Merge pull request #10259 from taosdata/feature/3.0_liaohj
Feature/3.0 liaohj
This commit is contained in:
Haojun Liao
GitHub
commit
378aaf3817
|
|
@ -582,9 +582,9 @@ typedef struct SOrderOperatorInfo {
|
|||
|
||||
void appendUpstream(SOperatorInfo* p, SOperatorInfo* pUpstream);
|
||||
|
||||
SOperatorInfo* createDataBlocksOptScanInfo(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv, int32_t repeatTime, int32_t reverseTime);
|
||||
SOperatorInfo* createTableScanOperatorInfo(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv, int32_t repeatTime, int32_t reverseTime);
|
||||
SOperatorInfo* createTableScanOperator(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv, int32_t repeatTime);
|
||||
SOperatorInfo* createTableSeqScanOperator(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv);
|
||||
SOperatorInfo* createTableSeqScanOperatorInfo(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv);
|
||||
|
||||
SOperatorInfo* createAggregateOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput);
|
||||
SOperatorInfo* createProjectOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput);
|
||||
|
|
@ -622,7 +622,7 @@ void doCompactSDataBlock(SSDataBlock* pBlock, int32_t numOfRows, int8_t* p);
|
|||
|
||||
SSDataBlock* createOutputBuf(SExprInfo* pExpr, int32_t numOfOutput, int32_t numOfRows);
|
||||
|
||||
void* destroyOutputBuf(SSDataBlock* pBlock);
|
||||
void* blockDataDestroy(SSDataBlock* pBlock);
|
||||
void* doDestroyFilterInfo(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols);
|
||||
|
||||
void setInputDataBlock(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order);
|
||||
|
|
|
|||
|
|
@ -336,7 +336,7 @@ SSDataBlock* createOutputBuf(SExprInfo* pExpr, int32_t numOfOutput, int32_t numO
|
|||
return res;
|
||||
}
|
||||
|
||||
void* destroyOutputBuf(SSDataBlock* pBlock) {
|
||||
void* blockDataDestroy(SSDataBlock* pBlock) {
|
||||
if (pBlock == NULL) {
|
||||
return NULL;
|
||||
}
|
||||
|
|
@ -4835,11 +4835,11 @@ int32_t doInitQInfo(SQInfo* pQInfo, STSBuf* pTsBuf, void* tsdb, void* sourceOptr
|
|||
break;
|
||||
}
|
||||
case OP_TableSeqScan: {
|
||||
pRuntimeEnv->proot = createTableSeqScanOperator(pRuntimeEnv->pTsdbReadHandle, pRuntimeEnv);
|
||||
pRuntimeEnv->proot = createTableSeqScanOperatorInfo(pRuntimeEnv->pTsdbReadHandle, pRuntimeEnv);
|
||||
break;
|
||||
}
|
||||
case OP_DataBlocksOptScan: {
|
||||
pRuntimeEnv->proot = createDataBlocksOptScanInfo(pRuntimeEnv->pTsdbReadHandle, pRuntimeEnv, getNumOfScanTimes(pQueryAttr), pQueryAttr->needReverseScan? 1:0);
|
||||
pRuntimeEnv->proot = createTableScanOperatorInfo(pRuntimeEnv->pTsdbReadHandle, pRuntimeEnv, getNumOfScanTimes(pQueryAttr), pQueryAttr->needReverseScan? 1:0);
|
||||
break;
|
||||
}
|
||||
case OP_TableScan: {
|
||||
|
|
@ -5162,7 +5162,7 @@ SOperatorInfo* createTableScanOperator(void* pTsdbQueryHandle, SQueryRuntimeEnv*
|
|||
return pOperator;
|
||||
}
|
||||
|
||||
SOperatorInfo* createTableSeqScanOperator(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv) {
|
||||
SOperatorInfo* createTableSeqScanOperatorInfo(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv) {
|
||||
STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo));
|
||||
|
||||
pInfo->pTsdbReadHandle = pTsdbQueryHandle;
|
||||
|
|
@ -5267,7 +5267,7 @@ void setTableScanFilterOperatorInfo(STableScanInfo* pTableScanInfo, SOperatorInf
|
|||
}
|
||||
}
|
||||
|
||||
SOperatorInfo* createDataBlocksOptScanInfo(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv, int32_t repeatTime, int32_t reverseTime) {
|
||||
SOperatorInfo* createTableScanOperatorInfo(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv, int32_t repeatTime, int32_t reverseTime) {
|
||||
assert(repeatTime > 0);
|
||||
|
||||
STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo));
|
||||
|
|
@ -5278,7 +5278,7 @@ SOperatorInfo* createDataBlocksOptScanInfo(void* pTsdbQueryHandle, SQueryRuntime
|
|||
pInfo->order = pRuntimeEnv->pQueryAttr->order.order;
|
||||
|
||||
SOperatorInfo* pOptr = calloc(1, sizeof(SOperatorInfo));
|
||||
pOptr->name = "DataBlocksOptimizedScanOperator";
|
||||
pOptr->name = "TableScanOperator";
|
||||
pOptr->operatorType = OP_DataBlocksOptScan;
|
||||
pOptr->pRuntimeEnv = pRuntimeEnv;
|
||||
pOptr->blockingOptr = false;
|
||||
|
|
@ -5373,7 +5373,7 @@ static void destroyGlobalAggOperatorInfo(void* param, int32_t numOfOutput) {
|
|||
static void destroySlimitOperatorInfo(void* param, int32_t numOfOutput) {
|
||||
SSLimitOperatorInfo *pInfo = (SSLimitOperatorInfo*) param;
|
||||
taosArrayDestroy(pInfo->orderColumnList);
|
||||
pInfo->pRes = destroyOutputBuf(pInfo->pRes);
|
||||
pInfo->pRes = blockDataDestroy(pInfo->pRes);
|
||||
tfree(pInfo->prevRow);
|
||||
}
|
||||
|
||||
|
|
@ -6566,7 +6566,7 @@ static void doDestroyBasicInfo(SOptrBasicInfo* pInfo, int32_t numOfOutput) {
|
|||
tfree(pInfo->rowCellInfoOffset);
|
||||
|
||||
cleanupResultRowInfo(&pInfo->resultRowInfo);
|
||||
pInfo->pRes = destroyOutputBuf(pInfo->pRes);
|
||||
pInfo->pRes = blockDataDestroy(pInfo->pRes);
|
||||
}
|
||||
|
||||
static void destroyBasicOperatorInfo(void* param, int32_t numOfOutput) {
|
||||
|
|
@ -6590,7 +6590,7 @@ static void destroySWindowOperatorInfo(void* param, int32_t numOfOutput) {
|
|||
static void destroySFillOperatorInfo(void* param, int32_t numOfOutput) {
|
||||
SFillOperatorInfo* pInfo = (SFillOperatorInfo*) param;
|
||||
pInfo->pFillInfo = taosDestroyFillInfo(pInfo->pFillInfo);
|
||||
pInfo->pRes = destroyOutputBuf(pInfo->pRes);
|
||||
pInfo->pRes = blockDataDestroy(pInfo->pRes);
|
||||
tfree(pInfo->p);
|
||||
}
|
||||
|
||||
|
|
@ -6607,12 +6607,12 @@ static void destroyProjectOperatorInfo(void* param, int32_t numOfOutput) {
|
|||
|
||||
static void destroyTagScanOperatorInfo(void* param, int32_t numOfOutput) {
|
||||
STagScanInfo* pInfo = (STagScanInfo*) param;
|
||||
pInfo->pRes = destroyOutputBuf(pInfo->pRes);
|
||||
pInfo->pRes = blockDataDestroy(pInfo->pRes);
|
||||
}
|
||||
|
||||
static void destroyOrderOperatorInfo(void* param, int32_t numOfOutput) {
|
||||
SOrderOperatorInfo* pInfo = (SOrderOperatorInfo*) param;
|
||||
pInfo->pDataBlock = destroyOutputBuf(pInfo->pDataBlock);
|
||||
pInfo->pDataBlock = blockDataDestroy(pInfo->pDataBlock);
|
||||
}
|
||||
|
||||
static void destroyConditionOperatorInfo(void* param, int32_t numOfOutput) {
|
||||
|
|
@ -6625,7 +6625,7 @@ static void destroyDistinctOperatorInfo(void* param, int32_t numOfOutput) {
|
|||
taosHashCleanup(pInfo->pSet);
|
||||
tfree(pInfo->buf);
|
||||
taosArrayDestroy(pInfo->pDistinctDataInfo);
|
||||
pInfo->pRes = destroyOutputBuf(pInfo->pRes);
|
||||
pInfo->pRes = blockDataDestroy(pInfo->pRes);
|
||||
}
|
||||
|
||||
SOperatorInfo* createMultiTableAggOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput) {
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@
|
|||
namespace {
|
||||
// simple test
|
||||
void simpleTest() {
|
||||
SDiskbasedResultBuf* pResultBuf = NULL;
|
||||
SDiskbasedBuf* pResultBuf = NULL;
|
||||
int32_t ret = createDiskbasedResultBuffer(&pResultBuf, 1024, 4096, 1);
|
||||
|
||||
int32_t pageId = 0;
|
||||
|
|
@ -22,40 +22,40 @@ void simpleTest() {
|
|||
tFilePage* pBufPage = getNewDataBuf(pResultBuf, groupId, &pageId);
|
||||
ASSERT_TRUE(pBufPage != NULL);
|
||||
|
||||
ASSERT_EQ(getResBufSize(pResultBuf), 1024);
|
||||
ASSERT_EQ(getTotalBufSize(pResultBuf), 1024);
|
||||
|
||||
SIDList list = getDataBufPagesIdList(pResultBuf, groupId);
|
||||
ASSERT_EQ(taosArrayGetSize(list), 1);
|
||||
ASSERT_EQ(getNumOfResultBufGroupId(pResultBuf), 1);
|
||||
|
||||
releaseResBufPage(pResultBuf, pBufPage);
|
||||
releaseBufPage(pResultBuf, pBufPage);
|
||||
|
||||
tFilePage* pBufPage1 = getNewDataBuf(pResultBuf, groupId, &pageId);
|
||||
|
||||
tFilePage* t = getResBufPage(pResultBuf, pageId);
|
||||
tFilePage* t = getBufPage(pResultBuf, pageId);
|
||||
ASSERT_TRUE(t == pBufPage1);
|
||||
|
||||
tFilePage* pBufPage2 = getNewDataBuf(pResultBuf, groupId, &pageId);
|
||||
tFilePage* t1 = getResBufPage(pResultBuf, pageId);
|
||||
tFilePage* t1 = getBufPage(pResultBuf, pageId);
|
||||
ASSERT_TRUE(t1 == pBufPage2);
|
||||
|
||||
tFilePage* pBufPage3 = getNewDataBuf(pResultBuf, groupId, &pageId);
|
||||
tFilePage* t2 = getResBufPage(pResultBuf, pageId);
|
||||
tFilePage* t2 = getBufPage(pResultBuf, pageId);
|
||||
ASSERT_TRUE(t2 == pBufPage3);
|
||||
|
||||
tFilePage* pBufPage4 = getNewDataBuf(pResultBuf, groupId, &pageId);
|
||||
tFilePage* t3 = getResBufPage(pResultBuf, pageId);
|
||||
tFilePage* t3 = getBufPage(pResultBuf, pageId);
|
||||
ASSERT_TRUE(t3 == pBufPage4);
|
||||
|
||||
tFilePage* pBufPage5 = getNewDataBuf(pResultBuf, groupId, &pageId);
|
||||
tFilePage* t4 = getResBufPage(pResultBuf, pageId);
|
||||
tFilePage* t4 = getBufPage(pResultBuf, pageId);
|
||||
ASSERT_TRUE(t4 == pBufPage5);
|
||||
|
||||
destroyResultBuf(pResultBuf);
|
||||
}
|
||||
|
||||
void writeDownTest() {
|
||||
SDiskbasedResultBuf* pResultBuf = NULL;
|
||||
SDiskbasedBuf* pResultBuf = NULL;
|
||||
int32_t ret = createDiskbasedResultBuffer(&pResultBuf, 1024, 4*1024, 1);
|
||||
|
||||
int32_t pageId = 0;
|
||||
|
|
@ -68,31 +68,31 @@ void writeDownTest() {
|
|||
|
||||
*(int32_t*)(pBufPage->data) = nx;
|
||||
writePageId = pageId;
|
||||
releaseResBufPage(pResultBuf, pBufPage);
|
||||
releaseBufPage(pResultBuf, pBufPage);
|
||||
|
||||
tFilePage* pBufPage1 = getNewDataBuf(pResultBuf, groupId, &pageId);
|
||||
tFilePage* t1 = getResBufPage(pResultBuf, pageId);
|
||||
tFilePage* t1 = getBufPage(pResultBuf, pageId);
|
||||
ASSERT_TRUE(t1 == pBufPage1);
|
||||
ASSERT_TRUE(pageId == 1);
|
||||
|
||||
tFilePage* pBufPage2 = getNewDataBuf(pResultBuf, groupId, &pageId);
|
||||
tFilePage* t2 = getResBufPage(pResultBuf, pageId);
|
||||
tFilePage* t2 = getBufPage(pResultBuf, pageId);
|
||||
ASSERT_TRUE(t2 == pBufPage2);
|
||||
ASSERT_TRUE(pageId == 2);
|
||||
|
||||
tFilePage* pBufPage3 = getNewDataBuf(pResultBuf, groupId, &pageId);
|
||||
tFilePage* t3 = getResBufPage(pResultBuf, pageId);
|
||||
tFilePage* t3 = getBufPage(pResultBuf, pageId);
|
||||
ASSERT_TRUE(t3 == pBufPage3);
|
||||
ASSERT_TRUE(pageId == 3);
|
||||
|
||||
tFilePage* pBufPage4 = getNewDataBuf(pResultBuf, groupId, &pageId);
|
||||
tFilePage* t4 = getResBufPage(pResultBuf, pageId);
|
||||
tFilePage* t4 = getBufPage(pResultBuf, pageId);
|
||||
ASSERT_TRUE(t4 == pBufPage4);
|
||||
ASSERT_TRUE(pageId == 4);
|
||||
releaseResBufPage(pResultBuf, t4);
|
||||
releaseBufPage(pResultBuf, t4);
|
||||
|
||||
// flush the written page to disk, and read it out again
|
||||
tFilePage* pBufPagex = getResBufPage(pResultBuf, writePageId);
|
||||
tFilePage* pBufPagex = getBufPage(pResultBuf, writePageId);
|
||||
ASSERT_EQ(*(int32_t*)pBufPagex->data, nx);
|
||||
|
||||
SArray* pa = getDataBufPagesIdList(pResultBuf, groupId);
|
||||
|
|
@ -102,7 +102,7 @@ void writeDownTest() {
|
|||
}
|
||||
|
||||
void recyclePageTest() {
|
||||
SDiskbasedResultBuf* pResultBuf = NULL;
|
||||
SDiskbasedBuf* pResultBuf = NULL;
|
||||
int32_t ret = createDiskbasedResultBuffer(&pResultBuf, 1024, 4*1024, 1);
|
||||
|
||||
int32_t pageId = 0;
|
||||
|
|
@ -112,41 +112,41 @@ void recyclePageTest() {
|
|||
|
||||
tFilePage* pBufPage = getNewDataBuf(pResultBuf, groupId, &pageId);
|
||||
ASSERT_TRUE(pBufPage != NULL);
|
||||
releaseResBufPage(pResultBuf, pBufPage);
|
||||
releaseBufPage(pResultBuf, pBufPage);
|
||||
|
||||
tFilePage* pBufPage1 = getNewDataBuf(pResultBuf, groupId, &pageId);
|
||||
tFilePage* t1 = getResBufPage(pResultBuf, pageId);
|
||||
tFilePage* t1 = getBufPage(pResultBuf, pageId);
|
||||
ASSERT_TRUE(t1 == pBufPage1);
|
||||
ASSERT_TRUE(pageId == 1);
|
||||
|
||||
tFilePage* pBufPage2 = getNewDataBuf(pResultBuf, groupId, &pageId);
|
||||
tFilePage* t2 = getResBufPage(pResultBuf, pageId);
|
||||
tFilePage* t2 = getBufPage(pResultBuf, pageId);
|
||||
ASSERT_TRUE(t2 == pBufPage2);
|
||||
ASSERT_TRUE(pageId == 2);
|
||||
|
||||
tFilePage* pBufPage3 = getNewDataBuf(pResultBuf, groupId, &pageId);
|
||||
tFilePage* t3 = getResBufPage(pResultBuf, pageId);
|
||||
tFilePage* t3 = getBufPage(pResultBuf, pageId);
|
||||
ASSERT_TRUE(t3 == pBufPage3);
|
||||
ASSERT_TRUE(pageId == 3);
|
||||
|
||||
tFilePage* pBufPage4 = getNewDataBuf(pResultBuf, groupId, &pageId);
|
||||
tFilePage* t4 = getResBufPage(pResultBuf, pageId);
|
||||
tFilePage* t4 = getBufPage(pResultBuf, pageId);
|
||||
ASSERT_TRUE(t4 == pBufPage4);
|
||||
ASSERT_TRUE(pageId == 4);
|
||||
releaseResBufPage(pResultBuf, t4);
|
||||
releaseBufPage(pResultBuf, t4);
|
||||
|
||||
tFilePage* pBufPage5 = getNewDataBuf(pResultBuf, groupId, &pageId);
|
||||
tFilePage* t5 = getResBufPage(pResultBuf, pageId);
|
||||
tFilePage* t5 = getBufPage(pResultBuf, pageId);
|
||||
ASSERT_TRUE(t5 == pBufPage5);
|
||||
ASSERT_TRUE(pageId == 5);
|
||||
|
||||
// flush the written page to disk, and read it out again
|
||||
tFilePage* pBufPagex = getResBufPage(pResultBuf, writePageId);
|
||||
tFilePage* pBufPagex = getBufPage(pResultBuf, writePageId);
|
||||
*(int32_t*)(pBufPagex->data) = nx;
|
||||
writePageId = pageId; // update the data
|
||||
releaseResBufPage(pResultBuf, pBufPagex);
|
||||
releaseBufPage(pResultBuf, pBufPagex);
|
||||
|
||||
tFilePage* pBufPagex1 = getResBufPage(pResultBuf, 1);
|
||||
tFilePage* pBufPagex1 = getBufPage(pResultBuf, 1);
|
||||
|
||||
SArray* pa = getDataBufPagesIdList(pResultBuf, groupId);
|
||||
ASSERT_EQ(taosArrayGetSize(pa), 6);
|
||||
|
|
|
|||
|
|
@ -16,6 +16,11 @@
|
|||
#ifndef TDENGINE_COMMON_H
|
||||
#define TDENGINE_COMMON_H
|
||||
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#include "taosdef.h"
|
||||
#include "tarray.h"
|
||||
#include "tmsg.h"
|
||||
|
|
@ -44,8 +49,8 @@
|
|||
|
||||
typedef struct {
|
||||
uint32_t numOfTables;
|
||||
SArray* pGroupList;
|
||||
SHashObj* map; // speedup acquire the tableQueryInfo by table uid
|
||||
SArray *pGroupList;
|
||||
SHashObj *map; // speedup acquire the tableQueryInfo by table uid
|
||||
} STableGroupInfo;
|
||||
|
||||
typedef struct SColumnDataAgg {
|
||||
|
|
@ -74,18 +79,28 @@ typedef struct SConstantItem {
|
|||
|
||||
// info.numOfCols = taosArrayGetSize(pDataBlock) + taosArrayGetSize(pConstantList);
|
||||
typedef struct SSDataBlock {
|
||||
SColumnDataAgg* pBlockAgg;
|
||||
SArray* pDataBlock; // SArray<SColumnInfoData>
|
||||
SArray* pConstantList; // SArray<SConstantItem>, it is a constant/tags value of the corresponding result value.
|
||||
SDataBlockInfo info;
|
||||
SColumnDataAgg *pBlockAgg;
|
||||
SArray *pDataBlock; // SArray<SColumnInfoData>
|
||||
SArray *pConstantList; // SArray<SConstantItem>, it is a constant/tags value of the corresponding result value.
|
||||
SDataBlockInfo info;
|
||||
} SSDataBlock;
|
||||
|
||||
typedef struct SVarColAttr {
|
||||
int32_t *offset; // start position for each entry in the list
|
||||
uint32_t length; // used buffer size that contain the valid data
|
||||
uint32_t allocLen; // allocated buffer size
|
||||
} SVarColAttr;
|
||||
|
||||
// pBlockAgg->numOfNull == info.rows, all data are null
|
||||
// pBlockAgg->numOfNull == 0, no data are null.
|
||||
typedef struct SColumnInfoData {
|
||||
SColumnInfo info; // TODO filter info needs to be removed
|
||||
char* nullbitmap; //
|
||||
char* pData; // the corresponding block data in memory
|
||||
SColumnInfo info; // TODO filter info needs to be removed
|
||||
bool hasNull;// if current column data has null value.
|
||||
char *pData; // the corresponding block data in memory
|
||||
union {
|
||||
char *nullbitmap; // bitmap, one bit for each item in the list
|
||||
SVarColAttr varmeta;
|
||||
};
|
||||
} SColumnInfoData;
|
||||
|
||||
static FORCE_INLINE int32_t tEncodeDataBlock(void** buf, const SSDataBlock* pBlock) {
|
||||
|
|
@ -235,11 +250,11 @@ typedef struct SSqlExpr {
|
|||
char token[TSDB_COL_NAME_LEN]; // original token
|
||||
SSchema resSchema;
|
||||
|
||||
int32_t numOfCols;
|
||||
SColumn* pColumns; // data columns that are required by query
|
||||
int32_t interBytes; // inter result buffer size
|
||||
int16_t numOfParams; // argument value of each function
|
||||
SVariant param[3]; // parameters are not more than 3
|
||||
int32_t numOfCols;
|
||||
SColumn* pColumns; // data columns that are required by query
|
||||
int32_t interBytes; // inter result buffer size
|
||||
int16_t numOfParams; // argument value of each function
|
||||
SVariant param[3]; // parameters are not more than 3
|
||||
} SSqlExpr;
|
||||
|
||||
typedef struct SExprInfo {
|
||||
|
|
@ -261,4 +276,8 @@ typedef struct SSessionWindow {
|
|||
|
||||
#define GET_FORWARD_DIRECTION_FACTOR(ord) (((ord) == TSDB_ORDER_ASC) ? QUERY_ASC_FORWARD_STEP : QUERY_DESC_FORWARD_STEP)
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif // TDENGINE_COMMON_H
|
||||
|
|
|
|||
|
|
@ -7,12 +7,22 @@ extern "C" {
|
|||
|
||||
#include "os.h"
|
||||
#include "tmsg.h"
|
||||
#include "common.h"
|
||||
|
||||
typedef struct SCorEpSet {
|
||||
int32_t version;
|
||||
SEpSet epSet;
|
||||
} SCorEpSet;
|
||||
|
||||
typedef struct SBlockOrderInfo {
|
||||
int32_t order;
|
||||
int32_t colIndex;
|
||||
SColumnInfoData *pColData;
|
||||
// int32_t type;
|
||||
// int32_t bytes;
|
||||
// bool hasNull;
|
||||
} SBlockOrderInfo;
|
||||
|
||||
int taosGetFqdnPortFromEp(const char *ep, SEp *pEp);
|
||||
void addEpIntoEpSet(SEpSet *pEpSet, const char *fqdn, uint16_t port);
|
||||
|
||||
|
|
@ -21,6 +31,77 @@ bool isEpsetEqual(const SEpSet *s1, const SEpSet *s2);
|
|||
void updateEpSet_s(SCorEpSet *pEpSet, SEpSet *pNewEpSet);
|
||||
SEpSet getEpSet_s(SCorEpSet *pEpSet);
|
||||
|
||||
#define NBIT (3u)
|
||||
#define BitPos(_n) ((_n) & ((1 << NBIT) - 1))
|
||||
#define BMCharPos(bm_, r_) ((bm_)[(r_) >> NBIT])
|
||||
#define colDataIsNull_f(bm_, r_) ((BMCharPos(bm_, r_) & (1u << (7u - BitPos(r_)))) == (1u << (7u - BitPos(r_))))
|
||||
|
||||
#define colDataSetNull_f(bm_, r_) \
|
||||
do { \
|
||||
BMCharPos(bm_, r_) |= (1u << (7u - BitPos(r_))); \
|
||||
} while (0)
|
||||
|
||||
static FORCE_INLINE bool colDataIsNull(const SColumnInfoData* pColumnInfoData, uint32_t totalRows, uint32_t row, SColumnDataAgg* pColAgg) {
|
||||
if (!pColumnInfoData->hasNull) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (pColAgg != NULL) {
|
||||
if (pColAgg->numOfNull == totalRows) {
|
||||
ASSERT(pColumnInfoData->nullbitmap == NULL);
|
||||
return true;
|
||||
} else if (pColAgg->numOfNull == 0) {
|
||||
ASSERT(pColumnInfoData->nullbitmap == NULL);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) {
|
||||
return pColumnInfoData->varmeta.offset[row] == -1;
|
||||
} else {
|
||||
if (pColumnInfoData->nullbitmap == NULL) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return colDataIsNull_f(pColumnInfoData->nullbitmap, row);
|
||||
}
|
||||
}
|
||||
|
||||
#define colDataGet(p1_, r_) \
|
||||
((IS_VAR_DATA_TYPE((p1_)->info.type)) ? (p1_)->pData + (p1_)->varmeta.offset[(r_)] \
|
||||
: (p1_)->pData + ((r_) * (p1_)->info.bytes));
|
||||
|
||||
int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, const char* pData, bool isNull);
|
||||
int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, const SColumnInfoData* pSource, uint32_t numOfRow2);
|
||||
int32_t blockDataUpdateTsWindow(SSDataBlock* pDataBlock);
|
||||
|
||||
int32_t colDataGetSize(const SColumnInfoData* pColumnInfoData, int32_t numOfRows);
|
||||
void colDataTrim(SColumnInfoData* pColumnInfoData);
|
||||
|
||||
size_t colDataGetNumOfCols(const SSDataBlock* pBlock);
|
||||
size_t colDataGetNumOfRows(const SSDataBlock* pBlock);
|
||||
|
||||
int32_t blockDataMerge(SSDataBlock* pDest, const SSDataBlock* pSrc);
|
||||
int32_t blockDataSplitRows(SSDataBlock* pBlock, bool hasVarCol, int32_t startIndex, int32_t* stopIndex, int32_t pageSize);
|
||||
SSDataBlock* blockDataExtractBlock(SSDataBlock* pBlock, int32_t startIndex, int32_t rowCount);
|
||||
|
||||
int32_t blockDataToBuf(char* buf, const SSDataBlock* pBlock);
|
||||
int32_t blockDataFromBuf(SSDataBlock* pBlock, const char* buf);
|
||||
|
||||
size_t blockDataGetSize(const SSDataBlock* pBlock);
|
||||
size_t blockDataGetRowSize(const SSDataBlock* pBlock);
|
||||
double blockDataGetSerialRowSize(const SSDataBlock* pBlock);
|
||||
size_t blockDataGetSerialMetaSize(const SSDataBlock* pBlock);
|
||||
|
||||
size_t blockDataNumOfRowsForSerialize(const SSDataBlock* pBlock, int32_t blockSize);
|
||||
|
||||
int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullFirst);
|
||||
int32_t blockDataSort_rv(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullFirst);
|
||||
|
||||
int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows);
|
||||
void blockDataClearup(SSDataBlock* pDataBlock, bool hasVarCol);
|
||||
void *blockDataDestroy(SSDataBlock *pBlock);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -473,7 +473,7 @@ static int32_t tdSRowResetBuf(SRowBuilder *pBuilder, void *pBuf) {
|
|||
terrno = TSDB_CODE_INVALID_PARA;
|
||||
return terrno;
|
||||
}
|
||||
|
||||
|
||||
TD_ROW_SET_TYPE(pBuilder->pBuf, pBuilder->rowType);
|
||||
|
||||
uint32_t len = 0;
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@
|
|||
#endif
|
||||
|
||||
OP_ENUM_MACRO(StreamScan)
|
||||
OP_ENUM_MACRO(DataBlocksOptScan)
|
||||
OP_ENUM_MACRO(TableScan)
|
||||
OP_ENUM_MACRO(TableSeqScan)
|
||||
OP_ENUM_MACRO(TagScan)
|
||||
OP_ENUM_MACRO(SystemTableScan)
|
||||
|
|
|
|||
|
|
@ -22,28 +22,31 @@ extern "C" {
|
|||
|
||||
typedef int (*__merge_compare_fn_t)(const void *, const void *, void *param);
|
||||
|
||||
typedef struct SLoserTreeNode {
|
||||
typedef struct STreeNode {
|
||||
int32_t index;
|
||||
void *pData;
|
||||
} SLoserTreeNode;
|
||||
void *pData; // TODO remove it?
|
||||
} STreeNode;
|
||||
|
||||
typedef struct SLoserTreeInfo {
|
||||
int32_t numOfEntries;
|
||||
int32_t totalEntries;
|
||||
typedef struct SMultiwayMergeTreeInfo {
|
||||
int32_t numOfSources;
|
||||
int32_t totalSources;
|
||||
__merge_compare_fn_t comparFn;
|
||||
void * param;
|
||||
SLoserTreeNode *pNode;
|
||||
} SLoserTreeInfo;
|
||||
struct STreeNode *pNode;
|
||||
} SMultiwayMergeTreeInfo;
|
||||
|
||||
uint32_t tLoserTreeCreate(SLoserTreeInfo **pTree, int32_t numOfEntries, void *param, __merge_compare_fn_t compareFn);
|
||||
#define tMergeTreeGetChosenIndex(t_) ((t_)->pNode[0].index)
|
||||
#define tMergeTreeGetAdjustIndex(t_) (tMergeTreeGetChosenIndex(t_) + (t_)->numOfSources)
|
||||
|
||||
void tLoserTreeInit(SLoserTreeInfo *pTree);
|
||||
int32_t tMergeTreeCreate(SMultiwayMergeTreeInfo **pTree, uint32_t numOfEntries, void *param, __merge_compare_fn_t compareFn);
|
||||
|
||||
void tLoserTreeAdjust(SLoserTreeInfo *pTree, int32_t idx);
|
||||
void tMergeTreeDestroy(SMultiwayMergeTreeInfo* pTree);
|
||||
|
||||
void tLoserTreeRebuild(SLoserTreeInfo *pTree);
|
||||
void tMergeTreeAdjust(SMultiwayMergeTreeInfo *pTree, int32_t idx);
|
||||
|
||||
void tLoserTreeDisplay(SLoserTreeInfo *pTree);
|
||||
void tMergeTreeRebuild(SMultiwayMergeTreeInfo *pTree);
|
||||
|
||||
void tMergeTreePrint(const SMultiwayMergeTreeInfo *pTree);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,171 @@
|
|||
/*
|
||||
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
|
||||
*
|
||||
* This program is free software: you can use, redistribute, and/or modify
|
||||
* it under the terms of the GNU Affero General Public License, version 3
|
||||
* or later ("AGPL"), as published by the Free Software Foundation.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful, but WITHOUT
|
||||
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#ifndef TDENGINE_TPAGEDBUF_H
|
||||
#define TDENGINE_TPAGEDBUF_H
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#include "tlist.h"
|
||||
#include "thash.h"
|
||||
#include "os.h"
|
||||
#include "tlockfree.h"
|
||||
|
||||
typedef struct SArray* SIDList;
|
||||
typedef struct SPageInfo SPageInfo;
|
||||
typedef struct SDiskbasedBuf SDiskbasedBuf;
|
||||
|
||||
#define DEFAULT_INTERN_BUF_PAGE_SIZE (1024L) // in bytes
|
||||
#define DEFAULT_PAGE_SIZE (16384L)
|
||||
|
||||
typedef struct SFilePage {
|
||||
int64_t num;
|
||||
char data[];
|
||||
} SFilePage;
|
||||
|
||||
typedef struct SDiskbasedBufStatis {
|
||||
int64_t flushBytes;
|
||||
int64_t loadBytes;
|
||||
int32_t loadPages;
|
||||
int32_t getPages;
|
||||
int32_t releasePages;
|
||||
int32_t flushPages;
|
||||
} SDiskbasedBufStatis;
|
||||
|
||||
/**
|
||||
* create disk-based result buffer
|
||||
* @param pBuf
|
||||
* @param rowSize
|
||||
* @param pagesize
|
||||
* @param inMemPages
|
||||
* @param handle
|
||||
* @return
|
||||
*/
|
||||
int32_t createDiskbasedBuffer(SDiskbasedBuf** pBuf, int32_t pagesize, int32_t inMemBufSize, uint64_t qId, const char* dir);
|
||||
|
||||
/**
|
||||
*
|
||||
* @param pBuf
|
||||
* @param groupId
|
||||
* @param pageId
|
||||
* @return
|
||||
*/
|
||||
SFilePage* getNewDataBuf(SDiskbasedBuf* pBuf, int32_t groupId, int32_t* pageId);
|
||||
|
||||
/**
|
||||
*
|
||||
* @param pBuf
|
||||
* @param groupId
|
||||
* @return
|
||||
*/
|
||||
SIDList getDataBufPagesIdList(SDiskbasedBuf* pBuf, int32_t groupId);
|
||||
|
||||
/**
|
||||
* get the specified buffer page by id
|
||||
* @param pBuf
|
||||
* @param id
|
||||
* @return
|
||||
*/
|
||||
SFilePage* getBufPage(SDiskbasedBuf* pBuf, int32_t id);
|
||||
|
||||
/**
|
||||
* release the referenced buf pages
|
||||
* @param pBuf
|
||||
* @param page
|
||||
*/
|
||||
void releaseBufPage(SDiskbasedBuf* pBuf, void* page);
|
||||
|
||||
/**
|
||||
*
|
||||
* @param pBuf
|
||||
* @param pi
|
||||
*/
|
||||
void releaseBufPageInfo(SDiskbasedBuf* pBuf, struct SPageInfo* pi);
|
||||
|
||||
/**
|
||||
* get the total buffer size in the format of disk file
|
||||
* @param pBuf
|
||||
* @return
|
||||
*/
|
||||
size_t getTotalBufSize(const SDiskbasedBuf* pBuf);
|
||||
|
||||
/**
|
||||
* get the number of groups in the result buffer
|
||||
* @param pBuf
|
||||
* @return
|
||||
*/
|
||||
size_t getNumOfResultBufGroupId(const SDiskbasedBuf* pBuf);
|
||||
|
||||
/**
|
||||
* destroy result buffer
|
||||
* @param pBuf
|
||||
*/
|
||||
void destroyResultBuf(SDiskbasedBuf* pBuf);
|
||||
|
||||
/**
|
||||
*
|
||||
* @param pList
|
||||
* @return
|
||||
*/
|
||||
SPageInfo* getLastPageInfo(SIDList pList);
|
||||
|
||||
/**
|
||||
*
|
||||
* @param pPgInfo
|
||||
* @return
|
||||
*/
|
||||
int32_t getPageId(const SPageInfo* pPgInfo);
|
||||
|
||||
/**
|
||||
* Return the buffer page size.
|
||||
* @param pBuf
|
||||
* @return
|
||||
*/
|
||||
int32_t getBufPageSize(const SDiskbasedBuf* pBuf);
|
||||
|
||||
int32_t getNumOfInMemBufPages(const SDiskbasedBuf* pBuf);
|
||||
|
||||
/**
|
||||
*
|
||||
* @param pBuf
|
||||
* @return
|
||||
*/
|
||||
bool isAllDataInMemBuf(const SDiskbasedBuf* pBuf);
|
||||
|
||||
/**
|
||||
* Set the buffer page is dirty, and needs to be flushed to disk when swap out.
|
||||
* @param pPageInfo
|
||||
* @param dirty
|
||||
*/
|
||||
void setBufPageDirty(SFilePage* pPageInfo, bool dirty);
|
||||
|
||||
/**
|
||||
* Print the statistics when closing this buffer
|
||||
* @param pBuf
|
||||
*/
|
||||
void printStatisBeforeClose(SDiskbasedBuf* pBuf);
|
||||
|
||||
/**
|
||||
* return buf statistics.
|
||||
*/
|
||||
SDiskbasedBufStatis getDBufStatis(const SDiskbasedBuf* pBuf);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif // TDENGINE_TPAGEDBUF_H
|
||||
|
|
@ -1,169 +0,0 @@
|
|||
/*
|
||||
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
|
||||
*
|
||||
* This program is free software: you can use, redistribute, and/or modify
|
||||
* it under the terms of the GNU Affero General Public License, version 3
|
||||
* or later ("AGPL"), as published by the Free Software Foundation.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful, but WITHOUT
|
||||
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#ifndef TDENGINE_TPAGEDFILE_H
|
||||
#define TDENGINE_TPAGEDFILE_H
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#include "tlist.h"
|
||||
#include "thash.h"
|
||||
#include "os.h"
|
||||
#include "tlockfree.h"
|
||||
|
||||
typedef struct SArray* SIDList;
|
||||
|
||||
typedef struct SPageDiskInfo {
|
||||
int32_t offset;
|
||||
int32_t length;
|
||||
} SPageDiskInfo;
|
||||
|
||||
typedef struct SPageInfo {
|
||||
SListNode* pn; // point to list node
|
||||
int32_t pageId;
|
||||
SPageDiskInfo info;
|
||||
void* pData;
|
||||
bool used; // set current page is in used
|
||||
} SPageInfo;
|
||||
|
||||
typedef struct SFreeListItem {
|
||||
int32_t offset;
|
||||
int32_t len;
|
||||
} SFreeListItem;
|
||||
|
||||
typedef struct SResultBufStatis {
|
||||
int32_t flushBytes;
|
||||
int32_t loadBytes;
|
||||
int32_t getPages;
|
||||
int32_t releasePages;
|
||||
int32_t flushPages;
|
||||
} SResultBufStatis;
|
||||
|
||||
typedef struct SDiskbasedResultBuf {
|
||||
int32_t numOfPages;
|
||||
int64_t totalBufSize;
|
||||
int64_t fileSize; // disk file size
|
||||
FILE* file;
|
||||
int32_t allocateId; // allocated page id
|
||||
char* path; // file path
|
||||
int32_t pageSize; // current used page size
|
||||
int32_t inMemPages; // numOfPages that are allocated in memory
|
||||
SHashObj* groupSet; // id hash table
|
||||
SHashObj* all;
|
||||
SList* lruList;
|
||||
void* emptyDummyIdList; // dummy id list
|
||||
void* assistBuf; // assistant buffer for compress/decompress data
|
||||
SArray* pFree; // free area in file
|
||||
bool comp; // compressed before flushed to disk
|
||||
int32_t nextPos; // next page flush position
|
||||
|
||||
uint64_t qId; // for debug purpose
|
||||
SResultBufStatis statis;
|
||||
} SDiskbasedResultBuf;
|
||||
|
||||
#define DEFAULT_INTERN_BUF_PAGE_SIZE (1024L) // in bytes
|
||||
#define PAGE_INFO_INITIALIZER (SPageDiskInfo){-1, -1}
|
||||
#define DEFAULT_PAGE_SIZE (16384L)
|
||||
|
||||
typedef struct SFilePage {
|
||||
int64_t num;
|
||||
char data[];
|
||||
} SFilePage;
|
||||
|
||||
/**
|
||||
* create disk-based result buffer
|
||||
* @param pResultBuf
|
||||
* @param rowSize
|
||||
* @param pagesize
|
||||
* @param inMemPages
|
||||
* @param handle
|
||||
* @return
|
||||
*/
|
||||
int32_t createDiskbasedResultBuffer(SDiskbasedResultBuf** pResultBuf, int32_t pagesize, int32_t inMemBufSize, uint64_t qId, const char* dir);
|
||||
|
||||
/**
|
||||
*
|
||||
* @param pResultBuf
|
||||
* @param groupId
|
||||
* @param pageId
|
||||
* @return
|
||||
*/
|
||||
SFilePage* getNewDataBuf(SDiskbasedResultBuf* pResultBuf, int32_t groupId, int32_t* pageId);
|
||||
|
||||
/**
|
||||
*
|
||||
* @param pResultBuf
|
||||
* @param groupId
|
||||
* @return
|
||||
*/
|
||||
SIDList getDataBufPagesIdList(SDiskbasedResultBuf* pResultBuf, int32_t groupId);
|
||||
|
||||
/**
|
||||
* get the specified buffer page by id
|
||||
* @param pResultBuf
|
||||
* @param id
|
||||
* @return
|
||||
*/
|
||||
SFilePage* getResBufPage(SDiskbasedResultBuf* pResultBuf, int32_t id);
|
||||
|
||||
/**
|
||||
* release the referenced buf pages
|
||||
* @param pResultBuf
|
||||
* @param page
|
||||
*/
|
||||
void releaseResBufPage(SDiskbasedResultBuf* pResultBuf, void* page);
|
||||
|
||||
/**
|
||||
*
|
||||
* @param pResultBuf
|
||||
* @param pi
|
||||
*/
|
||||
void releaseResBufPageInfo(SDiskbasedResultBuf* pResultBuf, SPageInfo* pi);
|
||||
|
||||
|
||||
/**
|
||||
* get the total buffer size in the format of disk file
|
||||
* @param pResultBuf
|
||||
* @return
|
||||
*/
|
||||
size_t getResBufSize(const SDiskbasedResultBuf* pResultBuf);
|
||||
|
||||
/**
|
||||
* get the number of groups in the result buffer
|
||||
* @param pResultBuf
|
||||
* @return
|
||||
*/
|
||||
size_t getNumOfResultBufGroupId(const SDiskbasedResultBuf* pResultBuf);
|
||||
|
||||
/**
|
||||
* destroy result buffer
|
||||
* @param pResultBuf
|
||||
*/
|
||||
void destroyResultBuf(SDiskbasedResultBuf* pResultBuf);
|
||||
|
||||
/**
|
||||
*
|
||||
* @param pList
|
||||
* @return
|
||||
*/
|
||||
SPageInfo* getLastPageInfo(SIDList pList);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif // TDENGINE_TPAGEDFILE_H
|
||||
|
|
@ -9,7 +9,7 @@
|
|||
#include "tglobal.h"
|
||||
#include "tmsgtype.h"
|
||||
#include "tnote.h"
|
||||
#include "tpagedfile.h"
|
||||
#include "tpagedbuf.h"
|
||||
#include "tref.h"
|
||||
|
||||
static int32_t initEpSetFromCfg(const char *firstEp, const char *secondEp, SCorEpSet *pEpSet);
|
||||
|
|
|
|||
|
|
@ -25,7 +25,7 @@
|
|||
#include "tglobal.h"
|
||||
#include "tmsgtype.h"
|
||||
#include "tnote.h"
|
||||
#include "tpagedfile.h"
|
||||
#include "tpagedbuf.h"
|
||||
#include "tref.h"
|
||||
|
||||
struct tmq_list_t {
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load Diff
|
|
@ -1,3 +1,4 @@
|
|||
#include <common.h>
|
||||
#include "os.h"
|
||||
#include "tutil.h"
|
||||
|
||||
|
|
@ -268,4 +269,5 @@ SSchema createSchema(uint8_t type, int32_t bytes, int32_t colId, const char* nam
|
|||
|
||||
tstrncpy(s.name, name, tListLen(s.name));
|
||||
return s;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -344,8 +344,8 @@ STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) {
|
|||
UNUSED(ret);
|
||||
}
|
||||
|
||||
fread(&pBlock->tag.nType, sizeof(pBlock->tag.nType), 1, pTSBuf->f);
|
||||
fread(&pBlock->tag.nLen, sizeof(pBlock->tag.nLen), 1, pTSBuf->f);
|
||||
int32_t ret = fread(&pBlock->tag.nType, sizeof(pBlock->tag.nType), 1, pTSBuf->f);
|
||||
ret = fread(&pBlock->tag.nLen, sizeof(pBlock->tag.nLen), 1, pTSBuf->f);
|
||||
|
||||
// NOTE: mix types tags are not supported
|
||||
size_t sz = 0;
|
||||
|
|
|
|||
|
|
@ -1,11 +1,12 @@
|
|||
#include <common.h>
|
||||
#include <gtest/gtest.h>
|
||||
#include <tep.h>
|
||||
#include <iostream>
|
||||
|
||||
#pragma GCC diagnostic push
|
||||
#pragma GCC diagnostic ignored "-Wwrite-strings"
|
||||
#pragma GCC diagnostic ignored "-Wunused-function"
|
||||
#pragma GCC diagnostic ignored "-Wunused-variable"
|
||||
#pragma GCC diagnostic ignored "-Wunused-but-set-variable"
|
||||
#pragma GCC diagnostic ignored "-Wsign-compare"
|
||||
#include "os.h"
|
||||
|
||||
|
|
@ -96,4 +97,199 @@ TEST(testCase, toInteger_test) {
|
|||
ASSERT_EQ(ret, -1);
|
||||
}
|
||||
|
||||
TEST(testCase, Datablock_test) {
|
||||
SSDataBlock* b = static_cast<SSDataBlock*>(calloc(1, sizeof(SSDataBlock)));
|
||||
b->info.numOfCols = 2;
|
||||
b->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData));
|
||||
|
||||
SColumnInfoData infoData = {0};
|
||||
infoData.info.bytes = 4;
|
||||
infoData.info.type = TSDB_DATA_TYPE_INT;
|
||||
infoData.info.colId = 1;
|
||||
|
||||
infoData.pData = (char*) calloc(40, infoData.info.bytes);
|
||||
infoData.nullbitmap = (char*) calloc(1, sizeof(char) * (40/8));
|
||||
taosArrayPush(b->pDataBlock, &infoData);
|
||||
|
||||
SColumnInfoData infoData1 = {0};
|
||||
infoData1.info.bytes = 40;
|
||||
infoData1.info.type = TSDB_DATA_TYPE_BINARY;
|
||||
infoData1.info.colId = 2;
|
||||
|
||||
infoData1.varmeta.offset = (int32_t*) calloc(40, sizeof(uint32_t));
|
||||
taosArrayPush(b->pDataBlock, &infoData1);
|
||||
|
||||
char* str = "the value of: %d";
|
||||
char buf[128] = {0};
|
||||
char varbuf[128] = {0};
|
||||
|
||||
for(int32_t i = 0; i < 40; ++i) {
|
||||
SColumnInfoData* p0 = (SColumnInfoData *) taosArrayGet(b->pDataBlock, 0);
|
||||
SColumnInfoData* p1 = (SColumnInfoData *) taosArrayGet(b->pDataBlock, 1);
|
||||
|
||||
if (i&0x01) {
|
||||
int32_t len = sprintf(buf, str, i);
|
||||
STR_TO_VARSTR(varbuf, buf)
|
||||
colDataAppend(p0, i, (const char*) &i, false);
|
||||
colDataAppend(p1, i, (const char*) varbuf, false);
|
||||
|
||||
memset(varbuf, 0, sizeof(varbuf));
|
||||
memset(buf, 0, sizeof(buf));
|
||||
} else {
|
||||
colDataAppend(p0, i, (const char*) &i, true);
|
||||
colDataAppend(p1, i, (const char*) varbuf, true);
|
||||
}
|
||||
|
||||
b->info.rows++;
|
||||
}
|
||||
|
||||
SColumnInfoData* p0 = (SColumnInfoData *) taosArrayGet(b->pDataBlock, 0);
|
||||
SColumnInfoData* p1 = (SColumnInfoData *) taosArrayGet(b->pDataBlock, 1);
|
||||
for(int32_t i = 0; i < 40; ++i) {
|
||||
if (i & 0x01) {
|
||||
ASSERT_EQ(colDataIsNull_f(p0->nullbitmap, i), false);
|
||||
ASSERT_EQ(colDataIsNull(p1, b->info.rows, i, nullptr), false);
|
||||
} else {
|
||||
ASSERT_EQ(colDataIsNull_f(p0->nullbitmap, i), true);
|
||||
|
||||
ASSERT_EQ(colDataIsNull(p0, b->info.rows, i, nullptr), true);
|
||||
ASSERT_EQ(colDataIsNull(p1, b->info.rows, i, nullptr), true);
|
||||
}
|
||||
}
|
||||
|
||||
printf("binary column length:%d\n", *(int32_t*) p1->pData);
|
||||
|
||||
ASSERT_EQ(colDataGetNumOfCols(b), 2);
|
||||
ASSERT_EQ(colDataGetNumOfRows(b), 40);
|
||||
|
||||
char* pData = colDataGet(p1, 3);
|
||||
printf("the second row of binary:%s, length:%d\n", (char*)varDataVal(pData), varDataLen(pData));
|
||||
|
||||
SArray* pOrderInfo = taosArrayInit(3, sizeof(SBlockOrderInfo));
|
||||
SBlockOrderInfo order = {.order = TSDB_ORDER_ASC, .colIndex = 0};
|
||||
taosArrayPush(pOrderInfo, &order);
|
||||
|
||||
blockDataSort(b, pOrderInfo, true);
|
||||
blockDataDestroy(b);
|
||||
|
||||
taosArrayDestroy(pOrderInfo);
|
||||
}
|
||||
|
||||
#if 0
|
||||
TEST(testCase, non_var_dataBlock_split_test) {
|
||||
SSDataBlock* b = static_cast<SSDataBlock*>(calloc(1, sizeof(SSDataBlock)));
|
||||
b->info.numOfCols = 2;
|
||||
b->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData));
|
||||
|
||||
SColumnInfoData infoData = {0};
|
||||
infoData.info.bytes = 4;
|
||||
infoData.info.type = TSDB_DATA_TYPE_INT;
|
||||
infoData.info.colId = 1;
|
||||
|
||||
int32_t numOfRows = 1000000;
|
||||
|
||||
infoData.pData = (char*) calloc(numOfRows, infoData.info.bytes);
|
||||
infoData.nullbitmap = (char*) calloc(1, sizeof(char) * (numOfRows/8));
|
||||
taosArrayPush(b->pDataBlock, &infoData);
|
||||
|
||||
SColumnInfoData infoData1 = {0};
|
||||
infoData1.info.bytes = 1;
|
||||
infoData1.info.type = TSDB_DATA_TYPE_TINYINT;
|
||||
infoData1.info.colId = 2;
|
||||
|
||||
infoData1.pData = (char*) calloc(numOfRows, infoData.info.bytes);
|
||||
infoData1.nullbitmap = (char*) calloc(1, sizeof(char) * (numOfRows/8));
|
||||
taosArrayPush(b->pDataBlock, &infoData1);
|
||||
|
||||
for(int32_t i = 0; i < numOfRows; ++i) {
|
||||
SColumnInfoData* p0 = (SColumnInfoData*)taosArrayGet(b->pDataBlock, 0);
|
||||
SColumnInfoData* p1 = (SColumnInfoData*)taosArrayGet(b->pDataBlock, 1);
|
||||
|
||||
int8_t v = i;
|
||||
colDataAppend(p0, i, (const char*)&i, false);
|
||||
colDataAppend(p1, i, (const char*)&v, false);
|
||||
b->info.rows++;
|
||||
}
|
||||
|
||||
int32_t pageSize = 64 * 1024;
|
||||
|
||||
int32_t startIndex= 0;
|
||||
int32_t stopIndex = 0;
|
||||
int32_t count = 1;
|
||||
while(1) {
|
||||
blockDataSplitRows(b, false, startIndex, &stopIndex, pageSize);
|
||||
printf("the %d split, from: %d to %d\n", count++, startIndex, stopIndex);
|
||||
|
||||
if (stopIndex == numOfRows - 1) {
|
||||
break;
|
||||
}
|
||||
|
||||
startIndex = stopIndex + 1;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
TEST(testCase, var_dataBlock_split_test) {
|
||||
SSDataBlock* b = static_cast<SSDataBlock*>(calloc(1, sizeof(SSDataBlock)));
|
||||
b->info.numOfCols = 2;
|
||||
b->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData));
|
||||
|
||||
int32_t numOfRows = 1000000;
|
||||
|
||||
SColumnInfoData infoData = {0};
|
||||
infoData.info.bytes = 4;
|
||||
infoData.info.type = TSDB_DATA_TYPE_INT;
|
||||
infoData.info.colId = 1;
|
||||
|
||||
infoData.pData = (char*) calloc(numOfRows, infoData.info.bytes);
|
||||
infoData.nullbitmap = (char*) calloc(1, sizeof(char) * (numOfRows/8));
|
||||
taosArrayPush(b->pDataBlock, &infoData);
|
||||
|
||||
SColumnInfoData infoData1 = {0};
|
||||
infoData1.info.bytes = 40;
|
||||
infoData1.info.type = TSDB_DATA_TYPE_BINARY;
|
||||
infoData1.info.colId = 2;
|
||||
|
||||
infoData1.varmeta.offset = (int32_t*) calloc(numOfRows, sizeof(uint32_t));
|
||||
taosArrayPush(b->pDataBlock, &infoData1);
|
||||
|
||||
char buf[41] = {0};
|
||||
char buf1[100] = {0};
|
||||
|
||||
for(int32_t i = 0; i < numOfRows; ++i) {
|
||||
SColumnInfoData* p0 = (SColumnInfoData*)taosArrayGet(b->pDataBlock, 0);
|
||||
SColumnInfoData* p1 = (SColumnInfoData*)taosArrayGet(b->pDataBlock, 1);
|
||||
|
||||
int8_t v = i;
|
||||
colDataAppend(p0, i, (const char*)&i, false);
|
||||
|
||||
sprintf(buf, "the number of row:%d", i);
|
||||
int32_t len = sprintf(buf1, buf, i);
|
||||
STR_TO_VARSTR(buf1, buf)
|
||||
colDataAppend(p1, i, buf1, false);
|
||||
b->info.rows++;
|
||||
|
||||
memset(buf, 0, sizeof(buf));
|
||||
memset(buf1, 0, sizeof(buf1));
|
||||
}
|
||||
|
||||
int32_t pageSize = 64 * 1024;
|
||||
|
||||
int32_t startIndex= 0;
|
||||
int32_t stopIndex = 0;
|
||||
int32_t count = 1;
|
||||
while(1) {
|
||||
blockDataSplitRows(b, true, startIndex, &stopIndex, pageSize);
|
||||
printf("the %d split, from: %d to %d\n", count++, startIndex, stopIndex);
|
||||
|
||||
if (stopIndex == numOfRows - 1) {
|
||||
break;
|
||||
}
|
||||
|
||||
startIndex = stopIndex + 1;
|
||||
}
|
||||
}
|
||||
|
||||
#pragma GCC diagnostic pop
|
||||
|
|
@ -2174,8 +2174,8 @@ static int32_t createDataBlocksInfo(STsdbReadHandle* pTsdbReadHandle, int32_t nu
|
|||
assert(cnt <= numOfBlocks && numOfQualTables <= numOfTables); // the pTableQueryInfo[j]->numOfBlocks may be 0
|
||||
sup.numOfTables = numOfQualTables;
|
||||
|
||||
SLoserTreeInfo* pTree = NULL;
|
||||
uint8_t ret = tLoserTreeCreate(&pTree, sup.numOfTables, &sup, dataBlockOrderCompar);
|
||||
SMultiwayMergeTreeInfo* pTree = NULL;
|
||||
uint8_t ret = tMergeTreeCreate(&pTree, sup.numOfTables, &sup, dataBlockOrderCompar);
|
||||
if (ret != TSDB_CODE_SUCCESS) {
|
||||
cleanBlockOrderSupporter(&sup, numOfTables);
|
||||
return TSDB_CODE_TDB_OUT_OF_MEMORY;
|
||||
|
|
@ -2184,7 +2184,7 @@ static int32_t createDataBlocksInfo(STsdbReadHandle* pTsdbReadHandle, int32_t nu
|
|||
int32_t numOfTotal = 0;
|
||||
|
||||
while (numOfTotal < cnt) {
|
||||
int32_t pos = pTree->pNode[0].index;
|
||||
int32_t pos = tMergeTreeGetChosenIndex(pTree);
|
||||
int32_t index = sup.blockIndexArray[pos]++;
|
||||
|
||||
STableBlockInfo* pBlocksInfo = sup.pDataBlockInfo[pos];
|
||||
|
|
@ -2195,7 +2195,7 @@ static int32_t createDataBlocksInfo(STsdbReadHandle* pTsdbReadHandle, int32_t nu
|
|||
sup.blockIndexArray[pos] = sup.numOfBlocksPerTable[pos] + 1;
|
||||
}
|
||||
|
||||
tLoserTreeAdjust(pTree, pos + sup.numOfTables);
|
||||
tMergeTreeAdjust(pTree, tMergeTreeGetAdjustIndex(pTree));
|
||||
}
|
||||
|
||||
/*
|
||||
|
|
@ -3643,13 +3643,13 @@ int32_t tsdbQuerySTableByTagCond(void* pMeta, uint64_t uid, TSKEY skey, const ch
|
|||
SColIndex* pColIndex, int32_t numOfCols, uint64_t reqId, uint64_t taskId) {
|
||||
STbCfg* pTbCfg = metaGetTbInfoByUid(pMeta, uid);
|
||||
if (pTbCfg == NULL) {
|
||||
// tsdbError("%p failed to get stable, uid:%"PRIu64", TID:0x%"PRIx64" QID:0x%"PRIx64, tsdb, uid, taskId, reqId);
|
||||
tsdbError("%p failed to get stable, uid:%"PRIu64", TID:0x%"PRIx64" QID:0x%"PRIx64, pMeta, uid, taskId, reqId);
|
||||
terrno = TSDB_CODE_TDB_INVALID_TABLE_ID;
|
||||
goto _error;
|
||||
}
|
||||
|
||||
if (pTbCfg->type != META_SUPER_TABLE) {
|
||||
// tsdbError("%p query normal tag not allowed, uid:%" PRIu64 ", TID:0x%"PRIx64" QID:0x%"PRIx64, tsdb, uid, taskId, reqId);
|
||||
tsdbError("%p query normal tag not allowed, uid:%" PRIu64 ", TID:0x%"PRIx64" QID:0x%"PRIx64, pMeta, uid, taskId, reqId);
|
||||
terrno = TSDB_CODE_OPS_NOT_SUPPORT; //basically, this error is caused by invalid sql issued by client
|
||||
goto _error;
|
||||
}
|
||||
|
|
@ -3668,8 +3668,8 @@ int32_t tsdbQuerySTableByTagCond(void* pMeta, uint64_t uid, TSKEY skey, const ch
|
|||
pGroupInfo->numOfTables = (uint32_t) taosArrayGetSize(res);
|
||||
pGroupInfo->pGroupList = createTableGroup(res, pTagSchema, pColIndex, numOfCols, skey);
|
||||
|
||||
// tsdbDebug("%p no table name/tag condition, all tables qualified, numOfTables:%u, group:%zu, TID:0x%"PRIx64" QID:0x%"PRIx64, tsdb,
|
||||
// pGroupInfo->numOfTables, taosArrayGetSize(pGroupInfo->pGroupList), taskId, reqId);
|
||||
tsdbDebug("%p no table name/tag condition, all tables qualified, numOfTables:%u, group:%zu, TID:0x%"PRIx64" QID:0x%"PRIx64, pMeta,
|
||||
pGroupInfo->numOfTables, taosArrayGetSize(pGroupInfo->pGroupList), taskId, reqId);
|
||||
|
||||
taosArrayDestroy(res);
|
||||
return ret;
|
||||
|
|
|
|||
|
|
@ -16,8 +16,8 @@
|
|||
#define TDENGINE_QUERYUTIL_H
|
||||
|
||||
#include "common.h"
|
||||
#include "tpagedfile.h"
|
||||
#include "tbuffer.h"
|
||||
#include "tpagedbuf.h"
|
||||
|
||||
#define SET_RES_WINDOW_KEY(_k, _ori, _len, _uid) \
|
||||
do { \
|
||||
|
|
@ -126,6 +126,13 @@ static FORCE_INLINE char* getPosInResultPage(struct STaskAttr* pQueryAttr, SFile
|
|||
// return ((char *)page->data) + rowOffset + offset * numOfRows;
|
||||
}
|
||||
|
||||
static FORCE_INLINE char* getPosInResultPage_rv(SFilePage* page, int32_t rowOffset, int32_t offset) {
|
||||
assert(rowOffset >= 0);
|
||||
|
||||
int32_t numOfRows = 1;//(int32_t)getRowNumForMultioutput(pQueryAttr, pQueryAttr->topBotQuery, pQueryAttr->stableQuery);
|
||||
return ((char *)page->data) + rowOffset + offset * numOfRows;
|
||||
}
|
||||
|
||||
//bool isNullOperator(SColumnFilterElem *pFilter, const char* minval, const char* maxval, int16_t type);
|
||||
//bool notNullOperator(SColumnFilterElem *pFilter, const char* minval, const char* maxval, int16_t type);
|
||||
|
||||
|
|
|
|||
|
|
@ -15,35 +15,40 @@
|
|||
#ifndef TDENGINE_EXECUTORIMPL_H
|
||||
#define TDENGINE_EXECUTORIMPL_H
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#include "os.h"
|
||||
#include "common.h"
|
||||
#include "tlosertree.h"
|
||||
#include "ttszip.h"
|
||||
#include "tvariant.h"
|
||||
|
||||
#include "dataSinkMgt.h"
|
||||
#include "executil.h"
|
||||
#include "executor.h"
|
||||
#include "planner.h"
|
||||
#include "taosdef.h"
|
||||
#include "tarray.h"
|
||||
#include "tfilter.h"
|
||||
#include "thash.h"
|
||||
#include "tlockfree.h"
|
||||
#include "tpagedfile.h"
|
||||
#include "executor.h"
|
||||
#include "tpagedbuf.h"
|
||||
|
||||
struct SColumnFilterElem;
|
||||
|
||||
typedef int32_t (*__block_search_fn_t)(char* data, int32_t num, int64_t key, int32_t order);
|
||||
|
||||
#define IS_QUERY_KILLED(_q) ((_q)->code == TSDB_CODE_TSC_QUERY_CANCELLED)
|
||||
#define Q_STATUS_EQUAL(p, s) (((p) & (s)) != 0u)
|
||||
#define Q_STATUS_EQUAL(p, s) (((p) & (s)) != 0u)
|
||||
#define QUERY_IS_ASC_QUERY(q) (GET_FORWARD_DIRECTION_FACTOR((q)->order.order) == QUERY_ASC_FORWARD_STEP)
|
||||
|
||||
#define GET_TABLEGROUP(q, _index) ((SArray*) taosArrayGetP((q)->tableqinfoGroupInfo.pGroupList, (_index)))
|
||||
#define GET_TABLEGROUP(q, _index) ((SArray*)taosArrayGetP((q)->tableqinfoGroupInfo.pGroupList, (_index)))
|
||||
|
||||
#define GET_NUM_OF_RESULTS(_r) (((_r)->outputBuf) == NULL? 0:((_r)->outputBuf)->info.rows)
|
||||
#define GET_NUM_OF_RESULTS(_r) (((_r)->outputBuf) == NULL ? 0 : ((_r)->outputBuf)->info.rows)
|
||||
|
||||
#define NEEDTO_COMPRESS_QUERY(size) ((size) > tsCompressColData? 1 : 0)
|
||||
#define NEEDTO_COMPRESS_QUERY(size) ((size) > tsCompressColData ? 1 : 0)
|
||||
|
||||
enum {
|
||||
// when this task starts to execute, this status will set
|
||||
|
|
@ -62,8 +67,8 @@ enum {
|
|||
};
|
||||
|
||||
typedef struct SResultRowCell {
|
||||
uint64_t groupId;
|
||||
SResultRow *pRow;
|
||||
uint64_t groupId;
|
||||
SResultRow* pRow;
|
||||
} SResultRowCell;
|
||||
|
||||
/**
|
||||
|
|
@ -80,25 +85,23 @@ typedef struct SColumnFilterElem {
|
|||
int16_t bytes; // column length
|
||||
__filter_func_t fp;
|
||||
SColumnFilterInfo filterInfo;
|
||||
void *q;
|
||||
void* q;
|
||||
} SColumnFilterElem;
|
||||
|
||||
typedef struct SSingleColumnFilterInfo {
|
||||
void* pData;
|
||||
void* pData2; //used for nchar column
|
||||
void* pData2; // used for nchar column
|
||||
int32_t numOfFilters;
|
||||
SColumnInfo info;
|
||||
SColumnFilterElem* pFilters;
|
||||
} SSingleColumnFilterInfo;
|
||||
|
||||
typedef struct STableQueryInfo {
|
||||
TSKEY lastKey;
|
||||
int32_t groupIndex; // group id in table list
|
||||
SVariant tag;
|
||||
STimeWindow win; // todo remove it later
|
||||
STSCursor cur;
|
||||
void* pTable; // for retrieve the page id list
|
||||
SResultRowInfo resInfo;
|
||||
TSKEY lastKey; // last check ts
|
||||
uint64_t uid; // table uid
|
||||
int32_t groupIndex; // group id in table list
|
||||
// SVariant tag;
|
||||
SResultRowInfo resInfo; // result info
|
||||
} STableQueryInfo;
|
||||
|
||||
typedef enum {
|
||||
|
|
@ -109,11 +112,11 @@ typedef enum {
|
|||
|
||||
typedef struct {
|
||||
EQueryProfEventType eventType;
|
||||
int64_t eventTime;
|
||||
int64_t eventTime;
|
||||
|
||||
union {
|
||||
uint8_t operatorType; //for operator event
|
||||
int32_t abortCode; //for query abort event
|
||||
uint8_t operatorType; // for operator event
|
||||
int32_t abortCode; // for query abort event
|
||||
};
|
||||
} SQueryProfEvent;
|
||||
|
||||
|
|
@ -124,33 +127,33 @@ typedef struct {
|
|||
} SOperatorProfResult;
|
||||
|
||||
typedef struct STaskCostInfo {
|
||||
int64_t created;
|
||||
int64_t start;
|
||||
int64_t end;
|
||||
int64_t created;
|
||||
int64_t start;
|
||||
int64_t end;
|
||||
|
||||
uint64_t loadStatisTime;
|
||||
uint64_t loadFileBlockTime;
|
||||
uint64_t loadDataInCacheTime;
|
||||
uint64_t loadStatisSize;
|
||||
uint64_t loadFileBlockSize;
|
||||
uint64_t loadDataInCacheSize;
|
||||
uint64_t loadStatisTime;
|
||||
uint64_t loadFileBlockTime;
|
||||
uint64_t loadDataInCacheTime;
|
||||
uint64_t loadStatisSize;
|
||||
uint64_t loadFileBlockSize;
|
||||
uint64_t loadDataInCacheSize;
|
||||
|
||||
uint64_t loadDataTime;
|
||||
uint64_t totalRows;
|
||||
uint64_t totalCheckedRows;
|
||||
uint32_t totalBlocks;
|
||||
uint32_t loadBlocks;
|
||||
uint32_t loadBlockStatis;
|
||||
uint32_t discardBlocks;
|
||||
uint64_t elapsedTime;
|
||||
uint64_t firstStageMergeTime;
|
||||
uint64_t winInfoSize;
|
||||
uint64_t tableInfoSize;
|
||||
uint64_t hashSize;
|
||||
uint64_t numOfTimeWindows;
|
||||
uint64_t loadDataTime;
|
||||
uint64_t totalRows;
|
||||
uint64_t totalCheckedRows;
|
||||
uint32_t totalBlocks;
|
||||
uint32_t loadBlocks;
|
||||
uint32_t loadBlockStatis;
|
||||
uint32_t discardBlocks;
|
||||
uint64_t elapsedTime;
|
||||
uint64_t firstStageMergeTime;
|
||||
uint64_t winInfoSize;
|
||||
uint64_t tableInfoSize;
|
||||
uint64_t hashSize;
|
||||
uint64_t numOfTimeWindows;
|
||||
|
||||
SArray *queryProfEvents; //SArray<SQueryProfEvent>
|
||||
SHashObj *operatorProfResults; //map<operator_type, SQueryProfEvent>
|
||||
SArray* queryProfEvents; // SArray<SQueryProfEvent>
|
||||
SHashObj* operatorProfResults; // map<operator_type, SQueryProfEvent>
|
||||
} STaskCostInfo;
|
||||
|
||||
typedef struct {
|
||||
|
|
@ -166,67 +169,67 @@ typedef struct {
|
|||
// The basic query information extracted from the SQueryInfo tree to support the
|
||||
// execution of query in a data node.
|
||||
typedef struct STaskAttr {
|
||||
SLimit limit;
|
||||
SLimit slimit;
|
||||
SLimit limit;
|
||||
SLimit slimit;
|
||||
|
||||
// todo comment it
|
||||
bool stableQuery; // super table query or not
|
||||
bool topBotQuery; // TODO used bitwise flag
|
||||
bool groupbyColumn; // denote if this is a groupby normal column query
|
||||
bool hasTagResults; // if there are tag values in final result or not
|
||||
bool timeWindowInterpo;// if the time window start/end required interpolation
|
||||
bool queryBlockDist; // if query data block distribution
|
||||
bool stabledev; // super table stddev query
|
||||
bool tsCompQuery; // is tscomp query
|
||||
bool diffQuery; // is diff query
|
||||
bool simpleAgg;
|
||||
bool pointInterpQuery; // point interpolation query
|
||||
bool needReverseScan; // need reverse scan
|
||||
bool distinct; // distinct query or not
|
||||
bool stateWindow; // window State on sub/normal table
|
||||
bool createFilterOperator; // if filter operator is needed
|
||||
bool multigroupResult; // multigroup result can exist in one SSDataBlock
|
||||
int32_t interBufSize; // intermediate buffer sizse
|
||||
bool stableQuery; // super table query or not
|
||||
bool topBotQuery; // TODO used bitwise flag
|
||||
bool groupbyColumn; // denote if this is a groupby normal column query
|
||||
bool hasTagResults; // if there are tag values in final result or not
|
||||
bool timeWindowInterpo; // if the time window start/end required interpolation
|
||||
bool queryBlockDist; // if query data block distribution
|
||||
bool stabledev; // super table stddev query
|
||||
bool tsCompQuery; // is tscomp query
|
||||
bool diffQuery; // is diff query
|
||||
bool simpleAgg;
|
||||
bool pointInterpQuery; // point interpolation query
|
||||
bool needReverseScan; // need reverse scan
|
||||
bool distinct; // distinct query or not
|
||||
bool stateWindow; // window State on sub/normal table
|
||||
bool createFilterOperator; // if filter operator is needed
|
||||
bool multigroupResult; // multigroup result can exist in one SSDataBlock
|
||||
int32_t interBufSize; // intermediate buffer sizse
|
||||
|
||||
int32_t havingNum; // having expr number
|
||||
int32_t havingNum; // having expr number
|
||||
|
||||
SOrder order;
|
||||
int16_t numOfCols;
|
||||
int16_t numOfTags;
|
||||
SOrder order;
|
||||
int16_t numOfCols;
|
||||
int16_t numOfTags;
|
||||
|
||||
STimeWindow window;
|
||||
SInterval interval;
|
||||
SSessionWindow sw;
|
||||
int16_t precision;
|
||||
int16_t numOfOutput;
|
||||
int16_t fillType;
|
||||
STimeWindow window;
|
||||
SInterval interval;
|
||||
SSessionWindow sw;
|
||||
int16_t precision;
|
||||
int16_t numOfOutput;
|
||||
int16_t fillType;
|
||||
|
||||
int32_t srcRowSize; // todo extract struct
|
||||
int32_t resultRowSize;
|
||||
int32_t intermediateResultRowSize; // intermediate result row size, in case of top-k query.
|
||||
int32_t maxTableColumnWidth;
|
||||
int32_t tagLen; // tag value length of current query
|
||||
SGroupbyExpr *pGroupbyExpr;
|
||||
int32_t srcRowSize; // todo extract struct
|
||||
int32_t resultRowSize;
|
||||
int32_t intermediateResultRowSize; // intermediate result row size, in case of top-k query.
|
||||
int32_t maxTableColumnWidth;
|
||||
int32_t tagLen; // tag value length of current query
|
||||
SGroupbyExpr* pGroupbyExpr;
|
||||
|
||||
SExprInfo* pExpr1;
|
||||
SExprInfo* pExpr2;
|
||||
int32_t numOfExpr2;
|
||||
SExprInfo* pExpr3;
|
||||
int32_t numOfExpr3;
|
||||
SExprInfo* pExpr1;
|
||||
SExprInfo* pExpr2;
|
||||
int32_t numOfExpr2;
|
||||
SExprInfo* pExpr3;
|
||||
int32_t numOfExpr3;
|
||||
|
||||
SColumnInfo* tableCols;
|
||||
SColumnInfo* tagColList;
|
||||
int32_t numOfFilterCols;
|
||||
int64_t* fillVal;
|
||||
SOrderedPrjQueryInfo prjInfo; // limit value for each vgroup, only available in global order projection query.
|
||||
SColumnInfo* tableCols;
|
||||
SColumnInfo* tagColList;
|
||||
int32_t numOfFilterCols;
|
||||
int64_t* fillVal;
|
||||
SOrderedPrjQueryInfo prjInfo; // limit value for each vgroup, only available in global order projection query.
|
||||
|
||||
SSingleColumnFilterInfo* pFilterInfo;
|
||||
// SFilterInfo *pFilters;
|
||||
|
||||
void* tsdb;
|
||||
STableGroupInfo tableGroupInfo; // table <tid, last_key> list SArray<STableKeyInfo>
|
||||
int32_t vgId;
|
||||
SArray *pUdfInfo; // no need to free
|
||||
// SFilterInfo *pFilters;
|
||||
|
||||
void* tsdb;
|
||||
STableGroupInfo tableGroupInfo; // table <tid, last_key> list SArray<STableKeyInfo>
|
||||
int32_t vgId;
|
||||
SArray* pUdfInfo; // no need to free
|
||||
} STaskAttr;
|
||||
|
||||
typedef int32_t (*__optr_prepare_fn_t)(void* param);
|
||||
|
|
@ -236,176 +239,185 @@ typedef void (*__optr_cleanup_fn_t)(void* param, int32_t num);
|
|||
struct SOperatorInfo;
|
||||
|
||||
typedef struct STaskIdInfo {
|
||||
uint64_t queryId; // this is also a request id
|
||||
uint64_t subplanId;
|
||||
uint64_t templateId;
|
||||
char *str;
|
||||
uint64_t queryId; // this is also a request id
|
||||
uint64_t subplanId;
|
||||
uint64_t templateId;
|
||||
char* str;
|
||||
} STaskIdInfo;
|
||||
|
||||
typedef struct SExecTaskInfo {
|
||||
STaskIdInfo id;
|
||||
char *content;
|
||||
char* content;
|
||||
uint32_t status;
|
||||
STimeWindow window;
|
||||
STaskCostInfo cost;
|
||||
int64_t owner; // if it is in execution
|
||||
int64_t owner; // if it is in execution
|
||||
int32_t code;
|
||||
uint64_t totalRows; // total number of rows
|
||||
uint64_t totalRows; // total number of rows
|
||||
STableGroupInfo tableqinfoGroupInfo; // this is a group array list, including SArray<STableQueryInfo*> structure
|
||||
char *sql; // query sql string
|
||||
jmp_buf env; //
|
||||
struct SOperatorInfo *pRoot;
|
||||
char* sql; // query sql string
|
||||
jmp_buf env; //
|
||||
struct SOperatorInfo* pRoot;
|
||||
} SExecTaskInfo;
|
||||
|
||||
typedef struct STaskRuntimeEnv {
|
||||
jmp_buf env;
|
||||
STaskAttr* pQueryAttr;
|
||||
uint32_t status; // query status
|
||||
void* qinfo;
|
||||
uint8_t scanFlag; // denotes reversed scan of data or not
|
||||
void* pTsdbReadHandle;
|
||||
jmp_buf env;
|
||||
STaskAttr* pQueryAttr;
|
||||
uint32_t status; // query status
|
||||
void* qinfo;
|
||||
uint8_t scanFlag; // denotes reversed scan of data or not
|
||||
void* pTsdbReadHandle;
|
||||
|
||||
int32_t prevGroupId; // previous executed group id
|
||||
bool enableGroupData;
|
||||
SDiskbasedResultBuf* pResultBuf; // query result buffer based on blocked-wised disk file
|
||||
SHashObj* pResultRowHashTable; // quick locate the window object for each result
|
||||
SHashObj* pResultRowListSet; // used to check if current ResultRowInfo has ResultRow object or not
|
||||
SArray* pResultRowArrayList; // The array list that contains the Result rows
|
||||
char* keyBuf; // window key buffer
|
||||
SResultRowPool* pool; // The window result objects pool, all the resultRow Objects are allocated and managed by this object.
|
||||
char** prevRow;
|
||||
int32_t prevGroupId; // previous executed group id
|
||||
bool enableGroupData;
|
||||
SDiskbasedBuf* pResultBuf; // query result buffer based on blocked-wised disk file
|
||||
SHashObj* pResultRowHashTable; // quick locate the window object for each result
|
||||
SHashObj* pResultRowListSet; // used to check if current ResultRowInfo has ResultRow object or not
|
||||
SArray* pResultRowArrayList; // The array list that contains the Result rows
|
||||
char* keyBuf; // window key buffer
|
||||
// The window result objects pool, all the resultRow Objects are allocated and managed by this object.
|
||||
char** prevRow;
|
||||
SResultRowPool* pool;
|
||||
|
||||
SArray* prevResult; // intermediate result, SArray<SInterResult>
|
||||
STSBuf* pTsBuf; // timestamp filter list
|
||||
STSCursor cur;
|
||||
SArray* prevResult; // intermediate result, SArray<SInterResult>
|
||||
STSBuf* pTsBuf; // timestamp filter list
|
||||
STSCursor cur;
|
||||
|
||||
char* tagVal; // tag value of current data block
|
||||
struct SScalarFunctionSupport * scalarSup;
|
||||
char* tagVal; // tag value of current data block
|
||||
struct SScalarFunctionSupport* scalarSup;
|
||||
|
||||
SSDataBlock *outputBuf;
|
||||
STableGroupInfo tableqinfoGroupInfo; // this is a group array list, including SArray<STableQueryInfo*> structure
|
||||
struct SOperatorInfo *proot;
|
||||
SSDataBlock* outputBuf;
|
||||
STableGroupInfo tableqinfoGroupInfo; // this is a group array list, including SArray<STableQueryInfo*> structure
|
||||
struct SOperatorInfo* proot;
|
||||
SGroupResInfo groupResInfo;
|
||||
int64_t currentOffset; // dynamic offset value
|
||||
int64_t currentOffset; // dynamic offset value
|
||||
|
||||
STableQueryInfo *current;
|
||||
SRspResultInfo resultInfo;
|
||||
SHashObj *pTableRetrieveTsMap;
|
||||
struct SUdfInfo *pUdfInfo;
|
||||
STableQueryInfo* current;
|
||||
SRspResultInfo resultInfo;
|
||||
SHashObj* pTableRetrieveTsMap;
|
||||
struct SUdfInfo* pUdfInfo;
|
||||
} STaskRuntimeEnv;
|
||||
|
||||
enum {
|
||||
OP_IN_EXECUTING = 1,
|
||||
OP_RES_TO_RETURN = 2,
|
||||
OP_EXEC_DONE = 3,
|
||||
OP_IN_EXECUTING = 1,
|
||||
OP_RES_TO_RETURN = 2,
|
||||
OP_EXEC_DONE = 3,
|
||||
};
|
||||
|
||||
typedef struct SOperatorInfo {
|
||||
uint8_t operatorType;
|
||||
bool blockingOptr; // block operator or not
|
||||
uint8_t status; // denote if current operator is completed
|
||||
int32_t numOfOutput; // number of columns of the current operator results
|
||||
char *name; // name, used to show the query execution plan
|
||||
void *info; // extension attribution
|
||||
SExprInfo *pExpr;
|
||||
STaskRuntimeEnv *pRuntimeEnv; // todo remove it
|
||||
SExecTaskInfo *pTaskInfo;
|
||||
uint8_t operatorType;
|
||||
bool blockingOptr; // block operator or not
|
||||
uint8_t status; // denote if current operator is completed
|
||||
int32_t numOfOutput; // number of columns of the current operator results
|
||||
char* name; // name, used to show the query execution plan
|
||||
void* info; // extension attribution
|
||||
SExprInfo* pExpr;
|
||||
STaskRuntimeEnv* pRuntimeEnv; // todo remove it
|
||||
SExecTaskInfo* pTaskInfo;
|
||||
|
||||
struct SOperatorInfo **pDownstream; // downstram pointer list
|
||||
int32_t numOfDownstream; // number of downstream. The value is always ONE expect for join operator
|
||||
__optr_prepare_fn_t prepareFn;
|
||||
__operator_fn_t exec;
|
||||
__optr_cleanup_fn_t cleanupFn;
|
||||
struct SOperatorInfo** pDownstream; // downstram pointer list
|
||||
int32_t numOfDownstream; // number of downstream. The value is always ONE expect for join operator
|
||||
__optr_prepare_fn_t prepareFn;
|
||||
__operator_fn_t exec;
|
||||
__optr_cleanup_fn_t cleanupFn;
|
||||
} SOperatorInfo;
|
||||
|
||||
enum {
|
||||
QUERY_RESULT_NOT_READY = 1,
|
||||
QUERY_RESULT_READY = 2,
|
||||
};
|
||||
|
||||
typedef struct {
|
||||
int32_t numOfTags;
|
||||
int32_t numOfCols;
|
||||
SColumnInfo *colList;
|
||||
SColumnInfo* colList;
|
||||
} SQueriedTableInfo;
|
||||
|
||||
typedef struct SQInfo {
|
||||
void* signature;
|
||||
uint64_t qId;
|
||||
int32_t code; // error code to returned to client
|
||||
int64_t owner; // if it is in execution
|
||||
void* signature;
|
||||
uint64_t qId;
|
||||
int32_t code; // error code to returned to client
|
||||
int64_t owner; // if it is in execution
|
||||
|
||||
STaskRuntimeEnv runtimeEnv;
|
||||
STaskAttr query;
|
||||
void* pBuf; // allocated buffer for STableQueryInfo, sizeof(STableQueryInfo)*numOfTables;
|
||||
void* pBuf; // allocated buffer for STableQueryInfo, sizeof(STableQueryInfo)*numOfTables;
|
||||
|
||||
pthread_mutex_t lock; // used to synchronize the rsp/query threads
|
||||
tsem_t ready;
|
||||
int32_t dataReady; // denote if query result is ready or not
|
||||
void* rspContext; // response context
|
||||
int64_t startExecTs; // start to exec timestamp
|
||||
char* sql; // query sql string
|
||||
STaskCostInfo summary;
|
||||
pthread_mutex_t lock; // used to synchronize the rsp/query threads
|
||||
tsem_t ready;
|
||||
int32_t dataReady; // denote if query result is ready or not
|
||||
void* rspContext; // response context
|
||||
int64_t startExecTs; // start to exec timestamp
|
||||
char* sql; // query sql string
|
||||
STaskCostInfo summary;
|
||||
} SQInfo;
|
||||
|
||||
typedef struct STaskParam {
|
||||
char *sql;
|
||||
char *tagCond;
|
||||
char *colCond;
|
||||
char *tbnameCond;
|
||||
char *prevResult;
|
||||
SArray *pTableIdList;
|
||||
SSqlExpr **pExpr;
|
||||
SSqlExpr **pSecExpr;
|
||||
SExprInfo *pExprs;
|
||||
SExprInfo *pSecExprs;
|
||||
char* sql;
|
||||
char* tagCond;
|
||||
char* colCond;
|
||||
char* tbnameCond;
|
||||
char* prevResult;
|
||||
SArray* pTableIdList;
|
||||
SSqlExpr** pExpr;
|
||||
SSqlExpr** pSecExpr;
|
||||
SExprInfo* pExprs;
|
||||
SExprInfo* pSecExprs;
|
||||
|
||||
SFilterInfo *pFilters;
|
||||
SFilterInfo* pFilters;
|
||||
|
||||
SColIndex *pGroupColIndex;
|
||||
SColumnInfo *pTagColumnInfo;
|
||||
SGroupbyExpr *pGroupbyExpr;
|
||||
SColIndex* pGroupColIndex;
|
||||
SColumnInfo* pTagColumnInfo;
|
||||
SGroupbyExpr* pGroupbyExpr;
|
||||
int32_t tableScanOperator;
|
||||
SArray *pOperator;
|
||||
struct SUdfInfo *pUdfInfo;
|
||||
SArray* pOperator;
|
||||
struct SUdfInfo* pUdfInfo;
|
||||
} STaskParam;
|
||||
|
||||
typedef struct SExchangeInfo {
|
||||
SArray *pSources;
|
||||
tsem_t ready;
|
||||
void *pTransporter;
|
||||
SRetrieveTableRsp *pRsp;
|
||||
SSDataBlock *pResult;
|
||||
int32_t current;
|
||||
uint64_t rowsOfCurrentSource;
|
||||
enum {
|
||||
DATA_NOT_READY = 0x1,
|
||||
DATA_READY = 0x2,
|
||||
DATA_EXHAUSTED = 0x3,
|
||||
};
|
||||
|
||||
uint64_t totalSize; // total load bytes from remote
|
||||
uint64_t totalRows; // total number of rows
|
||||
uint64_t totalElapsed;// total elapsed time
|
||||
typedef struct SSourceDataInfo {
|
||||
struct SExchangeInfo *pEx;
|
||||
int32_t index;
|
||||
SRetrieveTableRsp *pRsp;
|
||||
uint64_t totalRows;
|
||||
int32_t status;
|
||||
} SSourceDataInfo;
|
||||
|
||||
typedef struct SExchangeInfo {
|
||||
SArray* pSources;
|
||||
SArray* pSourceDataInfo;
|
||||
tsem_t ready;
|
||||
void* pTransporter;
|
||||
SSDataBlock* pResult;
|
||||
bool seqLoadData; // sequential load data or not, false by default
|
||||
int32_t current;
|
||||
uint64_t totalSize; // total load bytes from remote
|
||||
uint64_t totalRows; // total number of rows
|
||||
uint64_t totalElapsed; // total elapsed time
|
||||
} SExchangeInfo;
|
||||
|
||||
typedef struct STableScanInfo {
|
||||
void *pTsdbReadHandle;
|
||||
int32_t numOfBlocks; // extract basic running information.
|
||||
int32_t numOfSkipped;
|
||||
int32_t numOfBlockStatis;
|
||||
int64_t numOfRows;
|
||||
|
||||
int32_t order; // scan order
|
||||
int32_t times; // repeat counts
|
||||
int32_t current;
|
||||
int32_t reverseTimes; // 0 by default
|
||||
void* pTsdbReadHandle;
|
||||
int32_t numOfBlocks; // extract basic running information.
|
||||
int32_t numOfSkipped;
|
||||
int32_t numOfBlockStatis;
|
||||
int64_t numOfRows;
|
||||
|
||||
SqlFunctionCtx *pCtx; // next operator query context
|
||||
SResultRowInfo *pResultRowInfo;
|
||||
int32_t *rowCellInfoOffset;
|
||||
SExprInfo *pExpr;
|
||||
int32_t order; // scan order
|
||||
int32_t times; // repeat counts
|
||||
int32_t current;
|
||||
int32_t reverseTimes; // 0 by default
|
||||
|
||||
SqlFunctionCtx* pCtx; // next operator query context
|
||||
SResultRowInfo* pResultRowInfo;
|
||||
int32_t* rowCellInfoOffset;
|
||||
SExprInfo* pExpr;
|
||||
SSDataBlock block;
|
||||
int32_t numOfOutput;
|
||||
int64_t elapsedTime;
|
||||
int32_t prevGroupId; // previous table group id
|
||||
|
||||
int32_t scanFlag; // table scan flag to denote if it is a repeat/reverse/main scan
|
||||
int32_t scanFlag; // table scan flag to denote if it is a repeat/reverse/main scan
|
||||
} STableScanInfo;
|
||||
|
||||
typedef struct STagScanInfo {
|
||||
|
|
@ -416,32 +428,36 @@ typedef struct STagScanInfo {
|
|||
} STagScanInfo;
|
||||
|
||||
typedef struct SStreamBlockScanInfo {
|
||||
SSDataBlock *pRes; // result SSDataBlock
|
||||
SColumnInfo *pCols; // the output column info
|
||||
uint64_t numOfRows; // total scanned rows
|
||||
uint64_t numOfExec; // execution times
|
||||
void *readerHandle;// stream block reader handle
|
||||
SSDataBlock* pRes; // result SSDataBlock
|
||||
SColumnInfo* pCols; // the output column info
|
||||
uint64_t numOfRows; // total scanned rows
|
||||
uint64_t numOfExec; // execution times
|
||||
void* readerHandle; // stream block reader handle
|
||||
} SStreamBlockScanInfo;
|
||||
|
||||
typedef struct SOptrBasicInfo {
|
||||
SResultRowInfo resultRowInfo;
|
||||
int32_t *rowCellInfoOffset; // offset value for each row result cell info
|
||||
SqlFunctionCtx *pCtx;
|
||||
SSDataBlock *pRes;
|
||||
SResultRowInfo resultRowInfo;
|
||||
int32_t* rowCellInfoOffset; // offset value for each row result cell info
|
||||
SqlFunctionCtx* pCtx;
|
||||
SSDataBlock* pRes;
|
||||
uint32_t resRowSize;
|
||||
int32_t capacity;
|
||||
} SOptrBasicInfo;
|
||||
|
||||
typedef struct SOptrBasicInfo STableIntervalOperatorInfo;
|
||||
|
||||
typedef struct SAggOperatorInfo {
|
||||
SOptrBasicInfo binfo;
|
||||
uint32_t seed;
|
||||
SDiskbasedResultBuf *pResultBuf; // query result buffer based on blocked-wised disk file
|
||||
SHashObj* pResultRowHashTable; // quick locate the window object for each result
|
||||
SHashObj* pResultRowListSet; // used to check if current ResultRowInfo has ResultRow object or not
|
||||
SArray* pResultRowArrayList; // The array list that contains the Result rows
|
||||
char* keyBuf; // window key buffer
|
||||
SResultRowPool* pool; // The window result objects pool, all the resultRow Objects are allocated and managed by this object.
|
||||
STableQueryInfo *current;
|
||||
SOptrBasicInfo binfo;
|
||||
SDiskbasedBuf *pResultBuf; // query result buffer based on blocked-wised disk file
|
||||
SHashObj* pResultRowHashTable; // quick locate the window object for each result
|
||||
SHashObj* pResultRowListSet; // used to check if current ResultRowInfo has ResultRow object or not
|
||||
SArray* pResultRowArrayList; // The array list that contains the Result rows
|
||||
char* keyBuf; // window key buffer
|
||||
SResultRowPool *pool; // The window result objects pool, all the resultRow Objects are allocated and managed by this object.
|
||||
STableQueryInfo *current;
|
||||
uint32_t groupId;
|
||||
SGroupResInfo groupResInfo;
|
||||
STableQueryInfo *pTableQueryInfo;
|
||||
} SAggOperatorInfo;
|
||||
|
||||
typedef struct SProjectOperatorInfo {
|
||||
|
|
@ -449,52 +465,52 @@ typedef struct SProjectOperatorInfo {
|
|||
int32_t bufCapacity;
|
||||
uint32_t seed;
|
||||
|
||||
SSDataBlock *existDataBlock;
|
||||
SSDataBlock* existDataBlock;
|
||||
} SProjectOperatorInfo;
|
||||
|
||||
typedef struct SLimitOperatorInfo {
|
||||
int64_t limit;
|
||||
int64_t total;
|
||||
int64_t limit;
|
||||
int64_t total;
|
||||
} SLimitOperatorInfo;
|
||||
|
||||
typedef struct SSLimitOperatorInfo {
|
||||
int64_t groupTotal;
|
||||
int64_t currentGroupOffset;
|
||||
int64_t groupTotal;
|
||||
int64_t currentGroupOffset;
|
||||
|
||||
int64_t rowsTotal;
|
||||
int64_t currentOffset;
|
||||
SLimit limit;
|
||||
SLimit slimit;
|
||||
int64_t rowsTotal;
|
||||
int64_t currentOffset;
|
||||
SLimit limit;
|
||||
SLimit slimit;
|
||||
|
||||
char **prevRow;
|
||||
SArray *orderColumnList;
|
||||
char** prevRow;
|
||||
SArray* orderColumnList;
|
||||
bool hasPrev;
|
||||
bool ignoreCurrentGroup;
|
||||
bool multigroupResult;
|
||||
SSDataBlock *pRes; // result buffer
|
||||
SSDataBlock *pPrevBlock;
|
||||
SSDataBlock* pRes; // result buffer
|
||||
SSDataBlock* pPrevBlock;
|
||||
int64_t capacity;
|
||||
int64_t threshold;
|
||||
} SSLimitOperatorInfo;
|
||||
|
||||
typedef struct SFilterOperatorInfo {
|
||||
SSingleColumnFilterInfo *pFilterInfo;
|
||||
int32_t numOfFilterCols;
|
||||
SSingleColumnFilterInfo* pFilterInfo;
|
||||
int32_t numOfFilterCols;
|
||||
} SFilterOperatorInfo;
|
||||
|
||||
typedef struct SFillOperatorInfo {
|
||||
struct SFillInfo *pFillInfo;
|
||||
SSDataBlock *pRes;
|
||||
int64_t totalInputRows;
|
||||
void **p;
|
||||
SSDataBlock *existNewGroupBlock;
|
||||
bool multigroupResult;
|
||||
struct SFillInfo* pFillInfo;
|
||||
SSDataBlock* pRes;
|
||||
int64_t totalInputRows;
|
||||
void** p;
|
||||
SSDataBlock* existNewGroupBlock;
|
||||
bool multigroupResult;
|
||||
} SFillOperatorInfo;
|
||||
|
||||
typedef struct SGroupbyOperatorInfo {
|
||||
SOptrBasicInfo binfo;
|
||||
int32_t colIndex;
|
||||
char *prevData; // previous group by value
|
||||
char* prevData; // previous group by value
|
||||
} SGroupbyOperatorInfo;
|
||||
|
||||
typedef struct SSWindowOperatorInfo {
|
||||
|
|
@ -503,16 +519,16 @@ typedef struct SSWindowOperatorInfo {
|
|||
TSKEY prevTs; // previous timestamp
|
||||
int32_t numOfRows; // number of rows
|
||||
int32_t start; // start row index
|
||||
bool reptScan; // next round scan
|
||||
bool reptScan; // next round scan
|
||||
} SSWindowOperatorInfo;
|
||||
|
||||
typedef struct SStateWindowOperatorInfo {
|
||||
SOptrBasicInfo binfo;
|
||||
STimeWindow curWindow; // current time window
|
||||
int32_t numOfRows; // number of rows
|
||||
int32_t colIndex; // start row index
|
||||
int32_t colIndex; // start row index
|
||||
int32_t start;
|
||||
char* prevData; // previous data
|
||||
char* prevData; // previous data
|
||||
bool reptScan;
|
||||
} SStateWindowOperatorInfo;
|
||||
|
||||
|
|
@ -520,147 +536,186 @@ typedef struct SDistinctDataInfo {
|
|||
int32_t index;
|
||||
int32_t type;
|
||||
int32_t bytes;
|
||||
} SDistinctDataInfo;
|
||||
} SDistinctDataInfo;
|
||||
|
||||
typedef struct SDistinctOperatorInfo {
|
||||
SHashObj *pSet;
|
||||
SSDataBlock *pRes;
|
||||
bool recordNullVal; //has already record the null value, no need to try again
|
||||
int64_t threshold;
|
||||
int64_t outputCapacity;
|
||||
int32_t totalBytes;
|
||||
char* buf;
|
||||
SArray* pDistinctDataInfo;
|
||||
SHashObj* pSet;
|
||||
SSDataBlock* pRes;
|
||||
bool recordNullVal; // has already record the null value, no need to try again
|
||||
int64_t threshold;
|
||||
int64_t outputCapacity;
|
||||
int32_t totalBytes;
|
||||
char* buf;
|
||||
SArray* pDistinctDataInfo;
|
||||
} SDistinctOperatorInfo;
|
||||
|
||||
struct SGlobalMerger;
|
||||
|
||||
typedef struct SMultiwayMergeInfo {
|
||||
struct SGlobalMerger *pMerge;
|
||||
SOptrBasicInfo binfo;
|
||||
int32_t bufCapacity;
|
||||
int64_t seed;
|
||||
char **prevRow;
|
||||
SArray *orderColumnList;
|
||||
int32_t resultRowFactor;
|
||||
struct SGlobalMerger* pMerge;
|
||||
SOptrBasicInfo binfo;
|
||||
int32_t bufCapacity;
|
||||
int64_t seed;
|
||||
char** prevRow;
|
||||
SArray* orderColumnList;
|
||||
int32_t resultRowFactor;
|
||||
|
||||
bool hasGroupColData;
|
||||
char **currentGroupColData;
|
||||
SArray *groupColumnList;
|
||||
bool hasDataBlockForNewGroup;
|
||||
SSDataBlock *pExistBlock;
|
||||
bool hasGroupColData;
|
||||
char** currentGroupColData;
|
||||
SArray* groupColumnList;
|
||||
bool hasDataBlockForNewGroup;
|
||||
SSDataBlock* pExistBlock;
|
||||
|
||||
SArray *udfInfo;
|
||||
bool hasPrev;
|
||||
bool multiGroupResults;
|
||||
SArray* udfInfo;
|
||||
bool hasPrev;
|
||||
bool multiGroupResults;
|
||||
} SMultiwayMergeInfo;
|
||||
|
||||
// todo support the disk-based sort
|
||||
typedef struct SMsortComparParam {
|
||||
struct SExternalMemSource **pSources;
|
||||
int32_t numOfSources;
|
||||
SArray *orderInfo; // SArray<SBlockOrderInfo>
|
||||
bool nullFirst;
|
||||
} SMsortComparParam;
|
||||
|
||||
typedef struct SOrderOperatorInfo {
|
||||
int32_t colIndex;
|
||||
int32_t order;
|
||||
SSDataBlock *pDataBlock;
|
||||
int32_t sourceId;
|
||||
uint32_t sortBufSize; // max buffer size for in-memory sort
|
||||
SSDataBlock *pDataBlock;
|
||||
bool hasVarCol; // has variable length column, such as binary/varchar/nchar
|
||||
int32_t numOfCompleted;
|
||||
SDiskbasedBuf *pSortInternalBuf;
|
||||
SMultiwayMergeTreeInfo *pMergeTree;
|
||||
SArray *pSources; // SArray<SExternalMemSource*>
|
||||
int32_t bufPageSize;
|
||||
int32_t numOfRowsInRes;
|
||||
|
||||
SMsortComparParam cmpParam;
|
||||
|
||||
int64_t startTs; // sort start time
|
||||
uint64_t sortElapsed; // sort elapsed time, time to flush to disk not included.
|
||||
uint64_t totalSize; // total load bytes from remote
|
||||
uint64_t totalRows; // total number of rows
|
||||
uint64_t totalElapsed; // total elapsed time
|
||||
} SOrderOperatorInfo;
|
||||
|
||||
SOperatorInfo* createExchangeOperatorInfo(const SArray* pSources, const SArray* pSchema, SExecTaskInfo* pTaskInfo);
|
||||
|
||||
SOperatorInfo* createDataBlocksOptScanInfo(void* pTsdbReadHandle, int32_t order, int32_t numOfOutput, int32_t repeatTime, int32_t reverseTime, SExecTaskInfo* pTaskInfo);
|
||||
SOperatorInfo* createTableSeqScanOperator(void* pTsdbReadHandle, STaskRuntimeEnv* pRuntimeEnv);
|
||||
SOperatorInfo* createSubmitBlockScanOperatorInfo(void *pSubmitBlockReadHandle, int32_t numOfOutput, SExecTaskInfo* pTaskInfo);
|
||||
|
||||
SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SExecTaskInfo* pTaskInfo);
|
||||
SOperatorInfo* createProjectOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput);
|
||||
SOperatorInfo* createTableScanOperatorInfo(void* pTsdbReadHandle, int32_t order, int32_t numOfOutput,
|
||||
int32_t repeatTime, int32_t reverseTime, SExecTaskInfo* pTaskInfo);
|
||||
SOperatorInfo* createTableSeqScanOperatorInfo(void* pTsdbReadHandle, STaskRuntimeEnv* pRuntimeEnv);
|
||||
SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SExecTaskInfo* pTaskInfo, const STableGroupInfo* pTableGroupInfo);
|
||||
SOperatorInfo* createMultiTableAggOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SExecTaskInfo* pTaskInfo, const STableGroupInfo* pTableGroupInfo);
|
||||
SOperatorInfo* createProjectOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr,
|
||||
int32_t numOfOutput);
|
||||
SOperatorInfo* createLimitOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream);
|
||||
SOperatorInfo* createTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput);
|
||||
SOperatorInfo* createAllTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput);
|
||||
SOperatorInfo* createSWindowOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput);
|
||||
SOperatorInfo* createFillOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput, bool multigroupResult);
|
||||
SOperatorInfo* createGroupbyOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput);
|
||||
SOperatorInfo* createMultiTableAggOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput);
|
||||
SOperatorInfo* createMultiTableTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput);
|
||||
SOperatorInfo* createAllMultiTableTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput);
|
||||
SOperatorInfo* createTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr,
|
||||
int32_t numOfOutput);
|
||||
SOperatorInfo* createAllTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream,
|
||||
SExprInfo* pExpr, int32_t numOfOutput);
|
||||
SOperatorInfo* createSWindowOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr,
|
||||
int32_t numOfOutput);
|
||||
SOperatorInfo* createFillOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr,
|
||||
int32_t numOfOutput, bool multigroupResult);
|
||||
SOperatorInfo* createGroupbyOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr,
|
||||
int32_t numOfOutput);
|
||||
|
||||
SOperatorInfo* createMultiTableTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream,
|
||||
SExprInfo* pExpr, int32_t numOfOutput);
|
||||
SOperatorInfo* createAllMultiTableTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream,
|
||||
SExprInfo* pExpr, int32_t numOfOutput);
|
||||
SOperatorInfo* createTagScanOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SExprInfo* pExpr, int32_t numOfOutput);
|
||||
SOperatorInfo* createDistinctOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput);
|
||||
SOperatorInfo* createDistinctOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr,
|
||||
int32_t numOfOutput);
|
||||
SOperatorInfo* createTableBlockInfoScanOperator(void* pTsdbReadHandle, STaskRuntimeEnv* pRuntimeEnv);
|
||||
SOperatorInfo* createMultiwaySortOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SExprInfo* pExpr, int32_t numOfOutput,
|
||||
int32_t numOfRows, void* merger);
|
||||
SOperatorInfo* createGlobalAggregateOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput, void* param, SArray* pUdfInfo, bool groupResultMixedUp);
|
||||
SOperatorInfo* createStatewindowOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput);
|
||||
SOperatorInfo* createSLimitOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput, void* merger, bool multigroupResult);
|
||||
SOperatorInfo* createGlobalAggregateOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream,
|
||||
SExprInfo* pExpr, int32_t numOfOutput, void* param, SArray* pUdfInfo,
|
||||
bool groupResultMixedUp);
|
||||
SOperatorInfo* createStatewindowOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr,
|
||||
int32_t numOfOutput);
|
||||
SOperatorInfo* createSLimitOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr,
|
||||
int32_t numOfOutput, void* merger, bool multigroupResult);
|
||||
SOperatorInfo* createFilterOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr,
|
||||
int32_t numOfOutput, SColumnInfo* pCols, int32_t numOfFilter);
|
||||
|
||||
SOperatorInfo* createJoinOperatorInfo(SOperatorInfo** pdownstream, int32_t numOfDownstream, SSchema* pSchema, int32_t numOfOutput);
|
||||
SOperatorInfo* createOrderOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput, SOrder* pOrderVal);
|
||||
SOperatorInfo* createJoinOperatorInfo(SOperatorInfo** pdownstream, int32_t numOfDownstream, SSchema* pSchema,
|
||||
int32_t numOfOutput);
|
||||
SOperatorInfo* createOrderOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SArray* pOrderVal);
|
||||
SOperatorInfo* createMergeSortOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput,
|
||||
SOrder* pOrderVal);
|
||||
|
||||
//SSDataBlock* doGlobalAggregate(void* param, bool* newgroup);
|
||||
//SSDataBlock* doMultiwayMergeSort(void* param, bool* newgroup);
|
||||
//SSDataBlock* doSLimit(void* param, bool* newgroup);
|
||||
// SSDataBlock* doGlobalAggregate(void* param, bool* newgroup);
|
||||
// SSDataBlock* doMultiwayMergeSort(void* param, bool* newgroup);
|
||||
// SSDataBlock* doSLimit(void* param, bool* newgroup);
|
||||
|
||||
//int32_t doCreateFilterInfo(SColumnInfo* pCols, int32_t numOfCols, int32_t numOfFilterCols, SSingleColumnFilterInfo** pFilterInfo, uint64_t qId);
|
||||
// int32_t doCreateFilterInfo(SColumnInfo* pCols, int32_t numOfCols, int32_t numOfFilterCols, SSingleColumnFilterInfo** pFilterInfo, uint64_t qId);
|
||||
void doSetFilterColumnInfo(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols, SSDataBlock* pBlock);
|
||||
bool doFilterDataBlock(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols, int32_t numOfRows, int8_t* p);
|
||||
void doCompactSDataBlock(SSDataBlock* pBlock, int32_t numOfRows, int8_t* p);
|
||||
|
||||
SSDataBlock* createOutputBuf(SExprInfo* pExpr, int32_t numOfOutput, int32_t numOfRows);
|
||||
|
||||
void* destroyOutputBuf(SSDataBlock* pBlock);
|
||||
void* doDestroyFilterInfo(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols);
|
||||
|
||||
void setInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order);
|
||||
void finalizeQueryResult(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SResultRowInfo* pResultRowInfo, int32_t* rowCellInfoOffset);
|
||||
void updateOutputBuf(SOptrBasicInfo* pBInfo, int32_t *bufCapacity, int32_t numOfInputRows);
|
||||
void clearOutputBuf(SOptrBasicInfo* pBInfo, int32_t *bufCapacity);
|
||||
void finalizeQueryResult(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SResultRowInfo* pResultRowInfo,
|
||||
int32_t* rowCellInfoOffset);
|
||||
void updateOutputBuf(SOptrBasicInfo* pBInfo, int32_t* bufCapacity, int32_t numOfInputRows);
|
||||
void clearOutputBuf(SOptrBasicInfo* pBInfo, int32_t* bufCapacity);
|
||||
void copyTsColoum(SSDataBlock* pRes, SqlFunctionCtx* pCtx, int32_t numOfOutput);
|
||||
|
||||
void freeParam(STaskParam *param);
|
||||
int32_t createQueryFunc(SQueriedTableInfo* pTableInfo, int32_t numOfOutput, SExprInfo** pExprInfo,
|
||||
SSqlExpr** pExprMsg, SColumnInfo* pTagCols, int32_t queryType, void* pMsg, struct SUdfInfo* pUdfInfo);
|
||||
int32_t createQueryFunc(SQueriedTableInfo* pTableInfo, int32_t numOfOutput, SExprInfo** pExprInfo, SSqlExpr** pExprMsg,
|
||||
SColumnInfo* pTagCols, int32_t queryType, void* pMsg, struct SUdfInfo* pUdfInfo);
|
||||
|
||||
int32_t createIndirectQueryFuncExprFromMsg(SQueryTableReq *pQueryMsg, int32_t numOfOutput, SExprInfo **pExprInfo,
|
||||
SSqlExpr **pExpr, SExprInfo *prevExpr, struct SUdfInfo *pUdfInfo);
|
||||
int32_t createIndirectQueryFuncExprFromMsg(SQueryTableReq* pQueryMsg, int32_t numOfOutput, SExprInfo** pExprInfo,
|
||||
SSqlExpr** pExpr, SExprInfo* prevExpr, struct SUdfInfo* pUdfInfo);
|
||||
|
||||
int32_t createQueryFilter(char *data, uint16_t len, SFilterInfo** pFilters);
|
||||
int32_t createQueryFilter(char* data, uint16_t len, SFilterInfo** pFilters);
|
||||
|
||||
SGroupbyExpr *createGroupbyExprFromMsg(SQueryTableReq *pQueryMsg, SColIndex *pColIndex, int32_t *code);
|
||||
SGroupbyExpr* createGroupbyExprFromMsg(SQueryTableReq* pQueryMsg, SColIndex* pColIndex, int32_t* code);
|
||||
|
||||
int32_t initQInfo(STsBufInfo* pTsBufInfo, void* tsdb, void* sourceOptr, SQInfo* pQInfo, STaskParam* param, char* start,
|
||||
int32_t prevResultLen, void* merger);
|
||||
|
||||
int32_t createFilterInfo(STaskAttr* pQueryAttr, uint64_t qId);
|
||||
void freeColumnFilterInfo(SColumnFilterInfo* pFilter, int32_t numOfFilters);
|
||||
void freeColumnFilterInfo(SColumnFilterInfo* pFilter, int32_t numOfFilters);
|
||||
|
||||
STableQueryInfo *createTableQueryInfo(STaskAttr* pQueryAttr, void* pTable, bool groupbyColumn, STimeWindow win, void* buf);
|
||||
STableQueryInfo* createTableQueryInfo(void* buf, bool groupbyColumn, STimeWindow win);
|
||||
STableQueryInfo* createTmpTableQueryInfo(STimeWindow win);
|
||||
|
||||
int32_t buildArithmeticExprFromMsg(SExprInfo *pArithExprInfo, void *pQueryMsg);
|
||||
int32_t buildArithmeticExprFromMsg(SExprInfo* pArithExprInfo, void* pQueryMsg);
|
||||
|
||||
bool isTaskKilled(SExecTaskInfo *pTaskInfo);
|
||||
bool isTaskKilled(SExecTaskInfo* pTaskInfo);
|
||||
int32_t checkForQueryBuf(size_t numOfTables);
|
||||
bool checkNeedToCompressQueryCol(SQInfo *pQInfo);
|
||||
void setQueryStatus(STaskRuntimeEnv *pRuntimeEnv, int8_t status);
|
||||
bool checkNeedToCompressQueryCol(SQInfo* pQInfo);
|
||||
void setQueryStatus(STaskRuntimeEnv* pRuntimeEnv, int8_t status);
|
||||
|
||||
bool onlyQueryTags(STaskAttr* pQueryAttr);
|
||||
//void destroyUdfInfo(struct SUdfInfo* pUdfInfo);
|
||||
// void destroyUdfInfo(struct SUdfInfo* pUdfInfo);
|
||||
|
||||
int32_t doDumpQueryResult(SQInfo *pQInfo, char *data, int8_t compressed, int32_t *compLen);
|
||||
int32_t doDumpQueryResult(SQInfo* pQInfo, char* data, int8_t compressed, int32_t* compLen);
|
||||
|
||||
size_t getResultSize(SQInfo *pQInfo, int64_t *numOfRows);
|
||||
void setTaskKilled(SExecTaskInfo *pTaskInfo);
|
||||
size_t getResultSize(SQInfo* pQInfo, int64_t* numOfRows);
|
||||
void setTaskKilled(SExecTaskInfo* pTaskInfo);
|
||||
|
||||
void publishOperatorProfEvent(SOperatorInfo* operatorInfo, EQueryProfEventType eventType);
|
||||
void publishQueryAbortEvent(SExecTaskInfo * pTaskInfo, int32_t code);
|
||||
void publishQueryAbortEvent(SExecTaskInfo* pTaskInfo, int32_t code);
|
||||
|
||||
void calculateOperatorProfResults(SQInfo* pQInfo);
|
||||
void queryCostStatis(SExecTaskInfo *pTaskInfo);
|
||||
void queryCostStatis(SExecTaskInfo* pTaskInfo);
|
||||
|
||||
void doDestroyTask(SExecTaskInfo *pTaskInfo);
|
||||
void freeQueryAttr(STaskAttr *pQuery);
|
||||
void doDestroyTask(SExecTaskInfo* pTaskInfo);
|
||||
void freeQueryAttr(STaskAttr* pQuery);
|
||||
|
||||
int32_t getMaximumIdleDurationSec();
|
||||
|
||||
void doInvokeUdf(struct SUdfInfo* pUdfInfo, SqlFunctionCtx *pCtx, int32_t idx, int32_t type);
|
||||
void setTaskStatus(SExecTaskInfo *pTaskInfo, int8_t status);
|
||||
void doInvokeUdf(struct SUdfInfo* pUdfInfo, SqlFunctionCtx* pCtx, int32_t idx, int32_t type);
|
||||
void setTaskStatus(SExecTaskInfo* pTaskInfo, int8_t status);
|
||||
int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SReadHandle* pHandle, uint64_t taskId);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif // TDENGINE_EXECUTORIMPL_H
|
||||
|
|
|
|||
|
|
@ -141,9 +141,9 @@ void clearResultRow(STaskRuntimeEnv *pRuntimeEnv, SResultRow *pResultRow, int16_
|
|||
return;
|
||||
}
|
||||
|
||||
// the result does not put into the SDiskbasedResultBuf, ignore it.
|
||||
// the result does not put into the SDiskbasedBuf, ignore it.
|
||||
if (pResultRow->pageId >= 0) {
|
||||
SFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pResultRow->pageId);
|
||||
SFilePage *page = getBufPage(pRuntimeEnv->pResultBuf, pResultRow->pageId);
|
||||
|
||||
int16_t offset = 0;
|
||||
for (int32_t i = 0; i < pRuntimeEnv->pQueryAttr->numOfOutput; ++i) {
|
||||
|
|
@ -358,7 +358,6 @@ void initGroupResInfo(SGroupResInfo* pGroupResInfo, SResultRowInfo* pResultInfo)
|
|||
|
||||
pGroupResInfo->pRows = taosArrayFromList(pResultInfo->pResult, pResultInfo->size, POINTER_BYTES);
|
||||
pGroupResInfo->index = 0;
|
||||
|
||||
assert(pGroupResInfo->index <= getNumOfTotalRes(pGroupResInfo));
|
||||
}
|
||||
|
||||
|
|
@ -533,7 +532,7 @@ static UNUSED_FUNC int32_t mergeIntoGroupResultImpl(STaskRuntimeEnv *pRuntimeEnv
|
|||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
|
||||
int32_t *posList = NULL;
|
||||
SLoserTreeInfo *pTree = NULL;
|
||||
SMultiwayMergeTreeInfo *pTree = NULL;
|
||||
STableQueryInfo **pTableQueryInfoList = NULL;
|
||||
|
||||
size_t size = taosArrayGetSize(pTableList);
|
||||
|
|
@ -566,7 +565,7 @@ static UNUSED_FUNC int32_t mergeIntoGroupResultImpl(STaskRuntimeEnv *pRuntimeEnv
|
|||
|
||||
SCompSupporter cs = {pTableQueryInfoList, posList, pRuntimeEnv->pQueryAttr->order.order};
|
||||
|
||||
int32_t ret = tLoserTreeCreate(&pTree, numOfTables, &cs, tableResultComparFn);
|
||||
int32_t ret = tMergeTreeCreate(&pTree, numOfTables, &cs, tableResultComparFn);
|
||||
if (ret != TSDB_CODE_SUCCESS) {
|
||||
code = TSDB_CODE_QRY_OUT_OF_MEMORY;
|
||||
goto _end;
|
||||
|
|
@ -576,7 +575,7 @@ static UNUSED_FUNC int32_t mergeIntoGroupResultImpl(STaskRuntimeEnv *pRuntimeEnv
|
|||
int64_t startt = taosGetTimestampMs();
|
||||
|
||||
while (1) {
|
||||
int32_t tableIndex = pTree->pNode[0].index;
|
||||
int32_t tableIndex = tMergeTreeGetChosenIndex(pTree);
|
||||
|
||||
SResultRowInfo *pWindowResInfo = &pTableQueryInfoList[tableIndex]->resInfo;
|
||||
SResultRow *pWindowRes = getResultRow(pWindowResInfo, cs.rowIndex[tableIndex]);
|
||||
|
|
@ -612,7 +611,7 @@ static UNUSED_FUNC int32_t mergeIntoGroupResultImpl(STaskRuntimeEnv *pRuntimeEnv
|
|||
}
|
||||
}
|
||||
|
||||
tLoserTreeAdjust(pTree, tableIndex + pTree->numOfEntries);
|
||||
tMergeTreeAdjust(pTree, tMergeTreeGetAdjustIndex(pTree));
|
||||
}
|
||||
|
||||
int64_t endt = taosGetTimestampMs();
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load Diff
|
|
@ -33,193 +33,312 @@
|
|||
#include "stub.h"
|
||||
#include "executor.h"
|
||||
|
||||
/**
|
||||
{
|
||||
"Id": {
|
||||
"QueryId": 1.3108161807422521e+19,
|
||||
"TemplateId": 0,
|
||||
"SubplanId": 0
|
||||
},
|
||||
"Node": {
|
||||
"Name": "TableScan",
|
||||
"Targets": [{
|
||||
"Base": {
|
||||
"Schema": {
|
||||
"Type": 9,
|
||||
"ColId": 5000,
|
||||
"Bytes": 8
|
||||
},
|
||||
"Columns": [{
|
||||
"TableId": 1,
|
||||
"Flag": 0,
|
||||
"Info": {
|
||||
"ColId": 1,
|
||||
"Type": 9,
|
||||
"Bytes": 8
|
||||
}
|
||||
}],
|
||||
"InterBytes": 0
|
||||
},
|
||||
"Expr": {
|
||||
"Type": 4,
|
||||
"Column": {
|
||||
"Type": 9,
|
||||
"ColId": 1,
|
||||
"Bytes": 8
|
||||
}
|
||||
}
|
||||
}, {
|
||||
"Base": {
|
||||
"Schema": {
|
||||
"Type": 4,
|
||||
"ColId": 5001,
|
||||
"Bytes": 4
|
||||
},
|
||||
"Columns": [{
|
||||
"TableId": 1,
|
||||
"Flag": 0,
|
||||
"Info": {
|
||||
"ColId": 2,
|
||||
"Type": 4,
|
||||
"Bytes": 4
|
||||
}
|
||||
}],
|
||||
"InterBytes": 0
|
||||
},
|
||||
"Expr": {
|
||||
"Type": 4,
|
||||
"Column": {
|
||||
"Type": 4,
|
||||
"ColId": 2,
|
||||
"Bytes": 4
|
||||
}
|
||||
}
|
||||
}],
|
||||
"InputSchema": [{
|
||||
"Type": 9,
|
||||
"ColId": 5000,
|
||||
"Bytes": 8
|
||||
}, {
|
||||
"Type": 4,
|
||||
"ColId": 5001,
|
||||
"Bytes": 4
|
||||
}],
|
||||
"TableScan": {
|
||||
"TableId": 1,
|
||||
"TableType": 2,
|
||||
"Flag": 0,
|
||||
"Window": {
|
||||
"StartKey": -9.2233720368547758e+18,
|
||||
"EndKey": 9.2233720368547758e+18
|
||||
}
|
||||
}
|
||||
},
|
||||
"DataSink": {
|
||||
"Name": "Dispatch",
|
||||
"Dispatch": {
|
||||
}
|
||||
}
|
||||
}
|
||||
*/
|
||||
namespace {
|
||||
|
||||
typedef struct SDummyInputInfo {
|
||||
int32_t max;
|
||||
int32_t current;
|
||||
int32_t startVal;
|
||||
SSDataBlock* pBlock;
|
||||
} SDummyInputInfo;
|
||||
|
||||
SSDataBlock* getDummyBlock(void* param, bool* newgroup) {
|
||||
SOperatorInfo* pOperator = static_cast<SOperatorInfo*>(param);
|
||||
SDummyInputInfo* pInfo = static_cast<SDummyInputInfo*>(pOperator->info);
|
||||
if (pInfo->current >= pInfo->max) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
int32_t numOfRows = 1000;
|
||||
|
||||
if (pInfo->pBlock == NULL) {
|
||||
pInfo->pBlock = static_cast<SSDataBlock*>(calloc(1, sizeof(SSDataBlock)));
|
||||
|
||||
pInfo->pBlock->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData));
|
||||
|
||||
SColumnInfoData colInfo = {0};
|
||||
colInfo.info.type = TSDB_DATA_TYPE_INT;
|
||||
colInfo.info.bytes = sizeof(int32_t);
|
||||
colInfo.info.colId = 1;
|
||||
colInfo.pData = static_cast<char*>(calloc(numOfRows, sizeof(int32_t)));
|
||||
colInfo.nullbitmap = static_cast<char*>(calloc(1, (numOfRows + 7) / 8));
|
||||
|
||||
taosArrayPush(pInfo->pBlock->pDataBlock, &colInfo);
|
||||
|
||||
// SColumnInfoData colInfo1 = {0};
|
||||
// colInfo1.info.type = TSDB_DATA_TYPE_BINARY;
|
||||
// colInfo1.info.bytes = 40;
|
||||
// colInfo1.info.colId = 2;
|
||||
//
|
||||
// colInfo1.varmeta.allocLen = 0;//numOfRows * sizeof(int32_t);
|
||||
// colInfo1.varmeta.length = 0;
|
||||
// colInfo1.varmeta.offset = static_cast<int32_t*>(calloc(1, numOfRows * sizeof(int32_t)));
|
||||
//
|
||||
// taosArrayPush(pInfo->pBlock->pDataBlock, &colInfo1);
|
||||
} else {
|
||||
blockDataClearup(pInfo->pBlock, true);
|
||||
}
|
||||
|
||||
SSDataBlock* pBlock = pInfo->pBlock;
|
||||
|
||||
char buf[128] = {0};
|
||||
char b1[128] = {0};
|
||||
for(int32_t i = 0; i < numOfRows; ++i) {
|
||||
SColumnInfoData* pColInfo = static_cast<SColumnInfoData*>(TARRAY_GET_ELEM(pBlock->pDataBlock, 0));
|
||||
|
||||
int32_t v = (--pInfo->startVal);
|
||||
colDataAppend(pColInfo, i, reinterpret_cast<const char*>(&v), false);
|
||||
|
||||
// sprintf(buf, "this is %d row", i);
|
||||
// STR_TO_VARSTR(b1, buf);
|
||||
//
|
||||
// SColumnInfoData* pColInfo2 = static_cast<SColumnInfoData*>(TARRAY_GET_ELEM(pBlock->pDataBlock, 1));
|
||||
// colDataAppend(pColInfo2, i, b1, false);
|
||||
}
|
||||
|
||||
pBlock->info.rows = numOfRows;
|
||||
pBlock->info.numOfCols = 1;
|
||||
|
||||
pInfo->current += 1;
|
||||
return pBlock;
|
||||
}
|
||||
|
||||
SOperatorInfo* createDummyOperator(int32_t numOfBlocks) {
|
||||
SOperatorInfo* pOperator = static_cast<SOperatorInfo*>(calloc(1, sizeof(SOperatorInfo)));
|
||||
pOperator->name = "dummyInputOpertor4Test";
|
||||
pOperator->exec = getDummyBlock;
|
||||
|
||||
SDummyInputInfo *pInfo = (SDummyInputInfo*) calloc(1, sizeof(SDummyInputInfo));
|
||||
pInfo->max = numOfBlocks;
|
||||
pInfo->startVal = 1500000;
|
||||
|
||||
pOperator->info = pInfo;
|
||||
return pOperator;
|
||||
}
|
||||
}
|
||||
int main(int argc, char** argv) {
|
||||
testing::InitGoogleTest(&argc, argv);
|
||||
return RUN_ALL_TESTS();
|
||||
}
|
||||
|
||||
TEST(testCase, build_executor_tree_Test) {
|
||||
|
||||
|
||||
const char* msg = "{\n"
|
||||
"\t\"Id\":\t{\n"
|
||||
"\t\t\"QueryId\":\t1.3108161807422521e+19,\n"
|
||||
"\t\t\"TemplateId\":\t0,\n"
|
||||
"\t\t\"SubplanId\":\t0\n"
|
||||
"\t},\n"
|
||||
"\t\"Node\":\t{\n"
|
||||
"\t\t\"Name\":\t\"TableScan\",\n"
|
||||
"\t\t\"Targets\":\t[{\n"
|
||||
"\t\t\t\t\"Base\":\t{\n"
|
||||
"\t\t\t\t\t\"Schema\":\t{\n"
|
||||
"\t\t\t\t\t\t\"Type\":\t9,\n"
|
||||
"\t\t\t\t\t\t\"ColId\":\t5000,\n"
|
||||
"\t\t\t\t\t\t\"Bytes\":\t8\n"
|
||||
"\t\t\t\t\t},\n"
|
||||
"\t\t\t\t\t\"Columns\":\t[{\n"
|
||||
"\t\t\t\t\t\t\t\"TableId\":\t1,\n"
|
||||
"\t\t\t\t\t\t\t\"Flag\":\t0,\n"
|
||||
"\t\t\t\t\t\t\t\"Info\":\t{\n"
|
||||
"\t\t\t\t\t\t\t\t\"ColId\":\t1,\n"
|
||||
"\t\t\t\t\t\t\t\t\"Type\":\t9,\n"
|
||||
"\t\t\t\t\t\t\t\t\"Bytes\":\t8\n"
|
||||
"\t\t\t\t\t\t\t}\n"
|
||||
"\t\t\t\t\t\t}],\n"
|
||||
"\t\t\t\t\t\"InterBytes\":\t0\n"
|
||||
"\t\t\t\t},\n"
|
||||
"\t\t\t\t\"Expr\":\t{\n"
|
||||
"\t\t\t\t\t\"Type\":\t4,\n"
|
||||
"\t\t\t\t\t\"Column\":\t{\n"
|
||||
"\t\t\t\t\t\t\"Type\":\t9,\n"
|
||||
"\t\t\t\t\t\t\"ColId\":\t1,\n"
|
||||
"\t\t\t\t\t\t\"Bytes\":\t8\n"
|
||||
"\t\t\t\t\t}\n"
|
||||
"\t\t\t\t}\n"
|
||||
"\t\t\t}, {\n"
|
||||
"\t\t\t\t\"Base\":\t{\n"
|
||||
"\t\t\t\t\t\"Schema\":\t{\n"
|
||||
"\t\t\t\t\t\t\"Type\":\t4,\n"
|
||||
"\t\t\t\t\t\t\"ColId\":\t5001,\n"
|
||||
"\t\t\t\t\t\t\"Bytes\":\t4\n"
|
||||
"\t\t\t\t\t},\n"
|
||||
"\t\t\t\t\t\"Columns\":\t[{\n"
|
||||
"\t\t\t\t\t\t\t\"TableId\":\t1,\n"
|
||||
"\t\t\t\t\t\t\t\"Flag\":\t0,\n"
|
||||
"\t\t\t\t\t\t\t\"Info\":\t{\n"
|
||||
"\t\t\t\t\t\t\t\t\"ColId\":\t2,\n"
|
||||
"\t\t\t\t\t\t\t\t\"Type\":\t4,\n"
|
||||
"\t\t\t\t\t\t\t\t\"Bytes\":\t4\n"
|
||||
"\t\t\t\t\t\t\t}\n"
|
||||
"\t\t\t\t\t\t}],\n"
|
||||
"\t\t\t\t\t\"InterBytes\":\t0\n"
|
||||
"\t\t\t\t},\n"
|
||||
"\t\t\t\t\"Expr\":\t{\n"
|
||||
"\t\t\t\t\t\"Type\":\t4,\n"
|
||||
"\t\t\t\t\t\"Column\":\t{\n"
|
||||
"\t\t\t\t\t\t\"Type\":\t4,\n"
|
||||
"\t\t\t\t\t\t\"ColId\":\t2,\n"
|
||||
"\t\t\t\t\t\t\"Bytes\":\t4\n"
|
||||
"\t\t\t\t\t}\n"
|
||||
"\t\t\t\t}\n"
|
||||
"\t\t\t}],\n"
|
||||
"\t\t\"InputSchema\":\t[{\n"
|
||||
"\t\t\t\t\"Type\":\t9,\n"
|
||||
"\t\t\t\t\"ColId\":\t5000,\n"
|
||||
"\t\t\t\t\"Bytes\":\t8\n"
|
||||
"\t\t\t}, {\n"
|
||||
"\t\t\t\t\"Type\":\t4,\n"
|
||||
"\t\t\t\t\"ColId\":\t5001,\n"
|
||||
"\t\t\t\t\"Bytes\":\t4\n"
|
||||
"\t\t\t}],\n"
|
||||
"\t\t\"TableScan\":\t{\n"
|
||||
"\t\t\t\"TableId\":\t1,\n"
|
||||
"\t\t\t\"TableType\":\t2,\n"
|
||||
"\t\t\t\"Flag\":\t0,\n"
|
||||
"\t\t\t\"Window\":\t{\n"
|
||||
"\t\t\t\t\"StartKey\":\t-9.2233720368547758e+18,\n"
|
||||
"\t\t\t\t\"EndKey\":\t9.2233720368547758e+18\n"
|
||||
"\t\t\t}\n"
|
||||
"\t\t}\n"
|
||||
"\t},\n"
|
||||
"\t\"DataSink\":\t{\n"
|
||||
"\t\t\"Name\":\t\"Dispatch\",\n"
|
||||
"\t\t\"Dispatch\":\t{\n"
|
||||
"\t\t}\n"
|
||||
"\t}\n"
|
||||
"}";
|
||||
"\t\"Id\":\t{\n"
|
||||
"\t\t\"QueryId\":\t1.3108161807422521e+19,\n"
|
||||
"\t\t\"TemplateId\":\t0,\n"
|
||||
"\t\t\"SubplanId\":\t0\n"
|
||||
"\t},\n"
|
||||
"\t\"Node\":\t{\n"
|
||||
"\t\t\"Name\":\t\"TableScan\",\n"
|
||||
"\t\t\"Targets\":\t[{\n"
|
||||
"\t\t\t\t\"Base\":\t{\n"
|
||||
"\t\t\t\t\t\"Schema\":\t{\n"
|
||||
"\t\t\t\t\t\t\"Type\":\t9,\n"
|
||||
"\t\t\t\t\t\t\"ColId\":\t5000,\n"
|
||||
"\t\t\t\t\t\t\"Bytes\":\t8\n"
|
||||
"\t\t\t\t\t},\n"
|
||||
"\t\t\t\t\t\"Columns\":\t[{\n"
|
||||
"\t\t\t\t\t\t\t\"TableId\":\t1,\n"
|
||||
"\t\t\t\t\t\t\t\"Flag\":\t0,\n"
|
||||
"\t\t\t\t\t\t\t\"Info\":\t{\n"
|
||||
"\t\t\t\t\t\t\t\t\"ColId\":\t1,\n"
|
||||
"\t\t\t\t\t\t\t\t\"Type\":\t9,\n"
|
||||
"\t\t\t\t\t\t\t\t\"Bytes\":\t8\n"
|
||||
"\t\t\t\t\t\t\t}\n"
|
||||
"\t\t\t\t\t\t}],\n"
|
||||
"\t\t\t\t\t\"InterBytes\":\t0\n"
|
||||
"\t\t\t\t},\n"
|
||||
"\t\t\t\t\"Expr\":\t{\n"
|
||||
"\t\t\t\t\t\"Type\":\t4,\n"
|
||||
"\t\t\t\t\t\"Column\":\t{\n"
|
||||
"\t\t\t\t\t\t\"Type\":\t9,\n"
|
||||
"\t\t\t\t\t\t\"ColId\":\t1,\n"
|
||||
"\t\t\t\t\t\t\"Bytes\":\t8\n"
|
||||
"\t\t\t\t\t}\n"
|
||||
"\t\t\t\t}\n"
|
||||
"\t\t\t}, {\n"
|
||||
"\t\t\t\t\"Base\":\t{\n"
|
||||
"\t\t\t\t\t\"Schema\":\t{\n"
|
||||
"\t\t\t\t\t\t\"Type\":\t4,\n"
|
||||
"\t\t\t\t\t\t\"ColId\":\t5001,\n"
|
||||
"\t\t\t\t\t\t\"Bytes\":\t4\n"
|
||||
"\t\t\t\t\t},\n"
|
||||
"\t\t\t\t\t\"Columns\":\t[{\n"
|
||||
"\t\t\t\t\t\t\t\"TableId\":\t1,\n"
|
||||
"\t\t\t\t\t\t\t\"Flag\":\t0,\n"
|
||||
"\t\t\t\t\t\t\t\"Info\":\t{\n"
|
||||
"\t\t\t\t\t\t\t\t\"ColId\":\t2,\n"
|
||||
"\t\t\t\t\t\t\t\t\"Type\":\t4,\n"
|
||||
"\t\t\t\t\t\t\t\t\"Bytes\":\t4\n"
|
||||
"\t\t\t\t\t\t\t}\n"
|
||||
"\t\t\t\t\t\t}],\n"
|
||||
"\t\t\t\t\t\"InterBytes\":\t0\n"
|
||||
"\t\t\t\t},\n"
|
||||
"\t\t\t\t\"Expr\":\t{\n"
|
||||
"\t\t\t\t\t\"Type\":\t4,\n"
|
||||
"\t\t\t\t\t\"Column\":\t{\n"
|
||||
"\t\t\t\t\t\t\"Type\":\t4,\n"
|
||||
"\t\t\t\t\t\t\"ColId\":\t2,\n"
|
||||
"\t\t\t\t\t\t\"Bytes\":\t4\n"
|
||||
"\t\t\t\t\t}\n"
|
||||
"\t\t\t\t}\n"
|
||||
"\t\t\t}],\n"
|
||||
"\t\t\"InputSchema\":\t[{\n"
|
||||
"\t\t\t\t\"Type\":\t9,\n"
|
||||
"\t\t\t\t\"ColId\":\t5000,\n"
|
||||
"\t\t\t\t\"Bytes\":\t8\n"
|
||||
"\t\t\t}, {\n"
|
||||
"\t\t\t\t\"Type\":\t4,\n"
|
||||
"\t\t\t\t\"ColId\":\t5001,\n"
|
||||
"\t\t\t\t\"Bytes\":\t4\n"
|
||||
"\t\t\t}],\n"
|
||||
"\t\t\"TableScan\":\t{\n"
|
||||
"\t\t\t\"TableId\":\t1,\n"
|
||||
"\t\t\t\"TableType\":\t2,\n"
|
||||
"\t\t\t\"Flag\":\t0,\n"
|
||||
"\t\t\t\"Window\":\t{\n"
|
||||
"\t\t\t\t\"StartKey\":\t-9.2233720368547758e+18,\n"
|
||||
"\t\t\t\t\"EndKey\":\t9.2233720368547758e+18\n"
|
||||
"\t\t\t}\n"
|
||||
"\t\t}\n"
|
||||
"\t},\n"
|
||||
"\t\"DataSink\":\t{\n"
|
||||
"\t\t\"Name\":\t\"Dispatch\",\n"
|
||||
"\t\t\"Dispatch\":\t{\n"
|
||||
"\t\t}\n"
|
||||
"\t}\n"
|
||||
"}";
|
||||
|
||||
SExecTaskInfo* pTaskInfo = nullptr;
|
||||
DataSinkHandle sinkHandle = nullptr;
|
||||
int32_t code = qCreateExecTask((SReadHandle*) 1, 2, 1, NULL, (void**) &pTaskInfo, &sinkHandle);
|
||||
SReadHandle handle = {.reader = reinterpret_cast<void*>(0x1), .meta = reinterpret_cast<void*>(0x1)};
|
||||
|
||||
// int32_t code = qCreateExecTask(&handle, 2, 1, NULL, (void**) &pTaskInfo, &sinkHandle);
|
||||
}
|
||||
|
||||
//TEST(testCase, inMem_sort_Test) {
|
||||
// SArray* pOrderVal = taosArrayInit(4, sizeof(SOrder));
|
||||
// SOrder o = {.order = TSDB_ORDER_ASC};
|
||||
// o.col.info.colId = 1;
|
||||
// o.col.info.type = TSDB_DATA_TYPE_INT;
|
||||
// taosArrayPush(pOrderVal, &o);
|
||||
//
|
||||
// SArray* pExprInfo = taosArrayInit(4, sizeof(SExprInfo));
|
||||
// SExprInfo *exp = static_cast<SExprInfo*>(calloc(1, sizeof(SExprInfo)));
|
||||
// exp->base.resSchema = createSchema(TSDB_DATA_TYPE_INT, sizeof(int32_t), 1, "res");
|
||||
// taosArrayPush(pExprInfo, &exp);
|
||||
//
|
||||
// SExprInfo *exp1 = static_cast<SExprInfo*>(calloc(1, sizeof(SExprInfo)));
|
||||
// exp1->base.resSchema = createSchema(TSDB_DATA_TYPE_BINARY, 40, 2, "res1");
|
||||
// taosArrayPush(pExprInfo, &exp1);
|
||||
//
|
||||
// SOperatorInfo* pOperator = createOrderOperatorInfo(createDummyOperator(5), pExprInfo, pOrderVal);
|
||||
//
|
||||
// bool newgroup = false;
|
||||
// SSDataBlock* pRes = pOperator->exec(pOperator, &newgroup);
|
||||
//
|
||||
// SColumnInfoData* pCol1 = static_cast<SColumnInfoData*>(taosArrayGet(pRes->pDataBlock, 0));
|
||||
// SColumnInfoData* pCol2 = static_cast<SColumnInfoData*>(taosArrayGet(pRes->pDataBlock, 1));
|
||||
// for(int32_t i = 0; i < pRes->info.rows; ++i) {
|
||||
// char* p = colDataGet(pCol2, i);
|
||||
// printf("%d: %d, %s\n", i, ((int32_t*)pCol1->pData)[i], (char*)varDataVal(p));
|
||||
// }
|
||||
//}
|
||||
|
||||
typedef struct su {
|
||||
int32_t v;
|
||||
char *c;
|
||||
} su;
|
||||
|
||||
int32_t cmp(const void* p1, const void* p2) {
|
||||
su* v1 = (su*) p1;
|
||||
su* v2 = (su*) p2;
|
||||
|
||||
int32_t x1 = *(int32_t*) v1->c;
|
||||
int32_t x2 = *(int32_t*) v2->c;
|
||||
if (x1 == x2) {
|
||||
return 0;
|
||||
} else {
|
||||
return x1 < x2? -1:1;
|
||||
}
|
||||
}
|
||||
|
||||
TEST(testCase, external_sort_Test) {
|
||||
#if 0
|
||||
su* v = static_cast<su*>(calloc(1000000, sizeof(su)));
|
||||
for(int32_t i = 0; i < 1000000; ++i) {
|
||||
v[i].v = rand();
|
||||
v[i].c = static_cast<char*>(malloc(4));
|
||||
*(int32_t*) v[i].c = i;
|
||||
}
|
||||
|
||||
qsort(v, 1000000, sizeof(su), cmp);
|
||||
// for(int32_t i = 0; i < 1000; ++i) {
|
||||
// printf("%d ", v[i]);
|
||||
// }
|
||||
// printf("\n");
|
||||
return;
|
||||
#endif
|
||||
|
||||
srand(time(NULL));
|
||||
|
||||
SArray* pOrderVal = taosArrayInit(4, sizeof(SOrder));
|
||||
SOrder o = {0};
|
||||
o.order = TSDB_ORDER_ASC;
|
||||
o.col.info.colId = 1;
|
||||
o.col.info.type = TSDB_DATA_TYPE_INT;
|
||||
taosArrayPush(pOrderVal, &o);
|
||||
|
||||
SArray* pExprInfo = taosArrayInit(4, sizeof(SExprInfo));
|
||||
SExprInfo *exp = static_cast<SExprInfo*>(calloc(1, sizeof(SExprInfo)));
|
||||
exp->base.resSchema = createSchema(TSDB_DATA_TYPE_INT, sizeof(int32_t), 1, "res");
|
||||
taosArrayPush(pExprInfo, &exp);
|
||||
|
||||
SExprInfo *exp1 = static_cast<SExprInfo*>(calloc(1, sizeof(SExprInfo)));
|
||||
exp1->base.resSchema = createSchema(TSDB_DATA_TYPE_BINARY, 40, 2, "res1");
|
||||
// taosArrayPush(pExprInfo, &exp1);
|
||||
|
||||
SOperatorInfo* pOperator = createOrderOperatorInfo(createDummyOperator(1500), pExprInfo, pOrderVal);
|
||||
|
||||
bool newgroup = false;
|
||||
SSDataBlock* pRes = NULL;
|
||||
|
||||
int32_t total = 1;
|
||||
|
||||
int64_t s1 = taosGetTimestampUs();
|
||||
int32_t t = 1;
|
||||
|
||||
while(1) {
|
||||
int64_t s = taosGetTimestampUs();
|
||||
pRes = pOperator->exec(pOperator, &newgroup);
|
||||
|
||||
int64_t e = taosGetTimestampUs();
|
||||
if (t++ == 1) {
|
||||
printf("---------------elapsed:%ld\n", e - s);
|
||||
}
|
||||
|
||||
if (pRes == NULL) {
|
||||
break;
|
||||
}
|
||||
|
||||
SColumnInfoData* pCol1 = static_cast<SColumnInfoData*>(taosArrayGet(pRes->pDataBlock, 0));
|
||||
// SColumnInfoData* pCol2 = static_cast<SColumnInfoData*>(taosArrayGet(pRes->pDataBlock, 1));
|
||||
for (int32_t i = 0; i < pRes->info.rows; ++i) {
|
||||
// char* p = colDataGet(pCol2, i);
|
||||
printf("%d: %d\n", total++, ((int32_t*)pCol1->pData)[i]);
|
||||
// printf("%d: %d, %s\n", total++, ((int32_t*)pCol1->pData)[i], (char*)varDataVal(p));
|
||||
}
|
||||
}
|
||||
|
||||
printStatisBeforeClose(((SOrderOperatorInfo*) pOperator->info)->pSortInternalBuf);
|
||||
|
||||
int64_t s2 = taosGetTimestampUs();
|
||||
printf("total:%ld\n", s2 - s1);
|
||||
|
||||
pOperator->cleanupFn(pOperator->info, 2);
|
||||
tfree(exp);
|
||||
tfree(exp1);
|
||||
taosArrayDestroy(pExprInfo);
|
||||
taosArrayDestroy(pOrderVal);
|
||||
}
|
||||
#pragma GCC diagnostic pop
|
||||
|
|
|
|||
|
|
@ -49,7 +49,7 @@ typedef struct SHistogramInfo {
|
|||
SHistBin* elems;
|
||||
#else
|
||||
tSkipList* pList;
|
||||
SLoserTreeInfo* pLoserTree;
|
||||
SMultiwayMergeTreeInfo* pLoserTree;
|
||||
int32_t maxIndex;
|
||||
bool ordered;
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@
|
|||
extern "C" {
|
||||
#endif
|
||||
|
||||
#include "tpagedfile.h"
|
||||
#include "tpagedbuf.h"
|
||||
#include "ttszip.h"
|
||||
|
||||
typedef struct MinMaxEntry {
|
||||
|
|
@ -63,7 +63,7 @@ typedef struct tMemBucket {
|
|||
__compar_fn_t comparFn;
|
||||
|
||||
tMemBucketSlot * pSlots;
|
||||
SDiskbasedResultBuf *pBuffer;
|
||||
SDiskbasedBuf *pBuffer;
|
||||
__perc_hash_func_t hashFunc;
|
||||
} tMemBucket;
|
||||
|
||||
|
|
|
|||
|
|
@ -117,14 +117,14 @@ int32_t tHistogramAdd(SHistogramInfo** pHisto, double val) {
|
|||
|
||||
if ((*pHisto)->ordered) {
|
||||
int32_t lastIndex = (*pHisto)->maxIndex;
|
||||
SLoserTreeInfo* pTree = (*pHisto)->pLoserTree;
|
||||
SMultiwayMergeTreeInfo* pTree = (*pHisto)->pLoserTree;
|
||||
|
||||
(*pHisto)->pLoserTree->pNode[lastIndex + pTree->numOfEntries].pData = pResNode;
|
||||
pEntry1->index = (*pHisto)->pLoserTree->pNode[lastIndex + pTree->numOfEntries].index;
|
||||
|
||||
// update the loser tree
|
||||
if ((*pHisto)->ordered) {
|
||||
tLoserTreeAdjust(pTree, pEntry1->index + pTree->numOfEntries);
|
||||
tMergeTreeAdjust(pTree, pEntry1->index + pTree->numOfEntries);
|
||||
}
|
||||
|
||||
tSkipListKey kx =
|
||||
|
|
@ -142,10 +142,10 @@ int32_t tHistogramAdd(SHistogramInfo** pHisto, double val) {
|
|||
SHistBin* pPrevEntry = (SHistBin*)pResNode->pBackward[0]->pData;
|
||||
pPrevEntry->delta = val - pPrevEntry->val;
|
||||
|
||||
SLoserTreeInfo* pTree = (*pHisto)->pLoserTree;
|
||||
SMultiwayMergeTreeInfo* pTree = (*pHisto)->pLoserTree;
|
||||
if ((*pHisto)->ordered) {
|
||||
tLoserTreeAdjust(pTree, pPrevEntry->index + pTree->numOfEntries);
|
||||
tLoserTreeDisplay(pTree);
|
||||
tMergeTreeAdjust(pTree, pPrevEntry->index + pTree->numOfEntries);
|
||||
tMergeTreePrint(pTree);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -155,7 +155,7 @@ int32_t tHistogramAdd(SHistogramInfo** pHisto, double val) {
|
|||
if (!(*pHisto)->ordered) {
|
||||
SSkipListPrint((*pHisto)->pList, 1);
|
||||
|
||||
SLoserTreeInfo* pTree = (*pHisto)->pLoserTree;
|
||||
SMultiwayMergeTreeInfo* pTree = (*pHisto)->pLoserTree;
|
||||
tSkipListNode* pHead = (*pHisto)->pList->pHead.pForward[0];
|
||||
|
||||
tSkipListNode* p1 = pHead;
|
||||
|
|
@ -183,13 +183,13 @@ int32_t tHistogramAdd(SHistogramInfo** pHisto, double val) {
|
|||
pTree->pNode[i].index = -1;
|
||||
}
|
||||
|
||||
tLoserTreeDisplay(pTree);
|
||||
tMergeTreePrint(pTree);
|
||||
|
||||
for (int32_t i = pTree->totalEntries - 1; i >= pTree->numOfEntries; i--) {
|
||||
tLoserTreeAdjust(pTree, i);
|
||||
tMergeTreeAdjust(pTree, i);
|
||||
}
|
||||
|
||||
tLoserTreeDisplay(pTree);
|
||||
tMergeTreePrint(pTree);
|
||||
(*pHisto)->ordered = true;
|
||||
}
|
||||
|
||||
|
|
@ -219,7 +219,7 @@ int32_t tHistogramAdd(SHistogramInfo** pHisto, double val) {
|
|||
pPrevEntry->delta = pEntry->val - pPrevEntry->val;
|
||||
}
|
||||
|
||||
SLoserTreeInfo* pTree = (*pHisto)->pLoserTree;
|
||||
SMultiwayMergeTreeInfo* pTree = (*pHisto)->pLoserTree;
|
||||
if (pNextEntry->index != -1) {
|
||||
(*pHisto)->maxIndex = pNextEntry->index;
|
||||
|
||||
|
|
@ -230,12 +230,12 @@ int32_t tHistogramAdd(SHistogramInfo** pHisto, double val) {
|
|||
printf("disappear index is:%d\n", f);
|
||||
}
|
||||
|
||||
tLoserTreeAdjust(pTree, pEntry->index + pTree->numOfEntries);
|
||||
tMergeTreeAdjust(pTree, pEntry->index + pTree->numOfEntries);
|
||||
// remove the next node in skiplist
|
||||
tSkipListRemoveNode((*pHisto)->pList, pNext);
|
||||
SSkipListPrint((*pHisto)->pList, 1);
|
||||
|
||||
tLoserTreeDisplay((*pHisto)->pLoserTree);
|
||||
tMergeTreePrint((*pHisto)->pLoserTree);
|
||||
} else { // add to heap
|
||||
if (pResNode->pForward[0] != NULL) {
|
||||
pEntry1->delta = ((SHistBin*)pResNode->pForward[0]->pData)->val - val;
|
||||
|
|
|
|||
|
|
@ -15,10 +15,10 @@
|
|||
#include <tglobal.h>
|
||||
#include "os.h"
|
||||
|
||||
#include "tpercentile.h"
|
||||
#include "tpagedfile.h"
|
||||
#include "taosdef.h"
|
||||
#include "tcompare.h"
|
||||
#include "tpagedbuf.h"
|
||||
#include "tpercentile.h"
|
||||
#include "ttypes.h"
|
||||
|
||||
#define DEFAULT_NUM_OF_SLOT 1024
|
||||
|
|
@ -35,9 +35,9 @@ static SFilePage *loadDataFromFilePage(tMemBucket *pMemBucket, int32_t slotIdx)
|
|||
|
||||
int32_t offset = 0;
|
||||
for(int32_t i = 0; i < list->size; ++i) {
|
||||
SPageInfo* pgInfo = *(SPageInfo**) taosArrayGet(list, i);
|
||||
struct SPageInfo* pgInfo = *(struct SPageInfo**) taosArrayGet(list, i);
|
||||
|
||||
SFilePage* pg = getResBufPage(pMemBucket->pBuffer, pgInfo->pageId);
|
||||
SFilePage* pg = getBufPage(pMemBucket->pBuffer, getPageId(pgInfo));
|
||||
memcpy(buffer->data + offset, pg->data, (size_t)(pg->num * pMemBucket->bytes));
|
||||
|
||||
offset += (int32_t)(pg->num * pMemBucket->bytes);
|
||||
|
|
@ -98,8 +98,8 @@ double findOnlyResult(tMemBucket *pMemBucket) {
|
|||
SIDList list = getDataBufPagesIdList(pMemBucket->pBuffer, groupId);
|
||||
assert(list->size == 1);
|
||||
|
||||
SPageInfo* pgInfo = (SPageInfo*) taosArrayGetP(list, 0);
|
||||
SFilePage* pPage = getResBufPage(pMemBucket->pBuffer, pgInfo->pageId);
|
||||
struct SPageInfo* pgInfo = (struct SPageInfo*) taosArrayGetP(list, 0);
|
||||
SFilePage* pPage = getBufPage(pMemBucket->pBuffer, getPageId(pgInfo));
|
||||
assert(pPage->num == 1);
|
||||
|
||||
double v = 0;
|
||||
|
|
@ -254,7 +254,7 @@ tMemBucket *tMemBucketCreate(int16_t nElemSize, int16_t dataType, double minval,
|
|||
|
||||
resetSlotInfo(pBucket);
|
||||
|
||||
int32_t ret = createDiskbasedResultBuffer(&pBucket->pBuffer, pBucket->bufPageSize, pBucket->bufPageSize * 512, 1, tsTempDir);
|
||||
int32_t ret = createDiskbasedBuffer(&pBucket->pBuffer, pBucket->bufPageSize, pBucket->bufPageSize * 512, 1, tsTempDir);
|
||||
if (ret != 0) {
|
||||
tMemBucketDestroy(pBucket);
|
||||
return NULL;
|
||||
|
|
@ -343,7 +343,7 @@ int32_t tMemBucketPut(tMemBucket *pBucket, const void *data, size_t size) {
|
|||
assert(pSlot->info.data->num >= pBucket->elemPerPage && pSlot->info.size > 0);
|
||||
|
||||
// keep the pointer in memory
|
||||
releaseResBufPage(pBucket->pBuffer, pSlot->info.data);
|
||||
releaseBufPage(pBucket->pBuffer, pSlot->info.data);
|
||||
pSlot->info.data = NULL;
|
||||
}
|
||||
|
||||
|
|
@ -471,10 +471,10 @@ double getPercentileImpl(tMemBucket *pMemBucket, int32_t count, double fraction)
|
|||
|
||||
for (int32_t f = 0; f < list->size; ++f) {
|
||||
SPageInfo *pgInfo = *(SPageInfo **)taosArrayGet(list, f);
|
||||
SFilePage *pg = getResBufPage(pMemBucket->pBuffer, pgInfo->pageId);
|
||||
SFilePage *pg = getBufPage(pMemBucket->pBuffer, getPageId(pgInfo));
|
||||
|
||||
tMemBucketPut(pMemBucket, pg->data, (int32_t)pg->num);
|
||||
releaseResBufPageInfo(pMemBucket->pBuffer, pgInfo);
|
||||
releaseBufPageInfo(pMemBucket->pBuffer, pgInfo);
|
||||
}
|
||||
|
||||
return getPercentileImpl(pMemBucket, count - num, fraction);
|
||||
|
|
|
|||
|
|
@ -216,7 +216,7 @@ static SPhyNode* createMultiTableScanNode(SQueryPlanNode* pPlanNode, SQueryTable
|
|||
} else if (needSeqScan(pPlanNode)) {
|
||||
return createUserTableScanNode(pPlanNode, pTable, OP_TableSeqScan);
|
||||
}
|
||||
int32_t type = (pPlanNode->info.type == QNODE_TABLESCAN)? OP_DataBlocksOptScan:OP_StreamScan;
|
||||
int32_t type = (pPlanNode->info.type == QNODE_TABLESCAN)? OP_TableScan:OP_StreamScan;
|
||||
return createUserTableScanNode(pPlanNode, pTable, type);
|
||||
}
|
||||
|
||||
|
|
@ -288,7 +288,7 @@ static bool needMultiNodeScan(SQueryTableInfo* pTable) {
|
|||
static SPhyNode* createSingleTableScanNode(SQueryPlanNode* pPlanNode, SQueryTableInfo* pTableInfo, SSubplan* subplan) {
|
||||
SVgroupsInfo* pVgroupsInfo = pTableInfo->pMeta->vgroupList;
|
||||
vgroupInfoToNodeAddr(&(pVgroupsInfo->vgroups[0]), &subplan->execNode);
|
||||
int32_t type = (pPlanNode->info.type == QNODE_TABLESCAN)? OP_DataBlocksOptScan:OP_StreamScan;
|
||||
int32_t type = (pPlanNode->info.type == QNODE_TABLESCAN)? OP_TableScan:OP_StreamScan;
|
||||
return createUserTableScanNode(pPlanNode, pTableInfo, type);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -88,7 +88,7 @@ static const char* jkPnodeType = "Type";
|
|||
static int32_t getPnodeTypeSize(cJSON* json) {
|
||||
switch (getNumber(json, jkPnodeType)) {
|
||||
case OP_StreamScan:
|
||||
case OP_DataBlocksOptScan:
|
||||
case OP_TableScan:
|
||||
case OP_TableSeqScan:
|
||||
return sizeof(STableScanPhyNode);
|
||||
case OP_TagScan:
|
||||
|
|
@ -830,7 +830,7 @@ static bool specificPhyNodeToJson(const void* obj, cJSON* json) {
|
|||
const SPhyNode* phyNode = (const SPhyNode*)obj;
|
||||
switch (phyNode->info.type) {
|
||||
case OP_StreamScan:
|
||||
case OP_DataBlocksOptScan:
|
||||
case OP_TableScan:
|
||||
case OP_TableSeqScan:
|
||||
return tableScanNodeToJson(obj, json);
|
||||
case OP_TagScan:
|
||||
|
|
@ -868,7 +868,7 @@ static bool specificPhyNodeFromJson(const cJSON* json, void* obj) {
|
|||
SPhyNode* phyNode = (SPhyNode*)obj;
|
||||
switch (phyNode->info.type) {
|
||||
case OP_StreamScan:
|
||||
case OP_DataBlocksOptScan:
|
||||
case OP_TableScan:
|
||||
case OP_TableSeqScan:
|
||||
return tableScanNodeFromJson(json, obj);
|
||||
case OP_TagScan:
|
||||
|
|
|
|||
|
|
@ -14,82 +14,85 @@
|
|||
*/
|
||||
|
||||
#include "os.h"
|
||||
#include "tlosertree.h"
|
||||
#include "ulog.h"
|
||||
#include "tlosertree.h"
|
||||
#include "taoserror.h"
|
||||
|
||||
// set initial value for loser tree
|
||||
void tLoserTreeInit(SLoserTreeInfo* pTree) {
|
||||
assert((pTree->totalEntries & 0x01) == 0 && (pTree->numOfEntries << 1 == pTree->totalEntries));
|
||||
|
||||
for (int32_t i = 0; i < pTree->totalEntries; ++i) {
|
||||
if (i < pTree->numOfEntries) {
|
||||
|
||||
// Set the initial value of the multiway merge tree.
|
||||
static void tMergeTreeInit(SMultiwayMergeTreeInfo* pTree) {
|
||||
assert((pTree->totalSources & 0x01) == 0 && (pTree->numOfSources << 1 == pTree->totalSources));
|
||||
|
||||
for (int32_t i = 0; i < pTree->totalSources; ++i) {
|
||||
if (i < pTree->numOfSources) {
|
||||
pTree->pNode[i].index = -1;
|
||||
} else {
|
||||
pTree->pNode[i].index = i - pTree->numOfEntries;
|
||||
pTree->pNode[i].index = i - pTree->numOfSources;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* display whole loser tree on screen for debug purpose only.
|
||||
*/
|
||||
void tLoserTreeDisplay(SLoserTreeInfo* pTree) {
|
||||
printf("the value of loser tree:\t");
|
||||
for (int32_t i = 0; i < pTree->totalEntries; ++i) printf("%d\t", pTree->pNode[i].index);
|
||||
printf("\n");
|
||||
}
|
||||
int32_t tMergeTreeCreate(SMultiwayMergeTreeInfo** pTree, uint32_t numOfSources, void* param, __merge_compare_fn_t compareFn) {
|
||||
int32_t totalEntries = numOfSources << 1u;
|
||||
|
||||
uint32_t tLoserTreeCreate(SLoserTreeInfo** pTree, int32_t numOfEntries, void* param, __merge_compare_fn_t compareFn) {
|
||||
int32_t totalEntries = numOfEntries << 1;
|
||||
|
||||
*pTree = (SLoserTreeInfo*)calloc(1, sizeof(SLoserTreeInfo) + sizeof(SLoserTreeNode) * totalEntries);
|
||||
if ((*pTree) == NULL) {
|
||||
SMultiwayMergeTreeInfo* pTreeInfo = (SMultiwayMergeTreeInfo*)calloc(1, sizeof(SMultiwayMergeTreeInfo) + sizeof(STreeNode) * totalEntries);
|
||||
if (pTreeInfo == NULL) {
|
||||
uError("allocate memory for loser-tree failed. reason:%s", strerror(errno));
|
||||
return -1;
|
||||
return TAOS_SYSTEM_ERROR(errno);
|
||||
}
|
||||
|
||||
(*pTree)->pNode = (SLoserTreeNode*)(((char*)(*pTree)) + sizeof(SLoserTreeInfo));
|
||||
pTreeInfo->pNode = (STreeNode*)(((char*)pTreeInfo) + sizeof(SMultiwayMergeTreeInfo));
|
||||
|
||||
(*pTree)->numOfEntries = numOfEntries;
|
||||
(*pTree)->totalEntries = totalEntries;
|
||||
(*pTree)->param = param;
|
||||
(*pTree)->comparFn = compareFn;
|
||||
pTreeInfo->numOfSources = numOfSources;
|
||||
pTreeInfo->totalSources = totalEntries;
|
||||
pTreeInfo->param = param;
|
||||
pTreeInfo->comparFn = compareFn;
|
||||
|
||||
// set initial value for loser tree
|
||||
tLoserTreeInit(*pTree);
|
||||
tMergeTreeInit(pTreeInfo);
|
||||
|
||||
#ifdef _DEBUG_VIEW
|
||||
printf("the initial value of loser tree:\n");
|
||||
tLoserTreeDisplay(*pTree);
|
||||
tLoserTreeDisplaypTreeInfo;
|
||||
#endif
|
||||
|
||||
for (int32_t i = totalEntries - 1; i >= numOfEntries; i--) {
|
||||
tLoserTreeAdjust(*pTree, i);
|
||||
for (int32_t i = totalEntries - 1; i >= numOfSources; i--) {
|
||||
tMergeTreeAdjust(pTreeInfo, i);
|
||||
}
|
||||
|
||||
#if defined(_DEBUG_VIEW)
|
||||
printf("after adjust:\n");
|
||||
tLoserTreeDisplay(*pTree);
|
||||
tLoserTreeDisplaypTreeInfo;
|
||||
printf("initialize local reducer completed!\n");
|
||||
#endif
|
||||
|
||||
*pTree = pTreeInfo;
|
||||
return 0;
|
||||
}
|
||||
|
||||
void tLoserTreeAdjust(SLoserTreeInfo* pTree, int32_t idx) {
|
||||
assert(idx <= pTree->totalEntries - 1 && idx >= pTree->numOfEntries && pTree->totalEntries >= 2);
|
||||
void tMergeTreeDestroy(SMultiwayMergeTreeInfo* pTree) {
|
||||
if (pTree == NULL) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (pTree->totalEntries == 2) {
|
||||
tfree(pTree);
|
||||
}
|
||||
|
||||
void tMergeTreeAdjust(SMultiwayMergeTreeInfo* pTree, int32_t idx) {
|
||||
assert(idx <= pTree->totalSources - 1 && idx >= pTree->numOfSources && pTree->totalSources >= 2);
|
||||
|
||||
if (pTree->totalSources == 2) {
|
||||
pTree->pNode[0].index = 0;
|
||||
pTree->pNode[1].index = 0;
|
||||
return;
|
||||
}
|
||||
|
||||
int32_t parentId = idx >> 1;
|
||||
SLoserTreeNode kLeaf = pTree->pNode[idx];
|
||||
STreeNode kLeaf = pTree->pNode[idx];
|
||||
|
||||
while (parentId > 0) {
|
||||
SLoserTreeNode* pCur = &pTree->pNode[parentId];
|
||||
STreeNode* pCur = &pTree->pNode[parentId];
|
||||
if (pCur->index == -1) {
|
||||
pTree->pNode[parentId] = kLeaf;
|
||||
return;
|
||||
|
|
@ -97,7 +100,7 @@ void tLoserTreeAdjust(SLoserTreeInfo* pTree, int32_t idx) {
|
|||
|
||||
int32_t ret = pTree->comparFn(pCur, &kLeaf, pTree->param);
|
||||
if (ret < 0) {
|
||||
SLoserTreeNode t = pTree->pNode[parentId];
|
||||
STreeNode t = pTree->pNode[parentId];
|
||||
pTree->pNode[parentId] = kLeaf;
|
||||
kLeaf = t;
|
||||
}
|
||||
|
|
@ -111,11 +114,23 @@ void tLoserTreeAdjust(SLoserTreeInfo* pTree, int32_t idx) {
|
|||
}
|
||||
}
|
||||
|
||||
void tLoserTreeRebuild(SLoserTreeInfo* pTree) {
|
||||
assert((pTree->totalEntries & 0x1) == 0);
|
||||
void tMergeTreeRebuild(SMultiwayMergeTreeInfo* pTree) {
|
||||
assert((pTree->totalSources & 0x1) == 0);
|
||||
|
||||
tLoserTreeInit(pTree);
|
||||
for (int32_t i = pTree->totalEntries - 1; i >= pTree->numOfEntries; i--) {
|
||||
tLoserTreeAdjust(pTree, i);
|
||||
tMergeTreeInit(pTree);
|
||||
for (int32_t i = pTree->totalSources - 1; i >= pTree->numOfSources; i--) {
|
||||
tMergeTreeAdjust(pTree, i);
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* display whole loser tree on screen for debug purpose only.
|
||||
*/
|
||||
void tMergeTreePrint(const SMultiwayMergeTreeInfo* pTree) {
|
||||
printf("the value of loser tree:\t");
|
||||
for (int32_t i = 0; i < pTree->totalSources; ++i) {
|
||||
printf("%d\t", pTree->pNode[i].index);
|
||||
}
|
||||
|
||||
printf("\n");
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,597 @@
|
|||
#include "os.h"
|
||||
#include "ulog.h"
|
||||
#include "tpagedbuf.h"
|
||||
#include "taoserror.h"
|
||||
#include "tcompression.h"
|
||||
#include "thash.h"
|
||||
|
||||
#define GET_DATA_PAYLOAD(_p) ((char *)(_p)->pData + POINTER_BYTES)
|
||||
#define NO_IN_MEM_AVAILABLE_PAGES(_b) (listNEles((_b)->lruList) >= (_b)->inMemPages)
|
||||
|
||||
typedef struct SFreeListItem {
|
||||
int32_t offset;
|
||||
int32_t len;
|
||||
} SFreeListItem;
|
||||
|
||||
typedef struct SPageDiskInfo {
|
||||
int64_t offset;
|
||||
int32_t length;
|
||||
} SPageDiskInfo;
|
||||
|
||||
typedef struct SPageInfo {
|
||||
SListNode* pn; // point to list node
|
||||
void* pData;
|
||||
int64_t offset;
|
||||
int32_t pageId;
|
||||
int32_t length:30;
|
||||
bool used:1; // set current page is in used
|
||||
bool dirty:1; // set current buffer page is dirty or not
|
||||
} SPageInfo;
|
||||
|
||||
typedef struct SDiskbasedBuf {
|
||||
int32_t numOfPages;
|
||||
int64_t totalBufSize;
|
||||
uint64_t fileSize; // disk file size
|
||||
FILE* file;
|
||||
int32_t allocateId; // allocated page id
|
||||
char* path; // file path
|
||||
int32_t pageSize; // current used page size
|
||||
int32_t inMemPages; // numOfPages that are allocated in memory
|
||||
SHashObj* groupSet; // id hash table
|
||||
SHashObj* all;
|
||||
SList* lruList;
|
||||
void* emptyDummyIdList; // dummy id list
|
||||
void* assistBuf; // assistant buffer for compress/decompress data
|
||||
SArray* pFree; // free area in file
|
||||
bool comp; // compressed before flushed to disk
|
||||
uint64_t nextPos; // next page flush position
|
||||
|
||||
uint64_t qId; // for debug purpose
|
||||
bool printStatis; // Print statistics info when closing this buffer.
|
||||
SDiskbasedBufStatis statis;
|
||||
} SDiskbasedBuf;
|
||||
|
||||
static void printStatisData(const SDiskbasedBuf* pBuf);
|
||||
|
||||
int32_t createDiskbasedBuffer(SDiskbasedBuf** pBuf, int32_t pagesize, int32_t inMemBufSize, uint64_t qId, const char* dir) {
|
||||
*pBuf = calloc(1, sizeof(SDiskbasedBuf));
|
||||
|
||||
SDiskbasedBuf* pResBuf = *pBuf;
|
||||
if (pResBuf == NULL) {
|
||||
return TSDB_CODE_OUT_OF_MEMORY;
|
||||
}
|
||||
|
||||
pResBuf->pageSize = pagesize;
|
||||
pResBuf->numOfPages = 0; // all pages are in buffer in the first place
|
||||
pResBuf->totalBufSize = 0;
|
||||
pResBuf->inMemPages = inMemBufSize/pagesize; // maximum allowed pages, it is a soft limit.
|
||||
pResBuf->allocateId = -1;
|
||||
pResBuf->comp = true;
|
||||
pResBuf->file = NULL;
|
||||
pResBuf->qId = qId;
|
||||
pResBuf->fileSize = 0;
|
||||
|
||||
// at least more than 2 pages must be in memory
|
||||
assert(inMemBufSize >= pagesize * 2);
|
||||
|
||||
pResBuf->lruList = tdListNew(POINTER_BYTES);
|
||||
|
||||
// init id hash table
|
||||
pResBuf->groupSet = taosHashInit(10, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, false);
|
||||
pResBuf->assistBuf = malloc(pResBuf->pageSize + 2); // EXTRA BYTES
|
||||
pResBuf->all = taosHashInit(10, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, false);
|
||||
|
||||
char path[PATH_MAX] = {0};
|
||||
taosGetTmpfilePath(dir, "qbuf", path);
|
||||
pResBuf->path = strdup(path);
|
||||
|
||||
pResBuf->emptyDummyIdList = taosArrayInit(1, sizeof(int32_t));
|
||||
|
||||
// qDebug("QInfo:0x%"PRIx64" create resBuf for output, page size:%d, inmem buf pages:%d, file:%s", qId, pResBuf->pageSize,
|
||||
// pResBuf->inMemPages, pResBuf->path);
|
||||
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
static int32_t createDiskFile(SDiskbasedBuf* pBuf) {
|
||||
pBuf->file = fopen(pBuf->path, "wb+");
|
||||
if (pBuf->file == NULL) {
|
||||
// qError("failed to create tmp file: %s on disk. %s", pBuf->path, strerror(errno));
|
||||
return TAOS_SYSTEM_ERROR(errno);
|
||||
}
|
||||
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
static char* doCompressData(void* data, int32_t srcSize, int32_t *dst, SDiskbasedBuf* pBuf) { // do nothing
|
||||
if (!pBuf->comp) {
|
||||
*dst = srcSize;
|
||||
return data;
|
||||
}
|
||||
|
||||
*dst = tsCompressString(data, srcSize, 1, pBuf->assistBuf, srcSize, ONE_STAGE_COMP, NULL, 0);
|
||||
|
||||
memcpy(data, pBuf->assistBuf, *dst);
|
||||
return data;
|
||||
}
|
||||
|
||||
static char* doDecompressData(void* data, int32_t srcSize, int32_t *dst, SDiskbasedBuf* pBuf) { // do nothing
|
||||
if (!pBuf->comp) {
|
||||
*dst = srcSize;
|
||||
return data;
|
||||
}
|
||||
|
||||
*dst = tsDecompressString(data, srcSize, 1, pBuf->assistBuf, pBuf->pageSize+sizeof(SFilePage), ONE_STAGE_COMP, NULL, 0);
|
||||
if (*dst > 0) {
|
||||
memcpy(data, pBuf->assistBuf, *dst);
|
||||
}
|
||||
return data;
|
||||
}
|
||||
|
||||
static uint64_t allocatePositionInFile(SDiskbasedBuf* pBuf, size_t size) {
|
||||
if (pBuf->pFree == NULL) {
|
||||
return pBuf->nextPos;
|
||||
} else {
|
||||
int32_t offset = -1;
|
||||
|
||||
size_t num = taosArrayGetSize(pBuf->pFree);
|
||||
for(int32_t i = 0; i < num; ++i) {
|
||||
SFreeListItem* pi = taosArrayGet(pBuf->pFree, i);
|
||||
if (pi->len >= size) {
|
||||
offset = pi->offset;
|
||||
pi->offset += (int32_t)size;
|
||||
pi->len -= (int32_t)size;
|
||||
|
||||
return offset;
|
||||
}
|
||||
}
|
||||
|
||||
// no available recycle space, allocate new area in file
|
||||
return pBuf->nextPos;
|
||||
}
|
||||
}
|
||||
|
||||
static char* doFlushPageToDisk(SDiskbasedBuf* pBuf, SPageInfo* pg) {
|
||||
assert(!pg->used && pg->pData != NULL);
|
||||
|
||||
int32_t size = -1;
|
||||
char* t = NULL;
|
||||
if (pg->offset == -1 || pg->dirty) {
|
||||
SFilePage* pPage = (SFilePage*) GET_DATA_PAYLOAD(pg);
|
||||
t = doCompressData(pPage->data, pBuf->pageSize, &size, pBuf);
|
||||
}
|
||||
|
||||
// this page is flushed to disk for the first time
|
||||
if (pg->offset == -1) {
|
||||
assert(pg->dirty == true);
|
||||
|
||||
pg->offset = allocatePositionInFile(pBuf, size);
|
||||
pBuf->nextPos += size;
|
||||
|
||||
int32_t ret = fseek(pBuf->file, pg->offset, SEEK_SET);
|
||||
if (ret != 0) {
|
||||
terrno = TAOS_SYSTEM_ERROR(errno);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
ret = (int32_t) fwrite(t, 1, size, pBuf->file);
|
||||
if (ret != size) {
|
||||
terrno = TAOS_SYSTEM_ERROR(errno);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
if (pBuf->fileSize < pg->offset + size) {
|
||||
pBuf->fileSize = pg->offset + size;
|
||||
}
|
||||
|
||||
pBuf->statis.flushBytes += size;
|
||||
pBuf->statis.flushPages += 1;
|
||||
} else if (pg->dirty) {
|
||||
// length becomes greater, current space is not enough, allocate new place, otherwise, do nothing
|
||||
if (pg->length < size) {
|
||||
// 1. add current space to free list
|
||||
SPageDiskInfo dinfo = {.length = pg->length, .offset = pg->offset};
|
||||
taosArrayPush(pBuf->pFree, &dinfo);
|
||||
|
||||
// 2. allocate new position, and update the info
|
||||
pg->offset = allocatePositionInFile(pBuf, size);
|
||||
pBuf->nextPos += size;
|
||||
}
|
||||
|
||||
// 3. write to disk.
|
||||
int32_t ret = fseek(pBuf->file, pg->offset, SEEK_SET);
|
||||
if (ret != 0) {
|
||||
terrno = TAOS_SYSTEM_ERROR(errno);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
ret = (int32_t)fwrite(t, 1, size, pBuf->file);
|
||||
if (ret != size) {
|
||||
terrno = TAOS_SYSTEM_ERROR(errno);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
if (pBuf->fileSize < pg->offset + size) {
|
||||
pBuf->fileSize = pg->offset + size;
|
||||
}
|
||||
|
||||
pBuf->statis.flushBytes += size;
|
||||
pBuf->statis.flushPages += 1;
|
||||
}
|
||||
|
||||
char* pDataBuf = pg->pData;
|
||||
memset(pDataBuf, 0, pBuf->pageSize + sizeof(SFilePage));
|
||||
|
||||
pg->pData = NULL; // this means the data is not in buffer
|
||||
pg->length = size;
|
||||
pg->dirty = false;
|
||||
|
||||
return pDataBuf;
|
||||
}
|
||||
|
||||
static char* flushPageToDisk(SDiskbasedBuf* pBuf, SPageInfo* pg) {
|
||||
int32_t ret = TSDB_CODE_SUCCESS;
|
||||
assert(((int64_t) pBuf->numOfPages * pBuf->pageSize) == pBuf->totalBufSize && pBuf->numOfPages >= pBuf->inMemPages);
|
||||
|
||||
if (pBuf->file == NULL) {
|
||||
if ((ret = createDiskFile(pBuf)) != TSDB_CODE_SUCCESS) {
|
||||
terrno = ret;
|
||||
return NULL;
|
||||
}
|
||||
}
|
||||
|
||||
return doFlushPageToDisk(pBuf, pg);
|
||||
}
|
||||
|
||||
// load file block data in disk
|
||||
static int32_t loadPageFromDisk(SDiskbasedBuf* pBuf, SPageInfo* pg) {
|
||||
int32_t ret = fseek(pBuf->file, pg->offset, SEEK_SET);
|
||||
if (ret != 0) {
|
||||
ret = TAOS_SYSTEM_ERROR(errno);
|
||||
return ret;
|
||||
}
|
||||
|
||||
SFilePage* pPage = (SFilePage*) GET_DATA_PAYLOAD(pg);
|
||||
ret = (int32_t)fread(pPage->data, 1, pg->length, pBuf->file);
|
||||
if (ret != pg->length) {
|
||||
ret = TAOS_SYSTEM_ERROR(errno);
|
||||
return ret;
|
||||
}
|
||||
|
||||
pBuf->statis.loadBytes += pg->length;
|
||||
pBuf->statis.loadPages += 1;
|
||||
|
||||
int32_t fullSize = 0;
|
||||
doDecompressData(pPage->data, pg->length, &fullSize, pBuf);
|
||||
return 0;
|
||||
}
|
||||
|
||||
static SIDList addNewGroup(SDiskbasedBuf* pBuf, int32_t groupId) {
|
||||
assert(taosHashGet(pBuf->groupSet, (const char*) &groupId, sizeof(int32_t)) == NULL);
|
||||
|
||||
SArray* pa = taosArrayInit(1, POINTER_BYTES);
|
||||
int32_t ret = taosHashPut(pBuf->groupSet, (const char*)&groupId, sizeof(int32_t), &pa, POINTER_BYTES);
|
||||
assert(ret == 0);
|
||||
|
||||
return pa;
|
||||
}
|
||||
|
||||
static SPageInfo* registerPage(SDiskbasedBuf* pBuf, int32_t groupId, int32_t pageId) {
|
||||
SIDList list = NULL;
|
||||
|
||||
char** p = taosHashGet(pBuf->groupSet, (const char*)&groupId, sizeof(int32_t));
|
||||
if (p == NULL) { // it is a new group id
|
||||
list = addNewGroup(pBuf, groupId);
|
||||
} else {
|
||||
list = (SIDList) (*p);
|
||||
}
|
||||
|
||||
pBuf->numOfPages += 1;
|
||||
|
||||
SPageInfo* ppi = malloc(sizeof(SPageInfo));//{ .info = PAGE_INFO_INITIALIZER, .pageId = pageId, .pn = NULL};
|
||||
|
||||
ppi->pageId = pageId;
|
||||
ppi->pData = NULL;
|
||||
ppi->offset = -1;
|
||||
ppi->length = -1;
|
||||
ppi->used = true;
|
||||
ppi->pn = NULL;
|
||||
|
||||
return *(SPageInfo**) taosArrayPush(list, &ppi);
|
||||
}
|
||||
|
||||
static SListNode* getEldestUnrefedPage(SDiskbasedBuf* pBuf) {
|
||||
SListIter iter = {0};
|
||||
tdListInitIter(pBuf->lruList, &iter, TD_LIST_BACKWARD);
|
||||
|
||||
SListNode* pn = NULL;
|
||||
while((pn = tdListNext(&iter)) != NULL) {
|
||||
assert(pn != NULL);
|
||||
|
||||
SPageInfo* pageInfo = *(SPageInfo**) pn->data;
|
||||
assert(pageInfo->pageId >= 0 && pageInfo->pn == pn);
|
||||
|
||||
if (!pageInfo->used) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return pn;
|
||||
}
|
||||
|
||||
static char* evacOneDataPage(SDiskbasedBuf* pBuf) {
|
||||
char* bufPage = NULL;
|
||||
SListNode* pn = getEldestUnrefedPage(pBuf);
|
||||
|
||||
// all pages are referenced by user, try to allocate new space
|
||||
if (pn == NULL) {
|
||||
assert(0);
|
||||
int32_t prev = pBuf->inMemPages;
|
||||
|
||||
// increase by 50% of previous mem pages
|
||||
pBuf->inMemPages = (int32_t)(pBuf->inMemPages * 1.5f);
|
||||
|
||||
// qWarn("%p in memory buf page not sufficient, expand from %d to %d, page size:%d", pBuf, prev,
|
||||
// pBuf->inMemPages, pBuf->pageSize);
|
||||
} else {
|
||||
tdListPopNode(pBuf->lruList, pn);
|
||||
|
||||
SPageInfo* d = *(SPageInfo**) pn->data;
|
||||
assert(d->pn == pn);
|
||||
|
||||
d->pn = NULL;
|
||||
tfree(pn);
|
||||
|
||||
bufPage = flushPageToDisk(pBuf, d);
|
||||
}
|
||||
|
||||
return bufPage;
|
||||
}
|
||||
|
||||
static void lruListPushFront(SList *pList, SPageInfo* pi) {
|
||||
tdListPrepend(pList, &pi);
|
||||
SListNode* front = tdListGetHead(pList);
|
||||
pi->pn = front;
|
||||
}
|
||||
|
||||
static void lruListMoveToFront(SList *pList, SPageInfo* pi) {
|
||||
tdListPopNode(pList, pi->pn);
|
||||
tdListPrependNode(pList, pi->pn);
|
||||
}
|
||||
|
||||
static FORCE_INLINE size_t getAllocPageSize(int32_t pageSize) {
|
||||
return pageSize + POINTER_BYTES + 2 + sizeof(SFilePage);
|
||||
}
|
||||
|
||||
SFilePage* getNewDataBuf(SDiskbasedBuf* pBuf, int32_t groupId, int32_t* pageId) {
|
||||
pBuf->statis.getPages += 1;
|
||||
|
||||
char* availablePage = NULL;
|
||||
if (NO_IN_MEM_AVAILABLE_PAGES(pBuf)) {
|
||||
availablePage = evacOneDataPage(pBuf);
|
||||
|
||||
// Failed to allocate a new buffer page, and there is an error occurs.
|
||||
if (availablePage == NULL) {
|
||||
return NULL;
|
||||
}
|
||||
}
|
||||
|
||||
// register new id in this group
|
||||
*pageId = (++pBuf->allocateId);
|
||||
|
||||
// register page id info
|
||||
SPageInfo* pi = registerPage(pBuf, groupId, *pageId);
|
||||
|
||||
// add to LRU list
|
||||
assert(listNEles(pBuf->lruList) < pBuf->inMemPages && pBuf->inMemPages > 0);
|
||||
lruListPushFront(pBuf->lruList, pi);
|
||||
|
||||
// add to hash map
|
||||
taosHashPut(pBuf->all, pageId, sizeof(int32_t), &pi, POINTER_BYTES);
|
||||
|
||||
// allocate buf
|
||||
if (availablePage == NULL) {
|
||||
pi->pData = calloc(1, getAllocPageSize(pBuf->pageSize)); // add extract bytes in case of zipped buffer increased.
|
||||
} else {
|
||||
pi->pData = availablePage;
|
||||
}
|
||||
|
||||
pBuf->totalBufSize += pBuf->pageSize;
|
||||
|
||||
((void**)pi->pData)[0] = pi;
|
||||
pi->used = true;
|
||||
|
||||
return (void *)(GET_DATA_PAYLOAD(pi));
|
||||
}
|
||||
|
||||
SFilePage* getBufPage(SDiskbasedBuf* pBuf, int32_t id) {
|
||||
assert(pBuf != NULL && id >= 0);
|
||||
pBuf->statis.getPages += 1;
|
||||
|
||||
SPageInfo** pi = taosHashGet(pBuf->all, &id, sizeof(int32_t));
|
||||
assert(pi != NULL && *pi != NULL);
|
||||
|
||||
if ((*pi)->pData != NULL) { // it is in memory
|
||||
// no need to update the LRU list if only one page exists
|
||||
if (pBuf->numOfPages == 1) {
|
||||
(*pi)->used = true;
|
||||
return (void *)(GET_DATA_PAYLOAD(*pi));
|
||||
}
|
||||
|
||||
SPageInfo** pInfo = (SPageInfo**) ((*pi)->pn->data);
|
||||
assert(*pInfo == *pi);
|
||||
|
||||
lruListMoveToFront(pBuf->lruList, (*pi));
|
||||
(*pi)->used = true;
|
||||
|
||||
return (void *)(GET_DATA_PAYLOAD(*pi));
|
||||
|
||||
} else { // not in memory
|
||||
assert((*pi)->pData == NULL && (*pi)->pn == NULL && (*pi)->length >= 0 && (*pi)->offset >= 0);
|
||||
|
||||
char* availablePage = NULL;
|
||||
if (NO_IN_MEM_AVAILABLE_PAGES(pBuf)) {
|
||||
availablePage = evacOneDataPage(pBuf);
|
||||
if (availablePage == NULL) {
|
||||
return NULL;
|
||||
}
|
||||
}
|
||||
|
||||
if (availablePage == NULL) {
|
||||
(*pi)->pData = calloc(1, getAllocPageSize(pBuf->pageSize));
|
||||
} else {
|
||||
(*pi)->pData = availablePage;
|
||||
}
|
||||
|
||||
((void**)((*pi)->pData))[0] = (*pi);
|
||||
|
||||
lruListPushFront(pBuf->lruList, *pi);
|
||||
(*pi)->used = true;
|
||||
|
||||
int32_t code = loadPageFromDisk(pBuf, *pi);
|
||||
if (code != 0) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
return (void *)(GET_DATA_PAYLOAD(*pi));
|
||||
}
|
||||
}
|
||||
|
||||
void releaseBufPage(SDiskbasedBuf* pBuf, void* page) {
|
||||
assert(pBuf != NULL && page != NULL);
|
||||
int32_t offset = offsetof(SPageInfo, pData);
|
||||
char* p = page - offset;
|
||||
|
||||
SPageInfo* ppi = ((SPageInfo**) p)[0];
|
||||
releaseBufPageInfo(pBuf, ppi);
|
||||
}
|
||||
|
||||
void releaseBufPageInfo(SDiskbasedBuf* pBuf, SPageInfo* pi) {
|
||||
assert(pi->pData != NULL && pi->used);
|
||||
|
||||
pi->used = false;
|
||||
pBuf->statis.releasePages += 1;
|
||||
}
|
||||
|
||||
size_t getNumOfResultBufGroupId(const SDiskbasedBuf* pBuf) { return taosHashGetSize(pBuf->groupSet); }
|
||||
|
||||
size_t getTotalBufSize(const SDiskbasedBuf* pBuf) { return (size_t)pBuf->totalBufSize; }
|
||||
|
||||
SIDList getDataBufPagesIdList(SDiskbasedBuf* pBuf, int32_t groupId) {
|
||||
assert(pBuf != NULL);
|
||||
|
||||
char** p = taosHashGet(pBuf->groupSet, (const char*)&groupId, sizeof(int32_t));
|
||||
if (p == NULL) { // it is a new group id
|
||||
return pBuf->emptyDummyIdList;
|
||||
} else {
|
||||
return (SArray*) (*p);
|
||||
}
|
||||
}
|
||||
|
||||
void destroyResultBuf(SDiskbasedBuf* pBuf) {
|
||||
if (pBuf == NULL) {
|
||||
return;
|
||||
}
|
||||
|
||||
printStatisData(pBuf);
|
||||
|
||||
if (pBuf->file != NULL) {
|
||||
uDebug("Paged buffer closed, total:%.2f Kb (%d Pages), inmem size:%.2f Kb (%d Pages), file size:%.2f Kb, page size:%.2f Kb, %"PRIx64"\n",
|
||||
pBuf->totalBufSize/1024.0, pBuf->numOfPages, listNEles(pBuf->lruList) * pBuf->pageSize / 1024.0,
|
||||
listNEles(pBuf->lruList), pBuf->fileSize/1024.0, pBuf->pageSize/1024.0f, pBuf->qId);
|
||||
|
||||
fclose(pBuf->file);
|
||||
} else {
|
||||
uDebug("Paged buffer closed, total:%.2f Kb, no file created, %"PRIx64, pBuf->totalBufSize/1024.0, pBuf->qId);
|
||||
}
|
||||
|
||||
// print the statistics information
|
||||
{
|
||||
SDiskbasedBufStatis *ps = &pBuf->statis;
|
||||
uDebug("Get/Release pages:%d/%d, flushToDisk:%.2f Kb (%d Pages), loadFromDisk:%.2f Kb (%d Pages), avgPageSize:%.2f Kb\n"
|
||||
, ps->getPages, ps->releasePages, ps->flushBytes/1024.0f, ps->flushPages, ps->loadBytes/1024.0f, ps->loadPages
|
||||
, ps->loadBytes/(1024.0 * ps->loadPages));
|
||||
}
|
||||
|
||||
remove(pBuf->path);
|
||||
tfree(pBuf->path);
|
||||
|
||||
SArray** p = taosHashIterate(pBuf->groupSet, NULL);
|
||||
while(p) {
|
||||
size_t n = taosArrayGetSize(*p);
|
||||
for(int32_t i = 0; i < n; ++i) {
|
||||
SPageInfo* pi = taosArrayGetP(*p, i);
|
||||
tfree(pi->pData);
|
||||
tfree(pi);
|
||||
}
|
||||
|
||||
taosArrayDestroy(*p);
|
||||
p = taosHashIterate(pBuf->groupSet, p);
|
||||
}
|
||||
|
||||
tdListFree(pBuf->lruList);
|
||||
taosArrayDestroy(pBuf->emptyDummyIdList);
|
||||
taosHashCleanup(pBuf->groupSet);
|
||||
taosHashCleanup(pBuf->all);
|
||||
|
||||
tfree(pBuf->assistBuf);
|
||||
tfree(pBuf);
|
||||
}
|
||||
|
||||
SPageInfo* getLastPageInfo(SIDList pList) {
|
||||
size_t size = taosArrayGetSize(pList);
|
||||
SPageInfo* pPgInfo = taosArrayGetP(pList, size - 1);
|
||||
return pPgInfo;
|
||||
}
|
||||
|
||||
int32_t getPageId(const SPageInfo* pPgInfo) {
|
||||
ASSERT(pPgInfo != NULL);
|
||||
return pPgInfo->pageId;
|
||||
}
|
||||
|
||||
int32_t getBufPageSize(const SDiskbasedBuf* pBuf) {
|
||||
return pBuf->pageSize;
|
||||
}
|
||||
|
||||
int32_t getNumOfInMemBufPages(const SDiskbasedBuf* pBuf) {
|
||||
return pBuf->inMemPages;
|
||||
}
|
||||
|
||||
bool isAllDataInMemBuf(const SDiskbasedBuf* pBuf) {
|
||||
return pBuf->fileSize == 0;
|
||||
}
|
||||
|
||||
void setBufPageDirty(SFilePage* pPage, bool dirty) {
|
||||
int32_t offset = offsetof(SPageInfo, pData); // todo extract method
|
||||
char* p = (char*)pPage - offset;
|
||||
|
||||
SPageInfo* ppi = ((SPageInfo**) p)[0];
|
||||
ppi->dirty = dirty;
|
||||
}
|
||||
|
||||
void printStatisBeforeClose(SDiskbasedBuf* pBuf) {
|
||||
pBuf->printStatis = true;
|
||||
}
|
||||
|
||||
SDiskbasedBufStatis getDBufStatis(const SDiskbasedBuf* pBuf) {
|
||||
return pBuf->statis;
|
||||
}
|
||||
|
||||
void printStatisData(const SDiskbasedBuf* pBuf) {
|
||||
if (!pBuf->printStatis) {
|
||||
return;
|
||||
}
|
||||
|
||||
const SDiskbasedBufStatis* ps = &pBuf->statis;
|
||||
|
||||
printf(
|
||||
"Paged buffer closed, total:%.2f Kb (%d Pages), inmem size:%.2f Kb (%d Pages), file size:%.2f Kb, page size:%.2f "
|
||||
"Kb, %" PRIx64 "\n",
|
||||
pBuf->totalBufSize / 1024.0, pBuf->numOfPages, listNEles(pBuf->lruList) * pBuf->pageSize / 1024.0,
|
||||
listNEles(pBuf->lruList), pBuf->fileSize / 1024.0, pBuf->pageSize / 1024.0f, pBuf->qId);
|
||||
|
||||
printf(
|
||||
"Get/Release pages:%d/%d, flushToDisk:%.2f Kb (%d Pages), loadFromDisk:%.2f Kb (%d Pages), avgPageSize:%.2f Kb\n",
|
||||
ps->getPages, ps->releasePages, ps->flushBytes / 1024.0f, ps->flushPages, ps->loadBytes / 1024.0f, ps->loadPages,
|
||||
ps->loadBytes / (1024.0 * ps->loadPages));
|
||||
}
|
||||
|
|
@ -1,468 +0,0 @@
|
|||
/*
|
||||
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
|
||||
*
|
||||
* This program is free software: you can use, redistribute, and/or modify
|
||||
* it under the terms of the GNU Affero General Public License, version 3
|
||||
* or later ("AGPL"), as published by the Free Software Foundation.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful, but WITHOUT
|
||||
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#define _DEFAULT_SOURCE
|
||||
|
||||
#include "tpagedfile.h"
|
||||
#include "thash.h"
|
||||
#include "stddef.h"
|
||||
#include "taoserror.h"
|
||||
#include "tcompression.h"
|
||||
|
||||
#define GET_DATA_PAYLOAD(_p) ((char *)(_p)->pData + POINTER_BYTES)
|
||||
#define NO_IN_MEM_AVAILABLE_PAGES(_b) (listNEles((_b)->lruList) >= (_b)->inMemPages)
|
||||
|
||||
int32_t createDiskbasedResultBuffer(SDiskbasedResultBuf** pResultBuf, int32_t pagesize, int32_t inMemBufSize, uint64_t qId, const char* dir) {
|
||||
*pResultBuf = calloc(1, sizeof(SDiskbasedResultBuf));
|
||||
|
||||
SDiskbasedResultBuf* pResBuf = *pResultBuf;
|
||||
if (pResBuf == NULL) {
|
||||
return TSDB_CODE_OUT_OF_MEMORY;
|
||||
}
|
||||
|
||||
pResBuf->pageSize = pagesize;
|
||||
pResBuf->numOfPages = 0; // all pages are in buffer in the first place
|
||||
pResBuf->totalBufSize = 0;
|
||||
pResBuf->inMemPages = inMemBufSize/pagesize; // maximum allowed pages, it is a soft limit.
|
||||
pResBuf->allocateId = -1;
|
||||
pResBuf->comp = true;
|
||||
pResBuf->file = NULL;
|
||||
pResBuf->qId = qId;
|
||||
pResBuf->fileSize = 0;
|
||||
|
||||
// at least more than 2 pages must be in memory
|
||||
assert(inMemBufSize >= pagesize * 2);
|
||||
|
||||
pResBuf->lruList = tdListNew(POINTER_BYTES);
|
||||
|
||||
// init id hash table
|
||||
pResBuf->groupSet = taosHashInit(10, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, false);
|
||||
pResBuf->assistBuf = malloc(pResBuf->pageSize + 2); // EXTRA BYTES
|
||||
pResBuf->all = taosHashInit(10, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, false);
|
||||
|
||||
char path[PATH_MAX] = {0};
|
||||
taosGetTmpfilePath(dir, "qbuf", path);
|
||||
pResBuf->path = strdup(path);
|
||||
|
||||
pResBuf->emptyDummyIdList = taosArrayInit(1, sizeof(int32_t));
|
||||
|
||||
// qDebug("QInfo:0x%"PRIx64" create resBuf for output, page size:%d, inmem buf pages:%d, file:%s", qId, pResBuf->pageSize,
|
||||
// pResBuf->inMemPages, pResBuf->path);
|
||||
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
static int32_t createDiskFile(SDiskbasedResultBuf* pResultBuf) {
|
||||
pResultBuf->file = fopen(pResultBuf->path, "wb+");
|
||||
if (pResultBuf->file == NULL) {
|
||||
// qError("failed to create tmp file: %s on disk. %s", pResultBuf->path, strerror(errno));
|
||||
return TAOS_SYSTEM_ERROR(errno);
|
||||
}
|
||||
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
static char* doCompressData(void* data, int32_t srcSize, int32_t *dst, SDiskbasedResultBuf* pResultBuf) { // do nothing
|
||||
if (!pResultBuf->comp) {
|
||||
*dst = srcSize;
|
||||
return data;
|
||||
}
|
||||
|
||||
*dst = tsCompressString(data, srcSize, 1, pResultBuf->assistBuf, srcSize, ONE_STAGE_COMP, NULL, 0);
|
||||
|
||||
memcpy(data, pResultBuf->assistBuf, *dst);
|
||||
return data;
|
||||
}
|
||||
|
||||
static char* doDecompressData(void* data, int32_t srcSize, int32_t *dst, SDiskbasedResultBuf* pResultBuf) { // do nothing
|
||||
if (!pResultBuf->comp) {
|
||||
*dst = srcSize;
|
||||
return data;
|
||||
}
|
||||
|
||||
*dst = tsDecompressString(data, srcSize, 1, pResultBuf->assistBuf, pResultBuf->pageSize, ONE_STAGE_COMP, NULL, 0);
|
||||
if (*dst > 0) {
|
||||
memcpy(data, pResultBuf->assistBuf, *dst);
|
||||
}
|
||||
return data;
|
||||
}
|
||||
|
||||
static int32_t allocatePositionInFile(SDiskbasedResultBuf* pResultBuf, size_t size) {
|
||||
if (pResultBuf->pFree == NULL) {
|
||||
return pResultBuf->nextPos;
|
||||
} else {
|
||||
int32_t offset = -1;
|
||||
|
||||
size_t num = taosArrayGetSize(pResultBuf->pFree);
|
||||
for(int32_t i = 0; i < num; ++i) {
|
||||
SFreeListItem* pi = taosArrayGet(pResultBuf->pFree, i);
|
||||
if (pi->len >= size) {
|
||||
offset = pi->offset;
|
||||
pi->offset += (int32_t)size;
|
||||
pi->len -= (int32_t)size;
|
||||
|
||||
return offset;
|
||||
}
|
||||
}
|
||||
|
||||
// no available recycle space, allocate new area in file
|
||||
return pResultBuf->nextPos;
|
||||
}
|
||||
}
|
||||
|
||||
static char* doFlushPageToDisk(SDiskbasedResultBuf* pResultBuf, SPageInfo* pg) {
|
||||
assert(!pg->used && pg->pData != NULL);
|
||||
|
||||
int32_t size = -1;
|
||||
char* t = doCompressData(GET_DATA_PAYLOAD(pg), pResultBuf->pageSize, &size, pResultBuf);
|
||||
|
||||
// this page is flushed to disk for the first time
|
||||
if (pg->info.offset == -1) {
|
||||
pg->info.offset = allocatePositionInFile(pResultBuf, size);
|
||||
pResultBuf->nextPos += size;
|
||||
|
||||
int32_t ret = fseek(pResultBuf->file, pg->info.offset, SEEK_SET);
|
||||
assert(ret == 0);
|
||||
|
||||
ret = (int32_t) fwrite(t, 1, size, pResultBuf->file);
|
||||
assert(ret == size);
|
||||
|
||||
if (pResultBuf->fileSize < pg->info.offset + pg->info.length) {
|
||||
pResultBuf->fileSize = pg->info.offset + pg->info.length;
|
||||
}
|
||||
} else {
|
||||
// length becomes greater, current space is not enough, allocate new place, otherwise, do nothing
|
||||
if (pg->info.length < size) {
|
||||
// 1. add current space to free list
|
||||
taosArrayPush(pResultBuf->pFree, &pg->info);
|
||||
|
||||
// 2. allocate new position, and update the info
|
||||
pg->info.offset = allocatePositionInFile(pResultBuf, size);
|
||||
pResultBuf->nextPos += size;
|
||||
}
|
||||
|
||||
//3. write to disk.
|
||||
int32_t ret = fseek(pResultBuf->file, pg->info.offset, SEEK_SET);
|
||||
if (ret != 0) { // todo handle the error case
|
||||
|
||||
}
|
||||
|
||||
ret = (int32_t)fwrite(t, size, 1, pResultBuf->file);
|
||||
if (ret != size) { // todo handle the error case
|
||||
|
||||
}
|
||||
|
||||
if (pResultBuf->fileSize < pg->info.offset + pg->info.length) {
|
||||
pResultBuf->fileSize = pg->info.offset + pg->info.length;
|
||||
}
|
||||
}
|
||||
|
||||
char* ret = pg->pData;
|
||||
memset(ret, 0, pResultBuf->pageSize);
|
||||
|
||||
pg->pData = NULL;
|
||||
pg->info.length = size;
|
||||
|
||||
pResultBuf->statis.flushBytes += pg->info.length;
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
static char* flushPageToDisk(SDiskbasedResultBuf* pResultBuf, SPageInfo* pg) {
|
||||
int32_t ret = TSDB_CODE_SUCCESS;
|
||||
assert(((int64_t) pResultBuf->numOfPages * pResultBuf->pageSize) == pResultBuf->totalBufSize && pResultBuf->numOfPages >= pResultBuf->inMemPages);
|
||||
|
||||
if (pResultBuf->file == NULL) {
|
||||
if ((ret = createDiskFile(pResultBuf)) != TSDB_CODE_SUCCESS) {
|
||||
terrno = ret;
|
||||
return NULL;
|
||||
}
|
||||
}
|
||||
|
||||
return doFlushPageToDisk(pResultBuf, pg);
|
||||
}
|
||||
|
||||
// load file block data in disk
|
||||
static char* loadPageFromDisk(SDiskbasedResultBuf* pResultBuf, SPageInfo* pg) {
|
||||
int32_t ret = fseek(pResultBuf->file, pg->info.offset, SEEK_SET);
|
||||
ret = (int32_t)fread(GET_DATA_PAYLOAD(pg), 1, pg->info.length, pResultBuf->file);
|
||||
if (ret != pg->info.length) {
|
||||
terrno = errno;
|
||||
return NULL;
|
||||
}
|
||||
|
||||
pResultBuf->statis.loadBytes += pg->info.length;
|
||||
|
||||
int32_t fullSize = 0;
|
||||
doDecompressData(GET_DATA_PAYLOAD(pg), pg->info.length, &fullSize, pResultBuf);
|
||||
|
||||
return (char*)GET_DATA_PAYLOAD(pg);
|
||||
}
|
||||
|
||||
static SIDList addNewGroup(SDiskbasedResultBuf* pResultBuf, int32_t groupId) {
|
||||
assert(taosHashGet(pResultBuf->groupSet, (const char*) &groupId, sizeof(int32_t)) == NULL);
|
||||
|
||||
SArray* pa = taosArrayInit(1, POINTER_BYTES);
|
||||
int32_t ret = taosHashPut(pResultBuf->groupSet, (const char*)&groupId, sizeof(int32_t), &pa, POINTER_BYTES);
|
||||
assert(ret == 0);
|
||||
|
||||
return pa;
|
||||
}
|
||||
|
||||
static SPageInfo* registerPage(SDiskbasedResultBuf* pResultBuf, int32_t groupId, int32_t pageId) {
|
||||
SIDList list = NULL;
|
||||
|
||||
char** p = taosHashGet(pResultBuf->groupSet, (const char*)&groupId, sizeof(int32_t));
|
||||
if (p == NULL) { // it is a new group id
|
||||
list = addNewGroup(pResultBuf, groupId);
|
||||
} else {
|
||||
list = (SIDList) (*p);
|
||||
}
|
||||
|
||||
pResultBuf->numOfPages += 1;
|
||||
|
||||
SPageInfo* ppi = malloc(sizeof(SPageInfo));//{ .info = PAGE_INFO_INITIALIZER, .pageId = pageId, .pn = NULL};
|
||||
|
||||
ppi->pageId = pageId;
|
||||
ppi->pData = NULL;
|
||||
ppi->info = PAGE_INFO_INITIALIZER;
|
||||
ppi->used = true;
|
||||
ppi->pn = NULL;
|
||||
|
||||
return *(SPageInfo**) taosArrayPush(list, &ppi);
|
||||
}
|
||||
|
||||
static SListNode* getEldestUnrefedPage(SDiskbasedResultBuf* pResultBuf) {
|
||||
SListIter iter = {0};
|
||||
tdListInitIter(pResultBuf->lruList, &iter, TD_LIST_BACKWARD);
|
||||
|
||||
SListNode* pn = NULL;
|
||||
while((pn = tdListNext(&iter)) != NULL) {
|
||||
assert(pn != NULL);
|
||||
|
||||
SPageInfo* pageInfo = *(SPageInfo**) pn->data;
|
||||
assert(pageInfo->pageId >= 0 && pageInfo->pn == pn);
|
||||
|
||||
if (!pageInfo->used) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return pn;
|
||||
}
|
||||
|
||||
static char* evicOneDataPage(SDiskbasedResultBuf* pResultBuf) {
|
||||
char* bufPage = NULL;
|
||||
SListNode* pn = getEldestUnrefedPage(pResultBuf);
|
||||
|
||||
// all pages are referenced by user, try to allocate new space
|
||||
if (pn == NULL) {
|
||||
int32_t prev = pResultBuf->inMemPages;
|
||||
|
||||
// increase by 50% of previous mem pages
|
||||
pResultBuf->inMemPages = (int32_t)(pResultBuf->inMemPages * 1.5f);
|
||||
|
||||
// qWarn("%p in memory buf page not sufficient, expand from %d to %d, page size:%d", pResultBuf, prev,
|
||||
// pResultBuf->inMemPages, pResultBuf->pageSize);
|
||||
} else {
|
||||
pResultBuf->statis.flushPages += 1;
|
||||
tdListPopNode(pResultBuf->lruList, pn);
|
||||
|
||||
SPageInfo* d = *(SPageInfo**) pn->data;
|
||||
assert(d->pn == pn);
|
||||
|
||||
d->pn = NULL;
|
||||
tfree(pn);
|
||||
|
||||
bufPage = flushPageToDisk(pResultBuf, d);
|
||||
}
|
||||
|
||||
return bufPage;
|
||||
}
|
||||
|
||||
static void lruListPushFront(SList *pList, SPageInfo* pi) {
|
||||
tdListPrepend(pList, &pi);
|
||||
SListNode* front = tdListGetHead(pList);
|
||||
pi->pn = front;
|
||||
}
|
||||
|
||||
static void lruListMoveToFront(SList *pList, SPageInfo* pi) {
|
||||
tdListPopNode(pList, pi->pn);
|
||||
tdListPrependNode(pList, pi->pn);
|
||||
}
|
||||
|
||||
static FORCE_INLINE size_t getAllocPageSize(int32_t pageSize) {
|
||||
return pageSize + POINTER_BYTES + 2 + sizeof(SFilePage);
|
||||
}
|
||||
|
||||
SFilePage* getNewDataBuf(SDiskbasedResultBuf* pResultBuf, int32_t groupId, int32_t* pageId) {
|
||||
pResultBuf->statis.getPages += 1;
|
||||
|
||||
char* availablePage = NULL;
|
||||
if (NO_IN_MEM_AVAILABLE_PAGES(pResultBuf)) {
|
||||
availablePage = evicOneDataPage(pResultBuf);
|
||||
}
|
||||
|
||||
// register new id in this group
|
||||
*pageId = (++pResultBuf->allocateId);
|
||||
|
||||
// register page id info
|
||||
SPageInfo* pi = registerPage(pResultBuf, groupId, *pageId);
|
||||
|
||||
// add to LRU list
|
||||
assert(listNEles(pResultBuf->lruList) < pResultBuf->inMemPages && pResultBuf->inMemPages > 0);
|
||||
|
||||
lruListPushFront(pResultBuf->lruList, pi);
|
||||
|
||||
// add to hash map
|
||||
taosHashPut(pResultBuf->all, pageId, sizeof(int32_t), &pi, POINTER_BYTES);
|
||||
|
||||
// allocate buf
|
||||
if (availablePage == NULL) {
|
||||
pi->pData = calloc(1, getAllocPageSize(pResultBuf->pageSize)); // add extract bytes in case of zipped buffer increased.
|
||||
} else {
|
||||
pi->pData = availablePage;
|
||||
}
|
||||
|
||||
pResultBuf->totalBufSize += pResultBuf->pageSize;
|
||||
|
||||
((void**)pi->pData)[0] = pi;
|
||||
pi->used = true;
|
||||
|
||||
return (void *)(GET_DATA_PAYLOAD(pi));
|
||||
}
|
||||
|
||||
SFilePage* getResBufPage(SDiskbasedResultBuf* pResultBuf, int32_t id) {
|
||||
assert(pResultBuf != NULL && id >= 0);
|
||||
pResultBuf->statis.getPages += 1;
|
||||
|
||||
SPageInfo** pi = taosHashGet(pResultBuf->all, &id, sizeof(int32_t));
|
||||
assert(pi != NULL && *pi != NULL);
|
||||
|
||||
if ((*pi)->pData != NULL) { // it is in memory
|
||||
// no need to update the LRU list if only one page exists
|
||||
if (pResultBuf->numOfPages == 1) {
|
||||
(*pi)->used = true;
|
||||
return (void *)(GET_DATA_PAYLOAD(*pi));
|
||||
}
|
||||
|
||||
SPageInfo** pInfo = (SPageInfo**) ((*pi)->pn->data);
|
||||
assert(*pInfo == *pi);
|
||||
|
||||
lruListMoveToFront(pResultBuf->lruList, (*pi));
|
||||
(*pi)->used = true;
|
||||
|
||||
return (void *)(GET_DATA_PAYLOAD(*pi));
|
||||
|
||||
} else { // not in memory
|
||||
assert((*pi)->pData == NULL && (*pi)->pn == NULL && (*pi)->info.length >= 0 && (*pi)->info.offset >= 0);
|
||||
|
||||
char* availablePage = NULL;
|
||||
if (NO_IN_MEM_AVAILABLE_PAGES(pResultBuf)) {
|
||||
availablePage = evicOneDataPage(pResultBuf);
|
||||
}
|
||||
|
||||
if (availablePage == NULL) {
|
||||
(*pi)->pData = calloc(1, getAllocPageSize(pResultBuf->pageSize));
|
||||
} else {
|
||||
(*pi)->pData = availablePage;
|
||||
}
|
||||
|
||||
((void**)((*pi)->pData))[0] = (*pi);
|
||||
|
||||
lruListPushFront(pResultBuf->lruList, *pi);
|
||||
(*pi)->used = true;
|
||||
|
||||
loadPageFromDisk(pResultBuf, *pi);
|
||||
return (void *)(GET_DATA_PAYLOAD(*pi));
|
||||
}
|
||||
}
|
||||
|
||||
void releaseResBufPage(SDiskbasedResultBuf* pResultBuf, void* page) {
|
||||
assert(pResultBuf != NULL && page != NULL);
|
||||
char* p = (char*) page - POINTER_BYTES;
|
||||
|
||||
SPageInfo* ppi = ((SPageInfo**) p)[0];
|
||||
releaseResBufPageInfo(pResultBuf, ppi);
|
||||
}
|
||||
|
||||
void releaseResBufPageInfo(SDiskbasedResultBuf* pResultBuf, SPageInfo* pi) {
|
||||
assert(pi->pData != NULL && pi->used);
|
||||
|
||||
pi->used = false;
|
||||
pResultBuf->statis.releasePages += 1;
|
||||
}
|
||||
|
||||
size_t getNumOfResultBufGroupId(const SDiskbasedResultBuf* pResultBuf) { return taosHashGetSize(pResultBuf->groupSet); }
|
||||
|
||||
size_t getResBufSize(const SDiskbasedResultBuf* pResultBuf) { return (size_t)pResultBuf->totalBufSize; }
|
||||
|
||||
SIDList getDataBufPagesIdList(SDiskbasedResultBuf* pResultBuf, int32_t groupId) {
|
||||
assert(pResultBuf != NULL);
|
||||
|
||||
char** p = taosHashGet(pResultBuf->groupSet, (const char*)&groupId, sizeof(int32_t));
|
||||
if (p == NULL) { // it is a new group id
|
||||
return pResultBuf->emptyDummyIdList;
|
||||
} else {
|
||||
return (SArray*) (*p);
|
||||
}
|
||||
}
|
||||
|
||||
void destroyResultBuf(SDiskbasedResultBuf* pResultBuf) {
|
||||
if (pResultBuf == NULL) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (pResultBuf->file != NULL) {
|
||||
// qDebug("QInfo:0x%"PRIx64" res output buffer closed, total:%.2f Kb, inmem size:%.2f Kb, file size:%.2f Kb",
|
||||
// pResultBuf->qId, pResultBuf->totalBufSize/1024.0, listNEles(pResultBuf->lruList) * pResultBuf->pageSize / 1024.0,
|
||||
// pResultBuf->fileSize/1024.0);
|
||||
|
||||
fclose(pResultBuf->file);
|
||||
} else {
|
||||
// qDebug("QInfo:0x%"PRIx64" res output buffer closed, total:%.2f Kb, no file created", pResultBuf->qId,
|
||||
// pResultBuf->totalBufSize/1024.0);
|
||||
}
|
||||
|
||||
remove(pResultBuf->path);
|
||||
tfree(pResultBuf->path);
|
||||
|
||||
SArray** p = taosHashIterate(pResultBuf->groupSet, NULL);
|
||||
while(p) {
|
||||
size_t n = taosArrayGetSize(*p);
|
||||
for(int32_t i = 0; i < n; ++i) {
|
||||
SPageInfo* pi = taosArrayGetP(*p, i);
|
||||
tfree(pi->pData);
|
||||
tfree(pi);
|
||||
}
|
||||
|
||||
taosArrayDestroy(*p);
|
||||
p = taosHashIterate(pResultBuf->groupSet, p);
|
||||
}
|
||||
|
||||
tdListFree(pResultBuf->lruList);
|
||||
taosArrayDestroy(pResultBuf->emptyDummyIdList);
|
||||
taosHashCleanup(pResultBuf->groupSet);
|
||||
taosHashCleanup(pResultBuf->all);
|
||||
|
||||
tfree(pResultBuf->assistBuf);
|
||||
tfree(pResultBuf);
|
||||
}
|
||||
|
||||
SPageInfo* getLastPageInfo(SIDList pList) {
|
||||
size_t size = taosArrayGetSize(pList);
|
||||
return (SPageInfo*) taosArrayGetP(pList, size - 1);
|
||||
}
|
||||
|
||||
|
|
@ -0,0 +1,165 @@
|
|||
#include <gtest/gtest.h>
|
||||
#include <cassert>
|
||||
#include <iostream>
|
||||
|
||||
#include "taos.h"
|
||||
#include "tpagedbuf.h"
|
||||
|
||||
#pragma GCC diagnostic push
|
||||
#pragma GCC diagnostic ignored "-Wunused-function"
|
||||
#pragma GCC diagnostic ignored "-Wunused-variable"
|
||||
|
||||
namespace {
|
||||
// simple test
|
||||
void simpleTest() {
|
||||
SDiskbasedBuf* pResultBuf = NULL;
|
||||
int32_t ret = createDiskbasedBuffer(&pResultBuf, 1024, 4096, 1, "/tmp/");
|
||||
|
||||
int32_t pageId = 0;
|
||||
int32_t groupId = 0;
|
||||
|
||||
SFilePage* pBufPage = getNewDataBuf(pResultBuf, groupId, &pageId);
|
||||
ASSERT_TRUE(pBufPage != NULL);
|
||||
|
||||
ASSERT_EQ(getTotalBufSize(pResultBuf), 1024);
|
||||
|
||||
SIDList list = getDataBufPagesIdList(pResultBuf, groupId);
|
||||
ASSERT_EQ(taosArrayGetSize(list), 1);
|
||||
ASSERT_EQ(getNumOfResultBufGroupId(pResultBuf), 1);
|
||||
|
||||
releaseBufPage(pResultBuf, pBufPage);
|
||||
|
||||
SFilePage* pBufPage1 = getNewDataBuf(pResultBuf, groupId, &pageId);
|
||||
|
||||
SFilePage* t = getBufPage(pResultBuf, pageId);
|
||||
ASSERT_TRUE(t == pBufPage1);
|
||||
|
||||
SFilePage* pBufPage2 = getNewDataBuf(pResultBuf, groupId, &pageId);
|
||||
SFilePage* t1 = getBufPage(pResultBuf, pageId);
|
||||
ASSERT_TRUE(t1 == pBufPage2);
|
||||
|
||||
SFilePage* pBufPage3 = getNewDataBuf(pResultBuf, groupId, &pageId);
|
||||
SFilePage* t2 = getBufPage(pResultBuf, pageId);
|
||||
ASSERT_TRUE(t2 == pBufPage3);
|
||||
|
||||
SFilePage* pBufPage4 = getNewDataBuf(pResultBuf, groupId, &pageId);
|
||||
SFilePage* t3 = getBufPage(pResultBuf, pageId);
|
||||
ASSERT_TRUE(t3 == pBufPage4);
|
||||
|
||||
SFilePage* pBufPage5 = getNewDataBuf(pResultBuf, groupId, &pageId);
|
||||
SFilePage* t4 = getBufPage(pResultBuf, pageId);
|
||||
ASSERT_TRUE(t4 == pBufPage5);
|
||||
|
||||
destroyResultBuf(pResultBuf);
|
||||
}
|
||||
|
||||
void writeDownTest() {
|
||||
SDiskbasedBuf* pResultBuf = NULL;
|
||||
int32_t ret = createDiskbasedBuffer(&pResultBuf, 1024, 4*1024, 1, "/tmp/");
|
||||
|
||||
int32_t pageId = 0;
|
||||
int32_t writePageId = 0;
|
||||
int32_t groupId = 0;
|
||||
int32_t nx = 12345;
|
||||
|
||||
SFilePage* pBufPage = getNewDataBuf(pResultBuf, groupId, &pageId);
|
||||
ASSERT_TRUE(pBufPage != NULL);
|
||||
|
||||
*(int32_t*)(pBufPage->data) = nx;
|
||||
writePageId = pageId;
|
||||
releaseBufPage(pResultBuf, pBufPage);
|
||||
|
||||
SFilePage* pBufPage1 = getNewDataBuf(pResultBuf, groupId, &pageId);
|
||||
SFilePage* t1 = getBufPage(pResultBuf, pageId);
|
||||
ASSERT_TRUE(t1 == pBufPage1);
|
||||
ASSERT_TRUE(pageId == 1);
|
||||
|
||||
SFilePage* pBufPage2 = getNewDataBuf(pResultBuf, groupId, &pageId);
|
||||
SFilePage* t2 = getBufPage(pResultBuf, pageId);
|
||||
ASSERT_TRUE(t2 == pBufPage2);
|
||||
ASSERT_TRUE(pageId == 2);
|
||||
|
||||
SFilePage* pBufPage3 = getNewDataBuf(pResultBuf, groupId, &pageId);
|
||||
SFilePage* t3 = getBufPage(pResultBuf, pageId);
|
||||
ASSERT_TRUE(t3 == pBufPage3);
|
||||
ASSERT_TRUE(pageId == 3);
|
||||
|
||||
SFilePage* pBufPage4 = getNewDataBuf(pResultBuf, groupId, &pageId);
|
||||
SFilePage* t4 = getBufPage(pResultBuf, pageId);
|
||||
ASSERT_TRUE(t4 == pBufPage4);
|
||||
ASSERT_TRUE(pageId == 4);
|
||||
releaseBufPage(pResultBuf, t4);
|
||||
|
||||
// flush the written page to disk, and read it out again
|
||||
SFilePage* pBufPagex = getBufPage(pResultBuf, writePageId);
|
||||
ASSERT_EQ(*(int32_t*)pBufPagex->data, nx);
|
||||
|
||||
SArray* pa = getDataBufPagesIdList(pResultBuf, groupId);
|
||||
ASSERT_EQ(taosArrayGetSize(pa), 5);
|
||||
|
||||
destroyResultBuf(pResultBuf);
|
||||
}
|
||||
|
||||
void recyclePageTest() {
|
||||
SDiskbasedBuf* pResultBuf = NULL;
|
||||
int32_t ret = createDiskbasedBuffer(&pResultBuf, 1024, 4*1024, 1, "/tmp/");
|
||||
|
||||
int32_t pageId = 0;
|
||||
int32_t writePageId = 0;
|
||||
int32_t groupId = 0;
|
||||
int32_t nx = 12345;
|
||||
|
||||
SFilePage* pBufPage = getNewDataBuf(pResultBuf, groupId, &pageId);
|
||||
ASSERT_TRUE(pBufPage != NULL);
|
||||
releaseBufPage(pResultBuf, pBufPage);
|
||||
|
||||
SFilePage* pBufPage1 = getNewDataBuf(pResultBuf, groupId, &pageId);
|
||||
SFilePage* t1 = getBufPage(pResultBuf, pageId);
|
||||
ASSERT_TRUE(t1 == pBufPage1);
|
||||
ASSERT_TRUE(pageId == 1);
|
||||
|
||||
SFilePage* pBufPage2 = getNewDataBuf(pResultBuf, groupId, &pageId);
|
||||
SFilePage* t2 = getBufPage(pResultBuf, pageId);
|
||||
ASSERT_TRUE(t2 == pBufPage2);
|
||||
ASSERT_TRUE(pageId == 2);
|
||||
|
||||
SFilePage* pBufPage3 = getNewDataBuf(pResultBuf, groupId, &pageId);
|
||||
SFilePage* t3 = getBufPage(pResultBuf, pageId);
|
||||
ASSERT_TRUE(t3 == pBufPage3);
|
||||
ASSERT_TRUE(pageId == 3);
|
||||
|
||||
SFilePage* pBufPage4 = getNewDataBuf(pResultBuf, groupId, &pageId);
|
||||
SFilePage* t4 = getBufPage(pResultBuf, pageId);
|
||||
ASSERT_TRUE(t4 == pBufPage4);
|
||||
ASSERT_TRUE(pageId == 4);
|
||||
releaseBufPage(pResultBuf, t4);
|
||||
|
||||
SFilePage* pBufPage5 = getNewDataBuf(pResultBuf, groupId, &pageId);
|
||||
SFilePage* t5 = getBufPage(pResultBuf, pageId);
|
||||
ASSERT_TRUE(t5 == pBufPage5);
|
||||
ASSERT_TRUE(pageId == 5);
|
||||
|
||||
// flush the written page to disk, and read it out again
|
||||
SFilePage* pBufPagex = getBufPage(pResultBuf, writePageId);
|
||||
*(int32_t*)(pBufPagex->data) = nx;
|
||||
writePageId = pageId; // update the data
|
||||
releaseBufPage(pResultBuf, pBufPagex);
|
||||
|
||||
SFilePage* pBufPagex1 = getBufPage(pResultBuf, 1);
|
||||
|
||||
SArray* pa = getDataBufPagesIdList(pResultBuf, groupId);
|
||||
ASSERT_EQ(taosArrayGetSize(pa), 6);
|
||||
|
||||
destroyResultBuf(pResultBuf);
|
||||
}
|
||||
} // namespace
|
||||
|
||||
|
||||
TEST(testCase, resultBufferTest) {
|
||||
srand(time(NULL));
|
||||
simpleTest();
|
||||
writeDownTest();
|
||||
recyclePageTest();
|
||||
}
|
||||
|
||||
#pragma GCC diagnostic pop
|
||||
Loading…
Reference in New Issue