xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch '3.0' into 3.0test/jcy

This commit is contained in:
jiacy-jcy 2022-06-24 10:19:04 +08:00
parent b7bb08d665 81481d870e
commit e176d9a726
85 changed files with 1402 additions and 1326 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
examples/c/tmq.c
View File

@ -27,7 +27,11 @@ static void msg_process(TAOS_RES* msg) {
printf("db: %s\n", tmq_get_db_name(msg)); printf("db: %s\n", tmq_get_db_name(msg));
printf("vg: %d\n", tmq_get_vgroup_id(msg)); printf("vg: %d\n", tmq_get_vgroup_id(msg));
if (tmq_get_res_type(msg) == TMQ_RES_TABLE_META) { if (tmq_get_res_type(msg) == TMQ_RES_TABLE_META) {
printf("meta, skip\n"); void* meta;
int32_t metaLen;
tmq_get_raw_meta(msg, &meta, &metaLen);
printf("meta, len is %d\n", metaLen);
return; return;
} }
while (1) { while (1) {

2
include/client/taos.h
View File

@ -261,7 +261,7 @@ enum tmq_res_t {
typedef enum tmq_res_t tmq_res_t; typedef enum tmq_res_t tmq_res_t;
DLL_EXPORT tmq_res_t tmq_get_res_type(TAOS_RES *res); DLL_EXPORT tmq_res_t tmq_get_res_type(TAOS_RES *res);
DLL_EXPORT int32_t tmq_get_raw_meta(TAOS_RES *res, const void **raw_meta, int32_t *raw_meta_len); DLL_EXPORT int32_t tmq_get_raw_meta(TAOS_RES *res, void **raw_meta, int32_t *raw_meta_len);
DLL_EXPORT const char *tmq_get_topic_name(TAOS_RES *res); DLL_EXPORT const char *tmq_get_topic_name(TAOS_RES *res);
DLL_EXPORT const char *tmq_get_db_name(TAOS_RES *res); DLL_EXPORT const char *tmq_get_db_name(TAOS_RES *res);
DLL_EXPORT int32_t tmq_get_vgroup_id(TAOS_RES *res); DLL_EXPORT int32_t tmq_get_vgroup_id(TAOS_RES *res);

11
include/common/tcommon.h
View File

@ -69,14 +69,13 @@ typedef struct SColumnDataAgg {
typedef struct SDataBlockInfo { typedef struct SDataBlockInfo {
STimeWindow window; STimeWindow window;
int32_t rows; int32_t rows; // todo hide this attribute
int32_t rowSize; int32_t rowSize;
int64_t uid; // the uid of table, from which current data block comes uint64_t uid; // the uid of table, from which current data block comes
int64_t blockId; // block id, generated by physical planner uint16_t blockId; // block id, generated by physical planner
uint64_t groupId; // no need to serialize uint64_t groupId; // no need to serialize
int16_t numOfCols;
int16_t hasVarCol; int16_t hasVarCol;
int32_t capacity; uint32_t capacity;
// TODO: optimize and remove following // TODO: optimize and remove following
int32_t childId; // used for stream, do not serialize int32_t childId; // used for stream, do not serialize
EStreamType type; // used for stream, do not serialize EStreamType type; // used for stream, do not serialize
@ -97,7 +96,7 @@ typedef struct SVarColAttr {
// pBlockAgg->numOfNull == info.rows, all data are null // pBlockAgg->numOfNull == info.rows, all data are null
// pBlockAgg->numOfNull == 0, no data are null. // pBlockAgg->numOfNull == 0, no data are null.
typedef struct SColumnInfoData { typedef struct SColumnInfoData {
SColumnInfo info; // TODO filter info needs to be removed SColumnInfo info; // column info
bool hasNull; // if current column data has null value. bool hasNull; // if current column data has null value.
char* pData; // the corresponding block data in memory char* pData; // the corresponding block data in memory
union { union {

19
include/common/tdatablock.h
View File

@ -71,8 +71,8 @@ SEpSet getEpSet_s(SCorEpSet* pEpSet);
#define colDataGetData(p1_, r_) \ #define colDataGetData(p1_, r_) \
((IS_VAR_DATA_TYPE((p1_)->info.type)) ? colDataGetVarData(p1_, r_) : colDataGetNumData(p1_, r_)) ((IS_VAR_DATA_TYPE((p1_)->info.type)) ? colDataGetVarData(p1_, r_) : colDataGetNumData(p1_, r_))
#define IS_JSON_NULL(type, data) ((type) == TSDB_DATA_TYPE_JSON && \ #define IS_JSON_NULL(type, data) \
(*(data) == TSDB_DATA_TYPE_NULL || tTagIsJsonNull(data))) ((type) == TSDB_DATA_TYPE_JSON && (*(data) == TSDB_DATA_TYPE_NULL || tTagIsJsonNull(data)))
static FORCE_INLINE bool colDataIsNull_s(const SColumnInfoData* pColumnInfoData, uint32_t row) { static FORCE_INLINE bool colDataIsNull_s(const SColumnInfoData* pColumnInfoData, uint32_t row) {
if (!pColumnInfoData->hasNull) { if (!pColumnInfoData->hasNull) {
@ -184,9 +184,10 @@ static FORCE_INLINE void colDataAppendDouble(SColumnInfoData* pColumnInfoData, u
int32_t getJsonValueLen(const char* data); int32_t getJsonValueLen(const char* data);
int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, const char* pData, bool isNull); int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, const char* pData, bool isNull);
int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, int32_t* capacity, int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, uint32_t* capacity,
const SColumnInfoData* pSource, uint32_t numOfRow2); const SColumnInfoData* pSource, uint32_t numOfRow2);
int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* pSource, int32_t numOfRows); int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* pSource, int32_t numOfRows,
const SDataBlockInfo* pBlockInfo);
int32_t blockDataUpdateTsWindow(SSDataBlock* pDataBlock, int32_t tsColumnIndex); int32_t blockDataUpdateTsWindow(SSDataBlock* pDataBlock, int32_t tsColumnIndex);
int32_t colDataGetLength(const SColumnInfoData* pColumnInfoData, int32_t numOfRows); int32_t colDataGetLength(const SColumnInfoData* pColumnInfoData, int32_t numOfRows);
@ -212,7 +213,7 @@ size_t blockDataGetSerialMetaSize(uint32_t numOfCols);
int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo); int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo);
int32_t blockDataSort_rv(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullFirst); int32_t blockDataSort_rv(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullFirst);
int32_t colInfoDataEnsureCapacity(SColumnInfoData* pColumn, size_t existRows, uint32_t numOfRows); int32_t colInfoDataEnsureCapacity(SColumnInfoData* pColumn, uint32_t numOfRows);
int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows); int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows);
void colInfoDataCleanup(SColumnInfoData* pColumn, uint32_t numOfRows); void colInfoDataCleanup(SColumnInfoData* pColumn, uint32_t numOfRows);
@ -222,7 +223,13 @@ size_t blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize);
int32_t blockDataTrimFirstNRows(SSDataBlock* pBlock, size_t n); int32_t blockDataTrimFirstNRows(SSDataBlock* pBlock, size_t n);
int32_t assignOneDataBlock(SSDataBlock* dst, const SSDataBlock* src);
SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData); SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData);
SSDataBlock* createDataBlock();
int32_t blockDataAppendColInfo(SSDataBlock* pBlock, SColumnInfoData* pColInfoData);
SColumnInfoData createColumnInfoData(int16_t type, int32_t bytes, int16_t colId);
SColumnInfoData* bdGetColumnInfoData(SSDataBlock* pBlock, int32_t index);
void blockCompressEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_t numOfCols, void blockCompressEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_t numOfCols,
int8_t needCompress); int8_t needCompress);
@ -236,7 +243,7 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks
char* buildCtbNameByGroupId(const char* stbName, uint64_t groupId); char* buildCtbNameByGroupId(const char* stbName, uint64_t groupId);
static FORCE_INLINE int32_t blockGetEncodeSize(const SSDataBlock* pBlock) { static FORCE_INLINE int32_t blockGetEncodeSize(const SSDataBlock* pBlock) {
return blockDataGetSerialMetaSize(pBlock->info.numOfCols) + blockDataGetSize(pBlock); return blockDataGetSerialMetaSize(taosArrayGetSize(pBlock->pDataBlock)) + blockDataGetSize(pBlock);
} }
static FORCE_INLINE int32_t blockCompressColData(SColumnInfoData* pColRes, int32_t numOfRows, char* data, static FORCE_INLINE int32_t blockCompressColData(SColumnInfoData* pColRes, int32_t numOfRows, char* data,

15
include/common/tmsg.h
View File

@ -629,11 +629,6 @@ typedef struct {
uint8_t scale; uint8_t scale;
} SColumnInfo; } SColumnInfo;
typedef struct {
int64_t uid;
TSKEY key; // last accessed ts, for subscription
} STableIdInfo;
typedef struct STimeWindow { typedef struct STimeWindow {
TSKEY skey; TSKEY skey;
TSKEY ekey; TSKEY ekey;
@ -1346,11 +1341,19 @@ typedef struct {
int32_t dnodeId; int32_t dnodeId;
char config[TSDB_DNODE_CONFIG_LEN]; char config[TSDB_DNODE_CONFIG_LEN];
char value[TSDB_DNODE_VALUE_LEN]; char value[TSDB_DNODE_VALUE_LEN];
} SMCfgDnodeReq, SDCfgDnodeReq; } SMCfgDnodeReq;
int32_t tSerializeSMCfgDnodeReq(void* buf, int32_t bufLen, SMCfgDnodeReq* pReq); int32_t tSerializeSMCfgDnodeReq(void* buf, int32_t bufLen, SMCfgDnodeReq* pReq);
int32_t tDeserializeSMCfgDnodeReq(void* buf, int32_t bufLen, SMCfgDnodeReq* pReq); int32_t tDeserializeSMCfgDnodeReq(void* buf, int32_t bufLen, SMCfgDnodeReq* pReq);
typedef struct {
char config[TSDB_DNODE_CONFIG_LEN];
char value[TSDB_DNODE_VALUE_LEN];
} SDCfgDnodeReq;
int32_t tSerializeSDCfgDnodeReq(void* buf, int32_t bufLen, SDCfgDnodeReq* pReq);
int32_t tDeserializeSDCfgDnodeReq(void* buf, int32_t bufLen, SDCfgDnodeReq* pReq);
typedef struct { typedef struct {
int32_t dnodeId; int32_t dnodeId;
} SMCreateMnodeReq, SMDropMnodeReq, SDDropMnodeReq, SMCreateQnodeReq, SMDropQnodeReq, SDCreateQnodeReq, SDDropQnodeReq, } SMCreateMnodeReq, SMDropMnodeReq, SDDropMnodeReq, SMCreateQnodeReq, SMDropQnodeReq, SDCreateQnodeReq, SDDropQnodeReq,

4
include/dnode/mnode/mnode.h
View File

@ -73,10 +73,12 @@ void mndStop(SMnode *pMnode);
* @param pMnode The mnode object. * @param pMnode The mnode object.
* @param pCluster * @param pCluster
* @param pVgroup * @param pVgroup
* @param pStbInfo
* @param pGrant * @param pGrant
* @return int32_t 0 for success, -1 for failure. * @return int32_t 0 for success, -1 for failure.
*/ */
int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pCluster, SMonVgroupInfo *pVgroup, SMonGrantInfo *pGrant); int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo,
SMonStbInfo *pStbInfo, SMonGrantInfo *pGrantInfo);
/** /**
* @brief Get mnode loads for status msg. * @brief Get mnode loads for status msg.

17
include/libs/function/function.h
View File

@ -172,23 +172,6 @@ typedef struct tExprNode {
void tExprTreeDestroy(tExprNode *pNode, void (*fp)(void *)); void tExprTreeDestroy(tExprNode *pNode, void (*fp)(void *));
typedef struct SAggFunctionInfo {
char name[FUNCTIONS_NAME_MAX_LENGTH];
int8_t type; // Scalar function or aggregation function
uint32_t functionId; // Function Id
int8_t sFunctionId; // Transfer function for super table query
uint16_t status;
bool (*init)(SqlFunctionCtx *pCtx, struct SResultRowEntryInfo* pResultCellInfo); // setup the execute environment
void (*addInput)(SqlFunctionCtx *pCtx);
// finalizer must be called after all exec has been executed to generated final result.
void (*finalize)(SqlFunctionCtx *pCtx);
void (*combine)(SqlFunctionCtx *pCtx);
int32_t (*dataReqFunc)(SqlFunctionCtx *pCtx, STimeWindow* w, int32_t colId);
} SAggFunctionInfo;
struct SScalarParam { struct SScalarParam {
SColumnInfoData *columnData; SColumnInfoData *columnData;
SHashObj *pHashFilter; SHashObj *pHashFilter;

13
include/libs/monitor/monitor.h
View File

@ -109,6 +109,9 @@ typedef struct {
char version[MON_VER_LEN]; char version[MON_VER_LEN];
float master_uptime; // day float master_uptime; // day
int32_t monitor_interval; // sec int32_t monitor_interval; // sec
int32_t dbs_total;
int32_t stbs_total;
int64_t tbs_total;
int32_t vgroups_total; int32_t vgroups_total;
int32_t vgroups_alive; int32_t vgroups_alive;
int32_t vnodes_total; int32_t vnodes_total;
@ -135,6 +138,15 @@ typedef struct {
SArray *vgroups; // array of SMonVgroupDesc SArray *vgroups; // array of SMonVgroupDesc
} SMonVgroupInfo; } SMonVgroupInfo;
typedef struct {
char stb_name[TSDB_TABLE_NAME_LEN];
char database_name[TSDB_DB_NAME_LEN];
} SMonStbDesc;
typedef struct {
SArray *stbs; // array of SMonStbDesc
} SMonStbInfo;
typedef struct { typedef struct {
int32_t expire_time; int32_t expire_time;
int64_t timeseries_used; int64_t timeseries_used;
@ -144,6 +156,7 @@ typedef struct {
typedef struct { typedef struct {
SMonClusterInfo cluster; SMonClusterInfo cluster;
SMonVgroupInfo vgroup; SMonVgroupInfo vgroup;
SMonStbInfo stb;
SMonGrantInfo grant; SMonGrantInfo grant;
SMonSysInfo sys; SMonSysInfo sys;
SMonLogs log; SMonLogs log;

2
include/libs/stream/tstream.h
View File

@ -344,10 +344,12 @@ static FORCE_INLINE int32_t streamTaskOutput(SStreamTask* pTask, SStreamDataBloc
if (pTask->sinkType == TASK_SINK__TABLE) { if (pTask->sinkType == TASK_SINK__TABLE) {
ASSERT(pTask->dispatchType == TASK_DISPATCH__NONE); ASSERT(pTask->dispatchType == TASK_DISPATCH__NONE);
pTask->tbSink.tbSinkFunc(pTask, pTask->tbSink.vnode, 0, pBlock->blocks); pTask->tbSink.tbSinkFunc(pTask, pTask->tbSink.vnode, 0, pBlock->blocks);
taosArrayDestroyEx(pBlock->blocks, (FDelete)tDeleteSSDataBlock);
taosFreeQitem(pBlock); taosFreeQitem(pBlock);
} else if (pTask->sinkType == TASK_SINK__SMA) { } else if (pTask->sinkType == TASK_SINK__SMA) {
ASSERT(pTask->dispatchType == TASK_DISPATCH__NONE); ASSERT(pTask->dispatchType == TASK_DISPATCH__NONE);
pTask->smaSink.smaSink(pTask->smaSink.vnode, pTask->smaSink.smaId, pBlock->blocks); pTask->smaSink.smaSink(pTask->smaSink.vnode, pTask->smaSink.smaId, pBlock->blocks);
taosArrayDestroyEx(pBlock->blocks, (FDelete)tDeleteSSDataBlock);
taosFreeQitem(pBlock); taosFreeQitem(pBlock);
} else { } else {
ASSERT(pTask->dispatchType != TASK_DISPATCH__NONE); ASSERT(pTask->dispatchType != TASK_DISPATCH__NONE);

6
include/libs/sync/sync.h
View File

@ -157,13 +157,13 @@ typedef struct SSyncLogStore {
SyncIndex (*getCommitIndex)(struct SSyncLogStore* pLogStore); SyncIndex (*getCommitIndex)(struct SSyncLogStore* pLogStore);
// refactor, log[0 .. n] ==> log[m .. n] // refactor, log[0 .. n] ==> log[m .. n]
int32_t (*syncLogSetBeginIndex)(struct SSyncLogStore* pLogStore, SyncIndex beginIndex); // int32_t (*syncLogSetBeginIndex)(struct SSyncLogStore* pLogStore, SyncIndex beginIndex);
int32_t (*syncLogResetBeginIndex)(struct SSyncLogStore* pLogStore);
SyncIndex (*syncLogBeginIndex)(struct SSyncLogStore* pLogStore); SyncIndex (*syncLogBeginIndex)(struct SSyncLogStore* pLogStore);
SyncIndex (*syncLogEndIndex)(struct SSyncLogStore* pLogStore); SyncIndex (*syncLogEndIndex)(struct SSyncLogStore* pLogStore);
bool (*syncLogIsEmpty)(struct SSyncLogStore* pLogStore); bool (*syncLogIsEmpty)(struct SSyncLogStore* pLogStore);
int32_t (*syncLogEntryCount)(struct SSyncLogStore* pLogStore); int32_t (*syncLogEntryCount)(struct SSyncLogStore* pLogStore);
// bool (*syncLogInRange)(struct SSyncLogStore* pLogStore, SyncIndex index); int32_t (*syncLogRestoreFromSnapshot)(struct SSyncLogStore* pLogStore, SyncIndex index);
SyncIndex (*syncLogWriteIndex)(struct SSyncLogStore* pLogStore); SyncIndex (*syncLogWriteIndex)(struct SSyncLogStore* pLogStore);
SyncIndex (*syncLogLastIndex)(struct SSyncLogStore* pLogStore); SyncIndex (*syncLogLastIndex)(struct SSyncLogStore* pLogStore);

2
source/client/src/tmq.c
View File

@ -1871,7 +1871,7 @@ const char* tmq_get_table_name(TAOS_RES* res) {
return NULL; return NULL;
} }
int32_t tmq_get_raw_meta(TAOS_RES* res, const void** raw_meta, int32_t* raw_meta_len) { int32_t tmq_get_raw_meta(TAOS_RES* res, void** raw_meta, int32_t* raw_meta_len) {
if (TD_RES_TMQ_META(res)) { if (TD_RES_TMQ_META(res)) {
SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)res; SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)res;
*raw_meta = pMetaRspObj->metaRsp.metaRsp; *raw_meta = pMetaRspObj->metaRsp.metaRsp;

335
source/common/src/tdatablock.c
View File

@ -214,7 +214,7 @@ static void doBitmapMerge(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, c
} }
} }
int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, int32_t* capacity, int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, uint32_t* capacity,
const SColumnInfoData* pSource, uint32_t numOfRow2) { const SColumnInfoData* pSource, uint32_t numOfRow2) {
ASSERT(pColumnInfoData != NULL && pSource != NULL && pColumnInfoData->info.type == pSource->info.type); ASSERT(pColumnInfoData != NULL && pSource != NULL && pColumnInfoData->info.type == pSource->info.type);
if (numOfRow2 == 0) { if (numOfRow2 == 0) {
@ -263,7 +263,7 @@ int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, in
pColumnInfoData->varmeta.length = len + oldLen; pColumnInfoData->varmeta.length = len + oldLen;
} else { } else {
if (finalNumOfRows > *capacity) { if (finalNumOfRows > *capacity) {
ASSERT(finalNumOfRows*pColumnInfoData->info.bytes); ASSERT(finalNumOfRows * pColumnInfoData->info.bytes);
char* tmp = taosMemoryRealloc(pColumnInfoData->pData, finalNumOfRows * pColumnInfoData->info.bytes); char* tmp = taosMemoryRealloc(pColumnInfoData->pData, finalNumOfRows * pColumnInfoData->info.bytes);
if (tmp == NULL) { if (tmp == NULL) {
return TSDB_CODE_VND_OUT_OF_MEMORY; return TSDB_CODE_VND_OUT_OF_MEMORY;
@ -293,22 +293,19 @@ int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, in
return numOfRow1 + numOfRow2; return numOfRow1 + numOfRow2;
} }
int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* pSource, int32_t numOfRows) { int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* pSource, int32_t numOfRows,
const SDataBlockInfo* pBlockInfo) {
ASSERT(pColumnInfoData != NULL && pSource != NULL && pColumnInfoData->info.type == pSource->info.type); ASSERT(pColumnInfoData != NULL && pSource != NULL && pColumnInfoData->info.type == pSource->info.type);
if (numOfRows <= 0) { if (numOfRows <= 0) {
return numOfRows; return numOfRows;
} }
if (pBlockInfo != NULL) {
ASSERT(pBlockInfo->capacity >= numOfRows);
}
if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) {
// Handle the bitmap
char* p = taosMemoryRealloc(pColumnInfoData->varmeta.offset, sizeof(int32_t) * numOfRows);
if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pColumnInfoData->varmeta.offset = (int32_t*)p;
memcpy(pColumnInfoData->varmeta.offset, pSource->varmeta.offset, sizeof(int32_t) * numOfRows); memcpy(pColumnInfoData->varmeta.offset, pSource->varmeta.offset, sizeof(int32_t) * numOfRows);
if (pColumnInfoData->varmeta.allocLen < pSource->varmeta.length) { if (pColumnInfoData->varmeta.allocLen < pSource->varmeta.length) {
char* tmp = taosMemoryRealloc(pColumnInfoData->pData, pSource->varmeta.length); char* tmp = taosMemoryRealloc(pColumnInfoData->pData, pSource->varmeta.length);
if (tmp == NULL) { if (tmp == NULL) {
@ -319,27 +316,11 @@ int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* p
pColumnInfoData->varmeta.allocLen = pSource->varmeta.length; pColumnInfoData->varmeta.allocLen = pSource->varmeta.length;
} }
memcpy(pColumnInfoData->pData, pSource->pData, pSource->varmeta.length);
pColumnInfoData->varmeta.length = pSource->varmeta.length; pColumnInfoData->varmeta.length = pSource->varmeta.length;
memcpy(pColumnInfoData->pData, pSource->pData, pSource->varmeta.length);
} else { } else {
char* tmp = taosMemoryRealloc(pColumnInfoData->nullbitmap, BitmapLen(numOfRows));
if (tmp == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pColumnInfoData->nullbitmap = tmp;
memcpy(pColumnInfoData->nullbitmap, pSource->nullbitmap, BitmapLen(numOfRows)); memcpy(pColumnInfoData->nullbitmap, pSource->nullbitmap, BitmapLen(numOfRows));
memcpy(pColumnInfoData->pData, pSource->pData, pSource->info.bytes * numOfRows);
if (pSource->pData) {
int32_t newSize = numOfRows * pColumnInfoData->info.bytes;
tmp = taosMemoryRealloc(pColumnInfoData->pData, newSize);
if (tmp == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pColumnInfoData->pData = tmp;
memcpy(pColumnInfoData->pData, pSource->pData, pSource->info.bytes * numOfRows);
}
} }
pColumnInfoData->hasNull = pSource->hasNull; pColumnInfoData->hasNull = pSource->hasNull;
@ -347,10 +328,7 @@ int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* p
return 0; return 0;
} }
size_t blockDataGetNumOfCols(const SSDataBlock* pBlock) { size_t blockDataGetNumOfCols(const SSDataBlock* pBlock) { return taosArrayGetSize(pBlock->pDataBlock); }
ASSERT(pBlock && pBlock->info.numOfCols == taosArrayGetSize(pBlock->pDataBlock));
return pBlock->info.numOfCols;
}
size_t blockDataGetNumOfRows(const SSDataBlock* pBlock) { return pBlock->info.rows; } size_t blockDataGetNumOfRows(const SSDataBlock* pBlock) { return pBlock->info.rows; }
@ -359,7 +337,8 @@ int32_t blockDataUpdateTsWindow(SSDataBlock* pDataBlock, int32_t tsColumnIndex)
return 0; return 0;
} }
if (pDataBlock->info.numOfCols <= 0) { size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock);
if (numOfCols <= 0) {
return -1; return -1;
} }
@ -383,7 +362,8 @@ int32_t blockDataMerge(SSDataBlock* pDest, const SSDataBlock* pSrc) {
assert(pSrc != NULL && pDest != NULL); assert(pSrc != NULL && pDest != NULL);
int32_t capacity = pDest->info.capacity; int32_t capacity = pDest->info.capacity;
for (int32_t i = 0; i < pDest->info.numOfCols; ++i) { size_t numOfCols = taosArrayGetSize(pDest->pDataBlock);
for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pCol2 = taosArrayGet(pDest->pDataBlock, i); SColumnInfoData* pCol2 = taosArrayGet(pDest->pDataBlock, i);
SColumnInfoData* pCol1 = taosArrayGet(pSrc->pDataBlock, i); SColumnInfoData* pCol1 = taosArrayGet(pSrc->pDataBlock, i);
@ -399,9 +379,8 @@ int32_t blockDataMerge(SSDataBlock* pDest, const SSDataBlock* pSrc) {
size_t blockDataGetSize(const SSDataBlock* pBlock) { size_t blockDataGetSize(const SSDataBlock* pBlock) {
assert(pBlock != NULL); assert(pBlock != NULL);
size_t total = 0; size_t total = 0;
int32_t numOfCols = pBlock->info.numOfCols; size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
for (int32_t i = 0; i < numOfCols; ++i) { for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i);
total += colDataGetFullLength(pColInfoData, pBlock->info.rows); total += colDataGetFullLength(pColInfoData, pBlock->info.rows);
@ -416,7 +395,7 @@ int32_t blockDataSplitRows(SSDataBlock* pBlock, bool hasVarCol, int32_t startInd
int32_t pageSize) { int32_t pageSize) {
ASSERT(pBlock != NULL && stopIndex != NULL); ASSERT(pBlock != NULL && stopIndex != NULL);
int32_t numOfCols = pBlock->info.numOfCols; size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
int32_t numOfRows = pBlock->info.rows; int32_t numOfRows = pBlock->info.rows;
int32_t bitmapChar = 1; int32_t bitmapChar = 1;
@ -477,33 +456,24 @@ SSDataBlock* blockDataExtractBlock(SSDataBlock* pBlock, int32_t startIndex, int3
return NULL; return NULL;
} }
SSDataBlock* pDst = taosMemoryCalloc(1, sizeof(SSDataBlock)); SSDataBlock* pDst = createDataBlock();
if (pDst == NULL) { if (pDst == NULL) {
return NULL; return NULL;
} }
pDst->info = pBlock->info; pDst->info = pBlock->info;
pDst->info.rows = 0; pDst->info.rows = 0;
pDst->pDataBlock = taosArrayInit(pBlock->info.numOfCols, sizeof(SColumnInfoData)); size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
for (int32_t i = 0; i < numOfCols; ++i) {
for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) {
SColumnInfoData colInfo = {0}; SColumnInfoData colInfo = {0};
SColumnInfoData* pSrcCol = taosArrayGet(pBlock->pDataBlock, i); SColumnInfoData* pSrcCol = taosArrayGet(pBlock->pDataBlock, i);
colInfo.info = pSrcCol->info; colInfo.info = pSrcCol->info;
blockDataAppendColInfo(pDst, &colInfo);
if (IS_VAR_DATA_TYPE(pSrcCol->info.type)) {
SVarColAttr* pAttr = &colInfo.varmeta;
pAttr->offset = taosMemoryCalloc(rowCount, sizeof(int32_t));
} else {
colInfo.nullbitmap = taosMemoryCalloc(1, BitmapLen(rowCount));
colInfo.pData = taosMemoryCalloc(rowCount, colInfo.info.bytes);
}
taosArrayPush(pDst->pDataBlock, &colInfo);
} }
for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { blockDataEnsureCapacity(pDst, rowCount);
for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, i); SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, i);
SColumnInfoData* pDstCol = taosArrayGet(pDst->pDataBlock, i); SColumnInfoData* pDstCol = taosArrayGet(pDst->pDataBlock, i);
@ -541,7 +511,7 @@ int32_t blockDataToBuf(char* buf, const SSDataBlock* pBlock) {
// write the number of rows // write the number of rows
*(uint32_t*)buf = pBlock->info.rows; *(uint32_t*)buf = pBlock->info.rows;
int32_t numOfCols = pBlock->info.numOfCols; size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
int32_t numOfRows = pBlock->info.rows; int32_t numOfRows = pBlock->info.rows;
char* pStart = buf + sizeof(uint32_t); char* pStart = buf + sizeof(uint32_t);
@ -571,7 +541,7 @@ int32_t blockDataToBuf(char* buf, const SSDataBlock* pBlock) {
int32_t blockDataFromBuf(SSDataBlock* pBlock, const char* buf) { int32_t blockDataFromBuf(SSDataBlock* pBlock, const char* buf) {
pBlock->info.rows = *(int32_t*)buf; pBlock->info.rows = *(int32_t*)buf;
int32_t numOfCols = pBlock->info.numOfCols; size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
const char* pStart = buf + sizeof(uint32_t); const char* pStart = buf + sizeof(uint32_t);
for (int32_t i = 0; i < numOfCols; ++i) { for (int32_t i = 0; i < numOfCols; ++i) {
@ -620,7 +590,8 @@ int32_t blockDataFromBuf1(SSDataBlock* pBlock, const char* buf, size_t capacity)
pBlock->info.rows = *(int32_t*)buf; pBlock->info.rows = *(int32_t*)buf;
pBlock->info.groupId = *(uint64_t*)(buf + sizeof(int32_t)); pBlock->info.groupId = *(uint64_t*)(buf + sizeof(int32_t));
int32_t numOfCols = pBlock->info.numOfCols; size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
const char* pStart = buf + sizeof(uint32_t) + sizeof(uint64_t); const char* pStart = buf + sizeof(uint32_t) + sizeof(uint64_t);
for (int32_t i = 0; i < numOfCols; ++i) { for (int32_t i = 0; i < numOfCols; ++i) {
@ -666,7 +637,7 @@ size_t blockDataGetRowSize(SSDataBlock* pBlock) {
if (pBlock->info.rowSize == 0) { if (pBlock->info.rowSize == 0) {
size_t rowSize = 0; size_t rowSize = 0;
size_t numOfCols = pBlock->info.numOfCols; size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
for (int32_t i = 0; i < numOfCols; ++i) { for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i); SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i);
rowSize += pColInfo->info.bytes; rowSize += pColInfo->info.bytes;
@ -693,7 +664,7 @@ double blockDataGetSerialRowSize(const SSDataBlock* pBlock) {
ASSERT(pBlock != NULL); ASSERT(pBlock != NULL);
double rowSize = 0; double rowSize = 0;
size_t numOfCols = pBlock->info.numOfCols; size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
for (int32_t i = 0; i < numOfCols; ++i) { for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i); SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i);
rowSize += pColInfo->info.bytes; rowSize += pColInfo->info.bytes;
@ -762,7 +733,7 @@ int32_t dataBlockCompar(const void* p1, const void* p2, const void* param) {
static int32_t doAssignOneTuple(SColumnInfoData* pDstCols, int32_t numOfRows, const SSDataBlock* pSrcBlock, static int32_t doAssignOneTuple(SColumnInfoData* pDstCols, int32_t numOfRows, const SSDataBlock* pSrcBlock,
int32_t tupleIndex) { int32_t tupleIndex) {
int32_t code = 0; int32_t code = 0;
int32_t numOfCols = pSrcBlock->info.numOfCols; size_t numOfCols = taosArrayGetSize(pSrcBlock->pDataBlock);
for (int32_t i = 0; i < numOfCols; ++i) { for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pDst = &pDstCols[i]; SColumnInfoData* pDst = &pDstCols[i];
@ -785,7 +756,7 @@ static int32_t doAssignOneTuple(SColumnInfoData* pDstCols, int32_t numOfRows, co
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static int32_t blockDataAssign(SColumnInfoData* pCols, const SSDataBlock* pDataBlock, int32_t* index) { static int32_t blockDataAssign(SColumnInfoData* pCols, const SSDataBlock* pDataBlock, const int32_t* index) {
#if 0 #if 0
for (int32_t i = 0; i < pDataBlock->info.rows; ++i) { for (int32_t i = 0; i < pDataBlock->info.rows; ++i) {
int32_t code = doAssignOneTuple(pCols, i, pDataBlock, index[i]); int32_t code = doAssignOneTuple(pCols, i, pDataBlock, index[i]);
@ -794,7 +765,8 @@ static int32_t blockDataAssign(SColumnInfoData* pCols, const SSDataBlock* pDataB
} }
} }
#else #else
for (int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) { size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock);
for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pDst = &pCols[i]; SColumnInfoData* pDst = &pCols[i];
SColumnInfoData* pSrc = taosArrayGet(pDataBlock->pDataBlock, i); SColumnInfoData* pSrc = taosArrayGet(pDataBlock->pDataBlock, i);
@ -821,7 +793,7 @@ static int32_t blockDataAssign(SColumnInfoData* pCols, const SSDataBlock* pDataB
static SColumnInfoData* createHelpColInfoData(const SSDataBlock* pDataBlock) { static SColumnInfoData* createHelpColInfoData(const SSDataBlock* pDataBlock) {
int32_t rows = pDataBlock->info.rows; int32_t rows = pDataBlock->info.rows;
int32_t numOfCols = pDataBlock->info.numOfCols; size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock);
SColumnInfoData* pCols = taosMemoryCalloc(numOfCols, sizeof(SColumnInfoData)); SColumnInfoData* pCols = taosMemoryCalloc(numOfCols, sizeof(SColumnInfoData));
if (pCols == NULL) { if (pCols == NULL) {
@ -848,7 +820,7 @@ static SColumnInfoData* createHelpColInfoData(const SSDataBlock* pDataBlock) {
} }
static void copyBackToBlock(SSDataBlock* pDataBlock, SColumnInfoData* pCols) { static void copyBackToBlock(SSDataBlock* pDataBlock, SColumnInfoData* pCols) {
int32_t numOfCols = pDataBlock->info.numOfCols; size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock);
for (int32_t i = 0; i < numOfCols; ++i) { for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, i); SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, i);
@ -909,8 +881,10 @@ int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo) {
} }
} }
size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock);
if (taosArrayGetSize(pOrderInfo) == 1 && (!sortColumnHasNull)) { if (taosArrayGetSize(pOrderInfo) == 1 && (!sortColumnHasNull)) {
if (pDataBlock->info.numOfCols == 1) { if (numOfCols == 1) {
if (!varTypeSort) { if (!varTypeSort) {
SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, 0); SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, 0);
SBlockOrderInfo* pOrder = taosArrayGet(pOrderInfo, 0); SBlockOrderInfo* pOrder = taosArrayGet(pOrderInfo, 0);
@ -926,7 +900,7 @@ int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} else { // var data type } else { // var data type
} }
} else if (pDataBlock->info.numOfCols == 2) { } else if (numOfCols == 2) {
} }
} }
@ -1120,34 +1094,39 @@ int32_t blockDataSort_rv(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullF
void blockDataCleanup(SSDataBlock* pDataBlock) { void blockDataCleanup(SSDataBlock* pDataBlock) {
pDataBlock->info.rows = 0; pDataBlock->info.rows = 0;
for (int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) { size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock);
for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i); SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i);
colInfoDataCleanup(p, pDataBlock->info.capacity); colInfoDataCleanup(p, pDataBlock->info.capacity);
} }
} }
int32_t colInfoDataEnsureCapacity(SColumnInfoData* pColumn, size_t existRows, uint32_t numOfRows) { static int32_t doEnsureCapacity(SColumnInfoData* pColumn, const SDataBlockInfo* pBlockInfo, uint32_t numOfRows) {
ASSERT(numOfRows); ASSERT(numOfRows > 0 && pBlockInfo->capacity >= pBlockInfo->rows);
if (0 == numOfRows || numOfRows <= existRows) { if (numOfRows < pBlockInfo->capacity) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
// todo temp disable it
// ASSERT(pColumn->info.bytes != 0);
int32_t existedRows = pBlockInfo->rows;
if (IS_VAR_DATA_TYPE(pColumn->info.type)) { if (IS_VAR_DATA_TYPE(pColumn->info.type)) {
char* tmp = taosMemoryRealloc(pColumn->varmeta.offset, sizeof(int32_t) * numOfRows); char* tmp = taosMemoryRealloc(pColumn->varmeta.offset, sizeof(int32_t) * numOfRows);
if (tmp == NULL) { if (tmp == NULL) {
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
} }
pColumn->varmeta.offset = (int32_t*)tmp; pColumn->varmeta.offset = (int32_t*)tmp;
memset(&pColumn->varmeta.offset[existRows], 0, sizeof(int32_t) * (numOfRows - existRows)); memset(&pColumn->varmeta.offset[existedRows], 0, sizeof(int32_t) * (numOfRows - existedRows));
} else { } else {
char* tmp = taosMemoryRealloc(pColumn->nullbitmap, BitmapLen(numOfRows)); char* tmp = taosMemoryRealloc(pColumn->nullbitmap, BitmapLen(numOfRows));
if (tmp == NULL) { if (tmp == NULL) {
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
} }
int32_t oldLen = BitmapLen(existRows); int32_t oldLen = BitmapLen(existedRows);
pColumn->nullbitmap = tmp; pColumn->nullbitmap = tmp;
memset(&pColumn->nullbitmap[oldLen], 0, BitmapLen(numOfRows) - oldLen); memset(&pColumn->nullbitmap[oldLen], 0, BitmapLen(numOfRows) - oldLen);
@ -1160,8 +1139,8 @@ int32_t colInfoDataEnsureCapacity(SColumnInfoData* pColumn, size_t existRows, ui
if (tmp == NULL) { if (tmp == NULL) {
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
} }
memset(tmp + pColumn->info.bytes * existRows, 0, pColumn->info.bytes * (numOfRows - existRows));
memset(tmp + pColumn->info.bytes * existedRows, 0, pColumn->info.bytes * (numOfRows - existedRows));
pColumn->pData = tmp; pColumn->pData = tmp;
} }
@ -1178,6 +1157,11 @@ void colInfoDataCleanup(SColumnInfoData* pColumn, uint32_t numOfRows) {
} }
} }
int32_t colInfoDataEnsureCapacity(SColumnInfoData* pColumn, uint32_t numOfRows) {
SDataBlockInfo info = {0};
return doEnsureCapacity(pColumn, &info, numOfRows);
}
int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows) { int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows) {
int32_t code = 0; int32_t code = 0;
ASSERT(numOfRows > 0); ASSERT(numOfRows > 0);
@ -1186,11 +1170,14 @@ int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
pDataBlock->info.capacity = numOfRows; if (pDataBlock->info.capacity < numOfRows) {
pDataBlock->info.capacity = numOfRows;
}
for (int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) { size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock);
for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i); SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i);
code = colInfoDataEnsureCapacity(p, pDataBlock->info.rows, numOfRows); code = doEnsureCapacity(p, &pDataBlock->info, numOfRows);
if (code) { if (code) {
return code; return code;
} }
@ -1208,46 +1195,74 @@ void* blockDataDestroy(SSDataBlock* pBlock) {
taosMemoryFreeClear(pBlock); taosMemoryFreeClear(pBlock);
return NULL; return NULL;
} }
int32_t assignOneDataBlock(SSDataBlock* dst, const SSDataBlock* src) {
ASSERT(src != NULL);
dst->info = src->info;
dst->info.rows = 0;
dst->info.capacity = 0;
size_t numOfCols = taosArrayGetSize(src->pDataBlock);
for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* p = taosArrayGet(src->pDataBlock, i);
SColumnInfoData colInfo = {.hasNull = true, .info = p->info};
blockDataAppendColInfo(dst, &colInfo);
}
int32_t code = blockDataEnsureCapacity(dst, src->info.rows);
if (code != TSDB_CODE_SUCCESS) {
terrno = code;
return -1;
}
for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pDst = taosArrayGet(dst->pDataBlock, i);
SColumnInfoData* pSrc = taosArrayGet(src->pDataBlock, i);
if (pSrc->pData == NULL) {
continue;
}
colDataAssign(pDst, pSrc, src->info.rows, &src->info);
}
dst->info.rows = src->info.rows;
dst->info.capacity = src->info.rows;
return 0;
}
SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData) { SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData) {
if (pDataBlock == NULL) { if (pDataBlock == NULL) {
return NULL; return NULL;
} }
int32_t numOfCols = pDataBlock->info.numOfCols; SSDataBlock* pBlock = createDataBlock();
pBlock->info = pDataBlock->info;
SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); pBlock->info.rows = 0;
pBlock->pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); pBlock->info.capacity = 0;
pBlock->info.numOfCols = numOfCols;
pBlock->info.hasVarCol = pDataBlock->info.hasVarCol;
pBlock->info.rowSize = pDataBlock->info.rowSize;
pBlock->info.groupId = pDataBlock->info.groupId;
pBlock->info.childId = pDataBlock->info.childId;
pBlock->info.type = pDataBlock->info.type;
size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock);
for (int32_t i = 0; i < numOfCols; ++i) { for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData colInfo = {0};
SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i); SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i);
colInfo.info = p->info; SColumnInfoData colInfo = {.hasNull = true, .info = p->info};
colInfo.hasNull = true; blockDataAppendColInfo(pBlock, &colInfo);
taosArrayPush(pBlock->pDataBlock, &colInfo);
} }
if (copyData) { if (copyData) {
int32_t code = blockDataEnsureCapacity(pBlock, pDataBlock->info.rows);
if (code != TSDB_CODE_SUCCESS) {
terrno = code;
blockDataDestroy(pBlock);
return NULL;
}
for (int32_t i = 0; i < numOfCols; ++i) { for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pDst = taosArrayGet(pBlock->pDataBlock, i); SColumnInfoData* pDst = taosArrayGet(pBlock->pDataBlock, i);
SColumnInfoData* pSrc = taosArrayGet(pDataBlock->pDataBlock, i); SColumnInfoData* pSrc = taosArrayGet(pDataBlock->pDataBlock, i);
int32_t code = colInfoDataEnsureCapacity(pDst, 0, pDataBlock->info.rows);
if (code != TSDB_CODE_SUCCESS) {
return NULL;
}
if (pSrc->pData == NULL) { if (pSrc->pData == NULL) {
continue; continue;
} }
colDataAssign(pDst, pSrc, pDataBlock->info.rows);
colDataAssign(pDst, pSrc, pDataBlock->info.rows, &pDataBlock->info);
} }
pBlock->info.rows = pDataBlock->info.rows; pBlock->info.rows = pDataBlock->info.rows;
@ -1257,16 +1272,75 @@ SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData) {
return pBlock; return pBlock;
} }
SSDataBlock* createDataBlock() {
SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock));
if (pBlock == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
}
pBlock->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData));
if (pBlock->pDataBlock == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
taosMemoryFree(pBlock);
}
return pBlock;
}
int32_t blockDataAppendColInfo(SSDataBlock* pBlock, SColumnInfoData* pColInfoData) {
ASSERT(pBlock != NULL && pColInfoData != NULL);
if (pBlock->pDataBlock == NULL) {
pBlock->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData));
if (pBlock->pDataBlock == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return terrno;
}
}
void* p = taosArrayPush(pBlock->pDataBlock, pColInfoData);
if (p == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return terrno;
}
// todo disable it temporarily
// ASSERT(pColInfoData->info.type != 0);
if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) {
pBlock->info.hasVarCol = true;
}
pBlock->info.rowSize += pColInfoData->info.bytes;
return TSDB_CODE_SUCCESS;
}
SColumnInfoData createColumnInfoData(int16_t type, int32_t bytes, int16_t colId) {
SColumnInfoData col = {.hasNull = true};
col.info.colId = colId;
col.info.type = type;
col.info.bytes = bytes;
return col;
}
SColumnInfoData* bdGetColumnInfoData(SSDataBlock* pBlock, int32_t index) {
ASSERT(pBlock != NULL);
if (index >= taosArrayGetSize(pBlock->pDataBlock)) {
return NULL;
}
return taosArrayGet(pBlock->pDataBlock, index);
}
size_t blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize) { size_t blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize) {
int32_t payloadSize = pageSize - blockDataGetSerialMetaSize(pBlock->info.numOfCols); size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
int32_t payloadSize = pageSize - blockDataGetSerialMetaSize(numOfCols);
int32_t rowSize = pBlock->info.rowSize; int32_t rowSize = pBlock->info.rowSize;
int32_t nRows = payloadSize / rowSize; int32_t nRows = payloadSize / rowSize;
// the true value must be less than the value of nRows // the true value must be less than the value of nRows
int32_t additional = 0; int32_t additional = 0;
for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, i); SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, i);
if (IS_VAR_DATA_TYPE(pCol->info.type)) { if (IS_VAR_DATA_TYPE(pCol->info.type)) {
additional += nRows * sizeof(int32_t); additional += nRows * sizeof(int32_t);
@ -1350,7 +1424,8 @@ int32_t blockDataTrimFirstNRows(SSDataBlock* pBlock, size_t n) {
if (pBlock->info.rows <= n) { if (pBlock->info.rows <= n) {
blockDataCleanup(pBlock); blockDataCleanup(pBlock);
} else { } else {
for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i);
colDataTrimFirstNRows(pColInfoData, n, pBlock->info.rows); colDataTrimFirstNRows(pColInfoData, n, pBlock->info.rows);
} }
@ -1362,7 +1437,7 @@ int32_t blockDataTrimFirstNRows(SSDataBlock* pBlock, size_t n) {
int32_t tEncodeDataBlock(void** buf, const SSDataBlock* pBlock) { int32_t tEncodeDataBlock(void** buf, const SSDataBlock* pBlock) {
int64_t tbUid = pBlock->info.uid; int64_t tbUid = pBlock->info.uid;
int16_t numOfCols = pBlock->info.numOfCols; int16_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
int16_t hasVarCol = pBlock->info.hasVarCol; int16_t hasVarCol = pBlock->info.hasVarCol;
int32_t rows = pBlock->info.rows; int32_t rows = pBlock->info.rows;
int32_t sz = taosArrayGetSize(pBlock->pDataBlock); int32_t sz = taosArrayGetSize(pBlock->pDataBlock);
@ -1397,8 +1472,10 @@ int32_t tEncodeDataBlock(void** buf, const SSDataBlock* pBlock) {
void* tDecodeDataBlock(const void* buf, SSDataBlock* pBlock) { void* tDecodeDataBlock(const void* buf, SSDataBlock* pBlock) {
int32_t sz; int32_t sz;
buf = taosDecodeFixedI64(buf, &pBlock->info.uid); int16_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
buf = taosDecodeFixedI16(buf, &pBlock->info.numOfCols);
buf = taosDecodeFixedU64(buf, &pBlock->info.uid);
buf = taosDecodeFixedI16(buf, &numOfCols);
buf = taosDecodeFixedI16(buf, &pBlock->info.hasVarCol); buf = taosDecodeFixedI16(buf, &pBlock->info.hasVarCol);
buf = taosDecodeFixedI32(buf, &pBlock->info.rows); buf = taosDecodeFixedI32(buf, &pBlock->info.rows);
buf = taosDecodeFixedI32(buf, &sz); buf = taosDecodeFixedI32(buf, &sz);
@ -1507,12 +1584,13 @@ void blockDebugShowData(const SArray* dataBlocks, const char* flag) {
int32_t sz = taosArrayGetSize(dataBlocks); int32_t sz = taosArrayGetSize(dataBlocks);
for (int32_t i = 0; i < sz; i++) { for (int32_t i = 0; i < sz; i++) {
SSDataBlock* pDataBlock = taosArrayGet(dataBlocks, i); SSDataBlock* pDataBlock = taosArrayGet(dataBlocks, i);
int32_t colNum = pDataBlock->info.numOfCols; size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock);
int32_t rows = pDataBlock->info.rows;
int32_t rows = pDataBlock->info.rows;
printf("%s |block type %d |child id %d|\n", flag, (int32_t)pDataBlock->info.type, pDataBlock->info.childId); printf("%s |block type %d |child id %d|\n", flag, (int32_t)pDataBlock->info.type, pDataBlock->info.childId);
for (int32_t j = 0; j < rows; j++) { for (int32_t j = 0; j < rows; j++) {
printf("%s |", flag); printf("%s |", flag);
for (int32_t k = 0; k < colNum; k++) { for (int32_t k = 0; k < numOfCols; k++) {
SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, k); SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, k);
void* var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes); void* var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes);
if (colDataIsNull(pColInfoData, rows, j, NULL)) { if (colDataIsNull(pColInfoData, rows, j, NULL)) {
@ -1565,7 +1643,9 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks
int32_t bufSize = sizeof(SSubmitReq); int32_t bufSize = sizeof(SSubmitReq);
for (int32_t i = 0; i < sz; ++i) { for (int32_t i = 0; i < sz; ++i) {
SDataBlockInfo* pBlkInfo = &((SSDataBlock*)taosArrayGet(pDataBlocks, i))->info; SDataBlockInfo* pBlkInfo = &((SSDataBlock*)taosArrayGet(pDataBlocks, i))->info;
bufSize += pBlkInfo->rows * (TD_ROW_HEAD_LEN + pBlkInfo->rowSize + BitmapLen(pBlkInfo->numOfCols));
int32_t numOfCols = taosArrayGetSize(pDataBlocks);
bufSize += pBlkInfo->rows * (TD_ROW_HEAD_LEN + pBlkInfo->rowSize + BitmapLen(numOfCols));
bufSize += sizeof(SSubmitBlk); bufSize += sizeof(SSubmitBlk);
} }
@ -1583,10 +1663,10 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks
for (int32_t i = 0; i < sz; ++i) { for (int32_t i = 0; i < sz; ++i) {
SSDataBlock* pDataBlock = taosArrayGet(pDataBlocks, i); SSDataBlock* pDataBlock = taosArrayGet(pDataBlocks, i);
int32_t colNum = pDataBlock->info.numOfCols; int32_t colNum = taosArrayGetSize(pDataBlock->pDataBlock);
int32_t rows = pDataBlock->info.rows; int32_t rows = pDataBlock->info.rows;
int32_t rowSize = pDataBlock->info.rowSize; // int32_t rowSize = pDataBlock->info.rowSize;
int64_t groupId = pDataBlock->info.groupId; // int64_t groupId = pDataBlock->info.groupId;
if (colNum <= 1) { if (colNum <= 1) {
// invalid if only with TS col // invalid if only with TS col
@ -1804,8 +1884,6 @@ void blockCompressEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen
} }
const char* blockCompressDecode(SSDataBlock* pBlock, int32_t numOfCols, int32_t numOfRows, const char* pData) { const char* blockCompressDecode(SSDataBlock* pBlock, int32_t numOfCols, int32_t numOfRows, const char* pData) {
blockDataEnsureCapacity(pBlock, numOfRows);
const char* pStart = pData; const char* pStart = pData;
int32_t dataLen = *(int32_t*)pStart; int32_t dataLen = *(int32_t*)pStart;
@ -1819,9 +1897,6 @@ const char* blockCompressDecode(SSDataBlock* pBlock, int32_t numOfCols, int32_t
taosArraySetSize(pBlock->pDataBlock, numOfCols); taosArraySetSize(pBlock->pDataBlock, numOfCols);
} }
pBlock->info.numOfCols = taosArrayGetSize(pBlock->pDataBlock);
ASSERT(pBlock->pDataBlock->size >= numOfCols);
for (int32_t i = 0; i < numOfCols; ++i) { for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i);
pColInfoData->info.type = *(int16_t*)pStart; pColInfoData->info.type = *(int16_t*)pStart;
@ -1846,28 +1921,26 @@ const char* blockCompressDecode(SSDataBlock* pBlock, int32_t numOfCols, int32_t
SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i);
if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) {
pColInfoData->varmeta.length = colLen[i];
pColInfoData->varmeta.allocLen = colLen[i];
memcpy(pColInfoData->varmeta.offset, pStart, sizeof(int32_t) * numOfRows); memcpy(pColInfoData->varmeta.offset, pStart, sizeof(int32_t) * numOfRows);
pStart += sizeof(int32_t) * numOfRows; pStart += sizeof(int32_t) * numOfRows;
if (colLen[i] > 0) { if (colLen[i] > 0 && pColInfoData->varmeta.allocLen < colLen[i]) {
taosMemoryFreeClear(pColInfoData->pData); char* tmp = taosMemoryRealloc(pColInfoData->pData, colLen[i]);
pColInfoData->pData = taosMemoryMalloc(colLen[i]); if (tmp == NULL) {
return NULL;
}
pColInfoData->pData = tmp;
pColInfoData->varmeta.allocLen = colLen[i];
} }
pColInfoData->varmeta.length = colLen[i];
} else { } else {
if (pColInfoData->nullbitmap == NULL) {
pColInfoData->nullbitmap = taosMemoryCalloc(1, BitmapLen(numOfRows));
}
memcpy(pColInfoData->nullbitmap, pStart, BitmapLen(numOfRows)); memcpy(pColInfoData->nullbitmap, pStart, BitmapLen(numOfRows));
pStart += BitmapLen(numOfRows); pStart += BitmapLen(numOfRows);
} }
if (colLen[i] > 0) { if (colLen[i] > 0) {
if (pColInfoData->pData == NULL) {
pColInfoData->pData = taosMemoryCalloc(1, colLen[i]);
}
memcpy(pColInfoData->pData, pStart, colLen[i]); memcpy(pColInfoData->pData, pStart, colLen[i]);
} }

6
source/common/src/tglobal.c
View File

@ -760,9 +760,9 @@ void taosCleanupCfg() {
} }
void taosCfgDynamicOptions(const char *option, const char *value) { void taosCfgDynamicOptions(const char *option, const char *value) {
if (strcasecmp(option, "debugFlag") == 0) { if (strncasecmp(option, "debugFlag", 9) == 0) {
int32_t debugFlag = atoi(value); int32_t flag = atoi(value);
taosSetAllDebugFlag(debugFlag); taosSetAllDebugFlag(flag);
} }
if (strcasecmp(option, "resetlog") == 0) { if (strcasecmp(option, "resetlog") == 0) {

27
source/common/src/tmsg.c
View File

@ -1500,6 +1500,33 @@ int32_t tDeserializeSMCfgDnodeReq(void *buf, int32_t bufLen, SMCfgDnodeReq *pReq
return 0; return 0;
} }
int32_t tSerializeSDCfgDnodeReq(void *buf, int32_t bufLen, SDCfgDnodeReq *pReq) {
SEncoder encoder = {0};
tEncoderInit(&encoder, buf, bufLen);
if (tStartEncode(&encoder) < 0) return -1;
if (tEncodeCStr(&encoder, pReq->config) < 0) return -1;
if (tEncodeCStr(&encoder, pReq->value) < 0) return -1;
tEndEncode(&encoder);
int32_t tlen = encoder.pos;
tEncoderClear(&encoder);
return tlen;
}
int32_t tDeserializeSDCfgDnodeReq(void *buf, int32_t bufLen, SDCfgDnodeReq *pReq) {
SDecoder decoder = {0};
tDecoderInit(&decoder, buf, bufLen);
if (tStartDecode(&decoder) < 0) return -1;
if (tDecodeCStrTo(&decoder, pReq->config) < 0) return -1;
if (tDecodeCStrTo(&decoder, pReq->value) < 0) return -1;
tEndDecode(&decoder);
tDecoderClear(&decoder);
return 0;
}
int32_t tSerializeSCreateDnodeReq(void *buf, int32_t bufLen, SCreateDnodeReq *pReq) { int32_t tSerializeSCreateDnodeReq(void *buf, int32_t bufLen, SCreateDnodeReq *pReq) {
SEncoder encoder = {0}; SEncoder encoder = {0};
tEncoderInit(&encoder, buf, bufLen); tEncoderInit(&encoder, buf, bufLen);

46
source/common/test/commonTests.cpp
View File

@ -88,26 +88,15 @@ TEST(testCase, toInteger_test) {
} }
TEST(testCase, Datablock_test) { TEST(testCase, Datablock_test) {
SSDataBlock* b = static_cast<SSDataBlock*>(taosMemoryCalloc(1, sizeof(SSDataBlock))); SSDataBlock* b = createDataBlock();
b->info.numOfCols = 2;
b->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData));
SColumnInfoData infoData = {0}; SColumnInfoData infoData = createColumnInfoData(TSDB_DATA_TYPE_INT, 4, 1);
infoData.info.bytes = 4;
infoData.info.type = TSDB_DATA_TYPE_INT;
infoData.info.colId = 1;
infoData.pData = (char*)taosMemoryCalloc(40, infoData.info.bytes);
infoData.nullbitmap = (char*)taosMemoryCalloc(1, sizeof(char) * (40 / 8));
taosArrayPush(b->pDataBlock, &infoData); taosArrayPush(b->pDataBlock, &infoData);
blockDataAppendColInfo(b, &infoData);
SColumnInfoData infoData1 = {0}; SColumnInfoData infoData1 = createColumnInfoData(TSDB_DATA_TYPE_BINARY, 40, 2);
infoData1.info.bytes = 40; blockDataAppendColInfo(b, &infoData1);
infoData1.info.type = TSDB_DATA_TYPE_BINARY; blockDataEnsureCapacity(b, 40);
infoData1.info.colId = 2;
infoData1.varmeta.offset = (int32_t*)taosMemoryCalloc(40, sizeof(uint32_t));
taosArrayPush(b->pDataBlock, &infoData1);
char* str = "the value of: %d"; char* str = "the value of: %d";
char buf[128] = {0}; char buf[128] = {0};
@ -222,28 +211,17 @@ TEST(testCase, non_var_dataBlock_split_test) {
#endif #endif
TEST(testCase, var_dataBlock_split_test) { TEST(testCase, var_dataBlock_split_test) {
SSDataBlock* b = static_cast<SSDataBlock*>(taosMemoryCalloc(1, sizeof(SSDataBlock)));
b->info.numOfCols = 2;
b->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData));
int32_t numOfRows = 1000000; int32_t numOfRows = 1000000;
SColumnInfoData infoData = {0}; SSDataBlock* b = createDataBlock();
infoData.info.bytes = 4;
infoData.info.type = TSDB_DATA_TYPE_INT;
infoData.info.colId = 1;
infoData.pData = (char*)taosMemoryCalloc(numOfRows, infoData.info.bytes); SColumnInfoData infoData = createColumnInfoData(TSDB_DATA_TYPE_INT, 4, 1);
infoData.nullbitmap = (char*)taosMemoryCalloc(1, sizeof(char) * (numOfRows / 8)); blockDataAppendColInfo(b, &infoData);
taosArrayPush(b->pDataBlock, &infoData);
SColumnInfoData infoData1 = {0}; SColumnInfoData infoData1 = createColumnInfoData(TSDB_DATA_TYPE_BINARY, 40, 2);
infoData1.info.bytes = 40; blockDataAppendColInfo(b, &infoData1);
infoData1.info.type = TSDB_DATA_TYPE_BINARY;
infoData1.info.colId = 2;
infoData1.varmeta.offset = (int32_t*)taosMemoryCalloc(numOfRows, sizeof(uint32_t)); blockDataEnsureCapacity(b, numOfRows);
taosArrayPush(b->pDataBlock, &infoData1);
char buf[41] = {0}; char buf[41] = {0};
char buf1[100] = {0}; char buf1[100] = {0};

23
source/dnode/mgmt/mgmt_dnode/src/dmHandle.c
View File

@ -123,8 +123,15 @@ int32_t dmProcessGrantRsp(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) {
} }
int32_t dmProcessConfigReq(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) { int32_t dmProcessConfigReq(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) {
dError("config req is received, but not supported yet"); SDCfgDnodeReq cfgReq = {0};
return TSDB_CODE_OPS_NOT_SUPPORT; if (tDeserializeSDCfgDnodeReq(pMsg->pCont, pMsg->contLen, &cfgReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG;
return -1;
}
dInfo("start to config, option:%s, value:%s", cfgReq.config, cfgReq.value);
taosCfgDynamicOptions(cfgReq.config, cfgReq.value);
return 0;
} }
static void dmGetServerRunStatus(SDnodeMgmt *pMgmt, SServerStatusRsp *pStatus) { static void dmGetServerRunStatus(SDnodeMgmt *pMgmt, SServerStatusRsp *pStatus) {
@ -203,7 +210,6 @@ SSDataBlock* dmBuildVariablesBlock(void) {
taosArrayPush(pBlock->pDataBlock, &colInfoData); taosArrayPush(pBlock->pDataBlock, &colInfoData);
} }
pBlock->info.numOfCols = pMeta[index].colNum;
pBlock->info.hasVarCol = true; pBlock->info.hasVarCol = true;
return pBlock; return pBlock;
@ -262,8 +268,9 @@ int32_t dmProcessRetrieve(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) {
dmAppendVariablesToBlock(pBlock, pMgmt->pData->dnodeId); dmAppendVariablesToBlock(pBlock, pMgmt->pData->dnodeId);
size = sizeof(SRetrieveMetaTableRsp) + sizeof(int32_t) + sizeof(SSysTableSchema) * pBlock->info.numOfCols + size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
blockDataGetSize(pBlock) + blockDataGetSerialMetaSize(pBlock->info.numOfCols); size = sizeof(SRetrieveMetaTableRsp) + sizeof(int32_t) + sizeof(SSysTableSchema) * numOfCols +
blockDataGetSize(pBlock) + blockDataGetSerialMetaSize(numOfCols);
SRetrieveMetaTableRsp *pRsp = rpcMallocCont(size); SRetrieveMetaTableRsp *pRsp = rpcMallocCont(size);
if (pRsp == NULL) { if (pRsp == NULL) {
@ -274,10 +281,10 @@ int32_t dmProcessRetrieve(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) {
} }
char *pStart = pRsp->data; char *pStart = pRsp->data;
*(int32_t *)pStart = htonl(pBlock->info.numOfCols); *(int32_t *)pStart = htonl(numOfCols);
pStart += sizeof(int32_t); // number of columns pStart += sizeof(int32_t); // number of columns
for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { for (int32_t i = 0; i < numOfCols; ++i) {
SSysTableSchema *pSchema = (SSysTableSchema *)pStart; SSysTableSchema *pSchema = (SSysTableSchema *)pStart;
SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, i); SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, i);
@ -289,7 +296,7 @@ int32_t dmProcessRetrieve(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) {
} }
int32_t len = 0; int32_t len = 0;
blockCompressEncode(pBlock, pStart, &len, pBlock->info.numOfCols, false); blockCompressEncode(pBlock, pStart, &len, numOfCols, false);
pRsp->numOfRows = htonl(pBlock->info.rows); pRsp->numOfRows = htonl(pBlock->info.rows);
pRsp->precision = TSDB_TIME_PRECISION_MILLI; // millisecond time precision pRsp->precision = TSDB_TIME_PRECISION_MILLI; // millisecond time precision

2
source/dnode/mgmt/mgmt_mnode/src/mmHandle.c
View File

@ -17,7 +17,7 @@
#include "mmInt.h" #include "mmInt.h"
void mmGetMonitorInfo(SMnodeMgmt *pMgmt, SMonMmInfo *pInfo) { void mmGetMonitorInfo(SMnodeMgmt *pMgmt, SMonMmInfo *pInfo) {
mndGetMonitorInfo(pMgmt->pMnode, &pInfo->cluster, &pInfo->vgroup, &pInfo->grant); mndGetMonitorInfo(pMgmt->pMnode, &pInfo->cluster, &pInfo->vgroup, &pInfo->stb, &pInfo->grant);
} }
int32_t mmProcessGetMonitorInfoReq(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { int32_t mmProcessGetMonitorInfoReq(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) {

16
source/dnode/mgmt/node_util/src/dmFile.c
View File

@ -127,14 +127,24 @@ TdFilePtr dmCheckRunning(const char *dataDir) {
return NULL; return NULL;
} }
int32_t ret = taosLockFile(pFile); int32_t retryTimes = 0;
if (ret != 0) { int32_t ret = 0;
do {
ret = taosLockFile(pFile);
if (ret == 0) break;
terrno = TAOS_SYSTEM_ERROR(errno);
taosMsleep(100);
retryTimes++;
dError("failed to lock file:%s since %s, retryTimes:%d", filepath, terrstr(), retryTimes);
} while (retryTimes < 120);
if (ret < 0) {
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
dError("failed to lock file:%s since %s", filepath, terrstr());
taosCloseFile(&pFile); taosCloseFile(&pFile);
return NULL; return NULL;
} }
terrno = 0;
dDebug("lock file:%s to prevent repeated starts", filepath); dDebug("lock file:%s to prevent repeated starts", filepath);
return pFile; return pFile;
} }

43
source/dnode/mnode/impl/src/mndDnode.c
View File

@ -502,20 +502,20 @@ _OVER:
} }
static int32_t mndProcessDnodeListReq(SRpcMsg *pReq) { static int32_t mndProcessDnodeListReq(SRpcMsg *pReq) {
SMnode *pMnode = pReq->info.node; SMnode *pMnode = pReq->info.node;
SSdb *pSdb = pMnode->pSdb; SSdb *pSdb = pMnode->pSdb;
SDnodeObj *pObj = NULL; SDnodeObj *pObj = NULL;
void *pIter = NULL; void *pIter = NULL;
SDnodeListRsp rsp = {0}; SDnodeListRsp rsp = {0};
int32_t code = -1; int32_t code = -1;
rsp.dnodeList = taosArrayInit(5, sizeof(SEpSet)); rsp.dnodeList = taosArrayInit(5, sizeof(SEpSet));
if (NULL == rsp.dnodeList) { if (NULL == rsp.dnodeList) {
mError("failed to alloc epSet while process dnode list req"); mError("failed to alloc epSet while process dnode list req");
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER; goto _OVER;
} }
while (1) { while (1) {
pIter = sdbFetch(pSdb, SDB_DNODE, pIter, (void **)&pObj); pIter = sdbFetch(pSdb, SDB_DNODE, pIter, (void **)&pObj);
if (pIter == NULL) break; if (pIter == NULL) break;
@ -554,7 +554,6 @@ _OVER:
return code; return code;
} }
static int32_t mndProcessCreateDnodeReq(SRpcMsg *pReq) { static int32_t mndProcessCreateDnodeReq(SRpcMsg *pReq) {
SMnode *pMnode = pReq->info.node; SMnode *pMnode = pReq->info.node;
int32_t code = -1; int32_t code = -1;
@ -732,15 +731,37 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) {
mError("dnode:%d, failed to config since %s ", cfgReq.dnodeId, terrstr()); mError("dnode:%d, failed to config since %s ", cfgReq.dnodeId, terrstr());
return -1; return -1;
} }
SEpSet epSet = mndGetDnodeEpset(pDnode); SEpSet epSet = mndGetDnodeEpset(pDnode);
mndReleaseDnode(pMnode, pDnode); mndReleaseDnode(pMnode, pDnode);
int32_t bufLen = tSerializeSMCfgDnodeReq(NULL, 0, &cfgReq); SDCfgDnodeReq dcfgReq = {0};
if (strncasecmp(cfgReq.config, "debugFlag", 9) == 0) {
const char *value = cfgReq.value;
int32_t flag = atoi(value);
if (flag <= 0) {
flag = atoi(cfgReq.config + 10);
}
if (flag <= 0 || flag > 255) {
mError("dnode:%d, failed to config debugFlag since value:%d", cfgReq.dnodeId, flag);
terrno = TSDB_CODE_INVALID_CFG;
return -1;
}
strcpy(dcfgReq.config, "debugFlag");
snprintf(dcfgReq.value, TSDB_DNODE_VALUE_LEN, "%d", flag);
} else if (strcasecmp(cfgReq.config, "resetlog") == 0) {
strcpy(dcfgReq.config, "resetlog");
} else {
terrno = TSDB_CODE_INVALID_CFG;
mError("dnode:%d, failed to config since %s", cfgReq.dnodeId, terrstr());
return -1;
}
int32_t bufLen = tSerializeSDCfgDnodeReq(NULL, 0, &dcfgReq);
void *pBuf = rpcMallocCont(bufLen); void *pBuf = rpcMallocCont(bufLen);
if (pBuf == NULL) return -1; if (pBuf == NULL) return -1;
tSerializeSMCfgDnodeReq(pBuf, bufLen, &cfgReq); tSerializeSDCfgDnodeReq(pBuf, bufLen, &dcfgReq);
mDebug("dnode:%d, send config req to dnode, app:%p", cfgReq.dnodeId, pReq->info.ahandle); mDebug("dnode:%d, send config req to dnode, app:%p", cfgReq.dnodeId, pReq->info.ahandle);
SRpcMsg rpcMsg = {.msgType = TDMT_DND_CONFIG_DNODE, .pCont = pBuf, .contLen = bufLen}; SRpcMsg rpcMsg = {.msgType = TDMT_DND_CONFIG_DNODE, .pCont = pBuf, .contLen = bufLen};

30
source/dnode/mnode/impl/src/mndMain.c
View File

@ -614,7 +614,7 @@ int64_t mndGenerateUid(char *name, int32_t len) {
} }
int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo, int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo,
SMonGrantInfo *pGrantInfo) { SMonStbInfo *pStbInfo, SMonGrantInfo *pGrantInfo) {
if (mndAcquireRpcRef(pMnode) != 0) return -1; if (mndAcquireRpcRef(pMnode) != 0) return -1;
SSdb *pSdb = pMnode->pSdb; SSdb *pSdb = pMnode->pSdb;
@ -623,7 +623,9 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr
pClusterInfo->dnodes = taosArrayInit(sdbGetSize(pSdb, SDB_DNODE), sizeof(SMonDnodeDesc)); pClusterInfo->dnodes = taosArrayInit(sdbGetSize(pSdb, SDB_DNODE), sizeof(SMonDnodeDesc));
pClusterInfo->mnodes = taosArrayInit(sdbGetSize(pSdb, SDB_MNODE), sizeof(SMonMnodeDesc)); pClusterInfo->mnodes = taosArrayInit(sdbGetSize(pSdb, SDB_MNODE), sizeof(SMonMnodeDesc));
pVgroupInfo->vgroups = taosArrayInit(sdbGetSize(pSdb, SDB_VGROUP), sizeof(SMonVgroupDesc)); pVgroupInfo->vgroups = taosArrayInit(sdbGetSize(pSdb, SDB_VGROUP), sizeof(SMonVgroupDesc));
if (pClusterInfo->dnodes == NULL || pClusterInfo->mnodes == NULL || pVgroupInfo->vgroups == NULL) { pStbInfo->stbs = taosArrayInit(sdbGetSize(pSdb, SDB_STB), sizeof(SMonStbDesc));
if (pClusterInfo->dnodes == NULL || pClusterInfo->mnodes == NULL || pVgroupInfo->vgroups == NULL ||
pStbInfo->stbs == NULL) {
mndReleaseRpcRef(pMnode); mndReleaseRpcRef(pMnode);
return -1; return -1;
} }
@ -632,6 +634,8 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr
tstrncpy(pClusterInfo->version, version, sizeof(pClusterInfo->version)); tstrncpy(pClusterInfo->version, version, sizeof(pClusterInfo->version));
pClusterInfo->monitor_interval = tsMonitorInterval; pClusterInfo->monitor_interval = tsMonitorInterval;
pClusterInfo->connections_total = mndGetNumOfConnections(pMnode); pClusterInfo->connections_total = mndGetNumOfConnections(pMnode);
pClusterInfo->dbs_total = sdbGetSize(pSdb, SDB_DB);
pClusterInfo->stbs_total = sdbGetSize(pSdb, SDB_STB);
void *pIter = NULL; void *pIter = NULL;
while (1) { while (1) {
@ -681,6 +685,7 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr
if (pIter == NULL) break; if (pIter == NULL) break;
pClusterInfo->vgroups_total++; pClusterInfo->vgroups_total++;
pClusterInfo->tbs_total += pVgroup->numOfTables;
SMonVgroupDesc desc = {0}; SMonVgroupDesc desc = {0};
desc.vgroup_id = pVgroup->vgId; desc.vgroup_id = pVgroup->vgId;
@ -711,6 +716,27 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr
sdbRelease(pSdb, pVgroup); sdbRelease(pSdb, pVgroup);
} }
// stb info
pIter = NULL;
while (1) {
SStbObj *pStb = NULL;
pIter = sdbFetch(pSdb, SDB_STB, pIter, (void **)&pStb);
if (pIter == NULL) break;
SMonStbDesc desc = {0};
SName name1 = {0};
tNameFromString(&name1, pStb->db, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
tNameGetDbName(&name1, desc.database_name);
SName name2 = {0};
tNameFromString(&name2, pStb->name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
tstrncpy(desc.stb_name, tNameGetTableName(&name2), TSDB_TABLE_NAME_LEN);
taosArrayPush(pStbInfo->stbs, &desc);
sdbRelease(pSdb, pStb);
}
// grant info // grant info
pGrantInfo->expire_time = (pMnode->grant.expireTimeMS - ms) / 86400000.0f; pGrantInfo->expire_time = (pMnode->grant.expireTimeMS - ms) / 86400000.0f;
pGrantInfo->timeseries_total = pMnode->grant.timeseriesAllowed; pGrantInfo->timeseries_total = pMnode->grant.timeseriesAllowed;

3
source/dnode/mnode/impl/src/mndShow.c
View File

@ -236,7 +236,6 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) {
int32_t numOfCols = pShow->pMeta->numOfColumns; int32_t numOfCols = pShow->pMeta->numOfColumns;
SSDataBlock *pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); SSDataBlock *pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock));
pBlock->pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); pBlock->pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData));
pBlock->info.numOfCols = numOfCols;
for (int32_t i = 0; i < numOfCols; ++i) { for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData idata = {0}; SColumnInfoData idata = {0};
@ -271,7 +270,7 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) {
} }
size = sizeof(SRetrieveMetaTableRsp) + sizeof(int32_t) + sizeof(SSysTableSchema) * pShow->pMeta->numOfColumns + size = sizeof(SRetrieveMetaTableRsp) + sizeof(int32_t) + sizeof(SSysTableSchema) * pShow->pMeta->numOfColumns +
blockDataGetSize(pBlock) + blockDataGetSerialMetaSize(pBlock->info.numOfCols); blockDataGetSize(pBlock) + blockDataGetSerialMetaSize(taosArrayGetSize(pBlock->pDataBlock));
SRetrieveMetaTableRsp *pRsp = rpcMallocCont(size); SRetrieveMetaTableRsp *pRsp = rpcMallocCont(size);
if (pRsp == NULL) { if (pRsp == NULL) {

4
source/dnode/vnode/inc/vnode.h
View File

@ -149,8 +149,8 @@ int32_t tqReadHandleRemoveTbUidList(STqReadHandle *pHandle, const SArray *tbUidL
int32_t tqReadHandleSetMsg(STqReadHandle *pHandle, SSubmitReq *pMsg, int64_t ver); int32_t tqReadHandleSetMsg(STqReadHandle *pHandle, SSubmitReq *pMsg, int64_t ver);
bool tqNextDataBlock(STqReadHandle *pHandle); bool tqNextDataBlock(STqReadHandle *pHandle);
bool tqNextDataBlockFilterOut(STqReadHandle *pHandle, SHashObj *filterOutUids); bool tqNextDataBlockFilterOut(STqReadHandle *pHandle, SHashObj *filterOutUids);
int32_t tqRetrieveDataBlock(SArray **ppCols, STqReadHandle *pHandle, uint64_t *pGroupId, uint64_t *pUid, int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, STqReadHandle *pHandle, uint64_t *pGroupId, uint64_t *pUid,
int32_t *pNumOfRows, int16_t *pNumOfCols); int32_t *pNumOfRows);
// sma // sma
int32_t smaGetTSmaDays(SVnodeCfg *pCfg, void *pCont, uint32_t contLen, int32_t *days); int32_t smaGetTSmaDays(SVnodeCfg *pCfg, void *pCont, uint32_t contLen, int32_t *days);

2
source/dnode/vnode/src/tq/tq.c
View File

@ -306,7 +306,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
pHead->msgType == TDMT_VND_DROP_STB || pHead->msgType == TDMT_VND_CREATE_TABLE || pHead->msgType == TDMT_VND_DROP_STB || pHead->msgType == TDMT_VND_CREATE_TABLE ||
pHead->msgType == TDMT_VND_ALTER_TABLE || pHead->msgType == TDMT_VND_DROP_TABLE || pHead->msgType == TDMT_VND_ALTER_TABLE || pHead->msgType == TDMT_VND_DROP_TABLE ||
pHead->msgType == TDMT_VND_DROP_TTL_TABLE); pHead->msgType == TDMT_VND_DROP_TTL_TABLE);
// return tqInfo("fetch meta msg, ver: %ld, type: %d", pHead->version, pHead->msgType);
SMqMetaRsp metaRsp = {0}; SMqMetaRsp metaRsp = {0};
metaRsp.reqOffset = pReq->currentOffset; metaRsp.reqOffset = pReq->currentOffset;
metaRsp.rspOffset = fetchOffset; metaRsp.rspOffset = fetchOffset;

12
source/dnode/vnode/src/tq/tqExec.c
View File

@ -29,7 +29,7 @@ static int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataBlkRsp* pRs
// TODO enable compress // TODO enable compress
int32_t actualLen = 0; int32_t actualLen = 0;
blockCompressEncode(pBlock, pRetrieve->data, &actualLen, pBlock->info.numOfCols, false); blockCompressEncode(pBlock, pRetrieve->data, &actualLen, taosArrayGetSize(pBlock->pDataBlock), false);
actualLen += sizeof(SRetrieveTableRsp); actualLen += sizeof(SRetrieveTableRsp);
ASSERT(actualLen <= dataStrLen); ASSERT(actualLen <= dataStrLen);
taosArrayPush(pRsp->blockDataLen, &actualLen); taosArrayPush(pRsp->blockDataLen, &actualLen);
@ -66,6 +66,7 @@ int32_t tqScanSnapshot(STQ* pTq, const STqExecHandle* pExec, SMqDataBlkRsp* pRsp
if (qStreamScanSnapshot(task) < 0) { if (qStreamScanSnapshot(task) < 0) {
ASSERT(0); ASSERT(0);
} }
// set version
while (1) { while (1) {
SSDataBlock* pDataBlock = NULL; SSDataBlock* pDataBlock = NULL;
uint64_t ts = 0; uint64_t ts = 0;
@ -75,7 +76,7 @@ int32_t tqScanSnapshot(STQ* pTq, const STqExecHandle* pExec, SMqDataBlkRsp* pRsp
if (pDataBlock == NULL) break; if (pDataBlock == NULL) break;
ASSERT(pDataBlock->info.rows != 0); ASSERT(pDataBlock->info.rows != 0);
ASSERT(pDataBlock->info.numOfCols != 0); ASSERT(taosArrayGetSize(pDataBlock->pDataBlock) != 0);
tqAddBlockDataToRsp(pDataBlock, pRsp); tqAddBlockDataToRsp(pDataBlock, pRsp);
pRsp->blockNum++; pRsp->blockNum++;
@ -98,7 +99,6 @@ int32_t tqDataExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataBlkR
if (pDataBlock == NULL) break; if (pDataBlock == NULL) break;
ASSERT(pDataBlock->info.rows != 0); ASSERT(pDataBlock->info.rows != 0);
ASSERT(pDataBlock->info.numOfCols != 0);
tqAddBlockDataToRsp(pDataBlock, pRsp); tqAddBlockDataToRsp(pDataBlock, pRsp);
if (pRsp->withTbName) { if (pRsp->withTbName) {
@ -112,8 +112,7 @@ int32_t tqDataExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataBlkR
tqReadHandleSetMsg(pReader, pReq, 0); tqReadHandleSetMsg(pReader, pReq, 0);
while (tqNextDataBlock(pReader)) { while (tqNextDataBlock(pReader)) {
SSDataBlock block = {0}; SSDataBlock block = {0};
if (tqRetrieveDataBlock(&block.pDataBlock, pReader, &block.info.groupId, &block.info.uid, &block.info.rows, if (tqRetrieveDataBlock(&block, pReader, &block.info.groupId, &block.info.uid, &block.info.rows) < 0) {
&block.info.numOfCols) < 0) {
if (terrno == TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND) continue; if (terrno == TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND) continue;
ASSERT(0); ASSERT(0);
} }
@ -130,8 +129,7 @@ int32_t tqDataExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataBlkR
tqReadHandleSetMsg(pReader, pReq, 0); tqReadHandleSetMsg(pReader, pReq, 0);
while (tqNextDataBlockFilterOut(pReader, pExec->execDb.pFilterOutTbUid)) { while (tqNextDataBlockFilterOut(pReader, pExec->execDb.pFilterOutTbUid)) {
SSDataBlock block = {0}; SSDataBlock block = {0};
if (tqRetrieveDataBlock(&block.pDataBlock, pReader, &block.info.groupId, &block.info.uid, &block.info.rows, if (tqRetrieveDataBlock(&block, pReader, &block.info.groupId, &block.info.uid, &block.info.rows) < 0) {
&block.info.numOfCols) < 0) {
if (terrno == TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND) continue; if (terrno == TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND) continue;
ASSERT(0); ASSERT(0);
} }

4
source/dnode/vnode/src/tq/tqOffset.c
View File

@ -44,6 +44,7 @@ STqOffsetStore* tqOffsetOpen(STQ* pTq) {
} }
char* fname = buildFileName(pStore->pTq->path); char* fname = buildFileName(pStore->pTq->path);
TdFilePtr pFile = taosOpenFile(fname, TD_FILE_READ); TdFilePtr pFile = taosOpenFile(fname, TD_FILE_READ);
taosMemoryFree(fname);
if (pFile != NULL) { if (pFile != NULL) {
STqOffsetHead head = {0}; STqOffsetHead head = {0};
int64_t code; int64_t code;
@ -77,7 +78,6 @@ STqOffsetStore* tqOffsetOpen(STQ* pTq) {
} }
taosCloseFile(&pFile); taosCloseFile(&pFile);
taosMemoryFree(fname);
} }
return pStore; return pStore;
} }
@ -102,6 +102,7 @@ int32_t tqOffsetSnapshot(STqOffsetStore* pStore) {
// TODO file name should be with a version // TODO file name should be with a version
char* fname = buildFileName(pStore->pTq->path); char* fname = buildFileName(pStore->pTq->path);
TdFilePtr pFile = taosOpenFile(fname, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND); TdFilePtr pFile = taosOpenFile(fname, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND);
taosMemoryFree(fname);
if (pFile == NULL) { if (pFile == NULL) {
ASSERT(0); ASSERT(0);
return -1; return -1;
@ -140,6 +141,5 @@ int32_t tqOffsetSnapshot(STqOffsetStore* pStore) {
} }
// close and rename file // close and rename file
taosCloseFile(&pFile); taosCloseFile(&pFile);
taosMemoryFree(fname);
return 0; return 0;
} }

50
source/dnode/vnode/src/tq/tqRead.c
View File

@ -142,8 +142,8 @@ bool tqNextDataBlockFilterOut(STqReadHandle* pHandle, SHashObj* filterOutUids) {
return false; return false;
} }
int32_t tqRetrieveDataBlock(SArray** ppCols, STqReadHandle* pHandle, uint64_t* pGroupId, uint64_t* pUid, int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, STqReadHandle* pHandle, uint64_t* pGroupId, uint64_t* pUid,
int32_t* pNumOfRows, int16_t* pNumOfCols) { int32_t* pNumOfRows) {
*pUid = 0; *pUid = 0;
// TODO set to real sversion // TODO set to real sversion
@ -151,6 +151,7 @@ int32_t tqRetrieveDataBlock(SArray** ppCols, STqReadHandle* pHandle, uint64_t* p
int32_t sversion = htonl(pHandle->pBlock->sversion); int32_t sversion = htonl(pHandle->pBlock->sversion);
if (pHandle->cachedSchemaSuid == 0 || pHandle->cachedSchemaVer != sversion || if (pHandle->cachedSchemaSuid == 0 || pHandle->cachedSchemaVer != sversion ||
pHandle->cachedSchemaSuid != pHandle->msgIter.suid) { pHandle->cachedSchemaSuid != pHandle->msgIter.suid) {
if (pHandle->pSchema) taosMemoryFree(pHandle->pSchema);
pHandle->pSchema = metaGetTbTSchema(pHandle->pVnodeMeta, pHandle->msgIter.uid, sversion); pHandle->pSchema = metaGetTbTSchema(pHandle->pVnodeMeta, pHandle->msgIter.uid, sversion);
if (pHandle->pSchema == NULL) { if (pHandle->pSchema == NULL) {
tqWarn("cannot found tsschema for table: uid: %ld (suid: %ld), version %d, possibly dropped table", tqWarn("cannot found tsschema for table: uid: %ld (suid: %ld), version %d, possibly dropped table",
@ -161,6 +162,7 @@ int32_t tqRetrieveDataBlock(SArray** ppCols, STqReadHandle* pHandle, uint64_t* p
} }
// this interface use suid instead of uid // this interface use suid instead of uid
if (pHandle->pSchemaWrapper) tDeleteSSchemaWrapper(pHandle->pSchemaWrapper);
pHandle->pSchemaWrapper = metaGetTableSchema(pHandle->pVnodeMeta, pHandle->msgIter.uid, sversion, true); pHandle->pSchemaWrapper = metaGetTableSchema(pHandle->pVnodeMeta, pHandle->msgIter.uid, sversion, true);
if (pHandle->pSchemaWrapper == NULL) { if (pHandle->pSchemaWrapper == NULL) {
tqWarn("cannot found schema wrapper for table: suid: %ld, version %d, possibly dropped table", tqWarn("cannot found schema wrapper for table: suid: %ld, version %d, possibly dropped table",
@ -180,23 +182,14 @@ int32_t tqRetrieveDataBlock(SArray** ppCols, STqReadHandle* pHandle, uint64_t* p
int32_t colNumNeed = taosArrayGetSize(pHandle->pColIdList); int32_t colNumNeed = taosArrayGetSize(pHandle->pColIdList);
if (colNumNeed == 0) { if (colNumNeed == 0) {
*ppCols = taosArrayInit(pSchemaWrapper->nCols, sizeof(SColumnInfoData));
if (*ppCols == NULL) {
return -1;
}
int32_t colMeta = 0; int32_t colMeta = 0;
while (colMeta < pSchemaWrapper->nCols) { while (colMeta < pSchemaWrapper->nCols) {
SSchema* pColSchema = &pSchemaWrapper->pSchema[colMeta]; SSchema* pColSchema = &pSchemaWrapper->pSchema[colMeta];
SColumnInfoData colInfo = {0}; SColumnInfoData colInfo = createColumnInfoData(pColSchema->type, pColSchema->bytes, pColSchema->colId);
colInfo.info.bytes = pColSchema->bytes; int32_t code = blockDataAppendColInfo(pBlock, &colInfo);
colInfo.info.colId = pColSchema->colId; if (code != TSDB_CODE_SUCCESS) {
colInfo.info.type = pColSchema->type;
if (colInfoDataEnsureCapacity(&colInfo, 0, *pNumOfRows) < 0) {
goto FAIL; goto FAIL;
} }
taosArrayPush(*ppCols, &colInfo);
colMeta++; colMeta++;
} }
} else { } else {
@ -204,11 +197,6 @@ int32_t tqRetrieveDataBlock(SArray** ppCols, STqReadHandle* pHandle, uint64_t* p
colNumNeed = pSchemaWrapper->nCols; colNumNeed = pSchemaWrapper->nCols;
} }
*ppCols = taosArrayInit(colNumNeed, sizeof(SColumnInfoData));
if (*ppCols == NULL) {
return -1;
}
int32_t colMeta = 0; int32_t colMeta = 0;
int32_t colNeed = 0; int32_t colNeed = 0;
while (colMeta < pSchemaWrapper->nCols && colNeed < colNumNeed) { while (colMeta < pSchemaWrapper->nCols && colNeed < colNumNeed) {
@ -220,23 +208,22 @@ int32_t tqRetrieveDataBlock(SArray** ppCols, STqReadHandle* pHandle, uint64_t* p
} else if (colIdSchema > colIdNeed) { } else if (colIdSchema > colIdNeed) {
colNeed++; colNeed++;
} else { } else {
SColumnInfoData colInfo = {0}; SColumnInfoData colInfo = createColumnInfoData(pColSchema->type, pColSchema->bytes, pColSchema->colId);
colInfo.info.bytes = pColSchema->bytes; int32_t code = blockDataAppendColInfo(pBlock, &colInfo);
colInfo.info.colId = pColSchema->colId; if (code != TSDB_CODE_SUCCESS) {
colInfo.info.type = pColSchema->type;
if (colInfoDataEnsureCapacity(&colInfo, 0, *pNumOfRows) < 0) {
goto FAIL; goto FAIL;
} }
taosArrayPush(*ppCols, &colInfo);
colMeta++; colMeta++;
colNeed++; colNeed++;
} }
} }
} }
int32_t colActual = taosArrayGetSize(*ppCols); if (blockDataEnsureCapacity(pBlock, *pNumOfRows) < 0) {
*pNumOfCols = colActual; goto FAIL;
}
int32_t colActual = blockDataGetNumOfCols(pBlock);
// TODO in stream shuffle case, fetch groupId // TODO in stream shuffle case, fetch groupId
*pGroupId = 0; *pGroupId = 0;
@ -253,7 +240,7 @@ int32_t tqRetrieveDataBlock(SArray** ppCols, STqReadHandle* pHandle, uint64_t* p
tdSTSRowIterReset(&iter, row); tdSTSRowIterReset(&iter, row);
// get all wanted col of that block // get all wanted col of that block
for (int32_t i = 0; i < colActual; i++) { for (int32_t i = 0; i < colActual; i++) {
SColumnInfoData* pColData = taosArrayGet(*ppCols, i); SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, i);
SCellVal sVal = {0}; SCellVal sVal = {0};
if (!tdSTSRowIterNext(&iter, pColData->info.colId, pColData->info.type, &sVal)) { if (!tdSTSRowIterNext(&iter, pColData->info.colId, pColData->info.type, &sVal)) {
break; break;
@ -265,8 +252,9 @@ int32_t tqRetrieveDataBlock(SArray** ppCols, STqReadHandle* pHandle, uint64_t* p
curRow++; curRow++;
} }
return 0; return 0;
FAIL:
if (*ppCols) taosArrayDestroy(*ppCols); FAIL: // todo refactor here
// if (*ppCols) taosArrayDestroy(*ppCols);
return -1; return -1;
} }

4
source/dnode/vnode/src/tq/tqSink.c
View File

@ -44,7 +44,7 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema, boo
createTbReq.ctb.suid = suid; createTbReq.ctb.suid = suid;
STagVal tagVal = { STagVal tagVal = {
.cid = pDataBlock->info.numOfCols + 1, .cid = taosArrayGetSize(pDataBlock->pDataBlock) + 1,
.type = TSDB_DATA_TYPE_UBIGINT, .type = TSDB_DATA_TYPE_UBIGINT,
.i64 = (int64_t)pDataBlock->info.groupId, .i64 = (int64_t)pDataBlock->info.groupId,
}; };
@ -107,7 +107,7 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema, boo
createTbReq.ctb.suid = suid; createTbReq.ctb.suid = suid;
STagVal tagVal = { STagVal tagVal = {
.cid = pDataBlock->info.numOfCols + 1, .cid = taosArrayGetSize(pDataBlock->pDataBlock) + 1,
.type = TSDB_DATA_TYPE_UBIGINT, .type = TSDB_DATA_TYPE_UBIGINT,
.i64 = (int64_t)pDataBlock->info.groupId, .i64 = (int64_t)pDataBlock->info.groupId,
}; };

69
source/dnode/vnode/src/tsdb/tsdbRead.c
View File

@ -16,13 +16,11 @@
#include "tsdb.h" #include "tsdb.h"
#include "vnode.h" #include "vnode.h"
#define EXTRA_BYTES 2
#define ASCENDING_TRAVERSE(o) (o == TSDB_ORDER_ASC) #define ASCENDING_TRAVERSE(o) (o == TSDB_ORDER_ASC)
#define QH_GET_NUM_OF_COLS(handle) ((size_t)(taosArrayGetSize((handle)->pColumns))) #define QH_GET_NUM_OF_COLS(handle) ((size_t)(taosArrayGetSize((handle)->pResBlock->pDataBlock)))
#define GET_FILE_DATA_BLOCK_INFO(_checkInfo, _block) \ #define GET_FILE_DATA_BLOCK_INFO(_checkInfo, _block) \
((SDataBlockInfo){.window = {.skey = (_block)->minKey.ts, .ekey = (_block)->maxKey.ts}, \ ((SDataBlockInfo){.window = {.skey = (_block)->minKey.ts, .ekey = (_block)->maxKey.ts}, \
.numOfCols = (_block)->numOfCols, \
.rows = (_block)->numOfRows, \ .rows = (_block)->numOfRows, \
.uid = (_checkInfo)->tableId}) .uid = (_checkInfo)->tableId})
@ -115,7 +113,8 @@ typedef struct STsdbReadHandle {
// SColumnDataAgg* statis; // query level statistics, only one table block statistics info exists at any time // SColumnDataAgg* statis; // query level statistics, only one table block statistics info exists at any time
// SColumnDataAgg** pstatis;// the ptr array list to return to caller // SColumnDataAgg** pstatis;// the ptr array list to return to caller
int32_t numOfBlocks; int32_t numOfBlocks;
SArray* pColumns; // column list, SColumnInfoData array list SSDataBlock* pResBlock;
// SArray* pColumns; // column list, SColumnInfoData array list
bool locateStart; bool locateStart;
int32_t outputCapacity; int32_t outputCapacity;
int32_t realNumOfRows; int32_t realNumOfRows;
@ -177,7 +176,7 @@ static SArray* getColumnIdList(STsdbReadHandle* pTsdbReadHandle) {
SArray* pIdList = taosArrayInit(numOfCols, sizeof(int16_t)); SArray* pIdList = taosArrayInit(numOfCols, sizeof(int16_t));
for (int32_t i = 0; i < numOfCols; ++i) { for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pCol = taosArrayGet(pTsdbReadHandle->pColumns, i); SColumnInfoData* pCol = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, i);
taosArrayPush(pIdList, &pCol->info.colId); taosArrayPush(pIdList, &pCol->info.colId);
} }
@ -425,23 +424,21 @@ static STsdbReadHandle* tsdbQueryTablesImpl(SVnode* pVnode, SQueryTableDataCond*
} }
// todo: use list instead of array? // todo: use list instead of array?
pReadHandle->pColumns = taosArrayInit(pCond->numOfCols, sizeof(SColumnInfoData)); pReadHandle->pResBlock = createDataBlock();
if (pReadHandle->pColumns == NULL) { if (pReadHandle->pResBlock == NULL) {
goto _end; goto _end;
} }
for (int32_t i = 0; i < pCond->numOfCols; ++i) { for (int32_t i = 0; i < pCond->numOfCols; ++i) {
SColumnInfoData colInfo = {{0}, 0}; SColumnInfoData colInfo = {.info = pCond->colList[i], 0};
colInfo.info = pCond->colList[i]; int32_t code = blockDataAppendColInfo(pReadHandle->pResBlock, &colInfo);
if (code != TSDB_CODE_SUCCESS){
int32_t code = colInfoDataEnsureCapacity(&colInfo, 0, pReadHandle->outputCapacity);
if (code != TSDB_CODE_SUCCESS) {
goto _end; goto _end;
} }
taosArrayPush(pReadHandle->pColumns, &colInfo);
} }
blockDataEnsureCapacity(pReadHandle->pResBlock, pReadHandle->outputCapacity);
pReadHandle->suppInfo.defaultLoadColumn = getDefaultLoadColumns(pReadHandle, true); pReadHandle->suppInfo.defaultLoadColumn = getDefaultLoadColumns(pReadHandle, true);
size_t size = taosArrayGetSize(pReadHandle->suppInfo.defaultLoadColumn); size_t size = taosArrayGetSize(pReadHandle->suppInfo.defaultLoadColumn);
@ -1536,12 +1533,12 @@ static int32_t doCopyRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, int32_t
int32_t trueEnd = ascScan ? end : start; int32_t trueEnd = ascScan ? end : start;
int32_t step = ascScan ? 1 : -1; int32_t step = ascScan ? 1 : -1;
int32_t requiredNumOfCols = (int32_t)taosArrayGetSize(pTsdbReadHandle->pColumns); int32_t requiredNumOfCols = (int32_t)taosArrayGetSize(pTsdbReadHandle->pResBlock->pDataBlock);
// data in buffer has greater timestamp, copy data in file block // data in buffer has greater timestamp, copy data in file block
int32_t i = 0, j = 0; int32_t i = 0, j = 0;
while (i < requiredNumOfCols && j < pCols->numOfCols) { while (i < requiredNumOfCols && j < pCols->numOfCols) {
SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i); SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, i);
SDataCol* src = &pCols->cols[j]; SDataCol* src = &pCols->cols[j];
if (src->colId < pColInfo->info.colId) { if (src->colId < pColInfo->info.colId) {
@ -1592,7 +1589,7 @@ static int32_t doCopyRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, int32_t
} }
while (i < requiredNumOfCols) { // the remain columns are all null data while (i < requiredNumOfCols) { // the remain columns are all null data
SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i); SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, i);
colDataAppendNNULL(pColInfo, numOfRows, num); colDataAppendNNULL(pColInfo, numOfRows, num);
i++; i++;
} }
@ -1674,7 +1671,7 @@ static int32_t mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capa
int32_t i = 0, j = 0, k = 0; int32_t i = 0, j = 0, k = 0;
while (i < numOfCols && (j < numOfColsOfRow1 || k < numOfColsOfRow2)) { while (i < numOfCols && (j < numOfColsOfRow1 || k < numOfColsOfRow2)) {
SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i); SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, i);
int32_t colIdOfRow1; int32_t colIdOfRow1;
if (j >= numOfColsOfRow1) { if (j >= numOfColsOfRow1) {
@ -1805,7 +1802,7 @@ static int32_t mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capa
if (mergeOption == 1) { if (mergeOption == 1) {
while (i < numOfCols) { // the remain columns are all null data while (i < numOfCols) { // the remain columns are all null data
SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i); SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, i);
colDataAppend(pColInfo, *curRow, NULL, true); colDataAppend(pColInfo, *curRow, NULL, true);
++i; ++i;
} }
@ -1861,7 +1858,7 @@ static void doCheckGeneratedBlockRange(STsdbReadHandle* pTsdbReadHandle) {
assert(cur->win.skey >= pTsdbReadHandle->window.ekey && cur->win.ekey <= pTsdbReadHandle->window.skey); assert(cur->win.skey >= pTsdbReadHandle->window.ekey && cur->win.ekey <= pTsdbReadHandle->window.skey);
} }
SColumnInfoData* pColInfoData = taosArrayGet(pTsdbReadHandle->pColumns, 0); SColumnInfoData* pColInfoData = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, 0);
assert(cur->win.skey == ((TSKEY*)pColInfoData->pData)[0] && assert(cur->win.skey == ((TSKEY*)pColInfoData->pData)[0] &&
cur->win.ekey == ((TSKEY*)pColInfoData->pData)[cur->rows - 1]); cur->win.ekey == ((TSKEY*)pColInfoData->pData)[cur->rows - 1]);
} else { } else {
@ -2771,7 +2768,7 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int
STsdbReadHandle* pTsdbReadHandle) { STsdbReadHandle* pTsdbReadHandle) {
int numOfRows = 0; int numOfRows = 0;
int curRows = 0; int curRows = 0;
int32_t numOfCols = (int32_t)taosArrayGetSize(pTsdbReadHandle->pColumns); int32_t numOfCols = (int32_t)taosArrayGetSize(pTsdbReadHandle->pResBlock->pDataBlock);
STsdbCfg* pCfg = REPO_CFG(pTsdbReadHandle->pTsdb); STsdbCfg* pCfg = REPO_CFG(pTsdbReadHandle->pTsdb);
win->skey = TSKEY_INITIAL_VAL; win->skey = TSKEY_INITIAL_VAL;
@ -2901,7 +2898,7 @@ static bool loadBlockOfActiveTable(STsdbReadHandle* pTsdbReadHandle) {
if (exists) { if (exists) {
tsdbRetrieveDataBlock((tsdbReaderT*)pTsdbReadHandle, NULL); tsdbRetrieveDataBlock((tsdbReaderT*)pTsdbReadHandle, NULL);
if (pTsdbReadHandle->currentLoadExternalRows && pTsdbReadHandle->window.skey == pTsdbReadHandle->window.ekey) { if (pTsdbReadHandle->currentLoadExternalRows && pTsdbReadHandle->window.skey == pTsdbReadHandle->window.ekey) {
SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, 0); SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, 0);
assert(*(int64_t*)pColInfo->pData == pTsdbReadHandle->window.skey); assert(*(int64_t*)pColInfo->pData == pTsdbReadHandle->window.skey);
} }
@ -3003,7 +3000,7 @@ static bool loadCachedLastRow(STsdbReadHandle* pTsdbReadHandle) {
// //
// int32_t i = 0, j = 0; // int32_t i = 0, j = 0;
// while(i < tgNumOfCols && j < numOfCols) { // while(i < tgNumOfCols && j < numOfCols) {
// pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i); // pColInfo = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, i);
// if (pTable->lastCols[j].colId < pColInfo->info.colId) { // if (pTable->lastCols[j].colId < pColInfo->info.colId) {
// j++; // j++;
// continue; // continue;
@ -3066,7 +3063,7 @@ static bool loadCachedLastRow(STsdbReadHandle* pTsdbReadHandle) {
// continue; // continue;
// } // }
// //
// pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, n); // pColInfo = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, n);
// pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes;; // pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes;;
// //
// if (pColInfo->info.colId == PRIMARYKEY_TIMESTAMP_COL_ID) { // if (pColInfo->info.colId == PRIMARYKEY_TIMESTAMP_COL_ID) {
@ -3091,7 +3088,7 @@ static bool loadCachedLastRow(STsdbReadHandle* pTsdbReadHandle) {
// //
// // leave the real ts column as the last row, because last function only (not stable) use the last row as res // // leave the real ts column as the last row, because last function only (not stable) use the last row as res
// if (priKey != TSKEY_INITIAL_VAL) { // if (priKey != TSKEY_INITIAL_VAL) {
// pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, priIdx); // pColInfo = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, priIdx);
// pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes; // pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes;
// //
// *(TSKEY *)pData = priKey; // *(TSKEY *)pData = priKey;
@ -3101,7 +3098,7 @@ static bool loadCachedLastRow(STsdbReadHandle* pTsdbReadHandle) {
// continue; // continue;
// } // }
// //
// pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, n); // pColInfo = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, n);
// pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes;; // pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes;;
// //
// assert (pColInfo->info.colId != PRIMARYKEY_TIMESTAMP_COL_ID); // assert (pColInfo->info.colId != PRIMARYKEY_TIMESTAMP_COL_ID);
@ -3161,9 +3158,9 @@ static bool loadDataBlockFromTableSeq(STsdbReadHandle* pTsdbReadHandle) {
bool tsdbNextDataBlock(tsdbReaderT pHandle) { bool tsdbNextDataBlock(tsdbReaderT pHandle) {
STsdbReadHandle* pTsdbReadHandle = (STsdbReadHandle*)pHandle; STsdbReadHandle* pTsdbReadHandle = (STsdbReadHandle*)pHandle;
size_t numOfCols = taosArrayGetSize(pTsdbReadHandle->pColumns); size_t numOfCols = taosArrayGetSize(pTsdbReadHandle->pResBlock->pDataBlock);
for (int32_t i = 0; i < numOfCols; ++i) { for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i); SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, i);
colInfoDataCleanup(pColInfo, pTsdbReadHandle->outputCapacity); colInfoDataCleanup(pColInfo, pTsdbReadHandle->outputCapacity);
} }
@ -3250,7 +3247,7 @@ bool tsdbNextDataBlock(tsdbReaderT pHandle) {
// SArray* row = (type == TSDB_PREV_ROW)? pTsdbReadHandle->prev : pTsdbReadHandle->next; // SArray* row = (type == TSDB_PREV_ROW)? pTsdbReadHandle->prev : pTsdbReadHandle->next;
// //
// for (int32_t i = 0; i < numOfCols; ++i) { // for (int32_t i = 0; i < numOfCols; ++i) {
// SColumnInfoData* pCol = taosArrayGet(pTsdbReadHandle->pColumns, i); // SColumnInfoData* pCol = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, i);
// //
// SColumnInfoData colInfo = {{0}, 0}; // SColumnInfoData colInfo = {{0}, 0};
// colInfo.info = pCol->info; // colInfo.info = pCol->info;
@ -3280,7 +3277,7 @@ bool tsdbNextDataBlock(tsdbReaderT pHandle) {
// } // }
// //
// for (int32_t i = 0; i < cond.numOfCols; ++i) { // for (int32_t i = 0; i < cond.numOfCols; ++i) {
// SColumnInfoData* pColInfoData = taosArrayGet(pTsdbReadHandle->pColumns, i); // SColumnInfoData* pColInfoData = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, i);
// memcpy(&cond.colList[i], &pColInfoData->info, sizeof(SColumnInfo)); // memcpy(&cond.colList[i], &pColInfoData->info, sizeof(SColumnInfo));
// } // }
// //
@ -3342,7 +3339,7 @@ bool tsdbGetExternalRow(tsdbReaderT pHandle) {
int32_t numOfCols = (int32_t)QH_GET_NUM_OF_COLS(pTsdbReadHandle); int32_t numOfCols = (int32_t)QH_GET_NUM_OF_COLS(pTsdbReadHandle);
for (int32_t i = 0; i < numOfCols; ++i) { for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pColInfoData = taosArrayGet(pTsdbReadHandle->pColumns, i); SColumnInfoData* pColInfoData = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, i);
SColumnInfoData* first = taosArrayGet(pTsdbReadHandle->prev, i); SColumnInfoData* first = taosArrayGet(pTsdbReadHandle->prev, i);
memcpy(pColInfoData->pData, first->pData, pColInfoData->info.bytes); memcpy(pColInfoData->pData, first->pData, pColInfoData->info.bytes);
@ -3617,13 +3614,13 @@ SArray* tsdbRetrieveDataBlock(tsdbReaderT* pTsdbReadHandle, SArray* pIdList) {
*/ */
STsdbReadHandle* pHandle = (STsdbReadHandle*)pTsdbReadHandle; STsdbReadHandle* pHandle = (STsdbReadHandle*)pTsdbReadHandle;
if (pHandle->cur.fid == INT32_MIN) { if (pHandle->cur.fid == INT32_MIN) {
return pHandle->pColumns; return pHandle->pResBlock->pDataBlock;
} else { } else {
STableBlockInfo* pBlockInfo = &pHandle->pDataBlockInfo[pHandle->cur.slot]; STableBlockInfo* pBlockInfo = &pHandle->pDataBlockInfo[pHandle->cur.slot];
STableCheckInfo* pCheckInfo = pBlockInfo->pTableCheckInfo; STableCheckInfo* pCheckInfo = pBlockInfo->pTableCheckInfo;
if (pHandle->cur.mixBlock) { if (pHandle->cur.mixBlock) {
return pHandle->pColumns; return pHandle->pResBlock->pDataBlock;
} else { } else {
SDataBlockInfo binfo = GET_FILE_DATA_BLOCK_INFO(pCheckInfo, pBlockInfo->compBlock); SDataBlockInfo binfo = GET_FILE_DATA_BLOCK_INFO(pCheckInfo, pBlockInfo->compBlock);
assert(pHandle->realNumOfRows <= binfo.rows); assert(pHandle->realNumOfRows <= binfo.rows);
@ -3633,7 +3630,7 @@ SArray* tsdbRetrieveDataBlock(tsdbReaderT* pTsdbReadHandle, SArray* pIdList) {
if (pBlockLoadInfo->slot == pHandle->cur.slot && pBlockLoadInfo->fileGroup->fid == pHandle->cur.fid && if (pBlockLoadInfo->slot == pHandle->cur.slot && pBlockLoadInfo->fileGroup->fid == pHandle->cur.fid &&
pBlockLoadInfo->uid == pCheckInfo->tableId) { pBlockLoadInfo->uid == pCheckInfo->tableId) {
return pHandle->pColumns; return pHandle->pResBlock->pDataBlock;
} else { // only load the file block } else { // only load the file block
SBlock* pBlock = pBlockInfo->compBlock; SBlock* pBlock = pBlockInfo->compBlock;
if (doLoadFileDataBlock(pHandle, pBlock, pCheckInfo, pHandle->cur.slot) != TSDB_CODE_SUCCESS) { if (doLoadFileDataBlock(pHandle, pBlock, pCheckInfo, pHandle->cur.slot) != TSDB_CODE_SUCCESS) {
@ -3641,7 +3638,7 @@ SArray* tsdbRetrieveDataBlock(tsdbReaderT* pTsdbReadHandle, SArray* pIdList) {
} }
int32_t numOfRows = doCopyRowsFromFileBlock(pHandle, pHandle->outputCapacity, 0, 0, pBlock->numOfRows - 1); int32_t numOfRows = doCopyRowsFromFileBlock(pHandle, pHandle->outputCapacity, 0, 0, pBlock->numOfRows - 1);
return pHandle->pColumns; return pHandle->pResBlock->pDataBlock;
} }
} }
} }
@ -3692,7 +3689,7 @@ void tsdbCleanupReadHandle(tsdbReaderT queryHandle) {
return; return;
} }
pTsdbReadHandle->pColumns = doFreeColumnInfoData(pTsdbReadHandle->pColumns); pTsdbReadHandle->pResBlock->pDataBlock = doFreeColumnInfoData(pTsdbReadHandle->pResBlock->pDataBlock);
taosArrayDestroy(pTsdbReadHandle->suppInfo.defaultLoadColumn); taosArrayDestroy(pTsdbReadHandle->suppInfo.defaultLoadColumn);
taosMemoryFreeClear(pTsdbReadHandle->pDataBlockInfo); taosMemoryFreeClear(pTsdbReadHandle->pDataBlockInfo);

2
source/dnode/vnode/test/tsdbSmaTest.cpp
View File

@ -435,7 +435,7 @@ TEST(testCase, tSma_Data_Insert_Query_Test) {
SSDataBlock *pDataBlock = (SSDataBlock *)taosMemoryCalloc(1, sizeof(SSDataBlock)); SSDataBlock *pDataBlock = (SSDataBlock *)taosMemoryCalloc(1, sizeof(SSDataBlock));
EXPECT_NE(pDataBlock, nullptr); EXPECT_NE(pDataBlock, nullptr);
pDataBlock->pBlockAgg = NULL; pDataBlock->pBlockAgg = NULL;
pDataBlock->info.numOfCols = tSmaNumOfCols; taosArrayGetSize(pDataBlock->pDataBlock) = tSmaNumOfCols;
pDataBlock->info.rows = tSmaNumOfRows; pDataBlock->info.rows = tSmaNumOfRows;
pDataBlock->info.groupId = tSmaGroupId + g; pDataBlock->info.groupId = tSmaGroupId + g;

71
source/libs/command/src/command.c
View File

@ -32,31 +32,19 @@ static int32_t getSchemaBytes(const SSchema* pSchema) {
} }
static SSDataBlock* buildDescResultDataBlock() { static SSDataBlock* buildDescResultDataBlock() {
SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); SSDataBlock* pBlock = createDataBlock();
pBlock->info.numOfCols = DESCRIBE_RESULT_COLS;
pBlock->info.hasVarCol = true;
pBlock->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData)); SColumnInfoData infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, DESCRIBE_RESULT_FIELD_LEN, 1);
blockDataAppendColInfo(pBlock, &infoData);
SColumnInfoData infoData = {0}; infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, DESCRIBE_RESULT_TYPE_LEN, 2);
infoData.info.type = TSDB_DATA_TYPE_VARCHAR; blockDataAppendColInfo(pBlock, &infoData);
infoData.info.bytes = DESCRIBE_RESULT_FIELD_LEN;
taosArrayPush(pBlock->pDataBlock, &infoData); infoData = createColumnInfoData(TSDB_DATA_TYPE_INT, tDataTypes[TSDB_DATA_TYPE_INT].bytes, 3);
blockDataAppendColInfo(pBlock, &infoData);
infoData.info.type = TSDB_DATA_TYPE_VARCHAR;
infoData.info.bytes = DESCRIBE_RESULT_TYPE_LEN;
taosArrayPush(pBlock->pDataBlock, &infoData);
infoData.info.type = TSDB_DATA_TYPE_INT;
infoData.info.bytes = tDataTypes[TSDB_DATA_TYPE_INT].bytes;
taosArrayPush(pBlock->pDataBlock, &infoData);
infoData.info.type = TSDB_DATA_TYPE_VARCHAR;
infoData.info.bytes = DESCRIBE_RESULT_NOTE_LEN;
taosArrayPush(pBlock->pDataBlock, &infoData);
infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, DESCRIBE_RESULT_NOTE_LEN, 4);
blockDataAppendColInfo(pBlock, &infoData);
return pBlock; return pBlock;
} }
@ -127,22 +115,12 @@ static int32_t execResetQueryCache() { return catalogClearCache(); }
static SSDataBlock* buildCreateDBResultDataBlock() { static SSDataBlock* buildCreateDBResultDataBlock() {
SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); SSDataBlock* pBlock = createDataBlock();
pBlock->info.numOfCols = SHOW_CREATE_DB_RESULT_COLS; SColumnInfoData infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, SHOW_CREATE_DB_RESULT_COLS, 1);
pBlock->info.hasVarCol = true; blockDataAppendColInfo(pBlock, &infoData);
pBlock->pDataBlock = taosArrayInit(pBlock->info.numOfCols, sizeof(SColumnInfoData));
SColumnInfoData infoData = {0};
infoData.info.type = TSDB_DATA_TYPE_VARCHAR;
infoData.info.bytes = SHOW_CREATE_DB_RESULT_FIELD1_LEN;
taosArrayPush(pBlock->pDataBlock, &infoData);
infoData.info.type = TSDB_DATA_TYPE_VARCHAR;
infoData.info.bytes = SHOW_CREATE_DB_RESULT_FIELD2_LEN;
taosArrayPush(pBlock->pDataBlock, &infoData);
infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, SHOW_CREATE_DB_RESULT_FIELD2_LEN, 2);
blockDataAppendColInfo(pBlock, &infoData);
return pBlock; return pBlock;
} }
@ -284,21 +262,13 @@ static int32_t execShowCreateDatabase(SShowCreateDatabaseStmt* pStmt, SRetrieveT
} }
static SSDataBlock* buildCreateTbResultDataBlock() { static SSDataBlock* buildCreateTbResultDataBlock() {
SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); SSDataBlock* pBlock = createDataBlock();
pBlock->info.numOfCols = SHOW_CREATE_TB_RESULT_COLS;
pBlock->info.hasVarCol = true;
pBlock->pDataBlock = taosArrayInit(pBlock->info.numOfCols, sizeof(SColumnInfoData)); SColumnInfoData infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, SHOW_CREATE_TB_RESULT_FIELD1_LEN, 1);
blockDataAppendColInfo(pBlock, &infoData);
SColumnInfoData infoData = {0}; infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, SHOW_CREATE_TB_RESULT_FIELD2_LEN, 2);
infoData.info.type = TSDB_DATA_TYPE_VARCHAR; blockDataAppendColInfo(pBlock, &infoData);
infoData.info.bytes = SHOW_CREATE_TB_RESULT_FIELD1_LEN;
taosArrayPush(pBlock->pDataBlock, &infoData);
infoData.info.type = TSDB_DATA_TYPE_VARCHAR;
infoData.info.bytes = SHOW_CREATE_TB_RESULT_FIELD2_LEN;
taosArrayPush(pBlock->pDataBlock, &infoData);
return pBlock; return pBlock;
} }
@ -554,10 +524,9 @@ static int32_t execAlterLocal(SAlterLocalStmt* pStmt) { return TSDB_CODE_FAILED;
static SSDataBlock* buildLocalVariablesResultDataBlock() { static SSDataBlock* buildLocalVariablesResultDataBlock() {
SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock));
pBlock->info.numOfCols = SHOW_LOCAL_VARIABLES_RESULT_COLS;
pBlock->info.hasVarCol = true; pBlock->info.hasVarCol = true;
pBlock->pDataBlock = taosArrayInit(pBlock->info.numOfCols, sizeof(SColumnInfoData)); pBlock->pDataBlock = taosArrayInit(SHOW_LOCAL_VARIABLES_RESULT_COLS, sizeof(SColumnInfoData));
SColumnInfoData infoData = {0}; SColumnInfoData infoData = {0};
infoData.info.type = TSDB_DATA_TYPE_VARCHAR; infoData.info.type = TSDB_DATA_TYPE_VARCHAR;

17
source/libs/command/src/explain.c
View File

@ -1190,17 +1190,12 @@ int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) {
QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR); QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR);
} }
SSDataBlock *pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); SSDataBlock *pBlock = createDataBlock();
SColumnInfoData infoData = {0}; SColumnInfoData infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, TSDB_EXPLAIN_RESULT_ROW_SIZE, 1);
infoData.info.type = TSDB_DATA_TYPE_VARCHAR; blockDataAppendColInfo(pBlock, &infoData);
infoData.info.bytes = TSDB_EXPLAIN_RESULT_ROW_SIZE; blockDataEnsureCapacity(pBlock, rowNum);
pBlock->pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData));
taosArrayPush(pBlock->pDataBlock, &infoData);
SColumnInfoData* pInfoData = taosArrayGet(pBlock->pDataBlock, 0); SColumnInfoData* pInfoData = taosArrayGet(pBlock->pDataBlock, 0);
pInfoData->hasNull = false;
colInfoDataEnsureCapacity(pInfoData, 0, rowNum);
char buf[1024] = {0}; char buf[1024] = {0};
for (int32_t i = 0; i < rowNum; ++i) { for (int32_t i = 0; i < rowNum; ++i) {
@ -1210,9 +1205,7 @@ int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) {
colDataAppend(pInfoData, i, buf, false); colDataAppend(pInfoData, i, buf, false);
} }
pBlock->info.numOfCols = 1;
pBlock->info.rows = rowNum; pBlock->info.rows = rowNum;
pBlock->info.hasVarCol = true;
int32_t rspSize = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock); int32_t rspSize = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock);
@ -1226,7 +1219,7 @@ int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) {
rsp->numOfRows = htonl(rowNum); rsp->numOfRows = htonl(rowNum);
int32_t len = 0; int32_t len = 0;
blockCompressEncode(pBlock, rsp->data, &len, pBlock->info.numOfCols, 0); blockCompressEncode(pBlock, rsp->data, &len, taosArrayGetSize(pBlock->pDataBlock), 0);
ASSERT(len == rspSize - sizeof(SRetrieveTableRsp)); ASSERT(len == rspSize - sizeof(SRetrieveTableRsp));
rsp->compLen = htonl(len); rsp->compLen = htonl(len);

1
source/libs/executor/inc/executorimpl.h
View File

@ -336,7 +336,6 @@ typedef struct SStreamBlockScanInfo {
int32_t numOfPseudoExpr; int32_t numOfPseudoExpr;
int32_t primaryTsIndex; // primary time stamp slot id int32_t primaryTsIndex; // primary time stamp slot id
void* pDataReader;
SReadHandle readHandle; SReadHandle readHandle;
uint64_t tableUid; // queried super table uid uint64_t tableUid; // queried super table uid
EStreamScanMode scanMode; EStreamScanMode scanMode;

2
source/libs/executor/src/dataDeleter.c
View File

@ -75,7 +75,7 @@ static void toDataCacheEntry(SDataDeleterHandle* pHandle, const SInputData* pInp
SDataCacheEntry* pEntry = (SDataCacheEntry*)pBuf->pData; SDataCacheEntry* pEntry = (SDataCacheEntry*)pBuf->pData;
pEntry->compressed = 0; pEntry->compressed = 0;
pEntry->numOfRows = pInput->pData->info.rows; pEntry->numOfRows = pInput->pData->info.rows;
pEntry->numOfCols = pInput->pData->info.numOfCols; pEntry->numOfCols = taosArrayGetSize(pInput->pData->pDataBlock);
pEntry->dataLen = sizeof(SDeleterRes); pEntry->dataLen = sizeof(SDeleterRes);
ASSERT(1 == pEntry->numOfRows); ASSERT(1 == pEntry->numOfRows);

18
source/libs/executor/src/executil.c
View File

@ -183,34 +183,24 @@ SArray* createSortInfo(SNodeList* pNodeList) {
SSDataBlock* createResDataBlock(SDataBlockDescNode* pNode) { SSDataBlock* createResDataBlock(SDataBlockDescNode* pNode) {
int32_t numOfCols = LIST_LENGTH(pNode->pSlots); int32_t numOfCols = LIST_LENGTH(pNode->pSlots);
SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); SSDataBlock* pBlock = createDataBlock();
pBlock->pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData));
pBlock->info.blockId = pNode->dataBlockId; pBlock->info.blockId = pNode->dataBlockId;
pBlock->info.rowSize = pNode->totalRowSize; // todo ??
pBlock->info.type = STREAM_INVALID; pBlock->info.type = STREAM_INVALID;
for (int32_t i = 0; i < numOfCols; ++i) { for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData idata = {{0}};
SSlotDescNode* pDescNode = (SSlotDescNode*)nodesListGetNode(pNode->pSlots, i); SSlotDescNode* pDescNode = (SSlotDescNode*)nodesListGetNode(pNode->pSlots, i);
// if (!pDescNode->output) { // todo disable it temporarily // if (!pDescNode->output) { // todo disable it temporarily
// continue; // continue;
// } // }
idata.info.type = pDescNode->dataType.type; SColumnInfoData idata = createColumnInfoData(pDescNode->dataType.type, pDescNode->dataType.bytes, pDescNode->slotId);
idata.info.bytes = pDescNode->dataType.bytes;
idata.info.scale = pDescNode->dataType.scale; idata.info.scale = pDescNode->dataType.scale;
idata.info.slotId = pDescNode->slotId;
idata.info.precision = pDescNode->dataType.precision; idata.info.precision = pDescNode->dataType.precision;
if (IS_VAR_DATA_TYPE(idata.info.type)) { blockDataAppendColInfo(pBlock, &idata);
pBlock->info.hasVarCol = true;
}
taosArrayPush(pBlock->pDataBlock, &idata);
} }
pBlock->info.numOfCols = taosArrayGetSize(pBlock->pDataBlock);
return pBlock; return pBlock;
} }
@ -698,7 +688,7 @@ void relocateColumnData(SSDataBlock* pBlock, const SArray* pColMatchInfo, SArray
if (p->info.colId == pmInfo->colId) { if (p->info.colId == pmInfo->colId) {
SColumnInfoData* pDst = taosArrayGet(pBlock->pDataBlock, pmInfo->targetSlotId); SColumnInfoData* pDst = taosArrayGet(pBlock->pDataBlock, pmInfo->targetSlotId);
colDataAssign(pDst, p, pBlock->info.rows); colDataAssign(pDst, p, pBlock->info.rows, &pBlock->info);
i++; i++;
j++; j++;
} else if (p->info.colId < pmInfo->colId) { } else if (p->info.colId < pmInfo->colId) {

91
source/libs/executor/src/executorimpl.c
View File

@ -356,7 +356,7 @@ void initExecTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pQueryWindow
pColData->info.type = TSDB_DATA_TYPE_TIMESTAMP; pColData->info.type = TSDB_DATA_TYPE_TIMESTAMP;
pColData->info.bytes = sizeof(int64_t); pColData->info.bytes = sizeof(int64_t);
colInfoDataEnsureCapacity(pColData, 0, 5); colInfoDataEnsureCapacity(pColData, 5);
colDataAppendInt64(pColData, 0, &pQueryWindow->skey); colDataAppendInt64(pColData, 0, &pQueryWindow->skey);
colDataAppendInt64(pColData, 1, &pQueryWindow->ekey); colDataAppendInt64(pColData, 1, &pQueryWindow->ekey);
@ -458,7 +458,7 @@ static int32_t doCreateConstantValColumnInfo(SInputColumnInfoData* pInput, SFunc
pColInfo = pInput->pData[paramIndex]; pColInfo = pInput->pData[paramIndex];
} }
colInfoDataEnsureCapacity(pColInfo, 0, numOfRows); colInfoDataEnsureCapacity(pColInfo, numOfRows);
int8_t type = pFuncParam->param.nType; int8_t type = pFuncParam->param.nType;
if (type == TSDB_DATA_TYPE_BIGINT || type == TSDB_DATA_TYPE_UBIGINT) { if (type == TSDB_DATA_TYPE_BIGINT || type == TSDB_DATA_TYPE_UBIGINT) {
@ -579,7 +579,7 @@ int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBloc
colDataMergeCol(pColInfoData, pResult->info.rows, &pResult->info.capacity, pfCtx->input.pData[0], colDataMergeCol(pColInfoData, pResult->info.rows, &pResult->info.capacity, pfCtx->input.pData[0],
pfCtx->input.numOfRows); pfCtx->input.numOfRows);
} else { } else {
colDataAssign(pColInfoData, pfCtx->input.pData[0], pfCtx->input.numOfRows); colDataAssign(pColInfoData, pfCtx->input.pData[0], pfCtx->input.numOfRows, &pResult->info);
} }
numOfRows = pfCtx->input.numOfRows; numOfRows = pfCtx->input.numOfRows;
@ -1186,7 +1186,7 @@ int32_t loadDataBlockOnDemand(SExecTaskInfo* pTaskInfo, STableScanInfo* pTableSc
// } // }
// if (pQueryAttr->pFilters != NULL) { // if (pQueryAttr->pFilters != NULL) {
// filterSetColFieldData(pQueryAttr->pFilters, pBlock->info.numOfCols, pBlock->pDataBlock); // filterSetColFieldData(pQueryAttr->pFilters, taosArrayGetSize(pBlock->pDataBlock), pBlock->pDataBlock);
// } // }
// if (pQueryAttr->pFilters != NULL || pRuntimeEnv->pTsBuf != NULL) { // if (pQueryAttr->pFilters != NULL || pRuntimeEnv->pTsBuf != NULL) {
@ -1321,7 +1321,8 @@ void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock) {
// todo move to the initialization function // todo move to the initialization function
int32_t code = filterInitFromNode((SNode*)pFilterNode, &filter, 0); int32_t code = filterInitFromNode((SNode*)pFilterNode, &filter, 0);
SFilterColumnParam param1 = {.numOfCols = pBlock->info.numOfCols, .pDataBlock = pBlock->pDataBlock}; size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
SFilterColumnParam param1 = {.numOfCols = numOfCols, .pDataBlock = pBlock->pDataBlock};
code = filterSetDataFromSlotId(filter, &param1); code = filterSetDataFromSlotId(filter, &param1);
int8_t* rowRes = NULL; int8_t* rowRes = NULL;
@ -1340,10 +1341,11 @@ void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const int8_t* rowR
} }
if (rowRes != NULL) { if (rowRes != NULL) {
int32_t totalRows = pBlock->info.rows; int32_t totalRows = pBlock->info.rows;
SSDataBlock* px = createOneDataBlock(pBlock, true); SSDataBlock* px = createOneDataBlock(pBlock, true);
for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pSrc = taosArrayGet(px->pDataBlock, i); SColumnInfoData* pSrc = taosArrayGet(px->pDataBlock, i);
SColumnInfoData* pDst = taosArrayGet(pBlock->pDataBlock, i); SColumnInfoData* pDst = taosArrayGet(pBlock->pDataBlock, i);
// it is a reserved column for scalar function, and no data in this column yet. // it is a reserved column for scalar function, and no data in this column yet.
@ -2045,8 +2047,9 @@ int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLo
int32_t compLen, int32_t numOfOutput, int64_t startTs, uint64_t* total, int32_t compLen, int32_t numOfOutput, int64_t startTs, uint64_t* total,
SArray* pColList) { SArray* pColList) {
if (pColList == NULL) { // data from other sources if (pColList == NULL) { // data from other sources
blockDataCleanup(pRes);
// blockDataEnsureCapacity(pRes, numOfRows);
blockCompressDecode(pRes, numOfOutput, numOfRows, pData); blockCompressDecode(pRes, numOfOutput, numOfRows, pData);
pRes->info.rows = numOfRows;
} else { // extract data according to pColList } else { // extract data according to pColList
ASSERT(numOfOutput == taosArrayGetSize(pColList)); ASSERT(numOfOutput == taosArrayGetSize(pColList));
char* pStart = pData; char* pStart = pData;
@ -2064,32 +2067,19 @@ int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLo
pStart += sizeof(SSysTableSchema); pStart += sizeof(SSysTableSchema);
} }
SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); SSDataBlock* pBlock = createDataBlock();
pBlock->pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData));
pBlock->info.numOfCols = numOfCols;
for (int32_t i = 0; i < numOfCols; ++i) { for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData idata = {0}; SColumnInfoData idata = createColumnInfoData(pSchema[i].type, pSchema[i].bytes, pSchema[i].colId);
blockDataAppendColInfo(pBlock, &idata);
idata.info.type = pSchema[i].type;
idata.info.bytes = pSchema[i].bytes;
idata.info.colId = pSchema[i].colId;
idata.hasNull = true;
taosArrayPush(pBlock->pDataBlock, &idata);
if (IS_VAR_DATA_TYPE(idata.info.type)) {
pBlock->info.hasVarCol = true;
}
} }
blockCompressDecode(pBlock, numOfCols, numOfRows, pStart); blockCompressDecode(pBlock, numOfCols, numOfRows, pStart);
blockDataEnsureCapacity(pRes, numOfRows);
// data from mnode // data from mnode
pRes->info.rows = numOfRows; pRes->info.rows = numOfRows;
relocateColumnData(pRes, pColList, pBlock->pDataBlock, false); relocateColumnData(pRes, pColList, pBlock->pDataBlock, false);
taosArrayDestroy(pBlock->pDataBlock); blockDataDestroy(pBlock);
taosMemoryFree(pBlock);
// blockDataDestroy(pBlock);
} }
// todo move this to time window aggregator, since the primary timestamp may not be known by exchange operator. // todo move this to time window aggregator, since the primary timestamp may not be known by exchange operator.
@ -2412,15 +2402,15 @@ SOperatorInfo* createExchangeOperatorInfo(void* pTransporter, SExchangePhysiNode
tsem_init(&pInfo->ready, 0, 0); tsem_init(&pInfo->ready, 0, 0);
pInfo->seqLoadData = false; pInfo->seqLoadData = false;
pInfo->pTransporter = pTransporter; pInfo->pTransporter = pTransporter;
pInfo->pResult = createResDataBlock(pExNode->node.pOutputDataBlockDesc); pInfo->pResult = createResDataBlock(pExNode->node.pOutputDataBlockDesc);
pOperator->name = "ExchangeOperator"; pOperator->name = "ExchangeOperator";
pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_EXCHANGE; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_EXCHANGE;
pOperator->blocking = false; pOperator->blocking = false;
pOperator->status = OP_NOT_OPENED; pOperator->status = OP_NOT_OPENED;
pOperator->info = pInfo; pOperator->info = pInfo;
pOperator->exprSupp.numOfExprs = pInfo->pResult->info.numOfCols; pOperator->exprSupp.numOfExprs = taosArrayGetSize(pInfo->pResult->pDataBlock);
pOperator->pTaskInfo = pTaskInfo; pOperator->pTaskInfo = pTaskInfo;
pOperator->fpSet = createOperatorFpSet(prepareLoadRemoteData, doLoadRemoteData, NULL, NULL, pOperator->fpSet = createOperatorFpSet(prepareLoadRemoteData, doLoadRemoteData, NULL, NULL,
@ -2542,9 +2532,6 @@ static void doMergeImpl(SOperatorInfo* pOperator, int32_t numOfExpr, SSDataBlock
SSortedMergeOperatorInfo* pInfo = pOperator->info; SSortedMergeOperatorInfo* pInfo = pOperator->info;
SqlFunctionCtx* pCtx = pOperator->exprSupp.pCtx; SqlFunctionCtx* pCtx = pOperator->exprSupp.pCtx;
for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) {
// pCtx[i].size = 1;
}
for (int32_t i = 0; i < pBlock->info.rows; ++i) { for (int32_t i = 0; i < pBlock->info.rows; ++i) {
if (!pInfo->hasGroupVal) { if (!pInfo->hasGroupVal) {
@ -2653,7 +2640,7 @@ SSDataBlock* getSortedMergeBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlo
SColumnInfoData* pSrc = taosArrayGet(p->pDataBlock, pmInfo->srcSlotId); SColumnInfoData* pSrc = taosArrayGet(p->pDataBlock, pmInfo->srcSlotId);
SColumnInfoData* pDst = taosArrayGet(pDataBlock->pDataBlock, pmInfo->targetSlotId); SColumnInfoData* pDst = taosArrayGet(pDataBlock->pDataBlock, pmInfo->targetSlotId);
colDataAssign(pDst, pSrc, p->info.rows); colDataAssign(pDst, pSrc, p->info.rows, &pDataBlock->info);
} }
pDataBlock->info.rows = p->info.rows; pDataBlock->info.rows = p->info.rows;
@ -3967,7 +3954,7 @@ int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle,
} }
} }
} }
int32_t len = (int32_t)(pStart - (char*)keyBuf); int32_t len = (int32_t)(pStart - (char*)keyBuf);
uint64_t* pGroupId = taosHashGet(pTableListInfo->map, keyBuf, len); uint64_t* pGroupId = taosHashGet(pTableListInfo->map, keyBuf, len);
@ -3993,8 +3980,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
if (QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN == type) { if (QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN == type) {
STableScanPhysiNode* pTableScanNode = (STableScanPhysiNode*)pPhyNode; STableScanPhysiNode* pTableScanNode = (STableScanPhysiNode*)pPhyNode;
tsdbReaderT pDataReader = tsdbReaderT pDataReader = doCreateDataReader(pTableScanNode, pHandle, pTableListInfo, (uint64_t)queryId, taskId);
doCreateDataReader(pTableScanNode, pHandle, pTableListInfo, (uint64_t)queryId, taskId);
if (pDataReader == NULL && terrno != 0) { if (pDataReader == NULL && terrno != 0) {
pTaskInfo->code = terrno; pTaskInfo->code = terrno;
return NULL; return NULL;
@ -4020,16 +4006,20 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
STableScanInfo* pScanInfo = pOperator->info; STableScanInfo* pScanInfo = pOperator->info;
pTaskInfo->cost.pRecoder = &pScanInfo->readRecorder; pTaskInfo->cost.pRecoder = &pScanInfo->readRecorder;
return pOperator; return pOperator;
} else if (QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN == type) { } else if (QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN == type) {
STableMergeScanPhysiNode* pTableScanNode = (STableMergeScanPhysiNode*)pPhyNode; STableMergeScanPhysiNode* pTableScanNode = (STableMergeScanPhysiNode*)pPhyNode;
createScanTableListInfo(pTableScanNode, pHandle, pTableListInfo, queryId, taskId); createScanTableListInfo(pTableScanNode, pHandle, pTableListInfo, queryId, taskId);
extractTableSchemaVersion(pHandle, pTableScanNode->scan.uid, pTaskInfo); extractTableSchemaVersion(pHandle, pTableScanNode->scan.uid, pTaskInfo);
SOperatorInfo* pOperator = createTableMergeScanOperatorInfo(pTableScanNode, pTableListInfo, pHandle, pTaskInfo, queryId, taskId); SOperatorInfo* pOperator =
createTableMergeScanOperatorInfo(pTableScanNode, pTableListInfo, pHandle, pTaskInfo, queryId, taskId);
STableScanInfo* pScanInfo = pOperator->info; STableScanInfo* pScanInfo = pOperator->info;
pTaskInfo->cost.pRecoder = &pScanInfo->readRecorder; pTaskInfo->cost.pRecoder = &pScanInfo->readRecorder;
return pOperator; return pOperator;
} else if (QUERY_NODE_PHYSICAL_PLAN_EXCHANGE == type) { } else if (QUERY_NODE_PHYSICAL_PLAN_EXCHANGE == type) {
return createExchangeOperatorInfo(pHandle->pMsgCb->clientRpc, (SExchangePhysiNode*)pPhyNode, pTaskInfo); return createExchangeOperatorInfo(pHandle->pMsgCb->clientRpc, (SExchangePhysiNode*)pPhyNode, pTaskInfo);
} else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN == type) { } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN == type) {
SScanPhysiNode* pScanPhyNode = (SScanPhysiNode*)pPhyNode; // simple child table. SScanPhysiNode* pScanPhyNode = (SScanPhysiNode*)pPhyNode; // simple child table.
STableScanPhysiNode* pTableScanNode = (STableScanPhysiNode*)pPhyNode; STableScanPhysiNode* pTableScanNode = (STableScanPhysiNode*)pPhyNode;
@ -4044,20 +4034,22 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
if (pHandle->initTsdbReader) { if (pHandle->initTsdbReader) {
// for stream // for stream
ASSERT(pHandle->vnode); ASSERT(pHandle->vnode);
pDataReader = pDataReader = doCreateDataReader(pTableScanNode, pHandle, pTableListInfo, (uint64_t)queryId, taskId);
doCreateDataReader(pTableScanNode, pHandle, pTableListInfo, (uint64_t)queryId, taskId);
} else { } else {
// for tq // for tq
ASSERT(pHandle->meta); ASSERT(pHandle->meta);
getTableList(pHandle->meta, pScanPhyNode, pTableListInfo); getTableList(pHandle->meta, pScanPhyNode, pTableListInfo);
} }
} }
#if 0
if (pDataReader == NULL && terrno != 0) { if (pDataReader == NULL && terrno != 0) {
qDebug("%s pDataReader is NULL", GET_TASKID(pTaskInfo)); qDebug("%s pDataReader is NULL", GET_TASKID(pTaskInfo));
// return NULL; // return NULL;
} else { } else {
qDebug("%s pDataReader is not NULL", GET_TASKID(pTaskInfo)); qDebug("%s pDataReader is not NULL", GET_TASKID(pTaskInfo));
} }
#endif
SArray* groupKeys = extractPartitionColInfo(pTableScanNode->pPartitionTags); SArray* groupKeys = extractPartitionColInfo(pTableScanNode->pPartitionTags);
int32_t code = generateGroupIdMap(pTableListInfo, pHandle, groupKeys); // todo for json int32_t code = generateGroupIdMap(pTableListInfo, pHandle, groupKeys); // todo for json
@ -4070,9 +4062,11 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
SOperatorInfo* pOperator = createStreamScanOperatorInfo(pDataReader, pHandle, pTableScanNode, pTaskInfo, &twSup); SOperatorInfo* pOperator = createStreamScanOperatorInfo(pDataReader, pHandle, pTableScanNode, pTaskInfo, &twSup);
return pOperator; return pOperator;
} else if (QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN == type) { } else if (QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN == type) {
SSystemTableScanPhysiNode* pSysScanPhyNode = (SSystemTableScanPhysiNode*)pPhyNode; SSystemTableScanPhysiNode* pSysScanPhyNode = (SSystemTableScanPhysiNode*)pPhyNode;
return createSysTableScanOperatorInfo(pHandle, pSysScanPhyNode, pTaskInfo); return createSysTableScanOperatorInfo(pHandle, pSysScanPhyNode, pTaskInfo);
} else if (QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN == type) { } else if (QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN == type) {
STagScanPhysiNode* pScanPhyNode = (STagScanPhysiNode*)pPhyNode; STagScanPhysiNode* pScanPhyNode = (STagScanPhysiNode*)pPhyNode;
@ -4083,6 +4077,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
} }
return createTagScanOperatorInfo(pHandle, pScanPhyNode, pTableListInfo, pTaskInfo); return createTagScanOperatorInfo(pHandle, pScanPhyNode, pTableListInfo, pTaskInfo);
} else if (QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN == type) { } else if (QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN == type) {
SBlockDistScanPhysiNode* pBlockNode = (SBlockDistScanPhysiNode*)pPhyNode; SBlockDistScanPhysiNode* pBlockNode = (SBlockDistScanPhysiNode*)pPhyNode;
pTableListInfo->pTableList = taosArrayInit(4, sizeof(STableKeyInfo)); pTableListInfo->pTableList = taosArrayInit(4, sizeof(STableKeyInfo));
@ -4356,7 +4351,7 @@ tsdbReaderT doCreateDataReader(STableScanPhysiNode* pTableScanNode, SReadHandle*
goto _error; goto _error;
} }
tsdbReaderT* pReader = tsdbReaderOpen(pHandle->vnode, &cond, pTableListInfo, queryId, taskId); tsdbReaderT pReader = tsdbReaderOpen(pHandle->vnode, &cond, pTableListInfo, queryId, taskId);
cleanupQueryTableDataCond(&cond); cleanupQueryTableDataCond(&cond);
return pReader; return pReader;
@ -4500,18 +4495,14 @@ int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SRead
(*pTaskInfo)->sql = sql; (*pTaskInfo)->sql = sql;
(*pTaskInfo)->tableqinfoList.pTagCond = pPlan->pTagCond; (*pTaskInfo)->tableqinfoList.pTagCond = pPlan->pTagCond;
(*pTaskInfo)->tableqinfoList.pTagIndexCond = pPlan->pTagIndexCond; (*pTaskInfo)->tableqinfoList.pTagIndexCond = pPlan->pTagIndexCond;
(*pTaskInfo)->pRoot = createOperatorTree(pPlan->pNode, *pTaskInfo, pHandle, queryId, taskId, (*pTaskInfo)->pRoot =
&(*pTaskInfo)->tableqinfoList); createOperatorTree(pPlan->pNode, *pTaskInfo, pHandle, queryId, taskId, &(*pTaskInfo)->tableqinfoList);
if (NULL == (*pTaskInfo)->pRoot) { if (NULL == (*pTaskInfo)->pRoot) {
code = (*pTaskInfo)->code; code = (*pTaskInfo)->code;
goto _complete; goto _complete;
} }
if ((*pTaskInfo)->pRoot == NULL) {
code = TSDB_CODE_QRY_OUT_OF_MEMORY;
goto _complete;
}
return code; return code;
_complete: _complete:

4
source/libs/executor/src/groupoperator.c
View File

@ -559,8 +559,8 @@ uint64_t calcGroupId(char* pData, int32_t len) {
} }
int32_t* setupColumnOffset(const SSDataBlock* pBlock, int32_t rowCapacity) { int32_t* setupColumnOffset(const SSDataBlock* pBlock, int32_t rowCapacity) {
size_t numOfCols = pBlock->info.numOfCols; size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
int32_t* offset = taosMemoryCalloc(pBlock->info.numOfCols, sizeof(int32_t)); int32_t* offset = taosMemoryCalloc(numOfCols, sizeof(int32_t));
offset[0] = sizeof(int32_t) + sizeof(uint64_t); // the number of rows in current page, ref to SSDataBlock paged serialization format offset[0] = sizeof(int32_t) + sizeof(uint64_t); // the number of rows in current page, ref to SSDataBlock paged serialization format

83
source/libs/executor/src/scanoperator.c
View File

@ -203,7 +203,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca
pCost->skipBlocks += 1; pCost->skipBlocks += 1;
// clear all data in pBlock that are set when handing the previous block // clear all data in pBlock that are set when handing the previous block
for (int32_t i = 0; i < pBlockInfo->numOfCols; ++i) { for (int32_t i = 0; i < taosArrayGetSize(pBlock->pDataBlock); ++i) {
SColumnInfoData* pcol = taosArrayGet(pBlock->pDataBlock, i); SColumnInfoData* pcol = taosArrayGet(pBlock->pDataBlock, i);
pcol->pData = NULL; pcol->pData = NULL;
} }
@ -217,7 +217,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca
tsdbRetrieveDataBlockStatisInfo(pTableScanInfo->dataReader, &pColAgg, &allColumnsHaveAgg); tsdbRetrieveDataBlockStatisInfo(pTableScanInfo->dataReader, &pColAgg, &allColumnsHaveAgg);
if (allColumnsHaveAgg == true) { if (allColumnsHaveAgg == true) {
int32_t numOfCols = pBlock->info.numOfCols; int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
// todo create this buffer during creating operator // todo create this buffer during creating operator
if (pBlock->pBlockAgg == NULL) { if (pBlock->pBlockAgg == NULL) {
@ -316,7 +316,7 @@ void addTagPseudoColumnData(SReadHandle* pHandle, SExprInfo* pPseudoExpr, int32_
SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, dstSlotId); SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, dstSlotId);
colInfoDataEnsureCapacity(pColInfoData, 0, pBlock->info.rows); colInfoDataEnsureCapacity(pColInfoData, pBlock->info.rows);
colInfoDataCleanup(pColInfoData, pBlock->info.rows); colInfoDataCleanup(pColInfoData, pBlock->info.rows);
int32_t functionId = pExpr->pExpr->_function.functionId; int32_t functionId = pExpr->pExpr->_function.functionId;
@ -355,10 +355,8 @@ void setTbNameColData(void* pMeta, const SSDataBlock* pBlock, SColumnInfoData* p
struct SScalarFuncExecFuncs fpSet = {0}; struct SScalarFuncExecFuncs fpSet = {0};
fmGetScalarFuncExecFuncs(functionId, &fpSet); fmGetScalarFuncExecFuncs(functionId, &fpSet);
SColumnInfoData infoData = {0}; SColumnInfoData infoData = createColumnInfoData(TSDB_DATA_TYPE_BIGINT, sizeof(uint64_t), 1);
infoData.info.type = TSDB_DATA_TYPE_BIGINT; colInfoDataEnsureCapacity(&infoData, 1);
infoData.info.bytes = sizeof(uint64_t);
colInfoDataEnsureCapacity(&infoData, 0, 1);
colDataAppendInt64(&infoData, 0, (int64_t*)&pBlock->info.uid); colDataAppendInt64(&infoData, 0, (int64_t*)&pBlock->info.uid);
SScalarParam srcParam = {.numOfRows = pBlock->info.rows, .param = pMeta, .columnData = &infoData}; SScalarParam srcParam = {.numOfRows = pBlock->info.rows, .param = pMeta, .columnData = &infoData};
@ -786,7 +784,7 @@ static bool prepareDataScan(SStreamBlockScanInfo* pInfo) {
} }
static void copyOneRow(SSDataBlock* dest, SSDataBlock* source, int32_t sourceRowId) { static void copyOneRow(SSDataBlock* dest, SSDataBlock* source, int32_t sourceRowId) {
for (int32_t j = 0; j < source->info.numOfCols; j++) { for (int32_t j = 0; j < taosArrayGetSize(source->pDataBlock); j++) {
SColumnInfoData* pDestCol = (SColumnInfoData*)taosArrayGet(dest->pDataBlock, j); SColumnInfoData* pDestCol = (SColumnInfoData*)taosArrayGet(dest->pDataBlock, j);
SColumnInfoData* pSourceCol = (SColumnInfoData*)taosArrayGet(source->pDataBlock, j); SColumnInfoData* pSourceCol = (SColumnInfoData*)taosArrayGet(source->pDataBlock, j);
if (colDataIsNull_s(pSourceCol, sourceRowId)) { if (colDataIsNull_s(pSourceCol, sourceRowId)) {
@ -858,7 +856,6 @@ static void setUpdateData(SStreamBlockScanInfo* pInfo, SSDataBlock* pBlock, SSDa
SColumnInfoData* pCol = (SColumnInfoData*)taosArrayGet(pUpdateBlock->pDataBlock, pInfo->primaryTsIndex); SColumnInfoData* pCol = (SColumnInfoData*)taosArrayGet(pUpdateBlock->pDataBlock, pInfo->primaryTsIndex);
ASSERT(pCol->info.type == TSDB_DATA_TYPE_TIMESTAMP); ASSERT(pCol->info.type == TSDB_DATA_TYPE_TIMESTAMP);
blockDataEnsureCapacity(pUpdateBlock, size); blockDataEnsureCapacity(pUpdateBlock, size);
ASSERT(pBlock->info.numOfCols == pUpdateBlock->info.numOfCols);
int32_t rowId = *(int32_t*)taosArrayGet(pInfo->tsArray, pInfo->tsArrayIndex); int32_t rowId = *(int32_t*)taosArrayGet(pInfo->tsArray, pInfo->tsArrayIndex);
pInfo->groupId = getGroupId(pInfo->pSnapshotReadOp, pBlock, rowId); pInfo->groupId = getGroupId(pInfo->pSnapshotReadOp, pBlock, rowId);
@ -969,13 +966,13 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) {
blockDataCleanup(pInfo->pRes); blockDataCleanup(pInfo->pRes);
while (tqNextDataBlock(pInfo->streamBlockReader)) { while (tqNextDataBlock(pInfo->streamBlockReader)) {
SArray* pCols = NULL; SSDataBlock block = {0};
uint64_t groupId = 0; uint64_t groupId = 0;
uint64_t uid = 0; uint64_t uid = 0;
int32_t numOfRows = 0; int32_t numOfRows = 0;
int16_t outputCol = 0;
int32_t code = tqRetrieveDataBlock(&pCols, pInfo->streamBlockReader, &groupId, &uid, &numOfRows, &outputCol); // todo refactor
int32_t code = tqRetrieveDataBlock(&block, pInfo->streamBlockReader, &groupId, &uid, &numOfRows);
if (code != TSDB_CODE_SUCCESS || numOfRows == 0) { if (code != TSDB_CODE_SUCCESS || numOfRows == 0) {
pTaskInfo->code = code; pTaskInfo->code = code;
@ -986,6 +983,7 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) {
pInfo->pRes->info.rows = numOfRows; pInfo->pRes->info.rows = numOfRows;
pInfo->pRes->info.uid = uid; pInfo->pRes->info.uid = uid;
pInfo->pRes->info.type = STREAM_NORMAL; pInfo->pRes->info.type = STREAM_NORMAL;
pInfo->pRes->info.capacity = numOfRows;
// for generating rollup SMA result, each time is an independent time serie. // for generating rollup SMA result, each time is an independent time serie.
// TODO temporarily used, when the statement of "partition by tbname" is ready, remove this // TODO temporarily used, when the statement of "partition by tbname" is ready, remove this
@ -1000,6 +998,7 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) {
pInfo->pRes->info.groupId = *groupIdPre; pInfo->pRes->info.groupId = *groupIdPre;
} }
// todo extract method
for (int32_t i = 0; i < taosArrayGetSize(pInfo->pColMatchInfo); ++i) { for (int32_t i = 0; i < taosArrayGetSize(pInfo->pColMatchInfo); ++i) {
SColMatchInfo* pColMatchInfo = taosArrayGet(pInfo->pColMatchInfo, i); SColMatchInfo* pColMatchInfo = taosArrayGet(pInfo->pColMatchInfo, i);
if (!pColMatchInfo->output) { if (!pColMatchInfo->output) {
@ -1007,8 +1006,8 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) {
} }
bool colExists = false; bool colExists = false;
for (int32_t j = 0; j < taosArrayGetSize(pCols); ++j) { for (int32_t j = 0; j < blockDataGetNumOfCols(&block); ++j) {
SColumnInfoData* pResCol = taosArrayGet(pCols, j); SColumnInfoData* pResCol = bdGetColumnInfoData(&block, j);
if (pResCol->info.colId == pColMatchInfo->colId) { if (pResCol->info.colId == pColMatchInfo->colId) {
taosArraySet(pInfo->pRes->pDataBlock, pColMatchInfo->targetSlotId, pResCol); taosArraySet(pInfo->pRes->pDataBlock, pColMatchInfo->targetSlotId, pResCol);
colExists = true; colExists = true;
@ -1019,11 +1018,13 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) {
// the required column does not exists in submit block, let's set it to be all null value // the required column does not exists in submit block, let's set it to be all null value
if (!colExists) { if (!colExists) {
SColumnInfoData* pDst = taosArrayGet(pInfo->pRes->pDataBlock, pColMatchInfo->targetSlotId); SColumnInfoData* pDst = taosArrayGet(pInfo->pRes->pDataBlock, pColMatchInfo->targetSlotId);
colInfoDataEnsureCapacity(pDst, 0, pBlockInfo->rows);
colDataAppendNNULL(pDst, 0, pBlockInfo->rows); colDataAppendNNULL(pDst, 0, pBlockInfo->rows);
} }
} }
// TODO refactor @liao
taosArrayDestroy(block.pDataBlock);
if (pInfo->pRes->pDataBlock == NULL) { if (pInfo->pRes->pDataBlock == NULL) {
// TODO add log // TODO add log
pOperator->status = OP_EXEC_DONE; pOperator->status = OP_EXEC_DONE;
@ -1063,12 +1064,11 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) {
} }
return (pBlockInfo->rows == 0) ? NULL : pInfo->pRes; return (pBlockInfo->rows == 0) ? NULL : pInfo->pRes;
} else if (pInfo->blockType == STREAM_DATA_TYPE_FROM_SNAPSHOT) { } else if (pInfo->blockType == STREAM_DATA_TYPE_FROM_SNAPSHOT) {
SSDataBlock* pResult = doTableScan(pInfo->pSnapshotReadOp); SSDataBlock* pResult = doTableScan(pInfo->pSnapshotReadOp);
if (pResult) { return pResult && pResult->info.rows > 0 ? pResult : NULL;
return pResult->info.rows > 0 ? pResult : NULL;
}
return NULL;
} else { } else {
ASSERT(0); ASSERT(0);
return NULL; return NULL;
@ -1163,7 +1163,6 @@ SOperatorInfo* createStreamScanOperatorInfo(void* pDataReader, SReadHandle* pHan
pInfo->pRes = createResDataBlock(pDescNode); pInfo->pRes = createResDataBlock(pDescNode);
pInfo->pUpdateRes = createResDataBlock(pDescNode); pInfo->pUpdateRes = createResDataBlock(pDescNode);
pInfo->pCondition = pScanPhyNode->node.pConditions; pInfo->pCondition = pScanPhyNode->node.pConditions;
pInfo->pDataReader = pDataReader;
pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE; pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE;
pInfo->sessionSup = (SessionWindowSupporter){.pStreamAggSup = NULL, .gap = -1}; pInfo->sessionSup = (SessionWindowSupporter){.pStreamAggSup = NULL, .gap = -1};
pInfo->groupId = 0; pInfo->groupId = 0;
@ -1173,7 +1172,7 @@ SOperatorInfo* createStreamScanOperatorInfo(void* pDataReader, SReadHandle* pHan
pOperator->blocking = false; pOperator->blocking = false;
pOperator->status = OP_NOT_OPENED; pOperator->status = OP_NOT_OPENED;
pOperator->info = pInfo; pOperator->info = pInfo;
pOperator->exprSupp.numOfExprs = pInfo->pRes->info.numOfCols; pOperator->exprSupp.numOfExprs = taosArrayGetSize(pInfo->pRes->pDataBlock);
pOperator->pTaskInfo = pTaskInfo; pOperator->pTaskInfo = pTaskInfo;
pOperator->fpSet = pOperator->fpSet =
@ -1301,7 +1300,7 @@ static SSDataBlock* doFilterResult(SSysTableScanInfo* pInfo) {
SColumnInfoData* pSrc = taosArrayGet(pInfo->pRes->pDataBlock, i); SColumnInfoData* pSrc = taosArrayGet(pInfo->pRes->pDataBlock, i);
if (keep) { if (keep) {
colDataAssign(pDest, pSrc, pInfo->pRes->info.rows); colDataAssign(pDest, pSrc, pInfo->pRes->info.rows, &px->info);
numOfRow = pInfo->pRes->info.rows; numOfRow = pInfo->pRes->info.rows;
} else if (NULL != rowRes) { } else if (NULL != rowRes) {
numOfRow = 0; numOfRow = 0;
@ -1331,8 +1330,6 @@ static SSDataBlock* doFilterResult(SSysTableScanInfo* pInfo) {
} }
static SSDataBlock* buildSysTableMetaBlock() { static SSDataBlock* buildSysTableMetaBlock() {
SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock));
size_t size = 0; size_t size = 0;
const SSysTableMeta* pMeta = NULL; const SSysTableMeta* pMeta = NULL;
getInfosDbMeta(&pMeta, &size); getInfosDbMeta(&pMeta, &size);
@ -1345,19 +1342,13 @@ static SSDataBlock* buildSysTableMetaBlock() {
} }
} }
pBlock->pDataBlock = taosArrayInit(pBlock->info.numOfCols, sizeof(SColumnInfoData)); SSDataBlock* pBlock = createDataBlock();
for (int32_t i = 0; i < pMeta[index].colNum; ++i) { for (int32_t i = 0; i < pMeta[index].colNum; ++i) {
SColumnInfoData colInfoData = {0}; SColumnInfoData colInfoData =
colInfoData.info.colId = i + 1; createColumnInfoData(pMeta[index].schema[i].type, pMeta[index].schema[i].bytes, i + 1);
colInfoData.info.type = pMeta[index].schema[i].type; blockDataAppendColInfo(pBlock, &colInfoData);
colInfoData.info.bytes = pMeta[index].schema[i].bytes;
taosArrayPush(pBlock->pDataBlock, &colInfoData);
} }
pBlock->info.numOfCols = pMeta[index].colNum;
pBlock->info.hasVarCol = true;
return pBlock; return pBlock;
} }
@ -1388,7 +1379,6 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator) {
} }
blockDataCleanup(pInfo->pRes); blockDataCleanup(pInfo->pRes);
int32_t numOfRows = 0; int32_t numOfRows = 0;
const char* db = NULL; const char* db = NULL;
@ -1584,7 +1574,7 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator) {
if (pRsp->numOfRows == 0 || pRsp->completed) { if (pRsp->numOfRows == 0 || pRsp->completed) {
pOperator->status = OP_EXEC_DONE; pOperator->status = OP_EXEC_DONE;
qDebug("%s load meta data from mnode completed, rowsOfSource:%d, totalRows:%" PRIu64 " ", GET_TASKID(pTaskInfo), qDebug("%s load meta data from mnode completed, rowsOfSource:%d, totalRows:%" PRIu64, GET_TASKID(pTaskInfo),
pRsp->numOfRows, pInfo->loadInfo.totalRows); pRsp->numOfRows, pInfo->loadInfo.totalRows);
if (pRsp->numOfRows == 0) { if (pRsp->numOfRows == 0) {
@ -1711,7 +1701,7 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScan
pOperator->blocking = false; pOperator->blocking = false;
pOperator->status = OP_NOT_OPENED; pOperator->status = OP_NOT_OPENED;
pOperator->info = pInfo; pOperator->info = pInfo;
pOperator->exprSupp.numOfExprs = pResBlock->info.numOfCols; pOperator->exprSupp.numOfExprs = taosArrayGetSize(pResBlock->pDataBlock);
pOperator->pTaskInfo = pTaskInfo; pOperator->pTaskInfo = pTaskInfo;
pOperator->fpSet = pOperator->fpSet =
@ -2049,6 +2039,7 @@ int32_t createMultipleDataReaders(SQueryTableDataCond* pQueryCond, SReadHandle*
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
// todo refactor
static int32_t loadDataBlockFromOneTable(SOperatorInfo* pOperator, STableMergeScanInfo* pTableScanInfo, static int32_t loadDataBlockFromOneTable(SOperatorInfo* pOperator, STableMergeScanInfo* pTableScanInfo,
int32_t readerIdx, SSDataBlock* pBlock, uint32_t* status) { int32_t readerIdx, SSDataBlock* pBlock, uint32_t* status) {
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
@ -2079,7 +2070,7 @@ static int32_t loadDataBlockFromOneTable(SOperatorInfo* pOperator, STableMergeSc
pCost->skipBlocks += 1; pCost->skipBlocks += 1;
// clear all data in pBlock that are set when handing the previous block // clear all data in pBlock that are set when handing the previous block
for (int32_t i = 0; i < pBlockInfo->numOfCols; ++i) { for (int32_t i = 0; i < taosArrayGetSize(pBlock->pDataBlock); ++i) {
SColumnInfoData* pcol = taosArrayGet(pBlock->pDataBlock, i); SColumnInfoData* pcol = taosArrayGet(pBlock->pDataBlock, i);
pcol->pData = NULL; pcol->pData = NULL;
} }
@ -2094,7 +2085,7 @@ static int32_t loadDataBlockFromOneTable(SOperatorInfo* pOperator, STableMergeSc
tsdbRetrieveDataBlockStatisInfo(reader, &pColAgg, &allColumnsHaveAgg); tsdbRetrieveDataBlockStatisInfo(reader, &pColAgg, &allColumnsHaveAgg);
if (allColumnsHaveAgg == true) { if (allColumnsHaveAgg == true) {
int32_t numOfCols = pBlock->info.numOfCols; int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
// todo create this buffer during creating operator // todo create this buffer during creating operator
if (pBlock->pBlockAgg == NULL) { if (pBlock->pBlockAgg == NULL) {
@ -2189,7 +2180,13 @@ static SSDataBlock* getTableDataBlock(void* param) {
continue; continue;
} }
tsdbRetrieveDataBlockInfo(reader, &pBlock->info); blockDataCleanup(pBlock);
SDataBlockInfo binfo = pBlock->info;
tsdbRetrieveDataBlockInfo(reader, &binfo);
binfo.capacity = binfo.rows;
blockDataEnsureCapacity(pBlock, binfo.capacity);
pBlock->info = binfo;
uint32_t status = 0; uint32_t status = 0;
int32_t code = loadDataBlockFromOneTable(pOperator, pTableScanInfo, readerIdx, pBlock, &status); int32_t code = loadDataBlockFromOneTable(pOperator, pTableScanInfo, readerIdx, pBlock, &status);

23
source/libs/executor/src/sortoperator.c
View File

@ -80,7 +80,7 @@ _error:
} }
void appendOneRowToDataBlock(SSDataBlock* pBlock, STupleHandle* pTupleHandle) { void appendOneRowToDataBlock(SSDataBlock* pBlock, STupleHandle* pTupleHandle) {
for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { for (int32_t i = 0; i < taosArrayGetSize(pBlock->pDataBlock); ++i) {
SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i); SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i);
bool isNull = tsortIsNullVal(pTupleHandle, i); bool isNull = tsortIsNullVal(pTupleHandle, i);
if (isNull) { if (isNull) {
@ -120,6 +120,9 @@ SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, i
} }
if (p->info.rows > 0) { if (p->info.rows > 0) {
blockDataEnsureCapacity(pDataBlock, capacity);
// todo extract function to handle this
int32_t numOfCols = taosArrayGetSize(pColMatchInfo); int32_t numOfCols = taosArrayGetSize(pColMatchInfo);
for (int32_t i = 0; i < numOfCols; ++i) { for (int32_t i = 0; i < numOfCols; ++i) {
SColMatchInfo* pmInfo = taosArrayGet(pColMatchInfo, i); SColMatchInfo* pmInfo = taosArrayGet(pColMatchInfo, i);
@ -127,11 +130,10 @@ SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, i
SColumnInfoData* pSrc = taosArrayGet(p->pDataBlock, pmInfo->srcSlotId); SColumnInfoData* pSrc = taosArrayGet(p->pDataBlock, pmInfo->srcSlotId);
SColumnInfoData* pDst = taosArrayGet(pDataBlock->pDataBlock, pmInfo->targetSlotId); SColumnInfoData* pDst = taosArrayGet(pDataBlock->pDataBlock, pmInfo->targetSlotId);
colDataAssign(pDst, pSrc, p->info.rows); colDataAssign(pDst, pSrc, p->info.rows, &pDataBlock->info);
} }
pDataBlock->info.rows = p->info.rows; pDataBlock->info.rows = p->info.rows;
pDataBlock->info.capacity = p->info.rows;
} }
blockDataDestroy(p); blockDataDestroy(p);
@ -257,6 +259,7 @@ typedef struct SGroupSortOperatorInfo {
SSDataBlock* getGroupSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, int32_t capacity, SSDataBlock* getGroupSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, int32_t capacity,
SArray* pColMatchInfo, SGroupSortOperatorInfo* pInfo) { SArray* pColMatchInfo, SGroupSortOperatorInfo* pInfo) {
blockDataCleanup(pDataBlock); blockDataCleanup(pDataBlock);
blockDataEnsureCapacity(pDataBlock, capacity);
SSDataBlock* p = tsortGetSortedDataBlock(pHandle); SSDataBlock* p = tsortGetSortedDataBlock(pHandle);
if (p == NULL) { if (p == NULL) {
@ -285,7 +288,7 @@ SSDataBlock* getGroupSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlo
SColumnInfoData* pSrc = taosArrayGet(p->pDataBlock, pmInfo->srcSlotId); SColumnInfoData* pSrc = taosArrayGet(p->pDataBlock, pmInfo->srcSlotId);
SColumnInfoData* pDst = taosArrayGet(pDataBlock->pDataBlock, pmInfo->targetSlotId); SColumnInfoData* pDst = taosArrayGet(pDataBlock->pDataBlock, pmInfo->targetSlotId);
colDataAssign(pDst, pSrc, p->info.rows); colDataAssign(pDst, pSrc, p->info.rows, &pDataBlock->info);
} }
pDataBlock->info.rows = p->info.rows; pDataBlock->info.rows = p->info.rows;
@ -387,6 +390,10 @@ SSDataBlock* doGroupSort(SOperatorInfo* pOperator) {
pInfo->hasGroupId = true; pInfo->hasGroupId = true;
pInfo->prefetchedSortInput = pOperator->pDownstream[0]->fpSet.getNextFn(pOperator->pDownstream[0]); pInfo->prefetchedSortInput = pOperator->pDownstream[0]->fpSet.getNextFn(pOperator->pDownstream[0]);
if (pInfo->prefetchedSortInput == NULL) {
doSetOperatorCompleted(pOperator);
return NULL;
}
pInfo->currGroupId = pInfo->prefetchedSortInput->info.groupId; pInfo->currGroupId = pInfo->prefetchedSortInput->info.groupId;
pInfo->childOpStatus = CHILD_OP_NEW_GROUP; pInfo->childOpStatus = CHILD_OP_NEW_GROUP;
beginSortGroup(pOperator); beginSortGroup(pOperator);
@ -460,7 +467,7 @@ SOperatorInfo* createGroupSortOperatorInfo(SOperatorInfo* downstream, SGroupSort
pInfo->pColMatchInfo = pColMatchColInfo; pInfo->pColMatchInfo = pColMatchColInfo;
pOperator->name = "GroupSortOperator"; pOperator->name = "GroupSortOperator";
pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT;
pOperator->blocking = true; pOperator->blocking = false;
pOperator->status = OP_NOT_OPENED; pOperator->status = OP_NOT_OPENED;
pOperator->info = pInfo; pOperator->info = pInfo;
pOperator->exprSupp.pExprInfo = pExprInfo; pOperator->exprSupp.pExprInfo = pExprInfo;
@ -564,7 +571,8 @@ SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pData
} }
} }
if (p->info.rows > 0) { if (p->info.rows > 0) {// todo extract method
blockDataEnsureCapacity(pDataBlock, p->info.rows);
int32_t numOfCols = taosArrayGetSize(pColMatchInfo); int32_t numOfCols = taosArrayGetSize(pColMatchInfo);
for (int32_t i = 0; i < numOfCols; ++i) { for (int32_t i = 0; i < numOfCols; ++i) {
SColMatchInfo* pmInfo = taosArrayGet(pColMatchInfo, i); SColMatchInfo* pmInfo = taosArrayGet(pColMatchInfo, i);
@ -572,11 +580,10 @@ SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pData
SColumnInfoData* pSrc = taosArrayGet(p->pDataBlock, pmInfo->srcSlotId); SColumnInfoData* pSrc = taosArrayGet(p->pDataBlock, pmInfo->srcSlotId);
SColumnInfoData* pDst = taosArrayGet(pDataBlock->pDataBlock, pmInfo->targetSlotId); SColumnInfoData* pDst = taosArrayGet(pDataBlock->pDataBlock, pmInfo->targetSlotId);
colDataAssign(pDst, pSrc, p->info.rows); colDataAssign(pDst, pSrc, p->info.rows, &pDataBlock->info);
} }
pDataBlock->info.rows = p->info.rows; pDataBlock->info.rows = p->info.rows;
pDataBlock->info.capacity = p->info.rows;
pDataBlock->info.groupId = pInfo->groupId; pDataBlock->info.groupId = pInfo->groupId;
} }

2
source/libs/executor/src/tfill.c
View File

@ -50,7 +50,7 @@ static void setTagsValue(SFillInfo* pFillInfo, void** data, int32_t genRows) {
static void setNullRow(SSDataBlock* pBlock, int32_t numOfCol, int32_t rowIndex) { static void setNullRow(SSDataBlock* pBlock, int32_t numOfCol, int32_t rowIndex) {
// the first are always the timestamp column, so start from the second column. // the first are always the timestamp column, so start from the second column.
for (int32_t i = 1; i < pBlock->info.numOfCols; ++i) { for (int32_t i = 1; i < taosArrayGetSize(pBlock->pDataBlock); ++i) {
SColumnInfoData* p = taosArrayGet(pBlock->pDataBlock, i); SColumnInfoData* p = taosArrayGet(pBlock->pDataBlock, i);
colDataAppendNULL(p, rowIndex); colDataAppendNULL(p, rowIndex);
} }

18
source/libs/executor/src/timewindowoperator.c
View File

@ -1814,7 +1814,7 @@ static int32_t initPrevRowsKeeper(STimeSliceOperatorInfo* pInfo, SSDataBlock* pB
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
} }
int32_t numOfCols = pBlock->info.numOfCols; int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
for (int32_t i = 0; i < numOfCols; ++i) { for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i); SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i);
@ -2231,23 +2231,23 @@ static void clearStreamIntervalOperator(SStreamFinalIntervalOperatorInfo* pInfo)
} }
static void clearUpdateDataBlock(SSDataBlock* pBlock) { static void clearUpdateDataBlock(SSDataBlock* pBlock) {
if (pBlock->info.rows <= 0) {
return;
}
blockDataCleanup(pBlock); blockDataCleanup(pBlock);
} }
void copyUpdateDataBlock(SSDataBlock* pDest, SSDataBlock* pSource, int32_t tsColIndex) { void copyUpdateDataBlock(SSDataBlock* pDest, SSDataBlock* pSource, int32_t tsColIndex) {
ASSERT(pDest->info.capacity >= pSource->info.rows); // ASSERT(pDest->info.capacity >= pSource->info.rows);
blockDataEnsureCapacity(pDest, pSource->info.rows);
clearUpdateDataBlock(pDest); clearUpdateDataBlock(pDest);
SColumnInfoData* pDestCol = taosArrayGet(pDest->pDataBlock, 0); SColumnInfoData* pDestCol = taosArrayGet(pDest->pDataBlock, 0);
SColumnInfoData* pSourceCol = taosArrayGet(pSource->pDataBlock, tsColIndex); SColumnInfoData* pSourceCol = taosArrayGet(pSource->pDataBlock, tsColIndex);
// copy timestamp column // copy timestamp column
colDataAssign(pDestCol, pSourceCol, pSource->info.rows); colDataAssign(pDestCol, pSourceCol, pSource->info.rows, &pDest->info);
for (int32_t i = 1; i < pDest->info.numOfCols; i++) { for (int32_t i = 1; i < taosArrayGetSize(pDest->pDataBlock); i++) {
SColumnInfoData* pCol = taosArrayGet(pDest->pDataBlock, i); SColumnInfoData* pCol = taosArrayGet(pDest->pDataBlock, i);
colDataAppendNNULL(pCol, 0, pSource->info.rows); colDataAppendNNULL(pCol, 0, pSource->info.rows);
} }
pDest->info.rows = pSource->info.rows; pDest->info.rows = pSource->info.rows;
pDest->info.groupId = pSource->info.groupId; pDest->info.groupId = pSource->info.groupId;
pDest->info.type = pSource->info.type; pDest->info.type = pSource->info.type;
@ -2917,7 +2917,7 @@ void doBuildDeleteDataBlock(SHashObj* pStDeleted, SSDataBlock* pBlock, void** It
while (((*Ite) = taosHashIterate(pStDeleted, *Ite)) != NULL) { while (((*Ite) = taosHashIterate(pStDeleted, *Ite)) != NULL) {
SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, 0); SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, 0);
colDataAppend(pColInfoData, pBlock->info.rows, *Ite, false); colDataAppend(pColInfoData, pBlock->info.rows, *Ite, false);
for (int32_t i = 1; i < pBlock->info.numOfCols; i++) { for (int32_t i = 1; i < taosArrayGetSize(pBlock->pDataBlock); i++) {
pColInfoData = taosArrayGet(pBlock->pDataBlock, i); pColInfoData = taosArrayGet(pBlock->pDataBlock, i);
colDataAppendNULL(pColInfoData, pBlock->info.rows); colDataAppendNULL(pColInfoData, pBlock->info.rows);
} }
@ -4117,6 +4117,8 @@ SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SExprI
goto _error; goto _error;
} }
miaInfo->groupIntervalHash = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), true, HASH_NO_LOCK);
SIntervalAggOperatorInfo* iaInfo = &miaInfo->intervalAggOperatorInfo; SIntervalAggOperatorInfo* iaInfo = &miaInfo->intervalAggOperatorInfo;
iaInfo->win = pTaskInfo->window; iaInfo->win = pTaskInfo->window;

8
source/libs/executor/src/tsort.c
View File

@ -147,7 +147,7 @@ static int32_t doAddNewExternalMemSource(SDiskbasedBuf *pBuf, SArray* pAllSource
int32_t rowSize = blockDataGetSerialRowSize(pSource->src.pBlock); int32_t rowSize = blockDataGetSerialRowSize(pSource->src.pBlock);
// The value of numOfRows must be greater than 0, which is guaranteed by the previous memory allocation // The value of numOfRows must be greater than 0, which is guaranteed by the previous memory allocation
int32_t numOfRows = (getBufPageSize(pBuf) - blockDataGetSerialMetaSize(pBlock->info.numOfCols))/rowSize; int32_t numOfRows = (getBufPageSize(pBuf) - blockDataGetSerialMetaSize(taosArrayGetSize(pBlock->pDataBlock)))/rowSize;
ASSERT(numOfRows > 0); ASSERT(numOfRows > 0);
return blockDataEnsureCapacity(pSource->src.pBlock, numOfRows); return blockDataEnsureCapacity(pSource->src.pBlock, numOfRows);
} }
@ -178,7 +178,7 @@ static int32_t doAddToBuf(SSDataBlock* pDataBlock, SSortHandle* pHandle) {
return terrno; return terrno;
} }
int32_t size = blockDataGetSize(p) + sizeof(int32_t) + p->info.numOfCols * sizeof(int32_t); int32_t size = blockDataGetSize(p) + sizeof(int32_t) + taosArrayGetSize(p->pDataBlock) * sizeof(int32_t);
assert(size <= getBufPageSize(pHandle->pBuf)); assert(size <= getBufPageSize(pHandle->pBuf));
blockDataToBuf(pPage, p); blockDataToBuf(pPage, p);
@ -252,7 +252,7 @@ static int32_t sortComparInit(SMsortComparParam* cmpParam, SArray* pSources, int
} }
static void appendOneRowToDataBlock(SSDataBlock *pBlock, const SSDataBlock* pSource, int32_t* rowIndex) { static void appendOneRowToDataBlock(SSDataBlock *pBlock, const SSDataBlock* pSource, int32_t* rowIndex) {
for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { for (int32_t i = 0; i < taosArrayGetSize(pBlock->pDataBlock); ++i) {
SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i); SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i);
SColumnInfoData* pSrcColInfo = taosArrayGet(pSource->pDataBlock, i); SColumnInfoData* pSrcColInfo = taosArrayGet(pSource->pDataBlock, i);
@ -500,7 +500,7 @@ static int32_t doInternalMergeSort(SSortHandle* pHandle) {
return terrno; return terrno;
} }
int32_t size = blockDataGetSize(pDataBlock) + sizeof(int32_t) + pDataBlock->info.numOfCols * sizeof(int32_t); int32_t size = blockDataGetSize(pDataBlock) + sizeof(int32_t) + taosArrayGetSize(pDataBlock->pDataBlock) * sizeof(int32_t);
assert(size <= getBufPageSize(pHandle->pBuf)); assert(size <= getBufPageSize(pHandle->pBuf));
blockDataToBuf(pPage, pDataBlock); blockDataToBuf(pPage, pDataBlock);

42
source/libs/executor/test/executorTests.cpp
View File

@ -62,18 +62,11 @@ SSDataBlock* getDummyBlock(SOperatorInfo* pOperator) {
} }
if (pInfo->pBlock == NULL) { if (pInfo->pBlock == NULL) {
pInfo->pBlock = static_cast<SSDataBlock*>(taosMemoryCalloc(1, sizeof(SSDataBlock))); pInfo->pBlock = createDataBlock();
pInfo->pBlock->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData)); SColumnInfoData colInfo = createColumnInfoData(TSDB_DATA_TYPE_INT, sizeof(int32_t), 1);
blockDataAppendColInfo(pInfo->pBlock, &colInfo);
SColumnInfoData colInfo = {0}; blockDataEnsureCapacity(pInfo->pBlock, pInfo->numOfRowsPerPage);
colInfo.info.type = TSDB_DATA_TYPE_INT;
colInfo.info.bytes = sizeof(int32_t);
colInfo.info.colId = 1;
colInfo.pData = static_cast<char*>(taosMemoryCalloc(pInfo->numOfRowsPerPage, sizeof(int32_t)));
colInfo.nullbitmap = static_cast<char*>(taosMemoryCalloc(1, (pInfo->numOfRowsPerPage + 7) / 8));
taosArrayPush(pInfo->pBlock->pDataBlock, &colInfo);
// SColumnInfoData colInfo1 = {0}; // SColumnInfoData colInfo1 = {0};
// colInfo1.info.type = TSDB_DATA_TYPE_BINARY; // colInfo1.info.type = TSDB_DATA_TYPE_BINARY;
@ -115,7 +108,6 @@ SSDataBlock* getDummyBlock(SOperatorInfo* pOperator) {
} }
pBlock->info.rows = pInfo->numOfRowsPerPage; pBlock->info.rows = pInfo->numOfRowsPerPage;
pBlock->info.numOfCols = 1;
pInfo->current += 1; pInfo->current += 1;
return pBlock; return pBlock;
@ -128,28 +120,15 @@ SSDataBlock* get2ColsDummyBlock(SOperatorInfo* pOperator) {
} }
if (pInfo->pBlock == NULL) { if (pInfo->pBlock == NULL) {
pInfo->pBlock = static_cast<SSDataBlock*>(taosMemoryCalloc(1, sizeof(SSDataBlock))); pInfo->pBlock = createDataBlock();
pInfo->pBlock->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData)); SColumnInfoData colInfo = createColumnInfoData(TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), 1);
blockDataAppendColInfo(pInfo->pBlock, &colInfo);
SColumnInfoData colInfo = {0}; SColumnInfoData colInfo1 = createColumnInfoData(TSDB_DATA_TYPE_INT, 4, 2);
colInfo.info.type = TSDB_DATA_TYPE_TIMESTAMP; blockDataAppendColInfo(pInfo->pBlock, &colInfo1);
colInfo.info.bytes = sizeof(int64_t);
colInfo.info.colId = 1;
colInfo.pData = static_cast<char*>(taosMemoryCalloc(pInfo->numOfRowsPerPage, sizeof(int64_t)));
// colInfo.nullbitmap = static_cast<char*>(taosMemoryCalloc(1, (pInfo->numOfRowsPerPage + 7) / 8));
taosArrayPush(pInfo->pBlock->pDataBlock, &colInfo); blockDataEnsureCapacity(pInfo->pBlock, pInfo->numOfRowsPerPage);
SColumnInfoData colInfo1 = {0};
colInfo1.info.type = TSDB_DATA_TYPE_INT;
colInfo1.info.bytes = 4;
colInfo1.info.colId = 2;
colInfo1.pData = static_cast<char*>(taosMemoryCalloc(pInfo->numOfRowsPerPage, sizeof(int32_t)));
colInfo1.nullbitmap = static_cast<char*>(taosMemoryCalloc(1, (pInfo->numOfRowsPerPage + 7) / 8));
taosArrayPush(pInfo->pBlock->pDataBlock, &colInfo1);
} else { } else {
blockDataCleanup(pInfo->pBlock); blockDataCleanup(pInfo->pBlock);
} }
@ -185,7 +164,6 @@ SSDataBlock* get2ColsDummyBlock(SOperatorInfo* pOperator) {
} }
pBlock->info.rows = pInfo->numOfRowsPerPage; pBlock->info.rows = pInfo->numOfRowsPerPage;
pBlock->info.numOfCols = 1;
pInfo->current += 1; pInfo->current += 1;

24
source/libs/executor/test/sortTests.cpp
View File

@ -62,25 +62,21 @@ SSDataBlock* getSingleColDummyBlock(void* param) {
return NULL; return NULL;
} }
SSDataBlock* pBlock = static_cast<SSDataBlock*>(taosMemoryCalloc(1, sizeof(SSDataBlock))); SSDataBlock* pBlock = createDataBlock();
pBlock->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData));
SColumnInfoData colInfo = {0}; SColumnInfoData colInfo = {0};
colInfo.info.type = pInfo->type; colInfo.info.type = pInfo->type;
if (pInfo->type == TSDB_DATA_TYPE_NCHAR){ if (pInfo->type == TSDB_DATA_TYPE_NCHAR){
colInfo.info.bytes = TSDB_NCHAR_SIZE * VARCOUNT + VARSTR_HEADER_SIZE; colInfo.info.bytes = TSDB_NCHAR_SIZE * VARCOUNT + VARSTR_HEADER_SIZE;
colInfo.varmeta.offset = static_cast<int32_t *>(taosMemoryCalloc(pInfo->pageRows, sizeof(int32_t)));
} else if(pInfo->type == TSDB_DATA_TYPE_BINARY) { } else if(pInfo->type == TSDB_DATA_TYPE_BINARY) {
colInfo.info.bytes = VARCOUNT + VARSTR_HEADER_SIZE; colInfo.info.bytes = VARCOUNT + VARSTR_HEADER_SIZE;
colInfo.varmeta.offset = static_cast<int32_t *>(taosMemoryCalloc(pInfo->pageRows, sizeof(int32_t)));
} else{ } else{
colInfo.info.bytes = tDataTypes[pInfo->type].bytes; colInfo.info.bytes = tDataTypes[pInfo->type].bytes;
colInfo.pData = static_cast<char*>(taosMemoryCalloc(pInfo->pageRows, colInfo.info.bytes));
colInfo.nullbitmap = static_cast<char*>(taosMemoryCalloc(1, (pInfo->pageRows + 7) / 8));
} }
colInfo.info.colId = 1; colInfo.info.colId = 1;
taosArrayPush(pBlock->pDataBlock, &colInfo); blockDataAppendColInfo(pBlock, &colInfo);
blockDataEnsureCapacity(pBlock, pInfo->pageRows);
for (int32_t i = 0; i < pInfo->pageRows; ++i) { for (int32_t i = 0; i < pInfo->pageRows; ++i) {
SColumnInfoData* pColInfo = static_cast<SColumnInfoData*>(TARRAY_GET_ELEM(pBlock->pDataBlock, 0)); SColumnInfoData* pColInfo = static_cast<SColumnInfoData*>(TARRAY_GET_ELEM(pBlock->pDataBlock, 0));
@ -128,7 +124,6 @@ SSDataBlock* getSingleColDummyBlock(void* param) {
} }
pBlock->info.rows = pInfo->pageRows; pBlock->info.rows = pInfo->pageRows;
pBlock->info.numOfCols = 1;
return pBlock; return pBlock;
} }
@ -354,15 +349,10 @@ TEST(testCase, ordered_merge_sort_Test) {
SArray* orderInfo = taosArrayInit(1, sizeof(SBlockOrderInfo)); SArray* orderInfo = taosArrayInit(1, sizeof(SBlockOrderInfo));
taosArrayPush(orderInfo, &oi); taosArrayPush(orderInfo, &oi);
SSDataBlock* pBlock = static_cast<SSDataBlock*>(taosMemoryCalloc(1, sizeof(SSDataBlock))); SSDataBlock* pBlock = createDataBlock();
pBlock->pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData)); for (int32_t i = 0; i < 1; ++i) {
pBlock->info.numOfCols = 1; SColumnInfoData colInfo = createColumnInfoData(TSDB_DATA_TYPE_INT, sizeof(int32_t), 1);
for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { blockDataAppendColInfo(pBlock, &colInfo);
SColumnInfoData colInfo = {0};
colInfo.info.type = TSDB_DATA_TYPE_INT;
colInfo.info.bytes = sizeof(int32_t);
colInfo.info.colId = 1;
taosArrayPush(pBlock->pDataBlock, &colInfo);
} }
SSortHandle* phandle = tsortCreateSortHandle(orderInfo, SORT_MULTISOURCE_MERGE, 1024, 5, pBlock,"test_abc"); SSortHandle* phandle = tsortCreateSortHandle(orderInfo, SORT_MULTISOURCE_MERGE, 1024, 5, pBlock,"test_abc");

13
source/libs/function/inc/taggfunction.h
View File

@ -28,19 +28,6 @@ extern "C" {
#include "function.h" #include "function.h"
#include "tudf.h" #include "tudf.h"
extern SAggFunctionInfo aggFunc[35];
#define FUNCSTATE_SO 0x0u
#define FUNCSTATE_MO 0x1u // dynamic number of output, not multinumber of output e.g., TOP/BOTTOM
#define FUNCSTATE_STREAM 0x2u // function avail for stream
#define FUNCSTATE_STABLE 0x4u // function avail for super table
#define FUNCSTATE_NEED_TS 0x8u // timestamp is required during query processing
#define FUNCSTATE_SELECTIVITY 0x10u // selectivity functions, can exists along with tag columns
#define BASIC_FUNC_SO FUNCSTATE_SO | FUNCSTATE_STREAM | FUNCSTATE_STABLE
#define BASIC_FUNC_MO FUNCSTATE_MO | FUNCSTATE_STREAM | FUNCSTATE_STABLE
#define AVG_FUNCTION_INTER_BUFFER_SIZE 50 #define AVG_FUNCTION_INTER_BUFFER_SIZE 50
#define DATA_SET_FLAG ',' // to denote the output area has data, not null value #define DATA_SET_FLAG ',' // to denote the output area has data, not null value

32
source/libs/function/src/builtins.c
View File

@ -1267,6 +1267,29 @@ static bool validateMinuteRange(int8_t hour, int8_t minute, char sign) {
return false; return false;
} }
static bool validateTimestampDigits(const SValueNode* pVal) {
if (!IS_INTEGER_TYPE(pVal->node.resType.type)) {
return false;
}
int64_t tsVal = pVal->datum.i;
char fraction[20] = {0};
NUM_TO_STRING(pVal->node.resType.type, &tsVal, sizeof(fraction), fraction);
int32_t tsDigits = (int32_t)strlen(fraction);
if (tsDigits > TSDB_TIME_PRECISION_SEC_DIGITS) {
if (tsDigits == TSDB_TIME_PRECISION_MILLI_DIGITS ||
tsDigits == TSDB_TIME_PRECISION_MICRO_DIGITS ||
tsDigits == TSDB_TIME_PRECISION_NANO_DIGITS) {
return true;
} else {
return false;
}
}
return true;
}
static bool validateTimezoneFormat(const SValueNode* pVal) { static bool validateTimezoneFormat(const SValueNode* pVal) {
if (TSDB_DATA_TYPE_BINARY != pVal->node.resType.type) { if (TSDB_DATA_TYPE_BINARY != pVal->node.resType.type) {
return false; return false;
@ -1379,6 +1402,15 @@ static int32_t translateToIso8601(SFunctionNode* pFunc, char* pErrBuf, int32_t l
return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
} }
if (QUERY_NODE_VALUE == nodeType(nodesListGetNode(pFunc->pParameterList, 0))) {
SValueNode* pValue = (SValueNode*)nodesListGetNode(pFunc->pParameterList, 0);
if (!validateTimestampDigits(pValue)) {
pFunc->node.resType = (SDataType){.bytes = 0, .type = TSDB_DATA_TYPE_BINARY};
return TSDB_CODE_SUCCESS;
}
}
// param1 // param1
if (numOfParams == 2) { if (numOfParams == 2) {
SValueNode* pValue = (SValueNode*)nodesListGetNode(pFunc->pParameterList, 1); SValueNode* pValue = (SValueNode*)nodesListGetNode(pFunc->pParameterList, 1);

25
source/libs/function/src/builtinsimpl.c
View File

@ -561,7 +561,7 @@ int32_t sumFunction(SqlFunctionCtx* pCtx) {
// check for overflow // check for overflow
if (IS_FLOAT_TYPE(type) && (isinf(pSumRes->dsum) || isnan(pSumRes->dsum))) { if (IS_FLOAT_TYPE(type) && (isinf(pSumRes->dsum) || isnan(pSumRes->dsum))) {
GET_RES_INFO(pCtx)->isNullRes = 1; numOfElem = 0;
} }
_sum_over: _sum_over:
@ -927,7 +927,7 @@ int32_t avgFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
// check for overflow // check for overflow
if (isinf(pAvgRes->result) || isnan(pAvgRes->result)) { if (isinf(pAvgRes->result) || isnan(pAvgRes->result)) {
GET_RES_INFO(pCtx)->isNullRes = 1; GET_RES_INFO(pCtx)->numOfRes = 0;
} }
return functionFinalize(pCtx, pBlock); return functionFinalize(pCtx, pBlock);
@ -1449,11 +1449,13 @@ void setNullSelectivityValue(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t
void setSelectivityValue(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, const STuplePos* pTuplePos, int32_t rowIndex) { void setSelectivityValue(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, const STuplePos* pTuplePos, int32_t rowIndex) {
int32_t pageId = pTuplePos->pageId; int32_t pageId = pTuplePos->pageId;
int32_t offset = pTuplePos->offset; int32_t offset = pTuplePos->offset;
if (pTuplePos->pageId != -1) { if (pTuplePos->pageId != -1) {
int32_t numOfCols = taosArrayGetSize(pCtx->pSrcBlock->pDataBlock);
SFilePage* pPage = getBufPage(pCtx->pBuf, pageId); SFilePage* pPage = getBufPage(pCtx->pBuf, pageId);
bool* nullList = (bool*)((char*)pPage + offset); bool* nullList = (bool*)((char*)pPage + offset);
char* pStart = (char*)(nullList + pCtx->pSrcBlock->info.numOfCols * sizeof(bool)); char* pStart = (char*)(nullList + numOfCols * sizeof(bool));
// todo set the offset value to optimize the performance. // todo set the offset value to optimize the performance.
for (int32_t j = 0; j < pCtx->subsidiaries.num; ++j) { for (int32_t j = 0; j < pCtx->subsidiaries.num; ++j) {
@ -1774,6 +1776,11 @@ int32_t stddevFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
pStddevRes->result = sqrt(fabs(pStddevRes->quadraticDSum / ((double)pStddevRes->count) - avg * avg)); pStddevRes->result = sqrt(fabs(pStddevRes->quadraticDSum / ((double)pStddevRes->count) - avg * avg));
} }
// check for overflow
if (isinf(pStddevRes->result) || isnan(pStddevRes->result)) {
GET_RES_INFO(pCtx)->numOfRes = 0;
}
return functionFinalize(pCtx, pBlock); return functionFinalize(pCtx, pBlock);
} }
@ -3038,7 +3045,7 @@ void doAddIntoResult(SqlFunctionCtx* pCtx, void* pData, int32_t rowIndex, SSData
void saveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STuplePos* pPos) { void saveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STuplePos* pPos) {
SFilePage* pPage = NULL; SFilePage* pPage = NULL;
int32_t completeRowSize = pSrcBlock->info.rowSize + pSrcBlock->info.numOfCols * sizeof(bool); int32_t completeRowSize = pSrcBlock->info.rowSize + (int32_t) taosArrayGetSize(pSrcBlock->pDataBlock) * sizeof(bool);
if (pCtx->curBufPage == -1) { if (pCtx->curBufPage == -1) {
pPage = getNewBufPage(pCtx->pBuf, 0, &pCtx->curBufPage); pPage = getNewBufPage(pCtx->pBuf, 0, &pCtx->curBufPage);
@ -3056,8 +3063,8 @@ void saveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pS
// keep the current row data, extract method // keep the current row data, extract method
int32_t offset = 0; int32_t offset = 0;
bool* nullList = (bool*)((char*)pPage + pPage->num); bool* nullList = (bool*)((char*)pPage + pPage->num);
char* pStart = (char*)(nullList + sizeof(bool) * pSrcBlock->info.numOfCols); char* pStart = (char*)(nullList + sizeof(bool) * (int32_t) taosArrayGetSize(pSrcBlock->pDataBlock));
for (int32_t i = 0; i < pSrcBlock->info.numOfCols; ++i) { for (int32_t i = 0; i < (int32_t) taosArrayGetSize(pSrcBlock->pDataBlock); ++i) {
SColumnInfoData* pCol = taosArrayGet(pSrcBlock->pDataBlock, i); SColumnInfoData* pCol = taosArrayGet(pSrcBlock->pDataBlock, i);
bool isNull = colDataIsNull_s(pCol, rowIndex); bool isNull = colDataIsNull_s(pCol, rowIndex);
if (isNull) { if (isNull) {
@ -3086,11 +3093,13 @@ void saveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pS
void copyTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STuplePos* pPos) { void copyTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STuplePos* pPos) {
SFilePage* pPage = getBufPage(pCtx->pBuf, pPos->pageId); SFilePage* pPage = getBufPage(pCtx->pBuf, pPos->pageId);
int32_t numOfCols = taosArrayGetSize(pSrcBlock->pDataBlock);
bool* nullList = (bool*)((char*)pPage + pPos->offset); bool* nullList = (bool*)((char*)pPage + pPos->offset);
char* pStart = (char*)(nullList + pSrcBlock->info.numOfCols * sizeof(bool)); char* pStart = (char*)(nullList + numOfCols * sizeof(bool));
int32_t offset = 0; int32_t offset = 0;
for (int32_t i = 0; i < pSrcBlock->info.numOfCols; ++i) { for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pCol = taosArrayGet(pSrcBlock->pDataBlock, i); SColumnInfoData* pCol = taosArrayGet(pSrcBlock->pDataBlock, i);
if ((nullList[i] = colDataIsNull_s(pCol, rowIndex)) == true) { if ((nullList[i] = colDataIsNull_s(pCol, rowIndex)) == true) {
offset += pCol->info.bytes; offset += pCol->info.bytes;

45
source/libs/function/src/tudf.c
View File

@ -763,8 +763,8 @@ void freeUdfInterBuf(SUdfInterBuf *buf) {
int32_t convertDataBlockToUdfDataBlock(SSDataBlock *block, SUdfDataBlock *udfBlock) { int32_t convertDataBlockToUdfDataBlock(SSDataBlock *block, SUdfDataBlock *udfBlock) {
udfBlock->numOfRows = block->info.rows; udfBlock->numOfRows = block->info.rows;
udfBlock->numOfCols = block->info.numOfCols; udfBlock->numOfCols = taosArrayGetSize(block->pDataBlock);
udfBlock->udfCols = taosMemoryCalloc(udfBlock->numOfCols, sizeof(SUdfColumn*)); udfBlock->udfCols = taosMemoryCalloc(taosArrayGetSize(block->pDataBlock), sizeof(SUdfColumn*));
for (int32_t i = 0; i < udfBlock->numOfCols; ++i) { for (int32_t i = 0; i < udfBlock->numOfCols; ++i) {
udfBlock->udfCols[i] = taosMemoryCalloc(1, sizeof(SUdfColumn)); udfBlock->udfCols[i] = taosMemoryCalloc(1, sizeof(SUdfColumn));
SColumnInfoData *col= (SColumnInfoData*)taosArrayGet(block->pDataBlock, i); SColumnInfoData *col= (SColumnInfoData*)taosArrayGet(block->pDataBlock, i);
@ -799,7 +799,6 @@ int32_t convertDataBlockToUdfDataBlock(SSDataBlock *block, SUdfDataBlock *udfBlo
} }
int32_t convertUdfColumnToDataBlock(SUdfColumn *udfCol, SSDataBlock *block) { int32_t convertUdfColumnToDataBlock(SUdfColumn *udfCol, SSDataBlock *block) {
block->info.numOfCols = 1;
block->info.rows = udfCol->colData.numOfRows; block->info.rows = udfCol->colData.numOfRows;
block->info.hasVarCol = IS_VAR_DATA_TYPE(udfCol->colMeta.type); block->info.hasVarCol = IS_VAR_DATA_TYPE(udfCol->colMeta.type);
@ -830,25 +829,19 @@ int32_t convertUdfColumnToDataBlock(SUdfColumn *udfCol, SSDataBlock *block) {
int32_t convertScalarParamToDataBlock(SScalarParam *input, int32_t numOfCols, SSDataBlock *output) { int32_t convertScalarParamToDataBlock(SScalarParam *input, int32_t numOfCols, SSDataBlock *output) {
output->info.rows = input->numOfRows; output->info.rows = input->numOfRows;
output->info.numOfCols = numOfCols;
bool hasVarCol = false;
for (int32_t i = 0; i < numOfCols; ++i) {
if (IS_VAR_DATA_TYPE((input+i)->columnData->info.type)) {
hasVarCol = true;
break;
}
}
output->info.hasVarCol = hasVarCol;
output->pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); output->pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData));
for (int32_t i = 0; i < numOfCols; ++i) { for (int32_t i = 0; i < numOfCols; ++i) {
taosArrayPush(output->pDataBlock, (input + i)->columnData); taosArrayPush(output->pDataBlock, (input + i)->columnData);
if (IS_VAR_DATA_TYPE((input+i)->columnData->info.type)) {
output->info.hasVarCol = true;
}
} }
return 0; return 0;
} }
int32_t convertDataBlockToScalarParm(SSDataBlock *input, SScalarParam *output) { int32_t convertDataBlockToScalarParm(SSDataBlock *input, SScalarParam *output) {
if (input->info.numOfCols != 1) { if (taosArrayGetSize(input->pDataBlock) != 1) {
fnError("scalar function only support one column"); fnError("scalar function only support one column");
return -1; return -1;
} }
@ -1089,24 +1082,14 @@ int32_t udfAggProcess(struct SqlFunctionCtx *pCtx) {
int32_t start = pInput->startRowIndex; int32_t start = pInput->startRowIndex;
int32_t numOfRows = pInput->numOfRows; int32_t numOfRows = pInput->numOfRows;
SSDataBlock* pTempBlock = createDataBlock();
SSDataBlock tempBlock = {0}; pTempBlock->info.rows = pInput->totalRows;
tempBlock.info.numOfCols = numOfCols; pTempBlock->info.uid = pInput->uid;
tempBlock.info.rows = pInput->totalRows;
tempBlock.info.uid = pInput->uid;
bool hasVarCol = false;
tempBlock.pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData));
for (int32_t i = 0; i < numOfCols; ++i) { for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData *col = pInput->pData[i]; blockDataAppendColInfo(pTempBlock, pInput->pData[i]);
if (IS_VAR_DATA_TYPE(col->info.type)) {
hasVarCol = true;
}
taosArrayPush(tempBlock.pDataBlock, col);
} }
tempBlock.info.hasVarCol = hasVarCol;
SSDataBlock *inputBlock = blockDataExtractBlock(&tempBlock, start, numOfRows); SSDataBlock *inputBlock = blockDataExtractBlock(pTempBlock, start, numOfRows);
SUdfInterBuf state = {.buf = udfRes->interResBuf, SUdfInterBuf state = {.buf = udfRes->interResBuf,
.bufLen = session->bufSize, .bufLen = session->bufSize,
@ -1131,7 +1114,9 @@ int32_t udfAggProcess(struct SqlFunctionCtx *pCtx) {
} }
blockDataDestroy(inputBlock); blockDataDestroy(inputBlock);
taosArrayDestroy(tempBlock.pDataBlock);
taosArrayDestroy(pTempBlock->pDataBlock);
taosMemoryFree(pTempBlock);
releaseUdfFuncHandle(pCtx->udfName); releaseUdfFuncHandle(pCtx->udfName);
freeUdfInterBuf(&newState); freeUdfInterBuf(&newState);

5
source/libs/function/src/udfd.c
View File

@ -613,7 +613,9 @@ int32_t udfdOpenClientRpc() {
} }
int32_t udfdCloseClientRpc() { int32_t udfdCloseClientRpc() {
fnInfo("udfd begin closing rpc");
rpcClose(global.clientRpc); rpcClose(global.clientRpc);
fnInfo("udfd finish closing rpc");
return 0; return 0;
} }
@ -937,9 +939,8 @@ int main(int argc, char *argv[]) {
uv_thread_create(&mnodeConnectThread, udfdConnectMnodeThreadFunc, NULL); uv_thread_create(&mnodeConnectThread, udfdConnectMnodeThreadFunc, NULL);
udfdRun(); udfdRun();
removeListeningPipe(); removeListeningPipe();
uv_thread_join(&mnodeConnectThread);
udfdCloseClientRpc(); udfdCloseClientRpc();
return 0; return 0;

50
source/libs/function/test/runUdf.c
View File

@ -45,27 +45,27 @@ int scalarFuncTest() {
for (int k = 0; k < 1; ++k) { for (int k = 0; k < 1; ++k) {
SSDataBlock block = {0}; SSDataBlock block = {0};
SSDataBlock *pBlock = &block; SSDataBlock *pBlock = &block;
pBlock->pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData)); for (int32_t i = 0; i < 1; ++i) {
pBlock->info.numOfCols = 1; SColumnInfoData colInfo = createColumnInfoData(TSDB_DATA_TYPE_INT, sizeof(int32_t), 1);
blockDataAppendColInfo(pBlock, &colInfo);
}
blockDataEnsureCapacity(pBlock, 1024);
pBlock->info.rows = 1024; pBlock->info.rows = 1024;
for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) {
SColumnInfoData colInfo = {0}; SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, 0);
colInfo.info.type = TSDB_DATA_TYPE_INT; for (int32_t j = 0; j < pBlock->info.rows; ++j) {
colInfo.info.bytes = sizeof(int32_t); colDataAppendInt32(pCol, j, &j);
colInfo.info.colId = 1;
colInfoDataEnsureCapacity(&colInfo, 0, pBlock->info.rows);
for (int32_t j = 0; j < pBlock->info.rows; ++j) {
colDataAppendInt32(&colInfo, j, &j);
}
taosArrayPush(pBlock->pDataBlock, &colInfo);
} }
SScalarParam input = {0}; SScalarParam input = {0};
input.numOfRows = pBlock->info.rows; input.numOfRows = pBlock->info.rows;
input.columnData = taosArrayGet(pBlock->pDataBlock, 0); input.columnData = taosArrayGet(pBlock->pDataBlock, 0);
SScalarParam output = {0}; SScalarParam output = {0};
doCallUdfScalarFunc(handle, &input, 1, &output); doCallUdfScalarFunc(handle, &input, 1, &output);
taosArrayDestroy(pBlock->pDataBlock); taosArrayDestroy(pBlock->pDataBlock);
SColumnInfoData *col = output.columnData; SColumnInfoData *col = output.columnData;
for (int32_t i = 0; i < output.numOfRows; ++i) { for (int32_t i = 0; i < output.numOfRows; ++i) {
if (i % 100 == 0) if (i % 100 == 0)
@ -89,21 +89,18 @@ int aggregateFuncTest() {
return -1; return -1;
} }
SSDataBlock block = {0}; SSDataBlock *pBlock = createDataBlock();
SSDataBlock *pBlock = &block; for (int32_t i = 0; i < taosArrayGetSize(pBlock->pDataBlock); ++i) {
pBlock->pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData)); SColumnInfoData colInfo = createColumnInfoData(TSDB_DATA_TYPE_INT, sizeof(int32_t), 1);
pBlock->info.numOfCols = 1; blockDataAppendColInfo(pBlock, &colInfo);
}
blockDataEnsureCapacity(pBlock, 1024);
pBlock->info.rows = 1024; pBlock->info.rows = 1024;
for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) {
SColumnInfoData colInfo = {0}; SColumnInfoData* pColInfo = bdGetColumnInfoData(pBlock, 0);
colInfo.info.type = TSDB_DATA_TYPE_INT; for (int32_t j = 0; j < pBlock->info.rows; ++j) {
colInfo.info.bytes = sizeof(int32_t); colDataAppendInt32(pColInfo, j, &j);
colInfo.info.colId = 1;
colInfoDataEnsureCapacity(&colInfo, 0, pBlock->info.rows);
for (int32_t j = 0; j < pBlock->info.rows; ++j) {
colDataAppendInt32(&colInfo, j, &j);
}
taosArrayPush(pBlock->pDataBlock, &colInfo);
} }
SUdfInterBuf buf = {0}; SUdfInterBuf buf = {0};
@ -121,6 +118,7 @@ int aggregateFuncTest() {
freeUdfInterBuf(&resultBuf); freeUdfInterBuf(&resultBuf);
doTeardownUdf(handle); doTeardownUdf(handle);
blockDataDestroy(pBlock);
return 0; return 0;
} }

2
source/libs/index/test/index_executor_tests.cpp
View File

@ -86,7 +86,7 @@ void sifAppendReservedSlot(SArray *pBlockList, int16_t *dataBlockId, int16_t *sl
SColumnInfoData idata = {0}; SColumnInfoData idata = {0};
idata.info = *colInfo; idata.info = *colInfo;
colInfoDataEnsureCapacity(&idata, 0, rows); colInfoDataEnsureCapacity(&idata, rows);
taosArrayPush(res->pDataBlock, &idata); taosArrayPush(res->pDataBlock, &idata);

25
source/libs/monitor/src/monMain.c
View File

@ -201,6 +201,9 @@ static void monGenClusterJson(SMonInfo *pMonitor) {
tjsonAddStringToObject(pJson, "version", pInfo->version); tjsonAddStringToObject(pJson, "version", pInfo->version);
tjsonAddDoubleToObject(pJson, "master_uptime", pInfo->master_uptime); tjsonAddDoubleToObject(pJson, "master_uptime", pInfo->master_uptime);
tjsonAddDoubleToObject(pJson, "monitor_interval", pInfo->monitor_interval); tjsonAddDoubleToObject(pJson, "monitor_interval", pInfo->monitor_interval);
tjsonAddDoubleToObject(pJson, "dbs_total", pInfo->dbs_total);
tjsonAddDoubleToObject(pJson, "tbs_total", pInfo->tbs_total);
tjsonAddDoubleToObject(pJson, "stbs_total", pInfo->stbs_total);
tjsonAddDoubleToObject(pJson, "vgroups_total", pInfo->vgroups_total); tjsonAddDoubleToObject(pJson, "vgroups_total", pInfo->vgroups_total);
tjsonAddDoubleToObject(pJson, "vgroups_alive", pInfo->vgroups_alive); tjsonAddDoubleToObject(pJson, "vgroups_alive", pInfo->vgroups_alive);
tjsonAddDoubleToObject(pJson, "vnodes_total", pInfo->vnodes_total); tjsonAddDoubleToObject(pJson, "vnodes_total", pInfo->vnodes_total);
@ -277,6 +280,27 @@ static void monGenVgroupJson(SMonInfo *pMonitor) {
} }
} }
static void monGenStbJson(SMonInfo *pMonitor) {
SMonStbInfo *pInfo = &pMonitor->mmInfo.stb;
if (pMonitor->mmInfo.cluster.first_ep_dnode_id == 0) return;
SJson *pJson = tjsonAddArrayToObject(pMonitor->pJson, "stb_infos");
if (pJson == NULL) return;
for (int32_t i = 0; i < taosArrayGetSize(pInfo->stbs); ++i) {
SJson *pStbJson = tjsonCreateObject();
if (pStbJson == NULL) continue;
if (tjsonAddItemToArray(pJson, pStbJson) != 0) {
tjsonDelete(pStbJson);
continue;
}
SMonStbDesc *pStbDesc = taosArrayGet(pInfo->stbs, i);
tjsonAddStringToObject(pStbJson, "stb_name", pStbDesc->stb_name);
tjsonAddStringToObject(pStbJson, "database_name", pStbDesc->database_name);
}
}
static void monGenGrantJson(SMonInfo *pMonitor) { static void monGenGrantJson(SMonInfo *pMonitor) {
SMonGrantInfo *pInfo = &pMonitor->mmInfo.grant; SMonGrantInfo *pInfo = &pMonitor->mmInfo.grant;
if (pMonitor->mmInfo.cluster.first_ep_dnode_id == 0) return; if (pMonitor->mmInfo.cluster.first_ep_dnode_id == 0) return;
@ -524,6 +548,7 @@ void monSendReport() {
monGenBasicJson(pMonitor); monGenBasicJson(pMonitor);
monGenClusterJson(pMonitor); monGenClusterJson(pMonitor);
monGenVgroupJson(pMonitor); monGenVgroupJson(pMonitor);
monGenStbJson(pMonitor);
monGenGrantJson(pMonitor); monGenGrantJson(pMonitor);
monGenDnodeJson(pMonitor); monGenDnodeJson(pMonitor);
monGenDiskJson(pMonitor); monGenDiskJson(pMonitor);

36
source/libs/monitor/src/monMsg.c
View File

@ -102,6 +102,9 @@ int32_t tEncodeSMonClusterInfo(SEncoder *encoder, const SMonClusterInfo *pInfo)
if (tEncodeCStr(encoder, pInfo->version) < 0) return -1; if (tEncodeCStr(encoder, pInfo->version) < 0) return -1;
if (tEncodeFloat(encoder, pInfo->master_uptime) < 0) return -1; if (tEncodeFloat(encoder, pInfo->master_uptime) < 0) return -1;
if (tEncodeI32(encoder, pInfo->monitor_interval) < 0) return -1; if (tEncodeI32(encoder, pInfo->monitor_interval) < 0) return -1;
if (tEncodeI32(encoder, pInfo->dbs_total) < 0) return -1;
if (tEncodeI32(encoder, pInfo->stbs_total) < 0) return -1;
if (tEncodeI64(encoder, pInfo->tbs_total) < 0) return -1;
if (tEncodeI32(encoder, pInfo->vgroups_total) < 0) return -1; if (tEncodeI32(encoder, pInfo->vgroups_total) < 0) return -1;
if (tEncodeI32(encoder, pInfo->vgroups_alive) < 0) return -1; if (tEncodeI32(encoder, pInfo->vgroups_alive) < 0) return -1;
if (tEncodeI32(encoder, pInfo->vnodes_total) < 0) return -1; if (tEncodeI32(encoder, pInfo->vnodes_total) < 0) return -1;
@ -130,6 +133,9 @@ int32_t tDecodeSMonClusterInfo(SDecoder *decoder, SMonClusterInfo *pInfo) {
if (tDecodeCStrTo(decoder, pInfo->version) < 0) return -1; if (tDecodeCStrTo(decoder, pInfo->version) < 0) return -1;
if (tDecodeFloat(decoder, &pInfo->master_uptime) < 0) return -1; if (tDecodeFloat(decoder, &pInfo->master_uptime) < 0) return -1;
if (tDecodeI32(decoder, &pInfo->monitor_interval) < 0) return -1; if (tDecodeI32(decoder, &pInfo->monitor_interval) < 0) return -1;
if (tDecodeI32(decoder, &pInfo->dbs_total) < 0) return -1;
if (tDecodeI32(decoder, &pInfo->stbs_total) < 0) return -1;
if (tDecodeI64(decoder, &pInfo->tbs_total) < 0) return -1;
if (tDecodeI32(decoder, &pInfo->vgroups_total) < 0) return -1; if (tDecodeI32(decoder, &pInfo->vgroups_total) < 0) return -1;
if (tDecodeI32(decoder, &pInfo->vgroups_alive) < 0) return -1; if (tDecodeI32(decoder, &pInfo->vgroups_alive) < 0) return -1;
if (tDecodeI32(decoder, &pInfo->vnodes_total) < 0) return -1; if (tDecodeI32(decoder, &pInfo->vnodes_total) < 0) return -1;
@ -203,6 +209,32 @@ int32_t tDecodeSMonVgroupInfo(SDecoder *decoder, SMonVgroupInfo *pInfo) {
return 0; return 0;
} }
int32_t tEncodeSMonStbInfo(SEncoder *encoder, const SMonStbInfo *pInfo) {
if (tEncodeI32(encoder, taosArrayGetSize(pInfo->stbs)) < 0) return -1;
for (int32_t i = 0; i < taosArrayGetSize(pInfo->stbs); ++i) {
SMonStbDesc *pDesc = taosArrayGet(pInfo->stbs, i);
if (tEncodeCStr(encoder, pDesc->stb_name) < 0) return -1;
if (tEncodeCStr(encoder, pDesc->database_name) < 0) return -1;
}
return 0;
}
int32_t tDecodeSMonStbInfo(SDecoder *decoder, SMonStbInfo *pInfo) {
int32_t arraySize = 0;
if (tDecodeI32(decoder, &arraySize) < 0) return -1;
pInfo->stbs = taosArrayInit(arraySize, sizeof(SMonStbDesc));
if (pInfo->stbs == NULL) return -1;
for (int32_t i = 0; i < arraySize; ++i) {
SMonStbDesc desc = {0};
if (tDecodeCStrTo(decoder, desc.stb_name) < 0) return -1;
if (tDecodeCStrTo(decoder, desc.database_name) < 0) return -1;
taosArrayPush(pInfo->stbs, &desc);
}
return 0;
}
int32_t tEncodeSMonGrantInfo(SEncoder *encoder, const SMonGrantInfo *pInfo) { int32_t tEncodeSMonGrantInfo(SEncoder *encoder, const SMonGrantInfo *pInfo) {
if (tEncodeI32(encoder, pInfo->expire_time) < 0) return -1; if (tEncodeI32(encoder, pInfo->expire_time) < 0) return -1;
if (tEncodeI64(encoder, pInfo->timeseries_used) < 0) return -1; if (tEncodeI64(encoder, pInfo->timeseries_used) < 0) return -1;
@ -224,6 +256,7 @@ int32_t tSerializeSMonMmInfo(void *buf, int32_t bufLen, SMonMmInfo *pInfo) {
if (tStartEncode(&encoder) < 0) return -1; if (tStartEncode(&encoder) < 0) return -1;
if (tEncodeSMonClusterInfo(&encoder, &pInfo->cluster) < 0) return -1; if (tEncodeSMonClusterInfo(&encoder, &pInfo->cluster) < 0) return -1;
if (tEncodeSMonVgroupInfo(&encoder, &pInfo->vgroup) < 0) return -1; if (tEncodeSMonVgroupInfo(&encoder, &pInfo->vgroup) < 0) return -1;
if (tEncodeSMonStbInfo(&encoder, &pInfo->stb) < 0) return -1;
if (tEncodeSMonGrantInfo(&encoder, &pInfo->grant) < 0) return -1; if (tEncodeSMonGrantInfo(&encoder, &pInfo->grant) < 0) return -1;
if (tEncodeSMonSysInfo(&encoder, &pInfo->sys) < 0) return -1; if (tEncodeSMonSysInfo(&encoder, &pInfo->sys) < 0) return -1;
if (tEncodeSMonLogs(&encoder, &pInfo->log) < 0) return -1; if (tEncodeSMonLogs(&encoder, &pInfo->log) < 0) return -1;
@ -241,6 +274,7 @@ int32_t tDeserializeSMonMmInfo(void *buf, int32_t bufLen, SMonMmInfo *pInfo) {
if (tStartDecode(&decoder) < 0) return -1; if (tStartDecode(&decoder) < 0) return -1;
if (tDecodeSMonClusterInfo(&decoder, &pInfo->cluster) < 0) return -1; if (tDecodeSMonClusterInfo(&decoder, &pInfo->cluster) < 0) return -1;
if (tDecodeSMonVgroupInfo(&decoder, &pInfo->vgroup) < 0) return -1; if (tDecodeSMonVgroupInfo(&decoder, &pInfo->vgroup) < 0) return -1;
if (tDecodeSMonStbInfo(&decoder, &pInfo->stb) < 0) return -1;
if (tDecodeSMonGrantInfo(&decoder, &pInfo->grant) < 0) return -1; if (tDecodeSMonGrantInfo(&decoder, &pInfo->grant) < 0) return -1;
if (tDecodeSMonSysInfo(&decoder, &pInfo->sys) < 0) return -1; if (tDecodeSMonSysInfo(&decoder, &pInfo->sys) < 0) return -1;
if (tDecodeSMonLogs(&decoder, &pInfo->log) < 0) return -1; if (tDecodeSMonLogs(&decoder, &pInfo->log) < 0) return -1;
@ -255,9 +289,11 @@ void tFreeSMonMmInfo(SMonMmInfo *pInfo) {
taosArrayDestroy(pInfo->cluster.mnodes); taosArrayDestroy(pInfo->cluster.mnodes);
taosArrayDestroy(pInfo->cluster.dnodes); taosArrayDestroy(pInfo->cluster.dnodes);
taosArrayDestroy(pInfo->vgroup.vgroups); taosArrayDestroy(pInfo->vgroup.vgroups);
taosArrayDestroy(pInfo->stb.stbs);
pInfo->cluster.mnodes = NULL; pInfo->cluster.mnodes = NULL;
pInfo->cluster.dnodes = NULL; pInfo->cluster.dnodes = NULL;
pInfo->vgroup.vgroups = NULL; pInfo->vgroup.vgroups = NULL;
pInfo->stb.stbs = NULL;
pInfo->log.logs = NULL; pInfo->log.logs = NULL;
} }

2
source/libs/scalar/inc/sclInt.h
View File

@ -57,7 +57,7 @@ typedef struct SScalarCtx {
#define SCL_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _return; } } while (0) #define SCL_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _return; } } while (0)
int32_t doConvertDataType(SValueNode* pValueNode, SScalarParam* out); int32_t doConvertDataType(SValueNode* pValueNode, SScalarParam* out);
SColumnInfoData* createColumnInfoData(SDataType* pType, int32_t numOfRows); SColumnInfoData* sclCreateColumnInfoData(SDataType* pType, int32_t numOfRows);
int32_t sclConvertToTsValueNode(int8_t precision, SValueNode* valueNode); int32_t sclConvertToTsValueNode(int8_t precision, SValueNode* valueNode);
#define GET_PARAM_TYPE(_c) ((_c)->columnData ? (_c)->columnData->info.type : (_c)->hashValueType) #define GET_PARAM_TYPE(_c) ((_c)->columnData ? (_c)->columnData->info.type : (_c)->hashValueType)

2
source/libs/scalar/src/filter.c
View File

@ -3821,7 +3821,7 @@ bool filterExecute(SFilterInfo *info, SSDataBlock *pSrc, int8_t** p, SColumnData
SScalarParam output = {0}; SScalarParam output = {0};
SDataType type = {.type = TSDB_DATA_TYPE_BOOL, .bytes = sizeof(bool)}; SDataType type = {.type = TSDB_DATA_TYPE_BOOL, .bytes = sizeof(bool)};
output.columnData = createColumnInfoData(&type, pSrc->info.rows); output.columnData = sclCreateColumnInfoData(&type, pSrc->info.rows);
SArray *pList = taosArrayInit(1, POINTER_BYTES); SArray *pList = taosArrayInit(1, POINTER_BYTES);
taosArrayPush(pList, &pSrc); taosArrayPush(pList, &pSrc);

25
source/libs/scalar/src/scalar.c
View File

@ -36,7 +36,7 @@ int32_t sclConvertToTsValueNode(int8_t precision, SValueNode* valueNode) {
} }
SColumnInfoData* createColumnInfoData(SDataType* pType, int32_t numOfRows) { SColumnInfoData* sclCreateColumnInfoData(SDataType* pType, int32_t numOfRows) {
SColumnInfoData* pColumnData = taosMemoryCalloc(1, sizeof(SColumnInfoData)); SColumnInfoData* pColumnData = taosMemoryCalloc(1, sizeof(SColumnInfoData));
if (pColumnData == NULL) { if (pColumnData == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
@ -48,7 +48,7 @@ SColumnInfoData* createColumnInfoData(SDataType* pType, int32_t numOfRows) {
pColumnData->info.scale = pType->scale; pColumnData->info.scale = pType->scale;
pColumnData->info.precision = pType->precision; pColumnData->info.precision = pType->precision;
int32_t code = colInfoDataEnsureCapacity(pColumnData, 0, numOfRows); int32_t code = colInfoDataEnsureCapacity(pColumnData, numOfRows);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
taosMemoryFree(pColumnData); taosMemoryFree(pColumnData);
@ -60,10 +60,10 @@ SColumnInfoData* createColumnInfoData(SDataType* pType, int32_t numOfRows) {
int32_t doConvertDataType(SValueNode* pValueNode, SScalarParam* out) { int32_t doConvertDataType(SValueNode* pValueNode, SScalarParam* out) {
SScalarParam in = {.numOfRows = 1}; SScalarParam in = {.numOfRows = 1};
in.columnData = createColumnInfoData(&pValueNode->node.resType, 1); in.columnData = sclCreateColumnInfoData(&pValueNode->node.resType, 1);
colDataAppend(in.columnData, 0, nodesGetValueFromNode(pValueNode), false); colDataAppend(in.columnData, 0, nodesGetValueFromNode(pValueNode), false);
colInfoDataEnsureCapacity(out->columnData, 0, 1); colInfoDataEnsureCapacity(out->columnData, 1);
int32_t code = vectorConvertImpl(&in, out); int32_t code = vectorConvertImpl(&in, out);
sclFreeParam(&in); sclFreeParam(&in);
@ -193,7 +193,7 @@ int32_t sclInitParam(SNode* node, SScalarParam *param, SScalarCtx *ctx, int32_t
SValueNode *valueNode = (SValueNode *)node; SValueNode *valueNode = (SValueNode *)node;
param->numOfRows = 1; param->numOfRows = 1;
param->columnData = createColumnInfoData(&valueNode->node.resType, 1); param->columnData = sclCreateColumnInfoData(&valueNode->node.resType, 1);
if (TSDB_DATA_TYPE_NULL == valueNode->node.resType.type) { if (TSDB_DATA_TYPE_NULL == valueNode->node.resType.type) {
colDataAppendNULL(param->columnData, 0); colDataAppendNULL(param->columnData, 0);
} else { } else {
@ -245,7 +245,7 @@ int32_t sclInitParam(SNode* node, SScalarParam *param, SScalarCtx *ctx, int32_t
} }
SSDataBlock *block = *(SSDataBlock **)taosArrayGet(ctx->pBlockList, index); SSDataBlock *block = *(SSDataBlock **)taosArrayGet(ctx->pBlockList, index);
if (NULL == block || ref->slotId >= block->info.numOfCols) { if (NULL == block || ref->slotId >= taosArrayGetSize(block->pDataBlock)) {
sclError("column slotId is too big, slodId:%d, dataBlockSize:%d", ref->slotId, (int32_t)taosArrayGetSize(block->pDataBlock)); sclError("column slotId is too big, slodId:%d, dataBlockSize:%d", ref->slotId, (int32_t)taosArrayGetSize(block->pDataBlock));
SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
} }
@ -431,7 +431,7 @@ int32_t sclExecFunction(SFunctionNode *node, SScalarCtx *ctx, SScalarParam *outp
SCL_ERR_JRET(code); SCL_ERR_JRET(code);
} }
output->columnData = createColumnInfoData(&node->node.resType, rowNum); output->columnData = sclCreateColumnInfoData(&node->node.resType, rowNum);
if (output->columnData == NULL) { if (output->columnData == NULL) {
sclError("calloc %d failed", (int32_t)(rowNum * output->columnData->info.bytes)); sclError("calloc %d failed", (int32_t)(rowNum * output->columnData->info.bytes));
SCL_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); SCL_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY);
@ -484,7 +484,7 @@ int32_t sclExecLogic(SLogicConditionNode *node, SScalarCtx *ctx, SScalarParam *o
output->numOfRows = rowNum; output->numOfRows = rowNum;
SDataType t = {.type = type, .bytes = tDataTypes[type].bytes}; SDataType t = {.type = type, .bytes = tDataTypes[type].bytes};
output->columnData = createColumnInfoData(&t, rowNum); output->columnData = sclCreateColumnInfoData(&t, rowNum);
if (output->columnData == NULL) { if (output->columnData == NULL) {
sclError("calloc %d failed", (int32_t)(rowNum * sizeof(bool))); sclError("calloc %d failed", (int32_t)(rowNum * sizeof(bool)));
SCL_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); SCL_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY);
@ -539,7 +539,7 @@ int32_t sclExecOperator(SOperatorNode *node, SScalarCtx *ctx, SScalarParam *outp
int32_t code = 0; int32_t code = 0;
SCL_ERR_RET(sclInitOperatorParams(&params, node, ctx, &rowNum)); SCL_ERR_RET(sclInitOperatorParams(&params, node, ctx, &rowNum));
output->columnData = createColumnInfoData(&node->node.resType, rowNum); output->columnData = sclCreateColumnInfoData(&node->node.resType, rowNum);
if (output->columnData == NULL) { if (output->columnData == NULL) {
sclError("calloc failed, size:%d", (int32_t)rowNum * node->node.resType.bytes); sclError("calloc failed, size:%d", (int32_t)rowNum * node->node.resType.bytes);
SCL_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); SCL_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY);
@ -901,7 +901,7 @@ EDealRes sclWalkTarget(SNode* pNode, SScalarCtx *ctx) {
return DEAL_RES_ERROR; return DEAL_RES_ERROR;
} }
colDataAssign(col, res->columnData, res->numOfRows); colDataAssign(col, res->columnData, res->numOfRows, NULL);
block->info.rows = res->numOfRows; block->info.rows = res->numOfRows;
sclFreeParam(res); sclFreeParam(res);
@ -982,8 +982,9 @@ int32_t scalarCalculate(SNode *pNode, SArray *pBlockList, SScalarParam *pDst) {
sclError("no valid res in hash, node:%p, type:%d", pNode, nodeType(pNode)); sclError("no valid res in hash, node:%p, type:%d", pNode, nodeType(pNode));
SCL_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); SCL_ERR_JRET(TSDB_CODE_QRY_APP_ERROR);
} }
colDataAssign(pDst->columnData, res->columnData, res->numOfRows); colInfoDataEnsureCapacity(pDst->columnData, res->numOfRows);
colDataAssign(pDst->columnData, res->columnData, res->numOfRows, NULL);
pDst->numOfRows = res->numOfRows; pDst->numOfRows = res->numOfRows;
taosHashRemove(ctx.pRes, (void *)&pNode, POINTER_BYTES); taosHashRemove(ctx.pRes, (void *)&pNode, POINTER_BYTES);
} }

11
source/libs/scalar/src/sclfunc.c
View File

@ -120,7 +120,7 @@ int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutpu
} }
default: { default: {
colDataAssign(pOutputData, pInputData, pInput->numOfRows); colDataAssign(pOutputData, pInputData, pInput->numOfRows, NULL);
} }
} }
@ -273,7 +273,7 @@ static int32_t doScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarP
} }
default: { default: {
colDataAssign(pOutputData, pInputData, pInput->numOfRows); colDataAssign(pOutputData, pInputData, pInput->numOfRows, NULL);
} }
} }
@ -856,10 +856,13 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
int32_t toISO8601Function(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { int32_t toISO8601Function(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
int32_t type = GET_PARAM_TYPE(pInput); int32_t type = GET_PARAM_TYPE(pInput);
bool tzPresent = (inputNum == 2) ? true : false;
char* tz; char* tz;
int32_t tzLen; int32_t tzLen;
tz = varDataVal(pInput[1].columnData->pData); if (tzPresent) {
tzLen = varDataLen(pInput[1].columnData->pData); tz = varDataVal(pInput[1].columnData->pData);
tzLen = varDataLen(pInput[1].columnData->pData);
}
for (int32_t i = 0; i < pInput[0].numOfRows; ++i) { for (int32_t i = 0; i < pInput[0].numOfRows; ++i) {
if (colDataIsNull_s(pInput[0].columnData, i)) { if (colDataIsNull_s(pInput[0].columnData, i)) {

4
source/libs/scalar/src/sclvector.c
View File

@ -852,7 +852,7 @@ int32_t vectorConvertScalarParam(SScalarParam *input, SScalarParam *output, int3
SDataType t = {.type = type, .bytes = tDataTypes[type].bytes}; SDataType t = {.type = type, .bytes = tDataTypes[type].bytes};
output->numOfRows = input->numOfRows; output->numOfRows = input->numOfRows;
output->columnData = createColumnInfoData(&t, input->numOfRows); output->columnData = sclCreateColumnInfoData(&t, input->numOfRows);
if (output->columnData == NULL) { if (output->columnData == NULL) {
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
} }
@ -923,7 +923,7 @@ static int32_t doConvertHelper(SScalarParam* pDest, int32_t* convert, const SSca
pDest->numOfRows = pParam->numOfRows; pDest->numOfRows = pParam->numOfRows;
SDataType t = {.type = type, .bytes = tDataTypes[type].bytes}; SDataType t = {.type = type, .bytes = tDataTypes[type].bytes};
pDest->columnData = createColumnInfoData(&t, pParam->numOfRows); pDest->columnData = sclCreateColumnInfoData(&t, pParam->numOfRows);
if (pDest->columnData == NULL) { if (pDest->columnData == NULL) {
sclError("malloc %d failed", (int32_t)(pParam->numOfRows * sizeof(double))); sclError("malloc %d failed", (int32_t)(pParam->numOfRows * sizeof(double)));
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;

282
source/libs/scalar/test/filter/filterTests.cpp
View File

@ -105,29 +105,14 @@ void flttMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in
} }
if (NULL == *block) { if (NULL == *block) {
SSDataBlock *res = (SSDataBlock *)taosMemoryCalloc(1, sizeof(SSDataBlock)); SSDataBlock *res = createDataBlock();
res->info.numOfCols = 3;
res->info.rows = rowNum;
res->pDataBlock = taosArrayInit(3, sizeof(SColumnInfoData));
for (int32_t i = 0; i < 2; ++i) { for (int32_t i = 0; i < 2; ++i) {
SColumnInfoData idata = {{0}}; SColumnInfoData idata = createColumnInfoData(TSDB_DATA_TYPE_NULL, 10, 1+i);
idata.info.type = TSDB_DATA_TYPE_NULL; blockDataAppendColInfo(res, &idata);
idata.info.bytes = 10;
idata.info.colId = i + 1;
int32_t size = idata.info.bytes * rowNum;
idata.pData = (char *)taosMemoryCalloc(1, size);
taosArrayPush(res->pDataBlock, &idata);
} }
SColumnInfoData idata = {{0}}; SColumnInfoData idata = createColumnInfoData(dataType, dataBytes, 3);
idata.info.type = dataType; blockDataAppendColInfo(res, &idata);
idata.info.bytes = dataBytes;
idata.info.colId = 3;
int32_t size = idata.info.bytes * rowNum;
idata.pData = (char *)taosMemoryCalloc(1, size);
taosArrayPush(res->pDataBlock, &idata);
blockDataEnsureCapacity(res, rowNum); blockDataEnsureCapacity(res, rowNum);
SColumnInfoData *pColumn = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); SColumnInfoData *pColumn = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock);
@ -142,23 +127,18 @@ void flttMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in
rnode->slotId = 2; rnode->slotId = 2;
rnode->colId = 3; rnode->colId = 3;
res->info.rows = rowNum;
*block = res; *block = res;
} else { } else {
SSDataBlock *res = *block; SSDataBlock *res = *block;
int32_t idx = taosArrayGetSize(res->pDataBlock); int32_t idx = taosArrayGetSize(res->pDataBlock);
SColumnInfoData idata = {{0}}; SColumnInfoData idata = createColumnInfoData(dataType, dataBytes, 1+idx);
idata.info.type = dataType; blockDataAppendColInfo(res, &idata);
idata.info.bytes = dataBytes; blockDataEnsureCapacity(res, rowNum);
idata.info.colId = 1 + idx;
int32_t size = idata.info.bytes * rowNum;
idata.pData = (char *)taosMemoryCalloc(1, size);
taosArrayPush(res->pDataBlock, &idata);
res->info.numOfCols++;
SColumnInfoData *pColumn = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); SColumnInfoData *pColumn = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock);
colInfoDataEnsureCapacity(pColumn, 0, rowNum);
for (int32_t i = 0; i < rowNum; ++i) { for (int32_t i = 0; i < rowNum; ++i) {
colDataAppend(pColumn, i, (const char *)value, false); colDataAppend(pColumn, i, (const char *)value, false);
@ -399,7 +379,7 @@ TEST(columnTest, smallint_column_greater_double_value) {
keep = filterRangeExecute(filter, &stat, 1, rowNum); keep = filterRangeExecute(filter, &stat, 1, rowNum);
ASSERT_EQ(keep, true); ASSERT_EQ(keep, true);
SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; SFilterColumnParam param = { (int32_t) taosArrayGetSize(src->pDataBlock), src->pDataBlock };
code = filterSetDataFromSlotId(filter, &param); code = filterSetDataFromSlotId(filter, &param);
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
@ -407,9 +387,9 @@ TEST(columnTest, smallint_column_greater_double_value) {
stat.min = 1; stat.min = 1;
stat.numOfNull = 0; stat.numOfNull = 0;
int8_t *rowRes = NULL; int8_t *rowRes = NULL;
keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); keep = filterExecute(filter, src, &rowRes, &stat, (int32_t) taosArrayGetSize(src->pDataBlock));
ASSERT_EQ(keep, false); ASSERT_EQ(keep, false);
for (int32_t i = 0; i < rowNum; ++i) { for (int32_t i = 0; i < rowNum; ++i) {
ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]);
} }
@ -430,7 +410,7 @@ TEST(columnTest, int_column_greater_smallint_value) {
flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, leftv); flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, leftv);
flttMakeValueNode(&pRight, TSDB_DATA_TYPE_SMALLINT, &rightv); flttMakeValueNode(&pRight, TSDB_DATA_TYPE_SMALLINT, &rightv);
flttMakeOpNode(&opNode, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pLeft, pRight); flttMakeOpNode(&opNode, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pLeft, pRight);
SFilterInfo *filter = NULL; SFilterInfo *filter = NULL;
int32_t code = filterInitFromNode(opNode, &filter, 0); int32_t code = filterInitFromNode(opNode, &filter, 0);
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
@ -454,7 +434,7 @@ TEST(columnTest, int_column_greater_smallint_value) {
keep = filterRangeExecute(filter, &stat, 1, rowNum); keep = filterRangeExecute(filter, &stat, 1, rowNum);
ASSERT_EQ(keep, false); ASSERT_EQ(keep, false);
SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; SFilterColumnParam param = { (int32_t) taosArrayGetSize(src->pDataBlock), src->pDataBlock };
code = filterSetDataFromSlotId(filter, &param); code = filterSetDataFromSlotId(filter, &param);
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
@ -462,9 +442,9 @@ TEST(columnTest, int_column_greater_smallint_value) {
stat.min = 1; stat.min = 1;
stat.numOfNull = 0; stat.numOfNull = 0;
int8_t *rowRes = NULL; int8_t *rowRes = NULL;
keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); keep = filterExecute(filter, src, &rowRes, &stat, (int32_t) taosArrayGetSize(src->pDataBlock));
ASSERT_EQ(keep, false); ASSERT_EQ(keep, false);
for (int32_t i = 0; i < rowNum; ++i) { for (int32_t i = 0; i < rowNum; ++i) {
ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]);
} }
@ -478,11 +458,11 @@ TEST(columnTest, int_column_in_double_list) {
SNode *pLeft = NULL, *pRight = NULL, *listNode = NULL, *opNode = NULL; SNode *pLeft = NULL, *pRight = NULL, *listNode = NULL, *opNode = NULL;
int32_t leftv[5] = {1, 2, 3, 4, 5}; int32_t leftv[5] = {1, 2, 3, 4, 5};
double rightv1 = 1.1,rightv2 = 2.2,rightv3 = 3.3; double rightv1 = 1.1,rightv2 = 2.2,rightv3 = 3.3;
bool eRes[5] = {true, true, true, false, false}; bool eRes[5] = {true, true, true, false, false};
SSDataBlock *src = NULL; SSDataBlock *src = NULL;
SScalarParam res = {0}; SScalarParam res = {0};
int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]);
flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, leftv); flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, leftv);
SNodeList* list = nodesMakeList(); SNodeList* list = nodesMakeList();
flttMakeValueNode(&pRight, TSDB_DATA_TYPE_DOUBLE, &rightv1); flttMakeValueNode(&pRight, TSDB_DATA_TYPE_DOUBLE, &rightv1);
nodesListAppend(list, pRight); nodesListAppend(list, pRight);
@ -498,7 +478,7 @@ TEST(columnTest, int_column_in_double_list) {
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
SColumnDataAgg stat = {0}; SColumnDataAgg stat = {0};
SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; SFilterColumnParam param = { (int32_t) taosArrayGetSize(src->pDataBlock), src->pDataBlock };
code = filterSetDataFromSlotId(filter, &param); code = filterSetDataFromSlotId(filter, &param);
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
@ -506,9 +486,9 @@ TEST(columnTest, int_column_in_double_list) {
stat.min = 1; stat.min = 1;
stat.numOfNull = 0; stat.numOfNull = 0;
int8_t *rowRes = NULL; int8_t *rowRes = NULL;
bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); bool keep = filterExecute(filter, src, &rowRes, &stat, (int32_t) taosArrayGetSize(src->pDataBlock));
ASSERT_EQ(keep, false); ASSERT_EQ(keep, false);
for (int32_t i = 0; i < rowNum; ++i) { for (int32_t i = 0; i < rowNum; ++i) {
ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]);
} }
@ -521,8 +501,8 @@ TEST(columnTest, int_column_in_double_list) {
TEST(columnTest, binary_column_in_binary_list) { TEST(columnTest, binary_column_in_binary_list) {
SNode *pLeft = NULL, *pRight = NULL, *listNode = NULL, *opNode = NULL; SNode *pLeft = NULL, *pRight = NULL, *listNode = NULL, *opNode = NULL;
bool eRes[5] = {true, true, false, false, false}; bool eRes[5] = {true, true, false, false, false};
SSDataBlock *src = NULL; SSDataBlock *src = NULL;
SScalarParam res = {0}; SScalarParam res = {0};
char leftv[5][5]= {0}; char leftv[5][5]= {0};
char rightv[3][5]= {0}; char rightv[3][5]= {0};
@ -531,22 +511,22 @@ TEST(columnTest, binary_column_in_binary_list) {
leftv[i][3] = 'b' + i; leftv[i][3] = 'b' + i;
leftv[i][4] = '0' + i; leftv[i][4] = '0' + i;
varDataSetLen(leftv[i], 3); varDataSetLen(leftv[i], 3);
} }
for (int32_t i = 0; i < 2; ++i) { for (int32_t i = 0; i < 2; ++i) {
rightv[i][2] = 'a' + i; rightv[i][2] = 'a' + i;
rightv[i][3] = 'b' + i; rightv[i][3] = 'b' + i;
rightv[i][4] = '0' + i; rightv[i][4] = '0' + i;
varDataSetLen(rightv[i], 3); varDataSetLen(rightv[i], 3);
} }
for (int32_t i = 2; i < 3; ++i) { for (int32_t i = 2; i < 3; ++i) {
rightv[i][2] = 'a' + i; rightv[i][2] = 'a' + i;
rightv[i][3] = 'a' + i; rightv[i][3] = 'a' + i;
rightv[i][4] = 'a' + i; rightv[i][4] = 'a' + i;
varDataSetLen(rightv[i], 3); varDataSetLen(rightv[i], 3);
} }
int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]);
flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv);
SNodeList* list = nodesMakeList(); SNodeList* list = nodesMakeList();
flttMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, rightv[0]); flttMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, rightv[0]);
nodesListAppend(list, pRight); nodesListAppend(list, pRight);
@ -556,13 +536,13 @@ TEST(columnTest, binary_column_in_binary_list) {
nodesListAppend(list, pRight); nodesListAppend(list, pRight);
flttMakeListNode(&listNode,list, TSDB_DATA_TYPE_BINARY); flttMakeListNode(&listNode,list, TSDB_DATA_TYPE_BINARY);
flttMakeOpNode(&opNode, OP_TYPE_IN, TSDB_DATA_TYPE_BOOL, pLeft, listNode); flttMakeOpNode(&opNode, OP_TYPE_IN, TSDB_DATA_TYPE_BOOL, pLeft, listNode);
SFilterInfo *filter = NULL; SFilterInfo *filter = NULL;
int32_t code = filterInitFromNode(opNode, &filter, 0); int32_t code = filterInitFromNode(opNode, &filter, 0);
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
SColumnDataAgg stat = {0}; SColumnDataAgg stat = {0};
SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; SFilterColumnParam param = { (int32_t) taosArrayGetSize(src->pDataBlock), src->pDataBlock };
code = filterSetDataFromSlotId(filter, &param); code = filterSetDataFromSlotId(filter, &param);
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
@ -570,9 +550,9 @@ TEST(columnTest, binary_column_in_binary_list) {
stat.min = 1; stat.min = 1;
stat.numOfNull = 0; stat.numOfNull = 0;
int8_t *rowRes = NULL; int8_t *rowRes = NULL;
bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); bool keep = filterExecute(filter, src, &rowRes, &stat, (int32_t) taosArrayGetSize(src->pDataBlock));
ASSERT_EQ(keep, false); ASSERT_EQ(keep, false);
for (int32_t i = 0; i < rowNum; ++i) { for (int32_t i = 0; i < rowNum; ++i) {
ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]);
} }
@ -586,31 +566,31 @@ TEST(columnTest, binary_column_like_binary) {
SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL; SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL;
char rightv[64] = {0}; char rightv[64] = {0};
char leftv[5][5]= {0}; char leftv[5][5]= {0};
SSDataBlock *src = NULL; SSDataBlock *src = NULL;
SScalarParam res = {0}; SScalarParam res = {0};
bool eRes[5] = {true, false, true, false, true}; bool eRes[5] = {true, false, true, false, true};
for (int32_t i = 0; i < 5; ++i) { for (int32_t i = 0; i < 5; ++i) {
leftv[i][2] = 'a'; leftv[i][2] = 'a';
leftv[i][3] = 'a'; leftv[i][3] = 'a';
leftv[i][4] = '0' + i % 2; leftv[i][4] = '0' + i % 2;
varDataSetLen(leftv[i], 3); varDataSetLen(leftv[i], 3);
} }
int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]);
flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv);
sprintf(&rightv[2], "%s", "__0"); sprintf(&rightv[2], "%s", "__0");
varDataSetLen(rightv, strlen(&rightv[2])); varDataSetLen(rightv, strlen(&rightv[2]));
flttMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, rightv); flttMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, rightv);
flttMakeOpNode(&opNode, OP_TYPE_LIKE, TSDB_DATA_TYPE_BOOL, pLeft, pRight); flttMakeOpNode(&opNode, OP_TYPE_LIKE, TSDB_DATA_TYPE_BOOL, pLeft, pRight);
SFilterInfo *filter = NULL; SFilterInfo *filter = NULL;
int32_t code = filterInitFromNode(opNode, &filter, 0); int32_t code = filterInitFromNode(opNode, &filter, 0);
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
SColumnDataAgg stat = {0}; SColumnDataAgg stat = {0};
SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; SFilterColumnParam param = { (int32_t) taosArrayGetSize(src->pDataBlock), src->pDataBlock };
code = filterSetDataFromSlotId(filter, &param); code = filterSetDataFromSlotId(filter, &param);
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
@ -618,9 +598,9 @@ TEST(columnTest, binary_column_like_binary) {
stat.min = 1; stat.min = 1;
stat.numOfNull = 0; stat.numOfNull = 0;
int8_t *rowRes = NULL; int8_t *rowRes = NULL;
bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); bool keep = filterExecute(filter, src, &rowRes, &stat, (int32_t) taosArrayGetSize(src->pDataBlock));
ASSERT_EQ(keep, false); ASSERT_EQ(keep, false);
for (int32_t i = 0; i < rowNum; ++i) { for (int32_t i = 0; i < rowNum; ++i) {
ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]);
} }
@ -633,31 +613,31 @@ TEST(columnTest, binary_column_like_binary) {
TEST(columnTest, binary_column_is_null) { TEST(columnTest, binary_column_is_null) {
SNode *pLeft = NULL, *opNode = NULL; SNode *pLeft = NULL, *opNode = NULL;
char leftv[5][5]= {0}; char leftv[5][5]= {0};
SSDataBlock *src = NULL; SSDataBlock *src = NULL;
SScalarParam res = {0}; SScalarParam res = {0};
bool eRes[5] = {false, false, true, false, true}; bool eRes[5] = {false, false, true, false, true};
for (int32_t i = 0; i < 5; ++i) { for (int32_t i = 0; i < 5; ++i) {
leftv[i][2] = '0' + i % 2; leftv[i][2] = '0' + i % 2;
leftv[i][3] = 'a'; leftv[i][3] = 'a';
leftv[i][4] = '0' + i % 2; leftv[i][4] = '0' + i % 2;
varDataSetLen(leftv[i], 3); varDataSetLen(leftv[i], 3);
} }
int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]);
flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv);
SColumnInfoData *pcolumn = (SColumnInfoData *)taosArrayGetLast(src->pDataBlock); SColumnInfoData *pcolumn = (SColumnInfoData *)taosArrayGetLast(src->pDataBlock);
colDataAppend(pcolumn, 2, NULL, true); colDataAppend(pcolumn, 2, NULL, true);
colDataAppend(pcolumn, 4, NULL, true); colDataAppend(pcolumn, 4, NULL, true);
flttMakeOpNode(&opNode, OP_TYPE_IS_NULL, TSDB_DATA_TYPE_BOOL, pLeft, NULL); flttMakeOpNode(&opNode, OP_TYPE_IS_NULL, TSDB_DATA_TYPE_BOOL, pLeft, NULL);
SFilterInfo *filter = NULL; SFilterInfo *filter = NULL;
int32_t code = filterInitFromNode(opNode, &filter, 0); int32_t code = filterInitFromNode(opNode, &filter, 0);
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
SColumnDataAgg stat = {0}; SColumnDataAgg stat = {0};
SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; SFilterColumnParam param = { (int32_t) taosArrayGetSize(src->pDataBlock), src->pDataBlock };
code = filterSetDataFromSlotId(filter, &param); code = filterSetDataFromSlotId(filter, &param);
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
@ -665,9 +645,9 @@ TEST(columnTest, binary_column_is_null) {
stat.min = 1; stat.min = 1;
stat.numOfNull = 0; stat.numOfNull = 0;
int8_t *rowRes = NULL; int8_t *rowRes = NULL;
bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); bool keep = filterExecute(filter, src, &rowRes, &stat, (int32_t) taosArrayGetSize(src->pDataBlock));
ASSERT_EQ(keep, false); ASSERT_EQ(keep, false);
for (int32_t i = 0; i < rowNum; ++i) { for (int32_t i = 0; i < rowNum; ++i) {
ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]);
} }
@ -680,31 +660,31 @@ TEST(columnTest, binary_column_is_null) {
TEST(columnTest, binary_column_is_not_null) { TEST(columnTest, binary_column_is_not_null) {
SNode *pLeft = NULL, *opNode = NULL; SNode *pLeft = NULL, *opNode = NULL;
char leftv[5][5]= {0}; char leftv[5][5]= {0};
SSDataBlock *src = NULL; SSDataBlock *src = NULL;
SScalarParam res = {0}; SScalarParam res = {0};
bool eRes[5] = {true, true, true, true, false}; bool eRes[5] = {true, true, true, true, false};
for (int32_t i = 0; i < 5; ++i) { for (int32_t i = 0; i < 5; ++i) {
leftv[i][2] = '0' + i % 2; leftv[i][2] = '0' + i % 2;
leftv[i][3] = 'a'; leftv[i][3] = 'a';
leftv[i][4] = '0' + i % 2; leftv[i][4] = '0' + i % 2;
varDataSetLen(leftv[i], 3); varDataSetLen(leftv[i], 3);
} }
int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]);
flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv);
SColumnInfoData *pcolumn = (SColumnInfoData *)taosArrayGetLast(src->pDataBlock); SColumnInfoData *pcolumn = (SColumnInfoData *)taosArrayGetLast(src->pDataBlock);
colDataAppend(pcolumn, 4, NULL, true); colDataAppend(pcolumn, 4, NULL, true);
flttMakeOpNode(&opNode, OP_TYPE_IS_NOT_NULL, TSDB_DATA_TYPE_BOOL, pLeft, NULL); flttMakeOpNode(&opNode, OP_TYPE_IS_NOT_NULL, TSDB_DATA_TYPE_BOOL, pLeft, NULL);
SFilterInfo *filter = NULL; SFilterInfo *filter = NULL;
int32_t code = filterInitFromNode(opNode, &filter, 0); int32_t code = filterInitFromNode(opNode, &filter, 0);
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
SColumnDataAgg stat = {0}; SColumnDataAgg stat = {0};
SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; SFilterColumnParam param = { (int32_t) taosArrayGetSize(src->pDataBlock), src->pDataBlock };
code = filterSetDataFromSlotId(filter, &param); code = filterSetDataFromSlotId(filter, &param);
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
@ -712,9 +692,9 @@ TEST(columnTest, binary_column_is_not_null) {
stat.min = 1; stat.min = 1;
stat.numOfNull = 0; stat.numOfNull = 0;
int8_t *rowRes = NULL; int8_t *rowRes = NULL;
bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); bool keep = filterExecute(filter, src, &rowRes, &stat, (int32_t) taosArrayGetSize(src->pDataBlock));
ASSERT_EQ(keep, false); ASSERT_EQ(keep, false);
for (int32_t i = 0; i < rowNum; ++i) { for (int32_t i = 0; i < rowNum; ++i) {
ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]);
} }
@ -735,13 +715,13 @@ TEST(opTest, smallint_column_greater_int_column) {
flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv); flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv);
flttMakeColumnNode(&pRight, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, rightv); flttMakeColumnNode(&pRight, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, rightv);
flttMakeOpNode(&opNode, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pLeft, pRight); flttMakeOpNode(&opNode, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pLeft, pRight);
SFilterInfo *filter = NULL; SFilterInfo *filter = NULL;
int32_t code = filterInitFromNode(opNode, &filter, 0); int32_t code = filterInitFromNode(opNode, &filter, 0);
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
SColumnDataAgg stat = {0}; SColumnDataAgg stat = {0};
SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; SFilterColumnParam param = { (int32_t) taosArrayGetSize(src->pDataBlock), src->pDataBlock };
code = filterSetDataFromSlotId(filter, &param); code = filterSetDataFromSlotId(filter, &param);
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
@ -749,9 +729,9 @@ TEST(opTest, smallint_column_greater_int_column) {
stat.min = 1; stat.min = 1;
stat.numOfNull = 0; stat.numOfNull = 0;
int8_t *rowRes = NULL; int8_t *rowRes = NULL;
bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); bool keep = filterExecute(filter, src, &rowRes, &stat, (int32_t) taosArrayGetSize(src->pDataBlock));
ASSERT_EQ(keep, false); ASSERT_EQ(keep, false);
for (int32_t i = 0; i < rowNum; ++i) { for (int32_t i = 0; i < rowNum; ++i) {
ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]);
} }
@ -773,13 +753,13 @@ TEST(opTest, smallint_value_add_int_column) {
flttMakeColumnNode(&pRight, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, rightv); flttMakeColumnNode(&pRight, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, rightv);
flttMakeOpNode(&opNode, OP_TYPE_ADD, TSDB_DATA_TYPE_DOUBLE, pLeft, pRight); flttMakeOpNode(&opNode, OP_TYPE_ADD, TSDB_DATA_TYPE_DOUBLE, pLeft, pRight);
flttMakeOpNode(&opNode, OP_TYPE_IS_TRUE, TSDB_DATA_TYPE_BOOL, opNode, NULL); flttMakeOpNode(&opNode, OP_TYPE_IS_TRUE, TSDB_DATA_TYPE_BOOL, opNode, NULL);
SFilterInfo *filter = NULL; SFilterInfo *filter = NULL;
int32_t code = filterInitFromNode(opNode, &filter, 0); int32_t code = filterInitFromNode(opNode, &filter, 0);
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
SColumnDataAgg stat = {0}; SColumnDataAgg stat = {0};
SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; SFilterColumnParam param = { (int32_t) taosArrayGetSize(src->pDataBlock), src->pDataBlock };
code = filterSetDataFromSlotId(filter, &param); code = filterSetDataFromSlotId(filter, &param);
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
@ -787,9 +767,9 @@ TEST(opTest, smallint_value_add_int_column) {
stat.min = 1; stat.min = 1;
stat.numOfNull = 0; stat.numOfNull = 0;
int8_t *rowRes = NULL; int8_t *rowRes = NULL;
bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); bool keep = filterExecute(filter, src, &rowRes, &stat, (int32_t) taosArrayGetSize(src->pDataBlock));
ASSERT_EQ(keep, false); ASSERT_EQ(keep, false);
for (int32_t i = 0; i < rowNum; ++i) { for (int32_t i = 0; i < rowNum; ++i) {
ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]);
} }
@ -816,13 +796,13 @@ TEST(opTest, bigint_column_multi_binary_column) {
flttMakeColumnNode(&pRight, &src, TSDB_DATA_TYPE_BINARY, 5, rowNum, rightv); flttMakeColumnNode(&pRight, &src, TSDB_DATA_TYPE_BINARY, 5, rowNum, rightv);
flttMakeOpNode(&opNode, OP_TYPE_MULTI, TSDB_DATA_TYPE_DOUBLE, pLeft, pRight); flttMakeOpNode(&opNode, OP_TYPE_MULTI, TSDB_DATA_TYPE_DOUBLE, pLeft, pRight);
flttMakeOpNode(&opNode, OP_TYPE_IS_TRUE, TSDB_DATA_TYPE_BOOL, opNode, NULL); flttMakeOpNode(&opNode, OP_TYPE_IS_TRUE, TSDB_DATA_TYPE_BOOL, opNode, NULL);
SFilterInfo *filter = NULL; SFilterInfo *filter = NULL;
int32_t code = filterInitFromNode(opNode, &filter, 0); int32_t code = filterInitFromNode(opNode, &filter, 0);
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
SColumnDataAgg stat = {0}; SColumnDataAgg stat = {0};
SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; SFilterColumnParam param = { (int32_t) taosArrayGetSize(src->pDataBlock), src->pDataBlock };
code = filterSetDataFromSlotId(filter, &param); code = filterSetDataFromSlotId(filter, &param);
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
@ -830,9 +810,9 @@ TEST(opTest, bigint_column_multi_binary_column) {
stat.min = 1; stat.min = 1;
stat.numOfNull = 0; stat.numOfNull = 0;
int8_t *rowRes = NULL; int8_t *rowRes = NULL;
bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); bool keep = filterExecute(filter, src, &rowRes, &stat, (int32_t) taosArrayGetSize(src->pDataBlock));
ASSERT_EQ(keep, false); ASSERT_EQ(keep, false);
for (int32_t i = 0; i < rowNum; ++i) { for (int32_t i = 0; i < rowNum; ++i) {
ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]);
} }
@ -859,13 +839,13 @@ TEST(opTest, smallint_column_and_binary_column) {
flttMakeColumnNode(&pRight, &src, TSDB_DATA_TYPE_BINARY, 5, rowNum, rightv); flttMakeColumnNode(&pRight, &src, TSDB_DATA_TYPE_BINARY, 5, rowNum, rightv);
flttMakeOpNode(&opNode, OP_TYPE_BIT_AND, TSDB_DATA_TYPE_BIGINT, pLeft, pRight); flttMakeOpNode(&opNode, OP_TYPE_BIT_AND, TSDB_DATA_TYPE_BIGINT, pLeft, pRight);
flttMakeOpNode(&opNode, OP_TYPE_IS_TRUE, TSDB_DATA_TYPE_BOOL, opNode, NULL); flttMakeOpNode(&opNode, OP_TYPE_IS_TRUE, TSDB_DATA_TYPE_BOOL, opNode, NULL);
SFilterInfo *filter = NULL; SFilterInfo *filter = NULL;
int32_t code = filterInitFromNode(opNode, &filter, 0); int32_t code = filterInitFromNode(opNode, &filter, 0);
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
SColumnDataAgg stat = {0}; SColumnDataAgg stat = {0};
SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; SFilterColumnParam param = { (int32_t) taosArrayGetSize(src->pDataBlock), src->pDataBlock };
code = filterSetDataFromSlotId(filter, &param); code = filterSetDataFromSlotId(filter, &param);
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
@ -873,9 +853,9 @@ TEST(opTest, smallint_column_and_binary_column) {
stat.min = 1; stat.min = 1;
stat.numOfNull = 0; stat.numOfNull = 0;
int8_t *rowRes = NULL; int8_t *rowRes = NULL;
bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); bool keep = filterExecute(filter, src, &rowRes, &stat, (int32_t) taosArrayGetSize(src->pDataBlock));
ASSERT_EQ(keep, false); ASSERT_EQ(keep, false);
for (int32_t i = 0; i < rowNum; ++i) { for (int32_t i = 0; i < rowNum; ++i) {
ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]);
} }
@ -897,13 +877,13 @@ TEST(opTest, smallint_column_or_float_column) {
flttMakeColumnNode(&pRight, &src, TSDB_DATA_TYPE_FLOAT, sizeof(float), rowNum, rightv); flttMakeColumnNode(&pRight, &src, TSDB_DATA_TYPE_FLOAT, sizeof(float), rowNum, rightv);
flttMakeOpNode(&opNode, OP_TYPE_BIT_OR, TSDB_DATA_TYPE_BIGINT, pLeft, pRight); flttMakeOpNode(&opNode, OP_TYPE_BIT_OR, TSDB_DATA_TYPE_BIGINT, pLeft, pRight);
flttMakeOpNode(&opNode, OP_TYPE_IS_TRUE, TSDB_DATA_TYPE_BOOL, opNode, NULL); flttMakeOpNode(&opNode, OP_TYPE_IS_TRUE, TSDB_DATA_TYPE_BOOL, opNode, NULL);
SFilterInfo *filter = NULL; SFilterInfo *filter = NULL;
int32_t code = filterInitFromNode(opNode, &filter, 0); int32_t code = filterInitFromNode(opNode, &filter, 0);
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
SColumnDataAgg stat = {0}; SColumnDataAgg stat = {0};
SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; SFilterColumnParam param = { (int32_t)taosArrayGetSize(src->pDataBlock), src->pDataBlock };
code = filterSetDataFromSlotId(filter, &param); code = filterSetDataFromSlotId(filter, &param);
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
@ -911,9 +891,9 @@ TEST(opTest, smallint_column_or_float_column) {
stat.min = 1; stat.min = 1;
stat.numOfNull = 0; stat.numOfNull = 0;
int8_t *rowRes = NULL; int8_t *rowRes = NULL;
bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); bool keep = filterExecute(filter, src, &rowRes, &stat, taosArrayGetSize(src->pDataBlock));
ASSERT_EQ(keep, false); ASSERT_EQ(keep, false);
for (int32_t i = 0; i < rowNum; ++i) { for (int32_t i = 0; i < rowNum; ++i) {
ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]);
} }
@ -935,13 +915,13 @@ TEST(opTest, smallint_column_or_double_value) {
flttMakeValueNode(&pRight, TSDB_DATA_TYPE_DOUBLE, &rightv); flttMakeValueNode(&pRight, TSDB_DATA_TYPE_DOUBLE, &rightv);
flttMakeOpNode(&opNode, OP_TYPE_BIT_OR, TSDB_DATA_TYPE_BIGINT, pLeft, pRight); flttMakeOpNode(&opNode, OP_TYPE_BIT_OR, TSDB_DATA_TYPE_BIGINT, pLeft, pRight);
flttMakeOpNode(&opNode, OP_TYPE_IS_TRUE, TSDB_DATA_TYPE_BOOL, opNode, NULL); flttMakeOpNode(&opNode, OP_TYPE_IS_TRUE, TSDB_DATA_TYPE_BOOL, opNode, NULL);
SFilterInfo *filter = NULL; SFilterInfo *filter = NULL;
int32_t code = filterInitFromNode(opNode, &filter, 0); int32_t code = filterInitFromNode(opNode, &filter, 0);
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
SColumnDataAgg stat = {0}; SColumnDataAgg stat = {0};
SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; SFilterColumnParam param = { (int32_t)taosArrayGetSize(src->pDataBlock), src->pDataBlock };
code = filterSetDataFromSlotId(filter, &param); code = filterSetDataFromSlotId(filter, &param);
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
@ -949,9 +929,9 @@ TEST(opTest, smallint_column_or_double_value) {
stat.min = 1; stat.min = 1;
stat.numOfNull = 0; stat.numOfNull = 0;
int8_t *rowRes = NULL; int8_t *rowRes = NULL;
bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); bool keep = filterExecute(filter, src, &rowRes, &stat, taosArrayGetSize(src->pDataBlock));
ASSERT_EQ(keep, true); ASSERT_EQ(keep, true);
for (int32_t i = 0; i < rowNum; ++i) { for (int32_t i = 0; i < rowNum; ++i) {
ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]);
} }
@ -964,28 +944,28 @@ TEST(opTest, smallint_column_or_double_value) {
TEST(opTest, binary_column_is_true) { TEST(opTest, binary_column_is_true) {
SNode *pLeft = NULL, *opNode = NULL; SNode *pLeft = NULL, *opNode = NULL;
char leftv[5][5]= {0}; char leftv[5][5]= {0};
SSDataBlock *src = NULL; SSDataBlock *src = NULL;
SScalarParam res = {0}; SScalarParam res = {0};
bool eRes[5] = {false, true, false, true, false}; bool eRes[5] = {false, true, false, true, false};
for (int32_t i = 0; i < 5; ++i) { for (int32_t i = 0; i < 5; ++i) {
leftv[i][2] = '0' + i % 2; leftv[i][2] = '0' + i % 2;
leftv[i][3] = 'a'; leftv[i][3] = 'a';
leftv[i][4] = '0' + i % 2; leftv[i][4] = '0' + i % 2;
varDataSetLen(leftv[i], 3); varDataSetLen(leftv[i], 3);
} }
int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]);
flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv);
flttMakeOpNode(&opNode, OP_TYPE_IS_TRUE, TSDB_DATA_TYPE_BOOL, pLeft, NULL); flttMakeOpNode(&opNode, OP_TYPE_IS_TRUE, TSDB_DATA_TYPE_BOOL, pLeft, NULL);
SFilterInfo *filter = NULL; SFilterInfo *filter = NULL;
int32_t code = filterInitFromNode(opNode, &filter, 0); int32_t code = filterInitFromNode(opNode, &filter, 0);
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
SColumnDataAgg stat = {0}; SColumnDataAgg stat = {0};
SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; SFilterColumnParam param = { (int32_t)taosArrayGetSize(src->pDataBlock), src->pDataBlock };
code = filterSetDataFromSlotId(filter, &param); code = filterSetDataFromSlotId(filter, &param);
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
@ -993,9 +973,9 @@ TEST(opTest, binary_column_is_true) {
stat.min = 1; stat.min = 1;
stat.numOfNull = 0; stat.numOfNull = 0;
int8_t *rowRes = NULL; int8_t *rowRes = NULL;
bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); bool keep = filterExecute(filter, src, &rowRes, &stat, taosArrayGetSize(src->pDataBlock));
ASSERT_EQ(keep, false); ASSERT_EQ(keep, false);
for (int32_t i = 0; i < rowNum; ++i) { for (int32_t i = 0; i < rowNum; ++i) {
ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]);
} }
@ -1027,12 +1007,12 @@ TEST(filterModelogicTest, diff_columns_and_or_and) {
flttMakeValueNode(&pRight2, TSDB_DATA_TYPE_INT, &rightv2); flttMakeValueNode(&pRight2, TSDB_DATA_TYPE_INT, &rightv2);
flttMakeOpNode(&opNode2, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft2, pRight2); flttMakeOpNode(&opNode2, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft2, pRight2);
nodesListAppend(list, opNode2); nodesListAppend(list, opNode2);
flttMakeLogicNodeFromList(&logicNode1, LOGIC_COND_TYPE_AND, list); flttMakeLogicNodeFromList(&logicNode1, LOGIC_COND_TYPE_AND, list);
list = nodesMakeList(); list = nodesMakeList();
flttMakeColumnNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1); flttMakeColumnNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1);
flttMakeValueNode(&pRight1, TSDB_DATA_TYPE_INT, &rightv1); flttMakeValueNode(&pRight1, TSDB_DATA_TYPE_INT, &rightv1);
flttMakeOpNode(&opNode1, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft1, pRight1); flttMakeOpNode(&opNode1, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft1, pRight1);
@ -1042,7 +1022,7 @@ TEST(filterModelogicTest, diff_columns_and_or_and) {
flttMakeValueNode(&pRight2, TSDB_DATA_TYPE_INT, &rightv2); flttMakeValueNode(&pRight2, TSDB_DATA_TYPE_INT, &rightv2);
flttMakeOpNode(&opNode2, OP_TYPE_GREATER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft2, pRight2); flttMakeOpNode(&opNode2, OP_TYPE_GREATER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft2, pRight2);
nodesListAppend(list, opNode2); nodesListAppend(list, opNode2);
flttMakeLogicNodeFromList(&logicNode2, LOGIC_COND_TYPE_AND, list); flttMakeLogicNodeFromList(&logicNode2, LOGIC_COND_TYPE_AND, list);
list = nodesMakeList(); list = nodesMakeList();
@ -1050,13 +1030,13 @@ TEST(filterModelogicTest, diff_columns_and_or_and) {
nodesListAppend(list, logicNode2); nodesListAppend(list, logicNode2);
flttMakeLogicNodeFromList(&logicNode1, LOGIC_COND_TYPE_OR, list); flttMakeLogicNodeFromList(&logicNode1, LOGIC_COND_TYPE_OR, list);
SFilterInfo *filter = NULL; SFilterInfo *filter = NULL;
int32_t code = filterInitFromNode(logicNode1, &filter, 0); int32_t code = filterInitFromNode(logicNode1, &filter, 0);
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
SColumnDataAgg stat = {0}; SColumnDataAgg stat = {0};
SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; SFilterColumnParam param = { (int32_t)taosArrayGetSize(src->pDataBlock), src->pDataBlock };
code = filterSetDataFromSlotId(filter, &param); code = filterSetDataFromSlotId(filter, &param);
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
@ -1064,9 +1044,9 @@ TEST(filterModelogicTest, diff_columns_and_or_and) {
stat.min = 1; stat.min = 1;
stat.numOfNull = 0; stat.numOfNull = 0;
int8_t *rowRes = NULL; int8_t *rowRes = NULL;
bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); bool keep = filterExecute(filter, src, &rowRes, &stat, taosArrayGetSize(src->pDataBlock));
ASSERT_EQ(keep, false); ASSERT_EQ(keep, false);
for (int32_t i = 0; i < rowNum; ++i) { for (int32_t i = 0; i < rowNum; ++i) {
ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]);
} }
@ -1096,13 +1076,13 @@ TEST(filterModelogicTest, same_column_and_or_and) {
flttMakeValueNode(&pRight2, TSDB_DATA_TYPE_INT, &rightv2); flttMakeValueNode(&pRight2, TSDB_DATA_TYPE_INT, &rightv2);
flttMakeOpNode(&opNode2, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft1, pRight2); flttMakeOpNode(&opNode2, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft1, pRight2);
nodesListAppend(list, opNode2); nodesListAppend(list, opNode2);
flttMakeLogicNodeFromList(&logicNode1, LOGIC_COND_TYPE_AND, list); flttMakeLogicNodeFromList(&logicNode1, LOGIC_COND_TYPE_AND, list);
list = nodesMakeList(); list = nodesMakeList();
flttMakeColumnNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1); flttMakeColumnNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1);
flttMakeValueNode(&pRight1, TSDB_DATA_TYPE_INT, &rightv3); flttMakeValueNode(&pRight1, TSDB_DATA_TYPE_INT, &rightv3);
flttMakeOpNode(&opNode1, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft1, pRight1); flttMakeOpNode(&opNode1, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft1, pRight1);
nodesListAppend(list, opNode1); nodesListAppend(list, opNode1);
@ -1111,7 +1091,7 @@ TEST(filterModelogicTest, same_column_and_or_and) {
flttMakeValueNode(&pRight2, TSDB_DATA_TYPE_INT, &rightv4); flttMakeValueNode(&pRight2, TSDB_DATA_TYPE_INT, &rightv4);
flttMakeOpNode(&opNode2, OP_TYPE_GREATER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft1, pRight2); flttMakeOpNode(&opNode2, OP_TYPE_GREATER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft1, pRight2);
nodesListAppend(list, opNode2); nodesListAppend(list, opNode2);
flttMakeLogicNodeFromList(&logicNode2, LOGIC_COND_TYPE_AND, list); flttMakeLogicNodeFromList(&logicNode2, LOGIC_COND_TYPE_AND, list);
list = nodesMakeList(); list = nodesMakeList();
@ -1119,13 +1099,13 @@ TEST(filterModelogicTest, same_column_and_or_and) {
nodesListAppend(list, logicNode2); nodesListAppend(list, logicNode2);
flttMakeLogicNodeFromList(&logicNode1, LOGIC_COND_TYPE_OR, list); flttMakeLogicNodeFromList(&logicNode1, LOGIC_COND_TYPE_OR, list);
SFilterInfo *filter = NULL; SFilterInfo *filter = NULL;
int32_t code = filterInitFromNode(logicNode1, &filter, 0); int32_t code = filterInitFromNode(logicNode1, &filter, 0);
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
SColumnDataAgg stat = {0}; SColumnDataAgg stat = {0};
SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; SFilterColumnParam param = { (int32_t)taosArrayGetSize(src->pDataBlock), src->pDataBlock };
code = filterSetDataFromSlotId(filter, &param); code = filterSetDataFromSlotId(filter, &param);
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
@ -1133,9 +1113,9 @@ TEST(filterModelogicTest, same_column_and_or_and) {
stat.min = 1; stat.min = 1;
stat.numOfNull = 0; stat.numOfNull = 0;
int8_t *rowRes = NULL; int8_t *rowRes = NULL;
bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); bool keep = filterExecute(filter, src, &rowRes, &stat, taosArrayGetSize(src->pDataBlock));
ASSERT_EQ(keep, false); ASSERT_EQ(keep, false);
for (int32_t i = 0; i < rowNum; ++i) { for (int32_t i = 0; i < rowNum; ++i) {
ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]);
} }
@ -1165,12 +1145,12 @@ TEST(filterModelogicTest, diff_columns_or_and_or) {
flttMakeValueNode(&pRight2, TSDB_DATA_TYPE_INT, &rightv2); flttMakeValueNode(&pRight2, TSDB_DATA_TYPE_INT, &rightv2);
flttMakeOpNode(&opNode2, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft2, pRight2); flttMakeOpNode(&opNode2, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft2, pRight2);
nodesListAppend(list, opNode2); nodesListAppend(list, opNode2);
flttMakeLogicNodeFromList(&logicNode1, LOGIC_COND_TYPE_OR, list); flttMakeLogicNodeFromList(&logicNode1, LOGIC_COND_TYPE_OR, list);
list = nodesMakeList(); list = nodesMakeList();
flttMakeColumnNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1); flttMakeColumnNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1);
flttMakeValueNode(&pRight1, TSDB_DATA_TYPE_INT, &rightv1); flttMakeValueNode(&pRight1, TSDB_DATA_TYPE_INT, &rightv1);
flttMakeOpNode(&opNode1, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft1, pRight1); flttMakeOpNode(&opNode1, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft1, pRight1);
@ -1180,7 +1160,7 @@ TEST(filterModelogicTest, diff_columns_or_and_or) {
flttMakeValueNode(&pRight2, TSDB_DATA_TYPE_INT, &rightv2); flttMakeValueNode(&pRight2, TSDB_DATA_TYPE_INT, &rightv2);
flttMakeOpNode(&opNode2, OP_TYPE_GREATER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft2, pRight2); flttMakeOpNode(&opNode2, OP_TYPE_GREATER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft2, pRight2);
nodesListAppend(list, opNode2); nodesListAppend(list, opNode2);
flttMakeLogicNodeFromList(&logicNode2, LOGIC_COND_TYPE_OR, list); flttMakeLogicNodeFromList(&logicNode2, LOGIC_COND_TYPE_OR, list);
list = nodesMakeList(); list = nodesMakeList();
@ -1188,13 +1168,13 @@ TEST(filterModelogicTest, diff_columns_or_and_or) {
nodesListAppend(list, logicNode2); nodesListAppend(list, logicNode2);
flttMakeLogicNodeFromList(&logicNode1, LOGIC_COND_TYPE_AND, list); flttMakeLogicNodeFromList(&logicNode1, LOGIC_COND_TYPE_AND, list);
SFilterInfo *filter = NULL; SFilterInfo *filter = NULL;
int32_t code = filterInitFromNode(logicNode1, &filter, 0); int32_t code = filterInitFromNode(logicNode1, &filter, 0);
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
SColumnDataAgg stat = {0}; SColumnDataAgg stat = {0};
SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; SFilterColumnParam param = { (int32_t) taosArrayGetSize(src->pDataBlock), src->pDataBlock };
code = filterSetDataFromSlotId(filter, &param); code = filterSetDataFromSlotId(filter, &param);
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
@ -1202,9 +1182,9 @@ TEST(filterModelogicTest, diff_columns_or_and_or) {
stat.min = 1; stat.min = 1;
stat.numOfNull = 0; stat.numOfNull = 0;
int8_t *rowRes = NULL; int8_t *rowRes = NULL;
bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); bool keep = filterExecute(filter, src, &rowRes, &stat, taosArrayGetSize(src->pDataBlock));
ASSERT_EQ(keep, false); ASSERT_EQ(keep, false);
for (int32_t i = 0; i < rowNum; ++i) { for (int32_t i = 0; i < rowNum; ++i) {
ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]);
} }
@ -1234,12 +1214,12 @@ TEST(filterModelogicTest, same_column_or_and_or) {
flttMakeValueNode(&pRight2, TSDB_DATA_TYPE_INT, &rightv2); flttMakeValueNode(&pRight2, TSDB_DATA_TYPE_INT, &rightv2);
flttMakeOpNode(&opNode2, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft1, pRight2); flttMakeOpNode(&opNode2, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft1, pRight2);
nodesListAppend(list, opNode2); nodesListAppend(list, opNode2);
flttMakeLogicNodeFromList(&logicNode1, LOGIC_COND_TYPE_OR, list); flttMakeLogicNodeFromList(&logicNode1, LOGIC_COND_TYPE_OR, list);
list = nodesMakeList(); list = nodesMakeList();
flttMakeColumnNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1); flttMakeColumnNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1);
flttMakeValueNode(&pRight1, TSDB_DATA_TYPE_INT, &rightv3); flttMakeValueNode(&pRight1, TSDB_DATA_TYPE_INT, &rightv3);
flttMakeOpNode(&opNode1, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft1, pRight1); flttMakeOpNode(&opNode1, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft1, pRight1);
@ -1249,7 +1229,7 @@ TEST(filterModelogicTest, same_column_or_and_or) {
flttMakeValueNode(&pRight2, TSDB_DATA_TYPE_INT, &rightv4); flttMakeValueNode(&pRight2, TSDB_DATA_TYPE_INT, &rightv4);
flttMakeOpNode(&opNode2, OP_TYPE_GREATER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft1, pRight2); flttMakeOpNode(&opNode2, OP_TYPE_GREATER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft1, pRight2);
nodesListAppend(list, opNode2); nodesListAppend(list, opNode2);
flttMakeLogicNodeFromList(&logicNode2, LOGIC_COND_TYPE_OR, list); flttMakeLogicNodeFromList(&logicNode2, LOGIC_COND_TYPE_OR, list);
list = nodesMakeList(); list = nodesMakeList();
@ -1257,13 +1237,13 @@ TEST(filterModelogicTest, same_column_or_and_or) {
nodesListAppend(list, logicNode2); nodesListAppend(list, logicNode2);
flttMakeLogicNodeFromList(&logicNode1, LOGIC_COND_TYPE_AND, list); flttMakeLogicNodeFromList(&logicNode1, LOGIC_COND_TYPE_AND, list);
SFilterInfo *filter = NULL; SFilterInfo *filter = NULL;
int32_t code = filterInitFromNode(logicNode1, &filter, 0); int32_t code = filterInitFromNode(logicNode1, &filter, 0);
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
SColumnDataAgg stat = {0}; SColumnDataAgg stat = {0};
SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; SFilterColumnParam param = { (int32_t)taosArrayGetSize(src->pDataBlock), src->pDataBlock };
code = filterSetDataFromSlotId(filter, &param); code = filterSetDataFromSlotId(filter, &param);
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
@ -1271,9 +1251,9 @@ TEST(filterModelogicTest, same_column_or_and_or) {
stat.min = 1; stat.min = 1;
stat.numOfNull = 0; stat.numOfNull = 0;
int8_t *rowRes = NULL; int8_t *rowRes = NULL;
bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); bool keep = filterExecute(filter, src, &rowRes, &stat, taosArrayGetSize(src->pDataBlock));
ASSERT_EQ(keep, false); ASSERT_EQ(keep, false);
for (int32_t i = 0; i < rowNum; ++i) { for (int32_t i = 0; i < rowNum; ++i) {
ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]);
} }
@ -1305,12 +1285,12 @@ TEST(scalarModelogicTest, diff_columns_or_and_or) {
flttMakeColumnNode(&pRight2, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, rightv2); flttMakeColumnNode(&pRight2, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, rightv2);
flttMakeOpNode(&opNode2, OP_TYPE_LOWER_THAN, TSDB_DATA_TYPE_BOOL, pLeft2, pRight2); flttMakeOpNode(&opNode2, OP_TYPE_LOWER_THAN, TSDB_DATA_TYPE_BOOL, pLeft2, pRight2);
nodesListAppend(list, opNode2); nodesListAppend(list, opNode2);
flttMakeLogicNodeFromList(&logicNode1, LOGIC_COND_TYPE_OR, list); flttMakeLogicNodeFromList(&logicNode1, LOGIC_COND_TYPE_OR, list);
list = nodesMakeList(); list = nodesMakeList();
flttMakeColumnNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1); flttMakeColumnNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1);
flttMakeColumnNode(&pRight1, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, rightv1); flttMakeColumnNode(&pRight1, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, rightv1);
flttMakeOpNode(&opNode1, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pLeft1, pRight1); flttMakeOpNode(&opNode1, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pLeft1, pRight1);
@ -1320,7 +1300,7 @@ TEST(scalarModelogicTest, diff_columns_or_and_or) {
flttMakeColumnNode(&pRight2, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, rightv2); flttMakeColumnNode(&pRight2, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, rightv2);
flttMakeOpNode(&opNode2, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft2, pRight2); flttMakeOpNode(&opNode2, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft2, pRight2);
nodesListAppend(list, opNode2); nodesListAppend(list, opNode2);
flttMakeLogicNodeFromList(&logicNode2, LOGIC_COND_TYPE_OR, list); flttMakeLogicNodeFromList(&logicNode2, LOGIC_COND_TYPE_OR, list);
list = nodesMakeList(); list = nodesMakeList();
@ -1328,13 +1308,13 @@ TEST(scalarModelogicTest, diff_columns_or_and_or) {
nodesListAppend(list, logicNode2); nodesListAppend(list, logicNode2);
flttMakeLogicNodeFromList(&logicNode1, LOGIC_COND_TYPE_AND, list); flttMakeLogicNodeFromList(&logicNode1, LOGIC_COND_TYPE_AND, list);
SFilterInfo *filter = NULL; SFilterInfo *filter = NULL;
int32_t code = filterInitFromNode(logicNode1, &filter, 0); int32_t code = filterInitFromNode(logicNode1, &filter, 0);
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
SColumnDataAgg stat = {0}; SColumnDataAgg stat = {0};
SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; SFilterColumnParam param = { (int32_t)taosArrayGetSize(src->pDataBlock), src->pDataBlock };
code = filterSetDataFromSlotId(filter, &param); code = filterSetDataFromSlotId(filter, &param);
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
@ -1342,7 +1322,7 @@ TEST(scalarModelogicTest, diff_columns_or_and_or) {
stat.min = 1; stat.min = 1;
stat.numOfNull = 0; stat.numOfNull = 0;
int8_t *rowRes = NULL; int8_t *rowRes = NULL;
bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); bool keep = filterExecute(filter, src, &rowRes, &stat, taosArrayGetSize(src->pDataBlock));
ASSERT_EQ(keep, false); ASSERT_EQ(keep, false);
for (int32_t i = 0; i < rowNum; ++i) { for (int32_t i = 0; i < rowNum; ++i) {

79
source/libs/scalar/test/scalar/scalarTests.cpp
View File

@ -84,30 +84,29 @@ void scltInitLogFile() {
void scltAppendReservedSlot(SArray *pBlockList, int16_t *dataBlockId, int16_t *slotId, bool newBlock, int32_t rows, SColumnInfo *colInfo) { void scltAppendReservedSlot(SArray *pBlockList, int16_t *dataBlockId, int16_t *slotId, bool newBlock, int32_t rows, SColumnInfo *colInfo) {
if (newBlock) { if (newBlock) {
SSDataBlock *res = (SSDataBlock *)taosMemoryCalloc(1, sizeof(SSDataBlock)); SSDataBlock *res = createDataBlock();
res->info.numOfCols = 1;
res->info.rows = rows;
res->pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData));
SColumnInfoData idata = {0}; SColumnInfoData idata = {0};
idata.info = *colInfo; idata.info = *colInfo;
colInfoDataEnsureCapacity(&idata, rows);
blockDataAppendColInfo(res, &idata);
res->info.capacity = rows;
res->info.rows = rows;
SColumnInfoData* p = static_cast<SColumnInfoData *>(taosArrayGet(res->pDataBlock, 0));
ASSERT(p->pData != NULL && p->nullbitmap != NULL);
taosArrayPush(res->pDataBlock, &idata);
taosArrayPush(pBlockList, &res); taosArrayPush(pBlockList, &res);
blockDataEnsureCapacity(res, rows);
*dataBlockId = taosArrayGetSize(pBlockList) - 1; *dataBlockId = taosArrayGetSize(pBlockList) - 1;
res->info.blockId = *dataBlockId; res->info.blockId = *dataBlockId;
*slotId = 0; *slotId = 0;
} else { } else {
SSDataBlock *res = *(SSDataBlock **)taosArrayGetLast(pBlockList); SSDataBlock *res = *(SSDataBlock **)taosArrayGetLast(pBlockList);
res->info.numOfCols++;
SColumnInfoData idata = {0}; SColumnInfoData idata = {0};
idata.info = *colInfo; idata.info = *colInfo;
colInfoDataEnsureCapacity(&idata, rows);
colInfoDataEnsureCapacity(&idata, 0, rows); blockDataAppendColInfo(res, &idata);
taosArrayPush(res->pDataBlock, &idata);
*dataBlockId = taosArrayGetSize(pBlockList) - 1; *dataBlockId = taosArrayGetSize(pBlockList) - 1;
*slotId = taosArrayGetSize(res->pDataBlock) - 1; *slotId = taosArrayGetSize(res->pDataBlock) - 1;
@ -144,29 +143,19 @@ void scltMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in
} }
if (NULL == *block) { if (NULL == *block) {
SSDataBlock *res = (SSDataBlock *)taosMemoryCalloc(1, sizeof(SSDataBlock)); SSDataBlock *res = createDataBlock();
res->info.numOfCols = 3;
res->info.rows = rowNum;
res->pDataBlock = taosArrayInit(3, sizeof(SColumnInfoData));
for (int32_t i = 0; i < 2; ++i) { for (int32_t i = 0; i < 2; ++i) {
SColumnInfoData idata = {{0}}; SColumnInfoData idata = createColumnInfoData(TSDB_DATA_TYPE_INT, 10, i + 1);
idata.info.type = TSDB_DATA_TYPE_NULL; colInfoDataEnsureCapacity(&idata, rowNum);
idata.info.bytes = 10; blockDataAppendColInfo(res, &idata);
idata.info.colId = i + 1;
int32_t size = idata.info.bytes * rowNum;
idata.pData = (char *)taosMemoryCalloc(1, size);
taosArrayPush(res->pDataBlock, &idata);
} }
SColumnInfoData idata = {{0}}; SColumnInfoData idata = createColumnInfoData(dataType, dataBytes, 3);
idata.info.type = dataType; colInfoDataEnsureCapacity(&idata, rowNum);
idata.info.bytes = dataBytes; blockDataAppendColInfo(res, &idata);
idata.info.colId = 3; res->info.capacity = rowNum;
int32_t size = idata.info.bytes * rowNum;
idata.pData = (char *)taosMemoryCalloc(1, size); res->info.rows = rowNum;
colInfoDataEnsureCapacity(&idata, 0, rowNum);
taosArrayPush(res->pDataBlock, &idata);
SColumnInfoData *pColumn = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); SColumnInfoData *pColumn = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock);
for (int32_t i = 0; i < rowNum; ++i) { for (int32_t i = 0; i < rowNum; ++i) {
colDataAppend(pColumn, i, (const char *)value, false); colDataAppend(pColumn, i, (const char *)value, false);
@ -185,17 +174,13 @@ void scltMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in
SSDataBlock *res = *block; SSDataBlock *res = *block;
int32_t idx = taosArrayGetSize(res->pDataBlock); int32_t idx = taosArrayGetSize(res->pDataBlock);
SColumnInfoData idata = {{0}}; SColumnInfoData idata = createColumnInfoData(dataType, dataBytes, 1 + idx);
idata.info.type = dataType; colInfoDataEnsureCapacity(&idata, rowNum);
idata.info.bytes = dataBytes;
idata.info.colId = 1 + idx;
int32_t size = idata.info.bytes * rowNum;
idata.pData = (char *)taosMemoryCalloc(1, size);
taosArrayPush(res->pDataBlock, &idata);
res->info.numOfCols++;
SColumnInfoData *pColumn = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock);
colInfoDataEnsureCapacity(pColumn, 0, rowNum); res->info.capacity = rowNum;
blockDataAppendColInfo(res, &idata);
SColumnInfoData *pColumn = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock);
for (int32_t i = 0; i < rowNum; ++i) { for (int32_t i = 0; i < rowNum; ++i) {
colDataAppend(pColumn, i, (const char *)value, false); colDataAppend(pColumn, i, (const char *)value, false);
@ -279,9 +264,6 @@ void scltMakeTargetNode(SNode **pNode, int16_t dataBlockId, int16_t slotId, SNod
*pNode = (SNode *)onode; *pNode = (SNode *)onode;
} }
} }
TEST(constantTest, bigint_add_bigint) { TEST(constantTest, bigint_add_bigint) {
@ -1529,6 +1511,7 @@ TEST(columnTest, bigint_column_multi_binary_column) {
SArray *blockList = taosArrayInit(1, POINTER_BYTES); SArray *blockList = taosArrayInit(1, POINTER_BYTES);
taosArrayPush(blockList, &src); taosArrayPush(blockList, &src);
SColumnInfo colInfo = createColumnInfo(1, TSDB_DATA_TYPE_DOUBLE, sizeof(double)); SColumnInfo colInfo = createColumnInfo(1, TSDB_DATA_TYPE_DOUBLE, sizeof(double));
int16_t dataBlockId = 0, slotId = 0; int16_t dataBlockId = 0, slotId = 0;
scltAppendReservedSlot(blockList, &dataBlockId, &slotId, false, rowNum, &colInfo); scltAppendReservedSlot(blockList, &dataBlockId, &slotId, false, rowNum, &colInfo);
@ -2033,7 +2016,7 @@ void scltMakeDataBlock(SScalarParam **pInput, int32_t type, void *pVal, int32_t
input->numOfRows = num; input->numOfRows = num;
input->columnData->info = createColumnInfo(0, type, bytes); input->columnData->info = createColumnInfo(0, type, bytes);
colInfoDataEnsureCapacity(input->columnData, 0, num); colInfoDataEnsureCapacity(input->columnData, num);
if (setVal) { if (setVal) {
for (int32_t i = 0; i < num; ++i) { for (int32_t i = 0; i < num; ++i) {

2
source/libs/stream/inc/streamInc.h
View File

@ -34,7 +34,7 @@ int32_t streamExec(SStreamTask* pTask, SMsgCb* pMsgCb);
int32_t streamDispatch(SStreamTask* pTask, SMsgCb* pMsgCb); int32_t streamDispatch(SStreamTask* pTask, SMsgCb* pMsgCb);
int32_t streamDispatchReqToData(const SStreamDispatchReq* pReq, SStreamDataBlock* pData); int32_t streamDispatchReqToData(const SStreamDispatchReq* pReq, SStreamDataBlock* pData);
int32_t streamRetrieveReqToData(const SStreamRetrieveReq* pReq, SStreamDataBlock* pData); int32_t streamRetrieveReqToData(const SStreamRetrieveReq* pReq, SStreamDataBlock* pData);
int32_t streamBuildDispatchMsg(SStreamTask* pTask, SStreamDataBlock* data, SRpcMsg* pMsg, SEpSet** ppEpSet); int32_t streamBuildDispatchMsg(SStreamTask* pTask, const SStreamDataBlock* data, SRpcMsg* pMsg, SEpSet** ppEpSet);
int32_t streamBroadcastToChildren(SStreamTask* pTask, const SSDataBlock* pBlock); int32_t streamBroadcastToChildren(SStreamTask* pTask, const SSDataBlock* pBlock);

15
source/libs/stream/src/streamDispatch.c
View File

@ -96,16 +96,17 @@ int32_t streamBroadcastToChildren(SStreamTask* pTask, const SSDataBlock* pBlock)
pRetrieve = taosMemoryCalloc(1, dataStrLen); pRetrieve = taosMemoryCalloc(1, dataStrLen);
if (pRetrieve == NULL) return -1; if (pRetrieve == NULL) return -1;
int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
pRetrieve->useconds = 0; pRetrieve->useconds = 0;
pRetrieve->precision = TSDB_DEFAULT_PRECISION; pRetrieve->precision = TSDB_DEFAULT_PRECISION;
pRetrieve->compressed = 0; pRetrieve->compressed = 0;
pRetrieve->completed = 1; pRetrieve->completed = 1;
pRetrieve->streamBlockType = pBlock->info.type; pRetrieve->streamBlockType = pBlock->info.type;
pRetrieve->numOfRows = htonl(pBlock->info.rows); pRetrieve->numOfRows = htonl(pBlock->info.rows);
pRetrieve->numOfCols = htonl(pBlock->info.numOfCols); pRetrieve->numOfCols = htonl(numOfCols);
int32_t actualLen = 0; int32_t actualLen = 0;
blockCompressEncode(pBlock, pRetrieve->data, &actualLen, pBlock->info.numOfCols, false); blockCompressEncode(pBlock, pRetrieve->data, &actualLen, numOfCols, false);
SStreamRetrieveReq req = { SStreamRetrieveReq req = {
.streamId = pTask->streamId, .streamId = pTask->streamId,
@ -170,10 +171,12 @@ static int32_t streamAddBlockToDispatchMsg(const SSDataBlock* pBlock, SStreamDis
pRetrieve->completed = 1; pRetrieve->completed = 1;
pRetrieve->streamBlockType = pBlock->info.type; pRetrieve->streamBlockType = pBlock->info.type;
pRetrieve->numOfRows = htonl(pBlock->info.rows); pRetrieve->numOfRows = htonl(pBlock->info.rows);
pRetrieve->numOfCols = htonl(pBlock->info.numOfCols);
int32_t numOfCols = (int32_t)taosArrayGetSize(pBlock->pDataBlock);
pRetrieve->numOfCols = htonl(numOfCols);
int32_t actualLen = 0; int32_t actualLen = 0;
blockCompressEncode(pBlock, pRetrieve->data, &actualLen, pBlock->info.numOfCols, false); blockCompressEncode(pBlock, pRetrieve->data, &actualLen, numOfCols, false);
actualLen += sizeof(SRetrieveTableRsp); actualLen += sizeof(SRetrieveTableRsp);
ASSERT(actualLen <= dataStrLen); ASSERT(actualLen <= dataStrLen);
taosArrayPush(pReq->dataLen, &actualLen); taosArrayPush(pReq->dataLen, &actualLen);
@ -182,7 +185,7 @@ static int32_t streamAddBlockToDispatchMsg(const SSDataBlock* pBlock, SStreamDis
return 0; return 0;
} }
int32_t streamBuildDispatchMsg(SStreamTask* pTask, SStreamDataBlock* data, SRpcMsg* pMsg, SEpSet** ppEpSet) { int32_t streamBuildDispatchMsg(SStreamTask* pTask, const SStreamDataBlock* data, SRpcMsg* pMsg, SEpSet** ppEpSet) {
void* buf = NULL; void* buf = NULL;
int32_t code = -1; int32_t code = -1;
int32_t blockNum = taosArrayGetSize(data->blocks); int32_t blockNum = taosArrayGetSize(data->blocks);
@ -304,6 +307,8 @@ int32_t streamDispatch(SStreamTask* pTask, SMsgCb* pMsgCb) {
atomic_store_8(&pTask->outputStatus, TASK_OUTPUT_STATUS__NORMAL); atomic_store_8(&pTask->outputStatus, TASK_OUTPUT_STATUS__NORMAL);
return -1; return -1;
} }
taosArrayDestroyEx(pBlock->blocks, (FDelete)tDeleteSSDataBlock);
taosFreeQitem(pBlock);
tmsgSendReq(pEpSet, &dispatchMsg); tmsgSendReq(pEpSet, &dispatchMsg);
return 0; return 0;

45
source/libs/stream/src/streamExec.c
View File

@ -53,9 +53,13 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, void* data, SArray* pRes)
} }
// TODO: do we need free memory? // TODO: do we need free memory?
SSDataBlock* outputCopy = createOneDataBlock(output, true); SSDataBlock block = {0};
outputCopy->info.childId = pTask->selfChildId; assignOneDataBlock(&block, output);
taosArrayPush(pRes, outputCopy); block.info.childId = pTask->selfChildId;
taosArrayPush(pRes, &block);
/*SSDataBlock* outputCopy = createOneDataBlock(output, true);*/
/*outputCopy->info.childId = pTask->selfChildId;*/
/*taosArrayPush(pRes, outputCopy);*/
} }
return 0; return 0;
} }
@ -68,6 +72,7 @@ static SArray* streamExecForQall(SStreamTask* pTask, SArray* pRes) {
streamTaskExecImpl(pTask, data, pRes); streamTaskExecImpl(pTask, data, pRes);
if (pTask->taskStatus == TASK_STATUS__DROPPING) { if (pTask->taskStatus == TASK_STATUS__DROPPING) {
taosArrayDestroyEx(pRes, (FDelete)tDeleteSSDataBlock);
return NULL; return NULL;
} }
@ -82,25 +87,25 @@ static SArray* streamExecForQall(SStreamTask* pTask, SArray* pRes) {
qRes->blocks = pRes; qRes->blocks = pRes;
if (streamTaskOutput(pTask, qRes) < 0) { if (streamTaskOutput(pTask, qRes) < 0) {
streamQueueProcessFail(pTask->inputQueue); streamQueueProcessFail(pTask->inputQueue);
taosArrayDestroy(pRes); taosArrayDestroyEx(pRes, (FDelete)tDeleteSSDataBlock);
taosFreeQitem(qRes); taosFreeQitem(qRes);
return NULL; return NULL;
} }
int8_t type = ((SStreamQueueItem*)data)->type;
if (type == STREAM_INPUT__TRIGGER) {
blockDataDestroy(((SStreamTrigger*)data)->pBlock);
taosFreeQitem(data);
} else if (type == STREAM_INPUT__DATA_BLOCK) {
taosArrayDestroyEx(((SStreamDataBlock*)data)->blocks, (FDelete)tDeleteSSDataBlock);
taosFreeQitem(data);
} else if (type == STREAM_INPUT__DATA_SUBMIT) {
ASSERT(pTask->isDataScan);
streamDataSubmitRefDec((SStreamDataSubmit*)data);
taosFreeQitem(data);
}
streamQueueProcessSuccess(pTask->inputQueue); streamQueueProcessSuccess(pTask->inputQueue);
return taosArrayInit(0, sizeof(SSDataBlock)); pRes = taosArrayInit(0, sizeof(SSDataBlock));
}
int8_t type = ((SStreamQueueItem*)data)->type;
if (type == STREAM_INPUT__TRIGGER) {
blockDataDestroy(((SStreamTrigger*)data)->pBlock);
taosFreeQitem(data);
} else if (type == STREAM_INPUT__DATA_BLOCK) {
taosArrayDestroyEx(((SStreamDataBlock*)data)->blocks, (FDelete)tDeleteSSDataBlock);
taosFreeQitem(data);
} else if (type == STREAM_INPUT__DATA_SUBMIT) {
ASSERT(pTask->isDataScan);
streamDataSubmitRefDec((SStreamDataSubmit*)data);
taosFreeQitem(data);
} }
} }
return pRes; return pRes;
@ -125,14 +130,14 @@ int32_t streamExec(SStreamTask* pTask, SMsgCb* pMsgCb) {
pRes = streamExecForQall(pTask, pRes); pRes = streamExecForQall(pTask, pRes);
if (pRes == NULL) goto FAIL; if (pRes == NULL) goto FAIL;
taosArrayDestroy(pRes); taosArrayDestroyEx(pRes, (FDelete)tDeleteSSDataBlock);
atomic_store_8(&pTask->execStatus, TASK_EXEC_STATUS__IDLE); atomic_store_8(&pTask->execStatus, TASK_EXEC_STATUS__IDLE);
return 0; return 0;
} else if (execStatus == TASK_EXEC_STATUS__CLOSING) { } else if (execStatus == TASK_EXEC_STATUS__CLOSING) {
continue; continue;
} else if (execStatus == TASK_EXEC_STATUS__EXECUTING) { } else if (execStatus == TASK_EXEC_STATUS__EXECUTING) {
ASSERT(taosArrayGetSize(pRes) == 0); ASSERT(taosArrayGetSize(pRes) == 0);
taosArrayDestroy(pRes); taosArrayDestroyEx(pRes, (FDelete)tDeleteSSDataBlock);
return 0; return 0;
} else { } else {
ASSERT(0); ASSERT(0);

12
source/libs/sync/inc/syncEnv.h
View File

@ -28,13 +28,13 @@ extern "C" {
#include "trpc.h" #include "trpc.h"
#include "ttimer.h" #include "ttimer.h"
#define TIMER_MAX_MS 0x7FFFFFFF #define TIMER_MAX_MS 0x7FFFFFFF
#define ENV_TICK_TIMER_MS 1000 #define ENV_TICK_TIMER_MS 1000
#define PING_TIMER_MS 1000 #define PING_TIMER_MS 1000
#define ELECT_TIMER_MS_MIN 500 #define ELECT_TIMER_MS_MIN 1300
#define ELECT_TIMER_MS_MAX (ELECT_TIMER_MS_MIN * 2) #define ELECT_TIMER_MS_MAX (ELECT_TIMER_MS_MIN * 2)
#define ELECT_TIMER_MS_RANGE (ELECT_TIMER_MS_MAX - ELECT_TIMER_MS_MIN) #define ELECT_TIMER_MS_RANGE (ELECT_TIMER_MS_MAX - ELECT_TIMER_MS_MIN)
#define HEARTBEAT_TIMER_MS 100 #define HEARTBEAT_TIMER_MS 900
#define EMPTY_RAFT_ID ((SRaftId){.addr = 0, .vgId = 0}) #define EMPTY_RAFT_ID ((SRaftId){.addr = 0, .vgId = 0})

8
source/libs/sync/inc/syncRaftLog.h
View File

@ -26,11 +26,13 @@ extern "C" {
#include "syncInt.h" #include "syncInt.h"
#include "syncRaftEntry.h" #include "syncRaftEntry.h"
#include "taosdef.h" #include "taosdef.h"
#include "wal.h"
typedef struct SSyncLogStoreData { typedef struct SSyncLogStoreData {
SSyncNode* pSyncNode; SSyncNode* pSyncNode;
SWal* pWal; SWal* pWal;
SyncIndex beginIndex; // valid begin index, default 0, may be set beginIndex > 0 SWalReadHandle* pWalHandle;
// SyncIndex beginIndex; // valid begin index, default 0, may be set beginIndex > 0
} SSyncLogStoreData; } SSyncLogStoreData;
SSyncLogStore* logStoreCreate(SSyncNode* pSyncNode); SSyncLogStore* logStoreCreate(SSyncNode* pSyncNode);

71
source/libs/sync/src/syncAppendEntries.c
View File

@ -420,44 +420,26 @@ static int32_t syncNodePreCommit(SSyncNode* ths, SSyncRaftEntry* pEntry) {
// prevLogIndex == -1 // prevLogIndex == -1
static bool syncNodeOnAppendEntriesLogOK(SSyncNode* pSyncNode, SyncAppendEntries* pMsg) { static bool syncNodeOnAppendEntriesLogOK(SSyncNode* pSyncNode, SyncAppendEntries* pMsg) {
if (pMsg->prevLogIndex == SYNC_INDEX_INVALID) { if (pMsg->prevLogIndex == SYNC_INDEX_INVALID) {
if (gRaftDetailLog) {
sTrace("syncNodeOnAppendEntriesLogOK true, pMsg->prevLogIndex:%ld", pMsg->prevLogIndex);
}
return true; return true;
} }
SyncIndex myLastIndex = syncNodeGetLastIndex(pSyncNode); SyncIndex myLastIndex = syncNodeGetLastIndex(pSyncNode);
if (pMsg->prevLogIndex > myLastIndex) { if (pMsg->prevLogIndex > myLastIndex) {
if (gRaftDetailLog) { sDebug("vgId:%d sync log not ok, preindex:%ld", pSyncNode->vgId, pMsg->prevLogIndex);
sTrace("syncNodeOnAppendEntriesLogOK false, pMsg->prevLogIndex:%ld, myLastIndex:%ld", pMsg->prevLogIndex,
myLastIndex);
}
return false; return false;
} }
SyncTerm myPreLogTerm = syncNodeGetPreTerm(pSyncNode, pMsg->prevLogIndex + 1); SyncTerm myPreLogTerm = syncNodeGetPreTerm(pSyncNode, pMsg->prevLogIndex + 1);
if (myPreLogTerm == SYNC_TERM_INVALID) { if (myPreLogTerm == SYNC_TERM_INVALID) {
sError("vgId:%d sync get pre term error, preindex:%ld", pSyncNode->vgId, pMsg->prevLogIndex); sDebug("vgId:%d sync log not ok2, preindex:%ld", pSyncNode->vgId, pMsg->prevLogIndex);
return false; return false;
} }
if (pMsg->prevLogIndex <= myLastIndex && pMsg->prevLogTerm == myPreLogTerm) { if (pMsg->prevLogIndex <= myLastIndex && pMsg->prevLogTerm == myPreLogTerm) {
if (gRaftDetailLog) {
sTrace(
"syncNodeOnAppendEntriesLogOK true, pMsg->prevLogIndex:%ld, myLastIndex:%ld, pMsg->prevLogTerm:%lu, "
"myPreLogTerm:%lu",
pMsg->prevLogIndex, myLastIndex, pMsg->prevLogTerm, myPreLogTerm);
}
return true; return true;
} }
if (gRaftDetailLog) { sDebug("vgId:%d sync log not ok3, preindex:%ld", pSyncNode->vgId, pMsg->prevLogIndex);
sTrace(
"syncNodeOnAppendEntriesLogOK false, pMsg->prevLogIndex:%ld, myLastIndex:%ld, pMsg->prevLogTerm:%lu, "
"myPreLogTerm:%lu",
pMsg->prevLogIndex, myLastIndex, pMsg->prevLogTerm, myPreLogTerm);
}
return false; return false;
} }
@ -466,14 +448,11 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs
int32_t code = 0; int32_t code = 0;
// print log // print log
char logBuf[128] = {0}; syncAppendEntriesLog2("==syncNodeOnAppendEntriesSnapshotCb==", pMsg);
snprintf(logBuf, sizeof(logBuf), "recv SyncAppendEntries, vgId:%d, term:%lu", ths->vgId,
ths->pRaftStore->currentTerm);
syncAppendEntriesLog2(logBuf, pMsg);
// if already drop replica, do not process // if already drop replica, do not process
if (!syncNodeInRaftGroup(ths, &(pMsg->srcId)) && !ths->pRaftCfg->isStandBy) { if (!syncNodeInRaftGroup(ths, &(pMsg->srcId)) && !ths->pRaftCfg->isStandBy) {
sInfo("recv SyncAppendEntries maybe replica already dropped"); syncNodeEventLog(ths, "recv sync-append-entries, maybe replica already dropped");
return ret; return ret;
} }
@ -497,7 +476,7 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs
do { do {
bool condition = pMsg->term == ths->pRaftStore->currentTerm && ths->state == TAOS_SYNC_STATE_CANDIDATE; bool condition = pMsg->term == ths->pRaftStore->currentTerm && ths->state == TAOS_SYNC_STATE_CANDIDATE;
if (condition) { if (condition) {
sTrace("recv SyncAppendEntries, candidate to follower"); syncNodeEventLog(ths, "recv sync-append-entries, candidate to follower");
syncNodeBecomeFollower(ths, "from candidate by append entries"); syncNodeBecomeFollower(ths, "from candidate by append entries");
// do not reply? // do not reply?
@ -505,6 +484,7 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs
} }
} while (0); } while (0);
#if 0
// fake match // fake match
// //
// condition1: // condition1:
@ -530,7 +510,7 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs
bool condition0 = (pMsg->term == ths->pRaftStore->currentTerm) && (ths->state == TAOS_SYNC_STATE_FOLLOWER) && bool condition0 = (pMsg->term == ths->pRaftStore->currentTerm) && (ths->state == TAOS_SYNC_STATE_FOLLOWER) &&
syncNodeHasSnapshot(ths); syncNodeHasSnapshot(ths);
bool condition1 = bool condition1 =
condition0 && (ths->pLogStore->syncLogEntryCount(ths->pLogStore) == 0) && (pMsg->prevLogIndex > myLastIndex); condition0 && (ths->pLogStore->syncLogEntryCount(ths->pLogStore) == 0) && (pMsg->prevLogIndex > myLastIndex); // donot use syncLogEntryCount!!! use isEmpty
bool condition2 = condition0 && (ths->pLogStore->syncLogLastIndex(ths->pLogStore) <= snapshot.lastApplyIndex) && bool condition2 = condition0 && (ths->pLogStore->syncLogLastIndex(ths->pLogStore) <= snapshot.lastApplyIndex) &&
(pMsg->prevLogIndex > myLastIndex); (pMsg->prevLogIndex > myLastIndex);
bool condition3 = condition0 && (pMsg->prevLogIndex < snapshot.lastApplyIndex); bool condition3 = condition0 && (pMsg->prevLogIndex < snapshot.lastApplyIndex);
@ -538,11 +518,10 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs
bool condition = condition1 || condition2 || condition3 || condition4; bool condition = condition1 || condition2 || condition3 || condition4;
if (condition) { if (condition) {
sTrace( char logBuf[128];
"recv SyncAppendEntries, fake match, myLastIndex:%ld, syncLogBeginIndex:%ld, syncLogEndIndex:%ld, " snprintf(logBuf, sizeof(logBuf), "recv sync-append-entries, fake match, pre-index:%ld, pre-term:%lu",
"condition1:%d, condition2:%d, condition3:%d, condition4:%d", pMsg->prevLogIndex, pMsg->prevLogTerm);
myLastIndex, ths->pLogStore->syncLogBeginIndex(ths->pLogStore), syncNodeEventLog(ths, logBuf);
ths->pLogStore->syncLogEndIndex(ths->pLogStore), condition1, condition2, condition3, condition4);
// prepare response msg // prepare response msg
SyncAppendEntriesReply* pReply = syncAppendEntriesReplyBuild(ths->vgId); SyncAppendEntriesReply* pReply = syncAppendEntriesReplyBuild(ths->vgId);
@ -562,6 +541,7 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs
return ret; return ret;
} }
} while (0); } while (0);
#endif
// fake match2 // fake match2
// //
@ -576,8 +556,13 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs
bool condition = (pMsg->term == ths->pRaftStore->currentTerm) && (ths->state == TAOS_SYNC_STATE_FOLLOWER) && bool condition = (pMsg->term == ths->pRaftStore->currentTerm) && (ths->state == TAOS_SYNC_STATE_FOLLOWER) &&
(pMsg->prevLogIndex <= ths->commitIndex); (pMsg->prevLogIndex <= ths->commitIndex);
if (condition) { if (condition) {
sTrace("recv SyncAppendEntries, fake match2, msg-prevLogIndex:%ld, my-commitIndex:%ld", pMsg->prevLogIndex, do {
ths->commitIndex); char logBuf[128];
snprintf(logBuf, sizeof(logBuf),
"recv sync-append-entries, fake match2, pre-index:%ld, pre-term:%lu, datalen:%d", pMsg->prevLogIndex,
pMsg->prevLogTerm, pMsg->dataLen);
syncNodeEventLog(ths, logBuf);
} while (0);
SyncIndex matchIndex = ths->commitIndex; SyncIndex matchIndex = ths->commitIndex;
bool hasAppendEntries = pMsg->dataLen > 0; bool hasAppendEntries = pMsg->dataLen > 0;
@ -605,6 +590,7 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs
code = syncNodePreCommit(ths, pAppendEntry); code = syncNodePreCommit(ths, pAppendEntry);
ASSERT(code == 0); ASSERT(code == 0);
// update match index
matchIndex = pMsg->prevLogIndex + 1; matchIndex = pMsg->prevLogIndex + 1;
syncEntryDestory(pAppendEntry); syncEntryDestory(pAppendEntry);
@ -650,11 +636,10 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs
bool condition = condition1 || condition2; bool condition = condition1 || condition2;
if (condition) { if (condition) {
sTrace( char logBuf[128];
"recv SyncAppendEntries, not match, syncLogBeginIndex:%ld, syncLogEndIndex:%ld, condition1:%d, " snprintf(logBuf, sizeof(logBuf), "recv sync-append-entries, not match, pre-index:%ld, pre-term:%lu, datalen:%d",
"condition2:%d, logOK:%d", pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->dataLen);
ths->pLogStore->syncLogBeginIndex(ths->pLogStore), ths->pLogStore->syncLogEndIndex(ths->pLogStore), syncNodeEventLog(ths, logBuf);
condition1, condition2, logOK);
// prepare response msg // prepare response msg
SyncAppendEntriesReply* pReply = syncAppendEntriesReplyBuild(ths->vgId); SyncAppendEntriesReply* pReply = syncAppendEntriesReplyBuild(ths->vgId);
@ -693,8 +678,10 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs
// has entries in SyncAppendEntries msg // has entries in SyncAppendEntries msg
bool hasAppendEntries = pMsg->dataLen > 0; bool hasAppendEntries = pMsg->dataLen > 0;
sTrace("recv SyncAppendEntries, match, myLastIndex:%ld, hasExtraEntries:%d, hasAppendEntries:%d", myLastIndex, char logBuf[128];
hasExtraEntries, hasAppendEntries); snprintf(logBuf, sizeof(logBuf), "recv sync-append-entries, match, pre-index:%ld, pre-term:%lu, datalen:%d",
pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->dataLen);
syncNodeEventLog(ths, logBuf);
if (hasExtraEntries) { if (hasExtraEntries) {
// make log same, rollback deleted entries // make log same, rollback deleted entries

42
source/libs/sync/src/syncAppendEntriesReply.c
View File

@ -101,32 +101,27 @@ int32_t syncNodeOnAppendEntriesReplySnapshotCb(SSyncNode* ths, SyncAppendEntries
int32_t ret = 0; int32_t ret = 0;
// print log // print log
char logBuf[128] = {0}; syncAppendEntriesReplyLog2("==syncNodeOnAppendEntriesReplySnapshotCb==", pMsg);
snprintf(logBuf, sizeof(logBuf), "recv SyncAppendEntriesReply, vgId:%d, term:%lu", ths->vgId,
ths->pRaftStore->currentTerm);
syncAppendEntriesReplyLog2(logBuf, pMsg);
// if already drop replica, do not process // if already drop replica, do not process
if (!syncNodeInRaftGroup(ths, &(pMsg->srcId)) && !ths->pRaftCfg->isStandBy) { if (!syncNodeInRaftGroup(ths, &(pMsg->srcId)) && !ths->pRaftCfg->isStandBy) {
sInfo("recv SyncAppendEntriesReply, maybe replica already dropped"); syncNodeEventLog(ths, "recv sync-append-entries-reply, maybe replica already dropped");
return ret; return 0;
} }
// drop stale response // drop stale response
if (pMsg->term < ths->pRaftStore->currentTerm) { if (pMsg->term < ths->pRaftStore->currentTerm) {
sTrace("recv SyncAppendEntriesReply, drop stale response, receive_term:%lu current_term:%lu", pMsg->term, char logBuf[128];
ths->pRaftStore->currentTerm); snprintf(logBuf, sizeof(logBuf), "recv sync-append-entries-reply, recv-term:%lu, drop stale response", pMsg->term);
return ret; syncNodeEventLog(ths, logBuf);
return 0;
} }
syncIndexMgrLog2("recv SyncAppendEntriesReply, before pNextIndex:", ths->pNextIndex);
syncIndexMgrLog2("recv SyncAppendEntriesReply, before pMatchIndex:", ths->pMatchIndex);
if (gRaftDetailLog) { if (gRaftDetailLog) {
SSnapshot snapshot; syncNodeEventLog(ths, "recv sync-append-entries-reply, before");
ths->pFsm->FpGetSnapshotInfo(ths->pFsm, &snapshot);
sTrace("recv SyncAppendEntriesReply, before snapshot.lastApplyIndex:%ld, snapshot.lastApplyTerm:%lu",
snapshot.lastApplyIndex, snapshot.lastApplyTerm);
} }
syncIndexMgrLog2("recv sync-append-entries-reply, before pNextIndex:", ths->pNextIndex);
syncIndexMgrLog2("recv sync-append-entries-reply, before pMatchIndex:", ths->pMatchIndex);
// no need this code, because if I receive reply.term, then I must have sent for that term. // no need this code, because if I receive reply.term, then I must have sent for that term.
// if (pMsg->term > ths->pRaftStore->currentTerm) { // if (pMsg->term > ths->pRaftStore->currentTerm) {
@ -134,12 +129,10 @@ int32_t syncNodeOnAppendEntriesReplySnapshotCb(SSyncNode* ths, SyncAppendEntries
// } // }
if (pMsg->term > ths->pRaftStore->currentTerm) { if (pMsg->term > ths->pRaftStore->currentTerm) {
char logBuf[128] = {0}; char logBuf[128];
snprintf(logBuf, sizeof(logBuf), "recv SyncAppendEntriesReply, error term, receive_term:%lu current_term:%lu", snprintf(logBuf, sizeof(logBuf), "recv sync-append-entries-reply, error term, recv-term:%lu", pMsg->term);
pMsg->term, ths->pRaftStore->currentTerm); syncNodeErrorLog(ths, logBuf);
syncNodeLog2(logBuf, ths); return -1;
sError("%s", logBuf);
return ret;
} }
ASSERT(pMsg->term == ths->pRaftStore->currentTerm); ASSERT(pMsg->term == ths->pRaftStore->currentTerm);
@ -228,8 +221,11 @@ int32_t syncNodeOnAppendEntriesReplySnapshotCb(SSyncNode* ths, SyncAppendEntries
} }
} }
syncIndexMgrLog2("recv SyncAppendEntriesReply, after pNextIndex:", ths->pNextIndex); if (gRaftDetailLog) {
syncIndexMgrLog2("recv SyncAppendEntriesReply, after pMatchIndex:", ths->pMatchIndex); syncNodeEventLog(ths, "recv sync-append-entries-reply, after");
}
syncIndexMgrLog2("recv sync-append-entries-reply, after pNextIndex:", ths->pNextIndex);
syncIndexMgrLog2("recv sync-append-entries-reply, after pMatchIndex:", ths->pMatchIndex);
return ret; return ret;
} }

31
source/libs/sync/src/syncMain.c
View File

@ -416,7 +416,7 @@ int32_t syncGetSnapshotMetaByIndex(int64_t rid, SyncIndex snapshotIndex, struct
} }
} }
sMeta->lastConfigIndex = lastIndex; sMeta->lastConfigIndex = lastIndex;
sTrace("vgId:%d, get snapshot meta by index:%" PRId64 " lastConfigIndex:%" PRId64, pSyncNode->vgId, snapshotIndex, sTrace("vgId:%d, get snapshot meta by index:%" PRId64 " lcindex:%" PRId64, pSyncNode->vgId, snapshotIndex,
sMeta->lastConfigIndex); sMeta->lastConfigIndex);
taosReleaseRef(tsNodeRefId, pSyncNode->rid); taosReleaseRef(tsNodeRefId, pSyncNode->rid);
@ -433,8 +433,9 @@ SyncIndex syncNodeGetSnapshotConfigIndex(SSyncNode* pSyncNode, SyncIndex snapsho
lastIndex = (pSyncNode->pRaftCfg->configIndexArr)[i]; lastIndex = (pSyncNode->pRaftCfg->configIndexArr)[i];
} }
} }
sTrace("vgId:%d, sync get snapshot last config index, index:%ld lcindex:%ld", pSyncNode->vgId, snapshotLastApplyIndex,
lastIndex);
sTrace("sync syncNodeGetSnapshotConfigIndex index:%ld lastConfigIndex:%ld", snapshotLastApplyIndex, lastIndex);
return lastIndex; return lastIndex;
} }
@ -1310,6 +1311,10 @@ void syncNodeEventLog(const SSyncNode* pSyncNode, char* str) {
SyncIndex logBeginIndex = pSyncNode->pLogStore->syncLogBeginIndex(pSyncNode->pLogStore); SyncIndex logBeginIndex = pSyncNode->pLogStore->syncLogBeginIndex(pSyncNode->pLogStore);
char* pCfgStr = syncCfg2SimpleStr(&(pSyncNode->pRaftCfg->cfg)); char* pCfgStr = syncCfg2SimpleStr(&(pSyncNode->pRaftCfg->cfg));
char* printStr = "";
if (pCfgStr != NULL) {
printStr = pCfgStr;
}
if (userStrLen < 256) { if (userStrLen < 256) {
char logBuf[256 + 256]; char logBuf[256 + 256];
@ -1321,7 +1326,7 @@ void syncNodeEventLog(const SSyncNode* pSyncNode, char* str) {
pSyncNode->vgId, syncUtilState2String(pSyncNode->state), str, pSyncNode->pRaftStore->currentTerm, pSyncNode->vgId, syncUtilState2String(pSyncNode->state), str, pSyncNode->pRaftStore->currentTerm,
pSyncNode->commitIndex, logBeginIndex, logLastIndex, snapshot.lastApplyIndex, pSyncNode->commitIndex, logBeginIndex, logLastIndex, snapshot.lastApplyIndex,
pSyncNode->pRaftCfg->isStandBy, pSyncNode->replicaNum, pSyncNode->pRaftCfg->lastConfigIndex, pSyncNode->pRaftCfg->isStandBy, pSyncNode->replicaNum, pSyncNode->pRaftCfg->lastConfigIndex,
pSyncNode->changing, pCfgStr); pSyncNode->changing, printStr);
} else { } else {
snprintf(logBuf, sizeof(logBuf), "%s", str); snprintf(logBuf, sizeof(logBuf), "%s", str);
} }
@ -1338,7 +1343,7 @@ void syncNodeEventLog(const SSyncNode* pSyncNode, char* str) {
pSyncNode->vgId, syncUtilState2String(pSyncNode->state), str, pSyncNode->pRaftStore->currentTerm, pSyncNode->vgId, syncUtilState2String(pSyncNode->state), str, pSyncNode->pRaftStore->currentTerm,
pSyncNode->commitIndex, logBeginIndex, logLastIndex, snapshot.lastApplyIndex, pSyncNode->commitIndex, logBeginIndex, logLastIndex, snapshot.lastApplyIndex,
pSyncNode->pRaftCfg->isStandBy, pSyncNode->replicaNum, pSyncNode->pRaftCfg->lastConfigIndex, pSyncNode->pRaftCfg->isStandBy, pSyncNode->replicaNum, pSyncNode->pRaftCfg->lastConfigIndex,
pSyncNode->changing, pCfgStr); pSyncNode->changing, printStr);
} else { } else {
snprintf(s, len, "%s", str); snprintf(s, len, "%s", str);
} }
@ -1957,17 +1962,21 @@ SyncTerm syncNodeGetPreTerm(SSyncNode* pSyncNode, SyncIndex index) {
taosMemoryFree(pPreEntry); taosMemoryFree(pPreEntry);
return preTerm; return preTerm;
} else { } else {
if (terrno == TSDB_CODE_WAL_LOG_NOT_EXIST) { SSnapshot snapshot = {.data = NULL, .lastApplyIndex = -1, .lastApplyTerm = 0, .lastConfigIndex = -1};
SSnapshot snapshot = {.data = NULL, .lastApplyIndex = -1, .lastApplyTerm = 0, .lastConfigIndex = -1}; if (pSyncNode->pFsm->FpGetSnapshotInfo != NULL) {
if (pSyncNode->pFsm->FpGetSnapshotInfo != NULL) { pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, &snapshot);
pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, &snapshot); if (snapshot.lastApplyIndex == preIndex) {
if (snapshot.lastApplyIndex == preIndex) { return snapshot.lastApplyTerm;
return snapshot.lastApplyTerm;
}
} }
} }
} }
do {
char logBuf[128];
snprintf(logBuf, sizeof(logBuf), "sync node get pre term error, index:%ld", index);
syncNodeErrorLog(pSyncNode, logBuf);
} while (0);
return SYNC_TERM_INVALID; return SYNC_TERM_INVALID;
} }

38
source/libs/sync/src/syncRaftCfg.c
View File

@ -107,26 +107,30 @@ char *syncCfg2Str(SSyncCfg *pSyncCfg) {
} }
char *syncCfg2SimpleStr(SSyncCfg *pSyncCfg) { char *syncCfg2SimpleStr(SSyncCfg *pSyncCfg) {
int32_t len = 512; if (pSyncCfg != NULL) {
char *s = taosMemoryMalloc(len); int32_t len = 512;
memset(s, 0, len); char *s = taosMemoryMalloc(len);
memset(s, 0, len);
snprintf(s, len, "{replica-num:%d, my-index:%d, ", pSyncCfg->replicaNum, pSyncCfg->myIndex); snprintf(s, len, "{replica-num:%d, my-index:%d, ", pSyncCfg->replicaNum, pSyncCfg->myIndex);
char *p = s + strlen(s); char *p = s + strlen(s);
for (int i = 0; i < pSyncCfg->replicaNum; ++i) { for (int i = 0; i < pSyncCfg->replicaNum; ++i) {
/* /*
if (p + 128 + 32 > s + len) { if (p + 128 + 32 > s + len) {
break; break;
}
*/
char buf[128 + 32];
snprintf(buf, sizeof(buf), "%s:%d, ", pSyncCfg->nodeInfo[i].nodeFqdn, pSyncCfg->nodeInfo[i].nodePort);
strncpy(p, buf, sizeof(buf));
p = s + strlen(s);
} }
*/ strcpy(p - 2, "}");
char buf[128 + 32];
snprintf(buf, sizeof(buf), "%s:%d, ", pSyncCfg->nodeInfo[i].nodeFqdn, pSyncCfg->nodeInfo[i].nodePort);
strncpy(p, buf, sizeof(buf));
p = s + strlen(s);
}
strcpy(p - 2, "}");
return s; return s;
}
return NULL;
} }
int32_t syncCfgFromJson(const cJSON *pRoot, SSyncCfg *pSyncCfg) { int32_t syncCfgFromJson(const cJSON *pRoot, SSyncCfg *pSyncCfg) {

245
source/libs/sync/src/syncRaftLog.c
View File

@ -16,10 +16,10 @@
#include "syncRaftLog.h" #include "syncRaftLog.h"
#include "syncRaftCfg.h" #include "syncRaftCfg.h"
#include "syncRaftStore.h" #include "syncRaftStore.h"
#include "wal.h"
// refactor, log[0 .. n] ==> log[m .. n] // refactor, log[0 .. n] ==> log[m .. n]
static int32_t raftLogSetBeginIndex(struct SSyncLogStore* pLogStore, SyncIndex beginIndex); static int32_t raftLogRestoreFromSnapshot(struct SSyncLogStore* pLogStore, SyncIndex snapshotIndex);
// static int32_t raftLogSetBeginIndex(struct SSyncLogStore* pLogStore, SyncIndex beginIndex);
static SyncIndex raftLogBeginIndex(struct SSyncLogStore* pLogStore); static SyncIndex raftLogBeginIndex(struct SSyncLogStore* pLogStore);
static SyncIndex raftLogEndIndex(struct SSyncLogStore* pLogStore); static SyncIndex raftLogEndIndex(struct SSyncLogStore* pLogStore);
static SyncIndex raftLogWriteIndex(struct SSyncLogStore* pLogStore); static SyncIndex raftLogWriteIndex(struct SSyncLogStore* pLogStore);
@ -45,31 +45,43 @@ static int32_t logStoreUpdateCommitIndex(SSyncLogStore* pLogStore, SyncI
static SyncIndex logStoreGetCommitIndex(SSyncLogStore* pLogStore); static SyncIndex logStoreGetCommitIndex(SSyncLogStore* pLogStore);
// refactor, log[0 .. n] ==> log[m .. n] // refactor, log[0 .. n] ==> log[m .. n]
/*
static int32_t raftLogSetBeginIndex(struct SSyncLogStore* pLogStore, SyncIndex beginIndex) { static int32_t raftLogSetBeginIndex(struct SSyncLogStore* pLogStore, SyncIndex beginIndex) {
sTrace("raftLogSetBeginIndex beginIndex:%ld", beginIndex);
// if beginIndex == 0, donot need call this funciton // if beginIndex == 0, donot need call this funciton
ASSERT(beginIndex > 0); ASSERT(beginIndex > 0);
SSyncLogStoreData* pData = pLogStore->data; SSyncLogStoreData* pData = pLogStore->data;
SWal* pWal = pData->pWal; SWal* pWal = pData->pWal;
sTrace("vgId:%d, reset wal begin index:%ld", pData->pSyncNode->vgId, beginIndex);
pData->beginIndex = beginIndex; pData->beginIndex = beginIndex;
walRestoreFromSnapshot(pWal, beginIndex - 1); walRestoreFromSnapshot(pWal, beginIndex - 1);
return 0; return 0;
} }
*/
static int32_t raftLogRestoreFromSnapshot(struct SSyncLogStore* pLogStore, SyncIndex snapshotIndex) {
ASSERT(snapshotIndex >= 0);
SSyncLogStoreData* pData = pLogStore->data;
SWal* pWal = pData->pWal;
walRestoreFromSnapshot(pWal, snapshotIndex);
return 0;
}
static SyncIndex raftLogBeginIndex(struct SSyncLogStore* pLogStore) { static SyncIndex raftLogBeginIndex(struct SSyncLogStore* pLogStore) {
SSyncLogStoreData* pData = pLogStore->data; SSyncLogStoreData* pData = pLogStore->data;
SWal* pWal = pData->pWal; SWal* pWal = pData->pWal;
return pData->beginIndex; SyncIndex firstVer = walGetFirstVer(pWal);
return firstVer;
} }
static SyncIndex raftLogEndIndex(struct SSyncLogStore* pLogStore) { return raftLogLastIndex(pLogStore); } static SyncIndex raftLogEndIndex(struct SSyncLogStore* pLogStore) { return raftLogLastIndex(pLogStore); }
static bool raftLogIsEmpty(struct SSyncLogStore* pLogStore) { static bool raftLogIsEmpty(struct SSyncLogStore* pLogStore) {
SyncIndex beginIndex = raftLogBeginIndex(pLogStore); SSyncLogStoreData* pData = pLogStore->data;
SyncIndex endIndex = raftLogEndIndex(pLogStore); SWal* pWal = pData->pWal;
return (endIndex < beginIndex); return walIsEmpty(pWal);
} }
static int32_t raftLogEntryCount(struct SSyncLogStore* pLogStore) { static int32_t raftLogEntryCount(struct SSyncLogStore* pLogStore) {
@ -96,23 +108,8 @@ static SyncIndex raftLogLastIndex(struct SSyncLogStore* pLogStore) {
SSyncLogStoreData* pData = pLogStore->data; SSyncLogStoreData* pData = pLogStore->data;
SWal* pWal = pData->pWal; SWal* pWal = pData->pWal;
SyncIndex lastVer = walGetLastVer(pWal); SyncIndex lastVer = walGetLastVer(pWal);
SyncIndex firstVer = walGetFirstVer(pWal);
if (lastVer < firstVer) { return lastVer;
// no record
lastIndex = -1;
} else {
if (firstVer >= 0) {
lastIndex = lastVer;
} else if (firstVer == -1) {
lastIndex = -1;
} else {
ASSERT(0);
}
}
return lastIndex;
} }
static SyncIndex raftLogWriteIndex(struct SSyncLogStore* pLogStore) { static SyncIndex raftLogWriteIndex(struct SSyncLogStore* pLogStore) {
@ -122,6 +119,26 @@ static SyncIndex raftLogWriteIndex(struct SSyncLogStore* pLogStore) {
return lastVer + 1; return lastVer + 1;
} }
static SyncTerm raftLogLastTerm(struct SSyncLogStore* pLogStore) {
SSyncLogStoreData* pData = pLogStore->data;
SWal* pWal = pData->pWal;
if (walIsEmpty(pWal)) {
return 0;
} else {
SSyncRaftEntry* pLastEntry;
int32_t code = raftLogGetLastEntry(pLogStore, &pLastEntry);
ASSERT(code == 0);
ASSERT(pLastEntry != NULL);
SyncTerm lastTerm = pLastEntry->term;
taosMemoryFree(pLastEntry);
return lastTerm;
}
return 0;
}
/*
static SyncTerm raftLogLastTerm(struct SSyncLogStore* pLogStore) { static SyncTerm raftLogLastTerm(struct SSyncLogStore* pLogStore) {
SyncTerm lastTerm = 0; SyncTerm lastTerm = 0;
if (raftLogEntryCount(pLogStore) == 0) { if (raftLogEntryCount(pLogStore) == 0) {
@ -137,6 +154,7 @@ static SyncTerm raftLogLastTerm(struct SSyncLogStore* pLogStore) {
} }
return lastTerm; return lastTerm;
} }
*/
static int32_t raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry) { static int32_t raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry) {
SSyncLogStoreData* pData = pLogStore->data; SSyncLogStoreData* pData = pLogStore->data;
@ -160,8 +178,12 @@ static int32_t raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncRaftEntr
const char* errStr = tstrerror(err); const char* errStr = tstrerror(err);
int32_t sysErr = errno; int32_t sysErr = errno;
const char* sysErrStr = strerror(errno); const char* sysErrStr = strerror(errno);
sError("vgId:%d wal write error, index:%ld, err:%d %X, msg:%s, syserr:%d, sysmsg:%s", pData->pSyncNode->vgId,
pEntry->index, err, err, errStr, sysErr, sysErrStr); char logBuf[128];
snprintf(logBuf, sizeof(logBuf), "wal write error, index:%ld, err:%d %X, msg:%s, syserr:%d, sysmsg:%s",
pEntry->index, err, err, errStr, sysErr, sysErrStr);
syncNodeErrorLog(pData->pSyncNode, logBuf);
ASSERT(0); ASSERT(0);
} }
@ -229,7 +251,8 @@ static int32_t raftLogGetEntry(struct SSyncLogStore* pLogStore, SyncIndex index,
*ppEntry = NULL; *ppEntry = NULL;
SWalReadHandle* pWalHandle = walOpenReadHandle(pWal); // SWalReadHandle* pWalHandle = walOpenReadHandle(pWal);
SWalReadHandle* pWalHandle = pData->pWalHandle;
if (pWalHandle == NULL) { if (pWalHandle == NULL) {
return -1; return -1;
} }
@ -240,12 +263,23 @@ static int32_t raftLogGetEntry(struct SSyncLogStore* pLogStore, SyncIndex index,
const char* errStr = tstrerror(err); const char* errStr = tstrerror(err);
int32_t sysErr = errno; int32_t sysErr = errno;
const char* sysErrStr = strerror(errno); const char* sysErrStr = strerror(errno);
sError("vgId:%d wal read error, index:%ld, err:%d %X, msg:%s, syserr:%d, sysmsg:%s", pData->pSyncNode->vgId, index,
err, err, errStr, sysErr, sysErrStr);
int32_t saveErr = terrno; do {
walCloseReadHandle(pWalHandle); char logBuf[128];
terrno = saveErr; snprintf(logBuf, sizeof(logBuf), "wal read error, index:%ld, err:%d %X, msg:%s, syserr:%d, sysmsg:%s", index, err,
err, errStr, sysErr, sysErrStr);
if (terrno == TSDB_CODE_WAL_LOG_NOT_EXIST) {
syncNodeEventLog(pData->pSyncNode, logBuf);
} else {
syncNodeErrorLog(pData->pSyncNode, logBuf);
}
} while (0);
/*
int32_t saveErr = terrno;
walCloseReadHandle(pWalHandle);
terrno = saveErr;
*/
return code; return code;
} }
@ -261,9 +295,11 @@ static int32_t raftLogGetEntry(struct SSyncLogStore* pLogStore, SyncIndex index,
ASSERT((*ppEntry)->dataLen == pWalHandle->pHead->head.bodyLen); ASSERT((*ppEntry)->dataLen == pWalHandle->pHead->head.bodyLen);
memcpy((*ppEntry)->data, pWalHandle->pHead->head.body, pWalHandle->pHead->head.bodyLen); memcpy((*ppEntry)->data, pWalHandle->pHead->head.body, pWalHandle->pHead->head.bodyLen);
int32_t saveErr = terrno; /*
walCloseReadHandle(pWalHandle); int32_t saveErr = terrno;
terrno = saveErr; walCloseReadHandle(pWalHandle);
terrno = saveErr;
*/
return code; return code;
} }
@ -285,6 +321,25 @@ static int32_t raftLogTruncate(struct SSyncLogStore* pLogStore, SyncIndex fromIn
return code; return code;
} }
static int32_t raftLogGetLastEntry(SSyncLogStore* pLogStore, SSyncRaftEntry** ppLastEntry) {
SSyncLogStoreData* pData = pLogStore->data;
SWal* pWal = pData->pWal;
ASSERT(ppLastEntry != NULL);
*ppLastEntry = NULL;
if (walIsEmpty(pWal)) {
terrno = TSDB_CODE_WAL_LOG_NOT_EXIST;
return -1;
} else {
SyncIndex lastIndex = raftLogLastIndex(pLogStore);
int32_t code = raftLogGetEntry(pLogStore, lastIndex, ppLastEntry);
return code;
}
return -1;
}
/*
static int32_t raftLogGetLastEntry(SSyncLogStore* pLogStore, SSyncRaftEntry** ppLastEntry) { static int32_t raftLogGetLastEntry(SSyncLogStore* pLogStore, SSyncRaftEntry** ppLastEntry) {
*ppLastEntry = NULL; *ppLastEntry = NULL;
if (raftLogEntryCount(pLogStore) == 0) { if (raftLogEntryCount(pLogStore) == 0) {
@ -294,6 +349,7 @@ static int32_t raftLogGetLastEntry(SSyncLogStore* pLogStore, SSyncRaftEntry** pp
int32_t code = raftLogGetEntry(pLogStore, lastIndex, ppLastEntry); int32_t code = raftLogGetEntry(pLogStore, lastIndex, ppLastEntry);
return code; return code;
} }
*/
//------------------------------- //-------------------------------
SSyncLogStore* logStoreCreate(SSyncNode* pSyncNode) { SSyncLogStore* logStoreCreate(SSyncNode* pSyncNode) {
@ -306,16 +362,22 @@ SSyncLogStore* logStoreCreate(SSyncNode* pSyncNode) {
SSyncLogStoreData* pData = pLogStore->data; SSyncLogStoreData* pData = pLogStore->data;
pData->pSyncNode = pSyncNode; pData->pSyncNode = pSyncNode;
pData->pWal = pSyncNode->pWal; pData->pWal = pSyncNode->pWal;
ASSERT(pData->pWal != NULL);
SyncIndex firstVer = walGetFirstVer(pData->pWal); pData->pWalHandle = walOpenReadHandle(pData->pWal);
SyncIndex lastVer = walGetLastVer(pData->pWal); ASSERT(pData->pWalHandle != NULL);
if (firstVer >= 0) {
pData->beginIndex = firstVer; /*
} else if (firstVer == -1) { SyncIndex firstVer = walGetFirstVer(pData->pWal);
pData->beginIndex = lastVer + 1; SyncIndex lastVer = walGetLastVer(pData->pWal);
} else { if (firstVer >= 0) {
ASSERT(0); pData->beginIndex = firstVer;
} } else if (firstVer == -1) {
pData->beginIndex = lastVer + 1;
} else {
ASSERT(0);
}
*/
pLogStore->appendEntry = logStoreAppendEntry; pLogStore->appendEntry = logStoreAppendEntry;
pLogStore->getEntry = logStoreGetEntry; pLogStore->getEntry = logStoreGetEntry;
@ -325,7 +387,8 @@ SSyncLogStore* logStoreCreate(SSyncNode* pSyncNode) {
pLogStore->updateCommitIndex = logStoreUpdateCommitIndex; pLogStore->updateCommitIndex = logStoreUpdateCommitIndex;
pLogStore->getCommitIndex = logStoreGetCommitIndex; pLogStore->getCommitIndex = logStoreGetCommitIndex;
pLogStore->syncLogSetBeginIndex = raftLogSetBeginIndex; // pLogStore->syncLogSetBeginIndex = raftLogSetBeginIndex;
pLogStore->syncLogRestoreFromSnapshot = raftLogRestoreFromSnapshot;
pLogStore->syncLogBeginIndex = raftLogBeginIndex; pLogStore->syncLogBeginIndex = raftLogBeginIndex;
pLogStore->syncLogEndIndex = raftLogEndIndex; pLogStore->syncLogEndIndex = raftLogEndIndex;
pLogStore->syncLogIsEmpty = raftLogIsEmpty; pLogStore->syncLogIsEmpty = raftLogIsEmpty;
@ -344,6 +407,11 @@ SSyncLogStore* logStoreCreate(SSyncNode* pSyncNode) {
void logStoreDestory(SSyncLogStore* pLogStore) { void logStoreDestory(SSyncLogStore* pLogStore) {
if (pLogStore != NULL) { if (pLogStore != NULL) {
SSyncLogStoreData* pData = pLogStore->data;
if (pData->pWalHandle != NULL) {
walCloseReadHandle(pData->pWalHandle);
}
taosMemoryFree(pLogStore->data); taosMemoryFree(pLogStore->data);
taosMemoryFree(pLogStore); taosMemoryFree(pLogStore);
} }
@ -368,8 +436,11 @@ int32_t logStoreAppendEntry(SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry) {
const char* errStr = tstrerror(err); const char* errStr = tstrerror(err);
int32_t sysErr = errno; int32_t sysErr = errno;
const char* sysErrStr = strerror(errno); const char* sysErrStr = strerror(errno);
sError("vgId:%d wal write error, index:%ld, err:%d %X, msg:%s, syserr:%d, sysmsg:%s", pData->pSyncNode->vgId,
pEntry->index, err, err, errStr, sysErr, sysErrStr); char logBuf[128];
snprintf(logBuf, sizeof(logBuf), "wal write error, index:%ld, err:%d %X, msg:%s, syserr:%d, sysmsg:%s",
pEntry->index, err, err, errStr, sysErr, sysErrStr);
syncNodeErrorLog(pData->pSyncNode, logBuf);
ASSERT(0); ASSERT(0);
} }
@ -389,7 +460,8 @@ SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) {
SWal* pWal = pData->pWal; SWal* pWal = pData->pWal;
if (index >= SYNC_INDEX_BEGIN && index <= logStoreLastIndex(pLogStore)) { if (index >= SYNC_INDEX_BEGIN && index <= logStoreLastIndex(pLogStore)) {
SWalReadHandle* pWalHandle = walOpenReadHandle(pWal); // SWalReadHandle* pWalHandle = walOpenReadHandle(pWal);
SWalReadHandle* pWalHandle = pData->pWalHandle;
ASSERT(pWalHandle != NULL); ASSERT(pWalHandle != NULL);
int32_t code = walReadWithHandle(pWalHandle, index); int32_t code = walReadWithHandle(pWalHandle, index);
@ -398,12 +470,20 @@ SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) {
const char* errStr = tstrerror(err); const char* errStr = tstrerror(err);
int32_t sysErr = errno; int32_t sysErr = errno;
const char* sysErrStr = strerror(errno); const char* sysErrStr = strerror(errno);
sError("vgId:%d wal read error, index:%ld, err:%d %X, msg:%s, syserr:%d, sysmsg:%s", pData->pSyncNode->vgId,
index, err, err, errStr, sysErr, sysErrStr); do {
char logBuf[128];
snprintf(logBuf, sizeof(logBuf), "wal read error, index:%ld, err:%d %X, msg:%s, syserr:%d, sysmsg:%s", index,
err, err, errStr, sysErr, sysErrStr);
if (terrno == TSDB_CODE_WAL_LOG_NOT_EXIST) {
syncNodeEventLog(pData->pSyncNode, logBuf);
} else {
syncNodeErrorLog(pData->pSyncNode, logBuf);
}
} while (0);
ASSERT(0); ASSERT(0);
} }
// ASSERT(walReadWithHandle(pWalHandle, index) == 0);
SSyncRaftEntry* pEntry = syncEntryBuild(pWalHandle->pHead->head.bodyLen); SSyncRaftEntry* pEntry = syncEntryBuild(pWalHandle->pHead->head.bodyLen);
ASSERT(pEntry != NULL); ASSERT(pEntry != NULL);
@ -417,9 +497,11 @@ SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) {
ASSERT(pEntry->dataLen == pWalHandle->pHead->head.bodyLen); ASSERT(pEntry->dataLen == pWalHandle->pHead->head.bodyLen);
memcpy(pEntry->data, pWalHandle->pHead->head.body, pWalHandle->pHead->head.bodyLen); memcpy(pEntry->data, pWalHandle->pHead->head.body, pWalHandle->pHead->head.bodyLen);
int32_t saveErr = terrno; /*
walCloseReadHandle(pWalHandle); int32_t saveErr = terrno;
terrno = saveErr; walCloseReadHandle(pWalHandle);
terrno = saveErr;
*/
return pEntry; return pEntry;
@ -498,6 +580,7 @@ SSyncRaftEntry* logStoreGetLastEntry(SSyncLogStore* pLogStore) {
return pEntry; return pEntry;
} }
/*
cJSON* logStore2Json(SSyncLogStore* pLogStore) { cJSON* logStore2Json(SSyncLogStore* pLogStore) {
char u64buf[128] = {0}; char u64buf[128] = {0};
SSyncLogStoreData* pData = (SSyncLogStoreData*)pLogStore->data; SSyncLogStoreData* pData = (SSyncLogStoreData*)pLogStore->data;
@ -544,6 +627,57 @@ cJSON* logStore2Json(SSyncLogStore* pLogStore) {
cJSON_AddItemToObject(pJson, "SSyncLogStore", pRoot); cJSON_AddItemToObject(pJson, "SSyncLogStore", pRoot);
return pJson; return pJson;
} }
*/
cJSON* logStore2Json(SSyncLogStore* pLogStore) {
char u64buf[128] = {0};
SSyncLogStoreData* pData = (SSyncLogStoreData*)pLogStore->data;
cJSON* pRoot = cJSON_CreateObject();
if (pData != NULL && pData->pWal != NULL) {
snprintf(u64buf, sizeof(u64buf), "%p", pData->pSyncNode);
cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf);
snprintf(u64buf, sizeof(u64buf), "%p", pData->pWal);
cJSON_AddStringToObject(pRoot, "pWal", u64buf);
SyncIndex beginIndex = raftLogBeginIndex(pLogStore);
snprintf(u64buf, sizeof(u64buf), "%ld", beginIndex);
cJSON_AddStringToObject(pRoot, "beginIndex", u64buf);
SyncIndex endIndex = raftLogEndIndex(pLogStore);
snprintf(u64buf, sizeof(u64buf), "%ld", endIndex);
cJSON_AddStringToObject(pRoot, "endIndex", u64buf);
int32_t count = raftLogEntryCount(pLogStore);
cJSON_AddNumberToObject(pRoot, "entryCount", count);
snprintf(u64buf, sizeof(u64buf), "%ld", raftLogWriteIndex(pLogStore));
cJSON_AddStringToObject(pRoot, "WriteIndex", u64buf);
snprintf(u64buf, sizeof(u64buf), "%d", raftLogIsEmpty(pLogStore));
cJSON_AddStringToObject(pRoot, "IsEmpty", u64buf);
snprintf(u64buf, sizeof(u64buf), "%ld", raftLogLastIndex(pLogStore));
cJSON_AddStringToObject(pRoot, "LastIndex", u64buf);
snprintf(u64buf, sizeof(u64buf), "%lu", raftLogLastTerm(pLogStore));
cJSON_AddStringToObject(pRoot, "LastTerm", u64buf);
cJSON* pEntries = cJSON_CreateArray();
cJSON_AddItemToObject(pRoot, "pEntries", pEntries);
if (!raftLogIsEmpty(pLogStore)) {
for (SyncIndex i = beginIndex; i <= endIndex; ++i) {
SSyncRaftEntry* pEntry = logStoreGetEntry(pLogStore, i);
cJSON_AddItemToArray(pEntries, syncEntry2Json(pEntry));
syncEntryDestory(pEntry);
}
}
}
cJSON* pJson = cJSON_CreateObject();
cJSON_AddItemToObject(pJson, "SSyncLogStore", pRoot);
return pJson;
}
char* logStore2Str(SSyncLogStore* pLogStore) { char* logStore2Str(SSyncLogStore* pLogStore) {
cJSON* pJson = logStore2Json(pLogStore); cJSON* pJson = logStore2Json(pLogStore);
@ -563,7 +697,8 @@ cJSON* logStoreSimple2Json(SSyncLogStore* pLogStore) {
snprintf(u64buf, sizeof(u64buf), "%p", pData->pWal); snprintf(u64buf, sizeof(u64buf), "%p", pData->pWal);
cJSON_AddStringToObject(pRoot, "pWal", u64buf); cJSON_AddStringToObject(pRoot, "pWal", u64buf);
snprintf(u64buf, sizeof(u64buf), "%ld", pData->beginIndex); SyncIndex beginIndex = raftLogBeginIndex(pLogStore);
snprintf(u64buf, sizeof(u64buf), "%ld", beginIndex);
cJSON_AddStringToObject(pRoot, "beginIndex", u64buf); cJSON_AddStringToObject(pRoot, "beginIndex", u64buf);
SyncIndex endIndex = raftLogEndIndex(pLogStore); SyncIndex endIndex = raftLogEndIndex(pLogStore);

11
source/libs/sync/src/syncReplication.c
View File

@ -219,6 +219,17 @@ int32_t syncNodeReplicate(SSyncNode* pSyncNode) {
int32_t syncNodeAppendEntries(SSyncNode* pSyncNode, const SRaftId* destRaftId, const SyncAppendEntries* pMsg) { int32_t syncNodeAppendEntries(SSyncNode* pSyncNode, const SRaftId* destRaftId, const SyncAppendEntries* pMsg) {
int32_t ret = 0; int32_t ret = 0;
do {
char host[128];
uint16_t port;
syncUtilU642Addr(destRaftId->addr, host, sizeof(host), &port);
sDebug(
"vgId:%d, send sync-append-entries to %s:%d, term:%lu, pre-index:%ld, pre-term:%lu, pterm:%lu, commit:%ld, "
"datalen:%d",
pSyncNode->vgId, host, port, pMsg->term, pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->privateTerm,
pMsg->commitIndex, pMsg->dataLen);
} while (0);
SRpcMsg rpcMsg; SRpcMsg rpcMsg;
syncAppendEntries2RpcMsg(pMsg, &rpcMsg); syncAppendEntries2RpcMsg(pMsg, &rpcMsg);
syncNodeSendMsgById(destRaftId, pSyncNode, &rpcMsg); syncNodeSendMsgById(destRaftId, pSyncNode, &rpcMsg);

3
source/libs/sync/src/syncSnapshot.c
View File

@ -545,7 +545,8 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) {
pReceiver->pSyncNode->commitIndex = pReceiver->snapshot.lastApplyIndex; pReceiver->pSyncNode->commitIndex = pReceiver->snapshot.lastApplyIndex;
} }
pSyncNode->pLogStore->syncLogSetBeginIndex(pSyncNode->pLogStore, pMsg->lastIndex + 1); // pSyncNode->pLogStore->syncLogSetBeginIndex(pSyncNode->pLogStore, pMsg->lastIndex + 1);
pSyncNode->pLogStore->syncLogRestoreFromSnapshot(pSyncNode->pLogStore, pMsg->lastIndex);
// maybe update lastconfig // maybe update lastconfig
if (pMsg->lastConfigIndex >= SYNC_INDEX_BEGIN) { if (pMsg->lastConfigIndex >= SYNC_INDEX_BEGIN) {

33
source/libs/sync/test/syncRaftLogTest2.cpp
View File

@ -113,7 +113,8 @@ void test2() {
pLogStore = logStoreCreate(pSyncNode); pLogStore = logStoreCreate(pSyncNode);
assert(pLogStore); assert(pLogStore);
pSyncNode->pLogStore = pLogStore; pSyncNode->pLogStore = pLogStore;
pLogStore->syncLogSetBeginIndex(pLogStore, 5); //pLogStore->syncLogSetBeginIndex(pLogStore, 5);
pLogStore->syncLogRestoreFromSnapshot(pLogStore, 4);
logStoreLog2((char*)"\n\n\ntest2 ----- ", pLogStore); logStoreLog2((char*)"\n\n\ntest2 ----- ", pLogStore);
if (gAssert) { if (gAssert) {
@ -228,7 +229,8 @@ void test4() {
assert(pLogStore); assert(pLogStore);
pSyncNode->pLogStore = pLogStore; pSyncNode->pLogStore = pLogStore;
logStoreLog2((char*)"\n\n\ntest4 ----- ", pLogStore); logStoreLog2((char*)"\n\n\ntest4 ----- ", pLogStore);
pLogStore->syncLogSetBeginIndex(pLogStore, 5); //pLogStore->syncLogSetBeginIndex(pLogStore, 5);
pLogStore->syncLogRestoreFromSnapshot(pLogStore, 4);
for (int i = 5; i <= 9; ++i) { for (int i = 5; i <= 9; ++i) {
int32_t dataLen = 10; int32_t dataLen = 10;
@ -289,7 +291,8 @@ void test5() {
assert(pLogStore); assert(pLogStore);
pSyncNode->pLogStore = pLogStore; pSyncNode->pLogStore = pLogStore;
logStoreLog2((char*)"\n\n\ntest5 ----- ", pLogStore); logStoreLog2((char*)"\n\n\ntest5 ----- ", pLogStore);
pLogStore->syncLogSetBeginIndex(pLogStore, 5); //pLogStore->syncLogSetBeginIndex(pLogStore, 5);
pLogStore->syncLogRestoreFromSnapshot(pLogStore, 4);
for (int i = 5; i <= 9; ++i) { for (int i = 5; i <= 9; ++i) {
int32_t dataLen = 10; int32_t dataLen = 10;
@ -363,7 +366,8 @@ void test6() {
assert(pLogStore); assert(pLogStore);
pSyncNode->pLogStore = pLogStore; pSyncNode->pLogStore = pLogStore;
logStoreLog2((char*)"\n\n\ntest6 ----- ", pLogStore); logStoreLog2((char*)"\n\n\ntest6 ----- ", pLogStore);
pLogStore->syncLogSetBeginIndex(pLogStore, 5); // pLogStore->syncLogSetBeginIndex(pLogStore, 5);
pLogStore->syncLogRestoreFromSnapshot(pLogStore, 4);
for (int i = 5; i <= 9; ++i) { for (int i = 5; i <= 9; ++i) {
int32_t dataLen = 10; int32_t dataLen = 10;
@ -405,14 +409,32 @@ void test6() {
assert(pLogStore->syncLogLastTerm(pLogStore) == 0); assert(pLogStore->syncLogLastTerm(pLogStore) == 0);
} }
do {
SyncIndex firstVer = walGetFirstVer(pWal);
SyncIndex lastVer = walGetLastVer(pWal);
bool isEmpty = walIsEmpty(pWal);
printf("before -------- firstVer:%ld lastVer:%ld isEmpty:%d \n", firstVer, lastVer, isEmpty);
} while (0);
logStoreDestory(pLogStore); logStoreDestory(pLogStore);
cleanup(); cleanup();
// restart // restart
init(); init();
pLogStore = logStoreCreate(pSyncNode); pLogStore = logStoreCreate(pSyncNode);
assert(pLogStore); assert(pLogStore);
pSyncNode->pLogStore = pLogStore; pSyncNode->pLogStore = pLogStore;
do {
SyncIndex firstVer = walGetFirstVer(pWal);
SyncIndex lastVer = walGetLastVer(pWal);
bool isEmpty = walIsEmpty(pWal);
printf("after -------- firstVer:%ld lastVer:%ld isEmpty:%d \n", firstVer, lastVer, isEmpty);
} while (0);
logStoreLog2((char*)"\n\n\ntest6 restart ----- ", pLogStore); logStoreLog2((char*)"\n\n\ntest6 restart ----- ", pLogStore);
if (gAssert) { if (gAssert) {
@ -432,17 +454,20 @@ void test6() {
int main(int argc, char** argv) { int main(int argc, char** argv) {
tsAsyncLog = 0; tsAsyncLog = 0;
sDebugFlag = DEBUG_TRACE + DEBUG_INFO + DEBUG_SCREEN + DEBUG_FILE; sDebugFlag = DEBUG_TRACE + DEBUG_INFO + DEBUG_SCREEN + DEBUG_FILE;
gRaftDetailLog = true;
if (argc == 2) { if (argc == 2) {
gAssert = atoi(argv[1]); gAssert = atoi(argv[1]);
} }
sTrace("gAssert : %d", gAssert); sTrace("gAssert : %d", gAssert);
/*
test1(); test1();
test2(); test2();
test3(); test3();
test4(); test4();
test5(); test5();
*/
test6(); test6();
return 0; return 0;

4
source/libs/sync/test/syncRaftLogTest3.cpp
View File

@ -312,7 +312,8 @@ void test5() {
pSyncNode->pLogStore = pLogStore; pSyncNode->pLogStore = pLogStore;
logStoreLog2((char*)"\n\n\ntest5 ----- ", pLogStore); logStoreLog2((char*)"\n\n\ntest5 ----- ", pLogStore);
pSyncNode->pLogStore->syncLogSetBeginIndex(pSyncNode->pLogStore, 6); //pSyncNode->pLogStore->syncLogSetBeginIndex(pSyncNode->pLogStore, 6);
pLogStore->syncLogRestoreFromSnapshot(pSyncNode->pLogStore, 5);
for (int i = 6; i <= 10; ++i) { for (int i = 6; i <= 10; ++i) {
int32_t dataLen = 10; int32_t dataLen = 10;
SSyncRaftEntry* pEntry = syncEntryBuild(dataLen); SSyncRaftEntry* pEntry = syncEntryBuild(dataLen);
@ -372,6 +373,7 @@ void test5() {
int main(int argc, char** argv) { int main(int argc, char** argv) {
tsAsyncLog = 0; tsAsyncLog = 0;
sDebugFlag = DEBUG_TRACE + DEBUG_INFO + DEBUG_SCREEN + DEBUG_FILE; sDebugFlag = DEBUG_TRACE + DEBUG_INFO + DEBUG_SCREEN + DEBUG_FILE;
gRaftDetailLog = true;
if (argc == 2) { if (argc == 2) {
gAssert = atoi(argv[1]); gAssert = atoi(argv[1]);

38
source/libs/transport/src/transCli.c
View File

@ -983,17 +983,18 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) {
if (pResp->code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { if (pResp->code == TSDB_CODE_RPC_NETWORK_UNAVAIL) {
if (pCtx->retryCount < pEpSet->numOfEps * 3) { if (pCtx->retryCount < pEpSet->numOfEps * 3) {
pEpSet->inUse = (++pEpSet->inUse) % pEpSet->numOfEps; pEpSet->inUse = (++pEpSet->inUse) % pEpSet->numOfEps;
if (pThrd->quit == false) {
STaskArg* arg = taosMemoryMalloc(sizeof(STaskArg));
arg->param1 = pMsg;
arg->param2 = pThrd;
transDQSched(pThrd->delayQueue, doDelayTask, arg, TRANS_RETRY_INTERVAL);
transPrintEpSet(pEpSet);
tTrace("%s use local epset, inUse: %d, retry count:%d, limit: %d", pTransInst->label, pEpSet->inUse,
pCtx->retryCount + 1, pEpSet->numOfEps * 3);
STaskArg* arg = taosMemoryMalloc(sizeof(STaskArg)); transUnrefCliHandle(pConn);
arg->param1 = pMsg; return -1;
arg->param2 = pThrd; }
transDQSched(pThrd->delayQueue, doDelayTask, arg, TRANS_RETRY_INTERVAL);
transPrintEpSet(pEpSet);
tTrace("%s use local epset, inUse: %d, retry count:%d, limit: %d", pTransInst->label, pEpSet->inUse,
pCtx->retryCount + 1, pEpSet->numOfEps * 3);
transUnrefCliHandle(pConn);
return -1;
} }
} else if (pCtx->retryCount < TRANS_RETRY_COUNT_LIMIT) { } else if (pCtx->retryCount < TRANS_RETRY_COUNT_LIMIT) {
if (pResp->contLen == 0) { if (pResp->contLen == 0) {
@ -1010,15 +1011,16 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) {
tTrace("%s use remote epset, inUse: %d, retry count:%d, limit: %d", pTransInst->label, pEpSet->inUse, tTrace("%s use remote epset, inUse: %d, retry count:%d, limit: %d", pTransInst->label, pEpSet->inUse,
pCtx->retryCount + 1, TRANS_RETRY_COUNT_LIMIT); pCtx->retryCount + 1, TRANS_RETRY_COUNT_LIMIT);
} }
if (pConn->status != ConnInPool) { if (pThrd->quit == false) {
addConnToPool(pThrd->pool, pConn); if (pConn->status != ConnInPool) {
addConnToPool(pThrd->pool, pConn);
}
STaskArg* arg = taosMemoryMalloc(sizeof(STaskArg));
arg->param1 = pMsg;
arg->param2 = pThrd;
transDQSched(pThrd->delayQueue, doDelayTask, arg, TRANS_RETRY_INTERVAL);
return -1;
} }
STaskArg* arg = taosMemoryMalloc(sizeof(STaskArg));
arg->param1 = pMsg;
arg->param2 = pThrd;
transDQSched(pThrd->delayQueue, doDelayTask, arg, TRANS_RETRY_INTERVAL);
return -1;
} }
} }

15
source/libs/wal/src/walRead.c
View File

@ -272,14 +272,23 @@ int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver) {
} }
} }
if (ver > pRead->pWal->vers.lastVer || ver < pRead->pWal->vers.firstVer) {
wError("invalid version: % " PRId64 ", first ver %ld, last ver %ld", ver, pRead->pWal->vers.firstVer,
pRead->pWal->vers.lastVer);
terrno = TSDB_CODE_WAL_LOG_NOT_EXIST;
return -1;
}
ASSERT(taosValidFile(pRead->pReadLogTFile) == true); ASSERT(taosValidFile(pRead->pReadLogTFile) == true);
code = taosReadFile(pRead->pReadLogTFile, pRead->pHead, sizeof(SWalHead)); code = taosReadFile(pRead->pReadLogTFile, pRead->pHead, sizeof(SWalHead));
if (code != sizeof(SWalHead)) { if (code != sizeof(SWalHead)) {
if (code < 0) if (code < 0)
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
else else {
terrno = TSDB_CODE_WAL_FILE_CORRUPTED; terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
ASSERT(0);
}
return -1; return -1;
} }
@ -304,8 +313,10 @@ int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver) {
pRead->pHead->head.bodyLen) { pRead->pHead->head.bodyLen) {
if (code < 0) if (code < 0)
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
else else {
terrno = TSDB_CODE_WAL_FILE_CORRUPTED; terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
ASSERT(0);
}
return -1; return -1;
} }

1
source/libs/wal/src/walWrite.c
View File

@ -150,6 +150,7 @@ int32_t walRollback(SWal *pWal, int64_t ver) {
ASSERT(code == 0); ASSERT(code == 0);
if (code != 0) { if (code != 0) {
terrno = TSDB_CODE_WAL_FILE_CORRUPTED; terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
ASSERT(0);
return -1; return -1;
} }
if (head.head.version != ver) { if (head.head.version != ver) {

2
source/util/src/tlog.c
View File

@ -743,7 +743,7 @@ cmp_end:
} }
void taosSetAllDebugFlag(int32_t flag) { void taosSetAllDebugFlag(int32_t flag) {
if (!(flag & DEBUG_TRACE || flag & DEBUG_DEBUG || flag & DEBUG_DUMP)) return; if (flag <= 0) return;
dDebugFlag = flag; dDebugFlag = flag;
vDebugFlag = flag; vDebugFlag = flag;

262
tests/pytest/util/cluster.py
View File

@ -21,45 +21,39 @@ class ClusterDnodes(TDDnodes):
self.testCluster = False self.testCluster = False
self.valgrind = 0 self.valgrind = 0
self.killValgrind = 1 self.killValgrind = 1
self.independent = True
self.dnodeNums = 5
# def getTDDnodes(dnodeNums):
# return
class ConfigureyCluster: class ConfigureyCluster:
"""configure dnodes and return TDDnodes list, it can """ """This will create defined number of dnodes and create a cluset.
at the same time, it will return TDDnodes list: dnodes, """
hostname= socket.gethostname()
def __init__(self): def __init__(self):
self.dnodes = None self.dnodes = []
self.dnodes_nums = 5 self.dnodeNums = 5
self.independent = True self.independent = True
self.start_port = 6030 self.startPort = 6030
self.portStep = 100 self.portStep = 100
hostname1= socket.gethostname() self.mnodeNums = 0
def configure_cluster(self ,dnodes_nums=5,independent=True,start_port=6030,portStep=100,hostname="%s"%hostname1): def configure_cluster(self ,dnodeNums=5,mnodeNums=0,startPort=6030,portStep=100,hostname="%s"%hostname):
self.start_port=int(start_port) self.startPort=int(startPort)
self.portStep=int(portStep) self.portStep=int(portStep)
self.hostname=hostname self.hostname=hostname
self.dnodes_nums = int(dnodes_nums) self.dnodeNums = int(dnodeNums)
self.independent = independent self.mnodeNums = int(mnodeNums)
self.dnodes = [] self.dnodes = []
start_port_sec = 6130 startPort_sec = int(startPort+portStep)
for num in range(1, (self.dnodes_nums+1)): for num in range(1, (self.dnodeNums+1)):
dnode = TDDnode(num) dnode = TDDnode(num)
dnode.addExtraCfg("firstEp", f"{hostname}:{self.start_port}") dnode.addExtraCfg("firstEp", f"{hostname}:{self.startPort}")
dnode.addExtraCfg("fqdn", f"{hostname}") dnode.addExtraCfg("fqdn", f"{hostname}")
dnode.addExtraCfg("serverPort", f"{self.start_port + (num-1)*self.portStep}") dnode.addExtraCfg("serverPort", f"{self.startPort + (num-1)*self.portStep}")
# dnode.addExtraCfg("monitorFqdn", hostname) dnode.addExtraCfg("secondEp", f"{hostname}:{startPort_sec}")
# dnode.addExtraCfg("monitorPort", 7043)
dnode.addExtraCfg("secondEp", f"{hostname}:{start_port_sec}") # configure dnoe of independent mnodes
# configure three dnoe don't support vnodes if num <= self.mnodeNums and self.mnodeNums != 0 :
if self.dnodes_nums > 4 : dnode.addExtraCfg("supportVnodes", 0)
if self.independent and (num < 4):
dnode.addExtraCfg("supportVnodes", 0)
# print(dnode) # print(dnode)
self.dnodes.append(dnode) self.dnodes.append(dnode)
return self.dnodes return self.dnodes
@ -69,24 +63,7 @@ class ConfigureyCluster:
for dnode in self.dnodes[1:]: for dnode in self.dnodes[1:]:
# print(dnode.cfgDict) # print(dnode.cfgDict)
dnode_id = dnode.cfgDict["fqdn"] + ":" +dnode.cfgDict["serverPort"] dnode_id = dnode.cfgDict["fqdn"] + ":" +dnode.cfgDict["serverPort"]
print(dnode_id)
tdSql.execute(" create dnode '%s';"%dnode_id) tdSql.execute(" create dnode '%s';"%dnode_id)
# count=0
# while count < 10:
# time.sleep(1)
# tdSql.query("show dnodes;")
# if tdSql.checkRows(self.dnodes_nums) :
# print("mnode is three nodes")
# if tdSql.queryResult[0][4]=='leader' :
# if tdSql.queryResult[2][4]=='offline':
# if tdSql.queryResult[1][2]=='follower':
# print("stop mnodes on dnode 3 successfully in 10s")
# break
# count+=1
# else:
# print("stop mnodes on dnode 3 failed in 10s")
# return -1
checkstatus=False
@ -97,206 +74,19 @@ class ConfigureyCluster:
tdSql.query("show dnodes") tdSql.query("show dnodes")
# tdLog.debug(tdSql.queryResult) # tdLog.debug(tdSql.queryResult)
status=0 status=0
for i in range(self.dnodes_nums): for i in range(self.dnodeNums):
if tdSql.queryResult[i][4] == "ready": if tdSql.queryResult[i][4] == "ready":
status+=1 status+=1
tdLog.debug(status) # tdLog.debug(status)
if status == self.dnodes_nums: if status == self.dnodeNums:
tdLog.debug(" create cluster with %d dnode and check cluster dnode all ready within 5s! " %self.dnodes_nums) tdLog.debug(" create cluster with %d dnode and check cluster dnode all ready within 5s! " %self.dnodeNums)
break break
count+=1 count+=1
time.sleep(1) time.sleep(1)
else: else:
tdLog.debug("create cluster with %d dnode but check dnode not ready within 5s ! "%self.dnodes_nums) tdLog.debug("create cluster with %d dnode but check dnode not ready within 5s ! "%self.dnodeNums)
return -1 return -1
cluster = ConfigureyCluster() cluster = ConfigureyCluster()
# def start(self ,dnodes_nums):
# self.TDDnodes = ClusterDnodes(dnodes)
# self.TDDnodes.init("")
# self.TDDnodes.setTestCluster(testCluster)
# self.TDDnodes.setValgrind(valgrind)
# self.TDDnodes.stopAll()
# for dnode in self.TDDnodes.dnodes:
# self.TDDnodes.deploy(dnode.index,{})
# for dnode in self.TDDnodes.dnodes:
# self.TDDnodes.starttaosd(dnode.index)
# # create cluster
# for dnode in self.TDDnodes.dnodes[1:]:
# # print(dnode.cfgDict)
# dnode_id = dnode.cfgDict["fqdn"] + ":" +dnode.cfgDict["serverPort"]
# dnode_first_host = dnode.cfgDict["firstEp"].split(":")[0]
# dnode_first_port = dnode.cfgDict["firstEp"].split(":")[-1]
# cmd = f" taos -h {dnode_first_host} -P {dnode_first_port} -s ' create dnode \"{dnode_id} \" ' ;"
# print(cmd)
# os.system(cmd)
# time.sleep(2)
# tdLog.info(" create cluster with %d dnode done! " %dnodes_nums)
# def buildcluster(self,dnodenumber):
# self.depoly_cluster(dnodenumber)
# self.master_dnode = self.TDDnodes.dnodes[0]
# self.host=self.master_dnode.cfgDict["fqdn"]
# conn1 = taos.connect(self.master_dnode.cfgDict["fqdn"] , config=self.master_dnode.cfgDir)
# tdSql.init(conn1.cursor())
# def checkdnodes(self,dnodenumber):
# count=0
# while count < 10:
# time.sleep(1)
# statusReadyBumber=0
# tdSql.query("show dnodes;")
# if tdSql.checkRows(dnodenumber) :
# print("dnode is %d nodes"%dnodenumber)
# for i in range(dnodenumber):
# if tdSql.queryResult[i][4] !='ready' :
# status=tdSql.queryResult[i][4]
# print("dnode:%d status is %s "%(i,status))
# break
# else:
# statusReadyBumber+=1
# print(statusReadyBumber)
# if statusReadyBumber == dnodenumber :
# print("all of %d mnodes is ready in 10s "%dnodenumber)
# return True
# break
# count+=1
# else:
# print("%d mnodes is not ready in 10s "%dnodenumber)
# return False
# def check3mnode(self):
# count=0
# while count < 10:
# time.sleep(1)
# tdSql.query("show mnodes;")
# if tdSql.checkRows(3) :
# print("mnode is three nodes")
# if tdSql.queryResult[0][2]=='leader' :
# if tdSql.queryResult[1][2]=='follower':
# if tdSql.queryResult[2][2]=='follower':
# print("three mnodes is ready in 10s")
# break
# elif tdSql.queryResult[0][2]=='follower' :
# if tdSql.queryResult[1][2]=='leader':
# if tdSql.queryResult[2][2]=='follower':
# print("three mnodes is ready in 10s")
# break
# elif tdSql.queryResult[0][2]=='follower' :
# if tdSql.queryResult[1][2]=='follower':
# if tdSql.queryResult[2][2]=='leader':
# print("three mnodes is ready in 10s")
# break
# count+=1
# else:
# print("three mnodes is not ready in 10s ")
# return -1
# tdSql.query("show mnodes;")
# tdSql.checkRows(3)
# tdSql.checkData(0,1,'%s:6030'%self.host)
# tdSql.checkData(0,3,'ready')
# tdSql.checkData(1,1,'%s:6130'%self.host)
# tdSql.checkData(1,3,'ready')
# tdSql.checkData(2,1,'%s:6230'%self.host)
# tdSql.checkData(2,3,'ready')
# def check3mnode1off(self):
# count=0
# while count < 10:
# time.sleep(1)
# tdSql.query("show mnodes;")
# if tdSql.checkRows(3) :
# print("mnode is three nodes")
# if tdSql.queryResult[0][2]=='offline' :
# if tdSql.queryResult[1][2]=='leader':
# if tdSql.queryResult[2][2]=='follower':
# print("stop mnodes on dnode 2 successfully in 10s")
# break
# elif tdSql.queryResult[1][2]=='follower':
# if tdSql.queryResult[2][2]=='leader':
# print("stop mnodes on dnode 2 successfully in 10s")
# break
# count+=1
# else:
# print("stop mnodes on dnode 2 failed in 10s ")
# return -1
# tdSql.error("drop mnode on dnode 1;")
# tdSql.query("show mnodes;")
# tdSql.checkRows(3)
# tdSql.checkData(0,1,'%s:6030'%self.host)
# tdSql.checkData(0,2,'offline')
# tdSql.checkData(0,3,'ready')
# tdSql.checkData(1,1,'%s:6130'%self.host)
# tdSql.checkData(1,3,'ready')
# tdSql.checkData(2,1,'%s:6230'%self.host)
# tdSql.checkData(2,3,'ready')
# def check3mnode2off(self):
# count=0
# while count < 40:
# time.sleep(1)
# tdSql.query("show mnodes;")
# if tdSql.checkRows(3) :
# print("mnode is three nodes")
# if tdSql.queryResult[0][2]=='leader' :
# if tdSql.queryResult[1][2]=='offline':
# if tdSql.queryResult[2][2]=='follower':
# print("stop mnodes on dnode 2 successfully in 10s")
# break
# count+=1
# else:
# print("stop mnodes on dnode 2 failed in 10s ")
# return -1
# tdSql.error("drop mnode on dnode 2;")
# tdSql.query("show mnodes;")
# tdSql.checkRows(3)
# tdSql.checkData(0,1,'%s:6030'%self.host)
# tdSql.checkData(0,2,'leader')
# tdSql.checkData(0,3,'ready')
# tdSql.checkData(1,1,'%s:6130'%self.host)
# tdSql.checkData(1,2,'offline')
# tdSql.checkData(1,3,'ready')
# tdSql.checkData(2,1,'%s:6230'%self.host)
# tdSql.checkData(2,2,'follower')
# tdSql.checkData(2,3,'ready')
# def check3mnode3off(self):
# count=0
# while count < 10:
# time.sleep(1)
# tdSql.query("show mnodes;")
# if tdSql.checkRows(3) :
# print("mnode is three nodes")
# if tdSql.queryResult[0][2]=='leader' :
# if tdSql.queryResult[2][2]=='offline':
# if tdSql.queryResult[1][2]=='follower':
# print("stop mnodes on dnode 3 successfully in 10s")
# break
# count+=1
# else:
# print("stop mnodes on dnode 3 failed in 10s")
# return -1
# tdSql.error("drop mnode on dnode 3;")
# tdSql.query("show mnodes;")
# tdSql.checkRows(3)
# tdSql.checkData(0,1,'%s:6030'%self.host)
# tdSql.checkData(0,2,'leader')
# tdSql.checkData(0,3,'ready')
# tdSql.checkData(1,1,'%s:6130'%self.host)
# tdSql.checkData(1,2,'follower')
# tdSql.checkData(1,3,'ready')
# tdSql.checkData(2,1,'%s:6230'%self.host)
# tdSql.checkData(2,2,'offline')
# tdSql.checkData(2,3,'ready')

1
tests/script/general/parser/udf.sim
View File

@ -44,6 +44,7 @@ endi
sql create function n02 as '/tmp/normal' outputtype bool; sql create function n02 as '/tmp/normal' outputtype bool;
sql show functions; sql show functions;
if $rows != 1 then if $rows != 1 then
print expect 1, actual $rows
return -1 return -1
endi endi

2
tests/script/tmp/monitor.sim
View File

@ -12,6 +12,8 @@ sql connect
print =============== show dnodes print =============== show dnodes
sleep 2000 sleep 2000
sql create database db vgroups 2; sql create database db vgroups 2;
sql use db;
sql create table db.stb (ts timestamp, c1 int, c2 binary(4)) tags(t1 int, t2 binary(16)) comment "abd";
sleep 2000 sleep 2000
print =============== create drop qnode 1 print =============== create drop qnode 1

1
tests/script/tsim/mnode/basic5.sim
View File

@ -233,7 +233,6 @@ if $data(1)[3] != dropping then
endi endi
print =============== step9: start mnode1 and wait it dropped print =============== step9: start mnode1 and wait it dropped
sleep 3000
system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode1 -s start
$x = 0 $x = 0

1
tests/script/tsim/query/udf.sim
View File

@ -44,6 +44,7 @@ endi
sql select udf2(f) from t; sql select udf2(f) from t;
if $rows != 1 then if $rows != 1 then
print expect 1, actual $rows
return -1 return -1
endi endi
if $data00 != 2.236067977 then if $data00 != 2.236067977 then

10
tests/system-test/6-cluster/5dnode3mnodeStop.py
View File

@ -159,13 +159,10 @@ class TDTestCase:
tdSql.query("show mnodes;") tdSql.query("show mnodes;")
tdSql.checkRows(3) tdSql.checkRows(3)
tdSql.checkData(0,1,'%s:6030'%self.host) tdSql.checkData(0,1,'%s:6030'%self.host)
tdSql.checkData(0,2,'leader')
tdSql.checkData(0,3,'ready') tdSql.checkData(0,3,'ready')
tdSql.checkData(1,1,'%s:6130'%self.host) tdSql.checkData(1,1,'%s:6130'%self.host)
tdSql.checkData(1,2,'offline')
tdSql.checkData(1,3,'ready') tdSql.checkData(1,3,'ready')
tdSql.checkData(2,1,'%s:6230'%self.host) tdSql.checkData(2,1,'%s:6230'%self.host)
tdSql.checkData(2,2,'follower')
tdSql.checkData(2,3,'ready') tdSql.checkData(2,3,'ready')
def check3mnode3off(self): def check3mnode3off(self):
@ -265,13 +262,6 @@ class TDTestCase:
while stopcount <= 2: while stopcount <= 2:
for i in range(dnodenumber): for i in range(dnodenumber):
tdDnodes[i].stoptaosd() tdDnodes[i].stoptaosd()
# if i == 1 :
# self.check3mnode2off()
# elif i == 2 :
# self.check3mnode3off()
# elif i == 0:
# self.check3mnode1off()
tdDnodes[i].starttaosd() tdDnodes[i].starttaosd()
# self.check3mnode() # self.check3mnode()
stopcount+=1 stopcount+=1

2
tests/system-test/fulltest.sh
View File

@ -112,7 +112,7 @@ python3 ./test.py -f 2-query/twa.py
python3 ./test.py -f 6-cluster/5dnode1mnode.py python3 ./test.py -f 6-cluster/5dnode1mnode.py
python3 ./test.py -f 6-cluster/5dnode2mnode.py python3 ./test.py -f 6-cluster/5dnode2mnode.py
python3 ./test.py -f 6-cluster/5dnode3mnodeStop.py -N 5 python3 ./test.py -f 6-cluster/5dnode3mnodeStop.py -N 5 -M 3
# python3 ./test.py -f 6-cluster/5dnode3mnodeDrop.py -N 5 # python3 ./test.py -f 6-cluster/5dnode3mnodeDrop.py -N 5
# BUG python3 ./test.py -f 6-cluster/5dnode3mnodeStopInsert.py # BUG python3 ./test.py -f 6-cluster/5dnode3mnodeStopInsert.py

16
tests/system-test/test.py
View File

@ -60,10 +60,11 @@ if __name__ == "__main__":
stop = 0 stop = 0
restart = False restart = False
dnodeNums = 1 dnodeNums = 1
mnodeNums = 0
updateCfgDict = {} updateCfgDict = {}
execCmd = "" execCmd = ""
opts, args = getopt.gnu_getopt(sys.argv[1:], 'f:p:m:l:scghrd:k:e:N:', [ opts, args = getopt.gnu_getopt(sys.argv[1:], 'f:p:m:l:scghrd:k:e:N:M:', [
'file=', 'path=', 'master', 'logSql', 'stop', 'cluster', 'valgrind', 'help', 'restart', 'updateCfgDict', 'killv', 'execCmd','dnodeNums']) 'file=', 'path=', 'master', 'logSql', 'stop', 'cluster', 'valgrind', 'help', 'restart', 'updateCfgDict', 'killv', 'execCmd','dnodeNums','mnodeNums'])
for key, value in opts: for key, value in opts:
if key in ['-h', '--help']: if key in ['-h', '--help']:
tdLog.printNoPrefix( tdLog.printNoPrefix(
@ -79,7 +80,8 @@ if __name__ == "__main__":
tdLog.printNoPrefix('-d update cfg dict, base64 json str') tdLog.printNoPrefix('-d update cfg dict, base64 json str')
tdLog.printNoPrefix('-k not kill valgrind processer') tdLog.printNoPrefix('-k not kill valgrind processer')
tdLog.printNoPrefix('-e eval str to run') tdLog.printNoPrefix('-e eval str to run')
tdLog.printNoPrefix('-N create dnodes numbers clusters') tdLog.printNoPrefix('-N create dnodes numbers in clusters')
tdLog.printNoPrefix('-M create mnode numbers in clusters')
sys.exit(0) sys.exit(0)
@ -133,6 +135,9 @@ if __name__ == "__main__":
if key in ['-N', '--dnodeNums']: if key in ['-N', '--dnodeNums']:
dnodeNums = value dnodeNums = value
if key in ['-M', '--mnodeNums']:
mnodeNums = value
if not execCmd == "": if not execCmd == "":
tdDnodes.init(deployPath) tdDnodes.init(deployPath)
print(execCmd) print(execCmd)
@ -244,9 +249,8 @@ if __name__ == "__main__":
tdDnodes.start(1) tdDnodes.start(1)
tdCases.logSql(logSql) tdCases.logSql(logSql)
else : else :
print("start cluster and dnodes number") tdLog.debug("create an cluster with %s nodes and make %s dnode as independent mnode"%(dnodeNums,mnodeNums))
dnodeslist = cluster.configure_cluster(dnodeNums=dnodeNums,mnodeNums=mnodeNums)
dnodeslist = cluster.configure_cluster(dnodes_nums=dnodeNums,independent=True)
tdDnodes = ClusterDnodes(dnodeslist) tdDnodes = ClusterDnodes(dnodeslist)
tdDnodes.init(deployPath, masterIp) tdDnodes.init(deployPath, masterIp)
tdDnodes.setTestCluster(testCluster) tdDnodes.setTestCluster(testCluster)