xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch '3.0' into 3.0test/jcy

This commit is contained in:
jiacy-jcy 2022-06-24 10:19:04 +08:00
parent b7bb08d665 81481d870e
commit e176d9a726
85 changed files with 1402 additions and 1326 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
examples/c/tmq.c
View File

@ -27,7 +27,11 @@ static void msg_process(TAOS_RES* msg) {
printf("db: %s\n", tmq_get_db_name(msg));
printf("vg: %d\n", tmq_get_vgroup_id(msg));
if (tmq_get_res_type(msg) == TMQ_RES_TABLE_META) {
printf("meta, skip\n");
void* meta;
int32_t metaLen;
tmq_get_raw_meta(msg, &meta, &metaLen);
printf("meta, len is %d\n", metaLen);
return;
}
while (1) {

2
include/client/taos.h
View File

@ -261,7 +261,7 @@ enum tmq_res_t {
typedef enum tmq_res_t tmq_res_t;
DLL_EXPORT tmq_res_t tmq_get_res_type(TAOS_RES *res);
DLL_EXPORT int32_t tmq_get_raw_meta(TAOS_RES *res, const void **raw_meta, int32_t *raw_meta_len);
DLL_EXPORT int32_t tmq_get_raw_meta(TAOS_RES *res, void **raw_meta, int32_t *raw_meta_len);
DLL_EXPORT const char *tmq_get_topic_name(TAOS_RES *res);
DLL_EXPORT const char *tmq_get_db_name(TAOS_RES *res);
DLL_EXPORT int32_t tmq_get_vgroup_id(TAOS_RES *res);

11
include/common/tcommon.h
View File

@ -69,14 +69,13 @@ typedef struct SColumnDataAgg {
typedef struct SDataBlockInfo {
STimeWindow window;
int32_t rows;
int32_t rows; // todo hide this attribute
int32_t rowSize;
int64_t uid; // the uid of table, from which current data block comes
int64_t blockId; // block id, generated by physical planner
uint64_t uid; // the uid of table, from which current data block comes
uint16_t blockId; // block id, generated by physical planner
uint64_t groupId; // no need to serialize
int16_t numOfCols;
int16_t hasVarCol;
int32_t capacity;
uint32_t capacity;
// TODO: optimize and remove following
int32_t childId; // used for stream, do not serialize
EStreamType type; // used for stream, do not serialize
@ -97,7 +96,7 @@ typedef struct SVarColAttr {
// pBlockAgg->numOfNull == info.rows, all data are null
// pBlockAgg->numOfNull == 0, no data are null.
typedef struct SColumnInfoData {
SColumnInfo info; // TODO filter info needs to be removed
SColumnInfo info; // column info
bool hasNull; // if current column data has null value.
char* pData; // the corresponding block data in memory
union {

19
include/common/tdatablock.h
View File

@ -71,8 +71,8 @@ SEpSet getEpSet_s(SCorEpSet* pEpSet);
#define colDataGetData(p1_, r_) \
((IS_VAR_DATA_TYPE((p1_)->info.type)) ? colDataGetVarData(p1_, r_) : colDataGetNumData(p1_, r_))
#define IS_JSON_NULL(type, data) ((type) == TSDB_DATA_TYPE_JSON && \
(*(data) == TSDB_DATA_TYPE_NULL || tTagIsJsonNull(data)))
#define IS_JSON_NULL(type, data) \
((type) == TSDB_DATA_TYPE_JSON && (*(data) == TSDB_DATA_TYPE_NULL || tTagIsJsonNull(data)))
static FORCE_INLINE bool colDataIsNull_s(const SColumnInfoData* pColumnInfoData, uint32_t row) {
if (!pColumnInfoData->hasNull) {
@ -184,9 +184,10 @@ static FORCE_INLINE void colDataAppendDouble(SColumnInfoData* pColumnInfoData, u
int32_t getJsonValueLen(const char* data);
int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, const char* pData, bool isNull);
int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, int32_t* capacity,
int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, uint32_t* capacity,
const SColumnInfoData* pSource, uint32_t numOfRow2);
int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* pSource, int32_t numOfRows);
int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* pSource, int32_t numOfRows,
const SDataBlockInfo* pBlockInfo);
int32_t blockDataUpdateTsWindow(SSDataBlock* pDataBlock, int32_t tsColumnIndex);
int32_t colDataGetLength(const SColumnInfoData* pColumnInfoData, int32_t numOfRows);
@ -212,7 +213,7 @@ size_t blockDataGetSerialMetaSize(uint32_t numOfCols);
int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo);
int32_t blockDataSort_rv(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullFirst);
int32_t colInfoDataEnsureCapacity(SColumnInfoData* pColumn, size_t existRows, uint32_t numOfRows);
int32_t colInfoDataEnsureCapacity(SColumnInfoData* pColumn, uint32_t numOfRows);
int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows);
void colInfoDataCleanup(SColumnInfoData* pColumn, uint32_t numOfRows);
@ -222,7 +223,13 @@ size_t blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize);
int32_t blockDataTrimFirstNRows(SSDataBlock* pBlock, size_t n);
int32_t assignOneDataBlock(SSDataBlock* dst, const SSDataBlock* src);
SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData);
SSDataBlock* createDataBlock();
int32_t blockDataAppendColInfo(SSDataBlock* pBlock, SColumnInfoData* pColInfoData);
SColumnInfoData createColumnInfoData(int16_t type, int32_t bytes, int16_t colId);
SColumnInfoData* bdGetColumnInfoData(SSDataBlock* pBlock, int32_t index);
void blockCompressEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_t numOfCols,
int8_t needCompress);
@ -236,7 +243,7 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks
char* buildCtbNameByGroupId(const char* stbName, uint64_t groupId);
static FORCE_INLINE int32_t blockGetEncodeSize(const SSDataBlock* pBlock) {
return blockDataGetSerialMetaSize(pBlock->info.numOfCols) + blockDataGetSize(pBlock);
return blockDataGetSerialMetaSize(taosArrayGetSize(pBlock->pDataBlock)) + blockDataGetSize(pBlock);
}
static FORCE_INLINE int32_t blockCompressColData(SColumnInfoData* pColRes, int32_t numOfRows, char* data,

15
include/common/tmsg.h
View File

@ -629,11 +629,6 @@ typedef struct {
uint8_t scale;
} SColumnInfo;
typedef struct {
int64_t uid;
TSKEY key; // last accessed ts, for subscription
} STableIdInfo;
typedef struct STimeWindow {
TSKEY skey;
TSKEY ekey;
@ -1346,11 +1341,19 @@ typedef struct {
int32_t dnodeId;
char config[TSDB_DNODE_CONFIG_LEN];
char value[TSDB_DNODE_VALUE_LEN];
} SMCfgDnodeReq, SDCfgDnodeReq;
} SMCfgDnodeReq;
int32_t tSerializeSMCfgDnodeReq(void* buf, int32_t bufLen, SMCfgDnodeReq* pReq);
int32_t tDeserializeSMCfgDnodeReq(void* buf, int32_t bufLen, SMCfgDnodeReq* pReq);
typedef struct {
char config[TSDB_DNODE_CONFIG_LEN];
char value[TSDB_DNODE_VALUE_LEN];
} SDCfgDnodeReq;
int32_t tSerializeSDCfgDnodeReq(void* buf, int32_t bufLen, SDCfgDnodeReq* pReq);
int32_t tDeserializeSDCfgDnodeReq(void* buf, int32_t bufLen, SDCfgDnodeReq* pReq);
typedef struct {
int32_t dnodeId;
} SMCreateMnodeReq, SMDropMnodeReq, SDDropMnodeReq, SMCreateQnodeReq, SMDropQnodeReq, SDCreateQnodeReq, SDDropQnodeReq,

4
include/dnode/mnode/mnode.h
View File

@ -73,10 +73,12 @@ void mndStop(SMnode *pMnode);
* @param pMnode The mnode object.
* @param pCluster
* @param pVgroup
* @param pStbInfo
* @param pGrant
* @return int32_t 0 for success, -1 for failure.
*/
int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pCluster, SMonVgroupInfo *pVgroup, SMonGrantInfo *pGrant);
int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo,
SMonStbInfo *pStbInfo, SMonGrantInfo *pGrantInfo);
/**
* @brief Get mnode loads for status msg.

17
include/libs/function/function.h
View File

@ -172,23 +172,6 @@ typedef struct tExprNode {
void tExprTreeDestroy(tExprNode *pNode, void (*fp)(void *));
typedef struct SAggFunctionInfo {
char name[FUNCTIONS_NAME_MAX_LENGTH];
int8_t type; // Scalar function or aggregation function
uint32_t functionId; // Function Id
int8_t sFunctionId; // Transfer function for super table query
uint16_t status;
bool (*init)(SqlFunctionCtx *pCtx, struct SResultRowEntryInfo* pResultCellInfo); // setup the execute environment
void (*addInput)(SqlFunctionCtx *pCtx);
// finalizer must be called after all exec has been executed to generated final result.
void (*finalize)(SqlFunctionCtx *pCtx);
void (*combine)(SqlFunctionCtx *pCtx);
int32_t (*dataReqFunc)(SqlFunctionCtx *pCtx, STimeWindow* w, int32_t colId);
} SAggFunctionInfo;
struct SScalarParam {
SColumnInfoData *columnData;
SHashObj *pHashFilter;

13
include/libs/monitor/monitor.h
View File

@ -109,6 +109,9 @@ typedef struct {
char version[MON_VER_LEN];
float master_uptime; // day
int32_t monitor_interval; // sec
int32_t dbs_total;
int32_t stbs_total;
int64_t tbs_total;
int32_t vgroups_total;
int32_t vgroups_alive;
int32_t vnodes_total;
@ -135,6 +138,15 @@ typedef struct {
SArray *vgroups; // array of SMonVgroupDesc
} SMonVgroupInfo;
typedef struct {
char stb_name[TSDB_TABLE_NAME_LEN];
char database_name[TSDB_DB_NAME_LEN];
} SMonStbDesc;
typedef struct {
SArray *stbs; // array of SMonStbDesc
} SMonStbInfo;
typedef struct {
int32_t expire_time;
int64_t timeseries_used;
@ -144,6 +156,7 @@ typedef struct {
typedef struct {
SMonClusterInfo cluster;
SMonVgroupInfo vgroup;
SMonStbInfo stb;
SMonGrantInfo grant;
SMonSysInfo sys;
SMonLogs log;

2
include/libs/stream/tstream.h
View File

@ -344,10 +344,12 @@ static FORCE_INLINE int32_t streamTaskOutput(SStreamTask* pTask, SStreamDataBloc
if (pTask->sinkType == TASK_SINK__TABLE) {
ASSERT(pTask->dispatchType == TASK_DISPATCH__NONE);
pTask->tbSink.tbSinkFunc(pTask, pTask->tbSink.vnode, 0, pBlock->blocks);
taosArrayDestroyEx(pBlock->blocks, (FDelete)tDeleteSSDataBlock);
taosFreeQitem(pBlock);
} else if (pTask->sinkType == TASK_SINK__SMA) {
ASSERT(pTask->dispatchType == TASK_DISPATCH__NONE);
pTask->smaSink.smaSink(pTask->smaSink.vnode, pTask->smaSink.smaId, pBlock->blocks);
taosArrayDestroyEx(pBlock->blocks, (FDelete)tDeleteSSDataBlock);
taosFreeQitem(pBlock);
} else {
ASSERT(pTask->dispatchType != TASK_DISPATCH__NONE);

6
include/libs/sync/sync.h
View File

@ -157,13 +157,13 @@ typedef struct SSyncLogStore {
SyncIndex (*getCommitIndex)(struct SSyncLogStore* pLogStore);
// refactor, log[0 .. n] ==> log[m .. n]
int32_t (*syncLogSetBeginIndex)(struct SSyncLogStore* pLogStore, SyncIndex beginIndex);
int32_t (*syncLogResetBeginIndex)(struct SSyncLogStore* pLogStore);
// int32_t (*syncLogSetBeginIndex)(struct SSyncLogStore* pLogStore, SyncIndex beginIndex);
SyncIndex (*syncLogBeginIndex)(struct SSyncLogStore* pLogStore);
SyncIndex (*syncLogEndIndex)(struct SSyncLogStore* pLogStore);
bool (*syncLogIsEmpty)(struct SSyncLogStore* pLogStore);
int32_t (*syncLogEntryCount)(struct SSyncLogStore* pLogStore);
// bool (*syncLogInRange)(struct SSyncLogStore* pLogStore, SyncIndex index);
int32_t (*syncLogRestoreFromSnapshot)(struct SSyncLogStore* pLogStore, SyncIndex index);
SyncIndex (*syncLogWriteIndex)(struct SSyncLogStore* pLogStore);
SyncIndex (*syncLogLastIndex)(struct SSyncLogStore* pLogStore);

2
source/client/src/tmq.c
View File

@ -1871,7 +1871,7 @@ const char* tmq_get_table_name(TAOS_RES* res) {
return NULL;
}
int32_t tmq_get_raw_meta(TAOS_RES* res, const void** raw_meta, int32_t* raw_meta_len) {
int32_t tmq_get_raw_meta(TAOS_RES* res, void** raw_meta, int32_t* raw_meta_len) {
if (TD_RES_TMQ_META(res)) {
SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)res;
*raw_meta = pMetaRspObj->metaRsp.metaRsp;

321
source/common/src/tdatablock.c
View File

@ -214,7 +214,7 @@ static void doBitmapMerge(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, c
}
}
int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, int32_t* capacity,
int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, uint32_t* capacity,
const SColumnInfoData* pSource, uint32_t numOfRow2) {
ASSERT(pColumnInfoData != NULL && pSource != NULL && pColumnInfoData->info.type == pSource->info.type);
if (numOfRow2 == 0) {
@ -263,7 +263,7 @@ int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, in
pColumnInfoData->varmeta.length = len + oldLen;
} else {
if (finalNumOfRows > *capacity) {
ASSERT(finalNumOfRows*pColumnInfoData->info.bytes);
ASSERT(finalNumOfRows * pColumnInfoData->info.bytes);
char* tmp = taosMemoryRealloc(pColumnInfoData->pData, finalNumOfRows * pColumnInfoData->info.bytes);
if (tmp == NULL) {
return TSDB_CODE_VND_OUT_OF_MEMORY;
@ -293,22 +293,19 @@ int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, in
return numOfRow1 + numOfRow2;
}
int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* pSource, int32_t numOfRows) {
int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* pSource, int32_t numOfRows,
const SDataBlockInfo* pBlockInfo) {
ASSERT(pColumnInfoData != NULL && pSource != NULL && pColumnInfoData->info.type == pSource->info.type);
if (numOfRows <= 0) {
return numOfRows;
}
if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) {
// Handle the bitmap
char* p = taosMemoryRealloc(pColumnInfoData->varmeta.offset, sizeof(int32_t) * numOfRows);
if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
if (pBlockInfo != NULL) {
ASSERT(pBlockInfo->capacity >= numOfRows);
}
pColumnInfoData->varmeta.offset = (int32_t*)p;
if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) {
memcpy(pColumnInfoData->varmeta.offset, pSource->varmeta.offset, sizeof(int32_t) * numOfRows);
if (pColumnInfoData->varmeta.allocLen < pSource->varmeta.length) {
char* tmp = taosMemoryRealloc(pColumnInfoData->pData, pSource->varmeta.length);
if (tmp == NULL) {
@ -319,38 +316,19 @@ int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* p
pColumnInfoData->varmeta.allocLen = pSource->varmeta.length;
}
memcpy(pColumnInfoData->pData, pSource->pData, pSource->varmeta.length);
pColumnInfoData->varmeta.length = pSource->varmeta.length;
memcpy(pColumnInfoData->pData, pSource->pData, pSource->varmeta.length);
} else {
char* tmp = taosMemoryRealloc(pColumnInfoData->nullbitmap, BitmapLen(numOfRows));
if (tmp == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pColumnInfoData->nullbitmap = tmp;
memcpy(pColumnInfoData->nullbitmap, pSource->nullbitmap, BitmapLen(numOfRows));
if (pSource->pData) {
int32_t newSize = numOfRows * pColumnInfoData->info.bytes;
tmp = taosMemoryRealloc(pColumnInfoData->pData, newSize);
if (tmp == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pColumnInfoData->pData = tmp;
memcpy(pColumnInfoData->pData, pSource->pData, pSource->info.bytes * numOfRows);
}
}
pColumnInfoData->hasNull = pSource->hasNull;
pColumnInfoData->info = pSource->info;
return 0;
}
size_t blockDataGetNumOfCols(const SSDataBlock* pBlock) {
ASSERT(pBlock && pBlock->info.numOfCols == taosArrayGetSize(pBlock->pDataBlock));
return pBlock->info.numOfCols;
}
size_t blockDataGetNumOfCols(const SSDataBlock* pBlock) { return taosArrayGetSize(pBlock->pDataBlock); }
size_t blockDataGetNumOfRows(const SSDataBlock* pBlock) { return pBlock->info.rows; }
@ -359,7 +337,8 @@ int32_t blockDataUpdateTsWindow(SSDataBlock* pDataBlock, int32_t tsColumnIndex)
return 0;
}
if (pDataBlock->info.numOfCols <= 0) {
size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock);
if (numOfCols <= 0) {
return -1;
}
@ -383,7 +362,8 @@ int32_t blockDataMerge(SSDataBlock* pDest, const SSDataBlock* pSrc) {
assert(pSrc != NULL && pDest != NULL);
int32_t capacity = pDest->info.capacity;
for (int32_t i = 0; i < pDest->info.numOfCols; ++i) {
size_t numOfCols = taosArrayGetSize(pDest->pDataBlock);
for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pCol2 = taosArrayGet(pDest->pDataBlock, i);
SColumnInfoData* pCol1 = taosArrayGet(pSrc->pDataBlock, i);
@ -400,8 +380,7 @@ size_t blockDataGetSize(const SSDataBlock* pBlock) {
assert(pBlock != NULL);
size_t total = 0;
int32_t numOfCols = pBlock->info.numOfCols;
size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i);
total += colDataGetFullLength(pColInfoData, pBlock->info.rows);
@ -416,7 +395,7 @@ int32_t blockDataSplitRows(SSDataBlock* pBlock, bool hasVarCol, int32_t startInd
int32_t pageSize) {
ASSERT(pBlock != NULL && stopIndex != NULL);
int32_t numOfCols = pBlock->info.numOfCols;
size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
int32_t numOfRows = pBlock->info.rows;
int32_t bitmapChar = 1;
@ -477,33 +456,24 @@ SSDataBlock* blockDataExtractBlock(SSDataBlock* pBlock, int32_t startIndex, int3
return NULL;
}
SSDataBlock* pDst = taosMemoryCalloc(1, sizeof(SSDataBlock));
SSDataBlock* pDst = createDataBlock();
if (pDst == NULL) {
return NULL;
}
pDst->info = pBlock->info;
pDst->info.rows = 0;
pDst->pDataBlock = taosArrayInit(pBlock->info.numOfCols, sizeof(SColumnInfoData));
for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) {
size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData colInfo = {0};
SColumnInfoData* pSrcCol = taosArrayGet(pBlock->pDataBlock, i);
colInfo.info = pSrcCol->info;
if (IS_VAR_DATA_TYPE(pSrcCol->info.type)) {
SVarColAttr* pAttr = &colInfo.varmeta;
pAttr->offset = taosMemoryCalloc(rowCount, sizeof(int32_t));
} else {
colInfo.nullbitmap = taosMemoryCalloc(1, BitmapLen(rowCount));
colInfo.pData = taosMemoryCalloc(rowCount, colInfo.info.bytes);
blockDataAppendColInfo(pDst, &colInfo);
}
taosArrayPush(pDst->pDataBlock, &colInfo);
}
blockDataEnsureCapacity(pDst, rowCount);
for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) {
for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, i);
SColumnInfoData* pDstCol = taosArrayGet(pDst->pDataBlock, i);
@ -541,7 +511,7 @@ int32_t blockDataToBuf(char* buf, const SSDataBlock* pBlock) {
// write the number of rows
*(uint32_t*)buf = pBlock->info.rows;
int32_t numOfCols = pBlock->info.numOfCols;
size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
int32_t numOfRows = pBlock->info.rows;
char* pStart = buf + sizeof(uint32_t);
@ -571,7 +541,7 @@ int32_t blockDataToBuf(char* buf, const SSDataBlock* pBlock) {
int32_t blockDataFromBuf(SSDataBlock* pBlock, const char* buf) {
pBlock->info.rows = *(int32_t*)buf;
int32_t numOfCols = pBlock->info.numOfCols;
size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
const char* pStart = buf + sizeof(uint32_t);
for (int32_t i = 0; i < numOfCols; ++i) {
@ -620,7 +590,8 @@ int32_t blockDataFromBuf1(SSDataBlock* pBlock, const char* buf, size_t capacity)
pBlock->info.rows = *(int32_t*)buf;
pBlock->info.groupId = *(uint64_t*)(buf + sizeof(int32_t));
int32_t numOfCols = pBlock->info.numOfCols;
size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
const char* pStart = buf + sizeof(uint32_t) + sizeof(uint64_t);
for (int32_t i = 0; i < numOfCols; ++i) {
@ -666,7 +637,7 @@ size_t blockDataGetRowSize(SSDataBlock* pBlock) {
if (pBlock->info.rowSize == 0) {
size_t rowSize = 0;
size_t numOfCols = pBlock->info.numOfCols;
size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i);
rowSize += pColInfo->info.bytes;
@ -693,7 +664,7 @@ double blockDataGetSerialRowSize(const SSDataBlock* pBlock) {
ASSERT(pBlock != NULL);
double rowSize = 0;
size_t numOfCols = pBlock->info.numOfCols;
size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i);
rowSize += pColInfo->info.bytes;
@ -762,7 +733,7 @@ int32_t dataBlockCompar(const void* p1, const void* p2, const void* param) {
static int32_t doAssignOneTuple(SColumnInfoData* pDstCols, int32_t numOfRows, const SSDataBlock* pSrcBlock,
int32_t tupleIndex) {
int32_t code = 0;
int32_t numOfCols = pSrcBlock->info.numOfCols;
size_t numOfCols = taosArrayGetSize(pSrcBlock->pDataBlock);
for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pDst = &pDstCols[i];
@ -785,7 +756,7 @@ static int32_t doAssignOneTuple(SColumnInfoData* pDstCols, int32_t numOfRows, co
return TSDB_CODE_SUCCESS;
}
static int32_t blockDataAssign(SColumnInfoData* pCols, const SSDataBlock* pDataBlock, int32_t* index) {
static int32_t blockDataAssign(SColumnInfoData* pCols, const SSDataBlock* pDataBlock, const int32_t* index) {
#if 0
for (int32_t i = 0; i < pDataBlock->info.rows; ++i) {
int32_t code = doAssignOneTuple(pCols, i, pDataBlock, index[i]);
@ -794,7 +765,8 @@ static int32_t blockDataAssign(SColumnInfoData* pCols, const SSDataBlock* pDataB
}
}
#else
for (int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) {
size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock);
for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pDst = &pCols[i];
SColumnInfoData* pSrc = taosArrayGet(pDataBlock->pDataBlock, i);
@ -821,7 +793,7 @@ static int32_t blockDataAssign(SColumnInfoData* pCols, const SSDataBlock* pDataB
static SColumnInfoData* createHelpColInfoData(const SSDataBlock* pDataBlock) {
int32_t rows = pDataBlock->info.rows;
int32_t numOfCols = pDataBlock->info.numOfCols;
size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock);
SColumnInfoData* pCols = taosMemoryCalloc(numOfCols, sizeof(SColumnInfoData));
if (pCols == NULL) {
@ -848,7 +820,7 @@ static SColumnInfoData* createHelpColInfoData(const SSDataBlock* pDataBlock) {
}
static void copyBackToBlock(SSDataBlock* pDataBlock, SColumnInfoData* pCols) {
int32_t numOfCols = pDataBlock->info.numOfCols;
size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock);
for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, i);
@ -909,8 +881,10 @@ int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo) {
}
}
size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock);
if (taosArrayGetSize(pOrderInfo) == 1 && (!sortColumnHasNull)) {
if (pDataBlock->info.numOfCols == 1) {
if (numOfCols == 1) {
if (!varTypeSort) {
SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, 0);
SBlockOrderInfo* pOrder = taosArrayGet(pOrderInfo, 0);
@ -926,7 +900,7 @@ int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo) {
return TSDB_CODE_SUCCESS;
} else { // var data type
}
} else if (pDataBlock->info.numOfCols == 2) {
} else if (numOfCols == 2) {
}
}
@ -1120,34 +1094,39 @@ int32_t blockDataSort_rv(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullF
void blockDataCleanup(SSDataBlock* pDataBlock) {
pDataBlock->info.rows = 0;
for (int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) {
size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock);
for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i);
colInfoDataCleanup(p, pDataBlock->info.capacity);
}
}
int32_t colInfoDataEnsureCapacity(SColumnInfoData* pColumn, size_t existRows, uint32_t numOfRows) {
ASSERT(numOfRows);
if (0 == numOfRows || numOfRows <= existRows) {
static int32_t doEnsureCapacity(SColumnInfoData* pColumn, const SDataBlockInfo* pBlockInfo, uint32_t numOfRows) {
ASSERT(numOfRows > 0 && pBlockInfo->capacity >= pBlockInfo->rows);
if (numOfRows < pBlockInfo->capacity) {
return TSDB_CODE_SUCCESS;
}
// todo temp disable it
// ASSERT(pColumn->info.bytes != 0);
int32_t existedRows = pBlockInfo->rows;
if (IS_VAR_DATA_TYPE(pColumn->info.type)) {
char* tmp = taosMemoryRealloc(pColumn->varmeta.offset, sizeof(int32_t) * numOfRows);
if (tmp == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pColumn->varmeta.offset = (int32_t*)tmp;
memset(&pColumn->varmeta.offset[existRows], 0, sizeof(int32_t) * (numOfRows - existRows));
memset(&pColumn->varmeta.offset[existedRows], 0, sizeof(int32_t) * (numOfRows - existedRows));
} else {
char* tmp = taosMemoryRealloc(pColumn->nullbitmap, BitmapLen(numOfRows));
if (tmp == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
int32_t oldLen = BitmapLen(existRows);
int32_t oldLen = BitmapLen(existedRows);
pColumn->nullbitmap = tmp;
memset(&pColumn->nullbitmap[oldLen], 0, BitmapLen(numOfRows) - oldLen);
@ -1160,8 +1139,8 @@ int32_t colInfoDataEnsureCapacity(SColumnInfoData* pColumn, size_t existRows, ui
if (tmp == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
memset(tmp + pColumn->info.bytes * existRows, 0, pColumn->info.bytes * (numOfRows - existRows));
memset(tmp + pColumn->info.bytes * existedRows, 0, pColumn->info.bytes * (numOfRows - existedRows));
pColumn->pData = tmp;
}
@ -1178,6 +1157,11 @@ void colInfoDataCleanup(SColumnInfoData* pColumn, uint32_t numOfRows) {
}
}
int32_t colInfoDataEnsureCapacity(SColumnInfoData* pColumn, uint32_t numOfRows) {
SDataBlockInfo info = {0};
return doEnsureCapacity(pColumn, &info, numOfRows);
}
int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows) {
int32_t code = 0;
ASSERT(numOfRows > 0);
@ -1186,11 +1170,14 @@ int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows) {
return TSDB_CODE_SUCCESS;
}
if (pDataBlock->info.capacity < numOfRows) {
pDataBlock->info.capacity = numOfRows;
}
for (int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) {
size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock);
for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i);
code = colInfoDataEnsureCapacity(p, pDataBlock->info.rows, numOfRows);
code = doEnsureCapacity(p, &pDataBlock->info, numOfRows);
if (code) {
return code;
}
@ -1208,46 +1195,74 @@ void* blockDataDestroy(SSDataBlock* pBlock) {
taosMemoryFreeClear(pBlock);
return NULL;
}
int32_t assignOneDataBlock(SSDataBlock* dst, const SSDataBlock* src) {
ASSERT(src != NULL);
dst->info = src->info;
dst->info.rows = 0;
dst->info.capacity = 0;
size_t numOfCols = taosArrayGetSize(src->pDataBlock);
for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* p = taosArrayGet(src->pDataBlock, i);
SColumnInfoData colInfo = {.hasNull = true, .info = p->info};
blockDataAppendColInfo(dst, &colInfo);
}
int32_t code = blockDataEnsureCapacity(dst, src->info.rows);
if (code != TSDB_CODE_SUCCESS) {
terrno = code;
return -1;
}
for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pDst = taosArrayGet(dst->pDataBlock, i);
SColumnInfoData* pSrc = taosArrayGet(src->pDataBlock, i);
if (pSrc->pData == NULL) {
continue;
}
colDataAssign(pDst, pSrc, src->info.rows, &src->info);
}
dst->info.rows = src->info.rows;
dst->info.capacity = src->info.rows;
return 0;
}
SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData) {
if (pDataBlock == NULL) {
return NULL;
}
int32_t numOfCols = pDataBlock->info.numOfCols;
SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock));
pBlock->pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData));
pBlock->info.numOfCols = numOfCols;
pBlock->info.hasVarCol = pDataBlock->info.hasVarCol;
pBlock->info.rowSize = pDataBlock->info.rowSize;
pBlock->info.groupId = pDataBlock->info.groupId;
pBlock->info.childId = pDataBlock->info.childId;
pBlock->info.type = pDataBlock->info.type;
SSDataBlock* pBlock = createDataBlock();
pBlock->info = pDataBlock->info;
pBlock->info.rows = 0;
pBlock->info.capacity = 0;
size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock);
for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData colInfo = {0};
SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i);
colInfo.info = p->info;
colInfo.hasNull = true;
taosArrayPush(pBlock->pDataBlock, &colInfo);
SColumnInfoData colInfo = {.hasNull = true, .info = p->info};
blockDataAppendColInfo(pBlock, &colInfo);
}
if (copyData) {
for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pDst = taosArrayGet(pBlock->pDataBlock, i);
SColumnInfoData* pSrc = taosArrayGet(pDataBlock->pDataBlock, i);
int32_t code = colInfoDataEnsureCapacity(pDst, 0, pDataBlock->info.rows);
int32_t code = blockDataEnsureCapacity(pBlock, pDataBlock->info.rows);
if (code != TSDB_CODE_SUCCESS) {
terrno = code;
blockDataDestroy(pBlock);
return NULL;
}
for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pDst = taosArrayGet(pBlock->pDataBlock, i);
SColumnInfoData* pSrc = taosArrayGet(pDataBlock->pDataBlock, i);
if (pSrc->pData == NULL) {
continue;
}
colDataAssign(pDst, pSrc, pDataBlock->info.rows);
colDataAssign(pDst, pSrc, pDataBlock->info.rows, &pDataBlock->info);
}
pBlock->info.rows = pDataBlock->info.rows;
@ -1257,16 +1272,75 @@ SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData) {
return pBlock;
}
SSDataBlock* createDataBlock() {
SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock));
if (pBlock == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
}
pBlock->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData));
if (pBlock->pDataBlock == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
taosMemoryFree(pBlock);
}
return pBlock;
}
int32_t blockDataAppendColInfo(SSDataBlock* pBlock, SColumnInfoData* pColInfoData) {
ASSERT(pBlock != NULL && pColInfoData != NULL);
if (pBlock->pDataBlock == NULL) {
pBlock->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData));
if (pBlock->pDataBlock == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return terrno;
}
}
void* p = taosArrayPush(pBlock->pDataBlock, pColInfoData);
if (p == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return terrno;
}
// todo disable it temporarily
// ASSERT(pColInfoData->info.type != 0);
if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) {
pBlock->info.hasVarCol = true;
}
pBlock->info.rowSize += pColInfoData->info.bytes;
return TSDB_CODE_SUCCESS;
}
SColumnInfoData createColumnInfoData(int16_t type, int32_t bytes, int16_t colId) {
SColumnInfoData col = {.hasNull = true};
col.info.colId = colId;
col.info.type = type;
col.info.bytes = bytes;
return col;
}
SColumnInfoData* bdGetColumnInfoData(SSDataBlock* pBlock, int32_t index) {
ASSERT(pBlock != NULL);
if (index >= taosArrayGetSize(pBlock->pDataBlock)) {
return NULL;
}
return taosArrayGet(pBlock->pDataBlock, index);
}
size_t blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize) {
int32_t payloadSize = pageSize - blockDataGetSerialMetaSize(pBlock->info.numOfCols);
size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
int32_t payloadSize = pageSize - blockDataGetSerialMetaSize(numOfCols);
int32_t rowSize = pBlock->info.rowSize;
int32_t nRows = payloadSize / rowSize;
// the true value must be less than the value of nRows
int32_t additional = 0;
for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) {
for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, i);
if (IS_VAR_DATA_TYPE(pCol->info.type)) {
additional += nRows * sizeof(int32_t);
@ -1350,7 +1424,8 @@ int32_t blockDataTrimFirstNRows(SSDataBlock* pBlock, size_t n) {
if (pBlock->info.rows <= n) {
blockDataCleanup(pBlock);
} else {
for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) {
size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i);
colDataTrimFirstNRows(pColInfoData, n, pBlock->info.rows);
}
@ -1362,7 +1437,7 @@ int32_t blockDataTrimFirstNRows(SSDataBlock* pBlock, size_t n) {
int32_t tEncodeDataBlock(void** buf, const SSDataBlock* pBlock) {
int64_t tbUid = pBlock->info.uid;
int16_t numOfCols = pBlock->info.numOfCols;
int16_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
int16_t hasVarCol = pBlock->info.hasVarCol;
int32_t rows = pBlock->info.rows;
int32_t sz = taosArrayGetSize(pBlock->pDataBlock);
@ -1397,8 +1472,10 @@ int32_t tEncodeDataBlock(void** buf, const SSDataBlock* pBlock) {
void* tDecodeDataBlock(const void* buf, SSDataBlock* pBlock) {
int32_t sz;
buf = taosDecodeFixedI64(buf, &pBlock->info.uid);
buf = taosDecodeFixedI16(buf, &pBlock->info.numOfCols);
int16_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
buf = taosDecodeFixedU64(buf, &pBlock->info.uid);
buf = taosDecodeFixedI16(buf, &numOfCols);
buf = taosDecodeFixedI16(buf, &pBlock->info.hasVarCol);
buf = taosDecodeFixedI32(buf, &pBlock->info.rows);
buf = taosDecodeFixedI32(buf, &sz);
@ -1507,12 +1584,13 @@ void blockDebugShowData(const SArray* dataBlocks, const char* flag) {
int32_t sz = taosArrayGetSize(dataBlocks);
for (int32_t i = 0; i < sz; i++) {
SSDataBlock* pDataBlock = taosArrayGet(dataBlocks, i);
int32_t colNum = pDataBlock->info.numOfCols;
size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock);
int32_t rows = pDataBlock->info.rows;
printf("%s |block type %d |child id %d|\n", flag, (int32_t)pDataBlock->info.type, pDataBlock->info.childId);
for (int32_t j = 0; j < rows; j++) {
printf("%s |", flag);
for (int32_t k = 0; k < colNum; k++) {
for (int32_t k = 0; k < numOfCols; k++) {
SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, k);
void* var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes);
if (colDataIsNull(pColInfoData, rows, j, NULL)) {
@ -1565,7 +1643,9 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks
int32_t bufSize = sizeof(SSubmitReq);
for (int32_t i = 0; i < sz; ++i) {
SDataBlockInfo* pBlkInfo = &((SSDataBlock*)taosArrayGet(pDataBlocks, i))->info;
bufSize += pBlkInfo->rows * (TD_ROW_HEAD_LEN + pBlkInfo->rowSize + BitmapLen(pBlkInfo->numOfCols));
int32_t numOfCols = taosArrayGetSize(pDataBlocks);
bufSize += pBlkInfo->rows * (TD_ROW_HEAD_LEN + pBlkInfo->rowSize + BitmapLen(numOfCols));
bufSize += sizeof(SSubmitBlk);
}
@ -1583,10 +1663,10 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks
for (int32_t i = 0; i < sz; ++i) {
SSDataBlock* pDataBlock = taosArrayGet(pDataBlocks, i);
int32_t colNum = pDataBlock->info.numOfCols;
int32_t colNum = taosArrayGetSize(pDataBlock->pDataBlock);
int32_t rows = pDataBlock->info.rows;
int32_t rowSize = pDataBlock->info.rowSize;
int64_t groupId = pDataBlock->info.groupId;
// int32_t rowSize = pDataBlock->info.rowSize;
// int64_t groupId = pDataBlock->info.groupId;
if (colNum <= 1) {
// invalid if only with TS col
@ -1804,8 +1884,6 @@ void blockCompressEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen
}
const char* blockCompressDecode(SSDataBlock* pBlock, int32_t numOfCols, int32_t numOfRows, const char* pData) {
blockDataEnsureCapacity(pBlock, numOfRows);
const char* pStart = pData;
int32_t dataLen = *(int32_t*)pStart;
@ -1819,9 +1897,6 @@ const char* blockCompressDecode(SSDataBlock* pBlock, int32_t numOfCols, int32_t
taosArraySetSize(pBlock->pDataBlock, numOfCols);
}
pBlock->info.numOfCols = taosArrayGetSize(pBlock->pDataBlock);
ASSERT(pBlock->pDataBlock->size >= numOfCols);
for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i);
pColInfoData->info.type = *(int16_t*)pStart;
@ -1846,28 +1921,26 @@ const char* blockCompressDecode(SSDataBlock* pBlock, int32_t numOfCols, int32_t
SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i);
if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) {
pColInfoData->varmeta.length = colLen[i];
pColInfoData->varmeta.allocLen = colLen[i];
memcpy(pColInfoData->varmeta.offset, pStart, sizeof(int32_t) * numOfRows);
pStart += sizeof(int32_t) * numOfRows;
if (colLen[i] > 0) {
taosMemoryFreeClear(pColInfoData->pData);
pColInfoData->pData = taosMemoryMalloc(colLen[i]);
if (colLen[i] > 0 && pColInfoData->varmeta.allocLen < colLen[i]) {
char* tmp = taosMemoryRealloc(pColInfoData->pData, colLen[i]);
if (tmp == NULL) {
return NULL;
}
pColInfoData->pData = tmp;
pColInfoData->varmeta.allocLen = colLen[i];
}
pColInfoData->varmeta.length = colLen[i];
} else {
if (pColInfoData->nullbitmap == NULL) {
pColInfoData->nullbitmap = taosMemoryCalloc(1, BitmapLen(numOfRows));
}
memcpy(pColInfoData->nullbitmap, pStart, BitmapLen(numOfRows));
pStart += BitmapLen(numOfRows);
}
if (colLen[i] > 0) {
if (pColInfoData->pData == NULL) {
pColInfoData->pData = taosMemoryCalloc(1, colLen[i]);
}
memcpy(pColInfoData->pData, pStart, colLen[i]);
}

6
source/common/src/tglobal.c
View File

@ -760,9 +760,9 @@ void taosCleanupCfg() {
}
void taosCfgDynamicOptions(const char *option, const char *value) {
if (strcasecmp(option, "debugFlag") == 0) {
int32_t debugFlag = atoi(value);
taosSetAllDebugFlag(debugFlag);
if (strncasecmp(option, "debugFlag", 9) == 0) {
int32_t flag = atoi(value);
taosSetAllDebugFlag(flag);
}
if (strcasecmp(option, "resetlog") == 0) {

27
source/common/src/tmsg.c
View File

@ -1500,6 +1500,33 @@ int32_t tDeserializeSMCfgDnodeReq(void *buf, int32_t bufLen, SMCfgDnodeReq *pReq
return 0;
}
int32_t tSerializeSDCfgDnodeReq(void *buf, int32_t bufLen, SDCfgDnodeReq *pReq) {
SEncoder encoder = {0};
tEncoderInit(&encoder, buf, bufLen);
if (tStartEncode(&encoder) < 0) return -1;
if (tEncodeCStr(&encoder, pReq->config) < 0) return -1;
if (tEncodeCStr(&encoder, pReq->value) < 0) return -1;
tEndEncode(&encoder);
int32_t tlen = encoder.pos;
tEncoderClear(&encoder);
return tlen;
}
int32_t tDeserializeSDCfgDnodeReq(void *buf, int32_t bufLen, SDCfgDnodeReq *pReq) {
SDecoder decoder = {0};
tDecoderInit(&decoder, buf, bufLen);
if (tStartDecode(&decoder) < 0) return -1;
if (tDecodeCStrTo(&decoder, pReq->config) < 0) return -1;
if (tDecodeCStrTo(&decoder, pReq->value) < 0) return -1;
tEndDecode(&decoder);
tDecoderClear(&decoder);
return 0;
}
int32_t tSerializeSCreateDnodeReq(void *buf, int32_t bufLen, SCreateDnodeReq *pReq) {
SEncoder encoder = {0};
tEncoderInit(&encoder, buf, bufLen);

46
source/common/test/commonTests.cpp
View File

@ -88,26 +88,15 @@ TEST(testCase, toInteger_test) {
}
TEST(testCase, Datablock_test) {
SSDataBlock* b = static_cast<SSDataBlock*>(taosMemoryCalloc(1, sizeof(SSDataBlock)));
b->info.numOfCols = 2;
b->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData));
SSDataBlock* b = createDataBlock();
SColumnInfoData infoData = {0};
infoData.info.bytes = 4;
infoData.info.type = TSDB_DATA_TYPE_INT;
infoData.info.colId = 1;
infoData.pData = (char*)taosMemoryCalloc(40, infoData.info.bytes);
infoData.nullbitmap = (char*)taosMemoryCalloc(1, sizeof(char) * (40 / 8));
SColumnInfoData infoData = createColumnInfoData(TSDB_DATA_TYPE_INT, 4, 1);
taosArrayPush(b->pDataBlock, &infoData);
blockDataAppendColInfo(b, &infoData);
SColumnInfoData infoData1 = {0};
infoData1.info.bytes = 40;
infoData1.info.type = TSDB_DATA_TYPE_BINARY;
infoData1.info.colId = 2;
infoData1.varmeta.offset = (int32_t*)taosMemoryCalloc(40, sizeof(uint32_t));
taosArrayPush(b->pDataBlock, &infoData1);
SColumnInfoData infoData1 = createColumnInfoData(TSDB_DATA_TYPE_BINARY, 40, 2);
blockDataAppendColInfo(b, &infoData1);
blockDataEnsureCapacity(b, 40);
char* str = "the value of: %d";
char buf[128] = {0};
@ -222,28 +211,17 @@ TEST(testCase, non_var_dataBlock_split_test) {
#endif
TEST(testCase, var_dataBlock_split_test) {
SSDataBlock* b = static_cast<SSDataBlock*>(taosMemoryCalloc(1, sizeof(SSDataBlock)));
b->info.numOfCols = 2;
b->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData));
int32_t numOfRows = 1000000;
SColumnInfoData infoData = {0};
infoData.info.bytes = 4;
infoData.info.type = TSDB_DATA_TYPE_INT;
infoData.info.colId = 1;
SSDataBlock* b = createDataBlock();
infoData.pData = (char*)taosMemoryCalloc(numOfRows, infoData.info.bytes);
infoData.nullbitmap = (char*)taosMemoryCalloc(1, sizeof(char) * (numOfRows / 8));
taosArrayPush(b->pDataBlock, &infoData);
SColumnInfoData infoData = createColumnInfoData(TSDB_DATA_TYPE_INT, 4, 1);
blockDataAppendColInfo(b, &infoData);
SColumnInfoData infoData1 = {0};
infoData1.info.bytes = 40;
infoData1.info.type = TSDB_DATA_TYPE_BINARY;
infoData1.info.colId = 2;
SColumnInfoData infoData1 = createColumnInfoData(TSDB_DATA_TYPE_BINARY, 40, 2);
blockDataAppendColInfo(b, &infoData1);
infoData1.varmeta.offset = (int32_t*)taosMemoryCalloc(numOfRows, sizeof(uint32_t));
taosArrayPush(b->pDataBlock, &infoData1);
blockDataEnsureCapacity(b, numOfRows);
char buf[41] = {0};
char buf1[100] = {0};

23
source/dnode/mgmt/mgmt_dnode/src/dmHandle.c
View File

@ -123,8 +123,15 @@ int32_t dmProcessGrantRsp(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) {
}
int32_t dmProcessConfigReq(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) {
dError("config req is received, but not supported yet");
return TSDB_CODE_OPS_NOT_SUPPORT;
SDCfgDnodeReq cfgReq = {0};
if (tDeserializeSDCfgDnodeReq(pMsg->pCont, pMsg->contLen, &cfgReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG;
return -1;
}
dInfo("start to config, option:%s, value:%s", cfgReq.config, cfgReq.value);
taosCfgDynamicOptions(cfgReq.config, cfgReq.value);
return 0;
}
static void dmGetServerRunStatus(SDnodeMgmt *pMgmt, SServerStatusRsp *pStatus) {
@ -203,7 +210,6 @@ SSDataBlock* dmBuildVariablesBlock(void) {
taosArrayPush(pBlock->pDataBlock, &colInfoData);
}
pBlock->info.numOfCols = pMeta[index].colNum;
pBlock->info.hasVarCol = true;
return pBlock;
@ -262,8 +268,9 @@ int32_t dmProcessRetrieve(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) {
dmAppendVariablesToBlock(pBlock, pMgmt->pData->dnodeId);
size = sizeof(SRetrieveMetaTableRsp) + sizeof(int32_t) + sizeof(SSysTableSchema) * pBlock->info.numOfCols +
blockDataGetSize(pBlock) + blockDataGetSerialMetaSize(pBlock->info.numOfCols);
size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
size = sizeof(SRetrieveMetaTableRsp) + sizeof(int32_t) + sizeof(SSysTableSchema) * numOfCols +
blockDataGetSize(pBlock) + blockDataGetSerialMetaSize(numOfCols);
SRetrieveMetaTableRsp *pRsp = rpcMallocCont(size);
if (pRsp == NULL) {
@ -274,10 +281,10 @@ int32_t dmProcessRetrieve(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) {
}
char *pStart = pRsp->data;
*(int32_t *)pStart = htonl(pBlock->info.numOfCols);
*(int32_t *)pStart = htonl(numOfCols);
pStart += sizeof(int32_t); // number of columns
for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) {
for (int32_t i = 0; i < numOfCols; ++i) {
SSysTableSchema *pSchema = (SSysTableSchema *)pStart;
SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, i);
@ -289,7 +296,7 @@ int32_t dmProcessRetrieve(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) {
}
int32_t len = 0;
blockCompressEncode(pBlock, pStart, &len, pBlock->info.numOfCols, false);
blockCompressEncode(pBlock, pStart, &len, numOfCols, false);
pRsp->numOfRows = htonl(pBlock->info.rows);
pRsp->precision = TSDB_TIME_PRECISION_MILLI; // millisecond time precision

2
source/dnode/mgmt/mgmt_mnode/src/mmHandle.c
View File

@ -17,7 +17,7 @@
#include "mmInt.h"
void mmGetMonitorInfo(SMnodeMgmt *pMgmt, SMonMmInfo *pInfo) {
mndGetMonitorInfo(pMgmt->pMnode, &pInfo->cluster, &pInfo->vgroup, &pInfo->grant);
mndGetMonitorInfo(pMgmt->pMnode, &pInfo->cluster, &pInfo->vgroup, &pInfo->stb, &pInfo->grant);
}
int32_t mmProcessGetMonitorInfoReq(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) {

16
source/dnode/mgmt/node_util/src/dmFile.c
View File

@ -127,14 +127,24 @@ TdFilePtr dmCheckRunning(const char *dataDir) {
return NULL;
}
int32_t ret = taosLockFile(pFile);
if (ret != 0) {
int32_t retryTimes = 0;
int32_t ret = 0;
do {
ret = taosLockFile(pFile);
if (ret == 0) break;
terrno = TAOS_SYSTEM_ERROR(errno);
taosMsleep(100);
retryTimes++;
dError("failed to lock file:%s since %s, retryTimes:%d", filepath, terrstr(), retryTimes);
} while (retryTimes < 120);
if (ret < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
dError("failed to lock file:%s since %s", filepath, terrstr());
taosCloseFile(&pFile);
return NULL;
}
terrno = 0;
dDebug("lock file:%s to prevent repeated starts", filepath);
return pFile;
}

29
source/dnode/mnode/impl/src/mndDnode.c
View File

@ -554,7 +554,6 @@ _OVER:
return code;
}
static int32_t mndProcessCreateDnodeReq(SRpcMsg *pReq) {
SMnode *pMnode = pReq->info.node;
int32_t code = -1;
@ -732,15 +731,37 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) {
mError("dnode:%d, failed to config since %s ", cfgReq.dnodeId, terrstr());
return -1;
}
SEpSet epSet = mndGetDnodeEpset(pDnode);
mndReleaseDnode(pMnode, pDnode);
int32_t bufLen = tSerializeSMCfgDnodeReq(NULL, 0, &cfgReq);
SDCfgDnodeReq dcfgReq = {0};
if (strncasecmp(cfgReq.config, "debugFlag", 9) == 0) {
const char *value = cfgReq.value;
int32_t flag = atoi(value);
if (flag <= 0) {
flag = atoi(cfgReq.config + 10);
}
if (flag <= 0 || flag > 255) {
mError("dnode:%d, failed to config debugFlag since value:%d", cfgReq.dnodeId, flag);
terrno = TSDB_CODE_INVALID_CFG;
return -1;
}
strcpy(dcfgReq.config, "debugFlag");
snprintf(dcfgReq.value, TSDB_DNODE_VALUE_LEN, "%d", flag);
} else if (strcasecmp(cfgReq.config, "resetlog") == 0) {
strcpy(dcfgReq.config, "resetlog");
} else {
terrno = TSDB_CODE_INVALID_CFG;
mError("dnode:%d, failed to config since %s", cfgReq.dnodeId, terrstr());
return -1;
}
int32_t bufLen = tSerializeSDCfgDnodeReq(NULL, 0, &dcfgReq);
void *pBuf = rpcMallocCont(bufLen);
if (pBuf == NULL) return -1;
tSerializeSMCfgDnodeReq(pBuf, bufLen, &cfgReq);
tSerializeSDCfgDnodeReq(pBuf, bufLen, &dcfgReq);
mDebug("dnode:%d, send config req to dnode, app:%p", cfgReq.dnodeId, pReq->info.ahandle);
SRpcMsg rpcMsg = {.msgType = TDMT_DND_CONFIG_DNODE, .pCont = pBuf, .contLen = bufLen};

30
source/dnode/mnode/impl/src/mndMain.c
View File

@ -614,7 +614,7 @@ int64_t mndGenerateUid(char *name, int32_t len) {
}
int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo,
SMonGrantInfo *pGrantInfo) {
SMonStbInfo *pStbInfo, SMonGrantInfo *pGrantInfo) {
if (mndAcquireRpcRef(pMnode) != 0) return -1;
SSdb *pSdb = pMnode->pSdb;
@ -623,7 +623,9 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr
pClusterInfo->dnodes = taosArrayInit(sdbGetSize(pSdb, SDB_DNODE), sizeof(SMonDnodeDesc));
pClusterInfo->mnodes = taosArrayInit(sdbGetSize(pSdb, SDB_MNODE), sizeof(SMonMnodeDesc));
pVgroupInfo->vgroups = taosArrayInit(sdbGetSize(pSdb, SDB_VGROUP), sizeof(SMonVgroupDesc));
if (pClusterInfo->dnodes == NULL || pClusterInfo->mnodes == NULL || pVgroupInfo->vgroups == NULL) {
pStbInfo->stbs = taosArrayInit(sdbGetSize(pSdb, SDB_STB), sizeof(SMonStbDesc));
if (pClusterInfo->dnodes == NULL || pClusterInfo->mnodes == NULL || pVgroupInfo->vgroups == NULL ||
pStbInfo->stbs == NULL) {
mndReleaseRpcRef(pMnode);
return -1;
}
@ -632,6 +634,8 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr
tstrncpy(pClusterInfo->version, version, sizeof(pClusterInfo->version));
pClusterInfo->monitor_interval = tsMonitorInterval;
pClusterInfo->connections_total = mndGetNumOfConnections(pMnode);
pClusterInfo->dbs_total = sdbGetSize(pSdb, SDB_DB);
pClusterInfo->stbs_total = sdbGetSize(pSdb, SDB_STB);
void *pIter = NULL;
while (1) {
@ -681,6 +685,7 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr
if (pIter == NULL) break;
pClusterInfo->vgroups_total++;
pClusterInfo->tbs_total += pVgroup->numOfTables;
SMonVgroupDesc desc = {0};
desc.vgroup_id = pVgroup->vgId;
@ -711,6 +716,27 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr
sdbRelease(pSdb, pVgroup);
}
// stb info
pIter = NULL;
while (1) {
SStbObj *pStb = NULL;
pIter = sdbFetch(pSdb, SDB_STB, pIter, (void **)&pStb);
if (pIter == NULL) break;
SMonStbDesc desc = {0};
SName name1 = {0};
tNameFromString(&name1, pStb->db, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
tNameGetDbName(&name1, desc.database_name);
SName name2 = {0};
tNameFromString(&name2, pStb->name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
tstrncpy(desc.stb_name, tNameGetTableName(&name2), TSDB_TABLE_NAME_LEN);
taosArrayPush(pStbInfo->stbs, &desc);
sdbRelease(pSdb, pStb);
}
// grant info
pGrantInfo->expire_time = (pMnode->grant.expireTimeMS - ms) / 86400000.0f;
pGrantInfo->timeseries_total = pMnode->grant.timeseriesAllowed;

3
source/dnode/mnode/impl/src/mndShow.c
View File

@ -236,7 +236,6 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) {
int32_t numOfCols = pShow->pMeta->numOfColumns;
SSDataBlock *pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock));
pBlock->pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData));
pBlock->info.numOfCols = numOfCols;
for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData idata = {0};
@ -271,7 +270,7 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) {
}
size = sizeof(SRetrieveMetaTableRsp) + sizeof(int32_t) + sizeof(SSysTableSchema) * pShow->pMeta->numOfColumns +
blockDataGetSize(pBlock) + blockDataGetSerialMetaSize(pBlock->info.numOfCols);
blockDataGetSize(pBlock) + blockDataGetSerialMetaSize(taosArrayGetSize(pBlock->pDataBlock));
SRetrieveMetaTableRsp *pRsp = rpcMallocCont(size);
if (pRsp == NULL) {

4
source/dnode/vnode/inc/vnode.h
View File

@ -149,8 +149,8 @@ int32_t tqReadHandleRemoveTbUidList(STqReadHandle *pHandle, const SArray *tbUidL
int32_t tqReadHandleSetMsg(STqReadHandle *pHandle, SSubmitReq *pMsg, int64_t ver);
bool tqNextDataBlock(STqReadHandle *pHandle);
bool tqNextDataBlockFilterOut(STqReadHandle *pHandle, SHashObj *filterOutUids);
int32_t tqRetrieveDataBlock(SArray **ppCols, STqReadHandle *pHandle, uint64_t *pGroupId, uint64_t *pUid,
int32_t *pNumOfRows, int16_t *pNumOfCols);
int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, STqReadHandle *pHandle, uint64_t *pGroupId, uint64_t *pUid,
int32_t *pNumOfRows);
// sma
int32_t smaGetTSmaDays(SVnodeCfg *pCfg, void *pCont, uint32_t contLen, int32_t *days);

2
source/dnode/vnode/src/tq/tq.c
View File

@ -306,7 +306,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
pHead->msgType == TDMT_VND_DROP_STB || pHead->msgType == TDMT_VND_CREATE_TABLE ||
pHead->msgType == TDMT_VND_ALTER_TABLE || pHead->msgType == TDMT_VND_DROP_TABLE ||
pHead->msgType == TDMT_VND_DROP_TTL_TABLE);
// return
tqInfo("fetch meta msg, ver: %ld, type: %d", pHead->version, pHead->msgType);
SMqMetaRsp metaRsp = {0};
metaRsp.reqOffset = pReq->currentOffset;
metaRsp.rspOffset = fetchOffset;

12
source/dnode/vnode/src/tq/tqExec.c
View File

@ -29,7 +29,7 @@ static int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataBlkRsp* pRs
// TODO enable compress
int32_t actualLen = 0;
blockCompressEncode(pBlock, pRetrieve->data, &actualLen, pBlock->info.numOfCols, false);
blockCompressEncode(pBlock, pRetrieve->data, &actualLen, taosArrayGetSize(pBlock->pDataBlock), false);
actualLen += sizeof(SRetrieveTableRsp);
ASSERT(actualLen <= dataStrLen);
taosArrayPush(pRsp->blockDataLen, &actualLen);
@ -66,6 +66,7 @@ int32_t tqScanSnapshot(STQ* pTq, const STqExecHandle* pExec, SMqDataBlkRsp* pRsp
if (qStreamScanSnapshot(task) < 0) {
ASSERT(0);
}
// set version
while (1) {
SSDataBlock* pDataBlock = NULL;
uint64_t ts = 0;
@ -75,7 +76,7 @@ int32_t tqScanSnapshot(STQ* pTq, const STqExecHandle* pExec, SMqDataBlkRsp* pRsp
if (pDataBlock == NULL) break;
ASSERT(pDataBlock->info.rows != 0);
ASSERT(pDataBlock->info.numOfCols != 0);
ASSERT(taosArrayGetSize(pDataBlock->pDataBlock) != 0);
tqAddBlockDataToRsp(pDataBlock, pRsp);
pRsp->blockNum++;
@ -98,7 +99,6 @@ int32_t tqDataExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataBlkR
if (pDataBlock == NULL) break;
ASSERT(pDataBlock->info.rows != 0);
ASSERT(pDataBlock->info.numOfCols != 0);
tqAddBlockDataToRsp(pDataBlock, pRsp);
if (pRsp->withTbName) {
@ -112,8 +112,7 @@ int32_t tqDataExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataBlkR
tqReadHandleSetMsg(pReader, pReq, 0);
while (tqNextDataBlock(pReader)) {
SSDataBlock block = {0};
if (tqRetrieveDataBlock(&block.pDataBlock, pReader, &block.info.groupId, &block.info.uid, &block.info.rows,
&block.info.numOfCols) < 0) {
if (tqRetrieveDataBlock(&block, pReader, &block.info.groupId, &block.info.uid, &block.info.rows) < 0) {
if (terrno == TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND) continue;
ASSERT(0);
}
@ -130,8 +129,7 @@ int32_t tqDataExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataBlkR
tqReadHandleSetMsg(pReader, pReq, 0);
while (tqNextDataBlockFilterOut(pReader, pExec->execDb.pFilterOutTbUid)) {
SSDataBlock block = {0};
if (tqRetrieveDataBlock(&block.pDataBlock, pReader, &block.info.groupId, &block.info.uid, &block.info.rows,
&block.info.numOfCols) < 0) {
if (tqRetrieveDataBlock(&block, pReader, &block.info.groupId, &block.info.uid, &block.info.rows) < 0) {
if (terrno == TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND) continue;
ASSERT(0);
}

4
source/dnode/vnode/src/tq/tqOffset.c
View File

@ -44,6 +44,7 @@ STqOffsetStore* tqOffsetOpen(STQ* pTq) {
}
char* fname = buildFileName(pStore->pTq->path);
TdFilePtr pFile = taosOpenFile(fname, TD_FILE_READ);
taosMemoryFree(fname);
if (pFile != NULL) {
STqOffsetHead head = {0};
int64_t code;
@ -77,7 +78,6 @@ STqOffsetStore* tqOffsetOpen(STQ* pTq) {
}
taosCloseFile(&pFile);
taosMemoryFree(fname);
}
return pStore;
}
@ -102,6 +102,7 @@ int32_t tqOffsetSnapshot(STqOffsetStore* pStore) {
// TODO file name should be with a version
char* fname = buildFileName(pStore->pTq->path);
TdFilePtr pFile = taosOpenFile(fname, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND);
taosMemoryFree(fname);
if (pFile == NULL) {
ASSERT(0);
return -1;
@ -140,6 +141,5 @@ int32_t tqOffsetSnapshot(STqOffsetStore* pStore) {
}
// close and rename file
taosCloseFile(&pFile);
taosMemoryFree(fname);
return 0;
}

50
source/dnode/vnode/src/tq/tqRead.c
View File

@ -142,8 +142,8 @@ bool tqNextDataBlockFilterOut(STqReadHandle* pHandle, SHashObj* filterOutUids) {
return false;
}
int32_t tqRetrieveDataBlock(SArray** ppCols, STqReadHandle* pHandle, uint64_t* pGroupId, uint64_t* pUid,
int32_t* pNumOfRows, int16_t* pNumOfCols) {
int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, STqReadHandle* pHandle, uint64_t* pGroupId, uint64_t* pUid,
int32_t* pNumOfRows) {
*pUid = 0;
// TODO set to real sversion
@ -151,6 +151,7 @@ int32_t tqRetrieveDataBlock(SArray** ppCols, STqReadHandle* pHandle, uint64_t* p
int32_t sversion = htonl(pHandle->pBlock->sversion);
if (pHandle->cachedSchemaSuid == 0 || pHandle->cachedSchemaVer != sversion ||
pHandle->cachedSchemaSuid != pHandle->msgIter.suid) {
if (pHandle->pSchema) taosMemoryFree(pHandle->pSchema);
pHandle->pSchema = metaGetTbTSchema(pHandle->pVnodeMeta, pHandle->msgIter.uid, sversion);
if (pHandle->pSchema == NULL) {
tqWarn("cannot found tsschema for table: uid: %ld (suid: %ld), version %d, possibly dropped table",
@ -161,6 +162,7 @@ int32_t tqRetrieveDataBlock(SArray** ppCols, STqReadHandle* pHandle, uint64_t* p
}
// this interface use suid instead of uid
if (pHandle->pSchemaWrapper) tDeleteSSchemaWrapper(pHandle->pSchemaWrapper);
pHandle->pSchemaWrapper = metaGetTableSchema(pHandle->pVnodeMeta, pHandle->msgIter.uid, sversion, true);
if (pHandle->pSchemaWrapper == NULL) {
tqWarn("cannot found schema wrapper for table: suid: %ld, version %d, possibly dropped table",
@ -180,23 +182,14 @@ int32_t tqRetrieveDataBlock(SArray** ppCols, STqReadHandle* pHandle, uint64_t* p
int32_t colNumNeed = taosArrayGetSize(pHandle->pColIdList);
if (colNumNeed == 0) {
*ppCols = taosArrayInit(pSchemaWrapper->nCols, sizeof(SColumnInfoData));
if (*ppCols == NULL) {
return -1;
}
int32_t colMeta = 0;
while (colMeta < pSchemaWrapper->nCols) {
SSchema* pColSchema = &pSchemaWrapper->pSchema[colMeta];
SColumnInfoData colInfo = {0};
colInfo.info.bytes = pColSchema->bytes;
colInfo.info.colId = pColSchema->colId;
colInfo.info.type = pColSchema->type;
if (colInfoDataEnsureCapacity(&colInfo, 0, *pNumOfRows) < 0) {
SColumnInfoData colInfo = createColumnInfoData(pColSchema->type, pColSchema->bytes, pColSchema->colId);
int32_t code = blockDataAppendColInfo(pBlock, &colInfo);
if (code != TSDB_CODE_SUCCESS) {
goto FAIL;
}
taosArrayPush(*ppCols, &colInfo);
colMeta++;
}
} else {
@ -204,11 +197,6 @@ int32_t tqRetrieveDataBlock(SArray** ppCols, STqReadHandle* pHandle, uint64_t* p
colNumNeed = pSchemaWrapper->nCols;
}
*ppCols = taosArrayInit(colNumNeed, sizeof(SColumnInfoData));
if (*ppCols == NULL) {
return -1;
}
int32_t colMeta = 0;
int32_t colNeed = 0;
while (colMeta < pSchemaWrapper->nCols && colNeed < colNumNeed) {
@ -220,23 +208,22 @@ int32_t tqRetrieveDataBlock(SArray** ppCols, STqReadHandle* pHandle, uint64_t* p
} else if (colIdSchema > colIdNeed) {
colNeed++;
} else {
SColumnInfoData colInfo = {0};
colInfo.info.bytes = pColSchema->bytes;
colInfo.info.colId = pColSchema->colId;
colInfo.info.type = pColSchema->type;
if (colInfoDataEnsureCapacity(&colInfo, 0, *pNumOfRows) < 0) {
SColumnInfoData colInfo = createColumnInfoData(pColSchema->type, pColSchema->bytes, pColSchema->colId);
int32_t code = blockDataAppendColInfo(pBlock, &colInfo);
if (code != TSDB_CODE_SUCCESS) {
goto FAIL;
}
taosArrayPush(*ppCols, &colInfo);
colMeta++;
colNeed++;
}
}
}
int32_t colActual = taosArrayGetSize(*ppCols);
*pNumOfCols = colActual;
if (blockDataEnsureCapacity(pBlock, *pNumOfRows) < 0) {
goto FAIL;
}
int32_t colActual = blockDataGetNumOfCols(pBlock);
// TODO in stream shuffle case, fetch groupId
*pGroupId = 0;
@ -253,7 +240,7 @@ int32_t tqRetrieveDataBlock(SArray** ppCols, STqReadHandle* pHandle, uint64_t* p
tdSTSRowIterReset(&iter, row);
// get all wanted col of that block
for (int32_t i = 0; i < colActual; i++) {
SColumnInfoData* pColData = taosArrayGet(*ppCols, i);
SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, i);
SCellVal sVal = {0};
if (!tdSTSRowIterNext(&iter, pColData->info.colId, pColData->info.type, &sVal)) {
break;
@ -265,8 +252,9 @@ int32_t tqRetrieveDataBlock(SArray** ppCols, STqReadHandle* pHandle, uint64_t* p
curRow++;
}
return 0;
FAIL:
if (*ppCols) taosArrayDestroy(*ppCols);
FAIL: // todo refactor here
// if (*ppCols) taosArrayDestroy(*ppCols);
return -1;
}

4
source/dnode/vnode/src/tq/tqSink.c
View File

@ -44,7 +44,7 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema, boo
createTbReq.ctb.suid = suid;
STagVal tagVal = {
.cid = pDataBlock->info.numOfCols + 1,
.cid = taosArrayGetSize(pDataBlock->pDataBlock) + 1,
.type = TSDB_DATA_TYPE_UBIGINT,
.i64 = (int64_t)pDataBlock->info.groupId,
};
@ -107,7 +107,7 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema, boo
createTbReq.ctb.suid = suid;
STagVal tagVal = {
.cid = pDataBlock->info.numOfCols + 1,
.cid = taosArrayGetSize(pDataBlock->pDataBlock) + 1,
.type = TSDB_DATA_TYPE_UBIGINT,
.i64 = (int64_t)pDataBlock->info.groupId,
};

69
source/dnode/vnode/src/tsdb/tsdbRead.c
View File

@ -16,13 +16,11 @@
#include "tsdb.h"
#include "vnode.h"
#define EXTRA_BYTES 2
#define ASCENDING_TRAVERSE(o) (o == TSDB_ORDER_ASC)
#define QH_GET_NUM_OF_COLS(handle) ((size_t)(taosArrayGetSize((handle)->pColumns)))
#define QH_GET_NUM_OF_COLS(handle) ((size_t)(taosArrayGetSize((handle)->pResBlock->pDataBlock)))
#define GET_FILE_DATA_BLOCK_INFO(_checkInfo, _block) \
((SDataBlockInfo){.window = {.skey = (_block)->minKey.ts, .ekey = (_block)->maxKey.ts}, \
.numOfCols = (_block)->numOfCols, \
.rows = (_block)->numOfRows, \
.uid = (_checkInfo)->tableId})
@ -115,7 +113,8 @@ typedef struct STsdbReadHandle {
// SColumnDataAgg* statis; // query level statistics, only one table block statistics info exists at any time
// SColumnDataAgg** pstatis;// the ptr array list to return to caller
int32_t numOfBlocks;
SArray* pColumns; // column list, SColumnInfoData array list
SSDataBlock* pResBlock;
// SArray* pColumns; // column list, SColumnInfoData array list
bool locateStart;
int32_t outputCapacity;
int32_t realNumOfRows;
@ -177,7 +176,7 @@ static SArray* getColumnIdList(STsdbReadHandle* pTsdbReadHandle) {
SArray* pIdList = taosArrayInit(numOfCols, sizeof(int16_t));
for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pCol = taosArrayGet(pTsdbReadHandle->pColumns, i);
SColumnInfoData* pCol = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, i);
taosArrayPush(pIdList, &pCol->info.colId);
}
@ -425,23 +424,21 @@ static STsdbReadHandle* tsdbQueryTablesImpl(SVnode* pVnode, SQueryTableDataCond*
}
// todo: use list instead of array?
pReadHandle->pColumns = taosArrayInit(pCond->numOfCols, sizeof(SColumnInfoData));
if (pReadHandle->pColumns == NULL) {
pReadHandle->pResBlock = createDataBlock();
if (pReadHandle->pResBlock == NULL) {
goto _end;
}
for (int32_t i = 0; i < pCond->numOfCols; ++i) {
SColumnInfoData colInfo = {{0}, 0};
colInfo.info = pCond->colList[i];
int32_t code = colInfoDataEnsureCapacity(&colInfo, 0, pReadHandle->outputCapacity);
if (code != TSDB_CODE_SUCCESS) {
SColumnInfoData colInfo = {.info = pCond->colList[i], 0};
int32_t code = blockDataAppendColInfo(pReadHandle->pResBlock, &colInfo);
if (code != TSDB_CODE_SUCCESS){
goto _end;
}
taosArrayPush(pReadHandle->pColumns, &colInfo);
}
blockDataEnsureCapacity(pReadHandle->pResBlock, pReadHandle->outputCapacity);
pReadHandle->suppInfo.defaultLoadColumn = getDefaultLoadColumns(pReadHandle, true);
size_t size = taosArrayGetSize(pReadHandle->suppInfo.defaultLoadColumn);
@ -1536,12 +1533,12 @@ static int32_t doCopyRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, int32_t
int32_t trueEnd = ascScan ? end : start;
int32_t step = ascScan ? 1 : -1;
int32_t requiredNumOfCols = (int32_t)taosArrayGetSize(pTsdbReadHandle->pColumns);
int32_t requiredNumOfCols = (int32_t)taosArrayGetSize(pTsdbReadHandle->pResBlock->pDataBlock);
// data in buffer has greater timestamp, copy data in file block
int32_t i = 0, j = 0;
while (i < requiredNumOfCols && j < pCols->numOfCols) {
SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i);
SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, i);
SDataCol* src = &pCols->cols[j];
if (src->colId < pColInfo->info.colId) {
@ -1592,7 +1589,7 @@ static int32_t doCopyRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, int32_t
}
while (i < requiredNumOfCols) { // the remain columns are all null data
SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i);
SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, i);
colDataAppendNNULL(pColInfo, numOfRows, num);
i++;
}
@ -1674,7 +1671,7 @@ static int32_t mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capa
int32_t i = 0, j = 0, k = 0;
while (i < numOfCols && (j < numOfColsOfRow1 || k < numOfColsOfRow2)) {
SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i);
SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, i);
int32_t colIdOfRow1;
if (j >= numOfColsOfRow1) {
@ -1805,7 +1802,7 @@ static int32_t mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capa
if (mergeOption == 1) {
while (i < numOfCols) { // the remain columns are all null data
SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i);
SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, i);
colDataAppend(pColInfo, *curRow, NULL, true);
++i;
}
@ -1861,7 +1858,7 @@ static void doCheckGeneratedBlockRange(STsdbReadHandle* pTsdbReadHandle) {
assert(cur->win.skey >= pTsdbReadHandle->window.ekey && cur->win.ekey <= pTsdbReadHandle->window.skey);
}
SColumnInfoData* pColInfoData = taosArrayGet(pTsdbReadHandle->pColumns, 0);
SColumnInfoData* pColInfoData = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, 0);
assert(cur->win.skey == ((TSKEY*)pColInfoData->pData)[0] &&
cur->win.ekey == ((TSKEY*)pColInfoData->pData)[cur->rows - 1]);
} else {
@ -2771,7 +2768,7 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int
STsdbReadHandle* pTsdbReadHandle) {
int numOfRows = 0;
int curRows = 0;
int32_t numOfCols = (int32_t)taosArrayGetSize(pTsdbReadHandle->pColumns);
int32_t numOfCols = (int32_t)taosArrayGetSize(pTsdbReadHandle->pResBlock->pDataBlock);
STsdbCfg* pCfg = REPO_CFG(pTsdbReadHandle->pTsdb);
win->skey = TSKEY_INITIAL_VAL;
@ -2901,7 +2898,7 @@ static bool loadBlockOfActiveTable(STsdbReadHandle* pTsdbReadHandle) {
if (exists) {
tsdbRetrieveDataBlock((tsdbReaderT*)pTsdbReadHandle, NULL);
if (pTsdbReadHandle->currentLoadExternalRows && pTsdbReadHandle->window.skey == pTsdbReadHandle->window.ekey) {
SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, 0);
SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, 0);
assert(*(int64_t*)pColInfo->pData == pTsdbReadHandle->window.skey);
}
@ -3003,7 +3000,7 @@ static bool loadCachedLastRow(STsdbReadHandle* pTsdbReadHandle) {
//
// int32_t i = 0, j = 0;
// while(i < tgNumOfCols && j < numOfCols) {
// pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i);
// pColInfo = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, i);
// if (pTable->lastCols[j].colId < pColInfo->info.colId) {
// j++;
// continue;
@ -3066,7 +3063,7 @@ static bool loadCachedLastRow(STsdbReadHandle* pTsdbReadHandle) {
// continue;
// }
//
// pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, n);
// pColInfo = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, n);
// pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes;;
//
// if (pColInfo->info.colId == PRIMARYKEY_TIMESTAMP_COL_ID) {
@ -3091,7 +3088,7 @@ static bool loadCachedLastRow(STsdbReadHandle* pTsdbReadHandle) {
//
// // leave the real ts column as the last row, because last function only (not stable) use the last row as res
// if (priKey != TSKEY_INITIAL_VAL) {
// pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, priIdx);
// pColInfo = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, priIdx);
// pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes;
//
// *(TSKEY *)pData = priKey;
@ -3101,7 +3098,7 @@ static bool loadCachedLastRow(STsdbReadHandle* pTsdbReadHandle) {
// continue;
// }
//
// pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, n);
// pColInfo = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, n);
// pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes;;
//
// assert (pColInfo->info.colId != PRIMARYKEY_TIMESTAMP_COL_ID);
@ -3161,9 +3158,9 @@ static bool loadDataBlockFromTableSeq(STsdbReadHandle* pTsdbReadHandle) {
bool tsdbNextDataBlock(tsdbReaderT pHandle) {
STsdbReadHandle* pTsdbReadHandle = (STsdbReadHandle*)pHandle;
size_t numOfCols = taosArrayGetSize(pTsdbReadHandle->pColumns);
size_t numOfCols = taosArrayGetSize(pTsdbReadHandle->pResBlock->pDataBlock);
for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i);
SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, i);
colInfoDataCleanup(pColInfo, pTsdbReadHandle->outputCapacity);
}
@ -3250,7 +3247,7 @@ bool tsdbNextDataBlock(tsdbReaderT pHandle) {
// SArray* row = (type == TSDB_PREV_ROW)? pTsdbReadHandle->prev : pTsdbReadHandle->next;
//
// for (int32_t i = 0; i < numOfCols; ++i) {
// SColumnInfoData* pCol = taosArrayGet(pTsdbReadHandle->pColumns, i);
// SColumnInfoData* pCol = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, i);
//
// SColumnInfoData colInfo = {{0}, 0};
// colInfo.info = pCol->info;
@ -3280,7 +3277,7 @@ bool tsdbNextDataBlock(tsdbReaderT pHandle) {
// }
//
// for (int32_t i = 0; i < cond.numOfCols; ++i) {
// SColumnInfoData* pColInfoData = taosArrayGet(pTsdbReadHandle->pColumns, i);
// SColumnInfoData* pColInfoData = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, i);
// memcpy(&cond.colList[i], &pColInfoData->info, sizeof(SColumnInfo));
// }
//
@ -3342,7 +3339,7 @@ bool tsdbGetExternalRow(tsdbReaderT pHandle) {
int32_t numOfCols = (int32_t)QH_GET_NUM_OF_COLS(pTsdbReadHandle);
for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pColInfoData = taosArrayGet(pTsdbReadHandle->pColumns, i);
SColumnInfoData* pColInfoData = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, i);
SColumnInfoData* first = taosArrayGet(pTsdbReadHandle->prev, i);
memcpy(pColInfoData->pData, first->pData, pColInfoData->info.bytes);
@ -3617,13 +3614,13 @@ SArray* tsdbRetrieveDataBlock(tsdbReaderT* pTsdbReadHandle, SArray* pIdList) {
*/
STsdbReadHandle* pHandle = (STsdbReadHandle*)pTsdbReadHandle;
if (pHandle->cur.fid == INT32_MIN) {
return pHandle->pColumns;
return pHandle->pResBlock->pDataBlock;
} else {
STableBlockInfo* pBlockInfo = &pHandle->pDataBlockInfo[pHandle->cur.slot];
STableCheckInfo* pCheckInfo = pBlockInfo->pTableCheckInfo;
if (pHandle->cur.mixBlock) {
return pHandle->pColumns;
return pHandle->pResBlock->pDataBlock;
} else {
SDataBlockInfo binfo = GET_FILE_DATA_BLOCK_INFO(pCheckInfo, pBlockInfo->compBlock);
assert(pHandle->realNumOfRows <= binfo.rows);
@ -3633,7 +3630,7 @@ SArray* tsdbRetrieveDataBlock(tsdbReaderT* pTsdbReadHandle, SArray* pIdList) {
if (pBlockLoadInfo->slot == pHandle->cur.slot && pBlockLoadInfo->fileGroup->fid == pHandle->cur.fid &&
pBlockLoadInfo->uid == pCheckInfo->tableId) {
return pHandle->pColumns;
return pHandle->pResBlock->pDataBlock;
} else { // only load the file block
SBlock* pBlock = pBlockInfo->compBlock;
if (doLoadFileDataBlock(pHandle, pBlock, pCheckInfo, pHandle->cur.slot) != TSDB_CODE_SUCCESS) {
@ -3641,7 +3638,7 @@ SArray* tsdbRetrieveDataBlock(tsdbReaderT* pTsdbReadHandle, SArray* pIdList) {
}
int32_t numOfRows = doCopyRowsFromFileBlock(pHandle, pHandle->outputCapacity, 0, 0, pBlock->numOfRows - 1);
return pHandle->pColumns;
return pHandle->pResBlock->pDataBlock;
}
}
}
@ -3692,7 +3689,7 @@ void tsdbCleanupReadHandle(tsdbReaderT queryHandle) {
return;
}
pTsdbReadHandle->pColumns = doFreeColumnInfoData(pTsdbReadHandle->pColumns);
pTsdbReadHandle->pResBlock->pDataBlock = doFreeColumnInfoData(pTsdbReadHandle->pResBlock->pDataBlock);
taosArrayDestroy(pTsdbReadHandle->suppInfo.defaultLoadColumn);
taosMemoryFreeClear(pTsdbReadHandle->pDataBlockInfo);

2
source/dnode/vnode/test/tsdbSmaTest.cpp
View File

@ -435,7 +435,7 @@ TEST(testCase, tSma_Data_Insert_Query_Test) {
SSDataBlock *pDataBlock = (SSDataBlock *)taosMemoryCalloc(1, sizeof(SSDataBlock));
EXPECT_NE(pDataBlock, nullptr);
pDataBlock->pBlockAgg = NULL;
pDataBlock->info.numOfCols = tSmaNumOfCols;
taosArrayGetSize(pDataBlock->pDataBlock) = tSmaNumOfCols;
pDataBlock->info.rows = tSmaNumOfRows;
pDataBlock->info.groupId = tSmaGroupId + g;

71
source/libs/command/src/command.c
View File

@ -32,31 +32,19 @@ static int32_t getSchemaBytes(const SSchema* pSchema) {
}
static SSDataBlock* buildDescResultDataBlock() {
SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock));
pBlock->info.numOfCols = DESCRIBE_RESULT_COLS;
pBlock->info.hasVarCol = true;
SSDataBlock* pBlock = createDataBlock();
pBlock->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData));
SColumnInfoData infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, DESCRIBE_RESULT_FIELD_LEN, 1);
blockDataAppendColInfo(pBlock, &infoData);
SColumnInfoData infoData = {0};
infoData.info.type = TSDB_DATA_TYPE_VARCHAR;
infoData.info.bytes = DESCRIBE_RESULT_FIELD_LEN;
infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, DESCRIBE_RESULT_TYPE_LEN, 2);
blockDataAppendColInfo(pBlock, &infoData);
taosArrayPush(pBlock->pDataBlock, &infoData);
infoData.info.type = TSDB_DATA_TYPE_VARCHAR;
infoData.info.bytes = DESCRIBE_RESULT_TYPE_LEN;
taosArrayPush(pBlock->pDataBlock, &infoData);
infoData.info.type = TSDB_DATA_TYPE_INT;
infoData.info.bytes = tDataTypes[TSDB_DATA_TYPE_INT].bytes;
taosArrayPush(pBlock->pDataBlock, &infoData);
infoData.info.type = TSDB_DATA_TYPE_VARCHAR;
infoData.info.bytes = DESCRIBE_RESULT_NOTE_LEN;
taosArrayPush(pBlock->pDataBlock, &infoData);
infoData = createColumnInfoData(TSDB_DATA_TYPE_INT, tDataTypes[TSDB_DATA_TYPE_INT].bytes, 3);
blockDataAppendColInfo(pBlock, &infoData);
infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, DESCRIBE_RESULT_NOTE_LEN, 4);
blockDataAppendColInfo(pBlock, &infoData);
return pBlock;
}
@ -127,22 +115,12 @@ static int32_t execResetQueryCache() { return catalogClearCache(); }
static SSDataBlock* buildCreateDBResultDataBlock() {
SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock));
pBlock->info.numOfCols = SHOW_CREATE_DB_RESULT_COLS;
pBlock->info.hasVarCol = true;
pBlock->pDataBlock = taosArrayInit(pBlock->info.numOfCols, sizeof(SColumnInfoData));
SColumnInfoData infoData = {0};
infoData.info.type = TSDB_DATA_TYPE_VARCHAR;
infoData.info.bytes = SHOW_CREATE_DB_RESULT_FIELD1_LEN;
taosArrayPush(pBlock->pDataBlock, &infoData);
infoData.info.type = TSDB_DATA_TYPE_VARCHAR;
infoData.info.bytes = SHOW_CREATE_DB_RESULT_FIELD2_LEN;
taosArrayPush(pBlock->pDataBlock, &infoData);
SSDataBlock* pBlock = createDataBlock();
SColumnInfoData infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, SHOW_CREATE_DB_RESULT_COLS, 1);
blockDataAppendColInfo(pBlock, &infoData);
infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, SHOW_CREATE_DB_RESULT_FIELD2_LEN, 2);
blockDataAppendColInfo(pBlock, &infoData);
return pBlock;
}
@ -284,21 +262,13 @@ static int32_t execShowCreateDatabase(SShowCreateDatabaseStmt* pStmt, SRetrieveT
}
static SSDataBlock* buildCreateTbResultDataBlock() {
SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock));
pBlock->info.numOfCols = SHOW_CREATE_TB_RESULT_COLS;
pBlock->info.hasVarCol = true;
SSDataBlock* pBlock = createDataBlock();
pBlock->pDataBlock = taosArrayInit(pBlock->info.numOfCols, sizeof(SColumnInfoData));
SColumnInfoData infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, SHOW_CREATE_TB_RESULT_FIELD1_LEN, 1);
blockDataAppendColInfo(pBlock, &infoData);
SColumnInfoData infoData = {0};
infoData.info.type = TSDB_DATA_TYPE_VARCHAR;
infoData.info.bytes = SHOW_CREATE_TB_RESULT_FIELD1_LEN;
taosArrayPush(pBlock->pDataBlock, &infoData);
infoData.info.type = TSDB_DATA_TYPE_VARCHAR;
infoData.info.bytes = SHOW_CREATE_TB_RESULT_FIELD2_LEN;
taosArrayPush(pBlock->pDataBlock, &infoData);
infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, SHOW_CREATE_TB_RESULT_FIELD2_LEN, 2);
blockDataAppendColInfo(pBlock, &infoData);
return pBlock;
}
@ -554,10 +524,9 @@ static int32_t execAlterLocal(SAlterLocalStmt* pStmt) { return TSDB_CODE_FAILED;
static SSDataBlock* buildLocalVariablesResultDataBlock() {
SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock));
pBlock->info.numOfCols = SHOW_LOCAL_VARIABLES_RESULT_COLS;
pBlock->info.hasVarCol = true;
pBlock->pDataBlock = taosArrayInit(pBlock->info.numOfCols, sizeof(SColumnInfoData));
pBlock->pDataBlock = taosArrayInit(SHOW_LOCAL_VARIABLES_RESULT_COLS, sizeof(SColumnInfoData));
SColumnInfoData infoData = {0};
infoData.info.type = TSDB_DATA_TYPE_VARCHAR;

17
source/libs/command/src/explain.c
View File

@ -1190,17 +1190,12 @@ int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) {
QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR);
}
SSDataBlock *pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock));
SColumnInfoData infoData = {0};
infoData.info.type = TSDB_DATA_TYPE_VARCHAR;
infoData.info.bytes = TSDB_EXPLAIN_RESULT_ROW_SIZE;
pBlock->pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData));
taosArrayPush(pBlock->pDataBlock, &infoData);
SSDataBlock *pBlock = createDataBlock();
SColumnInfoData infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, TSDB_EXPLAIN_RESULT_ROW_SIZE, 1);
blockDataAppendColInfo(pBlock, &infoData);
blockDataEnsureCapacity(pBlock, rowNum);
SColumnInfoData* pInfoData = taosArrayGet(pBlock->pDataBlock, 0);
pInfoData->hasNull = false;
colInfoDataEnsureCapacity(pInfoData, 0, rowNum);
char buf[1024] = {0};
for (int32_t i = 0; i < rowNum; ++i) {
@ -1210,9 +1205,7 @@ int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) {
colDataAppend(pInfoData, i, buf, false);
}
pBlock->info.numOfCols = 1;
pBlock->info.rows = rowNum;
pBlock->info.hasVarCol = true;
int32_t rspSize = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock);
@ -1226,7 +1219,7 @@ int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) {
rsp->numOfRows = htonl(rowNum);
int32_t len = 0;
blockCompressEncode(pBlock, rsp->data, &len, pBlock->info.numOfCols, 0);
blockCompressEncode(pBlock, rsp->data, &len, taosArrayGetSize(pBlock->pDataBlock), 0);
ASSERT(len == rspSize - sizeof(SRetrieveTableRsp));
rsp->compLen = htonl(len);

1
source/libs/executor/inc/executorimpl.h
View File

@ -336,7 +336,6 @@ typedef struct SStreamBlockScanInfo {
int32_t numOfPseudoExpr;
int32_t primaryTsIndex; // primary time stamp slot id
void* pDataReader;
SReadHandle readHandle;
uint64_t tableUid; // queried super table uid
EStreamScanMode scanMode;

2
source/libs/executor/src/dataDeleter.c
View File

@ -75,7 +75,7 @@ static void toDataCacheEntry(SDataDeleterHandle* pHandle, const SInputData* pInp
SDataCacheEntry* pEntry = (SDataCacheEntry*)pBuf->pData;
pEntry->compressed = 0;
pEntry->numOfRows = pInput->pData->info.rows;
pEntry->numOfCols = pInput->pData->info.numOfCols;
pEntry->numOfCols = taosArrayGetSize(pInput->pData->pDataBlock);
pEntry->dataLen = sizeof(SDeleterRes);
ASSERT(1 == pEntry->numOfRows);

18
source/libs/executor/src/executil.c
View File

@ -183,34 +183,24 @@ SArray* createSortInfo(SNodeList* pNodeList) {
SSDataBlock* createResDataBlock(SDataBlockDescNode* pNode) {
int32_t numOfCols = LIST_LENGTH(pNode->pSlots);
SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock));
pBlock->pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData));
SSDataBlock* pBlock = createDataBlock();
pBlock->info.blockId = pNode->dataBlockId;
pBlock->info.rowSize = pNode->totalRowSize; // todo ??
pBlock->info.type = STREAM_INVALID;
for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData idata = {{0}};
SSlotDescNode* pDescNode = (SSlotDescNode*)nodesListGetNode(pNode->pSlots, i);
// if (!pDescNode->output) { // todo disable it temporarily
// continue;
// }
idata.info.type = pDescNode->dataType.type;
idata.info.bytes = pDescNode->dataType.bytes;
SColumnInfoData idata = createColumnInfoData(pDescNode->dataType.type, pDescNode->dataType.bytes, pDescNode->slotId);
idata.info.scale = pDescNode->dataType.scale;
idata.info.slotId = pDescNode->slotId;
idata.info.precision = pDescNode->dataType.precision;
if (IS_VAR_DATA_TYPE(idata.info.type)) {
pBlock->info.hasVarCol = true;
blockDataAppendColInfo(pBlock, &idata);
}
taosArrayPush(pBlock->pDataBlock, &idata);
}
pBlock->info.numOfCols = taosArrayGetSize(pBlock->pDataBlock);
return pBlock;
}
@ -698,7 +688,7 @@ void relocateColumnData(SSDataBlock* pBlock, const SArray* pColMatchInfo, SArray
if (p->info.colId == pmInfo->colId) {
SColumnInfoData* pDst = taosArrayGet(pBlock->pDataBlock, pmInfo->targetSlotId);
colDataAssign(pDst, p, pBlock->info.rows);
colDataAssign(pDst, p, pBlock->info.rows, &pBlock->info);
i++;
j++;
} else if (p->info.colId < pmInfo->colId) {

77
source/libs/executor/src/executorimpl.c
View File

@ -356,7 +356,7 @@ void initExecTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pQueryWindow
pColData->info.type = TSDB_DATA_TYPE_TIMESTAMP;
pColData->info.bytes = sizeof(int64_t);
colInfoDataEnsureCapacity(pColData, 0, 5);
colInfoDataEnsureCapacity(pColData, 5);
colDataAppendInt64(pColData, 0, &pQueryWindow->skey);
colDataAppendInt64(pColData, 1, &pQueryWindow->ekey);
@ -458,7 +458,7 @@ static int32_t doCreateConstantValColumnInfo(SInputColumnInfoData* pInput, SFunc
pColInfo = pInput->pData[paramIndex];
}
colInfoDataEnsureCapacity(pColInfo, 0, numOfRows);
colInfoDataEnsureCapacity(pColInfo, numOfRows);
int8_t type = pFuncParam->param.nType;
if (type == TSDB_DATA_TYPE_BIGINT || type == TSDB_DATA_TYPE_UBIGINT) {
@ -579,7 +579,7 @@ int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBloc
colDataMergeCol(pColInfoData, pResult->info.rows, &pResult->info.capacity, pfCtx->input.pData[0],
pfCtx->input.numOfRows);
} else {
colDataAssign(pColInfoData, pfCtx->input.pData[0], pfCtx->input.numOfRows);
colDataAssign(pColInfoData, pfCtx->input.pData[0], pfCtx->input.numOfRows, &pResult->info);
}
numOfRows = pfCtx->input.numOfRows;
@ -1186,7 +1186,7 @@ int32_t loadDataBlockOnDemand(SExecTaskInfo* pTaskInfo, STableScanInfo* pTableSc
// }
// if (pQueryAttr->pFilters != NULL) {
// filterSetColFieldData(pQueryAttr->pFilters, pBlock->info.numOfCols, pBlock->pDataBlock);
// filterSetColFieldData(pQueryAttr->pFilters, taosArrayGetSize(pBlock->pDataBlock), pBlock->pDataBlock);
// }
// if (pQueryAttr->pFilters != NULL || pRuntimeEnv->pTsBuf != NULL) {
@ -1321,7 +1321,8 @@ void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock) {
// todo move to the initialization function
int32_t code = filterInitFromNode((SNode*)pFilterNode, &filter, 0);
SFilterColumnParam param1 = {.numOfCols = pBlock->info.numOfCols, .pDataBlock = pBlock->pDataBlock};
size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
SFilterColumnParam param1 = {.numOfCols = numOfCols, .pDataBlock = pBlock->pDataBlock};
code = filterSetDataFromSlotId(filter, &param1);
int8_t* rowRes = NULL;
@ -1343,7 +1344,8 @@ void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const int8_t* rowR
int32_t totalRows = pBlock->info.rows;
SSDataBlock* px = createOneDataBlock(pBlock, true);
for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) {
size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pSrc = taosArrayGet(px->pDataBlock, i);
SColumnInfoData* pDst = taosArrayGet(pBlock->pDataBlock, i);
// it is a reserved column for scalar function, and no data in this column yet.
@ -2045,8 +2047,9 @@ int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLo
int32_t compLen, int32_t numOfOutput, int64_t startTs, uint64_t* total,
SArray* pColList) {
if (pColList == NULL) { // data from other sources
blockDataCleanup(pRes);
// blockDataEnsureCapacity(pRes, numOfRows);
blockCompressDecode(pRes, numOfOutput, numOfRows, pData);
pRes->info.rows = numOfRows;
} else { // extract data according to pColList
ASSERT(numOfOutput == taosArrayGetSize(pColList));
char* pStart = pData;
@ -2064,32 +2067,19 @@ int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLo
pStart += sizeof(SSysTableSchema);
}
SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock));
pBlock->pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData));
pBlock->info.numOfCols = numOfCols;
SSDataBlock* pBlock = createDataBlock();
for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData idata = {0};
idata.info.type = pSchema[i].type;
idata.info.bytes = pSchema[i].bytes;
idata.info.colId = pSchema[i].colId;
idata.hasNull = true;
taosArrayPush(pBlock->pDataBlock, &idata);
if (IS_VAR_DATA_TYPE(idata.info.type)) {
pBlock->info.hasVarCol = true;
}
SColumnInfoData idata = createColumnInfoData(pSchema[i].type, pSchema[i].bytes, pSchema[i].colId);
blockDataAppendColInfo(pBlock, &idata);
}
blockCompressDecode(pBlock, numOfCols, numOfRows, pStart);
blockDataEnsureCapacity(pRes, numOfRows);
// data from mnode
pRes->info.rows = numOfRows;
relocateColumnData(pRes, pColList, pBlock->pDataBlock, false);
taosArrayDestroy(pBlock->pDataBlock);
taosMemoryFree(pBlock);
// blockDataDestroy(pBlock);
blockDataDestroy(pBlock);
}
// todo move this to time window aggregator, since the primary timestamp may not be known by exchange operator.
@ -2420,7 +2410,7 @@ SOperatorInfo* createExchangeOperatorInfo(void* pTransporter, SExchangePhysiNode
pOperator->blocking = false;
pOperator->status = OP_NOT_OPENED;
pOperator->info = pInfo;
pOperator->exprSupp.numOfExprs = pInfo->pResult->info.numOfCols;
pOperator->exprSupp.numOfExprs = taosArrayGetSize(pInfo->pResult->pDataBlock);
pOperator->pTaskInfo = pTaskInfo;
pOperator->fpSet = createOperatorFpSet(prepareLoadRemoteData, doLoadRemoteData, NULL, NULL,
@ -2542,9 +2532,6 @@ static void doMergeImpl(SOperatorInfo* pOperator, int32_t numOfExpr, SSDataBlock
SSortedMergeOperatorInfo* pInfo = pOperator->info;
SqlFunctionCtx* pCtx = pOperator->exprSupp.pCtx;
for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) {
// pCtx[i].size = 1;
}
for (int32_t i = 0; i < pBlock->info.rows; ++i) {
if (!pInfo->hasGroupVal) {
@ -2653,7 +2640,7 @@ SSDataBlock* getSortedMergeBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlo
SColumnInfoData* pSrc = taosArrayGet(p->pDataBlock, pmInfo->srcSlotId);
SColumnInfoData* pDst = taosArrayGet(pDataBlock->pDataBlock, pmInfo->targetSlotId);
colDataAssign(pDst, pSrc, p->info.rows);
colDataAssign(pDst, pSrc, p->info.rows, &pDataBlock->info);
}
pDataBlock->info.rows = p->info.rows;
@ -3993,8 +3980,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
if (QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN == type) {
STableScanPhysiNode* pTableScanNode = (STableScanPhysiNode*)pPhyNode;
tsdbReaderT pDataReader =
doCreateDataReader(pTableScanNode, pHandle, pTableListInfo, (uint64_t)queryId, taskId);
tsdbReaderT pDataReader = doCreateDataReader(pTableScanNode, pHandle, pTableListInfo, (uint64_t)queryId, taskId);
if (pDataReader == NULL && terrno != 0) {
pTaskInfo->code = terrno;
return NULL;
@ -4020,16 +4006,20 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
STableScanInfo* pScanInfo = pOperator->info;
pTaskInfo->cost.pRecoder = &pScanInfo->readRecorder;
return pOperator;
} else if (QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN == type) {
STableMergeScanPhysiNode* pTableScanNode = (STableMergeScanPhysiNode*)pPhyNode;
createScanTableListInfo(pTableScanNode, pHandle, pTableListInfo, queryId, taskId);
extractTableSchemaVersion(pHandle, pTableScanNode->scan.uid, pTaskInfo);
SOperatorInfo* pOperator = createTableMergeScanOperatorInfo(pTableScanNode, pTableListInfo, pHandle, pTaskInfo, queryId, taskId);
SOperatorInfo* pOperator =
createTableMergeScanOperatorInfo(pTableScanNode, pTableListInfo, pHandle, pTaskInfo, queryId, taskId);
STableScanInfo* pScanInfo = pOperator->info;
pTaskInfo->cost.pRecoder = &pScanInfo->readRecorder;
return pOperator;
} else if (QUERY_NODE_PHYSICAL_PLAN_EXCHANGE == type) {
return createExchangeOperatorInfo(pHandle->pMsgCb->clientRpc, (SExchangePhysiNode*)pPhyNode, pTaskInfo);
} else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN == type) {
SScanPhysiNode* pScanPhyNode = (SScanPhysiNode*)pPhyNode; // simple child table.
STableScanPhysiNode* pTableScanNode = (STableScanPhysiNode*)pPhyNode;
@ -4044,20 +4034,22 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
if (pHandle->initTsdbReader) {
// for stream
ASSERT(pHandle->vnode);
pDataReader =
doCreateDataReader(pTableScanNode, pHandle, pTableListInfo, (uint64_t)queryId, taskId);
pDataReader = doCreateDataReader(pTableScanNode, pHandle, pTableListInfo, (uint64_t)queryId, taskId);
} else {
// for tq
ASSERT(pHandle->meta);
getTableList(pHandle->meta, pScanPhyNode, pTableListInfo);
}
}
#if 0
if (pDataReader == NULL && terrno != 0) {
qDebug("%s pDataReader is NULL", GET_TASKID(pTaskInfo));
// return NULL;
} else {
qDebug("%s pDataReader is not NULL", GET_TASKID(pTaskInfo));
}
#endif
SArray* groupKeys = extractPartitionColInfo(pTableScanNode->pPartitionTags);
int32_t code = generateGroupIdMap(pTableListInfo, pHandle, groupKeys); // todo for json
@ -4070,9 +4062,11 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
SOperatorInfo* pOperator = createStreamScanOperatorInfo(pDataReader, pHandle, pTableScanNode, pTaskInfo, &twSup);
return pOperator;
} else if (QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN == type) {
SSystemTableScanPhysiNode* pSysScanPhyNode = (SSystemTableScanPhysiNode*)pPhyNode;
return createSysTableScanOperatorInfo(pHandle, pSysScanPhyNode, pTaskInfo);
} else if (QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN == type) {
STagScanPhysiNode* pScanPhyNode = (STagScanPhysiNode*)pPhyNode;
@ -4083,6 +4077,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
}
return createTagScanOperatorInfo(pHandle, pScanPhyNode, pTableListInfo, pTaskInfo);
} else if (QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN == type) {
SBlockDistScanPhysiNode* pBlockNode = (SBlockDistScanPhysiNode*)pPhyNode;
pTableListInfo->pTableList = taosArrayInit(4, sizeof(STableKeyInfo));
@ -4356,7 +4351,7 @@ tsdbReaderT doCreateDataReader(STableScanPhysiNode* pTableScanNode, SReadHandle*
goto _error;
}
tsdbReaderT* pReader = tsdbReaderOpen(pHandle->vnode, &cond, pTableListInfo, queryId, taskId);
tsdbReaderT pReader = tsdbReaderOpen(pHandle->vnode, &cond, pTableListInfo, queryId, taskId);
cleanupQueryTableDataCond(&cond);
return pReader;
@ -4500,18 +4495,14 @@ int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SRead
(*pTaskInfo)->sql = sql;
(*pTaskInfo)->tableqinfoList.pTagCond = pPlan->pTagCond;
(*pTaskInfo)->tableqinfoList.pTagIndexCond = pPlan->pTagIndexCond;
(*pTaskInfo)->pRoot = createOperatorTree(pPlan->pNode, *pTaskInfo, pHandle, queryId, taskId,
&(*pTaskInfo)->tableqinfoList);
(*pTaskInfo)->pRoot =
createOperatorTree(pPlan->pNode, *pTaskInfo, pHandle, queryId, taskId, &(*pTaskInfo)->tableqinfoList);
if (NULL == (*pTaskInfo)->pRoot) {
code = (*pTaskInfo)->code;
goto _complete;
}
if ((*pTaskInfo)->pRoot == NULL) {
code = TSDB_CODE_QRY_OUT_OF_MEMORY;
goto _complete;
}
return code;
_complete:

4
source/libs/executor/src/groupoperator.c
View File

@ -559,8 +559,8 @@ uint64_t calcGroupId(char* pData, int32_t len) {
}
int32_t* setupColumnOffset(const SSDataBlock* pBlock, int32_t rowCapacity) {
size_t numOfCols = pBlock->info.numOfCols;
int32_t* offset = taosMemoryCalloc(pBlock->info.numOfCols, sizeof(int32_t));
size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
int32_t* offset = taosMemoryCalloc(numOfCols, sizeof(int32_t));
offset[0] = sizeof(int32_t) + sizeof(uint64_t); // the number of rows in current page, ref to SSDataBlock paged serialization format

77
source/libs/executor/src/scanoperator.c
View File

@ -203,7 +203,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca
pCost->skipBlocks += 1;
// clear all data in pBlock that are set when handing the previous block
for (int32_t i = 0; i < pBlockInfo->numOfCols; ++i) {
for (int32_t i = 0; i < taosArrayGetSize(pBlock->pDataBlock); ++i) {
SColumnInfoData* pcol = taosArrayGet(pBlock->pDataBlock, i);
pcol->pData = NULL;
}
@ -217,7 +217,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca
tsdbRetrieveDataBlockStatisInfo(pTableScanInfo->dataReader, &pColAgg, &allColumnsHaveAgg);
if (allColumnsHaveAgg == true) {
int32_t numOfCols = pBlock->info.numOfCols;
int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
// todo create this buffer during creating operator
if (pBlock->pBlockAgg == NULL) {
@ -316,7 +316,7 @@ void addTagPseudoColumnData(SReadHandle* pHandle, SExprInfo* pPseudoExpr, int32_
SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, dstSlotId);
colInfoDataEnsureCapacity(pColInfoData, 0, pBlock->info.rows);
colInfoDataEnsureCapacity(pColInfoData, pBlock->info.rows);
colInfoDataCleanup(pColInfoData, pBlock->info.rows);
int32_t functionId = pExpr->pExpr->_function.functionId;
@ -355,10 +355,8 @@ void setTbNameColData(void* pMeta, const SSDataBlock* pBlock, SColumnInfoData* p
struct SScalarFuncExecFuncs fpSet = {0};
fmGetScalarFuncExecFuncs(functionId, &fpSet);
SColumnInfoData infoData = {0};
infoData.info.type = TSDB_DATA_TYPE_BIGINT;
infoData.info.bytes = sizeof(uint64_t);
colInfoDataEnsureCapacity(&infoData, 0, 1);
SColumnInfoData infoData = createColumnInfoData(TSDB_DATA_TYPE_BIGINT, sizeof(uint64_t), 1);
colInfoDataEnsureCapacity(&infoData, 1);
colDataAppendInt64(&infoData, 0, (int64_t*)&pBlock->info.uid);
SScalarParam srcParam = {.numOfRows = pBlock->info.rows, .param = pMeta, .columnData = &infoData};
@ -786,7 +784,7 @@ static bool prepareDataScan(SStreamBlockScanInfo* pInfo) {
}
static void copyOneRow(SSDataBlock* dest, SSDataBlock* source, int32_t sourceRowId) {
for (int32_t j = 0; j < source->info.numOfCols; j++) {
for (int32_t j = 0; j < taosArrayGetSize(source->pDataBlock); j++) {
SColumnInfoData* pDestCol = (SColumnInfoData*)taosArrayGet(dest->pDataBlock, j);
SColumnInfoData* pSourceCol = (SColumnInfoData*)taosArrayGet(source->pDataBlock, j);
if (colDataIsNull_s(pSourceCol, sourceRowId)) {
@ -858,7 +856,6 @@ static void setUpdateData(SStreamBlockScanInfo* pInfo, SSDataBlock* pBlock, SSDa
SColumnInfoData* pCol = (SColumnInfoData*)taosArrayGet(pUpdateBlock->pDataBlock, pInfo->primaryTsIndex);
ASSERT(pCol->info.type == TSDB_DATA_TYPE_TIMESTAMP);
blockDataEnsureCapacity(pUpdateBlock, size);
ASSERT(pBlock->info.numOfCols == pUpdateBlock->info.numOfCols);
int32_t rowId = *(int32_t*)taosArrayGet(pInfo->tsArray, pInfo->tsArrayIndex);
pInfo->groupId = getGroupId(pInfo->pSnapshotReadOp, pBlock, rowId);
@ -969,13 +966,13 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) {
blockDataCleanup(pInfo->pRes);
while (tqNextDataBlock(pInfo->streamBlockReader)) {
SArray* pCols = NULL;
SSDataBlock block = {0};
uint64_t groupId = 0;
uint64_t uid = 0;
int32_t numOfRows = 0;
int16_t outputCol = 0;
int32_t code = tqRetrieveDataBlock(&pCols, pInfo->streamBlockReader, &groupId, &uid, &numOfRows, &outputCol);
// todo refactor
int32_t code = tqRetrieveDataBlock(&block, pInfo->streamBlockReader, &groupId, &uid, &numOfRows);
if (code != TSDB_CODE_SUCCESS || numOfRows == 0) {
pTaskInfo->code = code;
@ -986,6 +983,7 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) {
pInfo->pRes->info.rows = numOfRows;
pInfo->pRes->info.uid = uid;
pInfo->pRes->info.type = STREAM_NORMAL;
pInfo->pRes->info.capacity = numOfRows;
// for generating rollup SMA result, each time is an independent time serie.
// TODO temporarily used, when the statement of "partition by tbname" is ready, remove this
@ -1000,6 +998,7 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) {
pInfo->pRes->info.groupId = *groupIdPre;
}
// todo extract method
for (int32_t i = 0; i < taosArrayGetSize(pInfo->pColMatchInfo); ++i) {
SColMatchInfo* pColMatchInfo = taosArrayGet(pInfo->pColMatchInfo, i);
if (!pColMatchInfo->output) {
@ -1007,8 +1006,8 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) {
}
bool colExists = false;
for (int32_t j = 0; j < taosArrayGetSize(pCols); ++j) {
SColumnInfoData* pResCol = taosArrayGet(pCols, j);
for (int32_t j = 0; j < blockDataGetNumOfCols(&block); ++j) {
SColumnInfoData* pResCol = bdGetColumnInfoData(&block, j);
if (pResCol->info.colId == pColMatchInfo->colId) {
taosArraySet(pInfo->pRes->pDataBlock, pColMatchInfo->targetSlotId, pResCol);
colExists = true;
@ -1019,11 +1018,13 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) {
// the required column does not exists in submit block, let's set it to be all null value
if (!colExists) {
SColumnInfoData* pDst = taosArrayGet(pInfo->pRes->pDataBlock, pColMatchInfo->targetSlotId);
colInfoDataEnsureCapacity(pDst, 0, pBlockInfo->rows);
colDataAppendNNULL(pDst, 0, pBlockInfo->rows);
}
}
// TODO refactor @liao
taosArrayDestroy(block.pDataBlock);
if (pInfo->pRes->pDataBlock == NULL) {
// TODO add log
pOperator->status = OP_EXEC_DONE;
@ -1063,12 +1064,11 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) {
}
return (pBlockInfo->rows == 0) ? NULL : pInfo->pRes;
} else if (pInfo->blockType == STREAM_DATA_TYPE_FROM_SNAPSHOT) {
SSDataBlock* pResult = doTableScan(pInfo->pSnapshotReadOp);
if (pResult) {
return pResult->info.rows > 0 ? pResult : NULL;
}
return NULL;
return pResult && pResult->info.rows > 0 ? pResult : NULL;
} else {
ASSERT(0);
return NULL;
@ -1163,7 +1163,6 @@ SOperatorInfo* createStreamScanOperatorInfo(void* pDataReader, SReadHandle* pHan
pInfo->pRes = createResDataBlock(pDescNode);
pInfo->pUpdateRes = createResDataBlock(pDescNode);
pInfo->pCondition = pScanPhyNode->node.pConditions;
pInfo->pDataReader = pDataReader;
pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE;
pInfo->sessionSup = (SessionWindowSupporter){.pStreamAggSup = NULL, .gap = -1};
pInfo->groupId = 0;
@ -1173,7 +1172,7 @@ SOperatorInfo* createStreamScanOperatorInfo(void* pDataReader, SReadHandle* pHan
pOperator->blocking = false;
pOperator->status = OP_NOT_OPENED;
pOperator->info = pInfo;
pOperator->exprSupp.numOfExprs = pInfo->pRes->info.numOfCols;
pOperator->exprSupp.numOfExprs = taosArrayGetSize(pInfo->pRes->pDataBlock);
pOperator->pTaskInfo = pTaskInfo;
pOperator->fpSet =
@ -1301,7 +1300,7 @@ static SSDataBlock* doFilterResult(SSysTableScanInfo* pInfo) {
SColumnInfoData* pSrc = taosArrayGet(pInfo->pRes->pDataBlock, i);
if (keep) {
colDataAssign(pDest, pSrc, pInfo->pRes->info.rows);
colDataAssign(pDest, pSrc, pInfo->pRes->info.rows, &px->info);
numOfRow = pInfo->pRes->info.rows;
} else if (NULL != rowRes) {
numOfRow = 0;
@ -1331,8 +1330,6 @@ static SSDataBlock* doFilterResult(SSysTableScanInfo* pInfo) {
}
static SSDataBlock* buildSysTableMetaBlock() {
SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock));
size_t size = 0;
const SSysTableMeta* pMeta = NULL;
getInfosDbMeta(&pMeta, &size);
@ -1345,19 +1342,13 @@ static SSDataBlock* buildSysTableMetaBlock() {
}
}
pBlock->pDataBlock = taosArrayInit(pBlock->info.numOfCols, sizeof(SColumnInfoData));
SSDataBlock* pBlock = createDataBlock();
for (int32_t i = 0; i < pMeta[index].colNum; ++i) {
SColumnInfoData colInfoData = {0};
colInfoData.info.colId = i + 1;
colInfoData.info.type = pMeta[index].schema[i].type;
colInfoData.info.bytes = pMeta[index].schema[i].bytes;
taosArrayPush(pBlock->pDataBlock, &colInfoData);
SColumnInfoData colInfoData =
createColumnInfoData(pMeta[index].schema[i].type, pMeta[index].schema[i].bytes, i + 1);
blockDataAppendColInfo(pBlock, &colInfoData);
}
pBlock->info.numOfCols = pMeta[index].colNum;
pBlock->info.hasVarCol = true;
return pBlock;
}
@ -1388,7 +1379,6 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator) {
}
blockDataCleanup(pInfo->pRes);
int32_t numOfRows = 0;
const char* db = NULL;
@ -1584,7 +1574,7 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator) {
if (pRsp->numOfRows == 0 || pRsp->completed) {
pOperator->status = OP_EXEC_DONE;
qDebug("%s load meta data from mnode completed, rowsOfSource:%d, totalRows:%" PRIu64 " ", GET_TASKID(pTaskInfo),
qDebug("%s load meta data from mnode completed, rowsOfSource:%d, totalRows:%" PRIu64, GET_TASKID(pTaskInfo),
pRsp->numOfRows, pInfo->loadInfo.totalRows);
if (pRsp->numOfRows == 0) {
@ -1711,7 +1701,7 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScan
pOperator->blocking = false;
pOperator->status = OP_NOT_OPENED;
pOperator->info = pInfo;
pOperator->exprSupp.numOfExprs = pResBlock->info.numOfCols;
pOperator->exprSupp.numOfExprs = taosArrayGetSize(pResBlock->pDataBlock);
pOperator->pTaskInfo = pTaskInfo;
pOperator->fpSet =
@ -2049,6 +2039,7 @@ int32_t createMultipleDataReaders(SQueryTableDataCond* pQueryCond, SReadHandle*
return TSDB_CODE_SUCCESS;
}
// todo refactor
static int32_t loadDataBlockFromOneTable(SOperatorInfo* pOperator, STableMergeScanInfo* pTableScanInfo,
int32_t readerIdx, SSDataBlock* pBlock, uint32_t* status) {
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
@ -2079,7 +2070,7 @@ static int32_t loadDataBlockFromOneTable(SOperatorInfo* pOperator, STableMergeSc
pCost->skipBlocks += 1;
// clear all data in pBlock that are set when handing the previous block
for (int32_t i = 0; i < pBlockInfo->numOfCols; ++i) {
for (int32_t i = 0; i < taosArrayGetSize(pBlock->pDataBlock); ++i) {
SColumnInfoData* pcol = taosArrayGet(pBlock->pDataBlock, i);
pcol->pData = NULL;
}
@ -2094,7 +2085,7 @@ static int32_t loadDataBlockFromOneTable(SOperatorInfo* pOperator, STableMergeSc
tsdbRetrieveDataBlockStatisInfo(reader, &pColAgg, &allColumnsHaveAgg);
if (allColumnsHaveAgg == true) {
int32_t numOfCols = pBlock->info.numOfCols;
int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
// todo create this buffer during creating operator
if (pBlock->pBlockAgg == NULL) {
@ -2189,7 +2180,13 @@ static SSDataBlock* getTableDataBlock(void* param) {
continue;
}
tsdbRetrieveDataBlockInfo(reader, &pBlock->info);
blockDataCleanup(pBlock);
SDataBlockInfo binfo = pBlock->info;
tsdbRetrieveDataBlockInfo(reader, &binfo);
binfo.capacity = binfo.rows;
blockDataEnsureCapacity(pBlock, binfo.capacity);
pBlock->info = binfo;
uint32_t status = 0;
int32_t code = loadDataBlockFromOneTable(pOperator, pTableScanInfo, readerIdx, pBlock, &status);

23
source/libs/executor/src/sortoperator.c
View File

@ -80,7 +80,7 @@ _error:
}
void appendOneRowToDataBlock(SSDataBlock* pBlock, STupleHandle* pTupleHandle) {
for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) {
for (int32_t i = 0; i < taosArrayGetSize(pBlock->pDataBlock); ++i) {
SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i);
bool isNull = tsortIsNullVal(pTupleHandle, i);
if (isNull) {
@ -120,6 +120,9 @@ SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, i
}
if (p->info.rows > 0) {
blockDataEnsureCapacity(pDataBlock, capacity);
// todo extract function to handle this
int32_t numOfCols = taosArrayGetSize(pColMatchInfo);
for (int32_t i = 0; i < numOfCols; ++i) {
SColMatchInfo* pmInfo = taosArrayGet(pColMatchInfo, i);
@ -127,11 +130,10 @@ SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, i
SColumnInfoData* pSrc = taosArrayGet(p->pDataBlock, pmInfo->srcSlotId);
SColumnInfoData* pDst = taosArrayGet(pDataBlock->pDataBlock, pmInfo->targetSlotId);
colDataAssign(pDst, pSrc, p->info.rows);
colDataAssign(pDst, pSrc, p->info.rows, &pDataBlock->info);
}
pDataBlock->info.rows = p->info.rows;
pDataBlock->info.capacity = p->info.rows;
}
blockDataDestroy(p);
@ -257,6 +259,7 @@ typedef struct SGroupSortOperatorInfo {
SSDataBlock* getGroupSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, int32_t capacity,
SArray* pColMatchInfo, SGroupSortOperatorInfo* pInfo) {
blockDataCleanup(pDataBlock);
blockDataEnsureCapacity(pDataBlock, capacity);
SSDataBlock* p = tsortGetSortedDataBlock(pHandle);
if (p == NULL) {
@ -285,7 +288,7 @@ SSDataBlock* getGroupSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlo
SColumnInfoData* pSrc = taosArrayGet(p->pDataBlock, pmInfo->srcSlotId);
SColumnInfoData* pDst = taosArrayGet(pDataBlock->pDataBlock, pmInfo->targetSlotId);
colDataAssign(pDst, pSrc, p->info.rows);
colDataAssign(pDst, pSrc, p->info.rows, &pDataBlock->info);
}
pDataBlock->info.rows = p->info.rows;
@ -387,6 +390,10 @@ SSDataBlock* doGroupSort(SOperatorInfo* pOperator) {
pInfo->hasGroupId = true;
pInfo->prefetchedSortInput = pOperator->pDownstream[0]->fpSet.getNextFn(pOperator->pDownstream[0]);
if (pInfo->prefetchedSortInput == NULL) {
doSetOperatorCompleted(pOperator);
return NULL;
}
pInfo->currGroupId = pInfo->prefetchedSortInput->info.groupId;
pInfo->childOpStatus = CHILD_OP_NEW_GROUP;
beginSortGroup(pOperator);
@ -460,7 +467,7 @@ SOperatorInfo* createGroupSortOperatorInfo(SOperatorInfo* downstream, SGroupSort
pInfo->pColMatchInfo = pColMatchColInfo;
pOperator->name = "GroupSortOperator";
pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT;
pOperator->blocking = true;
pOperator->blocking = false;
pOperator->status = OP_NOT_OPENED;
pOperator->info = pInfo;
pOperator->exprSupp.pExprInfo = pExprInfo;
@ -564,7 +571,8 @@ SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pData
}
}
if (p->info.rows > 0) {
if (p->info.rows > 0) {// todo extract method
blockDataEnsureCapacity(pDataBlock, p->info.rows);
int32_t numOfCols = taosArrayGetSize(pColMatchInfo);
for (int32_t i = 0; i < numOfCols; ++i) {
SColMatchInfo* pmInfo = taosArrayGet(pColMatchInfo, i);
@ -572,11 +580,10 @@ SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pData
SColumnInfoData* pSrc = taosArrayGet(p->pDataBlock, pmInfo->srcSlotId);
SColumnInfoData* pDst = taosArrayGet(pDataBlock->pDataBlock, pmInfo->targetSlotId);
colDataAssign(pDst, pSrc, p->info.rows);
colDataAssign(pDst, pSrc, p->info.rows, &pDataBlock->info);
}
pDataBlock->info.rows = p->info.rows;
pDataBlock->info.capacity = p->info.rows;
pDataBlock->info.groupId = pInfo->groupId;
}

2
source/libs/executor/src/tfill.c
View File

@ -50,7 +50,7 @@ static void setTagsValue(SFillInfo* pFillInfo, void** data, int32_t genRows) {
static void setNullRow(SSDataBlock* pBlock, int32_t numOfCol, int32_t rowIndex) {
// the first are always the timestamp column, so start from the second column.
for (int32_t i = 1; i < pBlock->info.numOfCols; ++i) {
for (int32_t i = 1; i < taosArrayGetSize(pBlock->pDataBlock); ++i) {
SColumnInfoData* p = taosArrayGet(pBlock->pDataBlock, i);
colDataAppendNULL(p, rowIndex);
}

18
source/libs/executor/src/timewindowoperator.c
View File

@ -1814,7 +1814,7 @@ static int32_t initPrevRowsKeeper(STimeSliceOperatorInfo* pInfo, SSDataBlock* pB
return TSDB_CODE_OUT_OF_MEMORY;
}
int32_t numOfCols = pBlock->info.numOfCols;
int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i);
@ -2231,23 +2231,23 @@ static void clearStreamIntervalOperator(SStreamFinalIntervalOperatorInfo* pInfo)
}
static void clearUpdateDataBlock(SSDataBlock* pBlock) {
if (pBlock->info.rows <= 0) {
return;
}
blockDataCleanup(pBlock);
}
void copyUpdateDataBlock(SSDataBlock* pDest, SSDataBlock* pSource, int32_t tsColIndex) {
ASSERT(pDest->info.capacity >= pSource->info.rows);
// ASSERT(pDest->info.capacity >= pSource->info.rows);
blockDataEnsureCapacity(pDest, pSource->info.rows);
clearUpdateDataBlock(pDest);
SColumnInfoData* pDestCol = taosArrayGet(pDest->pDataBlock, 0);
SColumnInfoData* pSourceCol = taosArrayGet(pSource->pDataBlock, tsColIndex);
// copy timestamp column
colDataAssign(pDestCol, pSourceCol, pSource->info.rows);
for (int32_t i = 1; i < pDest->info.numOfCols; i++) {
colDataAssign(pDestCol, pSourceCol, pSource->info.rows, &pDest->info);
for (int32_t i = 1; i < taosArrayGetSize(pDest->pDataBlock); i++) {
SColumnInfoData* pCol = taosArrayGet(pDest->pDataBlock, i);
colDataAppendNNULL(pCol, 0, pSource->info.rows);
}
pDest->info.rows = pSource->info.rows;
pDest->info.groupId = pSource->info.groupId;
pDest->info.type = pSource->info.type;
@ -2917,7 +2917,7 @@ void doBuildDeleteDataBlock(SHashObj* pStDeleted, SSDataBlock* pBlock, void** It
while (((*Ite) = taosHashIterate(pStDeleted, *Ite)) != NULL) {
SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, 0);
colDataAppend(pColInfoData, pBlock->info.rows, *Ite, false);
for (int32_t i = 1; i < pBlock->info.numOfCols; i++) {
for (int32_t i = 1; i < taosArrayGetSize(pBlock->pDataBlock); i++) {
pColInfoData = taosArrayGet(pBlock->pDataBlock, i);
colDataAppendNULL(pColInfoData, pBlock->info.rows);
}
@ -4117,6 +4117,8 @@ SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SExprI
goto _error;
}
miaInfo->groupIntervalHash = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), true, HASH_NO_LOCK);
SIntervalAggOperatorInfo* iaInfo = &miaInfo->intervalAggOperatorInfo;
iaInfo->win = pTaskInfo->window;

8
source/libs/executor/src/tsort.c
View File

@ -147,7 +147,7 @@ static int32_t doAddNewExternalMemSource(SDiskbasedBuf *pBuf, SArray* pAllSource
int32_t rowSize = blockDataGetSerialRowSize(pSource->src.pBlock);
// The value of numOfRows must be greater than 0, which is guaranteed by the previous memory allocation
int32_t numOfRows = (getBufPageSize(pBuf) - blockDataGetSerialMetaSize(pBlock->info.numOfCols))/rowSize;
int32_t numOfRows = (getBufPageSize(pBuf) - blockDataGetSerialMetaSize(taosArrayGetSize(pBlock->pDataBlock)))/rowSize;
ASSERT(numOfRows > 0);
return blockDataEnsureCapacity(pSource->src.pBlock, numOfRows);
}
@ -178,7 +178,7 @@ static int32_t doAddToBuf(SSDataBlock* pDataBlock, SSortHandle* pHandle) {
return terrno;
}
int32_t size = blockDataGetSize(p) + sizeof(int32_t) + p->info.numOfCols * sizeof(int32_t);
int32_t size = blockDataGetSize(p) + sizeof(int32_t) + taosArrayGetSize(p->pDataBlock) * sizeof(int32_t);
assert(size <= getBufPageSize(pHandle->pBuf));
blockDataToBuf(pPage, p);
@ -252,7 +252,7 @@ static int32_t sortComparInit(SMsortComparParam* cmpParam, SArray* pSources, int
}
static void appendOneRowToDataBlock(SSDataBlock *pBlock, const SSDataBlock* pSource, int32_t* rowIndex) {
for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) {
for (int32_t i = 0; i < taosArrayGetSize(pBlock->pDataBlock); ++i) {
SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i);
SColumnInfoData* pSrcColInfo = taosArrayGet(pSource->pDataBlock, i);
@ -500,7 +500,7 @@ static int32_t doInternalMergeSort(SSortHandle* pHandle) {
return terrno;
}
int32_t size = blockDataGetSize(pDataBlock) + sizeof(int32_t) + pDataBlock->info.numOfCols * sizeof(int32_t);
int32_t size = blockDataGetSize(pDataBlock) + sizeof(int32_t) + taosArrayGetSize(pDataBlock->pDataBlock) * sizeof(int32_t);
assert(size <= getBufPageSize(pHandle->pBuf));
blockDataToBuf(pPage, pDataBlock);

42
source/libs/executor/test/executorTests.cpp
View File

@ -62,18 +62,11 @@ SSDataBlock* getDummyBlock(SOperatorInfo* pOperator) {
}
if (pInfo->pBlock == NULL) {
pInfo->pBlock = static_cast<SSDataBlock*>(taosMemoryCalloc(1, sizeof(SSDataBlock)));
pInfo->pBlock = createDataBlock();
pInfo->pBlock->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData));
SColumnInfoData colInfo = {0};
colInfo.info.type = TSDB_DATA_TYPE_INT;
colInfo.info.bytes = sizeof(int32_t);
colInfo.info.colId = 1;
colInfo.pData = static_cast<char*>(taosMemoryCalloc(pInfo->numOfRowsPerPage, sizeof(int32_t)));
colInfo.nullbitmap = static_cast<char*>(taosMemoryCalloc(1, (pInfo->numOfRowsPerPage + 7) / 8));
taosArrayPush(pInfo->pBlock->pDataBlock, &colInfo);
SColumnInfoData colInfo = createColumnInfoData(TSDB_DATA_TYPE_INT, sizeof(int32_t), 1);
blockDataAppendColInfo(pInfo->pBlock, &colInfo);
blockDataEnsureCapacity(pInfo->pBlock, pInfo->numOfRowsPerPage);
// SColumnInfoData colInfo1 = {0};
// colInfo1.info.type = TSDB_DATA_TYPE_BINARY;
@ -115,7 +108,6 @@ SSDataBlock* getDummyBlock(SOperatorInfo* pOperator) {
}
pBlock->info.rows = pInfo->numOfRowsPerPage;
pBlock->info.numOfCols = 1;
pInfo->current += 1;
return pBlock;
@ -128,28 +120,15 @@ SSDataBlock* get2ColsDummyBlock(SOperatorInfo* pOperator) {
}
if (pInfo->pBlock == NULL) {
pInfo->pBlock = static_cast<SSDataBlock*>(taosMemoryCalloc(1, sizeof(SSDataBlock)));
pInfo->pBlock = createDataBlock();
pInfo->pBlock->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData));
SColumnInfoData colInfo = createColumnInfoData(TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), 1);
blockDataAppendColInfo(pInfo->pBlock, &colInfo);
SColumnInfoData colInfo = {0};
colInfo.info.type = TSDB_DATA_TYPE_TIMESTAMP;
colInfo.info.bytes = sizeof(int64_t);
colInfo.info.colId = 1;
colInfo.pData = static_cast<char*>(taosMemoryCalloc(pInfo->numOfRowsPerPage, sizeof(int64_t)));
// colInfo.nullbitmap = static_cast<char*>(taosMemoryCalloc(1, (pInfo->numOfRowsPerPage + 7) / 8));
SColumnInfoData colInfo1 = createColumnInfoData(TSDB_DATA_TYPE_INT, 4, 2);
blockDataAppendColInfo(pInfo->pBlock, &colInfo1);
taosArrayPush(pInfo->pBlock->pDataBlock, &colInfo);
SColumnInfoData colInfo1 = {0};
colInfo1.info.type = TSDB_DATA_TYPE_INT;
colInfo1.info.bytes = 4;
colInfo1.info.colId = 2;
colInfo1.pData = static_cast<char*>(taosMemoryCalloc(pInfo->numOfRowsPerPage, sizeof(int32_t)));
colInfo1.nullbitmap = static_cast<char*>(taosMemoryCalloc(1, (pInfo->numOfRowsPerPage + 7) / 8));
taosArrayPush(pInfo->pBlock->pDataBlock, &colInfo1);
blockDataEnsureCapacity(pInfo->pBlock, pInfo->numOfRowsPerPage);
} else {
blockDataCleanup(pInfo->pBlock);
}
@ -185,7 +164,6 @@ SSDataBlock* get2ColsDummyBlock(SOperatorInfo* pOperator) {
}
pBlock->info.rows = pInfo->numOfRowsPerPage;
pBlock->info.numOfCols = 1;
pInfo->current += 1;

24
source/libs/executor/test/sortTests.cpp
View File

@ -62,25 +62,21 @@ SSDataBlock* getSingleColDummyBlock(void* param) {
return NULL;
}
SSDataBlock* pBlock = static_cast<SSDataBlock*>(taosMemoryCalloc(1, sizeof(SSDataBlock)));
pBlock->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData));
SSDataBlock* pBlock = createDataBlock();
SColumnInfoData colInfo = {0};
colInfo.info.type = pInfo->type;
if (pInfo->type == TSDB_DATA_TYPE_NCHAR){
colInfo.info.bytes = TSDB_NCHAR_SIZE * VARCOUNT + VARSTR_HEADER_SIZE;
colInfo.varmeta.offset = static_cast<int32_t *>(taosMemoryCalloc(pInfo->pageRows, sizeof(int32_t)));
} else if(pInfo->type == TSDB_DATA_TYPE_BINARY) {
colInfo.info.bytes = VARCOUNT + VARSTR_HEADER_SIZE;
colInfo.varmeta.offset = static_cast<int32_t *>(taosMemoryCalloc(pInfo->pageRows, sizeof(int32_t)));
} else{
colInfo.info.bytes = tDataTypes[pInfo->type].bytes;
colInfo.pData = static_cast<char*>(taosMemoryCalloc(pInfo->pageRows, colInfo.info.bytes));
colInfo.nullbitmap = static_cast<char*>(taosMemoryCalloc(1, (pInfo->pageRows + 7) / 8));
}
colInfo.info.colId = 1;
taosArrayPush(pBlock->pDataBlock, &colInfo);
blockDataAppendColInfo(pBlock, &colInfo);
blockDataEnsureCapacity(pBlock, pInfo->pageRows);
for (int32_t i = 0; i < pInfo->pageRows; ++i) {
SColumnInfoData* pColInfo = static_cast<SColumnInfoData*>(TARRAY_GET_ELEM(pBlock->pDataBlock, 0));
@ -128,7 +124,6 @@ SSDataBlock* getSingleColDummyBlock(void* param) {
}
pBlock->info.rows = pInfo->pageRows;
pBlock->info.numOfCols = 1;
return pBlock;
}
@ -354,15 +349,10 @@ TEST(testCase, ordered_merge_sort_Test) {
SArray* orderInfo = taosArrayInit(1, sizeof(SBlockOrderInfo));
taosArrayPush(orderInfo, &oi);
SSDataBlock* pBlock = static_cast<SSDataBlock*>(taosMemoryCalloc(1, sizeof(SSDataBlock)));
pBlock->pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData));
pBlock->info.numOfCols = 1;
for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) {
SColumnInfoData colInfo = {0};
colInfo.info.type = TSDB_DATA_TYPE_INT;
colInfo.info.bytes = sizeof(int32_t);
colInfo.info.colId = 1;
taosArrayPush(pBlock->pDataBlock, &colInfo);
SSDataBlock* pBlock = createDataBlock();
for (int32_t i = 0; i < 1; ++i) {
SColumnInfoData colInfo = createColumnInfoData(TSDB_DATA_TYPE_INT, sizeof(int32_t), 1);
blockDataAppendColInfo(pBlock, &colInfo);
}
SSortHandle* phandle = tsortCreateSortHandle(orderInfo, SORT_MULTISOURCE_MERGE, 1024, 5, pBlock,"test_abc");

13
source/libs/function/inc/taggfunction.h
View File

@ -28,19 +28,6 @@ extern "C" {
#include "function.h"
#include "tudf.h"
extern SAggFunctionInfo aggFunc[35];
#define FUNCSTATE_SO 0x0u
#define FUNCSTATE_MO 0x1u // dynamic number of output, not multinumber of output e.g., TOP/BOTTOM
#define FUNCSTATE_STREAM 0x2u // function avail for stream
#define FUNCSTATE_STABLE 0x4u // function avail for super table
#define FUNCSTATE_NEED_TS 0x8u // timestamp is required during query processing
#define FUNCSTATE_SELECTIVITY 0x10u // selectivity functions, can exists along with tag columns
#define BASIC_FUNC_SO FUNCSTATE_SO | FUNCSTATE_STREAM | FUNCSTATE_STABLE
#define BASIC_FUNC_MO FUNCSTATE_MO | FUNCSTATE_STREAM | FUNCSTATE_STABLE
#define AVG_FUNCTION_INTER_BUFFER_SIZE 50
#define DATA_SET_FLAG ',' // to denote the output area has data, not null value

32
source/libs/function/src/builtins.c
View File

@ -1267,6 +1267,29 @@ static bool validateMinuteRange(int8_t hour, int8_t minute, char sign) {
return false;
}
static bool validateTimestampDigits(const SValueNode* pVal) {
if (!IS_INTEGER_TYPE(pVal->node.resType.type)) {
return false;
}
int64_t tsVal = pVal->datum.i;
char fraction[20] = {0};
NUM_TO_STRING(pVal->node.resType.type, &tsVal, sizeof(fraction), fraction);
int32_t tsDigits = (int32_t)strlen(fraction);
if (tsDigits > TSDB_TIME_PRECISION_SEC_DIGITS) {
if (tsDigits == TSDB_TIME_PRECISION_MILLI_DIGITS ||
tsDigits == TSDB_TIME_PRECISION_MICRO_DIGITS ||
tsDigits == TSDB_TIME_PRECISION_NANO_DIGITS) {
return true;
} else {
return false;
}
}
return true;
}
static bool validateTimezoneFormat(const SValueNode* pVal) {
if (TSDB_DATA_TYPE_BINARY != pVal->node.resType.type) {
return false;
@ -1379,6 +1402,15 @@ static int32_t translateToIso8601(SFunctionNode* pFunc, char* pErrBuf, int32_t l
return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
}
if (QUERY_NODE_VALUE == nodeType(nodesListGetNode(pFunc->pParameterList, 0))) {
SValueNode* pValue = (SValueNode*)nodesListGetNode(pFunc->pParameterList, 0);
if (!validateTimestampDigits(pValue)) {
pFunc->node.resType = (SDataType){.bytes = 0, .type = TSDB_DATA_TYPE_BINARY};
return TSDB_CODE_SUCCESS;
}
}
// param1
if (numOfParams == 2) {
SValueNode* pValue = (SValueNode*)nodesListGetNode(pFunc->pParameterList, 1);

25
source/libs/function/src/builtinsimpl.c
View File

@ -561,7 +561,7 @@ int32_t sumFunction(SqlFunctionCtx* pCtx) {
// check for overflow
if (IS_FLOAT_TYPE(type) && (isinf(pSumRes->dsum) || isnan(pSumRes->dsum))) {
GET_RES_INFO(pCtx)->isNullRes = 1;
numOfElem = 0;
}
_sum_over:
@ -927,7 +927,7 @@ int32_t avgFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
// check for overflow
if (isinf(pAvgRes->result) || isnan(pAvgRes->result)) {
GET_RES_INFO(pCtx)->isNullRes = 1;
GET_RES_INFO(pCtx)->numOfRes = 0;
}
return functionFinalize(pCtx, pBlock);
@ -1449,11 +1449,13 @@ void setNullSelectivityValue(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t
void setSelectivityValue(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, const STuplePos* pTuplePos, int32_t rowIndex) {
int32_t pageId = pTuplePos->pageId;
int32_t offset = pTuplePos->offset;
if (pTuplePos->pageId != -1) {
int32_t numOfCols = taosArrayGetSize(pCtx->pSrcBlock->pDataBlock);
SFilePage* pPage = getBufPage(pCtx->pBuf, pageId);
bool* nullList = (bool*)((char*)pPage + offset);
char* pStart = (char*)(nullList + pCtx->pSrcBlock->info.numOfCols * sizeof(bool));
char* pStart = (char*)(nullList + numOfCols * sizeof(bool));
// todo set the offset value to optimize the performance.
for (int32_t j = 0; j < pCtx->subsidiaries.num; ++j) {
@ -1774,6 +1776,11 @@ int32_t stddevFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
pStddevRes->result = sqrt(fabs(pStddevRes->quadraticDSum / ((double)pStddevRes->count) - avg * avg));
}
// check for overflow
if (isinf(pStddevRes->result) || isnan(pStddevRes->result)) {
GET_RES_INFO(pCtx)->numOfRes = 0;
}
return functionFinalize(pCtx, pBlock);
}
@ -3038,7 +3045,7 @@ void doAddIntoResult(SqlFunctionCtx* pCtx, void* pData, int32_t rowIndex, SSData
void saveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STuplePos* pPos) {
SFilePage* pPage = NULL;
int32_t completeRowSize = pSrcBlock->info.rowSize + pSrcBlock->info.numOfCols * sizeof(bool);
int32_t completeRowSize = pSrcBlock->info.rowSize + (int32_t) taosArrayGetSize(pSrcBlock->pDataBlock) * sizeof(bool);
if (pCtx->curBufPage == -1) {
pPage = getNewBufPage(pCtx->pBuf, 0, &pCtx->curBufPage);
@ -3056,8 +3063,8 @@ void saveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pS
// keep the current row data, extract method
int32_t offset = 0;
bool* nullList = (bool*)((char*)pPage + pPage->num);
char* pStart = (char*)(nullList + sizeof(bool) * pSrcBlock->info.numOfCols);
for (int32_t i = 0; i < pSrcBlock->info.numOfCols; ++i) {
char* pStart = (char*)(nullList + sizeof(bool) * (int32_t) taosArrayGetSize(pSrcBlock->pDataBlock));
for (int32_t i = 0; i < (int32_t) taosArrayGetSize(pSrcBlock->pDataBlock); ++i) {
SColumnInfoData* pCol = taosArrayGet(pSrcBlock->pDataBlock, i);
bool isNull = colDataIsNull_s(pCol, rowIndex);
if (isNull) {
@ -3086,11 +3093,13 @@ void saveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pS
void copyTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STuplePos* pPos) {
SFilePage* pPage = getBufPage(pCtx->pBuf, pPos->pageId);
int32_t numOfCols = taosArrayGetSize(pSrcBlock->pDataBlock);
bool* nullList = (bool*)((char*)pPage + pPos->offset);
char* pStart = (char*)(nullList + pSrcBlock->info.numOfCols * sizeof(bool));
char* pStart = (char*)(nullList + numOfCols * sizeof(bool));
int32_t offset = 0;
for (int32_t i = 0; i < pSrcBlock->info.numOfCols; ++i) {
for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pCol = taosArrayGet(pSrcBlock->pDataBlock, i);
if ((nullList[i] = colDataIsNull_s(pCol, rowIndex)) == true) {
offset += pCol->info.bytes;

45
source/libs/function/src/tudf.c
View File

@ -763,8 +763,8 @@ void freeUdfInterBuf(SUdfInterBuf *buf) {
int32_t convertDataBlockToUdfDataBlock(SSDataBlock *block, SUdfDataBlock *udfBlock) {
udfBlock->numOfRows = block->info.rows;
udfBlock->numOfCols = block->info.numOfCols;
udfBlock->udfCols = taosMemoryCalloc(udfBlock->numOfCols, sizeof(SUdfColumn*));
udfBlock->numOfCols = taosArrayGetSize(block->pDataBlock);
udfBlock->udfCols = taosMemoryCalloc(taosArrayGetSize(block->pDataBlock), sizeof(SUdfColumn*));
for (int32_t i = 0; i < udfBlock->numOfCols; ++i) {
udfBlock->udfCols[i] = taosMemoryCalloc(1, sizeof(SUdfColumn));
SColumnInfoData *col= (SColumnInfoData*)taosArrayGet(block->pDataBlock, i);
@ -799,7 +799,6 @@ int32_t convertDataBlockToUdfDataBlock(SSDataBlock *block, SUdfDataBlock *udfBlo
}
int32_t convertUdfColumnToDataBlock(SUdfColumn *udfCol, SSDataBlock *block) {
block->info.numOfCols = 1;
block->info.rows = udfCol->colData.numOfRows;
block->info.hasVarCol = IS_VAR_DATA_TYPE(udfCol->colMeta.type);
@ -830,25 +829,19 @@ int32_t convertUdfColumnToDataBlock(SUdfColumn *udfCol, SSDataBlock *block) {
int32_t convertScalarParamToDataBlock(SScalarParam *input, int32_t numOfCols, SSDataBlock *output) {
output->info.rows = input->numOfRows;
output->info.numOfCols = numOfCols;
bool hasVarCol = false;
for (int32_t i = 0; i < numOfCols; ++i) {
if (IS_VAR_DATA_TYPE((input+i)->columnData->info.type)) {
hasVarCol = true;
break;
}
}
output->info.hasVarCol = hasVarCol;
output->pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData));
for (int32_t i = 0; i < numOfCols; ++i) {
taosArrayPush(output->pDataBlock, (input + i)->columnData);
if (IS_VAR_DATA_TYPE((input+i)->columnData->info.type)) {
output->info.hasVarCol = true;
}
}
return 0;
}
int32_t convertDataBlockToScalarParm(SSDataBlock *input, SScalarParam *output) {
if (input->info.numOfCols != 1) {
if (taosArrayGetSize(input->pDataBlock) != 1) {
fnError("scalar function only support one column");
return -1;
}
@ -1089,24 +1082,14 @@ int32_t udfAggProcess(struct SqlFunctionCtx *pCtx) {
int32_t start = pInput->startRowIndex;
int32_t numOfRows = pInput->numOfRows;
SSDataBlock tempBlock = {0};
tempBlock.info.numOfCols = numOfCols;
tempBlock.info.rows = pInput->totalRows;
tempBlock.info.uid = pInput->uid;
bool hasVarCol = false;
tempBlock.pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData));
SSDataBlock* pTempBlock = createDataBlock();
pTempBlock->info.rows = pInput->totalRows;
pTempBlock->info.uid = pInput->uid;
for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData *col = pInput->pData[i];
if (IS_VAR_DATA_TYPE(col->info.type)) {
hasVarCol = true;
blockDataAppendColInfo(pTempBlock, pInput->pData[i]);
}
taosArrayPush(tempBlock.pDataBlock, col);
}
tempBlock.info.hasVarCol = hasVarCol;
SSDataBlock *inputBlock = blockDataExtractBlock(&tempBlock, start, numOfRows);
SSDataBlock *inputBlock = blockDataExtractBlock(pTempBlock, start, numOfRows);
SUdfInterBuf state = {.buf = udfRes->interResBuf,
.bufLen = session->bufSize,
@ -1131,7 +1114,9 @@ int32_t udfAggProcess(struct SqlFunctionCtx *pCtx) {
}
blockDataDestroy(inputBlock);
taosArrayDestroy(tempBlock.pDataBlock);
taosArrayDestroy(pTempBlock->pDataBlock);
taosMemoryFree(pTempBlock);
releaseUdfFuncHandle(pCtx->udfName);
freeUdfInterBuf(&newState);

3
source/libs/function/src/udfd.c
View File

@ -613,7 +613,9 @@ int32_t udfdOpenClientRpc() {
}
int32_t udfdCloseClientRpc() {
fnInfo("udfd begin closing rpc");
rpcClose(global.clientRpc);
fnInfo("udfd finish closing rpc");
return 0;
}
@ -939,7 +941,6 @@ int main(int argc, char *argv[]) {
udfdRun();
removeListeningPipe();
uv_thread_join(&mnodeConnectThread);
udfdCloseClientRpc();
return 0;

50
source/libs/function/test/runUdf.c
View File

@ -45,27 +45,27 @@ int scalarFuncTest() {
for (int k = 0; k < 1; ++k) {
SSDataBlock block = {0};
SSDataBlock *pBlock = &block;
pBlock->pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData));
pBlock->info.numOfCols = 1;
pBlock->info.rows = 1024;
for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) {
SColumnInfoData colInfo = {0};
colInfo.info.type = TSDB_DATA_TYPE_INT;
colInfo.info.bytes = sizeof(int32_t);
colInfo.info.colId = 1;
colInfoDataEnsureCapacity(&colInfo, 0, pBlock->info.rows);
for (int32_t j = 0; j < pBlock->info.rows; ++j) {
colDataAppendInt32(&colInfo, j, &j);
for (int32_t i = 0; i < 1; ++i) {
SColumnInfoData colInfo = createColumnInfoData(TSDB_DATA_TYPE_INT, sizeof(int32_t), 1);
blockDataAppendColInfo(pBlock, &colInfo);
}
taosArrayPush(pBlock->pDataBlock, &colInfo);
blockDataEnsureCapacity(pBlock, 1024);
pBlock->info.rows = 1024;
SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, 0);
for (int32_t j = 0; j < pBlock->info.rows; ++j) {
colDataAppendInt32(pCol, j, &j);
}
SScalarParam input = {0};
input.numOfRows = pBlock->info.rows;
input.columnData = taosArrayGet(pBlock->pDataBlock, 0);
SScalarParam output = {0};
doCallUdfScalarFunc(handle, &input, 1, &output);
taosArrayDestroy(pBlock->pDataBlock);
SColumnInfoData *col = output.columnData;
for (int32_t i = 0; i < output.numOfRows; ++i) {
if (i % 100 == 0)
@ -89,21 +89,18 @@ int aggregateFuncTest() {
return -1;
}
SSDataBlock block = {0};
SSDataBlock *pBlock = &block;
pBlock->pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData));
pBlock->info.numOfCols = 1;
pBlock->info.rows = 1024;
for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) {
SColumnInfoData colInfo = {0};
colInfo.info.type = TSDB_DATA_TYPE_INT;
colInfo.info.bytes = sizeof(int32_t);
colInfo.info.colId = 1;
colInfoDataEnsureCapacity(&colInfo, 0, pBlock->info.rows);
for (int32_t j = 0; j < pBlock->info.rows; ++j) {
colDataAppendInt32(&colInfo, j, &j);
SSDataBlock *pBlock = createDataBlock();
for (int32_t i = 0; i < taosArrayGetSize(pBlock->pDataBlock); ++i) {
SColumnInfoData colInfo = createColumnInfoData(TSDB_DATA_TYPE_INT, sizeof(int32_t), 1);
blockDataAppendColInfo(pBlock, &colInfo);
}
taosArrayPush(pBlock->pDataBlock, &colInfo);
blockDataEnsureCapacity(pBlock, 1024);
pBlock->info.rows = 1024;
SColumnInfoData* pColInfo = bdGetColumnInfoData(pBlock, 0);
for (int32_t j = 0; j < pBlock->info.rows; ++j) {
colDataAppendInt32(pColInfo, j, &j);
}
SUdfInterBuf buf = {0};
@ -121,6 +118,7 @@ int aggregateFuncTest() {
freeUdfInterBuf(&resultBuf);
doTeardownUdf(handle);
blockDataDestroy(pBlock);
return 0;
}

2
source/libs/index/test/index_executor_tests.cpp
View File

@ -86,7 +86,7 @@ void sifAppendReservedSlot(SArray *pBlockList, int16_t *dataBlockId, int16_t *sl
SColumnInfoData idata = {0};
idata.info = *colInfo;
colInfoDataEnsureCapacity(&idata, 0, rows);
colInfoDataEnsureCapacity(&idata, rows);
taosArrayPush(res->pDataBlock, &idata);

25
source/libs/monitor/src/monMain.c
View File

@ -201,6 +201,9 @@ static void monGenClusterJson(SMonInfo *pMonitor) {
tjsonAddStringToObject(pJson, "version", pInfo->version);
tjsonAddDoubleToObject(pJson, "master_uptime", pInfo->master_uptime);
tjsonAddDoubleToObject(pJson, "monitor_interval", pInfo->monitor_interval);
tjsonAddDoubleToObject(pJson, "dbs_total", pInfo->dbs_total);
tjsonAddDoubleToObject(pJson, "tbs_total", pInfo->tbs_total);
tjsonAddDoubleToObject(pJson, "stbs_total", pInfo->stbs_total);
tjsonAddDoubleToObject(pJson, "vgroups_total", pInfo->vgroups_total);
tjsonAddDoubleToObject(pJson, "vgroups_alive", pInfo->vgroups_alive);
tjsonAddDoubleToObject(pJson, "vnodes_total", pInfo->vnodes_total);
@ -277,6 +280,27 @@ static void monGenVgroupJson(SMonInfo *pMonitor) {
}
}
static void monGenStbJson(SMonInfo *pMonitor) {
SMonStbInfo *pInfo = &pMonitor->mmInfo.stb;
if (pMonitor->mmInfo.cluster.first_ep_dnode_id == 0) return;
SJson *pJson = tjsonAddArrayToObject(pMonitor->pJson, "stb_infos");
if (pJson == NULL) return;
for (int32_t i = 0; i < taosArrayGetSize(pInfo->stbs); ++i) {
SJson *pStbJson = tjsonCreateObject();
if (pStbJson == NULL) continue;
if (tjsonAddItemToArray(pJson, pStbJson) != 0) {
tjsonDelete(pStbJson);
continue;
}
SMonStbDesc *pStbDesc = taosArrayGet(pInfo->stbs, i);
tjsonAddStringToObject(pStbJson, "stb_name", pStbDesc->stb_name);
tjsonAddStringToObject(pStbJson, "database_name", pStbDesc->database_name);
}
}
static void monGenGrantJson(SMonInfo *pMonitor) {
SMonGrantInfo *pInfo = &pMonitor->mmInfo.grant;
if (pMonitor->mmInfo.cluster.first_ep_dnode_id == 0) return;
@ -524,6 +548,7 @@ void monSendReport() {
monGenBasicJson(pMonitor);
monGenClusterJson(pMonitor);
monGenVgroupJson(pMonitor);
monGenStbJson(pMonitor);
monGenGrantJson(pMonitor);
monGenDnodeJson(pMonitor);
monGenDiskJson(pMonitor);

36
source/libs/monitor/src/monMsg.c
View File

@ -102,6 +102,9 @@ int32_t tEncodeSMonClusterInfo(SEncoder *encoder, const SMonClusterInfo *pInfo)
if (tEncodeCStr(encoder, pInfo->version) < 0) return -1;
if (tEncodeFloat(encoder, pInfo->master_uptime) < 0) return -1;
if (tEncodeI32(encoder, pInfo->monitor_interval) < 0) return -1;
if (tEncodeI32(encoder, pInfo->dbs_total) < 0) return -1;
if (tEncodeI32(encoder, pInfo->stbs_total) < 0) return -1;
if (tEncodeI64(encoder, pInfo->tbs_total) < 0) return -1;
if (tEncodeI32(encoder, pInfo->vgroups_total) < 0) return -1;
if (tEncodeI32(encoder, pInfo->vgroups_alive) < 0) return -1;
if (tEncodeI32(encoder, pInfo->vnodes_total) < 0) return -1;
@ -130,6 +133,9 @@ int32_t tDecodeSMonClusterInfo(SDecoder *decoder, SMonClusterInfo *pInfo) {
if (tDecodeCStrTo(decoder, pInfo->version) < 0) return -1;
if (tDecodeFloat(decoder, &pInfo->master_uptime) < 0) return -1;
if (tDecodeI32(decoder, &pInfo->monitor_interval) < 0) return -1;
if (tDecodeI32(decoder, &pInfo->dbs_total) < 0) return -1;
if (tDecodeI32(decoder, &pInfo->stbs_total) < 0) return -1;
if (tDecodeI64(decoder, &pInfo->tbs_total) < 0) return -1;
if (tDecodeI32(decoder, &pInfo->vgroups_total) < 0) return -1;
if (tDecodeI32(decoder, &pInfo->vgroups_alive) < 0) return -1;
if (tDecodeI32(decoder, &pInfo->vnodes_total) < 0) return -1;
@ -203,6 +209,32 @@ int32_t tDecodeSMonVgroupInfo(SDecoder *decoder, SMonVgroupInfo *pInfo) {
return 0;
}
int32_t tEncodeSMonStbInfo(SEncoder *encoder, const SMonStbInfo *pInfo) {
if (tEncodeI32(encoder, taosArrayGetSize(pInfo->stbs)) < 0) return -1;
for (int32_t i = 0; i < taosArrayGetSize(pInfo->stbs); ++i) {
SMonStbDesc *pDesc = taosArrayGet(pInfo->stbs, i);
if (tEncodeCStr(encoder, pDesc->stb_name) < 0) return -1;
if (tEncodeCStr(encoder, pDesc->database_name) < 0) return -1;
}
return 0;
}
int32_t tDecodeSMonStbInfo(SDecoder *decoder, SMonStbInfo *pInfo) {
int32_t arraySize = 0;
if (tDecodeI32(decoder, &arraySize) < 0) return -1;
pInfo->stbs = taosArrayInit(arraySize, sizeof(SMonStbDesc));
if (pInfo->stbs == NULL) return -1;
for (int32_t i = 0; i < arraySize; ++i) {
SMonStbDesc desc = {0};
if (tDecodeCStrTo(decoder, desc.stb_name) < 0) return -1;
if (tDecodeCStrTo(decoder, desc.database_name) < 0) return -1;
taosArrayPush(pInfo->stbs, &desc);
}
return 0;
}
int32_t tEncodeSMonGrantInfo(SEncoder *encoder, const SMonGrantInfo *pInfo) {
if (tEncodeI32(encoder, pInfo->expire_time) < 0) return -1;
if (tEncodeI64(encoder, pInfo->timeseries_used) < 0) return -1;
@ -224,6 +256,7 @@ int32_t tSerializeSMonMmInfo(void *buf, int32_t bufLen, SMonMmInfo *pInfo) {
if (tStartEncode(&encoder) < 0) return -1;
if (tEncodeSMonClusterInfo(&encoder, &pInfo->cluster) < 0) return -1;
if (tEncodeSMonVgroupInfo(&encoder, &pInfo->vgroup) < 0) return -1;
if (tEncodeSMonStbInfo(&encoder, &pInfo->stb) < 0) return -1;
if (tEncodeSMonGrantInfo(&encoder, &pInfo->grant) < 0) return -1;
if (tEncodeSMonSysInfo(&encoder, &pInfo->sys) < 0) return -1;
if (tEncodeSMonLogs(&encoder, &pInfo->log) < 0) return -1;
@ -241,6 +274,7 @@ int32_t tDeserializeSMonMmInfo(void *buf, int32_t bufLen, SMonMmInfo *pInfo) {
if (tStartDecode(&decoder) < 0) return -1;
if (tDecodeSMonClusterInfo(&decoder, &pInfo->cluster) < 0) return -1;
if (tDecodeSMonVgroupInfo(&decoder, &pInfo->vgroup) < 0) return -1;
if (tDecodeSMonStbInfo(&decoder, &pInfo->stb) < 0) return -1;
if (tDecodeSMonGrantInfo(&decoder, &pInfo->grant) < 0) return -1;
if (tDecodeSMonSysInfo(&decoder, &pInfo->sys) < 0) return -1;
if (tDecodeSMonLogs(&decoder, &pInfo->log) < 0) return -1;
@ -255,9 +289,11 @@ void tFreeSMonMmInfo(SMonMmInfo *pInfo) {
taosArrayDestroy(pInfo->cluster.mnodes);
taosArrayDestroy(pInfo->cluster.dnodes);
taosArrayDestroy(pInfo->vgroup.vgroups);
taosArrayDestroy(pInfo->stb.stbs);
pInfo->cluster.mnodes = NULL;
pInfo->cluster.dnodes = NULL;
pInfo->vgroup.vgroups = NULL;
pInfo->stb.stbs = NULL;
pInfo->log.logs = NULL;
}

2
source/libs/scalar/inc/sclInt.h
View File

@ -57,7 +57,7 @@ typedef struct SScalarCtx {
#define SCL_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _return; } } while (0)
int32_t doConvertDataType(SValueNode* pValueNode, SScalarParam* out);
SColumnInfoData* createColumnInfoData(SDataType* pType, int32_t numOfRows);
SColumnInfoData* sclCreateColumnInfoData(SDataType* pType, int32_t numOfRows);
int32_t sclConvertToTsValueNode(int8_t precision, SValueNode* valueNode);
#define GET_PARAM_TYPE(_c) ((_c)->columnData ? (_c)->columnData->info.type : (_c)->hashValueType)

2
source/libs/scalar/src/filter.c
View File

@ -3821,7 +3821,7 @@ bool filterExecute(SFilterInfo *info, SSDataBlock *pSrc, int8_t** p, SColumnData
SScalarParam output = {0};
SDataType type = {.type = TSDB_DATA_TYPE_BOOL, .bytes = sizeof(bool)};
output.columnData = createColumnInfoData(&type, pSrc->info.rows);
output.columnData = sclCreateColumnInfoData(&type, pSrc->info.rows);
SArray *pList = taosArrayInit(1, POINTER_BYTES);
taosArrayPush(pList, &pSrc);

23
source/libs/scalar/src/scalar.c
View File

@ -36,7 +36,7 @@ int32_t sclConvertToTsValueNode(int8_t precision, SValueNode* valueNode) {
}
SColumnInfoData* createColumnInfoData(SDataType* pType, int32_t numOfRows) {
SColumnInfoData* sclCreateColumnInfoData(SDataType* pType, int32_t numOfRows) {
SColumnInfoData* pColumnData = taosMemoryCalloc(1, sizeof(SColumnInfoData));
if (pColumnData == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
@ -48,7 +48,7 @@ SColumnInfoData* createColumnInfoData(SDataType* pType, int32_t numOfRows) {
pColumnData->info.scale = pType->scale;
pColumnData->info.precision = pType->precision;
int32_t code = colInfoDataEnsureCapacity(pColumnData, 0, numOfRows);
int32_t code = colInfoDataEnsureCapacity(pColumnData, numOfRows);
if (code != TSDB_CODE_SUCCESS) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
taosMemoryFree(pColumnData);
@ -60,10 +60,10 @@ SColumnInfoData* createColumnInfoData(SDataType* pType, int32_t numOfRows) {
int32_t doConvertDataType(SValueNode* pValueNode, SScalarParam* out) {
SScalarParam in = {.numOfRows = 1};
in.columnData = createColumnInfoData(&pValueNode->node.resType, 1);
in.columnData = sclCreateColumnInfoData(&pValueNode->node.resType, 1);
colDataAppend(in.columnData, 0, nodesGetValueFromNode(pValueNode), false);
colInfoDataEnsureCapacity(out->columnData, 0, 1);
colInfoDataEnsureCapacity(out->columnData, 1);
int32_t code = vectorConvertImpl(&in, out);
sclFreeParam(&in);
@ -193,7 +193,7 @@ int32_t sclInitParam(SNode* node, SScalarParam *param, SScalarCtx *ctx, int32_t
SValueNode *valueNode = (SValueNode *)node;
param->numOfRows = 1;
param->columnData = createColumnInfoData(&valueNode->node.resType, 1);
param->columnData = sclCreateColumnInfoData(&valueNode->node.resType, 1);
if (TSDB_DATA_TYPE_NULL == valueNode->node.resType.type) {
colDataAppendNULL(param->columnData, 0);
} else {
@ -245,7 +245,7 @@ int32_t sclInitParam(SNode* node, SScalarParam *param, SScalarCtx *ctx, int32_t
}
SSDataBlock *block = *(SSDataBlock **)taosArrayGet(ctx->pBlockList, index);
if (NULL == block || ref->slotId >= block->info.numOfCols) {
if (NULL == block || ref->slotId >= taosArrayGetSize(block->pDataBlock)) {
sclError("column slotId is too big, slodId:%d, dataBlockSize:%d", ref->slotId, (int32_t)taosArrayGetSize(block->pDataBlock));
SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
}
@ -431,7 +431,7 @@ int32_t sclExecFunction(SFunctionNode *node, SScalarCtx *ctx, SScalarParam *outp
SCL_ERR_JRET(code);
}
output->columnData = createColumnInfoData(&node->node.resType, rowNum);
output->columnData = sclCreateColumnInfoData(&node->node.resType, rowNum);
if (output->columnData == NULL) {
sclError("calloc %d failed", (int32_t)(rowNum * output->columnData->info.bytes));
SCL_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY);
@ -484,7 +484,7 @@ int32_t sclExecLogic(SLogicConditionNode *node, SScalarCtx *ctx, SScalarParam *o
output->numOfRows = rowNum;
SDataType t = {.type = type, .bytes = tDataTypes[type].bytes};
output->columnData = createColumnInfoData(&t, rowNum);
output->columnData = sclCreateColumnInfoData(&t, rowNum);
if (output->columnData == NULL) {
sclError("calloc %d failed", (int32_t)(rowNum * sizeof(bool)));
SCL_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY);
@ -539,7 +539,7 @@ int32_t sclExecOperator(SOperatorNode *node, SScalarCtx *ctx, SScalarParam *outp
int32_t code = 0;
SCL_ERR_RET(sclInitOperatorParams(&params, node, ctx, &rowNum));
output->columnData = createColumnInfoData(&node->node.resType, rowNum);
output->columnData = sclCreateColumnInfoData(&node->node.resType, rowNum);
if (output->columnData == NULL) {
sclError("calloc failed, size:%d", (int32_t)rowNum * node->node.resType.bytes);
SCL_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY);
@ -901,7 +901,7 @@ EDealRes sclWalkTarget(SNode* pNode, SScalarCtx *ctx) {
return DEAL_RES_ERROR;
}
colDataAssign(col, res->columnData, res->numOfRows);
colDataAssign(col, res->columnData, res->numOfRows, NULL);
block->info.rows = res->numOfRows;
sclFreeParam(res);
@ -983,7 +983,8 @@ int32_t scalarCalculate(SNode *pNode, SArray *pBlockList, SScalarParam *pDst) {
SCL_ERR_JRET(TSDB_CODE_QRY_APP_ERROR);
}
colDataAssign(pDst->columnData, res->columnData, res->numOfRows);
colInfoDataEnsureCapacity(pDst->columnData, res->numOfRows);
colDataAssign(pDst->columnData, res->columnData, res->numOfRows, NULL);
pDst->numOfRows = res->numOfRows;
taosHashRemove(ctx.pRes, (void *)&pNode, POINTER_BYTES);
}

7
source/libs/scalar/src/sclfunc.c
View File

@ -120,7 +120,7 @@ int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutpu
}
default: {
colDataAssign(pOutputData, pInputData, pInput->numOfRows);
colDataAssign(pOutputData, pInputData, pInput->numOfRows, NULL);
}
}
@ -273,7 +273,7 @@ static int32_t doScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarP
}
default: {
colDataAssign(pOutputData, pInputData, pInput->numOfRows);
colDataAssign(pOutputData, pInputData, pInput->numOfRows, NULL);
}
}
@ -856,10 +856,13 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
int32_t toISO8601Function(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
int32_t type = GET_PARAM_TYPE(pInput);
bool tzPresent = (inputNum == 2) ? true : false;
char* tz;
int32_t tzLen;
if (tzPresent) {
tz = varDataVal(pInput[1].columnData->pData);
tzLen = varDataLen(pInput[1].columnData->pData);
}
for (int32_t i = 0; i < pInput[0].numOfRows; ++i) {
if (colDataIsNull_s(pInput[0].columnData, i)) {

4
source/libs/scalar/src/sclvector.c
View File

@ -852,7 +852,7 @@ int32_t vectorConvertScalarParam(SScalarParam *input, SScalarParam *output, int3
SDataType t = {.type = type, .bytes = tDataTypes[type].bytes};
output->numOfRows = input->numOfRows;
output->columnData = createColumnInfoData(&t, input->numOfRows);
output->columnData = sclCreateColumnInfoData(&t, input->numOfRows);
if (output->columnData == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
@ -923,7 +923,7 @@ static int32_t doConvertHelper(SScalarParam* pDest, int32_t* convert, const SSca
pDest->numOfRows = pParam->numOfRows;
SDataType t = {.type = type, .bytes = tDataTypes[type].bytes};
pDest->columnData = createColumnInfoData(&t, pParam->numOfRows);
pDest->columnData = sclCreateColumnInfoData(&t, pParam->numOfRows);
if (pDest->columnData == NULL) {
sclError("malloc %d failed", (int32_t)(pParam->numOfRows * sizeof(double)));
return TSDB_CODE_OUT_OF_MEMORY;

116
source/libs/scalar/test/filter/filterTests.cpp
View File

@ -105,29 +105,14 @@ void flttMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in
}
if (NULL == *block) {
SSDataBlock *res = (SSDataBlock *)taosMemoryCalloc(1, sizeof(SSDataBlock));
res->info.numOfCols = 3;
res->info.rows = rowNum;
res->pDataBlock = taosArrayInit(3, sizeof(SColumnInfoData));
SSDataBlock *res = createDataBlock();
for (int32_t i = 0; i < 2; ++i) {
SColumnInfoData idata = {{0}};
idata.info.type = TSDB_DATA_TYPE_NULL;
idata.info.bytes = 10;
idata.info.colId = i + 1;
int32_t size = idata.info.bytes * rowNum;
idata.pData = (char *)taosMemoryCalloc(1, size);
taosArrayPush(res->pDataBlock, &idata);
SColumnInfoData idata = createColumnInfoData(TSDB_DATA_TYPE_NULL, 10, 1+i);
blockDataAppendColInfo(res, &idata);
}
SColumnInfoData idata = {{0}};
idata.info.type = dataType;
idata.info.bytes = dataBytes;
idata.info.colId = 3;
int32_t size = idata.info.bytes * rowNum;
idata.pData = (char *)taosMemoryCalloc(1, size);
taosArrayPush(res->pDataBlock, &idata);
SColumnInfoData idata = createColumnInfoData(dataType, dataBytes, 3);
blockDataAppendColInfo(res, &idata);
blockDataEnsureCapacity(res, rowNum);
SColumnInfoData *pColumn = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock);
@ -142,23 +127,18 @@ void flttMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in
rnode->slotId = 2;
rnode->colId = 3;
res->info.rows = rowNum;
*block = res;
} else {
SSDataBlock *res = *block;
int32_t idx = taosArrayGetSize(res->pDataBlock);
SColumnInfoData idata = {{0}};
idata.info.type = dataType;
idata.info.bytes = dataBytes;
idata.info.colId = 1 + idx;
int32_t size = idata.info.bytes * rowNum;
idata.pData = (char *)taosMemoryCalloc(1, size);
taosArrayPush(res->pDataBlock, &idata);
res->info.numOfCols++;
SColumnInfoData *pColumn = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock);
SColumnInfoData idata = createColumnInfoData(dataType, dataBytes, 1+idx);
blockDataAppendColInfo(res, &idata);
blockDataEnsureCapacity(res, rowNum);
colInfoDataEnsureCapacity(pColumn, 0, rowNum);
SColumnInfoData *pColumn = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock);
for (int32_t i = 0; i < rowNum; ++i) {
colDataAppend(pColumn, i, (const char *)value, false);
@ -399,7 +379,7 @@ TEST(columnTest, smallint_column_greater_double_value) {
keep = filterRangeExecute(filter, &stat, 1, rowNum);
ASSERT_EQ(keep, true);
SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock };
SFilterColumnParam param = { (int32_t) taosArrayGetSize(src->pDataBlock), src->pDataBlock };
code = filterSetDataFromSlotId(filter, &param);
ASSERT_EQ(code, 0);
@ -407,7 +387,7 @@ TEST(columnTest, smallint_column_greater_double_value) {
stat.min = 1;
stat.numOfNull = 0;
int8_t *rowRes = NULL;
keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols);
keep = filterExecute(filter, src, &rowRes, &stat, (int32_t) taosArrayGetSize(src->pDataBlock));
ASSERT_EQ(keep, false);
for (int32_t i = 0; i < rowNum; ++i) {
@ -454,7 +434,7 @@ TEST(columnTest, int_column_greater_smallint_value) {
keep = filterRangeExecute(filter, &stat, 1, rowNum);
ASSERT_EQ(keep, false);
SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock };
SFilterColumnParam param = { (int32_t) taosArrayGetSize(src->pDataBlock), src->pDataBlock };
code = filterSetDataFromSlotId(filter, &param);
ASSERT_EQ(code, 0);
@ -462,7 +442,7 @@ TEST(columnTest, int_column_greater_smallint_value) {
stat.min = 1;
stat.numOfNull = 0;
int8_t *rowRes = NULL;
keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols);
keep = filterExecute(filter, src, &rowRes, &stat, (int32_t) taosArrayGetSize(src->pDataBlock));
ASSERT_EQ(keep, false);
for (int32_t i = 0; i < rowNum; ++i) {
@ -498,7 +478,7 @@ TEST(columnTest, int_column_in_double_list) {
ASSERT_EQ(code, 0);
SColumnDataAgg stat = {0};
SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock };
SFilterColumnParam param = { (int32_t) taosArrayGetSize(src->pDataBlock), src->pDataBlock };
code = filterSetDataFromSlotId(filter, &param);
ASSERT_EQ(code, 0);
@ -506,7 +486,7 @@ TEST(columnTest, int_column_in_double_list) {
stat.min = 1;
stat.numOfNull = 0;
int8_t *rowRes = NULL;
bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols);
bool keep = filterExecute(filter, src, &rowRes, &stat, (int32_t) taosArrayGetSize(src->pDataBlock));
ASSERT_EQ(keep, false);
for (int32_t i = 0; i < rowNum; ++i) {
@ -562,7 +542,7 @@ TEST(columnTest, binary_column_in_binary_list) {
ASSERT_EQ(code, 0);
SColumnDataAgg stat = {0};
SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock };
SFilterColumnParam param = { (int32_t) taosArrayGetSize(src->pDataBlock), src->pDataBlock };
code = filterSetDataFromSlotId(filter, &param);
ASSERT_EQ(code, 0);
@ -570,7 +550,7 @@ TEST(columnTest, binary_column_in_binary_list) {
stat.min = 1;
stat.numOfNull = 0;
int8_t *rowRes = NULL;
bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols);
bool keep = filterExecute(filter, src, &rowRes, &stat, (int32_t) taosArrayGetSize(src->pDataBlock));
ASSERT_EQ(keep, false);
for (int32_t i = 0; i < rowNum; ++i) {
@ -610,7 +590,7 @@ TEST(columnTest, binary_column_like_binary) {
ASSERT_EQ(code, 0);
SColumnDataAgg stat = {0};
SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock };
SFilterColumnParam param = { (int32_t) taosArrayGetSize(src->pDataBlock), src->pDataBlock };
code = filterSetDataFromSlotId(filter, &param);
ASSERT_EQ(code, 0);
@ -618,7 +598,7 @@ TEST(columnTest, binary_column_like_binary) {
stat.min = 1;
stat.numOfNull = 0;
int8_t *rowRes = NULL;
bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols);
bool keep = filterExecute(filter, src, &rowRes, &stat, (int32_t) taosArrayGetSize(src->pDataBlock));
ASSERT_EQ(keep, false);
for (int32_t i = 0; i < rowNum; ++i) {
@ -657,7 +637,7 @@ TEST(columnTest, binary_column_is_null) {
ASSERT_EQ(code, 0);
SColumnDataAgg stat = {0};
SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock };
SFilterColumnParam param = { (int32_t) taosArrayGetSize(src->pDataBlock), src->pDataBlock };
code = filterSetDataFromSlotId(filter, &param);
ASSERT_EQ(code, 0);
@ -665,7 +645,7 @@ TEST(columnTest, binary_column_is_null) {
stat.min = 1;
stat.numOfNull = 0;
int8_t *rowRes = NULL;
bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols);
bool keep = filterExecute(filter, src, &rowRes, &stat, (int32_t) taosArrayGetSize(src->pDataBlock));
ASSERT_EQ(keep, false);
for (int32_t i = 0; i < rowNum; ++i) {
@ -704,7 +684,7 @@ TEST(columnTest, binary_column_is_not_null) {
ASSERT_EQ(code, 0);
SColumnDataAgg stat = {0};
SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock };
SFilterColumnParam param = { (int32_t) taosArrayGetSize(src->pDataBlock), src->pDataBlock };
code = filterSetDataFromSlotId(filter, &param);
ASSERT_EQ(code, 0);
@ -712,7 +692,7 @@ TEST(columnTest, binary_column_is_not_null) {
stat.min = 1;
stat.numOfNull = 0;
int8_t *rowRes = NULL;
bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols);
bool keep = filterExecute(filter, src, &rowRes, &stat, (int32_t) taosArrayGetSize(src->pDataBlock));
ASSERT_EQ(keep, false);
for (int32_t i = 0; i < rowNum; ++i) {
@ -741,7 +721,7 @@ TEST(opTest, smallint_column_greater_int_column) {
ASSERT_EQ(code, 0);
SColumnDataAgg stat = {0};
SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock };
SFilterColumnParam param = { (int32_t) taosArrayGetSize(src->pDataBlock), src->pDataBlock };
code = filterSetDataFromSlotId(filter, &param);
ASSERT_EQ(code, 0);
@ -749,7 +729,7 @@ TEST(opTest, smallint_column_greater_int_column) {
stat.min = 1;
stat.numOfNull = 0;
int8_t *rowRes = NULL;
bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols);
bool keep = filterExecute(filter, src, &rowRes, &stat, (int32_t) taosArrayGetSize(src->pDataBlock));
ASSERT_EQ(keep, false);
for (int32_t i = 0; i < rowNum; ++i) {
@ -779,7 +759,7 @@ TEST(opTest, smallint_value_add_int_column) {
ASSERT_EQ(code, 0);
SColumnDataAgg stat = {0};
SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock };
SFilterColumnParam param = { (int32_t) taosArrayGetSize(src->pDataBlock), src->pDataBlock };
code = filterSetDataFromSlotId(filter, &param);
ASSERT_EQ(code, 0);
@ -787,7 +767,7 @@ TEST(opTest, smallint_value_add_int_column) {
stat.min = 1;
stat.numOfNull = 0;
int8_t *rowRes = NULL;
bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols);
bool keep = filterExecute(filter, src, &rowRes, &stat, (int32_t) taosArrayGetSize(src->pDataBlock));
ASSERT_EQ(keep, false);
for (int32_t i = 0; i < rowNum; ++i) {
@ -822,7 +802,7 @@ TEST(opTest, bigint_column_multi_binary_column) {
ASSERT_EQ(code, 0);
SColumnDataAgg stat = {0};
SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock };
SFilterColumnParam param = { (int32_t) taosArrayGetSize(src->pDataBlock), src->pDataBlock };
code = filterSetDataFromSlotId(filter, &param);
ASSERT_EQ(code, 0);
@ -830,7 +810,7 @@ TEST(opTest, bigint_column_multi_binary_column) {
stat.min = 1;
stat.numOfNull = 0;
int8_t *rowRes = NULL;
bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols);
bool keep = filterExecute(filter, src, &rowRes, &stat, (int32_t) taosArrayGetSize(src->pDataBlock));
ASSERT_EQ(keep, false);
for (int32_t i = 0; i < rowNum; ++i) {
@ -865,7 +845,7 @@ TEST(opTest, smallint_column_and_binary_column) {
ASSERT_EQ(code, 0);
SColumnDataAgg stat = {0};
SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock };
SFilterColumnParam param = { (int32_t) taosArrayGetSize(src->pDataBlock), src->pDataBlock };
code = filterSetDataFromSlotId(filter, &param);
ASSERT_EQ(code, 0);
@ -873,7 +853,7 @@ TEST(opTest, smallint_column_and_binary_column) {
stat.min = 1;
stat.numOfNull = 0;
int8_t *rowRes = NULL;
bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols);
bool keep = filterExecute(filter, src, &rowRes, &stat, (int32_t) taosArrayGetSize(src->pDataBlock));
ASSERT_EQ(keep, false);
for (int32_t i = 0; i < rowNum; ++i) {
@ -903,7 +883,7 @@ TEST(opTest, smallint_column_or_float_column) {
ASSERT_EQ(code, 0);
SColumnDataAgg stat = {0};
SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock };
SFilterColumnParam param = { (int32_t)taosArrayGetSize(src->pDataBlock), src->pDataBlock };
code = filterSetDataFromSlotId(filter, &param);
ASSERT_EQ(code, 0);
@ -911,7 +891,7 @@ TEST(opTest, smallint_column_or_float_column) {
stat.min = 1;
stat.numOfNull = 0;
int8_t *rowRes = NULL;
bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols);
bool keep = filterExecute(filter, src, &rowRes, &stat, taosArrayGetSize(src->pDataBlock));
ASSERT_EQ(keep, false);
for (int32_t i = 0; i < rowNum; ++i) {
@ -941,7 +921,7 @@ TEST(opTest, smallint_column_or_double_value) {
ASSERT_EQ(code, 0);
SColumnDataAgg stat = {0};
SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock };
SFilterColumnParam param = { (int32_t)taosArrayGetSize(src->pDataBlock), src->pDataBlock };
code = filterSetDataFromSlotId(filter, &param);
ASSERT_EQ(code, 0);
@ -949,7 +929,7 @@ TEST(opTest, smallint_column_or_double_value) {
stat.min = 1;
stat.numOfNull = 0;
int8_t *rowRes = NULL;
bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols);
bool keep = filterExecute(filter, src, &rowRes, &stat, taosArrayGetSize(src->pDataBlock));
ASSERT_EQ(keep, true);
for (int32_t i = 0; i < rowNum; ++i) {
@ -985,7 +965,7 @@ TEST(opTest, binary_column_is_true) {
ASSERT_EQ(code, 0);
SColumnDataAgg stat = {0};
SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock };
SFilterColumnParam param = { (int32_t)taosArrayGetSize(src->pDataBlock), src->pDataBlock };
code = filterSetDataFromSlotId(filter, &param);
ASSERT_EQ(code, 0);
@ -993,7 +973,7 @@ TEST(opTest, binary_column_is_true) {
stat.min = 1;
stat.numOfNull = 0;
int8_t *rowRes = NULL;
bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols);
bool keep = filterExecute(filter, src, &rowRes, &stat, taosArrayGetSize(src->pDataBlock));
ASSERT_EQ(keep, false);
for (int32_t i = 0; i < rowNum; ++i) {
@ -1056,7 +1036,7 @@ TEST(filterModelogicTest, diff_columns_and_or_and) {
ASSERT_EQ(code, 0);
SColumnDataAgg stat = {0};
SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock };
SFilterColumnParam param = { (int32_t)taosArrayGetSize(src->pDataBlock), src->pDataBlock };
code = filterSetDataFromSlotId(filter, &param);
ASSERT_EQ(code, 0);
@ -1064,7 +1044,7 @@ TEST(filterModelogicTest, diff_columns_and_or_and) {
stat.min = 1;
stat.numOfNull = 0;
int8_t *rowRes = NULL;
bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols);
bool keep = filterExecute(filter, src, &rowRes, &stat, taosArrayGetSize(src->pDataBlock));
ASSERT_EQ(keep, false);
for (int32_t i = 0; i < rowNum; ++i) {
@ -1125,7 +1105,7 @@ TEST(filterModelogicTest, same_column_and_or_and) {
ASSERT_EQ(code, 0);
SColumnDataAgg stat = {0};
SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock };
SFilterColumnParam param = { (int32_t)taosArrayGetSize(src->pDataBlock), src->pDataBlock };
code = filterSetDataFromSlotId(filter, &param);
ASSERT_EQ(code, 0);
@ -1133,7 +1113,7 @@ TEST(filterModelogicTest, same_column_and_or_and) {
stat.min = 1;
stat.numOfNull = 0;
int8_t *rowRes = NULL;
bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols);
bool keep = filterExecute(filter, src, &rowRes, &stat, taosArrayGetSize(src->pDataBlock));
ASSERT_EQ(keep, false);
for (int32_t i = 0; i < rowNum; ++i) {
@ -1194,7 +1174,7 @@ TEST(filterModelogicTest, diff_columns_or_and_or) {
ASSERT_EQ(code, 0);
SColumnDataAgg stat = {0};
SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock };
SFilterColumnParam param = { (int32_t) taosArrayGetSize(src->pDataBlock), src->pDataBlock };
code = filterSetDataFromSlotId(filter, &param);
ASSERT_EQ(code, 0);
@ -1202,7 +1182,7 @@ TEST(filterModelogicTest, diff_columns_or_and_or) {
stat.min = 1;
stat.numOfNull = 0;
int8_t *rowRes = NULL;
bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols);
bool keep = filterExecute(filter, src, &rowRes, &stat, taosArrayGetSize(src->pDataBlock));
ASSERT_EQ(keep, false);
for (int32_t i = 0; i < rowNum; ++i) {
@ -1263,7 +1243,7 @@ TEST(filterModelogicTest, same_column_or_and_or) {
ASSERT_EQ(code, 0);
SColumnDataAgg stat = {0};
SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock };
SFilterColumnParam param = { (int32_t)taosArrayGetSize(src->pDataBlock), src->pDataBlock };
code = filterSetDataFromSlotId(filter, &param);
ASSERT_EQ(code, 0);
@ -1271,7 +1251,7 @@ TEST(filterModelogicTest, same_column_or_and_or) {
stat.min = 1;
stat.numOfNull = 0;
int8_t *rowRes = NULL;
bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols);
bool keep = filterExecute(filter, src, &rowRes, &stat, taosArrayGetSize(src->pDataBlock));
ASSERT_EQ(keep, false);
for (int32_t i = 0; i < rowNum; ++i) {
@ -1334,7 +1314,7 @@ TEST(scalarModelogicTest, diff_columns_or_and_or) {
ASSERT_EQ(code, 0);
SColumnDataAgg stat = {0};
SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock };
SFilterColumnParam param = { (int32_t)taosArrayGetSize(src->pDataBlock), src->pDataBlock };
code = filterSetDataFromSlotId(filter, &param);
ASSERT_EQ(code, 0);
@ -1342,7 +1322,7 @@ TEST(scalarModelogicTest, diff_columns_or_and_or) {
stat.min = 1;
stat.numOfNull = 0;
int8_t *rowRes = NULL;
bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols);
bool keep = filterExecute(filter, src, &rowRes, &stat, taosArrayGetSize(src->pDataBlock));
ASSERT_EQ(keep, false);
for (int32_t i = 0; i < rowNum; ++i) {

77
source/libs/scalar/test/scalar/scalarTests.cpp
View File

@ -84,30 +84,29 @@ void scltInitLogFile() {
void scltAppendReservedSlot(SArray *pBlockList, int16_t *dataBlockId, int16_t *slotId, bool newBlock, int32_t rows, SColumnInfo *colInfo) {
if (newBlock) {
SSDataBlock *res = (SSDataBlock *)taosMemoryCalloc(1, sizeof(SSDataBlock));
res->info.numOfCols = 1;
res->info.rows = rows;
res->pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData));
SSDataBlock *res = createDataBlock();
SColumnInfoData idata = {0};
idata.info = *colInfo;
colInfoDataEnsureCapacity(&idata, rows);
blockDataAppendColInfo(res, &idata);
res->info.capacity = rows;
res->info.rows = rows;
SColumnInfoData* p = static_cast<SColumnInfoData *>(taosArrayGet(res->pDataBlock, 0));
ASSERT(p->pData != NULL && p->nullbitmap != NULL);
taosArrayPush(res->pDataBlock, &idata);
taosArrayPush(pBlockList, &res);
blockDataEnsureCapacity(res, rows);
*dataBlockId = taosArrayGetSize(pBlockList) - 1;
res->info.blockId = *dataBlockId;
*slotId = 0;
} else {
SSDataBlock *res = *(SSDataBlock **)taosArrayGetLast(pBlockList);
res->info.numOfCols++;
SColumnInfoData idata = {0};
idata.info = *colInfo;
colInfoDataEnsureCapacity(&idata, 0, rows);
taosArrayPush(res->pDataBlock, &idata);
colInfoDataEnsureCapacity(&idata, rows);
blockDataAppendColInfo(res, &idata);
*dataBlockId = taosArrayGetSize(pBlockList) - 1;
*slotId = taosArrayGetSize(res->pDataBlock) - 1;
@ -144,29 +143,19 @@ void scltMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in
}
if (NULL == *block) {
SSDataBlock *res = (SSDataBlock *)taosMemoryCalloc(1, sizeof(SSDataBlock));
res->info.numOfCols = 3;
res->info.rows = rowNum;
res->pDataBlock = taosArrayInit(3, sizeof(SColumnInfoData));
SSDataBlock *res = createDataBlock();
for (int32_t i = 0; i < 2; ++i) {
SColumnInfoData idata = {{0}};
idata.info.type = TSDB_DATA_TYPE_NULL;
idata.info.bytes = 10;
idata.info.colId = i + 1;
int32_t size = idata.info.bytes * rowNum;
idata.pData = (char *)taosMemoryCalloc(1, size);
taosArrayPush(res->pDataBlock, &idata);
SColumnInfoData idata = createColumnInfoData(TSDB_DATA_TYPE_INT, 10, i + 1);
colInfoDataEnsureCapacity(&idata, rowNum);
blockDataAppendColInfo(res, &idata);
}
SColumnInfoData idata = {{0}};
idata.info.type = dataType;
idata.info.bytes = dataBytes;
idata.info.colId = 3;
int32_t size = idata.info.bytes * rowNum;
idata.pData = (char *)taosMemoryCalloc(1, size);
colInfoDataEnsureCapacity(&idata, 0, rowNum);
taosArrayPush(res->pDataBlock, &idata);
SColumnInfoData idata = createColumnInfoData(dataType, dataBytes, 3);
colInfoDataEnsureCapacity(&idata, rowNum);
blockDataAppendColInfo(res, &idata);
res->info.capacity = rowNum;
res->info.rows = rowNum;
SColumnInfoData *pColumn = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock);
for (int32_t i = 0; i < rowNum; ++i) {
colDataAppend(pColumn, i, (const char *)value, false);
@ -185,17 +174,13 @@ void scltMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in
SSDataBlock *res = *block;
int32_t idx = taosArrayGetSize(res->pDataBlock);
SColumnInfoData idata = {{0}};
idata.info.type = dataType;
idata.info.bytes = dataBytes;
idata.info.colId = 1 + idx;
int32_t size = idata.info.bytes * rowNum;
idata.pData = (char *)taosMemoryCalloc(1, size);
taosArrayPush(res->pDataBlock, &idata);
res->info.numOfCols++;
SColumnInfoData *pColumn = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock);
SColumnInfoData idata = createColumnInfoData(dataType, dataBytes, 1 + idx);
colInfoDataEnsureCapacity(&idata, rowNum);
colInfoDataEnsureCapacity(pColumn, 0, rowNum);
res->info.capacity = rowNum;
blockDataAppendColInfo(res, &idata);
SColumnInfoData *pColumn = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock);
for (int32_t i = 0; i < rowNum; ++i) {
colDataAppend(pColumn, i, (const char *)value, false);
@ -279,9 +264,6 @@ void scltMakeTargetNode(SNode **pNode, int16_t dataBlockId, int16_t slotId, SNod
*pNode = (SNode *)onode;
}
}
TEST(constantTest, bigint_add_bigint) {
@ -1529,6 +1511,7 @@ TEST(columnTest, bigint_column_multi_binary_column) {
SArray *blockList = taosArrayInit(1, POINTER_BYTES);
taosArrayPush(blockList, &src);
SColumnInfo colInfo = createColumnInfo(1, TSDB_DATA_TYPE_DOUBLE, sizeof(double));
int16_t dataBlockId = 0, slotId = 0;
scltAppendReservedSlot(blockList, &dataBlockId, &slotId, false, rowNum, &colInfo);
@ -2033,7 +2016,7 @@ void scltMakeDataBlock(SScalarParam **pInput, int32_t type, void *pVal, int32_t
input->numOfRows = num;
input->columnData->info = createColumnInfo(0, type, bytes);
colInfoDataEnsureCapacity(input->columnData, 0, num);
colInfoDataEnsureCapacity(input->columnData, num);
if (setVal) {
for (int32_t i = 0; i < num; ++i) {

2
source/libs/stream/inc/streamInc.h
View File

@ -34,7 +34,7 @@ int32_t streamExec(SStreamTask* pTask, SMsgCb* pMsgCb);
int32_t streamDispatch(SStreamTask* pTask, SMsgCb* pMsgCb);
int32_t streamDispatchReqToData(const SStreamDispatchReq* pReq, SStreamDataBlock* pData);
int32_t streamRetrieveReqToData(const SStreamRetrieveReq* pReq, SStreamDataBlock* pData);
int32_t streamBuildDispatchMsg(SStreamTask* pTask, SStreamDataBlock* data, SRpcMsg* pMsg, SEpSet** ppEpSet);
int32_t streamBuildDispatchMsg(SStreamTask* pTask, const SStreamDataBlock* data, SRpcMsg* pMsg, SEpSet** ppEpSet);
int32_t streamBroadcastToChildren(SStreamTask* pTask, const SSDataBlock* pBlock);

15
source/libs/stream/src/streamDispatch.c
View File

@ -96,16 +96,17 @@ int32_t streamBroadcastToChildren(SStreamTask* pTask, const SSDataBlock* pBlock)
pRetrieve = taosMemoryCalloc(1, dataStrLen);
if (pRetrieve == NULL) return -1;
int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
pRetrieve->useconds = 0;
pRetrieve->precision = TSDB_DEFAULT_PRECISION;
pRetrieve->compressed = 0;
pRetrieve->completed = 1;
pRetrieve->streamBlockType = pBlock->info.type;
pRetrieve->numOfRows = htonl(pBlock->info.rows);
pRetrieve->numOfCols = htonl(pBlock->info.numOfCols);
pRetrieve->numOfCols = htonl(numOfCols);
int32_t actualLen = 0;
blockCompressEncode(pBlock, pRetrieve->data, &actualLen, pBlock->info.numOfCols, false);
blockCompressEncode(pBlock, pRetrieve->data, &actualLen, numOfCols, false);
SStreamRetrieveReq req = {
.streamId = pTask->streamId,
@ -170,10 +171,12 @@ static int32_t streamAddBlockToDispatchMsg(const SSDataBlock* pBlock, SStreamDis
pRetrieve->completed = 1;
pRetrieve->streamBlockType = pBlock->info.type;
pRetrieve->numOfRows = htonl(pBlock->info.rows);
pRetrieve->numOfCols = htonl(pBlock->info.numOfCols);
int32_t numOfCols = (int32_t)taosArrayGetSize(pBlock->pDataBlock);
pRetrieve->numOfCols = htonl(numOfCols);
int32_t actualLen = 0;
blockCompressEncode(pBlock, pRetrieve->data, &actualLen, pBlock->info.numOfCols, false);
blockCompressEncode(pBlock, pRetrieve->data, &actualLen, numOfCols, false);
actualLen += sizeof(SRetrieveTableRsp);
ASSERT(actualLen <= dataStrLen);
taosArrayPush(pReq->dataLen, &actualLen);
@ -182,7 +185,7 @@ static int32_t streamAddBlockToDispatchMsg(const SSDataBlock* pBlock, SStreamDis
return 0;
}
int32_t streamBuildDispatchMsg(SStreamTask* pTask, SStreamDataBlock* data, SRpcMsg* pMsg, SEpSet** ppEpSet) {
int32_t streamBuildDispatchMsg(SStreamTask* pTask, const SStreamDataBlock* data, SRpcMsg* pMsg, SEpSet** ppEpSet) {
void* buf = NULL;
int32_t code = -1;
int32_t blockNum = taosArrayGetSize(data->blocks);
@ -304,6 +307,8 @@ int32_t streamDispatch(SStreamTask* pTask, SMsgCb* pMsgCb) {
atomic_store_8(&pTask->outputStatus, TASK_OUTPUT_STATUS__NORMAL);
return -1;
}
taosArrayDestroyEx(pBlock->blocks, (FDelete)tDeleteSSDataBlock);
taosFreeQitem(pBlock);
tmsgSendReq(pEpSet, &dispatchMsg);
return 0;

23
source/libs/stream/src/streamExec.c
View File

@ -53,9 +53,13 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, void* data, SArray* pRes)
}
// TODO: do we need free memory?
SSDataBlock* outputCopy = createOneDataBlock(output, true);
outputCopy->info.childId = pTask->selfChildId;
taosArrayPush(pRes, outputCopy);
SSDataBlock block = {0};
assignOneDataBlock(&block, output);
block.info.childId = pTask->selfChildId;
taosArrayPush(pRes, &block);
/*SSDataBlock* outputCopy = createOneDataBlock(output, true);*/
/*outputCopy->info.childId = pTask->selfChildId;*/
/*taosArrayPush(pRes, outputCopy);*/
}
return 0;
}
@ -68,6 +72,7 @@ static SArray* streamExecForQall(SStreamTask* pTask, SArray* pRes) {
streamTaskExecImpl(pTask, data, pRes);
if (pTask->taskStatus == TASK_STATUS__DROPPING) {
taosArrayDestroyEx(pRes, (FDelete)tDeleteSSDataBlock);
return NULL;
}
@ -82,10 +87,13 @@ static SArray* streamExecForQall(SStreamTask* pTask, SArray* pRes) {
qRes->blocks = pRes;
if (streamTaskOutput(pTask, qRes) < 0) {
streamQueueProcessFail(pTask->inputQueue);
taosArrayDestroy(pRes);
taosArrayDestroyEx(pRes, (FDelete)tDeleteSSDataBlock);
taosFreeQitem(qRes);
return NULL;
}
streamQueueProcessSuccess(pTask->inputQueue);
pRes = taosArrayInit(0, sizeof(SSDataBlock));
}
int8_t type = ((SStreamQueueItem*)data)->type;
if (type == STREAM_INPUT__TRIGGER) {
@ -99,9 +107,6 @@ static SArray* streamExecForQall(SStreamTask* pTask, SArray* pRes) {
streamDataSubmitRefDec((SStreamDataSubmit*)data);
taosFreeQitem(data);
}
streamQueueProcessSuccess(pTask->inputQueue);
return taosArrayInit(0, sizeof(SSDataBlock));
}
}
return pRes;
}
@ -125,14 +130,14 @@ int32_t streamExec(SStreamTask* pTask, SMsgCb* pMsgCb) {
pRes = streamExecForQall(pTask, pRes);
if (pRes == NULL) goto FAIL;
taosArrayDestroy(pRes);
taosArrayDestroyEx(pRes, (FDelete)tDeleteSSDataBlock);
atomic_store_8(&pTask->execStatus, TASK_EXEC_STATUS__IDLE);
return 0;
} else if (execStatus == TASK_EXEC_STATUS__CLOSING) {
continue;
} else if (execStatus == TASK_EXEC_STATUS__EXECUTING) {
ASSERT(taosArrayGetSize(pRes) == 0);
taosArrayDestroy(pRes);
taosArrayDestroyEx(pRes, (FDelete)tDeleteSSDataBlock);
return 0;
} else {
ASSERT(0);

4
source/libs/sync/inc/syncEnv.h
View File

@ -31,10 +31,10 @@ extern "C" {
#define TIMER_MAX_MS 0x7FFFFFFF
#define ENV_TICK_TIMER_MS 1000
#define PING_TIMER_MS 1000
#define ELECT_TIMER_MS_MIN 500
#define ELECT_TIMER_MS_MIN 1300
#define ELECT_TIMER_MS_MAX (ELECT_TIMER_MS_MIN * 2)
#define ELECT_TIMER_MS_RANGE (ELECT_TIMER_MS_MAX - ELECT_TIMER_MS_MIN)
#define HEARTBEAT_TIMER_MS 100
#define HEARTBEAT_TIMER_MS 900
#define EMPTY_RAFT_ID ((SRaftId){.addr = 0, .vgId = 0})

4
source/libs/sync/inc/syncRaftLog.h
View File

@ -26,11 +26,13 @@ extern "C" {
#include "syncInt.h"
#include "syncRaftEntry.h"
#include "taosdef.h"
#include "wal.h"
typedef struct SSyncLogStoreData {
SSyncNode* pSyncNode;
SWal* pWal;
SyncIndex beginIndex; // valid begin index, default 0, may be set beginIndex > 0
SWalReadHandle* pWalHandle;
// SyncIndex beginIndex; // valid begin index, default 0, may be set beginIndex > 0
} SSyncLogStoreData;
SSyncLogStore* logStoreCreate(SSyncNode* pSyncNode);

71
source/libs/sync/src/syncAppendEntries.c
View File

@ -420,44 +420,26 @@ static int32_t syncNodePreCommit(SSyncNode* ths, SSyncRaftEntry* pEntry) {
// prevLogIndex == -1
static bool syncNodeOnAppendEntriesLogOK(SSyncNode* pSyncNode, SyncAppendEntries* pMsg) {
if (pMsg->prevLogIndex == SYNC_INDEX_INVALID) {
if (gRaftDetailLog) {
sTrace("syncNodeOnAppendEntriesLogOK true, pMsg->prevLogIndex:%ld", pMsg->prevLogIndex);
}
return true;
}
SyncIndex myLastIndex = syncNodeGetLastIndex(pSyncNode);
if (pMsg->prevLogIndex > myLastIndex) {
if (gRaftDetailLog) {
sTrace("syncNodeOnAppendEntriesLogOK false, pMsg->prevLogIndex:%ld, myLastIndex:%ld", pMsg->prevLogIndex,
myLastIndex);
}
sDebug("vgId:%d sync log not ok, preindex:%ld", pSyncNode->vgId, pMsg->prevLogIndex);
return false;
}
SyncTerm myPreLogTerm = syncNodeGetPreTerm(pSyncNode, pMsg->prevLogIndex + 1);
if (myPreLogTerm == SYNC_TERM_INVALID) {
sError("vgId:%d sync get pre term error, preindex:%ld", pSyncNode->vgId, pMsg->prevLogIndex);
sDebug("vgId:%d sync log not ok2, preindex:%ld", pSyncNode->vgId, pMsg->prevLogIndex);
return false;
}
if (pMsg->prevLogIndex <= myLastIndex && pMsg->prevLogTerm == myPreLogTerm) {
if (gRaftDetailLog) {
sTrace(
"syncNodeOnAppendEntriesLogOK true, pMsg->prevLogIndex:%ld, myLastIndex:%ld, pMsg->prevLogTerm:%lu, "
"myPreLogTerm:%lu",
pMsg->prevLogIndex, myLastIndex, pMsg->prevLogTerm, myPreLogTerm);
}
return true;
}
if (gRaftDetailLog) {
sTrace(
"syncNodeOnAppendEntriesLogOK false, pMsg->prevLogIndex:%ld, myLastIndex:%ld, pMsg->prevLogTerm:%lu, "
"myPreLogTerm:%lu",
pMsg->prevLogIndex, myLastIndex, pMsg->prevLogTerm, myPreLogTerm);
}
sDebug("vgId:%d sync log not ok3, preindex:%ld", pSyncNode->vgId, pMsg->prevLogIndex);
return false;
}
@ -466,14 +448,11 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs
int32_t code = 0;
// print log
char logBuf[128] = {0};
snprintf(logBuf, sizeof(logBuf), "recv SyncAppendEntries, vgId:%d, term:%lu", ths->vgId,
ths->pRaftStore->currentTerm);
syncAppendEntriesLog2(logBuf, pMsg);
syncAppendEntriesLog2("==syncNodeOnAppendEntriesSnapshotCb==", pMsg);
// if already drop replica, do not process
if (!syncNodeInRaftGroup(ths, &(pMsg->srcId)) && !ths->pRaftCfg->isStandBy) {
sInfo("recv SyncAppendEntries maybe replica already dropped");
syncNodeEventLog(ths, "recv sync-append-entries, maybe replica already dropped");
return ret;
}
@ -497,7 +476,7 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs
do {
bool condition = pMsg->term == ths->pRaftStore->currentTerm && ths->state == TAOS_SYNC_STATE_CANDIDATE;
if (condition) {
sTrace("recv SyncAppendEntries, candidate to follower");
syncNodeEventLog(ths, "recv sync-append-entries, candidate to follower");
syncNodeBecomeFollower(ths, "from candidate by append entries");
// do not reply?
@ -505,6 +484,7 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs
}
} while (0);
#if 0
// fake match
//
// condition1:
@ -530,7 +510,7 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs
bool condition0 = (pMsg->term == ths->pRaftStore->currentTerm) && (ths->state == TAOS_SYNC_STATE_FOLLOWER) &&
syncNodeHasSnapshot(ths);
bool condition1 =
condition0 && (ths->pLogStore->syncLogEntryCount(ths->pLogStore) == 0) && (pMsg->prevLogIndex > myLastIndex);
condition0 && (ths->pLogStore->syncLogEntryCount(ths->pLogStore) == 0) && (pMsg->prevLogIndex > myLastIndex); // donot use syncLogEntryCount!!! use isEmpty
bool condition2 = condition0 && (ths->pLogStore->syncLogLastIndex(ths->pLogStore) <= snapshot.lastApplyIndex) &&
(pMsg->prevLogIndex > myLastIndex);
bool condition3 = condition0 && (pMsg->prevLogIndex < snapshot.lastApplyIndex);
@ -538,11 +518,10 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs
bool condition = condition1 || condition2 || condition3 || condition4;
if (condition) {
sTrace(
"recv SyncAppendEntries, fake match, myLastIndex:%ld, syncLogBeginIndex:%ld, syncLogEndIndex:%ld, "
"condition1:%d, condition2:%d, condition3:%d, condition4:%d",
myLastIndex, ths->pLogStore->syncLogBeginIndex(ths->pLogStore),
ths->pLogStore->syncLogEndIndex(ths->pLogStore), condition1, condition2, condition3, condition4);
char logBuf[128];
snprintf(logBuf, sizeof(logBuf), "recv sync-append-entries, fake match, pre-index:%ld, pre-term:%lu",
pMsg->prevLogIndex, pMsg->prevLogTerm);
syncNodeEventLog(ths, logBuf);
// prepare response msg
SyncAppendEntriesReply* pReply = syncAppendEntriesReplyBuild(ths->vgId);
@ -562,6 +541,7 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs
return ret;
}
} while (0);
#endif
// fake match2
//
@ -576,8 +556,13 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs
bool condition = (pMsg->term == ths->pRaftStore->currentTerm) && (ths->state == TAOS_SYNC_STATE_FOLLOWER) &&
(pMsg->prevLogIndex <= ths->commitIndex);
if (condition) {
sTrace("recv SyncAppendEntries, fake match2, msg-prevLogIndex:%ld, my-commitIndex:%ld", pMsg->prevLogIndex,
ths->commitIndex);
do {
char logBuf[128];
snprintf(logBuf, sizeof(logBuf),
"recv sync-append-entries, fake match2, pre-index:%ld, pre-term:%lu, datalen:%d", pMsg->prevLogIndex,
pMsg->prevLogTerm, pMsg->dataLen);
syncNodeEventLog(ths, logBuf);
} while (0);
SyncIndex matchIndex = ths->commitIndex;
bool hasAppendEntries = pMsg->dataLen > 0;
@ -605,6 +590,7 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs
code = syncNodePreCommit(ths, pAppendEntry);
ASSERT(code == 0);
// update match index
matchIndex = pMsg->prevLogIndex + 1;
syncEntryDestory(pAppendEntry);
@ -650,11 +636,10 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs
bool condition = condition1 || condition2;
if (condition) {
sTrace(
"recv SyncAppendEntries, not match, syncLogBeginIndex:%ld, syncLogEndIndex:%ld, condition1:%d, "
"condition2:%d, logOK:%d",
ths->pLogStore->syncLogBeginIndex(ths->pLogStore), ths->pLogStore->syncLogEndIndex(ths->pLogStore),
condition1, condition2, logOK);
char logBuf[128];
snprintf(logBuf, sizeof(logBuf), "recv sync-append-entries, not match, pre-index:%ld, pre-term:%lu, datalen:%d",
pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->dataLen);
syncNodeEventLog(ths, logBuf);
// prepare response msg
SyncAppendEntriesReply* pReply = syncAppendEntriesReplyBuild(ths->vgId);
@ -693,8 +678,10 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs
// has entries in SyncAppendEntries msg
bool hasAppendEntries = pMsg->dataLen > 0;
sTrace("recv SyncAppendEntries, match, myLastIndex:%ld, hasExtraEntries:%d, hasAppendEntries:%d", myLastIndex,
hasExtraEntries, hasAppendEntries);
char logBuf[128];
snprintf(logBuf, sizeof(logBuf), "recv sync-append-entries, match, pre-index:%ld, pre-term:%lu, datalen:%d",
pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->dataLen);
syncNodeEventLog(ths, logBuf);
if (hasExtraEntries) {
// make log same, rollback deleted entries

42
source/libs/sync/src/syncAppendEntriesReply.c
View File

@ -101,32 +101,27 @@ int32_t syncNodeOnAppendEntriesReplySnapshotCb(SSyncNode* ths, SyncAppendEntries
int32_t ret = 0;
// print log
char logBuf[128] = {0};
snprintf(logBuf, sizeof(logBuf), "recv SyncAppendEntriesReply, vgId:%d, term:%lu", ths->vgId,
ths->pRaftStore->currentTerm);
syncAppendEntriesReplyLog2(logBuf, pMsg);
syncAppendEntriesReplyLog2("==syncNodeOnAppendEntriesReplySnapshotCb==", pMsg);
// if already drop replica, do not process
if (!syncNodeInRaftGroup(ths, &(pMsg->srcId)) && !ths->pRaftCfg->isStandBy) {
sInfo("recv SyncAppendEntriesReply, maybe replica already dropped");
return ret;
syncNodeEventLog(ths, "recv sync-append-entries-reply, maybe replica already dropped");
return 0;
}
// drop stale response
if (pMsg->term < ths->pRaftStore->currentTerm) {
sTrace("recv SyncAppendEntriesReply, drop stale response, receive_term:%lu current_term:%lu", pMsg->term,
ths->pRaftStore->currentTerm);
return ret;
char logBuf[128];
snprintf(logBuf, sizeof(logBuf), "recv sync-append-entries-reply, recv-term:%lu, drop stale response", pMsg->term);
syncNodeEventLog(ths, logBuf);
return 0;
}
syncIndexMgrLog2("recv SyncAppendEntriesReply, before pNextIndex:", ths->pNextIndex);
syncIndexMgrLog2("recv SyncAppendEntriesReply, before pMatchIndex:", ths->pMatchIndex);
if (gRaftDetailLog) {
SSnapshot snapshot;
ths->pFsm->FpGetSnapshotInfo(ths->pFsm, &snapshot);
sTrace("recv SyncAppendEntriesReply, before snapshot.lastApplyIndex:%ld, snapshot.lastApplyTerm:%lu",
snapshot.lastApplyIndex, snapshot.lastApplyTerm);
syncNodeEventLog(ths, "recv sync-append-entries-reply, before");
}
syncIndexMgrLog2("recv sync-append-entries-reply, before pNextIndex:", ths->pNextIndex);
syncIndexMgrLog2("recv sync-append-entries-reply, before pMatchIndex:", ths->pMatchIndex);
// no need this code, because if I receive reply.term, then I must have sent for that term.
// if (pMsg->term > ths->pRaftStore->currentTerm) {
@ -134,12 +129,10 @@ int32_t syncNodeOnAppendEntriesReplySnapshotCb(SSyncNode* ths, SyncAppendEntries
// }
if (pMsg->term > ths->pRaftStore->currentTerm) {
char logBuf[128] = {0};
snprintf(logBuf, sizeof(logBuf), "recv SyncAppendEntriesReply, error term, receive_term:%lu current_term:%lu",
pMsg->term, ths->pRaftStore->currentTerm);
syncNodeLog2(logBuf, ths);
sError("%s", logBuf);
return ret;
char logBuf[128];
snprintf(logBuf, sizeof(logBuf), "recv sync-append-entries-reply, error term, recv-term:%lu", pMsg->term);
syncNodeErrorLog(ths, logBuf);
return -1;
}
ASSERT(pMsg->term == ths->pRaftStore->currentTerm);
@ -228,8 +221,11 @@ int32_t syncNodeOnAppendEntriesReplySnapshotCb(SSyncNode* ths, SyncAppendEntries
}
}
syncIndexMgrLog2("recv SyncAppendEntriesReply, after pNextIndex:", ths->pNextIndex);
syncIndexMgrLog2("recv SyncAppendEntriesReply, after pMatchIndex:", ths->pMatchIndex);
if (gRaftDetailLog) {
syncNodeEventLog(ths, "recv sync-append-entries-reply, after");
}
syncIndexMgrLog2("recv sync-append-entries-reply, after pNextIndex:", ths->pNextIndex);
syncIndexMgrLog2("recv sync-append-entries-reply, after pMatchIndex:", ths->pMatchIndex);
return ret;
}

21
source/libs/sync/src/syncMain.c
View File

@ -416,7 +416,7 @@ int32_t syncGetSnapshotMetaByIndex(int64_t rid, SyncIndex snapshotIndex, struct
}
}
sMeta->lastConfigIndex = lastIndex;
sTrace("vgId:%d, get snapshot meta by index:%" PRId64 " lastConfigIndex:%" PRId64, pSyncNode->vgId, snapshotIndex,
sTrace("vgId:%d, get snapshot meta by index:%" PRId64 " lcindex:%" PRId64, pSyncNode->vgId, snapshotIndex,
sMeta->lastConfigIndex);
taosReleaseRef(tsNodeRefId, pSyncNode->rid);
@ -433,8 +433,9 @@ SyncIndex syncNodeGetSnapshotConfigIndex(SSyncNode* pSyncNode, SyncIndex snapsho
lastIndex = (pSyncNode->pRaftCfg->configIndexArr)[i];
}
}
sTrace("vgId:%d, sync get snapshot last config index, index:%ld lcindex:%ld", pSyncNode->vgId, snapshotLastApplyIndex,
lastIndex);
sTrace("sync syncNodeGetSnapshotConfigIndex index:%ld lastConfigIndex:%ld", snapshotLastApplyIndex, lastIndex);
return lastIndex;
}
@ -1310,6 +1311,10 @@ void syncNodeEventLog(const SSyncNode* pSyncNode, char* str) {
SyncIndex logBeginIndex = pSyncNode->pLogStore->syncLogBeginIndex(pSyncNode->pLogStore);
char* pCfgStr = syncCfg2SimpleStr(&(pSyncNode->pRaftCfg->cfg));
char* printStr = "";
if (pCfgStr != NULL) {
printStr = pCfgStr;
}
if (userStrLen < 256) {
char logBuf[256 + 256];
@ -1321,7 +1326,7 @@ void syncNodeEventLog(const SSyncNode* pSyncNode, char* str) {
pSyncNode->vgId, syncUtilState2String(pSyncNode->state), str, pSyncNode->pRaftStore->currentTerm,
pSyncNode->commitIndex, logBeginIndex, logLastIndex, snapshot.lastApplyIndex,
pSyncNode->pRaftCfg->isStandBy, pSyncNode->replicaNum, pSyncNode->pRaftCfg->lastConfigIndex,
pSyncNode->changing, pCfgStr);
pSyncNode->changing, printStr);
} else {
snprintf(logBuf, sizeof(logBuf), "%s", str);
}
@ -1338,7 +1343,7 @@ void syncNodeEventLog(const SSyncNode* pSyncNode, char* str) {
pSyncNode->vgId, syncUtilState2String(pSyncNode->state), str, pSyncNode->pRaftStore->currentTerm,
pSyncNode->commitIndex, logBeginIndex, logLastIndex, snapshot.lastApplyIndex,
pSyncNode->pRaftCfg->isStandBy, pSyncNode->replicaNum, pSyncNode->pRaftCfg->lastConfigIndex,
pSyncNode->changing, pCfgStr);
pSyncNode->changing, printStr);
} else {
snprintf(s, len, "%s", str);
}
@ -1957,7 +1962,6 @@ SyncTerm syncNodeGetPreTerm(SSyncNode* pSyncNode, SyncIndex index) {
taosMemoryFree(pPreEntry);
return preTerm;
} else {
if (terrno == TSDB_CODE_WAL_LOG_NOT_EXIST) {
SSnapshot snapshot = {.data = NULL, .lastApplyIndex = -1, .lastApplyTerm = 0, .lastConfigIndex = -1};
if (pSyncNode->pFsm->FpGetSnapshotInfo != NULL) {
pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, &snapshot);
@ -1966,7 +1970,12 @@ SyncTerm syncNodeGetPreTerm(SSyncNode* pSyncNode, SyncIndex index) {
}
}
}
}
do {
char logBuf[128];
snprintf(logBuf, sizeof(logBuf), "sync node get pre term error, index:%ld", index);
syncNodeErrorLog(pSyncNode, logBuf);
} while (0);
return SYNC_TERM_INVALID;
}

4
source/libs/sync/src/syncRaftCfg.c
View File

@ -107,6 +107,7 @@ char *syncCfg2Str(SSyncCfg *pSyncCfg) {
}
char *syncCfg2SimpleStr(SSyncCfg *pSyncCfg) {
if (pSyncCfg != NULL) {
int32_t len = 512;
char *s = taosMemoryMalloc(len);
memset(s, 0, len);
@ -127,6 +128,9 @@ char *syncCfg2SimpleStr(SSyncCfg *pSyncCfg) {
strcpy(p - 2, "}");
return s;
}
return NULL;
}
int32_t syncCfgFromJson(const cJSON *pRoot, SSyncCfg *pSyncCfg) {

205
source/libs/sync/src/syncRaftLog.c
View File

@ -16,10 +16,10 @@
#include "syncRaftLog.h"
#include "syncRaftCfg.h"
#include "syncRaftStore.h"
#include "wal.h"
// refactor, log[0 .. n] ==> log[m .. n]
static int32_t raftLogSetBeginIndex(struct SSyncLogStore* pLogStore, SyncIndex beginIndex);
static int32_t raftLogRestoreFromSnapshot(struct SSyncLogStore* pLogStore, SyncIndex snapshotIndex);
// static int32_t raftLogSetBeginIndex(struct SSyncLogStore* pLogStore, SyncIndex beginIndex);
static SyncIndex raftLogBeginIndex(struct SSyncLogStore* pLogStore);
static SyncIndex raftLogEndIndex(struct SSyncLogStore* pLogStore);
static SyncIndex raftLogWriteIndex(struct SSyncLogStore* pLogStore);
@ -45,31 +45,43 @@ static int32_t logStoreUpdateCommitIndex(SSyncLogStore* pLogStore, SyncI
static SyncIndex logStoreGetCommitIndex(SSyncLogStore* pLogStore);
// refactor, log[0 .. n] ==> log[m .. n]
/*
static int32_t raftLogSetBeginIndex(struct SSyncLogStore* pLogStore, SyncIndex beginIndex) {
sTrace("raftLogSetBeginIndex beginIndex:%ld", beginIndex);
// if beginIndex == 0, donot need call this funciton
ASSERT(beginIndex > 0);
SSyncLogStoreData* pData = pLogStore->data;
SWal* pWal = pData->pWal;
sTrace("vgId:%d, reset wal begin index:%ld", pData->pSyncNode->vgId, beginIndex);
pData->beginIndex = beginIndex;
walRestoreFromSnapshot(pWal, beginIndex - 1);
return 0;
}
*/
static int32_t raftLogRestoreFromSnapshot(struct SSyncLogStore* pLogStore, SyncIndex snapshotIndex) {
ASSERT(snapshotIndex >= 0);
SSyncLogStoreData* pData = pLogStore->data;
SWal* pWal = pData->pWal;
walRestoreFromSnapshot(pWal, snapshotIndex);
return 0;
}
static SyncIndex raftLogBeginIndex(struct SSyncLogStore* pLogStore) {
SSyncLogStoreData* pData = pLogStore->data;
SWal* pWal = pData->pWal;
return pData->beginIndex;
SyncIndex firstVer = walGetFirstVer(pWal);
return firstVer;
}
static SyncIndex raftLogEndIndex(struct SSyncLogStore* pLogStore) { return raftLogLastIndex(pLogStore); }
static bool raftLogIsEmpty(struct SSyncLogStore* pLogStore) {
SyncIndex beginIndex = raftLogBeginIndex(pLogStore);
SyncIndex endIndex = raftLogEndIndex(pLogStore);
return (endIndex < beginIndex);
SSyncLogStoreData* pData = pLogStore->data;
SWal* pWal = pData->pWal;
return walIsEmpty(pWal);
}
static int32_t raftLogEntryCount(struct SSyncLogStore* pLogStore) {
@ -96,23 +108,8 @@ static SyncIndex raftLogLastIndex(struct SSyncLogStore* pLogStore) {
SSyncLogStoreData* pData = pLogStore->data;
SWal* pWal = pData->pWal;
SyncIndex lastVer = walGetLastVer(pWal);
SyncIndex firstVer = walGetFirstVer(pWal);
if (lastVer < firstVer) {
// no record
lastIndex = -1;
} else {
if (firstVer >= 0) {
lastIndex = lastVer;
} else if (firstVer == -1) {
lastIndex = -1;
} else {
ASSERT(0);
}
}
return lastIndex;
return lastVer;
}
static SyncIndex raftLogWriteIndex(struct SSyncLogStore* pLogStore) {
@ -122,6 +119,26 @@ static SyncIndex raftLogWriteIndex(struct SSyncLogStore* pLogStore) {
return lastVer + 1;
}
static SyncTerm raftLogLastTerm(struct SSyncLogStore* pLogStore) {
SSyncLogStoreData* pData = pLogStore->data;
SWal* pWal = pData->pWal;
if (walIsEmpty(pWal)) {
return 0;
} else {
SSyncRaftEntry* pLastEntry;
int32_t code = raftLogGetLastEntry(pLogStore, &pLastEntry);
ASSERT(code == 0);
ASSERT(pLastEntry != NULL);
SyncTerm lastTerm = pLastEntry->term;
taosMemoryFree(pLastEntry);
return lastTerm;
}
return 0;
}
/*
static SyncTerm raftLogLastTerm(struct SSyncLogStore* pLogStore) {
SyncTerm lastTerm = 0;
if (raftLogEntryCount(pLogStore) == 0) {
@ -137,6 +154,7 @@ static SyncTerm raftLogLastTerm(struct SSyncLogStore* pLogStore) {
}
return lastTerm;
}
*/
static int32_t raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry) {
SSyncLogStoreData* pData = pLogStore->data;
@ -160,8 +178,12 @@ static int32_t raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncRaftEntr
const char* errStr = tstrerror(err);
int32_t sysErr = errno;
const char* sysErrStr = strerror(errno);
sError("vgId:%d wal write error, index:%ld, err:%d %X, msg:%s, syserr:%d, sysmsg:%s", pData->pSyncNode->vgId,
char logBuf[128];
snprintf(logBuf, sizeof(logBuf), "wal write error, index:%ld, err:%d %X, msg:%s, syserr:%d, sysmsg:%s",
pEntry->index, err, err, errStr, sysErr, sysErrStr);
syncNodeErrorLog(pData->pSyncNode, logBuf);
ASSERT(0);
}
@ -229,7 +251,8 @@ static int32_t raftLogGetEntry(struct SSyncLogStore* pLogStore, SyncIndex index,
*ppEntry = NULL;
SWalReadHandle* pWalHandle = walOpenReadHandle(pWal);
// SWalReadHandle* pWalHandle = walOpenReadHandle(pWal);
SWalReadHandle* pWalHandle = pData->pWalHandle;
if (pWalHandle == NULL) {
return -1;
}
@ -240,12 +263,23 @@ static int32_t raftLogGetEntry(struct SSyncLogStore* pLogStore, SyncIndex index,
const char* errStr = tstrerror(err);
int32_t sysErr = errno;
const char* sysErrStr = strerror(errno);
sError("vgId:%d wal read error, index:%ld, err:%d %X, msg:%s, syserr:%d, sysmsg:%s", pData->pSyncNode->vgId, index,
err, err, errStr, sysErr, sysErrStr);
do {
char logBuf[128];
snprintf(logBuf, sizeof(logBuf), "wal read error, index:%ld, err:%d %X, msg:%s, syserr:%d, sysmsg:%s", index, err,
err, errStr, sysErr, sysErrStr);
if (terrno == TSDB_CODE_WAL_LOG_NOT_EXIST) {
syncNodeEventLog(pData->pSyncNode, logBuf);
} else {
syncNodeErrorLog(pData->pSyncNode, logBuf);
}
} while (0);
/*
int32_t saveErr = terrno;
walCloseReadHandle(pWalHandle);
terrno = saveErr;
*/
return code;
}
@ -261,9 +295,11 @@ static int32_t raftLogGetEntry(struct SSyncLogStore* pLogStore, SyncIndex index,
ASSERT((*ppEntry)->dataLen == pWalHandle->pHead->head.bodyLen);
memcpy((*ppEntry)->data, pWalHandle->pHead->head.body, pWalHandle->pHead->head.bodyLen);
/*
int32_t saveErr = terrno;
walCloseReadHandle(pWalHandle);
terrno = saveErr;
*/
return code;
}
@ -285,6 +321,25 @@ static int32_t raftLogTruncate(struct SSyncLogStore* pLogStore, SyncIndex fromIn
return code;
}
static int32_t raftLogGetLastEntry(SSyncLogStore* pLogStore, SSyncRaftEntry** ppLastEntry) {
SSyncLogStoreData* pData = pLogStore->data;
SWal* pWal = pData->pWal;
ASSERT(ppLastEntry != NULL);
*ppLastEntry = NULL;
if (walIsEmpty(pWal)) {
terrno = TSDB_CODE_WAL_LOG_NOT_EXIST;
return -1;
} else {
SyncIndex lastIndex = raftLogLastIndex(pLogStore);
int32_t code = raftLogGetEntry(pLogStore, lastIndex, ppLastEntry);
return code;
}
return -1;
}
/*
static int32_t raftLogGetLastEntry(SSyncLogStore* pLogStore, SSyncRaftEntry** ppLastEntry) {
*ppLastEntry = NULL;
if (raftLogEntryCount(pLogStore) == 0) {
@ -294,6 +349,7 @@ static int32_t raftLogGetLastEntry(SSyncLogStore* pLogStore, SSyncRaftEntry** pp
int32_t code = raftLogGetEntry(pLogStore, lastIndex, ppLastEntry);
return code;
}
*/
//-------------------------------
SSyncLogStore* logStoreCreate(SSyncNode* pSyncNode) {
@ -306,7 +362,12 @@ SSyncLogStore* logStoreCreate(SSyncNode* pSyncNode) {
SSyncLogStoreData* pData = pLogStore->data;
pData->pSyncNode = pSyncNode;
pData->pWal = pSyncNode->pWal;
ASSERT(pData->pWal != NULL);
pData->pWalHandle = walOpenReadHandle(pData->pWal);
ASSERT(pData->pWalHandle != NULL);
/*
SyncIndex firstVer = walGetFirstVer(pData->pWal);
SyncIndex lastVer = walGetLastVer(pData->pWal);
if (firstVer >= 0) {
@ -316,6 +377,7 @@ SSyncLogStore* logStoreCreate(SSyncNode* pSyncNode) {
} else {
ASSERT(0);
}
*/
pLogStore->appendEntry = logStoreAppendEntry;
pLogStore->getEntry = logStoreGetEntry;
@ -325,7 +387,8 @@ SSyncLogStore* logStoreCreate(SSyncNode* pSyncNode) {
pLogStore->updateCommitIndex = logStoreUpdateCommitIndex;
pLogStore->getCommitIndex = logStoreGetCommitIndex;
pLogStore->syncLogSetBeginIndex = raftLogSetBeginIndex;
// pLogStore->syncLogSetBeginIndex = raftLogSetBeginIndex;
pLogStore->syncLogRestoreFromSnapshot = raftLogRestoreFromSnapshot;
pLogStore->syncLogBeginIndex = raftLogBeginIndex;
pLogStore->syncLogEndIndex = raftLogEndIndex;
pLogStore->syncLogIsEmpty = raftLogIsEmpty;
@ -344,6 +407,11 @@ SSyncLogStore* logStoreCreate(SSyncNode* pSyncNode) {
void logStoreDestory(SSyncLogStore* pLogStore) {
if (pLogStore != NULL) {
SSyncLogStoreData* pData = pLogStore->data;
if (pData->pWalHandle != NULL) {
walCloseReadHandle(pData->pWalHandle);
}
taosMemoryFree(pLogStore->data);
taosMemoryFree(pLogStore);
}
@ -368,8 +436,11 @@ int32_t logStoreAppendEntry(SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry) {
const char* errStr = tstrerror(err);
int32_t sysErr = errno;
const char* sysErrStr = strerror(errno);
sError("vgId:%d wal write error, index:%ld, err:%d %X, msg:%s, syserr:%d, sysmsg:%s", pData->pSyncNode->vgId,
char logBuf[128];
snprintf(logBuf, sizeof(logBuf), "wal write error, index:%ld, err:%d %X, msg:%s, syserr:%d, sysmsg:%s",
pEntry->index, err, err, errStr, sysErr, sysErrStr);
syncNodeErrorLog(pData->pSyncNode, logBuf);
ASSERT(0);
}
@ -389,7 +460,8 @@ SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) {
SWal* pWal = pData->pWal;
if (index >= SYNC_INDEX_BEGIN && index <= logStoreLastIndex(pLogStore)) {
SWalReadHandle* pWalHandle = walOpenReadHandle(pWal);
// SWalReadHandle* pWalHandle = walOpenReadHandle(pWal);
SWalReadHandle* pWalHandle = pData->pWalHandle;
ASSERT(pWalHandle != NULL);
int32_t code = walReadWithHandle(pWalHandle, index);
@ -398,12 +470,20 @@ SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) {
const char* errStr = tstrerror(err);
int32_t sysErr = errno;
const char* sysErrStr = strerror(errno);
sError("vgId:%d wal read error, index:%ld, err:%d %X, msg:%s, syserr:%d, sysmsg:%s", pData->pSyncNode->vgId,
index, err, err, errStr, sysErr, sysErrStr);
do {
char logBuf[128];
snprintf(logBuf, sizeof(logBuf), "wal read error, index:%ld, err:%d %X, msg:%s, syserr:%d, sysmsg:%s", index,
err, err, errStr, sysErr, sysErrStr);
if (terrno == TSDB_CODE_WAL_LOG_NOT_EXIST) {
syncNodeEventLog(pData->pSyncNode, logBuf);
} else {
syncNodeErrorLog(pData->pSyncNode, logBuf);
}
} while (0);
ASSERT(0);
}
// ASSERT(walReadWithHandle(pWalHandle, index) == 0);
SSyncRaftEntry* pEntry = syncEntryBuild(pWalHandle->pHead->head.bodyLen);
ASSERT(pEntry != NULL);
@ -417,9 +497,11 @@ SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) {
ASSERT(pEntry->dataLen == pWalHandle->pHead->head.bodyLen);
memcpy(pEntry->data, pWalHandle->pHead->head.body, pWalHandle->pHead->head.bodyLen);
/*
int32_t saveErr = terrno;
walCloseReadHandle(pWalHandle);
terrno = saveErr;
*/
return pEntry;
@ -498,6 +580,7 @@ SSyncRaftEntry* logStoreGetLastEntry(SSyncLogStore* pLogStore) {
return pEntry;
}
/*
cJSON* logStore2Json(SSyncLogStore* pLogStore) {
char u64buf[128] = {0};
SSyncLogStoreData* pData = (SSyncLogStoreData*)pLogStore->data;
@ -544,6 +627,57 @@ cJSON* logStore2Json(SSyncLogStore* pLogStore) {
cJSON_AddItemToObject(pJson, "SSyncLogStore", pRoot);
return pJson;
}
*/
cJSON* logStore2Json(SSyncLogStore* pLogStore) {
char u64buf[128] = {0};
SSyncLogStoreData* pData = (SSyncLogStoreData*)pLogStore->data;
cJSON* pRoot = cJSON_CreateObject();
if (pData != NULL && pData->pWal != NULL) {
snprintf(u64buf, sizeof(u64buf), "%p", pData->pSyncNode);
cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf);
snprintf(u64buf, sizeof(u64buf), "%p", pData->pWal);
cJSON_AddStringToObject(pRoot, "pWal", u64buf);
SyncIndex beginIndex = raftLogBeginIndex(pLogStore);
snprintf(u64buf, sizeof(u64buf), "%ld", beginIndex);
cJSON_AddStringToObject(pRoot, "beginIndex", u64buf);
SyncIndex endIndex = raftLogEndIndex(pLogStore);
snprintf(u64buf, sizeof(u64buf), "%ld", endIndex);
cJSON_AddStringToObject(pRoot, "endIndex", u64buf);
int32_t count = raftLogEntryCount(pLogStore);
cJSON_AddNumberToObject(pRoot, "entryCount", count);
snprintf(u64buf, sizeof(u64buf), "%ld", raftLogWriteIndex(pLogStore));
cJSON_AddStringToObject(pRoot, "WriteIndex", u64buf);
snprintf(u64buf, sizeof(u64buf), "%d", raftLogIsEmpty(pLogStore));
cJSON_AddStringToObject(pRoot, "IsEmpty", u64buf);
snprintf(u64buf, sizeof(u64buf), "%ld", raftLogLastIndex(pLogStore));
cJSON_AddStringToObject(pRoot, "LastIndex", u64buf);
snprintf(u64buf, sizeof(u64buf), "%lu", raftLogLastTerm(pLogStore));
cJSON_AddStringToObject(pRoot, "LastTerm", u64buf);
cJSON* pEntries = cJSON_CreateArray();
cJSON_AddItemToObject(pRoot, "pEntries", pEntries);
if (!raftLogIsEmpty(pLogStore)) {
for (SyncIndex i = beginIndex; i <= endIndex; ++i) {
SSyncRaftEntry* pEntry = logStoreGetEntry(pLogStore, i);
cJSON_AddItemToArray(pEntries, syncEntry2Json(pEntry));
syncEntryDestory(pEntry);
}
}
}
cJSON* pJson = cJSON_CreateObject();
cJSON_AddItemToObject(pJson, "SSyncLogStore", pRoot);
return pJson;
}
char* logStore2Str(SSyncLogStore* pLogStore) {
cJSON* pJson = logStore2Json(pLogStore);
@ -563,7 +697,8 @@ cJSON* logStoreSimple2Json(SSyncLogStore* pLogStore) {
snprintf(u64buf, sizeof(u64buf), "%p", pData->pWal);
cJSON_AddStringToObject(pRoot, "pWal", u64buf);
snprintf(u64buf, sizeof(u64buf), "%ld", pData->beginIndex);
SyncIndex beginIndex = raftLogBeginIndex(pLogStore);
snprintf(u64buf, sizeof(u64buf), "%ld", beginIndex);
cJSON_AddStringToObject(pRoot, "beginIndex", u64buf);
SyncIndex endIndex = raftLogEndIndex(pLogStore);

11
source/libs/sync/src/syncReplication.c
View File

@ -219,6 +219,17 @@ int32_t syncNodeReplicate(SSyncNode* pSyncNode) {
int32_t syncNodeAppendEntries(SSyncNode* pSyncNode, const SRaftId* destRaftId, const SyncAppendEntries* pMsg) {
int32_t ret = 0;
do {
char host[128];
uint16_t port;
syncUtilU642Addr(destRaftId->addr, host, sizeof(host), &port);
sDebug(
"vgId:%d, send sync-append-entries to %s:%d, term:%lu, pre-index:%ld, pre-term:%lu, pterm:%lu, commit:%ld, "
"datalen:%d",
pSyncNode->vgId, host, port, pMsg->term, pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->privateTerm,
pMsg->commitIndex, pMsg->dataLen);
} while (0);
SRpcMsg rpcMsg;
syncAppendEntries2RpcMsg(pMsg, &rpcMsg);
syncNodeSendMsgById(destRaftId, pSyncNode, &rpcMsg);

3
source/libs/sync/src/syncSnapshot.c
View File

@ -545,7 +545,8 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) {
pReceiver->pSyncNode->commitIndex = pReceiver->snapshot.lastApplyIndex;
}
pSyncNode->pLogStore->syncLogSetBeginIndex(pSyncNode->pLogStore, pMsg->lastIndex + 1);
// pSyncNode->pLogStore->syncLogSetBeginIndex(pSyncNode->pLogStore, pMsg->lastIndex + 1);
pSyncNode->pLogStore->syncLogRestoreFromSnapshot(pSyncNode->pLogStore, pMsg->lastIndex);
// maybe update lastconfig
if (pMsg->lastConfigIndex >= SYNC_INDEX_BEGIN) {

33
source/libs/sync/test/syncRaftLogTest2.cpp
View File

@ -113,7 +113,8 @@ void test2() {
pLogStore = logStoreCreate(pSyncNode);
assert(pLogStore);
pSyncNode->pLogStore = pLogStore;
pLogStore->syncLogSetBeginIndex(pLogStore, 5);
//pLogStore->syncLogSetBeginIndex(pLogStore, 5);
pLogStore->syncLogRestoreFromSnapshot(pLogStore, 4);
logStoreLog2((char*)"\n\n\ntest2 ----- ", pLogStore);
if (gAssert) {
@ -228,7 +229,8 @@ void test4() {
assert(pLogStore);
pSyncNode->pLogStore = pLogStore;
logStoreLog2((char*)"\n\n\ntest4 ----- ", pLogStore);
pLogStore->syncLogSetBeginIndex(pLogStore, 5);
//pLogStore->syncLogSetBeginIndex(pLogStore, 5);
pLogStore->syncLogRestoreFromSnapshot(pLogStore, 4);
for (int i = 5; i <= 9; ++i) {
int32_t dataLen = 10;
@ -289,7 +291,8 @@ void test5() {
assert(pLogStore);
pSyncNode->pLogStore = pLogStore;
logStoreLog2((char*)"\n\n\ntest5 ----- ", pLogStore);
pLogStore->syncLogSetBeginIndex(pLogStore, 5);
//pLogStore->syncLogSetBeginIndex(pLogStore, 5);
pLogStore->syncLogRestoreFromSnapshot(pLogStore, 4);
for (int i = 5; i <= 9; ++i) {
int32_t dataLen = 10;
@ -363,7 +366,8 @@ void test6() {
assert(pLogStore);
pSyncNode->pLogStore = pLogStore;
logStoreLog2((char*)"\n\n\ntest6 ----- ", pLogStore);
pLogStore->syncLogSetBeginIndex(pLogStore, 5);
// pLogStore->syncLogSetBeginIndex(pLogStore, 5);
pLogStore->syncLogRestoreFromSnapshot(pLogStore, 4);
for (int i = 5; i <= 9; ++i) {
int32_t dataLen = 10;
@ -405,14 +409,32 @@ void test6() {
assert(pLogStore->syncLogLastTerm(pLogStore) == 0);
}
do {
SyncIndex firstVer = walGetFirstVer(pWal);
SyncIndex lastVer = walGetLastVer(pWal);
bool isEmpty = walIsEmpty(pWal);
printf("before -------- firstVer:%ld lastVer:%ld isEmpty:%d \n", firstVer, lastVer, isEmpty);
} while (0);
logStoreDestory(pLogStore);
cleanup();
// restart
init();
pLogStore = logStoreCreate(pSyncNode);
assert(pLogStore);
pSyncNode->pLogStore = pLogStore;
do {
SyncIndex firstVer = walGetFirstVer(pWal);
SyncIndex lastVer = walGetLastVer(pWal);
bool isEmpty = walIsEmpty(pWal);
printf("after -------- firstVer:%ld lastVer:%ld isEmpty:%d \n", firstVer, lastVer, isEmpty);
} while (0);
logStoreLog2((char*)"\n\n\ntest6 restart ----- ", pLogStore);
if (gAssert) {
@ -432,17 +454,20 @@ void test6() {
int main(int argc, char** argv) {
tsAsyncLog = 0;
sDebugFlag = DEBUG_TRACE + DEBUG_INFO + DEBUG_SCREEN + DEBUG_FILE;
gRaftDetailLog = true;
if (argc == 2) {
gAssert = atoi(argv[1]);
}
sTrace("gAssert : %d", gAssert);
/*
test1();
test2();
test3();
test4();
test5();
*/
test6();
return 0;

4
source/libs/sync/test/syncRaftLogTest3.cpp
View File

@ -312,7 +312,8 @@ void test5() {
pSyncNode->pLogStore = pLogStore;
logStoreLog2((char*)"\n\n\ntest5 ----- ", pLogStore);
pSyncNode->pLogStore->syncLogSetBeginIndex(pSyncNode->pLogStore, 6);
//pSyncNode->pLogStore->syncLogSetBeginIndex(pSyncNode->pLogStore, 6);
pLogStore->syncLogRestoreFromSnapshot(pSyncNode->pLogStore, 5);
for (int i = 6; i <= 10; ++i) {
int32_t dataLen = 10;
SSyncRaftEntry* pEntry = syncEntryBuild(dataLen);
@ -372,6 +373,7 @@ void test5() {
int main(int argc, char** argv) {
tsAsyncLog = 0;
sDebugFlag = DEBUG_TRACE + DEBUG_INFO + DEBUG_SCREEN + DEBUG_FILE;
gRaftDetailLog = true;
if (argc == 2) {
gAssert = atoi(argv[1]);

6
source/libs/transport/src/transCli.c
View File

@ -983,7 +983,7 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) {
if (pResp->code == TSDB_CODE_RPC_NETWORK_UNAVAIL) {
if (pCtx->retryCount < pEpSet->numOfEps * 3) {
pEpSet->inUse = (++pEpSet->inUse) % pEpSet->numOfEps;
if (pThrd->quit == false) {
STaskArg* arg = taosMemoryMalloc(sizeof(STaskArg));
arg->param1 = pMsg;
arg->param2 = pThrd;
@ -995,6 +995,7 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) {
transUnrefCliHandle(pConn);
return -1;
}
}
} else if (pCtx->retryCount < TRANS_RETRY_COUNT_LIMIT) {
if (pResp->contLen == 0) {
pEpSet->inUse = (++pEpSet->inUse) % pEpSet->numOfEps;
@ -1010,10 +1011,10 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) {
tTrace("%s use remote epset, inUse: %d, retry count:%d, limit: %d", pTransInst->label, pEpSet->inUse,
pCtx->retryCount + 1, TRANS_RETRY_COUNT_LIMIT);
}
if (pThrd->quit == false) {
if (pConn->status != ConnInPool) {
addConnToPool(pThrd->pool, pConn);
}
STaskArg* arg = taosMemoryMalloc(sizeof(STaskArg));
arg->param1 = pMsg;
arg->param2 = pThrd;
@ -1021,6 +1022,7 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) {
return -1;
}
}
}
STraceId* trace = &pResp->info.traceId;
if (pCtx->pSem != NULL) {

15
source/libs/wal/src/walRead.c
View File

@ -272,14 +272,23 @@ int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver) {
}
}
if (ver > pRead->pWal->vers.lastVer || ver < pRead->pWal->vers.firstVer) {
wError("invalid version: % " PRId64 ", first ver %ld, last ver %ld", ver, pRead->pWal->vers.firstVer,
pRead->pWal->vers.lastVer);
terrno = TSDB_CODE_WAL_LOG_NOT_EXIST;
return -1;
}
ASSERT(taosValidFile(pRead->pReadLogTFile) == true);
code = taosReadFile(pRead->pReadLogTFile, pRead->pHead, sizeof(SWalHead));
if (code != sizeof(SWalHead)) {
if (code < 0)
terrno = TAOS_SYSTEM_ERROR(errno);
else
else {
terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
ASSERT(0);
}
return -1;
}
@ -304,8 +313,10 @@ int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver) {
pRead->pHead->head.bodyLen) {
if (code < 0)
terrno = TAOS_SYSTEM_ERROR(errno);
else
else {
terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
ASSERT(0);
}
return -1;
}

1
source/libs/wal/src/walWrite.c
View File

@ -150,6 +150,7 @@ int32_t walRollback(SWal *pWal, int64_t ver) {
ASSERT(code == 0);
if (code != 0) {
terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
ASSERT(0);
return -1;
}
if (head.head.version != ver) {

2
source/util/src/tlog.c
View File

@ -743,7 +743,7 @@ cmp_end:
}
void taosSetAllDebugFlag(int32_t flag) {
if (!(flag & DEBUG_TRACE || flag & DEBUG_DEBUG || flag & DEBUG_DUMP)) return;
if (flag <= 0) return;
dDebugFlag = flag;
vDebugFlag = flag;

258
tests/pytest/util/cluster.py
View File

@ -21,44 +21,38 @@ class ClusterDnodes(TDDnodes):
self.testCluster = False
self.valgrind = 0
self.killValgrind = 1
self.independent = True
self.dnodeNums = 5
# def getTDDnodes(dnodeNums):
# return
class ConfigureyCluster:
"""configure dnodes and return TDDnodes list, it can """
"""This will create defined number of dnodes and create a cluset.
at the same time, it will return TDDnodes list: dnodes, """
hostname= socket.gethostname()
def __init__(self):
self.dnodes = None
self.dnodes_nums = 5
self.dnodes = []
self.dnodeNums = 5
self.independent = True
self.start_port = 6030
self.startPort = 6030
self.portStep = 100
hostname1= socket.gethostname()
self.mnodeNums = 0
def configure_cluster(self ,dnodes_nums=5,independent=True,start_port=6030,portStep=100,hostname="%s"%hostname1):
self.start_port=int(start_port)
def configure_cluster(self ,dnodeNums=5,mnodeNums=0,startPort=6030,portStep=100,hostname="%s"%hostname):
self.startPort=int(startPort)
self.portStep=int(portStep)
self.hostname=hostname
self.dnodes_nums = int(dnodes_nums)
self.independent = independent
self.dnodeNums = int(dnodeNums)
self.mnodeNums = int(mnodeNums)
self.dnodes = []
start_port_sec = 6130
for num in range(1, (self.dnodes_nums+1)):
startPort_sec = int(startPort+portStep)
for num in range(1, (self.dnodeNums+1)):
dnode = TDDnode(num)
dnode.addExtraCfg("firstEp", f"{hostname}:{self.start_port}")
dnode.addExtraCfg("firstEp", f"{hostname}:{self.startPort}")
dnode.addExtraCfg("fqdn", f"{hostname}")
dnode.addExtraCfg("serverPort", f"{self.start_port + (num-1)*self.portStep}")
# dnode.addExtraCfg("monitorFqdn", hostname)
# dnode.addExtraCfg("monitorPort", 7043)
dnode.addExtraCfg("secondEp", f"{hostname}:{start_port_sec}")
# configure three dnoe don't support vnodes
if self.dnodes_nums > 4 :
if self.independent and (num < 4):
dnode.addExtraCfg("serverPort", f"{self.startPort + (num-1)*self.portStep}")
dnode.addExtraCfg("secondEp", f"{hostname}:{startPort_sec}")
# configure dnoe of independent mnodes
if num <= self.mnodeNums and self.mnodeNums != 0 :
dnode.addExtraCfg("supportVnodes", 0)
# print(dnode)
self.dnodes.append(dnode)
@ -69,24 +63,7 @@ class ConfigureyCluster:
for dnode in self.dnodes[1:]:
# print(dnode.cfgDict)
dnode_id = dnode.cfgDict["fqdn"] + ":" +dnode.cfgDict["serverPort"]
print(dnode_id)
tdSql.execute(" create dnode '%s';"%dnode_id)
# count=0
# while count < 10:
# time.sleep(1)
# tdSql.query("show dnodes;")
# if tdSql.checkRows(self.dnodes_nums) :
# print("mnode is three nodes")
# if tdSql.queryResult[0][4]=='leader' :
# if tdSql.queryResult[2][4]=='offline':
# if tdSql.queryResult[1][2]=='follower':
# print("stop mnodes on dnode 3 successfully in 10s")
# break
# count+=1
# else:
# print("stop mnodes on dnode 3 failed in 10s")
# return -1
checkstatus=False
@ -97,206 +74,19 @@ class ConfigureyCluster:
tdSql.query("show dnodes")
# tdLog.debug(tdSql.queryResult)
status=0
for i in range(self.dnodes_nums):
for i in range(self.dnodeNums):
if tdSql.queryResult[i][4] == "ready":
status+=1
tdLog.debug(status)
# tdLog.debug(status)
if status == self.dnodes_nums:
tdLog.debug(" create cluster with %d dnode and check cluster dnode all ready within 5s! " %self.dnodes_nums)
if status == self.dnodeNums:
tdLog.debug(" create cluster with %d dnode and check cluster dnode all ready within 5s! " %self.dnodeNums)
break
count+=1
time.sleep(1)
else:
tdLog.debug("create cluster with %d dnode but check dnode not ready within 5s ! "%self.dnodes_nums)
tdLog.debug("create cluster with %d dnode but check dnode not ready within 5s ! "%self.dnodeNums)
return -1
cluster = ConfigureyCluster()
# def start(self ,dnodes_nums):
# self.TDDnodes = ClusterDnodes(dnodes)
# self.TDDnodes.init("")
# self.TDDnodes.setTestCluster(testCluster)
# self.TDDnodes.setValgrind(valgrind)
# self.TDDnodes.stopAll()
# for dnode in self.TDDnodes.dnodes:
# self.TDDnodes.deploy(dnode.index,{})
# for dnode in self.TDDnodes.dnodes:
# self.TDDnodes.starttaosd(dnode.index)
# # create cluster
# for dnode in self.TDDnodes.dnodes[1:]:
# # print(dnode.cfgDict)
# dnode_id = dnode.cfgDict["fqdn"] + ":" +dnode.cfgDict["serverPort"]
# dnode_first_host = dnode.cfgDict["firstEp"].split(":")[0]
# dnode_first_port = dnode.cfgDict["firstEp"].split(":")[-1]
# cmd = f" taos -h {dnode_first_host} -P {dnode_first_port} -s ' create dnode \"{dnode_id} \" ' ;"
# print(cmd)
# os.system(cmd)
# time.sleep(2)
# tdLog.info(" create cluster with %d dnode done! " %dnodes_nums)
# def buildcluster(self,dnodenumber):
# self.depoly_cluster(dnodenumber)
# self.master_dnode = self.TDDnodes.dnodes[0]
# self.host=self.master_dnode.cfgDict["fqdn"]
# conn1 = taos.connect(self.master_dnode.cfgDict["fqdn"] , config=self.master_dnode.cfgDir)
# tdSql.init(conn1.cursor())
# def checkdnodes(self,dnodenumber):
# count=0
# while count < 10:
# time.sleep(1)
# statusReadyBumber=0
# tdSql.query("show dnodes;")
# if tdSql.checkRows(dnodenumber) :
# print("dnode is %d nodes"%dnodenumber)
# for i in range(dnodenumber):
# if tdSql.queryResult[i][4] !='ready' :
# status=tdSql.queryResult[i][4]
# print("dnode:%d status is %s "%(i,status))
# break
# else:
# statusReadyBumber+=1
# print(statusReadyBumber)
# if statusReadyBumber == dnodenumber :
# print("all of %d mnodes is ready in 10s "%dnodenumber)
# return True
# break
# count+=1
# else:
# print("%d mnodes is not ready in 10s "%dnodenumber)
# return False
# def check3mnode(self):
# count=0
# while count < 10:
# time.sleep(1)
# tdSql.query("show mnodes;")
# if tdSql.checkRows(3) :
# print("mnode is three nodes")
# if tdSql.queryResult[0][2]=='leader' :
# if tdSql.queryResult[1][2]=='follower':
# if tdSql.queryResult[2][2]=='follower':
# print("three mnodes is ready in 10s")
# break
# elif tdSql.queryResult[0][2]=='follower' :
# if tdSql.queryResult[1][2]=='leader':
# if tdSql.queryResult[2][2]=='follower':
# print("three mnodes is ready in 10s")
# break
# elif tdSql.queryResult[0][2]=='follower' :
# if tdSql.queryResult[1][2]=='follower':
# if tdSql.queryResult[2][2]=='leader':
# print("three mnodes is ready in 10s")
# break
# count+=1
# else:
# print("three mnodes is not ready in 10s ")
# return -1
# tdSql.query("show mnodes;")
# tdSql.checkRows(3)
# tdSql.checkData(0,1,'%s:6030'%self.host)
# tdSql.checkData(0,3,'ready')
# tdSql.checkData(1,1,'%s:6130'%self.host)
# tdSql.checkData(1,3,'ready')
# tdSql.checkData(2,1,'%s:6230'%self.host)
# tdSql.checkData(2,3,'ready')
# def check3mnode1off(self):
# count=0
# while count < 10:
# time.sleep(1)
# tdSql.query("show mnodes;")
# if tdSql.checkRows(3) :
# print("mnode is three nodes")
# if tdSql.queryResult[0][2]=='offline' :
# if tdSql.queryResult[1][2]=='leader':
# if tdSql.queryResult[2][2]=='follower':
# print("stop mnodes on dnode 2 successfully in 10s")
# break
# elif tdSql.queryResult[1][2]=='follower':
# if tdSql.queryResult[2][2]=='leader':
# print("stop mnodes on dnode 2 successfully in 10s")
# break
# count+=1
# else:
# print("stop mnodes on dnode 2 failed in 10s ")
# return -1
# tdSql.error("drop mnode on dnode 1;")
# tdSql.query("show mnodes;")
# tdSql.checkRows(3)
# tdSql.checkData(0,1,'%s:6030'%self.host)
# tdSql.checkData(0,2,'offline')
# tdSql.checkData(0,3,'ready')
# tdSql.checkData(1,1,'%s:6130'%self.host)
# tdSql.checkData(1,3,'ready')
# tdSql.checkData(2,1,'%s:6230'%self.host)
# tdSql.checkData(2,3,'ready')
# def check3mnode2off(self):
# count=0
# while count < 40:
# time.sleep(1)
# tdSql.query("show mnodes;")
# if tdSql.checkRows(3) :
# print("mnode is three nodes")
# if tdSql.queryResult[0][2]=='leader' :
# if tdSql.queryResult[1][2]=='offline':
# if tdSql.queryResult[2][2]=='follower':
# print("stop mnodes on dnode 2 successfully in 10s")
# break
# count+=1
# else:
# print("stop mnodes on dnode 2 failed in 10s ")
# return -1
# tdSql.error("drop mnode on dnode 2;")
# tdSql.query("show mnodes;")
# tdSql.checkRows(3)
# tdSql.checkData(0,1,'%s:6030'%self.host)
# tdSql.checkData(0,2,'leader')
# tdSql.checkData(0,3,'ready')
# tdSql.checkData(1,1,'%s:6130'%self.host)
# tdSql.checkData(1,2,'offline')
# tdSql.checkData(1,3,'ready')
# tdSql.checkData(2,1,'%s:6230'%self.host)
# tdSql.checkData(2,2,'follower')
# tdSql.checkData(2,3,'ready')
# def check3mnode3off(self):
# count=0
# while count < 10:
# time.sleep(1)
# tdSql.query("show mnodes;")
# if tdSql.checkRows(3) :
# print("mnode is three nodes")
# if tdSql.queryResult[0][2]=='leader' :
# if tdSql.queryResult[2][2]=='offline':
# if tdSql.queryResult[1][2]=='follower':
# print("stop mnodes on dnode 3 successfully in 10s")
# break
# count+=1
# else:
# print("stop mnodes on dnode 3 failed in 10s")
# return -1
# tdSql.error("drop mnode on dnode 3;")
# tdSql.query("show mnodes;")
# tdSql.checkRows(3)
# tdSql.checkData(0,1,'%s:6030'%self.host)
# tdSql.checkData(0,2,'leader')
# tdSql.checkData(0,3,'ready')
# tdSql.checkData(1,1,'%s:6130'%self.host)
# tdSql.checkData(1,2,'follower')
# tdSql.checkData(1,3,'ready')
# tdSql.checkData(2,1,'%s:6230'%self.host)
# tdSql.checkData(2,2,'offline')
# tdSql.checkData(2,3,'ready')

1
tests/script/general/parser/udf.sim
View File

@ -44,6 +44,7 @@ endi
sql create function n02 as '/tmp/normal' outputtype bool;
sql show functions;
if $rows != 1 then
print expect 1, actual $rows
return -1
endi

2
tests/script/tmp/monitor.sim
View File

@ -12,6 +12,8 @@ sql connect
print =============== show dnodes
sleep 2000
sql create database db vgroups 2;
sql use db;
sql create table db.stb (ts timestamp, c1 int, c2 binary(4)) tags(t1 int, t2 binary(16)) comment "abd";
sleep 2000
print =============== create drop qnode 1

1
tests/script/tsim/mnode/basic5.sim
View File

@ -233,7 +233,6 @@ if $data(1)[3] != dropping then
endi
print =============== step9: start mnode1 and wait it dropped
sleep 3000
system sh/exec.sh -n dnode1 -s start
$x = 0

1
tests/script/tsim/query/udf.sim
View File

@ -44,6 +44,7 @@ endi
sql select udf2(f) from t;
if $rows != 1 then
print expect 1, actual $rows
return -1
endi
if $data00 != 2.236067977 then

10
tests/system-test/6-cluster/5dnode3mnodeStop.py
View File

@ -159,13 +159,10 @@ class TDTestCase:
tdSql.query("show mnodes;")
tdSql.checkRows(3)
tdSql.checkData(0,1,'%s:6030'%self.host)
tdSql.checkData(0,2,'leader')
tdSql.checkData(0,3,'ready')
tdSql.checkData(1,1,'%s:6130'%self.host)
tdSql.checkData(1,2,'offline')
tdSql.checkData(1,3,'ready')
tdSql.checkData(2,1,'%s:6230'%self.host)
tdSql.checkData(2,2,'follower')
tdSql.checkData(2,3,'ready')
def check3mnode3off(self):
@ -265,13 +262,6 @@ class TDTestCase:
while stopcount <= 2:
for i in range(dnodenumber):
tdDnodes[i].stoptaosd()
# if i == 1 :
# self.check3mnode2off()
# elif i == 2 :
# self.check3mnode3off()
# elif i == 0:
# self.check3mnode1off()
tdDnodes[i].starttaosd()
# self.check3mnode()
stopcount+=1

2
tests/system-test/fulltest.sh
View File

@ -112,7 +112,7 @@ python3 ./test.py -f 2-query/twa.py
python3 ./test.py -f 6-cluster/5dnode1mnode.py
python3 ./test.py -f 6-cluster/5dnode2mnode.py
python3 ./test.py -f 6-cluster/5dnode3mnodeStop.py -N 5
python3 ./test.py -f 6-cluster/5dnode3mnodeStop.py -N 5 -M 3
# python3 ./test.py -f 6-cluster/5dnode3mnodeDrop.py -N 5
# BUG python3 ./test.py -f 6-cluster/5dnode3mnodeStopInsert.py

16
tests/system-test/test.py
View File

@ -60,10 +60,11 @@ if __name__ == "__main__":
stop = 0
restart = False
dnodeNums = 1
mnodeNums = 0
updateCfgDict = {}
execCmd = ""
opts, args = getopt.gnu_getopt(sys.argv[1:], 'f:p:m:l:scghrd:k:e:N:', [
'file=', 'path=', 'master', 'logSql', 'stop', 'cluster', 'valgrind', 'help', 'restart', 'updateCfgDict', 'killv', 'execCmd','dnodeNums'])
opts, args = getopt.gnu_getopt(sys.argv[1:], 'f:p:m:l:scghrd:k:e:N:M:', [
'file=', 'path=', 'master', 'logSql', 'stop', 'cluster', 'valgrind', 'help', 'restart', 'updateCfgDict', 'killv', 'execCmd','dnodeNums','mnodeNums'])
for key, value in opts:
if key in ['-h', '--help']:
tdLog.printNoPrefix(
@ -79,7 +80,8 @@ if __name__ == "__main__":
tdLog.printNoPrefix('-d update cfg dict, base64 json str')
tdLog.printNoPrefix('-k not kill valgrind processer')
tdLog.printNoPrefix('-e eval str to run')
tdLog.printNoPrefix('-N create dnodes numbers clusters')
tdLog.printNoPrefix('-N create dnodes numbers in clusters')
tdLog.printNoPrefix('-M create mnode numbers in clusters')
sys.exit(0)
@ -133,6 +135,9 @@ if __name__ == "__main__":
if key in ['-N', '--dnodeNums']:
dnodeNums = value
if key in ['-M', '--mnodeNums']:
mnodeNums = value
if not execCmd == "":
tdDnodes.init(deployPath)
print(execCmd)
@ -244,9 +249,8 @@ if __name__ == "__main__":
tdDnodes.start(1)
tdCases.logSql(logSql)
else :
print("start cluster and dnodes number")
dnodeslist = cluster.configure_cluster(dnodes_nums=dnodeNums,independent=True)
tdLog.debug("create an cluster with %s nodes and make %s dnode as independent mnode"%(dnodeNums,mnodeNums))
dnodeslist = cluster.configure_cluster(dnodeNums=dnodeNums,mnodeNums=mnodeNums)
tdDnodes = ClusterDnodes(dnodeslist)
tdDnodes.init(deployPath, masterIp)
tdDnodes.setTestCluster(testCluster)