diff --git a/include/client/taos.h b/include/client/taos.h index d31d5c582c..5e7f12de0a 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -128,7 +128,6 @@ typedef struct setConfRet { DLL_EXPORT void taos_cleanup(void); DLL_EXPORT int taos_options(TSDB_OPTION option, const void *arg, ...); -DLL_EXPORT setConfRet taos_set_config(const char *config); DLL_EXPORT int taos_init(void); DLL_EXPORT TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port); DLL_EXPORT TAOS *taos_connect_auth(const char *ip, const char *user, const char *auth, const char *db, uint16_t port); diff --git a/include/common/tcommon.h b/include/common/tcommon.h index 460f7b3b58..09f24cc555 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -69,14 +69,13 @@ typedef struct SColumnDataAgg { typedef struct SDataBlockInfo { STimeWindow window; - int32_t rows; + int32_t rows; // todo hide this attribute int32_t rowSize; - int64_t uid; // the uid of table, from which current data block comes - int64_t blockId; // block id, generated by physical planner + uint64_t uid; // the uid of table, from which current data block comes + uint16_t blockId; // block id, generated by physical planner uint64_t groupId; // no need to serialize - int16_t numOfCols; int16_t hasVarCol; - int32_t capacity; + uint32_t capacity; // TODO: optimize and remove following int32_t childId; // used for stream, do not serialize EStreamType type; // used for stream, do not serialize @@ -97,7 +96,7 @@ typedef struct SVarColAttr { // pBlockAgg->numOfNull == info.rows, all data are null // pBlockAgg->numOfNull == 0, no data are null. typedef struct SColumnInfoData { - SColumnInfo info; // TODO filter info needs to be removed + SColumnInfo info; // column info bool hasNull; // if current column data has null value. char* pData; // the corresponding block data in memory union { diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index 92af7208a2..2a0d4e7ff6 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -71,8 +71,8 @@ SEpSet getEpSet_s(SCorEpSet* pEpSet); #define colDataGetData(p1_, r_) \ ((IS_VAR_DATA_TYPE((p1_)->info.type)) ? colDataGetVarData(p1_, r_) : colDataGetNumData(p1_, r_)) -#define IS_JSON_NULL(type, data) ((type) == TSDB_DATA_TYPE_JSON && \ - (*(data) == TSDB_DATA_TYPE_NULL || tTagIsJsonNull(data))) +#define IS_JSON_NULL(type, data) \ + ((type) == TSDB_DATA_TYPE_JSON && (*(data) == TSDB_DATA_TYPE_NULL || tTagIsJsonNull(data))) static FORCE_INLINE bool colDataIsNull_s(const SColumnInfoData* pColumnInfoData, uint32_t row) { if (!pColumnInfoData->hasNull) { @@ -184,9 +184,10 @@ static FORCE_INLINE void colDataAppendDouble(SColumnInfoData* pColumnInfoData, u int32_t getJsonValueLen(const char* data); int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, const char* pData, bool isNull); -int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, int32_t* capacity, +int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, uint32_t* capacity, const SColumnInfoData* pSource, uint32_t numOfRow2); -int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* pSource, int32_t numOfRows); +int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* pSource, int32_t numOfRows, + const SDataBlockInfo* pBlockInfo); int32_t blockDataUpdateTsWindow(SSDataBlock* pDataBlock, int32_t tsColumnIndex); int32_t colDataGetLength(const SColumnInfoData* pColumnInfoData, int32_t numOfRows); @@ -212,7 +213,7 @@ size_t blockDataGetSerialMetaSize(uint32_t numOfCols); int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo); int32_t blockDataSort_rv(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullFirst); -int32_t colInfoDataEnsureCapacity(SColumnInfoData* pColumn, size_t existRows, uint32_t numOfRows); +int32_t colInfoDataEnsureCapacity(SColumnInfoData* pColumn, uint32_t numOfRows); int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows); void colInfoDataCleanup(SColumnInfoData* pColumn, uint32_t numOfRows); @@ -222,7 +223,13 @@ size_t blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize); int32_t blockDataTrimFirstNRows(SSDataBlock* pBlock, size_t n); +int32_t assignOneDataBlock(SSDataBlock* dst, const SSDataBlock* src); SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData); +SSDataBlock* createDataBlock(); +int32_t blockDataAppendColInfo(SSDataBlock* pBlock, SColumnInfoData* pColInfoData); + +SColumnInfoData createColumnInfoData(int16_t type, int32_t bytes, int16_t colId); +SColumnInfoData* bdGetColumnInfoData(SSDataBlock* pBlock, int32_t index); void blockCompressEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_t numOfCols, int8_t needCompress); @@ -236,7 +243,7 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks char* buildCtbNameByGroupId(const char* stbName, uint64_t groupId); static FORCE_INLINE int32_t blockGetEncodeSize(const SSDataBlock* pBlock) { - return blockDataGetSerialMetaSize(pBlock->info.numOfCols) + blockDataGetSize(pBlock); + return blockDataGetSerialMetaSize(taosArrayGetSize(pBlock->pDataBlock)) + blockDataGetSize(pBlock); } static FORCE_INLINE int32_t blockCompressColData(SColumnInfoData* pColRes, int32_t numOfRows, char* data, diff --git a/include/common/tglobal.h b/include/common/tglobal.h index e083ebcf78..41674b7a70 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -151,6 +151,7 @@ void taosCfgDynamicOptions(const char *option, const char *value); void taosAddDataDir(int32_t index, char *v1, int32_t level, int32_t primary); struct SConfig *taosGetCfg(); +int32_t taosSetCfg(SConfig *pCfg, char* name); #ifdef __cplusplus } diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 93eab2fd22..e9ee8b5955 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -629,11 +629,6 @@ typedef struct { uint8_t scale; } SColumnInfo; -typedef struct { - int64_t uid; - TSKEY key; // last accessed ts, for subscription -} STableIdInfo; - typedef struct STimeWindow { TSKEY skey; TSKEY ekey; @@ -1346,11 +1341,19 @@ typedef struct { int32_t dnodeId; char config[TSDB_DNODE_CONFIG_LEN]; char value[TSDB_DNODE_VALUE_LEN]; -} SMCfgDnodeReq, SDCfgDnodeReq; +} SMCfgDnodeReq; int32_t tSerializeSMCfgDnodeReq(void* buf, int32_t bufLen, SMCfgDnodeReq* pReq); int32_t tDeserializeSMCfgDnodeReq(void* buf, int32_t bufLen, SMCfgDnodeReq* pReq); +typedef struct { + char config[TSDB_DNODE_CONFIG_LEN]; + char value[TSDB_DNODE_VALUE_LEN]; +} SDCfgDnodeReq; + +int32_t tSerializeSDCfgDnodeReq(void* buf, int32_t bufLen, SDCfgDnodeReq* pReq); +int32_t tDeserializeSDCfgDnodeReq(void* buf, int32_t bufLen, SDCfgDnodeReq* pReq); + typedef struct { int32_t dnodeId; } SMCreateMnodeReq, SMDropMnodeReq, SDDropMnodeReq, SMCreateQnodeReq, SMDropQnodeReq, SDCreateQnodeReq, SDDropQnodeReq, diff --git a/include/common/tname.h b/include/common/tname.h index c691c2f7b2..3bf1cee870 100644 --- a/include/common/tname.h +++ b/include/common/tname.h @@ -57,6 +57,8 @@ void tNameAssign(SName* dst, const SName* src); int32_t tNameSetDbName(SName* dst, int32_t acctId, const char* dbName, size_t nameLen); +int32_t tNameAddTbName(SName* dst, const char* tbName, size_t nameLen); + int32_t tNameFromString(SName* dst, const char* str, uint32_t type); int32_t tNameSetAcctId(SName* dst, int32_t acctId); diff --git a/include/dnode/mnode/mnode.h b/include/dnode/mnode/mnode.h index 2509c10601..70056783ea 100644 --- a/include/dnode/mnode/mnode.h +++ b/include/dnode/mnode/mnode.h @@ -73,10 +73,12 @@ void mndStop(SMnode *pMnode); * @param pMnode The mnode object. * @param pCluster * @param pVgroup + * @param pStbInfo * @param pGrant * @return int32_t 0 for success, -1 for failure. */ -int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pCluster, SMonVgroupInfo *pVgroup, SMonGrantInfo *pGrant); +int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo, + SMonStbInfo *pStbInfo, SMonGrantInfo *pGrantInfo); /** * @brief Get mnode loads for status msg. diff --git a/include/libs/function/function.h b/include/libs/function/function.h index e543f0f653..a569c8de54 100644 --- a/include/libs/function/function.h +++ b/include/libs/function/function.h @@ -172,23 +172,6 @@ typedef struct tExprNode { void tExprTreeDestroy(tExprNode *pNode, void (*fp)(void *)); -typedef struct SAggFunctionInfo { - char name[FUNCTIONS_NAME_MAX_LENGTH]; - int8_t type; // Scalar function or aggregation function - uint32_t functionId; // Function Id - int8_t sFunctionId; // Transfer function for super table query - uint16_t status; - - bool (*init)(SqlFunctionCtx *pCtx, struct SResultRowEntryInfo* pResultCellInfo); // setup the execute environment - void (*addInput)(SqlFunctionCtx *pCtx); - - // finalizer must be called after all exec has been executed to generated final result. - void (*finalize)(SqlFunctionCtx *pCtx); - void (*combine)(SqlFunctionCtx *pCtx); - - int32_t (*dataReqFunc)(SqlFunctionCtx *pCtx, STimeWindow* w, int32_t colId); -} SAggFunctionInfo; - struct SScalarParam { SColumnInfoData *columnData; SHashObj *pHashFilter; diff --git a/include/libs/monitor/monitor.h b/include/libs/monitor/monitor.h index 39e8042b93..41d80c1a83 100644 --- a/include/libs/monitor/monitor.h +++ b/include/libs/monitor/monitor.h @@ -109,6 +109,9 @@ typedef struct { char version[MON_VER_LEN]; float master_uptime; // day int32_t monitor_interval; // sec + int32_t dbs_total; + int32_t stbs_total; + int64_t tbs_total; int32_t vgroups_total; int32_t vgroups_alive; int32_t vnodes_total; @@ -135,6 +138,15 @@ typedef struct { SArray *vgroups; // array of SMonVgroupDesc } SMonVgroupInfo; +typedef struct { + char stb_name[TSDB_TABLE_NAME_LEN]; + char database_name[TSDB_DB_NAME_LEN]; +} SMonStbDesc; + +typedef struct { + SArray *stbs; // array of SMonStbDesc +} SMonStbInfo; + typedef struct { int32_t expire_time; int64_t timeseries_used; @@ -144,6 +156,7 @@ typedef struct { typedef struct { SMonClusterInfo cluster; SMonVgroupInfo vgroup; + SMonStbInfo stb; SMonGrantInfo grant; SMonSysInfo sys; SMonLogs log; diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index 26f2758033..d86a8dae6e 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -266,19 +266,19 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t #define qDebug(...) \ do { \ if (qDebugFlag & DEBUG_DEBUG) { \ - taosPrintLog("QRY ", DEBUG_DEBUG, tsLogEmbedded ? 255 : qDebugFlag, __VA_ARGS__); \ + taosPrintLog("QRY ", DEBUG_DEBUG, qDebugFlag, __VA_ARGS__); \ } \ } while (0) #define qTrace(...) \ do { \ if (qDebugFlag & DEBUG_TRACE) { \ - taosPrintLog("QRY ", DEBUG_TRACE, tsLogEmbedded ? 255 : qDebugFlag, __VA_ARGS__); \ + taosPrintLog("QRY ", DEBUG_TRACE, qDebugFlag, __VA_ARGS__); \ } \ } while (0) #define qDebugL(...) \ do { \ if (qDebugFlag & DEBUG_DEBUG) { \ - taosPrintLongString("QRY ", DEBUG_DEBUG, tsLogEmbedded ? 255 : qDebugFlag, __VA_ARGS__); \ + taosPrintLongString("QRY ", DEBUG_DEBUG, qDebugFlag, __VA_ARGS__); \ } \ } while (0) diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 7673e73035..a47810e7c1 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -344,10 +344,12 @@ static FORCE_INLINE int32_t streamTaskOutput(SStreamTask* pTask, SStreamDataBloc if (pTask->sinkType == TASK_SINK__TABLE) { ASSERT(pTask->dispatchType == TASK_DISPATCH__NONE); pTask->tbSink.tbSinkFunc(pTask, pTask->tbSink.vnode, 0, pBlock->blocks); + taosArrayDestroyEx(pBlock->blocks, (FDelete)tDeleteSSDataBlock); taosFreeQitem(pBlock); } else if (pTask->sinkType == TASK_SINK__SMA) { ASSERT(pTask->dispatchType == TASK_DISPATCH__NONE); pTask->smaSink.smaSink(pTask->smaSink.vnode, pTask->smaSink.smaId, pBlock->blocks); + taosArrayDestroyEx(pBlock->blocks, (FDelete)tDeleteSSDataBlock); taosFreeQitem(pBlock); } else { ASSERT(pTask->dispatchType != TASK_DISPATCH__NONE); diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index a0c07f0c09..2bf49fa006 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -157,13 +157,13 @@ typedef struct SSyncLogStore { SyncIndex (*getCommitIndex)(struct SSyncLogStore* pLogStore); // refactor, log[0 .. n] ==> log[m .. n] - int32_t (*syncLogSetBeginIndex)(struct SSyncLogStore* pLogStore, SyncIndex beginIndex); - int32_t (*syncLogResetBeginIndex)(struct SSyncLogStore* pLogStore); + // int32_t (*syncLogSetBeginIndex)(struct SSyncLogStore* pLogStore, SyncIndex beginIndex); + SyncIndex (*syncLogBeginIndex)(struct SSyncLogStore* pLogStore); SyncIndex (*syncLogEndIndex)(struct SSyncLogStore* pLogStore); bool (*syncLogIsEmpty)(struct SSyncLogStore* pLogStore); int32_t (*syncLogEntryCount)(struct SSyncLogStore* pLogStore); - // bool (*syncLogInRange)(struct SSyncLogStore* pLogStore, SyncIndex index); + int32_t (*syncLogRestoreFromSnapshot)(struct SSyncLogStore* pLogStore, SyncIndex index); SyncIndex (*syncLogWriteIndex)(struct SSyncLogStore* pLogStore); SyncIndex (*syncLogLastIndex)(struct SSyncLogStore* pLogStore); diff --git a/include/util/tconfig.h b/include/util/tconfig.h index fd6360aaf6..102656da2b 100644 --- a/include/util/tconfig.h +++ b/include/util/tconfig.h @@ -33,7 +33,8 @@ typedef enum { CFG_STYPE_ENV_CMD, CFG_STYPE_APOLLO_URL, CFG_STYPE_ARG_LIST, - CFG_STYPE_TAOS_OPTIONS + CFG_STYPE_TAOS_OPTIONS, + CFG_STYPE_ALTER_CMD, } ECfgSrcType; typedef enum { diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index cfe507e505..2b918412fe 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -215,6 +215,7 @@ typedef struct SRequestObj { SQueryExecMetric metric; SRequestSendRecvBody body; bool stableQuery; + bool validateOnly; bool killed; uint32_t prevCode; // previous error code: todo refactor, add update flag for catalog @@ -235,7 +236,12 @@ int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveT bool freeAfterUse); void setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t numOfCols); void doFreeReqResultInfo(SReqResultInfo* pResInfo); -SRequestObj* execQuery(STscObj* pTscObj, const char* sql, int sqlLen); +int32_t transferTableNameList(const char* tbList, int32_t acctId, char* dbName, SArray** pReq); +void syncCatalogFn(SMetaData* pResult, void* param, int32_t code); + +SRequestObj* execQuery(STscObj* pTscObj, const char* sql, int sqlLen, bool validateOnly); +TAOS_RES *taosQueryImpl(TAOS *taos, const char *sql, bool validateOnly); +void taosAsyncQueryImpl(TAOS *taos, const char *sql, __taos_async_fn_t fp, void *param, bool validateOnly); static FORCE_INLINE SReqResultInfo* tmqGetCurResInfo(TAOS_RES* res) { SMqRspObj* msg = (SMqRspObj*)res; @@ -301,7 +307,7 @@ void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet); STscObj* taos_connect_internal(const char* ip, const char* user, const char* pass, const char* auth, const char* db, uint16_t port, int connType); -SRequestObj* launchQuery(STscObj* pTscObj, const char* sql, int sqlLen); +SRequestObj* launchQuery(STscObj* pTscObj, const char* sql, int sqlLen, bool validateOnly); int32_t parseSql(SRequestObj* pRequest, bool topicQuery, SQuery** pQuery, SStmtCallback* pStmtCb); diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 191bfe088a..51e709ac58 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -237,6 +237,10 @@ static SAppInstInfo* getAppInfo(SRequestObj* pRequest) { return pRequest->pTscOb void asyncExecLocalCmd(SRequestObj* pRequest, SQuery* pQuery) { SRetrieveTableRsp* pRsp = NULL; + if (pRequest->validateOnly) { + pRequest->body.queryFp(pRequest->body.param, pRequest, 0); + return; + } int32_t code = qExecCommand(pQuery->pRoot, &pRsp); if (TSDB_CODE_SUCCESS == code && NULL != pRsp) { @@ -261,6 +265,11 @@ void asyncExecLocalCmd(SRequestObj* pRequest, SQuery* pQuery) { } int32_t asyncExecDdlQuery(SRequestObj* pRequest, SQuery* pQuery) { + if (pRequest->validateOnly) { + pRequest->body.queryFp(pRequest->body.param, pRequest, 0); + return TSDB_CODE_SUCCESS; + } + // drop table if exists not_exists_table if (NULL == pQuery->pCmdMsg) { pRequest->body.queryFp(pRequest->body.param, pRequest, 0); @@ -276,8 +285,11 @@ int32_t asyncExecDdlQuery(SRequestObj* pRequest, SQuery* pQuery) { SMsgSendInfo* pSendMsg = buildMsgInfoImpl(pRequest); int64_t transporterId = 0; - asyncSendMsgToServer(pAppInfo->pTransporter, &pMsgInfo->epSet, &transporterId, pSendMsg); - return TSDB_CODE_SUCCESS; + int32_t code = asyncSendMsgToServer(pAppInfo->pTransporter, &pMsgInfo->epSet, &transporterId, pSendMsg); + if (code) { + pRequest->body.queryFp(pRequest->body.param, pRequest, code); + } + return code; } int compareQueryNodeLoad(const void* elem1, const void* elem2) { @@ -851,15 +863,19 @@ SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, bool keepQue switch (pQuery->execMode) { case QUERY_EXEC_MODE_LOCAL: - code = execLocalCmd(pRequest, pQuery); + if (!pRequest->validateOnly) { + code = execLocalCmd(pRequest, pQuery); + } break; case QUERY_EXEC_MODE_RPC: - code = execDdlQuery(pRequest, pQuery); + if (!pRequest->validateOnly) { + code = execDdlQuery(pRequest, pQuery); + } break; case QUERY_EXEC_MODE_SCHEDULE: { SArray* pMnodeList = taosArrayInit(4, sizeof(SQueryNodeLoad)); code = getPlan(pRequest, pQuery, &pRequest->body.pDag, pMnodeList); - if (TSDB_CODE_SUCCESS == code) { + if (TSDB_CODE_SUCCESS == code && !pRequest->validateOnly) { SArray* pNodeList = NULL; buildSyncExecNodeList(pRequest, &pNodeList, pMnodeList); @@ -894,7 +910,7 @@ SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, bool keepQue return pRequest; } -SRequestObj* launchQuery(STscObj* pTscObj, const char* sql, int sqlLen) { +SRequestObj* launchQuery(STscObj* pTscObj, const char* sql, int sqlLen, bool validateOnly) { SRequestObj* pRequest = NULL; SQuery* pQuery = NULL; @@ -904,6 +920,8 @@ SRequestObj* launchQuery(STscObj* pTscObj, const char* sql, int sqlLen) { return NULL; } + pRequest->validateOnly = validateOnly; + code = parseSql(pRequest, false, &pQuery, NULL); if (code != TSDB_CODE_SUCCESS) { pRequest->code = code; @@ -945,7 +963,7 @@ void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData *pResultM pRequest->requestId); } - if (TSDB_CODE_SUCCESS == code) { + if (TSDB_CODE_SUCCESS == code && !pRequest->validateOnly) { SArray* pNodeList = NULL; buildAsyncExecNodeList(pRequest, &pNodeList, pMnodeList, pResultMeta); @@ -962,7 +980,7 @@ void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData *pResultM code = schedulerAsyncExecJob(&req, &pRequest->body.queryJob); taosArrayDestroy(pNodeList); } else { - tscError("0x%" PRIx64 " failed to create query plan, code:%s 0x%" PRIx64, pRequest->self, tstrerror(code), + tscDebug("0x%" PRIx64 " plan not executed, code:%s 0x%" PRIx64, pRequest->self, tstrerror(code), pRequest->requestId); pRequest->body.queryFp(pRequest->body.param, pRequest, code); } @@ -1045,14 +1063,14 @@ int32_t removeMeta(STscObj* pTscObj, SArray* tbList) { return TSDB_CODE_SUCCESS; } -SRequestObj* execQuery(STscObj* pTscObj, const char* sql, int sqlLen) { +SRequestObj* execQuery(STscObj* pTscObj, const char* sql, int sqlLen, bool validateOnly) { SRequestObj* pRequest = NULL; int32_t retryNum = 0; int32_t code = 0; do { destroyRequest(pRequest); - pRequest = launchQuery(pTscObj, sql, sqlLen); + pRequest = launchQuery(pTscObj, sql, sqlLen, validateOnly); if (pRequest == NULL || TSDB_CODE_SUCCESS == pRequest->code || !NEED_CLIENT_HANDLE_ERROR(pRequest->code)) { break; } @@ -1819,3 +1837,251 @@ _OVER: } return code; } + +int32_t appendTbToReq(SArray* pList, int32_t pos1, int32_t len1, int32_t pos2, int32_t len2, const char* str, int32_t acctId, char* db) { + SName name; + + if (len1 <= 0) { + return -1; + } + + const char *dbName = db; + const char *tbName = NULL; + int32_t dbLen = 0; + int32_t tbLen = 0; + if (len2 > 0) { + dbName = str + pos1; + dbLen = len1; + tbName = str + pos2; + tbLen = len2; + } else { + dbLen = strlen(db); + tbName = str + pos1; + tbLen = len1; + } + + if (tNameSetDbName(&name, acctId, dbName, dbLen)) { + return -1; + } + + if (tNameAddTbName(&name, tbName, tbLen)) { + return -1; + } + + taosArrayPush(pList, &name); + + return TSDB_CODE_SUCCESS; +} + +int32_t transferTableNameList(const char* tbList, int32_t acctId, char* dbName, SArray** pReq) { + *pReq = taosArrayInit(10, sizeof(SName)); + if (NULL == *pReq) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return terrno; + } + + bool inEscape = false; + int32_t code = 0; + + int32_t vIdx = 0; + int32_t vPos[2]; + int32_t vLen[2]; + + memset(vPos, -1, sizeof(vPos)); + memset(vLen, 0, sizeof(vLen)); + + for (int32_t i = 0; ; ++i) { + if (0 == *(tbList + i)) { + if (vPos[vIdx] >= 0 && vLen[vIdx] <= 0) { + vLen[vIdx] = i - vPos[vIdx]; + } + + code = appendTbToReq(*pReq, vPos[0], vLen[0], vPos[1], vLen[1], tbList, acctId, dbName); + if (code) { + goto _return; + } + + break; + } + + if ('`' == *(tbList + i)) { + inEscape = !inEscape; + if (!inEscape) { + if (vPos[vIdx] >= 0) { + vLen[vIdx] = i - vPos[vIdx]; + } else { + goto _return; + } + } + + continue; + } + + if (inEscape) { + if (vPos[vIdx] < 0) { + vPos[vIdx] = i; + } + continue; + } + + if ('.' == *(tbList + i)) { + if (vPos[vIdx] < 0) { + goto _return; + } + if (vLen[vIdx] <= 0) { + vLen[vIdx] = i - vPos[vIdx]; + } + vIdx++; + if (vIdx >= 2) { + goto _return; + } + continue; + } + + if (',' == *(tbList + i)) { + if (vPos[vIdx] < 0) { + goto _return; + } + if (vLen[vIdx] <= 0) { + vLen[vIdx] = i - vPos[vIdx]; + } + + code = appendTbToReq(*pReq, vPos[0], vLen[0], vPos[1], vLen[1], tbList, acctId, dbName); + if (code) { + goto _return; + } + + memset(vPos, -1, sizeof(vPos)); + memset(vLen, 0, sizeof(vLen)); + vIdx = 0; + continue; + } + + if (' ' == *(tbList + i) || '\r' == *(tbList + i) || '\t' == *(tbList + i) || '\n' == *(tbList + i)) { + if (vPos[vIdx] >= 0 && vLen[vIdx] <= 0) { + vLen[vIdx] = i - vPos[vIdx]; + } + continue; + } + + if (('a' <= *(tbList + i) && 'z' >= *(tbList + i)) || + ('A' <= *(tbList + i) && 'Z' >= *(tbList + i)) || + ('0' <= *(tbList + i) && '9' >= *(tbList + i))) { + if (vLen[vIdx] > 0) { + goto _return; + } + if (vPos[vIdx] < 0) { + vPos[vIdx] = i; + } + continue; + } + + goto _return; + } + + return TSDB_CODE_SUCCESS; + +_return: + + terrno = TSDB_CODE_TSC_INVALID_OPERATION; + + taosArrayDestroy(*pReq); + *pReq = NULL; + + return terrno; +} + +void syncCatalogFn(SMetaData* pResult, void* param, int32_t code) { + SSyncQueryParam *pParam = param; + pParam->pRequest->code = code; + + tsem_post(&pParam->sem); +} + + +void syncQueryFn(void *param, void *res, int32_t code) { + SSyncQueryParam *pParam = param; + pParam->pRequest = res; + pParam->pRequest->code = code; + + tsem_post(&pParam->sem); +} + +void taosAsyncQueryImpl(TAOS *taos, const char *sql, __taos_async_fn_t fp, void *param, bool validateOnly) { + STscObj *pTscObj = acquireTscObj(*(int64_t *)taos); + if (pTscObj == NULL || sql == NULL || NULL == fp) { + terrno = TSDB_CODE_INVALID_PARA; + if (pTscObj) { + releaseTscObj(*(int64_t *)taos); + } else { + terrno = TSDB_CODE_TSC_DISCONNECTED; + } + fp(param, NULL, terrno); + return; + } + + size_t sqlLen = strlen(sql); + if (sqlLen > (size_t)TSDB_MAX_ALLOWED_SQL_LEN) { + tscError("sql string exceeds max length:%d", TSDB_MAX_ALLOWED_SQL_LEN); + terrno = TSDB_CODE_TSC_EXCEED_SQL_LIMIT; + + fp(param, NULL, terrno); + return; + } + + SRequestObj *pRequest = NULL; + int32_t code = buildRequest(pTscObj, sql, sqlLen, &pRequest); + if (code != TSDB_CODE_SUCCESS) { + terrno = code; + fp(param, NULL, terrno); + return; + } + + pRequest->validateOnly = validateOnly; + pRequest->body.queryFp = fp; + pRequest->body.param = param; + doAsyncQuery(pRequest, false); +} + + +TAOS_RES *taosQueryImpl(TAOS *taos, const char *sql, bool validateOnly) { + if (NULL == taos) { + terrno = TSDB_CODE_TSC_DISCONNECTED; + return NULL; + } + + STscObj *pTscObj = acquireTscObj(*(int64_t *)taos); + if (pTscObj == NULL || sql == NULL) { + terrno = TSDB_CODE_TSC_DISCONNECTED; + return NULL; + } + +#if SYNC_ON_TOP_OF_ASYNC + SSyncQueryParam *param = taosMemoryCalloc(1, sizeof(SSyncQueryParam)); + tsem_init(¶m->sem, 0, 0); + + taosAsyncQueryImpl(taos, sql, syncQueryFn, param, validateOnly); + tsem_wait(¶m->sem); + + releaseTscObj(*(int64_t *)taos); + + return param->pRequest; +#else + size_t sqlLen = strlen(sql); + if (sqlLen > (size_t)TSDB_MAX_ALLOWED_SQL_LEN) { + releaseTscObj(*(int64_t *)taos); + tscError("sql string exceeds max length:%d", TSDB_MAX_ALLOWED_SQL_LEN); + terrno = TSDB_CODE_TSC_EXCEED_SQL_LIMIT; + return NULL; + } + + TAOS_RES *pRes = execQuery(pTscObj, sql, sqlLen, validateOnly); + + releaseTscObj(*(int64_t *)taos); + + return pRes; +#endif +} + + + diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 22747425f0..515136bb27 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -81,12 +81,6 @@ void taos_cleanup(void) { taosCloseLog(); } -setConfRet taos_set_config(const char *config) { - // TODO - setConfRet ret = {SET_CONF_RET_SUCC, {0}}; - return ret; -} - TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port) { tscDebug("try to connect to %s:%u, user:%s db:%s", ip, port, user, db); if (user == NULL) { @@ -205,51 +199,9 @@ TAOS_FIELD *taos_fetch_fields(TAOS_RES *res) { return pResInfo->userFields; } -static void syncQueryFn(void *param, void *res, int32_t code) { - SSyncQueryParam *pParam = param; - pParam->pRequest = res; - pParam->pRequest->code = code; - - tsem_post(&pParam->sem); -} TAOS_RES *taos_query(TAOS *taos, const char *sql) { - if (NULL == taos) { - terrno = TSDB_CODE_TSC_DISCONNECTED; - return NULL; - } - - STscObj *pTscObj = acquireTscObj(*(int64_t *)taos); - if (pTscObj == NULL || sql == NULL) { - terrno = TSDB_CODE_TSC_DISCONNECTED; - return NULL; - } - -#if SYNC_ON_TOP_OF_ASYNC - SSyncQueryParam *param = taosMemoryCalloc(1, sizeof(SSyncQueryParam)); - tsem_init(¶m->sem, 0, 0); - - taos_query_a(taos, sql, syncQueryFn, param); - tsem_wait(¶m->sem); - - releaseTscObj(*(int64_t *)taos); - - return param->pRequest; -#else - size_t sqlLen = strlen(sql); - if (sqlLen > (size_t)TSDB_MAX_ALLOWED_SQL_LEN) { - releaseTscObj(*(int64_t *)taos); - tscError("sql string exceeds max length:%d", TSDB_MAX_ALLOWED_SQL_LEN); - terrno = TSDB_CODE_TSC_EXCEED_SQL_LIMIT; - return NULL; - } - - TAOS_RES *pRes = execQuery(pTscObj, sql, sqlLen); - - releaseTscObj(*(int64_t *)taos); - - return pRes; -#endif + return taosQueryImpl(taos, sql, false); } TAOS_ROW taos_fetch_row(TAOS_RES *res) { @@ -641,7 +593,14 @@ int *taos_get_column_data_offset(TAOS_RES *res, int columnIndex) { return pResInfo->pCol[columnIndex].offset; } -int taos_validate_sql(TAOS *taos, const char *sql) { return true; } +int taos_validate_sql(TAOS *taos, const char *sql) { + TAOS_RES* pObj = taosQueryImpl(taos, sql, true); + + int code = taos_errno(pObj); + + taos_free_result(pObj); + return code; +} void taos_reset_current_db(TAOS *taos) { STscObj *pTscObj = acquireTscObj(*(int64_t *)taos); @@ -731,38 +690,7 @@ void retrieveMetaCallback(SMetaData *pResultMeta, void *param, int32_t code) { } void taos_query_a(TAOS *taos, const char *sql, __taos_async_fn_t fp, void *param) { - STscObj *pTscObj = acquireTscObj(*(int64_t *)taos); - if (pTscObj == NULL || sql == NULL || NULL == fp) { - terrno = TSDB_CODE_INVALID_PARA; - if (pTscObj) { - releaseTscObj(*(int64_t *)taos); - } else { - terrno = TSDB_CODE_TSC_DISCONNECTED; - } - fp(param, NULL, terrno); - return; - } - - size_t sqlLen = strlen(sql); - if (sqlLen > (size_t)TSDB_MAX_ALLOWED_SQL_LEN) { - tscError("sql string exceeds max length:%d", TSDB_MAX_ALLOWED_SQL_LEN); - terrno = TSDB_CODE_TSC_EXCEED_SQL_LIMIT; - - fp(param, NULL, terrno); - return; - } - - SRequestObj *pRequest = NULL; - int32_t code = buildRequest(pTscObj, sql, sqlLen, &pRequest); - if (code != TSDB_CODE_SUCCESS) { - terrno = code; - fp(param, NULL, terrno); - return; - } - - pRequest->body.queryFp = fp; - pRequest->body.param = param; - doAsyncQuery(pRequest, false); + taosAsyncQueryImpl(taos, sql, fp, param, false); } int32_t createParseContext(const SRequestObj *pRequest, SParseContext **pCxt) { @@ -955,10 +883,75 @@ void taos_unsubscribe(TAOS_SUB *tsub, int keepProgress) { } int taos_load_table_info(TAOS *taos, const char *tableNameList) { - // TODO - return -1; + const int32_t MAX_TABLE_NAME_LENGTH = 12 * 1024 * 1024; // 12MB list + int32_t code = 0; + SRequestObj *pRequest = NULL; + SCatalogReq catalogReq = {0}; + + if (NULL == tableNameList) { + return TSDB_CODE_SUCCESS; + } + + int32_t length = (int32_t)strlen(tableNameList); + if (0 == length) { + return TSDB_CODE_SUCCESS; + } else if (length > MAX_TABLE_NAME_LENGTH) { + tscError("tableNameList too long, length:%d, maximum allowed:%d", length, MAX_TABLE_NAME_LENGTH); + return TSDB_CODE_TSC_INVALID_OPERATION; + } + + STscObj *pTscObj = acquireTscObj(*(int64_t *)taos); + if (pTscObj == NULL) { + terrno = TSDB_CODE_TSC_DISCONNECTED; + return terrno; + } + + code = transferTableNameList(tableNameList, pTscObj->acctId, pTscObj->db, &catalogReq.pTableMeta); + if (code) { + goto _return; + } + + SCatalog* pCtg = NULL; + code = catalogGetHandle(pTscObj->pAppInfo->clusterId, &pCtg); + if (code != TSDB_CODE_SUCCESS) { + goto _return; + } + + char* sql = "taos_load_table_info"; + code = buildRequest(pTscObj, sql, strlen(sql), &pRequest); + if (code != TSDB_CODE_SUCCESS) { + terrno = code; + goto _return; + } + + SSyncQueryParam param = {0}; + tsem_init(¶m.sem, 0, 0); + param.pRequest = pRequest; + + SRequestConnInfo conn = {.pTrans = pTscObj->pAppInfo->pTransporter, + .requestId = pRequest->requestId, + .requestObjRefId = pRequest->self}; + + conn.mgmtEps = getEpSet_s(&pTscObj->pAppInfo->mgmtEp); + + code = catalogAsyncGetAllMeta(pCtg, &conn, pRequest->requestId, &catalogReq, syncCatalogFn, ¶m, NULL); + if (code) { + goto _return; + } + + tsem_wait(¶m.sem); + +_return: + + taosArrayDestroy(catalogReq.pTableMeta); + destroyRequest(pRequest); + + releaseTscObj(*(int64_t *)taos); + + return code; } + TAOS_STMT *taos_stmt_init(TAOS *taos) { STscObj *pObj = acquireTscObj(*(int64_t *)taos); if (NULL == pObj) { diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index d5843f699f..593f8c5c0b 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -214,7 +214,7 @@ static void doBitmapMerge(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, c } } -int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, int32_t* capacity, +int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, uint32_t* capacity, const SColumnInfoData* pSource, uint32_t numOfRow2) { ASSERT(pColumnInfoData != NULL && pSource != NULL && pColumnInfoData->info.type == pSource->info.type); if (numOfRow2 == 0) { @@ -263,7 +263,7 @@ int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, in pColumnInfoData->varmeta.length = len + oldLen; } else { if (finalNumOfRows > *capacity) { - ASSERT(finalNumOfRows*pColumnInfoData->info.bytes); + ASSERT(finalNumOfRows * pColumnInfoData->info.bytes); char* tmp = taosMemoryRealloc(pColumnInfoData->pData, finalNumOfRows * pColumnInfoData->info.bytes); if (tmp == NULL) { return TSDB_CODE_VND_OUT_OF_MEMORY; @@ -293,22 +293,19 @@ int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, in return numOfRow1 + numOfRow2; } -int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* pSource, int32_t numOfRows) { +int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* pSource, int32_t numOfRows, + const SDataBlockInfo* pBlockInfo) { ASSERT(pColumnInfoData != NULL && pSource != NULL && pColumnInfoData->info.type == pSource->info.type); if (numOfRows <= 0) { return numOfRows; } + if (pBlockInfo != NULL) { + ASSERT(pBlockInfo->capacity >= numOfRows); + } + if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { - // Handle the bitmap - char* p = taosMemoryRealloc(pColumnInfoData->varmeta.offset, sizeof(int32_t) * numOfRows); - if (p == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } - - pColumnInfoData->varmeta.offset = (int32_t*)p; memcpy(pColumnInfoData->varmeta.offset, pSource->varmeta.offset, sizeof(int32_t) * numOfRows); - if (pColumnInfoData->varmeta.allocLen < pSource->varmeta.length) { char* tmp = taosMemoryRealloc(pColumnInfoData->pData, pSource->varmeta.length); if (tmp == NULL) { @@ -319,27 +316,11 @@ int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* p pColumnInfoData->varmeta.allocLen = pSource->varmeta.length; } - memcpy(pColumnInfoData->pData, pSource->pData, pSource->varmeta.length); pColumnInfoData->varmeta.length = pSource->varmeta.length; + memcpy(pColumnInfoData->pData, pSource->pData, pSource->varmeta.length); } else { - char* tmp = taosMemoryRealloc(pColumnInfoData->nullbitmap, BitmapLen(numOfRows)); - if (tmp == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } - - pColumnInfoData->nullbitmap = tmp; memcpy(pColumnInfoData->nullbitmap, pSource->nullbitmap, BitmapLen(numOfRows)); - - if (pSource->pData) { - int32_t newSize = numOfRows * pColumnInfoData->info.bytes; - tmp = taosMemoryRealloc(pColumnInfoData->pData, newSize); - if (tmp == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } - - pColumnInfoData->pData = tmp; - memcpy(pColumnInfoData->pData, pSource->pData, pSource->info.bytes * numOfRows); - } + memcpy(pColumnInfoData->pData, pSource->pData, pSource->info.bytes * numOfRows); } pColumnInfoData->hasNull = pSource->hasNull; @@ -347,10 +328,7 @@ int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* p return 0; } -size_t blockDataGetNumOfCols(const SSDataBlock* pBlock) { - ASSERT(pBlock && pBlock->info.numOfCols == taosArrayGetSize(pBlock->pDataBlock)); - return pBlock->info.numOfCols; -} +size_t blockDataGetNumOfCols(const SSDataBlock* pBlock) { return taosArrayGetSize(pBlock->pDataBlock); } size_t blockDataGetNumOfRows(const SSDataBlock* pBlock) { return pBlock->info.rows; } @@ -359,7 +337,8 @@ int32_t blockDataUpdateTsWindow(SSDataBlock* pDataBlock, int32_t tsColumnIndex) return 0; } - if (pDataBlock->info.numOfCols <= 0) { + size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock); + if (numOfCols <= 0) { return -1; } @@ -383,7 +362,8 @@ int32_t blockDataMerge(SSDataBlock* pDest, const SSDataBlock* pSrc) { assert(pSrc != NULL && pDest != NULL); int32_t capacity = pDest->info.capacity; - for (int32_t i = 0; i < pDest->info.numOfCols; ++i) { + size_t numOfCols = taosArrayGetSize(pDest->pDataBlock); + for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pCol2 = taosArrayGet(pDest->pDataBlock, i); SColumnInfoData* pCol1 = taosArrayGet(pSrc->pDataBlock, i); @@ -399,9 +379,8 @@ int32_t blockDataMerge(SSDataBlock* pDest, const SSDataBlock* pSrc) { size_t blockDataGetSize(const SSDataBlock* pBlock) { assert(pBlock != NULL); - size_t total = 0; - int32_t numOfCols = pBlock->info.numOfCols; - + size_t total = 0; + size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); total += colDataGetFullLength(pColInfoData, pBlock->info.rows); @@ -416,7 +395,7 @@ int32_t blockDataSplitRows(SSDataBlock* pBlock, bool hasVarCol, int32_t startInd int32_t pageSize) { ASSERT(pBlock != NULL && stopIndex != NULL); - int32_t numOfCols = pBlock->info.numOfCols; + size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); int32_t numOfRows = pBlock->info.rows; int32_t bitmapChar = 1; @@ -477,33 +456,24 @@ SSDataBlock* blockDataExtractBlock(SSDataBlock* pBlock, int32_t startIndex, int3 return NULL; } - SSDataBlock* pDst = taosMemoryCalloc(1, sizeof(SSDataBlock)); + SSDataBlock* pDst = createDataBlock(); if (pDst == NULL) { return NULL; } pDst->info = pBlock->info; - pDst->info.rows = 0; - pDst->pDataBlock = taosArrayInit(pBlock->info.numOfCols, sizeof(SColumnInfoData)); - - for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { + size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); + for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData colInfo = {0}; SColumnInfoData* pSrcCol = taosArrayGet(pBlock->pDataBlock, i); colInfo.info = pSrcCol->info; - - if (IS_VAR_DATA_TYPE(pSrcCol->info.type)) { - SVarColAttr* pAttr = &colInfo.varmeta; - pAttr->offset = taosMemoryCalloc(rowCount, sizeof(int32_t)); - } else { - colInfo.nullbitmap = taosMemoryCalloc(1, BitmapLen(rowCount)); - colInfo.pData = taosMemoryCalloc(rowCount, colInfo.info.bytes); - } - - taosArrayPush(pDst->pDataBlock, &colInfo); + blockDataAppendColInfo(pDst, &colInfo); } - for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { + blockDataEnsureCapacity(pDst, rowCount); + + for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, i); SColumnInfoData* pDstCol = taosArrayGet(pDst->pDataBlock, i); @@ -541,7 +511,7 @@ int32_t blockDataToBuf(char* buf, const SSDataBlock* pBlock) { // write the number of rows *(uint32_t*)buf = pBlock->info.rows; - int32_t numOfCols = pBlock->info.numOfCols; + size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); int32_t numOfRows = pBlock->info.rows; char* pStart = buf + sizeof(uint32_t); @@ -571,7 +541,7 @@ int32_t blockDataToBuf(char* buf, const SSDataBlock* pBlock) { int32_t blockDataFromBuf(SSDataBlock* pBlock, const char* buf) { pBlock->info.rows = *(int32_t*)buf; - int32_t numOfCols = pBlock->info.numOfCols; + size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); const char* pStart = buf + sizeof(uint32_t); for (int32_t i = 0; i < numOfCols; ++i) { @@ -620,7 +590,8 @@ int32_t blockDataFromBuf1(SSDataBlock* pBlock, const char* buf, size_t capacity) pBlock->info.rows = *(int32_t*)buf; pBlock->info.groupId = *(uint64_t*)(buf + sizeof(int32_t)); - int32_t numOfCols = pBlock->info.numOfCols; + size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); + const char* pStart = buf + sizeof(uint32_t) + sizeof(uint64_t); for (int32_t i = 0; i < numOfCols; ++i) { @@ -666,7 +637,7 @@ size_t blockDataGetRowSize(SSDataBlock* pBlock) { if (pBlock->info.rowSize == 0) { size_t rowSize = 0; - size_t numOfCols = pBlock->info.numOfCols; + size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i); rowSize += pColInfo->info.bytes; @@ -693,7 +664,7 @@ double blockDataGetSerialRowSize(const SSDataBlock* pBlock) { ASSERT(pBlock != NULL); double rowSize = 0; - size_t numOfCols = pBlock->info.numOfCols; + size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i); rowSize += pColInfo->info.bytes; @@ -762,7 +733,7 @@ int32_t dataBlockCompar(const void* p1, const void* p2, const void* param) { static int32_t doAssignOneTuple(SColumnInfoData* pDstCols, int32_t numOfRows, const SSDataBlock* pSrcBlock, int32_t tupleIndex) { int32_t code = 0; - int32_t numOfCols = pSrcBlock->info.numOfCols; + size_t numOfCols = taosArrayGetSize(pSrcBlock->pDataBlock); for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pDst = &pDstCols[i]; @@ -785,7 +756,7 @@ static int32_t doAssignOneTuple(SColumnInfoData* pDstCols, int32_t numOfRows, co return TSDB_CODE_SUCCESS; } -static int32_t blockDataAssign(SColumnInfoData* pCols, const SSDataBlock* pDataBlock, int32_t* index) { +static int32_t blockDataAssign(SColumnInfoData* pCols, const SSDataBlock* pDataBlock, const int32_t* index) { #if 0 for (int32_t i = 0; i < pDataBlock->info.rows; ++i) { int32_t code = doAssignOneTuple(pCols, i, pDataBlock, index[i]); @@ -794,7 +765,8 @@ static int32_t blockDataAssign(SColumnInfoData* pCols, const SSDataBlock* pDataB } } #else - for (int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) { + size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock); + for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pDst = &pCols[i]; SColumnInfoData* pSrc = taosArrayGet(pDataBlock->pDataBlock, i); @@ -821,7 +793,7 @@ static int32_t blockDataAssign(SColumnInfoData* pCols, const SSDataBlock* pDataB static SColumnInfoData* createHelpColInfoData(const SSDataBlock* pDataBlock) { int32_t rows = pDataBlock->info.rows; - int32_t numOfCols = pDataBlock->info.numOfCols; + size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock); SColumnInfoData* pCols = taosMemoryCalloc(numOfCols, sizeof(SColumnInfoData)); if (pCols == NULL) { @@ -848,7 +820,7 @@ static SColumnInfoData* createHelpColInfoData(const SSDataBlock* pDataBlock) { } static void copyBackToBlock(SSDataBlock* pDataBlock, SColumnInfoData* pCols) { - int32_t numOfCols = pDataBlock->info.numOfCols; + size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock); for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, i); @@ -909,8 +881,10 @@ int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo) { } } + size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock); + if (taosArrayGetSize(pOrderInfo) == 1 && (!sortColumnHasNull)) { - if (pDataBlock->info.numOfCols == 1) { + if (numOfCols == 1) { if (!varTypeSort) { SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, 0); SBlockOrderInfo* pOrder = taosArrayGet(pOrderInfo, 0); @@ -926,7 +900,7 @@ int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo) { return TSDB_CODE_SUCCESS; } else { // var data type } - } else if (pDataBlock->info.numOfCols == 2) { + } else if (numOfCols == 2) { } } @@ -1120,34 +1094,39 @@ int32_t blockDataSort_rv(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullF void blockDataCleanup(SSDataBlock* pDataBlock) { pDataBlock->info.rows = 0; - for (int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) { + size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock); + for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i); colInfoDataCleanup(p, pDataBlock->info.capacity); } } -int32_t colInfoDataEnsureCapacity(SColumnInfoData* pColumn, size_t existRows, uint32_t numOfRows) { - ASSERT(numOfRows); - if (0 == numOfRows || numOfRows <= existRows) { +static int32_t doEnsureCapacity(SColumnInfoData* pColumn, const SDataBlockInfo* pBlockInfo, uint32_t numOfRows) { + ASSERT(numOfRows > 0 && pBlockInfo->capacity >= pBlockInfo->rows); + if (numOfRows < pBlockInfo->capacity) { return TSDB_CODE_SUCCESS; } + // todo temp disable it + // ASSERT(pColumn->info.bytes != 0); + + int32_t existedRows = pBlockInfo->rows; + if (IS_VAR_DATA_TYPE(pColumn->info.type)) { char* tmp = taosMemoryRealloc(pColumn->varmeta.offset, sizeof(int32_t) * numOfRows); - if (tmp == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } pColumn->varmeta.offset = (int32_t*)tmp; - memset(&pColumn->varmeta.offset[existRows], 0, sizeof(int32_t) * (numOfRows - existRows)); + memset(&pColumn->varmeta.offset[existedRows], 0, sizeof(int32_t) * (numOfRows - existedRows)); } else { char* tmp = taosMemoryRealloc(pColumn->nullbitmap, BitmapLen(numOfRows)); if (tmp == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } - int32_t oldLen = BitmapLen(existRows); + int32_t oldLen = BitmapLen(existedRows); pColumn->nullbitmap = tmp; memset(&pColumn->nullbitmap[oldLen], 0, BitmapLen(numOfRows) - oldLen); @@ -1160,8 +1139,8 @@ int32_t colInfoDataEnsureCapacity(SColumnInfoData* pColumn, size_t existRows, ui if (tmp == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } - memset(tmp + pColumn->info.bytes * existRows, 0, pColumn->info.bytes * (numOfRows - existRows)); + memset(tmp + pColumn->info.bytes * existedRows, 0, pColumn->info.bytes * (numOfRows - existedRows)); pColumn->pData = tmp; } @@ -1178,6 +1157,11 @@ void colInfoDataCleanup(SColumnInfoData* pColumn, uint32_t numOfRows) { } } +int32_t colInfoDataEnsureCapacity(SColumnInfoData* pColumn, uint32_t numOfRows) { + SDataBlockInfo info = {0}; + return doEnsureCapacity(pColumn, &info, numOfRows); +} + int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows) { int32_t code = 0; ASSERT(numOfRows > 0); @@ -1186,11 +1170,14 @@ int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows) { return TSDB_CODE_SUCCESS; } - pDataBlock->info.capacity = numOfRows; + if (pDataBlock->info.capacity < numOfRows) { + pDataBlock->info.capacity = numOfRows; + } - for (int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) { + size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock); + for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i); - code = colInfoDataEnsureCapacity(p, pDataBlock->info.rows, numOfRows); + code = doEnsureCapacity(p, &pDataBlock->info, numOfRows); if (code) { return code; } @@ -1208,46 +1195,74 @@ void* blockDataDestroy(SSDataBlock* pBlock) { taosMemoryFreeClear(pBlock); return NULL; } +int32_t assignOneDataBlock(SSDataBlock* dst, const SSDataBlock* src) { + ASSERT(src != NULL); + + dst->info = src->info; + dst->info.rows = 0; + dst->info.capacity = 0; + + size_t numOfCols = taosArrayGetSize(src->pDataBlock); + for (int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoData* p = taosArrayGet(src->pDataBlock, i); + SColumnInfoData colInfo = {.hasNull = true, .info = p->info}; + blockDataAppendColInfo(dst, &colInfo); + } + + int32_t code = blockDataEnsureCapacity(dst, src->info.rows); + if (code != TSDB_CODE_SUCCESS) { + terrno = code; + return -1; + } + + for (int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoData* pDst = taosArrayGet(dst->pDataBlock, i); + SColumnInfoData* pSrc = taosArrayGet(src->pDataBlock, i); + if (pSrc->pData == NULL) { + continue; + } + + colDataAssign(pDst, pSrc, src->info.rows, &src->info); + } + + dst->info.rows = src->info.rows; + dst->info.capacity = src->info.rows; + return 0; +} SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData) { if (pDataBlock == NULL) { return NULL; } - int32_t numOfCols = pDataBlock->info.numOfCols; - - SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); - pBlock->pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); - - pBlock->info.numOfCols = numOfCols; - pBlock->info.hasVarCol = pDataBlock->info.hasVarCol; - pBlock->info.rowSize = pDataBlock->info.rowSize; - pBlock->info.groupId = pDataBlock->info.groupId; - pBlock->info.childId = pDataBlock->info.childId; - pBlock->info.type = pDataBlock->info.type; + SSDataBlock* pBlock = createDataBlock(); + pBlock->info = pDataBlock->info; + pBlock->info.rows = 0; + pBlock->info.capacity = 0; + size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock); for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData colInfo = {0}; SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i); - colInfo.info = p->info; - colInfo.hasNull = true; - taosArrayPush(pBlock->pDataBlock, &colInfo); + SColumnInfoData colInfo = {.hasNull = true, .info = p->info}; + blockDataAppendColInfo(pBlock, &colInfo); } if (copyData) { + int32_t code = blockDataEnsureCapacity(pBlock, pDataBlock->info.rows); + if (code != TSDB_CODE_SUCCESS) { + terrno = code; + blockDataDestroy(pBlock); + return NULL; + } + for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pDst = taosArrayGet(pBlock->pDataBlock, i); SColumnInfoData* pSrc = taosArrayGet(pDataBlock->pDataBlock, i); - - int32_t code = colInfoDataEnsureCapacity(pDst, 0, pDataBlock->info.rows); - if (code != TSDB_CODE_SUCCESS) { - return NULL; - } - if (pSrc->pData == NULL) { continue; } - colDataAssign(pDst, pSrc, pDataBlock->info.rows); + + colDataAssign(pDst, pSrc, pDataBlock->info.rows, &pDataBlock->info); } pBlock->info.rows = pDataBlock->info.rows; @@ -1257,16 +1272,75 @@ SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData) { return pBlock; } +SSDataBlock* createDataBlock() { + SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); + if (pBlock == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + } + + pBlock->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData)); + if (pBlock->pDataBlock == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + taosMemoryFree(pBlock); + } + + return pBlock; +} + +int32_t blockDataAppendColInfo(SSDataBlock* pBlock, SColumnInfoData* pColInfoData) { + ASSERT(pBlock != NULL && pColInfoData != NULL); + if (pBlock->pDataBlock == NULL) { + pBlock->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData)); + if (pBlock->pDataBlock == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return terrno; + } + } + + void* p = taosArrayPush(pBlock->pDataBlock, pColInfoData); + if (p == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return terrno; + } + + // todo disable it temporarily + // ASSERT(pColInfoData->info.type != 0); + if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { + pBlock->info.hasVarCol = true; + } + + pBlock->info.rowSize += pColInfoData->info.bytes; + return TSDB_CODE_SUCCESS; +} + +SColumnInfoData createColumnInfoData(int16_t type, int32_t bytes, int16_t colId) { + SColumnInfoData col = {.hasNull = true}; + col.info.colId = colId; + col.info.type = type; + col.info.bytes = bytes; + + return col; +} + +SColumnInfoData* bdGetColumnInfoData(SSDataBlock* pBlock, int32_t index) { + ASSERT(pBlock != NULL); + if (index >= taosArrayGetSize(pBlock->pDataBlock)) { + return NULL; + } + + return taosArrayGet(pBlock->pDataBlock, index); +} + size_t blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize) { - int32_t payloadSize = pageSize - blockDataGetSerialMetaSize(pBlock->info.numOfCols); + size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); + int32_t payloadSize = pageSize - blockDataGetSerialMetaSize(numOfCols); int32_t rowSize = pBlock->info.rowSize; - int32_t nRows = payloadSize / rowSize; // the true value must be less than the value of nRows int32_t additional = 0; - for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { + for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, i); if (IS_VAR_DATA_TYPE(pCol->info.type)) { additional += nRows * sizeof(int32_t); @@ -1350,7 +1424,8 @@ int32_t blockDataTrimFirstNRows(SSDataBlock* pBlock, size_t n) { if (pBlock->info.rows <= n) { blockDataCleanup(pBlock); } else { - for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { + size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); + for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); colDataTrimFirstNRows(pColInfoData, n, pBlock->info.rows); } @@ -1362,7 +1437,7 @@ int32_t blockDataTrimFirstNRows(SSDataBlock* pBlock, size_t n) { int32_t tEncodeDataBlock(void** buf, const SSDataBlock* pBlock) { int64_t tbUid = pBlock->info.uid; - int16_t numOfCols = pBlock->info.numOfCols; + int16_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); int16_t hasVarCol = pBlock->info.hasVarCol; int32_t rows = pBlock->info.rows; int32_t sz = taosArrayGetSize(pBlock->pDataBlock); @@ -1397,8 +1472,10 @@ int32_t tEncodeDataBlock(void** buf, const SSDataBlock* pBlock) { void* tDecodeDataBlock(const void* buf, SSDataBlock* pBlock) { int32_t sz; - buf = taosDecodeFixedI64(buf, &pBlock->info.uid); - buf = taosDecodeFixedI16(buf, &pBlock->info.numOfCols); + int16_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); + + buf = taosDecodeFixedU64(buf, &pBlock->info.uid); + buf = taosDecodeFixedI16(buf, &numOfCols); buf = taosDecodeFixedI16(buf, &pBlock->info.hasVarCol); buf = taosDecodeFixedI32(buf, &pBlock->info.rows); buf = taosDecodeFixedI32(buf, &sz); @@ -1507,12 +1584,13 @@ void blockDebugShowData(const SArray* dataBlocks, const char* flag) { int32_t sz = taosArrayGetSize(dataBlocks); for (int32_t i = 0; i < sz; i++) { SSDataBlock* pDataBlock = taosArrayGet(dataBlocks, i); - int32_t colNum = pDataBlock->info.numOfCols; - int32_t rows = pDataBlock->info.rows; + size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock); + + int32_t rows = pDataBlock->info.rows; printf("%s |block type %d |child id %d|\n", flag, (int32_t)pDataBlock->info.type, pDataBlock->info.childId); for (int32_t j = 0; j < rows; j++) { printf("%s |", flag); - for (int32_t k = 0; k < colNum; k++) { + for (int32_t k = 0; k < numOfCols; k++) { SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, k); void* var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes); if (colDataIsNull(pColInfoData, rows, j, NULL)) { @@ -1565,7 +1643,9 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks int32_t bufSize = sizeof(SSubmitReq); for (int32_t i = 0; i < sz; ++i) { SDataBlockInfo* pBlkInfo = &((SSDataBlock*)taosArrayGet(pDataBlocks, i))->info; - bufSize += pBlkInfo->rows * (TD_ROW_HEAD_LEN + pBlkInfo->rowSize + BitmapLen(pBlkInfo->numOfCols)); + + int32_t numOfCols = taosArrayGetSize(pDataBlocks); + bufSize += pBlkInfo->rows * (TD_ROW_HEAD_LEN + pBlkInfo->rowSize + BitmapLen(numOfCols)); bufSize += sizeof(SSubmitBlk); } @@ -1583,10 +1663,10 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks for (int32_t i = 0; i < sz; ++i) { SSDataBlock* pDataBlock = taosArrayGet(pDataBlocks, i); - int32_t colNum = pDataBlock->info.numOfCols; + int32_t colNum = taosArrayGetSize(pDataBlock->pDataBlock); int32_t rows = pDataBlock->info.rows; - int32_t rowSize = pDataBlock->info.rowSize; - int64_t groupId = pDataBlock->info.groupId; + // int32_t rowSize = pDataBlock->info.rowSize; + // int64_t groupId = pDataBlock->info.groupId; if (colNum <= 1) { // invalid if only with TS col @@ -1804,8 +1884,6 @@ void blockCompressEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen } const char* blockCompressDecode(SSDataBlock* pBlock, int32_t numOfCols, int32_t numOfRows, const char* pData) { - blockDataEnsureCapacity(pBlock, numOfRows); - const char* pStart = pData; int32_t dataLen = *(int32_t*)pStart; @@ -1819,9 +1897,6 @@ const char* blockCompressDecode(SSDataBlock* pBlock, int32_t numOfCols, int32_t taosArraySetSize(pBlock->pDataBlock, numOfCols); } - pBlock->info.numOfCols = taosArrayGetSize(pBlock->pDataBlock); - ASSERT(pBlock->pDataBlock->size >= numOfCols); - for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); pColInfoData->info.type = *(int16_t*)pStart; @@ -1846,28 +1921,26 @@ const char* blockCompressDecode(SSDataBlock* pBlock, int32_t numOfCols, int32_t SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { - pColInfoData->varmeta.length = colLen[i]; - pColInfoData->varmeta.allocLen = colLen[i]; - memcpy(pColInfoData->varmeta.offset, pStart, sizeof(int32_t) * numOfRows); pStart += sizeof(int32_t) * numOfRows; - if (colLen[i] > 0) { - taosMemoryFreeClear(pColInfoData->pData); - pColInfoData->pData = taosMemoryMalloc(colLen[i]); + if (colLen[i] > 0 && pColInfoData->varmeta.allocLen < colLen[i]) { + char* tmp = taosMemoryRealloc(pColInfoData->pData, colLen[i]); + if (tmp == NULL) { + return NULL; + } + + pColInfoData->pData = tmp; + pColInfoData->varmeta.allocLen = colLen[i]; } + + pColInfoData->varmeta.length = colLen[i]; } else { - if (pColInfoData->nullbitmap == NULL) { - pColInfoData->nullbitmap = taosMemoryCalloc(1, BitmapLen(numOfRows)); - } memcpy(pColInfoData->nullbitmap, pStart, BitmapLen(numOfRows)); pStart += BitmapLen(numOfRows); } if (colLen[i] > 0) { - if (pColInfoData->pData == NULL) { - pColInfoData->pData = taosMemoryCalloc(1, colLen[i]); - } memcpy(pColInfoData->pData, pStart, colLen[i]); } diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 6cae3a13e6..e4bb30b001 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -636,6 +636,373 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { return 0; } +int32_t taosSetCfg(SConfig *pCfg, char* name) { + int32_t len = strlen(name); + char lowcaseName[CFG_NAME_MAX_LEN + 1] = {0}; + strntolower(lowcaseName, name, TMIN(CFG_NAME_MAX_LEN, len)); + + switch (lowcaseName[0]) { + case 'a': { + if (strcasecmp("asyncLog", name) == 0) { + tsAsyncLog = cfgGetItem(pCfg, "asyncLog")->bval; + } + break; + } + case 'b': { + if (strcasecmp("bnodeShmSize", name) == 0) { + tsBnodeShmSize = cfgGetItem(pCfg, "bnodeShmSize")->i32; + } + break; + } + case 'c': { + if (strcasecmp("charset", name) == 0) { + const char *locale = cfgGetItem(pCfg, "locale")->str; + const char *charset = cfgGetItem(pCfg, "charset")->str; + taosSetSystemLocale(locale, charset); + osSetSystemLocale(locale, charset); + } else if (strcasecmp("compressMsgSize", name) == 0) { + tsCompressMsgSize = cfgGetItem(pCfg, "compressMsgSize")->i32; + } else if (strcasecmp("compressColData", name) == 0) { + tsCompressColData = cfgGetItem(pCfg, "compressColData")->i32; + } else if (strcasecmp("countAlwaysReturnValue", name) == 0) { + tsCountAlwaysReturnValue = cfgGetItem(pCfg, "countAlwaysReturnValue")->i32; + } else if (strcasecmp("cDebugFlag", name) == 0) { + cDebugFlag = cfgGetItem(pCfg, "cDebugFlag")->i32; + } + break; + } + case 'd': { + if (strcasecmp("deadLockKillQuery", name) == 0) { + tsDeadLockKillQuery = cfgGetItem(pCfg, "deadLockKillQuery")->i32; + } else if (strcasecmp("dDebugFlag", name) == 0) { + dDebugFlag = cfgGetItem(pCfg, "dDebugFlag")->i32; + } + break; + } + case 'e': { + if (strcasecmp("enableCoreFile", name) == 0) { + bool enableCore = cfgGetItem(pCfg, "enableCoreFile")->bval; + taosSetConsoleEcho(enableCore); + } + break; + } + case 'f': { + if (strcasecmp("fqdn", name) == 0) { + tstrncpy(tsLocalFqdn, cfgGetItem(pCfg, "fqdn")->str, TSDB_FQDN_LEN); + tsServerPort = (uint16_t)cfgGetItem(pCfg, "serverPort")->i32; + snprintf(tsLocalEp, sizeof(tsLocalEp), "%s:%u", tsLocalFqdn, tsServerPort); + + char defaultFirstEp[TSDB_EP_LEN] = {0}; + snprintf(defaultFirstEp, TSDB_EP_LEN, "%s:%u", tsLocalFqdn, tsServerPort); + + SConfigItem *pFirstEpItem = cfgGetItem(pCfg, "firstEp"); + SEp firstEp = {0}; + taosGetFqdnPortFromEp(strlen(pFirstEpItem->str) == 0 ? defaultFirstEp : pFirstEpItem->str, &firstEp); + snprintf(tsFirst, sizeof(tsFirst), "%s:%u", firstEp.fqdn, firstEp.port); + cfgSetItem(pCfg, "firstEp", tsFirst, pFirstEpItem->stype); + } else if (strcasecmp("firstEp", name) == 0) { + tstrncpy(tsLocalFqdn, cfgGetItem(pCfg, "fqdn")->str, TSDB_FQDN_LEN); + tsServerPort = (uint16_t)cfgGetItem(pCfg, "serverPort")->i32; + snprintf(tsLocalEp, sizeof(tsLocalEp), "%s:%u", tsLocalFqdn, tsServerPort); + + char defaultFirstEp[TSDB_EP_LEN] = {0}; + snprintf(defaultFirstEp, TSDB_EP_LEN, "%s:%u", tsLocalFqdn, tsServerPort); + + SConfigItem *pFirstEpItem = cfgGetItem(pCfg, "firstEp"); + SEp firstEp = {0}; + taosGetFqdnPortFromEp(strlen(pFirstEpItem->str) == 0 ? defaultFirstEp : pFirstEpItem->str, &firstEp); + snprintf(tsFirst, sizeof(tsFirst), "%s:%u", firstEp.fqdn, firstEp.port); + cfgSetItem(pCfg, "firstEp", tsFirst, pFirstEpItem->stype); + } else if (strcasecmp("fsDebugFlag", name) == 0) { + fsDebugFlag = cfgGetItem(pCfg, "fsDebugFlag")->i32; + } else if (strcasecmp("fnDebugFlag", name) == 0) { + fnDebugFlag = cfgGetItem(pCfg, "fnDebugFlag")->i32; + } + break; + } + case 'i': { + if (strcasecmp("idxDebugFlag", name) == 0) { + idxDebugFlag = cfgGetItem(pCfg, "idxDebugFlag")->i32; + } + break; + } + case 'j': { + if (strcasecmp("jniDebugFlag", name) == 0) { + jniDebugFlag = cfgGetItem(pCfg, "jniDebugFlag")->i32; + } + break; + } + case 'k': { + if (strcasecmp("keepColumnName", name) == 0) { + tsKeepOriginalColumnName = cfgGetItem(pCfg, "keepColumnName")->bval; + } + break; + } + case 'l': { + if (strcasecmp("locale", name) == 0) { + const char *locale = cfgGetItem(pCfg, "locale")->str; + const char *charset = cfgGetItem(pCfg, "charset")->str; + taosSetSystemLocale(locale, charset); + osSetSystemLocale(locale, charset); + } else if (strcasecmp("logDir", name) == 0) { + tstrncpy(tsLogDir, cfgGetItem(pCfg, "logDir")->str, PATH_MAX); + taosExpandDir(tsLogDir, tsLogDir, PATH_MAX); + } else if (strcasecmp("logKeepDays", name) == 0) { + tsLogKeepDays = cfgGetItem(pCfg, "logKeepDays")->i32; + } + break; + } + case 'm': { + switch (lowcaseName[1]) { + case 'a': { + if (strcasecmp("maxShellConns", name) == 0) { + tsMaxShellConns = cfgGetItem(pCfg, "maxShellConns")->i32; + } else if (strcasecmp("maxNumOfDistinctRes", name) == 0) { + tsMaxNumOfDistinctResults = cfgGetItem(pCfg, "maxNumOfDistinctRes")->i32; + } else if (strcasecmp("maxStreamCompDelay", name) == 0) { + tsMaxStreamComputDelay = cfgGetItem(pCfg, "maxStreamCompDelay")->i32; + } else if (strcasecmp("maxFirstStreamCompDelay", name) == 0) { + tsStreamCompStartDelay = cfgGetItem(pCfg, "maxFirstStreamCompDelay")->i32; + } + break; + } + case 'd': { + if (strcasecmp("mDebugFlag", name) == 0) { + mDebugFlag = cfgGetItem(pCfg, "mDebugFlag")->i32; + } + break; + } + case 'i': { + if (strcasecmp("minimalTempDirGB", name) == 0) { + tsTempSpace.reserved = cfgGetItem(pCfg, "minimalTempDirGB")->fval; + } else if (strcasecmp("minimalDataDirGB", name) == 0) { + tsDataSpace.reserved = cfgGetItem(pCfg, "minimalDataDirGB")->fval; + } else if (strcasecmp("minSlidingTime", name) == 0) { + tsMinSlidingTime = cfgGetItem(pCfg, "minSlidingTime")->i32; + } else if (strcasecmp("minIntervalTime", name) == 0) { + tsMinIntervalTime = cfgGetItem(pCfg, "minIntervalTime")->i32; + } else if (strcasecmp("minimalLogDirGB", name) == 0) { + tsLogSpace.reserved = cfgGetItem(pCfg, "minimalLogDirGB")->fval; + } + break; + } + case 'n': { + if (strcasecmp("mnodeShmSize", name) == 0) { + tsMnodeShmSize = cfgGetItem(pCfg, "mnodeShmSize")->i32; + } + break; + } + case 'o': { + if (strcasecmp("monitor", name) == 0) { + tsEnableMonitor = cfgGetItem(pCfg, "monitor")->bval; + } else if (strcasecmp("monitorInterval", name) == 0) { + tsMonitorInterval = cfgGetItem(pCfg, "monitorInterval")->i32; + } else if (strcasecmp("monitorFqdn", name) == 0) { + tstrncpy(tsMonitorFqdn, cfgGetItem(pCfg, "monitorFqdn")->str, TSDB_FQDN_LEN); + } else if (strcasecmp("monitorPort", name) == 0) { + tsMonitorPort = (uint16_t)cfgGetItem(pCfg, "monitorPort")->i32; + } else if (strcasecmp("monitorMaxLogs", name) == 0) { + tsMonitorMaxLogs = cfgGetItem(pCfg, "monitorMaxLogs")->i32; + } else if (strcasecmp("monitorComp", name) == 0) { + tsMonitorComp = cfgGetItem(pCfg, "monitorComp")->bval; + } + break; + } + case 'q': { + if (strcasecmp("mqRebalanceInterval", name) == 0) { + tsMqRebalanceInterval = cfgGetItem(pCfg, "mqRebalanceInterval")->i32; + } + break; + } + case 'u': { + if (strcasecmp("multiProcess", name) == 0) { + tsMultiProcess = cfgGetItem(pCfg, "multiProcess")->bval; + } + break; + } + default: + terrno = TSDB_CODE_CFG_NOT_FOUND; + return -1; + } + break; + } + case 'n': { + if (strcasecmp("numOfTaskQueueThreads", name) == 0) { + tsNumOfTaskQueueThreads = cfgGetItem(pCfg, "numOfTaskQueueThreads")->i32; + } else if (strcasecmp("numOfRpcThreads", name) == 0) { + tsNumOfRpcThreads = cfgGetItem(pCfg, "numOfRpcThreads")->i32; + } else if (strcasecmp("numOfCommitThreads", name) == 0) { + tsNumOfCommitThreads = cfgGetItem(pCfg, "numOfCommitThreads")->i32; + } else if (strcasecmp("numOfMnodeQueryThreads", name) == 0) { + tsNumOfMnodeQueryThreads = cfgGetItem(pCfg, "numOfMnodeQueryThreads")->i32; + } else if (strcasecmp("numOfMnodeReadThreads", name) == 0) { + tsNumOfMnodeReadThreads = cfgGetItem(pCfg, "numOfMnodeReadThreads")->i32; + } else if (strcasecmp("numOfVnodeQueryThreads", name) == 0) { + tsNumOfVnodeQueryThreads = cfgGetItem(pCfg, "numOfVnodeQueryThreads")->i32; + } else if (strcasecmp("numOfVnodeFetchThreads", name) == 0) { + tsNumOfVnodeFetchThreads = cfgGetItem(pCfg, "numOfVnodeFetchThreads")->i32; + } else if (strcasecmp("numOfVnodeWriteThreads", name) == 0) { + tsNumOfVnodeWriteThreads = cfgGetItem(pCfg, "numOfVnodeWriteThreads")->i32; + } else if (strcasecmp("numOfVnodeSyncThreads", name) == 0) { + tsNumOfVnodeSyncThreads = cfgGetItem(pCfg, "numOfVnodeSyncThreads")->i32; + } else if (strcasecmp("numOfVnodeMergeThreads", name) == 0) { + tsNumOfVnodeMergeThreads = cfgGetItem(pCfg, "numOfVnodeMergeThreads")->i32; + } else if (strcasecmp("numOfQnodeQueryThreads", name) == 0) { + tsNumOfQnodeQueryThreads = cfgGetItem(pCfg, "numOfQnodeQueryThreads")->i32; + } else if (strcasecmp("numOfQnodeFetchThreads", name) == 0) { + tsNumOfQnodeFetchThreads = cfgGetItem(pCfg, "numOfQnodeFetchThreads")->i32; + } else if (strcasecmp("numOfSnodeSharedThreads", name) == 0) { + tsNumOfSnodeSharedThreads = cfgGetItem(pCfg, "numOfSnodeSharedThreads")->i32; + } else if (strcasecmp("numOfSnodeUniqueThreads", name) == 0) { + tsNumOfSnodeUniqueThreads = cfgGetItem(pCfg, "numOfSnodeUniqueThreads")->i32; + } else if (strcasecmp("numOfLogLines", name) == 0) { + tsNumOfLogLines = cfgGetItem(pCfg, "numOfLogLines")->i32; + } + break; + } + case 'p': { + if (strcasecmp("printAuth", name) == 0) { + tsPrintAuth = cfgGetItem(pCfg, "printAuth")->bval; + } + break; + } + case 'q': { + if (strcasecmp("queryPolicy", name) == 0) { + tsQueryPolicy = cfgGetItem(pCfg, "queryPolicy")->i32; + } else if (strcasecmp("querySmaOptimize", name) == 0) { + tsQuerySmaOptimize = cfgGetItem(pCfg, "querySmaOptimize")->i32; + } else if (strcasecmp("queryBufferSize", name) == 0) { + tsQueryBufferSize = cfgGetItem(pCfg, "queryBufferSize")->i32; + if (tsQueryBufferSize >= 0) { + tsQueryBufferSizeBytes = tsQueryBufferSize * 1048576UL; + } + } else if (strcasecmp("qnodeShmSize", name) == 0) { + tsQnodeShmSize = cfgGetItem(pCfg, "qnodeShmSize")->i32; + } else if (strcasecmp("qDebugFlag", name) == 0) { + qDebugFlag = cfgGetItem(pCfg, "qDebugFlag")->i32; + } + break; + } + case 'r': { + if (strcasecmp("retryStreamCompDelay", name) == 0) { + tsRetryStreamCompDelay = cfgGetItem(pCfg, "retryStreamCompDelay")->i32; + } else if (strcasecmp("retrieveBlockingModel", name) == 0) { + tsRetrieveBlockingModel = cfgGetItem(pCfg, "retrieveBlockingModel")->bval; + } else if (strcasecmp("rpcQueueMemoryAllowed", name) == 0) { + tsRpcQueueMemoryAllowed = cfgGetItem(pCfg, "rpcQueueMemoryAllowed")->i64; + } else if (strcasecmp("rpcDebugFlag", name) == 0) { + rpcDebugFlag = cfgGetItem(pCfg, "rpcDebugFlag")->i32; + } + break; + } + case 's': { + if (strcasecmp("secondEp", name) == 0) { + SConfigItem *pSecondpItem = cfgGetItem(pCfg, "secondEp"); + SEp secondEp = {0}; + taosGetFqdnPortFromEp(strlen(pSecondpItem->str) == 0 ? tsFirst : pSecondpItem->str, &secondEp); + snprintf(tsSecond, sizeof(tsSecond), "%s:%u", secondEp.fqdn, secondEp.port); + cfgSetItem(pCfg, "secondEp", tsSecond, pSecondpItem->stype); + } else if (strcasecmp("smlChildTableName", name) == 0) { + tstrncpy(tsSmlChildTableName, cfgGetItem(pCfg, "smlChildTableName")->str, TSDB_TABLE_NAME_LEN); + } else if (strcasecmp("smlTagName", name) == 0) { + tstrncpy(tsSmlTagName, cfgGetItem(pCfg, "smlTagName")->str, TSDB_COL_NAME_LEN); + } else if (strcasecmp("smlDataFormat", name) == 0) { + tsSmlDataFormat = cfgGetItem(pCfg, "smlDataFormat")->bval; + } else if (strcasecmp("shellActivityTimer", name) == 0) { + tsShellActivityTimer = cfgGetItem(pCfg, "shellActivityTimer")->i32; + } else if (strcasecmp("supportVnodes", name) == 0) { + tsNumOfSupportVnodes = cfgGetItem(pCfg, "supportVnodes")->i32; + } else if (strcasecmp("statusInterval", name) == 0) { + tsStatusInterval = cfgGetItem(pCfg, "statusInterval")->i32; + } else if (strcasecmp("streamCompDelayRatio", name) == 0) { + tsStreamComputDelayRatio = cfgGetItem(pCfg, "streamCompDelayRatio")->fval; + } else if (strcasecmp("slaveQuery", name) == 0) { + tsEnableSlaveQuery = cfgGetItem(pCfg, "slaveQuery")->bval; + } else if (strcasecmp("snodeShmSize", name) == 0) { + tsSnodeShmSize = cfgGetItem(pCfg, "snodeShmSize")->i32; + } else if (strcasecmp("serverPort", name) == 0) { + tstrncpy(tsLocalFqdn, cfgGetItem(pCfg, "fqdn")->str, TSDB_FQDN_LEN); + tsServerPort = (uint16_t)cfgGetItem(pCfg, "serverPort")->i32; + snprintf(tsLocalEp, sizeof(tsLocalEp), "%s:%u", tsLocalFqdn, tsServerPort); + + char defaultFirstEp[TSDB_EP_LEN] = {0}; + snprintf(defaultFirstEp, TSDB_EP_LEN, "%s:%u", tsLocalFqdn, tsServerPort); + + SConfigItem *pFirstEpItem = cfgGetItem(pCfg, "firstEp"); + SEp firstEp = {0}; + taosGetFqdnPortFromEp(strlen(pFirstEpItem->str) == 0 ? defaultFirstEp : pFirstEpItem->str, &firstEp); + snprintf(tsFirst, sizeof(tsFirst), "%s:%u", firstEp.fqdn, firstEp.port); + cfgSetItem(pCfg, "firstEp", tsFirst, pFirstEpItem->stype); + } else if (strcasecmp("sDebugFlag", name) == 0) { + sDebugFlag = cfgGetItem(pCfg, "sDebugFlag")->i32; + } else if (strcasecmp("smaDebugFlag", name) == 0) { + smaDebugFlag = cfgGetItem(pCfg, "smaDebugFlag")->i32; + } + break; + } + case 't': { + if (strcasecmp("timezone", name) == 0) { + SConfigItem *pItem = cfgGetItem(pCfg, "timezone"); + osSetTimezone(pItem->str); + uDebug("timezone format changed from %s to %s", pItem->str, tsTimezoneStr); + cfgSetItem(pCfg, "timezone", tsTimezoneStr, pItem->stype); + } else if (strcasecmp("tempDir", name) == 0) { + tstrncpy(tsTempDir, cfgGetItem(pCfg, "tempDir")->str, PATH_MAX); + taosExpandDir(tsTempDir, tsTempDir, PATH_MAX); + if (taosMulMkDir(tsTempDir) != 0) { + uError("failed to create tempDir:%s since %s", tsTempDir, terrstr()); + return -1; + } + } else if (strcasecmp("telemetryReporting", name) == 0) { + tsEnableTelem = cfgGetItem(pCfg, "telemetryReporting")->bval; + } else if (strcasecmp("telemetryInterval", name) == 0) { + tsTelemInterval = cfgGetItem(pCfg, "telemetryInterval")->i32; + } else if (strcasecmp("telemetryServer", name) == 0) { + tstrncpy(tsTelemServer, cfgGetItem(pCfg, "telemetryServer")->str, TSDB_FQDN_LEN); + } else if (strcasecmp("telemetryPort", name) == 0) { + tsTelemPort = (uint16_t)cfgGetItem(pCfg, "telemetryPort")->i32; + } else if (strcasecmp("transPullupInterval", name) == 0) { + tsTransPullupInterval = cfgGetItem(pCfg, "transPullupInterval")->i32; + } else if (strcasecmp("tmrDebugFlag", name) == 0) { + tmrDebugFlag = cfgGetItem(pCfg, "tmrDebugFlag")->i32; + } else if (strcasecmp("tsdbDebugFlag", name) == 0) { + tsdbDebugFlag = cfgGetItem(pCfg, "tsdbDebugFlag")->i32; + } else if (strcasecmp("tqDebugFlag", name) == 0) { + tqDebugFlag = cfgGetItem(pCfg, "tqDebugFlag")->i32; + } + break; + } + case 'u': { + if (strcasecmp("udf", name) == 0) { + tsStartUdfd = cfgGetItem(pCfg, "udf")->bval; + } else if (strcasecmp("uDebugFlag", name) == 0) { + uDebugFlag = cfgGetItem(pCfg, "uDebugFlag")->i32; + } + break; + } + case 'v': { + if (strcasecmp("vnodeShmSize", name) == 0) { + tsVnodeShmSize = cfgGetItem(pCfg, "vnodeShmSize")->i32; + } else if (strcasecmp("vDebugFlag", name) == 0) { + vDebugFlag = cfgGetItem(pCfg, "vDebugFlag")->i32; + } + break; + } + case 'w': { + if (strcasecmp("wDebugFlag", name) == 0) { + wDebugFlag = cfgGetItem(pCfg, "wDebugFlag")->i32; + } + break; + } + default: + terrno = TSDB_CODE_CFG_NOT_FOUND; + return -1; + } + + return 0; +} + + int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDir, const char **envCmd, const char *envFile, char *apolloUrl, SArray *pArgs, bool tsc) { if (tsCfg == NULL) osDefaultInit(); @@ -760,9 +1127,9 @@ void taosCleanupCfg() { } void taosCfgDynamicOptions(const char *option, const char *value) { - if (strcasecmp(option, "debugFlag") == 0) { - int32_t debugFlag = atoi(value); - taosSetAllDebugFlag(debugFlag); + if (strncasecmp(option, "debugFlag", 9) == 0) { + int32_t flag = atoi(value); + taosSetAllDebugFlag(flag); } if (strcasecmp(option, "resetlog") == 0) { diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index eb02191c26..dde282c4e3 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -1504,6 +1504,33 @@ int32_t tDeserializeSMCfgDnodeReq(void *buf, int32_t bufLen, SMCfgDnodeReq *pReq return 0; } +int32_t tSerializeSDCfgDnodeReq(void *buf, int32_t bufLen, SDCfgDnodeReq *pReq) { + SEncoder encoder = {0}; + tEncoderInit(&encoder, buf, bufLen); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->config) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->value) < 0) return -1; + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tEncoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSDCfgDnodeReq(void *buf, int32_t bufLen, SDCfgDnodeReq *pReq) { + SDecoder decoder = {0}; + tDecoderInit(&decoder, buf, bufLen); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->config) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->value) < 0) return -1; + tEndDecode(&decoder); + + tDecoderClear(&decoder); + return 0; +} + int32_t tSerializeSCreateDnodeReq(void *buf, int32_t bufLen, SCreateDnodeReq *pReq) { SEncoder encoder = {0}; tEncoderInit(&encoder, buf, bufLen); diff --git a/source/common/src/tname.c b/source/common/src/tname.c index ce454d439d..fb3a2d7464 100644 --- a/source/common/src/tname.c +++ b/source/common/src/tname.c @@ -214,6 +214,20 @@ int32_t tNameSetDbName(SName* dst, int32_t acct, const char* dbName, size_t name return 0; } +int32_t tNameAddTbName(SName* dst, const char* tbName, size_t nameLen) { + assert(dst != NULL && tbName != NULL && nameLen > 0); + + // too long account id or too long db name + if (nameLen >= tListLen(dst->tname) || nameLen <= 0) { + return -1; + } + + dst->type = TSDB_TABLE_NAME_T; + tstrncpy(dst->tname, tbName, nameLen + 1); + return 0; +} + + int32_t tNameSetAcctId(SName* dst, int32_t acctId) { assert(dst != NULL); dst->acctId = acctId; diff --git a/source/common/test/commonTests.cpp b/source/common/test/commonTests.cpp index 2adb558d01..8460d71e56 100644 --- a/source/common/test/commonTests.cpp +++ b/source/common/test/commonTests.cpp @@ -88,26 +88,15 @@ TEST(testCase, toInteger_test) { } TEST(testCase, Datablock_test) { - SSDataBlock* b = static_cast(taosMemoryCalloc(1, sizeof(SSDataBlock))); - b->info.numOfCols = 2; - b->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData)); + SSDataBlock* b = createDataBlock(); - SColumnInfoData infoData = {0}; - infoData.info.bytes = 4; - infoData.info.type = TSDB_DATA_TYPE_INT; - infoData.info.colId = 1; - - infoData.pData = (char*)taosMemoryCalloc(40, infoData.info.bytes); - infoData.nullbitmap = (char*)taosMemoryCalloc(1, sizeof(char) * (40 / 8)); + SColumnInfoData infoData = createColumnInfoData(TSDB_DATA_TYPE_INT, 4, 1); taosArrayPush(b->pDataBlock, &infoData); + blockDataAppendColInfo(b, &infoData); - SColumnInfoData infoData1 = {0}; - infoData1.info.bytes = 40; - infoData1.info.type = TSDB_DATA_TYPE_BINARY; - infoData1.info.colId = 2; - - infoData1.varmeta.offset = (int32_t*)taosMemoryCalloc(40, sizeof(uint32_t)); - taosArrayPush(b->pDataBlock, &infoData1); + SColumnInfoData infoData1 = createColumnInfoData(TSDB_DATA_TYPE_BINARY, 40, 2); + blockDataAppendColInfo(b, &infoData1); + blockDataEnsureCapacity(b, 40); char* str = "the value of: %d"; char buf[128] = {0}; @@ -222,28 +211,17 @@ TEST(testCase, non_var_dataBlock_split_test) { #endif TEST(testCase, var_dataBlock_split_test) { - SSDataBlock* b = static_cast(taosMemoryCalloc(1, sizeof(SSDataBlock))); - b->info.numOfCols = 2; - b->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData)); - int32_t numOfRows = 1000000; - SColumnInfoData infoData = {0}; - infoData.info.bytes = 4; - infoData.info.type = TSDB_DATA_TYPE_INT; - infoData.info.colId = 1; + SSDataBlock* b = createDataBlock(); - infoData.pData = (char*)taosMemoryCalloc(numOfRows, infoData.info.bytes); - infoData.nullbitmap = (char*)taosMemoryCalloc(1, sizeof(char) * (numOfRows / 8)); - taosArrayPush(b->pDataBlock, &infoData); + SColumnInfoData infoData = createColumnInfoData(TSDB_DATA_TYPE_INT, 4, 1); + blockDataAppendColInfo(b, &infoData); - SColumnInfoData infoData1 = {0}; - infoData1.info.bytes = 40; - infoData1.info.type = TSDB_DATA_TYPE_BINARY; - infoData1.info.colId = 2; + SColumnInfoData infoData1 = createColumnInfoData(TSDB_DATA_TYPE_BINARY, 40, 2); + blockDataAppendColInfo(b, &infoData1); - infoData1.varmeta.offset = (int32_t*)taosMemoryCalloc(numOfRows, sizeof(uint32_t)); - taosArrayPush(b->pDataBlock, &infoData1); + blockDataEnsureCapacity(b, numOfRows); char buf[41] = {0}; char buf1[100] = {0}; diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index 9c8918a445..3d1ece0b64 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -123,8 +123,15 @@ int32_t dmProcessGrantRsp(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) { } int32_t dmProcessConfigReq(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) { - dError("config req is received, but not supported yet"); - return TSDB_CODE_OPS_NOT_SUPPORT; + SDCfgDnodeReq cfgReq = {0}; + if (tDeserializeSDCfgDnodeReq(pMsg->pCont, pMsg->contLen, &cfgReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } + + dInfo("start to config, option:%s, value:%s", cfgReq.config, cfgReq.value); + taosCfgDynamicOptions(cfgReq.config, cfgReq.value); + return 0; } static void dmGetServerRunStatus(SDnodeMgmt *pMgmt, SServerStatusRsp *pStatus) { @@ -203,7 +210,6 @@ SSDataBlock* dmBuildVariablesBlock(void) { taosArrayPush(pBlock->pDataBlock, &colInfoData); } - pBlock->info.numOfCols = pMeta[index].colNum; pBlock->info.hasVarCol = true; return pBlock; @@ -262,8 +268,9 @@ int32_t dmProcessRetrieve(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) { dmAppendVariablesToBlock(pBlock, pMgmt->pData->dnodeId); - size = sizeof(SRetrieveMetaTableRsp) + sizeof(int32_t) + sizeof(SSysTableSchema) * pBlock->info.numOfCols + - blockDataGetSize(pBlock) + blockDataGetSerialMetaSize(pBlock->info.numOfCols); + size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); + size = sizeof(SRetrieveMetaTableRsp) + sizeof(int32_t) + sizeof(SSysTableSchema) * numOfCols + + blockDataGetSize(pBlock) + blockDataGetSerialMetaSize(numOfCols); SRetrieveMetaTableRsp *pRsp = rpcMallocCont(size); if (pRsp == NULL) { @@ -274,10 +281,10 @@ int32_t dmProcessRetrieve(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) { } char *pStart = pRsp->data; - *(int32_t *)pStart = htonl(pBlock->info.numOfCols); + *(int32_t *)pStart = htonl(numOfCols); pStart += sizeof(int32_t); // number of columns - for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { + for (int32_t i = 0; i < numOfCols; ++i) { SSysTableSchema *pSchema = (SSysTableSchema *)pStart; SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, i); @@ -289,7 +296,7 @@ int32_t dmProcessRetrieve(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) { } int32_t len = 0; - blockCompressEncode(pBlock, pStart, &len, pBlock->info.numOfCols, false); + blockCompressEncode(pBlock, pStart, &len, numOfCols, false); pRsp->numOfRows = htonl(pBlock->info.rows); pRsp->precision = TSDB_TIME_PRECISION_MILLI; // millisecond time precision diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c index d44a7d79bf..f9c54674f3 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c @@ -17,7 +17,7 @@ #include "mmInt.h" void mmGetMonitorInfo(SMnodeMgmt *pMgmt, SMonMmInfo *pInfo) { - mndGetMonitorInfo(pMgmt->pMnode, &pInfo->cluster, &pInfo->vgroup, &pInfo->grant); + mndGetMonitorInfo(pMgmt->pMnode, &pInfo->cluster, &pInfo->vgroup, &pInfo->stb, &pInfo->grant); } int32_t mmProcessGetMonitorInfoReq(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { diff --git a/source/dnode/mgmt/node_util/src/dmFile.c b/source/dnode/mgmt/node_util/src/dmFile.c index 78e706f908..2185adc18b 100644 --- a/source/dnode/mgmt/node_util/src/dmFile.c +++ b/source/dnode/mgmt/node_util/src/dmFile.c @@ -127,14 +127,24 @@ TdFilePtr dmCheckRunning(const char *dataDir) { return NULL; } - int32_t ret = taosLockFile(pFile); - if (ret != 0) { + int32_t retryTimes = 0; + int32_t ret = 0; + do { + ret = taosLockFile(pFile); + if (ret == 0) break; + terrno = TAOS_SYSTEM_ERROR(errno); + taosMsleep(100); + retryTimes++; + dError("failed to lock file:%s since %s, retryTimes:%d", filepath, terrstr(), retryTimes); + } while (retryTimes < 120); + + if (ret < 0) { terrno = TAOS_SYSTEM_ERROR(errno); - dError("failed to lock file:%s since %s", filepath, terrstr()); taosCloseFile(&pFile); return NULL; } + terrno = 0; dDebug("lock file:%s to prevent repeated starts", filepath); return pFile; } diff --git a/source/dnode/mnode/impl/inc/mndScheduler.h b/source/dnode/mnode/impl/inc/mndScheduler.h index 15d2c6cd5e..db81d8843e 100644 --- a/source/dnode/mnode/impl/inc/mndScheduler.h +++ b/source/dnode/mnode/impl/inc/mndScheduler.h @@ -27,12 +27,10 @@ void mndCleanupScheduler(SMnode* pMnode); int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscribeObj* pSub); -int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream); - int32_t mndConvertRsmaTask(char** pDst, int32_t* pDstLen, const char* ast, int64_t uid, int8_t triggerType, int64_t watermark); -int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream); +int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index b159061ce3..d3723326b9 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -502,20 +502,20 @@ _OVER: } static int32_t mndProcessDnodeListReq(SRpcMsg *pReq) { - SMnode *pMnode = pReq->info.node; - SSdb *pSdb = pMnode->pSdb; - SDnodeObj *pObj = NULL; - void *pIter = NULL; + SMnode *pMnode = pReq->info.node; + SSdb *pSdb = pMnode->pSdb; + SDnodeObj *pObj = NULL; + void *pIter = NULL; SDnodeListRsp rsp = {0}; - int32_t code = -1; - + int32_t code = -1; + rsp.dnodeList = taosArrayInit(5, sizeof(SEpSet)); if (NULL == rsp.dnodeList) { mError("failed to alloc epSet while process dnode list req"); terrno = TSDB_CODE_OUT_OF_MEMORY; goto _OVER; } - + while (1) { pIter = sdbFetch(pSdb, SDB_DNODE, pIter, (void **)&pObj); if (pIter == NULL) break; @@ -554,7 +554,6 @@ _OVER: return code; } - static int32_t mndProcessCreateDnodeReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; int32_t code = -1; @@ -732,15 +731,37 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) { mError("dnode:%d, failed to config since %s ", cfgReq.dnodeId, terrstr()); return -1; } - SEpSet epSet = mndGetDnodeEpset(pDnode); mndReleaseDnode(pMnode, pDnode); - int32_t bufLen = tSerializeSMCfgDnodeReq(NULL, 0, &cfgReq); + SDCfgDnodeReq dcfgReq = {0}; + if (strncasecmp(cfgReq.config, "debugFlag", 9) == 0) { + const char *value = cfgReq.value; + int32_t flag = atoi(value); + if (flag <= 0) { + flag = atoi(cfgReq.config + 10); + } + if (flag <= 0 || flag > 255) { + mError("dnode:%d, failed to config debugFlag since value:%d", cfgReq.dnodeId, flag); + terrno = TSDB_CODE_INVALID_CFG; + return -1; + } + + strcpy(dcfgReq.config, "debugFlag"); + snprintf(dcfgReq.value, TSDB_DNODE_VALUE_LEN, "%d", flag); + } else if (strcasecmp(cfgReq.config, "resetlog") == 0) { + strcpy(dcfgReq.config, "resetlog"); + } else { + terrno = TSDB_CODE_INVALID_CFG; + mError("dnode:%d, failed to config since %s", cfgReq.dnodeId, terrstr()); + return -1; + } + + int32_t bufLen = tSerializeSDCfgDnodeReq(NULL, 0, &dcfgReq); void *pBuf = rpcMallocCont(bufLen); if (pBuf == NULL) return -1; - tSerializeSMCfgDnodeReq(pBuf, bufLen, &cfgReq); + tSerializeSDCfgDnodeReq(pBuf, bufLen, &dcfgReq); mDebug("dnode:%d, send config req to dnode, app:%p", cfgReq.dnodeId, pReq->info.ahandle); SRpcMsg rpcMsg = {.msgType = TDMT_DND_CONFIG_DNODE, .pCont = pBuf, .contLen = bufLen}; diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index dede1c45e6..000e1041d0 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -614,7 +614,7 @@ int64_t mndGenerateUid(char *name, int32_t len) { } int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo, - SMonGrantInfo *pGrantInfo) { + SMonStbInfo *pStbInfo, SMonGrantInfo *pGrantInfo) { if (mndAcquireRpcRef(pMnode) != 0) return -1; SSdb *pSdb = pMnode->pSdb; @@ -623,7 +623,9 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr pClusterInfo->dnodes = taosArrayInit(sdbGetSize(pSdb, SDB_DNODE), sizeof(SMonDnodeDesc)); pClusterInfo->mnodes = taosArrayInit(sdbGetSize(pSdb, SDB_MNODE), sizeof(SMonMnodeDesc)); pVgroupInfo->vgroups = taosArrayInit(sdbGetSize(pSdb, SDB_VGROUP), sizeof(SMonVgroupDesc)); - if (pClusterInfo->dnodes == NULL || pClusterInfo->mnodes == NULL || pVgroupInfo->vgroups == NULL) { + pStbInfo->stbs = taosArrayInit(sdbGetSize(pSdb, SDB_STB), sizeof(SMonStbDesc)); + if (pClusterInfo->dnodes == NULL || pClusterInfo->mnodes == NULL || pVgroupInfo->vgroups == NULL || + pStbInfo->stbs == NULL) { mndReleaseRpcRef(pMnode); return -1; } @@ -632,6 +634,8 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr tstrncpy(pClusterInfo->version, version, sizeof(pClusterInfo->version)); pClusterInfo->monitor_interval = tsMonitorInterval; pClusterInfo->connections_total = mndGetNumOfConnections(pMnode); + pClusterInfo->dbs_total = sdbGetSize(pSdb, SDB_DB); + pClusterInfo->stbs_total = sdbGetSize(pSdb, SDB_STB); void *pIter = NULL; while (1) { @@ -681,6 +685,7 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr if (pIter == NULL) break; pClusterInfo->vgroups_total++; + pClusterInfo->tbs_total += pVgroup->numOfTables; SMonVgroupDesc desc = {0}; desc.vgroup_id = pVgroup->vgId; @@ -711,6 +716,27 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr sdbRelease(pSdb, pVgroup); } + // stb info + pIter = NULL; + while (1) { + SStbObj *pStb = NULL; + pIter = sdbFetch(pSdb, SDB_STB, pIter, (void **)&pStb); + if (pIter == NULL) break; + + SMonStbDesc desc = {0}; + + SName name1 = {0}; + tNameFromString(&name1, pStb->db, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + tNameGetDbName(&name1, desc.database_name); + + SName name2 = {0}; + tNameFromString(&name2, pStb->name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + tstrncpy(desc.stb_name, tNameGetTableName(&name2), TSDB_TABLE_NAME_LEN); + + taosArrayPush(pStbInfo->stbs, &desc); + sdbRelease(pSdb, pStb); + } + // grant info pGrantInfo->expire_time = (pMnode->grant.expireTimeMS - ms) / 86400000.0f; pGrantInfo->timeseries_total = pMnode->grant.timeseriesAllowed; diff --git a/source/dnode/mnode/impl/src/mndScheduler.c b/source/dnode/mnode/impl/src/mndScheduler.c index 9d8ed3701e..eb0ccbb5f7 100644 --- a/source/dnode/mnode/impl/src/mndScheduler.c +++ b/source/dnode/mnode/impl/src/mndScheduler.c @@ -97,37 +97,7 @@ END: return terrno; } -int32_t mndPersistTaskDeployReq(STrans* pTrans, SStreamTask* pTask, const SEpSet* pEpSet, tmsg_t type, int32_t nodeId) { - SEncoder encoder; - tEncoderInit(&encoder, NULL, 0); - tEncodeSStreamTask(&encoder, pTask); - int32_t size = encoder.pos; - int32_t tlen = sizeof(SMsgHead) + size; - tEncoderClear(&encoder); - void* buf = taosMemoryCalloc(1, tlen); - if (buf == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - ((SMsgHead*)buf)->vgId = htonl(nodeId); - void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); - tEncoderInit(&encoder, abuf, size); - tEncodeSStreamTask(&encoder, pTask); - tEncoderClear(&encoder); - - STransAction action = {0}; - memcpy(&action.epSet, pEpSet, sizeof(SEpSet)); - action.pCont = buf; - action.contLen = tlen; - action.msgType = type; - if (mndTransAppendRedoAction(pTrans, &action) != 0) { - taosMemoryFree(buf); - return -1; - } - return 0; -} - -int32_t mndAddSinkToTask(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream, SStreamTask* pTask) { +int32_t mndAddSinkToTask(SMnode* pMnode, SStreamObj* pStream, SStreamTask* pTask) { pTask->dispatchType = TASK_DISPATCH__NONE; // sink if (pStream->smaId != 0) { @@ -142,7 +112,7 @@ int32_t mndAddSinkToTask(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream, SS return 0; } -int32_t mndAddDispatcherToInnerTask(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream, SStreamTask* pTask) { +int32_t mndAddDispatcherToInnerTask(SMnode* pMnode, SStreamObj* pStream, SStreamTask* pTask) { pTask->sinkType = TASK_SINK__NONE; if (pStream->fixedSinkVgId == 0) { pTask->dispatchType = TASK_DISPATCH__SHUFFLE; @@ -187,7 +157,7 @@ int32_t mndAddDispatcherToInnerTask(SMnode* pMnode, STrans* pTrans, SStreamObj* return 0; } -int32_t mndAssignTaskToVg(SMnode* pMnode, STrans* pTrans, SStreamTask* pTask, SSubplan* plan, const SVgObj* pVgroup) { +int32_t mndAssignTaskToVg(SMnode* pMnode, SStreamTask* pTask, SSubplan* plan, const SVgObj* pVgroup) { int32_t msgLen; pTask->nodeId = pVgroup->vgId; pTask->epSet = mndGetVgroupEpset(pMnode, pVgroup); @@ -196,11 +166,11 @@ int32_t mndAssignTaskToVg(SMnode* pMnode, STrans* pTrans, SStreamTask* pTask, SS plan->execNode.epSet = pTask->epSet; if (qSubPlanToString(plan, &pTask->exec.qmsg, &msgLen) < 0) { + ASSERT(0); terrno = TSDB_CODE_QRY_INVALID_INPUT; return -1; } ASSERT(pTask->dispatchType != TASK_DISPATCH__NONE || pTask->sinkType != TASK_SINK__NONE); - mndPersistTaskDeployReq(pTrans, pTask, &plan->execNode.epSet, TDMT_STREAM_TASK_DEPLOY, pVgroup->vgId); return 0; } @@ -212,8 +182,7 @@ SSnodeObj* mndSchedFetchOneSnode(SMnode* pMnode) { return pObj; } -int32_t mndAssignTaskToSnode(SMnode* pMnode, STrans* pTrans, SStreamTask* pTask, SSubplan* plan, - const SSnodeObj* pSnode) { +int32_t mndAssignTaskToSnode(SMnode* pMnode, SStreamTask* pTask, SSubplan* plan, const SSnodeObj* pSnode) { int32_t msgLen; pTask->nodeId = SNODE_HANDLE; @@ -223,10 +192,10 @@ int32_t mndAssignTaskToSnode(SMnode* pMnode, STrans* pTrans, SStreamTask* pTask, plan->execNode.epSet = pTask->epSet; if (qSubPlanToString(plan, &pTask->exec.qmsg, &msgLen) < 0) { + ASSERT(0); terrno = TSDB_CODE_QRY_INVALID_INPUT; return -1; } - mndPersistTaskDeployReq(pTrans, pTask, &plan->execNode.epSet, TDMT_STREAM_TASK_DEPLOY, SNODE_HANDLE); return 0; } @@ -245,7 +214,7 @@ SVgObj* mndSchedFetchOneVg(SMnode* pMnode, int64_t dbUid) { return pVgroup; } -int32_t mndAddShuffleSinkTasksToStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) { +int32_t mndAddShuffleSinkTasksToStream(SMnode* pMnode, SStreamObj* pStream) { SSdb* pSdb = pMnode->pSdb; void* pIter = NULL; SArray* tasks = taosArrayGetP(pStream->tasks, 0); @@ -262,6 +231,7 @@ int32_t mndAddShuffleSinkTasksToStream(SMnode* pMnode, STrans* pTrans, SStreamOb } SStreamTask* pTask = tNewSStreamTask(pStream->uid); if (pTask == NULL) { + sdbRelease(pSdb, pVgroup); terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } @@ -290,13 +260,11 @@ int32_t mndAddShuffleSinkTasksToStream(SMnode* pMnode, STrans* pTrans, SStreamOb // dispatch pTask->dispatchType = TASK_DISPATCH__NONE; - - mndPersistTaskDeployReq(pTrans, pTask, &pTask->epSet, TDMT_STREAM_TASK_DEPLOY, pVgroup->vgId); } return 0; } -int32_t mndAddFixedSinkTaskToStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) { +int32_t mndAddFixedSinkTaskToStream(SMnode* pMnode, SStreamObj* pStream) { ASSERT(pStream->fixedSinkVgId != 0); SArray* tasks = taosArrayGetP(pStream->tasks, 0); SStreamTask* pTask = tNewSStreamTask(pStream->uid); @@ -337,13 +305,10 @@ int32_t mndAddFixedSinkTaskToStream(SMnode* pMnode, STrans* pTrans, SStreamObj* // dispatch pTask->dispatchType = TASK_DISPATCH__NONE; - - mndPersistTaskDeployReq(pTrans, pTask, &pTask->epSet, TDMT_STREAM_TASK_DEPLOY, pStream->fixedSinkVg.vgId); - return 0; } -int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) { +int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) { SSdb* pSdb = pMnode->pSdb; SQueryPlan* pPlan = qStringToQueryPlan(pStream->physicalPlan); if (pPlan == NULL) { @@ -368,9 +333,15 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) { // add extra sink hasExtraSink = true; if (pStream->fixedSinkVgId == 0) { - mndAddShuffleSinkTasksToStream(pMnode, pTrans, pStream); + if (mndAddShuffleSinkTasksToStream(pMnode, pStream) < 0) { + // TODO free + return -1; + } } else { - mndAddFixedSinkTaskToStream(pMnode, pTrans, pStream); + if (mndAddFixedSinkTaskToStream(pMnode, pStream) < 0) { + // TODO free + return -1; + } } } @@ -386,6 +357,11 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) { ASSERT(plan->subplanType == SUBPLAN_TYPE_MERGE); pInnerTask = tNewSStreamTask(pStream->uid); + if (pInnerTask == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + qDestroyQueryPlan(pPlan); + return -1; + } mndAddTaskToTaskSet(taskInnerLevel, pInnerTask); pInnerTask->childEpInfo = taosArrayInit(0, sizeof(void*)); @@ -397,7 +373,7 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) { pInnerTask->triggerParam = pStream->triggerParam; // dispatch - if (mndAddDispatcherToInnerTask(pMnode, pTrans, pStream, pInnerTask) < 0) { + if (mndAddDispatcherToInnerTask(pMnode, pStream, pInnerTask) < 0) { qDestroyQueryPlan(pPlan); return -1; } @@ -409,14 +385,13 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) { SSnodeObj* pSnode = mndSchedFetchOneSnode(pMnode); if (pSnode == NULL) { SVgObj* pVgroup = mndSchedFetchOneVg(pMnode, pStream->sourceDbUid); - if (mndAssignTaskToVg(pMnode, pTrans, pInnerTask, plan, pVgroup) < 0) { + if (mndAssignTaskToVg(pMnode, pInnerTask, plan, pVgroup) < 0) { sdbRelease(pSdb, pVgroup); qDestroyQueryPlan(pPlan); return -1; } } else { - if (mndAssignTaskToSnode(pMnode, pTrans, pInnerTask, plan, pSnode) < 0) { - ASSERT(0); + if (mndAssignTaskToSnode(pMnode, pInnerTask, plan, pSnode) < 0) { sdbRelease(pSdb, pSnode); qDestroyQueryPlan(pPlan); return -1; @@ -424,7 +399,7 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) { } } else { SVgObj* pVgroup = mndSchedFetchOneVg(pMnode, pStream->sourceDbUid); - if (mndAssignTaskToVg(pMnode, pTrans, pInnerTask, plan, pVgroup) < 0) { + if (mndAssignTaskToVg(pMnode, pInnerTask, plan, pVgroup) < 0) { sdbRelease(pSdb, pVgroup); qDestroyQueryPlan(pPlan); return -1; @@ -450,6 +425,12 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) { continue; } SStreamTask* pTask = tNewSStreamTask(pStream->uid); + if (pInnerTask == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + sdbRelease(pSdb, pVgroup); + qDestroyQueryPlan(pPlan); + return -1; + } mndAddTaskToTaskSet(taskSourceLevel, pTask); // source @@ -466,7 +447,7 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) { // exec pTask->execType = TASK_EXEC__PIPE; - if (mndAssignTaskToVg(pMnode, pTrans, pTask, plan, pVgroup) < 0) { + if (mndAssignTaskToVg(pMnode, pTask, plan, pVgroup) < 0) { sdbRelease(pSdb, pVgroup); qDestroyQueryPlan(pPlan); return -1; @@ -507,6 +488,11 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) { continue; } SStreamTask* pTask = tNewSStreamTask(pStream->uid); + if (pTask == NULL) { + sdbRelease(pSdb, pVgroup); + qDestroyQueryPlan(pPlan); + return -1; + } mndAddTaskToTaskSet(taskOneLevel, pTask); // source @@ -517,14 +503,14 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) { // sink or dispatch if (hasExtraSink) { - mndAddDispatcherToInnerTask(pMnode, pTrans, pStream, pTask); + mndAddDispatcherToInnerTask(pMnode, pStream, pTask); } else { - mndAddSinkToTask(pMnode, pTrans, pStream, pTask); + mndAddSinkToTask(pMnode, pStream, pTask); } // exec pTask->execType = TASK_EXEC__PIPE; - if (mndAssignTaskToVg(pMnode, pTrans, pTask, plan, pVgroup) < 0) { + if (mndAssignTaskToVg(pMnode, pTask, plan, pVgroup) < 0) { sdbRelease(pSdb, pVgroup); qDestroyQueryPlan(pPlan); return -1; diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index fed6a5a721..27de3883e9 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -236,7 +236,6 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) { int32_t numOfCols = pShow->pMeta->numOfColumns; SSDataBlock *pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); pBlock->pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); - pBlock->info.numOfCols = numOfCols; for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData idata = {0}; @@ -271,7 +270,7 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) { } size = sizeof(SRetrieveMetaTableRsp) + sizeof(int32_t) + sizeof(SSysTableSchema) * pShow->pMeta->numOfColumns + - blockDataGetSize(pBlock) + blockDataGetSerialMetaSize(pBlock->info.numOfCols); + blockDataGetSize(pBlock) + blockDataGetSerialMetaSize(taosArrayGetSize(pBlock->pDataBlock)); SRetrieveMetaTableRsp *pRsp = rpcMallocCont(size); if (pRsp == NULL) { diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index c44fb03be2..05603f8554 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -533,7 +533,7 @@ static int32_t mndCreateSma(SMnode *pMnode, SRpcMsg *pReq, SMCreateSmaReq *pCrea #if 0 smaObj.timezone = pCreate->timezone; #endif - smaObj.timezone = tsTimezone; // use timezone of server + smaObj.timezone = tsTimezone; // use timezone of server smaObj.interval = pCreate->interval; smaObj.offset = pCreate->offset; smaObj.sliding = pCreate->sliding; @@ -623,7 +623,7 @@ static int32_t mndCreateSma(SMnode *pMnode, SRpcMsg *pReq, SMCreateSmaReq *pCrea if (mndSetUpdateSmaStbCommitLogs(pMnode, pTrans, pStb) != 0) goto _OVER; // if (mndSetCreateSmaRedoActions(pMnode, pTrans, pDb, &smaObj) != 0) goto _OVER; if (mndSetCreateSmaVgroupRedoActions(pMnode, pTrans, pDb, &streamObj.fixedSinkVg, &smaObj) != 0) goto _OVER; - if (mndScheduleStream(pMnode, pTrans, &streamObj) != 0) goto _OVER; + if (mndScheduleStream(pMnode, &streamObj) != 0) goto _OVER; if (mndPersistStream(pMnode, pTrans, &streamObj) != 0) goto _OVER; if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 7e9069d5fc..96a82e2c18 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -320,11 +320,58 @@ FAIL: return 0; } +int32_t mndPersistTaskDeployReq(STrans *pTrans, const SStreamTask *pTask) { + SEncoder encoder; + tEncoderInit(&encoder, NULL, 0); + tEncodeSStreamTask(&encoder, pTask); + int32_t size = encoder.pos; + int32_t tlen = sizeof(SMsgHead) + size; + tEncoderClear(&encoder); + void *buf = taosMemoryCalloc(1, tlen); + if (buf == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + ((SMsgHead *)buf)->vgId = htonl(pTask->nodeId); + void *abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); + tEncoderInit(&encoder, abuf, size); + tEncodeSStreamTask(&encoder, pTask); + tEncoderClear(&encoder); + + STransAction action = {0}; + memcpy(&action.epSet, &pTask->epSet, sizeof(SEpSet)); + action.pCont = buf; + action.contLen = tlen; + action.msgType = TDMT_STREAM_TASK_DEPLOY; + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + taosMemoryFree(buf); + return -1; + } + return 0; +} + +int32_t mndPersistStreamTasks(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream) { + int32_t level = taosArrayGetSize(pStream->tasks); + for (int32_t i = 0; i < level; i++) { + SArray *pLevel = taosArrayGetP(pStream->tasks, i); + int32_t sz = taosArrayGetSize(pLevel); + for (int32_t j = 0; j < sz; j++) { + SStreamTask *pTask = taosArrayGetP(pLevel, j); + if (mndPersistTaskDeployReq(pTrans, pTask) < 0) { + return -1; + } + } + } + return 0; +} + int32_t mndPersistStream(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream) { + if (mndPersistStreamTasks(pMnode, pTrans, pStream) < 0) { + return -1; + } SSdbRaw *pCommitRaw = mndStreamActionEncode(pStream); if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr()); - mndTransDrop(pTrans); return -1; } sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); @@ -342,51 +389,6 @@ int32_t mndPersistDropStreamLog(SMnode *pMnode, STrans *pTrans, SStreamObj *pStr return 0; } -int32_t mndAddStreamToTrans(SMnode *pMnode, SStreamObj *pStream, const char *ast, STrans *pTrans) { - SNode *pAst = NULL; - - if (nodesStringToNode(ast, &pAst) < 0) { - return -1; - } - - if (qExtractResultSchema(pAst, (int32_t *)&pStream->outputSchema.nCols, &pStream->outputSchema.pSchema) != 0) { - nodesDestroyNode(pAst); - return -1; - } - // free - nodesDestroyNode(pAst); - -#if 0 - printf("|"); - for (int i = 0; i < pStream->outputSchema.nCols; i++) { - printf(" %15s |", (char *)pStream->outputSchema.pSchema[i].name); - } - printf("\n=======================================================\n"); - -#endif - - if (TSDB_CODE_SUCCESS != mndStreamGetPlanString(ast, pStream->trigger, pStream->watermark, &pStream->physicalPlan)) { - mError("topic:%s, failed to get plan since %s", pStream->name, terrstr()); - return -1; - } - - if (mndScheduleStream(pMnode, pTrans, pStream) < 0) { - mError("stream:%ld, schedule stream since %s", pStream->uid, terrstr()); - return -1; - } - mDebug("trans:%d, used to create stream:%s", pTrans->id, pStream->name); - - SSdbRaw *pCommitRaw = mndStreamActionEncode(pStream); - if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { - mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr()); - mndTransDrop(pTrans); - return -1; - } - sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); - - return 0; -} - static int32_t mndCreateStbForStream(SMnode *pMnode, STrans *pTrans, const SStreamObj *pStream, const char *user) { SStbObj *pStb = NULL; SDbObj *pDb = NULL; @@ -503,64 +505,6 @@ static int32_t mndDropStreamTasks(SMnode *pMnode, STrans *pTrans, SStreamObj *pS return 0; } -static int32_t mndCreateStream(SMnode *pMnode, SRpcMsg *pReq, SCMCreateStreamReq *pCreate, SDbObj *pDb) { - mDebug("stream:%s to create", pCreate->name); - SStreamObj streamObj = {0}; - tstrncpy(streamObj.name, pCreate->name, TSDB_STREAM_FNAME_LEN); - tstrncpy(streamObj.sourceDb, pDb->name, TSDB_DB_FNAME_LEN); - tstrncpy(streamObj.targetSTbName, pCreate->targetStbFullName, TSDB_TABLE_FNAME_LEN); - streamObj.createTime = taosGetTimestampMs(); - streamObj.updateTime = streamObj.createTime; - streamObj.uid = mndGenerateUid(pCreate->name, strlen(pCreate->name)); - streamObj.targetStbUid = mndGenerateUid(pCreate->targetStbFullName, TSDB_TABLE_FNAME_LEN); - streamObj.sourceDbUid = pDb->uid; - streamObj.version = 1; - streamObj.sql = pCreate->sql; - // TODO - streamObj.fixedSinkVgId = 0; - streamObj.smaId = 0; - streamObj.trigger = pCreate->triggerType; - streamObj.watermark = pCreate->watermark; - streamObj.triggerParam = pCreate->maxDelay; - - if (streamObj.targetSTbName[0]) { - pDb = mndAcquireDbByStb(pMnode, streamObj.targetSTbName); - if (pDb == NULL) { - terrno = TSDB_CODE_MND_DB_NOT_SELECTED; - return -1; - } - tstrncpy(streamObj.targetDb, pDb->name, TSDB_DB_FNAME_LEN); - } - - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq); - if (pTrans == NULL) { - mError("stream:%s, failed to create since %s", pCreate->name, terrstr()); - return -1; - } - mDebug("trans:%d, used to create stream:%s", pTrans->id, pCreate->name); - - if (mndAddStreamToTrans(pMnode, &streamObj, pCreate->ast, pTrans) != 0) { - mError("trans:%d, failed to add stream since %s", pTrans->id, terrstr()); - mndTransDrop(pTrans); - return -1; - } - - if (streamObj.targetSTbName[0] && mndCreateStbForStream(pMnode, pTrans, &streamObj, pReq->info.conn.user) < 0) { - mError("trans:%d, failed to create stb for stream since %s", pTrans->id, terrstr()); - mndTransDrop(pTrans); - return -1; - } - - if (mndTransPrepare(pMnode, pTrans) != 0) { - mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); - mndTransDrop(pTrans); - return -1; - } - - mndTransDrop(pTrans); - return 0; -} - static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; int32_t code = -1; @@ -631,7 +575,7 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) { } // schedule stream task for stream obj - if (mndScheduleStream(pMnode, pTrans, &streamObj) < 0) { + if (mndScheduleStream(pMnode, &streamObj) < 0) { mError("stream:%s, failed to schedule since %s", createStreamReq.name, terrstr()); mndTransDrop(pTrans); goto _OVER; @@ -653,8 +597,6 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) { mndTransDrop(pTrans); - /*code = mndCreateStream(pMnode, pReq, &createStreamReq, pDb);*/ - /*if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS;*/ code = TSDB_CODE_ACTION_IN_PROGRESS; _OVER: diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index d5d6f7c58a..c7e8e8dc86 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -149,8 +149,8 @@ int32_t tqReadHandleRemoveTbUidList(STqReadHandle *pHandle, const SArray *tbUidL int32_t tqReadHandleSetMsg(STqReadHandle *pHandle, SSubmitReq *pMsg, int64_t ver); bool tqNextDataBlock(STqReadHandle *pHandle); bool tqNextDataBlockFilterOut(STqReadHandle *pHandle, SHashObj *filterOutUids); -int32_t tqRetrieveDataBlock(SArray **ppCols, STqReadHandle *pHandle, uint64_t *pGroupId, uint64_t *pUid, - int32_t *pNumOfRows, int16_t *pNumOfCols); +int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, STqReadHandle *pHandle, uint64_t *pGroupId, uint64_t *pUid, + int32_t *pNumOfRows); // sma int32_t smaGetTSmaDays(SVnodeCfg *pCfg, void *pCont, uint32_t contLen, int32_t *days); diff --git a/source/dnode/vnode/src/tq/tqExec.c b/source/dnode/vnode/src/tq/tqExec.c index e6f960c8c1..be8fef1249 100644 --- a/source/dnode/vnode/src/tq/tqExec.c +++ b/source/dnode/vnode/src/tq/tqExec.c @@ -29,7 +29,7 @@ static int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataBlkRsp* pRs // TODO enable compress int32_t actualLen = 0; - blockCompressEncode(pBlock, pRetrieve->data, &actualLen, pBlock->info.numOfCols, false); + blockCompressEncode(pBlock, pRetrieve->data, &actualLen, taosArrayGetSize(pBlock->pDataBlock), false); actualLen += sizeof(SRetrieveTableRsp); ASSERT(actualLen <= dataStrLen); taosArrayPush(pRsp->blockDataLen, &actualLen); @@ -66,6 +66,7 @@ int32_t tqScanSnapshot(STQ* pTq, const STqExecHandle* pExec, SMqDataBlkRsp* pRsp if (qStreamScanSnapshot(task) < 0) { ASSERT(0); } + // set version while (1) { SSDataBlock* pDataBlock = NULL; uint64_t ts = 0; @@ -75,7 +76,7 @@ int32_t tqScanSnapshot(STQ* pTq, const STqExecHandle* pExec, SMqDataBlkRsp* pRsp if (pDataBlock == NULL) break; ASSERT(pDataBlock->info.rows != 0); - ASSERT(pDataBlock->info.numOfCols != 0); + ASSERT(taosArrayGetSize(pDataBlock->pDataBlock) != 0); tqAddBlockDataToRsp(pDataBlock, pRsp); pRsp->blockNum++; @@ -98,7 +99,6 @@ int32_t tqDataExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataBlkR if (pDataBlock == NULL) break; ASSERT(pDataBlock->info.rows != 0); - ASSERT(pDataBlock->info.numOfCols != 0); tqAddBlockDataToRsp(pDataBlock, pRsp); if (pRsp->withTbName) { @@ -112,8 +112,7 @@ int32_t tqDataExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataBlkR tqReadHandleSetMsg(pReader, pReq, 0); while (tqNextDataBlock(pReader)) { SSDataBlock block = {0}; - if (tqRetrieveDataBlock(&block.pDataBlock, pReader, &block.info.groupId, &block.info.uid, &block.info.rows, - &block.info.numOfCols) < 0) { + if (tqRetrieveDataBlock(&block, pReader, &block.info.groupId, &block.info.uid, &block.info.rows) < 0) { if (terrno == TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND) continue; ASSERT(0); } @@ -130,8 +129,7 @@ int32_t tqDataExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataBlkR tqReadHandleSetMsg(pReader, pReq, 0); while (tqNextDataBlockFilterOut(pReader, pExec->execDb.pFilterOutTbUid)) { SSDataBlock block = {0}; - if (tqRetrieveDataBlock(&block.pDataBlock, pReader, &block.info.groupId, &block.info.uid, &block.info.rows, - &block.info.numOfCols) < 0) { + if (tqRetrieveDataBlock(&block, pReader, &block.info.groupId, &block.info.uid, &block.info.rows) < 0) { if (terrno == TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND) continue; ASSERT(0); } diff --git a/source/dnode/vnode/src/tq/tqOffset.c b/source/dnode/vnode/src/tq/tqOffset.c index e5475d7c30..e536e87032 100644 --- a/source/dnode/vnode/src/tq/tqOffset.c +++ b/source/dnode/vnode/src/tq/tqOffset.c @@ -44,6 +44,7 @@ STqOffsetStore* tqOffsetOpen(STQ* pTq) { } char* fname = buildFileName(pStore->pTq->path); TdFilePtr pFile = taosOpenFile(fname, TD_FILE_READ); + taosMemoryFree(fname); if (pFile != NULL) { STqOffsetHead head = {0}; int64_t code; @@ -77,7 +78,6 @@ STqOffsetStore* tqOffsetOpen(STQ* pTq) { } taosCloseFile(&pFile); - taosMemoryFree(fname); } return pStore; } @@ -102,6 +102,7 @@ int32_t tqOffsetSnapshot(STqOffsetStore* pStore) { // TODO file name should be with a version char* fname = buildFileName(pStore->pTq->path); TdFilePtr pFile = taosOpenFile(fname, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND); + taosMemoryFree(fname); if (pFile == NULL) { ASSERT(0); return -1; @@ -140,6 +141,5 @@ int32_t tqOffsetSnapshot(STqOffsetStore* pStore) { } // close and rename file taosCloseFile(&pFile); - taosMemoryFree(fname); return 0; } diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index f7558cc00b..0243f2a9f0 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -142,15 +142,15 @@ bool tqNextDataBlockFilterOut(STqReadHandle* pHandle, SHashObj* filterOutUids) { return false; } -int32_t tqRetrieveDataBlock(SArray** ppCols, STqReadHandle* pHandle, uint64_t* pGroupId, uint64_t* pUid, - int32_t* pNumOfRows, int16_t* pNumOfCols) { +int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, STqReadHandle* pHandle, uint64_t* pGroupId, uint64_t* pUid, + int32_t* pNumOfRows) { *pUid = 0; - // TODO set to real sversion - /*int32_t sversion = 1;*/ + // TODO: cache multiple schema int32_t sversion = htonl(pHandle->pBlock->sversion); if (pHandle->cachedSchemaSuid == 0 || pHandle->cachedSchemaVer != sversion || pHandle->cachedSchemaSuid != pHandle->msgIter.suid) { + if (pHandle->pSchema) taosMemoryFree(pHandle->pSchema); pHandle->pSchema = metaGetTbTSchema(pHandle->pVnodeMeta, pHandle->msgIter.uid, sversion); if (pHandle->pSchema == NULL) { tqWarn("cannot found tsschema for table: uid: %ld (suid: %ld), version %d, possibly dropped table", @@ -160,7 +160,7 @@ int32_t tqRetrieveDataBlock(SArray** ppCols, STqReadHandle* pHandle, uint64_t* p return -1; } - // this interface use suid instead of uid + if (pHandle->pSchemaWrapper) tDeleteSSchemaWrapper(pHandle->pSchemaWrapper); pHandle->pSchemaWrapper = metaGetTableSchema(pHandle->pVnodeMeta, pHandle->msgIter.uid, sversion, true); if (pHandle->pSchemaWrapper == NULL) { tqWarn("cannot found schema wrapper for table: suid: %ld, version %d, possibly dropped table", @@ -180,23 +180,14 @@ int32_t tqRetrieveDataBlock(SArray** ppCols, STqReadHandle* pHandle, uint64_t* p int32_t colNumNeed = taosArrayGetSize(pHandle->pColIdList); if (colNumNeed == 0) { - *ppCols = taosArrayInit(pSchemaWrapper->nCols, sizeof(SColumnInfoData)); - if (*ppCols == NULL) { - return -1; - } - int32_t colMeta = 0; while (colMeta < pSchemaWrapper->nCols) { SSchema* pColSchema = &pSchemaWrapper->pSchema[colMeta]; - SColumnInfoData colInfo = {0}; - colInfo.info.bytes = pColSchema->bytes; - colInfo.info.colId = pColSchema->colId; - colInfo.info.type = pColSchema->type; - - if (colInfoDataEnsureCapacity(&colInfo, 0, *pNumOfRows) < 0) { + SColumnInfoData colInfo = createColumnInfoData(pColSchema->type, pColSchema->bytes, pColSchema->colId); + int32_t code = blockDataAppendColInfo(pBlock, &colInfo); + if (code != TSDB_CODE_SUCCESS) { goto FAIL; } - taosArrayPush(*ppCols, &colInfo); colMeta++; } } else { @@ -204,11 +195,6 @@ int32_t tqRetrieveDataBlock(SArray** ppCols, STqReadHandle* pHandle, uint64_t* p colNumNeed = pSchemaWrapper->nCols; } - *ppCols = taosArrayInit(colNumNeed, sizeof(SColumnInfoData)); - if (*ppCols == NULL) { - return -1; - } - int32_t colMeta = 0; int32_t colNeed = 0; while (colMeta < pSchemaWrapper->nCols && colNeed < colNumNeed) { @@ -220,23 +206,22 @@ int32_t tqRetrieveDataBlock(SArray** ppCols, STqReadHandle* pHandle, uint64_t* p } else if (colIdSchema > colIdNeed) { colNeed++; } else { - SColumnInfoData colInfo = {0}; - colInfo.info.bytes = pColSchema->bytes; - colInfo.info.colId = pColSchema->colId; - colInfo.info.type = pColSchema->type; - - if (colInfoDataEnsureCapacity(&colInfo, 0, *pNumOfRows) < 0) { + SColumnInfoData colInfo = createColumnInfoData(pColSchema->type, pColSchema->bytes, pColSchema->colId); + int32_t code = blockDataAppendColInfo(pBlock, &colInfo); + if (code != TSDB_CODE_SUCCESS) { goto FAIL; } - taosArrayPush(*ppCols, &colInfo); colMeta++; colNeed++; } } } - int32_t colActual = taosArrayGetSize(*ppCols); - *pNumOfCols = colActual; + if (blockDataEnsureCapacity(pBlock, *pNumOfRows) < 0) { + goto FAIL; + } + + int32_t colActual = blockDataGetNumOfCols(pBlock); // TODO in stream shuffle case, fetch groupId *pGroupId = 0; @@ -253,7 +238,7 @@ int32_t tqRetrieveDataBlock(SArray** ppCols, STqReadHandle* pHandle, uint64_t* p tdSTSRowIterReset(&iter, row); // get all wanted col of that block for (int32_t i = 0; i < colActual; i++) { - SColumnInfoData* pColData = taosArrayGet(*ppCols, i); + SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, i); SCellVal sVal = {0}; if (!tdSTSRowIterNext(&iter, pColData->info.colId, pColData->info.type, &sVal)) { break; @@ -265,8 +250,9 @@ int32_t tqRetrieveDataBlock(SArray** ppCols, STqReadHandle* pHandle, uint64_t* p curRow++; } return 0; -FAIL: - if (*ppCols) taosArrayDestroy(*ppCols); + +FAIL: // todo refactor here + // if (*ppCols) taosArrayDestroy(*ppCols); return -1; } diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index ece66d30fb..ef3b205b3e 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -44,7 +44,7 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema, boo createTbReq.ctb.suid = suid; STagVal tagVal = { - .cid = pDataBlock->info.numOfCols + 1, + .cid = taosArrayGetSize(pDataBlock->pDataBlock) + 1, .type = TSDB_DATA_TYPE_UBIGINT, .i64 = (int64_t)pDataBlock->info.groupId, }; @@ -107,7 +107,7 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema, boo createTbReq.ctb.suid = suid; STagVal tagVal = { - .cid = pDataBlock->info.numOfCols + 1, + .cid = taosArrayGetSize(pDataBlock->pDataBlock) + 1, .type = TSDB_DATA_TYPE_UBIGINT, .i64 = (int64_t)pDataBlock->info.groupId, }; diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index ce39b35aaa..e16360a58b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -16,13 +16,11 @@ #include "tsdb.h" #include "vnode.h" -#define EXTRA_BYTES 2 #define ASCENDING_TRAVERSE(o) (o == TSDB_ORDER_ASC) -#define QH_GET_NUM_OF_COLS(handle) ((size_t)(taosArrayGetSize((handle)->pColumns))) +#define QH_GET_NUM_OF_COLS(handle) ((size_t)(taosArrayGetSize((handle)->pResBlock->pDataBlock))) #define GET_FILE_DATA_BLOCK_INFO(_checkInfo, _block) \ ((SDataBlockInfo){.window = {.skey = (_block)->minKey.ts, .ekey = (_block)->maxKey.ts}, \ - .numOfCols = (_block)->numOfCols, \ .rows = (_block)->numOfRows, \ .uid = (_checkInfo)->tableId}) @@ -115,7 +113,8 @@ typedef struct STsdbReadHandle { // SColumnDataAgg* statis; // query level statistics, only one table block statistics info exists at any time // SColumnDataAgg** pstatis;// the ptr array list to return to caller int32_t numOfBlocks; - SArray* pColumns; // column list, SColumnInfoData array list + SSDataBlock* pResBlock; +// SArray* pColumns; // column list, SColumnInfoData array list bool locateStart; int32_t outputCapacity; int32_t realNumOfRows; @@ -177,7 +176,7 @@ static SArray* getColumnIdList(STsdbReadHandle* pTsdbReadHandle) { SArray* pIdList = taosArrayInit(numOfCols, sizeof(int16_t)); for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData* pCol = taosArrayGet(pTsdbReadHandle->pColumns, i); + SColumnInfoData* pCol = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, i); taosArrayPush(pIdList, &pCol->info.colId); } @@ -425,23 +424,21 @@ static STsdbReadHandle* tsdbQueryTablesImpl(SVnode* pVnode, SQueryTableDataCond* } // todo: use list instead of array? - pReadHandle->pColumns = taosArrayInit(pCond->numOfCols, sizeof(SColumnInfoData)); - if (pReadHandle->pColumns == NULL) { + pReadHandle->pResBlock = createDataBlock(); + if (pReadHandle->pResBlock == NULL) { goto _end; } for (int32_t i = 0; i < pCond->numOfCols; ++i) { - SColumnInfoData colInfo = {{0}, 0}; - colInfo.info = pCond->colList[i]; - - int32_t code = colInfoDataEnsureCapacity(&colInfo, 0, pReadHandle->outputCapacity); - if (code != TSDB_CODE_SUCCESS) { + SColumnInfoData colInfo = {.info = pCond->colList[i], 0}; + int32_t code = blockDataAppendColInfo(pReadHandle->pResBlock, &colInfo); + if (code != TSDB_CODE_SUCCESS){ goto _end; } - - taosArrayPush(pReadHandle->pColumns, &colInfo); } + blockDataEnsureCapacity(pReadHandle->pResBlock, pReadHandle->outputCapacity); + pReadHandle->suppInfo.defaultLoadColumn = getDefaultLoadColumns(pReadHandle, true); size_t size = taosArrayGetSize(pReadHandle->suppInfo.defaultLoadColumn); @@ -1536,12 +1533,12 @@ static int32_t doCopyRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, int32_t int32_t trueEnd = ascScan ? end : start; int32_t step = ascScan ? 1 : -1; - int32_t requiredNumOfCols = (int32_t)taosArrayGetSize(pTsdbReadHandle->pColumns); + int32_t requiredNumOfCols = (int32_t)taosArrayGetSize(pTsdbReadHandle->pResBlock->pDataBlock); // data in buffer has greater timestamp, copy data in file block int32_t i = 0, j = 0; while (i < requiredNumOfCols && j < pCols->numOfCols) { - SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i); + SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, i); SDataCol* src = &pCols->cols[j]; if (src->colId < pColInfo->info.colId) { @@ -1592,7 +1589,7 @@ static int32_t doCopyRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, int32_t } while (i < requiredNumOfCols) { // the remain columns are all null data - SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i); + SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, i); colDataAppendNNULL(pColInfo, numOfRows, num); i++; } @@ -1674,7 +1671,7 @@ static int32_t mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capa int32_t i = 0, j = 0, k = 0; while (i < numOfCols && (j < numOfColsOfRow1 || k < numOfColsOfRow2)) { - SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i); + SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, i); int32_t colIdOfRow1; if (j >= numOfColsOfRow1) { @@ -1805,7 +1802,7 @@ static int32_t mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capa if (mergeOption == 1) { while (i < numOfCols) { // the remain columns are all null data - SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i); + SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, i); colDataAppend(pColInfo, *curRow, NULL, true); ++i; } @@ -1861,7 +1858,7 @@ static void doCheckGeneratedBlockRange(STsdbReadHandle* pTsdbReadHandle) { assert(cur->win.skey >= pTsdbReadHandle->window.ekey && cur->win.ekey <= pTsdbReadHandle->window.skey); } - SColumnInfoData* pColInfoData = taosArrayGet(pTsdbReadHandle->pColumns, 0); + SColumnInfoData* pColInfoData = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, 0); assert(cur->win.skey == ((TSKEY*)pColInfoData->pData)[0] && cur->win.ekey == ((TSKEY*)pColInfoData->pData)[cur->rows - 1]); } else { @@ -2771,7 +2768,7 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int STsdbReadHandle* pTsdbReadHandle) { int numOfRows = 0; int curRows = 0; - int32_t numOfCols = (int32_t)taosArrayGetSize(pTsdbReadHandle->pColumns); + int32_t numOfCols = (int32_t)taosArrayGetSize(pTsdbReadHandle->pResBlock->pDataBlock); STsdbCfg* pCfg = REPO_CFG(pTsdbReadHandle->pTsdb); win->skey = TSKEY_INITIAL_VAL; @@ -2901,7 +2898,7 @@ static bool loadBlockOfActiveTable(STsdbReadHandle* pTsdbReadHandle) { if (exists) { tsdbRetrieveDataBlock((tsdbReaderT*)pTsdbReadHandle, NULL); if (pTsdbReadHandle->currentLoadExternalRows && pTsdbReadHandle->window.skey == pTsdbReadHandle->window.ekey) { - SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, 0); + SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, 0); assert(*(int64_t*)pColInfo->pData == pTsdbReadHandle->window.skey); } @@ -3003,7 +3000,7 @@ static bool loadCachedLastRow(STsdbReadHandle* pTsdbReadHandle) { // // int32_t i = 0, j = 0; // while(i < tgNumOfCols && j < numOfCols) { -// pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i); +// pColInfo = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, i); // if (pTable->lastCols[j].colId < pColInfo->info.colId) { // j++; // continue; @@ -3066,7 +3063,7 @@ static bool loadCachedLastRow(STsdbReadHandle* pTsdbReadHandle) { // continue; // } // -// pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, n); +// pColInfo = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, n); // pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes;; // // if (pColInfo->info.colId == PRIMARYKEY_TIMESTAMP_COL_ID) { @@ -3091,7 +3088,7 @@ static bool loadCachedLastRow(STsdbReadHandle* pTsdbReadHandle) { // // // leave the real ts column as the last row, because last function only (not stable) use the last row as res // if (priKey != TSKEY_INITIAL_VAL) { -// pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, priIdx); +// pColInfo = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, priIdx); // pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes; // // *(TSKEY *)pData = priKey; @@ -3101,7 +3098,7 @@ static bool loadCachedLastRow(STsdbReadHandle* pTsdbReadHandle) { // continue; // } // -// pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, n); +// pColInfo = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, n); // pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes;; // // assert (pColInfo->info.colId != PRIMARYKEY_TIMESTAMP_COL_ID); @@ -3161,9 +3158,9 @@ static bool loadDataBlockFromTableSeq(STsdbReadHandle* pTsdbReadHandle) { bool tsdbNextDataBlock(tsdbReaderT pHandle) { STsdbReadHandle* pTsdbReadHandle = (STsdbReadHandle*)pHandle; - size_t numOfCols = taosArrayGetSize(pTsdbReadHandle->pColumns); + size_t numOfCols = taosArrayGetSize(pTsdbReadHandle->pResBlock->pDataBlock); for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i); + SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, i); colInfoDataCleanup(pColInfo, pTsdbReadHandle->outputCapacity); } @@ -3250,7 +3247,7 @@ bool tsdbNextDataBlock(tsdbReaderT pHandle) { // SArray* row = (type == TSDB_PREV_ROW)? pTsdbReadHandle->prev : pTsdbReadHandle->next; // // for (int32_t i = 0; i < numOfCols; ++i) { -// SColumnInfoData* pCol = taosArrayGet(pTsdbReadHandle->pColumns, i); +// SColumnInfoData* pCol = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, i); // // SColumnInfoData colInfo = {{0}, 0}; // colInfo.info = pCol->info; @@ -3280,7 +3277,7 @@ bool tsdbNextDataBlock(tsdbReaderT pHandle) { // } // // for (int32_t i = 0; i < cond.numOfCols; ++i) { -// SColumnInfoData* pColInfoData = taosArrayGet(pTsdbReadHandle->pColumns, i); +// SColumnInfoData* pColInfoData = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, i); // memcpy(&cond.colList[i], &pColInfoData->info, sizeof(SColumnInfo)); // } // @@ -3342,7 +3339,7 @@ bool tsdbGetExternalRow(tsdbReaderT pHandle) { int32_t numOfCols = (int32_t)QH_GET_NUM_OF_COLS(pTsdbReadHandle); for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData* pColInfoData = taosArrayGet(pTsdbReadHandle->pColumns, i); + SColumnInfoData* pColInfoData = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, i); SColumnInfoData* first = taosArrayGet(pTsdbReadHandle->prev, i); memcpy(pColInfoData->pData, first->pData, pColInfoData->info.bytes); @@ -3617,13 +3614,13 @@ SArray* tsdbRetrieveDataBlock(tsdbReaderT* pTsdbReadHandle, SArray* pIdList) { */ STsdbReadHandle* pHandle = (STsdbReadHandle*)pTsdbReadHandle; if (pHandle->cur.fid == INT32_MIN) { - return pHandle->pColumns; + return pHandle->pResBlock->pDataBlock; } else { STableBlockInfo* pBlockInfo = &pHandle->pDataBlockInfo[pHandle->cur.slot]; STableCheckInfo* pCheckInfo = pBlockInfo->pTableCheckInfo; if (pHandle->cur.mixBlock) { - return pHandle->pColumns; + return pHandle->pResBlock->pDataBlock; } else { SDataBlockInfo binfo = GET_FILE_DATA_BLOCK_INFO(pCheckInfo, pBlockInfo->compBlock); assert(pHandle->realNumOfRows <= binfo.rows); @@ -3633,7 +3630,7 @@ SArray* tsdbRetrieveDataBlock(tsdbReaderT* pTsdbReadHandle, SArray* pIdList) { if (pBlockLoadInfo->slot == pHandle->cur.slot && pBlockLoadInfo->fileGroup->fid == pHandle->cur.fid && pBlockLoadInfo->uid == pCheckInfo->tableId) { - return pHandle->pColumns; + return pHandle->pResBlock->pDataBlock; } else { // only load the file block SBlock* pBlock = pBlockInfo->compBlock; if (doLoadFileDataBlock(pHandle, pBlock, pCheckInfo, pHandle->cur.slot) != TSDB_CODE_SUCCESS) { @@ -3641,7 +3638,7 @@ SArray* tsdbRetrieveDataBlock(tsdbReaderT* pTsdbReadHandle, SArray* pIdList) { } int32_t numOfRows = doCopyRowsFromFileBlock(pHandle, pHandle->outputCapacity, 0, 0, pBlock->numOfRows - 1); - return pHandle->pColumns; + return pHandle->pResBlock->pDataBlock; } } } @@ -3692,7 +3689,7 @@ void tsdbCleanupReadHandle(tsdbReaderT queryHandle) { return; } - pTsdbReadHandle->pColumns = doFreeColumnInfoData(pTsdbReadHandle->pColumns); + pTsdbReadHandle->pResBlock->pDataBlock = doFreeColumnInfoData(pTsdbReadHandle->pResBlock->pDataBlock); taosArrayDestroy(pTsdbReadHandle->suppInfo.defaultLoadColumn); taosMemoryFreeClear(pTsdbReadHandle->pDataBlockInfo); diff --git a/source/dnode/vnode/test/tsdbSmaTest.cpp b/source/dnode/vnode/test/tsdbSmaTest.cpp index 2c1e6fbbbd..2a8c7a583c 100644 --- a/source/dnode/vnode/test/tsdbSmaTest.cpp +++ b/source/dnode/vnode/test/tsdbSmaTest.cpp @@ -435,7 +435,7 @@ TEST(testCase, tSma_Data_Insert_Query_Test) { SSDataBlock *pDataBlock = (SSDataBlock *)taosMemoryCalloc(1, sizeof(SSDataBlock)); EXPECT_NE(pDataBlock, nullptr); pDataBlock->pBlockAgg = NULL; - pDataBlock->info.numOfCols = tSmaNumOfCols; + taosArrayGetSize(pDataBlock->pDataBlock) = tSmaNumOfCols; pDataBlock->info.rows = tSmaNumOfRows; pDataBlock->info.groupId = tSmaGroupId + g; diff --git a/source/libs/command/src/command.c b/source/libs/command/src/command.c index 6b0a84a37e..cb14b916af 100644 --- a/source/libs/command/src/command.c +++ b/source/libs/command/src/command.c @@ -32,31 +32,19 @@ static int32_t getSchemaBytes(const SSchema* pSchema) { } static SSDataBlock* buildDescResultDataBlock() { - SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); - pBlock->info.numOfCols = DESCRIBE_RESULT_COLS; - pBlock->info.hasVarCol = true; + SSDataBlock* pBlock = createDataBlock(); - pBlock->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData)); + SColumnInfoData infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, DESCRIBE_RESULT_FIELD_LEN, 1); + blockDataAppendColInfo(pBlock, &infoData); - SColumnInfoData infoData = {0}; - infoData.info.type = TSDB_DATA_TYPE_VARCHAR; - infoData.info.bytes = DESCRIBE_RESULT_FIELD_LEN; + infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, DESCRIBE_RESULT_TYPE_LEN, 2); + blockDataAppendColInfo(pBlock, &infoData); - taosArrayPush(pBlock->pDataBlock, &infoData); - - infoData.info.type = TSDB_DATA_TYPE_VARCHAR; - infoData.info.bytes = DESCRIBE_RESULT_TYPE_LEN; - taosArrayPush(pBlock->pDataBlock, &infoData); - - infoData.info.type = TSDB_DATA_TYPE_INT; - infoData.info.bytes = tDataTypes[TSDB_DATA_TYPE_INT].bytes; - - taosArrayPush(pBlock->pDataBlock, &infoData); - - infoData.info.type = TSDB_DATA_TYPE_VARCHAR; - infoData.info.bytes = DESCRIBE_RESULT_NOTE_LEN; - taosArrayPush(pBlock->pDataBlock, &infoData); + infoData = createColumnInfoData(TSDB_DATA_TYPE_INT, tDataTypes[TSDB_DATA_TYPE_INT].bytes, 3); + blockDataAppendColInfo(pBlock, &infoData); + infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, DESCRIBE_RESULT_NOTE_LEN, 4); + blockDataAppendColInfo(pBlock, &infoData); return pBlock; } @@ -127,22 +115,12 @@ static int32_t execResetQueryCache() { return catalogClearCache(); } static SSDataBlock* buildCreateDBResultDataBlock() { - SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); - pBlock->info.numOfCols = SHOW_CREATE_DB_RESULT_COLS; - pBlock->info.hasVarCol = true; - - pBlock->pDataBlock = taosArrayInit(pBlock->info.numOfCols, sizeof(SColumnInfoData)); - - SColumnInfoData infoData = {0}; - infoData.info.type = TSDB_DATA_TYPE_VARCHAR; - infoData.info.bytes = SHOW_CREATE_DB_RESULT_FIELD1_LEN; - - taosArrayPush(pBlock->pDataBlock, &infoData); - - infoData.info.type = TSDB_DATA_TYPE_VARCHAR; - infoData.info.bytes = SHOW_CREATE_DB_RESULT_FIELD2_LEN; - taosArrayPush(pBlock->pDataBlock, &infoData); + SSDataBlock* pBlock = createDataBlock(); + SColumnInfoData infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, SHOW_CREATE_DB_RESULT_COLS, 1); + blockDataAppendColInfo(pBlock, &infoData); + infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, SHOW_CREATE_DB_RESULT_FIELD2_LEN, 2); + blockDataAppendColInfo(pBlock, &infoData); return pBlock; } @@ -284,21 +262,13 @@ static int32_t execShowCreateDatabase(SShowCreateDatabaseStmt* pStmt, SRetrieveT } static SSDataBlock* buildCreateTbResultDataBlock() { - SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); - pBlock->info.numOfCols = SHOW_CREATE_TB_RESULT_COLS; - pBlock->info.hasVarCol = true; + SSDataBlock* pBlock = createDataBlock(); - pBlock->pDataBlock = taosArrayInit(pBlock->info.numOfCols, sizeof(SColumnInfoData)); + SColumnInfoData infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, SHOW_CREATE_TB_RESULT_FIELD1_LEN, 1); + blockDataAppendColInfo(pBlock, &infoData); - SColumnInfoData infoData = {0}; - infoData.info.type = TSDB_DATA_TYPE_VARCHAR; - infoData.info.bytes = SHOW_CREATE_TB_RESULT_FIELD1_LEN; - - taosArrayPush(pBlock->pDataBlock, &infoData); - - infoData.info.type = TSDB_DATA_TYPE_VARCHAR; - infoData.info.bytes = SHOW_CREATE_TB_RESULT_FIELD2_LEN; - taosArrayPush(pBlock->pDataBlock, &infoData); + infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, SHOW_CREATE_TB_RESULT_FIELD2_LEN, 2); + blockDataAppendColInfo(pBlock, &infoData); return pBlock; } @@ -550,14 +520,23 @@ static int32_t execShowCreateSTable(SShowCreateTableStmt* pStmt, SRetrieveTableR return execShowCreateTable(pStmt, pRsp); } -static int32_t execAlterLocal(SAlterLocalStmt* pStmt) { return TSDB_CODE_FAILED; } +static int32_t execAlterLocal(SAlterLocalStmt* pStmt) { + if (cfgSetItem(tsCfg, pStmt->config, pStmt->value, CFG_STYPE_ALTER_CMD)) { + return terrno; + } + + if (taosSetCfg(tsCfg, pStmt->config)) { + return terrno; + } + + return TSDB_CODE_SUCCESS; +} static SSDataBlock* buildLocalVariablesResultDataBlock() { SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); - pBlock->info.numOfCols = SHOW_LOCAL_VARIABLES_RESULT_COLS; pBlock->info.hasVarCol = true; - pBlock->pDataBlock = taosArrayInit(pBlock->info.numOfCols, sizeof(SColumnInfoData)); + pBlock->pDataBlock = taosArrayInit(SHOW_LOCAL_VARIABLES_RESULT_COLS, sizeof(SColumnInfoData)); SColumnInfoData infoData = {0}; infoData.info.type = TSDB_DATA_TYPE_VARCHAR; diff --git a/source/libs/command/src/explain.c b/source/libs/command/src/explain.c index 2d55b06f06..ef4ac0b639 100644 --- a/source/libs/command/src/explain.c +++ b/source/libs/command/src/explain.c @@ -1190,17 +1190,12 @@ int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) { QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR); } - SSDataBlock *pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); - SColumnInfoData infoData = {0}; - infoData.info.type = TSDB_DATA_TYPE_VARCHAR; - infoData.info.bytes = TSDB_EXPLAIN_RESULT_ROW_SIZE; - - pBlock->pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData)); - taosArrayPush(pBlock->pDataBlock, &infoData); + SSDataBlock *pBlock = createDataBlock(); + SColumnInfoData infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, TSDB_EXPLAIN_RESULT_ROW_SIZE, 1); + blockDataAppendColInfo(pBlock, &infoData); + blockDataEnsureCapacity(pBlock, rowNum); SColumnInfoData* pInfoData = taosArrayGet(pBlock->pDataBlock, 0); - pInfoData->hasNull = false; - colInfoDataEnsureCapacity(pInfoData, 0, rowNum); char buf[1024] = {0}; for (int32_t i = 0; i < rowNum; ++i) { @@ -1210,9 +1205,7 @@ int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) { colDataAppend(pInfoData, i, buf, false); } - pBlock->info.numOfCols = 1; pBlock->info.rows = rowNum; - pBlock->info.hasVarCol = true; int32_t rspSize = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock); @@ -1226,7 +1219,7 @@ int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) { rsp->numOfRows = htonl(rowNum); int32_t len = 0; - blockCompressEncode(pBlock, rsp->data, &len, pBlock->info.numOfCols, 0); + blockCompressEncode(pBlock, rsp->data, &len, taosArrayGetSize(pBlock->pDataBlock), 0); ASSERT(len == rspSize - sizeof(SRetrieveTableRsp)); rsp->compLen = htonl(len); diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 91931c2fd8..57a2c57b16 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -336,7 +336,6 @@ typedef struct SStreamBlockScanInfo { int32_t numOfPseudoExpr; int32_t primaryTsIndex; // primary time stamp slot id - void* pDataReader; SReadHandle readHandle; uint64_t tableUid; // queried super table uid EStreamScanMode scanMode; diff --git a/source/libs/executor/src/dataDeleter.c b/source/libs/executor/src/dataDeleter.c index 9db2783ccd..8c220134f0 100644 --- a/source/libs/executor/src/dataDeleter.c +++ b/source/libs/executor/src/dataDeleter.c @@ -75,7 +75,7 @@ static void toDataCacheEntry(SDataDeleterHandle* pHandle, const SInputData* pInp SDataCacheEntry* pEntry = (SDataCacheEntry*)pBuf->pData; pEntry->compressed = 0; pEntry->numOfRows = pInput->pData->info.rows; - pEntry->numOfCols = pInput->pData->info.numOfCols; + pEntry->numOfCols = taosArrayGetSize(pInput->pData->pDataBlock); pEntry->dataLen = sizeof(SDeleterRes); ASSERT(1 == pEntry->numOfRows); diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index ac38fd8de9..8669c2c28c 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -183,34 +183,24 @@ SArray* createSortInfo(SNodeList* pNodeList) { SSDataBlock* createResDataBlock(SDataBlockDescNode* pNode) { int32_t numOfCols = LIST_LENGTH(pNode->pSlots); - SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); - pBlock->pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); + SSDataBlock* pBlock = createDataBlock(); pBlock->info.blockId = pNode->dataBlockId; - pBlock->info.rowSize = pNode->totalRowSize; // todo ?? pBlock->info.type = STREAM_INVALID; for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData idata = {{0}}; SSlotDescNode* pDescNode = (SSlotDescNode*)nodesListGetNode(pNode->pSlots, i); // if (!pDescNode->output) { // todo disable it temporarily // continue; // } - idata.info.type = pDescNode->dataType.type; - idata.info.bytes = pDescNode->dataType.bytes; + SColumnInfoData idata = createColumnInfoData(pDescNode->dataType.type, pDescNode->dataType.bytes, pDescNode->slotId); idata.info.scale = pDescNode->dataType.scale; - idata.info.slotId = pDescNode->slotId; idata.info.precision = pDescNode->dataType.precision; - if (IS_VAR_DATA_TYPE(idata.info.type)) { - pBlock->info.hasVarCol = true; - } - - taosArrayPush(pBlock->pDataBlock, &idata); + blockDataAppendColInfo(pBlock, &idata); } - pBlock->info.numOfCols = taosArrayGetSize(pBlock->pDataBlock); return pBlock; } @@ -698,7 +688,7 @@ void relocateColumnData(SSDataBlock* pBlock, const SArray* pColMatchInfo, SArray if (p->info.colId == pmInfo->colId) { SColumnInfoData* pDst = taosArrayGet(pBlock->pDataBlock, pmInfo->targetSlotId); - colDataAssign(pDst, p, pBlock->info.rows); + colDataAssign(pDst, p, pBlock->info.rows, &pBlock->info); i++; j++; } else if (p->info.colId < pmInfo->colId) { diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 1bd1ce14b1..ad400d720a 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -40,15 +40,8 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu SStreamBlockScanInfo* pInfo = pOperator->info; pInfo->assignBlockUid = assignUid; - // no need to check -#if 0 - if (pInfo->blockType == 0) { - pInfo->blockType = type; - } else if (pInfo->blockType != type) { - ASSERT(0); - return TSDB_CODE_QRY_APP_ERROR; - } -#endif + // TODO: if a block was set but not consumed, + // prevent setting a different type of block pInfo->blockType = type; if (type == STREAM_DATA_TYPE_SUBMIT_BLOCK) { diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 51ba7cb87f..6c416ae9c4 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -356,7 +356,7 @@ void initExecTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pQueryWindow pColData->info.type = TSDB_DATA_TYPE_TIMESTAMP; pColData->info.bytes = sizeof(int64_t); - colInfoDataEnsureCapacity(pColData, 0, 5); + colInfoDataEnsureCapacity(pColData, 5); colDataAppendInt64(pColData, 0, &pQueryWindow->skey); colDataAppendInt64(pColData, 1, &pQueryWindow->ekey); @@ -458,7 +458,7 @@ static int32_t doCreateConstantValColumnInfo(SInputColumnInfoData* pInput, SFunc pColInfo = pInput->pData[paramIndex]; } - colInfoDataEnsureCapacity(pColInfo, 0, numOfRows); + colInfoDataEnsureCapacity(pColInfo, numOfRows); int8_t type = pFuncParam->param.nType; if (type == TSDB_DATA_TYPE_BIGINT || type == TSDB_DATA_TYPE_UBIGINT) { @@ -579,7 +579,7 @@ int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBloc colDataMergeCol(pColInfoData, pResult->info.rows, &pResult->info.capacity, pfCtx->input.pData[0], pfCtx->input.numOfRows); } else { - colDataAssign(pColInfoData, pfCtx->input.pData[0], pfCtx->input.numOfRows); + colDataAssign(pColInfoData, pfCtx->input.pData[0], pfCtx->input.numOfRows, &pResult->info); } numOfRows = pfCtx->input.numOfRows; @@ -1186,7 +1186,7 @@ int32_t loadDataBlockOnDemand(SExecTaskInfo* pTaskInfo, STableScanInfo* pTableSc // } // if (pQueryAttr->pFilters != NULL) { -// filterSetColFieldData(pQueryAttr->pFilters, pBlock->info.numOfCols, pBlock->pDataBlock); +// filterSetColFieldData(pQueryAttr->pFilters, taosArrayGetSize(pBlock->pDataBlock), pBlock->pDataBlock); // } // if (pQueryAttr->pFilters != NULL || pRuntimeEnv->pTsBuf != NULL) { @@ -1321,7 +1321,8 @@ void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock) { // todo move to the initialization function int32_t code = filterInitFromNode((SNode*)pFilterNode, &filter, 0); - SFilterColumnParam param1 = {.numOfCols = pBlock->info.numOfCols, .pDataBlock = pBlock->pDataBlock}; + size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); + SFilterColumnParam param1 = {.numOfCols = numOfCols, .pDataBlock = pBlock->pDataBlock}; code = filterSetDataFromSlotId(filter, ¶m1); int8_t* rowRes = NULL; @@ -1343,7 +1344,8 @@ void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const int8_t* rowR int32_t totalRows = pBlock->info.rows; SSDataBlock* px = createOneDataBlock(pBlock, true); - for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { + size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); + for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pSrc = taosArrayGet(px->pDataBlock, i); SColumnInfoData* pDst = taosArrayGet(pBlock->pDataBlock, i); // it is a reserved column for scalar function, and no data in this column yet. @@ -2045,8 +2047,9 @@ int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLo int32_t compLen, int32_t numOfOutput, int64_t startTs, uint64_t* total, SArray* pColList) { if (pColList == NULL) { // data from other sources + blockDataCleanup(pRes); + // blockDataEnsureCapacity(pRes, numOfRows); blockCompressDecode(pRes, numOfOutput, numOfRows, pData); - pRes->info.rows = numOfRows; } else { // extract data according to pColList ASSERT(numOfOutput == taosArrayGetSize(pColList)); char* pStart = pData; @@ -2064,32 +2067,19 @@ int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLo pStart += sizeof(SSysTableSchema); } - SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); - pBlock->pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); - pBlock->info.numOfCols = numOfCols; - + SSDataBlock* pBlock = createDataBlock(); for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData idata = {0}; - - idata.info.type = pSchema[i].type; - idata.info.bytes = pSchema[i].bytes; - idata.info.colId = pSchema[i].colId; - idata.hasNull = true; - - taosArrayPush(pBlock->pDataBlock, &idata); - if (IS_VAR_DATA_TYPE(idata.info.type)) { - pBlock->info.hasVarCol = true; - } + SColumnInfoData idata = createColumnInfoData(pSchema[i].type, pSchema[i].bytes, pSchema[i].colId); + blockDataAppendColInfo(pBlock, &idata); } blockCompressDecode(pBlock, numOfCols, numOfRows, pStart); + blockDataEnsureCapacity(pRes, numOfRows); // data from mnode pRes->info.rows = numOfRows; relocateColumnData(pRes, pColList, pBlock->pDataBlock, false); - taosArrayDestroy(pBlock->pDataBlock); - taosMemoryFree(pBlock); - // blockDataDestroy(pBlock); + blockDataDestroy(pBlock); } // todo move this to time window aggregator, since the primary timestamp may not be known by exchange operator. @@ -2420,7 +2410,7 @@ SOperatorInfo* createExchangeOperatorInfo(void* pTransporter, SExchangePhysiNode pOperator->blocking = false; pOperator->status = OP_NOT_OPENED; pOperator->info = pInfo; - pOperator->exprSupp.numOfExprs = pInfo->pResult->info.numOfCols; + pOperator->exprSupp.numOfExprs = taosArrayGetSize(pInfo->pResult->pDataBlock); pOperator->pTaskInfo = pTaskInfo; pOperator->fpSet = createOperatorFpSet(prepareLoadRemoteData, doLoadRemoteData, NULL, NULL, @@ -2542,9 +2532,6 @@ static void doMergeImpl(SOperatorInfo* pOperator, int32_t numOfExpr, SSDataBlock SSortedMergeOperatorInfo* pInfo = pOperator->info; SqlFunctionCtx* pCtx = pOperator->exprSupp.pCtx; - for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { - // pCtx[i].size = 1; - } for (int32_t i = 0; i < pBlock->info.rows; ++i) { if (!pInfo->hasGroupVal) { @@ -2653,7 +2640,7 @@ SSDataBlock* getSortedMergeBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlo SColumnInfoData* pSrc = taosArrayGet(p->pDataBlock, pmInfo->srcSlotId); SColumnInfoData* pDst = taosArrayGet(pDataBlock->pDataBlock, pmInfo->targetSlotId); - colDataAssign(pDst, pSrc, p->info.rows); + colDataAssign(pDst, pSrc, p->info.rows, &pDataBlock->info); } pDataBlock->info.rows = p->info.rows; @@ -3971,7 +3958,7 @@ int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle, } } } - int32_t len = (int32_t)(pStart - (char*)keyBuf); + int32_t len = (int32_t)(pStart - (char*)keyBuf); uint64_t* pGroupId = taosHashGet(pTableListInfo->map, keyBuf, len); @@ -3997,8 +3984,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo if (QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN == type) { STableScanPhysiNode* pTableScanNode = (STableScanPhysiNode*)pPhyNode; - tsdbReaderT pDataReader = - doCreateDataReader(pTableScanNode, pHandle, pTableListInfo, (uint64_t)queryId, taskId); + tsdbReaderT pDataReader = doCreateDataReader(pTableScanNode, pHandle, pTableListInfo, (uint64_t)queryId, taskId); if (pDataReader == NULL && terrno != 0) { pTaskInfo->code = terrno; return NULL; @@ -4024,16 +4010,20 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo STableScanInfo* pScanInfo = pOperator->info; pTaskInfo->cost.pRecoder = &pScanInfo->readRecorder; return pOperator; + } else if (QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN == type) { STableMergeScanPhysiNode* pTableScanNode = (STableMergeScanPhysiNode*)pPhyNode; createScanTableListInfo(pTableScanNode, pHandle, pTableListInfo, queryId, taskId); extractTableSchemaVersion(pHandle, pTableScanNode->scan.uid, pTaskInfo); - SOperatorInfo* pOperator = createTableMergeScanOperatorInfo(pTableScanNode, pTableListInfo, pHandle, pTaskInfo, queryId, taskId); + SOperatorInfo* pOperator = + createTableMergeScanOperatorInfo(pTableScanNode, pTableListInfo, pHandle, pTaskInfo, queryId, taskId); STableScanInfo* pScanInfo = pOperator->info; pTaskInfo->cost.pRecoder = &pScanInfo->readRecorder; return pOperator; + } else if (QUERY_NODE_PHYSICAL_PLAN_EXCHANGE == type) { return createExchangeOperatorInfo(pHandle->pMsgCb->clientRpc, (SExchangePhysiNode*)pPhyNode, pTaskInfo); + } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN == type) { SScanPhysiNode* pScanPhyNode = (SScanPhysiNode*)pPhyNode; // simple child table. STableScanPhysiNode* pTableScanNode = (STableScanPhysiNode*)pPhyNode; @@ -4048,20 +4038,22 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo if (pHandle->initTsdbReader) { // for stream ASSERT(pHandle->vnode); - pDataReader = - doCreateDataReader(pTableScanNode, pHandle, pTableListInfo, (uint64_t)queryId, taskId); + pDataReader = doCreateDataReader(pTableScanNode, pHandle, pTableListInfo, (uint64_t)queryId, taskId); } else { // for tq ASSERT(pHandle->meta); getTableList(pHandle->meta, pScanPhyNode, pTableListInfo); } } + +#if 0 if (pDataReader == NULL && terrno != 0) { qDebug("%s pDataReader is NULL", GET_TASKID(pTaskInfo)); // return NULL; } else { qDebug("%s pDataReader is not NULL", GET_TASKID(pTaskInfo)); } +#endif SArray* groupKeys = extractPartitionColInfo(pTableScanNode->pPartitionTags); int32_t code = generateGroupIdMap(pTableListInfo, pHandle, groupKeys); // todo for json @@ -4074,9 +4066,11 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo SOperatorInfo* pOperator = createStreamScanOperatorInfo(pDataReader, pHandle, pTableScanNode, pTaskInfo, &twSup); return pOperator; + } else if (QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN == type) { SSystemTableScanPhysiNode* pSysScanPhyNode = (SSystemTableScanPhysiNode*)pPhyNode; return createSysTableScanOperatorInfo(pHandle, pSysScanPhyNode, pTaskInfo); + } else if (QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN == type) { STagScanPhysiNode* pScanPhyNode = (STagScanPhysiNode*)pPhyNode; @@ -4087,6 +4081,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo } return createTagScanOperatorInfo(pHandle, pScanPhyNode, pTableListInfo, pTaskInfo); + } else if (QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN == type) { SBlockDistScanPhysiNode* pBlockNode = (SBlockDistScanPhysiNode*)pPhyNode; pTableListInfo->pTableList = taosArrayInit(4, sizeof(STableKeyInfo)); @@ -4360,7 +4355,7 @@ tsdbReaderT doCreateDataReader(STableScanPhysiNode* pTableScanNode, SReadHandle* goto _error; } - tsdbReaderT* pReader = tsdbReaderOpen(pHandle->vnode, &cond, pTableListInfo, queryId, taskId); + tsdbReaderT pReader = tsdbReaderOpen(pHandle->vnode, &cond, pTableListInfo, queryId, taskId); cleanupQueryTableDataCond(&cond); return pReader; @@ -4370,6 +4365,62 @@ _error: return NULL; } +int32_t extractTableScanNode(SPhysiNode* pNode, STableScanPhysiNode** ppNode) { + if (pNode->pChildren == NULL || LIST_LENGTH(pNode->pChildren) == 0) { + if (QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN == pNode->type) { + *ppNode = (STableScanPhysiNode*)pNode; + return 0; + } else { + ASSERT(0); + terrno = TSDB_CODE_QRY_APP_ERROR; + return -1; + } + } else { + if (LIST_LENGTH(pNode->pChildren) != 1) { + ASSERT(0); + terrno = TSDB_CODE_QRY_APP_ERROR; + return -1; + } + SPhysiNode* pChildNode = (SPhysiNode*)nodesListGetNode(pNode->pChildren, 0); + return extractTableScanNode(pChildNode, ppNode); + } + return -1; +} + +int32_t doRebuildReader(SOperatorInfo* pOperator, SSubplan* plan, SReadHandle* pHandle) { + if (pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { + if (pOperator->numOfDownstream == 0) { + qError("failed to find stream scan operator"); + return TSDB_CODE_QRY_APP_ERROR; + } + + if (pOperator->numOfDownstream > 1) { + qError("join not supported for stream block scan"); + return TSDB_CODE_QRY_APP_ERROR; + } + return doRebuildReader(pOperator->pDownstream[0], plan, pHandle); + } else { + SStreamBlockScanInfo* pInfo = pOperator->info; + ASSERT(pInfo->pSnapshotReadOp->operatorType == QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN); + STableScanInfo* pTableScanInfo = pInfo->pSnapshotReadOp->info; + + tsdbCleanupReadHandle(pTableScanInfo->dataReader); + STableScanPhysiNode* pNode = NULL; + if (extractTableScanNode(plan->pNode, &pNode) < 0) { + ASSERT(0); + } + + STableListInfo info = {0}; + pTableScanInfo->dataReader = doCreateDataReader(pNode, pHandle, &info, 0, 0); + if (pTableScanInfo->dataReader == NULL) { + ASSERT(0); + qError("failed to create data reader"); + return TSDB_CODE_QRY_APP_ERROR; + } + } + return 0; +} + int32_t encodeOperator(SOperatorInfo* ops, char** result, int32_t* length) { int32_t code = TDB_CODE_SUCCESS; char* pCurrent = NULL; @@ -4504,18 +4555,14 @@ int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SRead (*pTaskInfo)->sql = sql; (*pTaskInfo)->tableqinfoList.pTagCond = pPlan->pTagCond; (*pTaskInfo)->tableqinfoList.pTagIndexCond = pPlan->pTagIndexCond; - (*pTaskInfo)->pRoot = createOperatorTree(pPlan->pNode, *pTaskInfo, pHandle, queryId, taskId, - &(*pTaskInfo)->tableqinfoList); + (*pTaskInfo)->pRoot = + createOperatorTree(pPlan->pNode, *pTaskInfo, pHandle, queryId, taskId, &(*pTaskInfo)->tableqinfoList); + if (NULL == (*pTaskInfo)->pRoot) { code = (*pTaskInfo)->code; goto _complete; } - if ((*pTaskInfo)->pRoot == NULL) { - code = TSDB_CODE_QRY_OUT_OF_MEMORY; - goto _complete; - } - return code; _complete: diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index 064a36df1c..4e4aaba7f4 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -559,8 +559,8 @@ uint64_t calcGroupId(char* pData, int32_t len) { } int32_t* setupColumnOffset(const SSDataBlock* pBlock, int32_t rowCapacity) { - size_t numOfCols = pBlock->info.numOfCols; - int32_t* offset = taosMemoryCalloc(pBlock->info.numOfCols, sizeof(int32_t)); + size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); + int32_t* offset = taosMemoryCalloc(numOfCols, sizeof(int32_t)); offset[0] = sizeof(int32_t) + sizeof(uint64_t); // the number of rows in current page, ref to SSDataBlock paged serialization format diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 80dd01cf4c..07212bc018 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -203,7 +203,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca pCost->skipBlocks += 1; // clear all data in pBlock that are set when handing the previous block - for (int32_t i = 0; i < pBlockInfo->numOfCols; ++i) { + for (int32_t i = 0; i < taosArrayGetSize(pBlock->pDataBlock); ++i) { SColumnInfoData* pcol = taosArrayGet(pBlock->pDataBlock, i); pcol->pData = NULL; } @@ -217,7 +217,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca tsdbRetrieveDataBlockStatisInfo(pTableScanInfo->dataReader, &pColAgg, &allColumnsHaveAgg); if (allColumnsHaveAgg == true) { - int32_t numOfCols = pBlock->info.numOfCols; + int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); // todo create this buffer during creating operator if (pBlock->pBlockAgg == NULL) { @@ -316,7 +316,7 @@ void addTagPseudoColumnData(SReadHandle* pHandle, SExprInfo* pPseudoExpr, int32_ SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, dstSlotId); - colInfoDataEnsureCapacity(pColInfoData, 0, pBlock->info.rows); + colInfoDataEnsureCapacity(pColInfoData, pBlock->info.rows); colInfoDataCleanup(pColInfoData, pBlock->info.rows); int32_t functionId = pExpr->pExpr->_function.functionId; @@ -355,10 +355,8 @@ void setTbNameColData(void* pMeta, const SSDataBlock* pBlock, SColumnInfoData* p struct SScalarFuncExecFuncs fpSet = {0}; fmGetScalarFuncExecFuncs(functionId, &fpSet); - SColumnInfoData infoData = {0}; - infoData.info.type = TSDB_DATA_TYPE_BIGINT; - infoData.info.bytes = sizeof(uint64_t); - colInfoDataEnsureCapacity(&infoData, 0, 1); + SColumnInfoData infoData = createColumnInfoData(TSDB_DATA_TYPE_BIGINT, sizeof(uint64_t), 1); + colInfoDataEnsureCapacity(&infoData, 1); colDataAppendInt64(&infoData, 0, (int64_t*)&pBlock->info.uid); SScalarParam srcParam = {.numOfRows = pBlock->info.rows, .param = pMeta, .columnData = &infoData}; @@ -786,7 +784,7 @@ static bool prepareDataScan(SStreamBlockScanInfo* pInfo) { } static void copyOneRow(SSDataBlock* dest, SSDataBlock* source, int32_t sourceRowId) { - for (int32_t j = 0; j < source->info.numOfCols; j++) { + for (int32_t j = 0; j < taosArrayGetSize(source->pDataBlock); j++) { SColumnInfoData* pDestCol = (SColumnInfoData*)taosArrayGet(dest->pDataBlock, j); SColumnInfoData* pSourceCol = (SColumnInfoData*)taosArrayGet(source->pDataBlock, j); if (colDataIsNull_s(pSourceCol, sourceRowId)) { @@ -858,7 +856,6 @@ static void setUpdateData(SStreamBlockScanInfo* pInfo, SSDataBlock* pBlock, SSDa SColumnInfoData* pCol = (SColumnInfoData*)taosArrayGet(pUpdateBlock->pDataBlock, pInfo->primaryTsIndex); ASSERT(pCol->info.type == TSDB_DATA_TYPE_TIMESTAMP); blockDataEnsureCapacity(pUpdateBlock, size); - ASSERT(pBlock->info.numOfCols == pUpdateBlock->info.numOfCols); int32_t rowId = *(int32_t*)taosArrayGet(pInfo->tsArray, pInfo->tsArrayIndex); pInfo->groupId = getGroupId(pInfo->pSnapshotReadOp, pBlock, rowId); @@ -969,13 +966,13 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { blockDataCleanup(pInfo->pRes); while (tqNextDataBlock(pInfo->streamBlockReader)) { - SArray* pCols = NULL; - uint64_t groupId = 0; - uint64_t uid = 0; - int32_t numOfRows = 0; - int16_t outputCol = 0; + SSDataBlock block = {0}; + uint64_t groupId = 0; + uint64_t uid = 0; + int32_t numOfRows = 0; - int32_t code = tqRetrieveDataBlock(&pCols, pInfo->streamBlockReader, &groupId, &uid, &numOfRows, &outputCol); + // todo refactor + int32_t code = tqRetrieveDataBlock(&block, pInfo->streamBlockReader, &groupId, &uid, &numOfRows); if (code != TSDB_CODE_SUCCESS || numOfRows == 0) { pTaskInfo->code = code; @@ -986,6 +983,7 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { pInfo->pRes->info.rows = numOfRows; pInfo->pRes->info.uid = uid; pInfo->pRes->info.type = STREAM_NORMAL; + pInfo->pRes->info.capacity = numOfRows; // for generating rollup SMA result, each time is an independent time serie. // TODO temporarily used, when the statement of "partition by tbname" is ready, remove this @@ -1000,6 +998,7 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { pInfo->pRes->info.groupId = *groupIdPre; } + // todo extract method for (int32_t i = 0; i < taosArrayGetSize(pInfo->pColMatchInfo); ++i) { SColMatchInfo* pColMatchInfo = taosArrayGet(pInfo->pColMatchInfo, i); if (!pColMatchInfo->output) { @@ -1007,8 +1006,8 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { } bool colExists = false; - for (int32_t j = 0; j < taosArrayGetSize(pCols); ++j) { - SColumnInfoData* pResCol = taosArrayGet(pCols, j); + for (int32_t j = 0; j < blockDataGetNumOfCols(&block); ++j) { + SColumnInfoData* pResCol = bdGetColumnInfoData(&block, j); if (pResCol->info.colId == pColMatchInfo->colId) { taosArraySet(pInfo->pRes->pDataBlock, pColMatchInfo->targetSlotId, pResCol); colExists = true; @@ -1019,11 +1018,13 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { // the required column does not exists in submit block, let's set it to be all null value if (!colExists) { SColumnInfoData* pDst = taosArrayGet(pInfo->pRes->pDataBlock, pColMatchInfo->targetSlotId); - colInfoDataEnsureCapacity(pDst, 0, pBlockInfo->rows); colDataAppendNNULL(pDst, 0, pBlockInfo->rows); } } + // TODO refactor @liao + taosArrayDestroy(block.pDataBlock); + if (pInfo->pRes->pDataBlock == NULL) { // TODO add log pOperator->status = OP_EXEC_DONE; @@ -1063,12 +1064,11 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { } return (pBlockInfo->rows == 0) ? NULL : pInfo->pRes; + } else if (pInfo->blockType == STREAM_DATA_TYPE_FROM_SNAPSHOT) { SSDataBlock* pResult = doTableScan(pInfo->pSnapshotReadOp); - if (pResult) { - return pResult->info.rows > 0 ? pResult : NULL; - } - return NULL; + return pResult && pResult->info.rows > 0 ? pResult : NULL; + } else { ASSERT(0); return NULL; @@ -1163,7 +1163,6 @@ SOperatorInfo* createStreamScanOperatorInfo(void* pDataReader, SReadHandle* pHan pInfo->pRes = createResDataBlock(pDescNode); pInfo->pUpdateRes = createResDataBlock(pDescNode); pInfo->pCondition = pScanPhyNode->node.pConditions; - pInfo->pDataReader = pDataReader; pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE; pInfo->sessionSup = (SessionWindowSupporter){.pStreamAggSup = NULL, .gap = -1}; pInfo->groupId = 0; @@ -1173,7 +1172,7 @@ SOperatorInfo* createStreamScanOperatorInfo(void* pDataReader, SReadHandle* pHan pOperator->blocking = false; pOperator->status = OP_NOT_OPENED; pOperator->info = pInfo; - pOperator->exprSupp.numOfExprs = pInfo->pRes->info.numOfCols; + pOperator->exprSupp.numOfExprs = taosArrayGetSize(pInfo->pRes->pDataBlock); pOperator->pTaskInfo = pTaskInfo; pOperator->fpSet = @@ -1301,7 +1300,7 @@ static SSDataBlock* doFilterResult(SSysTableScanInfo* pInfo) { SColumnInfoData* pSrc = taosArrayGet(pInfo->pRes->pDataBlock, i); if (keep) { - colDataAssign(pDest, pSrc, pInfo->pRes->info.rows); + colDataAssign(pDest, pSrc, pInfo->pRes->info.rows, &px->info); numOfRow = pInfo->pRes->info.rows; } else if (NULL != rowRes) { numOfRow = 0; @@ -1331,8 +1330,6 @@ static SSDataBlock* doFilterResult(SSysTableScanInfo* pInfo) { } static SSDataBlock* buildSysTableMetaBlock() { - SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); - size_t size = 0; const SSysTableMeta* pMeta = NULL; getInfosDbMeta(&pMeta, &size); @@ -1345,19 +1342,13 @@ static SSDataBlock* buildSysTableMetaBlock() { } } - pBlock->pDataBlock = taosArrayInit(pBlock->info.numOfCols, sizeof(SColumnInfoData)); - + SSDataBlock* pBlock = createDataBlock(); for (int32_t i = 0; i < pMeta[index].colNum; ++i) { - SColumnInfoData colInfoData = {0}; - colInfoData.info.colId = i + 1; - colInfoData.info.type = pMeta[index].schema[i].type; - colInfoData.info.bytes = pMeta[index].schema[i].bytes; - taosArrayPush(pBlock->pDataBlock, &colInfoData); + SColumnInfoData colInfoData = + createColumnInfoData(pMeta[index].schema[i].type, pMeta[index].schema[i].bytes, i + 1); + blockDataAppendColInfo(pBlock, &colInfoData); } - pBlock->info.numOfCols = pMeta[index].colNum; - pBlock->info.hasVarCol = true; - return pBlock; } @@ -1388,7 +1379,6 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator) { } blockDataCleanup(pInfo->pRes); - int32_t numOfRows = 0; const char* db = NULL; @@ -1584,7 +1574,7 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator) { if (pRsp->numOfRows == 0 || pRsp->completed) { pOperator->status = OP_EXEC_DONE; - qDebug("%s load meta data from mnode completed, rowsOfSource:%d, totalRows:%" PRIu64 " ", GET_TASKID(pTaskInfo), + qDebug("%s load meta data from mnode completed, rowsOfSource:%d, totalRows:%" PRIu64, GET_TASKID(pTaskInfo), pRsp->numOfRows, pInfo->loadInfo.totalRows); if (pRsp->numOfRows == 0) { @@ -1711,7 +1701,7 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScan pOperator->blocking = false; pOperator->status = OP_NOT_OPENED; pOperator->info = pInfo; - pOperator->exprSupp.numOfExprs = pResBlock->info.numOfCols; + pOperator->exprSupp.numOfExprs = taosArrayGetSize(pResBlock->pDataBlock); pOperator->pTaskInfo = pTaskInfo; pOperator->fpSet = @@ -2049,6 +2039,7 @@ int32_t createMultipleDataReaders(SQueryTableDataCond* pQueryCond, SReadHandle* return TSDB_CODE_SUCCESS; } +// todo refactor static int32_t loadDataBlockFromOneTable(SOperatorInfo* pOperator, STableMergeScanInfo* pTableScanInfo, int32_t readerIdx, SSDataBlock* pBlock, uint32_t* status) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; @@ -2079,7 +2070,7 @@ static int32_t loadDataBlockFromOneTable(SOperatorInfo* pOperator, STableMergeSc pCost->skipBlocks += 1; // clear all data in pBlock that are set when handing the previous block - for (int32_t i = 0; i < pBlockInfo->numOfCols; ++i) { + for (int32_t i = 0; i < taosArrayGetSize(pBlock->pDataBlock); ++i) { SColumnInfoData* pcol = taosArrayGet(pBlock->pDataBlock, i); pcol->pData = NULL; } @@ -2094,7 +2085,7 @@ static int32_t loadDataBlockFromOneTable(SOperatorInfo* pOperator, STableMergeSc tsdbRetrieveDataBlockStatisInfo(reader, &pColAgg, &allColumnsHaveAgg); if (allColumnsHaveAgg == true) { - int32_t numOfCols = pBlock->info.numOfCols; + int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); // todo create this buffer during creating operator if (pBlock->pBlockAgg == NULL) { @@ -2189,7 +2180,13 @@ static SSDataBlock* getTableDataBlock(void* param) { continue; } - tsdbRetrieveDataBlockInfo(reader, &pBlock->info); + blockDataCleanup(pBlock); + SDataBlockInfo binfo = pBlock->info; + tsdbRetrieveDataBlockInfo(reader, &binfo); + + binfo.capacity = binfo.rows; + blockDataEnsureCapacity(pBlock, binfo.capacity); + pBlock->info = binfo; uint32_t status = 0; int32_t code = loadDataBlockFromOneTable(pOperator, pTableScanInfo, readerIdx, pBlock, &status); diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c index 4b5ad7b123..09cc4e47be 100644 --- a/source/libs/executor/src/sortoperator.c +++ b/source/libs/executor/src/sortoperator.c @@ -80,7 +80,7 @@ _error: } void appendOneRowToDataBlock(SSDataBlock* pBlock, STupleHandle* pTupleHandle) { - for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { + for (int32_t i = 0; i < taosArrayGetSize(pBlock->pDataBlock); ++i) { SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i); bool isNull = tsortIsNullVal(pTupleHandle, i); if (isNull) { @@ -120,6 +120,9 @@ SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, i } if (p->info.rows > 0) { + blockDataEnsureCapacity(pDataBlock, capacity); + + // todo extract function to handle this int32_t numOfCols = taosArrayGetSize(pColMatchInfo); for (int32_t i = 0; i < numOfCols; ++i) { SColMatchInfo* pmInfo = taosArrayGet(pColMatchInfo, i); @@ -127,11 +130,10 @@ SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, i SColumnInfoData* pSrc = taosArrayGet(p->pDataBlock, pmInfo->srcSlotId); SColumnInfoData* pDst = taosArrayGet(pDataBlock->pDataBlock, pmInfo->targetSlotId); - colDataAssign(pDst, pSrc, p->info.rows); + colDataAssign(pDst, pSrc, p->info.rows, &pDataBlock->info); } pDataBlock->info.rows = p->info.rows; - pDataBlock->info.capacity = p->info.rows; } blockDataDestroy(p); @@ -257,6 +259,7 @@ typedef struct SGroupSortOperatorInfo { SSDataBlock* getGroupSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, int32_t capacity, SArray* pColMatchInfo, SGroupSortOperatorInfo* pInfo) { blockDataCleanup(pDataBlock); + blockDataEnsureCapacity(pDataBlock, capacity); SSDataBlock* p = tsortGetSortedDataBlock(pHandle); if (p == NULL) { @@ -285,7 +288,7 @@ SSDataBlock* getGroupSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlo SColumnInfoData* pSrc = taosArrayGet(p->pDataBlock, pmInfo->srcSlotId); SColumnInfoData* pDst = taosArrayGet(pDataBlock->pDataBlock, pmInfo->targetSlotId); - colDataAssign(pDst, pSrc, p->info.rows); + colDataAssign(pDst, pSrc, p->info.rows, &pDataBlock->info); } pDataBlock->info.rows = p->info.rows; @@ -387,6 +390,10 @@ SSDataBlock* doGroupSort(SOperatorInfo* pOperator) { pInfo->hasGroupId = true; pInfo->prefetchedSortInput = pOperator->pDownstream[0]->fpSet.getNextFn(pOperator->pDownstream[0]); + if (pInfo->prefetchedSortInput == NULL) { + doSetOperatorCompleted(pOperator); + return NULL; + } pInfo->currGroupId = pInfo->prefetchedSortInput->info.groupId; pInfo->childOpStatus = CHILD_OP_NEW_GROUP; beginSortGroup(pOperator); @@ -460,7 +467,7 @@ SOperatorInfo* createGroupSortOperatorInfo(SOperatorInfo* downstream, SGroupSort pInfo->pColMatchInfo = pColMatchColInfo; pOperator->name = "GroupSortOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT; - pOperator->blocking = true; + pOperator->blocking = false; pOperator->status = OP_NOT_OPENED; pOperator->info = pInfo; pOperator->exprSupp.pExprInfo = pExprInfo; @@ -564,7 +571,8 @@ SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pData } } - if (p->info.rows > 0) { + if (p->info.rows > 0) {// todo extract method + blockDataEnsureCapacity(pDataBlock, p->info.rows); int32_t numOfCols = taosArrayGetSize(pColMatchInfo); for (int32_t i = 0; i < numOfCols; ++i) { SColMatchInfo* pmInfo = taosArrayGet(pColMatchInfo, i); @@ -572,11 +580,10 @@ SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pData SColumnInfoData* pSrc = taosArrayGet(p->pDataBlock, pmInfo->srcSlotId); SColumnInfoData* pDst = taosArrayGet(pDataBlock->pDataBlock, pmInfo->targetSlotId); - colDataAssign(pDst, pSrc, p->info.rows); + colDataAssign(pDst, pSrc, p->info.rows, &pDataBlock->info); } pDataBlock->info.rows = p->info.rows; - pDataBlock->info.capacity = p->info.rows; pDataBlock->info.groupId = pInfo->groupId; } diff --git a/source/libs/executor/src/tfill.c b/source/libs/executor/src/tfill.c index 110c2d6a8f..1d82d6a644 100644 --- a/source/libs/executor/src/tfill.c +++ b/source/libs/executor/src/tfill.c @@ -50,7 +50,7 @@ static void setTagsValue(SFillInfo* pFillInfo, void** data, int32_t genRows) { static void setNullRow(SSDataBlock* pBlock, int32_t numOfCol, int32_t rowIndex) { // the first are always the timestamp column, so start from the second column. - for (int32_t i = 1; i < pBlock->info.numOfCols; ++i) { + for (int32_t i = 1; i < taosArrayGetSize(pBlock->pDataBlock); ++i) { SColumnInfoData* p = taosArrayGet(pBlock->pDataBlock, i); colDataAppendNULL(p, rowIndex); } diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 6813c63542..1a56eacda8 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -1814,7 +1814,7 @@ static int32_t initPrevRowsKeeper(STimeSliceOperatorInfo* pInfo, SSDataBlock* pB return TSDB_CODE_OUT_OF_MEMORY; } - int32_t numOfCols = pBlock->info.numOfCols; + int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i); @@ -2231,9 +2231,6 @@ static void clearStreamIntervalOperator(SStreamFinalIntervalOperatorInfo* pInfo) } static void clearUpdateDataBlock(SSDataBlock* pBlock) { - if (pBlock->info.rows <= 0) { - return; - } blockDataCleanup(pBlock); } @@ -2243,12 +2240,14 @@ void copyUpdateDataBlock(SSDataBlock* pDest, SSDataBlock* pSource, int32_t tsCol clearUpdateDataBlock(pDest); SColumnInfoData* pDestCol = taosArrayGet(pDest->pDataBlock, 0); SColumnInfoData* pSourceCol = taosArrayGet(pSource->pDataBlock, tsColIndex); + // copy timestamp column - colDataAssign(pDestCol, pSourceCol, pSource->info.rows); - for (int32_t i = 1; i < pDest->info.numOfCols; i++) { + colDataAssign(pDestCol, pSourceCol, pSource->info.rows, &pDest->info); + for (int32_t i = 1; i < taosArrayGetSize(pDest->pDataBlock); i++) { SColumnInfoData* pCol = taosArrayGet(pDest->pDataBlock, i); colDataAppendNNULL(pCol, 0, pSource->info.rows); } + pDest->info.rows = pSource->info.rows; pDest->info.groupId = pSource->info.groupId; pDest->info.type = pSource->info.type; @@ -2918,7 +2917,7 @@ void doBuildDeleteDataBlock(SHashObj* pStDeleted, SSDataBlock* pBlock, void** It while (((*Ite) = taosHashIterate(pStDeleted, *Ite)) != NULL) { SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, 0); colDataAppend(pColInfoData, pBlock->info.rows, *Ite, false); - for (int32_t i = 1; i < pBlock->info.numOfCols; i++) { + for (int32_t i = 1; i < taosArrayGetSize(pBlock->pDataBlock); i++) { pColInfoData = taosArrayGet(pBlock->pDataBlock, i); colDataAppendNULL(pColInfoData, pBlock->info.rows); } @@ -4118,6 +4117,8 @@ SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SExprI goto _error; } + miaInfo->groupIntervalHash = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), true, HASH_NO_LOCK); + SIntervalAggOperatorInfo* iaInfo = &miaInfo->intervalAggOperatorInfo; iaInfo->win = pTaskInfo->window; diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c index 1dd72fd83f..50474ff62e 100644 --- a/source/libs/executor/src/tsort.c +++ b/source/libs/executor/src/tsort.c @@ -147,7 +147,7 @@ static int32_t doAddNewExternalMemSource(SDiskbasedBuf *pBuf, SArray* pAllSource int32_t rowSize = blockDataGetSerialRowSize(pSource->src.pBlock); // The value of numOfRows must be greater than 0, which is guaranteed by the previous memory allocation - int32_t numOfRows = (getBufPageSize(pBuf) - blockDataGetSerialMetaSize(pBlock->info.numOfCols))/rowSize; + int32_t numOfRows = (getBufPageSize(pBuf) - blockDataGetSerialMetaSize(taosArrayGetSize(pBlock->pDataBlock)))/rowSize; ASSERT(numOfRows > 0); return blockDataEnsureCapacity(pSource->src.pBlock, numOfRows); } @@ -178,7 +178,7 @@ static int32_t doAddToBuf(SSDataBlock* pDataBlock, SSortHandle* pHandle) { return terrno; } - int32_t size = blockDataGetSize(p) + sizeof(int32_t) + p->info.numOfCols * sizeof(int32_t); + int32_t size = blockDataGetSize(p) + sizeof(int32_t) + taosArrayGetSize(p->pDataBlock) * sizeof(int32_t); assert(size <= getBufPageSize(pHandle->pBuf)); blockDataToBuf(pPage, p); @@ -252,7 +252,7 @@ static int32_t sortComparInit(SMsortComparParam* cmpParam, SArray* pSources, int } static void appendOneRowToDataBlock(SSDataBlock *pBlock, const SSDataBlock* pSource, int32_t* rowIndex) { - for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { + for (int32_t i = 0; i < taosArrayGetSize(pBlock->pDataBlock); ++i) { SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i); SColumnInfoData* pSrcColInfo = taosArrayGet(pSource->pDataBlock, i); @@ -500,7 +500,7 @@ static int32_t doInternalMergeSort(SSortHandle* pHandle) { return terrno; } - int32_t size = blockDataGetSize(pDataBlock) + sizeof(int32_t) + pDataBlock->info.numOfCols * sizeof(int32_t); + int32_t size = blockDataGetSize(pDataBlock) + sizeof(int32_t) + taosArrayGetSize(pDataBlock->pDataBlock) * sizeof(int32_t); assert(size <= getBufPageSize(pHandle->pBuf)); blockDataToBuf(pPage, pDataBlock); diff --git a/source/libs/executor/test/executorTests.cpp b/source/libs/executor/test/executorTests.cpp index dba2fb897a..3c12889aca 100644 --- a/source/libs/executor/test/executorTests.cpp +++ b/source/libs/executor/test/executorTests.cpp @@ -62,18 +62,11 @@ SSDataBlock* getDummyBlock(SOperatorInfo* pOperator) { } if (pInfo->pBlock == NULL) { - pInfo->pBlock = static_cast(taosMemoryCalloc(1, sizeof(SSDataBlock))); + pInfo->pBlock = createDataBlock(); - pInfo->pBlock->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData)); - - SColumnInfoData colInfo = {0}; - colInfo.info.type = TSDB_DATA_TYPE_INT; - colInfo.info.bytes = sizeof(int32_t); - colInfo.info.colId = 1; - colInfo.pData = static_cast(taosMemoryCalloc(pInfo->numOfRowsPerPage, sizeof(int32_t))); - colInfo.nullbitmap = static_cast(taosMemoryCalloc(1, (pInfo->numOfRowsPerPage + 7) / 8)); - - taosArrayPush(pInfo->pBlock->pDataBlock, &colInfo); + SColumnInfoData colInfo = createColumnInfoData(TSDB_DATA_TYPE_INT, sizeof(int32_t), 1); + blockDataAppendColInfo(pInfo->pBlock, &colInfo); + blockDataEnsureCapacity(pInfo->pBlock, pInfo->numOfRowsPerPage); // SColumnInfoData colInfo1 = {0}; // colInfo1.info.type = TSDB_DATA_TYPE_BINARY; @@ -115,7 +108,6 @@ SSDataBlock* getDummyBlock(SOperatorInfo* pOperator) { } pBlock->info.rows = pInfo->numOfRowsPerPage; - pBlock->info.numOfCols = 1; pInfo->current += 1; return pBlock; @@ -128,28 +120,15 @@ SSDataBlock* get2ColsDummyBlock(SOperatorInfo* pOperator) { } if (pInfo->pBlock == NULL) { - pInfo->pBlock = static_cast(taosMemoryCalloc(1, sizeof(SSDataBlock))); + pInfo->pBlock = createDataBlock(); - pInfo->pBlock->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData)); + SColumnInfoData colInfo = createColumnInfoData(TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), 1); + blockDataAppendColInfo(pInfo->pBlock, &colInfo); - SColumnInfoData colInfo = {0}; - colInfo.info.type = TSDB_DATA_TYPE_TIMESTAMP; - colInfo.info.bytes = sizeof(int64_t); - colInfo.info.colId = 1; - colInfo.pData = static_cast(taosMemoryCalloc(pInfo->numOfRowsPerPage, sizeof(int64_t))); - // colInfo.nullbitmap = static_cast(taosMemoryCalloc(1, (pInfo->numOfRowsPerPage + 7) / 8)); + SColumnInfoData colInfo1 = createColumnInfoData(TSDB_DATA_TYPE_INT, 4, 2); + blockDataAppendColInfo(pInfo->pBlock, &colInfo1); - taosArrayPush(pInfo->pBlock->pDataBlock, &colInfo); - - SColumnInfoData colInfo1 = {0}; - colInfo1.info.type = TSDB_DATA_TYPE_INT; - colInfo1.info.bytes = 4; - colInfo1.info.colId = 2; - - colInfo1.pData = static_cast(taosMemoryCalloc(pInfo->numOfRowsPerPage, sizeof(int32_t))); - colInfo1.nullbitmap = static_cast(taosMemoryCalloc(1, (pInfo->numOfRowsPerPage + 7) / 8)); - - taosArrayPush(pInfo->pBlock->pDataBlock, &colInfo1); + blockDataEnsureCapacity(pInfo->pBlock, pInfo->numOfRowsPerPage); } else { blockDataCleanup(pInfo->pBlock); } @@ -185,7 +164,6 @@ SSDataBlock* get2ColsDummyBlock(SOperatorInfo* pOperator) { } pBlock->info.rows = pInfo->numOfRowsPerPage; - pBlock->info.numOfCols = 1; pInfo->current += 1; diff --git a/source/libs/executor/test/sortTests.cpp b/source/libs/executor/test/sortTests.cpp index 66ed078bbe..70def78263 100644 --- a/source/libs/executor/test/sortTests.cpp +++ b/source/libs/executor/test/sortTests.cpp @@ -62,25 +62,21 @@ SSDataBlock* getSingleColDummyBlock(void* param) { return NULL; } - SSDataBlock* pBlock = static_cast(taosMemoryCalloc(1, sizeof(SSDataBlock))); - pBlock->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData)); + SSDataBlock* pBlock = createDataBlock(); SColumnInfoData colInfo = {0}; colInfo.info.type = pInfo->type; if (pInfo->type == TSDB_DATA_TYPE_NCHAR){ colInfo.info.bytes = TSDB_NCHAR_SIZE * VARCOUNT + VARSTR_HEADER_SIZE; - colInfo.varmeta.offset = static_cast(taosMemoryCalloc(pInfo->pageRows, sizeof(int32_t))); } else if(pInfo->type == TSDB_DATA_TYPE_BINARY) { colInfo.info.bytes = VARCOUNT + VARSTR_HEADER_SIZE; - colInfo.varmeta.offset = static_cast(taosMemoryCalloc(pInfo->pageRows, sizeof(int32_t))); } else{ colInfo.info.bytes = tDataTypes[pInfo->type].bytes; - colInfo.pData = static_cast(taosMemoryCalloc(pInfo->pageRows, colInfo.info.bytes)); - colInfo.nullbitmap = static_cast(taosMemoryCalloc(1, (pInfo->pageRows + 7) / 8)); } colInfo.info.colId = 1; - taosArrayPush(pBlock->pDataBlock, &colInfo); + blockDataAppendColInfo(pBlock, &colInfo); + blockDataEnsureCapacity(pBlock, pInfo->pageRows); for (int32_t i = 0; i < pInfo->pageRows; ++i) { SColumnInfoData* pColInfo = static_cast(TARRAY_GET_ELEM(pBlock->pDataBlock, 0)); @@ -128,7 +124,6 @@ SSDataBlock* getSingleColDummyBlock(void* param) { } pBlock->info.rows = pInfo->pageRows; - pBlock->info.numOfCols = 1; return pBlock; } @@ -354,15 +349,10 @@ TEST(testCase, ordered_merge_sort_Test) { SArray* orderInfo = taosArrayInit(1, sizeof(SBlockOrderInfo)); taosArrayPush(orderInfo, &oi); - SSDataBlock* pBlock = static_cast(taosMemoryCalloc(1, sizeof(SSDataBlock))); - pBlock->pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData)); - pBlock->info.numOfCols = 1; - for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { - SColumnInfoData colInfo = {0}; - colInfo.info.type = TSDB_DATA_TYPE_INT; - colInfo.info.bytes = sizeof(int32_t); - colInfo.info.colId = 1; - taosArrayPush(pBlock->pDataBlock, &colInfo); + SSDataBlock* pBlock = createDataBlock(); + for (int32_t i = 0; i < 1; ++i) { + SColumnInfoData colInfo = createColumnInfoData(TSDB_DATA_TYPE_INT, sizeof(int32_t), 1); + blockDataAppendColInfo(pBlock, &colInfo); } SSortHandle* phandle = tsortCreateSortHandle(orderInfo, SORT_MULTISOURCE_MERGE, 1024, 5, pBlock,"test_abc"); diff --git a/source/libs/function/inc/taggfunction.h b/source/libs/function/inc/taggfunction.h index c3d61d426d..669c2635b5 100644 --- a/source/libs/function/inc/taggfunction.h +++ b/source/libs/function/inc/taggfunction.h @@ -28,19 +28,6 @@ extern "C" { #include "function.h" #include "tudf.h" - -extern SAggFunctionInfo aggFunc[35]; - -#define FUNCSTATE_SO 0x0u -#define FUNCSTATE_MO 0x1u // dynamic number of output, not multinumber of output e.g., TOP/BOTTOM -#define FUNCSTATE_STREAM 0x2u // function avail for stream -#define FUNCSTATE_STABLE 0x4u // function avail for super table -#define FUNCSTATE_NEED_TS 0x8u // timestamp is required during query processing -#define FUNCSTATE_SELECTIVITY 0x10u // selectivity functions, can exists along with tag columns - -#define BASIC_FUNC_SO FUNCSTATE_SO | FUNCSTATE_STREAM | FUNCSTATE_STABLE -#define BASIC_FUNC_MO FUNCSTATE_MO | FUNCSTATE_STREAM | FUNCSTATE_STABLE - #define AVG_FUNCTION_INTER_BUFFER_SIZE 50 #define DATA_SET_FLAG ',' // to denote the output area has data, not null value diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index acf2ffe3f8..6516a10795 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -79,7 +79,8 @@ static int32_t translateIn2NumOutDou(SFunctionNode* pFunc, char* pErrBuf, int32_ uint8_t para1Type = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type; uint8_t para2Type = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 1))->resType.type; - if (!IS_NUMERIC_TYPE(para1Type) || !IS_NUMERIC_TYPE(para2Type)) { + if ((!IS_NUMERIC_TYPE(para1Type) && !IS_NULL_TYPE(para1Type)) || + (!IS_NUMERIC_TYPE(para2Type) && !IS_NULL_TYPE(para2Type))) { return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); } @@ -109,13 +110,13 @@ static int32_t translateLogarithm(SFunctionNode* pFunc, char* pErrBuf, int32_t l } uint8_t para1Type = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type; - if (!IS_NUMERIC_TYPE(para1Type)) { + if (!IS_NUMERIC_TYPE(para1Type) && !IS_NULL_TYPE(para1Type)) { return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); } if (2 == numOfParams) { uint8_t para2Type = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 1))->resType.type; - if (!IS_NUMERIC_TYPE(para2Type)) { + if (!IS_NUMERIC_TYPE(para2Type) && !IS_NULL_TYPE(para2Type)) { return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); } } @@ -1267,6 +1268,29 @@ static bool validateMinuteRange(int8_t hour, int8_t minute, char sign) { return false; } +static bool validateTimestampDigits(const SValueNode* pVal) { + if (!IS_INTEGER_TYPE(pVal->node.resType.type)) { + return false; + } + + int64_t tsVal = pVal->datum.i; + char fraction[20] = {0}; + NUM_TO_STRING(pVal->node.resType.type, &tsVal, sizeof(fraction), fraction); + int32_t tsDigits = (int32_t)strlen(fraction); + + if (tsDigits > TSDB_TIME_PRECISION_SEC_DIGITS) { + if (tsDigits == TSDB_TIME_PRECISION_MILLI_DIGITS || + tsDigits == TSDB_TIME_PRECISION_MICRO_DIGITS || + tsDigits == TSDB_TIME_PRECISION_NANO_DIGITS) { + return true; + } else { + return false; + } + } + + return true; +} + static bool validateTimezoneFormat(const SValueNode* pVal) { if (TSDB_DATA_TYPE_BINARY != pVal->node.resType.type) { return false; @@ -1379,6 +1403,15 @@ static int32_t translateToIso8601(SFunctionNode* pFunc, char* pErrBuf, int32_t l return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); } + if (QUERY_NODE_VALUE == nodeType(nodesListGetNode(pFunc->pParameterList, 0))) { + SValueNode* pValue = (SValueNode*)nodesListGetNode(pFunc->pParameterList, 0); + + if (!validateTimestampDigits(pValue)) { + pFunc->node.resType = (SDataType){.bytes = 0, .type = TSDB_DATA_TYPE_BINARY}; + return TSDB_CODE_SUCCESS; + } + } + // param1 if (numOfParams == 2) { SValueNode* pValue = (SValueNode*)nodesListGetNode(pFunc->pParameterList, 1); diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index d7ff03ec59..4ec6432fc8 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -561,7 +561,7 @@ int32_t sumFunction(SqlFunctionCtx* pCtx) { // check for overflow if (IS_FLOAT_TYPE(type) && (isinf(pSumRes->dsum) || isnan(pSumRes->dsum))) { - GET_RES_INFO(pCtx)->isNullRes = 1; + numOfElem = 0; } _sum_over: @@ -927,7 +927,7 @@ int32_t avgFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { // check for overflow if (isinf(pAvgRes->result) || isnan(pAvgRes->result)) { - GET_RES_INFO(pCtx)->isNullRes = 1; + GET_RES_INFO(pCtx)->numOfRes = 0; } return functionFinalize(pCtx, pBlock); @@ -1449,11 +1449,13 @@ void setNullSelectivityValue(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t void setSelectivityValue(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, const STuplePos* pTuplePos, int32_t rowIndex) { int32_t pageId = pTuplePos->pageId; int32_t offset = pTuplePos->offset; + if (pTuplePos->pageId != -1) { + int32_t numOfCols = taosArrayGetSize(pCtx->pSrcBlock->pDataBlock); SFilePage* pPage = getBufPage(pCtx->pBuf, pageId); bool* nullList = (bool*)((char*)pPage + offset); - char* pStart = (char*)(nullList + pCtx->pSrcBlock->info.numOfCols * sizeof(bool)); + char* pStart = (char*)(nullList + numOfCols * sizeof(bool)); // todo set the offset value to optimize the performance. for (int32_t j = 0; j < pCtx->subsidiaries.num; ++j) { @@ -1774,6 +1776,11 @@ int32_t stddevFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { pStddevRes->result = sqrt(fabs(pStddevRes->quadraticDSum / ((double)pStddevRes->count) - avg * avg)); } + // check for overflow + if (isinf(pStddevRes->result) || isnan(pStddevRes->result)) { + GET_RES_INFO(pCtx)->numOfRes = 0; + } + return functionFinalize(pCtx, pBlock); } @@ -3038,7 +3045,7 @@ void doAddIntoResult(SqlFunctionCtx* pCtx, void* pData, int32_t rowIndex, SSData void saveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STuplePos* pPos) { SFilePage* pPage = NULL; - int32_t completeRowSize = pSrcBlock->info.rowSize + pSrcBlock->info.numOfCols * sizeof(bool); + int32_t completeRowSize = pSrcBlock->info.rowSize + (int32_t) taosArrayGetSize(pSrcBlock->pDataBlock) * sizeof(bool); if (pCtx->curBufPage == -1) { pPage = getNewBufPage(pCtx->pBuf, 0, &pCtx->curBufPage); @@ -3056,8 +3063,8 @@ void saveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pS // keep the current row data, extract method int32_t offset = 0; bool* nullList = (bool*)((char*)pPage + pPage->num); - char* pStart = (char*)(nullList + sizeof(bool) * pSrcBlock->info.numOfCols); - for (int32_t i = 0; i < pSrcBlock->info.numOfCols; ++i) { + char* pStart = (char*)(nullList + sizeof(bool) * (int32_t) taosArrayGetSize(pSrcBlock->pDataBlock)); + for (int32_t i = 0; i < (int32_t) taosArrayGetSize(pSrcBlock->pDataBlock); ++i) { SColumnInfoData* pCol = taosArrayGet(pSrcBlock->pDataBlock, i); bool isNull = colDataIsNull_s(pCol, rowIndex); if (isNull) { @@ -3086,11 +3093,13 @@ void saveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pS void copyTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STuplePos* pPos) { SFilePage* pPage = getBufPage(pCtx->pBuf, pPos->pageId); + int32_t numOfCols = taosArrayGetSize(pSrcBlock->pDataBlock); + bool* nullList = (bool*)((char*)pPage + pPos->offset); - char* pStart = (char*)(nullList + pSrcBlock->info.numOfCols * sizeof(bool)); + char* pStart = (char*)(nullList + numOfCols * sizeof(bool)); int32_t offset = 0; - for (int32_t i = 0; i < pSrcBlock->info.numOfCols; ++i) { + for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pCol = taosArrayGet(pSrcBlock->pDataBlock, i); if ((nullList[i] = colDataIsNull_s(pCol, rowIndex)) == true) { offset += pCol->info.bytes; diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index f6370be6f4..da9474ede0 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -763,8 +763,8 @@ void freeUdfInterBuf(SUdfInterBuf *buf) { int32_t convertDataBlockToUdfDataBlock(SSDataBlock *block, SUdfDataBlock *udfBlock) { udfBlock->numOfRows = block->info.rows; - udfBlock->numOfCols = block->info.numOfCols; - udfBlock->udfCols = taosMemoryCalloc(udfBlock->numOfCols, sizeof(SUdfColumn*)); + udfBlock->numOfCols = taosArrayGetSize(block->pDataBlock); + udfBlock->udfCols = taosMemoryCalloc(taosArrayGetSize(block->pDataBlock), sizeof(SUdfColumn*)); for (int32_t i = 0; i < udfBlock->numOfCols; ++i) { udfBlock->udfCols[i] = taosMemoryCalloc(1, sizeof(SUdfColumn)); SColumnInfoData *col= (SColumnInfoData*)taosArrayGet(block->pDataBlock, i); @@ -799,7 +799,6 @@ int32_t convertDataBlockToUdfDataBlock(SSDataBlock *block, SUdfDataBlock *udfBlo } int32_t convertUdfColumnToDataBlock(SUdfColumn *udfCol, SSDataBlock *block) { - block->info.numOfCols = 1; block->info.rows = udfCol->colData.numOfRows; block->info.hasVarCol = IS_VAR_DATA_TYPE(udfCol->colMeta.type); @@ -830,25 +829,19 @@ int32_t convertUdfColumnToDataBlock(SUdfColumn *udfCol, SSDataBlock *block) { int32_t convertScalarParamToDataBlock(SScalarParam *input, int32_t numOfCols, SSDataBlock *output) { output->info.rows = input->numOfRows; - output->info.numOfCols = numOfCols; - bool hasVarCol = false; - for (int32_t i = 0; i < numOfCols; ++i) { - if (IS_VAR_DATA_TYPE((input+i)->columnData->info.type)) { - hasVarCol = true; - break; - } - } - output->info.hasVarCol = hasVarCol; - output->pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); for (int32_t i = 0; i < numOfCols; ++i) { taosArrayPush(output->pDataBlock, (input + i)->columnData); + + if (IS_VAR_DATA_TYPE((input+i)->columnData->info.type)) { + output->info.hasVarCol = true; + } } return 0; } int32_t convertDataBlockToScalarParm(SSDataBlock *input, SScalarParam *output) { - if (input->info.numOfCols != 1) { + if (taosArrayGetSize(input->pDataBlock) != 1) { fnError("scalar function only support one column"); return -1; } @@ -1089,24 +1082,14 @@ int32_t udfAggProcess(struct SqlFunctionCtx *pCtx) { int32_t start = pInput->startRowIndex; int32_t numOfRows = pInput->numOfRows; - - SSDataBlock tempBlock = {0}; - tempBlock.info.numOfCols = numOfCols; - tempBlock.info.rows = pInput->totalRows; - tempBlock.info.uid = pInput->uid; - bool hasVarCol = false; - tempBlock.pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); - + SSDataBlock* pTempBlock = createDataBlock(); + pTempBlock->info.rows = pInput->totalRows; + pTempBlock->info.uid = pInput->uid; for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData *col = pInput->pData[i]; - if (IS_VAR_DATA_TYPE(col->info.type)) { - hasVarCol = true; - } - taosArrayPush(tempBlock.pDataBlock, col); + blockDataAppendColInfo(pTempBlock, pInput->pData[i]); } - tempBlock.info.hasVarCol = hasVarCol; - SSDataBlock *inputBlock = blockDataExtractBlock(&tempBlock, start, numOfRows); + SSDataBlock *inputBlock = blockDataExtractBlock(pTempBlock, start, numOfRows); SUdfInterBuf state = {.buf = udfRes->interResBuf, .bufLen = session->bufSize, @@ -1131,7 +1114,9 @@ int32_t udfAggProcess(struct SqlFunctionCtx *pCtx) { } blockDataDestroy(inputBlock); - taosArrayDestroy(tempBlock.pDataBlock); + + taosArrayDestroy(pTempBlock->pDataBlock); + taosMemoryFree(pTempBlock); releaseUdfFuncHandle(pCtx->udfName); freeUdfInterBuf(&newState); diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c index 7fffa84e0b..838071dbf1 100644 --- a/source/libs/function/src/udfd.c +++ b/source/libs/function/src/udfd.c @@ -613,7 +613,9 @@ int32_t udfdOpenClientRpc() { } int32_t udfdCloseClientRpc() { + fnInfo("udfd begin closing rpc"); rpcClose(global.clientRpc); + fnInfo("udfd finish closing rpc"); return 0; } @@ -937,9 +939,8 @@ int main(int argc, char *argv[]) { uv_thread_create(&mnodeConnectThread, udfdConnectMnodeThreadFunc, NULL); udfdRun(); - + removeListeningPipe(); - uv_thread_join(&mnodeConnectThread); udfdCloseClientRpc(); return 0; diff --git a/source/libs/function/test/runUdf.c b/source/libs/function/test/runUdf.c index 15109db220..7395f1e14c 100644 --- a/source/libs/function/test/runUdf.c +++ b/source/libs/function/test/runUdf.c @@ -45,27 +45,27 @@ int scalarFuncTest() { for (int k = 0; k < 1; ++k) { SSDataBlock block = {0}; SSDataBlock *pBlock = █ - pBlock->pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData)); - pBlock->info.numOfCols = 1; + for (int32_t i = 0; i < 1; ++i) { + SColumnInfoData colInfo = createColumnInfoData(TSDB_DATA_TYPE_INT, sizeof(int32_t), 1); + blockDataAppendColInfo(pBlock, &colInfo); + } + + blockDataEnsureCapacity(pBlock, 1024); pBlock->info.rows = 1024; - for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { - SColumnInfoData colInfo = {0}; - colInfo.info.type = TSDB_DATA_TYPE_INT; - colInfo.info.bytes = sizeof(int32_t); - colInfo.info.colId = 1; - colInfoDataEnsureCapacity(&colInfo, 0, pBlock->info.rows); - for (int32_t j = 0; j < pBlock->info.rows; ++j) { - colDataAppendInt32(&colInfo, j, &j); - } - taosArrayPush(pBlock->pDataBlock, &colInfo); + + SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, 0); + for (int32_t j = 0; j < pBlock->info.rows; ++j) { + colDataAppendInt32(pCol, j, &j); } SScalarParam input = {0}; input.numOfRows = pBlock->info.rows; input.columnData = taosArrayGet(pBlock->pDataBlock, 0); + SScalarParam output = {0}; doCallUdfScalarFunc(handle, &input, 1, &output); taosArrayDestroy(pBlock->pDataBlock); + SColumnInfoData *col = output.columnData; for (int32_t i = 0; i < output.numOfRows; ++i) { if (i % 100 == 0) @@ -89,21 +89,18 @@ int aggregateFuncTest() { return -1; } - SSDataBlock block = {0}; - SSDataBlock *pBlock = █ - pBlock->pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData)); - pBlock->info.numOfCols = 1; + SSDataBlock *pBlock = createDataBlock(); + for (int32_t i = 0; i < taosArrayGetSize(pBlock->pDataBlock); ++i) { + SColumnInfoData colInfo = createColumnInfoData(TSDB_DATA_TYPE_INT, sizeof(int32_t), 1); + blockDataAppendColInfo(pBlock, &colInfo); + } + + blockDataEnsureCapacity(pBlock, 1024); pBlock->info.rows = 1024; - for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { - SColumnInfoData colInfo = {0}; - colInfo.info.type = TSDB_DATA_TYPE_INT; - colInfo.info.bytes = sizeof(int32_t); - colInfo.info.colId = 1; - colInfoDataEnsureCapacity(&colInfo, 0, pBlock->info.rows); - for (int32_t j = 0; j < pBlock->info.rows; ++j) { - colDataAppendInt32(&colInfo, j, &j); - } - taosArrayPush(pBlock->pDataBlock, &colInfo); + + SColumnInfoData* pColInfo = bdGetColumnInfoData(pBlock, 0); + for (int32_t j = 0; j < pBlock->info.rows; ++j) { + colDataAppendInt32(pColInfo, j, &j); } SUdfInterBuf buf = {0}; @@ -121,6 +118,7 @@ int aggregateFuncTest() { freeUdfInterBuf(&resultBuf); doTeardownUdf(handle); + blockDataDestroy(pBlock); return 0; } diff --git a/source/libs/index/test/index_executor_tests.cpp b/source/libs/index/test/index_executor_tests.cpp index b88ffe5b8b..a0ac79a297 100644 --- a/source/libs/index/test/index_executor_tests.cpp +++ b/source/libs/index/test/index_executor_tests.cpp @@ -86,7 +86,7 @@ void sifAppendReservedSlot(SArray *pBlockList, int16_t *dataBlockId, int16_t *sl SColumnInfoData idata = {0}; idata.info = *colInfo; - colInfoDataEnsureCapacity(&idata, 0, rows); + colInfoDataEnsureCapacity(&idata, rows); taosArrayPush(res->pDataBlock, &idata); diff --git a/source/libs/monitor/src/monMain.c b/source/libs/monitor/src/monMain.c index bf857ad718..d387a432e1 100644 --- a/source/libs/monitor/src/monMain.c +++ b/source/libs/monitor/src/monMain.c @@ -201,6 +201,9 @@ static void monGenClusterJson(SMonInfo *pMonitor) { tjsonAddStringToObject(pJson, "version", pInfo->version); tjsonAddDoubleToObject(pJson, "master_uptime", pInfo->master_uptime); tjsonAddDoubleToObject(pJson, "monitor_interval", pInfo->monitor_interval); + tjsonAddDoubleToObject(pJson, "dbs_total", pInfo->dbs_total); + tjsonAddDoubleToObject(pJson, "tbs_total", pInfo->tbs_total); + tjsonAddDoubleToObject(pJson, "stbs_total", pInfo->stbs_total); tjsonAddDoubleToObject(pJson, "vgroups_total", pInfo->vgroups_total); tjsonAddDoubleToObject(pJson, "vgroups_alive", pInfo->vgroups_alive); tjsonAddDoubleToObject(pJson, "vnodes_total", pInfo->vnodes_total); @@ -277,6 +280,27 @@ static void monGenVgroupJson(SMonInfo *pMonitor) { } } +static void monGenStbJson(SMonInfo *pMonitor) { + SMonStbInfo *pInfo = &pMonitor->mmInfo.stb; + if (pMonitor->mmInfo.cluster.first_ep_dnode_id == 0) return; + + SJson *pJson = tjsonAddArrayToObject(pMonitor->pJson, "stb_infos"); + if (pJson == NULL) return; + + for (int32_t i = 0; i < taosArrayGetSize(pInfo->stbs); ++i) { + SJson *pStbJson = tjsonCreateObject(); + if (pStbJson == NULL) continue; + if (tjsonAddItemToArray(pJson, pStbJson) != 0) { + tjsonDelete(pStbJson); + continue; + } + + SMonStbDesc *pStbDesc = taosArrayGet(pInfo->stbs, i); + tjsonAddStringToObject(pStbJson, "stb_name", pStbDesc->stb_name); + tjsonAddStringToObject(pStbJson, "database_name", pStbDesc->database_name); + } +} + static void monGenGrantJson(SMonInfo *pMonitor) { SMonGrantInfo *pInfo = &pMonitor->mmInfo.grant; if (pMonitor->mmInfo.cluster.first_ep_dnode_id == 0) return; @@ -524,6 +548,7 @@ void monSendReport() { monGenBasicJson(pMonitor); monGenClusterJson(pMonitor); monGenVgroupJson(pMonitor); + monGenStbJson(pMonitor); monGenGrantJson(pMonitor); monGenDnodeJson(pMonitor); monGenDiskJson(pMonitor); diff --git a/source/libs/monitor/src/monMsg.c b/source/libs/monitor/src/monMsg.c index a041b582a9..8fa7e88605 100644 --- a/source/libs/monitor/src/monMsg.c +++ b/source/libs/monitor/src/monMsg.c @@ -102,6 +102,9 @@ int32_t tEncodeSMonClusterInfo(SEncoder *encoder, const SMonClusterInfo *pInfo) if (tEncodeCStr(encoder, pInfo->version) < 0) return -1; if (tEncodeFloat(encoder, pInfo->master_uptime) < 0) return -1; if (tEncodeI32(encoder, pInfo->monitor_interval) < 0) return -1; + if (tEncodeI32(encoder, pInfo->dbs_total) < 0) return -1; + if (tEncodeI32(encoder, pInfo->stbs_total) < 0) return -1; + if (tEncodeI64(encoder, pInfo->tbs_total) < 0) return -1; if (tEncodeI32(encoder, pInfo->vgroups_total) < 0) return -1; if (tEncodeI32(encoder, pInfo->vgroups_alive) < 0) return -1; if (tEncodeI32(encoder, pInfo->vnodes_total) < 0) return -1; @@ -130,6 +133,9 @@ int32_t tDecodeSMonClusterInfo(SDecoder *decoder, SMonClusterInfo *pInfo) { if (tDecodeCStrTo(decoder, pInfo->version) < 0) return -1; if (tDecodeFloat(decoder, &pInfo->master_uptime) < 0) return -1; if (tDecodeI32(decoder, &pInfo->monitor_interval) < 0) return -1; + if (tDecodeI32(decoder, &pInfo->dbs_total) < 0) return -1; + if (tDecodeI32(decoder, &pInfo->stbs_total) < 0) return -1; + if (tDecodeI64(decoder, &pInfo->tbs_total) < 0) return -1; if (tDecodeI32(decoder, &pInfo->vgroups_total) < 0) return -1; if (tDecodeI32(decoder, &pInfo->vgroups_alive) < 0) return -1; if (tDecodeI32(decoder, &pInfo->vnodes_total) < 0) return -1; @@ -203,6 +209,32 @@ int32_t tDecodeSMonVgroupInfo(SDecoder *decoder, SMonVgroupInfo *pInfo) { return 0; } +int32_t tEncodeSMonStbInfo(SEncoder *encoder, const SMonStbInfo *pInfo) { + if (tEncodeI32(encoder, taosArrayGetSize(pInfo->stbs)) < 0) return -1; + for (int32_t i = 0; i < taosArrayGetSize(pInfo->stbs); ++i) { + SMonStbDesc *pDesc = taosArrayGet(pInfo->stbs, i); + if (tEncodeCStr(encoder, pDesc->stb_name) < 0) return -1; + if (tEncodeCStr(encoder, pDesc->database_name) < 0) return -1; + } + return 0; +} + +int32_t tDecodeSMonStbInfo(SDecoder *decoder, SMonStbInfo *pInfo) { + int32_t arraySize = 0; + if (tDecodeI32(decoder, &arraySize) < 0) return -1; + + pInfo->stbs = taosArrayInit(arraySize, sizeof(SMonStbDesc)); + if (pInfo->stbs == NULL) return -1; + + for (int32_t i = 0; i < arraySize; ++i) { + SMonStbDesc desc = {0}; + if (tDecodeCStrTo(decoder, desc.stb_name) < 0) return -1; + if (tDecodeCStrTo(decoder, desc.database_name) < 0) return -1; + taosArrayPush(pInfo->stbs, &desc); + } + return 0; +} + int32_t tEncodeSMonGrantInfo(SEncoder *encoder, const SMonGrantInfo *pInfo) { if (tEncodeI32(encoder, pInfo->expire_time) < 0) return -1; if (tEncodeI64(encoder, pInfo->timeseries_used) < 0) return -1; @@ -224,6 +256,7 @@ int32_t tSerializeSMonMmInfo(void *buf, int32_t bufLen, SMonMmInfo *pInfo) { if (tStartEncode(&encoder) < 0) return -1; if (tEncodeSMonClusterInfo(&encoder, &pInfo->cluster) < 0) return -1; if (tEncodeSMonVgroupInfo(&encoder, &pInfo->vgroup) < 0) return -1; + if (tEncodeSMonStbInfo(&encoder, &pInfo->stb) < 0) return -1; if (tEncodeSMonGrantInfo(&encoder, &pInfo->grant) < 0) return -1; if (tEncodeSMonSysInfo(&encoder, &pInfo->sys) < 0) return -1; if (tEncodeSMonLogs(&encoder, &pInfo->log) < 0) return -1; @@ -241,6 +274,7 @@ int32_t tDeserializeSMonMmInfo(void *buf, int32_t bufLen, SMonMmInfo *pInfo) { if (tStartDecode(&decoder) < 0) return -1; if (tDecodeSMonClusterInfo(&decoder, &pInfo->cluster) < 0) return -1; if (tDecodeSMonVgroupInfo(&decoder, &pInfo->vgroup) < 0) return -1; + if (tDecodeSMonStbInfo(&decoder, &pInfo->stb) < 0) return -1; if (tDecodeSMonGrantInfo(&decoder, &pInfo->grant) < 0) return -1; if (tDecodeSMonSysInfo(&decoder, &pInfo->sys) < 0) return -1; if (tDecodeSMonLogs(&decoder, &pInfo->log) < 0) return -1; @@ -255,9 +289,11 @@ void tFreeSMonMmInfo(SMonMmInfo *pInfo) { taosArrayDestroy(pInfo->cluster.mnodes); taosArrayDestroy(pInfo->cluster.dnodes); taosArrayDestroy(pInfo->vgroup.vgroups); + taosArrayDestroy(pInfo->stb.stbs); pInfo->cluster.mnodes = NULL; pInfo->cluster.dnodes = NULL; pInfo->vgroup.vgroups = NULL; + pInfo->stb.stbs = NULL; pInfo->log.logs = NULL; } diff --git a/source/libs/scalar/inc/sclInt.h b/source/libs/scalar/inc/sclInt.h index 4dd16098ff..c2d984c7dd 100644 --- a/source/libs/scalar/inc/sclInt.h +++ b/source/libs/scalar/inc/sclInt.h @@ -57,7 +57,7 @@ typedef struct SScalarCtx { #define SCL_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _return; } } while (0) int32_t doConvertDataType(SValueNode* pValueNode, SScalarParam* out); -SColumnInfoData* createColumnInfoData(SDataType* pType, int32_t numOfRows); +SColumnInfoData* sclCreateColumnInfoData(SDataType* pType, int32_t numOfRows); int32_t sclConvertToTsValueNode(int8_t precision, SValueNode* valueNode); #define GET_PARAM_TYPE(_c) ((_c)->columnData ? (_c)->columnData->info.type : (_c)->hashValueType) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index a446aa7bdb..b8bcae161b 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -3821,7 +3821,7 @@ bool filterExecute(SFilterInfo *info, SSDataBlock *pSrc, int8_t** p, SColumnData SScalarParam output = {0}; SDataType type = {.type = TSDB_DATA_TYPE_BOOL, .bytes = sizeof(bool)}; - output.columnData = createColumnInfoData(&type, pSrc->info.rows); + output.columnData = sclCreateColumnInfoData(&type, pSrc->info.rows); SArray *pList = taosArrayInit(1, POINTER_BYTES); taosArrayPush(pList, &pSrc); diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index 66e4af5ded..75a9bc3809 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -36,7 +36,7 @@ int32_t sclConvertToTsValueNode(int8_t precision, SValueNode* valueNode) { } -SColumnInfoData* createColumnInfoData(SDataType* pType, int32_t numOfRows) { +SColumnInfoData* sclCreateColumnInfoData(SDataType* pType, int32_t numOfRows) { SColumnInfoData* pColumnData = taosMemoryCalloc(1, sizeof(SColumnInfoData)); if (pColumnData == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -48,7 +48,7 @@ SColumnInfoData* createColumnInfoData(SDataType* pType, int32_t numOfRows) { pColumnData->info.scale = pType->scale; pColumnData->info.precision = pType->precision; - int32_t code = colInfoDataEnsureCapacity(pColumnData, 0, numOfRows); + int32_t code = colInfoDataEnsureCapacity(pColumnData, numOfRows); if (code != TSDB_CODE_SUCCESS) { terrno = TSDB_CODE_OUT_OF_MEMORY; taosMemoryFree(pColumnData); @@ -60,10 +60,10 @@ SColumnInfoData* createColumnInfoData(SDataType* pType, int32_t numOfRows) { int32_t doConvertDataType(SValueNode* pValueNode, SScalarParam* out) { SScalarParam in = {.numOfRows = 1}; - in.columnData = createColumnInfoData(&pValueNode->node.resType, 1); + in.columnData = sclCreateColumnInfoData(&pValueNode->node.resType, 1); colDataAppend(in.columnData, 0, nodesGetValueFromNode(pValueNode), false); - colInfoDataEnsureCapacity(out->columnData, 0, 1); + colInfoDataEnsureCapacity(out->columnData, 1); int32_t code = vectorConvertImpl(&in, out); sclFreeParam(&in); @@ -193,7 +193,7 @@ int32_t sclInitParam(SNode* node, SScalarParam *param, SScalarCtx *ctx, int32_t SValueNode *valueNode = (SValueNode *)node; param->numOfRows = 1; - param->columnData = createColumnInfoData(&valueNode->node.resType, 1); + param->columnData = sclCreateColumnInfoData(&valueNode->node.resType, 1); if (TSDB_DATA_TYPE_NULL == valueNode->node.resType.type) { colDataAppendNULL(param->columnData, 0); } else { @@ -245,7 +245,7 @@ int32_t sclInitParam(SNode* node, SScalarParam *param, SScalarCtx *ctx, int32_t } SSDataBlock *block = *(SSDataBlock **)taosArrayGet(ctx->pBlockList, index); - if (NULL == block || ref->slotId >= block->info.numOfCols) { + if (NULL == block || ref->slotId >= taosArrayGetSize(block->pDataBlock)) { sclError("column slotId is too big, slodId:%d, dataBlockSize:%d", ref->slotId, (int32_t)taosArrayGetSize(block->pDataBlock)); SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } @@ -431,7 +431,7 @@ int32_t sclExecFunction(SFunctionNode *node, SScalarCtx *ctx, SScalarParam *outp SCL_ERR_JRET(code); } - output->columnData = createColumnInfoData(&node->node.resType, rowNum); + output->columnData = sclCreateColumnInfoData(&node->node.resType, rowNum); if (output->columnData == NULL) { sclError("calloc %d failed", (int32_t)(rowNum * output->columnData->info.bytes)); SCL_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -484,7 +484,7 @@ int32_t sclExecLogic(SLogicConditionNode *node, SScalarCtx *ctx, SScalarParam *o output->numOfRows = rowNum; SDataType t = {.type = type, .bytes = tDataTypes[type].bytes}; - output->columnData = createColumnInfoData(&t, rowNum); + output->columnData = sclCreateColumnInfoData(&t, rowNum); if (output->columnData == NULL) { sclError("calloc %d failed", (int32_t)(rowNum * sizeof(bool))); SCL_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -539,7 +539,7 @@ int32_t sclExecOperator(SOperatorNode *node, SScalarCtx *ctx, SScalarParam *outp int32_t code = 0; SCL_ERR_RET(sclInitOperatorParams(¶ms, node, ctx, &rowNum)); - output->columnData = createColumnInfoData(&node->node.resType, rowNum); + output->columnData = sclCreateColumnInfoData(&node->node.resType, rowNum); if (output->columnData == NULL) { sclError("calloc failed, size:%d", (int32_t)rowNum * node->node.resType.bytes); SCL_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -901,7 +901,7 @@ EDealRes sclWalkTarget(SNode* pNode, SScalarCtx *ctx) { return DEAL_RES_ERROR; } - colDataAssign(col, res->columnData, res->numOfRows); + colDataAssign(col, res->columnData, res->numOfRows, NULL); block->info.rows = res->numOfRows; sclFreeParam(res); @@ -982,8 +982,9 @@ int32_t scalarCalculate(SNode *pNode, SArray *pBlockList, SScalarParam *pDst) { sclError("no valid res in hash, node:%p, type:%d", pNode, nodeType(pNode)); SCL_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); } - - colDataAssign(pDst->columnData, res->columnData, res->numOfRows); + + colInfoDataEnsureCapacity(pDst->columnData, res->numOfRows); + colDataAssign(pDst->columnData, res->columnData, res->numOfRows, NULL); pDst->numOfRows = res->numOfRows; taosHashRemove(ctx.pRes, (void *)&pNode, POINTER_BYTES); } diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index d335b04ea4..d7f0733129 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -36,9 +36,6 @@ int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutpu SColumnInfoData *pOutputData = pOutput->columnData; int32_t type = GET_PARAM_TYPE(pInput); - if (!IS_NUMERIC_TYPE(type)) { - return TSDB_CODE_FAILED; - } switch (type) { case TSDB_DATA_TYPE_FLOAT: { @@ -119,8 +116,15 @@ int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutpu break; } + case TSDB_DATA_TYPE_NULL: { + for (int32_t i = 0; i < pInput->numOfRows; ++i) { + colDataAppendNULL(pOutputData, i); + } + break; + } + default: { - colDataAssign(pOutputData, pInputData, pInput->numOfRows); + colDataAssign(pOutputData, pInputData, pInput->numOfRows, NULL); } } @@ -130,9 +134,6 @@ int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutpu static int32_t doScalarFunctionUnique(SScalarParam *pInput, int32_t inputNum, SScalarParam* pOutput, _double_fn valFn) { int32_t type = GET_PARAM_TYPE(pInput); - if (inputNum != 1 || !IS_NUMERIC_TYPE(type)) { - return TSDB_CODE_FAILED; - } SColumnInfoData *pInputData = pInput->columnData; SColumnInfoData *pOutputData = pOutput->columnData; @@ -142,7 +143,7 @@ static int32_t doScalarFunctionUnique(SScalarParam *pInput, int32_t inputNum, SS double *out = (double *)pOutputData->pData; for (int32_t i = 0; i < pInput->numOfRows; ++i) { - if (colDataIsNull_s(pInputData, i)) { + if (colDataIsNull_s(pInputData, i) || IS_NULL_TYPE(type)) { colDataAppendNULL(pOutputData, i); continue; } @@ -159,10 +160,6 @@ static int32_t doScalarFunctionUnique(SScalarParam *pInput, int32_t inputNum, SS } static int32_t doScalarFunctionUnique2(SScalarParam *pInput, int32_t inputNum, SScalarParam* pOutput, _double_fn_2 valFn) { - if (inputNum != 2 || !IS_NUMERIC_TYPE(GET_PARAM_TYPE(&pInput[0])) || !IS_NUMERIC_TYPE(GET_PARAM_TYPE(&pInput[1]))) { - return TSDB_CODE_FAILED; - } - SColumnInfoData *pInputData[2]; SColumnInfoData *pOutputData = pOutput->columnData; _getDoubleValue_fn_t getValueFn[2]; @@ -175,11 +172,15 @@ static int32_t doScalarFunctionUnique2(SScalarParam *pInput, int32_t inputNum, S double *out = (double *)pOutputData->pData; double result; + bool hasNullType = (IS_NULL_TYPE(GET_PARAM_TYPE(&pInput[0])) || + IS_NULL_TYPE(GET_PARAM_TYPE(&pInput[1]))); + int32_t numOfRows = TMAX(pInput[0].numOfRows, pInput[1].numOfRows); if (pInput[0].numOfRows == pInput[1].numOfRows) { for (int32_t i = 0; i < numOfRows; ++i) { if (colDataIsNull_s(pInputData[0], i) || - colDataIsNull_s(pInputData[1], i)) { + colDataIsNull_s(pInputData[1], i) || + hasNullType) { colDataAppendNULL(pOutputData, i); continue; } @@ -191,7 +192,7 @@ static int32_t doScalarFunctionUnique2(SScalarParam *pInput, int32_t inputNum, S } } } else if (pInput[0].numOfRows == 1) { //left operand is constant - if (colDataIsNull_s(pInputData[0], 0)) { + if (colDataIsNull_s(pInputData[0], 0) || hasNullType) { colDataAppendNNULL(pOutputData, 0, pInput[1].numOfRows); } else { for (int32_t i = 0; i < numOfRows; ++i) { @@ -210,7 +211,7 @@ static int32_t doScalarFunctionUnique2(SScalarParam *pInput, int32_t inputNum, S } } } else if (pInput[1].numOfRows == 1) { - if (colDataIsNull_s(pInputData[1], 0)) { + if (colDataIsNull_s(pInputData[1], 0) || hasNullType) { colDataAppendNNULL(pOutputData, 0, pInput[0].numOfRows); } else { for (int32_t i = 0; i < numOfRows; ++i) { @@ -236,9 +237,6 @@ static int32_t doScalarFunctionUnique2(SScalarParam *pInput, int32_t inputNum, S static int32_t doScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam* pOutput, _float_fn f1, _double_fn d1) { int32_t type = GET_PARAM_TYPE(pInput); - if (inputNum != 1 || !IS_NUMERIC_TYPE(type)) { - return TSDB_CODE_FAILED; - } SColumnInfoData *pInputData = pInput->columnData; SColumnInfoData *pOutputData = pOutput->columnData; @@ -272,8 +270,15 @@ static int32_t doScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarP break; } + case TSDB_DATA_TYPE_NULL: { + for (int32_t i = 0; i < pInput->numOfRows; ++i) { + colDataAppendNULL(pOutputData, i); + } + break; + } + default: { - colDataAssign(pOutputData, pInputData, pInput->numOfRows); + colDataAssign(pOutputData, pInputData, pInput->numOfRows, NULL); } } @@ -856,10 +861,13 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp int32_t toISO8601Function(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { int32_t type = GET_PARAM_TYPE(pInput); + bool tzPresent = (inputNum == 2) ? true : false; char* tz; int32_t tzLen; - tz = varDataVal(pInput[1].columnData->pData); - tzLen = varDataLen(pInput[1].columnData->pData); + if (tzPresent) { + tz = varDataVal(pInput[1].columnData->pData); + tzLen = varDataLen(pInput[1].columnData->pData); + } for (int32_t i = 0; i < pInput[0].numOfRows; ++i) { if (colDataIsNull_s(pInput[0].columnData, i)) { diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index 50f2fd59d8..f9a9ec0f72 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -852,7 +852,7 @@ int32_t vectorConvertScalarParam(SScalarParam *input, SScalarParam *output, int3 SDataType t = {.type = type, .bytes = tDataTypes[type].bytes}; output->numOfRows = input->numOfRows; - output->columnData = createColumnInfoData(&t, input->numOfRows); + output->columnData = sclCreateColumnInfoData(&t, input->numOfRows); if (output->columnData == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -923,7 +923,7 @@ static int32_t doConvertHelper(SScalarParam* pDest, int32_t* convert, const SSca pDest->numOfRows = pParam->numOfRows; SDataType t = {.type = type, .bytes = tDataTypes[type].bytes}; - pDest->columnData = createColumnInfoData(&t, pParam->numOfRows); + pDest->columnData = sclCreateColumnInfoData(&t, pParam->numOfRows); if (pDest->columnData == NULL) { sclError("malloc %d failed", (int32_t)(pParam->numOfRows * sizeof(double))); return TSDB_CODE_OUT_OF_MEMORY; diff --git a/source/libs/scalar/test/filter/filterTests.cpp b/source/libs/scalar/test/filter/filterTests.cpp index 7fb1ffbd64..8f9332cef8 100644 --- a/source/libs/scalar/test/filter/filterTests.cpp +++ b/source/libs/scalar/test/filter/filterTests.cpp @@ -105,29 +105,14 @@ void flttMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in } if (NULL == *block) { - SSDataBlock *res = (SSDataBlock *)taosMemoryCalloc(1, sizeof(SSDataBlock)); - res->info.numOfCols = 3; - res->info.rows = rowNum; - res->pDataBlock = taosArrayInit(3, sizeof(SColumnInfoData)); + SSDataBlock *res = createDataBlock(); for (int32_t i = 0; i < 2; ++i) { - SColumnInfoData idata = {{0}}; - idata.info.type = TSDB_DATA_TYPE_NULL; - idata.info.bytes = 10; - idata.info.colId = i + 1; - - int32_t size = idata.info.bytes * rowNum; - idata.pData = (char *)taosMemoryCalloc(1, size); - taosArrayPush(res->pDataBlock, &idata); + SColumnInfoData idata = createColumnInfoData(TSDB_DATA_TYPE_NULL, 10, 1+i); + blockDataAppendColInfo(res, &idata); } - SColumnInfoData idata = {{0}}; - idata.info.type = dataType; - idata.info.bytes = dataBytes; - idata.info.colId = 3; - int32_t size = idata.info.bytes * rowNum; - idata.pData = (char *)taosMemoryCalloc(1, size); - taosArrayPush(res->pDataBlock, &idata); - + SColumnInfoData idata = createColumnInfoData(dataType, dataBytes, 3); + blockDataAppendColInfo(res, &idata); blockDataEnsureCapacity(res, rowNum); SColumnInfoData *pColumn = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); @@ -142,23 +127,18 @@ void flttMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in rnode->slotId = 2; rnode->colId = 3; + res->info.rows = rowNum; *block = res; } else { SSDataBlock *res = *block; int32_t idx = taosArrayGetSize(res->pDataBlock); - SColumnInfoData idata = {{0}}; - idata.info.type = dataType; - idata.info.bytes = dataBytes; - idata.info.colId = 1 + idx; - int32_t size = idata.info.bytes * rowNum; - idata.pData = (char *)taosMemoryCalloc(1, size); - taosArrayPush(res->pDataBlock, &idata); - res->info.numOfCols++; + SColumnInfoData idata = createColumnInfoData(dataType, dataBytes, 1+idx); + blockDataAppendColInfo(res, &idata); + blockDataEnsureCapacity(res, rowNum); + SColumnInfoData *pColumn = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); - - colInfoDataEnsureCapacity(pColumn, 0, rowNum); for (int32_t i = 0; i < rowNum; ++i) { colDataAppend(pColumn, i, (const char *)value, false); @@ -399,7 +379,7 @@ TEST(columnTest, smallint_column_greater_double_value) { keep = filterRangeExecute(filter, &stat, 1, rowNum); ASSERT_EQ(keep, true); - SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; + SFilterColumnParam param = { (int32_t) taosArrayGetSize(src->pDataBlock), src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -407,9 +387,9 @@ TEST(columnTest, smallint_column_greater_double_value) { stat.min = 1; stat.numOfNull = 0; int8_t *rowRes = NULL; - keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + keep = filterExecute(filter, src, &rowRes, &stat, (int32_t) taosArrayGetSize(src->pDataBlock)); ASSERT_EQ(keep, false); - + for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } @@ -430,7 +410,7 @@ TEST(columnTest, int_column_greater_smallint_value) { flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, leftv); flttMakeValueNode(&pRight, TSDB_DATA_TYPE_SMALLINT, &rightv); flttMakeOpNode(&opNode, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pLeft, pRight); - + SFilterInfo *filter = NULL; int32_t code = filterInitFromNode(opNode, &filter, 0); ASSERT_EQ(code, 0); @@ -454,7 +434,7 @@ TEST(columnTest, int_column_greater_smallint_value) { keep = filterRangeExecute(filter, &stat, 1, rowNum); ASSERT_EQ(keep, false); - SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; + SFilterColumnParam param = { (int32_t) taosArrayGetSize(src->pDataBlock), src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -462,9 +442,9 @@ TEST(columnTest, int_column_greater_smallint_value) { stat.min = 1; stat.numOfNull = 0; int8_t *rowRes = NULL; - keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + keep = filterExecute(filter, src, &rowRes, &stat, (int32_t) taosArrayGetSize(src->pDataBlock)); ASSERT_EQ(keep, false); - + for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } @@ -478,11 +458,11 @@ TEST(columnTest, int_column_in_double_list) { SNode *pLeft = NULL, *pRight = NULL, *listNode = NULL, *opNode = NULL; int32_t leftv[5] = {1, 2, 3, 4, 5}; double rightv1 = 1.1,rightv2 = 2.2,rightv3 = 3.3; - bool eRes[5] = {true, true, true, false, false}; - SSDataBlock *src = NULL; + bool eRes[5] = {true, true, true, false, false}; + SSDataBlock *src = NULL; SScalarParam res = {0}; int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); - flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, leftv); + flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, leftv); SNodeList* list = nodesMakeList(); flttMakeValueNode(&pRight, TSDB_DATA_TYPE_DOUBLE, &rightv1); nodesListAppend(list, pRight); @@ -498,7 +478,7 @@ TEST(columnTest, int_column_in_double_list) { ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; + SFilterColumnParam param = { (int32_t) taosArrayGetSize(src->pDataBlock), src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -506,9 +486,9 @@ TEST(columnTest, int_column_in_double_list) { stat.min = 1; stat.numOfNull = 0; int8_t *rowRes = NULL; - bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + bool keep = filterExecute(filter, src, &rowRes, &stat, (int32_t) taosArrayGetSize(src->pDataBlock)); ASSERT_EQ(keep, false); - + for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } @@ -521,8 +501,8 @@ TEST(columnTest, int_column_in_double_list) { TEST(columnTest, binary_column_in_binary_list) { SNode *pLeft = NULL, *pRight = NULL, *listNode = NULL, *opNode = NULL; - bool eRes[5] = {true, true, false, false, false}; - SSDataBlock *src = NULL; + bool eRes[5] = {true, true, false, false, false}; + SSDataBlock *src = NULL; SScalarParam res = {0}; char leftv[5][5]= {0}; char rightv[3][5]= {0}; @@ -531,22 +511,22 @@ TEST(columnTest, binary_column_in_binary_list) { leftv[i][3] = 'b' + i; leftv[i][4] = '0' + i; varDataSetLen(leftv[i], 3); - } + } for (int32_t i = 0; i < 2; ++i) { rightv[i][2] = 'a' + i; rightv[i][3] = 'b' + i; rightv[i][4] = '0' + i; varDataSetLen(rightv[i], 3); - } + } for (int32_t i = 2; i < 3; ++i) { rightv[i][2] = 'a' + i; rightv[i][3] = 'a' + i; rightv[i][4] = 'a' + i; varDataSetLen(rightv[i], 3); } - + int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); - flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); + flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); SNodeList* list = nodesMakeList(); flttMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, rightv[0]); nodesListAppend(list, pRight); @@ -556,13 +536,13 @@ TEST(columnTest, binary_column_in_binary_list) { nodesListAppend(list, pRight); flttMakeListNode(&listNode,list, TSDB_DATA_TYPE_BINARY); flttMakeOpNode(&opNode, OP_TYPE_IN, TSDB_DATA_TYPE_BOOL, pLeft, listNode); - + SFilterInfo *filter = NULL; int32_t code = filterInitFromNode(opNode, &filter, 0); ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; + SFilterColumnParam param = { (int32_t) taosArrayGetSize(src->pDataBlock), src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -570,9 +550,9 @@ TEST(columnTest, binary_column_in_binary_list) { stat.min = 1; stat.numOfNull = 0; int8_t *rowRes = NULL; - bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + bool keep = filterExecute(filter, src, &rowRes, &stat, (int32_t) taosArrayGetSize(src->pDataBlock)); ASSERT_EQ(keep, false); - + for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } @@ -586,31 +566,31 @@ TEST(columnTest, binary_column_like_binary) { SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL; char rightv[64] = {0}; char leftv[5][5]= {0}; - SSDataBlock *src = NULL; + SSDataBlock *src = NULL; SScalarParam res = {0}; - bool eRes[5] = {true, false, true, false, true}; - + bool eRes[5] = {true, false, true, false, true}; + for (int32_t i = 0; i < 5; ++i) { leftv[i][2] = 'a'; leftv[i][3] = 'a'; leftv[i][4] = '0' + i % 2; varDataSetLen(leftv[i], 3); - } - + } + int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); - flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); + flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); sprintf(&rightv[2], "%s", "__0"); varDataSetLen(rightv, strlen(&rightv[2])); flttMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, rightv); flttMakeOpNode(&opNode, OP_TYPE_LIKE, TSDB_DATA_TYPE_BOOL, pLeft, pRight); - + SFilterInfo *filter = NULL; int32_t code = filterInitFromNode(opNode, &filter, 0); ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; + SFilterColumnParam param = { (int32_t) taosArrayGetSize(src->pDataBlock), src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -618,9 +598,9 @@ TEST(columnTest, binary_column_like_binary) { stat.min = 1; stat.numOfNull = 0; int8_t *rowRes = NULL; - bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + bool keep = filterExecute(filter, src, &rowRes, &stat, (int32_t) taosArrayGetSize(src->pDataBlock)); ASSERT_EQ(keep, false); - + for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } @@ -633,31 +613,31 @@ TEST(columnTest, binary_column_like_binary) { TEST(columnTest, binary_column_is_null) { SNode *pLeft = NULL, *opNode = NULL; char leftv[5][5]= {0}; - SSDataBlock *src = NULL; + SSDataBlock *src = NULL; SScalarParam res = {0}; - bool eRes[5] = {false, false, true, false, true}; - + bool eRes[5] = {false, false, true, false, true}; + for (int32_t i = 0; i < 5; ++i) { leftv[i][2] = '0' + i % 2; leftv[i][3] = 'a'; leftv[i][4] = '0' + i % 2; varDataSetLen(leftv[i], 3); - } - + } + int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); - flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); + flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); SColumnInfoData *pcolumn = (SColumnInfoData *)taosArrayGetLast(src->pDataBlock); colDataAppend(pcolumn, 2, NULL, true); colDataAppend(pcolumn, 4, NULL, true); flttMakeOpNode(&opNode, OP_TYPE_IS_NULL, TSDB_DATA_TYPE_BOOL, pLeft, NULL); - + SFilterInfo *filter = NULL; int32_t code = filterInitFromNode(opNode, &filter, 0); ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; + SFilterColumnParam param = { (int32_t) taosArrayGetSize(src->pDataBlock), src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -665,9 +645,9 @@ TEST(columnTest, binary_column_is_null) { stat.min = 1; stat.numOfNull = 0; int8_t *rowRes = NULL; - bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + bool keep = filterExecute(filter, src, &rowRes, &stat, (int32_t) taosArrayGetSize(src->pDataBlock)); ASSERT_EQ(keep, false); - + for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } @@ -680,31 +660,31 @@ TEST(columnTest, binary_column_is_null) { TEST(columnTest, binary_column_is_not_null) { SNode *pLeft = NULL, *opNode = NULL; char leftv[5][5]= {0}; - SSDataBlock *src = NULL; + SSDataBlock *src = NULL; SScalarParam res = {0}; - bool eRes[5] = {true, true, true, true, false}; - + bool eRes[5] = {true, true, true, true, false}; + for (int32_t i = 0; i < 5; ++i) { leftv[i][2] = '0' + i % 2; leftv[i][3] = 'a'; leftv[i][4] = '0' + i % 2; varDataSetLen(leftv[i], 3); - } - + } + int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); - flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); + flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); SColumnInfoData *pcolumn = (SColumnInfoData *)taosArrayGetLast(src->pDataBlock); colDataAppend(pcolumn, 4, NULL, true); flttMakeOpNode(&opNode, OP_TYPE_IS_NOT_NULL, TSDB_DATA_TYPE_BOOL, pLeft, NULL); - + SFilterInfo *filter = NULL; int32_t code = filterInitFromNode(opNode, &filter, 0); ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; + SFilterColumnParam param = { (int32_t) taosArrayGetSize(src->pDataBlock), src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -712,9 +692,9 @@ TEST(columnTest, binary_column_is_not_null) { stat.min = 1; stat.numOfNull = 0; int8_t *rowRes = NULL; - bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + bool keep = filterExecute(filter, src, &rowRes, &stat, (int32_t) taosArrayGetSize(src->pDataBlock)); ASSERT_EQ(keep, false); - + for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } @@ -735,13 +715,13 @@ TEST(opTest, smallint_column_greater_int_column) { flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv); flttMakeColumnNode(&pRight, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, rightv); flttMakeOpNode(&opNode, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pLeft, pRight); - + SFilterInfo *filter = NULL; int32_t code = filterInitFromNode(opNode, &filter, 0); ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; + SFilterColumnParam param = { (int32_t) taosArrayGetSize(src->pDataBlock), src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -749,9 +729,9 @@ TEST(opTest, smallint_column_greater_int_column) { stat.min = 1; stat.numOfNull = 0; int8_t *rowRes = NULL; - bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + bool keep = filterExecute(filter, src, &rowRes, &stat, (int32_t) taosArrayGetSize(src->pDataBlock)); ASSERT_EQ(keep, false); - + for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } @@ -773,13 +753,13 @@ TEST(opTest, smallint_value_add_int_column) { flttMakeColumnNode(&pRight, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, rightv); flttMakeOpNode(&opNode, OP_TYPE_ADD, TSDB_DATA_TYPE_DOUBLE, pLeft, pRight); flttMakeOpNode(&opNode, OP_TYPE_IS_TRUE, TSDB_DATA_TYPE_BOOL, opNode, NULL); - + SFilterInfo *filter = NULL; int32_t code = filterInitFromNode(opNode, &filter, 0); ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; + SFilterColumnParam param = { (int32_t) taosArrayGetSize(src->pDataBlock), src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -787,9 +767,9 @@ TEST(opTest, smallint_value_add_int_column) { stat.min = 1; stat.numOfNull = 0; int8_t *rowRes = NULL; - bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + bool keep = filterExecute(filter, src, &rowRes, &stat, (int32_t) taosArrayGetSize(src->pDataBlock)); ASSERT_EQ(keep, false); - + for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } @@ -816,13 +796,13 @@ TEST(opTest, bigint_column_multi_binary_column) { flttMakeColumnNode(&pRight, &src, TSDB_DATA_TYPE_BINARY, 5, rowNum, rightv); flttMakeOpNode(&opNode, OP_TYPE_MULTI, TSDB_DATA_TYPE_DOUBLE, pLeft, pRight); flttMakeOpNode(&opNode, OP_TYPE_IS_TRUE, TSDB_DATA_TYPE_BOOL, opNode, NULL); - + SFilterInfo *filter = NULL; int32_t code = filterInitFromNode(opNode, &filter, 0); ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; + SFilterColumnParam param = { (int32_t) taosArrayGetSize(src->pDataBlock), src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -830,9 +810,9 @@ TEST(opTest, bigint_column_multi_binary_column) { stat.min = 1; stat.numOfNull = 0; int8_t *rowRes = NULL; - bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + bool keep = filterExecute(filter, src, &rowRes, &stat, (int32_t) taosArrayGetSize(src->pDataBlock)); ASSERT_EQ(keep, false); - + for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } @@ -859,13 +839,13 @@ TEST(opTest, smallint_column_and_binary_column) { flttMakeColumnNode(&pRight, &src, TSDB_DATA_TYPE_BINARY, 5, rowNum, rightv); flttMakeOpNode(&opNode, OP_TYPE_BIT_AND, TSDB_DATA_TYPE_BIGINT, pLeft, pRight); flttMakeOpNode(&opNode, OP_TYPE_IS_TRUE, TSDB_DATA_TYPE_BOOL, opNode, NULL); - + SFilterInfo *filter = NULL; int32_t code = filterInitFromNode(opNode, &filter, 0); ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; + SFilterColumnParam param = { (int32_t) taosArrayGetSize(src->pDataBlock), src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -873,9 +853,9 @@ TEST(opTest, smallint_column_and_binary_column) { stat.min = 1; stat.numOfNull = 0; int8_t *rowRes = NULL; - bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + bool keep = filterExecute(filter, src, &rowRes, &stat, (int32_t) taosArrayGetSize(src->pDataBlock)); ASSERT_EQ(keep, false); - + for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } @@ -897,13 +877,13 @@ TEST(opTest, smallint_column_or_float_column) { flttMakeColumnNode(&pRight, &src, TSDB_DATA_TYPE_FLOAT, sizeof(float), rowNum, rightv); flttMakeOpNode(&opNode, OP_TYPE_BIT_OR, TSDB_DATA_TYPE_BIGINT, pLeft, pRight); flttMakeOpNode(&opNode, OP_TYPE_IS_TRUE, TSDB_DATA_TYPE_BOOL, opNode, NULL); - + SFilterInfo *filter = NULL; int32_t code = filterInitFromNode(opNode, &filter, 0); ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; + SFilterColumnParam param = { (int32_t)taosArrayGetSize(src->pDataBlock), src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -911,9 +891,9 @@ TEST(opTest, smallint_column_or_float_column) { stat.min = 1; stat.numOfNull = 0; int8_t *rowRes = NULL; - bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + bool keep = filterExecute(filter, src, &rowRes, &stat, taosArrayGetSize(src->pDataBlock)); ASSERT_EQ(keep, false); - + for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } @@ -935,13 +915,13 @@ TEST(opTest, smallint_column_or_double_value) { flttMakeValueNode(&pRight, TSDB_DATA_TYPE_DOUBLE, &rightv); flttMakeOpNode(&opNode, OP_TYPE_BIT_OR, TSDB_DATA_TYPE_BIGINT, pLeft, pRight); flttMakeOpNode(&opNode, OP_TYPE_IS_TRUE, TSDB_DATA_TYPE_BOOL, opNode, NULL); - + SFilterInfo *filter = NULL; int32_t code = filterInitFromNode(opNode, &filter, 0); ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; + SFilterColumnParam param = { (int32_t)taosArrayGetSize(src->pDataBlock), src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -949,9 +929,9 @@ TEST(opTest, smallint_column_or_double_value) { stat.min = 1; stat.numOfNull = 0; int8_t *rowRes = NULL; - bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + bool keep = filterExecute(filter, src, &rowRes, &stat, taosArrayGetSize(src->pDataBlock)); ASSERT_EQ(keep, true); - + for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } @@ -964,28 +944,28 @@ TEST(opTest, smallint_column_or_double_value) { TEST(opTest, binary_column_is_true) { SNode *pLeft = NULL, *opNode = NULL; char leftv[5][5]= {0}; - SSDataBlock *src = NULL; + SSDataBlock *src = NULL; SScalarParam res = {0}; - bool eRes[5] = {false, true, false, true, false}; - + bool eRes[5] = {false, true, false, true, false}; + for (int32_t i = 0; i < 5; ++i) { leftv[i][2] = '0' + i % 2; leftv[i][3] = 'a'; leftv[i][4] = '0' + i % 2; varDataSetLen(leftv[i], 3); - } - + } + int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); - flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); + flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); flttMakeOpNode(&opNode, OP_TYPE_IS_TRUE, TSDB_DATA_TYPE_BOOL, pLeft, NULL); - + SFilterInfo *filter = NULL; int32_t code = filterInitFromNode(opNode, &filter, 0); ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; + SFilterColumnParam param = { (int32_t)taosArrayGetSize(src->pDataBlock), src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -993,9 +973,9 @@ TEST(opTest, binary_column_is_true) { stat.min = 1; stat.numOfNull = 0; int8_t *rowRes = NULL; - bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + bool keep = filterExecute(filter, src, &rowRes, &stat, taosArrayGetSize(src->pDataBlock)); ASSERT_EQ(keep, false); - + for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } @@ -1027,12 +1007,12 @@ TEST(filterModelogicTest, diff_columns_and_or_and) { flttMakeValueNode(&pRight2, TSDB_DATA_TYPE_INT, &rightv2); flttMakeOpNode(&opNode2, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft2, pRight2); nodesListAppend(list, opNode2); - + flttMakeLogicNodeFromList(&logicNode1, LOGIC_COND_TYPE_AND, list); list = nodesMakeList(); - + flttMakeColumnNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1); flttMakeValueNode(&pRight1, TSDB_DATA_TYPE_INT, &rightv1); flttMakeOpNode(&opNode1, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft1, pRight1); @@ -1042,7 +1022,7 @@ TEST(filterModelogicTest, diff_columns_and_or_and) { flttMakeValueNode(&pRight2, TSDB_DATA_TYPE_INT, &rightv2); flttMakeOpNode(&opNode2, OP_TYPE_GREATER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft2, pRight2); nodesListAppend(list, opNode2); - + flttMakeLogicNodeFromList(&logicNode2, LOGIC_COND_TYPE_AND, list); list = nodesMakeList(); @@ -1050,13 +1030,13 @@ TEST(filterModelogicTest, diff_columns_and_or_and) { nodesListAppend(list, logicNode2); flttMakeLogicNodeFromList(&logicNode1, LOGIC_COND_TYPE_OR, list); - + SFilterInfo *filter = NULL; int32_t code = filterInitFromNode(logicNode1, &filter, 0); ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; + SFilterColumnParam param = { (int32_t)taosArrayGetSize(src->pDataBlock), src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -1064,9 +1044,9 @@ TEST(filterModelogicTest, diff_columns_and_or_and) { stat.min = 1; stat.numOfNull = 0; int8_t *rowRes = NULL; - bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + bool keep = filterExecute(filter, src, &rowRes, &stat, taosArrayGetSize(src->pDataBlock)); ASSERT_EQ(keep, false); - + for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } @@ -1096,13 +1076,13 @@ TEST(filterModelogicTest, same_column_and_or_and) { flttMakeValueNode(&pRight2, TSDB_DATA_TYPE_INT, &rightv2); flttMakeOpNode(&opNode2, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft1, pRight2); nodesListAppend(list, opNode2); - + flttMakeLogicNodeFromList(&logicNode1, LOGIC_COND_TYPE_AND, list); list = nodesMakeList(); - flttMakeColumnNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1); + flttMakeColumnNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1); flttMakeValueNode(&pRight1, TSDB_DATA_TYPE_INT, &rightv3); flttMakeOpNode(&opNode1, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft1, pRight1); nodesListAppend(list, opNode1); @@ -1111,7 +1091,7 @@ TEST(filterModelogicTest, same_column_and_or_and) { flttMakeValueNode(&pRight2, TSDB_DATA_TYPE_INT, &rightv4); flttMakeOpNode(&opNode2, OP_TYPE_GREATER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft1, pRight2); nodesListAppend(list, opNode2); - + flttMakeLogicNodeFromList(&logicNode2, LOGIC_COND_TYPE_AND, list); list = nodesMakeList(); @@ -1119,13 +1099,13 @@ TEST(filterModelogicTest, same_column_and_or_and) { nodesListAppend(list, logicNode2); flttMakeLogicNodeFromList(&logicNode1, LOGIC_COND_TYPE_OR, list); - + SFilterInfo *filter = NULL; int32_t code = filterInitFromNode(logicNode1, &filter, 0); ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; + SFilterColumnParam param = { (int32_t)taosArrayGetSize(src->pDataBlock), src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -1133,9 +1113,9 @@ TEST(filterModelogicTest, same_column_and_or_and) { stat.min = 1; stat.numOfNull = 0; int8_t *rowRes = NULL; - bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + bool keep = filterExecute(filter, src, &rowRes, &stat, taosArrayGetSize(src->pDataBlock)); ASSERT_EQ(keep, false); - + for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } @@ -1165,12 +1145,12 @@ TEST(filterModelogicTest, diff_columns_or_and_or) { flttMakeValueNode(&pRight2, TSDB_DATA_TYPE_INT, &rightv2); flttMakeOpNode(&opNode2, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft2, pRight2); nodesListAppend(list, opNode2); - + flttMakeLogicNodeFromList(&logicNode1, LOGIC_COND_TYPE_OR, list); list = nodesMakeList(); - + flttMakeColumnNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1); flttMakeValueNode(&pRight1, TSDB_DATA_TYPE_INT, &rightv1); flttMakeOpNode(&opNode1, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft1, pRight1); @@ -1180,7 +1160,7 @@ TEST(filterModelogicTest, diff_columns_or_and_or) { flttMakeValueNode(&pRight2, TSDB_DATA_TYPE_INT, &rightv2); flttMakeOpNode(&opNode2, OP_TYPE_GREATER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft2, pRight2); nodesListAppend(list, opNode2); - + flttMakeLogicNodeFromList(&logicNode2, LOGIC_COND_TYPE_OR, list); list = nodesMakeList(); @@ -1188,13 +1168,13 @@ TEST(filterModelogicTest, diff_columns_or_and_or) { nodesListAppend(list, logicNode2); flttMakeLogicNodeFromList(&logicNode1, LOGIC_COND_TYPE_AND, list); - + SFilterInfo *filter = NULL; int32_t code = filterInitFromNode(logicNode1, &filter, 0); ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; + SFilterColumnParam param = { (int32_t) taosArrayGetSize(src->pDataBlock), src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -1202,9 +1182,9 @@ TEST(filterModelogicTest, diff_columns_or_and_or) { stat.min = 1; stat.numOfNull = 0; int8_t *rowRes = NULL; - bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + bool keep = filterExecute(filter, src, &rowRes, &stat, taosArrayGetSize(src->pDataBlock)); ASSERT_EQ(keep, false); - + for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } @@ -1234,12 +1214,12 @@ TEST(filterModelogicTest, same_column_or_and_or) { flttMakeValueNode(&pRight2, TSDB_DATA_TYPE_INT, &rightv2); flttMakeOpNode(&opNode2, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft1, pRight2); nodesListAppend(list, opNode2); - + flttMakeLogicNodeFromList(&logicNode1, LOGIC_COND_TYPE_OR, list); list = nodesMakeList(); - + flttMakeColumnNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1); flttMakeValueNode(&pRight1, TSDB_DATA_TYPE_INT, &rightv3); flttMakeOpNode(&opNode1, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft1, pRight1); @@ -1249,7 +1229,7 @@ TEST(filterModelogicTest, same_column_or_and_or) { flttMakeValueNode(&pRight2, TSDB_DATA_TYPE_INT, &rightv4); flttMakeOpNode(&opNode2, OP_TYPE_GREATER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft1, pRight2); nodesListAppend(list, opNode2); - + flttMakeLogicNodeFromList(&logicNode2, LOGIC_COND_TYPE_OR, list); list = nodesMakeList(); @@ -1257,13 +1237,13 @@ TEST(filterModelogicTest, same_column_or_and_or) { nodesListAppend(list, logicNode2); flttMakeLogicNodeFromList(&logicNode1, LOGIC_COND_TYPE_AND, list); - + SFilterInfo *filter = NULL; int32_t code = filterInitFromNode(logicNode1, &filter, 0); ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; + SFilterColumnParam param = { (int32_t)taosArrayGetSize(src->pDataBlock), src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -1271,9 +1251,9 @@ TEST(filterModelogicTest, same_column_or_and_or) { stat.min = 1; stat.numOfNull = 0; int8_t *rowRes = NULL; - bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + bool keep = filterExecute(filter, src, &rowRes, &stat, taosArrayGetSize(src->pDataBlock)); ASSERT_EQ(keep, false); - + for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } @@ -1305,12 +1285,12 @@ TEST(scalarModelogicTest, diff_columns_or_and_or) { flttMakeColumnNode(&pRight2, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, rightv2); flttMakeOpNode(&opNode2, OP_TYPE_LOWER_THAN, TSDB_DATA_TYPE_BOOL, pLeft2, pRight2); nodesListAppend(list, opNode2); - + flttMakeLogicNodeFromList(&logicNode1, LOGIC_COND_TYPE_OR, list); list = nodesMakeList(); - + flttMakeColumnNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1); flttMakeColumnNode(&pRight1, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, rightv1); flttMakeOpNode(&opNode1, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pLeft1, pRight1); @@ -1320,7 +1300,7 @@ TEST(scalarModelogicTest, diff_columns_or_and_or) { flttMakeColumnNode(&pRight2, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, rightv2); flttMakeOpNode(&opNode2, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft2, pRight2); nodesListAppend(list, opNode2); - + flttMakeLogicNodeFromList(&logicNode2, LOGIC_COND_TYPE_OR, list); list = nodesMakeList(); @@ -1328,13 +1308,13 @@ TEST(scalarModelogicTest, diff_columns_or_and_or) { nodesListAppend(list, logicNode2); flttMakeLogicNodeFromList(&logicNode1, LOGIC_COND_TYPE_AND, list); - + SFilterInfo *filter = NULL; int32_t code = filterInitFromNode(logicNode1, &filter, 0); ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; + SFilterColumnParam param = { (int32_t)taosArrayGetSize(src->pDataBlock), src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -1342,7 +1322,7 @@ TEST(scalarModelogicTest, diff_columns_or_and_or) { stat.min = 1; stat.numOfNull = 0; int8_t *rowRes = NULL; - bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + bool keep = filterExecute(filter, src, &rowRes, &stat, taosArrayGetSize(src->pDataBlock)); ASSERT_EQ(keep, false); for (int32_t i = 0; i < rowNum; ++i) { diff --git a/source/libs/scalar/test/scalar/scalarTests.cpp b/source/libs/scalar/test/scalar/scalarTests.cpp index bfd1e60d26..07440e7435 100644 --- a/source/libs/scalar/test/scalar/scalarTests.cpp +++ b/source/libs/scalar/test/scalar/scalarTests.cpp @@ -84,30 +84,29 @@ void scltInitLogFile() { void scltAppendReservedSlot(SArray *pBlockList, int16_t *dataBlockId, int16_t *slotId, bool newBlock, int32_t rows, SColumnInfo *colInfo) { if (newBlock) { - SSDataBlock *res = (SSDataBlock *)taosMemoryCalloc(1, sizeof(SSDataBlock)); - res->info.numOfCols = 1; - res->info.rows = rows; - res->pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData)); + SSDataBlock *res = createDataBlock(); + SColumnInfoData idata = {0}; - idata.info = *colInfo; + idata.info = *colInfo; + colInfoDataEnsureCapacity(&idata, rows); + + blockDataAppendColInfo(res, &idata); + + res->info.capacity = rows; + res->info.rows = rows; + SColumnInfoData* p = static_cast(taosArrayGet(res->pDataBlock, 0)); + ASSERT(p->pData != NULL && p->nullbitmap != NULL); - taosArrayPush(res->pDataBlock, &idata); taosArrayPush(pBlockList, &res); - - blockDataEnsureCapacity(res, rows); - *dataBlockId = taosArrayGetSize(pBlockList) - 1; res->info.blockId = *dataBlockId; *slotId = 0; } else { SSDataBlock *res = *(SSDataBlock **)taosArrayGetLast(pBlockList); - res->info.numOfCols++; SColumnInfoData idata = {0}; idata.info = *colInfo; - - colInfoDataEnsureCapacity(&idata, 0, rows); - - taosArrayPush(res->pDataBlock, &idata); + colInfoDataEnsureCapacity(&idata, rows); + blockDataAppendColInfo(res, &idata); *dataBlockId = taosArrayGetSize(pBlockList) - 1; *slotId = taosArrayGetSize(res->pDataBlock) - 1; @@ -144,29 +143,19 @@ void scltMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in } if (NULL == *block) { - SSDataBlock *res = (SSDataBlock *)taosMemoryCalloc(1, sizeof(SSDataBlock)); - res->info.numOfCols = 3; - res->info.rows = rowNum; - res->pDataBlock = taosArrayInit(3, sizeof(SColumnInfoData)); + SSDataBlock *res = createDataBlock(); for (int32_t i = 0; i < 2; ++i) { - SColumnInfoData idata = {{0}}; - idata.info.type = TSDB_DATA_TYPE_NULL; - idata.info.bytes = 10; - idata.info.colId = i + 1; - - int32_t size = idata.info.bytes * rowNum; - idata.pData = (char *)taosMemoryCalloc(1, size); - taosArrayPush(res->pDataBlock, &idata); + SColumnInfoData idata = createColumnInfoData(TSDB_DATA_TYPE_INT, 10, i + 1); + colInfoDataEnsureCapacity(&idata, rowNum); + blockDataAppendColInfo(res, &idata); } - SColumnInfoData idata = {{0}}; - idata.info.type = dataType; - idata.info.bytes = dataBytes; - idata.info.colId = 3; - int32_t size = idata.info.bytes * rowNum; - idata.pData = (char *)taosMemoryCalloc(1, size); - colInfoDataEnsureCapacity(&idata, 0, rowNum); - taosArrayPush(res->pDataBlock, &idata); + SColumnInfoData idata = createColumnInfoData(dataType, dataBytes, 3); + colInfoDataEnsureCapacity(&idata, rowNum); + blockDataAppendColInfo(res, &idata); + res->info.capacity = rowNum; + + res->info.rows = rowNum; SColumnInfoData *pColumn = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); for (int32_t i = 0; i < rowNum; ++i) { colDataAppend(pColumn, i, (const char *)value, false); @@ -185,17 +174,13 @@ void scltMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in SSDataBlock *res = *block; int32_t idx = taosArrayGetSize(res->pDataBlock); - SColumnInfoData idata = {{0}}; - idata.info.type = dataType; - idata.info.bytes = dataBytes; - idata.info.colId = 1 + idx; - int32_t size = idata.info.bytes * rowNum; - idata.pData = (char *)taosMemoryCalloc(1, size); - taosArrayPush(res->pDataBlock, &idata); - res->info.numOfCols++; - SColumnInfoData *pColumn = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); + SColumnInfoData idata = createColumnInfoData(dataType, dataBytes, 1 + idx); + colInfoDataEnsureCapacity(&idata, rowNum); - colInfoDataEnsureCapacity(pColumn, 0, rowNum); + res->info.capacity = rowNum; + blockDataAppendColInfo(res, &idata); + + SColumnInfoData *pColumn = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); for (int32_t i = 0; i < rowNum; ++i) { colDataAppend(pColumn, i, (const char *)value, false); @@ -279,9 +264,6 @@ void scltMakeTargetNode(SNode **pNode, int16_t dataBlockId, int16_t slotId, SNod *pNode = (SNode *)onode; } - - - } TEST(constantTest, bigint_add_bigint) { @@ -1529,6 +1511,7 @@ TEST(columnTest, bigint_column_multi_binary_column) { SArray *blockList = taosArrayInit(1, POINTER_BYTES); taosArrayPush(blockList, &src); + SColumnInfo colInfo = createColumnInfo(1, TSDB_DATA_TYPE_DOUBLE, sizeof(double)); int16_t dataBlockId = 0, slotId = 0; scltAppendReservedSlot(blockList, &dataBlockId, &slotId, false, rowNum, &colInfo); @@ -2033,7 +2016,7 @@ void scltMakeDataBlock(SScalarParam **pInput, int32_t type, void *pVal, int32_t input->numOfRows = num; input->columnData->info = createColumnInfo(0, type, bytes); - colInfoDataEnsureCapacity(input->columnData, 0, num); + colInfoDataEnsureCapacity(input->columnData, num); if (setVal) { for (int32_t i = 0; i < num; ++i) { diff --git a/source/libs/stream/inc/streamInc.h b/source/libs/stream/inc/streamInc.h index 9654e21d24..2f41c08354 100644 --- a/source/libs/stream/inc/streamInc.h +++ b/source/libs/stream/inc/streamInc.h @@ -34,7 +34,7 @@ int32_t streamExec(SStreamTask* pTask, SMsgCb* pMsgCb); int32_t streamDispatch(SStreamTask* pTask, SMsgCb* pMsgCb); int32_t streamDispatchReqToData(const SStreamDispatchReq* pReq, SStreamDataBlock* pData); int32_t streamRetrieveReqToData(const SStreamRetrieveReq* pReq, SStreamDataBlock* pData); -int32_t streamBuildDispatchMsg(SStreamTask* pTask, SStreamDataBlock* data, SRpcMsg* pMsg, SEpSet** ppEpSet); +int32_t streamBuildDispatchMsg(SStreamTask* pTask, const SStreamDataBlock* data, SRpcMsg* pMsg, SEpSet** ppEpSet); int32_t streamBroadcastToChildren(SStreamTask* pTask, const SSDataBlock* pBlock); diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index 993fd3dbd7..11cd089606 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -96,16 +96,17 @@ int32_t streamBroadcastToChildren(SStreamTask* pTask, const SSDataBlock* pBlock) pRetrieve = taosMemoryCalloc(1, dataStrLen); if (pRetrieve == NULL) return -1; + int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); pRetrieve->useconds = 0; pRetrieve->precision = TSDB_DEFAULT_PRECISION; pRetrieve->compressed = 0; pRetrieve->completed = 1; pRetrieve->streamBlockType = pBlock->info.type; pRetrieve->numOfRows = htonl(pBlock->info.rows); - pRetrieve->numOfCols = htonl(pBlock->info.numOfCols); + pRetrieve->numOfCols = htonl(numOfCols); int32_t actualLen = 0; - blockCompressEncode(pBlock, pRetrieve->data, &actualLen, pBlock->info.numOfCols, false); + blockCompressEncode(pBlock, pRetrieve->data, &actualLen, numOfCols, false); SStreamRetrieveReq req = { .streamId = pTask->streamId, @@ -170,10 +171,12 @@ static int32_t streamAddBlockToDispatchMsg(const SSDataBlock* pBlock, SStreamDis pRetrieve->completed = 1; pRetrieve->streamBlockType = pBlock->info.type; pRetrieve->numOfRows = htonl(pBlock->info.rows); - pRetrieve->numOfCols = htonl(pBlock->info.numOfCols); + + int32_t numOfCols = (int32_t)taosArrayGetSize(pBlock->pDataBlock); + pRetrieve->numOfCols = htonl(numOfCols); int32_t actualLen = 0; - blockCompressEncode(pBlock, pRetrieve->data, &actualLen, pBlock->info.numOfCols, false); + blockCompressEncode(pBlock, pRetrieve->data, &actualLen, numOfCols, false); actualLen += sizeof(SRetrieveTableRsp); ASSERT(actualLen <= dataStrLen); taosArrayPush(pReq->dataLen, &actualLen); @@ -182,7 +185,7 @@ static int32_t streamAddBlockToDispatchMsg(const SSDataBlock* pBlock, SStreamDis return 0; } -int32_t streamBuildDispatchMsg(SStreamTask* pTask, SStreamDataBlock* data, SRpcMsg* pMsg, SEpSet** ppEpSet) { +int32_t streamBuildDispatchMsg(SStreamTask* pTask, const SStreamDataBlock* data, SRpcMsg* pMsg, SEpSet** ppEpSet) { void* buf = NULL; int32_t code = -1; int32_t blockNum = taosArrayGetSize(data->blocks); @@ -304,6 +307,8 @@ int32_t streamDispatch(SStreamTask* pTask, SMsgCb* pMsgCb) { atomic_store_8(&pTask->outputStatus, TASK_OUTPUT_STATUS__NORMAL); return -1; } + taosArrayDestroyEx(pBlock->blocks, (FDelete)tDeleteSSDataBlock); + taosFreeQitem(pBlock); tmsgSendReq(pEpSet, &dispatchMsg); return 0; diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index 6fada59c84..cb7dd241c1 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -53,9 +53,13 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, void* data, SArray* pRes) } // TODO: do we need free memory? - SSDataBlock* outputCopy = createOneDataBlock(output, true); - outputCopy->info.childId = pTask->selfChildId; - taosArrayPush(pRes, outputCopy); + SSDataBlock block = {0}; + assignOneDataBlock(&block, output); + block.info.childId = pTask->selfChildId; + taosArrayPush(pRes, &block); + /*SSDataBlock* outputCopy = createOneDataBlock(output, true);*/ + /*outputCopy->info.childId = pTask->selfChildId;*/ + /*taosArrayPush(pRes, outputCopy);*/ } return 0; } @@ -68,6 +72,7 @@ static SArray* streamExecForQall(SStreamTask* pTask, SArray* pRes) { streamTaskExecImpl(pTask, data, pRes); if (pTask->taskStatus == TASK_STATUS__DROPPING) { + taosArrayDestroyEx(pRes, (FDelete)tDeleteSSDataBlock); return NULL; } @@ -82,25 +87,25 @@ static SArray* streamExecForQall(SStreamTask* pTask, SArray* pRes) { qRes->blocks = pRes; if (streamTaskOutput(pTask, qRes) < 0) { streamQueueProcessFail(pTask->inputQueue); - taosArrayDestroy(pRes); + taosArrayDestroyEx(pRes, (FDelete)tDeleteSSDataBlock); taosFreeQitem(qRes); return NULL; } - - int8_t type = ((SStreamQueueItem*)data)->type; - if (type == STREAM_INPUT__TRIGGER) { - blockDataDestroy(((SStreamTrigger*)data)->pBlock); - taosFreeQitem(data); - } else if (type == STREAM_INPUT__DATA_BLOCK) { - taosArrayDestroyEx(((SStreamDataBlock*)data)->blocks, (FDelete)tDeleteSSDataBlock); - taosFreeQitem(data); - } else if (type == STREAM_INPUT__DATA_SUBMIT) { - ASSERT(pTask->isDataScan); - streamDataSubmitRefDec((SStreamDataSubmit*)data); - taosFreeQitem(data); - } streamQueueProcessSuccess(pTask->inputQueue); - return taosArrayInit(0, sizeof(SSDataBlock)); + pRes = taosArrayInit(0, sizeof(SSDataBlock)); + } + + int8_t type = ((SStreamQueueItem*)data)->type; + if (type == STREAM_INPUT__TRIGGER) { + blockDataDestroy(((SStreamTrigger*)data)->pBlock); + taosFreeQitem(data); + } else if (type == STREAM_INPUT__DATA_BLOCK) { + taosArrayDestroyEx(((SStreamDataBlock*)data)->blocks, (FDelete)tDeleteSSDataBlock); + taosFreeQitem(data); + } else if (type == STREAM_INPUT__DATA_SUBMIT) { + ASSERT(pTask->isDataScan); + streamDataSubmitRefDec((SStreamDataSubmit*)data); + taosFreeQitem(data); } } return pRes; @@ -125,14 +130,14 @@ int32_t streamExec(SStreamTask* pTask, SMsgCb* pMsgCb) { pRes = streamExecForQall(pTask, pRes); if (pRes == NULL) goto FAIL; - taosArrayDestroy(pRes); + taosArrayDestroyEx(pRes, (FDelete)tDeleteSSDataBlock); atomic_store_8(&pTask->execStatus, TASK_EXEC_STATUS__IDLE); return 0; } else if (execStatus == TASK_EXEC_STATUS__CLOSING) { continue; } else if (execStatus == TASK_EXEC_STATUS__EXECUTING) { ASSERT(taosArrayGetSize(pRes) == 0); - taosArrayDestroy(pRes); + taosArrayDestroyEx(pRes, (FDelete)tDeleteSSDataBlock); return 0; } else { ASSERT(0); diff --git a/source/libs/sync/inc/syncEnv.h b/source/libs/sync/inc/syncEnv.h index 4f91990ccf..2e8e0e1227 100644 --- a/source/libs/sync/inc/syncEnv.h +++ b/source/libs/sync/inc/syncEnv.h @@ -28,13 +28,13 @@ extern "C" { #include "trpc.h" #include "ttimer.h" -#define TIMER_MAX_MS 0x7FFFFFFF -#define ENV_TICK_TIMER_MS 1000 -#define PING_TIMER_MS 1000 -#define ELECT_TIMER_MS_MIN 500 -#define ELECT_TIMER_MS_MAX (ELECT_TIMER_MS_MIN * 2) +#define TIMER_MAX_MS 0x7FFFFFFF +#define ENV_TICK_TIMER_MS 1000 +#define PING_TIMER_MS 1000 +#define ELECT_TIMER_MS_MIN 1300 +#define ELECT_TIMER_MS_MAX (ELECT_TIMER_MS_MIN * 2) #define ELECT_TIMER_MS_RANGE (ELECT_TIMER_MS_MAX - ELECT_TIMER_MS_MIN) -#define HEARTBEAT_TIMER_MS 100 +#define HEARTBEAT_TIMER_MS 900 #define EMPTY_RAFT_ID ((SRaftId){.addr = 0, .vgId = 0}) diff --git a/source/libs/sync/inc/syncRaftLog.h b/source/libs/sync/inc/syncRaftLog.h index aec1f77b42..f124867097 100644 --- a/source/libs/sync/inc/syncRaftLog.h +++ b/source/libs/sync/inc/syncRaftLog.h @@ -26,11 +26,13 @@ extern "C" { #include "syncInt.h" #include "syncRaftEntry.h" #include "taosdef.h" +#include "wal.h" typedef struct SSyncLogStoreData { - SSyncNode* pSyncNode; - SWal* pWal; - SyncIndex beginIndex; // valid begin index, default 0, may be set beginIndex > 0 + SSyncNode* pSyncNode; + SWal* pWal; + SWalReadHandle* pWalHandle; + // SyncIndex beginIndex; // valid begin index, default 0, may be set beginIndex > 0 } SSyncLogStoreData; SSyncLogStore* logStoreCreate(SSyncNode* pSyncNode); diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c index bc5342cc7e..47d0976aca 100644 --- a/source/libs/sync/src/syncAppendEntries.c +++ b/source/libs/sync/src/syncAppendEntries.c @@ -420,44 +420,26 @@ static int32_t syncNodePreCommit(SSyncNode* ths, SSyncRaftEntry* pEntry) { // prevLogIndex == -1 static bool syncNodeOnAppendEntriesLogOK(SSyncNode* pSyncNode, SyncAppendEntries* pMsg) { if (pMsg->prevLogIndex == SYNC_INDEX_INVALID) { - if (gRaftDetailLog) { - sTrace("syncNodeOnAppendEntriesLogOK true, pMsg->prevLogIndex:%ld", pMsg->prevLogIndex); - } return true; } SyncIndex myLastIndex = syncNodeGetLastIndex(pSyncNode); if (pMsg->prevLogIndex > myLastIndex) { - if (gRaftDetailLog) { - sTrace("syncNodeOnAppendEntriesLogOK false, pMsg->prevLogIndex:%ld, myLastIndex:%ld", pMsg->prevLogIndex, - myLastIndex); - } + sDebug("vgId:%d sync log not ok, preindex:%ld", pSyncNode->vgId, pMsg->prevLogIndex); return false; } SyncTerm myPreLogTerm = syncNodeGetPreTerm(pSyncNode, pMsg->prevLogIndex + 1); if (myPreLogTerm == SYNC_TERM_INVALID) { - sError("vgId:%d sync get pre term error, preindex:%ld", pSyncNode->vgId, pMsg->prevLogIndex); + sDebug("vgId:%d sync log not ok2, preindex:%ld", pSyncNode->vgId, pMsg->prevLogIndex); return false; } if (pMsg->prevLogIndex <= myLastIndex && pMsg->prevLogTerm == myPreLogTerm) { - if (gRaftDetailLog) { - sTrace( - "syncNodeOnAppendEntriesLogOK true, pMsg->prevLogIndex:%ld, myLastIndex:%ld, pMsg->prevLogTerm:%lu, " - "myPreLogTerm:%lu", - pMsg->prevLogIndex, myLastIndex, pMsg->prevLogTerm, myPreLogTerm); - } return true; } - if (gRaftDetailLog) { - sTrace( - "syncNodeOnAppendEntriesLogOK false, pMsg->prevLogIndex:%ld, myLastIndex:%ld, pMsg->prevLogTerm:%lu, " - "myPreLogTerm:%lu", - pMsg->prevLogIndex, myLastIndex, pMsg->prevLogTerm, myPreLogTerm); - } - + sDebug("vgId:%d sync log not ok3, preindex:%ld", pSyncNode->vgId, pMsg->prevLogIndex); return false; } @@ -466,14 +448,11 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs int32_t code = 0; // print log - char logBuf[128] = {0}; - snprintf(logBuf, sizeof(logBuf), "recv SyncAppendEntries, vgId:%d, term:%lu", ths->vgId, - ths->pRaftStore->currentTerm); - syncAppendEntriesLog2(logBuf, pMsg); + syncAppendEntriesLog2("==syncNodeOnAppendEntriesSnapshotCb==", pMsg); // if already drop replica, do not process if (!syncNodeInRaftGroup(ths, &(pMsg->srcId)) && !ths->pRaftCfg->isStandBy) { - sInfo("recv SyncAppendEntries maybe replica already dropped"); + syncNodeEventLog(ths, "recv sync-append-entries, maybe replica already dropped"); return ret; } @@ -497,7 +476,7 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs do { bool condition = pMsg->term == ths->pRaftStore->currentTerm && ths->state == TAOS_SYNC_STATE_CANDIDATE; if (condition) { - sTrace("recv SyncAppendEntries, candidate to follower"); + syncNodeEventLog(ths, "recv sync-append-entries, candidate to follower"); syncNodeBecomeFollower(ths, "from candidate by append entries"); // do not reply? @@ -505,6 +484,7 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs } } while (0); +#if 0 // fake match // // condition1: @@ -530,7 +510,7 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs bool condition0 = (pMsg->term == ths->pRaftStore->currentTerm) && (ths->state == TAOS_SYNC_STATE_FOLLOWER) && syncNodeHasSnapshot(ths); bool condition1 = - condition0 && (ths->pLogStore->syncLogEntryCount(ths->pLogStore) == 0) && (pMsg->prevLogIndex > myLastIndex); + condition0 && (ths->pLogStore->syncLogEntryCount(ths->pLogStore) == 0) && (pMsg->prevLogIndex > myLastIndex); // donot use syncLogEntryCount!!! use isEmpty bool condition2 = condition0 && (ths->pLogStore->syncLogLastIndex(ths->pLogStore) <= snapshot.lastApplyIndex) && (pMsg->prevLogIndex > myLastIndex); bool condition3 = condition0 && (pMsg->prevLogIndex < snapshot.lastApplyIndex); @@ -538,11 +518,10 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs bool condition = condition1 || condition2 || condition3 || condition4; if (condition) { - sTrace( - "recv SyncAppendEntries, fake match, myLastIndex:%ld, syncLogBeginIndex:%ld, syncLogEndIndex:%ld, " - "condition1:%d, condition2:%d, condition3:%d, condition4:%d", - myLastIndex, ths->pLogStore->syncLogBeginIndex(ths->pLogStore), - ths->pLogStore->syncLogEndIndex(ths->pLogStore), condition1, condition2, condition3, condition4); + char logBuf[128]; + snprintf(logBuf, sizeof(logBuf), "recv sync-append-entries, fake match, pre-index:%ld, pre-term:%lu", + pMsg->prevLogIndex, pMsg->prevLogTerm); + syncNodeEventLog(ths, logBuf); // prepare response msg SyncAppendEntriesReply* pReply = syncAppendEntriesReplyBuild(ths->vgId); @@ -562,6 +541,7 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs return ret; } } while (0); +#endif // fake match2 // @@ -576,8 +556,13 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs bool condition = (pMsg->term == ths->pRaftStore->currentTerm) && (ths->state == TAOS_SYNC_STATE_FOLLOWER) && (pMsg->prevLogIndex <= ths->commitIndex); if (condition) { - sTrace("recv SyncAppendEntries, fake match2, msg-prevLogIndex:%ld, my-commitIndex:%ld", pMsg->prevLogIndex, - ths->commitIndex); + do { + char logBuf[128]; + snprintf(logBuf, sizeof(logBuf), + "recv sync-append-entries, fake match2, pre-index:%ld, pre-term:%lu, datalen:%d", pMsg->prevLogIndex, + pMsg->prevLogTerm, pMsg->dataLen); + syncNodeEventLog(ths, logBuf); + } while (0); SyncIndex matchIndex = ths->commitIndex; bool hasAppendEntries = pMsg->dataLen > 0; @@ -605,6 +590,7 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs code = syncNodePreCommit(ths, pAppendEntry); ASSERT(code == 0); + // update match index matchIndex = pMsg->prevLogIndex + 1; syncEntryDestory(pAppendEntry); @@ -650,11 +636,10 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs bool condition = condition1 || condition2; if (condition) { - sTrace( - "recv SyncAppendEntries, not match, syncLogBeginIndex:%ld, syncLogEndIndex:%ld, condition1:%d, " - "condition2:%d, logOK:%d", - ths->pLogStore->syncLogBeginIndex(ths->pLogStore), ths->pLogStore->syncLogEndIndex(ths->pLogStore), - condition1, condition2, logOK); + char logBuf[128]; + snprintf(logBuf, sizeof(logBuf), "recv sync-append-entries, not match, pre-index:%ld, pre-term:%lu, datalen:%d", + pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->dataLen); + syncNodeEventLog(ths, logBuf); // prepare response msg SyncAppendEntriesReply* pReply = syncAppendEntriesReplyBuild(ths->vgId); @@ -693,8 +678,10 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs // has entries in SyncAppendEntries msg bool hasAppendEntries = pMsg->dataLen > 0; - sTrace("recv SyncAppendEntries, match, myLastIndex:%ld, hasExtraEntries:%d, hasAppendEntries:%d", myLastIndex, - hasExtraEntries, hasAppendEntries); + char logBuf[128]; + snprintf(logBuf, sizeof(logBuf), "recv sync-append-entries, match, pre-index:%ld, pre-term:%lu, datalen:%d", + pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->dataLen); + syncNodeEventLog(ths, logBuf); if (hasExtraEntries) { // make log same, rollback deleted entries diff --git a/source/libs/sync/src/syncAppendEntriesReply.c b/source/libs/sync/src/syncAppendEntriesReply.c index f13a3604da..6ca356b4f6 100644 --- a/source/libs/sync/src/syncAppendEntriesReply.c +++ b/source/libs/sync/src/syncAppendEntriesReply.c @@ -101,32 +101,27 @@ int32_t syncNodeOnAppendEntriesReplySnapshotCb(SSyncNode* ths, SyncAppendEntries int32_t ret = 0; // print log - char logBuf[128] = {0}; - snprintf(logBuf, sizeof(logBuf), "recv SyncAppendEntriesReply, vgId:%d, term:%lu", ths->vgId, - ths->pRaftStore->currentTerm); - syncAppendEntriesReplyLog2(logBuf, pMsg); + syncAppendEntriesReplyLog2("==syncNodeOnAppendEntriesReplySnapshotCb==", pMsg); // if already drop replica, do not process if (!syncNodeInRaftGroup(ths, &(pMsg->srcId)) && !ths->pRaftCfg->isStandBy) { - sInfo("recv SyncAppendEntriesReply, maybe replica already dropped"); - return ret; + syncNodeEventLog(ths, "recv sync-append-entries-reply, maybe replica already dropped"); + return 0; } // drop stale response if (pMsg->term < ths->pRaftStore->currentTerm) { - sTrace("recv SyncAppendEntriesReply, drop stale response, receive_term:%lu current_term:%lu", pMsg->term, - ths->pRaftStore->currentTerm); - return ret; + char logBuf[128]; + snprintf(logBuf, sizeof(logBuf), "recv sync-append-entries-reply, recv-term:%lu, drop stale response", pMsg->term); + syncNodeEventLog(ths, logBuf); + return 0; } - syncIndexMgrLog2("recv SyncAppendEntriesReply, before pNextIndex:", ths->pNextIndex); - syncIndexMgrLog2("recv SyncAppendEntriesReply, before pMatchIndex:", ths->pMatchIndex); if (gRaftDetailLog) { - SSnapshot snapshot; - ths->pFsm->FpGetSnapshotInfo(ths->pFsm, &snapshot); - sTrace("recv SyncAppendEntriesReply, before snapshot.lastApplyIndex:%ld, snapshot.lastApplyTerm:%lu", - snapshot.lastApplyIndex, snapshot.lastApplyTerm); + syncNodeEventLog(ths, "recv sync-append-entries-reply, before"); } + syncIndexMgrLog2("recv sync-append-entries-reply, before pNextIndex:", ths->pNextIndex); + syncIndexMgrLog2("recv sync-append-entries-reply, before pMatchIndex:", ths->pMatchIndex); // no need this code, because if I receive reply.term, then I must have sent for that term. // if (pMsg->term > ths->pRaftStore->currentTerm) { @@ -134,12 +129,10 @@ int32_t syncNodeOnAppendEntriesReplySnapshotCb(SSyncNode* ths, SyncAppendEntries // } if (pMsg->term > ths->pRaftStore->currentTerm) { - char logBuf[128] = {0}; - snprintf(logBuf, sizeof(logBuf), "recv SyncAppendEntriesReply, error term, receive_term:%lu current_term:%lu", - pMsg->term, ths->pRaftStore->currentTerm); - syncNodeLog2(logBuf, ths); - sError("%s", logBuf); - return ret; + char logBuf[128]; + snprintf(logBuf, sizeof(logBuf), "recv sync-append-entries-reply, error term, recv-term:%lu", pMsg->term); + syncNodeErrorLog(ths, logBuf); + return -1; } ASSERT(pMsg->term == ths->pRaftStore->currentTerm); @@ -228,8 +221,11 @@ int32_t syncNodeOnAppendEntriesReplySnapshotCb(SSyncNode* ths, SyncAppendEntries } } - syncIndexMgrLog2("recv SyncAppendEntriesReply, after pNextIndex:", ths->pNextIndex); - syncIndexMgrLog2("recv SyncAppendEntriesReply, after pMatchIndex:", ths->pMatchIndex); + if (gRaftDetailLog) { + syncNodeEventLog(ths, "recv sync-append-entries-reply, after"); + } + syncIndexMgrLog2("recv sync-append-entries-reply, after pNextIndex:", ths->pNextIndex); + syncIndexMgrLog2("recv sync-append-entries-reply, after pMatchIndex:", ths->pMatchIndex); return ret; } \ No newline at end of file diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 732f450f4c..42ba2b85b1 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -416,7 +416,7 @@ int32_t syncGetSnapshotMetaByIndex(int64_t rid, SyncIndex snapshotIndex, struct } } sMeta->lastConfigIndex = lastIndex; - sTrace("vgId:%d, get snapshot meta by index:%" PRId64 " lastConfigIndex:%" PRId64, pSyncNode->vgId, snapshotIndex, + sTrace("vgId:%d, get snapshot meta by index:%" PRId64 " lcindex:%" PRId64, pSyncNode->vgId, snapshotIndex, sMeta->lastConfigIndex); taosReleaseRef(tsNodeRefId, pSyncNode->rid); @@ -433,8 +433,9 @@ SyncIndex syncNodeGetSnapshotConfigIndex(SSyncNode* pSyncNode, SyncIndex snapsho lastIndex = (pSyncNode->pRaftCfg->configIndexArr)[i]; } } + sTrace("vgId:%d, sync get snapshot last config index, index:%ld lcindex:%ld", pSyncNode->vgId, snapshotLastApplyIndex, + lastIndex); - sTrace("sync syncNodeGetSnapshotConfigIndex index:%ld lastConfigIndex:%ld", snapshotLastApplyIndex, lastIndex); return lastIndex; } @@ -1310,6 +1311,10 @@ void syncNodeEventLog(const SSyncNode* pSyncNode, char* str) { SyncIndex logBeginIndex = pSyncNode->pLogStore->syncLogBeginIndex(pSyncNode->pLogStore); char* pCfgStr = syncCfg2SimpleStr(&(pSyncNode->pRaftCfg->cfg)); + char* printStr = ""; + if (pCfgStr != NULL) { + printStr = pCfgStr; + } if (userStrLen < 256) { char logBuf[256 + 256]; @@ -1317,11 +1322,11 @@ void syncNodeEventLog(const SSyncNode* pSyncNode, char* str) { snprintf(logBuf, sizeof(logBuf), "vgId:%d, sync %s %s, term:%lu, commit:%ld, beginlog:%ld, lastlog:%ld, lastsnapshot:%ld, standby:%d, " "replica-num:%d, " - "lconfig:%ld, changing:%d, %s", + "lconfig:%ld, changing:%d, restore:%d, %s", pSyncNode->vgId, syncUtilState2String(pSyncNode->state), str, pSyncNode->pRaftStore->currentTerm, pSyncNode->commitIndex, logBeginIndex, logLastIndex, snapshot.lastApplyIndex, pSyncNode->pRaftCfg->isStandBy, pSyncNode->replicaNum, pSyncNode->pRaftCfg->lastConfigIndex, - pSyncNode->changing, pCfgStr); + pSyncNode->changing, pSyncNode->restoreFinish, printStr); } else { snprintf(logBuf, sizeof(logBuf), "%s", str); } @@ -1334,11 +1339,11 @@ void syncNodeEventLog(const SSyncNode* pSyncNode, char* str) { snprintf(s, len, "vgId:%d, sync %s %s, term:%lu, commit:%ld, beginlog:%ld, lastlog:%ld, lastsnapshot:%ld, standby:%d, " "replica-num:%d, " - "lconfig:%ld, changing:%d, %s", + "lconfig:%ld, changing:%d, restore:%d, %s", pSyncNode->vgId, syncUtilState2String(pSyncNode->state), str, pSyncNode->pRaftStore->currentTerm, pSyncNode->commitIndex, logBeginIndex, logLastIndex, snapshot.lastApplyIndex, pSyncNode->pRaftCfg->isStandBy, pSyncNode->replicaNum, pSyncNode->pRaftCfg->lastConfigIndex, - pSyncNode->changing, pCfgStr); + pSyncNode->changing, pSyncNode->restoreFinish, printStr); } else { snprintf(s, len, "%s", str); } @@ -1398,14 +1403,34 @@ void syncNodeErrorLog(const SSyncNode* pSyncNode, char* str) { char* syncNode2SimpleStr(const SSyncNode* pSyncNode) { int len = 256; char* s = (char*)taosMemoryMalloc(len); + + SSnapshot snapshot = {.data = NULL, .lastApplyIndex = -1, .lastApplyTerm = 0}; + if (pSyncNode->pFsm->FpGetSnapshotInfo != NULL) { + pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, &snapshot); + } + SyncIndex logLastIndex = pSyncNode->pLogStore->syncLogLastIndex(pSyncNode->pLogStore); + SyncIndex logBeginIndex = pSyncNode->pLogStore->syncLogBeginIndex(pSyncNode->pLogStore); + snprintf(s, len, - "syncNode: vgId:%d, currentTerm:%lu, commitIndex:%ld, state:%d %s, isStandBy:%d, " - "electTimerLogicClock:%lu, " - "electTimerLogicClockUser:%lu, " - "electTimerMS:%d, replicaNum:%d", - pSyncNode->vgId, pSyncNode->pRaftStore->currentTerm, pSyncNode->commitIndex, pSyncNode->state, - syncUtilState2String(pSyncNode->state), pSyncNode->pRaftCfg->isStandBy, pSyncNode->electTimerLogicClock, - pSyncNode->electTimerLogicClockUser, pSyncNode->electTimerMS, pSyncNode->replicaNum); + "vgId:%d, sync %s, term:%lu, commit:%ld, beginlog:%ld, lastlog:%ld, lastsnapshot:%ld, standby:%d, " + "replica-num:%d, " + "lconfig:%ld, changing:%d, restore:%d", + pSyncNode->vgId, syncUtilState2String(pSyncNode->state), pSyncNode->pRaftStore->currentTerm, + pSyncNode->commitIndex, logBeginIndex, logLastIndex, snapshot.lastApplyIndex, pSyncNode->pRaftCfg->isStandBy, + pSyncNode->replicaNum, pSyncNode->pRaftCfg->lastConfigIndex, pSyncNode->changing, pSyncNode->restoreFinish); + + /* + snprintf(s, len, + "syncNode: vgId:%d, term:%lu, commit:%ld, state:%d %s, standby:%d, " + "lc:%lu, " + "lc-user:%lu, " + "ems:%d, replica-num:%d, restore:%d, changing:%d", + pSyncNode->vgId, pSyncNode->pRaftStore->currentTerm, pSyncNode->commitIndex, pSyncNode->state, + syncUtilState2String(pSyncNode->state), pSyncNode->pRaftCfg->isStandBy, pSyncNode->electTimerLogicClock, + pSyncNode->electTimerLogicClockUser, pSyncNode->electTimerMS, pSyncNode->replicaNum, + pSyncNode->restoreFinish, pSyncNode->changing); + */ + return s; } @@ -1957,17 +1982,21 @@ SyncTerm syncNodeGetPreTerm(SSyncNode* pSyncNode, SyncIndex index) { taosMemoryFree(pPreEntry); return preTerm; } else { - if (terrno == TSDB_CODE_WAL_LOG_NOT_EXIST) { - SSnapshot snapshot = {.data = NULL, .lastApplyIndex = -1, .lastApplyTerm = 0, .lastConfigIndex = -1}; - if (pSyncNode->pFsm->FpGetSnapshotInfo != NULL) { - pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, &snapshot); - if (snapshot.lastApplyIndex == preIndex) { - return snapshot.lastApplyTerm; - } + SSnapshot snapshot = {.data = NULL, .lastApplyIndex = -1, .lastApplyTerm = 0, .lastConfigIndex = -1}; + if (pSyncNode->pFsm->FpGetSnapshotInfo != NULL) { + pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, &snapshot); + if (snapshot.lastApplyIndex == preIndex) { + return snapshot.lastApplyTerm; } } } + do { + char logBuf[128]; + snprintf(logBuf, sizeof(logBuf), "sync node get pre term error, index:%ld", index); + syncNodeErrorLog(pSyncNode, logBuf); + } while (0); + return SYNC_TERM_INVALID; } diff --git a/source/libs/sync/src/syncRaftCfg.c b/source/libs/sync/src/syncRaftCfg.c index 2cc1eb0239..f40eff57d9 100644 --- a/source/libs/sync/src/syncRaftCfg.c +++ b/source/libs/sync/src/syncRaftCfg.c @@ -107,26 +107,30 @@ char *syncCfg2Str(SSyncCfg *pSyncCfg) { } char *syncCfg2SimpleStr(SSyncCfg *pSyncCfg) { - int32_t len = 512; - char *s = taosMemoryMalloc(len); - memset(s, 0, len); + if (pSyncCfg != NULL) { + int32_t len = 512; + char *s = taosMemoryMalloc(len); + memset(s, 0, len); - snprintf(s, len, "{replica-num:%d, my-index:%d, ", pSyncCfg->replicaNum, pSyncCfg->myIndex); - char *p = s + strlen(s); - for (int i = 0; i < pSyncCfg->replicaNum; ++i) { - /* - if (p + 128 + 32 > s + len) { - break; + snprintf(s, len, "{replica-num:%d, my-index:%d, ", pSyncCfg->replicaNum, pSyncCfg->myIndex); + char *p = s + strlen(s); + for (int i = 0; i < pSyncCfg->replicaNum; ++i) { + /* + if (p + 128 + 32 > s + len) { + break; + } + */ + char buf[128 + 32]; + snprintf(buf, sizeof(buf), "%s:%d, ", pSyncCfg->nodeInfo[i].nodeFqdn, pSyncCfg->nodeInfo[i].nodePort); + strncpy(p, buf, sizeof(buf)); + p = s + strlen(s); } - */ - char buf[128 + 32]; - snprintf(buf, sizeof(buf), "%s:%d, ", pSyncCfg->nodeInfo[i].nodeFqdn, pSyncCfg->nodeInfo[i].nodePort); - strncpy(p, buf, sizeof(buf)); - p = s + strlen(s); - } - strcpy(p - 2, "}"); + strcpy(p - 2, "}"); - return s; + return s; + } + + return NULL; } int32_t syncCfgFromJson(const cJSON *pRoot, SSyncCfg *pSyncCfg) { diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index 701b4a7b93..cc606f4df8 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -16,10 +16,10 @@ #include "syncRaftLog.h" #include "syncRaftCfg.h" #include "syncRaftStore.h" -#include "wal.h" // refactor, log[0 .. n] ==> log[m .. n] -static int32_t raftLogSetBeginIndex(struct SSyncLogStore* pLogStore, SyncIndex beginIndex); +static int32_t raftLogRestoreFromSnapshot(struct SSyncLogStore* pLogStore, SyncIndex snapshotIndex); +// static int32_t raftLogSetBeginIndex(struct SSyncLogStore* pLogStore, SyncIndex beginIndex); static SyncIndex raftLogBeginIndex(struct SSyncLogStore* pLogStore); static SyncIndex raftLogEndIndex(struct SSyncLogStore* pLogStore); static SyncIndex raftLogWriteIndex(struct SSyncLogStore* pLogStore); @@ -45,31 +45,43 @@ static int32_t logStoreUpdateCommitIndex(SSyncLogStore* pLogStore, SyncI static SyncIndex logStoreGetCommitIndex(SSyncLogStore* pLogStore); // refactor, log[0 .. n] ==> log[m .. n] +/* static int32_t raftLogSetBeginIndex(struct SSyncLogStore* pLogStore, SyncIndex beginIndex) { - sTrace("raftLogSetBeginIndex beginIndex:%ld", beginIndex); - // if beginIndex == 0, donot need call this funciton ASSERT(beginIndex > 0); SSyncLogStoreData* pData = pLogStore->data; SWal* pWal = pData->pWal; + sTrace("vgId:%d, reset wal begin index:%ld", pData->pSyncNode->vgId, beginIndex); + pData->beginIndex = beginIndex; walRestoreFromSnapshot(pWal, beginIndex - 1); return 0; } +*/ + +static int32_t raftLogRestoreFromSnapshot(struct SSyncLogStore* pLogStore, SyncIndex snapshotIndex) { + ASSERT(snapshotIndex >= 0); + + SSyncLogStoreData* pData = pLogStore->data; + SWal* pWal = pData->pWal; + walRestoreFromSnapshot(pWal, snapshotIndex); + return 0; +} static SyncIndex raftLogBeginIndex(struct SSyncLogStore* pLogStore) { SSyncLogStoreData* pData = pLogStore->data; SWal* pWal = pData->pWal; - return pData->beginIndex; + SyncIndex firstVer = walGetFirstVer(pWal); + return firstVer; } static SyncIndex raftLogEndIndex(struct SSyncLogStore* pLogStore) { return raftLogLastIndex(pLogStore); } static bool raftLogIsEmpty(struct SSyncLogStore* pLogStore) { - SyncIndex beginIndex = raftLogBeginIndex(pLogStore); - SyncIndex endIndex = raftLogEndIndex(pLogStore); - return (endIndex < beginIndex); + SSyncLogStoreData* pData = pLogStore->data; + SWal* pWal = pData->pWal; + return walIsEmpty(pWal); } static int32_t raftLogEntryCount(struct SSyncLogStore* pLogStore) { @@ -96,23 +108,8 @@ static SyncIndex raftLogLastIndex(struct SSyncLogStore* pLogStore) { SSyncLogStoreData* pData = pLogStore->data; SWal* pWal = pData->pWal; SyncIndex lastVer = walGetLastVer(pWal); - SyncIndex firstVer = walGetFirstVer(pWal); - if (lastVer < firstVer) { - // no record - lastIndex = -1; - - } else { - if (firstVer >= 0) { - lastIndex = lastVer; - } else if (firstVer == -1) { - lastIndex = -1; - } else { - ASSERT(0); - } - } - - return lastIndex; + return lastVer; } static SyncIndex raftLogWriteIndex(struct SSyncLogStore* pLogStore) { @@ -122,6 +119,26 @@ static SyncIndex raftLogWriteIndex(struct SSyncLogStore* pLogStore) { return lastVer + 1; } +static SyncTerm raftLogLastTerm(struct SSyncLogStore* pLogStore) { + SSyncLogStoreData* pData = pLogStore->data; + SWal* pWal = pData->pWal; + if (walIsEmpty(pWal)) { + return 0; + } else { + SSyncRaftEntry* pLastEntry; + int32_t code = raftLogGetLastEntry(pLogStore, &pLastEntry); + ASSERT(code == 0); + ASSERT(pLastEntry != NULL); + + SyncTerm lastTerm = pLastEntry->term; + taosMemoryFree(pLastEntry); + return lastTerm; + } + + return 0; +} + +/* static SyncTerm raftLogLastTerm(struct SSyncLogStore* pLogStore) { SyncTerm lastTerm = 0; if (raftLogEntryCount(pLogStore) == 0) { @@ -137,6 +154,7 @@ static SyncTerm raftLogLastTerm(struct SSyncLogStore* pLogStore) { } return lastTerm; } +*/ static int32_t raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry) { SSyncLogStoreData* pData = pLogStore->data; @@ -160,8 +178,12 @@ static int32_t raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncRaftEntr const char* errStr = tstrerror(err); int32_t sysErr = errno; const char* sysErrStr = strerror(errno); - sError("vgId:%d wal write error, index:%ld, err:%d %X, msg:%s, syserr:%d, sysmsg:%s", pData->pSyncNode->vgId, - pEntry->index, err, err, errStr, sysErr, sysErrStr); + + char logBuf[128]; + snprintf(logBuf, sizeof(logBuf), "wal write error, index:%ld, err:%d %X, msg:%s, syserr:%d, sysmsg:%s", + pEntry->index, err, err, errStr, sysErr, sysErrStr); + syncNodeErrorLog(pData->pSyncNode, logBuf); + ASSERT(0); } @@ -229,7 +251,8 @@ static int32_t raftLogGetEntry(struct SSyncLogStore* pLogStore, SyncIndex index, *ppEntry = NULL; - SWalReadHandle* pWalHandle = walOpenReadHandle(pWal); + // SWalReadHandle* pWalHandle = walOpenReadHandle(pWal); + SWalReadHandle* pWalHandle = pData->pWalHandle; if (pWalHandle == NULL) { return -1; } @@ -240,12 +263,23 @@ static int32_t raftLogGetEntry(struct SSyncLogStore* pLogStore, SyncIndex index, const char* errStr = tstrerror(err); int32_t sysErr = errno; const char* sysErrStr = strerror(errno); - sError("vgId:%d wal read error, index:%ld, err:%d %X, msg:%s, syserr:%d, sysmsg:%s", pData->pSyncNode->vgId, index, - err, err, errStr, sysErr, sysErrStr); - int32_t saveErr = terrno; - walCloseReadHandle(pWalHandle); - terrno = saveErr; + do { + char logBuf[128]; + snprintf(logBuf, sizeof(logBuf), "wal read error, index:%ld, err:%d %X, msg:%s, syserr:%d, sysmsg:%s", index, err, + err, errStr, sysErr, sysErrStr); + if (terrno == TSDB_CODE_WAL_LOG_NOT_EXIST) { + syncNodeEventLog(pData->pSyncNode, logBuf); + } else { + syncNodeErrorLog(pData->pSyncNode, logBuf); + } + } while (0); + + /* + int32_t saveErr = terrno; + walCloseReadHandle(pWalHandle); + terrno = saveErr; + */ return code; } @@ -261,9 +295,11 @@ static int32_t raftLogGetEntry(struct SSyncLogStore* pLogStore, SyncIndex index, ASSERT((*ppEntry)->dataLen == pWalHandle->pHead->head.bodyLen); memcpy((*ppEntry)->data, pWalHandle->pHead->head.body, pWalHandle->pHead->head.bodyLen); - int32_t saveErr = terrno; - walCloseReadHandle(pWalHandle); - terrno = saveErr; + /* + int32_t saveErr = terrno; + walCloseReadHandle(pWalHandle); + terrno = saveErr; + */ return code; } @@ -285,6 +321,25 @@ static int32_t raftLogTruncate(struct SSyncLogStore* pLogStore, SyncIndex fromIn return code; } +static int32_t raftLogGetLastEntry(SSyncLogStore* pLogStore, SSyncRaftEntry** ppLastEntry) { + SSyncLogStoreData* pData = pLogStore->data; + SWal* pWal = pData->pWal; + ASSERT(ppLastEntry != NULL); + + *ppLastEntry = NULL; + if (walIsEmpty(pWal)) { + terrno = TSDB_CODE_WAL_LOG_NOT_EXIST; + return -1; + } else { + SyncIndex lastIndex = raftLogLastIndex(pLogStore); + int32_t code = raftLogGetEntry(pLogStore, lastIndex, ppLastEntry); + return code; + } + + return -1; +} + +/* static int32_t raftLogGetLastEntry(SSyncLogStore* pLogStore, SSyncRaftEntry** ppLastEntry) { *ppLastEntry = NULL; if (raftLogEntryCount(pLogStore) == 0) { @@ -294,6 +349,7 @@ static int32_t raftLogGetLastEntry(SSyncLogStore* pLogStore, SSyncRaftEntry** pp int32_t code = raftLogGetEntry(pLogStore, lastIndex, ppLastEntry); return code; } +*/ //------------------------------- SSyncLogStore* logStoreCreate(SSyncNode* pSyncNode) { @@ -306,16 +362,22 @@ SSyncLogStore* logStoreCreate(SSyncNode* pSyncNode) { SSyncLogStoreData* pData = pLogStore->data; pData->pSyncNode = pSyncNode; pData->pWal = pSyncNode->pWal; + ASSERT(pData->pWal != NULL); - SyncIndex firstVer = walGetFirstVer(pData->pWal); - SyncIndex lastVer = walGetLastVer(pData->pWal); - if (firstVer >= 0) { - pData->beginIndex = firstVer; - } else if (firstVer == -1) { - pData->beginIndex = lastVer + 1; - } else { - ASSERT(0); - } + pData->pWalHandle = walOpenReadHandle(pData->pWal); + ASSERT(pData->pWalHandle != NULL); + + /* + SyncIndex firstVer = walGetFirstVer(pData->pWal); + SyncIndex lastVer = walGetLastVer(pData->pWal); + if (firstVer >= 0) { + pData->beginIndex = firstVer; + } else if (firstVer == -1) { + pData->beginIndex = lastVer + 1; + } else { + ASSERT(0); + } + */ pLogStore->appendEntry = logStoreAppendEntry; pLogStore->getEntry = logStoreGetEntry; @@ -325,7 +387,8 @@ SSyncLogStore* logStoreCreate(SSyncNode* pSyncNode) { pLogStore->updateCommitIndex = logStoreUpdateCommitIndex; pLogStore->getCommitIndex = logStoreGetCommitIndex; - pLogStore->syncLogSetBeginIndex = raftLogSetBeginIndex; + // pLogStore->syncLogSetBeginIndex = raftLogSetBeginIndex; + pLogStore->syncLogRestoreFromSnapshot = raftLogRestoreFromSnapshot; pLogStore->syncLogBeginIndex = raftLogBeginIndex; pLogStore->syncLogEndIndex = raftLogEndIndex; pLogStore->syncLogIsEmpty = raftLogIsEmpty; @@ -344,6 +407,11 @@ SSyncLogStore* logStoreCreate(SSyncNode* pSyncNode) { void logStoreDestory(SSyncLogStore* pLogStore) { if (pLogStore != NULL) { + SSyncLogStoreData* pData = pLogStore->data; + if (pData->pWalHandle != NULL) { + walCloseReadHandle(pData->pWalHandle); + } + taosMemoryFree(pLogStore->data); taosMemoryFree(pLogStore); } @@ -368,8 +436,11 @@ int32_t logStoreAppendEntry(SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry) { const char* errStr = tstrerror(err); int32_t sysErr = errno; const char* sysErrStr = strerror(errno); - sError("vgId:%d wal write error, index:%ld, err:%d %X, msg:%s, syserr:%d, sysmsg:%s", pData->pSyncNode->vgId, - pEntry->index, err, err, errStr, sysErr, sysErrStr); + + char logBuf[128]; + snprintf(logBuf, sizeof(logBuf), "wal write error, index:%ld, err:%d %X, msg:%s, syserr:%d, sysmsg:%s", + pEntry->index, err, err, errStr, sysErr, sysErrStr); + syncNodeErrorLog(pData->pSyncNode, logBuf); ASSERT(0); } @@ -389,7 +460,8 @@ SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) { SWal* pWal = pData->pWal; if (index >= SYNC_INDEX_BEGIN && index <= logStoreLastIndex(pLogStore)) { - SWalReadHandle* pWalHandle = walOpenReadHandle(pWal); + // SWalReadHandle* pWalHandle = walOpenReadHandle(pWal); + SWalReadHandle* pWalHandle = pData->pWalHandle; ASSERT(pWalHandle != NULL); int32_t code = walReadWithHandle(pWalHandle, index); @@ -398,12 +470,20 @@ SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) { const char* errStr = tstrerror(err); int32_t sysErr = errno; const char* sysErrStr = strerror(errno); - sError("vgId:%d wal read error, index:%ld, err:%d %X, msg:%s, syserr:%d, sysmsg:%s", pData->pSyncNode->vgId, - index, err, err, errStr, sysErr, sysErrStr); + + do { + char logBuf[128]; + snprintf(logBuf, sizeof(logBuf), "wal read error, index:%ld, err:%d %X, msg:%s, syserr:%d, sysmsg:%s", index, + err, err, errStr, sysErr, sysErrStr); + if (terrno == TSDB_CODE_WAL_LOG_NOT_EXIST) { + syncNodeEventLog(pData->pSyncNode, logBuf); + } else { + syncNodeErrorLog(pData->pSyncNode, logBuf); + } + } while (0); ASSERT(0); } - // ASSERT(walReadWithHandle(pWalHandle, index) == 0); SSyncRaftEntry* pEntry = syncEntryBuild(pWalHandle->pHead->head.bodyLen); ASSERT(pEntry != NULL); @@ -417,9 +497,11 @@ SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) { ASSERT(pEntry->dataLen == pWalHandle->pHead->head.bodyLen); memcpy(pEntry->data, pWalHandle->pHead->head.body, pWalHandle->pHead->head.bodyLen); - int32_t saveErr = terrno; - walCloseReadHandle(pWalHandle); - terrno = saveErr; + /* + int32_t saveErr = terrno; + walCloseReadHandle(pWalHandle); + terrno = saveErr; + */ return pEntry; @@ -498,6 +580,7 @@ SSyncRaftEntry* logStoreGetLastEntry(SSyncLogStore* pLogStore) { return pEntry; } +/* cJSON* logStore2Json(SSyncLogStore* pLogStore) { char u64buf[128] = {0}; SSyncLogStoreData* pData = (SSyncLogStoreData*)pLogStore->data; @@ -544,6 +627,57 @@ cJSON* logStore2Json(SSyncLogStore* pLogStore) { cJSON_AddItemToObject(pJson, "SSyncLogStore", pRoot); return pJson; } +*/ + +cJSON* logStore2Json(SSyncLogStore* pLogStore) { + char u64buf[128] = {0}; + SSyncLogStoreData* pData = (SSyncLogStoreData*)pLogStore->data; + cJSON* pRoot = cJSON_CreateObject(); + + if (pData != NULL && pData->pWal != NULL) { + snprintf(u64buf, sizeof(u64buf), "%p", pData->pSyncNode); + cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); + snprintf(u64buf, sizeof(u64buf), "%p", pData->pWal); + cJSON_AddStringToObject(pRoot, "pWal", u64buf); + + SyncIndex beginIndex = raftLogBeginIndex(pLogStore); + snprintf(u64buf, sizeof(u64buf), "%ld", beginIndex); + cJSON_AddStringToObject(pRoot, "beginIndex", u64buf); + + SyncIndex endIndex = raftLogEndIndex(pLogStore); + snprintf(u64buf, sizeof(u64buf), "%ld", endIndex); + cJSON_AddStringToObject(pRoot, "endIndex", u64buf); + + int32_t count = raftLogEntryCount(pLogStore); + cJSON_AddNumberToObject(pRoot, "entryCount", count); + + snprintf(u64buf, sizeof(u64buf), "%ld", raftLogWriteIndex(pLogStore)); + cJSON_AddStringToObject(pRoot, "WriteIndex", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%d", raftLogIsEmpty(pLogStore)); + cJSON_AddStringToObject(pRoot, "IsEmpty", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%ld", raftLogLastIndex(pLogStore)); + cJSON_AddStringToObject(pRoot, "LastIndex", u64buf); + snprintf(u64buf, sizeof(u64buf), "%lu", raftLogLastTerm(pLogStore)); + cJSON_AddStringToObject(pRoot, "LastTerm", u64buf); + + cJSON* pEntries = cJSON_CreateArray(); + cJSON_AddItemToObject(pRoot, "pEntries", pEntries); + + if (!raftLogIsEmpty(pLogStore)) { + for (SyncIndex i = beginIndex; i <= endIndex; ++i) { + SSyncRaftEntry* pEntry = logStoreGetEntry(pLogStore, i); + cJSON_AddItemToArray(pEntries, syncEntry2Json(pEntry)); + syncEntryDestory(pEntry); + } + } + } + + cJSON* pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "SSyncLogStore", pRoot); + return pJson; +} char* logStore2Str(SSyncLogStore* pLogStore) { cJSON* pJson = logStore2Json(pLogStore); @@ -563,7 +697,8 @@ cJSON* logStoreSimple2Json(SSyncLogStore* pLogStore) { snprintf(u64buf, sizeof(u64buf), "%p", pData->pWal); cJSON_AddStringToObject(pRoot, "pWal", u64buf); - snprintf(u64buf, sizeof(u64buf), "%ld", pData->beginIndex); + SyncIndex beginIndex = raftLogBeginIndex(pLogStore); + snprintf(u64buf, sizeof(u64buf), "%ld", beginIndex); cJSON_AddStringToObject(pRoot, "beginIndex", u64buf); SyncIndex endIndex = raftLogEndIndex(pLogStore); diff --git a/source/libs/sync/src/syncReplication.c b/source/libs/sync/src/syncReplication.c index c378926e28..02cd977e1e 100644 --- a/source/libs/sync/src/syncReplication.c +++ b/source/libs/sync/src/syncReplication.c @@ -219,6 +219,17 @@ int32_t syncNodeReplicate(SSyncNode* pSyncNode) { int32_t syncNodeAppendEntries(SSyncNode* pSyncNode, const SRaftId* destRaftId, const SyncAppendEntries* pMsg) { int32_t ret = 0; + do { + char host[128]; + uint16_t port; + syncUtilU642Addr(destRaftId->addr, host, sizeof(host), &port); + sDebug( + "vgId:%d, send sync-append-entries to %s:%d, term:%lu, pre-index:%ld, pre-term:%lu, pterm:%lu, commit:%ld, " + "datalen:%d", + pSyncNode->vgId, host, port, pMsg->term, pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->privateTerm, + pMsg->commitIndex, pMsg->dataLen); + } while (0); + SRpcMsg rpcMsg; syncAppendEntries2RpcMsg(pMsg, &rpcMsg); syncNodeSendMsgById(destRaftId, pSyncNode, &rpcMsg); diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index ed9d62fd14..4fc0128400 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -545,7 +545,8 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { pReceiver->pSyncNode->commitIndex = pReceiver->snapshot.lastApplyIndex; } - pSyncNode->pLogStore->syncLogSetBeginIndex(pSyncNode->pLogStore, pMsg->lastIndex + 1); + // pSyncNode->pLogStore->syncLogSetBeginIndex(pSyncNode->pLogStore, pMsg->lastIndex + 1); + pSyncNode->pLogStore->syncLogRestoreFromSnapshot(pSyncNode->pLogStore, pMsg->lastIndex); // maybe update lastconfig if (pMsg->lastConfigIndex >= SYNC_INDEX_BEGIN) { diff --git a/source/libs/sync/test/syncRaftLogTest2.cpp b/source/libs/sync/test/syncRaftLogTest2.cpp index bd6fa7c2c3..e3ceae0615 100644 --- a/source/libs/sync/test/syncRaftLogTest2.cpp +++ b/source/libs/sync/test/syncRaftLogTest2.cpp @@ -113,7 +113,8 @@ void test2() { pLogStore = logStoreCreate(pSyncNode); assert(pLogStore); pSyncNode->pLogStore = pLogStore; - pLogStore->syncLogSetBeginIndex(pLogStore, 5); + //pLogStore->syncLogSetBeginIndex(pLogStore, 5); + pLogStore->syncLogRestoreFromSnapshot(pLogStore, 4); logStoreLog2((char*)"\n\n\ntest2 ----- ", pLogStore); if (gAssert) { @@ -228,7 +229,8 @@ void test4() { assert(pLogStore); pSyncNode->pLogStore = pLogStore; logStoreLog2((char*)"\n\n\ntest4 ----- ", pLogStore); - pLogStore->syncLogSetBeginIndex(pLogStore, 5); + //pLogStore->syncLogSetBeginIndex(pLogStore, 5); + pLogStore->syncLogRestoreFromSnapshot(pLogStore, 4); for (int i = 5; i <= 9; ++i) { int32_t dataLen = 10; @@ -289,7 +291,8 @@ void test5() { assert(pLogStore); pSyncNode->pLogStore = pLogStore; logStoreLog2((char*)"\n\n\ntest5 ----- ", pLogStore); - pLogStore->syncLogSetBeginIndex(pLogStore, 5); + //pLogStore->syncLogSetBeginIndex(pLogStore, 5); + pLogStore->syncLogRestoreFromSnapshot(pLogStore, 4); for (int i = 5; i <= 9; ++i) { int32_t dataLen = 10; @@ -363,7 +366,8 @@ void test6() { assert(pLogStore); pSyncNode->pLogStore = pLogStore; logStoreLog2((char*)"\n\n\ntest6 ----- ", pLogStore); - pLogStore->syncLogSetBeginIndex(pLogStore, 5); + // pLogStore->syncLogSetBeginIndex(pLogStore, 5); + pLogStore->syncLogRestoreFromSnapshot(pLogStore, 4); for (int i = 5; i <= 9; ++i) { int32_t dataLen = 10; @@ -405,14 +409,32 @@ void test6() { assert(pLogStore->syncLogLastTerm(pLogStore) == 0); } + do { + SyncIndex firstVer = walGetFirstVer(pWal); + SyncIndex lastVer = walGetLastVer(pWal); + bool isEmpty = walIsEmpty(pWal); + printf("before -------- firstVer:%ld lastVer:%ld isEmpty:%d \n", firstVer, lastVer, isEmpty); + } while (0); + logStoreDestory(pLogStore); cleanup(); + + // restart init(); pLogStore = logStoreCreate(pSyncNode); assert(pLogStore); pSyncNode->pLogStore = pLogStore; + + + do { + SyncIndex firstVer = walGetFirstVer(pWal); + SyncIndex lastVer = walGetLastVer(pWal); + bool isEmpty = walIsEmpty(pWal); + printf("after -------- firstVer:%ld lastVer:%ld isEmpty:%d \n", firstVer, lastVer, isEmpty); + } while (0); + logStoreLog2((char*)"\n\n\ntest6 restart ----- ", pLogStore); if (gAssert) { @@ -432,17 +454,20 @@ void test6() { int main(int argc, char** argv) { tsAsyncLog = 0; sDebugFlag = DEBUG_TRACE + DEBUG_INFO + DEBUG_SCREEN + DEBUG_FILE; + gRaftDetailLog = true; if (argc == 2) { gAssert = atoi(argv[1]); } sTrace("gAssert : %d", gAssert); +/* test1(); test2(); test3(); test4(); test5(); +*/ test6(); return 0; diff --git a/source/libs/sync/test/syncRaftLogTest3.cpp b/source/libs/sync/test/syncRaftLogTest3.cpp index c81a6d3ca9..302e29a091 100644 --- a/source/libs/sync/test/syncRaftLogTest3.cpp +++ b/source/libs/sync/test/syncRaftLogTest3.cpp @@ -312,7 +312,8 @@ void test5() { pSyncNode->pLogStore = pLogStore; logStoreLog2((char*)"\n\n\ntest5 ----- ", pLogStore); - pSyncNode->pLogStore->syncLogSetBeginIndex(pSyncNode->pLogStore, 6); + //pSyncNode->pLogStore->syncLogSetBeginIndex(pSyncNode->pLogStore, 6); + pLogStore->syncLogRestoreFromSnapshot(pSyncNode->pLogStore, 5); for (int i = 6; i <= 10; ++i) { int32_t dataLen = 10; SSyncRaftEntry* pEntry = syncEntryBuild(dataLen); @@ -372,6 +373,7 @@ void test5() { int main(int argc, char** argv) { tsAsyncLog = 0; sDebugFlag = DEBUG_TRACE + DEBUG_INFO + DEBUG_SCREEN + DEBUG_FILE; + gRaftDetailLog = true; if (argc == 2) { gAssert = atoi(argv[1]); diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index bf03ea26ba..0dd00e0dc2 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -983,17 +983,18 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { if (pResp->code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { if (pCtx->retryCount < pEpSet->numOfEps * 3) { pEpSet->inUse = (++pEpSet->inUse) % pEpSet->numOfEps; + if (pThrd->quit == false) { + STaskArg* arg = taosMemoryMalloc(sizeof(STaskArg)); + arg->param1 = pMsg; + arg->param2 = pThrd; + transDQSched(pThrd->delayQueue, doDelayTask, arg, TRANS_RETRY_INTERVAL); + transPrintEpSet(pEpSet); + tTrace("%s use local epset, inUse: %d, retry count:%d, limit: %d", pTransInst->label, pEpSet->inUse, + pCtx->retryCount + 1, pEpSet->numOfEps * 3); - STaskArg* arg = taosMemoryMalloc(sizeof(STaskArg)); - arg->param1 = pMsg; - arg->param2 = pThrd; - transDQSched(pThrd->delayQueue, doDelayTask, arg, TRANS_RETRY_INTERVAL); - transPrintEpSet(pEpSet); - tTrace("%s use local epset, inUse: %d, retry count:%d, limit: %d", pTransInst->label, pEpSet->inUse, - pCtx->retryCount + 1, pEpSet->numOfEps * 3); - - transUnrefCliHandle(pConn); - return -1; + transUnrefCliHandle(pConn); + return -1; + } } } else if (pCtx->retryCount < TRANS_RETRY_COUNT_LIMIT) { if (pResp->contLen == 0) { @@ -1010,15 +1011,16 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { tTrace("%s use remote epset, inUse: %d, retry count:%d, limit: %d", pTransInst->label, pEpSet->inUse, pCtx->retryCount + 1, TRANS_RETRY_COUNT_LIMIT); } - if (pConn->status != ConnInPool) { - addConnToPool(pThrd->pool, pConn); + if (pThrd->quit == false) { + if (pConn->status != ConnInPool) { + addConnToPool(pThrd->pool, pConn); + } + STaskArg* arg = taosMemoryMalloc(sizeof(STaskArg)); + arg->param1 = pMsg; + arg->param2 = pThrd; + transDQSched(pThrd->delayQueue, doDelayTask, arg, TRANS_RETRY_INTERVAL); + return -1; } - - STaskArg* arg = taosMemoryMalloc(sizeof(STaskArg)); - arg->param1 = pMsg; - arg->param2 = pThrd; - transDQSched(pThrd->delayQueue, doDelayTask, arg, TRANS_RETRY_INTERVAL); - return -1; } } diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index 9bed6d230b..682afbb785 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -272,14 +272,23 @@ int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver) { } } + if (ver > pRead->pWal->vers.lastVer || ver < pRead->pWal->vers.firstVer) { + wError("invalid version: % " PRId64 ", first ver %ld, last ver %ld", ver, pRead->pWal->vers.firstVer, + pRead->pWal->vers.lastVer); + terrno = TSDB_CODE_WAL_LOG_NOT_EXIST; + return -1; + } + ASSERT(taosValidFile(pRead->pReadLogTFile) == true); code = taosReadFile(pRead->pReadLogTFile, pRead->pHead, sizeof(SWalHead)); if (code != sizeof(SWalHead)) { if (code < 0) terrno = TAOS_SYSTEM_ERROR(errno); - else + else { terrno = TSDB_CODE_WAL_FILE_CORRUPTED; + ASSERT(0); + } return -1; } @@ -304,8 +313,10 @@ int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver) { pRead->pHead->head.bodyLen) { if (code < 0) terrno = TAOS_SYSTEM_ERROR(errno); - else + else { terrno = TSDB_CODE_WAL_FILE_CORRUPTED; + ASSERT(0); + } return -1; } diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index 031289f93e..f82f27e3af 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -150,6 +150,7 @@ int32_t walRollback(SWal *pWal, int64_t ver) { ASSERT(code == 0); if (code != 0) { terrno = TSDB_CODE_WAL_FILE_CORRUPTED; + ASSERT(0); return -1; } if (head.head.version != ver) { @@ -178,6 +179,10 @@ int32_t walRollback(SWal *pWal, int64_t ver) { } ((SWalFileInfo *)taosArrayGetLast(pWal->fileInfoSet))->lastVer = ver - 1; ((SWalFileInfo *)taosArrayGetLast(pWal->fileInfoSet))->fileSize = entry.offset; + if (((SWalFileInfo *)taosArrayGetLast(pWal->fileInfoSet))->lastVer < ver - 1) { + ASSERT(((SWalFileInfo *)taosArrayGetLast(pWal->fileInfoSet))->fileSize == 0); + ((SWalFileInfo *)taosArrayGetLast(pWal->fileInfoSet))->firstVer = -1; + } taosCloseFile(&pIdxTFile); taosCloseFile(&pLogTFile); @@ -395,8 +400,12 @@ int64_t walWriteWithSyncInfo(SWal *pWal, int64_t index, tmsg_t msgType, SSyncLog } // set status + if (pWal->vers.firstVer == -1) pWal->vers.firstVer = index; pWal->vers.lastVer = index; pWal->totSize += sizeof(SWalHead) + bodyLen; + if (walGetCurFileInfo(pWal)->firstVer == -1) { + walGetCurFileInfo(pWal)->firstVer = index; + } walGetCurFileInfo(pWal)->lastVer = index; walGetCurFileInfo(pWal)->fileSize += sizeof(SWalHead) + bodyLen; diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index 353e94a490..0439ed148b 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -743,7 +743,7 @@ cmp_end: } void taosSetAllDebugFlag(int32_t flag) { - if (!(flag & DEBUG_TRACE || flag & DEBUG_DEBUG || flag & DEBUG_DUMP)) return; + if (flag <= 0) return; dDebugFlag = flag; vDebugFlag = flag; diff --git a/tests/pytest/util/cluster.py b/tests/pytest/util/cluster.py index 5debfab2c4..6eb78f0771 100644 --- a/tests/pytest/util/cluster.py +++ b/tests/pytest/util/cluster.py @@ -21,45 +21,39 @@ class ClusterDnodes(TDDnodes): self.testCluster = False self.valgrind = 0 self.killValgrind = 1 - self.independent = True - self.dnodeNums = 5 - - # def getTDDnodes(dnodeNums): - - # return class ConfigureyCluster: - """configure dnodes and return TDDnodes list, it can """ + """This will create defined number of dnodes and create a cluset. + at the same time, it will return TDDnodes list: dnodes, """ + hostname= socket.gethostname() def __init__(self): - self.dnodes = None - self.dnodes_nums = 5 + self.dnodes = [] + self.dnodeNums = 5 self.independent = True - self.start_port = 6030 + self.startPort = 6030 self.portStep = 100 - hostname1= socket.gethostname() + self.mnodeNums = 0 - def configure_cluster(self ,dnodes_nums=5,independent=True,start_port=6030,portStep=100,hostname="%s"%hostname1): - self.start_port=int(start_port) + def configure_cluster(self ,dnodeNums=5,mnodeNums=0,startPort=6030,portStep=100,hostname="%s"%hostname): + self.startPort=int(startPort) self.portStep=int(portStep) self.hostname=hostname - self.dnodes_nums = int(dnodes_nums) - self.independent = independent + self.dnodeNums = int(dnodeNums) + self.mnodeNums = int(mnodeNums) self.dnodes = [] - start_port_sec = 6130 - for num in range(1, (self.dnodes_nums+1)): + startPort_sec = int(startPort+portStep) + for num in range(1, (self.dnodeNums+1)): dnode = TDDnode(num) - dnode.addExtraCfg("firstEp", f"{hostname}:{self.start_port}") + dnode.addExtraCfg("firstEp", f"{hostname}:{self.startPort}") dnode.addExtraCfg("fqdn", f"{hostname}") - dnode.addExtraCfg("serverPort", f"{self.start_port + (num-1)*self.portStep}") - # dnode.addExtraCfg("monitorFqdn", hostname) - # dnode.addExtraCfg("monitorPort", 7043) - dnode.addExtraCfg("secondEp", f"{hostname}:{start_port_sec}") - # configure three dnoe don't support vnodes - if self.dnodes_nums > 4 : - if self.independent and (num < 4): - dnode.addExtraCfg("supportVnodes", 0) + dnode.addExtraCfg("serverPort", f"{self.startPort + (num-1)*self.portStep}") + dnode.addExtraCfg("secondEp", f"{hostname}:{startPort_sec}") + + # configure dnoe of independent mnodes + if num <= self.mnodeNums and self.mnodeNums != 0 : + dnode.addExtraCfg("supportVnodes", 0) # print(dnode) self.dnodes.append(dnode) return self.dnodes @@ -69,24 +63,7 @@ class ConfigureyCluster: for dnode in self.dnodes[1:]: # print(dnode.cfgDict) dnode_id = dnode.cfgDict["fqdn"] + ":" +dnode.cfgDict["serverPort"] - print(dnode_id) tdSql.execute(" create dnode '%s';"%dnode_id) - # count=0 - # while count < 10: - # time.sleep(1) - # tdSql.query("show dnodes;") - # if tdSql.checkRows(self.dnodes_nums) : - # print("mnode is three nodes") - # if tdSql.queryResult[0][4]=='leader' : - # if tdSql.queryResult[2][4]=='offline': - # if tdSql.queryResult[1][2]=='follower': - # print("stop mnodes on dnode 3 successfully in 10s") - # break - # count+=1 - # else: - # print("stop mnodes on dnode 3 failed in 10s") - # return -1 - checkstatus=False @@ -97,206 +74,19 @@ class ConfigureyCluster: tdSql.query("show dnodes") # tdLog.debug(tdSql.queryResult) status=0 - for i in range(self.dnodes_nums): + for i in range(self.dnodeNums): if tdSql.queryResult[i][4] == "ready": status+=1 - tdLog.debug(status) + # tdLog.debug(status) - if status == self.dnodes_nums: - tdLog.debug(" create cluster with %d dnode and check cluster dnode all ready within 5s! " %self.dnodes_nums) + if status == self.dnodeNums: + tdLog.debug(" create cluster with %d dnode and check cluster dnode all ready within 5s! " %self.dnodeNums) break count+=1 time.sleep(1) else: - tdLog.debug("create cluster with %d dnode but check dnode not ready within 5s ! "%self.dnodes_nums) + tdLog.debug("create cluster with %d dnode but check dnode not ready within 5s ! "%self.dnodeNums) return -1 -cluster = ConfigureyCluster() - - # def start(self ,dnodes_nums): - - # self.TDDnodes = ClusterDnodes(dnodes) - # self.TDDnodes.init("") - # self.TDDnodes.setTestCluster(testCluster) - # self.TDDnodes.setValgrind(valgrind) - # self.TDDnodes.stopAll() - # for dnode in self.TDDnodes.dnodes: - # self.TDDnodes.deploy(dnode.index,{}) - - # for dnode in self.TDDnodes.dnodes: - # self.TDDnodes.starttaosd(dnode.index) - - # # create cluster - # for dnode in self.TDDnodes.dnodes[1:]: - # # print(dnode.cfgDict) - # dnode_id = dnode.cfgDict["fqdn"] + ":" +dnode.cfgDict["serverPort"] - # dnode_first_host = dnode.cfgDict["firstEp"].split(":")[0] - # dnode_first_port = dnode.cfgDict["firstEp"].split(":")[-1] - # cmd = f" taos -h {dnode_first_host} -P {dnode_first_port} -s ' create dnode \"{dnode_id} \" ' ;" - # print(cmd) - # os.system(cmd) - - # time.sleep(2) - # tdLog.info(" create cluster with %d dnode done! " %dnodes_nums) - - # def buildcluster(self,dnodenumber): - # self.depoly_cluster(dnodenumber) - # self.master_dnode = self.TDDnodes.dnodes[0] - # self.host=self.master_dnode.cfgDict["fqdn"] - # conn1 = taos.connect(self.master_dnode.cfgDict["fqdn"] , config=self.master_dnode.cfgDir) - # tdSql.init(conn1.cursor()) - - # def checkdnodes(self,dnodenumber): - # count=0 - # while count < 10: - # time.sleep(1) - # statusReadyBumber=0 - # tdSql.query("show dnodes;") - # if tdSql.checkRows(dnodenumber) : - # print("dnode is %d nodes"%dnodenumber) - # for i in range(dnodenumber): - # if tdSql.queryResult[i][4] !='ready' : - # status=tdSql.queryResult[i][4] - # print("dnode:%d status is %s "%(i,status)) - # break - # else: - # statusReadyBumber+=1 - # print(statusReadyBumber) - # if statusReadyBumber == dnodenumber : - # print("all of %d mnodes is ready in 10s "%dnodenumber) - # return True - # break - # count+=1 - # else: - # print("%d mnodes is not ready in 10s "%dnodenumber) - # return False - - - # def check3mnode(self): - # count=0 - # while count < 10: - # time.sleep(1) - # tdSql.query("show mnodes;") - # if tdSql.checkRows(3) : - # print("mnode is three nodes") - # if tdSql.queryResult[0][2]=='leader' : - # if tdSql.queryResult[1][2]=='follower': - # if tdSql.queryResult[2][2]=='follower': - # print("three mnodes is ready in 10s") - # break - # elif tdSql.queryResult[0][2]=='follower' : - # if tdSql.queryResult[1][2]=='leader': - # if tdSql.queryResult[2][2]=='follower': - # print("three mnodes is ready in 10s") - # break - # elif tdSql.queryResult[0][2]=='follower' : - # if tdSql.queryResult[1][2]=='follower': - # if tdSql.queryResult[2][2]=='leader': - # print("three mnodes is ready in 10s") - # break - # count+=1 - # else: - # print("three mnodes is not ready in 10s ") - # return -1 - - # tdSql.query("show mnodes;") - # tdSql.checkRows(3) - # tdSql.checkData(0,1,'%s:6030'%self.host) - # tdSql.checkData(0,3,'ready') - # tdSql.checkData(1,1,'%s:6130'%self.host) - # tdSql.checkData(1,3,'ready') - # tdSql.checkData(2,1,'%s:6230'%self.host) - # tdSql.checkData(2,3,'ready') - - # def check3mnode1off(self): - # count=0 - # while count < 10: - # time.sleep(1) - # tdSql.query("show mnodes;") - # if tdSql.checkRows(3) : - # print("mnode is three nodes") - # if tdSql.queryResult[0][2]=='offline' : - # if tdSql.queryResult[1][2]=='leader': - # if tdSql.queryResult[2][2]=='follower': - # print("stop mnodes on dnode 2 successfully in 10s") - # break - # elif tdSql.queryResult[1][2]=='follower': - # if tdSql.queryResult[2][2]=='leader': - # print("stop mnodes on dnode 2 successfully in 10s") - # break - # count+=1 - # else: - # print("stop mnodes on dnode 2 failed in 10s ") - # return -1 - # tdSql.error("drop mnode on dnode 1;") - - # tdSql.query("show mnodes;") - # tdSql.checkRows(3) - # tdSql.checkData(0,1,'%s:6030'%self.host) - # tdSql.checkData(0,2,'offline') - # tdSql.checkData(0,3,'ready') - # tdSql.checkData(1,1,'%s:6130'%self.host) - # tdSql.checkData(1,3,'ready') - # tdSql.checkData(2,1,'%s:6230'%self.host) - # tdSql.checkData(2,3,'ready') - - # def check3mnode2off(self): - # count=0 - # while count < 40: - # time.sleep(1) - # tdSql.query("show mnodes;") - # if tdSql.checkRows(3) : - # print("mnode is three nodes") - # if tdSql.queryResult[0][2]=='leader' : - # if tdSql.queryResult[1][2]=='offline': - # if tdSql.queryResult[2][2]=='follower': - # print("stop mnodes on dnode 2 successfully in 10s") - # break - # count+=1 - # else: - # print("stop mnodes on dnode 2 failed in 10s ") - # return -1 - # tdSql.error("drop mnode on dnode 2;") - - # tdSql.query("show mnodes;") - # tdSql.checkRows(3) - # tdSql.checkData(0,1,'%s:6030'%self.host) - # tdSql.checkData(0,2,'leader') - # tdSql.checkData(0,3,'ready') - # tdSql.checkData(1,1,'%s:6130'%self.host) - # tdSql.checkData(1,2,'offline') - # tdSql.checkData(1,3,'ready') - # tdSql.checkData(2,1,'%s:6230'%self.host) - # tdSql.checkData(2,2,'follower') - # tdSql.checkData(2,3,'ready') - - # def check3mnode3off(self): - # count=0 - # while count < 10: - # time.sleep(1) - # tdSql.query("show mnodes;") - # if tdSql.checkRows(3) : - # print("mnode is three nodes") - # if tdSql.queryResult[0][2]=='leader' : - # if tdSql.queryResult[2][2]=='offline': - # if tdSql.queryResult[1][2]=='follower': - # print("stop mnodes on dnode 3 successfully in 10s") - # break - # count+=1 - # else: - # print("stop mnodes on dnode 3 failed in 10s") - # return -1 - # tdSql.error("drop mnode on dnode 3;") - # tdSql.query("show mnodes;") - # tdSql.checkRows(3) - # tdSql.checkData(0,1,'%s:6030'%self.host) - # tdSql.checkData(0,2,'leader') - # tdSql.checkData(0,3,'ready') - # tdSql.checkData(1,1,'%s:6130'%self.host) - # tdSql.checkData(1,2,'follower') - # tdSql.checkData(1,3,'ready') - # tdSql.checkData(2,1,'%s:6230'%self.host) - # tdSql.checkData(2,2,'offline') - # tdSql.checkData(2,3,'ready') - +cluster = ConfigureyCluster() \ No newline at end of file diff --git a/tests/script/general/parser/udf.sim b/tests/script/general/parser/udf.sim index c048a7c37a..4b767dbf60 100644 --- a/tests/script/general/parser/udf.sim +++ b/tests/script/general/parser/udf.sim @@ -44,6 +44,7 @@ endi sql create function n02 as '/tmp/normal' outputtype bool; sql show functions; if $rows != 1 then + print expect 1, actual $rows return -1 endi diff --git a/tests/script/tmp/monitor.sim b/tests/script/tmp/monitor.sim index ba98bec2d0..036d4cc6db 100644 --- a/tests/script/tmp/monitor.sim +++ b/tests/script/tmp/monitor.sim @@ -12,6 +12,8 @@ sql connect print =============== show dnodes sleep 2000 sql create database db vgroups 2; +sql use db; +sql create table db.stb (ts timestamp, c1 int, c2 binary(4)) tags(t1 int, t2 binary(16)) comment "abd"; sleep 2000 print =============== create drop qnode 1 diff --git a/tests/script/tsim/dnode/redistribute_vgroup_replica3_v1_follower.sim b/tests/script/tsim/dnode/redistribute_vgroup_replica3_v1_follower.sim index 788a23991a..9b11e16616 100644 --- a/tests/script/tsim/dnode/redistribute_vgroup_replica3_v1_follower.sim +++ b/tests/script/tsim/dnode/redistribute_vgroup_replica3_v1_follower.sim @@ -112,7 +112,7 @@ if $data(5)[4] != ready then goto step2 endi -print =============== step31: move follower +print =============== step3: create database $leaderExist = 0 $leaderVnode = 0 $follower1 = 0 @@ -165,7 +165,7 @@ if $rows != 1 then return -1 endi -print =============== step32: move follower2 +print =============== step31: print redistribute vgroup 2 dnode $leaderVnode dnode $follower2 dnode 5 sql redistribute vgroup 2 dnode $leaderVnode dnode $follower2 dnode 5 sql show d1.vgroups @@ -178,7 +178,7 @@ endi return -print =============== step33: move follower1 +print =============== step32: print redistribute vgroup 2 dnode $leaderVnode dnode $follower1 dnode 5 sql redistribute vgroup 2 dnode $leaderVnode dnode $follower1 dnode 5 sql show d1.vgroups @@ -189,7 +189,7 @@ if $rows != 1 then return -1 endi -print =============== step34: move follower2 +print =============== step33: print redistribute vgroup 2 dnode $leaderVnode dnode 5 dnode $follower2 sql redistribute vgroup 2 dnode $leaderVnode dnode 5 dnode $follower2 sql show d1.vgroups @@ -200,7 +200,7 @@ if $rows != 1 then return -1 endi -print =============== step35: move follower1 +print =============== step34: print redistribute vgroup 2 dnode $leaderVnode dnode 5 dnode $follower1 sql redistribute vgroup 2 dnode $leaderVnode dnode 5 dnode $follower1 sql show d1.vgroups @@ -211,9 +211,9 @@ if $rows != 1 then return -1 endi -print =============== step36: move follower2 -print redistribute vgroup 2 dnode $leaderVnode dnode $follower2 dnode 5 -sql redistribute vgroup 2 dnode $leaderVnode dnode $follower2 dnode 5 +print =============== step34: +print redistribute vgroup 2 dnode $leaderVnode dnode $follower2 dnode $follower1 +sql redistribute vgroup 2 dnode $leaderVnode dnode $follower2 dnode $follower1 sql show d1.vgroups print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 @@ -222,7 +222,7 @@ if $rows != 1 then return -1 endi -print =============== step37: move follower1 +print =============== step36: print redistribute vgroup 2 dnode $leaderVnode dnode $follower1 dnode 5 sql redistribute vgroup 2 dnode $leaderVnode dnode $follower1 dnode 5 sql show d1.vgroups @@ -233,7 +233,7 @@ if $rows != 1 then return -1 endi -print =============== step38: move follower2 +print =============== step37: print redistribute vgroup 2 dnode $leaderVnode dnode 5 dnode $follower2 sql redistribute vgroup 2 dnode $leaderVnode dnode 5 dnode $follower2 sql show d1.vgroups @@ -244,7 +244,7 @@ if $rows != 1 then return -1 endi -print =============== step39: move follower1 +print =============== step38: print redistribute vgroup 2 dnode $leaderVnode dnode 5 dnode $follower1 sql redistribute vgroup 2 dnode $leaderVnode dnode 5 dnode $follower1 sql show d1.vgroups @@ -255,6 +255,17 @@ if $rows != 1 then return -1 endi +print =============== step39: +print redistribute vgroup 2 dnode $leaderVnode dnode $follower1 dnode $follower2 +sql redistribute vgroup 2 dnode $leaderVnode dnode $follower1 dnode $follower2 +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show d1.tables +if $rows != 1 then + return -1 +endi + system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT system sh/exec.sh -n dnode3 -s stop -x SIGINT diff --git a/tests/script/tsim/dnode/redistribute_vgroup_replica3_v1_leader.sim b/tests/script/tsim/dnode/redistribute_vgroup_replica3_v1_leader.sim index e9dd82cad9..947482b3d4 100644 --- a/tests/script/tsim/dnode/redistribute_vgroup_replica3_v1_leader.sim +++ b/tests/script/tsim/dnode/redistribute_vgroup_replica3_v1_leader.sim @@ -91,7 +91,7 @@ if $data(5)[4] != ready then goto step2 endi -print =============== step31: move follower +print =============== step3: create database $leaderExist = 0 $leaderVnode = 0 $follower1 = 0 @@ -144,7 +144,7 @@ if $rows != 1 then return -1 endi -print =============== step32: move leader +print =============== step40: move leader print redistribute vgroup 2 dnode $follower1 dnode $follower2 dnode 5 sql redistribute vgroup 2 dnode $follower1 dnode $follower2 dnode 5 sql show d1.vgroups @@ -155,9 +155,7 @@ if $rows != 1 then return -1 endi -return - -print =============== step33: move follower1 +print =============== step41: print redistribute vgroup 2 dnode $leaderVnode dnode $follower2 dnode 5 sql redistribute vgroup 2 dnode $leaderVnode dnode $follower2 dnode 5 sql show d1.vgroups @@ -168,7 +166,7 @@ if $rows != 1 then return -1 endi -print =============== step34: move follower2 +print =============== step42: print redistribute vgroup 2 dnode $follower1 dnode 5 dnode $leaderVnode sql redistribute vgroup 2 dnode $follower1 dnode 5 dnode $leaderVnode sql show d1.vgroups @@ -179,7 +177,18 @@ if $rows != 1 then return -1 endi -print =============== step35: move leader +print =============== step43: +print redistribute vgroup 2 dnode $follower1 dnode $follower2 dnode $leaderVnode +sql redistribute vgroup 2 dnode $follower1 dnode $follower2 dnode $leaderVnode +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show d1.tables +if $rows != 1 then + return -1 +endi + +print =============== step44: print redistribute vgroup 2 dnode $follower2 dnode 5 dnode $follower1 sql redistribute vgroup 2 dnode $follower2 dnode 5 dnode $follower1 sql show d1.vgroups @@ -190,7 +199,7 @@ if $rows != 1 then return -1 endi -print =============== step36: move follower1 +print =============== step45: print redistribute vgroup 2 dnode $leaderVnode dnode $follower2 dnode 5 sql redistribute vgroup 2 dnode $leaderVnode dnode $follower2 dnode 5 sql show d1.vgroups @@ -201,7 +210,7 @@ if $rows != 1 then return -1 endi -print =============== step37: move follower2 +print =============== step46: print redistribute vgroup 2 dnode $leaderVnode dnode $follower1 dnode 5 sql redistribute vgroup 2 dnode $leaderVnode dnode $follower1 dnode 5 sql show d1.vgroups @@ -212,7 +221,7 @@ if $rows != 1 then return -1 endi -print =============== step38: move leader +print =============== step47: print redistribute vgroup 2 dnode $follower1 dnode 5 dnode $follower2 sql redistribute vgroup 2 dnode $follower1 dnode 5 dnode $follower2 sql show d1.vgroups @@ -223,7 +232,18 @@ if $rows != 1 then return -1 endi -print =============== step39: move follower1 +print =============== step48: +print redistribute vgroup 2 dnode $leaderVnode dnode 5 dnode $follower2 +sql redistribute vgroup 2 dnode $leaderVnode dnode 5 dnode $follower2 +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show d1.tables +if $rows != 1 then + return -1 +endi + +print =============== step49: print redistribute vgroup 2 dnode $leaderVnode dnode 5 dnode $follower2 sql redistribute vgroup 2 dnode $leaderVnode dnode 5 dnode $follower2 sql show d1.vgroups diff --git a/tests/script/tsim/mnode/basic5.sim b/tests/script/tsim/mnode/basic5.sim index e298dcbe55..23f5f6d782 100644 --- a/tests/script/tsim/mnode/basic5.sim +++ b/tests/script/tsim/mnode/basic5.sim @@ -233,7 +233,6 @@ if $data(1)[3] != dropping then endi print =============== step9: start mnode1 and wait it dropped -sleep 3000 system sh/exec.sh -n dnode1 -s start $x = 0 diff --git a/tests/script/tsim/query/udf.sim b/tests/script/tsim/query/udf.sim index d9821a0495..4e95095172 100644 --- a/tests/script/tsim/query/udf.sim +++ b/tests/script/tsim/query/udf.sim @@ -44,6 +44,7 @@ endi sql select udf2(f) from t; if $rows != 1 then + print expect 1, actual $rows return -1 endi if $data00 != 2.236067977 then diff --git a/tests/system-test/6-cluster/5dnode3mnodeStop.py b/tests/system-test/6-cluster/5dnode3mnodeStop.py index e1f66ceacb..69b9c3d879 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeStop.py +++ b/tests/system-test/6-cluster/5dnode3mnodeStop.py @@ -159,13 +159,10 @@ class TDTestCase: tdSql.query("show mnodes;") tdSql.checkRows(3) tdSql.checkData(0,1,'%s:6030'%self.host) - tdSql.checkData(0,2,'leader') tdSql.checkData(0,3,'ready') tdSql.checkData(1,1,'%s:6130'%self.host) - tdSql.checkData(1,2,'offline') tdSql.checkData(1,3,'ready') tdSql.checkData(2,1,'%s:6230'%self.host) - tdSql.checkData(2,2,'follower') tdSql.checkData(2,3,'ready') def check3mnode3off(self): @@ -265,13 +262,6 @@ class TDTestCase: while stopcount <= 2: for i in range(dnodenumber): tdDnodes[i].stoptaosd() - # if i == 1 : - # self.check3mnode2off() - # elif i == 2 : - # self.check3mnode3off() - # elif i == 0: - # self.check3mnode1off() - tdDnodes[i].starttaosd() # self.check3mnode() stopcount+=1 diff --git a/tests/system-test/7-tmq/tmq3mnodeSwitch.py b/tests/system-test/7-tmq/tmq3mnodeSwitch.py new file mode 100644 index 0000000000..2769c3867b --- /dev/null +++ b/tests/system-test/7-tmq/tmq3mnodeSwitch.py @@ -0,0 +1,272 @@ + +from ntpath import join +import taos +import sys +import time +import socket +import os +import threading + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +from util.cluster import * + +sys.path.append("./7-tmq") +from tmqCommon import * + +class TDTestCase: + def __init__(self): + self.dnodes = 5 + self.mnodes = 3 + self.idIndex = 0 + self.roleIndex = 2 + self.mnodeStatusIndex = 3 + self.mnodeEpIndex = 1 + self.dnodeStatusIndex = 4 + self.mnodeCheckCnt = 10 + self.host = socket.gethostname() + self.startPort = 6030 + self.portStep = 100 + self.dnodeOfLeader = 0 + + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + #tdSql.init(conn.cursor(), logSql) # output sql.txt file + + def checkDnodesStatusAndCreateMnode(self,dnodeNumbers): + count=0 + while count < dnodeNumbers: + tdSql.query("show dnodes") + # tdLog.debug(tdSql.queryResult) + dCnt = 0 + for i in range(dnodeNumbers): + if tdSql.queryResult[i][self.dnodeStatusIndex] != "ready": + break + else: + dCnt += 1 + if dCnt == dnodeNumbers: + break + time.sleep(1) + tdLog.debug("............... waiting for all dnodes ready!") + + tdLog.info("==============create two new mnodes ========") + tdSql.execute("create mnode on dnode 2") + tdSql.execute("create mnode on dnode 3") + self.check3mnode() + return + + def check3mnode(self): + count=0 + while count < self.mnodeCheckCnt: + time.sleep(1) + tdSql.query("show mnodes;") + if tdSql.checkRows(self.mnodes) : + tdLog.debug("mnode is three nodes") + else: + tdLog.exit("mnode number is correct") + + roleOfMnode0 = tdSql.queryResult[0][self.roleIndex] + roleOfMnode1 = tdSql.queryResult[1][self.roleIndex] + roleOfMnode2 = tdSql.queryResult[2][self.roleIndex] + + if roleOfMnode0=='leader' and roleOfMnode1=='follower' and roleOfMnode2 == 'follower' : + self.dnodeOfLeader = tdSql.queryResult[0][self.idIndex] + break + elif roleOfMnode0=='follower' and roleOfMnode1=='leader' and roleOfMnode2 == 'follower' : + self.dnodeOfLeader = tdSql.queryResult[1][self.idIndex] + break + elif roleOfMnode0=='follower' and roleOfMnode1=='follower' and roleOfMnode2 == 'leader' : + self.dnodeOfLeader = tdSql.queryResult[2][self.idIndex] + break + else: + count+=1 + else: + tdLog.exit("three mnodes is not ready in 10s ") + + tdSql.query("show mnodes;") + tdSql.checkRows(self.mnodes) + tdSql.checkData(0,self.mnodeEpIndex,'%s:%d'%(self.host,self.startPort)) + tdSql.checkData(0,self.mnodeStatusIndex,'ready') + tdSql.checkData(1,self.mnodeEpIndex,'%s:%d'%(self.host,self.startPort+self.portStep)) + tdSql.checkData(1,self.mnodeStatusIndex,'ready') + tdSql.checkData(2,self.mnodeEpIndex,'%s:%d'%(self.host,self.startPort+self.portStep*2)) + tdSql.checkData(2,self.mnodeStatusIndex,'ready') + + def check3mnode1off(self): + count=0 + while count < self.mnodeCheckCnt: + time.sleep(1) + tdSql.query("show mnodes") + tdLog.debug(tdSql.queryResult) + # if tdSql.checkRows(self.mnodes) : + # tdLog.debug("mnode is three nodes") + # else: + # tdLog.exit("mnode number is correct") + + roleOfMnode0 = tdSql.queryResult[0][self.roleIndex] + roleOfMnode1 = tdSql.queryResult[1][self.roleIndex] + roleOfMnode2 = tdSql.queryResult[2][self.roleIndex] + + if roleOfMnode0=='offline' : + if roleOfMnode1=='leader' and roleOfMnode2 == 'follower' : + self.dnodeOfLeader = tdSql.queryResult[1][self.idIndex] + break + elif roleOfMnode1=='follower' and roleOfMnode2 == 'leader' : + self.dnodeOfLeader = tdSql.queryResult[2][self.idIndex] + break + elif roleOfMnode1=='offline' : + if roleOfMnode0=='leader' and roleOfMnode2 == 'follower' : + self.dnodeOfLeader = tdSql.queryResult[0][self.idIndex] + break + elif roleOfMnode0=='follower' and roleOfMnode2 == 'leader' : + self.dnodeOfLeader = tdSql.queryResult[2][self.idIndex] + break + elif roleOfMnode2=='offline' : + if roleOfMnode0=='leader' and roleOfMnode1 == 'follower' : + self.dnodeOfLeader = tdSql.queryResult[0][self.idIndex] + break + elif roleOfMnode0=='follower' and roleOfMnode1 == 'leader' : + self.dnodeOfLeader = tdSql.queryResult[1][self.idIndex] + break + + count+=1 + else: + tdLog.exit("three mnodes is not ready in 10s ") + + def checkFileContent(self, consumerId, queryString): + buildPath = tdCom.getBuildPath() + cfgPath = tdCom.getClientCfgPath() + dstFile = '%s/../log/dstrows_%d.txt'%(cfgPath, consumerId) + cmdStr = '%s/build/bin/taos -c %s -s "%s >> %s"'%(buildPath, cfgPath, queryString, dstFile) + tdLog.info(cmdStr) + os.system(cmdStr) + + consumeRowsFile = '%s/../log/consumerid_%d.txt'%(cfgPath, consumerId) + tdLog.info("rows file: %s, %s"%(consumeRowsFile, dstFile)) + + consumeFile = open(consumeRowsFile, mode='r') + queryFile = open(dstFile, mode='r') + + # skip first line for it is schema + queryFile.readline() + + while True: + dst = queryFile.readline() + src = consumeFile.readline() + + if dst: + if dst != src: + tdLog.exit("consumerId %d consume rows is not match the rows by direct query"%consumerId) + else: + break + return + + def tmqCase1(self): + tdLog.printNoPrefix("======== test case 1: ") + paraDict = {'dbName': 'db1', + 'dropFlag': 1, + 'event': '', + 'vgroups': 4, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':2}, {'type': 'binary', 'len':20, 'count':1}, {'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbNum': 1, + 'rowsPerTbl': 100000, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 10, + 'showMsg': 1, + 'showRow': 1} + + topicNameList = ['topic1'] + expectRowsList = [] + tmqCom.initConsumerTable() + tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=4,replica=1) + tdLog.info("create stb") + tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema']) + tdLog.info("create ctb") + tdCom.create_ctable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"],tag_elm_list=paraDict['tagSchema'],count=paraDict["ctbNum"], default_ctbname_prefix=paraDict['ctbPrefix']) + tdLog.info("async insert data") + pThread = tmqCom.asyncInsertData(paraDict) + + tdLog.info("create topics from stb with filter") + queryString = "select ts, log(c1), ceil(pow(c1,3)) from %s.%s where c1 %% 7 == 0" %(paraDict['dbName'], paraDict['stbName']) + sqlString = "create topic %s as %s" %(topicNameList[0], queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + + # init consume info, and start tmq_sim, then check consume result + tdLog.info("insert consume info to consume processor") + consumerId = 0 + expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] + topicList = topicNameList[0] + ifcheckdata = 1 + ifManualCommit = 1 + keyList = 'group.id:cgrp1, enable.auto.commit:false, auto.commit.interval.ms:6000, auto.offset.reset:earliest' + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(paraDict['pollDelay'],paraDict["dbName"],paraDict['showMsg'], paraDict['showRow']) + + tdLog.info("wait the notify info of start consume") + tmqCom.getStartConsumeNotifyFromTmqsim() + + tdLog.info("start switch mnode ................") + tdDnodes = cluster.dnodes + + tdLog.info("1. stop dnode 0") + tdDnodes[0].stoptaosd() + time.sleep(10) + self.check3mnode1off() + + tdLog.info("2. start dnode 0") + tdDnodes[0].starttaosd() + self.check3mnode() + + tdLog.info("3. stop dnode 1") + tdDnodes[1].stoptaosd() + time.sleep(10) + self.check3mnode1off() + + tdLog.info("switch end and wait insert data end ................") + pThread.join() + + tdLog.info("check the consume result") + tdSql.query(queryString) + expectRowsList.append(tdSql.getRows()) + + expectRows = 1 + resultList = tmqCom.selectConsumeResult(expectRows) + + if expectRowsList[0] != resultList[0]: + tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[0], resultList[0])) + tdLog.exit("0 tmq consume rows error!") + + self.checkFileContent(consumerId, queryString) + + time.sleep(10) + for i in range(len(topicNameList)): + tdSql.query("drop topic %s"%topicNameList[i]) + + tdLog.printNoPrefix("======== test case 1 end ...... ") + + def run(self): + tdLog.printNoPrefix("======== Notes: must add '-N 5' for run the script ========") + self.checkDnodesStatusAndCreateMnode(self.dnodes) + self.tmqCase1() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +event = threading.Event() + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/7-tmq/tmqCommon.py b/tests/system-test/7-tmq/tmqCommon.py index 9b23056546..d17e36fc97 100644 --- a/tests/system-test/7-tmq/tmqCommon.py +++ b/tests/system-test/7-tmq/tmqCommon.py @@ -204,6 +204,35 @@ class TMQCom: tdLog.debug("insert data ............ [OK]") return + def insert_data_2(self,tsql,dbName,ctbPrefix,ctbNum,rowsPerTbl,batchNum,startTs): + tdLog.debug("start to insert data ............") + tsql.execute("use %s" %dbName) + pre_insert = "insert into " + sql = pre_insert + + t = time.time() + startTs = int(round(t * 1000)) + #tdLog.debug("doing insert data into stable:%s rows:%d ..."%(stbName, allRows)) + for i in range(ctbNum): + sql += " %s%d values "%(ctbPrefix,i) + for j in range(rowsPerTbl): + if (j % 2 == 0): + sql += "(%d, %d, %d, 'tmqrow_%d', now) "%(startTs + j, j, j, j) + else: + sql += "(%d, %d, %d, 'tmqrow_%d', now) "%(startTs + j, j, -j, j) + if (j > 0) and ((j%batchNum == 0) or (j == rowsPerTbl - 1)): + tsql.execute(sql) + if j < rowsPerTbl - 1: + sql = "insert into %s%d values " %(ctbPrefix,i) + else: + sql = "insert into " + #end sql + if sql != pre_insert: + #print("insert sql:%s"%sql) + tsql.execute(sql) + tdLog.debug("insert data ............ [OK]") + return + def insert_data_interlaceByMultiTbl(self,tsql,dbName,ctbPrefix,ctbNum,rowsPerTbl,batchNum,startTs=0): tdLog.debug("start to insert data ............") tsql.execute("use %s" %dbName) @@ -291,6 +320,17 @@ class TMQCom: pThread.start() return pThread + def threadFunctionForInsert(self, **paraDict): + # create new connector for new tdSql instance in my thread + newTdSql = tdCom.newTdSql() + self.insert_data_2(newTdSql,paraDict["dbName"],paraDict["ctbPrefix"],paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],paraDict["startTs"]) + return + + def asyncInsertData(self, paraDict): + pThread = threading.Thread(target=self.threadFunctionForInsert, kwargs=paraDict) + pThread.start() + return pThread + def close(self): self.cursor.close() diff --git a/tests/system-test/7-tmq/tmqUdf.py b/tests/system-test/7-tmq/tmqUdf.py index f1c451de85..d241a97e0c 100644 --- a/tests/system-test/7-tmq/tmqUdf.py +++ b/tests/system-test/7-tmq/tmqUdf.py @@ -113,7 +113,7 @@ class TDTestCase: tmqCom.insert_data_1(tdSql,paraDict["dbName"],paraDict["ctbPrefix"],paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],paraDict["startTs"]) tdLog.info("create topics from stb with filter") - queryString = "select ts, c1,udf1(c1),c2,udf1(c2) from %s.%s where c1 %% 7 == 0" %(paraDict['dbName'], paraDict['stbName']) + queryString = "select ts,c1,udf1(c1),c2,udf1(c2) from %s.%s where c1 %% 7 == 0" %(paraDict['dbName'], paraDict['stbName']) sqlString = "create topic %s as %s" %(topicNameList[0], queryString) tdLog.info("create topic sql: %s"%sqlString) tdSql.execute(sqlString) diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 19a67e924c..73e6716cad 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -112,7 +112,7 @@ python3 ./test.py -f 2-query/twa.py python3 ./test.py -f 6-cluster/5dnode1mnode.py python3 ./test.py -f 6-cluster/5dnode2mnode.py -python3 ./test.py -f 6-cluster/5dnode3mnodeStop.py -N 5 +python3 ./test.py -f 6-cluster/5dnode3mnodeStop.py -N 5 -M 3 # python3 ./test.py -f 6-cluster/5dnode3mnodeDrop.py -N 5 # BUG python3 ./test.py -f 6-cluster/5dnode3mnodeStopInsert.py @@ -134,3 +134,4 @@ python3 ./test.py -f 7-tmq/schema.py python3 ./test.py -f 7-tmq/stbFilter.py python3 ./test.py -f 7-tmq/tmqCheckData.py python3 ./test.py -f 7-tmq/tmqUdf.py +#python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 5 diff --git a/tests/system-test/test.py b/tests/system-test/test.py index 4ca16e7a57..7d78f5ab21 100644 --- a/tests/system-test/test.py +++ b/tests/system-test/test.py @@ -60,10 +60,11 @@ if __name__ == "__main__": stop = 0 restart = False dnodeNums = 1 + mnodeNums = 0 updateCfgDict = {} execCmd = "" - opts, args = getopt.gnu_getopt(sys.argv[1:], 'f:p:m:l:scghrd:k:e:N:', [ - 'file=', 'path=', 'master', 'logSql', 'stop', 'cluster', 'valgrind', 'help', 'restart', 'updateCfgDict', 'killv', 'execCmd','dnodeNums']) + opts, args = getopt.gnu_getopt(sys.argv[1:], 'f:p:m:l:scghrd:k:e:N:M:', [ + 'file=', 'path=', 'master', 'logSql', 'stop', 'cluster', 'valgrind', 'help', 'restart', 'updateCfgDict', 'killv', 'execCmd','dnodeNums','mnodeNums']) for key, value in opts: if key in ['-h', '--help']: tdLog.printNoPrefix( @@ -79,7 +80,8 @@ if __name__ == "__main__": tdLog.printNoPrefix('-d update cfg dict, base64 json str') tdLog.printNoPrefix('-k not kill valgrind processer') tdLog.printNoPrefix('-e eval str to run') - tdLog.printNoPrefix('-N create dnodes numbers clusters') + tdLog.printNoPrefix('-N create dnodes numbers in clusters') + tdLog.printNoPrefix('-M create mnode numbers in clusters') sys.exit(0) @@ -133,6 +135,9 @@ if __name__ == "__main__": if key in ['-N', '--dnodeNums']: dnodeNums = value + if key in ['-M', '--mnodeNums']: + mnodeNums = value + if not execCmd == "": tdDnodes.init(deployPath) print(execCmd) @@ -244,9 +249,8 @@ if __name__ == "__main__": tdDnodes.start(1) tdCases.logSql(logSql) else : - print("start cluster and dnodes number") - - dnodeslist = cluster.configure_cluster(dnodes_nums=dnodeNums,independent=True) + tdLog.debug("create an cluster with %s nodes and make %s dnode as independent mnode"%(dnodeNums,mnodeNums)) + dnodeslist = cluster.configure_cluster(dnodeNums=dnodeNums,mnodeNums=mnodeNums) tdDnodes = ClusterDnodes(dnodeslist) tdDnodes.init(deployPath, masterIp) tdDnodes.setTestCluster(testCluster) diff --git a/tests/test/c/tmqSim.c b/tests/test/c/tmqSim.c index 053cd79432..e96f5d4793 100644 --- a/tests/test/c/tmqSim.c +++ b/tests/test/c/tmqSim.c @@ -453,7 +453,17 @@ static int32_t msg_process(TAOS_RES* msg, SThreadInfo* pInfo, int32_t msgIndex) int32_t precision = taos_result_precision(msg); const char* tbName = tmq_get_table_name(msg); - dumpToFileForCheck(pInfo->pConsumeRowsFile, row, fields, length, numOfFields, precision); + #if 0 + // get schema + //============================== stub =================================================// + for (int32_t i = 0; i < numOfFields; i++) { + taosFprintfFile(g_fp, "%02d: name: %s, type: %d, len: %d\n", i, fields[i].name, fields[i].type, fields[i].bytes); + } + //============================== stub =================================================// + #endif + + dumpToFileForCheck(pInfo->pConsumeRowsFile, row, fields, length, numOfFields, precision); + taos_print_row(buf, row, fields, numOfFields); if (0 != g_stConfInfo.showRowFlag) { @@ -656,12 +666,13 @@ void* consumeThreadFunc(void* param) { pInfo->taos = taos_connect(NULL, "root", "taosdata", NULL, 0); if (pInfo->taos == NULL) { taosFprintfFile(g_fp, "taos_connect() fail, can not notify and save consume result to main scripte\n"); - exit(-1); + return NULL; } build_consumer(pInfo); build_topic_list(pInfo); if ((NULL == pInfo->tmq) || (NULL == pInfo->topicList)) { + taosFprintfFile(g_fp, "create consumer fail! tmq is null or topicList is null\n"); assert(0); return NULL; } @@ -669,7 +680,9 @@ void* consumeThreadFunc(void* param) { int32_t err = tmq_subscribe(pInfo->tmq, pInfo->topicList); if (err != 0) { pError("tmq_subscribe() fail, reason: %s\n", tmq_err2str(err)); - exit(-1); + taosFprintfFile(g_fp, "tmq_subscribe()! reason: %s\n", tmq_err2str(err)); + assert(0); + return NULL; } tmq_list_destroy(pInfo->topicList); @@ -688,14 +701,13 @@ void* consumeThreadFunc(void* param) { err = tmq_unsubscribe(pInfo->tmq); if (err != 0) { pError("tmq_unsubscribe() fail, reason: %s\n", tmq_err2str(err)); - /*pInfo->consumeMsgCnt = -1;*/ - /*return NULL;*/ + taosFprintfFile(g_fp, "tmq_unsubscribe()! reason: %s\n", tmq_err2str(err)); } err = tmq_consumer_close(pInfo->tmq); if (err != 0) { pError("tmq_consumer_close() fail, reason: %s\n", tmq_err2str(err)); - /*exit(-1);*/ + taosFprintfFile(g_fp, "tmq_consumer_close()! reason: %s\n", tmq_err2str(err)); } pInfo->tmq = NULL;