diff --git a/example/src/tmq.c b/example/src/tmq.c index efb4d1830e..ca80c8fe5a 100644 --- a/example/src/tmq.c +++ b/example/src/tmq.c @@ -163,12 +163,13 @@ void basic_consume_loop(tmq_t* tmq, tmq_list_t* topics) { printf("subscribe err\n"); return; } - /*int32_t cnt = 0;*/ + int32_t cnt = 0; /*clock_t startTime = clock();*/ while (running) { tmq_message_t* tmqmessage = tmq_consumer_poll(tmq, 500); if (tmqmessage) { - /*cnt++;*/ + cnt++; + printf("get data\n"); msg_process(tmqmessage); tmq_message_destroy(tmqmessage); /*} else {*/ diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index b2b8cff19a..5e5a8826e5 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -136,7 +136,8 @@ static FORCE_INLINE void colDataAppendInt8(SColumnInfoData* pColumnInfoData, uin } static FORCE_INLINE void colDataAppendInt16(SColumnInfoData* pColumnInfoData, uint32_t currentRow, int16_t* v) { - ASSERT(pColumnInfoData->info.type == TSDB_DATA_TYPE_SMALLINT || pColumnInfoData->info.type == TSDB_DATA_TYPE_USMALLINT); + ASSERT(pColumnInfoData->info.type == TSDB_DATA_TYPE_SMALLINT || + pColumnInfoData->info.type == TSDB_DATA_TYPE_USMALLINT); char* p = pColumnInfoData->pData + pColumnInfoData->info.bytes * currentRow; *(int16_t*)p = *(int16_t*)v; } @@ -210,15 +211,19 @@ SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock); void blockDebugShowData(const SArray* dataBlocks); +static FORCE_INLINE int32_t blockEstimateEncodeSize(const SSDataBlock* pBlock) { + return blockDataGetSerialMetaSize(pBlock) + (int32_t)ceil(blockDataGetSerialRowSize(pBlock) * pBlock->info.rows); +} + static FORCE_INLINE int32_t blockCompressColData(SColumnInfoData* pColRes, int32_t numOfRows, char* data, - int8_t compressed) { + int8_t compressed) { int32_t colSize = colDataGetLength(pColRes, numOfRows); return (*(tDataTypes[pColRes->info.type].compFunc))(pColRes->pData, colSize, numOfRows, data, colSize + COMP_OVERFLOW_BYTES, compressed, NULL, 0); } static FORCE_INLINE void blockCompressEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_t numOfCols, - int8_t needCompress) { + int8_t needCompress) { int32_t* colSizes = (int32_t*)data; data += numOfCols * sizeof(int32_t); diff --git a/include/common/tglobal.h b/include/common/tglobal.h index f03943e105..5022cb7be8 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -18,6 +18,7 @@ #include "tarray.h" #include "tdef.h" +#include "tconfig.h" #ifdef __cplusplus extern "C" { @@ -129,6 +130,7 @@ void taosCfgDynamicOptions(const char *option, const char *value); void taosAddDataDir(int32_t index, char *v1, int32_t level, int32_t primary); struct SConfig *taosGetCfg(); +int32_t taosAddClientLogCfg(SConfig *pCfg); #ifdef __cplusplus } diff --git a/include/common/ttypes.h b/include/common/ttypes.h index 5f1cfc030a..032cb44122 100644 --- a/include/common/ttypes.h +++ b/include/common/ttypes.h @@ -142,6 +142,43 @@ typedef struct { } \ } while (0) +#define NUM_TO_STRING(_inputType, _input, _outputBytes, _output) \ + do { \ + switch (_inputType) { \ + case TSDB_DATA_TYPE_TINYINT: \ + snprintf(_output, (int32_t)(_outputBytes), "%d", *(int8_t *)(_input)); \ + break; \ + case TSDB_DATA_TYPE_UTINYINT: \ + snprintf(_output, (int32_t)(_outputBytes), "%d", *(uint8_t *)(_input)); \ + break; \ + case TSDB_DATA_TYPE_SMALLINT: \ + snprintf(_output, (int32_t)(_outputBytes), "%d", *(int16_t *)(_input)); \ + break; \ + case TSDB_DATA_TYPE_USMALLINT: \ + snprintf(_output, (int32_t)(_outputBytes), "%d", *(uint16_t *)(_input)); \ + break; \ + case TSDB_DATA_TYPE_TIMESTAMP: \ + case TSDB_DATA_TYPE_BIGINT: \ + snprintf(_output, (int32_t)(_outputBytes), "%" PRId64, *(int64_t *)(_input)); \ + break; \ + case TSDB_DATA_TYPE_UBIGINT: \ + snprintf(_output, (int32_t)(_outputBytes), "%" PRIu64, *(uint64_t *)(_input)); \ + break; \ + case TSDB_DATA_TYPE_FLOAT: \ + snprintf(_output, (int32_t)(_outputBytes), "%f", *(float *)(_input)); \ + break; \ + case TSDB_DATA_TYPE_DOUBLE: \ + snprintf(_output, (int32_t)(_outputBytes), "%f", *(double *)(_input)); \ + break; \ + case TSDB_DATA_TYPE_UINT: \ + snprintf(_output, (int32_t)(_outputBytes), "%u", *(uint32_t *)(_input)); \ + break; \ + default: \ + snprintf(_output, (int32_t)(_outputBytes), "%d", *(int32_t *)(_input)); \ + break; \ + } \ + } while (0) + #define IS_SIGNED_NUMERIC_TYPE(_t) ((_t) >= TSDB_DATA_TYPE_TINYINT && (_t) <= TSDB_DATA_TYPE_BIGINT) #define IS_UNSIGNED_NUMERIC_TYPE(_t) ((_t) >= TSDB_DATA_TYPE_UTINYINT && (_t) <= TSDB_DATA_TYPE_UBIGINT) #define IS_FLOAT_TYPE(_t) ((_t) == TSDB_DATA_TYPE_FLOAT || (_t) == TSDB_DATA_TYPE_DOUBLE) diff --git a/include/libs/function/functionMgt.h b/include/libs/function/functionMgt.h index 57a3862a9a..9d1b2c9ea0 100644 --- a/include/libs/function/functionMgt.h +++ b/include/libs/function/functionMgt.h @@ -135,8 +135,17 @@ bool fmIsTimeorderFunc(int32_t funcId); bool fmIsPseudoColumnFunc(int32_t funcId); bool fmIsWindowPseudoColumnFunc(int32_t funcId); bool fmIsWindowClauseFunc(int32_t funcId); +bool fmIsSpecialDataRequiredFunc(int32_t funcId); +bool fmIsDynamicScanOptimizedFunc(int32_t funcId); -int32_t fmFuncScanType(int32_t funcId); +typedef enum EFuncDataRequired { + FUNC_DATA_REQUIRED_ALL_NEEDED = 1, + FUNC_DATA_REQUIRED_STATIS_NEEDED, + FUNC_DATA_REQUIRED_NO_NEEDED, + FUNC_DATA_REQUIRED_DISCARD +} EFuncDataRequired; + +EFuncDataRequired fmFuncDataRequired(SFunctionNode* pFunc, STimeWindow* pTimeWindow); int32_t fmGetFuncExecFuncs(int32_t funcId, SFuncExecFuncs* pFpSet); int32_t fmGetScalarFuncExecFuncs(int32_t funcId, SScalarFuncExecFuncs* pFpSet); diff --git a/include/libs/monitor/monitor.h b/include/libs/monitor/monitor.h index f5080fbe7b..af0580674d 100644 --- a/include/libs/monitor/monitor.h +++ b/include/libs/monitor/monitor.h @@ -78,6 +78,9 @@ typedef struct { typedef struct { float uptime; // day int8_t has_mnode; + int8_t has_qnode; + int8_t has_snode; + int8_t has_bnode; SMonDiskDesc logdir; SMonDiskDesc tempdir; } SMonDnodeInfo; @@ -134,8 +137,8 @@ typedef struct { typedef struct { int32_t expire_time; - int32_t timeseries_used; - int32_t timeseries_total; + int64_t timeseries_used; + int64_t timeseries_total; } SMonGrantInfo; typedef struct { diff --git a/include/libs/nodes/nodes.h b/include/libs/nodes/nodes.h index c91779ba8b..3a1d7954a7 100644 --- a/include/libs/nodes/nodes.h +++ b/include/libs/nodes/nodes.h @@ -216,6 +216,7 @@ SNodeList* nodesMakeList(); int32_t nodesListAppend(SNodeList* pList, SNodeptr pNode); int32_t nodesListStrictAppend(SNodeList* pList, SNodeptr pNode); int32_t nodesListMakeAppend(SNodeList** pList, SNodeptr pNode); +int32_t nodesListMakeStrictAppend(SNodeList** pList, SNodeptr pNode); int32_t nodesListAppendList(SNodeList* pTarget, SNodeList* pSrc); int32_t nodesListStrictAppendList(SNodeList* pTarget, SNodeList* pSrc); int32_t nodesListPushFront(SNodeList* pList, SNodeptr pNode); diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index 3931be7da5..3b5f9abe81 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -30,6 +30,7 @@ typedef struct SLogicNode { SNode* pConditions; SNodeList* pChildren; struct SLogicNode* pParent; + int32_t optimizedFlag; } SLogicNode; typedef enum EScanType { @@ -50,6 +51,8 @@ typedef struct SScanLogicNode { SName tableName; bool showRewrite; double ratio; + SNodeList* pDynamicScanFuncs; + int32_t dataRequired; } SScanLogicNode; typedef struct SJoinLogicNode { @@ -196,20 +199,13 @@ typedef struct SSystemTableScanPhysiNode { int32_t accountId; } SSystemTableScanPhysiNode; -typedef enum EScanRequired { - SCAN_REQUIRED_DATA_NO_NEEDED = 1, - SCAN_REQUIRED_DATA_STATIS_NEEDED, - SCAN_REQUIRED_DATA_ALL_NEEDED, - SCAN_REQUIRED_DATA_DISCARD, -} EScanRequired; - typedef struct STableScanPhysiNode { SScanPhysiNode scan; uint8_t scanFlag; // denotes reversed scan of data or not STimeWindow scanRange; double ratio; - EScanRequired scanRequired; - SNodeList* pScanReferFuncs; + int32_t dataRequired; + SNodeList* pDynamicScanFuncs; } STableScanPhysiNode; typedef STableScanPhysiNode STableSeqScanPhysiNode; diff --git a/include/libs/scalar/scalar.h b/include/libs/scalar/scalar.h index b5acc64f0b..3c5b164648 100644 --- a/include/libs/scalar/scalar.h +++ b/include/libs/scalar/scalar.h @@ -70,6 +70,9 @@ int32_t ltrimFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOut int32_t rtrimFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t substrFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +/* Conversion functions */ +int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); + bool getTimePseudoFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); int32_t winStartTsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); diff --git a/include/os/osSysinfo.h b/include/os/osSysinfo.h index 022f11bb0e..c009bcf350 100644 --- a/include/os/osSysinfo.h +++ b/include/os/osSysinfo.h @@ -39,7 +39,7 @@ int32_t taosGetEmail(char *email, int32_t maxLen); int32_t taosGetOsReleaseName(char *releaseName, int32_t maxLen); int32_t taosGetCpuInfo(char *cpuModel, int32_t maxLen, float *numOfCores); int32_t taosGetCpuCores(float *numOfCores); -int32_t taosGetCpuUsage(double *cpu_system, double *cpu_engine); +void taosGetCpuUsage(double *cpu_system, double *cpu_engine); int32_t taosGetTotalMemory(int64_t *totalKB); int32_t taosGetProcMemory(int64_t *usedKB); int32_t taosGetSysMemory(int64_t *usedKB); diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 04baa3c09d..60535e2f49 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -597,6 +597,8 @@ int32_t* taosGetErrno(); #define TSDB_CODE_PAR_INVALID_ROLLUP_OPTION TAOS_DEF_ERROR_CODE(0, 0x2622) #define TSDB_CODE_PAR_INVALID_RETENTIONS_OPTION TAOS_DEF_ERROR_CODE(0, 0x2623) #define TSDB_CODE_PAR_GROUPBY_WINDOW_COEXIST TAOS_DEF_ERROR_CODE(0, 0x2624) +#define TSDB_CODE_PAR_INVALID_OPTION_UNIT TAOS_DEF_ERROR_CODE(0, 0x2625) +#define TSDB_CODE_PAR_INVALID_KEEP_UNIT TAOS_DEF_ERROR_CODE(0, 0x2626) //planner #define TSDB_CODE_PLAN_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x2700) diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index f244059b7c..185b5824d9 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -75,12 +75,12 @@ typedef int32_t (*FHbReqHandle)(SClientHbKey* connKey, void* param, SClientHbReq typedef struct { int8_t inited; // ctl - int8_t threadStop; - TdThread thread; + int8_t threadStop; + TdThread thread; TdThreadMutex lock; // used when app init and cleanup - SArray* appHbMgrs; // SArray one for each cluster - FHbReqHandle reqHandle[HEARTBEAT_TYPE_MAX]; - FHbRspHandle rspHandle[HEARTBEAT_TYPE_MAX]; + SArray* appHbMgrs; // SArray one for each cluster + FHbReqHandle reqHandle[HEARTBEAT_TYPE_MAX]; + FHbRspHandle rspHandle[HEARTBEAT_TYPE_MAX]; } SClientHbMgr; typedef struct SQueryExecMetric { @@ -118,42 +118,42 @@ struct SAppInstInfo { }; typedef struct SAppInfo { - int64_t startTime; - char appName[TSDB_APP_NAME_LEN]; - char* ep; - int32_t pid; - int32_t numOfThreads; - SHashObj* pInstMap; + int64_t startTime; + char appName[TSDB_APP_NAME_LEN]; + char* ep; + int32_t pid; + int32_t numOfThreads; + SHashObj* pInstMap; TdThreadMutex mutex; } SAppInfo; typedef struct STscObj { - char user[TSDB_USER_LEN]; - char pass[TSDB_PASSWORD_LEN]; - char db[TSDB_DB_FNAME_LEN]; - char ver[128]; - int32_t acctId; - uint32_t connId; - int32_t connType; - uint64_t id; // ref ID returned by taosAddRef - TdThreadMutex mutex; // used to protect the operation on db - int32_t numOfReqs; // number of sqlObj bound to this connection - SAppInstInfo* pAppInfo; + char user[TSDB_USER_LEN]; + char pass[TSDB_PASSWORD_LEN]; + char db[TSDB_DB_FNAME_LEN]; + char ver[128]; + int32_t acctId; + uint32_t connId; + int32_t connType; + uint64_t id; // ref ID returned by taosAddRef + TdThreadMutex mutex; // used to protect the operation on db + int32_t numOfReqs; // number of sqlObj bound to this connection + SAppInstInfo* pAppInfo; } STscObj; typedef struct SResultColumn { union { - char* nullbitmap; // bitmap, one bit for each item in the list - int32_t* offset; + char* nullbitmap; // bitmap, one bit for each item in the list + int32_t* offset; }; - char* pData; + char* pData; } SResultColumn; typedef struct SReqResultInfo { const char* pRspMsg; const char* pData; - TAOS_FIELD* fields; // todo, column names are not needed. - TAOS_FIELD* userFields; // the fields info that return to user + TAOS_FIELD* fields; // todo, column names are not needed. + TAOS_FIELD* userFields; // the fields info that return to user uint32_t numOfCols; int32_t* length; char** convertBuf; @@ -180,13 +180,30 @@ typedef struct SRequestSendRecvBody { SShowReqInfo showInfo; // todo this attribute will be removed after the query framework being completed. SDataBuf requestMsg; int64_t queryJob; // query job, created according to sql query DAG. - struct SQueryPlan* pDag; // the query dag, generated according to the sql statement. + struct SQueryPlan* pDag; // the query dag, generated according to the sql statement. SReqResultInfo resInfo; } SRequestSendRecvBody; #define ERROR_MSG_BUF_DEFAULT_SIZE 512 +enum { + RES_TYPE__QUERY = 1, + RES_TYPE__TMQ, +}; + +#define TD_RES_QUERY(res) (*(int8_t*)res == RES_TYPE__QUERY) +#define TD_RES_TMQ(res) (*(int8_t*)res == RES_TYPE__TMQ) + +typedef struct SMqRspObj { + int8_t resType; + char* topic; + void* vg; + SArray* res; // SArray + int32_t resIter; +} SMqRspObj; + typedef struct SRequestObj { + int8_t resType; // query or tmq uint64_t requestId; int32_t type; // request type STscObj* pTscObj; @@ -203,6 +220,25 @@ typedef struct SRequestObj { SRequestSendRecvBody body; } SRequestObj; +static FORCE_INLINE SReqResultInfo* tmqGetCurResInfo(TAOS_RES* res) { + SMqRspObj* msg = (SMqRspObj*)res; + int32_t resIter = msg->resIter == -1 ? 0 : msg->resIter; + return (SReqResultInfo*)taosArrayGet(msg->res, resIter); +} + +static FORCE_INLINE SReqResultInfo* tmqGetNextResInfo(TAOS_RES* res) { + SMqRspObj* msg = (SMqRspObj*)res; + if (++msg->resIter < taosArrayGetSize(msg->res)) { + return (SReqResultInfo*)taosArrayGet(msg->res, msg->resIter); + } + return NULL; +} + +static FORCE_INLINE SReqResultInfo* tscGetCurResInfo(TAOS_RES* res) { + if (TD_RES_QUERY(res)) return &(((SRequestObj*)res)->body.resInfo); + return tmqGetCurResInfo(res); +} + extern SAppInfo appInfo; extern int32_t clientReqRefPool; extern int32_t clientConnRefPool; @@ -238,14 +274,17 @@ void initMsgHandleFp(); TAOS* taos_connect_internal(const char* ip, const char* user, const char* pass, const char* auth, const char* db, uint16_t port); -void* doFetchRow(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4); - -int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32_t numOfCols, int32_t numOfRows, bool convertUcs4); +int32_t parseSql(SRequestObj* pRequest, bool topicQuery, SQuery** pQuery); +int32_t getPlan(SRequestObj* pRequest, SQuery* pQuery, SQueryPlan** pPlan, SArray* pNodeList); int32_t buildRequest(STscObj* pTscObj, const char* sql, int sqlLen, SRequestObj** pRequest); -int32_t parseSql(SRequestObj* pRequest, bool topicQuery, SQuery** pQuery); -int32_t getPlan(SRequestObj* pRequest, SQuery* pQuery, SQueryPlan** pPlan, SArray* pNodeList); +void* doFetchRow(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4); +void doSetOneRowPtr(SReqResultInfo* pResultInfo); +int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32_t numOfCols, int32_t numOfRows, + bool convertUcs4); +void setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t numOfCols); +int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp, bool convertUcs4); // --- heartbeat // global, called by mgmt diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 359649884f..fec6c8e5db 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -149,6 +149,7 @@ void *createRequest(STscObj *pObj, __taos_async_fn_t fp, void *param, int32_t ty return NULL; } + pRequest->resType = RES_TYPE__QUERY; pRequest->pDb = getDbOfConnection(pObj); pRequest->requestId = generateRequestId(); pRequest->metric.start = taosGetTimestampUs(); diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index c396ab40ac..63bb3637cf 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -13,7 +13,6 @@ static int32_t initEpSetFromCfg(const char* firstEp, const char* secondEp, SCorEpSet* pEpSet); static SMsgSendInfo* buildConnectMsg(SRequestObj* pRequest); static void destroySendMsgInfo(SMsgSendInfo* pMsgBody); -static int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp, bool convertUcs4); static bool stringLengthCheck(const char* str, size_t maxsize) { if (str == NULL) { @@ -42,7 +41,6 @@ static char* getClusterKey(const char* user, const char* auth, const char* ip, i static STscObj* taosConnectImpl(const char* user, const char* auth, const char* db, __taos_async_fn_t fp, void* param, SAppInstInfo* pAppInfo); -static void setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t numOfCols); TAOS* taos_connect_internal(const char* ip, const char* user, const char* pass, const char* auth, const char* db, uint16_t port) { @@ -174,7 +172,7 @@ int32_t parseSql(SRequestObj* pRequest, bool topicQuery, SQuery** pQuery) { int32_t execLocalCmd(SRequestObj* pRequest, SQuery* pQuery) { SRetrieveTableRsp* pRsp = NULL; - int32_t code = qExecCommand(pQuery->pRoot, &pRsp); + int32_t code = qExecCommand(pQuery->pRoot, &pRsp); if (TSDB_CODE_SUCCESS == code && NULL != pRsp) { code = setQueryResultFromRsp(&pRequest->body.resInfo, pRsp, false); } @@ -190,7 +188,7 @@ int32_t execDdlQuery(SRequestObj* pRequest, SQuery* pQuery) { SCmdMsgInfo* pMsgInfo = pQuery->pCmdMsg; pRequest->type = pMsgInfo->msgType; pRequest->body.requestMsg = (SDataBuf){.pData = pMsgInfo->pMsg, .len = pMsgInfo->msgLen, .handle = NULL}; - pMsgInfo->pMsg = NULL; // pMsg transferred to SMsgSendInfo management + pMsgInfo->pMsg = NULL; // pMsg transferred to SMsgSendInfo management STscObj* pTscObj = pRequest->pTscObj; SMsgSendInfo* pSendMsg = buildMsgInfoImpl(pRequest); @@ -211,14 +209,12 @@ int32_t execDdlQuery(SRequestObj* pRequest, SQuery* pQuery) { int32_t getPlan(SRequestObj* pRequest, SQuery* pQuery, SQueryPlan** pPlan, SArray* pNodeList) { pRequest->type = pQuery->msgType; - SPlanContext cxt = { - .queryId = pRequest->requestId, - .acctId = pRequest->pTscObj->acctId, - .mgmtEpSet = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp), - .pAstRoot = pQuery->pRoot, - .showRewrite = pQuery->showRewrite - }; - int32_t code = qCreateQueryPlan(&cxt, pPlan, pNodeList); + SPlanContext cxt = {.queryId = pRequest->requestId, + .acctId = pRequest->pTscObj->acctId, + .mgmtEpSet = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp), + .pAstRoot = pQuery->pRoot, + .showRewrite = pQuery->showRewrite}; + int32_t code = qCreateQueryPlan(&cxt, pPlan, pNodeList); if (code != 0) { return code; } @@ -234,10 +230,10 @@ void setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t for (int32_t i = 0; i < pResInfo->numOfCols; ++i) { pResInfo->fields[i].bytes = pSchema[i].bytes; - pResInfo->fields[i].type = pSchema[i].type; + pResInfo->fields[i].type = pSchema[i].type; pResInfo->userFields[i].bytes = pSchema[i].bytes; - pResInfo->userFields[i].type = pSchema[i].type; + pResInfo->userFields[i].type = pSchema[i].type; if (pSchema[i].type == TSDB_DATA_TYPE_VARCHAR) { pResInfo->userFields[i].bytes -= VARSTR_HEADER_SIZE; @@ -254,7 +250,8 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList void* pTransporter = pRequest->pTscObj->pAppInfo->pTransporter; SQueryResult res = {.code = 0, .numOfRows = 0, .msgSize = ERROR_MSG_BUF_DEFAULT_SIZE, .msg = pRequest->msgBuf}; - int32_t code = schedulerExecJob(pTransporter, pNodeList, pDag, &pRequest->body.queryJob, pRequest->sqlstr, pRequest->metric.start, &res); + int32_t code = schedulerExecJob(pTransporter, pNodeList, pDag, &pRequest->body.queryJob, pRequest->sqlstr, + pRequest->metric.start, &res); if (code != TSDB_CODE_SUCCESS) { if (pRequest->body.queryJob != 0) { schedulerFreeJob(pRequest->body.queryJob); @@ -274,14 +271,14 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList } pRequest->code = res.code; - terrno = res.code; + terrno = res.code; return pRequest->code; } SRequestObj* execQueryImpl(STscObj* pTscObj, const char* sql, int sqlLen) { SRequestObj* pRequest = NULL; - SQuery* pQuery = NULL; - SArray* pNodeList = taosArrayInit(4, sizeof(struct SQueryNodeAddr)); + SQuery* pQuery = NULL; + SArray* pNodeList = taosArrayInit(4, sizeof(struct SQueryNodeAddr)); int32_t code = buildRequest(pTscObj, sql, sqlLen, &pRequest); if (TSDB_CODE_SUCCESS == code) { @@ -320,15 +317,15 @@ SRequestObj* execQueryImpl(STscObj* pTscObj, const char* sql, int sqlLen) { } int32_t refreshMeta(STscObj* pTscObj, SRequestObj* pRequest) { - SCatalog *pCatalog = NULL; - int32_t code = 0; - int32_t dbNum = taosArrayGetSize(pRequest->dbList); - int32_t tblNum = taosArrayGetSize(pRequest->tableList); + SCatalog* pCatalog = NULL; + int32_t code = 0; + int32_t dbNum = taosArrayGetSize(pRequest->dbList); + int32_t tblNum = taosArrayGetSize(pRequest->tableList); if (dbNum <= 0 && tblNum <= 0) { return TSDB_CODE_QRY_APP_ERROR; } - + code = catalogGetHandle(pTscObj->pAppInfo->clusterId, &pCatalog); if (code != TSDB_CODE_SUCCESS) { return code; @@ -337,8 +334,8 @@ int32_t refreshMeta(STscObj* pTscObj, SRequestObj* pRequest) { SEpSet epset = getEpSet_s(&pTscObj->pAppInfo->mgmtEp); for (int32_t i = 0; i < dbNum; ++i) { - char *dbFName = taosArrayGet(pRequest->dbList, i); - + char* dbFName = taosArrayGet(pRequest->dbList, i); + code = catalogRefreshDBVgInfo(pCatalog, pTscObj->pAppInfo->pTransporter, &epset, dbFName); if (code != TSDB_CODE_SUCCESS) { return code; @@ -346,7 +343,7 @@ int32_t refreshMeta(STscObj* pTscObj, SRequestObj* pRequest) { } for (int32_t i = 0; i < tblNum; ++i) { - SName *tableName = taosArrayGet(pRequest->tableList, i); + SName* tableName = taosArrayGet(pRequest->tableList, i); code = catalogRefreshTableMeta(pCatalog, pTscObj->pAppInfo->pTransporter, &epset, tableName, -1); if (code != TSDB_CODE_SUCCESS) { @@ -357,11 +354,10 @@ int32_t refreshMeta(STscObj* pTscObj, SRequestObj* pRequest) { return code; } - SRequestObj* execQuery(STscObj* pTscObj, const char* sql, int sqlLen) { SRequestObj* pRequest = NULL; - int32_t retryNum = 0; - int32_t code = 0; + int32_t retryNum = 0; + int32_t code = 0; while (retryNum++ < REQUEST_MAX_TRY_TIMES) { pRequest = execQueryImpl(pTscObj, sql, sqlLen); @@ -377,7 +373,7 @@ SRequestObj* execQuery(STscObj* pTscObj, const char* sql, int sqlLen) { destroyRequest(pRequest); } - + return pRequest; } @@ -509,7 +505,8 @@ static void destroySendMsgInfo(SMsgSendInfo* pMsgBody) { } bool persistConnForSpecificMsg(void* parenct, tmsg_t msgType) { - return msgType == TDMT_VND_QUERY_RSP || msgType == TDMT_VND_FETCH_RSP || msgType == TDMT_VND_RES_READY_RSP || msgType == TDMT_VND_QUERY_HEARTBEAT_RSP; + return msgType == TDMT_VND_QUERY_RSP || msgType == TDMT_VND_FETCH_RSP || msgType == TDMT_VND_RES_READY_RSP || + msgType == TDMT_VND_QUERY_HEARTBEAT_RSP; } void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) { @@ -536,10 +533,10 @@ void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) { int32_t elapsed = pRequest->metric.rsp - pRequest->metric.start; if (pMsg->code == TSDB_CODE_SUCCESS) { tscDebug("0x%" PRIx64 " message:%s, code:%s rspLen:%d, elapsed:%d ms, reqId:0x%" PRIx64, pRequest->self, - TMSG_INFO(pMsg->msgType), tstrerror(pMsg->code), pMsg->contLen, elapsed/1000, pRequest->requestId); + TMSG_INFO(pMsg->msgType), tstrerror(pMsg->code), pMsg->contLen, elapsed / 1000, pRequest->requestId); } else { tscError("0x%" PRIx64 " SQL cmd:%s, code:%s rspLen:%d, elapsed time:%d ms, reqId:0x%" PRIx64, pRequest->self, - TMSG_INFO(pMsg->msgType), tstrerror(pMsg->code), pMsg->contLen, elapsed/1000, pRequest->requestId); + TMSG_INFO(pMsg->msgType), tstrerror(pMsg->code), pMsg->contLen, elapsed / 1000, pRequest->requestId); } taosReleaseRef(clientReqRefPool, pSendInfo->requestObjRefId); @@ -590,7 +587,7 @@ TAOS* taos_connect_l(const char* ip, int ipLen, const char* user, int userLen, c return taos_connect(ipStr, userStr, passStr, dbStr, port); } -static void doSetOneRowPtr(SReqResultInfo* pResultInfo) { +void doSetOneRowPtr(SReqResultInfo* pResultInfo) { for (int32_t i = 0; i < pResultInfo->numOfCols; ++i) { SResultColumn* pCol = &pResultInfo->pCol[i]; @@ -722,8 +719,8 @@ _return: static int32_t doPrepareResPtr(SReqResultInfo* pResInfo) { if (pResInfo->row == NULL) { - pResInfo->row = taosMemoryCalloc(pResInfo->numOfCols, POINTER_BYTES); - pResInfo->pCol = taosMemoryCalloc(pResInfo->numOfCols, sizeof(SResultColumn)); + pResInfo->row = taosMemoryCalloc(pResInfo->numOfCols, POINTER_BYTES); + pResInfo->pCol = taosMemoryCalloc(pResInfo->numOfCols, sizeof(SResultColumn)); pResInfo->length = taosMemoryCalloc(pResInfo->numOfCols, sizeof(int32_t)); pResInfo->convertBuf = taosMemoryCalloc(pResInfo->numOfCols, POINTER_BYTES); @@ -740,7 +737,7 @@ static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t numOfRows, int int32_t type = pResultInfo->fields[i].type; int32_t bytes = pResultInfo->fields[i].bytes; - if (type == TSDB_DATA_TYPE_NCHAR) { + if (type == TSDB_DATA_TYPE_NCHAR && colLength[i] > 0) { char* p = taosMemoryRealloc(pResultInfo->convertBuf[i], colLength[i]); if (p == NULL) { return TSDB_CODE_OUT_OF_MEMORY; @@ -770,7 +767,8 @@ static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t numOfRows, int return TSDB_CODE_SUCCESS; } -int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32_t numOfCols, int32_t numOfRows, bool convertUcs4) { +int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32_t numOfCols, int32_t numOfRows, + bool convertUcs4) { assert(numOfCols > 0 && pFields != NULL && pResultInfo != NULL); if (numOfRows == 0) { return TSDB_CODE_SUCCESS; @@ -841,15 +839,16 @@ void resetConnectDB(STscObj* pTscObj) { int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp, bool convertUcs4) { assert(pResultInfo != NULL && pRsp != NULL); - pResultInfo->pRspMsg = (const char*)pRsp; - pResultInfo->pData = (void*)pRsp->data; - pResultInfo->numOfRows = htonl(pRsp->numOfRows); - pResultInfo->current = 0; - pResultInfo->completed = (pRsp->completed == 1); + pResultInfo->pRspMsg = (const char*)pRsp; + pResultInfo->pData = (void*)pRsp->data; + pResultInfo->numOfRows = htonl(pRsp->numOfRows); + pResultInfo->current = 0; + pResultInfo->completed = (pRsp->completed == 1); pResultInfo->payloadLen = htonl(pRsp->compLen); - pResultInfo->precision = pRsp->precision; + pResultInfo->precision = pRsp->precision; // TODO handle the compressed case pResultInfo->totalRows += pResultInfo->numOfRows; - return setResultDataPtr(pResultInfo, pResultInfo->fields, pResultInfo->numOfCols, pResultInfo->numOfRows, convertUcs4); + return setResultDataPtr(pResultInfo, pResultInfo->fields, pResultInfo->numOfCols, pResultInfo->numOfRows, + convertUcs4); } diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 51629510d0..040ddde630 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -71,7 +71,7 @@ void taos_cleanup(void) { tscInfo("all local resources released"); } -setConfRet taos_set_config(const char *config) { +setConfRet taos_set_config(const char *config) { // TODO setConfRet ret = {SET_CONF_RET_SUCC, {0}}; return ret; @@ -133,8 +133,7 @@ int taos_field_count(TAOS_RES *res) { return 0; } - SRequestObj *pRequest = (SRequestObj *)res; - SReqResultInfo *pResInfo = &pRequest->body.resInfo; + SReqResultInfo *pResInfo = tscGetCurResInfo(res); return pResInfo->numOfCols; } @@ -145,7 +144,7 @@ TAOS_FIELD *taos_fetch_fields(TAOS_RES *res) { return NULL; } - SReqResultInfo *pResInfo = &(((SRequestObj *)res)->body.resInfo); + SReqResultInfo *pResInfo = tscGetCurResInfo(res); return pResInfo->userFields; } @@ -162,13 +161,36 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) { return NULL; } - SRequestObj *pRequest = (SRequestObj *)res; - if (pRequest->type == TSDB_SQL_RETRIEVE_EMPTY_RESULT || pRequest->type == TSDB_SQL_INSERT || - pRequest->code != TSDB_CODE_SUCCESS || taos_num_fields(res) == 0) { - return NULL; - } + if (TD_RES_QUERY(res)) { + SRequestObj *pRequest = (SRequestObj *)res; + if (pRequest->type == TSDB_SQL_RETRIEVE_EMPTY_RESULT || pRequest->type == TSDB_SQL_INSERT || + pRequest->code != TSDB_CODE_SUCCESS || taos_num_fields(res) == 0) { + return NULL; + } - return doFetchRow(pRequest, true, true); + return doFetchRow(pRequest, true, true); + + } else if (TD_RES_TMQ(res)) { + SMqRspObj *msg = ((SMqRspObj *)res); + SReqResultInfo *pResultInfo = taosArrayGet(msg->res, msg->resIter); + + doSetOneRowPtr(pResultInfo); + pResultInfo->current += 1; + + if (pResultInfo->row == NULL) { + msg->resIter++; + pResultInfo = taosArrayGet(msg->res, msg->resIter); + doSetOneRowPtr(pResultInfo); + pResultInfo->current += 1; + } + + return pResultInfo->row; + + } else { + // assert to avoid uninitialization error + ASSERT(0); + } + return NULL; } int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields) { @@ -260,12 +282,12 @@ int *taos_fetch_lengths(TAOS_RES *res) { return NULL; } - return ((SRequestObj *)res)->body.resInfo.length; + SReqResultInfo *pResInfo = tscGetCurResInfo(res); + return pResInfo->length; } TAOS_ROW *taos_result_block(TAOS_RES *res) { - SRequestObj* pRequest = (SRequestObj*) res; - if (pRequest == NULL) { + if (res == NULL) { terrno = TSDB_CODE_INVALID_PARA; return NULL; } @@ -274,7 +296,8 @@ TAOS_ROW *taos_result_block(TAOS_RES *res) { return NULL; } - return &pRequest->body.resInfo.row; + SReqResultInfo *pResInfo = tscGetCurResInfo(res); + return &pResInfo->row; } // todo intergrate with tDataTypes @@ -313,7 +336,7 @@ const char *taos_data_type(int type) { const char *taos_get_client_info() { return version; } int taos_affected_rows(TAOS_RES *res) { - if (res == NULL) { + if (res == NULL || TD_RES_TMQ(res)) { return 0; } @@ -323,12 +346,17 @@ int taos_affected_rows(TAOS_RES *res) { } int taos_result_precision(TAOS_RES *res) { - SRequestObj* pRequest = (SRequestObj*) res; - if (pRequest == NULL) { + if (res == NULL) { return TSDB_TIME_PRECISION_MILLI; } - - return pRequest->body.resInfo.precision; + if (TD_RES_QUERY(res)) { + SRequestObj *pRequest = (SRequestObj *)res; + return pRequest->body.resInfo.precision; + } else if (TD_RES_TMQ(res)) { + SReqResultInfo *info = tmqGetCurResInfo(res); + return info->precision; + } + return TSDB_TIME_PRECISION_MILLI; } int taos_select_db(TAOS *taos, const char *db) { @@ -370,90 +398,115 @@ void taos_stop_query(TAOS_RES *res) { } bool taos_is_null(TAOS_RES *res, int32_t row, int32_t col) { - SRequestObj *pRequestObj = res; - SReqResultInfo *pResultInfo = &pRequestObj->body.resInfo; + SReqResultInfo *pResultInfo = tscGetCurResInfo(res); if (col >= pResultInfo->numOfCols || col < 0 || row >= pResultInfo->numOfRows || row < 0) { return true; } - SResultColumn *pCol = &pRequestObj->body.resInfo.pCol[col]; + SResultColumn *pCol = &pResultInfo->pCol[col]; return colDataIsNull_f(pCol->nullbitmap, row); } -bool taos_is_update_query(TAOS_RES *res) { - return taos_num_fields(res) == 0; -} +bool taos_is_update_query(TAOS_RES *res) { return taos_num_fields(res) == 0; } int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows) { int32_t numOfRows = 0; - /*int32_t code = */taos_fetch_block_s(res, &numOfRows, rows); + /*int32_t code = */ taos_fetch_block_s(res, &numOfRows, rows); return numOfRows; } -int taos_fetch_block_s(TAOS_RES *res, int* numOfRows, TAOS_ROW *rows) { - SRequestObj *pRequest = (SRequestObj *)res; - if (pRequest == NULL) { +int taos_fetch_block_s(TAOS_RES *res, int *numOfRows, TAOS_ROW *rows) { + if (res == NULL) { return 0; } + if (TD_RES_QUERY(res)) { + SRequestObj *pRequest = (SRequestObj *)res; - (*rows) = NULL; - (*numOfRows) = 0; + (*rows) = NULL; + (*numOfRows) = 0; - if (pRequest->type == TSDB_SQL_RETRIEVE_EMPTY_RESULT || pRequest->type == TSDB_SQL_INSERT || - pRequest->code != TSDB_CODE_SUCCESS || taos_num_fields(res) == 0) { + if (pRequest->type == TSDB_SQL_RETRIEVE_EMPTY_RESULT || pRequest->type == TSDB_SQL_INSERT || + pRequest->code != TSDB_CODE_SUCCESS || taos_num_fields(res) == 0) { + return 0; + } + + doFetchRow(pRequest, false, true); + + // TODO refactor + SReqResultInfo *pResultInfo = &pRequest->body.resInfo; + pResultInfo->current = pResultInfo->numOfRows; + + (*rows) = pResultInfo->row; + (*numOfRows) = pResultInfo->numOfRows; + return pRequest->code; + } else if (TD_RES_TMQ(res)) { + SReqResultInfo *pResultInfo = tmqGetNextResInfo(res); + if (pResultInfo == NULL) return -1; + + pResultInfo->current = pResultInfo->numOfRows; + (*rows) = pResultInfo->row; + (*numOfRows) = pResultInfo->numOfRows; return 0; + } else { + ASSERT(0); + return -1; } - - doFetchRow(pRequest, false, true); - - // TODO refactor - SReqResultInfo *pResultInfo = &pRequest->body.resInfo; - pResultInfo->current = pResultInfo->numOfRows; - - (*rows) = pResultInfo->row; - (*numOfRows) = pResultInfo->numOfRows; - return pRequest->code; } -int taos_fetch_raw_block(TAOS_RES *res, int* numOfRows, void** pData) { - SRequestObj *pRequest = (SRequestObj *)res; - if (pRequest == NULL) { +int taos_fetch_raw_block(TAOS_RES *res, int *numOfRows, void **pData) { + if (res == NULL) { return 0; } + if (TD_RES_QUERY(res)) { + SRequestObj *pRequest = (SRequestObj *)res; + + if (pRequest->type == TSDB_SQL_RETRIEVE_EMPTY_RESULT || pRequest->type == TSDB_SQL_INSERT || + pRequest->code != TSDB_CODE_SUCCESS || taos_num_fields(res) == 0) { + return 0; + } + + doFetchRow(pRequest, false, false); + + SReqResultInfo *pResultInfo = &pRequest->body.resInfo; + + pResultInfo->current = pResultInfo->numOfRows; + (*numOfRows) = pResultInfo->numOfRows; + (*pData) = (void *)pResultInfo->pData; - if (pRequest->type == TSDB_SQL_RETRIEVE_EMPTY_RESULT || pRequest->type == TSDB_SQL_INSERT || - pRequest->code != TSDB_CODE_SUCCESS || taos_num_fields(res) == 0) { return 0; + + } else if (TD_RES_TMQ(res)) { + SReqResultInfo *pResultInfo = tmqGetNextResInfo(res); + if (pResultInfo == NULL) return -1; + + pResultInfo->current = pResultInfo->numOfRows; + (*numOfRows) = pResultInfo->numOfRows; + (*pData) = (void *)pResultInfo->pData; + return 0; + + } else { + ASSERT(0); + return -1; } - - doFetchRow(pRequest, false, false); - - SReqResultInfo *pResultInfo = &pRequest->body.resInfo; - - pResultInfo->current = pResultInfo->numOfRows; - (*numOfRows) = pResultInfo->numOfRows; - (*pData) = (void*) pResultInfo->pData; - - return 0; } int *taos_get_column_data_offset(TAOS_RES *res, int columnIndex) { - SRequestObj *pRequest = (SRequestObj *)res; - if (pRequest == NULL) { + if (res == NULL) { return 0; } - int32_t numOfFields = taos_num_fields(pRequest); + int32_t numOfFields = taos_num_fields(res); if (columnIndex < 0 || columnIndex >= numOfFields || numOfFields == 0) { return 0; } - TAOS_FIELD* pField = &pRequest->body.resInfo.userFields[columnIndex]; + SReqResultInfo *pResInfo = tscGetCurResInfo(res); + TAOS_FIELD *pField = &pResInfo->userFields[columnIndex]; if (!IS_VAR_DATA_TYPE(pField->type)) { return 0; } - return pRequest->body.resInfo.pCol[columnIndex].offset; + return pResInfo->pCol[columnIndex].offset; } int taos_validate_sql(TAOS *taos, const char *sql) { return true; } @@ -483,18 +536,19 @@ void taos_fetch_rows_a(TAOS_RES *res, __taos_async_fn_t fp, void *param) { // TODO } -TAOS_SUB *taos_subscribe(TAOS *taos, int restart, const char* topic, const char *sql, TAOS_SUBSCRIBE_CALLBACK fp, void *param, int interval) { - // TODO - return NULL; +TAOS_SUB *taos_subscribe(TAOS *taos, int restart, const char *topic, const char *sql, TAOS_SUBSCRIBE_CALLBACK fp, + void *param, int interval) { + // TODO + return NULL; } TAOS_RES *taos_consume(TAOS_SUB *tsub) { - // TODO - return NULL; + // TODO + return NULL; } void taos_unsubscribe(TAOS_SUB *tsub, int keepProgress) { - // TODO + // TODO } int taos_load_table_info(TAOS *taos, const char *tableNameList) { @@ -553,26 +607,26 @@ int taos_stmt_set_tbname(TAOS_STMT *stmt, const char *name) { } int taos_stmt_is_insert(TAOS_STMT *stmt, int *insert) { - // TODO - return -1; + // TODO + return -1; } int taos_stmt_num_params(TAOS_STMT *stmt, int *nums) { - // TODO - return -1; + // TODO + return -1; } -int taos_stmt_add_batch(TAOS_STMT* stmt) { - // TODO - return -1; +int taos_stmt_add_batch(TAOS_STMT *stmt) { + // TODO + return -1; } TAOS_RES *taos_stmt_use_result(TAOS_STMT *stmt) { - // TODO - return NULL; + // TODO + return NULL; } -int taos_stmt_bind_param_batch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind) { - // TODO - return -1; +int taos_stmt_bind_param_batch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind) { + // TODO + return -1; } diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index 955e25fd71..dbe78782f5 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -17,25 +17,19 @@ #include "clientLog.h" #include "parser.h" #include "planner.h" -#include "scheduler.h" #include "tdatablock.h" #include "tdef.h" #include "tglobal.h" #include "tmsgtype.h" -#include "tpagedbuf.h" #include "tqueue.h" #include "tref.h" -typedef struct { - int32_t curBlock; - int32_t curRow; - void** uData; -} SMqRowIter; - struct tmq_message_t { SMqPollRsp msg; + char* topic; void* vg; - SMqRowIter iter; + SArray* res; // SArray + int32_t resIter; }; struct tmq_list_t { @@ -849,7 +843,8 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) { if (msgEpoch < tmqEpoch) { /*printf("discard rsp epoch %d, current epoch %d\n", msgEpoch, tmqEpoch);*/ /*tsem_post(&tmq->rspSem);*/ - tscWarn("msg discard from vg %d since from earlier epoch, rsp epoch %d, current epoch %d", pParam->vgId, msgEpoch, tmqEpoch); + tscWarn("msg discard from vg %d since from earlier epoch, rsp epoch %d, current epoch %d", pParam->vgId, msgEpoch, + tmqEpoch); return 0; } @@ -886,8 +881,8 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) { } memcpy(pRsp, pMsg->pData, sizeof(SMqRspHead)); tDecodeSMqPollRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pRsp->msg); - pRsp->iter.curBlock = 0; - pRsp->iter.curRow = 0; + /*pRsp->iter.curBlock = 0;*/ + /*pRsp->iter.curRow = 0;*/ // TODO: alloc mem /*pRsp->*/ /*printf("rsp commit off:%ld rsp off:%ld has data:%d\n", pRsp->committedOffset, pRsp->rspOffset, pRsp->numOfTopics);*/ @@ -899,8 +894,8 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) { } #endif - tscDebug("consumer %ld recv poll: vg %d, req offset %ld, rsp offset %ld", tmq->consumerId, pParam->pVg->vgId, pRsp->msg.reqOffset, - pRsp->msg.rspOffset); + tscDebug("consumer %ld recv poll: vg %d, req offset %ld, rsp offset %ld", tmq->consumerId, pParam->pVg->vgId, + pRsp->msg.reqOffset, pRsp->msg.rspOffset); pRsp->vg = pParam->pVg; taosWriteQitem(tmq->mqueue, pRsp); @@ -921,7 +916,8 @@ bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqCMGetSubEpRsp* pRsp) { bool set = false; int32_t topicNumGet = taosArrayGetSize(pRsp->topics); char vgKey[TSDB_TOPIC_FNAME_LEN + 22]; - tscDebug("consumer %ld update ep epoch %d to epoch %d, topic num: %d", tmq->consumerId, tmq->epoch, epoch, topicNumGet); + tscDebug("consumer %ld update ep epoch %d to epoch %d, topic num: %d", tmq->consumerId, tmq->epoch, epoch, + topicNumGet); SArray* newTopics = taosArrayInit(topicNumGet, sizeof(SMqClientTopic)); if (newTopics == NULL) { return false; @@ -1289,7 +1285,8 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) { int64_t transporterId = 0; /*printf("send poll\n");*/ atomic_add_fetch_32(&tmq->waitingRequest, 1); - tscDebug("consumer %ld send poll to %s : vg %d, epoch %d, req offset %ld, reqId %lu", tmq->consumerId, pTopic->topicName, pVg->vgId, tmq->epoch, pVg->currentOffset, pReq->reqId); + tscDebug("consumer %ld send poll to %s : vg %d, epoch %d, req offset %ld, reqId %lu", tmq->consumerId, + pTopic->topicName, pVg->vgId, tmq->epoch, pVg->currentOffset, pReq->reqId); /*printf("send vg %d %ld\n", pVg->vgId, pVg->currentOffset);*/ asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, sendInfo); pVg->pollCnt++; @@ -1566,30 +1563,4 @@ const char* tmq_err2str(tmq_resp_err_t err) { return "fail"; } -TAOS_ROW tmq_get_row(tmq_message_t* message) { - SMqPollRsp* rsp = &message->msg; - while (1) { - if (message->iter.curBlock < taosArrayGetSize(rsp->pBlockData)) { - SSDataBlock* pBlock = taosArrayGet(rsp->pBlockData, message->iter.curBlock); - if (message->iter.curRow < pBlock->info.rows) { - for (int i = 0; i < pBlock->info.numOfCols; i++) { - SColumnInfoData* pData = taosArrayGet(pBlock->pDataBlock, i); - if (colDataIsNull_s(pData, message->iter.curRow)) - message->iter.uData[i] = NULL; - else { - message->iter.uData[i] = colDataGetData(pData, message->iter.curRow); - } - } - message->iter.curRow++; - return message->iter.uData; - } else { - message->iter.curBlock++; - message->iter.curRow = 0; - continue; - } - } - return NULL; - } -} - char* tmq_get_topic_name(tmq_message_t* message) { return "not implemented yet"; } diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 0bc1fa09f5..df4433e6e4 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -256,7 +256,7 @@ static int32_t taosLoadCfg(SConfig *pCfg, const char *inputCfgDir, const char *e return 0; } -static int32_t taosAddClientLogCfg(SConfig *pCfg) { +int32_t taosAddClientLogCfg(SConfig *pCfg) { if (cfgAddDir(pCfg, "configDir", configDir, 1) != 0) return -1; if (cfgAddDir(pCfg, "scriptDir", configDir, 1) != 0) return -1; if (cfgAddDir(pCfg, "logDir", tsLogDir, 1) != 0) return -1; diff --git a/source/common/src/tvariant.c b/source/common/src/tvariant.c index 8c5010e577..a0503e43a1 100644 --- a/source/common/src/tvariant.c +++ b/source/common/src/tvariant.c @@ -1028,13 +1028,18 @@ int32_t taosVariantTypeSetType(SVariant *pVariant, char type) { char * taosVariantGet(SVariant *pVar, int32_t type) { switch (type) { - case TSDB_DATA_TYPE_BOOL: + case TSDB_DATA_TYPE_BOOL: case TSDB_DATA_TYPE_TINYINT: case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_INT: case TSDB_DATA_TYPE_BIGINT: - case TSDB_DATA_TYPE_INT: case TSDB_DATA_TYPE_TIMESTAMP: return (char *)&pVar->i; + case TSDB_DATA_TYPE_UTINYINT: + case TSDB_DATA_TYPE_USMALLINT: + case TSDB_DATA_TYPE_UINT: + case TSDB_DATA_TYPE_UBIGINT: + return (char *)&pVar->u; case TSDB_DATA_TYPE_DOUBLE: case TSDB_DATA_TYPE_FLOAT: return (char *)&pVar->d; @@ -1042,7 +1047,7 @@ char * taosVariantGet(SVariant *pVar, int32_t type) { return (char *)pVar->pz; case TSDB_DATA_TYPE_NCHAR: return (char *)pVar->ucs4; - default: + default: return NULL; } diff --git a/source/dnode/mgmt/bm/bmHandle.c b/source/dnode/mgmt/bm/bmHandle.c index 645a1d09c2..d110592603 100644 --- a/source/dnode/mgmt/bm/bmHandle.c +++ b/source/dnode/mgmt/bm/bmHandle.c @@ -58,7 +58,7 @@ int32_t bmProcessCreateReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { dError("failed to create bnode since %s, input:%d cur:%d", terrstr(), createReq.dnodeId, pDnode->dnodeId); return -1; } else { - return bmOpen(pWrapper); + return dndOpenNode(pWrapper); } } @@ -77,6 +77,7 @@ int32_t bmProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { dError("failed to drop bnode since %s", terrstr()); return -1; } else { + // dndCloseNode(pWrapper); return bmDrop(pWrapper); } } diff --git a/source/dnode/mgmt/dm/dmMonitor.c b/source/dnode/mgmt/dm/dmMonitor.c index fb5855070c..6edf85106b 100644 --- a/source/dnode/mgmt/dm/dmMonitor.c +++ b/source/dnode/mgmt/dm/dmMonitor.c @@ -25,11 +25,10 @@ static void dmGetMonitorBasicInfo(SDnode *pDnode, SMonBasicInfo *pInfo) { static void dmGetMonitorDnodeInfo(SDnode *pDnode, SMonDnodeInfo *pInfo) { pInfo->uptime = (taosGetTimestampMs() - pDnode->rebootTime) / (86400000.0f); - SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, MNODE); - if (pWrapper != NULL) { - pInfo->has_mnode = pWrapper->required; - dndReleaseWrapper(pWrapper); - } + pInfo->has_mnode = pDnode->wrappers[MNODE].required; + pInfo->has_qnode = pDnode->wrappers[QNODE].required; + pInfo->has_snode = pDnode->wrappers[SNODE].required; + pInfo->has_bnode = pDnode->wrappers[BNODE].required; tstrncpy(pInfo->logdir.name, tsLogDir, sizeof(pInfo->logdir.name)); pInfo->logdir.size = tsLogSpace.size; tstrncpy(pInfo->tempdir.name, tsTempDir, sizeof(pInfo->tempdir.name)); @@ -65,7 +64,7 @@ void dmSendMonitorReport(SDnode *pDnode) { bool getFromAPI = !tsMultiProcess; pWrapper = &pDnode->wrappers[MNODE]; if (getFromAPI) { - if (dndMarkWrapper(pWrapper) != 0) { + if (dndMarkWrapper(pWrapper) == 0) { mmGetMonitorInfo(pWrapper, &mmInfo); dndReleaseWrapper(pWrapper); } @@ -82,7 +81,7 @@ void dmSendMonitorReport(SDnode *pDnode) { pWrapper = &pDnode->wrappers[VNODES]; if (getFromAPI) { - if (dndMarkWrapper(pWrapper) != 0) { + if (dndMarkWrapper(pWrapper) == 0) { vmGetMonitorInfo(pWrapper, &vmInfo); dndReleaseWrapper(pWrapper); } @@ -99,7 +98,7 @@ void dmSendMonitorReport(SDnode *pDnode) { pWrapper = &pDnode->wrappers[QNODE]; if (getFromAPI) { - if (dndMarkWrapper(pWrapper) != 0) { + if (dndMarkWrapper(pWrapper) == 0) { qmGetMonitorInfo(pWrapper, &qmInfo); dndReleaseWrapper(pWrapper); } @@ -116,7 +115,7 @@ void dmSendMonitorReport(SDnode *pDnode) { pWrapper = &pDnode->wrappers[SNODE]; if (getFromAPI) { - if (dndMarkWrapper(pWrapper) != 0) { + if (dndMarkWrapper(pWrapper) == 0) { smGetMonitorInfo(pWrapper, &smInfo); dndReleaseWrapper(pWrapper); } @@ -133,7 +132,7 @@ void dmSendMonitorReport(SDnode *pDnode) { pWrapper = &pDnode->wrappers[BNODE]; if (getFromAPI) { - if (dndMarkWrapper(pWrapper) != 0) { + if (dndMarkWrapper(pWrapper) == 0) { bmGetMonitorInfo(pWrapper, &bmInfo); dndReleaseWrapper(pWrapper); } diff --git a/source/dnode/mgmt/inc/bmInt.h b/source/dnode/mgmt/inc/bmInt.h index 84a6a53e99..3158fe7d34 100644 --- a/source/dnode/mgmt/inc/bmInt.h +++ b/source/dnode/mgmt/inc/bmInt.h @@ -27,7 +27,7 @@ extern "C" { typedef struct SBnodeMgmt { SBnode *pBnode; SDnode *pDnode; - SMgmtWrapper *pWrapper; + SMgmtWrapper *pWrapper; const char *path; SMultiWorker writeWorker; SSingleWorker monitorWorker; diff --git a/source/dnode/mgmt/inc/dndInt.h b/source/dnode/mgmt/inc/dndInt.h index 7faf1e4276..a38fe87b59 100644 --- a/source/dnode/mgmt/inc/dndInt.h +++ b/source/dnode/mgmt/inc/dndInt.h @@ -126,6 +126,7 @@ typedef struct SDnode { int32_t numOfDisks; uint16_t serverPort; bool dropped; + EProcType procType; EDndType ntype; EDndStatus status; EDndEvent event; diff --git a/source/dnode/mgmt/main/dndExec.c b/source/dnode/mgmt/main/dndExec.c index 6c0d0456c9..51569d2ed4 100644 --- a/source/dnode/mgmt/main/dndExec.c +++ b/source/dnode/mgmt/main/dndExec.c @@ -27,46 +27,42 @@ static bool dndRequireNode(SMgmtWrapper *pWrapper) { return required; } -int32_t dndOpenNode(SMgmtWrapper *pWrapper) { - if (taosMkDir(pWrapper->path) != 0) { - terrno = TAOS_SYSTEM_ERROR(errno); - dError("node:%s, failed to create dir:%s since %s", pWrapper->name, pWrapper->path, terrstr()); +static int32_t dndInitNodeProc(SMgmtWrapper *pWrapper) { + int32_t shmsize = tsMnodeShmSize; + if (pWrapper->ntype == VNODES) { + shmsize = tsVnodeShmSize; + } else if (pWrapper->ntype == QNODE) { + shmsize = tsQnodeShmSize; + } else if (pWrapper->ntype == SNODE) { + shmsize = tsSnodeShmSize; + } else if (pWrapper->ntype == MNODE) { + shmsize = tsMnodeShmSize; + } else if (pWrapper->ntype == BNODE) { + shmsize = tsBnodeShmSize; + } else { return -1; } - if ((*pWrapper->fp.openFp)(pWrapper) != 0) { - dError("node:%s, failed to open since %s", pWrapper->name, terrstr()); + if (taosCreateShm(&pWrapper->shm, pWrapper->ntype, shmsize) != 0) { + terrno = TAOS_SYSTEM_ERROR(terrno); + dError("node:%s, failed to create shm size:%d since %s", pWrapper->name, shmsize, terrstr()); + return -1; + } + dInfo("node:%s, shm:%d is created, size:%d", pWrapper->name, pWrapper->shm.id, shmsize); + + SProcCfg cfg = dndGenProcCfg(pWrapper); + cfg.isChild = false; + pWrapper->procType = PROC_PARENT; + pWrapper->pProc = taosProcInit(&cfg); + if (pWrapper->pProc == NULL) { + dError("node:%s, failed to create proc since %s", pWrapper->name, terrstr()); return -1; } - dDebug("node:%s, has been opened", pWrapper->name); - pWrapper->deployed = true; return 0; } -void dndCloseNode(SMgmtWrapper *pWrapper) { - dDebug("node:%s, mgmt start to close", pWrapper->name); - pWrapper->required = false; - taosWLockLatch(&pWrapper->latch); - if (pWrapper->deployed) { - (*pWrapper->fp.closeFp)(pWrapper); - pWrapper->deployed = false; - } - taosWUnLockLatch(&pWrapper->latch); - - while (pWrapper->refCount > 0) { - taosMsleep(10); - } - - if (pWrapper->pProc) { - taosProcCleanup(pWrapper->pProc); - pWrapper->pProc = NULL; - } - dDebug("node:%s, mgmt has been closed", pWrapper->name); -} - - -static int32_t dndNewProc(SMgmtWrapper *pWrapper, EDndType n) { +static int32_t dndNewNodeProc(SMgmtWrapper *pWrapper, EDndType n) { char tstr[8] = {0}; char *args[6] = {0}; snprintf(tstr, sizeof(tstr), "%d", n); @@ -88,6 +84,86 @@ static int32_t dndNewProc(SMgmtWrapper *pWrapper, EDndType n) { return 0; } +static int32_t dndRunNodeProc(SMgmtWrapper *pWrapper) { + if (pWrapper->pDnode->ntype == NODE_MAX) { + dInfo("node:%s, should be started manually", pWrapper->name); + } else { + if (dndNewNodeProc(pWrapper, pWrapper->ntype) != 0) { + return -1; + } + } + + if (taosProcRun(pWrapper->pProc) != 0) { + dError("node:%s, failed to run proc since %s", pWrapper->name, terrstr()); + return -1; + } + + return 0; +} + +static int32_t dndOpenNodeImp(SMgmtWrapper *pWrapper) { + if (taosMkDir(pWrapper->path) != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + dError("node:%s, failed to create dir:%s since %s", pWrapper->name, pWrapper->path, terrstr()); + return -1; + } + + if ((*pWrapper->fp.openFp)(pWrapper) != 0) { + dError("node:%s, failed to open since %s", pWrapper->name, terrstr()); + return -1; + } + + dDebug("node:%s, has been opened", pWrapper->name); + pWrapper->deployed = true; + return 0; +} + +int32_t dndOpenNode(SMgmtWrapper *pWrapper) { + SDnode *pDnode = pWrapper->pDnode; + if (pDnode->procType == PROC_SINGLE) { + return dndOpenNodeImp(pWrapper); + } else if (pDnode->procType == PROC_PARENT) { + if (dndInitNodeProc(pWrapper) != 0) return -1; + if (dndWriteShmFile(pDnode) != 0) return -1; + if (dndRunNodeProc(pWrapper) != 0) return -1; + } + return 0; +} + +static void dndCloseNodeImp(SMgmtWrapper *pWrapper) { + dDebug("node:%s, mgmt start to close", pWrapper->name); + pWrapper->required = false; + taosWLockLatch(&pWrapper->latch); + if (pWrapper->deployed) { + (*pWrapper->fp.closeFp)(pWrapper); + pWrapper->deployed = false; + } + taosWUnLockLatch(&pWrapper->latch); + + while (pWrapper->refCount > 0) { + taosMsleep(10); + } + + if (pWrapper->pProc) { + taosProcCleanup(pWrapper->pProc); + pWrapper->pProc = NULL; + } + dDebug("node:%s, mgmt has been closed", pWrapper->name); +} + +void dndCloseNode(SMgmtWrapper *pWrapper) { + if (pWrapper->pDnode->procType == PROC_PARENT) { + if (pWrapper->procId > 0 && taosProcExist(pWrapper->procId)) { + dInfo("node:%s, send kill signal to the child process:%d", pWrapper->name, pWrapper->procId); + taosKillProc(pWrapper->procId); + dInfo("node:%s, wait for child process:%d to stop", pWrapper->name, pWrapper->procId); + taosWaitProc(pWrapper->procId); + dInfo("node:%s, child process:%d is stopped", pWrapper->name, pWrapper->procId); + } + } + dndCloseNodeImp(pWrapper); +} + static void dndProcessProcHandle(void *handle) { dWarn("handle:%p, the child process dies and send an offline rsp", handle); SRpcMsg rpcMsg = {.handle = handle, .code = TSDB_CODE_NODE_OFFLINE}; @@ -96,13 +172,14 @@ static void dndProcessProcHandle(void *handle) { static int32_t dndRunInSingleProcess(SDnode *pDnode) { dInfo("dnode run in single process"); + pDnode->procType = PROC_SINGLE; for (EDndType n = DNODE; n < NODE_MAX; ++n) { SMgmtWrapper *pWrapper = &pDnode->wrappers[n]; pWrapper->required = dndRequireNode(pWrapper); if (!pWrapper->required) continue; - if (dndOpenNode(pWrapper) != 0) { + if (dndOpenNodeImp(pWrapper) != 0) { dError("node:%s, failed to start since %s", pWrapper->name, terrstr()); return -1; } @@ -136,8 +213,10 @@ static int32_t dndRunInSingleProcess(SDnode *pDnode) { static int32_t dndRunInParentProcess(SDnode *pDnode) { dInfo("dnode run in parent process"); + pDnode->procType = PROC_PARENT; + SMgmtWrapper *pDWrapper = &pDnode->wrappers[DNODE]; - if (dndOpenNode(pDWrapper) != 0) { + if (dndOpenNodeImp(pDWrapper) != 0) { dError("node:%s, failed to start since %s", pDWrapper->name, terrstr()); return -1; } @@ -146,36 +225,7 @@ static int32_t dndRunInParentProcess(SDnode *pDnode) { SMgmtWrapper *pWrapper = &pDnode->wrappers[n]; pWrapper->required = dndRequireNode(pWrapper); if (!pWrapper->required) continue; - - int32_t shmsize = tsMnodeShmSize; - if (n == VNODES) { - shmsize = tsVnodeShmSize; - } else if (n == QNODE) { - shmsize = tsQnodeShmSize; - } else if (n == SNODE) { - shmsize = tsSnodeShmSize; - } else if (n == MNODE) { - shmsize = tsMnodeShmSize; - } else if (n == BNODE) { - shmsize = tsBnodeShmSize; - } else { - } - - if (taosCreateShm(&pWrapper->shm, n, shmsize) != 0) { - terrno = TAOS_SYSTEM_ERROR(terrno); - dError("node:%s, failed to create shm size:%d since %s", pWrapper->name, shmsize, terrstr()); - return -1; - } - dInfo("node:%s, shm:%d is created, size:%d", pWrapper->name, pWrapper->shm.id, shmsize); - - SProcCfg cfg = dndGenProcCfg(pWrapper); - cfg.isChild = false; - pWrapper->procType = PROC_PARENT; - pWrapper->pProc = taosProcInit(&cfg); - if (pWrapper->pProc == NULL) { - dError("node:%s, failed to create proc since %s", pWrapper->name, terrstr()); - return -1; - } + if (dndInitNodeProc(pWrapper) != 0) return -1; } if (dndWriteShmFile(pDnode) != 0) { @@ -186,19 +236,7 @@ static int32_t dndRunInParentProcess(SDnode *pDnode) { for (EDndType n = DNODE + 1; n < NODE_MAX; ++n) { SMgmtWrapper *pWrapper = &pDnode->wrappers[n]; if (!pWrapper->required) continue; - - if (pDnode->ntype == NODE_MAX) { - dInfo("node:%s, should be started manually", pWrapper->name); - } else { - if (dndNewProc(pWrapper, n) != 0) { - return -1; - } - } - - if (taosProcRun(pWrapper->pProc) != 0) { - dError("node:%s, failed to run proc since %s", pWrapper->name, terrstr()); - return -1; - } + if (dndRunNodeProc(pWrapper) != 0) return -1; } dndSetStatus(pDnode, DND_STAT_RUNNING); @@ -239,7 +277,7 @@ static int32_t dndRunInParentProcess(SDnode *pDnode) { if (pWrapper->procId <= 0 || !taosProcExist(pWrapper->procId)) { dWarn("node:%s, process:%d is killed and needs to be restarted", pWrapper->name, pWrapper->procId); taosProcCloseHandles(pWrapper->pProc, dndProcessProcHandle); - dndNewProc(pWrapper, n); + dndNewNodeProc(pWrapper, n); } } } @@ -253,6 +291,7 @@ static int32_t dndRunInParentProcess(SDnode *pDnode) { static int32_t dndRunInChildProcess(SDnode *pDnode) { SMgmtWrapper *pWrapper = &pDnode->wrappers[pDnode->ntype]; dInfo("%s run in child process", pWrapper->name); + pDnode->procType = PROC_CHILD; pWrapper->required = dndRequireNode(pWrapper); if (!pWrapper->required) { @@ -264,7 +303,7 @@ static int32_t dndRunInChildProcess(SDnode *pDnode) { tmsgSetDefaultMsgCb(&msgCb); pWrapper->procType = PROC_CHILD; - if (dndOpenNode(pWrapper) != 0) { + if (dndOpenNodeImp(pWrapper) != 0) { dError("node:%s, failed to start since %s", pWrapper->name, terrstr()); return -1; } diff --git a/source/dnode/mgmt/mm/mmHandle.c b/source/dnode/mgmt/mm/mmHandle.c index 6ad0b8c0ed..63240c3224 100644 --- a/source/dnode/mgmt/mm/mmHandle.c +++ b/source/dnode/mgmt/mm/mmHandle.c @@ -80,6 +80,7 @@ int32_t mmProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { dError("failed to drop mnode since %s", terrstr()); return -1; } else { + // dndCloseNode(pWrapper); return mmDrop(pWrapper); } } diff --git a/source/dnode/mgmt/qm/qmHandle.c b/source/dnode/mgmt/qm/qmHandle.c index 96fc338529..4fda72759a 100644 --- a/source/dnode/mgmt/qm/qmHandle.c +++ b/source/dnode/mgmt/qm/qmHandle.c @@ -58,7 +58,7 @@ int32_t qmProcessCreateReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { dError("failed to create qnode since %s", terrstr()); return -1; } else { - return qmOpen(pWrapper); + return dndOpenNode(pWrapper); } } @@ -77,6 +77,7 @@ int32_t qmProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { dError("failed to drop qnode since %s", terrstr()); return -1; } else { + // dndCloseNode(pWrapper); return qmDrop(pWrapper); } } diff --git a/source/dnode/mgmt/qm/qmWorker.c b/source/dnode/mgmt/qm/qmWorker.c index 974052cdf6..6b27af4fbd 100644 --- a/source/dnode/mgmt/qm/qmWorker.c +++ b/source/dnode/mgmt/qm/qmWorker.c @@ -32,7 +32,7 @@ static void qmProcessMonitorQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) { SRpcMsg *pRpc = &pMsg->rpcMsg; int32_t code = -1; - if (pMsg->rpcMsg.msgType == TDMT_MON_SM_INFO) { + if (pMsg->rpcMsg.msgType == TDMT_MON_QM_INFO) { code = qmProcessGetMonQmInfoReq(pMgmt->pWrapper, pMsg); } else { terrno = TSDB_CODE_MSG_NOT_PROCESSED; diff --git a/source/dnode/mgmt/sm/smHandle.c b/source/dnode/mgmt/sm/smHandle.c index 36345cf490..5b30dc04bc 100644 --- a/source/dnode/mgmt/sm/smHandle.c +++ b/source/dnode/mgmt/sm/smHandle.c @@ -58,7 +58,7 @@ int32_t smProcessCreateReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { dError("failed to create snode since %s", terrstr()); return -1; } else { - return smOpen(pWrapper); + return dndOpenNode(pWrapper); } } @@ -78,6 +78,7 @@ int32_t smProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { return -1; } else { return smDrop(pWrapper); + // return dndCloseNode(pWrapper); } } diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index e308c9c2b9..b81872f20c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -1414,7 +1414,7 @@ static int32_t doCopyRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, int32_t } if (sVal.valType == TD_VTYPE_NULL) { - colDataAppend(pColInfo, k, NULL, true); + colDataAppendNULL(pColInfo, k); } else { colDataAppend(pColInfo, k, sVal.val, false); } @@ -1427,7 +1427,11 @@ static int32_t doCopyRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, int32_t TASSERT(0); } - colDataAppend(pColInfo, k, sVal.val, false); + if (sVal.valType == TD_VTYPE_NULL) { + colDataAppendNULL(pColInfo, k); + } else { + colDataAppend(pColInfo, k, sVal.val, false); + } } } diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c index 1edebcd7db..626cb1b5f0 100644 --- a/source/libs/executor/src/dataDispatcher.c +++ b/source/libs/executor/src/dataDispatcher.c @@ -92,9 +92,7 @@ static bool allocBuf(SDataDispatchHandle* pDispatcher, const SInputData* pInput, return false; } - // NOTE: there are four bytes of an integer more than the required buffer space. - // struct size + data payload + length for each column + bitmap length - pBuf->allocSize = sizeof(SRetrieveTableRsp) + blockDataGetSerialMetaSize(pInput->pData) + blockDataGetSize(pInput->pData); + pBuf->allocSize = sizeof(SRetrieveTableRsp) + blockEstimateEncodeSize(pInput->pData); pBuf->pData = taosMemoryMalloc(pBuf->allocSize); if (pBuf->pData == NULL) { diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index b86d75cc7b..b7bbb605c5 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -6682,38 +6682,6 @@ bool validateExprColumnInfo(SQueriedTableInfo* pTableInfo, SExprBasicInfo* pExpr return j != INT32_MIN; } -static int32_t deserializeColFilterInfo(SColumnFilterInfo* pColFilters, int16_t numOfFilters, char** pMsg) { - for (int32_t f = 0; f < numOfFilters; ++f) { - SColumnFilterInfo* pFilterMsg = (SColumnFilterInfo*)(*pMsg); - - SColumnFilterInfo* pColFilter = &pColFilters[f]; - pColFilter->filterstr = htons(pFilterMsg->filterstr); - - (*pMsg) += sizeof(SColumnFilterInfo); - - if (pColFilter->filterstr) { - pColFilter->len = htobe64(pFilterMsg->len); - - pColFilter->pz = - (int64_t)taosMemoryCalloc(1, (size_t)(pColFilter->len + 1 * TSDB_NCHAR_SIZE)); // note: null-terminator - if (pColFilter->pz == 0) { - return TSDB_CODE_QRY_OUT_OF_MEMORY; - } - - memcpy((void*)pColFilter->pz, (*pMsg), (size_t)pColFilter->len); - (*pMsg) += (pColFilter->len + 1); - } else { - pColFilter->lowerBndi = htobe64(pFilterMsg->lowerBndi); - pColFilter->upperBndi = htobe64(pFilterMsg->upperBndi); - } - - pColFilter->lowerRelOptr = htons(pFilterMsg->lowerRelOptr); - pColFilter->upperRelOptr = htons(pFilterMsg->upperRelOptr); - } - - return TSDB_CODE_SUCCESS; -} - static SResSchema createResSchema(int32_t type, int32_t bytes, int32_t slotId, int32_t scale, int32_t precision, const char* name) { SResSchema s = {0}; diff --git a/source/libs/function/inc/builtins.h b/source/libs/function/inc/builtins.h index f0349c55b9..ab5a02c438 100644 --- a/source/libs/function/inc/builtins.h +++ b/source/libs/function/inc/builtins.h @@ -41,12 +41,14 @@ extern "C" { #define FUNC_MGT_TEST_MASK(val, mask) (((val) & (mask)) != 0) typedef int32_t (*FCheckAndGetResultType)(SFunctionNode* pFunc); +typedef EFuncDataRequired (*FFuncDataRequired)(SFunctionNode* pFunc, STimeWindow* pTimeWindow); typedef struct SBuiltinFuncDefinition { char name[FUNCTION_NAME_MAX_LENGTH]; EFunctionType type; uint64_t classification; FCheckAndGetResultType checkFunc; + FFuncDataRequired dataRequiredFunc; FExecGetEnv getEnvFunc; FExecInit initFunc; FExecProcess processFunc; diff --git a/source/libs/function/inc/builtinsimpl.h b/source/libs/function/inc/builtinsimpl.h index 607bd279c1..09c468b610 100644 --- a/source/libs/function/inc/builtinsimpl.h +++ b/source/libs/function/inc/builtinsimpl.h @@ -21,10 +21,12 @@ extern "C" { #endif #include "function.h" +#include "functionMgt.h" bool functionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo); void functionFinalize(SqlFunctionCtx *pCtx); +EFuncDataRequired countDataRequired(SFunctionNode* pFunc, STimeWindow* pTimeWindow); bool getCountFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); int32_t countFunction(SqlFunctionCtx *pCtx); diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index b4d886dcdf..4e87725014 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -25,8 +25,9 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "count", .type = FUNCTION_TYPE_COUNT, - .classification = FUNC_MGT_AGG_FUNC, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SPECIAL_DATA_REQUIRED, .checkFunc = checkAndGetResultType, + .dataRequiredFunc = countDataRequired, .getEnvFunc = getCountFuncEnv, .initFunc = functionSetup, .processFunc = countFunction, @@ -389,7 +390,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .checkFunc = checkAndGetResultType, .getEnvFunc = NULL, .initFunc = NULL, - .sprocessFunc = NULL, + .sprocessFunc = castFunction, .finalizeFunc = NULL }, { @@ -599,7 +600,13 @@ int32_t checkAndGetResultType(SFunctionNode* pFunc) { break; } case FUNCTION_TYPE_CAST: { - pFunc->node.resType = (SDataType) { .bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes, .type = TSDB_DATA_TYPE_BIGINT }; + //type + SValueNode* pParam = nodesListGetNode(pFunc->pParameterList, 1); + int32_t paraType = pParam->datum.i; + //bytes + pParam = nodesListGetNode(pFunc->pParameterList, 2); + int32_t paraBytes = pParam->datum.i; + pFunc->node.resType = (SDataType) { .bytes = paraBytes, .type = paraType}; break; } diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 133e8f9c93..50c65439c0 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -55,6 +55,14 @@ void functionFinalize(SqlFunctionCtx *pCtx) { pResInfo->isNullRes = (pResInfo->numOfRes == 0)? 1:0; } +EFuncDataRequired countDataRequired(SFunctionNode* pFunc, STimeWindow* pTimeWindow) { + SNode* pParam = nodesListGetNode(pFunc->pParameterList, 0); + if (QUERY_NODE_COLUMN == nodeType(pParam) && PRIMARYKEY_TIMESTAMP_COL_ID == ((SColumnNode*)pParam)->colId) { + return FUNC_DATA_REQUIRED_NO_NEEDED; + } + return FUNC_DATA_REQUIRED_STATIS_NEEDED; +} + bool getCountFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) { pEnv->calcMemSize = sizeof(int64_t); return true; diff --git a/source/libs/function/src/functionMgt.c b/source/libs/function/src/functionMgt.c index c50dea5a9d..ea9b3bdf18 100644 --- a/source/libs/function/src/functionMgt.c +++ b/source/libs/function/src/functionMgt.c @@ -76,6 +76,16 @@ int32_t fmGetFuncResultType(SFunctionNode* pFunc) { return funcMgtBuiltins[pFunc->funcId].checkFunc(pFunc); } +EFuncDataRequired fmFuncDataRequired(SFunctionNode* pFunc, STimeWindow* pTimeWindow) { + if (pFunc->funcId < 0 || pFunc->funcId >= funcMgtBuiltinsNum) { + return FUNC_DATA_REQUIRED_ALL_NEEDED; + } + if (NULL == funcMgtBuiltins[pFunc->funcId].dataRequiredFunc) { + return FUNC_DATA_REQUIRED_ALL_NEEDED; + } + return funcMgtBuiltins[pFunc->funcId].dataRequiredFunc(pFunc, pTimeWindow); +} + int32_t fmGetFuncExecFuncs(int32_t funcId, SFuncExecFuncs* pFpSet) { if (funcId < 0 || funcId >= funcMgtBuiltinsNum) { return TSDB_CODE_FAILED; @@ -120,6 +130,13 @@ bool fmIsNonstandardSQLFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_NONSTANDARD_SQL_FUNC); } +bool fmIsSpecialDataRequiredFunc(int32_t funcId) { + return isSpecificClassifyFunc(funcId, FUNC_MGT_SPECIAL_DATA_REQUIRED); +} + +bool fmIsDynamicScanOptimizedFunc(int32_t funcId) { + return isSpecificClassifyFunc(funcId, FUNC_MGT_DYNAMIC_SCAN_OPTIMIZED); +} void fmFuncMgtDestroy() { void* m = gFunMgtService.pFuncNameHashTable; diff --git a/source/libs/monitor/src/monMain.c b/source/libs/monitor/src/monMain.c index c90b1f58e8..3ece089a28 100644 --- a/source/libs/monitor/src/monMain.c +++ b/source/libs/monitor/src/monMain.c @@ -375,6 +375,9 @@ static void monGenDnodeJson(SMonInfo *pMonitor) { tjsonAddDoubleToObject(pJson, "vnodes_num", pStat->totalVnodes); tjsonAddDoubleToObject(pJson, "masters", pStat->masterNum); tjsonAddDoubleToObject(pJson, "has_mnode", pInfo->has_mnode); + tjsonAddDoubleToObject(pJson, "has_qnode", pInfo->has_qnode); + tjsonAddDoubleToObject(pJson, "has_snode", pInfo->has_snode); + tjsonAddDoubleToObject(pJson, "has_bnode", pInfo->has_bnode); } static void monGenDiskJson(SMonInfo *pMonitor) { @@ -530,7 +533,7 @@ void monSendReport() { if (pCont != NULL) { EHttpCompFlag flag = tsMonitor.cfg.comp ? HTTP_GZIP : HTTP_FLAT; if (taosSendHttpReport(tsMonitor.cfg.server, tsMonitor.cfg.port, pCont, strlen(pCont), flag) != 0) { - uError("failed to send monitor msg since %s", terrstr()); + uError("failed to send monitor msg"); } taosMemoryFree(pCont); } diff --git a/source/libs/monitor/src/monMsg.c b/source/libs/monitor/src/monMsg.c index 3aafcf071d..adacbf479b 100644 --- a/source/libs/monitor/src/monMsg.c +++ b/source/libs/monitor/src/monMsg.c @@ -194,9 +194,9 @@ int32_t tDecodeSMonVgroupInfo(SCoder *decoder, SMonVgroupInfo *pInfo) { if (tDecodeCStrTo(decoder, desc.database_name) < 0) return -1; if (tDecodeCStrTo(decoder, desc.status) < 0) return -1; for (int32_t j = 0; j < TSDB_MAX_REPLICA; ++j) { - SMonVnodeDesc vdesc = {0}; - if (tDecodeI32(decoder, &vdesc.dnode_id) < 0) return -1; - if (tDecodeCStrTo(decoder, vdesc.vnode_role) < 0) return -1; + SMonVnodeDesc *pVDesc = &desc.vnodes[j]; + if (tDecodeI32(decoder, &pVDesc->dnode_id) < 0) return -1; + if (tDecodeCStrTo(decoder, pVDesc->vnode_role) < 0) return -1; } taosArrayPush(pInfo->vgroups, &desc); } @@ -205,15 +205,15 @@ int32_t tDecodeSMonVgroupInfo(SCoder *decoder, SMonVgroupInfo *pInfo) { int32_t tEncodeSMonGrantInfo(SCoder *encoder, const SMonGrantInfo *pInfo) { if (tEncodeI32(encoder, pInfo->expire_time) < 0) return -1; - if (tEncodeI32(encoder, pInfo->timeseries_used) < 0) return -1; - if (tEncodeI32(encoder, pInfo->timeseries_total) < 0) return -1; + if (tEncodeI64(encoder, pInfo->timeseries_used) < 0) return -1; + if (tEncodeI64(encoder, pInfo->timeseries_total) < 0) return -1; return 0; } int32_t tDecodeSMonGrantInfo(SCoder *decoder, SMonGrantInfo *pInfo) { if (tDecodeI32(decoder, &pInfo->expire_time) < 0) return -1; - if (tDecodeI32(decoder, &pInfo->timeseries_used) < 0) return -1; - if (tDecodeI32(decoder, &pInfo->timeseries_total) < 0) return -1; + if (tDecodeI64(decoder, &pInfo->timeseries_used) < 0) return -1; + if (tDecodeI64(decoder, &pInfo->timeseries_total) < 0) return -1; return 0; } diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index 13093d63ac..8ab76f5714 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -723,6 +723,9 @@ static int32_t jsonToPhysiTagScanNode(const SJson* pJson, void* pObj) { static const char* jkTableScanPhysiPlanScanFlag = "ScanFlag"; static const char* jkTableScanPhysiPlanStartKey = "StartKey"; static const char* jkTableScanPhysiPlanEndKey = "EndKey"; +static const char* jkTableScanPhysiPlanRatio = "Ratio"; +static const char* jkTableScanPhysiPlanDataRequired = "DataRequired"; +static const char* jkTableScanPhysiPlanDynamicScanFuncs = "DynamicScanFuncs"; static int32_t physiTableScanNodeToJson(const void* pObj, SJson* pJson) { const STableScanPhysiNode* pNode = (const STableScanPhysiNode*)pObj; @@ -737,6 +740,15 @@ static int32_t physiTableScanNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = tjsonAddIntegerToObject(pJson, jkTableScanPhysiPlanEndKey, pNode->scanRange.ekey); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddDoubleToObject(pJson, jkTableScanPhysiPlanRatio, pNode->ratio); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkTableScanPhysiPlanDataRequired, pNode->dataRequired); + } + if (TSDB_CODE_SUCCESS == code) { + code = nodeListToJson(pJson, jkTableScanPhysiPlanDynamicScanFuncs, pNode->pDynamicScanFuncs); + } return code; } @@ -754,6 +766,15 @@ static int32_t jsonToPhysiTableScanNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = tjsonGetBigIntValue(pJson, jkTableScanPhysiPlanEndKey, &pNode->scanRange.ekey); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetDoubleValue(pJson, jkTableScanPhysiPlanRatio, &pNode->ratio); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetNumberValue(pJson, jkTableScanPhysiPlanDataRequired, pNode->dataRequired); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeList(pJson, jkTableScanPhysiPlanDynamicScanFuncs, &pNode->pDynamicScanFuncs); + } return code; } @@ -2767,6 +2788,7 @@ int32_t nodesStringToList(const char* pStr, SNodeList** pList) { return TSDB_CODE_FAILED; } int32_t code = jsonToNodeListImpl(pJson, pList); + tjsonDelete(pJson); if (TSDB_CODE_SUCCESS != code) { nodesDestroyList(*pList); terrno = code; diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 9f81b34274..e459f04634 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -694,6 +694,17 @@ int32_t nodesListMakeAppend(SNodeList** pList, SNodeptr pNode) { return nodesListAppend(*pList, pNode); } +int32_t nodesListMakeStrictAppend(SNodeList** pList, SNodeptr pNode) { + if (NULL == *pList) { + *pList = nodesMakeList(); + if (NULL == *pList) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; + } + } + return nodesListStrictAppend(*pList, pNode); +} + int32_t nodesListAppendList(SNodeList* pTarget, SNodeList* pSrc) { if (NULL == pTarget || NULL == pSrc) { return TSDB_CODE_FAILED; diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index 318f7ca5f7..a14053ada7 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -257,13 +257,20 @@ SNodeList* addValueNodeFromTypeToList(SAstCreateContext* pCxt, SDataType dataTyp char buf[64] = {0}; //add value node for type snprintf(buf, sizeof(buf), "%u", dataType.type); - SToken token = {.type = TSDB_DATA_TYPE_TINYINT, .n = strlen(buf), .z = buf}; + SToken token = {.type = TSDB_DATA_TYPE_SMALLINT, .n = strlen(buf), .z = buf}; SNode* pNode = createValueNode(pCxt, token.type, &token); addNodeToList(pCxt, pList, pNode); //add value node for bytes memset(buf, 0, sizeof(buf)); - snprintf(buf, sizeof(buf), "%u", dataType.bytes); + int32_t bytes; + if (IS_VAR_DATA_TYPE(dataType.type)) { + bytes = (dataType.type == TSDB_DATA_TYPE_NCHAR) ? dataType.bytes * TSDB_NCHAR_SIZE : dataType.bytes; + bytes += VARSTR_HEADER_SIZE; + } else { + bytes = dataType.bytes; + } + snprintf(buf, sizeof(buf), "%d", bytes); token.type = TSDB_DATA_TYPE_BIGINT; token.n = strlen(buf); token.z = buf; diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 8885373b2b..acf465bfa5 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -21,7 +21,7 @@ #include "parUtil.h" #include "ttime.h" -#define GET_OPTION_VAL(pVal, defaultVal) (NULL == (pVal) ? (defaultVal) : ((SValueNode*)(pVal))->datum.i) +#define GET_OPTION_VAL(pVal, defaultVal) (NULL == (pVal) ? (defaultVal) : getBigintFromValueNode((SValueNode*)(pVal))) typedef struct STranslateContext { SParseContext* pParseCxt; @@ -380,6 +380,7 @@ static EDealRes translateColumn(STranslateContext* pCxt, SColumnNode* pCol) { static EDealRes translateValue(STranslateContext* pCxt, SValueNode* pVal) { uint8_t precision = (NULL != pCxt->pCurrStmt ? pCxt->pCurrStmt->precision : pVal->node.resType.precision); + pVal->node.resType.precision = precision; if (pVal->isDuration) { if (parseNatualDuration(pVal->literal, strlen(pVal->literal), &pVal->datum.i, &pVal->unit, precision) != TSDB_CODE_SUCCESS) { @@ -452,6 +453,9 @@ static EDealRes translateOperator(STranslateContext* pCxt, SOperatorNode* pOp) { } pOp->node.resType.type = TSDB_DATA_TYPE_DOUBLE; pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes; + } else { + pOp->node.resType.type = TSDB_DATA_TYPE_BOOL; + pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BOOL].bytes; } return DEAL_RES_CONTINUE; } @@ -1041,6 +1045,27 @@ static int32_t translateSelect(STranslateContext* pCxt, SSelectStmt* pSelect) { return code; } +static int64_t getUnitPerMinute(uint8_t precision) { + switch (precision) { + case TSDB_TIME_PRECISION_MILLI: + return MILLISECOND_PER_MINUTE; + case TSDB_TIME_PRECISION_MICRO: + return MILLISECOND_PER_MINUTE * 1000L; + case TSDB_TIME_PRECISION_NANO: + return NANOSECOND_PER_MINUTE; + default: + break; + } + return MILLISECOND_PER_MINUTE; +} + +static int64_t getBigintFromValueNode(SValueNode* pVal) { + if (pVal->isDuration) { + return pVal->datum.i / getUnitPerMinute(pVal->node.resType.precision); + } + return pVal->datum.i; +} + static int32_t buildCreateDbRetentions(const SNodeList* pRetentions, SCreateDbReq* pReq) { if (NULL != pRetentions) { pReq->pRetensions = taosArrayInit(LIST_LENGTH(pRetentions), sizeof(SRetention)); @@ -1098,7 +1123,10 @@ static int32_t checkRangeOption(STranslateContext* pCxt, const char* pName, SVal if (DEAL_RES_ERROR == translateValue(pCxt, pVal)) { return pCxt->errCode; } - int64_t val = pVal->datum.i; + if (pVal->isDuration && (TIME_UNIT_MINUTE != pVal->unit && TIME_UNIT_HOUR != pVal->unit && TIME_UNIT_DAY != pVal->unit)) { + return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_INVALID_OPTION_UNIT, pName, pVal->unit); + } + int64_t val = getBigintFromValueNode(pVal); if (val < minVal || val > maxVal) { return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_INVALID_RANGE_OPTION, pName, val, minVal, maxVal); } @@ -1187,9 +1215,18 @@ static int32_t checkKeepOption(STranslateContext* pCxt, SNodeList* pKeep) { } } - int32_t daysToKeep0 = ((SValueNode*)nodesListGetNode(pKeep, 0))->datum.i; - int32_t daysToKeep1 = ((SValueNode*)nodesListGetNode(pKeep, 1))->datum.i; - int32_t daysToKeep2 = ((SValueNode*)nodesListGetNode(pKeep, 2))->datum.i; + SValueNode* pKeep0 = (SValueNode*)nodesListGetNode(pKeep, 0); + SValueNode* pKeep1 = (SValueNode*)nodesListGetNode(pKeep, 1); + SValueNode* pKeep2 = (SValueNode*)nodesListGetNode(pKeep, 2); + if ((pKeep0->isDuration && (TIME_UNIT_MINUTE != pKeep0->unit && TIME_UNIT_HOUR != pKeep0->unit && TIME_UNIT_DAY != pKeep0->unit)) || + (pKeep1->isDuration && (TIME_UNIT_MINUTE != pKeep1->unit && TIME_UNIT_HOUR != pKeep1->unit && TIME_UNIT_DAY != pKeep1->unit)) || + (pKeep2->isDuration && (TIME_UNIT_MINUTE != pKeep2->unit && TIME_UNIT_HOUR != pKeep2->unit && TIME_UNIT_DAY != pKeep2->unit))) { + return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_INVALID_KEEP_UNIT, pKeep0->unit, pKeep1->unit, pKeep2->unit); + } + + int32_t daysToKeep0 = getBigintFromValueNode(pKeep0); + int32_t daysToKeep1 = getBigintFromValueNode(pKeep1); + int32_t daysToKeep2 = getBigintFromValueNode(pKeep2); if (daysToKeep0 < TSDB_MIN_KEEP || daysToKeep1 < TSDB_MIN_KEEP || daysToKeep2 < TSDB_MIN_KEEP || daysToKeep0 > TSDB_MAX_KEEP || daysToKeep1 > TSDB_MAX_KEEP || daysToKeep2 > TSDB_MAX_KEEP) { return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_INVALID_KEEP_VALUE, daysToKeep0, daysToKeep1, daysToKeep2, @@ -1240,8 +1277,7 @@ static int32_t checkDatabaseOptions(STranslateContext* pCxt, SDatabaseOptions* p code = checkRangeOption(pCxt, "compression", pOptions->pCompressionLevel, TSDB_MIN_COMP_LEVEL, TSDB_MAX_COMP_LEVEL); } if (TSDB_CODE_SUCCESS == code) { - code = - checkRangeOption(pCxt, "daysPerFile", pOptions->pDaysPerFile, TSDB_MIN_DAYS_PER_FILE, TSDB_MAX_DAYS_PER_FILE); + code = checkRangeOption(pCxt, "daysPerFile", pOptions->pDaysPerFile, TSDB_MIN_DAYS_PER_FILE, TSDB_MAX_DAYS_PER_FILE); } if (TSDB_CODE_SUCCESS == code) { code = checkRangeOption(pCxt, "fsyncPeriod", pOptions->pFsyncPeriod, TSDB_MIN_FSYNC_PERIOD, TSDB_MAX_FSYNC_PERIOD); diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index 171324aa99..85d574d9e7 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -62,35 +62,39 @@ static char* getSyntaxErrFormat(int32_t errCode) { case TSDB_CODE_PAR_INTERVAL_VALUE_TOO_SMALL: return "This interval value is too small : %s"; case TSDB_CODE_PAR_DB_NOT_SPECIFIED: - return "db not specified"; + return "Database not specified"; case TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME: return "Invalid identifier name : %s"; case TSDB_CODE_PAR_CORRESPONDING_STABLE_ERR: - return "corresponding super table not in this db"; + return "Corresponding super table not in this db"; case TSDB_CODE_PAR_INVALID_RANGE_OPTION: - return "invalid option %s: %"PRId64" valid range: [%d, %d]"; + return "Invalid option %s: %"PRId64" valid range: [%d, %d]"; case TSDB_CODE_PAR_INVALID_STR_OPTION: - return "invalid option %s: %s"; + return "Invalid option %s: %s"; case TSDB_CODE_PAR_INVALID_ENUM_OPTION: - return "invalid option %s: %"PRId64", only %d, %d allowed"; + return "Invalid option %s: %"PRId64", only %d, %d allowed"; case TSDB_CODE_PAR_INVALID_TTL_OPTION: - return "invalid option ttl: %"PRId64", should be greater than or equal to %d"; + return "Invalid option ttl: %"PRId64", should be greater than or equal to %d"; case TSDB_CODE_PAR_INVALID_KEEP_NUM: - return "invalid number of keep options"; + return "Invalid number of keep options"; case TSDB_CODE_PAR_INVALID_KEEP_ORDER: - return "invalid keep value, should be keep0 <= keep1 <= keep2"; + return "Invalid keep value, should be keep0 <= keep1 <= keep2"; case TSDB_CODE_PAR_INVALID_KEEP_VALUE: - return "invalid option keep: %d, %d, %d valid range: [%d, %d]"; + return "Invalid option keep: %d, %d, %d valid range: [%d, %d]"; case TSDB_CODE_PAR_INVALID_COMMENT_OPTION: - return "invalid option comment, length cannot exceed %d"; + return "Invalid option comment, length cannot exceed %d"; case TSDB_CODE_PAR_INVALID_F_RANGE_OPTION: - return "invalid option %s: %f valid range: [%d, %d]"; + return "Invalid option %s: %f valid range: [%d, %d]"; case TSDB_CODE_PAR_INVALID_ROLLUP_OPTION: - return "invalid option rollup: only one function is allowed"; + return "Invalid option rollup: only one function is allowed"; case TSDB_CODE_PAR_INVALID_RETENTIONS_OPTION: - return "invalid option retentions"; + return "Invalid option retentions"; case TSDB_CODE_PAR_GROUPBY_WINDOW_COEXIST: return "GROUP BY and WINDOW-clause can't be used together"; + case TSDB_CODE_PAR_INVALID_OPTION_UNIT: + return "Invalid option %s unit: %c, only m, h, d allowed"; + case TSDB_CODE_PAR_INVALID_KEEP_UNIT: + return "Invalid option keep unit: %c, %c, %c, only m, h, d allowed"; case TSDB_CODE_OUT_OF_MEMORY: return "Out of memory"; default: diff --git a/source/libs/parser/test/parserAstTest.cpp b/source/libs/parser/test/parserAstTest.cpp index d7e9b4a24d..149c317bd4 100644 --- a/source/libs/parser/test/parserAstTest.cpp +++ b/source/libs/parser/test/parserAstTest.cpp @@ -226,6 +226,16 @@ TEST_F(ParserTest, selectExpression) { ASSERT_TRUE(run()); } +TEST_F(ParserTest, selectCondition) { + setDatabase("root", "test"); + + bind("SELECT c1 FROM t1 where ts in (true, false)"); + ASSERT_TRUE(run()); + + bind("SELECT * FROM t1 where c1 > 10 and c1 is not null"); + ASSERT_TRUE(run()); +} + TEST_F(ParserTest, selectPseudoColumn) { setDatabase("root", "test"); diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index bd5ce0f494..1d8400e1eb 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -200,6 +200,7 @@ static int32_t createScanLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect strcpy(pScan->tableName.tname, pRealTable->table.tableName); pScan->showRewrite = pCxt->pPlanCxt->showRewrite; pScan->ratio = pRealTable->ratio; + pScan->dataRequired = FUNC_DATA_REQUIRED_ALL_NEEDED; // set columns to scan SNodeList* pCols = NULL; diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 2a36e38ce1..2df1b97c89 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -14,7 +14,159 @@ */ #include "planInt.h" +#include "functionMgt.h" -int32_t optimizeLogicPlan(SPlanContext* pCxt, SLogicNode* pLogicNode) { +#define OPTIMIZE_FLAG_MASK(n) (1 << n) + +#define OPTIMIZE_FLAG_OSD OPTIMIZE_FLAG_MASK(0) + +#define OPTIMIZE_FLAG_SET_MASK(val, mask) (val) |= (mask) +#define OPTIMIZE_FLAG_TEST_MASK(val, mask) (((val) & (mask)) != 0) + +typedef struct SOptimizeContext { + bool optimized; +} SOptimizeContext; + +typedef int32_t (*FMatch)(SOptimizeContext* pCxt, SLogicNode* pLogicNode); +typedef int32_t (*FOptimize)(SOptimizeContext* pCxt, SLogicNode* pLogicNode); + +typedef struct SOptimizeRule { + char* pName; + FOptimize optimizeFunc; +} SOptimizeRule; + +typedef struct SOsdInfo { + SScanLogicNode* pScan; + SNodeList* pSdrFuncs; + SNodeList* pDsoFuncs; +} SOsdInfo; + +static bool osdMayBeOptimized(SLogicNode* pNode) { + if (OPTIMIZE_FLAG_TEST_MASK(pNode->optimizedFlag, OPTIMIZE_FLAG_OSD)) { + return false; + } + if (QUERY_NODE_LOGIC_PLAN_SCAN != nodeType(pNode)) { + return false; + } + if (NULL == pNode->pParent || + (QUERY_NODE_LOGIC_PLAN_WINDOW != nodeType(pNode->pParent) && QUERY_NODE_LOGIC_PLAN_AGG == nodeType(pNode->pParent))) { + return false; + } + return true; +} + +static SLogicNode* osdFindPossibleScanNode(SLogicNode* pNode) { + if (osdMayBeOptimized(pNode)) { + return pNode; + } + SNode* pChild; + FOREACH(pChild, pNode->pChildren) { + SLogicNode* pScanNode = osdFindPossibleScanNode((SLogicNode*)pChild); + if (NULL != pScanNode) { + return pScanNode; + } + } + return NULL; +} + +static SNodeList* osdGetAllFuncs(SLogicNode* pNode) { + switch (nodeType(pNode)) { + case QUERY_NODE_LOGIC_PLAN_WINDOW: + return ((SWindowLogicNode*)pNode)->pFuncs; + case QUERY_NODE_LOGIC_PLAN_AGG: + return ((SAggLogicNode*)pNode)->pAggFuncs; + default: + break; + } + return NULL; +} + +static int32_t osdGetRelatedFuncs(SScanLogicNode* pScan, SNodeList** pSdrFuncs, SNodeList** pDsoFuncs) { + SNodeList* pAllFuncs = osdGetAllFuncs(pScan->node.pParent); + SNode* pFunc = NULL; + FOREACH(pFunc, pAllFuncs) { + int32_t code = TSDB_CODE_SUCCESS; + if (fmIsSpecialDataRequiredFunc(((SFunctionNode*)pFunc)->funcId)) { + code = nodesListMakeStrictAppend(pSdrFuncs, nodesCloneNode(pFunc)); + } else if (fmIsDynamicScanOptimizedFunc(((SFunctionNode*)pFunc)->funcId)) { + code = nodesListMakeStrictAppend(pDsoFuncs, nodesCloneNode(pFunc)); + } + if (TSDB_CODE_SUCCESS != code) { + nodesDestroyList(*pSdrFuncs); + nodesDestroyList(*pDsoFuncs); + return code; + } + } return TSDB_CODE_SUCCESS; } + +static int32_t osdMatch(SOptimizeContext* pCxt, SLogicNode* pLogicNode, SOsdInfo* pInfo) { + pInfo->pScan = (SScanLogicNode*)osdFindPossibleScanNode(pLogicNode); + if (NULL == pInfo->pScan) { + return TSDB_CODE_SUCCESS; + } + return osdGetRelatedFuncs(pInfo->pScan, &pInfo->pSdrFuncs, &pInfo->pDsoFuncs); +} + +static EFuncDataRequired osdPromoteDataRequired(EFuncDataRequired l , EFuncDataRequired r) { + switch (l) { + case FUNC_DATA_REQUIRED_ALL_NEEDED: + return l; + case FUNC_DATA_REQUIRED_STATIS_NEEDED: + return FUNC_DATA_REQUIRED_ALL_NEEDED == r ? r : l; + case FUNC_DATA_REQUIRED_NO_NEEDED: + return FUNC_DATA_REQUIRED_DISCARD == r ? l : r; + default: + break; + } + return r; +} + +static int32_t osdGetDataRequired(SNodeList* pFuncs) { + if (NULL == pFuncs) { + return FUNC_DATA_REQUIRED_ALL_NEEDED; + } + EFuncDataRequired dataRequired = FUNC_DATA_REQUIRED_DISCARD; + SNode* pFunc = NULL; + FOREACH(pFunc, pFuncs) { + dataRequired = osdPromoteDataRequired(dataRequired, fmFuncDataRequired((SFunctionNode*)pFunc, NULL)); + } + return dataRequired; +} + +static int32_t osdOptimize(SOptimizeContext* pCxt, SLogicNode* pLogicNode) { + SOsdInfo info = {0}; + int32_t code = osdMatch(pCxt, pLogicNode, &info); + if (TSDB_CODE_SUCCESS == code && (NULL != info.pDsoFuncs || NULL != info.pSdrFuncs)) { + info.pScan->dataRequired = osdGetDataRequired(info.pSdrFuncs); + info.pScan->pDynamicScanFuncs = info.pDsoFuncs; + OPTIMIZE_FLAG_SET_MASK(info.pScan->node.optimizedFlag, OPTIMIZE_FLAG_OSD); + pCxt->optimized = true; + } + nodesDestroyList(info.pSdrFuncs); + return code; +} + +static const SOptimizeRule optimizeRuleSet[] = { + { .pName = "OptimizeScanData", .optimizeFunc = osdOptimize } +}; + +static const int32_t optimizeRuleNum = (sizeof(optimizeRuleSet) / sizeof(SOptimizeRule)); + +static int32_t applyOptimizeRule(SLogicNode* pLogicNode) { + SOptimizeContext cxt = { .optimized = false }; + do { + cxt.optimized = false; + for (int32_t i = 0; i < optimizeRuleNum; ++i) { + int32_t code = optimizeRuleSet[i].optimizeFunc(&cxt, pLogicNode); + if (TSDB_CODE_SUCCESS != code) { + return code; + } + } + } while (cxt.optimized); + return TSDB_CODE_SUCCESS; +} + +int32_t optimizeLogicPlan(SPlanContext* pCxt, SLogicNode* pLogicNode) { + return applyOptimizeRule(pLogicNode); +} diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index f34452f84a..3bdbc0b908 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -437,6 +437,12 @@ static int32_t createTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubp taosArrayPush(pCxt->pExecNodeList, &pSubplan->execNode); pSubplan->execNodeStat.tableNum = pScanLogicNode->pVgroupList->vgroups[0].numOfTable; tNameGetFullDbName(&pScanLogicNode->tableName, pSubplan->dbFName); + pTableScan->dataRequired = pScanLogicNode->dataRequired; + pTableScan->pDynamicScanFuncs = nodesCloneList(pScanLogicNode->pDynamicScanFuncs); + if (NULL != pScanLogicNode->pDynamicScanFuncs && NULL == pTableScan->pDynamicScanFuncs) { + nodesDestroyNode(pTableScan); + return TSDB_CODE_OUT_OF_MEMORY; + } return createScanPhysiNodeFinalize(pCxt, pScanLogicNode, (SScanPhysiNode*)pTableScan, pPhyNode); } diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index aadbaf6fd8..ad1a8c01c8 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -23,18 +23,14 @@ #define SPLIT_FLAG_TEST_MASK(val, mask) (((val) & (mask)) != 0) typedef struct SSplitContext { - int32_t errCode; int32_t groupId; - bool match; - void* pInfo; + bool split; } SSplitContext; -typedef int32_t (*FMatch)(SSplitContext* pCxt, SLogicSubplan* pSubplan); -typedef int32_t (*FSplit)(SSplitContext* pCxt); +typedef int32_t (*FSplit)(SSplitContext* pCxt, SLogicSubplan* pSubplan); typedef struct SSplitRule { char* pName; - FMatch matchFunc; FSplit splitFunc; } SSplitRule; @@ -58,30 +54,25 @@ static SLogicNode* stsMatchByNode(SLogicNode* pNode) { return NULL; } -static int32_t stsMatch(SSplitContext* pCxt, SLogicSubplan* pSubplan) { - if (SPLIT_FLAG_TEST_MASK(pSubplan->splitFlag, SPLIT_FLAG_STS)) { - return TSDB_CODE_SUCCESS; - } +static void stsFindSplitNode(SLogicSubplan* pSubplan, SStsInfo* pInfo) { SLogicNode* pSplitNode = stsMatchByNode(pSubplan->pNode); if (NULL != pSplitNode) { - SStsInfo* pInfo = taosMemoryCalloc(1, sizeof(SStsInfo)); - if (NULL == pInfo) { - return TSDB_CODE_OUT_OF_MEMORY; - } pInfo->pScan = (SScanLogicNode*)pSplitNode; pInfo->pSubplan = pSubplan; - pCxt->pInfo = pInfo; - pCxt->match = true; - return TSDB_CODE_SUCCESS; + } +} +static void stsMatch(SSplitContext* pCxt, SLogicSubplan* pSubplan, SStsInfo* pInfo) { + if (!SPLIT_FLAG_TEST_MASK(pSubplan->splitFlag, SPLIT_FLAG_STS)) { + stsFindSplitNode(pSubplan, pInfo); } SNode* pChild; FOREACH(pChild, pSubplan->pChildren) { - int32_t code = stsMatch(pCxt, (SLogicSubplan*)pChild); - if (TSDB_CODE_SUCCESS != code || pCxt->match) { - return code; + stsMatch(pCxt, (SLogicSubplan*)pChild, pInfo); + if (NULL != pInfo->pScan) { + break; } } - return TSDB_CODE_SUCCESS; + return; } static SLogicSubplan* stsCreateScanSubplan(SSplitContext* pCxt, SScanLogicNode* pScan) { @@ -128,46 +119,44 @@ static int32_t stsCreateExchangeNode(SSplitContext* pCxt, SLogicSubplan* pSubpla return TSDB_CODE_FAILED; } -static int32_t stsSplit(SSplitContext* pCxt) { - SStsInfo* pInfo = pCxt->pInfo; - if (NULL == pInfo->pSubplan->pChildren) { - pInfo->pSubplan->pChildren = nodesMakeList(); - if (NULL == pInfo->pSubplan->pChildren) { +static int32_t stsSplit(SSplitContext* pCxt, SLogicSubplan* pSubplan) { + SStsInfo info = {0}; + stsMatch(pCxt, pSubplan, &info); + if (NULL == info.pScan) { + return TSDB_CODE_SUCCESS; + } + if (NULL == info.pSubplan->pChildren) { + info.pSubplan->pChildren = nodesMakeList(); + if (NULL == info.pSubplan->pChildren) { return TSDB_CODE_OUT_OF_MEMORY; } } - int32_t code = nodesListStrictAppend(pInfo->pSubplan->pChildren, stsCreateScanSubplan(pCxt, pInfo->pScan)); + int32_t code = nodesListStrictAppend(info.pSubplan->pChildren, stsCreateScanSubplan(pCxt, info.pScan)); if (TSDB_CODE_SUCCESS == code) { - code = stsCreateExchangeNode(pCxt, pInfo->pSubplan, pInfo->pScan); + code = stsCreateExchangeNode(pCxt, info.pSubplan, info.pScan); } ++(pCxt->groupId); - taosMemoryFreeClear(pCxt->pInfo); + pCxt->split = true; return code; } static const SSplitRule splitRuleSet[] = { - { .pName = "SuperTableScan", .matchFunc = stsMatch, .splitFunc = stsSplit } + { .pName = "SuperTableScan", .splitFunc = stsSplit } }; static const int32_t splitRuleNum = (sizeof(splitRuleSet) / sizeof(SSplitRule)); static int32_t applySplitRule(SLogicSubplan* pSubplan) { - SSplitContext cxt = { .errCode = TSDB_CODE_SUCCESS, .groupId = pSubplan->id.groupId + 1, .match = false, .pInfo = NULL }; - bool split = false; + SSplitContext cxt = { .groupId = pSubplan->id.groupId + 1, .split = false }; do { - split = false; + cxt.split = false; for (int32_t i = 0; i < splitRuleNum; ++i) { - cxt.match = false; - int32_t code = splitRuleSet[i].matchFunc(&cxt, pSubplan); - if (TSDB_CODE_SUCCESS == code && cxt.match) { - code = splitRuleSet[i].splitFunc(&cxt); - split = true; - } + int32_t code = splitRuleSet[i].splitFunc(&cxt, pSubplan); if (TSDB_CODE_SUCCESS != code) { return code; } } - } while (split); + } while (cxt.split); return TSDB_CODE_SUCCESS; } diff --git a/source/libs/planner/test/plannerTest.cpp b/source/libs/planner/test/plannerTest.cpp index 697c562b1e..51267d5825 100644 --- a/source/libs/planner/test/plannerTest.cpp +++ b/source/libs/planner/test/plannerTest.cpp @@ -177,14 +177,14 @@ TEST_F(PlannerTest, groupBy) { bind("SELECT count(*) FROM t1"); ASSERT_TRUE(run()); - bind("SELECT c1, max(c3), min(c2), count(*) FROM t1 GROUP BY c1"); - ASSERT_TRUE(run()); + // bind("SELECT c1, max(c3), min(c2), count(*) FROM t1 GROUP BY c1"); + // ASSERT_TRUE(run()); - bind("SELECT c1 + c3, c1 + count(*) FROM t1 where c2 = 'abc' GROUP BY c1, c3"); - ASSERT_TRUE(run()); + // bind("SELECT c1 + c3, c1 + count(*) FROM t1 where c2 = 'abc' GROUP BY c1, c3"); + // ASSERT_TRUE(run()); - bind("SELECT c1 + c3, sum(c4 * c5) FROM t1 where concat(c2, 'wwww') = 'abcwww' GROUP BY c1 + c3"); - ASSERT_TRUE(run()); + // bind("SELECT c1 + c3, sum(c4 * c5) FROM t1 where concat(c2, 'wwww') = 'abcwww' GROUP BY c1 + c3"); + // ASSERT_TRUE(run()); } TEST_F(PlannerTest, subquery) { diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 0956c2add5..7b861f9b79 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -647,6 +647,159 @@ int32_t substrFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu return TSDB_CODE_SUCCESS; } +int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + if (inputNum!= 3) { + return TSDB_CODE_FAILED; + } + + int16_t inputType = pInput[0].columnData->info.type; + int16_t outputType = *(int16_t *)pInput[1].columnData->pData; + if (outputType != TSDB_DATA_TYPE_BIGINT && outputType != TSDB_DATA_TYPE_UBIGINT && + outputType != TSDB_DATA_TYPE_VARCHAR && outputType != TSDB_DATA_TYPE_NCHAR && + outputType != TSDB_DATA_TYPE_TIMESTAMP) { + return TSDB_CODE_FAILED; + } + int64_t outputLen = *(int64_t *)pInput[2].columnData->pData; + + char *input = NULL; + char *outputBuf = taosMemoryCalloc(outputLen * pInput[0].numOfRows, 1); + char *output = outputBuf; + if (IS_VAR_DATA_TYPE(inputType)) { + input = pInput[0].columnData->pData + pInput[0].columnData->varmeta.offset[0]; + } else { + input = pInput[0].columnData->pData; + } + + for (int32_t i = 0; i < pInput[0].numOfRows; ++i) { + if (colDataIsNull_s(pInput[0].columnData, i)) { + colDataAppendNULL(pOutput->columnData, i); + continue; + } + + switch(outputType) { + case TSDB_DATA_TYPE_BIGINT: { + if (inputType == TSDB_DATA_TYPE_BINARY) { + memcpy(output, varDataVal(input), varDataLen(input)); + *(int64_t *)output = strtoll(output, NULL, 10); + } else if (inputType == TSDB_DATA_TYPE_NCHAR) { + char *newBuf = taosMemoryCalloc(1, outputLen * TSDB_NCHAR_SIZE + 1); + int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf); + if (len < 0) { + taosMemoryFree(newBuf); + return TSDB_CODE_FAILED; + } + newBuf[len] = 0; + *(int64_t *)output = strtoll(newBuf, NULL, 10); + taosMemoryFree(newBuf); + } else { + GET_TYPED_DATA(*(int64_t *)output, int64_t, inputType, input); + } + break; + } + case TSDB_DATA_TYPE_UBIGINT: { + if (inputType == TSDB_DATA_TYPE_BINARY) { + memcpy(output, varDataVal(input), varDataLen(input)); + *(uint64_t *)output = strtoull(output, NULL, 10); + } else if (inputType == TSDB_DATA_TYPE_NCHAR) { + char *newBuf = taosMemoryCalloc(1, outputLen * TSDB_NCHAR_SIZE + 1); + int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf); + if (len < 0) { + taosMemoryFree(newBuf); + return TSDB_CODE_FAILED; + } + newBuf[len] = 0; + *(uint64_t *)output = strtoull(newBuf, NULL, 10); + taosMemoryFree(newBuf); + } else { + GET_TYPED_DATA(*(uint64_t *)output, uint64_t, inputType, input); + } + break; + } + case TSDB_DATA_TYPE_TIMESTAMP: { + if (inputType == TSDB_DATA_TYPE_BINARY || inputType == TSDB_DATA_TYPE_NCHAR) { + //not support + return TSDB_CODE_FAILED; + } else { + GET_TYPED_DATA(*(int64_t *)output, int64_t, inputType, input); + } + break; + } + case TSDB_DATA_TYPE_BINARY: { + if (inputType == TSDB_DATA_TYPE_BOOL) { + int32_t len = sprintf(varDataVal(output), "%.*s", (int32_t)(outputLen - VARSTR_HEADER_SIZE), *(int8_t *)input ? "true" : "false"); + varDataSetLen(output, len); + } else if (inputType == TSDB_DATA_TYPE_BINARY) { + int32_t len = sprintf(varDataVal(output), "%.*s", (int32_t)(outputLen - VARSTR_HEADER_SIZE), varDataVal(input)); + varDataSetLen(output, len); + } else if (inputType == TSDB_DATA_TYPE_BINARY || inputType == TSDB_DATA_TYPE_NCHAR) { + //not support + return TSDB_CODE_FAILED; + } else { + char tmp[400] = {0}; + NUM_TO_STRING(inputType, input, sizeof(tmp), tmp); + int32_t len = (int32_t)strlen(tmp); + len = (outputLen - VARSTR_HEADER_SIZE) > len ? len : (outputLen - VARSTR_HEADER_SIZE); + memcpy(varDataVal(output), tmp, len); + varDataSetLen(output, len); + } + break; + } + case TSDB_DATA_TYPE_NCHAR: { + int32_t outputCharLen = (outputLen - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE; + if (inputType == TSDB_DATA_TYPE_BOOL) { + char tmp[8] = {0}; + int32_t len = sprintf(tmp, "%.*s", outputCharLen, *(int8_t *)input ? "true" : "false" ); + bool ret = taosMbsToUcs4(tmp, len, (TdUcs4 *)varDataVal(output), outputLen - VARSTR_HEADER_SIZE, &len); + if (!ret) { + return TSDB_CODE_FAILED; + } + varDataSetLen(output, len); + } else if (inputType == TSDB_DATA_TYPE_BINARY) { + int32_t len = outputCharLen > varDataLen(input) ? varDataLen(input) : outputCharLen; + bool ret = taosMbsToUcs4(input + VARSTR_HEADER_SIZE, len, (TdUcs4 *)varDataVal(output), outputLen - VARSTR_HEADER_SIZE, &len); + if (!ret) { + return TSDB_CODE_FAILED; + } + varDataSetLen(output, len); + } else if (inputType == TSDB_DATA_TYPE_NCHAR) { + int32_t len = MIN(outputLen, varDataLen(input) + VARSTR_HEADER_SIZE); + memcpy(output, input, len); + varDataSetLen(output, len - VARSTR_HEADER_SIZE); + } else { + char tmp[400] = {0}; + NUM_TO_STRING(inputType, input, sizeof(tmp), tmp); + int32_t len = (int32_t)strlen(tmp); + len = outputCharLen > len ? len : outputCharLen; + bool ret = taosMbsToUcs4(tmp, len, (TdUcs4 *)varDataVal(output), outputLen - VARSTR_HEADER_SIZE, &len); + if (!ret) { + return TSDB_CODE_FAILED; + } + varDataSetLen(output, len); + } + break; + } + default: { + return TSDB_CODE_FAILED; + } + } + + colDataAppend(pOutput->columnData, i, output, false); + if (IS_VAR_DATA_TYPE(inputType)) { + input += varDataTLen(input); + } else { + input += tDataTypes[inputType].bytes; + } + if (IS_VAR_DATA_TYPE(outputType)) { + output += varDataTLen(output); + } else { + output += tDataTypes[outputType].bytes; + } + } + + pOutput->numOfRows = pInput->numOfRows; + taosMemoryFree(outputBuf); + return TSDB_CODE_SUCCESS; +} int32_t atanFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { return doScalarFunctionUnique(pInput, inputNum, pOutput, atan); diff --git a/source/libs/transport/src/thttp.c b/source/libs/transport/src/thttp.c index cd1fbf8e0e..c747e69339 100644 --- a/source/libs/transport/src/thttp.c +++ b/source/libs/transport/src/thttp.c @@ -164,8 +164,8 @@ int32_t taosSendHttpReport(const char* server, uint16_t port, char* pCont, int32 wb[1] = uv_buf_init((char*)pCont, contLen); connect->data = wb; - uv_tcp_connect(connect, &socket_tcp, (const struct sockaddr*)&dest, clientConnCb); terrno = 0; + uv_tcp_connect(connect, &socket_tcp, (const struct sockaddr*)&dest, clientConnCb); uv_run(loop, UV_RUN_DEFAULT); uv_loop_close(loop); taosMemoryFree(connect); diff --git a/source/os/src/osProc.c b/source/os/src/osProc.c index 1cdd41ad78..b6de638ac2 100644 --- a/source/os/src/osProc.c +++ b/source/os/src/osProc.c @@ -24,7 +24,7 @@ int32_t taosNewProc(char **args) { if (pid == 0) { args[0] = tsProcPath; // close(STDIN_FILENO); - close(STDOUT_FILENO); + // close(STDOUT_FILENO); // close(STDERR_FILENO); return execvp(tsProcPath, args); } else { diff --git a/source/os/src/osSysinfo.c b/source/os/src/osSysinfo.c index 26de26ab67..4ffbc13fb3 100644 --- a/source/os/src/osSysinfo.c +++ b/source/os/src/osSysinfo.c @@ -369,53 +369,33 @@ int32_t taosGetCpuCores(float *numOfCores) { #endif } -int32_t taosGetCpuUsage(double *cpu_system, double *cpu_engine) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +void taosGetCpuUsage(double *cpu_system, double *cpu_engine) { + static int64_t lastSysUsed = 0; + static int64_t lastSysTotal = 0; + static int64_t lastProcTotal = 0; + static int64_t curSysUsed = 0; + static int64_t curSysTotal = 0; + static int64_t curProcTotal = 0; + *cpu_system = 0; *cpu_engine = 0; - return 0; -#elif defined(_TD_DARWIN_64) - *cpu_system = 0; - *cpu_engine = 0; - return 0; -#else - static uint64_t lastSysUsed = 0; - static uint64_t lastSysTotal = 0; - static uint64_t lastProcTotal = 0; - SysCpuInfo sysCpu; - ProcCpuInfo procCpu; - if (taosGetSysCpuInfo(&sysCpu) != 0) { - return -1; + SysCpuInfo sysCpu = {0}; + ProcCpuInfo procCpu = {0}; + if (taosGetSysCpuInfo(&sysCpu) == 0 && taosGetProcCpuInfo(&procCpu) == 0) { + curSysUsed = sysCpu.user + sysCpu.nice + sysCpu.system; + curSysTotal = curSysUsed + sysCpu.idle; + curProcTotal = procCpu.utime + procCpu.stime + procCpu.cutime + procCpu.cstime; + + if (curSysTotal > lastSysTotal && curSysUsed >= lastSysUsed && curProcTotal >= lastProcTotal) { + *cpu_engine = (curSysUsed - lastSysUsed) / (double)(curSysTotal - lastSysTotal) * 100; + *cpu_system = (curProcTotal - lastProcTotal) / (double)(curSysTotal - lastSysTotal) * 100; + } + + lastSysUsed = curSysUsed; + lastSysTotal = curSysTotal; + lastProcTotal = curProcTotal; } - if (taosGetProcCpuInfo(&procCpu) != 0) { - return -1; - } - - uint64_t curSysUsed = sysCpu.user + sysCpu.nice + sysCpu.system; - uint64_t curSysTotal = curSysUsed + sysCpu.idle; - uint64_t curProcTotal = procCpu.utime + procCpu.stime + procCpu.cutime + procCpu.cstime; - - if (lastSysUsed == 0 || lastSysTotal == 0 || lastProcTotal == 0) { - lastSysUsed = curSysUsed > 1 ? curSysUsed : 1; - lastSysTotal = curSysTotal > 1 ? curSysTotal : 1; - lastProcTotal = curProcTotal > 1 ? curProcTotal : 1; - return -1; - } - - if (curSysTotal == lastSysTotal) { - return -1; - } - - *cpu_engine = (curSysUsed - lastSysUsed) / (double)(curSysTotal - lastSysTotal) * 100; - *cpu_system = (curProcTotal - lastProcTotal) / (double)(curSysTotal - lastSysTotal) * 100; - - lastSysUsed = curSysUsed; - lastSysTotal = curSysTotal; - lastProcTotal = curProcTotal; - - return 0; -#endif } int32_t taosGetTotalMemory(int64_t *totalKB) { @@ -618,7 +598,6 @@ void taosGetProcIODelta(int64_t *rchars, int64_t *wchars, int64_t *read_bytes, i static int64_t last_wchars = 0; static int64_t last_read_bytes = 0; static int64_t last_write_bytes = 0; - static int64_t cur_rchars = 0; static int64_t cur_wchars = 0; static int64_t cur_read_bytes = 0; @@ -632,6 +611,11 @@ void taosGetProcIODelta(int64_t *rchars, int64_t *wchars, int64_t *read_bytes, i last_wchars = cur_wchars; last_read_bytes = cur_read_bytes; last_write_bytes = cur_write_bytes; + } else { + *rchars = 0; + *wchars = 0; + *read_bytes = 0; + *write_bytes = 0; } } @@ -693,7 +677,6 @@ int32_t taosGetCardInfo(int64_t *receive_bytes, int64_t *transmit_bytes) { void taosGetCardInfoDelta(int64_t *receive_bytes, int64_t *transmit_bytes) { static int64_t last_receive_bytes = 0; static int64_t last_transmit_bytes = 0; - static int64_t cur_receive_bytes = 0; static int64_t cur_transmit_bytes = 0; if (taosGetCardInfo(&cur_receive_bytes, &cur_transmit_bytes) == 0) { @@ -701,6 +684,9 @@ void taosGetCardInfoDelta(int64_t *receive_bytes, int64_t *transmit_bytes) { *transmit_bytes = cur_transmit_bytes - last_transmit_bytes; last_receive_bytes = cur_receive_bytes; last_transmit_bytes = cur_transmit_bytes; + } else { + *receive_bytes = 0; + *transmit_bytes = 0; } } diff --git a/tests/script/tmp/monitor.sim b/tests/script/tmp/monitor.sim new file mode 100644 index 0000000000..ba98bec2d0 --- /dev/null +++ b/tests/script/tmp/monitor.sim @@ -0,0 +1,35 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 -c monitorfqdn -v localhost +system sh/cfg.sh -n dnode1 -c monitorport -v 80 +system sh/cfg.sh -n dnode1 -c monitorInterval -v 1 +system sh/cfg.sh -n dnode1 -c monitorComp -v 1 +#system sh/cfg.sh -n dnode1 -c supportVnodes -v 128 + +system sh/exec.sh -n dnode1 -s start +sql connect + +print =============== show dnodes +sleep 2000 +sql create database db vgroups 2; +sleep 2000 + +print =============== create drop qnode 1 +sql create qnode on dnode 1 +sql create snode on dnode 1 +sql create bnode on dnode 1 + +return +print =============== restart +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode1 -s start + + +return +system sh/deploy.sh -n dnode2 -i 2 +system sh/exec.sh -n dnode2 -s start +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s start + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT diff --git a/tests/script/tsim/query/diff.sim b/tests/script/tsim/query/diff.sim index ebbd470944..c55b080e44 100644 --- a/tests/script/tsim/query/diff.sim +++ b/tests/script/tsim/query/diff.sim @@ -63,20 +63,21 @@ print =============== step2 $i = 1 $tb = $tbPrefix . $i -print ===> select diff(tbcol) from $tb -sql select diff(tbcol) from $tb +print ===> select _rowts, diff(tbcol) from $tb +sql select _rowts, diff(tbcol) from $tb print ===> rows: $rows print ===> $data00 $data01 $data02 $data03 $data04 $data05 print ===> $data10 $data11 $data12 $data13 $data14 $data15 -if $data11 != 1 then +if $data11 != 1 then + print expect 1, actual: $data11 return -1 endi print =============== step3 $cc = 4 * 60000 $ms = 1601481600000 + $cc -print ===> select diff(tbcol) from $tb where ts > $ms -sql select diff(tbcol) from $tb where ts > $ms +print ===> select _rowts, diff(tbcol) from $tb where ts > $ms +sql select _rowts, diff(tbcol) from $tb where ts > $ms print ===> rows: $rows print ===> $data00 $data01 $data02 $data03 $data04 $data05 print ===> $data10 $data11 $data12 $data13 $data14 $data15 @@ -86,8 +87,8 @@ endi $cc = 4 * 60000 $ms = 1601481600000 + $cc -print ===> select diff(tbcol) from $tb where ts <= $ms -sql select diff(tbcol) from $tb where ts <= $ms +print ===> select _rowts, diff(tbcol) from $tb where ts <= $ms +sql select _rowts, diff(tbcol) from $tb where ts <= $ms print ===> rows: $rows print ===> $data00 $data01 $data02 $data03 $data04 $data05 print ===> $data10 $data11 $data12 $data13 $data14 $data15 @@ -96,8 +97,8 @@ if $data11 != 1 then endi print =============== step4 -print ===> select diff(tbcol) as b from $tb -sql select diff(tbcol) as b from $tb +print ===> select _rowts, diff(tbcol) as b from $tb +sql select _rowts, diff(tbcol) as b from $tb print ===> rows: $rows print ===> $data00 $data01 $data02 $data03 $data04 $data05 print ===> $data10 $data11 $data12 $data13 $data14 $data15 @@ -105,24 +106,24 @@ if $data11 != 1 then return -1 endi -print =============== step5 -print ===> select diff(tbcol) as b from $tb interval(1m) -sql select diff(tbcol) as b from $tb interval(1m) -x step5 - return -1 -step5: - -print =============== step6 -$cc = 4 * 60000 -$ms = 1601481600000 + $cc -print ===> select diff(tbcol) as b from $tb where ts <= $ms interval(1m) -sql select diff(tbcol) as b from $tb where ts <= $ms interval(1m) -x step6 - return -1 +#print =============== step5 +#print ===> select diff(tbcol) as b from $tb interval(1m) +#sql select diff(tbcol) as b from $tb interval(1m) -x step5 +# return -1 +#step5: +# +#print =============== step6 +#$cc = 4 * 60000 +#$ms = 1601481600000 + $cc +#print ===> select diff(tbcol) as b from $tb where ts <= $ms interval(1m) +#sql select diff(tbcol) as b from $tb where ts <= $ms interval(1m) -x step6 +# return -1 step6: print =============== clear sql drop database $db sql show databases -if $rows != 0 then +if $rows != 1 then return -1 endi diff --git a/tests/script/tsim/tmq/consumerMain.sim b/tests/script/tsim/tmq/consumerMain.sim new file mode 100644 index 0000000000..51b90971fd --- /dev/null +++ b/tests/script/tsim/tmq/consumerMain.sim @@ -0,0 +1,267 @@ +#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406 +# scene1: vgroups=1, one topic for one consumer, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb +# scene2: vgroups=1, multi topics for one consumer, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb +# scene3: vgroups=4, one topic for one consumer, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb +# scene4: vgroups=4, multi topics for one consumer, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb +# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN +# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5; +# +# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval). +# +######## ######## ######## ######## ######## ######## ######## ######## ######## ######## +######## This test case include scene1 and scene3 +######## ######## ######## ######## ######## ######## ######## ######## ######## ######## + +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 +system sh/exec.sh -n dnode1 -s start + +$loop_cnt = 0 +check_dnode_ready: + $loop_cnt = $loop_cnt + 1 + sleep 200 + if $loop_cnt == 10 then + print ====> dnode not ready! + return -1 + endi +sql show dnodes +print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05 +if $data00 != 1 then + return -1 +endi +if $data04 != ready then + goto check_dnode_ready +endi + +sql connect + +$loop_cnt = 0 +$vgroups = 1 +$dbNamme = d0 +loop_vgroups: +print =============== create database $dbNamme vgroups $vgroups +sql create database $dbNamme vgroups $vgroups +sql show databases +print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +print $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19 +print $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29 + +if $loop_cnt == 0 then + if $rows != 2 then + return -1 + endi + if $data02 != 1 then # vgroups + print vgroups: $data02 + return -1 + endi +else + if $rows != 3 then + return -1 + endi + if $data00 == d1 then + if $data02 != 4 then # vgroups + print vgroups: $data02 + return -1 + endi + else + if $data12 != 4 then # vgroups + print vgroups: $data12 + return -1 + endi + endi +endi + +sql use $dbNamme + +print =============== create super table +sql create table if not exists stb (ts timestamp, c1 int, c2 float, c3 binary(10)) tags (t1 int) + +sql show stables +if $rows != 1 then + return -1 +endi + +print =============== create child table +$tbPrefix = ct +$tbNum = 2 + +$i = 0 +while $i < $tbNum + $tb = $tbPrefix . $i + sql create table $tb using stb tags( $i ) + $i = $i + 1 +endw + +print =============== create normal table +sql create table ntb (ts timestamp, c1 int, c2 float, c3 binary(10)) + +print =============== create multi topics. notes: now only support: +print =============== 1. columns from stb/ctb/ntb; 2. * from ctb/ntb; 3. function from stb/ctb/ntb +print =============== will support: * from stb + +sql create topic topic_stb_column as select ts, c1, c3 from stb +#sql create topic topic_stb_all as select * from stb +sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb + +sql create topic topic_ctb_column as select ts, c1, c3 from ct0 +sql create topic topic_ctb_all as select * from ct0 +sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ct0 + +sql create topic topic_ntb_column as select ts, c1, c3 from ntb +sql create topic topic_ntb_all as select * from ntb +sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb + +sql show tables +if $rows != 3 then + return -1 +endi + +print =============== run_back insert data + +if $loop_cnt == 0 then + run_back tsim/tmq/insertDataV1.sim +else + run_back tsim/tmq/insertDataV4.sim +endi + +sleep 1000 + +#$rowNum = 1000 +#$tstart = 1640966400000 # 2022-01-01 00:00:00.000 +# +#$i = 0 +#while $i < $tbNum +# $tb = $tbPrefix . $i +# +# $x = 0 +# while $x < $rowNum +# $c = $x / 10 +# $c = $c * 10 +# $c = $x - $c +# +# $binary = ' . binary +# $binary = $binary . $c +# $binary = $binary . ' +# +# sql insert into $tb values ($tstart , $c , $x , $binary ) +# sql insert into ntb values ($tstart , $c , $x , $binary ) +# $tstart = $tstart + 1 +# $x = $x + 1 +# endw +# +# $i = $i + 1 +# $tstart = 1640966400000 +#endw + +#root@trd02 /home $ tmq_sim --help +# -c Configuration directory, default is +# -d The name of the database for cosumer, no default +# -t The topic string for cosumer, no default +# -k The key-value string for cosumer, no default +# -g showMsgFlag, default is 0 +# + +$consumeDelay = 5000 +$expectConsumeMsgCnt = 1000 +print expectConsumeMsgCnt: $expectConsumeMsgCnt, consumeDelay: $consumeDelay + +# supported key: +# group.id: +# enable.auto.commit: +# auto.offset.reset: +# td.connect.ip: +# td.connect.user:root +# td.connect.pass:taosdata +# td.connect.port:6030 +# td.connect.db:db + +$expect_result = @{consume success: @ +$expect_result = $expect_result . $expectConsumeMsgCnt +$expect_result = $expect_result . @, @ +$expect_result = $expect_result . 0} +print expect_result----> $expect_result +print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_column" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt +system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_column" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt +print cmd result----> $system_content +if $system_content >= $expect_result then + return -1 +endi + +#print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_all" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt +#system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_all" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt +#print cmd result----> $system_content +##if $system_content != @{consume success: 10000, 0}@ then +#if $system_content != $expect_result then +# return -1 +#endi + +print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_function" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt +system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_function" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt +print cmd result----> $system_content +#if $system_content != @{consume success: 10000, 0}@ then +if $system_content >= $expect_result then + return -1 +endi + +$expect_result = @{consume success: @ +$expect_result = $expect_result . $rowNum +$expect_result = $expect_result . @, @ +$expect_result = $expect_result . 0} +print expect_result----> $expect_result +print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_column" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt +system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_column" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt +print cmd result----> $system_content +if $system_content >= $expect_result then + return -1 +endi + +print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_all" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt +system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_all" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt +print cmd result----> $system_content +if $system_content >= $expect_result then + return -1 +endi + +print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_function" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt +system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_function" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt +print cmd result----> $system_content +if $system_content >= $expect_result then + return -1 +endi + +$expect_result = @{consume success: @ +$expect_result = $expect_result . $totalMsgCnt +$expect_result = $expect_result . @, @ +$expect_result = $expect_result . 0} +print expect_result----> $expect_result +print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_column" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt +system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_column" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt +print cmd result----> $system_content +if $system_content >= $expect_result then + return -1 +endi + +print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_all" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt +system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_all" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt +print cmd result----> $system_content +if $system_content >= $expect_result then + return -1 +endi + +print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_function" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt +system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_function" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt +print cmd result----> $system_content +if $system_content >= $expect_result then + return -1 +endi + +if $loop_cnt == 0 then + $loop_cnt = 1 + $vgroups = 4 + $dbNamme = d1 + goto loop_vgroups +endi + + +#system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/tmq/insertDataV1.sim b/tests/script/tsim/tmq/insertDataV1.sim new file mode 100644 index 0000000000..a349c55dbd --- /dev/null +++ b/tests/script/tsim/tmq/insertDataV1.sim @@ -0,0 +1,48 @@ + +sql connect + +print ================ insert data +$dbNamme = d0 +$tbPrefix = ct +$tbNum = 10 +$rowNum = 100 +$tstart = 1640966400000 # 2022-01-01 00:00:00.000 + +sql use $dbNamme + +$loop_cnt = 0 + +loop_insert: +print ====> loop $loop_cnt insert +$loop_cnt = $loop_cnt + 1 + +$i = 0 +while $i < $tbNum + $tb = $tbPrefix . $i + + $x = 0 + while $x < $rowNum + $c = $x / 10 + $c = $c * 10 + $c = $x - $c + + $binary = ' . binary + $binary = $binary . $c + $binary = $binary . ' + + #print ====> insert into $tb values ($tstart , $c , $x , $binary ) + #print ====> insert into ntb values ($tstart , $c , $x , $binary ) + sql insert into $tb values ($tstart , $c , $x , $binary ) + sql insert into ntb values ($tstart , $c , $x , $binary ) + $tstart = $tstart + 1 + $x = $x + 1 + endw + + #print ====> insert rows: $rowNum into $tb and ntb + + $i = $i + 1 +# $tstart = 1640966400000 +endw +goto loop_insert + + diff --git a/tests/script/tsim/tmq/insertDataV4.sim b/tests/script/tsim/tmq/insertDataV4.sim new file mode 100644 index 0000000000..72c1358e7f --- /dev/null +++ b/tests/script/tsim/tmq/insertDataV4.sim @@ -0,0 +1,48 @@ + +sql connect + +print ================ insert data +$dbNamme = d1 +$tbPrefix = ct +$tbNum = 10 +$rowNum = 100 +$tstart = 1640966400000 # 2022-01-01 00:00:00.000 + +sql use $dbNamme + +$loop_cnt = 0 + +loop_insert: +print ====> loop $loop_cnt insert +$loop_cnt = $loop_cnt + 1 + +$i = 0 +while $i < $tbNum + $tb = $tbPrefix . $i + + $x = 0 + while $x < $rowNum + $c = $x / 10 + $c = $c * 10 + $c = $x - $c + + $binary = ' . binary + $binary = $binary . $c + $binary = $binary . ' + + #print ====> insert into $tb values ($tstart , $c , $x , $binary ) + #print ====> insert into ntb values ($tstart , $c , $x , $binary ) + sql insert into $tb values ($tstart , $c , $x , $binary ) + sql insert into ntb values ($tstart , $c , $x , $binary ) + $tstart = $tstart + 1 + $x = $x + 1 + endw + + #print ====> insert rows: $rowNum into $tb and ntb + + $i = $i + 1 +# $tstart = 1640966400000 +endw +goto loop_insert + + diff --git a/tests/script/tsim/tmq/mainConsumerInMultiTopic.sim b/tests/script/tsim/tmq/mainConsumerInMultiTopic.sim new file mode 100644 index 0000000000..0df7a8ba57 --- /dev/null +++ b/tests/script/tsim/tmq/mainConsumerInMultiTopic.sim @@ -0,0 +1,234 @@ +#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406 +# scene1: vgroups=1, one topic for one consumer, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb +# scene2: vgroups=1, multi topics for one consumer, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb +# scene3: vgroups=4, one topic for one consumer, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb +# scene4: vgroups=4, multi topics for one consumer, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb +# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN +# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5; +# +# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval). +# +######## ######## ######## ######## ######## ######## ######## ######## ######## ######## +######## This test case include scene2 and scene4 +######## ######## ######## ######## ######## ######## ######## ######## ######## ######## + +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 +system sh/exec.sh -n dnode1 -s start + +$loop_cnt = 0 +check_dnode_ready: + $loop_cnt = $loop_cnt + 1 + sleep 200 + if $loop_cnt == 10 then + print ====> dnode not ready! + return -1 + endi +sql show dnodes +print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05 +if $data00 != 1 then + return -1 +endi +if $data04 != ready then + goto check_dnode_ready +endi + +sql connect + +$loop_cnt = 0 +$vgroups = 1 +$dbNamme = d0 +loop_vgroups: +print =============== create database $dbNamme vgroups $vgroups +sql create database $dbNamme vgroups $vgroups +sql show databases +print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +print $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19 +print $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29 + +if $loop_cnt == 0 then + if $rows != 2 then + return -1 + endi + if $data02 != 1 then # vgroups + print vgroups: $data02 + return -1 + endi +else + if $rows != 3 then + return -1 + endi + if $data00 == d1 then + if $data02 != 4 then # vgroups + print vgroups: $data02 + return -1 + endi + else + if $data12 != 4 then # vgroups + print vgroups: $data12 + return -1 + endi + endi +endi + +sql use $dbNamme + +print =============== create super table +sql create table if not exists stb (ts timestamp, c1 int, c2 float, c3 binary(10)) tags (t1 int) + +sql show stables +if $rows != 1 then + return -1 +endi + +print =============== create child table +$tbPrefix = ct +$tbNum = 100 + +$i = 0 +while $i < $tbNum + $tb = $tbPrefix . $i + sql create table $tb using stb tags( $i ) + $i = $i + 1 +endw + +print =============== create normal table +sql create table ntb (ts timestamp, c1 int, c2 float, c3 binary(10)) + +print =============== create multi topics. notes: now only support: +print =============== 1. columns from stb/ctb/ntb; 2. * from ctb/ntb; 3. function from stb/ctb/ntb +print =============== will support: * from stb + +sql create topic topic_stb_column as select ts, c1, c3 from stb +#sql create topic topic_stb_all as select * from stb +sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb + +sql create topic topic_ctb_column as select ts, c1, c3 from ct0 +sql create topic topic_ctb_all as select * from ct0 +sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ct0 + +sql create topic topic_ntb_column as select ts, c1, c3 from ntb +sql create topic topic_ntb_all as select * from ntb +sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb + +sql show tables +if $rows != 101 then + return -1 +endi + +print =============== run_back insert data + +if $loop_cnt == 0 then + run_back tsim/tmq/insertDataV1.sim +else + run_back tsim/tmq/insertDataV4.sim +endi + +#sleep 1000 + +#$rowNum = 1000 +#$tstart = 1640966400000 # 2022-01-01 00:00:00.000 +# +#$i = 0 +#while $i < $tbNum +# $tb = $tbPrefix . $i +# +# $x = 0 +# while $x < $rowNum +# $c = $x / 10 +# $c = $c * 10 +# $c = $x - $c +# +# $binary = ' . binary +# $binary = $binary . $c +# $binary = $binary . ' +# +# sql insert into $tb values ($tstart , $c , $x , $binary ) +# sql insert into ntb values ($tstart , $c , $x , $binary ) +# $tstart = $tstart + 1 +# $x = $x + 1 +# endw +# +# $i = $i + 1 +# $tstart = 1640966400000 +#endw + +#root@trd02 /home $ tmq_sim --help +# -c Configuration directory, default is +# -d The name of the database for cosumer, no default +# -t The topic string for cosumer, no default +# -k The key-value string for cosumer, no default +# -g showMsgFlag, default is 0 +# + +$consumeDelay = 50 +$consumeMsgCntFromTopic = 1000 +print consumeMsgCntFromTopic: $consumeMsgCntFromTopic , consumeDelay: $consumeDelay + +# supported key: +# group.id: +# enable.auto.commit: +# auto.offset.reset: +# td.connect.ip: +# td.connect.user:root +# td.connect.pass:taosdata +# td.connect.port:6030 +# td.connect.db:db + +$numOfTopics = 2 +$expectConsumeMsgCnt = $consumeMsgCntFromTopic * $numOfTopics +$expect_result = @{consume success: @ +$expect_result = $expect_result . $expectConsumeMsgCnt +$expect_result = $expect_result . @, @ +$expect_result = $expect_result . 0} +print expect_result----> $expect_result +#print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_column, topic_stb_function, topic_stb_all" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt +#system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_column, topic_stb_function, topic_stb_all" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt +print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_column, topic_stb_function" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt +system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_column, topic_stb_function" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt +print cmd result----> $system_content +#if $system_content != @{consume success: 20000, 0}@ then +if $system_content < $expect_result then + return -1 +endi + +$numOfTopics = 3 +$expectConsumeMsgCnt = $consumeMsgCntFromTopic * $numOfTopics +$expect_result = @{consume success: @ +$expect_result = $expect_result . $expectConsumeMsgCnt +$expect_result = $expect_result . @, @ +$expect_result = $expect_result . 0} +print expect_result----> $expect_result +print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_column, topic_ctb_function, topic_ctb_all" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt +system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_column, topic_ctb_function, topic_ctb_all" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt +print cmd result----> $system_content +#if $system_content != @{consume success: 300, 0}@ then +if $system_content < $expectConsumeMsgCnt then + return -1 +endi + +$numOfTopics = 3 +$expectConsumeMsgCnt = $consumeMsgCntFromTopic * $numOfTopics +$expect_result = @{consume success: @ +$expect_result = $expect_result . $expectConsumeMsgCnt +$expect_result = $expect_result . @, @ +$expect_result = $expect_result . 0} +print expect_result----> $expect_result +print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_column, topic_ntb_all, topic_ntb_function" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt +system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_column, topic_ntb_all, topic_ntb_function" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt +print cmd result----> $system_content +#if $system_content != @{consume success: 30000, 0}@ then +if $system_content < $expectConsumeMsgCnt then + return -1 +endi + +if $loop_cnt == 0 then + $loop_cnt = 1 + $vgroups = 4 + $dbNamme = d1 + goto loop_vgroups +endi + + +#system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/tmq/mainConsumerInOneTopic.sim b/tests/script/tsim/tmq/mainConsumerInOneTopic.sim new file mode 100644 index 0000000000..b9a867921e --- /dev/null +++ b/tests/script/tsim/tmq/mainConsumerInOneTopic.sim @@ -0,0 +1,266 @@ +#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406 +# scene1: vgroups=1, one topic for one consumer, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb +# scene2: vgroups=1, multi topics for one consumer, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb +# scene3: vgroups=4, one topic for one consumer, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb +# scene4: vgroups=4, multi topics for one consumer, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb +# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN +# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5; +# +# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval). +# +######## ######## ######## ######## ######## ######## ######## ######## ######## ######## +######## This test case include scene1 and scene3 +######## ######## ######## ######## ######## ######## ######## ######## ######## ######## + +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 +system sh/exec.sh -n dnode1 -s start + +$loop_cnt = 0 +check_dnode_ready: + $loop_cnt = $loop_cnt + 1 + sleep 200 + if $loop_cnt == 10 then + print ====> dnode not ready! + return -1 + endi +sql show dnodes +print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05 +if $data00 != 1 then + return -1 +endi +if $data04 != ready then + goto check_dnode_ready +endi + +sql connect + +$loop_cnt = 0 +$vgroups = 1 +$dbNamme = d0 +loop_vgroups: +print =============== create database $dbNamme vgroups $vgroups +sql create database $dbNamme vgroups $vgroups +sql show databases +print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +print $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19 +print $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29 + +if $loop_cnt == 0 then + if $rows != 2 then + return -1 + endi + if $data02 != 1 then # vgroups + print vgroups: $data02 + return -1 + endi +else + if $rows != 3 then + return -1 + endi + if $data00 == d1 then + if $data02 != 4 then # vgroups + print vgroups: $data02 + return -1 + endi + else + if $data12 != 4 then # vgroups + print vgroups: $data12 + return -1 + endi + endi +endi + +sql use $dbNamme + +print =============== create super table +sql create table if not exists stb (ts timestamp, c1 int, c2 float, c3 binary(10)) tags (t1 int) + +sql show stables +if $rows != 1 then + return -1 +endi + +print =============== create child table +$tbPrefix = ct +$tbNum = 100 + +$i = 0 +while $i < $tbNum + $tb = $tbPrefix . $i + sql create table $tb using stb tags( $i ) + $i = $i + 1 +endw + +print =============== create normal table +sql create table ntb (ts timestamp, c1 int, c2 float, c3 binary(10)) + +print =============== create multi topics. notes: now only support: +print =============== 1. columns from stb/ctb/ntb; 2. * from ctb/ntb; 3. function from stb/ctb/ntb +print =============== will support: * from stb + +sql create topic topic_stb_column as select ts, c1, c3 from stb +#sql create topic topic_stb_all as select * from stb +sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb + +sql create topic topic_ctb_column as select ts, c1, c3 from ct0 +sql create topic topic_ctb_all as select * from ct0 +sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ct0 + +sql create topic topic_ntb_column as select ts, c1, c3 from ntb +sql create topic topic_ntb_all as select * from ntb +sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb + +sql show tables +if $rows != 101 then + return -1 +endi + +print =============== run_back insert data + +if $loop_cnt == 0 then + run_back tsim/tmq/insertDataV1.sim +else + run_back tsim/tmq/insertDataV4.sim +endi + +#sleep 1000 + +#$rowNum = 1000 +#$tstart = 1640966400000 # 2022-01-01 00:00:00.000 +# +#$i = 0 +#while $i < $tbNum +# $tb = $tbPrefix . $i +# +# $x = 0 +# while $x < $rowNum +# $c = $x / 10 +# $c = $c * 10 +# $c = $x - $c +# +# $binary = ' . binary +# $binary = $binary . $c +# $binary = $binary . ' +# +# sql insert into $tb values ($tstart , $c , $x , $binary ) +# sql insert into ntb values ($tstart , $c , $x , $binary ) +# $tstart = $tstart + 1 +# $x = $x + 1 +# endw +# +# $i = $i + 1 +# $tstart = 1640966400000 +#endw + +#root@trd02 /home $ tmq_sim --help +# -c Configuration directory, default is +# -d The name of the database for cosumer, no default +# -t The topic string for cosumer, no default +# -k The key-value string for cosumer, no default +# -g showMsgFlag, default is 0 +# + +$consumeDelay = 50 +$expectConsumeMsgCnt = 1000 +print expectConsumeMsgCnt: $expectConsumeMsgCnt , consumeDelay: $consumeDelay + +# supported key: +# group.id: +# enable.auto.commit: +# auto.offset.reset: +# td.connect.ip: +# td.connect.user:root +# td.connect.pass:taosdata +# td.connect.port:6030 +# td.connect.db:db + +$expect_result = @{consume success: @ +$expect_result = $expect_result . $expectConsumeMsgCnt +$expect_result = $expect_result . @, @ +$expect_result = $expect_result . 0} +print expect_result----> $expect_result +print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_column" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt +system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_column" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt +print cmd result----> $system_content +if $system_content < $expectConsumeMsgCnt then + return -1 +endi + +#print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_all" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt +#system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_all" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt +#print cmd result----> $system_content +##if $system_content != @{consume success: 10000, 0}@ then +#if $system_content < $expectConsumeMsgCnt then +# return -1 +#endi + +print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_function" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt +system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_function" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt +print cmd result----> $system_content +#if $system_content != @{consume success: 10000, 0}@ then +if $system_content < $expectConsumeMsgCnt then + return -1 +endi + +$expect_result = @{consume success: @ +$expect_result = $expect_result . $rowNum +$expect_result = $expect_result . @, @ +$expect_result = $expect_result . 0} +print expect_result----> $expect_result +print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_column" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt +system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_column" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt +print cmd result----> $system_content +if $system_content < $expectConsumeMsgCnt then + return -1 +endi + +print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_all" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt +system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_all" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt +print cmd result----> $system_content +if $system_content < $expectConsumeMsgCnt then + return -1 +endi + +print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_function" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt +system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_function" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt +print cmd result----> $system_content +if $system_content < $expectConsumeMsgCnt then + return -1 +endi + +$expect_result = @{consume success: @ +$expect_result = $expect_result . $totalMsgCnt +$expect_result = $expect_result . @, @ +$expect_result = $expect_result . 0} +print expect_result----> $expect_result +print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_column" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt +system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_column" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt +print cmd result----> $system_content +if $system_content < $expectConsumeMsgCnt then + return -1 +endi + +print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_all" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt +system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_all" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt +print cmd result----> $system_content +if $system_content < $expectConsumeMsgCnt then + return -1 +endi + +print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_function" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt +system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_function" -k "group.id:tg2" -y $consumeDelay -m $expectConsumeMsgCnt +print cmd result----> $system_content +if $system_content < $expectConsumeMsgCnt then + return -1 +endi + +if $loop_cnt == 0 then + $loop_cnt = 1 + $vgroups = 4 + $dbNamme = d1 + goto loop_vgroups +endi + +#system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/test/c/tmqSim.c b/tests/test/c/tmqSim.c index 236d1e2eed..dc375dd35a 100644 --- a/tests/test/c/tmqSim.c +++ b/tests/test/c/tmqSim.c @@ -42,6 +42,8 @@ typedef struct { char topicString[256]; char keyString[1024]; int32_t showMsgFlag; + int32_t consumeDelay; // unit s + int32_t consumeMsgCnt; // save result after parse agrvs int32_t numOfTopic; @@ -71,12 +73,19 @@ static void printHelp() { printf("%s%s%s\n", indent, indent, "The key-value string for cosumer, no default "); printf("%s%s\n", indent, "-g"); printf("%s%s%s%d\n", indent, indent, "showMsgFlag, default is ", g_stConfInfo.showMsgFlag); + printf("%s%s\n", indent, "-y"); + printf("%s%s%s%d\n", indent, indent, "consume delay, default is s", g_stConfInfo.consumeDelay); + printf("%s%s\n", indent, "-m"); + printf("%s%s%s%d\n", indent, indent, "consume msg count, default is s", g_stConfInfo.consumeMsgCnt); exit(EXIT_SUCCESS); } void parseArgument(int32_t argc, char *argv[]) { memset(&g_stConfInfo, 0, sizeof(SConfInfo)); + g_stConfInfo.showMsgFlag = 0; + g_stConfInfo.consumeDelay = 8000; + g_stConfInfo.consumeMsgCnt = 0; for (int32_t i = 1; i < argc; i++) { if (strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "--help") == 0) { @@ -92,6 +101,10 @@ void parseArgument(int32_t argc, char *argv[]) { strcpy(g_stConfInfo.keyString, argv[++i]); } else if (strcmp(argv[i], "-g") == 0) { g_stConfInfo.showMsgFlag = atol(argv[++i]); + } else if (strcmp(argv[i], "-y") == 0) { + g_stConfInfo.consumeDelay = atol(argv[++i]); + } else if (strcmp(argv[i], "-m") == 0) { + g_stConfInfo.consumeMsgCnt = atol(argv[++i]); } else { printf("%s unknow para: %s %s", GREEN, argv[++i], NC); exit(-1); @@ -256,6 +269,48 @@ void loop_consume(tmq_t* tmq) { printf("{consume success: %d, %d}", totalMsgs, totalRows); } + +void parallel_consume(tmq_t* tmq) { + tmq_resp_err_t err; + + int32_t totalMsgs = 0; + int32_t totalRows = 0; + int32_t skipLogNum = 0; + while (running) { + tmq_message_t* tmqMsg = tmq_consumer_poll(tmq, g_stConfInfo.consumeDelay * 1000); + if (tmqMsg) { + totalMsgs++; + + #if 0 + TAOS_ROW row; + while (NULL != (row = tmq_get_row(tmqMsg))) { + totalRows++; + } + #endif + + skipLogNum += tmqGetSkipLogNum(tmqMsg); + if (0 != g_stConfInfo.showMsgFlag) { + msg_process(tmqMsg); + } + tmq_message_destroy(tmqMsg); + + if (totalMsgs >= g_stConfInfo.consumeMsgCnt) { + break; + } + } else { + break; + } + } + + err = tmq_consumer_close(tmq); + if (err) { + printf("tmq_consumer_close() fail, reason: %s\n", tmq_err2str(err)); + exit(-1); + } + + printf("%d", totalMsgs); // output to sim for check result +} + int main(int32_t argc, char *argv[]) { parseArgument(argc, argv); parseInputString(); @@ -271,8 +326,12 @@ int main(int32_t argc, char *argv[]) { printf("tmq_subscribe() fail, reason: %s\n", tmq_err2str(err)); exit(-1); } - - loop_consume(tmq); + + if (0 == g_stConfInfo.consumeMsgCnt) { + loop_consume(tmq); + } else { + parallel_consume(tmq); + } err = tmq_unsubscribe(tmq); if (err) { diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 0e611f3794..b89d517ad3 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -26,6 +26,7 @@ #include "tglobal.h" #include "ttypes.h" #include "tutil.h" +#include "tconfig.h" #include #include @@ -90,6 +91,11 @@ TAOS *shellInit(SShellArguments *_args) { _args->user = TSDB_DEFAULT_USER; } + SConfig *pCfg = cfgInit(); + if (NULL == pCfg) return NULL; + + if (0 != taosAddClientLogCfg(pCfg)) return NULL; + // Connect to the database. TAOS *con = NULL; if (_args->auth == NULL) { diff --git a/tools/taos-tools b/tools/taos-tools index 33cdfe4f90..bf6c766986 160000 --- a/tools/taos-tools +++ b/tools/taos-tools @@ -1 +1 @@ -Subproject commit 33cdfe4f90a209f105c1b6091439798a9cde1e93 +Subproject commit bf6c766986c61ff4fc80421fdea682a8fd4b5b32