diff --git a/examples/c/stream_demo.c b/examples/c/stream_demo.c index 961eb6c93a..a4fc30ff65 100644 --- a/examples/c/stream_demo.c +++ b/examples/c/stream_demo.c @@ -25,7 +25,7 @@ int32_t init_env() { return -1; } - TAOS_RES* pRes = taos_query(pConn, "create database if not exists abc1 vgroups 1"); + TAOS_RES* pRes = taos_query(pConn, "create database if not exists abc1 vgroups 2"); if (taos_errno(pRes) != 0) { printf("error in create db, reason:%s\n", taos_errstr(pRes)); return -1; @@ -68,6 +68,14 @@ int32_t init_env() { return -1; } taos_free_result(pRes); + + pRes = taos_query(pConn, "create table if not exists tu3 using st1 tags(3)"); + if (taos_errno(pRes) != 0) { + printf("failed to create child table tu3, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + return 0; } @@ -90,10 +98,9 @@ int32_t create_stream() { /*const char* sql = "select min(k), max(k), sum(k) as sum_of_k from st1";*/ /*const char* sql = "select sum(k) from tu1 interval(10m)";*/ /*pRes = tmq_create_stream(pConn, "stream1", "out1", sql);*/ - pRes = taos_query( - pConn, - "create stream stream1 trigger window_close watermark 10s into outstb as select _wstartts, sum(k) from st1 " - "interval(10s) "); + pRes = taos_query(pConn, + "create stream stream1 trigger at_once into outstb as select _wstartts, sum(k) from st1 " + "partition by tbname interval(10s) "); if (taos_errno(pRes) != 0) { printf("failed to create stream stream1, reason:%s\n", taos_errstr(pRes)); return -1; diff --git a/include/common/tcommon.h b/include/common/tcommon.h index 09f24cc555..9c84d0dd99 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -50,8 +50,10 @@ typedef enum EStreamType { } EStreamType; typedef struct { + SArray* pGroupList; SArray* pTableList; SHashObj* map; // speedup acquire the tableQueryInfo by table uid + bool needSortTableByGroupId; void* pTagCond; void* pTagIndexCond; uint64_t suid; diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index 6738fc23bc..00acc4741d 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -36,7 +36,7 @@ typedef struct SReadHandle { void* vnode; void* mnd; SMsgCb* pMsgCb; - int8_t initTsdbReader; +// int8_t initTsdbReader; } SReadHandle; enum { diff --git a/include/libs/function/functionMgt.h b/include/libs/function/functionMgt.h index 1ca51b3043..1ed78750d1 100644 --- a/include/libs/function/functionMgt.h +++ b/include/libs/function/functionMgt.h @@ -124,6 +124,7 @@ typedef enum EFunctionType { FUNCTION_TYPE_BLOCK_DIST, // block distribution aggregate function FUNCTION_TYPE_BLOCK_DIST_INFO, // block distribution pseudo column function FUNCTION_TYPE_TO_COLUMN, + FUNCTION_TYPE_GROUP_KEY, // distributed splitting functions FUNCTION_TYPE_APERCENTILE_PARTIAL = 4000, diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index f8cf58d8cb..b7583b41b9 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -458,6 +458,7 @@ typedef struct SSubplan { int32_t msgType; // message type for subplan, used to denote the send message type to vnode. int32_t level; // the execution level of current subplan, starting from 0 in a top-down manner. char dbFName[TSDB_DB_FNAME_LEN]; + char user[TSDB_USER_LEN]; SQueryNodeAddr execNode; // for the scan/modify subplan, the optional execution node SQueryNodeStat execNodeStat; // only for scan subplan SNodeList* pChildren; // the datasource subplan,from which to fetch the result diff --git a/include/libs/planner/planner.h b/include/libs/planner/planner.h index c4f71e57a8..b350837551 100644 --- a/include/libs/planner/planner.h +++ b/include/libs/planner/planner.h @@ -24,18 +24,19 @@ extern "C" { #include "taos.h" typedef struct SPlanContext { - uint64_t queryId; - int32_t acctId; - SEpSet mgmtEpSet; - SNode* pAstRoot; - bool topicQuery; - bool streamQuery; - bool rSmaQuery; - bool showRewrite; - int8_t triggerType; - int64_t watermark; - char* pMsg; - int32_t msgLen; + uint64_t queryId; + int32_t acctId; + SEpSet mgmtEpSet; + SNode* pAstRoot; + bool topicQuery; + bool streamQuery; + bool rSmaQuery; + bool showRewrite; + int8_t triggerType; + int64_t watermark; + char* pMsg; + int32_t msgLen; + const char* pUser; } SPlanContext; // Create the physical plan for the query, according to the AST. diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 3a8b8ca438..ac9daa5119 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -376,7 +376,8 @@ int32_t getPlan(SRequestObj* pRequest, SQuery* pQuery, SQueryPlan** pPlan, SArra .pAstRoot = pQuery->pRoot, .showRewrite = pQuery->showRewrite, .pMsg = pRequest->msgBuf, - .msgLen = ERROR_MSG_BUF_DEFAULT_SIZE}; + .msgLen = ERROR_MSG_BUF_DEFAULT_SIZE, + .pUser = pRequest->pTscObj->user}; return qCreateQueryPlan(&cxt, pPlan, pNodeList); } @@ -954,7 +955,8 @@ void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData* pResultM .pAstRoot = pQuery->pRoot, .showRewrite = pQuery->showRewrite, .pMsg = pRequest->msgBuf, - .msgLen = ERROR_MSG_BUF_DEFAULT_SIZE}; + .msgLen = ERROR_MSG_BUF_DEFAULT_SIZE, + .pUser = pRequest->pTscObj->user}; SAppInstInfo* pAppInfo = getAppInfo(pRequest); code = qCreateQueryPlan(&cxt, &pRequest->body.pDag, pMnodeList); @@ -1836,17 +1838,18 @@ _OVER: return code; } -int32_t appendTbToReq(SArray* pList, int32_t pos1, int32_t len1, int32_t pos2, int32_t len2, const char* str, int32_t acctId, char* db) { +int32_t appendTbToReq(SArray* pList, int32_t pos1, int32_t len1, int32_t pos2, int32_t len2, const char* str, + int32_t acctId, char* db) { SName name; - + if (len1 <= 0) { return -1; } - const char *dbName = db; - const char *tbName = NULL; - int32_t dbLen = 0; - int32_t tbLen = 0; + const char* dbName = db; + const char* tbName = NULL; + int32_t dbLen = 0; + int32_t tbLen = 0; if (len2 > 0) { dbName = str + pos1; dbLen = len1; @@ -1857,7 +1860,7 @@ int32_t appendTbToReq(SArray* pList, int32_t pos1, int32_t len1, int32_t pos2, i tbName = str + pos1; tbLen = len1; } - + if (tNameSetDbName(&name, acctId, dbName, dbLen)) { return -1; } @@ -1877,18 +1880,18 @@ int32_t transferTableNameList(const char* tbList, int32_t acctId, char* dbName, terrno = TSDB_CODE_OUT_OF_MEMORY; return terrno; } - - bool inEscape = false; + + bool inEscape = false; int32_t code = 0; - + int32_t vIdx = 0; int32_t vPos[2]; int32_t vLen[2]; memset(vPos, -1, sizeof(vPos)); memset(vLen, 0, sizeof(vLen)); - - for (int32_t i = 0; ; ++i) { + + for (int32_t i = 0;; ++i) { if (0 == *(tbList + i)) { if (vPos[vIdx] >= 0 && vLen[vIdx] <= 0) { vLen[vIdx] = i - vPos[vIdx]; @@ -1901,7 +1904,7 @@ int32_t transferTableNameList(const char* tbList, int32_t acctId, char* dbName, break; } - + if ('`' == *(tbList + i)) { inEscape = !inEscape; if (!inEscape) { @@ -1948,7 +1951,7 @@ int32_t transferTableNameList(const char* tbList, int32_t acctId, char* dbName, if (code) { goto _return; } - + memset(vPos, -1, sizeof(vPos)); memset(vLen, 0, sizeof(vLen)); vIdx = 0; @@ -1962,8 +1965,7 @@ int32_t transferTableNameList(const char* tbList, int32_t acctId, char* dbName, continue; } - if (('a' <= *(tbList + i) && 'z' >= *(tbList + i)) || - ('A' <= *(tbList + i) && 'Z' >= *(tbList + i)) || + if (('a' <= *(tbList + i) && 'z' >= *(tbList + i)) || ('A' <= *(tbList + i) && 'Z' >= *(tbList + i)) || ('0' <= *(tbList + i) && '9' >= *(tbList + i))) { if (vLen[vIdx] > 0) { goto _return; @@ -1985,32 +1987,31 @@ _return: taosArrayDestroy(*pReq); *pReq = NULL; - + return terrno; } void syncCatalogFn(SMetaData* pResult, void* param, int32_t code) { - SSyncQueryParam *pParam = param; + SSyncQueryParam* pParam = param; pParam->pRequest->code = code; tsem_post(&pParam->sem); } - -void syncQueryFn(void *param, void *res, int32_t code) { - SSyncQueryParam *pParam = param; +void syncQueryFn(void* param, void* res, int32_t code) { + SSyncQueryParam* pParam = param; pParam->pRequest = res; pParam->pRequest->code = code; tsem_post(&pParam->sem); } -void taosAsyncQueryImpl(TAOS *taos, const char *sql, __taos_async_fn_t fp, void *param, bool validateOnly) { - STscObj *pTscObj = acquireTscObj(*(int64_t *)taos); +void taosAsyncQueryImpl(TAOS* taos, const char* sql, __taos_async_fn_t fp, void* param, bool validateOnly) { + STscObj* pTscObj = acquireTscObj(*(int64_t*)taos); if (pTscObj == NULL || sql == NULL || NULL == fp) { terrno = TSDB_CODE_INVALID_PARA; if (pTscObj) { - releaseTscObj(*(int64_t *)taos); + releaseTscObj(*(int64_t*)taos); } else { terrno = TSDB_CODE_TSC_DISCONNECTED; } @@ -2027,7 +2028,7 @@ void taosAsyncQueryImpl(TAOS *taos, const char *sql, __taos_async_fn_t fp, void return; } - SRequestObj *pRequest = NULL; + SRequestObj* pRequest = NULL; int32_t code = buildRequest(pTscObj, sql, sqlLen, &pRequest); if (code != TSDB_CODE_SUCCESS) { terrno = code; @@ -2041,45 +2042,41 @@ void taosAsyncQueryImpl(TAOS *taos, const char *sql, __taos_async_fn_t fp, void doAsyncQuery(pRequest, false); } - -TAOS_RES *taosQueryImpl(TAOS *taos, const char *sql, bool validateOnly) { +TAOS_RES* taosQueryImpl(TAOS* taos, const char* sql, bool validateOnly) { if (NULL == taos) { terrno = TSDB_CODE_TSC_DISCONNECTED; return NULL; } - STscObj *pTscObj = acquireTscObj(*(int64_t *)taos); + STscObj* pTscObj = acquireTscObj(*(int64_t*)taos); if (pTscObj == NULL || sql == NULL) { terrno = TSDB_CODE_TSC_DISCONNECTED; return NULL; } #if SYNC_ON_TOP_OF_ASYNC - SSyncQueryParam *param = taosMemoryCalloc(1, sizeof(SSyncQueryParam)); + SSyncQueryParam* param = taosMemoryCalloc(1, sizeof(SSyncQueryParam)); tsem_init(¶m->sem, 0, 0); taosAsyncQueryImpl(taos, sql, syncQueryFn, param, validateOnly); tsem_wait(¶m->sem); - releaseTscObj(*(int64_t *)taos); + releaseTscObj(*(int64_t*)taos); return param->pRequest; #else size_t sqlLen = strlen(sql); if (sqlLen > (size_t)TSDB_MAX_ALLOWED_SQL_LEN) { - releaseTscObj(*(int64_t *)taos); + releaseTscObj(*(int64_t*)taos); tscError("sql string exceeds max length:%d", TSDB_MAX_ALLOWED_SQL_LEN); terrno = TSDB_CODE_TSC_EXCEED_SQL_LIMIT; return NULL; } - TAOS_RES *pRes = execQuery(pTscObj, sql, sqlLen, validateOnly); + TAOS_RES* pRes = execQuery(pTscObj, sql, sqlLen, validateOnly); - releaseTscObj(*(int64_t *)taos); + releaseTscObj(*(int64_t*)taos); return pRes; #endif } - - - diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index e4bb30b001..b104e1c2be 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -963,6 +963,10 @@ int32_t taosSetCfg(SConfig *pCfg, char* name) { tsTelemPort = (uint16_t)cfgGetItem(pCfg, "telemetryPort")->i32; } else if (strcasecmp("transPullupInterval", name) == 0) { tsTransPullupInterval = cfgGetItem(pCfg, "transPullupInterval")->i32; + } else if (strcasecmp("ttlUnit", name) == 0) { + tsTtlUnit = cfgGetItem(pCfg, "ttlUnit")->i32; + } else if (strcasecmp("ttlPushInterval", name) == 0) { + tsTtlPushInterval = cfgGetItem(pCfg, "ttlPushInterval")->i32; } else if (strcasecmp("tmrDebugFlag", name) == 0) { tmrDebugFlag = cfgGetItem(pCfg, "tmrDebugFlag")->i32; } else if (strcasecmp("tsdbDebugFlag", name) == 0) { diff --git a/source/dnode/mnode/impl/inc/mndAuth.h b/source/dnode/mnode/impl/inc/mndPrivilege.h similarity index 66% rename from source/dnode/mnode/impl/inc/mndAuth.h rename to source/dnode/mnode/impl/inc/mndPrivilege.h index 9af4792665..15f9e4e6b5 100644 --- a/source/dnode/mnode/impl/inc/mndAuth.h +++ b/source/dnode/mnode/impl/inc/mndPrivilege.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef _TD_MND_AUTH_H_ -#define _TD_MND_AUTH_H_ +#ifndef _TD_MND_PRIVILEGE_H +#define _TD_MND_PRIVILEGE_H #include "mndInt.h" @@ -24,6 +24,9 @@ extern "C" { typedef enum { MND_OPER_CONNECT = 1, + MND_OPER_CREATE_ACCT, + MND_OPER_DROP_ACCT, + MND_OPER_ALTER_ACCT, MND_OPER_CREATE_USER, MND_OPER_DROP_USER, MND_OPER_ALTER_USER, @@ -31,6 +34,7 @@ typedef enum { MND_OPER_DROP_BNODE, MND_OPER_CREATE_DNODE, MND_OPER_DROP_DNODE, + MND_OPER_CONFIG_DNODE, MND_OPER_CREATE_MNODE, MND_OPER_DROP_MNODE, MND_OPER_CREATE_QNODE, @@ -38,11 +42,14 @@ typedef enum { MND_OPER_CREATE_SNODE, MND_OPER_DROP_SNODE, MND_OPER_REDISTRIBUTE_VGROUP, + MND_OPER_MERGE_VGROUP, MND_OPER_SPLIT_VGROUP, MND_OPER_BALANCE_VGROUP, MND_OPER_CREATE_FUNC, MND_OPER_DROP_FUNC, MND_OPER_KILL_TRANS, + MND_OPER_KILL_CONN, + MND_OPER_KILL_QUERY, MND_OPER_CREATE_DB, MND_OPER_ALTER_DB, MND_OPER_DROP_DB, @@ -52,16 +59,16 @@ typedef enum { MND_OPER_READ_DB, } EOperType; -int32_t mndInitAuth(SMnode *pMnode); -void mndCleanupAuth(SMnode *pMnode); +int32_t mndInitPrivilege(SMnode *pMnode); +void mndCleanupPrivilege(SMnode *pMnode); -int32_t mndCheckOperAuth(SMnode *pMnode, const char *user, EOperType operType); -int32_t mndCheckDbAuth(SMnode *pMnode, const char *user, EOperType operType, SDbObj *pDb); -int32_t mndCheckShowAuth(SMnode *pMnode, const char *user, int32_t showType); -int32_t mndCheckAlterUserAuth(SUserObj *pOperUser, SUserObj *pUser, SAlterUserReq *pAlter); +int32_t mndCheckOperPrivilege(SMnode *pMnode, const char *user, EOperType operType); +int32_t mndCheckDbPrivilege(SMnode *pMnode, const char *user, EOperType operType, SDbObj *pDb); +int32_t mndCheckShowPrivilege(SMnode *pMnode, const char *user, int32_t showType); +int32_t mndCheckAlterUserPrivilege(SUserObj *pOperUser, SUserObj *pUser, SAlterUserReq *pAlter); #ifdef __cplusplus } #endif -#endif /*_TD_MND_AUTH_H_*/ +#endif /*_TD_MND_PRIVILEGE_H*/ diff --git a/source/dnode/mnode/impl/src/mndAcct.c b/source/dnode/mnode/impl/src/mndAcct.c index 0ce4a8c76e..33f0bb7a34 100644 --- a/source/dnode/mnode/impl/src/mndAcct.c +++ b/source/dnode/mnode/impl/src/mndAcct.c @@ -15,6 +15,7 @@ #define _DEFAULT_SOURCE #include "mndAcct.h" +#include "mndPrivilege.h" #include "mndShow.h" #include "mndTrans.h" @@ -212,18 +213,30 @@ static int32_t mndAcctActionUpdate(SSdb *pSdb, SAcctObj *pOld, SAcctObj *pNew) { } static int32_t mndProcessCreateAcctReq(SRpcMsg *pReq) { + if (mndCheckOperPrivilege(pReq->info.node, pReq->info.conn.user, MND_OPER_CREATE_ACCT) != 0) { + return -1; + } + terrno = TSDB_CODE_MSG_NOT_PROCESSED; mError("failed to process create acct request since %s", terrstr()); return -1; } static int32_t mndProcessAlterAcctReq(SRpcMsg *pReq) { + if (mndCheckOperPrivilege(pReq->info.node, pReq->info.conn.user, MND_OPER_ALTER_ACCT) != 0) { + return -1; + } + terrno = TSDB_CODE_MSG_NOT_PROCESSED; mError("failed to process create acct request since %s", terrstr()); return -1; } static int32_t mndProcessDropAcctReq(SRpcMsg *pReq) { + if (mndCheckOperPrivilege(pReq->info.node, pReq->info.conn.user, MND_OPER_DROP_ACCT) != 0) { + return -1; + } + terrno = TSDB_CODE_MSG_NOT_PROCESSED; mError("failed to process create acct request since %s", terrstr()); return -1; diff --git a/source/dnode/mnode/impl/src/mndBnode.c b/source/dnode/mnode/impl/src/mndBnode.c index 0de40ca671..aafcd19992 100644 --- a/source/dnode/mnode/impl/src/mndBnode.c +++ b/source/dnode/mnode/impl/src/mndBnode.c @@ -15,7 +15,7 @@ #define _DEFAULT_SOURCE #include "mndBnode.h" -#include "mndAuth.h" +#include "mndPrivilege.h" #include "mndDnode.h" #include "mndShow.h" #include "mndTrans.h" @@ -277,6 +277,9 @@ static int32_t mndProcessCreateBnodeReq(SRpcMsg *pReq) { } mDebug("bnode:%d, start to create", createReq.dnodeId); + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_BNODE) != 0) { + goto _OVER; + } pObj = mndAcquireBnode(pMnode, createReq.dnodeId); if (pObj != NULL) { @@ -292,10 +295,6 @@ static int32_t mndProcessCreateBnodeReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_CREATE_BNODE) != 0) { - goto _OVER; - } - code = mndCreateBnode(pMnode, pReq, pDnode, &createReq); if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; @@ -383,6 +382,9 @@ static int32_t mndProcessDropBnodeReq(SRpcMsg *pReq) { } mDebug("bnode:%d, start to drop", dropReq.dnodeId); + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_BNODE) != 0) { + goto _OVER; + } if (dropReq.dnodeId <= 0) { terrno = TSDB_CODE_INVALID_MSG; @@ -394,10 +396,6 @@ static int32_t mndProcessDropBnodeReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_DROP_BNODE) != 0) { - goto _OVER; - } - code = mndDropBnode(pMnode, pReq, pObj); if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index 4da3c906d7..7dc5ee1ea1 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -15,7 +15,7 @@ #define _DEFAULT_SOURCE #include "mndConsumer.h" -#include "mndAuth.h" +#include "mndPrivilege.h" #include "mndDb.h" #include "mndDnode.h" #include "mndMnode.h" diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 345464399e..0345f1b345 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -15,7 +15,7 @@ #define _DEFAULT_SOURCE #include "mndDb.h" -#include "mndAuth.h" +#include "mndPrivilege.h" #include "mndDnode.h" #include "mndOffset.h" #include "mndShow.h" @@ -506,6 +506,9 @@ static int32_t mndProcessCreateDbReq(SRpcMsg *pReq) { } mDebug("db:%s, start to create, vgroups:%d", createReq.db, createReq.numOfVgroups); + if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_DB, NULL) != 0) { + goto _OVER; + } pDb = mndAcquireDb(pMnode, createReq.db); if (pDb != NULL) { @@ -526,10 +529,6 @@ static int32_t mndProcessCreateDbReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckDbAuth(pMnode, pReq->info.conn.user, MND_OPER_CREATE_DB, NULL) != 0) { - goto _OVER; - } - code = mndCreateDb(pMnode, pReq, &createReq, pUser); if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; @@ -700,7 +699,7 @@ static int32_t mndProcessAlterDbReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckDbAuth(pMnode, pReq->info.conn.user, MND_OPER_ALTER_DB, pDb) != 0) { + if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_ALTER_DB, pDb) != 0) { goto _OVER; } @@ -980,7 +979,7 @@ static int32_t mndProcessDropDbReq(SRpcMsg *pReq) { } } - if (mndCheckDbAuth(pMnode, pReq->info.conn.user, MND_OPER_DROP_DB, pDb) != 0) { + if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_DB, pDb) != 0) { goto _OVER; } @@ -1127,7 +1126,7 @@ static int32_t mndProcessUseDbReq(SRpcMsg *pReq) { mError("db:%s, failed to process use db req since %s", usedbReq.db, terrstr()); } else { - if (mndCheckDbAuth(pMnode, pReq->info.conn.user, MND_OPER_USE_DB, pDb) != 0) { + if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_USE_DB, pDb) != 0) { goto _OVER; } @@ -1252,7 +1251,7 @@ static int32_t mndProcessCompactDbReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckDbAuth(pMnode, pReq->info.conn.user, MND_OPER_COMPACT_DB, pDb) != 0) { + if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_COMPACT_DB, pDb) != 0) { goto _OVER; } diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 0eab364e90..af1d641ebf 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -15,7 +15,7 @@ #define _DEFAULT_SOURCE #include "mndDnode.h" -#include "mndAuth.h" +#include "mndPrivilege.h" #include "mndMnode.h" #include "mndQnode.h" #include "mndShow.h" @@ -609,7 +609,6 @@ _OVER: return code; } - static int32_t mndProcessCreateDnodeReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; int32_t code = -1; @@ -622,6 +621,9 @@ static int32_t mndProcessCreateDnodeReq(SRpcMsg *pReq) { } mInfo("dnode:%s:%d, start to create", createReq.fqdn, createReq.port); + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_DNODE) != 0) { + goto _OVER; + } if (createReq.fqdn[0] == 0 || createReq.port <= 0 || createReq.port > UINT16_MAX) { terrno = TSDB_CODE_MND_INVALID_DNODE_EP; @@ -635,10 +637,6 @@ static int32_t mndProcessCreateDnodeReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_CREATE_DNODE) != 0) { - goto _OVER; - } - code = mndCreateDnode(pMnode, pReq, &createReq); if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; @@ -717,6 +715,9 @@ static int32_t mndProcessDropDnodeReq(SRpcMsg *pReq) { } mInfo("dnode:%d, start to drop, ep:%s:%d", dropReq.dnodeId, dropReq.fqdn, dropReq.port); + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_MNODE) != 0) { + goto _OVER; + } pDnode = mndAcquireDnode(pMnode, dropReq.dnodeId); if (pDnode == NULL) { @@ -753,10 +754,6 @@ static int32_t mndProcessDropDnodeReq(SRpcMsg *pReq) { } } - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_DROP_MNODE) != 0) { - goto _OVER; - } - code = mndDropDnode(pMnode, pReq, pDnode, pMObj, pQObj, pSObj, numOfVnodes); if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; @@ -782,6 +779,10 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) { } mInfo("dnode:%d, start to config, option:%s, value:%s", cfgReq.dnodeId, cfgReq.config, cfgReq.value); + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CONFIG_DNODE) != 0) { + return -1; + } + SDnodeObj *pDnode = mndAcquireDnode(pMnode, cfgReq.dnodeId); if (pDnode == NULL) { mError("dnode:%d, failed to config since %s ", cfgReq.dnodeId, terrstr()); diff --git a/source/dnode/mnode/impl/src/mndFunc.c b/source/dnode/mnode/impl/src/mndFunc.c index 832f1b8e68..b626c1fb04 100644 --- a/source/dnode/mnode/impl/src/mndFunc.c +++ b/source/dnode/mnode/impl/src/mndFunc.c @@ -15,7 +15,7 @@ #define _DEFAULT_SOURCE #include "mndFunc.h" -#include "mndAuth.h" +#include "mndPrivilege.h" #include "mndShow.h" #include "mndSync.h" #include "mndTrans.h" @@ -283,6 +283,9 @@ static int32_t mndProcessCreateFuncReq(SRpcMsg *pReq) { } mDebug("func:%s, start to create", createReq.name); + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_FUNC) != 0) { + goto _OVER; + } pFunc = mndAcquireFunc(pMnode, createReq.name); if (pFunc != NULL) { @@ -318,10 +321,6 @@ static int32_t mndProcessCreateFuncReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_CREATE_FUNC) != 0) { - goto _OVER; - } - code = mndCreateFunc(pMnode, pReq, &createReq); if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; @@ -347,6 +346,9 @@ static int32_t mndProcessDropFuncReq(SRpcMsg *pReq) { } mDebug("func:%s, start to drop", dropReq.name); + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_FUNC) != 0) { + goto _OVER; + } if (dropReq.name[0] == 0) { terrno = TSDB_CODE_MND_INVALID_FUNC_NAME; @@ -365,10 +367,6 @@ static int32_t mndProcessDropFuncReq(SRpcMsg *pReq) { } } - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_DROP_FUNC) != 0) { - goto _OVER; - } - code = mndDropFunc(pMnode, pReq, pFunc); if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index d07de7c048..4c9974ba1a 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -15,7 +15,7 @@ #define _DEFAULT_SOURCE #include "mndAcct.h" -#include "mndAuth.h" +#include "mndPrivilege.h" #include "mndBnode.h" #include "mndCluster.h" #include "mndConsumer.h" @@ -100,7 +100,7 @@ static void *mndThreadFp(void *param) { taosMsleep(100); if (mndGetStop(pMnode)) break; - if (lastTime % (tsTransPullupInterval * 10) == 1) { + if (lastTime % (tsTtlPushInterval * 10) == 1) { mndTtlTimer(pMnode); } @@ -239,7 +239,7 @@ static int32_t mndInitSteps(SMnode *pMnode) { if (mndAllocStep(pMnode, "mnode-dnode", mndInitDnode, mndCleanupDnode) != 0) return -1; if (mndAllocStep(pMnode, "mnode-user", mndInitUser, mndCleanupUser) != 0) return -1; if (mndAllocStep(pMnode, "mnode-grant", mndInitGrant, mndCleanupGrant) != 0) return -1; - if (mndAllocStep(pMnode, "mnode-auth", mndInitAuth, mndCleanupAuth) != 0) return -1; + if (mndAllocStep(pMnode, "mnode-privilege", mndInitPrivilege, mndCleanupPrivilege) != 0) return -1; if (mndAllocStep(pMnode, "mnode-acct", mndInitAcct, mndCleanupAcct) != 0) return -1; if (mndAllocStep(pMnode, "mnode-stream", mndInitStream, mndCleanupStream) != 0) return -1; if (mndAllocStep(pMnode, "mnode-topic", mndInitTopic, mndCleanupTopic) != 0) return -1; @@ -539,18 +539,23 @@ static int32_t mndCheckMnodeState(SRpcMsg *pMsg) { const STraceId *trace = &pMsg->info.traceId; mError("msg:%p, failed to check mnode state since %s, type:%s, numOfMnodes:%d inUse:%d", pMsg, terrstr(), TMSG_INFO(pMsg->msgType), epSet.numOfEps, epSet.inUse); - for (int32_t i = 0; i < epSet.numOfEps; ++i) { - mInfo("mnode index:%d, ep:%s:%u", i, epSet.eps[i].fqdn, epSet.eps[i].port); - } - int32_t contLen = tSerializeSEpSet(NULL, 0, &epSet); - pMsg->info.rsp = rpcMallocCont(contLen); - if (pMsg->info.rsp != NULL) { - tSerializeSEpSet(pMsg->info.rsp, contLen, &epSet); - pMsg->info.rspLen = contLen; - terrno = TSDB_CODE_RPC_REDIRECT; + if (epSet.numOfEps > 0) { + for (int32_t i = 0; i < epSet.numOfEps; ++i) { + mInfo("mnode index:%d, ep:%s:%u", i, epSet.eps[i].fqdn, epSet.eps[i].port); + } + + int32_t contLen = tSerializeSEpSet(NULL, 0, &epSet); + pMsg->info.rsp = rpcMallocCont(contLen); + if (pMsg->info.rsp != NULL) { + tSerializeSEpSet(pMsg->info.rsp, contLen, &epSet); + pMsg->info.rspLen = contLen; + terrno = TSDB_CODE_RPC_REDIRECT; + } else { + terrno = TSDB_CODE_OUT_OF_MEMORY; + } } else { - terrno = TSDB_CODE_OUT_OF_MEMORY; + terrno = TSDB_CODE_APP_NOT_READY; } return -1; diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index b40cd713e5..c03951b1d8 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -15,7 +15,7 @@ #define _DEFAULT_SOURCE #include "mndMnode.h" -#include "mndAuth.h" +#include "mndPrivilege.h" #include "mndDnode.h" #include "mndShow.h" #include "mndSync.h" @@ -389,6 +389,9 @@ static int32_t mndProcessCreateMnodeReq(SRpcMsg *pReq) { } mDebug("mnode:%d, start to create", createReq.dnodeId); + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_MNODE) != 0) { + goto _OVER; + } pObj = mndAcquireMnode(pMnode, createReq.dnodeId); if (pObj != NULL) { @@ -414,10 +417,6 @@ static int32_t mndProcessCreateMnodeReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_CREATE_MNODE) != 0) { - goto _OVER; - } - code = mndCreateMnode(pMnode, pReq, pDnode, &createReq); if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; @@ -495,7 +494,7 @@ static int32_t mndSetDropMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnode { int32_t contLen = tSerializeSSetStandbyReq(NULL, 0, &standbyReq) + sizeof(SMsgHead); void *pReq = taosMemoryMalloc(contLen); - tSerializeSSetStandbyReq((char*)pReq + sizeof(SMsgHead), contLen, &standbyReq); + tSerializeSSetStandbyReq((char *)pReq + sizeof(SMsgHead), contLen, &standbyReq); SMsgHead *pHead = pReq; pHead->contLen = htonl(contLen); pHead->vgId = htonl(MNODE_HANDLE); @@ -595,6 +594,9 @@ static int32_t mndProcessDropMnodeReq(SRpcMsg *pReq) { } mDebug("mnode:%d, start to drop", dropReq.dnodeId); + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_MNODE) != 0) { + goto _OVER; + } if (dropReq.dnodeId <= 0) { terrno = TSDB_CODE_INVALID_MSG; @@ -621,10 +623,6 @@ static int32_t mndProcessDropMnodeReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_DROP_MNODE) != 0) { - goto _OVER; - } - code = mndDropMnode(pMnode, pReq, pObj); if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; diff --git a/source/dnode/mnode/impl/src/mndOffset.c b/source/dnode/mnode/impl/src/mndOffset.c index 18f2e993b2..e2b20b2163 100644 --- a/source/dnode/mnode/impl/src/mndOffset.c +++ b/source/dnode/mnode/impl/src/mndOffset.c @@ -15,7 +15,7 @@ #define _DEFAULT_SOURCE #include "mndOffset.h" -#include "mndAuth.h" +#include "mndPrivilege.h" #include "mndDb.h" #include "mndDnode.h" #include "mndMnode.h" @@ -36,13 +36,15 @@ static int32_t mndOffsetActionUpdate(SSdb *pSdb, SMqOffsetObj *pOffset, SMqOffse static int32_t mndProcessCommitOffsetReq(SRpcMsg *pReq); int32_t mndInitOffset(SMnode *pMnode) { - SSdbTable table = {.sdbType = SDB_OFFSET, - .keyType = SDB_KEY_BINARY, - .encodeFp = (SdbEncodeFp)mndOffsetActionEncode, - .decodeFp = (SdbDecodeFp)mndOffsetActionDecode, - .insertFp = (SdbInsertFp)mndOffsetActionInsert, - .updateFp = (SdbUpdateFp)mndOffsetActionUpdate, - .deleteFp = (SdbDeleteFp)mndOffsetActionDelete}; + SSdbTable table = { + .sdbType = SDB_OFFSET, + .keyType = SDB_KEY_BINARY, + .encodeFp = (SdbEncodeFp)mndOffsetActionEncode, + .decodeFp = (SdbDecodeFp)mndOffsetActionDecode, + .insertFp = (SdbInsertFp)mndOffsetActionInsert, + .updateFp = (SdbUpdateFp)mndOffsetActionUpdate, + .deleteFp = (SdbDeleteFp)mndOffsetActionDelete, + }; mndSetMsgHandle(pMnode, TDMT_MND_MQ_COMMIT_OFFSET, mndProcessCommitOffsetReq); diff --git a/source/dnode/mnode/impl/src/mndAuth.c b/source/dnode/mnode/impl/src/mndPrivilege.c similarity index 66% rename from source/dnode/mnode/impl/src/mndAuth.c rename to source/dnode/mnode/impl/src/mndPrivilege.c index 4445e3b9f7..478ba2bee4 100644 --- a/source/dnode/mnode/impl/src/mndAuth.c +++ b/source/dnode/mnode/impl/src/mndPrivilege.c @@ -14,66 +14,14 @@ */ #define _DEFAULT_SOURCE -#include "mndAuth.h" +#include "mndPrivilege.h" #include "mndUser.h" -static int32_t mndProcessAuthReq(SRpcMsg *pReq); +int32_t mndInitPrivilege(SMnode *pMnode) { return 0; } -int32_t mndInitAuth(SMnode *pMnode) { - mndSetMsgHandle(pMnode, TDMT_MND_AUTH, mndProcessAuthReq); - return 0; -} +void mndCleanupPrivilege(SMnode *pMnode) {} -void mndCleanupAuth(SMnode *pMnode) {} - -static int32_t mndRetriveAuth(SMnode *pMnode, SAuthRsp *pRsp) { - SUserObj *pUser = mndAcquireUser(pMnode, pRsp->user); - if (pUser == NULL) { - *pRsp->secret = 0; - mError("user:%s, failed to auth user since %s", pRsp->user, terrstr()); - return -1; - } - - pRsp->spi = 1; - pRsp->encrypt = 0; - *pRsp->ckey = 0; - - memcpy(pRsp->secret, pUser->pass, TSDB_PASSWORD_LEN); - mndReleaseUser(pMnode, pUser); - - mDebug("user:%s, auth info is returned", pRsp->user); - return 0; -} - -static int32_t mndProcessAuthReq(SRpcMsg *pReq) { - SAuthReq authReq = {0}; - if (tDeserializeSAuthReq(pReq->pCont, pReq->contLen, &authReq) != 0) { - terrno = TSDB_CODE_INVALID_MSG; - return -1; - } - - SAuthReq authRsp = {0}; - memcpy(authRsp.user, authReq.user, TSDB_USER_LEN); - - int32_t code = mndRetriveAuth(pReq->info.node, &authRsp); - mTrace("user:%s, auth req received, spi:%d encrypt:%d ruser:%s", pReq->info.conn.user, authRsp.spi, authRsp.encrypt, - authRsp.user); - - int32_t contLen = tSerializeSAuthReq(NULL, 0, &authRsp); - void *pRsp = rpcMallocCont(contLen); - if (pRsp == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - tSerializeSAuthReq(pRsp, contLen, &authRsp); - - pReq->info.rsp = pRsp; - pReq->info.rspLen = contLen; - return code; -} - -int32_t mndCheckOperAuth(SMnode *pMnode, const char *user, EOperType operType) { +int32_t mndCheckOperPrivilege(SMnode *pMnode, const char *user, EOperType operType) { int32_t code = 0; SUserObj *pUser = mndAcquireUser(pMnode, user); @@ -95,6 +43,8 @@ int32_t mndCheckOperAuth(SMnode *pMnode, const char *user, EOperType operType) { switch (operType) { case MND_OPER_CONNECT: + case MND_OPER_CREATE_FUNC: + case MND_OPER_DROP_FUNC: break; default: terrno = TSDB_CODE_MND_NO_RIGHTS; @@ -106,7 +56,7 @@ _OVER: return code; } -int32_t mndCheckAlterUserAuth(SUserObj *pOperUser, SUserObj *pUser, SAlterUserReq *pAlter) { +int32_t mndCheckAlterUserPrivilege(SUserObj *pOperUser, SUserObj *pUser, SAlterUserReq *pAlter) { if (pUser->superUser && pAlter->alterType != TSDB_ALTER_USER_PASSWD) { terrno = TSDB_CODE_MND_NO_RIGHTS; return -1; @@ -129,7 +79,7 @@ int32_t mndCheckAlterUserAuth(SUserObj *pOperUser, SUserObj *pUser, SAlterUserRe return -1; } -int32_t mndCheckShowAuth(SMnode *pMnode, const char *user, int32_t showType) { +int32_t mndCheckShowPrivilege(SMnode *pMnode, const char *user, int32_t showType) { int32_t code = 0; SUserObj *pUser = mndAcquireUser(pMnode, user); @@ -162,7 +112,7 @@ _OVER: return code; } -int32_t mndCheckDbAuth(SMnode *pMnode, const char *user, EOperType operType, SDbObj *pDb) { +int32_t mndCheckDbPrivilege(SMnode *pMnode, const char *user, EOperType operType, SDbObj *pDb) { int32_t code = 0; SUserObj *pUser = mndAcquireUser(pMnode, user); diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index e9df4ae1d0..f2e599b073 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -15,7 +15,7 @@ #define _DEFAULT_SOURCE #include "mndProfile.h" -#include "mndAuth.h" +#include "mndPrivilege.h" #include "mndDb.h" #include "mndDnode.h" #include "mndMnode.h" @@ -227,6 +227,10 @@ static int32_t mndProcessConnectReq(SRpcMsg *pReq) { } taosIp2String(pReq->info.conn.clientIp, ip); + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CONNECT) != 0) { + mGError("user:%s, failed to login from %s since %s", pReq->info.conn.user, ip, terrstr()); + goto _OVER; + } pUser = mndAcquireUser(pMnode, pReq->info.conn.user); if (pUser == NULL) { @@ -240,11 +244,6 @@ static int32_t mndProcessConnectReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_CONNECT) != 0) { - mGError("user:%s, failed to login from %s since %s", pReq->info.conn.user, ip, terrstr()); - goto _OVER; - } - if (connReq.db[0]) { char db[TSDB_DB_FNAME_LEN] = {0}; snprintf(db, TSDB_DB_FNAME_LEN, "%d%s%s", pUser->acctId, TS_PATH_DELIMITER, connReq.db); @@ -271,7 +270,7 @@ static int32_t mndProcessConnectReq(SRpcMsg *pReq) { connectRsp.connId = pConn->id; connectRsp.connType = connReq.connType; connectRsp.dnodeNum = mndGetDnodeSize(pMnode); - + strcpy(connectRsp.sVer, version); snprintf(connectRsp.sDetailVer, sizeof(connectRsp.sDetailVer), "ver:%s\nbuild:%s\ngitinfo:%s", version, buildinfo, gitinfo); @@ -475,16 +474,16 @@ static int32_t mndGetOnlineDnodeNum(SMnode *pMnode, int32_t *num) { SDnodeObj *pDnode = NULL; int64_t curMs = taosGetTimestampMs(); void *pIter = NULL; - + while (true) { pIter = sdbFetch(pSdb, SDB_DNODE, pIter, (void **)&pDnode); if (pIter == NULL) break; - + bool online = mndIsDnodeOnline(pDnode, curMs); if (online) { (*num)++; } - + sdbRelease(pSdb, pDnode); } @@ -652,15 +651,6 @@ static int32_t mndProcessKillQueryReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; SProfileMgmt *pMgmt = &pMnode->profileMgmt; - SUserObj *pUser = mndAcquireUser(pMnode, pReq->info.conn.user); - if (pUser == NULL) return 0; - if (!pUser->superUser) { - mndReleaseUser(pMnode, pUser); - terrno = TSDB_CODE_MND_NO_RIGHTS; - return -1; - } - mndReleaseUser(pMnode, pUser); - SKillQueryReq killReq = {0}; if (tDeserializeSKillQueryReq(pReq->pCont, pReq->contLen, &killReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; @@ -668,6 +658,10 @@ static int32_t mndProcessKillQueryReq(SRpcMsg *pReq) { } mInfo("kill query msg is received, queryId:%s", killReq.queryStrId); + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_KILL_QUERY) != 0) { + return -1; + } + int32_t connId = 0; uint64_t queryId = 0; char *p = strchr(killReq.queryStrId, ':'); @@ -697,21 +691,16 @@ static int32_t mndProcessKillConnReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; SProfileMgmt *pMgmt = &pMnode->profileMgmt; - SUserObj *pUser = mndAcquireUser(pMnode, pReq->info.conn.user); - if (pUser == NULL) return 0; - if (!pUser->superUser) { - mndReleaseUser(pMnode, pUser); - terrno = TSDB_CODE_MND_NO_RIGHTS; - return -1; - } - mndReleaseUser(pMnode, pUser); - SKillConnReq killReq = {0}; if (tDeserializeSKillConnReq(pReq->pCont, pReq->contLen, &killReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; return -1; } + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_KILL_CONN) != 0) { + return -1; + } + SConnObj *pConn = taosCacheAcquireByKey(pMgmt->connCache, &killReq.connId, sizeof(uint32_t)); if (pConn == NULL) { mError("connId:%u, failed to kill connection, conn not exist", killReq.connId); @@ -726,10 +715,10 @@ static int32_t mndProcessKillConnReq(SRpcMsg *pReq) { } static int32_t mndProcessSvrVerReq(SRpcMsg *pReq) { - int32_t code = -1; + int32_t code = -1; SServerVerRsp rsp = {0}; strcpy(rsp.ver, version); - + int32_t contLen = tSerializeSServerVerRsp(NULL, 0, &rsp); if (contLen < 0) goto _over; void *pRsp = rpcMallocCont(contLen); @@ -746,7 +735,6 @@ _over: return code; } - static int32_t mndRetrieveConns(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { SMnode *pMnode = pReq->info.node; SSdb *pSdb = pMnode->pSdb; diff --git a/source/dnode/mnode/impl/src/mndQnode.c b/source/dnode/mnode/impl/src/mndQnode.c index 0a6c97e63c..f057f6190d 100644 --- a/source/dnode/mnode/impl/src/mndQnode.c +++ b/source/dnode/mnode/impl/src/mndQnode.c @@ -15,7 +15,7 @@ #define _DEFAULT_SOURCE #include "mndQnode.h" -#include "mndAuth.h" +#include "mndPrivilege.h" #include "mndDnode.h" #include "mndShow.h" #include "mndTrans.h" @@ -279,6 +279,9 @@ static int32_t mndProcessCreateQnodeReq(SRpcMsg *pReq) { } mDebug("qnode:%d, start to create", createReq.dnodeId); + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_QNODE) != 0) { + goto _OVER; + } pObj = mndAcquireQnode(pMnode, createReq.dnodeId); if (pObj != NULL) { @@ -294,10 +297,6 @@ static int32_t mndProcessCreateQnodeReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_CREATE_QNODE) != 0) { - goto _OVER; - } - code = mndCreateQnode(pMnode, pReq, pDnode, &createReq); if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; @@ -391,6 +390,9 @@ static int32_t mndProcessDropQnodeReq(SRpcMsg *pReq) { } mDebug("qnode:%d, start to drop", dropReq.dnodeId); + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_QNODE) != 0) { + goto _OVER; + } if (dropReq.dnodeId <= 0) { terrno = TSDB_CODE_INVALID_MSG; @@ -402,10 +404,6 @@ static int32_t mndProcessDropQnodeReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_DROP_QNODE) != 0) { - goto _OVER; - } - code = mndDropQnode(pMnode, pReq, pObj); if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; @@ -418,19 +416,19 @@ _OVER: return code; } -int32_t mndCreateQnodeList(SMnode *pMnode, SArray** pList, int32_t limit) { - SSdb *pSdb = pMnode->pSdb; - void *pIter = NULL; - SQnodeObj *pObj = NULL; - int32_t numOfRows = 0; +int32_t mndCreateQnodeList(SMnode *pMnode, SArray **pList, int32_t limit) { + SSdb *pSdb = pMnode->pSdb; + void *pIter = NULL; + SQnodeObj *pObj = NULL; + int32_t numOfRows = 0; - SArray* qnodeList = taosArrayInit(5, sizeof(SQueryNodeLoad)); + SArray *qnodeList = taosArrayInit(5, sizeof(SQueryNodeLoad)); if (NULL == qnodeList) { mError("failed to alloc epSet while process qnode list req"); terrno = TSDB_CODE_OUT_OF_MEMORY; return terrno; } - + while (1) { pIter = sdbFetch(pSdb, SDB_QNODE, pIter, (void **)&pObj); if (pIter == NULL) break; @@ -457,7 +455,6 @@ int32_t mndCreateQnodeList(SMnode *pMnode, SArray** pList, int32_t limit) return TSDB_CODE_SUCCESS; } - static int32_t mndProcessQnodeListReq(SRpcMsg *pReq) { int32_t code = -1; SMnode *pMnode = pReq->info.node; diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index 27de3883e9..5c2531c25f 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -16,7 +16,7 @@ #define _DEFAULT_SOURCE #include "mndShow.h" #include "systable.h" -#include "mndAuth.h" +#include "mndPrivilege.h" #define SHOW_STEP_SIZE 100 @@ -231,7 +231,7 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) { mDebug("show:0x%" PRIx64 ", start retrieve data, type:%d", pShow->id, pShow->type); - // if (mndCheckShowAuth(pMnode, pReq->info.conn.user, pShow->type) != 0) return -1; + // if (mndCheckShowPrivilege(pMnode, pReq->info.conn.user, pShow->type) != 0) return -1; int32_t numOfCols = pShow->pMeta->numOfColumns; SSDataBlock *pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index 05603f8554..ef24cd0ba4 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -15,7 +15,7 @@ #define _DEFAULT_SOURCE #include "mndSma.h" -#include "mndAuth.h" +#include "mndPrivilege.h" #include "mndDb.h" #include "mndDnode.h" #include "mndInfoSchema.h" @@ -713,7 +713,7 @@ static int32_t mndProcessCreateSmaReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckDbAuth(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) { + if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) { goto _OVER; } @@ -974,7 +974,7 @@ static int32_t mndProcessDropSmaReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckDbAuth(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) { + if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) { goto _OVER; } diff --git a/source/dnode/mnode/impl/src/mndSnode.c b/source/dnode/mnode/impl/src/mndSnode.c index df1330197a..2dd8592bf8 100644 --- a/source/dnode/mnode/impl/src/mndSnode.c +++ b/source/dnode/mnode/impl/src/mndSnode.c @@ -15,7 +15,7 @@ #define _DEFAULT_SOURCE #include "mndSnode.h" -#include "mndAuth.h" +#include "mndPrivilege.h" #include "mndDnode.h" #include "mndShow.h" #include "mndTrans.h" @@ -285,6 +285,9 @@ static int32_t mndProcessCreateSnodeReq(SRpcMsg *pReq) { } mDebug("snode:%d, start to create", createReq.dnodeId); + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_SNODE) != 0) { + goto _OVER; + } pObj = mndAcquireSnode(pMnode, createReq.dnodeId); if (pObj != NULL) { @@ -300,10 +303,6 @@ static int32_t mndProcessCreateSnodeReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_CREATE_SNODE) != 0) { - goto _OVER; - } - code = mndCreateSnode(pMnode, pReq, pDnode, &createReq); if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; @@ -398,6 +397,9 @@ static int32_t mndProcessDropSnodeReq(SRpcMsg *pReq) { } mDebug("snode:%d, start to drop", dropReq.dnodeId); + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_SNODE) != 0) { + goto _OVER; + } if (dropReq.dnodeId <= 0) { terrno = TSDB_CODE_INVALID_MSG; @@ -409,10 +411,6 @@ static int32_t mndProcessDropSnodeReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_DROP_SNODE) != 0) { - goto _OVER; - } - // check deletable code = mndDropSnode(pMnode, pReq, pObj); if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index f1bae14c07..77b13cd82d 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -15,7 +15,7 @@ #define _DEFAULT_SOURCE #include "mndStb.h" -#include "mndAuth.h" +#include "mndPrivilege.h" #include "mndDb.h" #include "mndDnode.h" #include "mndInfoSchema.h" @@ -876,7 +876,7 @@ static int32_t mndProcessCreateStbReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckDbAuth(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) { + if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) { goto _OVER; } @@ -1607,7 +1607,7 @@ static int32_t mndProcessAlterStbReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckDbAuth(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) { + if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) { goto _OVER; } @@ -1737,7 +1737,7 @@ static int32_t mndProcessDropStbReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckDbAuth(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) { + if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) { goto _OVER; } diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 5e2f5bc2dd..e49756c837 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -14,7 +14,7 @@ */ #include "mndStream.h" -#include "mndAuth.h" +#include "mndPrivilege.h" #include "mndDb.h" #include "mndDnode.h" #include "mndMnode.h" @@ -437,7 +437,7 @@ static int32_t mndCreateStbForStream(SMnode *pMnode, STrans *pTrans, const SStre goto _OVER; } - if (mndCheckDbAuth(pMnode, user, MND_OPER_WRITE_DB, pDb) != 0) { + if (mndCheckDbPrivilege(pMnode, user, MND_OPER_WRITE_DB, pDb) != 0) { goto _OVER; } @@ -550,7 +550,7 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckDbAuth(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) { + if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) { goto _OVER; } #endif diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index a650ed29f1..b8c17378c4 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -14,7 +14,7 @@ */ #include "mndTopic.h" -#include "mndAuth.h" +#include "mndPrivilege.h" #include "mndConsumer.h" #include "mndDb.h" #include "mndDnode.h" @@ -480,7 +480,7 @@ static int32_t mndProcessCreateTopicReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckDbAuth(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) { + if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) { goto _OVER; } diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index d1d88fdc90..a9de1a05a7 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -15,7 +15,7 @@ #define _DEFAULT_SOURCE #include "mndTrans.h" -#include "mndAuth.h" +#include "mndPrivilege.h" #include "mndConsumer.h" #include "mndDb.h" #include "mndShow.h" @@ -1384,8 +1384,7 @@ static int32_t mndProcessKillTransReq(SRpcMsg *pReq) { } mInfo("trans:%d, start to kill", killReq.transId); - - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_KILL_TRANS) != 0) { + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_KILL_TRANS) != 0) { goto _OVER; } diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index 03c9647bfe..921dba422d 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -15,7 +15,7 @@ #define _DEFAULT_SOURCE #include "mndUser.h" -#include "mndAuth.h" +#include "mndPrivilege.h" #include "mndDb.h" #include "mndShow.h" #include "mndTrans.h" @@ -295,7 +295,7 @@ static int32_t mndCreateUser(SMnode *pMnode, char *acct, SCreateUserReq *pCreate tstrncpy(userObj.acct, acct, TSDB_USER_LEN); userObj.createdTime = taosGetTimestampMs(); userObj.updateTime = userObj.createdTime; - userObj.superUser = 0;//pCreate->superUser; + userObj.superUser = 0; // pCreate->superUser; userObj.sysInfo = pCreate->sysInfo; userObj.enable = pCreate->enable; @@ -337,6 +337,9 @@ static int32_t mndProcessCreateUserReq(SRpcMsg *pReq) { } mDebug("user:%s, start to create", createReq.user); + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_USER) != 0) { + goto _OVER; + } if (createReq.user[0] == 0) { terrno = TSDB_CODE_MND_INVALID_USER_FORMAT; @@ -360,10 +363,6 @@ static int32_t mndProcessCreateUserReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_CREATE_USER) != 0) { - goto _OVER; - } - code = mndCreateUser(pMnode, pOperUser->acct, &createReq, pReq); if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; @@ -466,7 +465,7 @@ static int32_t mndProcessAlterUserReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckAlterUserAuth(pOperUser, pUser, &alterReq) != 0) { + if (mndCheckAlterUserPrivilege(pOperUser, pUser, &alterReq) != 0) { goto _OVER; } @@ -631,6 +630,9 @@ static int32_t mndProcessDropUserReq(SRpcMsg *pReq) { } mDebug("user:%s, start to drop", dropReq.user); + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_USER) != 0) { + goto _OVER; + } if (dropReq.user[0] == 0) { terrno = TSDB_CODE_MND_INVALID_USER_FORMAT; @@ -643,10 +645,6 @@ static int32_t mndProcessDropUserReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_DROP_USER) != 0) { - goto _OVER; - } - code = mndDropUser(pMnode, pReq, pUser); if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index ae13987d25..0e931e0a9c 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -15,7 +15,7 @@ #define _DEFAULT_SOURCE #include "mndVgroup.h" -#include "mndAuth.h" +#include "mndPrivilege.h" #include "mndDb.h" #include "mndDnode.h" #include "mndMnode.h" @@ -1212,8 +1212,9 @@ static int32_t mndProcessRedistributeVgroupMsg(SRpcMsg *pReq) { } mInfo("vgId:%d, start to redistribute vgroup to dnode %d:%d:%d", req.vgId, req.dnodeId1, req.dnodeId2, req.dnodeId3); - - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_REDISTRIBUTE_VGROUP) != 0) goto _OVER; + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_REDISTRIBUTE_VGROUP) != 0) { + goto _OVER; + } pVgroup = mndAcquireVgroup(pMnode, req.vgId); if (pVgroup == NULL) goto _OVER; @@ -1506,6 +1507,9 @@ static int32_t mndProcessSplitVgroupMsg(SRpcMsg *pReq) { SDbObj *pDb = NULL; mDebug("vgId:%d, start to split", vgId); + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_SPLIT_VGROUP) != 0) { + goto _OVER; + } pVgroup = mndAcquireVgroup(pMnode, vgId); if (pVgroup == NULL) goto _OVER; @@ -1513,8 +1517,6 @@ static int32_t mndProcessSplitVgroupMsg(SRpcMsg *pReq) { pDb = mndAcquireDb(pMnode, pVgroup->dbName); if (pDb == NULL) goto _OVER; - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_SPLIT_VGROUP) != 0) goto _OVER; - code = mndSplitVgroup(pMnode, pReq, pDb, pVgroup); if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; @@ -1655,8 +1657,9 @@ static int32_t mndProcessBalanceVgroupMsg(SRpcMsg *pReq) { } mInfo("start to balance vgroup"); - - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_BALANCE_VGROUP) != 0) goto _OVER; + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_BALANCE_VGROUP) != 0) { + goto _OVER; + } while (1) { SDnodeObj *pDnode = NULL; diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index c7e8e8dc86..a32bf0ecdb 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -116,7 +116,8 @@ typedef void *tsdbReaderT; #define BLOCK_LOAD_TABLE_SEQ_ORDER 2 #define BLOCK_LOAD_TABLE_RR_ORDER 3 -tsdbReaderT tsdbReaderOpen(SVnode *pVnode, SQueryTableDataCond *pCond, STableListInfo *tableInfoGroup, uint64_t qId, +int32_t tsdbSetTableList(tsdbReaderT reader, SArray* tableList); +tsdbReaderT tsdbReaderOpen(SVnode *pVnode, SQueryTableDataCond *pCond, SArray *tableList, uint64_t qId, uint64_t taskId); tsdbReaderT tsdbQueryCacheLast(SVnode *pVnode, SQueryTableDataCond *pCond, STableListInfo *groupList, uint64_t qId, void *pMemRef); @@ -195,7 +196,6 @@ struct SVnodeCfg { typedef struct { TSKEY lastKey; uint64_t uid; - uint64_t groupId; } STableKeyInfo; struct SMetaEntry { diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index f7965f0902..3b30224e17 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -121,7 +121,7 @@ int tsdbInsertData(STsdb* pTsdb, int64_t version, SSubmitReq* pMsg, SSub int32_t tsdbInsertTableData(STsdb* pTsdb, int64_t version, SSubmitMsgIter* pMsgIter, SSubmitBlk* pBlock, SSubmitBlkRsp* pRsp); int32_t tsdbDeleteTableData(STsdb* pTsdb, int64_t version, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKEY eKey); -tsdbReaderT tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, STableListInfo* tableList, uint64_t qId, +tsdbReaderT tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, SArray* tableList, uint64_t qId, uint64_t taskId); tsdbReaderT tsdbQueryCacheLastT(STsdb* tsdb, SQueryTableDataCond* pCond, STableListInfo* tableList, uint64_t qId, void* pMemRef); diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index ab512f7774..4a3feca8d0 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -381,6 +381,7 @@ int metaTtlDropTable(SMeta *pMeta, int64_t ttl, SArray *tbUids) { for (int i = 0; i < taosArrayGetSize(tbUids); ++i) { tb_uid_t *uid = (tb_uid_t *)taosArrayGet(tbUids, i); metaDropTableByUid(pMeta, *uid, NULL); + metaDebug("ttl drop table:%"PRId64, *uid); } metaULock(pMeta); return 0; @@ -443,7 +444,6 @@ static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type) { // drop schema.db (todo) } - metaError("ttl drop table:%s", e.name); tDecoderClear(&dc); tdbFree(pData); @@ -976,7 +976,9 @@ static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) { SDecoder dc = {0}; // get super table - tdbTbGet(pMeta->pUidIdx, &pCtbEntry->ctbEntry.suid, sizeof(tb_uid_t), &pData, &nData); + if(tdbTbGet(pMeta->pUidIdx, &pCtbEntry->ctbEntry.suid, sizeof(tb_uid_t), &pData, &nData) != 0){ + return -1; + } tbDbKey.uid = pCtbEntry->ctbEntry.suid; tbDbKey.version = *(int64_t *)pData; tdbTbGet(pMeta->pTbDb, &tbDbKey, sizeof(tbDbKey), &pData, &nData); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index ef6ab00cda..766cf7af35 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -403,7 +403,7 @@ int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen) { .reader = pHandle->execHandle.pExecReader[i], .meta = pTq->pVnode->pMeta, .vnode = pTq->pVnode, - .initTsdbReader = 1, +// .initTsdbReader = 1, }; pHandle->execHandle.execCol.task[i] = qCreateStreamExecTaskInfo(pHandle->execHandle.execCol.qmsg, &handle); ASSERT(pHandle->execHandle.execCol.task[i]); @@ -479,7 +479,7 @@ int32_t tqProcessTaskDeployReq(STQ* pTq, char* msg, int32_t msgLen) { .reader = pStreamReader, .meta = pTq->pVnode->pMeta, .vnode = pTq->pVnode, - .initTsdbReader = 1, +// .initTsdbReader = 1, }; /*pTask->exec.inputHandle = pStreamReader;*/ pTask->exec.executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &handle); diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 0243f2a9f0..6184d8f810 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -243,7 +243,7 @@ int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, STqReadHandle* pHandle, uint64_ if (!tdSTSRowIterNext(&iter, pColData->info.colId, pColData->info.type, &sVal)) { break; } - if (colDataAppend(pColData, curRow, sVal.val, sVal.valType == TD_VTYPE_NULL) < 0) { + if (colDataAppend(pColData, curRow, sVal.val, sVal.valType != TD_VTYPE_NORM) < 0) { goto FAIL; } } diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index e16360a58b..540810f876 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -223,9 +223,8 @@ int64_t tsdbGetNumOfRowsInMemTable(tsdbReaderT* pHandle) { return rows; } -static SArray* createCheckInfoFromTableGroup(STsdbReadHandle* pTsdbReadHandle, STableListInfo* pTableList) { - size_t tableSize = taosArrayGetSize(pTableList->pTableList); - assert(tableSize >= 1); +static SArray* createCheckInfoFromTableGroup(STsdbReadHandle* pTsdbReadHandle, SArray* pTableList) { + size_t tableSize = taosArrayGetSize(pTableList); // allocate buffer in order to load data blocks from file SArray* pTableCheckInfo = taosArrayInit(tableSize, sizeof(STableCheckInfo)); @@ -235,7 +234,7 @@ static SArray* createCheckInfoFromTableGroup(STsdbReadHandle* pTsdbReadHandle, S // todo apply the lastkey of table check to avoid to load header file for (int32_t j = 0; j < tableSize; ++j) { - STableKeyInfo* pKeyInfo = (STableKeyInfo*)taosArrayGet(pTableList->pTableList, j); + STableKeyInfo* pKeyInfo = (STableKeyInfo*)taosArrayGet(pTableList, j); STableCheckInfo info = {.lastKey = pKeyInfo->lastKey, .tableId = pKeyInfo->uid}; info.suid = pTsdbReadHandle->suid; @@ -254,8 +253,6 @@ static SArray* createCheckInfoFromTableGroup(STsdbReadHandle* pTsdbReadHandle, S pTsdbReadHandle->idStr); } - // TODO group table according to the tag value. - taosArraySort(pTableCheckInfo, tsdbCheckInfoCompar); return pTableCheckInfo; } @@ -497,8 +494,21 @@ static int32_t setCurrentSchema(SVnode* pVnode, STsdbReadHandle* pTsdbReadHandle return TSDB_CODE_SUCCESS; } -tsdbReaderT tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, STableListInfo* tableList, uint64_t qId, +int32_t tsdbSetTableList(tsdbReaderT reader, SArray* tableList){ + STsdbReadHandle* pTsdbReadHandle = reader; + if(pTsdbReadHandle->pTableCheckInfo) taosArrayDestroy(pTsdbReadHandle->pTableCheckInfo); + pTsdbReadHandle->pTableCheckInfo = createCheckInfoFromTableGroup(pTsdbReadHandle, tableList); + if (pTsdbReadHandle->pTableCheckInfo == NULL) { + return TSDB_CODE_TDB_OUT_OF_MEMORY; + } + return TDB_CODE_SUCCESS; +} + +tsdbReaderT tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, SArray* tableList, uint64_t qId, uint64_t taskId) { + if(taosArrayGetSize(tableList) == 0){ + return NULL; + } STsdbReadHandle* pTsdbReadHandle = tsdbQueryTablesImpl(pVnode, pCond, qId, taskId); if (pTsdbReadHandle == NULL) { return NULL; @@ -543,7 +553,7 @@ tsdbReaderT tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, STableLis } tsdbDebug("%p total numOfTable:%" PRIzu " in this query, table %" PRIzu " %s", pTsdbReadHandle, - taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo), taosArrayGetSize(tableList->pTableList), + taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo), taosArrayGetSize(tableList), pTsdbReadHandle->idStr); return (tsdbReaderT)pTsdbReadHandle; @@ -639,7 +649,7 @@ tsdbReaderT tsdbQueryLastRow(SVnode* pVnode, SQueryTableDataCond* pCond, STableL return NULL; } - STsdbReadHandle* pTsdbReadHandle = (STsdbReadHandle*)tsdbReaderOpen(pVnode, pCond, pList, qId, taskId); + STsdbReadHandle* pTsdbReadHandle = (STsdbReadHandle*)tsdbReaderOpen(pVnode, pCond, pList->pTableList, qId, taskId); if (pTsdbReadHandle == NULL) { return NULL; } @@ -2842,7 +2852,7 @@ int32_t tsdbGetAllTableList(SMeta* pMeta, uint64_t uid, SArray* list) { break; } - STableKeyInfo info = {.lastKey = TSKEY_INITIAL_VAL, uid = id, .groupId = 0}; + STableKeyInfo info = {.lastKey = TSKEY_INITIAL_VAL, uid = id}; taosArrayPush(list, &info); } @@ -3644,17 +3654,6 @@ SArray* tsdbRetrieveDataBlock(tsdbReaderT* pTsdbReadHandle, SArray* pIdList) { } } -static int tsdbCheckInfoCompar(const void* key1, const void* key2) { - if (((STableCheckInfo*)key1)->tableId < ((STableCheckInfo*)key2)->tableId) { - return -1; - } else if (((STableCheckInfo*)key1)->tableId > ((STableCheckInfo*)key2)->tableId) { - return 1; - } else { - ASSERT(false); - return 0; - } -} - static void* doFreeColumnInfoData(SArray* pColumnInfoData) { if (pColumnInfoData == NULL) { return NULL; diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 57a2c57b16..eb7beb19db 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -273,6 +273,10 @@ typedef struct STableScanInfo { SSampleExecInfo sample; // sample execution info int32_t curTWinIdx; + + int32_t currentGroupId; + uint64_t queryId; + uint64_t taskId; } STableScanInfo; typedef struct STagScanInfo { @@ -706,7 +710,7 @@ SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pR SOperatorInfo* createExchangeOperatorInfo(void* pTransporter, SExchangePhysiNode* pExNode, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, tsdbReaderT pDataReader, SReadHandle* pHandle, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle, SExecTaskInfo* pTaskInfo, uint64_t queryId, uint64_t taskId); SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysiNode* pPhyNode, STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo); SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScanPhysiNode *pScanPhyNode, SExecTaskInfo* pTaskInfo); @@ -749,8 +753,8 @@ SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SExprInfo* pEx SOperatorInfo* createDataBlockInfoScanOperator(void* dataReader, SReadHandle* readHandle, uint64_t uid, SBlockDistScanPhysiNode* pBlockScanNode, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createStreamScanOperatorInfo(void* pDataReader, SReadHandle* pHandle, - STableScanPhysiNode* pTableScanNode, SExecTaskInfo* pTaskInfo, STimeWindowAggSupp* pTwSup); +SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, + STableScanPhysiNode* pTableScanNode, SExecTaskInfo* pTaskInfo, STimeWindowAggSupp* pTwSup, uint64_t queryId, uint64_t taskId); SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode* pPhyFillNode, bool multigroupResult, SExecTaskInfo* pTaskInfo); @@ -845,7 +849,7 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN void copyUpdateDataBlock(SSDataBlock* pDest, SSDataBlock* pSource, int32_t tsColIndex); -int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle, SArray* groupKey); +int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle, SNodeList* groupKey); #ifdef __cplusplus } diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 8669c2c28c..5ac5957f2b 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -287,6 +287,7 @@ static bool isTableOk(STableKeyInfo* info, SNode *pTagCond, SMeta *metaHandle){ int32_t getTableList(void* metaHandle, SScanPhysiNode* pScanNode, STableListInfo* pListInfo) { int32_t code = TSDB_CODE_SUCCESS; pListInfo->pTableList = taosArrayInit(8, sizeof(STableKeyInfo)); + if(pListInfo->pTableList == NULL) return TSDB_CODE_OUT_OF_MEMORY; uint64_t tableUid = pScanNode->uid; @@ -314,7 +315,7 @@ int32_t getTableList(void* metaHandle, SScanPhysiNode* pScanNode, STableListInfo } for (int i = 0; i < taosArrayGetSize(res); i++) { - STableKeyInfo info = {.lastKey = TSKEY_INITIAL_VAL, .uid = *(uint64_t*)taosArrayGet(res, i), .groupId = 0}; + STableKeyInfo info = {.lastKey = TSKEY_INITIAL_VAL, .uid = *(uint64_t*)taosArrayGet(res, i)}; taosArrayPush(pListInfo->pTableList, &info); } taosArrayDestroy(res); @@ -335,9 +336,14 @@ int32_t getTableList(void* metaHandle, SScanPhysiNode* pScanNode, STableListInfo } } }else { // Create one table group. - STableKeyInfo info = {.lastKey = 0, .uid = tableUid, .groupId = 0}; + STableKeyInfo info = {.lastKey = 0, .uid = tableUid}; taosArrayPush(pListInfo->pTableList, &info); } + pListInfo->pGroupList = taosArrayInit(4, POINTER_BYTES); + if(pListInfo->pGroupList == NULL) return TSDB_CODE_OUT_OF_MEMORY; + + //put into list as default group, remove it if grouping sorting is required later + taosArrayPush(pListInfo->pGroupList, &pListInfo->pTableList); return code; } diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 57ca814974..c23d9a5040 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -3862,9 +3862,6 @@ static SExecTaskInfo* createExecTaskInfo(uint64_t queryId, uint64_t taskId, EOPT return pTaskInfo; } -static tsdbReaderT doCreateDataReader(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle, - STableListInfo* pTableListInfo, uint64_t queryId, uint64_t taskId); - static SArray* extractColumnInfo(SNodeList* pNodeList); int32_t extractTableSchemaVersion(SReadHandle* pHandle, uint64_t uid, SExecTaskInfo* pTaskInfo) { @@ -3895,8 +3892,67 @@ int32_t extractTableSchemaVersion(SReadHandle* pHandle, uint64_t uid, SExecTaskI return TSDB_CODE_SUCCESS; } -int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle, SArray* groupKey) { - if (groupKey == NULL) { +static int32_t sortTableGroup(STableListInfo* pTableListInfo, int32_t groupNum){ + taosArrayClear(pTableListInfo->pGroupList); + SArray *sortSupport = taosArrayInit(groupNum, sizeof(uint64_t)); + if(sortSupport == NULL) return TSDB_CODE_OUT_OF_MEMORY; + for (int32_t i = 0; i < taosArrayGetSize(pTableListInfo->pTableList); i++) { + STableKeyInfo* info = taosArrayGet(pTableListInfo->pTableList, i); + uint64_t* groupId = taosHashGet(pTableListInfo->map, &info->uid, sizeof(uint64_t)); + + int32_t index = taosArraySearchIdx(sortSupport, groupId, compareUint64Val, TD_EQ); + if (index == -1){ + void *p = taosArraySearch(sortSupport, groupId, compareUint64Val, TD_GT); + SArray *tGroup = taosArrayInit(8, sizeof(STableKeyInfo)); + if(tGroup == NULL) { + taosArrayDestroy(sortSupport); + return TSDB_CODE_OUT_OF_MEMORY; + } + if(taosArrayPush(tGroup, info) == NULL){ + qError("taos push info array error"); + taosArrayDestroy(sortSupport); + return TSDB_CODE_QRY_APP_ERROR; + } + if(p == NULL){ + if(taosArrayPush(sortSupport, groupId) != NULL){ + qError("taos push support array error"); + taosArrayDestroy(sortSupport); + return TSDB_CODE_QRY_APP_ERROR; + } + if(taosArrayPush(pTableListInfo->pGroupList, &tGroup) != NULL){ + qError("taos push group array error"); + taosArrayDestroy(sortSupport); + return TSDB_CODE_QRY_APP_ERROR; + } + }else{ + int32_t pos = TARRAY_ELEM_IDX(sortSupport, p); + if(taosArrayInsert(sortSupport, pos, groupId) == NULL){ + qError("taos insert support array error"); + taosArrayDestroy(sortSupport); + return TSDB_CODE_QRY_APP_ERROR; + } + if(taosArrayInsert(pTableListInfo->pGroupList, pos, &tGroup) == NULL){ + qError("taos insert group array error"); + taosArrayDestroy(sortSupport); + return TSDB_CODE_QRY_APP_ERROR; + } + } + }else{ + SArray* tGroup = (SArray*)taosArrayGetP(pTableListInfo->pGroupList, index); + if(taosArrayPush(tGroup, info) == NULL){ + qError("taos push uid array error"); + taosArrayDestroy(sortSupport); + return TSDB_CODE_QRY_APP_ERROR; + } + } + + } + taosArrayDestroy(sortSupport); + return TDB_CODE_SUCCESS; +} + +int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle, SNodeList* group) { + if (group == NULL) { return TDB_CODE_SUCCESS; } @@ -3906,13 +3962,14 @@ int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle, } int32_t keyLen = 0; void* keyBuf = NULL; - int32_t numOfGroupCols = taosArrayGetSize(groupKey); - for (int32_t j = 0; j < numOfGroupCols; ++j) { - SColumn* pCol = taosArrayGet(groupKey, j); - keyLen += pCol->bytes; // actual data + null_flag + + SNode* node; + FOREACH(node, group) { + SExprNode *pExpr = (SExprNode *)node; + keyLen += pExpr->resType.bytes; } - int32_t nullFlagSize = sizeof(int8_t) * numOfGroupCols; + int32_t nullFlagSize = sizeof(int8_t) * LIST_LENGTH(group); keyLen += nullFlagSize; keyBuf = taosMemoryCalloc(1, keyLen); @@ -3920,59 +3977,68 @@ int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle, return TSDB_CODE_OUT_OF_MEMORY; } + int32_t groupNum = 0; for (int32_t i = 0; i < taosArrayGetSize(pTableListInfo->pTableList); i++) { STableKeyInfo* info = taosArrayGet(pTableListInfo->pTableList, i); SMetaReader mr = {0}; metaReaderInit(&mr, pHandle->meta, 0); metaGetTableEntryByUid(&mr, info->uid); - char* isNull = (char*)keyBuf; - char* pStart = (char*)keyBuf + sizeof(int8_t) * numOfGroupCols; - for (int32_t j = 0; j < numOfGroupCols; ++j) { - SColumn* pCol = taosArrayGet(groupKey, j); + SNodeList *groupNew = nodesCloneList(group); - if (strcmp(pCol->name, "tbname") == 0) { - isNull[i] = 0; - memcpy(pStart, mr.me.name, strlen(mr.me.name)); - pStart += strlen(mr.me.name); + nodesRewriteExprsPostOrder(groupNew, doTranslateTagExpr, &mr); + char* isNull = (char*)keyBuf; + char* pStart = (char*)keyBuf + nullFlagSize; + + SNode* pNode; + int32_t index = 0; + FOREACH(pNode, groupNew){ + SNode* pNew = NULL; + int32_t code = scalarCalculateConstants(pNode, &pNew); + if (TSDB_CODE_SUCCESS == code) { + REPLACE_NODE(pNew); } else { - STagVal tagVal = {0}; - tagVal.cid = pCol->colId; - const char* p = metaGetTableTagVal(&mr.me, pCol->type, &tagVal); - if (p == NULL) { - isNull[j] = 1; - continue; - } - isNull[i] = 0; - if (pCol->type == TSDB_DATA_TYPE_JSON) { - // int32_t dataLen = getJsonValueLen(pkey->pData); - // memcpy(pStart, (pkey->pData), dataLen); - // pStart += dataLen; - } else if (IS_VAR_DATA_TYPE(pCol->type)) { - memcpy(pStart, tagVal.pData, tagVal.nData); - pStart += tagVal.nData; - ASSERT(tagVal.nData <= pCol->bytes); + taosMemoryFree(keyBuf); + nodesClearList(groupNew); + return code; + } + + ASSERT(nodeType(pNew) == QUERY_NODE_VALUE); + SValueNode *pValue = (SValueNode *)pNew; + + if (pValue->node.resType.type == TSDB_DATA_TYPE_NULL) { + isNull[index++] = 1; + continue; + } else { + isNull[index++] = 0; + char* data = nodesGetValueFromNode(pValue); + if (pValue->node.resType.type == TSDB_DATA_TYPE_JSON){ + int32_t len = getJsonValueLen(data); + memcpy(pStart, data, len); + pStart += len; + } else if (IS_VAR_DATA_TYPE(pValue->node.resType.type)) { + memcpy(pStart, data, varDataTLen(data)); + pStart += varDataTLen(data); } else { - memcpy(pStart, &(tagVal.i64), pCol->bytes); - pStart += pCol->bytes; + memcpy(pStart, data, pValue->node.resType.bytes); + pStart += pValue->node.resType.bytes; } } } - int32_t len = (int32_t)(pStart - (char*)keyBuf); - - uint64_t* pGroupId = taosHashGet(pTableListInfo->map, keyBuf, len); - - if (!pGroupId) { - uint64_t tmpId = calcGroupId(keyBuf, len); - info->groupId = tmpId; - taosHashPut(pTableListInfo->map, &(info->uid), sizeof(uint64_t), &tmpId, sizeof(uint64_t)); - } else { - info->groupId = *pGroupId; - } + int32_t len = (int32_t)(pStart - (char*)keyBuf); + uint64_t groupId = calcGroupId(keyBuf, len); + taosHashPut(pTableListInfo->map, &(info->uid), sizeof(uint64_t), &groupId, sizeof(uint64_t)); + groupNum++; + nodesClearList(groupNew); metaReaderClear(&mr); } taosMemoryFree(keyBuf); + + if(pTableListInfo->needSortTableByGroupId){ + return sortTableGroup(pTableListInfo, groupNum); + } + return TDB_CODE_SUCCESS; } @@ -3984,39 +4050,36 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo if (QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN == type) { STableScanPhysiNode* pTableScanNode = (STableScanPhysiNode*)pPhyNode; - tsdbReaderT pDataReader = doCreateDataReader(pTableScanNode, pHandle, pTableListInfo, (uint64_t)queryId, taskId); - if (pDataReader == NULL && terrno != 0) { - pTaskInfo->code = terrno; + int32_t code = createScanTableListInfo(pTableScanNode, pHandle, pTableListInfo, queryId, taskId); + if(code){ + pTaskInfo->code = code; return NULL; } - - int32_t code = extractTableSchemaVersion(pHandle, pTableScanNode->scan.uid, pTaskInfo); + code = extractTableSchemaVersion(pHandle, pTableScanNode->scan.uid, pTaskInfo); if (code) { - tsdbCleanupReadHandle(pDataReader); pTaskInfo->code = terrno; return NULL; } - SArray* groupKeys = extractPartitionColInfo(pTableScanNode->pPartitionTags); - code = generateGroupIdMap(pTableListInfo, pHandle, groupKeys); // todo for json - taosArrayDestroy(groupKeys); - if (code) { - tsdbCleanupReadHandle(pDataReader); - pTaskInfo->code = terrno; - return NULL; - } - - SOperatorInfo* pOperator = createTableScanOperatorInfo(pTableScanNode, pDataReader, pHandle, pTaskInfo); + SOperatorInfo* pOperator = createTableScanOperatorInfo(pTableScanNode, pHandle, pTaskInfo, queryId, taskId); STableScanInfo* pScanInfo = pOperator->info; pTaskInfo->cost.pRecoder = &pScanInfo->readRecorder; return pOperator; } else if (QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN == type) { STableMergeScanPhysiNode* pTableScanNode = (STableMergeScanPhysiNode*)pPhyNode; - createScanTableListInfo(pTableScanNode, pHandle, pTableListInfo, queryId, taskId); - extractTableSchemaVersion(pHandle, pTableScanNode->scan.uid, pTaskInfo); - SOperatorInfo* pOperator = - createTableMergeScanOperatorInfo(pTableScanNode, pTableListInfo, pHandle, pTaskInfo, queryId, taskId); + int32_t code = createScanTableListInfo(pTableScanNode, pHandle, pTableListInfo, queryId, taskId); + if(code){ + return NULL; + } + code = extractTableSchemaVersion(pHandle, pTableScanNode->scan.uid, pTaskInfo); + if (code) { + pTaskInfo->code = terrno; + return NULL; + } + + SOperatorInfo* pOperator = createTableMergeScanOperatorInfo(pTableScanNode, pTableListInfo, pHandle, pTaskInfo, queryId, taskId); + STableScanInfo* pScanInfo = pOperator->info; pTaskInfo->cost.pRecoder = &pScanInfo->readRecorder; return pOperator; @@ -4025,46 +4088,17 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo return createExchangeOperatorInfo(pHandle->pMsgCb->clientRpc, (SExchangePhysiNode*)pPhyNode, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN == type) { - SScanPhysiNode* pScanPhyNode = (SScanPhysiNode*)pPhyNode; // simple child table. STableScanPhysiNode* pTableScanNode = (STableScanPhysiNode*)pPhyNode; STimeWindowAggSupp twSup = { .waterMark = pTableScanNode->watermark, .calTrigger = pTableScanNode->triggerType, .maxTs = INT64_MIN, }; - tsdbReaderT pDataReader = NULL; - if (pHandle) { - if (pHandle->initTsdbReader) { - // for stream - ASSERT(pHandle->vnode); - pDataReader = doCreateDataReader(pTableScanNode, pHandle, pTableListInfo, (uint64_t)queryId, taskId); - } else { - // for tq - ASSERT(pHandle->meta); - getTableList(pHandle->meta, pScanPhyNode, pTableListInfo); - } + createScanTableListInfo(pTableScanNode, pHandle, pTableListInfo, queryId, taskId); } -#if 0 - if (pDataReader == NULL && terrno != 0) { - qDebug("%s pDataReader is NULL", GET_TASKID(pTaskInfo)); - // return NULL; - } else { - qDebug("%s pDataReader is not NULL", GET_TASKID(pTaskInfo)); - } -#endif - - SArray* groupKeys = extractPartitionColInfo(pTableScanNode->pPartitionTags); - int32_t code = generateGroupIdMap(pTableListInfo, pHandle, groupKeys); // todo for json - taosArrayDestroy(groupKeys); - if (code) { - tsdbCleanupReadHandle(pDataReader); - return NULL; - } - - SOperatorInfo* pOperator = createStreamScanOperatorInfo(pDataReader, pHandle, pTableScanNode, pTaskInfo, &twSup); - + SOperatorInfo* pOperator = createStreamScanOperatorInfo(pHandle, pTableScanNode, pTaskInfo, &twSup, queryId, taskId); return pOperator; } else if (QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN == type) { @@ -4093,7 +4127,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo return NULL; } } else { // Create one table group. - STableKeyInfo info = {.lastKey = 0, .uid = pBlockNode->uid, .groupId = 0}; + STableKeyInfo info = {.lastKey = 0, .uid = pBlockNode->uid}; taosArrayPush(pTableListInfo->pTableList, &info); } @@ -4118,7 +4152,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo cond.suid = pBlockNode->suid; cond.type = BLOCK_LOAD_OFFSET_SEQ_ORDER; } - tsdbReaderT* pReader = tsdbReaderOpen(pHandle->vnode, &cond, pTableListInfo, queryId, taskId); + tsdbReaderT* pReader = tsdbReaderOpen(pHandle->vnode, &cond, pTableListInfo->pTableList, queryId, taskId); cleanupQueryTableDataCond(&cond); return createDataBlockInfoScanOperator(pReader, pHandle, cond.suid, pBlockNode, pTaskInfo); @@ -4355,7 +4389,7 @@ tsdbReaderT doCreateDataReader(STableScanPhysiNode* pTableScanNode, SReadHandle* goto _error; } - tsdbReaderT pReader = tsdbReaderOpen(pHandle->vnode, &cond, pTableListInfo, queryId, taskId); + tsdbReaderT pReader = tsdbReaderOpen(pHandle->vnode, &cond, pTableListInfo->pTableList, queryId, taskId); cleanupQueryTableDataCond(&cond); return pReader; @@ -4584,6 +4618,13 @@ _complete: static void doDestroyTableList(STableListInfo* pTableqinfoList) { taosArrayDestroy(pTableqinfoList->pTableList); taosHashCleanup(pTableqinfoList->map); + if(pTableqinfoList->needSortTableByGroupId){ + for(int32_t i = 0; i < taosArrayGetSize(pTableqinfoList->pGroupList); i++){ + SArray* tmp = taosArrayGetP(pTableqinfoList->pGroupList, i); + taosArrayDestroy(tmp); + } + } + taosArrayDestroy(pTableqinfoList->pGroupList); pTableqinfoList->pTableList = NULL; pTableqinfoList->map = NULL; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 07212bc018..9c0ed40c30 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -418,7 +418,7 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) { return NULL; } -static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { +static SSDataBlock* doTableScanGroup(SOperatorInfo* pOperator) { STableScanInfo* pTableScanInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; @@ -500,6 +500,48 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { } } + return NULL; +} + +static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { + STableScanInfo* pInfo = pOperator->info; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + + if(pInfo->currentGroupId == -1){ + pInfo->currentGroupId++; + if (pInfo->currentGroupId >= taosArrayGetSize(pTaskInfo->tableqinfoList.pGroupList)) { + setTaskStatus(pTaskInfo, TASK_COMPLETED); + return NULL; + } + SArray *tableList = taosArrayGetP(pTaskInfo->tableqinfoList.pGroupList, pInfo->currentGroupId); + tsdbCleanupReadHandle(pInfo->dataReader); + tsdbReaderT* pReader = tsdbReaderOpen(pInfo->readHandle.vnode, &pInfo->cond, tableList, pInfo->queryId, pInfo->taskId); + pInfo->dataReader = pReader; + } + + SSDataBlock* result = doTableScanGroup(pOperator); + if(result){ + return result; + } + + pInfo->currentGroupId++; + if (pInfo->currentGroupId >= taosArrayGetSize(pTaskInfo->tableqinfoList.pGroupList)) { + setTaskStatus(pTaskInfo, TASK_COMPLETED); + return NULL; + } + + SArray *tableList = taosArrayGetP(pTaskInfo->tableqinfoList.pGroupList, pInfo->currentGroupId); + tsdbSetTableList(pInfo->dataReader, tableList); + + tsdbResetReadHandle(pInfo->dataReader, &pInfo->cond, 0); + pInfo->curTWinIdx = 0; + pInfo->scanTimes = 0; + + result = doTableScanGroup(pOperator); + if(result){ + return result; + } + setTaskStatus(pTaskInfo, TASK_COMPLETED); return NULL; } @@ -525,8 +567,8 @@ static void destroyTableScanOperatorInfo(void* param, int32_t numOfOutput) { } } -SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, tsdbReaderT pDataReader, - SReadHandle* readHandle, SExecTaskInfo* pTaskInfo) { +SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* readHandle, + SExecTaskInfo* pTaskInfo, uint64_t queryId, uint64_t taskId) { STableScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STableScanInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { @@ -561,10 +603,12 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, pInfo->dataBlockLoadFlag = pTableScanNode->dataRequired; pInfo->pResBlock = createResDataBlock(pDescNode); pInfo->pFilterNode = pTableScanNode->scan.node.pConditions; - pInfo->dataReader = pDataReader; pInfo->scanFlag = MAIN_SCAN; pInfo->pColMatchInfo = pColList; pInfo->curTWinIdx = 0; + pInfo->queryId = queryId; + pInfo->taskId = taskId; + pInfo->currentGroupId = -1; pOperator->name = "TableScanOperator"; // for debug purpose pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN; @@ -778,8 +822,9 @@ static bool prepareDataScan(SStreamBlockScanInfo* pInfo) { STableScanInfo* pTableScanInfo = pInfo->pSnapshotReadOp->info; pTableScanInfo->cond.twindows[0] = win; pTableScanInfo->curTWinIdx = 0; - tsdbResetReadHandle(pTableScanInfo->dataReader, &pTableScanInfo->cond, 0); +// tsdbResetReadHandle(pTableScanInfo->dataReader, &pTableScanInfo->cond, 0); pTableScanInfo->scanTimes = 0; + pTableScanInfo->currentGroupId = -1; return true; } @@ -1087,9 +1132,9 @@ static SArray* extractTableIdList(const STableListInfo* pTableGroupInfo) { return tableIdList; } -SOperatorInfo* createStreamScanOperatorInfo(void* pDataReader, SReadHandle* pHandle, +SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhysiNode* pTableScanNode, SExecTaskInfo* pTaskInfo, - STimeWindowAggSupp* pTwSup) { + STimeWindowAggSupp* pTwSup, uint64_t queryId, uint64_t taskId) { SStreamBlockScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamBlockScanInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); @@ -1129,7 +1174,7 @@ SOperatorInfo* createStreamScanOperatorInfo(void* pDataReader, SReadHandle* pHan } if (pHandle) { - SOperatorInfo* pTableScanDummy = createTableScanOperatorInfo(pTableScanNode, pDataReader, pHandle, pTaskInfo); + SOperatorInfo* pTableScanDummy = createTableScanOperatorInfo(pTableScanNode, pHandle, pTaskInfo, queryId, taskId); STableScanInfo* pSTInfo = (STableScanInfo*)pTableScanDummy->info; if (pSTInfo->interval.interval > 0) { pInfo->pUpdateInfo = updateInfoInitP(&pSTInfo->interval, pTwSup->waterMark); @@ -1889,11 +1934,12 @@ SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysi goto _error; } - pInfo->pTableList = pTableListInfo; - pInfo->pColMatchInfo = colList; - pInfo->pRes = createResDataBlock(pDescNode); - pInfo->readHandle = *pReadHandle; - pInfo->curPos = 0; + pInfo->pTableList = pTableListInfo; + pInfo->pColMatchInfo = colList; + pInfo->pRes = createResDataBlock(pDescNode); + pInfo->readHandle = *pReadHandle; + pInfo->curPos = 0; + pOperator->name = "TagScanOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN; @@ -1919,10 +1965,7 @@ _error: typedef struct STableMergeScanInfo { STableListInfo* tableListInfo; - int32_t tableStartIndex; - int32_t tableEndIndex; - bool hasGroupId; - uint64_t groupId; + int32_t currentGroupId; SArray* dataReaders; // array of tsdbReaderT* SReadHandle readHandle; @@ -1968,12 +2011,6 @@ typedef struct STableMergeScanInfo { SSampleExecInfo sample; // sample execution info } STableMergeScanInfo; -int32_t compareTableKeyInfoByGid(const void* p1, const void* p2) { - const STableKeyInfo* info1 = p1; - const STableKeyInfo* info2 = p2; - return info1->groupId - info2->groupId; -} - int32_t createScanTableListInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle, STableListInfo* pTableListInfo, uint64_t queryId, uint64_t taskId) { int32_t code = getTableList(pHandle->meta, &pTableScanNode->scan, pTableListInfo); @@ -1985,55 +2022,9 @@ int32_t createScanTableListInfo(STableScanPhysiNode* pTableScanNode, SReadHandle qDebug("no table qualified for query, TID:0x%" PRIx64 ", QID:0x%" PRIx64, taskId, queryId); return TSDB_CODE_SUCCESS; } - SArray* groupKeys = extractPartitionColInfo(pTableScanNode->pPartitionTags); - generateGroupIdMap(pTableListInfo, pHandle, groupKeys); // todo for json - if (groupKeys) { - taosArraySort(pTableListInfo->pTableList, compareTableKeyInfoByGid); - } - taosArrayDestroy(groupKeys); - return TSDB_CODE_SUCCESS; -} - -int32_t doCreateMultipleDataReaders(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle, - STableListInfo* pTableListInfo, SArray* arrayReader, uint64_t queryId, - uint64_t taskId) { - SQueryTableDataCond cond = {0}; - int32_t code = initQueryTableDataCond(&cond, pTableScanNode); + code = generateGroupIdMap(pTableListInfo, pHandle, pTableScanNode->pPartitionTags); if (code != TSDB_CODE_SUCCESS) { - goto _error; - } - for (int32_t i = 0; i < taosArrayGetSize(pTableListInfo->pTableList); ++i) { - STableListInfo* subListInfo = taosMemoryCalloc(1, sizeof(subListInfo)); - subListInfo->pTableList = taosArrayInit(1, sizeof(STableKeyInfo)); - taosArrayPush(subListInfo->pTableList, taosArrayGet(pTableListInfo->pTableList, i)); - - tsdbReaderT* pReader = tsdbReaderOpen(pHandle->vnode, &cond, subListInfo, queryId, taskId); - taosArrayPush(arrayReader, &pReader); - - taosArrayDestroy(subListInfo->pTableList); - taosMemoryFree(subListInfo); - } - cleanupQueryTableDataCond(&cond); - - return TSDB_CODE_SUCCESS; - -_error: - return code; -} - -int32_t createMultipleDataReaders(SQueryTableDataCond* pQueryCond, SReadHandle* pHandle, STableListInfo* pTableListInfo, - int32_t tableStartIdx, int32_t tableEndIdx, SArray* arrayReader, uint64_t queryId, - uint64_t taskId) { - for (int32_t i = tableStartIdx; i <= tableEndIdx; ++i) { - STableListInfo* subListInfo = taosMemoryCalloc(1, sizeof(subListInfo)); - subListInfo->pTableList = taosArrayInit(1, sizeof(STableKeyInfo)); - taosArrayPush(subListInfo->pTableList, taosArrayGet(pTableListInfo->pTableList, i)); - - tsdbReaderT* pReader = tsdbReaderOpen(pHandle->vnode, pQueryCond, subListInfo, queryId, taskId); - taosArrayPush(arrayReader, &pReader); - - taosArrayDestroy(subListInfo->pTableList); - taosMemoryFree(subListInfo); + return code; } return TSDB_CODE_SUCCESS; @@ -2225,32 +2216,34 @@ SArray* generateSortByTsInfo(int32_t order) { return pList; } +static int32_t createMultipleDataReaders(SQueryTableDataCond* pQueryCond, SReadHandle* pHandle, SArray* tableList, SArray* arrayReader, uint64_t queryId, + uint64_t taskId) { + for (int32_t i = 0; i < taosArrayGetSize(tableList); ++i) { + SArray* tmp = taosArrayInit(1, sizeof(STableKeyInfo)); + taosArrayPush(tmp, taosArrayGet(tableList, i)); + + tsdbReaderT* pReader = tsdbReaderOpen(pHandle->vnode, pQueryCond, tmp, queryId, taskId); + taosArrayPush(arrayReader, &pReader); + + taosArrayDestroy(tmp); + } + + return TSDB_CODE_SUCCESS; +} + int32_t startGroupTableMergeScan(SOperatorInfo* pOperator) { STableMergeScanInfo* pInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - { - size_t tableListSize = taosArrayGetSize(pInfo->tableListInfo->pTableList); - int32_t i = pInfo->tableStartIndex + 1; - for (; i < tableListSize; ++i) { - STableKeyInfo* tableKeyInfo = taosArrayGet(pInfo->tableListInfo->pTableList, i); - if (tableKeyInfo->groupId != pInfo->groupId) { - break; - } - } - pInfo->tableEndIndex = i - 1; - } + SArray* tableList = taosArrayGetP(pInfo->tableListInfo->pGroupList, pInfo->currentGroupId); - int32_t tableStartIdx = pInfo->tableStartIndex; - int32_t tableEndIdx = pInfo->tableEndIndex; - - STableListInfo* tableListInfo = pInfo->tableListInfo; - createMultipleDataReaders(&pInfo->cond, &pInfo->readHandle, tableListInfo, tableStartIdx, tableEndIdx, + createMultipleDataReaders(&pInfo->cond, &pInfo->readHandle, tableList, pInfo->dataReaders, pInfo->queryId, pInfo->taskId); // todo the total available buffer should be determined by total capacity of buffer of this task. // the additional one is reserved for merge result - pInfo->sortBufSize = pInfo->bufPageSize * (tableEndIdx - tableStartIdx + 1 + 1); + int32_t tableLen = taosArrayGetSize(tableList); + pInfo->sortBufSize = pInfo->bufPageSize * ((tableLen==0?1:tableLen) + 1); int32_t numOfBufPage = pInfo->sortBufSize / pInfo->bufPageSize; pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, SORT_MULTISOURCE_MERGE, pInfo->bufPageSize, numOfBufPage, pInfo->pSortInputBlock, pTaskInfo->id.str); @@ -2337,38 +2330,43 @@ SSDataBlock* doTableMergeScan(SOperatorInfo* pOperator) { if (code != TSDB_CODE_SUCCESS) { longjmp(pTaskInfo->env, code); } - size_t tableListSize = taosArrayGetSize(pInfo->tableListInfo->pTableList); - if (!pInfo->hasGroupId) { - pInfo->hasGroupId = true; - if (tableListSize == 0) { + if (pInfo->currentGroupId == -1) { + pInfo->currentGroupId++; + if (pInfo->currentGroupId >= taosArrayGetSize(pInfo->tableListInfo->pGroupList)) { doSetOperatorCompleted(pOperator); return NULL; } - pInfo->tableStartIndex = 0; - pInfo->groupId = ((STableKeyInfo*)taosArrayGet(pInfo->tableListInfo->pTableList, pInfo->tableStartIndex))->groupId; startGroupTableMergeScan(pOperator); } - SSDataBlock* pBlock = NULL; - while (pInfo->tableStartIndex < tableListSize) { - pBlock = getSortedTableMergeScanBlockData(pInfo->pSortHandle, pOperator->resultInfo.capacity, pOperator); - if (pBlock != NULL) { - pBlock->info.groupId = pInfo->groupId; - pOperator->resultInfo.totalRows += pBlock->info.rows; - return pBlock; - } else { - stopGroupTableMergeScan(pOperator); - if (pInfo->tableEndIndex >= tableListSize - 1) { - doSetOperatorCompleted(pOperator); - break; - } - pInfo->tableStartIndex = pInfo->tableEndIndex + 1; - pInfo->groupId = - ((STableKeyInfo*)taosArrayGet(pInfo->tableListInfo->pTableList, pInfo->tableStartIndex))->groupId; - startGroupTableMergeScan(pOperator); - } + SSDataBlock* pBlock = getSortedTableMergeScanBlockData(pInfo->pSortHandle, pOperator->resultInfo.capacity, pOperator); + if (pBlock != NULL) { + uint64_t* groupId = taosHashGet(pInfo->tableListInfo->map, &(pBlock->info.uid), sizeof(uint64_t)); + if(groupId) pBlock->info.groupId = *groupId; + + pOperator->resultInfo.totalRows += pBlock->info.rows; + return pBlock; } + stopGroupTableMergeScan(pOperator); + pInfo->currentGroupId++; + if (pInfo->currentGroupId >= taosArrayGetSize(pInfo->tableListInfo->pGroupList)) { + doSetOperatorCompleted(pOperator); + return NULL; + } + startGroupTableMergeScan(pOperator); + + pBlock = getSortedTableMergeScanBlockData(pInfo->pSortHandle, pOperator->resultInfo.capacity, pOperator); + if (pBlock != NULL) { + uint64_t* groupId = taosHashGet(pInfo->tableListInfo->map, &(pBlock->info.uid), sizeof(uint64_t)); + if(groupId) pBlock->info.groupId = *groupId; + + pOperator->resultInfo.totalRows += pBlock->info.rows; + return pBlock; + } + + doSetOperatorCompleted(pOperator); + return pBlock; } @@ -2445,6 +2443,7 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN pInfo->dataReaders = taosArrayInit(64, POINTER_BYTES); pInfo->queryId = queryId; pInfo->taskId = taskId; + pInfo->currentGroupId = -1; pInfo->sortSourceParams = taosArrayInit(64, sizeof(STableMergeScanSortSourceParam)); diff --git a/source/libs/function/inc/builtinsimpl.h b/source/libs/function/inc/builtinsimpl.h index 4d2f926c12..0820e884ce 100644 --- a/source/libs/function/inc/builtinsimpl.h +++ b/source/libs/function/inc/builtinsimpl.h @@ -202,6 +202,10 @@ bool blockDistSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo); int32_t blockDistFunction(SqlFunctionCtx *pCtx); int32_t blockDistFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); +bool getGroupKeyFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv); +int32_t groupKeyFunction(SqlFunctionCtx* pCtx); +int32_t groupKeyFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); + #ifdef __cplusplus } #endif diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 6516a10795..6744ebe4b9 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -1515,6 +1515,16 @@ static bool getBlockDistFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv return true; } +static int32_t translateGroupKey(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { + if (1 != LIST_LENGTH(pFunc->pParameterList)) { + return TSDB_CODE_SUCCESS; + } + + SNode* pPara = nodesListGetNode(pFunc->pParameterList, 0); + pFunc->node.resType = ((SExprNode*)pPara)->resType; + return TSDB_CODE_SUCCESS; +} + // clang-format off const SBuiltinFuncDefinition funcMgtBuiltins[] = { { @@ -2499,7 +2509,17 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .type = FUNCTION_TYPE_BLOCK_DIST_INFO, .classification = FUNC_MGT_PSEUDO_COLUMN_FUNC | FUNC_MGT_SCAN_PC_FUNC, .translateFunc = translateBlockDistInfoFunc, - } + }, + { + .name = "_group_key", + .type = FUNCTION_TYPE_GROUP_KEY, + .classification = FUNC_MGT_AGG_FUNC, + .translateFunc = translateGroupKey, + .getEnvFunc = getGroupKeyFuncEnv, + .initFunc = functionSetup, + .processFunc = groupKeyFunction, + .finalizeFunc = groupKeyFinalize, + }, }; // clang-format on diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 4ec6432fc8..61f3154811 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -262,6 +262,12 @@ typedef struct SRateInfo { int8_t hasResult; // flag to denote has value } SRateInfo; +typedef struct SGroupKeyInfo{ + bool hasResult; + char data[]; +} SGroupKeyInfo; + + #define SET_VAL(_info, numOfElem, res) \ do { \ if ((numOfElem) <= 0) { \ @@ -2402,6 +2408,12 @@ bool getSelectivityFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) { return true; } +bool getGroupKeyFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) { + SColumnNode* pNode = (SColumnNode*)nodesListGetNode(pFunc->pParameterList, 0); + pEnv->calcMemSize = sizeof(SGroupKeyInfo) + pNode->node.resType.bytes; + return true; +} + static FORCE_INLINE TSKEY getRowPTs(SColumnInfoData* pTsColInfo, int32_t rowIndex) { if (pTsColInfo == NULL) { return 0; @@ -5349,6 +5361,43 @@ int32_t irateFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { return pResInfo->numOfRes; } +int32_t groupKeyFunction(SqlFunctionCtx* pCtx) { + SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); + SGroupKeyInfo* pInfo = GET_ROWCELL_INTERBUF(pResInfo); + + SInputColumnInfoData* pInput = &pCtx->input; + SColumnInfoData* pInputCol = pInput->pData[0]; + + int32_t bytes = pInputCol->info.bytes; + + int32_t startIndex = pInput->startRowIndex; + if (colDataIsNull_s(pInputCol, startIndex)) { + pInfo->hasResult = false; + goto _group_key_over; + } + + pInfo->hasResult = true; + char* data = colDataGetData(pInputCol, startIndex); + memcpy(pInfo->data, data, bytes); + +_group_key_over: + + SET_VAL(pResInfo, 1, 1); + return TSDB_CODE_SUCCESS; +} + +int32_t groupKeyFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { + int32_t slotId = pCtx->pExpr->base.resSchema.slotId; + SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); + + SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); + + SGroupKeyInfo* pInfo = GET_ROWCELL_INTERBUF(pResInfo); + colDataAppend(pCol, pBlock->info.rows, pInfo->data, pInfo->hasResult ? false : true); + + return pResInfo->numOfRes; +} + int32_t interpFunction(SqlFunctionCtx* pCtx) { #if 0 int32_t fillType = (int32_t) pCtx->param[2].i64; diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index 1858918e4a..514d65d52e 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -2294,6 +2294,7 @@ static const char* jkSubplanType = "SubplanType"; static const char* jkSubplanMsgType = "MsgType"; static const char* jkSubplanLevel = "Level"; static const char* jkSubplanDbFName = "DbFName"; +static const char* jkSubplanUser = "User"; static const char* jkSubplanNodeAddr = "NodeAddr"; static const char* jkSubplanRootNode = "RootNode"; static const char* jkSubplanDataSink = "DataSink"; @@ -2316,6 +2317,9 @@ static int32_t subplanToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = tjsonAddStringToObject(pJson, jkSubplanDbFName, pNode->dbFName); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddStringToObject(pJson, jkSubplanUser, pNode->user); + } if (TSDB_CODE_SUCCESS == code) { code = tjsonAddObject(pJson, jkSubplanNodeAddr, queryNodeAddrToJson, &pNode->execNode); } @@ -2352,6 +2356,9 @@ static int32_t jsonToSubplan(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = tjsonGetStringValue(pJson, jkSubplanDbFName, pNode->dbFName); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetStringValue(pJson, jkSubplanUser, pNode->user); + } if (TSDB_CODE_SUCCESS == code) { code = tjsonToObject(pJson, jkSubplanNodeAddr, jsonToQueryNodeAddr, &pNode->execNode); } diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 1d8baf5373..227914449a 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -1220,6 +1220,7 @@ int32_t nodesSetValueNodeValue(SValueNode* pNode, void* value) { case TSDB_DATA_TYPE_NCHAR: case TSDB_DATA_TYPE_VARCHAR: case TSDB_DATA_TYPE_VARBINARY: + case TSDB_DATA_TYPE_JSON: pNode->datum.p = (char*)value; break; default: diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 7862ad5445..10275ffd9d 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -1061,6 +1061,7 @@ static EDealRes partTagsOptRebuildTbanmeImpl(SNode** pNode, void* pContext) { } strcpy(pFunc->functionName, "tbname"); pFunc->funcType = FUNCTION_TYPE_TBNAME; + pFunc->node.resType = ((SColumnNode*)*pNode)->node.resType; nodesDestroyNode(*pNode); *pNode = (SNode*)pFunc; return DEAL_RES_IGNORE_CHILD; @@ -1188,7 +1189,7 @@ static const SOptimizeRule optimizeRuleSet[] = { {.pName = "ConditionPushDown", .optimizeFunc = cpdOptimize}, {.pName = "OrderByPrimaryKey", .optimizeFunc = opkOptimize}, {.pName = "SmaIndex", .optimizeFunc = smaOptimize}, - // {.pName = "PartitionTags", .optimizeFunc = partTagsOptimize}, + {.pName = "PartitionTags", .optimizeFunc = partTagsOptimize}, {.pName = "EliminateProject", .optimizeFunc = eliminateProjOptimize} }; // clang-format on diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index efb0f790e1..37b69ffdfe 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -1453,6 +1453,9 @@ static SSubplan* makeSubplan(SPhysiPlanContext* pCxt, SLogicSubplan* pLogicSubpl pSubplan->id = pLogicSubplan->id; pSubplan->subplanType = pLogicSubplan->subplanType; pSubplan->level = pLogicSubplan->level; + if (NULL != pCxt->pPlanCxt->pUser) { + strcpy(pSubplan->user, pCxt->pPlanCxt->pUser); + } return pSubplan; } diff --git a/source/libs/planner/test/planTestUtil.cpp b/source/libs/planner/test/planTestUtil.cpp index f1ab9c8196..c197a02dd1 100644 --- a/source/libs/planner/test/planTestUtil.cpp +++ b/source/libs/planner/test/planTestUtil.cpp @@ -85,8 +85,9 @@ class PlannerTestBaseImpl { public: PlannerTestBaseImpl() : sqlNo_(0) {} - void useDb(const string& acctId, const string& db) { - caseEnv_.acctId_ = acctId; + void useDb(const string& user, const string& db) { + caseEnv_.acctId_ = 0; + caseEnv_.user_ = user; caseEnv_.db_ = db; caseEnv_.nsql_ = g_skipSql; } @@ -193,7 +194,8 @@ class PlannerTestBaseImpl { private: struct caseEnv { - string acctId_; + int32_t acctId_; + string user_; string db_; int32_t nsql_; @@ -295,7 +297,7 @@ class PlannerTestBaseImpl { transform(stmtEnv_.sql_.begin(), stmtEnv_.sql_.end(), stmtEnv_.sql_.begin(), ::tolower); SParseContext cxt = {0}; - cxt.acctId = atoi(caseEnv_.acctId_.c_str()); + cxt.acctId = caseEnv_.acctId_; cxt.db = caseEnv_.db_.c_str(); cxt.pSql = stmtEnv_.sql_.c_str(); cxt.sqlLen = stmtEnv_.sql_.length(); @@ -319,12 +321,13 @@ class PlannerTestBaseImpl { void doParseBoundSql(SQuery* pQuery) { SParseContext cxt = {0}; - cxt.acctId = atoi(caseEnv_.acctId_.c_str()); + cxt.acctId = caseEnv_.acctId_; cxt.db = caseEnv_.db_.c_str(); cxt.pSql = stmtEnv_.sql_.c_str(); cxt.sqlLen = stmtEnv_.sql_.length(); cxt.pMsg = stmtEnv_.msgBuf_.data(); cxt.msgLen = stmtEnv_.msgBuf_.max_size(); + cxt.pUser = caseEnv_.user_.c_str(); DO_WITH_THROW(qStmtParseQuerySql, &cxt, pQuery); res_.ast_ = toString(pQuery->pRoot); @@ -364,6 +367,7 @@ class PlannerTestBaseImpl { void setPlanContext(SQuery* pQuery, SPlanContext* pCxt) { pCxt->queryId = 1; + pCxt->pUser = caseEnv_.user_.c_str(); if (QUERY_NODE_CREATE_TOPIC_STMT == nodeType(pQuery->pRoot)) { pCxt->pAstRoot = ((SCreateTopicStmt*)pQuery->pRoot)->pQuery; pCxt->topicQuery = true; @@ -403,7 +407,7 @@ PlannerTestBase::PlannerTestBase() : impl_(new PlannerTestBaseImpl()) {} PlannerTestBase::~PlannerTestBase() {} -void PlannerTestBase::useDb(const std::string& acctId, const std::string& db) { impl_->useDb(acctId, db); } +void PlannerTestBase::useDb(const std::string& user, const std::string& db) { impl_->useDb(user, db); } void PlannerTestBase::run(const std::string& sql) { return impl_->run(sql); } diff --git a/source/libs/planner/test/planTestUtil.h b/source/libs/planner/test/planTestUtil.h index b188a7a054..7d2a9e533f 100644 --- a/source/libs/planner/test/planTestUtil.h +++ b/source/libs/planner/test/planTestUtil.h @@ -30,7 +30,7 @@ class PlannerTestBase : public testing::Test { PlannerTestBase(); virtual ~PlannerTestBase(); - void useDb(const std::string& acctId, const std::string& db); + void useDb(const std::string& user, const std::string& db); void run(const std::string& sql); // stmt mode APIs void prepare(const std::string& sql); diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index 179dc88d2a..a8b18210dd 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -63,7 +63,6 @@ int32_t tDecodeStreamDispatchReq(SDecoder* pDecoder, SStreamDispatchReq* pReq) { } int32_t tEncodeStreamRetrieveReq(SEncoder* pEncoder, const SStreamRetrieveReq* pReq) { - // if (tStartEncode(pEncoder) < 0) return -1; if (tEncodeI64(pEncoder, pReq->streamId) < 0) return -1; if (tEncodeI32(pEncoder, pReq->dstNodeId) < 0) return -1; @@ -84,7 +83,7 @@ int32_t tDecodeStreamRetrieveReq(SDecoder* pDecoder, SStreamRetrieveReq* pReq) { if (tDecodeI32(pDecoder, &pReq->srcTaskId) < 0) return -1; uint64_t len = 0; if (tDecodeBinaryAlloc(pDecoder, (void**)&pReq->pRetrieve, &len) < 0) return -1; - pReq->retrieveLen = len; + pReq->retrieveLen = (int32_t)len; tEndDecode(pDecoder); return 0; } diff --git a/tests/script/tsim/mnode/basic5.sim b/tests/script/tsim/mnode/basic5.sim index 23f5f6d782..c017d7f23f 100644 --- a/tests/script/tsim/mnode/basic5.sim +++ b/tests/script/tsim/mnode/basic5.sim @@ -157,7 +157,7 @@ step61: if $x == 10 then return -1 endi -sql show mnodes +sql show mnodes -x step61 print ===> $data00 $data01 $data02 $data03 $data04 $data05 print ===> $data10 $data11 $data12 $data13 $data14 $data15 print ===> $data20 $data21 $data22 $data23 $data24 $data25 diff --git a/tests/script/tsim/user/privilege_sysinfo.sim b/tests/script/tsim/user/privilege_sysinfo.sim index 9ddfce8a97..ea3294765c 100644 --- a/tests/script/tsim/user/privilege_sysinfo.sim +++ b/tests/script/tsim/user/privilege_sysinfo.sim @@ -22,5 +22,26 @@ sql_error drop user sysinfo1 sql_error alter user sysinfo1 pass '1' sql_error alter user sysinfo0 pass '1' +sql_error create dnode $hostname port 7200 +sql_error drop dnode 1 + +sql_error create qnode on dnode 1 +sql_error drop qnode on dnode 1 + +sql_error create mnode on dnode 1 +sql_error drop mnode on dnode 1 + +sql_error create snode on dnode 1 +sql_error drop snode on dnode 1 + +sql_error redistribute vgroup 2 dnode 1 dnode 2 +sql_error balance vgroup + +sql_error kill transaction 1 +sql_error kill connection 1 +sql_error kill query 1 + +print =============== check db +sql_error create database db system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/system-test/2-query/json_tag.py b/tests/system-test/2-query/json_tag.py index 0c649f2008..2ef1b8dad2 100644 --- a/tests/system-test/2-query/json_tag.py +++ b/tests/system-test/2-query/json_tag.py @@ -412,52 +412,59 @@ class TDTestCase: tdSql.checkColNameList(res, cname_list) # # test group by & order by json tag + tdSql.query("select ts,jtag->'tag1' from jsons1 partition by jtag->'tag1' order by jtag->'tag1' desc") + tdSql.checkRows(11) + tdSql.checkData(0, 1, '"femail"') + tdSql.checkData(2, 1, '"收到货"') + tdSql.checkData(7, 1, "false") + + # tdSql.error("select count(*) from jsons1 group by jtag") # tdSql.error("select count(*) from jsons1 partition by jtag") # tdSql.error("select count(*) from jsons1 group by jtag order by jtag") tdSql.error("select count(*) from jsons1 group by jtag->'tag1' order by jtag->'tag2'") tdSql.error("select count(*) from jsons1 group by jtag->'tag1' order by jtag") - tdSql.query("select count(*),jtag->'tag1' from jsons1 group by jtag->'tag1' order by jtag->'tag1' desc") - tdSql.checkRows(8) - tdSql.checkData(0, 0, 2) - tdSql.checkData(0, 1, '"femail"') - tdSql.checkData(1, 0, 2) - tdSql.checkData(1, 1, '"收到货"') - tdSql.checkData(2, 0, 1) - tdSql.checkData(2, 1, "11.000000000") - tdSql.checkData(5, 0, 1) - tdSql.checkData(5, 1, "false") - tdSql.checkData(6, 0, 1) - tdSql.checkData(6, 1, "null") - tdSql.checkData(7, 0, 2) - tdSql.checkData(7, 1, None) + # tdSql.query("select count(*),jtag->'tag1' from jsons1 group by jtag->'tag1' order by jtag->'tag1' desc") + # tdSql.checkRows(8) + # tdSql.checkData(0, 0, 2) + # tdSql.checkData(0, 1, '"femail"') + # tdSql.checkData(1, 0, 2) + # tdSql.checkData(1, 1, '"收到货"') + # tdSql.checkData(2, 0, 1) + # tdSql.checkData(2, 1, "11.000000000") + # tdSql.checkData(5, 0, 1) + # tdSql.checkData(5, 1, "false") + # tdSql.checkData(6, 0, 1) + # tdSql.checkData(6, 1, "null") + # tdSql.checkData(7, 0, 2) + # tdSql.checkData(7, 1, None) - tdSql.query("select count(*),jtag->'tag1' from jsons1 group by jtag->'tag1' order by jtag->'tag1' asc") - tdSql.checkRows(8) - tdSql.checkData(0, 0, 2) - tdSql.checkData(0, 1, None) - tdSql.checkData(2, 0, 1) - tdSql.checkData(2, 1, "false") - tdSql.checkData(5, 0, 1) - tdSql.checkData(5, 1, "11.000000000") - tdSql.checkData(7, 0, 2) - tdSql.checkData(7, 1, '"femail"') + # tdSql.query("select count(*),jtag->'tag1' from jsons1 group by jtag->'tag1' order by jtag->'tag1' asc") + # tdSql.checkRows(8) + # tdSql.checkData(0, 0, 2) + # tdSql.checkData(0, 1, None) + # tdSql.checkData(2, 0, 1) + # tdSql.checkData(2, 1, "false") + # tdSql.checkData(5, 0, 1) + # tdSql.checkData(5, 1, "11.000000000") + # tdSql.checkData(7, 0, 2) + # tdSql.checkData(7, 1, '"femail"') # # test stddev with group by json tag - tdSql.query("select stddev(dataint),jtag->'tag1' from jsons1 group by jtag->'tag1' order by jtag->'tag1'") - tdSql.checkRows(8) - tdSql.checkData(0, 0, 10) - tdSql.checkData(0, 1, None) - tdSql.checkData(4, 0, 0) - tdSql.checkData(4, 1, "5.000000000") - tdSql.checkData(7, 0, 11) - tdSql.checkData(7, 1, '"femail"') - - res = tdSql.getColNameList("select stddev(dataint),jsons1.jtag->'tag1' from jsons1 group by jsons1.jtag->'tag1' order by jtag->'tag1'") - cname_list = [] - cname_list.append("stddev(dataint)") - cname_list.append("jsons1.jtag->'tag1'") - tdSql.checkColNameList(res, cname_list) + # tdSql.query("select stddev(dataint),jtag->'tag1' from jsons1 group by jtag->'tag1' order by jtag->'tag1'") + # tdSql.checkRows(8) + # tdSql.checkData(0, 0, 10) + # tdSql.checkData(0, 1, None) + # tdSql.checkData(4, 0, 0) + # tdSql.checkData(4, 1, "5.000000000") + # tdSql.checkData(7, 0, 11) + # tdSql.checkData(7, 1, '"femail"') + # + # res = tdSql.getColNameList("select stddev(dataint),jsons1.jtag->'tag1' from jsons1 group by jsons1.jtag->'tag1' order by jtag->'tag1'") + # cname_list = [] + # cname_list.append("stddev(dataint)") + # cname_list.append("jsons1.jtag->'tag1'") + # tdSql.checkColNameList(res, cname_list) # test top/bottom with group by json tag # tdSql.query("select top(dataint,2),jtag->'tag1' from jsons1 group by jtag->'tag1' order by jtag->'tag1'") @@ -470,8 +477,8 @@ class TDTestCase: # tdSql.checkData(10, 1, '"femail"') # test having - tdSql.query("select count(*),jtag->'tag1' from jsons1 group by jtag->'tag1' having count(*) > 1") - tdSql.checkRows(3) + # tdSql.query("select count(*),jtag->'tag1' from jsons1 group by jtag->'tag1' having count(*) > 1") + # tdSql.checkRows(3) # subquery with json tag tdSql.query("select * from (select jtag, dataint from jsons1) order by dataint") diff --git a/tests/system-test/6-cluster/5dnode3mnodeStop.py b/tests/system-test/6-cluster/5dnode3mnodeStop.py index 69b9c3d879..3a85207af6 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeStop.py +++ b/tests/system-test/6-cluster/5dnode3mnodeStop.py @@ -227,6 +227,7 @@ class TDTestCase: # fisr add three mnodes; tdSql.execute("create mnode on dnode 2") + time.sleep(10) tdSql.execute("create mnode on dnode 3") # fisrt check statut ready diff --git a/tests/system-test/7-tmq/tmqCommon.py b/tests/system-test/7-tmq/tmqCommon.py index d17e36fc97..b8aa78e3ac 100644 --- a/tests/system-test/7-tmq/tmqCommon.py +++ b/tests/system-test/7-tmq/tmqCommon.py @@ -11,7 +11,9 @@ # -*- coding: utf-8 -*- +from asyncore import loop from collections import defaultdict +import subprocess import random import string import threading @@ -75,7 +77,7 @@ class TMQCom: return resultList - def startTmqSimProcess(self,pollDelay,dbName,showMsg=1,showRow=1,cdbName='cdb',valgrind=0): + def startTmqSimProcess(self,pollDelay,dbName,showMsg=1,showRow=1,cdbName='cdb',valgrind=0,alias=0): buildPath = tdCom.getBuildPath() cfgPath = tdCom.getClientCfgPath() if valgrind == 1: @@ -88,30 +90,53 @@ class TMQCom: shellCmd += " -y %d -d %s -g %d -r %d -w %s "%(pollDelay, dbName, showMsg, showRow, cdbName) shellCmd += "> nul 2>&1 &" else: - shellCmd = 'nohup ' + buildPath + '/build/bin/tmq_sim -c ' + cfgPath + processorName = buildPath + '/build/bin/tmq_sim' + if alias != 0: + processorNameNew = buildPath + '/build/bin/tmq_sim_new' + shellCmd = 'cp %s %s'%(processorName, processorNameNew) + os.system(shellCmd) + processorName = processorNameNew + shellCmd = 'nohup ' + processorName + ' -c ' + cfgPath shellCmd += " -y %d -d %s -g %d -r %d -w %s "%(pollDelay, dbName, showMsg, showRow, cdbName) shellCmd += "> /dev/null 2>&1 &" tdLog.info(shellCmd) - os.system(shellCmd) + os.system(shellCmd) - def getStartConsumeNotifyFromTmqsim(self,cdbName='cdb'): - while 1: + def stopTmqSimProcess(self, processorName): + psCmd = "ps -ef|grep -w %s|grep -v grep | awk '{print $2}'"%(processorName) + processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") + while(processID): + killCmd = "kill -INT %s > /dev/null 2>&1" % processID + os.system(killCmd) + time.sleep(0.2) + processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") + tdLog.debug("%s is stopped by kill -INT" % (processorName)) + + def getStartConsumeNotifyFromTmqsim(self,cdbName='cdb',rows=1): + loopFlag = 1 + while loopFlag: tdSql.query("select * from %s.notifyinfo"%cdbName) #tdLog.info("row: %d, %l64d, %l64d"%(tdSql.getData(0, 1),tdSql.getData(0, 2),tdSql.getData(0, 3)) - if (tdSql.getRows() == 1) and (tdSql.getData(0, 1) == 0): - break - else: - time.sleep(0.1) + actRows = tdSql.getRows() + if (actRows >= rows): + for i in range(actRows): + if tdSql.getData(i, 1) == 0: + loopFlag = 0 + break + time.sleep(0.1) return - def getStartCommitNotifyFromTmqsim(self,cdbName='cdb'): - while 1: + def getStartCommitNotifyFromTmqsim(self,cdbName='cdb',rows=2): + loopFlag = 1 + while loopFlag: tdSql.query("select * from %s.notifyinfo"%cdbName) #tdLog.info("row: %d, %l64d, %l64d"%(tdSql.getData(0, 1),tdSql.getData(0, 2),tdSql.getData(0, 3)) - if tdSql.getRows() == 2 : - print(tdSql.getData(0, 1), tdSql.getData(1, 1)) - if tdSql.getData(1, 1) == 1: - break + actRows = tdSql.getRows() + if (actRows >= rows): + for i in range(actRows): + if tdSql.getData(i, 1) == 1: + loopFlag = 0 + break time.sleep(0.1) return diff --git a/tests/system-test/7-tmq/tmqConsumerGroup.py b/tests/system-test/7-tmq/tmqConsumerGroup.py new file mode 100644 index 0000000000..bfd63fd4a2 --- /dev/null +++ b/tests/system-test/7-tmq/tmqConsumerGroup.py @@ -0,0 +1,170 @@ + +import taos +import sys +import time +import socket +import os +import threading + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +sys.path.append("./7-tmq") +from tmqCommon import * + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + #tdSql.init(conn.cursor(), logSql) # output sql.txt file + + def checkFileContent(self, consumerId, queryString): + buildPath = tdCom.getBuildPath() + cfgPath = tdCom.getClientCfgPath() + dstFile = '%s/../log/dstrows_%d.txt'%(cfgPath, consumerId) + cmdStr = '%s/build/bin/taos -c %s -s "%s >> %s"'%(buildPath, cfgPath, queryString, dstFile) + tdLog.info(cmdStr) + os.system(cmdStr) + + consumeRowsFile = '%s/../log/consumerid_%d.txt'%(cfgPath, consumerId) + tdLog.info("rows file: %s, %s"%(consumeRowsFile, dstFile)) + + consumeFile = open(consumeRowsFile, mode='r') + queryFile = open(dstFile, mode='r') + + # skip first line for it is schema + queryFile.readline() + + while True: + dst = queryFile.readline() + src = consumeFile.readline() + + if dst: + if dst != src: + tdLog.exit("consumerId %d consume rows is not match the rows by direct query"%consumerId) + else: + break + return + + def tmqCase1(self): + tdLog.printNoPrefix("======== test case 1: ") + paraDict = {'dbName': 'db1', + 'dropFlag': 1, + 'event': '', + 'vgroups': 4, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':2}, {'type': 'binary', 'len':20, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbNum': 10, + 'rowsPerTbl': 1000, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 20, + 'showMsg': 1, + 'showRow': 1} + + topicNameList = ['topic1', 'topic2'] + expectRowsList = [] + tmqCom.initConsumerTable() + tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=4,replica=1) + tdLog.info("create stb") + tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema']) + tdLog.info("create ctb") + tdCom.create_ctable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"],tag_elm_list=paraDict['tagSchema'],count=paraDict["ctbNum"], default_ctbname_prefix=paraDict['ctbPrefix']) + tdLog.info("insert data") + tmqCom.insert_data_2(tdSql,paraDict["dbName"],paraDict["ctbPrefix"],paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],paraDict["startTs"]) + + tdLog.info("create topics from stb with filter") + # queryString = "select ts, log(c1), ceil(pow(c1,3)) from %s.%s where c1 %% 7 == 0" %(paraDict['dbName'], paraDict['stbName']) + queryString = "select ts, log(c1), ceil(pow(c1,3)) from %s.%s" %(paraDict['dbName'], paraDict['stbName']) + sqlString = "create topic %s as %s" %(topicNameList[0], queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + tdSql.query(queryString) + expectRowsList.append(tdSql.getRows()) + + # create one stb2 + paraDict["stbName"] = 'stb2' + paraDict["ctbPrefix"] = 'ctbx' + paraDict["rowsPerTbl"] = 5000 + tdLog.info("create stb2") + tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema']) + tdLog.info("create ctb2") + tdCom.create_ctable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"],tag_elm_list=paraDict['tagSchema'],count=paraDict["ctbNum"], default_ctbname_prefix=paraDict['ctbPrefix']) + + # queryString = "select ts, sin(c1), abs(pow(c1,3)) from %s.%s where c1 %% 7 == 0" %(paraDict['dbName'], paraDict['stbName']) + queryString = "select ts, sin(c1), abs(pow(c1,3)) from %s.%s" %(paraDict['dbName'], paraDict['stbName']) + sqlString = "create topic %s as %s" %(topicNameList[1], queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + # tdSql.query(queryString) + # expectRowsList.append(tdSql.getRows()) + + # init consume info, and start tmq_sim, then check consume result + tdLog.info("insert consume info to consume processor") + consumerId = 0 + expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2 + topicList = "%s,%s"%(topicNameList[0],topicNameList[1]) + ifcheckdata = 1 + ifManualCommit = 1 + keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:3000, auto.offset.reset:earliest' + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor 1") + tmqCom.startTmqSimProcess(paraDict['pollDelay'],paraDict["dbName"],paraDict['showMsg'], paraDict['showRow']) + + tdLog.info("start consume processor 2") + tmqCom.startTmqSimProcess(paraDict['pollDelay'],paraDict["dbName"],paraDict['showMsg'], paraDict['showRow'],'cdb',0,1) + + tdLog.info("async insert data") + pThread = tmqCom.asyncInsertData(paraDict) + + tdLog.info("wait consumer commit notify") + tmqCom.getStartCommitNotifyFromTmqsim(rows=4) + + tdLog.info("pkill one consume processor") + tmqCom.stopTmqSimProcess('tmq_sim_new') + + pThread.join() + + tdLog.info("wait the consume result") + expectRows = 2 + resultList = tmqCom.selectConsumeResult(expectRows) + actTotalRows = 0 + for i in range(len(resultList)): + actTotalRows += resultList[i] + + tdSql.query(queryString) + expectRowsList.append(tdSql.getRows()) + expectTotalRows = 0 + for i in range(len(expectRowsList)): + expectTotalRows += expectRowsList[i] + + tdLog.info("act consume rows: %d, expect consume rows: %d"%(actTotalRows, expectTotalRows)) + if expectTotalRows <= resultList[0]: + tdLog.info("act consume rows: %d should >= expect consume rows: %d"%(actTotalRows, expectTotalRows)) + tdLog.exit("0 tmq consume rows error!") + + # time.sleep(10) + # for i in range(len(topicNameList)): + # tdSql.query("drop topic %s"%topicNameList[i]) + + tdLog.printNoPrefix("======== test case 1 end ...... ") + + def run(self): + tdSql.prepare() + self.tmqCase1() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +event = threading.Event() + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 30a1ccb33a..ef217b828f 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -138,3 +138,4 @@ python3 ./test.py -f 7-tmq/stbFilter.py python3 ./test.py -f 7-tmq/tmqCheckData.py python3 ./test.py -f 7-tmq/tmqUdf.py #python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 5 +python3 ./test.py -f 7-tmq/tmqConsumerGroup.py diff --git a/tests/test/c/tmqSim.c b/tests/test/c/tmqSim.c index e96f5d4793..c2fba52e6e 100644 --- a/tests/test/c/tmqSim.c +++ b/tests/test/c/tmqSim.c @@ -93,8 +93,6 @@ static SConfInfo g_stConfInfo; TdFilePtr g_fp = NULL; static int running = 1; -int8_t useSnapshot = 0; - // char* g_pRowValue = NULL; // TdFilePtr g_fp = NULL; @@ -126,11 +124,23 @@ char* getCurrentTimeString(char* timeString) { return timeString; } +static void tmqStop(int signum, void *info, void *ctx) { + running = 0; + char tmpString[128]; + taosFprintfFile(g_fp, "%s tmqStop() receive stop signal[%d]\n", getCurrentTimeString(tmpString), signum); +} + +static void tmqSetSignalHandle() { + taosSetSignal(SIGINT, tmqStop); +} + void initLogFile() { char filename[256]; char tmpString[128]; - sprintf(filename, "%s/../log/tmqlog_%s.txt", configDir, getCurrentTimeString(tmpString)); + pid_t process_id = getpid(); + + sprintf(filename, "%s/../log/tmqlog-%d-%s.txt", configDir, process_id, getCurrentTimeString(tmpString)); #ifdef WINDOWS for (int i = 2; i < sizeof(filename); i++) { if (filename[i] == ':') filename[i] = '-'; @@ -205,7 +215,7 @@ void parseArgument(int32_t argc, char* argv[]) { } else if (strcmp(argv[i], "-y") == 0) { g_stConfInfo.consumeDelay = atol(argv[++i]); } else if (strcmp(argv[i], "-e") == 0) { - useSnapshot = (int8_t)atol(argv[++i]); + g_stConfInfo.useSnapshot = atol(argv[++i]); } else { pError("%s unknow para: %s %s", GREEN, argv[++i], NC); exit(-1); @@ -519,7 +529,9 @@ static void tmq_commit_cb_print(tmq_t* tmq, int32_t code, void* param) { g_once_commit_flag = 1; notifyMainScript((SThreadInfo*)param, (int32_t)NOTIFY_CMD_START_COMMIT); } - taosFprintfFile(g_fp, "tmq_commit_cb_print() be called\n"); + + char tmpString[128]; + taosFprintfFile(g_fp, "%s tmq_commit_cb_print() be called\n", getCurrentTimeString(tmpString)); } void build_consumer(SThreadInfo* pInfo) { @@ -552,7 +564,7 @@ void build_consumer(SThreadInfo* pInfo) { // tmq_conf_set(conf, "auto.offset.reset", "earliest"); // tmq_conf_set(conf, "auto.offset.reset", "latest"); // - if (useSnapshot) { + if (g_stConfInfo.useSnapshot) { tmq_conf_set(conf, "experiment.use.snapshot", "true"); } @@ -651,6 +663,10 @@ void loop_consume(SThreadInfo* pInfo) { } } + if (0 == running) { + taosFprintfFile(g_fp, "receive stop signal and not continue consume\n"); + } + pInfo->consumeMsgCnt = totalMsgs; pInfo->consumeRowCnt = totalRows; @@ -680,7 +696,7 @@ void* consumeThreadFunc(void* param) { int32_t err = tmq_subscribe(pInfo->tmq, pInfo->topicList); if (err != 0) { pError("tmq_subscribe() fail, reason: %s\n", tmq_err2str(err)); - taosFprintfFile(g_fp, "tmq_subscribe()! reason: %s\n", tmq_err2str(err)); + taosFprintfFile(g_fp, "tmq_subscribe() fail! reason: %s\n", tmq_err2str(err)); assert(0); return NULL; } @@ -829,6 +845,8 @@ int main(int32_t argc, char* argv[]) { getConsumeInfo(); saveConfigToLogFile(); + tmqSetSignalHandle(); + TdThreadAttr thattr; taosThreadAttrInit(&thattr); taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE);