From cc3a7375ed8d47706fc94f1d326de7c0914eb992 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 1 Aug 2022 20:47:08 +0800 Subject: [PATCH 01/18] enh: support app level stat --- include/common/systable.h | 60 +++++++++---------- include/common/tmsg.h | 1 + include/libs/qcom/query.h | 1 + source/client/inc/clientInt.h | 4 +- source/client/src/clientEnv.c | 12 ++++ source/client/src/clientImpl.c | 38 +++++++++++- source/client/src/clientMain.c | 17 ++++++ source/common/src/systable.c | 18 +++--- source/dnode/mnode/impl/src/mndShow.c | 16 ++--- source/libs/executor/src/scanoperator.c | 22 +++---- source/libs/parser/src/parAstParser.c | 14 ++--- source/libs/parser/src/parInsert.c | 3 + source/libs/parser/src/parTranslater.c | 28 ++++----- source/libs/parser/test/mockCatalog.cpp | 14 ++--- source/libs/planner/src/planPhysiCreater.c | 6 +- source/libs/scheduler/src/schJob.c | 1 + source/libs/scheduler/src/schRemote.c | 1 + tests/script/tsim/query/explain.sim | 10 ++-- tests/script/tsim/show/basic.sim | 52 ++++++++-------- .../script/tsim/tmq/basic2Of2ConsOverlap.sim | 4 +- tests/script/tsim/tmq/snapshot1.sim | 4 +- tests/script/tsim/valgrind/checkError1.sim | 14 ++--- tests/script/tsim/valgrind/checkError6.sim | 6 +- 23 files changed, 208 insertions(+), 138 deletions(-) diff --git a/include/common/systable.h b/include/common/systable.h index ca097cd8d2..ed2e6a46c3 100644 --- a/include/common/systable.h +++ b/include/common/systable.h @@ -23,38 +23,38 @@ extern "C" { #define TDENGINE_SYSTABLE_H #define TSDB_INFORMATION_SCHEMA_DB "information_schema" -#define TSDB_INS_TABLE_DNODES "dnodes" -#define TSDB_INS_TABLE_MNODES "mnodes" -#define TSDB_INS_TABLE_MODULES "modules" -#define TSDB_INS_TABLE_QNODES "qnodes" -#define TSDB_INS_TABLE_BNODES "bnodes" -#define TSDB_INS_TABLE_SNODES "snodes" -#define TSDB_INS_TABLE_CLUSTER "cluster" -#define TSDB_INS_TABLE_USER_DATABASES "user_databases" -#define TSDB_INS_TABLE_USER_FUNCTIONS "user_functions" -#define TSDB_INS_TABLE_USER_INDEXES "user_indexes" -#define TSDB_INS_TABLE_USER_STABLES "user_stables" -#define TSDB_INS_TABLE_USER_TABLES "user_tables" -#define TSDB_INS_TABLE_USER_TAGS "user_tags" -#define TSDB_INS_TABLE_USER_TABLE_DISTRIBUTED "user_table_distributed" -#define TSDB_INS_TABLE_USER_USERS "user_users" -#define TSDB_INS_TABLE_LICENCES "grants" -#define TSDB_INS_TABLE_VGROUPS "vgroups" -#define TSDB_INS_TABLE_VNODES "vnodes" -#define TSDB_INS_TABLE_CONFIGS "configs" -#define TSDB_INS_TABLE_DNODE_VARIABLES "dnode_variables" +#define TSDB_INS_TABLE_DNODES "ins_dnodes" +#define TSDB_INS_TABLE_MNODES "ins_mnodes" +#define TSDB_INS_TABLE_MODULES "ins_modules" +#define TSDB_INS_TABLE_QNODES "ins_qnodes" +#define TSDB_INS_TABLE_BNODES "ins_bnodes" +#define TSDB_INS_TABLE_SNODES "ins_snodes" +#define TSDB_INS_TABLE_CLUSTER "ins_cluster" +#define TSDB_INS_TABLE_DATABASES "ins_databases" +#define TSDB_INS_TABLE_FUNCTIONS "ins_functions" +#define TSDB_INS_TABLE_INDEXES "ins_indexes" +#define TSDB_INS_TABLE_STABLES "ins_stables" +#define TSDB_INS_TABLE_TABLES "ins_tables" +#define TSDB_INS_TABLE_TAGS "ins_tags" +#define TSDB_INS_TABLE_TABLE_DISTRIBUTED "ins_table_distributed" +#define TSDB_INS_TABLE_USERS "ins_users" +#define TSDB_INS_TABLE_LICENCES "ins_grants" +#define TSDB_INS_TABLE_VGROUPS "ins_vgroups" +#define TSDB_INS_TABLE_VNODES "ins_vnodes" +#define TSDB_INS_TABLE_CONFIGS "ins_configs" +#define TSDB_INS_TABLE_DNODE_VARIABLES "ins_dnode_variables" #define TSDB_PERFORMANCE_SCHEMA_DB "performance_schema" -#define TSDB_PERFS_TABLE_SMAS "smas" -#define TSDB_PERFS_TABLE_CONNECTIONS "connections" -#define TSDB_PERFS_TABLE_QUERIES "queries" -#define TSDB_PERFS_TABLE_TOPICS "topics" -#define TSDB_PERFS_TABLE_CONSUMERS "consumers" -#define TSDB_PERFS_TABLE_SUBSCRIPTIONS "subscriptions" -#define TSDB_PERFS_TABLE_OFFSETS "offsets" -#define TSDB_PERFS_TABLE_TRANS "trans" -#define TSDB_PERFS_TABLE_STREAMS "streams" -#define TSDB_PERFS_TABLE_APPS "apps" +#define TSDB_PERFS_TABLE_SMAS "perf_smas" +#define TSDB_PERFS_TABLE_CONNECTIONS "perf_connections" +#define TSDB_PERFS_TABLE_QUERIES "perf_queries" +#define TSDB_PERFS_TABLE_TOPICS "perf_topics" +#define TSDB_PERFS_TABLE_CONSUMERS "perf_consumers" +#define TSDB_PERFS_TABLE_SUBSCRIPTIONS "perf_subscriptions" +#define TSDB_PERFS_TABLE_OFFSETS "perf_offsets" +#define TSDB_PERFS_TABLE_TRANS "perf_trans" +#define TSDB_PERFS_TABLE_STREAMS "perf_streams" +#define TSDB_PERFS_TABLE_APPS "perf_apps" typedef struct SSysDbTableSchema { const char* name; diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 7eafc4c3d8..9bc2ae4268 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -2223,6 +2223,7 @@ typedef struct SAppClusterSummary { uint64_t insertBytes; // submit to tsdb since launched. uint64_t fetchBytes; + uint64_t numOfQueryReq; uint64_t queryElapsedTime; uint64_t numOfSlowQueries; uint64_t totalRequests; diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index b62f822313..34d870397f 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -71,6 +71,7 @@ typedef struct SIndexMeta { typedef struct SExecResult { int32_t code; uint64_t numOfRows; + uint64_t numOfBytes; int32_t msgType; void* res; } SExecResult; diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index 2831e7a904..e7a099cf23 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -225,10 +225,12 @@ typedef struct SRequestObj { SArray* targetTableList; SQueryExecMetric metric; SRequestSendRecvBody body; + int32_t stmtType; bool syncQuery; // todo refactor: async query object bool stableQuery; // todo refactor bool validateOnly; // todo refactor bool killed; + bool inRetry; uint32_t prevCode; // previous error code: todo refactor, add update flag for catalog uint32_t retry; } SRequestObj; @@ -326,7 +328,7 @@ void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet); STscObj* taos_connect_internal(const char* ip, const char* user, const char* pass, const char* auth, const char* db, uint16_t port, int connType); -SRequestObj* launchQuery(uint64_t connId, const char* sql, int sqlLen, bool validateOnly); +SRequestObj* launchQuery(uint64_t connId, const char* sql, int sqlLen, bool validateOnly, bool inRetry); int32_t parseSql(SRequestObj* pRequest, bool topicQuery, SQuery** pQuery, SStmtCallback* pStmtCb); diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 95ce5c8338..d88a587f6a 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -60,6 +60,7 @@ static int32_t registerRequest(SRequestObj *pRequest, STscObj *pTscObj) { } static void deregisterRequest(SRequestObj *pRequest) { + const static int64_t SLOW_QUERY_INTERVAL = 3000000L; // todo configurable assert(pRequest != NULL); STscObj *pTscObj = pRequest->pTscObj; @@ -72,6 +73,17 @@ static void deregisterRequest(SRequestObj *pRequest) { tscDebug("0x%" PRIx64 " free Request from connObj: 0x%" PRIx64 ", reqId:0x%" PRIx64 " elapsed:%" PRIu64 " ms, current:%d, app current:%d", pRequest->self, pTscObj->id, pRequest->requestId, duration / 1000, num, currentInst); + + if (QUERY_NODE_VNODE_MODIF_STMT == pRequest->stmtType) { + atomic_add_fetch_64((int64_t *)&pActivity->insertElapsedTime, duration); + } else if (QUERY_NODE_SELECT_STMT == pRequest->stmtType) { + atomic_add_fetch_64((int64_t *)&pActivity->queryElapsedTime, duration); + } + + if (duration >= SLOW_QUERY_INTERVAL) { + atomic_add_fetch_64((int64_t *)&pActivity->numOfSlowQueries, 1); + } + releaseTscObj(pTscObj->id); } diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 6f8e7d0237..d2c0a70731 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -688,7 +688,12 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList if (TDMT_VND_SUBMIT == pRequest->type || TDMT_VND_DELETE == pRequest->type || TDMT_VND_CREATE_TABLE == pRequest->type) { pRequest->body.resInfo.numOfRows = res.numOfRows; - + if (TDMT_VND_SUBMIT == pRequest->type) { + STscObj *pTscObj = pRequest->pTscObj; + SAppClusterSummary *pActivity = &pTscObj->pAppInfo->summary; + atomic_add_fetch_64((int64_t *)&pActivity->numOfInsertRows, res.numOfRows); + } + schedulerFreeJob(&pRequest->body.queryJob, 0); } @@ -795,6 +800,8 @@ int32_t handleQueryExecRsp(SRequestObj* pRequest) { break; } case TDMT_VND_SUBMIT: { + atomic_add_fetch_64((int64_t *)&pAppInfo->summary.insertBytes, pRes->numOfBytes); + code = handleSubmitExecRes(pRequest, pRes->res, pCatalog, &epset); break; } @@ -824,6 +831,11 @@ void schedulerExecCb(SExecResult* pResult, void* param, int32_t code) { TDMT_VND_CREATE_TABLE == pRequest->type) { if (pResult) { pRequest->body.resInfo.numOfRows = pResult->numOfRows; + if (TDMT_VND_SUBMIT == pRequest->type) { + STscObj *pTscObj = pRequest->pTscObj; + SAppClusterSummary *pActivity = &pTscObj->pAppInfo->summary; + atomic_add_fetch_64((int64_t *)&pActivity->numOfInsertRows, pResult->numOfRows); + } } schedulerFreeJob(&pRequest->body.queryJob, 0); @@ -862,6 +874,18 @@ void schedulerExecCb(SExecResult* pResult, void* param, int32_t code) { SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, bool keepQuery, void** res) { int32_t code = 0; + pRequest->stmtType = pQuery->pRoot->type; + + if (!pRequest->inRetry) { + STscObj *pTscObj = pRequest->pTscObj; + SAppClusterSummary *pActivity = &pTscObj->pAppInfo->summary; + if (QUERY_NODE_VNODE_MODIF_STMT == pQuery->pRoot->type) { + atomic_add_fetch_64((int64_t *)&pActivity->numOfInsertsReq, 1); + } else if (QUERY_NODE_SELECT_STMT == pQuery->pRoot->type) { + atomic_add_fetch_64((int64_t *)&pActivity->numOfQueryReq, 1); + } + } + switch (pQuery->execMode) { case QUERY_EXEC_MODE_LOCAL: if (!pRequest->validateOnly) { @@ -915,7 +939,7 @@ SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, bool keepQue return pRequest; } -SRequestObj* launchQuery(uint64_t connId, const char* sql, int sqlLen, bool validateOnly) { +SRequestObj* launchQuery(uint64_t connId, const char* sql, int sqlLen, bool validateOnly, bool inRetry) { SRequestObj* pRequest = NULL; SQuery* pQuery = NULL; @@ -931,6 +955,7 @@ SRequestObj* launchQuery(uint64_t connId, const char* sql, int sqlLen, bool vali return pRequest; } + pRequest->inRetry = inRetry; pRequest->stableQuery = pQuery->stableQuery; return launchQueryImpl(pRequest, pQuery, false, NULL); @@ -1079,10 +1104,11 @@ SRequestObj* execQuery(uint64_t connId, const char* sql, int sqlLen, bool valida SRequestObj* pRequest = NULL; int32_t retryNum = 0; int32_t code = 0; + bool inRetry = false; do { destroyRequest(pRequest); - pRequest = launchQuery(connId, sql, sqlLen, validateOnly); + pRequest = launchQuery(connId, sql, sqlLen, validateOnly, inRetry); if (pRequest == NULL || TSDB_CODE_SUCCESS == pRequest->code || !NEED_CLIENT_HANDLE_ERROR(pRequest->code)) { break; } @@ -1092,6 +1118,8 @@ SRequestObj* execQuery(uint64_t connId, const char* sql, int sqlLen, bool valida pRequest->code = code; break; } + + inRetry = true; } while (retryNum++ < REQUEST_TOTAL_EXEC_TIMES); if (NEED_CLIENT_RM_TBLMETA_REQ(pRequest->type)) { @@ -1437,6 +1465,10 @@ void* doFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4) tscDebug("0x%" PRIx64 " fetch results, numOfRows:%d total Rows:%" PRId64 ", complete:%d, reqId:0x%" PRIx64, pRequest->self, pResInfo->numOfRows, pResInfo->totalRows, pResInfo->completed, pRequest->requestId); + STscObj *pTscObj = pRequest->pTscObj; + SAppClusterSummary *pActivity = &pTscObj->pAppInfo->summary; + atomic_add_fetch_64((int64_t *)&pActivity->fetchBytes, pRequest->body.resInfo.payloadLen); + if (pResultInfo->numOfRows == 0) { return NULL; } diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index a4eaf057d1..c2d621b311 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -679,6 +679,9 @@ void retrieveMetaCallback(SMetaData *pResultMeta, void *param, int32_t code) { if (code == TSDB_CODE_SUCCESS) { code = qAnalyseSqlSemantic(pWrapper->pCtx, &pWrapper->catalogReq, pResultMeta, pQuery); pRequest->stableQuery = pQuery->stableQuery; + if (pQuery->pRoot) { + pRequest->stmtType = pQuery->pRoot->type; + } } if (code == TSDB_CODE_SUCCESS) { @@ -778,6 +781,16 @@ void doAsyncQuery(SRequestObj *pRequest, bool updateMetaForce) { goto _error; } + if (!updateMetaForce) { + STscObj *pTscObj = pRequest->pTscObj; + SAppClusterSummary *pActivity = &pTscObj->pAppInfo->summary; + if (NULL == pQuery->pRoot) { + atomic_add_fetch_64((int64_t *)&pActivity->numOfInsertsReq, 1); + } else if (QUERY_NODE_SELECT_STMT == pQuery->pRoot->type) { + atomic_add_fetch_64((int64_t *)&pActivity->numOfQueryReq, 1); + } + } + SqlParseWrapper *pWrapper = taosMemoryCalloc(1, sizeof(SqlParseWrapper)); if (pWrapper == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; @@ -841,6 +854,10 @@ static void fetchCallback(void *pResult, void *param, int32_t code) { tscDebug("0x%" PRIx64 " fetch results, numOfRows:%d total Rows:%" PRId64 ", complete:%d, reqId:0x%" PRIx64, pRequest->self, pResultInfo->numOfRows, pResultInfo->totalRows, pResultInfo->completed, pRequest->requestId); + + STscObj *pTscObj = pRequest->pTscObj; + SAppClusterSummary *pActivity = &pTscObj->pAppInfo->summary; + atomic_add_fetch_64((int64_t *)&pActivity->fetchBytes, pRequest->body.resInfo.payloadLen); } pRequest->body.fetchFp(pRequest->body.param, pRequest, pResultInfo->numOfRows); diff --git a/source/common/src/systable.c b/source/common/src/systable.c index 7d07946dc9..679e0dbb8a 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -260,14 +260,14 @@ static const SSysTableMeta infosMeta[] = { // {TSDB_INS_TABLE_SNODES, snodesSchema, tListLen(snodesSchema)}, // {TSDB_INS_TABLE_BNODES, bnodesSchema, tListLen(bnodesSchema)}, {TSDB_INS_TABLE_CLUSTER, clusterSchema, tListLen(clusterSchema)}, - {TSDB_INS_TABLE_USER_DATABASES, userDBSchema, tListLen(userDBSchema)}, - {TSDB_INS_TABLE_USER_FUNCTIONS, userFuncSchema, tListLen(userFuncSchema)}, - {TSDB_INS_TABLE_USER_INDEXES, userIdxSchema, tListLen(userIdxSchema)}, - {TSDB_INS_TABLE_USER_STABLES, userStbsSchema, tListLen(userStbsSchema)}, - {TSDB_INS_TABLE_USER_TABLES, userTblsSchema, tListLen(userTblsSchema)}, - {TSDB_INS_TABLE_USER_TAGS, userTagsSchema, tListLen(userTagsSchema)}, - // {TSDB_INS_TABLE_USER_TABLE_DISTRIBUTED, userTblDistSchema, tListLen(userTblDistSchema)}, - {TSDB_INS_TABLE_USER_USERS, userUsersSchema, tListLen(userUsersSchema)}, + {TSDB_INS_TABLE_DATABASES, userDBSchema, tListLen(userDBSchema)}, + {TSDB_INS_TABLE_FUNCTIONS, userFuncSchema, tListLen(userFuncSchema)}, + {TSDB_INS_TABLE_INDEXES, userIdxSchema, tListLen(userIdxSchema)}, + {TSDB_INS_TABLE_STABLES, userStbsSchema, tListLen(userStbsSchema)}, + {TSDB_INS_TABLE_TABLES, userTblsSchema, tListLen(userTblsSchema)}, + {TSDB_INS_TABLE_TAGS, userTagsSchema, tListLen(userTagsSchema)}, + // {TSDB_INS_TABLE_TABLE_DISTRIBUTED, userTblDistSchema, tListLen(userTblDistSchema)}, + {TSDB_INS_TABLE_USERS, userUsersSchema, tListLen(userUsersSchema)}, {TSDB_INS_TABLE_LICENCES, grantsSchema, tListLen(grantsSchema)}, {TSDB_INS_TABLE_VGROUPS, vgroupsSchema, tListLen(vgroupsSchema)}, {TSDB_INS_TABLE_CONFIGS, configSchema, tListLen(configSchema)}, @@ -349,7 +349,7 @@ static const SSysDbTableSchema appSchema[] = { {.name = "insert_bytes", .bytes = 8, .type = TSDB_DATA_TYPE_UBIGINT}, {.name = "fetch_bytes", .bytes = 8, .type = TSDB_DATA_TYPE_UBIGINT}, {.name = "query_time", .bytes = 8, .type = TSDB_DATA_TYPE_UBIGINT}, - {.name = "show_query", .bytes = 8, .type = TSDB_DATA_TYPE_UBIGINT}, + {.name = "slow_query", .bytes = 8, .type = TSDB_DATA_TYPE_UBIGINT}, {.name = "total_req", .bytes = 8, .type = TSDB_DATA_TYPE_UBIGINT}, {.name = "current_req", .bytes = 8, .type = TSDB_DATA_TYPE_UBIGINT}, {.name = "last_access", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index 0711f7e06d..05d0fe66c3 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -66,21 +66,21 @@ static int32_t convertToRetrieveType(char *name, int32_t len) { type = TSDB_MGMT_TABLE_SNODE; } else if (strncasecmp(name, TSDB_INS_TABLE_CLUSTER, len) == 0) { type = TSDB_MGMT_TABLE_CLUSTER; - } else if (strncasecmp(name, TSDB_INS_TABLE_USER_DATABASES, len) == 0) { + } else if (strncasecmp(name, TSDB_INS_TABLE_DATABASES, len) == 0) { type = TSDB_MGMT_TABLE_DB; - } else if (strncasecmp(name, TSDB_INS_TABLE_USER_FUNCTIONS, len) == 0) { + } else if (strncasecmp(name, TSDB_INS_TABLE_FUNCTIONS, len) == 0) { type = TSDB_MGMT_TABLE_FUNC; - } else if (strncasecmp(name, TSDB_INS_TABLE_USER_INDEXES, len) == 0) { + } else if (strncasecmp(name, TSDB_INS_TABLE_INDEXES, len) == 0) { type = TSDB_MGMT_TABLE_INDEX; - } else if (strncasecmp(name, TSDB_INS_TABLE_USER_STABLES, len) == 0) { + } else if (strncasecmp(name, TSDB_INS_TABLE_STABLES, len) == 0) { type = TSDB_MGMT_TABLE_STB; - } else if (strncasecmp(name, TSDB_INS_TABLE_USER_TABLES, len) == 0) { + } else if (strncasecmp(name, TSDB_INS_TABLE_TABLES, len) == 0) { type = TSDB_MGMT_TABLE_TABLE; - } else if (strncasecmp(name, TSDB_INS_TABLE_USER_TAGS, len) == 0) { + } else if (strncasecmp(name, TSDB_INS_TABLE_TAGS, len) == 0) { type = TSDB_MGMT_TABLE_TAG; - } else if (strncasecmp(name, TSDB_INS_TABLE_USER_TABLE_DISTRIBUTED, len) == 0) { + } else if (strncasecmp(name, TSDB_INS_TABLE_TABLE_DISTRIBUTED, len) == 0) { // type = TSDB_MGMT_TABLE_DIST; - } else if (strncasecmp(name, TSDB_INS_TABLE_USER_USERS, len) == 0) { + } else if (strncasecmp(name, TSDB_INS_TABLE_USERS, len) == 0) { type = TSDB_MGMT_TABLE_USER; } else if (strncasecmp(name, TSDB_INS_TABLE_LICENCES, len) == 0) { type = TSDB_MGMT_TABLE_GRANTS; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index e38034f4aa..b0d08f72e7 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1622,8 +1622,8 @@ static void destroySysScanOperator(void* param, int32_t numOfOutput) { blockDataDestroy(pInfo->pRes); const char* name = tNameGetTableName(&pInfo->name); - if (strncasecmp(name, TSDB_INS_TABLE_USER_TABLES, TSDB_TABLE_FNAME_LEN) == 0 || - strncasecmp(name, TSDB_INS_TABLE_USER_TAGS, TSDB_TABLE_FNAME_LEN) == 0 || pInfo->pCur != NULL) { + if (strncasecmp(name, TSDB_INS_TABLE_TABLES, TSDB_TABLE_FNAME_LEN) == 0 || + strncasecmp(name, TSDB_INS_TABLE_TAGS, TSDB_TABLE_FNAME_LEN) == 0 || pInfo->pCur != NULL) { metaCloseTbCursor(pInfo->pCur); pInfo->pCur = NULL; } @@ -1635,7 +1635,7 @@ static void destroySysScanOperator(void* param, int32_t numOfOutput) { } static int32_t getSysTableDbNameColId(const char* pTable) { - // if (0 == strcmp(TSDB_INS_TABLE_USER_INDEXES, pTable)) { + // if (0 == strcmp(TSDB_INS_TABLE_INDEXES, pTable)) { // return 1; // } return TSDB_INS_USER_STABLES_DBNAME_COLID; @@ -1851,7 +1851,7 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { tNameGetDbName(&sn, varDataVal(dbname)); varDataSetLen(dbname, strlen(varDataVal(dbname))); - SSDataBlock* p = buildInfoSchemaTableMetaBlock(TSDB_INS_TABLE_USER_TAGS); + SSDataBlock* p = buildInfoSchemaTableMetaBlock(TSDB_INS_TABLE_TAGS); blockDataEnsureCapacity(p, pOperator->resultInfo.capacity); int32_t ret = 0; @@ -2022,7 +2022,7 @@ static SSDataBlock* sysTableScanUserTables(SOperatorInfo* pOperator) { tNameGetDbName(&sn, varDataVal(dbname)); varDataSetLen(dbname, strlen(varDataVal(dbname))); - SSDataBlock* p = buildInfoSchemaTableMetaBlock(TSDB_INS_TABLE_USER_TABLES); + SSDataBlock* p = buildInfoSchemaTableMetaBlock(TSDB_INS_TABLE_TABLES); blockDataEnsureCapacity(p, pOperator->resultInfo.capacity); char n[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; @@ -2189,11 +2189,11 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator) { sprintf(pInfo->req.db, "%d.%s", pInfo->accountId, dbName); } - if (strncasecmp(name, TSDB_INS_TABLE_USER_TABLES, TSDB_TABLE_FNAME_LEN) == 0) { + if (strncasecmp(name, TSDB_INS_TABLE_TABLES, TSDB_TABLE_FNAME_LEN) == 0) { return sysTableScanUserTables(pOperator); - } else if (strncasecmp(name, TSDB_INS_TABLE_USER_TAGS, TSDB_TABLE_FNAME_LEN) == 0) { + } else if (strncasecmp(name, TSDB_INS_TABLE_TAGS, TSDB_TABLE_FNAME_LEN) == 0) { return sysTableScanUserTags(pOperator); - } else if (strncasecmp(name, TSDB_INS_TABLE_USER_STABLES, TSDB_TABLE_FNAME_LEN) == 0 && + } else if (strncasecmp(name, TSDB_INS_TABLE_STABLES, TSDB_TABLE_FNAME_LEN) == 0 && IS_SYS_DBNAME(pInfo->req.db)) { return sysTableScanUserSTables(pOperator); } else { // load the meta from mnode of the given epset @@ -2269,7 +2269,7 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator) { } int32_t buildSysDbTableInfo(const SSysTableScanInfo* pInfo, int32_t capacity) { - SSDataBlock* p = buildInfoSchemaTableMetaBlock(TSDB_INS_TABLE_USER_TABLES); + SSDataBlock* p = buildInfoSchemaTableMetaBlock(TSDB_INS_TABLE_TABLES); blockDataEnsureCapacity(p, capacity); size_t size = 0; @@ -2358,8 +2358,8 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScan tNameAssign(&pInfo->name, &pScanNode->tableName); const char* name = tNameGetTableName(&pInfo->name); - if (strncasecmp(name, TSDB_INS_TABLE_USER_TABLES, TSDB_TABLE_FNAME_LEN) == 0 || - strncasecmp(name, TSDB_INS_TABLE_USER_TAGS, TSDB_TABLE_FNAME_LEN) == 0) { + if (strncasecmp(name, TSDB_INS_TABLE_TABLES, TSDB_TABLE_FNAME_LEN) == 0 || + strncasecmp(name, TSDB_INS_TABLE_TAGS, TSDB_TABLE_FNAME_LEN) == 0) { pInfo->readHandle = *(SReadHandle*)readHandle; blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity); } else { diff --git a/source/libs/parser/src/parAstParser.c b/source/libs/parser/src/parAstParser.c index a93091707a..01d8d1f306 100644 --- a/source/libs/parser/src/parAstParser.c +++ b/source/libs/parser/src/parAstParser.c @@ -334,22 +334,22 @@ static int32_t collectMetaKeyFromShowBnodes(SCollectMetaKeyCxt* pCxt, SShowStmt* } static int32_t collectMetaKeyFromShowDatabases(SCollectMetaKeyCxt* pCxt, SShowStmt* pStmt) { - return reserveTableMetaInCache(pCxt->pParseCxt->acctId, TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_USER_DATABASES, + return reserveTableMetaInCache(pCxt->pParseCxt->acctId, TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_DATABASES, pCxt->pMetaCache); } static int32_t collectMetaKeyFromShowFunctions(SCollectMetaKeyCxt* pCxt, SShowStmt* pStmt) { - return reserveTableMetaInCache(pCxt->pParseCxt->acctId, TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_USER_FUNCTIONS, + return reserveTableMetaInCache(pCxt->pParseCxt->acctId, TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_FUNCTIONS, pCxt->pMetaCache); } static int32_t collectMetaKeyFromShowIndexes(SCollectMetaKeyCxt* pCxt, SShowStmt* pStmt) { - return reserveTableMetaInCache(pCxt->pParseCxt->acctId, TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_USER_INDEXES, + return reserveTableMetaInCache(pCxt->pParseCxt->acctId, TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_INDEXES, pCxt->pMetaCache); } static int32_t collectMetaKeyFromShowStables(SCollectMetaKeyCxt* pCxt, SShowStmt* pStmt) { - return reserveTableMetaInCache(pCxt->pParseCxt->acctId, TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_USER_STABLES, + return reserveTableMetaInCache(pCxt->pParseCxt->acctId, TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_STABLES, pCxt->pMetaCache); } @@ -360,7 +360,7 @@ static int32_t collectMetaKeyFromShowStreams(SCollectMetaKeyCxt* pCxt, SShowStmt static int32_t collectMetaKeyFromShowTables(SCollectMetaKeyCxt* pCxt, SShowStmt* pStmt) { int32_t code = reserveTableMetaInCache(pCxt->pParseCxt->acctId, TSDB_INFORMATION_SCHEMA_DB, - TSDB_INS_TABLE_USER_TABLES, pCxt->pMetaCache); + TSDB_INS_TABLE_TABLES, pCxt->pMetaCache); if (TSDB_CODE_SUCCESS == code) { if (NULL != pStmt->pDbName) { code = reserveDbVgInfoInCache(pCxt->pParseCxt->acctId, ((SValueNode*)pStmt->pDbName)->literal, pCxt->pMetaCache); @@ -372,7 +372,7 @@ static int32_t collectMetaKeyFromShowTables(SCollectMetaKeyCxt* pCxt, SShowStmt* } static int32_t collectMetaKeyFromShowTags(SCollectMetaKeyCxt* pCxt, SShowStmt* pStmt) { - int32_t code = reserveTableMetaInCache(pCxt->pParseCxt->acctId, TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_USER_TAGS, + int32_t code = reserveTableMetaInCache(pCxt->pParseCxt->acctId, TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_TAGS, pCxt->pMetaCache); if (TSDB_CODE_SUCCESS == code) { if (NULL != pStmt->pDbName) { @@ -385,7 +385,7 @@ static int32_t collectMetaKeyFromShowTags(SCollectMetaKeyCxt* pCxt, SShowStmt* p } static int32_t collectMetaKeyFromShowUsers(SCollectMetaKeyCxt* pCxt, SShowStmt* pStmt) { - return reserveTableMetaInCache(pCxt->pParseCxt->acctId, TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_USER_USERS, + return reserveTableMetaInCache(pCxt->pParseCxt->acctId, TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_USERS, pCxt->pMetaCache); } diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index 7066907250..97910249eb 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -21,6 +21,7 @@ #include "tglobal.h" #include "ttime.h" #include "ttypes.h" +#include "query.h" #define NEXT_TOKEN(pSql, sToken) \ do { \ @@ -1488,6 +1489,8 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) { return buildSyntaxErrMsg(&pCxt->msg, "keyword VALUES or FILE is expected", sToken.z); } + qDebug("0x%" PRIx64 " insert input rows: %d", pCxt->pComCxt->requestId, pCxt->totalNum); + if (TSDB_QUERY_HAS_TYPE(pCxt->pOutput->insertType, TSDB_QUERY_TYPE_STMT_INSERT)) { SParsedDataColInfo* tags = taosMemoryMalloc(sizeof(pCxt->tags)); if (NULL == tags) { diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 5d1889194e..d9041aafd5 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1731,9 +1731,9 @@ static int32_t dnodeToVgroupsInfo(SArray* pDnodes, SVgroupsInfo** pVgsInfo) { } static bool sysTableFromVnode(const char* pTable) { - return (0 == strcmp(pTable, TSDB_INS_TABLE_USER_TABLES)) || - (0 == strcmp(pTable, TSDB_INS_TABLE_USER_TABLE_DISTRIBUTED) || - (0 == strcmp(pTable, TSDB_INS_TABLE_USER_TAGS))); + return (0 == strcmp(pTable, TSDB_INS_TABLE_TABLES)) || + (0 == strcmp(pTable, TSDB_INS_TABLE_TABLE_DISTRIBUTED) || + (0 == strcmp(pTable, TSDB_INS_TABLE_TAGS))); } static bool sysTableFromDnode(const char* pTable) { return 0 == strcmp(pTable, TSDB_INS_TABLE_DNODE_VARIABLES); } @@ -1749,7 +1749,7 @@ static int32_t setVnodeSysTableVgroupList(STranslateContext* pCxt, SName* pName, code = getDBVgInfoImpl(pCxt, pName, &vgroupList); } - if (TSDB_CODE_SUCCESS == code && 0 == strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_USER_TABLES)) { + if (TSDB_CODE_SUCCESS == code && 0 == strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_TABLES)) { code = addMnodeToVgroupList(&pCxt->pParseCxt->mgmtEpSet, &vgroupList); } @@ -1837,9 +1837,9 @@ static bool joinTableIsSingleTable(SJoinTableNode* pJoinTable) { static bool isSingleTable(SRealTableNode* pRealTable) { int8_t tableType = pRealTable->pMeta->tableType; if (TSDB_SYSTEM_TABLE == tableType) { - return 0 != strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_USER_TABLES) && - 0 != strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_USER_TABLE_DISTRIBUTED) && - 0 != strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_USER_TAGS); + return 0 != strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_TABLES) && + 0 != strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_TABLE_DISTRIBUTED) && + 0 != strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_TAGS); } return (TSDB_CHILD_TABLE == tableType || TSDB_NORMAL_TABLE == tableType); } @@ -5229,15 +5229,15 @@ static const char* getSysDbName(ENodeType type) { static const char* getSysTableName(ENodeType type) { switch (type) { case QUERY_NODE_SHOW_DATABASES_STMT: - return TSDB_INS_TABLE_USER_DATABASES; + return TSDB_INS_TABLE_DATABASES; case QUERY_NODE_SHOW_TABLES_STMT: - return TSDB_INS_TABLE_USER_TABLES; + return TSDB_INS_TABLE_TABLES; case QUERY_NODE_SHOW_TAGS_STMT: - return TSDB_INS_TABLE_USER_TAGS; + return TSDB_INS_TABLE_TAGS; case QUERY_NODE_SHOW_STABLES_STMT: - return TSDB_INS_TABLE_USER_STABLES; + return TSDB_INS_TABLE_STABLES; case QUERY_NODE_SHOW_USERS_STMT: - return TSDB_INS_TABLE_USER_USERS; + return TSDB_INS_TABLE_USERS; case QUERY_NODE_SHOW_DNODES_STMT: return TSDB_INS_TABLE_DNODES; case QUERY_NODE_SHOW_VGROUPS_STMT: @@ -5249,9 +5249,9 @@ static const char* getSysTableName(ENodeType type) { case QUERY_NODE_SHOW_QNODES_STMT: return TSDB_INS_TABLE_QNODES; case QUERY_NODE_SHOW_FUNCTIONS_STMT: - return TSDB_INS_TABLE_USER_FUNCTIONS; + return TSDB_INS_TABLE_FUNCTIONS; case QUERY_NODE_SHOW_INDEXES_STMT: - return TSDB_INS_TABLE_USER_INDEXES; + return TSDB_INS_TABLE_INDEXES; case QUERY_NODE_SHOW_STREAMS_STMT: return TSDB_PERFS_TABLE_STREAMS; case QUERY_NODE_SHOW_BNODES_STMT: diff --git a/source/libs/parser/test/mockCatalog.cpp b/source/libs/parser/test/mockCatalog.cpp index f064566d5f..6834f9ccca 100644 --- a/source/libs/parser/test/mockCatalog.cpp +++ b/source/libs/parser/test/mockCatalog.cpp @@ -58,46 +58,46 @@ void generateInformationSchema(MockCatalogService* mcs) { } { ITableBuilder& builder = - mcs->createTableBuilder(TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_USER_DATABASES, TSDB_SYSTEM_TABLE, 1) + mcs->createTableBuilder(TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_DATABASES, TSDB_SYSTEM_TABLE, 1) .addColumn("name", TSDB_DATA_TYPE_BINARY, TSDB_DB_NAME_LEN); builder.done(); } { ITableBuilder& builder = - mcs->createTableBuilder(TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_USER_FUNCTIONS, TSDB_SYSTEM_TABLE, 1) + mcs->createTableBuilder(TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_FUNCTIONS, TSDB_SYSTEM_TABLE, 1) .addColumn("name", TSDB_DATA_TYPE_BINARY, TSDB_FUNC_NAME_LEN); builder.done(); } { ITableBuilder& builder = - mcs->createTableBuilder(TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_USER_INDEXES, TSDB_SYSTEM_TABLE, 2) + mcs->createTableBuilder(TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_INDEXES, TSDB_SYSTEM_TABLE, 2) .addColumn("db_name", TSDB_DATA_TYPE_BINARY, TSDB_DB_NAME_LEN) .addColumn("table_name", TSDB_DATA_TYPE_BINARY, TSDB_TABLE_NAME_LEN); builder.done(); } { ITableBuilder& builder = - mcs->createTableBuilder(TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_USER_STABLES, TSDB_SYSTEM_TABLE, 2) + mcs->createTableBuilder(TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_STABLES, TSDB_SYSTEM_TABLE, 2) .addColumn("db_name", TSDB_DATA_TYPE_BINARY, TSDB_DB_NAME_LEN) .addColumn("stable_name", TSDB_DATA_TYPE_BINARY, TSDB_TABLE_NAME_LEN); builder.done(); } { ITableBuilder& builder = - mcs->createTableBuilder(TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_USER_TABLES, TSDB_SYSTEM_TABLE, 2) + mcs->createTableBuilder(TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_TABLES, TSDB_SYSTEM_TABLE, 2) .addColumn("db_name", TSDB_DATA_TYPE_BINARY, TSDB_DB_NAME_LEN) .addColumn("table_name", TSDB_DATA_TYPE_BINARY, TSDB_TABLE_NAME_LEN); builder.done(); } { ITableBuilder& builder = - mcs->createTableBuilder(TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_USER_TABLE_DISTRIBUTED, TSDB_SYSTEM_TABLE, 1) + mcs->createTableBuilder(TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_TABLE_DISTRIBUTED, TSDB_SYSTEM_TABLE, 1) .addColumn("db_name", TSDB_DATA_TYPE_BINARY, TSDB_DB_NAME_LEN); builder.done(); } { ITableBuilder& builder = - mcs->createTableBuilder(TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_USER_USERS, TSDB_SYSTEM_TABLE, 1) + mcs->createTableBuilder(TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_USERS, TSDB_SYSTEM_TABLE, 1) .addColumn("user_name", TSDB_DATA_TYPE_BINARY, TSDB_USER_LEN); builder.done(); } diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 3771586b34..6d70aa60d5 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -568,9 +568,9 @@ static int32_t createSystemTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pScan->showRewrite = pScanLogicNode->showRewrite; pScan->accountId = pCxt->pPlanCxt->acctId; - if (0 == strcmp(pScanLogicNode->tableName.tname, TSDB_INS_TABLE_USER_TABLES) || - 0 == strcmp(pScanLogicNode->tableName.tname, TSDB_INS_TABLE_USER_TABLE_DISTRIBUTED) || - 0 == strcmp(pScanLogicNode->tableName.tname, TSDB_INS_TABLE_USER_TAGS)) { + if (0 == strcmp(pScanLogicNode->tableName.tname, TSDB_INS_TABLE_TABLES) || + 0 == strcmp(pScanLogicNode->tableName.tname, TSDB_INS_TABLE_TABLE_DISTRIBUTED) || + 0 == strcmp(pScanLogicNode->tableName.tname, TSDB_INS_TABLE_TAGS)) { vgroupInfoToNodeAddr(pScanLogicNode->pVgroupList->vgroups, &pSubplan->execNode); } else { pSubplan->execNode.nodeId = MNODE_HANDLE; diff --git a/source/libs/scheduler/src/schJob.c b/source/libs/scheduler/src/schJob.c index 82146c3741..98501427ab 100644 --- a/source/libs/scheduler/src/schJob.c +++ b/source/libs/scheduler/src/schJob.c @@ -380,6 +380,7 @@ int32_t schDumpJobExecRes(SSchJob* pJob, SExecResult* pRes) { pRes->numOfRows = pJob->resNumOfRows; pRes->res = pJob->execRes.res; pRes->msgType = pJob->execRes.msgType; + pRes->numOfBytes = pJob->execRes.numOfBytes; pJob->execRes.res = NULL; SCH_JOB_DLOG("execRes dumped, code: %s", tstrerror(pRes->code)); diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c index 544ecce175..8c9003a9b2 100644 --- a/source/libs/scheduler/src/schRemote.c +++ b/source/libs/scheduler/src/schRemote.c @@ -213,6 +213,7 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SDa pJob->execRes.res = rsp; pJob->execRes.msgType = TDMT_VND_SUBMIT; } + pJob->execRes.numOfBytes += pTask->msgLen; SCH_UNLOCK(SCH_WRITE, &pJob->resLock); } diff --git a/tests/script/tsim/query/explain.sim b/tests/script/tsim/query/explain.sim index fcf91c45da..7cc1458657 100644 --- a/tests/script/tsim/query/explain.sim +++ b/tests/script/tsim/query/explain.sim @@ -35,7 +35,7 @@ sql explain select * from st1 where -2; sql explain select ts from tb1; sql explain select * from st1; sql explain select * from st1 order by ts; -sql explain select * from information_schema.user_stables; +sql explain select * from information_schema.ins_stables; sql explain select count(*),sum(f1) from tb1; sql explain select count(*),sum(f1) from st1; sql explain select count(*),sum(f1) from st1 group by f1; @@ -46,7 +46,7 @@ sql explain verbose true select * from st1 where -2; sql explain verbose true select ts from tb1 where f1 > 0; sql explain verbose true select * from st1 where f1 > 0 and ts > '2020-10-31 00:00:00' and ts < '2021-10-31 00:00:00'; sql explain verbose true select count(*) from st1 partition by tbname slimit 1 soffset 2 limit 2 offset 1; -sql explain verbose true select * from information_schema.user_stables where db_name='db2'; +sql explain verbose true select * from information_schema.ins_stables where db_name='db2'; print ======== step4 sql explain analyze select ts from st1 where -2; @@ -54,7 +54,7 @@ sql explain analyze select ts from tb1; sql explain analyze select ts from st1; sql explain analyze select ts from st1; sql explain analyze select ts from st1 order by ts; -sql explain analyze select * from information_schema.user_stables; +sql explain analyze select * from information_schema.ins_stables; sql explain analyze select count(*),sum(f1) from tb1; sql explain analyze select count(*),sum(f1) from st1; sql explain analyze select count(*),sum(f1) from st1 group by f1; @@ -65,14 +65,14 @@ sql explain analyze verbose true select ts from tb1; sql explain analyze verbose true select ts from st1; sql explain analyze verbose true select ts from st1; sql explain analyze verbose true select ts from st1 order by ts; -sql explain analyze verbose true select * from information_schema.user_stables; +sql explain analyze verbose true select * from information_schema.ins_stables; sql explain analyze verbose true select count(*),sum(f1) from tb1; sql explain analyze verbose true select count(*),sum(f1) from st1; sql explain analyze verbose true select count(*),sum(f1) from st1 group by f1; #sql explain analyze verbose true select count(f1) from tb1 interval(10s, 2s) sliding(3s) fill(prev); sql explain analyze verbose true select ts from tb1 where f1 > 0; sql explain analyze verbose true select f1 from st1 where f1 > 0 and ts > '2020-10-31 00:00:00' and ts < '2021-10-31 00:00:00'; -sql explain analyze verbose true select * from information_schema.user_stables where db_name='db2'; +sql explain analyze verbose true select * from information_schema.ins_stables where db_name='db2'; sql explain analyze verbose true select * from (select min(f1),count(*) a from st1 where f1 > 0) where a < 0; #not pass case diff --git a/tests/script/tsim/show/basic.sim b/tests/script/tsim/show/basic.sim index c4af7f3f3c..cc2b847d1a 100644 --- a/tests/script/tsim/show/basic.sim +++ b/tests/script/tsim/show/basic.sim @@ -77,37 +77,37 @@ if $rows != 3 then endi print =============== run select * from information_schema.xxxx -sql select * from information_schema.`dnodes` +sql select * from information_schema.ins_dnodes if $rows != 2 then return -1 endi -sql select * from information_schema.`mnodes` +sql select * from information_schema.ins_mnodes if $rows != 1 then return -1 endi -#sql select * from information_schema.`modules` -#sql select * from information_schema.`qnodes` -sql select * from information_schema.user_databases +#sql select * from information_schema.ins_modules +#sql select * from information_schema.ins_qnodes +sql select * from information_schema.ins_databases if $rows != 3 then return -1 endi -#sql select * from information_schema.user_functions -#sql select * from information_schema.user_indexes -sql select * from information_schema.user_stables +#sql select * from information_schema.ins_functions +#sql select * from information_schema.ins_indexes +sql select * from information_schema.ins_stables if $rows != 1 then return -1 endi -#sql select * from information_schema.`streams` -sql select * from information_schema.user_tables +#sql select * from information_schema.ins_streams +sql select * from information_schema.ins_tables if $rows <= 0 then return -1 endi -#sql select * from information_schema.user_table_distributed -sql select * from information_schema.user_users +#sql select * from information_schema.ins_table_distributed +sql select * from information_schema.ins_users if $rows != 1 then return -1 endi -sql select * from information_schema.`vgroups` +sql select * from information_schema.ins_vgroups if $rows != 3 then return -1 endi @@ -175,37 +175,37 @@ if $rows != 3 then endi print =============== run select * from information_schema.xxxx -sql select * from information_schema.`dnodes` +sql select * from information_schema.ins_dnodes if $rows != 2 then return -1 endi -sql select * from information_schema.`mnodes` +sql select * from information_schema.ins_mnodes if $rows != 1 then return -1 endi -#sql select * from information_schema.`modules` -#sql select * from information_schema.`qnodes` -sql select * from information_schema.user_databases +#sql select * from information_schema.ins_modules +#sql select * from information_schema.ins_qnodes +sql select * from information_schema.ins_databases if $rows != 3 then return -1 endi -#sql select * from information_schema.user_functions -#sql select * from information_schema.user_indexes -sql select * from information_schema.user_stables +#sql select * from information_schema.ins_functions +#sql select * from information_schema.ins_indexes +sql select * from information_schema.ins_stables if $rows != 1 then return -1 endi -#sql select * from performance_schema.`streams` -sql select * from information_schema.user_tables +#sql select * from performance_schema.perf_streams +sql select * from information_schema.ins_tables if $rows <= 0 then return -1 endi -#sql select * from information_schema.user_table_distributed -sql select * from information_schema.user_users +#sql select * from information_schema.ins_table_distributed +sql select * from information_schema.ins_users if $rows != 1 then return -1 endi -sql select * from information_schema.`vgroups` +sql select * from information_schema.ins_vgroups if $rows != 3 then return -1 endi diff --git a/tests/script/tsim/tmq/basic2Of2ConsOverlap.sim b/tests/script/tsim/tmq/basic2Of2ConsOverlap.sim index d9c6b195d2..dda5e0059e 100644 --- a/tests/script/tsim/tmq/basic2Of2ConsOverlap.sim +++ b/tests/script/tsim/tmq/basic2Of2ConsOverlap.sim @@ -342,12 +342,12 @@ endi return -1 check_ok_3: -sql select * from performance_schema.`consumers` +sql select * from performance_schema.perf_consumers if $rows != 0 then return -1 endi -#sql select * from performance_schema.`subscriptions` +#sql select * from performance_schema.perf_subscriptions #if $rows != 0 then # return -1 #endi diff --git a/tests/script/tsim/tmq/snapshot1.sim b/tests/script/tsim/tmq/snapshot1.sim index 58541b725d..e586719db2 100644 --- a/tests/script/tsim/tmq/snapshot1.sim +++ b/tests/script/tsim/tmq/snapshot1.sim @@ -294,12 +294,12 @@ endi return -1 check_ok_3: -sql select * from performance_schema.`consumers` +sql select * from performance_schema.perf_consumers if $rows != 0 then return -1 endi -#sql select * from performance_schema.`subscriptions` +#sql select * from performance_schema.perf_subscriptions #if $rows != 0 then # return -1 #endi diff --git a/tests/script/tsim/valgrind/checkError1.sim b/tests/script/tsim/valgrind/checkError1.sim index 059808e4be..f06a26aec6 100644 --- a/tests/script/tsim/valgrind/checkError1.sim +++ b/tests/script/tsim/valgrind/checkError1.sim @@ -84,37 +84,37 @@ if $rows != 1 then endi print =============== run select * from information_schema.xxxx -sql select * from information_schema.`dnodes` +sql select * from information_schema.ins_dnodes if $rows != 2 then return -1 endi -sql select * from information_schema.`mnodes` +sql select * from information_schema.ins_mnodes if $rows != 1 then return -1 endi -sql select * from information_schema.user_databases +sql select * from information_schema.ins_databases if $rows != 3 then return -1 endi -sql select * from information_schema.user_stables +sql select * from information_schema.ins_stables if $rows != 1 then return -1 endi -sql select * from information_schema.user_tables +sql select * from information_schema.ins_tables if $rows <= 0 then return -1 endi -sql select * from information_schema.user_users +sql select * from information_schema.ins_users if $rows != 1 then return -1 endi -sql select * from information_schema.`vgroups` +sql select * from information_schema.ins_vgroups if $rows != 3 then return -1 endi diff --git a/tests/script/tsim/valgrind/checkError6.sim b/tests/script/tsim/valgrind/checkError6.sim index eccf5b31f3..5f67dddee1 100644 --- a/tests/script/tsim/valgrind/checkError6.sim +++ b/tests/script/tsim/valgrind/checkError6.sim @@ -109,12 +109,12 @@ print =============== step5: explain sql explain analyze select ts from stb where -2; sql explain analyze select ts from tb1; sql explain analyze select ts from stb order by ts; -sql explain analyze select * from information_schema.user_stables; +sql explain analyze select * from information_schema.ins_stables; sql explain analyze select count(*),sum(tbcol) from tb1; sql explain analyze select count(*),sum(tbcol) from stb; sql explain analyze select count(*),sum(tbcol) from stb group by tbcol; -sql explain analyze select * from information_schema.user_stables; -sql explain analyze verbose true select * from information_schema.user_stables where db_name='db2'; +sql explain analyze select * from information_schema.ins_stables; +sql explain analyze verbose true select * from information_schema.ins_stables where db_name='db2'; sql explain analyze verbose true select count(tbcol), avg(tbcol), max(tbcol), min(tbcol), count(tbcol) from stb where ts <= 1601481840000 and ts >= 1601481800000 partition by tgcol interval(1m) fill(value, 0) sql explain select count(tbcol), avg(tbcol), max(tbcol), min(tbcol), count(tbcol) from stb where ts <= 1601481840000 and ts >= 1601481800000 partition by tgcol interval(1m) fill(value, 0) From df872728196b6dc7dc8a6526e3a0a3c8ecb55662 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 2 Aug 2022 08:36:39 +0800 Subject: [PATCH 02/18] fix: fix sml crash issue --- source/client/src/clientImpl.c | 8 +++++--- source/client/src/clientSml.c | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index d2c0a70731..0c4dc1705c 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -874,9 +874,11 @@ void schedulerExecCb(SExecResult* pResult, void* param, int32_t code) { SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, bool keepQuery, void** res) { int32_t code = 0; - pRequest->stmtType = pQuery->pRoot->type; - - if (!pRequest->inRetry) { + if (pQuery->pRoot) { + pRequest->stmtType = pQuery->pRoot->type; + } + + if (pQuery->pRoot && !pRequest->inRetry) { STscObj *pTscObj = pRequest->pTscObj; SAppClusterSummary *pActivity = &pTscObj->pAppInfo->summary; if (QUERY_NODE_VNODE_MODIF_STMT == pQuery->pRoot->type) { diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 56d09850fc..d860224827 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -420,7 +420,7 @@ static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SSmlSTableMeta *s } tSerializeSMCreateStbReq(pCmdMsg.pMsg, pCmdMsg.msgLen, &pReq); - SQuery pQuery; + SQuery pQuery = {0}; pQuery.execMode = QUERY_EXEC_MODE_RPC; pQuery.pCmdMsg = &pCmdMsg; pQuery.msgType = pQuery.pCmdMsg->msgType; From 31b3c61b9f26c612e91728dc70328a4fddda95dc Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 2 Aug 2022 09:53:04 +0800 Subject: [PATCH 03/18] test: adjust case --- tests/script/jenkins/basic.txt | 8 ++++---- tests/script/tsim/table/delete_writing.sim | 4 ---- tests/script/tsim/table/smallint.sim | 1 - 3 files changed, 4 insertions(+), 9 deletions(-) diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 910b99ace3..82e9c1e14d 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -183,7 +183,7 @@ # ---- qnode ./test.sh -f tsim/qnode/basic1.sim -# ---- snode +# ---- snode ---- # unsupport ./test.sh -f tsim/snode/basic1.sim # ---- bnode @@ -199,7 +199,7 @@ # ---- show ---- ./test.sh -f tsim/show/basic.sim -# ---- table +# ---- table ---- ./test.sh -f tsim/table/autocreate.sim ./test.sh -f tsim/table/basic1.sim ./test.sh -f tsim/table/basic2.sim @@ -248,7 +248,7 @@ ./test.sh -f tsim/stream/ignoreExpiredData.sim ./test.sh -f tsim/stream/sliding.sim -# ---- transaction +# ---- transaction ---- ./test.sh -f tsim/trans/lossdata1.sim ./test.sh -f tsim/trans/create_db.sim @@ -307,7 +307,7 @@ ./test.sh -f tsim/sma/rsmaCreateInsertQuery.sim ./test.sh -f tsim/sma/rsmaPersistenceRecovery.sim -# --- valgrind +# --- valgrind ---- ./test.sh -f tsim/valgrind/checkError1.sim ./test.sh -f tsim/valgrind/checkError2.sim ./test.sh -f tsim/valgrind/checkError3.sim diff --git a/tests/script/tsim/table/delete_writing.sim b/tests/script/tsim/table/delete_writing.sim index df7c7f47e8..2f11424e8a 100644 --- a/tests/script/tsim/table/delete_writing.sim +++ b/tests/script/tsim/table/delete_writing.sim @@ -3,10 +3,6 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/exec.sh -n dnode1 -s start sql connect -print ========= start dnodes -system sh/exec.sh -n dnode1 -s start -sql connect - sql create database db sql create table db.tb (ts timestamp, i int) sql insert into db.tb values(now, 1) diff --git a/tests/script/tsim/table/smallint.sim b/tests/script/tsim/table/smallint.sim index 87140c561a..a7547c7282 100644 --- a/tests/script/tsim/table/smallint.sim +++ b/tests/script/tsim/table/smallint.sim @@ -90,7 +90,6 @@ endi if $data01 != 2 then return -1 endi -return sql drop database $db sql show databases if $rows != 2 then From 596c86dd95da20ccc9f70d10962cfba590ea9010 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 2 Aug 2022 13:47:04 +0800 Subject: [PATCH 04/18] fix: fix case issue --- source/client/src/clientSml.c | 3 ++- source/dnode/mnode/impl/test/db/db.cpp | 14 +++++------ source/dnode/mnode/impl/test/func/func.cpp | 10 ++++---- .../dnode/mnode/impl/test/profile/profile.cpp | 4 ++-- source/dnode/mnode/impl/test/show/show.cpp | 4 ++-- source/dnode/mnode/impl/test/sma/sma.cpp | 4 ++-- source/dnode/mnode/impl/test/stb/stb.cpp | 22 ++++++++--------- source/dnode/mnode/impl/test/user/user.cpp | 24 +++++++++---------- source/libs/parser/test/parSelectTest.cpp | 2 +- source/libs/planner/test/planSysTbTest.cpp | 2 +- 10 files changed, 45 insertions(+), 44 deletions(-) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index d860224827..9f905a8352 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -420,7 +420,8 @@ static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SSmlSTableMeta *s } tSerializeSMCreateStbReq(pCmdMsg.pMsg, pCmdMsg.msgLen, &pReq); - SQuery pQuery = {0}; + SQuery pQuery; + memset(&pQuery, 0, sizeof(pQuery)); pQuery.execMode = QUERY_EXEC_MODE_RPC; pQuery.pCmdMsg = &pCmdMsg; pQuery.msgType = pQuery.pCmdMsg->msgType; diff --git a/source/dnode/mnode/impl/test/db/db.cpp b/source/dnode/mnode/impl/test/db/db.cpp index 06916b375c..94ba7f2968 100644 --- a/source/dnode/mnode/impl/test/db/db.cpp +++ b/source/dnode/mnode/impl/test/db/db.cpp @@ -26,7 +26,7 @@ class MndTestDb : public ::testing::Test { Testbase MndTestDb::test; TEST_F(MndTestDb, 01_ShowDb) { - test.SendShowReq(TSDB_MGMT_TABLE_DB, "user_databases", ""); + test.SendShowReq(TSDB_MGMT_TABLE_DB, "ins_databases", ""); EXPECT_EQ(test.GetShowRows(), 2); } @@ -64,10 +64,10 @@ TEST_F(MndTestDb, 02_Create_Alter_Drop_Db) { ASSERT_EQ(pRsp->code, 0); } - test.SendShowReq(TSDB_MGMT_TABLE_DB, "user_databases", ""); + test.SendShowReq(TSDB_MGMT_TABLE_DB, "ins_databases", ""); EXPECT_EQ(test.GetShowRows(), 3); - test.SendShowReq(TSDB_MGMT_TABLE_VGROUP, "vgroups", "1.d1"); + test.SendShowReq(TSDB_MGMT_TABLE_VGROUP, "ins_vgroups", "1.d1"); EXPECT_EQ(test.GetShowRows(), 2); { @@ -96,13 +96,13 @@ TEST_F(MndTestDb, 02_Create_Alter_Drop_Db) { ASSERT_EQ(pRsp->code, TSDB_CODE_OPS_NOT_SUPPORT); } - test.SendShowReq(TSDB_MGMT_TABLE_DB, "user_databases", ""); + test.SendShowReq(TSDB_MGMT_TABLE_DB, "ins_databases", ""); EXPECT_EQ(test.GetShowRows(), 3); // restart test.Restart(); - test.SendShowReq(TSDB_MGMT_TABLE_DB, "user_databases", ""); + test.SendShowReq(TSDB_MGMT_TABLE_DB, "ins_databases", ""); EXPECT_EQ(test.GetShowRows(), 3); { @@ -122,7 +122,7 @@ TEST_F(MndTestDb, 02_Create_Alter_Drop_Db) { EXPECT_STREQ(dropdbRsp.db, "1.d1"); } - test.SendShowReq(TSDB_MGMT_TABLE_DB, "user_databases", ""); + test.SendShowReq(TSDB_MGMT_TABLE_DB, "ins_databases", ""); EXPECT_EQ(test.GetShowRows(), 2); } @@ -160,7 +160,7 @@ TEST_F(MndTestDb, 03_Create_Use_Restart_Use_Db) { ASSERT_EQ(pRsp->code, 0); } - test.SendShowReq(TSDB_MGMT_TABLE_DB, "user_databases", ""); + test.SendShowReq(TSDB_MGMT_TABLE_DB, "ins_databases", ""); EXPECT_EQ(test.GetShowRows(), 3); uint64_t d2_uid = 0; diff --git a/source/dnode/mnode/impl/test/func/func.cpp b/source/dnode/mnode/impl/test/func/func.cpp index 2bebe7ef19..53029f5963 100644 --- a/source/dnode/mnode/impl/test/func/func.cpp +++ b/source/dnode/mnode/impl/test/func/func.cpp @@ -46,7 +46,7 @@ void MndTestFunc::SetBufSize(SCreateFuncReq* pReq, int32_t size) { } TEST_F(MndTestFunc, 01_Show_Func) { - test.SendShowReq(TSDB_MGMT_TABLE_FUNC, "user_functions", ""); + test.SendShowReq(TSDB_MGMT_TABLE_FUNC, "ins_functions", ""); EXPECT_EQ(test.GetShowRows(), 0); } @@ -159,7 +159,7 @@ TEST_F(MndTestFunc, 02_Create_Func) { } } - test.SendShowReq(TSDB_MGMT_TABLE_FUNC, "user_functions", ""); + test.SendShowReq(TSDB_MGMT_TABLE_FUNC, "ins_functions", ""); EXPECT_EQ(test.GetShowRows(), 1); } @@ -270,7 +270,7 @@ TEST_F(MndTestFunc, 03_Retrieve_Func) { ASSERT_NE(pRsp, nullptr); ASSERT_EQ(pRsp->code, 0); - test.SendShowReq(TSDB_MGMT_TABLE_FUNC, "user_functions", ""); + test.SendShowReq(TSDB_MGMT_TABLE_FUNC, "ins_functions", ""); EXPECT_EQ(test.GetShowRows(), 2); } @@ -435,13 +435,13 @@ TEST_F(MndTestFunc, 04_Drop_Func) { ASSERT_EQ(pRsp->code, 0); } - test.SendShowReq(TSDB_MGMT_TABLE_FUNC, "user_functions", ""); + test.SendShowReq(TSDB_MGMT_TABLE_FUNC, "ins_functions", ""); EXPECT_EQ(test.GetShowRows(), 1); // restart test.Restart(); - test.SendShowReq(TSDB_MGMT_TABLE_FUNC, "user_functions", ""); + test.SendShowReq(TSDB_MGMT_TABLE_FUNC, "ins_functions", ""); EXPECT_EQ(test.GetShowRows(), 1); } diff --git a/source/dnode/mnode/impl/test/profile/profile.cpp b/source/dnode/mnode/impl/test/profile/profile.cpp index e784a41d6f..2ae113e051 100644 --- a/source/dnode/mnode/impl/test/profile/profile.cpp +++ b/source/dnode/mnode/impl/test/profile/profile.cpp @@ -87,7 +87,7 @@ TEST_F(MndTestProfile, 02_ConnectMsg_InvalidDB) { } TEST_F(MndTestProfile, 03_ConnectMsg_Show) { - test.SendShowReq(TSDB_MGMT_TABLE_CONNS, "connections", ""); + test.SendShowReq(TSDB_MGMT_TABLE_CONNS, "perf_connections", ""); EXPECT_EQ(test.GetShowRows(), 1); } @@ -307,6 +307,6 @@ TEST_F(MndTestProfile, 08_KillQueryMsg_InvalidConn) { } TEST_F(MndTestProfile, 09_KillQueryMsg) { - test.SendShowReq(TSDB_MGMT_TABLE_QUERIES, "queries", ""); + test.SendShowReq(TSDB_MGMT_TABLE_QUERIES, "perf_queries", ""); EXPECT_EQ(test.GetShowRows(), 0); } diff --git a/source/dnode/mnode/impl/test/show/show.cpp b/source/dnode/mnode/impl/test/show/show.cpp index 0de8c9dca8..0ccefa7ca2 100644 --- a/source/dnode/mnode/impl/test/show/show.cpp +++ b/source/dnode/mnode/impl/test/show/show.cpp @@ -73,11 +73,11 @@ TEST_F(MndTestShow, 03_ShowMsg_Conn) { ASSERT_NE(pRsp, nullptr); ASSERT_EQ(pRsp->code, 0); - test.SendShowReq(TSDB_MGMT_TABLE_CONNS, "connections", ""); + test.SendShowReq(TSDB_MGMT_TABLE_CONNS, "perf_connections", ""); // EXPECT_EQ(test.GetShowRows(), 1); } TEST_F(MndTestShow, 04_ShowMsg_Cluster) { - test.SendShowReq(TSDB_MGMT_TABLE_CLUSTER, "cluster", ""); + test.SendShowReq(TSDB_MGMT_TABLE_CLUSTER, "ins_cluster", ""); EXPECT_EQ(test.GetShowRows(), 1); } diff --git a/source/dnode/mnode/impl/test/sma/sma.cpp b/source/dnode/mnode/impl/test/sma/sma.cpp index 06bb735f54..9a2372d904 100644 --- a/source/dnode/mnode/impl/test/sma/sma.cpp +++ b/source/dnode/mnode/impl/test/sma/sma.cpp @@ -259,7 +259,7 @@ TEST_F(MndTestSma, 02_Create_Show_Meta_Drop_Restart_BSma) { pReq = BuildCreateBSmaStbReq(stbname, &contLen); pRsp = test.SendReq(TDMT_MND_CREATE_STB, pReq, contLen); ASSERT_EQ(pRsp->code, 0); - test.SendShowReq(TSDB_MGMT_TABLE_STB, "user_stables", dbname); + test.SendShowReq(TSDB_MGMT_TABLE_STB, "ins_stables", dbname); EXPECT_EQ(test.GetShowRows(), 1); } @@ -275,7 +275,7 @@ TEST_F(MndTestSma, 02_Create_Show_Meta_Drop_Restart_BSma) { pReq = BuildDropStbReq(stbname, &contLen); pRsp = test.SendReq(TDMT_MND_DROP_STB, pReq, contLen); ASSERT_EQ(pRsp->code, 0); - test.SendShowReq(TSDB_MGMT_TABLE_STB, "user_stables", dbname); + test.SendShowReq(TSDB_MGMT_TABLE_STB, "ins_stables", dbname); EXPECT_EQ(test.GetShowRows(), 0); } diff --git a/source/dnode/mnode/impl/test/stb/stb.cpp b/source/dnode/mnode/impl/test/stb/stb.cpp index d275231c70..64982951c9 100644 --- a/source/dnode/mnode/impl/test/stb/stb.cpp +++ b/source/dnode/mnode/impl/test/stb/stb.cpp @@ -313,7 +313,7 @@ TEST_F(MndTestStb, 01_Create_Show_Meta_Drop_Restart_Stb) { } { - test.SendShowReq(TSDB_MGMT_TABLE_STB, "user_stables", dbname); + test.SendShowReq(TSDB_MGMT_TABLE_STB, "ins_stables", dbname); EXPECT_EQ(test.GetShowRows(), 1); } @@ -394,7 +394,7 @@ TEST_F(MndTestStb, 01_Create_Show_Meta_Drop_Restart_Stb) { test.Restart(); { - test.SendShowReq(TSDB_MGMT_TABLE_STB, "user_stables", dbname); + test.SendShowReq(TSDB_MGMT_TABLE_STB, "ins_stables", dbname); EXPECT_EQ(test.GetShowRows(), 1); } @@ -412,7 +412,7 @@ TEST_F(MndTestStb, 01_Create_Show_Meta_Drop_Restart_Stb) { } { - test.SendShowReq(TSDB_MGMT_TABLE_STB, "user_stables", dbname); + test.SendShowReq(TSDB_MGMT_TABLE_STB, "ins_stables", dbname); EXPECT_EQ(test.GetShowRows(), 0); } @@ -474,7 +474,7 @@ TEST_F(MndTestStb, 02_Alter_Stb_AddTag) { ASSERT_NE(pRsp, nullptr); ASSERT_EQ(pRsp->code, 0); - test.SendShowReq(TSDB_MGMT_TABLE_STB, "user_stables", dbname); + test.SendShowReq(TSDB_MGMT_TABLE_STB, "ins_stables", dbname); } { @@ -514,7 +514,7 @@ TEST_F(MndTestStb, 03_Alter_Stb_DropTag) { ASSERT_NE(pRsp, nullptr); ASSERT_EQ(pRsp->code, 0); - test.SendShowReq(TSDB_MGMT_TABLE_STB, "user_stables", dbname); + test.SendShowReq(TSDB_MGMT_TABLE_STB, "ins_stables", dbname); EXPECT_EQ(test.GetShowRows(), 1); } @@ -578,7 +578,7 @@ TEST_F(MndTestStb, 04_Alter_Stb_AlterTagName) { ASSERT_NE(pRsp, nullptr); ASSERT_EQ(pRsp->code, 0); - test.SendShowReq(TSDB_MGMT_TABLE_STB, "user_stables", dbname); + test.SendShowReq(TSDB_MGMT_TABLE_STB, "ins_stables", dbname); EXPECT_EQ(test.GetShowRows(), 1); } @@ -630,7 +630,7 @@ TEST_F(MndTestStb, 05_Alter_Stb_AlterTagBytes) { SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen); ASSERT_EQ(pRsp->code, 0); - test.SendShowReq(TSDB_MGMT_TABLE_STB, "user_stables", dbname); + test.SendShowReq(TSDB_MGMT_TABLE_STB, "ins_stables", dbname); EXPECT_EQ(test.GetShowRows(), 1); } @@ -691,7 +691,7 @@ TEST_F(MndTestStb, 06_Alter_Stb_AddColumn) { ASSERT_NE(pRsp, nullptr); ASSERT_EQ(pRsp->code, 0); - test.SendShowReq(TSDB_MGMT_TABLE_STB, "user_stables", dbname); + test.SendShowReq(TSDB_MGMT_TABLE_STB, "ins_stables", dbname); EXPECT_EQ(test.GetShowRows(), 1); } @@ -751,7 +751,7 @@ TEST_F(MndTestStb, 07_Alter_Stb_DropColumn) { ASSERT_NE(pRsp, nullptr); ASSERT_EQ(pRsp->code, 0); - test.SendShowReq(TSDB_MGMT_TABLE_STB, "user_stables", dbname); + test.SendShowReq(TSDB_MGMT_TABLE_STB, "ins_stables", dbname); EXPECT_EQ(test.GetShowRows(), 1); } @@ -809,7 +809,7 @@ TEST_F(MndTestStb, 08_Alter_Stb_AlterTagBytes) { SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen); ASSERT_EQ(pRsp->code, 0); - test.SendShowReq(TSDB_MGMT_TABLE_STB, "user_stables", dbname); + test.SendShowReq(TSDB_MGMT_TABLE_STB, "ins_stables", dbname); EXPECT_EQ(test.GetShowRows(), 1); } @@ -818,7 +818,7 @@ TEST_F(MndTestStb, 08_Alter_Stb_AlterTagBytes) { SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen); ASSERT_EQ(pRsp->code, TSDB_CODE_MND_COLUMN_NOT_EXIST); - test.SendShowReq(TSDB_MGMT_TABLE_STB, "user_stables", dbname); + test.SendShowReq(TSDB_MGMT_TABLE_STB, "ins_stables", dbname); EXPECT_EQ(test.GetShowRows(), 1); } diff --git a/source/dnode/mnode/impl/test/user/user.cpp b/source/dnode/mnode/impl/test/user/user.cpp index dd2bda6b19..eb8b1d18e9 100644 --- a/source/dnode/mnode/impl/test/user/user.cpp +++ b/source/dnode/mnode/impl/test/user/user.cpp @@ -26,7 +26,7 @@ class MndTestUser : public ::testing::Test { Testbase MndTestUser::test; TEST_F(MndTestUser, 01_Show_User) { - test.SendShowReq(TSDB_MGMT_TABLE_USER, "user_users", ""); + test.SendShowReq(TSDB_MGMT_TABLE_USER, "ins_users", ""); EXPECT_EQ(test.GetShowRows(), 1); } @@ -94,7 +94,7 @@ TEST_F(MndTestUser, 02_Create_User) { ASSERT_NE(pRsp, nullptr); ASSERT_EQ(pRsp->code, 0); - test.SendShowReq(TSDB_MGMT_TABLE_USER, "user_users", ""); + test.SendShowReq(TSDB_MGMT_TABLE_USER, "ins_users", ""); EXPECT_EQ(test.GetShowRows(), 2); } @@ -110,7 +110,7 @@ TEST_F(MndTestUser, 02_Create_User) { ASSERT_NE(pRsp, nullptr); ASSERT_EQ(pRsp->code, 0); - test.SendShowReq(TSDB_MGMT_TABLE_USER, "user_users", ""); + test.SendShowReq(TSDB_MGMT_TABLE_USER, "ins_users", ""); EXPECT_EQ(test.GetShowRows(), 1); } @@ -130,7 +130,7 @@ TEST_F(MndTestUser, 02_Create_User) { ASSERT_NE(pRsp, nullptr); ASSERT_EQ(pRsp->code, 0); - test.SendShowReq(TSDB_MGMT_TABLE_USER, "user_users", ""); + test.SendShowReq(TSDB_MGMT_TABLE_USER, "ins_users", ""); EXPECT_EQ(test.GetShowRows(), 2); } @@ -146,7 +146,7 @@ TEST_F(MndTestUser, 02_Create_User) { ASSERT_NE(pRsp, nullptr); ASSERT_EQ(pRsp->code, 0); - test.SendShowReq(TSDB_MGMT_TABLE_USER, "user_users", ""); + test.SendShowReq(TSDB_MGMT_TABLE_USER, "ins_users", ""); EXPECT_EQ(test.GetShowRows(), 1); } } @@ -168,7 +168,7 @@ TEST_F(MndTestUser, 03_Alter_User) { ASSERT_NE(pRsp, nullptr); ASSERT_EQ(pRsp->code, 0); - test.SendShowReq(TSDB_MGMT_TABLE_USER, "user_users", ""); + test.SendShowReq(TSDB_MGMT_TABLE_USER, "ins_users", ""); EXPECT_EQ(test.GetShowRows(), 2); } @@ -414,7 +414,7 @@ TEST_F(MndTestUser, 03_Alter_User) { ASSERT_NE(pRsp, nullptr); ASSERT_EQ(pRsp->code, 0); - test.SendShowReq(TSDB_MGMT_TABLE_USER, "user_users", ""); + test.SendShowReq(TSDB_MGMT_TABLE_USER, "ins_users", ""); EXPECT_EQ(test.GetShowRows(), 1); } } @@ -475,7 +475,7 @@ TEST_F(MndTestUser, 05_Drop_User) { ASSERT_EQ(pRsp->code, 0); } - test.SendShowReq(TSDB_MGMT_TABLE_USER, "user_users", ""); + test.SendShowReq(TSDB_MGMT_TABLE_USER, "ins_users", ""); EXPECT_EQ(test.GetShowRows(), 1); } @@ -512,7 +512,7 @@ TEST_F(MndTestUser, 06_Create_Drop_Alter_User) { ASSERT_EQ(pRsp->code, 0); } - test.SendShowReq(TSDB_MGMT_TABLE_USER, "user_users", ""); + test.SendShowReq(TSDB_MGMT_TABLE_USER, "ins_users", ""); EXPECT_EQ(test.GetShowRows(), 3); { @@ -530,7 +530,7 @@ TEST_F(MndTestUser, 06_Create_Drop_Alter_User) { ASSERT_EQ(pRsp->code, 0); } - test.SendShowReq(TSDB_MGMT_TABLE_USER, "user_users", ""); + test.SendShowReq(TSDB_MGMT_TABLE_USER, "ins_users", ""); EXPECT_EQ(test.GetShowRows(), 3); { SDropUserReq dropReq = {0}; @@ -545,13 +545,13 @@ TEST_F(MndTestUser, 06_Create_Drop_Alter_User) { ASSERT_EQ(pRsp->code, 0); } - test.SendShowReq(TSDB_MGMT_TABLE_USER, "user_users", ""); + test.SendShowReq(TSDB_MGMT_TABLE_USER, "ins_users", ""); EXPECT_EQ(test.GetShowRows(), 2); // restart test.Restart(); taosMsleep(1000); - test.SendShowReq(TSDB_MGMT_TABLE_USER, "user_users", ""); + test.SendShowReq(TSDB_MGMT_TABLE_USER, "ins_users", ""); EXPECT_EQ(test.GetShowRows(), 2); } diff --git a/source/libs/parser/test/parSelectTest.cpp b/source/libs/parser/test/parSelectTest.cpp index ebf8012cf7..02a82f7602 100644 --- a/source/libs/parser/test/parSelectTest.cpp +++ b/source/libs/parser/test/parSelectTest.cpp @@ -426,7 +426,7 @@ TEST_F(ParserSelectTest, setOperatorSemanticCheck) { TEST_F(ParserSelectTest, informationSchema) { useDb("root", "test"); - run("SELECT * FROM information_schema.user_databases WHERE name = 'information_schema'"); + run("SELECT * FROM information_schema.ins_databases WHERE name = 'information_schema'"); } TEST_F(ParserSelectTest, withoutFrom) { diff --git a/source/libs/planner/test/planSysTbTest.cpp b/source/libs/planner/test/planSysTbTest.cpp index e5c30030b3..921f86f09a 100644 --- a/source/libs/planner/test/planSysTbTest.cpp +++ b/source/libs/planner/test/planSysTbTest.cpp @@ -30,5 +30,5 @@ TEST_F(PlanSysTableTest, show) { TEST_F(PlanSysTableTest, informationSchema) { useDb("root", "information_schema"); - run("SELECT * FROM information_schema.user_databases WHERE name = 'information_schema'"); + run("SELECT * FROM information_schema.ins_databases WHERE name = 'information_schema'"); } From d0487750c03f847418e309e98c621514a348b6ae Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 2 Aug 2022 13:57:01 +0800 Subject: [PATCH 05/18] test: valgrind case --- tests/script/tsim/valgrind/basic3.sim | 30 +++++++++++++++++++--- tests/script/tsim/valgrind/checkError6.sim | 5 ++++ 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/tests/script/tsim/valgrind/basic3.sim b/tests/script/tsim/valgrind/basic3.sim index b9ed1641c8..d1877776b3 100644 --- a/tests/script/tsim/valgrind/basic3.sim +++ b/tests/script/tsim/valgrind/basic3.sim @@ -29,7 +29,7 @@ $rowNum = 10 print =============== step2: prepare data sql create database db vgroups 2 sql use db -sql create table if not exists stb (ts timestamp, tbcol int, tbcol2 float, tbcol3 double) tags (tgcol int unsigned) +sql create table if not exists stb (ts timestamp, tbcol int, tbcol2 float, tbcol3 double, tbcol4 binary(30), tbcol5 binary(30)) tags (tgcol int unsigned) $i = 0 while $i < $tbNum @@ -39,17 +39,39 @@ while $i < $tbNum while $x < $rowNum $cc = $x * 60000 $ms = 1601481600000 + $cc - sql insert into $tb values ($ms , $x , $x , $x ) + sql insert into $tb values ($ms , $x , $x , $x , "abcd1234=-+*" , "123456 0" ) $x = $x + 1 endw + + $cc = $x * 60000 + $ms = 1601481600000 + $cc + sql insert into $tb values ($ms , NULL , NULL , NULL , NULL , NULL ) $i = $i + 1 endw print =============== step3: tb -sql select count(tbcol), avg(tbcol), max(tbcol), min(tbcol), count(tbcol) from tb1 where ts <= 1601481840000 and ts >= 1601481800000 partition by tgcol interval(1m) fill(value, 0) +sql select tbcol5 - tbcol3 from tb1 print =============== step4: stb -sql select count(tbcol), avg(tbcol), max(tbcol), min(tbcol), count(tbcol) from stb where ts <= 1601481840000 and ts >= 1601481800000 partition by tgcol interval(1m) fill(value, 0) +sql select tbcol5 - tbcol3 from stb + +print =============== check +$null= + +system_content sh/checkValgrind.sh -n dnode1 +print cmd return result ----> [ $system_content ] +if $system_content > 0 then + return -1 +endi + +if $system_content == $null then + return -1 +endi + +print =============== restart +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode1 -s start -v +sql select last_row(tbcol), stddev(tbcol) from stb where tbcol > 5 and tbcol < 20 group by tgcol _OVER: system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/valgrind/checkError6.sim b/tests/script/tsim/valgrind/checkError6.sim index 7a16f5668a..2682d7450b 100644 --- a/tests/script/tsim/valgrind/checkError6.sim +++ b/tests/script/tsim/valgrind/checkError6.sim @@ -77,6 +77,8 @@ sql select length("abcd1234"), char_length("abcd1234=-+*") from tb1 sql select tbcol4, length(tbcol4), lower(tbcol4), upper(tbcol4), ltrim(tbcol4), rtrim(tbcol4), concat(tbcol4, tbcol5), concat_ws('_', tbcol4, tbcol5), substr(tbcol4, 1, 4) from tb1 sql select * from tb1 where tbcol not in (1,2,3,null); sql select * from tb1 where tbcol + 3 <> null; +sql select count(tbcol), avg(tbcol), max(tbcol), min(tbcol), count(tbcol) from tb1 where ts <= 1601481840000 and ts >= 1601481800000 partition by tgcol interval(1m) fill(value, 0) +#sql select tbcol5 - tbcol3 from tb1 print =============== step4: stb sql select avg(tbcol) as c from stb @@ -105,6 +107,9 @@ sql select tbcol4, length(tbcol4), lower(tbcol4), upper(tbcol4), ltrim(tbcol4), sql select * from stb where tbcol not in (1,2,3,null); sql select * from stb where tbcol + 3 <> null; sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from stb where tbcol = 1 and tbcol2 = 1 and tbcol3 = 1 partition by tgcol interval(1d) +sql select _wstart, count(*) from tb1 session(ts, 1m) +sql select count(tbcol), avg(tbcol), max(tbcol), min(tbcol), count(tbcol) from stb where ts <= 1601481840000 and ts >= 1601481800000 partition by tgcol interval(1m) fill(value, 0) +#sql select tbcol5 - tbcol3 from stb print =============== step5: explain sql explain analyze select ts from stb where -2; From 3852aa7498f298942d280f5966342342e29d2a7b Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 2 Aug 2022 14:04:49 +0800 Subject: [PATCH 06/18] test: valgrind case --- tests/script/tsim/valgrind/basic3.sim | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/tests/script/tsim/valgrind/basic3.sim b/tests/script/tsim/valgrind/basic3.sim index d1877776b3..0913691a11 100644 --- a/tests/script/tsim/valgrind/basic3.sim +++ b/tests/script/tsim/valgrind/basic3.sim @@ -55,25 +55,6 @@ sql select tbcol5 - tbcol3 from tb1 print =============== step4: stb sql select tbcol5 - tbcol3 from stb -print =============== check -$null= - -system_content sh/checkValgrind.sh -n dnode1 -print cmd return result ----> [ $system_content ] -if $system_content > 0 then - return -1 -endi - -if $system_content == $null then - return -1 -endi - -print =============== restart -system sh/exec.sh -n dnode1 -s stop -x SIGINT -system sh/exec.sh -n dnode1 -s start -v -sql select last_row(tbcol), stddev(tbcol) from stb where tbcol > 5 and tbcol < 20 group by tgcol - -_OVER: system sh/exec.sh -n dnode1 -s stop -x SIGINT print =============== check $null= From 5b8207556c1c56126d9b678055da368d30441f3e Mon Sep 17 00:00:00 2001 From: slzhou Date: Tue, 2 Aug 2022 14:47:29 +0800 Subject: [PATCH 07/18] fix: taken onCondtion of join node into account when eliminate projects --- source/libs/planner/src/planOptimizer.c | 29 ++++++--- tests/script/tsim/parser/join_multitables.sim | 63 ++++++------------- 2 files changed, 37 insertions(+), 55 deletions(-) diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index c02ef64707..e6f7c4ceb8 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -1693,22 +1693,30 @@ static EDealRes eliminateProjOptCanUseNewChildTargetsImpl(SNode* pNode, void* pC CheckNewChildTargetsCxt* pCxt = pContext; SNode* pTarget = NULL; FOREACH(pTarget, pCxt->pNewChildTargets) { - if (!nodesEqualNode(pTarget, pNode)) { - pCxt->canUse = false; - return DEAL_RES_END; + if (nodesEqualNode(pTarget, pNode)) { + pCxt->canUse = true; + return DEAL_RES_CONTINUE; } } + pCxt->canUse = false; + return DEAL_RES_END; } return DEAL_RES_CONTINUE; } -static bool eliminateProjOptCanUseNewChildTargets(SLogicNode* pChild, SNodeList* pNewChildTargets) { - if (NULL == pChild->pConditions) { - return true; +static bool eliminateProjOptCanChildConditionUseChildTargets(SLogicNode* pChild, SNodeList* pNewChildTargets) { + if (NULL != pChild->pConditions) { + CheckNewChildTargetsCxt cxt = {.pNewChildTargets = pNewChildTargets, .canUse = false}; + nodesWalkExpr(pChild->pConditions, eliminateProjOptCanUseNewChildTargetsImpl, &cxt); + if (!cxt.canUse) return false; } - CheckNewChildTargetsCxt cxt = {.pNewChildTargets = pNewChildTargets, .canUse = true}; - nodesWalkExpr(pChild->pConditions, eliminateProjOptCanUseNewChildTargetsImpl, &cxt); - return cxt.canUse; + if (QUERY_NODE_LOGIC_PLAN_JOIN == nodeType(pChild) && NULL != ((SJoinLogicNode*)pChild)->pOnConditions) { + SJoinLogicNode* pJoinLogicNode = (SJoinLogicNode*)pChild; + CheckNewChildTargetsCxt cxt = {.pNewChildTargets = pNewChildTargets, .canUse = false}; + nodesWalkExpr(pJoinLogicNode->pOnConditions, eliminateProjOptCanUseNewChildTargetsImpl, &cxt); + if (!cxt.canUse) return false; + } + return true; } static void alignProjectionWithTarget(SLogicNode* pNode) { @@ -1748,7 +1756,7 @@ static int32_t eliminateProjOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* } } } - if (eliminateProjOptCanUseNewChildTargets(pChild, pNewChildTargets)) { + if (eliminateProjOptCanChildConditionUseChildTargets(pChild, pNewChildTargets)) { nodesDestroyList(pChild->pTargets); pChild->pTargets = pNewChildTargets; } else { @@ -1760,6 +1768,7 @@ static int32_t eliminateProjOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* if (TSDB_CODE_SUCCESS == code) { NODES_CLEAR_LIST(pProjectNode->node.pChildren); nodesDestroyNode((SNode*)pProjectNode); + //if pChild is a project logic node, remove its projection which is not reference by its target. alignProjectionWithTarget(pChild); } pCxt->optimized = true; diff --git a/tests/script/tsim/parser/join_multitables.sim b/tests/script/tsim/parser/join_multitables.sim index 6c5138f8cb..3f5526d16f 100644 --- a/tests/script/tsim/parser/join_multitables.sim +++ b/tests/script/tsim/parser/join_multitables.sim @@ -682,7 +682,7 @@ if $data08 != 3 then return -1 endi -sql select st0.f1,st1.f1 from st0, st1 where st0.ts=st1.ts and st0.id1=st1.id1; +sql select st0.f1,st1.f1 from st0, st1 where st0.ts=st1.ts and st0.id1=st1.id1 order by st0.f1; if $rows != 25 then return -1 endi @@ -721,22 +721,10 @@ endi if $data01 != @21-03-01 01:00:00.000@ then return -1 endi -if $data10 != @21-03-02 01:00:00.000@ then +if $data50 != @21-03-02 01:00:00.000@ then return -1 endi -if $data11 != @21-03-02 01:00:00.000@ then - return -1 -endi -if $data20 != @21-03-03 01:00:00.000@ then - return -1 -endi -if $data21 != @21-03-03 01:00:00.000@ then - return -1 -endi -if $data30 != @21-03-04 01:00:00.000@ then - return -1 -endi -if $data31 != @21-03-04 01:00:00.000@ then +if $data51 != @21-03-02 01:00:00.000@ then return -1 endi @@ -782,23 +770,23 @@ endi if $data04 != 01 then return -1 endi -if $data10 != @21-03-02 01:00:00.000@ then +if $data50 != @21-03-02 01:00:00.000@ then return -1 endi -if $data11 != 9901.000000000 then +if $data51 != 9901.000000000 then return -1 endi -if $data12 != 11 then +if $data52 != 11 then return -1 endi -if $data13 != 9911.000000000 then +if $data53 != 9911.000000000 then return -1 endi -if $data14 != 01 then +if $data54 != 01 then return -1 endi -sql select last(*) from st0, st1 where st0.ts=st1.ts and st0.id1=st1.id1 interval(10a); +sql select _wstart, last(*) from st0, st1 where st0.ts=st1.ts and st0.id1=st1.id1 interval(10a); if $rows != 25 then return -1 endi @@ -830,7 +818,7 @@ if $data08 != 11 then return -1 endi -sql select last(*) from st0, st1 where st0.ts=st1.ts and st0.id1=st1.id1 interval(1d) sliding(1d); +sql select _wstart, last(*) from st0, st1 where st0.ts=st1.ts and st0.id1=st1.id1 interval(1d) sliding(1d); if $rows != 5 then return -1 endi @@ -937,7 +925,7 @@ sql select st0.*,st1.* from st0, st1 where st1.id1=st0.id1 and st0.ts=st1.ts and if $rows != 5 then return -1 endi -if $data00 != @21-03-01 01:00:00.000@ then +if $data00 != @21-03-02 01:00:00.000@ then print $data00 return -1 endi @@ -965,7 +953,7 @@ endi if $data08 != 3 then return -1 endi -if $data09 != @21-03-01 01:00:00.000@ then +if $data09 != @21-03-02 01:00:00.000@ then return -1 endi @@ -973,38 +961,23 @@ sql select top(st1.f1, 5) from st0, st1 where st1.id1=st0.id1 and st0.ts=st1.ts if $rows != 5 then return -1 endi -if $data00 != @21-03-01 05:00:00.000@ then - return -1 -endi -if $data01 != 9915 then +if $data00 != 9915 then return -1 endi -if $data10 != @21-03-02 05:00:00.000@ then - return -1 -endi -if $data11 != 9915 then +if $data10 != 9915 then return -1 endi -if $data20 != @21-03-03 05:00:00.000@ then - return -1 -endi -if $data21 != 9915 then +if $data20 != 9915 then return -1 endi -if $data30 != @21-03-04 05:00:00.000@ then - return -1 -endi -if $data31 != 9915 then +if $data30 != 9915 then return -1 endi -if $data40 != @21-03-05 05:00:00.000@ then - return -1 -endi -if $data41 != 9915 then +if $data40 != 9915 then return -1 endi -sql select top(st0.f1,5) from st0, st1 where st1.id1=st0.id1 and st0.ts=st1.ts and st1.ts=st0.ts and st0.id1=st1.id1; +sql select st0.ts, top(st0.f1,5) from st0, st1 where st1.id1=st0.id1 and st0.ts=st1.ts and st1.ts=st0.ts and st0.id1=st1.id1 order by st0.ts; if $rows != 5 then return -1 endi From 081695668da452b10c1486d0b1cfe20a8e8d2136 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Tue, 2 Aug 2022 15:33:41 +0800 Subject: [PATCH 08/18] refactor apercentile logs --- source/libs/function/src/builtinsimpl.c | 29 +++++++++++++++---------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index fe4e9fd43e..76be8a05c5 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -2494,6 +2494,8 @@ bool apercentileFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResult } else { buildHistogramInfo(pInfo); pInfo->pHisto = tHistogramCreateFrom(tmp, MAX_HISTOGRAM_BIN); + qDebug("%s set up histogram, numOfElems:%" PRId64 ", numOfEntry:%d, pHisto:%p, elems:%p", __FUNCTION__, + pInfo->pHisto->numOfElems, pInfo->pHisto->numOfEntries, pInfo->pHisto, pInfo->pHisto->elems); } return true; @@ -2524,6 +2526,9 @@ int32_t apercentileFunction(SqlFunctionCtx* pCtx) { tdigestAdd(pInfo->pTDigest, v, w); } } else { + qDebug("%s before add %d elements into histogram, total:%d, numOfEntry:%d, pHisto:%p, elems: %p", __FUNCTION__, + numOfElems, pInfo->pHisto->numOfElems, pInfo->pHisto->numOfEntries, pInfo->pHisto, pInfo->pHisto->elems); + for (int32_t i = start; i < pInput->numOfRows + start; ++i) { if (colDataIsNull_f(pCol->nullbitmap, i)) { continue; @@ -2536,8 +2541,8 @@ int32_t apercentileFunction(SqlFunctionCtx* pCtx) { tHistogramAdd(&pInfo->pHisto, v); } - qDebug("add %d elements into histogram, total:%d, numOfEntry:%d, %p", numOfElems, pInfo->pHisto->numOfElems, - pInfo->pHisto->numOfEntries, pInfo->pHisto); + qDebug("%s after add %d elements into histogram, total:%d, numOfEntry:%d, pHisto:%p, elems: %p", __FUNCTION__, + numOfElems, pInfo->pHisto->numOfElems, pInfo->pHisto->numOfEntries, pInfo->pHisto, pInfo->pHisto->elems); } SET_VAL(pResInfo, numOfElems, 1); @@ -2577,17 +2582,19 @@ static void apercentileTransferInfo(SAPercentileInfo* pInput, SAPercentileInfo* memcpy(pHisto, pInput->pHisto, sizeof(SHistogramInfo) + sizeof(SHistBin) * (MAX_HISTOGRAM_BIN + 1)); pHisto->elems = (SHistBin*)((char*)pHisto + sizeof(SHistogramInfo)); - qDebug("merge histo, total:%" PRId64 ", entry:%d, %p", pHisto->numOfElems, pHisto->numOfEntries, pHisto); + qDebug("%s merge histo, total:%" PRId64 ", entry:%d, %p", __FUNCTION__, pHisto->numOfElems, + pHisto->numOfEntries, pHisto); } else { pHisto->elems = (SHistBin*)((char*)pHisto + sizeof(SHistogramInfo)); - qDebug("input histogram, elem:%" PRId64 ", entry:%d, %p", pHisto->numOfElems, pHisto->numOfEntries, - pInput->pHisto); + qDebug("%s input histogram, elem:%" PRId64 ", entry:%d, %p", __FUNCTION__, pHisto->numOfElems, + pHisto->numOfEntries, pInput->pHisto); SHistogramInfo* pRes = tHistogramMerge(pHisto, pInput->pHisto, MAX_HISTOGRAM_BIN); memcpy(pHisto, pRes, sizeof(SHistogramInfo) + sizeof(SHistBin) * MAX_HISTOGRAM_BIN); pHisto->elems = (SHistBin*)((char*)pHisto + sizeof(SHistogramInfo)); - qDebug("merge histo, total:%" PRId64 ", entry:%d, %p", pHisto->numOfElems, pHisto->numOfEntries, pHisto); + qDebug("%s merge histo, total:%" PRId64 ", entry:%d, %p", __FUNCTION__, pHisto->numOfElems, + pHisto->numOfEntries, pHisto); tHistogramDestroy(&pRes); } } @@ -2603,7 +2610,7 @@ int32_t apercentileFunctionMerge(SqlFunctionCtx* pCtx) { SAPercentileInfo* pInfo = GET_ROWCELL_INTERBUF(pResInfo); - qDebug("total %d rows will merge, %p", pInput->numOfRows, pInfo->pHisto); + qDebug("%s total %d rows will merge, %p", __FUNCTION__, pInput->numOfRows, pInfo->pHisto); int32_t start = pInput->startRowIndex; for (int32_t i = start; i < start + pInput->numOfRows; ++i) { @@ -2614,7 +2621,7 @@ int32_t apercentileFunctionMerge(SqlFunctionCtx* pCtx) { } if (pInfo->algo != APERCT_ALGO_TDIGEST) { - qDebug("after merge, total:%d, numOfEntry:%d, %p", pInfo->pHisto->numOfElems, pInfo->pHisto->numOfEntries, + qDebug("%s after merge, total:%d, numOfEntry:%d, %p", __FUNCTION__, pInfo->pHisto->numOfElems, pInfo->pHisto->numOfEntries, pInfo->pHisto); } @@ -2637,8 +2644,8 @@ int32_t apercentileFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { } else { buildHistogramInfo(pInfo); if (pInfo->pHisto->numOfElems > 0) { - qDebug("get the final res:%d, elements:%" PRId64 ", entry:%d", pInfo->pHisto->numOfElems, - pInfo->pHisto->numOfEntries); + qDebug("%s get the final res, elements:%" PRId64 ", numOfEntry:%d, pHisto:%p, elems:%p", __FUNCTION__, + pInfo->pHisto->numOfElems, pInfo->pHisto->numOfEntries, pInfo->pHisto, pInfo->pHisto->elems); double ratio[] = {pInfo->percent}; double* res = tHistogramUniform(pInfo->pHisto, ratio, 1); @@ -2693,7 +2700,7 @@ int32_t apercentileCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) SResultRowEntryInfo* pSResInfo = GET_RES_INFO(pSourceCtx); SAPercentileInfo* pSBuf = GET_ROWCELL_INTERBUF(pSResInfo); - qDebug("start to combine apercentile, %p", pDBuf->pHisto); + qDebug("%s start to combine apercentile, %p", __FUNCTION__, pDBuf->pHisto); apercentileTransferInfo(pSBuf, pDBuf); pDResInfo->numOfRes = TMAX(pDResInfo->numOfRes, pSResInfo->numOfRes); From b323d508e62b7e93bb0d2733ca4947d31e81a47e Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Tue, 2 Aug 2022 15:34:33 +0800 Subject: [PATCH 09/18] fix(query): fix apercentile crash in stream concurrent insert TD-18010 --- source/libs/function/src/builtinsimpl.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 76be8a05c5..a69d2924a9 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -2529,6 +2529,9 @@ int32_t apercentileFunction(SqlFunctionCtx* pCtx) { qDebug("%s before add %d elements into histogram, total:%d, numOfEntry:%d, pHisto:%p, elems: %p", __FUNCTION__, numOfElems, pInfo->pHisto->numOfElems, pInfo->pHisto->numOfEntries, pInfo->pHisto, pInfo->pHisto->elems); + // might be a race condition here that pHisto can be overwritten or setup function + // has not been called, need to relink the buffer pHisto points to. + buildHistogramInfo(pInfo); for (int32_t i = start; i < pInput->numOfRows + start; ++i) { if (colDataIsNull_f(pCol->nullbitmap, i)) { continue; From 602d2aa3778d12345cfcc9dd43fa4cf299fe48e6 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Tue, 2 Aug 2022 16:13:36 +0800 Subject: [PATCH 10/18] other: rsma snapshot and misc update --- include/os/osFile.h | 1 + source/common/src/tdatablock.c | 19 +++ source/dnode/vnode/src/inc/sma.h | 29 +++- source/dnode/vnode/src/inc/vnodeInt.h | 21 +-- source/dnode/vnode/src/sma/smaCommit.c | 35 ++++ source/dnode/vnode/src/sma/smaRollup.c | 34 +++- source/dnode/vnode/src/sma/smaSnapshot.c | 167 +++++++++++++++---- source/dnode/vnode/src/tq/tqOffsetSnapshot.c | 4 +- source/dnode/vnode/src/tq/tqSnapshot.c | 14 +- source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 4 +- source/dnode/vnode/src/vnd/vnodeSnapshot.c | 15 +- source/dnode/vnode/src/vnd/vnodeSvr.c | 5 +- source/os/src/osFile.c | 20 +++ 13 files changed, 293 insertions(+), 75 deletions(-) diff --git a/include/os/osFile.h b/include/os/osFile.h index 2f6a6ba480..21e3d2e6cf 100644 --- a/include/os/osFile.h +++ b/include/os/osFile.h @@ -54,6 +54,7 @@ typedef struct TdFile *TdFilePtr; #define TD_FILE_EXCL 0x0080 #define TD_FILE_STREAM 0x0100 // Only support taosFprintfFile, taosGetLineFile, taosEOFFile TdFilePtr taosOpenFile(const char *path, int32_t tdFileOptions); +TdFilePtr taosCreateFile(const char *path, int32_t tdFileOptions); #define TD_FILE_ACCESS_EXIST_OK 0x1 #define TD_FILE_ACCESS_READ_OK 0x2 diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 2a8f80f030..be216e0742 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1739,6 +1739,9 @@ void blockDebugShowDataBlocks(const SArray* dataBlocks, const char* flag) { formatTimestamp(pBuf, *(uint64_t*)var, TSDB_TIME_PRECISION_MILLI); printf(" %25s |", pBuf); break; + case TSDB_DATA_TYPE_BOOL: + printf(" %15d |", *(int32_t*)var); + break; case TSDB_DATA_TYPE_INT: printf(" %15d |", *(int32_t*)var); break; @@ -1757,6 +1760,22 @@ void blockDebugShowDataBlocks(const SArray* dataBlocks, const char* flag) { case TSDB_DATA_TYPE_DOUBLE: printf(" %15lf |", *(double*)var); break; + case TSDB_DATA_TYPE_VARCHAR: { + char* pData = colDataGetVarData(pColInfoData, j); + int32_t dataSize = TMIN(sizeof(pBuf), varDataLen(pData) + 1); + memset(pBuf, 0, dataSize); + strncpy(pBuf, varDataVal(pData), dataSize); + printf(" %15s |", pBuf); + } break; + case TSDB_DATA_TYPE_NCHAR: { + char* pData = colDataGetVarData(pColInfoData, j); + int32_t dataSize = TMIN(sizeof(pBuf), varDataLen(pData) + 1); + memset(pBuf, 0, dataSize); + taosUcs4ToMbs((TdUcs4*)varDataVal(pData), dataSize, pBuf); + printf(" %15s |", pBuf); + } break; + default: + break; } } printf("\n"); diff --git a/source/dnode/vnode/src/inc/sma.h b/source/dnode/vnode/src/inc/sma.h index c825ab6731..8cd70537de 100644 --- a/source/dnode/vnode/src/inc/sma.h +++ b/source/dnode/vnode/src/inc/sma.h @@ -41,6 +41,9 @@ typedef struct SRSmaStat SRSmaStat; typedef struct SSmaKey SSmaKey; typedef struct SRSmaInfo SRSmaInfo; typedef struct SRSmaInfoItem SRSmaInfoItem; +typedef struct SQTaskFile SQTaskFile; +typedef struct SQTaskFReader SQTaskFReader; +typedef struct SQTaskFWriter SQTaskFWriter; struct SSmaEnv { SRWLatch lock; @@ -64,12 +67,32 @@ struct STSmaStat { STSchema *pTSchema; }; +struct SQTaskFile { + volatile int32_t nRef; + int64_t commitID; + int64_t size; +}; + +struct SQTaskFReader { + SSma *pSma; + SQTaskFile fTask; + TdFilePtr pReadH; +}; +struct SQTaskFWriter { + SSma *pSma; + SQTaskFile fTask; + TdFilePtr pWriteH; + char *fname; +}; + struct SRSmaStat { SSma *pSma; int64_t commitAppliedVer; // vnode applied version for async commit int64_t refId; // shared by fetch tasks + SRWLatch lock; // r/w lock for rsma fs(e.g. qtaskinfo) int8_t triggerStat; // shared by fetch tasks int8_t commitStat; // 0 not in committing, 1 in committing + SArray *aTaskFile; // qTaskFiles committed recently(for recovery/snapshot r/w) SHashObj *rsmaInfoHash; // key: stbUid, value: SRSmaInfo; SHashObj *iRsmaInfoHash; // key: stbUid, value: SRSmaInfo; immutable rsmaInfoHash }; @@ -89,6 +112,7 @@ struct SSmaStat { #define RSMA_TRIGGER_STAT(r) (&(r)->triggerStat) #define RSMA_COMMIT_STAT(r) (&(r)->commitStat) #define RSMA_REF_ID(r) ((r)->refId) +#define RSMA_FS_LOCK(r) (&(r)->lock) struct SRSmaInfoItem { void *taskInfo; // qTaskInfo_t @@ -192,6 +216,8 @@ static FORCE_INLINE void tdSmaStatSetDropped(STSmaStat *pTStat) { } } +void tdRSmaQTaskInfoGetFileName(int32_t vid, int64_t version, char *outputName); +void tdRSmaQTaskInfoGetFullName(int32_t vid, int64_t version, const char *path, char *outputName); int32_t tdCloneRSmaInfo(SSma *pSma, SRSmaInfo *pDest, SRSmaInfo *pSrc); void tdFreeQTaskInfo(qTaskInfo_t *taskHandle, int32_t vgId, int32_t level); static int32_t tdDestroySmaState(SSmaStat *pSmaStat, int8_t smaType); @@ -209,9 +235,6 @@ int32_t tdProcessTSmaGetDaysImpl(SVnodeCfg *pCfg, void *pCont, uint32_t contLen, // smaFileUtil ================ -typedef struct SQTaskFReader SQTaskFReader; -typedef struct SQTaskFWriter SQTaskFWriter; - #define TD_FILE_HEAD_SIZE 512 typedef struct STFInfo STFInfo; diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index b90254e543..1a7e0f9a45 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -186,6 +186,7 @@ int32_t smaSyncPostCommit(SSma* pSma); int32_t smaAsyncPreCommit(SSma* pSma); int32_t smaAsyncCommit(SSma* pSma); int32_t smaAsyncPostCommit(SSma* pSma); +int32_t smaDoRetention(SSma* pSma, int64_t now); int32_t tdProcessTSmaCreate(SSma* pSma, int64_t version, const char* msg); int32_t tdProcessTSmaInsert(SSma* pSma, int64_t indexUid, const char* msg); @@ -354,16 +355,16 @@ struct SSma { void smaHandleRes(void* pVnode, int64_t smaId, const SArray* data); enum { - SNAP_DATA_META = 0, - SNAP_DATA_TSDB = 1, - SNAP_DATA_DEL = 2, - SNAP_DATA_RSMA1 = 3, - SNAP_DATA_RSMA2 = 4, - SNAP_DATA_QTASK = 5, - SNAP_DATA_TQ_HANDLE = 6, - SNAP_DATA_TQ_OFFSET = 7, - SNAP_DATA_STREAM_TASK = 8, - SNAP_DATA_STREAM_STATE = 9, + SNAP_DATA_META = 1, + SNAP_DATA_TSDB = 2, + SNAP_DATA_DEL = 3, + SNAP_DATA_RSMA1 = 4, + SNAP_DATA_RSMA2 = 5, + SNAP_DATA_QTASK = 6, + SNAP_DATA_TQ_HANDLE = 7, + SNAP_DATA_TQ_OFFSET = 8, + SNAP_DATA_STREAM_TASK = 9, + SNAP_DATA_STREAM_STATE = 10, }; struct SSnapDataHdr { diff --git a/source/dnode/vnode/src/sma/smaCommit.c b/source/dnode/vnode/src/sma/smaCommit.c index a0e3932245..3c88b6f5cb 100644 --- a/source/dnode/vnode/src/sma/smaCommit.c +++ b/source/dnode/vnode/src/sma/smaCommit.c @@ -241,6 +241,41 @@ static int32_t tdCleanupQTaskInfoFiles(SSma *pSma, SRSmaStat *pRSmaStat) { return TSDB_CODE_SUCCESS; } +// SQTaskFile ====================================================== +// int32_t tCmprQTaskFile(void const *lhs, void const *rhs) { +// int64_t *lCommitted = *(int64_t *)lhs; +// SQTaskFile *rQTaskF = (SQTaskFile *)rhs; + +// if (lCommitted < rQTaskF->commitID) { +// return -1; +// } else if (lCommitted > rQTaskF->commitID) { +// return 1; +// } + +// return 0; +// } + +#if 0 +/** + * @brief At most time, there is only one qtaskinfo file committed latest in aTaskFile. Sometimes, there would be + * multiple qtaskinfo files supporting snapshot replication. + * + * @param pSma + * @param pRSmaStat + * @return int32_t + */ +static int32_t tdCleanupQTaskInfoFiles(SSma *pSma, SRSmaStat *pRSmaStat) { + SVnode *pVnode = pSma->pVnode; + int64_t committed = pRSmaStat->commitAppliedVer; + SArray *aTaskFile = pRSmaStat->aTaskFile; + + void *qTaskFile = taosArraySearch(aTaskFile, committed, tCmprQTaskFile, TD_LE); + + + return TSDB_CODE_SUCCESS; +} +#endif + /** * @brief post-commit for rollup sma * 1) clean up the outdated qtaskinfo files diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index 3505711cd0..02e488d65a 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -38,7 +38,6 @@ static SRSmaInfo *tdGetRSmaInfoBySuid(SSma *pSma, int64_t suid); static int32_t tdRSmaFetchAndSubmitResult(SRSmaInfoItem *pItem, STSchema *pTSchema, int64_t suid, SRSmaStat *pStat, int8_t blkType); static void tdRSmaFetchTrigger(void *param, void *tmrId); -static void tdRSmaQTaskInfoGetFName(int32_t vid, int64_t version, char *outputName); static int32_t tdRSmaQTaskInfoIterInit(SRSmaQTaskInfoIter *pIter, STFile *pTFile); static int32_t tdRSmaQTaskInfoIterNextBlock(SRSmaQTaskInfoIter *pIter, bool *isFinish); @@ -77,10 +76,14 @@ struct SRSmaQTaskInfoIter { int32_t nBufPos; }; -static void tdRSmaQTaskInfoGetFName(int32_t vgId, int64_t version, char *outputName) { +void tdRSmaQTaskInfoGetFileName(int32_t vgId, int64_t version, char *outputName) { tdGetVndFileName(vgId, NULL, VNODE_RSMA_DIR, TD_QTASKINFO_FNAME_PREFIX, version, outputName); } +void tdRSmaQTaskInfoGetFullName(int32_t vgId, int64_t version, const char* path, char *outputName) { + tdGetVndFileName(vgId, path, VNODE_RSMA_DIR, TD_QTASKINFO_FNAME_PREFIX, version, outputName); +} + static FORCE_INLINE int32_t tdRSmaQTaskInfoContLen(int32_t lenWithHead) { return lenWithHead - RSMA_QTASKINFO_HEAD_LEN; } @@ -599,8 +602,8 @@ static int32_t tdRSmaFetchAndSubmitResult(SRSmaInfoItem *pItem, STSchema *pTSche SSubmitReq *pReq = NULL; // TODO: the schema update should be handled if (buildSubmitReqFromDataBlock(&pReq, pResult, pTSchema, SMA_VID(pSma), suid) < 0) { - smaError("vgId:%d, build submit req for rsma stable %" PRIi64 " level %" PRIi8 " failed since %s", SMA_VID(pSma), - suid, pItem->level, terrstr()); + smaError("vgId:%d, build submit req for rsma stable %" PRIi64 " level %" PRIi8 " failed since %s", + SMA_VID(pSma), suid, pItem->level, terrstr()); goto _err; } @@ -874,7 +877,7 @@ static int32_t tdRSmaRestoreQTaskInfoReload(SSma *pSma, int64_t *committed) { STFile tFile = {0}; char qTaskInfoFName[TSDB_FILENAME_LEN] = {0}; - tdRSmaQTaskInfoGetFName(TD_VID(pVnode), pVnode->state.committed, qTaskInfoFName); + tdRSmaQTaskInfoGetFileName(TD_VID(pVnode), pVnode->state.committed, qTaskInfoFName); if (tdInitTFile(&tFile, tfsGetPrimaryPath(pVnode->pTfs), qTaskInfoFName) < 0) { goto _err; } @@ -1172,7 +1175,7 @@ int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat, SHashObj *pInfoHash) { #if 0 if (pRSmaStat->commitAppliedVer > 0) { char qTaskInfoFName[TSDB_FILENAME_LEN]; - tdRSmaQTaskInfoGetFName(vid, pRSmaStat->commitAppliedVer, qTaskInfoFName); + tdRSmaQTaskInfoGetFileName(vid, pRSmaStat->commitAppliedVer, qTaskInfoFName); if (tdInitTFile(&tFile, tfsGetPrimaryPath(pVnode->pTfs), qTaskInfoFName) < 0) { smaError("vgId:%d, rsma persit, init %s failed since %s", vid, qTaskInfoFName, terrstr()); goto _err; @@ -1217,7 +1220,7 @@ int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat, SHashObj *pInfoHash) { if (!isFileCreated) { char qTaskInfoFName[TSDB_FILENAME_LEN]; - tdRSmaQTaskInfoGetFName(vid, pRSmaStat->commitAppliedVer, qTaskInfoFName); + tdRSmaQTaskInfoGetFileName(vid, pRSmaStat->commitAppliedVer, qTaskInfoFName); if (tdInitTFile(&tFile, tfsGetPrimaryPath(pVnode->pTfs), qTaskInfoFName) < 0) { smaError("vgId:%d, rsma persit, init %s failed since %s", vid, qTaskInfoFName, terrstr()); goto _err; @@ -1357,3 +1360,20 @@ static void tdRSmaFetchTrigger(void *param, void *tmrId) { _end: tdReleaseSmaRef(smaMgmt.rsetId, pItem->refId, __func__, __LINE__); } + +int32_t smaDoRetention(SSma *pSma, int64_t now) { + int32_t code = TSDB_CODE_SUCCESS; + if (VND_IS_RSMA(pSma->pVnode)) { + return code; + } + + for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) { + if (pSma->pRSmaTsdb[i]) { + code = tsdbDoRetention(pSma->pRSmaTsdb[i], now); + if (code) goto _end; + } + } + +_end: + return code; +} \ No newline at end of file diff --git a/source/dnode/vnode/src/sma/smaSnapshot.c b/source/dnode/vnode/src/sma/smaSnapshot.c index 06fe0074f3..4dda122bd7 100644 --- a/source/dnode/vnode/src/sma/smaSnapshot.c +++ b/source/dnode/vnode/src/sma/smaSnapshot.c @@ -35,6 +35,7 @@ struct SRsmaSnapReader { int32_t rsmaSnapReaderOpen(SSma* pSma, int64_t sver, int64_t ever, SRsmaSnapReader** ppReader) { int32_t code = 0; + SVnode* pVnode = pSma->pVnode; SRsmaSnapReader* pReader = NULL; // alloc @@ -47,6 +48,7 @@ int32_t rsmaSnapReaderOpen(SSma* pSma, int64_t sver, int64_t ever, SRsmaSnapRead pReader->sver = sver; pReader->ever = ever; + // rsma1/rsma2 for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) { if (pSma->pRSmaTsdb[i]) { code = tsdbSnapReaderOpen(pSma->pRSmaTsdb[i], sver, ever, i == 0 ? SNAP_DATA_RSMA1 : SNAP_DATA_RSMA2, @@ -56,23 +58,98 @@ int32_t rsmaSnapReaderOpen(SSma* pSma, int64_t sver, int64_t ever, SRsmaSnapRead } } } + + // qtaskinfo + // 1. add ref to qtaskinfo.v${ever} if exists and then start to replicate + char qTaskInfoFullName[TSDB_FILENAME_LEN]; + tdRSmaQTaskInfoGetFullName(TD_VID(pVnode), ever, tfsGetPrimaryPath(pVnode->pTfs), qTaskInfoFullName); + + if (!taosCheckExistFile(qTaskInfoFullName)) { + smaInfo("vgId:%d, vnode snapshot rsma reader for qtaskinfo not need as %s not exists", TD_VID(pVnode), + qTaskInfoFullName); + } else { + pReader->pQTaskFReader = taosMemoryCalloc(1, sizeof(SQTaskFReader)); + if (!pReader->pQTaskFReader) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + + TdFilePtr qTaskF = taosOpenFile(qTaskInfoFullName, TD_FILE_READ); + if (!qTaskF) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + pReader->pQTaskFReader->pReadH = qTaskF; +#if 0 + SQTaskFile* pQTaskF = &pReader->pQTaskFReader->fTask; + pQTaskF->nRef = 1; +#endif + } + *ppReader = pReader; - smaInfo("vgId:%d, vnode snapshot rsma reader opened succeed", SMA_VID(pSma)); + smaInfo("vgId:%d, vnode snapshot rsma reader opened %s succeed", TD_VID(pVnode), qTaskInfoFullName); return TSDB_CODE_SUCCESS; _err: - smaError("vgId:%d, vnode snapshot rsma reader opened failed since %s", SMA_VID(pSma), tstrerror(code)); + smaError("vgId:%d, vnode snapshot rsma reader opened failed since %s", TD_VID(pVnode), tstrerror(code)); return TSDB_CODE_FAILED; } -static int32_t rsmaSnapReadQTaskInfo(SRsmaSnapReader* pReader, uint8_t** ppData) { - int32_t code = 0; - SSma* pSma = pReader->pSma; +static int32_t rsmaSnapReadQTaskInfo(SRsmaSnapReader* pReader, uint8_t** ppBuf) { + int32_t code = 0; + SSma* pSma = pReader->pSma; + int64_t n = 0; + uint8_t* pBuf = NULL; + SQTaskFReader* qReader = pReader->pQTaskFReader; + + if (!qReader->pReadH) { + *ppBuf = NULL; + smaInfo("vgId:%d, vnode snapshot rsma reader qtaskinfo, readh is empty", SMA_VID(pSma)); + return 0; + } + + int64_t size = 0; + if (taosFStatFile(qReader->pReadH, &size, NULL) < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + // seek + if (taosLSeekFile(qReader->pReadH, 0, SEEK_SET) < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + ASSERT(!(*ppBuf)); + // alloc + *ppBuf = taosMemoryCalloc(1, sizeof(SSnapDataHdr) + size); + if (!(*ppBuf)) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + + // read + n = taosReadFile(qReader->pReadH, POINTER_SHIFT(*ppBuf, sizeof(SSnapDataHdr)), size); + if (n < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } else if (n != size) { + code = TSDB_CODE_FILE_CORRUPTED; + goto _err; + } + + smaInfo("vgId:%d, vnode snapshot rsma read qtaskinfo, size:%" PRIi64, SMA_VID(pSma), size); + + + SSnapDataHdr* pHdr = (SSnapDataHdr*)(*ppBuf); + pHdr->type = SNAP_DATA_QTASK; + pHdr->size = size; _exit: smaInfo("vgId:%d, vnode snapshot rsma read qtaskinfo succeed", SMA_VID(pSma)); return code; _err: + *ppBuf = NULL; smaError("vgId:%d, vnode snapshot rsma read qtaskinfo failed since %s", SMA_VID(pSma), tstrerror(code)); return code; } @@ -108,14 +185,14 @@ int32_t rsmaSnapRead(SRsmaSnapReader* pReader, uint8_t** ppData) { // read qtaskinfo file if (!pReader->qTaskDone) { + smaInfo("vgId:%d, vnode snapshot rsma qtaskinfo not done", SMA_VID(pReader->pSma)); code = rsmaSnapReadQTaskInfo(pReader, ppData); if (code) { goto _err; } else { + pReader->qTaskDone = 1; if (*ppData) { goto _exit; - } else { - pReader->qTaskDone = 1; } } } @@ -140,11 +217,11 @@ int32_t rsmaSnapReaderClose(SRsmaSnapReader** ppReader) { } if (pReader->pQTaskFReader) { - // TODO: close for qtaskinfo + taosCloseFile(&pReader->pQTaskFReader->pReadH); + taosMemoryFreeClear(pReader->pQTaskFReader); smaInfo("vgId:%d, vnode snapshot rsma reader closed for qTaskInfo", SMA_VID(pReader->pSma)); } - smaInfo("vgId:%d, vnode snapshot rsma reader closed", SMA_VID(pReader->pSma)); taosMemoryFreeClear(*ppReader); @@ -171,6 +248,7 @@ struct SRsmaSnapWriter { int32_t rsmaSnapWriterOpen(SSma* pSma, int64_t sver, int64_t ever, SRsmaSnapWriter** ppWriter) { int32_t code = 0; SRsmaSnapWriter* pWriter = NULL; + SVnode* pVnode = pSma->pVnode; // alloc pWriter = (SRsmaSnapWriter*)taosMemoryCalloc(1, sizeof(*pWriter)); @@ -182,6 +260,7 @@ int32_t rsmaSnapWriterOpen(SSma* pSma, int64_t sver, int64_t ever, SRsmaSnapWrit pWriter->sver = sver; pWriter->ever = ever; + // rsma1/rsma2 for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) { if (pSma->pRSmaTsdb[i]) { code = tsdbSnapWriterOpen(pSma->pRSmaTsdb[i], sver, ever, &pWriter->pDataWriter[i]); @@ -192,8 +271,25 @@ int32_t rsmaSnapWriterOpen(SSma* pSma, int64_t sver, int64_t ever, SRsmaSnapWrit } // qtaskinfo - // TODO + SQTaskFWriter* qWriter = (SQTaskFWriter*)taosMemoryCalloc(1, sizeof(SQTaskFWriter)); + qWriter->pSma = pSma; + char qTaskInfoFullName[TSDB_FILENAME_LEN]; + tdRSmaQTaskInfoGetFullName(TD_VID(pVnode), 1, tfsGetPrimaryPath(pVnode->pTfs), qTaskInfoFullName); + TdFilePtr qTaskF = taosCreateFile(qTaskInfoFullName, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (!qTaskF) { + code = TAOS_SYSTEM_ERROR(errno); + smaError("vgId:%d, rsma snapshot writer open %s failed since %s", TD_VID(pSma->pVnode), qTaskInfoFullName, tstrerror(code)); + goto _err; + } + qWriter->pWriteH = qTaskF; + int32_t fnameLen = strlen(qTaskInfoFullName) + 1; + qWriter->fname = taosMemoryCalloc(1, fnameLen); + strncpy(qWriter->fname, qTaskInfoFullName, fnameLen); + pWriter->pQTaskFWriter = qWriter; + smaDebug("vgId:%d, rsma snapshot writer open succeed for %s", TD_VID(pSma->pVnode), qTaskInfoFullName); + + // snapWriter *ppWriter = pWriter; smaInfo("vgId:%d, rsma snapshot writer open succeed", TD_VID(pSma->pVnode)); @@ -208,18 +304,30 @@ _err: int32_t rsmaSnapWriterClose(SRsmaSnapWriter** ppWriter, int8_t rollback) { int32_t code = 0; SRsmaSnapWriter* pWriter = *ppWriter; + SVnode* pVnode = pWriter->pSma->pVnode; if (rollback) { - ASSERT(0); - // code = tsdbFSRollback(pWriter->pTsdb->pFS); - // if (code) goto _err; + // TODO: rsma1/rsma2 + // qtaskinfo + if(pWriter->pQTaskFWriter) { + taosRemoveFile(pWriter->pQTaskFWriter->fname); + } } else { + // rsma1/rsma2 for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) { if (pWriter->pDataWriter[i]) { code = tsdbSnapWriterClose(&pWriter->pDataWriter[i], rollback); if (code) goto _err; } } + // qtaskinfo + if (pWriter->pQTaskFWriter) { + char qTaskInfoFullName[TSDB_FILENAME_LEN]; + tdRSmaQTaskInfoGetFullName(TD_VID(pVnode), 0, tfsGetPrimaryPath(pVnode->pTfs), qTaskInfoFullName); + taosRenameFile(pWriter->pQTaskFWriter->fname, qTaskInfoFullName); + smaInfo("vgId:%d, vnode snapshot rsma writer rename %s to %s", SMA_VID(pWriter->pSma), + pWriter->pQTaskFWriter->fname, qTaskInfoFullName); + } } smaInfo("vgId:%d, vnode snapshot rsma writer close succeed", SMA_VID(pWriter->pSma)); @@ -261,26 +369,23 @@ _err: } static int32_t rsmaSnapWriteQTaskInfo(SRsmaSnapWriter* pWriter, uint8_t* pData, uint32_t nData) { - int32_t code = 0; + int32_t code = 0; + SQTaskFWriter* qWriter = pWriter->pQTaskFWriter; - if (pWriter->pQTaskFWriter == NULL) { - // SDelFile* pDelFile = pWriter->fs.pDelFile; - - // // reader - // if (pDelFile) { - // code = tsdbDelFReaderOpen(&pWriter->pDelFReader, pDelFile, pTsdb, NULL); - // if (code) goto _err; - - // code = tsdbReadDelIdx(pWriter->pDelFReader, pWriter->aDelIdxR, NULL); - // if (code) goto _err; - // } - - // // writer - // SDelFile delFile = {.commitID = pWriter->commitID, .offset = 0, .size = 0}; - // code = tsdbDelFWriterOpen(&pWriter->pDelFWriter, &delFile, pTsdb); - // if (code) goto _err; + if (qWriter && qWriter->pWriteH) { + SSnapDataHdr* pHdr = (SSnapDataHdr*)pData; + int64_t size = pHdr->size; + ASSERT(size == (nData - sizeof(SSnapDataHdr))); + int64_t contLen = taosWriteFile(qWriter->pWriteH, pHdr->data, size); + if (contLen != size) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + } else { + smaInfo("vgId:%d, vnode snapshot rsma write qtaskinfo is not needed", SMA_VID(pWriter->pSma)); } - smaInfo("vgId:%d, vnode snapshot rsma write qtaskinfo succeed", SMA_VID(pWriter->pSma)); + + smaInfo("vgId:%d, vnode snapshot rsma write qtaskinfo %s succeed", SMA_VID(pWriter->pSma), qWriter->fname); _exit: return code; diff --git a/source/dnode/vnode/src/tq/tqOffsetSnapshot.c b/source/dnode/vnode/src/tq/tqOffsetSnapshot.c index cacb82b702..292c234f49 100644 --- a/source/dnode/vnode/src/tq/tqOffsetSnapshot.c +++ b/source/dnode/vnode/src/tq/tqOffsetSnapshot.c @@ -37,7 +37,7 @@ int32_t tqOffsetReaderOpen(STQ* pTq, int64_t sver, int64_t ever, STqOffsetReader pReader->sver = sver; pReader->ever = ever; - tqInfo("vgId:%d vnode snapshot tq offset reader opened", TD_VID(pTq->pVnode)); + tqInfo("vgId:%d, vnode snapshot tq offset reader opened", TD_VID(pTq->pVnode)); *ppReader = pReader; return 0; @@ -109,7 +109,7 @@ int32_t tqOffsetWriterOpen(STQ* pTq, int64_t sver, int64_t ever, STqOffsetWriter return code; _err: - tqError("vgId:%d tq snapshot writer open failed since %s", TD_VID(pTq->pVnode), tstrerror(code)); + tqError("vgId:%d, tq snapshot writer open failed since %s", TD_VID(pTq->pVnode), tstrerror(code)); *ppWriter = NULL; return code; } diff --git a/source/dnode/vnode/src/tq/tqSnapshot.c b/source/dnode/vnode/src/tq/tqSnapshot.c index 21172134ba..b4a7ce7737 100644 --- a/source/dnode/vnode/src/tq/tqSnapshot.c +++ b/source/dnode/vnode/src/tq/tqSnapshot.c @@ -52,13 +52,13 @@ int32_t tqSnapReaderOpen(STQ* pTq, int64_t sver, int64_t ever, STqSnapReader** p goto _err; } - tqInfo("vgId:%d vnode snapshot tq reader opened", TD_VID(pTq->pVnode)); + tqInfo("vgId:%d, vnode snapshot tq reader opened", TD_VID(pTq->pVnode)); *ppReader = pReader; return code; _err: - tqError("vgId:%d vnode snapshot tq reader open failed since %s", TD_VID(pTq->pVnode), tstrerror(code)); + tqError("vgId:%d, vnode snapshot tq reader open failed since %s", TD_VID(pTq->pVnode), tstrerror(code)); *ppReader = NULL; return code; } @@ -113,14 +113,14 @@ int32_t tqSnapRead(STqSnapReader* pReader, uint8_t** ppData) { pHdr->size = vLen; memcpy(pHdr->data, pVal, vLen); - tqInfo("vgId:%d vnode snapshot tq read data, version:%" PRId64 " subKey: %s vLen:%d", TD_VID(pReader->pTq->pVnode), + tqInfo("vgId:%d, vnode snapshot tq read data, version:%" PRId64 " subKey: %s vLen:%d", TD_VID(pReader->pTq->pVnode), handle.snapshotVer, handle.subKey, vLen); _exit: return code; _err: - tqError("vgId:%d vnode snapshot tq read data failed since %s", TD_VID(pReader->pTq->pVnode), tstrerror(code)); + tqError("vgId:%d, vnode snapshot tq read data failed since %s", TD_VID(pReader->pTq->pVnode), tstrerror(code)); return code; } @@ -154,7 +154,7 @@ int32_t tqSnapWriterOpen(STQ* pTq, int64_t sver, int64_t ever, STqSnapWriter** p return code; _err: - tqError("vgId:%d tq snapshot writer open failed since %s", TD_VID(pTq->pVnode), tstrerror(code)); + tqError("vgId:%d, tq snapshot writer open failed since %s", TD_VID(pTq->pVnode), tstrerror(code)); *ppWriter = NULL; return code; } @@ -182,7 +182,7 @@ int32_t tqSnapWriterClose(STqSnapWriter** ppWriter, int8_t rollback) { return code; _err: - tqError("vgId:%d tq snapshot writer close failed since %s", TD_VID(pWriter->pTq->pVnode), tstrerror(code)); + tqError("vgId:%d, tq snapshot writer close failed since %s", TD_VID(pWriter->pTq->pVnode), tstrerror(code)); return code; } @@ -204,6 +204,6 @@ int32_t tqSnapWrite(STqSnapWriter* pWriter, uint8_t* pData, uint32_t nData) { _err: tDecoderClear(pDecoder); - tqError("vgId:%d vnode snapshot tq write failed since %s", TD_VID(pTq->pVnode), tstrerror(code)); + tqError("vgId:%d, vnode snapshot tq write failed since %s", TD_VID(pTq->pVnode), tstrerror(code)); return code; } diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index 97ab410c1b..a9f07cbf24 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -1215,11 +1215,11 @@ int32_t tsdbSnapWrite(STsdbSnapWriter* pWriter, uint8_t* pData, uint32_t nData) } _exit: - tsdbDebug("vgId:%d, tsdb snapshow write for %s succeed", TD_VID(pWriter->pTsdb->pVnode), pWriter->pTsdb->path); + tsdbDebug("vgId:%d, tsdb snapshot write for %s succeed", TD_VID(pWriter->pTsdb->pVnode), pWriter->pTsdb->path); return code; _err: - tsdbError("vgId:%d, tsdb snapshow write for %s failed since %s", TD_VID(pWriter->pTsdb->pVnode), pWriter->pTsdb->path, + tsdbError("vgId:%d, tsdb snapshot write for %s failed since %s", TD_VID(pWriter->pTsdb->pVnode), pWriter->pTsdb->path, tstrerror(code)); return code; } diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c index d1b1b68ce4..5a81f91919 100644 --- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c +++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c @@ -194,7 +194,7 @@ int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData) if (*ppData) { goto _exit; } else { - pReader->tsdbDone = 1; + pReader->rsmaDone = 1; code = rsmaSnapReaderClose(&pReader->pRsmaReader); if (code) goto _err; } @@ -373,18 +373,9 @@ int32_t vnodeSnapWrite(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData) { case SNAP_DATA_STREAM_STATE: { } break; case SNAP_DATA_RSMA1: - case SNAP_DATA_RSMA2: { - // rsma1/rsma2 - if (pWriter->pRsmaSnapWriter == NULL) { - code = rsmaSnapWriterOpen(pVnode->pSma, pWriter->sver, pWriter->ever, &pWriter->pRsmaSnapWriter); - if (code) goto _err; - } - - code = rsmaSnapWrite(pWriter->pRsmaSnapWriter, pData, nData); - if (code) goto _err; - } break; + case SNAP_DATA_RSMA2: case SNAP_DATA_QTASK: { - // qtask for rsma + // rsma1/rsma2/qtask for rsma if (pWriter->pRsmaSnapWriter == NULL) { code = rsmaSnapWriterOpen(pVnode->pSma, pWriter->sver, pWriter->ever, &pWriter->pRsmaSnapWriter); if (code) goto _err; diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 8aa3a7d296..16bcb967cd 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -378,6 +378,9 @@ static int32_t vnodeProcessTrimReq(SVnode *pVnode, int64_t version, void *pReq, code = tsdbDoRetention(pVnode->pTsdb, trimReq.timestamp); if (code) goto _exit; + code = smaDoRetention(pVnode->pSma, trimReq.timestamp); + if (code) goto _exit; + _exit: return code; } @@ -908,7 +911,7 @@ _exit: static int32_t vnodeProcessCreateTSmaReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { SVCreateTSmaReq req = {0}; - SDecoder coder; + SDecoder coder = {0}; if (pRsp) { pRsp->msgType = TDMT_VND_CREATE_SMA_RSP; diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index 556fd78360..b76769bdb8 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -162,6 +162,26 @@ _err: #endif } +TdFilePtr taosCreateFile(const char *path, int32_t tdFileOptions) { + TdFilePtr fp = taosOpenFile(path, tdFileOptions); + if (!fp) { + if (errno == ENOENT) { + // Try to create directory recursively + char *s = strdup(path); + if (taosMulMkDir(taosDirName(s)) != 0) { + taosMemoryFree(s); + return NULL; + } + taosMemoryFree(s); + fp = taosOpenFile(path, tdFileOptions); + if (!fp) { + return NULL; + } + } + } + return fp; +} + int32_t taosRemoveFile(const char *path) { return remove(path); } int32_t taosRenameFile(const char *oldName, const char *newName) { From 2a17deec7cad108782e98587d1630a0ebc17bad1 Mon Sep 17 00:00:00 2001 From: slzhou Date: Tue, 2 Aug 2022 16:18:55 +0800 Subject: [PATCH 11/18] fix: extract time condition corretly when join tables together --- source/libs/executor/inc/executorimpl.h | 1 + source/libs/executor/src/executorimpl.c | 32 ++++++++-------- source/libs/executor/src/joinoperator.c | 51 ++++++++++++++----------- 3 files changed, 45 insertions(+), 39 deletions(-) diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index d587e201ef..f5a1f34366 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -224,6 +224,7 @@ typedef struct SOperatorInfo { struct SOperatorInfo** pDownstream; // downstram pointer list int32_t numOfDownstream; // number of downstream. The value is always ONE expect for join operator SOperatorFpSet fpSet; + int16_t resultDataBlockId; } SOperatorInfo; typedef enum { diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 9dd028eeb7..2dbe4c184c 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -3934,6 +3934,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo int32_t type = nodeType(pPhyNode); if (pPhyNode->pChildren == NULL || LIST_LENGTH(pPhyNode->pChildren) == 0) { + SOperatorInfo* pOperator = NULL; if (QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN == type) { STableScanPhysiNode* pTableScanNode = (STableScanPhysiNode*)pPhyNode; @@ -3951,11 +3952,9 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo return NULL; } - SOperatorInfo* pOperator = createTableScanOperatorInfo(pTableScanNode, pHandle, pTaskInfo); + pOperator = createTableScanOperatorInfo(pTableScanNode, pHandle, pTaskInfo); STableScanInfo* pScanInfo = pOperator->info; pTaskInfo->cost.pRecoder = &pScanInfo->readRecorder; - return pOperator; - } else if (QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN == type) { STableMergeScanPhysiNode* pTableScanNode = (STableMergeScanPhysiNode*)pPhyNode; int32_t code = @@ -3972,14 +3971,12 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo return NULL; } - SOperatorInfo* pOperator = createTableMergeScanOperatorInfo(pTableScanNode, pTableListInfo, pHandle, pTaskInfo); + pOperator = createTableMergeScanOperatorInfo(pTableScanNode, pTableListInfo, pHandle, pTaskInfo); STableScanInfo* pScanInfo = pOperator->info; pTaskInfo->cost.pRecoder = &pScanInfo->readRecorder; - return pOperator; - } else if (QUERY_NODE_PHYSICAL_PLAN_EXCHANGE == type) { - return createExchangeOperatorInfo(pHandle->pMsgCb->clientRpc, (SExchangePhysiNode*)pPhyNode, pTaskInfo); + pOperator = createExchangeOperatorInfo(pHandle->pMsgCb->clientRpc, (SExchangePhysiNode*)pPhyNode, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN == type) { STableScanPhysiNode* pTableScanNode = (STableScanPhysiNode*)pPhyNode; if (pHandle->vnode) { @@ -4001,12 +3998,10 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo #endif pTaskInfo->schemaInfo.qsw = extractQueriedColumnSchema(&pTableScanNode->scan); - SOperatorInfo* pOperator = createStreamScanOperatorInfo(pHandle, pTableScanNode, pTagCond, pTaskInfo); - return pOperator; - + pOperator = createStreamScanOperatorInfo(pHandle, pTableScanNode, pTagCond, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN == type) { SSystemTableScanPhysiNode* pSysScanPhyNode = (SSystemTableScanPhysiNode*)pPhyNode; - return createSysTableScanOperatorInfo(pHandle, pSysScanPhyNode, pUser, pTaskInfo); + pOperator = createSysTableScanOperatorInfo(pHandle, pSysScanPhyNode, pUser, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN == type) { STagScanPhysiNode* pScanPhyNode = (STagScanPhysiNode*)pPhyNode; int32_t code = getTableList(pHandle->meta, pHandle->vnode, pScanPhyNode, pTagCond, pTagIndexCond, pTableListInfo); @@ -4015,7 +4010,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo return NULL; } - return createTagScanOperatorInfo(pHandle, pScanPhyNode, pTableListInfo, pTaskInfo); + pOperator = createTagScanOperatorInfo(pHandle, pScanPhyNode, pTableListInfo, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN == type) { SBlockDistScanPhysiNode* pBlockNode = (SBlockDistScanPhysiNode*)pPhyNode; pTableListInfo->pTableList = taosArrayInit(4, sizeof(STableKeyInfo)); @@ -4041,7 +4036,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo tsdbReaderOpen(pHandle->vnode, &cond, pTableListInfo->pTableList, &pReader, ""); cleanupQueryTableDataCond(&cond); - return createDataBlockInfoScanOperator(pReader, pHandle, cond.suid, pBlockNode, pTaskInfo); + pOperator = createDataBlockInfoScanOperator(pReader, pHandle, cond.suid, pBlockNode, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN == type) { SLastRowScanPhysiNode* pScanNode = (SLastRowScanPhysiNode*)pPhyNode; @@ -4058,12 +4053,14 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo return NULL; } - return createLastrowScanOperator(pScanNode, pHandle, pTaskInfo); + pOperator = createLastrowScanOperator(pScanNode, pHandle, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_PROJECT == type) { - return createProjectOperatorInfo(NULL, (SProjectPhysiNode*)pPhyNode, pTaskInfo); + pOperator = createProjectOperatorInfo(NULL, (SProjectPhysiNode*)pPhyNode, pTaskInfo); } else { ASSERT(0); } + pOperator->resultDataBlockId = pPhyNode->pOutputDataBlockDesc->dataBlockId; + return pOperator; } int32_t num = 0; @@ -4075,6 +4072,8 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo ops[i] = createOperatorTree(pChildNode, pTaskInfo, pHandle, pTableListInfo, pTagCond, pTagIndexCond, pUser); if (ops[i] == NULL) { return NULL; + } else { + ops[i]->resultDataBlockId = pChildNode->pOutputDataBlockDesc->dataBlockId; } } @@ -4217,8 +4216,9 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo } else { ASSERT(0); } - taosMemoryFree(ops); + + pOptr->resultDataBlockId = pPhyNode->pOutputDataBlockDesc->dataBlockId; return pOptr; } diff --git a/source/libs/executor/src/joinoperator.c b/source/libs/executor/src/joinoperator.c index 4134ce5dbf..659940c696 100644 --- a/source/libs/executor/src/joinoperator.c +++ b/source/libs/executor/src/joinoperator.c @@ -26,7 +26,33 @@ static void setJoinColumnInfo(SColumnInfo* pColumn, const SColumnNode* pColumnNode); static SSDataBlock* doMergeJoin(struct SOperatorInfo* pOperator); static void destroyMergeJoinOperator(void* param, int32_t numOfOutput); -static void extractTimeCondition(SJoinOperatorInfo* Info, SLogicConditionNode* pLogicConditionNode); +static void extractTimeCondition(SJoinOperatorInfo* pInfo, SOperatorInfo** pDownstream, int32_t numOfDownstream, + SSortMergeJoinPhysiNode* pJoinNode); + +static void extractTimeCondition(SJoinOperatorInfo* pInfo, SOperatorInfo** pDownstream, int32_t numOfDownstream, SSortMergeJoinPhysiNode* pJoinNode) { + SNode* pMergeCondition = pJoinNode->pMergeCondition; + if (nodeType(pMergeCondition) == QUERY_NODE_OPERATOR) { + SOperatorNode* pNode = (SOperatorNode*)pMergeCondition; + SColumnNode* col1 = (SColumnNode*)pNode->pLeft; + SColumnNode* col2 = (SColumnNode*)pNode->pRight; + SColumnNode* leftTsCol = NULL; + SColumnNode* rightTsCol = NULL; + if (col1->dataBlockId == pDownstream[0]->resultDataBlockId) { + ASSERT(col2->dataBlockId == pDownstream[1]->resultDataBlockId); + leftTsCol = col1; + rightTsCol = col2; + } else { + ASSERT(col1->dataBlockId == pDownstream[1]->resultDataBlockId); + ASSERT(col2->dataBlockId == pDownstream[0]->resultDataBlockId); + leftTsCol = col2; + rightTsCol = col1; + } + setJoinColumnInfo(&pInfo->leftCol, leftTsCol); + setJoinColumnInfo(&pInfo->rightCol, rightTsCol); + } else { + ASSERT(false); + } +} SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t numOfDownstream, SSortMergeJoinPhysiNode* pJoinNode, SExecTaskInfo* pTaskInfo) { @@ -53,14 +79,7 @@ SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t pOperator->info = pInfo; pOperator->pTaskInfo = pTaskInfo; - SNode* pMergeCondition = pJoinNode->pMergeCondition; - if (nodeType(pMergeCondition) == QUERY_NODE_OPERATOR) { - SOperatorNode* pNode = (SOperatorNode*)pMergeCondition; - setJoinColumnInfo(&pInfo->leftCol, (SColumnNode*)pNode->pLeft); - setJoinColumnInfo(&pInfo->rightCol, (SColumnNode*)pNode->pRight); - } else { - ASSERT(false); - } + extractTimeCondition(pInfo, pDownstream, numOfDownstream, pJoinNode); if (pJoinNode->pOnConditions != NULL && pJoinNode->node.pConditions != NULL) { pInfo->pCondAfterMerge = nodesMakeNode(QUERY_NODE_LOGIC_CONDITION); @@ -367,17 +386,3 @@ SSDataBlock* doMergeJoin(struct SOperatorInfo* pOperator) { } return (pRes->info.rows > 0) ? pRes : NULL; } - -static void extractTimeCondition(SJoinOperatorInfo* pInfo, SLogicConditionNode* pLogicConditionNode) { - int32_t len = LIST_LENGTH(pLogicConditionNode->pParameterList); - - for (int32_t i = 0; i < len; ++i) { - SNode* pNode = nodesListGetNode(pLogicConditionNode->pParameterList, i); - if (nodeType(pNode) == QUERY_NODE_OPERATOR) { - SOperatorNode* pn1 = (SOperatorNode*)pNode; - setJoinColumnInfo(&pInfo->leftCol, (SColumnNode*)pn1->pLeft); - setJoinColumnInfo(&pInfo->rightCol, (SColumnNode*)pn1->pRight); - break; - } - } -} From c0d0efffef2bd24ff327230ad6f2addfe183ac57 Mon Sep 17 00:00:00 2001 From: slzhou Date: Tue, 2 Aug 2022 16:21:19 +0800 Subject: [PATCH 12/18] fix: fix multi table join test case --- tests/script/tsim/parser/join_multitables.sim | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/script/tsim/parser/join_multitables.sim b/tests/script/tsim/parser/join_multitables.sim index 3f5526d16f..4278be52f3 100644 --- a/tests/script/tsim/parser/join_multitables.sim +++ b/tests/script/tsim/parser/join_multitables.sim @@ -1302,25 +1302,25 @@ if $data09 != 9925 then endi sql_error select tb0_1.*, tb1_1.* from tb0_1, tb1_1 where tb0_1.f1=tb1_1.f1; -sql_error select tb0_1.*, tb1_1.* from tb0_1, tb1_1 where tb0_1.ts=tb1_1.ts and tb0_1.id1=tb1_1.id2; -sql_error select tb0_5.*, tb1_5.*,tb2_5.*,tb3_5.*,tb4_5.*,tb5_5.*, tb6_5.*,tb7_5.*,tb8_5.*,tb9_5.*,tba_5.* from tb0_5, tb1_5, tb2_5, tb3_5, tb4_5,tb5_5, tb6_5, tb7_5, tb8_5, tb9_5, tba_5 where tb9_5.ts=tb8_5.ts and tb8_5.ts=tb7_5.ts and tb7_5.ts=tb6_5.ts and tb6_5.ts=tb5_5.ts and tb5_5.ts=tb4_5.ts and tb4_5.ts=tb3_5.ts and tb3_5.ts=tb2_5.ts and tb2_5.ts=tb1_5.ts and tb1_5.ts=tb0_5.ts and tb0_5.ts=tba_5.ts; +sql select tb0_1.*, tb1_1.* from tb0_1, tb1_1 where tb0_1.ts=tb1_1.ts and tb0_1.id1=tb1_1.id2; +sql select tb0_5.*, tb1_5.*,tb2_5.*,tb3_5.*,tb4_5.*,tb5_5.*, tb6_5.*,tb7_5.*,tb8_5.*,tb9_5.*,tba_5.* from tb0_5, tb1_5, tb2_5, tb3_5, tb4_5,tb5_5, tb6_5, tb7_5, tb8_5, tb9_5, tba_5 where tb9_5.ts=tb8_5.ts and tb8_5.ts=tb7_5.ts and tb7_5.ts=tb6_5.ts and tb6_5.ts=tb5_5.ts and tb5_5.ts=tb4_5.ts and tb4_5.ts=tb3_5.ts and tb3_5.ts=tb2_5.ts and tb2_5.ts=tb1_5.ts and tb1_5.ts=tb0_5.ts and tb0_5.ts=tba_5.ts; -sql_error select * from st0, st1 where st0.ts=st1.ts; +sql select * from st0, st1 where st0.ts=st1.ts; sql_error select * from st0, st1 where st0.id1=st1.id1; sql_error select * from st0, st1 where st0.f1=st1.f1 and st0.id1=st1.id1; -sql_error select * from st0, st1, st2, st3 where st0.id1=st1.id1 and st2.id1=st3.id1 and st0.ts=st1.ts and st1.ts=st2.ts and st2.ts=st3.ts; +sql select * from st0, st1, st2, st3 where st0.id1=st1.id1 and st2.id1=st3.id1 and st0.ts=st1.ts and st1.ts=st2.ts and st2.ts=st3.ts; sql_error select * from st0, st1, st2 where st0.id1=st1.id1; sql_error select * from st0, st1 where st0.id1=st1.id1 and st0.id2=st1.id3; sql_error select * from st0, st1 where st0.id1=st1.id1 or st0.ts=st1.ts; sql_error select * from st0, st1 where st0.ts=st1.ts and st0.id1=st1.id1 or st0.id2=st1.id2; sql_error select * from st0, st1, st2 where st0.ts=st1.ts and st0.id1=st1.id1; sql_error select * from st0, st1 where st0.id1=st1.ts and st0.ts=st1.id1; -sql_error select * from st0, st1 where st0.id1=st1.id2 and st0.ts=st1.ts; -sql_error select * from st0, st1 where st1.id4=st0.id4 and st1.ts=st0.ts; -sql_error select * from st0, st1 where st0.id1=st1.id2 and st1.ts=st0.ts; +sql select * from st0, st1 where st0.id1=st1.id2 and st0.ts=st1.ts; +sql select * from st0, st1 where st1.id4=st0.id4 and st1.ts=st0.ts; +sql select * from st0, st1 where st0.id1=st1.id2 and st1.ts=st0.ts; sql_error select * from st0, st1 where st0.ts=st1.ts and st0.id1=st1.id1 interval 10a; sql_error select last(*) from st0, st1 where st0.ts=st1.ts and st0.id1=st1.id1 group by f1; sql_error select st0.*,st1.*,st2.*,st3.*,st4.*,st5.*,st6.*,st7.*,st8.*,st9.* from st0,st1,st2,st3,st4,st5,st6,st7,st8,st9 where st0.ts=st2.ts and st0.ts=st4.ts and st0.ts=st6.ts and st0.ts=st8.ts and st1.ts=st3.ts and st3.ts=st5.ts and st5.ts=st7.ts and st7.ts=st9.ts and st0.id1=st2.id1 and st0.id1=st4.id1 and st0.id1=st6.id1 and st0.id1=st8.id1 and st1.id1=st3.id1 and st3.id1=st5.id1 and st5.id1=st7.id1 and st7.id1=st9.id1; -sql_error select st0.*,st1.*,st2.*,st3.*,st4.*,st5.*,st6.*,st7.*,st8.*,st9.* from st0,st1,st2,st3,st4,st5,st6,st7,st8,st9,sta where st0.ts=st2.ts and st0.ts=st4.ts and st0.ts=st6.ts and st0.ts=st8.ts and st1.ts=st3.ts and st3.ts=st5.ts and st5.ts=st7.ts and st7.ts=st9.ts and st0.ts=st1.ts and st0.id1=st2.id1 and st0.id1=st4.id1 and st0.id1=st6.id1 and st0.id1=st8.id1 and st1.id1=st3.id1 and st3.id1=st5.id1 and st5.id1=st7.id1 and st7.id1=st9.id1 and st0.id1=st1.id1 and st0.id1=sta.id1 and st0.ts=sta.ts; +sql select st0.*,st1.*,st2.*,st3.*,st4.*,st5.*,st6.*,st7.*,st8.*,st9.* from st0,st1,st2,st3,st4,st5,st6,st7,st8,st9,sta where st0.ts=st2.ts and st0.ts=st4.ts and st0.ts=st6.ts and st0.ts=st8.ts and st1.ts=st3.ts and st3.ts=st5.ts and st5.ts=st7.ts and st7.ts=st9.ts and st0.ts=st1.ts and st0.id1=st2.id1 and st0.id1=st4.id1 and st0.id1=st6.id1 and st0.id1=st8.id1 and st1.id1=st3.id1 and st3.id1=st5.id1 and st5.id1=st7.id1 and st7.id1=st9.id1 and st0.id1=st1.id1 and st0.id1=sta.id1 and st0.ts=sta.ts; system sh/exec.sh -n dnode1 -s stop -x SIGINT From 9af4edc3e857417d6ed71dabbf8b590fcf72b9f0 Mon Sep 17 00:00:00 2001 From: slzhou Date: Tue, 2 Aug 2022 16:22:17 +0800 Subject: [PATCH 13/18] fix: add join multi table test case --- tests/script/jenkins/basic.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 910b99ace3..f60d5c6e79 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -131,7 +131,7 @@ ./test.sh -f tsim/parser/insert_tb.sim # TD-17038 ./test.sh -f tsim/parser/interp.sim ./test.sh -f tsim/parser/join_manyblocks.sim -# TD-18018 ./test.sh -f tsim/parser/join_multitables.sim +./test.sh -f tsim/parser/join_multitables.sim ./test.sh -f tsim/parser/join_multivnode.sim ./test.sh -f tsim/parser/join.sim ./test.sh -f tsim/parser/last_cache.sim From a3f95f566ff5505f363ed7fec4419d9b52f41077 Mon Sep 17 00:00:00 2001 From: slzhou Date: Tue, 2 Aug 2022 16:24:22 +0800 Subject: [PATCH 14/18] fix: format code --- source/libs/executor/src/joinoperator.c | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/source/libs/executor/src/joinoperator.c b/source/libs/executor/src/joinoperator.c index 659940c696..e96844f1e3 100644 --- a/source/libs/executor/src/joinoperator.c +++ b/source/libs/executor/src/joinoperator.c @@ -29,14 +29,15 @@ static void destroyMergeJoinOperator(void* param, int32_t numOfOutput); static void extractTimeCondition(SJoinOperatorInfo* pInfo, SOperatorInfo** pDownstream, int32_t numOfDownstream, SSortMergeJoinPhysiNode* pJoinNode); -static void extractTimeCondition(SJoinOperatorInfo* pInfo, SOperatorInfo** pDownstream, int32_t numOfDownstream, SSortMergeJoinPhysiNode* pJoinNode) { +static void extractTimeCondition(SJoinOperatorInfo* pInfo, SOperatorInfo** pDownstream, int32_t numOfDownstream, + SSortMergeJoinPhysiNode* pJoinNode) { SNode* pMergeCondition = pJoinNode->pMergeCondition; if (nodeType(pMergeCondition) == QUERY_NODE_OPERATOR) { SOperatorNode* pNode = (SOperatorNode*)pMergeCondition; - SColumnNode* col1 = (SColumnNode*)pNode->pLeft; - SColumnNode* col2 = (SColumnNode*)pNode->pRight; - SColumnNode* leftTsCol = NULL; - SColumnNode* rightTsCol = NULL; + SColumnNode* col1 = (SColumnNode*)pNode->pLeft; + SColumnNode* col2 = (SColumnNode*)pNode->pRight; + SColumnNode* leftTsCol = NULL; + SColumnNode* rightTsCol = NULL; if (col1->dataBlockId == pDownstream[0]->resultDataBlockId) { ASSERT(col2->dataBlockId == pDownstream[1]->resultDataBlockId); leftTsCol = col1; @@ -192,16 +193,16 @@ static int32_t mergeJoinGetBlockRowsEqualTs(SSDataBlock* pBlock, int16_t tsSlotI } SSDataBlock* block = pBlock; - bool createdNewBlock = false; + bool createdNewBlock = false; if (endPos == numRows) { - block = blockDataExtractBlock(pBlock, startPos, endPos-startPos); + block = blockDataExtractBlock(pBlock, startPos, endPos - startPos); taosArrayPush(createdBlocks, &block); createdNewBlock = true; } SRowLocation location = {0}; for (int32_t j = startPos; j < endPos; ++j) { location.pDataBlock = block; - location.pos = ( createdNewBlock ? j - startPos : j); + location.pos = (createdNewBlock ? j - startPos : j); taosArrayPush(rowLocations, &location); } return 0; From d5aea7dac23a098f17b17c68f3f166fcfd06739f Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 2 Aug 2022 15:57:37 +0800 Subject: [PATCH 15/18] refactor: adjust sync log --- source/dnode/mgmt/mgmt_vnode/src/vmWorker.c | 2 +- source/dnode/vnode/src/meta/metaOpen.c | 2 +- source/dnode/vnode/src/meta/metaQuery.c | 4 +- source/dnode/vnode/src/meta/metaSma.c | 4 +- source/dnode/vnode/src/meta/metaTable.c | 8 ++-- source/dnode/vnode/src/tsdb/tsdbOpen.c | 2 +- source/dnode/vnode/src/tsdb/tsdbRead.c | 4 +- source/dnode/vnode/src/vnd/vnodeBufPool.c | 2 +- source/dnode/vnode/src/vnd/vnodeCommit.c | 8 ++-- source/dnode/vnode/src/vnd/vnodeModule.c | 2 +- source/dnode/vnode/src/vnd/vnodeOpen.c | 4 +- source/dnode/vnode/src/vnd/vnodeSvr.c | 4 +- source/dnode/vnode/src/vnd/vnodeSync.c | 19 ++++----- source/libs/index/src/index.c | 4 +- source/libs/index/src/indexTfile.c | 8 ++-- source/libs/sync/src/syncMain.c | 40 +++++++++--------- source/libs/sync/src/syncRaftLog.c | 8 ++-- source/libs/sync/src/syncRespMgr.c | 15 +++---- tests/script/tmp/r1.sim | 47 +++++++++++++++++++++ 19 files changed, 114 insertions(+), 73 deletions(-) create mode 100644 tests/script/tmp/r1.sim diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c index 9d1142801d..b41633b0a8 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c @@ -146,7 +146,7 @@ static int32_t vmPutMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType qtyp SVnodeObj *pVnode = vmAcquireVnode(pMgmt, pHead->vgId); if (pVnode == NULL) { - dGError("vgId:%d, msg:%p failed to put into vnode queue since %s, msgtype:%s qtype:%d", pHead->vgId, pMsg, + dGError("vgId:%d, msg:%p failed to put into vnode queue since %s, type:%s qtype:%d", pHead->vgId, pMsg, terrstr(), TMSG_INFO(pMsg->msgType), qtype); return terrno != 0 ? terrno : -1; } diff --git a/source/dnode/vnode/src/meta/metaOpen.c b/source/dnode/vnode/src/meta/metaOpen.c index 85293eff30..2f026e71eb 100644 --- a/source/dnode/vnode/src/meta/metaOpen.c +++ b/source/dnode/vnode/src/meta/metaOpen.c @@ -133,7 +133,7 @@ int metaOpen(SVnode *pVnode, SMeta **ppMeta) { ret = tdbTbOpen("stream.task.db", sizeof(int64_t), -1, taskIdxKeyCmpr, pMeta->pEnv, &pMeta->pStreamDb); if (ret < 0) { - metaError("vgId: %d, failed to open meta stream task index since %s", TD_VID(pVnode), tstrerror(terrno)); + metaError("vgIdd, failed to open meta stream task index since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index dc16c2321b..b72195d333 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -599,7 +599,7 @@ STSmaWrapper *metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid, bool deepCopy) { for (int i = 0; i < pSW->number; ++i) { smaId = *(tb_uid_t *)taosArrayGet(pSmaIds, i); if (metaGetTableEntryByUid(&mr, smaId) < 0) { - metaWarn("vgId:%d, no entry for tbId: %" PRIi64 ", smaId: %" PRIi64, TD_VID(pMeta->pVnode), uid, smaId); + metaWarn("vgId:%d, no entry for tbId" PRIi64 ", smaId" PRIi64, TD_VID(pMeta->pVnode), uid, smaId); continue; } pTSma = pSW->tSma + smaIdx; @@ -647,7 +647,7 @@ STSma *metaGetSmaInfoByIndex(SMeta *pMeta, int64_t indexUid) { SMetaReader mr = {0}; metaReaderInit(&mr, pMeta, 0); if (metaGetTableEntryByUid(&mr, indexUid) < 0) { - metaWarn("vgId:%d, failed to get table entry for smaId: %" PRIi64, TD_VID(pMeta->pVnode), indexUid); + metaWarn("vgId:%d, failed to get table entry for smaId" PRIi64, TD_VID(pMeta->pVnode), indexUid); metaReaderClear(&mr); return NULL; } diff --git a/source/dnode/vnode/src/meta/metaSma.c b/source/dnode/vnode/src/meta/metaSma.c index 0b6a526d8c..4f0d0341bc 100644 --- a/source/dnode/vnode/src/meta/metaSma.c +++ b/source/dnode/vnode/src/meta/metaSma.c @@ -57,12 +57,12 @@ int32_t metaCreateTSma(SMeta *pMeta, int64_t version, SSmaCfg *pCfg) { if (metaHandleSmaEntry(pMeta, &me) < 0) goto _err; - metaDebug("vgId:%d, tsma is created, name:%s uid: %" PRId64, TD_VID(pMeta->pVnode), pCfg->indexName, pCfg->indexUid); + metaDebug("vgId:%d, tsma is created, name:%s uid:%" PRId64, TD_VID(pMeta->pVnode), pCfg->indexName, pCfg->indexUid); return 0; _err: - metaError("vgId:%d, failed to create tsma: %s uid: %" PRId64 " since %s", TD_VID(pMeta->pVnode), pCfg->indexName, + metaError("vgId:%d, failed to create tsmas uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pCfg->indexName, pCfg->indexUid, tstrerror(terrno)); return -1; } diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 7236ef9991..3f47128a4b 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -204,12 +204,12 @@ int metaCreateSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { ++pMeta->pVnode->config.vndStats.numOfSTables; - metaDebug("vgId:%d, super table is created, name:%s uid: %" PRId64, TD_VID(pMeta->pVnode), pReq->name, pReq->suid); + metaDebug("vgId:%d, super table is created, name:%s uid:%" PRId64, TD_VID(pMeta->pVnode), pReq->name, pReq->suid); return 0; _err: - metaError("vgId:%d, failed to create super table: %s uid: %" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, + metaError("vgId:%d, failed to create super tables uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, pReq->suid, tstrerror(terrno)); return -1; } @@ -996,7 +996,7 @@ static int metaSaveToTbDb(SMeta *pMeta, const SMetaEntry *pME) { tbDbKey.version = pME->version; tbDbKey.uid = pME->uid; - metaDebug("vgId:%d, start to save table version:%" PRId64 "uid: %" PRId64, TD_VID(pMeta->pVnode), pME->version, + metaDebug("vgId:%d, start to save table version:%" PRId64 " uid:%" PRId64, TD_VID(pMeta->pVnode), pME->version, pME->uid); pKey = &tbDbKey; @@ -1031,7 +1031,7 @@ static int metaSaveToTbDb(SMeta *pMeta, const SMetaEntry *pME) { return 0; _err: - metaError("vgId:%d, failed to save table version:%" PRId64 "uid: %" PRId64 " %s", TD_VID(pMeta->pVnode), pME->version, + metaError("vgId:%d, failed to save table version:%" PRId64 "uid:%" PRId64 " %s", TD_VID(pMeta->pVnode), pME->version, pME->uid, tstrerror(terrno)); taosMemoryFree(pVal); diff --git a/source/dnode/vnode/src/tsdb/tsdbOpen.c b/source/dnode/vnode/src/tsdb/tsdbOpen.c index 3c832e9c2a..be2828d187 100644 --- a/source/dnode/vnode/src/tsdb/tsdbOpen.c +++ b/source/dnode/vnode/src/tsdb/tsdbOpen.c @@ -71,7 +71,7 @@ int tsdbOpen(SVnode *pVnode, STsdb **ppTsdb, const char *dir, STsdbKeepCfg *pKee goto _err; } - tsdbDebug("vgId:%d, tsdb is opened for %s, days:%d, keep:%d,%d,%d", TD_VID(pVnode), pTsdb->path, pTsdb->keepCfg.days, + tsdbDebug("vgId:%d, tsdb is opened at %s, days:%d, keep:%d,%d,%d", TD_VID(pVnode), pTsdb->path, pTsdb->keepCfg.days, pTsdb->keepCfg.keep0, pTsdb->keepCfg.keep1, pTsdb->keepCfg.keep2); *ppTsdb = pTsdb; diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index bd1b2db145..f6e01f04cd 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -1197,7 +1197,7 @@ static int32_t buildDataBlockFromBuf(STsdbReader* pReader, STableBlockScanInfo* setComposedBlockFlag(pReader, true); double elapsedTime = (taosGetTimestampUs() - st) / 1000.0; - tsdbDebug("%p build data block from cache completed, elapsed time:%.2f ms, numOfRows:%d, brange: %" PRId64 + tsdbDebug("%p build data block from cache completed, elapsed time:%.2f ms, numOfRows:%d, brange" PRId64 " - %" PRId64 " %s", pReader, elapsedTime, pBlock->info.rows, pBlock->info.window.skey, pBlock->info.window.ekey, pReader->idStr); @@ -2647,7 +2647,7 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, SArray* pTabl return code; _err: - tsdbError("failed to create data reader, code: %s %s", tstrerror(code), pReader->idStr); + tsdbError("failed to create data reader, codes %s", tstrerror(code), pReader->idStr); return code; } diff --git a/source/dnode/vnode/src/vnd/vnodeBufPool.c b/source/dnode/vnode/src/vnd/vnodeBufPool.c index d1584f701e..0623b3bd10 100644 --- a/source/dnode/vnode/src/vnd/vnodeBufPool.c +++ b/source/dnode/vnode/src/vnd/vnodeBufPool.c @@ -40,7 +40,7 @@ int vnodeOpenBufPool(SVnode *pVnode, int64_t size) { pVnode->pPool = pPool; } - vDebug("vgId:%d, vnode buffer pool is opened, pool size: %" PRId64, TD_VID(pVnode), size); + vDebug("vgId:%d, vnode buffer pool is opened, size:%" PRId64, TD_VID(pVnode), size); return 0; } diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c index e8d029a88f..650c98eb88 100644 --- a/source/dnode/vnode/src/vnd/vnodeCommit.c +++ b/source/dnode/vnode/src/vnd/vnodeCommit.c @@ -95,19 +95,19 @@ int vnodeSaveInfo(const char *dir, const SVnodeInfo *pInfo) { // save info to a vnode_tmp.json pFile = taosOpenFile(fname, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); if (pFile == NULL) { - vError("failed to open info file: %s for write: %s", fname, terrstr()); + vError("failed to open info file:%s for write:%s", fname, terrstr()); terrno = TAOS_SYSTEM_ERROR(errno); return -1; } if (taosWriteFile(pFile, data, strlen(data)) < 0) { - vError("failed to write info file: %s data: %s", fname, terrstr()); + vError("failed to write info file:%s data:%s", fname, terrstr()); terrno = TAOS_SYSTEM_ERROR(errno); goto _err; } if (taosFsyncFile(pFile) < 0) { - vError("failed to fsync info file: %s error: %s", fname, terrstr()); + vError("failed to fsync info file:%s error:%s", fname, terrstr()); terrno = TAOS_SYSTEM_ERROR(errno); goto _err; } @@ -117,7 +117,7 @@ int vnodeSaveInfo(const char *dir, const SVnodeInfo *pInfo) { // free info binary taosMemoryFree(data); - vInfo("vgId:%d, vnode info is saved, fname: %s", pInfo->config.vgId, fname); + vInfo("vgId:%d, vnode info is saved, fname:%s", pInfo->config.vgId, fname); return 0; diff --git a/source/dnode/vnode/src/vnd/vnodeModule.c b/source/dnode/vnode/src/vnd/vnodeModule.c index 69f87a2c1a..314c0e7390 100644 --- a/source/dnode/vnode/src/vnd/vnodeModule.c +++ b/source/dnode/vnode/src/vnd/vnodeModule.c @@ -55,7 +55,7 @@ int vnodeInit(int nthreads) { vnodeGlobal.threads = taosMemoryCalloc(nthreads, sizeof(TdThread)); if (vnodeGlobal.threads == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; - vError("failed to init vnode module since: %s", tstrerror(terrno)); + vError("failed to init vnode module since:%s", tstrerror(terrno)); return -1; } diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index e273a41f48..1ba74ac3be 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -23,13 +23,13 @@ int vnodeCreate(const char *path, SVnodeCfg *pCfg, STfs *pTfs) { // check config if (vnodeCheckCfg(pCfg) < 0) { - vError("vgId:%d, failed to create vnode since: %s", pCfg->vgId, tstrerror(terrno)); + vError("vgId:%d, failed to create vnode since:%s", pCfg->vgId, tstrerror(terrno)); return -1; } // create vnode env if (tfsMkdirAt(pTfs, path, (SDiskID){0}) < 0) { - vError("vgId:%d, failed to create vnode since: %s", pCfg->vgId, tstrerror(terrno)); + vError("vgId:%d, failed to create vnode since:%s", pCfg->vgId, tstrerror(terrno)); return -1; } diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 8aa3a7d296..3fc66b4bf2 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -262,7 +262,7 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp return 0; _err: - vError("vgId:%d, process %s request failed since %s, version: %" PRId64, TD_VID(pVnode), TMSG_INFO(pMsg->msgType), + vError("vgId:%d, process %s request failed since %s, version" PRId64, TD_VID(pVnode), TMSG_INFO(pMsg->msgType), tstrerror(terrno), version); return -1; } @@ -296,7 +296,7 @@ int32_t vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg) { } int32_t vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) { - vTrace("message in fetch queue is processing"); + vTrace("vgId:%d, msg:%p in fetch queue is processing", pVnode->config.vgId, pMsg); if ((pMsg->msgType == TDMT_SCH_FETCH || pMsg->msgType == TDMT_VND_TABLE_META || pMsg->msgType == TDMT_VND_TABLE_CFG || pMsg->msgType == TDMT_VND_BATCH_META) && !vnodeIsLeader(pVnode)) { diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index 99173e87d7..6eb9c402ac 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -518,15 +518,15 @@ static void vnodeSyncCommitMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta c rpcMsg.info.conn.applyTerm = cbMeta.term; vInfo("vgId:%d, commit-cb is excuted, fsm:%p, index:%" PRId64 ", term:%" PRIu64 ", msg-index:%" PRId64 - ", isWeak:%d, code:%d, state:%d %s, msgtype:%d %s", + ", weak:%d, code:%d, state:%d %s, type:%s", syncGetVgId(pVnode->sync), pFsm, cbMeta.index, cbMeta.term, rpcMsg.info.conn.applyIndex, cbMeta.isWeak, - cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state), pMsg->msgType, TMSG_INFO(pMsg->msgType)); + cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state), TMSG_INFO(pMsg->msgType)); tmsgPutToQueue(&pVnode->msgCb, APPLY_QUEUE, &rpcMsg); } else { SRpcMsg rsp = {.code = cbMeta.code, .info = pMsg->info}; - vError("vgId:%d, sync commit error, msgtype:%d,%s, index:%ld, error:0x%X, errmsg:%s", syncGetVgId(pVnode->sync), - pMsg->msgType, TMSG_INFO(pMsg->msgType), cbMeta.index, cbMeta.code, tstrerror(cbMeta.code)); + vError("vgId:%d, commit-cb execute error, type:%s, index:%" PRId64 ", error:0x%x %s", syncGetVgId(pVnode->sync), + TMSG_INFO(pMsg->msgType), cbMeta.index, cbMeta.code, tstrerror(cbMeta.code)); if (rsp.info.handle != NULL) { tmsgSendRsp(&rsp); } @@ -537,10 +537,9 @@ static void vnodeSyncCommitMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta c static void vnodeSyncPreCommitMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) { if (cbMeta.isWeak == 1) { SVnode *pVnode = pFsm->data; - vTrace("vgId:%d, pre-commit-cb is excuted, fsm:%p, index:%" PRId64 - ", isWeak:%d, code:%d, state:%d %s, msgtype:%d %s", + vTrace("vgId:%d, pre-commit-cb is excuted, fsm:%p, index:%" PRId64 ", weak:%d, code:%d, state:%d %s, type:%s", syncGetVgId(pVnode->sync), pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, - syncUtilState2String(cbMeta.state), pMsg->msgType, TMSG_INFO(pMsg->msgType)); + syncUtilState2String(cbMeta.state), TMSG_INFO(pMsg->msgType)); if (cbMeta.code == 0) { SRpcMsg rpcMsg = {.msgType = pMsg->msgType, .contLen = pMsg->contLen}; @@ -552,8 +551,8 @@ static void vnodeSyncPreCommitMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMet tmsgPutToQueue(&pVnode->msgCb, APPLY_QUEUE, &rpcMsg); } else { SRpcMsg rsp = {.code = cbMeta.code, .info = pMsg->info}; - vError("vgId:%d, sync pre-commit error, msgtype:%d,%s, error:0x%X, errmsg:%s", syncGetVgId(pVnode->sync), - pMsg->msgType, TMSG_INFO(pMsg->msgType), cbMeta.code, tstrerror(cbMeta.code)); + vError("vgId:%d, pre-commit-cb execute error, type:%s, error:0x%x %s", syncGetVgId(pVnode->sync), + TMSG_INFO(pMsg->msgType), cbMeta.code, tstrerror(cbMeta.code)); if (rsp.info.handle != NULL) { tmsgSendRsp(&rsp); } @@ -563,7 +562,7 @@ static void vnodeSyncPreCommitMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMet static void vnodeSyncRollBackMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) { SVnode *pVnode = pFsm->data; - vTrace("vgId:%d, rollback-cb is excuted, fsm:%p, index:%" PRId64 ", isWeak:%d, code:%d, state:%d %s, msgtype:%d %s", + vTrace("vgId:%d, rollback-cb is excuted, fsm:%p, index:%" PRId64 ", isWeak:%d, code:%d, state:%d %s, msgType:%d %s", syncGetVgId(pVnode->sync), pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state), pMsg->msgType, TMSG_INFO(pMsg->msgType)); } diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index 2468dca86c..c61b5be7e9 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -220,7 +220,7 @@ int indexPut(SIndex* index, SIndexMultiTerm* fVals, uint64_t uid) { char buf[128] = {0}; ICacheKey key = {.suid = p->suid, .colName = p->colName, .nColName = strlen(p->colName), .colType = p->colType}; int32_t sz = idxSerialCacheKey(&key, buf); - indexDebug("w suid: %" PRIu64 ", colName: %s, colType: %d", key.suid, key.colName, key.colType); + indexDebug("w suid:%" PRIu64 ", colName: %s, colType: %d", key.suid, key.colName, key.colType); IndexCache** cache = taosHashGet(index->colObj, buf, sz); assert(*cache != NULL); @@ -395,7 +395,7 @@ static int idxTermSearch(SIndex* sIdx, SIndexTermQuery* query, SArray** result) char buf[128] = {0}; ICacheKey key = { .suid = term->suid, .colName = term->colName, .nColName = strlen(term->colName), .colType = term->colType}; - indexDebug("r suid: %" PRIu64 ", colName: %s, colType: %d", key.suid, key.colName, key.colType); + indexDebug("r suid:%" PRIu64 ", colName: %s, colType: %d", key.suid, key.colName, key.colType); int32_t sz = idxSerialCacheKey(&key, buf); taosThreadMutexLock(&sIdx->mtx); diff --git a/source/libs/index/src/indexTfile.c b/source/libs/index/src/indexTfile.c index b4e59d0f63..59c8a7bfb2 100644 --- a/source/libs/index/src/indexTfile.c +++ b/source/libs/index/src/indexTfile.c @@ -141,7 +141,7 @@ void tfileCacheDestroy(TFileCache* tcache) { TFileReader** reader = taosHashIterate(tcache->tableCache, NULL); while (reader) { TFileReader* p = *reader; - indexInfo("drop table cache suid: %" PRIu64 ", colName: %s, colType: %d", p->header.suid, p->header.colName, + indexInfo("drop table cache suid:%" PRIu64 ", colName: %s, colType: %d", p->header.suid, p->header.colName, p->header.colType); tfileReaderUnRef(p); reader = taosHashIterate(tcache->tableCache, reader); @@ -185,20 +185,20 @@ TFileReader* tfileReaderCreate(IFileCtx* ctx) { reader->ctx = ctx; if (0 != tfileReaderVerify(reader)) { - indexError("invalid tfile, suid: %" PRIu64 ", colName: %s", reader->header.suid, reader->header.colName); + indexError("invalid tfile, suid:%" PRIu64 ", colName: %s", reader->header.suid, reader->header.colName); tfileReaderDestroy(reader); return NULL; } // T_REF_INC(reader); if (0 != tfileReaderLoadHeader(reader)) { - indexError("failed to load index header, suid: %" PRIu64 ", colName: %s", reader->header.suid, + indexError("failed to load index header, suid:%" PRIu64 ", colName: %s", reader->header.suid, reader->header.colName); tfileReaderDestroy(reader); return NULL; } if (0 != tfileReaderLoadFst(reader)) { - indexError("failed to load index fst, suid: %" PRIu64 ", colName: %s, errno: %d", reader->header.suid, + indexError("failed to load index fst, suid:%" PRIu64 ", colName: %s, errno: %d", reader->header.suid, reader->header.colName, errno); tfileReaderDestroy(reader); return NULL; diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 4e3c9bf73d..ca03d54675 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -94,7 +94,7 @@ int64_t syncOpen(const SSyncInfo* pSyncInfo) { return -1; } - sDebug("vgId:%d, rid:%" PRId64 " is added to rsetId:%" PRId64, pSyncInfo->vgId, pSyncNode->rid, tsNodeRefId); + sDebug("vgId:%d, sync rid:%" PRId64 " is added to rsetId:%" PRId64, pSyncInfo->vgId, pSyncNode->rid, tsNodeRefId); return pSyncNode->rid; } @@ -142,7 +142,7 @@ void syncStop(int64_t rid) { taosReleaseRef(tsNodeRefId, pSyncNode->rid); taosRemoveRef(tsNodeRefId, rid); - sDebug("vgId:%d, rid:%" PRId64 " is removed from rsetId:%" PRId64, vgId, rid, tsNodeRefId); + sDebug("vgId:%d, sync rid:%" PRId64 " is removed from rsetId:%" PRId64, vgId, rid, tsNodeRefId); } int32_t syncSetStandby(int64_t rid) { @@ -730,8 +730,7 @@ int32_t syncNodeProposeBatch(SSyncNode* pSyncNode, SRpcMsg** pMsgPArr, bool* pIs for (int i = 0; i < arrSize; ++i) { do { char eventLog[128]; - snprintf(eventLog, sizeof(eventLog), "propose type:%s,%d, batch:%d", TMSG_INFO(pMsgPArr[i]->msgType), - pMsgPArr[i]->msgType, arrSize); + snprintf(eventLog, sizeof(eventLog), "propose type:%s, batch:%d", TMSG_INFO(pMsgPArr[i]->msgType), arrSize); syncNodeEventLog(pSyncNode, eventLog); } while (0); @@ -791,7 +790,7 @@ int32_t syncNodePropose(SSyncNode* pSyncNode, SRpcMsg* pMsg, bool isWeak) { do { char eventLog[128]; - snprintf(eventLog, sizeof(eventLog), "propose type:%s,%d", TMSG_INFO(pMsg->msgType), pMsg->msgType); + snprintf(eventLog, sizeof(eventLog), "propose type:%s", TMSG_INFO(pMsg->msgType)); syncNodeEventLog(pSyncNode, eventLog); } while (0); @@ -799,7 +798,7 @@ int32_t syncNodePropose(SSyncNode* pSyncNode, SRpcMsg* pMsg, bool isWeak) { if (pSyncNode->changing && pMsg->msgType != TDMT_SYNC_CONFIG_CHANGE_FINISH) { ret = -1; terrno = TSDB_CODE_SYN_PROPOSE_NOT_READY; - sError("vgId:%d, sync propose not ready, type:%s,%d", pSyncNode->vgId, TMSG_INFO(pMsg->msgType), pMsg->msgType); + sError("vgId:%d, failed to sync propose since not ready, type:%s", pSyncNode->vgId, TMSG_INFO(pMsg->msgType)); goto _END; } @@ -808,8 +807,7 @@ int32_t syncNodePropose(SSyncNode* pSyncNode, SRpcMsg* pMsg, bool isWeak) { if (!syncNodeCanChange(pSyncNode)) { ret = -1; terrno = TSDB_CODE_SYN_RECONFIG_NOT_READY; - sError("vgId:%d, sync reconfig not ready, type:%s,%d", pSyncNode->vgId, TMSG_INFO(pMsg->msgType), - pMsg->msgType); + sError("vgId:%d, failed to sync reconfig since not ready, type:%s", pSyncNode->vgId, TMSG_INFO(pMsg->msgType)); goto _END; } @@ -836,13 +834,12 @@ int32_t syncNodePropose(SSyncNode* pSyncNode, SRpcMsg* pMsg, bool isWeak) { rpcFreeCont(rpcMsg.pCont); syncRespMgrDel(pSyncNode->pSyncRespMgr, seqNum); ret = 1; - sDebug("vgId:%d, optimized index:%" PRId64 " success, msgtype:%s,%d", pSyncNode->vgId, retIndex, - TMSG_INFO(pMsg->msgType), pMsg->msgType); + sDebug("vgId:%d, sync optimize index:%" PRId64 ", type:%s", pSyncNode->vgId, retIndex, TMSG_INFO(pMsg->msgType)); } else { ret = -1; terrno = TSDB_CODE_SYN_INTERNAL_ERROR; - sError("vgId:%d, optimized index:%" PRId64 " error, msgtype:%s,%d", pSyncNode->vgId, retIndex, - TMSG_INFO(pMsg->msgType), pMsg->msgType); + sError("vgId:%d, failed to sync optimize index:%" PRId64 ", type:%s", pSyncNode->vgId, retIndex, + TMSG_INFO(pMsg->msgType)); } } else { @@ -851,7 +848,7 @@ int32_t syncNodePropose(SSyncNode* pSyncNode, SRpcMsg* pMsg, bool isWeak) { } else { ret = -1; terrno = TSDB_CODE_SYN_INTERNAL_ERROR; - sError("vgId:%d, enqueue msg error, FpEqMsg is NULL", pSyncNode->vgId); + sError("vgId:%d, failed to enqueue msg since its null", pSyncNode->vgId); } } @@ -861,8 +858,8 @@ int32_t syncNodePropose(SSyncNode* pSyncNode, SRpcMsg* pMsg, bool isWeak) { } else { ret = -1; terrno = TSDB_CODE_SYN_NOT_LEADER; - sError("vgId:%d, sync propose not leader, %s, msgtype:%s,%d", pSyncNode->vgId, - syncUtilState2String(pSyncNode->state), TMSG_INFO(pMsg->msgType), pMsg->msgType); + sError("vgId:%d, sync propose not leader, %s, msgType:%d,%s", pSyncNode->vgId, + syncUtilState2String(pSyncNode->state), pMsg->msgType, TMSG_INFO(pMsg->msgType)); goto _END; } @@ -887,7 +884,7 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pOldSyncInfo) { } } - snprintf(pSyncNode->configPath, sizeof(pSyncNode->configPath), "%s/raft_config.json", pSyncInfo->path); + snprintf(pSyncNode->configPath, sizeof(pSyncNode->configPath), "%s%sraft_config.json", pSyncInfo->path, TD_DIRSEP); if (!taosCheckExistFile(pSyncNode->configPath)) { // create a new raft config file SRaftCfgMeta meta; @@ -910,8 +907,9 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pOldSyncInfo) { // init by SSyncInfo pSyncNode->vgId = pSyncInfo->vgId; memcpy(pSyncNode->path, pSyncInfo->path, sizeof(pSyncNode->path)); - snprintf(pSyncNode->raftStorePath, sizeof(pSyncNode->raftStorePath), "%s/raft_store.json", pSyncInfo->path); - snprintf(pSyncNode->configPath, sizeof(pSyncNode->configPath), "%s/raft_config.json", pSyncInfo->path); + snprintf(pSyncNode->raftStorePath, sizeof(pSyncNode->raftStorePath), "%s%sraft_store.json", pSyncInfo->path, + TD_DIRSEP); + snprintf(pSyncNode->configPath, sizeof(pSyncNode->configPath), "%s%sraft_config.json", pSyncInfo->path, TD_DIRSEP); pSyncNode->pWal = pSyncInfo->pWal; pSyncNode->msgcb = pSyncInfo->msgcb; @@ -2764,7 +2762,7 @@ int32_t syncNodeCommit(SSyncNode* ths, SyncIndex beginIndex, SyncIndex endIndex, ESyncState state = flag; char eventLog[128]; - snprintf(eventLog, sizeof(eventLog), "commit by wal from index:%" PRId64 " to index:%" PRId64, beginIndex, endIndex); + snprintf(eventLog, sizeof(eventLog), "commit wal from index:%" PRId64 " to index:%" PRId64, beginIndex, endIndex); syncNodeEventLog(ths, eventLog); // execute fsm @@ -2782,13 +2780,13 @@ int32_t syncNodeCommit(SSyncNode* ths, SyncIndex beginIndex, SyncIndex endIndex, // user commit if ((ths->pFsm->FpCommitCb != NULL) && syncUtilUserCommit(pEntry->originalRpcType)) { bool internalExecute = true; - if ((ths->replicaNum == 1) && ths->restoreFinish && (ths->vgId != 1)) { + if ((ths->replicaNum == 1) && ths->restoreFinish && ths->vgId != 1) { internalExecute = false; } do { char logBuf[128]; - snprintf(logBuf, sizeof(logBuf), "index:%" PRId64 ", internalExecute:%d", i, internalExecute); + snprintf(logBuf, sizeof(logBuf), "commit index:%" PRId64 ", internal:%d", i, internalExecute); syncNodeEventLog(ths, logBuf); } while (0); diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index 36be371213..0e17c1db80 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -229,8 +229,8 @@ static int32_t raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncRaftEntr do { char eventLog[128]; - snprintf(eventLog, sizeof(eventLog), "write index:%" PRId64 ", type:%s,%d, type2:%s,%d", pEntry->index, - TMSG_INFO(pEntry->msgType), pEntry->msgType, TMSG_INFO(pEntry->originalRpcType), pEntry->originalRpcType); + snprintf(eventLog, sizeof(eventLog), "write index:%" PRId64 ", type:%s, origin type:%s", pEntry->index, + TMSG_INFO(pEntry->msgType), TMSG_INFO(pEntry->originalRpcType)); syncNodeEventLog(pData->pSyncNode, eventLog); } while (0); @@ -468,8 +468,8 @@ int32_t logStoreAppendEntry(SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry) { do { char eventLog[128]; - snprintf(eventLog, sizeof(eventLog), "write2 index:%" PRId64 ", type:%s,%d, type2:%s,%d", pEntry->index, - TMSG_INFO(pEntry->msgType), pEntry->msgType, TMSG_INFO(pEntry->originalRpcType), pEntry->originalRpcType); + snprintf(eventLog, sizeof(eventLog), "write2 index:%" PRId64 ", type:%s, origin type:%s", pEntry->index, + TMSG_INFO(pEntry->msgType), TMSG_INFO(pEntry->originalRpcType)); syncNodeEventLog(pData->pSyncNode, eventLog); } while (0); diff --git a/source/libs/sync/src/syncRespMgr.c b/source/libs/sync/src/syncRespMgr.c index 501f1fb435..db4c1e0ba5 100644 --- a/source/libs/sync/src/syncRespMgr.c +++ b/source/libs/sync/src/syncRespMgr.c @@ -50,9 +50,8 @@ int64_t syncRespMgrAdd(SSyncRespMgr *pObj, SRespStub *pStub) { SSyncNode *pSyncNode = pObj->data; char eventLog[128]; - snprintf(eventLog, sizeof(eventLog), "resp mgr add, type:%s,%d, seq:%" PRIu64 ", handle:%p, ahandle:%p", - TMSG_INFO(pStub->rpcMsg.msgType), pStub->rpcMsg.msgType, keyCode, pStub->rpcMsg.info.handle, - pStub->rpcMsg.info.ahandle); + snprintf(eventLog, sizeof(eventLog), "save response handle, type:%s, seq:%" PRIu64 ", handle:%p, ahandle:%p", + TMSG_INFO(pStub->rpcMsg.msgType), keyCode, pStub->rpcMsg.info.handle, pStub->rpcMsg.info.ahandle); syncNodeEventLog(pSyncNode, eventLog); taosThreadMutexUnlock(&(pObj->mutex)); @@ -77,9 +76,8 @@ int32_t syncRespMgrGet(SSyncRespMgr *pObj, uint64_t index, SRespStub *pStub) { SSyncNode *pSyncNode = pObj->data; char eventLog[128]; - snprintf(eventLog, sizeof(eventLog), "resp mgr get, type:%s,%d, seq:%" PRIu64 ", handle:%p, ahandle:%p", - TMSG_INFO(pStub->rpcMsg.msgType), pStub->rpcMsg.msgType, index, pStub->rpcMsg.info.handle, - pStub->rpcMsg.info.ahandle); + snprintf(eventLog, sizeof(eventLog), "get response handle, type:%s, seq:%" PRIu64 ", handle:%p, ahandle:%p", + TMSG_INFO(pStub->rpcMsg.msgType), index, pStub->rpcMsg.info.handle, pStub->rpcMsg.info.ahandle); syncNodeEventLog(pSyncNode, eventLog); taosThreadMutexUnlock(&(pObj->mutex)); @@ -98,9 +96,8 @@ int32_t syncRespMgrGetAndDel(SSyncRespMgr *pObj, uint64_t index, SRespStub *pStu SSyncNode *pSyncNode = pObj->data; char eventLog[128]; - snprintf(eventLog, sizeof(eventLog), "resp mgr get-and-del, type:%s,%d, seq:%" PRIu64 ", handle:%p, ahandle:%p", - TMSG_INFO(pStub->rpcMsg.msgType), pStub->rpcMsg.msgType, index, pStub->rpcMsg.info.handle, - pStub->rpcMsg.info.ahandle); + snprintf(eventLog, sizeof(eventLog), "get-and-del response handle, type:%s, seq:%" PRIu64 ", handle:%p, ahandle:%p", + TMSG_INFO(pStub->rpcMsg.msgType), index, pStub->rpcMsg.info.handle, pStub->rpcMsg.info.ahandle); syncNodeEventLog(pSyncNode, eventLog); taosHashRemove(pObj->pRespHash, &index, sizeof(index)); diff --git a/tests/script/tmp/r1.sim b/tests/script/tmp/r1.sim new file mode 100644 index 0000000000..3fc875ad23 --- /dev/null +++ b/tests/script/tmp/r1.sim @@ -0,0 +1,47 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 +system sh/cfg.sh -n dnode1 -c supportVnodes -v 0 +system sh/exec.sh -n dnode1 -s start +system sh/exec.sh -n dnode2 -s start +sql connect + +print =============== step1: create dnodes +sql create dnode $hostname port 7200 + +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + print ====> dnode not ready! + return -1 + endi +sql show dnodes +print ===> rows: $rows +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +if $rows != 2 then + return -1 +endi +if $data(1)[4] != ready then + goto step1 +endi +if $data(2)[4] != ready then + goto step1 +endi + +print =============== step2: create database +sql create database db vgroups 1 replica 1 +sql show databases +if $rows != 3 then + return -1 +endi + +sql use db; +sql create table stb (ts timestamp, c int) tags (t int); +sql create table t0 using stb tags (0); +sql insert into t0 values(now, 1); +sql insert into t0 values(now+1s, 1); + +return From bc8c7f6ea6bf63860e0a1882198d61bf83b431d3 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 2 Aug 2022 16:40:05 +0800 Subject: [PATCH 16/18] refactor: minor changes --- source/dnode/vnode/src/meta/metaOpen.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/meta/metaOpen.c b/source/dnode/vnode/src/meta/metaOpen.c index 2f026e71eb..e1eb7a9109 100644 --- a/source/dnode/vnode/src/meta/metaOpen.c +++ b/source/dnode/vnode/src/meta/metaOpen.c @@ -133,7 +133,7 @@ int metaOpen(SVnode *pVnode, SMeta **ppMeta) { ret = tdbTbOpen("stream.task.db", sizeof(int64_t), -1, taskIdxKeyCmpr, pMeta->pEnv, &pMeta->pStreamDb); if (ret < 0) { - metaError("vgIdd, failed to open meta stream task index since %s", TD_VID(pVnode), tstrerror(terrno)); + metaError("vgId%d, failed to open meta stream task index since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } From efb0127fbf72bcdfe56ce28238daade1c106a455 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 2 Aug 2022 16:48:49 +0800 Subject: [PATCH 17/18] refactor: adjust logs --- source/dnode/vnode/src/meta/metaOpen.c | 2 +- source/dnode/vnode/src/meta/metaQuery.c | 4 ++-- source/dnode/vnode/src/meta/metaSma.c | 2 +- source/dnode/vnode/src/meta/metaTable.c | 2 +- source/dnode/vnode/src/tsdb/tsdbRead.c | 4 ++-- source/dnode/vnode/src/vnd/vnodeSvr.c | 2 +- source/dnode/vnode/src/vnd/vnodeSync.c | 4 ++-- source/libs/index/src/index.c | 4 ++-- source/libs/index/src/indexTfile.c | 10 +++++----- source/libs/sync/src/syncMain.c | 4 ++-- source/os/src/osSemaphore.c | 2 +- 11 files changed, 20 insertions(+), 20 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaOpen.c b/source/dnode/vnode/src/meta/metaOpen.c index e1eb7a9109..941d2c6d72 100644 --- a/source/dnode/vnode/src/meta/metaOpen.c +++ b/source/dnode/vnode/src/meta/metaOpen.c @@ -133,7 +133,7 @@ int metaOpen(SVnode *pVnode, SMeta **ppMeta) { ret = tdbTbOpen("stream.task.db", sizeof(int64_t), -1, taskIdxKeyCmpr, pMeta->pEnv, &pMeta->pStreamDb); if (ret < 0) { - metaError("vgId%d, failed to open meta stream task index since %s", TD_VID(pVnode), tstrerror(terrno)); + metaError("vgId:%d, failed to open meta stream task index since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index b72195d333..db71c21615 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -599,7 +599,7 @@ STSmaWrapper *metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid, bool deepCopy) { for (int i = 0; i < pSW->number; ++i) { smaId = *(tb_uid_t *)taosArrayGet(pSmaIds, i); if (metaGetTableEntryByUid(&mr, smaId) < 0) { - metaWarn("vgId:%d, no entry for tbId" PRIi64 ", smaId" PRIi64, TD_VID(pMeta->pVnode), uid, smaId); + metaWarn("vgId:%d, no entry for tbId:%" PRIi64 ", smaId:%" PRIi64, TD_VID(pMeta->pVnode), uid, smaId); continue; } pTSma = pSW->tSma + smaIdx; @@ -647,7 +647,7 @@ STSma *metaGetSmaInfoByIndex(SMeta *pMeta, int64_t indexUid) { SMetaReader mr = {0}; metaReaderInit(&mr, pMeta, 0); if (metaGetTableEntryByUid(&mr, indexUid) < 0) { - metaWarn("vgId:%d, failed to get table entry for smaId" PRIi64, TD_VID(pMeta->pVnode), indexUid); + metaWarn("vgId:%d, failed to get table entry for smaId:%" PRIi64, TD_VID(pMeta->pVnode), indexUid); metaReaderClear(&mr); return NULL; } diff --git a/source/dnode/vnode/src/meta/metaSma.c b/source/dnode/vnode/src/meta/metaSma.c index 4f0d0341bc..1e5b699fce 100644 --- a/source/dnode/vnode/src/meta/metaSma.c +++ b/source/dnode/vnode/src/meta/metaSma.c @@ -62,7 +62,7 @@ int32_t metaCreateTSma(SMeta *pMeta, int64_t version, SSmaCfg *pCfg) { return 0; _err: - metaError("vgId:%d, failed to create tsmas uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pCfg->indexName, + metaError("vgId:%d, failed to create tsma:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pCfg->indexName, pCfg->indexUid, tstrerror(terrno)); return -1; } diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 3f47128a4b..ff455bfe33 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -209,7 +209,7 @@ int metaCreateSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { return 0; _err: - metaError("vgId:%d, failed to create super tables uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, + metaError("vgId:%d, failed to create super table:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, pReq->suid, tstrerror(terrno)); return -1; } diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index f6e01f04cd..265e20e55e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -1197,7 +1197,7 @@ static int32_t buildDataBlockFromBuf(STsdbReader* pReader, STableBlockScanInfo* setComposedBlockFlag(pReader, true); double elapsedTime = (taosGetTimestampUs() - st) / 1000.0; - tsdbDebug("%p build data block from cache completed, elapsed time:%.2f ms, numOfRows:%d, brange" PRId64 + tsdbDebug("%p build data block from cache completed, elapsed time:%.2f ms, numOfRows:%d, brange:%" PRId64 " - %" PRId64 " %s", pReader, elapsedTime, pBlock->info.rows, pBlock->info.window.skey, pBlock->info.window.ekey, pReader->idStr); @@ -2647,7 +2647,7 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, SArray* pTabl return code; _err: - tsdbError("failed to create data reader, codes %s", tstrerror(code), pReader->idStr); + tsdbError("failed to create data reader, code:%s %s", tstrerror(code), pReader->idStr); return code; } diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 3fc66b4bf2..1f3de502a9 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -262,7 +262,7 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp return 0; _err: - vError("vgId:%d, process %s request failed since %s, version" PRId64, TD_VID(pVnode), TMSG_INFO(pMsg->msgType), + vError("vgId:%d, process %s request failed since %s, version:%" PRId64, TD_VID(pVnode), TMSG_INFO(pMsg->msgType), tstrerror(terrno), version); return -1; } diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index 6eb9c402ac..5c5163eb41 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -562,9 +562,9 @@ static void vnodeSyncPreCommitMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMet static void vnodeSyncRollBackMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) { SVnode *pVnode = pFsm->data; - vTrace("vgId:%d, rollback-cb is excuted, fsm:%p, index:%" PRId64 ", isWeak:%d, code:%d, state:%d %s, msgType:%d %s", + vTrace("vgId:%d, rollback-cb is excuted, fsm:%p, index:%" PRId64 ", weak:%d, code:%d, state:%d %s, type:%s", syncGetVgId(pVnode->sync), pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, - syncUtilState2String(cbMeta.state), pMsg->msgType, TMSG_INFO(pMsg->msgType)); + syncUtilState2String(cbMeta.state), TMSG_INFO(pMsg->msgType)); } #define USE_TSDB_SNAPSHOT diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index c61b5be7e9..d61cd871ee 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -220,7 +220,7 @@ int indexPut(SIndex* index, SIndexMultiTerm* fVals, uint64_t uid) { char buf[128] = {0}; ICacheKey key = {.suid = p->suid, .colName = p->colName, .nColName = strlen(p->colName), .colType = p->colType}; int32_t sz = idxSerialCacheKey(&key, buf); - indexDebug("w suid:%" PRIu64 ", colName: %s, colType: %d", key.suid, key.colName, key.colType); + indexDebug("w suid:%" PRIu64 ", colName:%s, colType:%d", key.suid, key.colName, key.colType); IndexCache** cache = taosHashGet(index->colObj, buf, sz); assert(*cache != NULL); @@ -395,7 +395,7 @@ static int idxTermSearch(SIndex* sIdx, SIndexTermQuery* query, SArray** result) char buf[128] = {0}; ICacheKey key = { .suid = term->suid, .colName = term->colName, .nColName = strlen(term->colName), .colType = term->colType}; - indexDebug("r suid:%" PRIu64 ", colName: %s, colType: %d", key.suid, key.colName, key.colType); + indexDebug("r suid:%" PRIu64 ", colName:%s, colType:%d", key.suid, key.colName, key.colType); int32_t sz = idxSerialCacheKey(&key, buf); taosThreadMutexLock(&sIdx->mtx); diff --git a/source/libs/index/src/indexTfile.c b/source/libs/index/src/indexTfile.c index 59c8a7bfb2..0a47fc0f16 100644 --- a/source/libs/index/src/indexTfile.c +++ b/source/libs/index/src/indexTfile.c @@ -141,7 +141,7 @@ void tfileCacheDestroy(TFileCache* tcache) { TFileReader** reader = taosHashIterate(tcache->tableCache, NULL); while (reader) { TFileReader* p = *reader; - indexInfo("drop table cache suid:%" PRIu64 ", colName: %s, colType: %d", p->header.suid, p->header.colName, + indexInfo("drop table cache suid:%" PRIu64 ", colName:%s, colType:%d", p->header.suid, p->header.colName, p->header.colType); tfileReaderUnRef(p); reader = taosHashIterate(tcache->tableCache, reader); @@ -185,20 +185,20 @@ TFileReader* tfileReaderCreate(IFileCtx* ctx) { reader->ctx = ctx; if (0 != tfileReaderVerify(reader)) { - indexError("invalid tfile, suid:%" PRIu64 ", colName: %s", reader->header.suid, reader->header.colName); + indexError("invalid tfile, suid:%" PRIu64 ", colName:%s", reader->header.suid, reader->header.colName); tfileReaderDestroy(reader); return NULL; } // T_REF_INC(reader); if (0 != tfileReaderLoadHeader(reader)) { - indexError("failed to load index header, suid:%" PRIu64 ", colName: %s", reader->header.suid, + indexError("failed to load index header, suid:%" PRIu64 ", colName:%s", reader->header.suid, reader->header.colName); tfileReaderDestroy(reader); return NULL; } if (0 != tfileReaderLoadFst(reader)) { - indexError("failed to load index fst, suid:%" PRIu64 ", colName: %s, errno: %d", reader->header.suid, + indexError("failed to load index fst, suid:%" PRIu64 ", colName:%s, code:0x%x", reader->header.suid, reader->header.colName, errno); tfileReaderDestroy(reader); return NULL; @@ -874,7 +874,7 @@ static int tfileReaderLoadHeader(TFileReader* reader) { int64_t nread = reader->ctx->readFrom(reader->ctx, buf, sizeof(buf), 0); if (nread == -1) { - indexError("actual Read: %d, to read: %d, errno: %d, filename: %s", (int)(nread), (int)sizeof(buf), errno, + indexError("actual Read: %d, to read: %d, code:0x%x, filename: %s", (int)(nread), (int)sizeof(buf), errno, reader->ctx->file.buf); } else { indexInfo("actual Read: %d, to read: %d, filename: %s", (int)(nread), (int)sizeof(buf), reader->ctx->file.buf); diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index ca03d54675..1b802b9e5b 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -858,8 +858,8 @@ int32_t syncNodePropose(SSyncNode* pSyncNode, SRpcMsg* pMsg, bool isWeak) { } else { ret = -1; terrno = TSDB_CODE_SYN_NOT_LEADER; - sError("vgId:%d, sync propose not leader, %s, msgType:%d,%s", pSyncNode->vgId, - syncUtilState2String(pSyncNode->state), pMsg->msgType, TMSG_INFO(pMsg->msgType)); + sError("vgId:%d, sync propose not leader, %s, type:%s", pSyncNode->vgId, syncUtilState2String(pSyncNode->state), + TMSG_INFO(pMsg->msgType)); goto _END; } diff --git a/source/os/src/osSemaphore.c b/source/os/src/osSemaphore.c index 3275774cce..63c414386b 100644 --- a/source/os/src/osSemaphore.c +++ b/source/os/src/osSemaphore.c @@ -93,7 +93,7 @@ int32_t tsem_timewait(tsem_t* sem, int64_t nanosecs) { // // We specified a non-zero wait. Time must advance. // if (ft_before.dwLowDateTime == ft_after.dwLowDateTime && ft_before.dwHighDateTime == ft_after.dwHighDateTime) // { - // printf("nanoseconds: %d, rc: %d, errno: %d. before filetime: %d, %d; after filetime: %d, %d\n", + // printf("nanoseconds: %d, rc: %d, code:0x%x. before filetime: %d, %d; after filetime: %d, %d\n", // nanosecs, rc, errno, // (int)ft_before.dwLowDateTime, (int)ft_before.dwHighDateTime, // (int)ft_after.dwLowDateTime, (int)ft_after.dwHighDateTime); From c59b65965453b953f8b4cef84f16bd9ab4c5ace2 Mon Sep 17 00:00:00 2001 From: gccgdb1234 Date: Tue, 2 Aug 2022 17:42:20 +0800 Subject: [PATCH 18/18] doc: refine information schema tables --- docs/zh/12-taos-sql/22-meta.md | 197 ++++++++++++++++++++++----------- 1 file changed, 130 insertions(+), 67 deletions(-) diff --git a/docs/zh/12-taos-sql/22-meta.md b/docs/zh/12-taos-sql/22-meta.md index e5bc800de7..14a4d3c2bb 100644 --- a/docs/zh/12-taos-sql/22-meta.md +++ b/docs/zh/12-taos-sql/22-meta.md @@ -15,21 +15,21 @@ Note: 由于 SHOW 语句已经被开发者熟悉和广泛使用,所以它们 本章将详细介绍 `INFORMATION_SCHEMA` 这个内置元数据库中的表和表结构。 -## DNODES +## INS_DNODES 提供 dnode 的相关信息。也可以使用 SHOW DNODES 来查询这些信息。 -| # | **列名** | **数据类型** | **说明** | -| --- | :------------: | ------------ | --------------------- | -| 1 | vnodes | SMALLINT | dnode 中的 vnode 个数 | -| 2 | support_vnodes | SMALLINT | 支持的 vnode 个数 | -| 3 | status | BINARY(10) | 当前状态 | -| 4 | note | BINARY(256) | 离线原因等信息 | -| 5 | id | SMALLINT | dnode id | -| 6 | endpoint | BINARY(134) | dnode 的地址 | -| 7 | create | TIMESTAMP | 创建时间 | +| # | **列名** | **数据类型** | **说明** | +| --- | :------------: | ------------ | ------------------------- | +| 1 | vnodes | SMALLINT | dnode 中的实际 vnode 个数 | +| 2 | support_vnodes | SMALLINT | 最多支持的 vnode 个数 | +| 3 | status | BINARY(10) | 当前状态 | +| 4 | note | BINARY(256) | 离线原因等信息 | +| 5 | id | SMALLINT | dnode id | +| 6 | endpoint | BINARY(134) | dnode 的地址 | +| 7 | create | TIMESTAMP | 创建时间 | -## MNODES +## INS_MNODES 提供 mnode 的相关信息。也可以使用 SHOW MNODES 来查询这些信息。 @@ -41,7 +41,7 @@ Note: 由于 SHOW 语句已经被开发者熟悉和广泛使用,所以它们 | 4 | role_time | TIMESTAMP | 成为当前角色的时间 | | 5 | create_time | TIMESTAMP | 创建时间 | -## MODULES +## INS_MODULES 提供组件的相关信息。也可以使用 SHOW MODULES 来查询这些信息 @@ -51,46 +51,74 @@ Note: 由于 SHOW 语句已经被开发者熟悉和广泛使用,所以它们 | 2 | endpoint | BINARY(134) | 组件的地址 | | 3 | module | BINARY(10) | 组件状态 | -## QNODES +## INS_QNODES 当前系统中 QNODE 的信息。也可以使用 SHOW QNODES 来查询这些信息。 | # | **列名** | **数据类型** | **说明** | | --- | :---------: | ------------ | ------------ | -| 1 | id | SMALLINT | module id | +| 1 | id | SMALLINT | qnode id | | 2 | endpoint | BINARY(134) | qnode 的地址 | | 3 | create_time | TIMESTAMP | 创建时间 | -## USER_DATABASES +## INS_CLUSTER + +存储集群相关信息。 + +| # | **列名** | **数据类型** | **说明** | +| --- | :---------: | ------------ | ---------- | +| 1 | id | BIGINT | cluster id | +| 2 | name | BINARY(134) | 集群名称 | +| 3 | create_time | TIMESTAMP | 创建时间 | + +## INS_DATABASES 提供用户创建的数据库对象的相关信息。也可以使用 SHOW DATABASES 来查询这些信息。 -TODO +| # | **列名** | **数据类型** | **说明** | +| --- | :------------------: | ---------------- | ------------------------------------------------ | +| 1 | name | BINARY(32) | 数据库名 | +| 2 | create_time | TIMESTAMP | 创建时间 | +| 3 | ntables | INT | 数据库中表的数量,包含子表和普通表但不包含超级表 | +| 4 | vgroups | INT | 数据库中有多少个 vgroup | +| 6 | replica | INT | 副本数 | +| 7 | quorum | BINARY(3) | 强一致性 | +| 8 | duration | INT | 单文件存储数据的时间跨度 | +| 9 | keep | INT | 数据保留时长 | +| 10 | buffer | INT | 每个 vnode 写缓存的内存块大小,单位 MB | +| 11 | pagesize | INT | 每个 VNODE 中元数据存储引擎的页大小,单位为 KB | +| 12 | pages | INT | 每个 vnode 元数据存储引擎的缓存页个数 | +| 13 | minrows | INT | 文件块中记录的最大条数 | +| 14 | maxrows | INT | 文件块中记录的最小条数 | +| 15 | comp | INT | 数据压缩方式 | +| 16 | precision | BINARY(2) | 时间分辨率 | +| 17 | status | BINARY(10) | 数据库状态 | +| 18 | retention | BINARY (60) | 数据的聚合周期和保存时长 | +| 19 | single_stable | BOOL | 表示此数据库中是否只可以创建一个超级表 | +| 20 | cachemodel | BINARY(60) | 表示是否在内存中缓存子表的最近数据 | +| 21 | cachesize | INT | 表示每个 vnode 中用于缓存子表最近数据的内存大小 | +| 22 | wal_level | INT | WAL 级别 | +| 23 | wal_fsync_period | INT | 数据落盘周期 | +| 24 | wal_retention_period | INT | WAL 的保存时长 | +| 25 | wal_retention_size | INT | WAL 的保存上限 | +| 26 | wal_roll_period | INT | wal 文件切换时长 | +| 27 | wal_segment_size | wal 单个文件大小 | -| # | **列名** | **数据类型** | **说明** | -| --- | :---------: | ------------ | ------------------------------------------------ | -| 1 | name | BINARY(32) | 数据库名 | -| 2 | create_time | TIMESTAMP | 创建时间 | -| 3 | ntables | INT | 数据库中表的数量,包含子表和普通表但不包含超级表 | -| 4 | vgroups | INT | 数据库中有多少个 vgroup | -| 5 | replica | INT | 副本数 | -| 6 | quorum | INT | 写成功的确认数 | -| 7 | days | INT | 单文件存储数据的时间跨度 | -| 8 | keep | INT | 数据保留时长 | -| 9 | buffer | INT | 每个 vnode 写缓存的内存块大小,单位 MB | -| 10 | minrows | INT | 文件块中记录的最大条数 | -| 11 | maxrows | INT | 文件块中记录的最小条数 | -| 12 | wal_level | INT | WAL 级别 | -| 13 | walfsync_period | INT | 数据落盘周期 | -| 14 | comp | INT | 数据压缩方式 | -| 15 | precision | BINARY(2) | 时间分辨率 | -| 16 | status | BINARY(10) | 数据库状态 | +## INS_FUNCTIONS -## USER_FUNCTIONS +用户创建的自定义函数的信息。 -TODO +| # | **列名** | **数据类型** | **说明** | +| --- | :---------: | ------------ | -------------- | +| 1 | name | BINARY(64) | 函数名 | +| 2 | comment | BINARY(255) | 补充说明 | +| 3 | aggregate | INT | 是否为聚合函数 | +| 4 | output_type | BINARY(31) | 输出类型 | +| 5 | create_time | TIMESTAMP | 创建时间 | +| 6 | code_len | INT | 代码长度 | +| 7 | bufsize | INT | buffer 大小 | -## USER_INDEXES +## INS_INDEXES 提供用户创建的索引的相关信息。也可以使用 SHOW INDEX 来查询这些信息。 @@ -103,7 +131,7 @@ TODO | 5 | index_type | BINARY(10) | 目前有 SMA 和 FULLTEXT | | 6 | index_extensions | BINARY(256) | 索引的额外信息。对 SMA 类型的索引,是函数名的列表。对 FULLTEXT 类型的索引为 NULL。 | -## USER_STABLES +## INS_STABLES 提供用户创建的超级表的相关信息。 @@ -120,19 +148,7 @@ TODO | 9 | max_delay | BINARY(64) | 推送计算结果的最大延迟 | | 10 | rollup | BINARY(128) | rollup 聚合函数 | -## USER_STREAMS - -提供用户创建的流计算的相关信息。 - -| # | **列名** | **数据类型** | **说明** | -| --- | :---------: | ------------ | --------------------------- | -| 1 | stream_name | BINARY(192) | 流计算名称 | -| 2 | user_name | BINARY(23) | 创建流计算的用户 | -| 3 | dest_table | BINARY(192) | 流计算写入的目标表 | -| 4 | create_time | TIMESTAMP | 创建时间 | -| 5 | sql | BLOB | 创建流计算时提供的 SQL 语句 | - -## USER_TABLES +## INS_TABLES 提供用户创建的普通表和子表的相关信息 @@ -149,7 +165,30 @@ TODO | 9 | table_comment | BINARY(1024) | 表注释 | | 10 | type | BINARY(20) | 表类型 | -## USER_USERS +## INS_TAGS + +| # | **列名** | **数据类型** | **说明** | +| --- | :---------: | ------------- | ---------------------- | +| 1 | table_name | BINARY(192) | 表名 | +| 2 | db_name | BINARY(64) | 该表所在的数据库的名称 | +| 3 | stable_name | BINARY(192) | 所属的超级表表名 | +| 4 | tag_name | BINARY(64) | tag 的名称 | +| 5 | tag_type | BINARY(64) | tag 的类型 | +| 6 | tag_value | BINARY(16384) | tag 的值 | + +## USER_STREAMS + +提供用户创建的流计算的相关信息。 + +| # | **列名** | **数据类型** | **说明** | +| --- | :---------: | ------------ | --------------------------- | +| 1 | stream_name | BINARY(192) | 流计算名称 | +| 2 | user_name | BINARY(23) | 创建流计算的用户 | +| 3 | dest_table | BINARY(192) | 流计算写入的目标表 | +| 4 | create_time | TIMESTAMP | 创建时间 | +| 5 | sql | BLOB | 创建流计算时提供的 SQL 语句 | + +## INS_USERS 提供系统中创建的用户的相关信息。 @@ -159,21 +198,45 @@ TODO | 2 | privilege | BINARY(256) | 权限 | | 3 | create_time | TIMESTAMP | 创建时间 | -## VGROUPS +## INS_GRANTS + +TODO + +## INS_VGROUPS 系统中所有 vgroups 的信息。 -| # | **列名** | **数据类型** | **说明** | -| --- | :--------: | ------------ | ---------------------------- | -| 1 | vg_id | INT | vgroup id | -| 2 | db_name | BINARY(32) | 数据库名 | -| 3 | tables | INT | 此 vgroup 内有多少表 | -| 4 | status | BINARY(10) | 此 vgroup 的状态 | -| 5 | onlines | INT | 在线的成员数目 | -| 6 | v1_dnode | INT | 第一个成员所在的 dnode 的 id | -| 7 | v1_status | BINARY(10) | 第一个成员的状态 | -| 8 | v2_dnode | INT | 第二个成员所在的 dnode 的 id | -| 9 | v2_status | BINARY(10) | 第二个成员的状态 | -| 10 | v3_dnode | INT | 第三个成员所在的 dnode 的 id | -| 11 | v3_status | BINARY(10) | 第三个成员的状态 | -| 12 | compacting | INT | compact 状态 | +| # | **列名** | **数据类型** | **说明** | +| --- | :-------: | ------------ | ---------------------------- | +| 1 | vgroup_id | INT | vgroup id | +| 2 | db_name | BINARY(32) | 数据库名 | +| 3 | tables | INT | 此 vgroup 内有多少表 | +| 4 | status | BINARY(10) | 此 vgroup 的状态 | +| 5 | v1_dnode | INT | 第一个成员所在的 dnode 的 id | +| 6 | v1_status | BINARY(10) | 第一个成员的状态 | +| 7 | v2_dnode | INT | 第二个成员所在的 dnode 的 id | +| 8 | v2_status | BINARY(10) | 第二个成员的状态 | +| 9 | v3_dnode | INT | 第三个成员所在的 dnode 的 id | +| 10 | v3_status | BINARY(10) | 第三个成员的状态 | +| 11 | nfiles | INT | TODO | +| 12 | file_size | INT | TODO | +| 13 | tsma | TINYINT | TODO | + +## INS_CONFIGS + +系统配置参数。 + +| # | **列名** | **数据类型** | **说明** | +| --- | :------: | ------------ | ------------ | +| 1 | name | BINARY(32) | 配置项名称 | +| 2 | value | BINARY(64) | 该配置项的值 | + +## INS_DNODE_VARIABLES + +系统中每个 dnode 的配置参数。 + +| # | **列名** | **数据类型** | **说明** | +| --- | :------: | ------------ | ------------ | +| 1 | dnode_id | INT | dnode 的 ID | +| 2 | name | BINARY(32) | 配置项名称 | +| 3 | value | BINARY(64) | 该配置项的值 |