From 770e03318a9faaf13bd26ae5b69f76ef51a17278 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 9 Dec 2021 18:59:48 +0800 Subject: [PATCH 01/43] [td-11818] create basic driver launch environment. --- include/client/taos.h | 13 +- include/common/tep.h | 6 + include/common/tglobal.h | 10 - include/util/tdef.h | 4 +- source/client/CMakeLists.txt | 4 +- source/client/inc/clientInt.h | 114 +++++ source/client/inc/tscLog.h | 37 ++ source/client/src/clientImpl.c | 190 ++++++++ source/client/src/tscEnv.c | 487 +++++++++++++++++++++ source/common/src/tep.c | 20 + source/common/src/tglobal.c | 75 ---- source/dnode/mgmt/impl/test/sut/deploy.cpp | 1 - source/libs/transport/inc/rpcLog.h | 7 +- source/util/src/tnote.c | 9 - 14 files changed, 870 insertions(+), 107 deletions(-) create mode 100644 include/common/tep.h create mode 100644 source/client/inc/clientInt.h create mode 100644 source/client/inc/tscLog.h create mode 100644 source/client/src/clientImpl.c create mode 100644 source/client/src/tscEnv.c create mode 100644 source/common/src/tep.c diff --git a/include/client/taos.h b/include/client/taos.h index 0f7edc9fed..bfb557a239 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -73,10 +73,13 @@ typedef struct taosField { #define DLL_EXPORT #endif +typedef void (*__taos_async_fn_t)(void *param, TAOS_RES *, int code); + DLL_EXPORT int taos_init(); DLL_EXPORT void taos_cleanup(void); DLL_EXPORT int taos_options(TSDB_OPTION option, const void *arg, ...); DLL_EXPORT TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port); +DLL_EXPORT TAOS *taos_connect_l(const char *ip, int ipLen, const char *user, int userLen, const char *pass, int passLen, const char *db, int dbLen, uint16_t port); DLL_EXPORT TAOS *taos_connect_auth(const char *ip, const char *user, const char *auth, const char *db, uint16_t port); DLL_EXPORT void taos_close(TAOS *taos); @@ -154,14 +157,14 @@ DLL_EXPORT int* taos_fetch_lengths(TAOS_RES *res); // TAOS_RES *taos_list_dbs(TAOS *mysql, const char *wild); // TODO: the return value should be `const` -DLL_EXPORT char *taos_get_server_info(TAOS *taos); -DLL_EXPORT char *taos_get_client_info(); -DLL_EXPORT char *taos_errstr(TAOS_RES *tres); +DLL_EXPORT const char *taos_get_server_info(TAOS *taos); +DLL_EXPORT const char *taos_get_client_info(); +DLL_EXPORT const char *taos_errstr(TAOS_RES *tres); DLL_EXPORT int taos_errno(TAOS_RES *tres); -DLL_EXPORT void taos_query_a(TAOS *taos, const char *sql, void (*fp)(void *param, TAOS_RES *, int code), void *param); -DLL_EXPORT void taos_fetch_rows_a(TAOS_RES *res, void (*fp)(void *param, TAOS_RES *, int numOfRows), void *param); +DLL_EXPORT void taos_query_a(TAOS *taos, const char *sql, __taos_async_fn_t fp, void *param); +DLL_EXPORT void taos_fetch_rows_a(TAOS_RES *res, __taos_async_fn_t fp, void *param); typedef void (*TAOS_SUBSCRIBE_CALLBACK)(TAOS_SUB* tsub, TAOS_RES *res, void* param, int code); DLL_EXPORT TAOS_SUB *taos_subscribe(TAOS* taos, int restart, const char* topic, const char *sql, TAOS_SUBSCRIBE_CALLBACK fp, void *param, int interval); diff --git a/include/common/tep.h b/include/common/tep.h new file mode 100644 index 0000000000..bdc25f6b82 --- /dev/null +++ b/include/common/tep.h @@ -0,0 +1,6 @@ +#ifndef TDENGINE_TEP_H +#define TDENGINE_TEP_H + +int taosGetFqdnPortFromEp(const char *ep, char *fqdn, uint16_t *port); + +#endif // TDENGINE_TEP_H diff --git a/include/common/tglobal.h b/include/common/tglobal.h index d0f95b786a..df5f6b7c22 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -114,16 +114,8 @@ extern int8_t tsEnableSlaveQuery; extern int8_t tsEnableAdjustMaster; // restful -extern int8_t tsEnableHttpModule; extern int32_t tsRestRowLimit; -extern uint16_t tsHttpPort; -extern int32_t tsHttpCacheSessions; -extern int32_t tsHttpSessionExpire; -extern int32_t tsHttpMaxThreads; -extern int8_t tsHttpEnableCompress; -extern int8_t tsHttpEnableRecordSql; extern int8_t tsTelegrafUseFieldNum; -extern int8_t tsHttpDbNameMandatory; // mqtt extern int8_t tsEnableMqttModule; @@ -145,7 +137,6 @@ extern int8_t tsEnableStream; // internal extern int8_t tsPrintAuth; -extern int8_t tscEmbedded; extern char tsVnodeDir[]; extern char tsMnodeDir[]; extern int64_t tsTickPerDay[3]; @@ -196,7 +187,6 @@ extern SDiskCfg tsDiskCfg[]; void taosInitGlobalCfg(); int32_t taosCheckGlobalCfg(); int32_t taosCfgDynamicOptions(char *msg); -int taosGetFqdnPortFromEp(const char *ep, char *fqdn, uint16_t *port); bool taosCheckBalanceCfgOptions(const char *option, int32_t *vnodeId, int32_t *dnodeId); void taosAddDataDir(int index, char *v1, int level, int primary); void taosReadDataDirCfg(char *v1, char *v2, char *v3); diff --git a/include/util/tdef.h b/include/util/tdef.h index 897f51f5c1..d227888582 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -161,7 +161,7 @@ do { \ #define TSDB_NODE_NAME_LEN 64 #define TSDB_TABLE_NAME_LEN 193 // it is a null-terminated string -#define TSDB_DB_NAME_LEN 33 +#define TSDB_DB_NAME_LEN 65 #define TSDB_FULL_DB_NAME_LEN (TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN) #define TSDB_FUNC_NAME_LEN 65 #define TSDB_FUNC_CODE_LEN (65535 - 512) @@ -188,7 +188,7 @@ do { \ #define TSDB_MAX_TAG_CONDITIONS 1024 #define TSDB_AUTH_LEN 16 -#define TSDB_KEY_LEN 16 +#define TSDB_KEY_LEN 64 #define TSDB_VERSION_LEN 12 #define TSDB_LABEL_LEN 8 diff --git a/source/client/CMakeLists.txt b/source/client/CMakeLists.txt index bc0d439407..5a47602dcd 100644 --- a/source/client/CMakeLists.txt +++ b/source/client/CMakeLists.txt @@ -2,9 +2,11 @@ aux_source_directory(src CLIENT_SRC) add_library(taos ${CLIENT_SRC}) target_include_directories( taos - PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" + PUBLIC "${CMAKE_SOURCE_DIR}/include/client" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) target_link_libraries( taos INTERFACE api + PRIVATE os util common transport parser ) diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h new file mode 100644 index 0000000000..27a1c92d92 --- /dev/null +++ b/source/client/inc/clientInt.h @@ -0,0 +1,114 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_CLIENTINT_H +#define TDENGINE_CLIENTINT_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include "taos.h" +#include "taosmsg.h" +#include "thash.h" +#include "tlist.h" +#include "trpc.h" +#include "tdef.h" + +typedef struct SQueryExecMetric { + int64_t start; // start timestamp + int64_t parsed; // start to parse + int64_t send; // start to send to server + int64_t rsp; // receive response from server +} SQueryExecMetric; + +typedef struct SInstanceActivity { + uint64_t numOfInsertsReq; + uint64_t numOfInsertRows; + uint64_t insertElapsedTime; + uint64_t insertBytes; // submit to tsdb since launched. + + uint64_t fetchBytes; + uint64_t queryElapsedTime; + uint64_t numOfSlowQueries; + uint64_t totalRequests; + uint64_t currentRequests; // the number of SRequestObj +} SInstanceActivity; + +typedef struct SHeartBeatInfo { + void *pTimer; // timer, used to send request msg to mnode +} SHeartBeatInfo; + +typedef struct SAppInstInfo { + int64_t numOfConns; + SRpcCorEpSet mgmtEp; + SInstanceActivity summary; + SList *pConnList; // STscObj linked list + char clusterId[TSDB_CLUSTER_ID_LEN]; +} SAppInstInfo; + +typedef struct SAppInfo { + int64_t startTime; + char appName[TSDB_APPNAME_LEN]; + char *ep; + int32_t pid; + int32_t numOfThreads; + SHeartBeatInfo hb; + SHashObj *pInstMap; +} SAppInfo; + +typedef struct STscObj { + char user[TSDB_USER_LEN]; + char pass[TSDB_KEY_LEN]; + char acctId[TSDB_ACCT_ID_LEN]; + char db[TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN]; + uint32_t connId; + uint64_t id; // ref ID returned by taosAddRef +// struct SSqlObj *sqlList; +// SRpcObj *pRpcObj; + pthread_mutex_t mutex; // used to protect the operation on db + int32_t numOfReqs; // number of sqlObj from this tscObj + SAppInstInfo *pAppInfo; +} STscObj; + +typedef struct SReqBody { + tsem_t rspSem; // not used now + void* fp; + void* param; +} SRequestBody; + +typedef struct SRequestObj { + uint64_t requestId; + int32_t type; // request type + STscObj *pTscObj; + SQueryExecMetric metric; + char *sqlstr; // sql string + SRequestBody body; + int64_t self; + char *msgBuf; + int32_t code; + void *pInfo; // sql parse info, generated by parser module +} SRequestObj; + +void* createTscObj(const char* user, const char* auth, const char *ip, uint32_t port); +void destroyTscObj(void* pTscObj); + +TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass, const char *auth, const char *db, uint16_t port); + +#ifdef __cplusplus +} +#endif + +#endif // TDENGINE_CLIENTINT_H diff --git a/source/client/inc/tscLog.h b/source/client/inc/tscLog.h new file mode 100644 index 0000000000..f205a50227 --- /dev/null +++ b/source/client/inc/tscLog.h @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_TSCLOG_H +#define TDENGINE_TSCLOG_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include "tlog.h" + +#define tscFatal(...) do { if (cDebugFlag & DEBUG_FATAL) { taosPrintLog("TSC FATAL ", cDebugFlag, __VA_ARGS__); }} while(0) +#define tscError(...) do { if (cDebugFlag & DEBUG_ERROR) { taosPrintLog("TSC ERROR ", cDebugFlag, __VA_ARGS__); }} while(0) +#define tscWarn(...) do { if (cDebugFlag & DEBUG_WARN) { taosPrintLog("TSC WARN ", cDebugFlag, __VA_ARGS__); }} while(0) +#define tscInfo(...) do { if (cDebugFlag & DEBUG_INFO) { taosPrintLog("TSC ", cDebugFlag, __VA_ARGS__); }} while(0) +#define tscDebug(...) do { if (cDebugFlag & DEBUG_DEBUG) { taosPrintLog("TSC ", cDebugFlag, __VA_ARGS__); }} while(0) +#define tscTrace(...) do { if (cDebugFlag & DEBUG_TRACE) { taosPrintLog("TSC ", cDebugFlag, __VA_ARGS__); }} while(0) +#define tscDebugL(...) do { if (cDebugFlag & DEBUG_DEBUG) { taosPrintLongString("TSC ", cDebugFlag, __VA_ARGS__); }} while(0) + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c new file mode 100644 index 0000000000..e3e6e8d3ae --- /dev/null +++ b/source/client/src/clientImpl.c @@ -0,0 +1,190 @@ +#include "tglobal.h" +#include "clientInt.h" +#include "tdef.h" +#include "tep.h" +#include "tmsgtype.h" +#include "tref.h" +#include "tscLog.h" + +static int initEpSetFromCfg(const char *firstEp, const char *secondEp, SRpcCorEpSet *pEpSet); + +static bool stringLengthCheck(const char* str, size_t maxsize) { + if (str == NULL) { + return false; + } + + size_t len = strlen(str); + if (len <= 0 || len > maxsize) { + return false; + } + + return true; +} + +static bool validateUserName(const char* user) { + return stringLengthCheck(user, TSDB_USER_LEN - 1); +} + +static bool validatePassword(const char* passwd) { + return stringLengthCheck(passwd, TSDB_KEY_LEN - 1); +} + +static bool validateDbName(const char* db) { + return stringLengthCheck(db, TSDB_DB_NAME_LEN - 1); +} + +static SRequestObj* taosConnectImpl(const char *ip, const char *user, const char *auth, const char *db, uint16_t port, __taos_async_fn_t fp, void *param); + +TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass, const char *auth, const char *db, uint16_t port) { + STscObj *pObj = NULL; + + if (!validateUserName(user)) { + terrno = TSDB_CODE_TSC_INVALID_USER_LENGTH; + return NULL; + } + + char tmp[TSDB_DB_NAME_LEN] = {0}; + if (db != NULL) { + if(!validateDbName(db)) { + terrno = TSDB_CODE_TSC_INVALID_DB_LENGTH; + return NULL; + } + + tstrncpy(tmp, db, sizeof(tmp)); + strdequote(tmp); + } + + char secretEncrypt[32] = {0}; + if (auth == NULL) { + if (!validatePassword(pass)) { + terrno = TSDB_CODE_TSC_INVALID_PASS_LENGTH; + return NULL; + } + + taosEncryptPass((uint8_t *)pass, strlen(pass), secretEncrypt); + } else { + tstrncpy(secretEncrypt, auth, tListLen(secretEncrypt)); + } + + SRpcCorEpSet epSet; + if (ip) { + if (initEpSetFromCfg(ip, NULL, &epSet) < 0) { + return NULL; + } + + if (port) { + epSet.epSet.port[0] = port; + } + } else { + if (initEpSetFromCfg(tsFirst, tsSecond, &epSet) < 0) { + return NULL; + } + } + + SRequestObj *pRequest = taosConnectImpl(ip, user, auth, db, port, NULL, NULL); + if (pRequest != NULL) { + pObj = pRequest->pTscObj; + + pRequest->body.fp = NULL; + pRequest->body.param = pRequest; + +// tscBuildAndSendRequest(pRequest, NULL); + tsem_wait(&pRequest->body.rspSem); + + if (pRequest->code != TSDB_CODE_SUCCESS) { + if (pRequest->code == TSDB_CODE_RPC_FQDN_ERROR) { + printf("taos connect failed, reason: %s\n\n", taos_errstr(pRequest)); + } else { + printf("taos connect failed, reason: %s.\n\n", tstrerror(terrno)); + } + + taos_free_result(pRequest); + taos_close(pObj); + return NULL; + } + +// tscDebug("%p DB connection is opening, rpcObj: %p, dnodeConn:%p", pObj, pObj->pRpcObj, pObj->pRpcObj->pDnodeConn); + taos_free_result(pRequest); + return pObj; + } + + return NULL; +} + +int initEpSetFromCfg(const char *firstEp, const char *secondEp, SRpcCorEpSet *pEpSet) { + pEpSet->version = 0; + + // init mgmt ip set + SEpSet *mgmtEpSet = &(pEpSet->epSet); + mgmtEpSet->numOfEps = 0; + mgmtEpSet->inUse = 0; + + if (firstEp && firstEp[0] != 0) { + if (strlen(firstEp) >= TSDB_EP_LEN) { + terrno = TSDB_CODE_TSC_INVALID_FQDN; + return -1; + } + + taosGetFqdnPortFromEp(firstEp, mgmtEpSet->fqdn[0], &(mgmtEpSet->port[0])); + mgmtEpSet->numOfEps++; + } + + if (secondEp && secondEp[0] != 0) { + if (strlen(secondEp) >= TSDB_EP_LEN) { + terrno = TSDB_CODE_TSC_INVALID_FQDN; + return -1; + } + + taosGetFqdnPortFromEp(secondEp, mgmtEpSet->fqdn[mgmtEpSet->numOfEps], &(mgmtEpSet->port[mgmtEpSet->numOfEps])); + mgmtEpSet->numOfEps++; + } + + if (mgmtEpSet->numOfEps == 0) { + terrno = TSDB_CODE_TSC_INVALID_FQDN; + return -1; + } + + return 0; +} + +SRequestObj* taosConnectImpl(const char *ip, const char *user, const char *auth, const char *db, uint16_t port, __taos_async_fn_t fp, void *param) { + if (taos_init() != TSDB_CODE_SUCCESS) { + return NULL; + } + + STscObj *pObj = createTscObj(user, auth, ip, port); + if (NULL == pObj) { + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + return NULL; + } + + SRequestObj *pRequest = (SRequestObj *)calloc(1, sizeof(SRequestObj)); + if (NULL == pRequest) { + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + free(pObj); + return NULL; + } + + void *pRpcObj = NULL; + + char rpcKey[512] = {0}; + snprintf(rpcKey, sizeof(rpcKey), "%s:%s:%s:%d", user, auth, ip, port); + if (tscAcquireRpc(rpcKey, user, auth, &pRpcObj) != 0) { + terrno = TSDB_CODE_RPC_NETWORK_UNAVAIL; + return NULL; + } + + pObj->pRpcObj = (SRpcObj *)pRpcObj; + + pRequest->pTscObj = pObj; + pRequest->body.fp = fp; + pRequest->body.param = param; + pRequest->type = TSDB_SQL_CONNECT; + + tsem_init(&pRequest->body.rspSem, 0, 0); + + pObj->id = taosAddRef(tscConn, pObj); + registerSqlObj(pRequest); + + return pRequest; +} \ No newline at end of file diff --git a/source/client/src/tscEnv.c b/source/client/src/tscEnv.c new file mode 100644 index 0000000000..d77d8b1bec --- /dev/null +++ b/source/client/src/tscEnv.c @@ -0,0 +1,487 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "clientInt.h" +#include "trpc.h" +#include "os.h" +#include "taosmsg.h" +#include "tcache.h" +#include "tconfig.h" +#include "tglobal.h" +#include "tnote.h" +#include "tref.h" +#include "tscLog.h" +#include "tsched.h" +#include "ttime.h" +#include "ttimezone.h" + +#define TSC_VAR_NOT_RELEASE 1 +#define TSC_VAR_RELEASED 0 + +SAppInfo appInfo; +int32_t sentinel = TSC_VAR_NOT_RELEASE; + +int32_t tscReqRef = -1; +void *tscQhandle; +int32_t tscConnRef = -1; +void *tscRpcCache; // TODO removed from here. + +static pthread_mutex_t rpcObjMutex; // mutex to protect open the rpc obj concurrently +static pthread_once_t tscinit = PTHREAD_ONCE_INIT; +static pthread_mutex_t setConfMutex = PTHREAD_MUTEX_INITIALIZER; + +// pthread_once can not return result code, so result code is set to a global variable. +static volatile int tscInitRes = 0; + +void tscFreeRpcObj(void *param) { +#if 0 + assert(param); + SRpcObj *pRpcObj = (SRpcObj *)(param); + tscDebug("free rpcObj:%p and free pDnodeConn: %p", pRpcObj, pRpcObj->pDnodeConn); + rpcClose(pRpcObj->pDnodeConn); +#endif +} + +void tscReleaseRpc(void *param) { + if (param == NULL) { + return; + } + + taosCacheRelease(tscRpcCache, (void *)¶m, false); +} + +void* tscAcquireRpc(const char *key, const char *user, const char *secretEncrypt) { +#if 0 + SRpcObj *pRpcObj = (SRpcObj *)taosCacheAcquireByKey(tscRpcCache, key, strlen(key)); + pthread_mutex_lock(&rpcObjMutex); + if (pRpcObj != NULL) { + pthread_mutex_unlock(&rpcObjMutex); + return pRpcObj; + } + + SRpcInit rpcInit; + memset(&rpcInit, 0, sizeof(rpcInit)); + rpcInit.localPort = 0; + rpcInit.label = "TSC"; + rpcInit.numOfThreads = tscNumOfThreads; + rpcInit.cfp = tscProcessMsgFromServer; + rpcInit.sessions = tsMaxConnections; + rpcInit.connType = TAOS_CONN_CLIENT; + rpcInit.user = (char *)user; + rpcInit.idleTime = tsShellActivityTimer * 1000; + rpcInit.ckey = "key"; + rpcInit.spi = 1; + rpcInit.secret = (char *)secretEncrypt; + + SRpcObj rpcObj = {0}; + strncpy(rpcObj.key, key, strlen(key)); + rpcObj.pDnodeConn = rpcOpen(&rpcInit); + if (rpcObj.pDnodeConn == NULL) { + pthread_mutex_unlock(&rpcObjMutex); + tscError("failed to init connection to server"); + return NULL; + } + + pRpcObj = taosCachePut(tscRpcCache, rpcObj.key, strlen(rpcObj.key), &rpcObj, sizeof(rpcObj), 1000*5); + if (pRpcObj == NULL) { + rpcClose(rpcObj.pDnodeConn); + pthread_mutex_unlock(&rpcObjMutex); + return NULL; + } + + pthread_mutex_unlock(&rpcObjMutex); + return pRpcObj; +#endif + +} + +void destroyTscObj(void *pTscObj) { + STscObj *pObj = pTscObj; +// tfree(pObj->tscCorMgmtEpSet); +// tscReleaseRpc(pObj->pRpcObj); + pthread_mutex_destroy(&pObj->mutex); + tfree(pObj); +} + +void* createTscObj(const char* user, const char* auth, const char *ip, uint32_t port) { + STscObj *pObj = (STscObj *)calloc(1, sizeof(STscObj)); + if (NULL == pObj) { + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + return NULL; + } + +// char rpcKey[512] = {0}; +// snprintf(rpcKey, sizeof(rpcKey), "%s:%s:%s:%d", user, auth, ip, port); + +// pObj->tscCorMgmtEpSet = malloc(sizeof(SRpcCorEpSet)); +// if (pObj->tscCorMgmtEpSet == NULL) { +// terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; +// free(pObj); +// return NULL; +// } +// +// memcpy(pObj->tscCorMgmtEpSet, &corMgmtEpSet, sizeof(corMgmtEpSet)); + + tstrncpy(pObj->user, user, sizeof(pObj->user)); + int32_t len = MIN(strlen(auth) + 1, sizeof(pObj->pass)); + tstrncpy(pObj->pass, auth, len); + + pthread_mutex_init(&pObj->mutex, NULL); +} + +static void tscInitLogFile() { + taosReadGlobalLogCfg(); + if (mkdir(tsLogDir, 0755) != 0 && errno != EEXIST) { + printf("failed to create log dir:%s\n", tsLogDir); + } + + const char *defaultLogFileNamePrefix = "taoslog"; + const int32_t maxLogFileNum = 10; + + char temp[128] = {0}; + sprintf(temp, "%s/%s", tsLogDir, defaultLogFileNamePrefix); + if (taosInitLog(temp, tsNumOfLogLines, maxLogFileNum) < 0) { + printf("failed to open log file in directory:%s\n", tsLogDir); + } +} + +void taos_init_imp(void) { + // In the APIs of other program language, taos_cleanup is not available yet. + // So, to make sure taos_cleanup will be invoked to clean up the allocated resource to suppress the valgrind warning. + atexit(taos_cleanup); + + errno = TSDB_CODE_SUCCESS; + srand(taosGetTimestampSec()); + + deltaToUtcInitOnce(); + taosInitGlobalCfg(); + taosReadGlobalCfg(); + + tscInitLogFile(); + + if (taosCheckGlobalCfg()) { + tscInitRes = -1; + return; + } + + taosInitNotes(); + rpcInit(); + + tscDebug("starting to initialize TAOS client ..."); + tscDebug("Local End Point is:%s", tsLocalEp); + + taosSetCoreDump(true); + + double factor = 4.0; + int32_t numOfThreads = MAX((int)(tsNumOfCores * tsNumOfThreadsPerCore / factor), 2); + + int32_t queueSize = tsMaxConnections * 2; + tscQhandle = taosInitScheduler(queueSize, numOfThreads, "tsc"); + if (NULL == tscQhandle) { + tscError("failed to init task queue"); + tscInitRes = -1; + return; + } + + tscDebug("client task queue is initialized, numOfWorkers: %d", numOfThreads); + + int refreshTime = 5; + tscRpcCache = taosCacheInit(TSDB_DATA_TYPE_BINARY, refreshTime, true, tscFreeRpcObj, "rpcObj"); + pthread_mutex_init(&rpcObjMutex, NULL); + + tscConnRef = taosOpenRef(200, destroyTscObj); + tscReqRef = taosOpenRef(40960, tscFreeRegisteredSqlObj); + + taosGetCurrentAPPName(appInfo.appName, NULL); + appInfo.pid = taosGetPId(); + appInfo.startTime = taosGetTimestampMs(); + + tscDebug("client is initialized successfully"); +} + +int taos_init() { + pthread_once(&tscinit, taos_init_imp); + return tscInitRes; +} + +// this function may be called by user or system, or by both simultaneously. +void taos_cleanup(void) { + tscDebug("start to cleanup client environment"); + + if (atomic_val_compare_exchange_32(&sentinel, TSC_VAR_NOT_RELEASE, TSC_VAR_RELEASED) != TSC_VAR_NOT_RELEASE) { + return; + } + + int32_t id = tscReqRef; + tscReqRef = -1; + taosCloseRef(id); + + void* p = tscQhandle; + tscQhandle = NULL; + taosCleanUpScheduler(p); + + id = tscConnRef; + tscConnRef = -1; + taosCloseRef(id); + + p = tscRpcCache; + tscRpcCache = NULL; + + if (p != NULL) { + taosCacheCleanup(p); + pthread_mutex_destroy(&rpcObjMutex); + } + + pthread_mutex_destroy(&setConfMutex); + + rpcCleanup(); + taosCloseLog(); +} + +static int taos_options_imp(TSDB_OPTION option, const char *pStr) { + SGlobalCfg *cfg = NULL; + + switch (option) { + case TSDB_OPTION_CONFIGDIR: + cfg = taosGetConfigOption("configDir"); + assert(cfg != NULL); + + if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_OPTION) { + tstrncpy(configDir, pStr, TSDB_FILENAME_LEN); + cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION; + tscInfo("set config file directory:%s", pStr); + } else { + tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, pStr, tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr); + } + break; + + case TSDB_OPTION_SHELL_ACTIVITY_TIMER: + cfg = taosGetConfigOption("shellActivityTimer"); + assert(cfg != NULL); + + if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_OPTION) { + tsShellActivityTimer = atoi(pStr); + if (tsShellActivityTimer < 1) tsShellActivityTimer = 1; + if (tsShellActivityTimer > 3600) tsShellActivityTimer = 3600; + cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION; + tscInfo("set shellActivityTimer:%d", tsShellActivityTimer); + } else { + tscWarn("config option:%s, input value:%s, is configured by %s, use %d", cfg->option, pStr, tsCfgStatusStr[cfg->cfgStatus], *(int32_t *)cfg->ptr); + } + break; + + case TSDB_OPTION_LOCALE: { // set locale + cfg = taosGetConfigOption("locale"); + assert(cfg != NULL); + + size_t len = strlen(pStr); + if (len == 0 || len > TSDB_LOCALE_LEN) { + tscInfo("Invalid locale:%s, use default", pStr); + return -1; + } + + if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_OPTION) { + char sep = '.'; + + if (strlen(tsLocale) == 0) { // locale does not set yet + char* defaultLocale = setlocale(LC_CTYPE, ""); + + // The locale of the current OS does not be set correctly, so the default locale cannot be acquired. + // The launch of current system will abort soon. + if (defaultLocale == NULL) { + tscError("failed to get default locale, please set the correct locale in current OS"); + return -1; + } + + tstrncpy(tsLocale, defaultLocale, TSDB_LOCALE_LEN); + } + + // set the user specified locale + char *locale = setlocale(LC_CTYPE, pStr); + + if (locale != NULL) { // failed to set the user specified locale + tscInfo("locale set, prev locale:%s, new locale:%s", tsLocale, locale); + cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION; + } else { // set the user specified locale failed, use default LC_CTYPE as current locale + locale = setlocale(LC_CTYPE, tsLocale); + tscInfo("failed to set locale:%s, current locale:%s", pStr, tsLocale); + } + + tstrncpy(tsLocale, locale, TSDB_LOCALE_LEN); + + char *charset = strrchr(tsLocale, sep); + if (charset != NULL) { + charset += 1; + + charset = taosCharsetReplace(charset); + + if (taosValidateEncodec(charset)) { + if (strlen(tsCharset) == 0) { + tscInfo("charset set:%s", charset); + } else { + tscInfo("charset changed from %s to %s", tsCharset, charset); + } + + tstrncpy(tsCharset, charset, TSDB_LOCALE_LEN); + cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION; + + } else { + tscInfo("charset:%s is not valid in locale, charset remains:%s", charset, tsCharset); + } + + free(charset); + } else { // it may be windows system + tscInfo("charset remains:%s", tsCharset); + } + } else { + tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, pStr, tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr); + } + break; + } + + case TSDB_OPTION_CHARSET: { + /* set charset will override the value of charset, assigned during system locale changed */ + cfg = taosGetConfigOption("charset"); + assert(cfg != NULL); + + size_t len = strlen(pStr); + if (len == 0 || len > TSDB_LOCALE_LEN) { + tscInfo("failed to set charset:%s", pStr); + return -1; + } + + if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_OPTION) { + if (taosValidateEncodec(pStr)) { + if (strlen(tsCharset) == 0) { + tscInfo("charset is set:%s", pStr); + } else { + tscInfo("charset changed from %s to %s", tsCharset, pStr); + } + + tstrncpy(tsCharset, pStr, TSDB_LOCALE_LEN); + cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION; + } else { + tscInfo("charset:%s not valid", pStr); + } + } else { + tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, pStr, tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr); + } + + break; + } + + case TSDB_OPTION_TIMEZONE: + cfg = taosGetConfigOption("timezone"); + assert(cfg != NULL); + + if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_OPTION) { + tstrncpy(tsTimezone, pStr, TSDB_TIMEZONE_LEN); + tsSetTimeZone(); + cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION; + tscDebug("timezone set:%s, input:%s by taos_options", tsTimezone, pStr); + } else { + tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, pStr, tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr); + } + break; + + default: + // TODO return the correct error code to client in the format for taos_errstr() + tscError("Invalid option %d", option); + return -1; + } + + return 0; +} + +int taos_options(TSDB_OPTION option, const void *arg, ...) { + static int32_t lock = 0; + + for (int i = 1; atomic_val_compare_exchange_32(&lock, 0, 1) != 0; ++i) { + if (i % 1000 == 0) { + tscInfo("haven't acquire lock after spin %d times.", i); + sched_yield(); + } + } + + int ret = taos_options_imp(option, (const char*)arg); + + atomic_store_32(&lock, 0); + return ret; +} + +#if 0 +#include "cJSON.h" +static setConfRet taos_set_config_imp(const char *config){ + setConfRet ret = {SET_CONF_RET_SUCC, {0}}; + static bool setConfFlag = false; + if (setConfFlag) { + ret.retCode = SET_CONF_RET_ERR_ONLY_ONCE; + strcpy(ret.retMsg, "configuration can only set once"); + return ret; + } + taosInitGlobalCfg(); + cJSON *root = cJSON_Parse(config); + if (root == NULL){ + ret.retCode = SET_CONF_RET_ERR_JSON_PARSE; + strcpy(ret.retMsg, "parse json error"); + return ret; + } + + int size = cJSON_GetArraySize(root); + if(!cJSON_IsObject(root) || size == 0) { + ret.retCode = SET_CONF_RET_ERR_JSON_INVALID; + strcpy(ret.retMsg, "json content is invalid, must be not empty object"); + return ret; + } + + if(size >= 1000) { + ret.retCode = SET_CONF_RET_ERR_TOO_LONG; + strcpy(ret.retMsg, "json object size is too long"); + return ret; + } + + for(int i = 0; i < size; i++){ + cJSON *item = cJSON_GetArrayItem(root, i); + if(!item) { + ret.retCode = SET_CONF_RET_ERR_INNER; + strcpy(ret.retMsg, "inner error"); + return ret; + } + if(!taosReadConfigOption(item->string, item->valuestring, NULL, NULL, TAOS_CFG_CSTATUS_OPTION, TSDB_CFG_CTYPE_B_CLIENT)){ + ret.retCode = SET_CONF_RET_ERR_PART; + if (strlen(ret.retMsg) == 0){ + snprintf(ret.retMsg, RET_MSG_LENGTH, "part error|%s", item->string); + }else{ + int tmp = RET_MSG_LENGTH - 1 - (int)strlen(ret.retMsg); + size_t leftSize = tmp >= 0 ? tmp : 0; + strncat(ret.retMsg, "|", leftSize); + tmp = RET_MSG_LENGTH - 1 - (int)strlen(ret.retMsg); + leftSize = tmp >= 0 ? tmp : 0; + strncat(ret.retMsg, item->string, leftSize); + } + } + } + cJSON_Delete(root); + setConfFlag = true; + return ret; +} + +setConfRet taos_set_config(const char *config){ + pthread_mutex_lock(&setConfMutex); + setConfRet ret = taos_set_config_imp(config); + pthread_mutex_unlock(&setConfMutex); + return ret; +} +#endif \ No newline at end of file diff --git a/source/common/src/tep.c b/source/common/src/tep.c new file mode 100644 index 0000000000..7dda5f5f6f --- /dev/null +++ b/source/common/src/tep.c @@ -0,0 +1,20 @@ + + +int taosGetFqdnPortFromEp(const char *ep, char *fqdn, uint16_t *port) { + *port = 0; + strcpy(fqdn, ep); + + char *temp = strchr(fqdn, ':'); + if (temp) { + *temp = 0; + *port = atoi(temp+1); + } + + if (*port == 0) { + *port = tsServerPort; + return -1; + } + + return 0; +} + diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 1737bd9def..a2cbdbff4a 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -166,16 +166,8 @@ int8_t tsEnableSlaveQuery = 1; int8_t tsEnableAdjustMaster = 1; // restful -int8_t tsEnableHttpModule = 1; int32_t tsRestRowLimit = 10240; -uint16_t tsHttpPort = 6041; // only tcp, range tcp[6041] -int32_t tsHttpCacheSessions = 1000; -int32_t tsHttpSessionExpire = 36000; -int32_t tsHttpMaxThreads = 2; -int8_t tsHttpEnableCompress = 1; -int8_t tsHttpEnableRecordSql = 0; int8_t tsTelegrafUseFieldNum = 0; -int8_t tsHttpDbNameMandatory = 0; // mqtt int8_t tsEnableMqttModule = 0; // not finished yet, not started it by default @@ -198,7 +190,6 @@ int8_t tsEnableStream = 1; // internal int8_t tsCompactMnodeWal = 0; int8_t tsPrintAuth = 0; -int8_t tscEmbedded = 0; char tsVnodeDir[PATH_MAX] = {0}; char tsDnodeDir[PATH_MAX] = {0}; char tsMnodeDir[PATH_MAX] = {0}; @@ -261,7 +252,6 @@ void taosSetAllDebugFlag() { sdbDebugFlag = debugFlag; dDebugFlag = debugFlag; vDebugFlag = debugFlag; - cDebugFlag = debugFlag; jniDebugFlag = debugFlag; odbcDebugFlag = debugFlag; httpDebugFlag = debugFlag; @@ -1161,16 +1151,6 @@ static void doInitGlobalConfig(void) { cfg.unitType = TAOS_CFG_UTYPE_NONE; taosInitConfigOption(cfg); - cfg.option = "http"; - cfg.ptr = &tsEnableHttpModule; - cfg.valType = TAOS_CFG_VTYPE_INT8; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 0; - cfg.maxValue = 1; - cfg.ptrLength = 1; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); - cfg.option = "mqtt"; cfg.ptr = &tsEnableMqttModule; cfg.valType = TAOS_CFG_VTYPE_INT8; @@ -1211,16 +1191,6 @@ static void doInitGlobalConfig(void) { cfg.unitType = TAOS_CFG_UTYPE_NONE; taosInitConfigOption(cfg); - cfg.option = "httpEnableRecordSql"; - cfg.ptr = &tsHttpEnableRecordSql; - cfg.valType = TAOS_CFG_VTYPE_INT8; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; - cfg.minValue = 0; - cfg.maxValue = 1; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); - cfg.option = "telegrafUseFieldNum"; cfg.ptr = &tsTelegrafUseFieldNum; cfg.valType = TAOS_CFG_VTYPE_INT8; @@ -1231,16 +1201,6 @@ static void doInitGlobalConfig(void) { cfg.unitType = TAOS_CFG_UTYPE_NONE; taosInitConfigOption(cfg); - cfg.option = "httpMaxThreads"; - cfg.ptr = &tsHttpMaxThreads; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; - cfg.minValue = 2; - cfg.maxValue = 1000000; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); - cfg.option = "restfulRowLimit"; cfg.ptr = &tsRestRowLimit; cfg.valType = TAOS_CFG_VTYPE_INT32; @@ -1251,16 +1211,6 @@ static void doInitGlobalConfig(void) { cfg.unitType = TAOS_CFG_UTYPE_NONE; taosInitConfigOption(cfg); - cfg.option = "httpDbNameMandatory"; - cfg.ptr = &tsHttpDbNameMandatory; - cfg.valType = TAOS_CFG_VTYPE_INT8; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; - cfg.minValue = 0; - cfg.maxValue = 1; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); - // debug flag cfg.option = "numOfLogLines"; cfg.ptr = &tsNumOfLogLines; @@ -1697,13 +1647,6 @@ int32_t taosCheckGlobalCfg() { tsNumOfCores = 1; } - if (tsHttpMaxThreads == 2) { - int32_t halfNumOfCores = tsNumOfCores >> 1; - if (halfNumOfCores > 2) { - tsHttpMaxThreads = halfNumOfCores; - } - } - if (tsMaxTablePerVnode < tsMinTablePerVnode) { uError("maxTablesPerVnode(%d) < minTablesPerVnode(%d), reset to minTablesPerVnode(%d)", tsMaxTablePerVnode, tsMinTablePerVnode, tsMinTablePerVnode); @@ -1738,24 +1681,6 @@ int32_t taosCheckGlobalCfg() { return 0; } -int taosGetFqdnPortFromEp(const char *ep, char *fqdn, uint16_t *port) { - *port = 0; - strcpy(fqdn, ep); - - char *temp = strchr(fqdn, ':'); - if (temp) { - *temp = 0; - *port = atoi(temp+1); - } - - if (*port == 0) { - *port = tsServerPort; - return -1; - } - - return 0; -} - /* * alter dnode 1 balance "vnode:1-dnode:2" */ diff --git a/source/dnode/mgmt/impl/test/sut/deploy.cpp b/source/dnode/mgmt/impl/test/sut/deploy.cpp index ad1667eac5..c484ea122d 100644 --- a/source/dnode/mgmt/impl/test/sut/deploy.cpp +++ b/source/dnode/mgmt/impl/test/sut/deploy.cpp @@ -19,7 +19,6 @@ void initLog(const char* path) { dDebugFlag = 0; vDebugFlag = 0; mDebugFlag = 207; - cDebugFlag = 0; jniDebugFlag = 0; tmrDebugFlag = 0; sdbDebugFlag = 0; diff --git a/source/libs/transport/inc/rpcLog.h b/source/libs/transport/inc/rpcLog.h index 6c4a281d2c..904680bbe6 100644 --- a/source/libs/transport/inc/rpcLog.h +++ b/source/libs/transport/inc/rpcLog.h @@ -23,11 +23,10 @@ extern "C" { #include "tlog.h" extern int32_t rpcDebugFlag; -extern int8_t tscEmbedded; -#define tFatal(...) { if (rpcDebugFlag & DEBUG_FATAL) { taosPrintLog("RPC FATAL ", tscEmbedded ? 255 : rpcDebugFlag, __VA_ARGS__); }} -#define tError(...) { if (rpcDebugFlag & DEBUG_ERROR) { taosPrintLog("RPC ERROR ", tscEmbedded ? 255 : rpcDebugFlag, __VA_ARGS__); }} -#define tWarn(...) { if (rpcDebugFlag & DEBUG_WARN) { taosPrintLog("RPC WARN ", tscEmbedded ? 255 : rpcDebugFlag, __VA_ARGS__); }} +#define tFatal(...) { if (rpcDebugFlag & DEBUG_FATAL) { taosPrintLog("RPC FATAL ", rpcDebugFlag, __VA_ARGS__); }} +#define tError(...) { if (rpcDebugFlag & DEBUG_ERROR) { taosPrintLog("RPC ERROR ", rpcDebugFlag, __VA_ARGS__); }} +#define tWarn(...) { if (rpcDebugFlag & DEBUG_WARN) { taosPrintLog("RPC WARN ", rpcDebugFlag, __VA_ARGS__); }} #define tInfo(...) { if (rpcDebugFlag & DEBUG_INFO) { taosPrintLog("RPC ", tscEmbedded ? 255 : rpcDebugFlag, __VA_ARGS__); }} #define tDebug(...) { if (rpcDebugFlag & DEBUG_DEBUG) { taosPrintLog("RPC ", rpcDebugFlag, __VA_ARGS__); }} #define tTrace(...) { if (rpcDebugFlag & DEBUG_TRACE) { taosPrintLog("RPC ", rpcDebugFlag, __VA_ARGS__); }} diff --git a/source/util/src/tnote.c b/source/util/src/tnote.c index 5606ab248d..d9356b7e40 100644 --- a/source/util/src/tnote.c +++ b/source/util/src/tnote.c @@ -49,15 +49,6 @@ int32_t taosInitNotes() { taosInitNote(tsNumOfLogLines, 1, &tsTscNote, name); } - if (tsHttpEnableRecordSql) { - snprintf(name, TSDB_FILENAME_LEN * 2, "%s/httpsql", tsLogDir); - taosInitNote(tsNumOfLogLines, 1, &tsHttpNote, name); - } - - if (tscEmbedded == 1) { - snprintf(name, TSDB_FILENAME_LEN * 2, "%s/taosinfo", tsLogDir); - taosInitNote(tsNumOfLogLines, 1, &tsInfoNote, name); - } #endif return 0; } From cfc632893a8b4d25afc3307fb5603554ba82ede6 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 10 Dec 2021 16:12:28 +0800 Subject: [PATCH 02/43] [td-10564] refactor and add test cases. --- include/common/tep.h | 2 +- include/common/tglobal.h | 2 +- include/os/osSemaphore.h | 2 +- include/util/tconfig.h | 8 +- source/client/CMakeLists.txt | 2 + source/client/inc/clientInt.h | 14 +- source/client/src/client.c | 53 +++++- source/client/src/clientImpl.c | 91 ++++----- source/client/src/tscEnv.c | 140 +++++++------- source/client/test/clientTests.cpp | 37 ++++ source/common/src/tep.c | 4 +- source/common/src/tglobal.c | 264 ++++++++++++-------------- source/dnode/mgmt/daemon/src/daemon.c | 2 +- source/os/src/osSemaphore.c | 6 +- source/os/src/osSysinfo.c | 16 +- source/util/src/tconfig.c | 7 +- source/util/src/tlog.c | 4 +- source/util/src/ttimer.c | 10 +- 18 files changed, 360 insertions(+), 304 deletions(-) diff --git a/include/common/tep.h b/include/common/tep.h index bdc25f6b82..715a116bb7 100644 --- a/include/common/tep.h +++ b/include/common/tep.h @@ -1,6 +1,6 @@ #ifndef TDENGINE_TEP_H #define TDENGINE_TEP_H -int taosGetFqdnPortFromEp(const char *ep, char *fqdn, uint16_t *port); +int taosGetFqdnPortFromEp(const char *ep, char *fqdn, uint16_t *port); #endif // TDENGINE_TEP_H diff --git a/include/common/tglobal.h b/include/common/tglobal.h index df5f6b7c22..e66c66de07 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -185,7 +185,7 @@ extern SDiskCfg tsDiskCfg[]; #define NEEDTO_COMPRESSS_MSG(size) (tsCompressMsgSize != -1 && (size) > tsCompressMsgSize) void taosInitGlobalCfg(); -int32_t taosCheckGlobalCfg(); +int32_t taosCheckAndPrintCfg(); int32_t taosCfgDynamicOptions(char *msg); bool taosCheckBalanceCfgOptions(const char *option, int32_t *vnodeId, int32_t *dnodeId); void taosAddDataDir(int index, char *v1, int level, int primary); diff --git a/include/os/osSemaphore.h b/include/os/osSemaphore.h index 86a9f10819..78112fc7a0 100644 --- a/include/os/osSemaphore.h +++ b/include/os/osSemaphore.h @@ -57,7 +57,7 @@ int64_t taosGetPthreadId(pthread_t thread); void taosResetPthread(pthread_t* thread); bool taosComparePthread(pthread_t first, pthread_t second); int32_t taosGetPId(); -int32_t taosGetCurrentAPPName(char* name, int32_t* len); +int32_t taosGetAppName(char* name, int32_t* len); #ifdef __cplusplus } diff --git a/include/util/tconfig.h b/include/util/tconfig.h index 15100423d5..18c21d238c 100644 --- a/include/util/tconfig.h +++ b/include/util/tconfig.h @@ -20,7 +20,7 @@ extern "C" { #endif -#define TSDB_CFG_MAX_NUM 123 +#define TSDB_CFG_MAX_NUM 119 #define TSDB_CFG_PRINT_LEN 23 #define TSDB_CFG_OPTION_LEN 24 #define TSDB_CFG_VALUE_LEN 41 @@ -83,11 +83,11 @@ extern int32_t tsGlobalConfigNum; extern char * tsCfgStatusStr[]; void taosReadGlobalLogCfg(); -int32_t taosReadGlobalCfg(); -void taosPrintGlobalCfg(); +int32_t taosReadCfgFromFile(); +void taosPrintCfg(); void taosDumpGlobalCfg(); -void taosInitConfigOption(SGlobalCfg cfg); +void taosAddConfigOption(SGlobalCfg cfg); SGlobalCfg *taosGetConfigOption(const char *option); #ifdef __cplusplus diff --git a/source/client/CMakeLists.txt b/source/client/CMakeLists.txt index 5a47602dcd..1a67faea9f 100644 --- a/source/client/CMakeLists.txt +++ b/source/client/CMakeLists.txt @@ -10,3 +10,5 @@ target_link_libraries( INTERFACE api PRIVATE os util common transport parser ) + +ADD_SUBDIRECTORY(test) \ No newline at end of file diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index 27a1c92d92..4fed198ab6 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -61,7 +61,7 @@ typedef struct SAppInstInfo { typedef struct SAppInfo { int64_t startTime; - char appName[TSDB_APPNAME_LEN]; + char appName[TSDB_APP_NAME_LEN]; char *ep; int32_t pid; int32_t numOfThreads; @@ -102,11 +102,23 @@ typedef struct SRequestObj { void *pInfo; // sql parse info, generated by parser module } SRequestObj; +extern int32_t tscReqRef; +extern void *tscQhandle; +extern int32_t tscConnRef; +extern void *tscRpcCache; +extern pthread_mutex_t rpcObjMutex; + void* createTscObj(const char* user, const char* auth, const char *ip, uint32_t port); void destroyTscObj(void* pTscObj); +void* createRequest(STscObj* pObj, __taos_async_fn_t fp, void* param, int32_t type); +void destroyRequest(void* p); + TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass, const char *auth, const char *db, uint16_t port); +void taos_init_imp(void); +int taos_options_imp(TSDB_OPTION option, const char *pStr); + #ifdef __cplusplus } #endif diff --git a/source/client/src/client.c b/source/client/src/client.c index b1663239e6..8aef77e9e7 100644 --- a/source/client/src/client.c +++ b/source/client/src/client.c @@ -13,11 +13,52 @@ * along with this program. If not, see . */ -//#include "taos.h" +#include "os.h" +#include "tdef.h" -//TAOS_RES *taos_query(TAOS *taos, const char *sql) { -// -//} +#include "tglobal.h" +#include "clientInt.h" +#include "tscLog.h" + +TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port) { + int32_t p = (port != 0)? port:tsServerPort; + + tscDebug("try to connect to %s:%u, user:%s db:%s", ip, p, user, db); + if (user == NULL) { + user = TSDB_DEFAULT_USER; + } + + if (pass == NULL) { + pass = TSDB_DEFAULT_PASS; + } + + return taos_connect_internal(ip, user, pass, NULL, db, p); +} + +TAOS *taos_connect_auth(const char *ip, const char *user, const char *auth, const char *db, uint16_t port) { + tscDebug("try to connect to %s:%u by auth, user:%s db:%s", ip, port, user, db); + if (user == NULL) { + user = TSDB_DEFAULT_USER; + } + + if (auth == NULL) { + tscError("No auth info is given, failed to connect to server"); + return NULL; + } + + return taos_connect_internal(ip, user, NULL, auth, db, port); +} + +TAOS *taos_connect_l(const char *ip, int ipLen, const char *user, int userLen, const char *pass, int passLen, const char *db, int dbLen, uint16_t port) { + char ipStr[TSDB_EP_LEN] = {0}; + char dbStr[TSDB_DB_NAME_LEN] = {0}; + char userStr[TSDB_USER_LEN] = {0}; + char passStr[TSDB_KEY_LEN] = {0}; + + strncpy(ipStr, ip, MIN(TSDB_EP_LEN - 1, ipLen)); + strncpy(userStr, user, MIN(TSDB_USER_LEN - 1, userLen)); + strncpy(passStr, pass, MIN(TSDB_KEY_LEN - 1, passLen)); + strncpy(dbStr, db, MIN(TSDB_DB_NAME_LEN - 1, dbLen)); + return taos_connect(ipStr, userStr, passStr, dbStr, port); +} -int taos_init() { return 0; } -void taos_cleanup(void) {} diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index e3e6e8d3ae..e7940e27b0 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -33,11 +33,9 @@ static bool validateDbName(const char* db) { return stringLengthCheck(db, TSDB_DB_NAME_LEN - 1); } -static SRequestObj* taosConnectImpl(const char *ip, const char *user, const char *auth, const char *db, uint16_t port, __taos_async_fn_t fp, void *param); +static STscObj* taosConnectImpl(const char *ip, const char *user, const char *auth, const char *db, uint16_t port, __taos_async_fn_t fp, void *param); TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass, const char *auth, const char *db, uint16_t port) { - STscObj *pObj = NULL; - if (!validateUserName(user)) { terrno = TSDB_CODE_TSC_INVALID_USER_LENGTH; return NULL; @@ -81,34 +79,7 @@ TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass, } } - SRequestObj *pRequest = taosConnectImpl(ip, user, auth, db, port, NULL, NULL); - if (pRequest != NULL) { - pObj = pRequest->pTscObj; - - pRequest->body.fp = NULL; - pRequest->body.param = pRequest; - -// tscBuildAndSendRequest(pRequest, NULL); - tsem_wait(&pRequest->body.rspSem); - - if (pRequest->code != TSDB_CODE_SUCCESS) { - if (pRequest->code == TSDB_CODE_RPC_FQDN_ERROR) { - printf("taos connect failed, reason: %s\n\n", taos_errstr(pRequest)); - } else { - printf("taos connect failed, reason: %s.\n\n", tstrerror(terrno)); - } - - taos_free_result(pRequest); - taos_close(pObj); - return NULL; - } - -// tscDebug("%p DB connection is opening, rpcObj: %p, dnodeConn:%p", pObj, pObj->pRpcObj, pObj->pRpcObj->pDnodeConn); - taos_free_result(pRequest); - return pObj; - } - - return NULL; + return taosConnectImpl(ip, user, auth, db, port, NULL, NULL); } int initEpSetFromCfg(const char *firstEp, const char *secondEp, SRpcCorEpSet *pEpSet) { @@ -147,7 +118,7 @@ int initEpSetFromCfg(const char *firstEp, const char *secondEp, SRpcCorEpSet *pE return 0; } -SRequestObj* taosConnectImpl(const char *ip, const char *user, const char *auth, const char *db, uint16_t port, __taos_async_fn_t fp, void *param) { +STscObj* taosConnectImpl(const char *ip, const char *user, const char *auth, const char *db, uint16_t port, __taos_async_fn_t fp, void *param) { if (taos_init() != TSDB_CODE_SUCCESS) { return NULL; } @@ -155,36 +126,40 @@ SRequestObj* taosConnectImpl(const char *ip, const char *user, const char *auth, STscObj *pObj = createTscObj(user, auth, ip, port); if (NULL == pObj) { terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - return NULL; + return pObj; } - SRequestObj *pRequest = (SRequestObj *)calloc(1, sizeof(SRequestObj)); - if (NULL == pRequest) { + // void *pRpcObj = NULL; + // + // char rpcKey[512] = {0}; + // snprintf(rpcKey, sizeof(rpcKey), "%s:%s:%s:%d", user, auth, ip, port); + // if (tscAcquireRpc(rpcKey, user, auth, &pRpcObj) != 0) { + // terrno = TSDB_CODE_RPC_NETWORK_UNAVAIL; + // return NULL; + // } + + SRequestObj *pRequest = createRequest(pObj, fp, param, TSDB_SQL_CONNECT); + if (pRequest == NULL) { + destroyTscObj(pObj); terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - free(pObj); + } + + // tscBuildAndSendRequest(pRequest, NULL); + // tsem_wait(&pRequest->body.rspSem); + if (pRequest->code != TSDB_CODE_SUCCESS) { + const char *errorMsg = (pRequest->code == TSDB_CODE_RPC_FQDN_ERROR) + ? taos_errstr(pRequest) + : tstrerror(terrno); + + printf("connect failed, reason: %s\n\n", errorMsg); + + taos_free_result(pRequest); + taos_close(pObj); return NULL; } - void *pRpcObj = NULL; - - char rpcKey[512] = {0}; - snprintf(rpcKey, sizeof(rpcKey), "%s:%s:%s:%d", user, auth, ip, port); - if (tscAcquireRpc(rpcKey, user, auth, &pRpcObj) != 0) { - terrno = TSDB_CODE_RPC_NETWORK_UNAVAIL; - return NULL; - } - - pObj->pRpcObj = (SRpcObj *)pRpcObj; - - pRequest->pTscObj = pObj; - pRequest->body.fp = fp; - pRequest->body.param = param; - pRequest->type = TSDB_SQL_CONNECT; - - tsem_init(&pRequest->body.rspSem, 0, 0); - - pObj->id = taosAddRef(tscConn, pObj); - registerSqlObj(pRequest); - - return pRequest; +// tscDebug("0x%"PRIx64" connection is opening, rpcObj: %p, dnodeConn:%p", pObj, pObj->pRpcObj, +// pObj->pRpcObj->pDnodeConn); + destroyRequest(pRequest); + return pObj; } \ No newline at end of file diff --git a/source/client/src/tscEnv.c b/source/client/src/tscEnv.c index d77d8b1bec..26ae9a8006 100644 --- a/source/client/src/tscEnv.c +++ b/source/client/src/tscEnv.c @@ -31,14 +31,12 @@ #define TSC_VAR_RELEASED 0 SAppInfo appInfo; -int32_t sentinel = TSC_VAR_NOT_RELEASE; - int32_t tscReqRef = -1; void *tscQhandle; int32_t tscConnRef = -1; void *tscRpcCache; // TODO removed from here. -static pthread_mutex_t rpcObjMutex; // mutex to protect open the rpc obj concurrently +pthread_mutex_t rpcObjMutex; // mutex to protect open the rpc obj concurrently static pthread_once_t tscinit = PTHREAD_ONCE_INIT; static pthread_mutex_t setConfMutex = PTHREAD_MUTEX_INITIALIZER; @@ -104,7 +102,6 @@ void* tscAcquireRpc(const char *key, const char *user, const char *secretEncrypt pthread_mutex_unlock(&rpcObjMutex); return pRpcObj; #endif - } void destroyTscObj(void *pTscObj) { @@ -139,6 +136,72 @@ void* createTscObj(const char* user, const char* auth, const char *ip, uint32_t tstrncpy(pObj->pass, auth, len); pthread_mutex_init(&pObj->mutex, NULL); + pObj->id = taosAddRef(tscConnRef, pObj); +} + +static void registerRequest(SRequestObj* pRequest) { + STscObj*pTscObj = (STscObj*) taosAcquireRef(tscConnRef, pRequest->pTscObj->id); + assert(pTscObj != NULL); + + // connection has been released already, abort creating request. + pRequest->self = taosAddRef(tscReqRef, pRequest); + + int32_t num = atomic_add_fetch_32(&pTscObj->numOfReqs, 1); + + SInstanceActivity* pActivity = &pTscObj->pAppInfo->summary; + int32_t total = atomic_add_fetch_32(&pActivity->totalRequests, 1); + int32_t currentInst = atomic_add_fetch_32(&pActivity->currentRequests, 1); + + tscDebug("0x%"PRIx64" new Request from 0x%"PRIx64", current:%d, app current:%d, total:%d", pRequest->self, pRequest->pTscObj->id, num, currentInst, total); +} + +void* createRequest(STscObj* pObj, __taos_async_fn_t fp, void* param, int32_t type) { + assert(pObj != NULL); + + SRequestObj *pRequest = (SRequestObj *)calloc(1, sizeof(SRequestObj)); + if (NULL == pRequest) { + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + return NULL; + } + + // TODO generated request uuid + pRequest->requestId = 0; + + pRequest->type = type; + pRequest->pTscObj = pObj; + pRequest->body.fp = fp; + pRequest->body.param = param; + tsem_init(&pRequest->body.rspSem, 0, 0); + + registerRequest(pRequest); +} + +static void deregisterRequest(SRequestObj* pRequest) { + assert(pRequest != NULL); + + STscObj* pTscObj = pRequest->pTscObj; + SInstanceActivity* pActivity = &pTscObj->pAppInfo->summary; + + taosReleaseRef(tscReqRef, pRequest->self); + + int32_t currentInst = atomic_sub_fetch_32(&pActivity->currentRequests, 1); + int32_t num = atomic_sub_fetch_32(&pTscObj->numOfReqs, 1); + + tscDebug("0x%"PRIx64" free Request from 0x%"PRIx64", current:%d, app current:%d", pRequest->self, pTscObj->id, num, currentInst); + taosReleaseRef(tscConnRef, pTscObj->id); +} + +void destroyRequest(void* p) { + assert(p != NULL); + SRequestObj* pRequest = *(SRequestObj**)p; + + assert(RID_VALID(pRequest->self)); + + tfree(pRequest->msgBuf); + tfree(pRequest->sqlstr); + tfree(pRequest->pInfo); + + deregisterRequest(pRequest); } static void tscInitLogFile() { @@ -167,11 +230,10 @@ void taos_init_imp(void) { deltaToUtcInitOnce(); taosInitGlobalCfg(); - taosReadGlobalCfg(); + taosReadCfgFromFile(); tscInitLogFile(); - - if (taosCheckGlobalCfg()) { + if (taosCheckAndPrintCfg()) { tscInitRes = -1; return; } @@ -179,8 +241,7 @@ void taos_init_imp(void) { taosInitNotes(); rpcInit(); - tscDebug("starting to initialize TAOS client ..."); - tscDebug("Local End Point is:%s", tsLocalEp); + tscDebug("starting to initialize TAOS client ...\nLocal End Point is:%s", tsLocalEp); taosSetCoreDump(true); @@ -202,55 +263,16 @@ void taos_init_imp(void) { pthread_mutex_init(&rpcObjMutex, NULL); tscConnRef = taosOpenRef(200, destroyTscObj); - tscReqRef = taosOpenRef(40960, tscFreeRegisteredSqlObj); + tscReqRef = taosOpenRef(40960, destroyRequest); - taosGetCurrentAPPName(appInfo.appName, NULL); + taosGetAppName(appInfo.appName, NULL); appInfo.pid = taosGetPId(); appInfo.startTime = taosGetTimestampMs(); tscDebug("client is initialized successfully"); } -int taos_init() { - pthread_once(&tscinit, taos_init_imp); - return tscInitRes; -} - -// this function may be called by user or system, or by both simultaneously. -void taos_cleanup(void) { - tscDebug("start to cleanup client environment"); - - if (atomic_val_compare_exchange_32(&sentinel, TSC_VAR_NOT_RELEASE, TSC_VAR_RELEASED) != TSC_VAR_NOT_RELEASE) { - return; - } - - int32_t id = tscReqRef; - tscReqRef = -1; - taosCloseRef(id); - - void* p = tscQhandle; - tscQhandle = NULL; - taosCleanUpScheduler(p); - - id = tscConnRef; - tscConnRef = -1; - taosCloseRef(id); - - p = tscRpcCache; - tscRpcCache = NULL; - - if (p != NULL) { - taosCacheCleanup(p); - pthread_mutex_destroy(&rpcObjMutex); - } - - pthread_mutex_destroy(&setConfMutex); - - rpcCleanup(); - taosCloseLog(); -} - -static int taos_options_imp(TSDB_OPTION option, const char *pStr) { +int taos_options_imp(TSDB_OPTION option, const char *pStr) { SGlobalCfg *cfg = NULL; switch (option) { @@ -405,22 +427,6 @@ static int taos_options_imp(TSDB_OPTION option, const char *pStr) { return 0; } -int taos_options(TSDB_OPTION option, const void *arg, ...) { - static int32_t lock = 0; - - for (int i = 1; atomic_val_compare_exchange_32(&lock, 0, 1) != 0; ++i) { - if (i % 1000 == 0) { - tscInfo("haven't acquire lock after spin %d times.", i); - sched_yield(); - } - } - - int ret = taos_options_imp(option, (const char*)arg); - - atomic_store_32(&lock, 0); - return ret; -} - #if 0 #include "cJSON.h" static setConfRet taos_set_config_imp(const char *config){ diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index e69de29bb2..4f848a0750 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include +#include "tglobal.h" +#pragma GCC diagnostic ignored "-Wwrite-strings" + +#pragma GCC diagnostic ignored "-Wunused-function" +#pragma GCC diagnostic ignored "-Wunused-variable" +#pragma GCC diagnostic ignored "-Wsign-compare" + +#include "taos.h" + +namespace { +} // namespace + +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + +TEST(testCase, driverInit_Test) { + taos_init(); +} \ No newline at end of file diff --git a/source/common/src/tep.c b/source/common/src/tep.c index 7dda5f5f6f..3f500faf89 100644 --- a/source/common/src/tep.c +++ b/source/common/src/tep.c @@ -1,4 +1,6 @@ - +#include "os.h" +#include "tep.h" +#include "tglobal.h" int taosGetFqdnPortFromEp(const char *ep, char *fqdn, uint16_t *port) { *port = 0; diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index a2cbdbff4a..2e68fb2742 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -25,6 +25,7 @@ #include "tutil.h" #include "ttimezone.h" #include "tlocale.h" +#include "tep.h" // cluster char tsFirst[TSDB_EP_LEN] = {0}; @@ -325,7 +326,7 @@ int32_t taosCfgDynamicOptions(char *msg) { if (strncasecmp(option, "resetlog", 8) == 0) { taosResetLog(); - taosPrintGlobalCfg(); + taosPrintCfg(); return 0; } @@ -389,7 +390,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 0; cfg.ptrLength = TSDB_EP_LEN; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "secondEp"; cfg.ptr = tsSecond; @@ -399,7 +400,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 0; cfg.ptrLength = TSDB_EP_LEN; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "fqdn"; cfg.ptr = tsLocalFqdn; @@ -409,7 +410,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 0; cfg.ptrLength = TSDB_FQDN_LEN; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); // port cfg.option = "serverPort"; @@ -420,7 +421,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 65056; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); // directory cfg.option = "configDir"; @@ -431,7 +432,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 0; cfg.ptrLength = TSDB_FILENAME_LEN; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "logDir"; cfg.ptr = tsLogDir; @@ -441,7 +442,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 0; cfg.ptrLength = TSDB_FILENAME_LEN; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "scriptDir"; cfg.ptr = tsScriptDir; @@ -451,7 +452,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 0; cfg.ptrLength = TSDB_FILENAME_LEN; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "dataDir"; cfg.ptr = tsDataDir; @@ -461,7 +462,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 0; cfg.ptrLength = TSDB_FILENAME_LEN; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "arbitrator"; cfg.ptr = tsArbitrator; @@ -471,7 +472,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 0; cfg.ptrLength = TSDB_EP_LEN; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); // dnode configs cfg.option = "numOfThreadsPerCore"; @@ -482,7 +483,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 10; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "numOfCommitThreads"; cfg.ptr = &tsNumOfCommitThreads; @@ -492,7 +493,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 100; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "ratioOfQueryCores"; cfg.ptr = &tsRatioOfQueryCores; @@ -502,7 +503,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 2.0f; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "maxNumOfDistinctRes"; cfg.ptr = &tsMaxNumOfDistinctResults; @@ -512,7 +513,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 10000*10000; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "numOfMnodes"; cfg.ptr = &tsNumOfMnodes; @@ -522,7 +523,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 3; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "vnodeBak"; cfg.ptr = &tsEnableVnodeBak; @@ -532,7 +533,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 1; cfg.ptrLength = 1; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "telemetryReporting"; cfg.ptr = &tsEnableTelemetryReporting; @@ -542,7 +543,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 1; cfg.ptrLength = 1; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "balance"; cfg.ptr = &tsEnableBalance; @@ -552,7 +553,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 1; cfg.ptrLength = 1; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "balanceInterval"; cfg.ptr = &tsBalanceInterval; @@ -562,7 +563,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 30000; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); // 0-any; 1-mnode; 2-vnode cfg.option = "role"; @@ -573,7 +574,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 2; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); // timer cfg.option = "maxTmrCtrl"; @@ -584,7 +585,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 2048; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "monitorInterval"; cfg.ptr = &tsMonitorInterval; @@ -594,7 +595,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 600; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_SECOND; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "offlineThreshold"; cfg.ptr = &tsOfflineThreshold; @@ -604,7 +605,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 86400 * 365; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_SECOND; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "rpcTimer"; cfg.ptr = &tsRpcTimer; @@ -614,7 +615,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 3000; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_MS; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "rpcForceTcp"; cfg.ptr = &tsRpcForceTcp; @@ -624,7 +625,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 1; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "rpcMaxTime"; cfg.ptr = &tsRpcMaxTime; @@ -634,7 +635,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 7200; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_SECOND; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "statusInterval"; cfg.ptr = &tsStatusInterval; @@ -644,7 +645,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 10; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_SECOND; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "shellActivityTimer"; cfg.ptr = &tsShellActivityTimer; @@ -654,7 +655,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 120; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_SECOND; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "minSlidingTime"; cfg.ptr = &tsMinSlidingTime; @@ -664,7 +665,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 1000000; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_MS; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "minIntervalTime"; cfg.ptr = &tsMinIntervalTime; @@ -674,7 +675,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 1000000; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_MS; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "maxStreamCompDelay"; cfg.ptr = &tsMaxStreamComputDelay; @@ -684,7 +685,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 1000000000; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_MS; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "maxFirstStreamCompDelay"; cfg.ptr = &tsStreamCompStartDelay; @@ -694,7 +695,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 1000000000; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_MS; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "retryStreamCompDelay"; cfg.ptr = &tsRetryStreamCompDelay; @@ -705,7 +706,7 @@ static void doInitGlobalConfig(void) { cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_MS; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "streamCompDelayRatio"; cfg.ptr = &tsStreamComputDelayRatio; cfg.valType = TAOS_CFG_VTYPE_FLOAT; @@ -714,7 +715,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 0.9f; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "maxVgroupsPerDb"; cfg.ptr = &tsMaxVgroupsPerDb; @@ -724,7 +725,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 8192; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); // database configs cfg.option = "maxTablesPerVnode"; @@ -735,7 +736,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = TSDB_MAX_TABLES; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "minTablesPerVnode"; cfg.ptr = &tsMinTablePerVnode; @@ -745,7 +746,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = TSDB_MAX_TABLES; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "tableIncStepPerVnode"; cfg.ptr = &tsTableIncStepPerVnode; @@ -755,7 +756,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = TSDB_MAX_TABLES; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "cache"; cfg.ptr = &tsCacheBlockSize; @@ -765,7 +766,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = TSDB_MAX_CACHE_BLOCK_SIZE; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_MB; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "blocks"; cfg.ptr = &tsBlocksPerVnode; @@ -775,7 +776,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = TSDB_MAX_TOTAL_BLOCKS; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "days"; cfg.ptr = &tsDaysPerFile; @@ -785,7 +786,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = TSDB_MAX_DAYS_PER_FILE; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "keep"; cfg.ptr = &tsDaysToKeep; @@ -795,7 +796,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = TSDB_MAX_KEEP; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "minRows"; cfg.ptr = &tsMinRowsInFileBlock; @@ -805,7 +806,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = TSDB_MAX_MIN_ROW_FBLOCK; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "maxRows"; cfg.ptr = &tsMaxRowsInFileBlock; @@ -815,7 +816,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = TSDB_MAX_MAX_ROW_FBLOCK; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "precision"; cfg.ptr = &tsTimePrecision; @@ -825,7 +826,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = TSDB_MAX_PRECISION; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "comp"; cfg.ptr = &tsCompression; @@ -835,7 +836,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = TSDB_MAX_COMP_LEVEL; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "walLevel"; cfg.ptr = &tsWAL; @@ -845,7 +846,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = TSDB_MAX_WAL_LEVEL; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "fsync"; cfg.ptr = &tsFsyncPeriod; @@ -855,7 +856,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = TSDB_MAX_FSYNC_PERIOD; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "replica"; cfg.ptr = &tsReplications; @@ -865,7 +866,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = TSDB_MAX_DB_REPLICA_OPTION; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "partitions"; cfg.ptr = &tsPartitons; @@ -875,7 +876,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = TSDB_MAX_DB_PARTITON_OPTION; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "quorum"; cfg.ptr = &tsQuorum; @@ -885,7 +886,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = TSDB_MAX_DB_QUORUM_OPTION; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "update"; cfg.ptr = &tsUpdate; @@ -895,7 +896,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = TSDB_MAX_DB_UPDATE; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "cachelast"; cfg.ptr = &tsCacheLastRow; @@ -905,7 +906,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = TSDB_MAX_DB_CACHE_LAST_ROW; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "mqttHostName"; cfg.ptr = tsMqttHostName; @@ -915,7 +916,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 0; cfg.ptrLength = TSDB_MQTT_HOSTNAME_LEN; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "mqttPort"; cfg.ptr = tsMqttPort; @@ -925,7 +926,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 0; cfg.ptrLength = TSDB_MQTT_PORT_LEN; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "mqttTopic"; cfg.ptr = tsMqttTopic; @@ -935,7 +936,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 0; cfg.ptrLength = TSDB_MQTT_TOPIC_LEN; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "compressMsgSize"; cfg.ptr = &tsCompressMsgSize; @@ -945,7 +946,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 100000000.0f; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "compressColData"; cfg.ptr = &tsCompressColData; @@ -955,7 +956,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 100000000.0f; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "maxSQLLength"; cfg.ptr = &tsMaxSQLStringLen; @@ -965,7 +966,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = TSDB_MAX_ALLOWED_SQL_LEN; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_BYTE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "maxWildCardsLength"; cfg.ptr = &tsMaxWildCardsLen; @@ -975,7 +976,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = TSDB_MAX_FIELD_LEN; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_BYTE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "maxRegexStringLen"; cfg.ptr = &tsMaxRegexStringLen; @@ -985,7 +986,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = TSDB_MAX_FIELD_LEN; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_BYTE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "maxNumOfOrderedRes"; cfg.ptr = &tsMaxNumOfOrderedResults; @@ -995,7 +996,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = TSDB_MAX_ALLOWED_SQL_LEN; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "queryBufferSize"; cfg.ptr = &tsQueryBufferSize; @@ -1005,7 +1006,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 500000000000.0f; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_BYTE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "retrieveBlockingModel"; cfg.ptr = &tsRetrieveBlockingModel; @@ -1015,7 +1016,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 1; cfg.ptrLength = 1; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "keepColumnName"; cfg.ptr = &tsKeepOriginalColumnName; @@ -1025,7 +1026,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 1; cfg.ptrLength = 1; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); // locale & charset cfg.option = "timezone"; @@ -1036,7 +1037,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 0; cfg.ptrLength = TSDB_TIMEZONE_LEN; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "locale"; cfg.ptr = tsLocale; @@ -1046,7 +1047,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 0; cfg.ptrLength = TSDB_LOCALE_LEN; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "charset"; cfg.ptr = tsCharset; @@ -1056,7 +1057,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 0; cfg.ptrLength = TSDB_LOCALE_LEN; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); // connect configs cfg.option = "maxShellConns"; @@ -1067,7 +1068,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 50000000; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "maxConnections"; cfg.ptr = &tsMaxConnections; @@ -1077,7 +1078,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 100000; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "minimalLogDirGB"; cfg.ptr = &tsMinimalLogDirGB; @@ -1087,7 +1088,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 10000000; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_GB; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "minimalTmpDirGB"; cfg.ptr = &tsReservedTmpDirectorySpace; @@ -1097,7 +1098,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 10000000; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_GB; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "minimalDataDirGB"; cfg.ptr = &tsMinimalDataDirGB; @@ -1107,7 +1108,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 10000000; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_GB; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); // module configs cfg.option = "mnodeEqualVnodeNum"; @@ -1118,7 +1119,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 1000; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); // module configs cfg.option = "flowctrl"; @@ -1129,7 +1130,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 1; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "slaveQuery"; cfg.ptr = &tsEnableSlaveQuery; @@ -1139,7 +1140,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 1; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "adjustMaster"; cfg.ptr = &tsEnableAdjustMaster; @@ -1149,7 +1150,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 1; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "mqtt"; cfg.ptr = &tsEnableMqttModule; @@ -1159,7 +1160,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 1; cfg.ptrLength = 1; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "monitor"; cfg.ptr = &tsEnableMonitorModule; @@ -1169,7 +1170,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 1; cfg.ptrLength = 1; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "stream"; cfg.ptr = &tsEnableStream; @@ -1179,7 +1180,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 1; cfg.ptrLength = 1; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "topicBianryLen"; cfg.ptr = &tsTopicBianryLen; @@ -1189,7 +1190,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 16000; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "telegrafUseFieldNum"; cfg.ptr = &tsTelegrafUseFieldNum; @@ -1199,7 +1200,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 1; cfg.ptrLength = 1; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "restfulRowLimit"; cfg.ptr = &tsRestRowLimit; @@ -1209,7 +1210,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 10000000; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); // debug flag cfg.option = "numOfLogLines"; @@ -1220,7 +1221,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 2000000000; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "logKeepDays"; cfg.ptr = &tsLogKeepDays; @@ -1230,7 +1231,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 365000; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "asyncLog"; cfg.ptr = &tsAsyncLog; @@ -1240,7 +1241,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 1; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "debugFlag"; cfg.ptr = &debugFlag; @@ -1250,7 +1251,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 255; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "mDebugFlag"; cfg.ptr = &mDebugFlag; @@ -1260,7 +1261,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 255; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "dDebugFlag"; cfg.ptr = &dDebugFlag; @@ -1270,7 +1271,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 255; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "sDebugFlag"; cfg.ptr = &sDebugFlag; @@ -1280,7 +1281,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 255; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "wDebugFlag"; cfg.ptr = &wDebugFlag; @@ -1290,7 +1291,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 255; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "sdbDebugFlag"; @@ -1301,7 +1302,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 255; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "rpcDebugFlag"; cfg.ptr = &rpcDebugFlag; @@ -1311,7 +1312,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 255; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "tmrDebugFlag"; cfg.ptr = &tmrDebugFlag; @@ -1321,7 +1322,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 255; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "cDebugFlag"; cfg.ptr = &cDebugFlag; @@ -1331,7 +1332,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 255; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "jniDebugFlag"; cfg.ptr = &jniDebugFlag; @@ -1341,7 +1342,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 255; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "odbcDebugFlag"; cfg.ptr = &odbcDebugFlag; @@ -1351,7 +1352,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 255; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "uDebugFlag"; cfg.ptr = &uDebugFlag; @@ -1361,7 +1362,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 255; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "httpDebugFlag"; cfg.ptr = &httpDebugFlag; @@ -1371,7 +1372,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 255; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "mqttDebugFlag"; cfg.ptr = &mqttDebugFlag; @@ -1381,7 +1382,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 255; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "monDebugFlag"; cfg.ptr = &monDebugFlag; @@ -1391,7 +1392,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 255; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "qDebugFlag"; cfg.ptr = &qDebugFlag; @@ -1401,7 +1402,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 255; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "vDebugFlag"; cfg.ptr = &vDebugFlag; @@ -1411,7 +1412,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 255; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "tsdbDebugFlag"; cfg.ptr = &tsdbDebugFlag; @@ -1421,7 +1422,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 255; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "cqDebugFlag"; cfg.ptr = &cqDebugFlag; @@ -1431,7 +1432,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 255; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "enableRecordSql"; cfg.ptr = &tsTscEnableRecordSql; @@ -1441,7 +1442,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 1; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "enableCoreFile"; cfg.ptr = &tsEnableCoreFile; @@ -1451,7 +1452,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 1; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); // version info cfg.option = "gitinfo"; @@ -1462,7 +1463,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 0; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "gitinfoOfInternal"; cfg.ptr = gitinfoOfInternal; @@ -1472,7 +1473,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 0; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "buildinfo"; cfg.ptr = buildinfo; @@ -1482,7 +1483,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 0; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "version"; cfg.ptr = version; @@ -1492,7 +1493,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 0; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "maxBinaryDisplayWidth"; cfg.ptr = &tsMaxBinaryDisplayWidth; @@ -1502,7 +1503,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 65536; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "tempDir"; cfg.ptr = tsTempDir; @@ -1512,7 +1513,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 0; cfg.ptrLength = PATH_MAX; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "tsdbMetaCompactRatio"; cfg.ptr = &tsTsdbMetaCompactRatio; @@ -1522,7 +1523,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 100; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); // enable kill long query cfg.option = "deadLockKillQuery"; @@ -1533,7 +1534,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 1; cfg.ptrLength = 1; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); #ifdef TD_TSZ // lossy compress @@ -1545,7 +1546,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 0; cfg.ptrLength = tListLen(lossyColumns); cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "fPrecision"; cfg.ptr = &fPrecision; @@ -1557,7 +1558,7 @@ static void doInitGlobalConfig(void) { cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "dPrecision"; cfg.ptr = &dPrecision; @@ -1567,7 +1568,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = MAX_FLOAT; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "maxRange"; cfg.ptr = &maxRange; @@ -1577,7 +1578,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 65536; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "range"; cfg.ptr = &curRange; @@ -1587,10 +1588,10 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 65536; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); assert(tsGlobalConfigNum == TSDB_CFG_MAX_NUM); #else - assert(tsGlobalConfigNum == TSDB_CFG_MAX_NUM - 5); + assert(tsGlobalConfigNum == (TSDB_CFG_MAX_NUM - 5)); #endif } @@ -1599,7 +1600,7 @@ void taosInitGlobalCfg() { pthread_once(&tsInitGlobalCfgOnce, doInitGlobalConfig); } -int32_t taosCheckGlobalCfg() { +int32_t taosCheckAndPrintCfg() { char fqdn[TSDB_FQDN_LEN]; uint16_t port; @@ -1653,30 +1654,13 @@ int32_t taosCheckGlobalCfg() { tsMaxTablePerVnode = tsMinTablePerVnode; } - // todo refactor - tsVersion = 0; - for (int ver = 0, i = 0; i < TSDB_VERSION_LEN; ++i) { - if (version[i] >= '0' && version[i] <= '9') { - ver = ver * 10 + (version[i] - '0'); - } else if (version[i] == '.') { - tsVersion |= ver & 0xFF; - tsVersion <<= 8; - - ver = 0; - } else if (version[i] == 0) { - tsVersion |= ver & 0xFF; - - break; - } - } - if (tsQueryBufferSize >= 0) { tsQueryBufferSizeBytes = tsQueryBufferSize * 1048576UL; } uInfo(" check global cfg completed"); uInfo("=================================="); - taosPrintGlobalCfg(); + taosPrintCfg(); return 0; } diff --git a/source/dnode/mgmt/daemon/src/daemon.c b/source/dnode/mgmt/daemon/src/daemon.c index cf9a960062..32a5380ff8 100644 --- a/source/dnode/mgmt/daemon/src/daemon.c +++ b/source/dnode/mgmt/daemon/src/daemon.c @@ -117,7 +117,7 @@ int dmnReadConfig(const char *path) { return -1; } - if (taosCheckGlobalCfg() != 0) { + if (taosCheckAndPrintCfg() != 0) { uError("failed to check global config"); return -1; } diff --git a/source/os/src/osSemaphore.c b/source/os/src/osSemaphore.c index 2385d10285..0d7066b5c8 100644 --- a/source/os/src/osSemaphore.c +++ b/source/os/src/osSemaphore.c @@ -42,7 +42,7 @@ bool taosComparePthread(pthread_t first, pthread_t second) { return first.p == s int32_t taosGetPId() { return GetCurrentProcessId(); } -int32_t taosGetCurrentAPPName(char* name, int32_t* len) { +int32_t taosGetAppName(char* name, int32_t* len) { char filepath[1024] = {0}; GetModuleFileName(NULL, filepath, MAX_PATH); @@ -358,7 +358,7 @@ bool taosComparePthread(pthread_t first, pthread_t second) { return pthread_equa int32_t taosGetPId() { return (int32_t)getpid(); } -int32_t taosGetCurrentAPPName(char *name, int32_t *len) { +int32_t taosGetAppName(char *name, int32_t *len) { char buf[PATH_MAX + 1]; buf[0] = '\0'; proc_name(getpid(), buf, sizeof(buf) - 1); @@ -392,7 +392,7 @@ void taosResetPthread(pthread_t* thread) { *thread = 0; } bool taosComparePthread(pthread_t first, pthread_t second) { return first == second; } int32_t taosGetPId() { return getpid(); } -int32_t taosGetCurrentAPPName(char* name, int32_t* len) { +int32_t taosGetAppName(char* name, int32_t* len) { const char* self = "/proc/self/exe"; char path[PATH_MAX] = {0}; diff --git a/source/os/src/osSysinfo.c b/source/os/src/osSysinfo.c index ca817c4c1e..f892b4d8c0 100644 --- a/source/os/src/osSysinfo.c +++ b/source/os/src/osSysinfo.c @@ -714,7 +714,7 @@ static void taosGetSystemLocale() { // get and set default locale //printf("locale not configured, set to system default:%s", tsLocale); } - /* if user does not specify the charset, extract it from locale */ + // if user does not specify the charset, extract it from locale char *str = strrchr(tsLocale, sep); if (str != NULL) { str++; @@ -1118,13 +1118,13 @@ char *taosGetCmdlineByPID(int pid) { SysNameInfo taosGetSysNameInfo() { SysNameInfo info = {0}; - struct utsname buf; - if (!uname(&buf)) { - info.sysname = buf.sysname; - info.sysname == buf.nodename; - info.sysname = buf.release; - info.sysname = buf.version; - info.sysname = buf.machine; + struct utsname uts; + if (!uname(&uts)) { + info.sysname = strdup(uts.sysname); + info.nodename = strdup(uts.nodename); + info.release = strdup(uts.release); + info.version = strdup(uts.version); + info.machine = strdup(uts.machine); } return info; diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index 8a6f389366..726247d450 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -282,7 +282,7 @@ static void taosReadConfigOption(const char *option, char *value, char *value2, } } -void taosInitConfigOption(SGlobalCfg cfg) { +void taosAddConfigOption(SGlobalCfg cfg) { tsGlobalConfig[tsGlobalConfigNum++] = cfg; } @@ -335,7 +335,7 @@ void taosReadGlobalLogCfg() { fclose(fp); } -int32_t taosReadGlobalCfg() { +int32_t taosReadCfgFromFile() { char * line, *option, *value, *value2, *value3; int olen, vlen, vlen2, vlen3; char fileName[PATH_MAX] = {0}; @@ -396,7 +396,7 @@ int32_t taosReadGlobalCfg() { return 0; } -void taosPrintGlobalCfg() { +void taosPrintCfg() { uInfo(" taos config & system info:"); uInfo("=================================="); @@ -443,7 +443,6 @@ void taosPrintGlobalCfg() { } taosPrintOsInfo(); - // taosPrintDataDirCfg(); uInfo("=================================="); } diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index e6cc3a53af..24293fe9d3 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -112,7 +112,7 @@ static int32_t taosPushLogBuffer(SLogBuff *tLogBuff, char *msg, int32_t msgLen static SLogBuff *taosLogBuffNew(int32_t bufSize); static void taosCloseLogByFd(int32_t oldFd); static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum); -extern void taosPrintGlobalCfg(); +extern void taosPrintCfg(); static int32_t taosCompressFile(char *srcFileName, char *destFileName); static int32_t taosStartLog() { @@ -222,7 +222,7 @@ static void *taosThreadToOpenNewFile(void *param) { uInfo(" new log file:%d is opened", tsLogObj.flag); uInfo("=================================="); - taosPrintGlobalCfg(); + taosPrintCfg(); taosKeepOldLog(keepName); return NULL; diff --git a/source/util/src/ttimer.c b/source/util/src/ttimer.c index 56186d9b24..1fdc2257d7 100644 --- a/source/util/src/ttimer.c +++ b/source/util/src/ttimer.c @@ -20,12 +20,10 @@ #include "tutil.h" #include "taoserror.h" -extern int8_t tscEmbedded; - -#define tmrFatal(...) { if (tmrDebugFlag & DEBUG_FATAL) { taosPrintLog("TMR FATAL ", tscEmbedded ? 255 : tmrDebugFlag, __VA_ARGS__); }} -#define tmrError(...) { if (tmrDebugFlag & DEBUG_ERROR) { taosPrintLog("TMR ERROR ", tscEmbedded ? 255 : tmrDebugFlag, __VA_ARGS__); }} -#define tmrWarn(...) { if (tmrDebugFlag & DEBUG_WARN) { taosPrintLog("TMR WARN ", tscEmbedded ? 255 : tmrDebugFlag, __VA_ARGS__); }} -#define tmrInfo(...) { if (tmrDebugFlag & DEBUG_INFO) { taosPrintLog("TMR ", tscEmbedded ? 255 : tmrDebugFlag, __VA_ARGS__); }} +#define tmrFatal(...) { if (tmrDebugFlag & DEBUG_FATAL) { taosPrintLog("TMR FATAL ", tmrDebugFlag, __VA_ARGS__); }} +#define tmrError(...) { if (tmrDebugFlag & DEBUG_ERROR) { taosPrintLog("TMR ERROR ", tmrDebugFlag, __VA_ARGS__); }} +#define tmrWarn(...) { if (tmrDebugFlag & DEBUG_WARN) { taosPrintLog("TMR WARN ", tmrDebugFlag, __VA_ARGS__); }} +#define tmrInfo(...) { if (tmrDebugFlag & DEBUG_INFO) { taosPrintLog("TMR ", tmrDebugFlag, __VA_ARGS__); }} #define tmrDebug(...) { if (tmrDebugFlag & DEBUG_DEBUG) { taosPrintLog("TMR ", tmrDebugFlag, __VA_ARGS__); }} #define tmrTrace(...) { if (tmrDebugFlag & DEBUG_TRACE) { taosPrintLog("TMR ", tmrDebugFlag, __VA_ARGS__); }} From 33e384a7aac6e94b74fdf85406eef1dbc0c7470e Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 10 Dec 2021 16:27:57 +0800 Subject: [PATCH 03/43] [td-10564] refactor and add test cases. --- source/client/src/clientmain.c | 86 +++++++++++++++++++++++++++++++ source/client/test/CMakeLists.txt | 18 +++++++ 2 files changed, 104 insertions(+) create mode 100644 source/client/src/clientmain.c create mode 100644 source/client/test/CMakeLists.txt diff --git a/source/client/src/clientmain.c b/source/client/src/clientmain.c new file mode 100644 index 0000000000..7dac304c58 --- /dev/null +++ b/source/client/src/clientmain.c @@ -0,0 +1,86 @@ +#include "clientInt.h" +#include "trpc.h" +#include "os.h" +#include "taosmsg.h" +#include "tcache.h" +#include "tconfig.h" +#include "tglobal.h" +#include "tnote.h" +#include "tref.h" +#include "tscLog.h" +#include "tsched.h" +#include "ttime.h" +#include "ttimezone.h" + +#define TSC_VAR_NOT_RELEASE 1 +#define TSC_VAR_RELEASED 0 + +static int32_t sentinel = TSC_VAR_NOT_RELEASE; +static pthread_once_t tscinit = PTHREAD_ONCE_INIT; + +extern int32_t tscInitRes; + +int taos_options(TSDB_OPTION option, const void *arg, ...) { + static int32_t lock = 0; + + for (int i = 1; atomic_val_compare_exchange_32(&lock, 0, 1) != 0; ++i) { + if (i % 1000 == 0) { + tscInfo("haven't acquire lock after spin %d times.", i); + sched_yield(); + } + } + + int ret = taos_options_imp(option, (const char*)arg); + + atomic_store_32(&lock, 0); + return ret; +} + +int taos_init() { + pthread_once(&tscinit, taos_init_imp); + return tscInitRes; +} + +// this function may be called by user or system, or by both simultaneously. +void taos_cleanup(void) { + tscDebug("start to cleanup client environment"); + + if (atomic_val_compare_exchange_32(&sentinel, TSC_VAR_NOT_RELEASE, TSC_VAR_RELEASED) != TSC_VAR_NOT_RELEASE) { + return; + } + + int32_t id = tscReqRef; + tscReqRef = -1; + taosCloseRef(id); + + void* p = tscQhandle; + tscQhandle = NULL; + taosCleanUpScheduler(p); + + id = tscConnRef; + tscConnRef = -1; + taosCloseRef(id); + + p = tscRpcCache; + tscRpcCache = NULL; + + if (p != NULL) { + taosCacheCleanup(p); + pthread_mutex_destroy(&rpcObjMutex); + } + + rpcCleanup(); + taosCloseLog(); +} + +void taos_close(TAOS* taos) { + +} + +const char *taos_errstr(TAOS_RES *res) { + +} + +void taos_free_result(TAOS_RES *res) { + +} \ No newline at end of file diff --git a/source/client/test/CMakeLists.txt b/source/client/test/CMakeLists.txt new file mode 100644 index 0000000000..a3f2ad88bb --- /dev/null +++ b/source/client/test/CMakeLists.txt @@ -0,0 +1,18 @@ + +MESSAGE(STATUS "build parser unit test") + +# GoogleTest requires at least C++11 +SET(CMAKE_CXX_STANDARD 11) +AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) + +ADD_EXECUTABLE(clientTest ${SOURCE_LIST}) +TARGET_LINK_LIBRARIES( + clientTest + PUBLIC os util common transport gtest taos +) + +TARGET_INCLUDE_DIRECTORIES( + clientTest + PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/client/" + PRIVATE "${CMAKE_SOURCE_DIR}/source/libs/client/inc" +) From bf97ce3d9d348cf205cae0a93ba7311ccb3da5bb Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 10 Dec 2021 16:28:51 +0800 Subject: [PATCH 04/43] [td-10564] refactor and add test cases. --- source/client/src/tscEnv.c | 76 ++++++++++++++++++-------------------- 1 file changed, 36 insertions(+), 40 deletions(-) diff --git a/source/client/src/tscEnv.c b/source/client/src/tscEnv.c index 26ae9a8006..0d667b0d19 100644 --- a/source/client/src/tscEnv.c +++ b/source/client/src/tscEnv.c @@ -13,8 +13,6 @@ * along with this program. If not, see . */ -#include "clientInt.h" -#include "trpc.h" #include "os.h" #include "taosmsg.h" #include "tcache.h" @@ -25,23 +23,52 @@ #include "tscLog.h" #include "tsched.h" #include "ttime.h" +#include "trpc.h" #include "ttimezone.h" +#include "clientInt.h" #define TSC_VAR_NOT_RELEASE 1 #define TSC_VAR_RELEASED 0 SAppInfo appInfo; -int32_t tscReqRef = -1; -void *tscQhandle; +int32_t tscReqRef = -1; int32_t tscConnRef = -1; -void *tscRpcCache; // TODO removed from here. +void *tscQhandle = NULL; +void *tscRpcCache= NULL; // TODO removed from here. pthread_mutex_t rpcObjMutex; // mutex to protect open the rpc obj concurrently -static pthread_once_t tscinit = PTHREAD_ONCE_INIT; -static pthread_mutex_t setConfMutex = PTHREAD_MUTEX_INITIALIZER; +volatile int32_t tscInitRes = 0; -// pthread_once can not return result code, so result code is set to a global variable. -static volatile int tscInitRes = 0; +static void registerRequest(SRequestObj* pRequest) { + STscObj*pTscObj = (STscObj*) taosAcquireRef(tscConnRef, pRequest->pTscObj->id); + assert(pTscObj != NULL); + + // connection has been released already, abort creating request. + pRequest->self = taosAddRef(tscReqRef, pRequest); + + int32_t num = atomic_add_fetch_32(&pTscObj->numOfReqs, 1); + + SInstanceActivity* pActivity = &pTscObj->pAppInfo->summary; + int32_t total = atomic_add_fetch_32(&pActivity->totalRequests, 1); + int32_t currentInst = atomic_add_fetch_32(&pActivity->currentRequests, 1); + + tscDebug("0x%"PRIx64" new Request from 0x%"PRIx64", current:%d, app current:%d, total:%d", pRequest->self, pRequest->pTscObj->id, num, currentInst, total); +} + +static void deregisterRequest(SRequestObj* pRequest) { + assert(pRequest != NULL); + + STscObj* pTscObj = pRequest->pTscObj; + SInstanceActivity* pActivity = &pTscObj->pAppInfo->summary; + + taosReleaseRef(tscReqRef, pRequest->self); + + int32_t currentInst = atomic_sub_fetch_32(&pActivity->currentRequests, 1); + int32_t num = atomic_sub_fetch_32(&pTscObj->numOfReqs, 1); + + tscDebug("0x%"PRIx64" free Request from 0x%"PRIx64", current:%d, app current:%d", pRequest->self, pTscObj->id, num, currentInst); + taosReleaseRef(tscConnRef, pTscObj->id); +} void tscFreeRpcObj(void *param) { #if 0 @@ -139,22 +166,6 @@ void* createTscObj(const char* user, const char* auth, const char *ip, uint32_t pObj->id = taosAddRef(tscConnRef, pObj); } -static void registerRequest(SRequestObj* pRequest) { - STscObj*pTscObj = (STscObj*) taosAcquireRef(tscConnRef, pRequest->pTscObj->id); - assert(pTscObj != NULL); - - // connection has been released already, abort creating request. - pRequest->self = taosAddRef(tscReqRef, pRequest); - - int32_t num = atomic_add_fetch_32(&pTscObj->numOfReqs, 1); - - SInstanceActivity* pActivity = &pTscObj->pAppInfo->summary; - int32_t total = atomic_add_fetch_32(&pActivity->totalRequests, 1); - int32_t currentInst = atomic_add_fetch_32(&pActivity->currentRequests, 1); - - tscDebug("0x%"PRIx64" new Request from 0x%"PRIx64", current:%d, app current:%d, total:%d", pRequest->self, pRequest->pTscObj->id, num, currentInst, total); -} - void* createRequest(STscObj* pObj, __taos_async_fn_t fp, void* param, int32_t type) { assert(pObj != NULL); @@ -176,21 +187,6 @@ void* createRequest(STscObj* pObj, __taos_async_fn_t fp, void* param, int32_t ty registerRequest(pRequest); } -static void deregisterRequest(SRequestObj* pRequest) { - assert(pRequest != NULL); - - STscObj* pTscObj = pRequest->pTscObj; - SInstanceActivity* pActivity = &pTscObj->pAppInfo->summary; - - taosReleaseRef(tscReqRef, pRequest->self); - - int32_t currentInst = atomic_sub_fetch_32(&pActivity->currentRequests, 1); - int32_t num = atomic_sub_fetch_32(&pTscObj->numOfReqs, 1); - - tscDebug("0x%"PRIx64" free Request from 0x%"PRIx64", current:%d, app current:%d", pRequest->self, pTscObj->id, num, currentInst); - taosReleaseRef(tscConnRef, pTscObj->id); -} - void destroyRequest(void* p) { assert(p != NULL); SRequestObj* pRequest = *(SRequestObj**)p; From 8c416a5e4e3c990cb5c9c4c642a611eae1c21331 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 10 Dec 2021 17:28:09 +0800 Subject: [PATCH 05/43] [td-10564] refactor and add test cases. --- include/common/taosmsg.h | 8 +-- include/util/tdef.h | 2 +- include/util/tutil.h | 14 +++- source/client/inc/clientInt.h | 4 +- source/client/src/client.c | 4 +- source/client/src/clientImpl.c | 8 +-- source/client/src/clientmain.c | 4 ++ source/client/src/tscEnv.c | 76 +++++++++++----------- source/client/test/clientTests.cpp | 1 + source/dnode/mgmt/impl/src/dndTransport.c | 12 ++-- source/dnode/mgmt/impl/test/sut/deploy.cpp | 2 +- source/dnode/mnode/impl/inc/mndDef.h | 2 +- source/dnode/mnode/impl/src/mndUser.c | 6 +- source/libs/sync/src/sync.c | 2 +- source/libs/transport/src/rpcMain.c | 18 ++--- source/os/src/osString.c | 6 +- src/inc/tcq.h | 4 +- 17 files changed, 95 insertions(+), 78 deletions(-) diff --git a/include/common/taosmsg.h b/include/common/taosmsg.h index 2769f8bc7a..ca250ab169 100644 --- a/include/common/taosmsg.h +++ b/include/common/taosmsg.h @@ -379,7 +379,7 @@ typedef struct { typedef struct { char user[TSDB_USER_LEN]; - char pass[TSDB_KEY_LEN]; + char pass[TSDB_PASSWORD_LEN]; int32_t maxUsers; int32_t maxDbs; int32_t maxTimeSeries; @@ -394,7 +394,7 @@ typedef struct { typedef struct { char user[TSDB_USER_LEN]; - char pass[TSDB_KEY_LEN]; + char pass[TSDB_PASSWORD_LEN]; } SCreateUserMsg, SAlterUserMsg; typedef struct { @@ -912,8 +912,8 @@ typedef struct { char user[TSDB_USER_LEN]; char spi; char encrypt; - char secret[TSDB_KEY_LEN]; - char ckey[TSDB_KEY_LEN]; + char secret[TSDB_PASSWORD_LEN]; + char ckey[TSDB_PASSWORD_LEN]; } SAuthMsg, SAuthRsp; typedef struct { diff --git a/include/util/tdef.h b/include/util/tdef.h index d227888582..5be7df2c14 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -188,7 +188,7 @@ do { \ #define TSDB_MAX_TAG_CONDITIONS 1024 #define TSDB_AUTH_LEN 16 -#define TSDB_KEY_LEN 64 +#define TSDB_PASSWORD_LEN 64 #define TSDB_VERSION_LEN 12 #define TSDB_LABEL_LEN 8 diff --git a/include/util/tutil.h b/include/util/tutil.h index 573dee9339..189cc2b728 100644 --- a/include/util/tutil.h +++ b/include/util/tutil.h @@ -51,7 +51,19 @@ static FORCE_INLINE void taosEncryptPass(uint8_t *inBuf, size_t inLen, char *tar MD5Init(&context); MD5Update(&context, inBuf, (unsigned int)inLen); MD5Final(&context); - memcpy(target, context.digest, TSDB_KEY_LEN); + memcpy(target, context.digest, tListLen(context.digest)); +} + +static FORCE_INLINE void taosEncryptPass_c(uint8_t *inBuf, size_t len, char *target) { + MD5_CTX context; + MD5Init(&context); + MD5Update(&context, inBuf, (unsigned int)len); + + MD5Final(&context); + sprintf(target, "%x%x%x%x%x%x%x%x%x%x%x%x%x%x%x%x", context.digest[0], context.digest[1], context.digest[2], + context.digest[3], context.digest[4], context.digest[5], context.digest[6], context.digest[7], + context.digest[8], context.digest[9], context.digest[10], context.digest[11], context.digest[12], + context.digest[13], context.digest[14], context.digest[15]); } #ifdef __cplusplus diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index 4fed198ab6..fb405e50c2 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -71,7 +71,7 @@ typedef struct SAppInfo { typedef struct STscObj { char user[TSDB_USER_LEN]; - char pass[TSDB_KEY_LEN]; + char pass[TSDB_PASSWORD_LEN]; char acctId[TSDB_ACCT_ID_LEN]; char db[TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN]; uint32_t connId; @@ -117,7 +117,7 @@ void destroyRequest(void* p); TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass, const char *auth, const char *db, uint16_t port); void taos_init_imp(void); -int taos_options_imp(TSDB_OPTION option, const char *pStr); +int taos_options_imp(TSDB_OPTION option, const char *str); #ifdef __cplusplus } diff --git a/source/client/src/client.c b/source/client/src/client.c index 8aef77e9e7..a99c7c44bc 100644 --- a/source/client/src/client.c +++ b/source/client/src/client.c @@ -53,11 +53,11 @@ TAOS *taos_connect_l(const char *ip, int ipLen, const char *user, int userLen, c char ipStr[TSDB_EP_LEN] = {0}; char dbStr[TSDB_DB_NAME_LEN] = {0}; char userStr[TSDB_USER_LEN] = {0}; - char passStr[TSDB_KEY_LEN] = {0}; + char passStr[TSDB_PASSWORD_LEN] = {0}; strncpy(ipStr, ip, MIN(TSDB_EP_LEN - 1, ipLen)); strncpy(userStr, user, MIN(TSDB_USER_LEN - 1, userLen)); - strncpy(passStr, pass, MIN(TSDB_KEY_LEN - 1, passLen)); + strncpy(passStr, pass, MIN(TSDB_PASSWORD_LEN - 1, passLen)); strncpy(dbStr, db, MIN(TSDB_DB_NAME_LEN - 1, dbLen)); return taos_connect(ipStr, userStr, passStr, dbStr, port); } diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index e7940e27b0..89ca302099 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -26,7 +26,7 @@ static bool validateUserName(const char* user) { } static bool validatePassword(const char* passwd) { - return stringLengthCheck(passwd, TSDB_KEY_LEN - 1); + return stringLengthCheck(passwd, TSDB_PASSWORD_LEN - 1); } static bool validateDbName(const char* db) { @@ -52,14 +52,14 @@ TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass, strdequote(tmp); } - char secretEncrypt[32] = {0}; + char secretEncrypt[64] = {0}; if (auth == NULL) { if (!validatePassword(pass)) { terrno = TSDB_CODE_TSC_INVALID_PASS_LENGTH; return NULL; } - taosEncryptPass((uint8_t *)pass, strlen(pass), secretEncrypt); + taosEncryptPass_c((uint8_t *)pass, strlen(pass), secretEncrypt); } else { tstrncpy(secretEncrypt, auth, tListLen(secretEncrypt)); } @@ -79,7 +79,7 @@ TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass, } } - return taosConnectImpl(ip, user, auth, db, port, NULL, NULL); + return taosConnectImpl(ip, user, &secretEncrypt[0], db, port, NULL, NULL); } int initEpSetFromCfg(const char *firstEp, const char *secondEp, SRpcCorEpSet *pEpSet) { diff --git a/source/client/src/clientmain.c b/source/client/src/clientmain.c index 7dac304c58..ce3ef41f18 100644 --- a/source/client/src/clientmain.c +++ b/source/client/src/clientmain.c @@ -74,6 +74,10 @@ void taos_cleanup(void) { } void taos_close(TAOS* taos) { + if (taos == NULL) { + return; + } + } diff --git a/source/client/src/tscEnv.c b/source/client/src/tscEnv.c index 0d667b0d19..13ae76ea40 100644 --- a/source/client/src/tscEnv.c +++ b/source/client/src/tscEnv.c @@ -70,6 +70,22 @@ static void deregisterRequest(SRequestObj* pRequest) { taosReleaseRef(tscConnRef, pTscObj->id); } +static void tscInitLogFile() { + taosReadGlobalLogCfg(); + if (mkdir(tsLogDir, 0755) != 0 && errno != EEXIST) { + printf("failed to create log dir:%s\n", tsLogDir); + } + + const char *defaultLogFileNamePrefix = "taoslog"; + const int32_t maxLogFileNum = 10; + + char temp[128] = {0}; + sprintf(temp, "%s/%s", tsLogDir, defaultLogFileNamePrefix); + if (taosInitLog(temp, tsNumOfLogLines, maxLogFileNum) < 0) { + printf("failed to open log file in directory:%s\n", tsLogDir); + } +} + void tscFreeRpcObj(void *param) { #if 0 assert(param); @@ -200,22 +216,6 @@ void destroyRequest(void* p) { deregisterRequest(pRequest); } -static void tscInitLogFile() { - taosReadGlobalLogCfg(); - if (mkdir(tsLogDir, 0755) != 0 && errno != EEXIST) { - printf("failed to create log dir:%s\n", tsLogDir); - } - - const char *defaultLogFileNamePrefix = "taoslog"; - const int32_t maxLogFileNum = 10; - - char temp[128] = {0}; - sprintf(temp, "%s/%s", tsLogDir, defaultLogFileNamePrefix); - if (taosInitLog(temp, tsNumOfLogLines, maxLogFileNum) < 0) { - printf("failed to open log file in directory:%s\n", tsLogDir); - } -} - void taos_init_imp(void) { // In the APIs of other program language, taos_cleanup is not available yet. // So, to make sure taos_cleanup will be invoked to clean up the allocated resource to suppress the valgrind warning. @@ -268,7 +268,7 @@ void taos_init_imp(void) { tscDebug("client is initialized successfully"); } -int taos_options_imp(TSDB_OPTION option, const char *pStr) { +int taos_options_imp(TSDB_OPTION option, const char *str) { SGlobalCfg *cfg = NULL; switch (option) { @@ -277,11 +277,11 @@ int taos_options_imp(TSDB_OPTION option, const char *pStr) { assert(cfg != NULL); if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_OPTION) { - tstrncpy(configDir, pStr, TSDB_FILENAME_LEN); + tstrncpy(configDir, str, TSDB_FILENAME_LEN); cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION; - tscInfo("set config file directory:%s", pStr); + tscInfo("set config file directory:%s", str); } else { - tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, pStr, tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr); + tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, str, tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr); } break; @@ -290,13 +290,13 @@ int taos_options_imp(TSDB_OPTION option, const char *pStr) { assert(cfg != NULL); if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_OPTION) { - tsShellActivityTimer = atoi(pStr); + tsShellActivityTimer = atoi(str); if (tsShellActivityTimer < 1) tsShellActivityTimer = 1; if (tsShellActivityTimer > 3600) tsShellActivityTimer = 3600; cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION; tscInfo("set shellActivityTimer:%d", tsShellActivityTimer); } else { - tscWarn("config option:%s, input value:%s, is configured by %s, use %d", cfg->option, pStr, tsCfgStatusStr[cfg->cfgStatus], *(int32_t *)cfg->ptr); + tscWarn("config option:%s, input value:%s, is configured by %s, use %d", cfg->option, str, tsCfgStatusStr[cfg->cfgStatus], *(int32_t *)cfg->ptr); } break; @@ -304,9 +304,9 @@ int taos_options_imp(TSDB_OPTION option, const char *pStr) { cfg = taosGetConfigOption("locale"); assert(cfg != NULL); - size_t len = strlen(pStr); + size_t len = strlen(str); if (len == 0 || len > TSDB_LOCALE_LEN) { - tscInfo("Invalid locale:%s, use default", pStr); + tscInfo("Invalid locale:%s, use default", str); return -1; } @@ -327,14 +327,14 @@ int taos_options_imp(TSDB_OPTION option, const char *pStr) { } // set the user specified locale - char *locale = setlocale(LC_CTYPE, pStr); + char *locale = setlocale(LC_CTYPE, str); if (locale != NULL) { // failed to set the user specified locale tscInfo("locale set, prev locale:%s, new locale:%s", tsLocale, locale); cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION; } else { // set the user specified locale failed, use default LC_CTYPE as current locale locale = setlocale(LC_CTYPE, tsLocale); - tscInfo("failed to set locale:%s, current locale:%s", pStr, tsLocale); + tscInfo("failed to set locale:%s, current locale:%s", str, tsLocale); } tstrncpy(tsLocale, locale, TSDB_LOCALE_LEN); @@ -364,7 +364,7 @@ int taos_options_imp(TSDB_OPTION option, const char *pStr) { tscInfo("charset remains:%s", tsCharset); } } else { - tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, pStr, tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr); + tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, str, tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr); } break; } @@ -374,27 +374,27 @@ int taos_options_imp(TSDB_OPTION option, const char *pStr) { cfg = taosGetConfigOption("charset"); assert(cfg != NULL); - size_t len = strlen(pStr); + size_t len = strlen(str); if (len == 0 || len > TSDB_LOCALE_LEN) { - tscInfo("failed to set charset:%s", pStr); + tscInfo("failed to set charset:%s", str); return -1; } if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_OPTION) { - if (taosValidateEncodec(pStr)) { + if (taosValidateEncodec(str)) { if (strlen(tsCharset) == 0) { - tscInfo("charset is set:%s", pStr); + tscInfo("charset is set:%s", str); } else { - tscInfo("charset changed from %s to %s", tsCharset, pStr); + tscInfo("charset changed from %s to %s", tsCharset, str); } - tstrncpy(tsCharset, pStr, TSDB_LOCALE_LEN); + tstrncpy(tsCharset, str, TSDB_LOCALE_LEN); cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION; } else { - tscInfo("charset:%s not valid", pStr); + tscInfo("charset:%s not valid", str); } } else { - tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, pStr, tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr); + tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, str, tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr); } break; @@ -405,12 +405,12 @@ int taos_options_imp(TSDB_OPTION option, const char *pStr) { assert(cfg != NULL); if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_OPTION) { - tstrncpy(tsTimezone, pStr, TSDB_TIMEZONE_LEN); + tstrncpy(tsTimezone, str, TSDB_TIMEZONE_LEN); tsSetTimeZone(); cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION; - tscDebug("timezone set:%s, input:%s by taos_options", tsTimezone, pStr); + tscDebug("timezone set:%s, input:%s by taos_options", tsTimezone, str); } else { - tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, pStr, tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr); + tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, str, tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr); } break; diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index 4f848a0750..a828d2e079 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -34,4 +34,5 @@ int main(int argc, char** argv) { TEST(testCase, driverInit_Test) { taos_init(); + TAOS* pTaos = taos_connect("ubuntu", "root", "taosdata", NULL, 0); } \ No newline at end of file diff --git a/source/dnode/mgmt/impl/src/dndTransport.c b/source/dnode/mgmt/impl/src/dndTransport.c index 98a0b8e308..07bbe6c0f6 100644 --- a/source/dnode/mgmt/impl/src/dndTransport.c +++ b/source/dnode/mgmt/impl/src/dndTransport.c @@ -240,18 +240,18 @@ static void dndSendMsgToMnodeRecv(SDnode *pDnode, SRpcMsg *pRpcMsg, SRpcMsg *pRp static int32_t dndAuthInternalMsg(SDnode *pDnode, char *user, char *spi, char *encrypt, char *secret, char *ckey) { if (strcmp(user, INTERNAL_USER) == 0) { // A simple temporary implementation - char pass[32] = {0}; + char pass[TSDB_PASSWORD_LEN] = {0}; taosEncryptPass((uint8_t *)(INTERNAL_SECRET), strlen(INTERNAL_SECRET), pass); - memcpy(secret, pass, TSDB_KEY_LEN); + memcpy(secret, pass, TSDB_PASSWORD_LEN); *spi = 0; *encrypt = 0; *ckey = 0; return 0; } else if (strcmp(user, TSDB_NETTEST_USER) == 0) { // A simple temporary implementation - char pass[32] = {0}; + char pass[TSDB_PASSWORD_LEN] = {0}; taosEncryptPass((uint8_t *)(TSDB_NETTEST_USER), strlen(TSDB_NETTEST_USER), pass); - memcpy(secret, pass, TSDB_KEY_LEN); + memcpy(secret, pass, TSDB_PASSWORD_LEN); *spi = 0; *encrypt = 0; *ckey = 0; @@ -293,8 +293,8 @@ static int32_t dndRetrieveUserAuthInfo(void *parent, char *user, char *spi, char dError("user:%s, failed to get user auth from other mnodes since %s", user, terrstr()); } else { SAuthRsp *pRsp = rpcRsp.pCont; - memcpy(secret, pRsp->secret, TSDB_KEY_LEN); - memcpy(ckey, pRsp->ckey, TSDB_KEY_LEN); + memcpy(secret, pRsp->secret, TSDB_PASSWORD_LEN); + memcpy(ckey, pRsp->ckey, TSDB_PASSWORD_LEN); *spi = pRsp->spi; *encrypt = pRsp->encrypt; dDebug("user:%s, success to get user auth from other mnodes", user); diff --git a/source/dnode/mgmt/impl/test/sut/deploy.cpp b/source/dnode/mgmt/impl/test/sut/deploy.cpp index c484ea122d..8a84733fa3 100644 --- a/source/dnode/mgmt/impl/test/sut/deploy.cpp +++ b/source/dnode/mgmt/impl/test/sut/deploy.cpp @@ -107,7 +107,7 @@ SClient* createClient(const char* user, const char* pass, const char* fqdn, uint SClient* pClient = (SClient*)calloc(1, sizeof(SClient)); ASSERT(pClient); - char secretEncrypt[32] = {0}; + char secretEncrypt[TSDB_PASSWORD_LEN] = {0}; taosEncryptPass((uint8_t*)pass, strlen(pass), secretEncrypt); SRpcInit rpcInit; diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 9e4f922540..7da92c84bb 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -181,7 +181,7 @@ typedef struct SAcctObj { typedef struct SUserObj { char user[TSDB_USER_LEN]; - char pass[TSDB_KEY_LEN]; + char pass[TSDB_PASSWORD_LEN]; char acct[TSDB_USER_LEN]; int64_t createdTime; int64_t updateTime; diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index a3dd255577..c5162d8595 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -92,7 +92,7 @@ static SSdbRaw *mndUserActionEncode(SUserObj *pUser) { int32_t dataPos = 0; SDB_SET_BINARY(pRaw, dataPos, pUser->user, TSDB_USER_LEN) - SDB_SET_BINARY(pRaw, dataPos, pUser->pass, TSDB_KEY_LEN) + SDB_SET_BINARY(pRaw, dataPos, pUser->pass, TSDB_PASSWORD_LEN) SDB_SET_BINARY(pRaw, dataPos, pUser->acct, TSDB_USER_LEN) SDB_SET_INT64(pRaw, dataPos, pUser->createdTime) SDB_SET_INT64(pRaw, dataPos, pUser->updateTime) @@ -120,7 +120,7 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) { int32_t dataPos = 0; SDB_GET_BINARY(pRaw, pRow, dataPos, pUser->user, TSDB_USER_LEN) - SDB_GET_BINARY(pRaw, pRow, dataPos, pUser->pass, TSDB_KEY_LEN) + SDB_GET_BINARY(pRaw, pRow, dataPos, pUser->pass, TSDB_PASSWORD_LEN) SDB_GET_BINARY(pRaw, pRow, dataPos, pUser->acct, TSDB_USER_LEN) SDB_GET_INT64(pRaw, pRow, dataPos, &pUser->createdTime) SDB_GET_INT64(pRaw, pRow, dataPos, &pUser->updateTime) @@ -165,7 +165,7 @@ static int32_t mndUserActionDelete(SSdb *pSdb, SUserObj *pUser) { static int32_t mndUserActionUpdate(SSdb *pSdb, SUserObj *pSrcUser, SUserObj *pDstUser) { mTrace("user:%s, perform update action", pSrcUser->user); memcpy(pSrcUser->user, pDstUser->user, TSDB_USER_LEN); - memcpy(pSrcUser->pass, pDstUser->pass, TSDB_KEY_LEN); + memcpy(pSrcUser->pass, pDstUser->pass, TSDB_PASSWORD_LEN); memcpy(pSrcUser->acct, pDstUser->acct, TSDB_USER_LEN); pSrcUser->createdTime = pDstUser->createdTime; pSrcUser->updateTime = pDstUser->updateTime; diff --git a/source/libs/sync/src/sync.c b/source/libs/sync/src/sync.c index 06af8ff6c2..e49b1d7983 100644 --- a/source/libs/sync/src/sync.c +++ b/source/libs/sync/src/sync.c @@ -228,7 +228,7 @@ static int syncInitRpcServer(SSyncManager* syncManager, const SSyncCluster* pSyn } static int syncInitRpcClient(SSyncManager* syncManager) { - char secret[TSDB_KEY_LEN] = "secret"; + char secret[TSDB_PASSWORD_LEN] = "secret"; SRpcInit rpcInit; memset(&rpcInit, 0, sizeof(rpcInit)); rpcInit.label = "sync-client"; diff --git a/source/libs/transport/src/rpcMain.c b/source/libs/transport/src/rpcMain.c index e392351366..56d5228c2d 100644 --- a/source/libs/transport/src/rpcMain.c +++ b/source/libs/transport/src/rpcMain.c @@ -51,8 +51,8 @@ typedef struct { char user[TSDB_UNI_LEN]; // meter ID char spi; // security parameter index char encrypt; // encrypt algorithm - char secret[TSDB_KEY_LEN]; // secret for the link - char ckey[TSDB_KEY_LEN]; // ciphering key + char secret[TSDB_PASSWORD_LEN]; // secret for the link + char ckey[TSDB_PASSWORD_LEN]; // ciphering key void (*cfp)(void *parent, SRpcMsg *, SEpSet *); int (*afp)(void *parent, char *user, char *spi, char *encrypt, char *secret, char *ckey); @@ -97,8 +97,8 @@ typedef struct SRpcConn { char user[TSDB_UNI_LEN]; // user ID for the link char spi; // security parameter index char encrypt; // encryption, 0:1 - char secret[TSDB_KEY_LEN]; // secret for the link - char ckey[TSDB_KEY_LEN]; // ciphering key + char secret[TSDB_PASSWORD_LEN]; // secret for the link + char ckey[TSDB_PASSWORD_LEN]; // ciphering key char secured; // if set to 1, no authentication uint16_t localPort; // for UDP only uint32_t linkUid; // connection unique ID assigned by client @@ -698,7 +698,7 @@ static SRpcConn *rpcAllocateClientConn(SRpcInfo *pRpc) { pConn->linkUid = (uint32_t)((int64_t)pConn + taosGetPid() + (int64_t)pConn->tranId); pConn->spi = pRpc->spi; pConn->encrypt = pRpc->encrypt; - if (pConn->spi) memcpy(pConn->secret, pRpc->secret, TSDB_KEY_LEN); + if (pConn->spi) memcpy(pConn->secret, pRpc->secret, TSDB_PASSWORD_LEN); tDebug("%s %p client connection is allocated, uid:0x%x", pRpc->label, pConn, pConn->linkUid); } @@ -1527,9 +1527,9 @@ static int rpcAuthenticateMsg(void *pMsg, int msgLen, void *pAuth, void *pKey) { int ret = -1; MD5Init(&context); - MD5Update(&context, (uint8_t *)pKey, TSDB_KEY_LEN); + MD5Update(&context, (uint8_t *)pKey, TSDB_PASSWORD_LEN); MD5Update(&context, (uint8_t *)pMsg, msgLen); - MD5Update(&context, (uint8_t *)pKey, TSDB_KEY_LEN); + MD5Update(&context, (uint8_t *)pKey, TSDB_PASSWORD_LEN); MD5Final(&context); if (memcmp(context.digest, pAuth, sizeof(context.digest)) == 0) ret = 0; @@ -1541,9 +1541,9 @@ static void rpcBuildAuthHead(void *pMsg, int msgLen, void *pAuth, void *pKey) { MD5_CTX context; MD5Init(&context); - MD5Update(&context, (uint8_t *)pKey, TSDB_KEY_LEN); + MD5Update(&context, (uint8_t *)pKey, TSDB_PASSWORD_LEN); MD5Update(&context, (uint8_t *)pMsg, msgLen); - MD5Update(&context, (uint8_t *)pKey, TSDB_KEY_LEN); + MD5Update(&context, (uint8_t *)pKey, TSDB_PASSWORD_LEN); MD5Final(&context); memcpy(pAuth, context.digest, sizeof(context.digest)); diff --git a/source/os/src/osString.c b/source/os/src/osString.c index 8054dc42be..4d8f0d134f 100644 --- a/source/os/src/osString.c +++ b/source/os/src/osString.c @@ -275,13 +275,13 @@ char *strsep(char **stringp, const char *delim) { } char *getpass(const char *prefix) { - static char passwd[TSDB_KEY_LEN] = {0}; - memset(passwd, 0, TSDB_KEY_LEN); + static char passwd[TSDB_PASSWORD_LEN] = {0}; + memset(passwd, 0, TSDB_PASSWORD_LEN); //printf("%s", prefix); int32_t index = 0; char ch; - while (index < TSDB_KEY_LEN) { + while (index < TSDB_PASSWORD_LEN) { ch = getch(); if (ch == '\n' || ch == '\r') { break; diff --git a/src/inc/tcq.h b/src/inc/tcq.h index 7338cccfee..71efe33011 100644 --- a/src/inc/tcq.h +++ b/src/inc/tcq.h @@ -26,7 +26,7 @@ typedef int32_t (*FCqWrite)(int32_t vgId, void *pHead, int32_t qtype, void *pMsg typedef struct { int32_t vgId; char user[TSDB_USER_LEN]; - char pass[TSDB_KEY_LEN]; + char pass[TSDB_PASSWORD_LEN]; char db[TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN]; // size must same with SVnodeObj.db[TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN] FCqWrite cqWrite; } SCqCfg; @@ -37,7 +37,7 @@ typedef struct { int32_t master; int32_t num; // number of continuous streams char user[TSDB_USER_LEN]; - char pass[TSDB_KEY_LEN]; + char pass[TSDB_PASSWORD_LEN]; char db[TSDB_DB_NAME_LEN]; FCqWrite cqWrite; struct SCqObj *pHead; From d962a2715f70cc3e1022586d31b14ef936a3c192 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Sat, 11 Dec 2021 11:18:15 +0800 Subject: [PATCH 06/43] catalog init version --- include/libs/catalog/catalog.h | 33 ++++++++++++++++------- source/libs/catalog/inc/catalogInt.h | 8 ++++-- source/libs/catalog/src/catalog.c | 12 +++++++-- source/libs/parser/inc/parserInt.h | 4 +-- source/libs/parser/src/astValidate.c | 6 ++--- source/libs/parser/src/parser.c | 6 ++--- source/libs/parser/test/parserTests.cpp | 24 ++++++++--------- source/libs/parser/test/plannerTest.cpp | 6 ++--- source/libs/parser/test/tokenizerTest.cpp | 2 +- 9 files changed, 63 insertions(+), 38 deletions(-) diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index 050b9c904f..b04b4f5c8d 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -30,19 +30,19 @@ extern "C" { struct SCatalog; -typedef struct SMetaReq { +typedef struct SCatalogReq { char clusterId[TSDB_CLUSTER_ID_LEN]; SArray *pTableName; // table full name SArray *pUdf; // udf name bool qNodeEpset; // valid qnode -} SMetaReq; +} SCatalogReq; -typedef struct SMetaData { +typedef struct SCatalogRsp { SArray *pTableMeta; // tableMeta SArray *pVgroupInfo; // vgroupInfo list SArray *pUdfList; // udf info list SEpSet *pEpSet; // qnode epset list -} SMetaData; +} SCatalogRsp; typedef struct STableComInfo { uint8_t numOfTags; // the number of tags in schema @@ -78,32 +78,45 @@ typedef struct STableMeta { SSchema schema[]; } STableMeta; +typedef struct SCatalogCfg { + +} SCatalogCfg; + + +int32_t catalogInit(SCatalog *cfg); + /** * Catalog service object, which is utilized to hold tableMeta (meta/vgroupInfo/udfInfo) at the client-side. * There is ONLY one SCatalog object for one process space, and this function returns a singleton. - * @param pMgmtEps + * @param clusterId * @return */ -struct SCatalog* getCatalogHandle(const SEpSet* pMgmtEps); +struct SCatalog* catalogGetHandle(const char *clusterId); /** * Get the required meta data from mnode. * Note that this is a synchronized API and is also thread-safety. * @param pCatalog + * @param pMgmtEps * @param pMetaReq * @param pMetaData * @return */ -int32_t catalogGetMetaData(struct SCatalog* pCatalog, const SMetaReq* pMetaReq, SMetaData* pMetaData); +int32_t catalogGetAllMeta(struct SCatalog* pCatalog, const SEpSet* pMgmtEps, const SCatalogReq* pCatalogReq, SCatalogRsp* pCatalogData); + +int32_t catalogRenewTableMeta(struct SCatalog* pCatalog, const SEpSet* pMgmtEps, const STableMeta* pTableMeta); + +int32_t catalogRenewAndGetTableMeta(struct SCatalog* pCatalog, const SEpSet* pMgmtEps, const STableMeta* pTableMeta, SCatalogRsp* pCatalogData); + /** - * Destroy catalog service handle + * Destroy catalog and relase all resources * @param pCatalog */ -void destroyCatalog(struct SCatalog* pCatalog); +void catalogDestroy(void); #ifdef __cplusplus } #endif -#endif /*_TD_CATALOG_H_*/ \ No newline at end of file +#endif /*_TD_CATALOG_H_*/ diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 5b50bbff4c..60cc1771e2 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -23,10 +23,14 @@ extern "C" { #include "catalog.h" typedef struct SCatalog { - void *pMsgSender; // used to send messsage to mnode to fetch necessary metadata - SHashObj *pData; // items cached for each cluster, the hash key is the cluster-id, returned by mgmt node + } SCatalog; +typedef struct SCatalogMgmt { + void *pMsgSender; // used to send messsage to mnode to fetch necessary metadata + SHashObj *pMeta; // items cached for each cluster, the hash key is the cluster-id, returned by mgmt node +} SCatalogMgmt; + #ifdef __cplusplus } #endif diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 08e2172adb..cd6e357f43 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -15,10 +15,18 @@ #include "catalogInt.h" -struct SCatalog* getCatalogHandle(const SEpSet* pMgmtEps) { +SCatalogMgmt ctgMgmt = {0}; + + +int32_t catalogInit(SCatalog *cfg) { + ctgMgmt = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); +} + + +struct SCatalog* catalogGetHandle(const char *clusterId) { return (struct SCatalog*) 0x1; } -int32_t catalogGetMetaData(struct SCatalog* pCatalog, const SMetaReq* pMetaReq, SMetaData* pMetaData) { +int32_t catalogGetAllMeta(struct SCatalog* pCatalog, const SEpSet* pMgmtEps, const SCatalogReq* pMetaReq, SCatalogRsp* pMetaData) { return 0; } diff --git a/source/libs/parser/inc/parserInt.h b/source/libs/parser/inc/parserInt.h index ca02165382..e040c21879 100644 --- a/source/libs/parser/inc/parserInt.h +++ b/source/libs/parser/inc/parserInt.h @@ -87,13 +87,13 @@ int32_t checkForInvalidExpr(SQueryStmtInfo* pQueryInfo, SMsgBuf* pMsgBuf); * @param msgBufLen * @return */ -int32_t qParserExtractRequestedMetaInfo(const SSqlInfo* pSqlInfo, SMetaReq* pMetaInfo, char* msg, int32_t msgBufLen); +int32_t qParserExtractRequestedMetaInfo(const SSqlInfo* pSqlInfo, SCatalogReq* pMetaInfo, char* msg, int32_t msgBufLen); /** * Destroy the meta data request structure. * @param pMetaInfo */ -void qParserClearupMetaRequestInfo(SMetaReq* pMetaInfo); +void qParserClearupMetaRequestInfo(SCatalogReq* pMetaInfo); #ifdef __cplusplus } diff --git a/source/libs/parser/src/astValidate.c b/source/libs/parser/src/astValidate.c index 7b6c423fb6..758b83d820 100644 --- a/source/libs/parser/src/astValidate.c +++ b/source/libs/parser/src/astValidate.c @@ -4077,8 +4077,8 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer } #endif - SMetaReq req = {0}; - SMetaData data = {0}; + SCatalogReq req = {0}; + SCatalogRsp data = {0}; // TODO: check if the qnode info has been cached already req.qNodeEpset = true; @@ -4088,7 +4088,7 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer } // load the meta data from catalog - code = catalogGetMetaData(pCatalog, &req, &data); + code = catalogGetAllMeta(pCatalog, NULL, &req, &data); if (code != TSDB_CODE_SUCCESS) { return code; } diff --git a/source/libs/parser/src/parser.c b/source/libs/parser/src/parser.c index 29561f7f54..3faa06720b 100644 --- a/source/libs/parser/src/parser.c +++ b/source/libs/parser/src/parser.c @@ -42,7 +42,7 @@ int32_t qParseQuerySql(const char* pStr, size_t length, struct SQueryStmtInfo** return TSDB_CODE_TSC_SQL_SYNTAX_ERROR; } - struct SCatalog* pCatalog = getCatalogHandle(NULL); + struct SCatalog* pCatalog = catalogGetHandle(NULL); return qParserValidateSqlNode(pCatalog, &info, *pQueryInfo, id, msg, msgLen); } @@ -131,7 +131,7 @@ static void freePtrElem(void* p) { tfree(*(char**)p); } -int32_t qParserExtractRequestedMetaInfo(const SSqlInfo* pSqlInfo, SMetaReq* pMetaInfo, char* msg, int32_t msgBufLen) { +int32_t qParserExtractRequestedMetaInfo(const SSqlInfo* pSqlInfo, SCatalogReq* pMetaInfo, char* msg, int32_t msgBufLen) { int32_t code = TSDB_CODE_SUCCESS; SMsgBuf msgBuf = {.buf = msg, .len = msgBufLen}; @@ -188,7 +188,7 @@ int32_t qParserExtractRequestedMetaInfo(const SSqlInfo* pSqlInfo, SMetaReq* pMet return code; } -void qParserClearupMetaRequestInfo(SMetaReq* pMetaReq) { +void qParserClearupMetaRequestInfo(SCatalogReq* pMetaReq) { if (pMetaReq == NULL) { return; } diff --git a/source/libs/parser/test/parserTests.cpp b/source/libs/parser/test/parserTests.cpp index 2193a44604..6a402cd620 100644 --- a/source/libs/parser/test/parserTests.cpp +++ b/source/libs/parser/test/parserTests.cpp @@ -38,7 +38,7 @@ void setSchema(SSchema* p, int32_t type, int32_t bytes, const char* name, int32_ strcpy(p->name, name); } -void setTableMetaInfo(SQueryStmtInfo* pQueryInfo, SMetaReq* req) { +void setTableMetaInfo(SQueryStmtInfo* pQueryInfo, SCatalogReq* req) { pQueryInfo->numOfTables = 1; pQueryInfo->pTableMetaInfo = (STableMetaInfo**)calloc(1, POINTER_BYTES); @@ -80,7 +80,7 @@ void sqlCheck(const char* sql, bool valid) { int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); ASSERT_EQ(code, 0); - SMetaReq req = {0}; + SCatalogReq req = {0}; int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, msg, 128); ASSERT_EQ(ret, 0); ASSERT_EQ(taosArrayGetSize(req.pTableName), 1); @@ -117,7 +117,7 @@ TEST(testCase, validateAST_test) { int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); ASSERT_EQ(code, 0); - SMetaReq req = {0}; + SCatalogReq req = {0}; int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, msg, 128); ASSERT_EQ(ret, 0); ASSERT_EQ(taosArrayGetSize(req.pTableName), 1); @@ -175,7 +175,7 @@ TEST(testCase, function_Test) { int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); ASSERT_EQ(code, 0); - SMetaReq req = {0}; + SCatalogReq req = {0}; int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, msg, 128); ASSERT_EQ(ret, 0); ASSERT_EQ(taosArrayGetSize(req.pTableName), 1); @@ -221,7 +221,7 @@ TEST(testCase, function_Test2) { int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); ASSERT_EQ(code, 0); - SMetaReq req = {0}; + SCatalogReq req = {0}; int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, msg, 128); ASSERT_EQ(ret, 0); ASSERT_EQ(taosArrayGetSize(req.pTableName), 1); @@ -267,7 +267,7 @@ TEST(testCase, function_Test3) { int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); ASSERT_EQ(code, 0); - SMetaReq req = {0}; + SCatalogReq req = {0}; int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, msg, 128); ASSERT_EQ(ret, 0); ASSERT_EQ(taosArrayGetSize(req.pTableName), 1); @@ -312,7 +312,7 @@ TEST(testCase, function_Test4) { int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); ASSERT_EQ(code, 0); - SMetaReq req = {0}; + SCatalogReq req = {0}; int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, msg, 128); ASSERT_EQ(ret, 0); ASSERT_EQ(taosArrayGetSize(req.pTableName), 1); @@ -360,7 +360,7 @@ TEST(testCase, function_Test5) { int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); ASSERT_EQ(code, 0); - SMetaReq req = {0}; + SCatalogReq req = {0}; int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, msg, 128); ASSERT_EQ(ret, 0); ASSERT_EQ(taosArrayGetSize(req.pTableName), 1); @@ -445,7 +445,7 @@ TEST(testCase, function_Test6) { int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); ASSERT_EQ(code, 0); - SMetaReq req = {0}; + SCatalogReq req = {0}; int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, msg, 128); ASSERT_EQ(ret, 0); ASSERT_EQ(taosArrayGetSize(req.pTableName), 1); @@ -523,7 +523,7 @@ TEST(testCase, function_Test6) { int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); ASSERT_EQ(code, 0); - SMetaReq req = {0}; + SCatalogReq req = {0}; int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, msg, 128); ASSERT_EQ(ret, 0); ASSERT_EQ(taosArrayGetSize(req.pTableName), 1); @@ -585,7 +585,7 @@ TEST(testCase, function_Test6) { int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); ASSERT_EQ(code, 0); - SMetaReq req = {0}; + SCatalogReq req = {0}; int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, msg, 128); ASSERT_EQ(ret, 0); ASSERT_EQ(taosArrayGetSize(req.pTableName), 1); @@ -664,7 +664,7 @@ TEST(testCase, function_Test6) { int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); ASSERT_EQ(code, 0); - SMetaReq req = {0}; + SCatalogReq req = {0}; int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, msg, 128); ASSERT_EQ(ret, 0); ASSERT_EQ(taosArrayGetSize(req.pTableName), 1); diff --git a/source/libs/parser/test/plannerTest.cpp b/source/libs/parser/test/plannerTest.cpp index c86e687664..bb9271a3c8 100644 --- a/source/libs/parser/test/plannerTest.cpp +++ b/source/libs/parser/test/plannerTest.cpp @@ -39,7 +39,7 @@ void setSchema(SSchema* p, int32_t type, int32_t bytes, const char* name, int32_ strcpy(p->name, name); } -void setTableMetaInfo(SQueryStmtInfo* pQueryInfo, SMetaReq *req) { +void setTableMetaInfo(SQueryStmtInfo* pQueryInfo, SCatalogReq *req) { pQueryInfo->numOfTables = 1; pQueryInfo->pTableMetaInfo = (STableMetaInfo**)calloc(1, POINTER_BYTES); @@ -79,7 +79,7 @@ void generateLogicplan(const char* sql) { int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); ASSERT_EQ(code, 0); - SMetaReq req = {0}; + SCatalogReq req = {0}; int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, msg, 128); ASSERT_EQ(ret, 0); ASSERT_EQ(taosArrayGetSize(req.pTableName), 1); @@ -119,7 +119,7 @@ TEST(testCase, planner_test) { int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); ASSERT_EQ(code, 0); - SMetaReq req = {0}; + SCatalogReq req = {0}; int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, msg, 128); ASSERT_EQ(ret, 0); ASSERT_EQ(taosArrayGetSize(req.pTableName), 1); diff --git a/source/libs/parser/test/tokenizerTest.cpp b/source/libs/parser/test/tokenizerTest.cpp index 07ba46427f..54372ae133 100644 --- a/source/libs/parser/test/tokenizerTest.cpp +++ b/source/libs/parser/test/tokenizerTest.cpp @@ -714,7 +714,7 @@ TEST(testCase, extractMeta_test) { ASSERT_EQ(info1.valid, true); char msg[128] = {0}; - SMetaReq req = {0}; + SCatalogReq req = {0}; int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, msg, 128); ASSERT_EQ(ret, 0); ASSERT_EQ(taosArrayGetSize(req.pTableName), 1); From 3d5f9a244dafd4e1eca20fa1da5a63b14196c8bb Mon Sep 17 00:00:00 2001 From: dapan Date: Mon, 13 Dec 2021 08:10:06 +0800 Subject: [PATCH 07/43] catalog init --- include/libs/catalog/catalog.h | 14 +++-- include/util/taoserror.h | 7 +++ source/common/src/tmessage.c | 89 ++++++++++++++++++++++++++++ source/libs/catalog/inc/catalogInt.h | 20 ++++++- source/libs/catalog/src/catalog.c | 63 +++++++++++++++++++- source/util/src/terror.c | 11 +++- 6 files changed, 194 insertions(+), 10 deletions(-) diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index b04b4f5c8d..8aacede5fe 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -31,10 +31,10 @@ extern "C" { struct SCatalog; typedef struct SCatalogReq { - char clusterId[TSDB_CLUSTER_ID_LEN]; + char clusterId[TSDB_CLUSTER_ID_LEN]; //???? SArray *pTableName; // table full name SArray *pUdf; // udf name - bool qNodeEpset; // valid qnode + bool qNodeRequired; // valid qnode } SCatalogReq; typedef struct SCatalogRsp { @@ -93,6 +93,9 @@ int32_t catalogInit(SCatalog *cfg); */ struct SCatalog* catalogGetHandle(const char *clusterId); +int32_t catalogGetTableMeta(struct SCatalog* pCatalog, const SEpSet* pMgmtEps, const char* pTableName, const STagData* tagData, STableMeta* pTableMeta); + + /** * Get the required meta data from mnode. * Note that this is a synchronized API and is also thread-safety. @@ -102,11 +105,14 @@ struct SCatalog* catalogGetHandle(const char *clusterId); * @param pMetaData * @return */ -int32_t catalogGetAllMeta(struct SCatalog* pCatalog, const SEpSet* pMgmtEps, const SCatalogReq* pCatalogReq, SCatalogRsp* pCatalogData); +int32_t catalogGetAllMeta(struct SCatalog* pCatalog, const SEpSet* pMgmtEps, const SCatalogReq* pReq, SCatalogRsp* pRsp); int32_t catalogRenewTableMeta(struct SCatalog* pCatalog, const SEpSet* pMgmtEps, const STableMeta* pTableMeta); -int32_t catalogRenewAndGetTableMeta(struct SCatalog* pCatalog, const SEpSet* pMgmtEps, const STableMeta* pTableMeta, SCatalogRsp* pCatalogData); +int32_t catalogRenewAndGetTableMeta(struct SCatalog* pCatalog, const SEpSet* pMgmtEps, const STableMeta* pTableMeta, STableMeta* pNewTableMeta); + +int32_t catalogGetQnodeList(struct SCatalog* pCatalog, const SEpSet* pMgmtEps, SEpSet* pQnodeEpSet); + /** diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 36301466f8..065f1ee0ab 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -490,6 +490,13 @@ int32_t* taosGetErrno(); // monitor #define TSDB_CODE_MON_CONNECTION_INVALID TAOS_DEF_ERROR_CODE(0, 0x2300) //"monitor invalid monitor db connection") +// catalog +#define TSDB_CODE_CTG_INTERNAL_EROR TAOS_DEF_ERROR_CODE(0, 0x2400) //catalog interval error +#define TSDB_CODE_CTG_INVALID_INPUT TAOS_DEF_ERROR_CODE(0, 0x2401) //invalid catalog input parameters +#define TSDB_CODE_CTG_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x2402) //catalog is not ready +#define TSDB_CODE_CTG_MEM_ERROR TAOS_DEF_ERROR_CODE(0, 0x2403) //catalog memory error +#define TSDB_CODE_CTG_SYS_ERROR TAOS_DEF_ERROR_CODE(0, 0x2404) //catalog system error + #ifdef __cplusplus } #endif diff --git a/source/common/src/tmessage.c b/source/common/src/tmessage.c index 0b6dbfdb51..8609e8f09a 100644 --- a/source/common/src/tmessage.c +++ b/source/common/src/tmessage.c @@ -16,3 +16,92 @@ #define TAOS_MESSAGE_C #include "taosmsg.h" + +int32_t (*tscBuildMsg[TSDB_MSG_TYPE_MAX])(void* input, char **msg, int32_t msgSize) = {0}; + +int32_t (*tscProcessMsgRsp[TSDB_MSG_TYPE_MAX])(void* output, char *msg, int32_t msgSize) = {0}; + + +void msgInit() { + tscBuildMsg[TSDB_MSG_TYPE_TABLE_META] = buildTableMetaReqMsg; + + tscProcessMsgRsp[TSDB_MSG_TYPE_TABLE_META] = ; + +/* + tscBuildMsg[TSDB_SQL_SELECT] = tscBuildQueryMsg; + tscBuildMsg[TSDB_SQL_INSERT] = tscBuildSubmitMsg; + tscBuildMsg[TSDB_SQL_FETCH] = tscBuildFetchMsg; + + tscBuildMsg[TSDB_SQL_CREATE_DB] = tscBuildCreateDbMsg; + tscBuildMsg[TSDB_SQL_CREATE_USER] = tscBuildUserMsg; + tscBuildMsg[TSDB_SQL_CREATE_FUNCTION] = tscBuildCreateFuncMsg; + + tscBuildMsg[TSDB_SQL_CREATE_ACCT] = tscBuildAcctMsg; + tscBuildMsg[TSDB_SQL_ALTER_ACCT] = tscBuildAcctMsg; + + tscBuildMsg[TSDB_SQL_CREATE_TABLE] = tscBuildCreateTableMsg; + tscBuildMsg[TSDB_SQL_DROP_USER] = tscBuildDropUserAcctMsg; + tscBuildMsg[TSDB_SQL_DROP_ACCT] = tscBuildDropUserAcctMsg; + tscBuildMsg[TSDB_SQL_DROP_DB] = tscBuildDropDbMsg; + tscBuildMsg[TSDB_SQL_DROP_FUNCTION] = tscBuildDropFuncMsg; + tscBuildMsg[TSDB_SQL_SYNC_DB_REPLICA] = tscBuildSyncDbReplicaMsg; + tscBuildMsg[TSDB_SQL_DROP_TABLE] = tscBuildDropTableMsg; + tscBuildMsg[TSDB_SQL_ALTER_USER] = tscBuildUserMsg; + tscBuildMsg[TSDB_SQL_CREATE_DNODE] = tscBuildCreateDnodeMsg; + tscBuildMsg[TSDB_SQL_DROP_DNODE] = tscBuildDropDnodeMsg; + tscBuildMsg[TSDB_SQL_CFG_DNODE] = tscBuildCfgDnodeMsg; + tscBuildMsg[TSDB_SQL_ALTER_TABLE] = tscBuildAlterTableMsg; + tscBuildMsg[TSDB_SQL_UPDATE_TAGS_VAL] = tscBuildUpdateTagMsg; + tscBuildMsg[TSDB_SQL_ALTER_DB] = tscAlterDbMsg; + tscBuildMsg[TSDB_SQL_COMPACT_VNODE] = tscBuildCompactMsg; + + tscBuildMsg[TSDB_SQL_CONNECT] = tscBuildConnectMsg; + tscBuildMsg[TSDB_SQL_USE_DB] = tscBuildUseDbMsg; + tscBuildMsg[TSDB_SQL_STABLEVGROUP] = tscBuildSTableVgroupMsg; + tscBuildMsg[TSDB_SQL_RETRIEVE_FUNC] = tscBuildRetrieveFuncMsg; + + tscBuildMsg[TSDB_SQL_HB] = tscBuildHeartBeatMsg; + tscBuildMsg[TSDB_SQL_SHOW] = tscBuildShowMsg; + tscBuildMsg[TSDB_SQL_RETRIEVE] = tscBuildRetrieveFromMgmtMsg; + tscBuildMsg[TSDB_SQL_KILL_QUERY] = tscBuildKillMsg; + tscBuildMsg[TSDB_SQL_KILL_STREAM] = tscBuildKillMsg; + tscBuildMsg[TSDB_SQL_KILL_CONNECTION] = tscBuildKillMsg; + + tscProcessMsgRsp[TSDB_SQL_SELECT] = tscProcessQueryRsp; + tscProcessMsgRsp[TSDB_SQL_FETCH] = tscProcessRetrieveRspFromNode; + + tscProcessMsgRsp[TSDB_SQL_DROP_DB] = tscProcessDropDbRsp; + tscProcessMsgRsp[TSDB_SQL_DROP_TABLE] = tscProcessDropTableRsp; + tscProcessMsgRsp[TSDB_SQL_CONNECT] = tscProcessConnectRsp; + tscProcessMsgRsp[TSDB_SQL_USE_DB] = tscProcessUseDbRsp; + tscProcessMsgRsp[TSDB_SQL_META] = tscProcessTableMetaRsp; + tscProcessMsgRsp[TSDB_SQL_STABLEVGROUP] = tscProcessSTableVgroupRsp; + tscProcessMsgRsp[TSDB_SQL_MULTI_META] = tscProcessMultiTableMetaRsp; + tscProcessMsgRsp[TSDB_SQL_RETRIEVE_FUNC] = tscProcessRetrieveFuncRsp; + + tscProcessMsgRsp[TSDB_SQL_SHOW] = tscProcessShowRsp; + tscProcessMsgRsp[TSDB_SQL_RETRIEVE] = tscProcessRetrieveRspFromNode; // rsp handled by same function. + tscProcessMsgRsp[TSDB_SQL_DESCRIBE_TABLE] = tscProcessDescribeTableRsp; + + tscProcessMsgRsp[TSDB_SQL_CURRENT_DB] = tscProcessLocalRetrieveRsp; + tscProcessMsgRsp[TSDB_SQL_CURRENT_USER] = tscProcessLocalRetrieveRsp; + tscProcessMsgRsp[TSDB_SQL_SERV_VERSION] = tscProcessLocalRetrieveRsp; + tscProcessMsgRsp[TSDB_SQL_CLI_VERSION] = tscProcessLocalRetrieveRsp; + tscProcessMsgRsp[TSDB_SQL_SERV_STATUS] = tscProcessLocalRetrieveRsp; + + tscProcessMsgRsp[TSDB_SQL_RETRIEVE_EMPTY_RESULT] = tscProcessEmptyResultRsp; + + tscProcessMsgRsp[TSDB_SQL_RETRIEVE_GLOBALMERGE] = tscProcessRetrieveGlobalMergeRsp; + + tscProcessMsgRsp[TSDB_SQL_ALTER_TABLE] = tscProcessAlterTableMsgRsp; + tscProcessMsgRsp[TSDB_SQL_ALTER_DB] = tscProcessAlterDbMsgRsp; + tscProcessMsgRsp[TSDB_SQL_COMPACT_VNODE] = tscProcessCompactRsp; + + tscProcessMsgRsp[TSDB_SQL_SHOW_CREATE_TABLE] = tscProcessShowCreateRsp; + tscProcessMsgRsp[TSDB_SQL_SHOW_CREATE_STABLE] = tscProcessShowCreateRsp; + tscProcessMsgRsp[TSDB_SQL_SHOW_CREATE_DATABASE] = tscProcessShowCreateRsp; +*/ +} + + + diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 60cc1771e2..8703f1f0ce 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -22,17 +22,33 @@ extern "C" { #include "catalog.h" +#define CTG_DEFAULT_CLUSTER_NUMBER 3 + typedef struct SCatalog { } SCatalog; typedef struct SCatalogMgmt { void *pMsgSender; // used to send messsage to mnode to fetch necessary metadata - SHashObj *pMeta; // items cached for each cluster, the hash key is the cluster-id, returned by mgmt node + SHashObj *pCluster; // items cached for each cluster, the hash key is the cluster-id got from mgmt node } SCatalogMgmt; + + +#define ctgFatal(...) tscFatal(__VA_ARGS__) +#define ctgError(...) tscError(__VA_ARGS__) +#define ctgWarn(...) tscWarn(__VA_ARGS__) +#define ctgInfo(...) tscInfo(__VA_ARGS__) +#define ctgDebug(...) tscDebug(__VA_ARGS__) +#define ctgTrace(...) tscTrace(__VA_ARGS__) + +#define CTG_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { return _code; } } while (0) +#define CTG_ERR_LRET(c,...) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { ctgError(__VA_ARGS__); return _code; } } while (0) +#define CTG_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { goto _return; } } while (0) + + #ifdef __cplusplus } #endif -#endif /*_TD_CATALOG_INT_H_*/ \ No newline at end of file +#endif /*_TD_CATALOG_INT_H_*/ diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index cd6e357f43..a305df05d0 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -19,14 +19,71 @@ SCatalogMgmt ctgMgmt = {0}; int32_t catalogInit(SCatalog *cfg) { - ctgMgmt = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + ctgMgmt.pCluster = taosHashInit(CTG_DEFAULT_CLUSTER_NUMBER, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + if (NULL == ctgMgmt.pCluster) { + CTG_ERR_LRET(TSDB_CODE_CTG_INTERNAL_EROR, "init %d cluster cache failed", CTG_DEFAULT_CLUSTER_NUMBER); + } + + ctgGetVnodeInfo(); + + return TSDB_CODE_SUCCESS; } struct SCatalog* catalogGetHandle(const char *clusterId) { - return (struct SCatalog*) 0x1; + if (NULL == clusterId) { + return NULL; + } + + if (NULL == ctgMgmt.pCluster) { + ctgError("cluster cache are not ready"); + return NULL; + } + + size_t clen = strlen(clusterId); + SCatalog *clusterCtg = (SCatalog *)taosHashGet(ctgMgmt.pCluster, clusterId, clen); + + if (clusterCtg) { + return clusterCtg; + } + + clusterCtg = calloc(1, sizeof(*clusterCtg)); + if (NULL == clusterCtg) { + ctgError("calloc %d failed", sizeof(*clusterCtg)); + return NULL; + } + + if (taosHashPut(ctgMgmt.pCluster, clusterId, clen, &clusterCtg, POINTER_BYTES)) { + ctgError("put cluster %s cache to hash failed", clusterId); + tfree(clusterCtg); + return NULL; + } + + return clusterCtg; } -int32_t catalogGetAllMeta(struct SCatalog* pCatalog, const SEpSet* pMgmtEps, const SCatalogReq* pMetaReq, SCatalogRsp* pMetaData) { +int32_t catalogGetTableMeta(struct SCatalog* pCatalog, const SEpSet* pMgmtEps, const char* pTableName, STableMeta* pTableMeta) { + if (NULL == pCatalog || NULL == pMgmtEps || NULL == pTableName || NULL == pTableMeta) { + return TSDB_CODE_CTG_INVALID_INPUT; + } + +} + + +int32_t catalogGetAllMeta(struct SCatalog* pCatalog, const SEpSet* pMgmtEps, const SCatalogReq* pReq, SCatalogRsp* pRsp) { + if (NULL == pCatalog || NULL == pMgmtEps || NULL == pReq || NULL == pRsp) { + return TSDB_CODE_CTG_INVALID_INPUT; + } + return 0; } + +void catalogDestroy(void) { + if (ctgMgmt.pCluster) { + taosHashCleanup(ctgMgmt.pCluster); //TBD + ctgMgmt.pCluster = NULL; + } +} + + + diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 5110c8ba22..c4ca44f1d2 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -496,6 +496,15 @@ TAOS_DEFINE_ERROR(TSDB_CODE_FS_FILE_ALREADY_EXISTS, "tfs file already exis TAOS_DEFINE_ERROR(TSDB_CODE_FS_INVLD_LEVEL, "tfs invalid level") TAOS_DEFINE_ERROR(TSDB_CODE_FS_NO_VALID_DISK, "tfs no valid disk") +// catalog +TAOS_DEFINE_ERROR(TSDB_CODE_CTG_INTERNAL_EROR, "catalog interval error") +TAOS_DEFINE_ERROR(TSDB_CODE_CTG_INVALID_INPUT, "invalid catalog input parameters") +TAOS_DEFINE_ERROR(TSDB_CODE_CTG_NOT_READY, "catalog is not ready") +TAOS_DEFINE_ERROR(TSDB_CODE_CTG_MEM_ERROR, "catalog memory error") +TAOS_DEFINE_ERROR(TSDB_CODE_CTG_SYS_ERROR, "catalog system error") + + + #ifdef TAOS_ERROR_C }; #endif @@ -544,4 +553,4 @@ const char* tstrerror(int32_t err) { return ""; } -const char* terrstr() { return tstrerror(terrno); } \ No newline at end of file +const char* terrstr() { return tstrerror(terrno); } From 5c7332c2fcfd2989415a3674f1b046856d620423 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 13 Dec 2021 17:10:31 +0800 Subject: [PATCH 08/43] catalog update --- include/common/taosmsg.h | 7 +++ include/libs/catalog/catalog.h | 36 +++++++++++---- include/util/taoserror.h | 1 + source/common/src/tmessage.c | 67 ++++++++++++++++++++++++++-- source/libs/catalog/inc/catalogInt.h | 18 +++++++- source/libs/catalog/src/catalog.c | 24 ++++++++-- source/libs/parser/src/parserUtil.c | 17 ------- source/util/src/terror.c | 1 + 8 files changed, 138 insertions(+), 33 deletions(-) diff --git a/include/common/taosmsg.h b/include/common/taosmsg.h index 2ce6da9806..fd55a11b52 100644 --- a/include/common/taosmsg.h +++ b/include/common/taosmsg.h @@ -214,6 +214,12 @@ typedef enum _mgmt_table { extern char *taosMsg[]; +typedef struct SBuildTableMetaInput { + int32_t vgId; + STagData *tagData; + char *tableFullName; +} SBuildTableMetaInput; + #pragma pack(push, 1) // null-terminated string instead of char array to avoid too many memory consumption in case of more than 1M tableMeta @@ -768,6 +774,7 @@ typedef struct { } SStableInfoMsg; typedef struct { + SMsgHead msgHead; char tableFname[TSDB_TABLE_FNAME_LEN]; int8_t createFlag; char tags[]; diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index 8aacede5fe..2092f53ba1 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -30,6 +30,19 @@ extern "C" { struct SCatalog; +typedef struct SVgroupInfo { + int32_t vgId; + int8_t numOfEps; + SEpAddrMsg epAddr[TSDB_MAX_REPLICA]; +} SVgroupInfo; + +typedef struct SDBVgroupInfo { + int32_t vgroupVersion; + SArray *vgId; + int32_t hashRange; + int32_t hashNum; +} SDBVgroupInfo; + typedef struct SCatalogReq { char clusterId[TSDB_CLUSTER_ID_LEN]; //???? SArray *pTableName; // table full name @@ -38,8 +51,8 @@ typedef struct SCatalogReq { } SCatalogReq; typedef struct SCatalogRsp { - SArray *pTableMeta; // tableMeta - SArray *pVgroupInfo; // vgroupInfo list + SArray *pTableMeta; // STableMeta array + SArray *pVgroupInfo; // SVgroupInfo list SArray *pUdfList; // udf info list SEpSet *pEpSet; // qnode epset list } SCatalogRsp; @@ -78,11 +91,6 @@ typedef struct STableMeta { SSchema schema[]; } STableMeta; -typedef struct SCatalogCfg { - -} SCatalogCfg; - - int32_t catalogInit(SCatalog *cfg); /** @@ -91,9 +99,19 @@ int32_t catalogInit(SCatalog *cfg); * @param clusterId * @return */ -struct SCatalog* catalogGetHandle(const char *clusterId); +int32_t catalogGetHandle(const char *clusterId, struct SCatalog** catalogHandle); -int32_t catalogGetTableMeta(struct SCatalog* pCatalog, const SEpSet* pMgmtEps, const char* pTableName, const STagData* tagData, STableMeta* pTableMeta); +int32_t catalogGetVgroupVersion(struct SCatalog* pCatalog, int32_t* version); + +int32_t catalogUpdateVgroupList(struct SCatalog* pCatalog, int32_t version, SArray* vgroupList); + +int32_t catalogGetDBVgroupVersion(struct SCatalog* pCatalog, const char* dbName, int32_t* version); + +int32_t catalogGetDBVgroupInfo(struct SCatalog* pCatalog, const char* dbName, SDBVgroupInfo* dbInfo); + +int32_t catalogUpdateDBVgroupInfo(struct SCatalog* pCatalog, const char* dbName, SDBVgroupInfo* dbInfo); + +int32_t catalogGetTableMeta(struct SCatalog* pCatalog, SRpcObj *pRpcObj, const SEpSet* pMgmtEps, const char* pTableName, const STagData* tagData, STableMeta* pTableMeta); /** diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 065f1ee0ab..a3a4297115 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -117,6 +117,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_TSC_INVALID_JSON TAOS_DEF_ERROR_CODE(0, 0x0221) //"Invalid JSON format") #define TSDB_CODE_TSC_INVALID_JSON_TYPE TAOS_DEF_ERROR_CODE(0, 0x0222) //"Invalid JSON data type") #define TSDB_CODE_TSC_VALUE_OUT_OF_RANGE TAOS_DEF_ERROR_CODE(0, 0x0223) //"Value out of range") +#define TSDB_CODE_TSC_INVALID_INPUT TAOS_DEF_ERROR_CODE(0, 0X0224) //"Invalid tsc input") // mnode #define TSDB_CODE_MND_MSG_NOT_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0300) diff --git a/source/common/src/tmessage.c b/source/common/src/tmessage.c index 8609e8f09a..0e732caa26 100644 --- a/source/common/src/tmessage.c +++ b/source/common/src/tmessage.c @@ -17,14 +17,16 @@ #include "taosmsg.h" -int32_t (*tscBuildMsg[TSDB_MSG_TYPE_MAX])(void* input, char **msg, int32_t msgSize) = {0}; +int32_t (*tscBuildMsg[TSDB_MSG_TYPE_MAX])(void* input, char **msg, int32_t msgSize, int32_t *msgLen) = {0}; int32_t (*tscProcessMsgRsp[TSDB_MSG_TYPE_MAX])(void* output, char *msg, int32_t msgSize) = {0}; void msgInit() { - tscBuildMsg[TSDB_MSG_TYPE_TABLE_META] = buildTableMetaReqMsg; - + tscBuildMsg[TSDB_MSG_TYPE_TABLE_META] = tscBuildTableMetaReqMsg; + + + tscProcessMsgRsp[TSDB_MSG_TYPE_TABLE_META] = ; /* @@ -104,4 +106,63 @@ void msgInit() { } +char* msgSerializeTagData(STagData* pTagData, char* pMsg) { + int32_t n = (int32_t) strlen(pTagData->name); + *(int32_t*) pMsg = htonl(n); + pMsg += sizeof(n); + + memcpy(pMsg, pTagData->name, n); + pMsg += n; + + *(int32_t*)pMsg = htonl(pTagData->dataLen); + pMsg += sizeof(int32_t); + + memcpy(pMsg, pTagData->data, pTagData->dataLen); + pMsg += pTagData->dataLen; + + return pMsg; +} + + +int32_t tscBuildTableMetaReqMsg(void* input, char **msg, int32_t msgSize, int32_t *msgLen) { + if (NULL == input || NULL == msg || NULL == msgLen) { + return TSDB_CODE_TSC_INVALID_INPUT; + } + + SBuildTableMetaInput* bInput = (SBuildTableMetaInput *)input; + + int32_t estimateSize = sizeof(STableInfoMsg) + (bInput->tagData ? (sizeof(*bInput->tagData) + bInput->tagData->dataLen) : 0); + if (NULL == *msg || msgSize < estimateSize) { + tfree(*msg); + *msg = calloc(1, estimateSize); + if (NULL == *msg) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + } + + STableInfoMsg *bMsg = (STableInfoMsg *)*msg; + + bMsg->msgHead.vgId = bInput->vgId; + + strncpy(bMsg->tableFname, bInput->tableFullName, sizeof(bMsg->tableFname)); + bMsg->tableFname[sizeof(bMsg->tableFname) - 1] = 0; + + int32_t autoCreate = (bInput->tagData && bInput->tagData->dataLen > 0); + + bMsg->createFlag = htons(autoCreate ? 1 : 0); + + char *pMsg = NULL; + + // tag data exists + if (autoCreate) { + pMsg = msgSerializeTagData(bInput->tagData, (char *)bMsg->tags); + } + + *msgLen = (int32_t)(pMsg - (char*)bMsg); + + return TSDB_CODE_SUCCESS; +} + + + diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 8703f1f0ce..82d7d9c571 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -24,8 +24,24 @@ extern "C" { #define CTG_DEFAULT_CLUSTER_NUMBER 3 -typedef struct SCatalog { +typedef struct SVgroupListCache { + int32_t vgroupNum; + int32_t vgroupVersion; + SHashObj *cache; //key:vgId, value:SVgroupInfo +} SVgroupListCache; +typedef struct SDBVgroupCache { + SHashObj *cache; //key:dbname, value:SDBVgroupInfo +} SDBVgroupCache; + +typedef struct STableMetaCache { + SHashObj *cache; //key:fulltablename, value:STableMeta +} STableMetaCache; + +typedef struct SCatalog { + SVgroupListCache vgroupCache; + SDBVgroupCache dbCache; + STableMetaCache tableCache; } SCatalog; typedef struct SCatalogMgmt { diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index a305df05d0..e8b79bae4b 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -24,8 +24,6 @@ int32_t catalogInit(SCatalog *cfg) { CTG_ERR_LRET(TSDB_CODE_CTG_INTERNAL_EROR, "init %d cluster cache failed", CTG_DEFAULT_CLUSTER_NUMBER); } - ctgGetVnodeInfo(); - return TSDB_CODE_SUCCESS; } @@ -62,11 +60,31 @@ struct SCatalog* catalogGetHandle(const char *clusterId) { return clusterCtg; } -int32_t catalogGetTableMeta(struct SCatalog* pCatalog, const SEpSet* pMgmtEps, const char* pTableName, STableMeta* pTableMeta) { +int32_t catalogGetTableMeta(struct SCatalog* pCatalog, SRpcObj *pRpcObj, const SEpSet* pMgmtEps, const char* pTableName, const STagData* tagData, STableMeta* pTableMeta) { if (NULL == pCatalog || NULL == pMgmtEps || NULL == pTableName || NULL == pTableMeta) { return TSDB_CODE_CTG_INVALID_INPUT; } + SBuildTableMetaInput bInput = {0}; + char *msg = NULL; + SEpSet *pVnodeEpSet = NULL; + int32_t msgLen = 0; + + int32_t code = tscBuildMsg[TSDB_MSG_TYPE_TABLE_META](&bInput, &msg, 0, &msgLen); + if (code) { + return code; + } + + SRpcMsg rpcMsg = { + .msgType = TSDB_MSG_TYPE_TABLE_META, + .pCont = msg, + .contLen = msgLen, + .ahandle = (void*)pSql->self, + .handle = NULL, + .code = 0 + }; + + rpcSendRequest(pRpcObj->pDnodeConn, pVnodeEpSet, &rpcMsg, &pSql->rpcRid); } diff --git a/source/libs/parser/src/parserUtil.c b/source/libs/parser/src/parserUtil.c index 3e83381a76..f154599aae 100644 --- a/source/libs/parser/src/parserUtil.c +++ b/source/libs/parser/src/parserUtil.c @@ -1433,23 +1433,6 @@ void* vgroupInfoClear(SVgroupsInfo *vgroupList) { return NULL; } -char* serializeTagData(STagData* pTagData, char* pMsg) { - int32_t n = (int32_t) strlen(pTagData->name); - *(int32_t*) pMsg = htonl(n); - pMsg += sizeof(n); - - memcpy(pMsg, pTagData->name, n); - pMsg += n; - - *(int32_t*)pMsg = htonl(pTagData->dataLen); - pMsg += sizeof(int32_t); - - memcpy(pMsg, pTagData->data, pTagData->dataLen); - pMsg += pTagData->dataLen; - - return pMsg; -} - int32_t copyTagData(STagData* dst, const STagData* src) { dst->dataLen = src->dataLen; tstrncpy(dst->name, src->name, tListLen(dst->name)); diff --git a/source/util/src/terror.c b/source/util/src/terror.c index c4ca44f1d2..a75ce747b2 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -127,6 +127,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TSC_DUP_TAG_NAMES, "duplicated tag names" TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_JSON, "Invalid JSON format") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_JSON_TYPE, "Invalid JSON data type") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_VALUE_OUT_OF_RANGE, "Value out of range") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_INPUT, "Invalid tsc input") // mnode TAOS_DEFINE_ERROR(TSDB_CODE_MND_MSG_NOT_PROCESSED, "Message not processed") From 79006358d7e2860b084134f9a313d8ba6c47e8f5 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 14 Dec 2021 09:56:24 +0800 Subject: [PATCH 09/43] more --- include/dnode/vnode/vnode.h | 4 +++- source/dnode/vnode/impl/inc/vnodeCommit.h | 3 +++ source/dnode/vnode/impl/src/vnodeCommit.c | 9 +++++++++ source/dnode/vnode/impl/src/vnodeMain.c | 8 +++++++- source/dnode/vnode/impl/test/vnodeApiTests.cpp | 2 +- 5 files changed, 23 insertions(+), 3 deletions(-) diff --git a/include/dnode/vnode/vnode.h b/include/dnode/vnode/vnode.h index 007ce83812..8458ad9da3 100644 --- a/include/dnode/vnode/vnode.h +++ b/include/dnode/vnode/vnode.h @@ -68,9 +68,11 @@ typedef struct SVnodeCfg { /** * @brief Initialize the vnode module * + * @param nthreads number of commit threads. 0 for no threads and + * a schedule queue should be given (TODO) * @return int 0 for success and -1 for failure */ -int vnodeInit(); +int vnodeInit(uint16_t nthreads); /** * @brief clear a vnode diff --git a/source/dnode/vnode/impl/inc/vnodeCommit.h b/source/dnode/vnode/impl/inc/vnodeCommit.h index a60e8feac2..8f0af27513 100644 --- a/source/dnode/vnode/impl/inc/vnodeCommit.h +++ b/source/dnode/vnode/impl/inc/vnodeCommit.h @@ -22,6 +22,9 @@ extern "C" { #endif +int vnodeInitCommit(uint16_t nthreads); +void vnodeClearCommit(); + #define vnodeShouldCommit vnodeBufPoolIsFull int vnodeAsyncCommit(SVnode *pVnode); diff --git a/source/dnode/vnode/impl/src/vnodeCommit.c b/source/dnode/vnode/impl/src/vnodeCommit.c index cac7999f59..944fe80b31 100644 --- a/source/dnode/vnode/impl/src/vnodeCommit.c +++ b/source/dnode/vnode/impl/src/vnodeCommit.c @@ -18,6 +18,15 @@ static int vnodeStartCommit(SVnode *pVnode); static int vnodeEndCommit(SVnode *pVnode); +int vnodeInitCommit(uint16_t nthreads) { + // TODO + return 0; +} + +void vnodeClearCommit() { + // TODO +} + int vnodeAsyncCommit(SVnode *pVnode) { #if 0 if (vnodeStartCommit(pVnode) < 0) { diff --git a/source/dnode/vnode/impl/src/vnodeMain.c b/source/dnode/vnode/impl/src/vnodeMain.c index 9b94b4a361..63fc0d52f0 100644 --- a/source/dnode/vnode/impl/src/vnodeMain.c +++ b/source/dnode/vnode/impl/src/vnodeMain.c @@ -24,7 +24,7 @@ static void vnodeCloseImpl(SVnode *pVnode); TD_DEF_MOD_INIT_FLAG(vnode); TD_DEF_MOD_CLEAR_FLAG(vnode); -int vnodeInit() { +int vnodeInit(uint16_t nthreads) { if (TD_CHECK_AND_SET_MODE_INIT(vnode) == TD_MOD_INITIALIZED) { return 0; } @@ -33,6 +33,10 @@ int vnodeInit() { return -1; } + if (vnodeInitCommit(nthreads) < 0) { + return -1; + } + return 0; } @@ -42,6 +46,8 @@ void vnodeClear() { } walCleanUp(); + + vnodeClearCommit(); } SVnode *vnodeOpen(const char *path, const SVnodeCfg *pVnodeCfg) { diff --git a/source/dnode/vnode/impl/test/vnodeApiTests.cpp b/source/dnode/vnode/impl/test/vnodeApiTests.cpp index ac2ccbc132..a25b04e161 100644 --- a/source/dnode/vnode/impl/test/vnodeApiTests.cpp +++ b/source/dnode/vnode/impl/test/vnodeApiTests.cpp @@ -92,7 +92,7 @@ TEST(vnodeApiTest, test_create_table_encode_and_decode_function) { #endif TEST(vnodeApiTest, vnodeOpen_vnodeClose_test) { - GTEST_ASSERT_GE(vnodeInit(), 0); + GTEST_ASSERT_GE(vnodeInit(2), 0); // Create and open a vnode SVnode *pVnode = vnodeOpen("vnode1", NULL); From 8ff4a07d7bb8c5cdec5582a10f6b566c9bd94e39 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 14 Dec 2021 10:32:39 +0800 Subject: [PATCH 10/43] [td-10564] remove unused attributes in user structure, add new log for performance metric. --- include/common/taosmsg.h | 15 +++++----- source/client/inc/clientInt.h | 4 +-- source/client/src/clientImpl.c | 30 ++++---------------- source/client/src/clientmain.c | 8 ------ source/client/src/tscEnv.c | 35 ++++++++++++------------ source/dnode/mgmt/impl/src/dndDnode.c | 1 + source/dnode/mnode/impl/inc/mndDef.h | 4 +-- source/dnode/mnode/impl/src/mndProfile.c | 30 ++++++++++---------- source/dnode/mnode/impl/src/mndUser.c | 34 ++++++++--------------- source/libs/transport/src/rpcMain.c | 2 -- 10 files changed, 62 insertions(+), 101 deletions(-) diff --git a/include/common/taosmsg.h b/include/common/taosmsg.h index b4bbab376b..45f9514cf0 100644 --- a/include/common/taosmsg.h +++ b/include/common/taosmsg.h @@ -358,6 +358,7 @@ typedef struct { int32_t pid; char app[TSDB_APP_NAME_LEN]; char db[TSDB_DB_NAME_LEN]; + int64_t startTime; } SConnectMsg; typedef struct SEpSet { @@ -368,14 +369,12 @@ typedef struct SEpSet { } SEpSet; typedef struct { - int32_t acctId; - int32_t clusterId; - int32_t connId; - int8_t superAuth; - int8_t readAuth; - int8_t writeAuth; - int8_t reserved[5]; - SEpSet epSet; + int32_t acctId; + uint32_t clusterId; + int32_t connId; + int8_t superUser; + int8_t reserved[5]; + SEpSet epSet; } SConnectRsp; typedef struct { diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index 46620abcf2..749894444e 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -117,8 +117,6 @@ extern SAppInfo appInfo; extern int32_t tscReqRef; extern void *tscQhandle; extern int32_t tscConnRef; -extern void *tscRpcCache; -extern pthread_mutex_t rpcObjMutex; extern int (*tscBuildMsg[TSDB_SQL_MAX])(SRequestObj *pRequest, SRequestMsgBody *pMsg); extern int (*handleRequestRspFp[TSDB_SQL_MAX])(SRequestObj *pRequest, const char* pMsg, int32_t msgLen); @@ -126,7 +124,7 @@ extern int (*handleRequestRspFp[TSDB_SQL_MAX])(SRequestObj *pRequest, const char int taos_init(); void* createTscObj(const char* user, const char* auth, const char *ip, uint32_t port, SAppInstInfo* pAppInfo); -void destroyTscObj(void* pTscObj); +void destroyTscObj(void*pObj); void* createRequest(STscObj* pObj, __taos_async_fn_t fp, void* param, int32_t type); void destroyRequest(void* p); diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index e81494ae6f..0f9b40e262 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -201,11 +201,10 @@ static int32_t buildConnectMsg(SRequestObj *pRequest, SRequestMsgBody* pMsgBody) tstrncpy(pConnect->db, db, sizeof(pConnect->db)); pthread_mutex_unlock(&pObj->mutex); -// tstrncpy(pConnect->clientVersion, version, sizeof(pConnect->clientVersion)); -// tstrncpy(pConnect->msgVersion, "", sizeof(pConnect->msgVersion)); + pConnect->pid = htonl(appInfo.pid); + pConnect->startTime = htobe64(appInfo.startTime); + tstrncpy(pConnect->app, appInfo.appName, tListLen(pConnect->app)); -// pConnect->pid = htonl(taosGetPId()); -// taosGetCurrentAPPName(pConnect->appName, NULL); pMsgBody->pData = pConnect; return 0; } @@ -232,23 +231,6 @@ int32_t sendMsgToServer(void *pTransporter, SEpSet* epSet, const SRequestMsgBody return TSDB_CODE_SUCCESS; } -// -//int tscBuildAndSendRequest(SRequestObj *pRequest) { -// assert(pRequest != NULL); -// char name[TSDB_TABLE_FNAME_LEN] = {0}; -// -// uint32_t type = 0; -// tscDebug("0x%"PRIx64" SQL cmd:%s will be processed, name:%s, type:%d", pRequest->requestId, taosMsg[pRequest->type], name, type); -// if (pRequest->type < TSDB_SQL_MGMT) { // the pTableMetaInfo cannot be NULL -// -// } else if (pCmd->command >= TSDB_SQL_LOCAL) { -// return (*tscProcessMsgRsp[pCmd->command])(pSql); -// } -// -// return buildConnectMsg(pRequest); -//} - - void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) { int64_t requestRefId = (int64_t)pMsg->ahandle; @@ -275,13 +257,13 @@ void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) { * The actual inserted number of points is the first number. */ if (pMsg->code == TSDB_CODE_SUCCESS) { - tscDebug("0x%" PRIx64 " SQL cmd:%s, code:%s rspLen:%d", pRequest->requestId, taosMsg[pRequest->type], - tstrerror(pMsg->code), pMsg->contLen); + tscDebug("0x%" PRIx64 " message:%s, code:%s rspLen:%d, elapsed:%"PRId64 " ms", pRequest->requestId, taosMsg[pMsg->msgType], + tstrerror(pMsg->code), pMsg->contLen, pRequest->metric.rsp - pRequest->metric.start); if (handleRequestRspFp[pRequest->type]) { pMsg->code = (*handleRequestRspFp[pRequest->type])(pRequest, pMsg->pCont, pMsg->contLen); } } else { - tscError("0x%" PRIx64 " SQL cmd:%s, code:%s rspLen:%d", pRequest->requestId, taosMsg[pRequest->type], + tscError("0x%" PRIx64 " SQL cmd:%s, code:%s rspLen:%d", pRequest->requestId, taosMsg[pMsg->msgType], tstrerror(pMsg->code), pMsg->contLen); } diff --git a/source/client/src/clientmain.c b/source/client/src/clientmain.c index ce3ef41f18..bda2311f97 100644 --- a/source/client/src/clientmain.c +++ b/source/client/src/clientmain.c @@ -61,14 +61,6 @@ void taos_cleanup(void) { tscConnRef = -1; taosCloseRef(id); - p = tscRpcCache; - tscRpcCache = NULL; - - if (p != NULL) { - taosCacheCleanup(p); - pthread_mutex_destroy(&rpcObjMutex); - } - rpcCleanup(); taosCloseLog(); } diff --git a/source/client/src/tscEnv.c b/source/client/src/tscEnv.c index c3d52cfe9e..7565dd0c53 100644 --- a/source/client/src/tscEnv.c +++ b/source/client/src/tscEnv.c @@ -34,7 +34,6 @@ SAppInfo appInfo; int32_t tscReqRef = -1; int32_t tscConnRef = -1; void *tscQhandle = NULL; -void *tscRpcCache= NULL; // TODO removed from here. int32_t tsNumOfThreads = 1; @@ -55,7 +54,7 @@ static void registerRequest(SRequestObj* pRequest) { int32_t total = atomic_add_fetch_32(&pActivity->totalRequests, 1); int32_t currentInst = atomic_add_fetch_32(&pActivity->currentRequests, 1); - tscDebug("0x%" PRIx64 " new Request from 0x%" PRIx64 ", current:%d, app current:%d, total:%d", pRequest->self, + tscDebug("0x%" PRIx64 " new Request from connObj:0x%" PRIx64 ", current:%d, app current:%d, total:%d", pRequest->self, pRequest->pTscObj->id, num, currentInst, total); } } @@ -100,12 +99,14 @@ void tscFreeRpcObj(void *param) { #endif } -void tscReleaseRpc(void *param) { - if (param == NULL) { +void closeTransporter(STscObj* pTscObj) { + if (pTscObj == NULL || pTscObj->pTransporter == NULL) { return; } - taosCacheRelease(tscRpcCache, (void *)¶m, false); + tscDebug("free transporter:%p in connObj: 0x%"PRIx64, pTscObj->pTransporter, pTscObj->id); + rpcClose(pTscObj->pTransporter); + pTscObj->pTransporter = NULL; } // TODO refactor @@ -133,11 +134,13 @@ void* openTransporter(const char *user, const char *auth) { return pDnodeConn; } -void destroyTscObj(void *pTscObj) { - STscObj *pObj = pTscObj; -// tscReleaseRpc(pObj->pRpcObj); - pthread_mutex_destroy(&pObj->mutex); - tfree(pObj); +void destroyTscObj(void *pObj) { + STscObj *pTscObj = pObj; + tscDebug("connect obj destroyed, 0x%"PRIx64, pTscObj->id); + + closeTransporter(pTscObj); + pthread_mutex_destroy(&pTscObj->mutex); + tfree(pTscObj); } void* createTscObj(const char* user, const char* auth, const char *ip, uint32_t port, SAppInstInfo* pAppInfo) { @@ -157,6 +160,9 @@ void* createTscObj(const char* user, const char* auth, const char *ip, uint32_t pthread_mutex_init(&pObj->mutex, NULL); pObj->id = taosAddRef(tscConnRef, pObj); + + tscDebug("connect obj created, 0x%"PRIx64, pObj->id); + return pObj; } void* createRequest(STscObj* pObj, __taos_async_fn_t fp, void* param, int32_t type) { @@ -219,7 +225,7 @@ void taos_init_imp(void) { rpcInit(); - tscDebug("starting to initialize TAOS client ...\nLocal End Point is:%s", tsLocalEp); + tscDebug("starting to initialize TAOS driver, local ep: %s", tsLocalEp); taosSetCoreDump(true); @@ -234,12 +240,7 @@ void taos_init_imp(void) { return; } - tscDebug("client task queue is initialized, numOfWorkers: %d", numOfThreads); - - int refreshTime = 5; - tscRpcCache = taosCacheInit(TSDB_DATA_TYPE_BINARY, refreshTime, true, tscFreeRpcObj, "rpcObj"); - pthread_mutex_init(&rpcObjMutex, NULL); - + tscDebug("client task queue is initialized, numOfThreads: %d", numOfThreads); tscConnRef = taosOpenRef(200, destroyTscObj); tscReqRef = taosOpenRef(40960, destroyRequest); diff --git a/source/dnode/mgmt/impl/src/dndDnode.c b/source/dnode/mgmt/impl/src/dndDnode.c index 130cbbef07..9e2d4d4c9b 100644 --- a/source/dnode/mgmt/impl/src/dndDnode.c +++ b/source/dnode/mgmt/impl/src/dndDnode.c @@ -17,6 +17,7 @@ #include "dndDnode.h" #include "dndTransport.h" #include "dndVnodes.h" +#include "tep.h" int32_t dndGetDnodeId(SDnode *pDnode) { SDnodeMgmt *pMgmt = &pDnode->dmgmt; diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 426daf5298..5ddde3181e 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -184,9 +184,7 @@ typedef struct SUserObj { char acct[TSDB_USER_LEN]; int64_t createdTime; int64_t updateTime; - int8_t superAuth; - int8_t readAuth; - int8_t writeAuth; + int8_t superUser; int32_t acctId; SHashObj *prohibitDbHash; } SUserObj; diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index b429879789..59d84e5760 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -29,6 +29,7 @@ typedef struct { char user[TSDB_USER_LEN]; char app[TSDB_APP_NAME_LEN]; // app name that invokes taosc int32_t pid; // pid of app that invokes taosc + int64_t appStartTime; // app start time int32_t id; int8_t killed; int8_t align; @@ -44,7 +45,7 @@ typedef struct { SQueryDesc *pQueries; } SConnObj; -static SConnObj *mndCreateConn(SMnode *pMnode, char *user, uint32_t ip, uint16_t port, int32_t pid, const char *app); +static SConnObj *mndCreateConn(SMnode *pMnode, char *user, uint32_t ip, uint16_t port, int32_t pid, const char *app, int64_t startTime); static void mndFreeConn(SConnObj *pConn); static SConnObj *mndAcquireConn(SMnode *pMnode, int32_t connId); static void mndReleaseConn(SMnode *pMnode, SConnObj *pConn); @@ -102,13 +103,14 @@ void mndCleanupProfile(SMnode *pMnode) { } } -static SConnObj *mndCreateConn(SMnode *pMnode, char *user, uint32_t ip, uint16_t port, int32_t pid, const char *app) { +static SConnObj *mndCreateConn(SMnode *pMnode, char *user, uint32_t ip, uint16_t port, int32_t pid, const char *app, int64_t startTime) { SProfileMgmt *pMgmt = &pMnode->profileMgmt; int32_t connId = atomic_add_fetch_32(&pMgmt->connId, 1); if (connId == 0) atomic_add_fetch_32(&pMgmt->connId, 1); SConnObj connObj = {.pid = pid, + .appStartTime = startTime, .id = connId, .killed = 0, .port = port, @@ -195,6 +197,7 @@ static int32_t mndProcessConnectMsg(SMnodeMsg *pMsg) { SMnode *pMnode = pMsg->pMnode; SConnectMsg *pReq = pMsg->rpcMsg.pCont; pReq->pid = htonl(pReq->pid); + pReq->startTime = htobe64(pReq->startTime); SRpcConnInfo info = {0}; if (rpcGetConnInfo(pMsg->rpcMsg.handle, &info) != 0) { @@ -216,7 +219,7 @@ static int32_t mndProcessConnectMsg(SMnodeMsg *pMsg) { mndReleaseDb(pMnode, pDb); } - SConnObj *pConn = mndCreateConn(pMnode, info.user, info.clientIp, info.clientPort, pReq->pid, pReq->app); + SConnObj *pConn = mndCreateConn(pMnode, info.user, info.clientIp, info.clientPort, pReq->pid, pReq->app, pReq->startTime); if (pConn == NULL) { mError("user:%s, failed to login from %s while create connection since %s", pMsg->user, ip, terrstr()); return -1; @@ -233,9 +236,7 @@ static int32_t mndProcessConnectMsg(SMnodeMsg *pMsg) { SUserObj *pUser = mndAcquireUser(pMnode, pMsg->user); if (pUser != NULL) { pRsp->acctId = htonl(pUser->acctId); - pRsp->superAuth = pUser->superAuth; - pRsp->readAuth = pUser->readAuth; - pRsp->writeAuth = pUser->writeAuth; + pRsp->superUser = pUser->superUser; mndReleaseUser(pMnode, pUser); } @@ -246,7 +247,8 @@ static int32_t mndProcessConnectMsg(SMnodeMsg *pMsg) { pMsg->contLen = sizeof(SConnectRsp); pMsg->pCont = pRsp; - mDebug("user:%s, login from %s, conn:%d", info.user, ip, pConn->id); + + mDebug("user:%s, login from %s, conn:%d, app:%s", info.user, ip, pConn->id, pReq->app); return 0; } @@ -301,7 +303,7 @@ static int32_t mndProcessHeartBeatMsg(SMnodeMsg *pMsg) { SConnObj *pConn = mndAcquireConn(pMnode, pReq->connId); if (pConn == NULL) { - pConn = mndCreateConn(pMnode, info.user, info.clientIp, info.clientPort, pReq->pid, pReq->app); + pConn = mndCreateConn(pMnode, info.user, info.clientIp, info.clientPort, pReq->pid, pReq->app, 0); if (pConn == NULL) { mError("user:%s, conn:%d is freed and failed to create new conn since %s", pMsg->user, pReq->connId, terrstr()); return -1; @@ -368,7 +370,7 @@ static int32_t mndProcessKillQueryMsg(SMnodeMsg *pMsg) { SUserObj *pUser = mndAcquireUser(pMnode, pMsg->user); if (pUser == NULL) return 0; - if (!pUser->superAuth) { + if (!pUser->superUser) { mndReleaseUser(pMnode, pUser); terrno = TSDB_CODE_MND_NO_RIGHTS; return -1; @@ -399,7 +401,7 @@ static int32_t mndProcessKillStreamMsg(SMnodeMsg *pMsg) { SUserObj *pUser = mndAcquireUser(pMnode, pMsg->user); if (pUser == NULL) return 0; - if (!pUser->superAuth) { + if (!pUser->superUser) { mndReleaseUser(pMnode, pUser); terrno = TSDB_CODE_MND_NO_RIGHTS; return -1; @@ -430,7 +432,7 @@ static int32_t mndProcessKillConnectionMsg(SMnodeMsg *pMsg) { SUserObj *pUser = mndAcquireUser(pMnode, pMsg->user); if (pUser == NULL) return 0; - if (!pUser->superAuth) { + if (!pUser->superUser) { mndReleaseUser(pMnode, pUser); terrno = TSDB_CODE_MND_NO_RIGHTS; return -1; @@ -459,7 +461,7 @@ static int32_t mndGetConnsMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg * SUserObj *pUser = mndAcquireUser(pMnode, pMsg->user); if (pUser == NULL) return 0; - if (!pUser->superAuth) { + if (!pUser->superUser) { mndReleaseUser(pMnode, pUser); terrno = TSDB_CODE_MND_NO_RIGHTS; return -1; @@ -587,7 +589,7 @@ static int32_t mndGetQueryMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg * SUserObj *pUser = mndAcquireUser(pMnode, pMsg->user); if (pUser == NULL) return 0; - if (!pUser->superAuth) { + if (!pUser->superUser) { mndReleaseUser(pMnode, pUser); terrno = TSDB_CODE_MND_NO_RIGHTS; return -1; @@ -803,7 +805,7 @@ static int32_t mndGetStreamMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg SUserObj *pUser = mndAcquireUser(pMnode, pMsg->user); if (pUser == NULL) return 0; - if (!pUser->superAuth) { + if (!pUser->superUser) { mndReleaseUser(pMnode, pUser); terrno = TSDB_CODE_MND_NO_RIGHTS; return -1; diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index ef823a5f0b..e0c8c21c72 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -65,11 +65,9 @@ static int32_t mndCreateDefaultUser(SMnode *pMnode, char *acct, char *user, char taosEncryptPass((uint8_t *)pass, strlen(pass), userObj.pass); userObj.createdTime = taosGetTimestampMs(); userObj.updateTime = userObj.createdTime; - userObj.readAuth = 1; - userObj.writeAuth = 1; if (strcmp(user, TSDB_DEFAULT_USER) == 0) { - userObj.superAuth = 1; + userObj.superUser = 1; } SSdbRaw *pRaw = mndUserActionEncode(&userObj); @@ -102,9 +100,7 @@ static SSdbRaw *mndUserActionEncode(SUserObj *pUser) { SDB_SET_BINARY(pRaw, dataPos, pUser->acct, TSDB_USER_LEN) SDB_SET_INT64(pRaw, dataPos, pUser->createdTime) SDB_SET_INT64(pRaw, dataPos, pUser->updateTime) - SDB_SET_INT8(pRaw, dataPos, pUser->superAuth) - SDB_SET_INT8(pRaw, dataPos, pUser->readAuth) - SDB_SET_INT8(pRaw, dataPos, pUser->writeAuth) + SDB_SET_INT8(pRaw, dataPos, pUser->superUser) SDB_SET_DATALEN(pRaw, dataPos); return pRaw; @@ -130,9 +126,7 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) { SDB_GET_BINARY(pRaw, pRow, dataPos, pUser->acct, TSDB_USER_LEN) SDB_GET_INT64(pRaw, pRow, dataPos, &pUser->createdTime) SDB_GET_INT64(pRaw, pRow, dataPos, &pUser->updateTime) - SDB_GET_INT8(pRaw, pRow, dataPos, &pUser->superAuth) - SDB_GET_INT8(pRaw, pRow, dataPos, &pUser->readAuth) - SDB_GET_INT8(pRaw, pRow, dataPos, &pUser->writeAuth) + SDB_GET_INT8(pRaw, pRow, dataPos, &pUser->superUser) return pRow; } @@ -175,9 +169,7 @@ static int32_t mndUserActionUpdate(SSdb *pSdb, SUserObj *pOldUser, SUserObj *pNe memcpy(pOldUser->acct, pNewUser->acct, TSDB_USER_LEN); pOldUser->createdTime = pNewUser->createdTime; pOldUser->updateTime = pNewUser->updateTime; - pOldUser->superAuth = pNewUser->superAuth; - pOldUser->readAuth = pNewUser->readAuth; - pOldUser->writeAuth = pNewUser->writeAuth; + pOldUser->superUser = pNewUser->superUser; return 0; } @@ -198,9 +190,7 @@ static int32_t mndCreateUser(SMnode *pMnode, char *acct, char *user, char *pass, taosEncryptPass((uint8_t *)pass, strlen(pass), userObj.pass); userObj.createdTime = taosGetTimestampMs(); userObj.updateTime = userObj.createdTime; - userObj.superAuth = 0; - userObj.readAuth = 1; - userObj.writeAuth = 1; + userObj.superUser = 0; STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, pMsg->rpcMsg.handle); if (pTrans == NULL) { @@ -513,15 +503,15 @@ static int32_t mndRetrieveUsers(SMnodeMsg *pMsg, SShowObj *pShow, char *data, in cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - if (pUser->superAuth) { + if (pUser->superUser) { const char *src = "super"; STR_WITH_SIZE_TO_VARSTR(pWrite, src, strlen(src)); - } else if (pUser->writeAuth) { - const char *src = "writable"; - STR_WITH_SIZE_TO_VARSTR(pWrite, src, strlen(src)); - } else { - const char *src = "readable"; - STR_WITH_SIZE_TO_VARSTR(pWrite, src, strlen(src)); +// } else if (pUser->writeAuth) { +// const char *src = "writable"; +// STR_WITH_SIZE_TO_VARSTR(pWrite, src, strlen(src)); +// } else { +// const char *src = "readable"; +// STR_WITH_SIZE_TO_VARSTR(pWrite, src, strlen(src)); } cols++; diff --git a/source/libs/transport/src/rpcMain.c b/source/libs/transport/src/rpcMain.c index 67e468b1bd..c54415860a 100644 --- a/source/libs/transport/src/rpcMain.c +++ b/source/libs/transport/src/rpcMain.c @@ -229,8 +229,6 @@ static void rpcInitImp(void) { tsRpcOverhead = sizeof(SRpcReqContext); tsRpcRefId = taosOpenRef(200, rpcFree); - - return 0; } int32_t rpcInit(void) { From 14ae798fbdc2830498f6e528fa52bfb83bb255d8 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 14 Dec 2021 10:50:16 +0800 Subject: [PATCH 11/43] more --- include/util/tdlist.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/util/tdlist.h b/include/util/tdlist.h index a19f3bebec..d047a57770 100644 --- a/include/util/tdlist.h +++ b/include/util/tdlist.h @@ -58,8 +58,8 @@ extern "C" { // Double linked list #define TD_DLIST_NODE(TYPE) \ struct { \ - TYPE *dl_prev_; \ - TYPE *dl_next_; \ + struct TYPE *dl_prev_; \ + struct TYPE *dl_next_; \ } #define TD_DLIST(TYPE) \ From 032c9d7bb313f652454abe366496505609e8ca67 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Mon, 13 Dec 2021 23:57:54 -0500 Subject: [PATCH 12/43] TD-12034 Define physical plan data structure. --- include/libs/planner/planner.h | 12 ++++---- source/libs/planner/inc/plannerInt.h | 46 +++++++++++++++++++--------- source/libs/planner/src/planner.c | 9 +++--- 3 files changed, 43 insertions(+), 24 deletions(-) diff --git a/include/libs/planner/planner.h b/include/libs/planner/planner.h index 1ff3f02da5..87be26895e 100644 --- a/include/libs/planner/planner.h +++ b/include/libs/planner/planner.h @@ -54,7 +54,7 @@ enum OPERATOR_TYPE_E { struct SEpSet; struct SQueryPlanNode; -struct SQueryDistPlanNode; +struct SQueryPhyPlanNode; struct SQueryStmtInfo; typedef struct SSubquery { @@ -62,7 +62,7 @@ typedef struct SSubquery { int32_t type; // QUERY_TYPE_MERGE|QUERY_TYPE_PARTIAL int32_t level; // the execution level of current subquery, starting from 0. SArray *pUpstream; // the upstream,from which to fetch the result - struct SQueryDistPlanNode *pNode; // physical plan of current subquery + struct SQueryPhyPlanNode *pNode; // physical plan of current subquery } SSubquery; typedef struct SQueryJob { @@ -108,7 +108,7 @@ int32_t qQueryPlanToSql(struct SQueryPlanNode* pQueryNode, char** sql); * @param pPhyNode * @return */ -int32_t qCreatePhysicalPlan(struct SQueryPlanNode* pQueryNode, struct SEpSet* pQnode, struct SQueryDistPlanNode *pPhyNode); +int32_t qCreatePhysicalPlan(struct SQueryPlanNode* pQueryNode, struct SEpSet* pQnode, struct SQueryPhyPlanNode *pPhyNode); /** * Convert to physical plan to string to enable to print it out in the shell. @@ -116,7 +116,7 @@ int32_t qCreatePhysicalPlan(struct SQueryPlanNode* pQueryNode, struct SEpSet* pQ * @param str * @return */ -int32_t qPhyPlanToString(struct SQueryDistPlanNode *pPhyNode, char** str); +int32_t qPhyPlanToString(struct SQueryPhyPlanNode *pPhyNode, char** str); /** * Destroy the query plan object. @@ -129,7 +129,7 @@ void* qDestroyQueryPlan(struct SQueryPlanNode* pQueryNode); * @param pQueryPhyNode * @return */ -void* qDestroyQueryPhyPlan(struct SQueryDistPlanNode* pQueryPhyNode); +void* qDestroyQueryPhyPlan(struct SQueryPhyPlanNode* pQueryPhyNode); /** * Create the query job from the physical execution plan @@ -137,7 +137,7 @@ void* qDestroyQueryPhyPlan(struct SQueryDistPlanNode* pQueryPhyNode); * @param pJob * @return */ -int32_t qCreateQueryJob(const struct SQueryDistPlanNode* pPhyNode, struct SQueryJob** pJob); +int32_t qCreateQueryJob(const struct SQueryPhyPlanNode* pPhyNode, struct SQueryJob** pJob); #ifdef __cplusplus } diff --git a/source/libs/planner/inc/plannerInt.h b/source/libs/planner/inc/plannerInt.h index 6bd89905b1..c51a15509d 100644 --- a/source/libs/planner/inc/plannerInt.h +++ b/source/libs/planner/inc/plannerInt.h @@ -57,20 +57,38 @@ typedef struct SQueryPlanNode { struct SQueryPlanNode *nextNode; } SQueryPlanNode; -typedef struct SQueryDistPlanNode { +typedef struct SDataBlockSchema { + int32_t index; + SSchema *pSchema; // the schema of the SSDatablock + int32_t numOfCols; // number of columns +} SDataBlockSchema; + +typedef struct SQueryPhyPlanNode { SQueryNodeBasicInfo info; - SSchema *pSchema; // the schema of the input SSDatablock - int32_t numOfCols; // number of input columns - SArray *pExpr; // the query functions or sql aggregations - int32_t numOfExpr; // number of result columns, which is also the number of pExprs - void *pExtInfo; // additional information + SArray *pTarget; // target list to be computed at this node + SArray *qual; // implicitly-ANDed qual conditions + SDataBlockSchema targetSchema; + // children plan to generated result for current node to process + // in case of join, multiple plan nodes exist. + SArray *pChildren; +} SQueryPhyPlanNode; - // previous operator to generated result for current node to process - // in case of join, multiple prev nodes exist. - SArray *pPrevNodes; // upstream nodes, or exchange operator to load data from multiple sources. -} SQueryDistPlanNode; +typedef struct SQueryScanPhyNode { + SQueryPhyPlanNode node; + uint64_t uid; +} SQueryScanPhyNode; -typedef struct SQueryCostSummary { +typedef struct SQueryProjectPhyNode { + SQueryPhyPlanNode node; +} SQueryProjectPhyNode; + +typedef struct SQueryAggPhyNode { + SQueryPhyPlanNode node; + SArray *pGroup; + // SInterval +} SQueryAggPhyNode; + +typedef struct SQueryProfileSummary { int64_t startTs; // Object created and added into the message queue int64_t endTs; // the timestamp when the task is completed int64_t cputime; // total cpu cost, not execute elapsed time @@ -91,14 +109,14 @@ typedef struct SQueryCostSummary { uint32_t loadBlockAgg; uint32_t skipBlocks; uint64_t resultSize; // generated result size in Kb. -} SQueryCostSummary; +} SQueryProfileSummary; typedef struct SQueryTask { uint64_t queryId; // query id uint64_t taskId; // task id - SQueryDistPlanNode *pNode; // operator tree + SQueryPhyPlanNode *pNode; // operator tree uint64_t status; // task status - SQueryCostSummary summary; // task execution summary + SQueryProfileSummary summary; // task execution summary void *pOutputHandle; // result buffer handle, to temporarily keep the output result for next stage } SQueryTask; diff --git a/source/libs/planner/src/planner.c b/source/libs/planner/src/planner.c index 79c7691698..121a7d3c2c 100644 --- a/source/libs/planner/src/planner.c +++ b/source/libs/planner/src/planner.c @@ -66,11 +66,12 @@ int32_t qQueryPlanToSql(struct SQueryPlanNode* pQueryNode, char** sql) { return 0; } -int32_t qCreatePhysicalPlan(struct SQueryPlanNode* pQueryNode, struct SEpSet* pQnode, struct SQueryDistPlanNode *pPhyNode) { +int32_t qCreatePhysicalPlan(struct SQueryPlanNode* pQueryNode, struct SEpSet* pQnode, struct SQueryPhyPlanNode *pPhyNode) { + return 0; } -int32_t qPhyPlanToString(struct SQueryDistPlanNode *pPhyNode, char** str) { +int32_t qPhyPlanToString(struct SQueryPhyPlanNode *pPhyNode, char** str) { return 0; } @@ -83,11 +84,11 @@ void* qDestroyQueryPlan(SQueryPlanNode* pQueryNode) { return NULL; } -void* qDestroyQueryPhyPlan(struct SQueryDistPlanNode* pQueryPhyNode) { +void* qDestroyQueryPhyPlan(struct SQueryPhyPlanNode* pQueryPhyNode) { return NULL; } -int32_t qCreateQueryJob(const struct SQueryDistPlanNode* pPhyNode, struct SQueryJob** pJob) { +int32_t qCreateQueryJob(const struct SQueryPhyPlanNode* pPhyNode, struct SQueryJob** pJob) { return 0; } From 1d54f14c3bcf726e64807d1513c3066ed6f75e9c Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 14 Dec 2021 13:40:41 +0800 Subject: [PATCH 13/43] implement commit threads and queue --- include/util/tmacro.h | 8 +- source/dnode/vnode/impl/inc/vnodeCommit.h | 3 - source/dnode/vnode/impl/inc/vnodeDef.h | 25 ++++++ source/dnode/vnode/impl/src/vnodeCommit.c | 9 -- source/dnode/vnode/impl/src/vnodeMain.c | 30 ------- source/dnode/vnode/impl/src/vnodeMgr.c | 102 ++++++++++++++++++++++ 6 files changed, 130 insertions(+), 47 deletions(-) create mode 100644 source/dnode/vnode/impl/src/vnodeMgr.c diff --git a/include/util/tmacro.h b/include/util/tmacro.h index 74056cfe07..5cca8a1062 100644 --- a/include/util/tmacro.h +++ b/include/util/tmacro.h @@ -29,13 +29,11 @@ extern "C" { #define TD_MOD_UNCLEARD 0 #define TD_MOD_CLEARD 1 -#define TD_DEF_MOD_INIT_FLAG(MOD) static int8_t MOD##InitFlag = TD_MOD_UNINITIALIZED -#define TD_DEF_MOD_CLEAR_FLAG(MOD) static int8_t MOD##ClearFlag = TD_MOD_UNCLEARD +typedef int8_t td_mode_flag_t; -#define TD_CHECK_AND_SET_MODE_INIT(MOD) \ - atomic_val_compare_exchange_8(&(MOD##InitFlag), TD_MOD_UNINITIALIZED, TD_MOD_INITIALIZED) +#define TD_CHECK_AND_SET_MODE_INIT(FLAG) atomic_val_compare_exchange_8((FLAG), TD_MOD_UNINITIALIZED, TD_MOD_INITIALIZED) -#define TD_CHECK_AND_SET_MOD_CLEAR(MOD) atomic_val_compare_exchange_8(&(MOD##ClearFlag), TD_MOD_UNCLEARD, TD_MOD_CLEARD) +#define TD_CHECK_AND_SET_MOD_CLEAR(FLAG) atomic_val_compare_exchange_8((FLAG), TD_MOD_UNCLEARD, TD_MOD_CLEARD) #ifdef __cplusplus } diff --git a/source/dnode/vnode/impl/inc/vnodeCommit.h b/source/dnode/vnode/impl/inc/vnodeCommit.h index 8f0af27513..a60e8feac2 100644 --- a/source/dnode/vnode/impl/inc/vnodeCommit.h +++ b/source/dnode/vnode/impl/inc/vnodeCommit.h @@ -22,9 +22,6 @@ extern "C" { #endif -int vnodeInitCommit(uint16_t nthreads); -void vnodeClearCommit(); - #define vnodeShouldCommit vnodeBufPoolIsFull int vnodeAsyncCommit(SVnode *pVnode); diff --git a/source/dnode/vnode/impl/inc/vnodeDef.h b/source/dnode/vnode/impl/inc/vnodeDef.h index e3a3fac6b9..e6a88c7629 100644 --- a/source/dnode/vnode/impl/inc/vnodeDef.h +++ b/source/dnode/vnode/impl/inc/vnodeDef.h @@ -21,9 +21,11 @@ #include "tcoding.h" #include "tdlist.h" #include "tlockfree.h" +#include "tmacro.h" #include "wal.h" #include "vnode.h" + #include "vnodeBufferPool.h" #include "vnodeCfg.h" #include "vnodeCommit.h" @@ -37,6 +39,27 @@ extern "C" { #endif +typedef struct SVnodeTask { + TD_DLIST_NODE(SVnodeTask); + void* arg; + int (*execute)(void*); +} SVnodeTask; + +typedef struct SVnodeMgr { + td_mode_flag_t vnodeInitFlag; + td_mode_flag_t vnodeClearFlag; + // For commit + bool stop; + uint16_t nthreads; + pthread_t* threads; + pthread_mutex_t mutex; + pthread_cond_t hasTask; + TD_DLIST(SVnodeTask) queue; + // For vnode Mgmt +} SVnodeMgr; + +extern SVnodeMgr vnodeMgr; + struct SVnode { char* path; SVnodeCfg config; @@ -50,6 +73,8 @@ struct SVnode { SVnodeFS* pFs; }; +int vnodeScheduleTask(SVnodeTask* task); + #ifdef __cplusplus } #endif diff --git a/source/dnode/vnode/impl/src/vnodeCommit.c b/source/dnode/vnode/impl/src/vnodeCommit.c index 944fe80b31..cac7999f59 100644 --- a/source/dnode/vnode/impl/src/vnodeCommit.c +++ b/source/dnode/vnode/impl/src/vnodeCommit.c @@ -18,15 +18,6 @@ static int vnodeStartCommit(SVnode *pVnode); static int vnodeEndCommit(SVnode *pVnode); -int vnodeInitCommit(uint16_t nthreads) { - // TODO - return 0; -} - -void vnodeClearCommit() { - // TODO -} - int vnodeAsyncCommit(SVnode *pVnode) { #if 0 if (vnodeStartCommit(pVnode) < 0) { diff --git a/source/dnode/vnode/impl/src/vnodeMain.c b/source/dnode/vnode/impl/src/vnodeMain.c index 63fc0d52f0..59e3bae5d7 100644 --- a/source/dnode/vnode/impl/src/vnodeMain.c +++ b/source/dnode/vnode/impl/src/vnodeMain.c @@ -13,7 +13,6 @@ * along with this program. If not, see . */ -#include "tmacro.h" #include "vnodeDef.h" static SVnode *vnodeNew(const char *path, const SVnodeCfg *pVnodeCfg); @@ -21,35 +20,6 @@ static void vnodeFree(SVnode *pVnode); static int vnodeOpenImpl(SVnode *pVnode); static void vnodeCloseImpl(SVnode *pVnode); -TD_DEF_MOD_INIT_FLAG(vnode); -TD_DEF_MOD_CLEAR_FLAG(vnode); - -int vnodeInit(uint16_t nthreads) { - if (TD_CHECK_AND_SET_MODE_INIT(vnode) == TD_MOD_INITIALIZED) { - return 0; - } - - if (walInit() < 0) { - return -1; - } - - if (vnodeInitCommit(nthreads) < 0) { - return -1; - } - - return 0; -} - -void vnodeClear() { - if (TD_CHECK_AND_SET_MOD_CLEAR(vnode) == TD_MOD_CLEARD) { - return; - } - - walCleanUp(); - - vnodeClearCommit(); -} - SVnode *vnodeOpen(const char *path, const SVnodeCfg *pVnodeCfg) { SVnode *pVnode = NULL; diff --git a/source/dnode/vnode/impl/src/vnodeMgr.c b/source/dnode/vnode/impl/src/vnodeMgr.c new file mode 100644 index 0000000000..b5bd913c52 --- /dev/null +++ b/source/dnode/vnode/impl/src/vnodeMgr.c @@ -0,0 +1,102 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "vnodeDef.h" + +SVnodeMgr vnodeMgr = {.vnodeInitFlag = TD_MOD_UNINITIALIZED, .vnodeClearFlag = TD_MOD_UNCLEARD, .stop = false}; + +static void* loop(void* arg); + +int vnodeInit(uint16_t nthreads) { + if (TD_CHECK_AND_SET_MODE_INIT(&(vnodeMgr.vnodeInitFlag)) == TD_MOD_INITIALIZED) { + return 0; + } + + // Start commit handers + if (nthreads > 0) { + vnodeMgr.nthreads = nthreads; + vnodeMgr.threads = (pthread_t*)calloc(nthreads, sizeof(pthread_t)); + if (vnodeMgr.threads == NULL) { + return -1; + } + + pthread_mutex_init(&(vnodeMgr.mutex), NULL); + pthread_cond_init(&(vnodeMgr.hasTask), NULL); + tDListInit(&(vnodeMgr.queue)); + + for (uint16_t i = 0; i < nthreads; i++) { + pthread_create(&(vnodeMgr.threads[i]), NULL, loop, NULL); + } + } else { + // TODO: if no commit thread is set, then another mechanism should be + // given. Otherwise, it is a false. + ASSERT(0); + } + + if (walInit() < 0) { + return -1; + } + + return 0; +} + +void vnodeClear() { + if (TD_CHECK_AND_SET_MOD_CLEAR(&(vnodeMgr.vnodeClearFlag)) == TD_MOD_CLEARD) { + return; + } + + walCleanUp(); + + // Stop commit handler + pthread_mutex_lock(&(vnodeMgr.mutex)); + vnodeMgr.stop = true; + pthread_cond_broadcast(&(vnodeMgr.hasTask)); + pthread_mutex_unlock(&(vnodeMgr.mutex)); + + for (uint16_t i = 0; i < vnodeMgr.nthreads; i++) { + pthread_join(vnodeMgr.threads[i], NULL); + } + + tfree(vnodeMgr.threads); + pthread_cond_destroy(&(vnodeMgr.hasTask)); + pthread_mutex_destroy(&(vnodeMgr.mutex)); +} + +static void* loop(void* arg) { + SVnodeTask* pTask; + for (;;) { + pthread_mutex_lock(&(vnodeMgr.mutex)); + for (;;) { + pTask = TD_DLIST_HEAD(&(vnodeMgr.queue)); + if (pTask == NULL) { + if (vnodeMgr.stop) { + pthread_mutex_unlock(&(vnodeMgr.mutex)); + return NULL; + } else { + pthread_cond_wait(&(vnodeMgr.hasTask), &(vnodeMgr.mutex)); + } + } else { + tDListPop(&(vnodeMgr.queue), pTask); + break; + } + } + + pthread_mutex_unlock(&(vnodeMgr.mutex)); + + (*(pTask->execute))(pTask->arg); + } + + return NULL; +} \ No newline at end of file From df35435416440929f585336e7f0a6c6639e7cd9f Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 14 Dec 2021 14:32:07 +0800 Subject: [PATCH 14/43] add buffer recycle --- source/dnode/vnode/impl/inc/vnodeBufferPool.h | 2 ++ source/dnode/vnode/impl/inc/vnodeCommit.h | 1 + source/dnode/vnode/impl/src/vnodeBufferPool.c | 21 ++++++++++++++ source/dnode/vnode/impl/src/vnodeCommit.c | 29 +++++++------------ source/dnode/vnode/impl/src/vnodeMgr.c | 13 +++++++++ 5 files changed, 48 insertions(+), 18 deletions(-) diff --git a/source/dnode/vnode/impl/inc/vnodeBufferPool.h b/source/dnode/vnode/impl/inc/vnodeBufferPool.h index bfc4de9e12..d96671d2bd 100644 --- a/source/dnode/vnode/impl/inc/vnodeBufferPool.h +++ b/source/dnode/vnode/impl/inc/vnodeBufferPool.h @@ -27,6 +27,8 @@ typedef struct SVBufPool SVBufPool; int vnodeOpenBufPool(SVnode *pVnode); void vnodeCloseBufPool(SVnode *pVnode); +int vnodeBufPoolSwitch(SVnode *pVnode); +int vnodeBufPoolRecycle(SVnode *pVnode); void *vnodeMalloc(SVnode *pVnode, uint64_t size); bool vnodeBufPoolIsFull(SVnode *pVnode); diff --git a/source/dnode/vnode/impl/inc/vnodeCommit.h b/source/dnode/vnode/impl/inc/vnodeCommit.h index a60e8feac2..031089ba14 100644 --- a/source/dnode/vnode/impl/inc/vnodeCommit.h +++ b/source/dnode/vnode/impl/inc/vnodeCommit.h @@ -24,6 +24,7 @@ extern "C" { #define vnodeShouldCommit vnodeBufPoolIsFull int vnodeAsyncCommit(SVnode *pVnode); +int vnodeCommit(void *arg); #ifdef __cplusplus } diff --git a/source/dnode/vnode/impl/src/vnodeBufferPool.c b/source/dnode/vnode/impl/src/vnodeBufferPool.c index 1db15c3990..2affcbb95d 100644 --- a/source/dnode/vnode/impl/src/vnodeBufferPool.c +++ b/source/dnode/vnode/impl/src/vnodeBufferPool.c @@ -79,6 +79,27 @@ void vnodeCloseBufPool(SVnode *pVnode) { } } +int vnodeBufPoolSwitch(SVnode *pVnode) { + SVMemAllocator *pvma = pVnode->pBufPool->inuse; + + pVnode->pBufPool->inuse = NULL; + + tDListAppend(&(pVnode->pBufPool->incycle), pvma); + return 0; +} + +int vnodeBufPoolRecycle(SVnode *pVnode) { + SVBufPool * pBufPool = pVnode->pBufPool; + SVMemAllocator *pvma = TD_DLIST_HEAD(&(pBufPool->incycle)); + ASSERT(pvma != NULL); + + tDListPop(&(pBufPool->incycle), pvma); + vmaReset(pvma); + tDListAppend(&(pBufPool->free), pvma); + + return 0; +} + void *vnodeMalloc(SVnode *pVnode, uint64_t size) { SVBufPool *pBufPool = pVnode->pBufPool; diff --git a/source/dnode/vnode/impl/src/vnodeCommit.c b/source/dnode/vnode/impl/src/vnodeCommit.c index cac7999f59..a728de0ebb 100644 --- a/source/dnode/vnode/impl/src/vnodeCommit.c +++ b/source/dnode/vnode/impl/src/vnodeCommit.c @@ -19,28 +19,21 @@ static int vnodeStartCommit(SVnode *pVnode); static int vnodeEndCommit(SVnode *pVnode); int vnodeAsyncCommit(SVnode *pVnode) { -#if 0 - if (vnodeStartCommit(pVnode) < 0) { - // TODO - } + vnodeBufPoolSwitch(pVnode); + SVnodeTask *pTask = (SVnodeTask *)malloc(sizeof(*pTask)); - if (tqCommit(pVnode->pTQ) < 0) { - // TODO - } + pTask->execute = vnodeCommit; // TODO + pTask->arg = pVnode; // TODO - if (metaCommit(pVnode->pMeta) < 0) { - // TODO - } + vnodeScheduleTask(pTask); + return 0; +} - if (tsdbCommit(pVnode->pTsdb) < 0) { - // TODO - } +int vnodeCommit(void *arg) { + SVnode *pVnode = (SVnode *)arg; - if (vnodeEndCommit(pVnode) < 0) { - // TODO - } - -#endif + vnodeBufPoolRecycle(pVnode); + // TODO return 0; } diff --git a/source/dnode/vnode/impl/src/vnodeMgr.c b/source/dnode/vnode/impl/src/vnodeMgr.c index b5bd913c52..964cbe77da 100644 --- a/source/dnode/vnode/impl/src/vnodeMgr.c +++ b/source/dnode/vnode/impl/src/vnodeMgr.c @@ -74,6 +74,19 @@ void vnodeClear() { pthread_mutex_destroy(&(vnodeMgr.mutex)); } +int vnodeScheduleTask(SVnodeTask* pTask) { + pthread_mutex_lock(&(vnodeMgr.mutex)); + + tDListAppend(&(vnodeMgr.queue), pTask); + + pthread_cond_signal(&(vnodeMgr.hasTask)); + + pthread_mutex_unlock(&(vnodeMgr.mutex)); + + return 0; +} + +/* ------------------------ STATIC METHODS ------------------------ */ static void* loop(void* arg) { SVnodeTask* pTask; for (;;) { From 136b04a5f64eddebf389793719965167afcd56ad Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 14 Dec 2021 15:07:51 +0800 Subject: [PATCH 15/43] more --- source/dnode/vnode/impl/src/vnodeCfg.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/impl/src/vnodeCfg.c b/source/dnode/vnode/impl/src/vnodeCfg.c index f5bb7e35d2..6969ca092d 100644 --- a/source/dnode/vnode/impl/src/vnodeCfg.c +++ b/source/dnode/vnode/impl/src/vnodeCfg.c @@ -15,7 +15,7 @@ #include "vnodeDef.h" -const SVnodeCfg defaultVnodeOptions = {.wsize = 16 * 1024 * 1024, .walCfg = {.level = TAOS_WAL_WRITE}}; /* TODO */ +const SVnodeCfg defaultVnodeOptions = {.wsize = 96 * 1024 * 1024, .walCfg = {.level = TAOS_WAL_WRITE}}; /* TODO */ void vnodeOptionsInit(SVnodeCfg *pVnodeOptions) { /* TODO */ vnodeOptionsCopy(pVnodeOptions, &defaultVnodeOptions); From 14952d7fb4af670770bfb99a5841073d898ba8ce Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 14 Dec 2021 15:08:06 +0800 Subject: [PATCH 16/43] [td-10564] fix bug in taos_close --- source/client/inc/clientInt.h | 4 ++-- source/client/src/clientImpl.c | 6 +++-- source/client/src/clientmain.c | 3 +++ source/client/src/tscEnv.c | 35 ++++++++++++++---------------- source/client/test/clientTests.cpp | 5 ++++- 5 files changed, 29 insertions(+), 24 deletions(-) diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index 749894444e..3180923aff 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -58,7 +58,7 @@ typedef struct SAppInstInfo { SCorEpSet mgmtEp; SInstanceActivity summary; SList *pConnList; // STscObj linked list - char clusterId[TSDB_CLUSTER_ID_LEN]; + uint32_t clusterId; void *pTransporter; } SAppInstInfo; @@ -127,7 +127,7 @@ void* createTscObj(const char* user, const char* auth, const char *ip, uint32_t void destroyTscObj(void*pObj); void* createRequest(STscObj* pObj, __taos_async_fn_t fp, void* param, int32_t type); -void destroyRequest(void* p); +void destroyRequest(SRequestObj* pRequest); TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass, const char *auth, const char *db, uint16_t port); diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 0f9b40e262..47b2c61ffc 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -172,7 +172,7 @@ STscObj* taosConnectImpl(const char *ip, const char *user, const char *auth, con taos_close(pTscObj); pTscObj = NULL; } else { - tscDebug("%p connection is opening, dnodeConn:%p", pTscObj, pTscObj->pTransporter); + tscDebug("0x%"PRIx64" connection is opening, connId:%d, dnodeConn:%p", pTscObj->id, pTscObj->connId, pTscObj->pTransporter); destroyRequest(pRequest); } @@ -267,6 +267,8 @@ void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) { tstrerror(pMsg->code), pMsg->contLen); } - taosReleaseRef(requestRefId, requestRefId); + taosReleaseRef(tscReqRef, requestRefId); rpcFreeCont(pMsg->pCont); + + sem_post(&pRequest->body.rspSem); } diff --git a/source/client/src/clientmain.c b/source/client/src/clientmain.c index bda2311f97..ba80135850 100644 --- a/source/client/src/clientmain.c +++ b/source/client/src/clientmain.c @@ -70,7 +70,10 @@ void taos_close(TAOS* taos) { return; } + STscObj *pTscObj = (STscObj *)taos; + tscDebug("0x%"PRIx64" try to close connection, numOfReq:%d", pTscObj->id, pTscObj->numOfReqs); + taosRemoveRef(tscConnRef, pTscObj->id); } const char *taos_errstr(TAOS_RES *res) { diff --git a/source/client/src/tscEnv.c b/source/client/src/tscEnv.c index 7565dd0c53..d37eb1ebcb 100644 --- a/source/client/src/tscEnv.c +++ b/source/client/src/tscEnv.c @@ -36,8 +36,6 @@ int32_t tscConnRef = -1; void *tscQhandle = NULL; int32_t tsNumOfThreads = 1; - -pthread_mutex_t rpcObjMutex; // mutex to protect open the rpc obj concurrently volatile int32_t tscInitRes = 0; static void registerRequest(SRequestObj* pRequest) { @@ -65,12 +63,10 @@ static void deregisterRequest(SRequestObj* pRequest) { STscObj* pTscObj = pRequest->pTscObj; SInstanceActivity* pActivity = &pTscObj->pAppInfo->summary; - taosReleaseRef(tscReqRef, pRequest->self); - int32_t currentInst = atomic_sub_fetch_32(&pActivity->currentRequests, 1); int32_t num = atomic_sub_fetch_32(&pTscObj->numOfReqs, 1); - tscDebug("0x%"PRIx64" free Request from 0x%"PRIx64", current:%d, app current:%d", pRequest->self, pTscObj->id, num, currentInst); + tscDebug("0x%"PRIx64" free Request from connObj: 0x%"PRIx64", current:%d, app current:%d", pRequest->self, pTscObj->id, num, currentInst); taosReleaseRef(tscConnRef, pTscObj->id); } @@ -90,15 +86,6 @@ static void tscInitLogFile() { } } -void tscFreeRpcObj(void *param) { -#if 0 - assert(param); - SRpcObj *pRpcObj = (SRpcObj *)(param); - tscDebug("free rpcObj:%p and free pDnodeConn: %p", pRpcObj, pRpcObj->pDnodeConn); - rpcClose(pRpcObj->pDnodeConn); -#endif -} - void closeTransporter(STscObj* pTscObj) { if (pTscObj == NULL || pTscObj->pTransporter == NULL) { return; @@ -136,7 +123,9 @@ void* openTransporter(const char *user, const char *auth) { void destroyTscObj(void *pObj) { STscObj *pTscObj = pObj; - tscDebug("connect obj destroyed, 0x%"PRIx64, pTscObj->id); + + atomic_sub_fetch_64(&pTscObj->pAppInfo->numOfConns, 1); + tscDebug("connObj 0x%"PRIx64" destroyed, totalConn:%"PRId64, pTscObj->id, pTscObj->pAppInfo->numOfConns); closeTransporter(pTscObj); pthread_mutex_destroy(&pTscObj->mutex); @@ -161,7 +150,7 @@ void* createTscObj(const char* user, const char* auth, const char *ip, uint32_t pthread_mutex_init(&pObj->mutex, NULL); pObj->id = taosAddRef(tscConnRef, pObj); - tscDebug("connect obj created, 0x%"PRIx64, pObj->id); + tscDebug("connObj created, 0x%"PRIx64, pObj->id); return pObj; } @@ -189,9 +178,9 @@ void* createRequest(STscObj* pObj, __taos_async_fn_t fp, void* param, int32_t ty return pRequest; } -void destroyRequest(void* p) { +static void doDestroyRequest(void* p) { assert(p != NULL); - SRequestObj* pRequest = *(SRequestObj**)p; + SRequestObj* pRequest = (SRequestObj*)p; assert(RID_VALID(pRequest->self)); @@ -202,6 +191,14 @@ void destroyRequest(void* p) { deregisterRequest(pRequest); } +void destroyRequest(SRequestObj* pRequest) { + if (pRequest == NULL) { + return; + } + + taosReleaseRef(tscReqRef, pRequest->self); +} + void taos_init_imp(void) { // In the APIs of other program language, taos_cleanup is not available yet. // So, to make sure taos_cleanup will be invoked to clean up the allocated resource to suppress the valgrind warning. @@ -242,7 +239,7 @@ void taos_init_imp(void) { tscDebug("client task queue is initialized, numOfThreads: %d", numOfThreads); tscConnRef = taosOpenRef(200, destroyTscObj); - tscReqRef = taosOpenRef(40960, destroyRequest); + tscReqRef = taosOpenRef(40960, doDestroyRequest); taosGetAppName(appInfo.appName, NULL); appInfo.pid = taosGetPId(); diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index ca6e2b44a8..8da4caf7eb 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -33,5 +33,8 @@ int main(int argc, char** argv) { } TEST(testCase, driverInit_Test) { - TAOS* pTaos = taos_connect("ubuntu", "root", "taosdata", NULL, 0); + TAOS* pConn = taos_connect("ubuntu", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + taos_close(pConn); } \ No newline at end of file From 44c453b92867b26a7e534cddd59369194c3ae535 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 14 Dec 2021 15:08:56 +0800 Subject: [PATCH 17/43] [td-10564] add a new file. --- source/client/src/clientMsgHandler.c | 3212 ++++++++++++++++++++++++++ 1 file changed, 3212 insertions(+) create mode 100644 source/client/src/clientMsgHandler.c diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c new file mode 100644 index 0000000000..62466a096d --- /dev/null +++ b/source/client/src/clientMsgHandler.c @@ -0,0 +1,3212 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "os.h" +#include "clientInt.h" +#include "tmsgtype.h" +#include "trpc.h" +#include "tscLog.h" + +int (*buildRequestMsgFp[TSDB_SQL_MAX])(SRequestObj *pRequest, SRequestMsgBody *pMsgBody) = {0}; +int (*handleRequestRspFp[TSDB_SQL_MAX])(SRequestObj *pRequest, const char* pMsg, int32_t msgLen); + +#if 0 +void tscProcessActivityTimer(void *handle, void *tmrId); + +static int32_t extractSTableQueryVgroupId(STableMetaInfo* pTableMetaInfo); + +static int32_t minMsgSize() { return tsRpcHeadSize + 100; } +static int32_t getWaitingTimeInterval(int32_t count) { + int32_t initial = 100; // 100 ms by default + if (count <= 1) { + return 0; + } + + return initial * ((2u)<<(count - 2)); +} + +static int32_t vgIdCompare(const void *lhs, const void *rhs) { + int32_t left = *(int32_t *)lhs; + int32_t right = *(int32_t *)rhs; + + if (left == right) { + return 0; + } else { + return left > right ? 1 : -1; + } +} +static int32_t removeDupVgid(int32_t *src, int32_t sz) { + if (src == NULL || sz <= 0) { + return 0; + } + qsort(src, sz, sizeof(src[0]), vgIdCompare); + + int32_t ret = 1; + for (int i = 1; i < sz; i++) { + if (src[i] != src[i - 1]) { + src[ret++] = src[i]; + } + } + return ret; +} + +static void tscSetDnodeEpSet(SRpcEpSet* pEpSet, SVgroupMsg* pVgroupInfo) { + assert(pEpSet != NULL && pVgroupInfo != NULL && pVgroupInfo->numOfEps > 0); + + // Issue the query to one of the vnode among a vgroup randomly. + // change the inUse property would not affect the isUse attribute of STableMeta + pEpSet->inUse = rand() % pVgroupInfo->numOfEps; + + // apply the FQDN string length check here + bool existed = false; + + pEpSet->numOfEps = pVgroupInfo->numOfEps; + for(int32_t i = 0; i < pVgroupInfo->numOfEps; ++i) { + pEpSet->port[i] = pVgroupInfo->epAddr[i].port; + + int32_t len = (int32_t) strnlen(pVgroupInfo->epAddr[i].fqdn, TSDB_FQDN_LEN); + if (len > 0) { + tstrncpy(pEpSet->fqdn[i], pVgroupInfo->epAddr[i].fqdn, tListLen(pEpSet->fqdn[i])); + existed = true; + } + } + + assert(existed); +} + +static void tscDumpMgmtEpSet(SSqlObj *pSql) { + SCorEpSet *pCorEpSet = pSql->pTscObj->tscCorMgmtEpSet; + taosCorBeginRead(&pCorEpSet->version); + pSql->epSet = pCorEpSet->epSet; + taosCorEndRead(&pCorEpSet->version); +} +static void tscEpSetHtons(SRpcEpSet *s) { + for (int32_t i = 0; i < s->numOfEps; i++) { + s->port[i] = htons(s->port[i]); + } +} + +bool tscEpSetIsEqual(SRpcEpSet *s1, SRpcEpSet *s2) { + if (s1->numOfEps != s2->numOfEps || s1->inUse != s2->inUse) { + return false; + } + + for (int32_t i = 0; i < s1->numOfEps; i++) { + if (s1->port[i] != s2->port[i] + || strncmp(s1->fqdn[i], s2->fqdn[i], TSDB_FQDN_LEN) != 0) + return false; + } + return true; +} + +void tscUpdateMgmtEpSet(SSqlObj *pSql, SRpcEpSet *pEpSet) { + // no need to update if equal + SCorEpSet *pCorEpSet = pSql->pTscObj->tscCorMgmtEpSet; + taosCorBeginWrite(&pCorEpSet->version); + pCorEpSet->epSet = *pEpSet; + taosCorEndWrite(&pCorEpSet->version); +} + +static void tscDumpEpSetFromVgroupInfo(SRpcEpSet *pEpSet, SNewVgroupInfo *pVgroupInfo) { + if (pVgroupInfo == NULL) { return;} + int8_t inUse = pVgroupInfo->inUse; + pEpSet->inUse = (inUse >= 0 && inUse < TSDB_MAX_REPLICA) ? inUse: 0; + pEpSet->numOfEps = pVgroupInfo->numOfEps; + for (int32_t i = 0; i < pVgroupInfo->numOfEps; ++i) { + tstrncpy(pEpSet->fqdn[i], pVgroupInfo->ep[i].fqdn, sizeof(pEpSet->fqdn[i])); + pEpSet->port[i] = pVgroupInfo->ep[i].port; + } +} + +static void tscUpdateVgroupInfo(SSqlObj *pSql, SRpcEpSet *pEpSet) { + SSqlCmd *pCmd = &pSql->cmd; + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); + if (pTableMetaInfo == NULL || pTableMetaInfo->pTableMeta == NULL) { + return; + } + + int32_t vgId = -1; + if (pTableMetaInfo->pTableMeta->tableType == TSDB_SUPER_TABLE) { + vgId = extractSTableQueryVgroupId(pTableMetaInfo); + } else { + vgId = pTableMetaInfo->pTableMeta->vgId; + } + + assert(vgId > 0); + + SNewVgroupInfo vgroupInfo = {.vgId = -1}; + taosHashGetClone(tscVgroupMap, &vgId, sizeof(vgId), NULL, &vgroupInfo); + assert(vgroupInfo.numOfEps > 0 && vgroupInfo.vgId > 0); + + tscDebug("before: Endpoint in use:%d, numOfEps:%d", vgroupInfo.inUse, vgroupInfo.numOfEps); + vgroupInfo.inUse = pEpSet->inUse; + vgroupInfo.numOfEps = pEpSet->numOfEps; + for (int32_t i = 0; i < vgroupInfo.numOfEps; i++) { + tstrncpy(vgroupInfo.ep[i].fqdn, pEpSet->fqdn[i], TSDB_FQDN_LEN); + vgroupInfo.ep[i].port = pEpSet->port[i]; + } + + tscDebug("after: EndPoint in use:%d, numOfEps:%d", vgroupInfo.inUse, vgroupInfo.numOfEps); + taosHashPut(tscVgroupMap, &vgId, sizeof(vgId), &vgroupInfo, sizeof(SNewVgroupInfo)); + + // Update the local cached epSet info cached by SqlObj + int32_t inUse = pSql->epSet.inUse; + tscDumpEpSetFromVgroupInfo(&pSql->epSet, &vgroupInfo); + tscDebug("0x%"PRIx64" update the epSet in SqlObj, in use before:%d, after:%d", pSql->self, inUse, pSql->epSet.inUse); + +} + +int32_t extractSTableQueryVgroupId(STableMetaInfo* pTableMetaInfo) { + assert(pTableMetaInfo != NULL); + + int32_t vgIndex = pTableMetaInfo->vgroupIndex; + int32_t vgId = -1; + + if (pTableMetaInfo->pVgroupTables == NULL) { + SVgroupsInfo *pVgroupInfo = pTableMetaInfo->vgroupList; + assert(pVgroupInfo->vgroups[vgIndex].vgId > 0 && vgIndex < pTableMetaInfo->vgroupList->numOfVgroups); + vgId = pVgroupInfo->vgroups[vgIndex].vgId; + } else { + int32_t numOfVgroups = (int32_t)taosArrayGetSize(pTableMetaInfo->pVgroupTables); + assert(vgIndex >= 0 && vgIndex < numOfVgroups); + + SVgroupTableInfo *pTableIdList = taosArrayGet(pTableMetaInfo->pVgroupTables, vgIndex); + vgId = pTableIdList->vgInfo.vgId; + } + + return vgId; +} + +void tscProcessHeartBeatRsp(void *param, TAOS_RES *tres, int code) { + STscObj *pObj = (STscObj *)param; + if (pObj == NULL) return; + + if (pObj != pObj->signature) { + tscError("heartbeat msg, pObj:%p, signature:%p invalid", pObj, pObj->signature); + return; + } + + SSqlObj *pSql = tres; + SSqlRes *pRes = &pSql->res; + + if (code == TSDB_CODE_SUCCESS) { + SHeartBeatRsp *pRsp = (SHeartBeatRsp *)pRes->pRsp; + SRpcEpSet *epSet = &pRsp->epSet; + if (epSet->numOfEps > 0) { + tscEpSetHtons(epSet); + + //SCorEpSet *pCorEpSet = pSql->pTscObj->tscCorMgmtEpSet; + //if (!tscEpSetIsEqual(&pCorEpSet->epSet, epSet)) { + // tscTrace("%p updating epset: numOfEps: %d, inUse: %d", pSql, epSet->numOfEps, epSet->inUse); + // for (int8_t i = 0; i < epSet->numOfEps; i++) { + // tscTrace("endpoint %d: fqdn=%s, port=%d", i, epSet->fqdn[i], epSet->port[i]); + // } + //} + //concurrency problem, update mgmt epset anyway + tscUpdateMgmtEpSet(pSql, epSet); + } + + pSql->pTscObj->connId = htonl(pRsp->connId); + + if (pRsp->killConnection) { + tscKillConnection(pObj); + return; + } else { + if (pRsp->queryId) { + tscKillQuery(pObj, htonl(pRsp->queryId)); + } + + if (pRsp->streamId) { + tscKillStream(pObj, htonl(pRsp->streamId)); + } + } + + int32_t total = htonl(pRsp->totalDnodes); + int32_t online = htonl(pRsp->onlineDnodes); + assert(online <= total); + + if (online < total) { + tscError("0x%"PRIx64", HB, total dnode:%d, online dnode:%d", pSql->self, total, online); + pSql->res.code = TSDB_CODE_RPC_NETWORK_UNAVAIL; + } + + if (pRes->length == NULL) { + pRes->length = calloc(2, sizeof(int32_t)); + } + + pRes->length[0] = total; + pRes->length[1] = online; + } else { + tscDebug("%" PRId64 " heartbeat failed, code:%s", pObj->hbrid, tstrerror(code)); + if (pRes->length == NULL) { + pRes->length = calloc(2, sizeof(int32_t)); + } + + pRes->length[1] = 0; + if (pRes->length[0] == 0) { + pRes->length[0] = 1; // make sure that the value of the total node is greater than the online node + } + } + + if (pObj->hbrid != 0) { + int32_t waitingDuring = tsShellActivityTimer * 500; + tscDebug("0x%"PRIx64" send heartbeat in %dms", pSql->self, waitingDuring); + + taosTmrReset(tscProcessActivityTimer, waitingDuring, (void *)pObj->rid, tscTmr, &pObj->pTimer); + } else { + tscDebug("0x%"PRIx64" start to close tscObj:%p, not send heartbeat again", pSql->self, pObj); + } +} + +void tscProcessActivityTimer(void *handle, void *tmrId) { + int64_t rid = (int64_t) handle; + STscObj *pObj = taosAcquireRef(tscRefId, rid); + if (pObj == NULL) { + return; + } + + SSqlObj* pHB = taosAcquireRef(tscObjRef, pObj->hbrid); + if (pHB == NULL) { + taosReleaseRef(tscRefId, rid); + return; + } + + assert(pHB->self == pObj->hbrid); + + pHB->retry = 0; + int32_t code = tscBuildAndSendRequest(pHB, NULL); + taosReleaseRef(tscObjRef, pObj->hbrid); + + if (code != TSDB_CODE_SUCCESS) { + tscError("0x%"PRIx64" failed to sent HB to server, reason:%s", pHB->self, tstrerror(code)); + } + + taosReleaseRef(tscRefId, rid); +} + +int tscSendMsgToServer(SSqlObj *pSql) { + STscObj* pObj = pSql->pTscObj; + SSqlCmd* pCmd = &pSql->cmd; + + char *pMsg = rpcMallocCont(pCmd->payloadLen); + if (NULL == pMsg) { + tscError("0x%"PRIx64" msg:%s malloc failed", pSql->self, taosMsg[pSql->cmd.msgType]); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + // set the mgmt ip list + if (pSql->cmd.command >= TSDB_SQL_MGMT) { + tscDumpMgmtEpSet(pSql); + } + + memcpy(pMsg, pSql->cmd.payload, pSql->cmd.payloadLen); + + SRpcMsg rpcMsg = { + .msgType = pSql->cmd.msgType, + .pCont = pMsg, + .contLen = pSql->cmd.payloadLen, + .ahandle = (void*)pSql->self, + .handle = NULL, + .code = 0 + }; + + rpcSendRequest(pObj->pRpcObj->pDnodeConn, &pSql->epSet, &rpcMsg, &pSql->rpcRid); + return TSDB_CODE_SUCCESS; +} + +// handle three situation +// 1. epset retry, only return last failure ep +// 2. no epset retry, like 'taos -h invalidFqdn', return invalidFqdn +// 3. other situation, no expected +void tscSetFqdnErrorMsg(SSqlObj* pSql, SRpcEpSet* pEpSet) { + SSqlCmd* pCmd = &pSql->cmd; + SSqlRes* pRes = &pSql->res; + + char* msgBuf = tscGetErrorMsgPayload(pCmd); + + if (pEpSet) { + sprintf(msgBuf, "%s\"%s\"", tstrerror(pRes->code),pEpSet->fqdn[(pEpSet->inUse)%(pEpSet->numOfEps)]); + } else if (pCmd->command >= TSDB_SQL_MGMT) { + SRpcEpSet tEpset; + + SCorEpSet *pCorEpSet = pSql->pTscObj->tscCorMgmtEpSet; + taosCorBeginRead(&pCorEpSet->version); + tEpset = pCorEpSet->epSet; + taosCorEndRead(&pCorEpSet->version); + + sprintf(msgBuf, "%s\"%s\"", tstrerror(pRes->code),tEpset.fqdn[(tEpset.inUse)%(tEpset.numOfEps)]); + } else { + sprintf(msgBuf, "%s", tstrerror(pRes->code)); + } +} + +void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) { + TSDB_CACHE_PTR_TYPE handle = (TSDB_CACHE_PTR_TYPE) rpcMsg->ahandle; + SSqlObj* pSql = (SSqlObj*)taosAcquireRef(tscObjRef, handle); + if (pSql == NULL) { + rpcFreeCont(rpcMsg->pCont); + return; + } + + assert(pSql->self == handle); + + STscObj *pObj = pSql->pTscObj; + SSqlRes *pRes = &pSql->res; + SSqlCmd *pCmd = &pSql->cmd; + + pSql->rpcRid = -1; + if (pObj->signature != pObj) { + tscDebug("0x%"PRIx64" DB connection is closed, cmd:%d pObj:%p signature:%p", pSql->self, pCmd->command, pObj, pObj->signature); + + taosRemoveRef(tscObjRef, handle); + taosReleaseRef(tscObjRef, handle); + rpcFreeCont(rpcMsg->pCont); + return; + } + + SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); + if (pQueryInfo != NULL && pQueryInfo->type == TSDB_QUERY_TYPE_FREE_RESOURCE) { + tscDebug("0x%"PRIx64" sqlObj needs to be released or DB connection is closed, cmd:%d type:%d, pObj:%p signature:%p", + pSql->self, pCmd->command, pQueryInfo->type, pObj, pObj->signature); + + taosRemoveRef(tscObjRef, handle); + taosReleaseRef(tscObjRef, handle); + rpcFreeCont(rpcMsg->pCont); + return; + } + + if (pEpSet) { + if (!tscEpSetIsEqual(&pSql->epSet, pEpSet)) { + if (pCmd->command < TSDB_SQL_MGMT) { + tscUpdateVgroupInfo(pSql, pEpSet); + } else { + tscUpdateMgmtEpSet(pSql, pEpSet); + } + } + } + + int32_t cmd = pCmd->command; + + // set the flag to denote that sql string needs to be re-parsed and build submit block with table schema + if (cmd == TSDB_SQL_INSERT && rpcMsg->code == TSDB_CODE_TDB_TABLE_RECONFIGURE) { + pSql->cmd.insertParam.schemaAttached = 1; + } + + // single table query error need to be handled here. + if ((cmd == TSDB_SQL_SELECT || cmd == TSDB_SQL_UPDATE_TAGS_VAL) && + (((rpcMsg->code == TSDB_CODE_TDB_INVALID_TABLE_ID || rpcMsg->code == TSDB_CODE_VND_INVALID_VGROUP_ID)) || + rpcMsg->code == TSDB_CODE_RPC_NETWORK_UNAVAIL || rpcMsg->code == TSDB_CODE_APP_NOT_READY)) { + + // 1. super table subquery + // 2. nest queries are all not updated the tablemeta and retry parse the sql after cleanup local tablemeta/vgroup id buffer + if ((TSDB_QUERY_HAS_TYPE(pQueryInfo->type, (TSDB_QUERY_TYPE_STABLE_SUBQUERY | TSDB_QUERY_TYPE_SUBQUERY | + TSDB_QUERY_TYPE_TAG_FILTER_QUERY)) && + !TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_PROJECTION_QUERY)) || + (TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_NEST_SUBQUERY)) || (TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_STABLE_SUBQUERY) && pQueryInfo->distinct)) { + // do nothing in case of super table subquery + } else { + pSql->retry += 1; + tscWarn("0x%" PRIx64 " it shall renew table meta, code:%s, retry:%d", pSql->self, tstrerror(rpcMsg->code), pSql->retry); + + pSql->res.code = rpcMsg->code; // keep the previous error code + if (pSql->retry > pSql->maxRetry) { + tscError("0x%" PRIx64 " max retry %d reached, give up", pSql->self, pSql->maxRetry); + } else { + // wait for a little bit moment and then retry + // todo do not sleep in rpc callback thread, add this process into queue to process + if (rpcMsg->code == TSDB_CODE_APP_NOT_READY || rpcMsg->code == TSDB_CODE_VND_INVALID_VGROUP_ID) { + int32_t duration = getWaitingTimeInterval(pSql->retry); + taosMsleep(duration); + } + + pSql->retryReason = rpcMsg->code; + rpcMsg->code = tscRenewTableMeta(pSql, 0); + // if there is an error occurring, proceed to the following error handling procedure. + if (rpcMsg->code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { + taosReleaseRef(tscObjRef, handle); + rpcFreeCont(rpcMsg->pCont); + return; + } + } + } + } + + pRes->rspLen = 0; + + if (pRes->code == TSDB_CODE_TSC_QUERY_CANCELLED) { + tscDebug("0x%"PRIx64" query is cancelled, code:%s", pSql->self, tstrerror(pRes->code)); + } else { + pRes->code = rpcMsg->code; + } + + if (pRes->code == TSDB_CODE_SUCCESS) { + tscDebug("0x%"PRIx64" reset retry counter to be 0 due to success rsp, old:%d", pSql->self, pSql->retry); + pSql->retry = 0; + } + + if (pRes->code != TSDB_CODE_TSC_QUERY_CANCELLED) { + assert(rpcMsg->msgType == pCmd->msgType + 1); + pRes->code = rpcMsg->code; + pRes->rspType = rpcMsg->msgType; + pRes->rspLen = rpcMsg->contLen; + + if (pRes->rspLen > 0 && rpcMsg->pCont) { + char *tmp = (char *)realloc(pRes->pRsp, pRes->rspLen); + if (tmp == NULL) { + pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; + } else { + pRes->pRsp = tmp; + memcpy(pRes->pRsp, rpcMsg->pCont, pRes->rspLen); + } + } else { + tfree(pRes->pRsp); + } + + /* + * There is not response callback function for submit response. + * The actual inserted number of points is the first number. + */ + if (rpcMsg->msgType == TSDB_MSG_TYPE_SUBMIT_RSP && pRes->pRsp != NULL) { + SShellSubmitRspMsg *pMsg = (SShellSubmitRspMsg*)pRes->pRsp; + pMsg->code = htonl(pMsg->code); + pMsg->numOfRows = htonl(pMsg->numOfRows); + pMsg->affectedRows = htonl(pMsg->affectedRows); + pMsg->failedRows = htonl(pMsg->failedRows); + pMsg->numOfFailedBlocks = htonl(pMsg->numOfFailedBlocks); + + pRes->numOfRows += pMsg->affectedRows; + tscDebug("0x%"PRIx64" SQL cmd:%s, code:%s inserted rows:%d rspLen:%d", pSql->self, sqlCmd[pCmd->command], + tstrerror(pRes->code), pMsg->affectedRows, pRes->rspLen); + } else { + tscDebug("0x%"PRIx64" SQL cmd:%s, code:%s rspLen:%d", pSql->self, sqlCmd[pCmd->command], tstrerror(pRes->code), pRes->rspLen); + } + } + + if (pRes->code == TSDB_CODE_SUCCESS && tscProcessMsgRsp[pCmd->command]) { + rpcMsg->code = (*tscProcessMsgRsp[pCmd->command])(pSql); + } + + bool shouldFree = tscShouldBeFreed(pSql); + if (rpcMsg->code != TSDB_CODE_TSC_ACTION_IN_PROGRESS) { + if (rpcMsg->code != TSDB_CODE_SUCCESS) { + pRes->code = rpcMsg->code; + } + + rpcMsg->code = (pRes->code == TSDB_CODE_SUCCESS) ? (int32_t)pRes->numOfRows : pRes->code; + if (rpcMsg->code == TSDB_CODE_RPC_FQDN_ERROR) { + tscAllocPayload(pCmd, TSDB_FQDN_LEN + 64); + tscSetFqdnErrorMsg(pSql, pEpSet); + } + + (*pSql->fp)(pSql->param, pSql, rpcMsg->code); + } + + if (shouldFree) { // in case of table-meta/vgrouplist query, automatically free it + tscDebug("0x%"PRIx64" sqlObj is automatically freed", pSql->self); + taosRemoveRef(tscObjRef, handle); + } + + taosReleaseRef(tscObjRef, handle); + rpcFreeCont(rpcMsg->pCont); +} + +int doBuildAndSendMsg(SSqlObj *pSql) { + SSqlCmd *pCmd = &pSql->cmd; + SSqlRes *pRes = &pSql->res; + + if (pCmd->command == TSDB_SQL_SELECT || + pCmd->command == TSDB_SQL_FETCH || + pCmd->command == TSDB_SQL_RETRIEVE || + pCmd->command == TSDB_SQL_INSERT || + pCmd->command == TSDB_SQL_CONNECT || + pCmd->command == TSDB_SQL_HB || + pCmd->command == TSDB_SQL_RETRIEVE_FUNC || + pCmd->command == TSDB_SQL_STABLEVGROUP) { + pRes->code = tscBuildMsg[pCmd->command](pSql, NULL); + } + + if (pRes->code != TSDB_CODE_SUCCESS) { + tscAsyncResultOnError(pSql); + return TSDB_CODE_SUCCESS; + } + + int32_t code = tscSendMsgToServer(pSql); + + // NOTE: if code is TSDB_CODE_SUCCESS, pSql may have been released here already by other threads. + if (code != TSDB_CODE_SUCCESS) { + pRes->code = code; + tscAsyncResultOnError(pSql); + return TSDB_CODE_SUCCESS; + } + + return TSDB_CODE_SUCCESS; +} + +int tscBuildAndSendRequest(SSqlObj *pSql, SQueryInfo* pQueryInfo) { + char name[TSDB_TABLE_FNAME_LEN] = {0}; + + SSqlCmd *pCmd = &pSql->cmd; + uint32_t type = 0; + + if (pQueryInfo == NULL) { + pQueryInfo = tscGetQueryInfo(pCmd); + } + + STableMetaInfo *pTableMetaInfo = NULL; + + if (pQueryInfo != NULL) { + pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + if (pTableMetaInfo != NULL) { + tNameExtractFullName(&pTableMetaInfo->name, name); + } + + type = pQueryInfo->type; + + // while numOfTables equals to 0, it must be Heartbeat + assert((pQueryInfo->numOfTables == 0 && (pQueryInfo->command == TSDB_SQL_HB || pSql->cmd.command == TSDB_SQL_RETRIEVE_FUNC)) || pQueryInfo->numOfTables > 0); + } + + tscDebug("0x%"PRIx64" SQL cmd:%s will be processed, name:%s, type:%d", pSql->self, sqlCmd[pCmd->command], name, type); + if (pCmd->command < TSDB_SQL_MGMT) { // the pTableMetaInfo cannot be NULL + if (pTableMetaInfo == NULL) { + pSql->res.code = TSDB_CODE_TSC_APP_ERROR; + return pSql->res.code; + } + } else if (pCmd->command >= TSDB_SQL_LOCAL) { + return (*tscProcessMsgRsp[pCmd->command])(pSql); + } + + return doBuildAndSendMsg(pSql); +} + +int tscBuildFetchMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + SRetrieveTableMsg *pRetrieveMsg = (SRetrieveTableMsg *) pSql->cmd.payload; + + SQueryInfo *pQueryInfo = tscGetQueryInfo(&pSql->cmd); + + pRetrieveMsg->free = htons(pQueryInfo->type); + pRetrieveMsg->qId = htobe64(pSql->res.qId); + + // todo valid the vgroupId at the client side + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + + if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { + int32_t vgIndex = pTableMetaInfo->vgroupIndex; + int32_t vgId = -1; + + if (pTableMetaInfo->pVgroupTables == NULL) { + SVgroupsInfo *pVgroupInfo = pTableMetaInfo->vgroupList; + assert(pVgroupInfo->vgroups[vgIndex].vgId > 0 && vgIndex < pTableMetaInfo->vgroupList->numOfVgroups); + vgId = pVgroupInfo->vgroups[vgIndex].vgId; + } else { + int32_t numOfVgroups = (int32_t)taosArrayGetSize(pTableMetaInfo->pVgroupTables); + assert(vgIndex >= 0 && vgIndex < numOfVgroups); + + SVgroupTableInfo* pTableIdList = taosArrayGet(pTableMetaInfo->pVgroupTables, vgIndex); + vgId = pTableIdList->vgInfo.vgId; + } + + pRetrieveMsg->header.vgId = htonl(vgId); + tscDebug("0x%"PRIx64" build fetch msg from vgId:%d, vgIndex:%d, qId:0x%" PRIx64, pSql->self, vgId, vgIndex, pSql->res.qId); + } else { + STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; + pRetrieveMsg->header.vgId = htonl(pTableMeta->vgId); + tscDebug("0x%"PRIx64" build fetch msg from only one vgroup, vgId:%d, qId:0x%" PRIx64, pSql->self, pTableMeta->vgId, + pSql->res.qId); + } + + pSql->cmd.payloadLen = sizeof(SRetrieveTableMsg); + pSql->cmd.msgType = TSDB_MSG_TYPE_FETCH; + + pRetrieveMsg->header.contLen = htonl(sizeof(SRetrieveTableMsg)); + + return TSDB_CODE_SUCCESS; +} + +int tscBuildSubmitMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + SQueryInfo *pQueryInfo = tscGetQueryInfo(&pSql->cmd); + STableMeta* pTableMeta = tscGetMetaInfo(pQueryInfo, 0)->pTableMeta; + + // pSql->cmd.payloadLen is set during copying data into payload + pSql->cmd.msgType = TSDB_MSG_TYPE_SUBMIT; + + SNewVgroupInfo vgroupInfo = {0}; + taosHashGetClone(tscVgroupMap, &pTableMeta->vgId, sizeof(pTableMeta->vgId), NULL, &vgroupInfo); + tscDumpEpSetFromVgroupInfo(&pSql->epSet, &vgroupInfo); + + tscDebug("0x%"PRIx64" submit msg built, numberOfEP:%d", pSql->self, pSql->epSet.numOfEps); + + return TSDB_CODE_SUCCESS; +} + +/* + * for table query, simply return the size <= 1k + */ +static int32_t tscEstimateQueryMsgSize(SSqlObj *pSql) { + const static int32_t MIN_QUERY_MSG_PKT_SIZE = TSDB_MAX_BYTES_PER_ROW * 5; + + SSqlCmd* pCmd = &pSql->cmd; + SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); + + int32_t srcColListSize = (int32_t)(taosArrayGetSize(pQueryInfo->colList) * sizeof(SColumnInfo)); + int32_t srcColFilterSize = 0; + int32_t srcTagFilterSize = tscGetTagFilterSerializeLen(pQueryInfo); + + size_t numOfExprs = tscNumOfExprs(pQueryInfo); + int32_t exprSize = (int32_t)(sizeof(SSqlExpr) * numOfExprs * 2); + + int32_t tsBufSize = (pQueryInfo->tsBuf != NULL) ? pQueryInfo->tsBuf->fileSize : 0; + int32_t sqlLen = (int32_t) strlen(pSql->sqlstr) + 1; + + int32_t tableSerialize = 0; + STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + STableMeta * pTableMeta = pTableMetaInfo->pTableMeta; + if (pTableMetaInfo->pVgroupTables != NULL) { + size_t numOfGroups = taosArrayGetSize(pTableMetaInfo->pVgroupTables); + + int32_t totalTables = 0; + for (int32_t i = 0; i < numOfGroups; ++i) { + SVgroupTableInfo *pTableInfo = taosArrayGet(pTableMetaInfo->pVgroupTables, i); + totalTables += (int32_t) taosArrayGetSize(pTableInfo->itemList); + } + + tableSerialize = totalTables * sizeof(STableIdInfo); + } + + if (pQueryInfo->colCond && taosArrayGetSize(pQueryInfo->colCond) > 0) { + STblCond *pCond = tsGetTableFilter(pQueryInfo->colCond, pTableMeta->id.uid, 0); + if (pCond != NULL && pCond->cond != NULL) { + srcColFilterSize = pCond->len; + } + } + + SCond* pCond = &pQueryInfo->tagCond.tbnameCond; + if (pCond->len > 0) { + srcColListSize += pCond->len; + } + + return MIN_QUERY_MSG_PKT_SIZE + minMsgSize() + sizeof(SQueryTableMsg) + srcColListSize + srcColFilterSize + srcTagFilterSize + + exprSize + tsBufSize + tableSerialize + sqlLen + 4096 + pQueryInfo->bufLen; +} + +static char *doSerializeTableInfo(SQueryTableMsg *pQueryMsg, SSqlObj *pSql, STableMetaInfo *pTableMetaInfo, char *pMsg, + int32_t *succeed) { + TSKEY dfltKey = htobe64(pQueryMsg->window.skey); + + STableMeta * pTableMeta = pTableMetaInfo->pTableMeta; + if (UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo) || pTableMetaInfo->pVgroupTables == NULL) { + + int32_t vgId = -1; + if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { + int32_t index = pTableMetaInfo->vgroupIndex; + assert(index >= 0); + + SVgroupMsg* pVgroupInfo = NULL; + if (pTableMetaInfo->vgroupList && pTableMetaInfo->vgroupList->numOfVgroups > 0) { + assert(index < pTableMetaInfo->vgroupList->numOfVgroups); + pVgroupInfo = &pTableMetaInfo->vgroupList->vgroups[index]; + } else { + tscError("0x%"PRIx64" No vgroup info found", pSql->self); + + *succeed = 0; + return pMsg; + } + + vgId = pVgroupInfo->vgId; + tscSetDnodeEpSet(&pSql->epSet, pVgroupInfo); + tscDebug("0x%"PRIx64" query on stable, vgIndex:%d, numOfVgroups:%d", pSql->self, index, pTableMetaInfo->vgroupList->numOfVgroups); + } else { + vgId = pTableMeta->vgId; + + SNewVgroupInfo vgroupInfo = {0}; + taosHashGetClone(tscVgroupMap, &pTableMeta->vgId, sizeof(pTableMeta->vgId), NULL, &vgroupInfo); + tscDumpEpSetFromVgroupInfo(&pSql->epSet, &vgroupInfo); + } + + if (pSql->epSet.numOfEps > 0){ + pSql->epSet.inUse = rand()%pSql->epSet.numOfEps; + } + pQueryMsg->head.vgId = htonl(vgId); + + STableIdInfo *pTableIdInfo = (STableIdInfo *)pMsg; + pTableIdInfo->tid = htonl(pTableMeta->id.tid); + pTableIdInfo->uid = htobe64(pTableMeta->id.uid); + pTableIdInfo->key = htobe64(tscGetSubscriptionProgress(pSql->pSubscription, pTableMeta->id.uid, dfltKey)); + + pQueryMsg->numOfTables = htonl(1); // set the number of tables + pMsg += sizeof(STableIdInfo); + } else { // it is a subquery of the super table query, this EP info is acquired from vgroupInfo + int32_t index = pTableMetaInfo->vgroupIndex; + int32_t numOfVgroups = (int32_t)taosArrayGetSize(pTableMetaInfo->pVgroupTables); + assert(index >= 0 && index < numOfVgroups); + + SVgroupTableInfo* pTableIdList = taosArrayGet(pTableMetaInfo->pVgroupTables, index); + + // set the vgroup info + tscSetDnodeEpSet(&pSql->epSet, &pTableIdList->vgInfo); + pQueryMsg->head.vgId = htonl(pTableIdList->vgInfo.vgId); + + int32_t numOfTables = (int32_t)taosArrayGetSize(pTableIdList->itemList); + pQueryMsg->numOfTables = htonl(numOfTables); // set the number of tables + + tscDebug("0x%"PRIx64" query on stable, vgId:%d, numOfTables:%d, vgIndex:%d, numOfVgroups:%d", pSql->self, + pTableIdList->vgInfo.vgId, numOfTables, index, numOfVgroups); + + // serialize each table id info + for(int32_t i = 0; i < numOfTables; ++i) { + STableIdInfo* pItem = taosArrayGet(pTableIdList->itemList, i); + + STableIdInfo *pTableIdInfo = (STableIdInfo *)pMsg; + pTableIdInfo->tid = htonl(pItem->tid); + pTableIdInfo->uid = htobe64(pItem->uid); + pTableIdInfo->key = htobe64(tscGetSubscriptionProgress(pSql->pSubscription, pItem->uid, dfltKey)); + pMsg += sizeof(STableIdInfo); + } + } + + char n[TSDB_TABLE_FNAME_LEN] = {0}; + tNameExtractFullName(&pTableMetaInfo->name, n); + + tscDebug("0x%"PRIx64" vgId:%d, query on table:%s, tid:%d, uid:%" PRIu64, pSql->self, htonl(pQueryMsg->head.vgId), n, pTableMeta->id.tid, pTableMeta->id.uid); + return pMsg; +} + +// TODO refactor +static int32_t serializeColFilterInfo(SColumnFilterInfo* pColFilters, int16_t numOfFilters, char** pMsg) { + // append the filter information after the basic column information + for (int32_t f = 0; f < numOfFilters; ++f) { + SColumnFilterInfo *pColFilter = &pColFilters[f]; + + SColumnFilterInfo *pFilterMsg = (SColumnFilterInfo *)(*pMsg); + pFilterMsg->filterstr = htons(pColFilter->filterstr); + + (*pMsg) += sizeof(SColumnFilterInfo); + + if (pColFilter->filterstr) { + pFilterMsg->len = htobe64(pColFilter->len); + memcpy(*pMsg, (void *)pColFilter->pz, (size_t)(pColFilter->len + 1)); + (*pMsg) += (pColFilter->len + 1); // append the additional filter binary info + } else { + pFilterMsg->lowerBndi = htobe64(pColFilter->lowerBndi); + pFilterMsg->upperBndi = htobe64(pColFilter->upperBndi); + } + + pFilterMsg->lowerRelOptr = htons(pColFilter->lowerRelOptr); + pFilterMsg->upperRelOptr = htons(pColFilter->upperRelOptr); + + if (pColFilter->lowerRelOptr == TSDB_RELATION_INVALID && pColFilter->upperRelOptr == TSDB_RELATION_INVALID) { + tscError("invalid filter info"); + return TSDB_CODE_TSC_INVALID_OPERATION; + } + } + + return TSDB_CODE_SUCCESS; +} + +static int32_t serializeSqlExpr(SSqlExpr* pExpr, STableMetaInfo* pTableMetaInfo, char** pMsg, int64_t id, bool validateColumn) { + STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; + + // the queried table has been removed and a new table with the same name has already been created already + // return error msg + if (pExpr->uid != pTableMeta->id.uid) { + tscError("0x%"PRIx64" table has already been destroyed", id); + return TSDB_CODE_TSC_INVALID_TABLE_NAME; + } + + if (validateColumn && !tscValidateColumnId(pTableMetaInfo, pExpr->colInfo.colId, pExpr->numOfParams)) { + tscError("0x%"PRIx64" table schema is not matched with parsed sql", id); + return TSDB_CODE_TSC_INVALID_OPERATION; + } + + assert(pExpr->resColId < 0); + SSqlExpr* pSqlExpr = (SSqlExpr *)(*pMsg); + + SColIndex* pIndex = &pSqlExpr->colInfo; + + pIndex->colId = htons(pExpr->colInfo.colId); + pIndex->colIndex = htons(pExpr->colInfo.colIndex); + pIndex->flag = htons(pExpr->colInfo.flag); + pSqlExpr->uid = htobe64(pExpr->uid); + pSqlExpr->colType = htons(pExpr->colType); + pSqlExpr->colBytes = htons(pExpr->colBytes); + pSqlExpr->resType = htons(pExpr->resType); + pSqlExpr->resBytes = htons(pExpr->resBytes); + pSqlExpr->interBytes = htonl(pExpr->interBytes); + pSqlExpr->functionId = htons(pExpr->functionId); + pSqlExpr->numOfParams = htons(pExpr->numOfParams); + pSqlExpr->resColId = htons(pExpr->resColId); + pSqlExpr->flist.numOfFilters = htons(pExpr->flist.numOfFilters); + + (*pMsg) += sizeof(SSqlExpr); + for (int32_t j = 0; j < pExpr->numOfParams; ++j) { // todo add log + pSqlExpr->param[j].nType = htonl(pExpr->param[j].nType); + pSqlExpr->param[j].nLen = htonl(pExpr->param[j].nLen); + + if (pExpr->param[j].nType == TSDB_DATA_TYPE_BINARY) { + memcpy((*pMsg), pExpr->param[j].pz, pExpr->param[j].nLen); + (*pMsg) += pExpr->param[j].nLen; + } else { + pSqlExpr->param[j].i64 = htobe64(pExpr->param[j].i64); + } + } + + serializeColFilterInfo(pExpr->flist.filterInfo, pExpr->flist.numOfFilters, pMsg); + + return TSDB_CODE_SUCCESS; +} + +int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + SSqlCmd *pCmd = &pSql->cmd; + + SQueryInfo *pQueryInfo = NULL; + STableMeta *pTableMeta = NULL; + STableMetaInfo *pTableMetaInfo = NULL; + + int32_t code = TSDB_CODE_SUCCESS; + int32_t size = tscEstimateQueryMsgSize(pSql); + assert(size > 0); + + if (TSDB_CODE_SUCCESS != tscAllocPayloadFast(pCmd, size)) { + tscError("%p failed to malloc for query msg", pSql); + return TSDB_CODE_TSC_INVALID_OPERATION; // todo add test for this + } + + pQueryInfo = tscGetQueryInfo(pCmd); + pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + pTableMeta = pTableMetaInfo->pTableMeta; + + SQueryAttr query = {{0}}; + tscCreateQueryFromQueryInfo(pQueryInfo, &query, pSql); + query.vgId = pTableMeta->vgId; + + SArray* tableScanOperator = createTableScanPlan(&query); + SArray* queryOperator = createExecOperatorPlan(&query); + + SQueryTableMsg *pQueryMsg = (SQueryTableMsg *)pCmd->payload; + tstrncpy(pQueryMsg->version, version, tListLen(pQueryMsg->version)); + + int32_t numOfTags = query.numOfTags; + int32_t sqlLen = (int32_t) strlen(pSql->sqlstr); + + if (taosArrayGetSize(tableScanOperator) == 0) { + pQueryMsg->tableScanOperator = htonl(-1); + } else { + int32_t* tablescanOp = taosArrayGet(tableScanOperator, 0); + pQueryMsg->tableScanOperator = htonl(*tablescanOp); + } + + pQueryMsg->window.skey = htobe64(query.window.skey); + pQueryMsg->window.ekey = htobe64(query.window.ekey); + + pQueryMsg->order = htons(query.order.order); + pQueryMsg->orderColId = htons(query.order.orderColId); + pQueryMsg->fillType = htons(query.fillType); + pQueryMsg->limit = htobe64(query.limit.limit); + pQueryMsg->offset = htobe64(query.limit.offset); + pQueryMsg->numOfCols = htons(query.numOfCols); + + pQueryMsg->interval.interval = htobe64(query.interval.interval); + pQueryMsg->interval.sliding = htobe64(query.interval.sliding); + pQueryMsg->interval.offset = htobe64(query.interval.offset); + pQueryMsg->interval.intervalUnit = query.interval.intervalUnit; + pQueryMsg->interval.slidingUnit = query.interval.slidingUnit; + pQueryMsg->interval.offsetUnit = query.interval.offsetUnit; + + pQueryMsg->stableQuery = query.stableQuery; + pQueryMsg->topBotQuery = query.topBotQuery; + pQueryMsg->groupbyColumn = query.groupbyColumn; + pQueryMsg->hasTagResults = query.hasTagResults; + pQueryMsg->timeWindowInterpo = query.timeWindowInterpo; + pQueryMsg->queryBlockDist = query.queryBlockDist; + pQueryMsg->stabledev = query.stabledev; + pQueryMsg->tsCompQuery = query.tsCompQuery; + pQueryMsg->simpleAgg = query.simpleAgg; + pQueryMsg->pointInterpQuery = query.pointInterpQuery; + pQueryMsg->needReverseScan = query.needReverseScan; + pQueryMsg->stateWindow = query.stateWindow; + pQueryMsg->numOfTags = htonl(numOfTags); + pQueryMsg->sqlstrLen = htonl(sqlLen); + pQueryMsg->sw.gap = htobe64(query.sw.gap); + pQueryMsg->sw.primaryColId = htonl(PRIMARYKEY_TIMESTAMP_COL_INDEX); + + pQueryMsg->secondStageOutput = htonl(query.numOfExpr2); + pQueryMsg->numOfOutput = htons((int16_t)query.numOfOutput); // this is the stage one output column number + + pQueryMsg->numOfGroupCols = htons(pQueryInfo->groupbyExpr.numOfGroupCols); + pQueryMsg->tagNameRelType = htons(pQueryInfo->tagCond.relType); + pQueryMsg->tbnameCondLen = htonl(pQueryInfo->tagCond.tbnameCond.len); + pQueryMsg->queryType = htonl(pQueryInfo->type); + pQueryMsg->prevResultLen = htonl(pQueryInfo->bufLen); + + // set column list ids + size_t numOfCols = taosArrayGetSize(pQueryInfo->colList); + char *pMsg = (char *)(pQueryMsg->tableCols) + numOfCols * sizeof(SColumnInfo); + + for (int32_t i = 0; i < numOfCols; ++i) { + SColumnInfo *pCol = &query.tableCols[i]; + + pQueryMsg->tableCols[i].colId = htons(pCol->colId); + pQueryMsg->tableCols[i].bytes = htons(pCol->bytes); + pQueryMsg->tableCols[i].type = htons(pCol->type); + //pQueryMsg->tableCols[i].flist.numOfFilters = htons(pCol->flist.numOfFilters); + pQueryMsg->tableCols[i].flist.numOfFilters = 0; + pQueryMsg->tableCols[i].flist.filterInfo = 0; + // append the filter information after the basic column information + //serializeColFilterInfo(pCol->flist.filterInfo, pCol->flist.numOfFilters, &pMsg); + } + + if (pQueryInfo->colCond && taosArrayGetSize(pQueryInfo->colCond) > 0 && !onlyQueryTags(&query) ) { + STblCond *pCond = tsGetTableFilter(pQueryInfo->colCond, pTableMeta->id.uid, 0); + if (pCond != NULL && pCond->cond != NULL) { + pQueryMsg->colCondLen = htons(pCond->len); + memcpy(pMsg, pCond->cond, pCond->len); + + pMsg += pCond->len; + } + } else { + pQueryMsg->colCondLen = 0; + } + + for (int32_t i = 0; i < query.numOfOutput; ++i) { + code = serializeSqlExpr(&query.pExpr1[i].base, pTableMetaInfo, &pMsg, pSql->self, true); + if (code != TSDB_CODE_SUCCESS) { + goto _end; + } + } + + for (int32_t i = 0; i < query.numOfExpr2; ++i) { + code = serializeSqlExpr(&query.pExpr2[i].base, pTableMetaInfo, &pMsg, pSql->self, false); + if (code != TSDB_CODE_SUCCESS) { + goto _end; + } + } + + int32_t succeed = 1; + + // serialize the table info (sid, uid, tags) + pMsg = doSerializeTableInfo(pQueryMsg, pSql, pTableMetaInfo, pMsg, &succeed); + if (succeed == 0) { + code = TSDB_CODE_TSC_APP_ERROR; + goto _end; + } + + SGroupbyExpr *pGroupbyExpr = query.pGroupbyExpr; + if (pGroupbyExpr != NULL && pGroupbyExpr->numOfGroupCols > 0) { + pQueryMsg->orderByIdx = htons(pGroupbyExpr->orderIndex); + pQueryMsg->orderType = htons(pGroupbyExpr->orderType); + + for (int32_t j = 0; j < pGroupbyExpr->numOfGroupCols; ++j) { + SColIndex* pCol = taosArrayGet(pGroupbyExpr->columnInfo, j); + + *((int16_t *)pMsg) = htons(pCol->colId); + pMsg += sizeof(pCol->colId); + + *((int16_t *)pMsg) += htons(pCol->colIndex); + pMsg += sizeof(pCol->colIndex); + + *((int16_t *)pMsg) += htons(pCol->flag); + pMsg += sizeof(pCol->flag); + + memcpy(pMsg, pCol->name, tListLen(pCol->name)); + pMsg += tListLen(pCol->name); + } + } + + if (query.fillType != TSDB_FILL_NONE) { + for (int32_t i = 0; i < query.numOfOutput; ++i) { + *((int64_t *)pMsg) = htobe64(query.fillVal[i]); + pMsg += sizeof(query.fillVal[0]); + } + } + + if (query.numOfTags > 0 && query.tagColList != NULL) { + for (int32_t i = 0; i < query.numOfTags; ++i) { + SColumnInfo* pTag = &query.tagColList[i]; + + SColumnInfo* pTagCol = (SColumnInfo*) pMsg; + pTagCol->colId = htons(pTag->colId); + pTagCol->bytes = htons(pTag->bytes); + pTagCol->type = htons(pTag->type); + pTagCol->flist.numOfFilters = 0; + + pMsg += sizeof(SColumnInfo); + } + } + + // serialize tag column query condition + if (pQueryInfo->tagCond.pCond != NULL && taosArrayGetSize(pQueryInfo->tagCond.pCond) > 0) { + STagCond* pTagCond = &pQueryInfo->tagCond; + + SCond *pCond = tsGetSTableQueryCond(pTagCond, pTableMeta->id.uid); + if (pCond != NULL && pCond->cond != NULL) { + pQueryMsg->tagCondLen = htons(pCond->len); + memcpy(pMsg, pCond->cond, pCond->len); + + pMsg += pCond->len; + } + } else { + pQueryMsg->tagCondLen = 0; + } + + if (pQueryInfo->bufLen > 0) { + memcpy(pMsg, pQueryInfo->buf, pQueryInfo->bufLen); + pMsg += pQueryInfo->bufLen; + } + + SCond* pCond = &pQueryInfo->tagCond.tbnameCond; + if (pCond->len > 0) { + strncpy(pMsg, pCond->cond, pCond->len); + pMsg += pCond->len; + } + + // compressed ts block + pQueryMsg->tsBuf.tsOffset = htonl((int32_t)(pMsg - pCmd->payload)); + + if (pQueryInfo->tsBuf != NULL) { + // note: here used the index instead of actual vnode id. + int32_t vnodeIndex = pTableMetaInfo->vgroupIndex; + code = dumpFileBlockByGroupId(pQueryInfo->tsBuf, vnodeIndex, pMsg, &pQueryMsg->tsBuf.tsLen, &pQueryMsg->tsBuf.tsNumOfBlocks); + if (code != TSDB_CODE_SUCCESS) { + goto _end; + } + + pMsg += pQueryMsg->tsBuf.tsLen; + + pQueryMsg->tsBuf.tsOrder = htonl(pQueryInfo->tsBuf->tsOrder); + pQueryMsg->tsBuf.tsLen = htonl(pQueryMsg->tsBuf.tsLen); + pQueryMsg->tsBuf.tsNumOfBlocks = htonl(pQueryMsg->tsBuf.tsNumOfBlocks); + } else { + pQueryMsg->tsBuf.tsLen = 0; + pQueryMsg->tsBuf.tsNumOfBlocks = 0; + } + + int32_t numOfOperator = (int32_t) taosArrayGetSize(queryOperator); + pQueryMsg->numOfOperator = htonl(numOfOperator); + for(int32_t i = 0; i < numOfOperator; ++i) { + int32_t *operator = taosArrayGet(queryOperator, i); + *(int32_t*)pMsg = htonl(*operator); + + pMsg += sizeof(int32_t); + } + + // support only one udf + if (pQueryInfo->pUdfInfo != NULL && taosArrayGetSize(pQueryInfo->pUdfInfo) > 0) { + pQueryMsg->udfContentOffset = htonl((int32_t) (pMsg - pCmd->payload)); + for(int32_t i = 0; i < taosArrayGetSize(pQueryInfo->pUdfInfo); ++i) { + SUdfInfo* pUdfInfo = taosArrayGet(pQueryInfo->pUdfInfo, i); + *(int8_t*) pMsg = pUdfInfo->resType; + pMsg += sizeof(pUdfInfo->resType); + + *(int16_t*) pMsg = htons(pUdfInfo->resBytes); + pMsg += sizeof(pUdfInfo->resBytes); + + STR_TO_VARSTR(pMsg, pUdfInfo->name); + + pMsg += varDataTLen(pMsg); + + *(int32_t*) pMsg = htonl(pUdfInfo->funcType); + pMsg += sizeof(pUdfInfo->funcType); + + *(int32_t*) pMsg = htonl(pUdfInfo->bufSize); + pMsg += sizeof(pUdfInfo->bufSize); + + pQueryMsg->udfContentLen = htonl(pUdfInfo->contLen); + memcpy(pMsg, pUdfInfo->content, pUdfInfo->contLen); + + pMsg += pUdfInfo->contLen; + } + } else { + pQueryMsg->udfContentOffset = 0; + pQueryMsg->udfContentLen = 0; + } + + memcpy(pMsg, pSql->sqlstr, sqlLen); + pMsg += sqlLen; + + int32_t msgLen = (int32_t)(pMsg - pCmd->payload); + + tscDebug("0x%"PRIx64" msg built success, len:%d bytes", pSql->self, msgLen); + pCmd->payloadLen = msgLen; + pSql->cmd.msgType = TSDB_MSG_TYPE_QUERY; + + pQueryMsg->head.contLen = htonl(msgLen); + assert(msgLen + minMsgSize() <= (int32_t)pCmd->allocSize); + + _end: + freeQueryAttr(&query); + taosArrayDestroy(tableScanOperator); + taosArrayDestroy(queryOperator); + return code; +} + +int32_t tscBuildCreateDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + SSqlCmd *pCmd = &pSql->cmd; + pCmd->payloadLen = sizeof(SCreateDbMsg); + + pCmd->msgType = (pInfo->pMiscInfo->dbOpt.dbType == TSDB_DB_TYPE_DEFAULT) ? TSDB_MSG_TYPE_CREATE_DB : TSDB_MSG_TYPE_CREATE_TP; + + SCreateDbMsg *pCreateDbMsg = (SCreateDbMsg *)pCmd->payload; + +// assert(pCmd->numOfClause == 1); + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); + int32_t code = tNameExtractFullName(&pTableMetaInfo->name, pCreateDbMsg->db); + assert(code == TSDB_CODE_SUCCESS); + + return TSDB_CODE_SUCCESS; +} + +int32_t tscBuildCreateFuncMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + SSqlCmd *pCmd = &pSql->cmd; + SCreateFuncMsg *pCreateFuncMsg = (SCreateFuncMsg *)pCmd->payload; + + pCmd->msgType = TSDB_MSG_TYPE_CREATE_FUNCTION; + + pCmd->payloadLen = sizeof(SCreateFuncMsg) + htonl(pCreateFuncMsg->codeLen); + + return TSDB_CODE_SUCCESS; +} + +int32_t tscBuildCreateDnodeMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + SSqlCmd *pCmd = &pSql->cmd; + pCmd->payloadLen = sizeof(SCreateDnodeMsg); + if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { + tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + SCreateDnodeMsg *pCreate = (SCreateDnodeMsg *)pCmd->payload; + + SStrToken* t0 = taosArrayGet(pInfo->pMiscInfo->a, 0); + strncpy(pCreate->ep, t0->z, t0->n); + + pCmd->msgType = TSDB_MSG_TYPE_CREATE_DNODE; + + return TSDB_CODE_SUCCESS; +} + +int32_t tscBuildAcctMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + SSqlCmd *pCmd = &pSql->cmd; + pCmd->payloadLen = sizeof(SCreateAcctMsg); + if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { + tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + SCreateAcctMsg *pAlterMsg = (SCreateAcctMsg *)pCmd->payload; + + SStrToken *pName = &pInfo->pMiscInfo->user.user; + SStrToken *pPwd = &pInfo->pMiscInfo->user.passwd; + + strncpy(pAlterMsg->user, pName->z, pName->n); + strncpy(pAlterMsg->pass, pPwd->z, pPwd->n); + + SCreateAcctInfo *pAcctOpt = &pInfo->pMiscInfo->acctOpt; + + pAlterMsg->cfg.maxUsers = htonl(pAcctOpt->maxUsers); + pAlterMsg->cfg.maxDbs = htonl(pAcctOpt->maxDbs); + pAlterMsg->cfg.maxTimeSeries = htonl(pAcctOpt->maxTimeSeries); + pAlterMsg->cfg.maxStreams = htonl(pAcctOpt->maxStreams); + pAlterMsg->cfg.maxPointsPerSecond = htonl(pAcctOpt->maxPointsPerSecond); + pAlterMsg->cfg.maxStorage = htobe64(pAcctOpt->maxStorage); + pAlterMsg->cfg.maxQueryTime = htobe64(pAcctOpt->maxQueryTime); + pAlterMsg->cfg.maxConnections = htonl(pAcctOpt->maxConnections); + + if (pAcctOpt->stat.n == 0) { + pAlterMsg->cfg.accessState = -1; + } else { + if (pAcctOpt->stat.z[0] == 'r' && pAcctOpt->stat.n == 1) { + pAlterMsg->cfg.accessState = TSDB_VN_READ_ACCCESS; + } else if (pAcctOpt->stat.z[0] == 'w' && pAcctOpt->stat.n == 1) { + pAlterMsg->cfg.accessState = TSDB_VN_WRITE_ACCCESS; + } else if (strncmp(pAcctOpt->stat.z, "all", 3) == 0 && pAcctOpt->stat.n == 3) { + pAlterMsg->cfg.accessState = TSDB_VN_ALL_ACCCESS; + } else if (strncmp(pAcctOpt->stat.z, "no", 2) == 0 && pAcctOpt->stat.n == 2) { + pAlterMsg->cfg.accessState = 0; + } + } + + pCmd->msgType = TSDB_MSG_TYPE_CREATE_ACCT; + return TSDB_CODE_SUCCESS; +} + +int32_t tscBuildUserMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + SSqlCmd *pCmd = &pSql->cmd; + pCmd->payloadLen = sizeof(SCreateUserMsg); + + if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { + tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + SCreateUserMsg *pAlterMsg = (SCreateUserMsg *)pCmd->payload; + + SUserInfo *pUser = &pInfo->pMiscInfo->user; + strncpy(pAlterMsg->user, pUser->user.z, pUser->user.n); + pAlterMsg->flag = (int8_t)pUser->type; + + if (pUser->type == TSDB_ALTER_USER_PRIVILEGES) { + pAlterMsg->privilege = (char)pCmd->count; + } else if (pUser->type == TSDB_ALTER_USER_PASSWD) { + strncpy(pAlterMsg->pass, pUser->passwd.z, pUser->passwd.n); + } else { // create user password info + strncpy(pAlterMsg->pass, pUser->passwd.z, pUser->passwd.n); + } + + if (pUser->type == TSDB_ALTER_USER_PASSWD || pUser->type == TSDB_ALTER_USER_PRIVILEGES) { + pCmd->msgType = TSDB_MSG_TYPE_ALTER_USER; + } else { + pCmd->msgType = TSDB_MSG_TYPE_CREATE_USER; + } + + return TSDB_CODE_SUCCESS; +} + +int32_t tscBuildCfgDnodeMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + SSqlCmd *pCmd = &pSql->cmd; + pCmd->payloadLen = sizeof(SCfgDnodeMsg); + pCmd->msgType = TSDB_MSG_TYPE_CONFIG_DNODE; + return TSDB_CODE_SUCCESS; +} + +int32_t tscBuildDropDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + SSqlCmd *pCmd = &pSql->cmd; + pCmd->payloadLen = sizeof(SDropDbMsg); + + if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { + tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + SDropDbMsg *pDropDbMsg = (SDropDbMsg*)pCmd->payload; + + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); + + int32_t code = tNameExtractFullName(&pTableMetaInfo->name, pDropDbMsg->db); + assert(code == TSDB_CODE_SUCCESS && pTableMetaInfo->name.type == TSDB_DB_NAME_T); + + pDropDbMsg->ignoreNotExists = pInfo->pMiscInfo->existsCheck ? 1 : 0; + + pCmd->msgType = (pInfo->pMiscInfo->dbType == TSDB_DB_TYPE_DEFAULT) ? TSDB_MSG_TYPE_DROP_DB : TSDB_MSG_TYPE_DROP_TP; + return TSDB_CODE_SUCCESS; +} + +int32_t tscBuildDropFuncMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + SSqlCmd *pCmd = &pSql->cmd; + + pCmd->msgType = TSDB_MSG_TYPE_DROP_FUNCTION; + + pCmd->payloadLen = sizeof(SDropFuncMsg); + + return TSDB_CODE_SUCCESS; +} + + +int32_t tscBuildDropTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + SSqlCmd *pCmd = &pSql->cmd; + pCmd->payloadLen = sizeof(SCMDropTableMsg); + + if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { + tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + SCMDropTableMsg *pDropTableMsg = (SCMDropTableMsg*)pCmd->payload; + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); + tNameExtractFullName(&pTableMetaInfo->name, pDropTableMsg->name); + + pDropTableMsg->supertable = (pInfo->pMiscInfo->tableType == TSDB_SUPER_TABLE)? 1:0; + pDropTableMsg->igNotExists = pInfo->pMiscInfo->existsCheck ? 1 : 0; + pCmd->msgType = TSDB_MSG_TYPE_DROP_TABLE; + return TSDB_CODE_SUCCESS; +} + +int32_t tscBuildDropDnodeMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + SSqlCmd *pCmd = &pSql->cmd; + + char dnodeEp[TSDB_EP_LEN] = {0}; + tstrncpy(dnodeEp, pCmd->payload, TSDB_EP_LEN); + + pCmd->payloadLen = sizeof(SDropDnodeMsg); + if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { + tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + SDropDnodeMsg * pDrop = (SDropDnodeMsg *)pCmd->payload; + tstrncpy(pDrop->ep, dnodeEp, tListLen(pDrop->ep)); + pCmd->msgType = TSDB_MSG_TYPE_DROP_DNODE; + + return TSDB_CODE_SUCCESS; +} + +int32_t tscBuildDropUserAcctMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + SSqlCmd *pCmd = &pSql->cmd; + + char user[TSDB_USER_LEN] = {0}; + tstrncpy(user, pCmd->payload, TSDB_USER_LEN); + + pCmd->payloadLen = sizeof(SDropUserMsg); + pCmd->msgType = (pInfo->type == TSDB_SQL_DROP_USER)? TSDB_MSG_TYPE_DROP_USER:TSDB_MSG_TYPE_DROP_ACCT; + + if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { + tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + SDropUserMsg *pDropMsg = (SDropUserMsg *)pCmd->payload; + tstrncpy(pDropMsg->user, user, tListLen(user)); + + return TSDB_CODE_SUCCESS; +} + +int32_t tscBuildUseDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + SSqlCmd *pCmd = &pSql->cmd; + pCmd->payloadLen = sizeof(SUseDbMsg); + + if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { + tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + SUseDbMsg *pUseDbMsg = (SUseDbMsg *)pCmd->payload; + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); + tNameExtractFullName(&pTableMetaInfo->name, pUseDbMsg->db); + pCmd->msgType = TSDB_MSG_TYPE_USE_DB; + + return TSDB_CODE_SUCCESS; +} + +int32_t tscBuildSyncDbReplicaMsg(SSqlObj* pSql, SSqlInfo *pInfo) { + SSqlCmd *pCmd = &pSql->cmd; + pCmd->payloadLen = sizeof(SSyncDbMsg); + + if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { + tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + SSyncDbMsg *pSyncMsg = (SSyncDbMsg *)pCmd->payload; + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); + tNameExtractFullName(&pTableMetaInfo->name, pSyncMsg->db); + pCmd->msgType = TSDB_MSG_TYPE_SYNC_DB; + + return TSDB_CODE_SUCCESS; +} + +int32_t tscBuildShowMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + SSqlCmd *pCmd = &pSql->cmd; + pCmd->msgType = TSDB_MSG_TYPE_SHOW; + pCmd->payloadLen = sizeof(SShowMsg) + 100; + + if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { + tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + SShowInfo *pShowInfo = &pInfo->pMiscInfo->showOpt; + SShowMsg *pShowMsg = (SShowMsg *)pCmd->payload; + + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); + if (pShowInfo->showType == TSDB_MGMT_TABLE_FUNCTION) { + pShowMsg->type = pShowInfo->showType; + pShowMsg->payloadLen = 0; + pCmd->payloadLen = sizeof(SShowMsg); + + return TSDB_CODE_SUCCESS; + } + + if (tNameIsEmpty(&pTableMetaInfo->name)) { + char *p = cloneCurrentDBName(pSql); + tstrncpy(pShowMsg->db, p, sizeof(pShowMsg->db)); + tfree(p); + } else { + tNameGetFullDbName(&pTableMetaInfo->name, pShowMsg->db); + } + + pShowMsg->type = pShowInfo->showType; + + if (pShowInfo->showType != TSDB_MGMT_TABLE_VNODES) { + SStrToken *pPattern = &pShowInfo->pattern; + if (pPattern->type > 0) { // only show tables support wildcard query + strncpy(pShowMsg->payload, pPattern->z, pPattern->n); + pShowMsg->payloadLen = htons(pPattern->n); + } + } else { + SStrToken *pEpAddr = &pShowInfo->prefix; + assert(pEpAddr->n > 0 && pEpAddr->type > 0); + + strncpy(pShowMsg->payload, pEpAddr->z, pEpAddr->n); + pShowMsg->payloadLen = htons(pEpAddr->n); + } + + pCmd->payloadLen = sizeof(SShowMsg) + htons(pShowMsg->payloadLen); + return TSDB_CODE_SUCCESS; +} + +int32_t tscBuildKillMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + SSqlCmd *pCmd = &pSql->cmd; + pCmd->payloadLen = sizeof(SKillQueryMsg); + + switch (pCmd->command) { + case TSDB_SQL_KILL_QUERY: + pCmd->msgType = TSDB_MSG_TYPE_KILL_QUERY; + break; + case TSDB_SQL_KILL_CONNECTION: + pCmd->msgType = TSDB_MSG_TYPE_KILL_CONN; + break; + case TSDB_SQL_KILL_STREAM: + pCmd->msgType = TSDB_MSG_TYPE_KILL_STREAM; + break; + } + return TSDB_CODE_SUCCESS; +} + +int tscEstimateCreateTableMsgLength(SSqlObj *pSql, SSqlInfo *pInfo) { + SSqlCmd *pCmd = &(pSql->cmd); + int32_t size = minMsgSize() + sizeof(SCMCreateTableMsg) + sizeof(SCreateTableMsg); + + SCreateTableSql *pCreateTableInfo = pInfo->pCreateTableInfo; + if (pCreateTableInfo->type == TSQL_CREATE_TABLE_FROM_STABLE) { + int32_t numOfTables = (int32_t)taosArrayGetSize(pInfo->pCreateTableInfo->childTableInfo); + size += numOfTables * (sizeof(SCreateTableMsg) + TSDB_MAX_TAGS_LEN); + } else { + size += sizeof(SSchema) * (pCmd->numOfCols + pCmd->count); + } + + if (pCreateTableInfo->pSelect != NULL) { + size += (pCreateTableInfo->pSelect->sqlstr.n + 1); + } + + return size + TSDB_EXTRA_PAYLOAD_SIZE; +} + +int tscBuildCreateTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + int msgLen = 0; + int size = 0; + SSchema *pSchema; + SSqlCmd *pCmd = &pSql->cmd; + + SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); + STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + + // Reallocate the payload size + size = tscEstimateCreateTableMsgLength(pSql, pInfo); + if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, size)) { + tscError("0x%"PRIx64" failed to malloc for create table msg", pSql->self); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + SCMCreateTableMsg *pCreateTableMsg = (SCMCreateTableMsg *)pCmd->payload; + + SCreateTableMsg* pCreateMsg = (SCreateTableMsg*)((char*) pCreateTableMsg + sizeof(SCMCreateTableMsg)); + char* pMsg = NULL; + + int8_t type = pInfo->pCreateTableInfo->type; + if (type == TSQL_CREATE_TABLE_FROM_STABLE) { // create by using super table, tags value + SArray* list = pInfo->pCreateTableInfo->childTableInfo; + + int32_t numOfTables = (int32_t) taosArrayGetSize(list); + pCreateTableMsg->numOfTables = htonl(numOfTables); + + pMsg = (char*) pCreateMsg; + for(int32_t i = 0; i < numOfTables; ++i) { + SCreateTableMsg* pCreate = (SCreateTableMsg*) pMsg; + + pCreate->numOfColumns = htons(pCmd->numOfCols); + pCreate->numOfTags = htons(pCmd->count); + pMsg += sizeof(SCreateTableMsg); + + SCreatedTableInfo* p = taosArrayGet(list, i); + strcpy(pCreate->tableName, p->fullname); + pCreate->igExists = (p->igExist)? 1 : 0; + + // use dbinfo from table id without modifying current db info + pMsg = serializeTagData(&p->tagdata, pMsg); + + int32_t len = (int32_t)(pMsg - (char*) pCreate); + pCreate->len = htonl(len); + } + } else { // create (super) table + pCreateTableMsg->numOfTables = htonl(1); // only one table will be created + + int32_t code = tNameExtractFullName(&pTableMetaInfo->name, pCreateMsg->tableName); + assert(code == 0); + + SCreateTableSql *pCreateTable = pInfo->pCreateTableInfo; + + pCreateMsg->igExists = pCreateTable->existCheck ? 1 : 0; + pCreateMsg->numOfColumns = htons(pCmd->numOfCols); + pCreateMsg->numOfTags = htons(pCmd->count); + + pCreateMsg->sqlLen = 0; + pMsg = (char *)pCreateMsg->schema; + + pSchema = (SSchema *)pCreateMsg->schema; + + for (int i = 0; i < pCmd->numOfCols + pCmd->count; ++i) { + TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); + + pSchema->type = pField->type; + strcpy(pSchema->name, pField->name); + pSchema->bytes = htons(pField->bytes); + + pSchema++; + } + + pMsg = (char *)pSchema; + if (type == TSQL_CREATE_STREAM) { // check if it is a stream sql + SSqlNode *pQuerySql = pInfo->pCreateTableInfo->pSelect; + + strncpy(pMsg, pQuerySql->sqlstr.z, pQuerySql->sqlstr.n + 1); + pCreateMsg->sqlLen = htons(pQuerySql->sqlstr.n + 1); + pMsg += pQuerySql->sqlstr.n + 1; + } + } + + tscFieldInfoClear(&pQueryInfo->fieldsInfo); + + msgLen = (int32_t)(pMsg - (char*)pCreateTableMsg); + pCreateTableMsg->contLen = htonl(msgLen); + pCmd->payloadLen = msgLen; + pCmd->msgType = TSDB_MSG_TYPE_CREATE_TABLE; + + assert(msgLen + minMsgSize() <= size); + return TSDB_CODE_SUCCESS; +} + +int tscEstimateAlterTableMsgLength(SSqlCmd *pCmd) { + SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); + return minMsgSize() + sizeof(SAlterTableMsg) + sizeof(SSchema) * tscNumOfFields(pQueryInfo) + TSDB_EXTRA_PAYLOAD_SIZE; +} + +int tscBuildAlterTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + char *pMsg; + int msgLen = 0; + + SSqlCmd *pCmd = &pSql->cmd; + SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); + + STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + + SAlterTableInfo *pAlterInfo = pInfo->pAlterInfo; + int size = tscEstimateAlterTableMsgLength(pCmd); + if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, size)) { + tscError("0x%"PRIx64" failed to malloc for alter table msg", pSql->self); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + SAlterTableMsg *pAlterTableMsg = (SAlterTableMsg *)pCmd->payload; + + tNameExtractFullName(&pTableMetaInfo->name, pAlterTableMsg->tableFname); + pAlterTableMsg->type = htons(pAlterInfo->type); + + pAlterTableMsg->numOfCols = htons(tscNumOfFields(pQueryInfo)); + SSchema *pSchema = pAlterTableMsg->schema; + for (int i = 0; i < tscNumOfFields(pQueryInfo); ++i) { + TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); + + pSchema->type = pField->type; + strcpy(pSchema->name, pField->name); + pSchema->bytes = htons(pField->bytes); + pSchema++; + } + + pMsg = (char *)pSchema; + pAlterTableMsg->tagValLen = htonl(pAlterInfo->tagData.dataLen); + if (pAlterInfo->tagData.dataLen > 0) { + memcpy(pMsg, pAlterInfo->tagData.data, pAlterInfo->tagData.dataLen); + } + pMsg += pAlterInfo->tagData.dataLen; + + msgLen = (int32_t)(pMsg - (char*)pAlterTableMsg); + + pCmd->payloadLen = msgLen; + pCmd->msgType = TSDB_MSG_TYPE_ALTER_TABLE; + + assert(msgLen + minMsgSize() <= size); + + return TSDB_CODE_SUCCESS; +} + +int tscBuildUpdateTagMsg(SSqlObj* pSql, SSqlInfo *pInfo) { + SSqlCmd* pCmd = &pSql->cmd; + pCmd->msgType = TSDB_MSG_TYPE_UPDATE_TAG_VAL; + + SUpdateTableTagValMsg* pUpdateMsg = (SUpdateTableTagValMsg*) pCmd->payload; + pCmd->payloadLen = htonl(pUpdateMsg->head.contLen); + + SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); + STableMeta *pTableMeta = tscGetMetaInfo(pQueryInfo, 0)->pTableMeta; + + SNewVgroupInfo vgroupInfo = {.vgId = -1}; + taosHashGetClone(tscVgroupMap, &pTableMeta->vgId, sizeof(pTableMeta->vgId), NULL, &vgroupInfo); + assert(vgroupInfo.vgId > 0); + + tscDumpEpSetFromVgroupInfo(&pSql->epSet, &vgroupInfo); + + return TSDB_CODE_SUCCESS; +} + +int tscAlterDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + SSqlCmd *pCmd = &pSql->cmd; + pCmd->payloadLen = sizeof(SAlterDbMsg); + pCmd->msgType = (pInfo->pMiscInfo->dbOpt.dbType == TSDB_DB_TYPE_DEFAULT) ? TSDB_MSG_TYPE_ALTER_DB : TSDB_MSG_TYPE_ALTER_TP; + + SAlterDbMsg *pAlterDbMsg = (SAlterDbMsg* )pCmd->payload; + pAlterDbMsg->dbType = -1; + + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); + tNameExtractFullName(&pTableMetaInfo->name, pAlterDbMsg->db); + + return TSDB_CODE_SUCCESS; +} +int tscBuildCompactMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + if (pInfo->list == NULL || taosArrayGetSize(pInfo->list) <= 0) { + return TSDB_CODE_TSC_INVALID_OPERATION; + } + STscObj *pObj = pSql->pTscObj; + SSqlCmd *pCmd = &pSql->cmd; + SArray *pList = pInfo->list; + int32_t size = (int32_t)taosArrayGetSize(pList); + + int32_t *result = malloc(sizeof(int32_t) * size); + if (result == NULL) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + for (int32_t i = 0; i < size; i++) { + tSqlExprItem* pSub = taosArrayGet(pList, i); + tVariant* pVar = &pSub->pNode->value; + if (pVar->nType >= TSDB_DATA_TYPE_TINYINT && pVar->nType <= TSDB_DATA_TYPE_BIGINT) { + result[i] = (int32_t)(pVar->i64); + } else { + free(result); + return TSDB_CODE_TSC_INVALID_OPERATION; + } + } + + int count = removeDupVgid(result, size); + pCmd->payloadLen = sizeof(SCompactMsg) + count * sizeof(int32_t); + pCmd->msgType = TSDB_MSG_TYPE_COMPACT_VNODE; + + if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { + tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); + free(result); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + SCompactMsg *pCompactMsg = (SCompactMsg *)pCmd->payload; + + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); + + if (tNameIsEmpty(&pTableMetaInfo->name)) { + pthread_mutex_lock(&pObj->mutex); + tstrncpy(pCompactMsg->db, pObj->db, sizeof(pCompactMsg->db)); + pthread_mutex_unlock(&pObj->mutex); + } else { + tNameGetFullDbName(&pTableMetaInfo->name, pCompactMsg->db); + } + + pCompactMsg->numOfVgroup = htons(count); + for (int32_t i = 0; i < count; i++) { + pCompactMsg->vgid[i] = htons(result[i]); + } + free(result); + + return TSDB_CODE_SUCCESS; +} + +int tscBuildRetrieveFromMgmtMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + SSqlCmd *pCmd = &pSql->cmd; + pCmd->msgType = TSDB_MSG_TYPE_SHOW_RETRIEVE; + pCmd->payloadLen = sizeof(SRetrieveTableMsg); + + if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { + tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); + SRetrieveTableMsg *pRetrieveMsg = (SRetrieveTableMsg*)pCmd->payload; + pRetrieveMsg->qId = htobe64(pSql->res.qId); + pRetrieveMsg->free = htons(pQueryInfo->type); + + return TSDB_CODE_SUCCESS; +} + +/* + * this function can only be called once. + * by using pRes->rspType to denote its status + * + * if pRes->rspType is 1, no more result + */ +static int tscLocalResultCommonBuilder(SSqlObj *pSql, int32_t numOfRes) { + SSqlRes *pRes = &pSql->res; + SSqlCmd *pCmd = &pSql->cmd; + + pRes->code = TSDB_CODE_SUCCESS; + if (pRes->rspType == 0) { + pRes->numOfRows = numOfRes; + pRes->row = 0; + pRes->rspType = 1; + + SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); + if (tscCreateResPointerInfo(pRes, pQueryInfo) != TSDB_CODE_SUCCESS) { + return pRes->code; + } + + tscSetResRawPtr(pRes, pQueryInfo); + } else { + tscResetForNextRetrieve(pRes); + } + + uint8_t code = pSql->res.code; + if (pSql->fp) { + if (code == TSDB_CODE_SUCCESS) { + (*pSql->fp)(pSql->param, pSql, pSql->res.numOfRows); + } else { + tscAsyncResultOnError(pSql); + } + } + + return code; +} + +int tscProcessDescribeTableRsp(SSqlObj *pSql) { + SSqlCmd * pCmd = &pSql->cmd; + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); + + STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); + + int32_t numOfRes = tinfo.numOfColumns + tinfo.numOfTags; + return tscLocalResultCommonBuilder(pSql, numOfRes); +} + +int tscProcessLocalRetrieveRsp(SSqlObj *pSql) { + int32_t numOfRes = 1; + pSql->res.completed = true; + return tscLocalResultCommonBuilder(pSql, numOfRes); +} + +int tscProcessRetrieveGlobalMergeRsp(SSqlObj *pSql) { + SSqlRes *pRes = &pSql->res; + SSqlCmd* pCmd = &pSql->cmd; + + int32_t code = pRes->code; + if (pRes->code != TSDB_CODE_SUCCESS) { + tscAsyncResultOnError(pSql); + return code; + } + + if (pRes->pMerger == NULL) { // no result from subquery, so abort here directly. + (*pSql->fp)(pSql->param, pSql, pRes->numOfRows); + return code; + } + + // global aggregation may be the upstream for parent query + SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); + if (pQueryInfo->pQInfo == NULL) { + STableGroupInfo tableGroupInfo = {.numOfTables = 1, .pGroupList = taosArrayInit(1, POINTER_BYTES),}; + tableGroupInfo.map = taosHashInit(1, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); + + STableKeyInfo tableKeyInfo = {.pTable = NULL, .lastKey = INT64_MIN}; + + SArray* group = taosArrayInit(1, sizeof(STableKeyInfo)); + taosArrayPush(group, &tableKeyInfo); + taosArrayPush(tableGroupInfo.pGroupList, &group); + + tscDebug("0x%"PRIx64" create QInfo 0x%"PRIx64" to execute query processing", pSql->self, pSql->self); + pQueryInfo->pQInfo = createQInfoFromQueryNode(pQueryInfo, &tableGroupInfo, NULL, NULL, pRes->pMerger, MERGE_STAGE, pSql->self); + } + + uint64_t localQueryId = pSql->self; + qTableQuery(pQueryInfo->pQInfo, &localQueryId); + convertQueryResult(pRes, pQueryInfo, pSql->self, true); + + code = pRes->code; + if (pRes->code == TSDB_CODE_SUCCESS) { + (*pSql->fp)(pSql->param, pSql, pRes->numOfRows); + } else { + tscAsyncResultOnError(pSql); + } + + return code; +} + +int tscProcessEmptyResultRsp(SSqlObj *pSql) { return tscLocalResultCommonBuilder(pSql, 0); } + +int tscBuildConnectMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + STscObj *pObj = pSql->pTscObj; + SSqlCmd *pCmd = &pSql->cmd; + pCmd->msgType = TSDB_MSG_TYPE_CONNECT; + pCmd->payloadLen = sizeof(SConnectMsg); + + if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { + tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + SConnectMsg *pConnect = (SConnectMsg*)pCmd->payload; + + // TODO refactor full_name + char *db; // ugly code to move the space + + pthread_mutex_lock(&pObj->mutex); + db = strstr(pObj->db, TS_PATH_DELIMITER); + + db = (db == NULL) ? pObj->db : db + 1; + tstrncpy(pConnect->db, db, sizeof(pConnect->db)); + pthread_mutex_unlock(&pObj->mutex); + + tstrncpy(pConnect->clientVersion, version, sizeof(pConnect->clientVersion)); + tstrncpy(pConnect->msgVersion, "", sizeof(pConnect->msgVersion)); + + pConnect->pid = htonl(taosGetPId()); + taosGetCurrentAPPName(pConnect->appName, NULL); + + return TSDB_CODE_SUCCESS; +} + +/** + * multi table meta req pkg format: + * |SMultiTableInfoMsg | tableId0 | tableId1 | tableId2 | ...... + * 4B + **/ +int tscBuildMultiTableMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + SSqlCmd *pCmd = &pSql->cmd; + + pCmd->msgType = TSDB_MSG_TYPE_TABLES_META; + assert(pCmd->payloadLen + minMsgSize() <= pCmd->allocSize); + + tscDebug("0x%"PRIx64" build load multi-tablemeta msg completed, numOfTables:%d, msg size:%d", pSql->self, pCmd->count, + pCmd->payloadLen); + + return pCmd->payloadLen; +} + +int tscBuildSTableVgroupMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + SSqlCmd *pCmd = &pSql->cmd; + + char* pMsg = pCmd->payload; + SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); + + SSTableVgroupMsg *pStableVgroupMsg = (SSTableVgroupMsg *)pMsg; + pStableVgroupMsg->numOfTables = htonl(pQueryInfo->numOfTables); + pMsg += sizeof(SSTableVgroupMsg); + + for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, i); + int32_t code = tNameExtractFullName(&pTableMetaInfo->name, pMsg); + assert(code == TSDB_CODE_SUCCESS); + + pMsg += TSDB_TABLE_FNAME_LEN; + } + + pCmd->msgType = TSDB_MSG_TYPE_STABLE_VGROUP; + pCmd->payloadLen = (int32_t)(pMsg - pCmd->payload); + + return TSDB_CODE_SUCCESS; +} + +int tscBuildRetrieveFuncMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + SSqlCmd *pCmd = &pSql->cmd; + + char *pMsg = pCmd->payload; + SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); + int32_t numOfFuncs = (int32_t)taosArrayGetSize(pQueryInfo->pUdfInfo); + + SRetrieveFuncMsg *pRetrieveFuncMsg = (SRetrieveFuncMsg *)pMsg; + pRetrieveFuncMsg->num = htonl(numOfFuncs); + + pMsg += sizeof(SRetrieveFuncMsg); + for(int32_t i = 0; i < numOfFuncs; ++i) { + SUdfInfo* pUdf = taosArrayGet(pQueryInfo->pUdfInfo, i); + STR_TO_NET_VARSTR(pMsg, pUdf->name); + pMsg += varDataNetTLen(pMsg); + } + + pCmd->msgType = TSDB_MSG_TYPE_RETRIEVE_FUNC; + pCmd->payloadLen = (int32_t)(pMsg - pCmd->payload); + + return TSDB_CODE_SUCCESS; +} + +int tscBuildHeartBeatMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + SSqlCmd *pCmd = &pSql->cmd; + STscObj *pObj = pSql->pTscObj; + + pthread_mutex_lock(&pObj->mutex); + + int32_t numOfQueries = 2; + SSqlObj *tpSql = pObj->sqlList; + while (tpSql) { + tpSql = tpSql->next; + numOfQueries++; + } + + int32_t numOfStreams = 2; + SSqlStream *pStream = pObj->streamList; + while (pStream) { + pStream = pStream->next; + numOfStreams++; + } + + int size = numOfQueries * sizeof(SQueryDesc) + numOfStreams * sizeof(SStreamDesc) + sizeof(SHeartBeatMsg) + 100; + if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, size)) { + pthread_mutex_unlock(&pObj->mutex); + tscError("0x%"PRIx64" failed to create heartbeat msg", pSql->self); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + // TODO the expired hb and client can not be identified by server till now. + SHeartBeatMsg *pHeartbeat = (SHeartBeatMsg *)pCmd->payload; + tstrncpy(pHeartbeat->clientVer, version, tListLen(pHeartbeat->clientVer)); + + pHeartbeat->numOfQueries = numOfQueries; + pHeartbeat->numOfStreams = numOfStreams; + + pHeartbeat->pid = htonl(taosGetPId()); + taosGetCurrentAPPName(pHeartbeat->appName, NULL); + + int msgLen = tscBuildQueryStreamDesc(pHeartbeat, pObj); + + pthread_mutex_unlock(&pObj->mutex); + + pCmd->payloadLen = msgLen; + pCmd->msgType = TSDB_MSG_TYPE_HEARTBEAT; + + return TSDB_CODE_SUCCESS; +} + +static int32_t tableMetaMsgConvert(STableMetaMsg* pMetaMsg) { + pMetaMsg->tid = htonl(pMetaMsg->tid); + pMetaMsg->sversion = htons(pMetaMsg->sversion); + pMetaMsg->tversion = htons(pMetaMsg->tversion); + pMetaMsg->vgroup.vgId = htonl(pMetaMsg->vgroup.vgId); + + pMetaMsg->uid = htobe64(pMetaMsg->uid); + pMetaMsg->numOfColumns = htons(pMetaMsg->numOfColumns); + + if ((pMetaMsg->tableType != TSDB_SUPER_TABLE) && + (pMetaMsg->tid <= 0 || pMetaMsg->vgroup.vgId < 2 || pMetaMsg->vgroup.numOfEps <= 0)) { + tscError("invalid value in table numOfEps:%d, vgId:%d tid:%d, name:%s", pMetaMsg->vgroup.numOfEps, pMetaMsg->vgroup.vgId, + pMetaMsg->tid, pMetaMsg->tableFname); + return TSDB_CODE_TSC_INVALID_VALUE; + } + + if (pMetaMsg->numOfTags > TSDB_MAX_TAGS) { + tscError("invalid numOfTags:%d", pMetaMsg->numOfTags); + return TSDB_CODE_TSC_INVALID_VALUE; + } + + if (pMetaMsg->numOfColumns > TSDB_MAX_COLUMNS || pMetaMsg->numOfColumns <= 0) { + tscError("invalid numOfColumns:%d", pMetaMsg->numOfColumns); + return TSDB_CODE_TSC_INVALID_VALUE; + } + + for (int i = 0; i < pMetaMsg->vgroup.numOfEps; ++i) { + pMetaMsg->vgroup.epAddr[i].port = htons(pMetaMsg->vgroup.epAddr[i].port); + } + + SSchema* pSchema = pMetaMsg->schema; + + int32_t numOfTotalCols = pMetaMsg->numOfColumns + pMetaMsg->numOfTags; + for (int i = 0; i < numOfTotalCols; ++i) { + pSchema->bytes = htons(pSchema->bytes); + pSchema->colId = htons(pSchema->colId); + + if (pSchema->colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { + assert(i == 0); + } + + pSchema++; + } + + return TSDB_CODE_SUCCESS; +} + +// update the vgroupInfo if needed +static void doUpdateVgroupInfo(int32_t vgId, SVgroupMsg *pVgroupMsg) { + assert(vgId > 0); + + SNewVgroupInfo vgroupInfo = {.inUse = -1}; + taosHashGetClone(tscVgroupMap, &vgId, sizeof(vgId), NULL, &vgroupInfo); + + // vgroup info exists, compare with it + if (((vgroupInfo.inUse >= 0) && !vgroupInfoIdentical(&vgroupInfo, pVgroupMsg)) || (vgroupInfo.inUse < 0)) { + vgroupInfo = createNewVgroupInfo(pVgroupMsg); + taosHashPut(tscVgroupMap, &vgId, sizeof(vgId), &vgroupInfo, sizeof(vgroupInfo)); + tscDebug("add/update new VgroupInfo, vgId:%d, total cached:%d", vgId, (int32_t) taosHashGetSize(tscVgroupMap)); + } +} + +static void doAddTableMetaToLocalBuf(STableMeta* pTableMeta, STableMetaMsg* pMetaMsg, bool updateSTable) { + if (pTableMeta->tableType == TSDB_CHILD_TABLE) { + // add or update the corresponding super table meta data info + int32_t len = (int32_t) strnlen(pTableMeta->sTableName, TSDB_TABLE_FNAME_LEN); + + // The super tableMeta already exists, create it according to tableMeta and add it to hash map + if (updateSTable) { + STableMeta* pSupTableMeta = createSuperTableMeta(pMetaMsg); + uint32_t size = tscGetTableMetaSize(pSupTableMeta); + int32_t code = taosHashPut(tscTableMetaMap, pTableMeta->sTableName, len, pSupTableMeta, size); + assert(code == TSDB_CODE_SUCCESS); + + tfree(pSupTableMeta); + } + + CChildTableMeta* cMeta = tscCreateChildMeta(pTableMeta); + taosHashPut(tscTableMetaMap, pMetaMsg->tableFname, strlen(pMetaMsg->tableFname), cMeta, sizeof(CChildTableMeta)); + tfree(cMeta); + } else { + uint32_t s = tscGetTableMetaSize(pTableMeta); + taosHashPut(tscTableMetaMap, pMetaMsg->tableFname, strlen(pMetaMsg->tableFname), pTableMeta, s); + } +} + +int tscProcessTableMetaRsp(SSqlObj *pSql) { + STableMetaMsg *pMetaMsg = (STableMetaMsg *)pSql->res.pRsp; + int32_t code = tableMetaMsgConvert(pMetaMsg); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0); + assert(pTableMetaInfo->pTableMeta == NULL); + + STableMeta* pTableMeta = tscCreateTableMetaFromMsg(pMetaMsg); + if (pTableMeta == NULL){ + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + if (!tIsValidSchema(pTableMeta->schema, pTableMeta->tableInfo.numOfColumns, pTableMeta->tableInfo.numOfTags)) { + tscError("0x%"PRIx64" invalid table meta from mnode, name:%s", pSql->self, tNameGetTableName(&pTableMetaInfo->name)); + tfree(pTableMeta); + return TSDB_CODE_TSC_INVALID_VALUE; + } + + char name[TSDB_TABLE_FNAME_LEN] = {0}; + tNameExtractFullName(&pTableMetaInfo->name, name); + assert(strncmp(pMetaMsg->tableFname, name, tListLen(pMetaMsg->tableFname)) == 0); + + doAddTableMetaToLocalBuf(pTableMeta, pMetaMsg, true); + if (pTableMeta->tableType != TSDB_SUPER_TABLE) { + doUpdateVgroupInfo(pTableMeta->vgId, &pMetaMsg->vgroup); + } + + tscDebug("0x%"PRIx64" recv table meta, uid:%" PRIu64 ", tid:%d, name:%s, numOfCols:%d, numOfTags:%d", pSql->self, + pTableMeta->id.uid, pTableMeta->id.tid, tNameGetTableName(&pTableMetaInfo->name), pTableMeta->tableInfo.numOfColumns, + pTableMeta->tableInfo.numOfTags); + + free(pTableMeta); + return TSDB_CODE_SUCCESS; +} + +static SArray* createVgroupIdListFromMsg(char* pMsg, SHashObj* pSet, char* name, int32_t* size, uint64_t id) { + SVgroupsMsg *pVgroupMsg = (SVgroupsMsg *)pMsg; + + pVgroupMsg->numOfVgroups = htonl(pVgroupMsg->numOfVgroups); + *size = (int32_t)(sizeof(SVgroupMsg) * pVgroupMsg->numOfVgroups + sizeof(SVgroupsMsg)); + + SArray* vgroupIdList = taosArrayInit(pVgroupMsg->numOfVgroups, sizeof(int32_t)); + + if (pVgroupMsg->numOfVgroups <= 0) { + tscDebug("0x%" PRIx64 " empty vgroup id list, no corresponding tables for stable:%s", id, name); + } else { + // just init, no need to lock + for (int32_t j = 0; j < pVgroupMsg->numOfVgroups; ++j) { + SVgroupMsg *vmsg = &pVgroupMsg->vgroups[j]; + vmsg->vgId = htonl(vmsg->vgId); + for (int32_t k = 0; k < vmsg->numOfEps; ++k) { + vmsg->epAddr[k].port = htons(vmsg->epAddr[k].port); + } + + taosArrayPush(vgroupIdList, &vmsg->vgId); + + if (taosHashGet(pSet, &vmsg->vgId, sizeof(vmsg->vgId)) == NULL) { + taosHashPut(pSet, &vmsg->vgId, sizeof(vmsg->vgId), "", 0); + doUpdateVgroupInfo(vmsg->vgId, vmsg); + } + } + } + + return vgroupIdList; +} + +static SVgroupsInfo* createVgroupInfoFromMsg(char* pMsg, int32_t* size, uint64_t id) { + SVgroupsMsg *pVgroupMsg = (SVgroupsMsg *)pMsg; + pVgroupMsg->numOfVgroups = htonl(pVgroupMsg->numOfVgroups); + + *size = (int32_t)(sizeof(SVgroupMsg) * pVgroupMsg->numOfVgroups + sizeof(SVgroupsMsg)); + + size_t vgroupsz = sizeof(SVgroupMsg) * pVgroupMsg->numOfVgroups + sizeof(SVgroupsInfo); + SVgroupsInfo *pVgroupInfo = calloc(1, vgroupsz); + assert(pVgroupInfo != NULL); + + pVgroupInfo->numOfVgroups = pVgroupMsg->numOfVgroups; + if (pVgroupInfo->numOfVgroups <= 0) { + tscDebug("0x%" PRIx64 " empty vgroup info, no corresponding tables for stable", id); + } else { + for (int32_t j = 0; j < pVgroupInfo->numOfVgroups; ++j) { + // just init, no need to lock + SVgroupMsg *pVgroup = &pVgroupInfo->vgroups[j]; + + SVgroupMsg *vmsg = &pVgroupMsg->vgroups[j]; + vmsg->vgId = htonl(vmsg->vgId); + for (int32_t k = 0; k < vmsg->numOfEps; ++k) { + vmsg->epAddr[k].port = htons(vmsg->epAddr[k].port); + } + + pVgroup->numOfEps = vmsg->numOfEps; + pVgroup->vgId = vmsg->vgId; + for (int32_t k = 0; k < vmsg->numOfEps; ++k) { + pVgroup->epAddr[k].port = vmsg->epAddr[k].port; + tstrncpy(pVgroup->epAddr[k].fqdn, vmsg->epAddr[k].fqdn, TSDB_FQDN_LEN); +// pVgroup->epAddr[k].fqdn = strndup(vmsg->epAddr[k].fqdn, TSDB_FQDN_LEN); + } + + doUpdateVgroupInfo(pVgroup->vgId, vmsg); + } + } + + return pVgroupInfo; +} + +int tscProcessRetrieveFuncRsp(SSqlObj* pSql) { + SSqlCmd* pCmd = &pSql->cmd; + SUdfFuncMsg* pFuncMsg = (SUdfFuncMsg *)pSql->res.pRsp; + SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); + + pFuncMsg->num = htonl(pFuncMsg->num); + assert(pFuncMsg->num == taosArrayGetSize(pQueryInfo->pUdfInfo)); + + char* pMsg = pFuncMsg->content; + for(int32_t i = 0; i < pFuncMsg->num; ++i) { + SFunctionInfoMsg* pFunc = (SFunctionInfoMsg*) pMsg; + + for(int32_t j = 0; j < pFuncMsg->num; ++j) { + SUdfInfo* pUdfInfo = taosArrayGet(pQueryInfo->pUdfInfo, j); + if (strcmp(pUdfInfo->name, pFunc->name) != 0) { + continue; + } + + if (pUdfInfo->content) { + continue; + } + + pUdfInfo->resBytes = htons(pFunc->resBytes); + pUdfInfo->resType = pFunc->resType; + pUdfInfo->funcType = htonl(pFunc->funcType); + pUdfInfo->contLen = htonl(pFunc->len); + pUdfInfo->bufSize = htonl(pFunc->bufSize); + + pUdfInfo->content = malloc(pUdfInfo->contLen); + memcpy(pUdfInfo->content, pFunc->content, pUdfInfo->contLen); + + pMsg += sizeof(SFunctionInfoMsg) + pUdfInfo->contLen; + } + } + + // master sqlObj locates in param + SSqlObj* parent = (SSqlObj*)taosAcquireRef(tscObjRef, (int64_t)pSql->param); + if(parent == NULL) { + return pSql->res.code; + } + + SQueryInfo* parQueryInfo = tscGetQueryInfo(&parent->cmd); + + assert(parent->signature == parent && (int64_t)pSql->param == parent->self); + taosArrayDestroy(parQueryInfo->pUdfInfo); + + parQueryInfo->pUdfInfo = pQueryInfo->pUdfInfo; // assigned to parent sql obj. + pQueryInfo->pUdfInfo = NULL; + return TSDB_CODE_SUCCESS; +} + +int tscProcessMultiTableMetaRsp(SSqlObj *pSql) { + char *rsp = pSql->res.pRsp; + + SMultiTableMeta *pMultiMeta = (SMultiTableMeta *)rsp; + pMultiMeta->numOfTables = htonl(pMultiMeta->numOfTables); + pMultiMeta->numOfVgroup = htonl(pMultiMeta->numOfVgroup); + pMultiMeta->numOfUdf = htonl(pMultiMeta->numOfUdf); + + rsp += sizeof(SMultiTableMeta); + + SSqlObj* pParentSql = (SSqlObj*)taosAcquireRef(tscObjRef, (int64_t)pSql->param); + if(pParentSql == NULL) { + return pSql->res.code; + } + + SSqlCmd *pParentCmd = &pParentSql->cmd; + SHashObj *pSet = taosHashInit(pMultiMeta->numOfVgroup, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); + + char* buf = NULL; + char* pMsg = pMultiMeta->meta; + + // decompresss the message payload + if (pMultiMeta->compressed) { + buf = malloc(pMultiMeta->rawLen - sizeof(SMultiTableMeta)); + int32_t len = tsDecompressString(pMultiMeta->meta, pMultiMeta->contLen - sizeof(SMultiTableMeta), 1, + buf, pMultiMeta->rawLen - sizeof(SMultiTableMeta), ONE_STAGE_COMP, NULL, 0); + assert(len == pMultiMeta->rawLen - sizeof(SMultiTableMeta)); + + pMsg = buf; + } + + if (pParentCmd->pTableMetaMap == NULL) { + pParentCmd->pTableMetaMap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + } + + for (int32_t i = 0; i < pMultiMeta->numOfTables; i++) { + STableMetaMsg *pMetaMsg = (STableMetaMsg *)pMsg; + int32_t code = tableMetaMsgConvert(pMetaMsg); + if (code != TSDB_CODE_SUCCESS) { + taosHashCleanup(pSet); + taosReleaseRef(tscObjRef, pParentSql->self); + + tfree(buf); + return code; + } + + bool freeMeta = false; + STableMeta* pTableMeta = tscCreateTableMetaFromMsg(pMetaMsg); + if (!tIsValidSchema(pTableMeta->schema, pTableMeta->tableInfo.numOfColumns, pTableMeta->tableInfo.numOfTags)) { + tscError("0x%"PRIx64" invalid table meta from mnode, name:%s", pSql->self, pMetaMsg->tableFname); + tfree(pTableMeta); + taosHashCleanup(pSet); + taosReleaseRef(tscObjRef, pParentSql->self); + + tfree(buf); + return TSDB_CODE_TSC_INVALID_VALUE; + } + + if (pMultiMeta->metaClone == 1 || pTableMeta->tableType == TSDB_SUPER_TABLE) { + STableMetaVgroupInfo p = {.pTableMeta = pTableMeta,}; + size_t keyLen = strnlen(pMetaMsg->tableFname, TSDB_TABLE_FNAME_LEN); + void* t = taosHashGet(pParentCmd->pTableMetaMap, pMetaMsg->tableFname, keyLen); + assert(t == NULL); + + taosHashPut(pParentCmd->pTableMetaMap, pMetaMsg->tableFname, keyLen, &p, sizeof(STableMetaVgroupInfo)); + } else { + freeMeta = true; + } + + // for each super table, only update meta information once + bool updateStableMeta = false; + if (pTableMeta->tableType == TSDB_CHILD_TABLE && taosHashGet(pSet, &pMetaMsg->suid, sizeof(pMetaMsg->suid)) == NULL) { + updateStableMeta = true; + taosHashPut(pSet, &pTableMeta->suid, sizeof(pMetaMsg->suid), "", 0); + } + + // create the tableMeta and add it into the TableMeta map + doAddTableMetaToLocalBuf(pTableMeta, pMetaMsg, updateStableMeta); + + // for each vgroup, only update the information once. + int64_t vgId = pMetaMsg->vgroup.vgId; + if (pTableMeta->tableType != TSDB_SUPER_TABLE && taosHashGet(pSet, &vgId, sizeof(vgId)) == NULL) { + doUpdateVgroupInfo((int32_t) vgId, &pMetaMsg->vgroup); + taosHashPut(pSet, &vgId, sizeof(vgId), "", 0); + } + + pMsg += pMetaMsg->contLen; + if (freeMeta) { + tfree(pTableMeta); + } + } + + for(int32_t i = 0; i < pMultiMeta->numOfVgroup; ++i) { + char fname[TSDB_TABLE_FNAME_LEN] = {0}; + tstrncpy(fname, pMsg, TSDB_TABLE_FNAME_LEN); + size_t len = strnlen(fname, TSDB_TABLE_FNAME_LEN); + + pMsg += TSDB_TABLE_FNAME_LEN; + + STableMetaVgroupInfo* p = taosHashGet(pParentCmd->pTableMetaMap, fname, len); + assert(p != NULL); + + int32_t size = 0; + if (p->vgroupIdList!= NULL) { + taosArrayDestroy(p->vgroupIdList); + } + + p->vgroupIdList = createVgroupIdListFromMsg(pMsg, pSet, fname, &size, pSql->self); + + int32_t numOfVgId = (int32_t) taosArrayGetSize(p->vgroupIdList); + int32_t s = sizeof(tFilePage) + numOfVgId * sizeof(int32_t); + + tFilePage* idList = calloc(1, s); + idList->num = numOfVgId; + memcpy(idList->data, TARRAY_GET_START(p->vgroupIdList), numOfVgId * sizeof(int32_t)); + + void* idListInst = taosCachePut(tscVgroupListBuf, fname, len, idList, s, 5000); + taosCacheRelease(tscVgroupListBuf, (void*) &idListInst, false); + + tfree(idList); + pMsg += size; + } + + SQueryInfo* pQueryInfo = tscGetQueryInfo(pParentCmd); + if (pMultiMeta->numOfUdf > 0) { + assert(pQueryInfo->pUdfInfo != NULL); + } + + for(int32_t i = 0; i < pMultiMeta->numOfUdf; ++i) { + SFunctionInfoMsg* pFunc = (SFunctionInfoMsg*) pMsg; + + for(int32_t j = 0; j < pMultiMeta->numOfUdf; ++j) { + SUdfInfo* pUdfInfo = taosArrayGet(pQueryInfo->pUdfInfo, j); + if (strcmp(pUdfInfo->name, pFunc->name) != 0) { + continue; + } + + if (pUdfInfo->content) { + continue; + } + + pUdfInfo->resBytes = htons(pFunc->resBytes); + pUdfInfo->resType = pFunc->resType; + pUdfInfo->funcType = htonl(pFunc->funcType); + pUdfInfo->contLen = htonl(pFunc->len); + pUdfInfo->bufSize = htonl(pFunc->bufSize); + + pUdfInfo->content = malloc(pUdfInfo->contLen); + memcpy(pUdfInfo->content, pFunc->content, pUdfInfo->contLen); + + pMsg += sizeof(SFunctionInfoMsg) + pUdfInfo->contLen; + } + } + + pSql->res.code = TSDB_CODE_SUCCESS; + pSql->res.numOfTotal = pMultiMeta->numOfTables; + tscDebug("0x%"PRIx64" load multi-tableMeta from mnode, numOfTables:%d", pSql->self, pMultiMeta->numOfTables); + + taosHashCleanup(pSet); + taosReleaseRef(tscObjRef, pParentSql->self); + + tfree(buf); + return TSDB_CODE_SUCCESS; +} + +int tscProcessSTableVgroupRsp(SSqlObj *pSql) { + // master sqlObj locates in param + SSqlObj* parent = (SSqlObj*)taosAcquireRef(tscObjRef, (int64_t)pSql->param); + if(parent == NULL) { + return pSql->res.code; + } + + assert(parent->signature == parent && (int64_t)pSql->param == parent->self); + + SSqlRes* pRes = &pSql->res; + + // NOTE: the order of several table must be preserved. + SSTableVgroupRspMsg *pStableVgroup = (SSTableVgroupRspMsg *)pRes->pRsp; + pStableVgroup->numOfTables = htonl(pStableVgroup->numOfTables); + char *pMsg = pRes->pRsp + sizeof(SSTableVgroupRspMsg); + + SSqlCmd* pCmd = &parent->cmd; + SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); + + char fName[TSDB_TABLE_FNAME_LEN] = {0}; + for(int32_t i = 0; i < pStableVgroup->numOfTables; ++i) { + char* name = pMsg; + pMsg += TSDB_TABLE_FNAME_LEN; + + STableMetaInfo *pInfo = NULL; + for(int32_t j = 0; j < pQueryInfo->numOfTables; ++j) { + STableMetaInfo *pInfo1 = tscGetTableMetaInfoFromCmd(pCmd, j); + memset(fName, 0, tListLen(fName)); + + tNameExtractFullName(&pInfo1->name, fName); + if (strcmp(name, fName) != 0) { + continue; + } + + pInfo = pInfo1; + break; + } + + if (!pInfo){ + continue; + } + int32_t size = 0; + pInfo->vgroupList = createVgroupInfoFromMsg(pMsg, &size, pSql->self); + pMsg += size; + } + + taosReleaseRef(tscObjRef, parent->self); + return pSql->res.code; +} + +int tscProcessShowRsp(SSqlObj *pSql) { + STableMetaMsg *pMetaMsg; + SShowRsp * pShow; + SSchema * pSchema; + + SSqlRes *pRes = &pSql->res; + SSqlCmd *pCmd = &pSql->cmd; + + SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); + + STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + + pShow = (SShowRsp *)pRes->pRsp; + pShow->qhandle = htobe64(pShow->qhandle); + pRes->qId = pShow->qhandle; + + tscResetForNextRetrieve(pRes); + pMetaMsg = &(pShow->tableMeta); + + pMetaMsg->numOfColumns = ntohs(pMetaMsg->numOfColumns); + + pSchema = pMetaMsg->schema; + pMetaMsg->tid = ntohs(pMetaMsg->tid); + for (int i = 0; i < pMetaMsg->numOfColumns; ++i) { + pSchema->bytes = htons(pSchema->bytes); + pSchema++; + } + + tfree(pTableMetaInfo->pTableMeta); + pTableMetaInfo->pTableMeta = tscCreateTableMetaFromMsg(pMetaMsg); + + SSchema *pTableSchema = tscGetTableSchema(pTableMetaInfo->pTableMeta); + if (pQueryInfo->colList == NULL) { + pQueryInfo->colList = taosArrayInit(4, POINTER_BYTES); + } + + SFieldInfo* pFieldInfo = &pQueryInfo->fieldsInfo; + + SColumnIndex index = {0}; + pSchema = pMetaMsg->schema; + + uint64_t uid = pTableMetaInfo->pTableMeta->id.uid; + for (int16_t i = 0; i < pMetaMsg->numOfColumns; ++i, ++pSchema) { + index.columnIndex = i; + tscColumnListInsert(pQueryInfo->colList, i, uid, pSchema); + + TAOS_FIELD f = tscCreateField(pSchema->type, pSchema->name, pSchema->bytes); + SInternalField* pInfo = tscFieldInfoAppend(pFieldInfo, &f); + + pInfo->pExpr = tscExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, + pTableSchema[i].type, pTableSchema[i].bytes, getNewResColId(pCmd), pTableSchema[i].bytes, false); + } + + pCmd->numOfCols = pQueryInfo->fieldsInfo.numOfOutput; + tscFieldInfoUpdateOffset(pQueryInfo); + return 0; +} + +static void createHbObj(STscObj* pObj) { + if (pObj->hbrid != 0) { + return; + } + + SSqlObj *pSql = (SSqlObj *)calloc(1, sizeof(SSqlObj)); + if (NULL == pSql) return; + + pSql->fp = tscProcessHeartBeatRsp; + + SQueryInfo *pQueryInfo = tscGetQueryInfoS(&pSql->cmd); + if (pQueryInfo == NULL) { + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + tfree(pSql); + return; + } + + pQueryInfo->command = TSDB_SQL_HB; + + pSql->cmd.command = pQueryInfo->command; + if (TSDB_CODE_SUCCESS != tscAllocPayload(&(pSql->cmd), TSDB_DEFAULT_PAYLOAD_SIZE)) { + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + tfree(pSql); + return; + } + + pSql->param = pObj; + pSql->pTscObj = pObj; + pSql->signature = pSql; + + registerSqlObj(pSql); + tscDebug("0x%"PRIx64" HB is allocated, pObj:%p", pSql->self, pObj); + + pObj->hbrid = pSql->self; +} + +int tscProcessUseDbRsp(SSqlObj *pSql) { + STscObj * pObj = pSql->pTscObj; + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0); + + pthread_mutex_lock(&pObj->mutex); + int ret = tNameExtractFullName(&pTableMetaInfo->name, pObj->db); + pthread_mutex_unlock(&pObj->mutex); + + return ret; +} + +//todo only invalid the buffered data that belongs to dropped databases +int tscProcessDropDbRsp(SSqlObj *pSql) { + //TODO LOCK DB WHEN MODIFY IT + //pSql->pTscObj->db[0] = 0; + + taosHashClear(tscTableMetaMap); + taosHashClear(tscVgroupMap); + taosCacheEmpty(tscVgroupListBuf); + return 0; +} + +int tscProcessDropTableRsp(SSqlObj *pSql) { + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0); + tscRemoveCachedTableMeta(pTableMetaInfo, pSql->self); + tfree(pTableMetaInfo->pTableMeta); + return 0; +} + +int tscProcessAlterTableMsgRsp(SSqlObj *pSql) { + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0); + + char name[TSDB_TABLE_FNAME_LEN] = {0}; + tNameExtractFullName(&pTableMetaInfo->name, name); + + tscDebug("0x%"PRIx64" remove tableMeta in hashMap after alter-table: %s", pSql->self, name); + + bool isSuperTable = UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo); + taosHashRemove(tscTableMetaMap, name, strnlen(name, TSDB_TABLE_FNAME_LEN)); + tfree(pTableMetaInfo->pTableMeta); + + if (isSuperTable) { // if it is a super table, iterate the hashTable and remove all the childTableMeta + if (pSql->res.pRsp == NULL) { + tscDebug("0x%"PRIx64" unexpected resp from mnode, super table: %s failed to update super table meta ", pSql->self, name); + return 0; + } + return tscProcessTableMetaRsp(pSql); + } + + return 0; +} + +int tscProcessAlterDbMsgRsp(SSqlObj *pSql) { + UNUSED(pSql); + return 0; +} +int tscProcessCompactRsp(SSqlObj *pSql) { + UNUSED(pSql); + return TSDB_CODE_SUCCESS; +} + +int tscProcessShowCreateRsp(SSqlObj *pSql) { + return tscLocalResultCommonBuilder(pSql, 1); +} + +int tscProcessQueryRsp(SSqlObj *pSql) { + SSqlRes *pRes = &pSql->res; + + SQueryTableRsp *pQueryAttr = (SQueryTableRsp *)pRes->pRsp; + pQueryAttr->qId = htobe64(pQueryAttr->qId); + + pRes->qId = pQueryAttr->qId; + pRes->data = NULL; + + tscResetForNextRetrieve(pRes); + tscDebug("0x%"PRIx64" query rsp received, qId:0x%"PRIx64, pSql->self, pRes->qId); + return 0; +} + +static void decompressQueryColData(SSqlObj *pSql, SSqlRes *pRes, SQueryInfo* pQueryInfo, char **data, int8_t compressed, int32_t compLen) { + int32_t decompLen = 0; + int32_t numOfCols = pQueryInfo->fieldsInfo.numOfOutput; + int32_t *compSizes; + char *pData = *data; + compSizes = (int32_t *)(pData + compLen); + + TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, numOfCols - 1); + int16_t offset = tscFieldInfoGetOffset(pQueryInfo, numOfCols - 1); + char *outputBuf = tcalloc(pRes->numOfRows, (pField->bytes + offset)); + + char *p = outputBuf; + int32_t bufOffset; + for (int32_t i = 0; i < numOfCols; ++i) { + SInternalField* pInfo = (SInternalField*)TARRAY_GET_ELEM(pQueryInfo->fieldsInfo.internalField, i); + bufOffset = pInfo->field.bytes * pRes->numOfRows; + + int32_t flen = (*(tDataTypes[pInfo->field.type].decompFunc))(pData, htonl(compSizes[i]), pRes->numOfRows, p, bufOffset, + compressed, NULL, 0); + + p += flen; + decompLen +=flen; + pData += htonl(compSizes[i]); + } + + /* Resize rsp as decompressed data will occupy more space */ + pRes->rspLen = pRes->rspLen - (compLen + numOfCols * sizeof(int32_t)) + decompLen; + char *new_rsp = (char *)realloc(pRes->pRsp, pRes->rspLen); + if (new_rsp == NULL) { + pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; + return; + } else { + pRes->pRsp = new_rsp; + *data = ((SRetrieveTableRsp *)pRes->pRsp)->data; + pData = *data + compLen + numOfCols * sizeof(int32_t); + } + + tscDebug("0x%"PRIx64" decompress col data, compressed size:%d, decompressed size:%d", + pSql->self, (int32_t)(compLen + numOfCols * sizeof(int32_t)), decompLen); + + int32_t tailLen = pRes->rspLen - sizeof(SRetrieveTableRsp) - decompLen; + memmove(*data + decompLen, pData, tailLen); + memmove(*data, outputBuf, decompLen); + + tfree(outputBuf); +} + +int tscProcessRetrieveRspFromNode(SSqlObj *pSql) { + SSqlRes *pRes = &pSql->res; + SSqlCmd *pCmd = &pSql->cmd; + + assert(pRes->rspLen >= sizeof(SRetrieveTableRsp)); + + SRetrieveTableRsp *pRetrieve = (SRetrieveTableRsp *)pRes->pRsp; + if (pRetrieve == NULL) { + pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; + return pRes->code; + } + + pRes->numOfRows = htonl(pRetrieve->numOfRows); + pRes->precision = htons(pRetrieve->precision); + pRes->offset = htobe64(pRetrieve->offset); + pRes->useconds = htobe64(pRetrieve->useconds); + pRes->completed = (pRetrieve->completed == 1); + pRes->data = pRetrieve->data; + + SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); + if (tscCreateResPointerInfo(pRes, pQueryInfo) != TSDB_CODE_SUCCESS) { + return pRes->code; + } + + //Decompress col data if compressed from server + if (pRetrieve->compressed) { + int32_t compLen = htonl(pRetrieve->compLen); + decompressQueryColData(pSql, pRes, pQueryInfo, &pRes->data, pRetrieve->compressed, compLen); + } + + STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + if ((pCmd->command == TSDB_SQL_RETRIEVE) || + ((UTIL_TABLE_IS_CHILD_TABLE(pTableMetaInfo) || UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo)) && + !TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_SUBQUERY)) || + (tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0) && + !TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_JOIN_QUERY) && + !TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_JOIN_SEC_STAGE))) { + tscSetResRawPtr(pRes, pQueryInfo); + } + + if (pSql->pSubscription != NULL) { + int32_t numOfCols = pQueryInfo->fieldsInfo.numOfOutput; + + TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, numOfCols - 1); + int16_t offset = tscFieldInfoGetOffset(pQueryInfo, numOfCols - 1); + + char* p = pRes->data + (pField->bytes + offset) * pRes->numOfRows; + + int32_t numOfTables = htonl(*(int32_t*)p); + p += sizeof(int32_t); + for (int i = 0; i < numOfTables; i++) { + int64_t uid = htobe64(*(int64_t*)p); + p += sizeof(int64_t); + p += sizeof(int32_t); // skip tid + TSKEY key = htobe64(*(TSKEY*)p); + p += sizeof(TSKEY); + tscUpdateSubscriptionProgress(pSql->pSubscription, uid, key); + } + } + + pRes->row = 0; + tscDebug("0x%"PRIx64" numOfRows:%d, offset:%" PRId64 ", complete:%d, qId:0x%"PRIx64, pSql->self, pRes->numOfRows, pRes->offset, + pRes->completed, pRes->qId); + + return 0; +} + +void tscTableMetaCallBack(void *param, TAOS_RES *res, int code); + +static int32_t getTableMetaFromMnode(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo, bool autocreate) { + SSqlObj *pNew = calloc(1, sizeof(SSqlObj)); + if (NULL == pNew) { + tscError("0x%"PRIx64" malloc failed for new sqlobj to get table meta", pSql->self); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + pNew->pTscObj = pSql->pTscObj; + pNew->signature = pNew; + pNew->cmd.command = TSDB_SQL_META; + + tscAddQueryInfo(&pNew->cmd); + + SQueryInfo *pNewQueryInfo = tscGetQueryInfoS(&pNew->cmd); + if (TSDB_CODE_SUCCESS != tscAllocPayload(&pNew->cmd, TSDB_DEFAULT_PAYLOAD_SIZE + pSql->cmd.payloadLen)) { + tscError("0x%"PRIx64" malloc failed for payload to get table meta", pSql->self); + + tscFreeSqlObj(pNew); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + STableMetaInfo *pNewTableMetaInfo = tscAddEmptyMetaInfo(pNewQueryInfo); + assert(pNewQueryInfo->numOfTables == 1); + + tNameAssign(&pNewTableMetaInfo->name, &pTableMetaInfo->name); + + registerSqlObj(pNew); + + pNew->fp = tscTableMetaCallBack; + pNew->param = (void *)pSql->self; + + tscDebug("0x%"PRIx64" new pSqlObj:0x%"PRIx64" to get tableMeta, auto create:%d, metaRid from %"PRId64" to %"PRId64, + pSql->self, pNew->self, autocreate, pSql->metaRid, pNew->self); + pSql->metaRid = pNew->self; + + { + STableInfoMsg *pInfoMsg = (STableInfoMsg *)pNew->cmd.payload; + int32_t code = tNameExtractFullName(&pNewTableMetaInfo->name, pInfoMsg->tableFname); + if (code != TSDB_CODE_SUCCESS) { + return TSDB_CODE_TSC_INVALID_OPERATION; + } + + pInfoMsg->createFlag = htons(autocreate? 1 : 0); + char *pMsg = (char *)pInfoMsg + sizeof(STableInfoMsg); + + // tag data exists + if (autocreate && pSql->cmd.insertParam.tagData.dataLen != 0) { + pMsg = serializeTagData(&pSql->cmd.insertParam.tagData, pMsg); + } + + pNew->cmd.payloadLen = (int32_t)(pMsg - (char*)pInfoMsg); + pNew->cmd.msgType = TSDB_MSG_TYPE_TABLE_META; + } + + int32_t code = tscBuildAndSendRequest(pNew, NULL); + if (code == TSDB_CODE_SUCCESS) { + code = TSDB_CODE_TSC_ACTION_IN_PROGRESS; // notify application that current process needs to be terminated + } + + return code; +} + +int32_t getMultiTableMetaFromMnode(SSqlObj *pSql, SArray* pNameList, SArray* pVgroupNameList, SArray* pUdfList, __async_cb_func_t fp, bool metaClone) { + SSqlObj *pNew = calloc(1, sizeof(SSqlObj)); + if (NULL == pNew) { + tscError("0x%"PRIx64" failed to allocate sqlobj to get multiple table meta", pSql->self); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + pNew->pTscObj = pSql->pTscObj; + pNew->signature = pNew; + pNew->cmd.command = TSDB_SQL_MULTI_META; + + int32_t numOfTable = (int32_t) taosArrayGetSize(pNameList); + int32_t numOfVgroupList = (int32_t) taosArrayGetSize(pVgroupNameList); + int32_t numOfUdf = pUdfList ? (int32_t)taosArrayGetSize(pUdfList) : 0; + + int32_t size = (numOfTable + numOfVgroupList) * TSDB_TABLE_FNAME_LEN + TSDB_FUNC_NAME_LEN * numOfUdf + sizeof(SMultiTableInfoMsg); + if (TSDB_CODE_SUCCESS != tscAllocPayload(&pNew->cmd, size)) { + tscError("0x%"PRIx64" malloc failed for payload to get table meta", pSql->self); + tscFreeSqlObj(pNew); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + SMultiTableInfoMsg* pInfo = (SMultiTableInfoMsg*) pNew->cmd.payload; + pInfo->metaClone = metaClone? 1:0; + pInfo->numOfTables = htonl((uint32_t) taosArrayGetSize(pNameList)); + pInfo->numOfVgroups = htonl((uint32_t) taosArrayGetSize(pVgroupNameList)); + pInfo->numOfUdfs = htonl(numOfUdf); + + char* start = pInfo->tableNames; + int32_t len = 0; + for(int32_t i = 0; i < numOfTable; ++i) { + char* name = taosArrayGetP(pNameList, i); + if (i < numOfTable - 1 || numOfVgroupList > 0 || numOfUdf > 0) { + len = sprintf(start, "%s,", name); + } else { + len = sprintf(start, "%s", name); + } + + start += len; + } + + for(int32_t i = 0; i < numOfVgroupList; ++i) { + char* name = taosArrayGetP(pVgroupNameList, i); + if (i < numOfVgroupList - 1 || numOfUdf > 0) { + len = sprintf(start, "%s,", name); + } else { + len = sprintf(start, "%s", name); + } + + start += len; + } + + for(int32_t i = 0; i < numOfUdf; ++i) { + SUdfInfo * u = taosArrayGet(pUdfList, i); + if (i < numOfUdf - 1) { + len = sprintf(start, "%s,", u->name); + } else { + len = sprintf(start, "%s", u->name); + } + + start += len; + } + + pNew->cmd.payloadLen = (int32_t) ((start - pInfo->tableNames) + sizeof(SMultiTableInfoMsg)); + pNew->cmd.msgType = TSDB_MSG_TYPE_TABLES_META; + + registerSqlObj(pNew); + tscDebug("0x%"PRIx64" new pSqlObj:0x%"PRIx64" to get %d tableMeta, vgroupInfo:%d, udf:%d, msg size:%d", pSql->self, + pNew->self, numOfTable, numOfVgroupList, numOfUdf, pNew->cmd.payloadLen); + + pNew->fp = fp; + pNew->param = (void *)pSql->self; + + tscDebug("0x%"PRIx64" metaRid from 0x%" PRIx64 " to 0x%" PRIx64 , pSql->self, pSql->metaRid, pNew->self); + + pSql->metaRid = pNew->self; + int32_t code = tscBuildAndSendRequest(pNew, NULL); + if (code == TSDB_CODE_SUCCESS) { + code = TSDB_CODE_TSC_ACTION_IN_PROGRESS; // notify application that current process needs to be terminated + } + + return code; +} + +int32_t tscGetTableMetaImpl(SSqlObj* pSql, STableMetaInfo *pTableMetaInfo, bool autocreate, bool onlyLocal) { + assert(tIsValidName(&pTableMetaInfo->name)); + + char name[TSDB_TABLE_FNAME_LEN] = {0}; + tNameExtractFullName(&pTableMetaInfo->name, name); + + size_t len = strlen(name); + // just make runtime happy + if (pTableMetaInfo->tableMetaCapacity != 0 && pTableMetaInfo->pTableMeta != NULL) { + memset(pTableMetaInfo->pTableMeta, 0, pTableMetaInfo->tableMetaCapacity); + } + + if (NULL == taosHashGetCloneExt(tscTableMetaMap, name, len, NULL, (void **)&(pTableMetaInfo->pTableMeta), &pTableMetaInfo->tableMetaCapacity)) { + tfree(pTableMetaInfo->pTableMeta); + } + + STableMeta* pMeta = pTableMetaInfo->pTableMeta; + STableMeta* pSTMeta = (STableMeta *)(pSql->pBuf); + + if (pMeta && pMeta->id.uid > 0) { + // in case of child table, here only get the + if (pMeta->tableType == TSDB_CHILD_TABLE) { + int32_t code = tscCreateTableMetaFromSTableMeta(&pTableMetaInfo->pTableMeta, name, &pTableMetaInfo->tableMetaCapacity, (STableMeta **)(&pSTMeta)); + pSql->pBuf = (void *)(pSTMeta); + if (code != TSDB_CODE_SUCCESS) { + return getTableMetaFromMnode(pSql, pTableMetaInfo, autocreate); + } + } + + tscDebug("0x%"PRIx64 " %s retrieve tableMeta from cache, numOfCols:%d, numOfTags:%d", pSql->self, name, pMeta->tableInfo.numOfColumns, pMeta->tableInfo.numOfTags); + return TSDB_CODE_SUCCESS; + } + + if (onlyLocal) { + return TSDB_CODE_TSC_NO_META_CACHED; + } + + return getTableMetaFromMnode(pSql, pTableMetaInfo, autocreate); +} + +int32_t tscGetTableMeta(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo) { + return tscGetTableMetaImpl(pSql, pTableMetaInfo, false, false); +} + +int tscGetTableMetaEx(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo, bool createIfNotExists, bool onlyLocal) { + return tscGetTableMetaImpl(pSql, pTableMetaInfo, createIfNotExists, onlyLocal); +} + +int32_t tscGetUdfFromNode(SSqlObj *pSql, SQueryInfo* pQueryInfo) { + SSqlObj *pNew = calloc(1, sizeof(SSqlObj)); + if (NULL == pNew) { + tscError("%p malloc failed for new sqlobj to get user-defined functions", pSql); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + pNew->pTscObj = pSql->pTscObj; + pNew->signature = pNew; + pNew->cmd.command = TSDB_SQL_RETRIEVE_FUNC; + + if (tscAddQueryInfo(&pNew->cmd) != TSDB_CODE_SUCCESS) { + tscError("%p malloc failed for new queryinfo", pSql); + tscFreeSqlObj(pNew); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + SQueryInfo *pNewQueryInfo = tscGetQueryInfo(&pNew->cmd); + + pNewQueryInfo->pUdfInfo = taosArrayInit(4, sizeof(SUdfInfo)); + for(int32_t i = 0; i < taosArrayGetSize(pQueryInfo->pUdfInfo); ++i) { + SUdfInfo info = {0}; + SUdfInfo* p1 = taosArrayGet(pQueryInfo->pUdfInfo, i); + info = *p1; + info.name = strdup(p1->name); + taosArrayPush(pNewQueryInfo->pUdfInfo, &info); + } + + pNew->cmd.active = pNewQueryInfo; + + if (TSDB_CODE_SUCCESS != tscAllocPayload(&pNew->cmd, TSDB_DEFAULT_PAYLOAD_SIZE + pSql->cmd.payloadLen)) { + tscError("%p malloc failed for payload to get table meta", pSql); + tscFreeSqlObj(pNew); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + tscDebug("%p new pSqlObj:%p to retrieve udf", pSql, pNew); + registerSqlObj(pNew); + + pNew->fp = tscTableMetaCallBack; + pNew->param = (void *)pSql->self; + + tscDebug("%p metaRid from %" PRId64 " to %" PRId64 , pSql, pSql->metaRid, pNew->self); + + pSql->metaRid = pNew->self; + + int32_t code = tscBuildAndSendRequest(pNew, NULL); + if (code == TSDB_CODE_SUCCESS) { + code = TSDB_CODE_TSC_ACTION_IN_PROGRESS; // notify application that current process needs to be terminated + } + + return code; +} + +static void freeElem(void* p) { + tfree(*(char**)p); +} + +/** + * retrieve table meta from mnode, and then update the local table meta hashmap. + * @param pSql sql object + * @param tableIndex table index + * @return status code + */ +int tscRenewTableMeta(SSqlObj *pSql, int32_t tableIndex) { + SSqlCmd* pCmd = &pSql->cmd; + + SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); + STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, tableIndex); + + char name[TSDB_TABLE_FNAME_LEN] = {0}; + int32_t code = tNameExtractFullName(&pTableMetaInfo->name, name); + if (code != TSDB_CODE_SUCCESS) { + tscError("0x%"PRIx64" failed to generate the table full name", pSql->self); + return TSDB_CODE_TSC_INVALID_OPERATION; + } + + STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; + if (pTableMeta) { + tscDebug("0x%"PRIx64" update table meta:%s, old meta numOfTags:%d, numOfCols:%d, uid:%" PRIu64, pSql->self, name, + tscGetNumOfTags(pTableMeta), tscGetNumOfColumns(pTableMeta), pTableMeta->id.uid); + } + + + // remove stored tableMeta info in hash table + tscResetSqlCmd(pCmd, true, pSql->self); + + SArray* pNameList = taosArrayInit(1, POINTER_BYTES); + SArray* vgroupList = taosArrayInit(1, POINTER_BYTES); + + char* n = strdup(name); + taosArrayPush(pNameList, &n); + code = getMultiTableMetaFromMnode(pSql, pNameList, vgroupList, NULL, tscTableMetaCallBack, true); + taosArrayDestroyEx(pNameList, freeElem); + taosArrayDestroyEx(vgroupList, freeElem); + + return code; +} + +static bool allVgroupInfoRetrieved(SQueryInfo* pQueryInfo) { + for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { + STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, i); + if (pTableMetaInfo->vgroupList == NULL) { + return false; + } + } + + // all super tables vgroupinfo are retrieved, no need to retrieve vgroup info anymore + return true; +} + +int tscGetSTableVgroupInfo(SSqlObj *pSql, SQueryInfo* pQueryInfo) { + int32_t code = TSDB_CODE_RPC_NETWORK_UNAVAIL; + if (allVgroupInfoRetrieved(pQueryInfo)) { + return TSDB_CODE_SUCCESS; + } + SSqlObj *pNew = calloc(1, sizeof(SSqlObj)); + pNew->pTscObj = pSql->pTscObj; + pNew->signature = pNew; + + pNew->cmd.command = TSDB_SQL_STABLEVGROUP; + + // TODO TEST IT + SQueryInfo *pNewQueryInfo = tscGetQueryInfoS(&pNew->cmd); + if (pNewQueryInfo == NULL) { + tscFreeSqlObj(pNew); + return code; + } + + for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { + STableMetaInfo *pMInfo = tscGetMetaInfo(pQueryInfo, i); + STableMeta* pTableMeta = tscTableMetaDup(pMInfo->pTableMeta); + tscAddTableMetaInfo(pNewQueryInfo, &pMInfo->name, pTableMeta, NULL, pMInfo->tagColList, pMInfo->pVgroupTables); + } + + if ((code = tscAllocPayload(&pNew->cmd, TSDB_DEFAULT_PAYLOAD_SIZE)) != TSDB_CODE_SUCCESS) { + tscFreeSqlObj(pNew); + return code; + } + + pNewQueryInfo->numOfTables = pQueryInfo->numOfTables; + registerSqlObj(pNew); + + tscDebug("0x%"PRIx64" svgroupRid from %" PRId64 " to %" PRId64 , pSql->self, pSql->svgroupRid, pNew->self); + + pSql->svgroupRid = pNew->self; + tscDebug("0x%"PRIx64" new sqlObj:%p to get vgroupInfo, numOfTables:%d", pSql->self, pNew, pNewQueryInfo->numOfTables); + + pNew->fp = tscTableMetaCallBack; + pNew->param = (void *)pSql->self; + code = tscBuildAndSendRequest(pNew, NULL); + if (code == TSDB_CODE_SUCCESS) { + code = TSDB_CODE_TSC_ACTION_IN_PROGRESS; + } + + return code; +} + +#endif + +int processConnectRsp(SRequestObj *pRequest, const char* pMsg, int32_t msgLen) { + STscObj *pTscObj = pRequest->pTscObj; + + SConnectRsp *pConnect = (SConnectRsp *)pMsg; + pConnect->acctId = htonl(pConnect->acctId); + pConnect->connId = htonl(pConnect->connId); + pConnect->clusterId = htonl(pConnect->clusterId); + + // TODO refactor + pthread_mutex_lock(&pTscObj->mutex); + char temp[TSDB_TABLE_FNAME_LEN * 2] = {0}; + int32_t len = sprintf(temp, "%s%s%s", pTscObj->acctId, TS_PATH_DELIMITER, pTscObj->db); + + assert(len <= sizeof(pTscObj->db)); + tstrncpy(pTscObj->db, temp, sizeof(pTscObj->db)); + pthread_mutex_unlock(&pTscObj->mutex); + + assert(pConnect->epSet.numOfEps > 0); + if (!isEpsetEqual(&pTscObj->pAppInfo->mgmtEp.epSet, &pConnect->epSet)) { + updateEpSet_s(&pTscObj->pAppInfo->mgmtEp, &pConnect->epSet); + } + + for (int i = 0; i < pConnect->epSet.numOfEps; ++i) { + tscDebug("0x%" PRIx64 " epSet.fqdn[%d]: %s, connObj:0x%"PRIx64, pRequest->requestId, i, pConnect->epSet.fqdn[i], pTscObj->id); + } + + pTscObj->connId = pConnect->connId; + + // update the appInstInfo + pTscObj->pAppInfo->clusterId = pConnect->clusterId; + atomic_add_fetch_64(&pTscObj->pAppInfo->numOfConns, 1); + + tscDebug("0x%" PRIx64 " clusterId:%d, totalConn:%"PRId64, pRequest->requestId, pConnect->clusterId, pTscObj->pAppInfo->numOfConns); + // createHbObj(pTscObj); + + // launch a timer to send heartbeat to maintain the connection and send status to mnode + // taosTmrReset(tscProcessActivityTimer, tsShellActivityTimer * 500, (void *)pTscObj->rid, tscTmr, &pTscObj->pTimer); + return 0; +} + +void initMsgHandleFp() { +#if 0 + tscBuildMsg[TSDB_SQL_SELECT] = tscBuildQueryMsg; + tscBuildMsg[TSDB_SQL_INSERT] = tscBuildSubmitMsg; + tscBuildMsg[TSDB_SQL_FETCH] = tscBuildFetchMsg; + + tscBuildMsg[TSDB_SQL_CREATE_DB] = tscBuildCreateDbMsg; + tscBuildMsg[TSDB_SQL_CREATE_USER] = tscBuildUserMsg; + tscBuildMsg[TSDB_SQL_CREATE_FUNCTION] = tscBuildCreateFuncMsg; + + tscBuildMsg[TSDB_SQL_CREATE_ACCT] = tscBuildAcctMsg; + tscBuildMsg[TSDB_SQL_ALTER_ACCT] = tscBuildAcctMsg; + + tscBuildMsg[TSDB_SQL_CREATE_TABLE] = tscBuildCreateTableMsg; + tscBuildMsg[TSDB_SQL_DROP_USER] = tscBuildDropUserAcctMsg; + tscBuildMsg[TSDB_SQL_DROP_ACCT] = tscBuildDropUserAcctMsg; + tscBuildMsg[TSDB_SQL_DROP_DB] = tscBuildDropDbMsg; + tscBuildMsg[TSDB_SQL_DROP_FUNCTION] = tscBuildDropFuncMsg; + tscBuildMsg[TSDB_SQL_SYNC_DB_REPLICA] = tscBuildSyncDbReplicaMsg; + tscBuildMsg[TSDB_SQL_DROP_TABLE] = tscBuildDropTableMsg; + tscBuildMsg[TSDB_SQL_ALTER_USER] = tscBuildUserMsg; + tscBuildMsg[TSDB_SQL_CREATE_DNODE] = tscBuildCreateDnodeMsg; + tscBuildMsg[TSDB_SQL_DROP_DNODE] = tscBuildDropDnodeMsg; + tscBuildMsg[TSDB_SQL_CFG_DNODE] = tscBuildCfgDnodeMsg; + tscBuildMsg[TSDB_SQL_ALTER_TABLE] = tscBuildAlterTableMsg; + tscBuildMsg[TSDB_SQL_UPDATE_TAGS_VAL] = tscBuildUpdateTagMsg; + tscBuildMsg[TSDB_SQL_ALTER_DB] = tscAlterDbMsg; + tscBuildMsg[TSDB_SQL_COMPACT_VNODE] = tscBuildCompactMsg; + + + tscBuildMsg[TSDB_SQL_USE_DB] = tscBuildUseDbMsg; + tscBuildMsg[TSDB_SQL_STABLEVGROUP] = tscBuildSTableVgroupMsg; + tscBuildMsg[TSDB_SQL_RETRIEVE_FUNC] = tscBuildRetrieveFuncMsg; + + tscBuildMsg[TSDB_SQL_HB] = tscBuildHeartBeatMsg; + tscBuildMsg[TSDB_SQL_SHOW] = tscBuildShowMsg; + tscBuildMsg[TSDB_SQL_RETRIEVE] = tscBuildRetrieveFromMgmtMsg; + tscBuildMsg[TSDB_SQL_KILL_QUERY] = tscBuildKillMsg; + tscBuildMsg[TSDB_SQL_KILL_STREAM] = tscBuildKillMsg; + tscBuildMsg[TSDB_SQL_KILL_CONNECTION] = tscBuildKillMsg; + + tscProcessMsgRsp[TSDB_SQL_SELECT] = tscProcessQueryRsp; + tscProcessMsgRsp[TSDB_SQL_FETCH] = tscProcessRetrieveRspFromNode; + + tscProcessMsgRsp[TSDB_SQL_DROP_DB] = tscProcessDropDbRsp; + tscProcessMsgRsp[TSDB_SQL_DROP_TABLE] = tscProcessDropTableRsp; + + tscProcessMsgRsp[TSDB_SQL_USE_DB] = tscProcessUseDbRsp; + tscProcessMsgRsp[TSDB_SQL_META] = tscProcessTableMetaRsp; + tscProcessMsgRsp[TSDB_SQL_STABLEVGROUP] = tscProcessSTableVgroupRsp; + tscProcessMsgRsp[TSDB_SQL_MULTI_META] = tscProcessMultiTableMetaRsp; + tscProcessMsgRsp[TSDB_SQL_RETRIEVE_FUNC] = tscProcessRetrieveFuncRsp; + + tscProcessMsgRsp[TSDB_SQL_SHOW] = tscProcessShowRsp; + tscProcessMsgRsp[TSDB_SQL_RETRIEVE] = tscProcessRetrieveRspFromNode; // rsp handled by same function. + tscProcessMsgRsp[TSDB_SQL_DESCRIBE_TABLE] = tscProcessDescribeTableRsp; + + tscProcessMsgRsp[TSDB_SQL_CURRENT_DB] = tscProcessLocalRetrieveRsp; + tscProcessMsgRsp[TSDB_SQL_CURRENT_USER] = tscProcessLocalRetrieveRsp; + tscProcessMsgRsp[TSDB_SQL_SERV_VERSION] = tscProcessLocalRetrieveRsp; + tscProcessMsgRsp[TSDB_SQL_CLI_VERSION] = tscProcessLocalRetrieveRsp; + tscProcessMsgRsp[TSDB_SQL_SERV_STATUS] = tscProcessLocalRetrieveRsp; + + tscProcessMsgRsp[TSDB_SQL_RETRIEVE_EMPTY_RESULT] = tscProcessEmptyResultRsp; + + tscProcessMsgRsp[TSDB_SQL_RETRIEVE_GLOBALMERGE] = tscProcessRetrieveGlobalMergeRsp; + + tscProcessMsgRsp[TSDB_SQL_ALTER_TABLE] = tscProcessAlterTableMsgRsp; + tscProcessMsgRsp[TSDB_SQL_ALTER_DB] = tscProcessAlterDbMsgRsp; + tscProcessMsgRsp[TSDB_SQL_COMPACT_VNODE] = tscProcessCompactRsp; + + tscProcessMsgRsp[TSDB_SQL_SHOW_CREATE_TABLE] = tscProcessShowCreateRsp; + tscProcessMsgRsp[TSDB_SQL_SHOW_CREATE_STABLE] = tscProcessShowCreateRsp; + tscProcessMsgRsp[TSDB_SQL_SHOW_CREATE_DATABASE] = tscProcessShowCreateRsp; +#endif + +// buildRequestMsgFp[TSDB_SQL_CONNECT] = tscBuildConnectMsg; + handleRequestRspFp[TSDB_SQL_CONNECT] = processConnectRsp; +} \ No newline at end of file From fb5db901d70beb5670e309d23a2874586966a5c4 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 14 Dec 2021 15:13:10 +0800 Subject: [PATCH 18/43] [td-10564] fix memory leak. --- source/client/src/clientImpl.c | 9 +++++++++ source/client/src/tscEnv.c | 1 + 2 files changed, 10 insertions(+) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 47b2c61ffc..2ae79f1947 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -9,6 +9,8 @@ static int32_t initEpSetFromCfg(const char *firstEp, const char *secondEp, SCorEpSet *pEpSet); static int32_t buildConnectMsg(SRequestObj *pRequest, SRequestMsgBody* pMsgBody); +static void destroyConnectMsg(SRequestMsgBody* pMsgBody); + static int32_t sendMsgToServer(void *pTransporter, SEpSet* epSet, const SRequestMsgBody *pBody, int64_t* pTransporterId); static bool stringLengthCheck(const char* str, size_t maxsize) { @@ -164,6 +166,8 @@ STscObj* taosConnectImpl(const char *ip, const char *user, const char *auth, con sendMsgToServer(pTscObj->pTransporter, &pTscObj->pAppInfo->mgmtEp.epSet, &body, &transporterId); tsem_wait(&pRequest->body.rspSem); + destroyConnectMsg(&body); + if (pRequest->code != TSDB_CODE_SUCCESS) { const char *errorMsg = (pRequest->code == TSDB_CODE_RPC_FQDN_ERROR) ? taos_errstr(pRequest) : tstrerror(terrno); printf("failed to connect to server, reason: %s\n\n", errorMsg); @@ -209,6 +213,11 @@ static int32_t buildConnectMsg(SRequestObj *pRequest, SRequestMsgBody* pMsgBody) return 0; } +static void destroyConnectMsg(SRequestMsgBody* pMsgBody) { + assert(pMsgBody != NULL); + tfree(pMsgBody->pData); +} + int32_t sendMsgToServer(void *pTransporter, SEpSet* epSet, const SRequestMsgBody *pBody, int64_t* pTransporterId) { char *pMsg = rpcMallocCont(pBody->msgLen); if (NULL == pMsg) { diff --git a/source/client/src/tscEnv.c b/source/client/src/tscEnv.c index d37eb1ebcb..43d73bf3db 100644 --- a/source/client/src/tscEnv.c +++ b/source/client/src/tscEnv.c @@ -189,6 +189,7 @@ static void doDestroyRequest(void* p) { tfree(pRequest->pInfo); deregisterRequest(pRequest); + tfree(pRequest); } void destroyRequest(SRequestObj* pRequest) { From a352b93570689b954d8514c86678b95d9ce08eb1 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 14 Dec 2021 15:24:21 +0800 Subject: [PATCH 19/43] update catalog and message --- include/common/taosmsg.h | 19 +- include/common/tmessage.h | 32 +++ include/libs/catalog/catalog.h | 39 ++-- include/util/taoserror.h | 2 +- include/util/tlog.h | 2 + source/client/CMakeLists.txt | 1 + source/client/src/client.c | 3 + source/common/inc/commonInt.h | 16 +- source/common/src/taosmsg.c | 20 ++ source/common/src/tglobal.c | 1 + source/common/src/tmessage.c | 147 ++++++++------ source/dnode/mgmt/impl/src/dndTransport.c | 2 +- source/libs/catalog/inc/catalogInt.h | 26 ++- source/libs/catalog/src/catalog.c | 228 ++++++++++++++++++++-- source/libs/parser/src/astValidate.c | 2 +- source/libs/parser/src/parser.c | 7 +- source/libs/transport/src/rpcMain.c | 2 +- source/util/src/terror.c | 2 +- source/util/src/tlog.c | 2 + 19 files changed, 441 insertions(+), 112 deletions(-) create mode 100644 include/common/tmessage.h create mode 100644 source/common/src/taosmsg.c diff --git a/include/common/taosmsg.h b/include/common/taosmsg.h index fd55a11b52..906dd38c51 100644 --- a/include/common/taosmsg.h +++ b/include/common/taosmsg.h @@ -77,7 +77,7 @@ TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DROP_FUNCTION, "drop-function" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CREATE_STABLE, "create-stable" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_ALTER_STABLE, "alter-stable" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DROP_STABLE, "drop-stable" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_STABLE_VGROUP, "stable-vgroup" ) +TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_VGROUP_LIST, "vgroup-list" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_KILL_QUERY, "kill-query" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_KILL_STREAM, "kill-stream" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_KILL_CONN, "kill-conn" ) @@ -216,7 +216,6 @@ extern char *taosMsg[]; typedef struct SBuildTableMetaInput { int32_t vgId; - STagData *tagData; char *tableFullName; } SBuildTableMetaInput; @@ -776,8 +775,6 @@ typedef struct { typedef struct { SMsgHead msgHead; char tableFname[TSDB_TABLE_FNAME_LEN]; - int8_t createFlag; - char tags[]; } STableInfoMsg; typedef struct { @@ -792,6 +789,20 @@ typedef struct SSTableVgroupMsg { int32_t numOfTables; } SSTableVgroupMsg, SSTableVgroupRspMsg; +typedef struct SVgroupInfo { + int32_t vgId; + int8_t numOfEps; + SEpAddrMsg epAddr[TSDB_MAX_REPLICA]; +} SVgroupInfo; + +typedef struct SVgroupListRspMsg { + int32_t vgroupNum; + int32_t vgroupVersion; + SVgroupInfo vgroupInfo[]; +} SVgroupListRspMsg; + +typedef SVgroupListRspMsg SVgroupListInfo; + typedef struct { int32_t vgId; int8_t numOfEps; diff --git a/include/common/tmessage.h b/include/common/tmessage.h new file mode 100644 index 0000000000..c728ee026e --- /dev/null +++ b/include/common/tmessage.h @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_COMMON_TMESSAGE_H_ +#define _TD_COMMON_TMESSAGE_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +extern int32_t (*tscBuildMsg[TSDB_MSG_TYPE_MAX])(void* input, char **msg, int32_t msgSize, int32_t *msgLen); +extern int32_t (*tscProcessMsgRsp[TSDB_MSG_TYPE_MAX])(void* output, char *msg, int32_t msgSize); + + + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_COMMON_TMESSAGE_H_*/ diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index 2092f53ba1..2ceb893ce4 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -30,12 +30,6 @@ extern "C" { struct SCatalog; -typedef struct SVgroupInfo { - int32_t vgId; - int8_t numOfEps; - SEpAddrMsg epAddr[TSDB_MAX_REPLICA]; -} SVgroupInfo; - typedef struct SDBVgroupInfo { int32_t vgroupVersion; SArray *vgId; @@ -91,7 +85,11 @@ typedef struct STableMeta { SSchema schema[]; } STableMeta; -int32_t catalogInit(SCatalog *cfg); +typedef struct SCatalogCfg { + +} SCatalogCfg; + +int32_t catalogInit(SCatalogCfg *cfg); /** * Catalog service object, which is utilized to hold tableMeta (meta/vgroupInfo/udfInfo) at the client-side. @@ -101,17 +99,31 @@ int32_t catalogInit(SCatalog *cfg); */ int32_t catalogGetHandle(const char *clusterId, struct SCatalog** catalogHandle); -int32_t catalogGetVgroupVersion(struct SCatalog* pCatalog, int32_t* version); -int32_t catalogUpdateVgroupList(struct SCatalog* pCatalog, int32_t version, SArray* vgroupList); + +int32_t catalogGetVgroupVersion(struct SCatalog* pCatalog, int32_t* version); +int32_t catalogGetVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, SArray** pVgroupList); +int32_t catalogUpdateVgroup(struct SCatalog* pCatalog, SVgroupListInfo* pVgroup); + + int32_t catalogGetDBVgroupVersion(struct SCatalog* pCatalog, const char* dbName, int32_t* version); +int32_t catalogGetDBVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* dbName, int32_t forceUpdate, SDBVgroupInfo* dbInfo); +int32_t catalogUpdateDBVgroup(struct SCatalog* pCatalog, const char* dbName, SDBVgroupInfo* dbInfo); -int32_t catalogGetDBVgroupInfo(struct SCatalog* pCatalog, const char* dbName, SDBVgroupInfo* dbInfo); -int32_t catalogUpdateDBVgroupInfo(struct SCatalog* pCatalog, const char* dbName, SDBVgroupInfo* dbInfo); +int32_t catalogGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pTableName, STableMeta* pTableMeta); +int32_t catalogRenewTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const STableMeta* pTableMeta); +int32_t catalogRenewAndGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const STableMeta* pTableMeta, STableMeta* pNewTableMeta); -int32_t catalogGetTableMeta(struct SCatalog* pCatalog, SRpcObj *pRpcObj, const SEpSet* pMgmtEps, const char* pTableName, const STagData* tagData, STableMeta* pTableMeta); + +/** + * get table's vgroup list. + * @param clusterId + * @pVgroupList - array of SVgroupInfo + * @return + */ +int32_t catalogGetTableVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pTableName, SArray* pVgroupList); /** @@ -125,9 +137,6 @@ int32_t catalogGetTableMeta(struct SCatalog* pCatalog, SRpcObj *pRpcObj, const S */ int32_t catalogGetAllMeta(struct SCatalog* pCatalog, const SEpSet* pMgmtEps, const SCatalogReq* pReq, SCatalogRsp* pRsp); -int32_t catalogRenewTableMeta(struct SCatalog* pCatalog, const SEpSet* pMgmtEps, const STableMeta* pTableMeta); - -int32_t catalogRenewAndGetTableMeta(struct SCatalog* pCatalog, const SEpSet* pMgmtEps, const STableMeta* pTableMeta, STableMeta* pNewTableMeta); int32_t catalogGetQnodeList(struct SCatalog* pCatalog, const SEpSet* pMgmtEps, SEpSet* pQnodeEpSet); diff --git a/include/util/taoserror.h b/include/util/taoserror.h index a3a4297115..e1a1694fab 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -492,7 +492,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MON_CONNECTION_INVALID TAOS_DEF_ERROR_CODE(0, 0x2300) //"monitor invalid monitor db connection") // catalog -#define TSDB_CODE_CTG_INTERNAL_EROR TAOS_DEF_ERROR_CODE(0, 0x2400) //catalog interval error +#define TSDB_CODE_CTG_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x2400) //catalog interval error #define TSDB_CODE_CTG_INVALID_INPUT TAOS_DEF_ERROR_CODE(0, 0x2401) //invalid catalog input parameters #define TSDB_CODE_CTG_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x2402) //catalog is not ready #define TSDB_CODE_CTG_MEM_ERROR TAOS_DEF_ERROR_CODE(0, 0x2403) //catalog memory error diff --git a/include/util/tlog.h b/include/util/tlog.h index 2ee60e4324..9b6033e7fe 100644 --- a/include/util/tlog.h +++ b/include/util/tlog.h @@ -45,6 +45,8 @@ extern int32_t sDebugFlag; extern int32_t tsdbDebugFlag; extern int32_t cqDebugFlag; extern int32_t debugFlag; +extern int32_t ctgDebugFlag; + #define DEBUG_FATAL 1U #define DEBUG_ERROR DEBUG_FATAL diff --git a/source/client/CMakeLists.txt b/source/client/CMakeLists.txt index bc0d439407..bc18e41664 100644 --- a/source/client/CMakeLists.txt +++ b/source/client/CMakeLists.txt @@ -6,5 +6,6 @@ target_include_directories( ) target_link_libraries( taos + PRIVATE common INTERFACE api ) diff --git a/source/client/src/client.c b/source/client/src/client.c index b1663239e6..258fd91565 100644 --- a/source/client/src/client.c +++ b/source/client/src/client.c @@ -18,6 +18,9 @@ //TAOS_RES *taos_query(TAOS *taos, const char *sql) { // //} +#include "taosmsg.h" int taos_init() { return 0; } void taos_cleanup(void) {} + + diff --git a/source/common/inc/commonInt.h b/source/common/inc/commonInt.h index b8be8899f3..c1baa2a453 100644 --- a/source/common/inc/commonInt.h +++ b/source/common/inc/commonInt.h @@ -20,8 +20,22 @@ extern "C" { #endif + +#include "tlog.h" + +extern int32_t cDebugFlag; +extern int8_t tscEmbedded; + +#define tscFatal(...) do { if (cDebugFlag & DEBUG_FATAL) { taosPrintLog("TSC FATAL ", tscEmbedded ? 255 : cDebugFlag, __VA_ARGS__); }} while(0) +#define tscError(...) do { if (cDebugFlag & DEBUG_ERROR) { taosPrintLog("TSC ERROR ", tscEmbedded ? 255 : cDebugFlag, __VA_ARGS__); }} while(0) +#define tscWarn(...) do { if (cDebugFlag & DEBUG_WARN) { taosPrintLog("TSC WARN ", tscEmbedded ? 255 : cDebugFlag, __VA_ARGS__); }} while(0) +#define tscInfo(...) do { if (cDebugFlag & DEBUG_INFO) { taosPrintLog("TSC ", tscEmbedded ? 255 : cDebugFlag, __VA_ARGS__); }} while(0) +#define tscDebug(...) do { if (cDebugFlag & DEBUG_DEBUG) { taosPrintLog("TSC ", cDebugFlag, __VA_ARGS__); }} while(0) +#define tscTrace(...) do { if (cDebugFlag & DEBUG_TRACE) { taosPrintLog("TSC ", cDebugFlag, __VA_ARGS__); }} while(0) +#define tscDebugL(...) do { if (cDebugFlag & DEBUG_DEBUG) { taosPrintLongString("TSC ", cDebugFlag, __VA_ARGS__); }} while(0) + #ifdef __cplusplus } #endif -#endif /*_TD_COMMON_INT_H_*/ \ No newline at end of file +#endif /*_TD_COMMON_INT_H_*/ diff --git a/source/common/src/taosmsg.c b/source/common/src/taosmsg.c new file mode 100644 index 0000000000..b35e3f1478 --- /dev/null +++ b/source/common/src/taosmsg.c @@ -0,0 +1,20 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define TAOS_MESSAGE_C + +#include "taosmsg.h" + + diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 3cf5e52c44..60e8415392 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -1715,3 +1715,4 @@ bool taosCheckBalanceCfgOptions(const char *option, int32_t *vnodeId, int32_t *d return true; } + \ No newline at end of file diff --git a/source/common/src/tmessage.c b/source/common/src/tmessage.c index 0e732caa26..07adf1d599 100644 --- a/source/common/src/tmessage.c +++ b/source/common/src/tmessage.c @@ -13,21 +13,103 @@ * along with this program. If not, see . */ -#define TAOS_MESSAGE_C - #include "taosmsg.h" +#include "commonint.h" + int32_t (*tscBuildMsg[TSDB_MSG_TYPE_MAX])(void* input, char **msg, int32_t msgSize, int32_t *msgLen) = {0}; int32_t (*tscProcessMsgRsp[TSDB_MSG_TYPE_MAX])(void* output, char *msg, int32_t msgSize) = {0}; +int32_t tscBuildVgroupListReqMsg(void* input, char **msg, int32_t msgSize, int32_t *msgLen) { + if (NULL == msg || NULL == msgLen) { + return TSDB_CODE_TSC_INVALID_INPUT; + } + + *msgLen = 0; + + return TSDB_CODE_SUCCESS; +} + +int32_t tscBuildTableMetaReqMsg(void* input, char **msg, int32_t msgSize, int32_t *msgLen) { + if (NULL == input || NULL == msg || NULL == msgLen) { + return TSDB_CODE_TSC_INVALID_INPUT; + } + + SBuildTableMetaInput* bInput = (SBuildTableMetaInput *)input; + + int32_t estimateSize = sizeof(STableInfoMsg); + if (NULL == *msg || msgSize < estimateSize) { + tfree(*msg); + *msg = calloc(1, estimateSize); + if (NULL == *msg) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + } + + STableInfoMsg *bMsg = (STableInfoMsg *)*msg; + + bMsg->msgHead.vgId = bInput->vgId; + + strncpy(bMsg->tableFname, bInput->tableFullName, sizeof(bMsg->tableFname)); + bMsg->tableFname[sizeof(bMsg->tableFname) - 1] = 0; + + *msgLen = (int32_t)sizeof(*bMsg); + + return TSDB_CODE_SUCCESS; +} + + +int32_t tscProcessVgroupListRsp(void* output, char *msg, int32_t msgSize) { + if (NULL == output || NULL == msg || msgSize <= 0) { + return TSDB_CODE_TSC_INVALID_INPUT; + } + + SVgroupListRspMsg *pRsp = (SVgroupListRspMsg *)msg; + + pRsp->vgroupNum = htonl(pRsp->vgroupNum); + pRsp->vgroupVersion = htonl(pRsp->vgroupVersion); + + if (pRsp->vgroupNum < 0) { + tscError("vgroup number[%d] in rsp is invalid", pRsp->vgroupNum); + return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; + } + + if (pRsp->vgroupVersion < 0) { + tscError("vgroup vgroupVersion[%d] in rsp is invalid", pRsp->vgroupVersion); + return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; + } + + if (msgSize != (pRsp->vgroupNum * sizeof(pRsp->vgroupInfo[0]) + sizeof(*pRsp))) { + tscError("vgroup list msg size mis-match, msgSize:%d, vgroup number:%d", msgSize, pRsp->vgroupNum); + return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; + } + + // keep SVgroupListInfo/SVgroupListRspMsg the same + *(SVgroupListInfo **)output = (SVgroupListInfo *)msg; + + if (pRsp->vgroupNum == 0) { + return TSDB_CODE_SUCCESS; + } + + for (int32_t i = 0; i < pRsp->vgroupNum; ++i) { + pRsp->vgroupInfo[i].vgId = htonl(pRsp->vgroupInfo[i].vgId); + for (int32_t n = 0; n < pRsp->vgroupInfo[i].numOfEps; ++n) { + pRsp->vgroupInfo[i].epAddr[n].port = htonl(pRsp->vgroupInfo[i].epAddr[n].port); + } + } + + return TSDB_CODE_SUCCESS; +} + void msgInit() { tscBuildMsg[TSDB_MSG_TYPE_TABLE_META] = tscBuildTableMetaReqMsg; + tscBuildMsg[TSDB_MSG_TYPE_VGROUP_LIST] = tscBuildVgroupListReqMsg; - - tscProcessMsgRsp[TSDB_MSG_TYPE_TABLE_META] = ; + //tscProcessMsgRsp[TSDB_MSG_TYPE_TABLE_META] = tscProcessTableMetaRsp; + tscProcessMsgRsp[TSDB_MSG_TYPE_VGROUP_LIST] = tscProcessVgroupListRsp; /* tscBuildMsg[TSDB_SQL_SELECT] = tscBuildQueryMsg; @@ -106,63 +188,6 @@ void msgInit() { } -char* msgSerializeTagData(STagData* pTagData, char* pMsg) { - int32_t n = (int32_t) strlen(pTagData->name); - *(int32_t*) pMsg = htonl(n); - pMsg += sizeof(n); - - memcpy(pMsg, pTagData->name, n); - pMsg += n; - - *(int32_t*)pMsg = htonl(pTagData->dataLen); - pMsg += sizeof(int32_t); - - memcpy(pMsg, pTagData->data, pTagData->dataLen); - pMsg += pTagData->dataLen; - - return pMsg; -} - - -int32_t tscBuildTableMetaReqMsg(void* input, char **msg, int32_t msgSize, int32_t *msgLen) { - if (NULL == input || NULL == msg || NULL == msgLen) { - return TSDB_CODE_TSC_INVALID_INPUT; - } - - SBuildTableMetaInput* bInput = (SBuildTableMetaInput *)input; - - int32_t estimateSize = sizeof(STableInfoMsg) + (bInput->tagData ? (sizeof(*bInput->tagData) + bInput->tagData->dataLen) : 0); - if (NULL == *msg || msgSize < estimateSize) { - tfree(*msg); - *msg = calloc(1, estimateSize); - if (NULL == *msg) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - } - - STableInfoMsg *bMsg = (STableInfoMsg *)*msg; - - bMsg->msgHead.vgId = bInput->vgId; - - strncpy(bMsg->tableFname, bInput->tableFullName, sizeof(bMsg->tableFname)); - bMsg->tableFname[sizeof(bMsg->tableFname) - 1] = 0; - - int32_t autoCreate = (bInput->tagData && bInput->tagData->dataLen > 0); - - bMsg->createFlag = htons(autoCreate ? 1 : 0); - - char *pMsg = NULL; - - // tag data exists - if (autoCreate) { - pMsg = msgSerializeTagData(bInput->tagData, (char *)bMsg->tags); - } - - *msgLen = (int32_t)(pMsg - (char*)bMsg); - - return TSDB_CODE_SUCCESS; -} - diff --git a/source/dnode/mgmt/impl/src/dndTransport.c b/source/dnode/mgmt/impl/src/dndTransport.c index a36c56ea3d..0f8281a3a6 100644 --- a/source/dnode/mgmt/impl/src/dndTransport.c +++ b/source/dnode/mgmt/impl/src/dndTransport.c @@ -72,7 +72,7 @@ static void dndInitMsgFp(STransMgmt *pMgmt) { pMgmt->msgFp[TSDB_MSG_TYPE_CREATE_STABLE] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TSDB_MSG_TYPE_ALTER_STABLE] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TSDB_MSG_TYPE_DROP_STABLE] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_STABLE_VGROUP] = dndProcessMnodeReadMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_VGROUP_LIST] = dndProcessMnodeReadMsg; pMgmt->msgFp[TSDB_MSG_TYPE_KILL_QUERY] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TSDB_MSG_TYPE_KILL_STREAM] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TSDB_MSG_TYPE_KILL_CONN] = dndProcessMnodeWriteMsg; diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 82d7d9c571..720f197782 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -21,13 +21,18 @@ extern "C" { #endif #include "catalog.h" +#include "common.h" +#include "tlog.h" -#define CTG_DEFAULT_CLUSTER_NUMBER 3 +#define CTG_DEFAULT_CLUSTER_NUMBER 6 +#define CTG_DEFAULT_VGROUP_NUMBER 100 + +#define CTG_DEFAULT_INVALID_VERSION (-1) typedef struct SVgroupListCache { - int32_t vgroupNum; int32_t vgroupVersion; - SHashObj *cache; //key:vgId, value:SVgroupInfo + SHashObj *cache; // key:vgId, value:SVgroupInfo* + SArray *arrayCache; // SVgroupInfo } SVgroupListCache; typedef struct SDBVgroupCache { @@ -50,13 +55,16 @@ typedef struct SCatalogMgmt { } SCatalogMgmt; +extern int32_t ctgDebugFlag; + +#define ctgFatal(...) do { if (ctgDebugFlag & DEBUG_FATAL) { taosPrintLog("CTG FATAL ", ctgDebugFlag, __VA_ARGS__); }} while(0) +#define ctgError(...) do { if (ctgDebugFlag & DEBUG_ERROR) { taosPrintLog("CTG ERROR ", ctgDebugFlag, __VA_ARGS__); }} while(0) +#define ctgWarn(...) do { if (ctgDebugFlag & DEBUG_WARN) { taosPrintLog("CTG WARN ", ctgDebugFlag, __VA_ARGS__); }} while(0) +#define ctgInfo(...) do { if (ctgDebugFlag & DEBUG_INFO) { taosPrintLog("CTG ", ctgDebugFlag, __VA_ARGS__); }} while(0) +#define ctgDebug(...) do { if (ctgDebugFlag & DEBUG_DEBUG) { taosPrintLog("CTG ", ctgDebugFlag, __VA_ARGS__); }} while(0) +#define ctgTrace(...) do { if (ctgDebugFlag & DEBUG_TRACE) { taosPrintLog("CTG ", ctgDebugFlag, __VA_ARGS__); }} while(0) +#define ctgDebugL(...) do { if (ctgDebugFlag & DEBUG_DEBUG) { taosPrintLongString("CTG ", ctgDebugFlag, __VA_ARGS__); }} while(0) -#define ctgFatal(...) tscFatal(__VA_ARGS__) -#define ctgError(...) tscError(__VA_ARGS__) -#define ctgWarn(...) tscWarn(__VA_ARGS__) -#define ctgInfo(...) tscInfo(__VA_ARGS__) -#define ctgDebug(...) tscDebug(__VA_ARGS__) -#define ctgTrace(...) tscTrace(__VA_ARGS__) #define CTG_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { return _code; } } while (0) #define CTG_ERR_LRET(c,...) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { ctgError(__VA_ARGS__); return _code; } } while (0) diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index e8b79bae4b..3837e5408a 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -14,53 +14,248 @@ */ #include "catalogInt.h" +#include "trpc.h" +#include "tmessage.h" SCatalogMgmt ctgMgmt = {0}; +int32_t ctgGetVgroupFromMnode(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, SVgroupListInfo** pVgroup) { + char *msg = NULL; + SEpSet *pVnodeEpSet = NULL; + int32_t msgLen = 0; -int32_t catalogInit(SCatalog *cfg) { + int32_t code = tscBuildMsg[TSDB_MSG_TYPE_VGROUP_LIST](NULL, &msg, 0, &msgLen); + if (code) { + return code; + } + + SRpcMsg rpcMsg = { + .msgType = TSDB_MSG_TYPE_VGROUP_LIST, + .pCont = msg, + .contLen = msgLen, + }; + + SRpcMsg rpcRsp = {0}; + + rpcSendRecv(pRpc, (SEpSet*)pMgmtEps, &rpcMsg, &rpcRsp); + + code = tscProcessMsgRsp[TSDB_MSG_TYPE_VGROUP_LIST](pVgroup, rpcRsp.pCont, rpcRsp.contLen); + if (code) { + return code; + } + + return TSDB_CODE_SUCCESS; +} + +int32_t ctgGetVgroupFromCache(SCatalog* pCatalog, SArray** pVgroupList, int32_t* exist) { + if (NULL == pCatalog->vgroupCache.arrayCache || pCatalog->vgroupCache.vgroupVersion < 0) { + *exist = 0; + return TSDB_CODE_SUCCESS; + } + + if (pVgroupList) { + *pVgroupList = taosArrayDup(pCatalog->vgroupCache.arrayCache); + } + + *exist = 1; + + return TSDB_CODE_SUCCESS; +} + + + + + +int32_t catalogInit(SCatalogCfg *cfg) { ctgMgmt.pCluster = taosHashInit(CTG_DEFAULT_CLUSTER_NUMBER, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); if (NULL == ctgMgmt.pCluster) { - CTG_ERR_LRET(TSDB_CODE_CTG_INTERNAL_EROR, "init %d cluster cache failed", CTG_DEFAULT_CLUSTER_NUMBER); + CTG_ERR_LRET(TSDB_CODE_CTG_INTERNAL_ERROR, "init %d cluster cache failed", CTG_DEFAULT_CLUSTER_NUMBER); } return TSDB_CODE_SUCCESS; } -struct SCatalog* catalogGetHandle(const char *clusterId) { - if (NULL == clusterId) { - return NULL; +int32_t catalogGetHandle(const char *clusterId, struct SCatalog** catalogHandle) { + if (NULL == clusterId || NULL == catalogHandle) { + return TSDB_CODE_CTG_INVALID_INPUT; } if (NULL == ctgMgmt.pCluster) { ctgError("cluster cache are not ready"); - return NULL; + return TSDB_CODE_CTG_NOT_READY; } size_t clen = strlen(clusterId); SCatalog *clusterCtg = (SCatalog *)taosHashGet(ctgMgmt.pCluster, clusterId, clen); if (clusterCtg) { - return clusterCtg; + *catalogHandle = clusterCtg; + return TSDB_CODE_SUCCESS; } clusterCtg = calloc(1, sizeof(*clusterCtg)); if (NULL == clusterCtg) { - ctgError("calloc %d failed", sizeof(*clusterCtg)); - return NULL; + ctgError("calloc %d failed", (int32_t)sizeof(*clusterCtg)); + return TSDB_CODE_CTG_MEM_ERROR; } + clusterCtg->vgroupCache.vgroupVersion = CTG_DEFAULT_INVALID_VERSION; + if (taosHashPut(ctgMgmt.pCluster, clusterId, clen, &clusterCtg, POINTER_BYTES)) { ctgError("put cluster %s cache to hash failed", clusterId); tfree(clusterCtg); - return NULL; + return TSDB_CODE_CTG_INTERNAL_ERROR; } + + *catalogHandle = clusterCtg; - return clusterCtg; + return TSDB_CODE_SUCCESS; } -int32_t catalogGetTableMeta(struct SCatalog* pCatalog, SRpcObj *pRpcObj, const SEpSet* pMgmtEps, const char* pTableName, const STagData* tagData, STableMeta* pTableMeta) { + +int32_t catalogGetVgroupVersion(struct SCatalog* pCatalog, int32_t* version) { + if (NULL == pCatalog || NULL == version) { + return TSDB_CODE_CTG_INVALID_INPUT; + } + + *version = pCatalog->vgroupCache.vgroupVersion; + + return TSDB_CODE_SUCCESS; +} + + + +int32_t catalogUpdateVgroup(struct SCatalog* pCatalog, SVgroupListInfo* pVgroup) { + if (NULL == pVgroup) { + ctgError("vgroup get from mnode succeed, but no output"); + return TSDB_CODE_CTG_INTERNAL_ERROR; + } + + if (pVgroup->vgroupVersion < 0) { + ctgError("vgroup version[%d] is invalid", pVgroup->vgroupVersion); + return TSDB_CODE_CTG_INVALID_INPUT; + } + + + if (NULL == pCatalog->vgroupCache.arrayCache) { + pCatalog->vgroupCache.arrayCache = taosArrayInit(pVgroup->vgroupNum, sizeof(pVgroup->vgroupInfo[0])); + if (NULL == pCatalog->vgroupCache.arrayCache) { + ctgError("init array[%d] for cluster cache failed", pVgroup->vgroupNum); + return TSDB_CODE_CTG_MEM_ERROR; + } + } else { + taosArrayClear(pCatalog->vgroupCache.arrayCache); + } + + if (NULL == pCatalog->vgroupCache.cache) { + pCatalog->vgroupCache.cache = taosHashInit(CTG_DEFAULT_VGROUP_NUMBER, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); + if (NULL == pCatalog->vgroupCache.cache) { + ctgError("init hash[%d] for cluster cache failed", CTG_DEFAULT_VGROUP_NUMBER); + return TSDB_CODE_CTG_MEM_ERROR; + } + } else { + taosHashClear(pCatalog->vgroupCache.cache); + } + + SVgroupInfo *vInfo = NULL; + for (int32_t i = 0; i < pVgroup->vgroupNum; ++i) { + vInfo = taosArrayPush(pCatalog->vgroupCache.arrayCache, &pVgroup->vgroupInfo[i]); + if (NULL == vInfo) { + ctgError("push to vgroup array cache failed"); + goto error_exit; + } + + if (taosHashPut(pCatalog->vgroupCache.cache, &pVgroup->vgroupInfo[i].vgId, sizeof(pVgroup->vgroupInfo[i].vgId), &vInfo, POINTER_BYTES) != 0) { + ctgError("push to vgroup hash cache failed"); + goto error_exit; + } + } + + pCatalog->vgroupCache.vgroupVersion = pVgroup->vgroupVersion; + + return TSDB_CODE_SUCCESS; + +error_exit: + if (pCatalog->vgroupCache.arrayCache) { + taosArrayDestroy(pCatalog->vgroupCache.arrayCache); + pCatalog->vgroupCache.arrayCache = NULL; + } + + if (pCatalog->vgroupCache.cache) { + taosHashCleanup(pCatalog->vgroupCache.cache); + pCatalog->vgroupCache.cache = NULL; + } + + pCatalog->vgroupCache.vgroupVersion = CTG_DEFAULT_INVALID_VERSION; + + return TSDB_CODE_CTG_INTERNAL_ERROR; +} + + +int32_t catalogGetVgroup(SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, SArray** pVgroupList) { + if (NULL == pCatalog || NULL == pMgmtEps || NULL == pRpc) { + return TSDB_CODE_CTG_INVALID_INPUT; + } + + int32_t exist = 0; + + CTG_ERR_RET(ctgGetVgroupFromCache(pCatalog, pVgroupList, &exist)); + + if (exist) { + return TSDB_CODE_SUCCESS; + } + + SVgroupListInfo *pVgroup = NULL; + + CTG_ERR_RET(ctgGetVgroupFromMnode(pCatalog, pRpc, pMgmtEps, &pVgroup)); + + CTG_ERR_RET(catalogUpdateVgroup(pCatalog, pVgroup)); + + if (pVgroupList) { + CTG_ERR_RET(ctgGetVgroupFromCache(pCatalog, pVgroupList, &exist)); + } + + if (0 == exist) { + ctgError("catalog fetched but get from cache failed"); + return TSDB_CODE_CTG_INTERNAL_ERROR; + } + + return TSDB_CODE_SUCCESS; +} + +int32_t catalogGetDBVgroupVersion(struct SCatalog* pCatalog, const char* dbName, int32_t* version) { + if (NULL == pCatalog || NULL == dbName || NULL == version) { + return TSDB_CODE_CTG_INVALID_INPUT; + } + + if (NULL == pCatalog->dbCache.cache) { + *version = CTG_DEFAULT_INVALID_VERSION; + return TSDB_CODE_SUCCESS; + } + + SDBVgroupInfo * dbInfo = taosHashGet(pCatalog->dbCache.cache, dbName, strlen(dbName)); + if (NULL == dbInfo) { + *version = CTG_DEFAULT_INVALID_VERSION; + return TSDB_CODE_SUCCESS; + } + + *version = dbInfo->vgroupVersion; + + return TSDB_CODE_SUCCESS; +} + +int32_t catalogGetDBVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* dbName, int32_t forceUpdate, SDBVgroupInfo* dbInfo) { + +} + +int32_t catalogUpdateDBVgroup(struct SCatalog* pCatalog, const char* dbName, SDBVgroupInfo* dbInfo) { + +} + + + +int32_t catalogGetTableMetaFromMnode(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pTableName, const STagData* tagData, STableMeta* pTableMeta) { if (NULL == pCatalog || NULL == pMgmtEps || NULL == pTableName || NULL == pTableMeta) { return TSDB_CODE_CTG_INVALID_INPUT; } @@ -79,12 +274,13 @@ int32_t catalogGetTableMeta(struct SCatalog* pCatalog, SRpcObj *pRpcObj, const S .msgType = TSDB_MSG_TYPE_TABLE_META, .pCont = msg, .contLen = msgLen, - .ahandle = (void*)pSql->self, - .handle = NULL, - .code = 0 }; - rpcSendRequest(pRpcObj->pDnodeConn, pVnodeEpSet, &rpcMsg, &pSql->rpcRid); + SRpcMsg rpcRsp = {0}; + + rpcSendRecv(pRpc, (SEpSet*)pMgmtEps, &rpcMsg, &rpcRsp); + + return TSDB_CODE_SUCCESS; } diff --git a/source/libs/parser/src/astValidate.c b/source/libs/parser/src/astValidate.c index 758b83d820..ce47e50c1e 100644 --- a/source/libs/parser/src/astValidate.c +++ b/source/libs/parser/src/astValidate.c @@ -4081,7 +4081,7 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer SCatalogRsp data = {0}; // TODO: check if the qnode info has been cached already - req.qNodeEpset = true; + req.qNodeRequired = true; code = qParserExtractRequestedMetaInfo(pInfo, &req, msgBuf, msgBufLen); if (code != TSDB_CODE_SUCCESS) { return code; diff --git a/source/libs/parser/src/parser.c b/source/libs/parser/src/parser.c index 3faa06720b..548ab1b8df 100644 --- a/source/libs/parser/src/parser.c +++ b/source/libs/parser/src/parser.c @@ -42,7 +42,12 @@ int32_t qParseQuerySql(const char* pStr, size_t length, struct SQueryStmtInfo** return TSDB_CODE_TSC_SQL_SYNTAX_ERROR; } - struct SCatalog* pCatalog = catalogGetHandle(NULL); + struct SCatalog* pCatalog = NULL; + int32_t code = catalogGetHandle(NULL, &pCatalog); + if (code) { + return code; + } + return qParserValidateSqlNode(pCatalog, &info, *pQueryInfo, id, msg, msgLen); } diff --git a/source/libs/transport/src/rpcMain.c b/source/libs/transport/src/rpcMain.c index 911e8472ab..71e08ea76f 100644 --- a/source/libs/transport/src/rpcMain.c +++ b/source/libs/transport/src/rpcMain.c @@ -406,7 +406,7 @@ void rpcSendRequest(void *shandle, const SEpSet *pEpSet, SRpcMsg *pMsg, int64_t // for TDengine, all the query, show commands shall have TCP connection char type = pMsg->msgType; if (type == TSDB_MSG_TYPE_QUERY || type == TSDB_MSG_TYPE_SHOW_RETRIEVE - || type == TSDB_MSG_TYPE_FETCH || type == TSDB_MSG_TYPE_STABLE_VGROUP + || type == TSDB_MSG_TYPE_FETCH || type == TSDB_MSG_TYPE_VGROUP_LIST || type == TSDB_MSG_TYPE_TABLES_META || type == TSDB_MSG_TYPE_TABLE_META || type == TSDB_MSG_TYPE_SHOW || type == TSDB_MSG_TYPE_STATUS || type == TSDB_MSG_TYPE_ALTER_TABLE) pContext->connType = RPC_CONN_TCPC; diff --git a/source/util/src/terror.c b/source/util/src/terror.c index a75ce747b2..db94d26518 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -498,7 +498,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_FS_INVLD_LEVEL, "tfs invalid level") TAOS_DEFINE_ERROR(TSDB_CODE_FS_NO_VALID_DISK, "tfs no valid disk") // catalog -TAOS_DEFINE_ERROR(TSDB_CODE_CTG_INTERNAL_EROR, "catalog interval error") +TAOS_DEFINE_ERROR(TSDB_CODE_CTG_INTERNAL_ERROR, "catalog interval error") TAOS_DEFINE_ERROR(TSDB_CODE_CTG_INVALID_INPUT, "invalid catalog input parameters") TAOS_DEFINE_ERROR(TSDB_CODE_CTG_NOT_READY, "catalog is not ready") TAOS_DEFINE_ERROR(TSDB_CODE_CTG_MEM_ERROR, "catalog memory error") diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index e6cc3a53af..23b66af0c8 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -99,6 +99,8 @@ int32_t wDebugFlag = 135; int32_t tsdbDebugFlag = 131; int32_t cqDebugFlag = 131; int32_t fsDebugFlag = 135; +int32_t ctgDebugFlag = 131; + int64_t dbgEmptyW = 0; int64_t dbgWN = 0; From 43d1015c3734d2639a2e54b7f3e7af7309891537 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 14 Dec 2021 15:26:38 +0800 Subject: [PATCH 20/43] more --- source/dnode/vnode/impl/src/vnodeBufferPool.c | 4 + src/tsdb/inc/tsdbCommitQueue.h | 23 -- src/tsdb/src/tsdbCommitQueue.c | 213 ------------------ 3 files changed, 4 insertions(+), 236 deletions(-) delete mode 100644 src/tsdb/inc/tsdbCommitQueue.h delete mode 100644 src/tsdb/src/tsdbCommitQueue.c diff --git a/source/dnode/vnode/impl/src/vnodeBufferPool.c b/source/dnode/vnode/impl/src/vnodeBufferPool.c index 2affcbb95d..8fcc112a9f 100644 --- a/source/dnode/vnode/impl/src/vnodeBufferPool.c +++ b/source/dnode/vnode/impl/src/vnodeBufferPool.c @@ -19,6 +19,8 @@ #define VNODE_BUF_POOL_SHARDS 3 struct SVBufPool { + pthread_mutex_t mutex; + pthread_cond_t hasFree; TD_DLIST(SVMemAllocator) free; TD_DLIST(SVMemAllocator) incycle; SVMemAllocator *inuse; @@ -110,6 +112,8 @@ void *vnodeMalloc(SVnode *pVnode, uint64_t size) { if (pBufPool->inuse) { tDListPop(&(pBufPool->free), pBufPool->inuse); break; + } else { + // tsem_wait(&(pBufPool->hasFree)); } } } diff --git a/src/tsdb/inc/tsdbCommitQueue.h b/src/tsdb/inc/tsdbCommitQueue.h deleted file mode 100644 index b690e3bdc2..0000000000 --- a/src/tsdb/inc/tsdbCommitQueue.h +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_TSDB_COMMIT_QUEUE_H_ -#define _TD_TSDB_COMMIT_QUEUE_H_ - -typedef enum { COMMIT_REQ, COMPACT_REQ,COMMIT_CONFIG_REQ } TSDB_REQ_T; - -int tsdbScheduleCommit(STsdbRepo *pRepo, TSDB_REQ_T req); - -#endif /* _TD_TSDB_COMMIT_QUEUE_H_ */ \ No newline at end of file diff --git a/src/tsdb/src/tsdbCommitQueue.c b/src/tsdb/src/tsdbCommitQueue.c deleted file mode 100644 index 59fb4f334d..0000000000 --- a/src/tsdb/src/tsdbCommitQueue.c +++ /dev/null @@ -1,213 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "tsdbint.h" - -typedef struct { - bool stop; - pthread_mutex_t lock; - pthread_cond_t queueNotEmpty; - int nthreads; - int refCount; - SList * queue; - pthread_t * threads; -} SCommitQueue; - -typedef struct { - TSDB_REQ_T req; - STsdbRepo *pRepo; -} SReq; - -static void *tsdbLoopCommit(void *arg); - -static SCommitQueue tsCommitQueue = {0}; - -int tsdbInitCommitQueue() { - int nthreads = tsNumOfCommitThreads; - SCommitQueue *pQueue = &tsCommitQueue; - - if (nthreads < 1) nthreads = 1; - - pQueue->stop = false; - pQueue->nthreads = nthreads; - - pQueue->queue = tdListNew(0); - if (pQueue->queue == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; - } - - pQueue->threads = (pthread_t *)calloc(nthreads, sizeof(pthread_t)); - if (pQueue->threads == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - tdListFree(pQueue->queue); - return -1; - } - - pthread_mutex_init(&(pQueue->lock), NULL); - pthread_cond_init(&(pQueue->queueNotEmpty), NULL); - - for (int i = 0; i < nthreads; i++) { - pthread_create(pQueue->threads + i, NULL, tsdbLoopCommit, NULL); - } - - return 0; -} - -void tsdbDestroyCommitQueue() { - SCommitQueue *pQueue = &tsCommitQueue; - - pthread_mutex_lock(&(pQueue->lock)); - - if (pQueue->stop) { - pthread_mutex_unlock(&(pQueue->lock)); - return; - } - - pQueue->stop = true; - pthread_cond_broadcast(&(pQueue->queueNotEmpty)); - - pthread_mutex_unlock(&(pQueue->lock)); - - for (size_t i = 0; i < pQueue->nthreads; i++) { - pthread_join(pQueue->threads[i], NULL); - } - - free(pQueue->threads); - tdListFree(pQueue->queue); - pthread_cond_destroy(&(pQueue->queueNotEmpty)); - pthread_mutex_destroy(&(pQueue->lock)); -} - -int tsdbScheduleCommit(STsdbRepo *pRepo, TSDB_REQ_T req) { - SCommitQueue *pQueue = &tsCommitQueue; - - SListNode *pNode = (SListNode *)calloc(1, sizeof(SListNode) + sizeof(SReq)); - if (pNode == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; - } - - ((SReq *)pNode->data)->req = req; - ((SReq *)pNode->data)->pRepo = pRepo; - - pthread_mutex_lock(&(pQueue->lock)); - - // ASSERT(pQueue->stop); - - tdListAppendNode(pQueue->queue, pNode); - pthread_cond_signal(&(pQueue->queueNotEmpty)); - - pthread_mutex_unlock(&(pQueue->lock)); - return 0; -} - -static void tsdbApplyRepoConfig(STsdbRepo *pRepo) { - pthread_mutex_lock(&pRepo->save_mutex); - - pRepo->config_changed = false; - STsdbCfg * pSaveCfg = &pRepo->save_config; - STsdbCfg oldCfg; - int32_t oldTotalBlocks = pRepo->config.totalBlocks; - - memcpy(&oldCfg, &(pRepo->config), sizeof(STsdbCfg)); - - pRepo->config.compression = pRepo->save_config.compression; - pRepo->config.keep = pRepo->save_config.keep; - pRepo->config.keep1 = pRepo->save_config.keep1; - pRepo->config.keep2 = pRepo->save_config.keep2; - pRepo->config.cacheLastRow = pRepo->save_config.cacheLastRow; - pRepo->config.totalBlocks = pRepo->save_config.totalBlocks; - - pthread_mutex_unlock(&pRepo->save_mutex); - - tsdbInfo("vgId:%d apply new config: compression(%d), keep(%d,%d,%d), totalBlocks(%d), cacheLastRow(%d->%d),totalBlocks(%d->%d)", - REPO_ID(pRepo), - pSaveCfg->compression, pSaveCfg->keep,pSaveCfg->keep1, pSaveCfg->keep2, - pSaveCfg->totalBlocks, oldCfg.cacheLastRow, pSaveCfg->cacheLastRow, oldTotalBlocks, pSaveCfg->totalBlocks); - - int err = tsdbExpandPool(pRepo, oldTotalBlocks); - if (!TAOS_SUCCEEDED(err)) { - tsdbError("vgId:%d expand pool from %d to %d fail,reason:%s", - REPO_ID(pRepo), oldTotalBlocks, pSaveCfg->totalBlocks, tstrerror(err)); - } - - if (oldCfg.cacheLastRow != pRepo->config.cacheLastRow) { - if (tsdbLockRepo(pRepo) < 0) return; - tsdbCacheLastData(pRepo, &oldCfg); - tsdbUnlockRepo(pRepo); - } - -} - -static void *tsdbLoopCommit(void *arg) { - SCommitQueue *pQueue = &tsCommitQueue; - SListNode * pNode = NULL; - STsdbRepo * pRepo = NULL; - TSDB_REQ_T req; - - setThreadName("tsdbCommit"); - - while (true) { - pthread_mutex_lock(&(pQueue->lock)); - - while (true) { - pNode = tdListPopHead(pQueue->queue); - if (pNode == NULL) { - if (pQueue->stop && pQueue->refCount <= 0) { - pthread_mutex_unlock(&(pQueue->lock)); - goto _exit; - } else { - pthread_cond_wait(&(pQueue->queueNotEmpty), &(pQueue->lock)); - } - } else { - break; - } - } - - pthread_mutex_unlock(&(pQueue->lock)); - - req = ((SReq *)pNode->data)->req; - pRepo = ((SReq *)pNode->data)->pRepo; - - if (req == COMMIT_REQ) { - tsdbCommitData(pRepo); - } else if (req == COMPACT_REQ) { - tsdbCompactImpl(pRepo); - } else if (req == COMMIT_CONFIG_REQ) { - ASSERT(pRepo->config_changed); - tsdbApplyRepoConfig(pRepo); - tsem_post(&(pRepo->readyToCommit)); - } else { - ASSERT(0); - } - - listNodeFree(pNode); - } - -_exit: - return NULL; -} - -void tsdbIncCommitRef(int vgId) { - int refCount = atomic_add_fetch_32(&tsCommitQueue.refCount, 1); - tsdbDebug("vgId:%d, inc commit queue ref to %d", vgId, refCount); -} - -void tsdbDecCommitRef(int vgId) { - int refCount = atomic_sub_fetch_32(&tsCommitQueue.refCount, 1); - pthread_cond_broadcast(&(tsCommitQueue.queueNotEmpty)); - tsdbDebug("vgId:%d, dec commit queue ref to %d", vgId, refCount); -} From d64823d3a93be325e63786f08d34da798ccd1c43 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 14 Dec 2021 16:14:08 +0800 Subject: [PATCH 21/43] more --- source/dnode/vnode/impl/inc/vnodeMemAllocator.h | 1 + source/dnode/vnode/impl/src/vnodeBufferPool.c | 1 - source/dnode/vnode/impl/src/vnodeCfg.c | 3 ++- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/impl/inc/vnodeMemAllocator.h b/source/dnode/vnode/impl/inc/vnodeMemAllocator.h index c8c58e9f69..bdafbf31a7 100644 --- a/source/dnode/vnode/impl/inc/vnodeMemAllocator.h +++ b/source/dnode/vnode/impl/inc/vnodeMemAllocator.h @@ -33,6 +33,7 @@ struct SVArenaNode { }; struct SVMemAllocator { + T_REF_DECLARE() TD_DLIST_NODE(SVMemAllocator); uint64_t capacity; uint64_t ssize; diff --git a/source/dnode/vnode/impl/src/vnodeBufferPool.c b/source/dnode/vnode/impl/src/vnodeBufferPool.c index 8fcc112a9f..152a346f0a 100644 --- a/source/dnode/vnode/impl/src/vnodeBufferPool.c +++ b/source/dnode/vnode/impl/src/vnodeBufferPool.c @@ -30,7 +30,6 @@ struct SVBufPool { int vnodeOpenBufPool(SVnode *pVnode) { uint64_t capacity; - // EVMemAllocatorT type = E_V_ARENA_ALLOCATOR; if ((pVnode->pBufPool = (SVBufPool *)calloc(1, sizeof(SVBufPool))) == NULL) { /* TODO */ diff --git a/source/dnode/vnode/impl/src/vnodeCfg.c b/source/dnode/vnode/impl/src/vnodeCfg.c index 6969ca092d..97c3cc9cee 100644 --- a/source/dnode/vnode/impl/src/vnodeCfg.c +++ b/source/dnode/vnode/impl/src/vnodeCfg.c @@ -15,7 +15,8 @@ #include "vnodeDef.h" -const SVnodeCfg defaultVnodeOptions = {.wsize = 96 * 1024 * 1024, .walCfg = {.level = TAOS_WAL_WRITE}}; /* TODO */ +const SVnodeCfg defaultVnodeOptions = { + .wsize = 96 * 1024 * 1024, .ssize = 1 * 1024 * 1024, .lsize = 1024, .walCfg = {.level = TAOS_WAL_WRITE}}; /* TODO */ void vnodeOptionsInit(SVnodeCfg *pVnodeOptions) { /* TODO */ vnodeOptionsCopy(pVnodeOptions, &defaultVnodeOptions); From 6f3c49ee73ddcc40cb185de12ba371d195443f02 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 14 Dec 2021 16:17:38 +0800 Subject: [PATCH 22/43] update search framework --- source/libs/index/inc/index_fst_automation.h | 15 +++++- source/libs/index/src/index_fst.c | 22 ++++++--- source/libs/index/src/index_fst_automation.c | 52 +++++++++++++++++++- 3 files changed, 80 insertions(+), 9 deletions(-) diff --git a/source/libs/index/inc/index_fst_automation.h b/source/libs/index/inc/index_fst_automation.h index c2ab61bf5f..7eb2ba97ae 100644 --- a/source/libs/index/inc/index_fst_automation.h +++ b/source/libs/index/inc/index_fst_automation.h @@ -20,6 +20,8 @@ extern "C" { #endif #include "index_fst_util.h" + + typedef struct AutomationCtx AutomationCtx; typedef enum AutomationType { @@ -42,14 +44,23 @@ typedef struct AutomationCtx { } AutomationCtx; - +typedef enum ValueType { FST_INT, FST_CHAR, FST_ARRAY} ValueType; typedef enum StartWithStateKind { Done, Running } StartWithStateKind; typedef struct StartWithStateValue { StartWithStateKind kind; - void *value; + ValueType type; + union { + int val; + char *ptr; + SArray *arr; + // add more type + } ; } StartWithStateValue; +StartWithStateValue *startWithStateValueDump(StartWithStateValue *sv); + + typedef struct AutomationFunc { void* (*start)(AutomationCtx *ctx) ; bool (*isMatch)(AutomationCtx *ctx, void *); diff --git a/source/libs/index/src/index_fst.c b/source/libs/index/src/index_fst.c index 07f1e343bd..e2708c5919 100644 --- a/source/libs/index/src/index_fst.c +++ b/source/libs/index/src/index_fst.c @@ -1322,6 +1322,7 @@ StreamWithStateResult *streamWithStateNextWith(StreamWithState *sws, StreamCallb return swsResultCreate(&s, output, callback(start)); } } + SArray *nodes = taosArrayInit(8, sizeof(FstNode *)); while (taosArrayGetSize(sws->stack) > 0) { StreamState *p = (StreamState *)taosArrayPop(sws->stack); if (p->trans >= FST_NODE_LEN(p->node) || automFuncs[aut->type].canMatch(aut, p->autState)) { @@ -1337,8 +1338,8 @@ StreamWithStateResult *streamWithStateNextWith(StreamWithState *sws, StreamCallb void* nextState = automFuncs[aut->type].accept(aut, p->autState, trn.inp); void* tState = callback(nextState); bool isMatch = automFuncs[aut->type].isMatch(aut, nextState); - //bool isMatch = sws->aut->isMatch(nextState); FstNode *nextNode = fstGetNode(sws->fst, trn.addr); + taosArrayPush(nodes, &nextNode); taosArrayPush(sws->inp, &(trn.inp)); if (FST_NODE_IS_FINAL(nextNode)) { @@ -1354,26 +1355,35 @@ StreamWithStateResult *streamWithStateNextWith(StreamWithState *sws, StreamCallb StreamState s2 = {.node = nextNode, .trans = 0, .out = {.null = false, .out = out}, .autState = nextState}; taosArrayPush(sws->stack, &s2); - uint8_t *buf = (uint8_t *)malloc(taosArrayGetSize(sws->inp) * sizeof(uint8_t)); - for (uint32_t i = 0; i < taosArrayGetSize(sws->inp); i++) { - uint8_t *t = (uint8_t *)taosArrayGet(sws->inp, i); - buf[i] = *t; + + size_t isz = taosArrayGetSize(sws->inp); + uint8_t *buf = (uint8_t *)malloc(isz * sizeof(uint8_t)); + for (uint32_t i = 0; i < isz; i++) { + buf[i] = *(uint8_t *)taosArrayGet(sws->inp, i); } FstSlice slice = fstSliceCreate(buf, taosArrayGetSize(sws->inp)); if (fstBoundWithDataExceededBy(sws->endAt, &slice)) { taosArrayDestroyEx(sws->stack, streamStateDestroy); sws->stack = (SArray *)taosArrayInit(256, sizeof(StreamState)); + free(buf); fstSliceDestroy(&slice); return NULL; } if (FST_NODE_IS_FINAL(nextNode) && isMatch) { FstOutput fOutput = {.null = false, .out = out + FST_NODE_FINAL_OUTPUT(nextNode)}; - StreamWithStateResult *result = swsResultCreate(&slice, fOutput , tState); + StreamWithStateResult *result = swsResultCreate(&slice, fOutput, tState); + free(buf); fstSliceDestroy(&slice); return result; } + free(buf); fstSliceDestroy(&slice); } + for (size_t i = 0; i < taosArrayGetSize(nodes); i++) { + FstNode** node = (FstNode **)taosArrayGet(nodes, i); + fstNodeDestroy(*node); + } + taosArrayDestroy(nodes); return NULL; } diff --git a/source/libs/index/src/index_fst_automation.c b/source/libs/index/src/index_fst_automation.c index 6a08b41b12..bb0f0da76d 100644 --- a/source/libs/index/src/index_fst_automation.c +++ b/source/libs/index/src/index_fst_automation.c @@ -16,9 +16,59 @@ #include "index_fst_automation.h" +StartWithStateValue *startWithStateValueCreate(StartWithStateKind kind, ValueType ty, void *val) { + StartWithStateValue *nsv = calloc(1, sizeof(StartWithStateValue)); + if (nsv == NULL) { return NULL; } + + nsv->kind = kind; + nsv->type = ty; + if (ty == FST_INT) { + nsv->val = *(int *)val; + } else if (ty == FST_CHAR) { + size_t len = strlen((char *)val); + nsv->ptr = (char *)calloc(1, len + 1); + memcpy(nsv->ptr, val, len); + } else if (ty == FST_ARRAY) { + //TODO, + //nsv->arr = taosArrayFromList() + } + return nsv; +} +void startWithStateValueDestroy(StartWithStateValue *sv) { + if (sv == NULL) { return; } + + if (sv->type == FST_INT) { + // + } else if (sv->type == FST_CHAR) { + free(sv->ptr); + } else if (sv->type == FST_ARRAY) { + taosArrayDestroy(sv->arr); + } + free(sv); +} +StartWithStateValue *startWithStateValueDump(StartWithStateValue *sv) { + StartWithStateValue *nsv = calloc(1, sizeof(StartWithStateValue)); + if (nsv == NULL) { return NULL; } + + nsv->kind = sv->kind; + nsv->type= sv->type; + if (nsv->type == FST_INT) { + nsv->val = sv->val; + } else if (nsv->type == FST_CHAR) { + size_t len = strlen(sv->ptr); + nsv->ptr = (char *)calloc(1, len + 1); + memcpy(nsv->ptr, sv->ptr, len); + } else if (nsv->type == FST_ARRAY) { + } + return nsv; +} + + // prefix query, impl later + static void* prefixStart(AutomationCtx *ctx) { StartWithStateValue *data = (StartWithStateValue *)(ctx->data); + return data; }; static bool prefixIsMatch(AutomationCtx *ctx, void *data) { @@ -86,7 +136,7 @@ AutomationCtx* automCtxCreate(void *data, AutomationType type) { if (type == AUTOMATION_PREFIX) { StartWithStateValue *swsv = (StartWithStateValue *)calloc(1, sizeof(StartWithStateValue)); swsv->kind = Done; - swsv->value = NULL; + //swsv->value = NULL; ctx->data = (void *)swsv; } else if (type == AUTMMATION_MATCH) { From fb035bb3316405d0e9bf4e6e30f6ca3b82c49b06 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 14 Dec 2021 17:13:31 +0800 Subject: [PATCH 23/43] add query lib --- include/libs/catalog/catalog.h | 2 +- .../{common/tmessage.h => libs/query/query.h} | 10 ++-- source/common/inc/commonInt.h | 10 ---- source/libs/CMakeLists.txt | 3 +- source/libs/catalog/CMakeLists.txt | 4 +- source/libs/catalog/src/catalog.c | 57 +++++++++++++++++-- source/libs/query/CMakeLists.txt | 12 ++++ source/libs/query/inc/queryInt.h | 40 +++++++++++++ .../tmessage.c => libs/query/src/querymsg.c} | 25 ++++---- 9 files changed, 125 insertions(+), 38 deletions(-) rename include/{common/tmessage.h => libs/query/query.h} (69%) create mode 100644 source/libs/query/CMakeLists.txt create mode 100644 source/libs/query/inc/queryInt.h rename source/{common/src/tmessage.c => libs/query/src/querymsg.c} (86%) diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index 5019191fd1..0b45f71557 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -108,7 +108,7 @@ int32_t catalogUpdateVgroup(struct SCatalog* pCatalog, SVgroupListInfo* pVgroup) int32_t catalogGetDBVgroupVersion(struct SCatalog* pCatalog, const char* dbName, int32_t* version); -int32_t catalogGetDBVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* dbName, int32_t forceUpdate, SDBVgroupInfo* dbInfo); +int32_t catalogGetDBVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* dbName, int32_t forceUpdate, SDBVgroupInfo** dbInfo); int32_t catalogUpdateDBVgroup(struct SCatalog* pCatalog, const char* dbName, SDBVgroupInfo* dbInfo); diff --git a/include/common/tmessage.h b/include/libs/query/query.h similarity index 69% rename from include/common/tmessage.h rename to include/libs/query/query.h index c728ee026e..866a69ed8e 100644 --- a/include/common/tmessage.h +++ b/include/libs/query/query.h @@ -13,20 +13,20 @@ * along with this program. If not, see . */ -#ifndef _TD_COMMON_TMESSAGE_H_ -#define _TD_COMMON_TMESSAGE_H_ +#ifndef _TD_QUERY_H_ +#define _TD_QUERY_H_ #ifdef __cplusplus extern "C" { #endif -extern int32_t (*tscBuildMsg[TSDB_MSG_TYPE_MAX])(void* input, char **msg, int32_t msgSize, int32_t *msgLen); -extern int32_t (*tscProcessMsgRsp[TSDB_MSG_TYPE_MAX])(void* output, char *msg, int32_t msgSize); +extern int32_t (*queryBuildMsg[TSDB_MSG_TYPE_MAX])(void* input, char **msg, int32_t msgSize, int32_t *msgLen); +extern int32_t (*queryProcessMsgRsp[TSDB_MSG_TYPE_MAX])(void* output, char *msg, int32_t msgSize); #ifdef __cplusplus } #endif -#endif /*_TD_COMMON_TMESSAGE_H_*/ +#endif /*_TD_QUERY_H_*/ diff --git a/source/common/inc/commonInt.h b/source/common/inc/commonInt.h index 448600e22e..e7d2dba95c 100644 --- a/source/common/inc/commonInt.h +++ b/source/common/inc/commonInt.h @@ -21,17 +21,7 @@ extern "C" { #endif -#include "tlog.h" -extern int32_t cDebugFlag; - -#define tscFatal(...) do { if (cDebugFlag & DEBUG_FATAL) { taosPrintLog("TSC FATAL ", cDebugFlag, __VA_ARGS__); }} while(0) -#define tscError(...) do { if (cDebugFlag & DEBUG_ERROR) { taosPrintLog("TSC ERROR ", cDebugFlag, __VA_ARGS__); }} while(0) -#define tscWarn(...) do { if (cDebugFlag & DEBUG_WARN) { taosPrintLog("TSC WARN ", cDebugFlag, __VA_ARGS__); }} while(0) -#define tscInfo(...) do { if (cDebugFlag & DEBUG_INFO) { taosPrintLog("TSC ", cDebugFlag, __VA_ARGS__); }} while(0) -#define tscDebug(...) do { if (cDebugFlag & DEBUG_DEBUG) { taosPrintLog("TSC ", cDebugFlag, __VA_ARGS__); }} while(0) -#define tscTrace(...) do { if (cDebugFlag & DEBUG_TRACE) { taosPrintLog("TSC ", cDebugFlag, __VA_ARGS__); }} while(0) -#define tscDebugL(...) do { if (cDebugFlag & DEBUG_DEBUG) { taosPrintLongString("TSC ", cDebugFlag, __VA_ARGS__); }} while(0) #ifdef __cplusplus } diff --git a/source/libs/CMakeLists.txt b/source/libs/CMakeLists.txt index 007bb1e967..57a5023807 100644 --- a/source/libs/CMakeLists.txt +++ b/source/libs/CMakeLists.txt @@ -9,4 +9,5 @@ add_subdirectory(cache) add_subdirectory(catalog) add_subdirectory(executor) add_subdirectory(planner) -add_subdirectory(function) \ No newline at end of file +add_subdirectory(function) +add_subdirectory(query) diff --git a/source/libs/catalog/CMakeLists.txt b/source/libs/catalog/CMakeLists.txt index ff3e62700a..25c80d502a 100644 --- a/source/libs/catalog/CMakeLists.txt +++ b/source/libs/catalog/CMakeLists.txt @@ -8,5 +8,5 @@ target_include_directories( target_link_libraries( catalog - PRIVATE os util common transport -) \ No newline at end of file + PRIVATE os util common transport query +) diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index f4a4b01dab..b488ab8101 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -15,7 +15,7 @@ #include "catalogInt.h" #include "trpc.h" -#include "tmessage.h" +#include "query.h" SCatalogMgmt ctgMgmt = {0}; @@ -24,7 +24,7 @@ int32_t ctgGetVgroupFromMnode(struct SCatalog* pCatalog, void *pRpc, const SEpSe SEpSet *pVnodeEpSet = NULL; int32_t msgLen = 0; - int32_t code = tscBuildMsg[TSDB_MSG_TYPE_VGROUP_LIST](NULL, &msg, 0, &msgLen); + int32_t code = queryBuildMsg[TSDB_MSG_TYPE_VGROUP_LIST](NULL, &msg, 0, &msgLen); if (code) { return code; } @@ -39,7 +39,7 @@ int32_t ctgGetVgroupFromMnode(struct SCatalog* pCatalog, void *pRpc, const SEpSe rpcSendRecv(pRpc, (SEpSet*)pMgmtEps, &rpcMsg, &rpcRsp); - code = tscProcessMsgRsp[TSDB_MSG_TYPE_VGROUP_LIST](pVgroup, rpcRsp.pCont, rpcRsp.contLen); + code = queryProcessMsgRsp[TSDB_MSG_TYPE_VGROUP_LIST](pVgroup, rpcRsp.pCont, rpcRsp.contLen); if (code) { return code; } @@ -63,7 +63,23 @@ int32_t ctgGetVgroupFromCache(SCatalog* pCatalog, SArray** pVgroupList, int32_t* } +int32_t ctgGetDBVgroupFromCache(SCatalog* pCatalog, char *dbName, SDBVgroupInfo **dbInfo, int32_t *exist) { +/* + if (NULL == pCatalog->dbCache.cache) { + *exist = 0; + return TSDB_CODE_SUCCESS; + } + taosHashGet(SHashObj * pHashObj, const void * key, size_t keyLen) + + if (dbInfo) { + *pVgroupList = taosArrayDup(pCatalog->vgroupCache.arrayCache); + } + + *exist = 1; +*/ + return TSDB_CODE_SUCCESS; +} int32_t catalogInit(SCatalogCfg *cfg) { @@ -245,12 +261,41 @@ int32_t catalogGetDBVgroupVersion(struct SCatalog* pCatalog, const char* dbName, return TSDB_CODE_SUCCESS; } -int32_t catalogGetDBVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* dbName, int32_t forceUpdate, SDBVgroupInfo* dbInfo) { +int32_t catalogUpdateDBVgroup(struct SCatalog* pCatalog, const char* dbName, SDBVgroupInfo* dbInfo) { } -int32_t catalogUpdateDBVgroup(struct SCatalog* pCatalog, const char* dbName, SDBVgroupInfo* dbInfo) { + + +int32_t catalogGetDBVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* dbName, int32_t forceUpdate, SDBVgroupInfo** dbInfo) { + if (NULL == pCatalog || NULL == dbName || NULL == pRpc || NULL == pMgmtEps) { + return TSDB_CODE_CTG_INVALID_INPUT; + } + +/* + int32_t exist = 0; + + if (0 == forceUpdate) { + CTG_ERR_RET(ctgGetDBVgroupFromCache(pCatalog, dbName, dbInfo, &exist)); + + if (exist) { + return TSDB_CODE_SUCCESS; + } + } + + SDBVgroupInfo* newDbInfo = NULL; + + CTG_ERR_RET(ctgGetDBVgroupFromMnode(pCatalog, pRpc, pMgmtEps, dbName, &newDbInfo)); + + CTG_ERR_RET(catalogUpdateDBVgroup(pCatalog, dbName, newDbInfo)); + + if (dbInfo) { + *dbInfo = newDbInfo; + } +*/ + + return TSDB_CODE_SUCCESS; } @@ -265,7 +310,7 @@ int32_t catalogGetTableMetaFromMnode(struct SCatalog* pCatalog, void *pRpc, cons SEpSet *pVnodeEpSet = NULL; int32_t msgLen = 0; - int32_t code = tscBuildMsg[TSDB_MSG_TYPE_TABLE_META](&bInput, &msg, 0, &msgLen); + int32_t code = queryBuildMsg[TSDB_MSG_TYPE_TABLE_META](&bInput, &msg, 0, &msgLen); if (code) { return code; } diff --git a/source/libs/query/CMakeLists.txt b/source/libs/query/CMakeLists.txt new file mode 100644 index 0000000000..579a4b279c --- /dev/null +++ b/source/libs/query/CMakeLists.txt @@ -0,0 +1,12 @@ +aux_source_directory(src QUERY_SRC) +add_library(query ${QUERY_SRC}) +target_include_directories( + query + PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/query" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" +) + +target_link_libraries( + query + PRIVATE os util common transport +) diff --git a/source/libs/query/inc/queryInt.h b/source/libs/query/inc/queryInt.h new file mode 100644 index 0000000000..f3204b3785 --- /dev/null +++ b/source/libs/query/inc/queryInt.h @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_QUERY_INT_H_ +#define _TD_QUERY_INT_H_ + +#ifdef __cplusplus +extern "C" { +#endif + + +#include "tlog.h" + +extern int32_t qDebugFlag; + +#define qFatal(...) do { if (qDebugFlag & DEBUG_FATAL) { taosPrintLog("QRY FATAL ", qDebugFlag, __VA_ARGS__); }} while(0) +#define qError(...) do { if (qDebugFlag & DEBUG_ERROR) { taosPrintLog("QRY ERROR ", qDebugFlag, __VA_ARGS__); }} while(0) +#define qWarn(...) do { if (qDebugFlag & DEBUG_WARN) { taosPrintLog("QRY WARN ", qDebugFlag, __VA_ARGS__); }} while(0) +#define qInfo(...) do { if (qDebugFlag & DEBUG_INFO) { taosPrintLog("QRY ", qDebugFlag, __VA_ARGS__); }} while(0) +#define qDebug(...) do { if (qDebugFlag & DEBUG_DEBUG) { taosPrintLog("QRY ", qDebugFlag, __VA_ARGS__); }} while(0) +#define qTrace(...) do { if (qDebugFlag & DEBUG_TRACE) { taosPrintLog("QRY ", qDebugFlag, __VA_ARGS__); }} while(0) +#define qDebugL(...) do { if (qDebugFlag & DEBUG_DEBUG) { taosPrintLongString("QRY ", qDebugFlag, __VA_ARGS__); }} while(0) + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_QUERY_INT_H_*/ diff --git a/source/common/src/tmessage.c b/source/libs/query/src/querymsg.c similarity index 86% rename from source/common/src/tmessage.c rename to source/libs/query/src/querymsg.c index 07adf1d599..924878c872 100644 --- a/source/common/src/tmessage.c +++ b/source/libs/query/src/querymsg.c @@ -14,15 +14,15 @@ */ #include "taosmsg.h" -#include "commonint.h" +#include "queryInt.h" -int32_t (*tscBuildMsg[TSDB_MSG_TYPE_MAX])(void* input, char **msg, int32_t msgSize, int32_t *msgLen) = {0}; +int32_t (*queryBuildMsg[TSDB_MSG_TYPE_MAX])(void* input, char **msg, int32_t msgSize, int32_t *msgLen) = {0}; -int32_t (*tscProcessMsgRsp[TSDB_MSG_TYPE_MAX])(void* output, char *msg, int32_t msgSize) = {0}; +int32_t (*queryProcessMsgRsp[TSDB_MSG_TYPE_MAX])(void* output, char *msg, int32_t msgSize) = {0}; -int32_t tscBuildVgroupListReqMsg(void* input, char **msg, int32_t msgSize, int32_t *msgLen) { +int32_t queryBuildVgroupListReqMsg(void* input, char **msg, int32_t msgSize, int32_t *msgLen) { if (NULL == msg || NULL == msgLen) { return TSDB_CODE_TSC_INVALID_INPUT; } @@ -32,7 +32,7 @@ int32_t tscBuildVgroupListReqMsg(void* input, char **msg, int32_t msgSize, int32 return TSDB_CODE_SUCCESS; } -int32_t tscBuildTableMetaReqMsg(void* input, char **msg, int32_t msgSize, int32_t *msgLen) { +int32_t queryBuildTableMetaReqMsg(void* input, char **msg, int32_t msgSize, int32_t *msgLen) { if (NULL == input || NULL == msg || NULL == msgLen) { return TSDB_CODE_TSC_INVALID_INPUT; } @@ -61,7 +61,7 @@ int32_t tscBuildTableMetaReqMsg(void* input, char **msg, int32_t msgSize, int32_ } -int32_t tscProcessVgroupListRsp(void* output, char *msg, int32_t msgSize) { +int32_t queryProcessVgroupListRsp(void* output, char *msg, int32_t msgSize) { if (NULL == output || NULL == msg || msgSize <= 0) { return TSDB_CODE_TSC_INVALID_INPUT; } @@ -72,17 +72,17 @@ int32_t tscProcessVgroupListRsp(void* output, char *msg, int32_t msgSize) { pRsp->vgroupVersion = htonl(pRsp->vgroupVersion); if (pRsp->vgroupNum < 0) { - tscError("vgroup number[%d] in rsp is invalid", pRsp->vgroupNum); + qError("vgroup number[%d] in rsp is invalid", pRsp->vgroupNum); return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; } if (pRsp->vgroupVersion < 0) { - tscError("vgroup vgroupVersion[%d] in rsp is invalid", pRsp->vgroupVersion); + qError("vgroup vgroupVersion[%d] in rsp is invalid", pRsp->vgroupVersion); return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; } if (msgSize != (pRsp->vgroupNum * sizeof(pRsp->vgroupInfo[0]) + sizeof(*pRsp))) { - tscError("vgroup list msg size mis-match, msgSize:%d, vgroup number:%d", msgSize, pRsp->vgroupNum); + qError("vgroup list msg size mis-match, msgSize:%d, vgroup number:%d", msgSize, pRsp->vgroupNum); return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; } @@ -104,12 +104,11 @@ int32_t tscProcessVgroupListRsp(void* output, char *msg, int32_t msgSize) { } void msgInit() { - tscBuildMsg[TSDB_MSG_TYPE_TABLE_META] = tscBuildTableMetaReqMsg; - tscBuildMsg[TSDB_MSG_TYPE_VGROUP_LIST] = tscBuildVgroupListReqMsg; - + queryBuildMsg[TSDB_MSG_TYPE_TABLE_META] = queryBuildTableMetaReqMsg; + queryBuildMsg[TSDB_MSG_TYPE_VGROUP_LIST] = queryBuildVgroupListReqMsg; //tscProcessMsgRsp[TSDB_MSG_TYPE_TABLE_META] = tscProcessTableMetaRsp; - tscProcessMsgRsp[TSDB_MSG_TYPE_VGROUP_LIST] = tscProcessVgroupListRsp; + queryProcessMsgRsp[TSDB_MSG_TYPE_VGROUP_LIST] = queryProcessVgroupListRsp; /* tscBuildMsg[TSDB_SQL_SELECT] = tscBuildQueryMsg; From 2348b6e276266833dfe8840959908af43d7d1302 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 14 Dec 2021 17:13:18 +0800 Subject: [PATCH 24/43] TD-10431 refact trans --- include/dnode/mnode/sdb/sdb.h | 59 +-- source/dnode/mnode/impl/inc/mndDef.h | 2 +- source/dnode/mnode/impl/inc/mndSync.h | 2 +- source/dnode/mnode/impl/inc/mndTrans.h | 6 +- source/dnode/mnode/impl/src/mndDb.c | 6 +- source/dnode/mnode/impl/src/mndDnode.c | 4 +- source/dnode/mnode/impl/src/mndFunc.c | 4 +- source/dnode/mnode/impl/src/mndMnode.c | 4 +- source/dnode/mnode/impl/src/mndStb.c | 4 +- source/dnode/mnode/impl/src/mndSync.c | 14 +- source/dnode/mnode/impl/src/mndTrans.c | 505 +++++++++++++++---------- source/dnode/mnode/impl/src/mndUser.c | 6 +- 12 files changed, 346 insertions(+), 270 deletions(-) diff --git a/include/dnode/mnode/sdb/sdb.h b/include/dnode/mnode/sdb/sdb.h index 7e7afc9774..9620c4ea62 100644 --- a/include/dnode/mnode/sdb/sdb.h +++ b/include/dnode/mnode/sdb/sdb.h @@ -144,9 +144,10 @@ typedef struct SSdbRow SSdbRow; typedef enum { SDB_KEY_BINARY = 1, SDB_KEY_INT32 = 2, SDB_KEY_INT64 = 3 } EKeyType; typedef enum { SDB_STATUS_CREATING = 1, - SDB_STATUS_READY = 2, + SDB_STATUS_UPDATING = 2, SDB_STATUS_DROPPING = 3, - SDB_STATUS_DROPPED = 4 + SDB_STATUS_READY = 4, + SDB_STATUS_DROPPED = 5 } ESdbStatus; typedef enum { @@ -174,67 +175,19 @@ typedef SSdbRow *(*SdbDecodeFp)(SSdbRaw *pRaw); typedef SSdbRaw *(*SdbEncodeFp)(void *pObj); typedef struct { - /** - * @brief The sdb type of the table. - * - */ - ESdbType sdbType; - - /** - * @brief The key type of the table. - * - */ - EKeyType keyType; - - /** - * @brief The callback function when the table is first deployed. - * - */ + ESdbType sdbType; + EKeyType keyType; SdbDeployFp deployFp; - - /** - * @brief Encode one row of the table into rawdata. - * - */ SdbEncodeFp encodeFp; - - /** - * @brief Decode one row of the table from rawdata. - * - */ SdbDecodeFp decodeFp; - - /** - * @brief The callback function when insert a row to sdb. - * - */ SdbInsertFp insertFp; - - /** - * @brief The callback function when undate a row in sdb. - * - */ SdbUpdateFp updateFp; - - /** - * @brief The callback function when delete a row from sdb. - * - */ SdbDeleteFp deleteFp; } SSdbTable; typedef struct SSdbOpt { - /** - * @brief The path of the sdb file. - * - */ const char *path; - - /** - * @brief The mnode object. - * - */ - SMnode *pMnode; + SMnode *pMnode; } SSdbOpt; /** diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index aace3f4ee3..5c672c74be 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -103,7 +103,6 @@ typedef struct STrans { int32_t id; ETrnStage stage; ETrnPolicy policy; - SMnode *pMnode; void *rpcHandle; SArray *redoLogs; SArray *undoLogs; @@ -306,6 +305,7 @@ typedef struct SMnodeMsg { typedef struct { int32_t id; + int32_t code; void *rpcHandle; } STransMsg; diff --git a/source/dnode/mnode/impl/inc/mndSync.h b/source/dnode/mnode/impl/inc/mndSync.h index 02ba725be1..fe557cdeac 100644 --- a/source/dnode/mnode/impl/inc/mndSync.h +++ b/source/dnode/mnode/impl/inc/mndSync.h @@ -25,7 +25,7 @@ extern "C" { int32_t mndInitSync(SMnode *pMnode); void mndCleanupSync(SMnode *pMnode); bool mndIsMaster(SMnode *pMnode); -int32_t mndSyncPropose(SMnode *pMnode, SSdbRaw *pRaw, STransMsg *pMsg); +int32_t mndSyncPropose(SMnode *pMnode, SSdbRaw *pRaw); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/inc/mndTrans.h b/source/dnode/mnode/impl/inc/mndTrans.h index 878337e4be..5da1d1ca2b 100644 --- a/source/dnode/mnode/impl/inc/mndTrans.h +++ b/source/dnode/mnode/impl/inc/mndTrans.h @@ -32,10 +32,10 @@ int32_t mndTransAppendUndolog(STrans *pTrans, SSdbRaw *pRaw); int32_t mndTransAppendCommitlog(STrans *pTrans, SSdbRaw *pRaw); int32_t mndTransAppendRedoAction(STrans *pTrans, SEpSet *, void *pMsg); int32_t mndTransAppendUndoAction(STrans *pTrans, SEpSet *, void *pMsg); - -int32_t mndTransPrepare(STrans *pTrans); +int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans); void mndTransApply(SMnode *pMnode, SSdbRaw *pRaw, STransMsg *pMsg, int32_t code); -int32_t mndTransExecute(SSdb *pSdb, int32_t tranId); +char *mndTransStageStr(ETrnStage stage); +char *mndTransPolicyStr(ETrnPolicy policy); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index fd02c6e251..b638728647 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -357,7 +357,7 @@ static int32_t mndCreateDb(SMnode *pMnode, SMnodeMsg *pMsg, SCreateDbMsg *pCreat } sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); - if (mndTransPrepare(pTrans) != 0) { + if (mndTransPrepare(pMnode, pTrans) != 0) { mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); mndTransDrop(pTrans); return -1; @@ -491,7 +491,7 @@ static int32_t mndUpdateDb(SMnode *pMnode, SMnodeMsg *pMsg, SDbObj *pOldDb, SDbO } sdbSetRawStatus(pUndoRaw, SDB_STATUS_READY); - if (mndTransPrepare(pTrans) != 0) { + if (mndTransPrepare(pMnode, pTrans) != 0) { mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); mndTransDrop(pTrans); return -1; @@ -571,7 +571,7 @@ static int32_t mndDropDb(SMnode *pMnode, SMnodeMsg *pMsg, SDbObj *pDb) { } sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED); - if (mndTransPrepare(pTrans) != 0) { + if (mndTransPrepare(pMnode, pTrans) != 0) { mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); mndTransDrop(pTrans); return -1; diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index cf46d044ca..fbcf623d3c 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -431,7 +431,7 @@ static int32_t mndCreateDnode(SMnode *pMnode, SMnodeMsg *pMsg, SCreateDnodeMsg * } sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); - if (mndTransPrepare(pTrans) != 0) { + if (mndTransPrepare(pMnode, pTrans) != 0) { mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); mndTransDrop(pTrans); return -1; @@ -503,7 +503,7 @@ static int32_t mndDropDnode(SMnode *pMnode, SMnodeMsg *pMsg, SDnodeObj *pDnode) } sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED); - if (mndTransPrepare(pTrans) != 0) { + if (mndTransPrepare(pMnode, pTrans) != 0) { mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); mndTransDrop(pTrans); return -1; diff --git a/source/dnode/mnode/impl/src/mndFunc.c b/source/dnode/mnode/impl/src/mndFunc.c index e407b271fd..3fd7dcfba1 100644 --- a/source/dnode/mnode/impl/src/mndFunc.c +++ b/source/dnode/mnode/impl/src/mndFunc.c @@ -183,7 +183,7 @@ static int32_t mndCreateFunc(SMnode *pMnode, SMnodeMsg *pMsg, SCreateFuncMsg *pC } sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); - if (mndTransPrepare(pTrans) != 0) { + if (mndTransPrepare(pMnode, pTrans) != 0) { mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); mndTransDrop(pTrans); return -1; @@ -226,7 +226,7 @@ static int32_t mndDropFunc(SMnode *pMnode, SMnodeMsg *pMsg, SFuncObj *pFunc) { } sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED); - if (mndTransPrepare(pTrans) != 0) { + if (mndTransPrepare(pMnode, pTrans) != 0) { mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); mndTransDrop(pTrans); return -1; diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index 6278e3ffef..e91c51d301 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -238,7 +238,7 @@ static int32_t mndCreateMnode(SMnode *pMnode, SMnodeMsg *pMsg, SCreateMnodeMsg * } sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); - if (mndTransPrepare(pTrans) != 0) { + if (mndTransPrepare(pMnode, pTrans) != 0) { mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); mndTransDrop(pTrans); return -1; @@ -313,7 +313,7 @@ static int32_t mndDropMnode(SMnode *pMnode, SMnodeMsg *pMsg, SMnodeObj *pMnodeOb } sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED); - if (mndTransPrepare(pTrans) != 0) { + if (mndTransPrepare(pMnode, pTrans) != 0) { mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); mndTransDrop(pTrans); return -1; diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index c3afbf37c8..63bf186be5 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -285,7 +285,7 @@ static int32_t mndCreateStb(SMnode *pMnode, SMnodeMsg *pMsg, SCreateStbMsg *pCre } sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); - if (mndTransPrepare(pTrans) != 0) { + if (mndTransPrepare(pMnode, pTrans) != 0) { mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); mndTransDrop(pTrans); return -1; @@ -433,7 +433,7 @@ static int32_t mndDropStb(SMnode *pMnode, SMnodeMsg *pMsg, SStbObj *pStb) { } sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED); - if (mndTransPrepare(pTrans) != 0) { + if (mndTransPrepare(pMnode, pTrans) != 0) { mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); mndTransDrop(pTrans); return -1; diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c index 6e7ee662f8..59161b32f2 100644 --- a/source/dnode/mnode/impl/src/mndSync.c +++ b/source/dnode/mnode/impl/src/mndSync.c @@ -21,16 +21,16 @@ int32_t mndInitSync(SMnode *pMnode) { return 0; } void mndCleanupSync(SMnode *pMnode) {} -int32_t mndSyncPropose(SMnode *pMnode, SSdbRaw *pRaw, STransMsg *pMsg) { +int32_t mndSyncPropose(SMnode *pMnode, SSdbRaw *pRaw) { int32_t code = 0; - int32_t len = sdbGetRawTotalSize(pRaw); - SSdbRaw *pReceived = calloc(1, len); - memcpy(pReceived, pRaw, len); - mDebug("trans:%d, data:%p recv from sync, code:0x%x pMsg:%p", pMsg->id, pReceived, code & 0xFFFF, pMsg); + // int32_t len = sdbGetRawTotalSize(pRaw); + // SSdbRaw *pReceived = calloc(1, len); + // memcpy(pReceived, pRaw, len); + // mDebug("trans:%d, data:%p recv from sync, code:0x%x pMsg:%p", pMsg->id, pReceived, code & 0xFFFF, pMsg); - mndTransApply(pMnode, pReceived, pMsg, code); - return 0; + // mndTransApply(pMnode, pReceived, code); + return code; } bool mndIsMaster(SMnode *pMnode) { return true; } \ No newline at end of file diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 3a53472d45..65957e44bc 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -17,8 +17,9 @@ #include "mndTrans.h" #include "mndSync.h" -#define SDB_TRANS_VER 1 -#define TRN_DEFAULT_ARRAY_SIZE 8 +#define TSDB_TRANS_VER 1 +#define TSDB_TRN_ARRAY_SIZE 8 +#define TSDB_TRN_RESERVE_SIZE 64 static SSdbRaw *mndTransActionEncode(STrans *pTrans); static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw); @@ -26,6 +27,22 @@ static int32_t mndTransActionInsert(SSdb *pSdb, STrans *pTrans); static int32_t mndTransActionUpdate(SSdb *pSdb, STrans *OldTrans, STrans *pOldTrans); static int32_t mndTransActionDelete(SSdb *pSdb, STrans *pTrans); +static void mndTransSetRpcHandle(STrans *pTrans, void *rpcHandle); +static void mndTransSendRpcRsp(STrans *pTrans, int32_t code); +static int32_t mndTransAppendArray(SArray *pArray, SSdbRaw *pRaw); +static void mndTransDropArray(SArray *pArray); +static int32_t mndTransExecuteArray(SMnode *pMnode, SArray *pArray); +static int32_t mndTransExecuteRedoLogs(SMnode *pMnode, STrans *pTrans); +static int32_t mndTransExecuteUndoLogs(SMnode *pMnode, STrans *pTrans); +static int32_t mndTransExecuteCommitLogs(SMnode *pMnode, STrans *pTrans); +static int32_t mndTransExecuteRedoActions(SMnode *pMnode, STrans *pTrans); +static int32_t mndTransExecuteUndoActions(SMnode *pMnode, STrans *pTrans); +static int32_t mndTransPerformPrepareStage(SMnode *pMnode, STrans *pTrans); +static int32_t mndTransPerformExecuteStage(SMnode *pMnode, STrans *pTrans); +static int32_t mndTransPerformCommitStage(SMnode *pMnode, STrans *pTrans); +static int32_t mndTransPerformRollbackStage(SMnode *pMnode, STrans *pTrans); +static void mndTransExecute(SMnode *pMnode, STrans *pTrans); + int32_t mndInitTrans(SMnode *pMnode) { SSdbTable table = {.sdbType = SDB_TRANS, .keyType = SDB_KEY_INT32, @@ -63,7 +80,7 @@ static SSdbRaw *mndTransActionEncode(STrans *pTrans) { rawDataLen += sdbGetRawTotalSize(pTmp); } - SSdbRaw *pRaw = sdbAllocRaw(SDB_TRANS, SDB_TRANS_VER, rawDataLen); + SSdbRaw *pRaw = sdbAllocRaw(SDB_TRANS, TSDB_TRANS_VER, rawDataLen); if (pRaw == NULL) { mError("trans:%d, failed to alloc raw since %s", pTrans->id, terrstr()); return NULL; @@ -100,6 +117,8 @@ static SSdbRaw *mndTransActionEncode(STrans *pTrans) { SDB_SET_BINARY(pRaw, dataPos, (void *)pTmp, len) } + SDB_SET_RESERVE(pRaw, dataPos, TSDB_TRN_RESERVE_SIZE) + SDB_SET_DATALEN(pRaw, dataPos); mTrace("trans:%d, encode to raw:%p, len:%d", pTrans->id, pRaw, dataPos); return pRaw; } @@ -113,7 +132,7 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { return NULL; } - if (sver != SDB_TRANS_VER) { + if (sver != TSDB_TRANS_VER) { terrno = TSDB_CODE_SDB_INVALID_DATA_VER; mError("failed to get check soft ver from raw:%p since %s", pRaw, terrstr()); return NULL; @@ -126,11 +145,11 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { return NULL; } - pTrans->redoLogs = taosArrayInit(TRN_DEFAULT_ARRAY_SIZE, sizeof(void *)); - pTrans->undoLogs = taosArrayInit(TRN_DEFAULT_ARRAY_SIZE, sizeof(void *)); - pTrans->commitLogs = taosArrayInit(TRN_DEFAULT_ARRAY_SIZE, sizeof(void *)); - pTrans->redoActions = taosArrayInit(TRN_DEFAULT_ARRAY_SIZE, sizeof(void *)); - pTrans->undoActions = taosArrayInit(TRN_DEFAULT_ARRAY_SIZE, sizeof(void *)); + pTrans->redoLogs = taosArrayInit(TSDB_TRN_ARRAY_SIZE, sizeof(void *)); + pTrans->undoLogs = taosArrayInit(TSDB_TRN_ARRAY_SIZE, sizeof(void *)); + pTrans->commitLogs = taosArrayInit(TSDB_TRN_ARRAY_SIZE, sizeof(void *)); + pTrans->redoActions = taosArrayInit(TSDB_TRN_ARRAY_SIZE, sizeof(void *)); + pTrans->undoActions = taosArrayInit(TSDB_TRN_ARRAY_SIZE, sizeof(void *)); if (pTrans->redoLogs == NULL || pTrans->undoLogs == NULL || pTrans->commitLogs == NULL || pTrans->redoActions == NULL || pTrans->undoActions == NULL) { @@ -197,6 +216,8 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { } } + SDB_GET_RESERVE(pRaw, pRow, dataPos, TSDB_TRN_RESERVE_SIZE) + TRANS_DECODE_OVER: if (code != 0) { mError("trans:%d, failed to parse from raw:%p since %s", pTrans->id, pRaw, tstrerror(errno)); @@ -210,64 +231,71 @@ TRANS_DECODE_OVER: } static int32_t mndTransActionInsert(SSdb *pSdb, STrans *pTrans) { - mTrace("trans:%d, perform insert action, stage:%d", pTrans->id, pTrans->stage); - - SArray *pArray = pTrans->redoLogs; - int32_t arraySize = taosArrayGetSize(pArray); - - for (int32_t i = 0; i < arraySize; ++i) { - SSdbRaw *pRaw = taosArrayGetP(pArray, i); - int32_t code = sdbWrite(pSdb, pRaw); - if (code != 0) { - mError("trans:%d, failed to write raw:%p to sdb since %s", pTrans->id, pRaw, terrstr()); - return code; - } - } + mTrace("trans:%d, perform insert action, stage:%s", pTrans->id, mndTransStageStr(pTrans->stage)); return 0; } static int32_t mndTransActionDelete(SSdb *pSdb, STrans *pTrans) { - mTrace("trans:%d, perform delete action, stage:%d", pTrans->id, pTrans->stage); + mTrace("trans:%d, perform delete action, stage:%s", pTrans->id, mndTransStageStr(pTrans->stage)); - SArray *pArray = pTrans->undoLogs; - int32_t arraySize = taosArrayGetSize(pArray); - - for (int32_t i = 0; i < arraySize; ++i) { - SSdbRaw *pRaw = taosArrayGetP(pArray, i); - int32_t code = sdbWrite(pSdb, pRaw); - if (code != 0) { - mError("trans:%d, failed to write raw:%p to sdb since %s", pTrans->id, pRaw, terrstr()); - return code; - } - } + mndTransDropArray(pTrans->redoLogs); + mndTransDropArray(pTrans->undoLogs); + mndTransDropArray(pTrans->commitLogs); + mndTransDropArray(pTrans->redoActions); + mndTransDropArray(pTrans->undoActions); return 0; } static int32_t mndTransActionUpdate(SSdb *pSdb, STrans *pOldTrans, STrans *pNewTrans) { - mTrace("trans:%d, perform update action, stage:%d", pOldTrans->id, pNewTrans->stage); - - SArray *pArray = pOldTrans->commitLogs; - int32_t arraySize = taosArrayGetSize(pArray); - - for (int32_t i = 0; i < arraySize; ++i) { - SSdbRaw *pRaw = taosArrayGetP(pArray, i); - int32_t code = sdbWrite(pSdb, pRaw); - if (code != 0) { - mError("trans:%d, failed to write raw:%p to sdb since %s", pOldTrans->id, pRaw, terrstr()); - return code; - } - } - + mTrace("trans:%d, perform update action, stage:%s", pOldTrans->id, mndTransStageStr(pNewTrans->stage)); pOldTrans->stage = pNewTrans->stage; return 0; } +STrans *mndAcquireTrans(SMnode *pMnode, int32_t transId) { + SSdb *pSdb = pMnode->pSdb; + return sdbAcquire(pSdb, SDB_TRANS, &transId); +} + +void mndReleaseTrans(SMnode *pMnode, STrans *pTrans) { + SSdb *pSdb = pMnode->pSdb; + sdbRelease(pSdb, pTrans); +} + static int32_t trnGenerateTransId() { static int32_t tmp = 0; return ++tmp; } +char *mndTransStageStr(ETrnStage stage) { + switch (stage) { + case TRN_STAGE_PREPARE: + return "prepare"; + case TRN_STAGE_EXECUTE: + return "execute"; + case TRN_STAGE_COMMIT: + return "commit"; + case TRN_STAGE_ROLLBACK: + return "rollback"; + case TRN_STAGE_RETRY: + return "retry"; + default: + return "undefined"; + } +} + +char *mndTransPolicyStr(ETrnPolicy policy) { + switch (policy) { + case TRN_POLICY_ROLLBACK: + return "prepare"; + case TRN_POLICY_RETRY: + return "retry"; + default: + return "undefined"; + } +} + STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, void *rpcHandle) { STrans *pTrans = calloc(1, sizeof(STrans)); if (pTrans == NULL) { @@ -279,13 +307,12 @@ STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, void *rpcHandle) { pTrans->id = trnGenerateTransId(); pTrans->stage = TRN_STAGE_PREPARE; pTrans->policy = policy; - pTrans->pMnode = pMnode; pTrans->rpcHandle = rpcHandle; - pTrans->redoLogs = taosArrayInit(TRN_DEFAULT_ARRAY_SIZE, sizeof(void *)); - pTrans->undoLogs = taosArrayInit(TRN_DEFAULT_ARRAY_SIZE, sizeof(void *)); - pTrans->commitLogs = taosArrayInit(TRN_DEFAULT_ARRAY_SIZE, sizeof(void *)); - pTrans->redoActions = taosArrayInit(TRN_DEFAULT_ARRAY_SIZE, sizeof(void *)); - pTrans->undoActions = taosArrayInit(TRN_DEFAULT_ARRAY_SIZE, sizeof(void *)); + pTrans->redoLogs = taosArrayInit(TSDB_TRN_ARRAY_SIZE, sizeof(void *)); + pTrans->undoLogs = taosArrayInit(TSDB_TRN_ARRAY_SIZE, sizeof(void *)); + pTrans->commitLogs = taosArrayInit(TSDB_TRN_ARRAY_SIZE, sizeof(void *)); + pTrans->redoActions = taosArrayInit(TSDB_TRN_ARRAY_SIZE, sizeof(void *)); + pTrans->undoActions = taosArrayInit(TSDB_TRN_ARRAY_SIZE, sizeof(void *)); if (pTrans->redoLogs == NULL || pTrans->undoLogs == NULL || pTrans->commitLogs == NULL || pTrans->redoActions == NULL || pTrans->undoActions == NULL) { @@ -298,7 +325,7 @@ STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, void *rpcHandle) { return pTrans; } -static void trnDropArray(SArray *pArray) { +static void mndTransDropArray(SArray *pArray) { for (int32_t i = 0; i < pArray->size; ++i) { SSdbRaw *pRaw = taosArrayGetP(pArray, i); tfree(pRaw); @@ -308,17 +335,17 @@ static void trnDropArray(SArray *pArray) { } void mndTransDrop(STrans *pTrans) { - trnDropArray(pTrans->redoLogs); - trnDropArray(pTrans->undoLogs); - trnDropArray(pTrans->commitLogs); - trnDropArray(pTrans->redoActions); - trnDropArray(pTrans->undoActions); + mndTransDropArray(pTrans->redoLogs); + mndTransDropArray(pTrans->undoLogs); + mndTransDropArray(pTrans->commitLogs); + mndTransDropArray(pTrans->redoActions); + mndTransDropArray(pTrans->undoActions); mDebug("trans:%d, data:%p is dropped", pTrans->id, pTrans); tfree(pTrans); } -void mndTransSetRpcHandle(STrans *pTrans, void *rpcHandle) { +static void mndTransSetRpcHandle(STrans *pTrans, void *rpcHandle) { pTrans->rpcHandle = rpcHandle; mTrace("trans:%d, set rpc handle:%p", pTrans->id, rpcHandle); } @@ -340,19 +367,19 @@ static int32_t mndTransAppendArray(SArray *pArray, SSdbRaw *pRaw) { int32_t mndTransAppendRedolog(STrans *pTrans, SSdbRaw *pRaw) { int32_t code = mndTransAppendArray(pTrans->redoLogs, pRaw); - mTrace("trans:%d, raw:%p append to redo logs, code:%d", pTrans->id, pRaw, code); + mTrace("trans:%d, raw:%p append to redo logs, code:0x%x", pTrans->id, pRaw, code); return code; } int32_t mndTransAppendUndolog(STrans *pTrans, SSdbRaw *pRaw) { int32_t code = mndTransAppendArray(pTrans->undoLogs, pRaw); - mTrace("trans:%d, raw:%p append to undo logs, code:%d", pTrans->id, pRaw, code); + mTrace("trans:%d, raw:%p append to undo logs, code:0x%x", pTrans->id, pRaw, code); return code; } int32_t mndTransAppendCommitlog(STrans *pTrans, SSdbRaw *pRaw) { int32_t code = mndTransAppendArray(pTrans->commitLogs, pRaw); - mTrace("trans:%d, raw:%p append to commit logs, code:%d", pTrans->id, pRaw, code); + mTrace("trans:%d, raw:%p append to commit logs, code:0x%x", pTrans->id, pRaw, code); return code; } @@ -368,7 +395,7 @@ int32_t mndTransAppendUndoAction(STrans *pTrans, SEpSet *pEpSet, void *pMsg) { return code; } -int32_t mndTransPrepare(STrans *pTrans) { +int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans) { mDebug("trans:%d, prepare transaction", pTrans->id); SSdbRaw *pRaw = mndTransActionEncode(pTrans); @@ -376,180 +403,276 @@ int32_t mndTransPrepare(STrans *pTrans) { mError("trans:%d, failed to decode trans since %s", pTrans->id, terrstr()); return -1; } - sdbSetRawStatus(pRaw, SDB_STATUS_CREATING); + sdbSetRawStatus(pRaw, SDB_STATUS_READY); - if (sdbWriteNotFree(pTrans->pMnode->pSdb, pRaw) != 0) { - mError("trans:%d, failed to write trans since %s", pTrans->id, terrstr()); - return -1; - } - - STransMsg *pMsg = calloc(1, sizeof(STransMsg)); - pMsg->id = pTrans->id; - pMsg->rpcHandle = pTrans->rpcHandle; - - mDebug("trans:%d, start sync, RPC:%p pMsg:%p", pTrans->id, pTrans->rpcHandle, pMsg); - if (mndSyncPropose(pTrans->pMnode, pRaw, pMsg) != 0) { + mTrace("trans:%d, start sync", pTrans->id); + int32_t code = mndSyncPropose(pMnode, pRaw); + if (code != 0) { mError("trans:%d, failed to sync since %s", pTrans->id, terrstr()); - free(pMsg); sdbFreeRaw(pRaw); return -1; } - sdbFreeRaw(pRaw); + mTrace("trans:%d, sync finished", pTrans->id); + + code = sdbWrite(pMnode->pSdb, pRaw); + if (code != 0) { + mError("trans:%d, failed to write sdb since %s", pTrans->id, terrstr()); + return -1; + } + + STrans *pNewTrans = mndAcquireTrans(pMnode, pTrans->id); + if (pNewTrans == NULL) { + mError("trans:%d, failed to ready from sdb since %s", pTrans->id, terrstr()); + return -1; + } + + mDebug("trans:%d, prepare finished", pNewTrans->id); + mndTransExecute(pMnode, pNewTrans); + mndReleaseTrans(pMnode, pNewTrans); return 0; } -static void trnSendRpcRsp(STransMsg *pMsg, int32_t code) { - mDebug("trans:%d, send rpc rsp, RPC:%p code:0x%x pMsg:%p", pMsg->id, pMsg->rpcHandle, code & 0xFFFF, pMsg); - if (pMsg->rpcHandle != NULL) { - SRpcMsg rspMsg = {.handle = pMsg->rpcHandle, .code = code}; - rpcSendResponse(&rspMsg); +int32_t mndTransCommit(SMnode *pMnode, STrans *pTrans) { + mDebug("trans:%d, commit transaction", pTrans->id); + + SSdbRaw *pRaw = mndTransActionEncode(pTrans); + if (pRaw == NULL) { + mError("trans:%d, failed to decode trans since %s", pTrans->id, terrstr()); + return -1; + } + sdbSetRawStatus(pRaw, SDB_STATUS_DROPPED); + + mTrace("trans:%d, start sync", pTrans->id); + int32_t code = mndSyncPropose(pMnode, pRaw); + if (code != 0) { + mError("trans:%d, failed to sync since %s", pTrans->id, terrstr()); + sdbFreeRaw(pRaw); + return -1; } - free(pMsg); + mTrace("trans:%d, sync finished", pTrans->id); + code = sdbWrite(pMnode->pSdb, pRaw); + if (code != 0) { + mError("trans:%d, failed to write sdb since %s", pTrans->id, terrstr()); + return -1; + } + + mDebug("trans:%d, commit finished", pTrans->id); + return 0; +} + +int32_t mndTransRollback(SMnode *pMnode, STrans *pTrans) { + mDebug("trans:%d, rollback transaction", pTrans->id); + + SSdbRaw *pRaw = mndTransActionEncode(pTrans); + if (pRaw == NULL) { + mError("trans:%d, failed to decode trans since %s", pTrans->id, terrstr()); + return -1; + } + sdbSetRawStatus(pRaw, SDB_STATUS_DROPPED); + + mTrace("trans:%d, start sync", pTrans->id); + int32_t code = mndSyncPropose(pMnode, pRaw); + if (code != 0) { + mError("trans:%d, failed to sync since %s", pTrans->id, terrstr()); + sdbFreeRaw(pRaw); + return -1; + } + + mTrace("trans:%d, sync finished", pTrans->id); + code = sdbWrite(pMnode->pSdb, pRaw); + if (code != 0) { + mError("trans:%d, failed to write sdb since %s", pTrans->id, terrstr()); + return -1; + } + + mDebug("trans:%d, rollback finished", pTrans->id); + return 0; +} + +static void mndTransSendRpcRsp(STrans *pTrans, int32_t code) { + if (code == TSDB_CODE_MND_ACTION_IN_PROGRESS) return; + mDebug("trans:%d, send rpc rsp, RPC:%p code:0x%x", pTrans->id, pTrans->rpcHandle, code & 0xFFFF); + + if (pTrans->rpcHandle != NULL) { + SRpcMsg rspMsg = {.handle = pTrans->rpcHandle, .code = code}; + rpcSendResponse(&rspMsg); + } } void mndTransApply(SMnode *pMnode, SSdbRaw *pRaw, STransMsg *pMsg, int32_t code) { - if (code == 0) { - mDebug("trans:%d, commit transaction", pMsg->id); - sdbSetRawStatus(pRaw, SDB_STATUS_READY); - if (sdbWrite(pMnode->pSdb, pRaw) != 0) { - code = terrno; - mError("trans:%d, failed to write sdb while commit since %s", pMsg->id, terrstr()); - } - trnSendRpcRsp(pMsg, code); - } else { - mDebug("trans:%d, rollback transaction", pMsg->id); - sdbSetRawStatus(pRaw, SDB_STATUS_DROPPED); - if (sdbWrite(pMnode->pSdb, pRaw) != 0) { - mError("trans:%d, failed to write sdb while rollback since %s", pMsg->id, terrstr()); - } - trnSendRpcRsp(pMsg, code); - } + // todo } -static int32_t trnExecuteArray(SMnode *pMnode, SArray *pArray) { - for (int32_t i = 0; i < pArray->size; ++i) { +static int32_t mndTransExecuteArray(SMnode *pMnode, SArray *pArray) { + SSdb *pSdb = pMnode->pSdb; + int32_t arraySize = taosArrayGetSize(pArray); + + for (int32_t i = 0; i < arraySize; ++i) { SSdbRaw *pRaw = taosArrayGetP(pArray, i); - if (sdbWrite(pMnode->pSdb, pRaw) != 0) { - return -1; + int32_t code = sdbWriteNotFree(pSdb, pRaw); + if (code != 0) { + return code; } } return 0; } -static int32_t trnExecuteRedoLogs(STrans *pTrans) { return trnExecuteArray(pTrans->pMnode, pTrans->redoLogs); } - -static int32_t trnExecuteUndoLogs(STrans *pTrans) { return trnExecuteArray(pTrans->pMnode, pTrans->undoLogs); } - -static int32_t trnExecuteCommitLogs(STrans *pTrans) { return trnExecuteArray(pTrans->pMnode, pTrans->commitLogs); } - -static int32_t trnExecuteRedoActions(STrans *pTrans) { return trnExecuteArray(pTrans->pMnode, pTrans->redoActions); } - -static int32_t trnExecuteUndoActions(STrans *pTrans) { return trnExecuteArray(pTrans->pMnode, pTrans->undoActions); } - -static int32_t trnPerformPrepareStage(STrans *pTrans) { - if (trnExecuteRedoLogs(pTrans) == 0) { - pTrans->stage = TRN_STAGE_EXECUTE; - return 0; +static int32_t mndTransExecuteRedoLogs(SMnode *pMnode, STrans *pTrans) { + int32_t code = mndTransExecuteArray(pMnode, pTrans->redoLogs); + if (code != 0) { + mError("trans:%d, failed to execute redo logs since %s", pTrans->id, terrstr()) } else { - pTrans->stage = TRN_STAGE_ROLLBACK; - return -1; + mTrace("trans:%d, execute redo logs finished", pTrans->id) } + + return code; } -static int32_t trnPerformExecuteStage(STrans *pTrans) { - int32_t code = trnExecuteRedoActions(pTrans); +static int32_t mndTransExecuteUndoLogs(SMnode *pMnode, STrans *pTrans) { + int32_t code = mndTransExecuteArray(pMnode, pTrans->undoLogs); + if (code != 0) { + mError("trans:%d, failed to execute undo logs since %s", pTrans->id, terrstr()) + } else { + mTrace("trans:%d, execute undo logs finished", pTrans->id) + } + + return code; +} + +static int32_t mndTransExecuteCommitLogs(SMnode *pMnode, STrans *pTrans) { + int32_t code = mndTransExecuteArray(pMnode, pTrans->commitLogs); + if (code != 0) { + mError("trans:%d, failed to execute commit logs since %s", pTrans->id, terrstr()) + } else { + mTrace("trans:%d, execute commit logs finished", pTrans->id) + } + + return code; +} + +static int32_t mndTransExecuteRedoActions(SMnode *pMnode, STrans *pTrans) { + mTrace("trans:%d, execute redo actions finished", pTrans->id); + return 0; +} + +static int32_t mndTransExecuteUndoActions(SMnode *pMnode, STrans *pTrans) { + mTrace("trans:%d, execute undo actions finished", pTrans->id); + return 0; +} + +static int32_t mndTransPerformPrepareStage(SMnode *pMnode, STrans *pTrans) { + int32_t code = mndTransExecuteRedoLogs(pMnode, pTrans); + + if (code == 0) { + pTrans->stage = TRN_STAGE_EXECUTE; + mTrace("trans:%d, stage from prepare to execute", pTrans->id); + } else { + pTrans->stage = TRN_STAGE_ROLLBACK; + mError("trans:%d, stage from prepare to rollback since %s", pTrans->id, terrstr()); + } + + return 0; +} + +static int32_t mndTransPerformExecuteStage(SMnode *pMnode, STrans *pTrans) { + int32_t code = mndTransExecuteRedoActions(pMnode, pTrans); if (code == 0) { pTrans->stage = TRN_STAGE_COMMIT; - return 0; + mTrace("trans:%d, stage from execute to commit", pTrans->id); } else if (code == TSDB_CODE_MND_ACTION_IN_PROGRESS) { - return -1; + mTrace("trans:%d, stage keep on execute since %s", pTrans->id, terrstr(code)); + return code; } else { - if (pTrans->policy == TRN_POLICY_RETRY) { - pTrans->stage = TRN_STAGE_RETRY; - } else { + if (pTrans->policy == TRN_POLICY_ROLLBACK) { pTrans->stage = TRN_STAGE_ROLLBACK; + mError("trans:%d, stage from execute to rollback since %s", pTrans->id, terrstr()); + } else { + pTrans->stage = TRN_STAGE_RETRY; + mError("trans:%d, stage from execute to retry since %s", pTrans->id, terrstr()); } - return 0; } + + return 0; } -static int32_t trnPerformCommitStage(STrans *pTrans) { - if (trnExecuteCommitLogs(pTrans) == 0) { - pTrans->stage = TRN_STAGE_EXECUTE; - return 0; +static int32_t mndTransPerformCommitStage(SMnode *pMnode, STrans *pTrans) { + int32_t code = mndTransExecuteCommitLogs(pMnode, pTrans); + + if (code == 0) { + pTrans->stage = TRN_STAGE_COMMIT; + mTrace("trans:%d, commit stage finished", pTrans->id); + } else { + if (pTrans->policy == TRN_POLICY_ROLLBACK) { + pTrans->stage = TRN_STAGE_ROLLBACK; + mError("trans:%d, stage from commit to rollback since %s", pTrans->id, terrstr()); + } else { + pTrans->stage = TRN_STAGE_RETRY; + mError("trans:%d, stage from commit to retry since %s", pTrans->id, terrstr()); + } + } + + return code; +} + +static int32_t mndTransPerformRollbackStage(SMnode *pMnode, STrans *pTrans) { + int32_t code = mndTransExecuteUndoActions(pMnode, pTrans); + + if (code == 0) { + mTrace("trans:%d, rollbacked", pTrans->id); } else { pTrans->stage = TRN_STAGE_ROLLBACK; - return -1; + mError("trans:%d, stage keep on rollback since %s", pTrans->id, terrstr()); } + + return code; } -static int32_t trnPerformRollbackStage(STrans *pTrans) { - if (trnExecuteCommitLogs(pTrans) == 0) { - pTrans->stage = TRN_STAGE_EXECUTE; - return 0; +static int32_t mndTransPerformRetryStage(SMnode *pMnode, STrans *pTrans) { + int32_t code = mndTransExecuteRedoActions(pMnode, pTrans); + + if (code == 0) { + pTrans->stage = TRN_STAGE_COMMIT; + mTrace("trans:%d, stage from retry to commit", pTrans->id); } else { - pTrans->stage = TRN_STAGE_ROLLBACK; - return -1; + pTrans->stage = TRN_STAGE_RETRY; + mError("trans:%d, stage keep on retry since %s", pTrans->id, terrstr()); } + + return code; } -static int32_t trnPerformRetryStage(STrans *pTrans) { - if (trnExecuteCommitLogs(pTrans) == 0) { - pTrans->stage = TRN_STAGE_EXECUTE; - return 0; - } else { - pTrans->stage = TRN_STAGE_ROLLBACK; - return -1; - } -} - -int32_t mndTransExecute(SSdb *pSdb, int32_t tranId) { +static void mndTransExecute(SMnode *pMnode, STrans *pTrans) { int32_t code = 0; - STrans *pTrans = sdbAcquire(pSdb, SDB_TRANS, &tranId); - if (pTrans == NULL) { - return -1; - } - - if (pTrans->stage == TRN_STAGE_PREPARE) { - if (trnPerformPrepareStage(pTrans) != 0) { - sdbRelease(pSdb, pTrans); - return -1; + while (code == 0) { + switch (pTrans->stage) { + case TRN_STAGE_PREPARE: + code = mndTransPerformPrepareStage(pMnode, pTrans); + break; + case TRN_STAGE_EXECUTE: + code = mndTransPerformExecuteStage(pMnode, pTrans); + break; + case TRN_STAGE_COMMIT: + code = mndTransCommit(pMnode, pTrans); + if (code == 0) { + code = mndTransPerformCommitStage(pMnode, pTrans); + } + break; + case TRN_STAGE_ROLLBACK: + code = mndTransPerformRollbackStage(pMnode, pTrans); + if (code == 0) { + code = mndTransRollback(pMnode, pTrans); + } + break; + case TRN_STAGE_RETRY: + code = mndTransPerformRetryStage(pMnode, pTrans); + break; } } - if (pTrans->stage == TRN_STAGE_EXECUTE) { - if (trnPerformExecuteStage(pTrans) != 0) { - sdbRelease(pSdb, pTrans); - return -1; - } - } - - if (pTrans->stage == TRN_STAGE_COMMIT) { - if (trnPerformCommitStage(pTrans) != 0) { - sdbRelease(pSdb, pTrans); - return -1; - } - } - - if (pTrans->stage == TRN_STAGE_ROLLBACK) { - if (trnPerformRollbackStage(pTrans) != 0) { - sdbRelease(pSdb, pTrans); - return -1; - } - } - - if (pTrans->stage == TRN_STAGE_RETRY) { - if (trnPerformRetryStage(pTrans) != 0) { - sdbRelease(pSdb, pTrans); - return -1; - } - } - - sdbRelease(pSdb, pTrans); - return 0; -} \ No newline at end of file + mndTransSendRpcRsp(pTrans, code); +} diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index 2f582a810d..e6bf30f41b 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -235,7 +235,7 @@ static int32_t mndCreateUser(SMnode *pMnode, char *acct, char *user, char *pass, } sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); - if (mndTransPrepare(pTrans) != 0) { + if (mndTransPrepare(pMnode, pTrans) != 0) { mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); mndTransDrop(pTrans); return -1; @@ -269,7 +269,7 @@ static int32_t mndUpdateUser(SMnode *pMnode, SUserObj *pOldUser, SUserObj *pNewU } sdbSetRawStatus(pUndoRaw, SDB_STATUS_READY); - if (mndTransPrepare(pTrans) != 0) { + if (mndTransPrepare(pMnode, pTrans) != 0) { mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); mndTransDrop(pTrans); return -1; @@ -311,7 +311,7 @@ static int32_t mndDropUser(SMnode *pMnode, SMnodeMsg *pMsg, SUserObj *pUser) { } sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED); - if (mndTransPrepare(pTrans) != 0) { + if (mndTransPrepare(pMnode, pTrans) != 0) { mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); mndTransDrop(pTrans); return -1; From 9a0d27c6fcab9a3501dc387522e66dbc62b8990a Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 14 Dec 2021 18:07:29 +0800 Subject: [PATCH 25/43] TD-10431 test for create user --- source/dnode/mgmt/impl/test/dnode/dnode.cpp | 68 ++++++++-------- source/dnode/mgmt/impl/test/sut/deploy.cpp | 2 +- source/dnode/mnode/impl/inc/mndDef.h | 3 +- source/dnode/mnode/impl/src/mndTrans.c | 88 +++++++++++++-------- source/dnode/mnode/impl/src/mndUser.c | 18 +---- source/dnode/mnode/sdb/src/sdbHash.c | 11 +-- 6 files changed, 97 insertions(+), 93 deletions(-) diff --git a/source/dnode/mgmt/impl/test/dnode/dnode.cpp b/source/dnode/mgmt/impl/test/dnode/dnode.cpp index fba3794f6a..580fe8e131 100644 --- a/source/dnode/mgmt/impl/test/dnode/dnode.cpp +++ b/source/dnode/mgmt/impl/test/dnode/dnode.cpp @@ -379,41 +379,41 @@ TEST_F(DndTestDnode, RestartDnode_01) { const char* fqdn = "localhost"; const char* firstEp = "localhost:9521"; pServer1 = startServer("/tmp/dndTestDnode1", fqdn, 9521, firstEp); - // pServer1 = startServer("/tmp/dndTestDnode3", fqdn, 9523, firstEp); - // pServer1 = startServer("/tmp/dndTestDnode4", fqdn, 9524, firstEp); - // pServer1 = startServer("/tmp/dndTestDnode5", fqdn, 9525, firstEp); + pServer3 = startServer("/tmp/dndTestDnode3", fqdn, 9523, firstEp); + pServer4 = startServer("/tmp/dndTestDnode4", fqdn, 9524, firstEp); + pServer5 = startServer("/tmp/dndTestDnode5", fqdn, 9525, firstEp); uInfo("all server is running"); - // taosMsleep(1300); - // SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DNODE, "show dnodes", 7); - // SendThenCheckShowRetrieveMsg(4); - // CheckInt16(1); - // CheckInt16(3); - // CheckInt16(4); - // CheckInt16(5); - // CheckBinary("localhost:9521", TSDB_EP_LEN); - // CheckBinary("localhost:9523", TSDB_EP_LEN); - // CheckBinary("localhost:9524", TSDB_EP_LEN); - // CheckBinary("localhost:9525", TSDB_EP_LEN); - // CheckInt16(0); - // CheckInt16(0); - // CheckInt16(0); - // CheckInt16(0); - // CheckInt16(1); - // CheckInt16(1); - // CheckInt16(1); - // CheckInt16(1); - // CheckBinary("ready", 10); - // CheckBinary("ready", 10); - // CheckBinary("ready", 10); - // CheckBinary("ready", 10); - // CheckTimestamp(); - // CheckTimestamp(); - // CheckTimestamp(); - // CheckTimestamp(); - // CheckBinary("", 24); - // CheckBinary("", 24); - // CheckBinary("", 24); - // CheckBinary("", 24); + taosMsleep(1300); + SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DNODE, "show dnodes", 7); + SendThenCheckShowRetrieveMsg(4); + CheckInt16(1); + CheckInt16(3); + CheckInt16(4); + CheckInt16(5); + CheckBinary("localhost:9521", TSDB_EP_LEN); + CheckBinary("localhost:9523", TSDB_EP_LEN); + CheckBinary("localhost:9524", TSDB_EP_LEN); + CheckBinary("localhost:9525", TSDB_EP_LEN); + CheckInt16(0); + CheckInt16(0); + CheckInt16(0); + CheckInt16(0); + CheckInt16(1); + CheckInt16(1); + CheckInt16(1); + CheckInt16(1); + CheckBinary("ready", 10); + CheckBinary("ready", 10); + CheckBinary("ready", 10); + CheckBinary("ready", 10); + CheckTimestamp(); + CheckTimestamp(); + CheckTimestamp(); + CheckTimestamp(); + CheckBinary("", 24); + CheckBinary("", 24); + CheckBinary("", 24); + CheckBinary("", 24); } diff --git a/source/dnode/mgmt/impl/test/sut/deploy.cpp b/source/dnode/mgmt/impl/test/sut/deploy.cpp index 61fc27a595..640f300699 100644 --- a/source/dnode/mgmt/impl/test/sut/deploy.cpp +++ b/source/dnode/mgmt/impl/test/sut/deploy.cpp @@ -18,7 +18,7 @@ void initLog(const char* path) { dDebugFlag = 143; vDebugFlag = 0; - mDebugFlag = 143; + mDebugFlag = 207; cDebugFlag = 0; jniDebugFlag = 0; tmrDebugFlag = 0; diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 5c672c74be..74b4b1faa3 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -73,7 +73,8 @@ typedef enum { TRN_STAGE_EXECUTE = 2, TRN_STAGE_COMMIT = 3, TRN_STAGE_ROLLBACK = 4, - TRN_STAGE_RETRY = 5 + TRN_STAGE_RETRY = 5, + TRN_STAGE_OVER = 6, } ETrnStage; typedef enum { TRN_POLICY_ROLLBACK = 1, TRN_POLICY_RETRY = 2 } ETrnPolicy; diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 65957e44bc..9ab84d6557 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -58,7 +58,7 @@ int32_t mndInitTrans(SMnode *pMnode) { void mndCleanupTrans(SMnode *pMnode) {} static SSdbRaw *mndTransActionEncode(STrans *pTrans) { - int32_t rawDataLen = 16 * sizeof(int32_t); + int32_t rawDataLen = 16 * sizeof(int32_t) + TSDB_TRN_RESERVE_SIZE; int32_t redoLogNum = taosArrayGetSize(pTrans->redoLogs); int32_t undoLogNum = taosArrayGetSize(pTrans->undoLogs); int32_t commitLogNum = taosArrayGetSize(pTrans->commitLogs); @@ -88,7 +88,6 @@ static SSdbRaw *mndTransActionEncode(STrans *pTrans) { int32_t dataPos = 0; SDB_SET_INT32(pRaw, dataPos, pTrans->id) - SDB_SET_INT8(pRaw, dataPos, pTrans->stage) SDB_SET_INT8(pRaw, dataPos, pTrans->policy) SDB_SET_INT32(pRaw, dataPos, redoLogNum) SDB_SET_INT32(pRaw, dataPos, undoLogNum) @@ -166,7 +165,6 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { int32_t dataPos = 0; SDB_GET_INT32(pRaw, pRow, dataPos, &pTrans->id) - SDB_GET_INT8(pRaw, pRow, dataPos, (int8_t *)&pTrans->stage) SDB_GET_INT8(pRaw, pRow, dataPos, (int8_t *)&pTrans->policy) SDB_GET_INT32(pRaw, pRow, dataPos, &redoLogNum) SDB_GET_INT32(pRaw, pRow, dataPos, &undoLogNum) @@ -231,6 +229,7 @@ TRANS_DECODE_OVER: } static int32_t mndTransActionInsert(SSdb *pSdb, STrans *pTrans) { + pTrans->stage = TRN_STAGE_PREPARE; mTrace("trans:%d, perform insert action, stage:%s", pTrans->id, mndTransStageStr(pTrans->stage)); return 0; } @@ -263,7 +262,7 @@ void mndReleaseTrans(SMnode *pMnode, STrans *pTrans) { sdbRelease(pSdb, pTrans); } -static int32_t trnGenerateTransId() { +static int32_t mndGenerateTransId() { static int32_t tmp = 0; return ++tmp; } @@ -304,7 +303,7 @@ STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, void *rpcHandle) { return NULL; } - pTrans->id = trnGenerateTransId(); + pTrans->id = mndGenerateTransId(); pTrans->stage = TRN_STAGE_PREPARE; pTrans->policy = policy; pTrans->rpcHandle = rpcHandle; @@ -428,6 +427,7 @@ int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans) { } mDebug("trans:%d, prepare finished", pNewTrans->id); + pNewTrans->rpcHandle = pTrans->rpcHandle; mndTransExecute(pMnode, pNewTrans); mndReleaseTrans(pMnode, pNewTrans); return 0; @@ -443,19 +443,21 @@ int32_t mndTransCommit(SMnode *pMnode, STrans *pTrans) { } sdbSetRawStatus(pRaw, SDB_STATUS_DROPPED); - mTrace("trans:%d, start sync", pTrans->id); - int32_t code = mndSyncPropose(pMnode, pRaw); - if (code != 0) { - mError("trans:%d, failed to sync since %s", pTrans->id, terrstr()); - sdbFreeRaw(pRaw); - return -1; - } + if (taosArrayGetSize(pTrans->commitLogs) != 0) { + mTrace("trans:%d, start sync", pTrans->id); + int32_t code = mndSyncPropose(pMnode, pRaw); + if (code != 0) { + mError("trans:%d, failed to sync since %s", pTrans->id, terrstr()); + sdbFreeRaw(pRaw); + return -1; + } - mTrace("trans:%d, sync finished", pTrans->id); - code = sdbWrite(pMnode->pSdb, pRaw); - if (code != 0) { - mError("trans:%d, failed to write sdb since %s", pTrans->id, terrstr()); - return -1; + mTrace("trans:%d, sync finished", pTrans->id); + code = sdbWrite(pMnode->pSdb, pRaw); + if (code != 0) { + mError("trans:%d, failed to write sdb since %s", pTrans->id, terrstr()); + return -1; + } } mDebug("trans:%d, commit finished", pTrans->id); @@ -521,45 +523,58 @@ static int32_t mndTransExecuteArray(SMnode *pMnode, SArray *pArray) { } static int32_t mndTransExecuteRedoLogs(SMnode *pMnode, STrans *pTrans) { - int32_t code = mndTransExecuteArray(pMnode, pTrans->redoLogs); - if (code != 0) { - mError("trans:%d, failed to execute redo logs since %s", pTrans->id, terrstr()) - } else { - mTrace("trans:%d, execute redo logs finished", pTrans->id) + int32_t code = 0; + if (taosArrayGetSize(pTrans->redoLogs) != 0) { + code = mndTransExecuteArray(pMnode, pTrans->redoLogs); + if (code != 0) { + mError("trans:%d, failed to execute redo logs since %s", pTrans->id, terrstr()) + } else { + mTrace("trans:%d, execute redo logs finished", pTrans->id) + } } return code; } static int32_t mndTransExecuteUndoLogs(SMnode *pMnode, STrans *pTrans) { - int32_t code = mndTransExecuteArray(pMnode, pTrans->undoLogs); - if (code != 0) { - mError("trans:%d, failed to execute undo logs since %s", pTrans->id, terrstr()) - } else { - mTrace("trans:%d, execute undo logs finished", pTrans->id) + int32_t code = 0; + if (taosArrayGetSize(pTrans->undoLogs) != 0) { + code = mndTransExecuteArray(pMnode, pTrans->undoLogs); + if (code != 0) { + mError("trans:%d, failed to execute undo logs since %s", pTrans->id, terrstr()) + } else { + mTrace("trans:%d, execute undo logs finished", pTrans->id) + } } return code; } static int32_t mndTransExecuteCommitLogs(SMnode *pMnode, STrans *pTrans) { - int32_t code = mndTransExecuteArray(pMnode, pTrans->commitLogs); - if (code != 0) { - mError("trans:%d, failed to execute commit logs since %s", pTrans->id, terrstr()) - } else { - mTrace("trans:%d, execute commit logs finished", pTrans->id) + int32_t code = 0; + if (taosArrayGetSize(pTrans->commitLogs) != 0) { + code = mndTransExecuteArray(pMnode, pTrans->commitLogs); + if (code != 0) { + mError("trans:%d, failed to execute commit logs since %s", pTrans->id, terrstr()) + } else { + mTrace("trans:%d, execute commit logs finished", pTrans->id) + } } return code; } static int32_t mndTransExecuteRedoActions(SMnode *pMnode, STrans *pTrans) { - mTrace("trans:%d, execute redo actions finished", pTrans->id); + if (taosArrayGetSize(pTrans->redoActions) != 0) { + mTrace("trans:%d, execute redo actions finished", pTrans->id); + } return 0; } static int32_t mndTransExecuteUndoActions(SMnode *pMnode, STrans *pTrans) { - mTrace("trans:%d, execute undo actions finished", pTrans->id); + if (taosArrayGetSize(pTrans->undoActions) != 0) { + mTrace("trans:%d, execute undo actions finished", pTrans->id); + } return 0; } @@ -603,7 +618,7 @@ static int32_t mndTransPerformCommitStage(SMnode *pMnode, STrans *pTrans) { int32_t code = mndTransExecuteCommitLogs(pMnode, pTrans); if (code == 0) { - pTrans->stage = TRN_STAGE_COMMIT; + pTrans->stage = TRN_STAGE_OVER; mTrace("trans:%d, commit stage finished", pTrans->id); } else { if (pTrans->policy == TRN_POLICY_ROLLBACK) { @@ -671,6 +686,9 @@ static void mndTransExecute(SMnode *pMnode, STrans *pTrans) { case TRN_STAGE_RETRY: code = mndTransPerformRetryStage(pMnode, pTrans); break; + default: + mndTransSendRpcRsp(pTrans, 0); + return; } } diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index e6bf30f41b..c4599b8e49 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -217,23 +217,7 @@ static int32_t mndCreateUser(SMnode *pMnode, char *acct, char *user, char *pass, mndTransDrop(pTrans); return -1; } - sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING); - - SSdbRaw *pUndoRaw = mndUserActionEncode(&userObj); - if (pUndoRaw == NULL || mndTransAppendUndolog(pTrans, pUndoRaw) != 0) { - mError("trans:%d, failed to append undo log since %s", pTrans->id, terrstr()); - mndTransDrop(pTrans); - return -1; - } - sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED); - - SSdbRaw *pCommitRaw = mndUserActionEncode(&userObj); - if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { - mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr()); - mndTransDrop(pTrans); - return -1; - } - sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); + sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY); if (mndTransPrepare(pMnode, pTrans) != 0) { mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); diff --git a/source/dnode/mnode/sdb/src/sdbHash.c b/source/dnode/mnode/sdb/src/sdbHash.c index 8d8daf5ce5..c3874caffc 100644 --- a/source/dnode/mnode/sdb/src/sdbHash.c +++ b/source/dnode/mnode/sdb/src/sdbHash.c @@ -132,11 +132,6 @@ static int32_t sdbDeleteRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow * taosHashRemove(hash, pOldRow->pObj, keySize); taosWUnLockLatch(pLock); - SdbDeleteFp deleteFp = pSdb->deleteFps[pOldRow->type]; - if (deleteFp != NULL) { - code = (*deleteFp)(pSdb, pOldRow->pObj); - } - sdbRelease(pSdb, pOldRow->pObj); sdbFreeRow(pRow); return code; @@ -161,6 +156,7 @@ int32_t sdbWriteNotFree(SSdb *pSdb, SSdbRaw *pRaw) { case SDB_STATUS_CREATING: code = sdbInsertRow(pSdb, hash, pRaw, pRow, keySize); break; + case SDB_STATUS_UPDATING: case SDB_STATUS_READY: case SDB_STATUS_DROPPING: code = sdbUpdateRow(pSdb, hash, pRaw, pRow, keySize); @@ -228,6 +224,11 @@ void sdbRelease(SSdb *pSdb, void *pObj) { int32_t ref = atomic_sub_fetch_32(&pRow->refCount, 1); if (ref <= 0 && pRow->status == SDB_STATUS_DROPPED) { + SdbDeleteFp deleteFp = pSdb->deleteFps[pRow->type]; + if (deleteFp != NULL) { + (*deleteFp)(pSdb, pRow->pObj); + } + sdbFreeRow(pRow); } From b7a513954a3f9291227ca336493d55772b922860 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 14 Dec 2021 18:10:55 +0800 Subject: [PATCH 26/43] TD-10431 test for drop user --- source/dnode/mnode/impl/src/mndDnode.c | 36 ++------------------------ source/dnode/mnode/impl/src/mndUser.c | 26 +------------------ 2 files changed, 3 insertions(+), 59 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index fbcf623d3c..4ce557fc02 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -413,23 +413,7 @@ static int32_t mndCreateDnode(SMnode *pMnode, SMnodeMsg *pMsg, SCreateDnodeMsg * mndTransDrop(pTrans); return -1; } - sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING); - - SSdbRaw *pUndoRaw = mndDnodeActionEncode(&dnodeObj); - if (pUndoRaw == NULL || mndTransAppendUndolog(pTrans, pUndoRaw) != 0) { - mError("trans:%d, failed to append undo log since %s", pTrans->id, terrstr()); - mndTransDrop(pTrans); - return -1; - } - sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED); - - SSdbRaw *pCommitRaw = mndDnodeActionEncode(&dnodeObj); - if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { - mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr()); - mndTransDrop(pTrans); - return -1; - } - sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); + sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY); if (mndTransPrepare(pMnode, pTrans) != 0) { mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); @@ -485,23 +469,7 @@ static int32_t mndDropDnode(SMnode *pMnode, SMnodeMsg *pMsg, SDnodeObj *pDnode) mndTransDrop(pTrans); return -1; } - sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING); - - SSdbRaw *pUndoRaw = mndDnodeActionEncode(pDnode); - if (pUndoRaw == NULL || mndTransAppendUndolog(pTrans, pUndoRaw) != 0) { - mError("trans:%d, failed to append undo log since %s", pTrans->id, terrstr()); - mndTransDrop(pTrans); - return -1; - } - sdbSetRawStatus(pUndoRaw, SDB_STATUS_READY); - - SSdbRaw *pCommitRaw = mndDnodeActionEncode(pDnode); - if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { - mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr()); - mndTransDrop(pTrans); - return -1; - } - sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED); + sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPED); if (mndTransPrepare(pMnode, pTrans) != 0) { mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index c4599b8e49..43b9e94e06 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -245,14 +245,6 @@ static int32_t mndUpdateUser(SMnode *pMnode, SUserObj *pOldUser, SUserObj *pNewU } sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY); - SSdbRaw *pUndoRaw = mndUserActionEncode(pOldUser); - if (pUndoRaw == NULL || mndTransAppendUndolog(pTrans, pUndoRaw) != 0) { - mError("trans:%d, failed to append undo log since %s", pTrans->id, terrstr()); - mndTransDrop(pTrans); - return -1; - } - sdbSetRawStatus(pUndoRaw, SDB_STATUS_READY); - if (mndTransPrepare(pMnode, pTrans) != 0) { mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); mndTransDrop(pTrans); @@ -277,23 +269,7 @@ static int32_t mndDropUser(SMnode *pMnode, SMnodeMsg *pMsg, SUserObj *pUser) { mndTransDrop(pTrans); return -1; } - sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING); - - SSdbRaw *pUndoRaw = mndUserActionEncode(pUser); - if (pUndoRaw == NULL || mndTransAppendUndolog(pTrans, pUndoRaw) != 0) { - mError("trans:%d, failed to append undo log since %s", pTrans->id, terrstr()); - mndTransDrop(pTrans); - return -1; - } - sdbSetRawStatus(pUndoRaw, SDB_STATUS_READY); - - SSdbRaw *pCommitRaw = mndUserActionEncode(pUser); - if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { - mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr()); - mndTransDrop(pTrans); - return -1; - } - sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED); + sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPED); if (mndTransPrepare(pMnode, pTrans) != 0) { mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); From e5bc959fd7a5ebed313605eabb2041e63e6bef1f Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 14 Dec 2021 18:12:29 +0800 Subject: [PATCH 27/43] update fst search frame --- source/libs/index/inc/index_fst_automation.h | 7 ++- source/libs/index/src/index_fst.c | 2 + source/libs/index/src/index_fst_automation.c | 53 +++++++++++++------- 3 files changed, 41 insertions(+), 21 deletions(-) diff --git a/source/libs/index/inc/index_fst_automation.h b/source/libs/index/inc/index_fst_automation.h index 7eb2ba97ae..8050b85b08 100644 --- a/source/libs/index/inc/index_fst_automation.h +++ b/source/libs/index/inc/index_fst_automation.h @@ -40,7 +40,8 @@ typedef struct Complement { // automation typedef struct AutomationCtx { AutomationType type; - void *data; + void *stdata; + char *data; } AutomationCtx; @@ -58,7 +59,9 @@ typedef struct StartWithStateValue { } ; } StartWithStateValue; +StartWithStateValue *startWithStateValueCreate(StartWithStateKind kind, ValueType ty, void *val); StartWithStateValue *startWithStateValueDump(StartWithStateValue *sv); +void startWithStateValueDestroy(void *sv); typedef struct AutomationFunc { @@ -70,7 +73,7 @@ typedef struct AutomationFunc { void* (*acceptEof)(AutomationCtx *ct, void *state); } AutomationFunc; -AutomationCtx *automCtxCreate(void *data, AutomationType type); +AutomationCtx *automCtxCreate(void *data, AutomationType atype); void automCtxDestroy(AutomationCtx *ctx); extern AutomationFunc automFuncs[]; diff --git a/source/libs/index/src/index_fst.c b/source/libs/index/src/index_fst.c index e2708c5919..37bdcb0ecf 100644 --- a/source/libs/index/src/index_fst.c +++ b/source/libs/index/src/index_fst.c @@ -17,6 +17,7 @@ #include "tcoding.h" #include "tchecksum.h" #include "indexInt.h" +#include "index_fst_automation.h" static void fstPackDeltaIn(FstCountingWriter *wrt, CompiledAddr nodeAddr, CompiledAddr transAddr, uint8_t nBytes) { @@ -1402,6 +1403,7 @@ void swsResultDestroy(StreamWithStateResult *result) { if (NULL == result) { return; } fstSliceDestroy(&result->data); + startWithStateValueDestroy(result->state); free(result); } diff --git a/source/libs/index/src/index_fst_automation.c b/source/libs/index/src/index_fst_automation.c index bb0f0da76d..d905147654 100644 --- a/source/libs/index/src/index_fst_automation.c +++ b/source/libs/index/src/index_fst_automation.c @@ -34,7 +34,8 @@ StartWithStateValue *startWithStateValueCreate(StartWithStateKind kind, ValueTyp } return nsv; } -void startWithStateValueDestroy(StartWithStateValue *sv) { +void startWithStateValueDestroy(void *val) { + StartWithStateValue *sv = (StartWithStateValue *)val; if (sv == NULL) { return; } if (sv->type == FST_INT) { @@ -68,19 +69,28 @@ StartWithStateValue *startWithStateValueDump(StartWithStateValue *sv) { static void* prefixStart(AutomationCtx *ctx) { StartWithStateValue *data = (StartWithStateValue *)(ctx->data); - - return data; + return startWithStateValueDump(data); }; -static bool prefixIsMatch(AutomationCtx *ctx, void *data) { - return true; +static bool prefixIsMatch(AutomationCtx *ctx, void *sv) { + StartWithStateValue* ssv = (StartWithStateValue *)sv; + return ssv->val == strlen(ctx->data); } -static bool prefixCanMatch(AutomationCtx *ctx, void *data) { - return true; +static bool prefixCanMatch(AutomationCtx *ctx, void *sv) { + StartWithStateValue* ssv = (StartWithStateValue *)sv; + return ssv->val >= 0; } static bool prefixWillAlwaysMatch(AutomationCtx *ctx, void *state) { return true; } static void* prefixAccept(AutomationCtx *ctx, void *state, uint8_t byte) { + StartWithStateValue* ssv = (StartWithStateValue *)state; + if (ssv == NULL || ctx == NULL) {return NULL;} + + char *data = ctx->data; + if ((strlen(data) > ssv->val) && data[ssv->val] == byte) { + int val = ssv->val + 1; + return startWithStateValueCreate(Running, FST_INT, &val); + } return NULL; } static void* prefixAcceptEof(AutomationCtx *ctx, void *state) { @@ -129,28 +139,33 @@ AutomationFunc automFuncs[] = {{ // add more search type }; -AutomationCtx* automCtxCreate(void *data, AutomationType type) { +AutomationCtx* automCtxCreate(void *data,AutomationType atype) { AutomationCtx *ctx = calloc(1, sizeof(AutomationCtx)); if (ctx == NULL) { return NULL; } - if (type == AUTOMATION_PREFIX) { - StartWithStateValue *swsv = (StartWithStateValue *)calloc(1, sizeof(StartWithStateValue)); - swsv->kind = Done; - //swsv->value = NULL; - ctx->data = (void *)swsv; - } else if (type == AUTMMATION_MATCH) { + StartWithStateValue *sv = NULL; + if (atype == AUTOMATION_PREFIX) { + sv = startWithStateValueCreate(Running, FST_INT, 0); + ctx->stdata = (void *)sv; + } else if (atype == AUTMMATION_MATCH) { } else { // add more search type } - ctx->type = type; + char* src = (char *)data; + size_t len = strlen(src); + char* dst = (char *)malloc(len * sizeof(char) + 1); + memcpy(dst, src, len); + dst[len] = 0; + + ctx->data = dst; + ctx->type = atype; + ctx->stdata = (void *)sv; return ctx; } void automCtxDestroy(AutomationCtx *ctx) { - if (ctx->type == AUTOMATION_PREFIX) { - free(ctx->data); - } else if (ctx->type == AUTMMATION_MATCH) { - } + startWithStateValueDestroy(ctx->stdata); + free(ctx->data); free(ctx); } From c048dc5b60b5841557f1f43b00f33e6c65597841 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Tue, 14 Dec 2021 18:14:45 +0800 Subject: [PATCH 28/43] refactor wal --- include/libs/wal/wal.h | 44 ++-- include/util/tref.h | 2 + source/libs/wal/inc/walInt.h | 14 +- source/libs/wal/src/walMeta.c | 18 +- source/libs/wal/src/walMgmt.c | 97 ++++----- source/libs/wal/src/walRead.c | 8 +- source/libs/wal/src/walSeek.c | 6 +- source/libs/wal/src/walUtil.c | 120 ----------- source/libs/wal/src/walWrite.c | 310 +-------------------------- source/libs/wal/test/walMetaTest.cpp | 18 +- 10 files changed, 98 insertions(+), 539 deletions(-) delete mode 100644 source/libs/wal/src/walUtil.c diff --git a/include/libs/wal/wal.h b/include/libs/wal/wal.h index 744275e6ff..e19d65837a 100644 --- a/include/libs/wal/wal.h +++ b/include/libs/wal/wal.h @@ -32,23 +32,19 @@ extern int32_t wDebugFlag; #define wDebug(...) { if (wDebugFlag & DEBUG_DEBUG) { taosPrintLog("WAL ", wDebugFlag, __VA_ARGS__); }} #define wTrace(...) { if (wDebugFlag & DEBUG_TRACE) { taosPrintLog("WAL ", wDebugFlag, __VA_ARGS__); }} -#define WAL_PREFIX "wal" -#define WAL_PREFIX_LEN 3 +#define WAL_HEAD_VER 0 #define WAL_NOSUFFIX_LEN 20 #define WAL_SUFFIX_AT (WAL_NOSUFFIX_LEN+1) #define WAL_LOG_SUFFIX "log" #define WAL_INDEX_SUFFIX "idx" #define WAL_REFRESH_MS 1000 -#define WAL_MAX_SIZE (TSDB_MAX_WAL_SIZE + sizeof(SWalHead) + 16) +#define WAL_MAX_SIZE (TSDB_MAX_WAL_SIZE + sizeof(SWalHead)) #define WAL_PATH_LEN (TSDB_FILENAME_LEN + 12) #define WAL_FILE_LEN (WAL_PATH_LEN + 32) -#define WAL_IDX_ENTRY_SIZE (sizeof(int64_t)*2) -#define WAL_CUR_POS_WRITABLE 1 -#define WAL_CUR_FILE_WRITABLE 2 -#define WAL_CUR_FAILED 4 +#define WAL_CUR_FAILED 1 -#pragma pack(push,1) +#pragma pack(push, 1) typedef enum { TAOS_WAL_NOLOG = 0, TAOS_WAL_WRITE = 1, @@ -56,11 +52,11 @@ typedef enum { } EWalType; typedef struct SWalReadHead { - int8_t sver; + int8_t headVer; uint8_t msgType; int8_t reserved[2]; int32_t len; - //int64_t ingestTs; //not implemented + int64_t ingestTs; //not implemented int64_t version; char body[]; } SWalReadHead; @@ -72,18 +68,10 @@ typedef struct { int32_t rollPeriod; // secs int64_t retentionSize; int64_t segSize; - EWalType level; // wal level + EWalType level; // wal level } SWalCfg; typedef struct { - //union { - //uint32_t info; - //struct { - //uint32_t sver:3; - //uint32_t msgtype: 5; - //uint32_t reserved : 24; - //}; - //}; uint32_t cksumHead; uint32_t cksumBody; SWalReadHead head; @@ -102,16 +90,16 @@ typedef struct SWal { SWalCfg cfg; SWalVer vers; //file set - int32_t writeCur; int64_t writeLogTfd; int64_t writeIdxTfd; + int32_t writeCur; SArray* fileInfoSet; - //ctl - int32_t curStatus; - int32_t fsyncSeq; + //statistics int64_t totSize; - int64_t refId; int64_t lastRollSeq; + //ctl + int32_t fsyncSeq; + int64_t refId; pthread_mutex_t mutex; //path char path[WAL_PATH_LEN]; @@ -131,7 +119,7 @@ typedef struct SWalReadHandle { } SWalReadHandle; #pragma pack(pop) -typedef int32_t (*FWalWrite)(void *ahandle, void *pHead); +//typedef int32_t (*FWalWrite)(void *ahandle, void *pHead); // module initialization int32_t walInit(); @@ -151,8 +139,8 @@ int32_t walCommit(SWal *, int64_t ver); // truncate after int32_t walRollback(SWal *, int64_t ver); // notify that previous logs can be pruned safely -int32_t walBeginTakeSnapshot(SWal *, int64_t ver); -int32_t walEndTakeSnapshot(SWal *); +int32_t walBeginSnapshot(SWal *, int64_t ver); +int32_t walEndSnapshot(SWal *); //int32_t walDataCorrupted(SWal*); // read @@ -161,7 +149,7 @@ void walCloseReadHandle(SWalReadHandle *); int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver); int32_t walRead(SWal *, SWalHead **, int64_t ver); -int32_t walReadWithFp(SWal *, FWalWrite writeFp, int64_t verStart, int32_t readNum); +//int32_t walReadWithFp(SWal *, FWalWrite writeFp, int64_t verStart, int32_t readNum); // lifecycle check int64_t walGetFirstVer(SWal *); diff --git a/include/util/tref.h b/include/util/tref.h index cc7d075f52..6680204d63 100644 --- a/include/util/tref.h +++ b/include/util/tref.h @@ -17,6 +17,8 @@ #ifndef _TD_UTIL_REF_H #define _TD_UTIL_REF_H +#include "os.h" + #ifdef __cplusplus extern "C" { #endif diff --git a/source/libs/wal/inc/walInt.h b/source/libs/wal/inc/walInt.h index e546a87326..48142878c3 100644 --- a/source/libs/wal/inc/walInt.h +++ b/source/libs/wal/inc/walInt.h @@ -33,12 +33,10 @@ typedef struct WalFileInfo { int64_t fileSize; } WalFileInfo; -#pragma pack(push,1) typedef struct WalIdxEntry { int64_t ver; int64_t offset; } WalIdxEntry; -#pragma pack(pop) static inline int32_t compareWalFileInfo(const void* pLeft, const void* pRight) { WalFileInfo* pInfoLeft = (WalFileInfo*)pLeft; @@ -107,8 +105,16 @@ static inline uint32_t walCalcBodyCksum(const void* body, uint32_t len) { return taosCalcChecksum(0, (uint8_t*)body, len); } -int walReadMeta(SWal* pWal); -int walWriteMeta(SWal* pWal); +static inline void walResetVer(SWalVer* pVer) { + pVer->firstVer = -1; + pVer->verInSnapshotting = -1; + pVer->snapshotVer = -1; + pVer->commitVer = -1; + pVer->lastVer = -1; +} + +int walLoadMeta(SWal* pWal); +int walSaveMeta(SWal* pWal); int walRollFileInfo(SWal* pWal); char* walMetaSerialize(SWal* pWal); diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index 49f4fde3a0..aa592b4fe8 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -24,18 +24,22 @@ #include #include -int64_t walGetFirstVer(SWal *pWal) { +int64_t inline walGetFirstVer(SWal *pWal) { return pWal->vers.firstVer; } -int64_t walGetSnaphostVer(SWal *pWal) { +int64_t inline walGetSnaphostVer(SWal *pWal) { return pWal->vers.snapshotVer; } -int64_t walGetLastVer(SWal *pWal) { +int64_t inline walGetLastVer(SWal *pWal) { return pWal->vers.lastVer; } +static inline int walBuildMetaName(SWal* pWal, int metaVer, char* buf) { + return sprintf(buf, "%s/meta-ver%d", pWal->path, metaVer); +} + int walRollFileInfo(SWal* pWal) { int64_t ts = taosGetTimestampSec(); @@ -150,10 +154,6 @@ int walMetaDeserialize(SWal* pWal, const char* bytes) { return 0; } -static inline int walBuildMetaName(SWal* pWal, int metaVer, char* buf) { - return sprintf(buf, "%s/meta-ver%d", pWal->path, metaVer); -} - static int walFindCurMetaVer(SWal* pWal) { const char * pattern = "^meta-ver[0-9]+$"; regex_t walMetaRegexPattern; @@ -182,7 +182,7 @@ static int walFindCurMetaVer(SWal* pWal) { return metaVer; } -int walWriteMeta(SWal* pWal) { +int walSaveMeta(SWal* pWal) { int metaVer = walFindCurMetaVer(pWal); char fnameStr[WAL_FILE_LEN]; walBuildMetaName(pWal, metaVer+1, fnameStr); @@ -207,7 +207,7 @@ int walWriteMeta(SWal* pWal) { return 0; } -int walReadMeta(SWal* pWal) { +int walLoadMeta(SWal* pWal) { ASSERT(pWal->fileInfoSet->size == 0); //find existing meta file int metaVer = walFindCurMetaVer(pWal); diff --git a/source/libs/wal/src/walMgmt.c b/source/libs/wal/src/walMgmt.c index 629451a722..9efeb83cf0 100644 --- a/source/libs/wal/src/walMgmt.c +++ b/source/libs/wal/src/walMgmt.c @@ -21,23 +21,17 @@ #include "compare.h" #include "walInt.h" -//internal -int32_t walGetNextFile(SWal *pWal, int64_t *nextFileId); -int32_t walGetOldFile(SWal *pWal, int64_t curFileId, int32_t minDiff, int64_t *oldFileId); -int32_t walGetNewFile(SWal *pWal, int64_t *newFileId); - typedef struct { - int32_t refSetId; - uint32_t seq; int8_t stop; int8_t inited; + uint32_t seq; + int32_t refSetId; pthread_t thread; } SWalMgmt; static SWalMgmt tsWal = {0, .seq = 1}; static int32_t walCreateThread(); static void walStopThread(); -static int32_t walInitObj(SWal *pWal); static void walFreeObj(void *pWal); int64_t walGetSeq() { @@ -68,7 +62,7 @@ int32_t walInit() { } void walCleanUp() { - int old = atomic_val_compare_exchange_8(&tsWal.inited, 1, 0); + int8_t old = atomic_val_compare_exchange_8(&tsWal.inited, 1, 0); if(old == 0) { return; } @@ -83,48 +77,59 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { terrno = TAOS_SYSTEM_ERROR(errno); return NULL; } - memset(pWal, 0, sizeof(SWal)); - pWal->writeLogTfd = -1; - pWal->writeIdxTfd = -1; - pWal->writeCur = -1; //set config memcpy(&pWal->cfg, pCfg, sizeof(SWalCfg)); + pWal->fsyncSeq = pCfg->fsyncPeriod / 1000; + if(pWal->fsyncSeq <= 0) pWal->fsyncSeq = 1; - //init version info - pWal->vers.firstVer = -1; - pWal->vers.commitVer = -1; - pWal->vers.snapshotVer = -1; - pWal->vers.lastVer = -1; + tstrncpy(pWal->path, path, sizeof(pWal->path)); + if(taosMkDir(pWal->path) != 0) { + wError("vgId:%d, path:%s, failed to create directory since %s", pWal->cfg.vgId, pWal->path, strerror(errno)); + return NULL; + } - pWal->vers.verInSnapshotting = -1; - - pWal->totSize = 0; + //open meta + pWal->writeLogTfd = -1; + pWal->writeIdxTfd = -1; + pWal->writeCur = -1; + pWal->fileInfoSet = taosArrayInit(8, sizeof(WalFileInfo)); + if(pWal->fileInfoSet == NULL) { + wError("vgId:%d, path:%s, failed to init taosArray %s", pWal->cfg.vgId, pWal->path, strerror(errno)); + free(pWal); + return NULL; + } //init status + walResetVer(&pWal->vers); + pWal->totSize = 0; pWal->lastRollSeq = -1; //init write buffer memset(&pWal->writeHead, 0, sizeof(SWalHead)); - pWal->writeHead.head.sver = 0; + pWal->writeHead.head.headVer = WAL_HEAD_VER; - tstrncpy(pWal->path, path, sizeof(pWal->path)); - pthread_mutex_init(&pWal->mutex, NULL); - - pWal->fsyncSeq = pCfg->fsyncPeriod / 1000; - if (pWal->fsyncSeq <= 0) pWal->fsyncSeq = 1; - - if (walInitObj(pWal) != 0) { - walFreeObj(pWal); + if(pthread_mutex_init(&pWal->mutex, NULL) < 0) { + taosArrayDestroy(pWal->fileInfoSet); + free(pWal); return NULL; } - pWal->refId = taosAddRef(tsWal.refSetId, pWal); - if (pWal->refId < 0) { - walFreeObj(pWal); + pWal->refId = taosAddRef(tsWal.refSetId, pWal); + if(pWal->refId < 0) { + pthread_mutex_destroy(&pWal->mutex); + taosArrayDestroy(pWal->fileInfoSet); + free(pWal); + return NULL; + } + + if(walLoadMeta(pWal) < 0) { + taosRemoveRef(tsWal.refSetId, pWal->refId); + pthread_mutex_destroy(&pWal->mutex); + taosArrayDestroy(pWal->fileInfoSet); + free(pWal); return NULL; } - walReadMeta(pWal); wDebug("vgId:%d, wal:%p is opened, level:%d fsyncPeriod:%d", pWal->cfg.vgId, pWal, pWal->cfg.level, pWal->cfg.fsyncPeriod); @@ -152,43 +157,23 @@ int32_t walAlter(SWal *pWal, SWalCfg *pCfg) { } void walClose(SWal *pWal) { - if (pWal == NULL) return; - pthread_mutex_lock(&pWal->mutex); tfClose(pWal->writeLogTfd); pWal->writeLogTfd = -1; tfClose(pWal->writeIdxTfd); pWal->writeIdxTfd = -1; - walWriteMeta(pWal); + walSaveMeta(pWal); taosArrayDestroy(pWal->fileInfoSet); pWal->fileInfoSet = NULL; pthread_mutex_unlock(&pWal->mutex); + taosRemoveRef(tsWal.refSetId, pWal->refId); } -static int32_t walInitObj(SWal *pWal) { - if (taosMkDir(pWal->path) != 0) { - wError("vgId:%d, path:%s, failed to create directory since %s", pWal->cfg.vgId, pWal->path, strerror(errno)); - return TAOS_SYSTEM_ERROR(errno); - } - pWal->fileInfoSet = taosArrayInit(8, sizeof(WalFileInfo)); - if(pWal->fileInfoSet == NULL) { - wError("vgId:%d, path:%s, failed to init taosArray %s", pWal->cfg.vgId, pWal->path, strerror(errno)); - return TAOS_SYSTEM_ERROR(errno); - } - - wDebug("vgId:%d, object is initialized", pWal->cfg.vgId); - return 0; -} - static void walFreeObj(void *wal) { SWal *pWal = wal; wDebug("vgId:%d, wal:%p is freed", pWal->cfg.vgId, pWal); - tfClose(pWal->writeLogTfd); - tfClose(pWal->writeIdxTfd); - taosArrayDestroy(pWal->fileInfoSet); - pWal->fileInfoSet = NULL; pthread_mutex_destroy(&pWal->mutex); tfree(pWal); } diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index b6aafedea3..42fcb8c375 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -54,7 +54,7 @@ static int32_t walReadSeekFilePos(SWalReadHandle *pRead, int64_t fileFirstVer, i int64_t logTfd = pRead->readLogTfd; //seek position - int64_t offset = (ver - fileFirstVer) * WAL_IDX_ENTRY_SIZE; + int64_t offset = (ver - fileFirstVer) * sizeof(WalIdxEntry); code = tfLseek(idxTfd, offset, SEEK_SET); if(code < 0) { return -1; @@ -210,6 +210,6 @@ int32_t walRead(SWal *pWal, SWalHead **ppHead, int64_t ver) { return 0; } -int32_t walReadWithFp(SWal *pWal, FWalWrite writeFp, int64_t verStart, int32_t readNum) { - return 0; -} +/*int32_t walReadWithFp(SWal *pWal, FWalWrite writeFp, int64_t verStart, int32_t readNum) {*/ + /*return 0;*/ +/*}*/ diff --git a/source/libs/wal/src/walSeek.c b/source/libs/wal/src/walSeek.c index 953aae703c..7db5b90c1d 100644 --- a/source/libs/wal/src/walSeek.c +++ b/source/libs/wal/src/walSeek.c @@ -27,7 +27,7 @@ static int walSeekFilePos(SWal* pWal, int64_t ver) { int64_t logTfd = pWal->writeLogTfd; //seek position - int64_t offset = (ver - walGetCurFileFirstVer(pWal)) * WAL_IDX_ENTRY_SIZE; + int64_t offset = (ver - walGetCurFileFirstVer(pWal)) * sizeof(WalIdxEntry); code = tfLseek(idxTfd, offset, SEEK_SET); if(code != 0) { return -1; @@ -66,8 +66,6 @@ int walChangeFileToLast(SWal *pWal) { //switch file pWal->writeIdxTfd = idxTfd; pWal->writeLogTfd = logTfd; - //change status - pWal->curStatus = WAL_CUR_FILE_WRITABLE; return 0; } @@ -93,13 +91,11 @@ int walChangeFile(SWal *pWal, int64_t ver) { int64_t fileFirstVer = pRet->firstVer; //closed if(taosArrayGetLast(pWal->fileInfoSet) != pRet) { - pWal->curStatus &= ~WAL_CUR_FILE_WRITABLE; walBuildIdxName(pWal, fileFirstVer, fnameStr); idxTfd = tfOpenRead(fnameStr); walBuildLogName(pWal, fileFirstVer, fnameStr); logTfd = tfOpenRead(fnameStr); } else { - pWal->curStatus |= WAL_CUR_FILE_WRITABLE; walBuildIdxName(pWal, fileFirstVer, fnameStr); idxTfd = tfOpenReadWrite(fnameStr); walBuildLogName(pWal, fileFirstVer, fnameStr); diff --git a/source/libs/wal/src/walUtil.c b/source/libs/wal/src/walUtil.c deleted file mode 100644 index 849d0c3e51..0000000000 --- a/source/libs/wal/src/walUtil.c +++ /dev/null @@ -1,120 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "walInt.h" - -#if 0 -int32_t walGetNextFile(SWal *pWal, int64_t *nextFileId) { - int64_t curFileId = *nextFileId; - int64_t minFileId = INT64_MAX; - - DIR *dir = opendir(pWal->path); - if (dir == NULL) { - wError("vgId:%d, path:%s, failed to open since %s", pWal->vgId, pWal->path, strerror(errno)); - return -1; - } - - struct dirent *ent; - while ((ent = readdir(dir)) != NULL) { - char *name = ent->d_name; - - if (strncmp(name, WAL_PREFIX, WAL_PREFIX_LEN) == 0) { - int64_t id = atoll(name + WAL_PREFIX_LEN); - if (id <= curFileId) continue; - - if (id < minFileId) { - minFileId = id; - } - } - } - closedir(dir); - - if (minFileId == INT64_MAX) return -1; - - *nextFileId = minFileId; - wTrace("vgId:%d, path:%s, curFileId:%" PRId64 " nextFileId:%" PRId64, pWal->vgId, pWal->path, curFileId, *nextFileId); - - return 0; -} - -int32_t walGetOldFile(SWal *pWal, int64_t curFileId, int32_t minDiff, int64_t *oldFileId) { - int64_t minFileId = INT64_MAX; - - DIR *dir = opendir(pWal->path); - if (dir == NULL) { - wError("vgId:%d, path:%s, failed to open since %s", pWal->vgId, pWal->path, strerror(errno)); - return -1; - } - - struct dirent *ent; - while ((ent = readdir(dir)) != NULL) { - char *name = ent->d_name; - - if (strncmp(name, WAL_PREFIX, WAL_PREFIX_LEN) == 0) { - int64_t id = atoll(name + WAL_PREFIX_LEN); - if (id >= curFileId) continue; - - minDiff--; - if (id < minFileId) { - minFileId = id; - } - } - } - closedir(dir); - - if (minFileId == INT64_MAX) return -1; - if (minDiff > 0) return -1; - - *oldFileId = minFileId; - wTrace("vgId:%d, path:%s, curFileId:%" PRId64 " oldFildId:%" PRId64, pWal->vgId, pWal->path, curFileId, *oldFileId); - - return 0; -} - -int32_t walGetNewFile(SWal *pWal, int64_t *newFileId) { - int64_t maxFileId = INT64_MIN; - - DIR *dir = opendir(pWal->path); - if (dir == NULL) { - wError("vgId:%d, path:%s, failed to open since %s", pWal->vgId, pWal->path, strerror(errno)); - return -1; - } - - struct dirent *ent; - while ((ent = readdir(dir)) != NULL) { - char *name = ent->d_name; - - if (strncmp(name, WAL_PREFIX, WAL_PREFIX_LEN) == 0) { - int64_t id = atoll(name + WAL_PREFIX_LEN); - if (id > maxFileId) { - maxFileId = id; - } - } - } - closedir(dir); - - if (maxFileId == INT64_MIN) { - *newFileId = 0; - } else { - *newFileId = maxFileId; - } - - wTrace("vgId:%d, path:%s, newFileId:%" PRId64, pWal->vgId, pWal->path, *newFileId); - - return 0; -} -#endif diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index 994b8fc333..ffbb19c6b7 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -21,98 +21,6 @@ #include "tfile.h" #include "walInt.h" - -#if 0 -static int32_t walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp, char *name, int64_t fileId); - -int32_t walRenew(void *handle) { - if (handle == NULL) return 0; - - SWal * pWal = handle; - int32_t code = 0; - - /*if (pWal->stop) {*/ - /*wDebug("vgId:%d, do not create a new wal file", pWal->vgId);*/ - /*return 0;*/ - /*}*/ - - pthread_mutex_lock(&pWal->mutex); - - if (tfValid(pWal->logTfd)) { - tfClose(pWal->logTfd); - wDebug("vgId:%d, file:%s, it is closed while renew", pWal->vgId, pWal->logName); - } - - /*if (pWal->keep == TAOS_WAL_KEEP) {*/ - /*pWal->fileId = 0;*/ - /*} else {*/ - /*if (walGetNewFile(pWal, &pWal->fileId) != 0) pWal->fileId = 0;*/ - /*pWal->fileId++;*/ - /*}*/ - - snprintf(pWal->logName, sizeof(pWal->logName), "%s/%s%" PRId64, pWal->path, WAL_PREFIX, pWal->curFileId); - pWal->logTfd = tfOpenCreateWrite(pWal->logName); - - if (!tfValid(pWal->logTfd)) { - code = TAOS_SYSTEM_ERROR(errno); - wError("vgId:%d, file:%s, failed to open since %s", pWal->vgId, pWal->logName, strerror(errno)); - } else { - wDebug("vgId:%d, file:%s, it is created and open while renew", pWal->vgId, pWal->logName); - } - - pthread_mutex_unlock(&pWal->mutex); - - return code; -} - -void walRemoveOneOldFile(void *handle) { - SWal *pWal = handle; - if (pWal == NULL) return; - /*if (pWal->keep == TAOS_WAL_KEEP) return;*/ - if (!tfValid(pWal->logTfd)) return; - - pthread_mutex_lock(&pWal->mutex); - - // remove the oldest wal file - int64_t oldFileId = -1; - if (walGetOldFile(pWal, pWal->curFileId, WAL_FILE_NUM, &oldFileId) == 0) { - char walName[WAL_FILE_LEN] = {0}; - snprintf(walName, sizeof(walName), "%s/%s%" PRId64, pWal->path, WAL_PREFIX, oldFileId); - - if (remove(walName) < 0) { - wError("vgId:%d, file:%s, failed to remove since %s", pWal->vgId, walName, strerror(errno)); - } else { - wInfo("vgId:%d, file:%s, it is removed", pWal->vgId, walName); - } - } - - pthread_mutex_unlock(&pWal->mutex); -} - -void walRemoveAllOldFiles(void *handle) { - if (handle == NULL) return; - - SWal * pWal = handle; - int64_t fileId = -1; - - pthread_mutex_lock(&pWal->mutex); - - tfClose(pWal->logTfd); - wDebug("vgId:%d, file:%s, it is closed before remove all wals", pWal->vgId, pWal->logName); - - while (walGetNextFile(pWal, &fileId) >= 0) { - snprintf(pWal->logName, sizeof(pWal->logName), "%s/%s%" PRId64, pWal->path, WAL_PREFIX, fileId); - - if (remove(pWal->logName) < 0) { - wError("vgId:%d, wal:%p file:%s, failed to remove since %s", pWal->vgId, pWal, pWal->logName, strerror(errno)); - } else { - wInfo("vgId:%d, wal:%p file:%s, it is removed", pWal->vgId, pWal, pWal->logName); - } - } - pthread_mutex_unlock(&pWal->mutex); -} -#endif - int32_t walCommit(SWal *pWal, int64_t ver) { ASSERT(pWal->vers.commitVer >= pWal->vers.snapshotVer); ASSERT(pWal->vers.commitVer <= pWal->vers.lastVer); @@ -166,7 +74,7 @@ int32_t walRollback(SWal *pWal, int64_t ver) { pthread_mutex_unlock(&pWal->mutex); return -1; } - int idxOff = (ver - walGetCurFileFirstVer(pWal)) * WAL_IDX_ENTRY_SIZE; + int idxOff = (ver - walGetCurFileFirstVer(pWal)) * sizeof(WalIdxEntry); code = tfLseek(idxTfd, idxOff, SEEK_SET); if(code < 0) { pthread_mutex_unlock(&pWal->mutex); @@ -229,7 +137,7 @@ int32_t walRollback(SWal *pWal, int64_t ver) { return 0; } -int32_t walBeginTakeSnapshot(SWal* pWal, int64_t ver) { +int32_t walBeginSnapshot(SWal* pWal, int64_t ver) { pWal->vers.verInSnapshotting = ver; //check file rolling if(pWal->cfg.retentionPeriod == 0) { @@ -239,7 +147,7 @@ int32_t walBeginTakeSnapshot(SWal* pWal, int64_t ver) { return 0; } -int32_t walEndTakeSnapshot(SWal *pWal) { +int32_t walEndSnapshot(SWal *pWal) { int64_t ver = pWal->vers.verInSnapshotting; if(ver == -1) return -1; @@ -287,7 +195,7 @@ int32_t walEndTakeSnapshot(SWal *pWal) { pWal->vers.verInSnapshotting = -1; //save snapshot ver, commit ver - int code = walWriteMeta(pWal); + int code = walSaveMeta(pWal); if(code != 0) { return -1; } @@ -314,13 +222,13 @@ int walRoll(SWal *pWal) { int64_t newFileFirstVersion = pWal->vers.lastVer + 1; char fnameStr[WAL_FILE_LEN]; walBuildIdxName(pWal, newFileFirstVersion, fnameStr); - idxTfd = tfOpenCreateWrite(fnameStr); + idxTfd = tfOpenCreateWriteAppend(fnameStr); if(idxTfd < 0) { ASSERT(0); return -1; } walBuildLogName(pWal, newFileFirstVersion, fnameStr); - logTfd = tfOpenCreateWrite(fnameStr); + logTfd = tfOpenCreateWriteAppend(fnameStr); if(logTfd < 0) { ASSERT(0); return -1; @@ -335,8 +243,6 @@ int walRoll(SWal *pWal) { pWal->writeIdxTfd = idxTfd; pWal->writeLogTfd = logTfd; pWal->writeCur = taosArrayGetSize(pWal->fileInfoSet) - 1; - //change status - pWal->curStatus = WAL_CUR_FILE_WRITABLE & WAL_CUR_POS_WRITABLE; pWal->lastRollSeq = walGetSeq(); return 0; @@ -425,74 +331,6 @@ void walFsync(SWal *pWal, bool forceFsync) { } } -#if 0 -int32_t walRestore(void *handle, void *pVnode, FWalWrite writeFp) { - if (handle == NULL) return -1; - - SWal * pWal = handle; - int32_t count = 0; - int32_t code = 0; - int64_t fileId = -1; - - while ((code = walGetNextFile(pWal, &fileId)) >= 0) { - /*if (fileId == pWal->curFileId) continue;*/ - - char walName[WAL_FILE_LEN]; - snprintf(walName, sizeof(pWal->logName), "%s/%s%" PRId64, pWal->path, WAL_PREFIX, fileId); - - wInfo("vgId:%d, file:%s, will be restored", pWal->vgId, walName); - code = walRestoreWalFile(pWal, pVnode, writeFp, walName, fileId); - if (code != TSDB_CODE_SUCCESS) { - wError("vgId:%d, file:%s, failed to restore since %s", pWal->vgId, walName, tstrerror(code)); - continue; - } - - wInfo("vgId:%d, file:%s, restore success, wver:%" PRIu64, pWal->vgId, walName, pWal->curVersion); - - count++; - } - - /*if (pWal->keep != TAOS_WAL_KEEP) return TSDB_CODE_SUCCESS;*/ - - if (count == 0) { - wDebug("vgId:%d, wal file not exist, renew it", pWal->vgId); - return walRenew(pWal); - } else { - // open the existing WAL file in append mode - /*pWal->curFileId = 0;*/ - snprintf(pWal->logName, sizeof(pWal->logName), "%s/%s%" PRId64, pWal->path, WAL_PREFIX, pWal->curFileId); - pWal->logTfd = tfOpenCreateWriteAppend(pWal->logName); - if (!tfValid(pWal->logTfd)) { - wError("vgId:%d, file:%s, failed to open since %s", pWal->vgId, pWal->logName, strerror(errno)); - return TAOS_SYSTEM_ERROR(errno); - } - wDebug("vgId:%d, file:%s, it is created and open while restore", pWal->vgId, pWal->logName); - } - - return TSDB_CODE_SUCCESS; -} - -int32_t walGetWalFile(void *handle, char *fileName, int64_t *fileId) { - if (handle == NULL) return -1; - SWal *pWal = handle; - - if (*fileId == 0) *fileId = -1; - - pthread_mutex_lock(&(pWal->mutex)); - - int32_t code = walGetNextFile(pWal, fileId); - if (code >= 0) { - sprintf(fileName, "wal/%s%" PRId64, WAL_PREFIX, *fileId); - /*code = (*fileId == pWal->curFileId) ? 0 : 1;*/ - } - - wDebug("vgId:%d, get wal file, code:%d curId:%" PRId64 " outId:%" PRId64, pWal->vgId, code, pWal->curFileId, *fileId); - pthread_mutex_unlock(&(pWal->mutex)); - - return code; -} -#endif - /*static int walValidateOffset(SWal* pWal, int64_t ver) {*/ /*int code = 0;*/ /*SWalHead *pHead = NULL;*/ @@ -516,139 +354,3 @@ int32_t walGetWalFile(void *handle, char *fileName, int64_t *fileId) { /*return 0;*/ /*}*/ - -#if 0 -static int32_t walSkipCorruptedRecord(SWal *pWal, SWalHead *pHead, int64_t tfd, int64_t *offset) { - int64_t pos = *offset; - while (1) { - pos++; - - if (tfLseek(tfd, pos, SEEK_SET) < 0) { - wError("vgId:%d, failed to seek from corrupted wal file since %s", pWal->vgId, strerror(errno)); - return TSDB_CODE_WAL_FILE_CORRUPTED; - } - - if (tfRead(tfd, pHead, sizeof(SWalHead)) <= 0) { - wError("vgId:%d, read to end of corrupted wal file, offset:%" PRId64, pWal->vgId, pos); - return TSDB_CODE_WAL_FILE_CORRUPTED; - } - - if (pHead->signature != WAL_SIGNATURE) { - continue; - } - - if (pHead->sver >= 1) { - if (tfRead(tfd, pHead->cont, pHead->len) < pHead->len) { - wError("vgId:%d, read to end of corrupted wal file, offset:%" PRId64, pWal->vgId, pos); - return TSDB_CODE_WAL_FILE_CORRUPTED; - } - - if (walValidateChecksum(pHead)) { - wInfo("vgId:%d, wal whole cksum check passed, offset:%" PRId64, pWal->vgId, pos); - *offset = pos; - return TSDB_CODE_SUCCESS; - } - } - } - - return TSDB_CODE_WAL_FILE_CORRUPTED; -} - -static int32_t walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp, char *name, int64_t fileId) { - int32_t size = WAL_MAX_SIZE; - void * buffer = malloc(size); - if (buffer == NULL) { - wError("vgId:%d, file:%s, failed to open for restore since %s", pWal->vgId, name, strerror(errno)); - return TAOS_SYSTEM_ERROR(errno); - } - - int64_t tfd = tfOpenReadWrite(name); - if (!tfValid(tfd)) { - wError("vgId:%d, file:%s, failed to open for restore since %s", pWal->vgId, name, strerror(errno)); - tfree(buffer); - return TAOS_SYSTEM_ERROR(errno); - } else { - wDebug("vgId:%d, file:%s, open for restore", pWal->vgId, name); - } - - int32_t code = TSDB_CODE_SUCCESS; - int64_t offset = 0; - SWalHead *pHead = buffer; - - while (1) { - int32_t ret = (int32_t)tfRead(tfd, pHead, sizeof(SWalHead)); - if (ret == 0) break; - - if (ret < 0) { - wError("vgId:%d, file:%s, failed to read wal head since %s", pWal->vgId, name, strerror(errno)); - code = TAOS_SYSTEM_ERROR(errno); - break; - } - - if (ret < sizeof(SWalHead)) { - wError("vgId:%d, file:%s, failed to read wal head, ret is %d", pWal->vgId, name, ret); - walFtruncate(pWal, tfd, offset); - break; - } - - if ((pHead->sver == 0 && !walValidateChecksum(pHead)) || pHead->sver < 0 || pHead->sver > 2) { - wError("vgId:%d, file:%s, wal head cksum is messed up, hver:%" PRIu64 " len:%d offset:%" PRId64, pWal->vgId, name, - pHead->version, pHead->len, offset); - code = walSkipCorruptedRecord(pWal, pHead, tfd, &offset); - if (code != TSDB_CODE_SUCCESS) { - walFtruncate(pWal, tfd, offset); - break; - } - } - - if (pHead->len < 0 || pHead->len > size - sizeof(SWalHead)) { - wError("vgId:%d, file:%s, wal head len out of range, hver:%" PRIu64 " len:%d offset:%" PRId64, pWal->vgId, name, - pHead->version, pHead->len, offset); - code = walSkipCorruptedRecord(pWal, pHead, tfd, &offset); - if (code != TSDB_CODE_SUCCESS) { - walFtruncate(pWal, tfd, offset); - break; - } - } - - ret = (int32_t)tfRead(tfd, pHead->cont, pHead->len); - if (ret < 0) { - wError("vgId:%d, file:%s, failed to read wal body since %s", pWal->vgId, name, strerror(errno)); - code = TAOS_SYSTEM_ERROR(errno); - break; - } - - if (ret < pHead->len) { - wError("vgId:%d, file:%s, failed to read wal body, ret:%d len:%d", pWal->vgId, name, ret, pHead->len); - offset += sizeof(SWalHead); - continue; - } - - if ((pHead->sver >= 1) && !walValidateChecksum(pHead)) { - wError("vgId:%d, file:%s, wal whole cksum is messed up, hver:%" PRIu64 " len:%d offset:%" PRId64, pWal->vgId, name, - pHead->version, pHead->len, offset); - code = walSkipCorruptedRecord(pWal, pHead, tfd, &offset); - if (code != TSDB_CODE_SUCCESS) { - walFtruncate(pWal, tfd, offset); - break; - } - } - - offset = offset + sizeof(SWalHead) + pHead->len; - - wTrace("vgId:%d, restore wal, fileId:%" PRId64 " hver:%" PRIu64 " wver:%" PRIu64 " len:%d offset:%" PRId64, - pWal->vgId, fileId, pHead->version, pWal->curVersion, pHead->len, offset); - - pWal->curVersion = pHead->version; - - // wInfo("writeFp: %ld", offset); - (*writeFp)(pVnode, pHead); - } - - tfClose(tfd); - tfree(buffer); - - wDebug("vgId:%d, file:%s, it is closed after restore", pWal->vgId, name); - return code; -} -#endif diff --git a/source/libs/wal/test/walMetaTest.cpp b/source/libs/wal/test/walMetaTest.cpp index 200bf39c5a..d06388201e 100644 --- a/source/libs/wal/test/walMetaTest.cpp +++ b/source/libs/wal/test/walMetaTest.cpp @@ -142,7 +142,7 @@ TEST_F(WalCleanEnv, serialize) { char*ss = walMetaSerialize(pWal); printf("%s\n", ss); free(ss); - code = walWriteMeta(pWal); + code = walSaveMeta(pWal); ASSERT(code == 0); } @@ -150,11 +150,11 @@ TEST_F(WalCleanEnv, removeOldMeta) { int code = walRollFileInfo(pWal); ASSERT(code == 0); ASSERT(pWal->fileInfoSet != NULL); - code = walWriteMeta(pWal); + code = walSaveMeta(pWal); ASSERT(code == 0); code = walRollFileInfo(pWal); ASSERT(code == 0); - code = walWriteMeta(pWal); + code = walSaveMeta(pWal); ASSERT(code == 0); } @@ -199,7 +199,7 @@ TEST_F(WalCleanEnv, write) { ASSERT_EQ(code, -1); ASSERT_EQ(pWal->vers.lastVer, i); } - code = walWriteMeta(pWal); + code = walSaveMeta(pWal); ASSERT_EQ(code, 0); } @@ -216,7 +216,7 @@ TEST_F(WalCleanEnv, rollback) { code = walRollback(pWal, 3); ASSERT_EQ(code, 0); ASSERT_EQ(pWal->vers.lastVer, 2); - code = walWriteMeta(pWal); + code = walSaveMeta(pWal); ASSERT_EQ(code, 0); } @@ -231,9 +231,9 @@ TEST_F(WalCleanDeleteEnv, roll) { ASSERT_EQ(pWal->vers.commitVer, i); } - walBeginTakeSnapshot(pWal, i-1); + walBeginSnapshot(pWal, i-1); ASSERT_EQ(pWal->vers.verInSnapshotting, i-1); - walEndTakeSnapshot(pWal); + walEndSnapshot(pWal); ASSERT_EQ(pWal->vers.snapshotVer, i-1); ASSERT_EQ(pWal->vers.verInSnapshotting, -1); @@ -247,9 +247,9 @@ TEST_F(WalCleanDeleteEnv, roll) { ASSERT_EQ(pWal->vers.commitVer, i); } - code = walBeginTakeSnapshot(pWal, i - 1); + code = walBeginSnapshot(pWal, i - 1); ASSERT_EQ(code, 0); - code = walEndTakeSnapshot(pWal); + code = walEndSnapshot(pWal); ASSERT_EQ(code, 0); } From f8b8569bd1d63ac1c52c9d47305835f512c203f6 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Tue, 14 Dec 2021 05:54:46 -0500 Subject: [PATCH 29/43] TD-12034 Organize planner module interface. --- include/libs/planner/planner.h | 86 ++++------------ include/libs/scheduler/scheduler.h | 37 ++++++- source/libs/parser/test/plannerTest.cpp | 9 +- source/libs/planner/inc/plannerInt.h | 125 ++++++++++++++++-------- source/libs/planner/src/physicalPlan.c | 36 +++++++ source/libs/planner/src/planner.c | 20 ++-- 6 files changed, 184 insertions(+), 129 deletions(-) create mode 100644 source/libs/planner/src/physicalPlan.c diff --git a/include/libs/planner/planner.h b/include/libs/planner/planner.h index 87be26895e..8f217a0deb 100644 --- a/include/libs/planner/planner.h +++ b/include/libs/planner/planner.h @@ -22,6 +22,7 @@ extern "C" { #define QUERY_TYPE_MERGE 1 #define QUERY_TYPE_PARTIAL 2 +#define QUERY_TYPE_SCAN 3 enum OPERATOR_TYPE_E { OP_TableScan = 1, @@ -54,90 +55,37 @@ enum OPERATOR_TYPE_E { struct SEpSet; struct SQueryPlanNode; -struct SQueryPhyPlanNode; +struct SPhyNode; struct SQueryStmtInfo; -typedef struct SSubquery { - int64_t queryId; // the subquery id created by qnode - int32_t type; // QUERY_TYPE_MERGE|QUERY_TYPE_PARTIAL - int32_t level; // the execution level of current subquery, starting from 0. - SArray *pUpstream; // the upstream,from which to fetch the result - struct SQueryPhyPlanNode *pNode; // physical plan of current subquery -} SSubquery; - -typedef struct SQueryJob { - SArray **pSubqueries; - int32_t numOfLevels; - int32_t currentLevel; -} SQueryJob; +typedef struct SSubplan { + int32_t type; // QUERY_TYPE_MERGE|QUERY_TYPE_PARTIAL|QUERY_TYPE_SCAN + SArray *pDatasource; // the datasource subplan,from which to fetch the result + struct SPhyNode *pNode; // physical plan of current subplan +} SSubplan; +typedef struct SQueryDag { + SArray **pSubplans; +} SQueryDag; /** - * Optimize the query execution plan, currently not implement yet. - * @param pQueryNode - * @return + * Create the physical plan for the query, according to the AST. */ -int32_t qOptimizeQueryPlan(struct SQueryPlanNode* pQueryNode); +int32_t qCreateQueryDag(const struct SQueryStmtInfo* pQueryInfo, struct SEpSet* pQnode, struct SQueryDag** pDag); + +int32_t qExplainQuery(const struct SQueryStmtInfo* pQueryInfo, struct SEpSet* pQnode, char** str); /** - * Create the query plan according to the bound AST, which is in the form of pQueryInfo - * @param pQueryInfo - * @param pQueryNode - * @return + * Convert to subplan to string for the scheduler to send to the executor */ -int32_t qCreateQueryPlan(const struct SQueryStmtInfo* pQueryInfo, struct SQueryPlanNode** pQueryNode); - -/** - * Convert the query plan to string, in order to display it in the shell. - * @param pQueryNode - * @return - */ -int32_t qQueryPlanToString(struct SQueryPlanNode* pQueryNode, char** str); - -/** - * Restore the SQL statement according to the logic query plan. - * @param pQueryNode - * @param sql - * @return - */ -int32_t qQueryPlanToSql(struct SQueryPlanNode* pQueryNode, char** sql); - -/** - * Create the physical plan for the query, according to the logic plan. - * @param pQueryNode - * @param pPhyNode - * @return - */ -int32_t qCreatePhysicalPlan(struct SQueryPlanNode* pQueryNode, struct SEpSet* pQnode, struct SQueryPhyPlanNode *pPhyNode); - -/** - * Convert to physical plan to string to enable to print it out in the shell. - * @param pPhyNode - * @param str - * @return - */ -int32_t qPhyPlanToString(struct SQueryPhyPlanNode *pPhyNode, char** str); - -/** - * Destroy the query plan object. - * @return - */ -void* qDestroyQueryPlan(struct SQueryPlanNode* pQueryNode); +int32_t qSubPlanToString(struct SSubplan *pPhyNode, char** str); /** * Destroy the physical plan. * @param pQueryPhyNode * @return */ -void* qDestroyQueryPhyPlan(struct SQueryPhyPlanNode* pQueryPhyNode); - -/** - * Create the query job from the physical execution plan - * @param pPhyNode - * @param pJob - * @return - */ -int32_t qCreateQueryJob(const struct SQueryPhyPlanNode* pPhyNode, struct SQueryJob** pJob); +void* qDestroyQueryDag(struct SQueryDag* pDag); #ifdef __cplusplus } diff --git a/include/libs/scheduler/scheduler.h b/include/libs/scheduler/scheduler.h index d9653046cf..6b3c9ed021 100644 --- a/include/libs/scheduler/scheduler.h +++ b/include/libs/scheduler/scheduler.h @@ -20,7 +20,42 @@ extern "C" { #endif -struct SQueryJob; +typedef struct SQueryProfileSummary { + int64_t startTs; // Object created and added into the message queue + int64_t endTs; // the timestamp when the task is completed + int64_t cputime; // total cpu cost, not execute elapsed time + + int64_t loadRemoteDataDuration; // remote io time + int64_t loadNativeDataDuration; // native disk io time + + uint64_t loadNativeData; // blocks + SMA + header files + uint64_t loadRemoteData; // remote data acquired by exchange operator. + + uint64_t waitDuration; // the time to waiting to be scheduled in queue does matter, so we need to record it + int64_t addQTs; // the time to be added into the message queue, used to calculate the waiting duration in queue. + + uint64_t totalRows; + uint64_t loadRows; + uint32_t totalBlocks; + uint32_t loadBlocks; + uint32_t loadBlockAgg; + uint32_t skipBlocks; + uint64_t resultSize; // generated result size in Kb. +} SQueryProfileSummary; + +typedef struct SQueryTask { + uint64_t queryId; // query id + uint64_t taskId; // task id + char *pSubplan; // operator tree + uint64_t status; // task status + SQueryProfileSummary summary; // task execution summary + void *pOutputHandle; // result buffer handle, to temporarily keep the output result for next stage +} SQueryTask; + +typedef struct SQueryJob { + SArray **pSubtasks; + // todo +} SQueryJob; /** * Process the query job, generated according to the query physical plan. diff --git a/source/libs/parser/test/plannerTest.cpp b/source/libs/parser/test/plannerTest.cpp index c86e687664..8a45e96248 100644 --- a/source/libs/parser/test/plannerTest.cpp +++ b/source/libs/parser/test/plannerTest.cpp @@ -30,6 +30,7 @@ #include "tdef.h" #include "tvariant.h" #include "planner.h" +#include "../../planner/inc/plannerInt.h" namespace { void setSchema(SSchema* p, int32_t type, int32_t bytes, const char* name, int32_t colId) { @@ -92,10 +93,10 @@ void generateLogicplan(const char* sql) { ASSERT_EQ(ret, 0); struct SQueryPlanNode* n = nullptr; - code = qCreateQueryPlan(pQueryInfo, &n); + code = createQueryPlan(pQueryInfo, &n); char* str = NULL; - qQueryPlanToString(n, &str); + queryPlanToString(n, &str); printf("--------SQL:%s\n", sql); printf("%s\n", str); @@ -155,10 +156,10 @@ TEST(testCase, planner_test) { ASSERT_EQ(pQueryInfo->fieldsInfo.numOfOutput, 2); struct SQueryPlanNode* n = nullptr; - code = qCreateQueryPlan(pQueryInfo, &n); + code = createQueryPlan(pQueryInfo, &n); char* str = NULL; - qQueryPlanToString(n, &str); + queryPlanToString(n, &str); printf("%s\n", str); destroyQueryInfo(pQueryInfo); diff --git a/source/libs/planner/inc/plannerInt.h b/source/libs/planner/inc/plannerInt.h index c51a15509d..2231c93362 100644 --- a/source/libs/planner/inc/plannerInt.h +++ b/source/libs/planner/inc/plannerInt.h @@ -25,6 +25,19 @@ extern "C" { #include "planner.h" #include "taosmsg.h" +enum LOGIC_PLAN_E { + LP_SCAN = 1, + LP_SESSION = 2, + LP_STATE = 3, + LP_INTERVAL = 4, + LP_FILL = 5, + LP_AGG = 6, + LP_JOIN = 7, + LP_PROJECT = 8, + LP_DISTINCT = 9, + LP_ORDER = 10 +}; + typedef struct SQueryNodeBasicInfo { int32_t type; // operator type char *name; // operator name @@ -57,68 +70,94 @@ typedef struct SQueryPlanNode { struct SQueryPlanNode *nextNode; } SQueryPlanNode; +typedef SSchema SSlotSchema; + typedef struct SDataBlockSchema { int32_t index; - SSchema *pSchema; // the schema of the SSDatablock + SSlotSchema *pSchema; int32_t numOfCols; // number of columns } SDataBlockSchema; -typedef struct SQueryPhyPlanNode { +typedef struct SPhyNode { SQueryNodeBasicInfo info; - SArray *pTarget; // target list to be computed at this node - SArray *qual; // implicitly-ANDed qual conditions + SArray *pTargets; // target list to be computed or scanned at this node + SArray *pConditions; // implicitly-ANDed qual conditions SDataBlockSchema targetSchema; // children plan to generated result for current node to process // in case of join, multiple plan nodes exist. SArray *pChildren; -} SQueryPhyPlanNode; +} SPhyNode; -typedef struct SQueryScanPhyNode { - SQueryPhyPlanNode node; - uint64_t uid; -} SQueryScanPhyNode; +typedef struct SScanPhyNode { + SPhyNode node; + uint64_t uid; // unique id of the table +} SScanPhyNode; -typedef struct SQueryProjectPhyNode { - SQueryPhyPlanNode node; -} SQueryProjectPhyNode; +typedef SScanPhyNode STagScanPhyNode; -typedef struct SQueryAggPhyNode { - SQueryPhyPlanNode node; - SArray *pGroup; - // SInterval -} SQueryAggPhyNode; +typedef SScanPhyNode SSystemTableScanPhyNode; -typedef struct SQueryProfileSummary { - int64_t startTs; // Object created and added into the message queue - int64_t endTs; // the timestamp when the task is completed - int64_t cputime; // total cpu cost, not execute elapsed time +typedef struct SMultiTableScanPhyNode { + SScanPhyNode scan; + SArray *pTagsConditions; // implicitly-ANDed tag qual conditions +} SMultiTableScanPhyNode; - int64_t loadRemoteDataDuration; // remote io time - int64_t loadNativeDataDuration; // native disk io time +typedef SMultiTableScanPhyNode SMultiTableSeqScanPhyNode; - uint64_t loadNativeData; // blocks + SMA + header files - uint64_t loadRemoteData; // remote data acquired by exchange operator. +typedef struct SProjectPhyNode { + SPhyNode node; +} SProjectPhyNode; - uint64_t waitDuration; // the time to waiting to be scheduled in queue does matter, so we need to record it - int64_t addQTs; // the time to be added into the message queue, used to calculate the waiting duration in queue. +/** + * Optimize the query execution plan, currently not implement yet. + * @param pQueryNode + * @return + */ +int32_t optimizeQueryPlan(struct SQueryPlanNode* pQueryNode); - uint64_t totalRows; - uint64_t loadRows; - uint32_t totalBlocks; - uint32_t loadBlocks; - uint32_t loadBlockAgg; - uint32_t skipBlocks; - uint64_t resultSize; // generated result size in Kb. -} SQueryProfileSummary; +/** + * Create the query plan according to the bound AST, which is in the form of pQueryInfo + * @param pQueryInfo + * @param pQueryNode + * @return + */ +int32_t createQueryPlan(const struct SQueryStmtInfo* pQueryInfo, struct SQueryPlanNode** pQueryNode); -typedef struct SQueryTask { - uint64_t queryId; // query id - uint64_t taskId; // task id - SQueryPhyPlanNode *pNode; // operator tree - uint64_t status; // task status - SQueryProfileSummary summary; // task execution summary - void *pOutputHandle; // result buffer handle, to temporarily keep the output result for next stage -} SQueryTask; +/** + * Convert the query plan to string, in order to display it in the shell. + * @param pQueryNode + * @return + */ +int32_t queryPlanToString(struct SQueryPlanNode* pQueryNode, char** str); + +/** + * Restore the SQL statement according to the logic query plan. + * @param pQueryNode + * @param sql + * @return + */ +int32_t queryPlanToSql(struct SQueryPlanNode* pQueryNode, char** sql); + +/** + * Convert to physical plan to string to enable to print it out in the shell. + * @param pPhyNode + * @param str + * @return + */ +int32_t phyPlanToString(struct SPhyNode *pPhyNode, char** str); + +/** + * Destroy the query plan object. + * @return + */ +void* destroyQueryPlan(struct SQueryPlanNode* pQueryNode); + +/** + * Destroy the physical plan. + * @param pQueryPhyNode + * @return + */ +void* destroyQueryPhyPlan(struct SPhyNode* pQueryPhyNode); #ifdef __cplusplus } diff --git a/source/libs/planner/src/physicalPlan.c b/source/libs/planner/src/physicalPlan.c new file mode 100644 index 0000000000..2bdc159af8 --- /dev/null +++ b/source/libs/planner/src/physicalPlan.c @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "plannerInt.h" + +SPhyNode* createScanNode(SQueryPlanNode* pPlanNode) { + return NULL; +} + +SPhyNode* createPhyNode(SQueryPlanNode* node) { + switch (node->info.type) { + case LP_SCAN: + return createScanNode(node); + } + return NULL; +} + +SPhyNode* createSubplan(SQueryPlanNode* pSubquery) { + return NULL; +} + +int32_t createDag(struct SQueryPlanNode* pQueryNode, struct SEpSet* pQnode, struct SQueryDag** pDag) { + return 0; +} diff --git a/source/libs/planner/src/planner.c b/source/libs/planner/src/planner.c index 121a7d3c2c..e54b847230 100644 --- a/source/libs/planner/src/planner.c +++ b/source/libs/planner/src/planner.c @@ -48,11 +48,11 @@ static SArray* createQueryPlanImpl(SQueryStmtInfo* pQueryInfo); static void doDestroyQueryNode(SQueryPlanNode* pQueryNode); int32_t printExprInfo(const char* buf, const SQueryPlanNode* pQueryNode, int32_t len); -int32_t qOptimizeQueryPlan(struct SQueryPlanNode* pQueryNode) { +int32_t optimizeQueryPlan(struct SQueryPlanNode* pQueryNode) { return 0; } -int32_t qCreateQueryPlan(const struct SQueryStmtInfo* pQueryInfo, struct SQueryPlanNode** pQueryNode) { +int32_t createQueryPlan(const struct SQueryStmtInfo* pQueryInfo, struct SQueryPlanNode** pQueryNode) { SArray* upstream = createQueryPlanImpl((struct SQueryStmtInfo*) pQueryInfo); assert(taosArrayGetSize(upstream) == 1); @@ -62,20 +62,20 @@ int32_t qCreateQueryPlan(const struct SQueryStmtInfo* pQueryInfo, struct SQueryP return TSDB_CODE_SUCCESS; } -int32_t qQueryPlanToSql(struct SQueryPlanNode* pQueryNode, char** sql) { +int32_t queryPlanToSql(struct SQueryPlanNode* pQueryNode, char** sql) { return 0; } -int32_t qCreatePhysicalPlan(struct SQueryPlanNode* pQueryNode, struct SEpSet* pQnode, struct SQueryPhyPlanNode *pPhyNode) { +int32_t qCreatePhysicalPlan(struct SQueryPlanNode* pQueryNode, struct SEpSet* pQnode, struct SQueryDag** pDag) { return 0; } -int32_t qPhyPlanToString(struct SQueryPhyPlanNode *pPhyNode, char** str) { +int32_t phyPlanToString(struct SPhyNode *pPhyNode, char** str) { return 0; } -void* qDestroyQueryPlan(SQueryPlanNode* pQueryNode) { +void* destroyQueryPlan(SQueryPlanNode* pQueryNode) { if (pQueryNode == NULL) { return NULL; } @@ -84,14 +84,10 @@ void* qDestroyQueryPlan(SQueryPlanNode* pQueryNode) { return NULL; } -void* qDestroyQueryPhyPlan(struct SQueryPhyPlanNode* pQueryPhyNode) { +void* destroyQueryPhyPlan(struct SPhyNode* pQueryPhyNode) { return NULL; } -int32_t qCreateQueryJob(const struct SQueryPhyPlanNode* pPhyNode, struct SQueryJob** pJob) { - return 0; -} - //====================================================================================================================== static SQueryPlanNode* createQueryNode(int32_t type, const char* name, SQueryPlanNode** prev, int32_t numOfPrev, @@ -620,7 +616,7 @@ int32_t queryPlanToStringImpl(char* buf, SQueryPlanNode* pQueryNode, int32_t lev return len; } -int32_t qQueryPlanToString(struct SQueryPlanNode* pQueryNode, char** str) { +int32_t queryPlanToString(struct SQueryPlanNode* pQueryNode, char** str) { assert(pQueryNode); *str = calloc(1, 4096); From b282dfe13e4b806a87560fc89e237526ca765dc2 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 14 Dec 2021 19:29:37 +0800 Subject: [PATCH 30/43] TD-10431 get sdbGetMaxId func --- include/dnode/mnode/sdb/sdb.h | 11 ++- source/dnode/mnode/impl/src/mndDnode.c | 6 +- source/dnode/mnode/impl/src/mndTrans.c | 7 +- source/dnode/mnode/impl/src/mndUser.c | 104 ++++++++++++------------- source/dnode/mnode/sdb/inc/sdbInt.h | 1 + source/dnode/mnode/sdb/src/sdb.c | 1 + source/dnode/mnode/sdb/src/sdbHash.c | 29 +++++++ source/dnode/vnode/impl/CMakeLists.txt | 2 +- 8 files changed, 97 insertions(+), 64 deletions(-) diff --git a/include/dnode/mnode/sdb/sdb.h b/include/dnode/mnode/sdb/sdb.h index 9620c4ea62..51d7ca7202 100644 --- a/include/dnode/mnode/sdb/sdb.h +++ b/include/dnode/mnode/sdb/sdb.h @@ -99,7 +99,7 @@ extern "C" { sdbFreeRaw(pRaw); \ return NULL; \ } \ - dataPos += sizeof(int16_t); \ + dataPos += sizeof(int16_t); \ } #define SDB_SET_INT8(pRaw, dataPos, val) \ @@ -295,6 +295,15 @@ void sdbCancelFetch(SSdb *pSdb, void *pIter); */ int32_t sdbGetSize(SSdb *pSdb, ESdbType type); +/** + * @brief Get the max id of the table, keyType of table should be INT32 + * + * @param pSdb The sdb object. + * @param pIter The type of the table. + * @record int32_t The max id of the table + */ +int32_t sdbGetMaxId(SSdb *pSdb, ESdbType type); + SSdbRaw *sdbAllocRaw(ESdbType type, int8_t sver, int32_t dataLen); void sdbFreeRaw(SSdbRaw *pRaw); int32_t sdbSetRawInt8(SSdbRaw *pRaw, int32_t dataPos, int8_t val); diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 4ce557fc02..e00244d05c 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -19,7 +19,7 @@ #include "mndShow.h" #include "mndTrans.h" #include "ttime.h" -#include "tutil.h" +#include "tep.h" #define TSDB_DNODE_VER 1 #define TSDB_DNODE_RESERVE_SIZE 64 @@ -27,8 +27,6 @@ #define TSDB_CONIIG_VALUE_LEN 48 #define TSDB_CONFIG_NUMBER 8 -static int32_t id = 2; - static const char *offlineReason[] = { "", "status msg timeout", @@ -389,7 +387,7 @@ static int32_t mndProcessStatusMsg(SMnodeMsg *pMsg) { static int32_t mndCreateDnode(SMnode *pMnode, SMnodeMsg *pMsg, SCreateDnodeMsg *pCreate) { SDnodeObj dnodeObj = {0}; - dnodeObj.id = id++; + dnodeObj.id = sdbGetMaxId(pMnode->pSdb, SDB_DNODE); dnodeObj.createdTime = taosGetTimestampMs(); dnodeObj.updateTime = dnodeObj.createdTime; taosGetFqdnPortFromEp(pCreate->ep, dnodeObj.fqdn, &dnodeObj.port); diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 9ab84d6557..32ac795301 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -262,11 +262,6 @@ void mndReleaseTrans(SMnode *pMnode, STrans *pTrans) { sdbRelease(pSdb, pTrans); } -static int32_t mndGenerateTransId() { - static int32_t tmp = 0; - return ++tmp; -} - char *mndTransStageStr(ETrnStage stage) { switch (stage) { case TRN_STAGE_PREPARE: @@ -303,7 +298,7 @@ STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, void *rpcHandle) { return NULL; } - pTrans->id = mndGenerateTransId(); + pTrans->id = sdbGetMaxId(pMnode->pSdb, SDB_TRANS); pTrans->stage = TRN_STAGE_PREPARE; pTrans->policy = policy; pTrans->rpcHandle = rpcHandle; diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index 452c14886d..6fb05653a5 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -219,58 +219,6 @@ static int32_t mndCreateUser(SMnode *pMnode, char *acct, char *user, char *pass, return 0; } -static int32_t mndUpdateUser(SMnode *pMnode, SUserObj *pOldUser, SUserObj *pNewUser, SMnodeMsg *pMsg) { - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, pMsg->rpcMsg.handle); - if (pTrans == NULL) { - mError("user:%s, failed to update since %s", pOldUser->user, terrstr()); - return -1; - } - mDebug("trans:%d, used to update user:%s", pTrans->id, pOldUser->user); - - SSdbRaw *pRedoRaw = mndUserActionEncode(pNewUser); - if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) { - mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr()); - mndTransDrop(pTrans); - return -1; - } - sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY); - - if (mndTransPrepare(pMnode, pTrans) != 0) { - mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); - mndTransDrop(pTrans); - return -1; - } - - mndTransDrop(pTrans); - return 0; -} - -static int32_t mndDropUser(SMnode *pMnode, SMnodeMsg *pMsg, SUserObj *pUser) { - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, pMsg->rpcMsg.handle); - if (pTrans == NULL) { - mError("user:%s, failed to drop since %s", pUser->user, terrstr()); - return -1; - } - mDebug("trans:%d, used to drop user:%s", pTrans->id, pUser->user); - - SSdbRaw *pRedoRaw = mndUserActionEncode(pUser); - if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) { - mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr()); - mndTransDrop(pTrans); - return -1; - } - sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPED); - - if (mndTransPrepare(pMnode, pTrans) != 0) { - mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); - mndTransDrop(pTrans); - return -1; - } - - mndTransDrop(pTrans); - return 0; -} - static int32_t mndProcessCreateUserMsg(SMnodeMsg *pMsg) { SMnode *pMnode = pMsg->pMnode; SCreateUserMsg *pCreate = pMsg->rpcMsg.pCont; @@ -315,6 +263,32 @@ static int32_t mndProcessCreateUserMsg(SMnodeMsg *pMsg) { return TSDB_CODE_MND_ACTION_IN_PROGRESS; } +static int32_t mndUpdateUser(SMnode *pMnode, SUserObj *pOldUser, SUserObj *pNewUser, SMnodeMsg *pMsg) { + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, pMsg->rpcMsg.handle); + if (pTrans == NULL) { + mError("user:%s, failed to update since %s", pOldUser->user, terrstr()); + return -1; + } + mDebug("trans:%d, used to update user:%s", pTrans->id, pOldUser->user); + + SSdbRaw *pRedoRaw = mndUserActionEncode(pNewUser); + if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) { + mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY); + + if (mndTransPrepare(pMnode, pTrans) != 0) { + mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + + mndTransDrop(pTrans); + return 0; +} + static int32_t mndProcessAlterUserMsg(SMnodeMsg *pMsg) { SMnode *pMnode = pMsg->pMnode; SAlterUserMsg *pAlter = pMsg->rpcMsg.pCont; @@ -363,6 +337,32 @@ static int32_t mndProcessAlterUserMsg(SMnodeMsg *pMsg) { return TSDB_CODE_MND_ACTION_IN_PROGRESS; } +static int32_t mndDropUser(SMnode *pMnode, SMnodeMsg *pMsg, SUserObj *pUser) { + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, pMsg->rpcMsg.handle); + if (pTrans == NULL) { + mError("user:%s, failed to drop since %s", pUser->user, terrstr()); + return -1; + } + mDebug("trans:%d, used to drop user:%s", pTrans->id, pUser->user); + + SSdbRaw *pRedoRaw = mndUserActionEncode(pUser); + if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) { + mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPED); + + if (mndTransPrepare(pMnode, pTrans) != 0) { + mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + + mndTransDrop(pTrans); + return 0; +} + static int32_t mndProcessDropUserMsg(SMnodeMsg *pMsg) { SMnode *pMnode = pMsg->pMnode; SDropUserMsg *pDrop = pMsg->rpcMsg.pCont; diff --git a/source/dnode/mnode/sdb/inc/sdbInt.h b/source/dnode/mnode/sdb/inc/sdbInt.h index a160533bf2..7e2e2602a8 100644 --- a/source/dnode/mnode/sdb/inc/sdbInt.h +++ b/source/dnode/mnode/sdb/inc/sdbInt.h @@ -59,6 +59,7 @@ typedef struct SSdb { char *tmpDir; int64_t lastCommitVer; int64_t curVer; + int32_t maxId[SDB_MAX]; EKeyType keyTypes[SDB_MAX]; SHashObj *hashObjs[SDB_MAX]; SRWLatch locks[SDB_MAX]; diff --git a/source/dnode/mnode/sdb/src/sdb.c b/source/dnode/mnode/sdb/src/sdb.c index ccff5b6c82..77614e399e 100644 --- a/source/dnode/mnode/sdb/src/sdb.c +++ b/source/dnode/mnode/sdb/src/sdb.c @@ -127,6 +127,7 @@ int32_t sdbSetTable(SSdb *pSdb, SSdbTable table) { return -1; } + pSdb->maxId[sdbType] = 0; pSdb->hashObjs[sdbType] = hash; taosInitRWLatch(&pSdb->locks[sdbType]); mDebug("sdb table:%d is initialized", sdbType); diff --git a/source/dnode/mnode/sdb/src/sdbHash.c b/source/dnode/mnode/sdb/src/sdbHash.c index c3874caffc..f63f7236fb 100644 --- a/source/dnode/mnode/sdb/src/sdbHash.c +++ b/source/dnode/mnode/sdb/src/sdbHash.c @@ -72,6 +72,10 @@ static int32_t sdbInsertRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow * taosWUnLockLatch(pLock); + if (pSdb->keyTypes[pRow->type] == SDB_KEY_INT32) { + pSdb->maxId[pRow->type] = MAX(pSdb->maxId[pRow->type], *((int32_t *)pRow->pObj)); + } + SdbInsertFp insertFp = pSdb->insertFps[pRow->type]; if (insertFp != NULL) { code = (*insertFp)(pSdb, pRow->pObj); @@ -290,3 +294,28 @@ int32_t sdbGetSize(SSdb *pSdb, ESdbType type) { return size; } + +int32_t sdbGetMaxId(SSdb *pSdb, ESdbType type) { + SHashObj *hash = sdbGetHash(pSdb, type); + if (hash == NULL) return -1; + + if (pSdb->keyTypes[type] != SDB_KEY_INT32) return -1; + + int32_t maxId = 0; + + SRWLatch *pLock = &pSdb->locks[type]; + taosRLockLatch(pLock); + + SSdbRow **ppRow = taosHashIterate(hash, NULL); + while (ppRow != NULL) { + SSdbRow *pRow = *ppRow; + int32_t id = *(int32_t *)pRow->pObj; + maxId = MAX(id, maxId); + ppRow = taosHashIterate(hash, ppRow); + } + + taosRUnLockLatch(pLock); + + maxId = MAX(maxId, pSdb->maxId[type]); + return maxId + 1; +} diff --git a/source/dnode/vnode/impl/CMakeLists.txt b/source/dnode/vnode/impl/CMakeLists.txt index 6972605afd..dd01c94da7 100644 --- a/source/dnode/vnode/impl/CMakeLists.txt +++ b/source/dnode/vnode/impl/CMakeLists.txt @@ -19,5 +19,5 @@ target_link_libraries( # test if(${BUILD_TEST}) - add_subdirectory(test) +# add_subdirectory(test) endif(${BUILD_TEST}) \ No newline at end of file From 924e3ad6bde95b866b863d679491e85ec9456560 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 15 Dec 2021 09:52:37 +0800 Subject: [PATCH 31/43] use db to get vgroup list and db info --- include/common/taosmsg.h | 25 +++- include/libs/catalog/catalog.h | 8 +- include/libs/query/query.h | 16 +++ include/os/osMemory.h | 4 +- source/libs/catalog/inc/catalogInt.h | 1 + source/libs/catalog/src/catalog.c | 118 ++++++++++++++++--- source/libs/parser/CMakeLists.txt | 4 +- source/libs/parser/test/CMakeLists.txt | 2 +- source/libs/planner/CMakeLists.txt | 4 +- source/libs/planner/test/CMakeLists.txt | 2 +- source/libs/query/src/querymsg.c | 146 +++++++++++++++++++++++- 11 files changed, 298 insertions(+), 32 deletions(-) diff --git a/include/common/taosmsg.h b/include/common/taosmsg.h index a75c4a7bac..caf872689c 100644 --- a/include/common/taosmsg.h +++ b/include/common/taosmsg.h @@ -219,6 +219,13 @@ typedef struct SBuildTableMetaInput { char *tableFullName; } SBuildTableMetaInput; +typedef struct SBuildUseDBInput { + char db[TSDB_TABLE_FNAME_LEN]; + int32_t vgroupVersion; + int32_t dbGroupVersion; +} SBuildUseDBInput; + + #pragma pack(push, 1) // null-terminated string instead of char array to avoid too many memory consumption in case of more than 1M tableMeta @@ -617,9 +624,12 @@ typedef struct { typedef struct { char db[TSDB_TABLE_FNAME_LEN]; int8_t ignoreNotExists; + int32_t vgroupVersion; + int32_t dbGroupVersion; int32_t reserve[8]; } SUseDbMsg; + typedef struct { char db[TSDB_TABLE_FNAME_LEN]; int32_t reserve[8]; @@ -806,8 +816,6 @@ typedef struct SVgroupListRspMsg { SVgroupInfo vgroupInfo[]; } SVgroupListRspMsg; -typedef SVgroupListRspMsg SVgroupListInfo; - typedef struct { int32_t vgId; int8_t numOfEps; @@ -852,6 +860,19 @@ typedef struct { char *data; } STagData; +typedef struct { + int32_t vgroupNum; + int32_t vgroupVersion; + char db[TSDB_TABLE_FNAME_LEN]; + int32_t dbVgroupVersion; + int32_t dbVgroupNum; + int32_t dbHashRange; + SVgroupInfo vgroupInfo[]; +//int32_t vgIdList[]; +} SUseDbRspMsg; + + + /* * sql: show tables like '%a_%' * payload is the query condition, e.g., '%a_%' diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index 0b45f71557..f9d3b3c8c1 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -27,16 +27,10 @@ extern "C" { #include "transport.h" #include "common.h" #include "taosmsg.h" +#include "query.h" struct SCatalog; -typedef struct SDBVgroupInfo { - int32_t vgroupVersion; - SArray *vgId; - int32_t hashRange; - int32_t hashNum; -} SDBVgroupInfo; - typedef struct SCatalogReq { char clusterId[TSDB_CLUSTER_ID_LEN]; //???? SArray *pTableName; // table full name diff --git a/include/libs/query/query.h b/include/libs/query/query.h index 866a69ed8e..02ae708874 100644 --- a/include/libs/query/query.h +++ b/include/libs/query/query.h @@ -20,6 +20,22 @@ extern "C" { #endif +#include "tarray.h" + +typedef SVgroupListRspMsg SVgroupListInfo; + +typedef struct SDBVgroupInfo { + int32_t vgroupVersion; + SArray *vgId; + int32_t hashRange; +} SDBVgroupInfo; + +typedef struct SUseDbOutput { + SVgroupListInfo *vgroupList; + char db[TSDB_TABLE_FNAME_LEN]; + SDBVgroupInfo *dbVgroup; +} SUseDbOutput; + extern int32_t (*queryBuildMsg[TSDB_MSG_TYPE_MAX])(void* input, char **msg, int32_t msgSize, int32_t *msgLen); extern int32_t (*queryProcessMsgRsp[TSDB_MSG_TYPE_MAX])(void* output, char *msg, int32_t msgSize); diff --git a/include/os/osMemory.h b/include/os/osMemory.h index 10c90cd9aa..5f1d5a9a8a 100644 --- a/include/os/osMemory.h +++ b/include/os/osMemory.h @@ -23,8 +23,8 @@ extern "C" { #define tfree(x) \ do { \ if (x) { \ - free((void *)x); \ - x = 0; \ + free((void *)(x)); \ + (x) = 0; \ } \ } while (0) diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 720f197782..a08b64f9a9 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -26,6 +26,7 @@ extern "C" { #define CTG_DEFAULT_CLUSTER_NUMBER 6 #define CTG_DEFAULT_VGROUP_NUMBER 100 +#define CTG_DEFAULT_DB_NUMBER 20 #define CTG_DEFAULT_INVALID_VERSION (-1) diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index b488ab8101..92b6094529 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -63,21 +63,69 @@ int32_t ctgGetVgroupFromCache(SCatalog* pCatalog, SArray** pVgroupList, int32_t* } -int32_t ctgGetDBVgroupFromCache(SCatalog* pCatalog, char *dbName, SDBVgroupInfo **dbInfo, int32_t *exist) { -/* +int32_t ctgGetDBVgroupFromCache(SCatalog* pCatalog, const char *dbName, SDBVgroupInfo **dbInfo, int32_t *exist) { if (NULL == pCatalog->dbCache.cache) { *exist = 0; return TSDB_CODE_SUCCESS; } - taosHashGet(SHashObj * pHashObj, const void * key, size_t keyLen) + SDBVgroupInfo *info = taosHashGet(pCatalog->dbCache.cache, dbName, strlen(dbName)); + + if (NULL == info || info->vgroupVersion < pCatalog->vgroupCache.vgroupVersion) { + *exist = 0; + return TSDB_CODE_SUCCESS; + } if (dbInfo) { - *pVgroupList = taosArrayDup(pCatalog->vgroupCache.arrayCache); + *dbInfo = calloc(1, sizeof(**dbInfo)); + if (NULL == *dbInfo) { + ctgError("calloc size[%d] failed", (int32_t)sizeof(**dbInfo)); + return TSDB_CODE_CTG_MEM_ERROR; + } + + (*dbInfo)->vgId = taosArrayDup(info->vgId); + if (NULL == (*dbInfo)->vgId) { + ctgError("taos array duplicate failed"); + tfree(*dbInfo); + return TSDB_CODE_CTG_MEM_ERROR; + } + + (*dbInfo)->vgroupVersion = info->vgroupVersion; + (*dbInfo)->hashRange = info->hashRange; } *exist = 1; -*/ + + return TSDB_CODE_SUCCESS; +} + + + +int32_t ctgGetDBVgroupFromMnode(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, SBuildUseDBInput *input, SUseDbOutput *out) { + char *msg = NULL; + SEpSet *pVnodeEpSet = NULL; + int32_t msgLen = 0; + + int32_t code = queryBuildMsg[TSDB_MSG_TYPE_USE_DB](input, &msg, 0, &msgLen); + if (code) { + return code; + } + + SRpcMsg rpcMsg = { + .msgType = TSDB_MSG_TYPE_USE_DB, + .pCont = msg, + .contLen = msgLen, + }; + + SRpcMsg rpcRsp = {0}; + + rpcSendRecv(pRpc, (SEpSet*)pMgmtEps, &rpcMsg, &rpcRsp); + + code = queryProcessMsgRsp[TSDB_MSG_TYPE_USE_DB](out, rpcRsp.pCont, rpcRsp.contLen); + if (code) { + return code; + } + return TSDB_CODE_SUCCESS; } @@ -144,7 +192,7 @@ int32_t catalogGetVgroupVersion(struct SCatalog* pCatalog, int32_t* version) { int32_t catalogUpdateVgroup(struct SCatalog* pCatalog, SVgroupListInfo* pVgroup) { if (NULL == pVgroup) { - ctgError("vgroup get from mnode succeed, but no output"); + ctgError("no valid vgroup list info to update"); return TSDB_CODE_CTG_INTERNAL_ERROR; } @@ -262,7 +310,33 @@ int32_t catalogGetDBVgroupVersion(struct SCatalog* pCatalog, const char* dbName, } int32_t catalogUpdateDBVgroup(struct SCatalog* pCatalog, const char* dbName, SDBVgroupInfo* dbInfo) { + if (NULL == pCatalog || NULL == dbName || NULL == dbInfo) { + return TSDB_CODE_CTG_INVALID_INPUT; + } + if (dbInfo->vgroupVersion < 0) { + if (pCatalog->dbCache.cache) { + taosHashRemove(pCatalog->dbCache.cache, dbName, strlen(dbName)); + } + + ctgWarn("remove db [%s] from cache", dbName); + return TSDB_CODE_SUCCESS; + } + + if (NULL == pCatalog->dbCache.cache) { + pCatalog->dbCache.cache = taosHashInit(CTG_DEFAULT_DB_NUMBER, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + if (NULL == pCatalog->dbCache.cache) { + ctgError("init hash[%d] for db cache failed", CTG_DEFAULT_DB_NUMBER); + return TSDB_CODE_CTG_MEM_ERROR; + } + } + + if (taosHashPut(pCatalog->dbCache.cache, dbName, strlen(dbName), dbInfo, sizeof(*dbInfo)) != 0) { + ctgError("push to vgroup hash cache failed"); + return TSDB_CODE_CTG_MEM_ERROR; + } + + return TSDB_CODE_SUCCESS; } @@ -273,8 +347,8 @@ int32_t catalogGetDBVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* return TSDB_CODE_CTG_INVALID_INPUT; } -/* int32_t exist = 0; + int32_t code = 0; if (0 == forceUpdate) { CTG_ERR_RET(ctgGetDBVgroupFromCache(pCatalog, dbName, dbInfo, &exist)); @@ -284,18 +358,34 @@ int32_t catalogGetDBVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* } } - SDBVgroupInfo* newDbInfo = NULL; - - CTG_ERR_RET(ctgGetDBVgroupFromMnode(pCatalog, pRpc, pMgmtEps, dbName, &newDbInfo)); + SUseDbOutput DbOut = {0}; + SBuildUseDBInput input = {0}; - CTG_ERR_RET(catalogUpdateDBVgroup(pCatalog, dbName, newDbInfo)); + strncpy(input.db, dbName, sizeof(input.db)); + input.db[sizeof(input.db) - 1] = 0; + input.vgroupVersion = pCatalog->vgroupCache.vgroupVersion; + input.dbGroupVersion = CTG_DEFAULT_INVALID_VERSION; + + CTG_ERR_RET(ctgGetDBVgroupFromMnode(pCatalog, pRpc, pMgmtEps, &input, &DbOut)); + + if (DbOut.vgroupList) { + CTG_ERR_JRET(catalogUpdateVgroup(pCatalog, DbOut.vgroupList)); + } + + if (DbOut.dbVgroup) { + CTG_ERR_JRET(catalogUpdateDBVgroup(pCatalog, dbName, DbOut.dbVgroup)); + } if (dbInfo) { - *dbInfo = newDbInfo; + *dbInfo = DbOut.dbVgroup; + DbOut.dbVgroup = NULL; } -*/ - return TSDB_CODE_SUCCESS; +_return: + tfree(DbOut.dbVgroup); + tfree(DbOut.vgroupList); + + return code; } diff --git a/source/libs/parser/CMakeLists.txt b/source/libs/parser/CMakeLists.txt index 155b72c1f9..5e635aa6a1 100644 --- a/source/libs/parser/CMakeLists.txt +++ b/source/libs/parser/CMakeLists.txt @@ -8,7 +8,7 @@ target_include_directories( target_link_libraries( parser - PRIVATE os util common catalog function transport + PRIVATE os util common catalog function transport query ) -ADD_SUBDIRECTORY(test) \ No newline at end of file +ADD_SUBDIRECTORY(test) diff --git a/source/libs/parser/test/CMakeLists.txt b/source/libs/parser/test/CMakeLists.txt index f7d7113243..4b9e586be3 100644 --- a/source/libs/parser/test/CMakeLists.txt +++ b/source/libs/parser/test/CMakeLists.txt @@ -8,7 +8,7 @@ AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) ADD_EXECUTABLE(parserTest ${SOURCE_LIST}) TARGET_LINK_LIBRARIES( parserTest - PUBLIC os util common parser catalog transport gtest function planner + PUBLIC os util common parser catalog transport gtest function planner query ) TARGET_INCLUDE_DIRECTORIES( diff --git a/source/libs/planner/CMakeLists.txt b/source/libs/planner/CMakeLists.txt index 23efce38f4..4e0d03d07a 100644 --- a/source/libs/planner/CMakeLists.txt +++ b/source/libs/planner/CMakeLists.txt @@ -8,7 +8,7 @@ target_include_directories( target_link_libraries( planner - PRIVATE os util common catalog parser transport function + PRIVATE os util common catalog parser transport function query ) -ADD_SUBDIRECTORY(test) \ No newline at end of file +ADD_SUBDIRECTORY(test) diff --git a/source/libs/planner/test/CMakeLists.txt b/source/libs/planner/test/CMakeLists.txt index a83d7a39d9..f00adfaeb2 100644 --- a/source/libs/planner/test/CMakeLists.txt +++ b/source/libs/planner/test/CMakeLists.txt @@ -8,7 +8,7 @@ AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) ADD_EXECUTABLE(plannerTest ${SOURCE_LIST}) TARGET_LINK_LIBRARIES( plannerTest - PUBLIC os util common planner parser catalog transport gtest function + PUBLIC os util common planner parser catalog transport gtest function query ) TARGET_INCLUDE_DIRECTORIES( diff --git a/source/libs/query/src/querymsg.c b/source/libs/query/src/querymsg.c index 924878c872..8f35fd9c3e 100644 --- a/source/libs/query/src/querymsg.c +++ b/source/libs/query/src/querymsg.c @@ -15,7 +15,7 @@ #include "taosmsg.h" #include "queryInt.h" - +#include "query.h" int32_t (*queryBuildMsg[TSDB_MSG_TYPE_MAX])(void* input, char **msg, int32_t msgSize, int32_t *msgLen) = {0}; @@ -60,6 +60,36 @@ int32_t queryBuildTableMetaReqMsg(void* input, char **msg, int32_t msgSize, int3 return TSDB_CODE_SUCCESS; } +int32_t queryBuildUseDbMsg(void* input, char **msg, int32_t msgSize, int32_t *msgLen) { + if (NULL == input || NULL == msg || NULL == msgLen) { + return TSDB_CODE_TSC_INVALID_INPUT; + } + + SBuildUseDBInput* bInput = (SBuildUseDBInput *)input; + + int32_t estimateSize = sizeof(SUseDbMsg); + if (NULL == *msg || msgSize < estimateSize) { + tfree(*msg); + *msg = calloc(1, estimateSize); + if (NULL == *msg) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + } + + SUseDbMsg *bMsg = (SUseDbMsg *)*msg; + + strncpy(bMsg->db, bInput->db, sizeof(bMsg->db)); + bMsg->db[sizeof(bMsg->db) - 1] = 0; + + bMsg->vgroupVersion = bInput->vgroupVersion; + bMsg->dbGroupVersion = bInput->dbGroupVersion; + + *msgLen = (int32_t)sizeof(*bMsg); + + return TSDB_CODE_SUCCESS; +} + + int32_t queryProcessVgroupListRsp(void* output, char *msg, int32_t msgSize) { if (NULL == output || NULL == msg || msgSize <= 0) { @@ -103,12 +133,126 @@ int32_t queryProcessVgroupListRsp(void* output, char *msg, int32_t msgSize) { return TSDB_CODE_SUCCESS; } + + + +int32_t queryProcessUseDBRsp(void* output, char *msg, int32_t msgSize) { + if (NULL == output || NULL == msg || msgSize <= 0) { + return TSDB_CODE_TSC_INVALID_INPUT; + } + + SUseDbRspMsg *pRsp = (SUseDbRspMsg *)msg; + SUseDbOutput *pOut = (SUseDbOutput *)output; + int32_t code = 0; + + if (msgSize <= sizeof(*pRsp)) { + qError("invalid use db rsp msg size, msgSize:%d", msgSize); + return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; + } + + pRsp->vgroupVersion = htonl(pRsp->vgroupVersion); + pRsp->dbVgroupVersion = htonl(pRsp->dbVgroupVersion); + + pRsp->vgroupNum = htonl(pRsp->vgroupNum); + pRsp->dbVgroupNum = htonl(pRsp->dbVgroupNum); + + if (pRsp->vgroupNum < 0) { + qError("invalid vgroup number[%d]", pRsp->vgroupNum); + return TSDB_CODE_TSC_INVALID_VALUE; + } + + if (pRsp->dbVgroupNum < 0) { + qError("invalid db vgroup number[%d]", pRsp->dbVgroupNum); + return TSDB_CODE_TSC_INVALID_VALUE; + } + + int32_t expectSize = pRsp->vgroupNum * sizeof(pRsp->vgroupInfo[0]) + pRsp->dbVgroupNum * sizeof(int32_t) + sizeof(*pRsp); + if (msgSize != expectSize) { + qError("vgroup list msg size mis-match, msgSize:%d, expected:%d, vgroup number:%d, db vgroup number:%d", msgSize, expectSize, pRsp->vgroupNum, pRsp->dbVgroupNum); + return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; + } + + if (pRsp->vgroupVersion < 0) { + qInfo("no new vgroup list info"); + if (pRsp->vgroupNum != 0) { + qError("invalid vgroup number[%d] for no new vgroup list case", pRsp->vgroupNum); + return TSDB_CODE_TSC_INVALID_VALUE; + } + } else { + int32_t s = sizeof(*pOut->vgroupList) + sizeof(pOut->vgroupList->vgroupInfo[0]) * pRsp->vgroupNum; + pOut->vgroupList = calloc(1, s); + if (NULL == pOut->vgroupList) { + qError("calloc size[%d] failed", s); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + pOut->vgroupList->vgroupNum = pRsp->vgroupNum; + pOut->vgroupList->vgroupVersion = pRsp->vgroupVersion; + + for (int32_t i = 0; i < pRsp->vgroupNum; ++i) { + pRsp->vgroupInfo[i].vgId = htonl(pRsp->vgroupInfo[i].vgId); + for (int32_t n = 0; n < pRsp->vgroupInfo[i].numOfEps; ++n) { + pRsp->vgroupInfo[i].epAddr[n].port = htonl(pRsp->vgroupInfo[i].epAddr[n].port); + } + + memcpy(&pOut->vgroupList->vgroupInfo[i], &pRsp->vgroupInfo[i], sizeof(pRsp->vgroupInfo[i])); + } + } + + int32_t *vgIdList = (int32_t *)((char *)pRsp->vgroupInfo + sizeof(pRsp->vgroupInfo[0]) * pRsp->vgroupNum); + + memcpy(pOut->db, pRsp->db, sizeof(pOut->db)); + + if (pRsp->dbVgroupVersion < 0) { + qInfo("no new vgroup info for db[%s]", pRsp->db); + } else { + pOut->dbVgroup = calloc(1, sizeof(*pOut->dbVgroup)); + if (NULL == pOut->dbVgroup) { + qError("calloc size[%d] failed", (int32_t)sizeof(*pOut->dbVgroup)); + code = TSDB_CODE_TSC_OUT_OF_MEMORY; + goto _exit; + } + + pOut->dbVgroup->vgId = taosArrayInit(pRsp->dbVgroupNum, sizeof(int32_t)); + if (NULL == pOut->dbVgroup->vgId) { + qError("taosArrayInit size[%d] failed", pRsp->dbVgroupNum); + code = TSDB_CODE_TSC_OUT_OF_MEMORY; + goto _exit; + } + + pOut->dbVgroup->vgroupVersion = pRsp->dbVgroupVersion; + pOut->dbVgroup->hashRange = htonl(pRsp->dbHashRange); + + for (int32_t i = 0; i < pRsp->dbVgroupNum; ++i) { + *(vgIdList + i) = htonl(*(vgIdList + i)); + + taosArrayPush(pOut->dbVgroup->vgId, vgIdList + i) ; + } + } + + return code; + +_exit: + if (pOut->dbVgroup && pOut->dbVgroup->vgId) { + taosArrayDestroy(pOut->dbVgroup->vgId); + pOut->dbVgroup->vgId = NULL; + } + + tfree(pOut->dbVgroup); + tfree(pOut->vgroupList); + + return code; +} + + void msgInit() { queryBuildMsg[TSDB_MSG_TYPE_TABLE_META] = queryBuildTableMetaReqMsg; queryBuildMsg[TSDB_MSG_TYPE_VGROUP_LIST] = queryBuildVgroupListReqMsg; + queryBuildMsg[TSDB_MSG_TYPE_USE_DB] = queryBuildUseDbMsg; //tscProcessMsgRsp[TSDB_MSG_TYPE_TABLE_META] = tscProcessTableMetaRsp; queryProcessMsgRsp[TSDB_MSG_TYPE_VGROUP_LIST] = queryProcessVgroupListRsp; + queryProcessMsgRsp[TSDB_MSG_TYPE_USE_DB] = queryProcessUseDBRsp; /* tscBuildMsg[TSDB_SQL_SELECT] = tscBuildQueryMsg; From ed68f95c11729c9690a8b3e17e6b9090d9cd8bcb Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 15 Dec 2021 10:48:49 +0800 Subject: [PATCH 32/43] minor changes --- source/dnode/mnode/impl/inc/mndDef.h | 14 +++++++------- source/dnode/mnode/impl/src/mndDb.c | 2 +- source/dnode/mnode/impl/src/mndDnode.c | 2 +- source/dnode/mnode/impl/src/mndUser.c | 9 ++++++--- 4 files changed, 15 insertions(+), 12 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 0f4839392e..e86ab1c448 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -17,14 +17,16 @@ #define _TD_MND_DEF_H_ #include "os.h" + +#include "cJSON.h" +#include "sync.h" #include "taosmsg.h" +#include "thash.h" #include "tlog.h" #include "trpc.h" #include "ttimer.h" -#include "thash.h" -#include "cJSON.h" + #include "mnode.h" -#include "sync.h" #ifdef __cplusplus extern "C" { @@ -41,10 +43,8 @@ extern int32_t mDebugFlag; #define mTrace(...) { if (mDebugFlag & DEBUG_TRACE) { taosPrintLog("MND ", mDebugFlag, __VA_ARGS__); }} typedef struct SClusterObj SClusterObj; -typedef struct SDnodeObj SDnodeObj; typedef struct SMnodeObj SMnodeObj; typedef struct SAcctObj SAcctObj; -typedef struct SUserObj SUserObj; typedef struct SDbObj SDbObj; typedef struct SVgObj SVgObj; typedef struct SFuncObj SFuncObj; @@ -119,7 +119,7 @@ typedef struct SClusterObj { int64_t updateTime; } SClusterObj; -typedef struct SDnodeObj { +typedef struct { int32_t id; int64_t createdTime; int64_t updateTime; @@ -178,7 +178,7 @@ typedef struct SAcctObj { SAcctInfo info; } SAcctObj; -typedef struct SUserObj { +typedef struct { char user[TSDB_USER_LEN]; char pass[TSDB_PASSWORD_LEN]; char acct[TSDB_USER_LEN]; diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index b638728647..2d1bfefc28 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -65,7 +65,7 @@ int32_t mndInitDb(SMnode *pMnode) { void mndCleanupDb(SMnode *pMnode) {} static SSdbRaw *mndDbActionEncode(SDbObj *pDb) { - SSdbRaw *pRaw = sdbAllocRaw(SDB_DB, TSDB_DB_VER_NUM, sizeof(SDbObj)); + SSdbRaw *pRaw = sdbAllocRaw(SDB_DB, TSDB_DB_VER_NUM, sizeof(SDbObj) + TSDB_DB_RESERVE_SIZE); if (pRaw == NULL) return NULL; int32_t dataPos = 0; diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index e00244d05c..72be28d2b3 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -108,7 +108,7 @@ static int32_t mndCreateDefaultDnode(SMnode *pMnode) { } static SSdbRaw *mndDnodeActionEncode(SDnodeObj *pDnode) { - SSdbRaw *pRaw = sdbAllocRaw(SDB_DNODE, TSDB_DNODE_VER, sizeof(SDnodeObj)); + SSdbRaw *pRaw = sdbAllocRaw(SDB_DNODE, TSDB_DNODE_VER, sizeof(SDnodeObj) + TSDB_DNODE_RESERVE_SIZE); if (pRaw == NULL) return NULL; int32_t dataPos = 0; diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index 6fb05653a5..2243f787b9 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -20,7 +20,8 @@ #include "mndTrans.h" #include "tkey.h" -#define SDB_USER_VER 1 +#define TSDB_USER_VER 1 +#define TSDB_USER_RESERVE_SIZE 64 static int32_t mndCreateDefaultUsers(SMnode *pMnode); static SSdbRaw *mndUserActionEncode(SUserObj *pUser); @@ -93,7 +94,7 @@ static int32_t mndCreateDefaultUsers(SMnode *pMnode) { } static SSdbRaw *mndUserActionEncode(SUserObj *pUser) { - SSdbRaw *pRaw = sdbAllocRaw(SDB_USER, SDB_USER_VER, sizeof(SUserObj)); + SSdbRaw *pRaw = sdbAllocRaw(SDB_USER, TSDB_USER_VER, sizeof(SUserObj) + TSDB_USER_RESERVE_SIZE); if (pRaw == NULL) return NULL; int32_t dataPos = 0; @@ -103,6 +104,7 @@ static SSdbRaw *mndUserActionEncode(SUserObj *pUser) { SDB_SET_INT64(pRaw, dataPos, pUser->createdTime) SDB_SET_INT64(pRaw, dataPos, pUser->updateTime) SDB_SET_INT8(pRaw, dataPos, pUser->superUser) + SDB_SET_RESERVE(pRaw, dataPos, TSDB_USER_RESERVE_SIZE) SDB_SET_DATALEN(pRaw, dataPos); return pRaw; @@ -112,7 +114,7 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) { int8_t sver = 0; if (sdbGetRawSoftVer(pRaw, &sver) != 0) return NULL; - if (sver != SDB_USER_VER) { + if (sver != TSDB_USER_VER) { mError("failed to decode user since %s", terrstr()); terrno = TSDB_CODE_SDB_INVALID_DATA_VER; return NULL; @@ -129,6 +131,7 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) { SDB_GET_INT64(pRaw, pRow, dataPos, &pUser->createdTime) SDB_GET_INT64(pRaw, pRow, dataPos, &pUser->updateTime) SDB_GET_INT8(pRaw, pRow, dataPos, &pUser->superUser) + SDB_GET_RESERVE(pRaw, pRow, dataPos, TSDB_USER_RESERVE_SIZE) return pRow; } From 9673e0edf25a33404966046d1229e516493fb637 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 15 Dec 2021 13:22:04 +0800 Subject: [PATCH 33/43] minor changes --- include/common/taosmsg.h | 16 +++++++++++----- source/dnode/mnode/impl/inc/mndDef.h | 3 +-- source/dnode/mnode/impl/src/mndDb.c | 2 +- source/dnode/mnode/impl/src/mndDnode.c | 1 + source/dnode/mnode/impl/src/mndUser.c | 5 +---- 5 files changed, 15 insertions(+), 12 deletions(-) diff --git a/include/common/taosmsg.h b/include/common/taosmsg.h index a75c4a7bac..2dc08e85da 100644 --- a/include/common/taosmsg.h +++ b/include/common/taosmsg.h @@ -389,17 +389,20 @@ typedef struct { int32_t maxDbs; int32_t maxTimeSeries; int32_t maxStreams; - int64_t maxStorage; // In unit of GB int32_t accessState; // Configured only by command + int64_t maxStorage; // In unit of GB + int32_t reserve[8]; } SCreateAcctMsg, SAlterAcctMsg; typedef struct { - char user[TSDB_USER_LEN]; + char user[TSDB_USER_LEN]; + int32_t reserve[8]; } SDropUserMsg, SDropAcctMsg; typedef struct { - char user[TSDB_USER_LEN]; - char pass[TSDB_PASSWORD_LEN]; + char user[TSDB_USER_LEN]; + char pass[TSDB_PASSWORD_LEN]; + int32_t reserve[8]; } SCreateUserMsg, SAlterUserMsg; typedef struct { @@ -876,16 +879,19 @@ typedef struct SShowRsp { } SShowRsp; typedef struct { - char ep[TSDB_EP_LEN]; // end point, hostname:port + char ep[TSDB_EP_LEN]; // end point, hostname:port + int32_t reserve[8]; } SCreateDnodeMsg; typedef struct { int32_t dnodeId; + int32_t reserve[8]; } SDropDnodeMsg; typedef struct { int32_t dnodeId; char config[TSDB_DNODE_CONFIG_LEN]; + int32_t reserve[8]; } SCfgDnodeMsg; typedef struct { diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index e86ab1c448..30c18d3e18 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -45,7 +45,6 @@ extern int32_t mDebugFlag; typedef struct SClusterObj SClusterObj; typedef struct SMnodeObj SMnodeObj; typedef struct SAcctObj SAcctObj; -typedef struct SDbObj SDbObj; typedef struct SVgObj SVgObj; typedef struct SFuncObj SFuncObj; typedef struct SOperObj SOperObj; @@ -209,7 +208,7 @@ typedef struct { int8_t cacheLastRow; } SDbCfg; -typedef struct SDbObj { +typedef struct { char name[TSDB_FULL_DB_NAME_LEN]; char acct[TSDB_USER_LEN]; int64_t createdTime; diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 2d1bfefc28..dd8998399e 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -300,7 +300,7 @@ static int32_t mndCreateDb(SMnode *pMnode, SMnodeMsg *pMsg, SCreateDbMsg *pCreat tstrncpy(dbObj.acct, acct, TSDB_USER_LEN); dbObj.createdTime = taosGetTimestampMs(); dbObj.updateTime = dbObj.createdTime; - dbObj.uid = 1234; + dbObj.uid = mndGenerateUid(dbObj.name, TSDB_FULL_DB_NAME_LEN); dbObj.cfg = (SDbCfg){.cacheBlockSize = pCreate->cacheBlockSize, .totalBlocks = pCreate->totalBlocks, .daysPerFile = pCreate->daysPerFile, diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 72be28d2b3..504be58d09 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -175,6 +175,7 @@ static int32_t mndDnodeActionDelete(SSdb *pSdb, SDnodeObj *pDnode) { static int32_t mndDnodeActionUpdate(SSdb *pSdb, SDnodeObj *pOldDnode, SDnodeObj *pNewDnode) { mTrace("dnode:%d, perform update action", pOldDnode->id); + pOldDnode->updateTime = pNewDnode->updateTime; return 0; } diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index 2243f787b9..91adc02d72 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -169,12 +169,8 @@ static int32_t mndUserActionDelete(SSdb *pSdb, SUserObj *pUser) { static int32_t mndUserActionUpdate(SSdb *pSdb, SUserObj *pOldUser, SUserObj *pNewUser) { mTrace("user:%s, perform update action", pOldUser->user); - memcpy(pOldUser->user, pNewUser->user, TSDB_USER_LEN); memcpy(pOldUser->pass, pNewUser->pass, TSDB_PASSWORD_LEN); - memcpy(pOldUser->acct, pNewUser->acct, TSDB_USER_LEN); - pOldUser->createdTime = pNewUser->createdTime; pOldUser->updateTime = pNewUser->updateTime; - pOldUser->superUser = pNewUser->superUser; return 0; } @@ -328,6 +324,7 @@ static int32_t mndProcessAlterUserMsg(SMnodeMsg *pMsg) { memcpy(&newUser, pUser, sizeof(SUserObj)); memset(pUser->pass, 0, sizeof(pUser->pass)); taosEncryptPass((uint8_t *)pAlter->pass, strlen(pAlter->pass), pUser->pass); + newUser.updateTime = taosGetTimestampMs(); int32_t code = mndUpdateUser(pMnode, pUser, &newUser, pMsg); sdbRelease(pMnode->pSdb, pOperUser); From 21c8fae2210f52d4f699511a00d4951a1c80227f Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 15 Dec 2021 13:49:55 +0800 Subject: [PATCH 34/43] minor changes --- include/util/taoserror.h | 174 +++++++++++++++++++++------------------ source/util/src/terror.c | 101 +++++++++++++---------- 2 files changed, 149 insertions(+), 126 deletions(-) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 8400031f9b..c53e7931ac 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -119,71 +119,107 @@ int32_t* taosGetErrno(); #define TSDB_CODE_TSC_VALUE_OUT_OF_RANGE TAOS_DEF_ERROR_CODE(0, 0x0223) //"Value out of range") #define TSDB_CODE_TSC_INVALID_INPUT TAOS_DEF_ERROR_CODE(0, 0X0224) //"Invalid tsc input") -// mnode -#define TSDB_CODE_MND_MSG_NOT_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0300) -#define TSDB_CODE_MND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0301) -#define TSDB_CODE_MND_ACTION_NEED_REPROCESSED TAOS_DEF_ERROR_CODE(0, 0x0302) -#define TSDB_CODE_MND_NO_RIGHTS TAOS_DEF_ERROR_CODE(0, 0x0303) -#define TSDB_CODE_MND_INVALID_OPTIONS TAOS_DEF_ERROR_CODE(0, 0x0304) -#define TSDB_CODE_MND_INVALID_CONNECTION TAOS_DEF_ERROR_CODE(0, 0x0305) -#define TSDB_CODE_MND_INVALID_MSG_VERSION TAOS_DEF_ERROR_CODE(0, 0x0306) -#define TSDB_CODE_MND_INVALID_MSG_LEN TAOS_DEF_ERROR_CODE(0, 0x0307) -#define TSDB_CODE_MND_INVALID_MSG_TYPE TAOS_DEF_ERROR_CODE(0, 0x0308) -#define TSDB_CODE_MND_TOO_MANY_SHELL_CONNS TAOS_DEF_ERROR_CODE(0, 0x0309) -#define TSDB_CODE_MND_INVALID_SHOWOBJ TAOS_DEF_ERROR_CODE(0, 0x030B) -#define TSDB_CODE_MND_INVALID_QUERY_ID TAOS_DEF_ERROR_CODE(0, 0x030C) -#define TSDB_CODE_MND_INVALID_STREAM_ID TAOS_DEF_ERROR_CODE(0, 0x030D) -#define TSDB_CODE_MND_INVALID_CONN_ID TAOS_DEF_ERROR_CODE(0, 0x030E) -#define TSDB_CODE_MND_MNODE_IS_RUNNING TAOS_DEF_ERROR_CODE(0, 0x0310) -#define TSDB_CODE_MND_FAILED_TO_CONFIG_SYNC TAOS_DEF_ERROR_CODE(0, 0x0311) -#define TSDB_CODE_MND_FAILED_TO_START_SYNC TAOS_DEF_ERROR_CODE(0, 0x0312) -#define TSDB_CODE_MND_FAILED_TO_CREATE_DIR TAOS_DEF_ERROR_CODE(0, 0x0313) -#define TSDB_CODE_MND_FAILED_TO_INIT_STEP TAOS_DEF_ERROR_CODE(0, 0x0314) +// mnode-common +#define TSDB_CODE_MND_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x0300) +#define TSDB_CODE_MND_MSG_NOT_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0301) +#define TSDB_CODE_MND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0302) +#define TSDB_CODE_MND_ACTION_NEED_REPROCESSED TAOS_DEF_ERROR_CODE(0, 0x0303) +#define TSDB_CODE_MND_NO_RIGHTS TAOS_DEF_ERROR_CODE(0, 0x0304) +#define TSDB_CODE_MND_INVALID_OPTIONS TAOS_DEF_ERROR_CODE(0, 0x0305) +#define TSDB_CODE_MND_INVALID_CONNECTION TAOS_DEF_ERROR_CODE(0, 0x0306) +#define TSDB_CODE_MND_INVALID_MSG_VERSION TAOS_DEF_ERROR_CODE(0, 0x0307) +#define TSDB_CODE_MND_INVALID_MSG_LEN TAOS_DEF_ERROR_CODE(0, 0x0308) +#define TSDB_CODE_MND_INVALID_MSG_TYPE TAOS_DEF_ERROR_CODE(0, 0x0309) +#define TSDB_CODE_MND_TOO_MANY_SHELL_CONNS TAOS_DEF_ERROR_CODE(0, 0x030A) -#define TSDB_CODE_SDB_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x0320) -#define TSDB_CODE_SDB_OBJ_ALREADY_THERE TAOS_DEF_ERROR_CODE(0, 0x0321) -#define TSDB_CODE_SDB_OBJ_NOT_THERE TAOS_DEF_ERROR_CODE(0, 0x0322) -#define TSDB_CODE_SDB_OBJ_CREATING TAOS_DEF_ERROR_CODE(0, 0x0323) -#define TSDB_CODE_SDB_OBJ_DROPPING TAOS_DEF_ERROR_CODE(0, 0x0324) -#define TSDB_CODE_SDB_INVALID_TABLE_TYPE TAOS_DEF_ERROR_CODE(0, 0x0325) -#define TSDB_CODE_SDB_INVALID_KEY_TYPE TAOS_DEF_ERROR_CODE(0, 0x0326) -#define TSDB_CODE_SDB_INVALID_ACTION_TYPE TAOS_DEF_ERROR_CODE(0, 0x0327) -#define TSDB_CODE_SDB_INVALID_STATUS_TYPE TAOS_DEF_ERROR_CODE(0, 0x0328) -#define TSDB_CODE_SDB_INVALID_DATA_VER TAOS_DEF_ERROR_CODE(0, 0x0329) -#define TSDB_CODE_SDB_INVALID_DATA_LEN TAOS_DEF_ERROR_CODE(0, 0x032A) -#define TSDB_CODE_SDB_INVALID_DATA_CONTENT TAOS_DEF_ERROR_CODE(0, 0x032B) +// mnode-show +#define TSDB_CODE_MND_INVALID_SHOWOBJ TAOS_DEF_ERROR_CODE(0, 0x0310) + +// mnode-profile +#define TSDB_CODE_MND_INVALID_QUERY_ID TAOS_DEF_ERROR_CODE(0, 0x0320) +#define TSDB_CODE_MND_INVALID_STREAM_ID TAOS_DEF_ERROR_CODE(0, 0x0321) +#define TSDB_CODE_MND_INVALID_CONN_ID TAOS_DEF_ERROR_CODE(0, 0x0322) +#define TSDB_CODE_MND_MNODE_IS_RUNNING TAOS_DEF_ERROR_CODE(0, 0x0323) +#define TSDB_CODE_MND_FAILED_TO_CONFIG_SYNC TAOS_DEF_ERROR_CODE(0, 0x0324) +#define TSDB_CODE_MND_FAILED_TO_START_SYNC TAOS_DEF_ERROR_CODE(0, 0x0325) +#define TSDB_CODE_MND_FAILED_TO_CREATE_DIR TAOS_DEF_ERROR_CODE(0, 0x0326) +#define TSDB_CODE_MND_FAILED_TO_INIT_STEP TAOS_DEF_ERROR_CODE(0, 0x0327) + +// mnode-sdb +#define TSDB_CODE_SDB_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x0330) +#define TSDB_CODE_SDB_OBJ_ALREADY_THERE TAOS_DEF_ERROR_CODE(0, 0x0331) +#define TSDB_CODE_SDB_OBJ_NOT_THERE TAOS_DEF_ERROR_CODE(0, 0x0332) +#define TSDB_CODE_SDB_OBJ_CREATING TAOS_DEF_ERROR_CODE(0, 0x0333) +#define TSDB_CODE_SDB_OBJ_DROPPING TAOS_DEF_ERROR_CODE(0, 0x0334) +#define TSDB_CODE_SDB_INVALID_TABLE_TYPE TAOS_DEF_ERROR_CODE(0, 0x0335) +#define TSDB_CODE_SDB_INVALID_KEY_TYPE TAOS_DEF_ERROR_CODE(0, 0x0336) +#define TSDB_CODE_SDB_INVALID_ACTION_TYPE TAOS_DEF_ERROR_CODE(0, 0x0337) +#define TSDB_CODE_SDB_INVALID_STATUS_TYPE TAOS_DEF_ERROR_CODE(0, 0x0338) +#define TSDB_CODE_SDB_INVALID_DATA_VER TAOS_DEF_ERROR_CODE(0, 0x0339) +#define TSDB_CODE_SDB_INVALID_DATA_LEN TAOS_DEF_ERROR_CODE(0, 0x033A) +#define TSDB_CODE_SDB_INVALID_DATA_CONTENT TAOS_DEF_ERROR_CODE(0, 0x033B) // mnode-dnode -#define TSDB_CODE_MND_DNODE_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0330) -#define TSDB_CODE_MND_DNODE_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0331) -#define TSDB_CODE_MND_NO_ENOUGH_DNODES TAOS_DEF_ERROR_CODE(0, 0x0332) -#define TSDB_CODE_MND_INVALID_CLUSTER_CFG TAOS_DEF_ERROR_CODE(0, 0x0333) -#define TSDB_CODE_MND_INVALID_CLUSTER_ID TAOS_DEF_ERROR_CODE(0, 0x0334) -#define TSDB_CODE_MND_INVALID_DNODE_CFG TAOS_DEF_ERROR_CODE(0, 0x0335) -#define TSDB_CODE_MND_INVALID_DNODE_EP TAOS_DEF_ERROR_CODE(0, 0x0336) -#define TSDB_CODE_MND_INVALID_DNODE_ID TAOS_DEF_ERROR_CODE(0, 0x0337) +#define TSDB_CODE_MND_DNODE_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0340) +#define TSDB_CODE_MND_DNODE_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0341) +#define TSDB_CODE_MND_TOO_MANY_DNODES TAOS_DEF_ERROR_CODE(0, 0x0342) +#define TSDB_CODE_MND_NO_ENOUGH_DNODES TAOS_DEF_ERROR_CODE(0, 0x0343) +#define TSDB_CODE_MND_INVALID_CLUSTER_CFG TAOS_DEF_ERROR_CODE(0, 0x0344) +#define TSDB_CODE_MND_INVALID_CLUSTER_ID TAOS_DEF_ERROR_CODE(0, 0x0345) +#define TSDB_CODE_MND_INVALID_DNODE_CFG TAOS_DEF_ERROR_CODE(0, 0x0346) +#define TSDB_CODE_MND_INVALID_DNODE_EP TAOS_DEF_ERROR_CODE(0, 0x0347) +#define TSDB_CODE_MND_INVALID_DNODE_ID TAOS_DEF_ERROR_CODE(0, 0x0348) -// mnode-vgroup -#define TSDB_CODE_MND_VGROUP_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0332) //"VGroup does not exist") -#define TSDB_CODE_MND_VGROUP_NOT_IN_DNODE TAOS_DEF_ERROR_CODE(0, 0x0338) //"Vgroup not in dnode") -#define TSDB_CODE_MND_VGROUP_ALREADY_IN_DNODE TAOS_DEF_ERROR_CODE(0, 0x0339) //"Vgroup already in dnode") -#define TSDB_CODE_MND_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x033C) //"Cluster not ready") +// mnode-mnode +#define TSDB_CODE_MND_MNODE_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0350) +#define TSDB_CODE_MND_MNODE_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0351) +#define TSDB_CODE_MND_TOO_MANY_MNODES TAOS_DEF_ERROR_CODE(0, 0x0352) // mnode-acct -#define TSDB_CODE_MND_ACCT_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0340) //"Account already exists") -#define TSDB_CODE_MND_ACCT_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0341) //"Invalid account") -#define TSDB_CODE_MND_INVALID_ACCT_OPTION TAOS_DEF_ERROR_CODE(0, 0x0342) //"Invalid account options") -#define TSDB_CODE_MND_ACCT_EXPIRED TAOS_DEF_ERROR_CODE(0, 0x0343) //"Account authorization has expired") +#define TSDB_CODE_MND_ACCT_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0360) +#define TSDB_CODE_MND_ACCT_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0361) +#define TSDB_CODE_MND_TOO_MANY_ACCTS TAOS_DEF_ERROR_CODE(0, 0x0362) +#define TSDB_CODE_MND_INVALID_ACCT_OPTION TAOS_DEF_ERROR_CODE(0, 0x0363) +#define TSDB_CODE_MND_ACCT_EXPIRED TAOS_DEF_ERROR_CODE(0, 0x0364) -#define TSDB_CODE_MND_USER_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0350) //"User already exists") -#define TSDB_CODE_MND_USER_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0351) //"Invalid user") -#define TSDB_CODE_MND_INVALID_USER_FORMAT TAOS_DEF_ERROR_CODE(0, 0x0352) //"Invalid user format") -#define TSDB_CODE_MND_INVALID_PASS_FORMAT TAOS_DEF_ERROR_CODE(0, 0x0353) //"Invalid password format") -#define TSDB_CODE_MND_NO_USER_FROM_CONN TAOS_DEF_ERROR_CODE(0, 0x0354) //"Can not get user from conn") -#define TSDB_CODE_MND_TOO_MANY_USERS TAOS_DEF_ERROR_CODE(0, 0x0355) //"Too many users") +// mnode-user +#define TSDB_CODE_MND_USER_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0370) +#define TSDB_CODE_MND_USER_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0371) +#define TSDB_CODE_MND_TOO_MANY_USERS TAOS_DEF_ERROR_CODE(0, 0x0372) +#define TSDB_CODE_MND_INVALID_USER_FORMAT TAOS_DEF_ERROR_CODE(0, 0x0373) +#define TSDB_CODE_MND_INVALID_PASS_FORMAT TAOS_DEF_ERROR_CODE(0, 0x0374) +#define TSDB_CODE_MND_NO_USER_FROM_CONN TAOS_DEF_ERROR_CODE(0, 0x0375) -#define TSDB_CODE_MND_MNODE_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0348) //"Mnode already exists") -#define TSDB_CODE_MND_MNODE_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0349) //"Mnode not there") +// mnode-db +#define TSDB_CODE_MND_DB_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0380) +#define TSDB_CODE_MND_DB_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0381) +#define TSDB_CODE_MND_TOO_MANY_DATABASES TAOS_DEF_ERROR_CODE(0, 0x0382) +#define TSDB_CODE_MND_DB_NOT_SELECTED TAOS_DEF_ERROR_CODE(0, 0x0383) +#define TSDB_CODE_MND_INVALID_DB TAOS_DEF_ERROR_CODE(0, 0x0384) +#define TSDB_CODE_MND_INVALID_DB_OPTION TAOS_DEF_ERROR_CODE(0, 0x0385) +#define TSDB_CODE_MND_INVALID_DB_CACHE_SIZE TAOS_DEF_ERROR_CODE(0, 0x0386) +#define TSDB_CODE_MND_INVALID_DB_TOTAL_BLOCKS TAOS_DEF_ERROR_CODE(0, 0x0387) +#define TSDB_CODE_MND_INVALID_DB_DAYS TAOS_DEF_ERROR_CODE(0, 0x0388) +#define TSDB_CODE_MND_INVALID_DB_KEEP0 TAOS_DEF_ERROR_CODE(0, 0x0389) +#define TSDB_CODE_MND_INVALID_DB_KEEP1 TAOS_DEF_ERROR_CODE(0, 0x038A) +#define TSDB_CODE_MND_INVALID_DB_KEEP2 TAOS_DEF_ERROR_CODE(0, 0x038B) +#define TSDB_CODE_MND_INVALID_DB_MIN_ROWS TAOS_DEF_ERROR_CODE(0, 0x038C) +#define TSDB_CODE_MND_INVALID_DB_MAX_ROWS TAOS_DEF_ERROR_CODE(0, 0x038D) +#define TSDB_CODE_MND_INVALID_DB_COMMIT_TIME TAOS_DEF_ERROR_CODE(0, 0x038E) +#define TSDB_CODE_MND_INVALID_DB_FSYNC_PERIOD TAOS_DEF_ERROR_CODE(0, 0x038F) +#define TSDB_CODE_MND_INVALID_DB_WAL_LEVEL TAOS_DEF_ERROR_CODE(0, 0x0390) +#define TSDB_CODE_MND_INVALID_DB_PRECISION TAOS_DEF_ERROR_CODE(0, 0x0391) +#define TSDB_CODE_MND_INVALID_DB_COMP TAOS_DEF_ERROR_CODE(0, 0x0392) +#define TSDB_CODE_MND_INVALID_DB_REPLICA TAOS_DEF_ERROR_CODE(0, 0x0393) +#define TSDB_CODE_MND_INVALID_DB_QUORUM TAOS_DEF_ERROR_CODE(0, 0x0394) +#define TSDB_CODE_MND_INVALID_DB_UPDATE TAOS_DEF_ERROR_CODE(0, 0x0395) +#define TSDB_CODE_MND_INVALID_DB_CACHE_LAST TAOS_DEF_ERROR_CODE(0, 0x0396) +#define TSDB_CODE_MND_DB_OPTION_UNCHANGED TAOS_DEF_ERROR_CODE(0, 0x0397) + +// mnode-vgroup +#define TSDB_CODE_MND_VGROUP_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03A0) +#define TSDB_CODE_MND_VGROUP_NOT_IN_DNODE TAOS_DEF_ERROR_CODE(0, 0x03A1) +#define TSDB_CODE_MND_VGROUP_ALREADY_IN_DNODE TAOS_DEF_ERROR_CODE(0, 0x03A2) // mnode-stable #define TSDB_CODE_MND_STB_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0360) @@ -224,30 +260,6 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_INVALID_TAG_LENGTH TAOS_DEF_ERROR_CODE(0, 0x0376) //"invalid tag length") #define TSDB_CODE_MND_INVALID_COLUMN_LENGTH TAOS_DEF_ERROR_CODE(0, 0x0377) //"invalid column length") -#define TSDB_CODE_MND_DB_NOT_SELECTED TAOS_DEF_ERROR_CODE(0, 0x0380) -#define TSDB_CODE_MND_DB_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0381) -#define TSDB_CODE_MND_DB_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0381) -#define TSDB_CODE_MND_INVALID_DB TAOS_DEF_ERROR_CODE(0, 0x0383) -#define TSDB_CODE_MND_INVALID_DB_OPTION TAOS_DEF_ERROR_CODE(0, 0x0382) -#define TSDB_CODE_MND_INVALID_DB_CACHE_SIZE TAOS_DEF_ERROR_CODE(0, 0x0383) -#define TSDB_CODE_MND_INVALID_DB_TOTAL_BLOCKS TAOS_DEF_ERROR_CODE(0, 0x0383) -#define TSDB_CODE_MND_INVALID_DB_DAYS TAOS_DEF_ERROR_CODE(0, 0x0383) -#define TSDB_CODE_MND_INVALID_DB_KEEP0 TAOS_DEF_ERROR_CODE(0, 0x0383) -#define TSDB_CODE_MND_INVALID_DB_KEEP1 TAOS_DEF_ERROR_CODE(0, 0x0383) -#define TSDB_CODE_MND_INVALID_DB_KEEP2 TAOS_DEF_ERROR_CODE(0, 0x0383) -#define TSDB_CODE_MND_INVALID_DB_MIN_ROWS TAOS_DEF_ERROR_CODE(0, 0x0383) -#define TSDB_CODE_MND_INVALID_DB_MAX_ROWS TAOS_DEF_ERROR_CODE(0, 0x0383) -#define TSDB_CODE_MND_INVALID_DB_COMMIT_TIME TAOS_DEF_ERROR_CODE(0, 0x0383) -#define TSDB_CODE_MND_INVALID_DB_FSYNC_PERIOD TAOS_DEF_ERROR_CODE(0, 0x0383) -#define TSDB_CODE_MND_INVALID_DB_WAL_LEVEL TAOS_DEF_ERROR_CODE(0, 0x0383) -#define TSDB_CODE_MND_INVALID_DB_PRECISION TAOS_DEF_ERROR_CODE(0, 0x0383) -#define TSDB_CODE_MND_INVALID_DB_COMP TAOS_DEF_ERROR_CODE(0, 0x0383) -#define TSDB_CODE_MND_INVALID_DB_REPLICA TAOS_DEF_ERROR_CODE(0, 0x0383) -#define TSDB_CODE_MND_INVALID_DB_QUORUM TAOS_DEF_ERROR_CODE(0, 0x0383) -#define TSDB_CODE_MND_INVALID_DB_UPDATE TAOS_DEF_ERROR_CODE(0, 0x0383) -#define TSDB_CODE_MND_INVALID_DB_CACHE_LAST TAOS_DEF_ERROR_CODE(0, 0x0383) -#define TSDB_CODE_MND_DB_OPTION_UNCHANGED TAOS_DEF_ERROR_CODE(0, 0x0383) -#define TSDB_CODE_MND_TOO_MANY_DATABASES TAOS_DEF_ERROR_CODE(0, 0x0385) //"Too many databases for account") // dnode #define TSDB_CODE_DND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0400) diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 42fde042e7..b8520821fc 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -129,7 +129,8 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_JSON_TYPE, "Invalid JSON data typ TAOS_DEFINE_ERROR(TSDB_CODE_TSC_VALUE_OUT_OF_RANGE, "Value out of range") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_INPUT, "Invalid tsc input") -// mnode +// mnode-common +TAOS_DEFINE_ERROR(TSDB_CODE_MND_NOT_READY, "Cluster not ready") TAOS_DEFINE_ERROR(TSDB_CODE_MND_MSG_NOT_PROCESSED, "Message not processed") TAOS_DEFINE_ERROR(TSDB_CODE_MND_ACTION_IN_PROGRESS, "Message is progressing") TAOS_DEFINE_ERROR(TSDB_CODE_MND_ACTION_NEED_REPROCESSED, "Message need to be reprocessed") @@ -140,7 +141,11 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_MSG_VERSION, "Incompatible protocol TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_MSG_LEN, "Invalid message length") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_MSG_TYPE, "Invalid message type") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_SHELL_CONNS, "Too many connections") + +// mnode-show TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_SHOWOBJ, "Data expired") + +// mnode-profile TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_QUERY_ID, "Invalid query id") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_STREAM_ID, "Invalid stream id") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_CONN_ID, "Invalid connection id") @@ -150,6 +155,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_FAILED_TO_START_SYNC, "failed to start sync" TAOS_DEFINE_ERROR(TSDB_CODE_MND_FAILED_TO_CREATE_DIR, "failed to create mnode dir") TAOS_DEFINE_ERROR(TSDB_CODE_MND_FAILED_TO_INIT_STEP, "failed to init components") +// mnode-sdb TAOS_DEFINE_ERROR(TSDB_CODE_SDB_APP_ERROR, "Unexpected generic error in sdb") TAOS_DEFINE_ERROR(TSDB_CODE_SDB_OBJ_ALREADY_THERE, "Object already there") TAOS_DEFINE_ERROR(TSDB_CODE_SDB_OBJ_NOT_THERE, "Object not there") @@ -164,69 +170,41 @@ TAOS_DEFINE_ERROR(TSDB_CODE_SDB_INVALID_DATA_LEN, "Invalid raw data len" TAOS_DEFINE_ERROR(TSDB_CODE_SDB_INVALID_DATA_CONTENT, "Invalid raw data content") // mnode-dnode -TAOS_DEFINE_ERROR(TSDB_CODE_MND_DNODE_ALREADY_EXIST, "DNode already exists") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_DNODE_NOT_EXIST, "DNode does not exist") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_NO_ENOUGH_DNODES, "Out of DNodes") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_DNODE_ALREADY_EXIST, "Dnode already exists") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_DNODE_NOT_EXIST, "Dnode does not exist") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_DNODES, "Too many dnodes") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_NO_ENOUGH_DNODES, "Out of dnodes") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_CLUSTER_CFG, "Cluster cfg inconsistent") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_CLUSTER_ID, "Cluster id not match") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DNODE_CFG, "Invalid dnode cfg") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DNODE_EP, "Invalid dnode end point") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DNODE_ID, "Invalid dnode id") -// mnode-vgroup -TAOS_DEFINE_ERROR(TSDB_CODE_MND_VGROUP_NOT_EXIST, "VGroup does not exist") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_VGROUP_NOT_IN_DNODE, "Vgroup not in dnode") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_VGROUP_ALREADY_IN_DNODE, "Vgroup already in dnode") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_NOT_READY, "Cluster not ready") +// mnode-mnode +TAOS_DEFINE_ERROR(TSDB_CODE_MND_MNODE_ALREADY_EXIST, "Mnode already exists") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_MNODE_NOT_EXIST, "Mnode not there") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_MNODES, "Too many mnodes") // mnode-acct TAOS_DEFINE_ERROR(TSDB_CODE_MND_ACCT_ALREADY_EXIST, "Account already exists") TAOS_DEFINE_ERROR(TSDB_CODE_MND_ACCT_NOT_EXIST, "Invalid account") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_ACCTS, "Too many accounts") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_ACCT_OPTION, "Invalid account options") TAOS_DEFINE_ERROR(TSDB_CODE_MND_ACCT_EXPIRED, "Account authorization has expired") +// mnode-user TAOS_DEFINE_ERROR(TSDB_CODE_MND_USER_ALREADY_EXIST, "User already exists") TAOS_DEFINE_ERROR(TSDB_CODE_MND_USER_NOT_EXIST, "Invalid user") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_USERS, "Too many users") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_USER_FORMAT, "Invalid user format") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_PASS_FORMAT, "Invalid password format") TAOS_DEFINE_ERROR(TSDB_CODE_MND_NO_USER_FROM_CONN, "Can not get user from conn") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_USERS, "Too many users") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_MNODE_ALREADY_EXIST, "Mnode already exists") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_MNODE_NOT_EXIST, "Mnode not there") - -// mnode-stable -TAOS_DEFINE_ERROR(TSDB_CODE_MND_STB_ALREADY_EXIST, "Stable already exists") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TABLE_ID, "Table name too long") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TABLE_NAME, "Table does not exist") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TABLE_TYPE, "Invalid table type in tsdb") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_TAGS, "Too many tags") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_COLUMNS, "Too many columns") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_TIMESERIES, "Too many time series") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_NOT_SUPER_TABLE, "Not super table") // operation only available for super table -TAOS_DEFINE_ERROR(TSDB_CODE_MND_COL_NAME_TOO_LONG, "Tag name too long") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_TAG_ALREAY_EXIST, "Tag already exists") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_TAG_NOT_EXIST, "Tag does not exist") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_FIELD_ALREAY_EXIST, "Field already exists") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_FIELD_NOT_EXIST, "Field does not exist") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_STABLE_NAME, "Super table does not exist") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_CREATE_TABLE_MSG, "Invalid create table message") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_EXCEED_MAX_ROW_BYTES, "Exceed max row bytes") - -TAOS_DEFINE_ERROR(TSDB_CODE_MND_FUNC_ALREADY_EXIST, "Func already exists") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_FUNC_NOT_EXIST, "Func not exists") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC, "Invalid func") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_NAME, "Invalid func name") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_COMMENT, "Invalid func comment") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_CODE, "Invalid func code") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_BUFSIZE, "Invalid func bufSize") - -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TAG_LENGTH, "invalid tag length") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_COLUMN_LENGTH, "invalid column length") - -TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_NOT_SELECTED, "Database not specified or available") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_NOT_EXIST, "Database not exist") +// mnode-db TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_ALREADY_EXIST, "Database already exists") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_NOT_EXIST, "Database not exist") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_DATABASES, "Too many databases for account") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_NOT_SELECTED, "Database not specified or available") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB, "Invalid database name") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_OPTION, "Invalid database options") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_CACHE_SIZE, "Invalid database cache block size option") @@ -247,7 +225,40 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_QUORUM, "Invalid database quor TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_UPDATE, "Invalid database update option") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_CACHE_LAST, "Invalid database cache last option") TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_OPTION_UNCHANGED, "Database options not changed") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_DATABASES, "Too many databases for account") + +// mnode-vgroup +TAOS_DEFINE_ERROR(TSDB_CODE_MND_VGROUP_ALREADY_IN_DNODE, "Vgroup already in dnode") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_VGROUP_NOT_IN_DNODE, "Vgroup not in dnode") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_VGROUP_NOT_EXIST, "VGroup does not exist") + +// mnode-stable +TAOS_DEFINE_ERROR(TSDB_CODE_MND_STB_ALREADY_EXIST, "Stable already exists") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TABLE_ID, "Table name too long") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TABLE_NAME, "Table does not exist") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TABLE_TYPE, "Invalid table type in tsdb") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_TAGS, "Too many tags") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_COLUMNS, "Too many columns") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_TIMESERIES, "Too many time series") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_NOT_SUPER_TABLE, "Not super table") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_COL_NAME_TOO_LONG, "Tag name too long") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_TAG_ALREAY_EXIST, "Tag already exists") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_TAG_NOT_EXIST, "Tag does not exist") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_FIELD_ALREAY_EXIST, "Field already exists") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_FIELD_NOT_EXIST, "Field does not exist") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_STABLE_NAME, "Super table does not exist") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_CREATE_TABLE_MSG, "Invalid create table message") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_EXCEED_MAX_ROW_BYTES, "Exceed max row bytes") + +TAOS_DEFINE_ERROR(TSDB_CODE_MND_FUNC_ALREADY_EXIST, "Func already exists") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_FUNC_NOT_EXIST, "Func not exists") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC, "Invalid func") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_NAME, "Invalid func name") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_COMMENT, "Invalid func comment") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_CODE, "Invalid func code") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_BUFSIZE, "Invalid func bufSize") + +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TAG_LENGTH, "invalid tag length") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_COLUMN_LENGTH, "invalid column length") // dnode TAOS_DEFINE_ERROR(TSDB_CODE_DND_ACTION_IN_PROGRESS, "Action in progress") From d22296f2a62f18baa63b3b7fc55dafe34d0eadb3 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 15 Dec 2021 14:20:49 +0800 Subject: [PATCH 35/43] rename test name --- include/common/taosmsg.h | 1 - source/dnode/mgmt/impl/test/CMakeLists.txt | 1 + source/dnode/mgmt/impl/test/db/CMakeLists.txt | 29 ++ source/dnode/mgmt/impl/test/db/db.cpp | 315 ++++++++++++++++++ .../dnode/mgmt/impl/test/dnode/CMakeLists.txt | 12 +- source/dnode/mgmt/impl/test/dnode/dnode.cpp | 20 +- .../dnode/mgmt/impl/test/user/CMakeLists.txt | 12 +- source/dnode/mgmt/impl/test/user/user.cpp | 6 +- 8 files changed, 370 insertions(+), 26 deletions(-) create mode 100644 source/dnode/mgmt/impl/test/db/CMakeLists.txt create mode 100644 source/dnode/mgmt/impl/test/db/db.cpp diff --git a/include/common/taosmsg.h b/include/common/taosmsg.h index 9dce9193c5..38b9c6fc8c 100644 --- a/include/common/taosmsg.h +++ b/include/common/taosmsg.h @@ -632,7 +632,6 @@ typedef struct { int32_t reserve[8]; } SUseDbMsg; - typedef struct { char db[TSDB_TABLE_FNAME_LEN]; int32_t reserve[8]; diff --git a/source/dnode/mgmt/impl/test/CMakeLists.txt b/source/dnode/mgmt/impl/test/CMakeLists.txt index b340029044..93b0bc83e0 100644 --- a/source/dnode/mgmt/impl/test/CMakeLists.txt +++ b/source/dnode/mgmt/impl/test/CMakeLists.txt @@ -1,5 +1,6 @@ # add_subdirectory(acct) # add_subdirectory(cluster) +add_subdirectory(db) add_subdirectory(dnode) # add_subdirectory(profile) # add_subdirectory(show) diff --git a/source/dnode/mgmt/impl/test/db/CMakeLists.txt b/source/dnode/mgmt/impl/test/db/CMakeLists.txt new file mode 100644 index 0000000000..b778e3854f --- /dev/null +++ b/source/dnode/mgmt/impl/test/db/CMakeLists.txt @@ -0,0 +1,29 @@ +add_executable(dnode_test_db "") + +target_sources(dnode_test_db + PRIVATE + "db.cpp" + "../sut/deploy.cpp" +) + +target_link_libraries( + dnode_test_db + PUBLIC dnode + PUBLIC util + PUBLIC os + PUBLIC gtest_main +) + +target_include_directories(dnode_test_db + PUBLIC + "${CMAKE_SOURCE_DIR}/include/server/dnode/mgmt" + "${CMAKE_CURRENT_SOURCE_DIR}/../../inc" + "${CMAKE_CURRENT_SOURCE_DIR}/../sut" +) + +enable_testing() + +add_test( + NAME dnode_test_db + COMMAND dnode_test_db +) diff --git a/source/dnode/mgmt/impl/test/db/db.cpp b/source/dnode/mgmt/impl/test/db/db.cpp new file mode 100644 index 0000000000..4756ea508f --- /dev/null +++ b/source/dnode/mgmt/impl/test/db/db.cpp @@ -0,0 +1,315 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "deploy.h" + +class DndTestDb : public ::testing::Test { + protected: + static SServer* CreateServer(const char* path, const char* fqdn, uint16_t port, const char* firstEp) { + SServer* pServer = createServer(path, fqdn, port, firstEp); + ASSERT(pServer); + return pServer; + } + + static void SetUpTestSuite() { + initLog("/tmp/dnode_test_db"); + + const char* fqdn = "localhost"; + const char* firstEp = "localhost:9530"; + pServer = CreateServer("/tmp/dnode_test_db", fqdn, 9530, firstEp); + pClient = createClient("root", "taosdata", fqdn, 9530); + taosMsleep(300); + } + + static void TearDownTestSuite() { + stopServer(pServer); + dropClient(pClient); + pServer = NULL; + pClient = NULL; + } + + static SServer* pServer; + static SClient* pClient; + static int32_t connId; + + public: + void SetUp() override {} + void TearDown() override {} + + void SendTheCheckShowMetaMsg(int8_t showType, const char* showName, int32_t columns) { + SShowMsg* pShow = (SShowMsg*)rpcMallocCont(sizeof(SShowMsg)); + pShow->type = showType; + strcpy(pShow->db, ""); + + SRpcMsg showRpcMsg = {0}; + showRpcMsg.pCont = pShow; + showRpcMsg.contLen = sizeof(SShowMsg); + showRpcMsg.msgType = TSDB_MSG_TYPE_SHOW; + + sendMsg(pClient, &showRpcMsg); + ASSERT_NE(pClient->pRsp, nullptr); + ASSERT_EQ(pClient->pRsp->code, 0); + ASSERT_NE(pClient->pRsp->pCont, nullptr); + + SShowRsp* pShowRsp = (SShowRsp*)pClient->pRsp->pCont; + ASSERT_NE(pShowRsp, nullptr); + pShowRsp->showId = htonl(pShowRsp->showId); + pMeta = &pShowRsp->tableMeta; + pMeta->numOfTags = htons(pMeta->numOfTags); + pMeta->numOfColumns = htons(pMeta->numOfColumns); + pMeta->sversion = htons(pMeta->sversion); + pMeta->tversion = htons(pMeta->tversion); + pMeta->tuid = htobe64(pMeta->tuid); + pMeta->suid = htobe64(pMeta->suid); + + showId = pShowRsp->showId; + + EXPECT_NE(pShowRsp->showId, 0); + EXPECT_STREQ(pMeta->tbFname, showName); + EXPECT_EQ(pMeta->numOfTags, 0); + EXPECT_EQ(pMeta->numOfColumns, columns); + EXPECT_EQ(pMeta->precision, 0); + EXPECT_EQ(pMeta->tableType, 0); + EXPECT_EQ(pMeta->update, 0); + EXPECT_EQ(pMeta->sversion, 0); + EXPECT_EQ(pMeta->tversion, 0); + EXPECT_EQ(pMeta->tuid, 0); + EXPECT_EQ(pMeta->suid, 0); + } + + void CheckSchema(int32_t index, int8_t type, int32_t bytes, const char* name) { + SSchema* pSchema = &pMeta->pSchema[index]; + pSchema->bytes = htons(pSchema->bytes); + EXPECT_EQ(pSchema->colId, 0); + EXPECT_EQ(pSchema->type, type); + EXPECT_EQ(pSchema->bytes, bytes); + EXPECT_STREQ(pSchema->name, name); + } + + void SendThenCheckShowRetrieveMsg(int32_t rows) { + SRetrieveTableMsg* pRetrieve = (SRetrieveTableMsg*)rpcMallocCont(sizeof(SRetrieveTableMsg)); + pRetrieve->showId = htonl(showId); + pRetrieve->free = 0; + + SRpcMsg retrieveRpcMsg = {0}; + retrieveRpcMsg.pCont = pRetrieve; + retrieveRpcMsg.contLen = sizeof(SRetrieveTableMsg); + retrieveRpcMsg.msgType = TSDB_MSG_TYPE_SHOW_RETRIEVE; + + sendMsg(pClient, &retrieveRpcMsg); + + ASSERT_NE(pClient->pRsp, nullptr); + ASSERT_EQ(pClient->pRsp->code, 0); + ASSERT_NE(pClient->pRsp->pCont, nullptr); + + pRetrieveRsp = (SRetrieveTableRsp*)pClient->pRsp->pCont; + ASSERT_NE(pRetrieveRsp, nullptr); + pRetrieveRsp->numOfRows = htonl(pRetrieveRsp->numOfRows); + pRetrieveRsp->offset = htobe64(pRetrieveRsp->offset); + pRetrieveRsp->useconds = htobe64(pRetrieveRsp->useconds); + pRetrieveRsp->compLen = htonl(pRetrieveRsp->compLen); + + EXPECT_EQ(pRetrieveRsp->numOfRows, rows); + EXPECT_EQ(pRetrieveRsp->offset, 0); + EXPECT_EQ(pRetrieveRsp->useconds, 0); + // EXPECT_EQ(pRetrieveRsp->completed, completed); + EXPECT_EQ(pRetrieveRsp->precision, TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(pRetrieveRsp->compressed, 0); + EXPECT_EQ(pRetrieveRsp->reserved, 0); + EXPECT_EQ(pRetrieveRsp->compLen, 0); + + pData = pRetrieveRsp->data; + pos = 0; + } + + void CheckInt16(int16_t val) { + int16_t data = *((int16_t*)(pData + pos)); + pos += sizeof(int16_t); + EXPECT_EQ(data, val); + } + + void CheckInt64(int64_t val) { + int64_t data = *((int64_t*)(pData + pos)); + pos += sizeof(int64_t); + EXPECT_EQ(data, val); + } + + void CheckTimestamp() { + int64_t data = *((int64_t*)(pData + pos)); + pos += sizeof(int64_t); + EXPECT_GT(data, 0); + } + + void CheckBinary(const char* val, int32_t len) { + pos += sizeof(VarDataLenT); + char* data = (char*)(pData + pos); + pos += len; + EXPECT_STREQ(data, val); + } + + int32_t showId; + STableMetaMsg* pMeta; + SRetrieveTableRsp* pRetrieveRsp; + char* pData; + int32_t pos; +}; + +SServer* DndTestDb::pServer; +SClient* DndTestDb::pClient; +int32_t DndTestDb::connId; + +TEST_F(DndTestDb, ShowUser) { + SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_USER, "show users", 4); + CheckSchema(0, TSDB_DATA_TYPE_BINARY, TSDB_USER_LEN + VARSTR_HEADER_SIZE, "name"); + CheckSchema(1, TSDB_DATA_TYPE_BINARY, 10 + VARSTR_HEADER_SIZE, "privilege"); + CheckSchema(2, TSDB_DATA_TYPE_TIMESTAMP, 8, "create time"); + CheckSchema(3, TSDB_DATA_TYPE_BINARY, TSDB_USER_LEN + VARSTR_HEADER_SIZE, "account"); + + SendThenCheckShowRetrieveMsg(1); + CheckBinary("root", TSDB_USER_LEN); + CheckBinary("super", 10); + CheckTimestamp(); + CheckBinary("root", TSDB_USER_LEN); +} + +TEST_F(DndTestDb, CreateUser_01) { + { + SCreateUserMsg* pReq = (SCreateUserMsg*)rpcMallocCont(sizeof(SCreateUserMsg)); + strcpy(pReq->user, "u1"); + strcpy(pReq->pass, "p1"); + + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pReq; + rpcMsg.contLen = sizeof(SCreateUserMsg); + rpcMsg.msgType = TSDB_MSG_TYPE_CREATE_USER; + + sendMsg(pClient, &rpcMsg); + SRpcMsg* pMsg = pClient->pRsp; + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); + } + + { + SCreateUserMsg* pReq = (SCreateUserMsg*)rpcMallocCont(sizeof(SCreateUserMsg)); + strcpy(pReq->user, "u2"); + strcpy(pReq->pass, "p2"); + + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pReq; + rpcMsg.contLen = sizeof(SCreateUserMsg); + rpcMsg.msgType = TSDB_MSG_TYPE_CREATE_USER; + + sendMsg(pClient, &rpcMsg); + SRpcMsg* pMsg = pClient->pRsp; + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); + } + + SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_USER, "show users", 4); + SendThenCheckShowRetrieveMsg(3); + CheckBinary("u1", TSDB_USER_LEN); + CheckBinary("root", TSDB_USER_LEN); + CheckBinary("u2", TSDB_USER_LEN); + CheckBinary("normal", 10); + CheckBinary("super", 10); + CheckBinary("normal", 10); + CheckTimestamp(); + CheckTimestamp(); + CheckTimestamp(); + CheckBinary("root", TSDB_USER_LEN); + CheckBinary("root", TSDB_USER_LEN); + CheckBinary("root", TSDB_USER_LEN); +} + +TEST_F(DndTestDb, AlterUser_01) { + SAlterUserMsg* pReq = (SAlterUserMsg*)rpcMallocCont(sizeof(SAlterUserMsg)); + strcpy(pReq->user, "u1"); + strcpy(pReq->pass, "p2"); + + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pReq; + rpcMsg.contLen = sizeof(SAlterUserMsg); + rpcMsg.msgType = TSDB_MSG_TYPE_ALTER_USER; + + sendMsg(pClient, &rpcMsg); + SRpcMsg* pMsg = pClient->pRsp; + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); + + SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_USER, "show users", 4); + SendThenCheckShowRetrieveMsg(3); + CheckBinary("u1", TSDB_USER_LEN); + CheckBinary("root", TSDB_USER_LEN); + CheckBinary("u2", TSDB_USER_LEN); + CheckBinary("normal", 10); + CheckBinary("super", 10); + CheckBinary("normal", 10); + CheckTimestamp(); + CheckTimestamp(); + CheckTimestamp(); + CheckBinary("root", TSDB_USER_LEN); + CheckBinary("root", TSDB_USER_LEN); + CheckBinary("root", TSDB_USER_LEN); +} + +TEST_F(DndTestDb, DropUser_01) { + SDropUserMsg* pReq = (SDropUserMsg*)rpcMallocCont(sizeof(SDropUserMsg)); + strcpy(pReq->user, "u1"); + + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pReq; + rpcMsg.contLen = sizeof(SDropUserMsg); + rpcMsg.msgType = TSDB_MSG_TYPE_DROP_USER; + + sendMsg(pClient, &rpcMsg); + SRpcMsg* pMsg = pClient->pRsp; + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); + + SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_USER, "show users", 4); + SendThenCheckShowRetrieveMsg(2); + CheckBinary("root", TSDB_USER_LEN); + CheckBinary("u2", TSDB_USER_LEN); + CheckBinary("super", 10); + CheckBinary("normal", 10); + CheckTimestamp(); + CheckTimestamp(); + CheckBinary("root", TSDB_USER_LEN); + CheckBinary("root", TSDB_USER_LEN); +} + +TEST_F(DndTestDb, RestartDnode) { + stopServer(pServer); + pServer = NULL; + + uInfo("start all server"); + + const char* fqdn = "localhost"; + const char* firstEp = "localhost:9530"; + pServer = startServer("/tmp/dnode_test_db", fqdn, 9530, firstEp); + + uInfo("all server is running"); + + SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_USER, "show users", 4); + SendThenCheckShowRetrieveMsg(2); + CheckBinary("root", TSDB_USER_LEN); + CheckBinary("u2", TSDB_USER_LEN); + CheckBinary("super", 10); + CheckBinary("normal", 10); + CheckTimestamp(); + CheckTimestamp(); + CheckBinary("root", TSDB_USER_LEN); + CheckBinary("root", TSDB_USER_LEN); +} diff --git a/source/dnode/mgmt/impl/test/dnode/CMakeLists.txt b/source/dnode/mgmt/impl/test/dnode/CMakeLists.txt index e118cb8725..ebe2f3c5eb 100644 --- a/source/dnode/mgmt/impl/test/dnode/CMakeLists.txt +++ b/source/dnode/mgmt/impl/test/dnode/CMakeLists.txt @@ -1,20 +1,20 @@ -add_executable(dndTestDnode "") +add_executable(dnode_test_dnode "") -target_sources(dndTestDnode +target_sources(dnode_test_dnode PRIVATE "dnode.cpp" "../sut/deploy.cpp" ) target_link_libraries( - dndTestDnode + dnode_test_dnode PUBLIC dnode PUBLIC util PUBLIC os PUBLIC gtest_main ) -target_include_directories(dndTestDnode +target_include_directories(dnode_test_dnode PUBLIC "${CMAKE_SOURCE_DIR}/include/server/dnode/mgmt" "${CMAKE_CURRENT_SOURCE_DIR}/../../inc" @@ -24,6 +24,6 @@ target_include_directories(dndTestDnode enable_testing() add_test( - NAME dndTestDnode - COMMAND dndTestDnode + NAME dnode_test_dnode + COMMAND dnode_test_dnode ) diff --git a/source/dnode/mgmt/impl/test/dnode/dnode.cpp b/source/dnode/mgmt/impl/test/dnode/dnode.cpp index 580fe8e131..dda991b906 100644 --- a/source/dnode/mgmt/impl/test/dnode/dnode.cpp +++ b/source/dnode/mgmt/impl/test/dnode/dnode.cpp @@ -24,15 +24,15 @@ class DndTestDnode : public ::testing::Test { } static void SetUpTestSuite() { - initLog("/tmp/dndTestDnode"); + initLog("/tmp/dnode_test_dnode"); const char* fqdn = "localhost"; const char* firstEp = "localhost:9521"; - pServer1 = CreateServer("/tmp/dndTestDnode1", fqdn, 9521, firstEp); - pServer2 = CreateServer("/tmp/dndTestDnode2", fqdn, 9522, firstEp); - pServer3 = CreateServer("/tmp/dndTestDnode3", fqdn, 9523, firstEp); - pServer4 = CreateServer("/tmp/dndTestDnode4", fqdn, 9524, firstEp); - pServer5 = CreateServer("/tmp/dndTestDnode5", fqdn, 9525, firstEp); + pServer1 = CreateServer("/tmp/dnode_test_dnode1", fqdn, 9521, firstEp); + pServer2 = CreateServer("/tmp/dnode_test_dnode2", fqdn, 9522, firstEp); + pServer3 = CreateServer("/tmp/dnode_test_dnode3", fqdn, 9523, firstEp); + pServer4 = CreateServer("/tmp/dnode_test_dnode4", fqdn, 9524, firstEp); + pServer5 = CreateServer("/tmp/dnode_test_dnode5", fqdn, 9525, firstEp); pClient = createClient("root", "taosdata", fqdn, 9521); taosMsleep(300); } @@ -378,10 +378,10 @@ TEST_F(DndTestDnode, RestartDnode_01) { const char* fqdn = "localhost"; const char* firstEp = "localhost:9521"; - pServer1 = startServer("/tmp/dndTestDnode1", fqdn, 9521, firstEp); - pServer3 = startServer("/tmp/dndTestDnode3", fqdn, 9523, firstEp); - pServer4 = startServer("/tmp/dndTestDnode4", fqdn, 9524, firstEp); - pServer5 = startServer("/tmp/dndTestDnode5", fqdn, 9525, firstEp); + pServer1 = startServer("/tmp/dnode_test_dnode1", fqdn, 9521, firstEp); + pServer3 = startServer("/tmp/dnode_test_dnode3", fqdn, 9523, firstEp); + pServer4 = startServer("/tmp/dnode_test_dnode4", fqdn, 9524, firstEp); + pServer5 = startServer("/tmp/dnode_test_dnode5", fqdn, 9525, firstEp); uInfo("all server is running"); diff --git a/source/dnode/mgmt/impl/test/user/CMakeLists.txt b/source/dnode/mgmt/impl/test/user/CMakeLists.txt index b5f02e41f4..ca8f2ec6db 100644 --- a/source/dnode/mgmt/impl/test/user/CMakeLists.txt +++ b/source/dnode/mgmt/impl/test/user/CMakeLists.txt @@ -1,20 +1,20 @@ -add_executable(dndTestUser "") +add_executable(dnode_test_user "") -target_sources(dndTestUser +target_sources(dnode_test_user PRIVATE "user.cpp" "../sut/deploy.cpp" ) target_link_libraries( - dndTestUser + dnode_test_user PUBLIC dnode PUBLIC util PUBLIC os PUBLIC gtest_main ) -target_include_directories(dndTestUser +target_include_directories(dnode_test_user PUBLIC "${CMAKE_SOURCE_DIR}/include/server/dnode/mgmt" "${CMAKE_CURRENT_SOURCE_DIR}/../../inc" @@ -24,6 +24,6 @@ target_include_directories(dndTestUser enable_testing() add_test( - NAME dndTestUser - COMMAND dndTestUser + NAME dnode_test_user + COMMAND dnode_test_user ) diff --git a/source/dnode/mgmt/impl/test/user/user.cpp b/source/dnode/mgmt/impl/test/user/user.cpp index 48be2635cd..8e25181ae1 100644 --- a/source/dnode/mgmt/impl/test/user/user.cpp +++ b/source/dnode/mgmt/impl/test/user/user.cpp @@ -24,11 +24,11 @@ class DndTestUser : public ::testing::Test { } static void SetUpTestSuite() { - initLog("/tmp/dndTestUser"); + initLog("/tmp/dnode_test_user"); const char* fqdn = "localhost"; const char* firstEp = "localhost:9530"; - pServer = CreateServer("/tmp/dndTestUser", fqdn, 9530, firstEp); + pServer = CreateServer("/tmp/dnode_test_user", fqdn, 9530, firstEp); pClient = createClient("root", "taosdata", fqdn, 9530); taosMsleep(300); } @@ -298,7 +298,7 @@ TEST_F(DndTestUser, RestartDnode) { const char* fqdn = "localhost"; const char* firstEp = "localhost:9530"; - pServer = startServer("/tmp/dndTestUser", fqdn, 9530, firstEp); + pServer = startServer("/tmp/dnode_test_user", fqdn, 9530, firstEp); uInfo("all server is running"); From d68899e5e9d3b8be8c23b2c7422b97725a498fd3 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 15 Dec 2021 14:48:47 +0800 Subject: [PATCH 36/43] more work --- include/dnode/vnode/tsdb/tsdb.h | 10 +- include/util/tdef.h | 4 +- source/dnode/vnode/impl/CMakeLists.txt | 2 +- source/dnode/vnode/tsdb/inc/tsdbDef.h | 4 + source/dnode/vnode/tsdb/inc/tsdbMemTable.h | 9 +- source/dnode/vnode/tsdb/src/tsdbCommit.c | 21 + source/dnode/vnode/tsdb/src/tsdbMemTable.c | 55 +- source/dnode/vnode/tsdb/src/tsdbWrite.c | 25 + src/tsdb/inc/tsdbMemTable.h | 97 -- src/tsdb/src/tsdbBuffer.c | 214 ---- src/tsdb/src/tsdbMemTable.c | 1067 -------------------- 11 files changed, 117 insertions(+), 1391 deletions(-) create mode 100644 source/dnode/vnode/tsdb/src/tsdbCommit.c create mode 100644 source/dnode/vnode/tsdb/src/tsdbWrite.c delete mode 100644 src/tsdb/inc/tsdbMemTable.h delete mode 100644 src/tsdb/src/tsdbBuffer.c delete mode 100644 src/tsdb/src/tsdbMemTable.c diff --git a/include/dnode/vnode/tsdb/tsdb.h b/include/dnode/vnode/tsdb/tsdb.h index 26efce5d43..f8eac9768f 100644 --- a/include/dnode/vnode/tsdb/tsdb.h +++ b/include/dnode/vnode/tsdb/tsdb.h @@ -21,15 +21,14 @@ extern "C" { #endif // TYPES EXPOSED -typedef struct STsdb STsdb; -typedef struct STsdbCfg STsdbCfg; -typedef struct STsdbMemAllocator STsdbMemAllocator; +typedef struct STsdb STsdb; +typedef struct STsdbCfg STsdbCfg; // STsdb -STsdb *tsdbOpen(const char *path, const STsdbCfg *); +STsdb *tsdbOpen(const char *path, const STsdbCfg *pTsdbCfg); void tsdbClose(STsdb *); void tsdbRemove(const char *path); -int tsdbInsertData(STsdb *pTsdb, void *pData, int len); +int tsdbInsertData(STsdb *pTsdb, SSubmitMsg *pMsg); // STsdbCfg int tsdbOptionsInit(STsdbCfg *); @@ -41,7 +40,6 @@ struct STsdbCfg { uint32_t keep0; uint32_t keep1; uint32_t keep2; - /* TODO */ }; #ifdef __cplusplus diff --git a/include/util/tdef.h b/include/util/tdef.h index 300b190e97..5d6e238b6f 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -25,7 +25,9 @@ extern "C" { #define TSDB__packed #define TSKEY int64_t -#define TSKEY_INITIAL_VAL INT64_MIN +#define TSKEY_MIN INT64_MIN +#define TSKEY_MAX (INT64_MAX - 1) +#define TSKEY_INITIAL_VAL TSKEY_MIN // Bytes for each type. extern const int32_t TYPE_BYTES[15]; diff --git a/source/dnode/vnode/impl/CMakeLists.txt b/source/dnode/vnode/impl/CMakeLists.txt index dd01c94da7..6972605afd 100644 --- a/source/dnode/vnode/impl/CMakeLists.txt +++ b/source/dnode/vnode/impl/CMakeLists.txt @@ -19,5 +19,5 @@ target_link_libraries( # test if(${BUILD_TEST}) -# add_subdirectory(test) + add_subdirectory(test) endif(${BUILD_TEST}) \ No newline at end of file diff --git a/source/dnode/vnode/tsdb/inc/tsdbDef.h b/source/dnode/vnode/tsdb/inc/tsdbDef.h index 0a57f5f670..882a62e097 100644 --- a/source/dnode/vnode/tsdb/inc/tsdbDef.h +++ b/source/dnode/vnode/tsdb/inc/tsdbDef.h @@ -17,6 +17,8 @@ #define _TD_TSDB_DEF_H_ #include "mallocator.h" +#include "taosmsg.h" +#include "thash.h" #include "tsdb.h" #include "tsdbMemTable.h" @@ -29,6 +31,8 @@ extern "C" { struct STsdb { char * path; STsdbCfg options; + STsdbMemTable * mem; + STsdbMemTable * imem; SMemAllocatorFactory *pmaf; }; diff --git a/source/dnode/vnode/tsdb/inc/tsdbMemTable.h b/source/dnode/vnode/tsdb/inc/tsdbMemTable.h index 6be90ef53c..e7787af7cf 100644 --- a/source/dnode/vnode/tsdb/inc/tsdbMemTable.h +++ b/source/dnode/vnode/tsdb/inc/tsdbMemTable.h @@ -22,10 +22,11 @@ extern "C" { #endif -typedef struct SMemTable { - /* TODO */ - SMemAllocator *pma; -} SMemTable; +typedef struct STsdbMemTable STsdbMemTable; + +STsdbMemTable *tsdbNewMemTable(SMemAllocatorFactory *pMAF); +void tsdbFreeMemTable(SMemAllocatorFactory *pMAF, STsdbMemTable *pMemTable); +int tsdbInsertDataToMemTable(STsdbMemTable *pMemTable, SSubmitMsg *pMsg); #ifdef __cplusplus } diff --git a/source/dnode/vnode/tsdb/src/tsdbCommit.c b/source/dnode/vnode/tsdb/src/tsdbCommit.c new file mode 100644 index 0000000000..b124197736 --- /dev/null +++ b/source/dnode/vnode/tsdb/src/tsdbCommit.c @@ -0,0 +1,21 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tsdbDef.h" + +int tsdbCommit(STsdb *pTsdb) { + // TODO + return 0; +} \ No newline at end of file diff --git a/source/dnode/vnode/tsdb/src/tsdbMemTable.c b/source/dnode/vnode/tsdb/src/tsdbMemTable.c index 6dea4a4e57..e13b881ee5 100644 --- a/source/dnode/vnode/tsdb/src/tsdbMemTable.c +++ b/source/dnode/vnode/tsdb/src/tsdbMemTable.c @@ -11,4 +11,57 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . - */ \ No newline at end of file + */ + +#include "tsdbDef.h" + +struct STsdbMemTable { + T_REF_DECLARE() + SRWLatch latch; + TSKEY keyMin; + TSKEY keyMax; + uint32_t nRow; + SHashObj * pHash; + SMemAllocator *pMA; +}; + +STsdbMemTable *tsdbNewMemTable(SMemAllocatorFactory *pMAF) { + STsdbMemTable *pMemTable; + SMemAllocator *pMA; + + pMA = (*pMAF->create)(pMAF); + ASSERT(pMA != NULL); + + pMemTable = (STsdbMemTable *)((*pMA->malloc)(pMA, sizeof(*pMemTable))); + if (pMemTable == NULL) { + (*pMAF->destroy)(pMAF, pMA); + return NULL; + } + + T_REF_INIT_VAL(pMemTable, 1); + taosInitRWLatch(&(pMemTable->latch)); + pMemTable->keyMin = TSKEY_MAX; + pMemTable->keyMax = TSKEY_MIN; + pMemTable->nRow = 0; + pMemTable->pHash = NULL; /// TODO + pMemTable->pMA = pMA; + + // TODO + return pMemTable; +} + +void tsdbFreeMemTable(SMemAllocatorFactory *pMAF, STsdbMemTable *pMemTable) { + SMemAllocator *pMA = pMemTable->pMA; + + if (pMA->free) { + // TODO + ASSERT(0); + } + + (*pMAF->destroy)(pMAF, pMA); +} + +int tsdbInsertDataToMemTable(STsdbMemTable *pMemTable, SSubmitMsg *pMsg) { + // TODO + return 0; +} \ No newline at end of file diff --git a/source/dnode/vnode/tsdb/src/tsdbWrite.c b/source/dnode/vnode/tsdb/src/tsdbWrite.c new file mode 100644 index 0000000000..f9441cbe44 --- /dev/null +++ b/source/dnode/vnode/tsdb/src/tsdbWrite.c @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tsdbDef.h" + +int tsdbInsertData(STsdb *pTsdb, SSubmitMsg *pMsg) { + // Check if mem is there. If not, create one. + pTsdb->mem = tsdbNewMemTable(pTsdb->pmaf); + if (pTsdb->mem == NULL) { + return -1; + } + return tsdbInsertDataToMemTable(pTsdb->mem, pMsg); +} \ No newline at end of file diff --git a/src/tsdb/inc/tsdbMemTable.h b/src/tsdb/inc/tsdbMemTable.h deleted file mode 100644 index 67e9976c70..0000000000 --- a/src/tsdb/inc/tsdbMemTable.h +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_TSDB_MEMTABLE_H_ -#define _TD_TSDB_MEMTABLE_H_ - -typedef struct { - int rowsInserted; - int rowsUpdated; - int rowsDeleteSucceed; - int rowsDeleteFailed; - int nOperations; - TSKEY keyFirst; - TSKEY keyLast; -} SMergeInfo; - -typedef struct { - STable * pTable; - SSkipListIterator *pIter; -} SCommitIter; - -struct STableData { - uint64_t uid; - TSKEY keyFirst; - TSKEY keyLast; - int64_t numOfRows; - SSkipList* pData; - T_REF_DECLARE() -}; - -enum { TSDB_UPDATE_META, TSDB_DROP_META }; - -#ifdef WINDOWS -#pragma pack(push ,1) -typedef struct { -#else -typedef struct __attribute__((packed)){ -#endif - char act; - uint64_t uid; -} SActObj; -#ifdef WINDOWS -#pragma pack(pop) -#endif - -typedef struct { - int len; - char cont[]; -} SActCont; - -int tsdbRefMemTable(STsdbRepo* pRepo, SMemTable* pMemTable); -int tsdbUnRefMemTable(STsdbRepo* pRepo, SMemTable* pMemTable); -int tsdbTakeMemSnapshot(STsdbRepo* pRepo, SMemSnapshot* pSnapshot, SArray* pATable); -void tsdbUnTakeMemSnapShot(STsdbRepo* pRepo, SMemSnapshot* pSnapshot); -void* tsdbAllocBytes(STsdbRepo* pRepo, int bytes); -int tsdbAsyncCommit(STsdbRepo* pRepo); -int tsdbSyncCommitConfig(STsdbRepo* pRepo); -int tsdbLoadDataFromCache(STable* pTable, SSkipListIterator* pIter, TSKEY maxKey, int maxRowsToRead, SDataCols* pCols, - TKEY* filterKeys, int nFilterKeys, bool keepDup, SMergeInfo* pMergeInfo); -void* tsdbCommitData(STsdbRepo* pRepo); - -static FORCE_INLINE SMemRow tsdbNextIterRow(SSkipListIterator* pIter) { - if (pIter == NULL) return NULL; - - SSkipListNode* node = tSkipListIterGet(pIter); - if (node == NULL) return NULL; - - return (SMemRow)SL_GET_NODE_DATA(node); -} - -static FORCE_INLINE TSKEY tsdbNextIterKey(SSkipListIterator* pIter) { - SMemRow row = tsdbNextIterRow(pIter); - if (row == NULL) return TSDB_DATA_TIMESTAMP_NULL; - - return memRowKey(row); -} - -static FORCE_INLINE TKEY tsdbNextIterTKey(SSkipListIterator* pIter) { - SMemRow row = tsdbNextIterRow(pIter); - if (row == NULL) return TKEY_NULL; - - return memRowTKey(row); -} - -#endif /* _TD_TSDB_MEMTABLE_H_ */ \ No newline at end of file diff --git a/src/tsdb/src/tsdbBuffer.c b/src/tsdb/src/tsdbBuffer.c deleted file mode 100644 index 70589031f6..0000000000 --- a/src/tsdb/src/tsdbBuffer.c +++ /dev/null @@ -1,214 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "tsdbint.h" -#include "tsdbHealth.h" - -#define POOL_IS_EMPTY(b) (listNEles((b)->bufBlockList) == 0) - -// ---------------- INTERNAL FUNCTIONS ---------------- -STsdbBufPool *tsdbNewBufPool() { - STsdbBufPool *pBufPool = (STsdbBufPool *)calloc(1, sizeof(*pBufPool)); - if (pBufPool == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - goto _err; - } - - int code = pthread_cond_init(&(pBufPool->poolNotEmpty), NULL); - if (code != 0) { - terrno = TAOS_SYSTEM_ERROR(code); - goto _err; - } - - pBufPool->bufBlockList = tdListNew(sizeof(STsdbBufBlock *)); - if (pBufPool->bufBlockList == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - goto _err; - } - - return pBufPool; - -_err: - tsdbFreeBufPool(pBufPool); - return NULL; -} - -void tsdbFreeBufPool(STsdbBufPool *pBufPool) { - if (pBufPool) { - if (pBufPool->bufBlockList) { - ASSERT(listNEles(pBufPool->bufBlockList) == 0); - tdListFree(pBufPool->bufBlockList); - } - - pthread_cond_destroy(&pBufPool->poolNotEmpty); - - free(pBufPool); - } -} - -int tsdbOpenBufPool(STsdbRepo *pRepo) { - STsdbCfg * pCfg = &(pRepo->config); - STsdbBufPool *pPool = pRepo->pPool; - - ASSERT(pPool != NULL); - pPool->bufBlockSize = pCfg->cacheBlockSize * 1024 * 1024; // MB - pPool->tBufBlocks = pCfg->totalBlocks; - pPool->nBufBlocks = 0; - pPool->nElasticBlocks = 0; - pPool->index = 0; - pPool->nRecycleBlocks = 0; - - for (int i = 0; i < pCfg->totalBlocks; i++) { - STsdbBufBlock *pBufBlock = tsdbNewBufBlock(pPool->bufBlockSize); - if (pBufBlock == NULL) goto _err; - - if (tdListAppend(pPool->bufBlockList, (void *)(&pBufBlock)) < 0) { - tsdbFreeBufBlock(pBufBlock); - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - goto _err; - } - - pPool->nBufBlocks++; - } - - tsdbDebug("vgId:%d buffer pool is opened! bufBlockSize:%d tBufBlocks:%d nBufBlocks:%d", REPO_ID(pRepo), - pPool->bufBlockSize, pPool->tBufBlocks, pPool->nBufBlocks); - - return 0; - -_err: - tsdbCloseBufPool(pRepo); - return -1; -} - -void tsdbCloseBufPool(STsdbRepo *pRepo) { - if (pRepo == NULL) return; - - STsdbBufPool * pBufPool = pRepo->pPool; - STsdbBufBlock *pBufBlock = NULL; - - if (pBufPool) { - SListNode *pNode = NULL; - while ((pNode = tdListPopHead(pBufPool->bufBlockList)) != NULL) { - tdListNodeGetData(pBufPool->bufBlockList, pNode, (void *)(&pBufBlock)); - tsdbFreeBufBlock(pBufBlock); - free(pNode); - } - } - - tsdbDebug("vgId:%d, buffer pool is closed", REPO_ID(pRepo)); -} - -SListNode *tsdbAllocBufBlockFromPool(STsdbRepo *pRepo) { - ASSERT(pRepo != NULL && pRepo->pPool != NULL); - ASSERT(IS_REPO_LOCKED(pRepo)); - - STsdbBufPool *pBufPool = pRepo->pPool; - - while (POOL_IS_EMPTY(pBufPool)) { - if(tsDeadLockKillQuery) { - // supply new Block - if(tsdbInsertNewBlock(pRepo) > 0) { - tsdbWarn("vgId:%d add new elastic block . elasticBlocks=%d cur free Blocks=%d", REPO_ID(pRepo), pBufPool->nElasticBlocks, pBufPool->bufBlockList->numOfEles); - break; - } else { - // no newBlock, kill query free - if(!tsdbUrgeQueryFree(pRepo)) - tsdbWarn("vgId:%d Urge query free thread start failed.", REPO_ID(pRepo)); - } - } - - pRepo->repoLocked = false; - pthread_cond_wait(&(pBufPool->poolNotEmpty), &(pRepo->mutex)); - pRepo->repoLocked = true; - } - - SListNode * pNode = tdListPopHead(pBufPool->bufBlockList); - ASSERT(pNode != NULL); - STsdbBufBlock *pBufBlock = NULL; - tdListNodeGetData(pBufPool->bufBlockList, pNode, (void *)(&pBufBlock)); - - pBufBlock->blockId = pBufPool->index++; - pBufBlock->offset = 0; - pBufBlock->remain = pBufPool->bufBlockSize; - - tsdbDebug("vgId:%d, buffer block is allocated, blockId:%" PRId64, REPO_ID(pRepo), pBufBlock->blockId); - return pNode; -} - -// ---------------- LOCAL FUNCTIONS ---------------- -STsdbBufBlock *tsdbNewBufBlock(int bufBlockSize) { - STsdbBufBlock *pBufBlock = (STsdbBufBlock *)malloc(sizeof(*pBufBlock) + bufBlockSize); - if (pBufBlock == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return NULL; - } - - pBufBlock->blockId = 0; - pBufBlock->offset = 0; - pBufBlock->remain = bufBlockSize; - - return pBufBlock; -} - - void tsdbFreeBufBlock(STsdbBufBlock *pBufBlock) { tfree(pBufBlock); } - -int tsdbExpandPool(STsdbRepo* pRepo, int32_t oldTotalBlocks) { - if (oldTotalBlocks == pRepo->config.totalBlocks) { - return TSDB_CODE_SUCCESS; - } - - int err = TSDB_CODE_SUCCESS; - - if (tsdbLockRepo(pRepo) < 0) return terrno; - STsdbBufPool* pPool = pRepo->pPool; - - if (pRepo->config.totalBlocks > oldTotalBlocks) { - for (int i = 0; i < pRepo->config.totalBlocks - oldTotalBlocks; i++) { - STsdbBufBlock *pBufBlock = tsdbNewBufBlock(pPool->bufBlockSize); - if (pBufBlock == NULL) goto err; - - if (tdListAppend(pPool->bufBlockList, (void *)(&pBufBlock)) < 0) { - tsdbFreeBufBlock(pBufBlock); - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - err = TSDB_CODE_TDB_OUT_OF_MEMORY; - goto err; - } - - pPool->nBufBlocks++; - } - pthread_cond_signal(&pPool->poolNotEmpty); - } else { - pPool->nRecycleBlocks = oldTotalBlocks - pRepo->config.totalBlocks; - } - -err: - tsdbUnlockRepo(pRepo); - return err; -} - -void tsdbRecycleBufferBlock(STsdbBufPool* pPool, SListNode *pNode, bool bELastic) { - STsdbBufBlock *pBufBlock = NULL; - tdListNodeGetData(pPool->bufBlockList, pNode, (void *)(&pBufBlock)); - tsdbFreeBufBlock(pBufBlock); - free(pNode); - if(bELastic) - { - pPool->nElasticBlocks--; - tsdbWarn("pPool=%p elastic block reduce one . nElasticBlocks=%d cur free Blocks=%d", pPool, pPool->nElasticBlocks, pPool->bufBlockList->numOfEles); - } - else - pPool->nBufBlocks--; -} \ No newline at end of file diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c deleted file mode 100644 index 3890dca5b9..0000000000 --- a/src/tsdb/src/tsdbMemTable.c +++ /dev/null @@ -1,1067 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "tdataformat.h" -#include "tfunctional.h" -#include "tsdbint.h" -#include "tskiplist.h" -#include "tsdbRowMergeBuf.h" - -#define TSDB_DATA_SKIPLIST_LEVEL 5 -#define TSDB_MAX_INSERT_BATCH 512 - -typedef struct { - int32_t totalLen; - int32_t len; - SMemRow row; -} SSubmitBlkIter; - -typedef struct { - int32_t totalLen; - int32_t len; - void * pMsg; -} SSubmitMsgIter; - -static SMemTable * tsdbNewMemTable(STsdbRepo *pRepo); -static void tsdbFreeMemTable(SMemTable *pMemTable); -static STableData* tsdbNewTableData(STsdbCfg *pCfg, STable *pTable); -static void tsdbFreeTableData(STableData *pTableData); -static char * tsdbGetTsTupleKey(const void *data); -static int tsdbAdjustMemMaxTables(SMemTable *pMemTable, int maxTables); -static int tsdbAppendTableRowToCols(STable *pTable, SDataCols *pCols, STSchema **ppSchema, SMemRow row); -static int tsdbInitSubmitBlkIter(SSubmitBlk *pBlock, SSubmitBlkIter *pIter); -static SMemRow tsdbGetSubmitBlkNext(SSubmitBlkIter *pIter); -static int tsdbScanAndConvertSubmitMsg(STsdbRepo *pRepo, SSubmitMsg *pMsg); -static int tsdbInsertDataToTable(STsdbRepo *pRepo, SSubmitBlk *pBlock, int32_t *affectedrows); -static int tsdbInitSubmitMsgIter(SSubmitMsg *pMsg, SSubmitMsgIter *pIter); -static int tsdbGetSubmitMsgNext(SSubmitMsgIter *pIter, SSubmitBlk **pPBlock); -static int tsdbCheckTableSchema(STsdbRepo *pRepo, SSubmitBlk *pBlock, STable *pTable); -static int tsdbUpdateTableLatestInfo(STsdbRepo *pRepo, STable *pTable, SMemRow row); - -static FORCE_INLINE int tsdbCheckRowRange(STsdbRepo *pRepo, STable *pTable, SMemRow row, TSKEY minKey, TSKEY maxKey, - TSKEY now); - -int32_t tsdbInsertData(STsdbRepo *repo, SSubmitMsg *pMsg, SShellSubmitRspMsg *pRsp) { - STsdbRepo * pRepo = repo; - SSubmitMsgIter msgIter = {0}; - SSubmitBlk * pBlock = NULL; - int32_t affectedrows = 0; - - if (tsdbScanAndConvertSubmitMsg(pRepo, pMsg) < 0) { - if (terrno != TSDB_CODE_TDB_TABLE_RECONFIGURE) { - tsdbError("vgId:%d failed to insert data since %s", REPO_ID(pRepo), tstrerror(terrno)); - } - return -1; - } - - tsdbInitSubmitMsgIter(pMsg, &msgIter); - while (true) { - tsdbGetSubmitMsgNext(&msgIter, &pBlock); - if (pBlock == NULL) break; - if (tsdbInsertDataToTable(pRepo, pBlock, &affectedrows) < 0) { - return -1; - } - } - - if (pRsp != NULL) pRsp->affectedRows = htonl(affectedrows); - - if (tsdbCheckCommit(pRepo) < 0) return -1; - return 0; -} - -// ---------------- INTERNAL FUNCTIONS ---------------- -int tsdbRefMemTable(STsdbRepo *pRepo, SMemTable *pMemTable) { - if (pMemTable == NULL) return 0; - int ref = T_REF_INC(pMemTable); - tsdbDebug("vgId:%d ref memtable %p ref %d", REPO_ID(pRepo), pMemTable, ref); - return 0; -} - -// Need to lock the repository -int tsdbUnRefMemTable(STsdbRepo *pRepo, SMemTable *pMemTable) { - if (pMemTable == NULL) return 0; - - int ref = T_REF_DEC(pMemTable); - tsdbDebug("vgId:%d unref memtable %p ref %d", REPO_ID(pRepo), pMemTable, ref); - if (ref == 0) { - STsdbBufPool *pBufPool = pRepo->pPool; - - SListNode *pNode = NULL; - bool addNew = false; - if (tsdbLockRepo(pRepo) < 0) return -1; - while ((pNode = tdListPopHead(pMemTable->bufBlockList)) != NULL) { - if (pBufPool->nRecycleBlocks > 0) { - tsdbRecycleBufferBlock(pBufPool, pNode, false); - pBufPool->nRecycleBlocks -= 1; - } else { - if(pBufPool->nElasticBlocks > 0 && listNEles(pBufPool->bufBlockList) > 2) { - tsdbRecycleBufferBlock(pBufPool, pNode, true); - } else { - tdListAppendNode(pBufPool->bufBlockList, pNode); - addNew = true; - } - } - } - if (addNew) { - int code = pthread_cond_signal(&pBufPool->poolNotEmpty); - if (code != 0) { - if (tsdbUnlockRepo(pRepo) < 0) return -1; - tsdbError("vgId:%d failed to signal pool not empty since %s", REPO_ID(pRepo), strerror(code)); - terrno = TAOS_SYSTEM_ERROR(code); - return -1; - } - } - - if (tsdbUnlockRepo(pRepo) < 0) return -1; - - for (int i = 0; i < pMemTable->maxTables; i++) { - if (pMemTable->tData[i] != NULL) { - tsdbFreeTableData(pMemTable->tData[i]); - } - } - - tdListDiscard(pMemTable->actList); - tdListDiscard(pMemTable->bufBlockList); - tsdbFreeMemTable(pMemTable); - } - return 0; -} - -int tsdbTakeMemSnapshot(STsdbRepo *pRepo, SMemSnapshot *pSnapshot, SArray *pATable) { - memset(pSnapshot, 0, sizeof(*pSnapshot)); - - if (tsdbLockRepo(pRepo) < 0) return -1; - - pSnapshot->omem = pRepo->mem; - pSnapshot->imem = pRepo->imem; - tsdbRefMemTable(pRepo, pRepo->mem); - tsdbRefMemTable(pRepo, pRepo->imem); - - if (tsdbUnlockRepo(pRepo) < 0) return -1; - - if (pSnapshot->omem) { - taosRLockLatch(&(pSnapshot->omem->latch)); - - pSnapshot->mem = &(pSnapshot->mtable); - - pSnapshot->mem->tData = (STableData **)calloc(pSnapshot->omem->maxTables, sizeof(STableData *)); - if (pSnapshot->mem->tData == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - taosRUnLockLatch(&(pSnapshot->omem->latch)); - tsdbUnRefMemTable(pRepo, pSnapshot->omem); - tsdbUnRefMemTable(pRepo, pSnapshot->imem); - pSnapshot->mem = NULL; - pSnapshot->imem = NULL; - pSnapshot->omem = NULL; - return -1; - } - - pSnapshot->mem->keyFirst = pSnapshot->omem->keyFirst; - pSnapshot->mem->keyLast = pSnapshot->omem->keyLast; - pSnapshot->mem->numOfRows = pSnapshot->omem->numOfRows; - pSnapshot->mem->maxTables = pSnapshot->omem->maxTables; - - for (size_t i = 0; i < taosArrayGetSize(pATable); i++) { - STable * pTable = *(STable **)taosArrayGet(pATable, i); - int32_t tid = TABLE_TID(pTable); - STableData *pTableData = (tid < pSnapshot->omem->maxTables) ? pSnapshot->omem->tData[tid] : NULL; - - if ((pTableData == NULL) || (TABLE_UID(pTable) != pTableData->uid)) continue; - - pSnapshot->mem->tData[tid] = pTableData; - T_REF_INC(pTableData); - } - - taosRUnLockLatch(&(pSnapshot->omem->latch)); - } - - tsdbDebug("vgId:%d take memory snapshot, pMem %p pIMem %p", REPO_ID(pRepo), pSnapshot->omem, pSnapshot->imem); - return 0; -} - -void tsdbUnTakeMemSnapShot(STsdbRepo *pRepo, SMemSnapshot *pSnapshot) { - tsdbDebug("vgId:%d untake memory snapshot, pMem %p pIMem %p", REPO_ID(pRepo), pSnapshot->omem, pSnapshot->imem); - - if (pSnapshot->mem) { - ASSERT(pSnapshot->omem != NULL); - - for (size_t i = 0; i < pSnapshot->mem->maxTables; i++) { - STableData *pTableData = pSnapshot->mem->tData[i]; - if (pTableData) { - tsdbFreeTableData(pTableData); - } - } - tfree(pSnapshot->mem->tData); - - tsdbUnRefMemTable(pRepo, pSnapshot->omem); - } - - tsdbUnRefMemTable(pRepo, pSnapshot->imem); - - pSnapshot->mem = NULL; - pSnapshot->imem = NULL; - pSnapshot->omem = NULL; -} - -void *tsdbAllocBytes(STsdbRepo *pRepo, int bytes) { - STsdbCfg * pCfg = &pRepo->config; - STsdbBufBlock *pBufBlock = NULL; - void * ptr = NULL; - - // Either allocate from buffer blocks or from SYSTEM memory pool - if (pRepo->mem == NULL) { - SMemTable *pMemTable = tsdbNewMemTable(pRepo); - if (pMemTable == NULL) return NULL; - pRepo->mem = pMemTable; - } - - ASSERT(pRepo->mem != NULL); - - pBufBlock = tsdbGetCurrBufBlock(pRepo); - if ((pRepo->mem->extraBuffList != NULL) || - ((listNEles(pRepo->mem->bufBlockList) >= pCfg->totalBlocks / 3) && (pBufBlock->remain < bytes))) { - // allocate from SYSTEM buffer pool - if (pRepo->mem->extraBuffList == NULL) { - pRepo->mem->extraBuffList = tdListNew(0); - if (pRepo->mem->extraBuffList == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return NULL; - } - } - - ASSERT(pRepo->mem->extraBuffList != NULL); - SListNode *pNode = (SListNode *)malloc(sizeof(SListNode) + bytes); - if (pNode == NULL) { - if (listNEles(pRepo->mem->extraBuffList) == 0) { - tdListFree(pRepo->mem->extraBuffList); - pRepo->mem->extraBuffList = NULL; - } - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return NULL; - } - - pNode->next = pNode->prev = NULL; - tdListAppendNode(pRepo->mem->extraBuffList, pNode); - ptr = (void *)(pNode->data); - tsdbTrace("vgId:%d allocate %d bytes from SYSTEM buffer block", REPO_ID(pRepo), bytes); - } else { // allocate from TSDB buffer pool - if (pBufBlock == NULL || pBufBlock->remain < bytes) { - ASSERT(listNEles(pRepo->mem->bufBlockList) < pCfg->totalBlocks / 3); - if (tsdbLockRepo(pRepo) < 0) return NULL; - SListNode *pNode = tsdbAllocBufBlockFromPool(pRepo); - tdListAppendNode(pRepo->mem->bufBlockList, pNode); - if (tsdbUnlockRepo(pRepo) < 0) return NULL; - pBufBlock = tsdbGetCurrBufBlock(pRepo); - } - - ASSERT(pBufBlock->remain >= bytes); - ptr = POINTER_SHIFT(pBufBlock->data, pBufBlock->offset); - pBufBlock->offset += bytes; - pBufBlock->remain -= bytes; - tsdbTrace("vgId:%d allocate %d bytes from TSDB buffer block, nBlocks %d offset %d remain %d", REPO_ID(pRepo), bytes, - listNEles(pRepo->mem->bufBlockList), pBufBlock->offset, pBufBlock->remain); - } - - return ptr; -} - -int tsdbSyncCommitConfig(STsdbRepo* pRepo) { - ASSERT(pRepo->config_changed == true); - tsem_wait(&(pRepo->readyToCommit)); - - if (pRepo->code != TSDB_CODE_SUCCESS) { - tsdbWarn("vgId:%d try to commit config when TSDB not in good state: %s", REPO_ID(pRepo), tstrerror(terrno)); - } - - if (tsdbLockRepo(pRepo) < 0) return -1; - tsdbScheduleCommit(pRepo, COMMIT_CONFIG_REQ); - if (tsdbUnlockRepo(pRepo) < 0) return -1; - - tsem_wait(&(pRepo->readyToCommit)); - tsem_post(&(pRepo->readyToCommit)); - - if (pRepo->code != TSDB_CODE_SUCCESS) { - terrno = pRepo->code; - return -1; - } - - terrno = TSDB_CODE_SUCCESS; - return 0; -} - -int tsdbAsyncCommit(STsdbRepo *pRepo) { - tsem_wait(&(pRepo->readyToCommit)); - - ASSERT(pRepo->imem == NULL); - if (pRepo->mem == NULL) { - tsem_post(&(pRepo->readyToCommit)); - return 0; - } - - if (pRepo->code != TSDB_CODE_SUCCESS) { - tsdbWarn("vgId:%d try to commit when TSDB not in good state: %s", REPO_ID(pRepo), tstrerror(terrno)); - } - - if (pRepo->appH.notifyStatus) pRepo->appH.notifyStatus(pRepo->appH.appH, TSDB_STATUS_COMMIT_START, TSDB_CODE_SUCCESS); - if (tsdbLockRepo(pRepo) < 0) return -1; - pRepo->imem = pRepo->mem; - pRepo->mem = NULL; - tsdbScheduleCommit(pRepo, COMMIT_REQ); - if (tsdbUnlockRepo(pRepo) < 0) return -1; - - return 0; -} - -int tsdbSyncCommit(STsdbRepo *repo) { - STsdbRepo *pRepo = repo; - - tsdbAsyncCommit(pRepo); - tsem_wait(&(pRepo->readyToCommit)); - tsem_post(&(pRepo->readyToCommit)); - - if (pRepo->code != TSDB_CODE_SUCCESS) { - terrno = pRepo->code; - return -1; - } else { - terrno = TSDB_CODE_SUCCESS; - return 0; - } -} - -/** - * This is an important function to load data or try to load data from memory skiplist iterator. - * - * This function load memory data until: - * 1. iterator ends - * 2. data key exceeds maxKey - * 3. rowsIncreased = rowsInserted - rowsDeleteSucceed >= maxRowsToRead - * 4. operations in pCols not exceeds its max capacity if pCols is given - * - * The function tries to procceed AS MUCH AS POSSIBLE. - */ -int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey, int maxRowsToRead, SDataCols *pCols, - TKEY *filterKeys, int nFilterKeys, bool keepDup, SMergeInfo *pMergeInfo) { - ASSERT(maxRowsToRead > 0 && nFilterKeys >= 0); - if (pIter == NULL) return 0; - STSchema * pSchema = NULL; - TSKEY rowKey = 0; - TSKEY fKey = 0; - bool isRowDel = false; - int filterIter = 0; - SMemRow row = NULL; - SMergeInfo mInfo; - - if (pMergeInfo == NULL) pMergeInfo = &mInfo; - - memset(pMergeInfo, 0, sizeof(*pMergeInfo)); - pMergeInfo->keyFirst = INT64_MAX; - pMergeInfo->keyLast = INT64_MIN; - if (pCols) tdResetDataCols(pCols); - - row = tsdbNextIterRow(pIter); - if (row == NULL || memRowKey(row) > maxKey) { - rowKey = INT64_MAX; - isRowDel = false; - } else { - rowKey = memRowKey(row); - isRowDel = memRowDeleted(row); - } - - if (filterIter >= nFilterKeys) { - fKey = INT64_MAX; - } else { - fKey = tdGetKey(filterKeys[filterIter]); - } - - while (true) { - if (fKey == INT64_MAX && rowKey == INT64_MAX) break; - - if (fKey < rowKey) { - pMergeInfo->keyFirst = MIN(pMergeInfo->keyFirst, fKey); - pMergeInfo->keyLast = MAX(pMergeInfo->keyLast, fKey); - - filterIter++; - if (filterIter >= nFilterKeys) { - fKey = INT64_MAX; - } else { - fKey = tdGetKey(filterKeys[filterIter]); - } - } else if (fKey > rowKey) { - if (isRowDel) { - pMergeInfo->rowsDeleteFailed++; - } else { - if (pMergeInfo->rowsInserted - pMergeInfo->rowsDeleteSucceed >= maxRowsToRead) break; - if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break; - pMergeInfo->rowsInserted++; - pMergeInfo->nOperations++; - pMergeInfo->keyFirst = MIN(pMergeInfo->keyFirst, rowKey); - pMergeInfo->keyLast = MAX(pMergeInfo->keyLast, rowKey); - tsdbAppendTableRowToCols(pTable, pCols, &pSchema, row); - } - - tSkipListIterNext(pIter); - row = tsdbNextIterRow(pIter); - if (row == NULL || memRowKey(row) > maxKey) { - rowKey = INT64_MAX; - isRowDel = false; - } else { - rowKey = memRowKey(row); - isRowDel = memRowDeleted(row); - } - } else { - if (isRowDel) { - ASSERT(!keepDup); - if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break; - pMergeInfo->rowsDeleteSucceed++; - pMergeInfo->nOperations++; - tsdbAppendTableRowToCols(pTable, pCols, &pSchema, row); - } else { - if (keepDup) { - if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break; - pMergeInfo->rowsUpdated++; - pMergeInfo->nOperations++; - pMergeInfo->keyFirst = MIN(pMergeInfo->keyFirst, rowKey); - pMergeInfo->keyLast = MAX(pMergeInfo->keyLast, rowKey); - tsdbAppendTableRowToCols(pTable, pCols, &pSchema, row); - } else { - pMergeInfo->keyFirst = MIN(pMergeInfo->keyFirst, fKey); - pMergeInfo->keyLast = MAX(pMergeInfo->keyLast, fKey); - } - } - - tSkipListIterNext(pIter); - row = tsdbNextIterRow(pIter); - if (row == NULL || memRowKey(row) > maxKey) { - rowKey = INT64_MAX; - isRowDel = false; - } else { - rowKey = memRowKey(row); - isRowDel = memRowDeleted(row); - } - - filterIter++; - if (filterIter >= nFilterKeys) { - fKey = INT64_MAX; - } else { - fKey = tdGetKey(filterKeys[filterIter]); - } - } - } - - return 0; -} - -// ---------------- LOCAL FUNCTIONS ---------------- -static SMemTable* tsdbNewMemTable(STsdbRepo *pRepo) { - STsdbMeta *pMeta = pRepo->tsdbMeta; - - SMemTable *pMemTable = (SMemTable *)calloc(1, sizeof(*pMemTable)); - if (pMemTable == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - goto _err; - } - - pMemTable->keyFirst = INT64_MAX; - pMemTable->keyLast = 0; - pMemTable->numOfRows = 0; - - pMemTable->maxTables = pMeta->maxTables; - pMemTable->tData = (STableData **)calloc(pMemTable->maxTables, sizeof(STableData *)); - if (pMemTable->tData == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - goto _err; - } - - pMemTable->actList = tdListNew(0); - if (pMemTable->actList == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - goto _err; - } - - pMemTable->bufBlockList = tdListNew(sizeof(STsdbBufBlock*)); - if (pMemTable->bufBlockList == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - goto _err; - } - - T_REF_INC(pMemTable); - - return pMemTable; - -_err: - tsdbFreeMemTable(pMemTable); - return NULL; -} - -static void tsdbFreeMemTable(SMemTable* pMemTable) { - if (pMemTable) { - ASSERT((pMemTable->bufBlockList == NULL) ? true : (listNEles(pMemTable->bufBlockList) == 0)); - ASSERT((pMemTable->actList == NULL) ? true : (listNEles(pMemTable->actList) == 0)); - - tdListFree(pMemTable->extraBuffList); - tdListFree(pMemTable->bufBlockList); - tdListFree(pMemTable->actList); - tfree(pMemTable->tData); - free(pMemTable); - } -} - -static STableData *tsdbNewTableData(STsdbCfg *pCfg, STable *pTable) { - STableData *pTableData = (STableData *)calloc(1, sizeof(*pTableData)); - if (pTableData == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return NULL; - } - - pTableData->uid = TABLE_UID(pTable); - pTableData->keyFirst = INT64_MAX; - pTableData->keyLast = 0; - pTableData->numOfRows = 0; - - uint8_t skipListCreateFlags; - if(pCfg->update == TD_ROW_DISCARD_UPDATE) - skipListCreateFlags = SL_DISCARD_DUP_KEY; - else - skipListCreateFlags = SL_UPDATE_DUP_KEY; - - pTableData->pData = - tSkipListCreate(TSDB_DATA_SKIPLIST_LEVEL, TSDB_DATA_TYPE_TIMESTAMP, TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP], - tkeyComparFn, skipListCreateFlags, tsdbGetTsTupleKey); - if (pTableData->pData == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - free(pTableData); - return NULL; - } - - T_REF_INC(pTableData); - - return pTableData; -} - -static void tsdbFreeTableData(STableData *pTableData) { - if (pTableData) { - int32_t ref = T_REF_DEC(pTableData); - if (ref == 0) { - tSkipListDestroy(pTableData->pData); - free(pTableData); - } - } -} - -static char *tsdbGetTsTupleKey(const void *data) { return memRowTuple((SMemRow)data); } - -static int tsdbAdjustMemMaxTables(SMemTable *pMemTable, int maxTables) { - ASSERT(pMemTable->maxTables < maxTables); - - STableData **pTableData = (STableData **)calloc(maxTables, sizeof(STableData *)); - if (pTableData == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; - } - memcpy((void *)pTableData, (void *)pMemTable->tData, sizeof(STableData *) * pMemTable->maxTables); - - STableData **tData = pMemTable->tData; - - taosWLockLatch(&(pMemTable->latch)); - pMemTable->maxTables = maxTables; - pMemTable->tData = pTableData; - taosWUnLockLatch(&(pMemTable->latch)); - - tfree(tData); - - return 0; -} - -static int tsdbAppendTableRowToCols(STable *pTable, SDataCols *pCols, STSchema **ppSchema, SMemRow row) { - if (pCols) { - if (*ppSchema == NULL || schemaVersion(*ppSchema) != memRowVersion(row)) { - *ppSchema = tsdbGetTableSchemaImpl(pTable, false, false, memRowVersion(row)); - if (*ppSchema == NULL) { - ASSERT(false); - return -1; - } - } - - tdAppendMemRowToDataCol(row, *ppSchema, pCols, true); - } - - return 0; -} - -static int tsdbInitSubmitBlkIter(SSubmitBlk *pBlock, SSubmitBlkIter *pIter) { - if (pBlock->dataLen <= 0) return -1; - pIter->totalLen = pBlock->dataLen; - pIter->len = 0; - pIter->row = (SMemRow)(pBlock->data + pBlock->schemaLen); - return 0; -} - -static SMemRow tsdbGetSubmitBlkNext(SSubmitBlkIter *pIter) { - SMemRow row = pIter->row; // firstly, get current row - if (row == NULL) return NULL; - - pIter->len += memRowTLen(row); - if (pIter->len >= pIter->totalLen) { // reach the end - pIter->row = NULL; - } else { - pIter->row = (char *)row + memRowTLen(row); // secondly, move to next row - } - - return row; -} - -static FORCE_INLINE int tsdbCheckRowRange(STsdbRepo *pRepo, STable *pTable, SMemRow row, TSKEY minKey, TSKEY maxKey, - TSKEY now) { - TSKEY rowKey = memRowKey(row); - if (rowKey < minKey || rowKey > maxKey) { - tsdbError("vgId:%d table %s tid %d uid %" PRIu64 " timestamp is out of range! now %" PRId64 " minKey %" PRId64 - " maxKey %" PRId64 " row key %" PRId64, - REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), TABLE_UID(pTable), now, minKey, maxKey, - rowKey); - terrno = TSDB_CODE_TDB_TIMESTAMP_OUT_OF_RANGE; - return -1; - } - - return 0; -} - -static int tsdbScanAndConvertSubmitMsg(STsdbRepo *pRepo, SSubmitMsg *pMsg) { - ASSERT(pMsg != NULL); - STsdbMeta * pMeta = pRepo->tsdbMeta; - SSubmitMsgIter msgIter = {0}; - SSubmitBlk * pBlock = NULL; - SSubmitBlkIter blkIter = {0}; - SMemRow row = NULL; - TSKEY now = taosGetTimestamp(pRepo->config.precision); - TSKEY minKey = now - tsTickPerDay[pRepo->config.precision] * pRepo->config.keep; - TSKEY maxKey = now + tsTickPerDay[pRepo->config.precision] * pRepo->config.daysPerFile; - - terrno = TSDB_CODE_SUCCESS; - pMsg->length = htonl(pMsg->length); - pMsg->numOfBlocks = htonl(pMsg->numOfBlocks); - - if (tsdbInitSubmitMsgIter(pMsg, &msgIter) < 0) return -1; - while (true) { - if (tsdbGetSubmitMsgNext(&msgIter, &pBlock) < 0) return -1; - if (pBlock == NULL) break; - - pBlock->uid = htobe64(pBlock->uid); - pBlock->tid = htonl(pBlock->tid); - pBlock->sversion = htonl(pBlock->sversion); - pBlock->dataLen = htonl(pBlock->dataLen); - pBlock->schemaLen = htonl(pBlock->schemaLen); - pBlock->numOfRows = htons(pBlock->numOfRows); - - if (pBlock->tid <= 0 || pBlock->tid >= pMeta->maxTables) { - tsdbError("vgId:%d failed to get table to insert data, uid %" PRIu64 " tid %d", REPO_ID(pRepo), pBlock->uid, - pBlock->tid); - terrno = TSDB_CODE_TDB_INVALID_TABLE_ID; - return -1; - } - - STable *pTable = pMeta->tables[pBlock->tid]; - if (pTable == NULL || TABLE_UID(pTable) != pBlock->uid) { - tsdbError("vgId:%d failed to get table to insert data, uid %" PRIu64 " tid %d", REPO_ID(pRepo), pBlock->uid, - pBlock->tid); - terrno = TSDB_CODE_TDB_INVALID_TABLE_ID; - return -1; - } - - if (TABLE_TYPE(pTable) == TSDB_SUPER_TABLE) { - tsdbError("vgId:%d invalid action trying to insert a super table %s", REPO_ID(pRepo), TABLE_CHAR_NAME(pTable)); - terrno = TSDB_CODE_TDB_INVALID_ACTION; - return -1; - } - - // Check schema version and update schema if needed - if (tsdbCheckTableSchema(pRepo, pBlock, pTable) < 0) { - if (terrno == TSDB_CODE_TDB_TABLE_RECONFIGURE) { - continue; - } else { - return -1; - } - } - - tsdbInitSubmitBlkIter(pBlock, &blkIter); - while ((row = tsdbGetSubmitBlkNext(&blkIter)) != NULL) { - if (tsdbCheckRowRange(pRepo, pTable, row, minKey, maxKey, now) < 0) { - return -1; - } - } - } - - if (terrno != TSDB_CODE_SUCCESS) return -1; - return 0; -} - -//row1 has higher priority -static SMemRow tsdbInsertDupKeyMerge(SMemRow row1, SMemRow row2, STsdbRepo* pRepo, - STSchema **ppSchema1, STSchema **ppSchema2, - STable* pTable, int32_t* pPoints, SMemRow* pLastRow) { - - //for compatiblity, duplicate key inserted when update=0 should be also calculated as affected rows! - if(row1 == NULL && row2 == NULL && pRepo->config.update == TD_ROW_DISCARD_UPDATE) { - (*pPoints)++; - return NULL; - } - - if(row2 == NULL || pRepo->config.update != TD_ROW_PARTIAL_UPDATE) { - void* pMem = tsdbAllocBytes(pRepo, memRowTLen(row1)); - if(pMem == NULL) return NULL; - memRowCpy(pMem, row1); - (*pPoints)++; - *pLastRow = pMem; - return pMem; - } - - STSchema *pSchema1 = *ppSchema1; - STSchema *pSchema2 = *ppSchema2; - SMergeBuf * pBuf = &pRepo->mergeBuf; - int dv1 = memRowVersion(row1); - int dv2 = memRowVersion(row2); - if(pSchema1 == NULL || schemaVersion(pSchema1) != dv1) { - if(pSchema2 != NULL && schemaVersion(pSchema2) == dv1) { - *ppSchema1 = pSchema2; - } else { - *ppSchema1 = tsdbGetTableSchemaImpl(pTable, false, false, memRowVersion(row1)); - } - pSchema1 = *ppSchema1; - } - - if(pSchema2 == NULL || schemaVersion(pSchema2) != dv2) { - if(schemaVersion(pSchema1) == dv2) { - pSchema2 = pSchema1; - } else { - *ppSchema2 = tsdbGetTableSchemaImpl(pTable, false, false, memRowVersion(row2)); - pSchema2 = *ppSchema2; - } - } - - SMemRow tmp = tsdbMergeTwoRows(pBuf, row1, row2, pSchema1, pSchema2); - - void* pMem = tsdbAllocBytes(pRepo, memRowTLen(tmp)); - if(pMem == NULL) return NULL; - memRowCpy(pMem, tmp); - - (*pPoints)++; - *pLastRow = pMem; - return pMem; -} - -static void* tsdbInsertDupKeyMergePacked(void** args) { - return tsdbInsertDupKeyMerge(args[0], args[1], args[2], (STSchema**)&args[3], (STSchema**)&args[4], args[5], args[6], args[7]); -} - -static void tsdbSetupSkipListHookFns(SSkipList* pSkipList, STsdbRepo *pRepo, STable *pTable, int32_t* pPoints, SMemRow* pLastRow) { - - if(pSkipList->insertHandleFn == NULL) { - tGenericSavedFunc *dupHandleSavedFunc = genericSavedFuncInit((GenericVaFunc)&tsdbInsertDupKeyMergePacked, 9); - dupHandleSavedFunc->args[2] = pRepo; - dupHandleSavedFunc->args[3] = NULL; - dupHandleSavedFunc->args[4] = NULL; - dupHandleSavedFunc->args[5] = pTable; - pSkipList->insertHandleFn = dupHandleSavedFunc; - } - pSkipList->insertHandleFn->args[6] = pPoints; - pSkipList->insertHandleFn->args[7] = pLastRow; -} - -static int tsdbInsertDataToTable(STsdbRepo* pRepo, SSubmitBlk* pBlock, int32_t *pAffectedRows) { - - STsdbMeta *pMeta = pRepo->tsdbMeta; - int32_t points = 0; - STable *pTable = NULL; - SSubmitBlkIter blkIter = {0}; - SMemTable *pMemTable = NULL; - STableData *pTableData = NULL; - STsdbCfg *pCfg = &(pRepo->config); - - tsdbInitSubmitBlkIter(pBlock, &blkIter); - if(blkIter.row == NULL) return 0; - TSKEY firstRowKey = memRowKey(blkIter.row); - - tsdbAllocBytes(pRepo, 0); - pMemTable = pRepo->mem; - - ASSERT(pMemTable != NULL); - ASSERT(pBlock->tid < pMeta->maxTables); - - pTable = pMeta->tables[pBlock->tid]; - - ASSERT(pTable != NULL && TABLE_UID(pTable) == pBlock->uid); - - - if (TABLE_TID(pTable) >= pMemTable->maxTables) { - if (tsdbAdjustMemMaxTables(pMemTable, pMeta->maxTables) < 0) { - return -1; - } - } - pTableData = pMemTable->tData[TABLE_TID(pTable)]; - - if (pTableData == NULL || pTableData->uid != TABLE_UID(pTable)) { - if (pTableData != NULL) { - taosWLockLatch(&(pMemTable->latch)); - pMemTable->tData[TABLE_TID(pTable)] = NULL; - tsdbFreeTableData(pTableData); - taosWUnLockLatch(&(pMemTable->latch)); - } - - pTableData = tsdbNewTableData(pCfg, pTable); - if (pTableData == NULL) { - tsdbError("vgId:%d failed to insert data to table %s uid %" PRId64 " tid %d since %s", REPO_ID(pRepo), - TABLE_CHAR_NAME(pTable), TABLE_UID(pTable), TABLE_TID(pTable), tstrerror(terrno)); - return -1; - } - - pRepo->mem->tData[TABLE_TID(pTable)] = pTableData; - } - - ASSERT((pTableData != NULL) && pTableData->uid == TABLE_UID(pTable)); - - SMemRow lastRow = NULL; - int64_t osize = SL_SIZE(pTableData->pData); - tsdbSetupSkipListHookFns(pTableData->pData, pRepo, pTable, &points, &lastRow); - tSkipListPutBatchByIter(pTableData->pData, &blkIter, (iter_next_fn_t)tsdbGetSubmitBlkNext); - int64_t dsize = SL_SIZE(pTableData->pData) - osize; - (*pAffectedRows) += points; - - - if(lastRow != NULL) { - TSKEY lastRowKey = memRowKey(lastRow); - if (pMemTable->keyFirst > firstRowKey) pMemTable->keyFirst = firstRowKey; - pMemTable->numOfRows += dsize; - - if (pTableData->keyFirst > firstRowKey) pTableData->keyFirst = firstRowKey; - pTableData->numOfRows += dsize; - if (pMemTable->keyLast < lastRowKey) pMemTable->keyLast = lastRowKey; - if (pTableData->keyLast < lastRowKey) pTableData->keyLast = lastRowKey; - if (tsdbUpdateTableLatestInfo(pRepo, pTable, lastRow) < 0) { - return -1; - } - } - - STSchema *pSchema = tsdbGetTableSchemaByVersion(pTable, pBlock->sversion); - pRepo->stat.pointsWritten += points * schemaNCols(pSchema); - pRepo->stat.totalStorage += points * schemaVLen(pSchema); - - return 0; -} - - -static int tsdbInitSubmitMsgIter(SSubmitMsg *pMsg, SSubmitMsgIter *pIter) { - if (pMsg == NULL) { - terrno = TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP; - return -1; - } - - pIter->totalLen = pMsg->length; - pIter->len = 0; - pIter->pMsg = pMsg; - if (pMsg->length <= TSDB_SUBMIT_MSG_HEAD_SIZE) { - terrno = TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP; - return -1; - } - - return 0; -} - -static int tsdbGetSubmitMsgNext(SSubmitMsgIter *pIter, SSubmitBlk **pPBlock) { - if (pIter->len == 0) { - pIter->len += TSDB_SUBMIT_MSG_HEAD_SIZE; - } else { - SSubmitBlk *pSubmitBlk = (SSubmitBlk *)POINTER_SHIFT(pIter->pMsg, pIter->len); - pIter->len += (sizeof(SSubmitBlk) + pSubmitBlk->dataLen + pSubmitBlk->schemaLen); - } - - if (pIter->len > pIter->totalLen) { - terrno = TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP; - *pPBlock = NULL; - return -1; - } - - *pPBlock = (pIter->len == pIter->totalLen) ? NULL : (SSubmitBlk *)POINTER_SHIFT(pIter->pMsg, pIter->len); - - return 0; -} - -static int tsdbCheckTableSchema(STsdbRepo *pRepo, SSubmitBlk *pBlock, STable *pTable) { - ASSERT(pTable != NULL); - - STSchema *pSchema = tsdbGetTableSchemaImpl(pTable, false, false, -1); - int sversion = schemaVersion(pSchema); - - if (pBlock->sversion == sversion) { - return 0; - } else { - if (TABLE_TYPE(pTable) == TSDB_STREAM_TABLE) { // stream table is not allowed to change schema - terrno = TSDB_CODE_TDB_IVD_TB_SCHEMA_VERSION; - return -1; - } - } - - if (pBlock->sversion > sversion) { // may need to update table schema - if (pBlock->schemaLen > 0) { - tsdbDebug( - "vgId:%d table %s tid %d uid %" PRIu64 " schema version %d is out of data, client version %d, update...", - REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), TABLE_UID(pTable), sversion, pBlock->sversion); - ASSERT(pBlock->schemaLen % sizeof(STColumn) == 0); - int numOfCols = pBlock->schemaLen / sizeof(STColumn); - STColumn *pTCol = (STColumn *)pBlock->data; - - STSchemaBuilder schemaBuilder = {0}; - if (tdInitTSchemaBuilder(&schemaBuilder, pBlock->sversion) < 0) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - tsdbError("vgId:%d failed to update schema of table %s since %s", REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), - tstrerror(terrno)); - return -1; - } - - for (int i = 0; i < numOfCols; i++) { - if (tdAddColToSchema(&schemaBuilder, pTCol[i].type, htons(pTCol[i].colId), htons(pTCol[i].bytes)) < 0) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - tsdbError("vgId:%d failed to update schema of table %s since %s", REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), - tstrerror(terrno)); - tdDestroyTSchemaBuilder(&schemaBuilder); - return -1; - } - } - - STSchema *pNSchema = tdGetSchemaFromBuilder(&schemaBuilder); - if (pNSchema == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - tdDestroyTSchemaBuilder(&schemaBuilder); - return -1; - } - - tdDestroyTSchemaBuilder(&schemaBuilder); - tsdbUpdateTableSchema(pRepo, pTable, pNSchema, true); - } else { - tsdbDebug( - "vgId:%d table %s tid %d uid %" PRIu64 " schema version %d is out of data, client version %d, reconfigure...", - REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), TABLE_UID(pTable), sversion, pBlock->sversion); - terrno = TSDB_CODE_TDB_TABLE_RECONFIGURE; - return -1; - } - } else { - ASSERT(pBlock->sversion >= 0); - if (tsdbGetTableSchemaImpl(pTable, false, false, pBlock->sversion) == NULL) { - tsdbError("vgId:%d invalid submit schema version %d to table %s tid %d from client", REPO_ID(pRepo), - pBlock->sversion, TABLE_CHAR_NAME(pTable), TABLE_TID(pTable)); - terrno = TSDB_CODE_TDB_IVD_TB_SCHEMA_VERSION; - return -1; - } - } - - return 0; -} - - -static void updateTableLatestColumn(STsdbRepo *pRepo, STable *pTable, SMemRow row) { - tsdbDebug("vgId:%d updateTableLatestColumn, %s row version:%d", REPO_ID(pRepo), pTable->name->data, - memRowVersion(row)); - - STSchema* pSchema = tsdbGetTableLatestSchema(pTable); - if (tsdbUpdateLastColSchema(pTable, pSchema) < 0) { - return; - } - - pSchema = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row)); - if (pSchema == NULL) { - return; - } - - SDataCol *pLatestCols = pTable->lastCols; - int32_t kvIdx = 0; - - for (int16_t j = 0; j < schemaNCols(pSchema); j++) { - STColumn *pTCol = schemaColAt(pSchema, j); - // ignore not exist colId - int16_t idx = tsdbGetLastColumnsIndexByColId(pTable, pTCol->colId); - if (idx == -1) { - continue; - } - - void *value = NULL; - - value = tdGetMemRowDataOfColEx(row, pTCol->colId, (int8_t)pTCol->type, - TD_DATA_ROW_HEAD_SIZE + pSchema->columns[j].offset, &kvIdx); - - if ((value == NULL) || isNull(value, pTCol->type)) { - continue; - } - - SDataCol *pDataCol = &(pLatestCols[idx]); - if (pDataCol->pData == NULL) { - pDataCol->pData = malloc(pTCol->bytes); - pDataCol->bytes = pTCol->bytes; - } else if (pDataCol->bytes < pTCol->bytes) { - pDataCol->pData = realloc(pDataCol->pData, pTCol->bytes); - pDataCol->bytes = pTCol->bytes; - } - // the actual value size - uint16_t bytes = IS_VAR_DATA_TYPE(pTCol->type) ? varDataTLen(value) : pTCol->bytes; - // the actual data size CANNOT larger than column size - assert(pTCol->bytes >= bytes); - memcpy(pDataCol->pData, value, bytes); - //tsdbInfo("updateTableLatestColumn vgId:%d cache column %d for %d,%s", REPO_ID(pRepo), j, pDataCol->bytes, (char*)pDataCol->pData); - pDataCol->ts = memRowKey(row); - } -} - -static int tsdbUpdateTableLatestInfo(STsdbRepo *pRepo, STable *pTable, SMemRow row) { - STsdbCfg *pCfg = &pRepo->config; - - // if cacheLastRow config has been reset, free the lastRow - if (!pCfg->cacheLastRow && pTable->lastRow != NULL) { - SMemRow cachedLastRow = pTable->lastRow; - TSDB_WLOCK_TABLE(pTable); - pTable->lastRow = NULL; - TSDB_WUNLOCK_TABLE(pTable); - taosTZfree(cachedLastRow); - } - - if (tsdbGetTableLastKeyImpl(pTable) <= memRowKey(row)) { - if (CACHE_LAST_ROW(pCfg) || pTable->lastRow != NULL) { - SMemRow nrow = pTable->lastRow; - if (taosTSizeof(nrow) < memRowTLen(row)) { - SMemRow orow = nrow; - nrow = taosTMalloc(memRowTLen(row)); - if (nrow == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; - } - - memRowCpy(nrow, row); - TSDB_WLOCK_TABLE(pTable); - pTable->lastKey = memRowKey(row); - pTable->lastRow = nrow; - TSDB_WUNLOCK_TABLE(pTable); - taosTZfree(orow); - } else { - TSDB_WLOCK_TABLE(pTable); - pTable->lastKey = memRowKey(row); - memRowCpy(nrow, row); - TSDB_WUNLOCK_TABLE(pTable); - } - } else { - pTable->lastKey = memRowKey(row); - } - - if (CACHE_LAST_NULL_COLUMN(pCfg)) { - updateTableLatestColumn(pRepo, pTable, row); - } - } - return 0; -} From f57151d2dcfaeb16da9cc7d9b37a465148193672 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 15 Dec 2021 14:54:54 +0800 Subject: [PATCH 37/43] TD-10431 process create db msg --- include/common/taosmsg.h | 2 +- source/dnode/mgmt/impl/test/CMakeLists.txt | 12 ++ source/dnode/mgmt/impl/test/db/db.cpp | 126 ++++++++++++-------- source/dnode/mgmt/impl/test/dnode/dnode.cpp | 56 ++++----- source/dnode/mgmt/impl/test/user/user.cpp | 10 +- source/dnode/mnode/impl/src/mndDb.c | 3 +- source/dnode/mnode/impl/src/mndShow.c | 2 + 7 files changed, 127 insertions(+), 84 deletions(-) diff --git a/include/common/taosmsg.h b/include/common/taosmsg.h index 38b9c6fc8c..2fc458fd90 100644 --- a/include/common/taosmsg.h +++ b/include/common/taosmsg.h @@ -168,8 +168,8 @@ typedef enum _mgmt_table { TSDB_MGMT_TABLE_SCORES, TSDB_MGMT_TABLE_GRANTS, TSDB_MGMT_TABLE_VNODES, - TSDB_MGMT_TABLE_STREAMTABLES, TSDB_MGMT_TABLE_CLUSTER, + TSDB_MGMT_TABLE_STREAMTABLES, TSDB_MGMT_TABLE_TP, TSDB_MGMT_TABLE_FUNCTION, TSDB_MGMT_TABLE_MAX, diff --git a/source/dnode/mgmt/impl/test/CMakeLists.txt b/source/dnode/mgmt/impl/test/CMakeLists.txt index 93b0bc83e0..80055ee013 100644 --- a/source/dnode/mgmt/impl/test/CMakeLists.txt +++ b/source/dnode/mgmt/impl/test/CMakeLists.txt @@ -1,7 +1,19 @@ + # add_subdirectory(acct) +# add_subdirectory(auth) +# add_subdirectory(balance) # add_subdirectory(cluster) add_subdirectory(db) add_subdirectory(dnode) +# add_subdirectory(func) +# add_subdirectory(mnode) # add_subdirectory(profile) # add_subdirectory(show) +# add_subdirectory(stb) +# add_subdirectory(sync) +# add_subdirectory(telem) +# add_subdirectory(trans) add_subdirectory(user) +# add_subdirectory(vgroup) + +# add_subdirectory(common) diff --git a/source/dnode/mgmt/impl/test/db/db.cpp b/source/dnode/mgmt/impl/test/db/db.cpp index 4756ea508f..d88d9d644f 100644 --- a/source/dnode/mgmt/impl/test/db/db.cpp +++ b/source/dnode/mgmt/impl/test/db/db.cpp @@ -27,9 +27,9 @@ class DndTestDb : public ::testing::Test { initLog("/tmp/dnode_test_db"); const char* fqdn = "localhost"; - const char* firstEp = "localhost:9530"; - pServer = CreateServer("/tmp/dnode_test_db", fqdn, 9530, firstEp); - pClient = createClient("root", "taosdata", fqdn, 9530); + const char* firstEp = "localhost:9040"; + pServer = CreateServer("/tmp/dnode_test_db", fqdn, 9040, firstEp); + pClient = createClient("root", "taosdata", fqdn, 9040); taosMsleep(300); } @@ -134,12 +134,24 @@ class DndTestDb : public ::testing::Test { pos = 0; } + void CheckInt8(int8_t val) { + int8_t data = *((int8_t*)(pData + pos)); + pos += sizeof(int8_t); + EXPECT_EQ(data, val); + } + void CheckInt16(int16_t val) { int16_t data = *((int16_t*)(pData + pos)); pos += sizeof(int16_t); EXPECT_EQ(data, val); } + void CheckInt32(int32_t val) { + int32_t data = *((int32_t*)(pData + pos)); + pos += sizeof(int32_t); + EXPECT_EQ(data, val); + } + void CheckInt64(int64_t val) { int64_t data = *((int64_t*)(pData + pos)); pos += sizeof(int64_t); @@ -170,30 +182,55 @@ SServer* DndTestDb::pServer; SClient* DndTestDb::pClient; int32_t DndTestDb::connId; -TEST_F(DndTestDb, ShowUser) { - SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_USER, "show users", 4); - CheckSchema(0, TSDB_DATA_TYPE_BINARY, TSDB_USER_LEN + VARSTR_HEADER_SIZE, "name"); - CheckSchema(1, TSDB_DATA_TYPE_BINARY, 10 + VARSTR_HEADER_SIZE, "privilege"); - CheckSchema(2, TSDB_DATA_TYPE_TIMESTAMP, 8, "create time"); - CheckSchema(3, TSDB_DATA_TYPE_BINARY, TSDB_USER_LEN + VARSTR_HEADER_SIZE, "account"); +TEST_F(DndTestDb, ShowDb) { + SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DB, "show databases", 16); + CheckSchema(0, TSDB_DATA_TYPE_BINARY, TSDB_DB_NAME_LEN - 1 + VARSTR_HEADER_SIZE, "name"); + CheckSchema(1, TSDB_DATA_TYPE_TIMESTAMP, 8, "create time"); + CheckSchema(2, TSDB_DATA_TYPE_SMALLINT, 2, "replica"); + CheckSchema(3, TSDB_DATA_TYPE_SMALLINT, 2, "quorum"); + CheckSchema(4, TSDB_DATA_TYPE_SMALLINT, 2, "days"); + CheckSchema(5, TSDB_DATA_TYPE_BINARY, 24 + VARSTR_HEADER_SIZE, "keep0,keep1,keep2"); + CheckSchema(6, TSDB_DATA_TYPE_INT, 4, "cache(MB)"); + CheckSchema(7, TSDB_DATA_TYPE_INT, 4, "blocks"); + CheckSchema(8, TSDB_DATA_TYPE_INT, 4, "minrows"); + CheckSchema(9, TSDB_DATA_TYPE_INT, 4, "maxrows"); + CheckSchema(10, TSDB_DATA_TYPE_TINYINT, 1, "wallevel"); + CheckSchema(11, TSDB_DATA_TYPE_INT, 4, "fsync"); + CheckSchema(12, TSDB_DATA_TYPE_TINYINT, 1, "comp"); + CheckSchema(13, TSDB_DATA_TYPE_TINYINT, 1, "cachelast"); + CheckSchema(14, TSDB_DATA_TYPE_BINARY, 3 + VARSTR_HEADER_SIZE, "precision"); + CheckSchema(15, TSDB_DATA_TYPE_TINYINT, 1, "update"); - SendThenCheckShowRetrieveMsg(1); - CheckBinary("root", TSDB_USER_LEN); - CheckBinary("super", 10); - CheckTimestamp(); - CheckBinary("root", TSDB_USER_LEN); + SendThenCheckShowRetrieveMsg(0); } -TEST_F(DndTestDb, CreateUser_01) { +TEST_F(DndTestDb, CreateDb_01) { { - SCreateUserMsg* pReq = (SCreateUserMsg*)rpcMallocCont(sizeof(SCreateUserMsg)); - strcpy(pReq->user, "u1"); - strcpy(pReq->pass, "p1"); + SCreateDbMsg* pReq = (SCreateDbMsg*)rpcMallocCont(sizeof(SCreateDbMsg)); + strcpy(pReq->db, "d1"); + pReq->cacheBlockSize = 16; + pReq->totalBlocks = 10; + pReq->daysPerFile = 10; + pReq->daysToKeep0 = 3650; + pReq->daysToKeep1 = 3650; + pReq->daysToKeep2 = 3650; + pReq->minRowsPerFileBlock = 100; + pReq->maxRowsPerFileBlock = 4096; + pReq->commitTime = 3600; + pReq->fsyncPeriod = 3000; + pReq->walLevel = 1; + pReq->precision = 0; + pReq->compression = 2; + pReq->replications = 1; + pReq->quorum = 1; + pReq->update = 0; + pReq->cacheLastRow = 0; + pReq->ignoreExist = 1; SRpcMsg rpcMsg = {0}; rpcMsg.pCont = pReq; rpcMsg.contLen = sizeof(SCreateUserMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_CREATE_USER; + rpcMsg.msgType = TSDB_MSG_TYPE_CREATE_DB; sendMsg(pClient, &rpcMsg); SRpcMsg* pMsg = pClient->pRsp; @@ -201,38 +238,27 @@ TEST_F(DndTestDb, CreateUser_01) { ASSERT_EQ(pMsg->code, 0); } - { - SCreateUserMsg* pReq = (SCreateUserMsg*)rpcMallocCont(sizeof(SCreateUserMsg)); - strcpy(pReq->user, "u2"); - strcpy(pReq->pass, "p2"); - - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SCreateUserMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_CREATE_USER; - - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, 0); - } - - SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_USER, "show users", 4); + SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_USER, "show databases", 6); SendThenCheckShowRetrieveMsg(3); - CheckBinary("u1", TSDB_USER_LEN); - CheckBinary("root", TSDB_USER_LEN); - CheckBinary("u2", TSDB_USER_LEN); - CheckBinary("normal", 10); - CheckBinary("super", 10); - CheckBinary("normal", 10); + CheckBinary("d1", TSDB_DB_NAME_LEN - 1); CheckTimestamp(); - CheckTimestamp(); - CheckTimestamp(); - CheckBinary("root", TSDB_USER_LEN); - CheckBinary("root", TSDB_USER_LEN); - CheckBinary("root", TSDB_USER_LEN); + CheckInt16(1); // replica + CheckInt16(1); // quorum + CheckInt16(10); // days + CheckBinary("3650,3650,3650", 24); // days + CheckInt32(16); // cache + CheckInt32(10); // blocks + CheckInt32(100); // minrows + CheckInt32(4096); // maxrows + CheckInt8(1); // wallevel + CheckInt32(3000); // fsync + CheckInt8(2); // comp + CheckInt8(0); // cachelast + CheckBinary("ms", 3); // precision + CheckInt8(0); // update } +#if 0 TEST_F(DndTestDb, AlterUser_01) { SAlterUserMsg* pReq = (SAlterUserMsg*)rpcMallocCont(sizeof(SAlterUserMsg)); strcpy(pReq->user, "u1"); @@ -297,8 +323,8 @@ TEST_F(DndTestDb, RestartDnode) { uInfo("start all server"); const char* fqdn = "localhost"; - const char* firstEp = "localhost:9530"; - pServer = startServer("/tmp/dnode_test_db", fqdn, 9530, firstEp); + const char* firstEp = "localhost:9040"; + pServer = startServer("/tmp/dnode_test_db", fqdn, 9040, firstEp); uInfo("all server is running"); @@ -313,3 +339,5 @@ TEST_F(DndTestDb, RestartDnode) { CheckBinary("root", TSDB_USER_LEN); CheckBinary("root", TSDB_USER_LEN); } + +#endif \ No newline at end of file diff --git a/source/dnode/mgmt/impl/test/dnode/dnode.cpp b/source/dnode/mgmt/impl/test/dnode/dnode.cpp index dda991b906..c2b122d054 100644 --- a/source/dnode/mgmt/impl/test/dnode/dnode.cpp +++ b/source/dnode/mgmt/impl/test/dnode/dnode.cpp @@ -27,13 +27,13 @@ class DndTestDnode : public ::testing::Test { initLog("/tmp/dnode_test_dnode"); const char* fqdn = "localhost"; - const char* firstEp = "localhost:9521"; - pServer1 = CreateServer("/tmp/dnode_test_dnode1", fqdn, 9521, firstEp); - pServer2 = CreateServer("/tmp/dnode_test_dnode2", fqdn, 9522, firstEp); - pServer3 = CreateServer("/tmp/dnode_test_dnode3", fqdn, 9523, firstEp); - pServer4 = CreateServer("/tmp/dnode_test_dnode4", fqdn, 9524, firstEp); - pServer5 = CreateServer("/tmp/dnode_test_dnode5", fqdn, 9525, firstEp); - pClient = createClient("root", "taosdata", fqdn, 9521); + const char* firstEp = "localhost:9041"; + pServer1 = CreateServer("/tmp/dnode_test_dnode1", fqdn, 9041, firstEp); + pServer2 = CreateServer("/tmp/dnode_test_dnode2", fqdn, 9042, firstEp); + pServer3 = CreateServer("/tmp/dnode_test_dnode3", fqdn, 9043, firstEp); + pServer4 = CreateServer("/tmp/dnode_test_dnode4", fqdn, 9044, firstEp); + pServer5 = CreateServer("/tmp/dnode_test_dnode5", fqdn, 9045, firstEp); + pClient = createClient("root", "taosdata", fqdn, 9041); taosMsleep(300); } @@ -200,7 +200,7 @@ TEST_F(DndTestDnode, ShowDnode) { SendThenCheckShowRetrieveMsg(1); CheckInt16(1); - CheckBinary("localhost:9521", TSDB_EP_LEN); + CheckBinary("localhost:9041", TSDB_EP_LEN); CheckInt16(0); CheckInt16(1); CheckBinary("ready", 10); @@ -226,7 +226,7 @@ TEST_F(DndTestDnode, ConfigDnode_01) { TEST_F(DndTestDnode, CreateDnode_01) { SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(sizeof(SCreateDnodeMsg)); - strcpy(pReq->ep, "localhost:9522"); + strcpy(pReq->ep, "localhost:9042"); SRpcMsg rpcMsg = {0}; rpcMsg.pCont = pReq; @@ -243,8 +243,8 @@ TEST_F(DndTestDnode, CreateDnode_01) { SendThenCheckShowRetrieveMsg(2); CheckInt16(1); CheckInt16(2); - CheckBinary("localhost:9521", TSDB_EP_LEN); - CheckBinary("localhost:9522", TSDB_EP_LEN); + CheckBinary("localhost:9041", TSDB_EP_LEN); + CheckBinary("localhost:9042", TSDB_EP_LEN); CheckInt16(0); CheckInt16(0); CheckInt16(1); @@ -274,7 +274,7 @@ TEST_F(DndTestDnode, DropDnode_01) { SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DNODE, "show dnodes", 7); SendThenCheckShowRetrieveMsg(1); CheckInt16(1); - CheckBinary("localhost:9521", TSDB_EP_LEN); + CheckBinary("localhost:9041", TSDB_EP_LEN); CheckInt16(0); CheckInt16(1); CheckBinary("ready", 10); @@ -285,7 +285,7 @@ TEST_F(DndTestDnode, DropDnode_01) { TEST_F(DndTestDnode, CreateDnode_02) { { SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(sizeof(SCreateDnodeMsg)); - strcpy(pReq->ep, "localhost:9523"); + strcpy(pReq->ep, "localhost:9043"); SRpcMsg rpcMsg = {0}; rpcMsg.pCont = pReq; @@ -300,7 +300,7 @@ TEST_F(DndTestDnode, CreateDnode_02) { { SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(sizeof(SCreateDnodeMsg)); - strcpy(pReq->ep, "localhost:9524"); + strcpy(pReq->ep, "localhost:9044"); SRpcMsg rpcMsg = {0}; rpcMsg.pCont = pReq; @@ -315,7 +315,7 @@ TEST_F(DndTestDnode, CreateDnode_02) { { SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(sizeof(SCreateDnodeMsg)); - strcpy(pReq->ep, "localhost:9525"); + strcpy(pReq->ep, "localhost:9045"); SRpcMsg rpcMsg = {0}; rpcMsg.pCont = pReq; @@ -335,10 +335,10 @@ TEST_F(DndTestDnode, CreateDnode_02) { CheckInt16(3); CheckInt16(4); CheckInt16(5); - CheckBinary("localhost:9521", TSDB_EP_LEN); - CheckBinary("localhost:9523", TSDB_EP_LEN); - CheckBinary("localhost:9524", TSDB_EP_LEN); - CheckBinary("localhost:9525", TSDB_EP_LEN); + CheckBinary("localhost:9041", TSDB_EP_LEN); + CheckBinary("localhost:9043", TSDB_EP_LEN); + CheckBinary("localhost:9044", TSDB_EP_LEN); + CheckBinary("localhost:9045", TSDB_EP_LEN); CheckInt16(0); CheckInt16(0); CheckInt16(0); @@ -377,11 +377,11 @@ TEST_F(DndTestDnode, RestartDnode_01) { uInfo("start all server"); const char* fqdn = "localhost"; - const char* firstEp = "localhost:9521"; - pServer1 = startServer("/tmp/dnode_test_dnode1", fqdn, 9521, firstEp); - pServer3 = startServer("/tmp/dnode_test_dnode3", fqdn, 9523, firstEp); - pServer4 = startServer("/tmp/dnode_test_dnode4", fqdn, 9524, firstEp); - pServer5 = startServer("/tmp/dnode_test_dnode5", fqdn, 9525, firstEp); + const char* firstEp = "localhost:9041"; + pServer1 = startServer("/tmp/dnode_test_dnode1", fqdn, 9041, firstEp); + pServer3 = startServer("/tmp/dnode_test_dnode3", fqdn, 9043, firstEp); + pServer4 = startServer("/tmp/dnode_test_dnode4", fqdn, 9044, firstEp); + pServer5 = startServer("/tmp/dnode_test_dnode5", fqdn, 9045, firstEp); uInfo("all server is running"); @@ -392,10 +392,10 @@ TEST_F(DndTestDnode, RestartDnode_01) { CheckInt16(3); CheckInt16(4); CheckInt16(5); - CheckBinary("localhost:9521", TSDB_EP_LEN); - CheckBinary("localhost:9523", TSDB_EP_LEN); - CheckBinary("localhost:9524", TSDB_EP_LEN); - CheckBinary("localhost:9525", TSDB_EP_LEN); + CheckBinary("localhost:9041", TSDB_EP_LEN); + CheckBinary("localhost:9043", TSDB_EP_LEN); + CheckBinary("localhost:9044", TSDB_EP_LEN); + CheckBinary("localhost:9045", TSDB_EP_LEN); CheckInt16(0); CheckInt16(0); CheckInt16(0); diff --git a/source/dnode/mgmt/impl/test/user/user.cpp b/source/dnode/mgmt/impl/test/user/user.cpp index 8e25181ae1..9d3acbc273 100644 --- a/source/dnode/mgmt/impl/test/user/user.cpp +++ b/source/dnode/mgmt/impl/test/user/user.cpp @@ -27,9 +27,9 @@ class DndTestUser : public ::testing::Test { initLog("/tmp/dnode_test_user"); const char* fqdn = "localhost"; - const char* firstEp = "localhost:9530"; - pServer = CreateServer("/tmp/dnode_test_user", fqdn, 9530, firstEp); - pClient = createClient("root", "taosdata", fqdn, 9530); + const char* firstEp = "localhost:9140"; + pServer = CreateServer("/tmp/dnode_test_user", fqdn, 9140, firstEp); + pClient = createClient("root", "taosdata", fqdn, 9140); taosMsleep(300); } @@ -297,8 +297,8 @@ TEST_F(DndTestUser, RestartDnode) { uInfo("start all server"); const char* fqdn = "localhost"; - const char* firstEp = "localhost:9530"; - pServer = startServer("/tmp/dnode_test_user", fqdn, 9530, firstEp); + const char* firstEp = "localhost:9140"; + pServer = startServer("/tmp/dnode_test_user", fqdn, 9140, firstEp); uInfo("all server is running"); diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index dd8998399e..83975ca77a 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -151,7 +151,8 @@ static int32_t mndDbActionDelete(SSdb *pSdb, SDbObj *pDb) { static int32_t mndDbActionUpdate(SSdb *pSdb, SDbObj *pOldDb, SDbObj *pNewDb) { mTrace("db:%s, perform update action", pOldDb->name); - memcpy(pOldDb, pNewDb, sizeof(SDbObj)); + pOldDb->updateTime = pNewDb->createdTime; + memcpy(&pOldDb->cfg, &pNewDb->cfg, sizeof(SDbObj)); return 0; } diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index c2b2acfa0c..15ae0d1ee4 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -296,6 +296,8 @@ char *mndShowStr(int32_t showType) { return "show streamtables"; case TSDB_MGMT_TABLE_TP: return "show topics"; + case TSDB_MGMT_TABLE_FUNCTION: + return "show functions"; default: return "undefined"; } From 63f03cfadf17737eaee91d81bdacf34d6b65bcfc Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 15 Dec 2021 15:22:39 +0800 Subject: [PATCH 38/43] more work --- source/dnode/vnode/tsdb/inc/tsdbDef.h | 2 ++ source/dnode/vnode/tsdb/src/tsdbMemTable.c | 37 +++++++++++++++++++--- 2 files changed, 34 insertions(+), 5 deletions(-) diff --git a/source/dnode/vnode/tsdb/inc/tsdbDef.h b/source/dnode/vnode/tsdb/inc/tsdbDef.h index 882a62e097..7c593cb4c7 100644 --- a/source/dnode/vnode/tsdb/inc/tsdbDef.h +++ b/source/dnode/vnode/tsdb/inc/tsdbDef.h @@ -18,7 +18,9 @@ #include "mallocator.h" #include "taosmsg.h" +#include "tdlist.h" #include "thash.h" +#include "tskiplist.h" #include "tsdb.h" #include "tsdbMemTable.h" diff --git a/source/dnode/vnode/tsdb/src/tsdbMemTable.c b/source/dnode/vnode/tsdb/src/tsdbMemTable.c index e13b881ee5..e3d1f8673e 100644 --- a/source/dnode/vnode/tsdb/src/tsdbMemTable.c +++ b/source/dnode/vnode/tsdb/src/tsdbMemTable.c @@ -15,14 +15,32 @@ #include "tsdbDef.h" +#if 1 +typedef struct STbData { + TD_SLIST_NODE(STbData); + SSubmitMsg *pMsg; +} STbData; +#else +typedef struct STbData { + TD_SLIST_NODE(STbData); + uint64_t uid; // TODO: change here as tb_uid_t + TSKEY keyMin; + TSKEY keyMax; + uint64_t nRows; + SSkipList *pData; // Here need a container, may not use the SL + T_REF_DECLARE() +} STbData; +#endif + struct STsdbMemTable { T_REF_DECLARE() SRWLatch latch; TSKEY keyMin; TSKEY keyMax; - uint32_t nRow; - SHashObj * pHash; + uint64_t nRow; SMemAllocator *pMA; + // Container + TD_SLIST(STbData) list; }; STsdbMemTable *tsdbNewMemTable(SMemAllocatorFactory *pMAF) { @@ -43,8 +61,8 @@ STsdbMemTable *tsdbNewMemTable(SMemAllocatorFactory *pMAF) { pMemTable->keyMin = TSKEY_MAX; pMemTable->keyMax = TSKEY_MIN; pMemTable->nRow = 0; - pMemTable->pHash = NULL; /// TODO pMemTable->pMA = pMA; + tSListInit(&(pMemTable->list)); // TODO return pMemTable; @@ -62,6 +80,15 @@ void tsdbFreeMemTable(SMemAllocatorFactory *pMAF, STsdbMemTable *pMemTable) { } int tsdbInsertDataToMemTable(STsdbMemTable *pMemTable, SSubmitMsg *pMsg) { - // TODO + SMemAllocator *pMA = pMemTable->pMA; + STbData * pTbData = (STbData *)((*pMA->malloc)(pMA, sizeof(*pTbData))); + if (pTbData == NULL) { + // TODO + } + + tSListPush(&(pMemTable->list), pTbData); + return 0; -} \ No newline at end of file +} + +/* ------------------------ STATIC METHODS ------------------------ */ \ No newline at end of file From e43aad8b5e78bb86197414a822032391ae671093 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 15 Dec 2021 15:30:14 +0800 Subject: [PATCH 39/43] more --- source/dnode/vnode/impl/test/vnodeApiTests.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/dnode/vnode/impl/test/vnodeApiTests.cpp b/source/dnode/vnode/impl/test/vnodeApiTests.cpp index a25b04e161..f0bca3aa2e 100644 --- a/source/dnode/vnode/impl/test/vnodeApiTests.cpp +++ b/source/dnode/vnode/impl/test/vnodeApiTests.cpp @@ -92,6 +92,8 @@ TEST(vnodeApiTest, test_create_table_encode_and_decode_function) { #endif TEST(vnodeApiTest, vnodeOpen_vnodeClose_test) { + vnodeDestroy("vnode1"); + GTEST_ASSERT_GE(vnodeInit(2), 0); // Create and open a vnode From 56c513cd90a5d812cac8458c4629096848d7eef1 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 15 Dec 2021 15:35:18 +0800 Subject: [PATCH 40/43] rename some funcs --- source/dnode/mnode/impl/inc/mndShow.h | 2 +- source/dnode/mnode/impl/src/mndAcct.c | 38 ++++++++++++------------ source/dnode/mnode/impl/src/mndCluster.c | 2 +- source/dnode/mnode/impl/src/mndDb.c | 11 ++++--- source/dnode/mnode/impl/src/mndDnode.c | 4 +-- source/dnode/mnode/impl/src/mndFunc.c | 2 +- source/dnode/mnode/impl/src/mndMnode.c | 2 +- source/dnode/mnode/impl/src/mndProfile.c | 4 +-- source/dnode/mnode/impl/src/mndShow.c | 2 +- source/dnode/mnode/impl/src/mndStb.c | 4 +-- source/dnode/mnode/impl/src/mndUser.c | 2 +- source/dnode/mnode/impl/src/mndVgroup.c | 4 +-- source/dnode/mnode/impl/src/mnode.c | 4 +-- 13 files changed, 40 insertions(+), 41 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndShow.h b/source/dnode/mnode/impl/inc/mndShow.h index 37fb9159dc..67e277677c 100644 --- a/source/dnode/mnode/impl/inc/mndShow.h +++ b/source/dnode/mnode/impl/inc/mndShow.h @@ -27,7 +27,7 @@ void mndCleanupShow(SMnode *pMnode); void mndAddShowMetaHandle(SMnode *pMnode, EShowType showType, ShowMetaFp fp); void mndAddShowRetrieveHandle(SMnode *pMnode, EShowType showType, ShowRetrieveFp fp); void mndAddShowFreeIterHandle(SMnode *pMnode, EShowType msgType, ShowFreeIterFp fp); -void mnodeVacuumResult(char *data, int32_t numOfCols, int32_t rows, int32_t capacity, SShowObj *pShow); +void mndVacuumResult(char *data, int32_t numOfCols, int32_t rows, int32_t capacity, SShowObj *pShow); char *mndShowStr(int32_t showType); #ifdef __cplusplus diff --git a/source/dnode/mnode/impl/src/mndAcct.c b/source/dnode/mnode/impl/src/mndAcct.c index a365ad7326..75b0acbd0a 100644 --- a/source/dnode/mnode/impl/src/mndAcct.c +++ b/source/dnode/mnode/impl/src/mndAcct.c @@ -19,12 +19,12 @@ #define SDB_ACCT_VER 1 -static int32_t mnodeCreateDefaultAcct(SMnode *pMnode); -static SSdbRaw *mnodeAcctActionEncode(SAcctObj *pAcct); -static SSdbRow *mnodeAcctActionDecode(SSdbRaw *pRaw); -static int32_t mnodeAcctActionInsert(SSdb *pSdb, SAcctObj *pAcct); -static int32_t mnodeAcctActionDelete(SSdb *pSdb, SAcctObj *pAcct); -static int32_t mnodeAcctActionUpdate(SSdb *pSdb, SAcctObj *pOldAcct, SAcctObj *pNewAcct); +static int32_t mndCreateDefaultAcct(SMnode *pMnode); +static SSdbRaw *mndAcctActionEncode(SAcctObj *pAcct); +static SSdbRow *mndAcctActionDecode(SSdbRaw *pRaw); +static int32_t mndAcctActionInsert(SSdb *pSdb, SAcctObj *pAcct); +static int32_t mndAcctActionDelete(SSdb *pSdb, SAcctObj *pAcct); +static int32_t mndAcctActionUpdate(SSdb *pSdb, SAcctObj *pOldAcct, SAcctObj *pNewAcct); static int32_t mndProcessCreateAcctMsg(SMnodeMsg *pMnodeMsg); static int32_t mndProcessAlterAcctMsg(SMnodeMsg *pMnodeMsg); static int32_t mndProcessDropAcctMsg(SMnodeMsg *pMnodeMsg); @@ -32,12 +32,12 @@ static int32_t mndProcessDropAcctMsg(SMnodeMsg *pMnodeMsg); int32_t mndInitAcct(SMnode *pMnode) { SSdbTable table = {.sdbType = SDB_ACCT, .keyType = SDB_KEY_BINARY, - .deployFp = mnodeCreateDefaultAcct, - .encodeFp = (SdbEncodeFp)mnodeAcctActionEncode, - .decodeFp = (SdbDecodeFp)mnodeAcctActionDecode, - .insertFp = (SdbInsertFp)mnodeAcctActionInsert, - .updateFp = (SdbUpdateFp)mnodeAcctActionUpdate, - .deleteFp = (SdbDeleteFp)mnodeAcctActionDelete}; + .deployFp = mndCreateDefaultAcct, + .encodeFp = (SdbEncodeFp)mndAcctActionEncode, + .decodeFp = (SdbDecodeFp)mndAcctActionDecode, + .insertFp = (SdbInsertFp)mndAcctActionInsert, + .updateFp = (SdbUpdateFp)mndAcctActionUpdate, + .deleteFp = (SdbDeleteFp)mndAcctActionDelete}; mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_CREATE_ACCT, mndProcessCreateAcctMsg); mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_ALTER_ACCT, mndProcessAlterAcctMsg); @@ -48,7 +48,7 @@ int32_t mndInitAcct(SMnode *pMnode) { void mndCleanupAcct(SMnode *pMnode) {} -static int32_t mnodeCreateDefaultAcct(SMnode *pMnode) { +static int32_t mndCreateDefaultAcct(SMnode *pMnode) { SAcctObj acctObj = {0}; tstrncpy(acctObj.acct, TSDB_DEFAULT_USER, TSDB_USER_LEN); acctObj.createdTime = taosGetTimestampMs(); @@ -61,7 +61,7 @@ static int32_t mnodeCreateDefaultAcct(SMnode *pMnode) { .maxStorage = INT64_MAX, .accessState = TSDB_VN_ALL_ACCCESS}; - SSdbRaw *pRaw = mnodeAcctActionEncode(&acctObj); + SSdbRaw *pRaw = mndAcctActionEncode(&acctObj); if (pRaw == NULL) return -1; sdbSetRawStatus(pRaw, SDB_STATUS_READY); @@ -69,7 +69,7 @@ static int32_t mnodeCreateDefaultAcct(SMnode *pMnode) { return sdbWrite(pMnode->pSdb, pRaw); } -static SSdbRaw *mnodeAcctActionEncode(SAcctObj *pAcct) { +static SSdbRaw *mndAcctActionEncode(SAcctObj *pAcct) { SSdbRaw *pRaw = sdbAllocRaw(SDB_ACCT, SDB_ACCT_VER, sizeof(SAcctObj)); if (pRaw == NULL) return NULL; @@ -90,7 +90,7 @@ static SSdbRaw *mnodeAcctActionEncode(SAcctObj *pAcct) { return pRaw; } -static SSdbRow *mnodeAcctActionDecode(SSdbRaw *pRaw) { +static SSdbRow *mndAcctActionDecode(SSdbRaw *pRaw) { int8_t sver = 0; if (sdbGetRawSoftVer(pRaw, &sver) != 0) return NULL; @@ -120,18 +120,18 @@ static SSdbRow *mnodeAcctActionDecode(SSdbRaw *pRaw) { return pRow; } -static int32_t mnodeAcctActionInsert(SSdb *pSdb, SAcctObj *pAcct) { +static int32_t mndAcctActionInsert(SSdb *pSdb, SAcctObj *pAcct) { mTrace("acct:%s, perform insert action", pAcct->acct); memset(&pAcct->info, 0, sizeof(SAcctInfo)); return 0; } -static int32_t mnodeAcctActionDelete(SSdb *pSdb, SAcctObj *pAcct) { +static int32_t mndAcctActionDelete(SSdb *pSdb, SAcctObj *pAcct) { mTrace("acct:%s, perform delete action", pAcct->acct); return 0; } -static int32_t mnodeAcctActionUpdate(SSdb *pSdb, SAcctObj *pOldAcct, SAcctObj *pNewAcct) { +static int32_t mndAcctActionUpdate(SSdb *pSdb, SAcctObj *pOldAcct, SAcctObj *pNewAcct) { mTrace("acct:%s, perform update action", pOldAcct->acct); memcpy(pOldAcct->acct, pNewAcct->acct, TSDB_USER_LEN); diff --git a/source/dnode/mnode/impl/src/mndCluster.c b/source/dnode/mnode/impl/src/mndCluster.c index 7edeb73647..0eaa184907 100644 --- a/source/dnode/mnode/impl/src/mndCluster.c +++ b/source/dnode/mnode/impl/src/mndCluster.c @@ -203,7 +203,7 @@ static int32_t mndRetrieveClusters(SMnodeMsg *pMsg, SShowObj *pShow, char *data, numOfRows++; } - mnodeVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); + mndVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); pShow->numOfReads += numOfRows; return numOfRows; } diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 83975ca77a..e44e25cbab 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -408,7 +408,6 @@ static int32_t mndProcessCreateDbMsg(SMnodeMsg *pMsg) { mndReleaseUser(pMnode, pOperUser); if (code != 0) { - terrno = code; mError("db:%s, failed to create since %s", pCreate->db, terrstr()); return -1; } @@ -416,7 +415,7 @@ static int32_t mndProcessCreateDbMsg(SMnodeMsg *pMsg) { return TSDB_CODE_MND_ACTION_IN_PROGRESS; } -static int32_t mnodeSetDbCfgFromAlterDbMsg(SDbObj *pDb, SAlterDbMsg *pAlter) { +static int32_t mndSetDbCfgFromAlterDbMsg(SDbObj *pDb, SAlterDbMsg *pAlter) { bool changed = false; if (pAlter->totalBlocks >= 0 && pAlter->totalBlocks != pDb->cfg.totalBlocks) { @@ -522,7 +521,7 @@ static int32_t mndProcessAlterDbMsg(SMnodeMsg *pMsg) { SDbObj dbObj = {0}; memcpy(&dbObj, pDb, sizeof(SDbObj)); - int32_t code = mnodeSetDbCfgFromAlterDbMsg(&dbObj, pAlter); + int32_t code = mndSetDbCfgFromAlterDbMsg(&dbObj, pAlter); if (code != 0) { mndReleaseDb(pMnode, pDb); mError("db:%s, failed to alter since %s", pAlter->db, tstrerror(code)); @@ -773,7 +772,7 @@ static int32_t mndGetDbMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMe return 0; } -char *mnodeGetDbStr(char *src) { +char *mnGetDbStr(char *src) { char *pos = strstr(src, TS_PATH_DELIMITER); if (pos != NULL) ++pos; @@ -795,7 +794,7 @@ static int32_t mndRetrieveDbs(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int3 cols = 0; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - char *name = mnodeGetDbStr(pDb->name); + char *name = mnGetDbStr(pDb->name); if (name != NULL) { STR_WITH_MAXSIZE_TO_VARSTR(pWrite, name, pShow->bytes[cols]); } else { @@ -888,7 +887,7 @@ static int32_t mndRetrieveDbs(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int3 sdbRelease(pSdb, pDb); } - mnodeVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); + mndVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); pShow->numOfReads += numOfRows; return numOfRows; diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 504be58d09..ec5f68a713 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -612,7 +612,7 @@ static int32_t mndRetrieveConfigs(SMnodeMsg *pMsg, SShowObj *pShow, char *data, cols++; } - mnodeVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); + mndVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); pShow->numOfReads += numOfRows; return numOfRows; } @@ -734,7 +734,7 @@ static int32_t mndRetrieveDnodes(SMnodeMsg *pMsg, SShowObj *pShow, char *data, i sdbRelease(pSdb, pDnode); } - mnodeVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); + mndVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); pShow->numOfReads += numOfRows; return numOfRows; diff --git a/source/dnode/mnode/impl/src/mndFunc.c b/source/dnode/mnode/impl/src/mndFunc.c index 3fd7dcfba1..b7158bb094 100644 --- a/source/dnode/mnode/impl/src/mndFunc.c +++ b/source/dnode/mnode/impl/src/mndFunc.c @@ -500,7 +500,7 @@ static int32_t mndRetrieveFuncs(SMnodeMsg *pMsg, SShowObj *pShow, char *data, in sdbRelease(pSdb, pFunc); } - mnodeVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); + mndVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); pShow->numOfReads += numOfRows; return numOfRows; } diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index e91c51d301..7b6804d43e 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -457,7 +457,7 @@ static int32_t mndRetrieveMnodes(SMnodeMsg *pMsg, SShowObj *pShow, char *data, i sdbRelease(pSdb, pMnodeObj); } - mnodeVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); + mndVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); pShow->numOfReads += numOfRows; return numOfRows; diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index 59d84e5760..45a63f2dc5 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -252,7 +252,7 @@ static int32_t mndProcessConnectMsg(SMnodeMsg *pMsg) { return 0; } -static int32_t mnodeSaveQueryStreamList(SConnObj *pConn, SHeartBeatMsg *pMsg) { +static int32_t mndSaveQueryStreamList(SConnObj *pConn, SHeartBeatMsg *pMsg) { pConn->numOfQueries = 0; pConn->numOfStreams = 0; int32_t numOfQueries = htonl(pMsg->numOfQueries); @@ -338,7 +338,7 @@ static int32_t mndProcessHeartBeatMsg(SMnodeMsg *pMsg) { return -1; } - mnodeSaveQueryStreamList(pConn, pReq); + mndSaveQueryStreamList(pConn, pReq); if (pConn->killed != 0) { pRsp->killConnection = 1; } diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index 15ae0d1ee4..91a229956f 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -310,7 +310,7 @@ static bool mndCheckRetrieveFinished(SShowObj *pShow) { return false; } -void mnodeVacuumResult(char *data, int32_t numOfCols, int32_t rows, int32_t capacity, SShowObj *pShow) { +void mndVacuumResult(char *data, int32_t numOfCols, int32_t rows, int32_t capacity, SShowObj *pShow) { if (rows < capacity) { for (int32_t i = 0; i < numOfCols; ++i) { memmove(data + pShow->offset[i] * rows, data + pShow->offset[i] * capacity, pShow->bytes[i] * rows); diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 63bf186be5..d06099416b 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -605,7 +605,7 @@ static int32_t mndGetStbMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pM return 0; } -static void mnodeExtractTableName(char *tableId, char *name) { +static void mndExtractTableName(char *tableId, char *name) { int pos = -1; int num = 0; for (pos = 0; tableId[pos] != 0; ++pos) { @@ -665,7 +665,7 @@ static int32_t mndRetrieveStb(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int3 } pShow->numOfReads += numOfRows; - mnodeVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); + mndVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); return numOfRows; } diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index 91adc02d72..7219342d59 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -486,7 +486,7 @@ static int32_t mndRetrieveUsers(SMnodeMsg *pMsg, SShowObj *pShow, char *data, in sdbRelease(pSdb, pUser); } - mnodeVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); + mndVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); pShow->numOfReads += numOfRows; return numOfRows; } diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 44d13d1fb4..ca244fed9f 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -284,7 +284,7 @@ static int32_t mndRetrieveVgroups(SMnodeMsg *pMsg, SShowObj *pShow, char *data, numOfRows++; } - mnodeVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); + mndVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); pShow->numOfReads += numOfRows; return numOfRows; } @@ -374,7 +374,7 @@ static int32_t mndRetrieveVnodes(SMnodeMsg *pMsg, SShowObj *pShow, char *data, i sdbRelease(pSdb, pVgroup); } - mnodeVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); + mndVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); pShow->numOfReads += numOfRows; return numOfRows; } diff --git a/source/dnode/mnode/impl/src/mnode.c b/source/dnode/mnode/impl/src/mnode.c index e543478608..24155a4e53 100644 --- a/source/dnode/mnode/impl/src/mnode.c +++ b/source/dnode/mnode/impl/src/mnode.c @@ -69,7 +69,7 @@ static void mndCleanupTimer(SMnode *pMnode) { } } -static int32_t mnodeCreateDir(SMnode *pMnode, const char *path) { +static int32_t mndCreateDir(SMnode *pMnode, const char *path) { pMnode->path = strdup(path); if (pMnode->path == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -245,7 +245,7 @@ SMnode *mndOpen(const char *path, const SMnodeOpt *pOption) { return NULL; } - int32_t code = mnodeCreateDir(pMnode, path); + int32_t code = mndCreateDir(pMnode, path); if (code != 0) { code = terrno; mError("failed to open mnode since %s", terrstr()); From ac7bcea22fd1c9824868505cae93f4edee36fa56 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 15 Dec 2021 16:24:43 +0800 Subject: [PATCH 41/43] more --- source/dnode/vnode/impl/src/vnodeWrite.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/impl/src/vnodeWrite.c b/source/dnode/vnode/impl/src/vnodeWrite.c index cdeb894932..85460e8d91 100644 --- a/source/dnode/vnode/impl/src/vnodeWrite.c +++ b/source/dnode/vnode/impl/src/vnodeWrite.c @@ -72,7 +72,9 @@ int vnodeProcessWMsgs(SVnode *pVnode, SArray *pMsgs) { } break; case TSDB_MSG_TYPE_SUBMIT: - /* code */ + if (tsdbInsertData(pVnode->pTsdb, (SSubmitMsg *)ptr) < 0) { + // TODO: handle error + } break; default: break; From c8a746574da5e35bc88f918a469aba0e394c4f69 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 15 Dec 2021 16:32:24 +0800 Subject: [PATCH 42/43] TD-10431 create db test --- include/util/taoserror.h | 26 ++---- include/util/tdef.h | 1 + source/dnode/mgmt/impl/test/db/db.cpp | 30 +++---- source/dnode/mgmt/impl/test/user/user.cpp | 2 +- source/dnode/mnode/impl/src/mndDb.c | 101 ++++++++++++++++------ source/util/src/terror.c | 18 +--- 6 files changed, 97 insertions(+), 81 deletions(-) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index c53e7931ac..4f1ef7da7b 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -197,29 +197,13 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_DB_NOT_SELECTED TAOS_DEF_ERROR_CODE(0, 0x0383) #define TSDB_CODE_MND_INVALID_DB TAOS_DEF_ERROR_CODE(0, 0x0384) #define TSDB_CODE_MND_INVALID_DB_OPTION TAOS_DEF_ERROR_CODE(0, 0x0385) -#define TSDB_CODE_MND_INVALID_DB_CACHE_SIZE TAOS_DEF_ERROR_CODE(0, 0x0386) -#define TSDB_CODE_MND_INVALID_DB_TOTAL_BLOCKS TAOS_DEF_ERROR_CODE(0, 0x0387) -#define TSDB_CODE_MND_INVALID_DB_DAYS TAOS_DEF_ERROR_CODE(0, 0x0388) -#define TSDB_CODE_MND_INVALID_DB_KEEP0 TAOS_DEF_ERROR_CODE(0, 0x0389) -#define TSDB_CODE_MND_INVALID_DB_KEEP1 TAOS_DEF_ERROR_CODE(0, 0x038A) -#define TSDB_CODE_MND_INVALID_DB_KEEP2 TAOS_DEF_ERROR_CODE(0, 0x038B) -#define TSDB_CODE_MND_INVALID_DB_MIN_ROWS TAOS_DEF_ERROR_CODE(0, 0x038C) -#define TSDB_CODE_MND_INVALID_DB_MAX_ROWS TAOS_DEF_ERROR_CODE(0, 0x038D) -#define TSDB_CODE_MND_INVALID_DB_COMMIT_TIME TAOS_DEF_ERROR_CODE(0, 0x038E) -#define TSDB_CODE_MND_INVALID_DB_FSYNC_PERIOD TAOS_DEF_ERROR_CODE(0, 0x038F) -#define TSDB_CODE_MND_INVALID_DB_WAL_LEVEL TAOS_DEF_ERROR_CODE(0, 0x0390) -#define TSDB_CODE_MND_INVALID_DB_PRECISION TAOS_DEF_ERROR_CODE(0, 0x0391) -#define TSDB_CODE_MND_INVALID_DB_COMP TAOS_DEF_ERROR_CODE(0, 0x0392) -#define TSDB_CODE_MND_INVALID_DB_REPLICA TAOS_DEF_ERROR_CODE(0, 0x0393) -#define TSDB_CODE_MND_INVALID_DB_QUORUM TAOS_DEF_ERROR_CODE(0, 0x0394) -#define TSDB_CODE_MND_INVALID_DB_UPDATE TAOS_DEF_ERROR_CODE(0, 0x0395) -#define TSDB_CODE_MND_INVALID_DB_CACHE_LAST TAOS_DEF_ERROR_CODE(0, 0x0396) -#define TSDB_CODE_MND_DB_OPTION_UNCHANGED TAOS_DEF_ERROR_CODE(0, 0x0397) +#define TSDB_CODE_MND_INVALID_DB_ACCT TAOS_DEF_ERROR_CODE(0, 0x0386) +#define TSDB_CODE_MND_DB_OPTION_UNCHANGED TAOS_DEF_ERROR_CODE(0, 0x0387) // mnode-vgroup -#define TSDB_CODE_MND_VGROUP_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03A0) -#define TSDB_CODE_MND_VGROUP_NOT_IN_DNODE TAOS_DEF_ERROR_CODE(0, 0x03A1) -#define TSDB_CODE_MND_VGROUP_ALREADY_IN_DNODE TAOS_DEF_ERROR_CODE(0, 0x03A2) +#define TSDB_CODE_MND_VGROUP_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0390) +#define TSDB_CODE_MND_VGROUP_NOT_IN_DNODE TAOS_DEF_ERROR_CODE(0, 0x0391) +#define TSDB_CODE_MND_VGROUP_ALREADY_IN_DNODE TAOS_DEF_ERROR_CODE(0, 0x0392) // mnode-stable #define TSDB_CODE_MND_STB_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0360) diff --git a/include/util/tdef.h b/include/util/tdef.h index 300b190e97..1511190c3e 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -209,6 +209,7 @@ do { \ #define TSDB_STEP_NAME_LEN 32 #define TSDB_STEP_DESC_LEN 128 +#define TSDB_ERROR_MSG_LEN 1024 #define TSDB_DNODE_CONFIG_LEN 128 #define TSDB_MQTT_HOSTNAME_LEN 64 diff --git a/source/dnode/mgmt/impl/test/db/db.cpp b/source/dnode/mgmt/impl/test/db/db.cpp index d88d9d644f..4fa5e43883 100644 --- a/source/dnode/mgmt/impl/test/db/db.cpp +++ b/source/dnode/mgmt/impl/test/db/db.cpp @@ -24,7 +24,7 @@ class DndTestDb : public ::testing::Test { } static void SetUpTestSuite() { - initLog("/tmp/dnode_test_db"); + initLog("/tmp/tdlog"); const char* fqdn = "localhost"; const char* firstEp = "localhost:9040"; @@ -207,17 +207,17 @@ TEST_F(DndTestDb, ShowDb) { TEST_F(DndTestDb, CreateDb_01) { { SCreateDbMsg* pReq = (SCreateDbMsg*)rpcMallocCont(sizeof(SCreateDbMsg)); - strcpy(pReq->db, "d1"); - pReq->cacheBlockSize = 16; - pReq->totalBlocks = 10; - pReq->daysPerFile = 10; - pReq->daysToKeep0 = 3650; - pReq->daysToKeep1 = 3650; - pReq->daysToKeep2 = 3650; - pReq->minRowsPerFileBlock = 100; - pReq->maxRowsPerFileBlock = 4096; - pReq->commitTime = 3600; - pReq->fsyncPeriod = 3000; + strcpy(pReq->db, "1.d1"); + pReq->cacheBlockSize = htonl(16); + pReq->totalBlocks = htonl(10); + pReq->daysPerFile = htonl(10); + pReq->daysToKeep0 = htonl(3650); + pReq->daysToKeep1 = htonl(3650); + pReq->daysToKeep2 = htonl(3650); + pReq->minRowsPerFileBlock = htonl(100); + pReq->maxRowsPerFileBlock = htonl(4096); + pReq->commitTime = htonl(3600); + pReq->fsyncPeriod = htonl(3000); pReq->walLevel = 1; pReq->precision = 0; pReq->compression = 2; @@ -229,7 +229,7 @@ TEST_F(DndTestDb, CreateDb_01) { SRpcMsg rpcMsg = {0}; rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SCreateUserMsg); + rpcMsg.contLen = sizeof(SCreateDbMsg); rpcMsg.msgType = TSDB_MSG_TYPE_CREATE_DB; sendMsg(pClient, &rpcMsg); @@ -238,8 +238,8 @@ TEST_F(DndTestDb, CreateDb_01) { ASSERT_EQ(pMsg->code, 0); } - SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_USER, "show databases", 6); - SendThenCheckShowRetrieveMsg(3); + SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DB, "show databases", 16); + SendThenCheckShowRetrieveMsg(1); CheckBinary("d1", TSDB_DB_NAME_LEN - 1); CheckTimestamp(); CheckInt16(1); // replica diff --git a/source/dnode/mgmt/impl/test/user/user.cpp b/source/dnode/mgmt/impl/test/user/user.cpp index 9d3acbc273..6a48a2cb49 100644 --- a/source/dnode/mgmt/impl/test/user/user.cpp +++ b/source/dnode/mgmt/impl/test/user/user.cpp @@ -24,7 +24,7 @@ class DndTestUser : public ::testing::Test { } static void SetUpTestSuite() { - initLog("/tmp/dnode_test_user"); + initLog("/tmp/tdlog"); const char* fqdn = "localhost"; const char* firstEp = "localhost:9140"; diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index e44e25cbab..6426c3f33f 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -152,7 +152,7 @@ static int32_t mndDbActionDelete(SSdb *pSdb, SDbObj *pDb) { static int32_t mndDbActionUpdate(SSdb *pSdb, SDbObj *pOldDb, SDbObj *pNewDb) { mTrace("db:%s, perform update action", pOldDb->name); pOldDb->updateTime = pNewDb->createdTime; - memcpy(&pOldDb->cfg, &pNewDb->cfg, sizeof(SDbObj)); + memcpy(&pOldDb->cfg, &pNewDb->cfg, sizeof(SDbCfg)); return 0; } @@ -166,109 +166,146 @@ void mndReleaseDb(SMnode *pMnode, SDbObj *pDb) { sdbRelease(pSdb, pDb); } -static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg) { +static int32_t mndCheckDbName(char *dbName, SUserObj *pUser) { + char *pos = strstr(dbName, TS_PATH_DELIMITER); + if (pos == NULL) { + terrno = TSDB_CODE_MND_INVALID_DB; + return -1; + } + + int32_t acctId = atoi(dbName); + if (acctId != pUser->acctId) { + terrno = TSDB_CODE_MND_INVALID_DB_ACCT; + return -1; + } + + return 0; +} + +static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg, char *errMsg, int32_t len) { if (pCfg->cacheBlockSize < TSDB_MIN_CACHE_BLOCK_SIZE || pCfg->cacheBlockSize > TSDB_MAX_CACHE_BLOCK_SIZE) { - terrno = TSDB_CODE_MND_INVALID_DB_CACHE_SIZE; + terrno = TSDB_CODE_MND_INVALID_DB_OPTION; + tstrncpy(errMsg, "Invalid database cache block size option", len); return -1; } if (pCfg->totalBlocks < TSDB_MIN_TOTAL_BLOCKS || pCfg->totalBlocks > TSDB_MAX_TOTAL_BLOCKS) { - terrno = TSDB_CODE_MND_INVALID_DB_TOTAL_BLOCKS; + terrno = TSDB_CODE_MND_INVALID_DB_OPTION; + tstrncpy(errMsg, "Invalid database total blocks option", len); return -1; } if (pCfg->daysPerFile < TSDB_MIN_DAYS_PER_FILE || pCfg->daysPerFile > TSDB_MAX_DAYS_PER_FILE) { - terrno = TSDB_CODE_MND_INVALID_DB_DAYS; + terrno = TSDB_CODE_MND_INVALID_DB_OPTION; + tstrncpy(errMsg, "Invalid database days option", len); return -1; } if (pCfg->daysToKeep0 < pCfg->daysPerFile) { - terrno = TSDB_CODE_MND_INVALID_DB_KEEP0; + terrno = TSDB_CODE_MND_INVALID_DB_OPTION; + tstrncpy(errMsg, "Invalid database days option", len); return -1; } if (pCfg->daysToKeep0 < TSDB_MIN_KEEP || pCfg->daysToKeep0 > TSDB_MAX_KEEP || pCfg->daysToKeep0 > pCfg->daysToKeep1) { - terrno = TSDB_CODE_MND_INVALID_DB_KEEP0; + terrno = TSDB_CODE_MND_INVALID_DB_OPTION; + tstrncpy(errMsg, "Invalid database keep0 option", len); return -1; } if (pCfg->daysToKeep1 < TSDB_MIN_KEEP || pCfg->daysToKeep1 > TSDB_MAX_KEEP || pCfg->daysToKeep1 > pCfg->daysToKeep2) { - terrno = TSDB_CODE_MND_INVALID_DB_KEEP1; + terrno = TSDB_CODE_MND_INVALID_DB_OPTION; + tstrncpy(errMsg, "Invalid database keep1 option", len); return -1; } if (pCfg->daysToKeep2 < TSDB_MIN_KEEP || pCfg->daysToKeep2 > TSDB_MAX_KEEP) { - terrno = TSDB_CODE_MND_INVALID_DB_KEEP1; + terrno = TSDB_CODE_MND_INVALID_DB_OPTION; + tstrncpy(errMsg, "Invalid database keep2 option", len); return -1; } if (pCfg->minRowsPerFileBlock < TSDB_MIN_MIN_ROW_FBLOCK || pCfg->minRowsPerFileBlock > TSDB_MAX_MIN_ROW_FBLOCK) { - terrno = TSDB_CODE_MND_INVALID_DB_MIN_ROWS; + terrno = TSDB_CODE_MND_INVALID_DB_OPTION; + tstrncpy(errMsg, "Invalid database minrows option", len); return -1; } if (pCfg->maxRowsPerFileBlock < TSDB_MIN_MAX_ROW_FBLOCK || pCfg->maxRowsPerFileBlock > TSDB_MAX_MAX_ROW_FBLOCK) { - terrno = TSDB_CODE_MND_INVALID_DB_MAX_ROWS; + terrno = TSDB_CODE_MND_INVALID_DB_OPTION; + tstrncpy(errMsg, "Invalid database maxrows option", len); return -1; } if (pCfg->minRowsPerFileBlock > pCfg->maxRowsPerFileBlock) { - terrno = TSDB_CODE_MND_INVALID_DB_MIN_ROWS; + terrno = TSDB_CODE_MND_INVALID_DB_OPTION; + tstrncpy(errMsg, "Invalid database minrows option", len); return -1; } if (pCfg->commitTime < TSDB_MIN_COMMIT_TIME || pCfg->commitTime > TSDB_MAX_COMMIT_TIME) { - terrno = TSDB_CODE_MND_INVALID_DB_COMMIT_TIME; + terrno = TSDB_CODE_MND_INVALID_DB_OPTION; + tstrncpy(errMsg, "Invalid database commit option", len); return -1; } if (pCfg->fsyncPeriod < TSDB_MIN_FSYNC_PERIOD || pCfg->fsyncPeriod > TSDB_MAX_FSYNC_PERIOD) { - terrno = TSDB_CODE_MND_INVALID_DB_FSYNC_PERIOD; + terrno = TSDB_CODE_MND_INVALID_DB_OPTION; + tstrncpy(errMsg, "Invalid database fsync option", len); return -1; } if (pCfg->walLevel < TSDB_MIN_WAL_LEVEL || pCfg->walLevel > TSDB_MAX_WAL_LEVEL) { - terrno = TSDB_CODE_MND_INVALID_DB_WAL_LEVEL; + terrno = TSDB_CODE_MND_INVALID_DB_OPTION; + tstrncpy(errMsg, "Invalid database wal level option", len); return -1; } if (pCfg->precision < TSDB_MIN_PRECISION && pCfg->precision > TSDB_MAX_PRECISION) { - terrno = TSDB_CODE_MND_INVALID_DB_PRECISION; + terrno = TSDB_CODE_MND_INVALID_DB_OPTION; + tstrncpy(errMsg, "Invalid precision option", len); return -1; } if (pCfg->compression < TSDB_MIN_COMP_LEVEL || pCfg->compression > TSDB_MAX_COMP_LEVEL) { - terrno = TSDB_CODE_MND_INVALID_DB_COMP; + terrno = TSDB_CODE_MND_INVALID_DB_OPTION; + tstrncpy(errMsg, "Invalid database compression option", len); return -1; } if (pCfg->replications < TSDB_MIN_DB_REPLICA_OPTION || pCfg->replications > TSDB_MAX_DB_REPLICA_OPTION) { - terrno = TSDB_CODE_MND_INVALID_DB_REPLICA; + terrno = TSDB_CODE_MND_INVALID_DB_OPTION; + tstrncpy(errMsg, "Invalid database replication option", len); return -1; } if (pCfg->replications > mndGetDnodeSize(pMnode)) { - terrno = TSDB_CODE_MND_INVALID_DB_REPLICA; + terrno = TSDB_CODE_MND_INVALID_DB_OPTION; + tstrncpy(errMsg, "Invalid database replication option", len); return -1; } if (pCfg->quorum < TSDB_MIN_DB_QUORUM_OPTION || pCfg->quorum > TSDB_MAX_DB_QUORUM_OPTION) { - terrno = TSDB_CODE_MND_INVALID_DB_QUORUM; + terrno = TSDB_CODE_MND_INVALID_DB_OPTION; + tstrncpy(errMsg, "Invalid database quorum option", len); return -1; } if (pCfg->quorum > pCfg->replications) { - terrno = TSDB_CODE_MND_INVALID_DB_QUORUM; + terrno = TSDB_CODE_MND_INVALID_DB_OPTION; + tstrncpy(errMsg, "Invalid database quorum option", len); return -1; } if (pCfg->update < TSDB_MIN_DB_UPDATE || pCfg->update > TSDB_MAX_DB_UPDATE) { - terrno = TSDB_CODE_MND_INVALID_DB_UPDATE; + terrno = TSDB_CODE_MND_INVALID_DB_OPTION; + tstrncpy(errMsg, "Invalid database update option", len); return -1; } if (pCfg->cacheLastRow < TSDB_MIN_DB_CACHE_LAST_ROW || pCfg->cacheLastRow > TSDB_MAX_DB_CACHE_LAST_ROW) { - terrno = TSDB_CODE_MND_INVALID_DB_CACHE_LAST; + terrno = TSDB_CODE_MND_INVALID_DB_OPTION; + tstrncpy(errMsg, "Invalid database cachelast option", len); return -1; } @@ -295,10 +332,10 @@ static void mndSetDefaultDbCfg(SDbCfg *pCfg) { if (pCfg->cacheLastRow < 0) pCfg->cacheLastRow = TSDB_DEFAULT_CACHE_LAST_ROW; } -static int32_t mndCreateDb(SMnode *pMnode, SMnodeMsg *pMsg, SCreateDbMsg *pCreate, char *acct) { +static int32_t mndCreateDb(SMnode *pMnode, SMnodeMsg *pMsg, SCreateDbMsg *pCreate, SUserObj *pUser) { SDbObj dbObj = {0}; tstrncpy(dbObj.name, pCreate->db, TSDB_FULL_DB_NAME_LEN); - tstrncpy(dbObj.acct, acct, TSDB_USER_LEN); + tstrncpy(dbObj.acct, pUser->acct, TSDB_USER_LEN); dbObj.createdTime = taosGetTimestampMs(); dbObj.updateTime = dbObj.createdTime; dbObj.uid = mndGenerateUid(dbObj.name, TSDB_FULL_DB_NAME_LEN); @@ -322,7 +359,13 @@ static int32_t mndCreateDb(SMnode *pMnode, SMnodeMsg *pMsg, SCreateDbMsg *pCreat mndSetDefaultDbCfg(&dbObj.cfg); - if (mndCheckDbCfg(pMnode, &dbObj.cfg) != 0) { + if (mndCheckDbName(dbObj.name, pUser) != 0) { + mError("db:%s, failed to create since %s", pCreate->db, terrstr()); + return -1; + } + + char errMsg[TSDB_ERROR_MSG_LEN] = {0}; + if (mndCheckDbCfg(pMnode, &dbObj.cfg, errMsg, TSDB_ERROR_MSG_LEN) != 0) { mError("db:%s, failed to create since %s", pCreate->db, terrstr()); return -1; } @@ -404,7 +447,7 @@ static int32_t mndProcessCreateDbMsg(SMnodeMsg *pMsg) { return -1; } - int32_t code = mndCreateDb(pMnode, pMsg, pCreate, pOperUser->acct); + int32_t code = mndCreateDb(pMnode, pMsg, pCreate, pOperUser); mndReleaseUser(pMnode, pOperUser); if (code != 0) { @@ -776,6 +819,10 @@ char *mnGetDbStr(char *src) { char *pos = strstr(src, TS_PATH_DELIMITER); if (pos != NULL) ++pos; + if (pos == NULL) { + return src; + } + return pos; } diff --git a/source/util/src/terror.c b/source/util/src/terror.c index b8520821fc..70a3dc622f 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -207,23 +207,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_DATABASES, "Too many databases fo TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_NOT_SELECTED, "Database not specified or available") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB, "Invalid database name") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_OPTION, "Invalid database options") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_CACHE_SIZE, "Invalid database cache block size option") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_TOTAL_BLOCKS, "Invalid database total blocks option") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_DAYS, "Invalid database days option") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_KEEP0, "Invalid database keep0 option") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_KEEP1, "Invalid database keep1 option") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_KEEP2, "Invalid database keep2 option") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_MIN_ROWS, "Invalid database min rows option") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_MAX_ROWS, "Invalid database max rows option") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_COMMIT_TIME, "Invalid database commit time option") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_FSYNC_PERIOD, "Invalid database fsync periodoptions") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_WAL_LEVEL, "Invalid database wal level option") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_PRECISION, "Invalid database precisin option") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_COMP, "Invalid database compression option") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_REPLICA, "Invalid database replication option") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_QUORUM, "Invalid database quorum option") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_UPDATE, "Invalid database update option") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_CACHE_LAST, "Invalid database cache last option") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_ACCT, "Invalid database account") TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_OPTION_UNCHANGED, "Database options not changed") // mnode-vgroup From ac7a543b3e57c656c2d89aa7cfdf74f540949d0c Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 15 Dec 2021 16:56:56 +0800 Subject: [PATCH 43/43] TD-10431 acct test --- source/dnode/mgmt/impl/test/CMakeLists.txt | 2 +- .../dnode/mgmt/impl/test/acct/CMakeLists.txt | 12 +- source/dnode/mgmt/impl/test/acct/acct.cpp | 23 +-- source/dnode/mgmt/impl/test/db/db.cpp | 142 ++++++++++-------- source/dnode/mgmt/impl/test/dnode/dnode.cpp | 14 +- source/dnode/mgmt/impl/test/user/user.cpp | 10 +- source/dnode/mnode/impl/src/mndDb.c | 1 - 7 files changed, 109 insertions(+), 95 deletions(-) diff --git a/source/dnode/mgmt/impl/test/CMakeLists.txt b/source/dnode/mgmt/impl/test/CMakeLists.txt index 80055ee013..86186f77de 100644 --- a/source/dnode/mgmt/impl/test/CMakeLists.txt +++ b/source/dnode/mgmt/impl/test/CMakeLists.txt @@ -1,5 +1,5 @@ -# add_subdirectory(acct) +add_subdirectory(acct) # add_subdirectory(auth) # add_subdirectory(balance) # add_subdirectory(cluster) diff --git a/source/dnode/mgmt/impl/test/acct/CMakeLists.txt b/source/dnode/mgmt/impl/test/acct/CMakeLists.txt index 3e963df2e6..a548c2adc2 100644 --- a/source/dnode/mgmt/impl/test/acct/CMakeLists.txt +++ b/source/dnode/mgmt/impl/test/acct/CMakeLists.txt @@ -1,20 +1,20 @@ -add_executable(dndTestAcct "") +add_executable(dnode_test_acct "") -target_sources(dndTestAcct +target_sources(dnode_test_acct PRIVATE "acct.cpp" "../sut/deploy.cpp" ) target_link_libraries( - dndTestAcct + dnode_test_acct PUBLIC dnode PUBLIC util PUBLIC os PUBLIC gtest_main ) -target_include_directories(dndTestAcct +target_include_directories(dnode_test_acct PUBLIC "${CMAKE_SOURCE_DIR}/include/server/dnode/mgmt" "${CMAKE_CURRENT_SOURCE_DIR}/../../inc" @@ -24,6 +24,6 @@ target_include_directories(dndTestAcct enable_testing() add_test( - NAME dndTestAcct - COMMAND dndTestAcct + NAME dnode_test_acct + COMMAND dnode_test_acct ) diff --git a/source/dnode/mgmt/impl/test/acct/acct.cpp b/source/dnode/mgmt/impl/test/acct/acct.cpp index 9050a938fa..e1a71c5c9d 100644 --- a/source/dnode/mgmt/impl/test/acct/acct.cpp +++ b/source/dnode/mgmt/impl/test/acct/acct.cpp @@ -17,24 +17,27 @@ class DndTestAcct : public ::testing::Test { protected: - void SetUp() override {} - void TearDown() override {} + static SServer* CreateServer(const char* path, const char* fqdn, uint16_t port, const char* firstEp) { + SServer* pServer = createServer(path, fqdn, port, firstEp); + ASSERT(pServer); + return pServer; + } static void SetUpTestSuite() { - const char* user = "root"; - const char* pass = "taosdata"; - const char* path = "/tmp/dndTestAcct"; - const char* fqdn = "localhost"; - uint16_t port = 9520; + initLog("/tmp/tdlog"); - pServer = createServer(path, fqdn, port); - ASSERT(pServer); - pClient = createClient(user, pass, fqdn, port); + const char* fqdn = "localhost"; + const char* firstEp = "localhost:9012"; + pServer = CreateServer("/tmp/dnode_test_user", fqdn, 9012, firstEp); + pClient = createClient("root", "taosdata", fqdn, 9012); + taosMsleep(300); } static void TearDownTestSuite() { stopServer(pServer); dropClient(pClient); + pServer = NULL; + pClient = NULL; } static SServer* pServer; diff --git a/source/dnode/mgmt/impl/test/db/db.cpp b/source/dnode/mgmt/impl/test/db/db.cpp index 4fa5e43883..b0ee00cb64 100644 --- a/source/dnode/mgmt/impl/test/db/db.cpp +++ b/source/dnode/mgmt/impl/test/db/db.cpp @@ -182,7 +182,7 @@ SServer* DndTestDb::pServer; SClient* DndTestDb::pClient; int32_t DndTestDb::connId; -TEST_F(DndTestDb, ShowDb) { +TEST_F(DndTestDb, 01_ShowDb) { SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DB, "show databases", 16); CheckSchema(0, TSDB_DATA_TYPE_BINARY, TSDB_DB_NAME_LEN - 1 + VARSTR_HEADER_SIZE, "name"); CheckSchema(1, TSDB_DATA_TYPE_TIMESTAMP, 8, "create time"); @@ -204,7 +204,7 @@ TEST_F(DndTestDb, ShowDb) { SendThenCheckShowRetrieveMsg(0); } -TEST_F(DndTestDb, CreateDb_01) { +TEST_F(DndTestDb, 02_CreateDb) { { SCreateDbMsg* pReq = (SCreateDbMsg*)rpcMallocCont(sizeof(SCreateDbMsg)); strcpy(pReq->db, "1.d1"); @@ -258,65 +258,51 @@ TEST_F(DndTestDb, CreateDb_01) { CheckInt8(0); // update } -#if 0 -TEST_F(DndTestDb, AlterUser_01) { - SAlterUserMsg* pReq = (SAlterUserMsg*)rpcMallocCont(sizeof(SAlterUserMsg)); - strcpy(pReq->user, "u1"); - strcpy(pReq->pass, "p2"); +TEST_F(DndTestDb, 03_AlterDb) { + { + SAlterDbMsg* pReq = (SAlterDbMsg*)rpcMallocCont(sizeof(SAlterDbMsg)); + strcpy(pReq->db, "1.d1"); + pReq->totalBlocks = htonl(12); + pReq->daysToKeep0 = htonl(300); + pReq->daysToKeep1 = htonl(400); + pReq->daysToKeep2 = htonl(500); + pReq->fsyncPeriod = htonl(4000); + pReq->walLevel = 2; + pReq->quorum = 2; + pReq->cacheLastRow = 1; - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SAlterUserMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_ALTER_USER; + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pReq; + rpcMsg.contLen = sizeof(SAlterDbMsg); + rpcMsg.msgType = TSDB_MSG_TYPE_ALTER_DB; - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, 0); + sendMsg(pClient, &rpcMsg); + SRpcMsg* pMsg = pClient->pRsp; + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); + } - SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_USER, "show users", 4); - SendThenCheckShowRetrieveMsg(3); - CheckBinary("u1", TSDB_USER_LEN); - CheckBinary("root", TSDB_USER_LEN); - CheckBinary("u2", TSDB_USER_LEN); - CheckBinary("normal", 10); - CheckBinary("super", 10); - CheckBinary("normal", 10); + SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DB, "show databases", 16); + SendThenCheckShowRetrieveMsg(1); + CheckBinary("d1", TSDB_DB_NAME_LEN - 1); CheckTimestamp(); - CheckTimestamp(); - CheckTimestamp(); - CheckBinary("root", TSDB_USER_LEN); - CheckBinary("root", TSDB_USER_LEN); - CheckBinary("root", TSDB_USER_LEN); + CheckInt16(1); // replica + CheckInt16(2); // quorum + CheckInt16(10); // days + CheckBinary("300,400,500", 24); // days + CheckInt32(16); // cache + CheckInt32(12); // blocks + CheckInt32(100); // minrows + CheckInt32(4096); // maxrows + CheckInt8(2); // wallevel + CheckInt32(4000); // fsync + CheckInt8(2); // comp + CheckInt8(1); // cachelast + CheckBinary("ms", 3); // precision + CheckInt8(0); // update } -TEST_F(DndTestDb, DropUser_01) { - SDropUserMsg* pReq = (SDropUserMsg*)rpcMallocCont(sizeof(SDropUserMsg)); - strcpy(pReq->user, "u1"); - - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SDropUserMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_DROP_USER; - - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, 0); - - SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_USER, "show users", 4); - SendThenCheckShowRetrieveMsg(2); - CheckBinary("root", TSDB_USER_LEN); - CheckBinary("u2", TSDB_USER_LEN); - CheckBinary("super", 10); - CheckBinary("normal", 10); - CheckTimestamp(); - CheckTimestamp(); - CheckBinary("root", TSDB_USER_LEN); - CheckBinary("root", TSDB_USER_LEN); -} - -TEST_F(DndTestDb, RestartDnode) { +TEST_F(DndTestDb, 04_RestartDnode) { stopServer(pServer); pServer = NULL; @@ -328,16 +314,42 @@ TEST_F(DndTestDb, RestartDnode) { uInfo("all server is running"); - SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_USER, "show users", 4); - SendThenCheckShowRetrieveMsg(2); - CheckBinary("root", TSDB_USER_LEN); - CheckBinary("u2", TSDB_USER_LEN); - CheckBinary("super", 10); - CheckBinary("normal", 10); + SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DB, "show databases", 16); + SendThenCheckShowRetrieveMsg(1); + CheckBinary("d1", TSDB_DB_NAME_LEN - 1); CheckTimestamp(); - CheckTimestamp(); - CheckBinary("root", TSDB_USER_LEN); - CheckBinary("root", TSDB_USER_LEN); + CheckInt16(1); // replica + CheckInt16(2); // quorum + CheckInt16(10); // days + CheckBinary("300,400,500", 24); // days + CheckInt32(16); // cache + CheckInt32(12); // blocks + CheckInt32(100); // minrows + CheckInt32(4096); // maxrows + CheckInt8(2); // wallevel + CheckInt32(4000); // fsync + CheckInt8(2); // comp + CheckInt8(1); // cachelast + CheckBinary("ms", 3); // precision + CheckInt8(0); // update } -#endif \ No newline at end of file +TEST_F(DndTestDb, 05_DropDb) { + { + SDropDbMsg* pReq = (SDropDbMsg*)rpcMallocCont(sizeof(SAlterDbMsg)); + strcpy(pReq->db, "1.d1"); + + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pReq; + rpcMsg.contLen = sizeof(SDropDbMsg); + rpcMsg.msgType = TSDB_MSG_TYPE_DROP_DB; + + sendMsg(pClient, &rpcMsg); + SRpcMsg* pMsg = pClient->pRsp; + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); + } + + SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DB, "show databases", 16); + SendThenCheckShowRetrieveMsg(0); +} \ No newline at end of file diff --git a/source/dnode/mgmt/impl/test/dnode/dnode.cpp b/source/dnode/mgmt/impl/test/dnode/dnode.cpp index c2b122d054..5a4512e16c 100644 --- a/source/dnode/mgmt/impl/test/dnode/dnode.cpp +++ b/source/dnode/mgmt/impl/test/dnode/dnode.cpp @@ -24,7 +24,7 @@ class DndTestDnode : public ::testing::Test { } static void SetUpTestSuite() { - initLog("/tmp/dnode_test_dnode"); + initLog("/tmp/tdlog"); const char* fqdn = "localhost"; const char* firstEp = "localhost:9041"; @@ -188,7 +188,7 @@ SServer* DndTestDnode::pServer4; SServer* DndTestDnode::pServer5; SClient* DndTestDnode::pClient; -TEST_F(DndTestDnode, ShowDnode) { +TEST_F(DndTestDnode, 01_ShowDnode) { SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DNODE, "show dnodes", 7); CheckSchema(0, TSDB_DATA_TYPE_SMALLINT, 2, "id"); CheckSchema(1, TSDB_DATA_TYPE_BINARY, TSDB_EP_LEN + VARSTR_HEADER_SIZE, "end point"); @@ -208,7 +208,7 @@ TEST_F(DndTestDnode, ShowDnode) { CheckBinary("", 24); } -TEST_F(DndTestDnode, ConfigDnode_01) { +TEST_F(DndTestDnode, 02_ConfigDnode) { SCfgDnodeMsg* pReq = (SCfgDnodeMsg*)rpcMallocCont(sizeof(SCfgDnodeMsg)); pReq->dnodeId = htonl(1); strcpy(pReq->config, "ddebugflag 131"); @@ -224,7 +224,7 @@ TEST_F(DndTestDnode, ConfigDnode_01) { ASSERT_EQ(pMsg->code, 0); } -TEST_F(DndTestDnode, CreateDnode_01) { +TEST_F(DndTestDnode, 03_CreateDnode) { SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(sizeof(SCreateDnodeMsg)); strcpy(pReq->ep, "localhost:9042"); @@ -257,7 +257,7 @@ TEST_F(DndTestDnode, CreateDnode_01) { CheckBinary("", 24); } -TEST_F(DndTestDnode, DropDnode_01) { +TEST_F(DndTestDnode, 04_DropDnode) { SDropDnodeMsg* pReq = (SDropDnodeMsg*)rpcMallocCont(sizeof(SDropDnodeMsg)); pReq->dnodeId = htonl(2); @@ -282,7 +282,7 @@ TEST_F(DndTestDnode, DropDnode_01) { CheckBinary("", 24); } -TEST_F(DndTestDnode, CreateDnode_02) { +TEST_F(DndTestDnode, 05_CreateDnode) { { SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(sizeof(SCreateDnodeMsg)); strcpy(pReq->ep, "localhost:9043"); @@ -361,7 +361,7 @@ TEST_F(DndTestDnode, CreateDnode_02) { CheckBinary("", 24); } -TEST_F(DndTestDnode, RestartDnode_01) { +TEST_F(DndTestDnode, 06_RestartDnode) { uInfo("stop all server"); stopServer(pServer1); stopServer(pServer2); diff --git a/source/dnode/mgmt/impl/test/user/user.cpp b/source/dnode/mgmt/impl/test/user/user.cpp index 6a48a2cb49..2b30dcbb1b 100644 --- a/source/dnode/mgmt/impl/test/user/user.cpp +++ b/source/dnode/mgmt/impl/test/user/user.cpp @@ -170,7 +170,7 @@ SServer* DndTestUser::pServer; SClient* DndTestUser::pClient; int32_t DndTestUser::connId; -TEST_F(DndTestUser, ShowUser) { +TEST_F(DndTestUser, 01_ShowUser) { SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_USER, "show users", 4); CheckSchema(0, TSDB_DATA_TYPE_BINARY, TSDB_USER_LEN + VARSTR_HEADER_SIZE, "name"); CheckSchema(1, TSDB_DATA_TYPE_BINARY, 10 + VARSTR_HEADER_SIZE, "privilege"); @@ -184,7 +184,7 @@ TEST_F(DndTestUser, ShowUser) { CheckBinary("root", TSDB_USER_LEN); } -TEST_F(DndTestUser, CreateUser_01) { +TEST_F(DndTestUser, 02_CreateUser) { { SCreateUserMsg* pReq = (SCreateUserMsg*)rpcMallocCont(sizeof(SCreateUserMsg)); strcpy(pReq->user, "u1"); @@ -233,7 +233,7 @@ TEST_F(DndTestUser, CreateUser_01) { CheckBinary("root", TSDB_USER_LEN); } -TEST_F(DndTestUser, AlterUser_01) { +TEST_F(DndTestUser, 03_AlterUser) { SAlterUserMsg* pReq = (SAlterUserMsg*)rpcMallocCont(sizeof(SAlterUserMsg)); strcpy(pReq->user, "u1"); strcpy(pReq->pass, "p2"); @@ -264,7 +264,7 @@ TEST_F(DndTestUser, AlterUser_01) { CheckBinary("root", TSDB_USER_LEN); } -TEST_F(DndTestUser, DropUser_01) { +TEST_F(DndTestUser, 04_DropUser) { SDropUserMsg* pReq = (SDropUserMsg*)rpcMallocCont(sizeof(SDropUserMsg)); strcpy(pReq->user, "u1"); @@ -290,7 +290,7 @@ TEST_F(DndTestUser, DropUser_01) { CheckBinary("root", TSDB_USER_LEN); } -TEST_F(DndTestUser, RestartDnode) { +TEST_F(DndTestUser, 05_RestartDnode) { stopServer(pServer); pServer = NULL; diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 6426c3f33f..1cf9423da8 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -646,7 +646,6 @@ static int32_t mndProcessDropDbMsg(SMnodeMsg *pMsg) { mndReleaseDb(pMnode, pDb); if (code != 0) { - terrno = code; mError("db:%s, failed to drop since %s", pDrop->db, terrstr()); return code; }