diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 19923a5ad6..e797911add 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -110,7 +110,7 @@ execute_process(COMMAND "${CMAKE_COMMAND}" --build . # ================================================================================================ # googletest if(${BUILD_TEST}) - add_subdirectory(googletest) + add_subdirectory(googletest EXCLUDE_FROM_ALL) target_include_directories( gtest PUBLIC $ @@ -143,7 +143,7 @@ set(CMAKE_PROJECT_INCLUDE_BEFORE "${TD_SUPPORT_DIR}/EnableCMP0048.txt.in") option(ENABLE_CJSON_TEST "Enable building cJSON test" OFF) option(CJSON_OVERRIDE_BUILD_SHARED_LIBS "Override BUILD_SHARED_LIBS with CJSON_BUILD_SHARED_LIBS" ON) option(CJSON_BUILD_SHARED_LIBS "Overrides BUILD_SHARED_LIBS if CJSON_OVERRIDE_BUILD_SHARED_LIBS is enabled" OFF) -add_subdirectory(cJson) +add_subdirectory(cJson EXCLUDE_FROM_ALL) target_include_directories( cjson # see https://stackoverflow.com/questions/25676277/cmake-target-include-directories-prints-an-error-when-i-try-to-add-the-source @@ -152,7 +152,7 @@ target_include_directories( unset(CMAKE_PROJECT_INCLUDE_BEFORE) # lz4 -add_subdirectory(lz4/build/cmake) +add_subdirectory(lz4/build/cmake EXCLUDE_FROM_ALL) target_include_directories( lz4_static PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/lz4/lib @@ -160,7 +160,7 @@ target_include_directories( # zlib set(CMAKE_PROJECT_INCLUDE_BEFORE "${TD_SUPPORT_DIR}/EnableCMP0048.txt.in") -add_subdirectory(zlib) +add_subdirectory(zlib EXCLUDE_FROM_ALL) target_include_directories( zlibstatic PUBLIC ${CMAKE_CURRENT_BINARY_DIR}/zlib @@ -176,7 +176,7 @@ unset(CMAKE_PROJECT_INCLUDE_BEFORE) # leveldb if(${BUILD_WITH_LEVELDB}) option(LEVELDB_BUILD_TESTS "" OFF) - add_subdirectory(leveldb) + add_subdirectory(leveldb EXCLUDE_FROM_ALL) target_include_directories( leveldb PUBLIC $ @@ -192,7 +192,7 @@ if(${BUILD_WITH_ROCKSDB}) option(WITH_TOOLS "" OFF) option(WITH_LIBURING "" OFF) option(ROCKSDB_BUILD_SHARED "Build shared versions of the RocksDB libraries" OFF) - add_subdirectory(rocksdb) + add_subdirectory(rocksdb EXCLUDE_FROM_ALL) target_include_directories( rocksdb PUBLIC $ @@ -203,7 +203,7 @@ endif(${BUILD_WITH_ROCKSDB}) # To support build on ubuntu: sudo apt-get install libboost-all-dev if(${BUILD_WITH_LUCENE}) option(ENABLE_TEST "Enable the tests" OFF) - add_subdirectory(lucene) + add_subdirectory(lucene EXCLUDE_FROM_ALL) target_include_directories( lucene++ PUBLIC $ @@ -213,13 +213,13 @@ endif(${BUILD_WITH_LUCENE}) # NuRaft if(${BUILD_WITH_NURAFT}) - add_subdirectory(nuraft) + add_subdirectory(nuraft EXCLUDE_FROM_ALL) endif(${BUILD_WITH_NURAFT}) # pthread if(${BUILD_PTHREAD}) set(CMAKE_BUILD_TYPE release) - add_definitions(-DPTW32_STATIC_LIB) + add_definitions(-DPTW32_STATIC_LIB) add_subdirectory(pthread) set_target_properties(libpthreadVC3 PROPERTIES OUTPUT_NAME pthread) add_library(pthread STATIC IMPORTED GLOBAL) @@ -228,12 +228,12 @@ endif() # iconv if(${BUILD_WITH_ICONV}) - add_subdirectory(iconv) + add_subdirectory(iconv EXCLUDE_FROM_ALL) endif(${BUILD_WITH_ICONV}) # wingetopt if(${BUILD_WINGETOPT}) - add_subdirectory(wingetopt) + add_subdirectory(wingetopt EXCLUDE_FROM_ALL) endif(${BUILD_WINGETOPT}) # msvcregex @@ -293,7 +293,7 @@ if(${BUILD_WITH_UV}) MESSAGE("Windows need set no-sign-compare") add_compile_options(-Wno-sign-compare) endif () - add_subdirectory(libuv) + add_subdirectory(libuv EXCLUDE_FROM_ALL) endif(${BUILD_WITH_UV}) # BDB @@ -334,5 +334,5 @@ endif(${BUILD_WITH_SQLITE}) # Build test # ================================================================================================ if(${BUILD_DEPENDENCY_TESTS}) - add_subdirectory(test) + add_subdirectory(test EXCLUDE_FROM_ALL) endif(${BUILD_DEPENDENCY_TESTS}) diff --git a/include/client/taos.h b/include/client/taos.h index 0f1633ed6f..1ac92c61a5 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -121,13 +121,14 @@ typedef struct setConfRet { DLL_EXPORT void taos_cleanup(void); DLL_EXPORT int taos_options(TSDB_OPTION option, const void *arg, ...); DLL_EXPORT setConfRet taos_set_config(const char *config); +DLL_EXPORT int taos_init(void); DLL_EXPORT TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port); -DLL_EXPORT TAOS *taos_connect_l(const char *ip, int ipLen, const char *user, int userLen, const char *pass, int passLen, +DLL_EXPORT TAOS *taos_connect_l(const char *ip, int ipLen, const char *user, int userLen, const char *pass, int passLen, const char *db, int dbLen, uint16_t port); -DLL_EXPORT TAOS *taos_connect_auth(const char *ip, const char *user, const char *auth, const char *db, uint16_t port); -DLL_EXPORT void taos_close(TAOS *taos); +DLL_EXPORT TAOS *taos_connect_auth(const char *ip, const char *user, const char *auth, const char *db, uint16_t port); +DLL_EXPORT void taos_close(TAOS *taos); -const char *taos_data_type(int type); +const char *taos_data_type(int type); DLL_EXPORT TAOS_STMT *taos_stmt_init(TAOS *taos); DLL_EXPORT int taos_stmt_prepare(TAOS_STMT *stmt, const char *sql, unsigned long length); diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 05536d0f83..da48846a8f 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -438,6 +438,7 @@ typedef struct { int32_t tSerializeSGetUserAuthRsp(void* buf, int32_t bufLen, SGetUserAuthRsp* pRsp); int32_t tDeserializeSGetUserAuthRsp(void* buf, int32_t bufLen, SGetUserAuthRsp* pRsp); +void tFreeSGetUserAuthRsp(SGetUserAuthRsp* pRsp); typedef struct { int16_t colId; // column id @@ -1332,7 +1333,7 @@ int32_t tDeserializeSCMCreateTopicRsp(void* buf, int32_t bufLen, SCMCreateTopicR typedef struct { int64_t consumerId; -} SMqConsumerLostMsg; +} SMqConsumerLostMsg, SMqConsumerRecoverMsg; typedef struct { int64_t consumerId; @@ -1537,6 +1538,10 @@ int tDecodeSVCreateStbReq(SCoder* pCoder, SVCreateStbReq* pReq); typedef struct SVDropStbReq { // data +#ifdef WINDOWS + size_t avoidCompilationErrors; +#endif + } SVDropStbReq; typedef struct SVCreateStbRsp { @@ -2116,7 +2121,7 @@ static FORCE_INLINE int32_t tDecodeSSchemaWrapper(SCoder* pDecoder, SSchemaWrapp if (tDecodeI32v(pDecoder, &pSW->nCols) < 0) return -1; if (tDecodeI32v(pDecoder, &pSW->sver) < 0) return -1; - pSW->pSchema = (SSchema*)TCODER_MALLOC(pDecoder, sizeof(SSchema) * pSW->nCols); + pSW->pSchema = (SSchema*)tCoderMalloc(pDecoder, sizeof(SSchema) * pSW->nCols); if (pSW->pSchema == NULL) return -1; for (int32_t i = 0; i < pSW->nCols; i++) { if (tDecodeSSchema(pDecoder, &pSW->pSchema[i]) < 0) return -1; diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index 1976db3a12..e1ee946ce4 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -145,10 +145,11 @@ enum { TD_DEF_MSG_TYPE(TDMT_MND_ALTER_TOPIC, "mnode-alter-topic", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_DROP_TOPIC, "mnode-drop-topic", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_SUBSCRIBE, "mnode-subscribe", SCMSubscribeReq, SCMSubscribeRsp) - TD_DEF_MSG_TYPE(TDMT_MND_MQ_ASK_EP, "mnode-mq-ask-ep", SMqAskEpReq, SMqAskEpReq) - TD_DEF_MSG_TYPE(TDMT_MND_MQ_TIMER, "mnode-mq-tmr", SMTimerReq, SMTimerReq) - TD_DEF_MSG_TYPE(TDMT_MND_MQ_CONSUMER_LOST, "mnode-mq-consumer-lost", SMTimerReq, SMTimerReq) - TD_DEF_MSG_TYPE(TDMT_MND_MQ_DO_REBALANCE, "mnode-mq-do-rebalance", SMqDoRebalanceMsg, SMqDoRebalanceMsg) + TD_DEF_MSG_TYPE(TDMT_MND_MQ_ASK_EP, "mnode-mq-ask-ep", SMqAskEpReq, SMqAskEpRsp) + TD_DEF_MSG_TYPE(TDMT_MND_MQ_TIMER, "mnode-mq-tmr", SMTimerReq, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_MQ_CONSUMER_LOST, "mnode-mq-consumer-lost", SMqConsumerLostMsg, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_MQ_CONSUMER_RECOVER, "mnode-mq-consumer-recover", SMqConsumerRecoverMsg, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_MQ_DO_REBALANCE, "mnode-mq-do-rebalance", SMqDoRebalanceMsg, NULL) TD_DEF_MSG_TYPE(TDMT_MND_MQ_COMMIT_OFFSET, "mnode-mq-commit-offset", SMqCMCommitOffsetReq, SMqCMCommitOffsetRsp) TD_DEF_MSG_TYPE(TDMT_MND_CREATE_STREAM, "mnode-create-stream", SCMCreateStreamReq, SCMCreateStreamRsp) TD_DEF_MSG_TYPE(TDMT_MND_ALTER_STREAM, "mnode-alter-stream", NULL, NULL) diff --git a/include/dnode/mnode/sdb/sdb.h b/include/dnode/mnode/sdb/sdb.h index d534b0405f..1613339d10 100644 --- a/include/dnode/mnode/sdb/sdb.h +++ b/include/dnode/mnode/sdb/sdb.h @@ -18,6 +18,11 @@ #include "os.h" +#include "thash.h" +#include "tlockfree.h" +#include "tlog.h" +#include "tmsg.h" + #ifdef __cplusplus extern "C" { #endif @@ -135,7 +140,7 @@ typedef enum { typedef struct SSdb SSdb; typedef int32_t (*SdbInsertFp)(SSdb *pSdb, void *pObj); typedef int32_t (*SdbUpdateFp)(SSdb *pSdb, void *pSrcObj, void *pDstObj); -typedef int32_t (*SdbDeleteFp)(SSdb *pSdb, void *pObj); +typedef int32_t (*SdbDeleteFp)(SSdb *pSdb, void *pObj, bool callFunc); typedef int32_t (*SdbDeployFp)(SMnode *pMnode); typedef SSdbRow *(*SdbDecodeFp)(SSdbRaw *pRaw); typedef SSdbRaw *(*SdbEncodeFp)(void *pObj); @@ -326,9 +331,29 @@ int32_t sdbGetRawSoftVer(SSdbRaw *pRaw, int8_t *sver); int32_t sdbGetRawTotalSize(SSdbRaw *pRaw); SSdbRow *sdbAllocRow(int32_t objSize); -void sdbFreeRow(SSdb *pSdb, SSdbRow *pRow); +void sdbFreeRow(SSdb *pSdb, SSdbRow *pRow, bool callFunc); void *sdbGetRowObj(SSdbRow *pRow); +typedef struct SSdb { + SMnode *pMnode; + char *currDir; + char *syncDir; + char *tmpDir; + int64_t lastCommitVer; + int64_t curVer; + int64_t tableVer[SDB_MAX]; + int64_t maxId[SDB_MAX]; + EKeyType keyTypes[SDB_MAX]; + SHashObj *hashObjs[SDB_MAX]; + SRWLatch locks[SDB_MAX]; + SdbInsertFp insertFps[SDB_MAX]; + SdbUpdateFp updateFps[SDB_MAX]; + SdbDeleteFp deleteFps[SDB_MAX]; + SdbDeployFp deployFps[SDB_MAX]; + SdbEncodeFp encodeFps[SDB_MAX]; + SdbDecodeFp decodeFps[SDB_MAX]; +} SSdb; + #ifdef __cplusplus } #endif diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index 4d289147d0..0cd1a9265d 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -32,6 +32,7 @@ typedef struct SReadHandle { void* reader; void* meta; void* config; + void* vnode; } SReadHandle; #define STREAM_DATA_TYPE_SUBMIT_BLOCK 0x1 diff --git a/include/os/os.h b/include/os/os.h index 86abcc15f5..329ad481aa 100644 --- a/include/os/os.h +++ b/include/os/os.h @@ -59,6 +59,7 @@ extern "C" { #include #endif +#define __typeof(a) auto #endif diff --git a/include/os/osMath.h b/include/os/osMath.h index cabb821844..829bbd847b 100644 --- a/include/os/osMath.h +++ b/include/os/osMath.h @@ -23,27 +23,21 @@ extern "C" { #define TPOW2(x) ((x) * (x)) #define TABS(x) ((x) > 0 ? (x) : -(x)) +#define TSWAP(a, b) \ + do { \ + __typeof(a) __tmp = (a); \ + (a) = (b); \ + (b) = __tmp; \ + } while (0) + #ifdef WINDOWS - #define TSWAP(a, b, c) \ - do { \ - c __tmp = (c)(a); \ - (a) = (c)(b); \ - (b) = __tmp; \ - } while (0) #define TMAX(a, b) (((a) > (b)) ? (a) : (b)) #define TMIN(a, b) (((a) < (b)) ? (a) : (b)) #define TRANGE(aa, bb, cc) ((aa) = TMAX((aa), (bb)),(aa) = TMIN((aa), (cc))) #else - #define TSWAP(a, b, c) \ - do { \ - __typeof(a) __tmp = (a); \ - (a) = (b); \ - (b) = __tmp; \ - } while (0) - #define TMAX(a, b) \ ({ \ __typeof(a) __a = (a); \ @@ -51,12 +45,12 @@ extern "C" { (__a > __b) ? __a : __b; \ }) - #define TMIN(a, b) \ - ({ \ - __typeof(a) __a = (a); \ - __typeof(b) __b = (b); \ - (__a < __b) ? __a : __b; \ - }) +#define TMIN(a, b) \ + ({ \ + __typeof(a) __a = (a); \ + __typeof(b) __b = (b); \ + (__a < __b) ? __a : __b; \ + }) #define TRANGE(a, b, c) \ ({ \ diff --git a/include/os/osSocket.h b/include/os/osSocket.h index a47239089a..62c3771669 100644 --- a/include/os/osSocket.h +++ b/include/os/osSocket.h @@ -19,53 +19,53 @@ // If the error is in a third-party library, place this header file under the third-party library header file. // When you want to use this feature, you should find or add the same function in the following section. #ifndef ALLOW_FORBID_FUNC - #define socket SOCKET_FUNC_TAOS_FORBID - #define bind BIND_FUNC_TAOS_FORBID - #define listen LISTEN_FUNC_TAOS_FORBID - #define accept ACCEPT_FUNC_TAOS_FORBID - #define epoll_create EPOLL_CREATE_FUNC_TAOS_FORBID - #define epoll_ctl EPOLL_CTL_FUNC_TAOS_FORBID - #define epoll_wait EPOLL_WAIT_FUNC_TAOS_FORBID - #define inet_addr INET_ADDR_FUNC_TAOS_FORBID - #define inet_ntoa INET_NTOA_FUNC_TAOS_FORBID +#define socket SOCKET_FUNC_TAOS_FORBID +#define bind BIND_FUNC_TAOS_FORBID +#define listen LISTEN_FUNC_TAOS_FORBID +#define accept ACCEPT_FUNC_TAOS_FORBID +#define epoll_create EPOLL_CREATE_FUNC_TAOS_FORBID +#define epoll_ctl EPOLL_CTL_FUNC_TAOS_FORBID +#define epoll_wait EPOLL_WAIT_FUNC_TAOS_FORBID +#define inet_addr INET_ADDR_FUNC_TAOS_FORBID +#define inet_ntoa INET_NTOA_FUNC_TAOS_FORBID #endif #if defined(WINDOWS) - #if BYTE_ORDER == LITTLE_ENDIAN - #include - #define htobe16(x) _byteswap_ushort(x) - #define htole16(x) (x) - #define be16toh(x) _byteswap_ushort(x) - #define le16toh(x) (x) - - #define htobe32(x) _byteswap_ulong(x) - #define htole32(x) (x) - #define be32toh(x) _byteswap_ulong(x) - #define le32toh(x) (x) - - #define htobe64(x) _byteswap_uint64(x) - #define htole64(x) (x) - #define be64toh(x) _byteswap_uint64(x) - #define le64toh(x) (x) - #else - #error byte order not supported - #endif +#if BYTE_ORDER == LITTLE_ENDIAN +#include +#define htobe16(x) _byteswap_ushort(x) +#define htole16(x) (x) +#define be16toh(x) _byteswap_ushort(x) +#define le16toh(x) (x) - #define __BYTE_ORDER BYTE_ORDER - #define __BIG_ENDIAN BIG_ENDIAN - #define __LITTLE_ENDIAN LITTLE_ENDIAN - #define __PDP_ENDIAN PDP_ENDIAN +#define htobe32(x) _byteswap_ulong(x) +#define htole32(x) (x) +#define be32toh(x) _byteswap_ulong(x) +#define le32toh(x) (x) + +#define htobe64(x) _byteswap_uint64(x) +#define htole64(x) (x) +#define be64toh(x) _byteswap_uint64(x) +#define le64toh(x) (x) +#else +#error byte order not supported +#endif + +#define __BYTE_ORDER BYTE_ORDER +#define __BIG_ENDIAN BIG_ENDIAN +#define __LITTLE_ENDIAN LITTLE_ENDIAN +#define __PDP_ENDIAN PDP_ENDIAN #else - #include - #include +#include +#include - #if defined(_TD_DARWIN_64) - #include - #else - #include - #include - #endif +#if defined(_TD_DARWIN_64) +#include +#else +#include +#include +#endif #endif #ifdef __cplusplus @@ -73,24 +73,24 @@ extern "C" { #endif #if defined(WINDOWS) - typedef int socklen_t; - #define TAOS_EPOLL_WAIT_TIME 100 - typedef SOCKET eventfd_t; - #define eventfd(a, b) -1 - #define EpollClose(pollFd) epoll_close(pollFd) - #ifndef EPOLLWAKEUP - #define EPOLLWAKEUP (1u << 29) - #endif +typedef int socklen_t; +#define TAOS_EPOLL_WAIT_TIME 100 +typedef SOCKET eventfd_t; +#define eventfd(a, b) -1 +#define EpollClose(pollFd) epoll_close(pollFd) +#ifndef EPOLLWAKEUP +#define EPOLLWAKEUP (1u << 29) +#endif #elif defined(_TD_DARWIN_64) - #define TAOS_EPOLL_WAIT_TIME 500 - typedef int32_t SOCKET; - typedef SOCKET EpollFd; - #define EpollClose(pollFd) epoll_close(pollFd) +#define TAOS_EPOLL_WAIT_TIME 500 +typedef int32_t SOCKET; +typedef SOCKET EpollFd; +#define EpollClose(pollFd) epoll_close(pollFd) #else - #define TAOS_EPOLL_WAIT_TIME 500 - typedef int32_t SOCKET; - typedef SOCKET EpollFd; - #define EpollClose(pollFd) taosCloseSocket(pollFd) +#define TAOS_EPOLL_WAIT_TIME 500 +typedef int32_t SOCKET; +typedef SOCKET EpollFd; +#define EpollClose(pollFd) taosCloseSocket(pollFd) #endif #if defined(_TD_DARWIN_64) @@ -119,8 +119,8 @@ extern "C" { #define __PDP_ENDIAN PDP_ENDIAN #endif -typedef int32_t SocketFd; -typedef SocketFd EpollFd; +typedef int32_t SocketFd; +typedef SocketFd EpollFd; typedef struct TdSocket { #if SOCKET_WITH_LOCK @@ -128,16 +128,17 @@ typedef struct TdSocket { #endif int refId; SocketFd fd; -} *TdSocketPtr, TdSocket; +} * TdSocketPtr, TdSocket; typedef struct TdSocketServer *TdSocketServerPtr; -typedef struct TdSocket *TdSocketPtr; -typedef struct TdEpoll *TdEpollPtr; +typedef struct TdSocket * TdSocketPtr; +typedef struct TdEpoll * TdEpollPtr; -int32_t taosSendto(TdSocketPtr pSocket, void * msg, int len, unsigned int flags, const struct sockaddr * to, int tolen); +int32_t taosSendto(TdSocketPtr pSocket, void *msg, int len, unsigned int flags, const struct sockaddr *to, int tolen); int32_t taosWriteSocket(TdSocketPtr pSocket, void *msg, int len); int32_t taosReadSocket(TdSocketPtr pSocket, void *msg, int len); -int32_t taosReadFromSocket(TdSocketPtr pSocket, void *buf, int32_t len, int32_t flags, struct sockaddr *destAddr, int *addrLen); +int32_t taosReadFromSocket(TdSocketPtr pSocket, void *buf, int32_t len, int32_t flags, struct sockaddr *destAddr, + int *addrLen); int32_t taosCloseSocketNoCheck1(SocketFd fd); int32_t taosCloseSocket(TdSocketPtr *ppSocket); int32_t taosCloseSocketServer(TdSocketServerPtr *ppSocketServer); @@ -154,30 +155,32 @@ int32_t taosWriteMsg(TdSocketPtr pSocket, void *ptr, int32_t nbytes); int32_t taosReadMsg(TdSocketPtr pSocket, void *ptr, int32_t nbytes); int32_t taosNonblockwrite(TdSocketPtr pSocket, char *ptr, int32_t nbytes); int64_t taosCopyFds(TdSocketPtr pSrcSocket, TdSocketPtr pDestSocket, int64_t len); -void taosWinSocketInit(); +void taosWinSocketInit(); -TdSocketPtr taosOpenUdpSocket(uint32_t localIp, uint16_t localPort); -TdSocketPtr taosOpenTcpClientSocket(uint32_t ip, uint16_t port, uint32_t localIp); +int taosCreateSocketWithTimeOutOpt(uint32_t conn_timeout_sec); + +TdSocketPtr taosOpenUdpSocket(uint32_t localIp, uint16_t localPort); +TdSocketPtr taosOpenTcpClientSocket(uint32_t ip, uint16_t port, uint32_t localIp); TdSocketServerPtr taosOpenTcpServerSocket(uint32_t ip, uint16_t port); -int32_t taosKeepTcpAlive(TdSocketPtr pSocket); -TdSocketPtr taosAcceptTcpConnectSocket(TdSocketServerPtr pServerSocket, struct sockaddr *destAddr, int *addrLen); +int32_t taosKeepTcpAlive(TdSocketPtr pSocket); +TdSocketPtr taosAcceptTcpConnectSocket(TdSocketServerPtr pServerSocket, struct sockaddr *destAddr, int *addrLen); -int32_t taosGetSocketName(TdSocketPtr pSocket,struct sockaddr *destAddr, int *addrLen); +int32_t taosGetSocketName(TdSocketPtr pSocket, struct sockaddr *destAddr, int *addrLen); -void taosBlockSIGPIPE(); -uint32_t taosGetIpv4FromFqdn(const char *); -int32_t taosGetFqdn(char *); -void tinet_ntoa(char *ipstr, uint32_t ip); -uint32_t ip2uint(const char *const ip_addr); -void taosIgnSIGPIPE(); -void taosSetMaskSIGPIPE(); -uint32_t taosInetAddr(const char *ipAddr); +void taosBlockSIGPIPE(); +uint32_t taosGetIpv4FromFqdn(const char *); +int32_t taosGetFqdn(char *); +void tinet_ntoa(char *ipstr, uint32_t ip); +uint32_t ip2uint(const char *const ip_addr); +void taosIgnSIGPIPE(); +void taosSetMaskSIGPIPE(); +uint32_t taosInetAddr(const char *ipAddr); const char *taosInetNtoa(struct in_addr ipInt); TdEpollPtr taosCreateEpoll(int32_t size); -int32_t taosCtlEpoll(TdEpollPtr pEpoll, int32_t epollOperate, TdSocketPtr pSocket, struct epoll_event *event); -int32_t taosWaitEpoll(TdEpollPtr pEpoll, struct epoll_event *event, int32_t maxEvents, int32_t timeout); -int32_t taosCloseEpoll(TdEpollPtr *ppEpoll); +int32_t taosCtlEpoll(TdEpollPtr pEpoll, int32_t epollOperate, TdSocketPtr pSocket, struct epoll_event *event); +int32_t taosWaitEpoll(TdEpollPtr pEpoll, struct epoll_event *event, int32_t maxEvents, int32_t timeout); +int32_t taosCloseEpoll(TdEpollPtr *ppEpoll); #ifdef __cplusplus } diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 2c249a2d8d..0ddc5b91d3 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -62,6 +62,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_APP_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x0014) #define TSDB_CODE_RPC_FQDN_ERROR TAOS_DEF_ERROR_CODE(0, 0x0015) #define TSDB_CODE_RPC_INVALID_VERSION TAOS_DEF_ERROR_CODE(0, 0x0016) +#define TSDB_CODE_RPC_PORT_EADDRINUSE TAOS_DEF_ERROR_CODE(0, 0x0017) //common & util #define TSDB_CODE_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0100) @@ -268,14 +269,13 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_TOPIC_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03E1) #define TSDB_CODE_MND_TOO_MANY_TOPICS TAOS_DEF_ERROR_CODE(0, 0x03E2) #define TSDB_CODE_MND_INVALID_TOPIC TAOS_DEF_ERROR_CODE(0, 0x03E3) -#define TSDB_CODE_MND_INVALID_TOPIC_OPTION TAOS_DEF_ERROR_CODE(0, 0x03E4) -#define TSDB_CODE_MND_TOPIC_OPTION_UNCHNAGED TAOS_DEF_ERROR_CODE(0, 0x03E5) -#define TSDB_CODE_MND_NAME_CONFLICT_WITH_STB TAOS_DEF_ERROR_CODE(0, 0x03E6) -#define TSDB_CODE_MND_CONSUMER_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03E7) -#define TSDB_CODE_MND_UNSUPPORTED_TOPIC TAOS_DEF_ERROR_CODE(0, 0x03E8) -#define TSDB_CODE_MND_SUBSCRIBE_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03E9) -#define TSDB_CODE_MND_OFFSET_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03EA) -#define TSDB_CODE_MND_CONSUMER_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x03EB) +#define TSDB_CODE_MND_INVALID_TOPIC_QUERY TAOS_DEF_ERROR_CODE(0, 0x03E4) +#define TSDB_CODE_MND_INVALID_TOPIC_OPTION TAOS_DEF_ERROR_CODE(0, 0x03E5) +#define TSDB_CODE_MND_CONSUMER_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03E6) +#define TSDB_CODE_MND_TOPIC_OPTION_UNCHNAGED TAOS_DEF_ERROR_CODE(0, 0x03E7) +#define TSDB_CODE_MND_SUBSCRIBE_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03E8) +#define TSDB_CODE_MND_OFFSET_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03E9) +#define TSDB_CODE_MND_CONSUMER_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x03EA) // mnode-stream #define TSDB_CODE_MND_STREAM_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x03F0) diff --git a/include/util/tarray.h b/include/util/tarray.h index a41bcd9349..bbde90f28f 100644 --- a/include/util/tarray.h +++ b/include/util/tarray.h @@ -205,7 +205,6 @@ SArray* taosArrayDup(const SArray* pSrc); */ SArray* taosArrayDeepCopy(const SArray* pSrc, FCopy deepCopy); - /** * clear the array (remove all element) * @param pArray @@ -272,6 +271,8 @@ void taosArraySortPWithExt(SArray* pArray, __ext_compar_fn_t fn, const void* par int32_t taosEncodeArray(void** buf, const SArray* pArray, FEncode encode); void* taosDecodeArray(const void* buf, SArray** pArray, FDecode decode, int32_t dataSz); +char* taosShowStrArray(const SArray* pArray); + #ifdef __cplusplus } #endif diff --git a/include/util/tdef.h b/include/util/tdef.h index 7a1fe5dd7b..cf0c75e58f 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -131,6 +131,7 @@ extern const int32_t TYPE_BYTES[15]; #define TSDB_PERFS_TABLE_TOPICS "topics" #define TSDB_PERFS_TABLE_CONSUMERS "consumers" #define TSDB_PERFS_TABLE_SUBSCRIPTIONS "subscriptions" +#define TSDB_PERFS_TABLE_OFFSETS "offsets" #define TSDB_INDEX_TYPE_SMA "SMA" #define TSDB_INDEX_TYPE_FULLTEXT "FULLTEXT" diff --git a/include/util/tencode.h b/include/util/tencode.h index 0236fe58da..c5066996f3 100644 --- a/include/util/tencode.h +++ b/include/util/tencode.h @@ -79,31 +79,52 @@ typedef struct { #define TD_CODER_CURRENT(CODER) ((CODER)->data + (CODER)->pos) #define TD_CODER_MOVE_POS(CODER, MOVE) ((CODER)->pos += (MOVE)) #define TD_CODER_CHECK_CAPACITY_FAILED(CODER, EXPSIZE) (((CODER)->size - (CODER)->pos) < (EXPSIZE)) -#define TCODER_MALLOC(PCODER, SIZE) \ - ({ \ - void* ptr = NULL; \ - SCoderMem* pMem = (SCoderMem*)taosMemoryMalloc(sizeof(*pMem) + (SIZE)); \ - if (pMem) { \ - pMem->next = (PCODER)->mList; \ - (PCODER)->mList = pMem; \ - ptr = (void*)&pMem[1]; \ - } \ - ptr; \ - }) +// #define TCODER_MALLOC(PCODER, SIZE) \ +// ({ \ +// void* ptr = NULL; \ +// SCoderMem* pMem = (SCoderMem*)taosMemoryMalloc(sizeof(*pMem) + (SIZE)); \ +// if (pMem) { \ +// pMem->next = (PCODER)->mList; \ +// (PCODER)->mList = pMem; \ +// ptr = (void*)&pMem[1]; \ +// } \ +// ptr; \ +// }) +static FORCE_INLINE void* tCoderMalloc(SCoder* pCoder, int32_t size) { + void* ptr = NULL; + SCoderMem* pMem = (SCoderMem*)taosMemoryMalloc(sizeof(SCoderMem*) + size); + if (pMem) { + pMem->next = pCoder->mList; + pCoder->mList = pMem; + ptr = (void*)&pMem[1]; + } + return ptr; +} -#define tEncodeSize(E, S, SIZE) \ - ({ \ +#define tEncodeSize(E, S, SIZE, RET) \ + do{ \ SCoder coder = {0}; \ - int ret = 0; \ tCoderInit(&coder, TD_LITTLE_ENDIAN, NULL, 0, TD_ENCODER); \ if ((E)(&coder, S) == 0) { \ SIZE = coder.pos; \ } else { \ - ret = -1; \ + RET = -1; \ } \ tCoderClear(&coder); \ - ret; \ - }) + }while(0) +// #define tEncodeSize(E, S, SIZE) \ +// ({ \ +// SCoder coder = {0}; \ +// int ret = 0; \ +// tCoderInit(&coder, TD_LITTLE_ENDIAN, NULL, 0, TD_ENCODER); \ +// if ((E)(&coder, S) == 0) { \ +// SIZE = coder.pos; \ +// } else { \ +// ret = -1; \ +// } \ +// tCoderClear(&coder); \ +// ret; \ +// }) void tCoderInit(SCoder* pCoder, td_endian_t endian, uint8_t* data, int32_t size, td_coder_t type); void tCoderClear(SCoder* pCoder); diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index 41448a4391..0ff542358d 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -254,8 +254,6 @@ extern int (*handleRequestRspFp[TDMT_MAX])(void*, const SDataBuf* pMsg, int32_t int genericRspCallback(void* param, const SDataBuf* pMsg, int32_t code); SMsgSendInfo* buildMsgInfoImpl(SRequestObj* pReqObj); -int taos_init(); - void* createTscObj(const char* user, const char* auth, const char* db, SAppInstInfo* pAppInfo); void destroyTscObj(void* pObj); STscObj* acquireTscObj(int64_t rid); diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 7c873acadb..b876f9b65c 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -103,6 +103,7 @@ TAOS* taos_connect_internal(const char* ip, const char* user, const char* pass, if (port) { epSet.epSet.eps[0].port = port; + epSet.epSet.eps[1].port = port; } char* key = getClusterKey(user, secretEncrypt, ip, port); @@ -187,8 +188,8 @@ int32_t parseSql(SRequestObj* pRequest, bool topicQuery, SQuery** pQuery, SStmtC setResPrecision(&pRequest->body.resInfo, (*pQuery)->precision); } - TSWAP(pRequest->dbList, (*pQuery)->pDbList, SArray*); - TSWAP(pRequest->tableList, (*pQuery)->pTableList, SArray*); + TSWAP(pRequest->dbList, (*pQuery)->pDbList); + TSWAP(pRequest->tableList, (*pQuery)->pTableList); } return code; @@ -245,7 +246,6 @@ void setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t ASSERT(pSchema != NULL && numOfCols > 0); pResInfo->numOfCols = numOfCols; - // TODO handle memory leak if (pResInfo->fields != NULL) { taosMemoryFree(pResInfo->fields); } @@ -358,8 +358,15 @@ SRequestObj* launchQuery(STscObj* pTscObj, const char* sql, int sqlLen) { SQuery* pQuery = NULL; int32_t code = buildRequest(pTscObj, sql, sqlLen, &pRequest); - if (TSDB_CODE_SUCCESS == code) { - code = parseSql(pRequest, false, &pQuery, NULL); + if (code != TSDB_CODE_SUCCESS) { + terrno = code; + return NULL; + } + + code = parseSql(pRequest, false, &pQuery, NULL); + if (code != TSDB_CODE_SUCCESS) { + pRequest->code = code; + return pRequest; } return launchQueryImpl(pRequest, pQuery, code, false); @@ -410,7 +417,7 @@ SRequestObj* execQuery(STscObj* pTscObj, const char* sql, int sqlLen) { while (retryNum++ < REQUEST_MAX_TRY_TIMES) { pRequest = launchQuery(pTscObj, sql, sqlLen); - if (TSDB_CODE_SUCCESS == pRequest->code || !NEED_CLIENT_HANDLE_ERROR(pRequest->code)) { + if (pRequest == NULL || TSDB_CODE_SUCCESS == pRequest->code || !NEED_CLIENT_HANDLE_ERROR(pRequest->code)) { break; } diff --git a/source/client/src/taos.def b/source/client/src/taos.def index bc78b241d2..994dd75090 100644 --- a/source/client/src/taos.def +++ b/source/client/src/taos.def @@ -1,6 +1,7 @@ taos_cleanup taos_options taos_set_config +taos_init taos_connect taos_connect_l taos_connect_auth diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index d0f6a296d9..b03947e2ca 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -666,7 +666,6 @@ tmq_resp_err_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { code = param.rspErr; if (code != 0) goto FAIL; - // TODO: add max retry cnt while (TSDB_CODE_MND_CONSUMER_NOT_READY == tmqAskEp(tmq, false)) { tscDebug("not ready, retry"); taosMsleep(500); @@ -683,7 +682,7 @@ tmq_resp_err_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { code = 0; FAIL: if (req.topicNames != NULL) taosArrayDestroyP(req.topicNames, taosMemoryFree); - if (code != 0) { + if (code != 0 && buf) { taosMemoryFree(buf); } return code; @@ -1265,6 +1264,7 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t wait_time) { return (TAOS_RES*)rspObj; } + // in no topic status also need process delayed task if (atomic_load_8(&tmq->status) == TMQ_CONSUMER_STATUS__INIT) { return NULL; } @@ -1285,6 +1285,9 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t wait_time) { return NULL; } tsem_timewait(&tmq->rspSem, leftTime * 1000); + } else { + // use tsem_timewait instead of tsem_wait to avoid unexpected stuck + tsem_timewait(&tmq->rspSem, 500 * 1000); } } } diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index d6a6ba5b13..63e009ed0a 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -117,22 +117,23 @@ int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, con int32_t type = pColumnInfoData->info.type; if (IS_VAR_DATA_TYPE(type)) { int32_t dataLen = varDataTLen(pData); - if(type == TSDB_DATA_TYPE_JSON) { - if(*pData == TSDB_DATA_TYPE_NULL) { + if (type == TSDB_DATA_TYPE_JSON) { + if (*pData == TSDB_DATA_TYPE_NULL) { dataLen = 0; - }else if(*pData == TSDB_DATA_TYPE_NCHAR) { - dataLen = varDataTLen(pData+CHAR_BYTES); - }else if(*pData == TSDB_DATA_TYPE_BIGINT || *pData == TSDB_DATA_TYPE_DOUBLE) { + } else if (*pData == TSDB_DATA_TYPE_NCHAR) { + dataLen = varDataTLen(pData + CHAR_BYTES); + } else if (*pData == TSDB_DATA_TYPE_BIGINT || *pData == TSDB_DATA_TYPE_DOUBLE) { dataLen = LONG_BYTES; - }else if(*pData == TSDB_DATA_TYPE_BOOL) { + } else if (*pData == TSDB_DATA_TYPE_BOOL) { dataLen = CHAR_BYTES; } dataLen += CHAR_BYTES; } + SVarColAttr* pAttr = &pColumnInfoData->varmeta; if (pAttr->allocLen < pAttr->length + dataLen) { uint32_t newSize = pAttr->allocLen; - if (newSize == 0) { + if (newSize <= 1) { newSize = 8; } diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 3e7d9dbdec..afab703e77 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -1302,6 +1302,11 @@ int32_t tDeserializeSGetUserAuthRsp(void *buf, int32_t bufLen, SGetUserAuthRsp * return 0; } +void tFreeSGetUserAuthRsp(SGetUserAuthRsp *pRsp) { + taosHashCleanup(pRsp->readDbs); + taosHashCleanup(pRsp->writeDbs); +} + int32_t tSerializeSCreateDropMQSBNodeReq(void *buf, int32_t bufLen, SMCreateQnodeReq *pReq) { SCoder encoder = {0}; tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); @@ -3226,7 +3231,7 @@ int32_t tEncodeSMqCMCommitOffsetReq(SCoder *encoder, const SMqCMCommitOffsetReq int32_t tDecodeSMqCMCommitOffsetReq(SCoder *decoder, SMqCMCommitOffsetReq *pReq) { if (tStartDecode(decoder) < 0) return -1; if (tDecodeI32(decoder, &pReq->num) < 0) return -1; - pReq->offsets = (SMqOffset *)TCODER_MALLOC(decoder, sizeof(SMqOffset) * pReq->num); + pReq->offsets = (SMqOffset *)tCoderMalloc(decoder, sizeof(SMqOffset) * pReq->num); if (pReq->offsets == NULL) return -1; for (int32_t i = 0; i < pReq->num; i++) { tDecodeSMqOffset(decoder, &pReq->offsets[i]); @@ -3509,7 +3514,7 @@ int tDecodeSVCreateTbBatchRsp(SCoder *pCoder, SVCreateTbBatchRsp *pRsp) { if (tStartDecode(pCoder) < 0) return -1; if (tDecodeI32v(pCoder, &pRsp->nRsps) < 0) return -1; - pRsp->pRsps = (SVCreateTbRsp *)TCODER_MALLOC(pCoder, sizeof(*pRsp->pRsps) * pRsp->nRsps); + pRsp->pRsps = (SVCreateTbRsp *)tCoderMalloc(pCoder, sizeof(*pRsp->pRsps) * pRsp->nRsps); for (int32_t i = 0; i < pRsp->nRsps; i++) { if (tDecodeSVCreateTbRsp(pCoder, pRsp->pRsps + i) < 0) return -1; } @@ -3775,7 +3780,7 @@ int tDecodeSVCreateTbBatchReq(SCoder *pCoder, SVCreateTbBatchReq *pReq) { if (tStartDecode(pCoder) < 0) return -1; if (tDecodeI32v(pCoder, &pReq->nReqs) < 0) return -1; - pReq->pReqs = (SVCreateTbReq *)TCODER_MALLOC(pCoder, sizeof(SVCreateTbReq) * pReq->nReqs); + pReq->pReqs = (SVCreateTbReq *)tCoderMalloc(pCoder, sizeof(SVCreateTbReq) * pReq->nReqs); if (pReq->pReqs == NULL) return -1; for (int iReq = 0; iReq < pReq->nReqs; iReq++) { if (tDecodeSVCreateTbReq(pCoder, pReq->pReqs + iReq) < 0) return -1; diff --git a/source/common/src/ttypes.c b/source/common/src/ttypes.c index 1c7a7986e9..e3d67cd488 100644 --- a/source/common/src/ttypes.c +++ b/source/common/src/ttypes.c @@ -651,35 +651,35 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void *buf switch (type) { case TSDB_DATA_TYPE_INT: case TSDB_DATA_TYPE_UINT: { - TSWAP(*(int32_t *)(pLeft), *(int32_t *)(pRight), int32_t); + TSWAP(*(int32_t *)(pLeft), *(int32_t *)(pRight)); break; } case TSDB_DATA_TYPE_BIGINT: case TSDB_DATA_TYPE_UBIGINT: case TSDB_DATA_TYPE_TIMESTAMP: { - TSWAP(*(int64_t *)(pLeft), *(int64_t *)(pRight), int64_t); + TSWAP(*(int64_t *)(pLeft), *(int64_t *)(pRight)); break; } case TSDB_DATA_TYPE_DOUBLE: { - TSWAP(*(double *)(pLeft), *(double *)(pRight), double); + TSWAP(*(double *)(pLeft), *(double *)(pRight)); break; } case TSDB_DATA_TYPE_SMALLINT: case TSDB_DATA_TYPE_USMALLINT: { - TSWAP(*(int16_t *)(pLeft), *(int16_t *)(pRight), int16_t); + TSWAP(*(int16_t *)(pLeft), *(int16_t *)(pRight)); break; } case TSDB_DATA_TYPE_FLOAT: { - TSWAP(*(float *)(pLeft), *(float *)(pRight), float); + TSWAP(*(float *)(pLeft), *(float *)(pRight)); break; } case TSDB_DATA_TYPE_BOOL: case TSDB_DATA_TYPE_TINYINT: case TSDB_DATA_TYPE_UTINYINT: { - TSWAP(*(int8_t *)(pLeft), *(int8_t *)(pRight), int8_t); + TSWAP(*(int8_t *)(pLeft), *(int8_t *)(pRight)); break; } diff --git a/source/dnode/mgmt/exe/dmMain.c b/source/dnode/mgmt/exe/dmMain.c index 6d9ead4c87..4a2d02d25d 100644 --- a/source/dnode/mgmt/exe/dmMain.c +++ b/source/dnode/mgmt/exe/dmMain.c @@ -69,8 +69,8 @@ static void dmSetSignalHandle() { static int32_t dmParseArgs(int32_t argc, char const *argv[]) { int32_t cmdEnvIndex = 0; if (argc < 2) return 0; - global.envCmd = taosMemoryMalloc(argc-1); - memset(global.envCmd, 0, argc-1); + global.envCmd = taosMemoryMalloc((argc-1)*sizeof(char*)); + memset(global.envCmd, 0, (argc-1)*sizeof(char*)); for (int32_t i = 1; i < argc; ++i) { if (strcmp(argv[i], "-c") == 0) { if (i < argc - 1) { diff --git a/source/dnode/mgmt/implement/src/dmEps.c b/source/dnode/mgmt/implement/src/dmEps.c index ae1cd513b5..853c238316 100644 --- a/source/dnode/mgmt/implement/src/dmEps.c +++ b/source/dnode/mgmt/implement/src/dmEps.c @@ -120,7 +120,7 @@ int32_t dmReadEps(SDnode *pDnode) { goto PRASE_DNODE_OVER; } - dnodeEp.id = dnodeId->valueint; + dnodeEp.id = did->valueint; cJSON *dnodeFqdn = cJSON_GetObjectItem(node, "fqdn"); if (!dnodeFqdn || dnodeFqdn->type != cJSON_String || dnodeFqdn->valuestring == NULL) { @@ -156,11 +156,6 @@ PRASE_DNODE_OVER: if (root != NULL) cJSON_Delete(root); if (pFile != NULL) taosCloseFile(&pFile); - if (dmIsEpChanged(pDnode, pDnode->data.dnodeId, pDnode->data.localEp)) { - dError("localEp %s different with %s and need reconfigured", pDnode->data.localEp, file); - return -1; - } - if (taosArrayGetSize(pDnode->data.dnodeEps) == 0) { SDnodeEp dnodeEp = {0}; dnodeEp.isMnode = 1; @@ -170,6 +165,11 @@ PRASE_DNODE_OVER: dmResetEps(pDnode, pDnode->data.dnodeEps); + if (dmIsEpChanged(pDnode, pDnode->data.dnodeId, pDnode->data.localEp)) { + dError("localEp %s different with %s and need reconfigured", pDnode->data.localEp, file); + return -1; + } + terrno = code; return code; } @@ -291,13 +291,17 @@ static void dmPrintEps(SDnode *pDnode) { static bool dmIsEpChanged(SDnode *pDnode, int32_t dnodeId, const char *ep) { bool changed = false; + if (dnodeId == 0) return changed; taosRLockLatch(&pDnode->data.latch); SDnodeEp *pDnodeEp = taosHashGet(pDnode->data.dnodeHash, &dnodeId, sizeof(int32_t)); if (pDnodeEp != NULL) { - char epstr[TSDB_EP_LEN + 1]; + char epstr[TSDB_EP_LEN + 1] = {0}; snprintf(epstr, TSDB_EP_LEN, "%s:%u", pDnodeEp->ep.fqdn, pDnodeEp->ep.port); - changed = strcmp(ep, epstr) != 0; + changed = (strcmp(ep, epstr) != 0); + if (changed) { + dError("dnode:%d, localEp %s different from %s", dnodeId, ep, epstr); + } } taosRUnLockLatch(&pDnode->data.latch); diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index 2da98a41df..185aa071d7 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -114,7 +114,6 @@ static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) { pCfg->tsdbCfg.keep2 = 3650; // pCreate->daysToKeep0; pCfg->tsdbCfg.keep0 = 3650; // pCreate->daysToKeep2; pCfg->tsdbCfg.keep1 = 3650; // pCreate->daysToKeep0; - pCfg->tsdbCfg.lruCacheSize = pCreate->cacheBlockSize; pCfg->tsdbCfg.retentions = pCreate->pRetensions; pCfg->walCfg.vgId = pCreate->vgId; pCfg->hashBegin = pCreate->hashBegin; diff --git a/source/dnode/mnode/impl/inc/mndConsumer.h b/source/dnode/mnode/impl/inc/mndConsumer.h index a818f5d7c3..19e202dddc 100644 --- a/source/dnode/mnode/impl/inc/mndConsumer.h +++ b/source/dnode/mnode/impl/inc/mndConsumer.h @@ -23,10 +23,8 @@ extern "C" { #endif enum { - // MQ_CONSUMER_STATUS__INIT = 1, MQ_CONSUMER_STATUS__MODIFY = 1, MQ_CONSUMER_STATUS__MODIFY_IN_REB, - // MQ_CONSUMER_STATUS__IDLE, MQ_CONSUMER_STATUS__READY, MQ_CONSUMER_STATUS__LOST, MQ_CONSUMER_STATUS__LOST_IN_REB, diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 65cc4353e8..055327570e 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -89,6 +89,7 @@ typedef enum { TRN_TYPE_DROP_STREAM = 1020, TRN_TYPE_ALTER_STREAM = 1021, TRN_TYPE_CONSUMER_LOST = 1022, + TRN_TYPE_CONSUMER_RECOVER = 1023, TRN_TYPE_BASIC_SCOPE_END, TRN_TYPE_GLOBAL_SCOPE = 2000, TRN_TYPE_CREATE_DNODE = 2001, @@ -126,8 +127,6 @@ typedef enum { DND_REASON_OTHERS } EDndReason; -typedef void (*TransCbFp)(SMnode* pMnode, void* param); - typedef struct { int32_t id; ETrnStage stage; @@ -150,8 +149,10 @@ typedef struct { int64_t dbUid; char dbname[TSDB_DB_FNAME_LEN]; char lastError[TSDB_TRANS_ERROR_LEN]; - TransCbFp transCbFp; - void* transCbParam; + int32_t startFunc; + int32_t stopFunc; + int32_t paramLen; + void* param; } STrans; typedef struct { @@ -463,12 +464,14 @@ enum { CONSUMER_UPDATE__ADD, CONSUMER_UPDATE__REMOVE, CONSUMER_UPDATE__LOST, + CONSUMER_UPDATE__RECOVER, CONSUMER_UPDATE__MODIFY, }; typedef struct { int64_t consumerId; char cgroup[TSDB_CGROUP_LEN]; + char appId[TSDB_CGROUP_LEN]; int8_t updateType; // used only for update int32_t epoch; int32_t status; @@ -479,6 +482,17 @@ typedef struct { SArray* currentTopics; // SArray SArray* rebNewTopics; // SArray SArray* rebRemovedTopics; // SArray + + // subscribed by user + SArray* assignedTopics; // SArray + + // data for display + int32_t pid; + SEpSet ep; + int64_t upTime; + int64_t subscribeTime; + int64_t rebalanceTime; + } SMqConsumerObj; SMqConsumerObj* tNewSMqConsumerObj(int64_t consumerId, char cgroup[TSDB_CGROUP_LEN]); diff --git a/source/dnode/mnode/impl/inc/mndTrans.h b/source/dnode/mnode/impl/inc/mndTrans.h index 2fcc82e861..5b5aff7c86 100644 --- a/source/dnode/mnode/impl/inc/mndTrans.h +++ b/source/dnode/mnode/impl/inc/mndTrans.h @@ -33,6 +33,15 @@ typedef struct { void *pCont; } STransAction; +typedef enum { + TEST_TRANS_START_FUNC = 1, + TEST_TRANS_STOP_FUNC = 2, + CONSUME_TRANS_START_FUNC = 3, + CONSUME_TRANS_STOP_FUNC = 4, +} ETrnFuncType; + +typedef void (*TransCbFp)(SMnode *pMnode, void *param, int32_t paramLen); + int32_t mndInitTrans(SMnode *pMnode); void mndCleanupTrans(SMnode *pMnode); @@ -44,7 +53,7 @@ int32_t mndTransAppendCommitlog(STrans *pTrans, SSdbRaw *pRaw); int32_t mndTransAppendRedoAction(STrans *pTrans, STransAction *pAction); int32_t mndTransAppendUndoAction(STrans *pTrans, STransAction *pAction); void mndTransSetRpcRsp(STrans *pTrans, void *pCont, int32_t contLen); -void mndTransSetCb(STrans *pTrans, TransCbFp fp, void *param); +void mndTransSetCb(STrans *pTrans, ETrnFuncType startFunc, ETrnFuncType stopFunc, void *param, int32_t paramLen); void mndTransSetDbInfo(STrans *pTrans, SDbObj *pDb); int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans); diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index ff7a757007..ac75baeb35 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -37,6 +37,8 @@ static int8_t mqInRebFlag = 0; +static const char *mndConsumerStatusName(int status); + static int32_t mndConsumerActionInsert(SSdb *pSdb, SMqConsumerObj *pConsumer); static int32_t mndConsumerActionDelete(SSdb *pSdb, SMqConsumerObj *pConsumer); static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pConsumer, SMqConsumerObj *pNewConsumer); @@ -48,6 +50,7 @@ static int32_t mndProcessSubscribeReq(SNodeMsg *pMsg); static int32_t mndProcessAskEpReq(SNodeMsg *pMsg); static int32_t mndProcessMqTimerMsg(SNodeMsg *pMsg); static int32_t mndProcessConsumerLostMsg(SNodeMsg *pMsg); +static int32_t mndProcessConsumerRecoverMsg(SNodeMsg *pMsg); int32_t mndInitConsumer(SMnode *pMnode) { SSdbTable table = {.sdbType = SDB_CONSUMER, @@ -62,6 +65,11 @@ int32_t mndInitConsumer(SMnode *pMnode) { mndSetMsgHandle(pMnode, TDMT_MND_MQ_ASK_EP, mndProcessAskEpReq); mndSetMsgHandle(pMnode, TDMT_MND_MQ_TIMER, mndProcessMqTimerMsg); mndSetMsgHandle(pMnode, TDMT_MND_MQ_CONSUMER_LOST, mndProcessConsumerLostMsg); + mndSetMsgHandle(pMnode, TDMT_MND_MQ_CONSUMER_RECOVER, mndProcessConsumerRecoverMsg); + + mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_CONSUMERS, mndRetrieveConsumer); + mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_CONSUMERS, mndCancelGetNextConsumer); + return sdbSetTable(pMnode->pSdb, table); } @@ -91,6 +99,30 @@ FAIL: return -1; } +static int32_t mndProcessConsumerRecoverMsg(SNodeMsg *pMsg) { + SMnode *pMnode = pMsg->pNode; + SMqConsumerRecoverMsg *pRecoverMsg = pMsg->rpcMsg.pCont; + SMqConsumerObj *pConsumer = mndAcquireConsumer(pMnode, pRecoverMsg->consumerId); + ASSERT(pConsumer); + + SMqConsumerObj *pConsumerNew = tNewSMqConsumerObj(pConsumer->consumerId, pConsumer->cgroup); + pConsumerNew->updateType = CONSUMER_UPDATE__RECOVER; + + mndReleaseConsumer(pMnode, pConsumer); + + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_CONSUMER_RECOVER, &pMsg->rpcMsg); + if (pTrans == NULL) goto FAIL; + if (mndSetConsumerCommitLogs(pMnode, pTrans, pConsumerNew) != 0) goto FAIL; + if (mndTransPrepare(pMnode, pTrans) != 0) goto FAIL; + + mndTransDrop(pTrans); + return 0; +FAIL: + tDeleteSMqConsumerObj(pConsumerNew); + mndTransDrop(pTrans); + return -1; +} + static SMqRebSubscribe *mndGetOrCreateRebSub(SHashObj *pHash, const char *key) { SMqRebSubscribe *pRebSub = taosHashGet(pHash, key, strlen(key) + 1); if (pRebSub == NULL) { @@ -216,8 +248,15 @@ static int32_t mndProcessAskEpReq(SNodeMsg *pMsg) { // 1. check consumer status int32_t status = atomic_load_32(&pConsumer->status); - if (status == MQ_CONSUMER_STATUS__LOST) { - // TODO: recover consumer + if (status == MQ_CONSUMER_STATUS__LOST_REBD) { + SMqConsumerRecoverMsg *pRecoverMsg = rpcMallocCont(sizeof(SMqConsumerRecoverMsg)); + + pRecoverMsg->consumerId = consumerId; + SRpcMsg *pRpcMsg = taosMemoryCalloc(1, sizeof(SRpcMsg)); + pRpcMsg->msgType = TDMT_MND_MQ_CONSUMER_RECOVER; + pRpcMsg->pCont = pRecoverMsg; + pRpcMsg->contLen = sizeof(SMqConsumerRecoverMsg); + tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, pRpcMsg); } if (status != MQ_CONSUMER_STATUS__READY) { @@ -366,10 +405,14 @@ static int32_t mndProcessSubscribeReq(SNodeMsg *pMsg) { if (pConsumerOld == NULL) { pConsumerNew = tNewSMqConsumerObj(consumerId, cgroup); pConsumerNew->updateType = CONSUMER_UPDATE__MODIFY; - /*pConsumerNew->waitingRebTopics = newSub;*/ pConsumerNew->rebNewTopics = newSub; subscribe.topicNames = NULL; + for (int32_t i = 0; i < newTopicNum; i++) { + char *newTopicCopy = strdup(taosArrayGetP(newSub, i)); + taosArrayPush(pConsumerNew->assignedTopics, &newTopicCopy); + } + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_SUBSCRIBE, &pMsg->rpcMsg); if (pTrans == NULL) goto SUBSCRIBE_OVER; if (mndSetConsumerCommitLogs(pMnode, pTrans, pConsumerNew) != 0) goto SUBSCRIBE_OVER; @@ -389,7 +432,11 @@ static int32_t mndProcessSubscribeReq(SNodeMsg *pMsg) { goto SUBSCRIBE_OVER; } pConsumerNew->updateType = CONSUMER_UPDATE__MODIFY; - /*pConsumerOld->waitingRebTopics = newSub;*/ + + for (int32_t i = 0; i < newTopicNum; i++) { + char *newTopicCopy = strdup(taosArrayGetP(newSub, i)); + taosArrayPush(pConsumerNew->assignedTopics, &newTopicCopy); + } int32_t oldTopicNum = 0; if (pConsumerOld->currentTopics) { @@ -532,6 +579,7 @@ CM_DECODE_OVER: static int32_t mndConsumerActionInsert(SSdb *pSdb, SMqConsumerObj *pConsumer) { mTrace("consumer:%" PRId64 ", perform insert action", pConsumer->consumerId); + pConsumer->subscribeTime = pConsumer->upTime; return 0; } @@ -557,17 +605,45 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, pOldConsumer->rebRemovedTopics = pNewConsumer->rebRemovedTopics; pNewConsumer->rebRemovedTopics = tmp; + tmp = pOldConsumer->assignedTopics; + pOldConsumer->assignedTopics = pNewConsumer->assignedTopics; + pNewConsumer->assignedTopics = tmp; + + pOldConsumer->subscribeTime = pNewConsumer->upTime; + pOldConsumer->status = MQ_CONSUMER_STATUS__MODIFY; } else if (pNewConsumer->updateType == CONSUMER_UPDATE__LOST) { + ASSERT(taosArrayGetSize(pOldConsumer->rebNewTopics) == 0); + ASSERT(taosArrayGetSize(pOldConsumer->rebRemovedTopics) == 0); + int32_t sz = taosArrayGetSize(pOldConsumer->currentTopics); - pOldConsumer->rebRemovedTopics = taosArrayInit(sz, sizeof(void *)); + /*pOldConsumer->rebRemovedTopics = taosArrayInit(sz, sizeof(void *));*/ for (int32_t i = 0; i < sz; i++) { char *topic = strdup(taosArrayGetP(pOldConsumer->currentTopics, i)); - taosArrayPush(pNewConsumer->rebRemovedTopics, &topic); + taosArrayPush(pOldConsumer->rebRemovedTopics, &topic); } + + pOldConsumer->rebalanceTime = pNewConsumer->upTime; + pOldConsumer->status = MQ_CONSUMER_STATUS__LOST; + } else if (pNewConsumer->updateType == CONSUMER_UPDATE__RECOVER) { + ASSERT(taosArrayGetSize(pOldConsumer->currentTopics) == 0); + ASSERT(taosArrayGetSize(pOldConsumer->rebNewTopics) == 0); + + int32_t sz = taosArrayGetSize(pOldConsumer->assignedTopics); + for (int32_t i = 0; i < sz; i++) { + char *topic = strdup(taosArrayGetP(pOldConsumer->assignedTopics, i)); + taosArrayPush(pOldConsumer->rebNewTopics, &topic); + } + + pOldConsumer->rebalanceTime = pNewConsumer->upTime; + + pOldConsumer->status = MQ_CONSUMER_STATUS__MODIFY; } else if (pNewConsumer->updateType == CONSUMER_UPDATE__TOUCH) { atomic_add_fetch_32(&pOldConsumer->epoch, 1); + + pOldConsumer->rebalanceTime = pNewConsumer->upTime; + } else if (pNewConsumer->updateType == CONSUMER_UPDATE__ADD) { ASSERT(taosArrayGetSize(pNewConsumer->rebNewTopics) == 1); ASSERT(taosArrayGetSize(pNewConsumer->rebRemovedTopics) == 0); @@ -612,6 +688,9 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, pOldConsumer->status = MQ_CONSUMER_STATUS__LOST_IN_REB; } } + + pOldConsumer->rebalanceTime = pNewConsumer->upTime; + atomic_add_fetch_32(&pOldConsumer->epoch, 1); } else if (pNewConsumer->updateType == CONSUMER_UPDATE__REMOVE) { ASSERT(taosArrayGetSize(pNewConsumer->rebNewTopics) == 0); @@ -668,6 +747,9 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, pOldConsumer->status = MQ_CONSUMER_STATUS__LOST_IN_REB; } } + + pOldConsumer->rebalanceTime = pNewConsumer->upTime; + atomic_add_fetch_32(&pOldConsumer->epoch, 1); } @@ -688,3 +770,104 @@ void mndReleaseConsumer(SMnode *pMnode, SMqConsumerObj *pConsumer) { SSdb *pSdb = pMnode->pSdb; sdbRelease(pSdb, pConsumer); } + +static int32_t mndRetrieveConsumer(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rowsCapacity) { + SMnode *pMnode = pReq->pNode; + SSdb *pSdb = pMnode->pSdb; + int32_t numOfRows = 0; + SMqConsumerObj *pConsumer = NULL; + + while (numOfRows < rowsCapacity) { + pShow->pIter = sdbFetch(pSdb, SDB_CONSUMER, pShow->pIter, (void **)&pConsumer); + if (pShow->pIter == NULL) break; + + SColumnInfoData *pColInfo; + int32_t cols = 0; + + taosRLockLatch(&pConsumer->lock); + + // consumer id + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, (const char *)&pConsumer->consumerId, false); + + // group id + char groupId[TSDB_CGROUP_LEN + VARSTR_HEADER_SIZE] = {0}; + tstrncpy(varDataVal(groupId), pConsumer->cgroup, TSDB_CGROUP_LEN); + varDataSetLen(groupId, strlen(varDataVal(groupId))); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, (const char *)groupId, false); + + // app id + char appId[TSDB_CGROUP_LEN + VARSTR_HEADER_SIZE] = {0}; + tstrncpy(varDataVal(appId), pConsumer->appId, TSDB_CGROUP_LEN); + varDataSetLen(appId, strlen(varDataVal(appId))); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, (const char *)appId, false); + + // status + char status[20 + VARSTR_HEADER_SIZE] = {0}; + tstrncpy(varDataVal(status), mndConsumerStatusName(pConsumer->status), 20); + varDataSetLen(status, strlen(varDataVal(status))); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, (const char *)status, false); + + // subscribed topics + char topics[TSDB_SHOW_LIST_LEN + VARSTR_HEADER_SIZE] = {0}; + char *showStr = taosShowStrArray(pConsumer->assignedTopics); + tstrncpy(varDataVal(topics), showStr, TSDB_SHOW_LIST_LEN); + taosMemoryFree(showStr); + varDataSetLen(topics, strlen(varDataVal(topics))); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, (const char *)topics, false); + + // pid + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, (const char *)&pConsumer->pid, true); + + // end point + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, (const char *)&pConsumer->ep, true); + + // up time + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, (const char *)&pConsumer->upTime, false); + + // subscribe time + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, (const char *)&pConsumer->subscribeTime, false); + + // rebalance time + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, (const char *)&pConsumer->rebalanceTime, pConsumer->rebalanceTime == 0); + + taosRUnLockLatch(&pConsumer->lock); + sdbRelease(pSdb, pConsumer); + + numOfRows++; + } + + pShow->numOfRows += numOfRows; + return numOfRows; +} + +static void mndCancelGetNextConsumer(SMnode *pMnode, void *pIter) { + SSdb *pSdb = pMnode->pSdb; + sdbCancelFetch(pSdb, pIter); +} + +static const char *mndConsumerStatusName(int status) { + switch (status) { + case MQ_CONSUMER_STATUS__READY: + return "ready"; + case MQ_CONSUMER_STATUS__LOST: + case MQ_CONSUMER_STATUS__LOST_REBD: + case MQ_CONSUMER_STATUS__LOST_IN_REB: + return "lost"; + case MQ_CONSUMER_STATUS__MODIFY: + case MQ_CONSUMER_STATUS__MODIFY_IN_REB: + return "rebalancing"; + default: + return "unknown"; + } +} diff --git a/source/dnode/mnode/impl/src/mndDef.c b/source/dnode/mnode/impl/src/mndDef.c index 12de1f5bbc..767e59e4f6 100644 --- a/source/dnode/mnode/impl/src/mndDef.c +++ b/source/dnode/mnode/impl/src/mndDef.c @@ -34,15 +34,20 @@ SMqConsumerObj *tNewSMqConsumerObj(int64_t consumerId, char cgroup[TSDB_CGROUP_L pConsumer->currentTopics = taosArrayInit(0, sizeof(void *)); pConsumer->rebNewTopics = taosArrayInit(0, sizeof(void *)); pConsumer->rebRemovedTopics = taosArrayInit(0, sizeof(void *)); + pConsumer->assignedTopics = taosArrayInit(0, sizeof(void *)); - if (pConsumer->currentTopics == NULL || pConsumer->rebNewTopics == NULL || pConsumer->rebRemovedTopics == NULL) { + if (pConsumer->currentTopics == NULL || pConsumer->rebNewTopics == NULL || pConsumer->rebRemovedTopics == NULL || + pConsumer->assignedTopics == NULL) { taosArrayDestroy(pConsumer->currentTopics); taosArrayDestroy(pConsumer->rebNewTopics); taosArrayDestroy(pConsumer->rebRemovedTopics); + taosArrayDestroy(pConsumer->assignedTopics); taosMemoryFree(pConsumer); return NULL; } + pConsumer->upTime = taosGetTimestampMs(); + return pConsumer; } @@ -56,6 +61,9 @@ void tDeleteSMqConsumerObj(SMqConsumerObj *pConsumer) { if (pConsumer->rebRemovedTopics) { taosArrayDestroyP(pConsumer->rebRemovedTopics, (FDelete)taosMemoryFree); } + if (pConsumer->assignedTopics) { + taosArrayDestroyP(pConsumer->assignedTopics, (FDelete)taosMemoryFree); + } } int32_t tEncodeSMqConsumerObj(void **buf, const SMqConsumerObj *pConsumer) { @@ -67,6 +75,12 @@ int32_t tEncodeSMqConsumerObj(void **buf, const SMqConsumerObj *pConsumer) { tlen += taosEncodeFixedI32(buf, pConsumer->epoch); tlen += taosEncodeFixedI32(buf, pConsumer->status); + tlen += taosEncodeFixedI32(buf, pConsumer->pid); + tlen += taosEncodeSEpSet(buf, &pConsumer->ep); + tlen += taosEncodeFixedI64(buf, pConsumer->upTime); + tlen += taosEncodeFixedI64(buf, pConsumer->subscribeTime); + tlen += taosEncodeFixedI64(buf, pConsumer->rebalanceTime); + // current topics if (pConsumer->currentTopics) { sz = taosArrayGetSize(pConsumer->currentTopics); @@ -103,6 +117,18 @@ int32_t tEncodeSMqConsumerObj(void **buf, const SMqConsumerObj *pConsumer) { tlen += taosEncodeFixedI32(buf, 0); } + // lost topics + if (pConsumer->assignedTopics) { + sz = taosArrayGetSize(pConsumer->assignedTopics); + tlen += taosEncodeFixedI32(buf, sz); + for (int32_t i = 0; i < sz; i++) { + char *topic = taosArrayGetP(pConsumer->assignedTopics, i); + tlen += taosEncodeString(buf, topic); + } + } else { + tlen += taosEncodeFixedI32(buf, 0); + } + return tlen; } @@ -114,6 +140,12 @@ void *tDecodeSMqConsumerObj(const void *buf, SMqConsumerObj *pConsumer) { buf = taosDecodeFixedI32(buf, &pConsumer->epoch); buf = taosDecodeFixedI32(buf, &pConsumer->status); + buf = taosDecodeFixedI32(buf, &pConsumer->pid); + buf = taosDecodeSEpSet(buf, &pConsumer->ep); + buf = taosDecodeFixedI64(buf, &pConsumer->upTime); + buf = taosDecodeFixedI64(buf, &pConsumer->subscribeTime); + buf = taosDecodeFixedI64(buf, &pConsumer->rebalanceTime); + // current topics buf = taosDecodeFixedI32(buf, &sz); pConsumer->currentTopics = taosArrayInit(sz, sizeof(void *)); @@ -141,6 +173,15 @@ void *tDecodeSMqConsumerObj(const void *buf, SMqConsumerObj *pConsumer) { taosArrayPush(pConsumer->rebRemovedTopics, &topic); } + // reb removed topics + buf = taosDecodeFixedI32(buf, &sz); + pConsumer->assignedTopics = taosArrayInit(sz, sizeof(void *)); + for (int32_t i = 0; i < sz; i++) { + char *topic; + buf = taosDecodeString(buf, &topic); + taosArrayPush(pConsumer->assignedTopics, &topic); + } + return (void *)buf; } @@ -329,6 +370,7 @@ int32_t tEncodeSMqSubActionLogEntry(void **buf, const SMqSubActionLogEntry *pEnt tlen += taosEncodeArray(buf, pEntry->consumers, (FEncode)tEncodeSMqSubActionLogEntry); return tlen; } + void *tDecodeSMqSubActionLogEntry(const void *buf, SMqSubActionLogEntry *pEntry) { buf = taosDecodeFixedI32(buf, &pEntry->epoch); buf = taosDecodeArray(buf, &pEntry->consumers, (FDecode)tDecodeSMqSubActionLogEntry, sizeof(SMqSubActionLogEntry)); diff --git a/source/dnode/mnode/impl/src/mndInfoSchema.c b/source/dnode/mnode/impl/src/mndInfoSchema.c index 2b46fc9274..07f8228cda 100644 --- a/source/dnode/mnode/impl/src/mndInfoSchema.c +++ b/source/dnode/mnode/impl/src/mndInfoSchema.c @@ -142,6 +142,7 @@ static const SInfosTableSchema userTblsSchema[] = { {.name = "vgroup_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, {.name = "ttl", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, {.name = "table_comment", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "type", .bytes = 20 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, }; static const SInfosTableSchema userTblDistSchema[] = { @@ -164,7 +165,6 @@ static const SInfosTableSchema userUsersSchema[] = { {.name = "name", .bytes = TSDB_USER_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, {.name = "privilege", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, - {.name = "account", .bytes = TSDB_USER_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, }; static const SInfosTableSchema grantsSchema[] = { @@ -199,23 +199,6 @@ static const SInfosTableSchema vgroupsSchema[] = { {.name = "file_size", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, }; -static const SInfosTableSchema consumerSchema[] = { - {.name = "client_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "group_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "pid", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "status", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - // ep - // up time - // topics -}; - -static const SInfosTableSchema subscribeSchema[] = { - {.name = "topic_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "group_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "vgroup_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "client_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR}, -}; - static const SInfosTableSchema smaSchema[] = { {.name = "sma_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR}, {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, @@ -282,8 +265,6 @@ static const SInfosTableMeta infosMeta[] = { {TSDB_INS_TABLE_USER_USERS, userUsersSchema, tListLen(userUsersSchema)}, {TSDB_INS_TABLE_LICENCES, grantsSchema, tListLen(grantsSchema)}, {TSDB_INS_TABLE_VGROUPS, vgroupsSchema, tListLen(vgroupsSchema)}, - {TSDB_INS_TABLE_CONSUMERS, consumerSchema, tListLen(consumerSchema)}, - {TSDB_INS_TABLE_SUBSCRIBES, subscribeSchema, tListLen(subscribeSchema)}, {TSDB_INS_TABLE_TRANS, transSchema, tListLen(transSchema)}, {TSDB_INS_TABLE_SMAS, smaSchema, tListLen(smaSchema)}, {TSDB_INS_TABLE_CONFIGS, configSchema, tListLen(configSchema)}, diff --git a/source/dnode/mnode/impl/src/mndPerfSchema.c b/source/dnode/mnode/impl/src/mndPerfSchema.c index cf1cb34115..2cb8c4351e 100644 --- a/source/dnode/mnode/impl/src/mndPerfSchema.c +++ b/source/dnode/mnode/impl/src/mndPerfSchema.c @@ -49,13 +49,15 @@ static const SPerfsTableSchema topicSchema[] = { static const SPerfsTableSchema consumerSchema[] = { {.name = "consumer_id", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT}, - {.name = "app_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY}, {.name = "group_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY}, - {.name = "status", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "app_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "status", .bytes = 20 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY}, {.name = "topics", .bytes = TSDB_SHOW_LIST_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY}, {.name = "pid", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, {.name = "end_point", .bytes = TSDB_EP_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY}, {.name = "up_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, + {.name = "subscribe_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, + {.name = "rebalance_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, }; static const SPerfsTableSchema subscriptionSchema[] = { @@ -63,6 +65,12 @@ static const SPerfsTableSchema subscriptionSchema[] = { {.name = "group_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY}, {.name = "vgroup_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, {.name = "consumer_id", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT}, +}; + +static const SPerfsTableSchema offsetSchema[] = { + {.name = "topic_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "group_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "vgroup_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, {.name = "committed_offset", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT}, {.name = "current_offset", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT}, {.name = "skip_log_cnt", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT}, @@ -74,6 +82,7 @@ static const SPerfsTableMeta perfsMeta[] = { {TSDB_PERFS_TABLE_TOPICS, topicSchema, tListLen(topicSchema)}, {TSDB_PERFS_TABLE_CONSUMERS, consumerSchema, tListLen(consumerSchema)}, {TSDB_PERFS_TABLE_SUBSCRIPTIONS, subscriptionSchema, tListLen(subscriptionSchema)}, + {TSDB_PERFS_TABLE_OFFSETS, offsetSchema, tListLen(offsetSchema)}, }; // connection/application/ diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index 805419129b..7bd22c2de5 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -196,7 +196,7 @@ static int32_t mndProcessConnectReq(SNodeMsg *pReq) { goto CONN_OVER; } if (0 != strncmp(connReq.passwd, pUser->pass, TSDB_PASSWORD_LEN - 1)) { - mError("user:%s, failed to auth while acquire user\n %s \r\n %s", pReq->user, connReq.passwd, pUser->pass); + mError("user:%s, failed to auth while acquire user, input:%s saved:%s", pReq->user, connReq.passwd, pUser->pass); code = TSDB_CODE_RPC_AUTH_FAILURE; goto CONN_OVER; } diff --git a/source/dnode/mnode/impl/src/mndScheduler.c b/source/dnode/mnode/impl/src/mndScheduler.c index a106cf348d..06aa3cec07 100644 --- a/source/dnode/mnode/impl/src/mndScheduler.c +++ b/source/dnode/mnode/impl/src/mndScheduler.c @@ -489,7 +489,7 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib int32_t levelNum = LIST_LENGTH(pPlan->pSubplans); if (levelNum != 1) { qDestroyQueryPlan(pPlan); - terrno = TSDB_CODE_MND_UNSUPPORTED_TOPIC; + terrno = TSDB_CODE_MND_INVALID_TOPIC_QUERY; return -1; } @@ -498,7 +498,7 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib int32_t opNum = LIST_LENGTH(inner->pNodeList); if (opNum != 1) { qDestroyQueryPlan(pPlan); - terrno = TSDB_CODE_MND_UNSUPPORTED_TOPIC; + terrno = TSDB_CODE_MND_INVALID_TOPIC_QUERY; return -1; } plan = nodesListGetNode(inner->pNodeList, 0); diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 3b6b264d1b..41bcef25d1 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -395,7 +395,9 @@ static void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pSt } } // get length - if (tEncodeSize(tEncodeSVCreateStbReq, &req, contLen) < 0) { + int32_t ret = 0; + tEncodeSize(tEncodeSVCreateStbReq, &req, contLen, ret); + if (ret < 0) { return NULL; } diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index 886ba028de..731ee69105 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -261,7 +261,7 @@ static SDDropTopicReq *mndBuildDropTopicMsg(SMnode *pMnode, SVgObj *pVgroup, SMq static int32_t mndCheckCreateTopicReq(SCMCreateTopicReq *pCreate) { if (pCreate->name[0] == 0 || pCreate->sql == NULL || pCreate->sql[0] == 0 || pCreate->subscribeDbName[0] == 0) { - terrno = TSDB_CODE_MND_INVALID_TOPIC_OPTION; + terrno = TSDB_CODE_MND_INVALID_TOPIC; return -1; } diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 582b60f7b2..c08b0f6db9 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -29,7 +29,7 @@ static SSdbRaw *mndTransActionEncode(STrans *pTrans); static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw); static int32_t mndTransActionInsert(SSdb *pSdb, STrans *pTrans); static int32_t mndTransActionUpdate(SSdb *pSdb, STrans *OldTrans, STrans *pOld); -static int32_t mndTransActionDelete(SSdb *pSdb, STrans *pTrans); +static int32_t mndTransActionDelete(SSdb *pSdb, STrans *pTrans, bool callFunc); static int32_t mndTransAppendLog(SArray *pArray, SSdbRaw *pRaw); static int32_t mndTransAppendAction(SArray *pArray, STransAction *pAction); @@ -174,6 +174,13 @@ static SSdbRaw *mndTransActionEncode(STrans *pTrans) { SDB_SET_BINARY(pRaw, dataPos, (void *)pAction->pCont, pAction->contLen, TRANS_ENCODE_OVER) } + SDB_SET_INT32(pRaw, dataPos, pTrans->startFunc, TRANS_ENCODE_OVER) + SDB_SET_INT32(pRaw, dataPos, pTrans->stopFunc, TRANS_ENCODE_OVER) + SDB_SET_INT32(pRaw, dataPos, pTrans->paramLen, TRANS_ENCODE_OVER) + if (pTrans->param != NULL) { + SDB_SET_BINARY(pRaw, dataPos, pTrans->param, pTrans->paramLen, TRANS_ENCODE_OVER) + } + SDB_SET_RESERVE(pRaw, dataPos, MND_TRANS_RESERVE_SIZE, TRANS_ENCODE_OVER) SDB_SET_DATALEN(pRaw, dataPos, TRANS_ENCODE_OVER) @@ -305,6 +312,14 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { action.pCont = NULL; } + SDB_GET_INT32(pRaw, dataPos, &pTrans->startFunc, TRANS_DECODE_OVER) + SDB_GET_INT32(pRaw, dataPos, &pTrans->stopFunc, TRANS_DECODE_OVER) + SDB_GET_INT32(pRaw, dataPos, &pTrans->paramLen, TRANS_DECODE_OVER) + if (pTrans->paramLen != 0) { + pTrans->param = taosMemoryMalloc(pTrans->paramLen); + SDB_GET_BINARY(pRaw, dataPos, pTrans->param, pTrans->paramLen, TRANS_DECODE_OVER); + } + SDB_GET_RESERVE(pRaw, dataPos, MND_TRANS_RESERVE_SIZE, TRANS_DECODE_OVER) terrno = 0; @@ -413,9 +428,36 @@ static const char *mndTransType(ETrnType type) { } } +static void mndTransTestStartFunc(SMnode *pMnode, void *param, int32_t paramLen) { + mInfo("test trans start, param:%s, len:%d", (char *)param, paramLen); +} + +static void mndTransTestStopFunc(SMnode *pMnode, void *param, int32_t paramLen) { + mInfo("test trans stop, param:%s, len:%d", (char *)param, paramLen); +} + +static TransCbFp mndTransGetCbFp(ETrnFuncType ftype) { + switch (ftype) { + case TEST_TRANS_START_FUNC: + return mndTransTestStartFunc; + case TEST_TRANS_STOP_FUNC: + return mndTransTestStopFunc; + default: + return NULL; + } +} + static int32_t mndTransActionInsert(SSdb *pSdb, STrans *pTrans) { // pTrans->stage = TRN_STAGE_PREPARE; mTrace("trans:%d, perform insert action, row:%p stage:%s", pTrans->id, pTrans, mndTransStr(pTrans->stage)); + + if (pTrans->startFunc > 0) { + TransCbFp fp = mndTransGetCbFp(pTrans->startFunc); + if (fp) { + (*fp)(pSdb->pMnode, pTrans->param, pTrans->paramLen); + } + } + return 0; } @@ -430,10 +472,23 @@ static void mndTransDropData(STrans *pTrans) { pTrans->rpcRsp = NULL; pTrans->rpcRspLen = 0; } + if (pTrans->param != NULL) { + taosMemoryFree(pTrans->param); + pTrans->param = NULL; + pTrans->paramLen = 0; + } } -static int32_t mndTransActionDelete(SSdb *pSdb, STrans *pTrans) { - mTrace("trans:%d, perform delete action, row:%p stage:%s", pTrans->id, pTrans, mndTransStr(pTrans->stage)); +static int32_t mndTransActionDelete(SSdb *pSdb, STrans *pTrans, bool callFunc) { + mDebug("trans:%d, perform delete action, row:%p stage:%s callfunc:%d", pTrans->id, pTrans, mndTransStr(pTrans->stage), + callFunc); + if (pTrans->stopFunc > 0 && callFunc) { + TransCbFp fp = mndTransGetCbFp(pTrans->stopFunc); + if (fp) { + (*fp)(pSdb->pMnode, pTrans->param, pTrans->paramLen); + } + } + mndTransDropData(pTrans); return 0; } @@ -498,7 +553,7 @@ STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, ETrnType type, const S return NULL; } - mDebug("trans:%d, is created, data:%p", pTrans->id, pTrans); + mDebug("trans:%d, local var is created, data:%p", pTrans->id, pTrans); return pTrans; } @@ -525,7 +580,7 @@ static void mndTransDropActions(SArray *pArray) { void mndTransDrop(STrans *pTrans) { if (pTrans != NULL) { mndTransDropData(pTrans); - mDebug("trans:%d, is dropped, data:%p", pTrans->id, pTrans); + mDebug("trans:%d, local var is freed, data:%p", pTrans->id, pTrans); taosMemoryFreeClear(pTrans); } } @@ -574,9 +629,11 @@ void mndTransSetRpcRsp(STrans *pTrans, void *pCont, int32_t contLen) { pTrans->rpcRspLen = contLen; } -void mndTransSetCb(STrans *pTrans, TransCbFp fp, void *param) { - pTrans->transCbFp = fp; - pTrans->transCbParam = param; +void mndTransSetCb(STrans *pTrans, ETrnFuncType startFunc, ETrnFuncType stopFunc, void *param, int32_t paramLen) { + pTrans->startFunc = startFunc; + pTrans->stopFunc = stopFunc; + pTrans->param = param; + pTrans->paramLen = paramLen; } void mndTransSetDbInfo(STrans *pTrans, SDbObj *pDb) { @@ -712,8 +769,6 @@ int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans) { pNew->rpcRefId = pTrans->rpcRefId; pNew->rpcRsp = pTrans->rpcRsp; pNew->rpcRspLen = pTrans->rpcRspLen; - pNew->transCbFp = pTrans->transCbFp; - pNew->transCbParam = pTrans->transCbParam; pTrans->rpcRsp = NULL; pTrans->rpcRspLen = 0; @@ -1125,10 +1180,6 @@ static bool mndTransPerfromFinishedStage(SMnode *pMnode, STrans *pTrans) { mDebug("trans:%d, finished, code:0x%04x, failedTimes:%d", pTrans->id, pTrans->code, pTrans->failedTimes); - if (pTrans->transCbFp != NULL) { - (*pTrans->transCbFp)(pMnode, pTrans->transCbParam); - } - return continueExec; } diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index d6876782f6..f739810065 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -96,36 +96,36 @@ static SSdbRaw *mndUserActionEncode(SUserObj *pUser) { int32_t size = sizeof(SUserObj) + USER_RESERVE_SIZE + (numOfReadDbs + numOfWriteDbs) * TSDB_DB_FNAME_LEN; SSdbRaw *pRaw = sdbAllocRaw(SDB_USER, USER_VER_NUMBER, size); - if (pRaw == NULL) goto USER_ENCODE_OVER; + if (pRaw == NULL) goto _OVER; int32_t dataPos = 0; - SDB_SET_BINARY(pRaw, dataPos, pUser->user, TSDB_USER_LEN, USER_ENCODE_OVER) - SDB_SET_BINARY(pRaw, dataPos, pUser->pass, TSDB_PASSWORD_LEN, USER_ENCODE_OVER) - SDB_SET_BINARY(pRaw, dataPos, pUser->acct, TSDB_USER_LEN, USER_ENCODE_OVER) - SDB_SET_INT64(pRaw, dataPos, pUser->createdTime, USER_ENCODE_OVER) - SDB_SET_INT64(pRaw, dataPos, pUser->updateTime, USER_ENCODE_OVER) - SDB_SET_INT8(pRaw, dataPos, pUser->superUser, USER_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, numOfReadDbs, USER_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, numOfWriteDbs, USER_ENCODE_OVER) + SDB_SET_BINARY(pRaw, dataPos, pUser->user, TSDB_USER_LEN, _OVER) + SDB_SET_BINARY(pRaw, dataPos, pUser->pass, TSDB_PASSWORD_LEN, _OVER) + SDB_SET_BINARY(pRaw, dataPos, pUser->acct, TSDB_USER_LEN, _OVER) + SDB_SET_INT64(pRaw, dataPos, pUser->createdTime, _OVER) + SDB_SET_INT64(pRaw, dataPos, pUser->updateTime, _OVER) + SDB_SET_INT8(pRaw, dataPos, pUser->superUser, _OVER) + SDB_SET_INT32(pRaw, dataPos, numOfReadDbs, _OVER) + SDB_SET_INT32(pRaw, dataPos, numOfWriteDbs, _OVER) char *db = taosHashIterate(pUser->readDbs, NULL); while (db != NULL) { - SDB_SET_BINARY(pRaw, dataPos, db, TSDB_DB_FNAME_LEN, USER_ENCODE_OVER); + SDB_SET_BINARY(pRaw, dataPos, db, TSDB_DB_FNAME_LEN, _OVER); db = taosHashIterate(pUser->readDbs, db); } db = taosHashIterate(pUser->writeDbs, NULL); while (db != NULL) { - SDB_SET_BINARY(pRaw, dataPos, db, TSDB_DB_FNAME_LEN, USER_ENCODE_OVER); + SDB_SET_BINARY(pRaw, dataPos, db, TSDB_DB_FNAME_LEN, _OVER); db = taosHashIterate(pUser->writeDbs, db); } - SDB_SET_RESERVE(pRaw, dataPos, USER_RESERVE_SIZE, USER_ENCODE_OVER) - SDB_SET_DATALEN(pRaw, dataPos, USER_ENCODE_OVER) + SDB_SET_RESERVE(pRaw, dataPos, USER_RESERVE_SIZE, _OVER) + SDB_SET_DATALEN(pRaw, dataPos, _OVER) terrno = 0; -USER_ENCODE_OVER: +_OVER: if (terrno != 0) { mError("user:%s, failed to encode to raw:%p since %s", pUser->user, pRaw, terrstr()); sdbFreeRaw(pRaw); @@ -140,55 +140,55 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) { terrno = TSDB_CODE_OUT_OF_MEMORY; int8_t sver = 0; - if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto USER_DECODE_OVER; + if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER; if (sver != USER_VER_NUMBER) { terrno = TSDB_CODE_SDB_INVALID_DATA_VER; - goto USER_DECODE_OVER; + goto _OVER; } SSdbRow *pRow = sdbAllocRow(sizeof(SUserObj)); - if (pRow == NULL) goto USER_DECODE_OVER; + if (pRow == NULL) goto _OVER; SUserObj *pUser = sdbGetRowObj(pRow); - if (pUser == NULL) goto USER_DECODE_OVER; - - pUser->readDbs = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, true); - pUser->writeDbs = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, true); - if (pUser->readDbs == NULL || pUser->writeDbs == NULL) goto USER_DECODE_OVER; + if (pUser == NULL) goto _OVER; int32_t dataPos = 0; - SDB_GET_BINARY(pRaw, dataPos, pUser->user, TSDB_USER_LEN, USER_DECODE_OVER) - SDB_GET_BINARY(pRaw, dataPos, pUser->pass, TSDB_PASSWORD_LEN, USER_DECODE_OVER) - SDB_GET_BINARY(pRaw, dataPos, pUser->acct, TSDB_USER_LEN, USER_DECODE_OVER) - SDB_GET_INT64(pRaw, dataPos, &pUser->createdTime, USER_DECODE_OVER) - SDB_GET_INT64(pRaw, dataPos, &pUser->updateTime, USER_DECODE_OVER) - SDB_GET_INT8(pRaw, dataPos, &pUser->superUser, USER_DECODE_OVER) + SDB_GET_BINARY(pRaw, dataPos, pUser->user, TSDB_USER_LEN, _OVER) + SDB_GET_BINARY(pRaw, dataPos, pUser->pass, TSDB_PASSWORD_LEN, _OVER) + SDB_GET_BINARY(pRaw, dataPos, pUser->acct, TSDB_USER_LEN, _OVER) + SDB_GET_INT64(pRaw, dataPos, &pUser->createdTime, _OVER) + SDB_GET_INT64(pRaw, dataPos, &pUser->updateTime, _OVER) + SDB_GET_INT8(pRaw, dataPos, &pUser->superUser, _OVER) int32_t numOfReadDbs = 0; int32_t numOfWriteDbs = 0; - SDB_GET_INT32(pRaw, dataPos, &numOfReadDbs, USER_DECODE_OVER) - SDB_GET_INT32(pRaw, dataPos, &numOfWriteDbs, USER_DECODE_OVER) + SDB_GET_INT32(pRaw, dataPos, &numOfReadDbs, _OVER) + SDB_GET_INT32(pRaw, dataPos, &numOfWriteDbs, _OVER) + pUser->readDbs = taosHashInit(numOfReadDbs, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + pUser->writeDbs = + taosHashInit(numOfWriteDbs, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + if (pUser->readDbs == NULL || pUser->writeDbs == NULL) goto _OVER; for (int32_t i = 0; i < numOfReadDbs; ++i) { char db[TSDB_DB_FNAME_LEN] = {0}; - SDB_GET_BINARY(pRaw, dataPos, db, TSDB_DB_FNAME_LEN, USER_DECODE_OVER) + SDB_GET_BINARY(pRaw, dataPos, db, TSDB_DB_FNAME_LEN, _OVER) int32_t len = strlen(db) + 1; taosHashPut(pUser->readDbs, db, len, db, TSDB_DB_FNAME_LEN); } for (int32_t i = 0; i < numOfWriteDbs; ++i) { char db[TSDB_DB_FNAME_LEN] = {0}; - SDB_GET_BINARY(pRaw, dataPos, db, TSDB_DB_FNAME_LEN, USER_DECODE_OVER) + SDB_GET_BINARY(pRaw, dataPos, db, TSDB_DB_FNAME_LEN, _OVER) int32_t len = strlen(db) + 1; taosHashPut(pUser->writeDbs, db, len, db, TSDB_DB_FNAME_LEN); } - SDB_GET_RESERVE(pRaw, dataPos, USER_RESERVE_SIZE, USER_DECODE_OVER) + SDB_GET_RESERVE(pRaw, dataPos, USER_RESERVE_SIZE, _OVER) terrno = 0; -USER_DECODE_OVER: +_OVER: if (terrno != 0) { mError("user:%s, failed to decode from raw:%p since %s", pUser->user, pRaw, terrstr()); taosHashCleanup(pUser->readDbs); @@ -220,6 +220,8 @@ static int32_t mndUserActionDelete(SSdb *pSdb, SUserObj *pUser) { mTrace("user:%s, perform delete action, row:%p", pUser->user, pUser); taosHashCleanup(pUser->readDbs); taosHashCleanup(pUser->writeDbs); + pUser->readDbs = NULL; + pUser->writeDbs = NULL; return 0; } @@ -228,13 +230,8 @@ static int32_t mndUserActionUpdate(SSdb *pSdb, SUserObj *pOld, SUserObj *pNew) { memcpy(pOld->pass, pNew->pass, TSDB_PASSWORD_LEN); pOld->updateTime = pNew->updateTime; - void *tmp1 = pOld->readDbs; - pOld->readDbs = pNew->readDbs; - pNew->readDbs = tmp1; - - void *tmp2 = pOld->writeDbs; - pOld->writeDbs = pNew->writeDbs; - pNew->writeDbs = tmp2; + TSWAP(pOld->readDbs, pNew->readDbs); + TSWAP(pOld->writeDbs, pNew->writeDbs); return 0; } @@ -277,6 +274,9 @@ static int32_t mndCreateUser(SMnode *pMnode, char *acct, SCreateUserReq *pCreate } sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY); + char *param = strdup("====> test code to be deleted later <====="); + mndTransSetCb(pTrans, TEST_TRANS_START_FUNC, TEST_TRANS_STOP_FUNC, param, strlen(param) + 1); + if (mndTransPrepare(pMnode, pTrans) != 0) { mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); mndTransDrop(pTrans); @@ -296,41 +296,41 @@ static int32_t mndProcessCreateUserReq(SNodeMsg *pReq) { if (tDeserializeSCreateUserReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &createReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; - goto CREATE_USER_OVER; + goto _OVER; } mDebug("user:%s, start to create", createReq.user); if (createReq.user[0] == 0) { terrno = TSDB_CODE_MND_INVALID_USER_FORMAT; - goto CREATE_USER_OVER; + goto _OVER; } if (createReq.pass[0] == 0) { terrno = TSDB_CODE_MND_INVALID_PASS_FORMAT; - goto CREATE_USER_OVER; + goto _OVER; } pUser = mndAcquireUser(pMnode, createReq.user); if (pUser != NULL) { terrno = TSDB_CODE_MND_USER_ALREADY_EXIST; - goto CREATE_USER_OVER; + goto _OVER; } pOperUser = mndAcquireUser(pMnode, pReq->user); if (pOperUser == NULL) { terrno = TSDB_CODE_MND_NO_USER_FROM_CONN; - goto CREATE_USER_OVER; + goto _OVER; } if (mndCheckCreateUserAuth(pOperUser) != 0) { - goto CREATE_USER_OVER; + goto _OVER; } code = mndCreateUser(pMnode, pOperUser->acct, &createReq, pReq); if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; -CREATE_USER_OVER: +_OVER: if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { mError("user:%s, failed to create since %s", createReq.user, terrstr()); } @@ -341,13 +341,13 @@ CREATE_USER_OVER: return code; } -static int32_t mndUpdateUser(SMnode *pMnode, SUserObj *pOld, SUserObj *pNew, SNodeMsg *pReq) { +static int32_t mndAlterUser(SMnode *pMnode, SUserObj *pOld, SUserObj *pNew, SNodeMsg *pReq) { STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_ALTER_USER, &pReq->rpcMsg); if (pTrans == NULL) { - mError("user:%s, failed to update since %s", pOld->user, terrstr()); + mError("user:%s, failed to alter since %s", pOld->user, terrstr()); return -1; } - mDebug("trans:%d, used to update user:%s", pTrans->id, pOld->user); + mDebug("trans:%d, used to alter user:%s", pTrans->id, pOld->user); SSdbRaw *pRedoRaw = mndUserActionEncode(pNew); if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) { @@ -368,7 +368,8 @@ static int32_t mndUpdateUser(SMnode *pMnode, SUserObj *pOld, SUserObj *pNew, SNo } static SHashObj *mndDupDbHash(SHashObj *pOld) { - SHashObj *pNew = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, true); + SHashObj *pNew = + taosHashInit(taosHashGetSize(pOld), taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); if (pNew == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -379,8 +380,8 @@ static SHashObj *mndDupDbHash(SHashObj *pOld) { int32_t len = strlen(db) + 1; if (taosHashPut(pNew, db, len, db, TSDB_DB_FNAME_LEN) != 0) { taosHashCancelIterate(pOld, db); - terrno = TSDB_CODE_OUT_OF_MEMORY; taosHashCleanup(pNew); + terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } db = taosHashIterate(pOld, db); @@ -399,38 +400,38 @@ static int32_t mndProcessAlterUserReq(SNodeMsg *pReq) { if (tDeserializeSAlterUserReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &alterReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; - goto ALTER_USER_OVER; + goto _OVER; } mDebug("user:%s, start to alter", alterReq.user); if (alterReq.user[0] == 0) { terrno = TSDB_CODE_MND_INVALID_USER_FORMAT; - goto ALTER_USER_OVER; + goto _OVER; } if (alterReq.pass[0] == 0) { terrno = TSDB_CODE_MND_INVALID_PASS_FORMAT; - goto ALTER_USER_OVER; + goto _OVER; } pUser = mndAcquireUser(pMnode, alterReq.user); if (pUser == NULL) { terrno = TSDB_CODE_MND_USER_NOT_EXIST; - goto ALTER_USER_OVER; + goto _OVER; } pOperUser = mndAcquireUser(pMnode, pReq->user); if (pOperUser == NULL) { terrno = TSDB_CODE_MND_NO_USER_FROM_CONN; - goto ALTER_USER_OVER; + goto _OVER; } memcpy(&newUser, pUser, sizeof(SUserObj)); newUser.readDbs = mndDupDbHash(pUser->readDbs); newUser.writeDbs = mndDupDbHash(pUser->writeDbs); if (newUser.readDbs == NULL || newUser.writeDbs == NULL) { - goto ALTER_USER_OVER; + goto _OVER; } int32_t len = strlen(alterReq.dbname) + 1; @@ -440,56 +441,56 @@ static int32_t mndProcessAlterUserReq(SNodeMsg *pReq) { if (alterReq.alterType == TSDB_ALTER_USER_PASSWD) { char pass[TSDB_PASSWORD_LEN + 1] = {0}; taosEncryptPass_c((uint8_t *)alterReq.pass, strlen(alterReq.pass), pass); - memcpy(pUser->pass, pass, TSDB_PASSWORD_LEN); + memcpy(newUser.pass, pass, TSDB_PASSWORD_LEN); } else if (alterReq.alterType == TSDB_ALTER_USER_SUPERUSER) { newUser.superUser = alterReq.superUser; } else if (alterReq.alterType == TSDB_ALTER_USER_ADD_READ_DB) { if (pDb == NULL) { terrno = TSDB_CODE_MND_DB_NOT_EXIST; - goto ALTER_USER_OVER; + goto _OVER; } if (taosHashPut(newUser.readDbs, alterReq.dbname, len, alterReq.dbname, TSDB_DB_FNAME_LEN) != 0) { terrno = TSDB_CODE_OUT_OF_MEMORY; - goto ALTER_USER_OVER; + goto _OVER; } } else if (alterReq.alterType == TSDB_ALTER_USER_REMOVE_READ_DB) { if (taosHashRemove(newUser.readDbs, alterReq.dbname, len) != 0) { terrno = TSDB_CODE_MND_DB_NOT_EXIST; - goto ALTER_USER_OVER; + goto _OVER; } } else if (alterReq.alterType == TSDB_ALTER_USER_CLEAR_READ_DB) { taosHashClear(newUser.readDbs); } else if (alterReq.alterType == TSDB_ALTER_USER_ADD_WRITE_DB) { if (pDb == NULL) { terrno = TSDB_CODE_MND_DB_NOT_EXIST; - goto ALTER_USER_OVER; + goto _OVER; } if (taosHashPut(newUser.writeDbs, alterReq.dbname, len, alterReq.dbname, TSDB_DB_FNAME_LEN) != 0) { terrno = TSDB_CODE_OUT_OF_MEMORY; - goto ALTER_USER_OVER; + goto _OVER; } } else if (alterReq.alterType == TSDB_ALTER_USER_REMOVE_WRITE_DB) { if (taosHashRemove(newUser.writeDbs, alterReq.dbname, len) != 0) { terrno = TSDB_CODE_MND_DB_NOT_EXIST; - goto ALTER_USER_OVER; + goto _OVER; } } else if (alterReq.alterType == TSDB_ALTER_USER_CLEAR_WRITE_DB) { taosHashClear(newUser.writeDbs); } else { terrno = TSDB_CODE_MND_INVALID_ALTER_OPER; - goto ALTER_USER_OVER; + goto _OVER; } newUser.updateTime = taosGetTimestampMs(); if (mndCheckAlterUserAuth(pOperUser, pUser, pDb, &alterReq) != 0) { - goto ALTER_USER_OVER; + goto _OVER; } - code = mndUpdateUser(pMnode, pUser, &newUser, pReq); + code = mndAlterUser(pMnode, pUser, &newUser, pReq); if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; -ALTER_USER_OVER: +_OVER: if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { mError("user:%s, failed to alter since %s", alterReq.user, terrstr()); } @@ -537,36 +538,36 @@ static int32_t mndProcessDropUserReq(SNodeMsg *pReq) { if (tDeserializeSDropUserReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &dropReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; - goto DROP_USER_OVER; + goto _OVER; } mDebug("user:%s, start to drop", dropReq.user); if (dropReq.user[0] == 0) { terrno = TSDB_CODE_MND_INVALID_USER_FORMAT; - goto DROP_USER_OVER; + goto _OVER; } pUser = mndAcquireUser(pMnode, dropReq.user); if (pUser == NULL) { terrno = TSDB_CODE_MND_USER_NOT_EXIST; - goto DROP_USER_OVER; + goto _OVER; } pOperUser = mndAcquireUser(pMnode, pReq->user); if (pOperUser == NULL) { terrno = TSDB_CODE_MND_NO_USER_FROM_CONN; - goto DROP_USER_OVER; + goto _OVER; } if (mndCheckDropUserAuth(pOperUser) != 0) { - goto DROP_USER_OVER; + goto _OVER; } code = mndDropUser(pMnode, pReq, pUser); if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; -DROP_USER_OVER: +_OVER: if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { mError("user:%s, failed to drop since %s", dropReq.user, terrstr()); } @@ -586,7 +587,7 @@ static int32_t mndProcessGetUserAuthReq(SNodeMsg *pReq) { if (tDeserializeSGetUserAuthReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &authReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; - goto GET_AUTH_OVER; + goto _OVER; } mTrace("user:%s, start to get auth", authReq.user); @@ -594,7 +595,7 @@ static int32_t mndProcessGetUserAuthReq(SNodeMsg *pReq) { pUser = mndAcquireUser(pMnode, authReq.user); if (pUser == NULL) { terrno = TSDB_CODE_MND_USER_NOT_EXIST; - goto GET_AUTH_OVER; + goto _OVER; } memcpy(authRsp.user, pUser->user, TSDB_USER_LEN); @@ -622,7 +623,7 @@ static int32_t mndProcessGetUserAuthReq(SNodeMsg *pReq) { void *pRsp = rpcMallocCont(contLen); if (pRsp == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; - goto GET_AUTH_OVER; + goto _OVER; } tSerializeSGetUserAuthRsp(pRsp, contLen, &authRsp); @@ -631,10 +632,9 @@ static int32_t mndProcessGetUserAuthReq(SNodeMsg *pReq) { pReq->rspLen = contLen; code = 0; -GET_AUTH_OVER: +_OVER: mndReleaseUser(pMnode, pUser); - taosHashCleanup(authRsp.readDbs); - taosHashCleanup(authRsp.writeDbs); + tFreeSGetUserAuthRsp(&authRsp); return code; } @@ -671,11 +671,6 @@ static int32_t mndRetrieveUsers(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pB pColInfo = taosArrayGet(pBlock->pDataBlock, cols); colDataAppend(pColInfo, numOfRows, (const char *)&pUser->createdTime, false); - cols++; - pColInfo = taosArrayGet(pBlock->pDataBlock, cols); - STR_WITH_MAXSIZE_TO_VARSTR(name, pUser->acct, pShow->bytes[cols]); - colDataAppend(pColInfo, numOfRows, (const char *)name, false); - numOfRows++; sdbRelease(pSdb, pUser); } diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 3a4fde992f..aab9ac61a2 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -45,8 +45,8 @@ int32_t mndInitVgroup(SMnode *pMnode) { .encodeFp = (SdbEncodeFp)mndVgroupActionEncode, .decodeFp = (SdbDecodeFp)mndVgroupActionDecode, .insertFp = (SdbInsertFp)mndVgroupActionInsert, - .updateFp = (SdbUpdateFp)mndVgroupActionDelete, - .deleteFp = (SdbDeleteFp)mndVgroupActionUpdate}; + .updateFp = (SdbUpdateFp)mndVgroupActionUpdate, + .deleteFp = (SdbDeleteFp)mndVgroupActionDelete}; mndSetMsgHandle(pMnode, TDMT_DND_CREATE_VNODE_RSP, mndProcessCreateVnodeRsp); mndSetMsgHandle(pMnode, TDMT_VND_ALTER_VNODE_RSP, mndProcessAlterVnodeRsp); diff --git a/source/dnode/mnode/impl/src/mnode.c b/source/dnode/mnode/impl/src/mnode.c index 41a309e941..b38fd6178c 100644 --- a/source/dnode/mnode/impl/src/mnode.c +++ b/source/dnode/mnode/impl/src/mnode.c @@ -43,7 +43,7 @@ #include "mndUser.h" #include "mndVgroup.h" -#define MQ_TIMER_MS 3000 +#define MQ_TIMER_MS 2000 #define TRNAS_TIMER_MS 6000 static void *mndBuildTimerMsg(int32_t *pContLen) { @@ -418,7 +418,6 @@ int64_t mndGenerateUid(char *name, int32_t len) { } while (true); } - int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo, SMonGrantInfo *pGrantInfo) { if (!mndIsMaster(pMnode)) return -1; @@ -528,4 +527,4 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr int32_t mndGetLoad(SMnode *pMnode, SMnodeLoad *pLoad) { pLoad->syncState = pMnode->syncMgmt.state; return 0; -} \ No newline at end of file +} diff --git a/source/dnode/mnode/impl/test/CMakeLists.txt b/source/dnode/mnode/impl/test/CMakeLists.txt index 15f2aed22a..feeebad674 100644 --- a/source/dnode/mnode/impl/test/CMakeLists.txt +++ b/source/dnode/mnode/impl/test/CMakeLists.txt @@ -8,6 +8,7 @@ add_subdirectory(func) add_subdirectory(mnode) add_subdirectory(profile) add_subdirectory(qnode) +add_subdirectory(sdb) add_subdirectory(show) add_subdirectory(sma) add_subdirectory(snode) diff --git a/source/dnode/mnode/impl/test/sdb/CMakeLists.txt b/source/dnode/mnode/impl/test/sdb/CMakeLists.txt new file mode 100644 index 0000000000..371c4d5766 --- /dev/null +++ b/source/dnode/mnode/impl/test/sdb/CMakeLists.txt @@ -0,0 +1,12 @@ +aux_source_directory(. MNODE_SDB_TEST_SRC) +add_executable(sdbTest ${MNODE_SDB_TEST_SRC}) +target_link_libraries( + sdbTest + PUBLIC sut + PUBLIC sdb +) + +add_test( + NAME sdbTest + COMMAND sdbTest +) diff --git a/source/dnode/mnode/impl/test/sdb/sdbTest.cpp b/source/dnode/mnode/impl/test/sdb/sdbTest.cpp new file mode 100644 index 0000000000..998dca1401 --- /dev/null +++ b/source/dnode/mnode/impl/test/sdb/sdbTest.cpp @@ -0,0 +1,83 @@ +/** + * @file sdbTest.cpp + * @author slguan (slguan@taosdata.com) + * @brief MNODE module sdb tests + * @version 1.0 + * @date 2022-04-27 + * + * @copyright Copyright (c) 2022 + * + */ + +#include "sut.h" + +class MndTestShow : public ::testing::Test { + protected: + static void SetUpTestSuite() { test.Init("/tmp/mnode_test_show", 9021); } + static void TearDownTestSuite() { test.Cleanup(); } + + static Testbase test; + + public: + void SetUp() override {} + void TearDown() override {} +}; + +Testbase MndTestShow::test; + +TEST_F(MndTestShow, 01_ShowMsg_InvalidMsgMax) { + SShowReq showReq = {0}; + showReq.type = TSDB_MGMT_TABLE_MAX; + + int32_t contLen = tSerializeSShowReq(NULL, 0, &showReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSShowReq(pReq, contLen, &showReq); + tFreeSShowReq(&showReq); + + SRpcMsg* pRsp = test.SendReq(TDMT_MND_SYSTABLE_RETRIEVE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_NE(pRsp->code, 0); +} + +TEST_F(MndTestShow, 02_ShowMsg_InvalidMsgStart) { + SShowReq showReq = {0}; + showReq.type = TSDB_MGMT_TABLE_START; + + int32_t contLen = tSerializeSShowReq(NULL, 0, &showReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSShowReq(pReq, contLen, &showReq); + tFreeSShowReq(&showReq); + + SRpcMsg* pRsp = test.SendReq(TDMT_MND_SYSTABLE_RETRIEVE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_NE(pRsp->code, 0); +} + +TEST_F(MndTestShow, 03_ShowMsg_Conn) { + char passwd[] = "taosdata"; + char secretEncrypt[TSDB_PASSWORD_LEN] = {0}; + taosEncryptPass_c((uint8_t*)passwd, strlen(passwd), secretEncrypt); + + SConnectReq connectReq = {0}; + connectReq.pid = 1234; + strcpy(connectReq.app, "mnode_test_show"); + strcpy(connectReq.db, ""); + strcpy(connectReq.user, "root"); + strcpy(connectReq.passwd, secretEncrypt); + + int32_t contLen = tSerializeSConnectReq(NULL, 0, &connectReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSConnectReq(pReq, contLen, &connectReq); + + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CONNECT, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); + + test.SendShowReq(TSDB_MGMT_TABLE_CONNS, "connections", ""); + // EXPECT_EQ(test.GetShowRows(), 1); +} + +TEST_F(MndTestShow, 04_ShowMsg_Cluster) { + test.SendShowReq(TSDB_MGMT_TABLE_CLUSTER, "cluster", ""); + EXPECT_EQ(test.GetShowRows(), 1); +} diff --git a/source/dnode/mnode/sdb/inc/sdbInt.h b/source/dnode/mnode/sdb/inc/sdbInt.h index d268b93051..de71ac47b6 100644 --- a/source/dnode/mnode/sdb/inc/sdbInt.h +++ b/source/dnode/mnode/sdb/inc/sdbInt.h @@ -19,10 +19,6 @@ #include "os.h" #include "sdb.h" -#include "thash.h" -#include "tlockfree.h" -#include "tlog.h" -#include "tmsg.h" #ifdef __cplusplus extern "C" { @@ -53,26 +49,6 @@ typedef struct SSdbRow { char pObj[]; } SSdbRow; -typedef struct SSdb { - SMnode *pMnode; - char *currDir; - char *syncDir; - char *tmpDir; - int64_t lastCommitVer; - int64_t curVer; - int64_t tableVer[SDB_MAX]; - int64_t maxId[SDB_MAX]; - EKeyType keyTypes[SDB_MAX]; - SHashObj *hashObjs[SDB_MAX]; - SRWLatch locks[SDB_MAX]; - SdbInsertFp insertFps[SDB_MAX]; - SdbUpdateFp updateFps[SDB_MAX]; - SdbDeleteFp deleteFps[SDB_MAX]; - SdbDeployFp deployFps[SDB_MAX]; - SdbEncodeFp encodeFps[SDB_MAX]; - SdbDecodeFp decodeFps[SDB_MAX]; -} SSdb; - const char *sdbTableName(ESdbType type); void sdbPrintOper(SSdb *pSdb, SSdbRow *pRow, const char *oper); diff --git a/source/dnode/mnode/sdb/src/sdb.c b/source/dnode/mnode/sdb/src/sdb.c index bc878d7b31..f85ff7a26e 100644 --- a/source/dnode/mnode/sdb/src/sdb.c +++ b/source/dnode/mnode/sdb/src/sdb.c @@ -87,7 +87,7 @@ void sdbCleanup(SSdb *pSdb) { SSdbRow *pRow = *ppRow; if (pRow == NULL) continue; - sdbFreeRow(pSdb, pRow); + sdbFreeRow(pSdb, pRow, true); ppRow = taosHashIterate(hash, ppRow); } } diff --git a/source/dnode/mnode/sdb/src/sdbHash.c b/source/dnode/mnode/sdb/src/sdbHash.c index 1158bedc21..b34c2a82d8 100644 --- a/source/dnode/mnode/sdb/src/sdbHash.c +++ b/source/dnode/mnode/sdb/src/sdbHash.c @@ -137,7 +137,7 @@ static int32_t sdbInsertRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow * SSdbRow *pOldRow = taosHashGet(hash, pRow->pObj, keySize); if (pOldRow != NULL) { taosWUnLockLatch(pLock); - sdbFreeRow(pSdb, pRow); + sdbFreeRow(pSdb, pRow, false); terrno = TSDB_CODE_SDB_OBJ_ALREADY_THERE; return terrno; } @@ -148,7 +148,7 @@ static int32_t sdbInsertRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow * if (taosHashPut(hash, pRow->pObj, keySize, &pRow, sizeof(void *)) != 0) { taosWUnLockLatch(pLock); - sdbFreeRow(pSdb, pRow); + sdbFreeRow(pSdb, pRow, false); terrno = TSDB_CODE_SDB_OBJ_ALREADY_THERE; return terrno; } @@ -164,7 +164,7 @@ static int32_t sdbInsertRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow * taosWLockLatch(pLock); taosHashRemove(hash, pRow->pObj, keySize); taosWUnLockLatch(pLock); - sdbFreeRow(pSdb, pRow); + sdbFreeRow(pSdb, pRow, false); terrno = code; return terrno; } @@ -202,7 +202,7 @@ static int32_t sdbUpdateRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow * code = (*updateFp)(pSdb, pOldRow->pObj, pNewRow->pObj); } - sdbFreeRow(pSdb, pNewRow); + sdbFreeRow(pSdb, pNewRow, false); pSdb->tableVer[pOldRow->type]++; return code; @@ -215,7 +215,7 @@ static int32_t sdbDeleteRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow * SSdbRow **ppOldRow = taosHashGet(hash, pRow->pObj, keySize); if (ppOldRow == NULL || *ppOldRow == NULL) { taosWUnLockLatch(pLock); - sdbFreeRow(pSdb, pRow); + sdbFreeRow(pSdb, pRow, false); terrno = TSDB_CODE_SDB_OBJ_NOT_THERE; return terrno; } @@ -228,7 +228,7 @@ static int32_t sdbDeleteRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow * taosWUnLockLatch(pLock); pSdb->tableVer[pOldRow->type]++; - sdbFreeRow(pSdb, pRow); + sdbFreeRow(pSdb, pRow, false); sdbCheck(pSdb, pOldRow); // sdbRelease(pSdb, pOldRow->pObj); @@ -322,7 +322,7 @@ static void sdbCheck(SSdb *pSdb, SSdbRow *pRow) { int32_t ref = atomic_load_32(&pRow->refCount); sdbPrintOper(pSdb, pRow, "check"); if (ref <= 0 && pRow->status == SDB_STATUS_DROPPED) { - sdbFreeRow(pSdb, pRow); + sdbFreeRow(pSdb, pRow, true); } taosRUnLockLatch(pLock); @@ -340,7 +340,7 @@ void sdbRelease(SSdb *pSdb, void *pObj) { int32_t ref = atomic_sub_fetch_32(&pRow->refCount, 1); sdbPrintOper(pSdb, pRow, "release"); if (ref <= 0 && pRow->status == SDB_STATUS_DROPPED) { - sdbFreeRow(pSdb, pRow); + sdbFreeRow(pSdb, pRow, true); } taosRUnLockLatch(pLock); diff --git a/source/dnode/mnode/sdb/src/sdbRaw.c b/source/dnode/mnode/sdb/src/sdbRaw.c index 326fe53fc7..45464c3bd3 100644 --- a/source/dnode/mnode/sdb/src/sdbRaw.c +++ b/source/dnode/mnode/sdb/src/sdbRaw.c @@ -107,7 +107,9 @@ int32_t sdbSetRawBinary(SSdbRaw *pRaw, int32_t dataPos, const char *pVal, int32_ return -1; } - memcpy(pRaw->pData + dataPos, pVal, valLen); + if (pVal != NULL) { + memcpy(pRaw->pData + dataPos, pVal, valLen); + } return 0; } diff --git a/source/dnode/mnode/sdb/src/sdbRow.c b/source/dnode/mnode/sdb/src/sdbRow.c index 94f87cb350..43f70cb245 100644 --- a/source/dnode/mnode/sdb/src/sdbRow.c +++ b/source/dnode/mnode/sdb/src/sdbRow.c @@ -36,11 +36,11 @@ void *sdbGetRowObj(SSdbRow *pRow) { return pRow->pObj; } -void sdbFreeRow(SSdb *pSdb, SSdbRow *pRow) { +void sdbFreeRow(SSdb *pSdb, SSdbRow *pRow, bool callFunc) { // remove attached object such as trans SdbDeleteFp deleteFp = pSdb->deleteFps[pRow->type]; if (deleteFp != NULL) { - (*deleteFp)(pSdb, pRow->pObj); + (*deleteFp)(pSdb, pRow->pObj, callFunc); } sdbPrintOper(pSdb, pRow, "free"); diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index 2a784c112e..724c7b312e 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -27,10 +27,9 @@ target_sources( "src/tsdb/tsdbTDBImpl.c" "src/tsdb/tsdbCommit.c" "src/tsdb/tsdbCommit2.c" - "src/tsdb/tsdbCompact.c" "src/tsdb/tsdbFile.c" "src/tsdb/tsdbFS.c" - "src/tsdb/tsdbMain.c" + "src/tsdb/tsdbOpen.c" "src/tsdb/tsdbMemTable.c" "src/tsdb/tsdbRead.c" "src/tsdb/tsdbReadImpl.c" diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 3314d2f264..4213ce78bd 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -68,6 +68,7 @@ void vnodeStop(SVnode *pVnode); int64_t vnodeGetSyncHandle(SVnode *pVnode); void vnodeGetSnapshot(SVnode *pVnode, SSnapshot *pSnapshot); +void vnodeGetInfo(SVnode *pVnode, const char **dbname, int32_t *vgId); // meta typedef struct SMeta SMeta; // todo: remove @@ -76,6 +77,7 @@ typedef struct SMetaEntry SMetaEntry; void metaReaderInit(SMetaReader *pReader, SMeta *pMeta, int32_t flags); void metaReaderClear(SMetaReader *pReader); +int metaGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid); int metaReadNext(SMetaReader *pReader); #if 1 // refact APIs below (TODO) @@ -133,23 +135,18 @@ int32_t tqRetrieveDataBlock(SArray **ppCols, STqReadHandle *pHandle, uint64_t *p // need to reposition // structs -struct SMetaCfg { - uint64_t lruSize; -}; - struct STsdbCfg { - int8_t precision; - int8_t update; - int8_t compression; - int8_t slLevel; - int32_t days; - int32_t minRows; - int32_t maxRows; - int32_t keep2; - int32_t keep0; - int32_t keep1; - uint64_t lruCacheSize; - SArray *retentions; + int8_t precision; + int8_t update; + int8_t compression; + int8_t slLevel; + int32_t days; + int32_t minRows; + int32_t maxRows; + int32_t keep0; + int32_t keep1; + int32_t keep2; + SArray *retentions; }; struct SVnodeCfg { @@ -160,8 +157,6 @@ struct SVnodeCfg { int32_t szCache; uint64_t szBuf; bool isHeap; - uint32_t ttl; - uint32_t keep; int8_t streamMode; bool isWeak; STsdbCfg tsdbCfg; diff --git a/source/dnode/vnode/src/inc/meta.h b/source/dnode/vnode/src/inc/meta.h index 9a0b1f4578..0001a43231 100644 --- a/source/dnode/vnode/src/inc/meta.h +++ b/source/dnode/vnode/src/inc/meta.h @@ -77,21 +77,25 @@ typedef struct { tb_uid_t uid; } STbDbKey; -typedef struct __attribute__((__packed__)) { +#pragma pack(push, 1) +typedef struct { tb_uid_t uid; int32_t sver; } SSkmDbKey; +#pragma pack(pop) typedef struct { tb_uid_t suid; tb_uid_t uid; } SCtbIdxKey; -typedef struct __attribute__((__packed__)) { +#pragma pack(push, 1) +typedef struct { tb_uid_t suid; int16_t cid; char data[]; } STagIdxKey; +#pragma pack(pop) typedef struct { int64_t dtime; diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 3349accb88..4fc94da06a 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -46,12 +46,45 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKe // tsdbCommit ================ -#if 1 +// tsdbFS ================ +typedef struct STsdbFS STsdbFS; -typedef struct SSmaStat SSmaStat; +// tsdbSma ================ typedef struct SSmaEnv SSmaEnv; typedef struct SSmaEnvs SSmaEnvs; +// structs +typedef struct { + int minFid; + int midFid; + int maxFid; + TSKEY minKey; +} SRtn; + +struct SSmaEnvs { + int16_t nTSma; + int16_t nRSma; + SSmaEnv *pTSmaEnv; + SSmaEnv *pRSmaEnv; +}; + +struct STsdb { + char *path; + SVnode *pVnode; + bool repoLocked; + TdThreadMutex mutex; + STsdbCfg config; + STsdbMemTable *mem; + STsdbMemTable *imem; + SRtn rtn; + STsdbFS *fs; + SSmaEnvs smaEnvs; +}; + +#if 1 // ====================================== + +typedef struct SSmaStat SSmaStat; + struct STable { uint64_t tid; uint64_t uid; @@ -97,13 +130,6 @@ typedef struct { uint8_t state; } SDFile; -struct SSmaEnvs { - int16_t nTSma; - int16_t nRSma; - SSmaEnv *pTSmaEnv; - SSmaEnv *pRSmaEnv; -}; - typedef struct { int fid; int8_t state; // -128~127 @@ -112,13 +138,6 @@ typedef struct { SDFile files[TSDB_FILE_MAX]; } SDFileSet; -typedef struct { - int minFid; - int midFid; - int maxFid; - TSKEY minKey; -} SRtn; - struct STbData { tb_uid_t uid; TSKEY keyMin; @@ -155,7 +174,7 @@ typedef struct { SArray *sf; // sma data file array v2f1900.index_name_1 } SFSStatus; -typedef struct { +struct STsdbFS { TdThreadRwlock lock; SFSStatus *cstatus; // current status @@ -163,23 +182,9 @@ typedef struct { SHashObj *metaCacheComp; // meta cache for compact bool intxn; SFSStatus *nstatus; // new status -} STsdbFS; - -struct STsdb { - char *path; - SVnode *pVnode; - int32_t vgId; - bool repoLocked; - TdThreadMutex mutex; - STsdbCfg config; - STsdbMemTable *mem; - STsdbMemTable *imem; - SRtn rtn; - STsdbFS *fs; - SSmaEnvs smaEnvs; }; -#define REPO_ID(r) ((r)->vgId) +#define REPO_ID(r) TD_VID((r)->pVnode) #define REPO_CFG(r) (&(r)->config) #define REPO_FS(r) ((r)->fs) #define REPO_META(r) ((r)->pVnode->pMeta) @@ -507,12 +512,6 @@ static FORCE_INLINE void *taosTZfree(void *ptr) { // tsdbCommit -typedef struct { - uint64_t uid; - int64_t offset; - int64_t size; -} SKVRecord; - void tsdbGetRtnSnap(STsdb *pRepo, SRtn *pRtn); static FORCE_INLINE int TSDB_KEY_FID(TSKEY key, int32_t days, int8_t precision) { @@ -891,66 +890,6 @@ static FORCE_INLINE int tsdbUnLockFS(STsdbFS *pFs) { return 0; } -// tsdbSma -// #define TSDB_SMA_TEST // remove after test finished - -// struct SSmaEnv { -// TdThreadRwlock lock; -// SDiskID did; -// TDBEnv dbEnv; // TODO: If it's better to put it in smaIndex level? -// char *path; // relative path -// SSmaStat *pStat; -// }; - -// #define SMA_ENV_LOCK(env) ((env)->lock) -// #define SMA_ENV_DID(env) ((env)->did) -// #define SMA_ENV_ENV(env) ((env)->dbEnv) -// #define SMA_ENV_PATH(env) ((env)->path) -// #define SMA_ENV_STAT(env) ((env)->pStat) -// #define SMA_ENV_STAT_ITEMS(env) ((env)->pStat->smaStatItems) - -// void tsdbDestroySmaEnv(SSmaEnv *pSmaEnv); -// void *tsdbFreeSmaEnv(SSmaEnv *pSmaEnv); -// #if 0 -// int32_t tsdbGetTSmaStatus(STsdb *pTsdb, STSma *param, void *result); -// int32_t tsdbRemoveTSmaData(STsdb *pTsdb, STSma *param, STimeWindow *pWin); -// #endif - -// // internal func -// static FORCE_INLINE int32_t tsdbEncodeTSmaKey(int64_t groupId, TSKEY tsKey, void **pData) { -// int32_t len = 0; -// len += taosEncodeFixedI64(pData, tsKey); -// len += taosEncodeFixedI64(pData, groupId); -// return len; -// } - -// static FORCE_INLINE int32_t tsdbRLockSma(SSmaEnv *pEnv) { -// int code = taosThreadRwlockRdlock(&(pEnv->lock)); -// if (code != 0) { -// terrno = TAOS_SYSTEM_ERROR(code); -// return -1; -// } -// return 0; -// } - -// static FORCE_INLINE int32_t tsdbWLockSma(SSmaEnv *pEnv) { -// int code = taosThreadRwlockWrlock(&(pEnv->lock)); -// if (code != 0) { -// terrno = TAOS_SYSTEM_ERROR(code); -// return -1; -// } -// return 0; -// } - -// static FORCE_INLINE int32_t tsdbUnLockSma(SSmaEnv *pEnv) { -// int code = taosThreadRwlockUnlock(&(pEnv->lock)); -// if (code != 0) { -// terrno = TAOS_SYSTEM_ERROR(code); -// return -1; -// } -// return 0; -// } - typedef struct SSmaKey SSmaKey; struct SSmaKey { diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index ec98a37669..b5066799ca 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -75,7 +75,6 @@ int metaCreateSTable(SMeta* pMeta, int64_t version, SVCreateStbReq* int metaCreateTable(SMeta* pMeta, int64_t version, SVCreateTbReq* pReq); SSchemaWrapper* metaGetTableSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver, bool isinline); STSchema* metaGetTbTSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver); -int metaGetTableEntryByUid(SMetaReader* pReader, tb_uid_t uid); int metaGetTableEntryByName(SMetaReader* pReader, const char* name); int metaGetTbNum(SMeta* pMeta); SMCtbCursor* metaOpenCtbCursor(SMeta* pMeta, tb_uid_t uid); diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 452d52ce5e..3e70bbb479 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -88,7 +88,7 @@ int metaCreateTable(SMeta *pMeta, int64_t version, SVCreateTbReq *pReq) { // preprocess req pReq->uid = tGenIdPI64(); - pReq->ctime = taosGetTimestampSec(); + pReq->ctime = taosGetTimestampMs(); // validate req metaReaderInit(&mr, pMeta, 0); @@ -158,7 +158,9 @@ static int metaSaveToTbDb(SMeta *pMeta, const SMetaEntry *pME) { pKey = &tbDbKey; kLen = sizeof(tbDbKey); - if (tEncodeSize(metaEncodeEntry, pME, vLen) < 0) { + int32_t ret = 0; + tEncodeSize(metaEncodeEntry, pME, vLen, ret); + if (ret < 0) { goto _err; } @@ -250,7 +252,9 @@ static int metaSaveToSkmDb(SMeta *pMeta, const SMetaEntry *pME) { skmDbKey.sver = pSW->sver; // encode schema - if (tEncodeSize(tEncodeSSchemaWrapper, pSW, vLen) < 0) return -1; + int32_t ret = 0; + tEncodeSize(tEncodeSSchemaWrapper, pSW, vLen, ret); + if (ret < 0) return -1; pVal = taosMemoryMalloc(vLen); if (pVal == NULL) { rcode = -1; diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 29f148b64d..8bcb848516 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -752,334 +752,7 @@ int tsdbWriteBlockIdx(SDFile *pHeadf, SArray *pIdxA, void **ppBuf) { return 0; } -// // =================== Commit Meta Data -// static int tsdbInitCommitMetaFile(STsdbRepo *pRepo, SMFile* pMf, bool open) { -// STsdbFS * pfs = REPO_FS(pRepo); -// SMFile * pOMFile = pfs->cstatus->pmf; -// SDiskID did; - -// // Create/Open a meta file or open the existing file -// if (pOMFile == NULL) { -// // Create a new meta file -// did.level = TFS_PRIMARY_LEVEL; -// did.id = TFS_PRIMARY_ID; -// tsdbInitMFile(pMf, did, REPO_ID(pRepo), FS_TXN_VERSION(REPO_FS(pRepo))); - -// if (open && tsdbCreateMFile(pMf, true) < 0) { -// tsdbError("vgId:%d failed to create META file since %s", REPO_ID(pRepo), tstrerror(terrno)); -// return -1; -// } - -// tsdbInfo("vgId:%d meta file %s is created to commit", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMf)); -// } else { -// tsdbInitMFileEx(pMf, pOMFile); -// if (open && tsdbOpenMFile(pMf, O_WRONLY) < 0) { -// tsdbError("vgId:%d failed to open META file since %s", REPO_ID(pRepo), tstrerror(terrno)); -// return -1; -// } -// } - -// return 0; -// } - -// static int tsdbCommitMeta(STsdbRepo *pRepo) { -// STsdbFS * pfs = REPO_FS(pRepo); -// SMemTable *pMem = pRepo->imem; -// SMFile * pOMFile = pfs->cstatus->pmf; -// SMFile mf; -// SActObj * pAct = NULL; -// SActCont * pCont = NULL; -// SListNode *pNode = NULL; - -// ASSERT(pOMFile != NULL || listNEles(pMem->actList) > 0); - -// if (listNEles(pMem->actList) <= 0) { -// // no meta data to commit, just keep the old meta file -// tsdbUpdateMFile(pfs, pOMFile); -// if (tsTsdbMetaCompactRatio > 0) { -// if (tsdbInitCommitMetaFile(pRepo, &mf, false) < 0) { -// return -1; -// } -// int ret = tsdbCompactMetaFile(pRepo, pfs, &mf); -// if (ret < 0) tsdbError("compact meta file error"); - -// return ret; -// } -// return 0; -// } else { -// if (tsdbInitCommitMetaFile(pRepo, &mf, true) < 0) { -// return -1; -// } -// } - -// // Loop to write -// while ((pNode = tdListPopHead(pMem->actList)) != NULL) { -// pAct = (SActObj *)pNode->data; -// if (pAct->act == TSDB_UPDATE_META) { -// pCont = (SActCont *)POINTER_SHIFT(pAct, sizeof(SActObj)); -// if (tsdbUpdateMetaRecord(pfs, &mf, pAct->uid, (void *)(pCont->cont), pCont->len, false) < 0) { -// tsdbError("vgId:%d failed to update META record, uid %" PRIu64 " since %s", REPO_ID(pRepo), pAct->uid, -// tstrerror(terrno)); -// tsdbCloseMFile(&mf); -// (void)tsdbApplyMFileChange(&mf, pOMFile); -// // TODO: need to reload metaCache -// return -1; -// } -// } else if (pAct->act == TSDB_DROP_META) { -// if (tsdbDropMetaRecord(pfs, &mf, pAct->uid) < 0) { -// tsdbError("vgId:%d failed to drop META record, uid %" PRIu64 " since %s", REPO_ID(pRepo), pAct->uid, -// tstrerror(terrno)); -// tsdbCloseMFile(&mf); -// tsdbApplyMFileChange(&mf, pOMFile); -// // TODO: need to reload metaCache -// return -1; -// } -// } else { -// ASSERT(false); -// } -// } - -// if (tsdbUpdateMFileHeader(&mf) < 0) { -// tsdbError("vgId:%d failed to update META file header since %s, revert it", REPO_ID(pRepo), tstrerror(terrno)); -// tsdbApplyMFileChange(&mf, pOMFile); -// // TODO: need to reload metaCache -// return -1; -// } - -// TSDB_FILE_FSYNC(&mf); -// tsdbCloseMFile(&mf); -// tsdbUpdateMFile(pfs, &mf); - -// if (tsTsdbMetaCompactRatio > 0 && tsdbCompactMetaFile(pRepo, pfs, &mf) < 0) { -// tsdbError("compact meta file error"); -// } - -// return 0; -// } - -// int tsdbEncodeKVRecord(void **buf, SKVRecord *pRecord) { -// int tlen = 0; -// tlen += taosEncodeFixedU64(buf, pRecord->uid); -// tlen += taosEncodeFixedI64(buf, pRecord->offset); -// tlen += taosEncodeFixedI64(buf, pRecord->size); - -// return tlen; -// } - -// void *tsdbDecodeKVRecord(void *buf, SKVRecord *pRecord) { -// buf = taosDecodeFixedU64(buf, &(pRecord->uid)); -// buf = taosDecodeFixedI64(buf, &(pRecord->offset)); -// buf = taosDecodeFixedI64(buf, &(pRecord->size)); - -// return buf; -// } - -// static int tsdbUpdateMetaRecord(STsdbFS *pfs, SMFile *pMFile, uint64_t uid, void *cont, int contLen, bool compact) { -// char buf[64] = "\0"; -// void * pBuf = buf; -// SKVRecord rInfo; -// int64_t offset; - -// // Seek to end of meta file -// offset = tsdbSeekMFile(pMFile, 0, SEEK_END); -// if (offset < 0) { -// return -1; -// } - -// rInfo.offset = offset; -// rInfo.uid = uid; -// rInfo.size = contLen; - -// int tlen = tsdbEncodeKVRecord((void **)(&pBuf), &rInfo); -// if (tsdbAppendMFile(pMFile, buf, tlen, NULL) < tlen) { -// return -1; -// } - -// if (tsdbAppendMFile(pMFile, cont, contLen, NULL) < contLen) { -// return -1; -// } - -// tsdbUpdateMFileMagic(pMFile, POINTER_SHIFT(cont, contLen - sizeof(TSCKSUM))); - -// SHashObj* cache = compact ? pfs->metaCacheComp : pfs->metaCache; - -// pMFile->info.nRecords++; - -// SKVRecord *pRecord = taosHashGet(cache, (void *)&uid, sizeof(uid)); -// if (pRecord != NULL) { -// pMFile->info.tombSize += (pRecord->size + sizeof(SKVRecord)); -// } else { -// pMFile->info.nRecords++; -// } -// taosHashPut(cache, (void *)(&uid), sizeof(uid), (void *)(&rInfo), sizeof(rInfo)); - -// return 0; -// } - -// static int tsdbDropMetaRecord(STsdbFS *pfs, SMFile *pMFile, uint64_t uid) { -// SKVRecord rInfo = {0}; -// char buf[128] = "\0"; - -// SKVRecord *pRecord = taosHashGet(pfs->metaCache, (void *)(&uid), sizeof(uid)); -// if (pRecord == NULL) { -// tsdbError("failed to drop META record with key %" PRIu64 " since not find", uid); -// return -1; -// } - -// rInfo.offset = -pRecord->offset; -// rInfo.uid = pRecord->uid; -// rInfo.size = pRecord->size; - -// void *pBuf = buf; -// tsdbEncodeKVRecord(&pBuf, &rInfo); - -// if (tsdbAppendMFile(pMFile, buf, sizeof(SKVRecord), NULL) < 0) { -// return -1; -// } - -// pMFile->info.magic = taosCalcChecksum(pMFile->info.magic, (uint8_t *)buf, sizeof(SKVRecord)); -// pMFile->info.nDels++; -// pMFile->info.nRecords--; -// pMFile->info.tombSize += (rInfo.size + sizeof(SKVRecord) * 2); - -// taosHashRemove(pfs->metaCache, (void *)(&uid), sizeof(uid)); -// return 0; -// } - -// static int tsdbCompactMetaFile(STsdbRepo *pRepo, STsdbFS *pfs, SMFile *pMFile) { -// float delPercent = (float)(pMFile->info.nDels) / (float)(pMFile->info.nRecords); -// float tombPercent = (float)(pMFile->info.tombSize) / (float)(pMFile->info.size); -// float compactRatio = (float)(tsTsdbMetaCompactRatio)/100; - -// if (delPercent < compactRatio && tombPercent < compactRatio) { -// return 0; -// } - -// if (tsdbOpenMFile(pMFile, O_RDONLY) < 0) { -// tsdbError("open meta file %s compact fail", pMFile->f.rname); -// return -1; -// } - -// tsdbInfo("begin compact tsdb meta file, ratio:%d, nDels:%" PRId64 ",nRecords:%" PRId64 ",tombSize:%" PRId64 -// ",size:%" PRId64, -// tsTsdbMetaCompactRatio, pMFile->info.nDels,pMFile->info.nRecords,pMFile->info.tombSize,pMFile->info.size); - -// SMFile mf; -// SDiskID did; - -// // first create tmp meta file -// did.level = TFS_PRIMARY_LEVEL; -// did.id = TFS_PRIMARY_ID; -// tsdbInitMFile(&mf, did, REPO_ID(pRepo), FS_TXN_VERSION(REPO_FS(pRepo)) + 1); - -// if (tsdbCreateMFile(&mf, true) < 0) { -// tsdbError("vgId:%d failed to create META file since %s", REPO_ID(pRepo), tstrerror(terrno)); -// return -1; -// } - -// tsdbInfo("vgId:%d meta file %s is created to compact meta data", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(&mf)); - -// // second iterator metaCache -// int code = -1; -// int64_t maxBufSize = 1024; -// SKVRecord *pRecord; -// void *pBuf = NULL; - -// pBuf = taosMemoryMalloc((size_t)maxBufSize); -// if (pBuf == NULL) { -// goto _err; -// } - -// // init Comp -// assert(pfs->metaCacheComp == NULL); -// pfs->metaCacheComp = taosHashInit(4096, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); -// if (pfs->metaCacheComp == NULL) { -// goto _err; -// } - -// pRecord = taosHashIterate(pfs->metaCache, NULL); -// while (pRecord) { -// if (tsdbSeekMFile(pMFile, pRecord->offset + sizeof(SKVRecord), SEEK_SET) < 0) { -// tsdbError("vgId:%d failed to seek file %s since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile), -// tstrerror(terrno)); -// goto _err; -// } -// if (pRecord->size > maxBufSize) { -// maxBufSize = pRecord->size; -// void* tmp = taosMemoryRealloc(pBuf, (size_t)maxBufSize); -// if (tmp == NULL) { -// goto _err; -// } -// pBuf = tmp; -// } -// int nread = (int)tsdbReadMFile(pMFile, pBuf, pRecord->size); -// if (nread < 0) { -// tsdbError("vgId:%d failed to read file %s since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile), -// tstrerror(terrno)); -// goto _err; -// } - -// if (nread < pRecord->size) { -// tsdbError("vgId:%d failed to read file %s since file corrupted, expected read:%" PRId64 " actual read:%d", -// REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile), pRecord->size, nread); -// goto _err; -// } - -// if (tsdbUpdateMetaRecord(pfs, &mf, pRecord->uid, pBuf, (int)pRecord->size, true) < 0) { -// tsdbError("vgId:%d failed to update META record, uid %" PRIu64 " since %s", REPO_ID(pRepo), pRecord->uid, -// tstrerror(terrno)); -// goto _err; -// } - -// pRecord = taosHashIterate(pfs->metaCache, pRecord); -// } -// code = 0; - -// _err: -// if (code == 0) TSDB_FILE_FSYNC(&mf); -// tsdbCloseMFile(&mf); -// tsdbCloseMFile(pMFile); - -// if (code == 0) { -// // rename meta.tmp -> meta -// tsdbInfo("vgId:%d meta file rename %s -> %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(&mf), -// TSDB_FILE_FULL_NAME(pMFile)); taosRename(mf.f.aname,pMFile->f.aname); tstrncpy(mf.f.aname, pMFile->f.aname, -// TSDB_FILENAME_LEN); tstrncpy(mf.f.rname, pMFile->f.rname, TSDB_FILENAME_LEN); -// // update current meta file info -// pfs->nstatus->pmf = NULL; -// tsdbUpdateMFile(pfs, &mf); - -// taosHashCleanup(pfs->metaCache); -// pfs->metaCache = pfs->metaCacheComp; -// pfs->metaCacheComp = NULL; -// } else { -// // remove meta.tmp file -// taosRemoveFile(mf.f.aname); -// taosHashCleanup(pfs->metaCacheComp); -// pfs->metaCacheComp = NULL; -// } - -// taosMemoryFreeClear(pBuf); - -// ASSERT(mf.info.nDels == 0); -// ASSERT(mf.info.tombSize == 0); - -// tsdbInfo("end compact tsdb meta file,code:%d,nRecords:%" PRId64 ",size:%" PRId64, -// code,mf.info.nRecords,mf.info.size); -// return code; -// } - -// // =================== Commit Time-Series Data -// #if 0 -// static bool tsdbHasDataToCommit(SCommitIter *iters, int nIters, TSKEY minKey, TSKEY maxKey) { -// for (int i = 0; i < nIters; i++) { -// TSKEY nextKey = tsdbNextIterKey((iters + i)->pIter); -// if (nextKey != TSDB_DATA_TIMESTAMP_NULL && (nextKey >= minKey && nextKey <= maxKey)) return true; -// } -// return false; -// } -// #endif - +// =================== Commit Time-Series Data static int tsdbCommitToTable(SCommitH *pCommith, int tid) { SCommitIter *pIter = pCommith->iters + tid; TSKEY nextKey = tsdbNextIterKey(pIter->pIter); diff --git a/source/dnode/vnode/src/tsdb/tsdbCompact.c b/source/dnode/vnode/src/tsdb/tsdbCompact.c deleted file mode 100644 index 9e0721815a..0000000000 --- a/source/dnode/vnode/src/tsdb/tsdbCompact.c +++ /dev/null @@ -1,533 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -#if 0 -#include "tsdb.h" - -typedef struct { - STable * pTable; - SBlockIdx * pBlkIdx; - SBlockIdx bindex; - SBlockInfo *pInfo; -} STableCompactH; - -typedef struct { - SRtn rtn; - SFSIter fsIter; - SArray * tbArray; // table array to cache table obj and block indexes - SReadH readh; - SDFileSet wSet; - SArray * aBlkIdx; - SArray * aSupBlk; - SDataCols *pDataCols; -} SCompactH; - -#define TSDB_COMPACT_WSET(pComph) (&((pComph)->wSet)) -#define TSDB_COMPACT_REPO(pComph) TSDB_READ_REPO(&((pComph)->readh)) -#define TSDB_COMPACT_HEAD_FILE(pComph) TSDB_DFILE_IN_SET(TSDB_COMPACT_WSET(pComph), TSDB_FILE_HEAD) -#define TSDB_COMPACT_DATA_FILE(pComph) TSDB_DFILE_IN_SET(TSDB_COMPACT_WSET(pComph), TSDB_FILE_DATA) -#define TSDB_COMPACT_LAST_FILE(pComph) TSDB_DFILE_IN_SET(TSDB_COMPACT_WSET(pComph), TSDB_FILE_LAST) -#define TSDB_COMPACT_SMAD_FILE(pComph) TSDB_DFILE_IN_SET(TSDB_COMPACT_WSET(pComph), TSDB_FILE_SMAD) -#define TSDB_COMPACT_SMAL_FILE(pComph) TSDB_DFILE_IN_SET(TSDB_COMPACT_WSET(pComph), TSDB_FILE_SMAL) -#define TSDB_COMPACT_BUF(pComph) TSDB_READ_BUF(&((pComph)->readh)) -#define TSDB_COMPACT_COMP_BUF(pComph) TSDB_READ_COMP_BUF(&((pComph)->readh)) - -static int tsdbAsyncCompact(STsdbRepo *pRepo); -static void tsdbStartCompact(STsdbRepo *pRepo); -static void tsdbEndCompact(STsdbRepo *pRepo, int eno); -static int tsdbCompactMeta(STsdbRepo *pRepo); -static int tsdbCompactTSData(STsdbRepo *pRepo); -static int tsdbCompactFSet(SCompactH *pComph, SDFileSet *pSet); -static bool tsdbShouldCompact(SCompactH *pComph); -static int tsdbInitCompactH(SCompactH *pComph, STsdbRepo *pRepo); -static void tsdbDestroyCompactH(SCompactH *pComph); -static int tsdbInitCompTbArray(SCompactH *pComph); -static void tsdbDestroyCompTbArray(SCompactH *pComph); -static int tsdbCacheFSetIndex(SCompactH *pComph); -static int tsdbCompactFSetInit(SCompactH *pComph, SDFileSet *pSet); -static void tsdbCompactFSetEnd(SCompactH *pComph); -static int tsdbCompactFSetImpl(SCompactH *pComph); -static int tsdbWriteBlockToRightFile(SCompactH *pComph, STable *pTable, SDataCols *pDataCols, void **ppBuf, - void **ppCBuf); - -enum { TSDB_NO_COMPACT, TSDB_IN_COMPACT, TSDB_WAITING_COMPACT}; -int tsdbCompact(STsdbRepo *pRepo) { return tsdbAsyncCompact(pRepo); } - -void *tsdbCompactImpl(STsdbRepo *pRepo) { - // Check if there are files in TSDB FS to compact - if (REPO_FS(pRepo)->cstatus->pmf == NULL) { - tsdbInfo("vgId:%d no file to compact in FS", REPO_ID(pRepo)); - return NULL; - } - - tsdbStartCompact(pRepo); - - if (tsdbCompactMeta(pRepo) < 0) { - tsdbError("vgId:%d failed to compact META data since %s", REPO_ID(pRepo), tstrerror(terrno)); - goto _err; - } - - if (tsdbCompactTSData(pRepo) < 0) { - tsdbError("vgId:%d failed to compact TS data since %s", REPO_ID(pRepo), tstrerror(terrno)); - goto _err; - } - - tsdbEndCompact(pRepo, TSDB_CODE_SUCCESS); - return NULL; - -_err: - pRepo->code = terrno; - tsdbEndCompact(pRepo, terrno); - return NULL; -} - -static int tsdbAsyncCompact(STsdbRepo *pRepo) { - if (pRepo->compactState != TSDB_NO_COMPACT) { - tsdbInfo("vgId:%d not compact tsdb again ", REPO_ID(pRepo)); - return 0; - } - pRepo->compactState = TSDB_WAITING_COMPACT; - tsem_wait(&(pRepo->readyToCommit)); - return tsdbScheduleCommit(pRepo, COMPACT_REQ); -} - -static void tsdbStartCompact(STsdbRepo *pRepo) { - assert(pRepo->compactState != TSDB_IN_COMPACT); - tsdbInfo("vgId:%d start to compact!", REPO_ID(pRepo)); - tsdbStartFSTxn(pRepo, 0, 0); - pRepo->code = TSDB_CODE_SUCCESS; - pRepo->compactState = TSDB_IN_COMPACT; -} - -static void tsdbEndCompact(STsdbRepo *pRepo, int eno) { - if (eno != TSDB_CODE_SUCCESS) { - tsdbEndFSTxnWithError(REPO_FS(pRepo)); - } else { - tsdbEndFSTxn(pRepo); - } - pRepo->compactState = TSDB_NO_COMPACT; - tsdbInfo("vgId:%d compact over, %s", REPO_ID(pRepo), (eno == TSDB_CODE_SUCCESS) ? "succeed" : "failed"); - tsem_post(&(pRepo->readyToCommit)); -} - -static int tsdbCompactMeta(STsdbRepo *pRepo) { - STsdbFS *pfs = REPO_FS(pRepo); - tsdbUpdateMFile(pfs, pfs->cstatus->pmf); - return 0; -} - - static int tsdbCompactTSData(STsdbRepo *pRepo) { - SCompactH compactH; - SDFileSet *pSet = NULL; - - tsdbDebug("vgId:%d start to compact TS data", REPO_ID(pRepo)); - - // If no file, just return 0; - if (taosArrayGetSize(REPO_FS(pRepo)->cstatus->df) <= 0) { - tsdbDebug("vgId:%d no TS data file to compact, compact over", REPO_ID(pRepo)); - return 0; - } - - if (tsdbInitCompactH(&compactH, pRepo) < 0) { - return -1; - } - - while ((pSet = tsdbFSIterNext(&(compactH.fsIter)))) { - // Remove those expired files - if (pSet->fid < compactH.rtn.minFid) { - tsdbInfo("vgId:%d FSET %d on level %d disk id %d expires, remove it", REPO_ID(pRepo), pSet->fid, - TSDB_FSET_LEVEL(pSet), TSDB_FSET_ID(pSet)); - continue; - } - - if (TSDB_FSET_LEVEL(pSet) == TFS_MAX_LEVEL) { - tsdbDebug("vgId:%d FSET %d on level %d, should not compact", REPO_ID(pRepo), pSet->fid, TFS_MAX_LEVEL); - tsdbUpdateDFileSet(REPO_FS(pRepo), pSet); - continue; - } - - if (tsdbCompactFSet(&compactH, pSet) < 0) { - tsdbDestroyCompactH(&compactH); - tsdbError("vgId:%d failed to compact FSET %d since %s", REPO_ID(pRepo), pSet->fid, tstrerror(terrno)); - return -1; - } - } - - tsdbDestroyCompactH(&compactH); - tsdbDebug("vgId:%d compact TS data over", REPO_ID(pRepo)); - return 0; - } - - static int tsdbCompactFSet(SCompactH *pComph, SDFileSet *pSet) { - STsdbRepo *pRepo = TSDB_COMPACT_REPO(pComph); - SDiskID did; - - tsdbDebug("vgId:%d start to compact FSET %d on level %d id %d", REPO_ID(pRepo), pSet->fid, TSDB_FSET_LEVEL(pSet), - TSDB_FSET_ID(pSet)); - - if (tsdbCompactFSetInit(pComph, pSet) < 0) { - return -1; - } - - if (!tsdbShouldCompact(pComph)) { - tsdbDebug("vgId:%d no need to compact FSET %d", REPO_ID(pRepo), pSet->fid); - if (tsdbApplyRtnOnFSet(TSDB_COMPACT_REPO(pComph), pSet, &(pComph->rtn)) < 0) { - tsdbCompactFSetEnd(pComph); - return -1; - } - } else { - // Create new fset as compacted fset - if (tfsAllocDisk(pRepo->pTfs, tsdbGetFidLevel(pSet->fid, &(pComph->rtn)), &did) < 0) { - terrno = TSDB_CODE_TDB_NO_AVAIL_DISK; - tsdbError("vgId:%d failed to compact FSET %d since %s", REPO_ID(pRepo), pSet->fid, tstrerror(terrno)); - tsdbCompactFSetEnd(pComph); - return -1; - } - - tsdbInitDFileSet(TSDB_COMPACT_WSET(pComph), did, REPO_ID(pRepo), TSDB_FSET_FID(pSet), - FS_TXN_VERSION(REPO_FS(pRepo))); - if (tsdbCreateDFileSet(TSDB_COMPACT_WSET(pComph), true) < 0) { - tsdbError("vgId:%d failed to compact FSET %d since %s", REPO_ID(pRepo), pSet->fid, tstrerror(terrno)); - tsdbCompactFSetEnd(pComph); - return -1; - } - - if (tsdbCompactFSetImpl(pComph) < 0) { - tsdbCloseDFileSet(TSDB_COMPACT_WSET(pComph)); - tsdbRemoveDFileSet(TSDB_COMPACT_WSET(pComph)); - tsdbCompactFSetEnd(pComph); - return -1; - } - - tsdbCloseDFileSet(TSDB_COMPACT_WSET(pComph)); - tsdbUpdateDFileSet(REPO_FS(pRepo), TSDB_COMPACT_WSET(pComph)); - tsdbDebug("vgId:%d FSET %d compact over", REPO_ID(pRepo), pSet->fid); - } - - tsdbCompactFSetEnd(pComph); - return 0; - } - - static bool tsdbShouldCompact(SCompactH *pComph) { - STsdbRepo * pRepo = TSDB_COMPACT_REPO(pComph); - STsdbCfg * pCfg = REPO_CFG(pRepo); - SReadH * pReadh = &(pComph->readh); - STableCompactH *pTh; - SBlock * pBlock; - int defaultRows = TSDB_DEFAULT_BLOCK_ROWS(pCfg->maxRowsPerFileBlock); - SDFile * pDataF = TSDB_READ_DATA_FILE(pReadh); - SDFile * pLastF = TSDB_READ_LAST_FILE(pReadh); - - int tblocks = 0; // total blocks - int nSubBlocks = 0; // # of blocks with sub-blocks - int nSmallBlocks = 0; // # of blocks with rows < defaultRows - int64_t tsize = 0; - - for (size_t i = 0; i < taosArrayGetSize(pComph->tbArray); i++) { - pTh = (STableCompactH *)taosArrayGet(pComph->tbArray, i); - - if (pTh->pTable == NULL || pTh->pBlkIdx == NULL) continue; - - for (size_t bidx = 0; bidx < pTh->pBlkIdx->numOfBlocks; bidx++) { - tblocks++; - pBlock = pTh->pInfo->blocks + bidx; - - if (pBlock->numOfRows < defaultRows) { - nSmallBlocks++; - } - - if (pBlock->numOfSubBlocks > 1) { - nSubBlocks++; - for (int k = 0; k < pBlock->numOfSubBlocks; k++) { - SBlock *iBlock = ((SBlock *)POINTER_SHIFT(pTh->pInfo, pBlock->offset)) + k; - tsize = tsize + iBlock->len; - } - } else if (pBlock->numOfSubBlocks == 1) { - tsize += pBlock->len; - } else { - ASSERT(0); - } - } - } - - return (((nSubBlocks * 1.0 / tblocks) > 0.33) || ((nSmallBlocks * 1.0 / tblocks) > 0.33) || - (tsize * 1.0 / (pDataF->info.size + pLastF->info.size - 2 * TSDB_FILE_HEAD_SIZE) < 0.85)); - } - - static int tsdbInitCompactH(SCompactH *pComph, STsdbRepo *pRepo) { - STsdbCfg *pCfg = REPO_CFG(pRepo); - - memset(pComph, 0, sizeof(*pComph)); - - TSDB_FSET_SET_CLOSED(TSDB_COMPACT_WSET(pComph)); - - tsdbGetRtnSnap(pRepo, &(pComph->rtn)); - tsdbFSIterInit(&(pComph->fsIter), REPO_FS(pRepo), TSDB_FS_ITER_FORWARD); - - if (tsdbInitReadH(&(pComph->readh), pRepo) < 0) { - return -1; - } - - if (tsdbInitCompTbArray(pComph) < 0) { - tsdbDestroyCompactH(pComph); - return -1; - } - - pComph->aBlkIdx = taosArrayInit(1024, sizeof(SBlockIdx)); - if (pComph->aBlkIdx == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - tsdbDestroyCompactH(pComph); - return -1; - } - - pComph->aSupBlk = taosArrayInit(1024, sizeof(SBlock)); - if (pComph->aSupBlk == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - tsdbDestroyCompactH(pComph); - return -1; - } - - pComph->pDataCols = tdNewDataCols(0, pCfg->maxRowsPerFileBlock); - if (pComph->pDataCols == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - tsdbDestroyCompactH(pComph); - return -1; - } - - return 0; - } - - static void tsdbDestroyCompactH(SCompactH *pComph) { - pComph->pDataCols = tdFreeDataCols(pComph->pDataCols); - pComph->aSupBlk = taosArrayDestroy(pComph->aSupBlk); - pComph->aBlkIdx = taosArrayDestroy(pComph->aBlkIdx); - tsdbDestroyCompTbArray(pComph); - tsdbDestroyReadH(&(pComph->readh)); - tsdbCloseDFileSet(TSDB_COMPACT_WSET(pComph)); - } - - static int tsdbInitCompTbArray(SCompactH *pComph) { // Init pComp->tbArray - STsdbRepo *pRepo = TSDB_COMPACT_REPO(pComph); - STsdbMeta *pMeta = pRepo->tsdbMeta; - - if (tsdbRLockRepoMeta(pRepo) < 0) return -1; - - pComph->tbArray = taosArrayInit(pMeta->maxTables, sizeof(STableCompactH)); - if (pComph->tbArray == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - tsdbUnlockRepoMeta(pRepo); - return -1; - } - - // Note here must start from 0 - for (int i = 0; i < pMeta->maxTables; i++) { - STableCompactH ch = {0}; - if (pMeta->tables[i] != NULL) { - tsdbRefTable(pMeta->tables[i]); - ch.pTable = pMeta->tables[i]; - } - - if (taosArrayPush(pComph->tbArray, &ch) == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - tsdbUnlockRepoMeta(pRepo); - return -1; - } - } - - if (tsdbUnlockRepoMeta(pRepo) < 0) return -1; - return 0; - } - - static void tsdbDestroyCompTbArray(SCompactH *pComph) { - STableCompactH *pTh; - - if (pComph->tbArray == NULL) return; - - for (size_t i = 0; i < taosArrayGetSize(pComph->tbArray); i++) { - pTh = (STableCompactH *)taosArrayGet(pComph->tbArray, i); - if (pTh->pTable) { - tsdbUnRefTable(pTh->pTable); - } - - pTh->pInfo = taosTZfree(pTh->pInfo); - } - - pComph->tbArray = taosArrayDestroy(pComph->tbArray); - } - - static int tsdbCacheFSetIndex(SCompactH *pComph) { - SReadH *pReadH = &(pComph->readh); - - if (tsdbLoadBlockIdx(pReadH) < 0) { - return -1; - } - - for (int tid = 1; tid < taosArrayGetSize(pComph->tbArray); tid++) { - STableCompactH *pTh = (STableCompactH *)taosArrayGet(pComph->tbArray, tid); - pTh->pBlkIdx = NULL; - - if (pTh->pTable == NULL) continue; - if (tsdbSetReadTable(pReadH, pTh->pTable) < 0) { - return -1; - } - - if (pReadH->pBlkIdx == NULL) continue; - pTh->bindex = *(pReadH->pBlkIdx); - pTh->pBlkIdx = &(pTh->bindex); - - if (tsdbMakeRoom((void **)(&(pTh->pInfo)), pTh->pBlkIdx->len) < 0) { - return -1; - } - - if (tsdbLoadBlockInfo(pReadH, (void *)(pTh->pInfo)) < 0) { - return -1; - } - } - - return 0; - } - - static int tsdbCompactFSetInit(SCompactH *pComph, SDFileSet *pSet) { - taosArrayClear(pComph->aBlkIdx); - taosArrayClear(pComph->aSupBlk); - - if (tsdbSetAndOpenReadFSet(&(pComph->readh), pSet) < 0) { - return -1; - } - - if (tsdbCacheFSetIndex(pComph) < 0) { - tsdbCloseAndUnsetFSet(&(pComph->readh)); - return -1; - } - - return 0; - } - - static void tsdbCompactFSetEnd(SCompactH *pComph) { tsdbCloseAndUnsetFSet(&(pComph->readh)); } - - static int tsdbCompactFSetImpl(SCompactH *pComph) { - STsdbRepo *pRepo = TSDB_COMPACT_REPO(pComph); - STsdbCfg * pCfg = REPO_CFG(pRepo); - SReadH * pReadh = &(pComph->readh); - SBlockIdx blkIdx; - void ** ppBuf = &(TSDB_COMPACT_BUF(pComph)); - void ** ppCBuf = &(TSDB_COMPACT_COMP_BUF(pComph)); - int defaultRows = TSDB_DEFAULT_BLOCK_ROWS(pCfg->maxRowsPerFileBlock); - - taosArrayClear(pComph->aBlkIdx); - - for (int tid = 1; tid < taosArrayGetSize(pComph->tbArray); tid++) { - STableCompactH *pTh = (STableCompactH *)taosArrayGet(pComph->tbArray, tid); - STSchema * pSchema; - - if (pTh->pTable == NULL || pTh->pBlkIdx == NULL) continue; - - pSchema = tsdbGetTableSchemaImpl(pTh->pTable, true, true, -1); - taosArrayClear(pComph->aSupBlk); - if ((tdInitDataCols(pComph->pDataCols, pSchema) < 0) || (tdInitDataCols(pReadh->pDCols[0], pSchema) < 0) || - (tdInitDataCols(pReadh->pDCols[1], pSchema) < 0)) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; - } - tdFreeSchema(pSchema); - - // Loop to compact each block data - for (int i = 0; i < pTh->pBlkIdx->numOfBlocks; i++) { - SBlock *pBlock = pTh->pInfo->blocks + i; - - // Load the block data - if (tsdbLoadBlockData(pReadh, pBlock, pTh->pInfo) < 0) { - return -1; - } - - // Merge pComph->pDataCols and pReadh->pDCols[0] and write data to file - if (pComph->pDataCols->numOfRows == 0 && pBlock->numOfRows >= defaultRows) { - if (tsdbWriteBlockToRightFile(pComph, pTh->pTable, pReadh->pDCols[0], ppBuf, ppCBuf) < 0) { - return -1; - } - } else { - int ridx = 0; - - while (true) { - if (pReadh->pDCols[0]->numOfRows - ridx == 0) break; - int rowsToMerge = TMIN(pReadh->pDCols[0]->numOfRows - ridx, defaultRows - pComph->pDataCols->numOfRows); - - tdMergeDataCols(pComph->pDataCols, pReadh->pDCols[0], rowsToMerge, &ridx, pCfg->update != TD_ROW_PARTIAL_UPDATE); - - if (pComph->pDataCols->numOfRows < defaultRows) { - break; - } - - if (tsdbWriteBlockToRightFile(pComph, pTh->pTable, pComph->pDataCols, ppBuf, ppCBuf) < 0) { - return -1; - } - tdResetDataCols(pComph->pDataCols); - } - } - } - - if (pComph->pDataCols->numOfRows > 0 && - tsdbWriteBlockToRightFile(pComph, pTh->pTable, pComph->pDataCols, ppBuf, ppCBuf) < 0) { - return -1; - } - - if (tsdbWriteBlockInfoImpl(TSDB_COMPACT_HEAD_FILE(pComph), pTh->pTable, pComph->aSupBlk, NULL, ppBuf, &blkIdx) < - 0) { - return -1; - } - - if ((blkIdx.numOfBlocks > 0) && (taosArrayPush(pComph->aBlkIdx, (void *)(&blkIdx)) == NULL)) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; - } - } - - if (tsdbWriteBlockIdx(TSDB_COMPACT_HEAD_FILE(pComph), pComph->aBlkIdx, ppBuf) < 0) { - return -1; - } - - return 0; - } - - static int tsdbWriteBlockToRightFile(SCompactH *pComph, STable *pTable, SDataCols *pDataCols, void **ppBuf, - void **ppCBuf) { - STsdbRepo *pRepo = TSDB_COMPACT_REPO(pComph); - STsdbCfg * pCfg = REPO_CFG(pRepo); - SDFile * pDFile; - bool isLast; - SBlock block; - - ASSERT(pDataCols->numOfRows > 0); - - if (pDataCols->numOfRows < pCfg->minRowsPerFileBlock) { - pDFile = TSDB_COMPACT_LAST_FILE(pComph); - isLast = true; - } else { - pDFile = TSDB_COMPACT_DATA_FILE(pComph); - isLast = false; - } - - if (tsdbWriteBlockImpl(pRepo, pTable, pDFile, pDataCols, &block, isLast, true, ppBuf, ppCBuf) < 0) { - return -1; - } - - if (taosArrayPush(pComph->aSupBlk, (void *)(&block)) == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; - } - - return 0; -} - -#endif diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index 91a7e1cd54..4ff3227e08 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -246,62 +246,6 @@ void *tsdbFreeFS(STsdbFS *pfs) { return NULL; } -// static int tsdbProcessExpiredFS(STsdb *pRepo) { -// tsdbStartFSTxn(pRepo, 0, 0); -// // if (tsdbCreateMeta(pRepo) < 0) { -// // tsdbError("vgId:%d failed to create meta since %s", REPO_ID(pRepo), tstrerror(terrno)); -// // return -1; -// // } - -// if (tsdbApplyRtn(pRepo) < 0) { -// tsdbEndFSTxnWithError(REPO_FS(pRepo)); -// tsdbError("vgId:%d failed to apply rtn since %s", REPO_ID(pRepo), tstrerror(terrno)); -// return -1; -// } -// if (tsdbEndFSTxn(pRepo) < 0) { -// tsdbError("vgId:%d failed to end fs txn since %s", REPO_ID(pRepo), tstrerror(terrno)); -// return -1; -// } -// return 0; -// } - -// static int tsdbCreateMeta(STsdb *pRepo) { -// STsdbFS *pfs = REPO_FS(pRepo); -// SMFile * pOMFile = pfs->cstatus->pmf; -// SMFile mf; -// SDiskID did; - -// if (pOMFile != NULL) { -// // keep the old meta file -// tsdbUpdateMFile(pfs, pOMFile); -// return 0; -// } - -// // Create a new meta file -// did.level = TFS_PRIMARY_LEVEL; -// did.id = TFS_PRIMARY_ID; -// tsdbInitMFile(&mf, did, REPO_ID(pRepo), FS_TXN_VERSION(REPO_FS(pRepo))); - -// if (tsdbCreateMFile(&mf, true) < 0) { -// tsdbError("vgId:%d failed to create META file since %s", REPO_ID(pRepo), tstrerror(terrno)); -// return -1; -// } - -// tsdbInfo("vgId:%d meta file %s is created", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(&mf)); - -// if (tsdbUpdateMFileHeader(&mf) < 0) { -// tsdbError("vgId:%d failed to update META file header since %s, revert it", REPO_ID(pRepo), tstrerror(terrno)); -// tsdbApplyMFileChange(&mf, pOMFile); -// return -1; -// } - -// TSDB_FILE_FSYNC(&mf); -// tsdbCloseMFile(&mf); -// tsdbUpdateMFile(pfs, &mf); - -// return 0; -// } - int tsdbOpenFS(STsdb *pRepo) { STsdbFS *pfs = REPO_FS(pRepo); char current[TSDB_FILENAME_LEN] = "\0"; @@ -644,7 +588,7 @@ static int tsdbComparFidFSet(const void *arg1, const void *arg2) { } static void tsdbGetTxnFname(STsdb *pRepo, TSDB_TXN_FILE_T ftype, char fname[]) { - snprintf(fname, TSDB_FILENAME_LEN, "%s/vnode/vnode%d/tsdb/%s", tfsGetPrimaryPath(REPO_TFS(pRepo)), pRepo->vgId, + snprintf(fname, TSDB_FILENAME_LEN, "%s/vnode/vnode%d/tsdb/%s", tfsGetPrimaryPath(REPO_TFS(pRepo)), REPO_ID(pRepo), tsdbTxnFname[ftype]); } @@ -769,142 +713,6 @@ static int tsdbScanAndTryFixFS(STsdb *pRepo) { return 0; } -// int tsdbLoadMetaCache(STsdb *pRepo, bool recoverMeta) { -// char tbuf[128]; -// STsdbFS * pfs = REPO_FS(pRepo); -// SMFile mf; -// SMFile * pMFile = &mf; -// void * pBuf = NULL; -// SKVRecord rInfo; -// int64_t maxBufSize = 0; -// SMFInfo minfo; - -// taosHashClear(pfs->metaCache); - -// // No meta file, just return -// if (pfs->cstatus->pmf == NULL) return 0; - -// mf = pfs->cstatus->mf; -// // Load cache first -// if (tsdbOpenMFile(pMFile, O_RDONLY) < 0) { -// return -1; -// } - -// if (tsdbLoadMFileHeader(pMFile, &minfo) < 0) { -// tsdbCloseMFile(pMFile); -// return -1; -// } - -// while (true) { -// int64_t tsize = tsdbReadMFile(pMFile, tbuf, sizeof(SKVRecord)); -// if (tsize == 0) break; - -// if (tsize < 0) { -// tsdbError("vgId:%d failed to read META file since %s", REPO_ID(pRepo), tstrerror(terrno)); -// return -1; -// } - -// if (tsize < sizeof(SKVRecord)) { -// tsdbError("vgId:%d failed to read %" PRIzu " bytes from file %s", REPO_ID(pRepo), sizeof(SKVRecord), -// TSDB_FILE_FULL_NAME(pMFile)); -// terrno = TSDB_CODE_TDB_FILE_CORRUPTED; -// tsdbCloseMFile(pMFile); -// return -1; -// } - -// void *ptr = tsdbDecodeKVRecord(tbuf, &rInfo); -// ASSERT(POINTER_DISTANCE(ptr, tbuf) == sizeof(SKVRecord)); -// // ASSERT((rInfo.offset > 0) ? (pStore->info.size == rInfo.offset) : true); - -// if (rInfo.offset < 0) { -// taosHashRemove(pfs->metaCache, (void *)(&rInfo.uid), sizeof(rInfo.uid)); -// #if 0 -// pStore->info.size += sizeof(SKVRecord); -// pStore->info.nRecords--; -// pStore->info.nDels++; -// pStore->info.tombSize += (rInfo.size + sizeof(SKVRecord) * 2); -// #endif -// } else { -// ASSERT(rInfo.offset > 0 && rInfo.size > 0); -// if (taosHashPut(pfs->metaCache, (void *)(&rInfo.uid), sizeof(rInfo.uid), &rInfo, sizeof(rInfo)) < 0) { -// tsdbError("vgId:%d failed to load meta cache from file %s since OOM", REPO_ID(pRepo), -// TSDB_FILE_FULL_NAME(pMFile)); -// terrno = TSDB_CODE_COM_OUT_OF_MEMORY; -// tsdbCloseMFile(pMFile); -// return -1; -// } - -// maxBufSize = TMAX(maxBufSize, rInfo.size); - -// if (tsdbSeekMFile(pMFile, rInfo.size, SEEK_CUR) < 0) { -// tsdbError("vgId:%d failed to lseek file %s since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile), -// tstrerror(terrno)); -// tsdbCloseMFile(pMFile); -// return -1; -// } - -// #if 0 -// pStore->info.size += (sizeof(SKVRecord) + rInfo.size); -// pStore->info.nRecords++; -// #endif -// } -// } - -// if (recoverMeta) { -// pBuf = taosMemoryMalloc((size_t)maxBufSize); -// if (pBuf == NULL) { -// terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; -// tsdbCloseMFile(pMFile); -// return -1; -// } - -// SKVRecord *pRecord = taosHashIterate(pfs->metaCache, NULL); -// while (pRecord) { -// if (tsdbSeekMFile(pMFile, pRecord->offset + sizeof(SKVRecord), SEEK_SET) < 0) { -// tsdbError("vgId:%d failed to seek file %s since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile), -// tstrerror(terrno)); -// taosMemoryFreeClear(pBuf); -// tsdbCloseMFile(pMFile); -// return -1; -// } - -// int nread = (int)tsdbReadMFile(pMFile, pBuf, pRecord->size); -// if (nread < 0) { -// tsdbError("vgId:%d failed to read file %s since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile), -// tstrerror(terrno)); -// taosMemoryFreeClear(pBuf); -// tsdbCloseMFile(pMFile); -// return -1; -// } - -// if (nread < pRecord->size) { -// tsdbError("vgId:%d failed to read file %s since file corrupted, expected read:%" PRId64 " actual read:%d", -// REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile), pRecord->size, nread); -// terrno = TSDB_CODE_TDB_FILE_CORRUPTED; -// taosMemoryFreeClear(pBuf); -// tsdbCloseMFile(pMFile); -// return -1; -// } - -// if (tsdbRestoreTable(pRepo, pBuf, (int)pRecord->size) < 0) { -// tsdbError("vgId:%d failed to restore table, uid %" PRId64 ", since %s" PRIu64, REPO_ID(pRepo), pRecord->uid, -// tstrerror(terrno)); -// taosMemoryFreeClear(pBuf); -// tsdbCloseMFile(pMFile); -// return -1; -// } - -// pRecord = taosHashIterate(pfs->metaCache, pRecord); -// } - -// tsdbOrgMeta(pRepo); -// } - -// tsdbCloseMFile(pMFile); -// taosMemoryFreeClear(pBuf); -// return 0; -// } - static int tsdbScanRootDir(STsdb *pRepo) { char rootDir[TSDB_FILENAME_LEN]; char bname[TSDB_FILENAME_LEN]; @@ -983,127 +791,6 @@ static bool tsdbIsTFileInFS(STsdbFS *pfs, const STfsFile *pf) { return false; } -// static int tsdbRestoreMeta(STsdb *pRepo) { -// char rootDir[TSDB_FILENAME_LEN]; -// char bname[TSDB_FILENAME_LEN]; -// STfsDir * tdir = NULL; -// const STfsFile *pf = NULL; -// const char * pattern = "^meta(-ver[0-9]+)?$"; -// regex_t regex; -// STsdbFS * pfs = REPO_FS(pRepo); - -// regcomp(®ex, pattern, REG_EXTENDED); - -// tsdbInfo("vgId:%d try to restore meta", REPO_ID(pRepo)); - -// tsdbGetRootDir(REPO_ID(pRepo), rootDir); - -// tdir = tfsOpendir(rootDir); -// if (tdir == NULL) { -// tsdbError("vgId:%d failed to open dir %s since %s", REPO_ID(pRepo), rootDir, tstrerror(terrno)); -// regfree(®ex); -// return -1; -// } - -// while ((pf = tfsReaddir(tdir))) { -// tfsBasename(pf, bname); - -// if (strcmp(bname, "data") == 0) { -// // Skip the data/ directory -// continue; -// } - -// if (strcmp(bname, tsdbTxnFname[TSDB_TXN_TEMP_FILE]) == 0) { -// // Skip current.t file -// tsdbInfo("vgId:%d file %s exists, remove it", REPO_ID(pRepo), pf->aname); -// (void)tfsremove(pf); -// continue; -// } - -// int code = regexec(®ex, bname, 0, NULL, 0); -// if (code == 0) { -// // Match -// if (pfs->cstatus->pmf != NULL) { -// tsdbError("vgId:%d failed to restore meta since two file exists, file1 %s and file2 %s", REPO_ID(pRepo), -// TSDB_FILE_FULL_NAME(pfs->cstatus->pmf), pf->aname); -// terrno = TSDB_CODE_TDB_FILE_CORRUPTED; -// tfsClosedir(tdir); -// regfree(®ex); -// return -1; -// } else { -// uint32_t _version = 0; -// if (strcmp(bname, "meta") != 0) { -// sscanf(bname, "meta-ver%" PRIu32, &_version); -// pfs->cstatus->meta.version = _version; -// } - -// pfs->cstatus->pmf = &(pfs->cstatus->mf); -// pfs->cstatus->pmf->f = *pf; -// TSDB_FILE_SET_CLOSED(pfs->cstatus->pmf); - -// if (tsdbOpenMFile(pfs->cstatus->pmf, O_RDONLY) < 0) { -// tsdbError("vgId:%d failed to restore meta since %s", REPO_ID(pRepo), tstrerror(terrno)); -// tfsClosedir(tdir); -// regfree(®ex); -// return -1; -// } - -// if (tsdbLoadMFileHeader(pfs->cstatus->pmf, &(pfs->cstatus->pmf->info)) < 0) { -// tsdbError("vgId:%d failed to restore meta since %s", REPO_ID(pRepo), tstrerror(terrno)); -// tsdbCloseMFile(pfs->cstatus->pmf); -// tfsClosedir(tdir); -// regfree(®ex); -// return -1; -// } - -// if (tsdbForceKeepFile) { -// struct stat tfstat; - -// // Get real file size -// if (fstat(pfs->cstatus->pmf->fd, &tfstat) < 0) { -// terrno = TAOS_SYSTEM_ERROR(errno); -// tsdbCloseMFile(pfs->cstatus->pmf); -// tfsClosedir(tdir); -// regfree(®ex); -// return -1; -// } - -// if (pfs->cstatus->pmf->info.size != tfstat.st_size) { -// int64_t tfsize = pfs->cstatus->pmf->info.size; -// pfs->cstatus->pmf->info.size = tfstat.st_size; -// tsdbInfo("vgId:%d file %s header size is changed from %" PRId64 " to %" PRId64, REPO_ID(pRepo), -// TSDB_FILE_FULL_NAME(pfs->cstatus->pmf), tfsize, pfs->cstatus->pmf->info.size); -// } -// } - -// tsdbCloseMFile(pfs->cstatus->pmf); -// } -// } else if (code == REG_NOMATCH) { -// // Not match -// tsdbInfo("vgId:%d invalid file %s exists, remove it", REPO_ID(pRepo), pf->aname); -// tfsremove(pf); -// continue; -// } else { -// // Has other error -// tsdbError("vgId:%d failed to restore meta file while run regexec since %s", REPO_ID(pRepo), strerror(code)); -// terrno = TAOS_SYSTEM_ERROR(code); -// tfsClosedir(tdir); -// regfree(®ex); -// return -1; -// } -// } - -// if (pfs->cstatus->pmf) { -// tsdbInfo("vgId:%d meta file %s is restored", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pfs->cstatus->pmf)); -// } else { -// tsdbInfo("vgId:%d no meta file is restored", REPO_ID(pRepo)); -// } - -// tfsClosedir(tdir); -// regfree(®ex); -// return 0; -// } - static int tsdbRestoreDFileSet(STsdb *pRepo) { char dataDir[TSDB_FILENAME_LEN]; char bname[TSDB_FILENAME_LEN]; diff --git a/source/dnode/vnode/src/tsdb/tsdbFile.c b/source/dnode/vnode/src/tsdb/tsdbFile.c index 74e3c66a9d..b1af3e0461 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFile.c +++ b/source/dnode/vnode/src/tsdb/tsdbFile.c @@ -33,271 +33,6 @@ static int tsdbEncodeDFInfo(void **buf, SDFInfo *pInfo); static void *tsdbDecodeDFInfo(void *buf, SDFInfo *pInfo); static int tsdbRollBackDFile(SDFile *pDFile); -#if 0 -// ============== SMFile -void tsdbInitMFile(SMFile *pMFile, SDiskID did, int vid, uint32_t ver) { - char fname[TSDB_FILENAME_LEN]; - - TSDB_FILE_SET_STATE(pMFile, TSDB_FILE_STATE_OK); - - memset(&(pMFile->info), 0, sizeof(pMFile->info)); - pMFile->info.magic = TSDB_FILE_INIT_MAGIC; - - tsdbGetFilename(vid, 0, ver, TSDB_FILE_META, fname); - tfsInitFile(TSDB_FILE_F(pMFile), did.level, did.id, fname); -} - -void tsdbInitMFileEx(SMFile *pMFile, const SMFile *pOMFile) { - *pMFile = *pOMFile; - TSDB_FILE_SET_CLOSED(pMFile); -} - -int tsdbEncodeSMFile(void **buf, SMFile *pMFile) { - int tlen = 0; - - tlen += tsdbEncodeMFInfo(buf, &(pMFile->info)); - tlen += tfsEncodeFile(buf, &(pMFile->f)); - - return tlen; -} - -void *tsdbDecodeSMFile(void *buf, SMFile *pMFile) { - buf = tsdbDecodeMFInfo(buf, &(pMFile->info)); - buf = tfsDecodeFile(buf, &(pMFile->f)); - TSDB_FILE_SET_CLOSED(pMFile); - - return buf; -} - -int tsdbEncodeSMFileEx(void **buf, SMFile *pMFile) { - int tlen = 0; - - tlen += tsdbEncodeMFInfo(buf, &(pMFile->info)); - tlen += taosEncodeString(buf, TSDB_FILE_FULL_NAME(pMFile)); - - return tlen; -} - -void *tsdbDecodeSMFileEx(void *buf, SMFile *pMFile) { - char *aname; - buf = tsdbDecodeMFInfo(buf, &(pMFile->info)); - buf = taosDecodeString(buf, &aname); - strncpy(TSDB_FILE_FULL_NAME(pMFile), aname, TSDB_FILENAME_LEN); - TSDB_FILE_SET_CLOSED(pMFile); - - taosMemoryFreeClear(aname); - - return buf; -} - -int tsdbApplyMFileChange(SMFile *from, SMFile *to) { - if (from == NULL && to == NULL) return 0; - - if (from != NULL) { - if (to == NULL) { - return tsdbRemoveMFile(from); - } else { - if (tfsIsSameFile(TSDB_FILE_F(from), TSDB_FILE_F(to))) { - if (from->info.size > to->info.size) { - tsdbRollBackMFile(to); - } - } else { - return tsdbRemoveMFile(from); - } - } - } - - return 0; -} - -int tsdbCreateMFile(SMFile *pMFile, bool updateHeader) { - ASSERT(pMFile->info.size == 0 && pMFile->info.magic == TSDB_FILE_INIT_MAGIC); - - pMFile->fd = open(TSDB_FILE_FULL_NAME(pMFile), O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0755); - if (pMFile->fd < 0) { - if (errno == ENOENT) { - // Try to create directory recursively - char *s = strdup(TFILE_REL_NAME(&(pMFile->f))); - if (tfsMkdirRecurAt(dirname(s), TSDB_FILE_LEVEL(pMFile), TSDB_FILE_ID(pMFile)) < 0) { - taosMemoryFreeClear(s); - return -1; - } - taosMemoryFreeClear(s); - - pMFile->fd = open(TSDB_FILE_FULL_NAME(pMFile), O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0755); - if (pMFile->fd < 0) { - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - } else { - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - } - - if (!updateHeader) { - return 0; - } - - pMFile->info.size += TSDB_FILE_HEAD_SIZE; - - if (tsdbUpdateMFileHeader(pMFile) < 0) { - tsdbCloseMFile(pMFile); - tsdbRemoveMFile(pMFile); - return -1; - } - - return 0; -} - -int tsdbUpdateMFileHeader(SMFile *pMFile) { - char buf[TSDB_FILE_HEAD_SIZE] = "\0"; - - if (tsdbSeekMFile(pMFile, 0, SEEK_SET) < 0) { - return -1; - } - - void *ptr = buf; - tsdbEncodeMFInfo(&ptr, TSDB_FILE_INFO(pMFile)); - - taosCalcChecksumAppend(0, (uint8_t *)buf, TSDB_FILE_HEAD_SIZE); - if (tsdbWriteMFile(pMFile, buf, TSDB_FILE_HEAD_SIZE) < 0) { - return -1; - } - - return 0; -} - -int tsdbLoadMFileHeader(SMFile *pMFile, SMFInfo *pInfo) { - char buf[TSDB_FILE_HEAD_SIZE] = "\0"; - - ASSERT(TSDB_FILE_OPENED(pMFile)); - - if (tsdbSeekMFile(pMFile, 0, SEEK_SET) < 0) { - return -1; - } - - if (tsdbReadMFile(pMFile, buf, TSDB_FILE_HEAD_SIZE) < 0) { - return -1; - } - - if (!taosCheckChecksumWhole((uint8_t *)buf, TSDB_FILE_HEAD_SIZE)) { - terrno = TSDB_CODE_TDB_FILE_CORRUPTED; - return -1; - } - - tsdbDecodeMFInfo(buf, pInfo); - return 0; -} - -int tsdbScanAndTryFixMFile(STsdb *pRepo) { - SMFile * pMFile = pRepo->fs->cstatus->pmf; - struct stat mfstat; - SMFile mf; - - if (pMFile == NULL) { - // No meta file, no need to scan - return 0; - } - - tsdbInitMFileEx(&mf, pMFile); - - if (access(TSDB_FILE_FULL_NAME(pMFile), F_OK) != 0) { - tsdbError("vgId:%d meta file %s not exit, report to upper layer to fix it", REPO_ID(pRepo), - TSDB_FILE_FULL_NAME(pMFile)); - pRepo->state |= TSDB_STATE_BAD_META; - TSDB_FILE_SET_STATE(pMFile, TSDB_FILE_STATE_BAD); - return 0; - } - - if (stat(TSDB_FILE_FULL_NAME(&mf), &mfstat) < 0) { - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - if (pMFile->info.size < mfstat.st_size) { - if (tsdbOpenMFile(&mf, O_WRONLY) < 0) { - return -1; - } - - if (taosFtruncate(mf.fd, mf.info.size) < 0) { - terrno = TAOS_SYSTEM_ERROR(errno); - tsdbCloseMFile(&mf); - return -1; - } - - if (tsdbUpdateMFileHeader(&mf) < 0) { - tsdbCloseMFile(&mf); - return -1; - } - - tsdbCloseMFile(&mf); - tsdbInfo("vgId:%d file %s is truncated from %" PRId64 " to %" PRId64, REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile), - mfstat.st_size, pMFile->info.size); - } else if (pMFile->info.size > mfstat.st_size) { - tsdbError("vgId:%d meta file %s has wrong size %" PRId64 " expected %" PRId64 ", report to upper layer to fix it", - REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile), mfstat.st_size, pMFile->info.size); - pRepo->state |= TSDB_STATE_BAD_META; - TSDB_FILE_SET_STATE(pMFile, TSDB_FILE_STATE_BAD); - terrno = TSDB_CODE_TDB_FILE_CORRUPTED; - return 0; - } else { - tsdbDebug("vgId:%d meta file %s passes the scan", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile)); - } - - return 0; -} - -int tsdbEncodeMFInfo(void **buf, SMFInfo *pInfo) { - int tlen = 0; - - tlen += taosEncodeVariantI64(buf, pInfo->size); - tlen += taosEncodeVariantI64(buf, pInfo->tombSize); - tlen += taosEncodeVariantI64(buf, pInfo->nRecords); - tlen += taosEncodeVariantI64(buf, pInfo->nDels); - tlen += taosEncodeFixedU32(buf, pInfo->magic); - - return tlen; -} - -void *tsdbDecodeMFInfo(void *buf, SMFInfo *pInfo) { - buf = taosDecodeVariantI64(buf, &(pInfo->size)); - buf = taosDecodeVariantI64(buf, &(pInfo->tombSize)); - buf = taosDecodeVariantI64(buf, &(pInfo->nRecords)); - buf = taosDecodeVariantI64(buf, &(pInfo->nDels)); - buf = taosDecodeFixedU32(buf, &(pInfo->magic)); - - return buf; -} - -static int tsdbRollBackMFile(SMFile *pMFile) { - SMFile mf; - - tsdbInitMFileEx(&mf, pMFile); - - if (tsdbOpenMFile(&mf, O_WRONLY) < 0) { - return -1; - } - - if (taosFtruncate(TSDB_FILE_FD(&mf), pMFile->info.size) < 0) { - terrno = TAOS_SYSTEM_ERROR(errno); - tsdbCloseMFile(&mf); - return -1; - } - - if (tsdbUpdateMFileHeader(&mf) < 0) { - tsdbCloseMFile(&mf); - return -1; - } - - TSDB_FILE_FSYNC(&mf); - - tsdbCloseMFile(&mf); - return 0; -} - -#endif - // ============== Operations on SDFile void tsdbInitDFile(STsdb *pRepo, SDFile *pDFile, SDiskID did, int fid, uint32_t ver, TSDB_FILE_T ftype) { char fname[TSDB_FILENAME_LEN]; @@ -310,7 +45,7 @@ void tsdbInitDFile(STsdb *pRepo, SDFile *pDFile, SDiskID did, int fid, uint32_t pDFile->info.magic = TSDB_FILE_INIT_MAGIC; pDFile->info.fver = tsdbGetDFSVersion(ftype); - tsdbGetFilename(pRepo->vgId, fid, ver, ftype, fname); + tsdbGetFilename(REPO_ID(pRepo), fid, ver, ftype, fname); tfsInitFile(REPO_TFS(pRepo), &(pDFile->f), did, fname); } diff --git a/source/dnode/vnode/src/tsdb/tsdbMain.c b/source/dnode/vnode/src/tsdb/tsdbMain.c deleted file mode 100644 index de5ff9ac91..0000000000 --- a/source/dnode/vnode/src/tsdb/tsdbMain.c +++ /dev/null @@ -1,1105 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "tsdb.h" - -int tsdbOpen(SVnode *pVnode, STsdb **ppTsdb) { - STsdb *pTsdb = NULL; - int slen = 0; - - *ppTsdb = NULL; - slen = strlen(tfsGetPrimaryPath(pVnode->pTfs)) + strlen(pVnode->path) + strlen(VNODE_TSDB_DIR) + 3; - - // create handle - pTsdb = (STsdb *)taosMemoryCalloc(1, sizeof(*pTsdb) + slen); - if (pTsdb == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - pTsdb->path = (char *)&pTsdb[1]; - sprintf(pTsdb->path, "%s%s%s%s%s", tfsGetPrimaryPath(pVnode->pTfs), TD_DIRSEP, pVnode->path, TD_DIRSEP, - VNODE_TSDB_DIR); - pTsdb->pVnode = pVnode; - pTsdb->vgId = TD_VID(pVnode); - pTsdb->repoLocked = false; - tdbMutexInit(&pTsdb->mutex, NULL); - pTsdb->config = pVnode->config.tsdbCfg; - pTsdb->fs = tsdbNewFS(&pTsdb->config); - - // create dir (TODO: use tfsMkdir) - taosMkDir(pTsdb->path); - - // open tsdb - if (tsdbOpenFS(pTsdb) < 0) { - goto _err; - } - - tsdbDebug("vgId: %d tsdb is opened", TD_VID(pVnode)); - - *ppTsdb = pTsdb; - return 0; - -_err: - taosMemoryFree(pTsdb); - return -1; -} - -int tsdbClose(STsdb *pTsdb) { - if (pTsdb) { - tsdbCloseFS(pTsdb); - // tsdbFreeSmaEnv(REPO_TSMA_ENV(pTsdb)); - // tsdbFreeSmaEnv(REPO_RSMA_ENV(pTsdb)); - tsdbFreeFS(pTsdb->fs); - // taosMemoryFreeClear(pTsdb->path); - taosMemoryFree(pTsdb); - } - return 0; -} - -int tsdbLockRepo(STsdb *pTsdb) { - int code = taosThreadMutexLock(&pTsdb->mutex); - if (code != 0) { - tsdbError("vgId:%d failed to lock tsdb since %s", REPO_ID(pTsdb), strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(code); - return -1; - } - pTsdb->repoLocked = true; - return 0; -} - -int tsdbUnlockRepo(STsdb *pTsdb) { - ASSERT(IS_REPO_LOCKED(pTsdb)); - pTsdb->repoLocked = false; - int code = taosThreadMutexUnlock(&pTsdb->mutex); - if (code != 0) { - tsdbError("vgId:%d failed to unlock tsdb since %s", REPO_ID(pTsdb), strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(code); - return -1; - } - return 0; -} - -#if 0 -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -// no test file errors here -#include "taosdef.h" -#include "tsdbint.h" -#include "tthread.h" -#include "ttimer.h" - -#define IS_VALID_PRECISION(precision) \ - (((precision) >= TSDB_TIME_PRECISION_MILLI) && ((precision) <= TSDB_TIME_PRECISION_NANO)) -#define TSDB_DEFAULT_COMPRESSION TWO_STAGE_COMP -#define IS_VALID_COMPRESSION(compression) (((compression) >= NO_COMPRESSION) && ((compression) <= TWO_STAGE_COMP)) - -static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg); -static STsdbRepo *tsdbNewRepo(STsdbCfg *pCfg, STsdbAppH *pAppH); -static void tsdbFreeRepo(STsdbRepo *pRepo); -static void tsdbStartStream(STsdbRepo *pRepo); -static void tsdbStopStream(STsdbRepo *pRepo); -static int tsdbRestoreLastColumns(STsdbRepo *pRepo, STable *pTable, SReadH* pReadh); -static int tsdbRestoreLastRow(STsdbRepo *pRepo, STable *pTable, SReadH* pReadh, SBlockIdx *pIdx); - -// Function declaration -int32_t tsdbCreateRepo(int repoid) { - char tsdbDir[TSDB_FILENAME_LEN] = "\0"; - char dataDir[TSDB_FILENAME_LEN] = "\0"; - - tsdbGetRootDir(repoid, tsdbDir); - if (tfsMkdir(tsdbDir) < 0) { - goto _err; - } - - tsdbGetDataDir(repoid, dataDir); - if (tfsMkdir(dataDir) < 0) { - goto _err; - } - - // TODO: need to create current file with nothing in - - return 0; - -_err: - tsdbError("vgId:%d failed to create TSDB repository since %s", repoid, tstrerror(terrno)); - return -1; -} - -int32_t tsdbDropRepo(int repoid) { - char tsdbDir[TSDB_FILENAME_LEN] = "\0"; - - tsdbGetRootDir(repoid, tsdbDir); - return tfsRmdir(tsdbDir); -} - -STsdbRepo *tsdbOpenRepo(STsdbCfg *pCfg, STsdbAppH *pAppH) { - STsdbRepo *pRepo; - STsdbCfg config = *pCfg; - - terrno = TSDB_CODE_SUCCESS; - - // Check and set default configurations - if (tsdbCheckAndSetDefaultCfg(&config) < 0) { - tsdbError("vgId:%d failed to open TSDB repository since %s", config.tsdbId, tstrerror(terrno)); - return NULL; - } - - // Create new TSDB object - if ((pRepo = tsdbNewRepo(&config, pAppH)) == NULL) { - tsdbError("vgId:%d failed to open TSDB repository while creating TSDB object since %s", config.tsdbId, - tstrerror(terrno)); - return NULL; - } - - // Open meta - if (tsdbOpenMeta(pRepo) < 0) { - tsdbError("vgId:%d failed to open TSDB repository while opening Meta since %s", config.tsdbId, tstrerror(terrno)); - tsdbCloseRepo(pRepo, false); - return NULL; - } - - if (tsdbOpenBufPool(pRepo) < 0) { - tsdbError("vgId:%d failed to open TSDB repository while opening buffer pool since %s", config.tsdbId, - tstrerror(terrno)); - tsdbCloseRepo(pRepo, false); - return NULL; - } - - if (tsdbOpenFS(pRepo) < 0) { - tsdbError("vgId:%d failed to open TSDB repository while opening FS since %s", config.tsdbId, tstrerror(terrno)); - tsdbCloseRepo(pRepo, false); - return NULL; - } - - // TODO: Restore information from data - if ((!(pRepo->state & TSDB_STATE_BAD_DATA)) && tsdbRestoreInfo(pRepo) < 0) { - tsdbError("vgId:%d failed to open TSDB repository while restore info since %s", config.tsdbId, tstrerror(terrno)); - tsdbCloseRepo(pRepo, false); - return NULL; - } - - pRepo->mergeBuf = NULL; - - tsdbStartStream(pRepo); - - tsdbDebug("vgId:%d, TSDB repository opened", REPO_ID(pRepo)); - - return pRepo; -} - -// Note: all working thread and query thread must stopped when calling this function -int tsdbCloseRepo(STsdbRepo *repo, int toCommit) { - if (repo == NULL) return 0; - - STsdbRepo *pRepo = repo; - int vgId = REPO_ID(pRepo); - - terrno = TSDB_CODE_SUCCESS; - - tsdbStopStream(pRepo); - if(pRepo->pthread){ - taosDestoryThread(pRepo->pthread); - pRepo->pthread = NULL; - } - - if (toCommit) { - tsdbSyncCommit(repo); - } - - tsem_wait(&(pRepo->readyToCommit)); - - tsdbUnRefMemTable(pRepo, pRepo->mem); - tsdbUnRefMemTable(pRepo, pRepo->imem); - pRepo->mem = NULL; - pRepo->imem = NULL; - - tsdbCloseFS(pRepo); - tsdbCloseBufPool(pRepo); - tsdbCloseMeta(pRepo); - tsdbFreeRepo(pRepo); - tsdbDebug("vgId:%d repository is closed", vgId); - - if (terrno != TSDB_CODE_SUCCESS) { - return -1; - } else { - return 0; - } -} - -STsdbCfg *tsdbGetCfg(const STsdbRepo *repo) { - ASSERT(repo != NULL); - return &((STsdbRepo *)repo)->config; -} - -int tsdbLockRepo(STsdbRepo *pRepo) { - int code = taosThreadMutexLock(&pRepo->mutex); - if (code != 0) { - tsdbError("vgId:%d failed to lock tsdb since %s", REPO_ID(pRepo), strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(code); - return -1; - } - pRepo->repoLocked = true; - return 0; -} - -int tsdbUnlockRepo(STsdbRepo *pRepo) { - ASSERT(IS_REPO_LOCKED(pRepo)); - pRepo->repoLocked = false; - int code = taosThreadMutexUnlock(&pRepo->mutex); - if (code != 0) { - tsdbError("vgId:%d failed to unlock tsdb since %s", REPO_ID(pRepo), strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(code); - return -1; - } - return 0; -} - -int tsdbCheckCommit(STsdbRepo *pRepo) { - ASSERT(pRepo->mem != NULL); - STsdbCfg *pCfg = &(pRepo->config); - - STsdbBufBlock *pBufBlock = tsdbGetCurrBufBlock(pRepo); - ASSERT(pBufBlock != NULL); - if ((pRepo->mem->extraBuffList != NULL) || - ((listNEles(pRepo->mem->bufBlockList) >= pCfg->totalBlocks / 3) && (pBufBlock->remain < TSDB_BUFFER_RESERVE))) { - // trigger commit - if (tsdbAsyncCommit(pRepo) < 0) return -1; - } - - return 0; -} - -STsdbMeta *tsdbGetMeta(STsdbRepo *pRepo) { return pRepo->tsdbMeta; } - -STsdbRepoInfo *tsdbGetStatus(STsdbRepo *pRepo) { return NULL; } - -int tsdbGetState(STsdbRepo *repo) { return repo->state; } - -int8_t tsdbGetCompactState(STsdbRepo *repo) { return (int8_t)(repo->compactState); } - -void tsdbReportStat(void *repo, int64_t *totalPoints, int64_t *totalStorage, int64_t *compStorage) { - ASSERT(repo != NULL); - STsdbRepo *pRepo = repo; - *totalPoints = pRepo->stat.pointsWritten; - *totalStorage = pRepo->stat.totalStorage; - *compStorage = pRepo->stat.compStorage; -} - -int32_t tsdbConfigRepo(STsdbRepo *repo, STsdbCfg *pCfg) { - // TODO: think about multithread cases - if (tsdbCheckAndSetDefaultCfg(pCfg) < 0) return -1; - - STsdbCfg * pRCfg = &repo->config; - - ASSERT(pRCfg->tsdbId == pCfg->tsdbId); - ASSERT(pRCfg->cacheBlockSize == pCfg->cacheBlockSize); - ASSERT(pRCfg->daysPerFile == pCfg->daysPerFile); - ASSERT(pRCfg->minRowsPerFileBlock == pCfg->minRowsPerFileBlock); - ASSERT(pRCfg->maxRowsPerFileBlock == pCfg->maxRowsPerFileBlock); - ASSERT(pRCfg->precision == pCfg->precision); - - bool configChanged = false; - if (pRCfg->compression != pCfg->compression) { - configChanged = true; - } - if (pRCfg->keep != pCfg->keep) { - configChanged = true; - } - if (pRCfg->keep1 != pCfg->keep1) { - configChanged = true; - } - if (pRCfg->keep2 != pCfg->keep2) { - configChanged = true; - } - if (pRCfg->cacheLastRow != pCfg->cacheLastRow) { - configChanged = true; - } - if (pRCfg->totalBlocks != pCfg->totalBlocks) { - configChanged = true; - } - - if (!configChanged) { - tsdbError("vgId:%d no config changed", REPO_ID(repo)); - } - - int code = taosThreadMutexLock(&repo->save_mutex); - if (code != 0) { - tsdbError("vgId:%d failed to lock tsdb save config mutex since %s", REPO_ID(repo), strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(code); - return -1; - } - - STsdbCfg * pSaveCfg = &repo->save_config; - *pSaveCfg = repo->config; - - pSaveCfg->compression = pCfg->compression; - pSaveCfg->keep = pCfg->keep; - pSaveCfg->keep1 = pCfg->keep1; - pSaveCfg->keep2 = pCfg->keep2; - pSaveCfg->cacheLastRow = pCfg->cacheLastRow; - pSaveCfg->totalBlocks = pCfg->totalBlocks; - - tsdbInfo("vgId:%d old config: compression(%d), keep(%d,%d,%d), cacheLastRow(%d),totalBlocks(%d)", - REPO_ID(repo), - pRCfg->compression, pRCfg->keep, pRCfg->keep1,pRCfg->keep2, - pRCfg->cacheLastRow, pRCfg->totalBlocks); - tsdbInfo("vgId:%d new config: compression(%d), keep(%d,%d,%d), cacheLastRow(%d),totalBlocks(%d)", - REPO_ID(repo), - pSaveCfg->compression, pSaveCfg->keep,pSaveCfg->keep1, pSaveCfg->keep2, - pSaveCfg->cacheLastRow,pSaveCfg->totalBlocks); - - repo->config_changed = true; - - taosThreadMutexUnlock(&repo->save_mutex); - - // schedule a commit msg and wait for the new config applied - tsdbSyncCommitConfig(repo); - - return 0; -#if 0 - STsdbRepo *pRepo = (STsdbRepo *)repo; - STsdbCfg config = pRepo->config; - STsdbCfg * pRCfg = &pRepo->config; - - if (tsdbCheckAndSetDefaultCfg(pCfg) < 0) return -1; - - ASSERT(pRCfg->tsdbId == pCfg->tsdbId); - ASSERT(pRCfg->cacheBlockSize == pCfg->cacheBlockSize); - ASSERT(pRCfg->daysPerFile == pCfg->daysPerFile); - ASSERT(pRCfg->minRowsPerFileBlock == pCfg->minRowsPerFileBlock); - ASSERT(pRCfg->maxRowsPerFileBlock == pCfg->maxRowsPerFileBlock); - ASSERT(pRCfg->precision == pCfg->precision); - - bool configChanged = false; - if (pRCfg->compression != pCfg->compression) { - tsdbAlterCompression(pRepo, pCfg->compression); - config.compression = pCfg->compression; - configChanged = true; - } - if (pRCfg->keep != pCfg->keep) { - if (tsdbAlterKeep(pRepo, pCfg->keep) < 0) { - tsdbError("vgId:%d failed to configure repo when alter keep since %s", REPO_ID(pRepo), tstrerror(terrno)); - config.keep = pCfg->keep; - return -1; - } - configChanged = true; - } - if (pRCfg->totalBlocks != pCfg->totalBlocks) { - tsdbAlterCacheTotalBlocks(pRepo, pCfg->totalBlocks); - config.totalBlocks = pCfg->totalBlocks; - configChanged = true; - } - if (pRCfg->cacheLastRow != pCfg->cacheLastRow) { - config.cacheLastRow = pCfg->cacheLastRow; - configChanged = true; - } - - if (configChanged) { - if (tsdbSaveConfig(pRepo->rootDir, &config) < 0) { - tsdbError("vgId:%d failed to configure repository while save config since %s", REPO_ID(pRepo), tstrerror(terrno)); - return -1; - } - } - - return 0; -#endif -} - -uint32_t tsdbGetFileInfo(STsdbRepo *repo, char *name, uint32_t *index, uint32_t eindex, int64_t *size) { - // TODO - return 0; -#if 0 - STsdbRepo *pRepo = (STsdbRepo *)repo; - // STsdbMeta *pMeta = pRepo->tsdbMeta; - STsdbFileH *pFileH = pRepo->tsdbFileH; - uint32_t magic = 0; - char * fname = NULL; - - struct stat fState; - - tsdbDebug("vgId:%d name:%s index:%d eindex:%d", pRepo->config.tsdbId, name, *index, eindex); - ASSERT(*index <= eindex); - - if (name[0] == 0) { // get the file from index or after, but not larger than eindex - int fid = (*index) / TSDB_FILE_TYPE_MAX; - - if (pFileH->nFGroups == 0 || fid > pFileH->pFGroup[pFileH->nFGroups - 1].fileId) { - if (*index <= TSDB_META_FILE_INDEX && TSDB_META_FILE_INDEX <= eindex) { - fname = tsdbGetMetaFileName(pRepo->rootDir); - *index = TSDB_META_FILE_INDEX; - magic = TSDB_META_FILE_MAGIC(pRepo->tsdbMeta); - sprintf(name, "tsdb/%s", TSDB_META_FILE_NAME); - } else { - return 0; - } - } else { - SFileGroup *pFGroup = - taosbsearch(&fid, pFileH->pFGroup, pFileH->nFGroups, sizeof(SFileGroup), keyFGroupCompFunc, TD_GE); - if (pFGroup->fileId == fid) { - SFile *pFile = &pFGroup->files[(*index) % TSDB_FILE_TYPE_MAX]; - fname = strdup(TSDB_FILE_NAME(pFile)); - magic = pFile->info.magic; - char *tfname = strdup(fname); - sprintf(name, "tsdb/%s/%s", TSDB_DATA_DIR_NAME, basename(tfname)); - taosMemoryFreeClear(tfname); - } else { - if ((pFGroup->fileId + 1) * TSDB_FILE_TYPE_MAX - 1 < (int)eindex) { - SFile *pFile = &pFGroup->files[0]; - fname = strdup(TSDB_FILE_NAME(pFile)); - *index = pFGroup->fileId * TSDB_FILE_TYPE_MAX; - magic = pFile->info.magic; - char *tfname = strdup(fname); - sprintf(name, "tsdb/%s/%s", TSDB_DATA_DIR_NAME, basename(tfname)); - taosMemoryFreeClear(tfname); - } else { - return 0; - } - } - } - } else { // get the named file at the specified index. If not there, return 0 - fname = taosMemoryMalloc(256); - sprintf(fname, "%s/vnode/vnode%d/%s", tfsGetPrimaryPath(pRepo->pTfs), REPO_ID(pRepo), name); - if (access(fname, F_OK) != 0) { - taosMemoryFreeClear(fname); - return 0; - } - if (*index == TSDB_META_FILE_INDEX) { // get meta file - tsdbGetStoreInfo(fname, &magic, size); - } else { - char tfname[TSDB_FILENAME_LEN] = "\0"; - sprintf(tfname, "vnode/vnode%d/tsdb/%s/%s", REPO_ID(pRepo), TSDB_DATA_DIR_NAME, basename(name)); - tsdbGetFileInfoImpl(tfname, &magic, size); - } - taosMemoryFreeClear(fname); - return magic; - } - - if (stat(fname, &fState) < 0) { - taosMemoryFreeClear(fname); - return 0; - } - - *size = fState.st_size; - // magic = *size; - - taosMemoryFreeClear(fname); - return magic; -#endif -} - -void tsdbGetRootDir(int repoid, char dirName[]) { - snprintf(dirName, TSDB_FILENAME_LEN, "vnode/vnode%d/tsdb", repoid); -} - -void tsdbGetDataDir(int repoid, char dirName[]) { - snprintf(dirName, TSDB_FILENAME_LEN, "vnode/vnode%d/tsdb/data", repoid); -} - -static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg) { - // Check tsdbId - if (pCfg->tsdbId < 0) { - tsdbError("vgId:%d invalid vgroup ID", pCfg->tsdbId); - terrno = TSDB_CODE_TDB_INVALID_CONFIG; - return -1; - } - - // Check precision - if (pCfg->precision == -1) { - pCfg->precision = TSDB_DEFAULT_PRECISION; - } else { - if (!IS_VALID_PRECISION(pCfg->precision)) { - tsdbError("vgId:%d invalid precision configuration %d", pCfg->tsdbId, pCfg->precision); - terrno = TSDB_CODE_TDB_INVALID_CONFIG; - return -1; - } - } - - // Check compression - if (pCfg->compression == -1) { - pCfg->compression = TSDB_DEFAULT_COMPRESSION; - } else { - if (!IS_VALID_COMPRESSION(pCfg->compression)) { - tsdbError("vgId:%d invalid compression configuration %d", pCfg->tsdbId, pCfg->precision); - terrno = TSDB_CODE_TDB_INVALID_CONFIG; - return -1; - } - } - - // Check daysPerFile - if (pCfg->daysPerFile == -1) { - pCfg->daysPerFile = TSDB_DEFAULT_DAYS_PER_FILE; - } else { - if (pCfg->daysPerFile < TSDB_MIN_DAYS_PER_FILE || pCfg->daysPerFile > TSDB_MAX_DAYS_PER_FILE) { - tsdbError( - "vgId:%d invalid daysPerFile configuration! daysPerFile %d TSDB_MIN_DAYS_PER_FILE %d TSDB_MAX_DAYS_PER_FILE " - "%d", - pCfg->tsdbId, pCfg->daysPerFile, TSDB_MIN_DAYS_PER_FILE, TSDB_MAX_DAYS_PER_FILE); - terrno = TSDB_CODE_TDB_INVALID_CONFIG; - return -1; - } - } - - // Check minRowsPerFileBlock and maxRowsPerFileBlock - if (pCfg->minRowsPerFileBlock == -1) { - pCfg->minRowsPerFileBlock = TSDB_DEFAULT_MINROWS_FBLOCK; - } else { - if (pCfg->minRowsPerFileBlock < TSDB_MIN_MINROWS_FBLOCK || pCfg->minRowsPerFileBlock > TSDB_MAX_MINROWS_FBLOCK) { - tsdbError( - "vgId:%d invalid minRowsPerFileBlock configuration! minRowsPerFileBlock %d TSDB_MIN_MINROWS_FBLOCK %d " - "TSDB_MAX_MINROWS_FBLOCK %d", - pCfg->tsdbId, pCfg->minRowsPerFileBlock, TSDB_MIN_MINROWS_FBLOCK, TSDB_MAX_MINROWS_FBLOCK); - terrno = TSDB_CODE_TDB_INVALID_CONFIG; - return -1; - } - } - - if (pCfg->maxRowsPerFileBlock == -1) { - pCfg->maxRowsPerFileBlock = TSDB_DEFAULT_MAXROWS_FBLOCK; - } else { - if (pCfg->maxRowsPerFileBlock < TSDB_MIN_MAXROWS_FBLOCK || pCfg->maxRowsPerFileBlock > TSDB_MAX_MAXROWS_FBLOCK) { - tsdbError( - "vgId:%d invalid maxRowsPerFileBlock configuration! maxRowsPerFileBlock %d TSDB_MIN_MAXROWS_FBLOCK %d " - "TSDB_MAX_MAXROWS_FBLOCK %d", - pCfg->tsdbId, pCfg->maxRowsPerFileBlock, TSDB_MIN_MINROWS_FBLOCK, TSDB_MAX_MINROWS_FBLOCK); - terrno = TSDB_CODE_TDB_INVALID_CONFIG; - return -1; - } - } - - if (pCfg->minRowsPerFileBlock > pCfg->maxRowsPerFileBlock) { - tsdbError("vgId:%d invalid configuration! minRowsPerFileBlock %d maxRowsPerFileBlock %d", pCfg->tsdbId, - pCfg->minRowsPerFileBlock, pCfg->maxRowsPerFileBlock); - terrno = TSDB_CODE_TDB_INVALID_CONFIG; - return -1; - } - - // Check keep - if (pCfg->keep == -1) { - pCfg->keep = TSDB_DEFAULT_KEEP; - } else { - if (pCfg->keep < TSDB_MIN_KEEP || pCfg->keep > TSDB_MAX_KEEP) { - tsdbError( - "vgId:%d invalid keep configuration! keep %d TSDB_MIN_KEEP %d " - "TSDB_MAX_KEEP %d", - pCfg->tsdbId, pCfg->keep, TSDB_MIN_KEEP, TSDB_MAX_KEEP); - terrno = TSDB_CODE_TDB_INVALID_CONFIG; - return -1; - } - } - - if (pCfg->keep1 == 0) { - pCfg->keep1 = pCfg->keep; - } - - if (pCfg->keep2 == 0) { - pCfg->keep2 = pCfg->keep; - } - - // update check - if (pCfg->update < TD_ROW_DISCARD_UPDATE || pCfg->update > TD_ROW_PARTIAL_UPDATE) - pCfg->update = TD_ROW_DISCARD_UPDATE; - - // update cacheLastRow - if (pCfg->cacheLastRow != 0) { - if (pCfg->cacheLastRow > 3) - pCfg->cacheLastRow = 1; - } - return 0; -} - -static STsdbRepo *tsdbNewRepo(STsdbCfg *pCfg, STsdbAppH *pAppH) { - STsdbRepo *pRepo = (STsdbRepo *)taosMemoryCalloc(1, sizeof(*pRepo)); - if (pRepo == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return NULL; - } - - pRepo->state = TSDB_STATE_OK; - pRepo->code = TSDB_CODE_SUCCESS; - pRepo->compactState = 0; - pRepo->config = *pCfg; - if (pAppH) { - pRepo->appH = *pAppH; - } - pRepo->repoLocked = false; - pRepo->pthread = NULL; - - int code = taosThreadMutexInit(&(pRepo->mutex), NULL); - if (code != 0) { - terrno = TAOS_SYSTEM_ERROR(code); - tsdbFreeRepo(pRepo); - return NULL; - } - - code = taosThreadMutexInit(&(pRepo->save_mutex), NULL); - if (code != 0) { - terrno = TAOS_SYSTEM_ERROR(code); - tsdbFreeRepo(pRepo); - return NULL; - } - pRepo->config_changed = false; - atomic_store_8(&pRepo->hasCachedLastColumn, 0); - - code = tsem_init(&(pRepo->readyToCommit), 0, 1); - if (code != 0) { - code = errno; - terrno = TAOS_SYSTEM_ERROR(code); - tsdbFreeRepo(pRepo); - return NULL; - } - - pRepo->tsdbMeta = tsdbNewMeta(pCfg); - if (pRepo->tsdbMeta == NULL) { - tsdbError("vgId:%d failed to create meta since %s", REPO_ID(pRepo), tstrerror(terrno)); - tsdbFreeRepo(pRepo); - return NULL; - } - - pRepo->pPool = tsdbNewBufPool(pCfg); - if (pRepo->pPool == NULL) { - tsdbError("vgId:%d failed to create buffer pool since %s", REPO_ID(pRepo), tstrerror(terrno)); - tsdbFreeRepo(pRepo); - return NULL; - } - - pRepo->fs = tsdbNewFS(pCfg); - if (pRepo->fs == NULL) { - tsdbError("vgId:%d failed to TSDB file system since %s", REPO_ID(pRepo), tstrerror(terrno)); - tsdbFreeRepo(pRepo); - return NULL; - } - - return pRepo; -} - -static void tsdbFreeRepo(STsdbRepo *pRepo) { - if (pRepo) { - tsdbFreeFS(pRepo->fs); - tsdbFreeBufPool(pRepo->pPool); - tsdbFreeMeta(pRepo->tsdbMeta); - tsdbFreeMergeBuf(pRepo->mergeBuf); - // tsdbFreeMemTable(pRepo->mem); - // tsdbFreeMemTable(pRepo->imem); - tsem_destroy(&(pRepo->readyToCommit)); - taosThreadMutexDestroy(&pRepo->mutex); - taosMemoryFree(pRepo); - } -} - -static void tsdbStartStream(STsdbRepo *pRepo) { - STsdbMeta *pMeta = pRepo->tsdbMeta; - - for (int i = 0; i < pMeta->maxTables; i++) { - STable *pTable = pMeta->tables[i]; - if (pTable && pTable->type == TSDB_STREAM_TABLE) { - pTable->cqhandle = (*pRepo->appH.cqCreateFunc)(pRepo->appH.cqH, TABLE_UID(pTable), TABLE_TID(pTable), TABLE_NAME(pTable)->data, pTable->sql, - tsdbGetTableSchemaImpl(pTable, false, false, -1), 0); - } - } -} - -static void tsdbStopStream(STsdbRepo *pRepo) { - STsdbMeta *pMeta = pRepo->tsdbMeta; - - for (int i = 0; i < pMeta->maxTables; i++) { - STable *pTable = pMeta->tables[i]; - if (pTable && pTable->type == TSDB_STREAM_TABLE) { - (*pRepo->appH.cqDropFunc)(pTable->cqhandle); - } - } -} - -static int tsdbRestoreLastColumns(STsdbRepo *pRepo, STable *pTable, SReadH* pReadh) { - //tsdbInfo("tsdbRestoreLastColumns of table %s", pTable->name->data); - - STSchema *pSchema = tsdbGetTableLatestSchema(pTable); - if (pSchema == NULL) { - tsdbError("tsdbGetTableLatestSchema of table %s fail", pTable->name->data); - return 0; - } - - SBlock* pBlock; - int numColumns; - int32_t blockIdx; - SDataStatis* pBlockStatis = NULL; - STSRow* row = NULL; - // restore last column data with last schema - - int err = 0; - - numColumns = schemaNCols(pSchema); - if (numColumns <= pTable->restoreColumnNum) { - pTable->hasRestoreLastColumn = true; - return 0; - } - if (pTable->lastColSVersion != schemaVersion(pSchema)) { - if (tsdbInitColIdCacheWithSchema(pTable, pSchema) < 0) { - return -1; - } - } - - row = taosTMalloc(memRowMaxBytesFromSchema(pSchema)); - if (row == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - err = -1; - goto out; - } - - memRowSetType(row, SMEM_ROW_DATA); - tdInitDataRow(memRowDataBody(row), pSchema); - - // first load block index info - if (tsdbLoadBlockInfo(pReadh, NULL) < 0) { - err = -1; - goto out; - } - - pBlockStatis = taosMemoryCalloc(numColumns, sizeof(SDataStatis)); - if (pBlockStatis == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - err = -1; - goto out; - } - memset(pBlockStatis, 0, numColumns * sizeof(SDataStatis)); - for(int32_t i = 0; i < numColumns; ++i) { - STColumn *pCol = schemaColAt(pSchema, i); - pBlockStatis[i].colId = pCol->colId; - } - - // load block from backward - SBlockIdx *pIdx = pReadh->pBlkIdx; - blockIdx = (int32_t)(pIdx->numOfBlocks - 1); - - while (numColumns > pTable->restoreColumnNum && blockIdx >= 0) { - bool loadStatisData = false; - pBlock = pReadh->pBlkInfo->blocks + blockIdx; - blockIdx -= 1; - - // load block data - if (tsdbLoadBlockData(pReadh, pBlock, NULL) < 0) { - err = -1; - goto out; - } - - // file block with sub-blocks has no statistics data - if (pBlock->numOfSubBlocks <= 1) { - if (tsdbLoadBlockStatis(pReadh, pBlock) == TSDB_STATIS_OK) { - tsdbGetBlockStatis(pReadh, pBlockStatis, (int)numColumns, pBlock); - loadStatisData = true; - } - } - - for (int16_t i = 0; i < numColumns && numColumns > pTable->restoreColumnNum; ++i) { - STColumn *pCol = schemaColAt(pSchema, i); - // ignore loaded columns - if (pTable->lastCols[i].bytes != 0) { - continue; - } - - // ignore block which has no not-null colId column - if (loadStatisData && pBlockStatis[i].numOfNull == pBlock->numOfRows) { - continue; - } - - // OK,let's load row from backward to get not-null column - for (int32_t rowId = pBlock->numOfRows - 1; rowId >= 0; rowId--) { - SDataCol *pDataCol = pReadh->pDCols[0]->cols + i; - const void* pColData = tdGetColDataOfRow(pDataCol, rowId); - tdAppendColVal(memRowDataBody(row), pColData, pCol->type, pCol->offset); - //SDataCol *pDataCol = readh.pDCols[0]->cols + j; - void *value = tdGetRowDataOfCol(memRowDataBody(row), (int8_t)pCol->type, TD_DATA_ROW_HEAD_SIZE + pCol->offset); - if (isNull(value, pCol->type)) { - continue; - } - - int16_t idx = tsdbGetLastColumnsIndexByColId(pTable, pCol->colId); - if (idx == -1) { - tsdbError("tsdbRestoreLastColumns restore vgId:%d,table:%s cache column %d fail", REPO_ID(pRepo), pTable->name->data, pCol->colId); - continue; - } - // save not-null column - uint16_t bytes = IS_VAR_DATA_TYPE(pCol->type) ? varDataTLen(pColData) : pCol->bytes; - SDataCol *pLastCol = &(pTable->lastCols[idx]); - pLastCol->pData = taosMemoryMalloc(bytes); - pLastCol->bytes = bytes; - pLastCol->colId = pCol->colId; - memcpy(pLastCol->pData, value, bytes); - - // save row ts(in column 0) - pDataCol = pReadh->pDCols[0]->cols + 0; - pCol = schemaColAt(pSchema, 0); - tdAppendColVal(memRowDataBody(row), tdGetColDataOfRow(pDataCol, rowId), pCol->type, pCol->offset); - pLastCol->ts = TD_ROW_KEY(row); - - pTable->restoreColumnNum += 1; - - tsdbDebug("tsdbRestoreLastColumns restore vgId:%d,table:%s cache column %d, %" PRId64, REPO_ID(pRepo), pTable->name->data, pLastCol->colId, pLastCol->ts); - break; - } - } - } - -out: - taosTZfree(row); - taosMemoryFreeClear(pBlockStatis); - - if (err == 0 && numColumns <= pTable->restoreColumnNum) { - pTable->hasRestoreLastColumn = true; - } - - return err; -} - -static int tsdbRestoreLastRow(STsdbRepo *pRepo, STable *pTable, SReadH* pReadh, SBlockIdx *pIdx) { - ASSERT(pTable->lastRow == NULL); - if (tsdbLoadBlockInfo(pReadh, NULL) < 0) { - return -1; - } - - SBlock* pBlock = pReadh->pBlkInfo->blocks + pIdx->numOfBlocks - 1; - - if (tsdbLoadBlockData(pReadh, pBlock, NULL) < 0) { - return -1; - } - - // Get the data in row - - STSchema *pSchema = tsdbGetTableSchema(pTable); - pTable->lastRow = taosTMalloc(memRowMaxBytesFromSchema(pSchema)); - if (pTable->lastRow == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; - } - memRowSetType(pTable->lastRow, SMEM_ROW_DATA); - tdInitDataRow(memRowDataBody(pTable->lastRow), pSchema); - for (int icol = 0; icol < schemaNCols(pSchema); icol++) { - STColumn *pCol = schemaColAt(pSchema, icol); - SDataCol *pDataCol = pReadh->pDCols[0]->cols + icol; - tdAppendColVal(memRowDataBody(pTable->lastRow), tdGetColDataOfRow(pDataCol, pBlock->numOfRows - 1), pCol->type, - pCol->offset); - } - - return 0; -} - -int tsdbRestoreInfo(STsdbRepo *pRepo) { - SFSIter fsiter; - SReadH readh; - SDFileSet *pSet; - STsdbMeta *pMeta = pRepo->tsdbMeta; - STsdbCfg * pCfg = REPO_CFG(pRepo); - - if (tsdbInitReadH(&readh, pRepo) < 0) { - return -1; - } - - tsdbFSIterInit(&fsiter, REPO_FS(pRepo), TSDB_FS_ITER_BACKWARD); - - if (CACHE_LAST_NULL_COLUMN(pCfg)) { - for (int i = 1; i < pMeta->maxTables; i++) { - STable *pTable = pMeta->tables[i]; - if (pTable == NULL) continue; - pTable->restoreColumnNum = 0; - pTable->hasRestoreLastColumn = false; - } - } - - while ((pSet = tsdbFSIterNext(&fsiter)) != NULL) { - if (tsdbSetAndOpenReadFSet(&readh, pSet) < 0) { - tsdbDestroyReadH(&readh); - return -1; - } - - if (tsdbLoadBlockIdx(&readh) < 0) { - tsdbDestroyReadH(&readh); - return -1; - } - - for (int i = 1; i < pMeta->maxTables; i++) { - STable *pTable = pMeta->tables[i]; - if (pTable == NULL) continue; - - //tsdbInfo("tsdbRestoreInfo restore vgId:%d,table:%s", REPO_ID(pRepo), pTable->name->data); - - if (tsdbSetReadTable(&readh, pTable) < 0) { - tsdbDestroyReadH(&readh); - return -1; - } - - TSKEY lastKey = tsdbGetTableLastKeyImpl(pTable); - SBlockIdx *pIdx = readh.pBlkIdx; - if (pIdx && lastKey < pIdx->maxKey) { - pTable->lastKey = pIdx->maxKey; - - if (CACHE_LAST_ROW(pCfg) && tsdbRestoreLastRow(pRepo, pTable, &readh, pIdx) != 0) { - tsdbDestroyReadH(&readh); - return -1; - } - } - - // restore NULL columns - if (pIdx && CACHE_LAST_NULL_COLUMN(pCfg) && !pTable->hasRestoreLastColumn) { - if (tsdbRestoreLastColumns(pRepo, pTable, &readh) != 0) { - tsdbDestroyReadH(&readh); - return -1; - } - } - } - } - - tsdbDestroyReadH(&readh); - - if (CACHE_LAST_NULL_COLUMN(pCfg)) { - atomic_store_8(&pRepo->hasCachedLastColumn, 1); - } - - return 0; -} - -int tsdbCacheLastData(STsdbRepo *pRepo, STsdbCfg* oldCfg) { - bool cacheLastRow = false, cacheLastCol = false; - SFSIter fsiter; - SReadH readh; - SDFileSet *pSet; - STsdbMeta *pMeta = pRepo->tsdbMeta; - int tableNum = 0; - int maxTableIdx = 0; - int cacheLastRowTableNum = 0; - int cacheLastColTableNum = 0; - - bool need_free_last_row = CACHE_LAST_ROW(oldCfg) && !CACHE_LAST_ROW(&(pRepo->config)); - bool need_free_last_col = CACHE_LAST_NULL_COLUMN(oldCfg) && !CACHE_LAST_NULL_COLUMN(&(pRepo->config)); - - if (CACHE_LAST_ROW(&(pRepo->config)) || CACHE_LAST_NULL_COLUMN(&(pRepo->config))) { - tsdbInfo("tsdbCacheLastData cache last data since cacheLast option changed"); - cacheLastRow = !CACHE_LAST_ROW(oldCfg) && CACHE_LAST_ROW(&(pRepo->config)); - cacheLastCol = !CACHE_LAST_NULL_COLUMN(oldCfg) && CACHE_LAST_NULL_COLUMN(&(pRepo->config)); - } - - // calc max table idx and table num - for (int i = 1; i < pMeta->maxTables; i++) { - STable *pTable = pMeta->tables[i]; - if (pTable == NULL) continue; - tableNum += 1; - maxTableIdx = i; - if (cacheLastCol) { - pTable->restoreColumnNum = 0; - pTable->hasRestoreLastColumn = false; - } - } - - // if close last option,need to free data - if (need_free_last_row || need_free_last_col) { - if (need_free_last_col) { - atomic_store_8(&pRepo->hasCachedLastColumn, 0); - } - tsdbInfo("free cache last data since cacheLast option changed"); - for (int i = 1; i <= maxTableIdx; i++) { - STable *pTable = pMeta->tables[i]; - if (pTable == NULL) continue; - if (need_free_last_row) { - taosTZfree(pTable->lastRow); - pTable->lastRow = NULL; - } - if (need_free_last_col) { - tsdbFreeLastColumns(pTable); - pTable->hasRestoreLastColumn = false; - } - } - } - - if (!cacheLastRow && !cacheLastCol) { - return 0; - } - - cacheLastRowTableNum = cacheLastRow ? tableNum : 0; - cacheLastColTableNum = cacheLastCol ? tableNum : 0; - - if (tsdbInitReadH(&readh, pRepo) < 0) { - return -1; - } - - tsdbFSIterInit(&fsiter, REPO_FS(pRepo), TSDB_FS_ITER_BACKWARD); - - while ((pSet = tsdbFSIterNext(&fsiter)) != NULL && (cacheLastRowTableNum > 0 || cacheLastColTableNum > 0)) { - if (tsdbSetAndOpenReadFSet(&readh, pSet) < 0) { - tsdbDestroyReadH(&readh); - return -1; - } - - if (tsdbLoadBlockIdx(&readh) < 0) { - tsdbDestroyReadH(&readh); - return -1; - } - - for (int i = 1; i <= maxTableIdx; i++) { - STable *pTable = pMeta->tables[i]; - if (pTable == NULL) continue; - - //tsdbInfo("tsdbRestoreInfo restore vgId:%d,table:%s", REPO_ID(pRepo), pTable->name->data); - - if (tsdbSetReadTable(&readh, pTable) < 0) { - tsdbDestroyReadH(&readh); - return -1; - } - - SBlockIdx *pIdx = readh.pBlkIdx; - - if (pIdx && cacheLastRowTableNum > 0 && pTable->lastRow == NULL) { - pTable->lastKey = pIdx->maxKey; - - if (tsdbRestoreLastRow(pRepo, pTable, &readh, pIdx) != 0) { - tsdbDestroyReadH(&readh); - return -1; - } - cacheLastRowTableNum -= 1; - } - - // restore NULL columns - if (pIdx && cacheLastColTableNum > 0 && !pTable->hasRestoreLastColumn) { - if (tsdbRestoreLastColumns(pRepo, pTable, &readh) != 0) { - tsdbDestroyReadH(&readh); - return -1; - } - if (pTable->hasRestoreLastColumn) { - cacheLastColTableNum -= 1; - } - } - } - } - - tsdbDestroyReadH(&readh); - - if (cacheLastCol) { - atomic_store_8(&pRepo->hasCachedLastColumn, 1); - } - - return 0; -} -#endif \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index 323fc7970b..48e672d9bc 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -285,21 +285,6 @@ static STbData *tsdbNewTbData(tb_uid_t uid) { pTbData->keyMax = TSKEY_MIN; pTbData->nrows = 0; - // uint8_t skipListCreateFlags; - // if (pCfg->update == TD_ROW_DISCARD_UPDATE) - // skipListCreateFlags = SL_DISCARD_DUP_KEY; - // else - // skipListCreateFlags = SL_UPDATE_DUP_KEY; - - // pTableData->pData = - // tSkipListCreate(TSDB_DATA_SKIPLIST_LEVEL, TSDB_DATA_TYPE_TIMESTAMP, TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP], - // tkeyComparFn, skipListCreateFlags, tsdbGetTsTupleKey); - // if (pTableData->pData == NULL) { - // terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - // taosMemoryFree(pTableData); - // return NULL; - // } - pTbData->pData = tSkipListCreate(5, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), tkeyComparFn, SL_DISCARD_DUP_KEY, tsdbGetTsTupleKey); if (pTbData->pData == NULL) { @@ -350,609 +335,4 @@ static int tsdbAppendTableRowToCols(STable *pTable, SDataCols *pCols, STSchema * } return 0; -} - -/* ------------------------ REFACTORING ------------------------ */ -#if 0 -int tsdbInsertDataToMemTable(STsdbMemTable *pMemTable, SSubmitReq *pMsg) { - SMemAllocator *pMA = pMemTable->pMA; - STbData * pTbData = (STbData *)TD_MA_MALLOC(pMA, sizeof(*pTbData)); - if (pTbData == NULL) { - // TODO - } - - TD_SLIST_PUSH(&(pMemTable->list), pTbData); - - return 0; -} - -#include "tdataformat.h" -#include "tfunctional.h" -#include "tsdbRowMergeBuf.h" -#include "tsdbint.h" -#include "tskiplist.h" - -#define TSDB_DATA_SKIPLIST_LEVEL 5 -#define TSDB_MAX_INSERT_BATCH 512 - -typedef struct { - int32_t totalLen; - int32_t len; - STSRow* row; -} SSubmitBlkIter; - -typedef struct { - int32_t totalLen; - int32_t len; - void * pMsg; -} SSubmitMsgIter; - -static SMemTable * tsdbNewMemTable(STsdbRepo *pRepo); -static void tsdbFreeMemTable(SMemTable *pMemTable); -static STableData* tsdbNewTableData(STsdbCfg *pCfg, STable *pTable); -static void tsdbFreeTableData(STableData *pTableData); -static int tsdbAdjustMemMaxTables(SMemTable *pMemTable, int maxTables); -static int tsdbAppendTableRowToCols(STable *pTable, SDataCols *pCols, STSchema **ppSchema, STSRow* row); -static int tsdbInitSubmitBlkIter(SSubmitBlk *pBlock, SSubmitBlkIter *pIter); -static STSRow* tsdbGetSubmitBlkNext(SSubmitBlkIter *pIter); -static int tsdbInsertDataToTable(STsdbRepo *pRepo, SSubmitBlk *pBlock, int32_t *affectedrows); -static int tsdbInitSubmitMsgIter(SSubmitReq *pMsg, SSubmitMsgIter *pIter); -static int tsdbGetSubmitMsgNext(SSubmitMsgIter *pIter, SSubmitBlk **pPBlock); -static int tsdbCheckTableSchema(STsdbRepo *pRepo, SSubmitBlk *pBlock, STable *pTable); -static int tsdbUpdateTableLatestInfo(STsdbRepo *pRepo, STable *pTable, STSRow* row); - -static FORCE_INLINE int tsdbCheckRowRange(STsdbRepo *pRepo, STable *pTable, STSRow* row, TSKEY minKey, TSKEY maxKey, - TSKEY now); - - -// ---------------- INTERNAL FUNCTIONS ---------------- -int tsdbRefMemTable(STsdbRepo *pRepo, SMemTable *pMemTable) { - if (pMemTable == NULL) return 0; - int ref = T_REF_INC(pMemTable); - tsdbDebug("vgId:%d ref memtable %p ref %d", REPO_ID(pRepo), pMemTable, ref); - return 0; -} - -// Need to lock the repository -int tsdbUnRefMemTable(STsdbRepo *pRepo, SMemTable *pMemTable) { - if (pMemTable == NULL) return 0; - - int ref = T_REF_DEC(pMemTable); - tsdbDebug("vgId:%d unref memtable %p ref %d", REPO_ID(pRepo), pMemTable, ref); - if (ref == 0) { - STsdbBufPool *pBufPool = pRepo->pPool; - - SListNode *pNode = NULL; - bool addNew = false; - if (tsdbLockRepo(pRepo) < 0) return -1; - while ((pNode = tdListPopHead(pMemTable->bufBlockList)) != NULL) { - if (pBufPool->nRecycleBlocks > 0) { - tsdbRecycleBufferBlock(pBufPool, pNode, false); - pBufPool->nRecycleBlocks -= 1; - } else { - if(pBufPool->nElasticBlocks > 0 && listNEles(pBufPool->bufBlockList) > 2) { - tsdbRecycleBufferBlock(pBufPool, pNode, true); - } else { - tdListAppendNode(pBufPool->bufBlockList, pNode); - addNew = true; - } - } - } - if (addNew) { - int code = taosThreadCondSignal(&pBufPool->poolNotEmpty); - if (code != 0) { - if (tsdbUnlockRepo(pRepo) < 0) return -1; - tsdbError("vgId:%d failed to signal pool not empty since %s", REPO_ID(pRepo), strerror(code)); - terrno = TAOS_SYSTEM_ERROR(code); - return -1; - } - } - - if (tsdbUnlockRepo(pRepo) < 0) return -1; - - for (int i = 0; i < pMemTable->maxTables; i++) { - if (pMemTable->tData[i] != NULL) { - tsdbFreeTableData(pMemTable->tData[i]); - } - } - - tdListDiscard(pMemTable->actList); - tdListDiscard(pMemTable->bufBlockList); - tsdbFreeMemTable(pMemTable); - } - return 0; -} - -int tsdbTakeMemSnapshot(STsdbRepo *pRepo, SMemSnapshot *pSnapshot, SArray *pATable) { - memset(pSnapshot, 0, sizeof(*pSnapshot)); - - if (tsdbLockRepo(pRepo) < 0) return -1; - - pSnapshot->omem = pRepo->mem; - pSnapshot->imem = pRepo->imem; - tsdbRefMemTable(pRepo, pRepo->mem); - tsdbRefMemTable(pRepo, pRepo->imem); - - if (tsdbUnlockRepo(pRepo) < 0) return -1; - - if (pSnapshot->omem) { - taosRLockLatch(&(pSnapshot->omem->latch)); - - pSnapshot->mem = &(pSnapshot->mtable); - - pSnapshot->mem->tData = (STableData **)taosMemoryCalloc(pSnapshot->omem->maxTables, sizeof(STableData *)); - if (pSnapshot->mem->tData == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - taosRUnLockLatch(&(pSnapshot->omem->latch)); - tsdbUnRefMemTable(pRepo, pSnapshot->omem); - tsdbUnRefMemTable(pRepo, pSnapshot->imem); - pSnapshot->mem = NULL; - pSnapshot->imem = NULL; - pSnapshot->omem = NULL; - return -1; - } - - pSnapshot->mem->keyFirst = pSnapshot->omem->keyFirst; - pSnapshot->mem->keyLast = pSnapshot->omem->keyLast; - pSnapshot->mem->numOfRows = pSnapshot->omem->numOfRows; - pSnapshot->mem->maxTables = pSnapshot->omem->maxTables; - - for (size_t i = 0; i < taosArrayGetSize(pATable); i++) { - STable * pTable = *(STable **)taosArrayGet(pATable, i); - int32_t tid = TABLE_TID(pTable); - STableData *pTableData = (tid < pSnapshot->omem->maxTables) ? pSnapshot->omem->tData[tid] : NULL; - - if ((pTableData == NULL) || (TABLE_UID(pTable) != pTableData->uid)) continue; - - pSnapshot->mem->tData[tid] = pTableData; - T_REF_INC(pTableData); - } - - taosRUnLockLatch(&(pSnapshot->omem->latch)); - } - - tsdbDebug("vgId:%d take memory snapshot, pMem %p pIMem %p", REPO_ID(pRepo), pSnapshot->omem, pSnapshot->imem); - return 0; -} - -void tsdbUnTakeMemSnapShot(STsdbRepo *pRepo, SMemSnapshot *pSnapshot) { - tsdbDebug("vgId:%d untake memory snapshot, pMem %p pIMem %p", REPO_ID(pRepo), pSnapshot->omem, pSnapshot->imem); - - if (pSnapshot->mem) { - ASSERT(pSnapshot->omem != NULL); - - for (size_t i = 0; i < pSnapshot->mem->maxTables; i++) { - STableData *pTableData = pSnapshot->mem->tData[i]; - if (pTableData) { - tsdbFreeTableData(pTableData); - } - } - taosMemoryFreeClear(pSnapshot->mem->tData); - - tsdbUnRefMemTable(pRepo, pSnapshot->omem); - } - - tsdbUnRefMemTable(pRepo, pSnapshot->imem); - - pSnapshot->mem = NULL; - pSnapshot->imem = NULL; - pSnapshot->omem = NULL; -} - -int tsdbSyncCommitConfig(STsdbRepo* pRepo) { - ASSERT(pRepo->config_changed == true); - tsem_wait(&(pRepo->readyToCommit)); - - if (pRepo->code != TSDB_CODE_SUCCESS) { - tsdbWarn("vgId:%d try to commit config when TSDB not in good state: %s", REPO_ID(pRepo), tstrerror(terrno)); - } - - if (tsdbLockRepo(pRepo) < 0) return -1; - tsdbScheduleCommit(pRepo, COMMIT_CONFIG_REQ); - if (tsdbUnlockRepo(pRepo) < 0) return -1; - - tsem_wait(&(pRepo->readyToCommit)); - tsem_post(&(pRepo->readyToCommit)); - - if (pRepo->code != TSDB_CODE_SUCCESS) { - terrno = pRepo->code; - return -1; - } - - terrno = TSDB_CODE_SUCCESS; - return 0; -} - -/** - * This is an important function to load data or try to load data from memory skiplist iterator. - * - * This function load memory data until: - * 1. iterator ends - * 2. data key exceeds maxKey - * 3. rowsIncreased = rowsInserted - rowsDeleteSucceed >= maxRowsToRead - * 4. operations in pCols not exceeds its max capacity if pCols is given - * - * The function tries to procceed AS MUCH AS POSSIBLE. - */ -int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey, int maxRowsToRead, SDataCols *pCols, - TKEY *filterKeys, int nFilterKeys, bool keepDup, SMergeInfo *pMergeInfo) { - ASSERT(maxRowsToRead > 0 && nFilterKeys >= 0); - if (pIter == NULL) return 0; - STSchema * pSchema = NULL; - TSKEY rowKey = 0; - TSKEY fKey = 0; - bool isRowDel = false; - int filterIter = 0; - STSRow* row = NULL; - SMergeInfo mInfo; - - if (pMergeInfo == NULL) pMergeInfo = &mInfo; - - memset(pMergeInfo, 0, sizeof(*pMergeInfo)); - pMergeInfo->keyFirst = INT64_MAX; - pMergeInfo->keyLast = INT64_MIN; - if (pCols) tdResetDataCols(pCols); - - row = tsdbNextIterRow(pIter); - if (row == NULL || TD_ROW_KEY(row) > maxKey) { - rowKey = INT64_MAX; - isRowDel = false; - } else { - rowKey = TD_ROW_KEY(row); - isRowDel = memRowDeleted(row); - } - - if (filterIter >= nFilterKeys) { - fKey = INT64_MAX; - } else { - fKey = tdGetKey(filterKeys[filterIter]); - } - - while (true) { - if (fKey == INT64_MAX && rowKey == INT64_MAX) break; - - if (fKey < rowKey) { - pMergeInfo->keyFirst = TMIN(pMergeInfo->keyFirst, fKey); - pMergeInfo->keyLast = TMAX(pMergeInfo->keyLast, fKey); - - filterIter++; - if (filterIter >= nFilterKeys) { - fKey = INT64_MAX; - } else { - fKey = tdGetKey(filterKeys[filterIter]); - } - } else if (fKey > rowKey) { - if (isRowDel) { - pMergeInfo->rowsDeleteFailed++; - } else { - if (pMergeInfo->rowsInserted - pMergeInfo->rowsDeleteSucceed >= maxRowsToRead) break; - if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break; - pMergeInfo->rowsInserted++; - pMergeInfo->nOperations++; - pMergeInfo->keyFirst = TMIN(pMergeInfo->keyFirst, rowKey); - pMergeInfo->keyLast = TMAX(pMergeInfo->keyLast, rowKey); - tsdbAppendTableRowToCols(pTable, pCols, &pSchema, row); - } - - tSkipListIterNext(pIter); - row = tsdbNextIterRow(pIter); - if (row == NULL || TD_ROW_KEY(row) > maxKey) { - rowKey = INT64_MAX; - isRowDel = false; - } else { - rowKey = TD_ROW_KEY(row); - isRowDel = memRowDeleted(row); - } - } else { - if (isRowDel) { - ASSERT(!keepDup); - if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break; - pMergeInfo->rowsDeleteSucceed++; - pMergeInfo->nOperations++; - tsdbAppendTableRowToCols(pTable, pCols, &pSchema, row); - } else { - if (keepDup) { - if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break; - pMergeInfo->rowsUpdated++; - pMergeInfo->nOperations++; - pMergeInfo->keyFirst = TMIN(pMergeInfo->keyFirst, rowKey); - pMergeInfo->keyLast = TMAX(pMergeInfo->keyLast, rowKey); - tsdbAppendTableRowToCols(pTable, pCols, &pSchema, row); - } else { - pMergeInfo->keyFirst = TMIN(pMergeInfo->keyFirst, fKey); - pMergeInfo->keyLast = TMAX(pMergeInfo->keyLast, fKey); - } - } - - tSkipListIterNext(pIter); - row = tsdbNextIterRow(pIter); - if (row == NULL || TD_ROW_KEY(row) > maxKey) { - rowKey = INT64_MAX; - isRowDel = false; - } else { - rowKey = TD_ROW_KEY(row); - isRowDel = memRowDeleted(row); - } - - filterIter++; - if (filterIter >= nFilterKeys) { - fKey = INT64_MAX; - } else { - fKey = tdGetKey(filterKeys[filterIter]); - } - } - } - - return 0; -} - -// ---------------- LOCAL FUNCTIONS ---------------- - -static FORCE_INLINE int tsdbCheckRowRange(STsdbRepo *pRepo, STable *pTable, STSRow* row, TSKEY minKey, TSKEY maxKey, - TSKEY now) { - TSKEY rowKey = TD_ROW_KEY(row); - if (rowKey < minKey || rowKey > maxKey) { - tsdbError("vgId:%d table %s tid %d uid %" PRIu64 " timestamp is out of range! now %" PRId64 " minKey %" PRId64 - " maxKey %" PRId64 " row key %" PRId64, - REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), TABLE_UID(pTable), now, minKey, maxKey, - rowKey); - terrno = TSDB_CODE_TDB_TIMESTAMP_OUT_OF_RANGE; - return -1; - } - - return 0; -} - - -//row1 has higher priority -static STSRow* tsdbInsertDupKeyMerge(STSRow* row1, STSRow* row2, STsdbRepo* pRepo, - STSchema **ppSchema1, STSchema **ppSchema2, - STable* pTable, int32_t* pPoints, STSRow** pLastRow) { - - //for compatiblity, duplicate key inserted when update=0 should be also calculated as affected rows! - if(row1 == NULL && row2 == NULL && pRepo->config.update == TD_ROW_DISCARD_UPDATE) { - (*pPoints)++; - return NULL; - } - - tsdbTrace("vgId:%d a row is %s table %s tid %d uid %" PRIu64 " key %" PRIu64, REPO_ID(pRepo), - "updated in", TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), TABLE_UID(pTable), - TD_ROW_KEY(row1)); - - if(row2 == NULL || pRepo->config.update != TD_ROW_PARTIAL_UPDATE) { - void* pMem = tsdbAllocBytes(pRepo, TD_ROW_LEN(row1)); - if(pMem == NULL) return NULL; - memRowCpy(pMem, row1); - (*pPoints)++; - *pLastRow = pMem; - return pMem; - } - - STSchema *pSchema1 = *ppSchema1; - STSchema *pSchema2 = *ppSchema2; - SMergeBuf * pBuf = &pRepo->mergeBuf; - int dv1 = memRowVersion(row1); - int dv2 = memRowVersion(row2); - if(pSchema1 == NULL || schemaVersion(pSchema1) != dv1) { - if(pSchema2 != NULL && schemaVersion(pSchema2) == dv1) { - *ppSchema1 = pSchema2; - } else { - *ppSchema1 = tsdbGetTableSchemaImpl(pTable, false, false, memRowVersion(row1), (int8_t)memRowType(row1)); - } - pSchema1 = *ppSchema1; - } - - if(pSchema2 == NULL || schemaVersion(pSchema2) != dv2) { - if(schemaVersion(pSchema1) == dv2) { - pSchema2 = pSchema1; - } else { - *ppSchema2 = tsdbGetTableSchemaImpl(pTable, false, false, memRowVersion(row2), (int8_t)memRowType(row2)); - pSchema2 = *ppSchema2; - } - } - - STSRow* tmp = tsdbMergeTwoRows(pBuf, row1, row2, pSchema1, pSchema2); - - void* pMem = tsdbAllocBytes(pRepo, TD_ROW_LEN(tmp)); - if(pMem == NULL) return NULL; - memRowCpy(pMem, tmp); - - (*pPoints)++; - *pLastRow = pMem; - return pMem; -} - -static void* tsdbInsertDupKeyMergePacked(void** args) { - return tsdbInsertDupKeyMerge(args[0], args[1], args[2], (STSchema**)&args[3], (STSchema**)&args[4], args[5], args[6], args[7]); -} - -static void tsdbSetupSkipListHookFns(SSkipList* pSkipList, STsdbRepo *pRepo, STable *pTable, int32_t* pPoints, STSRow** pLastRow) { - - if(pSkipList->insertHandleFn == NULL) { - tGenericSavedFunc *dupHandleSavedFunc = genericSavedFuncInit((GenericVaFunc)&tsdbInsertDupKeyMergePacked, 9); - dupHandleSavedFunc->args[2] = pRepo; - dupHandleSavedFunc->args[3] = NULL; - dupHandleSavedFunc->args[4] = NULL; - dupHandleSavedFunc->args[5] = pTable; - pSkipList->insertHandleFn = dupHandleSavedFunc; - } - pSkipList->insertHandleFn->args[6] = pPoints; - pSkipList->insertHandleFn->args[7] = pLastRow; -} - -static int tsdbCheckTableSchema(STsdbRepo *pRepo, SSubmitBlk *pBlock, STable *pTable) { - ASSERT(pTable != NULL); - - STSchema *pSchema = tsdbGetTableSchemaImpl(pTable, false, false, -1, -1); - int sversion = schemaVersion(pSchema); - - if (pBlock->sversion == sversion) { - return 0; - } else { - if (TABLE_TYPE(pTable) == TSDB_STREAM_TABLE) { // stream table is not allowed to change schema - terrno = TSDB_CODE_TDB_IVD_TB_SCHEMA_VERSION; - return -1; - } - } - - if (pBlock->sversion > sversion) { // may need to update table schema - if (pBlock->schemaLen > 0) { - tsdbDebug( - "vgId:%d table %s tid %d uid %" PRIu64 " schema version %d is out of data, client version %d, update...", - REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), TABLE_UID(pTable), sversion, pBlock->sversion); - ASSERT(pBlock->schemaLen % sizeof(STColumn) == 0); - int numOfCols = pBlock->schemaLen / sizeof(STColumn); - STColumn *pTCol = (STColumn *)pBlock->data; - - STSchemaBuilder schemaBuilder = {0}; - if (tdInitTSchemaBuilder(&schemaBuilder, pBlock->sversion) < 0) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - tsdbError("vgId:%d failed to update schema of table %s since %s", REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), - tstrerror(terrno)); - return -1; - } - - for (int i = 0; i < numOfCols; i++) { - if (tdAddColToSchema(&schemaBuilder, pTCol[i].type, htons(pTCol[i].colId), htons(pTCol[i].bytes)) < 0) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - tsdbError("vgId:%d failed to update schema of table %s since %s", REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), - tstrerror(terrno)); - tdDestroyTSchemaBuilder(&schemaBuilder); - return -1; - } - } - - STSchema *pNSchema = tdGetSchemaFromBuilder(&schemaBuilder); - if (pNSchema == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - tdDestroyTSchemaBuilder(&schemaBuilder); - return -1; - } - - tdDestroyTSchemaBuilder(&schemaBuilder); - tsdbUpdateTableSchema(pRepo, pTable, pNSchema, true); - } else { - tsdbDebug( - "vgId:%d table %s tid %d uid %" PRIu64 " schema version %d is out of data, client version %d, reconfigure...", - REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), TABLE_UID(pTable), sversion, pBlock->sversion); - terrno = TSDB_CODE_TDB_TABLE_RECONFIGURE; - return -1; - } - } else { - ASSERT(pBlock->sversion >= 0); - if (tsdbGetTableSchemaImpl(pTable, false, false, pBlock->sversion, -1) == NULL) { - tsdbError("vgId:%d invalid submit schema version %d to table %s tid %d from client", REPO_ID(pRepo), - pBlock->sversion, TABLE_CHAR_NAME(pTable), TABLE_TID(pTable)); - terrno = TSDB_CODE_TDB_IVD_TB_SCHEMA_VERSION; - return -1; - } - } - - return 0; -} - -static void updateTableLatestColumn(STsdbRepo *pRepo, STable *pTable, STSRow* row) { - tsdbDebug("vgId:%d updateTableLatestColumn, %s row version:%d", REPO_ID(pRepo), pTable->name->data, - memRowVersion(row)); - - STSchema* pSchema = tsdbGetTableLatestSchema(pTable); - if (tsdbUpdateLastColSchema(pTable, pSchema) < 0) { - return; - } - - pSchema = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row), (int8_t)memRowType(row)); - if (pSchema == NULL) { - return; - } - - SDataCol *pLatestCols = pTable->lastCols; - int32_t kvIdx = 0; - - for (int16_t j = 0; j < schemaNCols(pSchema); j++) { - STColumn *pTCol = schemaColAt(pSchema, j); - // ignore not exist colId - int16_t idx = tsdbGetLastColumnsIndexByColId(pTable, pTCol->colId); - if (idx == -1) { - continue; - } - - void *value = NULL; - - value = tdGetMemRowDataOfColEx(row, pTCol->colId, (int8_t)pTCol->type, - TD_DATA_ROW_HEAD_SIZE + pSchema->columns[j].offset, &kvIdx); - - if ((value == NULL) || isNull(value, pTCol->type)) { - continue; - } - // lock - TSDB_WLOCK_TABLE(pTable); - SDataCol *pDataCol = &(pLatestCols[idx]); - if (pDataCol->pData == NULL) { - pDataCol->pData = taosMemoryMalloc(pTCol->bytes); - pDataCol->bytes = pTCol->bytes; - } else if (pDataCol->bytes < pTCol->bytes) { - pDataCol->pData = taosMemoryRealloc(pDataCol->pData, pTCol->bytes); - pDataCol->bytes = pTCol->bytes; - } - // the actual value size - uint16_t bytes = IS_VAR_DATA_TYPE(pTCol->type) ? varDataTLen(value) : pTCol->bytes; - // the actual data size CANNOT larger than column size - assert(pTCol->bytes >= bytes); - memcpy(pDataCol->pData, value, bytes); - //tsdbInfo("updateTableLatestColumn vgId:%d cache column %d for %d,%s", REPO_ID(pRepo), j, pDataCol->bytes, (char*)pDataCol->pData); - pDataCol->ts = TD_ROW_KEY(row); - // unlock - TSDB_WUNLOCK_TABLE(pTable); - } -} - -static int tsdbUpdateTableLatestInfo(STsdbRepo *pRepo, STable *pTable, STSRow* row) { - STsdbCfg *pCfg = &pRepo->config; - - // if cacheLastRow config has been reset, free the lastRow - if (!pCfg->cacheLastRow && pTable->lastRow != NULL) { - STSRow* cachedLastRow = pTable->lastRow; - TSDB_WLOCK_TABLE(pTable); - pTable->lastRow = NULL; - TSDB_WUNLOCK_TABLE(pTable); - taosTZfree(cachedLastRow); - } - - if (tsdbGetTableLastKeyImpl(pTable) <= TD_ROW_KEY(row)) { - if (CACHE_LAST_ROW(pCfg) || pTable->lastRow != NULL) { - STSRow* nrow = pTable->lastRow; - if (taosTSizeof(nrow) < TD_ROW_LEN(row)) { - STSRow* orow = nrow; - nrow = taosTMalloc(TD_ROW_LEN(row)); - if (nrow == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; - } - - memRowCpy(nrow, row); - TSDB_WLOCK_TABLE(pTable); - pTable->lastKey = TD_ROW_KEY(row); - pTable->lastRow = nrow; - TSDB_WUNLOCK_TABLE(pTable); - taosTZfree(orow); - } else { - TSDB_WLOCK_TABLE(pTable); - pTable->lastKey = TD_ROW_KEY(row); - memRowCpy(nrow, row); - TSDB_WUNLOCK_TABLE(pTable); - } - } else { - pTable->lastKey = TD_ROW_KEY(row); - } - - if (CACHE_LAST_NULL_COLUMN(pCfg)) { - updateTableLatestColumn(pRepo, pTable, row); - } - } - - pTable->cacheLastConfigVersion = pRepo->cacheLastConfigVersion; - - return 0; -} - -#endif \ No newline at end of file +} \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbOpen.c b/source/dnode/vnode/src/tsdb/tsdbOpen.c new file mode 100644 index 0000000000..e5b2518415 --- /dev/null +++ b/source/dnode/vnode/src/tsdb/tsdbOpen.c @@ -0,0 +1,89 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tsdb.h" + +int tsdbOpen(SVnode *pVnode, STsdb **ppTsdb) { + STsdb *pTsdb = NULL; + int slen = 0; + + *ppTsdb = NULL; + slen = strlen(tfsGetPrimaryPath(pVnode->pTfs)) + strlen(pVnode->path) + strlen(VNODE_TSDB_DIR) + 3; + + // create handle + pTsdb = (STsdb *)taosMemoryCalloc(1, sizeof(*pTsdb) + slen); + if (pTsdb == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + pTsdb->path = (char *)&pTsdb[1]; + sprintf(pTsdb->path, "%s%s%s%s%s", tfsGetPrimaryPath(pVnode->pTfs), TD_DIRSEP, pVnode->path, TD_DIRSEP, + VNODE_TSDB_DIR); + pTsdb->pVnode = pVnode; + pTsdb->repoLocked = false; + tdbMutexInit(&pTsdb->mutex, NULL); + pTsdb->config = pVnode->config.tsdbCfg; + pTsdb->fs = tsdbNewFS(&pTsdb->config); + + // create dir (TODO: use tfsMkdir) + taosMkDir(pTsdb->path); + + // open tsdb + if (tsdbOpenFS(pTsdb) < 0) { + goto _err; + } + + tsdbDebug("vgId: %d tsdb is opened", TD_VID(pVnode)); + + *ppTsdb = pTsdb; + return 0; + +_err: + taosMemoryFree(pTsdb); + return -1; +} + +int tsdbClose(STsdb *pTsdb) { + if (pTsdb) { + tsdbCloseFS(pTsdb); + tsdbFreeFS(pTsdb->fs); + taosMemoryFree(pTsdb); + } + return 0; +} + +int tsdbLockRepo(STsdb *pTsdb) { + int code = taosThreadMutexLock(&pTsdb->mutex); + if (code != 0) { + tsdbError("vgId:%d failed to lock tsdb since %s", REPO_ID(pTsdb), strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(code); + return -1; + } + pTsdb->repoLocked = true; + return 0; +} + +int tsdbUnlockRepo(STsdb *pTsdb) { + ASSERT(IS_REPO_LOCKED(pTsdb)); + pTsdb->repoLocked = false; + int code = taosThreadMutexUnlock(&pTsdb->mutex); + if (code != 0) { + tsdbError("vgId:%d failed to unlock tsdb since %s", REPO_ID(pTsdb), strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(code); + return -1; + } + return 0; +} \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 5022f5249d..6243eb4891 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -454,7 +454,7 @@ void tsdbResetReadHandle(tsdbReaderT queryHandle, SQueryTableDataCond* pCond) { if (emptyQueryTimewindow(pTsdbReadHandle)) { if (pCond->order != pTsdbReadHandle->order) { pTsdbReadHandle->order = pCond->order; - TSWAP(pTsdbReadHandle->window.skey, pTsdbReadHandle->window.ekey, int64_t); + TSWAP(pTsdbReadHandle->window.skey, pTsdbReadHandle->window.ekey); } return; @@ -924,7 +924,7 @@ static bool hasMoreDataInCache(STsdbReadHandle* pHandle) { pHandle->cur.mixBlock = true; if (!ASCENDING_TRAVERSE(pHandle->order)) { - TSWAP(win->skey, win->ekey, TSKEY); + TSWAP(win->skey, win->ekey); } return true; @@ -1203,7 +1203,7 @@ static int32_t handleDataMergeIfNeeded(STsdbReadHandle* pTsdbReadHandle, SBlock* // update the last key value pCheckInfo->lastKey = cur->win.ekey + step; if (!ASCENDING_TRAVERSE(pTsdbReadHandle->order)) { - TSWAP(cur->win.skey, cur->win.ekey, TSKEY); + TSWAP(cur->win.skey, cur->win.ekey); } cur->mixBlock = true; @@ -1699,7 +1699,7 @@ static void copyAllRemainRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, STa int32_t end = endPos; if (!ASCENDING_TRAVERSE(pTsdbReadHandle->order)) { - TSWAP(start, end, int32_t); + TSWAP(start, end); } assert(pTsdbReadHandle->outputCapacity >= (end - start + 1)); @@ -1930,7 +1930,7 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf ((pos < endPos || cur->lastKey < pTsdbReadHandle->window.ekey) && !ASCENDING_TRAVERSE(pTsdbReadHandle->order))); if (!ASCENDING_TRAVERSE(pTsdbReadHandle->order)) { - TSWAP(cur->win.skey, cur->win.ekey, TSKEY); + TSWAP(cur->win.skey, cur->win.ekey); } moveDataToFront(pTsdbReadHandle, numOfRows, numOfCols); diff --git a/source/dnode/vnode/src/vnd/vnodeCfg.c b/source/dnode/vnode/src/vnd/vnodeCfg.c index df8cf8d503..0e55113dc3 100644 --- a/source/dnode/vnode/src/vnd/vnodeCfg.c +++ b/source/dnode/vnode/src/vnd/vnodeCfg.c @@ -23,8 +23,6 @@ const SVnodeCfg vnodeCfgDefault = { .szCache = 256, .szBuf = 96 * 1024 * 1024, .isHeap = false, - .ttl = 0, - .keep = 0, .streamMode = 0, .isWeak = 0, .tsdbCfg = {.precision = TSDB_TIME_PRECISION_MILLI, @@ -58,8 +56,6 @@ int vnodeEncodeConfig(const void *pObj, SJson *pJson) { if (tjsonAddIntegerToObject(pJson, "szCache", pCfg->szCache) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "szBuf", pCfg->szBuf) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "isHeap", pCfg->isHeap) < 0) return -1; - if (tjsonAddIntegerToObject(pJson, "ttl", pCfg->ttl) < 0) return -1; - if (tjsonAddIntegerToObject(pJson, "keep", pCfg->keep) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "streamMode", pCfg->streamMode) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "isWeak", pCfg->isWeak) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "precision", pCfg->tsdbCfg.precision) < 0) return -1; @@ -72,7 +68,6 @@ int vnodeEncodeConfig(const void *pObj, SJson *pJson) { if (tjsonAddIntegerToObject(pJson, "keep0", pCfg->tsdbCfg.keep0) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "keep1", pCfg->tsdbCfg.keep1) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "keep2", pCfg->tsdbCfg.keep2) < 0) return -1; - if (tjsonAddIntegerToObject(pJson, "lruCacheSize", pCfg->tsdbCfg.lruCacheSize) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "wal.vgId", pCfg->walCfg.vgId) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "wal.fsyncPeriod", pCfg->walCfg.fsyncPeriod) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "wal.retentionPeriod", pCfg->walCfg.retentionPeriod) < 0) return -1; @@ -109,8 +104,6 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) { if (tjsonGetNumberValue(pJson, "szCache", pCfg->szCache) < 0) return -1; if (tjsonGetNumberValue(pJson, "szBuf", pCfg->szBuf) < 0) return -1; if (tjsonGetNumberValue(pJson, "isHeap", pCfg->isHeap) < 0) return -1; - if (tjsonGetNumberValue(pJson, "ttl", pCfg->ttl) < 0) return -1; - if (tjsonGetNumberValue(pJson, "keep", pCfg->keep) < 0) return -1; if (tjsonGetNumberValue(pJson, "streamMode", pCfg->streamMode) < 0) return -1; if (tjsonGetNumberValue(pJson, "isWeak", pCfg->isWeak) < 0) return -1; if (tjsonGetNumberValue(pJson, "precision", pCfg->tsdbCfg.precision) < 0) return -1; @@ -123,7 +116,6 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) { if (tjsonGetNumberValue(pJson, "keep0", pCfg->tsdbCfg.keep0) < 0) return -1; if (tjsonGetNumberValue(pJson, "keep1", pCfg->tsdbCfg.keep1) < 0) return -1; if (tjsonGetNumberValue(pJson, "keep2", pCfg->tsdbCfg.keep2) < 0) return -1; - if (tjsonGetNumberValue(pJson, "lruCacheSize", pCfg->tsdbCfg.lruCacheSize) < 0) return -1; if (tjsonGetNumberValue(pJson, "wal.vgId", pCfg->walCfg.vgId) < 0) return -1; if (tjsonGetNumberValue(pJson, "wal.fsyncPeriod", pCfg->walCfg.fsyncPeriod) < 0) return -1; if (tjsonGetNumberValue(pJson, "wal.retentionPeriod", pCfg->walCfg.retentionPeriod) < 0) return -1; diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index ef7b9c97b5..605185c9d9 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -138,3 +138,13 @@ int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad) { pLoad->numOfBatchInsertSuccessReqs = 4; return 0; } + +void vnodeGetInfo(SVnode *pVnode, const char **dbname, int32_t *vgId) { + if (dbname) { + *dbname = pVnode->config.dbname; + } + + if (vgId) { + *vgId = TD_VID(pVnode); + } +} \ No newline at end of file diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index b890b873a8..7f7023fccd 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -142,7 +142,7 @@ _err: int vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg) { vTrace("message in vnode query queue is processing"); - SReadHandle handle = {.reader = pVnode->pTsdb, .meta = pVnode->pMeta, .config = &pVnode->config}; + SReadHandle handle = {.reader = pVnode->pTsdb, .meta = pVnode->pMeta, .config = &pVnode->config, .vnode = pVnode}; switch (pMsg->msgType) { case TDMT_VND_QUERY: @@ -374,7 +374,8 @@ static int vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pReq, tsdbUidStoreFree(pStore); // prepare rsp - tEncodeSize(tEncodeSVCreateTbBatchRsp, &rsp, pRsp->contLen); + int32_t ret = 0; + tEncodeSize(tEncodeSVCreateTbBatchRsp, &rsp, pRsp->contLen, ret); pRsp->pCont = rpcMallocCont(pRsp->contLen); if (pRsp->pCont == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; diff --git a/source/libs/command/src/command.c b/source/libs/command/src/command.c index 4d4ac6c1e4..621ea7b7fc 100644 --- a/source/libs/command/src/command.c +++ b/source/libs/command/src/command.c @@ -27,9 +27,14 @@ static int32_t getSchemaBytes(const SSchema* pSchema) { } } +// todo : to convert data according to SSDatablock static void buildRspData(const STableMeta* pMeta, char* pData) { - int32_t* pColSizes = (int32_t*)pData; - pData += DESCRIBE_RESULT_COLS * sizeof(int32_t); + int32_t* payloadLen = (int32_t*) pData; + uint64_t* groupId = (uint64_t*)(pData + sizeof(int32_t)); + + int32_t* pColSizes = (int32_t*)(pData + sizeof(int32_t) + sizeof(uint64_t)); + pData = (char*) pColSizes + DESCRIBE_RESULT_COLS * sizeof(int32_t); + int32_t numOfRows = TABLE_TOTAL_COL_NUM(pMeta); // Field @@ -79,6 +84,9 @@ static void buildRspData(const STableMeta* pMeta, char* pData) { for (int32_t i = 0; i < DESCRIBE_RESULT_COLS; ++i) { pColSizes[i] = htonl(pColSizes[i]); } + + + *payloadLen = (int32_t)(pData - (char*)payloadLen); } static int32_t calcRspSize(const STableMeta* pMeta) { @@ -87,7 +95,8 @@ static int32_t calcRspSize(const STableMeta* pMeta) { (numOfRows * sizeof(int32_t) + numOfRows * DESCRIBE_RESULT_FIELD_LEN) + (numOfRows * sizeof(int32_t) + numOfRows * DESCRIBE_RESULT_TYPE_LEN) + (BitmapLen(numOfRows) + numOfRows * sizeof(int32_t)) + - (numOfRows * sizeof(int32_t) + numOfRows * DESCRIBE_RESULT_NOTE_LEN); + (numOfRows * sizeof(int32_t) + numOfRows * DESCRIBE_RESULT_NOTE_LEN) + + sizeof(int32_t) + sizeof(uint64_t); } static int32_t execDescribe(SNode* pStmt, SRetrieveTableRsp** pRsp) { diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 00ec92465f..7a6432a06c 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -254,6 +254,17 @@ enum { OP_EXEC_DONE = 0x9, }; +typedef struct SOperatorFpSet { + __optr_open_fn_t _openFn; // DO NOT invoke this function directly + __optr_fn_t getNextFn; + __optr_fn_t getStreamResFn; // execute the aggregate in the stream model, todo remove it + __optr_fn_t cleanupFn; // call this function to release the allocated resources ASAP + __optr_close_fn_t closeFn; + __optr_encode_fn_t encodeResultRow; + __optr_decode_fn_t decodeResultRow; + __optr_get_explain_fn_t getExplainFn; +} SOperatorFpSet; + typedef struct SOperatorInfo { uint8_t operatorType; bool blockingOptr; // block operator or not @@ -267,15 +278,7 @@ typedef struct SOperatorInfo { SResultInfo resultInfo; struct SOperatorInfo** pDownstream; // downstram pointer list int32_t numOfDownstream; // number of downstream. The value is always ONE expect for join operator - // todo extract struct - __optr_open_fn_t _openFn; // DO NOT invoke this function directly - __optr_fn_t getNextFn; - __optr_fn_t getStreamResFn; // execute the aggregate in the stream model. - __optr_fn_t cleanupFn; // call this function to release the allocated resources ASAP - __optr_close_fn_t closeFn; - __optr_encode_fn_t encodeResultRow; - __optr_decode_fn_t decodeResultRow; - __optr_get_explain_fn_t getExplainFn; + SOperatorFpSet fpSet; } SOperatorInfo; typedef struct { @@ -377,7 +380,7 @@ typedef struct SStreamBlockScanInfo { typedef struct SSysTableScanInfo { union { void* pTransporter; - void* readHandle; + SReadHandle readHandle; }; SRetrieveMetaTableRsp* pRsp; @@ -609,6 +612,10 @@ typedef struct SJoinOperatorInfo { SNode *pOnCondition; } SJoinOperatorInfo; +SOperatorFpSet createOperatorFpSet(__optr_open_fn_t openFn, __optr_fn_t nextFn, __optr_fn_t streamFn, + __optr_fn_t cleanup, __optr_close_fn_t closeFn, __optr_encode_fn_t encode, + __optr_decode_fn_t decode, __optr_get_explain_fn_t explain); + int32_t operatorDummyOpenFn(SOperatorInfo* pOperator); void operatorDummyCloseFn(void* param, int32_t numOfCols); int32_t appendDownstream(SOperatorInfo* p, SOperatorInfo** pDownstream, int32_t num); @@ -653,6 +660,9 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* pSysTableReadHandle, SSDataB SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, STimeWindowAggSupp *pTwAggSupp, const STableGroupInfo* pTableGroupInfo, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, + SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, + STimeWindowAggSupp *pTwAggSupp, const STableGroupInfo* pTableGroupInfo, SExecTaskInfo* pTaskInfo); SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, int64_t gap, STimeWindowAggSupp *pTwAggSupp, SExecTaskInfo* pTaskInfo); SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, diff --git a/source/libs/executor/src/executorMain.c b/source/libs/executor/src/executorMain.c index 5cbda90733..516afe5553 100644 --- a/source/libs/executor/src/executorMain.c +++ b/source/libs/executor/src/executorMain.c @@ -159,7 +159,7 @@ int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t *useconds) { int64_t st = 0; st = taosGetTimestampUs(); - *pRes = pTaskInfo->pRoot->getNextFn(pTaskInfo->pRoot, &newgroup); + *pRes = pTaskInfo->pRoot->fpSet.getNextFn(pTaskInfo->pRoot, &newgroup); uint64_t el = (taosGetTimestampUs() - st); pTaskInfo->cost.elapsedTime += el; diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 3541013015..cee84c65d0 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -226,6 +226,23 @@ int32_t operatorDummyOpenFn(SOperatorInfo* pOperator) { return TSDB_CODE_SUCCESS; } +SOperatorFpSet createOperatorFpSet(__optr_open_fn_t openFn, __optr_fn_t nextFn, __optr_fn_t streamFn, + __optr_fn_t cleanup, __optr_close_fn_t closeFn, __optr_encode_fn_t encode, + __optr_decode_fn_t decode, __optr_get_explain_fn_t explain) { + SOperatorFpSet fpSet = { + ._openFn = openFn, + .getNextFn = nextFn, + .getStreamResFn = streamFn, + .cleanupFn = cleanup, + .closeFn = closeFn, + .encodeResultRow = encode, + .decodeResultRow = decode, + .getExplainFn = explain, + }; + + return fpSet; +} + void operatorDummyCloseFn(void* param, int32_t numOfCols) {} static int32_t doCopyToSDataBlock(SSDataBlock* pBlock, SExprInfo* pExprInfo, SDiskbasedBuf* pBuf, @@ -1017,7 +1034,7 @@ void setInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SSDataBlo } } -static int32_t doCreateConstantValColumnInfo(SInputColumnInfoData* pInput, SFunctParam* pFuncParam, int32_t type, +static int32_t doCreateConstantValColumnInfo(SInputColumnInfoData* pInput, SFunctParam* pFuncParam, int32_t paramIndex, int32_t numOfRows) { SColumnInfoData* pColInfo = NULL; if (pInput->pData[paramIndex] == NULL) { @@ -1027,17 +1044,17 @@ static int32_t doCreateConstantValColumnInfo(SInputColumnInfoData* pInput, SFunc } // Set the correct column info (data type and bytes) - pColInfo->info.type = type; - pColInfo->info.bytes = tDataTypes[type].bytes; + pColInfo->info.type = pFuncParam->param.nType; + pColInfo->info.bytes = pFuncParam->param.nLen; pInput->pData[paramIndex] = pColInfo; } else { pColInfo = pInput->pData[paramIndex]; } - ASSERT(!IS_VAR_DATA_TYPE(type)); colInfoDataEnsureCapacity(pColInfo, 0, numOfRows); + int8_t type = pFuncParam->param.nType; if (type == TSDB_DATA_TYPE_BIGINT || type == TSDB_DATA_TYPE_UBIGINT) { int64_t v = pFuncParam->param.i; for (int32_t i = 0; i < numOfRows; ++i) { @@ -1048,6 +1065,12 @@ static int32_t doCreateConstantValColumnInfo(SInputColumnInfoData* pInput, SFunc for (int32_t i = 0; i < numOfRows; ++i) { colDataAppendDouble(pColInfo, i, &v); } + } else if (type == TSDB_DATA_TYPE_VARCHAR) { + char *tmp = taosMemoryMalloc(pFuncParam->param.nLen + VARSTR_HEADER_SIZE); + STR_WITH_SIZE_TO_VARSTR(tmp, pFuncParam->param.pz, pFuncParam->param.nLen); + for(int32_t i = 0; i < numOfRows; ++i) { + colDataAppend(pColInfo, i, tmp, false); + } } return TSDB_CODE_SUCCESS; @@ -1087,7 +1110,7 @@ static int32_t doSetInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCt pInput->numOfRows = pBlock->info.rows; pInput->startRowIndex = 0; - code = doCreateConstantValColumnInfo(pInput, pFuncParam, pFuncParam->param.nType, j, pBlock->info.rows); + code = doCreateConstantValColumnInfo(pInput, pFuncParam, j, pBlock->info.rows); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2077,7 +2100,7 @@ static int32_t updateBlockLoadStatus(STaskAttr* pQuery, int32_t status) { // // pQueryAttr->order.order = TSDB_ORDER_ASC; // if (pQueryAttr->window.skey > pQueryAttr->window.ekey) { -// TSWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); +// TSWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey); // } // // pQueryAttr->needReverseScan = false; @@ -2087,7 +2110,7 @@ static int32_t updateBlockLoadStatus(STaskAttr* pQuery, int32_t status) { // if (pQueryAttr->groupbyColumn && pQueryAttr->order.order == TSDB_ORDER_DESC) { // pQueryAttr->order.order = TSDB_ORDER_ASC; // if (pQueryAttr->window.skey > pQueryAttr->window.ekey) { -// TSWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); +// TSWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey); // } // // pQueryAttr->needReverseScan = false; @@ -2112,7 +2135,7 @@ static int32_t updateBlockLoadStatus(STaskAttr* pQuery, int32_t status) { // //qDebug(msg, pQInfo->qId, "only-first", pQueryAttr->order.order, TSDB_ORDER_ASC, pQueryAttr->window.skey, //// pQueryAttr->window.ekey, pQueryAttr->window.ekey, pQueryAttr->window.skey); // -// TSWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); +// TSWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey); // doUpdateLastKey(pQueryAttr); // } // @@ -2123,7 +2146,7 @@ static int32_t updateBlockLoadStatus(STaskAttr* pQuery, int32_t status) { // //qDebug(msg, pQInfo->qId, "only-last", pQueryAttr->order.order, TSDB_ORDER_DESC, pQueryAttr->window.skey, //// pQueryAttr->window.ekey, pQueryAttr->window.ekey, pQueryAttr->window.skey); // -// TSWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); +// TSWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey); // doUpdateLastKey(pQueryAttr); // } // @@ -2139,7 +2162,7 @@ static int32_t updateBlockLoadStatus(STaskAttr* pQuery, int32_t status) { //// pQueryAttr->window.skey, pQueryAttr->window.ekey, pQueryAttr->window.ekey, /// pQueryAttr->window.skey); // -// TSWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); +// TSWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey); // doUpdateLastKey(pQueryAttr); // } // @@ -2151,7 +2174,7 @@ static int32_t updateBlockLoadStatus(STaskAttr* pQuery, int32_t status) { //// pQueryAttr->window.skey, pQueryAttr->window.ekey, pQueryAttr->window.ekey, /// pQueryAttr->window.skey); // -// TSWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); +// TSWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey); // doUpdateLastKey(pQueryAttr); // } // @@ -2650,7 +2673,7 @@ static void updateTableQueryInfoForReverseScan(STableQueryInfo* pTableQueryInfo) return; } - // TSWAP(pTableQueryInfo->win.skey, pTableQueryInfo->win.ekey, TSKEY); + // TSWAP(pTableQueryInfo->win.skey, pTableQueryInfo->win.ekey); // pTableQueryInfo->lastKey = pTableQueryInfo->win.skey; // SWITCH_ORDER(pTableQueryInfo->cur.order); @@ -3715,7 +3738,32 @@ static int32_t doSendFetchDataRequest(SExchangeInfo* pExchangeInfo, SExecTaskInf return TSDB_CODE_SUCCESS; } -// TODO if only one or two columns required, how to extract data? +// NOTE: sources columns are more than the destination SSDatablock columns. +static void relocateColumnData(SSDataBlock* pBlock, const SArray* pColMatchInfo, SArray* pCols) { + size_t numOfSrcCols = taosArrayGetSize(pCols); + ASSERT(numOfSrcCols >= pBlock->info.numOfCols); + + int32_t i = 0, j = 0; + while(i < numOfSrcCols && j < taosArrayGetSize(pColMatchInfo)) { + SColumnInfoData* p = taosArrayGet(pCols, i); + SColMatchInfo* pmInfo = taosArrayGet(pColMatchInfo, j); + if (!pmInfo->output) { + j++; + continue; + } + + if (p->info.colId == pmInfo->colId) { + taosArraySet(pBlock->pDataBlock, pmInfo->targetSlotId, p); + i++; + j++; + } else if (p->info.colId < pmInfo->colId) { + i++; + } else { + ASSERT(0); + } + } +} + int32_t setSDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLoadInfo, int32_t numOfRows, char* pData, int32_t compLen, int32_t numOfOutput, int64_t startTs, uint64_t* total, SArray* pColList) { @@ -3733,7 +3781,7 @@ int32_t setSDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLoadI char* pStart = pData + sizeof(int32_t) * numOfOutput; for (int32_t i = 0; i < numOfOutput; ++i) { colLen[i] = htonl(colLen[i]); - ASSERT(colLen[i] > 0); + ASSERT(colLen[i] >= 0); SColumnInfoData* pColInfoData = taosArrayGet(pRes->pDataBlock, i); if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { @@ -3743,13 +3791,18 @@ int32_t setSDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLoadI memcpy(pColInfoData->varmeta.offset, pStart, sizeof(int32_t) * numOfRows); pStart += sizeof(int32_t) * numOfRows; - pColInfoData->pData = taosMemoryMalloc(colLen[i]); + if (colLen[i] > 0) { + pColInfoData->pData = taosMemoryMalloc(colLen[i]); + } } else { memcpy(pColInfoData->nullbitmap, pStart, BitmapLen(numOfRows)); pStart += BitmapLen(numOfRows); } - memcpy(pColInfoData->pData, pStart, colLen[i]); + if (colLen[i] > 0) { + memcpy(pColInfoData->pData, pStart, colLen[i]); + } + //TODO setting this flag to true temporarily so aggregate function on stable will //examine NULL value for non-primary key column pColInfoData->hasNull = true; @@ -3762,6 +3815,7 @@ int32_t setSDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLoadI int32_t numOfCols = htonl(*(int32_t*)pStart); pStart += sizeof(int32_t); + // todo refactor:extract method SSysTableSchema* pSchema = (SSysTableSchema*)pStart; for (int32_t i = 0; i < numOfCols; ++i) { SSysTableSchema* p = (SSysTableSchema*)pStart; @@ -3816,19 +3870,7 @@ int32_t setSDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLoadI } // data from mnode - for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData* pSrc = taosArrayGet(block.pDataBlock, i); - - for (int32_t j = 0; j < numOfOutput; ++j) { - int16_t colIndex = *(int16_t*)taosArrayGet(pColList, j); - - if (colIndex - 1 == i) { - SColumnInfoData* pColInfoData = taosArrayGet(pRes->pDataBlock, j); - colDataAssign(pColInfoData, pSrc, numOfRows); - break; - } - } - } + relocateColumnData(pRes, pColList, block.pDataBlock); } pRes->info.rows = numOfRows; @@ -4081,7 +4123,7 @@ static SSDataBlock* doLoadRemoteData(SOperatorInfo* pOperator, bool* newgroup) { SExchangeInfo* pExchangeInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - pTaskInfo->code = pOperator->_openFn(pOperator); + pTaskInfo->code = pOperator->fpSet._openFn(pOperator); if (pTaskInfo->code != TSDB_CODE_SUCCESS) { return NULL; } @@ -4176,9 +4218,9 @@ SOperatorInfo* createExchangeOperatorInfo(const SNodeList* pSources, SSDataBlock pOperator->info = pInfo; pOperator->numOfOutput = pBlock->info.numOfCols; pOperator->pTaskInfo = pTaskInfo; - pOperator->_openFn = prepareLoadRemoteData; // assign a dummy function. - pOperator->getNextFn = doLoadRemoteData; - pOperator->closeFn = destroyExchangeOperatorInfo; + + pOperator->fpSet = createOperatorFpSet(prepareLoadRemoteData, doLoadRemoteData, NULL, NULL, destroyExchangeOperatorInfo, + NULL, NULL, NULL); #if 1 { // todo refactor @@ -4289,7 +4331,7 @@ SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, i SSDataBlock* loadNextDataBlock(void* param) { SOperatorInfo* pOperator = (SOperatorInfo*)param; bool newgroup = false; - return pOperator->getNextFn(pOperator, &newgroup); + return pOperator->fpSet.getNextFn(pOperator, &newgroup); } static bool needToMerge(SSDataBlock* pBlock, SArray* groupInfo, char** buf, int32_t rowIndex) { @@ -4586,9 +4628,9 @@ SOperatorInfo* createSortedMergeOperatorInfo(SOperatorInfo** downstream, int32_t pOperator->pExpr = pExprInfo; pOperator->pTaskInfo = pTaskInfo; - pOperator->getNextFn = doSortedMerge; - pOperator->closeFn = destroySortedMergeOperatorInfo; + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doSortedMerge, NULL, NULL, destroySortedMergeOperatorInfo, + NULL, NULL, NULL); code = appendDownstream(pOperator, downstream, numOfDownstream); if (code != TSDB_CODE_SUCCESS) { goto _error; @@ -4667,8 +4709,8 @@ SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSDataBlock* pR pOperator->info = pInfo; pOperator->pTaskInfo = pTaskInfo; - pOperator->getNextFn = doSort; - pOperator->closeFn = destroyOrderOperatorInfo; + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doSort, NULL, NULL, destroyOrderOperatorInfo, + NULL, NULL, NULL); int32_t code = appendDownstream(pOperator, &downstream, 1); return pOperator; @@ -4710,7 +4752,7 @@ static int32_t doOpenAggregateOptr(SOperatorInfo* pOperator) { bool newgroup = true; while (1) { publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = downstream->getNextFn(downstream, &newgroup); + SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream, &newgroup); publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { @@ -4765,7 +4807,7 @@ static SSDataBlock* getAggregateResult(SOperatorInfo* pOperator, bool* newgroup) } SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - pTaskInfo->code = pOperator->_openFn(pOperator); + pTaskInfo->code = pOperator->fpSet._openFn(pOperator); if (pTaskInfo->code != TSDB_CODE_SUCCESS) { return NULL; } @@ -4952,13 +4994,20 @@ static int32_t handleLimitOffset(SOperatorInfo* pOperator, SSDataBlock* pBlock) pProjectInfo->curOffset = 0; } - if (pRes->info.rows >= pOperator->resultInfo.threshold) { + // check for the limitation in each group + if (pProjectInfo->limit.limit > 0 && pProjectInfo->curOutput + pRes->info.rows >= pProjectInfo->limit.limit) { + pRes->info.rows = (int32_t)(pProjectInfo->limit.limit - pProjectInfo->curOutput); - // check for the limitation in each group - if (pProjectInfo->limit.limit > 0 && pProjectInfo->curOutput + pRes->info.rows >= pProjectInfo->limit.limit) { - pRes->info.rows = (int32_t)(pProjectInfo->limit.limit - pProjectInfo->curOutput); + if (pProjectInfo->slimit.limit == -1 || pProjectInfo->slimit.limit <= pProjectInfo->curGroupOutput) { + pOperator->status = OP_EXEC_DONE; } + return PROJECT_RETRIEVE_DONE; + } + + // If there are slimit/soffset value exists, multi-round result can not be packed into one group, since the + // they may not belong to the same group the limit/offset value is not valid in this case. + if (pRes->info.rows >= pOperator->resultInfo.threshold || pProjectInfo->slimit.offset != -1 || pProjectInfo->slimit.limit != -1) { return PROJECT_RETRIEVE_DONE; } else { // not full enough, continue to accumulate the output data in the buffer. return PROJECT_RETRIEVE_CONTINUE; @@ -5007,7 +5056,7 @@ static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator, bool* newgroup) // The downstream exec may change the value of the newgroup, so use a local variable instead. publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = downstream->getNextFn(downstream, newgroup); + SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream, newgroup); publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { @@ -5070,7 +5119,7 @@ static int32_t doOpenIntervalAgg(SOperatorInfo* pOperator) { while (1) { publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = downstream->getNextFn(downstream, &newgroup); + SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream, &newgroup); publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { @@ -5121,9 +5170,9 @@ static SSDataBlock* doBuildIntervalResult(SOperatorInfo* pOperator, bool* newgro SSDataBlock* pBlock = pInfo->binfo.pRes; if (pInfo->execModel == OPTR_EXEC_MODEL_STREAM) { - return pOperator->getStreamResFn(pOperator, newgroup); + return pOperator->fpSet.getStreamResFn(pOperator, newgroup); } else { - pTaskInfo->code = pOperator->_openFn(pOperator); + pTaskInfo->code = pOperator->fpSet._openFn(pOperator); if (pTaskInfo->code != TSDB_CODE_SUCCESS) { return NULL; } @@ -5165,7 +5214,7 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator, bool* newgroup while (1) { publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = downstream->getNextFn(downstream, newgroup); + SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream, newgroup); publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { @@ -5216,7 +5265,7 @@ static SSDataBlock* doAllIntervalAgg(SOperatorInfo* pOperator, bool* newgroup) { while (1) { publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = downstream->getNextFn(downstream, newgroup); + SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream, newgroup); publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { break; @@ -5265,7 +5314,7 @@ static SSDataBlock* doSTableIntervalAgg(SOperatorInfo* pOperator, bool* newgroup while (1) { publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = downstream->getNextFn(downstream, newgroup); + SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream, newgroup); publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { @@ -5400,7 +5449,7 @@ static SSDataBlock* doStateWindowAgg(SOperatorInfo* pOperator, bool* newgroup) { SOperatorInfo* downstream = pOperator->pDownstream[0]; while (1) { publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = downstream->getNextFn(downstream, newgroup); + SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream, newgroup); publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { @@ -5451,7 +5500,7 @@ static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator, bool* newgroup) while (1) { publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = downstream->getNextFn(downstream, newgroup); + SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream, newgroup); publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { break; @@ -5531,7 +5580,7 @@ static SSDataBlock* doFill(SOperatorInfo* pOperator, bool* newgroup) { SOperatorInfo* pDownstream = pOperator->pDownstream[0]; while (1) { publishOperatorProfEvent(pDownstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = pDownstream->getNextFn(pDownstream, newgroup); + SSDataBlock* pBlock = pDownstream->fpSet.getNextFn(pDownstream, newgroup); publishOperatorProfEvent(pDownstream, QUERY_PROF_AFTER_OPERATOR_EXEC); if (*newgroup) { @@ -5603,8 +5652,8 @@ static void destroyOperatorInfo(SOperatorInfo* pOperator) { return; } - if (pOperator->closeFn != NULL) { - pOperator->closeFn(pOperator->info, pOperator->numOfOutput); + if (pOperator->fpSet.closeFn != NULL) { + pOperator->fpSet.closeFn(pOperator->info, pOperator->numOfOutput); } if (pOperator->pDownstream != NULL) { @@ -5739,12 +5788,9 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo* pOperator->pExpr = pExprInfo; pOperator->numOfOutput = numOfCols; pOperator->pTaskInfo = pTaskInfo; - pOperator->_openFn = doOpenAggregateOptr; - pOperator->getNextFn = getAggregateResult; - pOperator->closeFn = destroyAggOperatorInfo; - pOperator->encodeResultRow = aggEncodeResultRow; - pOperator->decodeResultRow = aggDecodeResultRow; + pOperator->fpSet = createOperatorFpSet(doOpenAggregateOptr, getAggregateResult, NULL, NULL, destroyAggOperatorInfo, + aggEncodeResultRow, aggDecodeResultRow, NULL); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { @@ -5871,9 +5917,9 @@ SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SExprInfo* p pOperator->info = pInfo; pOperator->pExpr = pExprInfo; pOperator->numOfOutput = num; - pOperator->_openFn = operatorDummyOpenFn; - pOperator->getNextFn = doProjectOperation; - pOperator->closeFn = destroyProjectOperatorInfo; + + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doProjectOperation, NULL, NULL, destroyProjectOperatorInfo, + NULL, NULL, NULL); pOperator->pTaskInfo = pTaskInfo; int32_t code = appendDownstream(pOperator, &downstream, 1); @@ -5929,12 +5975,9 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pOperator->pTaskInfo = pTaskInfo; pOperator->numOfOutput = numOfCols; pOperator->info = pInfo; - pOperator->_openFn = doOpenIntervalAgg; - pOperator->getNextFn = doBuildIntervalResult; - pOperator->getStreamResFn = doStreamIntervalAgg; - pOperator->closeFn = destroyIntervalOperatorInfo; - pOperator->encodeResultRow = aggEncodeResultRow; - pOperator->decodeResultRow = aggDecodeResultRow; + + pOperator->fpSet = createOperatorFpSet(doOpenIntervalAgg, doBuildIntervalResult, doStreamIntervalAgg, NULL, destroyIntervalOperatorInfo, + aggEncodeResultRow, aggDecodeResultRow, NULL); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { @@ -5951,6 +5994,65 @@ _error: return NULL; } +SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, + SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, + STimeWindowAggSupp *pTwAggSupp, const STableGroupInfo* pTableGroupInfo, SExecTaskInfo* pTaskInfo) { + STableIntervalOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(STableIntervalOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + if (pInfo == NULL || pOperator == NULL) { + goto _error; + } + + pInfo->order = TSDB_ORDER_ASC; + pInfo->interval = *pInterval; + pInfo->execModel = OPTR_EXEC_MODEL_STREAM; + pInfo->win = pTaskInfo->window; + pInfo->twAggSup = *pTwAggSupp; + pInfo->primaryTsIndex = primaryTsSlotId; + + int32_t numOfRows = 4096; + size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; + + initResultSizeInfo(pOperator, numOfRows); + int32_t code = + initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExprInfo, numOfCols, pResBlock, keyBufSize, pTaskInfo->id.str); + initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pInfo->win); + + // pInfo->pTableQueryInfo = initTableQueryInfo(pTableGroupInfo); + if (code != TSDB_CODE_SUCCESS /* || pInfo->pTableQueryInfo == NULL*/) { + goto _error; + } + + initResultRowInfo(&pInfo->binfo.resultRowInfo, (int32_t)1); + + pOperator->name = "StreamTimeIntervalAggOperator"; + pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_INTERVAL; + pOperator->blockingOptr = true; + pOperator->status = OP_NOT_OPENED; + pOperator->pExpr = pExprInfo; + pOperator->pTaskInfo = pTaskInfo; + pOperator->numOfOutput = numOfCols; + pOperator->info = pInfo; + + pOperator->fpSet = createOperatorFpSet(doOpenIntervalAgg, doStreamIntervalAgg, doStreamIntervalAgg, NULL, destroyIntervalOperatorInfo, + aggEncodeResultRow, aggDecodeResultRow, NULL); + + code = appendDownstream(pOperator, &downstream, 1); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + + return pOperator; + + _error: + destroyIntervalOperatorInfo(pInfo, numOfCols); + taosMemoryFreeClear(pInfo); + taosMemoryFreeClear(pOperator); + pTaskInfo->code = code; + return NULL; + +} + SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock, SExecTaskInfo* pTaskInfo) { STimeSliceOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(STimeSliceOperatorInfo)); @@ -5969,8 +6071,9 @@ SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SExprInfo* pOperator->numOfOutput = numOfCols; pOperator->info = pInfo; pOperator->pTaskInfo = pTaskInfo; - pOperator->getNextFn = doAllIntervalAgg; - pOperator->closeFn = destroyBasicOperatorInfo; + + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doAllIntervalAgg, NULL, NULL, destroyBasicOperatorInfo, + NULL, NULL, NULL); int32_t code = appendDownstream(pOperator, &downstream, 1); return pOperator; @@ -6010,10 +6113,9 @@ SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SExprInf pOperator->pTaskInfo = pTaskInfo; pOperator->info = pInfo; - pOperator->getNextFn = doStateWindowAgg; - pOperator->closeFn = destroyStateWindowOperatorInfo; - pOperator->encodeResultRow = aggEncodeResultRow; - pOperator->decodeResultRow = aggDecodeResultRow; + + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStateWindowAgg, NULL, NULL, destroyStateWindowOperatorInfo, + aggEncodeResultRow, aggDecodeResultRow, NULL); int32_t code = appendDownstream(pOperator, &downstream, 1); return pOperator; @@ -6057,10 +6159,9 @@ SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SExprInfo pOperator->pExpr = pExprInfo; pOperator->numOfOutput = numOfCols; pOperator->info = pInfo; - pOperator->getNextFn = doSessionWindowAgg; - pOperator->closeFn = destroySWindowOperatorInfo; - pOperator->encodeResultRow = aggEncodeResultRow; - pOperator->decodeResultRow = aggDecodeResultRow; + + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doSessionWindowAgg, NULL, NULL, destroySWindowOperatorInfo, + aggEncodeResultRow, aggDecodeResultRow, NULL); pOperator->pTaskInfo = pTaskInfo; code = appendDownstream(pOperator, &downstream, 1); @@ -6147,12 +6248,10 @@ SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExp pOperator->pExpr = pExpr; pOperator->numOfOutput = numOfCols; pOperator->info = pInfo; - pOperator->_openFn = operatorDummyOpenFn; - pOperator->getNextFn = doFill; + + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doFill, NULL, NULL, destroySFillOperatorInfo, + NULL, NULL, NULL); pOperator->pTaskInfo = pTaskInfo; - - pOperator->closeFn = destroySFillOperatorInfo; - code = appendDownstream(pOperator, &downstream, 1); return pOperator; @@ -6350,7 +6449,6 @@ static tsdbReaderT doCreateDataReader(STableScanPhysiNode* pTableScanNode, SRead static int32_t doCreateTableGroup(void* metaHandle, int32_t tableType, uint64_t tableUid, STableGroupInfo* pGroupInfo, uint64_t queryId, uint64_t taskId); static SArray* extractTableIdList(const STableGroupInfo* pTableGroupInfo); -static SArray* extractScanColumnId(SNodeList* pNodeList); static SArray* extractColumnInfo(SNodeList* pNodeList); static SArray* extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNodeList, int32_t* numOfOutputCols); @@ -6421,10 +6519,11 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo SScanPhysiNode* pScanNode = &pSysScanPhyNode->scan; SSDataBlock* pResBlock = createResDataBlock(pScanNode->node.pOutputDataBlockDesc); - SArray* colList = extractScanColumnId(pScanNode->pScanCols); + int32_t numOfOutputCols = 0; + SArray* colList = extractColMatchInfo(pScanNode->pScanCols, pScanNode->node.pOutputDataBlockDesc, &numOfOutputCols); SOperatorInfo* pOperator = createSysTableScanOperatorInfo( - pHandle->meta, pResBlock, &pScanNode->tableName, pScanNode->node.pConditions, pSysScanPhyNode->mgmtEpSet, + pHandle, pResBlock, &pScanNode->tableName, pScanNode->node.pConditions, pSysScanPhyNode->mgmtEpSet, colList, pTaskInfo, pSysScanPhyNode->showRewrite, pSysScanPhyNode->accountId); return pOperator; } else { @@ -6560,7 +6659,7 @@ static int32_t initQueryTableDataCond(SQueryTableDataCond* pCond, const STableS //todo work around a problem, remove it later if ((pCond->order == TSDB_ORDER_ASC && pCond->twindow.skey > pCond->twindow.ekey) || (pCond->order == TSDB_ORDER_DESC && pCond->twindow.skey < pCond->twindow.ekey)) { - TSWAP(pCond->twindow.skey, pCond->twindow.ekey, int64_t); + TSWAP(pCond->twindow.skey, pCond->twindow.ekey); } #endif @@ -6585,28 +6684,6 @@ static int32_t initQueryTableDataCond(SQueryTableDataCond* pCond, const STableS return TSDB_CODE_SUCCESS; } -SArray* extractScanColumnId(SNodeList* pNodeList) { - size_t numOfCols = LIST_LENGTH(pNodeList); - SArray* pList = taosArrayInit(numOfCols, sizeof(int16_t)); - if (pList == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; - } - - for (int32_t i = 0; i < numOfCols; ++i) { - for (int32_t j = 0; j < numOfCols; ++j) { - STargetNode* pNode = (STargetNode*)nodesListGetNode(pNodeList, j); - if (pNode->slotId == i) { - SColumnNode* pColNode = (SColumnNode*)pNode->pExpr; - taosArrayPush(pList, &pColNode->colId); - break; - } - } - } - - return pList; -} - SArray* extractColumnInfo(SNodeList* pNodeList) { size_t numOfCols = LIST_LENGTH(pNodeList); SArray* pList = taosArrayInit(numOfCols, sizeof(SColumn)); @@ -6617,18 +6694,32 @@ SArray* extractColumnInfo(SNodeList* pNodeList) { for (int32_t i = 0; i < numOfCols; ++i) { STargetNode* pNode = (STargetNode*)nodesListGetNode(pNodeList, i); - SColumnNode* pColNode = (SColumnNode*)pNode->pExpr; - // todo extract method - SColumn c = {0}; - c.slotId = pColNode->slotId; - c.colId = pColNode->colId; - c.type = pColNode->node.resType.type; - c.bytes = pColNode->node.resType.bytes; - c.precision = pColNode->node.resType.precision; - c.scale = pColNode->node.resType.scale; + if (nodeType(pNode->pExpr) == QUERY_NODE_COLUMN) { + SColumnNode* pColNode = (SColumnNode*)pNode->pExpr; - taosArrayPush(pList, &c); + // todo extract method + SColumn c = {0}; + c.slotId = pColNode->slotId; + c.colId = pColNode->colId; + c.type = pColNode->node.resType.type; + c.bytes = pColNode->node.resType.bytes; + c.scale = pColNode->node.resType.scale; + c.precision = pColNode->node.resType.precision; + + taosArrayPush(pList, &c); + } else if (nodeType(pNode->pExpr) == QUERY_NODE_VALUE) { + SValueNode* pValNode = (SValueNode*) pNode->pExpr; + SColumn c = {0}; + c.slotId = pNode->slotId; + c.colId = pNode->slotId; + c.type = pValNode->node.type; + c.bytes = pValNode->node.resType.bytes; + c.scale = pValNode->node.resType.scale; + c.precision = pValNode->node.resType.precision; + + taosArrayPush(pList, &c); + } } return pList; @@ -6728,9 +6819,9 @@ SArray* extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNod SColumnNode* pColNode = (SColumnNode*)pNode->pExpr; SColMatchInfo c = {0}; + c.output = true; c.colId = pColNode->colId; c.targetSlotId = pNode->slotId; - c.output = true; taosArrayPush(pList, &c); } @@ -6738,8 +6829,10 @@ SArray* extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNod int32_t num = LIST_LENGTH(pOutputNodeList->pSlots); for (int32_t i = 0; i < num; ++i) { SSlotDescNode* pNode = (SSlotDescNode*)nodesListGetNode(pOutputNodeList->pSlots, i); + // todo: add reserve flag check - if (pNode->slotId >= numOfCols) { // it is a column reserved for the arithmetic expression calculation + // it is a column reserved for the arithmetic expression calculation + if (pNode->slotId >= numOfCols) { (*numOfOutputCols) += 1; continue; } @@ -7017,8 +7110,8 @@ int32_t getOperatorExplainExecInfo(SOperatorInfo* operatorInfo, SExplainExecInfo (*pRes)[*resNum].startupCost = operatorInfo->cost.openCost; (*pRes)[*resNum].totalCost = operatorInfo->cost.totalCost; - if (operatorInfo->getExplainFn) { - int32_t code = (*operatorInfo->getExplainFn)(operatorInfo, &(*pRes)->verboseInfo); + if (operatorInfo->fpSet.getExplainFn) { + int32_t code = (*operatorInfo->fpSet.getExplainFn)(operatorInfo, &(*pRes)->verboseInfo); if (code) { qError("operator getExplainFn failed, error:%s", tstrerror(code)); return code; @@ -7055,7 +7148,7 @@ static SSDataBlock* doMergeJoin(struct SOperatorInfo* pOperator, bool* newgroup) if (pJoinInfo->pLeft == NULL || pJoinInfo->leftPos >= pJoinInfo->pLeft->info.rows) { SOperatorInfo* ds1 = pOperator->pDownstream[0]; publishOperatorProfEvent(ds1, QUERY_PROF_BEFORE_OPERATOR_EXEC); - pJoinInfo->pLeft = ds1->getNextFn(ds1, newgroup); + pJoinInfo->pLeft = ds1->fpSet.getNextFn(ds1, newgroup); publishOperatorProfEvent(ds1, QUERY_PROF_AFTER_OPERATOR_EXEC); pJoinInfo->leftPos = 0; @@ -7068,7 +7161,7 @@ static SSDataBlock* doMergeJoin(struct SOperatorInfo* pOperator, bool* newgroup) if (pJoinInfo->pRight == NULL || pJoinInfo->rightPos >= pJoinInfo->pRight->info.rows) { SOperatorInfo* ds2 = pOperator->pDownstream[1]; publishOperatorProfEvent(ds2, QUERY_PROF_BEFORE_OPERATOR_EXEC); - pJoinInfo->pRight = ds2->getNextFn(ds2, newgroup); + pJoinInfo->pRight = ds2->fpSet.getNextFn(ds2, newgroup); publishOperatorProfEvent(ds2, QUERY_PROF_AFTER_OPERATOR_EXEC); pJoinInfo->rightPos = 0; @@ -7161,9 +7254,9 @@ SOperatorInfo* createJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t numOf pOperator->numOfOutput = numOfCols; pOperator->info = pInfo; pOperator->pTaskInfo = pTaskInfo; - pOperator->getNextFn = doMergeJoin; - pOperator->closeFn = destroyBasicOperatorInfo; + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doMergeJoin, NULL, NULL, destroyBasicOperatorInfo, + NULL, NULL, NULL); int32_t code = appendDownstream(pOperator, pDownstream, numOfDownstream); return pOperator; diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index 27c616498e..beee11ec18 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -277,7 +277,7 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator, bool* newgrou while (1) { publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = downstream->getNextFn(downstream, newgroup); + SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream, newgroup); publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { break; @@ -360,12 +360,8 @@ SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SExprInfo* pEx pOperator->numOfOutput = numOfCols; pOperator->info = pInfo; pOperator->pTaskInfo = pTaskInfo; - pOperator->_openFn = operatorDummyOpenFn; - pOperator->getNextFn = hashGroupbyAggregate; - pOperator->closeFn = destroyGroupOperatorInfo; - pOperator->encodeResultRow = aggEncodeResultRow; - pOperator->decodeResultRow = aggDecodeResultRow; + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, hashGroupbyAggregate, NULL, NULL, destroyGroupOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL); code = appendDownstream(pOperator, &downstream, 1); return pOperator; @@ -562,7 +558,7 @@ static SSDataBlock* hashPartition(SOperatorInfo* pOperator, bool* newgroup) { while (1) { publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = downstream->getNextFn(downstream, newgroup); + SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream, newgroup); publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { break; @@ -618,14 +614,13 @@ SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SExprInfo* pOperator->blockingOptr = true; pOperator->status = OP_NOT_OPENED; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_PARTITION; - pInfo->binfo.pRes = pResultBlock; pOperator->numOfOutput = numOfCols; pOperator->pExpr = pExprInfo; pOperator->info = pInfo; - pOperator->_openFn = operatorDummyOpenFn; - pOperator->getNextFn = hashPartition; - pOperator->closeFn = destroyPartitionOperatorInfo; + + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, hashPartition, NULL, NULL, destroyPartitionOperatorInfo, + NULL, NULL, NULL); code = appendDownstream(pOperator, &downstream, 1); return pOperator; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 277ab9bc6f..8291826e69 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -266,7 +266,7 @@ static void prepareForDescendingScan(STableScanInfo* pTableScanInfo, SqlFunction // setupQueryRangeForReverseScan(pTableScanInfo); STimeWindow* pTWindow = &pTableScanInfo->cond.twindow; - TSWAP(pTWindow->skey, pTWindow->ekey, int64_t); + TSWAP(pTWindow->skey, pTWindow->ekey); pTableScanInfo->cond.order = TSDB_ORDER_DESC; } @@ -405,7 +405,7 @@ SOperatorInfo* createTableScanOperatorInfo(void* pDataReader, SQueryTableDataCon pOperator->status = OP_NOT_OPENED; pOperator->info = pInfo; pOperator->numOfOutput = numOfOutput; - pOperator->getNextFn = doTableScan; + pOperator->fpSet.getNextFn = doTableScan; pOperator->pTaskInfo = pTaskInfo; static int32_t cost = 0; @@ -429,7 +429,7 @@ SOperatorInfo* createTableSeqScanOperatorInfo(void* pTsdbReadHandle) { pOperator->blockingOptr = false; pOperator->status = OP_NOT_OPENED; pOperator->info = pInfo; - pOperator->getNextFn = doTableScanImpl; + pOperator->fpSet.getNextFn = doTableScanImpl; return pOperator; } @@ -502,8 +502,8 @@ SOperatorInfo* createDataBlockInfoScanOperator(void* dataReader, SExecTaskInfo* // pOperator->operatorType = OP_TableBlockInfoScan; pOperator->blockingOptr = false; pOperator->status = OP_NOT_OPENED; - pOperator->_openFn = operatorDummyOpenFn; - pOperator->getNextFn = doBlockInfoScan; + pOperator->fpSet._openFn = operatorDummyOpenFn; + pOperator->fpSet.getNextFn = doBlockInfoScan; pOperator->info = pInfo; pOperator->pTaskInfo = pTaskInfo; @@ -532,7 +532,7 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator, bool* newgroup) SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SStreamBlockScanInfo* pInfo = pOperator->info; - pTaskInfo->code = pOperator->_openFn(pOperator); + pTaskInfo->code = pOperator->fpSet._openFn(pOperator); if (pTaskInfo->code != TSDB_CODE_SUCCESS || pOperator->status == OP_EXEC_DONE) { return NULL; } @@ -659,9 +659,9 @@ SOperatorInfo* createStreamScanOperatorInfo(void* streamReadHandle, SSDataBlock* pOperator->status = OP_NOT_OPENED; pOperator->info = pInfo; pOperator->numOfOutput = pResBlock->info.numOfCols; - pOperator->_openFn = operatorDummyOpenFn; - pOperator->getNextFn = doStreamBlockScan; - pOperator->closeFn = operatorDummyCloseFn; + pOperator->fpSet._openFn = operatorDummyOpenFn; + pOperator->fpSet.getNextFn = doStreamBlockScan; + pOperator->fpSet.closeFn = operatorDummyCloseFn; pOperator->pTaskInfo = pTaskInfo; return pOperator; @@ -811,7 +811,7 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator, bool* newgroup) { const char* name = tNameGetTableName(&pInfo->name); if (strncasecmp(name, TSDB_INS_TABLE_USER_TABLES, TSDB_TABLE_FNAME_LEN) == 0) { if (pInfo->pCur == NULL) { - pInfo->pCur = metaOpenTbCursor(pInfo->readHandle); + pInfo->pCur = metaOpenTbCursor(pInfo->readHandle.meta); } blockDataCleanup(pInfo->pRes); @@ -819,32 +819,97 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator, bool* newgroup) { int32_t tableNameSlotId = 1; SColumnInfoData* pTableNameCol = taosArrayGet(pInfo->pRes->pDataBlock, tableNameSlotId); - char* tb = NULL; int32_t numOfRows = 0; + const char* db = NULL; + int32_t vgId = 0; + vnodeGetInfo(pInfo->readHandle.vnode, &db, &vgId); + + SName sn = {0}; + char dbname[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; + tNameFromString(&sn, db, T_NAME_ACCT|T_NAME_DB); + + tNameGetDbName(&sn, varDataVal(dbname)); + varDataSetLen(dbname, strlen(varDataVal(dbname))); + char n[TSDB_TABLE_NAME_LEN] = {0}; while (metaTbCursorNext(pInfo->pCur) == 0) { STR_TO_VARSTR(n, pInfo->pCur->mr.me.name); colDataAppend(pTableNameCol, numOfRows, n, false); - numOfRows += 1; - if (numOfRows >= pInfo->capacity) { - break; + + int32_t tableType = pInfo->pCur->mr.me.type; + + // database name + SColumnInfoData* pColInfoData = taosArrayGet(pInfo->pRes->pDataBlock, 0); + colDataAppend(pColInfoData, numOfRows, dbname, false); + + // vgId + pColInfoData = taosArrayGet(pInfo->pRes->pDataBlock, 6); + colDataAppend(pColInfoData, numOfRows, (char*) &vgId, false); + + // table comment + // todo: set the correct comment + pColInfoData = taosArrayGet(pInfo->pRes->pDataBlock, 8); + colDataAppendNULL(pColInfoData, numOfRows); + + char str[256] = {0}; + if (tableType == TSDB_CHILD_TABLE) { + SMetaReader mr = {0}; + metaReaderInit(&mr, pInfo->readHandle.meta, 0); + metaGetTableEntryByUid(&mr, pInfo->pCur->mr.me.ctbEntry.suid); + + pColInfoData = taosArrayGet(pInfo->pRes->pDataBlock, 3); + colDataAppend(pColInfoData, numOfRows, (char*) &mr.me.stbEntry.schema.nCols, false); + + // create time + int64_t ts = pInfo->pCur->mr.me.ctbEntry.ctime; + pColInfoData = taosArrayGet(pInfo->pRes->pDataBlock, 2); + colDataAppend(pColInfoData, numOfRows, (char*) &ts, false); + + // super table name + STR_TO_VARSTR(str, mr.me.name); + pColInfoData = taosArrayGet(pInfo->pRes->pDataBlock, 4); + colDataAppend(pColInfoData, numOfRows, str, false); + metaReaderClear(&mr); + + // uid + pColInfoData = taosArrayGet(pInfo->pRes->pDataBlock, 5); + colDataAppend(pColInfoData, numOfRows, (char*) &pInfo->pCur->mr.me.uid, false); + + // ttl + pColInfoData = taosArrayGet(pInfo->pRes->pDataBlock, 7); + colDataAppend(pColInfoData, numOfRows, (char*) &pInfo->pCur->mr.me.ctbEntry.ttlDays, false); + + STR_TO_VARSTR(str, "CHILD_TABLE"); + } else if (tableType == TSDB_NORMAL_TABLE) { + // create time + pColInfoData = taosArrayGet(pInfo->pRes->pDataBlock, 2); + colDataAppend(pColInfoData, numOfRows, (char*) &pInfo->pCur->mr.me.ntbEntry.ctime, false); + + // number of columns + pColInfoData = taosArrayGet(pInfo->pRes->pDataBlock, 3); + colDataAppend(pColInfoData, numOfRows, (char*) &pInfo->pCur->mr.me.ntbEntry.schema.nCols, false); + + // super table name + pColInfoData = taosArrayGet(pInfo->pRes->pDataBlock, 4); + colDataAppendNULL(pColInfoData, numOfRows); + + // uid + pColInfoData = taosArrayGet(pInfo->pRes->pDataBlock, 5); + colDataAppend(pColInfoData, numOfRows, (char*) &pInfo->pCur->mr.me.uid, false); + + // ttl + pColInfoData = taosArrayGet(pInfo->pRes->pDataBlock, 7); + colDataAppend(pColInfoData, numOfRows, (char*) &pInfo->pCur->mr.me.ntbEntry.ttlDays, false); + + STR_TO_VARSTR(str, "NORMAL_TABLE"); } - for (int32_t i = 0; i < pInfo->pRes->info.numOfCols; ++i) { - if (i == tableNameSlotId) { - continue; - } + pColInfoData = taosArrayGet(pInfo->pRes->pDataBlock, 9); + colDataAppend(pColInfoData, numOfRows, str, false); - SColumnInfoData* pColInfoData = taosArrayGet(pInfo->pRes->pDataBlock, i); - int64_t tmp = 0; - char t[10] = {0}; - STR_TO_VARSTR(t, "_"); // TODO - if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { - colDataAppend(pColInfoData, numOfRows, t, false); - } else { - colDataAppend(pColInfoData, numOfRows, (char*)&tmp, false); - } + if (++numOfRows >= pInfo->capacity) { + break; } } @@ -923,7 +988,7 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator, bool* newgroup) { } } -SOperatorInfo* createSysTableScanOperatorInfo(void* pSysTableReadHandle, SSDataBlock* pResBlock, const SName* pName, +SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSDataBlock* pResBlock, const SName* pName, SNode* pCondition, SEpSet epset, SArray* colList, SExecTaskInfo* pTaskInfo, bool showRewrite, int32_t accountId) { SSysTableScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SSysTableScanInfo)); @@ -945,7 +1010,7 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* pSysTableReadHandle, SSDataB tNameAssign(&pInfo->name, pName); const char* name = tNameGetTableName(&pInfo->name); if (strncasecmp(name, TSDB_INS_TABLE_USER_TABLES, TSDB_TABLE_FNAME_LEN) == 0) { - pInfo->readHandle = pSysTableReadHandle; + pInfo->readHandle = *(SReadHandle*) readHandle; blockDataEnsureCapacity(pInfo->pRes, pInfo->capacity); } else { tsem_init(&pInfo->ready, 0, 0); @@ -981,8 +1046,8 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* pSysTableReadHandle, SSDataB pOperator->status = OP_NOT_OPENED; pOperator->info = pInfo; pOperator->numOfOutput = pResBlock->info.numOfCols; - pOperator->getNextFn = doSysTableScan; - pOperator->closeFn = destroySysScanOperator; + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doSysTableScan, NULL, NULL, destroySysScanOperator, + NULL, NULL, NULL); pOperator->pTaskInfo = pTaskInfo; return pOperator; @@ -1139,11 +1204,12 @@ SOperatorInfo* createTagScanOperatorInfo(void* pReaderHandle, SExprInfo* pExpr, pOperator->blockingOptr = false; pOperator->status = OP_NOT_OPENED; pOperator->info = pInfo; - pOperator->getNextFn = doTagScan; + + pOperator->fpSet = + createOperatorFpSet(operatorDummyOpenFn, doTagScan, NULL, NULL, destroyTagScanOperatorInfo, NULL, NULL, NULL); pOperator->pExpr = pExpr; pOperator->numOfOutput = numOfOutput; pOperator->pTaskInfo = pTaskInfo; - pOperator->closeFn = destroyTagScanOperatorInfo; return pOperator; _error: diff --git a/source/libs/executor/test/executorTests.cpp b/source/libs/executor/test/executorTests.cpp index 543df30686..de8df7b916 100644 --- a/source/libs/executor/test/executorTests.cpp +++ b/source/libs/executor/test/executorTests.cpp @@ -199,9 +199,9 @@ SOperatorInfo* createDummyOperator(int32_t startVal, int32_t numOfBlocks, int32_ pOperator->name = "dummyInputOpertor4Test"; if (numOfCols == 1) { - pOperator->getNextFn = getDummyBlock; + pOperator->fpSet.getNextFn = getDummyBlock; } else { - pOperator->getNextFn = get2ColsDummyBlock; + pOperator->fpSet.getNextFn = get2ColsDummyBlock; } SDummyInputInfo *pInfo = (SDummyInputInfo*) taosMemoryCalloc(1, sizeof(SDummyInputInfo)); diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index f8a7e77814..f4a724afa3 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -213,7 +213,7 @@ enum { int32_t getUdfdPipeName(char* pipeName, int32_t size) { char dnodeId[8] = {0}; - size_t dnodeIdSize; + size_t dnodeIdSize = sizeof(dnodeId); int32_t err = uv_os_getenv(UDF_DNODE_ID_ENV_NAME, dnodeId, &dnodeIdSize); if (err != 0) { dnodeId[0] = '1'; diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index dfeb2df911..62bb96f1cb 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -307,7 +307,7 @@ static int32_t buildOutput(SInsertParseContext* pCxt) { taosHashGetDup(pCxt->pVgroupsHashObj, (const char*)&src->vgId, sizeof(src->vgId), &dst->vg); dst->numOfTables = src->numOfTables; dst->size = src->size; - TSWAP(dst->pData, src->pData, char*); + TSWAP(dst->pData, src->pData); buildMsgHeader(src, dst); taosArrayPush(pCxt->pOutput->pDataBlocks, &dst); } @@ -1069,7 +1069,6 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) { if (TSDB_QUERY_HAS_TYPE(pCxt->pOutput->insertType, TSDB_QUERY_TYPE_STMT_INSERT) && tbNum > 0) { return buildInvalidOperationMsg(&pCxt->msg, "single table allowed in one stmt"); - ; } destroyInsertParseContextForTable(pCxt); diff --git a/source/libs/parser/src/parInsertData.c b/source/libs/parser/src/parInsertData.c index aee4566fa0..ae05d2293e 100644 --- a/source/libs/parser/src/parInsertData.c +++ b/source/libs/parser/src/parInsertData.c @@ -159,9 +159,10 @@ static int32_t createDataBlock(size_t defaultSize, int32_t rowSize, int32_t star int32_t buildCreateTbMsg(STableDataBlocks* pBlocks, SVCreateTbReq* pCreateTbReq) { SCoder coder = {0}; char* pBuf; - int32_t len; + int32_t len; - tEncodeSize(tEncodeSVCreateTbReq, pCreateTbReq, len); + int32_t ret = 0; + tEncodeSize(tEncodeSVCreateTbReq, pCreateTbReq, len, ret); if (pBlocks->nAllocSize - pBlocks->size < len) { pBlocks->nAllocSize += len + pBlocks->rowSize; char* pTmp = taosMemoryRealloc(pBlocks->pData, pBlocks->nAllocSize); diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index aae932471d..cb514e974a 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -2049,10 +2049,10 @@ static int32_t buildSampleAst(STranslateContext* pCxt, SSampleAstInfo* pInfo, ch } strcpy(pTable->table.dbName, pInfo->pDbName); strcpy(pTable->table.tableName, pInfo->pTableName); - TSWAP(pTable->pMeta, pInfo->pRollupTableMeta, STableMeta*); + TSWAP(pTable->pMeta, pInfo->pRollupTableMeta); pSelect->pFromTable = (SNode*)pTable; - TSWAP(pSelect->pProjectionList, pInfo->pFuncs, SNodeList*); + TSWAP(pSelect->pProjectionList, pInfo->pFuncs); SFunctionNode* pFunc = nodesMakeNode(QUERY_NODE_FUNCTION); if (NULL == pSelect->pProjectionList || NULL == pFunc) { nodesDestroyNode(pSelect); @@ -2069,9 +2069,9 @@ static int32_t buildSampleAst(STranslateContext* pCxt, SSampleAstInfo* pInfo, ch return TSDB_CODE_OUT_OF_MEMORY; } pSelect->pWindow = (SNode*)pInterval; - TSWAP(pInterval->pInterval, pInfo->pInterval, SNode*); - TSWAP(pInterval->pOffset, pInfo->pOffset, SNode*); - TSWAP(pInterval->pSliding, pInfo->pSliding, SNode*); + TSWAP(pInterval->pInterval, pInfo->pInterval); + TSWAP(pInterval->pOffset, pInfo->pOffset); + TSWAP(pInterval->pSliding, pInfo->pSliding); pInterval->pCol = nodesMakeNode(QUERY_NODE_COLUMN); if (NULL == pInterval->pCol) { nodesDestroyNode(pSelect); @@ -3282,7 +3282,8 @@ static int32_t serializeVgroupTablesBatch(SVgroupTablesBatch* pTbBatch, SArray* int tlen; SCoder coder = {0}; - tEncodeSize(tEncodeSVCreateTbBatchReq, &pTbBatch->req, tlen); + int32_t ret = 0; + tEncodeSize(tEncodeSVCreateTbBatchReq, &pTbBatch->req, tlen, ret); tlen += sizeof(SMsgHead); //+ tSerializeSVCreateTbBatchReq(NULL, &(pTbBatch->req)); void* buf = taosMemoryMalloc(tlen); if (NULL == buf) { @@ -3696,7 +3697,7 @@ static int32_t setQuery(STranslateContext* pCxt, SQuery* pQuery) { default: pQuery->execMode = QUERY_EXEC_MODE_RPC; if (NULL != pCxt->pCmdMsg) { - TSWAP(pQuery->pCmdMsg, pCxt->pCmdMsg, SCmdMsgInfo*); + TSWAP(pQuery->pCmdMsg, pCxt->pCmdMsg); pQuery->msgType = pQuery->pCmdMsg->msgType; } break; diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 450fc307ea..18e59859ac 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -251,8 +251,8 @@ static int32_t createScanLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect return TSDB_CODE_OUT_OF_MEMORY; } - TSWAP(pScan->pMeta, pRealTable->pMeta, STableMeta*); - TSWAP(pScan->pVgroupList, pRealTable->pVgroupList, SVgroupsInfo*); + TSWAP(pScan->pMeta, pRealTable->pMeta); + TSWAP(pScan->pVgroupList, pRealTable->pVgroupList); pScan->scanSeq[0] = 1; pScan->scanSeq[1] = 0; pScan->scanRange = TSWINDOW_INITIALIZER; @@ -954,7 +954,7 @@ static int32_t createVnodeModifLogicNode(SLogicPlanContext* pCxt, SVnodeModifOpS if (NULL == pModif) { return TSDB_CODE_OUT_OF_MEMORY; } - TSWAP(pModif->pDataBlocks, pStmt->pDataBlocks, SArray*); + TSWAP(pModif->pDataBlocks, pStmt->pDataBlocks); pModif->msgType = getMsgType(pStmt->sqlNodeType); *pLogicNode = (SLogicNode*)pModif; return TSDB_CODE_SUCCESS; diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index f1b16353b6..042b914927 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -246,7 +246,7 @@ static int32_t cpdMergeConds(SNode** pDst, SNodeList** pSrc) { static int32_t cpdCondAppend(SNode** pCond, SNode** pAdditionalCond) { if (NULL == *pCond) { - TSWAP(*pCond, *pAdditionalCond, SNode*); + TSWAP(*pCond, *pAdditionalCond); return TSDB_CODE_SUCCESS; } diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 707e6aac14..548957ab8e 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -1097,7 +1097,7 @@ static int32_t createDataInserter(SPhysiPlanContext* pCxt, SVgDataBlocks* pBlock pInserter->numOfTables = pBlocks->numOfTables; pInserter->size = pBlocks->size; - TSWAP(pInserter->pData, pBlocks->pData, char*); + TSWAP(pInserter->pData, pBlocks->pData); *pSink = (SDataSinkNode*)pInserter; return TSDB_CODE_SUCCESS; diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index 51bd36f9f9..1266e8ae4b 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -65,7 +65,7 @@ static SLogicSubplan* splCreateScanSubplan(SSplitContext* pCxt, SScanLogicNode* pSubplan->id.groupId = pCxt->groupId; pSubplan->subplanType = SUBPLAN_TYPE_SCAN; pSubplan->pNode = (SLogicNode*)nodesCloneNode(pScan); - TSWAP(pSubplan->pVgroupList, ((SScanLogicNode*)pSubplan->pNode)->pVgroupList, SVgroupsInfo*); + TSWAP(pSubplan->pVgroupList, ((SScanLogicNode*)pSubplan->pNode)->pVgroupList); SPLIT_FLAG_SET_MASK(pSubplan->splitFlag, flag); return pSubplan; } @@ -406,8 +406,7 @@ int32_t splitLogicPlan(SPlanContext* pCxt, SLogicNode* pLogicNode, SLogicSubplan } if (QUERY_NODE_LOGIC_PLAN_VNODE_MODIF == nodeType(pLogicNode)) { pSubplan->subplanType = SUBPLAN_TYPE_MODIFY; - TSWAP(((SVnodeModifLogicNode*)pLogicNode)->pDataBlocks, ((SVnodeModifLogicNode*)pSubplan->pNode)->pDataBlocks, - SArray*); + TSWAP(((SVnodeModifLogicNode*)pLogicNode)->pDataBlocks, ((SVnodeModifLogicNode*)pSubplan->pNode)->pDataBlocks); } else { pSubplan->subplanType = SUBPLAN_TYPE_SCAN; } diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index aa8a03f3d2..37f21e2511 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -21,6 +21,7 @@ extern "C" { #include #include "lz4.h" #include "os.h" +#include "osSocket.h" #include "rpcCache.h" #include "rpcHead.h" #include "rpcLog.h" @@ -105,6 +106,7 @@ typedef void* queue[2]; #define TRANS_RETRY_COUNT_LIMIT 10 // retry count limit #define TRANS_RETRY_INTERVAL 5 // ms retry interval +#define TRANS_CONN_TIMEOUT 3 // connect timeout typedef struct { SRpcInfo* pRpc; // associated SRpcInfo diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index c0da3f9c1f..fecb5d9279 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -49,6 +49,10 @@ void* rpcOpen(const SRpcInit* pInit) { pRpc->connType = pInit->connType; pRpc->idleTime = pInit->idleTime; pRpc->tcphandle = (*taosInitHandle[pRpc->connType])(0, pInit->localPort, pRpc->label, pRpc->numOfThreads, NULL, pRpc); + if (pRpc->tcphandle == NULL) { + taosMemoryFree(pRpc); + return NULL; + } pRpc->parent = pInit->parent; if (pInit->user) { memcpy(pRpc->user, pInit->user, strlen(pInit->user)); diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index ab57f7d017..20406763de 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -103,6 +103,10 @@ static void cliDestroyConn(SCliConn* pConn, bool clear /*clear tcp handle o static void cliDestroy(uv_handle_t* handle); static void cliSend(SCliConn* pConn); +/* + * set TCP connection timeout per-socket level + */ +static int cliCreateSocket(); // process data read from server, add decompress etc later static void cliHandleResp(SCliConn* conn); // handle except about conn @@ -729,9 +733,15 @@ void cliHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd) { if (ret) { tError("%s cli conn %p failed to set conn option, errmsg %s", pTransInst->label, conn, uv_err_name(ret)); } + int fd = taosCreateSocketWithTimeOutOpt(TRANS_CONN_TIMEOUT); + if (fd == -1) { + tTrace("%s cli conn %p failed to create socket", pTransInst->label, conn); + cliHandleExcept(conn); + return; + } + uv_tcp_open((uv_tcp_t*)conn->stream, fd); struct sockaddr_in addr; - addr.sin_family = AF_INET; addr.sin_addr.s_addr = taosGetIpv4FromFqdn(conn->ip); addr.sin_port = (uint16_t)htons((uint16_t)conn->port); @@ -902,7 +912,6 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { cliDestroy((uv_handle_t*)pConn->stream); return -1; } - } else if (pCtx->retryCount < TRANS_RETRY_COUNT_LIMIT) { if (pResp->contLen == 0) { pEpSet->inUse = (pEpSet->inUse++) % pEpSet->numOfEps; diff --git a/source/libs/transport/src/transSrv.c b/source/libs/transport/src/transSrv.c index c02cb07101..b78edf6cd0 100644 --- a/source/libs/transport/src/transSrv.c +++ b/source/libs/transport/src/transSrv.c @@ -93,6 +93,8 @@ typedef struct SServerObj { uint32_t ip; uint32_t port; uv_async_t* pAcceptAsync; // just to quit from from accept thread + + bool inited; } SServerObj; // handle @@ -143,7 +145,7 @@ static void (*transAsyncHandle[])(SSrvMsg* msg, SWorkThrdObj* thrd) = {uvHandleR static int32_t exHandlesMgt; -void uvInitExHandleMgt(); +void uvInitEnv(); void uvOpenExHandleMgt(int size); void uvCloseExHandleMgt(); int64_t uvAddExHandle(void* p); @@ -716,6 +718,7 @@ static bool addHandleToAcceptloop(void* arg) { } if ((err = uv_listen((uv_stream_t*)&srv->server, 512, uvOnAcceptCb)) != 0) { tError("failed to listen: %s", uv_err_name(err)); + terrno = TSDB_CODE_RPC_PORT_EADDRINUSE; return false; } return true; @@ -800,7 +803,7 @@ void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, srv->port = port; uv_loop_init(srv->loop); - taosThreadOnce(&transModuleInit, uvInitExHandleMgt); + taosThreadOnce(&transModuleInit, uvInitEnv); transSrvInst++; for (int i = 0; i < srv->numOfThreads; i++) { @@ -844,15 +847,15 @@ void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, goto End; // clear all resource later } - + srv->inited = true; return srv; End: transCloseServer(srv); return NULL; } -void uvInitExHandleMgt() { - // init exhandle mgt +void uvInitEnv() { + uv_os_setenv("UV_TCP_SINGLE_ACCEPT", "1"); uvOpenExHandleMgt(10000); } void uvOpenExHandleMgt(int size) { @@ -958,9 +961,10 @@ void transCloseServer(void* arg) { SServerObj* srv = arg; tDebug("send quit msg to accept thread"); - uv_async_send(srv->pAcceptAsync); - taosThreadJoin(srv->thread, NULL); - + if (srv->inited) { + uv_async_send(srv->pAcceptAsync); + taosThreadJoin(srv->thread, NULL); + } SRV_RELEASE_UV(srv->loop); for (int i = 0; i < srv->numOfThreads; i++) { diff --git a/source/os/src/osSemaphore.c b/source/os/src/osSemaphore.c index 9475625413..7df4c26afd 100644 --- a/source/os/src/osSemaphore.c +++ b/source/os/src/osSemaphore.c @@ -67,6 +67,12 @@ int32_t tsem_wait(tsem_t* sem) { return ret; } +int32_t tsem_timewait(tsem_t* sem, int64_t nanosecs) { + int ret = 0; + + return ret; +} + #elif defined(_TD_DARWIN_64) /* diff --git a/source/os/src/osSocket.c b/source/os/src/osSocket.c index 0430f68a26..6aa8520082 100644 --- a/source/os/src/osSocket.c +++ b/source/os/src/osSocket.c @@ -37,31 +37,35 @@ #include #if defined(DARWIN) - #include - #include "osEok.h" +#include +#include "osEok.h" #else - #include +#include #endif #endif +#ifndef INVALID_SOCKET +#define INVALID_SOCKET -1 +#endif + typedef struct TdSocketServer { #if SOCKET_WITH_LOCK TdThreadRwlock rwlock; #endif int refId; SocketFd fd; -} *TdSocketServerPtr, TdSocketServer; +} * TdSocketServerPtr, TdSocketServer; typedef struct TdEpoll { #if SOCKET_WITH_LOCK TdThreadRwlock rwlock; #endif - int refId; - EpollFd fd; -} *TdEpollPtr, TdEpoll; + int refId; + EpollFd fd; +} * TdEpollPtr, TdEpoll; int32_t taosSendto(TdSocketPtr pSocket, void *buf, int len, unsigned int flags, const struct sockaddr *dest_addr, - int addrlen) { + int addrlen) { if (pSocket == NULL || pSocket->fd < 0) { return -1; } @@ -94,7 +98,8 @@ int32_t taosReadSocket(TdSocketPtr pSocket, void *buf, int len) { #endif } -int32_t taosReadFromSocket(TdSocketPtr pSocket, void *buf, int32_t len, int32_t flags, struct sockaddr *destAddr, int *addrLen) { +int32_t taosReadFromSocket(TdSocketPtr pSocket, void *buf, int32_t len, int32_t flags, struct sockaddr *destAddr, + int *addrLen) { if (pSocket == NULL || pSocket->fd < 0) { return -1; } @@ -318,7 +323,7 @@ int32_t taosWriteMsg(TdSocketPtr pSocket, void *buf, int32_t nbytes) { return -1; } int32_t nleft, nwritten; - char *ptr = (char *)buf; + char * ptr = (char *)buf; nleft = nbytes; @@ -347,7 +352,7 @@ int32_t taosReadMsg(TdSocketPtr pSocket, void *buf, int32_t nbytes) { return -1; } int32_t nleft, nread; - char *ptr = (char *)buf; + char * ptr = (char *)buf; nleft = nbytes; @@ -689,8 +694,7 @@ TdSocketServerPtr taosOpenTcpServerSocket(uint32_t ip, uint16_t port) { return (TdSocketServerPtr)pSocket; } -TdSocketPtr taosAcceptTcpConnectSocket(TdSocketServerPtr pServerSocket, struct sockaddr *destAddr, - int *addrLen) { +TdSocketPtr taosAcceptTcpConnectSocket(TdSocketServerPtr pServerSocket, struct sockaddr *destAddr, int *addrLen) { if (pServerSocket == NULL || pServerSocket->fd < 0) { return NULL; } @@ -771,7 +775,7 @@ uint32_t taosGetIpv4FromFqdn(const char *fqdn) { int32_t ret = getaddrinfo(fqdn, NULL, &hints, &result); if (result) { - struct sockaddr *sa = result->ai_addr; + struct sockaddr * sa = result->ai_addr; struct sockaddr_in *si = (struct sockaddr_in *)sa; struct in_addr ia = si->sin_addr; uint32_t ip = ia.s_addr; @@ -887,7 +891,6 @@ int32_t taosGetSocketName(TdSocketPtr pSocket, struct sockaddr *destAddr, int *a return getsockname(pSocket->fd, destAddr, addrLen); } - TdEpollPtr taosCreateEpoll(int32_t size) { EpollFd fd = -1; #ifdef WINDOWS @@ -939,3 +942,28 @@ int32_t taosCloseEpoll(TdEpollPtr *ppEpoll) { taosMemoryFree(*ppEpoll); return code; } +/* + * Set TCP connection timeout per-socket level. + * ref [https://github.com/libuv/help/issues/54] + */ +int taosCreateSocketWithTimeOutOpt(uint32_t conn_timeout_sec) { +#if defined(WINDOWS) + SOCKET fd; +#else + int fd; +#endif + if ((fd = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP)) == INVALID_SOCKET) { + return -1; + } +#if defined(WINDOWS) + if (0 != setsockopt(fd, IPPROTO_TCP, TCP_MAXRT, (char *)&conn_timeout_sec, sizeof(conn_timeout_sec))) { + return -1; + } +#else // Linux like systems + uint32_t conn_timeout_ms = conn_timeout_sec * 1000; + if (0 != setsockopt(fd, IPPROTO_TCP, TCP_USER_TIMEOUT, (char *)&conn_timeout_ms, sizeof(conn_timeout_ms))) { + return -1; + } +#endif + return (int)fd; +} diff --git a/source/util/src/tarray.c b/source/util/src/tarray.c index 2d741b18f6..18bc883143 100644 --- a/source/util/src/tarray.c +++ b/source/util/src/tarray.c @@ -482,3 +482,22 @@ void taosArraySortPWithExt(SArray* pArray, __ext_compar_fn_t fn, const void* par taosArrayGetSize(pArray) > 8 ? taosArrayQuickSort(pArray, fn, param) : taosArrayInsertSort(pArray, fn, param); } // TODO(yihaoDeng) add order array +// + +char* taosShowStrArray(const SArray* pArray) { + int32_t sz = pArray->size; + int32_t tlen = 0; + for (int32_t i = 0; i < sz; i++) { + tlen += strlen(taosArrayGetP(pArray, i)) + 1; + } + char* res = taosMemoryCalloc(1, tlen); + char* buf = res; + for (int32_t i = 0; i < sz; i++) { + char* str = taosArrayGetP(pArray, i); + int32_t len = strlen(str); + memcpy(buf, str, len); + buf += len; + if (i != sz - 1) *buf = ','; + } + return res; +} diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index b864f48522..cb9ef87701 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -823,7 +823,7 @@ int32_t cfgLoadFromApollUrl(SConfig *pConfig, const char *url) { } p++; - if (memcmp(url, "jsonFile", 8) == 0) { + if (strncmp(url, "jsonFile", 8) == 0) { char *filepath = p; if (!taosCheckExistFile(filepath)) { uError("fial to load json file: %s", filepath); @@ -893,8 +893,8 @@ int32_t cfgLoadFromApollUrl(SConfig *pConfig, const char *url) { } tjsonDelete(pJson); - // } else if (memcmp(url, "jsonUrl", 7) == 0) { - // } else if (memcmp(url, "etcdUrl", 7) == 0) { + // } else if (strncmp(url, "jsonUrl", 7) == 0) { + // } else if (strncmp(url, "etcdUrl", 7) == 0) { } else { uError("Unsupported url: %s", url); return -1; @@ -908,7 +908,7 @@ int32_t cfgGetApollUrl(const char **envCmd, const char *envFile, char* apolloUrl int32_t index = 0; if (envCmd == NULL) return 0; while (envCmd[index]!=NULL) { - if (memcmp(envCmd[index], "TAOS_APOLLO_URL", 14) == 0) { + if (strncmp(envCmd[index], "TAOS_APOLLO_URL", 14) == 0) { char *p = strchr(envCmd[index], '='); if (p != NULL) { p++; @@ -934,7 +934,7 @@ int32_t cfgGetApollUrl(const char **envCmd, const char *envFile, char* apolloUrl break; } if(line[_bytes - 1] == '\n') line[_bytes - 1] = 0; - if (memcmp(line, "TAOS_APOLLO_URL", 14) == 0) { + if (strncmp(line, "TAOS_APOLLO_URL", 14) == 0) { char *p = strchr(line, '='); if (p != NULL) { p++; @@ -975,7 +975,7 @@ int32_t cfgGetApollUrl(const char **envCmd, const char *envFile, char* apolloUrl break; } if(line[_bytes - 1] == '\n') line[_bytes - 1] = 0; - if (memcmp(line, "TAOS_APOLLO_URL", 14) == 0) { + if (strncmp(line, "TAOS_APOLLO_URL", 14) == 0) { char *p = strchr(line, '='); if (p != NULL) { p++; diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 61f2015fe5..8f2ba2dcc4 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -68,6 +68,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_RPC_INVALID_TIME_STAMP, "Client and server's t TAOS_DEFINE_ERROR(TSDB_CODE_APP_NOT_READY, "Database not ready") TAOS_DEFINE_ERROR(TSDB_CODE_RPC_FQDN_ERROR, "Unable to resolve FQDN") TAOS_DEFINE_ERROR(TSDB_CODE_RPC_INVALID_VERSION, "Invalid app version") +TAOS_DEFINE_ERROR(TSDB_CODE_RPC_PORT_EADDRINUSE, "port already in use") //common & util TAOS_DEFINE_ERROR(TSDB_CODE_OUT_OF_MEMORY, "Out of Memory") @@ -271,9 +272,14 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_INVALID_STAGE, "Invalid stage to kill TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_CANT_PARALLEL, "Invalid stage to kill") // mnode-mq -TAOS_DEFINE_ERROR(TSDB_CODE_MND_UNSUPPORTED_TOPIC, "Topic with aggregation is unsupported") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_CONSUMER_NOT_READY, "Consumer waiting for rebalance") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOPIC_ALREADY_EXIST, "Topic already exists") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOPIC_NOT_EXIST, "Topic not exist") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_TOPICS, "Too many Topics") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TOPIC, "Invalid topic") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TOPIC_QUERY, "Topic with invalid query") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TOPIC_OPTION, "Topic with invalid option") TAOS_DEFINE_ERROR(TSDB_CODE_MND_CONSUMER_NOT_EXIST, "Consumer not exist") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_CONSUMER_NOT_READY, "Consumer waiting for rebalance") // mnode-sma TAOS_DEFINE_ERROR(TSDB_CODE_MND_SMA_ALREADY_EXIST, "SMA already exists") diff --git a/source/util/test/encodeTest.cpp b/source/util/test/encodeTest.cpp index 9ddbd24353..95ea6b1674 100644 --- a/source/util/test/encodeTest.cpp +++ b/source/util/test/encodeTest.cpp @@ -230,7 +230,7 @@ static int32_t tSStructA_v1_decode(SCoder *pCoder, SStructA_v1 *pSAV1) { const char *tstr; uint64_t len; if (tDecodeCStrAndLen(pCoder, &tstr, &len) < 0) return -1; - pSAV1->A_c = (char *)TCODER_MALLOC(pCoder, len + 1); + pSAV1->A_c = (char *)tCoderMalloc(pCoder, len + 1); memcpy(pSAV1->A_c, tstr, len + 1); tEndDecode(pCoder); @@ -269,7 +269,7 @@ static int32_t tSStructA_v2_decode(SCoder *pCoder, SStructA_v2 *pSAV2) { const char *tstr; uint64_t len; if (tDecodeCStrAndLen(pCoder, &tstr, &len) < 0) return -1; - pSAV2->A_c = (char *)TCODER_MALLOC(pCoder, len + 1); + pSAV2->A_c = (char *)tCoderMalloc(pCoder, len + 1); memcpy(pSAV2->A_c, tstr, len + 1); // ------------------------NEW FIELDS DECODE------------------------------- @@ -305,7 +305,7 @@ static int32_t tSFinalReq_v1_encode(SCoder *pCoder, const SFinalReq_v1 *ps1) { static int32_t tSFinalReq_v1_decode(SCoder *pCoder, SFinalReq_v1 *ps1) { if (tStartDecode(pCoder) < 0) return -1; - ps1->pA = (SStructA_v1 *)TCODER_MALLOC(pCoder, sizeof(*(ps1->pA))); + ps1->pA = (SStructA_v1 *)tCoderMalloc(pCoder, sizeof(*(ps1->pA))); if (tSStructA_v1_decode(pCoder, ps1->pA) < 0) return -1; if (tDecodeI32(pCoder, &ps1->v_a) < 0) return -1; if (tDecodeI8(pCoder, &ps1->v_b) < 0) return -1; @@ -339,7 +339,7 @@ static int32_t tSFinalReq_v2_encode(SCoder *pCoder, const SFinalReq_v2 *ps2) { static int32_t tSFinalReq_v2_decode(SCoder *pCoder, SFinalReq_v2 *ps2) { if (tStartDecode(pCoder) < 0) return -1; - ps2->pA = (SStructA_v2 *)TCODER_MALLOC(pCoder, sizeof(*(ps2->pA))); + ps2->pA = (SStructA_v2 *)tCoderMalloc(pCoder, sizeof(*(ps2->pA))); if (tSStructA_v2_decode(pCoder, ps2->pA) < 0) return -1; if (tDecodeI32(pCoder, &ps2->v_a) < 0) return -1; if (tDecodeI8(pCoder, &ps2->v_b) < 0) return -1; diff --git a/tests/pytest/util/dnodes.py b/tests/pytest/util/dnodes.py index c3c2424397..8a29a0a7a6 100644 --- a/tests/pytest/util/dnodes.py +++ b/tests/pytest/util/dnodes.py @@ -63,7 +63,7 @@ class TDSimClient: if os.system(cmd) != 0: tdLog.exit(cmd) - def deploy(self): + def deploy(self, *updatecfgDict): self.logDir = "%s/sim/psim/log" % (self.path) self.cfgDir = "%s/sim/psim/cfg" % (self.path) self.cfgPath = "%s/sim/psim/cfg/taos.cfg" % (self.path) @@ -95,6 +95,14 @@ class TDSimClient: for key, value in self.cfgDict.items(): self.cfg(key, value) + + try: + if bool(updatecfgDict) and updatecfgDict[0] and updatecfgDict[0][0]: + clientCfg = dict (updatecfgDict[0][0].get('clientCfg')) + for key, value in clientCfg.items(): + self.cfg(key, value) + except Exception: + pass tdLog.debug("psim is deployed and configured by %s" % (self.cfgPath)) @@ -214,9 +222,11 @@ class TDDnode: # self.cfg("logDir",self.logDir) # print(updatecfgDict) isFirstDir = 1 - if updatecfgDict[0] and updatecfgDict[0][0]: + if bool(updatecfgDict) and updatecfgDict[0] and updatecfgDict[0][0]: print(updatecfgDict[0][0]) for key, value in updatecfgDict[0][0].items(): + if key == "clientCfg": + continue if value == 'dataDir': if isFirstDir: self.cfgDict.pop('dataDir') @@ -491,7 +501,7 @@ class TDDnodes: self.sim.setTestCluster(self.testCluster) if (self.simDeployed == False): - self.sim.deploy() + self.sim.deploy(updatecfgDict) self.simDeployed = True self.check(index) diff --git a/tests/script/general/user/authority.sim b/tests/script/general/user/authority.sim deleted file mode 100644 index 45230e3e8a..0000000000 --- a/tests/script/general/user/authority.sim +++ /dev/null @@ -1,66 +0,0 @@ -system sh/stop_dnodes.sh -system sh/deploy.sh -n dnode1 -i 1 -system sh/exec.sh -n dnode1 -s start -sql connect -sleep 2000 - -print ============= step1 -sql create user read pass 'taosdata' -sql create user write pass 'taosdata' -sql show users -if $rows != 5 then - return -1 -endi - -print ============= step2 -sql close -sql connect read -sleep 2000 - -sql create database dread -sql show databases -if $rows != 1 then - return -1 -endi - -print ============= step3 -sql close -sql connect write -sleep 2000 - -sql create database dwrite -sql show databases -if $rows != 2 then - return -1 -endi - -print ============ step4 -sql close -sql connect -sleep 2000 - -sql show databases -if $row != 2 then - return -1 -endi - -print ============ step5 -sql close -sql connect read -sleep 2000 -sql drop database dread -sql drop database dwrite - - -sql close -sql connect -sql show databases -if $rows != 0 then - return -1 -endi - -sql close -sql connect -sleep 2000 - -system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/user/user_create.sim b/tests/script/general/user/user_create.sim deleted file mode 100644 index 34934d09e6..0000000000 --- a/tests/script/general/user/user_create.sim +++ /dev/null @@ -1,84 +0,0 @@ -system sh/stop_dnodes.sh - -system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c wallevel -v 0 -system sh/exec.sh -n dnode1 -s start -sql connect - -print =============== step1 -sql show users -if $rows != 3 then - return -1 -endi - -sql create user read PASS 'pass123' -sql create user read PASS 'pass123' -x step1 - return -1 -step1: - -sql show users -if $rows != 4 then - return -1 -endi - -sql alter user read PASS 'taosdata' - -print =============== step2 -sql close -sql connect read -sleep 2000 - -sql alter user read PASS 'taosdata' - -print =============== step3 -sql drop user read -x step31 - return -1 -step31: -sql drop user _root -x step32 - return -1 -step32: -sql drop user monitor -x step33 - return -1 -step33: - -print =============== step4 -sql close -sql connect -sleep 2000 - -sql alter user read privilege read -sql show users -print $data1_read -if $data1_read != readable then - return -1 -endi - -sql_error alter user read privilege super -sql show users -print $data1_read -if $data1_read != readable then - return -1 -endi - -sql alter user read privilege write -sql show users -print $data1_read -if $data1_read != writable then - return -1 -endi - -sql alter user read privilege 1 -x step43 - return -1 -step43: - -sql drop user _root -x step41 - return -1 -step41: - -sql drop user monitor -x step42 - return -1 -step42: - -sql drop user read - -system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 72ff6179db..ba4296f9bf 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -3,10 +3,13 @@ # ---- user ./test.sh -f tsim/user/basic1.sim +./test.sh -f tsim/user/pass_alter.sim +./test.sh -f tsim/user/pass_len.sim +./test.sh -f tsim/user/user_len.sim # ---- db ./test.sh -f tsim/db/create_all_options.sim -#./test.sh -f tsim/db/alter_option.sim +./test.sh -f tsim/db/alter_option.sim ./test.sh -f tsim/db/basic1.sim ./test.sh -f tsim/db/basic2.sim ./test.sh -f tsim/db/basic3.sim @@ -87,6 +90,6 @@ # ./test.sh -f tsim/sma/tsmaCreateInsertData.sim # --- valgrind -#./test.sh -f tsim/valgrind/checkError.sim -v +./test.sh -f tsim/valgrind/checkError.sim -v #======================b1-end=============== diff --git a/tests/script/tsim/db/alter_option.sim b/tests/script/tsim/db/alter_option.sim index 3b61b907de..1f0c2043e8 100644 --- a/tests/script/tsim/db/alter_option.sim +++ b/tests/script/tsim/db/alter_option.sim @@ -66,7 +66,7 @@ print ============= create database # | REPLICA value [1 | 3] # | WAL value [1 | 2] -sql create database db BLOCKS 7 CACHE 3 CACHELAST 3 COMP 0 DAYS 345600 FSYNC 1000 MAXROWS 8000 MINROWS 10 KEEP 1440000 PRECISION 'ns' REPLICA 3 TTL 7 WAL 2 VGROUPS 6 SINGLE_STABLE 1 STREAM_MODE 1 +sql create database db BLOCKS 7 CACHE 3 CACHELAST 3 COMP 0 DAYS 345600 FSYNC 1000 MAXROWS 8000 MINROWS 10 KEEP 1440000 PRECISION 'ns' REPLICA 1 TTL 7 WAL 2 VGROUPS 6 SINGLE_STABLE 1 STREAM_MODE 1 sql show databases print rows: $rows print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 @@ -86,7 +86,7 @@ endi if $data3_db != 0 then # ntables return -1 endi -if $data4_db != 3 then # replica +if $data4_db != 1 then # replica return -1 endi if $data5_db != nostrict then # strict @@ -127,47 +127,47 @@ if $data16_db != ns then # precision endi sleep 3000 -sql show db.vgroups -if $data[0][4] == LEADER then - if $data[0][6] != FOLLOWER then - return -1 - endi - if $data[0][8] != FOLLOWER then - return -1 - endi -endi -if $data[0][6] == LEADER then - if $data[0][4] != FOLLOWER then - return -1 - endi - if $data[0][8] != FOLLOWER then - return -1 - endi -endi -if $data[0][8] == LEADER then - if $data[0][4] != FOLLOWER then - return -1 - endi - if $data[0][6] != FOLLOWER then - return -1 - endi -endi - -if $data[0][4] != LEADER then - if $data[0][4] != FOLLOWER then - return -1 - endi -endi -if $data[0][6] != LEADER then - if $data[0][6] != FOLLOWER then - return -1 - endi -endi -if $data[0][8] != LEADER then - if $data[0][8] != FOLLOWER then - return -1 - endi -endi +#sql show db.vgroups +#if $data[0][4] == LEADER then +# if $data[0][6] != FOLLOWER then +# return -1 +# endi +# if $data[0][8] != FOLLOWER then +# return -1 +# endi +#endi +#if $data[0][6] == LEADER then +# if $data[0][4] != FOLLOWER then +# return -1 +# endi +# if $data[0][8] != FOLLOWER then +# return -1 +# endi +#endi +#if $data[0][8] == LEADER then +# if $data[0][4] != FOLLOWER then +# return -1 +# endi +# if $data[0][6] != FOLLOWER then +# return -1 +# endi +#endi +# +#if $data[0][4] != LEADER then +# if $data[0][4] != FOLLOWER then +# return -1 +# endi +#endi +#if $data[0][6] != LEADER then +# if $data[0][6] != FOLLOWER then +# return -1 +# endi +#endi +#if $data[0][8] != LEADER then +# if $data[0][8] != FOLLOWER then +# return -1 +# endi +#endi print ============== not support modify options: name, create_time, vgroups, ntables sql_error alter database db name dba diff --git a/tests/script/general/user/pass_alter.sim b/tests/script/tsim/user/pass_alter.sim similarity index 73% rename from tests/script/general/user/pass_alter.sim rename to tests/script/tsim/user/pass_alter.sim index 964e311ec0..db0667971c 100644 --- a/tests/script/general/user/pass_alter.sim +++ b/tests/script/tsim/user/pass_alter.sim @@ -1,8 +1,6 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c wallevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 2000 sql connect print ============= step1 @@ -13,7 +11,7 @@ sql alter user read pass 'taosdata' sql alter user write pass 'taosdata' sql show users -if $rows != 5 then +if $rows != 3 then return -1 endi @@ -27,11 +25,11 @@ sql alter user write pass 'taosdata1' -x step2 return -1 step2: -sql_error create user read pass 'taosdata1' -sql_error create user write pass 'taosdata1' +sql_error create user read1 pass 'taosdata1' +sql_error create user write1 pass 'taosdata1' sql show users -if $rows != 5 then +if $rows != 3 then return -1 endi @@ -41,27 +39,27 @@ sleep 2500 print user write login sql connect write -sql_error create user read pass 'taosdata1' -sql_error create user write pass 'taosdata1' +sql_error create user read2 pass 'taosdata1' +sql_error create user write2 pass 'taosdata1' sql alter user write pass 'taosdata' sql alter user read pass 'taosdata' -x step3 return -1 step3: sql show users -if $rows != 5 then +if $rows != 3 then return -1 endi print ============= step4 sql close sleep 2500 -print root write login +print user root login sql connect sql create user oroot pass 'taosdata' sql show users -if $rows != 6 then +if $rows != 4 then return -1 endi diff --git a/tests/script/general/user/pass_len.sim b/tests/script/tsim/user/pass_len.sim similarity index 85% rename from tests/script/general/user/pass_len.sim rename to tests/script/tsim/user/pass_len.sim index 649b3efa48..66c378c6cb 100644 --- a/tests/script/general/user/pass_len.sim +++ b/tests/script/tsim/user/pass_len.sim @@ -1,7 +1,5 @@ system sh/stop_dnodes.sh - system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c wallevel -v 0 system sh/exec.sh -n dnode1 -s start sql connect @@ -50,15 +48,16 @@ step3: sql create user $user PASS 'abc0123456789' sql show users -if $rows != 3 then +if $rows != 4 then return -1 endi print =============== step4 $i = 3 $user = $userPrefix . $i -sql create user $user PASS 'abcd012345678901234567891234567890' -x step4 +sql create user $user PASS 'abcd012345678901234567891234567890abcd012345678901234567891234567890abcd012345678901234567891234567890abcd012345678901234567891234567890123' -x step4 return -1 + step4: sql show users if $rows != 4 then diff --git a/tests/script/general/user/user_len.sim b/tests/script/tsim/user/user_len.sim similarity index 84% rename from tests/script/general/user/user_len.sim rename to tests/script/tsim/user/user_len.sim index 55e5eb19ef..0e44f94294 100644 --- a/tests/script/general/user/user_len.sim +++ b/tests/script/tsim/user/user_len.sim @@ -1,11 +1,6 @@ system sh/stop_dnodes.sh - - system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c wallevel -v 0 system sh/exec.sh -n dnode1 -s start - -sleep 2000 sql connect $i = 0 @@ -24,7 +19,7 @@ sql create user PASS '123' -x step1 step1: sql show users -if $rows != 3 then +if $rows != 1 then return -1 endi @@ -33,13 +28,13 @@ sql drop user a -x step2 step2: sql create user a PASS '123' sql show users -if $rows != 4 then +if $rows != 2 then return -1 endi sql drop user a sql show users -if $rows != 3 then +if $rows != 1 then return -1 endi @@ -49,13 +44,13 @@ step3: sql create user abc01234567890123456789 PASS '123' sql show users -if $rows != 4 then +if $rows != 2 then return -1 endi sql drop user abc01234567890123456789 sql show users -if $rows != 3 then +if $rows != 1 then return -1 endi @@ -64,7 +59,7 @@ sql create user abcd0123456789012345678901234567890111 PASS '123' -x step4 return -1 step4: sql show users -if $rows != 3 then +if $rows != 1 then return -1 endi @@ -77,13 +72,13 @@ step61: sql create user a123 PASS '123' sql show users -if $rows != 4 then +if $rows != 2 then return -1 endi sql drop user a123 sql show users -if $rows != 3 then +if $rows != 1 then return -1 endi diff --git a/tests/system-test/0-others/taosShell.py b/tests/system-test/0-others/taosShell.py new file mode 100644 index 0000000000..3c3508a5c3 --- /dev/null +++ b/tests/system-test/0-others/taosShell.py @@ -0,0 +1,324 @@ + +import taos +import sys +import time +import socket +import pexpect +import os + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * + +def taos_command (key, value, expectString, cfgDir, sqlString='', key1='', value1=''): + if len(key) == 0: + tdLog.exit("taos test key is null!") + + if len(cfgDir) != 0: + taosCmd = 'taos -c ' + cfgDir + ' -' + key + + if len(value) != 0: + if key == 'p': + taosCmd = taosCmd + value + else: + taosCmd = taosCmd + ' ' + value + + if len(key1) != 0: + taosCmd = taosCmd + ' -' + key1 + if key1 == 'p': + taosCmd = taosCmd + value1 + else: + if len(value1) != 0: + taosCmd = taosCmd + ' ' + value1 + + tdLog.info ("taos cmd: %s" % taosCmd) + + child = pexpect.spawn(taosCmd, timeout=3) + #output = child.readline() + #print (output.decode()) + if len(expectString) != 0: + i = child.expect([expectString, pexpect.TIMEOUT, pexpect.EOF], timeout=6) + else: + i = child.expect([pexpect.TIMEOUT, pexpect.EOF], timeout=6) + + retResult = child.before.decode() + print(retResult) + #print(child.after.decode()) + if i == 0: + print ('taos login success! Here can run sql, taos> ') + if len(sqlString) != 0: + child.sendline (sqlString) + w = child.expect(["Query OK", pexpect.TIMEOUT, pexpect.EOF], timeout=1) + if w == 0: + return "TAOS_OK" + else: + return "TAOS_FAIL" + else: + if key == 'A' or key1 == 'A' or key == 'C' or key1 == 'C': + return "TAOS_OK", retResult + else: + return "TAOS_OK" + else: + if key == 'A' or key1 == 'A' or key == 'C' or key1 == 'C': + return "TAOS_OK", retResult + else: + return "TAOS_FAIL" + +class TDTestCase: + #updatecfgDict = {'clientCfg': {'serverPort': 7080, 'firstEp': 'trd02:7080', 'secondEp':'trd02:7080'},\ + # 'serverPort': 7080, 'firstEp': 'trd02:7080'} + hostname = socket.gethostname() + serverPort = '7080' + clientCfgDict = {'serverPort': '', 'firstEp': '', 'secondEp':''} + clientCfgDict["serverPort"] = serverPort + clientCfgDict["firstEp"] = hostname + ':' + serverPort + clientCfgDict["secondEp"] = hostname + ':' + serverPort + + + updatecfgDict = {'clientCfg': {}, 'serverPort': '', 'firstEp': '', 'secondEp':''} + updatecfgDict["clientCfg"] = clientCfgDict + updatecfgDict["serverPort"] = serverPort + updatecfgDict["firstEp"] = hostname + ':' + serverPort + updatecfgDict["secondEp"] = hostname + ':' + serverPort + + print ("===================: ", updatecfgDict) + + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/build/bin")] + break + return buildPath + + def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring + tdSql.prepare() + # time.sleep(2) + tdSql.query("create user testpy pass 'testpy'") + + hostname = socket.gethostname() + tdLog.info ("hostname: %s" % hostname) + + buildPath = self.getBuildPath() + if (buildPath == ""): + tdLog.exit("taosd not found!") + else: + tdLog.info("taosd found in %s" % buildPath) + cfgPath = buildPath + "/../sim/psim/cfg" + tdLog.info("cfgPath: %s" % cfgPath) + + checkNetworkStatus = ['0: unavailable', '1: network ok', '2: service ok', '3: service degraded', '4: exiting'] + netrole = ['client', 'server'] + + keyDict = {'h':'', 'P':'6030', 'p':'testpy', 'u':'testpy', 'a':'', 'A':'', 'c':'', 'C':'', 's':'', 'r':'', 'f':'', \ + 'k':'', 't':'', 'n':'', 'l':'1024', 'N':'100', 'V':'', 'd':'db', 'w':'30', '-help':'', '-usage':'', '?':''} + + keyDict['h'] = hostname + keyDict['c'] = cfgPath + + tdLog.printNoPrefix("================================ parameter: -h") + newDbName="dbh" + sqlString = 'create database ' + newDbName + ';' + retCode = taos_command("h", keyDict['h'], "taos>", keyDict['c'], sqlString) + if retCode != "TAOS_OK": + tdLog.exit("taos -h %s fail"%keyDict['h']) + else: + #dataDbName = ["information_schema", "performance_schema", "db", newDbName] + tdSql.query("show databases") + #tdSql.getResult("show databases") + for i in range(tdSql.queryRows): + if tdSql.getData(i, 0) == newDbName: + break + else: + tdLog.exit("create db fail after taos -h %s fail"%keyDict['h']) + + tdSql.query('drop database %s'%newDbName) + + tdLog.printNoPrefix("================================ parameter: -P") + #tdDnodes.stop(1) + #sleep(3) + #tdDnodes.start(1) + #sleep(3) + #keyDict['P'] = 6030 + newDbName = "dbpp" + sqlString = 'create database ' + newDbName + ';' + retCode = taos_command("P", keyDict['P'], "taos>", keyDict['c'], sqlString) + if retCode != "TAOS_OK": + tdLog.exit("taos -P %s fail"%keyDict['P']) + else: + tdSql.query("show databases") + for i in range(tdSql.queryRows): + if tdSql.getData(i, 0) == newDbName: + break + else: + tdLog.exit("create db fail after taos -P %s fail"%keyDict['P']) + + tdSql.query('drop database %s'%newDbName) + + tdLog.printNoPrefix("================================ parameter: -u") + newDbName="dbu" + sqlString = 'create database ' + newDbName + ';' + retCode = taos_command("u", keyDict['u'], "taos>", keyDict['c'], sqlString, "p", keyDict['p']) + if retCode != "TAOS_OK": + tdLog.exit("taos -u %s -p%s fail"%(keyDict['u'], keyDict['p'])) + else: + tdSql.query("show databases") + for i in range(tdSql.queryRows): + if tdSql.getData(i, 0) == newDbName: + break + else: + tdLog.exit("create db fail after taos -u %s -p%s fail"%(keyDict['u'], keyDict['p'])) + + tdSql.query('drop database %s'%newDbName) + + tdLog.printNoPrefix("================================ parameter: -A") + newDbName="dbaa" + retCode, retVal = taos_command("p", keyDict['p'], "taos>", keyDict['c'], '', "A", '') + if retCode != "TAOS_OK": + tdLog.exit("taos -A fail") + + sqlString = 'create database ' + newDbName + ';' + retCode = taos_command("u", keyDict['u'], "taos>", keyDict['c'], sqlString, 'a', retVal) + if retCode != "TAOS_OK": + tdLog.exit("taos -u %s -a %s"%(keyDict['u'], retVal)) + + tdSql.query("show databases") + for i in range(tdSql.queryRows): + if tdSql.getData(i, 0) == newDbName: + break + else: + tdLog.exit("create db fail after taos -u %s -a %s fail"%(keyDict['u'], retVal)) + + tdSql.query('drop database %s'%newDbName) + + tdLog.printNoPrefix("================================ parameter: -s") + newDbName="dbss" + keyDict['s'] = "\"create database " + newDbName + "\"" + retCode = taos_command("s", keyDict['s'], "Query OK", keyDict['c'], '', '', '') + if retCode != "TAOS_OK": + tdLog.exit("taos -s fail") + + print ("========== check new db ==========") + tdSql.query("show databases") + for i in range(tdSql.queryRows): + if tdSql.getData(i, 0) == newDbName: + break + else: + tdLog.exit("create db fail after taos -s %s fail"%(keyDict['s'])) + + keyDict['s'] = "\"create table " + newDbName + ".stb (ts timestamp, c int) tags (t int)\"" + retCode = taos_command("s", keyDict['s'], "Query OK", keyDict['c'], '', '', '') + if retCode != "TAOS_OK": + tdLog.exit("taos -s create table fail") + + keyDict['s'] = "\"create table " + newDbName + ".ctb0 using " + newDbName + ".stb tags (0) " + newDbName + ".ctb1 using " + newDbName + ".stb tags (1)\"" + retCode = taos_command("s", keyDict['s'], "Query OK", keyDict['c'], '', '', '') + if retCode != "TAOS_OK": + tdLog.exit("taos -s create table fail") + + keyDict['s'] = "\"insert into " + newDbName + ".ctb0 values('2021-04-01 08:00:00.000', 10)('2021-04-01 08:00:01.000', 20) " + newDbName + ".ctb1 values('2021-04-01 08:00:00.000', 11)('2021-04-01 08:00:01.000', 21)\"" + retCode = taos_command("s", keyDict['s'], "Query OK", keyDict['c'], '', '', '') + if retCode != "TAOS_OK": + tdLog.exit("taos -s insert data fail") + + sqlString = "select * from " + newDbName + ".ctb0" + tdSql.query(sqlString) + tdSql.checkData(0, 0, '2021-04-01 08:00:00.000') + tdSql.checkData(0, 1, 10) + tdSql.checkData(1, 0, '2021-04-01 08:00:01.000') + tdSql.checkData(1, 1, 20) + sqlString = "select * from " + newDbName + ".ctb1" + tdSql.query(sqlString) + tdSql.checkData(0, 0, '2021-04-01 08:00:00.000') + tdSql.checkData(0, 1, 11) + tdSql.checkData(1, 0, '2021-04-01 08:00:01.000') + tdSql.checkData(1, 1, 21) + + keyDict['s'] = "\"select * from " + newDbName + ".ctb0\"" + retCode = taos_command("s", keyDict['s'], "2021-04-01 08:00:01.000", keyDict['c'], '', '', '') + if retCode != "TAOS_OK": + tdLog.exit("taos -r show fail") + + tdLog.printNoPrefix("================================ parameter: -r") + keyDict['s'] = "\"select * from " + newDbName + ".ctb0\"" + retCode = taos_command("s", keyDict['s'], "1617235200000", keyDict['c'], '', 'r', '') + if retCode != "TAOS_OK": + tdLog.exit("taos -r show fail") + + keyDict['s'] = "\"select * from " + newDbName + ".ctb1\"" + retCode = taos_command("s", keyDict['s'], "1617235201000", keyDict['c'], '', 'r', '') + if retCode != "TAOS_OK": + tdLog.exit("taos -r show fail") + + tdSql.query('drop database %s'%newDbName) + + tdLog.printNoPrefix("================================ parameter: -f") + pwd=os.getcwd() + newDbName="dbf" + sqlFile = pwd + "/0-others/sql.txt" + sql1 = "echo 'create database " + newDbName + "' > " + sqlFile + sql2 = "echo 'use " + newDbName + "' >> " + sqlFile + sql3 = "echo 'create table ntbf (ts timestamp, c binary(40))' >> " + sqlFile + sql4 = "echo 'insert into ntbf values (\"2021-04-01 08:00:00.000\", \"test taos -f1\")(\"2021-04-01 08:00:01.000\", \"test taos -f2\")' >> " + sqlFile + sql5 = "echo 'show databases' >> " + sqlFile + os.system(sql1) + os.system(sql2) + os.system(sql3) + os.system(sql4) + os.system(sql5) + + keyDict['f'] = pwd + "/0-others/sql.txt" + retCode = taos_command("f", keyDict['f'], 'performance_schema', keyDict['c'], '', '', '') + print("============ ret code: ", retCode) + if retCode != "TAOS_OK": + tdLog.exit("taos -s fail") + + print ("========== check new db ==========") + tdSql.query("show databases") + for i in range(tdSql.queryRows): + #print ("dbseq: %d, dbname: %s"%(i, tdSql.getData(i, 0))) + if tdSql.getData(i, 0) == newDbName: + break + else: + tdLog.exit("create db fail after taos -f fail") + + sqlString = "select * from " + newDbName + ".ntbf" + tdSql.query(sqlString) + tdSql.checkData(0, 0, '2021-04-01 08:00:00.000') + tdSql.checkData(0, 1, 'test taos -f1') + tdSql.checkData(1, 0, '2021-04-01 08:00:01.000') + tdSql.checkData(1, 1, 'test taos -f2') + + shellCmd = "rm -f " + sqlFile + os.system(shellCmd) + tdSql.query('drop database %s'%newDbName) + + tdLog.printNoPrefix("================================ parameter: -C") + newDbName="dbcc" + retCode, retVal = taos_command("C", keyDict['C'], "buildinfo", keyDict['c'], '', '', '') + if retCode != "TAOS_OK": + tdLog.exit("taos -C fail") + + print ("-C return content:\n ", retVal) + + + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/2-query/Today.py b/tests/system-test/2-query/Today.py index ad4e48bda5..09f018dc11 100644 --- a/tests/system-test/2-query/Today.py +++ b/tests/system-test/2-query/Today.py @@ -8,6 +8,7 @@ from util.cases import * class TDTestCase: + updatecfgDict = {'rpcDebugFlag': '143'} def init(self, conn, logSql): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) diff --git a/tools/shell/inc/shellInt.h b/tools/shell/inc/shellInt.h index d218bbf373..382009905c 100644 --- a/tools/shell/inc/shellInt.h +++ b/tools/shell/inc/shellInt.h @@ -111,6 +111,5 @@ void shellTestNetWork(); // shellMain.c extern SShellObj shell; -extern void taos_init(); #endif /*_TD_SHELL_INT_H_*/ diff --git a/tools/shell/src/shellMain.c b/tools/shell/src/shellMain.c index d277920877..6672cee367 100644 --- a/tools/shell/src/shellMain.c +++ b/tools/shell/src/shellMain.c @@ -42,6 +42,8 @@ int main(int argc, char *argv[]) { return 0; } + taos_init(); + if (shell.args.is_dump_config) { shellDumpConfig(); taos_cleanup();