diff --git a/.gitignore b/.gitignore index c651127992..2308ea7896 100644 --- a/.gitignore +++ b/.gitignore @@ -101,4 +101,4 @@ TAGS contrib/* !contrib/CMakeLists.txt -!contrib/test +!contrib/test \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt index b626977588..1a8cb1d710 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -10,7 +10,7 @@ set(CMAKE_SUPPORT_DIR "${CMAKE_SOURCE_DIR}/cmake") set(CMAKE_CONTRIB_DIR "${CMAKE_SOURCE_DIR}/contrib") include(${CMAKE_SUPPORT_DIR}/cmake.options) -SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fPIC -gdwarf-2 -msse4.2 -mfma") +SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fPIC -gdwarf-2 -msse4.2 -mfma -g3") # contrib add_subdirectory(contrib) @@ -20,9 +20,13 @@ add_library(api INTERFACE) target_include_directories(api INTERFACE "include/client") # src +if(${BUILD_TEST}) + include(CTest) + enable_testing() +endif(${BUILD_TEST}) add_subdirectory(source) # docs add_subdirectory(docs) -# tests (TODO) +# tests (TODO) \ No newline at end of file diff --git a/include/client/taos.h b/include/client/taos.h index 0f7edc9fed..19d191b84e 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -73,10 +73,12 @@ typedef struct taosField { #define DLL_EXPORT #endif -DLL_EXPORT int taos_init(); +typedef void (*__taos_async_fn_t)(void *param, TAOS_RES *, int code); + DLL_EXPORT void taos_cleanup(void); DLL_EXPORT int taos_options(TSDB_OPTION option, const void *arg, ...); DLL_EXPORT TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port); +DLL_EXPORT TAOS *taos_connect_l(const char *ip, int ipLen, const char *user, int userLen, const char *pass, int passLen, const char *db, int dbLen, uint16_t port); DLL_EXPORT TAOS *taos_connect_auth(const char *ip, const char *user, const char *auth, const char *db, uint16_t port); DLL_EXPORT void taos_close(TAOS *taos); @@ -154,14 +156,14 @@ DLL_EXPORT int* taos_fetch_lengths(TAOS_RES *res); // TAOS_RES *taos_list_dbs(TAOS *mysql, const char *wild); // TODO: the return value should be `const` -DLL_EXPORT char *taos_get_server_info(TAOS *taos); -DLL_EXPORT char *taos_get_client_info(); -DLL_EXPORT char *taos_errstr(TAOS_RES *tres); +DLL_EXPORT const char *taos_get_server_info(TAOS *taos); +DLL_EXPORT const char *taos_get_client_info(); +DLL_EXPORT const char *taos_errstr(TAOS_RES *tres); DLL_EXPORT int taos_errno(TAOS_RES *tres); -DLL_EXPORT void taos_query_a(TAOS *taos, const char *sql, void (*fp)(void *param, TAOS_RES *, int code), void *param); -DLL_EXPORT void taos_fetch_rows_a(TAOS_RES *res, void (*fp)(void *param, TAOS_RES *, int numOfRows), void *param); +DLL_EXPORT void taos_query_a(TAOS *taos, const char *sql, __taos_async_fn_t fp, void *param); +DLL_EXPORT void taos_fetch_rows_a(TAOS_RES *res, __taos_async_fn_t fp, void *param); typedef void (*TAOS_SUBSCRIBE_CALLBACK)(TAOS_SUB* tsub, TAOS_RES *res, void* param, int code); DLL_EXPORT TAOS_SUB *taos_subscribe(TAOS* taos, int restart, const char* topic, const char *sql, TAOS_SUBSCRIBE_CALLBACK fp, void *param, int interval); diff --git a/include/common/taosmsg.h b/include/common/taosmsg.h index 6f25d0fb87..a75c4a7bac 100644 --- a/include/common/taosmsg.h +++ b/include/common/taosmsg.h @@ -77,7 +77,7 @@ TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DROP_FUNCTION, "drop-function" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CREATE_STB, "create-stb" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_ALTER_STB, "alter-stb" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DROP_STB, "drop-stb" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_STB_VGROUP, "stb-vgroup" ) +TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_VGROUP_LIST, "vgroup-list" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_KILL_QUERY, "kill-query" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_KILL_STREAM, "kill-stream" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_KILL_CONN, "kill-conn" ) @@ -214,6 +214,11 @@ typedef enum _mgmt_table { extern char *taosMsg[]; +typedef struct SBuildTableMetaInput { + int32_t vgId; + char *tableFullName; +} SBuildTableMetaInput; + #pragma pack(push, 1) // null-terminated string instead of char array to avoid too many memory consumption in case of more than 1M tableMeta @@ -358,6 +363,7 @@ typedef struct { int32_t pid; char app[TSDB_APP_NAME_LEN]; char db[TSDB_DB_NAME_LEN]; + int64_t startTime; } SConnectMsg; typedef struct SEpSet { @@ -368,19 +374,17 @@ typedef struct SEpSet { } SEpSet; typedef struct { - int32_t acctId; - int32_t clusterId; - int32_t connId; - int8_t superAuth; - int8_t readAuth; - int8_t writeAuth; - int8_t reserved[5]; - SEpSet epSet; + int32_t acctId; + uint32_t clusterId; + int32_t connId; + int8_t superUser; + int8_t reserved[5]; + SEpSet epSet; } SConnectRsp; typedef struct { char user[TSDB_USER_LEN]; - char pass[TSDB_KEY_LEN]; + char pass[TSDB_PASSWORD_LEN]; int32_t maxUsers; int32_t maxDbs; int32_t maxTimeSeries; @@ -395,7 +399,7 @@ typedef struct { typedef struct { char user[TSDB_USER_LEN]; - char pass[TSDB_KEY_LEN]; + char pass[TSDB_PASSWORD_LEN]; } SCreateUserMsg, SAlterUserMsg; typedef struct { @@ -774,9 +778,8 @@ typedef struct { } SStbInfoMsg; typedef struct { + SMsgHead msgHead; char tableFname[TSDB_TABLE_FNAME_LEN]; - int8_t createFlag; - char tags[]; } STableInfoMsg; typedef struct { @@ -791,6 +794,20 @@ typedef struct SSTableVgroupMsg { int32_t numOfTables; } SSTableVgroupMsg, SSTableVgroupRspMsg; +typedef struct SVgroupInfo { + int32_t vgId; + int8_t numOfEps; + SEpAddrMsg epAddr[TSDB_MAX_REPLICA]; +} SVgroupInfo; + +typedef struct SVgroupListRspMsg { + int32_t vgroupNum; + int32_t vgroupVersion; + SVgroupInfo vgroupInfo[]; +} SVgroupListRspMsg; + +typedef SVgroupListRspMsg SVgroupListInfo; + typedef struct { int32_t vgId; int8_t numOfEps; @@ -961,8 +978,8 @@ typedef struct { char user[TSDB_USER_LEN]; char spi; char encrypt; - char secret[TSDB_KEY_LEN]; - char ckey[TSDB_KEY_LEN]; + char secret[TSDB_PASSWORD_LEN]; + char ckey[TSDB_PASSWORD_LEN]; } SAuthMsg, SAuthRsp; typedef struct { diff --git a/include/common/tep.h b/include/common/tep.h new file mode 100644 index 0000000000..2addc11e5a --- /dev/null +++ b/include/common/tep.h @@ -0,0 +1,17 @@ +#ifndef TDENGINE_TEP_H +#define TDENGINE_TEP_H + +#include "os.h" +#include "taosmsg.h" + +typedef struct SCorEpSet { + int32_t version; + SEpSet epSet; +} SCorEpSet; + +int taosGetFqdnPortFromEp(const char *ep, char *fqdn, uint16_t *port); +bool isEpsetEqual(const SEpSet *s1, const SEpSet *s2); + +void updateEpSet_s(SCorEpSet *pEpSet, SEpSet *pNewEpSet); + +#endif // TDENGINE_TEP_H diff --git a/include/common/tglobal.h b/include/common/tglobal.h index aedf4122bb..a9b0e60761 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -81,8 +81,6 @@ extern int64_t tsMaxRetentWindow; // db parameters in client extern int32_t tsCacheBlockSize; extern int32_t tsBlocksPerVnode; -extern int32_t tsMinTablePerVnode; -extern int32_t tsMaxTablePerVnode; extern int32_t tsTableIncStepPerVnode; extern int32_t tsMaxVgroupsPerDb; extern int16_t tsDaysPerFile; @@ -113,16 +111,8 @@ extern int8_t tsEnableSlaveQuery; extern int8_t tsEnableAdjustMaster; // restful -extern int8_t tsEnableHttpModule; extern int32_t tsRestRowLimit; -extern uint16_t tsHttpPort; -extern int32_t tsHttpCacheSessions; -extern int32_t tsHttpSessionExpire; -extern int32_t tsHttpMaxThreads; -extern int8_t tsHttpEnableCompress; -extern int8_t tsHttpEnableRecordSql; extern int8_t tsTelegrafUseFieldNum; -extern int8_t tsHttpDbNameMandatory; // mqtt extern int8_t tsEnableMqttModule; @@ -144,7 +134,6 @@ extern int8_t tsEnableStream; // internal extern int8_t tsPrintAuth; -extern int8_t tscEmbedded; extern char tsVnodeDir[]; extern char tsMnodeDir[]; extern int64_t tsTickPerDay[3]; @@ -193,7 +182,7 @@ extern SDiskCfg tsDiskCfg[]; #define NEEDTO_COMPRESSS_MSG(size) (tsCompressMsgSize != -1 && (size) > tsCompressMsgSize) void taosInitGlobalCfg(); -int32_t taosCheckGlobalCfg(); +int32_t taosCheckAndPrintCfg(); int32_t taosCfgDynamicOptions(char *msg); bool taosCheckBalanceCfgOptions(const char *option, int32_t *vnodeId, int32_t *dnodeId); void taosAddDataDir(int index, char *v1, int level, int primary); diff --git a/include/dnode/vnode/tq/tq.h b/include/dnode/vnode/tq/tq.h index 85533f65bc..7359df92cc 100644 --- a/include/dnode/vnode/tq/tq.h +++ b/include/dnode/vnode/tq/tq.h @@ -256,7 +256,7 @@ typedef struct STQ { // the collection of group handle // the handle of kvstore char* path; - STqCfg* tqConfig; + STqCfg* tqConfig; TqLogReader* tqLogReader; TqMemRef tqMemRef; TqMetaStore* tqMeta; diff --git a/include/dnode/vnode/vnode.h b/include/dnode/vnode/vnode.h index 30531ad738..8458ad9da3 100644 --- a/include/dnode/vnode/vnode.h +++ b/include/dnode/vnode/vnode.h @@ -36,6 +36,8 @@ typedef struct SVnodeCfg { struct { /** write buffer size */ uint64_t wsize; + uint64_t ssize; + uint64_t lsize; /** use heap allocator or arena allocator */ bool isHeapAllocator; }; @@ -66,9 +68,11 @@ typedef struct SVnodeCfg { /** * @brief Initialize the vnode module * + * @param nthreads number of commit threads. 0 for no threads and + * a schedule queue should be given (TODO) * @return int 0 for success and -1 for failure */ -int vnodeInit(); +int vnodeInit(uint16_t nthreads); /** * @brief clear a vnode diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index 050b9c904f..0b45f71557 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -30,16 +30,23 @@ extern "C" { struct SCatalog; -typedef struct SMetaReq { - char clusterId[TSDB_CLUSTER_ID_LEN]; +typedef struct SDBVgroupInfo { + int32_t vgroupVersion; + SArray *vgId; + int32_t hashRange; + int32_t hashNum; +} SDBVgroupInfo; + +typedef struct SCatalogReq { + char clusterId[TSDB_CLUSTER_ID_LEN]; //???? SArray *pTableName; // table full name SArray *pUdf; // udf name - bool qNodeEpset; // valid qnode -} SMetaReq; + bool qNodeRequired; // valid qnode +} SCatalogReq; typedef struct SMetaData { - SArray *pTableMeta; // tableMeta - SArray *pVgroupInfo; // vgroupInfo list + SArray *pTableMeta; // STableMeta array + SArray *pVgroupInfo; // SVgroupInfo list SArray *pUdfList; // udf info list SEpSet *pEpSet; // qnode epset list } SMetaData; @@ -78,32 +85,71 @@ typedef struct STableMeta { SSchema schema[]; } STableMeta; +typedef struct SCatalogCfg { + +} SCatalogCfg; + +int32_t catalogInit(SCatalogCfg *cfg); + /** * Catalog service object, which is utilized to hold tableMeta (meta/vgroupInfo/udfInfo) at the client-side. * There is ONLY one SCatalog object for one process space, and this function returns a singleton. - * @param pMgmtEps + * @param clusterId * @return */ -struct SCatalog* getCatalogHandle(const SEpSet* pMgmtEps); +int32_t catalogGetHandle(const char *clusterId, struct SCatalog** catalogHandle); + + + +int32_t catalogGetVgroupVersion(struct SCatalog* pCatalog, int32_t* version); +int32_t catalogGetVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, SArray** pVgroupList); +int32_t catalogUpdateVgroup(struct SCatalog* pCatalog, SVgroupListInfo* pVgroup); + + + +int32_t catalogGetDBVgroupVersion(struct SCatalog* pCatalog, const char* dbName, int32_t* version); +int32_t catalogGetDBVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* dbName, int32_t forceUpdate, SDBVgroupInfo** dbInfo); +int32_t catalogUpdateDBVgroup(struct SCatalog* pCatalog, const char* dbName, SDBVgroupInfo* dbInfo); + + +int32_t catalogGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pTableName, STableMeta* pTableMeta); +int32_t catalogRenewTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const STableMeta* pTableMeta); +int32_t catalogRenewAndGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const STableMeta* pTableMeta, STableMeta* pNewTableMeta); + + +/** + * get table's vgroup list. + * @param clusterId + * @pVgroupList - array of SVgroupInfo + * @return + */ +int32_t catalogGetTableVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pTableName, SArray* pVgroupList); + /** * Get the required meta data from mnode. * Note that this is a synchronized API and is also thread-safety. * @param pCatalog + * @param pMgmtEps * @param pMetaReq * @param pMetaData * @return */ -int32_t catalogGetMetaData(struct SCatalog* pCatalog, const SMetaReq* pMetaReq, SMetaData* pMetaData); +int32_t catalogGetAllMeta(struct SCatalog* pCatalog, const SEpSet* pMgmtEps, const SCatalogReq* pReq, SMetaData* pRsp); + + +int32_t catalogGetQnodeList(struct SCatalog* pCatalog, const SEpSet* pMgmtEps, SEpSet* pQnodeEpSet); + + /** - * Destroy catalog service handle + * Destroy catalog and relase all resources * @param pCatalog */ -void destroyCatalog(struct SCatalog* pCatalog); +void catalogDestroy(void); #ifdef __cplusplus } #endif -#endif /*_TD_CATALOG_H_*/ \ No newline at end of file +#endif /*_TD_CATALOG_H_*/ diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index e2817baeae..2f152c3e2b 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -162,7 +162,7 @@ typedef struct SVgDataBlocks { int64_t vgId; // virtual group id int32_t numOfTables; // number of tables in current submit block uint32_t size; - char *pData; + char *pData; // SMsgDesc + SSubmitMsg + SSubmitBlk + ... } SVgDataBlocks; typedef struct SInsertStmtInfo { diff --git a/src/tsdb/inc/tsdbCommitQueue.h b/include/libs/query/query.h similarity index 64% rename from src/tsdb/inc/tsdbCommitQueue.h rename to include/libs/query/query.h index b690e3bdc2..866a69ed8e 100644 --- a/src/tsdb/inc/tsdbCommitQueue.h +++ b/include/libs/query/query.h @@ -13,11 +13,20 @@ * along with this program. If not, see . */ -#ifndef _TD_TSDB_COMMIT_QUEUE_H_ -#define _TD_TSDB_COMMIT_QUEUE_H_ +#ifndef _TD_QUERY_H_ +#define _TD_QUERY_H_ -typedef enum { COMMIT_REQ, COMPACT_REQ,COMMIT_CONFIG_REQ } TSDB_REQ_T; +#ifdef __cplusplus +extern "C" { +#endif -int tsdbScheduleCommit(STsdbRepo *pRepo, TSDB_REQ_T req); -#endif /* _TD_TSDB_COMMIT_QUEUE_H_ */ \ No newline at end of file +extern int32_t (*queryBuildMsg[TSDB_MSG_TYPE_MAX])(void* input, char **msg, int32_t msgSize, int32_t *msgLen); +extern int32_t (*queryProcessMsgRsp[TSDB_MSG_TYPE_MAX])(void* output, char *msg, int32_t msgSize); + + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_QUERY_H_*/ diff --git a/include/libs/transport/trpc.h b/include/libs/transport/trpc.h index 2a8a7aad4b..2afd9d48b7 100644 --- a/include/libs/transport/trpc.h +++ b/include/libs/transport/trpc.h @@ -29,11 +29,6 @@ extern "C" { extern int tsRpcHeadSize; -typedef struct SRpcCorEpSet { - int32_t version; - SEpSet epSet; -} SRpcCorEpSet; - typedef struct SRpcConnInfo { uint32_t clientIp; uint16_t clientPort; diff --git a/include/libs/wal/wal.h b/include/libs/wal/wal.h index fe3ee17d00..744275e6ff 100644 --- a/include/libs/wal/wal.h +++ b/include/libs/wal/wal.h @@ -32,6 +32,23 @@ extern int32_t wDebugFlag; #define wDebug(...) { if (wDebugFlag & DEBUG_DEBUG) { taosPrintLog("WAL ", wDebugFlag, __VA_ARGS__); }} #define wTrace(...) { if (wDebugFlag & DEBUG_TRACE) { taosPrintLog("WAL ", wDebugFlag, __VA_ARGS__); }} +#define WAL_PREFIX "wal" +#define WAL_PREFIX_LEN 3 +#define WAL_NOSUFFIX_LEN 20 +#define WAL_SUFFIX_AT (WAL_NOSUFFIX_LEN+1) +#define WAL_LOG_SUFFIX "log" +#define WAL_INDEX_SUFFIX "idx" +#define WAL_REFRESH_MS 1000 +#define WAL_MAX_SIZE (TSDB_MAX_WAL_SIZE + sizeof(SWalHead) + 16) +#define WAL_PATH_LEN (TSDB_FILENAME_LEN + 12) +#define WAL_FILE_LEN (WAL_PATH_LEN + 32) + +#define WAL_IDX_ENTRY_SIZE (sizeof(int64_t)*2) +#define WAL_CUR_POS_WRITABLE 1 +#define WAL_CUR_FILE_WRITABLE 2 +#define WAL_CUR_FAILED 4 + +#pragma pack(push,1) typedef enum { TAOS_WAL_NOLOG = 0, TAOS_WAL_WRITE = 1, @@ -43,8 +60,9 @@ typedef struct SWalReadHead { uint8_t msgType; int8_t reserved[2]; int32_t len; + //int64_t ingestTs; //not implemented int64_t version; - char cont[]; + char body[]; } SWalReadHead; typedef struct { @@ -52,8 +70,9 @@ typedef struct { int32_t fsyncPeriod; // millisecond int32_t retentionPeriod; // secs int32_t rollPeriod; // secs + int64_t retentionSize; int64_t segSize; - EWalType walLevel; // wal level + EWalType level; // wal level } SWalCfg; typedef struct { @@ -70,64 +89,48 @@ typedef struct { SWalReadHead head; } SWalHead; -#define WAL_PREFIX "wal" -#define WAL_PREFIX_LEN 3 -#define WAL_NOSUFFIX_LEN 20 -#define WAL_SUFFIX_AT (WAL_NOSUFFIX_LEN+1) -#define WAL_LOG_SUFFIX "log" -#define WAL_INDEX_SUFFIX "idx" -#define WAL_REFRESH_MS 1000 -#define WAL_MAX_SIZE (TSDB_MAX_WAL_SIZE + sizeof(SWalHead) + 16) -#define WAL_SIGNATURE ((uint32_t)(0xFAFBFDFEUL)) -#define WAL_PATH_LEN (TSDB_FILENAME_LEN + 12) -#define WAL_FILE_LEN (WAL_PATH_LEN + 32) -//#define WAL_FILE_NUM 1 // 3 -#define WAL_FILESET_MAX 128 - -#define WAL_IDX_ENTRY_SIZE (sizeof(int64_t)*2) -#define WAL_CUR_POS_WRITABLE 1 -#define WAL_CUR_FILE_WRITABLE 2 -#define WAL_CUR_FAILED 4 +typedef struct SWalVer { + int64_t firstVer; + int64_t verInSnapshotting; + int64_t snapshotVer; + int64_t commitVer; + int64_t lastVer; +} SWalVer; typedef struct SWal { // cfg - int32_t vgId; - int32_t fsyncPeriod; // millisecond - int32_t rollPeriod; // second - int64_t segSize; - int64_t retentionSize; - int32_t retentionPeriod; - EWalType level; - //total size - int64_t totSize; - //fsync seq - int32_t fsyncSeq; - //reference - int64_t refId; - //write tfd - int64_t writeLogTfd; - int64_t writeIdxTfd; - //wal lifecycle - int64_t firstVersion; - int64_t snapshotVersion; - int64_t commitVersion; - int64_t lastVersion; - //snapshotting version - int64_t snapshottingVer; - //roll status - int64_t lastRollSeq; + SWalCfg cfg; + SWalVer vers; //file set int32_t writeCur; + int64_t writeLogTfd; + int64_t writeIdxTfd; SArray* fileInfoSet; //ctl int32_t curStatus; + int32_t fsyncSeq; + int64_t totSize; + int64_t refId; + int64_t lastRollSeq; pthread_mutex_t mutex; //path char path[WAL_PATH_LEN]; //reusable write head - SWalHead head; + SWalHead writeHead; } SWal; // WAL HANDLE +typedef struct SWalReadHandle { + SWal* pWal; + int64_t readLogTfd; + int64_t readIdxTfd; + int64_t curFileFirstVer; + int64_t curVersion; + int64_t capacity; + int64_t status; //if cursor valid + SWalHead* pHead; +} SWalReadHandle; +#pragma pack(pop) + typedef int32_t (*FWalWrite)(void *ahandle, void *pHead); // module initialization @@ -153,6 +156,10 @@ int32_t walEndTakeSnapshot(SWal *); //int32_t walDataCorrupted(SWal*); // read +SWalReadHandle* walOpenReadHandle(SWal *); +void walCloseReadHandle(SWalReadHandle *); +int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver); + int32_t walRead(SWal *, SWalHead **, int64_t ver); int32_t walReadWithFp(SWal *, FWalWrite writeFp, int64_t verStart, int32_t readNum); diff --git a/include/os/osSemaphore.h b/include/os/osSemaphore.h index 86a9f10819..78112fc7a0 100644 --- a/include/os/osSemaphore.h +++ b/include/os/osSemaphore.h @@ -57,7 +57,7 @@ int64_t taosGetPthreadId(pthread_t thread); void taosResetPthread(pthread_t* thread); bool taosComparePthread(pthread_t first, pthread_t second); int32_t taosGetPId(); -int32_t taosGetCurrentAPPName(char* name, int32_t* len); +int32_t taosGetAppName(char* name, int32_t* len); #ifdef __cplusplus } diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 64a1fa84ad..8400031f9b 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -117,6 +117,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_TSC_INVALID_JSON TAOS_DEF_ERROR_CODE(0, 0x0221) //"Invalid JSON format") #define TSDB_CODE_TSC_INVALID_JSON_TYPE TAOS_DEF_ERROR_CODE(0, 0x0222) //"Invalid JSON data type") #define TSDB_CODE_TSC_VALUE_OUT_OF_RANGE TAOS_DEF_ERROR_CODE(0, 0x0223) //"Value out of range") +#define TSDB_CODE_TSC_INVALID_INPUT TAOS_DEF_ERROR_CODE(0, 0X0224) //"Invalid tsc input") // mnode #define TSDB_CODE_MND_MSG_NOT_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0300) @@ -501,6 +502,13 @@ int32_t* taosGetErrno(); // monitor #define TSDB_CODE_MON_CONNECTION_INVALID TAOS_DEF_ERROR_CODE(0, 0x2300) //"monitor invalid monitor db connection") +// catalog +#define TSDB_CODE_CTG_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x2400) //catalog interval error +#define TSDB_CODE_CTG_INVALID_INPUT TAOS_DEF_ERROR_CODE(0, 0x2401) //invalid catalog input parameters +#define TSDB_CODE_CTG_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x2402) //catalog is not ready +#define TSDB_CODE_CTG_MEM_ERROR TAOS_DEF_ERROR_CODE(0, 0x2403) //catalog memory error +#define TSDB_CODE_CTG_SYS_ERROR TAOS_DEF_ERROR_CODE(0, 0x2404) //catalog system error + #ifdef __cplusplus } #endif diff --git a/include/util/tarray.h b/include/util/tarray.h index ad02b20868..e0f14dcd25 100644 --- a/include/util/tarray.h +++ b/include/util/tarray.h @@ -43,6 +43,13 @@ typedef struct SArray { */ void* taosArrayInit(size_t size, size_t elemSize); +/** + * + * @param tsize + * @return + */ +int32_t taosArrayEnsureCap(SArray* pArray, size_t tsize); + /** * * @param pArray diff --git a/include/util/tconfig.h b/include/util/tconfig.h index 15100423d5..7dee01247b 100644 --- a/include/util/tconfig.h +++ b/include/util/tconfig.h @@ -20,7 +20,7 @@ extern "C" { #endif -#define TSDB_CFG_MAX_NUM 123 +#define TSDB_CFG_MAX_NUM 115 #define TSDB_CFG_PRINT_LEN 23 #define TSDB_CFG_OPTION_LEN 24 #define TSDB_CFG_VALUE_LEN 41 @@ -83,11 +83,11 @@ extern int32_t tsGlobalConfigNum; extern char * tsCfgStatusStr[]; void taosReadGlobalLogCfg(); -int32_t taosReadGlobalCfg(); -void taosPrintGlobalCfg(); +int32_t taosReadCfgFromFile(); +void taosPrintCfg(); void taosDumpGlobalCfg(); -void taosInitConfigOption(SGlobalCfg cfg); +void taosAddConfigOption(SGlobalCfg cfg); SGlobalCfg *taosGetConfigOption(const char *option); #ifdef __cplusplus diff --git a/include/util/tdef.h b/include/util/tdef.h index 28a1e24581..300b190e97 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -161,7 +161,7 @@ do { \ #define TSDB_NODE_NAME_LEN 64 #define TSDB_TABLE_NAME_LEN 193 // it is a null-terminated string -#define TSDB_DB_NAME_LEN 33 +#define TSDB_DB_NAME_LEN 65 #define TSDB_FULL_DB_NAME_LEN (TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN) #define TSDB_FUNC_NAME_LEN 65 @@ -193,7 +193,7 @@ do { \ #define TSDB_MAX_TAG_CONDITIONS 1024 #define TSDB_AUTH_LEN 16 -#define TSDB_KEY_LEN 16 +#define TSDB_PASSWORD_LEN 32 #define TSDB_VERSION_LEN 12 #define TSDB_LABEL_LEN 8 diff --git a/include/util/tdlist.h b/include/util/tdlist.h new file mode 100644 index 0000000000..d047a57770 --- /dev/null +++ b/include/util/tdlist.h @@ -0,0 +1,169 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_UTIL_TDLIST_H_ +#define _TD_UTIL_TDLIST_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +// Single linked list +#define TD_SLIST_NODE(TYPE) \ + struct { \ + struct TYPE *sl_next_; \ + } + +#define TD_SLIST(TYPE) \ + struct { \ + struct TYPE *sl_head_; \ + int sl_neles_; \ + } + +#define TD_SLIST_HEAD(sl) ((sl)->sl_head_) +#define TD_SLIST_NELES(sl) ((sl)->sl_neles_) +#define TD_SLIST_NODE_NEXT(sln) ((sln)->sl_next_) + +#define tSListInit(sl) \ + do { \ + (sl)->sl_head_ = NULL; \ + (sl)->sl_neles_ = 0; \ + } while (0) + +#define tSListPush(sl, sln) \ + do { \ + TD_SLIST_NODE_NEXT(sln) = TD_SLIST_HEAD(sl); \ + TD_SLIST_HEAD(sl) = (sln); \ + TD_SLIST_NELES(sl) += 1; \ + } while (0) + +#define tSListPop(sl) \ + do { \ + TD_SLIST_HEAD(sl) = TD_SLIST_NODE_NEXT(TD_SLIST_HEAD(sl)); \ + TD_SLIST_NELES(sl) -= 1; \ + } while (0) + +// Double linked list +#define TD_DLIST_NODE(TYPE) \ + struct { \ + struct TYPE *dl_prev_; \ + struct TYPE *dl_next_; \ + } + +#define TD_DLIST(TYPE) \ + struct { \ + struct TYPE *dl_head_; \ + struct TYPE *dl_tail_; \ + int dl_neles_; \ + } + +#define TD_DLIST_NODE_PREV(dln) ((dln)->dl_prev_) +#define TD_DLIST_NODE_NEXT(dln) ((dln)->dl_next_) +#define TD_DLIST_HEAD(dl) ((dl)->dl_head_) +#define TD_DLIST_TAIL(dl) ((dl)->dl_tail_) +#define TD_DLIST_NELES(dl) ((dl)->dl_neles_) + +#define tDListInit(dl) \ + do { \ + TD_DLIST_HEAD(dl) = TD_DLIST_TAIL(dl) = NULL; \ + TD_DLIST_NELES(dl) = 0; \ + } while (0) + +#define tDListAppend(dl, dln) \ + do { \ + if (TD_DLIST_HEAD(dl) == NULL) { \ + TD_DLIST_NODE_PREV(dln) = TD_DLIST_NODE_NEXT(dln) = NULL; \ + TD_DLIST_HEAD(dl) = TD_DLIST_TAIL(dl) = (dln); \ + } else { \ + TD_DLIST_NODE_PREV(dln) = TD_DLIST_TAIL(dl); \ + TD_DLIST_NODE_NEXT(dln) = NULL; \ + TD_DLIST_NODE_NEXT(TD_DLIST_TAIL(dl)) = (dln); \ + TD_DLIST_TAIL(dl) = (dln); \ + } \ + TD_DLIST_NELES(dl) += 1; \ + } while (0) + +#define tDListPrepend(dl, dln) \ + do { \ + if (TD_DLIST_HEAD(dl) == NULL) { \ + TD_DLIST_NODE_PREV(dln) = TD_DLIST_NODE_NEXT(dln) = NULL; \ + TD_DLIST_HEAD(dl) = TD_DLIST_TAIL(dl) = (dln); \ + } else { \ + TD_DLIST_NODE_PREV(dln) = NULL; \ + TD_DLIST_NODE_NEXT(dln) = TD_DLIST_HEAD(dl); \ + TD_DLIST_NODE_PREV(TD_DLIST_HEAD(dl)) = (dln); \ + TD_DLIST_HEAD(dl) = (dln); \ + } \ + TD_DLIST_NELES(dl) += 1; \ + } while (0) + +#define tDListPop(dl, dln) \ + do { \ + if (TD_DLIST_HEAD(dl) == (dln)) { \ + TD_DLIST_HEAD(dl) = TD_DLIST_NODE_NEXT(dln); \ + } \ + if (TD_DLIST_TAIL(dl) == (dln)) { \ + TD_DLIST_TAIL(dl) = TD_DLIST_NODE_PREV(dln); \ + } \ + if (TD_DLIST_NODE_PREV(dln) != NULL) { \ + TD_DLIST_NODE_NEXT(TD_DLIST_NODE_PREV(dln)) = TD_DLIST_NODE_NEXT(dln); \ + } \ + if (TD_DLIST_NODE_NEXT(dln) != NULL) { \ + TD_DLIST_NODE_PREV(TD_DLIST_NODE_NEXT(dln)) = TD_DLIST_NODE_PREV(dln); \ + } \ + TD_DLIST_NELES(dl) -= 1; \ + TD_DLIST_NODE_PREV(dln) = TD_DLIST_NODE_NEXT(dln) = NULL; \ + } while (0) + +#if 0 +// List iterator +#define TD_LIST_FITER 0 +#define TD_LIST_BITER 1 +#define TD_LIST_ITER(S) \ + struct { \ + int it_dir_; \ + S * it_next_; \ + S * it_ptr_; \ + TD_DLIST(S) * it_list_; \ + } + +#define tlistIterInit(it, l, dir) \ + (it)->it_dir_ = (dir); \ + (it)->it_list_ = l; \ + if ((dir) == TD_LIST_FITER) { \ + (it)->it_next_ = (l)->dl_head_; \ + } else { \ + (it)->it_next_ = (l)->dl_tail_; \ + } + +#define tlistIterNext(it) \ + ({ \ + (it)->it_ptr_ = (it)->it_next_; \ + if ((it)->it_next_ != NULL) { \ + if ((it)->it_dir_ == TD_LIST_FITER) { \ + (it)->it_next_ = (it)->it_next_->next_; \ + } else { \ + (it)->it_next_ = (it)->it_next_->prev_; \ + } \ + } \ + (it)->it_ptr_; \ + }) +#endif + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_UTIL_TDLIST_H_*/ \ No newline at end of file diff --git a/include/util/tlog.h b/include/util/tlog.h index 2ee60e4324..9b6033e7fe 100644 --- a/include/util/tlog.h +++ b/include/util/tlog.h @@ -45,6 +45,8 @@ extern int32_t sDebugFlag; extern int32_t tsdbDebugFlag; extern int32_t cqDebugFlag; extern int32_t debugFlag; +extern int32_t ctgDebugFlag; + #define DEBUG_FATAL 1U #define DEBUG_ERROR DEBUG_FATAL diff --git a/include/util/tmacro.h b/include/util/tmacro.h new file mode 100644 index 0000000000..5cca8a1062 --- /dev/null +++ b/include/util/tmacro.h @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_UTIL_MACRO_H_ +#define _TD_UTIL_MACRO_H_ + +#include "os.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// Module init/clear MACRO definitions +#define TD_MOD_UNINITIALIZED 0 +#define TD_MOD_INITIALIZED 1 + +#define TD_MOD_UNCLEARD 0 +#define TD_MOD_CLEARD 1 + +typedef int8_t td_mode_flag_t; + +#define TD_CHECK_AND_SET_MODE_INIT(FLAG) atomic_val_compare_exchange_8((FLAG), TD_MOD_UNINITIALIZED, TD_MOD_INITIALIZED) + +#define TD_CHECK_AND_SET_MOD_CLEAR(FLAG) atomic_val_compare_exchange_8((FLAG), TD_MOD_UNCLEARD, TD_MOD_CLEARD) + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_UTIL_MACRO_H_*/ \ No newline at end of file diff --git a/include/util/tutil.h b/include/util/tutil.h index 3a07b898c0..c315948702 100644 --- a/include/util/tutil.h +++ b/include/util/tutil.h @@ -45,14 +45,25 @@ char *taosIpStr(uint32_t ipInt); uint32_t ip2uint(const char *const ip_addr); void taosIp2String(uint32_t ip, char *str); void taosIpPort2String(uint32_t ip, uint16_t port, char *str); -int32_t taosGetFqdnPortFromEp(const char *ep, char *fqdn, uint16_t *port); static FORCE_INLINE void taosEncryptPass(uint8_t *inBuf, size_t inLen, char *target) { T_MD5_CTX context; tMD5Init(&context); tMD5Update(&context, inBuf, (unsigned int)inLen); tMD5Final(&context); - memcpy(target, context.digest, TSDB_KEY_LEN); + memcpy(target, context.digest, tListLen(context.digest)); +} + +static FORCE_INLINE void taosEncryptPass_c(uint8_t *inBuf, size_t len, char *target) { + T_MD5_CTX context; + tMD5Init(&context); + tMD5Update(&context, inBuf, (unsigned int)len); + tMD5Final(&context); + + sprintf(target, "%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x", context.digest[0], context.digest[1], context.digest[2], + context.digest[3], context.digest[4], context.digest[5], context.digest[6], context.digest[7], + context.digest[8], context.digest[9], context.digest[10], context.digest[11], context.digest[12], + context.digest[13], context.digest[14], context.digest[15]); } #ifdef __cplusplus diff --git a/source/client/CMakeLists.txt b/source/client/CMakeLists.txt index bc0d439407..b46afa31b9 100644 --- a/source/client/CMakeLists.txt +++ b/source/client/CMakeLists.txt @@ -2,9 +2,14 @@ aux_source_directory(src CLIENT_SRC) add_library(taos ${CLIENT_SRC}) target_include_directories( taos - PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" + PUBLIC "${CMAKE_SOURCE_DIR}/include/client" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) target_link_libraries( taos + PRIVATE common INTERFACE api + PRIVATE os util common transport parser ) + +ADD_SUBDIRECTORY(test) \ No newline at end of file diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h new file mode 100644 index 0000000000..3180923aff --- /dev/null +++ b/source/client/inc/clientInt.h @@ -0,0 +1,146 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_CLIENTINT_H +#define TDENGINE_CLIENTINT_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include "taos.h" +#include "taosmsg.h" +#include "thash.h" +#include "tlist.h" +#include "trpc.h" +#include "tdef.h" +#include "tmsgtype.h" +#include "tep.h" + +typedef struct SQueryExecMetric { + int64_t start; // start timestamp + int64_t parsed; // start to parse + int64_t send; // start to send to server + int64_t rsp; // receive response from server +} SQueryExecMetric; + +typedef struct SInstanceActivity { + uint64_t numOfInsertsReq; + uint64_t numOfInsertRows; + uint64_t insertElapsedTime; + uint64_t insertBytes; // submit to tsdb since launched. + + uint64_t fetchBytes; + uint64_t queryElapsedTime; + uint64_t numOfSlowQueries; + uint64_t totalRequests; + uint64_t currentRequests; // the number of SRequestObj +} SInstanceActivity; + +typedef struct SHeartBeatInfo { + void *pTimer; // timer, used to send request msg to mnode +} SHeartBeatInfo; + +typedef struct SAppInstInfo { + int64_t numOfConns; + SCorEpSet mgmtEp; + SInstanceActivity summary; + SList *pConnList; // STscObj linked list + uint32_t clusterId; + void *pTransporter; +} SAppInstInfo; + +typedef struct SAppInfo { + int64_t startTime; + char appName[TSDB_APP_NAME_LEN]; + char *ep; + int32_t pid; + int32_t numOfThreads; + SHeartBeatInfo hb; + SHashObj *pInstMap; +} SAppInfo; + +typedef struct STscObj { + char user[TSDB_USER_LEN]; + char pass[TSDB_PASSWORD_LEN]; + char acctId[TSDB_ACCT_ID_LEN]; + char db[TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN]; + uint32_t connId; + uint64_t id; // ref ID returned by taosAddRef +// struct SSqlObj *sqlList; + void *pTransporter; + pthread_mutex_t mutex; // used to protect the operation on db + int32_t numOfReqs; // number of sqlObj from this tscObj + SAppInstInfo *pAppInfo; +} STscObj; + +typedef struct SReqBody { + tsem_t rspSem; // not used now + void* fp; + void* param; +} SRequestBody; + +typedef struct SRequestObj { + uint64_t requestId; + int32_t type; // request type + STscObj *pTscObj; + SQueryExecMetric metric; + char *sqlstr; // sql string + SRequestBody body; + int64_t self; + char *msgBuf; + int32_t code; + void *pInfo; // sql parse info, generated by parser module +} SRequestObj; + +typedef struct SRequestMsgBody { + int32_t msgType; + void *pData; + int32_t msgLen; + uint64_t requestId; + uint64_t requestObjRefId; +} SRequestMsgBody; + +extern SAppInfo appInfo; +extern int32_t tscReqRef; +extern void *tscQhandle; +extern int32_t tscConnRef; + +extern int (*tscBuildMsg[TSDB_SQL_MAX])(SRequestObj *pRequest, SRequestMsgBody *pMsg); +extern int (*handleRequestRspFp[TSDB_SQL_MAX])(SRequestObj *pRequest, const char* pMsg, int32_t msgLen); + +int taos_init(); + +void* createTscObj(const char* user, const char* auth, const char *ip, uint32_t port, SAppInstInfo* pAppInfo); +void destroyTscObj(void*pObj); + +void* createRequest(STscObj* pObj, __taos_async_fn_t fp, void* param, int32_t type); +void destroyRequest(SRequestObj* pRequest); + +TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass, const char *auth, const char *db, uint16_t port); + +void taos_init_imp(void); +int taos_options_imp(TSDB_OPTION option, const char *str); + +void* openTransporter(const char *user, const char *auth); + +void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet); +void initMsgHandleFp(); + +#ifdef __cplusplus +} +#endif + +#endif // TDENGINE_CLIENTINT_H diff --git a/source/client/inc/tscLog.h b/source/client/inc/tscLog.h new file mode 100644 index 0000000000..f205a50227 --- /dev/null +++ b/source/client/inc/tscLog.h @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_TSCLOG_H +#define TDENGINE_TSCLOG_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include "tlog.h" + +#define tscFatal(...) do { if (cDebugFlag & DEBUG_FATAL) { taosPrintLog("TSC FATAL ", cDebugFlag, __VA_ARGS__); }} while(0) +#define tscError(...) do { if (cDebugFlag & DEBUG_ERROR) { taosPrintLog("TSC ERROR ", cDebugFlag, __VA_ARGS__); }} while(0) +#define tscWarn(...) do { if (cDebugFlag & DEBUG_WARN) { taosPrintLog("TSC WARN ", cDebugFlag, __VA_ARGS__); }} while(0) +#define tscInfo(...) do { if (cDebugFlag & DEBUG_INFO) { taosPrintLog("TSC ", cDebugFlag, __VA_ARGS__); }} while(0) +#define tscDebug(...) do { if (cDebugFlag & DEBUG_DEBUG) { taosPrintLog("TSC ", cDebugFlag, __VA_ARGS__); }} while(0) +#define tscTrace(...) do { if (cDebugFlag & DEBUG_TRACE) { taosPrintLog("TSC ", cDebugFlag, __VA_ARGS__); }} while(0) +#define tscDebugL(...) do { if (cDebugFlag & DEBUG_DEBUG) { taosPrintLongString("TSC ", cDebugFlag, __VA_ARGS__); }} while(0) + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/source/client/src/client.c b/source/client/src/client.c index b1663239e6..863e96aa74 100644 --- a/source/client/src/client.c +++ b/source/client/src/client.c @@ -13,11 +13,51 @@ * along with this program. If not, see . */ -//#include "taos.h" +#include "os.h" +#include "tdef.h" +#include "tglobal.h" +#include "clientInt.h" +#include "tscLog.h" -//TAOS_RES *taos_query(TAOS *taos, const char *sql) { -// -//} +TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port) { + int32_t p = (port != 0)? port:tsServerPort; + + tscDebug("try to connect to %s:%u, user:%s db:%s", ip, p, user, db); + if (user == NULL) { + user = TSDB_DEFAULT_USER; + } + + if (pass == NULL) { + pass = TSDB_DEFAULT_PASS; + } + + return taos_connect_internal(ip, user, pass, NULL, db, p); +} + +TAOS *taos_connect_auth(const char *ip, const char *user, const char *auth, const char *db, uint16_t port) { + tscDebug("try to connect to %s:%u by auth, user:%s db:%s", ip, port, user, db); + if (user == NULL) { + user = TSDB_DEFAULT_USER; + } + + if (auth == NULL) { + tscError("No auth info is given, failed to connect to server"); + return NULL; + } + + return taos_connect_internal(ip, user, NULL, auth, db, port); +} + +TAOS *taos_connect_l(const char *ip, int ipLen, const char *user, int userLen, const char *pass, int passLen, const char *db, int dbLen, uint16_t port) { + char ipStr[TSDB_EP_LEN] = {0}; + char dbStr[TSDB_DB_NAME_LEN] = {0}; + char userStr[TSDB_USER_LEN] = {0}; + char passStr[TSDB_PASSWORD_LEN] = {0}; + + strncpy(ipStr, ip, MIN(TSDB_EP_LEN - 1, ipLen)); + strncpy(userStr, user, MIN(TSDB_USER_LEN - 1, userLen)); + strncpy(passStr, pass, MIN(TSDB_PASSWORD_LEN - 1, passLen)); + strncpy(dbStr, db, MIN(TSDB_DB_NAME_LEN - 1, dbLen)); + return taos_connect(ipStr, userStr, passStr, dbStr, port); +} -int taos_init() { return 0; } -void taos_cleanup(void) {} diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c new file mode 100644 index 0000000000..2ae79f1947 --- /dev/null +++ b/source/client/src/clientImpl.c @@ -0,0 +1,283 @@ +#include +#include "clientInt.h" +#include "tdef.h" +#include "tep.h" +#include "tglobal.h" +#include "tmsgtype.h" +#include "tref.h" +#include "tscLog.h" + +static int32_t initEpSetFromCfg(const char *firstEp, const char *secondEp, SCorEpSet *pEpSet); +static int32_t buildConnectMsg(SRequestObj *pRequest, SRequestMsgBody* pMsgBody); +static void destroyConnectMsg(SRequestMsgBody* pMsgBody); + +static int32_t sendMsgToServer(void *pTransporter, SEpSet* epSet, const SRequestMsgBody *pBody, int64_t* pTransporterId); + +static bool stringLengthCheck(const char* str, size_t maxsize) { + if (str == NULL) { + return false; + } + + size_t len = strlen(str); + if (len <= 0 || len > maxsize) { + return false; + } + + return true; +} + +static bool validateUserName(const char* user) { + return stringLengthCheck(user, TSDB_USER_LEN - 1); +} + +static bool validatePassword(const char* passwd) { + return stringLengthCheck(passwd, TSDB_PASSWORD_LEN - 1); +} + +static bool validateDbName(const char* db) { + return stringLengthCheck(db, TSDB_DB_NAME_LEN - 1); +} + +static char* getClusterKey(const char* user, const char* auth, const char* ip, int32_t port) { + char key[512] = {0}; + snprintf(key, sizeof(key), "%s:%s:%s:%d", user, auth, ip, port); + return strdup(key); +} + +static STscObj* taosConnectImpl(const char *ip, const char *user, const char *auth, const char *db, uint16_t port, __taos_async_fn_t fp, void *param, SAppInstInfo* pAppInfo); + +TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass, const char *auth, const char *db, uint16_t port) { + if (taos_init() != TSDB_CODE_SUCCESS) { + return NULL; + } + + if (!validateUserName(user)) { + terrno = TSDB_CODE_TSC_INVALID_USER_LENGTH; + return NULL; + } + + char tmp[TSDB_DB_NAME_LEN] = {0}; + if (db != NULL) { + if(!validateDbName(db)) { + terrno = TSDB_CODE_TSC_INVALID_DB_LENGTH; + return NULL; + } + + tstrncpy(tmp, db, sizeof(tmp)); + strdequote(tmp); + } + + char secretEncrypt[32] = {0}; + if (auth == NULL) { + if (!validatePassword(pass)) { + terrno = TSDB_CODE_TSC_INVALID_PASS_LENGTH; + return NULL; + } + + taosEncryptPass_c((uint8_t *)pass, strlen(pass), secretEncrypt); + } else { + tstrncpy(secretEncrypt, auth, tListLen(secretEncrypt)); + } + + SCorEpSet epSet = {0}; + if (ip) { + if (initEpSetFromCfg(ip, NULL, &epSet) < 0) { + return NULL; + } + + if (port) { + epSet.epSet.port[0] = port; + } + } else { + if (initEpSetFromCfg(tsFirst, tsSecond, &epSet) < 0) { + return NULL; + } + } + + char* key = getClusterKey(user, secretEncrypt, ip, port); + + SAppInstInfo* pInst = taosHashGet(appInfo.pInstMap, key, strlen(key)); + if (pInst == NULL) { + pInst = calloc(1, sizeof(struct SAppInstInfo)); + + pInst->mgmtEp = epSet; + pInst->pTransporter = openTransporter(user, secretEncrypt); + + taosHashPut(appInfo.pInstMap, key, strlen(key), &pInst, POINTER_BYTES); + } + + return taosConnectImpl(ip, user, &secretEncrypt[0], db, port, NULL, NULL, pInst); +} + +int initEpSetFromCfg(const char *firstEp, const char *secondEp, SCorEpSet *pEpSet) { + pEpSet->version = 0; + + // init mgmt ip set + SEpSet *mgmtEpSet = &(pEpSet->epSet); + mgmtEpSet->numOfEps = 0; + mgmtEpSet->inUse = 0; + + if (firstEp && firstEp[0] != 0) { + if (strlen(firstEp) >= TSDB_EP_LEN) { + terrno = TSDB_CODE_TSC_INVALID_FQDN; + return -1; + } + + taosGetFqdnPortFromEp(firstEp, mgmtEpSet->fqdn[0], &(mgmtEpSet->port[0])); + mgmtEpSet->numOfEps++; + } + + if (secondEp && secondEp[0] != 0) { + if (strlen(secondEp) >= TSDB_EP_LEN) { + terrno = TSDB_CODE_TSC_INVALID_FQDN; + return -1; + } + + taosGetFqdnPortFromEp(secondEp, mgmtEpSet->fqdn[mgmtEpSet->numOfEps], &(mgmtEpSet->port[mgmtEpSet->numOfEps])); + mgmtEpSet->numOfEps++; + } + + if (mgmtEpSet->numOfEps == 0) { + terrno = TSDB_CODE_TSC_INVALID_FQDN; + return -1; + } + + return 0; +} + +STscObj* taosConnectImpl(const char *ip, const char *user, const char *auth, const char *db, uint16_t port, __taos_async_fn_t fp, void *param, SAppInstInfo* pAppInfo) { + STscObj *pTscObj = createTscObj(user, auth, ip, port, pAppInfo); + if (NULL == pTscObj) { + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + return pTscObj; + } + + SRequestObj *pRequest = createRequest(pTscObj, fp, param, TSDB_SQL_CONNECT); + if (pRequest == NULL) { + destroyTscObj(pTscObj); + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + return NULL; + } + + SRequestMsgBody body = {0}; + buildConnectMsg(pRequest, &body); + + int64_t transporterId = 0; + sendMsgToServer(pTscObj->pTransporter, &pTscObj->pAppInfo->mgmtEp.epSet, &body, &transporterId); + + tsem_wait(&pRequest->body.rspSem); + destroyConnectMsg(&body); + + if (pRequest->code != TSDB_CODE_SUCCESS) { + const char *errorMsg = (pRequest->code == TSDB_CODE_RPC_FQDN_ERROR) ? taos_errstr(pRequest) : tstrerror(terrno); + printf("failed to connect to server, reason: %s\n\n", errorMsg); + + destroyRequest(pRequest); + taos_close(pTscObj); + pTscObj = NULL; + } else { + tscDebug("0x%"PRIx64" connection is opening, connId:%d, dnodeConn:%p", pTscObj->id, pTscObj->connId, pTscObj->pTransporter); + destroyRequest(pRequest); + } + + return pTscObj; +} + +static int32_t buildConnectMsg(SRequestObj *pRequest, SRequestMsgBody* pMsgBody) { + pMsgBody->msgType = TSDB_MSG_TYPE_CONNECT; + pMsgBody->msgLen = sizeof(SConnectMsg); + pMsgBody->requestObjRefId = pRequest->self; + + SConnectMsg *pConnect = calloc(1, sizeof(SConnectMsg)); + if (pConnect == NULL) { + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + return -1; + } + + // TODO refactor full_name + char *db; // ugly code to move the space + + STscObj *pObj = pRequest->pTscObj; + pthread_mutex_lock(&pObj->mutex); + db = strstr(pObj->db, TS_PATH_DELIMITER); + + db = (db == NULL) ? pObj->db : db + 1; + tstrncpy(pConnect->db, db, sizeof(pConnect->db)); + pthread_mutex_unlock(&pObj->mutex); + + pConnect->pid = htonl(appInfo.pid); + pConnect->startTime = htobe64(appInfo.startTime); + tstrncpy(pConnect->app, appInfo.appName, tListLen(pConnect->app)); + + pMsgBody->pData = pConnect; + return 0; +} + +static void destroyConnectMsg(SRequestMsgBody* pMsgBody) { + assert(pMsgBody != NULL); + tfree(pMsgBody->pData); +} + +int32_t sendMsgToServer(void *pTransporter, SEpSet* epSet, const SRequestMsgBody *pBody, int64_t* pTransporterId) { + char *pMsg = rpcMallocCont(pBody->msgLen); + if (NULL == pMsg) { + tscError("0x%"PRIx64" msg:%s malloc failed", pBody->requestId, taosMsg[pBody->msgType]); + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + return -1; + } + + memcpy(pMsg, pBody->pData, pBody->msgLen); + SRpcMsg rpcMsg = { + .msgType = pBody->msgType, + .pCont = pMsg, + .contLen = pBody->msgLen, + .ahandle = (void*) pBody->requestObjRefId, + .handle = NULL, + .code = 0 + }; + + rpcSendRequest(pTransporter, epSet, &rpcMsg, pTransporterId); + return TSDB_CODE_SUCCESS; +} + +void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) { + int64_t requestRefId = (int64_t)pMsg->ahandle; + + SRequestObj *pRequest = (SRequestObj *)taosAcquireRef(tscReqRef, requestRefId); + if (pRequest == NULL) { + rpcFreeCont(pMsg->pCont); + return; + } + + assert(pRequest->self == requestRefId); + pRequest->metric.rsp = taosGetTimestampMs(); + + pRequest->code = pMsg->code; + + STscObj *pTscObj = pRequest->pTscObj; + if (pEpSet) { + if (!isEpsetEqual(&pTscObj->pAppInfo->mgmtEp.epSet, pEpSet)) { + updateEpSet_s(&pTscObj->pAppInfo->mgmtEp, pEpSet); + } + } + + /* + * There is not response callback function for submit response. + * The actual inserted number of points is the first number. + */ + if (pMsg->code == TSDB_CODE_SUCCESS) { + tscDebug("0x%" PRIx64 " message:%s, code:%s rspLen:%d, elapsed:%"PRId64 " ms", pRequest->requestId, taosMsg[pMsg->msgType], + tstrerror(pMsg->code), pMsg->contLen, pRequest->metric.rsp - pRequest->metric.start); + if (handleRequestRspFp[pRequest->type]) { + pMsg->code = (*handleRequestRspFp[pRequest->type])(pRequest, pMsg->pCont, pMsg->contLen); + } + } else { + tscError("0x%" PRIx64 " SQL cmd:%s, code:%s rspLen:%d", pRequest->requestId, taosMsg[pMsg->msgType], + tstrerror(pMsg->code), pMsg->contLen); + } + + taosReleaseRef(tscReqRef, requestRefId); + rpcFreeCont(pMsg->pCont); + + sem_post(&pRequest->body.rspSem); +} diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c new file mode 100644 index 0000000000..62466a096d --- /dev/null +++ b/source/client/src/clientMsgHandler.c @@ -0,0 +1,3212 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "os.h" +#include "clientInt.h" +#include "tmsgtype.h" +#include "trpc.h" +#include "tscLog.h" + +int (*buildRequestMsgFp[TSDB_SQL_MAX])(SRequestObj *pRequest, SRequestMsgBody *pMsgBody) = {0}; +int (*handleRequestRspFp[TSDB_SQL_MAX])(SRequestObj *pRequest, const char* pMsg, int32_t msgLen); + +#if 0 +void tscProcessActivityTimer(void *handle, void *tmrId); + +static int32_t extractSTableQueryVgroupId(STableMetaInfo* pTableMetaInfo); + +static int32_t minMsgSize() { return tsRpcHeadSize + 100; } +static int32_t getWaitingTimeInterval(int32_t count) { + int32_t initial = 100; // 100 ms by default + if (count <= 1) { + return 0; + } + + return initial * ((2u)<<(count - 2)); +} + +static int32_t vgIdCompare(const void *lhs, const void *rhs) { + int32_t left = *(int32_t *)lhs; + int32_t right = *(int32_t *)rhs; + + if (left == right) { + return 0; + } else { + return left > right ? 1 : -1; + } +} +static int32_t removeDupVgid(int32_t *src, int32_t sz) { + if (src == NULL || sz <= 0) { + return 0; + } + qsort(src, sz, sizeof(src[0]), vgIdCompare); + + int32_t ret = 1; + for (int i = 1; i < sz; i++) { + if (src[i] != src[i - 1]) { + src[ret++] = src[i]; + } + } + return ret; +} + +static void tscSetDnodeEpSet(SRpcEpSet* pEpSet, SVgroupMsg* pVgroupInfo) { + assert(pEpSet != NULL && pVgroupInfo != NULL && pVgroupInfo->numOfEps > 0); + + // Issue the query to one of the vnode among a vgroup randomly. + // change the inUse property would not affect the isUse attribute of STableMeta + pEpSet->inUse = rand() % pVgroupInfo->numOfEps; + + // apply the FQDN string length check here + bool existed = false; + + pEpSet->numOfEps = pVgroupInfo->numOfEps; + for(int32_t i = 0; i < pVgroupInfo->numOfEps; ++i) { + pEpSet->port[i] = pVgroupInfo->epAddr[i].port; + + int32_t len = (int32_t) strnlen(pVgroupInfo->epAddr[i].fqdn, TSDB_FQDN_LEN); + if (len > 0) { + tstrncpy(pEpSet->fqdn[i], pVgroupInfo->epAddr[i].fqdn, tListLen(pEpSet->fqdn[i])); + existed = true; + } + } + + assert(existed); +} + +static void tscDumpMgmtEpSet(SSqlObj *pSql) { + SCorEpSet *pCorEpSet = pSql->pTscObj->tscCorMgmtEpSet; + taosCorBeginRead(&pCorEpSet->version); + pSql->epSet = pCorEpSet->epSet; + taosCorEndRead(&pCorEpSet->version); +} +static void tscEpSetHtons(SRpcEpSet *s) { + for (int32_t i = 0; i < s->numOfEps; i++) { + s->port[i] = htons(s->port[i]); + } +} + +bool tscEpSetIsEqual(SRpcEpSet *s1, SRpcEpSet *s2) { + if (s1->numOfEps != s2->numOfEps || s1->inUse != s2->inUse) { + return false; + } + + for (int32_t i = 0; i < s1->numOfEps; i++) { + if (s1->port[i] != s2->port[i] + || strncmp(s1->fqdn[i], s2->fqdn[i], TSDB_FQDN_LEN) != 0) + return false; + } + return true; +} + +void tscUpdateMgmtEpSet(SSqlObj *pSql, SRpcEpSet *pEpSet) { + // no need to update if equal + SCorEpSet *pCorEpSet = pSql->pTscObj->tscCorMgmtEpSet; + taosCorBeginWrite(&pCorEpSet->version); + pCorEpSet->epSet = *pEpSet; + taosCorEndWrite(&pCorEpSet->version); +} + +static void tscDumpEpSetFromVgroupInfo(SRpcEpSet *pEpSet, SNewVgroupInfo *pVgroupInfo) { + if (pVgroupInfo == NULL) { return;} + int8_t inUse = pVgroupInfo->inUse; + pEpSet->inUse = (inUse >= 0 && inUse < TSDB_MAX_REPLICA) ? inUse: 0; + pEpSet->numOfEps = pVgroupInfo->numOfEps; + for (int32_t i = 0; i < pVgroupInfo->numOfEps; ++i) { + tstrncpy(pEpSet->fqdn[i], pVgroupInfo->ep[i].fqdn, sizeof(pEpSet->fqdn[i])); + pEpSet->port[i] = pVgroupInfo->ep[i].port; + } +} + +static void tscUpdateVgroupInfo(SSqlObj *pSql, SRpcEpSet *pEpSet) { + SSqlCmd *pCmd = &pSql->cmd; + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); + if (pTableMetaInfo == NULL || pTableMetaInfo->pTableMeta == NULL) { + return; + } + + int32_t vgId = -1; + if (pTableMetaInfo->pTableMeta->tableType == TSDB_SUPER_TABLE) { + vgId = extractSTableQueryVgroupId(pTableMetaInfo); + } else { + vgId = pTableMetaInfo->pTableMeta->vgId; + } + + assert(vgId > 0); + + SNewVgroupInfo vgroupInfo = {.vgId = -1}; + taosHashGetClone(tscVgroupMap, &vgId, sizeof(vgId), NULL, &vgroupInfo); + assert(vgroupInfo.numOfEps > 0 && vgroupInfo.vgId > 0); + + tscDebug("before: Endpoint in use:%d, numOfEps:%d", vgroupInfo.inUse, vgroupInfo.numOfEps); + vgroupInfo.inUse = pEpSet->inUse; + vgroupInfo.numOfEps = pEpSet->numOfEps; + for (int32_t i = 0; i < vgroupInfo.numOfEps; i++) { + tstrncpy(vgroupInfo.ep[i].fqdn, pEpSet->fqdn[i], TSDB_FQDN_LEN); + vgroupInfo.ep[i].port = pEpSet->port[i]; + } + + tscDebug("after: EndPoint in use:%d, numOfEps:%d", vgroupInfo.inUse, vgroupInfo.numOfEps); + taosHashPut(tscVgroupMap, &vgId, sizeof(vgId), &vgroupInfo, sizeof(SNewVgroupInfo)); + + // Update the local cached epSet info cached by SqlObj + int32_t inUse = pSql->epSet.inUse; + tscDumpEpSetFromVgroupInfo(&pSql->epSet, &vgroupInfo); + tscDebug("0x%"PRIx64" update the epSet in SqlObj, in use before:%d, after:%d", pSql->self, inUse, pSql->epSet.inUse); + +} + +int32_t extractSTableQueryVgroupId(STableMetaInfo* pTableMetaInfo) { + assert(pTableMetaInfo != NULL); + + int32_t vgIndex = pTableMetaInfo->vgroupIndex; + int32_t vgId = -1; + + if (pTableMetaInfo->pVgroupTables == NULL) { + SVgroupsInfo *pVgroupInfo = pTableMetaInfo->vgroupList; + assert(pVgroupInfo->vgroups[vgIndex].vgId > 0 && vgIndex < pTableMetaInfo->vgroupList->numOfVgroups); + vgId = pVgroupInfo->vgroups[vgIndex].vgId; + } else { + int32_t numOfVgroups = (int32_t)taosArrayGetSize(pTableMetaInfo->pVgroupTables); + assert(vgIndex >= 0 && vgIndex < numOfVgroups); + + SVgroupTableInfo *pTableIdList = taosArrayGet(pTableMetaInfo->pVgroupTables, vgIndex); + vgId = pTableIdList->vgInfo.vgId; + } + + return vgId; +} + +void tscProcessHeartBeatRsp(void *param, TAOS_RES *tres, int code) { + STscObj *pObj = (STscObj *)param; + if (pObj == NULL) return; + + if (pObj != pObj->signature) { + tscError("heartbeat msg, pObj:%p, signature:%p invalid", pObj, pObj->signature); + return; + } + + SSqlObj *pSql = tres; + SSqlRes *pRes = &pSql->res; + + if (code == TSDB_CODE_SUCCESS) { + SHeartBeatRsp *pRsp = (SHeartBeatRsp *)pRes->pRsp; + SRpcEpSet *epSet = &pRsp->epSet; + if (epSet->numOfEps > 0) { + tscEpSetHtons(epSet); + + //SCorEpSet *pCorEpSet = pSql->pTscObj->tscCorMgmtEpSet; + //if (!tscEpSetIsEqual(&pCorEpSet->epSet, epSet)) { + // tscTrace("%p updating epset: numOfEps: %d, inUse: %d", pSql, epSet->numOfEps, epSet->inUse); + // for (int8_t i = 0; i < epSet->numOfEps; i++) { + // tscTrace("endpoint %d: fqdn=%s, port=%d", i, epSet->fqdn[i], epSet->port[i]); + // } + //} + //concurrency problem, update mgmt epset anyway + tscUpdateMgmtEpSet(pSql, epSet); + } + + pSql->pTscObj->connId = htonl(pRsp->connId); + + if (pRsp->killConnection) { + tscKillConnection(pObj); + return; + } else { + if (pRsp->queryId) { + tscKillQuery(pObj, htonl(pRsp->queryId)); + } + + if (pRsp->streamId) { + tscKillStream(pObj, htonl(pRsp->streamId)); + } + } + + int32_t total = htonl(pRsp->totalDnodes); + int32_t online = htonl(pRsp->onlineDnodes); + assert(online <= total); + + if (online < total) { + tscError("0x%"PRIx64", HB, total dnode:%d, online dnode:%d", pSql->self, total, online); + pSql->res.code = TSDB_CODE_RPC_NETWORK_UNAVAIL; + } + + if (pRes->length == NULL) { + pRes->length = calloc(2, sizeof(int32_t)); + } + + pRes->length[0] = total; + pRes->length[1] = online; + } else { + tscDebug("%" PRId64 " heartbeat failed, code:%s", pObj->hbrid, tstrerror(code)); + if (pRes->length == NULL) { + pRes->length = calloc(2, sizeof(int32_t)); + } + + pRes->length[1] = 0; + if (pRes->length[0] == 0) { + pRes->length[0] = 1; // make sure that the value of the total node is greater than the online node + } + } + + if (pObj->hbrid != 0) { + int32_t waitingDuring = tsShellActivityTimer * 500; + tscDebug("0x%"PRIx64" send heartbeat in %dms", pSql->self, waitingDuring); + + taosTmrReset(tscProcessActivityTimer, waitingDuring, (void *)pObj->rid, tscTmr, &pObj->pTimer); + } else { + tscDebug("0x%"PRIx64" start to close tscObj:%p, not send heartbeat again", pSql->self, pObj); + } +} + +void tscProcessActivityTimer(void *handle, void *tmrId) { + int64_t rid = (int64_t) handle; + STscObj *pObj = taosAcquireRef(tscRefId, rid); + if (pObj == NULL) { + return; + } + + SSqlObj* pHB = taosAcquireRef(tscObjRef, pObj->hbrid); + if (pHB == NULL) { + taosReleaseRef(tscRefId, rid); + return; + } + + assert(pHB->self == pObj->hbrid); + + pHB->retry = 0; + int32_t code = tscBuildAndSendRequest(pHB, NULL); + taosReleaseRef(tscObjRef, pObj->hbrid); + + if (code != TSDB_CODE_SUCCESS) { + tscError("0x%"PRIx64" failed to sent HB to server, reason:%s", pHB->self, tstrerror(code)); + } + + taosReleaseRef(tscRefId, rid); +} + +int tscSendMsgToServer(SSqlObj *pSql) { + STscObj* pObj = pSql->pTscObj; + SSqlCmd* pCmd = &pSql->cmd; + + char *pMsg = rpcMallocCont(pCmd->payloadLen); + if (NULL == pMsg) { + tscError("0x%"PRIx64" msg:%s malloc failed", pSql->self, taosMsg[pSql->cmd.msgType]); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + // set the mgmt ip list + if (pSql->cmd.command >= TSDB_SQL_MGMT) { + tscDumpMgmtEpSet(pSql); + } + + memcpy(pMsg, pSql->cmd.payload, pSql->cmd.payloadLen); + + SRpcMsg rpcMsg = { + .msgType = pSql->cmd.msgType, + .pCont = pMsg, + .contLen = pSql->cmd.payloadLen, + .ahandle = (void*)pSql->self, + .handle = NULL, + .code = 0 + }; + + rpcSendRequest(pObj->pRpcObj->pDnodeConn, &pSql->epSet, &rpcMsg, &pSql->rpcRid); + return TSDB_CODE_SUCCESS; +} + +// handle three situation +// 1. epset retry, only return last failure ep +// 2. no epset retry, like 'taos -h invalidFqdn', return invalidFqdn +// 3. other situation, no expected +void tscSetFqdnErrorMsg(SSqlObj* pSql, SRpcEpSet* pEpSet) { + SSqlCmd* pCmd = &pSql->cmd; + SSqlRes* pRes = &pSql->res; + + char* msgBuf = tscGetErrorMsgPayload(pCmd); + + if (pEpSet) { + sprintf(msgBuf, "%s\"%s\"", tstrerror(pRes->code),pEpSet->fqdn[(pEpSet->inUse)%(pEpSet->numOfEps)]); + } else if (pCmd->command >= TSDB_SQL_MGMT) { + SRpcEpSet tEpset; + + SCorEpSet *pCorEpSet = pSql->pTscObj->tscCorMgmtEpSet; + taosCorBeginRead(&pCorEpSet->version); + tEpset = pCorEpSet->epSet; + taosCorEndRead(&pCorEpSet->version); + + sprintf(msgBuf, "%s\"%s\"", tstrerror(pRes->code),tEpset.fqdn[(tEpset.inUse)%(tEpset.numOfEps)]); + } else { + sprintf(msgBuf, "%s", tstrerror(pRes->code)); + } +} + +void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) { + TSDB_CACHE_PTR_TYPE handle = (TSDB_CACHE_PTR_TYPE) rpcMsg->ahandle; + SSqlObj* pSql = (SSqlObj*)taosAcquireRef(tscObjRef, handle); + if (pSql == NULL) { + rpcFreeCont(rpcMsg->pCont); + return; + } + + assert(pSql->self == handle); + + STscObj *pObj = pSql->pTscObj; + SSqlRes *pRes = &pSql->res; + SSqlCmd *pCmd = &pSql->cmd; + + pSql->rpcRid = -1; + if (pObj->signature != pObj) { + tscDebug("0x%"PRIx64" DB connection is closed, cmd:%d pObj:%p signature:%p", pSql->self, pCmd->command, pObj, pObj->signature); + + taosRemoveRef(tscObjRef, handle); + taosReleaseRef(tscObjRef, handle); + rpcFreeCont(rpcMsg->pCont); + return; + } + + SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); + if (pQueryInfo != NULL && pQueryInfo->type == TSDB_QUERY_TYPE_FREE_RESOURCE) { + tscDebug("0x%"PRIx64" sqlObj needs to be released or DB connection is closed, cmd:%d type:%d, pObj:%p signature:%p", + pSql->self, pCmd->command, pQueryInfo->type, pObj, pObj->signature); + + taosRemoveRef(tscObjRef, handle); + taosReleaseRef(tscObjRef, handle); + rpcFreeCont(rpcMsg->pCont); + return; + } + + if (pEpSet) { + if (!tscEpSetIsEqual(&pSql->epSet, pEpSet)) { + if (pCmd->command < TSDB_SQL_MGMT) { + tscUpdateVgroupInfo(pSql, pEpSet); + } else { + tscUpdateMgmtEpSet(pSql, pEpSet); + } + } + } + + int32_t cmd = pCmd->command; + + // set the flag to denote that sql string needs to be re-parsed and build submit block with table schema + if (cmd == TSDB_SQL_INSERT && rpcMsg->code == TSDB_CODE_TDB_TABLE_RECONFIGURE) { + pSql->cmd.insertParam.schemaAttached = 1; + } + + // single table query error need to be handled here. + if ((cmd == TSDB_SQL_SELECT || cmd == TSDB_SQL_UPDATE_TAGS_VAL) && + (((rpcMsg->code == TSDB_CODE_TDB_INVALID_TABLE_ID || rpcMsg->code == TSDB_CODE_VND_INVALID_VGROUP_ID)) || + rpcMsg->code == TSDB_CODE_RPC_NETWORK_UNAVAIL || rpcMsg->code == TSDB_CODE_APP_NOT_READY)) { + + // 1. super table subquery + // 2. nest queries are all not updated the tablemeta and retry parse the sql after cleanup local tablemeta/vgroup id buffer + if ((TSDB_QUERY_HAS_TYPE(pQueryInfo->type, (TSDB_QUERY_TYPE_STABLE_SUBQUERY | TSDB_QUERY_TYPE_SUBQUERY | + TSDB_QUERY_TYPE_TAG_FILTER_QUERY)) && + !TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_PROJECTION_QUERY)) || + (TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_NEST_SUBQUERY)) || (TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_STABLE_SUBQUERY) && pQueryInfo->distinct)) { + // do nothing in case of super table subquery + } else { + pSql->retry += 1; + tscWarn("0x%" PRIx64 " it shall renew table meta, code:%s, retry:%d", pSql->self, tstrerror(rpcMsg->code), pSql->retry); + + pSql->res.code = rpcMsg->code; // keep the previous error code + if (pSql->retry > pSql->maxRetry) { + tscError("0x%" PRIx64 " max retry %d reached, give up", pSql->self, pSql->maxRetry); + } else { + // wait for a little bit moment and then retry + // todo do not sleep in rpc callback thread, add this process into queue to process + if (rpcMsg->code == TSDB_CODE_APP_NOT_READY || rpcMsg->code == TSDB_CODE_VND_INVALID_VGROUP_ID) { + int32_t duration = getWaitingTimeInterval(pSql->retry); + taosMsleep(duration); + } + + pSql->retryReason = rpcMsg->code; + rpcMsg->code = tscRenewTableMeta(pSql, 0); + // if there is an error occurring, proceed to the following error handling procedure. + if (rpcMsg->code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { + taosReleaseRef(tscObjRef, handle); + rpcFreeCont(rpcMsg->pCont); + return; + } + } + } + } + + pRes->rspLen = 0; + + if (pRes->code == TSDB_CODE_TSC_QUERY_CANCELLED) { + tscDebug("0x%"PRIx64" query is cancelled, code:%s", pSql->self, tstrerror(pRes->code)); + } else { + pRes->code = rpcMsg->code; + } + + if (pRes->code == TSDB_CODE_SUCCESS) { + tscDebug("0x%"PRIx64" reset retry counter to be 0 due to success rsp, old:%d", pSql->self, pSql->retry); + pSql->retry = 0; + } + + if (pRes->code != TSDB_CODE_TSC_QUERY_CANCELLED) { + assert(rpcMsg->msgType == pCmd->msgType + 1); + pRes->code = rpcMsg->code; + pRes->rspType = rpcMsg->msgType; + pRes->rspLen = rpcMsg->contLen; + + if (pRes->rspLen > 0 && rpcMsg->pCont) { + char *tmp = (char *)realloc(pRes->pRsp, pRes->rspLen); + if (tmp == NULL) { + pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; + } else { + pRes->pRsp = tmp; + memcpy(pRes->pRsp, rpcMsg->pCont, pRes->rspLen); + } + } else { + tfree(pRes->pRsp); + } + + /* + * There is not response callback function for submit response. + * The actual inserted number of points is the first number. + */ + if (rpcMsg->msgType == TSDB_MSG_TYPE_SUBMIT_RSP && pRes->pRsp != NULL) { + SShellSubmitRspMsg *pMsg = (SShellSubmitRspMsg*)pRes->pRsp; + pMsg->code = htonl(pMsg->code); + pMsg->numOfRows = htonl(pMsg->numOfRows); + pMsg->affectedRows = htonl(pMsg->affectedRows); + pMsg->failedRows = htonl(pMsg->failedRows); + pMsg->numOfFailedBlocks = htonl(pMsg->numOfFailedBlocks); + + pRes->numOfRows += pMsg->affectedRows; + tscDebug("0x%"PRIx64" SQL cmd:%s, code:%s inserted rows:%d rspLen:%d", pSql->self, sqlCmd[pCmd->command], + tstrerror(pRes->code), pMsg->affectedRows, pRes->rspLen); + } else { + tscDebug("0x%"PRIx64" SQL cmd:%s, code:%s rspLen:%d", pSql->self, sqlCmd[pCmd->command], tstrerror(pRes->code), pRes->rspLen); + } + } + + if (pRes->code == TSDB_CODE_SUCCESS && tscProcessMsgRsp[pCmd->command]) { + rpcMsg->code = (*tscProcessMsgRsp[pCmd->command])(pSql); + } + + bool shouldFree = tscShouldBeFreed(pSql); + if (rpcMsg->code != TSDB_CODE_TSC_ACTION_IN_PROGRESS) { + if (rpcMsg->code != TSDB_CODE_SUCCESS) { + pRes->code = rpcMsg->code; + } + + rpcMsg->code = (pRes->code == TSDB_CODE_SUCCESS) ? (int32_t)pRes->numOfRows : pRes->code; + if (rpcMsg->code == TSDB_CODE_RPC_FQDN_ERROR) { + tscAllocPayload(pCmd, TSDB_FQDN_LEN + 64); + tscSetFqdnErrorMsg(pSql, pEpSet); + } + + (*pSql->fp)(pSql->param, pSql, rpcMsg->code); + } + + if (shouldFree) { // in case of table-meta/vgrouplist query, automatically free it + tscDebug("0x%"PRIx64" sqlObj is automatically freed", pSql->self); + taosRemoveRef(tscObjRef, handle); + } + + taosReleaseRef(tscObjRef, handle); + rpcFreeCont(rpcMsg->pCont); +} + +int doBuildAndSendMsg(SSqlObj *pSql) { + SSqlCmd *pCmd = &pSql->cmd; + SSqlRes *pRes = &pSql->res; + + if (pCmd->command == TSDB_SQL_SELECT || + pCmd->command == TSDB_SQL_FETCH || + pCmd->command == TSDB_SQL_RETRIEVE || + pCmd->command == TSDB_SQL_INSERT || + pCmd->command == TSDB_SQL_CONNECT || + pCmd->command == TSDB_SQL_HB || + pCmd->command == TSDB_SQL_RETRIEVE_FUNC || + pCmd->command == TSDB_SQL_STABLEVGROUP) { + pRes->code = tscBuildMsg[pCmd->command](pSql, NULL); + } + + if (pRes->code != TSDB_CODE_SUCCESS) { + tscAsyncResultOnError(pSql); + return TSDB_CODE_SUCCESS; + } + + int32_t code = tscSendMsgToServer(pSql); + + // NOTE: if code is TSDB_CODE_SUCCESS, pSql may have been released here already by other threads. + if (code != TSDB_CODE_SUCCESS) { + pRes->code = code; + tscAsyncResultOnError(pSql); + return TSDB_CODE_SUCCESS; + } + + return TSDB_CODE_SUCCESS; +} + +int tscBuildAndSendRequest(SSqlObj *pSql, SQueryInfo* pQueryInfo) { + char name[TSDB_TABLE_FNAME_LEN] = {0}; + + SSqlCmd *pCmd = &pSql->cmd; + uint32_t type = 0; + + if (pQueryInfo == NULL) { + pQueryInfo = tscGetQueryInfo(pCmd); + } + + STableMetaInfo *pTableMetaInfo = NULL; + + if (pQueryInfo != NULL) { + pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + if (pTableMetaInfo != NULL) { + tNameExtractFullName(&pTableMetaInfo->name, name); + } + + type = pQueryInfo->type; + + // while numOfTables equals to 0, it must be Heartbeat + assert((pQueryInfo->numOfTables == 0 && (pQueryInfo->command == TSDB_SQL_HB || pSql->cmd.command == TSDB_SQL_RETRIEVE_FUNC)) || pQueryInfo->numOfTables > 0); + } + + tscDebug("0x%"PRIx64" SQL cmd:%s will be processed, name:%s, type:%d", pSql->self, sqlCmd[pCmd->command], name, type); + if (pCmd->command < TSDB_SQL_MGMT) { // the pTableMetaInfo cannot be NULL + if (pTableMetaInfo == NULL) { + pSql->res.code = TSDB_CODE_TSC_APP_ERROR; + return pSql->res.code; + } + } else if (pCmd->command >= TSDB_SQL_LOCAL) { + return (*tscProcessMsgRsp[pCmd->command])(pSql); + } + + return doBuildAndSendMsg(pSql); +} + +int tscBuildFetchMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + SRetrieveTableMsg *pRetrieveMsg = (SRetrieveTableMsg *) pSql->cmd.payload; + + SQueryInfo *pQueryInfo = tscGetQueryInfo(&pSql->cmd); + + pRetrieveMsg->free = htons(pQueryInfo->type); + pRetrieveMsg->qId = htobe64(pSql->res.qId); + + // todo valid the vgroupId at the client side + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + + if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { + int32_t vgIndex = pTableMetaInfo->vgroupIndex; + int32_t vgId = -1; + + if (pTableMetaInfo->pVgroupTables == NULL) { + SVgroupsInfo *pVgroupInfo = pTableMetaInfo->vgroupList; + assert(pVgroupInfo->vgroups[vgIndex].vgId > 0 && vgIndex < pTableMetaInfo->vgroupList->numOfVgroups); + vgId = pVgroupInfo->vgroups[vgIndex].vgId; + } else { + int32_t numOfVgroups = (int32_t)taosArrayGetSize(pTableMetaInfo->pVgroupTables); + assert(vgIndex >= 0 && vgIndex < numOfVgroups); + + SVgroupTableInfo* pTableIdList = taosArrayGet(pTableMetaInfo->pVgroupTables, vgIndex); + vgId = pTableIdList->vgInfo.vgId; + } + + pRetrieveMsg->header.vgId = htonl(vgId); + tscDebug("0x%"PRIx64" build fetch msg from vgId:%d, vgIndex:%d, qId:0x%" PRIx64, pSql->self, vgId, vgIndex, pSql->res.qId); + } else { + STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; + pRetrieveMsg->header.vgId = htonl(pTableMeta->vgId); + tscDebug("0x%"PRIx64" build fetch msg from only one vgroup, vgId:%d, qId:0x%" PRIx64, pSql->self, pTableMeta->vgId, + pSql->res.qId); + } + + pSql->cmd.payloadLen = sizeof(SRetrieveTableMsg); + pSql->cmd.msgType = TSDB_MSG_TYPE_FETCH; + + pRetrieveMsg->header.contLen = htonl(sizeof(SRetrieveTableMsg)); + + return TSDB_CODE_SUCCESS; +} + +int tscBuildSubmitMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + SQueryInfo *pQueryInfo = tscGetQueryInfo(&pSql->cmd); + STableMeta* pTableMeta = tscGetMetaInfo(pQueryInfo, 0)->pTableMeta; + + // pSql->cmd.payloadLen is set during copying data into payload + pSql->cmd.msgType = TSDB_MSG_TYPE_SUBMIT; + + SNewVgroupInfo vgroupInfo = {0}; + taosHashGetClone(tscVgroupMap, &pTableMeta->vgId, sizeof(pTableMeta->vgId), NULL, &vgroupInfo); + tscDumpEpSetFromVgroupInfo(&pSql->epSet, &vgroupInfo); + + tscDebug("0x%"PRIx64" submit msg built, numberOfEP:%d", pSql->self, pSql->epSet.numOfEps); + + return TSDB_CODE_SUCCESS; +} + +/* + * for table query, simply return the size <= 1k + */ +static int32_t tscEstimateQueryMsgSize(SSqlObj *pSql) { + const static int32_t MIN_QUERY_MSG_PKT_SIZE = TSDB_MAX_BYTES_PER_ROW * 5; + + SSqlCmd* pCmd = &pSql->cmd; + SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); + + int32_t srcColListSize = (int32_t)(taosArrayGetSize(pQueryInfo->colList) * sizeof(SColumnInfo)); + int32_t srcColFilterSize = 0; + int32_t srcTagFilterSize = tscGetTagFilterSerializeLen(pQueryInfo); + + size_t numOfExprs = tscNumOfExprs(pQueryInfo); + int32_t exprSize = (int32_t)(sizeof(SSqlExpr) * numOfExprs * 2); + + int32_t tsBufSize = (pQueryInfo->tsBuf != NULL) ? pQueryInfo->tsBuf->fileSize : 0; + int32_t sqlLen = (int32_t) strlen(pSql->sqlstr) + 1; + + int32_t tableSerialize = 0; + STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + STableMeta * pTableMeta = pTableMetaInfo->pTableMeta; + if (pTableMetaInfo->pVgroupTables != NULL) { + size_t numOfGroups = taosArrayGetSize(pTableMetaInfo->pVgroupTables); + + int32_t totalTables = 0; + for (int32_t i = 0; i < numOfGroups; ++i) { + SVgroupTableInfo *pTableInfo = taosArrayGet(pTableMetaInfo->pVgroupTables, i); + totalTables += (int32_t) taosArrayGetSize(pTableInfo->itemList); + } + + tableSerialize = totalTables * sizeof(STableIdInfo); + } + + if (pQueryInfo->colCond && taosArrayGetSize(pQueryInfo->colCond) > 0) { + STblCond *pCond = tsGetTableFilter(pQueryInfo->colCond, pTableMeta->id.uid, 0); + if (pCond != NULL && pCond->cond != NULL) { + srcColFilterSize = pCond->len; + } + } + + SCond* pCond = &pQueryInfo->tagCond.tbnameCond; + if (pCond->len > 0) { + srcColListSize += pCond->len; + } + + return MIN_QUERY_MSG_PKT_SIZE + minMsgSize() + sizeof(SQueryTableMsg) + srcColListSize + srcColFilterSize + srcTagFilterSize + + exprSize + tsBufSize + tableSerialize + sqlLen + 4096 + pQueryInfo->bufLen; +} + +static char *doSerializeTableInfo(SQueryTableMsg *pQueryMsg, SSqlObj *pSql, STableMetaInfo *pTableMetaInfo, char *pMsg, + int32_t *succeed) { + TSKEY dfltKey = htobe64(pQueryMsg->window.skey); + + STableMeta * pTableMeta = pTableMetaInfo->pTableMeta; + if (UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo) || pTableMetaInfo->pVgroupTables == NULL) { + + int32_t vgId = -1; + if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { + int32_t index = pTableMetaInfo->vgroupIndex; + assert(index >= 0); + + SVgroupMsg* pVgroupInfo = NULL; + if (pTableMetaInfo->vgroupList && pTableMetaInfo->vgroupList->numOfVgroups > 0) { + assert(index < pTableMetaInfo->vgroupList->numOfVgroups); + pVgroupInfo = &pTableMetaInfo->vgroupList->vgroups[index]; + } else { + tscError("0x%"PRIx64" No vgroup info found", pSql->self); + + *succeed = 0; + return pMsg; + } + + vgId = pVgroupInfo->vgId; + tscSetDnodeEpSet(&pSql->epSet, pVgroupInfo); + tscDebug("0x%"PRIx64" query on stable, vgIndex:%d, numOfVgroups:%d", pSql->self, index, pTableMetaInfo->vgroupList->numOfVgroups); + } else { + vgId = pTableMeta->vgId; + + SNewVgroupInfo vgroupInfo = {0}; + taosHashGetClone(tscVgroupMap, &pTableMeta->vgId, sizeof(pTableMeta->vgId), NULL, &vgroupInfo); + tscDumpEpSetFromVgroupInfo(&pSql->epSet, &vgroupInfo); + } + + if (pSql->epSet.numOfEps > 0){ + pSql->epSet.inUse = rand()%pSql->epSet.numOfEps; + } + pQueryMsg->head.vgId = htonl(vgId); + + STableIdInfo *pTableIdInfo = (STableIdInfo *)pMsg; + pTableIdInfo->tid = htonl(pTableMeta->id.tid); + pTableIdInfo->uid = htobe64(pTableMeta->id.uid); + pTableIdInfo->key = htobe64(tscGetSubscriptionProgress(pSql->pSubscription, pTableMeta->id.uid, dfltKey)); + + pQueryMsg->numOfTables = htonl(1); // set the number of tables + pMsg += sizeof(STableIdInfo); + } else { // it is a subquery of the super table query, this EP info is acquired from vgroupInfo + int32_t index = pTableMetaInfo->vgroupIndex; + int32_t numOfVgroups = (int32_t)taosArrayGetSize(pTableMetaInfo->pVgroupTables); + assert(index >= 0 && index < numOfVgroups); + + SVgroupTableInfo* pTableIdList = taosArrayGet(pTableMetaInfo->pVgroupTables, index); + + // set the vgroup info + tscSetDnodeEpSet(&pSql->epSet, &pTableIdList->vgInfo); + pQueryMsg->head.vgId = htonl(pTableIdList->vgInfo.vgId); + + int32_t numOfTables = (int32_t)taosArrayGetSize(pTableIdList->itemList); + pQueryMsg->numOfTables = htonl(numOfTables); // set the number of tables + + tscDebug("0x%"PRIx64" query on stable, vgId:%d, numOfTables:%d, vgIndex:%d, numOfVgroups:%d", pSql->self, + pTableIdList->vgInfo.vgId, numOfTables, index, numOfVgroups); + + // serialize each table id info + for(int32_t i = 0; i < numOfTables; ++i) { + STableIdInfo* pItem = taosArrayGet(pTableIdList->itemList, i); + + STableIdInfo *pTableIdInfo = (STableIdInfo *)pMsg; + pTableIdInfo->tid = htonl(pItem->tid); + pTableIdInfo->uid = htobe64(pItem->uid); + pTableIdInfo->key = htobe64(tscGetSubscriptionProgress(pSql->pSubscription, pItem->uid, dfltKey)); + pMsg += sizeof(STableIdInfo); + } + } + + char n[TSDB_TABLE_FNAME_LEN] = {0}; + tNameExtractFullName(&pTableMetaInfo->name, n); + + tscDebug("0x%"PRIx64" vgId:%d, query on table:%s, tid:%d, uid:%" PRIu64, pSql->self, htonl(pQueryMsg->head.vgId), n, pTableMeta->id.tid, pTableMeta->id.uid); + return pMsg; +} + +// TODO refactor +static int32_t serializeColFilterInfo(SColumnFilterInfo* pColFilters, int16_t numOfFilters, char** pMsg) { + // append the filter information after the basic column information + for (int32_t f = 0; f < numOfFilters; ++f) { + SColumnFilterInfo *pColFilter = &pColFilters[f]; + + SColumnFilterInfo *pFilterMsg = (SColumnFilterInfo *)(*pMsg); + pFilterMsg->filterstr = htons(pColFilter->filterstr); + + (*pMsg) += sizeof(SColumnFilterInfo); + + if (pColFilter->filterstr) { + pFilterMsg->len = htobe64(pColFilter->len); + memcpy(*pMsg, (void *)pColFilter->pz, (size_t)(pColFilter->len + 1)); + (*pMsg) += (pColFilter->len + 1); // append the additional filter binary info + } else { + pFilterMsg->lowerBndi = htobe64(pColFilter->lowerBndi); + pFilterMsg->upperBndi = htobe64(pColFilter->upperBndi); + } + + pFilterMsg->lowerRelOptr = htons(pColFilter->lowerRelOptr); + pFilterMsg->upperRelOptr = htons(pColFilter->upperRelOptr); + + if (pColFilter->lowerRelOptr == TSDB_RELATION_INVALID && pColFilter->upperRelOptr == TSDB_RELATION_INVALID) { + tscError("invalid filter info"); + return TSDB_CODE_TSC_INVALID_OPERATION; + } + } + + return TSDB_CODE_SUCCESS; +} + +static int32_t serializeSqlExpr(SSqlExpr* pExpr, STableMetaInfo* pTableMetaInfo, char** pMsg, int64_t id, bool validateColumn) { + STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; + + // the queried table has been removed and a new table with the same name has already been created already + // return error msg + if (pExpr->uid != pTableMeta->id.uid) { + tscError("0x%"PRIx64" table has already been destroyed", id); + return TSDB_CODE_TSC_INVALID_TABLE_NAME; + } + + if (validateColumn && !tscValidateColumnId(pTableMetaInfo, pExpr->colInfo.colId, pExpr->numOfParams)) { + tscError("0x%"PRIx64" table schema is not matched with parsed sql", id); + return TSDB_CODE_TSC_INVALID_OPERATION; + } + + assert(pExpr->resColId < 0); + SSqlExpr* pSqlExpr = (SSqlExpr *)(*pMsg); + + SColIndex* pIndex = &pSqlExpr->colInfo; + + pIndex->colId = htons(pExpr->colInfo.colId); + pIndex->colIndex = htons(pExpr->colInfo.colIndex); + pIndex->flag = htons(pExpr->colInfo.flag); + pSqlExpr->uid = htobe64(pExpr->uid); + pSqlExpr->colType = htons(pExpr->colType); + pSqlExpr->colBytes = htons(pExpr->colBytes); + pSqlExpr->resType = htons(pExpr->resType); + pSqlExpr->resBytes = htons(pExpr->resBytes); + pSqlExpr->interBytes = htonl(pExpr->interBytes); + pSqlExpr->functionId = htons(pExpr->functionId); + pSqlExpr->numOfParams = htons(pExpr->numOfParams); + pSqlExpr->resColId = htons(pExpr->resColId); + pSqlExpr->flist.numOfFilters = htons(pExpr->flist.numOfFilters); + + (*pMsg) += sizeof(SSqlExpr); + for (int32_t j = 0; j < pExpr->numOfParams; ++j) { // todo add log + pSqlExpr->param[j].nType = htonl(pExpr->param[j].nType); + pSqlExpr->param[j].nLen = htonl(pExpr->param[j].nLen); + + if (pExpr->param[j].nType == TSDB_DATA_TYPE_BINARY) { + memcpy((*pMsg), pExpr->param[j].pz, pExpr->param[j].nLen); + (*pMsg) += pExpr->param[j].nLen; + } else { + pSqlExpr->param[j].i64 = htobe64(pExpr->param[j].i64); + } + } + + serializeColFilterInfo(pExpr->flist.filterInfo, pExpr->flist.numOfFilters, pMsg); + + return TSDB_CODE_SUCCESS; +} + +int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + SSqlCmd *pCmd = &pSql->cmd; + + SQueryInfo *pQueryInfo = NULL; + STableMeta *pTableMeta = NULL; + STableMetaInfo *pTableMetaInfo = NULL; + + int32_t code = TSDB_CODE_SUCCESS; + int32_t size = tscEstimateQueryMsgSize(pSql); + assert(size > 0); + + if (TSDB_CODE_SUCCESS != tscAllocPayloadFast(pCmd, size)) { + tscError("%p failed to malloc for query msg", pSql); + return TSDB_CODE_TSC_INVALID_OPERATION; // todo add test for this + } + + pQueryInfo = tscGetQueryInfo(pCmd); + pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + pTableMeta = pTableMetaInfo->pTableMeta; + + SQueryAttr query = {{0}}; + tscCreateQueryFromQueryInfo(pQueryInfo, &query, pSql); + query.vgId = pTableMeta->vgId; + + SArray* tableScanOperator = createTableScanPlan(&query); + SArray* queryOperator = createExecOperatorPlan(&query); + + SQueryTableMsg *pQueryMsg = (SQueryTableMsg *)pCmd->payload; + tstrncpy(pQueryMsg->version, version, tListLen(pQueryMsg->version)); + + int32_t numOfTags = query.numOfTags; + int32_t sqlLen = (int32_t) strlen(pSql->sqlstr); + + if (taosArrayGetSize(tableScanOperator) == 0) { + pQueryMsg->tableScanOperator = htonl(-1); + } else { + int32_t* tablescanOp = taosArrayGet(tableScanOperator, 0); + pQueryMsg->tableScanOperator = htonl(*tablescanOp); + } + + pQueryMsg->window.skey = htobe64(query.window.skey); + pQueryMsg->window.ekey = htobe64(query.window.ekey); + + pQueryMsg->order = htons(query.order.order); + pQueryMsg->orderColId = htons(query.order.orderColId); + pQueryMsg->fillType = htons(query.fillType); + pQueryMsg->limit = htobe64(query.limit.limit); + pQueryMsg->offset = htobe64(query.limit.offset); + pQueryMsg->numOfCols = htons(query.numOfCols); + + pQueryMsg->interval.interval = htobe64(query.interval.interval); + pQueryMsg->interval.sliding = htobe64(query.interval.sliding); + pQueryMsg->interval.offset = htobe64(query.interval.offset); + pQueryMsg->interval.intervalUnit = query.interval.intervalUnit; + pQueryMsg->interval.slidingUnit = query.interval.slidingUnit; + pQueryMsg->interval.offsetUnit = query.interval.offsetUnit; + + pQueryMsg->stableQuery = query.stableQuery; + pQueryMsg->topBotQuery = query.topBotQuery; + pQueryMsg->groupbyColumn = query.groupbyColumn; + pQueryMsg->hasTagResults = query.hasTagResults; + pQueryMsg->timeWindowInterpo = query.timeWindowInterpo; + pQueryMsg->queryBlockDist = query.queryBlockDist; + pQueryMsg->stabledev = query.stabledev; + pQueryMsg->tsCompQuery = query.tsCompQuery; + pQueryMsg->simpleAgg = query.simpleAgg; + pQueryMsg->pointInterpQuery = query.pointInterpQuery; + pQueryMsg->needReverseScan = query.needReverseScan; + pQueryMsg->stateWindow = query.stateWindow; + pQueryMsg->numOfTags = htonl(numOfTags); + pQueryMsg->sqlstrLen = htonl(sqlLen); + pQueryMsg->sw.gap = htobe64(query.sw.gap); + pQueryMsg->sw.primaryColId = htonl(PRIMARYKEY_TIMESTAMP_COL_INDEX); + + pQueryMsg->secondStageOutput = htonl(query.numOfExpr2); + pQueryMsg->numOfOutput = htons((int16_t)query.numOfOutput); // this is the stage one output column number + + pQueryMsg->numOfGroupCols = htons(pQueryInfo->groupbyExpr.numOfGroupCols); + pQueryMsg->tagNameRelType = htons(pQueryInfo->tagCond.relType); + pQueryMsg->tbnameCondLen = htonl(pQueryInfo->tagCond.tbnameCond.len); + pQueryMsg->queryType = htonl(pQueryInfo->type); + pQueryMsg->prevResultLen = htonl(pQueryInfo->bufLen); + + // set column list ids + size_t numOfCols = taosArrayGetSize(pQueryInfo->colList); + char *pMsg = (char *)(pQueryMsg->tableCols) + numOfCols * sizeof(SColumnInfo); + + for (int32_t i = 0; i < numOfCols; ++i) { + SColumnInfo *pCol = &query.tableCols[i]; + + pQueryMsg->tableCols[i].colId = htons(pCol->colId); + pQueryMsg->tableCols[i].bytes = htons(pCol->bytes); + pQueryMsg->tableCols[i].type = htons(pCol->type); + //pQueryMsg->tableCols[i].flist.numOfFilters = htons(pCol->flist.numOfFilters); + pQueryMsg->tableCols[i].flist.numOfFilters = 0; + pQueryMsg->tableCols[i].flist.filterInfo = 0; + // append the filter information after the basic column information + //serializeColFilterInfo(pCol->flist.filterInfo, pCol->flist.numOfFilters, &pMsg); + } + + if (pQueryInfo->colCond && taosArrayGetSize(pQueryInfo->colCond) > 0 && !onlyQueryTags(&query) ) { + STblCond *pCond = tsGetTableFilter(pQueryInfo->colCond, pTableMeta->id.uid, 0); + if (pCond != NULL && pCond->cond != NULL) { + pQueryMsg->colCondLen = htons(pCond->len); + memcpy(pMsg, pCond->cond, pCond->len); + + pMsg += pCond->len; + } + } else { + pQueryMsg->colCondLen = 0; + } + + for (int32_t i = 0; i < query.numOfOutput; ++i) { + code = serializeSqlExpr(&query.pExpr1[i].base, pTableMetaInfo, &pMsg, pSql->self, true); + if (code != TSDB_CODE_SUCCESS) { + goto _end; + } + } + + for (int32_t i = 0; i < query.numOfExpr2; ++i) { + code = serializeSqlExpr(&query.pExpr2[i].base, pTableMetaInfo, &pMsg, pSql->self, false); + if (code != TSDB_CODE_SUCCESS) { + goto _end; + } + } + + int32_t succeed = 1; + + // serialize the table info (sid, uid, tags) + pMsg = doSerializeTableInfo(pQueryMsg, pSql, pTableMetaInfo, pMsg, &succeed); + if (succeed == 0) { + code = TSDB_CODE_TSC_APP_ERROR; + goto _end; + } + + SGroupbyExpr *pGroupbyExpr = query.pGroupbyExpr; + if (pGroupbyExpr != NULL && pGroupbyExpr->numOfGroupCols > 0) { + pQueryMsg->orderByIdx = htons(pGroupbyExpr->orderIndex); + pQueryMsg->orderType = htons(pGroupbyExpr->orderType); + + for (int32_t j = 0; j < pGroupbyExpr->numOfGroupCols; ++j) { + SColIndex* pCol = taosArrayGet(pGroupbyExpr->columnInfo, j); + + *((int16_t *)pMsg) = htons(pCol->colId); + pMsg += sizeof(pCol->colId); + + *((int16_t *)pMsg) += htons(pCol->colIndex); + pMsg += sizeof(pCol->colIndex); + + *((int16_t *)pMsg) += htons(pCol->flag); + pMsg += sizeof(pCol->flag); + + memcpy(pMsg, pCol->name, tListLen(pCol->name)); + pMsg += tListLen(pCol->name); + } + } + + if (query.fillType != TSDB_FILL_NONE) { + for (int32_t i = 0; i < query.numOfOutput; ++i) { + *((int64_t *)pMsg) = htobe64(query.fillVal[i]); + pMsg += sizeof(query.fillVal[0]); + } + } + + if (query.numOfTags > 0 && query.tagColList != NULL) { + for (int32_t i = 0; i < query.numOfTags; ++i) { + SColumnInfo* pTag = &query.tagColList[i]; + + SColumnInfo* pTagCol = (SColumnInfo*) pMsg; + pTagCol->colId = htons(pTag->colId); + pTagCol->bytes = htons(pTag->bytes); + pTagCol->type = htons(pTag->type); + pTagCol->flist.numOfFilters = 0; + + pMsg += sizeof(SColumnInfo); + } + } + + // serialize tag column query condition + if (pQueryInfo->tagCond.pCond != NULL && taosArrayGetSize(pQueryInfo->tagCond.pCond) > 0) { + STagCond* pTagCond = &pQueryInfo->tagCond; + + SCond *pCond = tsGetSTableQueryCond(pTagCond, pTableMeta->id.uid); + if (pCond != NULL && pCond->cond != NULL) { + pQueryMsg->tagCondLen = htons(pCond->len); + memcpy(pMsg, pCond->cond, pCond->len); + + pMsg += pCond->len; + } + } else { + pQueryMsg->tagCondLen = 0; + } + + if (pQueryInfo->bufLen > 0) { + memcpy(pMsg, pQueryInfo->buf, pQueryInfo->bufLen); + pMsg += pQueryInfo->bufLen; + } + + SCond* pCond = &pQueryInfo->tagCond.tbnameCond; + if (pCond->len > 0) { + strncpy(pMsg, pCond->cond, pCond->len); + pMsg += pCond->len; + } + + // compressed ts block + pQueryMsg->tsBuf.tsOffset = htonl((int32_t)(pMsg - pCmd->payload)); + + if (pQueryInfo->tsBuf != NULL) { + // note: here used the index instead of actual vnode id. + int32_t vnodeIndex = pTableMetaInfo->vgroupIndex; + code = dumpFileBlockByGroupId(pQueryInfo->tsBuf, vnodeIndex, pMsg, &pQueryMsg->tsBuf.tsLen, &pQueryMsg->tsBuf.tsNumOfBlocks); + if (code != TSDB_CODE_SUCCESS) { + goto _end; + } + + pMsg += pQueryMsg->tsBuf.tsLen; + + pQueryMsg->tsBuf.tsOrder = htonl(pQueryInfo->tsBuf->tsOrder); + pQueryMsg->tsBuf.tsLen = htonl(pQueryMsg->tsBuf.tsLen); + pQueryMsg->tsBuf.tsNumOfBlocks = htonl(pQueryMsg->tsBuf.tsNumOfBlocks); + } else { + pQueryMsg->tsBuf.tsLen = 0; + pQueryMsg->tsBuf.tsNumOfBlocks = 0; + } + + int32_t numOfOperator = (int32_t) taosArrayGetSize(queryOperator); + pQueryMsg->numOfOperator = htonl(numOfOperator); + for(int32_t i = 0; i < numOfOperator; ++i) { + int32_t *operator = taosArrayGet(queryOperator, i); + *(int32_t*)pMsg = htonl(*operator); + + pMsg += sizeof(int32_t); + } + + // support only one udf + if (pQueryInfo->pUdfInfo != NULL && taosArrayGetSize(pQueryInfo->pUdfInfo) > 0) { + pQueryMsg->udfContentOffset = htonl((int32_t) (pMsg - pCmd->payload)); + for(int32_t i = 0; i < taosArrayGetSize(pQueryInfo->pUdfInfo); ++i) { + SUdfInfo* pUdfInfo = taosArrayGet(pQueryInfo->pUdfInfo, i); + *(int8_t*) pMsg = pUdfInfo->resType; + pMsg += sizeof(pUdfInfo->resType); + + *(int16_t*) pMsg = htons(pUdfInfo->resBytes); + pMsg += sizeof(pUdfInfo->resBytes); + + STR_TO_VARSTR(pMsg, pUdfInfo->name); + + pMsg += varDataTLen(pMsg); + + *(int32_t*) pMsg = htonl(pUdfInfo->funcType); + pMsg += sizeof(pUdfInfo->funcType); + + *(int32_t*) pMsg = htonl(pUdfInfo->bufSize); + pMsg += sizeof(pUdfInfo->bufSize); + + pQueryMsg->udfContentLen = htonl(pUdfInfo->contLen); + memcpy(pMsg, pUdfInfo->content, pUdfInfo->contLen); + + pMsg += pUdfInfo->contLen; + } + } else { + pQueryMsg->udfContentOffset = 0; + pQueryMsg->udfContentLen = 0; + } + + memcpy(pMsg, pSql->sqlstr, sqlLen); + pMsg += sqlLen; + + int32_t msgLen = (int32_t)(pMsg - pCmd->payload); + + tscDebug("0x%"PRIx64" msg built success, len:%d bytes", pSql->self, msgLen); + pCmd->payloadLen = msgLen; + pSql->cmd.msgType = TSDB_MSG_TYPE_QUERY; + + pQueryMsg->head.contLen = htonl(msgLen); + assert(msgLen + minMsgSize() <= (int32_t)pCmd->allocSize); + + _end: + freeQueryAttr(&query); + taosArrayDestroy(tableScanOperator); + taosArrayDestroy(queryOperator); + return code; +} + +int32_t tscBuildCreateDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + SSqlCmd *pCmd = &pSql->cmd; + pCmd->payloadLen = sizeof(SCreateDbMsg); + + pCmd->msgType = (pInfo->pMiscInfo->dbOpt.dbType == TSDB_DB_TYPE_DEFAULT) ? TSDB_MSG_TYPE_CREATE_DB : TSDB_MSG_TYPE_CREATE_TP; + + SCreateDbMsg *pCreateDbMsg = (SCreateDbMsg *)pCmd->payload; + +// assert(pCmd->numOfClause == 1); + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); + int32_t code = tNameExtractFullName(&pTableMetaInfo->name, pCreateDbMsg->db); + assert(code == TSDB_CODE_SUCCESS); + + return TSDB_CODE_SUCCESS; +} + +int32_t tscBuildCreateFuncMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + SSqlCmd *pCmd = &pSql->cmd; + SCreateFuncMsg *pCreateFuncMsg = (SCreateFuncMsg *)pCmd->payload; + + pCmd->msgType = TSDB_MSG_TYPE_CREATE_FUNCTION; + + pCmd->payloadLen = sizeof(SCreateFuncMsg) + htonl(pCreateFuncMsg->codeLen); + + return TSDB_CODE_SUCCESS; +} + +int32_t tscBuildCreateDnodeMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + SSqlCmd *pCmd = &pSql->cmd; + pCmd->payloadLen = sizeof(SCreateDnodeMsg); + if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { + tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + SCreateDnodeMsg *pCreate = (SCreateDnodeMsg *)pCmd->payload; + + SStrToken* t0 = taosArrayGet(pInfo->pMiscInfo->a, 0); + strncpy(pCreate->ep, t0->z, t0->n); + + pCmd->msgType = TSDB_MSG_TYPE_CREATE_DNODE; + + return TSDB_CODE_SUCCESS; +} + +int32_t tscBuildAcctMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + SSqlCmd *pCmd = &pSql->cmd; + pCmd->payloadLen = sizeof(SCreateAcctMsg); + if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { + tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + SCreateAcctMsg *pAlterMsg = (SCreateAcctMsg *)pCmd->payload; + + SStrToken *pName = &pInfo->pMiscInfo->user.user; + SStrToken *pPwd = &pInfo->pMiscInfo->user.passwd; + + strncpy(pAlterMsg->user, pName->z, pName->n); + strncpy(pAlterMsg->pass, pPwd->z, pPwd->n); + + SCreateAcctInfo *pAcctOpt = &pInfo->pMiscInfo->acctOpt; + + pAlterMsg->cfg.maxUsers = htonl(pAcctOpt->maxUsers); + pAlterMsg->cfg.maxDbs = htonl(pAcctOpt->maxDbs); + pAlterMsg->cfg.maxTimeSeries = htonl(pAcctOpt->maxTimeSeries); + pAlterMsg->cfg.maxStreams = htonl(pAcctOpt->maxStreams); + pAlterMsg->cfg.maxPointsPerSecond = htonl(pAcctOpt->maxPointsPerSecond); + pAlterMsg->cfg.maxStorage = htobe64(pAcctOpt->maxStorage); + pAlterMsg->cfg.maxQueryTime = htobe64(pAcctOpt->maxQueryTime); + pAlterMsg->cfg.maxConnections = htonl(pAcctOpt->maxConnections); + + if (pAcctOpt->stat.n == 0) { + pAlterMsg->cfg.accessState = -1; + } else { + if (pAcctOpt->stat.z[0] == 'r' && pAcctOpt->stat.n == 1) { + pAlterMsg->cfg.accessState = TSDB_VN_READ_ACCCESS; + } else if (pAcctOpt->stat.z[0] == 'w' && pAcctOpt->stat.n == 1) { + pAlterMsg->cfg.accessState = TSDB_VN_WRITE_ACCCESS; + } else if (strncmp(pAcctOpt->stat.z, "all", 3) == 0 && pAcctOpt->stat.n == 3) { + pAlterMsg->cfg.accessState = TSDB_VN_ALL_ACCCESS; + } else if (strncmp(pAcctOpt->stat.z, "no", 2) == 0 && pAcctOpt->stat.n == 2) { + pAlterMsg->cfg.accessState = 0; + } + } + + pCmd->msgType = TSDB_MSG_TYPE_CREATE_ACCT; + return TSDB_CODE_SUCCESS; +} + +int32_t tscBuildUserMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + SSqlCmd *pCmd = &pSql->cmd; + pCmd->payloadLen = sizeof(SCreateUserMsg); + + if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { + tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + SCreateUserMsg *pAlterMsg = (SCreateUserMsg *)pCmd->payload; + + SUserInfo *pUser = &pInfo->pMiscInfo->user; + strncpy(pAlterMsg->user, pUser->user.z, pUser->user.n); + pAlterMsg->flag = (int8_t)pUser->type; + + if (pUser->type == TSDB_ALTER_USER_PRIVILEGES) { + pAlterMsg->privilege = (char)pCmd->count; + } else if (pUser->type == TSDB_ALTER_USER_PASSWD) { + strncpy(pAlterMsg->pass, pUser->passwd.z, pUser->passwd.n); + } else { // create user password info + strncpy(pAlterMsg->pass, pUser->passwd.z, pUser->passwd.n); + } + + if (pUser->type == TSDB_ALTER_USER_PASSWD || pUser->type == TSDB_ALTER_USER_PRIVILEGES) { + pCmd->msgType = TSDB_MSG_TYPE_ALTER_USER; + } else { + pCmd->msgType = TSDB_MSG_TYPE_CREATE_USER; + } + + return TSDB_CODE_SUCCESS; +} + +int32_t tscBuildCfgDnodeMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + SSqlCmd *pCmd = &pSql->cmd; + pCmd->payloadLen = sizeof(SCfgDnodeMsg); + pCmd->msgType = TSDB_MSG_TYPE_CONFIG_DNODE; + return TSDB_CODE_SUCCESS; +} + +int32_t tscBuildDropDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + SSqlCmd *pCmd = &pSql->cmd; + pCmd->payloadLen = sizeof(SDropDbMsg); + + if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { + tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + SDropDbMsg *pDropDbMsg = (SDropDbMsg*)pCmd->payload; + + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); + + int32_t code = tNameExtractFullName(&pTableMetaInfo->name, pDropDbMsg->db); + assert(code == TSDB_CODE_SUCCESS && pTableMetaInfo->name.type == TSDB_DB_NAME_T); + + pDropDbMsg->ignoreNotExists = pInfo->pMiscInfo->existsCheck ? 1 : 0; + + pCmd->msgType = (pInfo->pMiscInfo->dbType == TSDB_DB_TYPE_DEFAULT) ? TSDB_MSG_TYPE_DROP_DB : TSDB_MSG_TYPE_DROP_TP; + return TSDB_CODE_SUCCESS; +} + +int32_t tscBuildDropFuncMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + SSqlCmd *pCmd = &pSql->cmd; + + pCmd->msgType = TSDB_MSG_TYPE_DROP_FUNCTION; + + pCmd->payloadLen = sizeof(SDropFuncMsg); + + return TSDB_CODE_SUCCESS; +} + + +int32_t tscBuildDropTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + SSqlCmd *pCmd = &pSql->cmd; + pCmd->payloadLen = sizeof(SCMDropTableMsg); + + if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { + tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + SCMDropTableMsg *pDropTableMsg = (SCMDropTableMsg*)pCmd->payload; + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); + tNameExtractFullName(&pTableMetaInfo->name, pDropTableMsg->name); + + pDropTableMsg->supertable = (pInfo->pMiscInfo->tableType == TSDB_SUPER_TABLE)? 1:0; + pDropTableMsg->igNotExists = pInfo->pMiscInfo->existsCheck ? 1 : 0; + pCmd->msgType = TSDB_MSG_TYPE_DROP_TABLE; + return TSDB_CODE_SUCCESS; +} + +int32_t tscBuildDropDnodeMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + SSqlCmd *pCmd = &pSql->cmd; + + char dnodeEp[TSDB_EP_LEN] = {0}; + tstrncpy(dnodeEp, pCmd->payload, TSDB_EP_LEN); + + pCmd->payloadLen = sizeof(SDropDnodeMsg); + if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { + tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + SDropDnodeMsg * pDrop = (SDropDnodeMsg *)pCmd->payload; + tstrncpy(pDrop->ep, dnodeEp, tListLen(pDrop->ep)); + pCmd->msgType = TSDB_MSG_TYPE_DROP_DNODE; + + return TSDB_CODE_SUCCESS; +} + +int32_t tscBuildDropUserAcctMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + SSqlCmd *pCmd = &pSql->cmd; + + char user[TSDB_USER_LEN] = {0}; + tstrncpy(user, pCmd->payload, TSDB_USER_LEN); + + pCmd->payloadLen = sizeof(SDropUserMsg); + pCmd->msgType = (pInfo->type == TSDB_SQL_DROP_USER)? TSDB_MSG_TYPE_DROP_USER:TSDB_MSG_TYPE_DROP_ACCT; + + if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { + tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + SDropUserMsg *pDropMsg = (SDropUserMsg *)pCmd->payload; + tstrncpy(pDropMsg->user, user, tListLen(user)); + + return TSDB_CODE_SUCCESS; +} + +int32_t tscBuildUseDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + SSqlCmd *pCmd = &pSql->cmd; + pCmd->payloadLen = sizeof(SUseDbMsg); + + if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { + tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + SUseDbMsg *pUseDbMsg = (SUseDbMsg *)pCmd->payload; + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); + tNameExtractFullName(&pTableMetaInfo->name, pUseDbMsg->db); + pCmd->msgType = TSDB_MSG_TYPE_USE_DB; + + return TSDB_CODE_SUCCESS; +} + +int32_t tscBuildSyncDbReplicaMsg(SSqlObj* pSql, SSqlInfo *pInfo) { + SSqlCmd *pCmd = &pSql->cmd; + pCmd->payloadLen = sizeof(SSyncDbMsg); + + if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { + tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + SSyncDbMsg *pSyncMsg = (SSyncDbMsg *)pCmd->payload; + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); + tNameExtractFullName(&pTableMetaInfo->name, pSyncMsg->db); + pCmd->msgType = TSDB_MSG_TYPE_SYNC_DB; + + return TSDB_CODE_SUCCESS; +} + +int32_t tscBuildShowMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + SSqlCmd *pCmd = &pSql->cmd; + pCmd->msgType = TSDB_MSG_TYPE_SHOW; + pCmd->payloadLen = sizeof(SShowMsg) + 100; + + if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { + tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + SShowInfo *pShowInfo = &pInfo->pMiscInfo->showOpt; + SShowMsg *pShowMsg = (SShowMsg *)pCmd->payload; + + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); + if (pShowInfo->showType == TSDB_MGMT_TABLE_FUNCTION) { + pShowMsg->type = pShowInfo->showType; + pShowMsg->payloadLen = 0; + pCmd->payloadLen = sizeof(SShowMsg); + + return TSDB_CODE_SUCCESS; + } + + if (tNameIsEmpty(&pTableMetaInfo->name)) { + char *p = cloneCurrentDBName(pSql); + tstrncpy(pShowMsg->db, p, sizeof(pShowMsg->db)); + tfree(p); + } else { + tNameGetFullDbName(&pTableMetaInfo->name, pShowMsg->db); + } + + pShowMsg->type = pShowInfo->showType; + + if (pShowInfo->showType != TSDB_MGMT_TABLE_VNODES) { + SStrToken *pPattern = &pShowInfo->pattern; + if (pPattern->type > 0) { // only show tables support wildcard query + strncpy(pShowMsg->payload, pPattern->z, pPattern->n); + pShowMsg->payloadLen = htons(pPattern->n); + } + } else { + SStrToken *pEpAddr = &pShowInfo->prefix; + assert(pEpAddr->n > 0 && pEpAddr->type > 0); + + strncpy(pShowMsg->payload, pEpAddr->z, pEpAddr->n); + pShowMsg->payloadLen = htons(pEpAddr->n); + } + + pCmd->payloadLen = sizeof(SShowMsg) + htons(pShowMsg->payloadLen); + return TSDB_CODE_SUCCESS; +} + +int32_t tscBuildKillMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + SSqlCmd *pCmd = &pSql->cmd; + pCmd->payloadLen = sizeof(SKillQueryMsg); + + switch (pCmd->command) { + case TSDB_SQL_KILL_QUERY: + pCmd->msgType = TSDB_MSG_TYPE_KILL_QUERY; + break; + case TSDB_SQL_KILL_CONNECTION: + pCmd->msgType = TSDB_MSG_TYPE_KILL_CONN; + break; + case TSDB_SQL_KILL_STREAM: + pCmd->msgType = TSDB_MSG_TYPE_KILL_STREAM; + break; + } + return TSDB_CODE_SUCCESS; +} + +int tscEstimateCreateTableMsgLength(SSqlObj *pSql, SSqlInfo *pInfo) { + SSqlCmd *pCmd = &(pSql->cmd); + int32_t size = minMsgSize() + sizeof(SCMCreateTableMsg) + sizeof(SCreateTableMsg); + + SCreateTableSql *pCreateTableInfo = pInfo->pCreateTableInfo; + if (pCreateTableInfo->type == TSQL_CREATE_TABLE_FROM_STABLE) { + int32_t numOfTables = (int32_t)taosArrayGetSize(pInfo->pCreateTableInfo->childTableInfo); + size += numOfTables * (sizeof(SCreateTableMsg) + TSDB_MAX_TAGS_LEN); + } else { + size += sizeof(SSchema) * (pCmd->numOfCols + pCmd->count); + } + + if (pCreateTableInfo->pSelect != NULL) { + size += (pCreateTableInfo->pSelect->sqlstr.n + 1); + } + + return size + TSDB_EXTRA_PAYLOAD_SIZE; +} + +int tscBuildCreateTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + int msgLen = 0; + int size = 0; + SSchema *pSchema; + SSqlCmd *pCmd = &pSql->cmd; + + SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); + STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + + // Reallocate the payload size + size = tscEstimateCreateTableMsgLength(pSql, pInfo); + if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, size)) { + tscError("0x%"PRIx64" failed to malloc for create table msg", pSql->self); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + SCMCreateTableMsg *pCreateTableMsg = (SCMCreateTableMsg *)pCmd->payload; + + SCreateTableMsg* pCreateMsg = (SCreateTableMsg*)((char*) pCreateTableMsg + sizeof(SCMCreateTableMsg)); + char* pMsg = NULL; + + int8_t type = pInfo->pCreateTableInfo->type; + if (type == TSQL_CREATE_TABLE_FROM_STABLE) { // create by using super table, tags value + SArray* list = pInfo->pCreateTableInfo->childTableInfo; + + int32_t numOfTables = (int32_t) taosArrayGetSize(list); + pCreateTableMsg->numOfTables = htonl(numOfTables); + + pMsg = (char*) pCreateMsg; + for(int32_t i = 0; i < numOfTables; ++i) { + SCreateTableMsg* pCreate = (SCreateTableMsg*) pMsg; + + pCreate->numOfColumns = htons(pCmd->numOfCols); + pCreate->numOfTags = htons(pCmd->count); + pMsg += sizeof(SCreateTableMsg); + + SCreatedTableInfo* p = taosArrayGet(list, i); + strcpy(pCreate->tableName, p->fullname); + pCreate->igExists = (p->igExist)? 1 : 0; + + // use dbinfo from table id without modifying current db info + pMsg = serializeTagData(&p->tagdata, pMsg); + + int32_t len = (int32_t)(pMsg - (char*) pCreate); + pCreate->len = htonl(len); + } + } else { // create (super) table + pCreateTableMsg->numOfTables = htonl(1); // only one table will be created + + int32_t code = tNameExtractFullName(&pTableMetaInfo->name, pCreateMsg->tableName); + assert(code == 0); + + SCreateTableSql *pCreateTable = pInfo->pCreateTableInfo; + + pCreateMsg->igExists = pCreateTable->existCheck ? 1 : 0; + pCreateMsg->numOfColumns = htons(pCmd->numOfCols); + pCreateMsg->numOfTags = htons(pCmd->count); + + pCreateMsg->sqlLen = 0; + pMsg = (char *)pCreateMsg->schema; + + pSchema = (SSchema *)pCreateMsg->schema; + + for (int i = 0; i < pCmd->numOfCols + pCmd->count; ++i) { + TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); + + pSchema->type = pField->type; + strcpy(pSchema->name, pField->name); + pSchema->bytes = htons(pField->bytes); + + pSchema++; + } + + pMsg = (char *)pSchema; + if (type == TSQL_CREATE_STREAM) { // check if it is a stream sql + SSqlNode *pQuerySql = pInfo->pCreateTableInfo->pSelect; + + strncpy(pMsg, pQuerySql->sqlstr.z, pQuerySql->sqlstr.n + 1); + pCreateMsg->sqlLen = htons(pQuerySql->sqlstr.n + 1); + pMsg += pQuerySql->sqlstr.n + 1; + } + } + + tscFieldInfoClear(&pQueryInfo->fieldsInfo); + + msgLen = (int32_t)(pMsg - (char*)pCreateTableMsg); + pCreateTableMsg->contLen = htonl(msgLen); + pCmd->payloadLen = msgLen; + pCmd->msgType = TSDB_MSG_TYPE_CREATE_TABLE; + + assert(msgLen + minMsgSize() <= size); + return TSDB_CODE_SUCCESS; +} + +int tscEstimateAlterTableMsgLength(SSqlCmd *pCmd) { + SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); + return minMsgSize() + sizeof(SAlterTableMsg) + sizeof(SSchema) * tscNumOfFields(pQueryInfo) + TSDB_EXTRA_PAYLOAD_SIZE; +} + +int tscBuildAlterTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + char *pMsg; + int msgLen = 0; + + SSqlCmd *pCmd = &pSql->cmd; + SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); + + STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + + SAlterTableInfo *pAlterInfo = pInfo->pAlterInfo; + int size = tscEstimateAlterTableMsgLength(pCmd); + if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, size)) { + tscError("0x%"PRIx64" failed to malloc for alter table msg", pSql->self); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + SAlterTableMsg *pAlterTableMsg = (SAlterTableMsg *)pCmd->payload; + + tNameExtractFullName(&pTableMetaInfo->name, pAlterTableMsg->tableFname); + pAlterTableMsg->type = htons(pAlterInfo->type); + + pAlterTableMsg->numOfCols = htons(tscNumOfFields(pQueryInfo)); + SSchema *pSchema = pAlterTableMsg->schema; + for (int i = 0; i < tscNumOfFields(pQueryInfo); ++i) { + TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); + + pSchema->type = pField->type; + strcpy(pSchema->name, pField->name); + pSchema->bytes = htons(pField->bytes); + pSchema++; + } + + pMsg = (char *)pSchema; + pAlterTableMsg->tagValLen = htonl(pAlterInfo->tagData.dataLen); + if (pAlterInfo->tagData.dataLen > 0) { + memcpy(pMsg, pAlterInfo->tagData.data, pAlterInfo->tagData.dataLen); + } + pMsg += pAlterInfo->tagData.dataLen; + + msgLen = (int32_t)(pMsg - (char*)pAlterTableMsg); + + pCmd->payloadLen = msgLen; + pCmd->msgType = TSDB_MSG_TYPE_ALTER_TABLE; + + assert(msgLen + minMsgSize() <= size); + + return TSDB_CODE_SUCCESS; +} + +int tscBuildUpdateTagMsg(SSqlObj* pSql, SSqlInfo *pInfo) { + SSqlCmd* pCmd = &pSql->cmd; + pCmd->msgType = TSDB_MSG_TYPE_UPDATE_TAG_VAL; + + SUpdateTableTagValMsg* pUpdateMsg = (SUpdateTableTagValMsg*) pCmd->payload; + pCmd->payloadLen = htonl(pUpdateMsg->head.contLen); + + SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); + STableMeta *pTableMeta = tscGetMetaInfo(pQueryInfo, 0)->pTableMeta; + + SNewVgroupInfo vgroupInfo = {.vgId = -1}; + taosHashGetClone(tscVgroupMap, &pTableMeta->vgId, sizeof(pTableMeta->vgId), NULL, &vgroupInfo); + assert(vgroupInfo.vgId > 0); + + tscDumpEpSetFromVgroupInfo(&pSql->epSet, &vgroupInfo); + + return TSDB_CODE_SUCCESS; +} + +int tscAlterDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + SSqlCmd *pCmd = &pSql->cmd; + pCmd->payloadLen = sizeof(SAlterDbMsg); + pCmd->msgType = (pInfo->pMiscInfo->dbOpt.dbType == TSDB_DB_TYPE_DEFAULT) ? TSDB_MSG_TYPE_ALTER_DB : TSDB_MSG_TYPE_ALTER_TP; + + SAlterDbMsg *pAlterDbMsg = (SAlterDbMsg* )pCmd->payload; + pAlterDbMsg->dbType = -1; + + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); + tNameExtractFullName(&pTableMetaInfo->name, pAlterDbMsg->db); + + return TSDB_CODE_SUCCESS; +} +int tscBuildCompactMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + if (pInfo->list == NULL || taosArrayGetSize(pInfo->list) <= 0) { + return TSDB_CODE_TSC_INVALID_OPERATION; + } + STscObj *pObj = pSql->pTscObj; + SSqlCmd *pCmd = &pSql->cmd; + SArray *pList = pInfo->list; + int32_t size = (int32_t)taosArrayGetSize(pList); + + int32_t *result = malloc(sizeof(int32_t) * size); + if (result == NULL) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + for (int32_t i = 0; i < size; i++) { + tSqlExprItem* pSub = taosArrayGet(pList, i); + tVariant* pVar = &pSub->pNode->value; + if (pVar->nType >= TSDB_DATA_TYPE_TINYINT && pVar->nType <= TSDB_DATA_TYPE_BIGINT) { + result[i] = (int32_t)(pVar->i64); + } else { + free(result); + return TSDB_CODE_TSC_INVALID_OPERATION; + } + } + + int count = removeDupVgid(result, size); + pCmd->payloadLen = sizeof(SCompactMsg) + count * sizeof(int32_t); + pCmd->msgType = TSDB_MSG_TYPE_COMPACT_VNODE; + + if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { + tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); + free(result); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + SCompactMsg *pCompactMsg = (SCompactMsg *)pCmd->payload; + + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); + + if (tNameIsEmpty(&pTableMetaInfo->name)) { + pthread_mutex_lock(&pObj->mutex); + tstrncpy(pCompactMsg->db, pObj->db, sizeof(pCompactMsg->db)); + pthread_mutex_unlock(&pObj->mutex); + } else { + tNameGetFullDbName(&pTableMetaInfo->name, pCompactMsg->db); + } + + pCompactMsg->numOfVgroup = htons(count); + for (int32_t i = 0; i < count; i++) { + pCompactMsg->vgid[i] = htons(result[i]); + } + free(result); + + return TSDB_CODE_SUCCESS; +} + +int tscBuildRetrieveFromMgmtMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + SSqlCmd *pCmd = &pSql->cmd; + pCmd->msgType = TSDB_MSG_TYPE_SHOW_RETRIEVE; + pCmd->payloadLen = sizeof(SRetrieveTableMsg); + + if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { + tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); + SRetrieveTableMsg *pRetrieveMsg = (SRetrieveTableMsg*)pCmd->payload; + pRetrieveMsg->qId = htobe64(pSql->res.qId); + pRetrieveMsg->free = htons(pQueryInfo->type); + + return TSDB_CODE_SUCCESS; +} + +/* + * this function can only be called once. + * by using pRes->rspType to denote its status + * + * if pRes->rspType is 1, no more result + */ +static int tscLocalResultCommonBuilder(SSqlObj *pSql, int32_t numOfRes) { + SSqlRes *pRes = &pSql->res; + SSqlCmd *pCmd = &pSql->cmd; + + pRes->code = TSDB_CODE_SUCCESS; + if (pRes->rspType == 0) { + pRes->numOfRows = numOfRes; + pRes->row = 0; + pRes->rspType = 1; + + SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); + if (tscCreateResPointerInfo(pRes, pQueryInfo) != TSDB_CODE_SUCCESS) { + return pRes->code; + } + + tscSetResRawPtr(pRes, pQueryInfo); + } else { + tscResetForNextRetrieve(pRes); + } + + uint8_t code = pSql->res.code; + if (pSql->fp) { + if (code == TSDB_CODE_SUCCESS) { + (*pSql->fp)(pSql->param, pSql, pSql->res.numOfRows); + } else { + tscAsyncResultOnError(pSql); + } + } + + return code; +} + +int tscProcessDescribeTableRsp(SSqlObj *pSql) { + SSqlCmd * pCmd = &pSql->cmd; + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); + + STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); + + int32_t numOfRes = tinfo.numOfColumns + tinfo.numOfTags; + return tscLocalResultCommonBuilder(pSql, numOfRes); +} + +int tscProcessLocalRetrieveRsp(SSqlObj *pSql) { + int32_t numOfRes = 1; + pSql->res.completed = true; + return tscLocalResultCommonBuilder(pSql, numOfRes); +} + +int tscProcessRetrieveGlobalMergeRsp(SSqlObj *pSql) { + SSqlRes *pRes = &pSql->res; + SSqlCmd* pCmd = &pSql->cmd; + + int32_t code = pRes->code; + if (pRes->code != TSDB_CODE_SUCCESS) { + tscAsyncResultOnError(pSql); + return code; + } + + if (pRes->pMerger == NULL) { // no result from subquery, so abort here directly. + (*pSql->fp)(pSql->param, pSql, pRes->numOfRows); + return code; + } + + // global aggregation may be the upstream for parent query + SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); + if (pQueryInfo->pQInfo == NULL) { + STableGroupInfo tableGroupInfo = {.numOfTables = 1, .pGroupList = taosArrayInit(1, POINTER_BYTES),}; + tableGroupInfo.map = taosHashInit(1, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); + + STableKeyInfo tableKeyInfo = {.pTable = NULL, .lastKey = INT64_MIN}; + + SArray* group = taosArrayInit(1, sizeof(STableKeyInfo)); + taosArrayPush(group, &tableKeyInfo); + taosArrayPush(tableGroupInfo.pGroupList, &group); + + tscDebug("0x%"PRIx64" create QInfo 0x%"PRIx64" to execute query processing", pSql->self, pSql->self); + pQueryInfo->pQInfo = createQInfoFromQueryNode(pQueryInfo, &tableGroupInfo, NULL, NULL, pRes->pMerger, MERGE_STAGE, pSql->self); + } + + uint64_t localQueryId = pSql->self; + qTableQuery(pQueryInfo->pQInfo, &localQueryId); + convertQueryResult(pRes, pQueryInfo, pSql->self, true); + + code = pRes->code; + if (pRes->code == TSDB_CODE_SUCCESS) { + (*pSql->fp)(pSql->param, pSql, pRes->numOfRows); + } else { + tscAsyncResultOnError(pSql); + } + + return code; +} + +int tscProcessEmptyResultRsp(SSqlObj *pSql) { return tscLocalResultCommonBuilder(pSql, 0); } + +int tscBuildConnectMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + STscObj *pObj = pSql->pTscObj; + SSqlCmd *pCmd = &pSql->cmd; + pCmd->msgType = TSDB_MSG_TYPE_CONNECT; + pCmd->payloadLen = sizeof(SConnectMsg); + + if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { + tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + SConnectMsg *pConnect = (SConnectMsg*)pCmd->payload; + + // TODO refactor full_name + char *db; // ugly code to move the space + + pthread_mutex_lock(&pObj->mutex); + db = strstr(pObj->db, TS_PATH_DELIMITER); + + db = (db == NULL) ? pObj->db : db + 1; + tstrncpy(pConnect->db, db, sizeof(pConnect->db)); + pthread_mutex_unlock(&pObj->mutex); + + tstrncpy(pConnect->clientVersion, version, sizeof(pConnect->clientVersion)); + tstrncpy(pConnect->msgVersion, "", sizeof(pConnect->msgVersion)); + + pConnect->pid = htonl(taosGetPId()); + taosGetCurrentAPPName(pConnect->appName, NULL); + + return TSDB_CODE_SUCCESS; +} + +/** + * multi table meta req pkg format: + * |SMultiTableInfoMsg | tableId0 | tableId1 | tableId2 | ...... + * 4B + **/ +int tscBuildMultiTableMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + SSqlCmd *pCmd = &pSql->cmd; + + pCmd->msgType = TSDB_MSG_TYPE_TABLES_META; + assert(pCmd->payloadLen + minMsgSize() <= pCmd->allocSize); + + tscDebug("0x%"PRIx64" build load multi-tablemeta msg completed, numOfTables:%d, msg size:%d", pSql->self, pCmd->count, + pCmd->payloadLen); + + return pCmd->payloadLen; +} + +int tscBuildSTableVgroupMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + SSqlCmd *pCmd = &pSql->cmd; + + char* pMsg = pCmd->payload; + SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); + + SSTableVgroupMsg *pStableVgroupMsg = (SSTableVgroupMsg *)pMsg; + pStableVgroupMsg->numOfTables = htonl(pQueryInfo->numOfTables); + pMsg += sizeof(SSTableVgroupMsg); + + for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, i); + int32_t code = tNameExtractFullName(&pTableMetaInfo->name, pMsg); + assert(code == TSDB_CODE_SUCCESS); + + pMsg += TSDB_TABLE_FNAME_LEN; + } + + pCmd->msgType = TSDB_MSG_TYPE_STABLE_VGROUP; + pCmd->payloadLen = (int32_t)(pMsg - pCmd->payload); + + return TSDB_CODE_SUCCESS; +} + +int tscBuildRetrieveFuncMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + SSqlCmd *pCmd = &pSql->cmd; + + char *pMsg = pCmd->payload; + SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); + int32_t numOfFuncs = (int32_t)taosArrayGetSize(pQueryInfo->pUdfInfo); + + SRetrieveFuncMsg *pRetrieveFuncMsg = (SRetrieveFuncMsg *)pMsg; + pRetrieveFuncMsg->num = htonl(numOfFuncs); + + pMsg += sizeof(SRetrieveFuncMsg); + for(int32_t i = 0; i < numOfFuncs; ++i) { + SUdfInfo* pUdf = taosArrayGet(pQueryInfo->pUdfInfo, i); + STR_TO_NET_VARSTR(pMsg, pUdf->name); + pMsg += varDataNetTLen(pMsg); + } + + pCmd->msgType = TSDB_MSG_TYPE_RETRIEVE_FUNC; + pCmd->payloadLen = (int32_t)(pMsg - pCmd->payload); + + return TSDB_CODE_SUCCESS; +} + +int tscBuildHeartBeatMsg(SSqlObj *pSql, SSqlInfo *pInfo) { + SSqlCmd *pCmd = &pSql->cmd; + STscObj *pObj = pSql->pTscObj; + + pthread_mutex_lock(&pObj->mutex); + + int32_t numOfQueries = 2; + SSqlObj *tpSql = pObj->sqlList; + while (tpSql) { + tpSql = tpSql->next; + numOfQueries++; + } + + int32_t numOfStreams = 2; + SSqlStream *pStream = pObj->streamList; + while (pStream) { + pStream = pStream->next; + numOfStreams++; + } + + int size = numOfQueries * sizeof(SQueryDesc) + numOfStreams * sizeof(SStreamDesc) + sizeof(SHeartBeatMsg) + 100; + if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, size)) { + pthread_mutex_unlock(&pObj->mutex); + tscError("0x%"PRIx64" failed to create heartbeat msg", pSql->self); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + // TODO the expired hb and client can not be identified by server till now. + SHeartBeatMsg *pHeartbeat = (SHeartBeatMsg *)pCmd->payload; + tstrncpy(pHeartbeat->clientVer, version, tListLen(pHeartbeat->clientVer)); + + pHeartbeat->numOfQueries = numOfQueries; + pHeartbeat->numOfStreams = numOfStreams; + + pHeartbeat->pid = htonl(taosGetPId()); + taosGetCurrentAPPName(pHeartbeat->appName, NULL); + + int msgLen = tscBuildQueryStreamDesc(pHeartbeat, pObj); + + pthread_mutex_unlock(&pObj->mutex); + + pCmd->payloadLen = msgLen; + pCmd->msgType = TSDB_MSG_TYPE_HEARTBEAT; + + return TSDB_CODE_SUCCESS; +} + +static int32_t tableMetaMsgConvert(STableMetaMsg* pMetaMsg) { + pMetaMsg->tid = htonl(pMetaMsg->tid); + pMetaMsg->sversion = htons(pMetaMsg->sversion); + pMetaMsg->tversion = htons(pMetaMsg->tversion); + pMetaMsg->vgroup.vgId = htonl(pMetaMsg->vgroup.vgId); + + pMetaMsg->uid = htobe64(pMetaMsg->uid); + pMetaMsg->numOfColumns = htons(pMetaMsg->numOfColumns); + + if ((pMetaMsg->tableType != TSDB_SUPER_TABLE) && + (pMetaMsg->tid <= 0 || pMetaMsg->vgroup.vgId < 2 || pMetaMsg->vgroup.numOfEps <= 0)) { + tscError("invalid value in table numOfEps:%d, vgId:%d tid:%d, name:%s", pMetaMsg->vgroup.numOfEps, pMetaMsg->vgroup.vgId, + pMetaMsg->tid, pMetaMsg->tableFname); + return TSDB_CODE_TSC_INVALID_VALUE; + } + + if (pMetaMsg->numOfTags > TSDB_MAX_TAGS) { + tscError("invalid numOfTags:%d", pMetaMsg->numOfTags); + return TSDB_CODE_TSC_INVALID_VALUE; + } + + if (pMetaMsg->numOfColumns > TSDB_MAX_COLUMNS || pMetaMsg->numOfColumns <= 0) { + tscError("invalid numOfColumns:%d", pMetaMsg->numOfColumns); + return TSDB_CODE_TSC_INVALID_VALUE; + } + + for (int i = 0; i < pMetaMsg->vgroup.numOfEps; ++i) { + pMetaMsg->vgroup.epAddr[i].port = htons(pMetaMsg->vgroup.epAddr[i].port); + } + + SSchema* pSchema = pMetaMsg->schema; + + int32_t numOfTotalCols = pMetaMsg->numOfColumns + pMetaMsg->numOfTags; + for (int i = 0; i < numOfTotalCols; ++i) { + pSchema->bytes = htons(pSchema->bytes); + pSchema->colId = htons(pSchema->colId); + + if (pSchema->colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { + assert(i == 0); + } + + pSchema++; + } + + return TSDB_CODE_SUCCESS; +} + +// update the vgroupInfo if needed +static void doUpdateVgroupInfo(int32_t vgId, SVgroupMsg *pVgroupMsg) { + assert(vgId > 0); + + SNewVgroupInfo vgroupInfo = {.inUse = -1}; + taosHashGetClone(tscVgroupMap, &vgId, sizeof(vgId), NULL, &vgroupInfo); + + // vgroup info exists, compare with it + if (((vgroupInfo.inUse >= 0) && !vgroupInfoIdentical(&vgroupInfo, pVgroupMsg)) || (vgroupInfo.inUse < 0)) { + vgroupInfo = createNewVgroupInfo(pVgroupMsg); + taosHashPut(tscVgroupMap, &vgId, sizeof(vgId), &vgroupInfo, sizeof(vgroupInfo)); + tscDebug("add/update new VgroupInfo, vgId:%d, total cached:%d", vgId, (int32_t) taosHashGetSize(tscVgroupMap)); + } +} + +static void doAddTableMetaToLocalBuf(STableMeta* pTableMeta, STableMetaMsg* pMetaMsg, bool updateSTable) { + if (pTableMeta->tableType == TSDB_CHILD_TABLE) { + // add or update the corresponding super table meta data info + int32_t len = (int32_t) strnlen(pTableMeta->sTableName, TSDB_TABLE_FNAME_LEN); + + // The super tableMeta already exists, create it according to tableMeta and add it to hash map + if (updateSTable) { + STableMeta* pSupTableMeta = createSuperTableMeta(pMetaMsg); + uint32_t size = tscGetTableMetaSize(pSupTableMeta); + int32_t code = taosHashPut(tscTableMetaMap, pTableMeta->sTableName, len, pSupTableMeta, size); + assert(code == TSDB_CODE_SUCCESS); + + tfree(pSupTableMeta); + } + + CChildTableMeta* cMeta = tscCreateChildMeta(pTableMeta); + taosHashPut(tscTableMetaMap, pMetaMsg->tableFname, strlen(pMetaMsg->tableFname), cMeta, sizeof(CChildTableMeta)); + tfree(cMeta); + } else { + uint32_t s = tscGetTableMetaSize(pTableMeta); + taosHashPut(tscTableMetaMap, pMetaMsg->tableFname, strlen(pMetaMsg->tableFname), pTableMeta, s); + } +} + +int tscProcessTableMetaRsp(SSqlObj *pSql) { + STableMetaMsg *pMetaMsg = (STableMetaMsg *)pSql->res.pRsp; + int32_t code = tableMetaMsgConvert(pMetaMsg); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0); + assert(pTableMetaInfo->pTableMeta == NULL); + + STableMeta* pTableMeta = tscCreateTableMetaFromMsg(pMetaMsg); + if (pTableMeta == NULL){ + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + if (!tIsValidSchema(pTableMeta->schema, pTableMeta->tableInfo.numOfColumns, pTableMeta->tableInfo.numOfTags)) { + tscError("0x%"PRIx64" invalid table meta from mnode, name:%s", pSql->self, tNameGetTableName(&pTableMetaInfo->name)); + tfree(pTableMeta); + return TSDB_CODE_TSC_INVALID_VALUE; + } + + char name[TSDB_TABLE_FNAME_LEN] = {0}; + tNameExtractFullName(&pTableMetaInfo->name, name); + assert(strncmp(pMetaMsg->tableFname, name, tListLen(pMetaMsg->tableFname)) == 0); + + doAddTableMetaToLocalBuf(pTableMeta, pMetaMsg, true); + if (pTableMeta->tableType != TSDB_SUPER_TABLE) { + doUpdateVgroupInfo(pTableMeta->vgId, &pMetaMsg->vgroup); + } + + tscDebug("0x%"PRIx64" recv table meta, uid:%" PRIu64 ", tid:%d, name:%s, numOfCols:%d, numOfTags:%d", pSql->self, + pTableMeta->id.uid, pTableMeta->id.tid, tNameGetTableName(&pTableMetaInfo->name), pTableMeta->tableInfo.numOfColumns, + pTableMeta->tableInfo.numOfTags); + + free(pTableMeta); + return TSDB_CODE_SUCCESS; +} + +static SArray* createVgroupIdListFromMsg(char* pMsg, SHashObj* pSet, char* name, int32_t* size, uint64_t id) { + SVgroupsMsg *pVgroupMsg = (SVgroupsMsg *)pMsg; + + pVgroupMsg->numOfVgroups = htonl(pVgroupMsg->numOfVgroups); + *size = (int32_t)(sizeof(SVgroupMsg) * pVgroupMsg->numOfVgroups + sizeof(SVgroupsMsg)); + + SArray* vgroupIdList = taosArrayInit(pVgroupMsg->numOfVgroups, sizeof(int32_t)); + + if (pVgroupMsg->numOfVgroups <= 0) { + tscDebug("0x%" PRIx64 " empty vgroup id list, no corresponding tables for stable:%s", id, name); + } else { + // just init, no need to lock + for (int32_t j = 0; j < pVgroupMsg->numOfVgroups; ++j) { + SVgroupMsg *vmsg = &pVgroupMsg->vgroups[j]; + vmsg->vgId = htonl(vmsg->vgId); + for (int32_t k = 0; k < vmsg->numOfEps; ++k) { + vmsg->epAddr[k].port = htons(vmsg->epAddr[k].port); + } + + taosArrayPush(vgroupIdList, &vmsg->vgId); + + if (taosHashGet(pSet, &vmsg->vgId, sizeof(vmsg->vgId)) == NULL) { + taosHashPut(pSet, &vmsg->vgId, sizeof(vmsg->vgId), "", 0); + doUpdateVgroupInfo(vmsg->vgId, vmsg); + } + } + } + + return vgroupIdList; +} + +static SVgroupsInfo* createVgroupInfoFromMsg(char* pMsg, int32_t* size, uint64_t id) { + SVgroupsMsg *pVgroupMsg = (SVgroupsMsg *)pMsg; + pVgroupMsg->numOfVgroups = htonl(pVgroupMsg->numOfVgroups); + + *size = (int32_t)(sizeof(SVgroupMsg) * pVgroupMsg->numOfVgroups + sizeof(SVgroupsMsg)); + + size_t vgroupsz = sizeof(SVgroupMsg) * pVgroupMsg->numOfVgroups + sizeof(SVgroupsInfo); + SVgroupsInfo *pVgroupInfo = calloc(1, vgroupsz); + assert(pVgroupInfo != NULL); + + pVgroupInfo->numOfVgroups = pVgroupMsg->numOfVgroups; + if (pVgroupInfo->numOfVgroups <= 0) { + tscDebug("0x%" PRIx64 " empty vgroup info, no corresponding tables for stable", id); + } else { + for (int32_t j = 0; j < pVgroupInfo->numOfVgroups; ++j) { + // just init, no need to lock + SVgroupMsg *pVgroup = &pVgroupInfo->vgroups[j]; + + SVgroupMsg *vmsg = &pVgroupMsg->vgroups[j]; + vmsg->vgId = htonl(vmsg->vgId); + for (int32_t k = 0; k < vmsg->numOfEps; ++k) { + vmsg->epAddr[k].port = htons(vmsg->epAddr[k].port); + } + + pVgroup->numOfEps = vmsg->numOfEps; + pVgroup->vgId = vmsg->vgId; + for (int32_t k = 0; k < vmsg->numOfEps; ++k) { + pVgroup->epAddr[k].port = vmsg->epAddr[k].port; + tstrncpy(pVgroup->epAddr[k].fqdn, vmsg->epAddr[k].fqdn, TSDB_FQDN_LEN); +// pVgroup->epAddr[k].fqdn = strndup(vmsg->epAddr[k].fqdn, TSDB_FQDN_LEN); + } + + doUpdateVgroupInfo(pVgroup->vgId, vmsg); + } + } + + return pVgroupInfo; +} + +int tscProcessRetrieveFuncRsp(SSqlObj* pSql) { + SSqlCmd* pCmd = &pSql->cmd; + SUdfFuncMsg* pFuncMsg = (SUdfFuncMsg *)pSql->res.pRsp; + SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); + + pFuncMsg->num = htonl(pFuncMsg->num); + assert(pFuncMsg->num == taosArrayGetSize(pQueryInfo->pUdfInfo)); + + char* pMsg = pFuncMsg->content; + for(int32_t i = 0; i < pFuncMsg->num; ++i) { + SFunctionInfoMsg* pFunc = (SFunctionInfoMsg*) pMsg; + + for(int32_t j = 0; j < pFuncMsg->num; ++j) { + SUdfInfo* pUdfInfo = taosArrayGet(pQueryInfo->pUdfInfo, j); + if (strcmp(pUdfInfo->name, pFunc->name) != 0) { + continue; + } + + if (pUdfInfo->content) { + continue; + } + + pUdfInfo->resBytes = htons(pFunc->resBytes); + pUdfInfo->resType = pFunc->resType; + pUdfInfo->funcType = htonl(pFunc->funcType); + pUdfInfo->contLen = htonl(pFunc->len); + pUdfInfo->bufSize = htonl(pFunc->bufSize); + + pUdfInfo->content = malloc(pUdfInfo->contLen); + memcpy(pUdfInfo->content, pFunc->content, pUdfInfo->contLen); + + pMsg += sizeof(SFunctionInfoMsg) + pUdfInfo->contLen; + } + } + + // master sqlObj locates in param + SSqlObj* parent = (SSqlObj*)taosAcquireRef(tscObjRef, (int64_t)pSql->param); + if(parent == NULL) { + return pSql->res.code; + } + + SQueryInfo* parQueryInfo = tscGetQueryInfo(&parent->cmd); + + assert(parent->signature == parent && (int64_t)pSql->param == parent->self); + taosArrayDestroy(parQueryInfo->pUdfInfo); + + parQueryInfo->pUdfInfo = pQueryInfo->pUdfInfo; // assigned to parent sql obj. + pQueryInfo->pUdfInfo = NULL; + return TSDB_CODE_SUCCESS; +} + +int tscProcessMultiTableMetaRsp(SSqlObj *pSql) { + char *rsp = pSql->res.pRsp; + + SMultiTableMeta *pMultiMeta = (SMultiTableMeta *)rsp; + pMultiMeta->numOfTables = htonl(pMultiMeta->numOfTables); + pMultiMeta->numOfVgroup = htonl(pMultiMeta->numOfVgroup); + pMultiMeta->numOfUdf = htonl(pMultiMeta->numOfUdf); + + rsp += sizeof(SMultiTableMeta); + + SSqlObj* pParentSql = (SSqlObj*)taosAcquireRef(tscObjRef, (int64_t)pSql->param); + if(pParentSql == NULL) { + return pSql->res.code; + } + + SSqlCmd *pParentCmd = &pParentSql->cmd; + SHashObj *pSet = taosHashInit(pMultiMeta->numOfVgroup, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); + + char* buf = NULL; + char* pMsg = pMultiMeta->meta; + + // decompresss the message payload + if (pMultiMeta->compressed) { + buf = malloc(pMultiMeta->rawLen - sizeof(SMultiTableMeta)); + int32_t len = tsDecompressString(pMultiMeta->meta, pMultiMeta->contLen - sizeof(SMultiTableMeta), 1, + buf, pMultiMeta->rawLen - sizeof(SMultiTableMeta), ONE_STAGE_COMP, NULL, 0); + assert(len == pMultiMeta->rawLen - sizeof(SMultiTableMeta)); + + pMsg = buf; + } + + if (pParentCmd->pTableMetaMap == NULL) { + pParentCmd->pTableMetaMap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + } + + for (int32_t i = 0; i < pMultiMeta->numOfTables; i++) { + STableMetaMsg *pMetaMsg = (STableMetaMsg *)pMsg; + int32_t code = tableMetaMsgConvert(pMetaMsg); + if (code != TSDB_CODE_SUCCESS) { + taosHashCleanup(pSet); + taosReleaseRef(tscObjRef, pParentSql->self); + + tfree(buf); + return code; + } + + bool freeMeta = false; + STableMeta* pTableMeta = tscCreateTableMetaFromMsg(pMetaMsg); + if (!tIsValidSchema(pTableMeta->schema, pTableMeta->tableInfo.numOfColumns, pTableMeta->tableInfo.numOfTags)) { + tscError("0x%"PRIx64" invalid table meta from mnode, name:%s", pSql->self, pMetaMsg->tableFname); + tfree(pTableMeta); + taosHashCleanup(pSet); + taosReleaseRef(tscObjRef, pParentSql->self); + + tfree(buf); + return TSDB_CODE_TSC_INVALID_VALUE; + } + + if (pMultiMeta->metaClone == 1 || pTableMeta->tableType == TSDB_SUPER_TABLE) { + STableMetaVgroupInfo p = {.pTableMeta = pTableMeta,}; + size_t keyLen = strnlen(pMetaMsg->tableFname, TSDB_TABLE_FNAME_LEN); + void* t = taosHashGet(pParentCmd->pTableMetaMap, pMetaMsg->tableFname, keyLen); + assert(t == NULL); + + taosHashPut(pParentCmd->pTableMetaMap, pMetaMsg->tableFname, keyLen, &p, sizeof(STableMetaVgroupInfo)); + } else { + freeMeta = true; + } + + // for each super table, only update meta information once + bool updateStableMeta = false; + if (pTableMeta->tableType == TSDB_CHILD_TABLE && taosHashGet(pSet, &pMetaMsg->suid, sizeof(pMetaMsg->suid)) == NULL) { + updateStableMeta = true; + taosHashPut(pSet, &pTableMeta->suid, sizeof(pMetaMsg->suid), "", 0); + } + + // create the tableMeta and add it into the TableMeta map + doAddTableMetaToLocalBuf(pTableMeta, pMetaMsg, updateStableMeta); + + // for each vgroup, only update the information once. + int64_t vgId = pMetaMsg->vgroup.vgId; + if (pTableMeta->tableType != TSDB_SUPER_TABLE && taosHashGet(pSet, &vgId, sizeof(vgId)) == NULL) { + doUpdateVgroupInfo((int32_t) vgId, &pMetaMsg->vgroup); + taosHashPut(pSet, &vgId, sizeof(vgId), "", 0); + } + + pMsg += pMetaMsg->contLen; + if (freeMeta) { + tfree(pTableMeta); + } + } + + for(int32_t i = 0; i < pMultiMeta->numOfVgroup; ++i) { + char fname[TSDB_TABLE_FNAME_LEN] = {0}; + tstrncpy(fname, pMsg, TSDB_TABLE_FNAME_LEN); + size_t len = strnlen(fname, TSDB_TABLE_FNAME_LEN); + + pMsg += TSDB_TABLE_FNAME_LEN; + + STableMetaVgroupInfo* p = taosHashGet(pParentCmd->pTableMetaMap, fname, len); + assert(p != NULL); + + int32_t size = 0; + if (p->vgroupIdList!= NULL) { + taosArrayDestroy(p->vgroupIdList); + } + + p->vgroupIdList = createVgroupIdListFromMsg(pMsg, pSet, fname, &size, pSql->self); + + int32_t numOfVgId = (int32_t) taosArrayGetSize(p->vgroupIdList); + int32_t s = sizeof(tFilePage) + numOfVgId * sizeof(int32_t); + + tFilePage* idList = calloc(1, s); + idList->num = numOfVgId; + memcpy(idList->data, TARRAY_GET_START(p->vgroupIdList), numOfVgId * sizeof(int32_t)); + + void* idListInst = taosCachePut(tscVgroupListBuf, fname, len, idList, s, 5000); + taosCacheRelease(tscVgroupListBuf, (void*) &idListInst, false); + + tfree(idList); + pMsg += size; + } + + SQueryInfo* pQueryInfo = tscGetQueryInfo(pParentCmd); + if (pMultiMeta->numOfUdf > 0) { + assert(pQueryInfo->pUdfInfo != NULL); + } + + for(int32_t i = 0; i < pMultiMeta->numOfUdf; ++i) { + SFunctionInfoMsg* pFunc = (SFunctionInfoMsg*) pMsg; + + for(int32_t j = 0; j < pMultiMeta->numOfUdf; ++j) { + SUdfInfo* pUdfInfo = taosArrayGet(pQueryInfo->pUdfInfo, j); + if (strcmp(pUdfInfo->name, pFunc->name) != 0) { + continue; + } + + if (pUdfInfo->content) { + continue; + } + + pUdfInfo->resBytes = htons(pFunc->resBytes); + pUdfInfo->resType = pFunc->resType; + pUdfInfo->funcType = htonl(pFunc->funcType); + pUdfInfo->contLen = htonl(pFunc->len); + pUdfInfo->bufSize = htonl(pFunc->bufSize); + + pUdfInfo->content = malloc(pUdfInfo->contLen); + memcpy(pUdfInfo->content, pFunc->content, pUdfInfo->contLen); + + pMsg += sizeof(SFunctionInfoMsg) + pUdfInfo->contLen; + } + } + + pSql->res.code = TSDB_CODE_SUCCESS; + pSql->res.numOfTotal = pMultiMeta->numOfTables; + tscDebug("0x%"PRIx64" load multi-tableMeta from mnode, numOfTables:%d", pSql->self, pMultiMeta->numOfTables); + + taosHashCleanup(pSet); + taosReleaseRef(tscObjRef, pParentSql->self); + + tfree(buf); + return TSDB_CODE_SUCCESS; +} + +int tscProcessSTableVgroupRsp(SSqlObj *pSql) { + // master sqlObj locates in param + SSqlObj* parent = (SSqlObj*)taosAcquireRef(tscObjRef, (int64_t)pSql->param); + if(parent == NULL) { + return pSql->res.code; + } + + assert(parent->signature == parent && (int64_t)pSql->param == parent->self); + + SSqlRes* pRes = &pSql->res; + + // NOTE: the order of several table must be preserved. + SSTableVgroupRspMsg *pStableVgroup = (SSTableVgroupRspMsg *)pRes->pRsp; + pStableVgroup->numOfTables = htonl(pStableVgroup->numOfTables); + char *pMsg = pRes->pRsp + sizeof(SSTableVgroupRspMsg); + + SSqlCmd* pCmd = &parent->cmd; + SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); + + char fName[TSDB_TABLE_FNAME_LEN] = {0}; + for(int32_t i = 0; i < pStableVgroup->numOfTables; ++i) { + char* name = pMsg; + pMsg += TSDB_TABLE_FNAME_LEN; + + STableMetaInfo *pInfo = NULL; + for(int32_t j = 0; j < pQueryInfo->numOfTables; ++j) { + STableMetaInfo *pInfo1 = tscGetTableMetaInfoFromCmd(pCmd, j); + memset(fName, 0, tListLen(fName)); + + tNameExtractFullName(&pInfo1->name, fName); + if (strcmp(name, fName) != 0) { + continue; + } + + pInfo = pInfo1; + break; + } + + if (!pInfo){ + continue; + } + int32_t size = 0; + pInfo->vgroupList = createVgroupInfoFromMsg(pMsg, &size, pSql->self); + pMsg += size; + } + + taosReleaseRef(tscObjRef, parent->self); + return pSql->res.code; +} + +int tscProcessShowRsp(SSqlObj *pSql) { + STableMetaMsg *pMetaMsg; + SShowRsp * pShow; + SSchema * pSchema; + + SSqlRes *pRes = &pSql->res; + SSqlCmd *pCmd = &pSql->cmd; + + SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); + + STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + + pShow = (SShowRsp *)pRes->pRsp; + pShow->qhandle = htobe64(pShow->qhandle); + pRes->qId = pShow->qhandle; + + tscResetForNextRetrieve(pRes); + pMetaMsg = &(pShow->tableMeta); + + pMetaMsg->numOfColumns = ntohs(pMetaMsg->numOfColumns); + + pSchema = pMetaMsg->schema; + pMetaMsg->tid = ntohs(pMetaMsg->tid); + for (int i = 0; i < pMetaMsg->numOfColumns; ++i) { + pSchema->bytes = htons(pSchema->bytes); + pSchema++; + } + + tfree(pTableMetaInfo->pTableMeta); + pTableMetaInfo->pTableMeta = tscCreateTableMetaFromMsg(pMetaMsg); + + SSchema *pTableSchema = tscGetTableSchema(pTableMetaInfo->pTableMeta); + if (pQueryInfo->colList == NULL) { + pQueryInfo->colList = taosArrayInit(4, POINTER_BYTES); + } + + SFieldInfo* pFieldInfo = &pQueryInfo->fieldsInfo; + + SColumnIndex index = {0}; + pSchema = pMetaMsg->schema; + + uint64_t uid = pTableMetaInfo->pTableMeta->id.uid; + for (int16_t i = 0; i < pMetaMsg->numOfColumns; ++i, ++pSchema) { + index.columnIndex = i; + tscColumnListInsert(pQueryInfo->colList, i, uid, pSchema); + + TAOS_FIELD f = tscCreateField(pSchema->type, pSchema->name, pSchema->bytes); + SInternalField* pInfo = tscFieldInfoAppend(pFieldInfo, &f); + + pInfo->pExpr = tscExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, + pTableSchema[i].type, pTableSchema[i].bytes, getNewResColId(pCmd), pTableSchema[i].bytes, false); + } + + pCmd->numOfCols = pQueryInfo->fieldsInfo.numOfOutput; + tscFieldInfoUpdateOffset(pQueryInfo); + return 0; +} + +static void createHbObj(STscObj* pObj) { + if (pObj->hbrid != 0) { + return; + } + + SSqlObj *pSql = (SSqlObj *)calloc(1, sizeof(SSqlObj)); + if (NULL == pSql) return; + + pSql->fp = tscProcessHeartBeatRsp; + + SQueryInfo *pQueryInfo = tscGetQueryInfoS(&pSql->cmd); + if (pQueryInfo == NULL) { + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + tfree(pSql); + return; + } + + pQueryInfo->command = TSDB_SQL_HB; + + pSql->cmd.command = pQueryInfo->command; + if (TSDB_CODE_SUCCESS != tscAllocPayload(&(pSql->cmd), TSDB_DEFAULT_PAYLOAD_SIZE)) { + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + tfree(pSql); + return; + } + + pSql->param = pObj; + pSql->pTscObj = pObj; + pSql->signature = pSql; + + registerSqlObj(pSql); + tscDebug("0x%"PRIx64" HB is allocated, pObj:%p", pSql->self, pObj); + + pObj->hbrid = pSql->self; +} + +int tscProcessUseDbRsp(SSqlObj *pSql) { + STscObj * pObj = pSql->pTscObj; + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0); + + pthread_mutex_lock(&pObj->mutex); + int ret = tNameExtractFullName(&pTableMetaInfo->name, pObj->db); + pthread_mutex_unlock(&pObj->mutex); + + return ret; +} + +//todo only invalid the buffered data that belongs to dropped databases +int tscProcessDropDbRsp(SSqlObj *pSql) { + //TODO LOCK DB WHEN MODIFY IT + //pSql->pTscObj->db[0] = 0; + + taosHashClear(tscTableMetaMap); + taosHashClear(tscVgroupMap); + taosCacheEmpty(tscVgroupListBuf); + return 0; +} + +int tscProcessDropTableRsp(SSqlObj *pSql) { + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0); + tscRemoveCachedTableMeta(pTableMetaInfo, pSql->self); + tfree(pTableMetaInfo->pTableMeta); + return 0; +} + +int tscProcessAlterTableMsgRsp(SSqlObj *pSql) { + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0); + + char name[TSDB_TABLE_FNAME_LEN] = {0}; + tNameExtractFullName(&pTableMetaInfo->name, name); + + tscDebug("0x%"PRIx64" remove tableMeta in hashMap after alter-table: %s", pSql->self, name); + + bool isSuperTable = UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo); + taosHashRemove(tscTableMetaMap, name, strnlen(name, TSDB_TABLE_FNAME_LEN)); + tfree(pTableMetaInfo->pTableMeta); + + if (isSuperTable) { // if it is a super table, iterate the hashTable and remove all the childTableMeta + if (pSql->res.pRsp == NULL) { + tscDebug("0x%"PRIx64" unexpected resp from mnode, super table: %s failed to update super table meta ", pSql->self, name); + return 0; + } + return tscProcessTableMetaRsp(pSql); + } + + return 0; +} + +int tscProcessAlterDbMsgRsp(SSqlObj *pSql) { + UNUSED(pSql); + return 0; +} +int tscProcessCompactRsp(SSqlObj *pSql) { + UNUSED(pSql); + return TSDB_CODE_SUCCESS; +} + +int tscProcessShowCreateRsp(SSqlObj *pSql) { + return tscLocalResultCommonBuilder(pSql, 1); +} + +int tscProcessQueryRsp(SSqlObj *pSql) { + SSqlRes *pRes = &pSql->res; + + SQueryTableRsp *pQueryAttr = (SQueryTableRsp *)pRes->pRsp; + pQueryAttr->qId = htobe64(pQueryAttr->qId); + + pRes->qId = pQueryAttr->qId; + pRes->data = NULL; + + tscResetForNextRetrieve(pRes); + tscDebug("0x%"PRIx64" query rsp received, qId:0x%"PRIx64, pSql->self, pRes->qId); + return 0; +} + +static void decompressQueryColData(SSqlObj *pSql, SSqlRes *pRes, SQueryInfo* pQueryInfo, char **data, int8_t compressed, int32_t compLen) { + int32_t decompLen = 0; + int32_t numOfCols = pQueryInfo->fieldsInfo.numOfOutput; + int32_t *compSizes; + char *pData = *data; + compSizes = (int32_t *)(pData + compLen); + + TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, numOfCols - 1); + int16_t offset = tscFieldInfoGetOffset(pQueryInfo, numOfCols - 1); + char *outputBuf = tcalloc(pRes->numOfRows, (pField->bytes + offset)); + + char *p = outputBuf; + int32_t bufOffset; + for (int32_t i = 0; i < numOfCols; ++i) { + SInternalField* pInfo = (SInternalField*)TARRAY_GET_ELEM(pQueryInfo->fieldsInfo.internalField, i); + bufOffset = pInfo->field.bytes * pRes->numOfRows; + + int32_t flen = (*(tDataTypes[pInfo->field.type].decompFunc))(pData, htonl(compSizes[i]), pRes->numOfRows, p, bufOffset, + compressed, NULL, 0); + + p += flen; + decompLen +=flen; + pData += htonl(compSizes[i]); + } + + /* Resize rsp as decompressed data will occupy more space */ + pRes->rspLen = pRes->rspLen - (compLen + numOfCols * sizeof(int32_t)) + decompLen; + char *new_rsp = (char *)realloc(pRes->pRsp, pRes->rspLen); + if (new_rsp == NULL) { + pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; + return; + } else { + pRes->pRsp = new_rsp; + *data = ((SRetrieveTableRsp *)pRes->pRsp)->data; + pData = *data + compLen + numOfCols * sizeof(int32_t); + } + + tscDebug("0x%"PRIx64" decompress col data, compressed size:%d, decompressed size:%d", + pSql->self, (int32_t)(compLen + numOfCols * sizeof(int32_t)), decompLen); + + int32_t tailLen = pRes->rspLen - sizeof(SRetrieveTableRsp) - decompLen; + memmove(*data + decompLen, pData, tailLen); + memmove(*data, outputBuf, decompLen); + + tfree(outputBuf); +} + +int tscProcessRetrieveRspFromNode(SSqlObj *pSql) { + SSqlRes *pRes = &pSql->res; + SSqlCmd *pCmd = &pSql->cmd; + + assert(pRes->rspLen >= sizeof(SRetrieveTableRsp)); + + SRetrieveTableRsp *pRetrieve = (SRetrieveTableRsp *)pRes->pRsp; + if (pRetrieve == NULL) { + pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; + return pRes->code; + } + + pRes->numOfRows = htonl(pRetrieve->numOfRows); + pRes->precision = htons(pRetrieve->precision); + pRes->offset = htobe64(pRetrieve->offset); + pRes->useconds = htobe64(pRetrieve->useconds); + pRes->completed = (pRetrieve->completed == 1); + pRes->data = pRetrieve->data; + + SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); + if (tscCreateResPointerInfo(pRes, pQueryInfo) != TSDB_CODE_SUCCESS) { + return pRes->code; + } + + //Decompress col data if compressed from server + if (pRetrieve->compressed) { + int32_t compLen = htonl(pRetrieve->compLen); + decompressQueryColData(pSql, pRes, pQueryInfo, &pRes->data, pRetrieve->compressed, compLen); + } + + STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + if ((pCmd->command == TSDB_SQL_RETRIEVE) || + ((UTIL_TABLE_IS_CHILD_TABLE(pTableMetaInfo) || UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo)) && + !TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_SUBQUERY)) || + (tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0) && + !TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_JOIN_QUERY) && + !TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_JOIN_SEC_STAGE))) { + tscSetResRawPtr(pRes, pQueryInfo); + } + + if (pSql->pSubscription != NULL) { + int32_t numOfCols = pQueryInfo->fieldsInfo.numOfOutput; + + TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, numOfCols - 1); + int16_t offset = tscFieldInfoGetOffset(pQueryInfo, numOfCols - 1); + + char* p = pRes->data + (pField->bytes + offset) * pRes->numOfRows; + + int32_t numOfTables = htonl(*(int32_t*)p); + p += sizeof(int32_t); + for (int i = 0; i < numOfTables; i++) { + int64_t uid = htobe64(*(int64_t*)p); + p += sizeof(int64_t); + p += sizeof(int32_t); // skip tid + TSKEY key = htobe64(*(TSKEY*)p); + p += sizeof(TSKEY); + tscUpdateSubscriptionProgress(pSql->pSubscription, uid, key); + } + } + + pRes->row = 0; + tscDebug("0x%"PRIx64" numOfRows:%d, offset:%" PRId64 ", complete:%d, qId:0x%"PRIx64, pSql->self, pRes->numOfRows, pRes->offset, + pRes->completed, pRes->qId); + + return 0; +} + +void tscTableMetaCallBack(void *param, TAOS_RES *res, int code); + +static int32_t getTableMetaFromMnode(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo, bool autocreate) { + SSqlObj *pNew = calloc(1, sizeof(SSqlObj)); + if (NULL == pNew) { + tscError("0x%"PRIx64" malloc failed for new sqlobj to get table meta", pSql->self); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + pNew->pTscObj = pSql->pTscObj; + pNew->signature = pNew; + pNew->cmd.command = TSDB_SQL_META; + + tscAddQueryInfo(&pNew->cmd); + + SQueryInfo *pNewQueryInfo = tscGetQueryInfoS(&pNew->cmd); + if (TSDB_CODE_SUCCESS != tscAllocPayload(&pNew->cmd, TSDB_DEFAULT_PAYLOAD_SIZE + pSql->cmd.payloadLen)) { + tscError("0x%"PRIx64" malloc failed for payload to get table meta", pSql->self); + + tscFreeSqlObj(pNew); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + STableMetaInfo *pNewTableMetaInfo = tscAddEmptyMetaInfo(pNewQueryInfo); + assert(pNewQueryInfo->numOfTables == 1); + + tNameAssign(&pNewTableMetaInfo->name, &pTableMetaInfo->name); + + registerSqlObj(pNew); + + pNew->fp = tscTableMetaCallBack; + pNew->param = (void *)pSql->self; + + tscDebug("0x%"PRIx64" new pSqlObj:0x%"PRIx64" to get tableMeta, auto create:%d, metaRid from %"PRId64" to %"PRId64, + pSql->self, pNew->self, autocreate, pSql->metaRid, pNew->self); + pSql->metaRid = pNew->self; + + { + STableInfoMsg *pInfoMsg = (STableInfoMsg *)pNew->cmd.payload; + int32_t code = tNameExtractFullName(&pNewTableMetaInfo->name, pInfoMsg->tableFname); + if (code != TSDB_CODE_SUCCESS) { + return TSDB_CODE_TSC_INVALID_OPERATION; + } + + pInfoMsg->createFlag = htons(autocreate? 1 : 0); + char *pMsg = (char *)pInfoMsg + sizeof(STableInfoMsg); + + // tag data exists + if (autocreate && pSql->cmd.insertParam.tagData.dataLen != 0) { + pMsg = serializeTagData(&pSql->cmd.insertParam.tagData, pMsg); + } + + pNew->cmd.payloadLen = (int32_t)(pMsg - (char*)pInfoMsg); + pNew->cmd.msgType = TSDB_MSG_TYPE_TABLE_META; + } + + int32_t code = tscBuildAndSendRequest(pNew, NULL); + if (code == TSDB_CODE_SUCCESS) { + code = TSDB_CODE_TSC_ACTION_IN_PROGRESS; // notify application that current process needs to be terminated + } + + return code; +} + +int32_t getMultiTableMetaFromMnode(SSqlObj *pSql, SArray* pNameList, SArray* pVgroupNameList, SArray* pUdfList, __async_cb_func_t fp, bool metaClone) { + SSqlObj *pNew = calloc(1, sizeof(SSqlObj)); + if (NULL == pNew) { + tscError("0x%"PRIx64" failed to allocate sqlobj to get multiple table meta", pSql->self); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + pNew->pTscObj = pSql->pTscObj; + pNew->signature = pNew; + pNew->cmd.command = TSDB_SQL_MULTI_META; + + int32_t numOfTable = (int32_t) taosArrayGetSize(pNameList); + int32_t numOfVgroupList = (int32_t) taosArrayGetSize(pVgroupNameList); + int32_t numOfUdf = pUdfList ? (int32_t)taosArrayGetSize(pUdfList) : 0; + + int32_t size = (numOfTable + numOfVgroupList) * TSDB_TABLE_FNAME_LEN + TSDB_FUNC_NAME_LEN * numOfUdf + sizeof(SMultiTableInfoMsg); + if (TSDB_CODE_SUCCESS != tscAllocPayload(&pNew->cmd, size)) { + tscError("0x%"PRIx64" malloc failed for payload to get table meta", pSql->self); + tscFreeSqlObj(pNew); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + SMultiTableInfoMsg* pInfo = (SMultiTableInfoMsg*) pNew->cmd.payload; + pInfo->metaClone = metaClone? 1:0; + pInfo->numOfTables = htonl((uint32_t) taosArrayGetSize(pNameList)); + pInfo->numOfVgroups = htonl((uint32_t) taosArrayGetSize(pVgroupNameList)); + pInfo->numOfUdfs = htonl(numOfUdf); + + char* start = pInfo->tableNames; + int32_t len = 0; + for(int32_t i = 0; i < numOfTable; ++i) { + char* name = taosArrayGetP(pNameList, i); + if (i < numOfTable - 1 || numOfVgroupList > 0 || numOfUdf > 0) { + len = sprintf(start, "%s,", name); + } else { + len = sprintf(start, "%s", name); + } + + start += len; + } + + for(int32_t i = 0; i < numOfVgroupList; ++i) { + char* name = taosArrayGetP(pVgroupNameList, i); + if (i < numOfVgroupList - 1 || numOfUdf > 0) { + len = sprintf(start, "%s,", name); + } else { + len = sprintf(start, "%s", name); + } + + start += len; + } + + for(int32_t i = 0; i < numOfUdf; ++i) { + SUdfInfo * u = taosArrayGet(pUdfList, i); + if (i < numOfUdf - 1) { + len = sprintf(start, "%s,", u->name); + } else { + len = sprintf(start, "%s", u->name); + } + + start += len; + } + + pNew->cmd.payloadLen = (int32_t) ((start - pInfo->tableNames) + sizeof(SMultiTableInfoMsg)); + pNew->cmd.msgType = TSDB_MSG_TYPE_TABLES_META; + + registerSqlObj(pNew); + tscDebug("0x%"PRIx64" new pSqlObj:0x%"PRIx64" to get %d tableMeta, vgroupInfo:%d, udf:%d, msg size:%d", pSql->self, + pNew->self, numOfTable, numOfVgroupList, numOfUdf, pNew->cmd.payloadLen); + + pNew->fp = fp; + pNew->param = (void *)pSql->self; + + tscDebug("0x%"PRIx64" metaRid from 0x%" PRIx64 " to 0x%" PRIx64 , pSql->self, pSql->metaRid, pNew->self); + + pSql->metaRid = pNew->self; + int32_t code = tscBuildAndSendRequest(pNew, NULL); + if (code == TSDB_CODE_SUCCESS) { + code = TSDB_CODE_TSC_ACTION_IN_PROGRESS; // notify application that current process needs to be terminated + } + + return code; +} + +int32_t tscGetTableMetaImpl(SSqlObj* pSql, STableMetaInfo *pTableMetaInfo, bool autocreate, bool onlyLocal) { + assert(tIsValidName(&pTableMetaInfo->name)); + + char name[TSDB_TABLE_FNAME_LEN] = {0}; + tNameExtractFullName(&pTableMetaInfo->name, name); + + size_t len = strlen(name); + // just make runtime happy + if (pTableMetaInfo->tableMetaCapacity != 0 && pTableMetaInfo->pTableMeta != NULL) { + memset(pTableMetaInfo->pTableMeta, 0, pTableMetaInfo->tableMetaCapacity); + } + + if (NULL == taosHashGetCloneExt(tscTableMetaMap, name, len, NULL, (void **)&(pTableMetaInfo->pTableMeta), &pTableMetaInfo->tableMetaCapacity)) { + tfree(pTableMetaInfo->pTableMeta); + } + + STableMeta* pMeta = pTableMetaInfo->pTableMeta; + STableMeta* pSTMeta = (STableMeta *)(pSql->pBuf); + + if (pMeta && pMeta->id.uid > 0) { + // in case of child table, here only get the + if (pMeta->tableType == TSDB_CHILD_TABLE) { + int32_t code = tscCreateTableMetaFromSTableMeta(&pTableMetaInfo->pTableMeta, name, &pTableMetaInfo->tableMetaCapacity, (STableMeta **)(&pSTMeta)); + pSql->pBuf = (void *)(pSTMeta); + if (code != TSDB_CODE_SUCCESS) { + return getTableMetaFromMnode(pSql, pTableMetaInfo, autocreate); + } + } + + tscDebug("0x%"PRIx64 " %s retrieve tableMeta from cache, numOfCols:%d, numOfTags:%d", pSql->self, name, pMeta->tableInfo.numOfColumns, pMeta->tableInfo.numOfTags); + return TSDB_CODE_SUCCESS; + } + + if (onlyLocal) { + return TSDB_CODE_TSC_NO_META_CACHED; + } + + return getTableMetaFromMnode(pSql, pTableMetaInfo, autocreate); +} + +int32_t tscGetTableMeta(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo) { + return tscGetTableMetaImpl(pSql, pTableMetaInfo, false, false); +} + +int tscGetTableMetaEx(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo, bool createIfNotExists, bool onlyLocal) { + return tscGetTableMetaImpl(pSql, pTableMetaInfo, createIfNotExists, onlyLocal); +} + +int32_t tscGetUdfFromNode(SSqlObj *pSql, SQueryInfo* pQueryInfo) { + SSqlObj *pNew = calloc(1, sizeof(SSqlObj)); + if (NULL == pNew) { + tscError("%p malloc failed for new sqlobj to get user-defined functions", pSql); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + pNew->pTscObj = pSql->pTscObj; + pNew->signature = pNew; + pNew->cmd.command = TSDB_SQL_RETRIEVE_FUNC; + + if (tscAddQueryInfo(&pNew->cmd) != TSDB_CODE_SUCCESS) { + tscError("%p malloc failed for new queryinfo", pSql); + tscFreeSqlObj(pNew); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + SQueryInfo *pNewQueryInfo = tscGetQueryInfo(&pNew->cmd); + + pNewQueryInfo->pUdfInfo = taosArrayInit(4, sizeof(SUdfInfo)); + for(int32_t i = 0; i < taosArrayGetSize(pQueryInfo->pUdfInfo); ++i) { + SUdfInfo info = {0}; + SUdfInfo* p1 = taosArrayGet(pQueryInfo->pUdfInfo, i); + info = *p1; + info.name = strdup(p1->name); + taosArrayPush(pNewQueryInfo->pUdfInfo, &info); + } + + pNew->cmd.active = pNewQueryInfo; + + if (TSDB_CODE_SUCCESS != tscAllocPayload(&pNew->cmd, TSDB_DEFAULT_PAYLOAD_SIZE + pSql->cmd.payloadLen)) { + tscError("%p malloc failed for payload to get table meta", pSql); + tscFreeSqlObj(pNew); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + tscDebug("%p new pSqlObj:%p to retrieve udf", pSql, pNew); + registerSqlObj(pNew); + + pNew->fp = tscTableMetaCallBack; + pNew->param = (void *)pSql->self; + + tscDebug("%p metaRid from %" PRId64 " to %" PRId64 , pSql, pSql->metaRid, pNew->self); + + pSql->metaRid = pNew->self; + + int32_t code = tscBuildAndSendRequest(pNew, NULL); + if (code == TSDB_CODE_SUCCESS) { + code = TSDB_CODE_TSC_ACTION_IN_PROGRESS; // notify application that current process needs to be terminated + } + + return code; +} + +static void freeElem(void* p) { + tfree(*(char**)p); +} + +/** + * retrieve table meta from mnode, and then update the local table meta hashmap. + * @param pSql sql object + * @param tableIndex table index + * @return status code + */ +int tscRenewTableMeta(SSqlObj *pSql, int32_t tableIndex) { + SSqlCmd* pCmd = &pSql->cmd; + + SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); + STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, tableIndex); + + char name[TSDB_TABLE_FNAME_LEN] = {0}; + int32_t code = tNameExtractFullName(&pTableMetaInfo->name, name); + if (code != TSDB_CODE_SUCCESS) { + tscError("0x%"PRIx64" failed to generate the table full name", pSql->self); + return TSDB_CODE_TSC_INVALID_OPERATION; + } + + STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; + if (pTableMeta) { + tscDebug("0x%"PRIx64" update table meta:%s, old meta numOfTags:%d, numOfCols:%d, uid:%" PRIu64, pSql->self, name, + tscGetNumOfTags(pTableMeta), tscGetNumOfColumns(pTableMeta), pTableMeta->id.uid); + } + + + // remove stored tableMeta info in hash table + tscResetSqlCmd(pCmd, true, pSql->self); + + SArray* pNameList = taosArrayInit(1, POINTER_BYTES); + SArray* vgroupList = taosArrayInit(1, POINTER_BYTES); + + char* n = strdup(name); + taosArrayPush(pNameList, &n); + code = getMultiTableMetaFromMnode(pSql, pNameList, vgroupList, NULL, tscTableMetaCallBack, true); + taosArrayDestroyEx(pNameList, freeElem); + taosArrayDestroyEx(vgroupList, freeElem); + + return code; +} + +static bool allVgroupInfoRetrieved(SQueryInfo* pQueryInfo) { + for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { + STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, i); + if (pTableMetaInfo->vgroupList == NULL) { + return false; + } + } + + // all super tables vgroupinfo are retrieved, no need to retrieve vgroup info anymore + return true; +} + +int tscGetSTableVgroupInfo(SSqlObj *pSql, SQueryInfo* pQueryInfo) { + int32_t code = TSDB_CODE_RPC_NETWORK_UNAVAIL; + if (allVgroupInfoRetrieved(pQueryInfo)) { + return TSDB_CODE_SUCCESS; + } + SSqlObj *pNew = calloc(1, sizeof(SSqlObj)); + pNew->pTscObj = pSql->pTscObj; + pNew->signature = pNew; + + pNew->cmd.command = TSDB_SQL_STABLEVGROUP; + + // TODO TEST IT + SQueryInfo *pNewQueryInfo = tscGetQueryInfoS(&pNew->cmd); + if (pNewQueryInfo == NULL) { + tscFreeSqlObj(pNew); + return code; + } + + for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { + STableMetaInfo *pMInfo = tscGetMetaInfo(pQueryInfo, i); + STableMeta* pTableMeta = tscTableMetaDup(pMInfo->pTableMeta); + tscAddTableMetaInfo(pNewQueryInfo, &pMInfo->name, pTableMeta, NULL, pMInfo->tagColList, pMInfo->pVgroupTables); + } + + if ((code = tscAllocPayload(&pNew->cmd, TSDB_DEFAULT_PAYLOAD_SIZE)) != TSDB_CODE_SUCCESS) { + tscFreeSqlObj(pNew); + return code; + } + + pNewQueryInfo->numOfTables = pQueryInfo->numOfTables; + registerSqlObj(pNew); + + tscDebug("0x%"PRIx64" svgroupRid from %" PRId64 " to %" PRId64 , pSql->self, pSql->svgroupRid, pNew->self); + + pSql->svgroupRid = pNew->self; + tscDebug("0x%"PRIx64" new sqlObj:%p to get vgroupInfo, numOfTables:%d", pSql->self, pNew, pNewQueryInfo->numOfTables); + + pNew->fp = tscTableMetaCallBack; + pNew->param = (void *)pSql->self; + code = tscBuildAndSendRequest(pNew, NULL); + if (code == TSDB_CODE_SUCCESS) { + code = TSDB_CODE_TSC_ACTION_IN_PROGRESS; + } + + return code; +} + +#endif + +int processConnectRsp(SRequestObj *pRequest, const char* pMsg, int32_t msgLen) { + STscObj *pTscObj = pRequest->pTscObj; + + SConnectRsp *pConnect = (SConnectRsp *)pMsg; + pConnect->acctId = htonl(pConnect->acctId); + pConnect->connId = htonl(pConnect->connId); + pConnect->clusterId = htonl(pConnect->clusterId); + + // TODO refactor + pthread_mutex_lock(&pTscObj->mutex); + char temp[TSDB_TABLE_FNAME_LEN * 2] = {0}; + int32_t len = sprintf(temp, "%s%s%s", pTscObj->acctId, TS_PATH_DELIMITER, pTscObj->db); + + assert(len <= sizeof(pTscObj->db)); + tstrncpy(pTscObj->db, temp, sizeof(pTscObj->db)); + pthread_mutex_unlock(&pTscObj->mutex); + + assert(pConnect->epSet.numOfEps > 0); + if (!isEpsetEqual(&pTscObj->pAppInfo->mgmtEp.epSet, &pConnect->epSet)) { + updateEpSet_s(&pTscObj->pAppInfo->mgmtEp, &pConnect->epSet); + } + + for (int i = 0; i < pConnect->epSet.numOfEps; ++i) { + tscDebug("0x%" PRIx64 " epSet.fqdn[%d]: %s, connObj:0x%"PRIx64, pRequest->requestId, i, pConnect->epSet.fqdn[i], pTscObj->id); + } + + pTscObj->connId = pConnect->connId; + + // update the appInstInfo + pTscObj->pAppInfo->clusterId = pConnect->clusterId; + atomic_add_fetch_64(&pTscObj->pAppInfo->numOfConns, 1); + + tscDebug("0x%" PRIx64 " clusterId:%d, totalConn:%"PRId64, pRequest->requestId, pConnect->clusterId, pTscObj->pAppInfo->numOfConns); + // createHbObj(pTscObj); + + // launch a timer to send heartbeat to maintain the connection and send status to mnode + // taosTmrReset(tscProcessActivityTimer, tsShellActivityTimer * 500, (void *)pTscObj->rid, tscTmr, &pTscObj->pTimer); + return 0; +} + +void initMsgHandleFp() { +#if 0 + tscBuildMsg[TSDB_SQL_SELECT] = tscBuildQueryMsg; + tscBuildMsg[TSDB_SQL_INSERT] = tscBuildSubmitMsg; + tscBuildMsg[TSDB_SQL_FETCH] = tscBuildFetchMsg; + + tscBuildMsg[TSDB_SQL_CREATE_DB] = tscBuildCreateDbMsg; + tscBuildMsg[TSDB_SQL_CREATE_USER] = tscBuildUserMsg; + tscBuildMsg[TSDB_SQL_CREATE_FUNCTION] = tscBuildCreateFuncMsg; + + tscBuildMsg[TSDB_SQL_CREATE_ACCT] = tscBuildAcctMsg; + tscBuildMsg[TSDB_SQL_ALTER_ACCT] = tscBuildAcctMsg; + + tscBuildMsg[TSDB_SQL_CREATE_TABLE] = tscBuildCreateTableMsg; + tscBuildMsg[TSDB_SQL_DROP_USER] = tscBuildDropUserAcctMsg; + tscBuildMsg[TSDB_SQL_DROP_ACCT] = tscBuildDropUserAcctMsg; + tscBuildMsg[TSDB_SQL_DROP_DB] = tscBuildDropDbMsg; + tscBuildMsg[TSDB_SQL_DROP_FUNCTION] = tscBuildDropFuncMsg; + tscBuildMsg[TSDB_SQL_SYNC_DB_REPLICA] = tscBuildSyncDbReplicaMsg; + tscBuildMsg[TSDB_SQL_DROP_TABLE] = tscBuildDropTableMsg; + tscBuildMsg[TSDB_SQL_ALTER_USER] = tscBuildUserMsg; + tscBuildMsg[TSDB_SQL_CREATE_DNODE] = tscBuildCreateDnodeMsg; + tscBuildMsg[TSDB_SQL_DROP_DNODE] = tscBuildDropDnodeMsg; + tscBuildMsg[TSDB_SQL_CFG_DNODE] = tscBuildCfgDnodeMsg; + tscBuildMsg[TSDB_SQL_ALTER_TABLE] = tscBuildAlterTableMsg; + tscBuildMsg[TSDB_SQL_UPDATE_TAGS_VAL] = tscBuildUpdateTagMsg; + tscBuildMsg[TSDB_SQL_ALTER_DB] = tscAlterDbMsg; + tscBuildMsg[TSDB_SQL_COMPACT_VNODE] = tscBuildCompactMsg; + + + tscBuildMsg[TSDB_SQL_USE_DB] = tscBuildUseDbMsg; + tscBuildMsg[TSDB_SQL_STABLEVGROUP] = tscBuildSTableVgroupMsg; + tscBuildMsg[TSDB_SQL_RETRIEVE_FUNC] = tscBuildRetrieveFuncMsg; + + tscBuildMsg[TSDB_SQL_HB] = tscBuildHeartBeatMsg; + tscBuildMsg[TSDB_SQL_SHOW] = tscBuildShowMsg; + tscBuildMsg[TSDB_SQL_RETRIEVE] = tscBuildRetrieveFromMgmtMsg; + tscBuildMsg[TSDB_SQL_KILL_QUERY] = tscBuildKillMsg; + tscBuildMsg[TSDB_SQL_KILL_STREAM] = tscBuildKillMsg; + tscBuildMsg[TSDB_SQL_KILL_CONNECTION] = tscBuildKillMsg; + + tscProcessMsgRsp[TSDB_SQL_SELECT] = tscProcessQueryRsp; + tscProcessMsgRsp[TSDB_SQL_FETCH] = tscProcessRetrieveRspFromNode; + + tscProcessMsgRsp[TSDB_SQL_DROP_DB] = tscProcessDropDbRsp; + tscProcessMsgRsp[TSDB_SQL_DROP_TABLE] = tscProcessDropTableRsp; + + tscProcessMsgRsp[TSDB_SQL_USE_DB] = tscProcessUseDbRsp; + tscProcessMsgRsp[TSDB_SQL_META] = tscProcessTableMetaRsp; + tscProcessMsgRsp[TSDB_SQL_STABLEVGROUP] = tscProcessSTableVgroupRsp; + tscProcessMsgRsp[TSDB_SQL_MULTI_META] = tscProcessMultiTableMetaRsp; + tscProcessMsgRsp[TSDB_SQL_RETRIEVE_FUNC] = tscProcessRetrieveFuncRsp; + + tscProcessMsgRsp[TSDB_SQL_SHOW] = tscProcessShowRsp; + tscProcessMsgRsp[TSDB_SQL_RETRIEVE] = tscProcessRetrieveRspFromNode; // rsp handled by same function. + tscProcessMsgRsp[TSDB_SQL_DESCRIBE_TABLE] = tscProcessDescribeTableRsp; + + tscProcessMsgRsp[TSDB_SQL_CURRENT_DB] = tscProcessLocalRetrieveRsp; + tscProcessMsgRsp[TSDB_SQL_CURRENT_USER] = tscProcessLocalRetrieveRsp; + tscProcessMsgRsp[TSDB_SQL_SERV_VERSION] = tscProcessLocalRetrieveRsp; + tscProcessMsgRsp[TSDB_SQL_CLI_VERSION] = tscProcessLocalRetrieveRsp; + tscProcessMsgRsp[TSDB_SQL_SERV_STATUS] = tscProcessLocalRetrieveRsp; + + tscProcessMsgRsp[TSDB_SQL_RETRIEVE_EMPTY_RESULT] = tscProcessEmptyResultRsp; + + tscProcessMsgRsp[TSDB_SQL_RETRIEVE_GLOBALMERGE] = tscProcessRetrieveGlobalMergeRsp; + + tscProcessMsgRsp[TSDB_SQL_ALTER_TABLE] = tscProcessAlterTableMsgRsp; + tscProcessMsgRsp[TSDB_SQL_ALTER_DB] = tscProcessAlterDbMsgRsp; + tscProcessMsgRsp[TSDB_SQL_COMPACT_VNODE] = tscProcessCompactRsp; + + tscProcessMsgRsp[TSDB_SQL_SHOW_CREATE_TABLE] = tscProcessShowCreateRsp; + tscProcessMsgRsp[TSDB_SQL_SHOW_CREATE_STABLE] = tscProcessShowCreateRsp; + tscProcessMsgRsp[TSDB_SQL_SHOW_CREATE_DATABASE] = tscProcessShowCreateRsp; +#endif + +// buildRequestMsgFp[TSDB_SQL_CONNECT] = tscBuildConnectMsg; + handleRequestRspFp[TSDB_SQL_CONNECT] = processConnectRsp; +} \ No newline at end of file diff --git a/source/client/src/clientmain.c b/source/client/src/clientmain.c new file mode 100644 index 0000000000..ba80135850 --- /dev/null +++ b/source/client/src/clientmain.c @@ -0,0 +1,85 @@ +#include "clientInt.h" +#include "trpc.h" +#include "os.h" +#include "taosmsg.h" +#include "tcache.h" +#include "tconfig.h" +#include "tglobal.h" +#include "tnote.h" +#include "tref.h" +#include "tscLog.h" +#include "tsched.h" +#include "ttime.h" +#include "ttimezone.h" + +#define TSC_VAR_NOT_RELEASE 1 +#define TSC_VAR_RELEASED 0 + +static int32_t sentinel = TSC_VAR_NOT_RELEASE; +static pthread_once_t tscinit = PTHREAD_ONCE_INIT; + +extern int32_t tscInitRes; + +int taos_options(TSDB_OPTION option, const void *arg, ...) { + static int32_t lock = 0; + + for (int i = 1; atomic_val_compare_exchange_32(&lock, 0, 1) != 0; ++i) { + if (i % 1000 == 0) { + tscInfo("haven't acquire lock after spin %d times.", i); + sched_yield(); + } + } + + int ret = taos_options_imp(option, (const char*)arg); + + atomic_store_32(&lock, 0); + return ret; +} + +int taos_init() { + pthread_once(&tscinit, taos_init_imp); + return tscInitRes; +} + +// this function may be called by user or system, or by both simultaneously. +void taos_cleanup(void) { + tscDebug("start to cleanup client environment"); + + if (atomic_val_compare_exchange_32(&sentinel, TSC_VAR_NOT_RELEASE, TSC_VAR_RELEASED) != TSC_VAR_NOT_RELEASE) { + return; + } + + int32_t id = tscReqRef; + tscReqRef = -1; + taosCloseRef(id); + + void* p = tscQhandle; + tscQhandle = NULL; + taosCleanUpScheduler(p); + + id = tscConnRef; + tscConnRef = -1; + taosCloseRef(id); + + rpcCleanup(); + taosCloseLog(); +} + +void taos_close(TAOS* taos) { + if (taos == NULL) { + return; + } + + STscObj *pTscObj = (STscObj *)taos; + tscDebug("0x%"PRIx64" try to close connection, numOfReq:%d", pTscObj->id, pTscObj->numOfReqs); + + taosRemoveRef(tscConnRef, pTscObj->id); +} + +const char *taos_errstr(TAOS_RES *res) { + +} + +void taos_free_result(TAOS_RES *res) { + +} \ No newline at end of file diff --git a/source/client/src/tscEnv.c b/source/client/src/tscEnv.c new file mode 100644 index 0000000000..43d73bf3db --- /dev/null +++ b/source/client/src/tscEnv.c @@ -0,0 +1,471 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "os.h" +#include "taosmsg.h" +#include "tcache.h" +#include "tconfig.h" +#include "tglobal.h" +#include "tnote.h" +#include "tref.h" +#include "tscLog.h" +#include "tsched.h" +#include "ttime.h" +#include "trpc.h" +#include "ttimezone.h" +#include "clientInt.h" + +#define TSC_VAR_NOT_RELEASE 1 +#define TSC_VAR_RELEASED 0 + +SAppInfo appInfo; +int32_t tscReqRef = -1; +int32_t tscConnRef = -1; +void *tscQhandle = NULL; + +int32_t tsNumOfThreads = 1; +volatile int32_t tscInitRes = 0; + +static void registerRequest(SRequestObj* pRequest) { + STscObj *pTscObj = (STscObj *)taosAcquireRef(tscConnRef, pRequest->pTscObj->id); + assert(pTscObj != NULL); + + // connection has been released already, abort creating request. + pRequest->self = taosAddRef(tscReqRef, pRequest); + + int32_t num = atomic_add_fetch_32(&pTscObj->numOfReqs, 1); + + if (pTscObj->pAppInfo) { + SInstanceActivity *pActivity = &pTscObj->pAppInfo->summary; + + int32_t total = atomic_add_fetch_32(&pActivity->totalRequests, 1); + int32_t currentInst = atomic_add_fetch_32(&pActivity->currentRequests, 1); + tscDebug("0x%" PRIx64 " new Request from connObj:0x%" PRIx64 ", current:%d, app current:%d, total:%d", pRequest->self, + pRequest->pTscObj->id, num, currentInst, total); + } +} + +static void deregisterRequest(SRequestObj* pRequest) { + assert(pRequest != NULL); + + STscObj* pTscObj = pRequest->pTscObj; + SInstanceActivity* pActivity = &pTscObj->pAppInfo->summary; + + int32_t currentInst = atomic_sub_fetch_32(&pActivity->currentRequests, 1); + int32_t num = atomic_sub_fetch_32(&pTscObj->numOfReqs, 1); + + tscDebug("0x%"PRIx64" free Request from connObj: 0x%"PRIx64", current:%d, app current:%d", pRequest->self, pTscObj->id, num, currentInst); + taosReleaseRef(tscConnRef, pTscObj->id); +} + +static void tscInitLogFile() { + taosReadGlobalLogCfg(); + if (mkdir(tsLogDir, 0755) != 0 && errno != EEXIST) { + printf("failed to create log dir:%s\n", tsLogDir); + } + + const char *defaultLogFileNamePrefix = "taoslog"; + const int32_t maxLogFileNum = 10; + + char temp[128] = {0}; + sprintf(temp, "%s/%s", tsLogDir, defaultLogFileNamePrefix); + if (taosInitLog(temp, tsNumOfLogLines, maxLogFileNum) < 0) { + printf("failed to open log file in directory:%s\n", tsLogDir); + } +} + +void closeTransporter(STscObj* pTscObj) { + if (pTscObj == NULL || pTscObj->pTransporter == NULL) { + return; + } + + tscDebug("free transporter:%p in connObj: 0x%"PRIx64, pTscObj->pTransporter, pTscObj->id); + rpcClose(pTscObj->pTransporter); + pTscObj->pTransporter = NULL; +} + +// TODO refactor +void* openTransporter(const char *user, const char *auth) { + SRpcInit rpcInit; + memset(&rpcInit, 0, sizeof(rpcInit)); + rpcInit.localPort = 0; + rpcInit.label = "TSC"; + rpcInit.numOfThreads = tsNumOfThreads; + rpcInit.cfp = processMsgFromServer; + rpcInit.sessions = tsMaxConnections; + rpcInit.connType = TAOS_CONN_CLIENT; + rpcInit.user = (char *)user; + rpcInit.idleTime = tsShellActivityTimer * 1000; + rpcInit.ckey = "key"; +// rpcInit.spi = 1; + rpcInit.secret = (char *)auth; + + void* pDnodeConn = rpcOpen(&rpcInit); + if (pDnodeConn == NULL) { + tscError("failed to init connection to server"); + return NULL; + } + + return pDnodeConn; +} + +void destroyTscObj(void *pObj) { + STscObj *pTscObj = pObj; + + atomic_sub_fetch_64(&pTscObj->pAppInfo->numOfConns, 1); + tscDebug("connObj 0x%"PRIx64" destroyed, totalConn:%"PRId64, pTscObj->id, pTscObj->pAppInfo->numOfConns); + + closeTransporter(pTscObj); + pthread_mutex_destroy(&pTscObj->mutex); + tfree(pTscObj); +} + +void* createTscObj(const char* user, const char* auth, const char *ip, uint32_t port, SAppInstInfo* pAppInfo) { + STscObj *pObj = (STscObj *)calloc(1, sizeof(STscObj)); + if (NULL == pObj) { + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + return NULL; + } + + pObj->pAppInfo = pAppInfo; + if (pAppInfo != NULL) { + pObj->pTransporter = pAppInfo->pTransporter; + } + + tstrncpy(pObj->user, user, sizeof(pObj->user)); + memcpy(pObj->pass, auth, TSDB_PASSWORD_LEN); + + pthread_mutex_init(&pObj->mutex, NULL); + pObj->id = taosAddRef(tscConnRef, pObj); + + tscDebug("connObj created, 0x%"PRIx64, pObj->id); + return pObj; +} + +void* createRequest(STscObj* pObj, __taos_async_fn_t fp, void* param, int32_t type) { + assert(pObj != NULL); + + SRequestObj *pRequest = (SRequestObj *)calloc(1, sizeof(SRequestObj)); + if (NULL == pRequest) { + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + return NULL; + } + + // TODO generated request uuid + pRequest->requestId = 0; + + pRequest->metric.start = taosGetTimestampMs(); + + pRequest->type = type; + pRequest->pTscObj = pObj; + pRequest->body.fp = fp; + pRequest->body.param = param; + tsem_init(&pRequest->body.rspSem, 0, 0); + + registerRequest(pRequest); + return pRequest; +} + +static void doDestroyRequest(void* p) { + assert(p != NULL); + SRequestObj* pRequest = (SRequestObj*)p; + + assert(RID_VALID(pRequest->self)); + + tfree(pRequest->msgBuf); + tfree(pRequest->sqlstr); + tfree(pRequest->pInfo); + + deregisterRequest(pRequest); + tfree(pRequest); +} + +void destroyRequest(SRequestObj* pRequest) { + if (pRequest == NULL) { + return; + } + + taosReleaseRef(tscReqRef, pRequest->self); +} + +void taos_init_imp(void) { + // In the APIs of other program language, taos_cleanup is not available yet. + // So, to make sure taos_cleanup will be invoked to clean up the allocated resource to suppress the valgrind warning. + atexit(taos_cleanup); + + errno = TSDB_CODE_SUCCESS; + srand(taosGetTimestampSec()); + + deltaToUtcInitOnce(); + taosInitGlobalCfg(); + taosReadCfgFromFile(); + + tscInitLogFile(); + if (taosCheckAndPrintCfg()) { + tscInitRes = -1; + return; + } + + taosInitNotes(); + initMsgHandleFp(); + + rpcInit(); + + tscDebug("starting to initialize TAOS driver, local ep: %s", tsLocalEp); + + taosSetCoreDump(true); + + double factor = 4.0; + int32_t numOfThreads = MAX((int)(tsNumOfCores * tsNumOfThreadsPerCore / factor), 2); + + int32_t queueSize = tsMaxConnections * 2; + tscQhandle = taosInitScheduler(queueSize, numOfThreads, "tsc"); + if (NULL == tscQhandle) { + tscError("failed to init task queue"); + tscInitRes = -1; + return; + } + + tscDebug("client task queue is initialized, numOfThreads: %d", numOfThreads); + tscConnRef = taosOpenRef(200, destroyTscObj); + tscReqRef = taosOpenRef(40960, doDestroyRequest); + + taosGetAppName(appInfo.appName, NULL); + appInfo.pid = taosGetPId(); + appInfo.startTime = taosGetTimestampMs(); + appInfo.pInstMap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + + tscDebug("client is initialized successfully"); +} + +int taos_options_imp(TSDB_OPTION option, const char *str) { + SGlobalCfg *cfg = NULL; + + switch (option) { + case TSDB_OPTION_CONFIGDIR: + cfg = taosGetConfigOption("configDir"); + assert(cfg != NULL); + + if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_OPTION) { + tstrncpy(configDir, str, TSDB_FILENAME_LEN); + cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION; + tscInfo("set config file directory:%s", str); + } else { + tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, str, tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr); + } + break; + + case TSDB_OPTION_SHELL_ACTIVITY_TIMER: + cfg = taosGetConfigOption("shellActivityTimer"); + assert(cfg != NULL); + + if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_OPTION) { + tsShellActivityTimer = atoi(str); + if (tsShellActivityTimer < 1) tsShellActivityTimer = 1; + if (tsShellActivityTimer > 3600) tsShellActivityTimer = 3600; + cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION; + tscInfo("set shellActivityTimer:%d", tsShellActivityTimer); + } else { + tscWarn("config option:%s, input value:%s, is configured by %s, use %d", cfg->option, str, tsCfgStatusStr[cfg->cfgStatus], *(int32_t *)cfg->ptr); + } + break; + + case TSDB_OPTION_LOCALE: { // set locale + cfg = taosGetConfigOption("locale"); + assert(cfg != NULL); + + size_t len = strlen(str); + if (len == 0 || len > TSDB_LOCALE_LEN) { + tscInfo("Invalid locale:%s, use default", str); + return -1; + } + + if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_OPTION) { + char sep = '.'; + + if (strlen(tsLocale) == 0) { // locale does not set yet + char* defaultLocale = setlocale(LC_CTYPE, ""); + + // The locale of the current OS does not be set correctly, so the default locale cannot be acquired. + // The launch of current system will abort soon. + if (defaultLocale == NULL) { + tscError("failed to get default locale, please set the correct locale in current OS"); + return -1; + } + + tstrncpy(tsLocale, defaultLocale, TSDB_LOCALE_LEN); + } + + // set the user specified locale + char *locale = setlocale(LC_CTYPE, str); + + if (locale != NULL) { // failed to set the user specified locale + tscInfo("locale set, prev locale:%s, new locale:%s", tsLocale, locale); + cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION; + } else { // set the user specified locale failed, use default LC_CTYPE as current locale + locale = setlocale(LC_CTYPE, tsLocale); + tscInfo("failed to set locale:%s, current locale:%s", str, tsLocale); + } + + tstrncpy(tsLocale, locale, TSDB_LOCALE_LEN); + + char *charset = strrchr(tsLocale, sep); + if (charset != NULL) { + charset += 1; + + charset = taosCharsetReplace(charset); + + if (taosValidateEncodec(charset)) { + if (strlen(tsCharset) == 0) { + tscInfo("charset set:%s", charset); + } else { + tscInfo("charset changed from %s to %s", tsCharset, charset); + } + + tstrncpy(tsCharset, charset, TSDB_LOCALE_LEN); + cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION; + + } else { + tscInfo("charset:%s is not valid in locale, charset remains:%s", charset, tsCharset); + } + + free(charset); + } else { // it may be windows system + tscInfo("charset remains:%s", tsCharset); + } + } else { + tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, str, tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr); + } + break; + } + + case TSDB_OPTION_CHARSET: { + /* set charset will override the value of charset, assigned during system locale changed */ + cfg = taosGetConfigOption("charset"); + assert(cfg != NULL); + + size_t len = strlen(str); + if (len == 0 || len > TSDB_LOCALE_LEN) { + tscInfo("failed to set charset:%s", str); + return -1; + } + + if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_OPTION) { + if (taosValidateEncodec(str)) { + if (strlen(tsCharset) == 0) { + tscInfo("charset is set:%s", str); + } else { + tscInfo("charset changed from %s to %s", tsCharset, str); + } + + tstrncpy(tsCharset, str, TSDB_LOCALE_LEN); + cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION; + } else { + tscInfo("charset:%s not valid", str); + } + } else { + tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, str, tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr); + } + + break; + } + + case TSDB_OPTION_TIMEZONE: + cfg = taosGetConfigOption("timezone"); + assert(cfg != NULL); + + if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_OPTION) { + tstrncpy(tsTimezone, str, TSDB_TIMEZONE_LEN); + tsSetTimeZone(); + cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION; + tscDebug("timezone set:%s, input:%s by taos_options", tsTimezone, str); + } else { + tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, str, tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr); + } + break; + + default: + // TODO return the correct error code to client in the format for taos_errstr() + tscError("Invalid option %d", option); + return -1; + } + + return 0; +} + +#if 0 +#include "cJSON.h" +static setConfRet taos_set_config_imp(const char *config){ + setConfRet ret = {SET_CONF_RET_SUCC, {0}}; + static bool setConfFlag = false; + if (setConfFlag) { + ret.retCode = SET_CONF_RET_ERR_ONLY_ONCE; + strcpy(ret.retMsg, "configuration can only set once"); + return ret; + } + taosInitGlobalCfg(); + cJSON *root = cJSON_Parse(config); + if (root == NULL){ + ret.retCode = SET_CONF_RET_ERR_JSON_PARSE; + strcpy(ret.retMsg, "parse json error"); + return ret; + } + + int size = cJSON_GetArraySize(root); + if(!cJSON_IsObject(root) || size == 0) { + ret.retCode = SET_CONF_RET_ERR_JSON_INVALID; + strcpy(ret.retMsg, "json content is invalid, must be not empty object"); + return ret; + } + + if(size >= 1000) { + ret.retCode = SET_CONF_RET_ERR_TOO_LONG; + strcpy(ret.retMsg, "json object size is too long"); + return ret; + } + + for(int i = 0; i < size; i++){ + cJSON *item = cJSON_GetArrayItem(root, i); + if(!item) { + ret.retCode = SET_CONF_RET_ERR_INNER; + strcpy(ret.retMsg, "inner error"); + return ret; + } + if(!taosReadConfigOption(item->string, item->valuestring, NULL, NULL, TAOS_CFG_CSTATUS_OPTION, TSDB_CFG_CTYPE_B_CLIENT)){ + ret.retCode = SET_CONF_RET_ERR_PART; + if (strlen(ret.retMsg) == 0){ + snprintf(ret.retMsg, RET_MSG_LENGTH, "part error|%s", item->string); + }else{ + int tmp = RET_MSG_LENGTH - 1 - (int)strlen(ret.retMsg); + size_t leftSize = tmp >= 0 ? tmp : 0; + strncat(ret.retMsg, "|", leftSize); + tmp = RET_MSG_LENGTH - 1 - (int)strlen(ret.retMsg); + leftSize = tmp >= 0 ? tmp : 0; + strncat(ret.retMsg, item->string, leftSize); + } + } + } + cJSON_Delete(root); + setConfFlag = true; + return ret; +} + +setConfRet taos_set_config(const char *config){ + pthread_mutex_lock(&setConfMutex); + setConfRet ret = taos_set_config_imp(config); + pthread_mutex_unlock(&setConfMutex); + return ret; +} +#endif \ No newline at end of file diff --git a/source/client/test/CMakeLists.txt b/source/client/test/CMakeLists.txt new file mode 100644 index 0000000000..a3f2ad88bb --- /dev/null +++ b/source/client/test/CMakeLists.txt @@ -0,0 +1,18 @@ + +MESSAGE(STATUS "build parser unit test") + +# GoogleTest requires at least C++11 +SET(CMAKE_CXX_STANDARD 11) +AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) + +ADD_EXECUTABLE(clientTest ${SOURCE_LIST}) +TARGET_LINK_LIBRARIES( + clientTest + PUBLIC os util common transport gtest taos +) + +TARGET_INCLUDE_DIRECTORIES( + clientTest + PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/client/" + PRIVATE "${CMAKE_SOURCE_DIR}/source/libs/client/inc" +) diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index e69de29bb2..8da4caf7eb 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include +#include "tglobal.h" +#pragma GCC diagnostic ignored "-Wwrite-strings" + +#pragma GCC diagnostic ignored "-Wunused-function" +#pragma GCC diagnostic ignored "-Wunused-variable" +#pragma GCC diagnostic ignored "-Wsign-compare" + +#include "taos.h" + +namespace { +} // namespace + +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + +TEST(testCase, driverInit_Test) { + TAOS* pConn = taos_connect("ubuntu", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + taos_close(pConn); +} \ No newline at end of file diff --git a/source/common/inc/commonInt.h b/source/common/inc/commonInt.h index b8be8899f3..e7d2dba95c 100644 --- a/source/common/inc/commonInt.h +++ b/source/common/inc/commonInt.h @@ -20,8 +20,11 @@ extern "C" { #endif + + + #ifdef __cplusplus } #endif -#endif /*_TD_COMMON_INT_H_*/ \ No newline at end of file +#endif /*_TD_COMMON_INT_H_*/ diff --git a/source/common/src/tmessage.c b/source/common/src/taosmsg.c similarity index 99% rename from source/common/src/tmessage.c rename to source/common/src/taosmsg.c index 0b6dbfdb51..b35e3f1478 100644 --- a/source/common/src/tmessage.c +++ b/source/common/src/taosmsg.c @@ -16,3 +16,5 @@ #define TAOS_MESSAGE_C #include "taosmsg.h" + + diff --git a/source/common/src/tep.c b/source/common/src/tep.c new file mode 100644 index 0000000000..ef47a0e0c2 --- /dev/null +++ b/source/common/src/tep.c @@ -0,0 +1,41 @@ +#include "tep.h" +#include "tglobal.h" +#include "tlockfree.h" + +int taosGetFqdnPortFromEp(const char *ep, char *fqdn, uint16_t *port) { + *port = 0; + strcpy(fqdn, ep); + + char *temp = strchr(fqdn, ':'); + if (temp) { + *temp = 0; + *port = atoi(temp+1); + } + + if (*port == 0) { + *port = tsServerPort; + return -1; + } + + return 0; +} + +bool isEpsetEqual(const SEpSet *s1, const SEpSet *s2) { + if (s1->numOfEps != s2->numOfEps || s1->inUse != s2->inUse) { + return false; + } + + for (int32_t i = 0; i < s1->numOfEps; i++) { + if (s1->port[i] != s2->port[i] + || strncmp(s1->fqdn[i], s2->fqdn[i], TSDB_FQDN_LEN) != 0) + return false; + } + return true; +} + +void updateEpSet_s(SCorEpSet *pEpSet, SEpSet *pNewEpSet) { + taosCorBeginWrite(&pEpSet->version); + pEpSet->epSet = *pNewEpSet; + taosCorEndWrite(&pEpSet->version); +} + diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 7e71c6bfc7..dfbbe33b59 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -25,6 +25,7 @@ #include "tutil.h" #include "ttimezone.h" #include "tlocale.h" +#include "tep.h" // cluster char tsFirst[TSDB_EP_LEN] = {0}; @@ -161,16 +162,8 @@ int8_t tsEnableSlaveQuery = 1; int8_t tsEnableAdjustMaster = 1; // restful -int8_t tsEnableHttpModule = 1; int32_t tsRestRowLimit = 10240; -uint16_t tsHttpPort = 6041; // only tcp, range tcp[6041] -int32_t tsHttpCacheSessions = 1000; -int32_t tsHttpSessionExpire = 36000; -int32_t tsHttpMaxThreads = 2; -int8_t tsHttpEnableCompress = 1; -int8_t tsHttpEnableRecordSql = 0; int8_t tsTelegrafUseFieldNum = 0; -int8_t tsHttpDbNameMandatory = 0; // mqtt int8_t tsEnableMqttModule = 0; // not finished yet, not started it by default @@ -193,7 +186,6 @@ int8_t tsEnableStream = 1; // internal int8_t tsCompactMnodeWal = 0; int8_t tsPrintAuth = 0; -int8_t tscEmbedded = 0; char tsVnodeDir[PATH_MAX] = {0}; char tsDnodeDir[PATH_MAX] = {0}; char tsMnodeDir[PATH_MAX] = {0}; @@ -256,7 +248,6 @@ void taosSetAllDebugFlag() { sdbDebugFlag = debugFlag; dDebugFlag = debugFlag; vDebugFlag = debugFlag; - cDebugFlag = debugFlag; jniDebugFlag = debugFlag; odbcDebugFlag = debugFlag; httpDebugFlag = debugFlag; @@ -330,7 +321,7 @@ int32_t taosCfgDynamicOptions(char *msg) { if (strncasecmp(option, "resetlog", 8) == 0) { taosResetLog(); - taosPrintGlobalCfg(); + taosPrintCfg(); return 0; } @@ -394,7 +385,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 0; cfg.ptrLength = TSDB_EP_LEN; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "secondEp"; cfg.ptr = tsSecond; @@ -404,7 +395,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 0; cfg.ptrLength = TSDB_EP_LEN; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "fqdn"; cfg.ptr = tsLocalFqdn; @@ -414,7 +405,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 0; cfg.ptrLength = TSDB_FQDN_LEN; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); // port cfg.option = "serverPort"; @@ -425,7 +416,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 65056; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); // directory cfg.option = "configDir"; @@ -436,7 +427,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 0; cfg.ptrLength = TSDB_FILENAME_LEN; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "logDir"; cfg.ptr = tsLogDir; @@ -446,7 +437,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 0; cfg.ptrLength = TSDB_FILENAME_LEN; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "scriptDir"; cfg.ptr = tsScriptDir; @@ -456,7 +447,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 0; cfg.ptrLength = TSDB_FILENAME_LEN; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "dataDir"; cfg.ptr = tsDataDir; @@ -466,7 +457,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 0; cfg.ptrLength = TSDB_FILENAME_LEN; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "arbitrator"; cfg.ptr = tsArbitrator; @@ -476,7 +467,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 0; cfg.ptrLength = TSDB_EP_LEN; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); // dnode configs cfg.option = "numOfThreadsPerCore"; @@ -487,7 +478,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 10; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "numOfCommitThreads"; cfg.ptr = &tsNumOfCommitThreads; @@ -497,7 +488,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 100; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "ratioOfQueryCores"; cfg.ptr = &tsRatioOfQueryCores; @@ -507,7 +498,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 2.0f; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "maxNumOfDistinctRes"; cfg.ptr = &tsMaxNumOfDistinctResults; @@ -517,7 +508,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 10000*10000; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "numOfMnodes"; cfg.ptr = &tsNumOfMnodes; @@ -527,7 +518,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 3; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "vnodeBak"; cfg.ptr = &tsEnableVnodeBak; @@ -537,7 +528,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 1; cfg.ptrLength = 1; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "telemetryReporting"; cfg.ptr = &tsEnableTelemetryReporting; @@ -547,7 +538,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 1; cfg.ptrLength = 1; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "balance"; cfg.ptr = &tsEnableBalance; @@ -557,7 +548,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 1; cfg.ptrLength = 1; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "balanceInterval"; cfg.ptr = &tsBalanceInterval; @@ -567,7 +558,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 30000; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); // 0-any; 1-mnode; 2-vnode cfg.option = "role"; @@ -578,7 +569,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 2; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); // timer cfg.option = "maxTmrCtrl"; @@ -589,7 +580,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 2048; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "monitorInterval"; cfg.ptr = &tsMonitorInterval; @@ -599,7 +590,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 600; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_SECOND; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "offlineThreshold"; cfg.ptr = &tsOfflineThreshold; @@ -609,7 +600,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 86400 * 365; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_SECOND; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "rpcTimer"; cfg.ptr = &tsRpcTimer; @@ -619,7 +610,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 3000; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_MS; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "rpcForceTcp"; cfg.ptr = &tsRpcForceTcp; @@ -629,7 +620,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 1; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "rpcMaxTime"; cfg.ptr = &tsRpcMaxTime; @@ -639,7 +630,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 7200; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_SECOND; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "statusInterval"; cfg.ptr = &tsStatusInterval; @@ -649,7 +640,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 10; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_SECOND; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "shellActivityTimer"; cfg.ptr = &tsShellActivityTimer; @@ -659,7 +650,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 120; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_SECOND; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "minSlidingTime"; cfg.ptr = &tsMinSlidingTime; @@ -669,7 +660,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 1000000; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_MS; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "minIntervalTime"; cfg.ptr = &tsMinIntervalTime; @@ -679,7 +670,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 1000000; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_MS; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "maxStreamCompDelay"; cfg.ptr = &tsMaxStreamComputDelay; @@ -689,7 +680,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 1000000000; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_MS; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "maxFirstStreamCompDelay"; cfg.ptr = &tsStreamCompStartDelay; @@ -699,7 +690,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 1000000000; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_MS; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "retryStreamCompDelay"; cfg.ptr = &tsRetryStreamCompDelay; @@ -710,7 +701,7 @@ static void doInitGlobalConfig(void) { cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_MS; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "streamCompDelayRatio"; cfg.ptr = &tsStreamComputDelayRatio; cfg.valType = TAOS_CFG_VTYPE_FLOAT; @@ -719,7 +710,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 0.9f; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "maxVgroupsPerDb"; cfg.ptr = &tsMaxVgroupsPerDb; @@ -729,7 +720,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 8192; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "cache"; cfg.ptr = &tsCacheBlockSize; @@ -739,7 +730,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = TSDB_MAX_CACHE_BLOCK_SIZE; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_MB; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "blocks"; cfg.ptr = &tsBlocksPerVnode; @@ -749,7 +740,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = TSDB_MAX_TOTAL_BLOCKS; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "days"; cfg.ptr = &tsDaysPerFile; @@ -759,7 +750,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = TSDB_MAX_DAYS_PER_FILE; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "keep"; cfg.ptr = &tsDaysToKeep; @@ -769,7 +760,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = TSDB_MAX_KEEP; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "minRows"; cfg.ptr = &tsMinRowsInFileBlock; @@ -779,7 +770,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = TSDB_MAX_MIN_ROW_FBLOCK; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "maxRows"; cfg.ptr = &tsMaxRowsInFileBlock; @@ -789,7 +780,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = TSDB_MAX_MAX_ROW_FBLOCK; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "precision"; cfg.ptr = &tsTimePrecision; @@ -799,7 +790,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = TSDB_MAX_PRECISION; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "comp"; cfg.ptr = &tsCompression; @@ -809,7 +800,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = TSDB_MAX_COMP_LEVEL; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "walLevel"; cfg.ptr = &tsWAL; @@ -819,7 +810,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = TSDB_MAX_WAL_LEVEL; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "fsync"; cfg.ptr = &tsFsyncPeriod; @@ -829,7 +820,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = TSDB_MAX_FSYNC_PERIOD; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "replica"; cfg.ptr = &tsReplications; @@ -839,7 +830,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = TSDB_MAX_DB_REPLICA_OPTION; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "quorum"; cfg.ptr = &tsQuorum; @@ -849,7 +840,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = TSDB_MAX_DB_QUORUM_OPTION; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "update"; cfg.ptr = &tsUpdate; @@ -859,7 +850,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = TSDB_MAX_DB_UPDATE; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "cachelast"; cfg.ptr = &tsCacheLastRow; @@ -869,7 +860,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = TSDB_MAX_DB_CACHE_LAST_ROW; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "mqttHostName"; cfg.ptr = tsMqttHostName; @@ -879,7 +870,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 0; cfg.ptrLength = TSDB_MQTT_HOSTNAME_LEN; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "mqttPort"; cfg.ptr = tsMqttPort; @@ -889,7 +880,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 0; cfg.ptrLength = TSDB_MQTT_PORT_LEN; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "mqttTopic"; cfg.ptr = tsMqttTopic; @@ -899,7 +890,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 0; cfg.ptrLength = TSDB_MQTT_TOPIC_LEN; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "compressMsgSize"; cfg.ptr = &tsCompressMsgSize; @@ -909,7 +900,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 100000000.0f; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "compressColData"; cfg.ptr = &tsCompressColData; @@ -919,7 +910,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 100000000.0f; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "maxSQLLength"; cfg.ptr = &tsMaxSQLStringLen; @@ -929,7 +920,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = TSDB_MAX_ALLOWED_SQL_LEN; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_BYTE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "maxWildCardsLength"; cfg.ptr = &tsMaxWildCardsLen; @@ -939,7 +930,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = TSDB_MAX_FIELD_LEN; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_BYTE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "maxRegexStringLen"; cfg.ptr = &tsMaxRegexStringLen; @@ -949,7 +940,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = TSDB_MAX_FIELD_LEN; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_BYTE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "maxNumOfOrderedRes"; cfg.ptr = &tsMaxNumOfOrderedResults; @@ -959,7 +950,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = TSDB_MAX_ALLOWED_SQL_LEN; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "queryBufferSize"; cfg.ptr = &tsQueryBufferSize; @@ -969,7 +960,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 500000000000.0f; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_BYTE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "retrieveBlockingModel"; cfg.ptr = &tsRetrieveBlockingModel; @@ -979,7 +970,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 1; cfg.ptrLength = 1; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "keepColumnName"; cfg.ptr = &tsKeepOriginalColumnName; @@ -989,7 +980,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 1; cfg.ptrLength = 1; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); // locale & charset cfg.option = "timezone"; @@ -1000,7 +991,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 0; cfg.ptrLength = TSDB_TIMEZONE_LEN; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "locale"; cfg.ptr = tsLocale; @@ -1010,7 +1001,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 0; cfg.ptrLength = TSDB_LOCALE_LEN; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "charset"; cfg.ptr = tsCharset; @@ -1020,7 +1011,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 0; cfg.ptrLength = TSDB_LOCALE_LEN; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); // connect configs cfg.option = "maxShellConns"; @@ -1031,7 +1022,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 50000000; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "maxConnections"; cfg.ptr = &tsMaxConnections; @@ -1041,7 +1032,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 100000; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "minimalLogDirGB"; cfg.ptr = &tsMinimalLogDirGB; @@ -1051,7 +1042,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 10000000; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_GB; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "minimalTmpDirGB"; cfg.ptr = &tsReservedTmpDirectorySpace; @@ -1061,7 +1052,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 10000000; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_GB; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "minimalDataDirGB"; cfg.ptr = &tsMinimalDataDirGB; @@ -1071,7 +1062,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 10000000; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_GB; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); // module configs cfg.option = "flowctrl"; @@ -1082,7 +1073,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 1; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "slaveQuery"; cfg.ptr = &tsEnableSlaveQuery; @@ -1092,7 +1083,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 1; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "adjustMaster"; cfg.ptr = &tsEnableAdjustMaster; @@ -1102,17 +1093,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 1; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); - - cfg.option = "http"; - cfg.ptr = &tsEnableHttpModule; - cfg.valType = TAOS_CFG_VTYPE_INT8; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 0; - cfg.maxValue = 1; - cfg.ptrLength = 1; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "mqtt"; cfg.ptr = &tsEnableMqttModule; @@ -1122,7 +1103,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 1; cfg.ptrLength = 1; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "monitor"; cfg.ptr = &tsEnableMonitorModule; @@ -1132,7 +1113,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 1; cfg.ptrLength = 1; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "stream"; cfg.ptr = &tsEnableStream; @@ -1142,7 +1123,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 1; cfg.ptrLength = 1; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "topicBianryLen"; cfg.ptr = &tsTopicBianryLen; @@ -1152,17 +1133,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 16000; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); - - cfg.option = "httpEnableRecordSql"; - cfg.ptr = &tsHttpEnableRecordSql; - cfg.valType = TAOS_CFG_VTYPE_INT8; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; - cfg.minValue = 0; - cfg.maxValue = 1; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "telegrafUseFieldNum"; cfg.ptr = &tsTelegrafUseFieldNum; @@ -1172,17 +1143,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 1; cfg.ptrLength = 1; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); - - cfg.option = "httpMaxThreads"; - cfg.ptr = &tsHttpMaxThreads; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; - cfg.minValue = 2; - cfg.maxValue = 1000000; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "restfulRowLimit"; cfg.ptr = &tsRestRowLimit; @@ -1192,17 +1153,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 10000000; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); - - cfg.option = "httpDbNameMandatory"; - cfg.ptr = &tsHttpDbNameMandatory; - cfg.valType = TAOS_CFG_VTYPE_INT8; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; - cfg.minValue = 0; - cfg.maxValue = 1; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); // debug flag cfg.option = "numOfLogLines"; @@ -1213,7 +1164,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 2000000000; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "logKeepDays"; cfg.ptr = &tsLogKeepDays; @@ -1223,7 +1174,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 365000; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "asyncLog"; cfg.ptr = &tsAsyncLog; @@ -1233,7 +1184,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 1; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "debugFlag"; cfg.ptr = &debugFlag; @@ -1243,7 +1194,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 255; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "mDebugFlag"; cfg.ptr = &mDebugFlag; @@ -1253,7 +1204,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 255; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "dDebugFlag"; cfg.ptr = &dDebugFlag; @@ -1263,7 +1214,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 255; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "sDebugFlag"; cfg.ptr = &sDebugFlag; @@ -1273,7 +1224,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 255; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "wDebugFlag"; cfg.ptr = &wDebugFlag; @@ -1283,7 +1234,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 255; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "sdbDebugFlag"; @@ -1294,7 +1245,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 255; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "rpcDebugFlag"; cfg.ptr = &rpcDebugFlag; @@ -1304,7 +1255,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 255; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "tmrDebugFlag"; cfg.ptr = &tmrDebugFlag; @@ -1314,7 +1265,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 255; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "cDebugFlag"; cfg.ptr = &cDebugFlag; @@ -1324,7 +1275,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 255; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "jniDebugFlag"; cfg.ptr = &jniDebugFlag; @@ -1334,7 +1285,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 255; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "odbcDebugFlag"; cfg.ptr = &odbcDebugFlag; @@ -1344,7 +1295,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 255; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "uDebugFlag"; cfg.ptr = &uDebugFlag; @@ -1354,7 +1305,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 255; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "httpDebugFlag"; cfg.ptr = &httpDebugFlag; @@ -1364,7 +1315,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 255; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "mqttDebugFlag"; cfg.ptr = &mqttDebugFlag; @@ -1374,7 +1325,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 255; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "monDebugFlag"; cfg.ptr = &monDebugFlag; @@ -1384,7 +1335,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 255; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "qDebugFlag"; cfg.ptr = &qDebugFlag; @@ -1394,7 +1345,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 255; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "vDebugFlag"; cfg.ptr = &vDebugFlag; @@ -1404,7 +1355,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 255; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "tsdbDebugFlag"; cfg.ptr = &tsdbDebugFlag; @@ -1414,7 +1365,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 255; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "cqDebugFlag"; cfg.ptr = &cqDebugFlag; @@ -1424,7 +1375,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 255; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "enableRecordSql"; cfg.ptr = &tsTscEnableRecordSql; @@ -1434,7 +1385,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 1; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "enableCoreFile"; cfg.ptr = &tsEnableCoreFile; @@ -1444,7 +1395,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 1; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); // version info cfg.option = "gitinfo"; @@ -1455,7 +1406,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 0; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "gitinfoOfInternal"; cfg.ptr = gitinfoOfInternal; @@ -1465,7 +1416,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 0; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "buildinfo"; cfg.ptr = buildinfo; @@ -1475,7 +1426,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 0; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "version"; cfg.ptr = version; @@ -1485,7 +1436,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 0; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "maxBinaryDisplayWidth"; cfg.ptr = &tsMaxBinaryDisplayWidth; @@ -1495,7 +1446,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 65536; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "tempDir"; cfg.ptr = tsTempDir; @@ -1505,7 +1456,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 0; cfg.ptrLength = PATH_MAX; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "tsdbMetaCompactRatio"; cfg.ptr = &tsTsdbMetaCompactRatio; @@ -1515,7 +1466,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 100; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); // enable kill long query cfg.option = "deadLockKillQuery"; @@ -1526,7 +1477,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 1; cfg.ptrLength = 1; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); #ifdef TD_TSZ // lossy compress @@ -1538,7 +1489,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 0; cfg.ptrLength = tListLen(lossyColumns); cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "fPrecision"; cfg.ptr = &fPrecision; @@ -1550,7 +1501,7 @@ static void doInitGlobalConfig(void) { cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "dPrecision"; cfg.ptr = &dPrecision; @@ -1560,7 +1511,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = MAX_FLOAT; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "maxRange"; cfg.ptr = &maxRange; @@ -1570,7 +1521,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 65536; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); cfg.option = "range"; cfg.ptr = &curRange; @@ -1580,7 +1531,7 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 65536; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); + taosAddConfigOption(cfg); assert(tsGlobalConfigNum == TSDB_CFG_MAX_NUM); #else //assert(tsGlobalConfigNum == TSDB_CFG_MAX_NUM - 5); @@ -1592,7 +1543,7 @@ void taosInitGlobalCfg() { pthread_once(&tsInitGlobalCfgOnce, doInitGlobalConfig); } -int32_t taosCheckGlobalCfg() { +int32_t taosCheckAndPrintCfg() { char fqdn[TSDB_FQDN_LEN]; uint16_t port; @@ -1640,37 +1591,13 @@ int32_t taosCheckGlobalCfg() { tsNumOfCores = 1; } - if (tsHttpMaxThreads == 2) { - int32_t halfNumOfCores = tsNumOfCores >> 1; - if (halfNumOfCores > 2) { - tsHttpMaxThreads = halfNumOfCores; - } - } - - // todo refactor - tsVersion = 0; - for (int ver = 0, i = 0; i < TSDB_VERSION_LEN; ++i) { - if (version[i] >= '0' && version[i] <= '9') { - ver = ver * 10 + (version[i] - '0'); - } else if (version[i] == '.') { - tsVersion |= ver & 0xFF; - tsVersion <<= 8; - - ver = 0; - } else if (version[i] == 0) { - tsVersion |= ver & 0xFF; - - break; - } - } - if (tsQueryBufferSize >= 0) { tsQueryBufferSizeBytes = tsQueryBufferSize * 1048576UL; } uInfo(" check global cfg completed"); uInfo("=================================="); - taosPrintGlobalCfg(); + taosPrintCfg(); return 0; } @@ -1703,3 +1630,4 @@ bool taosCheckBalanceCfgOptions(const char *option, int32_t *vnodeId, int32_t *d return true; } + \ No newline at end of file diff --git a/source/dnode/mgmt/daemon/src/daemon.c b/source/dnode/mgmt/daemon/src/daemon.c index 82201df072..0a725d9975 100644 --- a/source/dnode/mgmt/daemon/src/daemon.c +++ b/source/dnode/mgmt/daemon/src/daemon.c @@ -112,12 +112,12 @@ int dmnReadConfig(const char *path) { return -1; } - if (taosReadGlobalCfg() != 0) { + if (taosReadCfgFromFile() != 0) { uError("failed to read global config"); return -1; } - if (taosCheckGlobalCfg() != 0) { + if (taosCheckAndPrintCfg() != 0) { uError("failed to check global config"); return -1; } diff --git a/source/dnode/mgmt/impl/src/dndDnode.c b/source/dnode/mgmt/impl/src/dndDnode.c index 130cbbef07..9e2d4d4c9b 100644 --- a/source/dnode/mgmt/impl/src/dndDnode.c +++ b/source/dnode/mgmt/impl/src/dndDnode.c @@ -17,6 +17,7 @@ #include "dndDnode.h" #include "dndTransport.h" #include "dndVnodes.h" +#include "tep.h" int32_t dndGetDnodeId(SDnode *pDnode) { SDnodeMgmt *pMgmt = &pDnode->dmgmt; diff --git a/source/dnode/mgmt/impl/src/dndTransport.c b/source/dnode/mgmt/impl/src/dndTransport.c index 2ec02ac203..6dc46cefcd 100644 --- a/source/dnode/mgmt/impl/src/dndTransport.c +++ b/source/dnode/mgmt/impl/src/dndTransport.c @@ -72,7 +72,7 @@ static void dndInitMsgFp(STransMgmt *pMgmt) { pMgmt->msgFp[TSDB_MSG_TYPE_CREATE_STB] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TSDB_MSG_TYPE_ALTER_STB] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TSDB_MSG_TYPE_DROP_STB] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_STB_VGROUP] = dndProcessMnodeReadMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_VGROUP_LIST] = dndProcessMnodeReadMsg; pMgmt->msgFp[TSDB_MSG_TYPE_KILL_QUERY] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TSDB_MSG_TYPE_KILL_STREAM] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TSDB_MSG_TYPE_KILL_CONN] = dndProcessMnodeWriteMsg; @@ -235,18 +235,18 @@ static void dndSendMsgToMnodeRecv(SDnode *pDnode, SRpcMsg *pRpcMsg, SRpcMsg *pRp static int32_t dndAuthInternalMsg(SDnode *pDnode, char *user, char *spi, char *encrypt, char *secret, char *ckey) { if (strcmp(user, INTERNAL_USER) == 0) { // A simple temporary implementation - char pass[32] = {0}; + char pass[TSDB_PASSWORD_LEN] = {0}; taosEncryptPass((uint8_t *)(INTERNAL_SECRET), strlen(INTERNAL_SECRET), pass); - memcpy(secret, pass, TSDB_KEY_LEN); + memcpy(secret, pass, TSDB_PASSWORD_LEN); *spi = 0; *encrypt = 0; *ckey = 0; return 0; } else if (strcmp(user, TSDB_NETTEST_USER) == 0) { // A simple temporary implementation - char pass[32] = {0}; + char pass[TSDB_PASSWORD_LEN] = {0}; taosEncryptPass((uint8_t *)(TSDB_NETTEST_USER), strlen(TSDB_NETTEST_USER), pass); - memcpy(secret, pass, TSDB_KEY_LEN); + memcpy(secret, pass, TSDB_PASSWORD_LEN); *spi = 0; *encrypt = 0; *ckey = 0; @@ -288,8 +288,8 @@ static int32_t dndRetrieveUserAuthInfo(void *parent, char *user, char *spi, char dError("user:%s, failed to get user auth from other mnodes since %s", user, terrstr()); } else { SAuthRsp *pRsp = rpcRsp.pCont; - memcpy(secret, pRsp->secret, TSDB_KEY_LEN); - memcpy(ckey, pRsp->ckey, TSDB_KEY_LEN); + memcpy(secret, pRsp->secret, TSDB_PASSWORD_LEN); + memcpy(ckey, pRsp->ckey, TSDB_PASSWORD_LEN); *spi = pRsp->spi; *encrypt = pRsp->encrypt; dDebug("user:%s, success to get user auth from other mnodes", user); @@ -368,4 +368,4 @@ void dndSendMsgToMnode(SDnode *pDnode, SRpcMsg *pMsg) { SEpSet epSet = {0}; dndGetMnodeEpSet(pDnode, &epSet); dndSendMsgToDnode(pDnode, &epSet, pMsg); -} \ No newline at end of file +} diff --git a/source/dnode/mgmt/impl/test/sut/deploy.cpp b/source/dnode/mgmt/impl/test/sut/deploy.cpp index 640f300699..be1506bccf 100644 --- a/source/dnode/mgmt/impl/test/sut/deploy.cpp +++ b/source/dnode/mgmt/impl/test/sut/deploy.cpp @@ -120,7 +120,7 @@ SClient* createClient(const char* user, const char* pass, const char* fqdn, uint SClient* pClient = (SClient*)calloc(1, sizeof(SClient)); ASSERT(pClient); - char secretEncrypt[32] = {0}; + char secretEncrypt[TSDB_PASSWORD_LEN] = {0}; taosEncryptPass((uint8_t*)pass, strlen(pass), secretEncrypt); SRpcInit rpcInit; diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 74b4b1faa3..0f4839392e 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -180,13 +180,11 @@ typedef struct SAcctObj { typedef struct SUserObj { char user[TSDB_USER_LEN]; - char pass[TSDB_KEY_LEN]; + char pass[TSDB_PASSWORD_LEN]; char acct[TSDB_USER_LEN]; int64_t createdTime; int64_t updateTime; - int8_t superAuth; - int8_t readAuth; - int8_t writeAuth; + int8_t superUser; int32_t acctId; SHashObj *prohibitDbHash; } SUserObj; diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index b429879789..59d84e5760 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -29,6 +29,7 @@ typedef struct { char user[TSDB_USER_LEN]; char app[TSDB_APP_NAME_LEN]; // app name that invokes taosc int32_t pid; // pid of app that invokes taosc + int64_t appStartTime; // app start time int32_t id; int8_t killed; int8_t align; @@ -44,7 +45,7 @@ typedef struct { SQueryDesc *pQueries; } SConnObj; -static SConnObj *mndCreateConn(SMnode *pMnode, char *user, uint32_t ip, uint16_t port, int32_t pid, const char *app); +static SConnObj *mndCreateConn(SMnode *pMnode, char *user, uint32_t ip, uint16_t port, int32_t pid, const char *app, int64_t startTime); static void mndFreeConn(SConnObj *pConn); static SConnObj *mndAcquireConn(SMnode *pMnode, int32_t connId); static void mndReleaseConn(SMnode *pMnode, SConnObj *pConn); @@ -102,13 +103,14 @@ void mndCleanupProfile(SMnode *pMnode) { } } -static SConnObj *mndCreateConn(SMnode *pMnode, char *user, uint32_t ip, uint16_t port, int32_t pid, const char *app) { +static SConnObj *mndCreateConn(SMnode *pMnode, char *user, uint32_t ip, uint16_t port, int32_t pid, const char *app, int64_t startTime) { SProfileMgmt *pMgmt = &pMnode->profileMgmt; int32_t connId = atomic_add_fetch_32(&pMgmt->connId, 1); if (connId == 0) atomic_add_fetch_32(&pMgmt->connId, 1); SConnObj connObj = {.pid = pid, + .appStartTime = startTime, .id = connId, .killed = 0, .port = port, @@ -195,6 +197,7 @@ static int32_t mndProcessConnectMsg(SMnodeMsg *pMsg) { SMnode *pMnode = pMsg->pMnode; SConnectMsg *pReq = pMsg->rpcMsg.pCont; pReq->pid = htonl(pReq->pid); + pReq->startTime = htobe64(pReq->startTime); SRpcConnInfo info = {0}; if (rpcGetConnInfo(pMsg->rpcMsg.handle, &info) != 0) { @@ -216,7 +219,7 @@ static int32_t mndProcessConnectMsg(SMnodeMsg *pMsg) { mndReleaseDb(pMnode, pDb); } - SConnObj *pConn = mndCreateConn(pMnode, info.user, info.clientIp, info.clientPort, pReq->pid, pReq->app); + SConnObj *pConn = mndCreateConn(pMnode, info.user, info.clientIp, info.clientPort, pReq->pid, pReq->app, pReq->startTime); if (pConn == NULL) { mError("user:%s, failed to login from %s while create connection since %s", pMsg->user, ip, terrstr()); return -1; @@ -233,9 +236,7 @@ static int32_t mndProcessConnectMsg(SMnodeMsg *pMsg) { SUserObj *pUser = mndAcquireUser(pMnode, pMsg->user); if (pUser != NULL) { pRsp->acctId = htonl(pUser->acctId); - pRsp->superAuth = pUser->superAuth; - pRsp->readAuth = pUser->readAuth; - pRsp->writeAuth = pUser->writeAuth; + pRsp->superUser = pUser->superUser; mndReleaseUser(pMnode, pUser); } @@ -246,7 +247,8 @@ static int32_t mndProcessConnectMsg(SMnodeMsg *pMsg) { pMsg->contLen = sizeof(SConnectRsp); pMsg->pCont = pRsp; - mDebug("user:%s, login from %s, conn:%d", info.user, ip, pConn->id); + + mDebug("user:%s, login from %s, conn:%d, app:%s", info.user, ip, pConn->id, pReq->app); return 0; } @@ -301,7 +303,7 @@ static int32_t mndProcessHeartBeatMsg(SMnodeMsg *pMsg) { SConnObj *pConn = mndAcquireConn(pMnode, pReq->connId); if (pConn == NULL) { - pConn = mndCreateConn(pMnode, info.user, info.clientIp, info.clientPort, pReq->pid, pReq->app); + pConn = mndCreateConn(pMnode, info.user, info.clientIp, info.clientPort, pReq->pid, pReq->app, 0); if (pConn == NULL) { mError("user:%s, conn:%d is freed and failed to create new conn since %s", pMsg->user, pReq->connId, terrstr()); return -1; @@ -368,7 +370,7 @@ static int32_t mndProcessKillQueryMsg(SMnodeMsg *pMsg) { SUserObj *pUser = mndAcquireUser(pMnode, pMsg->user); if (pUser == NULL) return 0; - if (!pUser->superAuth) { + if (!pUser->superUser) { mndReleaseUser(pMnode, pUser); terrno = TSDB_CODE_MND_NO_RIGHTS; return -1; @@ -399,7 +401,7 @@ static int32_t mndProcessKillStreamMsg(SMnodeMsg *pMsg) { SUserObj *pUser = mndAcquireUser(pMnode, pMsg->user); if (pUser == NULL) return 0; - if (!pUser->superAuth) { + if (!pUser->superUser) { mndReleaseUser(pMnode, pUser); terrno = TSDB_CODE_MND_NO_RIGHTS; return -1; @@ -430,7 +432,7 @@ static int32_t mndProcessKillConnectionMsg(SMnodeMsg *pMsg) { SUserObj *pUser = mndAcquireUser(pMnode, pMsg->user); if (pUser == NULL) return 0; - if (!pUser->superAuth) { + if (!pUser->superUser) { mndReleaseUser(pMnode, pUser); terrno = TSDB_CODE_MND_NO_RIGHTS; return -1; @@ -459,7 +461,7 @@ static int32_t mndGetConnsMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg * SUserObj *pUser = mndAcquireUser(pMnode, pMsg->user); if (pUser == NULL) return 0; - if (!pUser->superAuth) { + if (!pUser->superUser) { mndReleaseUser(pMnode, pUser); terrno = TSDB_CODE_MND_NO_RIGHTS; return -1; @@ -587,7 +589,7 @@ static int32_t mndGetQueryMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg * SUserObj *pUser = mndAcquireUser(pMnode, pMsg->user); if (pUser == NULL) return 0; - if (!pUser->superAuth) { + if (!pUser->superUser) { mndReleaseUser(pMnode, pUser); terrno = TSDB_CODE_MND_NO_RIGHTS; return -1; @@ -803,7 +805,7 @@ static int32_t mndGetStreamMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg SUserObj *pUser = mndAcquireUser(pMnode, pMsg->user); if (pUser == NULL) return 0; - if (!pUser->superAuth) { + if (!pUser->superUser) { mndReleaseUser(pMnode, pUser); terrno = TSDB_CODE_MND_NO_RIGHTS; return -1; diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index 43b9e94e06..452c14886d 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -65,11 +65,9 @@ static int32_t mndCreateDefaultUser(SMnode *pMnode, char *acct, char *user, char taosEncryptPass((uint8_t *)pass, strlen(pass), userObj.pass); userObj.createdTime = taosGetTimestampMs(); userObj.updateTime = userObj.createdTime; - userObj.readAuth = 1; - userObj.writeAuth = 1; if (strcmp(user, TSDB_DEFAULT_USER) == 0) { - userObj.superAuth = 1; + userObj.superUser = 1; } SSdbRaw *pRaw = mndUserActionEncode(&userObj); @@ -89,7 +87,7 @@ static int32_t mndCreateDefaultUsers(SMnode *pMnode) { if (mndCreateDefaultUser(pMnode, TSDB_DEFAULT_USER, "_" TSDB_DEFAULT_USER, TSDB_DEFAULT_PASS) != 0) { return -1; } -#endif +#endif return 0; } @@ -100,13 +98,11 @@ static SSdbRaw *mndUserActionEncode(SUserObj *pUser) { int32_t dataPos = 0; SDB_SET_BINARY(pRaw, dataPos, pUser->user, TSDB_USER_LEN) - SDB_SET_BINARY(pRaw, dataPos, pUser->pass, TSDB_KEY_LEN) + SDB_SET_BINARY(pRaw, dataPos, pUser->pass, TSDB_PASSWORD_LEN) SDB_SET_BINARY(pRaw, dataPos, pUser->acct, TSDB_USER_LEN) SDB_SET_INT64(pRaw, dataPos, pUser->createdTime) SDB_SET_INT64(pRaw, dataPos, pUser->updateTime) - SDB_SET_INT8(pRaw, dataPos, pUser->superAuth) - SDB_SET_INT8(pRaw, dataPos, pUser->readAuth) - SDB_SET_INT8(pRaw, dataPos, pUser->writeAuth) + SDB_SET_INT8(pRaw, dataPos, pUser->superUser) SDB_SET_DATALEN(pRaw, dataPos); return pRaw; @@ -128,13 +124,11 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) { int32_t dataPos = 0; SDB_GET_BINARY(pRaw, pRow, dataPos, pUser->user, TSDB_USER_LEN) - SDB_GET_BINARY(pRaw, pRow, dataPos, pUser->pass, TSDB_KEY_LEN) + SDB_GET_BINARY(pRaw, pRow, dataPos, pUser->pass, TSDB_PASSWORD_LEN) SDB_GET_BINARY(pRaw, pRow, dataPos, pUser->acct, TSDB_USER_LEN) SDB_GET_INT64(pRaw, pRow, dataPos, &pUser->createdTime) SDB_GET_INT64(pRaw, pRow, dataPos, &pUser->updateTime) - SDB_GET_INT8(pRaw, pRow, dataPos, &pUser->superAuth) - SDB_GET_INT8(pRaw, pRow, dataPos, &pUser->readAuth) - SDB_GET_INT8(pRaw, pRow, dataPos, &pUser->writeAuth) + SDB_GET_INT8(pRaw, pRow, dataPos, &pUser->superUser) return pRow; } @@ -173,13 +167,11 @@ static int32_t mndUserActionDelete(SSdb *pSdb, SUserObj *pUser) { static int32_t mndUserActionUpdate(SSdb *pSdb, SUserObj *pOldUser, SUserObj *pNewUser) { mTrace("user:%s, perform update action", pOldUser->user); memcpy(pOldUser->user, pNewUser->user, TSDB_USER_LEN); - memcpy(pOldUser->pass, pNewUser->pass, TSDB_KEY_LEN); + memcpy(pOldUser->pass, pNewUser->pass, TSDB_PASSWORD_LEN); memcpy(pOldUser->acct, pNewUser->acct, TSDB_USER_LEN); pOldUser->createdTime = pNewUser->createdTime; pOldUser->updateTime = pNewUser->updateTime; - pOldUser->superAuth = pNewUser->superAuth; - pOldUser->readAuth = pNewUser->readAuth; - pOldUser->writeAuth = pNewUser->writeAuth; + pOldUser->superUser = pNewUser->superUser; return 0; } @@ -200,9 +192,7 @@ static int32_t mndCreateUser(SMnode *pMnode, char *acct, char *user, char *pass, taosEncryptPass((uint8_t *)pass, strlen(pass), userObj.pass); userObj.createdTime = taosGetTimestampMs(); userObj.updateTime = userObj.createdTime; - userObj.superAuth = 0; - userObj.readAuth = 1; - userObj.writeAuth = 1; + userObj.superUser = 0; STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, pMsg->rpcMsg.handle); if (pTrans == NULL) { @@ -475,7 +465,7 @@ static int32_t mndRetrieveUsers(SMnodeMsg *pMsg, SShowObj *pShow, char *data, in cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - if (pUser->superAuth) { + if (pUser->superUser) { const char *src = "super"; STR_WITH_SIZE_TO_VARSTR(pWrite, src, strlen(src)); } else { diff --git a/source/dnode/vnode/impl/CMakeLists.txt b/source/dnode/vnode/impl/CMakeLists.txt index d6d267c4d4..6972605afd 100644 --- a/source/dnode/vnode/impl/CMakeLists.txt +++ b/source/dnode/vnode/impl/CMakeLists.txt @@ -18,6 +18,6 @@ target_link_libraries( ) # test -#if(${BUILD_TEST}) -# add_subdirectory(test) -#endif(${BUILD_TEST}) \ No newline at end of file +if(${BUILD_TEST}) + add_subdirectory(test) +endif(${BUILD_TEST}) \ No newline at end of file diff --git a/source/dnode/vnode/impl/inc/vnodeBufferPool.h b/source/dnode/vnode/impl/inc/vnodeBufferPool.h index d64dc93847..d96671d2bd 100644 --- a/source/dnode/vnode/impl/inc/vnodeBufferPool.h +++ b/source/dnode/vnode/impl/inc/vnodeBufferPool.h @@ -27,7 +27,10 @@ typedef struct SVBufPool SVBufPool; int vnodeOpenBufPool(SVnode *pVnode); void vnodeCloseBufPool(SVnode *pVnode); +int vnodeBufPoolSwitch(SVnode *pVnode); +int vnodeBufPoolRecycle(SVnode *pVnode); void *vnodeMalloc(SVnode *pVnode, uint64_t size); +bool vnodeBufPoolIsFull(SVnode *pVnode); #ifdef __cplusplus } diff --git a/source/dnode/vnode/impl/inc/vnodeCommit.h b/source/dnode/vnode/impl/inc/vnodeCommit.h index c8ff4947aa..031089ba14 100644 --- a/source/dnode/vnode/impl/inc/vnodeCommit.h +++ b/source/dnode/vnode/impl/inc/vnodeCommit.h @@ -22,8 +22,9 @@ extern "C" { #endif -bool vnodeShouldCommit(SVnode *pVnode); -int vnodeAsyncCommit(SVnode *pVnode); +#define vnodeShouldCommit vnodeBufPoolIsFull +int vnodeAsyncCommit(SVnode *pVnode); +int vnodeCommit(void *arg); #ifdef __cplusplus } diff --git a/source/dnode/vnode/impl/inc/vnodeDef.h b/source/dnode/vnode/impl/inc/vnodeDef.h index c92de433c3..e6a88c7629 100644 --- a/source/dnode/vnode/impl/inc/vnodeDef.h +++ b/source/dnode/vnode/impl/inc/vnodeDef.h @@ -18,15 +18,19 @@ #include "mallocator.h" #include "sync.h" -#include "tlockfree.h" -#include "wal.h" #include "tcoding.h" +#include "tdlist.h" +#include "tlockfree.h" +#include "tmacro.h" +#include "wal.h" #include "vnode.h" + #include "vnodeBufferPool.h" #include "vnodeCfg.h" #include "vnodeCommit.h" #include "vnodeFS.h" +#include "vnodeMemAllocator.h" #include "vnodeRequest.h" #include "vnodeStateMgr.h" #include "vnodeSync.h" @@ -35,6 +39,27 @@ extern "C" { #endif +typedef struct SVnodeTask { + TD_DLIST_NODE(SVnodeTask); + void* arg; + int (*execute)(void*); +} SVnodeTask; + +typedef struct SVnodeMgr { + td_mode_flag_t vnodeInitFlag; + td_mode_flag_t vnodeClearFlag; + // For commit + bool stop; + uint16_t nthreads; + pthread_t* threads; + pthread_mutex_t mutex; + pthread_cond_t hasTask; + TD_DLIST(SVnodeTask) queue; + // For vnode Mgmt +} SVnodeMgr; + +extern SVnodeMgr vnodeMgr; + struct SVnode { char* path; SVnodeCfg config; @@ -48,6 +73,8 @@ struct SVnode { SVnodeFS* pFs; }; +int vnodeScheduleTask(SVnodeTask* task); + #ifdef __cplusplus } #endif diff --git a/source/dnode/vnode/impl/inc/vnodeMemAllocator.h b/source/dnode/vnode/impl/inc/vnodeMemAllocator.h index 9184eb416b..bdafbf31a7 100644 --- a/source/dnode/vnode/impl/inc/vnodeMemAllocator.h +++ b/source/dnode/vnode/impl/inc/vnodeMemAllocator.h @@ -16,15 +16,38 @@ #ifndef _TD_VNODE_MEM_ALLOCATOR_H_ #define _TD_VNODE_MEM_ALLOCATOR_H_ -#include "mallocator.h" -#include "vnode.h" +#include "os.h" #ifdef __cplusplus extern "C" { #endif -SMemAllocator *vnodeCreateMemAllocator(SVnode *pVnode); -void vnodeDestroyMemAllocator(SMemAllocator *pma); +typedef struct SVArenaNode SVArenaNode; +typedef struct SVMemAllocator SVMemAllocator; + +struct SVArenaNode { + TD_SLIST_NODE(SVArenaNode); + uint64_t size; // current node size + void * ptr; + char data[]; +}; + +struct SVMemAllocator { + T_REF_DECLARE() + TD_DLIST_NODE(SVMemAllocator); + uint64_t capacity; + uint64_t ssize; + uint64_t lsize; + SVArenaNode *pNode; + TD_SLIST(SVArenaNode) nlist; +}; + +SVMemAllocator *vmaCreate(uint64_t capacity, uint64_t ssize, uint64_t lsize); +void vmaDestroy(SVMemAllocator *pVMA); +void vmaReset(SVMemAllocator *pVMA); +void * vmaMalloc(SVMemAllocator *pVMA, uint64_t size); +void vmaFree(SVMemAllocator *pVMA, void *ptr); +bool vmaIsFull(SVMemAllocator *pVMA); #ifdef __cplusplus } diff --git a/source/dnode/vnode/impl/src/vnodeArenaMAImpl.c b/source/dnode/vnode/impl/src/vnodeArenaMAImpl.c new file mode 100644 index 0000000000..99d4781df9 --- /dev/null +++ b/source/dnode/vnode/impl/src/vnodeArenaMAImpl.c @@ -0,0 +1,117 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "vnodeDef.h" + +static SVArenaNode *vArenaNodeNew(uint64_t capacity); +static void vArenaNodeFree(SVArenaNode *pNode); + +SVMemAllocator *vmaCreate(uint64_t capacity, uint64_t ssize, uint64_t lsize) { + SVMemAllocator *pVMA = (SVMemAllocator *)malloc(sizeof(*pVMA)); + if (pVMA == NULL) { + return NULL; + } + + pVMA->capacity = capacity; + pVMA->ssize = ssize; + pVMA->lsize = lsize; + tSListInit(&(pVMA->nlist)); + + pVMA->pNode = vArenaNodeNew(capacity); + if (pVMA->pNode == NULL) { + free(pVMA); + return NULL; + } + + tSListPush(&(pVMA->nlist), pVMA->pNode); + + return pVMA; +} + +void vmaDestroy(SVMemAllocator *pVMA) { + if (pVMA) { + while (TD_SLIST_NELES(&(pVMA->nlist)) > 1) { + SVArenaNode *pNode = TD_SLIST_HEAD(&(pVMA->nlist)); + tSListPop(&(pVMA->nlist)); + vArenaNodeFree(pNode); + } + + free(pVMA); + } +} + +void vmaReset(SVMemAllocator *pVMA) { + while (TD_SLIST_NELES(&(pVMA->nlist)) > 1) { + SVArenaNode *pNode = TD_SLIST_HEAD(&(pVMA->nlist)); + tSListPop(&(pVMA->nlist)); + vArenaNodeFree(pNode); + } + + SVArenaNode *pNode = TD_SLIST_HEAD(&(pVMA->nlist)); + pNode->ptr = pNode->data; +} + +void *vmaMalloc(SVMemAllocator *pVMA, uint64_t size) { + SVArenaNode *pNode = TD_SLIST_HEAD(&(pVMA->nlist)); + void * ptr; + + if (pNode->size < POINTER_DISTANCE(pNode->ptr, pNode->data) + size) { + uint64_t capacity = MAX(pVMA->ssize, size); + pNode = vArenaNodeNew(capacity); + if (pNode == NULL) { + // TODO: handle error + return NULL; + } + + tSListPush(&(pVMA->nlist), pNode); + } + + ptr = pNode->ptr; + pNode->ptr = POINTER_SHIFT(ptr, size); + + return ptr; +} + +void vmaFree(SVMemAllocator *pVMA, void *ptr) { + // TODO +} + +bool vmaIsFull(SVMemAllocator *pVMA) { + SVArenaNode *pNode = TD_SLIST_HEAD(&(pVMA->nlist)); + + return (TD_SLIST_NELES(&(pVMA->nlist)) > 1) || + (pNode->size < POINTER_DISTANCE(pNode->ptr, pNode->data) + pVMA->lsize); +} + +/* ------------------------ STATIC METHODS ------------------------ */ +static SVArenaNode *vArenaNodeNew(uint64_t capacity) { + SVArenaNode *pNode = NULL; + + pNode = (SVArenaNode *)malloc(sizeof(*pNode) + capacity); + if (pNode == NULL) { + return NULL; + } + + pNode->size = capacity; + pNode->ptr = pNode->data; + + return pNode; +} + +static void vArenaNodeFree(SVArenaNode *pNode) { + if (pNode) { + free(pNode); + } +} \ No newline at end of file diff --git a/source/dnode/vnode/impl/src/vnodeBufferPool.c b/source/dnode/vnode/impl/src/vnodeBufferPool.c index 00203ed9b6..152a346f0a 100644 --- a/source/dnode/vnode/impl/src/vnodeBufferPool.c +++ b/source/dnode/vnode/impl/src/vnodeBufferPool.c @@ -19,14 +19,114 @@ #define VNODE_BUF_POOL_SHARDS 3 struct SVBufPool { - // buffer pool impl - SList free; - SList incycle; - SListNode *inuse; + pthread_mutex_t mutex; + pthread_cond_t hasFree; + TD_DLIST(SVMemAllocator) free; + TD_DLIST(SVMemAllocator) incycle; + SVMemAllocator *inuse; // MAF for submodules - SMemAllocatorFactory maf; + // SMemAllocatorFactory maf; }; +int vnodeOpenBufPool(SVnode *pVnode) { + uint64_t capacity; + + if ((pVnode->pBufPool = (SVBufPool *)calloc(1, sizeof(SVBufPool))) == NULL) { + /* TODO */ + return -1; + } + + tDListInit(&(pVnode->pBufPool->free)); + tDListInit(&(pVnode->pBufPool->incycle)); + + pVnode->pBufPool->inuse = NULL; + + // TODO + capacity = pVnode->config.wsize / VNODE_BUF_POOL_SHARDS; + + for (int i = 0; i < VNODE_BUF_POOL_SHARDS; i++) { + SVMemAllocator *pVMA = vmaCreate(capacity, pVnode->config.ssize, pVnode->config.lsize); + if (pVMA == NULL) { + // TODO: handle error + return -1; + } + + tDListAppend(&(pVnode->pBufPool->free), pVMA); + } + + return 0; +} + +void vnodeCloseBufPool(SVnode *pVnode) { + if (pVnode->pBufPool) { + vmaDestroy(pVnode->pBufPool->inuse); + + while (true) { + SVMemAllocator *pVMA = TD_DLIST_HEAD(&(pVnode->pBufPool->incycle)); + if (pVMA == NULL) break; + tDListPop(&(pVnode->pBufPool->incycle), pVMA); + vmaDestroy(pVMA); + } + + while (true) { + SVMemAllocator *pVMA = TD_DLIST_HEAD(&(pVnode->pBufPool->free)); + if (pVMA == NULL) break; + tDListPop(&(pVnode->pBufPool->free), pVMA); + vmaDestroy(pVMA); + } + + free(pVnode->pBufPool); + pVnode->pBufPool = NULL; + } +} + +int vnodeBufPoolSwitch(SVnode *pVnode) { + SVMemAllocator *pvma = pVnode->pBufPool->inuse; + + pVnode->pBufPool->inuse = NULL; + + tDListAppend(&(pVnode->pBufPool->incycle), pvma); + return 0; +} + +int vnodeBufPoolRecycle(SVnode *pVnode) { + SVBufPool * pBufPool = pVnode->pBufPool; + SVMemAllocator *pvma = TD_DLIST_HEAD(&(pBufPool->incycle)); + ASSERT(pvma != NULL); + + tDListPop(&(pBufPool->incycle), pvma); + vmaReset(pvma); + tDListAppend(&(pBufPool->free), pvma); + + return 0; +} + +void *vnodeMalloc(SVnode *pVnode, uint64_t size) { + SVBufPool *pBufPool = pVnode->pBufPool; + + if (pBufPool->inuse == NULL) { + while (true) { + // TODO: add sem_wait and sem_post + pBufPool->inuse = TD_DLIST_HEAD(&(pBufPool->free)); + if (pBufPool->inuse) { + tDListPop(&(pBufPool->free), pBufPool->inuse); + break; + } else { + // tsem_wait(&(pBufPool->hasFree)); + } + } + } + + return vmaMalloc(pBufPool->inuse, size); +} + +bool vnodeBufPoolIsFull(SVnode *pVnode) { + if (pVnode->pBufPool->inuse == NULL) return false; + return vmaIsFull(pVnode->pBufPool->inuse); +} + +#if 0 + typedef enum { // Heap allocator E_V_HEAP_ALLOCATOR = 0, @@ -57,15 +157,6 @@ typedef struct { SListNode *pNode; } SVMAWrapper; -typedef struct { - T_REF_DECLARE() - uint64_t capacity; - EVMemAllocatorT type; - union { - SVHeapAllocator vha; - SVArenaAllocator vaa; - }; -} SVMemAllocator; static SListNode * vBufPoolNewNode(uint64_t capacity, EVMemAllocatorT type); static void vBufPoolFreeNode(SListNode *pNode); @@ -73,106 +164,13 @@ static SMemAllocator *vBufPoolCreateMA(SMemAllocatorFactory *pmaf); static void vBufPoolDestroyMA(SMemAllocatorFactory *pmaf, SMemAllocator *pma); static void * vBufPoolMalloc(SVMemAllocator *pvma, uint64_t size); -int vnodeOpenBufPool(SVnode *pVnode) { - uint64_t capacity; - EVMemAllocatorT type = E_V_ARENA_ALLOCATOR; - - if ((pVnode->pBufPool = (SVBufPool *)calloc(1, sizeof(SVBufPool))) == NULL) { - /* TODO */ - return -1; - } - - tdListInit(&(pVnode->pBufPool->free), 0); - tdListInit(&(pVnode->pBufPool->incycle), 0); - - capacity = pVnode->config.wsize / VNODE_BUF_POOL_SHARDS; - if (pVnode->config.isHeapAllocator) { - type = E_V_HEAP_ALLOCATOR; - } - - for (int i = 0; i < VNODE_BUF_POOL_SHARDS; i++) { - SListNode *pNode = vBufPoolNewNode(capacity, type); - if (pNode == NULL) { - vnodeCloseBufPool(pVnode); - return -1; - } - - tdListAppendNode(&(pVnode->pBufPool->free), pNode); - } - - pVnode->pBufPool->maf.impl = pVnode; - pVnode->pBufPool->maf.create = vBufPoolCreateMA; - pVnode->pBufPool->maf.destroy = vBufPoolDestroyMA; - - return 0; -} - -void vnodeCloseBufPool(SVnode *pVnode) { - SListNode *pNode; - if (pVnode->pBufPool) { - // Clear free list - while ((pNode = tdListPopHead(&(pVnode->pBufPool->free))) != NULL) { - vBufPoolFreeNode(pNode); - } - - // Clear incycle list - while ((pNode = tdListPopHead(&(pVnode->pBufPool->incycle))) != NULL) { - vBufPoolFreeNode(pNode); - } - - // Free inuse node - if (pVnode->pBufPool->inuse) { - vBufPoolFreeNode(pVnode->pBufPool->inuse); - } - - free(pVnode->pBufPool); - pVnode->pBufPool = NULL; - } -} - -void *vnodeMalloc(SVnode *pVnode, uint64_t size) { - void *ptr; - - if (pVnode->pBufPool->inuse == NULL) { - SListNode *pNode; - while ((pNode = tdListPopHead(&(pVnode->pBufPool->free))) == NULL) { - // todo - // tsem_wait(); - ASSERT(0); - } - - pVnode->pBufPool->inuse = pNode; - } - - SVMemAllocator *pvma = (SVMemAllocator *)(pVnode->pBufPool->inuse->data); - return vBufPoolMalloc(pvma, size); -} - /* ------------------------ STATIC METHODS ------------------------ */ -static void vArenaAllocatorInit(SVArenaAllocator *pvaa, uint64_t capacity, uint64_t ssize, uint64_t lsize) { /* TODO */ - pvaa->ssize = ssize; - pvaa->lsize = lsize; - pvaa->inuse = &pvaa->node; - - pvaa->node.prev = NULL; - pvaa->node.size = capacity; - pvaa->node.ptr = pvaa->node.data; -} - -static void vArenaAllocatorClear(SVArenaAllocator *pvaa) { /* TODO */ - while (pvaa->inuse != &(pvaa->node)) { - SVArenaNode *pANode = pvaa->inuse; - pvaa->inuse = pANode->prev; - free(pANode); - } -} - static SListNode *vBufPoolNewNode(uint64_t capacity, EVMemAllocatorT type) { SListNode * pNode; SVMemAllocator *pvma; uint64_t msize; - uint64_t ssize = 0; // TODO - uint64_t lsize = 0; // TODO + uint64_t ssize = 4096; // TODO + uint64_t lsize = 1024; // TODO msize = sizeof(SListNode) + sizeof(SVMemAllocator); if (type == E_V_ARENA_ALLOCATOR) { @@ -317,4 +315,5 @@ static void vBufPoolDestroyMA(SMemAllocatorFactory *pmaf, SMemAllocator *pma) { tdListAppendNode(&(pVnode->pBufPool->free), tdListPopNode(&(pVnode->pBufPool->incycle), pNode)); // tsem_post(); todo: sem_post } -} \ No newline at end of file +} +#endif \ No newline at end of file diff --git a/source/dnode/vnode/impl/src/vnodeCfg.c b/source/dnode/vnode/impl/src/vnodeCfg.c index 01facba888..97c3cc9cee 100644 --- a/source/dnode/vnode/impl/src/vnodeCfg.c +++ b/source/dnode/vnode/impl/src/vnodeCfg.c @@ -15,7 +15,8 @@ #include "vnodeDef.h" -const SVnodeCfg defaultVnodeOptions = {0}; /* TODO */ +const SVnodeCfg defaultVnodeOptions = { + .wsize = 96 * 1024 * 1024, .ssize = 1 * 1024 * 1024, .lsize = 1024, .walCfg = {.level = TAOS_WAL_WRITE}}; /* TODO */ void vnodeOptionsInit(SVnodeCfg *pVnodeOptions) { /* TODO */ vnodeOptionsCopy(pVnodeOptions, &defaultVnodeOptions); diff --git a/source/dnode/vnode/impl/src/vnodeCommit.c b/source/dnode/vnode/impl/src/vnodeCommit.c index 18a0c6d91d..a728de0ebb 100644 --- a/source/dnode/vnode/impl/src/vnodeCommit.c +++ b/source/dnode/vnode/impl/src/vnodeCommit.c @@ -18,31 +18,22 @@ static int vnodeStartCommit(SVnode *pVnode); static int vnodeEndCommit(SVnode *pVnode); -bool vnodeShouldCommit(SVnode *pVnode) { return false; } - int vnodeAsyncCommit(SVnode *pVnode) { -#if 0 - if (vnodeStartCommit(pVnode) < 0) { - // TODO - } + vnodeBufPoolSwitch(pVnode); + SVnodeTask *pTask = (SVnodeTask *)malloc(sizeof(*pTask)); - if (tqCommit(pVnode->pTQ) < 0) { - // TODO - } + pTask->execute = vnodeCommit; // TODO + pTask->arg = pVnode; // TODO - if (metaCommit(pVnode->pMeta) < 0) { - // TODO - } + vnodeScheduleTask(pTask); + return 0; +} - if (tsdbCommit(pVnode->pTsdb) < 0) { - // TODO - } +int vnodeCommit(void *arg) { + SVnode *pVnode = (SVnode *)arg; - if (vnodeEndCommit(pVnode) < 0) { - // TODO - } - -#endif + vnodeBufPoolRecycle(pVnode); + // TODO return 0; } diff --git a/source/dnode/vnode/impl/src/vnodeMain.c b/source/dnode/vnode/impl/src/vnodeMain.c index ab33b58858..59e3bae5d7 100644 --- a/source/dnode/vnode/impl/src/vnodeMain.c +++ b/source/dnode/vnode/impl/src/vnodeMain.c @@ -20,19 +20,6 @@ static void vnodeFree(SVnode *pVnode); static int vnodeOpenImpl(SVnode *pVnode); static void vnodeCloseImpl(SVnode *pVnode); -int vnodeInit() { - // TODO - if (walInit() < 0) { - return -1; - } - - return 0; -} - -void vnodeClear() { - walCleanUp(); -} - SVnode *vnodeOpen(const char *path, const SVnodeCfg *pVnodeCfg) { SVnode *pVnode = NULL; diff --git a/source/dnode/vnode/impl/src/vnodeMemAllocator.c b/source/dnode/vnode/impl/src/vnodeMemAllocator.c deleted file mode 100644 index 902014eb47..0000000000 --- a/source/dnode/vnode/impl/src/vnodeMemAllocator.c +++ /dev/null @@ -1,113 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "vnodeDef.h" - -SMemAllocator *vnodeCreateMemAllocator(SVnode *pVnode) { - SMemAllocator *pma = NULL; - /* TODO */ - return pma; -} - -void vnodeDestroyMemAllocator(SMemAllocator *pma) { - // TODO -} - -#if 0 -#define VNODE_HEAP_ALLOCATOR 0 -#define VNODE_ARENA_ALLOCATOR 1 - -typedef struct { - uint64_t tsize; - uint64_t used; -} SVHeapAllocator; - -typedef struct SVArenaNode { - struct SVArenaNode *prev; - void * nptr; - char data[]; -} SVArenaNode; - -typedef struct { - SVArenaNode *inuse; - SVArenaNode node; -} SVArenaAllocator; - -typedef struct { - int8_t type; - uint64_t tsize; - T_REF_DECLARE() - union { - SVHeapAllocator vha; - SVArenaAllocator vaa; - }; -} SVMemAllocator; - -SMemAllocator *vnodeCreateMemAllocator(int8_t type, uint64_t tsize, uint64_t ssize /* step size only for arena */) { - SMemAllocator * pma; - uint64_t msize; - SVMemAllocator *pva; - - msize = sizeof(*pma) + sizeof(SVMemAllocator); - if (type == VNODE_ARENA_ALLOCATOR) { - msize += tsize; - } - - pma = (SMemAllocator *)calloc(1, msize); - if (pma == NULL) { - return NULL; - } - - pma->impl = POINTER_SHIFT(pma, sizeof(*pma)); - pva = (SVMemAllocator *)(pma->impl); - pva->type = type; - pva->tsize = tsize; - - if (type == VNODE_HEAP_ALLOCATOR) { - pma->malloc = NULL; - pma->calloc = NULL; - pma->realloc = NULL; - pma->free = NULL; - pma->usage = NULL; - } else if (type == VNODE_ARENA_ALLOCATOR) { - pma->malloc = NULL; - pma->calloc = NULL; - pma->realloc = NULL; - pma->free = NULL; - pma->usage = NULL; - } else { - ASSERT(0); - } - - return pma; -} - -void vnodeDestroyMemAllocator(SMemAllocator *pma) { - // TODO -} - -void vnodeRefMemAllocator(SMemAllocator *pma) { - // TODO -} - -void vnodeUnrefMemAllocator(SMemAllocator *pma) { - // TODO -} - -/* ------------------------ Heap Allocator IMPL ------------------------ */ - -/* ------------------------ Arena Allocator IMPL ------------------------ */ - -#endif \ No newline at end of file diff --git a/source/dnode/vnode/impl/src/vnodeMgr.c b/source/dnode/vnode/impl/src/vnodeMgr.c new file mode 100644 index 0000000000..964cbe77da --- /dev/null +++ b/source/dnode/vnode/impl/src/vnodeMgr.c @@ -0,0 +1,115 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "vnodeDef.h" + +SVnodeMgr vnodeMgr = {.vnodeInitFlag = TD_MOD_UNINITIALIZED, .vnodeClearFlag = TD_MOD_UNCLEARD, .stop = false}; + +static void* loop(void* arg); + +int vnodeInit(uint16_t nthreads) { + if (TD_CHECK_AND_SET_MODE_INIT(&(vnodeMgr.vnodeInitFlag)) == TD_MOD_INITIALIZED) { + return 0; + } + + // Start commit handers + if (nthreads > 0) { + vnodeMgr.nthreads = nthreads; + vnodeMgr.threads = (pthread_t*)calloc(nthreads, sizeof(pthread_t)); + if (vnodeMgr.threads == NULL) { + return -1; + } + + pthread_mutex_init(&(vnodeMgr.mutex), NULL); + pthread_cond_init(&(vnodeMgr.hasTask), NULL); + tDListInit(&(vnodeMgr.queue)); + + for (uint16_t i = 0; i < nthreads; i++) { + pthread_create(&(vnodeMgr.threads[i]), NULL, loop, NULL); + } + } else { + // TODO: if no commit thread is set, then another mechanism should be + // given. Otherwise, it is a false. + ASSERT(0); + } + + if (walInit() < 0) { + return -1; + } + + return 0; +} + +void vnodeClear() { + if (TD_CHECK_AND_SET_MOD_CLEAR(&(vnodeMgr.vnodeClearFlag)) == TD_MOD_CLEARD) { + return; + } + + walCleanUp(); + + // Stop commit handler + pthread_mutex_lock(&(vnodeMgr.mutex)); + vnodeMgr.stop = true; + pthread_cond_broadcast(&(vnodeMgr.hasTask)); + pthread_mutex_unlock(&(vnodeMgr.mutex)); + + for (uint16_t i = 0; i < vnodeMgr.nthreads; i++) { + pthread_join(vnodeMgr.threads[i], NULL); + } + + tfree(vnodeMgr.threads); + pthread_cond_destroy(&(vnodeMgr.hasTask)); + pthread_mutex_destroy(&(vnodeMgr.mutex)); +} + +int vnodeScheduleTask(SVnodeTask* pTask) { + pthread_mutex_lock(&(vnodeMgr.mutex)); + + tDListAppend(&(vnodeMgr.queue), pTask); + + pthread_cond_signal(&(vnodeMgr.hasTask)); + + pthread_mutex_unlock(&(vnodeMgr.mutex)); + + return 0; +} + +/* ------------------------ STATIC METHODS ------------------------ */ +static void* loop(void* arg) { + SVnodeTask* pTask; + for (;;) { + pthread_mutex_lock(&(vnodeMgr.mutex)); + for (;;) { + pTask = TD_DLIST_HEAD(&(vnodeMgr.queue)); + if (pTask == NULL) { + if (vnodeMgr.stop) { + pthread_mutex_unlock(&(vnodeMgr.mutex)); + return NULL; + } else { + pthread_cond_wait(&(vnodeMgr.hasTask), &(vnodeMgr.mutex)); + } + } else { + tDListPop(&(vnodeMgr.queue), pTask); + break; + } + } + + pthread_mutex_unlock(&(vnodeMgr.mutex)); + + (*(pTask->execute))(pTask->arg); + } + + return NULL; +} \ No newline at end of file diff --git a/source/dnode/vnode/impl/test/CMakeLists.txt b/source/dnode/vnode/impl/test/CMakeLists.txt index 83506a4fde..e1226331e9 100644 --- a/source/dnode/vnode/impl/test/CMakeLists.txt +++ b/source/dnode/vnode/impl/test/CMakeLists.txt @@ -4,4 +4,9 @@ target_sources(vnodeApiTests PRIVATE "vnodeApiTests.cpp" ) -target_link_libraries(vnodeApiTests vnode gtest gtest_main) \ No newline at end of file +target_link_libraries(vnodeApiTests vnode gtest gtest_main) + +add_test( + NAME vnode_api_tests + COMMAND ${CMAKE_CURRENT_BINARY_DIR}/vnodeApiTests + ) \ No newline at end of file diff --git a/source/dnode/vnode/impl/test/vnodeApiTests.cpp b/source/dnode/vnode/impl/test/vnodeApiTests.cpp index df784181b7..a25b04e161 100644 --- a/source/dnode/vnode/impl/test/vnodeApiTests.cpp +++ b/source/dnode/vnode/impl/test/vnodeApiTests.cpp @@ -1,3 +1,14 @@ +/** + * @file vnodeApiTests.cpp + * @author hzcheng (hzcheng@taosdata.com) + * @brief VNODE module API tests + * @version 0.1 + * @date 2021-12-13 + * + * @copyright Copyright (c) 2021 + * + */ + #include #include @@ -81,7 +92,7 @@ TEST(vnodeApiTest, test_create_table_encode_and_decode_function) { #endif TEST(vnodeApiTest, vnodeOpen_vnodeClose_test) { - GTEST_ASSERT_GE(vnodeInit(), 0); + GTEST_ASSERT_GE(vnodeInit(2), 0); // Create and open a vnode SVnode *pVnode = vnodeOpen("vnode1", NULL); diff --git a/source/libs/CMakeLists.txt b/source/libs/CMakeLists.txt index 007bb1e967..57a5023807 100644 --- a/source/libs/CMakeLists.txt +++ b/source/libs/CMakeLists.txt @@ -9,4 +9,5 @@ add_subdirectory(cache) add_subdirectory(catalog) add_subdirectory(executor) add_subdirectory(planner) -add_subdirectory(function) \ No newline at end of file +add_subdirectory(function) +add_subdirectory(query) diff --git a/source/libs/catalog/CMakeLists.txt b/source/libs/catalog/CMakeLists.txt index ff3e62700a..25c80d502a 100644 --- a/source/libs/catalog/CMakeLists.txt +++ b/source/libs/catalog/CMakeLists.txt @@ -8,5 +8,5 @@ target_include_directories( target_link_libraries( catalog - PRIVATE os util common transport -) \ No newline at end of file + PRIVATE os util common transport query +) diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 5b50bbff4c..720f197782 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -21,14 +21,58 @@ extern "C" { #endif #include "catalog.h" +#include "common.h" +#include "tlog.h" + +#define CTG_DEFAULT_CLUSTER_NUMBER 6 +#define CTG_DEFAULT_VGROUP_NUMBER 100 + +#define CTG_DEFAULT_INVALID_VERSION (-1) + +typedef struct SVgroupListCache { + int32_t vgroupVersion; + SHashObj *cache; // key:vgId, value:SVgroupInfo* + SArray *arrayCache; // SVgroupInfo +} SVgroupListCache; + +typedef struct SDBVgroupCache { + SHashObj *cache; //key:dbname, value:SDBVgroupInfo +} SDBVgroupCache; + +typedef struct STableMetaCache { + SHashObj *cache; //key:fulltablename, value:STableMeta +} STableMetaCache; typedef struct SCatalog { - void *pMsgSender; // used to send messsage to mnode to fetch necessary metadata - SHashObj *pData; // items cached for each cluster, the hash key is the cluster-id, returned by mgmt node + SVgroupListCache vgroupCache; + SDBVgroupCache dbCache; + STableMetaCache tableCache; } SCatalog; +typedef struct SCatalogMgmt { + void *pMsgSender; // used to send messsage to mnode to fetch necessary metadata + SHashObj *pCluster; // items cached for each cluster, the hash key is the cluster-id got from mgmt node +} SCatalogMgmt; + + +extern int32_t ctgDebugFlag; + +#define ctgFatal(...) do { if (ctgDebugFlag & DEBUG_FATAL) { taosPrintLog("CTG FATAL ", ctgDebugFlag, __VA_ARGS__); }} while(0) +#define ctgError(...) do { if (ctgDebugFlag & DEBUG_ERROR) { taosPrintLog("CTG ERROR ", ctgDebugFlag, __VA_ARGS__); }} while(0) +#define ctgWarn(...) do { if (ctgDebugFlag & DEBUG_WARN) { taosPrintLog("CTG WARN ", ctgDebugFlag, __VA_ARGS__); }} while(0) +#define ctgInfo(...) do { if (ctgDebugFlag & DEBUG_INFO) { taosPrintLog("CTG ", ctgDebugFlag, __VA_ARGS__); }} while(0) +#define ctgDebug(...) do { if (ctgDebugFlag & DEBUG_DEBUG) { taosPrintLog("CTG ", ctgDebugFlag, __VA_ARGS__); }} while(0) +#define ctgTrace(...) do { if (ctgDebugFlag & DEBUG_TRACE) { taosPrintLog("CTG ", ctgDebugFlag, __VA_ARGS__); }} while(0) +#define ctgDebugL(...) do { if (ctgDebugFlag & DEBUG_DEBUG) { taosPrintLongString("CTG ", ctgDebugFlag, __VA_ARGS__); }} while(0) + + +#define CTG_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { return _code; } } while (0) +#define CTG_ERR_LRET(c,...) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { ctgError(__VA_ARGS__); return _code; } } while (0) +#define CTG_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { goto _return; } } while (0) + + #ifdef __cplusplus } #endif -#endif /*_TD_CATALOG_INT_H_*/ \ No newline at end of file +#endif /*_TD_CATALOG_INT_H_*/ diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 08e2172adb..b488ab8101 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -14,11 +14,339 @@ */ #include "catalogInt.h" +#include "trpc.h" +#include "query.h" -struct SCatalog* getCatalogHandle(const SEpSet* pMgmtEps) { - return (struct SCatalog*) 0x1; +SCatalogMgmt ctgMgmt = {0}; + +int32_t ctgGetVgroupFromMnode(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, SVgroupListInfo** pVgroup) { + char *msg = NULL; + SEpSet *pVnodeEpSet = NULL; + int32_t msgLen = 0; + + int32_t code = queryBuildMsg[TSDB_MSG_TYPE_VGROUP_LIST](NULL, &msg, 0, &msgLen); + if (code) { + return code; + } + + SRpcMsg rpcMsg = { + .msgType = TSDB_MSG_TYPE_VGROUP_LIST, + .pCont = msg, + .contLen = msgLen, + }; + + SRpcMsg rpcRsp = {0}; + + rpcSendRecv(pRpc, (SEpSet*)pMgmtEps, &rpcMsg, &rpcRsp); + + code = queryProcessMsgRsp[TSDB_MSG_TYPE_VGROUP_LIST](pVgroup, rpcRsp.pCont, rpcRsp.contLen); + if (code) { + return code; + } + + return TSDB_CODE_SUCCESS; } -int32_t catalogGetMetaData(struct SCatalog* pCatalog, const SMetaReq* pMetaReq, SMetaData* pMetaData) { +int32_t ctgGetVgroupFromCache(SCatalog* pCatalog, SArray** pVgroupList, int32_t* exist) { + if (NULL == pCatalog->vgroupCache.arrayCache || pCatalog->vgroupCache.vgroupVersion < 0) { + *exist = 0; + return TSDB_CODE_SUCCESS; + } + + if (pVgroupList) { + *pVgroupList = taosArrayDup(pCatalog->vgroupCache.arrayCache); + } + + *exist = 1; + + return TSDB_CODE_SUCCESS; +} + + +int32_t ctgGetDBVgroupFromCache(SCatalog* pCatalog, char *dbName, SDBVgroupInfo **dbInfo, int32_t *exist) { +/* + if (NULL == pCatalog->dbCache.cache) { + *exist = 0; + return TSDB_CODE_SUCCESS; + } + + taosHashGet(SHashObj * pHashObj, const void * key, size_t keyLen) + + if (dbInfo) { + *pVgroupList = taosArrayDup(pCatalog->vgroupCache.arrayCache); + } + + *exist = 1; +*/ + return TSDB_CODE_SUCCESS; +} + + +int32_t catalogInit(SCatalogCfg *cfg) { + ctgMgmt.pCluster = taosHashInit(CTG_DEFAULT_CLUSTER_NUMBER, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + if (NULL == ctgMgmt.pCluster) { + CTG_ERR_LRET(TSDB_CODE_CTG_INTERNAL_ERROR, "init %d cluster cache failed", CTG_DEFAULT_CLUSTER_NUMBER); + } + + return TSDB_CODE_SUCCESS; +} + + +int32_t catalogGetHandle(const char *clusterId, struct SCatalog** catalogHandle) { + if (NULL == clusterId || NULL == catalogHandle) { + return TSDB_CODE_CTG_INVALID_INPUT; + } + + if (NULL == ctgMgmt.pCluster) { + ctgError("cluster cache are not ready"); + return TSDB_CODE_CTG_NOT_READY; + } + + size_t clen = strlen(clusterId); + SCatalog *clusterCtg = (SCatalog *)taosHashGet(ctgMgmt.pCluster, clusterId, clen); + + if (clusterCtg) { + *catalogHandle = clusterCtg; + return TSDB_CODE_SUCCESS; + } + + clusterCtg = calloc(1, sizeof(*clusterCtg)); + if (NULL == clusterCtg) { + ctgError("calloc %d failed", (int32_t)sizeof(*clusterCtg)); + return TSDB_CODE_CTG_MEM_ERROR; + } + + clusterCtg->vgroupCache.vgroupVersion = CTG_DEFAULT_INVALID_VERSION; + + if (taosHashPut(ctgMgmt.pCluster, clusterId, clen, &clusterCtg, POINTER_BYTES)) { + ctgError("put cluster %s cache to hash failed", clusterId); + tfree(clusterCtg); + return TSDB_CODE_CTG_INTERNAL_ERROR; + } + + *catalogHandle = clusterCtg; + + return TSDB_CODE_SUCCESS; +} + + +int32_t catalogGetVgroupVersion(struct SCatalog* pCatalog, int32_t* version) { + if (NULL == pCatalog || NULL == version) { + return TSDB_CODE_CTG_INVALID_INPUT; + } + + *version = pCatalog->vgroupCache.vgroupVersion; + + return TSDB_CODE_SUCCESS; +} + + + +int32_t catalogUpdateVgroup(struct SCatalog* pCatalog, SVgroupListInfo* pVgroup) { + if (NULL == pVgroup) { + ctgError("vgroup get from mnode succeed, but no output"); + return TSDB_CODE_CTG_INTERNAL_ERROR; + } + + if (pVgroup->vgroupVersion < 0) { + ctgError("vgroup version[%d] is invalid", pVgroup->vgroupVersion); + return TSDB_CODE_CTG_INVALID_INPUT; + } + + + if (NULL == pCatalog->vgroupCache.arrayCache) { + pCatalog->vgroupCache.arrayCache = taosArrayInit(pVgroup->vgroupNum, sizeof(pVgroup->vgroupInfo[0])); + if (NULL == pCatalog->vgroupCache.arrayCache) { + ctgError("init array[%d] for cluster cache failed", pVgroup->vgroupNum); + return TSDB_CODE_CTG_MEM_ERROR; + } + } else { + taosArrayClear(pCatalog->vgroupCache.arrayCache); + } + + if (NULL == pCatalog->vgroupCache.cache) { + pCatalog->vgroupCache.cache = taosHashInit(CTG_DEFAULT_VGROUP_NUMBER, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); + if (NULL == pCatalog->vgroupCache.cache) { + ctgError("init hash[%d] for cluster cache failed", CTG_DEFAULT_VGROUP_NUMBER); + return TSDB_CODE_CTG_MEM_ERROR; + } + } else { + taosHashClear(pCatalog->vgroupCache.cache); + } + + SVgroupInfo *vInfo = NULL; + for (int32_t i = 0; i < pVgroup->vgroupNum; ++i) { + vInfo = taosArrayPush(pCatalog->vgroupCache.arrayCache, &pVgroup->vgroupInfo[i]); + if (NULL == vInfo) { + ctgError("push to vgroup array cache failed"); + goto error_exit; + } + + if (taosHashPut(pCatalog->vgroupCache.cache, &pVgroup->vgroupInfo[i].vgId, sizeof(pVgroup->vgroupInfo[i].vgId), &vInfo, POINTER_BYTES) != 0) { + ctgError("push to vgroup hash cache failed"); + goto error_exit; + } + } + + pCatalog->vgroupCache.vgroupVersion = pVgroup->vgroupVersion; + + return TSDB_CODE_SUCCESS; + +error_exit: + if (pCatalog->vgroupCache.arrayCache) { + taosArrayDestroy(pCatalog->vgroupCache.arrayCache); + pCatalog->vgroupCache.arrayCache = NULL; + } + + if (pCatalog->vgroupCache.cache) { + taosHashCleanup(pCatalog->vgroupCache.cache); + pCatalog->vgroupCache.cache = NULL; + } + + pCatalog->vgroupCache.vgroupVersion = CTG_DEFAULT_INVALID_VERSION; + + return TSDB_CODE_CTG_INTERNAL_ERROR; +} + + +int32_t catalogGetVgroup(SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, SArray** pVgroupList) { + if (NULL == pCatalog || NULL == pMgmtEps || NULL == pRpc) { + return TSDB_CODE_CTG_INVALID_INPUT; + } + + int32_t exist = 0; + + CTG_ERR_RET(ctgGetVgroupFromCache(pCatalog, pVgroupList, &exist)); + + if (exist) { + return TSDB_CODE_SUCCESS; + } + + SVgroupListInfo *pVgroup = NULL; + + CTG_ERR_RET(ctgGetVgroupFromMnode(pCatalog, pRpc, pMgmtEps, &pVgroup)); + + CTG_ERR_RET(catalogUpdateVgroup(pCatalog, pVgroup)); + + if (pVgroupList) { + CTG_ERR_RET(ctgGetVgroupFromCache(pCatalog, pVgroupList, &exist)); + } + + if (0 == exist) { + ctgError("catalog fetched but get from cache failed"); + return TSDB_CODE_CTG_INTERNAL_ERROR; + } + + return TSDB_CODE_SUCCESS; +} + +int32_t catalogGetDBVgroupVersion(struct SCatalog* pCatalog, const char* dbName, int32_t* version) { + if (NULL == pCatalog || NULL == dbName || NULL == version) { + return TSDB_CODE_CTG_INVALID_INPUT; + } + + if (NULL == pCatalog->dbCache.cache) { + *version = CTG_DEFAULT_INVALID_VERSION; + return TSDB_CODE_SUCCESS; + } + + SDBVgroupInfo * dbInfo = taosHashGet(pCatalog->dbCache.cache, dbName, strlen(dbName)); + if (NULL == dbInfo) { + *version = CTG_DEFAULT_INVALID_VERSION; + return TSDB_CODE_SUCCESS; + } + + *version = dbInfo->vgroupVersion; + + return TSDB_CODE_SUCCESS; +} + +int32_t catalogUpdateDBVgroup(struct SCatalog* pCatalog, const char* dbName, SDBVgroupInfo* dbInfo) { + +} + + + + +int32_t catalogGetDBVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* dbName, int32_t forceUpdate, SDBVgroupInfo** dbInfo) { + if (NULL == pCatalog || NULL == dbName || NULL == pRpc || NULL == pMgmtEps) { + return TSDB_CODE_CTG_INVALID_INPUT; + } + +/* + int32_t exist = 0; + + if (0 == forceUpdate) { + CTG_ERR_RET(ctgGetDBVgroupFromCache(pCatalog, dbName, dbInfo, &exist)); + + if (exist) { + return TSDB_CODE_SUCCESS; + } + } + + SDBVgroupInfo* newDbInfo = NULL; + + CTG_ERR_RET(ctgGetDBVgroupFromMnode(pCatalog, pRpc, pMgmtEps, dbName, &newDbInfo)); + + CTG_ERR_RET(catalogUpdateDBVgroup(pCatalog, dbName, newDbInfo)); + + if (dbInfo) { + *dbInfo = newDbInfo; + } +*/ + + return TSDB_CODE_SUCCESS; +} + + + +int32_t catalogGetTableMetaFromMnode(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pTableName, const STagData* tagData, STableMeta* pTableMeta) { + if (NULL == pCatalog || NULL == pMgmtEps || NULL == pTableName || NULL == pTableMeta) { + return TSDB_CODE_CTG_INVALID_INPUT; + } + + SBuildTableMetaInput bInput = {0}; + char *msg = NULL; + SEpSet *pVnodeEpSet = NULL; + int32_t msgLen = 0; + + int32_t code = queryBuildMsg[TSDB_MSG_TYPE_TABLE_META](&bInput, &msg, 0, &msgLen); + if (code) { + return code; + } + + SRpcMsg rpcMsg = { + .msgType = TSDB_MSG_TYPE_TABLE_META, + .pCont = msg, + .contLen = msgLen, + }; + + SRpcMsg rpcRsp = {0}; + + rpcSendRecv(pRpc, (SEpSet*)pMgmtEps, &rpcMsg, &rpcRsp); + + return TSDB_CODE_SUCCESS; +} + +int32_t catalogGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pTableName, STableMeta* pTableMeta) { + +} + + +int32_t catalogGetAllMeta(struct SCatalog* pCatalog, const SEpSet* pMgmtEps, const SCatalogReq* pReq, SMetaData* pRsp) { + if (NULL == pCatalog || NULL == pMgmtEps || NULL == pReq || NULL == pRsp) { + return TSDB_CODE_CTG_INVALID_INPUT; + } + return 0; } + +void catalogDestroy(void) { + if (ctgMgmt.pCluster) { + taosHashCleanup(ctgMgmt.pCluster); //TBD + ctgMgmt.pCluster = NULL; + } +} + + + diff --git a/source/libs/index/inc/index_fst_automation.h b/source/libs/index/inc/index_fst_automation.h index c2ab61bf5f..7eb2ba97ae 100644 --- a/source/libs/index/inc/index_fst_automation.h +++ b/source/libs/index/inc/index_fst_automation.h @@ -20,6 +20,8 @@ extern "C" { #endif #include "index_fst_util.h" + + typedef struct AutomationCtx AutomationCtx; typedef enum AutomationType { @@ -42,14 +44,23 @@ typedef struct AutomationCtx { } AutomationCtx; - +typedef enum ValueType { FST_INT, FST_CHAR, FST_ARRAY} ValueType; typedef enum StartWithStateKind { Done, Running } StartWithStateKind; typedef struct StartWithStateValue { StartWithStateKind kind; - void *value; + ValueType type; + union { + int val; + char *ptr; + SArray *arr; + // add more type + } ; } StartWithStateValue; +StartWithStateValue *startWithStateValueDump(StartWithStateValue *sv); + + typedef struct AutomationFunc { void* (*start)(AutomationCtx *ctx) ; bool (*isMatch)(AutomationCtx *ctx, void *); diff --git a/source/libs/index/src/index_fst.c b/source/libs/index/src/index_fst.c index 07f1e343bd..e2708c5919 100644 --- a/source/libs/index/src/index_fst.c +++ b/source/libs/index/src/index_fst.c @@ -1322,6 +1322,7 @@ StreamWithStateResult *streamWithStateNextWith(StreamWithState *sws, StreamCallb return swsResultCreate(&s, output, callback(start)); } } + SArray *nodes = taosArrayInit(8, sizeof(FstNode *)); while (taosArrayGetSize(sws->stack) > 0) { StreamState *p = (StreamState *)taosArrayPop(sws->stack); if (p->trans >= FST_NODE_LEN(p->node) || automFuncs[aut->type].canMatch(aut, p->autState)) { @@ -1337,8 +1338,8 @@ StreamWithStateResult *streamWithStateNextWith(StreamWithState *sws, StreamCallb void* nextState = automFuncs[aut->type].accept(aut, p->autState, trn.inp); void* tState = callback(nextState); bool isMatch = automFuncs[aut->type].isMatch(aut, nextState); - //bool isMatch = sws->aut->isMatch(nextState); FstNode *nextNode = fstGetNode(sws->fst, trn.addr); + taosArrayPush(nodes, &nextNode); taosArrayPush(sws->inp, &(trn.inp)); if (FST_NODE_IS_FINAL(nextNode)) { @@ -1354,26 +1355,35 @@ StreamWithStateResult *streamWithStateNextWith(StreamWithState *sws, StreamCallb StreamState s2 = {.node = nextNode, .trans = 0, .out = {.null = false, .out = out}, .autState = nextState}; taosArrayPush(sws->stack, &s2); - uint8_t *buf = (uint8_t *)malloc(taosArrayGetSize(sws->inp) * sizeof(uint8_t)); - for (uint32_t i = 0; i < taosArrayGetSize(sws->inp); i++) { - uint8_t *t = (uint8_t *)taosArrayGet(sws->inp, i); - buf[i] = *t; + + size_t isz = taosArrayGetSize(sws->inp); + uint8_t *buf = (uint8_t *)malloc(isz * sizeof(uint8_t)); + for (uint32_t i = 0; i < isz; i++) { + buf[i] = *(uint8_t *)taosArrayGet(sws->inp, i); } FstSlice slice = fstSliceCreate(buf, taosArrayGetSize(sws->inp)); if (fstBoundWithDataExceededBy(sws->endAt, &slice)) { taosArrayDestroyEx(sws->stack, streamStateDestroy); sws->stack = (SArray *)taosArrayInit(256, sizeof(StreamState)); + free(buf); fstSliceDestroy(&slice); return NULL; } if (FST_NODE_IS_FINAL(nextNode) && isMatch) { FstOutput fOutput = {.null = false, .out = out + FST_NODE_FINAL_OUTPUT(nextNode)}; - StreamWithStateResult *result = swsResultCreate(&slice, fOutput , tState); + StreamWithStateResult *result = swsResultCreate(&slice, fOutput, tState); + free(buf); fstSliceDestroy(&slice); return result; } + free(buf); fstSliceDestroy(&slice); } + for (size_t i = 0; i < taosArrayGetSize(nodes); i++) { + FstNode** node = (FstNode **)taosArrayGet(nodes, i); + fstNodeDestroy(*node); + } + taosArrayDestroy(nodes); return NULL; } diff --git a/source/libs/index/src/index_fst_automation.c b/source/libs/index/src/index_fst_automation.c index 6a08b41b12..bb0f0da76d 100644 --- a/source/libs/index/src/index_fst_automation.c +++ b/source/libs/index/src/index_fst_automation.c @@ -16,9 +16,59 @@ #include "index_fst_automation.h" +StartWithStateValue *startWithStateValueCreate(StartWithStateKind kind, ValueType ty, void *val) { + StartWithStateValue *nsv = calloc(1, sizeof(StartWithStateValue)); + if (nsv == NULL) { return NULL; } + + nsv->kind = kind; + nsv->type = ty; + if (ty == FST_INT) { + nsv->val = *(int *)val; + } else if (ty == FST_CHAR) { + size_t len = strlen((char *)val); + nsv->ptr = (char *)calloc(1, len + 1); + memcpy(nsv->ptr, val, len); + } else if (ty == FST_ARRAY) { + //TODO, + //nsv->arr = taosArrayFromList() + } + return nsv; +} +void startWithStateValueDestroy(StartWithStateValue *sv) { + if (sv == NULL) { return; } + + if (sv->type == FST_INT) { + // + } else if (sv->type == FST_CHAR) { + free(sv->ptr); + } else if (sv->type == FST_ARRAY) { + taosArrayDestroy(sv->arr); + } + free(sv); +} +StartWithStateValue *startWithStateValueDump(StartWithStateValue *sv) { + StartWithStateValue *nsv = calloc(1, sizeof(StartWithStateValue)); + if (nsv == NULL) { return NULL; } + + nsv->kind = sv->kind; + nsv->type= sv->type; + if (nsv->type == FST_INT) { + nsv->val = sv->val; + } else if (nsv->type == FST_CHAR) { + size_t len = strlen(sv->ptr); + nsv->ptr = (char *)calloc(1, len + 1); + memcpy(nsv->ptr, sv->ptr, len); + } else if (nsv->type == FST_ARRAY) { + } + return nsv; +} + + // prefix query, impl later + static void* prefixStart(AutomationCtx *ctx) { StartWithStateValue *data = (StartWithStateValue *)(ctx->data); + return data; }; static bool prefixIsMatch(AutomationCtx *ctx, void *data) { @@ -86,7 +136,7 @@ AutomationCtx* automCtxCreate(void *data, AutomationType type) { if (type == AUTOMATION_PREFIX) { StartWithStateValue *swsv = (StartWithStateValue *)calloc(1, sizeof(StartWithStateValue)); swsv->kind = Done; - swsv->value = NULL; + //swsv->value = NULL; ctx->data = (void *)swsv; } else if (type == AUTMMATION_MATCH) { diff --git a/source/libs/parser/inc/dataBlockMgt.h b/source/libs/parser/inc/dataBlockMgt.h index 350610ec06..7d2e6e3aec 100644 --- a/source/libs/parser/inc/dataBlockMgt.h +++ b/source/libs/parser/inc/dataBlockMgt.h @@ -75,16 +75,7 @@ typedef struct { SMemRowInfo *rowInfo; } SMemRowBuilder; -typedef struct SParamInfo { - int32_t idx; - uint8_t type; - uint8_t timePrec; - int16_t bytes; - uint32_t offset; -} SParamInfo; - typedef struct STableDataBlocks { - SName tableName; int8_t tsSource; // where does the UNIX timestamp come from, server or client bool ordered; // if current rows are ordered or not int64_t vgId; // virtual group id @@ -100,11 +91,6 @@ typedef struct STableDataBlocks { STagData tagData; SParsedDataColInfo boundColumnInfo; - - // for parameter ('?') binding - uint32_t numOfAllocedParams; - uint32_t numOfParams; - SParamInfo * params; SMemRowBuilder rowBuilder; } STableDataBlocks; @@ -187,7 +173,7 @@ void destroyBoundColumnInfo(SParsedDataColInfo* pColList); int32_t initMemRowBuilder(SMemRowBuilder *pBuilder, uint32_t nRows, uint32_t nCols, uint32_t nBoundCols, int32_t allNullLen); int32_t allocateMemIfNeed(STableDataBlocks *pDataBlock, int32_t rowSize, int32_t * numOfRows); int32_t getDataBlockFromList(SHashObj* pHashList, int64_t id, int32_t size, int32_t startOffset, int32_t rowSize, - SName* name, const STableMeta* pTableMeta, STableDataBlocks** dataBlocks, SArray* pBlockList); + const STableMeta* pTableMeta, STableDataBlocks** dataBlocks, SArray* pBlockList); int32_t mergeTableDataBlocks(SHashObj* pHashObj, int8_t schemaAttached, uint8_t payloadType, bool freeBlockMap); #endif // TDENGINE_DATABLOCKMGT_H diff --git a/source/libs/parser/inc/parserInt.h b/source/libs/parser/inc/parserInt.h index 47f655d446..e3925d3446 100644 --- a/source/libs/parser/inc/parserInt.h +++ b/source/libs/parser/inc/parserInt.h @@ -79,13 +79,13 @@ int32_t checkForInvalidExpr(SQueryStmtInfo* pQueryInfo, SMsgBuf* pMsgBuf); * @param msgBufLen * @return */ -int32_t qParserExtractRequestedMetaInfo(const SSqlInfo* pSqlInfo, SMetaReq* pMetaInfo, char* msg, int32_t msgBufLen); +int32_t qParserExtractRequestedMetaInfo(const SSqlInfo* pSqlInfo, SCatalogReq* pMetaInfo, char* msg, int32_t msgBufLen); /** * Destroy the meta data request structure. * @param pMetaInfo */ -void qParserClearupMetaRequestInfo(SMetaReq* pMetaInfo); +void qParserClearupMetaRequestInfo(SCatalogReq* pMetaInfo); #ifdef __cplusplus } diff --git a/source/libs/parser/src/astValidate.c b/source/libs/parser/src/astValidate.c index 7b6c423fb6..d2813886b3 100644 --- a/source/libs/parser/src/astValidate.c +++ b/source/libs/parser/src/astValidate.c @@ -4077,18 +4077,18 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer } #endif - SMetaReq req = {0}; + SCatalogReq req = {0}; SMetaData data = {0}; // TODO: check if the qnode info has been cached already - req.qNodeEpset = true; + req.qNodeRequired = true; code = qParserExtractRequestedMetaInfo(pInfo, &req, msgBuf, msgBufLen); if (code != TSDB_CODE_SUCCESS) { return code; } // load the meta data from catalog - code = catalogGetMetaData(pCatalog, &req, &data); + code = catalogGetAllMeta(pCatalog, NULL, &req, &data); if (code != TSDB_CODE_SUCCESS) { return code; } diff --git a/source/libs/parser/src/dataBlockMgt.c b/source/libs/parser/src/dataBlockMgt.c index 4ece848888..3138f0c1d8 100644 --- a/source/libs/parser/src/dataBlockMgt.c +++ b/source/libs/parser/src/dataBlockMgt.c @@ -108,7 +108,7 @@ void destroyBoundColumnInfo(SParsedDataColInfo* pColList) { tfree(pColList->colIdxInfo); } -static int32_t createDataBlock(size_t defaultSize, int32_t rowSize, int32_t startOffset, SName* name, +static int32_t createDataBlock(size_t defaultSize, int32_t rowSize, int32_t startOffset, const STableMeta* pTableMeta, STableDataBlocks** dataBlocks) { STableDataBlocks* dataBuf = (STableDataBlocks*)calloc(1, sizeof(STableDataBlocks)); if (dataBuf == NULL) { @@ -145,7 +145,7 @@ static int32_t createDataBlock(size_t defaultSize, int32_t rowSize, int32_t star dataBuf->tsSource = -1; dataBuf->vgId = dataBuf->pTableMeta->vgId; - tNameAssign(&dataBuf->tableName, name); + // tNameAssign(&dataBuf->tableName, name); assert(defaultSize > 0 && pTableMeta != NULL && dataBuf->pTableMeta != NULL); @@ -154,8 +154,7 @@ static int32_t createDataBlock(size_t defaultSize, int32_t rowSize, int32_t star } int32_t getDataBlockFromList(SHashObj* pHashList, int64_t id, int32_t size, int32_t startOffset, int32_t rowSize, - SName* name, const STableMeta* pTableMeta, STableDataBlocks** dataBlocks, - SArray* pBlockList) { + const STableMeta* pTableMeta, STableDataBlocks** dataBlocks, SArray* pBlockList) { *dataBlocks = NULL; STableDataBlocks** t1 = (STableDataBlocks**)taosHashGet(pHashList, (const char*)&id, sizeof(id)); if (t1 != NULL) { @@ -163,7 +162,7 @@ int32_t getDataBlockFromList(SHashObj* pHashList, int64_t id, int32_t size, int3 } if (*dataBlocks == NULL) { - int32_t ret = createDataBlock((size_t)size, rowSize, startOffset, name, pTableMeta, dataBlocks); + int32_t ret = createDataBlock((size_t)size, rowSize, startOffset, pTableMeta, dataBlocks); if (ret != TSDB_CODE_SUCCESS) { return ret; } @@ -253,23 +252,13 @@ static FORCE_INLINE void convertSMemRow(SMemRow dest, SMemRow src, STableDataBlo } } -void destroyDataBlock(STableDataBlocks* pDataBlock, bool removeMeta) { +void destroyDataBlock(STableDataBlocks* pDataBlock) { if (pDataBlock == NULL) { return; } tfree(pDataBlock->pData); - - if (removeMeta) { - char name[TSDB_TABLE_FNAME_LEN] = {0}; - tNameExtractFullName(&pDataBlock->tableName, name); - - // taosHashRemove(tscTableMetaMap, name, strnlen(name, TSDB_TABLE_FNAME_LEN)); - } - if (!pDataBlock->cloned) { - tfree(pDataBlock->params); - // free the refcount for metermeta if (pDataBlock->pTableMeta != NULL) { tfree(pDataBlock->pTableMeta); @@ -277,7 +266,6 @@ void destroyDataBlock(STableDataBlocks* pDataBlock, bool removeMeta) { destroyBoundColumnInfo(&pDataBlock->boundColumnInfo); } - tfree(pDataBlock); } @@ -289,7 +277,7 @@ void* destroyBlockArrayList(SArray* pDataBlockList) { size_t size = taosArrayGetSize(pDataBlockList); for (int32_t i = 0; i < size; i++) { void* d = taosArrayGetP(pDataBlockList, i); - destroyDataBlock(d, false); + destroyDataBlock(d); } taosArrayDestroy(pDataBlockList); @@ -505,7 +493,7 @@ int32_t mergeTableDataBlocks(SHashObj* pHashObj, int8_t schemaAttached, uint8_t if (pBlocks->numOfRows > 0) { STableDataBlocks* dataBuf = NULL; int32_t ret = getDataBlockFromList(pVnodeDataBlockHashList, pOneTableBlock->vgId, TSDB_PAYLOAD_SIZE, - INSERT_HEAD_SIZE, 0, &pOneTableBlock->tableName, pOneTableBlock->pTableMeta, &dataBuf, pVnodeDataBlockList); + INSERT_HEAD_SIZE, 0, pOneTableBlock->pTableMeta, &dataBuf, pVnodeDataBlockList); if (ret != TSDB_CODE_SUCCESS) { taosHashCleanup(pVnodeDataBlockHashList); destroyBlockArrayList(pVnodeDataBlockList); @@ -580,7 +568,6 @@ int32_t mergeTableDataBlocks(SHashObj* pHashObj, int8_t schemaAttached, uint8_t extractTableNameList(pHashObj, freeBlockMap); // free the table data blocks; - // pInsertParam->pDataBlocks = pVnodeDataBlockList; taosHashCleanup(pVnodeDataBlockHashList); tfree(blkKeyInfo.pKeyTuple); diff --git a/source/libs/parser/src/insertParser.c b/source/libs/parser/src/insertParser.c index 33b02f4261..fa59bc6ca7 100644 --- a/source/libs/parser/src/insertParser.c +++ b/source/libs/parser/src/insertParser.c @@ -167,25 +167,26 @@ static int32_t skipInsertInto(SInsertParseContext* pCxt) { static int32_t buildTableName(SInsertParseContext* pCxt, SToken* pStname, SArray* tableNameList) { if (parserValidateIdToken(pStname) != TSDB_CODE_SUCCESS) { - return buildInvalidOperationMsg(&pCxt->msg, "invalid table name"); + return buildSyntaxErrMsg(&pCxt->msg, "invalid table name", pStname->z); } SName name = {0}; - strndequote(name.tname, pStname->z, pStname->n); + strcpy(name.dbname, pCxt->pComCxt->pDbname); + strncpy(name.tname, pStname->z, pStname->n); taosArrayPush(tableNameList, &name); return TSDB_CODE_SUCCESS; } -static int32_t buildMetaReq(SInsertParseContext* pCxt, SToken* pStname, SMetaReq* pMetaReq) { +static int32_t buildMetaReq(SInsertParseContext* pCxt, SToken* pStname, SCatalogReq* pMetaReq) { pMetaReq->pTableName = taosArrayInit(4, sizeof(SName)); return buildTableName(pCxt, pStname, pMetaReq->pTableName); } static int32_t getTableMeta(SInsertParseContext* pCxt, SToken* pTname) { - SMetaReq req; + SCatalogReq req; CHECK_CODE(buildMetaReq(pCxt, pTname, &req)); - CHECK_CODE(catalogGetMetaData(pCxt->pCatalog, &req, &pCxt->meta)); + CHECK_CODE(catalogGetTableMeta(pCxt->pCatalog, NULL, NULL, NULL, &pCxt->meta)); //TODO pCxt->pTableMeta = (STableMeta*)taosArrayGetP(pCxt->meta.pTableMeta, 0); return TSDB_CODE_SUCCESS; } @@ -686,7 +687,6 @@ static int parseOneRow(SInsertParseContext* pCxt, STableDataBlocks* pDataBlocks, // 1. set the parsed value from sql string for (int i = 0; i < spd->numOfBound; ++i) { NEXT_TOKEN(pCxt->pSql, sToken); - // todo bind param SSchema *pSchema = &schema[spd->boundedColumns[i]]; param.schema = pSchema; param.compareStat = pBuilder->compareStat; @@ -770,14 +770,6 @@ static int32_t parseValuesClause(SInsertParseContext* pCxt, STableDataBlocks* da int32_t numOfRows = 0; CHECK_CODE(parseValues(pCxt, dataBuf, maxNumOfRows, &numOfRows)); - for (uint32_t i = 0; i < dataBuf->numOfParams; ++i) { - SParamInfo *param = dataBuf->params + i; - if (param->idx == -1) { - // param->idx = pInsertParam->numOfParams++; - param->offset -= sizeof(SSubmitBlk); - } - } - SSubmitBlk *pBlocks = (SSubmitBlk *)(dataBuf->pData); if (TSDB_CODE_SUCCESS != setBlockInfo(pBlocks, dataBuf->pTableMeta, numOfRows)) { return buildInvalidOperationMsg(&pCxt->msg, "too many rows in sql, total number of rows should be less than 32767"); @@ -815,12 +807,12 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) { CHECK_CODE(parseUsingClause(pCxt, &tbnameToken)); NEXT_TOKEN(pCxt->pSql, sToken); } else { - CHECK_CODE(getTableMeta(pCxt, &sToken)); + CHECK_CODE(getTableMeta(pCxt, &tbnameToken)); } STableDataBlocks *dataBuf = NULL; CHECK_CODE(getDataBlockFromList(pCxt->pTableBlockHashObj, pCxt->pTableMeta->uid, TSDB_DEFAULT_PAYLOAD_SIZE, - sizeof(SSubmitBlk), getTableInfo(pCxt->pTableMeta).rowSize, NULL/* tbname */, pCxt->pTableMeta, &dataBuf, NULL)); + sizeof(SSubmitBlk), getTableInfo(pCxt->pTableMeta).rowSize, pCxt->pTableMeta, &dataBuf, NULL)); if (TK_LP == sToken.type) { // pSql -> field1_name, ...) @@ -831,6 +823,7 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) { if (TK_VALUES == sToken.type) { // pSql -> (field1_value, ...) [(field1_value2, ...) ...] CHECK_CODE(parseValuesClause(pCxt, dataBuf)); + pCxt->pOutput->insertType = TSDB_QUERY_TYPE_INSERT; continue; } @@ -842,6 +835,7 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) { return buildSyntaxErrMsg(&pCxt->msg, "file path is required following keyword FILE", sToken.z); } // todo + pCxt->pOutput->insertType = TSDB_QUERY_TYPE_FILE_INSERT; continue; } @@ -867,13 +861,15 @@ int32_t parseInsertSql(SParseContext* pContext, SInsertStmtInfo** pInfo) { .pComCxt = pContext, .pSql = pContext->pSql, .msg = {.buf = pContext->pMsg, .len = pContext->msgLen}, - .pCatalog = getCatalogHandle(pContext->pEpSet), + .pCatalog = NULL, .pTableMeta = NULL, .pTableBlockHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false), .totalNum = 0, .pOutput = *pInfo }; + CHECK_CODE(catalogGetHandle(NULL, &context.pCatalog)); //TODO + if (NULL == context.pTableBlockHashObj) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } diff --git a/source/libs/parser/src/parser.c b/source/libs/parser/src/parser.c index a8642e2535..d167f7ad3c 100644 --- a/source/libs/parser/src/parser.c +++ b/source/libs/parser/src/parser.c @@ -43,7 +43,12 @@ int32_t qParseQuerySql(const char* pStr, size_t length, struct SQueryStmtInfo** return TSDB_CODE_TSC_SQL_SYNTAX_ERROR; } - struct SCatalog* pCatalog = getCatalogHandle(NULL); + struct SCatalog* pCatalog = NULL; + int32_t code = catalogGetHandle(NULL, &pCatalog); + if (code) { + return code; + } + return qParserValidateSqlNode(pCatalog, &info, *pQueryInfo, id, msg, msgLen); } @@ -132,7 +137,7 @@ static void freePtrElem(void* p) { tfree(*(char**)p); } -int32_t qParserExtractRequestedMetaInfo(const SSqlInfo* pSqlInfo, SMetaReq* pMetaInfo, char* msg, int32_t msgBufLen) { +int32_t qParserExtractRequestedMetaInfo(const SSqlInfo* pSqlInfo, SCatalogReq* pMetaInfo, char* msg, int32_t msgBufLen) { int32_t code = TSDB_CODE_SUCCESS; SMsgBuf msgBuf = {.buf = msg, .len = msgBufLen}; @@ -189,7 +194,7 @@ int32_t qParserExtractRequestedMetaInfo(const SSqlInfo* pSqlInfo, SMetaReq* pMet return code; } -void qParserClearupMetaRequestInfo(SMetaReq* pMetaReq) { +void qParserClearupMetaRequestInfo(SCatalogReq* pMetaReq) { if (pMetaReq == NULL) { return; } diff --git a/source/libs/parser/src/parserUtil.c b/source/libs/parser/src/parserUtil.c index 7b3239c0dd..28d01b9e66 100644 --- a/source/libs/parser/src/parserUtil.c +++ b/source/libs/parser/src/parserUtil.c @@ -1448,23 +1448,6 @@ void* vgroupInfoClear(SVgroupsInfo *vgroupList) { return NULL; } -char* serializeTagData(STagData* pTagData, char* pMsg) { - int32_t n = (int32_t) strlen(pTagData->name); - *(int32_t*) pMsg = htonl(n); - pMsg += sizeof(n); - - memcpy(pMsg, pTagData->name, n); - pMsg += n; - - *(int32_t*)pMsg = htonl(pTagData->dataLen); - pMsg += sizeof(int32_t); - - memcpy(pMsg, pTagData->data, pTagData->dataLen); - pMsg += pTagData->dataLen; - - return pMsg; -} - int32_t copyTagData(STagData* dst, const STagData* src) { dst->dataLen = src->dataLen; tstrncpy(dst->name, src->name, tListLen(dst->name)); diff --git a/source/libs/parser/test/insertTest.cpp b/source/libs/parser/test/insertTest.cpp index 08ca515b17..9cf48da4eb 100644 --- a/source/libs/parser/test/insertTest.cpp +++ b/source/libs/parser/test/insertTest.cpp @@ -36,51 +36,67 @@ namespace { // [...]; class InsertTest : public Test { protected: + void setDatabase(const string& db) { + db_ = db; + } + void bind(const char* sql) { reset(); - cxt.pSql = sql; - cxt.sqlLen = strlen(sql); + cxt_.sqlLen = strlen(sql); + strcpy(sqlBuf_, sql); + sqlBuf_[cxt_.sqlLen] = '\0'; + cxt_.pSql = sqlBuf_; + cxt_.pDbname = db_.c_str(); } int32_t run() { - code = parseInsertSql(&cxt, &res); - if (code != TSDB_CODE_SUCCESS) { - cout << "code:" << toString(code) << ", msg:" << errMagBuf << endl; + code_ = parseInsertSql(&cxt_, &res_); + if (code_ != TSDB_CODE_SUCCESS) { + cout << "code:" << toString(code_) << ", msg:" << errMagBuf_ << endl; } - return code; + return code_; } SInsertStmtInfo* reslut() { - return res; + return res_; } private: static const int max_err_len = 1024; + static const int max_sql_len = 1024 * 1024; void reset() { - memset(&cxt, 0, sizeof(cxt)); - memset(errMagBuf, 0, max_err_len); - cxt.pMsg = errMagBuf; - cxt.msgLen = max_err_len; - code = TSDB_CODE_SUCCESS; - res = nullptr; + memset(&cxt_, 0, sizeof(cxt_)); + memset(errMagBuf_, 0, max_err_len); + cxt_.pMsg = errMagBuf_; + cxt_.msgLen = max_err_len; + code_ = TSDB_CODE_SUCCESS; + res_ = nullptr; } - char errMagBuf[max_err_len]; - SParseContext cxt; - int32_t code; - SInsertStmtInfo* res; + string db_; + char errMagBuf_[max_err_len]; + char sqlBuf_[max_sql_len]; + SParseContext cxt_; + int32_t code_; + SInsertStmtInfo* res_; }; // INSERT INTO tb_name VALUES (field1_value, ...) TEST_F(InsertTest, simpleTest) { - bind("insert into .. values (...)"); + setDatabase("test"); + + bind("insert into t1 values (now, 1, \"wxy\")"); ASSERT_EQ(run(), TSDB_CODE_SUCCESS); SInsertStmtInfo* res = reslut(); // todo check + ASSERT_EQ(res->insertType, TSDB_QUERY_TYPE_INSERT); + // ASSERT_EQ(taosArrayGetSize(res->pDataBlocks), 1); } TEST_F(InsertTest, toleranceTest) { + setDatabase("test"); + bind("insert into"); ASSERT_NE(run(), TSDB_CODE_SUCCESS); bind("insert into t"); diff --git a/source/libs/parser/test/mockCatalog.cpp b/source/libs/parser/test/mockCatalog.cpp index fc988512c9..f2b34971ef 100644 --- a/source/libs/parser/test/mockCatalog.cpp +++ b/source/libs/parser/test/mockCatalog.cpp @@ -17,29 +17,30 @@ #include +namespace { + +void generateTestT1(MockCatalogService* mcs) { + ITableBuilder& builder = mcs->createTableBuilder("test", "t1", TSDB_NORMAL_TABLE, 3) + .setPrecision(TSDB_TIME_PRECISION_MILLI).setVgid(1).addColumn("ts", TSDB_DATA_TYPE_TIMESTAMP) + .addColumn("c1", TSDB_DATA_TYPE_INT).addColumn("c2", TSDB_DATA_TYPE_BINARY, 10); + builder.done(); +} + +void generateTestST1(MockCatalogService* mcs) { + ITableBuilder& builder = mcs->createTableBuilder("test", "st1", TSDB_SUPER_TABLE, 3, 2) + .setPrecision(TSDB_TIME_PRECISION_MILLI).addColumn("ts", TSDB_DATA_TYPE_TIMESTAMP) + .addTag("tag1", TSDB_DATA_TYPE_INT).addTag("tag2", TSDB_DATA_TYPE_BINARY, 10) + .addColumn("c1", TSDB_DATA_TYPE_INT).addColumn("c2", TSDB_DATA_TYPE_BINARY, 10); + builder.done(); + mcs->createSubTable("test", "st1", "st1s1", 1); + mcs->createSubTable("test", "st1", "st1s2", 2); +} + +} + void generateMetaData(MockCatalogService* mcs) { - { - ITableBuilder& builder = mcs->createTableBuilder("test", "t1", TSDB_NORMAL_TABLE, MockCatalogService::numOfDataTypes) - .setPrecision(TSDB_TIME_PRECISION_MILLI).setVgid(1).addColumn("ts", TSDB_DATA_TYPE_TIMESTAMP); - for (int32_t i = 0; i < MockCatalogService::numOfDataTypes; ++i) { - if (TSDB_DATA_TYPE_NULL == tDataTypes[i].type) { - continue; - } - builder = builder.addColumn("c" + std::to_string(i + 1), tDataTypes[i].type); - } - builder.done(); - } - { - ITableBuilder& builder = mcs->createTableBuilder("test", "st1", TSDB_SUPER_TABLE, MockCatalogService::numOfDataTypes, 2) - .setPrecision(TSDB_TIME_PRECISION_MILLI).setVgid(2).addColumn("ts", TSDB_DATA_TYPE_TIMESTAMP); - for (int32_t i = 0; i < MockCatalogService::numOfDataTypes; ++i) { - if (TSDB_DATA_TYPE_NULL == tDataTypes[i].type) { - continue; - } - builder = builder.addColumn("c" + std::to_string(i + 1), tDataTypes[i].type); - } - builder.done(); - } + generateTestT1(mcs); + generateTestST1(mcs); mcs->showTables(); } @@ -47,6 +48,6 @@ struct SCatalog* getCatalogHandle(const SEpSet* pMgmtEps) { return mockCatalogService->getCatalogHandle(pMgmtEps); } -int32_t catalogGetMetaData(struct SCatalog* pCatalog, const SMetaReq* pMetaReq, SMetaData* pMetaData) { +int32_t catalogGetMetaData(struct SCatalog* pCatalog, const SCatalogReq* pMetaReq, SMetaData* pMetaData) { return mockCatalogService->catalogGetMetaData(pCatalog, pMetaReq, pMetaData); } diff --git a/source/libs/parser/test/mockCatalog.h b/source/libs/parser/test/mockCatalog.h index 32001d19fd..e60f7727e6 100644 --- a/source/libs/parser/test/mockCatalog.h +++ b/source/libs/parser/test/mockCatalog.h @@ -22,6 +22,6 @@ void generateMetaData(MockCatalogService* mcs); // mock struct SCatalog* getCatalogHandle(const SEpSet* pMgmtEps); -int32_t catalogGetMetaData(struct SCatalog* pCatalog, const SMetaReq* pMetaReq, SMetaData* pMetaData); +int32_t catalogGetMetaData(struct SCatalog* pCatalog, const SCatalogReq* pMetaReq, SMetaData* pMetaData); #endif // MOCK_CATALOG_H diff --git a/source/libs/parser/test/mockCatalogService.cpp b/source/libs/parser/test/mockCatalogService.cpp index de4fa8ee7e..457f8d88bd 100644 --- a/source/libs/parser/test/mockCatalogService.cpp +++ b/source/libs/parser/test/mockCatalogService.cpp @@ -19,6 +19,7 @@ #include #include +#include "tname.h" #include "ttypes.h" std::unique_ptr mockCatalogService; @@ -82,12 +83,26 @@ public: MockCatalogServiceImpl() { } - struct SCatalog* getCatalogHandle(const SEpSet* pMgmtEps) { + struct SCatalog* getCatalogHandle(const SEpSet* pMgmtEps) const { return (struct SCatalog*)0x01; } - int32_t catalogGetMetaData(struct SCatalog* pCatalog, const SMetaReq* pMetaReq, SMetaData* pMetaData) { - return 0; + int32_t catalogGetMetaData(struct SCatalog* pCatalog, const SCatalogReq* pMetaReq, SMetaData* pMetaData) const { + assert(nullptr != pMetaReq && 1 == taosArrayGetSize(pMetaReq->pTableName)); + SName* fullName = (SName*)taosArrayGet(pMetaReq->pTableName, 0); + std::unique_ptr table; + int32_t code = copyTableMeta(fullName->dbname, fullName->tname, &table); + if (TSDB_CODE_SUCCESS != code) { + return code; + } + std::unique_ptr tables((SArray*)taosArrayInit(1, sizeof(STableMeta*))); + if (!tables) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + STableMeta* elem = table.release(); + taosArrayPush(tables.get(), &elem); + pMetaData->pTableMeta = tables.release(); + return TSDB_CODE_SUCCESS; } TableBuilder& createTableBuilder(const std::string& db, const std::string& tbname, int8_t tableType, int32_t numOfColumns, int32_t numOfTags) { @@ -97,6 +112,15 @@ public: return *(builder_.get()); } + void createSubTable(const std::string& db, const std::string& stbname, const std::string& tbname, int16_t vgid) { + std::unique_ptr table; + if (TSDB_CODE_SUCCESS != copyTableMeta(db, stbname, &table)) { + throw std::runtime_error("copyTableMeta failed"); + } + meta_[db][tbname].reset(table.release()); + meta_[db][tbname]->uid = id_++; + } + void showTables() const { // number of forward fills #define NOF(n) ((n) / 2) @@ -120,20 +144,35 @@ public: #define SL(sn, in) std::setfill('=') << std::setw((sn) * (SFL + 1) + (in) * (IFL + 1)) << "" << std::setfill(' ') for (const auto& db : meta_) { - std::cout << SH("Database") << SH("Table") << SH("Type") << SH("Precision") << IH(std::string("Vgid")) << std::endl; - std::cout << SL(4, 1) << std::endl; + std::cout << "Databse:" << db.first << std::endl; + std::cout << SH("Table") << SH("Type") << SH("Precision") << IH("Vgid") << std::endl; + std::cout << SL(3, 1) << std::endl; for (const auto& table : db.second) { - std::cout << SF(db.first) << SF(table.first) << SF(ttToString(table.second->tableType)) << SF(pToString(table.second->tableInfo.precision)) << IF(table.second->vgId) << std::endl; - // int16_t numOfFields = table.second->tableInfo.numOfTags + table.second->tableInfo.numOfColumns; - // for (int16_t i = 0; i < numOfFields; ++i) { - // const SSchema* schema = table.second->schema + i; - // std::cout << schema->name << " " << schema->type << " " << schema->bytes << std::endl; - // } + std::cout << SF(table.first) << SF(ttToString(table.second->tableType)) << SF(pToString(table.second->tableInfo.precision)) << IF(table.second->vgId) << std::endl; + } + std::cout << std::endl; + } + + for (const auto& db : meta_) { + for (const auto& table : db.second) { + std::cout << "Table:" << table.first << std::endl; + std::cout << SH("Field") << SH("Type") << SH("DataType") << IH("Bytes") << std::endl; + std::cout << SL(3, 1) << std::endl; + int16_t numOfTags = table.second->tableInfo.numOfTags; + int16_t numOfFields = numOfTags + table.second->tableInfo.numOfColumns; + for (int16_t i = 0; i < numOfFields; ++i) { + const SSchema* schema = table.second->schema + i; + std::cout << SF(std::string(schema->name)) << SH(ftToString(i, numOfTags)) << SH(dtToString(schema->type)) << IF(schema->bytes) << std::endl; + } + std::cout << std::endl; } } } private: + typedef std::map > TableMetaCache; + typedef std::map DbMetaCache; + std::string ttToString(int8_t tableType) const { switch (tableType) { case TSDB_SUPER_TABLE: @@ -160,9 +199,43 @@ private: } } + std::string dtToString(int8_t type) const { + return tDataTypes[type].name; + } + + std::string ftToString(int16_t colid, int16_t numOfTags) const { + return (0 == colid ? "column" : (colid <= numOfTags ? "tag" : "column")); + } + + std::shared_ptr getTableMeta(const std::string& db, const std::string& tbname) const { + DbMetaCache::const_iterator it = meta_.find(db); + if (meta_.end() == it) { + return std::shared_ptr(); + } + TableMetaCache::const_iterator tit = it->second.find(tbname); + if (it->second.end() == tit) { + return std::shared_ptr(); + } + return tit->second; + } + + int32_t copyTableMeta(const std::string& db, const std::string& tbname, std::unique_ptr* dst) const { + std::shared_ptr src = getTableMeta(db, tbname); + if (!src) { + return TSDB_CODE_TSC_INVALID_TABLE_NAME; + } + int32_t len = sizeof(STableMeta) + sizeof(SSchema) * (src->tableInfo.numOfTags + src->tableInfo.numOfColumns); + dst->reset((STableMeta*)std::calloc(1, len)); + if (!dst) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + memcpy(dst->get(), src.get(), len); + return TSDB_CODE_SUCCESS; + } + uint64_t id_; std::unique_ptr builder_; - std::map > > meta_; + DbMetaCache meta_; }; MockCatalogService::MockCatalogService() : impl_(new MockCatalogServiceImpl()) { @@ -171,11 +244,11 @@ MockCatalogService::MockCatalogService() : impl_(new MockCatalogServiceImpl()) { MockCatalogService::~MockCatalogService() { } -struct SCatalog* MockCatalogService::getCatalogHandle(const SEpSet* pMgmtEps) { +struct SCatalog* MockCatalogService::getCatalogHandle(const SEpSet* pMgmtEps) const { return impl_->getCatalogHandle(pMgmtEps); } -int32_t MockCatalogService::catalogGetMetaData(struct SCatalog* pCatalog, const SMetaReq* pMetaReq, SMetaData* pMetaData) { +int32_t MockCatalogService::catalogGetMetaData(struct SCatalog* pCatalog, const SCatalogReq* pMetaReq, SMetaData* pMetaData) const { return impl_->catalogGetMetaData(pCatalog, pMetaReq, pMetaData); } @@ -183,6 +256,10 @@ ITableBuilder& MockCatalogService::createTableBuilder(const std::string& db, con return impl_->createTableBuilder(db, tbname, tableType, numOfColumns, numOfTags); } +void MockCatalogService::createSubTable(const std::string& db, const std::string& stbname, const std::string& tbname, int16_t vgid) { + impl_->createSubTable(db, stbname, tbname, vgid); +} + void MockCatalogService::showTables() const { impl_->showTables(); } \ No newline at end of file diff --git a/source/libs/parser/test/mockCatalogService.h b/source/libs/parser/test/mockCatalogService.h index ba974ab200..66b439b3e9 100644 --- a/source/libs/parser/test/mockCatalogService.h +++ b/source/libs/parser/test/mockCatalogService.h @@ -49,9 +49,10 @@ public: MockCatalogService(); ~MockCatalogService(); - struct SCatalog* getCatalogHandle(const SEpSet* pMgmtEps); - int32_t catalogGetMetaData(struct SCatalog* pCatalog, const SMetaReq* pMetaReq, SMetaData* pMetaData); + struct SCatalog* getCatalogHandle(const SEpSet* pMgmtEps) const; + int32_t catalogGetMetaData(struct SCatalog* pCatalog, const SCatalogReq* pMetaReq, SMetaData* pMetaData) const; ITableBuilder& createTableBuilder(const std::string& db, const std::string& tbname, int8_t tableType, int32_t numOfColumns, int32_t numOfTags = 0); + void createSubTable(const std::string& db, const std::string& stbname, const std::string& tbname, int16_t vgid); void showTables() const; private: diff --git a/source/libs/parser/test/parserTests.cpp b/source/libs/parser/test/parserTests.cpp index 2193a44604..6a402cd620 100644 --- a/source/libs/parser/test/parserTests.cpp +++ b/source/libs/parser/test/parserTests.cpp @@ -38,7 +38,7 @@ void setSchema(SSchema* p, int32_t type, int32_t bytes, const char* name, int32_ strcpy(p->name, name); } -void setTableMetaInfo(SQueryStmtInfo* pQueryInfo, SMetaReq* req) { +void setTableMetaInfo(SQueryStmtInfo* pQueryInfo, SCatalogReq* req) { pQueryInfo->numOfTables = 1; pQueryInfo->pTableMetaInfo = (STableMetaInfo**)calloc(1, POINTER_BYTES); @@ -80,7 +80,7 @@ void sqlCheck(const char* sql, bool valid) { int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); ASSERT_EQ(code, 0); - SMetaReq req = {0}; + SCatalogReq req = {0}; int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, msg, 128); ASSERT_EQ(ret, 0); ASSERT_EQ(taosArrayGetSize(req.pTableName), 1); @@ -117,7 +117,7 @@ TEST(testCase, validateAST_test) { int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); ASSERT_EQ(code, 0); - SMetaReq req = {0}; + SCatalogReq req = {0}; int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, msg, 128); ASSERT_EQ(ret, 0); ASSERT_EQ(taosArrayGetSize(req.pTableName), 1); @@ -175,7 +175,7 @@ TEST(testCase, function_Test) { int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); ASSERT_EQ(code, 0); - SMetaReq req = {0}; + SCatalogReq req = {0}; int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, msg, 128); ASSERT_EQ(ret, 0); ASSERT_EQ(taosArrayGetSize(req.pTableName), 1); @@ -221,7 +221,7 @@ TEST(testCase, function_Test2) { int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); ASSERT_EQ(code, 0); - SMetaReq req = {0}; + SCatalogReq req = {0}; int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, msg, 128); ASSERT_EQ(ret, 0); ASSERT_EQ(taosArrayGetSize(req.pTableName), 1); @@ -267,7 +267,7 @@ TEST(testCase, function_Test3) { int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); ASSERT_EQ(code, 0); - SMetaReq req = {0}; + SCatalogReq req = {0}; int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, msg, 128); ASSERT_EQ(ret, 0); ASSERT_EQ(taosArrayGetSize(req.pTableName), 1); @@ -312,7 +312,7 @@ TEST(testCase, function_Test4) { int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); ASSERT_EQ(code, 0); - SMetaReq req = {0}; + SCatalogReq req = {0}; int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, msg, 128); ASSERT_EQ(ret, 0); ASSERT_EQ(taosArrayGetSize(req.pTableName), 1); @@ -360,7 +360,7 @@ TEST(testCase, function_Test5) { int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); ASSERT_EQ(code, 0); - SMetaReq req = {0}; + SCatalogReq req = {0}; int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, msg, 128); ASSERT_EQ(ret, 0); ASSERT_EQ(taosArrayGetSize(req.pTableName), 1); @@ -445,7 +445,7 @@ TEST(testCase, function_Test6) { int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); ASSERT_EQ(code, 0); - SMetaReq req = {0}; + SCatalogReq req = {0}; int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, msg, 128); ASSERT_EQ(ret, 0); ASSERT_EQ(taosArrayGetSize(req.pTableName), 1); @@ -523,7 +523,7 @@ TEST(testCase, function_Test6) { int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); ASSERT_EQ(code, 0); - SMetaReq req = {0}; + SCatalogReq req = {0}; int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, msg, 128); ASSERT_EQ(ret, 0); ASSERT_EQ(taosArrayGetSize(req.pTableName), 1); @@ -585,7 +585,7 @@ TEST(testCase, function_Test6) { int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); ASSERT_EQ(code, 0); - SMetaReq req = {0}; + SCatalogReq req = {0}; int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, msg, 128); ASSERT_EQ(ret, 0); ASSERT_EQ(taosArrayGetSize(req.pTableName), 1); @@ -664,7 +664,7 @@ TEST(testCase, function_Test6) { int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); ASSERT_EQ(code, 0); - SMetaReq req = {0}; + SCatalogReq req = {0}; int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, msg, 128); ASSERT_EQ(ret, 0); ASSERT_EQ(taosArrayGetSize(req.pTableName), 1); diff --git a/source/libs/parser/test/plannerTest.cpp b/source/libs/parser/test/plannerTest.cpp index c86e687664..bb9271a3c8 100644 --- a/source/libs/parser/test/plannerTest.cpp +++ b/source/libs/parser/test/plannerTest.cpp @@ -39,7 +39,7 @@ void setSchema(SSchema* p, int32_t type, int32_t bytes, const char* name, int32_ strcpy(p->name, name); } -void setTableMetaInfo(SQueryStmtInfo* pQueryInfo, SMetaReq *req) { +void setTableMetaInfo(SQueryStmtInfo* pQueryInfo, SCatalogReq *req) { pQueryInfo->numOfTables = 1; pQueryInfo->pTableMetaInfo = (STableMetaInfo**)calloc(1, POINTER_BYTES); @@ -79,7 +79,7 @@ void generateLogicplan(const char* sql) { int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); ASSERT_EQ(code, 0); - SMetaReq req = {0}; + SCatalogReq req = {0}; int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, msg, 128); ASSERT_EQ(ret, 0); ASSERT_EQ(taosArrayGetSize(req.pTableName), 1); @@ -119,7 +119,7 @@ TEST(testCase, planner_test) { int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); ASSERT_EQ(code, 0); - SMetaReq req = {0}; + SCatalogReq req = {0}; int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, msg, 128); ASSERT_EQ(ret, 0); ASSERT_EQ(taosArrayGetSize(req.pTableName), 1); diff --git a/source/libs/parser/test/tokenizerTest.cpp b/source/libs/parser/test/tokenizerTest.cpp index 032a93cd32..23c0aae15f 100644 --- a/source/libs/parser/test/tokenizerTest.cpp +++ b/source/libs/parser/test/tokenizerTest.cpp @@ -709,7 +709,7 @@ TEST(testCase, extractMeta_test) { ASSERT_EQ(info1.valid, true); char msg[128] = {0}; - SMetaReq req = {0}; + SCatalogReq req = {0}; int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, msg, 128); ASSERT_EQ(ret, 0); ASSERT_EQ(taosArrayGetSize(req.pTableName), 1); diff --git a/source/libs/query/CMakeLists.txt b/source/libs/query/CMakeLists.txt new file mode 100644 index 0000000000..579a4b279c --- /dev/null +++ b/source/libs/query/CMakeLists.txt @@ -0,0 +1,12 @@ +aux_source_directory(src QUERY_SRC) +add_library(query ${QUERY_SRC}) +target_include_directories( + query + PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/query" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" +) + +target_link_libraries( + query + PRIVATE os util common transport +) diff --git a/source/libs/query/inc/queryInt.h b/source/libs/query/inc/queryInt.h new file mode 100644 index 0000000000..f3204b3785 --- /dev/null +++ b/source/libs/query/inc/queryInt.h @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_QUERY_INT_H_ +#define _TD_QUERY_INT_H_ + +#ifdef __cplusplus +extern "C" { +#endif + + +#include "tlog.h" + +extern int32_t qDebugFlag; + +#define qFatal(...) do { if (qDebugFlag & DEBUG_FATAL) { taosPrintLog("QRY FATAL ", qDebugFlag, __VA_ARGS__); }} while(0) +#define qError(...) do { if (qDebugFlag & DEBUG_ERROR) { taosPrintLog("QRY ERROR ", qDebugFlag, __VA_ARGS__); }} while(0) +#define qWarn(...) do { if (qDebugFlag & DEBUG_WARN) { taosPrintLog("QRY WARN ", qDebugFlag, __VA_ARGS__); }} while(0) +#define qInfo(...) do { if (qDebugFlag & DEBUG_INFO) { taosPrintLog("QRY ", qDebugFlag, __VA_ARGS__); }} while(0) +#define qDebug(...) do { if (qDebugFlag & DEBUG_DEBUG) { taosPrintLog("QRY ", qDebugFlag, __VA_ARGS__); }} while(0) +#define qTrace(...) do { if (qDebugFlag & DEBUG_TRACE) { taosPrintLog("QRY ", qDebugFlag, __VA_ARGS__); }} while(0) +#define qDebugL(...) do { if (qDebugFlag & DEBUG_DEBUG) { taosPrintLongString("QRY ", qDebugFlag, __VA_ARGS__); }} while(0) + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_QUERY_INT_H_*/ diff --git a/source/libs/query/src/querymsg.c b/source/libs/query/src/querymsg.c new file mode 100644 index 0000000000..924878c872 --- /dev/null +++ b/source/libs/query/src/querymsg.c @@ -0,0 +1,192 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "taosmsg.h" +#include "queryInt.h" + + +int32_t (*queryBuildMsg[TSDB_MSG_TYPE_MAX])(void* input, char **msg, int32_t msgSize, int32_t *msgLen) = {0}; + +int32_t (*queryProcessMsgRsp[TSDB_MSG_TYPE_MAX])(void* output, char *msg, int32_t msgSize) = {0}; + + +int32_t queryBuildVgroupListReqMsg(void* input, char **msg, int32_t msgSize, int32_t *msgLen) { + if (NULL == msg || NULL == msgLen) { + return TSDB_CODE_TSC_INVALID_INPUT; + } + + *msgLen = 0; + + return TSDB_CODE_SUCCESS; +} + +int32_t queryBuildTableMetaReqMsg(void* input, char **msg, int32_t msgSize, int32_t *msgLen) { + if (NULL == input || NULL == msg || NULL == msgLen) { + return TSDB_CODE_TSC_INVALID_INPUT; + } + + SBuildTableMetaInput* bInput = (SBuildTableMetaInput *)input; + + int32_t estimateSize = sizeof(STableInfoMsg); + if (NULL == *msg || msgSize < estimateSize) { + tfree(*msg); + *msg = calloc(1, estimateSize); + if (NULL == *msg) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + } + + STableInfoMsg *bMsg = (STableInfoMsg *)*msg; + + bMsg->msgHead.vgId = bInput->vgId; + + strncpy(bMsg->tableFname, bInput->tableFullName, sizeof(bMsg->tableFname)); + bMsg->tableFname[sizeof(bMsg->tableFname) - 1] = 0; + + *msgLen = (int32_t)sizeof(*bMsg); + + return TSDB_CODE_SUCCESS; +} + + +int32_t queryProcessVgroupListRsp(void* output, char *msg, int32_t msgSize) { + if (NULL == output || NULL == msg || msgSize <= 0) { + return TSDB_CODE_TSC_INVALID_INPUT; + } + + SVgroupListRspMsg *pRsp = (SVgroupListRspMsg *)msg; + + pRsp->vgroupNum = htonl(pRsp->vgroupNum); + pRsp->vgroupVersion = htonl(pRsp->vgroupVersion); + + if (pRsp->vgroupNum < 0) { + qError("vgroup number[%d] in rsp is invalid", pRsp->vgroupNum); + return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; + } + + if (pRsp->vgroupVersion < 0) { + qError("vgroup vgroupVersion[%d] in rsp is invalid", pRsp->vgroupVersion); + return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; + } + + if (msgSize != (pRsp->vgroupNum * sizeof(pRsp->vgroupInfo[0]) + sizeof(*pRsp))) { + qError("vgroup list msg size mis-match, msgSize:%d, vgroup number:%d", msgSize, pRsp->vgroupNum); + return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; + } + + // keep SVgroupListInfo/SVgroupListRspMsg the same + *(SVgroupListInfo **)output = (SVgroupListInfo *)msg; + + if (pRsp->vgroupNum == 0) { + return TSDB_CODE_SUCCESS; + } + + for (int32_t i = 0; i < pRsp->vgroupNum; ++i) { + pRsp->vgroupInfo[i].vgId = htonl(pRsp->vgroupInfo[i].vgId); + for (int32_t n = 0; n < pRsp->vgroupInfo[i].numOfEps; ++n) { + pRsp->vgroupInfo[i].epAddr[n].port = htonl(pRsp->vgroupInfo[i].epAddr[n].port); + } + } + + return TSDB_CODE_SUCCESS; +} + +void msgInit() { + queryBuildMsg[TSDB_MSG_TYPE_TABLE_META] = queryBuildTableMetaReqMsg; + queryBuildMsg[TSDB_MSG_TYPE_VGROUP_LIST] = queryBuildVgroupListReqMsg; + + //tscProcessMsgRsp[TSDB_MSG_TYPE_TABLE_META] = tscProcessTableMetaRsp; + queryProcessMsgRsp[TSDB_MSG_TYPE_VGROUP_LIST] = queryProcessVgroupListRsp; + +/* + tscBuildMsg[TSDB_SQL_SELECT] = tscBuildQueryMsg; + tscBuildMsg[TSDB_SQL_INSERT] = tscBuildSubmitMsg; + tscBuildMsg[TSDB_SQL_FETCH] = tscBuildFetchMsg; + + tscBuildMsg[TSDB_SQL_CREATE_DB] = tscBuildCreateDbMsg; + tscBuildMsg[TSDB_SQL_CREATE_USER] = tscBuildUserMsg; + tscBuildMsg[TSDB_SQL_CREATE_FUNCTION] = tscBuildCreateFuncMsg; + + tscBuildMsg[TSDB_SQL_CREATE_ACCT] = tscBuildAcctMsg; + tscBuildMsg[TSDB_SQL_ALTER_ACCT] = tscBuildAcctMsg; + + tscBuildMsg[TSDB_SQL_CREATE_TABLE] = tscBuildCreateTableMsg; + tscBuildMsg[TSDB_SQL_DROP_USER] = tscBuildDropUserAcctMsg; + tscBuildMsg[TSDB_SQL_DROP_ACCT] = tscBuildDropUserAcctMsg; + tscBuildMsg[TSDB_SQL_DROP_DB] = tscBuildDropDbMsg; + tscBuildMsg[TSDB_SQL_DROP_FUNCTION] = tscBuildDropFuncMsg; + tscBuildMsg[TSDB_SQL_SYNC_DB_REPLICA] = tscBuildSyncDbReplicaMsg; + tscBuildMsg[TSDB_SQL_DROP_TABLE] = tscBuildDropTableMsg; + tscBuildMsg[TSDB_SQL_ALTER_USER] = tscBuildUserMsg; + tscBuildMsg[TSDB_SQL_CREATE_DNODE] = tscBuildCreateDnodeMsg; + tscBuildMsg[TSDB_SQL_DROP_DNODE] = tscBuildDropDnodeMsg; + tscBuildMsg[TSDB_SQL_CFG_DNODE] = tscBuildCfgDnodeMsg; + tscBuildMsg[TSDB_SQL_ALTER_TABLE] = tscBuildAlterTableMsg; + tscBuildMsg[TSDB_SQL_UPDATE_TAGS_VAL] = tscBuildUpdateTagMsg; + tscBuildMsg[TSDB_SQL_ALTER_DB] = tscAlterDbMsg; + tscBuildMsg[TSDB_SQL_COMPACT_VNODE] = tscBuildCompactMsg; + + tscBuildMsg[TSDB_SQL_CONNECT] = tscBuildConnectMsg; + tscBuildMsg[TSDB_SQL_USE_DB] = tscBuildUseDbMsg; + tscBuildMsg[TSDB_SQL_STABLEVGROUP] = tscBuildSTableVgroupMsg; + tscBuildMsg[TSDB_SQL_RETRIEVE_FUNC] = tscBuildRetrieveFuncMsg; + + tscBuildMsg[TSDB_SQL_HB] = tscBuildHeartBeatMsg; + tscBuildMsg[TSDB_SQL_SHOW] = tscBuildShowMsg; + tscBuildMsg[TSDB_SQL_RETRIEVE] = tscBuildRetrieveFromMgmtMsg; + tscBuildMsg[TSDB_SQL_KILL_QUERY] = tscBuildKillMsg; + tscBuildMsg[TSDB_SQL_KILL_STREAM] = tscBuildKillMsg; + tscBuildMsg[TSDB_SQL_KILL_CONNECTION] = tscBuildKillMsg; + + tscProcessMsgRsp[TSDB_SQL_SELECT] = tscProcessQueryRsp; + tscProcessMsgRsp[TSDB_SQL_FETCH] = tscProcessRetrieveRspFromNode; + + tscProcessMsgRsp[TSDB_SQL_DROP_DB] = tscProcessDropDbRsp; + tscProcessMsgRsp[TSDB_SQL_DROP_TABLE] = tscProcessDropTableRsp; + tscProcessMsgRsp[TSDB_SQL_CONNECT] = tscProcessConnectRsp; + tscProcessMsgRsp[TSDB_SQL_USE_DB] = tscProcessUseDbRsp; + tscProcessMsgRsp[TSDB_SQL_META] = tscProcessTableMetaRsp; + tscProcessMsgRsp[TSDB_SQL_STABLEVGROUP] = tscProcessSTableVgroupRsp; + tscProcessMsgRsp[TSDB_SQL_MULTI_META] = tscProcessMultiTableMetaRsp; + tscProcessMsgRsp[TSDB_SQL_RETRIEVE_FUNC] = tscProcessRetrieveFuncRsp; + + tscProcessMsgRsp[TSDB_SQL_SHOW] = tscProcessShowRsp; + tscProcessMsgRsp[TSDB_SQL_RETRIEVE] = tscProcessRetrieveRspFromNode; // rsp handled by same function. + tscProcessMsgRsp[TSDB_SQL_DESCRIBE_TABLE] = tscProcessDescribeTableRsp; + + tscProcessMsgRsp[TSDB_SQL_CURRENT_DB] = tscProcessLocalRetrieveRsp; + tscProcessMsgRsp[TSDB_SQL_CURRENT_USER] = tscProcessLocalRetrieveRsp; + tscProcessMsgRsp[TSDB_SQL_SERV_VERSION] = tscProcessLocalRetrieveRsp; + tscProcessMsgRsp[TSDB_SQL_CLI_VERSION] = tscProcessLocalRetrieveRsp; + tscProcessMsgRsp[TSDB_SQL_SERV_STATUS] = tscProcessLocalRetrieveRsp; + + tscProcessMsgRsp[TSDB_SQL_RETRIEVE_EMPTY_RESULT] = tscProcessEmptyResultRsp; + + tscProcessMsgRsp[TSDB_SQL_RETRIEVE_GLOBALMERGE] = tscProcessRetrieveGlobalMergeRsp; + + tscProcessMsgRsp[TSDB_SQL_ALTER_TABLE] = tscProcessAlterTableMsgRsp; + tscProcessMsgRsp[TSDB_SQL_ALTER_DB] = tscProcessAlterDbMsgRsp; + tscProcessMsgRsp[TSDB_SQL_COMPACT_VNODE] = tscProcessCompactRsp; + + tscProcessMsgRsp[TSDB_SQL_SHOW_CREATE_TABLE] = tscProcessShowCreateRsp; + tscProcessMsgRsp[TSDB_SQL_SHOW_CREATE_STABLE] = tscProcessShowCreateRsp; + tscProcessMsgRsp[TSDB_SQL_SHOW_CREATE_DATABASE] = tscProcessShowCreateRsp; +*/ +} + + + + + diff --git a/source/libs/sync/src/sync.c b/source/libs/sync/src/sync.c index 06af8ff6c2..e49b1d7983 100644 --- a/source/libs/sync/src/sync.c +++ b/source/libs/sync/src/sync.c @@ -228,7 +228,7 @@ static int syncInitRpcServer(SSyncManager* syncManager, const SSyncCluster* pSyn } static int syncInitRpcClient(SSyncManager* syncManager) { - char secret[TSDB_KEY_LEN] = "secret"; + char secret[TSDB_PASSWORD_LEN] = "secret"; SRpcInit rpcInit; memset(&rpcInit, 0, sizeof(rpcInit)); rpcInit.label = "sync-client"; diff --git a/source/libs/transport/inc/rpcLog.h b/source/libs/transport/inc/rpcLog.h index 6c4a281d2c..904680bbe6 100644 --- a/source/libs/transport/inc/rpcLog.h +++ b/source/libs/transport/inc/rpcLog.h @@ -23,11 +23,10 @@ extern "C" { #include "tlog.h" extern int32_t rpcDebugFlag; -extern int8_t tscEmbedded; -#define tFatal(...) { if (rpcDebugFlag & DEBUG_FATAL) { taosPrintLog("RPC FATAL ", tscEmbedded ? 255 : rpcDebugFlag, __VA_ARGS__); }} -#define tError(...) { if (rpcDebugFlag & DEBUG_ERROR) { taosPrintLog("RPC ERROR ", tscEmbedded ? 255 : rpcDebugFlag, __VA_ARGS__); }} -#define tWarn(...) { if (rpcDebugFlag & DEBUG_WARN) { taosPrintLog("RPC WARN ", tscEmbedded ? 255 : rpcDebugFlag, __VA_ARGS__); }} +#define tFatal(...) { if (rpcDebugFlag & DEBUG_FATAL) { taosPrintLog("RPC FATAL ", rpcDebugFlag, __VA_ARGS__); }} +#define tError(...) { if (rpcDebugFlag & DEBUG_ERROR) { taosPrintLog("RPC ERROR ", rpcDebugFlag, __VA_ARGS__); }} +#define tWarn(...) { if (rpcDebugFlag & DEBUG_WARN) { taosPrintLog("RPC WARN ", rpcDebugFlag, __VA_ARGS__); }} #define tInfo(...) { if (rpcDebugFlag & DEBUG_INFO) { taosPrintLog("RPC ", tscEmbedded ? 255 : rpcDebugFlag, __VA_ARGS__); }} #define tDebug(...) { if (rpcDebugFlag & DEBUG_DEBUG) { taosPrintLog("RPC ", rpcDebugFlag, __VA_ARGS__); }} #define tTrace(...) { if (rpcDebugFlag & DEBUG_TRACE) { taosPrintLog("RPC ", rpcDebugFlag, __VA_ARGS__); }} diff --git a/source/libs/transport/src/rpcMain.c b/source/libs/transport/src/rpcMain.c index fb69a74876..2f710f3efd 100644 --- a/source/libs/transport/src/rpcMain.c +++ b/source/libs/transport/src/rpcMain.c @@ -51,8 +51,8 @@ typedef struct { char user[TSDB_UNI_LEN]; // meter ID char spi; // security parameter index char encrypt; // encrypt algorithm - char secret[TSDB_KEY_LEN]; // secret for the link - char ckey[TSDB_KEY_LEN]; // ciphering key + char secret[TSDB_PASSWORD_LEN]; // secret for the link + char ckey[TSDB_PASSWORD_LEN]; // ciphering key void (*cfp)(void *parent, SRpcMsg *, SEpSet *); int (*afp)(void *parent, char *user, char *spi, char *encrypt, char *secret, char *ckey); @@ -97,8 +97,8 @@ typedef struct SRpcConn { char user[TSDB_UNI_LEN]; // user ID for the link char spi; // security parameter index char encrypt; // encryption, 0:1 - char secret[TSDB_KEY_LEN]; // secret for the link - char ckey[TSDB_KEY_LEN]; // ciphering key + char secret[TSDB_PASSWORD_LEN]; // secret for the link + char ckey[TSDB_PASSWORD_LEN]; // ciphering key char secured; // if set to 1, no authentication uint16_t localPort; // for UDP only uint32_t linkUid; // connection unique ID assigned by client @@ -229,8 +229,6 @@ static void rpcInitImp(void) { tsRpcOverhead = sizeof(SRpcReqContext); tsRpcRefId = taosOpenRef(200, rpcFree); - - return 0; } int32_t rpcInit(void) { @@ -413,7 +411,7 @@ void rpcSendRequest(void *shandle, const SEpSet *pEpSet, SRpcMsg *pMsg, int64_t // for TDengine, all the query, show commands shall have TCP connection char type = pMsg->msgType; if (type == TSDB_MSG_TYPE_QUERY || type == TSDB_MSG_TYPE_SHOW_RETRIEVE - || type == TSDB_MSG_TYPE_FETCH || type == TSDB_MSG_TYPE_STB_VGROUP + || type == TSDB_MSG_TYPE_FETCH || type == TSDB_MSG_TYPE_VGROUP_LIST || type == TSDB_MSG_TYPE_TABLES_META || type == TSDB_MSG_TYPE_TABLE_META || type == TSDB_MSG_TYPE_SHOW || type == TSDB_MSG_TYPE_STATUS || type == TSDB_MSG_TYPE_ALTER_TABLE) pContext->connType = RPC_CONN_TCPC; @@ -705,7 +703,7 @@ static SRpcConn *rpcAllocateClientConn(SRpcInfo *pRpc) { pConn->linkUid = (uint32_t)((int64_t)pConn + taosGetPid() + (int64_t)pConn->tranId); pConn->spi = pRpc->spi; pConn->encrypt = pRpc->encrypt; - if (pConn->spi) memcpy(pConn->secret, pRpc->secret, TSDB_KEY_LEN); + if (pConn->spi) memcpy(pConn->secret, pRpc->secret, TSDB_PASSWORD_LEN); tDebug("%s %p client connection is allocated, uid:0x%x", pRpc->label, pConn, pConn->linkUid); } @@ -1534,9 +1532,9 @@ static int rpcAuthenticateMsg(void *pMsg, int msgLen, void *pAuth, void *pKey) { int ret = -1; tMD5Init(&context); - tMD5Update(&context, (uint8_t *)pKey, TSDB_KEY_LEN); + tMD5Update(&context, (uint8_t *)pKey, TSDB_PASSWORD_LEN); tMD5Update(&context, (uint8_t *)pMsg, msgLen); - tMD5Update(&context, (uint8_t *)pKey, TSDB_KEY_LEN); + tMD5Update(&context, (uint8_t *)pKey, TSDB_PASSWORD_LEN); tMD5Final(&context); if (memcmp(context.digest, pAuth, sizeof(context.digest)) == 0) ret = 0; @@ -1548,9 +1546,9 @@ static void rpcBuildAuthHead(void *pMsg, int msgLen, void *pAuth, void *pKey) { T_MD5_CTX context; tMD5Init(&context); - tMD5Update(&context, (uint8_t *)pKey, TSDB_KEY_LEN); + tMD5Update(&context, (uint8_t *)pKey, TSDB_PASSWORD_LEN); tMD5Update(&context, (uint8_t *)pMsg, msgLen); - tMD5Update(&context, (uint8_t *)pKey, TSDB_KEY_LEN); + tMD5Update(&context, (uint8_t *)pKey, TSDB_PASSWORD_LEN); tMD5Final(&context); memcpy(pAuth, context.digest, sizeof(context.digest)); diff --git a/source/libs/wal/inc/walInt.h b/source/libs/wal/inc/walInt.h index f8f2e2eadd..e546a87326 100644 --- a/source/libs/wal/inc/walInt.h +++ b/source/libs/wal/inc/walInt.h @@ -33,10 +33,12 @@ typedef struct WalFileInfo { int64_t fileSize; } WalFileInfo; +#pragma pack(push,1) typedef struct WalIdxEntry { int64_t ver; int64_t offset; } WalIdxEntry; +#pragma pack(pop) static inline int32_t compareWalFileInfo(const void* pLeft, const void* pRight) { WalFileInfo* pInfoLeft = (WalFileInfo*)pLeft; @@ -78,11 +80,11 @@ static inline WalFileInfo* walGetCurFileInfo(SWal* pWal) { } static inline int walBuildLogName(SWal*pWal, int64_t fileFirstVer, char* buf) { - return sprintf(buf, "%s/%" PRId64 "." WAL_LOG_SUFFIX, pWal->path, fileFirstVer); + return sprintf(buf, "%s/%020" PRId64 "." WAL_LOG_SUFFIX, pWal->path, fileFirstVer); } static inline int walBuildIdxName(SWal*pWal, int64_t fileFirstVer, char* buf) { - return sprintf(buf, "%s/%" PRId64 "." WAL_INDEX_SUFFIX, pWal->path, fileFirstVer); + return sprintf(buf, "%s/%020" PRId64 "." WAL_INDEX_SUFFIX, pWal->path, fileFirstVer); } static inline int walValidHeadCksum(SWalHead* pHead) { @@ -90,7 +92,7 @@ static inline int walValidHeadCksum(SWalHead* pHead) { } static inline int walValidBodyCksum(SWalHead* pHead) { - return taosCheckChecksum((uint8_t*)pHead->head.cont, pHead->head.len, pHead->cksumBody); + return taosCheckChecksum((uint8_t*)pHead->head.body, pHead->head.len, pHead->cksumBody); } static inline int walValidCksum(SWalHead *pHead, void* body, int64_t bodyLen) { diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index 89588b1ddf..49f4fde3a0 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -25,15 +25,15 @@ #include int64_t walGetFirstVer(SWal *pWal) { - return pWal->firstVersion; + return pWal->vers.firstVer; } int64_t walGetSnaphostVer(SWal *pWal) { - return pWal->snapshotVersion; + return pWal->vers.snapshotVer; } int64_t walGetLastVer(SWal *pWal) { - return pWal->lastVersion; + return pWal->vers.lastVer; } int walRollFileInfo(SWal* pWal) { @@ -42,26 +42,28 @@ int walRollFileInfo(SWal* pWal) { SArray* pArray = pWal->fileInfoSet; if(taosArrayGetSize(pArray) != 0) { WalFileInfo *pInfo = taosArrayGetLast(pArray); - pInfo->lastVer = pWal->lastVersion; + pInfo->lastVer = pWal->vers.lastVer; pInfo->closeTs = ts; } + //TODO: change to emplace back WalFileInfo *pNewInfo = malloc(sizeof(WalFileInfo)); if(pNewInfo == NULL) { return -1; } - pNewInfo->firstVer = pWal->lastVersion + 1; + pNewInfo->firstVer = pWal->vers.lastVer + 1; pNewInfo->lastVer = -1; pNewInfo->createTs = ts; pNewInfo->closeTs = -1; pNewInfo->fileSize = 0; taosArrayPush(pWal->fileInfoSet, pNewInfo); + free(pNewInfo); return 0; } char* walMetaSerialize(SWal* pWal) { char buf[30]; - if(pWal == NULL || pWal->fileInfoSet == NULL) return 0; + ASSERT(pWal->fileInfoSet); int sz = pWal->fileInfoSet->size; cJSON* pRoot = cJSON_CreateObject(); cJSON* pMeta = cJSON_CreateObject(); @@ -72,13 +74,13 @@ char* walMetaSerialize(SWal* pWal) { return NULL; } cJSON_AddItemToObject(pRoot, "meta", pMeta); - sprintf(buf, "%" PRId64, pWal->firstVersion); + sprintf(buf, "%" PRId64, pWal->vers.firstVer); cJSON_AddStringToObject(pMeta, "firstVer", buf); - sprintf(buf, "%" PRId64, pWal->snapshotVersion); + sprintf(buf, "%" PRId64, pWal->vers.snapshotVer); cJSON_AddStringToObject(pMeta, "snapshotVer", buf); - sprintf(buf, "%" PRId64, pWal->commitVersion); + sprintf(buf, "%" PRId64, pWal->vers.commitVer); cJSON_AddStringToObject(pMeta, "commitVer", buf); - sprintf(buf, "%" PRId64, pWal->lastVersion); + sprintf(buf, "%" PRId64, pWal->vers.lastVer); cJSON_AddStringToObject(pMeta, "lastVer", buf); cJSON_AddItemToObject(pRoot, "files", pFiles); @@ -103,7 +105,9 @@ char* walMetaSerialize(SWal* pWal) { sprintf(buf, "%" PRId64, pInfo->fileSize); cJSON_AddStringToObject(pField, "fileSize", buf); } - return cJSON_Print(pRoot); + char* serialized = cJSON_Print(pRoot); + cJSON_Delete(pRoot); + return serialized; } int walMetaDeserialize(SWal* pWal, const char* bytes) { @@ -112,18 +116,19 @@ int walMetaDeserialize(SWal* pWal, const char* bytes) { pRoot = cJSON_Parse(bytes); pMeta = cJSON_GetObjectItem(pRoot, "meta"); pField = cJSON_GetObjectItem(pMeta, "firstVer"); - pWal->firstVersion = atoll(cJSON_GetStringValue(pField)); + pWal->vers.firstVer = atoll(cJSON_GetStringValue(pField)); pField = cJSON_GetObjectItem(pMeta, "snapshotVer"); - pWal->snapshotVersion = atoll(cJSON_GetStringValue(pField)); + pWal->vers.snapshotVer = atoll(cJSON_GetStringValue(pField)); pField = cJSON_GetObjectItem(pMeta, "commitVer"); - pWal->commitVersion = atoll(cJSON_GetStringValue(pField)); + pWal->vers.commitVer = atoll(cJSON_GetStringValue(pField)); pField = cJSON_GetObjectItem(pMeta, "lastVer"); - pWal->lastVersion = atoll(cJSON_GetStringValue(pField)); + pWal->vers.lastVer = atoll(cJSON_GetStringValue(pField)); pFiles = cJSON_GetObjectItem(pRoot, "files"); int sz = cJSON_GetArraySize(pFiles); //deserialize - SArray* pArray = taosArrayInit(sz, sizeof(WalFileInfo)); + SArray* pArray = pWal->fileInfoSet; + taosArrayEnsureCap(pArray, sz); WalFileInfo *pData = pArray->pData; for(int i = 0; i < sz; i++) { cJSON* pInfoJson = cJSON_GetArrayItem(pFiles, i); @@ -141,6 +146,7 @@ int walMetaDeserialize(SWal* pWal, const char* bytes) { } taosArraySetSize(pArray, sz); pWal->fileInfoSet = pArray; + cJSON_Delete(pRoot); return 0; } @@ -155,7 +161,7 @@ static int walFindCurMetaVer(SWal* pWal) { DIR *dir = opendir(pWal->path); if(dir == NULL) { - wError("vgId:%d, path:%s, failed to open since %s", pWal->vgId, pWal->path, strerror(errno)); + wError("vgId:%d, path:%s, failed to open since %s", pWal->cfg.vgId, pWal->path, strerror(errno)); return -1; } @@ -171,6 +177,8 @@ static int walFindCurMetaVer(SWal* pWal) { break; } } + closedir(dir); + regfree(&walMetaRegexPattern); return metaVer; } @@ -195,6 +203,7 @@ int walWriteMeta(SWal* pWal) { walBuildMetaName(pWal, metaVer, fnameStr); remove(fnameStr); } + free(serialized); return 0; } @@ -215,6 +224,7 @@ int walReadMeta(SWal* pWal) { if(buf == NULL) { return -1; } + memset(buf, 0, size+5); int tfd = tfOpenRead(fnameStr); if(tfRead(tfd, buf, size) != size) { free(buf); diff --git a/source/libs/wal/src/walMgmt.c b/source/libs/wal/src/walMgmt.c index d18652233a..629451a722 100644 --- a/source/libs/wal/src/walMgmt.c +++ b/source/libs/wal/src/walMgmt.c @@ -68,9 +68,12 @@ int32_t walInit() { } void walCleanUp() { + int old = atomic_val_compare_exchange_8(&tsWal.inited, 1, 0); + if(old == 0) { + return; + } walStopThread(); taosCloseRef(tsWal.refSetId); - atomic_store_8(&tsWal.inited, 0); wInfo("wal module is cleaned up"); } @@ -80,31 +83,30 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { terrno = TAOS_SYSTEM_ERROR(errno); return NULL; } + memset(pWal, 0, sizeof(SWal)); pWal->writeLogTfd = -1; pWal->writeIdxTfd = -1; pWal->writeCur = -1; //set config - pWal->vgId = pCfg->vgId; - pWal->fsyncPeriod = pCfg->fsyncPeriod; - pWal->rollPeriod = pCfg->rollPeriod; - pWal->segSize = pCfg->segSize; - pWal->level = pCfg->walLevel; + memcpy(&pWal->cfg, pCfg, sizeof(SWalCfg)); //init version info - pWal->firstVersion = -1; - pWal->commitVersion = -1; - pWal->snapshotVersion = -1; - pWal->lastVersion = -1; + pWal->vers.firstVer = -1; + pWal->vers.commitVer = -1; + pWal->vers.snapshotVer = -1; + pWal->vers.lastVer = -1; - pWal->snapshottingVer = -1; + pWal->vers.verInSnapshotting = -1; + + pWal->totSize = 0; //init status pWal->lastRollSeq = -1; //init write buffer - memset(&pWal->head, 0, sizeof(SWalHead)); - pWal->head.head.sver = 0; + memset(&pWal->writeHead, 0, sizeof(SWalHead)); + pWal->writeHead.head.sver = 0; tstrncpy(pWal->path, path, sizeof(pWal->path)); pthread_mutex_init(&pWal->mutex, NULL); @@ -122,8 +124,9 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { walFreeObj(pWal); return NULL; } + walReadMeta(pWal); - wDebug("vgId:%d, wal:%p is opened, level:%d fsyncPeriod:%d", pWal->vgId, pWal, pWal->level, pWal->fsyncPeriod); + wDebug("vgId:%d, wal:%p is opened, level:%d fsyncPeriod:%d", pWal->cfg.vgId, pWal, pWal->cfg.level, pWal->cfg.fsyncPeriod); return pWal; } @@ -131,17 +134,17 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { int32_t walAlter(SWal *pWal, SWalCfg *pCfg) { if (pWal == NULL) return TSDB_CODE_WAL_APP_ERROR; - if (pWal->level == pCfg->walLevel && pWal->fsyncPeriod == pCfg->fsyncPeriod) { - wDebug("vgId:%d, old walLevel:%d fsync:%d, new walLevel:%d fsync:%d not change", pWal->vgId, pWal->level, - pWal->fsyncPeriod, pCfg->walLevel, pCfg->fsyncPeriod); + if (pWal->cfg.level == pCfg->level && pWal->cfg.fsyncPeriod == pCfg->fsyncPeriod) { + wDebug("vgId:%d, old walLevel:%d fsync:%d, new walLevel:%d fsync:%d not change", pWal->cfg.vgId, pWal->cfg.level, + pWal->cfg.fsyncPeriod, pCfg->level, pCfg->fsyncPeriod); return 0; } - wInfo("vgId:%d, change old walLevel:%d fsync:%d, new walLevel:%d fsync:%d", pWal->vgId, pWal->level, - pWal->fsyncPeriod, pCfg->walLevel, pCfg->fsyncPeriod); + wInfo("vgId:%d, change old walLevel:%d fsync:%d, new walLevel:%d fsync:%d", pWal->cfg.vgId, pWal->cfg.level, + pWal->cfg.fsyncPeriod, pCfg->level, pCfg->fsyncPeriod); - pWal->level = pCfg->walLevel; - pWal->fsyncPeriod = pCfg->fsyncPeriod; + pWal->cfg.level = pCfg->level; + pWal->cfg.fsyncPeriod = pCfg->fsyncPeriod; pWal->fsyncSeq = pCfg->fsyncPeriod / 1000; if (pWal->fsyncSeq <= 0) pWal->fsyncSeq = 1; @@ -153,31 +156,34 @@ void walClose(SWal *pWal) { pthread_mutex_lock(&pWal->mutex); tfClose(pWal->writeLogTfd); + pWal->writeLogTfd = -1; tfClose(pWal->writeIdxTfd); - /*taosArrayDestroy(pWal->fileInfoSet);*/ - /*pWal->fileInfoSet = NULL;*/ + pWal->writeIdxTfd = -1; + walWriteMeta(pWal); + taosArrayDestroy(pWal->fileInfoSet); + pWal->fileInfoSet = NULL; pthread_mutex_unlock(&pWal->mutex); taosRemoveRef(tsWal.refSetId, pWal->refId); } static int32_t walInitObj(SWal *pWal) { if (taosMkDir(pWal->path) != 0) { - wError("vgId:%d, path:%s, failed to create directory since %s", pWal->vgId, pWal->path, strerror(errno)); + wError("vgId:%d, path:%s, failed to create directory since %s", pWal->cfg.vgId, pWal->path, strerror(errno)); return TAOS_SYSTEM_ERROR(errno); } - pWal->fileInfoSet = taosArrayInit(0, sizeof(WalFileInfo)); + pWal->fileInfoSet = taosArrayInit(8, sizeof(WalFileInfo)); if(pWal->fileInfoSet == NULL) { - wError("vgId:%d, path:%s, failed to init taosArray %s", pWal->vgId, pWal->path, strerror(errno)); + wError("vgId:%d, path:%s, failed to init taosArray %s", pWal->cfg.vgId, pWal->path, strerror(errno)); return TAOS_SYSTEM_ERROR(errno); } - wDebug("vgId:%d, object is initialized", pWal->vgId); + wDebug("vgId:%d, object is initialized", pWal->cfg.vgId); return 0; } static void walFreeObj(void *wal) { SWal *pWal = wal; - wDebug("vgId:%d, wal:%p is freed", pWal->vgId, pWal); + wDebug("vgId:%d, wal:%p is freed", pWal->cfg.vgId, pWal); tfClose(pWal->writeLogTfd); tfClose(pWal->writeIdxTfd); @@ -188,7 +194,7 @@ static void walFreeObj(void *wal) { } static bool walNeedFsync(SWal *pWal) { - if (pWal->fsyncPeriod <= 0 || pWal->level != TAOS_WAL_FSYNC) { + if (pWal->cfg.fsyncPeriod <= 0 || pWal->cfg.level != TAOS_WAL_FSYNC) { return false; } @@ -208,10 +214,10 @@ static void walFsyncAll() { SWal *pWal = taosIterateRef(tsWal.refSetId, 0); while (pWal) { if (walNeedFsync(pWal)) { - wTrace("vgId:%d, do fsync, level:%d seq:%d rseq:%d", pWal->vgId, pWal->level, pWal->fsyncSeq, atomic_load_32(&tsWal.seq)); + wTrace("vgId:%d, do fsync, level:%d seq:%d rseq:%d", pWal->cfg.vgId, pWal->cfg.level, pWal->fsyncSeq, atomic_load_32(&tsWal.seq)); int32_t code = tfFsync(pWal->writeLogTfd); if (code != 0) { - wError("vgId:%d, file:%"PRId64".log, failed to fsync since %s", pWal->vgId, walGetLastFileFirstVer(pWal), strerror(code)); + wError("vgId:%d, file:%"PRId64".log, failed to fsync since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal), strerror(code)); } } pWal = taosIterateRef(tsWal.refSetId, pWal->refId); @@ -249,9 +255,8 @@ static int32_t walCreateThread() { static void walStopThread() { atomic_store_8(&tsWal.stop, 1); - if (tsWal.thread != NULL && taosCheckPthreadValid(tsWal.thread)) { + if (taosCheckPthreadValid(tsWal.thread)) { pthread_join(tsWal.thread, NULL); - tsWal.thread = NULL; } wDebug("wal thread is stopped"); diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index 27dffddf83..b6aafedea3 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -16,6 +16,163 @@ #include "walInt.h" #include "tfile.h" +SWalReadHandle* walOpenReadHandle(SWal* pWal) { + SWalReadHandle *pRead = malloc(sizeof(SWalReadHandle)); + if(pRead == NULL) { + return NULL; + } + pRead->pWal = pWal; + pRead->readIdxTfd = -1; + pRead->readLogTfd = -1; + pRead->curVersion = -1; + pRead->curFileFirstVer = -1; + pRead->capacity = 0; + pRead->status = 0; + pRead->pHead = malloc(sizeof(SWalHead)); + if(pRead->pHead == NULL) { + free(pRead); + return NULL; + } + return pRead; +} + +void walCloseReadHandle(SWalReadHandle *pRead) { + tfClose(pRead->readIdxTfd); + tfClose(pRead->readLogTfd); + tfree(pRead->pHead); + free(pRead); +} + +int32_t walRegisterRead(SWalReadHandle *pRead, int64_t ver) { + return 0; +} + +static int32_t walReadSeekFilePos(SWalReadHandle *pRead, int64_t fileFirstVer, int64_t ver) { + int code = 0; + + int64_t idxTfd = pRead->readIdxTfd; + int64_t logTfd = pRead->readLogTfd; + + //seek position + int64_t offset = (ver - fileFirstVer) * WAL_IDX_ENTRY_SIZE; + code = tfLseek(idxTfd, offset, SEEK_SET); + if(code < 0) { + return -1; + } + WalIdxEntry entry; + if(tfRead(idxTfd, &entry, sizeof(WalIdxEntry)) != sizeof(WalIdxEntry)) { + return -1; + } + //TODO:deserialize + ASSERT(entry.ver == ver); + code = tfLseek(logTfd, entry.offset, SEEK_SET); + if (code < 0) { + return -1; + } + return code; +} + +static int32_t walReadChangeFile(SWalReadHandle *pRead, int64_t fileFirstVer) { + char fnameStr[WAL_FILE_LEN]; + + tfClose(pRead->readIdxTfd); + tfClose(pRead->readLogTfd); + + walBuildLogName(pRead->pWal, fileFirstVer, fnameStr); + int64_t logTfd = tfOpenRead(fnameStr); + if(logTfd < 0) { + return -1; + } + + walBuildIdxName(pRead->pWal, fileFirstVer, fnameStr); + int64_t idxTfd = tfOpenRead(fnameStr); + if(idxTfd < 0) { + return -1; + } + + pRead->readLogTfd = logTfd; + pRead->readIdxTfd = idxTfd; + return 0; +} + +static int32_t walReadSeekVer(SWalReadHandle *pRead, int64_t ver) { + int code; + SWal *pWal = pRead->pWal; + if(ver == pRead->curVersion) { + return 0; + } + if(ver > pWal->vers.lastVer || ver < pWal->vers.firstVer) { + return -1; + } + if(ver < pWal->vers.snapshotVer) { + + } + + WalFileInfo tmpInfo; + tmpInfo.firstVer = ver; + //bsearch in fileSet + WalFileInfo* pRet = taosArraySearch(pWal->fileInfoSet, &tmpInfo, compareWalFileInfo, TD_LE); + ASSERT(pRet != NULL); + if(pRead->curFileFirstVer != pRet->firstVer) { + code = walReadChangeFile(pRead, pRet->firstVer); + if(code < 0) { + //TODO: set error flag + return -1; + } + } + + code = walReadSeekFilePos(pRead, pRet->firstVer, ver); + if(code < 0) { + return -1; + } + pRead->curVersion = ver; + + return 0; +} + +int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver) { + int code; + //TODO: check wal life + if(pRead->curVersion != ver) { + code = walReadSeekVer(pRead, ver); + if(code != 0) { + return -1; + } + } + + if(!tfValid(pRead->readLogTfd)) return -1; + + code = tfRead(pRead->readLogTfd, pRead->pHead, sizeof(SWalHead)); + if(code != sizeof(SWalHead)) { + return -1; + } + code = walValidHeadCksum(pRead->pHead); + if(code != 0) { + return -1; + } + if(pRead->capacity < pRead->pHead->head.len) { + void* ptr = realloc(pRead->pHead, sizeof(SWalHead) + pRead->pHead->head.len); + if(ptr == NULL) { + return -1; + } + pRead->pHead = ptr; + pRead->capacity = pRead->pHead->head.len; + } + if(pRead->pHead->head.len != tfRead(pRead->readLogTfd, pRead->pHead->head.body, pRead->pHead->head.len)) { + return -1; + } + + /*code = walValidBodyCksum(pRead->pHead);*/ + ASSERT(pRead->pHead->head.version == ver); + + if(code != 0) { + return -1; + } + pRead->curVersion++; + + return 0; +} + int32_t walRead(SWal *pWal, SWalHead **ppHead, int64_t ver) { int code; code = walSeekVer(pWal, ver); @@ -42,7 +199,7 @@ int32_t walRead(SWal *pWal, SWalHead **ppHead, int64_t ver) { *ppHead = NULL; return -1; } - if(tfRead(pWal->writeLogTfd, (*ppHead)->head.cont, (*ppHead)->head.len) != (*ppHead)->head.len) { + if(tfRead(pWal->writeLogTfd, (*ppHead)->head.body, (*ppHead)->head.len) != (*ppHead)->head.len) { return -1; } //TODO: endian compatibility processing after read diff --git a/source/libs/wal/src/walSeek.c b/source/libs/wal/src/walSeek.c index 48272f8f32..953aae703c 100644 --- a/source/libs/wal/src/walSeek.c +++ b/source/libs/wal/src/walSeek.c @@ -78,10 +78,12 @@ int walChangeFile(SWal *pWal, int64_t ver) { code = tfClose(pWal->writeLogTfd); if(code != 0) { //TODO + return -1; } code = tfClose(pWal->writeIdxTfd); if(code != 0) { //TODO + return -1; } WalFileInfo tmpInfo; tmpInfo.firstVer = ver; @@ -106,24 +108,19 @@ int walChangeFile(SWal *pWal, int64_t ver) { pWal->writeLogTfd = logTfd; pWal->writeIdxTfd = idxTfd; - return code; -} - -int walGetVerOffset(SWal* pWal, int64_t ver) { - int code; - return 0; + return fileFirstVer; } int walSeekVer(SWal *pWal, int64_t ver) { int code; - if(ver == pWal->lastVersion) { + if(ver == pWal->vers.lastVer) { return 0; } - if(ver > pWal->lastVersion || ver < pWal->firstVersion) { + if(ver > pWal->vers.lastVer|| ver < pWal->vers.firstVer) { return -1; } - if(ver < pWal->snapshotVersion) { - //TODO: set flag to prevent roll back + if(ver < pWal->vers.snapshotVer) { + } if(ver < walGetCurFileFirstVer(pWal) || (ver > walGetCurFileLastVer(pWal))) { code = walChangeFile(pWal, ver); diff --git a/source/libs/wal/src/walUtil.c b/source/libs/wal/src/walUtil.c index c88cc918fe..849d0c3e51 100644 --- a/source/libs/wal/src/walUtil.c +++ b/source/libs/wal/src/walUtil.c @@ -17,6 +17,7 @@ #include "os.h" #include "walInt.h" +#if 0 int32_t walGetNextFile(SWal *pWal, int64_t *nextFileId) { int64_t curFileId = *nextFileId; int64_t minFileId = INT64_MAX; @@ -116,3 +117,4 @@ int32_t walGetNewFile(SWal *pWal, int64_t *newFileId) { return 0; } +#endif diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index 87d52ad236..994b8fc333 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -114,22 +114,22 @@ void walRemoveAllOldFiles(void *handle) { #endif int32_t walCommit(SWal *pWal, int64_t ver) { - ASSERT(pWal->commitVersion >= pWal->snapshotVersion); - ASSERT(pWal->commitVersion <= pWal->lastVersion); - if(ver < pWal->commitVersion || ver > pWal->lastVersion) { + ASSERT(pWal->vers.commitVer >= pWal->vers.snapshotVer); + ASSERT(pWal->vers.commitVer <= pWal->vers.lastVer); + if(ver < pWal->vers.commitVer || ver > pWal->vers.lastVer) { return -1; } - pWal->commitVersion = ver; + pWal->vers.commitVer = ver; return 0; } int32_t walRollback(SWal *pWal, int64_t ver) { int code; char fnameStr[WAL_FILE_LEN]; - if(ver == pWal->lastVersion) { + if(ver == pWal->vers.lastVer) { return 0; } - if(ver > pWal->lastVersion || ver < pWal->commitVersion) { + if(ver > pWal->vers.lastVer || ver < pWal->vers.commitVer) { return -1; } pthread_mutex_lock(&pWal->mutex); @@ -220,7 +220,7 @@ int32_t walRollback(SWal *pWal, int64_t ver) { if(code < 0) { return -1; } - pWal->lastVersion = ver - 1; + pWal->vers.lastVer = ver - 1; ((WalFileInfo*)taosArrayGetLast(pWal->fileInfoSet))->lastVer = ver - 1; ((WalFileInfo*)taosArrayGetLast(pWal->fileInfoSet))->fileSize = entry.offset; @@ -228,10 +228,11 @@ int32_t walRollback(SWal *pWal, int64_t ver) { pthread_mutex_unlock(&pWal->mutex); return 0; } + int32_t walBeginTakeSnapshot(SWal* pWal, int64_t ver) { - pWal->snapshottingVer = ver; + pWal->vers.verInSnapshotting = ver; //check file rolling - if(pWal->retentionPeriod == 0) { + if(pWal->cfg.retentionPeriod == 0) { walRoll(pWal); } @@ -239,10 +240,10 @@ int32_t walBeginTakeSnapshot(SWal* pWal, int64_t ver) { } int32_t walEndTakeSnapshot(SWal *pWal) { - int64_t ver = pWal->snapshottingVer; + int64_t ver = pWal->vers.verInSnapshotting; if(ver == -1) return -1; - pWal->snapshotVersion = ver; + pWal->vers.snapshotVer = ver; int ts = taosGetTimestampSec(); int deleteCnt = 0; @@ -256,8 +257,8 @@ int32_t walEndTakeSnapshot(SWal *pWal) { } //iterate files, until the searched result for(WalFileInfo* iter = pWal->fileInfoSet->pData; iter < pInfo; iter++) { - if(pWal->totSize > pWal->retentionSize || - iter->closeTs + pWal->retentionPeriod > ts) { + if(pWal->totSize > pWal->cfg.retentionSize || + iter->closeTs + pWal->cfg.retentionPeriod > ts) { //delete according to file size or close time deleteCnt++; newTotSize -= iter->fileSize; @@ -276,12 +277,14 @@ int32_t walEndTakeSnapshot(SWal *pWal) { //make new array, remove files taosArrayPopFrontBatch(pWal->fileInfoSet, deleteCnt); if(taosArrayGetSize(pWal->fileInfoSet) == 0) { - pWal->firstVersion = -1; + pWal->writeCur = -1; + pWal->vers.firstVer = -1; } else { - pWal->firstVersion = ((WalFileInfo*)taosArrayGet(pWal->fileInfoSet, 0))->firstVer; + pWal->vers.firstVer = ((WalFileInfo*)taosArrayGet(pWal->fileInfoSet, 0))->firstVer; } + pWal->writeCur = taosArrayGetSize(pWal->fileInfoSet) - 1;; pWal->totSize = newTotSize; - pWal->snapshottingVer = -1; + pWal->vers.verInSnapshotting = -1; //save snapshot ver, commit ver int code = walWriteMeta(pWal); @@ -308,7 +311,7 @@ int walRoll(SWal *pWal) { } int64_t idxTfd, logTfd; //create new file - int64_t newFileFirstVersion = pWal->lastVersion + 1; + int64_t newFileFirstVersion = pWal->vers.lastVer + 1; char fnameStr[WAL_FILE_LEN]; walBuildIdxName(pWal, newFileFirstVersion, fnameStr); idxTfd = tfOpenCreateWrite(fnameStr); @@ -340,19 +343,10 @@ int walRoll(SWal *pWal) { } static int walWriteIndex(SWal *pWal, int64_t ver, int64_t offset) { - int code = 0; - //get index file - if(!tfValid(pWal->writeIdxTfd)) { - code = TAOS_SYSTEM_ERROR(errno); - - wError("vgId:%d, file:%"PRId64".idx, failed to open since %s", pWal->vgId, walGetLastFileFirstVer(pWal), strerror(errno)); - return code; - } - char fnameStr[WAL_FILE_LEN]; - walBuildIdxName(pWal, walGetCurFileFirstVer(pWal), fnameStr); WalIdxEntry entry = { .ver = ver, .offset = offset }; int size = tfWrite(pWal->writeIdxTfd, &entry, sizeof(WalIdxEntry)); if(size != sizeof(WalIdxEntry)) { + //TODO truncate return -1; } return 0; @@ -363,18 +357,18 @@ int64_t walWrite(SWal *pWal, int64_t index, uint8_t msgType, const void *body, i int code = 0; // no wal - if (pWal->level == TAOS_WAL_NOLOG) return 0; + if (pWal->cfg.level == TAOS_WAL_NOLOG) return 0; - if (index == pWal->lastVersion + 1) { + if (index == pWal->vers.lastVer + 1) { if(taosArrayGetSize(pWal->fileInfoSet) == 0) { - pWal->firstVersion = index; + pWal->vers.firstVer = index; code = walRoll(pWal); ASSERT(code == 0); } else { int64_t passed = walGetSeq() - pWal->lastRollSeq; - if(pWal->rollPeriod != -1 && pWal->rollPeriod != 0 && passed > pWal->rollPeriod) { + if(pWal->cfg.rollPeriod != -1 && pWal->cfg.rollPeriod != 0 && passed > pWal->cfg.rollPeriod) { walRoll(pWal); - } else if(pWal->segSize != -1 && pWal->segSize != 0 && walGetLastFileSize(pWal) > pWal->segSize) { + } else if(pWal->cfg.segSize != -1 && pWal->cfg.segSize != 0 && walGetLastFileSize(pWal) > pWal->cfg.segSize) { walRoll(pWal); } } @@ -383,35 +377,36 @@ int64_t walWrite(SWal *pWal, int64_t index, uint8_t msgType, const void *body, i //must truncate explicitly first return -1; } - /*if (!tfValid(pWal->curLogTfd)) return 0;*/ + /*if (!tfValid(pWal->writeLogTfd)) return -1;*/ pthread_mutex_lock(&pWal->mutex); - pWal->head.head.version = index; + pWal->writeHead.head.version = index; - pWal->head.head.len = bodyLen; - pWal->head.head.msgType = msgType; - pWal->head.cksumHead = walCalcHeadCksum(&pWal->head); - pWal->head.cksumBody = walCalcBodyCksum(body, bodyLen); + int64_t offset = walGetCurFileOffset(pWal); + pWal->writeHead.head.len = bodyLen; + pWal->writeHead.head.msgType = msgType; + pWal->writeHead.cksumHead = walCalcHeadCksum(&pWal->writeHead); + pWal->writeHead.cksumBody = walCalcBodyCksum(body, bodyLen); - if (tfWrite(pWal->writeLogTfd, &pWal->head, sizeof(SWalHead)) != sizeof(SWalHead)) { + if (tfWrite(pWal->writeLogTfd, &pWal->writeHead, sizeof(SWalHead)) != sizeof(SWalHead)) { //ftruncate code = TAOS_SYSTEM_ERROR(errno); - wError("vgId:%d, file:%"PRId64".log, failed to write since %s", pWal->vgId, walGetLastFileFirstVer(pWal), strerror(errno)); + wError("vgId:%d, file:%"PRId64".log, failed to write since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal), strerror(errno)); } - if (tfWrite(pWal->writeLogTfd, &body, bodyLen) != bodyLen) { + if (tfWrite(pWal->writeLogTfd, (char*)body, bodyLen) != bodyLen) { //ftruncate code = TAOS_SYSTEM_ERROR(errno); - wError("vgId:%d, file:%"PRId64".log, failed to write since %s", pWal->vgId, walGetLastFileFirstVer(pWal), strerror(errno)); + wError("vgId:%d, file:%"PRId64".log, failed to write since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal), strerror(errno)); } - code = walWriteIndex(pWal, index, walGetCurFileOffset(pWal)); + code = walWriteIndex(pWal, index, offset); if(code != 0) { //TODO return -1; } //set status - pWal->lastVersion = index; + pWal->vers.lastVer = index; pWal->totSize += sizeof(SWalHead) + bodyLen; walGetCurFileInfo(pWal)->lastVer = index; walGetCurFileInfo(pWal)->fileSize += sizeof(SWalHead) + bodyLen; @@ -422,10 +417,10 @@ int64_t walWrite(SWal *pWal, int64_t index, uint8_t msgType, const void *body, i } void walFsync(SWal *pWal, bool forceFsync) { - if (forceFsync || (pWal->level == TAOS_WAL_FSYNC && pWal->fsyncPeriod == 0)) { - wTrace("vgId:%d, fileId:%"PRId64".log, do fsync", pWal->vgId, walGetCurFileFirstVer(pWal)); + if (forceFsync || (pWal->cfg.level == TAOS_WAL_FSYNC && pWal->cfg.fsyncPeriod == 0)) { + wTrace("vgId:%d, fileId:%"PRId64".log, do fsync", pWal->cfg.vgId, walGetCurFileFirstVer(pWal)); if (tfFsync(pWal->writeLogTfd) < 0) { - wError("vgId:%d, file:%"PRId64".log, fsync failed since %s", pWal->vgId, walGetCurFileFirstVer(pWal), strerror(errno)); + wError("vgId:%d, file:%"PRId64".log, fsync failed since %s", pWal->cfg.vgId, walGetCurFileFirstVer(pWal), strerror(errno)); } } } @@ -498,29 +493,29 @@ int32_t walGetWalFile(void *handle, char *fileName, int64_t *fileId) { } #endif -static int walValidateOffset(SWal* pWal, int64_t ver) { - int code = 0; - SWalHead *pHead = NULL; - code = (int)walRead(pWal, &pHead, ver); - if(pHead->head.version != ver) { - return -1; - } - return 0; -} +/*static int walValidateOffset(SWal* pWal, int64_t ver) {*/ + /*int code = 0;*/ + /*SWalHead *pHead = NULL;*/ + /*code = (int)walRead(pWal, &pHead, ver);*/ + /*if(pHead->head.version != ver) {*/ + /*return -1;*/ + /*}*/ + /*return 0;*/ +/*}*/ -static int64_t walGetOffset(SWal* pWal, int64_t ver) { - int code = walSeekVer(pWal, ver); - if(code != 0) { - return -1; - } +/*static int64_t walGetOffset(SWal* pWal, int64_t ver) {*/ + /*int code = walSeekVer(pWal, ver);*/ + /*if(code != 0) {*/ + /*return -1;*/ + /*}*/ - code = walValidateOffset(pWal, ver); - if(code != 0) { - return -1; - } + /*code = walValidateOffset(pWal, ver);*/ + /*if(code != 0) {*/ + /*return -1;*/ + /*}*/ - return 0; -} + /*return 0;*/ +/*}*/ #if 0 static int32_t walSkipCorruptedRecord(SWal *pWal, SWalHead *pHead, int64_t tfd, int64_t *offset) { diff --git a/source/libs/wal/test/walMetaTest.cpp b/source/libs/wal/test/walMetaTest.cpp index f244917f77..200bf39c5a 100644 --- a/source/libs/wal/test/walMetaTest.cpp +++ b/source/libs/wal/test/walMetaTest.cpp @@ -5,6 +5,9 @@ #include "walInt.h" +const char* ranStr = "tvapq02tcp"; +const int ranStrLen = strlen(ranStr); + class WalCleanEnv : public ::testing::Test { protected: static void SetUpTestCase() { @@ -18,12 +21,15 @@ class WalCleanEnv : public ::testing::Test { void SetUp() override { taosRemoveDir(pathName); - SWalCfg* pCfg = (SWalCfg*)malloc(sizeof(SWal)); + SWalCfg* pCfg = (SWalCfg*)malloc(sizeof(SWalCfg)); memset(pCfg, 0, sizeof(SWalCfg)); pCfg->rollPeriod = -1; pCfg->segSize = -1; - pCfg->walLevel = TAOS_WAL_FSYNC; + pCfg->retentionPeriod = 0; + pCfg->retentionSize = 0; + pCfg->level = TAOS_WAL_FSYNC; pWal = walOpen(pathName, pCfg); + free(pCfg); ASSERT(pWal != NULL); } @@ -49,11 +55,13 @@ class WalCleanDeleteEnv : public ::testing::Test { void SetUp() override { taosRemoveDir(pathName); - SWalCfg* pCfg = (SWalCfg*)malloc(sizeof(SWal)); + SWalCfg* pCfg = (SWalCfg*)malloc(sizeof(SWalCfg)); memset(pCfg, 0, sizeof(SWalCfg)); pCfg->retentionPeriod = 0; - pCfg->walLevel = TAOS_WAL_FSYNC; + pCfg->retentionSize = 0; + pCfg->level = TAOS_WAL_FSYNC; pWal = walOpen(pathName, pCfg); + free(pCfg); ASSERT(pWal != NULL); } @@ -77,13 +85,22 @@ class WalKeepEnv : public ::testing::Test { walCleanUp(); } + void walResetEnv() { + TearDown(); + taosRemoveDir(pathName); + SetUp(); + } + void SetUp() override { - SWalCfg* pCfg = (SWalCfg*)malloc(sizeof(SWal)); + SWalCfg* pCfg = (SWalCfg*)malloc(sizeof(SWalCfg)); memset(pCfg, 0, sizeof(SWalCfg)); pCfg->rollPeriod = -1; pCfg->segSize = -1; - pCfg->walLevel = TAOS_WAL_FSYNC; + pCfg->retentionPeriod = 0; + pCfg->retentionSize = 0; + pCfg->level = TAOS_WAL_FSYNC; pWal = walOpen(pathName, pCfg); + free(pCfg); ASSERT(pWal != NULL); } @@ -124,6 +141,7 @@ TEST_F(WalCleanEnv, serialize) { ASSERT(code == 0); char*ss = walMetaSerialize(pWal); printf("%s\n", ss); + free(ss); code = walWriteMeta(pWal); ASSERT(code == 0); } @@ -140,94 +158,134 @@ TEST_F(WalCleanEnv, removeOldMeta) { ASSERT(code == 0); } -//TEST_F(WalKeepEnv, readOldMeta) { - //int code = walRollFileInfo(pWal); - //ASSERT(code == 0); - //code = walWriteMeta(pWal); - //ASSERT(code == 0); - //code = walRollFileInfo(pWal); - //ASSERT(code == 0); - //code = walWriteMeta(pWal); - //ASSERT(code == 0); - //char*oldss = walMetaSerialize(pWal); +TEST_F(WalKeepEnv, readOldMeta) { + walResetEnv(); + int code; - //TearDown(); - //SetUp(); - //code = walReadMeta(pWal); - //ASSERT(code == 0); - //char* newss = walMetaSerialize(pWal); + for(int i = 0; i < 10; i++) { + code = walWrite(pWal, i, i+1, (void*)ranStr, ranStrLen); + ASSERT_EQ(code, 0); + ASSERT_EQ(pWal->vers.lastVer, i); + code = walWrite(pWal, i+2, i, (void*)ranStr, ranStrLen); + ASSERT_EQ(code, -1); + ASSERT_EQ(pWal->vers.lastVer, i); + } + char* oldss = walMetaSerialize(pWal); - //int len = strlen(oldss); - //ASSERT_EQ(len, strlen(newss)); - //for(int i = 0; i < len; i++) { - //EXPECT_EQ(oldss[i], newss[i]); - //} -//} + TearDown(); + SetUp(); + + ASSERT_EQ(pWal->vers.firstVer, 0); + ASSERT_EQ(pWal->vers.lastVer, 9); + + char* newss = walMetaSerialize(pWal); + + int len = strlen(oldss); + ASSERT_EQ(len, strlen(newss)); + for(int i = 0; i < len; i++) { + EXPECT_EQ(oldss[i], newss[i]); + } + free(oldss); + free(newss); +} TEST_F(WalCleanEnv, write) { - const char* ranStr = "tvapq02tcp"; - const int len = strlen(ranStr); int code; for(int i = 0; i < 10; i++) { - code = walWrite(pWal, i, i+1, (void*)ranStr, len); + code = walWrite(pWal, i, i+1, (void*)ranStr, ranStrLen); ASSERT_EQ(code, 0); - ASSERT_EQ(pWal->lastVersion, i); - code = walWrite(pWal, i+2, i, (void*)ranStr, len); + ASSERT_EQ(pWal->vers.lastVer, i); + code = walWrite(pWal, i+2, i, (void*)ranStr, ranStrLen); ASSERT_EQ(code, -1); - ASSERT_EQ(pWal->lastVersion, i); + ASSERT_EQ(pWal->vers.lastVer, i); } code = walWriteMeta(pWal); ASSERT_EQ(code, 0); } TEST_F(WalCleanEnv, rollback) { - const char* ranStr = "tvapq02tcp"; - const int len = strlen(ranStr); int code; for(int i = 0; i < 10; i++) { - code = walWrite(pWal, i, i+1, (void*)ranStr, len); + code = walWrite(pWal, i, i+1, (void*)ranStr, ranStrLen); ASSERT_EQ(code, 0); - ASSERT_EQ(pWal->lastVersion, i); + ASSERT_EQ(pWal->vers.lastVer, i); } code = walRollback(pWal, 5); ASSERT_EQ(code, 0); - ASSERT_EQ(pWal->lastVersion, 4); + ASSERT_EQ(pWal->vers.lastVer, 4); code = walRollback(pWal, 3); ASSERT_EQ(code, 0); - ASSERT_EQ(pWal->lastVersion, 2); + ASSERT_EQ(pWal->vers.lastVer, 2); code = walWriteMeta(pWal); ASSERT_EQ(code, 0); } TEST_F(WalCleanDeleteEnv, roll) { - const char* ranStr = "tvapq02tcp"; - const int len = strlen(ranStr); int code; int i; for(i = 0; i < 100; i++) { - code = walWrite(pWal, i, 0, (void*)ranStr, len); + code = walWrite(pWal, i, 0, (void*)ranStr, ranStrLen); ASSERT_EQ(code, 0); - ASSERT_EQ(pWal->lastVersion, i); + ASSERT_EQ(pWal->vers.lastVer, i); code = walCommit(pWal, i); - ASSERT_EQ(pWal->commitVersion, i); + ASSERT_EQ(pWal->vers.commitVer, i); } walBeginTakeSnapshot(pWal, i-1); - ASSERT_EQ(pWal->snapshottingVer, i-1); + ASSERT_EQ(pWal->vers.verInSnapshotting, i-1); walEndTakeSnapshot(pWal); - ASSERT_EQ(pWal->snapshotVersion, i-1); - ASSERT_EQ(pWal->snapshottingVer, -1); + ASSERT_EQ(pWal->vers.snapshotVer, i-1); + ASSERT_EQ(pWal->vers.verInSnapshotting, -1); - code = walWrite(pWal, 5, 0, (void*)ranStr, len); + code = walWrite(pWal, 5, 0, (void*)ranStr, ranStrLen); ASSERT_NE(code, 0); for(; i < 200; i++) { - code = walWrite(pWal, i, 0, (void*)ranStr, len); + code = walWrite(pWal, i, 0, (void*)ranStr, ranStrLen); ASSERT_EQ(code, 0); code = walCommit(pWal, i); - ASSERT_EQ(pWal->commitVersion, i); + ASSERT_EQ(pWal->vers.commitVer, i); } - code = walWriteMeta(pWal); + code = walBeginTakeSnapshot(pWal, i - 1); + ASSERT_EQ(code, 0); + code = walEndTakeSnapshot(pWal); ASSERT_EQ(code, 0); } + +TEST_F(WalKeepEnv, readHandleRead) { + walResetEnv(); + int code; + SWalReadHandle* pRead = walOpenReadHandle(pWal); + ASSERT(pRead != NULL); + + int i ; + for(i = 0; i < 100; i++) { + char newStr[100]; + sprintf(newStr, "%s-%d", ranStr, i); + int len = strlen(newStr); + code = walWrite(pWal, i, 0, newStr, len); + ASSERT_EQ(code, 0); + } + for(int i = 0; i < 1000; i++) { + int ver = rand() % 100; + code = walReadWithHandle(pRead, ver); + ASSERT_EQ(code, 0); + + //printf("rrbody: \n"); + //for(int i = 0; i < pRead->pHead->head.len; i++) { + //printf("%d ", pRead->pHead->head.body[i]); + //} + //printf("\n"); + + ASSERT_EQ(pRead->pHead->head.version, ver); + ASSERT_EQ(pRead->curVersion, ver+1); + char newStr[100]; + sprintf(newStr, "%s-%d", ranStr, ver); + int len = strlen(newStr); + ASSERT_EQ(pRead->pHead->head.len, len); + for(int j = 0; j < len; j++) { + EXPECT_EQ(newStr[j], pRead->pHead->head.body[j]); + } + } +} diff --git a/source/os/src/osSemaphore.c b/source/os/src/osSemaphore.c index 2385d10285..0d7066b5c8 100644 --- a/source/os/src/osSemaphore.c +++ b/source/os/src/osSemaphore.c @@ -42,7 +42,7 @@ bool taosComparePthread(pthread_t first, pthread_t second) { return first.p == s int32_t taosGetPId() { return GetCurrentProcessId(); } -int32_t taosGetCurrentAPPName(char* name, int32_t* len) { +int32_t taosGetAppName(char* name, int32_t* len) { char filepath[1024] = {0}; GetModuleFileName(NULL, filepath, MAX_PATH); @@ -358,7 +358,7 @@ bool taosComparePthread(pthread_t first, pthread_t second) { return pthread_equa int32_t taosGetPId() { return (int32_t)getpid(); } -int32_t taosGetCurrentAPPName(char *name, int32_t *len) { +int32_t taosGetAppName(char *name, int32_t *len) { char buf[PATH_MAX + 1]; buf[0] = '\0'; proc_name(getpid(), buf, sizeof(buf) - 1); @@ -392,7 +392,7 @@ void taosResetPthread(pthread_t* thread) { *thread = 0; } bool taosComparePthread(pthread_t first, pthread_t second) { return first == second; } int32_t taosGetPId() { return getpid(); } -int32_t taosGetCurrentAPPName(char* name, int32_t* len) { +int32_t taosGetAppName(char* name, int32_t* len) { const char* self = "/proc/self/exe"; char path[PATH_MAX] = {0}; diff --git a/source/os/src/osString.c b/source/os/src/osString.c index 6063f81682..f714fd0dc4 100644 --- a/source/os/src/osString.c +++ b/source/os/src/osString.c @@ -275,13 +275,13 @@ char *strsep(char **stringp, const char *delim) { } char *getpass(const char *prefix) { - static char passwd[TSDB_KEY_LEN] = {0}; - memset(passwd, 0, TSDB_KEY_LEN); + static char passwd[TSDB_PASSWORD_LEN] = {0}; + memset(passwd, 0, TSDB_PASSWORD_LEN); //printf("%s", prefix); int32_t index = 0; char ch; - while (index < TSDB_KEY_LEN) { + while (index < TSDB_PASSWORD_LEN) { ch = getch(); if (ch == '\n' || ch == '\r') { break; diff --git a/source/os/src/osSysinfo.c b/source/os/src/osSysinfo.c index ca817c4c1e..f892b4d8c0 100644 --- a/source/os/src/osSysinfo.c +++ b/source/os/src/osSysinfo.c @@ -714,7 +714,7 @@ static void taosGetSystemLocale() { // get and set default locale //printf("locale not configured, set to system default:%s", tsLocale); } - /* if user does not specify the charset, extract it from locale */ + // if user does not specify the charset, extract it from locale char *str = strrchr(tsLocale, sep); if (str != NULL) { str++; @@ -1118,13 +1118,13 @@ char *taosGetCmdlineByPID(int pid) { SysNameInfo taosGetSysNameInfo() { SysNameInfo info = {0}; - struct utsname buf; - if (!uname(&buf)) { - info.sysname = buf.sysname; - info.sysname == buf.nodename; - info.sysname = buf.release; - info.sysname = buf.version; - info.sysname = buf.machine; + struct utsname uts; + if (!uname(&uts)) { + info.sysname = strdup(uts.sysname); + info.nodename = strdup(uts.nodename); + info.release = strdup(uts.release); + info.version = strdup(uts.version); + info.machine = strdup(uts.machine); } return info; diff --git a/source/util/src/tarray.c b/source/util/src/tarray.c index 2191419670..581a797343 100644 --- a/source/util/src/tarray.c +++ b/source/util/src/tarray.c @@ -58,24 +58,31 @@ static int32_t taosArrayResize(SArray* pArray) { return 0; } -void* taosArrayAddBatch(SArray* pArray, const void* pData, int nEles) { - if (pArray == NULL || pData == NULL) { - return NULL; - } - - if (pArray->size + nEles > pArray->capacity) { +int32_t taosArrayEnsureCap(SArray* pArray, size_t newCap) { + if (newCap > pArray->capacity) { size_t tsize = (pArray->capacity << 1u); - while (pArray->size + nEles > tsize) { + while (newCap > tsize) { tsize = (tsize << 1u); } pArray->pData = realloc(pArray->pData, tsize * pArray->elemSize); if (pArray->pData == NULL) { - return NULL; + return -1; } pArray->capacity = tsize; } + return 0; +} + +void* taosArrayAddBatch(SArray* pArray, const void* pData, int nEles) { + if (pArray == NULL || pData == NULL) { + return NULL; + } + + if(taosArrayEnsureCap(pArray, pArray->size + nEles) != 0){ + return NULL; + } void* dst = TARRAY_GET_ELEM(pArray, pArray->size); memcpy(dst, pData, pArray->elemSize * nEles); @@ -243,7 +250,7 @@ void taosArrayPopFrontBatch(SArray* pArray, size_t cnt) { if(pArray->size == 0) { return; } - memmove(pArray->pData, (char*)pArray->pData + cnt * pArray->elemSize, pArray->size); + memmove(pArray->pData, (char*)pArray->pData + cnt * pArray->elemSize, pArray->size * pArray->elemSize); } void taosArrayPopTailBatch(SArray* pArray, size_t cnt) { diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index 8a6f389366..726247d450 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -282,7 +282,7 @@ static void taosReadConfigOption(const char *option, char *value, char *value2, } } -void taosInitConfigOption(SGlobalCfg cfg) { +void taosAddConfigOption(SGlobalCfg cfg) { tsGlobalConfig[tsGlobalConfigNum++] = cfg; } @@ -335,7 +335,7 @@ void taosReadGlobalLogCfg() { fclose(fp); } -int32_t taosReadGlobalCfg() { +int32_t taosReadCfgFromFile() { char * line, *option, *value, *value2, *value3; int olen, vlen, vlen2, vlen3; char fileName[PATH_MAX] = {0}; @@ -396,7 +396,7 @@ int32_t taosReadGlobalCfg() { return 0; } -void taosPrintGlobalCfg() { +void taosPrintCfg() { uInfo(" taos config & system info:"); uInfo("=================================="); @@ -443,7 +443,6 @@ void taosPrintGlobalCfg() { } taosPrintOsInfo(); - // taosPrintDataDirCfg(); uInfo("=================================="); } diff --git a/source/util/src/terror.c b/source/util/src/terror.c index ceb344ffe8..42fde042e7 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -127,6 +127,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TSC_DUP_TAG_NAMES, "duplicated tag names" TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_JSON, "Invalid JSON format") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_JSON_TYPE, "Invalid JSON data type") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_VALUE_OUT_OF_RANGE, "Value out of range") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_INPUT, "Invalid tsc input") // mnode TAOS_DEFINE_ERROR(TSDB_CODE_MND_MSG_NOT_PROCESSED, "Message not processed") @@ -497,6 +498,15 @@ TAOS_DEFINE_ERROR(TSDB_CODE_FS_FILE_ALREADY_EXISTS, "tfs file already exis TAOS_DEFINE_ERROR(TSDB_CODE_FS_INVLD_LEVEL, "tfs invalid level") TAOS_DEFINE_ERROR(TSDB_CODE_FS_NO_VALID_DISK, "tfs no valid disk") +// catalog +TAOS_DEFINE_ERROR(TSDB_CODE_CTG_INTERNAL_ERROR, "catalog interval error") +TAOS_DEFINE_ERROR(TSDB_CODE_CTG_INVALID_INPUT, "invalid catalog input parameters") +TAOS_DEFINE_ERROR(TSDB_CODE_CTG_NOT_READY, "catalog is not ready") +TAOS_DEFINE_ERROR(TSDB_CODE_CTG_MEM_ERROR, "catalog memory error") +TAOS_DEFINE_ERROR(TSDB_CODE_CTG_SYS_ERROR, "catalog system error") + + + #ifdef TAOS_ERROR_C }; #endif @@ -545,4 +555,4 @@ const char* tstrerror(int32_t err) { return ""; } -const char* terrstr() { return tstrerror(terrno); } \ No newline at end of file +const char* terrstr() { return tstrerror(terrno); } diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index e6cc3a53af..2d7179c0d3 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -99,6 +99,8 @@ int32_t wDebugFlag = 135; int32_t tsdbDebugFlag = 131; int32_t cqDebugFlag = 131; int32_t fsDebugFlag = 135; +int32_t ctgDebugFlag = 131; + int64_t dbgEmptyW = 0; int64_t dbgWN = 0; @@ -112,7 +114,7 @@ static int32_t taosPushLogBuffer(SLogBuff *tLogBuff, char *msg, int32_t msgLen static SLogBuff *taosLogBuffNew(int32_t bufSize); static void taosCloseLogByFd(int32_t oldFd); static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum); -extern void taosPrintGlobalCfg(); +extern void taosPrintCfg(); static int32_t taosCompressFile(char *srcFileName, char *destFileName); static int32_t taosStartLog() { @@ -222,7 +224,7 @@ static void *taosThreadToOpenNewFile(void *param) { uInfo(" new log file:%d is opened", tsLogObj.flag); uInfo("=================================="); - taosPrintGlobalCfg(); + taosPrintCfg(); taosKeepOldLog(keepName); return NULL; diff --git a/source/util/src/tnote.c b/source/util/src/tnote.c index 5606ab248d..d9356b7e40 100644 --- a/source/util/src/tnote.c +++ b/source/util/src/tnote.c @@ -49,15 +49,6 @@ int32_t taosInitNotes() { taosInitNote(tsNumOfLogLines, 1, &tsTscNote, name); } - if (tsHttpEnableRecordSql) { - snprintf(name, TSDB_FILENAME_LEN * 2, "%s/httpsql", tsLogDir); - taosInitNote(tsNumOfLogLines, 1, &tsHttpNote, name); - } - - if (tscEmbedded == 1) { - snprintf(name, TSDB_FILENAME_LEN * 2, "%s/taosinfo", tsLogDir); - taosInitNote(tsNumOfLogLines, 1, &tsInfoNote, name); - } #endif return 0; } diff --git a/source/util/src/ttimer.c b/source/util/src/ttimer.c index 56186d9b24..1fdc2257d7 100644 --- a/source/util/src/ttimer.c +++ b/source/util/src/ttimer.c @@ -20,12 +20,10 @@ #include "tutil.h" #include "taoserror.h" -extern int8_t tscEmbedded; - -#define tmrFatal(...) { if (tmrDebugFlag & DEBUG_FATAL) { taosPrintLog("TMR FATAL ", tscEmbedded ? 255 : tmrDebugFlag, __VA_ARGS__); }} -#define tmrError(...) { if (tmrDebugFlag & DEBUG_ERROR) { taosPrintLog("TMR ERROR ", tscEmbedded ? 255 : tmrDebugFlag, __VA_ARGS__); }} -#define tmrWarn(...) { if (tmrDebugFlag & DEBUG_WARN) { taosPrintLog("TMR WARN ", tscEmbedded ? 255 : tmrDebugFlag, __VA_ARGS__); }} -#define tmrInfo(...) { if (tmrDebugFlag & DEBUG_INFO) { taosPrintLog("TMR ", tscEmbedded ? 255 : tmrDebugFlag, __VA_ARGS__); }} +#define tmrFatal(...) { if (tmrDebugFlag & DEBUG_FATAL) { taosPrintLog("TMR FATAL ", tmrDebugFlag, __VA_ARGS__); }} +#define tmrError(...) { if (tmrDebugFlag & DEBUG_ERROR) { taosPrintLog("TMR ERROR ", tmrDebugFlag, __VA_ARGS__); }} +#define tmrWarn(...) { if (tmrDebugFlag & DEBUG_WARN) { taosPrintLog("TMR WARN ", tmrDebugFlag, __VA_ARGS__); }} +#define tmrInfo(...) { if (tmrDebugFlag & DEBUG_INFO) { taosPrintLog("TMR ", tmrDebugFlag, __VA_ARGS__); }} #define tmrDebug(...) { if (tmrDebugFlag & DEBUG_DEBUG) { taosPrintLog("TMR ", tmrDebugFlag, __VA_ARGS__); }} #define tmrTrace(...) { if (tmrDebugFlag & DEBUG_TRACE) { taosPrintLog("TMR ", tmrDebugFlag, __VA_ARGS__); }} diff --git a/source/util/src/tutil.c b/source/util/src/tutil.c index e8a5cdd301..58a2e57f7c 100644 --- a/source/util/src/tutil.c +++ b/source/util/src/tutil.c @@ -417,16 +417,3 @@ void taosIp2String(uint32_t ip, char *str) { void taosIpPort2String(uint32_t ip, uint16_t port, char *str) { sprintf(str, "%u.%u.%u.%u:%u", ip & 0xFF, (ip >> 8) & 0xFF, (ip >> 16) & 0xFF, (uint8_t)(ip >> 24), port); } - -int32_t taosGetFqdnPortFromEp(const char *ep, char *fqdn, uint16_t *port) { - *port = 0; - strcpy(fqdn, ep); - - char *temp = strchr(fqdn, ':'); - if (temp) { - *temp = 0; - *port = atoi(temp + 1); - } - - return 0; -} \ No newline at end of file diff --git a/src/inc/tcq.h b/src/inc/tcq.h index 7338cccfee..71efe33011 100644 --- a/src/inc/tcq.h +++ b/src/inc/tcq.h @@ -26,7 +26,7 @@ typedef int32_t (*FCqWrite)(int32_t vgId, void *pHead, int32_t qtype, void *pMsg typedef struct { int32_t vgId; char user[TSDB_USER_LEN]; - char pass[TSDB_KEY_LEN]; + char pass[TSDB_PASSWORD_LEN]; char db[TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN]; // size must same with SVnodeObj.db[TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN] FCqWrite cqWrite; } SCqCfg; @@ -37,7 +37,7 @@ typedef struct { int32_t master; int32_t num; // number of continuous streams char user[TSDB_USER_LEN]; - char pass[TSDB_KEY_LEN]; + char pass[TSDB_PASSWORD_LEN]; char db[TSDB_DB_NAME_LEN]; FCqWrite cqWrite; struct SCqObj *pHead; diff --git a/src/tsdb/src/tsdbCommitQueue.c b/src/tsdb/src/tsdbCommitQueue.c deleted file mode 100644 index 59fb4f334d..0000000000 --- a/src/tsdb/src/tsdbCommitQueue.c +++ /dev/null @@ -1,213 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "tsdbint.h" - -typedef struct { - bool stop; - pthread_mutex_t lock; - pthread_cond_t queueNotEmpty; - int nthreads; - int refCount; - SList * queue; - pthread_t * threads; -} SCommitQueue; - -typedef struct { - TSDB_REQ_T req; - STsdbRepo *pRepo; -} SReq; - -static void *tsdbLoopCommit(void *arg); - -static SCommitQueue tsCommitQueue = {0}; - -int tsdbInitCommitQueue() { - int nthreads = tsNumOfCommitThreads; - SCommitQueue *pQueue = &tsCommitQueue; - - if (nthreads < 1) nthreads = 1; - - pQueue->stop = false; - pQueue->nthreads = nthreads; - - pQueue->queue = tdListNew(0); - if (pQueue->queue == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; - } - - pQueue->threads = (pthread_t *)calloc(nthreads, sizeof(pthread_t)); - if (pQueue->threads == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - tdListFree(pQueue->queue); - return -1; - } - - pthread_mutex_init(&(pQueue->lock), NULL); - pthread_cond_init(&(pQueue->queueNotEmpty), NULL); - - for (int i = 0; i < nthreads; i++) { - pthread_create(pQueue->threads + i, NULL, tsdbLoopCommit, NULL); - } - - return 0; -} - -void tsdbDestroyCommitQueue() { - SCommitQueue *pQueue = &tsCommitQueue; - - pthread_mutex_lock(&(pQueue->lock)); - - if (pQueue->stop) { - pthread_mutex_unlock(&(pQueue->lock)); - return; - } - - pQueue->stop = true; - pthread_cond_broadcast(&(pQueue->queueNotEmpty)); - - pthread_mutex_unlock(&(pQueue->lock)); - - for (size_t i = 0; i < pQueue->nthreads; i++) { - pthread_join(pQueue->threads[i], NULL); - } - - free(pQueue->threads); - tdListFree(pQueue->queue); - pthread_cond_destroy(&(pQueue->queueNotEmpty)); - pthread_mutex_destroy(&(pQueue->lock)); -} - -int tsdbScheduleCommit(STsdbRepo *pRepo, TSDB_REQ_T req) { - SCommitQueue *pQueue = &tsCommitQueue; - - SListNode *pNode = (SListNode *)calloc(1, sizeof(SListNode) + sizeof(SReq)); - if (pNode == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; - } - - ((SReq *)pNode->data)->req = req; - ((SReq *)pNode->data)->pRepo = pRepo; - - pthread_mutex_lock(&(pQueue->lock)); - - // ASSERT(pQueue->stop); - - tdListAppendNode(pQueue->queue, pNode); - pthread_cond_signal(&(pQueue->queueNotEmpty)); - - pthread_mutex_unlock(&(pQueue->lock)); - return 0; -} - -static void tsdbApplyRepoConfig(STsdbRepo *pRepo) { - pthread_mutex_lock(&pRepo->save_mutex); - - pRepo->config_changed = false; - STsdbCfg * pSaveCfg = &pRepo->save_config; - STsdbCfg oldCfg; - int32_t oldTotalBlocks = pRepo->config.totalBlocks; - - memcpy(&oldCfg, &(pRepo->config), sizeof(STsdbCfg)); - - pRepo->config.compression = pRepo->save_config.compression; - pRepo->config.keep = pRepo->save_config.keep; - pRepo->config.keep1 = pRepo->save_config.keep1; - pRepo->config.keep2 = pRepo->save_config.keep2; - pRepo->config.cacheLastRow = pRepo->save_config.cacheLastRow; - pRepo->config.totalBlocks = pRepo->save_config.totalBlocks; - - pthread_mutex_unlock(&pRepo->save_mutex); - - tsdbInfo("vgId:%d apply new config: compression(%d), keep(%d,%d,%d), totalBlocks(%d), cacheLastRow(%d->%d),totalBlocks(%d->%d)", - REPO_ID(pRepo), - pSaveCfg->compression, pSaveCfg->keep,pSaveCfg->keep1, pSaveCfg->keep2, - pSaveCfg->totalBlocks, oldCfg.cacheLastRow, pSaveCfg->cacheLastRow, oldTotalBlocks, pSaveCfg->totalBlocks); - - int err = tsdbExpandPool(pRepo, oldTotalBlocks); - if (!TAOS_SUCCEEDED(err)) { - tsdbError("vgId:%d expand pool from %d to %d fail,reason:%s", - REPO_ID(pRepo), oldTotalBlocks, pSaveCfg->totalBlocks, tstrerror(err)); - } - - if (oldCfg.cacheLastRow != pRepo->config.cacheLastRow) { - if (tsdbLockRepo(pRepo) < 0) return; - tsdbCacheLastData(pRepo, &oldCfg); - tsdbUnlockRepo(pRepo); - } - -} - -static void *tsdbLoopCommit(void *arg) { - SCommitQueue *pQueue = &tsCommitQueue; - SListNode * pNode = NULL; - STsdbRepo * pRepo = NULL; - TSDB_REQ_T req; - - setThreadName("tsdbCommit"); - - while (true) { - pthread_mutex_lock(&(pQueue->lock)); - - while (true) { - pNode = tdListPopHead(pQueue->queue); - if (pNode == NULL) { - if (pQueue->stop && pQueue->refCount <= 0) { - pthread_mutex_unlock(&(pQueue->lock)); - goto _exit; - } else { - pthread_cond_wait(&(pQueue->queueNotEmpty), &(pQueue->lock)); - } - } else { - break; - } - } - - pthread_mutex_unlock(&(pQueue->lock)); - - req = ((SReq *)pNode->data)->req; - pRepo = ((SReq *)pNode->data)->pRepo; - - if (req == COMMIT_REQ) { - tsdbCommitData(pRepo); - } else if (req == COMPACT_REQ) { - tsdbCompactImpl(pRepo); - } else if (req == COMMIT_CONFIG_REQ) { - ASSERT(pRepo->config_changed); - tsdbApplyRepoConfig(pRepo); - tsem_post(&(pRepo->readyToCommit)); - } else { - ASSERT(0); - } - - listNodeFree(pNode); - } - -_exit: - return NULL; -} - -void tsdbIncCommitRef(int vgId) { - int refCount = atomic_add_fetch_32(&tsCommitQueue.refCount, 1); - tsdbDebug("vgId:%d, inc commit queue ref to %d", vgId, refCount); -} - -void tsdbDecCommitRef(int vgId) { - int refCount = atomic_sub_fetch_32(&tsCommitQueue.refCount, 1); - pthread_cond_broadcast(&(tsCommitQueue.queueNotEmpty)); - tsdbDebug("vgId:%d, dec commit queue ref to %d", vgId, refCount); -} diff --git a/tests/pytest/crash_gen/valgrind_taos.supp b/tests/pytest/crash_gen/valgrind_taos.supp index ec44a85d5b..974103052e 100644 --- a/tests/pytest/crash_gen/valgrind_taos.supp +++ b/tests/pytest/crash_gen/valgrind_taos.supp @@ -17515,7 +17515,7 @@ fun:gaih_inet.constprop.0 fun:getaddrinfo fun:taosGetFqdn - fun:taosCheckGlobalCfg + fun:taosCheckAndPrintCfg fun:taos_init_imp } { @@ -17740,7 +17740,7 @@ fun:gaih_inet.constprop.7 fun:getaddrinfo fun:taosGetFqdn - fun:taosCheckGlobalCfg + fun:taosCheckAndPrintCfg fun:taos_init_imp } {