diff --git a/cmake/libuv_CMakeLists.txt.in b/cmake/libuv_CMakeLists.txt.in index ed406e089e..6c7ab79ed2 100644 --- a/cmake/libuv_CMakeLists.txt.in +++ b/cmake/libuv_CMakeLists.txt.in @@ -4,9 +4,9 @@ ExternalProject_Add(libuv GIT_REPOSITORY https://github.com/libuv/libuv.git GIT_TAG v1.42.0 SOURCE_DIR "${CMAKE_CONTRIB_DIR}/libuv" - BINARY_DIR "" - CONFIGURE_COMMAND "" - BUILD_COMMAND "" + BINARY_DIR "${CMAKE_CONTRIB_DIR}/libuv" + CONFIGURE_COMMAND "" + BUILD_COMMAND "" INSTALL_COMMAND "" TEST_COMMAND "" - ) \ No newline at end of file + ) diff --git a/include/dnode/bnode/bnode.h b/include/dnode/bnode/bnode.h index 3cc26861ab..7f7b8a8d53 100644 --- a/include/dnode/bnode/bnode.h +++ b/include/dnode/bnode/bnode.h @@ -23,22 +23,18 @@ extern "C" { /* ------------------------ TYPES EXPOSED ------------------------ */ typedef struct SDnode SDnode; typedef struct SBnode SBnode; -typedef int32_t (*SendReqToDnodeFp)(SDnode *pDnode, struct SEpSet *epSet, struct SRpcMsg *rpcMsg); -typedef int32_t (*SendReqToMnodeFp)(SDnode *pDnode, struct SRpcMsg *rpcMsg); -typedef void (*SendRedirectRspFp)(SDnode *pDnode, struct SRpcMsg *rpcMsg); +typedef int32_t (*SendReqToDnodeFp)(SDnode *pDnode, struct SEpSet *epSet, struct SRpcMsg *pMsg); +typedef int32_t (*SendReqToMnodeFp)(SDnode *pDnode, struct SRpcMsg *pMsg); +typedef void (*SendRedirectRspFp)(SDnode *pDnode, struct SRpcMsg *pMsg); typedef struct { int64_t numOfErrors; } SBnodeLoad; typedef struct { - int32_t sver; -} SBnodeCfg; - -typedef struct { + int32_t sver; int32_t dnodeId; int64_t clusterId; - SBnodeCfg cfg; SDnode *pDnode; SendReqToDnodeFp sendReqToDnodeFp; SendReqToMnodeFp sendReqToMnodeFp; diff --git a/include/dnode/mgmt/dnode.h b/include/dnode/mgmt/dnode.h index 2e3863c3a1..e64d358de4 100644 --- a/include/dnode/mgmt/dnode.h +++ b/include/dnode/mgmt/dnode.h @@ -22,15 +22,39 @@ extern "C" { #endif -/* ------------------------ TYPES EXPOSED ------------------------ */ +/* ------------------------ TYPES EXPOSED ---------------- */ typedef struct SDnode SDnode; +/* ------------------------ Environment ------------------ */ +typedef struct { + int32_t sver; + int32_t numOfCores; + int16_t numOfCommitThreads; + int8_t enableTelem; + char timezone[TSDB_TIMEZONE_LEN]; + char locale[TSDB_LOCALE_LEN]; + char charset[TSDB_LOCALE_LEN]; + char buildinfo[64]; + char gitinfo[48]; +} SDnodeEnvCfg; + +/** + * @brief Initialize the environment + * + * @param pOption Option of the environment + * @return int32_t 0 for success and -1 for failure + */ +int32_t dndInit(const SDnodeEnvCfg *pCfg); + +/** + * @brief clear the environment + * + */ +void dndCleanup(); + +/* ------------------------ SDnode ----------------------- */ typedef struct { - int32_t sver; - int32_t numOfCores; int32_t numOfSupportVnodes; - int16_t numOfCommitThreads; - int8_t enableTelem; int32_t statusInterval; float numOfThreadsPerCore; float ratioOfQueryCores; @@ -41,28 +65,22 @@ typedef struct { char localEp[TSDB_EP_LEN]; char localFqdn[TSDB_FQDN_LEN]; char firstEp[TSDB_EP_LEN]; - char timezone[TSDB_TIMEZONE_LEN]; - char locale[TSDB_LOCALE_LEN]; - char charset[TSDB_LOCALE_LEN]; - char buildinfo[64]; - char gitinfo[48]; -} SDnodeOpt; +} SDnodeObjCfg; -/* ------------------------ SDnode ------------------------ */ /** * @brief Initialize and start the dnode. * - * @param pOption Option of the dnode. + * @param pCfg Config of the dnode. * @return SDnode* The dnode object. */ -SDnode *dndInit(SDnodeOpt *pOption); +SDnode *dndCreate(SDnodeObjCfg *pCfg); /** * @brief Stop and cleanup the dnode. * * @param pDnode The dnode object to close. */ -void dndCleanup(SDnode *pDnode); +void dndClose(SDnode *pDnode); #ifdef __cplusplus } diff --git a/include/dnode/qnode/qnode.h b/include/dnode/qnode/qnode.h index 554c57a045..3de2986047 100644 --- a/include/dnode/qnode/qnode.h +++ b/include/dnode/qnode/qnode.h @@ -23,9 +23,9 @@ extern "C" { /* ------------------------ TYPES EXPOSED ------------------------ */ typedef struct SDnode SDnode; typedef struct SQnode SQnode; -typedef int32_t (*SendReqToDnodeFp)(SDnode *pDnode, struct SEpSet *epSet, struct SRpcMsg *rpcMsg); -typedef int32_t (*SendReqToMnodeFp)(SDnode *pDnode, struct SRpcMsg *rpcMsg); -typedef void (*SendRedirectRspFp)(SDnode *pDnode, struct SRpcMsg *rpcMsg); +typedef int32_t (*SendReqToDnodeFp)(SDnode *pDnode, struct SEpSet *epSet, struct SRpcMsg *pMsg); +typedef int32_t (*SendReqToMnodeFp)(SDnode *pDnode, struct SRpcMsg *pMsg); +typedef void (*SendRedirectRspFp)(SDnode *pDnode, struct SRpcMsg *pMsg); typedef struct { int64_t numOfStartTask; @@ -39,13 +39,9 @@ typedef struct { } SQnodeLoad; typedef struct { - int32_t sver; -} SQnodeCfg; - -typedef struct { + int32_t sver; int32_t dnodeId; int64_t clusterId; - SQnodeCfg cfg; SDnode *pDnode; SendReqToDnodeFp sendReqToDnodeFp; SendReqToMnodeFp sendReqToMnodeFp; diff --git a/include/dnode/snode/snode.h b/include/dnode/snode/snode.h index 43d3dd9b4b..319708decc 100644 --- a/include/dnode/snode/snode.h +++ b/include/dnode/snode/snode.h @@ -23,22 +23,18 @@ extern "C" { /* ------------------------ TYPES EXPOSED ------------------------ */ typedef struct SDnode SDnode; typedef struct SSnode SSnode; -typedef int32_t (*SendReqToDnodeFp)(SDnode *pDnode, struct SEpSet *epSet, struct SRpcMsg *rpcMsg); -typedef int32_t (*SendReqToMnodeFp)(SDnode *pDnode, struct SRpcMsg *rpcMsg); -typedef void (*SendRedirectRspFp)(SDnode *pDnode, struct SRpcMsg *rpcMsg); +typedef int32_t (*SendReqToDnodeFp)(SDnode *pDnode, struct SEpSet *epSet, struct SRpcMsg *pMsg); +typedef int32_t (*SendReqToMnodeFp)(SDnode *pDnode, struct SRpcMsg *pMsg); +typedef void (*SendRedirectRspFp)(SDnode *pDnode, struct SRpcMsg *pMsg); typedef struct { int64_t numOfErrors; } SSnodeLoad; typedef struct { - int32_t sver; -} SSnodeCfg; - -typedef struct { + int32_t sver; int32_t dnodeId; int64_t clusterId; - SSnodeCfg cfg; SDnode *pDnode; SendReqToDnodeFp sendReqToDnodeFp; SendReqToMnodeFp sendReqToMnodeFp; diff --git a/include/libs/parser/parsenodes.h b/include/libs/parser/parsenodes.h index ac8a10067d..374ae02e4a 100644 --- a/include/libs/parser/parsenodes.h +++ b/include/libs/parser/parsenodes.h @@ -43,15 +43,6 @@ typedef struct SField { int32_t bytes; } SField; -typedef struct SParseBasicCtx { - uint64_t requestId; - int32_t acctId; - const char *db; - void *pTransporter; - SEpSet mgmtEpSet; - struct SCatalog *pCatalog; -} SParseBasicCtx; - typedef struct SFieldInfo { int16_t numOfOutput; // number of column in result SField *final; diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index edf9cf461f..0cdbfdb07f 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -23,11 +23,17 @@ extern "C" { #include "parsenodes.h" typedef struct SParseContext { - SParseBasicCtx ctx; + uint64_t requestId; + int32_t acctId; + const char *db; + void *pTransporter; + SEpSet mgmtEpSet; const char *pSql; // sql string size_t sqlLen; // length of the sql string char *pMsg; // extended error message if exists to help identifying the problem in sql statement. int32_t msgLen; // max length of the msg + + struct SCatalog *pCatalog; } SParseContext; /** diff --git a/include/libs/planner/planner.h b/include/libs/planner/planner.h index da70f21498..d98c1ca595 100644 --- a/include/libs/planner/planner.h +++ b/include/libs/planner/planner.h @@ -91,8 +91,10 @@ typedef struct SPhyNode { typedef struct SScanPhyNode { SPhyNode node; - uint64_t uid; // unique id of the table + uint64_t uid; // unique id of the table int8_t tableType; + int32_t order; // scan order: TSDB_ORDER_ASC|TSDB_ORDER_DESC + int32_t count; // repeat count } SScanPhyNode; typedef SScanPhyNode SSystemTableScanPhyNode; diff --git a/include/libs/tfs/tfs.h b/include/libs/tfs/tfs.h index 6c850d1016..793c861363 100644 --- a/include/libs/tfs/tfs.h +++ b/include/libs/tfs/tfs.h @@ -49,7 +49,7 @@ typedef struct { } STierMeta; int tfsInit(SDiskCfg *pDiskCfg, int ndisk); -void tfsDestroy(); +void tfsCleanup(); void tfsUpdateInfo(SFSMeta *pFSMeta, STierMeta *tierMetas, int8_t numLevels); void tfsGetMeta(SFSMeta *pMeta); void tfsAllocDisk(int expLevel, int *level, int *id); diff --git a/include/libs/tkv/tkv.h b/include/libs/tkv/tkv.h deleted file mode 100644 index 98194f090c..0000000000 --- a/include/libs/tkv/tkv.h +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_TKV_H_ -#define _TD_TKV_H_ - -#if 0 -#include "os.h" - -#ifdef __cplusplus -extern "C" { -#endif - -// Types exported -typedef struct STkvDb STkvDb; -typedef struct STkvOpts STkvOpts; -typedef struct STkvCache STkvCache; -typedef struct STkvReadOpts STkvReadOpts; -typedef struct STkvWriteOpts STkvWriteOpts; - -// DB operations -STkvDb *tkvOpen(const STkvOpts *options, const char *path); -void tkvClose(STkvDb *db); -void tkvPut(STkvDb *db, const STkvWriteOpts *, const char *key, size_t keylen, const char *val, size_t vallen); -char * tkvGet(STkvDb *db, const STkvReadOpts *, const char *key, size_t keylen, size_t *vallen); -void tkvCommit(STkvDb *db); - -// DB options -STkvOpts *tkvOptsCreate(); -void tkvOptsDestroy(STkvOpts *); -void tkvOptionsSetCache(STkvOpts *, STkvCache *); -void tkvOptsSetCreateIfMissing(STkvOpts *, unsigned char); - -// DB cache -typedef enum { TKV_LRU_CACHE = 0, TKV_LFU_CACHE = 1 } ETkvCacheType; -STkvCache *tkvCacheCreate(size_t capacity, ETkvCacheType type); -void tkvCacheDestroy(STkvCache *); - -// STkvReadOpts -STkvReadOpts *tkvReadOptsCreate(); -void tkvReadOptsDestroy(STkvReadOpts *); - -// STkvWriteOpts -STkvWriteOpts *tkvWriteOptsCreate(); -void tkvWriteOptsDestroy(STkvWriteOpts *); - -#ifdef __cplusplus -} -#endif - -#endif -#endif /*_TD_TKV_H_*/ \ No newline at end of file diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 80241405a6..854d16f67d 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -70,6 +70,8 @@ int32_t* taosGetErrno(); #define TSDB_CODE_INVALID_MSG TAOS_DEF_ERROR_CODE(0, 0x0108) #define TSDB_CODE_MSG_NOT_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0109) #define TSDB_CODE_INVALID_PARA TAOS_DEF_ERROR_CODE(0, 0x010A) +#define TSDB_CODE_REPEAT_INIT TAOS_DEF_ERROR_CODE(0, 0x010B) + #define TSDB_CODE_REF_NO_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0110) #define TSDB_CODE_REF_FULL TAOS_DEF_ERROR_CODE(0, 0x0111) #define TSDB_CODE_REF_ID_REMOVED TAOS_DEF_ERROR_CODE(0, 0x0112) diff --git a/include/util/tarray.h b/include/util/tarray.h index f7c72add01..6d6120a49b 100644 --- a/include/util/tarray.h +++ b/include/util/tarray.h @@ -23,10 +23,23 @@ extern "C" { #include "os.h" #include "talgo.h" +#if 0 +#define TARRAY(TYPE) \ + struct { \ + int32_t tarray_size_; \ + int32_t tarray_neles_; \ + struct TYPE* td_array_data_; \ + } + +#define TARRAY_SIZE(ARRAY) (ARRAY)->tarray_size_ +#define TARRAY_NELES(ARRAY) (ARRAY)->tarray_neles_ +#define TARRAY_ELE_AT(ARRAY, IDX) ((ARRAY)->td_array_data_ + idx) +#endif + #define TARRAY_MIN_SIZE 8 #define TARRAY_GET_ELEM(array, index) ((void*)((char*)((array)->pData) + (index) * (array)->elemSize)) -#define TARRAY_ELEM_IDX(array, ele) (POINTER_DISTANCE(ele, (array)->pData) / (array)->elemSize) -#define TARRAY_GET_START(array) ((array)->pData) +#define TARRAY_ELEM_IDX(array, ele) (POINTER_DISTANCE(ele, (array)->pData) / (array)->elemSize) +#define TARRAY_GET_START(array) ((array)->pData) typedef struct SArray { size_t size; @@ -57,7 +70,7 @@ int32_t taosArrayEnsureCap(SArray* pArray, size_t tsize); * @param nEles * @return */ -void *taosArrayAddBatch(SArray *pArray, const void *pData, int nEles); +void* taosArrayAddBatch(SArray* pArray, const void* pData, int nEles); /** * @@ -65,7 +78,7 @@ void *taosArrayAddBatch(SArray *pArray, const void *pData, int nEles); * @param pData position array list * @param numOfElems the number of removed position */ -void taosArrayRemoveBatch(SArray *pArray, const int32_t* pData, int32_t numOfElems); +void taosArrayRemoveBatch(SArray* pArray, const int32_t* pData, int32_t numOfElems); /** * @@ -73,7 +86,7 @@ void taosArrayRemoveBatch(SArray *pArray, const int32_t* pData, int32_t numOfEle * @param comparFn * @param fp */ -void taosArrayRemoveDuplicate(SArray *pArray, __compar_fn_t comparFn, void (*fp)(void*)); +void taosArrayRemoveDuplicate(SArray* pArray, __compar_fn_t comparFn, void (*fp)(void*)); /** * add all element from the source array list into the destination @@ -242,19 +255,18 @@ int32_t taosArraySearchIdx(const SArray* pArray, const void* key, __compar_fn_t */ char* taosArraySearchString(const SArray* pArray, const char* key, __compar_fn_t comparFn, int flags); - /** * sort the pointer data in the array * @param pArray - * @param compar - * @param param + * @param compar + * @param param * @return */ -void taosArraySortPWithExt(SArray* pArray, __ext_compar_fn_t fn, const void *param); +void taosArraySortPWithExt(SArray* pArray, __ext_compar_fn_t fn, const void* param); #ifdef __cplusplus } #endif -#endif /*_TD_UTIL_ARRAY_H*/ +#endif /*_TD_UTIL_ARRAY_H*/ diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index 7292375e7f..39f84ffd86 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -93,13 +93,12 @@ typedef struct SReqResultInfo { const char *pData; TAOS_FIELD *fields; uint32_t numOfCols; - int32_t *length; TAOS_ROW row; char **pCol; - uint32_t numOfRows; uint32_t current; + bool completed; } SReqResultInfo; typedef struct SShowReqInfo { @@ -130,7 +129,6 @@ typedef struct SRequestObj { char *msgBuf; void *pInfo; // sql parse info, generated by parser module int32_t code; - uint64_t affectedRows; // todo remove it SQueryExecMetric metric; SRequestSendRecvBody body; } SRequestObj; diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 727b668f3a..ba08e67127 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -25,8 +25,9 @@ static int32_t initEpSetFromCfg(const char *firstEp, const char *secondEp, SCorEpSet *pEpSet); static SMsgSendInfo* buildConnectMsg(SRequestObj *pRequest); static void destroySendMsgInfo(SMsgSendInfo* pMsgBody); +static void setQueryResultByRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp); -static bool stringLengthCheck(const char* str, size_t maxsize) { + static bool stringLengthCheck(const char* str, size_t maxsize) { if (str == NULL) { return false; } @@ -150,23 +151,26 @@ int32_t parseSql(SRequestObj* pRequest, SQueryNode** pQuery) { STscObj* pTscObj = pRequest->pTscObj; SParseContext cxt = { - .ctx = {.requestId = pRequest->requestId, .acctId = pTscObj->acctId, .db = getConnectionDB(pTscObj), .pTransporter = pTscObj->pTransporter}, + .requestId = pRequest->requestId, + .acctId = pTscObj->acctId, + .db = getConnectionDB(pTscObj), + .pTransporter = pTscObj->pTransporter, .pSql = pRequest->sqlstr, .sqlLen = pRequest->sqlLen, .pMsg = pRequest->msgBuf, .msgLen = ERROR_MSG_BUF_DEFAULT_SIZE }; - cxt.ctx.mgmtEpSet = getEpSet_s(&pTscObj->pAppInfo->mgmtEp); - int32_t code = catalogGetHandle(pTscObj->pAppInfo->clusterId, &cxt.ctx.pCatalog); + cxt.mgmtEpSet = getEpSet_s(&pTscObj->pAppInfo->mgmtEp); + int32_t code = catalogGetHandle(pTscObj->pAppInfo->clusterId, &cxt.pCatalog); if (code != TSDB_CODE_SUCCESS) { - tfree(cxt.ctx.db); + tfree(cxt.db); return code; } code = qParseQuerySql(&cxt, pQuery); - tfree(cxt.ctx.db); + tfree(cxt.db); return code; } @@ -212,6 +216,7 @@ int32_t getPlan(SRequestObj* pRequest, SQueryNode* pQueryNode, SQueryDag** pDag) SSubplan* pPlan = taosArrayGetP(pa, 0); SDataBlockSchema* pDataBlockSchema = &(pPlan->pDataSink->schema); setResSchemaInfo(pResInfo, pDataBlockSchema); + pRequest->type = TDMT_VND_QUERY; } @@ -228,7 +233,7 @@ void setResSchemaInfo(SReqResultInfo* pResInfo, const SDataBlockSchema* pDataBlo SSchema* pSchema = &pDataBlockSchema->pSchema[i]; pResInfo->fields[i].bytes = pSchema->bytes; pResInfo->fields[i].type = pSchema->type; - tstrncpy(pResInfo->fields[i].name, pSchema[i].name, tListLen(pResInfo->fields[i].name)); + tstrncpy(pResInfo->fields[i].name, pSchema->name, tListLen(pResInfo->fields[i].name)); } } @@ -245,8 +250,9 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryDag* pDag) { } } - pRequest->affectedRows = res.numOfRows; - return res.code; + pRequest->body.resInfo.numOfRows = res.numOfRows; + pRequest->code = res.code; + return pRequest->code; } return scheduleAsyncExecJob(pRequest->pTscObj->pTransporter, NULL, pDag, &pRequest->body.pQueryJob); @@ -545,11 +551,15 @@ void* doFetchRow(SRequestObj* pRequest) { if (pResultInfo->pData == NULL || pResultInfo->current >= pResultInfo->numOfRows) { if (pRequest->type == TDMT_VND_QUERY) { - pRequest->type = TDMT_VND_FETCH; + // All data has returned to App already, no need to try again + if (pResultInfo->completed) { + return NULL; + } + scheduleFetchRows(pRequest->body.pQueryJob, (void **)&pRequest->body.resInfo.pData); - - pResultInfo->current = 0; - if (pResultInfo->numOfRows <= pResultInfo->current) { + setQueryResultByRsp(&pRequest->body.resInfo, (SRetrieveTableRsp*)pRequest->body.resInfo.pData); + + if (pResultInfo->numOfRows == 0) { return NULL; } @@ -611,12 +621,23 @@ _return: return pResultInfo->row; } +static void doPrepareResPtr(SReqResultInfo* pResInfo) { + if (pResInfo->row == NULL) { + pResInfo->row = calloc(pResInfo->numOfCols, POINTER_BYTES); + pResInfo->pCol = calloc(pResInfo->numOfCols, POINTER_BYTES); + pResInfo->length = calloc(pResInfo->numOfCols, sizeof(int32_t)); + } +} + void setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32_t numOfCols, int32_t numOfRows) { assert(numOfCols > 0 && pFields != NULL && pResultInfo != NULL); if (numOfRows == 0) { return; } + // todo check for the failure of malloc + doPrepareResPtr(pResultInfo); + int32_t offset = 0; for (int32_t i = 0; i < numOfCols; ++i) { pResultInfo->length[i] = pResultInfo->fields[i].bytes; @@ -642,3 +663,14 @@ void setConnectionDB(STscObj* pTscObj, const char* db) { pthread_mutex_unlock(&pTscObj->mutex); } +void setQueryResultByRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp) { + assert(pResultInfo != NULL && pRsp != NULL); + + pResultInfo->pRspMsg = (const char*) pRsp; + pResultInfo->pData = (void*) pRsp->data; + pResultInfo->numOfRows = htonl(pRsp->numOfRows); + pResultInfo->current = 0; + pResultInfo->completed = (pRsp->completed == 1); + + setResultDataPtr(pResultInfo, pResultInfo->fields, pResultInfo->numOfCols, pResultInfo->numOfRows); +} \ No newline at end of file diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 1238976b97..ba2a21d7ea 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -265,7 +265,13 @@ const char *taos_data_type(int type) { const char *taos_get_client_info() { return version; } int taos_affected_rows(TAOS_RES *res) { - return ((SRequestObj*)res)->affectedRows; + if (res == NULL) { + return 0; + } + + SRequestObj* pRequest = (SRequestObj*) res; + SReqResultInfo* pResInfo = &pRequest->body.resInfo; + return pResInfo->numOfRows; } int taos_result_precision(TAOS_RES *res) { return TSDB_TIME_PRECISION_MILLI; } diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index 85f3fb06a7..4f1a96ad29 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -154,9 +154,6 @@ int32_t processShowRsp(void* param, const SDataBuf* pMsg, int32_t code) { pResInfo->fields = pFields; pResInfo->numOfCols = pMetaMsg->numOfColumns; - pResInfo->row = calloc(pResInfo->numOfCols, POINTER_BYTES); - pResInfo->pCol = calloc(pResInfo->numOfCols, POINTER_BYTES); - pResInfo->length = calloc(pResInfo->numOfCols, sizeof(int32_t)); pRequest->body.showInfo.execId = pShow->showId; diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index d507c565df..d1093bb1a6 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -279,7 +279,7 @@ TEST(testCase, connect_Test) { // taos_free_result(pRes); // taos_close(pConn); //} -// + //TEST(testCase, create_table_Test) { // TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); // assert(pConn != NULL); @@ -292,26 +292,26 @@ TEST(testCase, connect_Test) { // // taos_close(pConn); //} -// -//TEST(testCase, create_ctable_Test) { -// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// assert(pConn != NULL); -// -// TAOS_RES* pRes = taos_query(pConn, "use abc1"); -// if (taos_errno(pRes) != 0) { -// printf("failed to use db, reason:%s\n", taos_errstr(pRes)); -// } -// taos_free_result(pRes); -// -// pRes = taos_query(pConn, "create table tm0 using st1 tags(1)"); -// if (taos_errno(pRes) != 0) { -// printf("failed to create child table tm0, reason:%s\n", taos_errstr(pRes)); -// } -// -// taos_free_result(pRes); -// taos_close(pConn); -//} -// + +TEST(testCase, create_ctable_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + TAOS_RES* pRes = taos_query(pConn, "use abc1"); + if (taos_errno(pRes) != 0) { + printf("failed to use db, reason:%s\n", taos_errstr(pRes)); + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table tm0 using st1 tags(1)"); + if (taos_errno(pRes) != 0) { + printf("failed to create child table tm0, reason:%s\n", taos_errstr(pRes)); + } + + taos_free_result(pRes); + taos_close(pConn); +} + //TEST(testCase, show_stable_Test) { // TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); // assert(pConn != NULL); @@ -533,6 +533,7 @@ TEST(testCase, connect_Test) { // tmq_create_topic(pConn, "test_topic_1", sql, strlen(sql)); // taos_close(pConn); //} + //TEST(testCase, insert_test) { // TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); // ASSERT_EQ(pConn, nullptr); @@ -550,49 +551,48 @@ TEST(testCase, connect_Test) { // taos_free_result(pRes); // taos_close(pConn); //} -//#endif -TEST(testCase, projection_query_tables) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - ASSERT_NE(pConn, nullptr); - -// TAOS_RES* pRes = taos_query(pConn, "create database abc1 vgroups 2"); +//TEST(testCase, projection_query_tables) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// ASSERT_NE(pConn, nullptr); +// +//// TAOS_RES* pRes = taos_query(pConn, "create database abc1 vgroups 2"); +//// if (taos_errno(pRes) != 0) { +//// printf("failed to use db, reason:%s\n", taos_errstr(pRes)); +//// taos_free_result(pRes); +//// return; +//// } +// +//// taos_free_result(pRes); +// +// TAOS_RES* pRes = taos_query(pConn, "use abc1"); +// +//// pRes = taos_query(pConn, "create table m1 (ts timestamp, k int) tags(a int)"); +// taos_free_result(pRes); +//// +//// pRes = taos_query(pConn, "create table tu using m1 tags(1)"); +//// taos_free_result(pRes); +//// +//// pRes = taos_query(pConn, "insert into tu values(now, 1)"); +//// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "select * from tu"); // if (taos_errno(pRes) != 0) { -// printf("failed to use db, reason:%s\n", taos_errstr(pRes)); +// printf("failed to select from table, reason:%s\n", taos_errstr(pRes)); // taos_free_result(pRes); -// return; +// ASSERT_TRUE(false); // } - -// taos_free_result(pRes); - - TAOS_RES* pRes = taos_query(pConn, "use abc1"); - -// pRes = taos_query(pConn, "create table m1 (ts timestamp, k int) tags(a int)"); - taos_free_result(pRes); // -// pRes = taos_query(pConn, "create table tu using m1 tags(1)"); -// taos_free_result(pRes); +// TAOS_ROW pRow = NULL; +// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +// int32_t numOfFields = taos_num_fields(pRes); +// +// char str[512] = {0}; +// while ((pRow = taos_fetch_row(pRes)) != NULL) { +// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); +// printf("%s\n", str); +// } // -// pRes = taos_query(pConn, "insert into tu values(now, 1)"); // taos_free_result(pRes); - - pRes = taos_query(pConn, "select * from tu"); - if (taos_errno(pRes) != 0) { - printf("failed to select from table, reason:%s\n", taos_errstr(pRes)); - taos_free_result(pRes); - ASSERT_TRUE(false); - } - - TAOS_ROW pRow = NULL; - TAOS_FIELD* pFields = taos_fetch_fields(pRes); - int32_t numOfFields = taos_num_fields(pRes); - - char str[512] = {0}; - while ((pRow = taos_fetch_row(pRes)) != NULL) { - int32_t code = taos_print_row(str, pRow, pFields, numOfFields); - printf("%s\n", str); - } - - taos_free_result(pRes); - taos_close(pConn); -} +// taos_close(pConn); +//} diff --git a/source/dnode/bnode/inc/bndInt.h b/source/dnode/bnode/inc/bndInt.h index cddb1e50f1..4a0d16a749 100644 --- a/source/dnode/bnode/inc/bndInt.h +++ b/source/dnode/bnode/inc/bndInt.h @@ -30,12 +30,7 @@ extern "C" { #endif typedef struct SBnode { - int32_t dnodeId; - int64_t clusterId; - SBnodeCfg cfg; - SendReqToDnodeFp sendReqToDnodeFp; - SendReqToMnodeFp sendReqToMnodeFp; - SendRedirectRspFp sendRedirectRspFp; + SBnodeOpt opt; } SBnode; #ifdef __cplusplus diff --git a/source/dnode/mgmt/daemon/src/daemon.c b/source/dnode/mgmt/daemon/src/daemon.c index cedab6266e..ce672bf7b8 100644 --- a/source/dnode/mgmt/daemon/src/daemon.c +++ b/source/dnode/mgmt/daemon/src/daemon.c @@ -28,11 +28,11 @@ static struct { bool printAuth; bool printVersion; char configDir[PATH_MAX]; -} global = {0}; +} dmn = {0}; void dmnSigintHandle(int signum, void *info, void *ctx) { uInfo("singal:%d is received", signum); - global.stop = true; + dmn.stop = true; } void dmnSetSignalHandle() { @@ -44,7 +44,7 @@ void dmnSetSignalHandle() { } int dmnParseOption(int argc, char const *argv[]) { - tstrncpy(global.configDir, "/etc/taos", PATH_MAX); + tstrncpy(dmn.configDir, "/etc/taos", PATH_MAX); for (int i = 1; i < argc; ++i) { if (strcmp(argv[i], "-c") == 0) { @@ -53,19 +53,19 @@ int dmnParseOption(int argc, char const *argv[]) { printf("config file path overflow"); return -1; } - tstrncpy(global.configDir, argv[i], PATH_MAX); + tstrncpy(dmn.configDir, argv[i], PATH_MAX); } else { printf("'-c' requires a parameter, default is %s\n", configDir); return -1; } } else if (strcmp(argv[i], "-C") == 0) { - global.dumpConfig = true; + dmn.dumpConfig = true; } else if (strcmp(argv[i], "-k") == 0) { - global.generateGrant = true; + dmn.generateGrant = true; } else if (strcmp(argv[i], "-A") == 0) { - global.printAuth = true; + dmn.printAuth = true; } else if (strcmp(argv[i], "-V") == 0) { - global.printVersion = true; + dmn.printVersion = true; } else { } } @@ -92,7 +92,7 @@ void dmnPrintVersion() { } int dmnReadConfig(const char *path) { - tstrncpy(configDir, global.configDir, PATH_MAX); + tstrncpy(configDir, dmn.configDir, PATH_MAX); taosInitGlobalCfg(); taosReadGlobalLogCfg(); @@ -114,12 +114,12 @@ int dmnReadConfig(const char *path) { } if (taosReadCfgFromFile() != 0) { - uError("failed to read global config"); + uError("failed to read config"); return -1; } if (taosCheckAndPrintCfg() != 0) { - uError("failed to check global config"); + uError("failed to check config"); return -1; } @@ -131,38 +131,50 @@ void dmnDumpConfig() { taosDumpGlobalCfg(); } void dmnWaitSignal() { dmnSetSignalHandle(); - while (!global.stop) { + while (!dmn.stop) { taosMsleep(100); } } -void dmnInitOption(SDnodeOpt *pOption) { - pOption->sver = 30000000; //3.0.0.0 - pOption->numOfCores = tsNumOfCores; - pOption->numOfSupportVnodes = tsNumOfSupportVnodes; - pOption->numOfCommitThreads = tsNumOfCommitThreads; - pOption->statusInterval = tsStatusInterval; - pOption->numOfThreadsPerCore = tsNumOfThreadsPerCore; - pOption->ratioOfQueryCores = tsRatioOfQueryCores; - pOption->maxShellConns = tsMaxShellConns; - pOption->shellActivityTimer = tsShellActivityTimer; - pOption->serverPort = tsServerPort; - tstrncpy(pOption->dataDir, tsDataDir, TSDB_FILENAME_LEN); - tstrncpy(pOption->localEp, tsLocalEp, TSDB_EP_LEN); - tstrncpy(pOption->localFqdn, tsLocalFqdn, TSDB_FQDN_LEN); - tstrncpy(pOption->firstEp, tsFirst, TSDB_EP_LEN); - tstrncpy(pOption->timezone, tsTimezone, TSDB_TIMEZONE_LEN); - tstrncpy(pOption->locale, tsLocale, TSDB_LOCALE_LEN); - tstrncpy(pOption->charset, tsCharset, TSDB_LOCALE_LEN); - tstrncpy(pOption->buildinfo, buildinfo, 64); - tstrncpy(pOption->gitinfo, gitinfo, 48); +void dnmInitEnvCfg(SDnodeEnvCfg *pCfg) { + pCfg->sver = 30000000; // 3.0.0.0 + pCfg->numOfCores = tsNumOfCores; + pCfg->numOfCommitThreads = tsNumOfCommitThreads; + pCfg->enableTelem = 0; + tstrncpy(pCfg->timezone, tsTimezone, TSDB_TIMEZONE_LEN); + tstrncpy(pCfg->locale, tsLocale, TSDB_LOCALE_LEN); + tstrncpy(pCfg->charset, tsCharset, TSDB_LOCALE_LEN); + tstrncpy(pCfg->buildinfo, buildinfo, 64); + tstrncpy(pCfg->gitinfo, gitinfo, 48); +} + +void dmnInitObjCfg(SDnodeObjCfg *pCfg) { + pCfg->numOfSupportVnodes = tsNumOfSupportVnodes; + pCfg->statusInterval = tsStatusInterval; + pCfg->numOfThreadsPerCore = tsNumOfThreadsPerCore; + pCfg->ratioOfQueryCores = tsRatioOfQueryCores; + pCfg->maxShellConns = tsMaxShellConns; + pCfg->shellActivityTimer = tsShellActivityTimer; + pCfg->serverPort = tsServerPort; + tstrncpy(pCfg->dataDir, tsDataDir, TSDB_FILENAME_LEN); + tstrncpy(pCfg->localEp, tsLocalEp, TSDB_EP_LEN); + tstrncpy(pCfg->localFqdn, tsLocalFqdn, TSDB_FQDN_LEN); + tstrncpy(pCfg->firstEp, tsFirst, TSDB_EP_LEN); } int dmnRunDnode() { - SDnodeOpt option = {0}; - dmnInitOption(&option); + SDnodeEnvCfg envCfg = {0}; + SDnodeObjCfg objCfg = {0}; - SDnode *pDnode = dndInit(&option); + dnmInitEnvCfg(&envCfg); + dmnInitObjCfg(&objCfg); + + if (dndInit(&envCfg) != 0) { + uInfo("Failed to start TDengine, please check the log at %s", tsLogDir); + return -1; + } + + SDnode *pDnode = dndCreate(&objCfg); if (pDnode == NULL) { uInfo("Failed to start TDengine, please check the log at %s", tsLogDir); return -1; @@ -172,7 +184,8 @@ int dmnRunDnode() { dmnWaitSignal(); uInfo("TDengine is shut down!"); - dndCleanup(pDnode); + dndClose(pDnode); + dndCleanup(); taosCloseLog(); return 0; } @@ -182,21 +195,21 @@ int main(int argc, char const *argv[]) { return -1; } - if (global.generateGrant) { + if (dmn.generateGrant) { dmnGenerateGrant(); return 0; } - if (global.printVersion) { + if (dmn.printVersion) { dmnPrintVersion(); return 0; } - if (dmnReadConfig(global.configDir) != 0) { + if (dmnReadConfig(dmn.configDir) != 0) { return -1; } - if (global.dumpConfig) { + if (dmn.dumpConfig) { dmnDumpConfig(); return 0; } diff --git a/source/dnode/mgmt/impl/inc/dndBnode.h b/source/dnode/mgmt/impl/inc/dndBnode.h index 853b54ff69..080cd2e487 100644 --- a/source/dnode/mgmt/impl/inc/dndBnode.h +++ b/source/dnode/mgmt/impl/inc/dndBnode.h @@ -19,7 +19,7 @@ #ifdef __cplusplus extern "C" { #endif -#include "dndInt.h" +#include "dndEnv.h" int32_t dndInitBnode(SDnode *pDnode); void dndCleanupBnode(SDnode *pDnode); diff --git a/source/dnode/mgmt/impl/inc/dndEnv.h b/source/dnode/mgmt/impl/inc/dndEnv.h new file mode 100644 index 0000000000..e80237bb9c --- /dev/null +++ b/source/dnode/mgmt/impl/inc/dndEnv.h @@ -0,0 +1,149 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_DND_ENV_H_ +#define _TD_DND_ENV_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include "dndInt.h" + +typedef struct { + EWorkerType type; + const char *name; + int32_t minNum; + int32_t maxNum; + void *queueFp; + SDnode *pDnode; + STaosQueue *queue; + union { + SWorkerPool pool; + SMWorkerPool mpool; + }; +} SDnodeWorker; + +typedef struct { + char *dnode; + char *mnode; + char *snode; + char *bnode; + char *vnodes; +} SDnodeDir; + +typedef struct { + int32_t dnodeId; + int32_t dropped; + int64_t clusterId; + int64_t dver; + int64_t rebootTime; + int64_t updateTime; + int8_t statusSent; + SEpSet mnodeEpSet; + char *file; + SHashObj *dnodeHash; + SDnodeEps *dnodeEps; + pthread_t *threadId; + SRWLatch latch; + SDnodeWorker mgmtWorker; + SDnodeWorker statusWorker; +} SDnodeMgmt; + +typedef struct { + int32_t refCount; + int8_t deployed; + int8_t dropped; + SMnode *pMnode; + SRWLatch latch; + SDnodeWorker readWorker; + SDnodeWorker writeWorker; + SDnodeWorker syncWorker; + int8_t replica; + int8_t selfIndex; + SReplica replicas[TSDB_MAX_REPLICA]; +} SMnodeMgmt; + +typedef struct { + int32_t refCount; + int8_t deployed; + int8_t dropped; + SQnode *pQnode; + SRWLatch latch; + SDnodeWorker queryWorker; + SDnodeWorker fetchWorker; +} SQnodeMgmt; + +typedef struct { + int32_t refCount; + int8_t deployed; + int8_t dropped; + SSnode *pSnode; + SRWLatch latch; + SDnodeWorker writeWorker; +} SSnodeMgmt; + +typedef struct { + int32_t refCount; + int8_t deployed; + int8_t dropped; + SBnode *pBnode; + SRWLatch latch; + SDnodeWorker writeWorker; +} SBnodeMgmt; + +typedef struct { + SHashObj *hash; + int32_t openVnodes; + int32_t totalVnodes; + SRWLatch latch; + SWorkerPool queryPool; + SWorkerPool fetchPool; + SMWorkerPool syncPool; + SMWorkerPool writePool; +} SVnodesMgmt; + +typedef struct { + void *serverRpc; + void *clientRpc; + DndMsgFp msgFp[TDMT_MAX]; +} STransMgmt; + +typedef struct SDnode { + EStat stat; + SDnodeObjCfg cfg; + SDnodeEnvCfg env; + SDnodeDir dir; + FileFd lockFd; + SDnodeMgmt dmgmt; + SMnodeMgmt mmgmt; + SQnodeMgmt qmgmt; + SSnodeMgmt smgmt; + SBnodeMgmt bmgmt; + SVnodesMgmt vmgmt; + STransMgmt tmgmt; + SStartupReq startup; +} SDnode; + +typedef struct { + int8_t once; + SDnodeEnvCfg cfg; +} SDnodeEnv; + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_DND_ENV_H_*/ \ No newline at end of file diff --git a/source/dnode/mgmt/impl/inc/dndInt.h b/source/dnode/mgmt/impl/inc/dndInt.h index e637b38815..c5ba494e5e 100644 --- a/source/dnode/mgmt/impl/inc/dndInt.h +++ b/source/dnode/mgmt/impl/inc/dndInt.h @@ -55,125 +55,12 @@ extern int32_t dDebugFlag; typedef enum { DND_STAT_INIT, DND_STAT_RUNNING, DND_STAT_STOPPED } EStat; typedef enum { DND_WORKER_SINGLE, DND_WORKER_MULTI } EWorkerType; +typedef enum { DND_ENV_INIT = 0, DND_ENV_READY = 1, DND_ENV_CLEANUP = 2 } EEnvStat; typedef void (*DndMsgFp)(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEps); -typedef struct { - EWorkerType type; - const char *name; - int32_t minNum; - int32_t maxNum; - void *queueFp; - SDnode *pDnode; - STaosQueue *queue; - union { - SWorkerPool pool; - SMWorkerPool mpool; - }; -} SDnodeWorker; - -typedef struct { - char *dnode; - char *mnode; - char *snode; - char *bnode; - char *vnodes; -} SDnodeDir; - -typedef struct { - int32_t dnodeId; - int32_t dropped; - int64_t clusterId; - int64_t dver; - int64_t rebootTime; - int64_t updateTime; - int8_t statusSent; - SEpSet mnodeEpSet; - char *file; - SHashObj *dnodeHash; - SDnodeEps *dnodeEps; - pthread_t *threadId; - SRWLatch latch; - SDnodeWorker mgmtWorker; - SDnodeWorker statusWorker; -} SDnodeMgmt; - -typedef struct { - int32_t refCount; - int8_t deployed; - int8_t dropped; - SMnode *pMnode; - SRWLatch latch; - SDnodeWorker readWorker; - SDnodeWorker writeWorker; - SDnodeWorker syncWorker; - int8_t replica; - int8_t selfIndex; - SReplica replicas[TSDB_MAX_REPLICA]; -} SMnodeMgmt; - -typedef struct { - int32_t refCount; - int8_t deployed; - int8_t dropped; - SQnode *pQnode; - SRWLatch latch; - SDnodeWorker queryWorker; - SDnodeWorker fetchWorker; -} SQnodeMgmt; - -typedef struct { - int32_t refCount; - int8_t deployed; - int8_t dropped; - SSnode *pSnode; - SRWLatch latch; - SDnodeWorker writeWorker; -} SSnodeMgmt; - -typedef struct { - int32_t refCount; - int8_t deployed; - int8_t dropped; - SBnode *pBnode; - SRWLatch latch; - SDnodeWorker writeWorker; -} SBnodeMgmt; - -typedef struct { - SHashObj *hash; - int32_t openVnodes; - int32_t totalVnodes; - SRWLatch latch; - SWorkerPool queryPool; - SWorkerPool fetchPool; - SMWorkerPool syncPool; - SMWorkerPool writePool; -} SVnodesMgmt; - -typedef struct { - void *serverRpc; - void *clientRpc; - DndMsgFp msgFp[TDMT_MAX]; -} STransMgmt; - -typedef struct SDnode { - EStat stat; - SDnodeOpt opt; - SDnodeDir dir; - FileFd lockFd; - SDnodeMgmt dmgmt; - SMnodeMgmt mmgmt; - SQnodeMgmt qmgmt; - SSnodeMgmt smgmt; - SBnodeMgmt bmgmt; - SVnodesMgmt vmgmt; - STransMgmt tmgmt; - SStartupReq startup; -} SDnode; - -EStat dndGetStat(SDnode *pDnode); -void dndSetStat(SDnode *pDnode, EStat stat); -char *dndStatStr(EStat stat); +EStat dndGetStat(SDnode *pDnode); +void dndSetStat(SDnode *pDnode, EStat stat); +const char *dndStatStr(EStat stat); void dndReportStartup(SDnode *pDnode, char *pName, char *pDesc); void dndGetStartup(SDnode *pDnode, SStartupReq *pStartup); diff --git a/source/dnode/mgmt/impl/inc/dndMgmt.h b/source/dnode/mgmt/impl/inc/dndMgmt.h index 0df3af23d5..9cc0c4ae66 100644 --- a/source/dnode/mgmt/impl/inc/dndMgmt.h +++ b/source/dnode/mgmt/impl/inc/dndMgmt.h @@ -19,7 +19,7 @@ #ifdef __cplusplus extern "C" { #endif -#include "dndInt.h" +#include "dndEnv.h" int32_t dndInitMgmt(SDnode *pDnode); void dndStopMgmt(SDnode *pDnode); diff --git a/source/dnode/mgmt/impl/inc/dndMnode.h b/source/dnode/mgmt/impl/inc/dndMnode.h index 0f7cec230e..dafbae10ad 100644 --- a/source/dnode/mgmt/impl/inc/dndMnode.h +++ b/source/dnode/mgmt/impl/inc/dndMnode.h @@ -19,7 +19,7 @@ #ifdef __cplusplus extern "C" { #endif -#include "dndInt.h" +#include "dndEnv.h" int32_t dndInitMnode(SDnode *pDnode); void dndCleanupMnode(SDnode *pDnode); diff --git a/source/dnode/mgmt/impl/inc/dndQnode.h b/source/dnode/mgmt/impl/inc/dndQnode.h index 677c234679..2a25dca1c6 100644 --- a/source/dnode/mgmt/impl/inc/dndQnode.h +++ b/source/dnode/mgmt/impl/inc/dndQnode.h @@ -19,7 +19,7 @@ #ifdef __cplusplus extern "C" { #endif -#include "dndInt.h" +#include "dndEnv.h" int32_t dndInitQnode(SDnode *pDnode); void dndCleanupQnode(SDnode *pDnode); diff --git a/source/dnode/mgmt/impl/inc/dndSnode.h b/source/dnode/mgmt/impl/inc/dndSnode.h index 8cb883794d..b21e9191e8 100644 --- a/source/dnode/mgmt/impl/inc/dndSnode.h +++ b/source/dnode/mgmt/impl/inc/dndSnode.h @@ -19,7 +19,7 @@ #ifdef __cplusplus extern "C" { #endif -#include "dndInt.h" +#include "dndEnv.h" int32_t dndInitSnode(SDnode *pDnode); void dndCleanupSnode(SDnode *pDnode); diff --git a/source/dnode/mgmt/impl/inc/dndTransport.h b/source/dnode/mgmt/impl/inc/dndTransport.h index 42fb379fc1..e0ea21cba8 100644 --- a/source/dnode/mgmt/impl/inc/dndTransport.h +++ b/source/dnode/mgmt/impl/inc/dndTransport.h @@ -19,7 +19,7 @@ #ifdef __cplusplus extern "C" { #endif -#include "dndInt.h" +#include "dndEnv.h" int32_t dndInitTrans(SDnode *pDnode); void dndCleanupTrans(SDnode *pDnode); diff --git a/source/dnode/mgmt/impl/inc/dndVnodes.h b/source/dnode/mgmt/impl/inc/dndVnodes.h index a78db602fe..f5177778ec 100644 --- a/source/dnode/mgmt/impl/inc/dndVnodes.h +++ b/source/dnode/mgmt/impl/inc/dndVnodes.h @@ -19,7 +19,7 @@ #ifdef __cplusplus extern "C" { #endif -#include "dndInt.h" +#include "dndEnv.h" int32_t dndInitVnodes(SDnode *pDnode); void dndCleanupVnodes(SDnode *pDnode); diff --git a/source/dnode/mgmt/impl/inc/dndWorker.h b/source/dnode/mgmt/impl/inc/dndWorker.h index 49ef88e67d..9c037d91c7 100644 --- a/source/dnode/mgmt/impl/inc/dndWorker.h +++ b/source/dnode/mgmt/impl/inc/dndWorker.h @@ -19,7 +19,7 @@ #ifdef __cplusplus extern "C" { #endif -#include "dndInt.h" +#include "dndEnv.h" int32_t dndInitWorker(SDnode *pDnode, SDnodeWorker *pWorker, EWorkerType type, const char *name, int32_t minNum, int32_t maxNum, void *queueFp); diff --git a/source/dnode/mgmt/impl/src/dndBnode.c b/source/dnode/mgmt/impl/src/dndBnode.c index 15be59a419..f26ec72f1d 100644 --- a/source/dnode/mgmt/impl/src/dndBnode.c +++ b/source/dnode/mgmt/impl/src/dndBnode.c @@ -179,7 +179,7 @@ static void dndBuildBnodeOption(SDnode *pDnode, SBnodeOpt *pOption) { pOption->sendRedirectRspFp = dndSendRedirectRsp; pOption->dnodeId = dndGetDnodeId(pDnode); pOption->clusterId = dndGetClusterId(pDnode); - pOption->cfg.sver = pDnode->opt.sver; + pOption->sver = pDnode->env.sver; } static int32_t dndOpenBnode(SDnode *pDnode) { diff --git a/source/dnode/mgmt/impl/src/dnode.c b/source/dnode/mgmt/impl/src/dndEnv.c similarity index 64% rename from source/dnode/mgmt/impl/src/dnode.c rename to source/dnode/mgmt/impl/src/dndEnv.c index f3d5e09564..1bf1ea2b92 100644 --- a/source/dnode/mgmt/impl/src/dnode.c +++ b/source/dnode/mgmt/impl/src/dndEnv.c @@ -25,6 +25,8 @@ #include "tfs.h" #include "wal.h" +static SDnodeEnv dndEnv = {0}; + EStat dndGetStat(SDnode *pDnode) { return pDnode->stat; } void dndSetStat(SDnode *pDnode, EStat stat) { @@ -32,7 +34,7 @@ void dndSetStat(SDnode *pDnode, EStat stat) { pDnode->stat = stat; } -char *dndStatStr(EStat stat) { +const char *dndStatStr(EStat stat) { switch (stat) { case DND_STAT_INIT: return "init"; @@ -79,25 +81,26 @@ static FileFd dndCheckRunning(char *dataDir) { return fd; } -static int32_t dndInitEnv(SDnode *pDnode, SDnodeOpt *pOption) { - pDnode->lockFd = dndCheckRunning(pOption->dataDir); +static int32_t dndCreateImp(SDnode *pDnode, SDnodeObjCfg *pCfg) { + pDnode->lockFd = dndCheckRunning(pCfg->dataDir); if (pDnode->lockFd < 0) { return -1; } char path[PATH_MAX + 100]; - snprintf(path, sizeof(path), "%s%smnode", pOption->dataDir, TD_DIRSEP); + snprintf(path, sizeof(path), "%s%smnode", pCfg->dataDir, TD_DIRSEP); pDnode->dir.mnode = tstrdup(path); - snprintf(path, sizeof(path), "%s%svnode", pOption->dataDir, TD_DIRSEP); + snprintf(path, sizeof(path), "%s%svnode", pCfg->dataDir, TD_DIRSEP); pDnode->dir.vnodes = tstrdup(path); - snprintf(path, sizeof(path), "%s%sdnode", pOption->dataDir, TD_DIRSEP); + snprintf(path, sizeof(path), "%s%sdnode", pCfg->dataDir, TD_DIRSEP); pDnode->dir.dnode = tstrdup(path); - snprintf(path, sizeof(path), "%s%ssnode", pOption->dataDir, TD_DIRSEP); + snprintf(path, sizeof(path), "%s%ssnode", pCfg->dataDir, TD_DIRSEP); pDnode->dir.snode = tstrdup(path); - snprintf(path, sizeof(path), "%s%sbnode", pOption->dataDir, TD_DIRSEP); + snprintf(path, sizeof(path), "%s%sbnode", pCfg->dataDir, TD_DIRSEP); pDnode->dir.bnode = tstrdup(path); - if (pDnode->dir.mnode == NULL || pDnode->dir.vnodes == NULL || pDnode->dir.dnode == NULL) { + if (pDnode->dir.mnode == NULL || pDnode->dir.vnodes == NULL || pDnode->dir.dnode == NULL || + pDnode->dir.snode == NULL || pDnode->dir.bnode == NULL) { dError("failed to malloc dir object"); terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -133,11 +136,12 @@ static int32_t dndInitEnv(SDnode *pDnode, SDnodeOpt *pOption) { return -1; } - memcpy(&pDnode->opt, pOption, sizeof(SDnodeOpt)); + memcpy(&pDnode->cfg, pCfg, sizeof(SDnodeObjCfg)); + memcpy(&pDnode->env, &dndEnv.cfg, sizeof(SDnodeEnvCfg)); return 0; } -static void dndCleanupEnv(SDnode *pDnode) { +static void dndCloseImp(SDnode *pDnode) { tfree(pDnode->dir.mnode); tfree(pDnode->dir.vnodes); tfree(pDnode->dir.dnode); @@ -149,126 +153,95 @@ static void dndCleanupEnv(SDnode *pDnode) { taosCloseFile(pDnode->lockFd); pDnode->lockFd = 0; } - - taosStopCacheRefreshWorker(); } -SDnode *dndInit(SDnodeOpt *pOption) { - taosIgnSIGPIPE(); - taosBlockSIGPIPE(); - taosResolveCRC(); +SDnode *dndCreate(SDnodeObjCfg *pCfg) { + dInfo("start to create dnode object"); SDnode *pDnode = calloc(1, sizeof(SDnode)); if (pDnode == NULL) { - dError("failed to create dnode object"); terrno = TSDB_CODE_OUT_OF_MEMORY; + dError("failed to create dnode object since %s", terrstr()); return NULL; } - dInfo("start to initialize TDengine"); dndSetStat(pDnode, DND_STAT_INIT); - if (dndInitEnv(pDnode, pOption) != 0) { - dError("failed to init env"); - dndCleanup(pDnode); - return NULL; - } - - if (rpcInit() != 0) { - dError("failed to init rpc env"); - dndCleanup(pDnode); - return NULL; - } - - if (walInit() != 0) { - dError("failed to init wal env"); - dndCleanup(pDnode); + if (dndCreateImp(pDnode, pCfg) != 0) { + dError("failed to init dnode dir since %s", terrstr()); + dndClose(pDnode); return NULL; } SDiskCfg dCfg; - strcpy(dCfg.dir, pDnode->opt.dataDir); + strcpy(dCfg.dir, pDnode->cfg.dataDir); dCfg.level = 0; dCfg.primary = 1; if (tfsInit(&dCfg, 1) != 0) { - dError("failed to init tfs env"); - dndCleanup(pDnode); - return NULL; - } - - SVnodeOpt vnodeOpt = { - .sver = pDnode->opt.sver, - .timezone = pDnode->opt.timezone, - .locale = pDnode->opt.locale, - .charset = pDnode->opt.charset, - .nthreads = pDnode->opt.numOfCommitThreads, - .putReqToVQueryQFp = dndPutReqToVQueryQ, - }; - if (vnodeInit(&vnodeOpt) != 0) { - dError("failed to init vnode env"); - dndCleanup(pDnode); + dError("failed to init tfs since %s", terrstr()); + dndClose(pDnode); return NULL; } if (dndInitMgmt(pDnode) != 0) { - dError("failed to init dnode"); - dndCleanup(pDnode); + dError("failed to init mgmt since %s", terrstr()); + dndClose(pDnode); return NULL; } if (dndInitVnodes(pDnode) != 0) { - dError("failed to init vnodes"); - dndCleanup(pDnode); + dError("failed to init vnodes since %s", terrstr()); + dndClose(pDnode); return NULL; } if (dndInitQnode(pDnode) != 0) { - dError("failed to init qnode"); - dndCleanup(pDnode); + dError("failed to init qnode since %s", terrstr()); + dndClose(pDnode); return NULL; } if (dndInitSnode(pDnode) != 0) { - dError("failed to init snode"); - dndCleanup(pDnode); + dError("failed to init snode since %s", terrstr()); + dndClose(pDnode); return NULL; } if (dndInitBnode(pDnode) != 0) { - dError("failed to init bnode"); - dndCleanup(pDnode); + dError("failed to init bnode since %s", terrstr()); + dndClose(pDnode); return NULL; } if (dndInitMnode(pDnode) != 0) { - dError("failed to init mnode"); - dndCleanup(pDnode); + dError("failed to init mnode since %s", terrstr()); + dndClose(pDnode); return NULL; } if (dndInitTrans(pDnode) != 0) { - dError("failed to init transport"); - dndCleanup(pDnode); + dError("failed to init transport since %s", terrstr()); + dndClose(pDnode); return NULL; } dndSetStat(pDnode, DND_STAT_RUNNING); dndSendStatusReq(pDnode); dndReportStartup(pDnode, "TDengine", "initialized successfully"); - dInfo("TDengine is initialized successfully, pDnode:%p", pDnode); + dInfo("dnode object is created, data:%p", pDnode); return pDnode; } -void dndCleanup(SDnode *pDnode) { +void dndClose(SDnode *pDnode) { if (pDnode == NULL) return; if (dndGetStat(pDnode) == DND_STAT_STOPPED) { - dError("dnode is shutting down"); + dError("dnode is shutting down, data:%p", pDnode); return; } - dInfo("start to cleanup TDengine"); + dInfo("start to close dnode, data:%p", pDnode); dndSetStat(pDnode, DND_STAT_STOPPED); dndCleanupTrans(pDnode); dndStopMgmt(pDnode); @@ -278,12 +251,66 @@ void dndCleanup(SDnode *pDnode) { dndCleanupQnode(pDnode); dndCleanupVnodes(pDnode); dndCleanupMgmt(pDnode); - vnodeClear(); - tfsDestroy(); + tfsCleanup(); + + dndCloseImp(pDnode); + free(pDnode); + dInfo("dnode object is closed, data:%p", pDnode); +} + +int32_t dndInit(const SDnodeEnvCfg *pCfg) { + if (atomic_val_compare_exchange_8(&dndEnv.once, DND_ENV_INIT, DND_ENV_READY) != DND_ENV_INIT) { + terrno = TSDB_CODE_REPEAT_INIT; + dError("failed to init dnode env since %s", terrstr()); + return -1; + } + + taosIgnSIGPIPE(); + taosBlockSIGPIPE(); + taosResolveCRC(); + + if (rpcInit() != 0) { + dError("failed to init rpc since %s", terrstr()); + dndCleanup(); + return -1; + } + + if (walInit() != 0) { + dError("failed to init wal since %s", terrstr()); + dndCleanup(); + return -1; + } + + SVnodeOpt vnodeOpt = { + .sver = pCfg->sver, + .timezone = pCfg->timezone, + .locale = pCfg->locale, + .charset = pCfg->charset, + .nthreads = pCfg->numOfCommitThreads, + .putReqToVQueryQFp = dndPutReqToVQueryQ, + }; + + if (vnodeInit(&vnodeOpt) != 0) { + dError("failed to init vnode since %s", terrstr()); + dndCleanup(); + return NULL; + } + + memcpy(&dndEnv.cfg, pCfg, sizeof(SDnodeEnvCfg)); + dInfo("dnode env is initialized"); + return 0; +} + +void dndCleanup() { + if (atomic_val_compare_exchange_8(&dndEnv.once, DND_ENV_READY, DND_ENV_CLEANUP) != DND_ENV_READY) { + dError("dnode env is already cleaned up"); + return; + } + walCleanUp(); + vnodeCleanup(); rpcCleanup(); - dndCleanupEnv(pDnode); - free(pDnode); - dInfo("TDengine is cleaned up successfully"); -} + taosStopCacheRefreshWorker(); + dInfo("dnode env is cleaned up"); +} \ No newline at end of file diff --git a/source/dnode/mgmt/impl/src/dndMgmt.c b/source/dnode/mgmt/impl/src/dndMgmt.c index d9edf39b73..507f9a2644 100644 --- a/source/dnode/mgmt/impl/src/dndMgmt.c +++ b/source/dnode/mgmt/impl/src/dndMgmt.c @@ -86,7 +86,7 @@ void dndSendRedirectRsp(SDnode *pDnode, SRpcMsg *pReq) { dDebug("RPC %p, req:%s is redirected, num:%d use:%d", pReq->handle, TMSG_INFO(msgType), epSet.numOfEps, epSet.inUse); for (int32_t i = 0; i < epSet.numOfEps; ++i) { dDebug("mnode index:%d %s:%u", i, epSet.fqdn[i], epSet.port[i]); - if (strcmp(epSet.fqdn[i], pDnode->opt.localFqdn) == 0 && epSet.port[i] == pDnode->opt.serverPort) { + if (strcmp(epSet.fqdn[i], pDnode->cfg.localFqdn) == 0 && epSet.port[i] == pDnode->cfg.serverPort) { epSet.inUse = (i + 1) % epSet.numOfEps; } @@ -289,8 +289,8 @@ PRASE_DNODE_OVER: if (root != NULL) cJSON_Delete(root); if (fp != NULL) fclose(fp); - if (dndIsEpChanged(pDnode, pMgmt->dnodeId, pDnode->opt.localEp)) { - dError("localEp %s different with %s and need reconfigured", pDnode->opt.localEp, pMgmt->file); + if (dndIsEpChanged(pDnode, pMgmt->dnodeId, pDnode->cfg.localEp)) { + dError("localEp %s different with %s and need reconfigured", pDnode->cfg.localEp, pMgmt->file); return -1; } @@ -298,7 +298,7 @@ PRASE_DNODE_OVER: pMgmt->dnodeEps = calloc(1, sizeof(SDnodeEps) + sizeof(SDnodeEp)); pMgmt->dnodeEps->num = 1; pMgmt->dnodeEps->eps[0].isMnode = 1; - taosGetFqdnPortFromEp(pDnode->opt.firstEp, pMgmt->dnodeEps->eps[0].fqdn, &pMgmt->dnodeEps->eps[0].port); + taosGetFqdnPortFromEp(pDnode->cfg.firstEp, pMgmt->dnodeEps->eps[0].fqdn, &pMgmt->dnodeEps->eps[0].port); } dndResetDnodes(pDnode, pMgmt->dnodeEps); @@ -362,24 +362,24 @@ void dndSendStatusReq(SDnode *pDnode) { SDnodeMgmt *pMgmt = &pDnode->dmgmt; taosRLockLatch(&pMgmt->latch); - pStatus->sver = htonl(pDnode->opt.sver); + pStatus->sver = htonl(pDnode->env.sver); pStatus->dver = htobe64(pMgmt->dver); pStatus->dnodeId = htonl(pMgmt->dnodeId); pStatus->clusterId = htobe64(pMgmt->clusterId); pStatus->rebootTime = htobe64(pMgmt->rebootTime); pStatus->updateTime = htobe64(pMgmt->updateTime); - pStatus->numOfCores = htonl(pDnode->opt.numOfCores); - pStatus->numOfSupportVnodes = htonl(pDnode->opt.numOfSupportVnodes); - tstrncpy(pStatus->dnodeEp, pDnode->opt.localEp, TSDB_EP_LEN); + pStatus->numOfCores = htonl(pDnode->env.numOfCores); + pStatus->numOfSupportVnodes = htonl(pDnode->cfg.numOfSupportVnodes); + tstrncpy(pStatus->dnodeEp, pDnode->cfg.localEp, TSDB_EP_LEN); - pStatus->clusterCfg.statusInterval = htonl(pDnode->opt.statusInterval); + pStatus->clusterCfg.statusInterval = htonl(pDnode->cfg.statusInterval); pStatus->clusterCfg.checkTime = 0; char timestr[32] = "1970-01-01 00:00:00.00"; (void)taosParseTime(timestr, &pStatus->clusterCfg.checkTime, (int32_t)strlen(timestr), TSDB_TIME_PRECISION_MILLI, 0); pStatus->clusterCfg.checkTime = htonl(pStatus->clusterCfg.checkTime); - tstrncpy(pStatus->clusterCfg.timezone, pDnode->opt.timezone, TSDB_TIMEZONE_LEN); - tstrncpy(pStatus->clusterCfg.locale, pDnode->opt.locale, TSDB_LOCALE_LEN); - tstrncpy(pStatus->clusterCfg.charset, pDnode->opt.charset, TSDB_LOCALE_LEN); + tstrncpy(pStatus->clusterCfg.timezone, pDnode->env.timezone, TSDB_TIMEZONE_LEN); + tstrncpy(pStatus->clusterCfg.locale, pDnode->env.locale, TSDB_LOCALE_LEN); + tstrncpy(pStatus->clusterCfg.charset, pDnode->env.charset, TSDB_LOCALE_LEN); taosRUnLockLatch(&pMgmt->latch); dndGetVnodeLoads(pDnode, &pStatus->vnodeLoads); @@ -485,7 +485,7 @@ void dndProcessStartupReq(SDnode *pDnode, SRpcMsg *pReq) { static void *dnodeThreadRoutine(void *param) { SDnode *pDnode = param; SDnodeMgmt *pMgmt = &pDnode->dmgmt; - int32_t ms = pDnode->opt.statusInterval * 1000; + int32_t ms = pDnode->cfg.statusInterval * 1000; while (true) { pthread_testcancel(); diff --git a/source/dnode/mgmt/impl/src/dndMnode.c b/source/dnode/mgmt/impl/src/dndMnode.c index 6c23af7f00..32730fc1e3 100644 --- a/source/dnode/mgmt/impl/src/dndMnode.c +++ b/source/dnode/mgmt/impl/src/dndMnode.c @@ -247,7 +247,7 @@ static bool dndNeedDeployMnode(SDnode *pDnode) { return false; } - if (strcmp(pDnode->opt.localEp, pDnode->opt.firstEp) != 0) { + if (strcmp(pDnode->cfg.localEp, pDnode->cfg.firstEp) != 0) { return false; } @@ -266,15 +266,15 @@ static void dndInitMnodeOption(SDnode *pDnode, SMnodeOpt *pOption) { pOption->putReqToMWriteQFp = dndPutMsgToMWriteQ; pOption->dnodeId = dndGetDnodeId(pDnode); pOption->clusterId = dndGetClusterId(pDnode); - pOption->cfg.sver = pDnode->opt.sver; - pOption->cfg.enableTelem = pDnode->opt.enableTelem; - pOption->cfg.statusInterval = pDnode->opt.statusInterval; - pOption->cfg.shellActivityTimer = pDnode->opt.shellActivityTimer; - pOption->cfg.timezone = pDnode->opt.timezone; - pOption->cfg.charset = pDnode->opt.charset; - pOption->cfg.locale = pDnode->opt.locale; - pOption->cfg.gitinfo = pDnode->opt.gitinfo; - pOption->cfg.buildinfo = pDnode->opt.buildinfo; + pOption->cfg.sver = pDnode->env.sver; + pOption->cfg.enableTelem = pDnode->env.enableTelem; + pOption->cfg.statusInterval = pDnode->cfg.statusInterval; + pOption->cfg.shellActivityTimer = pDnode->cfg.shellActivityTimer; + pOption->cfg.timezone = pDnode->env.timezone; + pOption->cfg.charset = pDnode->env.charset; + pOption->cfg.locale = pDnode->env.locale; + pOption->cfg.gitinfo = pDnode->env.gitinfo; + pOption->cfg.buildinfo = pDnode->env.buildinfo; } static void dndBuildMnodeDeployOption(SDnode *pDnode, SMnodeOpt *pOption) { @@ -283,8 +283,8 @@ static void dndBuildMnodeDeployOption(SDnode *pDnode, SMnodeOpt *pOption) { pOption->selfIndex = 0; SReplica *pReplica = &pOption->replicas[0]; pReplica->id = 1; - pReplica->port = pDnode->opt.serverPort; - memcpy(pReplica->fqdn, pDnode->opt.localFqdn, TSDB_FQDN_LEN); + pReplica->port = pDnode->cfg.serverPort; + memcpy(pReplica->fqdn, pDnode->cfg.localFqdn, TSDB_FQDN_LEN); SMnodeMgmt *pMgmt = &pDnode->mmgmt; pMgmt->selfIndex = pOption->selfIndex; diff --git a/source/dnode/mgmt/impl/src/dndQnode.c b/source/dnode/mgmt/impl/src/dndQnode.c index 9d2f623c45..fa53375381 100644 --- a/source/dnode/mgmt/impl/src/dndQnode.c +++ b/source/dnode/mgmt/impl/src/dndQnode.c @@ -185,7 +185,7 @@ static void dndBuildQnodeOption(SDnode *pDnode, SQnodeOpt *pOption) { pOption->sendRedirectRspFp = dndSendRedirectRsp; pOption->dnodeId = dndGetDnodeId(pDnode); pOption->clusterId = dndGetClusterId(pDnode); - pOption->cfg.sver = pDnode->opt.sver; + pOption->sver = pDnode->env.sver; } static int32_t dndOpenQnode(SDnode *pDnode) { diff --git a/source/dnode/mgmt/impl/src/dndSnode.c b/source/dnode/mgmt/impl/src/dndSnode.c index 00435d4c3e..6f22e5b00c 100644 --- a/source/dnode/mgmt/impl/src/dndSnode.c +++ b/source/dnode/mgmt/impl/src/dndSnode.c @@ -179,7 +179,7 @@ static void dndBuildSnodeOption(SDnode *pDnode, SSnodeOpt *pOption) { pOption->sendRedirectRspFp = dndSendRedirectRsp; pOption->dnodeId = dndGetDnodeId(pDnode); pOption->clusterId = dndGetClusterId(pDnode); - pOption->cfg.sver = pDnode->opt.sver; + pOption->sver = pDnode->env.sver; } static int32_t dndOpenSnode(SDnode *pDnode) { diff --git a/source/dnode/mgmt/impl/src/dndTransport.c b/source/dnode/mgmt/impl/src/dndTransport.c index 509e8f4cab..ab4ae4ac53 100644 --- a/source/dnode/mgmt/impl/src/dndTransport.c +++ b/source/dnode/mgmt/impl/src/dndTransport.c @@ -176,7 +176,7 @@ static int32_t dndInitClient(SDnode *pDnode) { rpcInit.cfp = dndProcessResponse; rpcInit.sessions = 1024; rpcInit.connType = TAOS_CONN_CLIENT; - rpcInit.idleTime = pDnode->opt.shellActivityTimer * 1000; + rpcInit.idleTime = pDnode->cfg.shellActivityTimer * 1000; rpcInit.user = INTERNAL_USER; rpcInit.ckey = INTERNAL_CKEY; rpcInit.secret = INTERNAL_SECRET; @@ -325,20 +325,20 @@ static int32_t dndInitServer(SDnode *pDnode) { STransMgmt *pMgmt = &pDnode->tmgmt; dndInitMsgFp(pMgmt); - int32_t numOfThreads = (int32_t)((pDnode->opt.numOfCores * pDnode->opt.numOfThreadsPerCore) / 2.0); + int32_t numOfThreads = (int32_t)((pDnode->env.numOfCores * pDnode->cfg.numOfThreadsPerCore) / 2.0); if (numOfThreads < 1) { numOfThreads = 1; } SRpcInit rpcInit; memset(&rpcInit, 0, sizeof(rpcInit)); - rpcInit.localPort = pDnode->opt.serverPort; + rpcInit.localPort = pDnode->cfg.serverPort; rpcInit.label = "DND-S"; rpcInit.numOfThreads = numOfThreads; rpcInit.cfp = dndProcessRequest; - rpcInit.sessions = pDnode->opt.maxShellConns; + rpcInit.sessions = pDnode->cfg.maxShellConns; rpcInit.connType = TAOS_CONN_SERVER; - rpcInit.idleTime = pDnode->opt.shellActivityTimer * 1000; + rpcInit.idleTime = pDnode->cfg.shellActivityTimer * 1000; rpcInit.afp = dndRetrieveUserAuthInfo; rpcInit.parent = pDnode; diff --git a/source/dnode/mgmt/impl/src/dndVnodes.c b/source/dnode/mgmt/impl/src/dndVnodes.c index 2d7999fe5a..156009fa68 100644 --- a/source/dnode/mgmt/impl/src/dndVnodes.c +++ b/source/dnode/mgmt/impl/src/dndVnodes.c @@ -420,7 +420,7 @@ static int32_t dndOpenVnodes(SDnode *pDnode) { pMgmt->totalVnodes = numOfVnodes; - int32_t threadNum = pDnode->opt.numOfCores; + int32_t threadNum = pDnode->env.numOfCores; int32_t vnodesPerThread = numOfVnodes / threadNum + 1; SVnodeThread *threads = calloc(threadNum, sizeof(SVnodeThread)); @@ -904,11 +904,11 @@ static int32_t dndInitVnodeWorkers(SDnode *pDnode) { SVnodesMgmt *pMgmt = &pDnode->vmgmt; int32_t maxFetchThreads = 4; - int32_t minFetchThreads = MIN(maxFetchThreads, pDnode->opt.numOfCores); - int32_t minQueryThreads = MAX((int32_t)(pDnode->opt.numOfCores * pDnode->opt.ratioOfQueryCores), 1); + int32_t minFetchThreads = MIN(maxFetchThreads, pDnode->env.numOfCores); + int32_t minQueryThreads = MAX((int32_t)(pDnode->env.numOfCores * pDnode->cfg.ratioOfQueryCores), 1); int32_t maxQueryThreads = minQueryThreads; - int32_t maxWriteThreads = MAX(pDnode->opt.numOfCores, 1); - int32_t maxSyncThreads = MAX(pDnode->opt.numOfCores / 2, 1); + int32_t maxWriteThreads = MAX(pDnode->env.numOfCores, 1); + int32_t maxSyncThreads = MAX(pDnode->env.numOfCores / 2, 1); SWorkerPool *pPool = &pMgmt->queryPool; pPool->name = "vnode-query"; diff --git a/source/dnode/mgmt/impl/test/sut/inc/server.h b/source/dnode/mgmt/impl/test/sut/inc/server.h index 5f9e4846a7..99554a7aa7 100644 --- a/source/dnode/mgmt/impl/test/sut/inc/server.h +++ b/source/dnode/mgmt/impl/test/sut/inc/server.h @@ -24,7 +24,7 @@ class TestServer { bool DoStart(); private: - SDnodeOpt BuildOption(const char* path, const char* fqdn, uint16_t port, const char* firstEp); + SDnodeObjCfg BuildOption(const char* path, const char* fqdn, uint16_t port, const char* firstEp); private: SDnode* pDnode; diff --git a/source/dnode/mgmt/impl/test/sut/src/server.cpp b/source/dnode/mgmt/impl/test/sut/src/server.cpp index fb2974294c..f7b5ab4599 100644 --- a/source/dnode/mgmt/impl/test/sut/src/server.cpp +++ b/source/dnode/mgmt/impl/test/sut/src/server.cpp @@ -22,30 +22,27 @@ void* serverLoop(void* param) { } } -SDnodeOpt TestServer::BuildOption(const char* path, const char* fqdn, uint16_t port, const char* firstEp) { - SDnodeOpt option = {0}; - option.sver = 1; - option.numOfCores = 1; - option.numOfSupportVnodes = 16; - option.numOfCommitThreads = 1; - option.statusInterval = 1; - option.numOfThreadsPerCore = 1; - option.ratioOfQueryCores = 1; - option.maxShellConns = 1000; - option.shellActivityTimer = 30; - option.serverPort = port; - strcpy(option.dataDir, path); - snprintf(option.localEp, TSDB_EP_LEN, "%s:%u", fqdn, port); - snprintf(option.localFqdn, TSDB_FQDN_LEN, "%s", fqdn); - snprintf(option.firstEp, TSDB_EP_LEN, "%s", firstEp); - return option; +SDnodeObjCfg TestServer::BuildOption(const char* path, const char* fqdn, uint16_t port, const char* firstEp) { + SDnodeObjCfg cfg = {0}; + cfg.numOfSupportVnodes = 16; + cfg.statusInterval = 1; + cfg.numOfThreadsPerCore = 1; + cfg.ratioOfQueryCores = 1; + cfg.maxShellConns = 1000; + cfg.shellActivityTimer = 30; + cfg.serverPort = port; + strcpy(cfg.dataDir, path); + snprintf(cfg.localEp, TSDB_EP_LEN, "%s:%u", fqdn, port); + snprintf(cfg.localFqdn, TSDB_FQDN_LEN, "%s", fqdn); + snprintf(cfg.firstEp, TSDB_EP_LEN, "%s", firstEp); + return cfg; } bool TestServer::DoStart() { - SDnodeOpt option = BuildOption(path, fqdn, port, firstEp); + SDnodeObjCfg cfg = BuildOption(path, fqdn, port, firstEp); taosMkDir(path); - pDnode = dndInit(&option); + pDnode = dndCreate(&cfg); if (pDnode != NULL) { return false; } @@ -81,7 +78,7 @@ void TestServer::Stop() { } if (pDnode != NULL) { - dndCleanup(pDnode); + dndClose(pDnode); pDnode = NULL; } } diff --git a/source/dnode/mgmt/impl/test/sut/src/sut.cpp b/source/dnode/mgmt/impl/test/sut/src/sut.cpp index ba530dbdd0..c6e6c0006d 100644 --- a/source/dnode/mgmt/impl/test/sut/src/sut.cpp +++ b/source/dnode/mgmt/impl/test/sut/src/sut.cpp @@ -43,6 +43,11 @@ void Testbase::InitLog(const char* path) { } void Testbase::Init(const char* path, int16_t port) { + SDnodeEnvCfg cfg = {0}; + cfg.numOfCommitThreads = 1; + cfg.numOfCores = 1; + dndInit(&cfg); + char fqdn[] = "localhost"; char firstEp[TSDB_EP_LEN] = {0}; snprintf(firstEp, TSDB_EP_LEN, "%s:%u", fqdn, port); @@ -56,6 +61,7 @@ void Testbase::Init(const char* path, int16_t port) { void Testbase::Cleanup() { server.Stop(); client.Cleanup(); + dndCleanup(); } void Testbase::Restart() { server.Restart(); } diff --git a/source/dnode/qnode/inc/qndInt.h b/source/dnode/qnode/inc/qndInt.h index 529c407efa..0f8e65de7e 100644 --- a/source/dnode/qnode/inc/qndInt.h +++ b/source/dnode/qnode/inc/qndInt.h @@ -29,12 +29,7 @@ extern "C" { #endif typedef struct SQnode { - int32_t dnodeId; - int64_t clusterId; - SQnodeCfg cfg; - SendReqToDnodeFp sendReqToDnodeFp; - SendReqToMnodeFp sendReqToMnodeFp; - SendRedirectRspFp sendRedirectRspFp; + SQnodeOpt opt; } SQnode; #ifdef __cplusplus diff --git a/source/dnode/snode/inc/sndInt.h b/source/dnode/snode/inc/sndInt.h index 3b41c7f4b1..aff82e4ae7 100644 --- a/source/dnode/snode/inc/sndInt.h +++ b/source/dnode/snode/inc/sndInt.h @@ -29,12 +29,7 @@ extern "C" { #endif typedef struct SSnode { - int32_t dnodeId; - int64_t clusterId; - SSnodeCfg cfg; - SendReqToDnodeFp sendReqToDnodeFp; - SendReqToMnodeFp sendReqToMnodeFp; - SendRedirectRspFp sendRedirectRspFp; + SSnodeOpt cfg; } SSnode; #ifdef __cplusplus diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index a4a9cff002..9dc4bb1873 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -1,4 +1,33 @@ -add_subdirectory(meta) -add_subdirectory(tq) -add_subdirectory(tsdb) -add_subdirectory(impl) \ No newline at end of file +aux_source_directory(src/meta META_SRC) +aux_source_directory(src/tq TQ_SRC) +aux_source_directory(src/tsdb TSDB_SRC) +aux_source_directory(src/vnd VND_SRC) +list(APPEND + VNODE_SRC + ${META_SRC} + ${TQ_SRC} + ${TSDB_SRC} + ${VND_SRC} +) + +add_library(vnode STATIC ${VNODE_SRC}) +target_include_directories( + vnode + PUBLIC inc + PRIVATE src/inc +) +target_link_libraries( + vnode + PUBLIC os + PUBLIC util + PUBLIC common + PUBLIC transport + PUBLIC bdb + PUBLIC tfs + PUBLIC wal + PUBLIC qworker +) + +if(${BUILD_TEST}) + # add_subdirectory(test) +endif(${BUILD_TEST}) diff --git a/source/dnode/vnode/impl/test/CMakeLists.txt b/source/dnode/vnode/impl/test/CMakeLists.txt deleted file mode 100644 index e1226331e9..0000000000 --- a/source/dnode/vnode/impl/test/CMakeLists.txt +++ /dev/null @@ -1,12 +0,0 @@ -# Vnode API test -add_executable(vnodeApiTests "") -target_sources(vnodeApiTests - PRIVATE - "vnodeApiTests.cpp" -) -target_link_libraries(vnodeApiTests vnode gtest gtest_main) - -add_test( - NAME vnode_api_tests - COMMAND ${CMAKE_CURRENT_BINARY_DIR}/vnodeApiTests - ) \ No newline at end of file diff --git a/source/dnode/vnode/impl/test/vBenchmarkTest.cpp b/source/dnode/vnode/impl/test/vBenchmarkTest.cpp deleted file mode 100644 index e218886231..0000000000 --- a/source/dnode/vnode/impl/test/vBenchmarkTest.cpp +++ /dev/null @@ -1,2 +0,0 @@ -// https://stackoverflow.com/questions/8565666/benchmarking-with-googletest -// https://github.com/google/benchmark \ No newline at end of file diff --git a/source/dnode/vnode/impl/test/vnodeApiTests.cpp b/source/dnode/vnode/impl/test/vnodeApiTests.cpp deleted file mode 100644 index 71ffdd1d0f..0000000000 --- a/source/dnode/vnode/impl/test/vnodeApiTests.cpp +++ /dev/null @@ -1,285 +0,0 @@ -/** - * @file vnodeApiTests.cpp - * @author hzcheng (hzcheng@taosdata.com) - * @brief VNODE module API tests - * @version 0.1 - * @date 2021-12-13 - * - * @copyright Copyright (c) 2021 - * - */ - -#include -#include - -#include "vnode.h" - -static STSchema *vtCreateBasicSchema() { - STSchemaBuilder sb; - STSchema * pSchema = NULL; - - tdInitTSchemaBuilder(&sb, 0); - - tdAddColToSchema(&sb, TSDB_DATA_TYPE_TIMESTAMP, 0, 0); - for (int i = 1; i < 10; i++) { - tdAddColToSchema(&sb, TSDB_DATA_TYPE_INT, i, 0); - } - - pSchema = tdGetSchemaFromBuilder(&sb); - - tdDestroyTSchemaBuilder(&sb); - - return pSchema; -} - -static STSchema *vtCreateBasicTagSchema() { - STSchemaBuilder sb; - STSchema * pSchema = NULL; - - tdInitTSchemaBuilder(&sb, 0); - - tdAddColToSchema(&sb, TSDB_DATA_TYPE_TIMESTAMP, 0, 0); - for (int i = 10; i < 12; i++) { - tdAddColToSchema(&sb, TSDB_DATA_TYPE_BINARY, i, 20); - } - - pSchema = tdGetSchemaFromBuilder(&sb); - - tdDestroyTSchemaBuilder(&sb); - - return pSchema; -} - -static SKVRow vtCreateBasicTag() { - SKVRowBuilder rb; - SKVRow pTag; - - tdInitKVRowBuilder(&rb); - - for (int i = 0; i < 2; i++) { - void *pVal = malloc(sizeof(VarDataLenT) + strlen("foo")); - varDataLen(pVal) = strlen("foo"); - memcpy(varDataVal(pVal), "foo", strlen("foo")); - - tdAddColToKVRow(&rb, i, TSDB_DATA_TYPE_BINARY, pVal); - free(pVal); - } - - pTag = tdGetKVRowFromBuilder(&rb); - tdDestroyKVRowBuilder(&rb); - - return pTag; -} - -static void vtBuildCreateStbReq(tb_uid_t suid, char *tbname, SRpcMsg **ppMsg) { - SRpcMsg * pMsg; - STSchema *pSchema; - STSchema *pTagSchema; - int zs; - void * pBuf; - - pSchema = vtCreateBasicSchema(); - pTagSchema = vtCreateBasicTagSchema(); - - SVnodeReq vCreateSTbReq; - vnodeSetCreateStbReq(&vCreateSTbReq, tbname, UINT32_MAX, UINT32_MAX, suid, pSchema, pTagSchema); - - zs = vnodeBuildReq(NULL, &vCreateSTbReq, TDMT_VND_CREATE_STB); - pMsg = (SRpcMsg *)malloc(sizeof(SRpcMsg) + zs); - pMsg->msgType = TDMT_VND_CREATE_STB; - pMsg->contLen = zs; - pMsg->pCont = POINTER_SHIFT(pMsg, sizeof(SRpcMsg)); - - pBuf = pMsg->pCont; - vnodeBuildReq(&pBuf, &vCreateSTbReq, TDMT_VND_CREATE_STB); - META_CLEAR_TB_CFG(&vCreateSTbReq); - - tdFreeSchema(pSchema); - tdFreeSchema(pTagSchema); - - *ppMsg = pMsg; -} - -static void vtBuildCreateCtbReq(tb_uid_t suid, char *tbname, SRpcMsg **ppMsg) { - SRpcMsg *pMsg; - int tz; - SKVRow pTag = vtCreateBasicTag(); - - SVnodeReq vCreateCTbReq; - vnodeSetCreateCtbReq(&vCreateCTbReq, tbname, UINT32_MAX, UINT32_MAX, suid, pTag); - - tz = vnodeBuildReq(NULL, &vCreateCTbReq, TDMT_VND_CREATE_TABLE); - pMsg = (SRpcMsg *)malloc(sizeof(SRpcMsg) + tz); - pMsg->msgType = TDMT_VND_CREATE_TABLE; - pMsg->contLen = tz; - pMsg->pCont = POINTER_SHIFT(pMsg, sizeof(*pMsg)); - void *pBuf = pMsg->pCont; - - vnodeBuildReq(&pBuf, &vCreateCTbReq, TDMT_VND_CREATE_TABLE); - META_CLEAR_TB_CFG(&vCreateCTbReq); - free(pTag); - - *ppMsg = pMsg; -} - -static void vtBuildCreateNtbReq(char *tbname, SRpcMsg **ppMsg) { - // TODO -} - -static void vtBuildSubmitReq(SRpcMsg **ppMsg) { - SRpcMsg * pMsg; - SSubmitMsg *pSubmitMsg; - SSubmitBlk *pSubmitBlk; - int tz = 1024; // TODO - - pMsg = (SRpcMsg *)malloc(sizeof(*pMsg) + tz); - pMsg->msgType = TDMT_VND_SUBMIT; - pMsg->contLen = tz; - pMsg->pCont = POINTER_SHIFT(pMsg, sizeof(*pMsg)); - - // For submit msg header - pSubmitMsg = (SSubmitMsg *)(pMsg->pCont); - // pSubmitMsg->header.contLen = 0; - // pSubmitMsg->header.vgId = 0; - // pSubmitMsg->length = 0; - pSubmitMsg->numOfBlocks = 1; - - // For submit blk - pSubmitBlk = (SSubmitBlk *)(pSubmitMsg->blocks); - pSubmitBlk->uid = 0; - pSubmitBlk->tid = 0; - pSubmitBlk->padding = 0; - pSubmitBlk->sversion = 0; - pSubmitBlk->dataLen = 0; - pSubmitBlk->numOfRows = 0; - - // For row batch - - *ppMsg = pMsg; -} - -static void vtClearMsgBatch(SArray *pMsgArr) { - SRpcMsg *pMsg; - for (size_t i = 0; i < taosArrayGetSize(pMsgArr); i++) { - pMsg = *(SRpcMsg **)taosArrayGet(pMsgArr, i); - free(pMsg); - } - - taosArrayClear(pMsgArr); -} - -static void vtProcessAndApplyReqs(SVnode *pVnode, SArray *pMsgArr) { - int rcode; - SRpcMsg *pReq; - SRpcMsg *pRsp; - - rcode = vnodeProcessWMsgs(pVnode, pMsgArr); - GTEST_ASSERT_EQ(rcode, 0); - - for (size_t i = 0; i < taosArrayGetSize(pMsgArr); i++) { - pReq = *(SRpcMsg **)taosArrayGet(pMsgArr, i); - rcode = vnodeApplyWMsg(pVnode, pReq, NULL); - GTEST_ASSERT_EQ(rcode, 0); - } -} - -TEST(vnodeApiTest, vnode_simple_create_table_test) { - tb_uid_t suid = 1638166374163; - SRpcMsg *pMsg; - SArray * pMsgArr = NULL; - SVnode * pVnode; - int rcode; - int ntables = 1000000; - int batch = 10; - char tbname[128]; - - pMsgArr = (SArray *)taosArrayInit(batch, sizeof(pMsg)); - - vnodeDestroy("vnode1"); - GTEST_ASSERT_GE(vnodeInit(2), 0); - - // CREATE AND OPEN A VNODE - pVnode = vnodeOpen("vnode1", NULL); - ASSERT_NE(pVnode, nullptr); - - // CREATE A SUPER TABLE - sprintf(tbname, "st"); - vtBuildCreateStbReq(suid, tbname, &pMsg); - taosArrayPush(pMsgArr, &pMsg); - vtProcessAndApplyReqs(pVnode, pMsgArr); - vtClearMsgBatch(pMsgArr); - - // CREATE A LOT OF CHILD TABLES - for (int i = 0; i < ntables / batch; i++) { - // Build request batch - for (int j = 0; j < batch; j++) { - sprintf(tbname, "ct%d", i * batch + j + 1); - vtBuildCreateCtbReq(suid, tbname, &pMsg); - taosArrayPush(pMsgArr, &pMsg); - } - - // Process request batch - vtProcessAndApplyReqs(pVnode, pMsgArr); - - // Clear request batch - vtClearMsgBatch(pMsgArr); - } - - // CLOSE THE VNODE - vnodeClose(pVnode); - vnodeClear(); - - taosArrayDestroy(pMsgArr); -} - -TEST(vnodeApiTest, vnode_simple_insert_test) { - const char *vname = "vnode2"; - char tbname[128]; - tb_uid_t suid = 1638166374163; - SRpcMsg * pMsg; - SArray * pMsgArr; - int rcode; - SVnode * pVnode; - int batch = 1; - int loop = 1000000; - - pMsgArr = (SArray *)taosArrayInit(0, sizeof(pMsg)); - - vnodeDestroy(vname); - - GTEST_ASSERT_GE(vnodeInit(2), 0); - - // Open a vnode - pVnode = vnodeOpen(vname, NULL); - GTEST_ASSERT_NE(pVnode, nullptr); - - // 1. CREATE A SUPER TABLE - sprintf(tbname, "st"); - vtBuildCreateStbReq(suid, tbname, &pMsg); - taosArrayPush(pMsgArr, &pMsg); - vtProcessAndApplyReqs(pVnode, pMsgArr); - vtClearMsgBatch(pMsgArr); - - // 2. CREATE A CHILD TABLE - sprintf(tbname, "t0"); - vtBuildCreateCtbReq(suid, tbname, &pMsg); - taosArrayPush(pMsgArr, &pMsg); - vtProcessAndApplyReqs(pVnode, pMsgArr); - vtClearMsgBatch(pMsgArr); - - // 3. WRITE A LOT OF TIME-SERIES DATA - for (int j = 0; j < loop; j++) { - for (int i = 0; i < batch; i++) { - vtBuildSubmitReq(&pMsg); - taosArrayPush(pMsgArr, &pMsg); - } - vtProcessAndApplyReqs(pVnode, pMsgArr); - vtClearMsgBatch(pMsgArr); - } - - // Close the vnode - vnodeClose(pVnode); - vnodeClear(); - - taosArrayDestroy(pMsgArr); -} \ No newline at end of file diff --git a/include/dnode/vnode/meta/meta.h b/source/dnode/vnode/inc/meta.h similarity index 100% rename from include/dnode/vnode/meta/meta.h rename to source/dnode/vnode/inc/meta.h diff --git a/include/dnode/vnode/tq/tq.h b/source/dnode/vnode/inc/tq.h similarity index 100% rename from include/dnode/vnode/tq/tq.h rename to source/dnode/vnode/inc/tq.h diff --git a/include/dnode/vnode/tsdb/tsdb.h b/source/dnode/vnode/inc/tsdb.h similarity index 100% rename from include/dnode/vnode/tsdb/tsdb.h rename to source/dnode/vnode/inc/tsdb.h diff --git a/include/dnode/vnode/vnode.h b/source/dnode/vnode/inc/vnode.h similarity index 98% rename from include/dnode/vnode/vnode.h rename to source/dnode/vnode/inc/vnode.h index 4332fc8e58..6113d0a536 100644 --- a/include/dnode/vnode/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -89,10 +89,10 @@ typedef struct { int vnodeInit(const SVnodeOpt *pOption); /** - * @brief clear a vnode + * @brief Cleanup the vnode module * */ -void vnodeClear(); +void vnodeCleanup(); /** * @brief Open a VNODE. diff --git a/source/dnode/vnode/meta/src/metaSQLiteImpl.c b/source/dnode/vnode/meta/src/metaSQLiteImpl.c deleted file mode 100644 index fe9ef22fb3..0000000000 --- a/source/dnode/vnode/meta/src/metaSQLiteImpl.c +++ /dev/null @@ -1,212 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "metaDef.h" -#include "sqlite3.h" - -struct SMetaDB { - sqlite3 *pDB; -}; - -int metaOpenDB(SMeta *pMeta) { - char dir[128]; - int rc; - char *err = NULL; - - pMeta->pDB = (SMetaDB *)calloc(1, sizeof(SMetaDB)); - if (pMeta->pDB == NULL) { - // TODO: handle error - return -1; - } - - sprintf(dir, "%s/meta.db", pMeta->path); - rc = sqlite3_open(dir, &(pMeta->pDB->pDB)); - if (rc != SQLITE_OK) { - // TODO: handle error - printf("failed to open meta.db\n"); - } - - // For all tables - rc = sqlite3_exec(pMeta->pDB->pDB, - "CREATE TABLE IF NOT EXISTS tb (" - " tbname VARCHAR(256) NOT NULL UNIQUE," - " tb_uid INTEGER NOT NULL UNIQUE " - ");", - NULL, NULL, &err); - if (rc != SQLITE_OK) { - // TODO: handle error - printf("failed to create meta table tb since %s\n", err); - } - - // For super tables - rc = sqlite3_exec(pMeta->pDB->pDB, - "CREATE TABLE IF NOT EXISTS stb (" - " tb_uid INTEGER NOT NULL UNIQUE," - " tbname VARCHAR(256) NOT NULL UNIQUE," - " tb_schema BLOB NOT NULL," - " tag_schema BLOB NOT NULL" - ");", - NULL, NULL, &err); - if (rc != SQLITE_OK) { - // TODO: handle error - printf("failed to create meta table stb since %s\n", err); - } - - // For normal tables - rc = sqlite3_exec(pMeta->pDB->pDB, - "CREATE TABLE IF NOT EXISTS ntb (" - " tb_uid INTEGER NOT NULL UNIQUE," - " tbname VARCHAR(256) NOT NULL," - " tb_schema BLOB NOT NULL" - ");", - NULL, NULL, &err); - if (rc != SQLITE_OK) { - // TODO: handle error - printf("failed to create meta table ntb since %s\n", err); - } - - sqlite3_exec(pMeta->pDB->pDB, "BEGIN;", NULL, NULL, &err); - - tfree(err); - - return 0; -} - -void metaCloseDB(SMeta *pMeta) { - if (pMeta->pDB) { - sqlite3_exec(pMeta->pDB->pDB, "COMMIT;", NULL, NULL, NULL); - sqlite3_close(pMeta->pDB->pDB); - free(pMeta->pDB); - pMeta->pDB = NULL; - } - - // TODO -} - -int metaSaveTableToDB(SMeta *pMeta, const STbCfg *pTbCfg) { - char sql[256]; - char * err = NULL; - int rc; - tb_uid_t uid; - sqlite3_stmt *stmt; - char buf[256]; - void * pBuf; - - switch (pTbCfg->type) { - case META_SUPER_TABLE: - uid = pTbCfg->stbCfg.suid; - sprintf(sql, - "INSERT INTO tb VALUES (\'%s\', %" PRIu64 - ");" - "CREATE TABLE IF NOT EXISTS stb_%" PRIu64 - " (" - " tb_uid INTEGER NOT NULL UNIQUE," - " tbname VARCHAR(256)," - " tag1 INTEGER);", - pTbCfg->name, uid, uid); - rc = sqlite3_exec(pMeta->pDB->pDB, sql, NULL, NULL, &err); - if (rc != SQLITE_OK) { - printf("failed to create normal table since %s\n", err); - } - - sprintf(sql, "INSERT INTO stb VALUES (%" PRIu64 ", %s, ?, ?)", uid, pTbCfg->name); - sqlite3_prepare_v2(pMeta->pDB->pDB, sql, -1, &stmt, NULL); - - pBuf = buf; - tdEncodeSchema(&pBuf, pTbCfg->stbCfg.pSchema); - sqlite3_bind_blob(stmt, 1, buf, POINTER_DISTANCE(pBuf, buf), NULL); - pBuf = buf; - tdEncodeSchema(&pBuf, pTbCfg->stbCfg.pTagSchema); - sqlite3_bind_blob(stmt, 2, buf, POINTER_DISTANCE(pBuf, buf), NULL); - - sqlite3_step(stmt); - - sqlite3_finalize(stmt); - -#if 0 - sprintf(sql, - "INSERT INTO tb VALUES (?, ?);" - // "INSERT INTO stb VALUES (?, ?, ?, ?);" - // "CREATE TABLE IF NOT EXISTS stb_%" PRIu64 - // " (" - // " tb_uid INTEGER NOT NULL UNIQUE," - // " tbname VARCHAR(256)," - // " tag1 INTEGER);" - , - uid); - rc = sqlite3_prepare_v2(pMeta->pDB->pDB, sql, -1, &stmt, NULL); - if (rc != SQLITE_OK) { - return -1; - } - sqlite3_bind_text(stmt, 1, pTbCfg->name, -1, SQLITE_TRANSIENT); - sqlite3_bind_int64(stmt, 2, uid); - sqlite3_step(stmt); - sqlite3_finalize(stmt); - - - // sqlite3_bind_int64(stmt, 3, uid); - // sqlite3_bind_text(stmt, 4, pTbCfg->name, -1, SQLITE_TRANSIENT); - // pBuf = buf; - // tdEncodeSchema(&pBuf, pTbCfg->stbCfg.pSchema); - // sqlite3_bind_blob(stmt, 5, buf, POINTER_DISTANCE(pBuf, buf), NULL); - // pBuf = buf; - // tdEncodeSchema(&pBuf, pTbCfg->stbCfg.pTagSchema); - // sqlite3_bind_blob(stmt, 6, buf, POINTER_DISTANCE(pBuf, buf), NULL); - - rc = sqliteVjj3_step(stmt); - if (rc != SQLITE_OK) { - printf("failed to create normal table since %s\n", sqlite3_errmsg(pMeta->pDB->pDB)); - } - sqlite3_finalize(stmt); -#endif - break; - case META_NORMAL_TABLE: - // uid = metaGenerateUid(pMeta); - // sprintf(sql, - // "INSERT INTO tb VALUES (\'%s\', %" PRIu64 - // ");" - // "INSERT INTO ntb VALUES (%" PRIu64 ", \'%s\', );", - // pTbCfg->name, uid, uid, pTbCfg->name, ); - - // rc = sqlite3_exec(pMeta->pDB->pDB, sql, NULL, NULL, &err); - // if (rc != SQLITE_OK) { - // printf("failed to create normal table since %s\n", err); - // } - break; - case META_CHILD_TABLE: -#if 0 - uid = metaGenerateUid(pMeta); - // sprintf(sql, "INSERT INTO tb VALUES (\'%s\', %" PRIu64 - // ");" - // "INSERT INTO stb_%" PRIu64 " VALUES (%" PRIu64 ", \'%s\', );"); - rc = sqlite3_exec(pMeta->pDB->pDB, sql, NULL, NULL, &err); - if (rc != SQLITE_OK) { - printf("failed to create child table since %s\n", err); - } -#endif - break; - default: - break; - } - - tfree(err); - - return 0; -} - -int metaRemoveTableFromDb(SMeta *pMeta, tb_uid_t uid) { - /* TODO */ - return 0; -} \ No newline at end of file diff --git a/source/dnode/vnode/meta/test/CMakeLists.txt b/source/dnode/vnode/meta/test/CMakeLists.txt deleted file mode 100644 index 625c07ad56..0000000000 --- a/source/dnode/vnode/meta/test/CMakeLists.txt +++ /dev/null @@ -1,24 +0,0 @@ -# add_executable(metaTest "") -# target_sources(metaTest -# PRIVATE -# "../src/metaMain.c" -# "../src/metaUid.c" -# "metaTests.cpp" -# ) -# target_include_directories(metaTest -# PUBLIC -# "${CMAKE_SOURCE_DIR}/include/server/vnode/meta" -# "${CMAKE_CURRENT_SOURCE_DIR}/../inc" -# ) -# target_link_libraries(metaTest -# os -# util -# common -# gtest_main -# tkv -# ) -# enable_testing() -# add_test( -# NAME meta_test -# COMMAND metaTest -# ) diff --git a/source/dnode/vnode/meta/test/metaAPITest.cpp b/source/dnode/vnode/meta/test/metaAPITest.cpp deleted file mode 100644 index 0d79882018..0000000000 --- a/source/dnode/vnode/meta/test/metaAPITest.cpp +++ /dev/null @@ -1,105 +0,0 @@ -#if 0 -#include -#include -#include - -#include "meta.h" - -static STSchema *metaGetSimpleSchema() { - STSchema * pSchema = NULL; - STSchemaBuilder sb = {0}; - - tdInitTSchemaBuilder(&sb, 0); - tdAddColToSchema(&sb, TSDB_DATA_TYPE_TIMESTAMP, 0, 8); - tdAddColToSchema(&sb, TSDB_DATA_TYPE_INT, 1, 4); - - pSchema = tdGetSchemaFromBuilder(&sb); - tdDestroyTSchemaBuilder(&sb); - - return pSchema; -} - -static SKVRow metaGetSimpleTags() { - SKVRowBuilder kvrb = {0}; - SKVRow row; - - tdInitKVRowBuilder(&kvrb); - int64_t ts = 1634287978000; - int32_t a = 10; - - tdAddColToKVRow(&kvrb, 0, TSDB_DATA_TYPE_TIMESTAMP, (void *)(&ts)); - tdAddColToKVRow(&kvrb, 0, TSDB_DATA_TYPE_INT, (void *)(&a)); - - row = tdGetKVRowFromBuilder(&kvrb); - - tdDestroyKVRowBuilder(&kvrb); - - return row; -} - -TEST(MetaTest, DISABLED_meta_create_1m_normal_tables_test) { - // Open Meta - SMeta *meta = metaOpen(NULL, NULL); - std::cout << "Meta is opened!" << std::endl; - - // Create 1000000 normal tables - META_TABLE_OPTS_DECLARE(tbOpts); - STSchema *pSchema = metaGetSimpleSchema(); - char tbname[128]; - - for (size_t i = 0; i < 1000000; i++) { - sprintf(tbname, "ntb%ld", i); - metaNormalTableOptsInit(&tbOpts, tbname, pSchema); - metaCreateTable(meta, &tbOpts); - metaTableOptsClear(&tbOpts); - } - - tdFreeSchema(pSchema); - - // Close Meta - metaClose(meta); - std::cout << "Meta is closed!" << std::endl; - - // Destroy Meta - metaDestroy("meta"); - std::cout << "Meta is destroyed!" << std::endl; -} - -TEST(MetaTest, meta_create_1m_child_tables_test) { - // Open Meta - SMeta *meta = metaOpen(NULL); - std::cout << "Meta is opened!" << std::endl; - - // Create a super tables - tb_uid_t uid = 477529885843758ul; - META_TABLE_OPTS_DECLARE(tbOpts); - STSchema *pSchema = metaGetSimpleSchema(); - STSchema *pTagSchema = metaGetSimpleSchema(); - - metaSuperTableOptsInit(&tbOpts, "st", uid, pSchema, pTagSchema); - metaCreateTable(meta, &tbOpts); - metaTableOptsClear(&tbOpts); - - tdFreeSchema(pSchema); - tdFreeSchema(pTagSchema); - - // Create 1000000 child tables - char name[128]; - SKVRow row = metaGetSimpleTags(); - for (size_t i = 0; i < 1000000; i++) { - sprintf(name, "ctb%ld", i); - metaChildTableOptsInit(&tbOpts, name, uid, row); - metaCreateTable(meta, &tbOpts); - metaTableOptsClear(&tbOpts); - } - kvRowFree(row); - - // Close Meta - metaClose(meta); - std::cout << "Meta is closed!" << std::endl; - - // Destroy Meta - metaDestroy("meta"); - std::cout << "Meta is destroyed!" << std::endl; -} -#endif \ No newline at end of file diff --git a/source/dnode/vnode/meta/inc/metaCache.h b/source/dnode/vnode/src/inc/metaCache.h similarity index 100% rename from source/dnode/vnode/meta/inc/metaCache.h rename to source/dnode/vnode/src/inc/metaCache.h diff --git a/source/dnode/vnode/meta/inc/metaCfg.h b/source/dnode/vnode/src/inc/metaCfg.h similarity index 100% rename from source/dnode/vnode/meta/inc/metaCfg.h rename to source/dnode/vnode/src/inc/metaCfg.h diff --git a/source/dnode/vnode/meta/inc/metaDB.h b/source/dnode/vnode/src/inc/metaDB.h similarity index 100% rename from source/dnode/vnode/meta/inc/metaDB.h rename to source/dnode/vnode/src/inc/metaDB.h diff --git a/source/dnode/vnode/meta/inc/metaDef.h b/source/dnode/vnode/src/inc/metaDef.h similarity index 100% rename from source/dnode/vnode/meta/inc/metaDef.h rename to source/dnode/vnode/src/inc/metaDef.h diff --git a/source/dnode/vnode/meta/inc/metaIdx.h b/source/dnode/vnode/src/inc/metaIdx.h similarity index 100% rename from source/dnode/vnode/meta/inc/metaIdx.h rename to source/dnode/vnode/src/inc/metaIdx.h diff --git a/source/dnode/vnode/meta/inc/metaQuery.h b/source/dnode/vnode/src/inc/metaQuery.h similarity index 100% rename from source/dnode/vnode/meta/inc/metaQuery.h rename to source/dnode/vnode/src/inc/metaQuery.h diff --git a/source/dnode/vnode/meta/inc/metaTbCfg.h b/source/dnode/vnode/src/inc/metaTbCfg.h similarity index 100% rename from source/dnode/vnode/meta/inc/metaTbCfg.h rename to source/dnode/vnode/src/inc/metaTbCfg.h diff --git a/source/dnode/vnode/meta/inc/metaTbTag.h b/source/dnode/vnode/src/inc/metaTbTag.h similarity index 100% rename from source/dnode/vnode/meta/inc/metaTbTag.h rename to source/dnode/vnode/src/inc/metaTbTag.h diff --git a/source/dnode/vnode/meta/inc/metaTbUid.h b/source/dnode/vnode/src/inc/metaTbUid.h similarity index 100% rename from source/dnode/vnode/meta/inc/metaTbUid.h rename to source/dnode/vnode/src/inc/metaTbUid.h diff --git a/source/dnode/vnode/tq/inc/tqCommit.h b/source/dnode/vnode/src/inc/tqCommit.h similarity index 100% rename from source/dnode/vnode/tq/inc/tqCommit.h rename to source/dnode/vnode/src/inc/tqCommit.h diff --git a/source/dnode/vnode/tq/inc/tqInt.h b/source/dnode/vnode/src/inc/tqInt.h similarity index 100% rename from source/dnode/vnode/tq/inc/tqInt.h rename to source/dnode/vnode/src/inc/tqInt.h diff --git a/source/dnode/vnode/tq/inc/tqMetaStore.h b/source/dnode/vnode/src/inc/tqMetaStore.h similarity index 100% rename from source/dnode/vnode/tq/inc/tqMetaStore.h rename to source/dnode/vnode/src/inc/tqMetaStore.h diff --git a/source/dnode/vnode/tsdb/inc/tsdbCommit.h b/source/dnode/vnode/src/inc/tsdbCommit.h similarity index 100% rename from source/dnode/vnode/tsdb/inc/tsdbCommit.h rename to source/dnode/vnode/src/inc/tsdbCommit.h diff --git a/source/dnode/vnode/tsdb/inc/tsdbCompact.h b/source/dnode/vnode/src/inc/tsdbCompact.h similarity index 100% rename from source/dnode/vnode/tsdb/inc/tsdbCompact.h rename to source/dnode/vnode/src/inc/tsdbCompact.h diff --git a/source/dnode/vnode/tsdb/inc/tsdbDef.h b/source/dnode/vnode/src/inc/tsdbDef.h similarity index 100% rename from source/dnode/vnode/tsdb/inc/tsdbDef.h rename to source/dnode/vnode/src/inc/tsdbDef.h diff --git a/source/dnode/vnode/tsdb/inc/tsdbFS.h b/source/dnode/vnode/src/inc/tsdbFS.h similarity index 100% rename from source/dnode/vnode/tsdb/inc/tsdbFS.h rename to source/dnode/vnode/src/inc/tsdbFS.h diff --git a/source/dnode/vnode/tsdb/inc/tsdbFile.h b/source/dnode/vnode/src/inc/tsdbFile.h similarity index 100% rename from source/dnode/vnode/tsdb/inc/tsdbFile.h rename to source/dnode/vnode/src/inc/tsdbFile.h diff --git a/source/dnode/vnode/tsdb/inc/tsdbHealth.h b/source/dnode/vnode/src/inc/tsdbHealth.h similarity index 100% rename from source/dnode/vnode/tsdb/inc/tsdbHealth.h rename to source/dnode/vnode/src/inc/tsdbHealth.h diff --git a/source/dnode/vnode/tsdb/inc/tsdbLog.h b/source/dnode/vnode/src/inc/tsdbLog.h similarity index 100% rename from source/dnode/vnode/tsdb/inc/tsdbLog.h rename to source/dnode/vnode/src/inc/tsdbLog.h diff --git a/source/dnode/vnode/tsdb/inc/tsdbMemTable.h b/source/dnode/vnode/src/inc/tsdbMemTable.h similarity index 100% rename from source/dnode/vnode/tsdb/inc/tsdbMemTable.h rename to source/dnode/vnode/src/inc/tsdbMemTable.h diff --git a/source/dnode/vnode/tsdb/inc/tsdbMemory.h b/source/dnode/vnode/src/inc/tsdbMemory.h similarity index 100% rename from source/dnode/vnode/tsdb/inc/tsdbMemory.h rename to source/dnode/vnode/src/inc/tsdbMemory.h diff --git a/source/dnode/vnode/tsdb/inc/tsdbOptions.h b/source/dnode/vnode/src/inc/tsdbOptions.h similarity index 100% rename from source/dnode/vnode/tsdb/inc/tsdbOptions.h rename to source/dnode/vnode/src/inc/tsdbOptions.h diff --git a/source/dnode/vnode/tsdb/inc/tsdbReadImpl.h b/source/dnode/vnode/src/inc/tsdbReadImpl.h similarity index 100% rename from source/dnode/vnode/tsdb/inc/tsdbReadImpl.h rename to source/dnode/vnode/src/inc/tsdbReadImpl.h diff --git a/source/dnode/vnode/tsdb/inc/tsdbRowMergeBuf.h b/source/dnode/vnode/src/inc/tsdbRowMergeBuf.h similarity index 100% rename from source/dnode/vnode/tsdb/inc/tsdbRowMergeBuf.h rename to source/dnode/vnode/src/inc/tsdbRowMergeBuf.h diff --git a/source/dnode/vnode/impl/inc/vnodeBufferPool.h b/source/dnode/vnode/src/inc/vnodeBufferPool.h similarity index 100% rename from source/dnode/vnode/impl/inc/vnodeBufferPool.h rename to source/dnode/vnode/src/inc/vnodeBufferPool.h diff --git a/source/dnode/vnode/impl/inc/vnodeCfg.h b/source/dnode/vnode/src/inc/vnodeCfg.h similarity index 100% rename from source/dnode/vnode/impl/inc/vnodeCfg.h rename to source/dnode/vnode/src/inc/vnodeCfg.h diff --git a/source/dnode/vnode/impl/inc/vnodeCommit.h b/source/dnode/vnode/src/inc/vnodeCommit.h similarity index 100% rename from source/dnode/vnode/impl/inc/vnodeCommit.h rename to source/dnode/vnode/src/inc/vnodeCommit.h diff --git a/source/dnode/vnode/impl/inc/vnodeDef.h b/source/dnode/vnode/src/inc/vnodeDef.h similarity index 98% rename from source/dnode/vnode/impl/inc/vnodeDef.h rename to source/dnode/vnode/src/inc/vnodeDef.h index f9172dd351..1c534a8aeb 100644 --- a/source/dnode/vnode/impl/inc/vnodeDef.h +++ b/source/dnode/vnode/src/inc/vnodeDef.h @@ -17,7 +17,7 @@ #define _TD_VNODE_DEF_H_ #include "mallocator.h" -#include "sync.h" +// #include "sync.h" #include "tcoding.h" #include "tlist.h" #include "tlockfree.h" diff --git a/source/dnode/vnode/impl/inc/vnodeFS.h b/source/dnode/vnode/src/inc/vnodeFS.h similarity index 100% rename from source/dnode/vnode/impl/inc/vnodeFS.h rename to source/dnode/vnode/src/inc/vnodeFS.h diff --git a/source/dnode/vnode/impl/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h similarity index 98% rename from source/dnode/vnode/impl/inc/vnodeInt.h rename to source/dnode/vnode/src/inc/vnodeInt.h index 48977ff046..028798bc3e 100644 --- a/source/dnode/vnode/impl/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -19,7 +19,7 @@ #include "vnode.h" #include "meta.h" -#include "sync.h" +// #include "sync.h" #include "tlog.h" #include "tq.h" #include "tsdb.h" diff --git a/source/dnode/vnode/impl/inc/vnodeMAF.h b/source/dnode/vnode/src/inc/vnodeMAF.h similarity index 100% rename from source/dnode/vnode/impl/inc/vnodeMAF.h rename to source/dnode/vnode/src/inc/vnodeMAF.h diff --git a/source/dnode/vnode/impl/inc/vnodeMemAllocator.h b/source/dnode/vnode/src/inc/vnodeMemAllocator.h similarity index 100% rename from source/dnode/vnode/impl/inc/vnodeMemAllocator.h rename to source/dnode/vnode/src/inc/vnodeMemAllocator.h diff --git a/source/dnode/vnode/impl/inc/vnodeQuery.h b/source/dnode/vnode/src/inc/vnodeQuery.h similarity index 100% rename from source/dnode/vnode/impl/inc/vnodeQuery.h rename to source/dnode/vnode/src/inc/vnodeQuery.h diff --git a/source/dnode/vnode/impl/inc/vnodeRead.h b/source/dnode/vnode/src/inc/vnodeRead.h similarity index 100% rename from source/dnode/vnode/impl/inc/vnodeRead.h rename to source/dnode/vnode/src/inc/vnodeRead.h diff --git a/source/dnode/vnode/impl/inc/vnodeRequest.h b/source/dnode/vnode/src/inc/vnodeRequest.h similarity index 100% rename from source/dnode/vnode/impl/inc/vnodeRequest.h rename to source/dnode/vnode/src/inc/vnodeRequest.h diff --git a/source/dnode/vnode/impl/inc/vnodeStateMgr.h b/source/dnode/vnode/src/inc/vnodeStateMgr.h similarity index 100% rename from source/dnode/vnode/impl/inc/vnodeStateMgr.h rename to source/dnode/vnode/src/inc/vnodeStateMgr.h diff --git a/source/dnode/vnode/impl/inc/vnodeSync.h b/source/dnode/vnode/src/inc/vnodeSync.h similarity index 97% rename from source/dnode/vnode/impl/inc/vnodeSync.h rename to source/dnode/vnode/src/inc/vnodeSync.h index a3eb004dfa..e82979551d 100644 --- a/source/dnode/vnode/impl/inc/vnodeSync.h +++ b/source/dnode/vnode/src/inc/vnodeSync.h @@ -16,7 +16,7 @@ #ifndef _TD_VNODE_SYNC_H_ #define _TD_VNODE_SYNC_H_ -#include "sync.h" +// #include "sync.h" #ifdef __cplusplus extern "C" { diff --git a/source/dnode/vnode/impl/inc/vnodeWrite.h b/source/dnode/vnode/src/inc/vnodeWrite.h similarity index 100% rename from source/dnode/vnode/impl/inc/vnodeWrite.h rename to source/dnode/vnode/src/inc/vnodeWrite.h diff --git a/source/dnode/vnode/meta/CMakeLists.txt b/source/dnode/vnode/src/meta/CMakeLists.txt similarity index 100% rename from source/dnode/vnode/meta/CMakeLists.txt rename to source/dnode/vnode/src/meta/CMakeLists.txt diff --git a/source/dnode/vnode/meta/src/metaBDBImpl.c b/source/dnode/vnode/src/meta/metaBDBImpl.c similarity index 100% rename from source/dnode/vnode/meta/src/metaBDBImpl.c rename to source/dnode/vnode/src/meta/metaBDBImpl.c diff --git a/source/dnode/vnode/meta/src/metaCache.c b/source/dnode/vnode/src/meta/metaCache.c similarity index 100% rename from source/dnode/vnode/meta/src/metaCache.c rename to source/dnode/vnode/src/meta/metaCache.c diff --git a/source/dnode/vnode/meta/src/metaCfg.c b/source/dnode/vnode/src/meta/metaCfg.c similarity index 100% rename from source/dnode/vnode/meta/src/metaCfg.c rename to source/dnode/vnode/src/meta/metaCfg.c diff --git a/source/dnode/vnode/meta/src/metaCommit.c b/source/dnode/vnode/src/meta/metaCommit.c similarity index 100% rename from source/dnode/vnode/meta/src/metaCommit.c rename to source/dnode/vnode/src/meta/metaCommit.c diff --git a/source/dnode/vnode/meta/src/metaIdx.c b/source/dnode/vnode/src/meta/metaIdx.c similarity index 98% rename from source/dnode/vnode/meta/src/metaIdx.c rename to source/dnode/vnode/src/meta/metaIdx.c index 3da56fc394..d9abb4bb7b 100644 --- a/source/dnode/vnode/meta/src/metaIdx.c +++ b/source/dnode/vnode/src/meta/metaIdx.c @@ -13,7 +13,9 @@ * along with this program. If not, see . */ +#ifdef USE_INVERTED_INDEX #include "index.h" +#endif #include "metaDef.h" struct SMetaIdx { diff --git a/source/dnode/vnode/meta/src/metaMain.c b/source/dnode/vnode/src/meta/metaMain.c similarity index 100% rename from source/dnode/vnode/meta/src/metaMain.c rename to source/dnode/vnode/src/meta/metaMain.c diff --git a/source/dnode/vnode/meta/src/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c similarity index 100% rename from source/dnode/vnode/meta/src/metaQuery.c rename to source/dnode/vnode/src/meta/metaQuery.c diff --git a/source/dnode/vnode/meta/src/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c similarity index 100% rename from source/dnode/vnode/meta/src/metaTable.c rename to source/dnode/vnode/src/meta/metaTable.c diff --git a/source/dnode/vnode/meta/src/metaTbCfg.c b/source/dnode/vnode/src/meta/metaTbCfg.c similarity index 100% rename from source/dnode/vnode/meta/src/metaTbCfg.c rename to source/dnode/vnode/src/meta/metaTbCfg.c diff --git a/source/dnode/vnode/meta/src/metaTbTag.c b/source/dnode/vnode/src/meta/metaTbTag.c similarity index 100% rename from source/dnode/vnode/meta/src/metaTbTag.c rename to source/dnode/vnode/src/meta/metaTbTag.c diff --git a/source/dnode/vnode/meta/src/metaTbUid.c b/source/dnode/vnode/src/meta/metaTbUid.c similarity index 100% rename from source/dnode/vnode/meta/src/metaTbUid.c rename to source/dnode/vnode/src/meta/metaTbUid.c diff --git a/source/dnode/vnode/tq/CMakeLists.txt b/source/dnode/vnode/src/tq/CMakeLists.txt similarity index 100% rename from source/dnode/vnode/tq/CMakeLists.txt rename to source/dnode/vnode/src/tq/CMakeLists.txt diff --git a/source/dnode/vnode/tq/src/tq.c b/source/dnode/vnode/src/tq/tq.c similarity index 100% rename from source/dnode/vnode/tq/src/tq.c rename to source/dnode/vnode/src/tq/tq.c diff --git a/source/dnode/vnode/tq/src/tqCommit.c b/source/dnode/vnode/src/tq/tqCommit.c similarity index 100% rename from source/dnode/vnode/tq/src/tqCommit.c rename to source/dnode/vnode/src/tq/tqCommit.c diff --git a/source/dnode/vnode/tq/src/tqMetaStore.c b/source/dnode/vnode/src/tq/tqMetaStore.c similarity index 100% rename from source/dnode/vnode/tq/src/tqMetaStore.c rename to source/dnode/vnode/src/tq/tqMetaStore.c diff --git a/source/dnode/vnode/tsdb/CMakeLists.txt b/source/dnode/vnode/src/tsdb/CMakeLists.txt similarity index 100% rename from source/dnode/vnode/tsdb/CMakeLists.txt rename to source/dnode/vnode/src/tsdb/CMakeLists.txt diff --git a/source/dnode/vnode/tsdb/src/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c similarity index 100% rename from source/dnode/vnode/tsdb/src/tsdbCommit.c rename to source/dnode/vnode/src/tsdb/tsdbCommit.c diff --git a/source/dnode/vnode/tsdb/src/tsdbCompact.c b/source/dnode/vnode/src/tsdb/tsdbCompact.c similarity index 100% rename from source/dnode/vnode/tsdb/src/tsdbCompact.c rename to source/dnode/vnode/src/tsdb/tsdbCompact.c diff --git a/source/dnode/vnode/tsdb/src/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c similarity index 100% rename from source/dnode/vnode/tsdb/src/tsdbFS.c rename to source/dnode/vnode/src/tsdb/tsdbFS.c diff --git a/source/dnode/vnode/tsdb/src/tsdbFile.c b/source/dnode/vnode/src/tsdb/tsdbFile.c similarity index 100% rename from source/dnode/vnode/tsdb/src/tsdbFile.c rename to source/dnode/vnode/src/tsdb/tsdbFile.c diff --git a/source/dnode/vnode/src/tsdb/tsdbHealth.c b/source/dnode/vnode/src/tsdb/tsdbHealth.c new file mode 100644 index 0000000000..99c1b925b0 --- /dev/null +++ b/source/dnode/vnode/src/tsdb/tsdbHealth.c @@ -0,0 +1,98 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +// #include "os.h" +// #include "tmsg.h" +// #include "tarray.h" +// #include "query.h" +// #include "tglobal.h" +// #include "tlist.h" +// #include "tsdbint.h" +// #include "tsdbBuffer.h" +// #include "tsdbLog.h" +// #include "tsdbHealth.h" +// #include "ttimer.h" +// #include "tthread.h" + + +// // return malloc new block count +// int32_t tsdbInsertNewBlock(STsdbRepo * pRepo) { +// STsdbBufPool *pPool = pRepo->pPool; +// int32_t cnt = 0; + +// if(tsdbAllowNewBlock(pRepo)) { +// STsdbBufBlock *pBufBlock = tsdbNewBufBlock(pPool->bufBlockSize); +// if (pBufBlock) { +// if (tdListAppend(pPool->bufBlockList, (void *)(&pBufBlock)) < 0) { +// // append error +// tsdbFreeBufBlock(pBufBlock); +// } else { +// pPool->nElasticBlocks ++; +// cnt ++ ; +// } +// } +// } +// return cnt; +// } + +// // switch anther thread to run +// void* cbKillQueryFree(void* param) { +// STsdbRepo* pRepo = (STsdbRepo*)param; +// // vnode +// if(pRepo->appH.notifyStatus) { +// pRepo->appH.notifyStatus(pRepo->appH.appH, TSDB_STATUS_COMMIT_NOBLOCK, TSDB_CODE_SUCCESS); +// } + +// // free +// if(pRepo->pthread){ +// void* p = pRepo->pthread; +// pRepo->pthread = NULL; +// free(p); +// } + +// return NULL; +// } + +// // return true do free , false do nothing +// bool tsdbUrgeQueryFree(STsdbRepo * pRepo) { +// // check previous running +// if(pRepo->pthread && taosThreadRunning(pRepo->pthread)) { +// tsdbWarn("vgId:%d pre urge thread is runing. nBlocks=%d nElasticBlocks=%d", REPO_ID(pRepo), pRepo->pPool->nBufBlocks, pRepo->pPool->nElasticBlocks); +// return false; +// } +// // create new +// pRepo->pthread = taosCreateThread(cbKillQueryFree, pRepo); +// if(pRepo->pthread == NULL) { +// tsdbError("vgId:%d create urge thread error.", REPO_ID(pRepo)); +// return false; +// } +// return true; +// } + +// bool tsdbAllowNewBlock(STsdbRepo* pRepo) { +// int32_t nMaxElastic = pRepo->config.totalBlocks/3; +// STsdbBufPool* pPool = pRepo->pPool; +// if(pPool->nElasticBlocks >= nMaxElastic) { +// tsdbWarn("vgId:%d tsdbAllowNewBlock return fasle. nElasticBlock(%d) >= MaxElasticBlocks(%d)", REPO_ID(pRepo), pPool->nElasticBlocks, nMaxElastic); +// return false; +// } +// return true; +// } + +// bool tsdbNoProblem(STsdbRepo* pRepo) { +// if(listNEles(pRepo->pPool->bufBlockList) == 0) +// return false; +// return true; +// } \ No newline at end of file diff --git a/source/dnode/vnode/tsdb/src/tsdbMain.c b/source/dnode/vnode/src/tsdb/tsdbMain.c similarity index 100% rename from source/dnode/vnode/tsdb/src/tsdbMain.c rename to source/dnode/vnode/src/tsdb/tsdbMain.c diff --git a/source/dnode/vnode/tsdb/src/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c similarity index 100% rename from source/dnode/vnode/tsdb/src/tsdbMemTable.c rename to source/dnode/vnode/src/tsdb/tsdbMemTable.c diff --git a/source/dnode/vnode/tsdb/src/tsdbOptions.c b/source/dnode/vnode/src/tsdb/tsdbOptions.c similarity index 100% rename from source/dnode/vnode/tsdb/src/tsdbOptions.c rename to source/dnode/vnode/src/tsdb/tsdbOptions.c diff --git a/source/dnode/vnode/tsdb/src/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c similarity index 99% rename from source/dnode/vnode/tsdb/src/tsdbRead.c rename to source/dnode/vnode/src/tsdb/tsdbRead.c index 906046ed9a..a8296e8f8e 100644 --- a/source/dnode/vnode/tsdb/src/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -28,7 +28,7 @@ #include "taosdef.h" #include "tlosertree.h" -#include "tsdbint.h" +#include "tsdbDef.h" #include "tmsg.h" #define EXTRA_BYTES 2 diff --git a/source/dnode/vnode/tsdb/src/tsdbReadImpl.c b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c similarity index 100% rename from source/dnode/vnode/tsdb/src/tsdbReadImpl.c rename to source/dnode/vnode/src/tsdb/tsdbReadImpl.c diff --git a/source/dnode/vnode/src/tsdb/tsdbRowMergeBuf.c b/source/dnode/vnode/src/tsdb/tsdbRowMergeBuf.c new file mode 100644 index 0000000000..1eebea22d4 --- /dev/null +++ b/source/dnode/vnode/src/tsdb/tsdbRowMergeBuf.c @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +// #include "tsdbRowMergeBuf.h" +// #include "tdataformat.h" + +// // row1 has higher priority +// SMemRow tsdbMergeTwoRows(SMergeBuf *pBuf, SMemRow row1, SMemRow row2, STSchema *pSchema1, STSchema *pSchema2) { +// if(row2 == NULL) return row1; +// if(row1 == NULL) return row2; +// ASSERT(pSchema1->version == memRowVersion(row1)); +// ASSERT(pSchema2->version == memRowVersion(row2)); + +// if(tsdbMergeBufMakeSureRoom(pBuf, pSchema1, pSchema2) < 0) { +// return NULL; +// } +// return mergeTwoMemRows(*pBuf, row1, row2, pSchema1, pSchema2); +// } diff --git a/source/dnode/vnode/tsdb/src/tsdbScan.c b/source/dnode/vnode/src/tsdb/tsdbScan.c similarity index 100% rename from source/dnode/vnode/tsdb/src/tsdbScan.c rename to source/dnode/vnode/src/tsdb/tsdbScan.c index 382f7b11ae..c0e468e640 100644 --- a/source/dnode/vnode/tsdb/src/tsdbScan.c +++ b/source/dnode/vnode/src/tsdb/tsdbScan.c @@ -13,9 +13,9 @@ * along with this program. If not, see . */ -#include "tsdbint.h" #if 0 +#include "tsdbint.h" #ifndef _TSDB_PLUGINS int tsdbScanFGroup(STsdbScanHandle* pScanHandle, char* rootDir, int fid) { return 0; } diff --git a/source/dnode/vnode/tsdb/src/tsdbWrite.c b/source/dnode/vnode/src/tsdb/tsdbWrite.c similarity index 100% rename from source/dnode/vnode/tsdb/src/tsdbWrite.c rename to source/dnode/vnode/src/tsdb/tsdbWrite.c diff --git a/source/dnode/vnode/impl/CMakeLists.txt b/source/dnode/vnode/src/vnd/CMakeLists.txt similarity index 100% rename from source/dnode/vnode/impl/CMakeLists.txt rename to source/dnode/vnode/src/vnd/CMakeLists.txt diff --git a/source/dnode/vnode/impl/src/vnodeArenaMAImpl.c b/source/dnode/vnode/src/vnd/vnodeArenaMAImpl.c similarity index 100% rename from source/dnode/vnode/impl/src/vnodeArenaMAImpl.c rename to source/dnode/vnode/src/vnd/vnodeArenaMAImpl.c diff --git a/source/dnode/vnode/impl/src/vnodeBufferPool.c b/source/dnode/vnode/src/vnd/vnodeBufferPool.c similarity index 100% rename from source/dnode/vnode/impl/src/vnodeBufferPool.c rename to source/dnode/vnode/src/vnd/vnodeBufferPool.c diff --git a/source/dnode/vnode/impl/src/vnodeCfg.c b/source/dnode/vnode/src/vnd/vnodeCfg.c similarity index 100% rename from source/dnode/vnode/impl/src/vnodeCfg.c rename to source/dnode/vnode/src/vnd/vnodeCfg.c diff --git a/source/dnode/vnode/impl/src/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c similarity index 100% rename from source/dnode/vnode/impl/src/vnodeCommit.c rename to source/dnode/vnode/src/vnd/vnodeCommit.c diff --git a/source/dnode/vnode/impl/src/vnodeFS.c b/source/dnode/vnode/src/vnd/vnodeFS.c similarity index 100% rename from source/dnode/vnode/impl/src/vnodeFS.c rename to source/dnode/vnode/src/vnd/vnodeFS.c diff --git a/source/dnode/vnode/impl/src/vnodeInt.c b/source/dnode/vnode/src/vnd/vnodeInt.c similarity index 100% rename from source/dnode/vnode/impl/src/vnodeInt.c rename to source/dnode/vnode/src/vnd/vnodeInt.c diff --git a/source/dnode/vnode/impl/src/vnodeMain.c b/source/dnode/vnode/src/vnd/vnodeMain.c similarity index 100% rename from source/dnode/vnode/impl/src/vnodeMain.c rename to source/dnode/vnode/src/vnd/vnodeMain.c diff --git a/source/dnode/vnode/impl/src/vnodeMgr.c b/source/dnode/vnode/src/vnd/vnodeMgr.c similarity index 98% rename from source/dnode/vnode/impl/src/vnodeMgr.c rename to source/dnode/vnode/src/vnd/vnodeMgr.c index cc369a0d15..fdb96e52e2 100644 --- a/source/dnode/vnode/impl/src/vnodeMgr.c +++ b/source/dnode/vnode/src/vnd/vnodeMgr.c @@ -56,13 +56,11 @@ int vnodeInit(const SVnodeOpt *pOption) { return 0; } -void vnodeClear() { +void vnodeCleanup() { if (TD_CHECK_AND_SET_MOD_CLEAR(&(vnodeMgr.vnodeInitFlag)) == TD_MOD_UNINITIALIZED) { return; } - walCleanUp(); - // Stop commit handler pthread_mutex_lock(&(vnodeMgr.mutex)); vnodeMgr.stop = true; diff --git a/source/dnode/vnode/impl/src/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c similarity index 100% rename from source/dnode/vnode/impl/src/vnodeQuery.c rename to source/dnode/vnode/src/vnd/vnodeQuery.c diff --git a/source/dnode/vnode/impl/src/vnodeRead.c b/source/dnode/vnode/src/vnd/vnodeRead.c similarity index 100% rename from source/dnode/vnode/impl/src/vnodeRead.c rename to source/dnode/vnode/src/vnd/vnodeRead.c diff --git a/source/dnode/vnode/impl/src/vnodeRequest.c b/source/dnode/vnode/src/vnd/vnodeRequest.c similarity index 100% rename from source/dnode/vnode/impl/src/vnodeRequest.c rename to source/dnode/vnode/src/vnd/vnodeRequest.c diff --git a/source/dnode/vnode/impl/src/vnodeStateMgr.c b/source/dnode/vnode/src/vnd/vnodeStateMgr.c similarity index 100% rename from source/dnode/vnode/impl/src/vnodeStateMgr.c rename to source/dnode/vnode/src/vnd/vnodeStateMgr.c diff --git a/source/dnode/vnode/impl/src/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c similarity index 100% rename from source/dnode/vnode/impl/src/vnodeSync.c rename to source/dnode/vnode/src/vnd/vnodeSync.c diff --git a/source/dnode/vnode/impl/src/vnodeWrite.c b/source/dnode/vnode/src/vnd/vnodeWrite.c similarity index 89% rename from source/dnode/vnode/impl/src/vnodeWrite.c rename to source/dnode/vnode/src/vnd/vnodeWrite.c index ddcb93863a..185487757f 100644 --- a/source/dnode/vnode/impl/src/vnodeWrite.c +++ b/source/dnode/vnode/src/vnd/vnodeWrite.c @@ -83,8 +83,18 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { if (metaCreateTable(pVnode->pMeta, pCreateTbReq) < 0) { // TODO: handle error } + if (pCreateTbReq->type == TD_SUPER_TABLE) { + free(pCreateTbReq->stbCfg.pSchema); + free(pCreateTbReq->stbCfg.pTagSchema); + } else if (pCreateTbReq->type == TD_CHILD_TABLE) { + free(pCreateTbReq->ctbCfg.pTag); + } else { + free(pCreateTbReq->ntbCfg.pSchema); + } } - + taosArrayDestroy(vCreateTbBatchReq.pArray); + break; + case TDMT_VND_DROP_STB: case TDMT_VND_DROP_TABLE: // if (metaDropTable(pVnode->pMeta, vReq.dtReq.uid) < 0) { diff --git a/source/dnode/vnode/tq/test/CMakeLists.txt b/source/dnode/vnode/test/CMakeLists.txt similarity index 100% rename from source/dnode/vnode/tq/test/CMakeLists.txt rename to source/dnode/vnode/test/CMakeLists.txt diff --git a/source/dnode/vnode/tq/test/tqMetaTest.cpp b/source/dnode/vnode/test/tqMetaTest.cpp similarity index 100% rename from source/dnode/vnode/tq/test/tqMetaTest.cpp rename to source/dnode/vnode/test/tqMetaTest.cpp diff --git a/source/dnode/vnode/tq/test/tqSerializerTest.cpp b/source/dnode/vnode/test/tqSerializerTest.cpp similarity index 100% rename from source/dnode/vnode/tq/test/tqSerializerTest.cpp rename to source/dnode/vnode/test/tqSerializerTest.cpp diff --git a/source/dnode/vnode/tsdb/inc/tsdbint.h b/source/dnode/vnode/tsdb/inc/tsdbint.h deleted file mode 100644 index bdd1bd6f71..0000000000 --- a/source/dnode/vnode/tsdb/inc/tsdbint.h +++ /dev/null @@ -1,149 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_TSDB_INT_H_ -#define _TD_TSDB_INT_H_ - -#if 0 -// // TODO: remove the include -// #include -// #include -// #include -// #include -// #include -// #include -// #include -// #include - -#include "hash.h" -#include "os.h" -#include "taosdef.h" -#include "taoserror.h" -#include "tarray.h" -#include "tchecksum.h" -#include "tcoding.h" -#include "tcompression.h" -#include "tdataformat.h" -#include "tfs.h" -#include "tlist.h" -#include "tlockfree.h" -#include "tlog.h" -#include "tskiplist.h" -#include "tsocket.h" - -#include "tsdb.h" - -#ifdef __cplusplus -extern "C" { -#endif - -// Log -#include "tsdbLog.h" -// Meta -#include "tsdbMeta.h" -// Buffer -#include "tsdbBuffer.h" -// MemTable -#include "tsdbMemTable.h" -// File -#include "tsdbFile.h" -// FS -#include "tsdbFS.h" -// ReadImpl -#include "tsdbReadImpl.h" -// Commit -#include "tsdbCommit.h" -// Compact -#include "tsdbCompact.h" -// Commit Queue -#include "tsdbCommitQueue.h" - -#include "tsdbRowMergeBuf.h" -// Main definitions -struct STsdbRepo { - uint8_t state; - - STsdbCfg config; - - STsdbCfg save_config; // save apply config - bool config_changed; // config changed flag - pthread_mutex_t save_mutex; // protect save config - - uint8_t hasCachedLastColumn; - - STsdbAppH appH; - STsdbStat stat; - STsdbMeta* tsdbMeta; - STsdbBufPool* pPool; - SMemTable* mem; - SMemTable* imem; - STsdbFS* fs; - SRtn rtn; - tsem_t readyToCommit; - pthread_mutex_t mutex; - bool repoLocked; - int32_t code; // Commit code - - SMergeBuf mergeBuf; //used when update=2 - int8_t compactState; // compact state: inCompact/noCompact/waitingCompact? - pthread_t* pthread; -}; - -#define REPO_ID(r) (r)->config.tsdbId -#define REPO_CFG(r) (&((r)->config)) -#define REPO_FS(r) ((r)->fs) -#define IS_REPO_LOCKED(r) (r)->repoLocked -#define TSDB_SUBMIT_MSG_HEAD_SIZE sizeof(SSubmitMsg) - -int tsdbLockRepo(STsdbRepo* pRepo); -int tsdbUnlockRepo(STsdbRepo* pRepo); -STsdbMeta* tsdbGetMeta(STsdbRepo* pRepo); -int tsdbCheckCommit(STsdbRepo* pRepo); -int tsdbRestoreInfo(STsdbRepo* pRepo); -int tsdbCacheLastData(STsdbRepo *pRepo, STsdbCfg* oldCfg); -void tsdbGetRootDir(int repoid, char dirName[]); -void tsdbGetDataDir(int repoid, char dirName[]); - -static FORCE_INLINE STsdbBufBlock* tsdbGetCurrBufBlock(STsdbRepo* pRepo) { - ASSERT(pRepo != NULL); - if (pRepo->mem == NULL) return NULL; - - SListNode* pNode = listTail(pRepo->mem->bufBlockList); - if (pNode == NULL) return NULL; - - STsdbBufBlock* pBufBlock = NULL; - tdListNodeGetData(pRepo->mem->bufBlockList, pNode, (void*)(&pBufBlock)); - - return pBufBlock; -} - -static FORCE_INLINE int tsdbGetNextMaxTables(int tid) { - ASSERT(tid >= 1 && tid <= TSDB_MAX_TABLES); - int maxTables = TSDB_INIT_NTABLES; - while (true) { - maxTables = MIN(maxTables, TSDB_MAX_TABLES); - if (tid <= maxTables) break; - maxTables *= 2; - } - - return maxTables + 1; -} - -#ifdef __cplusplus -} -#endif - -#endif -#endif /* _TD_TSDB_INT_H_ */ diff --git a/source/dnode/vnode/tsdb/src/tsdbHealth.c b/source/dnode/vnode/tsdb/src/tsdbHealth.c deleted file mode 100644 index 4205f3e90f..0000000000 --- a/source/dnode/vnode/tsdb/src/tsdbHealth.c +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "os.h" -#include "tmsg.h" -#include "tarray.h" -#include "query.h" -#include "tglobal.h" -#include "tlist.h" -#include "tsdbint.h" -#include "tsdbBuffer.h" -#include "tsdbLog.h" -#include "tsdbHealth.h" -#include "ttimer.h" -#include "tthread.h" - - -// return malloc new block count -int32_t tsdbInsertNewBlock(STsdbRepo * pRepo) { - STsdbBufPool *pPool = pRepo->pPool; - int32_t cnt = 0; - - if(tsdbAllowNewBlock(pRepo)) { - STsdbBufBlock *pBufBlock = tsdbNewBufBlock(pPool->bufBlockSize); - if (pBufBlock) { - if (tdListAppend(pPool->bufBlockList, (void *)(&pBufBlock)) < 0) { - // append error - tsdbFreeBufBlock(pBufBlock); - } else { - pPool->nElasticBlocks ++; - cnt ++ ; - } - } - } - return cnt; -} - -// switch anther thread to run -void* cbKillQueryFree(void* param) { - STsdbRepo* pRepo = (STsdbRepo*)param; - // vnode - if(pRepo->appH.notifyStatus) { - pRepo->appH.notifyStatus(pRepo->appH.appH, TSDB_STATUS_COMMIT_NOBLOCK, TSDB_CODE_SUCCESS); - } - - // free - if(pRepo->pthread){ - void* p = pRepo->pthread; - pRepo->pthread = NULL; - free(p); - } - - return NULL; -} - -// return true do free , false do nothing -bool tsdbUrgeQueryFree(STsdbRepo * pRepo) { - // check previous running - if(pRepo->pthread && taosThreadRunning(pRepo->pthread)) { - tsdbWarn("vgId:%d pre urge thread is runing. nBlocks=%d nElasticBlocks=%d", REPO_ID(pRepo), pRepo->pPool->nBufBlocks, pRepo->pPool->nElasticBlocks); - return false; - } - // create new - pRepo->pthread = taosCreateThread(cbKillQueryFree, pRepo); - if(pRepo->pthread == NULL) { - tsdbError("vgId:%d create urge thread error.", REPO_ID(pRepo)); - return false; - } - return true; -} - -bool tsdbAllowNewBlock(STsdbRepo* pRepo) { - int32_t nMaxElastic = pRepo->config.totalBlocks/3; - STsdbBufPool* pPool = pRepo->pPool; - if(pPool->nElasticBlocks >= nMaxElastic) { - tsdbWarn("vgId:%d tsdbAllowNewBlock return fasle. nElasticBlock(%d) >= MaxElasticBlocks(%d)", REPO_ID(pRepo), pPool->nElasticBlocks, nMaxElastic); - return false; - } - return true; -} - -bool tsdbNoProblem(STsdbRepo* pRepo) { - if(listNEles(pRepo->pPool->bufBlockList) == 0) - return false; - return true; -} \ No newline at end of file diff --git a/source/dnode/vnode/tsdb/src/tsdbSync.c b/source/dnode/vnode/tsdb/src/tsdbSync.c deleted file mode 100644 index edcb84d091..0000000000 --- a/source/dnode/vnode/tsdb/src/tsdbSync.c +++ /dev/null @@ -1,724 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "taoserror.h" -#include "tsdbint.h" - -// Sync handle -typedef struct { - STsdbRepo *pRepo; - SRtn rtn; - SOCKET socketFd; - void * pBuf; - bool mfChanged; - SMFile * pmf; - SMFile mf; - SDFileSet df; - SDFileSet *pdf; -} SSyncH; - -#define SYNC_BUFFER(sh) ((sh)->pBuf) - -static void tsdbInitSyncH(SSyncH *pSyncH, STsdbRepo *pRepo, SOCKET socketFd); -static void tsdbDestroySyncH(SSyncH *pSyncH); -static int32_t tsdbSyncSendMeta(SSyncH *pSynch); -static int32_t tsdbSyncRecvMeta(SSyncH *pSynch); -static int32_t tsdbSendMetaInfo(SSyncH *pSynch); -static int32_t tsdbRecvMetaInfo(SSyncH *pSynch); -static int32_t tsdbSendDecision(SSyncH *pSynch, bool toSend); -static int32_t tsdbRecvDecision(SSyncH *pSynch, bool *toSend); -static int32_t tsdbSyncSendDFileSetArray(SSyncH *pSynch); -static int32_t tsdbSyncRecvDFileSetArray(SSyncH *pSynch); -static bool tsdbIsTowFSetSame(SDFileSet *pSet1, SDFileSet *pSet2); -static int32_t tsdbSyncSendDFileSet(SSyncH *pSynch, SDFileSet *pSet); -static int32_t tsdbSendDFileSetInfo(SSyncH *pSynch, SDFileSet *pSet); -static int32_t tsdbRecvDFileSetInfo(SSyncH *pSynch); -static int tsdbReload(STsdbRepo *pRepo, bool isMfChanged); - -int32_t tsdbSyncSend(void *tsdb, SOCKET socketFd) { - STsdbRepo *pRepo = (STsdbRepo *)tsdb; - SSyncH synch = {0}; - - tsdbInitSyncH(&synch, pRepo, socketFd); - // Disable TSDB commit - tsem_wait(&(pRepo->readyToCommit)); - - if (tsdbSyncSendMeta(&synch) < 0) { - tsdbError("vgId:%d, failed to send metafile since %s", REPO_ID(pRepo), tstrerror(terrno)); - goto _err; - } - - if (tsdbSyncSendDFileSetArray(&synch) < 0) { - tsdbError("vgId:%d, failed to send filesets since %s", REPO_ID(pRepo), tstrerror(terrno)); - goto _err; - } - - // Enable TSDB commit - tsem_post(&(pRepo->readyToCommit)); - tsdbDestroySyncH(&synch); - return 0; - -_err: - tsem_post(&(pRepo->readyToCommit)); - tsdbDestroySyncH(&synch); - return -1; -} - -int32_t tsdbSyncRecv(void *tsdb, SOCKET socketFd) { - STsdbRepo *pRepo = (STsdbRepo *)tsdb; - SSyncH synch = {0}; - - pRepo->state = TSDB_STATE_OK; - - tsdbInitSyncH(&synch, pRepo, socketFd); - tsem_wait(&(pRepo->readyToCommit)); - tsdbStartFSTxn(pRepo, 0, 0); - - if (tsdbSyncRecvMeta(&synch) < 0) { - tsdbError("vgId:%d, failed to recv metafile since %s", REPO_ID(pRepo), tstrerror(terrno)); - goto _err; - } - - if (tsdbSyncRecvDFileSetArray(&synch) < 0) { - tsdbError("vgId:%d, failed to recv filesets since %s", REPO_ID(pRepo), tstrerror(terrno)); - goto _err; - } - - tsdbEndFSTxn(pRepo); - tsem_post(&(pRepo->readyToCommit)); - tsdbDestroySyncH(&synch); - - // Reload file change - tsdbReload(pRepo, synch.mfChanged); - - return 0; - -_err: - tsdbEndFSTxnWithError(REPO_FS(pRepo)); - tsem_post(&(pRepo->readyToCommit)); - tsdbDestroySyncH(&synch); - return -1; -} - -static void tsdbInitSyncH(SSyncH *pSyncH, STsdbRepo *pRepo, SOCKET socketFd) { - pSyncH->pRepo = pRepo; - pSyncH->socketFd = socketFd; - tsdbGetRtnSnap(pRepo, &(pSyncH->rtn)); -} - -static void tsdbDestroySyncH(SSyncH *pSyncH) { taosTZfree(pSyncH->pBuf); } - -static int32_t tsdbSyncSendMeta(SSyncH *pSynch) { - STsdbRepo *pRepo = pSynch->pRepo; - bool toSendMeta = false; - SMFile mf; - - // Send meta info to remote - tsdbInfo("vgId:%d, metainfo will be sent", REPO_ID(pRepo)); - if (tsdbSendMetaInfo(pSynch) < 0) { - tsdbError("vgId:%d, failed to send metainfo since %s", REPO_ID(pRepo), tstrerror(terrno)); - return -1; - } - - if (pRepo->fs->cstatus->pmf == NULL) { - // No meta file, not need to wait to retrieve meta file - tsdbInfo("vgId:%d, metafile not exist, no need to send", REPO_ID(pRepo)); - return 0; - } - - if (tsdbRecvDecision(pSynch, &toSendMeta) < 0) { - tsdbError("vgId:%d, failed to recv decision while send meta since %s", REPO_ID(pRepo), tstrerror(terrno)); - return -1; - } - - if (toSendMeta) { - tsdbInitMFileEx(&mf, pRepo->fs->cstatus->pmf); - if (tsdbOpenMFile(&mf, O_RDONLY) < 0) { - tsdbError("vgId:%d, failed to open file while send metafile since %s", REPO_ID(pRepo), tstrerror(terrno)); - return -1; - } - - int64_t writeLen = mf.info.size; - tsdbInfo("vgId:%d, metafile:%s will be sent, size:%" PRId64, REPO_ID(pRepo), mf.f.aname, writeLen); - - int64_t ret = taosSendFile(pSynch->socketFd, TSDB_FILE_FD(&mf), 0, writeLen); - if (ret != writeLen) { - terrno = TAOS_SYSTEM_ERROR(errno); - tsdbError("vgId:%d, failed to send metafile since %s, ret:%" PRId64 " writeLen:%" PRId64, REPO_ID(pRepo), - tstrerror(terrno), ret, writeLen); - tsdbCloseMFile(&mf); - return -1; - } - - tsdbCloseMFile(&mf); - tsdbInfo("vgId:%d, metafile is sent", REPO_ID(pRepo)); - } else { - tsdbInfo("vgId:%d, metafile is same, no need to send", REPO_ID(pRepo)); - } - - return 0; -} - -static int32_t tsdbSyncRecvMeta(SSyncH *pSynch) { - STsdbRepo *pRepo = pSynch->pRepo; - SMFile * pLMFile = pRepo->fs->cstatus->pmf; - - // Recv meta info from remote - if (tsdbRecvMetaInfo(pSynch) < 0) { - tsdbError("vgId:%d, failed to recv metainfo since %s", REPO_ID(pRepo), tstrerror(terrno)); - return -1; - } - - // No meta file, do nothing (rm local meta file) - if (pSynch->pmf == NULL) { - if (pLMFile == NULL) { - pSynch->mfChanged = false; - } else { - pSynch->mfChanged = true; - } - tsdbInfo("vgId:%d, metafile not exist in remote, no need to recv", REPO_ID(pRepo)); - return 0; - } - - if (pLMFile == NULL || pSynch->pmf->info.size != pLMFile->info.size || - pSynch->pmf->info.magic != pLMFile->info.magic || TSDB_FILE_IS_BAD(pLMFile)) { - // Local has no meta file or has a different meta file, need to copy from remote - pSynch->mfChanged = true; - - if (tsdbSendDecision(pSynch, true) < 0) { - tsdbError("vgId:%d, failed to send decision while recv metafile since %s", REPO_ID(pRepo), tstrerror(terrno)); - return -1; - } - - tsdbInfo("vgId:%d, metafile will be received", REPO_ID(pRepo)); - - // Recv from remote - SMFile mf; - SDiskID did = {.level = TFS_PRIMARY_LEVEL, .id = TFS_PRIMARY_ID}; - tsdbInitMFile(&mf, did, REPO_ID(pRepo), FS_TXN_VERSION(REPO_FS(pRepo))); - if (tsdbCreateMFile(&mf, false) < 0) { - tsdbError("vgId:%d, failed to create file while recv metafile since %s", REPO_ID(pRepo), tstrerror(terrno)); - return -1; - } - - tsdbInfo("vgId:%d, metafile:%s is created", REPO_ID(pRepo), mf.f.aname); - - int64_t readLen = pSynch->pmf->info.size; - int64_t ret = taosCopyFds(pSynch->socketFd, TSDB_FILE_FD(&mf), readLen); - if (ret != readLen) { - terrno = TAOS_SYSTEM_ERROR(errno); - tsdbError("vgId:%d, failed to recv metafile since %s, ret:%" PRId64 " readLen:%" PRId64, REPO_ID(pRepo), - tstrerror(terrno), ret, readLen); - tsdbCloseMFile(&mf); - tsdbRemoveMFile(&mf); - return -1; - } - - tsdbInfo("vgId:%d, metafile is received, size:%" PRId64, REPO_ID(pRepo), readLen); - - mf.info = pSynch->pmf->info; - tsdbCloseMFile(&mf); - tsdbUpdateMFile(REPO_FS(pRepo), &mf); - } else { - pSynch->mfChanged = false; - tsdbInfo("vgId:%d, metafile is same, no need to recv", REPO_ID(pRepo)); - if (tsdbSendDecision(pSynch, false) < 0) { - tsdbError("vgId:%d, failed to send decision while recv metafile since %s", REPO_ID(pRepo), tstrerror(terrno)); - return -1; - } - tsdbUpdateMFile(REPO_FS(pRepo), pLMFile); - } - - return 0; -} - -static int32_t tsdbSendMetaInfo(SSyncH *pSynch) { - STsdbRepo *pRepo = pSynch->pRepo; - uint32_t tlen = 0; - SMFile * pMFile = pRepo->fs->cstatus->pmf; - - if (pMFile) { - tlen = tlen + tsdbEncodeSMFileEx(NULL, pMFile) + sizeof(TSCKSUM); - } - - if (tsdbMakeRoom((void **)(&SYNC_BUFFER(pSynch)), tlen + sizeof(tlen)) < 0) { - tsdbError("vgId:%d, failed to makeroom while send metainfo since %s", REPO_ID(pRepo), tstrerror(terrno)); - return -1; - } - - void *ptr = SYNC_BUFFER(pSynch); - taosEncodeFixedU32(&ptr, tlen); - void *tptr = ptr; - if (pMFile) { - tsdbEncodeSMFileEx(&ptr, pMFile); - taosCalcChecksumAppend(0, (uint8_t *)tptr, tlen); - } - - int32_t writeLen = tlen + sizeof(uint32_t); - int32_t ret = taosWriteMsg(pSynch->socketFd, SYNC_BUFFER(pSynch), writeLen); - if (ret != writeLen) { - terrno = TAOS_SYSTEM_ERROR(errno); - tsdbError("vgId:%d, failed to send metainfo since %s, ret:%d writeLen:%d", REPO_ID(pRepo), tstrerror(terrno), ret, - writeLen); - return -1; - } - - tsdbInfo("vgId:%d, metainfo is sent, tlen:%d, writeLen:%d", REPO_ID(pRepo), tlen, writeLen); - return 0; -} - -static int32_t tsdbRecvMetaInfo(SSyncH *pSynch) { - STsdbRepo *pRepo = pSynch->pRepo; - uint32_t tlen = 0; - char buf[64] = {0}; - - int32_t readLen = sizeof(uint32_t); - int32_t ret = taosReadMsg(pSynch->socketFd, buf, readLen); - if (ret != readLen) { - terrno = TAOS_SYSTEM_ERROR(errno); - tsdbError("vgId:%d, failed to recv metalen, ret:%d readLen:%d", REPO_ID(pRepo), ret, readLen); - return -1; - } - - taosDecodeFixedU32(buf, &tlen); - - tsdbInfo("vgId:%d, metalen is received, readLen:%d, tlen:%d", REPO_ID(pRepo), readLen, tlen); - if (tlen == 0) { - pSynch->pmf = NULL; - return 0; - } - - if (tsdbMakeRoom((void **)(&SYNC_BUFFER(pSynch)), tlen) < 0) { - tsdbError("vgId:%d, failed to makeroom while recv metainfo since %s", REPO_ID(pRepo), tstrerror(terrno)); - return -1; - } - - ret = taosReadMsg(pSynch->socketFd, SYNC_BUFFER(pSynch), tlen); - if (ret != tlen) { - terrno = TAOS_SYSTEM_ERROR(errno); - tsdbError("vgId:%d, failed to recv metainfo, ret:%d tlen:%d", REPO_ID(pRepo), ret, tlen); - return -1; - } - - tsdbInfo("vgId:%d, metainfo is received, tlen:%d", REPO_ID(pRepo), tlen); - if (!taosCheckChecksumWhole((uint8_t *)SYNC_BUFFER(pSynch), tlen)) { - terrno = TSDB_CODE_TDB_MESSED_MSG; - tsdbError("vgId:%d, failed to checksum while recv metainfo since %s", REPO_ID(pRepo), tstrerror(terrno)); - return -1; - } - - pSynch->pmf = &(pSynch->mf); - tsdbDecodeSMFileEx(SYNC_BUFFER(pSynch), pSynch->pmf); - - return 0; -} - -static int32_t tsdbSendDecision(SSyncH *pSynch, bool toSend) { - STsdbRepo *pRepo = pSynch->pRepo; - uint8_t decision = toSend; - - int32_t writeLen = sizeof(uint8_t); - int32_t ret = taosWriteMsg(pSynch->socketFd, (void *)(&decision), writeLen); - if (ret != writeLen) { - terrno = TAOS_SYSTEM_ERROR(errno); - tsdbError("vgId:%d, failed to send decison, ret:%d writeLen:%d", REPO_ID(pRepo), ret, writeLen); - return -1; - } - - return 0; -} - -static int32_t tsdbRecvDecision(SSyncH *pSynch, bool *toSend) { - STsdbRepo *pRepo = pSynch->pRepo; - uint8_t decision = 0; - - int32_t readLen = sizeof(uint8_t); - int32_t ret = taosReadMsg(pSynch->socketFd, (void *)(&decision), readLen); - if (ret != readLen) { - terrno = TAOS_SYSTEM_ERROR(errno); - tsdbError("vgId:%d, failed to recv decison, ret:%d readLen:%d", REPO_ID(pRepo), ret, readLen); - return -1; - } - - *toSend = decision; - return 0; -} - -static int32_t tsdbSyncSendDFileSetArray(SSyncH *pSynch) { - STsdbRepo *pRepo = pSynch->pRepo; - STsdbFS * pfs = REPO_FS(pRepo); - SFSIter fsiter; - SDFileSet *pSet; - - tsdbFSIterInit(&fsiter, pfs, TSDB_FS_ITER_FORWARD); - - do { - pSet = tsdbFSIterNext(&fsiter); - if (tsdbSyncSendDFileSet(pSynch, pSet) < 0) { - tsdbError("vgId:%d, failed to send fileset:%d since %s", REPO_ID(pRepo), pSet ? pSet->fid : -1, - tstrerror(terrno)); - return -1; - } - - // No more file set to send, jut break - if (pSet == NULL) { - tsdbInfo("vgId:%d, no filesets any more", REPO_ID(pRepo)); - break; - } - } while (true); - - return 0; -} - -static int32_t tsdbSyncRecvDFileSetArray(SSyncH *pSynch) { - STsdbRepo *pRepo = pSynch->pRepo; - STsdbFS * pfs = REPO_FS(pRepo); - SFSIter fsiter; - SDFileSet *pLSet; // Local file set - - tsdbFSIterInit(&fsiter, pfs, TSDB_FS_ITER_FORWARD); - - pLSet = tsdbFSIterNext(&fsiter); - if (tsdbRecvDFileSetInfo(pSynch) < 0) { - tsdbError("vgId:%d, failed to recv fileset since %s", REPO_ID(pRepo), tstrerror(terrno)); - return -1; - } - - while (true) { - if (pLSet == NULL && pSynch->pdf == NULL) { - tsdbInfo("vgId:%d, all filesets is disposed", REPO_ID(pRepo)); - break; - } else { - tsdbInfo("vgId:%d, fileset local:%d remote:%d, will be disposed", REPO_ID(pRepo), pLSet != NULL ? pLSet->fid : -1, - pSynch->pdf != NULL ? pSynch->pdf->fid : -1); - } - - if (pLSet && (pSynch->pdf == NULL || pLSet->fid < pSynch->pdf->fid)) { - // remote not has pLSet->fid set, just remove local (do nothing to remote the fset) - tsdbInfo("vgId:%d, fileset:%d smaller than remote:%d, remove it", REPO_ID(pRepo), pLSet->fid, - pSynch->pdf != NULL ? pSynch->pdf->fid : -1); - pLSet = tsdbFSIterNext(&fsiter); - } else { - if (pLSet && pSynch->pdf && pLSet->fid == pSynch->pdf->fid && tsdbIsTowFSetSame(pLSet, pSynch->pdf) && - tsdbFSetIsOk(pLSet)) { - // Just keep local files and notify remote not to send - tsdbInfo("vgId:%d, fileset:%d is same and no need to recv", REPO_ID(pRepo), pLSet->fid); - - if (tsdbUpdateDFileSet(pfs, pLSet) < 0) { - tsdbError("vgId:%d, failed to update fileset since %s", REPO_ID(pRepo), tstrerror(terrno)); - return -1; - } - - if (tsdbSendDecision(pSynch, false) < 0) { - tsdbError("vgId:%d, failed to send decision since %s", REPO_ID(pRepo), tstrerror(terrno)); - return -1; - } - } else { - // Need to copy from remote - int fidLevel = tsdbGetFidLevel(pSynch->pdf->fid, &(pSynch->rtn)); - if (fidLevel < 0) { // expired fileset - tsdbInfo("vgId:%d, fileset:%d will be skipped as expired", REPO_ID(pRepo), pSynch->pdf->fid); - if (tsdbSendDecision(pSynch, false) < 0) { - tsdbError("vgId:%d, failed to send decision since %s", REPO_ID(pRepo), tstrerror(terrno)); - return -1; - } - // Move forward - if (tsdbRecvDFileSetInfo(pSynch) < 0) { - tsdbError("vgId:%d, failed to recv fileset since %s", REPO_ID(pRepo), tstrerror(terrno)); - return -1; - } - if (pLSet) { - pLSet = tsdbFSIterNext(&fsiter); - } - // Next loop - continue; - } else { - tsdbInfo("vgId:%d, fileset:%d will be received", REPO_ID(pRepo), pSynch->pdf->fid); - // Notify remote to send there file here - if (tsdbSendDecision(pSynch, true) < 0) { - tsdbError("vgId:%d, failed to send decision since %s", REPO_ID(pRepo), tstrerror(terrno)); - return -1; - } - } - - // Create local files and copy from remote - SDiskID did; - SDFileSet fset; - - tfsAllocDisk(fidLevel, &(did.level), &(did.id)); - if (did.level == TFS_UNDECIDED_LEVEL) { - terrno = TSDB_CODE_TDB_NO_AVAIL_DISK; - tsdbError("vgId:%d, failed allc disk since %s", REPO_ID(pRepo), tstrerror(terrno)); - return -1; - } - - tsdbInitDFileSet(&fset, did, REPO_ID(pRepo), pSynch->pdf->fid, FS_TXN_VERSION(pfs)); - - // Create new FSET - if (tsdbCreateDFileSet(&fset, false) < 0) { - tsdbError("vgId:%d, failed to create fileset since %s", REPO_ID(pRepo), tstrerror(terrno)); - return -1; - } - - for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { - SDFile *pDFile = TSDB_DFILE_IN_SET(&fset, ftype); // local file - SDFile *pRDFile = TSDB_DFILE_IN_SET(pSynch->pdf, ftype); // remote file - - tsdbInfo("vgId:%d, file:%s will be received, osize:%" PRIu64 " rsize:%" PRIu64, REPO_ID(pRepo), - pDFile->f.aname, pDFile->info.size, pRDFile->info.size); - - int64_t writeLen = pRDFile->info.size; - int64_t ret = taosCopyFds(pSynch->socketFd, pDFile->fd, writeLen); - if (ret != writeLen) { - terrno = TAOS_SYSTEM_ERROR(errno); - tsdbError("vgId:%d, failed to recv file:%s since %s, ret:%" PRId64 " writeLen:%" PRId64, REPO_ID(pRepo), - pDFile->f.aname, tstrerror(terrno), ret, writeLen); - tsdbCloseDFileSet(&fset); - tsdbRemoveDFileSet(&fset); - return -1; - } - - // Update new file info - pDFile->info = pRDFile->info; - tsdbInfo("vgId:%d, file:%s is received, size:%" PRId64, REPO_ID(pRepo), pDFile->f.aname, writeLen); - } - - tsdbCloseDFileSet(&fset); - if (tsdbUpdateDFileSet(pfs, &fset) < 0) { - tsdbInfo("vgId:%d, fileset:%d failed to update since %s", REPO_ID(pRepo), fset.fid, tstrerror(terrno)); - return -1; - } - - tsdbInfo("vgId:%d, fileset:%d is received", REPO_ID(pRepo), pSynch->pdf->fid); - } - - // Move forward - if (tsdbRecvDFileSetInfo(pSynch) < 0) { - tsdbError("vgId:%d, failed to recv fileset since %s", REPO_ID(pRepo), tstrerror(terrno)); - return -1; - } - - if (pLSet) { - pLSet = tsdbFSIterNext(&fsiter); - } - } - -#if 0 - if (pLSet == NULL) { - // Copy from remote >>>>>>>>>>> - } else { - if (pSynch->pdf == NULL) { - // Remove local file, just ignore ++++++++++++++ - pLSet = tsdbFSIterNext(&fsiter); - } else { - if (pLSet->fid < pSynch->pdf->fid) { - // Remove local file, just ignore ++++++++++++ - pLSet = tsdbFSIterNext(&fsiter); - } else if (pLSet->fid > pSynch->pdf->fid){ - // Copy from remote >>>>>>>>>>>>>> - if (tsdbRecvDFileSetInfo(pSynch) < 0) { - // TODO - return -1; - } - } else { - if (true/*TODO: is same fset*/) { - // No need to copy --------------------- - } else { - // copy from remote >>>>>>>>>>>>>. - } - } - } - } -#endif - } - - return 0; -} - -static bool tsdbIsTowFSetSame(SDFileSet *pSet1, SDFileSet *pSet2) { - for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { - SDFile *pDFile1 = TSDB_DFILE_IN_SET(pSet1, ftype); - SDFile *pDFile2 = TSDB_DFILE_IN_SET(pSet2, ftype); - - if (pDFile1->info.size != pDFile2->info.size || pDFile1->info.magic != pDFile2->info.magic) { - return false; - } - } - - return true; -} - -static int32_t tsdbSyncSendDFileSet(SSyncH *pSynch, SDFileSet *pSet) { - STsdbRepo *pRepo = pSynch->pRepo; - bool toSend = false; - - // skip expired fileset - if (pSet && tsdbGetFidLevel(pSet->fid, &(pSynch->rtn)) < 0) { - tsdbInfo("vgId:%d, don't sync send since fileset:%d smaller than minFid:%d", REPO_ID(pRepo), pSet->fid, - pSynch->rtn.minFid); - return 0; - } - - if (tsdbSendDFileSetInfo(pSynch, pSet) < 0) { - tsdbError("vgId:%d, failed to send fileset:%d info since %s", REPO_ID(pRepo), pSet ? pSet->fid : -1, tstrerror(terrno)); - return -1; - } - - // No file any more, no need to send file, just return - if (pSet == NULL) { - return 0; - } - - if (tsdbRecvDecision(pSynch, &toSend) < 0) { - tsdbError("vgId:%d, failed to recv decision while send fileset:%d since %s", REPO_ID(pRepo), pSet->fid, - tstrerror(terrno)); - return -1; - } - - if (toSend) { - tsdbInfo("vgId:%d, fileset:%d will be sent", REPO_ID(pRepo), pSet->fid); - - for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { - SDFile df = *TSDB_DFILE_IN_SET(pSet, ftype); - - if (tsdbOpenDFile(&df, O_RDONLY) < 0) { - tsdbError("vgId:%d, failed to file:%s since %s", REPO_ID(pRepo), df.f.aname, tstrerror(terrno)); - return -1; - } - - int64_t writeLen = df.info.size; - tsdbInfo("vgId:%d, file:%s will be sent, size:%" PRId64, REPO_ID(pRepo), df.f.aname, writeLen); - - int64_t ret = taosSendFile(pSynch->socketFd, TSDB_FILE_FD(&df), 0, writeLen); - if (ret != writeLen) { - terrno = TAOS_SYSTEM_ERROR(errno); - tsdbError("vgId:%d, failed to send file:%s since %s, ret:%" PRId64 " writeLen:%" PRId64, REPO_ID(pRepo), - df.f.aname, tstrerror(terrno), ret, writeLen); - tsdbCloseDFile(&df); - return -1; - } - - tsdbInfo("vgId:%d, file:%s is sent", REPO_ID(pRepo), df.f.aname); - tsdbCloseDFile(&df); - } - - tsdbInfo("vgId:%d, fileset:%d is sent", REPO_ID(pRepo), pSet->fid); - } else { - tsdbInfo("vgId:%d, fileset:%d is same, no need to send", REPO_ID(pRepo), pSet->fid); - } - - return 0; -} - -static int32_t tsdbSendDFileSetInfo(SSyncH *pSynch, SDFileSet *pSet) { - STsdbRepo *pRepo = pSynch->pRepo; - uint32_t tlen = 0; - - if (pSet) { - tlen = tsdbEncodeDFileSetEx(NULL, pSet) + sizeof(TSCKSUM); - } - - if (tsdbMakeRoom((void **)(&SYNC_BUFFER(pSynch)), tlen + sizeof(tlen)) < 0) { - tsdbError("vgId:%d, failed to makeroom while send fileinfo since %s", REPO_ID(pRepo), tstrerror(terrno)); - return -1; - } - - void *ptr = SYNC_BUFFER(pSynch); - taosEncodeFixedU32(&ptr, tlen); - void *tptr = ptr; - if (pSet) { - tsdbEncodeDFileSetEx(&ptr, pSet); - taosCalcChecksumAppend(0, (uint8_t *)tptr, tlen); - } - - int32_t writeLen = tlen + sizeof(uint32_t); - int32_t ret = taosWriteMsg(pSynch->socketFd, SYNC_BUFFER(pSynch), writeLen); - if (ret != writeLen) { - terrno = TAOS_SYSTEM_ERROR(errno); - tsdbError("vgId:%d, failed to send fileinfo, ret:%d writeLen:%d", REPO_ID(pRepo), ret, writeLen); - return -1; - } - - return 0; -} - -static int32_t tsdbRecvDFileSetInfo(SSyncH *pSynch) { - STsdbRepo *pRepo = pSynch->pRepo; - uint32_t tlen; - char buf[64] = {0}; - - int32_t readLen = sizeof(uint32_t); - int32_t ret = taosReadMsg(pSynch->socketFd, buf, readLen); - if (ret != readLen) { - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - taosDecodeFixedU32(buf, &tlen); - - tsdbInfo("vgId:%d, fileinfo len:%d is received", REPO_ID(pRepo), tlen); - if (tlen == 0) { - pSynch->pdf = NULL; - return 0; - } - - if (tsdbMakeRoom((void **)(&SYNC_BUFFER(pSynch)), tlen) < 0) { - tsdbError("vgId:%d, failed to makeroom while recv fileinfo since %s", REPO_ID(pRepo), tstrerror(terrno)); - return -1; - } - - ret = taosReadMsg(pSynch->socketFd, SYNC_BUFFER(pSynch), tlen); - if (ret != tlen) { - terrno = TAOS_SYSTEM_ERROR(errno); - tsdbError("vgId:%d, failed to recv fileinfo, ret:%d readLen:%d", REPO_ID(pRepo), ret, tlen); - return -1; - } - - if (!taosCheckChecksumWhole((uint8_t *)SYNC_BUFFER(pSynch), tlen)) { - terrno = TSDB_CODE_TDB_MESSED_MSG; - tsdbError("vgId:%d, failed to checksum while recv fileinfo since %s", REPO_ID(pRepo), tstrerror(terrno)); - return -1; - } - - pSynch->pdf = &(pSynch->df); - tsdbDecodeDFileSetEx(SYNC_BUFFER(pSynch), pSynch->pdf); - - return 0; -} - -static int tsdbReload(STsdbRepo *pRepo, bool isMfChanged) { - // TODO: may need to stop and restart stream - // if (isMfChanged) { - tsdbCloseMeta(pRepo); - tsdbFreeMeta(pRepo->tsdbMeta); - pRepo->tsdbMeta = tsdbNewMeta(REPO_CFG(pRepo)); - tsdbOpenMeta(pRepo); - tsdbLoadMetaCache(pRepo, true); - // } - - tsdbUnRefMemTable(pRepo, pRepo->mem); - tsdbUnRefMemTable(pRepo, pRepo->imem); - pRepo->mem = NULL; - pRepo->imem = NULL; - - if (tsdbRestoreInfo(pRepo) < 0) { - tsdbError("vgId:%d failed to restore info from file since %s", REPO_ID(pRepo), tstrerror(terrno)); - return -1; - } - - return 0; -} \ No newline at end of file diff --git a/source/dnode/vnode/tsdb/test/tsdbTests.cpp b/source/dnode/vnode/tsdb/test/tsdbTests.cpp deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/source/libs/executor/CMakeLists.txt b/source/libs/executor/CMakeLists.txt index 04b5fab4bf..9b53cc1fbb 100644 --- a/source/libs/executor/CMakeLists.txt +++ b/source/libs/executor/CMakeLists.txt @@ -13,7 +13,7 @@ add_library(executor STATIC ${EXECUTOR_SRC}) # INTERFACE_INCLUDE_DIRECTORIES "${CMAKE_SOURCE_DIR}/include/libs/executor" # ) target_link_libraries(executor - PRIVATE os util common function parser planner qcom tsdb + PRIVATE os util common function parser planner qcom vnode ) target_include_directories( diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 9aeb979806..0dfcc9b1d8 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -18,7 +18,6 @@ #include "ttime.h" #include "exception.h" -#include "../../../../contrib/cJson/cJSON.h" #include "executorimpl.h" #include "function.h" #include "tcompare.h" @@ -7188,31 +7187,47 @@ static SExecTaskInfo* createExecTaskInfo(uint64_t queryId) { SOperatorInfo* doCreateOperatorTreeNode(SPhyNode* pPhyNode, SExecTaskInfo* pTaskInfo, void* param) { if (pPhyNode->pChildren == NULL || taosArrayGetSize(pPhyNode->pChildren) == 0) { if (pPhyNode->info.type == OP_TableScan) { + SScanPhyNode* pScanPhyNode = (SScanPhyNode*) pPhyNode; size_t numOfCols = taosArrayGetSize(pPhyNode->pTargets); - SOperatorInfo* pOperatorInfo = createTableScanOperator(param, TSDB_ORDER_ASC, numOfCols, 1, pTaskInfo); + SOperatorInfo* pOperatorInfo = createTableScanOperator(param, pScanPhyNode->order, numOfCols, pScanPhyNode->count, pTaskInfo); pTaskInfo->pRoot = pOperatorInfo; + } else { + assert(0); } } } int32_t doCreateExecTaskInfo(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, void* readerHandle) { - STsdbQueryCond cond = {.order = TSDB_ORDER_ASC, .numOfCols = 2, .loadExternalRows = false}; + STsdbQueryCond cond = {.loadExternalRows = false}; cond.twindow.skey = INT64_MIN; cond.twindow.ekey = INT64_MAX; - cond.colList = calloc(cond.numOfCols, sizeof(SColumnInfo)); - // todo set the correct table column info - cond.colList[0].type = TSDB_DATA_TYPE_TIMESTAMP; - cond.colList[0].bytes = sizeof(uint64_t); - cond.colList[0].colId = 1; + uint64_t uid = 0; + SPhyNode* pPhyNode = pPlan->pNode; + if (pPhyNode->info.type == OP_TableScan) { - cond.colList[1].type = TSDB_DATA_TYPE_INT; - cond.colList[1].bytes = sizeof(int32_t); - cond.colList[1].colId = 2; + SScanPhyNode* pScanNode = (SScanPhyNode*) pPhyNode; + uid = pScanNode->uid; + cond.order = pScanNode->order; + cond.numOfCols = taosArrayGetSize(pScanNode->node.pTargets); + cond.colList = calloc(cond.numOfCols, sizeof(SColumnInfo)); + + for(int32_t i = 0; i < cond.numOfCols; ++i) { + SExprInfo* pExprInfo = taosArrayGetP(pScanNode->node.pTargets, i); + assert(pExprInfo->pExpr->nodeType == TEXPR_COL_NODE); + + SSchema* pSchema = pExprInfo->pExpr->pSchema; + cond.colList[i].type = pSchema->type; + cond.colList[i].bytes = pSchema->bytes; + cond.colList[i].colId = pSchema->colId; + } + } else { + assert(0); + } STableGroupInfo group = {.numOfTables = 1, .pGroupList = taosArrayInit(1, POINTER_BYTES)}; SArray* pa = taosArrayInit(1, sizeof(STableKeyInfo)); - STableKeyInfo info = {.pTable = NULL, .lastKey = 0, .uid = 1}; + STableKeyInfo info = {.pTable = NULL, .lastKey = 0, .uid = uid}; taosArrayPush(pa, &info); taosArrayPush(group.pGroupList, &pa); diff --git a/source/libs/parser/inc/astToMsg.h b/source/libs/parser/inc/astToMsg.h index 83683c2bfc..8acbc6bc11 100644 --- a/source/libs/parser/inc/astToMsg.h +++ b/source/libs/parser/inc/astToMsg.h @@ -8,10 +8,10 @@ SCreateUserReq* buildUserManipulationMsg(SSqlInfo* pInfo, int32_t* outputLen, int64_t id, char* msgBuf, int32_t msgLen); SCreateAcctReq* buildAcctManipulationMsg(SSqlInfo* pInfo, int32_t* outputLen, int64_t id, char* msgBuf, int32_t msgLen); SDropUserReq* buildDropUserMsg(SSqlInfo* pInfo, int32_t* outputLen, int64_t id, char* msgBuf, int32_t msgLen); -SShowReq* buildShowMsg(SShowInfo* pShowInfo, SParseBasicCtx* pParseCtx, char* msgBuf, int32_t msgLen); -SCreateDbReq* buildCreateDbMsg(SCreateDbInfo* pCreateDbInfo, SParseBasicCtx *pCtx, SMsgBuf* pMsgBuf); -SMCreateStbReq* buildCreateStbMsg(SCreateTableSql* pCreateTableSql, int32_t* len, SParseBasicCtx* pParseCtx, SMsgBuf* pMsgBuf); -SMDropStbReq* buildDropStableMsg(SSqlInfo* pInfo, int32_t* len, SParseBasicCtx* pParseCtx, SMsgBuf* pMsgBuf); +SShowReq* buildShowMsg(SShowInfo* pShowInfo, SParseContext* pParseCtx, char* msgBuf, int32_t msgLen); +SCreateDbReq* buildCreateDbMsg(SCreateDbInfo* pCreateDbInfo, SParseContext *pCtx, SMsgBuf* pMsgBuf); +SMCreateStbReq* buildCreateStbMsg(SCreateTableSql* pCreateTableSql, int32_t* len, SParseContext* pParseCtx, SMsgBuf* pMsgBuf); +SMDropStbReq* buildDropStableMsg(SSqlInfo* pInfo, int32_t* len, SParseContext* pParseCtx, SMsgBuf* pMsgBuf); SCreateDnodeReq *buildCreateDnodeMsg(SSqlInfo* pInfo, int32_t* len, SMsgBuf* pMsgBuf); SDropDnodeReq *buildDropDnodeMsg(SSqlInfo* pInfo, int32_t* len, SMsgBuf* pMsgBuf); diff --git a/source/libs/parser/inc/parserInt.h b/source/libs/parser/inc/parserInt.h index 10ec335fc8..abb19b3ab6 100644 --- a/source/libs/parser/inc/parserInt.h +++ b/source/libs/parser/inc/parserInt.h @@ -51,7 +51,7 @@ void clearAllTableMetaInfo(SQueryStmtInfo* pQueryInfo, bool removeMeta, uint64_t * @param msgBufLen * @return */ -int32_t qParserValidateSqlNode(SParseBasicCtx *pCtx, SSqlInfo* pInfo, SQueryStmtInfo* pQueryInfo, char* msgBuf, int32_t msgBufLen); +int32_t qParserValidateSqlNode(SParseContext *pCtx, SSqlInfo* pInfo, SQueryStmtInfo* pQueryInfo, char* msgBuf, int32_t msgBufLen); /** * validate the ddl ast, and convert the ast to the corresponding message format @@ -60,7 +60,7 @@ int32_t qParserValidateSqlNode(SParseBasicCtx *pCtx, SSqlInfo* pInfo, SQueryStmt * @param type * @return */ -SDclStmtInfo* qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, char* msgBuf, int32_t msgBufLen); +SDclStmtInfo* qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseContext* pCtx, char* msgBuf, int32_t msgBufLen); /** * @@ -70,7 +70,7 @@ SDclStmtInfo* qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, c * @param msgBufLen * @return */ -SVnodeModifOpStmtInfo* qParserValidateCreateTbSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, char* msgBuf, int32_t msgBufLen); +SVnodeModifOpStmtInfo* qParserValidateCreateTbSqlNode(SSqlInfo* pInfo, SParseContext* pCtx, char* msgBuf, int32_t msgBufLen); /** * Evaluate the numeric and timestamp arithmetic expression in the WHERE clause. @@ -98,7 +98,7 @@ int32_t checkForInvalidExpr(SQueryStmtInfo* pQueryInfo, SMsgBuf* pMsgBuf); * @param msgBufLen * @return */ -int32_t qParserExtractRequestedMetaInfo(const SSqlInfo* pSqlInfo, SCatalogReq* pMetaInfo, SParseBasicCtx *pCtx, char* msg, int32_t msgBufLen); +int32_t qParserExtractRequestedMetaInfo(const SSqlInfo* pSqlInfo, SCatalogReq* pMetaInfo, SParseContext *pCtx, char* msg, int32_t msgBufLen); /** * Destroy the meta data request structure. diff --git a/source/libs/parser/inc/parserUtil.h b/source/libs/parser/inc/parserUtil.h index 764b363394..d660d36d3f 100644 --- a/source/libs/parser/inc/parserUtil.h +++ b/source/libs/parser/inc/parserUtil.h @@ -81,7 +81,7 @@ int32_t KvRowAppend(const void *value, int32_t len, void *param); typedef int32_t (*_row_append_fn_t)(const void *value, int32_t len, void *param); int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int16_t timePrec, char* tmpTokenBuf, _row_append_fn_t func, void* param, SMsgBuf* pMsgBuf); -int32_t createSName(SName* pName, SToken* pTableName, SParseBasicCtx* pParseCtx, SMsgBuf* pMsgBuf); +int32_t createSName(SName* pName, SToken* pTableName, SParseContext* pParseCtx, SMsgBuf* pMsgBuf); #ifdef __cplusplus } diff --git a/source/libs/parser/src/astToMsg.c b/source/libs/parser/src/astToMsg.c index 5f45ce824e..5b841594e0 100644 --- a/source/libs/parser/src/astToMsg.c +++ b/source/libs/parser/src/astToMsg.c @@ -85,7 +85,7 @@ SDropUserReq* buildDropUserMsg(SSqlInfo* pInfo, int32_t *msgLen, int64_t id, cha return pMsg; } -SShowReq* buildShowMsg(SShowInfo* pShowInfo, SParseBasicCtx *pCtx, char* msgBuf, int32_t msgLen) { +SShowReq* buildShowMsg(SShowInfo* pShowInfo, SParseContext *pCtx, char* msgBuf, int32_t msgLen) { SShowReq* pShowMsg = calloc(1, sizeof(SShowReq)); pShowMsg->type = pShowInfo->showType; @@ -210,7 +210,7 @@ int32_t setDbOptions(SCreateDbReq* pCreateDbMsg, const SCreateDbInfo* pCreateDbS return TSDB_CODE_SUCCESS; } -SCreateDbReq* buildCreateDbMsg(SCreateDbInfo* pCreateDbInfo, SParseBasicCtx *pCtx, SMsgBuf* pMsgBuf) { +SCreateDbReq* buildCreateDbMsg(SCreateDbInfo* pCreateDbInfo, SParseContext *pCtx, SMsgBuf* pMsgBuf) { SCreateDbReq* pCreateMsg = calloc(1, sizeof(SCreateDbReq)); if (setDbOptions(pCreateMsg, pCreateDbInfo, pMsgBuf) != TSDB_CODE_SUCCESS) { tfree(pCreateMsg); @@ -230,7 +230,7 @@ SCreateDbReq* buildCreateDbMsg(SCreateDbInfo* pCreateDbInfo, SParseBasicCtx *pCt return pCreateMsg; } -SMCreateStbReq* buildCreateStbMsg(SCreateTableSql* pCreateTableSql, int32_t* len, SParseBasicCtx* pParseCtx, SMsgBuf* pMsgBuf) { +SMCreateStbReq* buildCreateStbMsg(SCreateTableSql* pCreateTableSql, int32_t* len, SParseContext* pParseCtx, SMsgBuf* pMsgBuf) { SSchema* pSchema; int32_t numOfTags = 0; @@ -315,7 +315,7 @@ SMCreateStbReq* buildCreateStbMsg(SCreateTableSql* pCreateTableSql, int32_t* len return pCreateStbMsg; } -SMDropStbReq* buildDropStableMsg(SSqlInfo* pInfo, int32_t* len, SParseBasicCtx* pParseCtx, SMsgBuf* pMsgBuf) { +SMDropStbReq* buildDropStableMsg(SSqlInfo* pInfo, int32_t* len, SParseContext* pParseCtx, SMsgBuf* pMsgBuf) { SToken* tableName = taosArrayGet(pInfo->pMiscInfo->a, 0); SName name = {0}; diff --git a/source/libs/parser/src/astValidate.c b/source/libs/parser/src/astValidate.c index faa8c526a0..5d56a88c14 100644 --- a/source/libs/parser/src/astValidate.c +++ b/source/libs/parser/src/astValidate.c @@ -3628,7 +3628,7 @@ int32_t evaluateSqlNode(SSqlNode* pNode, int32_t tsPrecision, SMsgBuf* pMsgBuf) return TSDB_CODE_SUCCESS; } -int32_t setTableVgroupList(SParseBasicCtx *pCtx, SName* name, SVgroupsInfo **pVgList) { +int32_t setTableVgroupList(SParseContext *pCtx, SName* name, SVgroupsInfo **pVgList) { SArray* vgroupList = NULL; int32_t code = catalogGetTableDistVgroup(pCtx->pCatalog, pCtx->pTransporter, &pCtx->mgmtEpSet, name, &vgroupList); if (code != TSDB_CODE_SUCCESS) { @@ -3655,7 +3655,7 @@ int32_t setTableVgroupList(SParseBasicCtx *pCtx, SName* name, SVgroupsInfo **pVg return TSDB_CODE_SUCCESS; } -int32_t qParserValidateSqlNode(SParseBasicCtx *pCtx, SSqlInfo* pInfo, SQueryStmtInfo* pQueryInfo, char* msgBuf, int32_t msgBufLen) { +int32_t qParserValidateSqlNode(SParseContext *pCtx, SSqlInfo* pInfo, SQueryStmtInfo* pQueryInfo, char* msgBuf, int32_t msgBufLen) { assert(pCtx != NULL && pInfo != NULL); int32_t code = 0; diff --git a/source/libs/parser/src/dCDAstProcess.c b/source/libs/parser/src/dCDAstProcess.c index 6676e1ebf6..4d4d68e962 100644 --- a/source/libs/parser/src/dCDAstProcess.c +++ b/source/libs/parser/src/dCDAstProcess.c @@ -18,7 +18,7 @@ static bool has(SArray* pFieldList, int32_t startIndex, const char* name) { return false; } -static int32_t setShowInfo(SShowInfo* pShowInfo, SParseBasicCtx* pCtx, void** output, int32_t* outputLen, +static int32_t setShowInfo(SShowInfo* pShowInfo, SParseContext* pCtx, void** output, int32_t* outputLen, SEpSet* pEpSet, void** pExtension, SMsgBuf* pMsgBuf) { const char* msg1 = "invalid name"; const char* msg2 = "wildcard string should be less than %d characters"; @@ -396,7 +396,7 @@ static void destroyCreateTbReqBatch(SVgroupTablesBatch* pTbBatch) { taosArrayDestroy(pTbBatch->req.pArray); } -static int32_t doCheckAndBuildCreateCTableReq(SCreateTableSql* pCreateTable, SParseBasicCtx* pCtx, SMsgBuf* pMsgBuf, SArray** pBufArray) { +static int32_t doCheckAndBuildCreateCTableReq(SCreateTableSql* pCreateTable, SParseContext* pCtx, SMsgBuf* pMsgBuf, SArray** pBufArray) { const char* msg1 = "invalid table name"; const char* msg2 = "tags number not matched"; const char* msg3 = "tag value too long"; @@ -634,7 +634,7 @@ static int32_t doBuildSingleTableBatchReq(SName* pTableName, SArray* pColumns, S return TSDB_CODE_SUCCESS; } -int32_t doCheckAndBuildCreateTableReq(SCreateTableSql* pCreateTable, SParseBasicCtx* pCtx, SMsgBuf* pMsgBuf, char** pOutput, int32_t* len) { +int32_t doCheckAndBuildCreateTableReq(SCreateTableSql* pCreateTable, SParseContext* pCtx, SMsgBuf* pMsgBuf, char** pOutput, int32_t* len) { SArray* pBufArray = NULL; int32_t code = 0; @@ -703,7 +703,7 @@ SArray* doSerializeVgroupCreateTableInfo(SHashObj* pVgroupHashmap) { return pBufArray; } -SDclStmtInfo* qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, char* msgBuf, int32_t msgBufLen) { +SDclStmtInfo* qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseContext* pCtx, char* msgBuf, int32_t msgBufLen) { int32_t code = 0; SDclStmtInfo* pDcl = calloc(1, sizeof(SDclStmtInfo)); @@ -961,7 +961,7 @@ SDclStmtInfo* qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, c return NULL; } -SVnodeModifOpStmtInfo* qParserValidateCreateTbSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, char* msgBuf, int32_t msgBufLen) { +SVnodeModifOpStmtInfo* qParserValidateCreateTbSqlNode(SSqlInfo* pInfo, SParseContext* pCtx, char* msgBuf, int32_t msgBufLen) { SCreateTableSql* pCreateTable = pInfo->pCreateTableInfo; assert(pCreateTable->type == TSDB_SQL_CREATE_TABLE); diff --git a/source/libs/parser/src/insertParser.c b/source/libs/parser/src/insertParser.c index 04c287baf1..8b1506bad1 100644 --- a/source/libs/parser/src/insertParser.c +++ b/source/libs/parser/src/insertParser.c @@ -84,11 +84,11 @@ static int32_t buildName(SInsertParseContext* pCxt, SToken* pStname, char* fullD char* p = strnchr(pStname->z, TS_PATH_DELIMITER[0], pStname->n, false); if (NULL != p) { // db.table - int32_t n = sprintf(fullDbName, "%d.", pCxt->pComCxt->ctx.acctId); + int32_t n = sprintf(fullDbName, "%d.", pCxt->pComCxt->acctId); strncpy(fullDbName + n, pStname->z, p - pStname->z); strncpy(tableName, p + 1, pStname->n - (p - pStname->z) - 1); } else { - snprintf(fullDbName, TSDB_DB_FNAME_LEN, "%d.%s", pCxt->pComCxt->ctx.acctId, pCxt->pComCxt->ctx.db); + snprintf(fullDbName, TSDB_DB_FNAME_LEN, "%d.%s", pCxt->pComCxt->acctId, pCxt->pComCxt->db); strncpy(tableName, pStname->z, pStname->n); } @@ -97,11 +97,11 @@ static int32_t buildName(SInsertParseContext* pCxt, SToken* pStname, char* fullD static int32_t getTableMeta(SInsertParseContext* pCxt, SToken* pTname) { SName name = {0}; - createSName(&name, pTname, &pCxt->pComCxt->ctx, &pCxt->msg); + createSName(&name, pTname, pCxt->pComCxt, &pCxt->msg); char tableName[TSDB_TABLE_FNAME_LEN] = {0}; tNameExtractFullName(&name, tableName); - SParseBasicCtx* pBasicCtx = &pCxt->pComCxt->ctx; + SParseContext* pBasicCtx = pCxt->pComCxt; CHECK_CODE(catalogGetTableMeta(pBasicCtx->pCatalog, pBasicCtx->pTransporter, &pBasicCtx->mgmtEpSet, &name, &pCxt->pTableMeta)); SVgroupInfo vg; CHECK_CODE(catalogGetTableHashVgroup(pBasicCtx->pCatalog, pBasicCtx->pTransporter, &pBasicCtx->mgmtEpSet, &name, &vg)); diff --git a/source/libs/parser/src/parser.c b/source/libs/parser/src/parser.c index 5f17fcfaee..fea215238c 100644 --- a/source/libs/parser/src/parser.c +++ b/source/libs/parser/src/parser.c @@ -45,14 +45,14 @@ int32_t parseQuerySql(SParseContext* pCxt, SQueryNode** pQuery) { if (!isDqlSqlStatement(&info)) { if (info.type == TSDB_SQL_CREATE_TABLE) { - SVnodeModifOpStmtInfo * pModifStmtInfo = qParserValidateCreateTbSqlNode(&info, &pCxt->ctx, pCxt->pMsg, pCxt->msgLen); + SVnodeModifOpStmtInfo * pModifStmtInfo = qParserValidateCreateTbSqlNode(&info, pCxt, pCxt->pMsg, pCxt->msgLen); if (pModifStmtInfo == NULL) { return terrno; } *pQuery = (SQueryNode*)pModifStmtInfo; } else { - SDclStmtInfo* pDcl = qParserValidateDclSqlNode(&info, &pCxt->ctx, pCxt->pMsg, pCxt->msgLen); + SDclStmtInfo* pDcl = qParserValidateDclSqlNode(&info, pCxt, pCxt->pMsg, pCxt->msgLen); if (pDcl == NULL) { return terrno; } @@ -67,7 +67,7 @@ int32_t parseQuerySql(SParseContext* pCxt, SQueryNode** pQuery) { return terrno; } - int32_t code = qParserValidateSqlNode(&pCxt->ctx, &info, pQueryInfo, pCxt->pMsg, pCxt->msgLen); + int32_t code = qParserValidateSqlNode(pCxt, &info, pQueryInfo, pCxt->pMsg, pCxt->msgLen); if (code == TSDB_CODE_SUCCESS) { *pQuery = (SQueryNode*)pQueryInfo; } else { @@ -92,7 +92,7 @@ int32_t qParserConvertSql(const char* pStr, size_t length, char** pConvertSql) { return 0; } -static int32_t getTableNameFromSqlNode(SSqlNode* pSqlNode, SArray* tableNameList, SParseBasicCtx *pCtx, SMsgBuf* pMsgBuf); +static int32_t getTableNameFromSqlNode(SSqlNode* pSqlNode, SArray* tableNameList, SParseContext *pCtx, SMsgBuf* pMsgBuf); static int32_t tnameComparFn(const void* p1, const void* p2) { SName* pn1 = (SName*)p1; @@ -116,7 +116,7 @@ static int32_t tnameComparFn(const void* p1, const void* p2) { } } -static int32_t getTableNameFromSubquery(SSqlNode* pSqlNode, SArray* tableNameList, SParseBasicCtx *pCtx, SMsgBuf* pMsgBuf) { +static int32_t getTableNameFromSubquery(SSqlNode* pSqlNode, SArray* tableNameList, SParseContext *pCtx, SMsgBuf* pMsgBuf) { int32_t numOfSub = (int32_t)taosArrayGetSize(pSqlNode->from->list); for (int32_t j = 0; j < numOfSub; ++j) { @@ -139,7 +139,7 @@ static int32_t getTableNameFromSubquery(SSqlNode* pSqlNode, SArray* tableNameLis return TSDB_CODE_SUCCESS; } -int32_t getTableNameFromSqlNode(SSqlNode* pSqlNode, SArray* tableNameList, SParseBasicCtx *pParseCtx, SMsgBuf* pMsgBuf) { +int32_t getTableNameFromSqlNode(SSqlNode* pSqlNode, SArray* tableNameList, SParseContext *pParseCtx, SMsgBuf* pMsgBuf) { const char* msg1 = "invalid table name"; int32_t numOfTables = (int32_t) taosArrayGetSize(pSqlNode->from->list); @@ -173,7 +173,7 @@ static void freePtrElem(void* p) { tfree(*(char**)p); } -int32_t qParserExtractRequestedMetaInfo(const SSqlInfo* pSqlInfo, SCatalogReq* pMetaInfo, SParseBasicCtx *pCtx, char* msg, int32_t msgBufLen) { +int32_t qParserExtractRequestedMetaInfo(const SSqlInfo* pSqlInfo, SCatalogReq* pMetaInfo, SParseContext *pCtx, char* msg, int32_t msgBufLen) { int32_t code = TSDB_CODE_SUCCESS; SMsgBuf msgBuf = {.buf = msg, .len = msgBufLen}; diff --git a/source/libs/parser/src/parserUtil.c b/source/libs/parser/src/parserUtil.c index 22545255a3..7bb0f8ee83 100644 --- a/source/libs/parser/src/parserUtil.c +++ b/source/libs/parser/src/parserUtil.c @@ -1943,7 +1943,7 @@ int32_t KvRowAppend(const void *value, int32_t len, void *param) { return TSDB_CODE_SUCCESS; } -int32_t createSName(SName* pName, SToken* pTableName, SParseBasicCtx* pParseCtx, SMsgBuf* pMsgBuf) { +int32_t createSName(SName* pName, SToken* pTableName, SParseContext* pParseCtx, SMsgBuf* pMsgBuf) { const char* msg1 = "name too long"; int32_t code = TSDB_CODE_SUCCESS; diff --git a/source/libs/parser/src/queryInfoUtil.c b/source/libs/parser/src/queryInfoUtil.c index d7aa758576..1aa8836cb2 100644 --- a/source/libs/parser/src/queryInfoUtil.c +++ b/source/libs/parser/src/queryInfoUtil.c @@ -160,11 +160,14 @@ void addExprInfo(SArray* pExprList, int32_t index, SExprInfo* pExprInfo, int32_t taosArrayInsert(pExprList, index, &pExprInfo); } +#if 0 if (pExprInfo->pExpr->nodeType == TEXPR_FUNCTION_NODE) { printf("add function: %s, level:%d, total:%ld\n", pExprInfo->pExpr->_function.functionName, level, taosArrayGetSize(pExprList)); } else { printf("add operator: %s, level:%d, total:%ld\n", pExprInfo->base.resSchema.name, level, taosArrayGetSize(pExprList)); } +#endif + } void updateExprInfo(SExprInfo* pExprInfo, int16_t functionId, int32_t colId, int16_t srcColumnIndex, int16_t resType, int16_t resSize) { diff --git a/source/libs/parser/test/insertParserTest.cpp b/source/libs/parser/test/insertParserTest.cpp index 9b007fb36d..86e8b1d7aa 100644 --- a/source/libs/parser/test/insertParserTest.cpp +++ b/source/libs/parser/test/insertParserTest.cpp @@ -43,8 +43,8 @@ protected: void bind(const char* sql) { reset(); - cxt_.ctx.acctId = atoi(acctId_.c_str()); - cxt_.ctx.db = (char*) db_.c_str(); + cxt_.acctId = atoi(acctId_.c_str()); + cxt_.db = (char*) db_.c_str(); strcpy(sqlBuf_, sql); cxt_.sqlLen = strlen(sql); sqlBuf_[cxt_.sqlLen] = '\0'; diff --git a/source/libs/parser/test/parserTests.cpp b/source/libs/parser/test/parserTests.cpp index 8758fdbc71..4847b50082 100644 --- a/source/libs/parser/test/parserTests.cpp +++ b/source/libs/parser/test/parserTests.cpp @@ -77,7 +77,7 @@ void sqlCheck(const char* sql, bool valid) { buf.len = 128; buf.buf = msg; - SParseBasicCtx ctx = {0}; + SParseContext ctx = {0}; ctx.db = "db1"; ctx.acctId = 1; SSqlNode* pNode = (SSqlNode*)taosArrayGetP(((SArray*)info1.sub.node), 0); @@ -122,7 +122,7 @@ TEST(testCase, validateAST_test) { ASSERT_EQ(code, 0); SCatalogReq req = {0}; - SParseBasicCtx ctx = {0}; + SParseContext ctx = {0}; ctx.db = "db1"; ctx.acctId = 1; int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, &ctx, msg, 128); @@ -184,7 +184,7 @@ TEST(testCase, function_Test) { ASSERT_EQ(code, 0); SCatalogReq req = {0}; - SParseBasicCtx ctx = {0}; + SParseContext ctx = {0}; ctx.db = "db1"; ctx.acctId = 1; int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, &ctx, msg, 128); @@ -234,7 +234,7 @@ TEST(testCase, function_Test2) { ASSERT_EQ(code, 0); SCatalogReq req = {0}; - SParseBasicCtx ctx = {0}; + SParseContext ctx = {0}; ctx.db = "db1"; ctx.acctId = 1; int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, &ctx, msg, 128); @@ -284,7 +284,7 @@ TEST(testCase, function_Test3) { ASSERT_EQ(code, 0); SCatalogReq req = {0}; - SParseBasicCtx ctx = {0}; + SParseContext ctx = {0}; ctx.db = "db1"; ctx.acctId = 1; int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, &ctx, msg, 128); @@ -333,7 +333,7 @@ TEST(testCase, function_Test4) { ASSERT_EQ(code, 0); SCatalogReq req = {0}; - SParseBasicCtx ctx = {0}; + SParseContext ctx = {0}; ctx.db = "db1"; ctx.acctId = 1; int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, &ctx, msg, 128); @@ -385,7 +385,7 @@ TEST(testCase, function_Test5) { ASSERT_EQ(code, 0); SCatalogReq req = {0}; - SParseBasicCtx ctx = {0}; + SParseContext ctx = {0}; ctx.db = "db1"; ctx.acctId = 1; int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, &ctx, msg, 128); @@ -474,7 +474,7 @@ TEST(testCase, function_Test6) { ASSERT_EQ(code, 0); SCatalogReq req = {0}; - SParseBasicCtx ctx = {0}; + SParseContext ctx = {0}; ctx.db = "db1"; ctx.acctId = 1; int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, &ctx, msg, 128); @@ -556,7 +556,7 @@ TEST(testCase, function_Test6) { ASSERT_EQ(code, 0); SCatalogReq req = {0}; - SParseBasicCtx ctx = {0}; + SParseContext ctx = {0}; ctx.db = "db1"; ctx.acctId = 1; int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, &ctx, msg, 128); @@ -622,7 +622,7 @@ TEST(testCase, function_Test6) { ASSERT_EQ(code, 0); SCatalogReq req = {0}; - SParseBasicCtx ctx = {0}; + SParseContext ctx = {0}; ctx.db = "db1"; ctx.acctId = 1; int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, &ctx, msg, 128); @@ -705,7 +705,7 @@ TEST(testCase, function_Test6) { ASSERT_EQ(code, 0); SCatalogReq req = {0}; - SParseBasicCtx ctx = {0}; + SParseContext ctx = {0}; ctx.db = "db1"; ctx.acctId = 1; int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, &ctx, msg, 128); @@ -756,7 +756,7 @@ TEST(testCase, show_user_Test) { SSqlInfo info1 = doGenerateAST(sql1); ASSERT_EQ(info1.valid, true); - SParseBasicCtx ct= {.requestId = 1, .acctId = 1, .db = "abc", .pTransporter = NULL}; + SParseContext ct= {.requestId = 1, .acctId = 1, .db = "abc", .pTransporter = NULL}; SDclStmtInfo* output = qParserValidateDclSqlNode(&info1, &ct, msg, buf.len); ASSERT_NE(output, nullptr); @@ -776,7 +776,7 @@ TEST(testCase, create_user_Test) { ASSERT_EQ(info1.valid, true); ASSERT_EQ(isDclSqlStatement(&info1), true); - SParseBasicCtx ct= {.requestId = 1, .acctId = 1, .db = "abc"}; + SParseContext ct= {.requestId = 1, .acctId = 1, .db = "abc"}; SDclStmtInfo* output = qParserValidateDclSqlNode(&info1, &ct, msg, buf.len); ASSERT_NE(output, nullptr); diff --git a/source/libs/parser/test/plannerTest.cpp b/source/libs/parser/test/plannerTest.cpp index 8d9fbadfad..a278e0053c 100644 --- a/source/libs/parser/test/plannerTest.cpp +++ b/source/libs/parser/test/plannerTest.cpp @@ -81,7 +81,7 @@ void generateLogicplan(const char* sql) { ASSERT_EQ(code, 0); SCatalogReq req = {0}; - SParseBasicCtx ctx = {0}; + SParseContext ctx = {0}; int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, &ctx, msg, 128); ASSERT_EQ(ret, 0); ASSERT_EQ(taosArrayGetSize(req.pTableName), 1); @@ -122,7 +122,7 @@ TEST(testCase, planner_test) { ASSERT_EQ(code, 0); SCatalogReq req = {0}; - SParseBasicCtx ctx = {0}; + SParseContext ctx = {0}; int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, &ctx, msg, 128); ASSERT_EQ(ret, 0); diff --git a/source/libs/parser/test/tokenizerTest.cpp b/source/libs/parser/test/tokenizerTest.cpp index ee01a50148..7ed052e8cc 100644 --- a/source/libs/parser/test/tokenizerTest.cpp +++ b/source/libs/parser/test/tokenizerTest.cpp @@ -711,7 +711,7 @@ TEST(testCase, extractMeta_test) { char msg[128] = {0}; SCatalogReq req = {0}; - SParseBasicCtx ctx = {0}; + SParseContext ctx = {0}; ctx.db = "db1"; ctx.acctId = 1; int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, &ctx, msg, 128); diff --git a/source/libs/planner/src/logicPlan.c b/source/libs/planner/src/logicPlan.c index 9a9b40473b..93f72bba95 100644 --- a/source/libs/planner/src/logicPlan.c +++ b/source/libs/planner/src/logicPlan.c @@ -38,10 +38,10 @@ int32_t optimizeQueryPlan(struct SQueryPlanNode* pQueryNode) { } static int32_t createModificationOpPlan(const SQueryNode* pNode, SQueryPlanNode** pQueryPlan) { - SVnodeModifOpStmtInfo* pInsert = (SVnodeModifOpStmtInfo*)pNode; + SVnodeModifOpStmtInfo* pModifStmtInfo = (SVnodeModifOpStmtInfo*)pNode; *pQueryPlan = calloc(1, sizeof(SQueryPlanNode)); - SArray* blocks = taosArrayInit(taosArrayGetSize(pInsert->pDataBlocks), POINTER_BYTES); + SArray* blocks = taosArrayInit(taosArrayGetSize(pModifStmtInfo->pDataBlocks), POINTER_BYTES); SDataPayloadInfo* pPayload = calloc(1, sizeof(SDataPayloadInfo)); if (NULL == *pQueryPlan || NULL == blocks || NULL == pPayload) { @@ -49,7 +49,7 @@ static int32_t createModificationOpPlan(const SQueryNode* pNode, SQueryPlanNode* } (*pQueryPlan)->info.type = QNODE_MODIFY; - taosArrayAddAll(blocks, pInsert->pDataBlocks); + taosArrayAddAll(blocks, pModifStmtInfo->pDataBlocks); if (pNode->type == TSDB_SQL_INSERT) { pPayload->msgType = TDMT_VND_SUBMIT; diff --git a/source/libs/planner/src/physicalPlan.c b/source/libs/planner/src/physicalPlan.c index 01fb3c2513..71e69c67d2 100644 --- a/source/libs/planner/src/physicalPlan.c +++ b/source/libs/planner/src/physicalPlan.c @@ -156,9 +156,14 @@ static SPhyNode* initPhyNode(SQueryPlanNode* pPlanNode, int32_t type, int32_t si } static SPhyNode* initScanNode(SQueryPlanNode* pPlanNode, SQueryTableInfo* pTable, int32_t type, int32_t size) { - SScanPhyNode* node = (SScanPhyNode*)initPhyNode(pPlanNode, type, size); - node->uid = pTable->pMeta->pTableMeta->uid; - node->tableType = pTable->pMeta->pTableMeta->tableType; + SScanPhyNode* node = (SScanPhyNode*) initPhyNode(pPlanNode, type, size); + + STableMeta *pTableMeta = pTable->pMeta->pTableMeta; + node->uid = pTableMeta->uid; + node->count = 1; + node->order = TSDB_ORDER_ASC; + node->tableType = pTableMeta->tableType; + return (SPhyNode*)node; } @@ -176,10 +181,10 @@ static uint8_t getScanFlag(SQueryPlanNode* pPlanNode, SQueryTableInfo* pTable) { return MAIN_SCAN; } -static SPhyNode* createUserTableScanNode(SQueryPlanNode* pPlanNode, SQueryTableInfo* pTable, int32_t op) { - STableScanPhyNode* node = (STableScanPhyNode*)initScanNode(pPlanNode, pTable, op, sizeof(STableScanPhyNode)); - node->scanFlag = getScanFlag(pPlanNode, pTable); - node->window = pTable->window; +static SPhyNode* createUserTableScanNode(SQueryPlanNode* pPlanNode, SQueryTableInfo* pQueryTableInfo, int32_t op) { + STableScanPhyNode* node = (STableScanPhyNode*)initScanNode(pPlanNode, pQueryTableInfo, op, sizeof(STableScanPhyNode)); + node->scanFlag = getScanFlag(pPlanNode, pQueryTableInfo); + node->window = pQueryTableInfo->window; // todo tag cond return (SPhyNode*)node; } @@ -364,9 +369,9 @@ int32_t createDag(SQueryPlanNode* pQueryNode, struct SCatalog* pCatalog, SQueryD TRY(TSDB_MAX_TAG_CONDITIONS) { SPlanContext context = { .pCatalog = pCatalog, - .pDag = validPointer(calloc(1, sizeof(SQueryDag))), + .pDag = validPointer(calloc(1, sizeof(SQueryDag))), .pCurrentSubplan = NULL, - .nextId = {.queryId = requestId}, + .nextId = {.queryId = requestId}, }; *pDag = context.pDag; diff --git a/source/libs/planner/src/physicalPlanJson.c b/source/libs/planner/src/physicalPlanJson.c index 895788c550..7e36e0e124 100644 --- a/source/libs/planner/src/physicalPlanJson.c +++ b/source/libs/planner/src/physicalPlanJson.c @@ -136,7 +136,7 @@ static const cJSON* getArray(const cJSON* json, const char* name, int32_t* size) static bool fromItem(const cJSON* jArray, FFromJson func, void* array, int32_t itemSize, int32_t size) { for (int32_t i = 0; i < size; ++i) { - if (!func(cJSON_GetArrayItem(jArray, i), (char*)array + itemSize)) { + if (!func(cJSON_GetArrayItem(jArray, i), (char*)array + itemSize * i)) { return false; } } @@ -165,9 +165,7 @@ static bool fromRawArray(const cJSON* json, const char* name, FFromJson func, vo static char* getString(const cJSON* json, const char* name) { char* p = cJSON_GetStringValue(cJSON_GetObjectItem(json, name)); - char* res = calloc(1, strlen(p) + 1); - strcpy(res, p); - return res; + return strdup(p); } static void copyString(const cJSON* json, const char* name, char* dst) { @@ -285,7 +283,7 @@ static bool columnInfoToJson(const void* obj, cJSON* jCol) { static bool columnInfoFromJson(const cJSON* json, void* obj) { SColumnInfo* col = (SColumnInfo*)obj; col->colId = getNumber(json, jkColumnInfoColId); - col->type = getNumber(json, jkColumnInfoType); + col->type = getNumber(json, jkColumnInfoType); col->bytes = getNumber(json, jkColumnInfoBytes); int32_t size = 0; bool res = fromRawArrayWithAlloc(json, jkColumnInfoFilterList, columnFilterInfoFromJson, (void**)&col->flist.filterInfo, sizeof(SColumnFilterInfo), &size); @@ -530,13 +528,25 @@ static bool timeWindowFromJson(const cJSON* json, void* obj) { static const char* jkScanNodeTableId = "TableId"; static const char* jkScanNodeTableType = "TableType"; +static const char* jkScanNodeTableOrder = "Order"; +static const char* jkScanNodeTableCount = "Count"; static bool scanNodeToJson(const void* obj, cJSON* json) { const SScanPhyNode* scan = (const SScanPhyNode*)obj; bool res = cJSON_AddNumberToObject(json, jkScanNodeTableId, scan->uid); + if (res) { res = cJSON_AddNumberToObject(json, jkScanNodeTableType, scan->tableType); } + + if (res) { + res = cJSON_AddNumberToObject(json, jkScanNodeTableOrder, scan->order); + } + + if (res) { + res = cJSON_AddNumberToObject(json, jkScanNodeTableCount, scan->count); + } + return res; } @@ -544,6 +554,8 @@ static bool scanNodeFromJson(const cJSON* json, void* obj) { SScanPhyNode* scan = (SScanPhyNode*)obj; scan->uid = getNumber(json, jkScanNodeTableId); scan->tableType = getNumber(json, jkScanNodeTableType); + scan->count = getNumber(json, jkScanNodeTableCount); + scan->order = getNumber(json, jkScanNodeTableOrder); return true; } @@ -748,9 +760,11 @@ static bool phyNodeToJson(const void* obj, cJSON* jNode) { } static bool phyNodeFromJson(const cJSON* json, void* obj) { - SPhyNode* node = (SPhyNode*)obj; + SPhyNode* node = (SPhyNode*) obj; + node->info.name = getString(json, jkPnodeName); node->info.type = opNameToOpType(node->info.name); + bool res = fromArray(json, jkPnodeTargets, exprInfoFromJson, &node->pTargets, sizeof(SExprInfo)); if (res) { res = fromArray(json, jkPnodeConditions, exprInfoFromJson, &node->pConditions, sizeof(SExprInfo)); @@ -896,7 +910,8 @@ static SSubplan* subplanFromJson(const cJSON* json) { } bool res = fromObject(json, jkSubplanId, subplanIdFromJson, &subplan->id, true); if (res) { - res = fromObjectWithAlloc(json, jkSubplanNode, phyNodeFromJson, (void**)&subplan->pNode, sizeof(SPhyNode), false); + size_t size = MAX(sizeof(SPhyNode), sizeof(SScanPhyNode)); + res = fromObjectWithAlloc(json, jkSubplanNode, phyNodeFromJson, (void**)&subplan->pNode, size, false); } if (res) { res = fromObjectWithAlloc(json, jkSubplanDataSink, dataSinkFromJson, (void**)&subplan->pDataSink, sizeof(SDataSink), false); @@ -925,8 +940,8 @@ int32_t subPlanToString(const SSubplan* subplan, char** str, int32_t* len) { } *str = cJSON_Print(json); - - printf("%s\n", *str); +// printf("====Physical plan:====\n") +// printf("%s\n", *str); *len = strlen(*str) + 1; return TSDB_CODE_SUCCESS; } diff --git a/source/libs/planner/src/planner.c b/source/libs/planner/src/planner.c index 3047ef4f5a..21f57d95d5 100644 --- a/source/libs/planner/src/planner.c +++ b/source/libs/planner/src/planner.c @@ -65,9 +65,9 @@ int32_t qCreateQueryDag(const struct SQueryNode* pNode, struct SQueryDag** pDag, } if (pLogicPlan->info.type != QNODE_MODIFY) { - char* str = NULL; - queryPlanToString(pLogicPlan, &str); - printf("%s\n", str); +// char* str = NULL; +// queryPlanToString(pLogicPlan, &str); +// printf("%s\n", str); } code = optimizeQueryPlan(pLogicPlan); diff --git a/source/libs/planner/test/phyPlanTests.cpp b/source/libs/planner/test/phyPlanTests.cpp index 2733a73a3f..29f6e48dc7 100644 --- a/source/libs/planner/test/phyPlanTests.cpp +++ b/source/libs/planner/test/phyPlanTests.cpp @@ -135,9 +135,9 @@ private: _sql = sql; memset(_msg, 0, _msgMaxLen); - pCxt->ctx.acctId = 1; - pCxt->ctx.db = _db.c_str(); - pCxt->ctx.requestId = 1; + pCxt->acctId = 1; + pCxt->db = _db.c_str(); + pCxt->requestId = 1; pCxt->pSql = _sql.c_str(); pCxt->sqlLen = _sql.length(); pCxt->pMsg = _msg; diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index aeec1ff5a0..095f179925 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -50,6 +50,10 @@ void schFreeTask(SSchTask* pTask) { if (pTask->parents) { taosArrayDestroy(pTask->parents); } + + if (pTask->execAddrs) { + taosArrayDestroy(pTask->execAddrs); + } } @@ -798,7 +802,7 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, ch SQueryTableRsp *rsp = (SQueryTableRsp *)msg; if (rspCode != TSDB_CODE_SUCCESS || NULL == msg || rsp->code != TSDB_CODE_SUCCESS) { - SCH_ERR_RET(schProcessOnTaskFailure(pJob, pTask, rsp->code)); + SCH_ERR_RET(schProcessOnTaskFailure(pJob, pTask, rspCode)); } SCH_ERR_JRET(schBuildAndSendMsg(pJob, pTask, NULL, TDMT_VND_RES_READY)); @@ -1364,16 +1368,10 @@ int32_t scheduleAsyncExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } - SSchJob *job = NULL; - - SCH_ERR_RET(schExecJobImpl(transport, nodeList, pDag, &job, false)); - - *pJob = job; - + SCH_ERR_RET(schExecJobImpl(transport, nodeList, pDag, pJob, false)); return TSDB_CODE_SUCCESS; } - int32_t scheduleFetchRows(SSchJob *pJob, void** pData) { if (NULL == pJob || NULL == pData) { SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); diff --git a/source/libs/scheduler/test/schedulerTests.cpp b/source/libs/scheduler/test/schedulerTests.cpp index 114b3c02b5..d72c4920d5 100644 --- a/source/libs/scheduler/test/schedulerTests.cpp +++ b/source/libs/scheduler/test/schedulerTests.cpp @@ -162,7 +162,7 @@ int32_t schtPlanToString(const SSubplan *subplan, char** str, int32_t* len) { } void schtExecNode(SSubplan* subplan, uint64_t templateId, SQueryNodeAddr* ep) { - + } void schtRpcSendRequest(void *shandle, const SEpSet *pEpSet, SRpcMsg *pMsg, int64_t *pRid) { @@ -239,8 +239,6 @@ void *schtSendRsp(void *param) { } struct SSchJob *pInsertJob = NULL; - - } TEST(queryTest, normalCase) { @@ -357,7 +355,7 @@ TEST(insertTest, normalCase) { uint64_t numOfRows = 0; schtInitLogFile(); - + SArray *qnodeList = taosArrayInit(1, sizeof(SEpAddr)); SEpAddr qnodeAddr = {0}; diff --git a/source/libs/tfs/src/tfs.c b/source/libs/tfs/src/tfs.c index 83d9905511..88d6d587a7 100644 --- a/source/libs/tfs/src/tfs.c +++ b/source/libs/tfs/src/tfs.c @@ -85,19 +85,19 @@ int tfsInit(SDiskCfg *pDiskCfg, int ndisk) { taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); if (pfs->map == NULL) { terrno = TSDB_CODE_FS_OUT_OF_MEMORY; - tfsDestroy(); + tfsCleanup(); return -1; } for (int idisk = 0; idisk < ndisk; idisk++) { if (tfsMount(pDiskCfg + idisk) < 0) { - tfsDestroy(); + tfsCleanup(); return -1; } } if (tfsCheck() < 0) { - tfsDestroy(); + tfsCleanup(); return -1; } @@ -109,7 +109,7 @@ int tfsInit(SDiskCfg *pDiskCfg, int ndisk) { return 0; } -void tfsDestroy() { +void tfsCleanup() { taosHashCleanup(pfs->map); pfs->map = NULL; diff --git a/source/libs/tkv/CMakeLists.txt b/source/libs/tkv/CMakeLists.txt index 0620e12f55..fec3f37cd5 100644 --- a/source/libs/tkv/CMakeLists.txt +++ b/source/libs/tkv/CMakeLists.txt @@ -1,11 +1,17 @@ aux_source_directory(src TKV_SRC) add_library(tkv STATIC ${TKV_SRC}) +# target_include_directories( +# tkv +# PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/tkv" +# PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" +# ) target_include_directories( tkv - PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/tkv" - PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" + PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/inc" + PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/src/inc" ) target_link_libraries( tkv PUBLIC os + PUBLIC util ) \ No newline at end of file diff --git a/source/dnode/vnode/tsdb/src/tsdbRowMergeBuf.c b/source/libs/tkv/inc/tkv.h similarity index 55% rename from source/dnode/vnode/tsdb/src/tsdbRowMergeBuf.c rename to source/libs/tkv/inc/tkv.h index 5ce580f70f..00534d2827 100644 --- a/source/dnode/vnode/tsdb/src/tsdbRowMergeBuf.c +++ b/source/libs/tkv/inc/tkv.h @@ -13,18 +13,27 @@ * along with this program. If not, see . */ -#include "tsdbRowMergeBuf.h" -#include "tdataformat.h" +#ifndef _TD_TKV_H_ +#define _TD_TKV_H_ -// row1 has higher priority -SMemRow tsdbMergeTwoRows(SMergeBuf *pBuf, SMemRow row1, SMemRow row2, STSchema *pSchema1, STSchema *pSchema2) { - if(row2 == NULL) return row1; - if(row1 == NULL) return row2; - ASSERT(pSchema1->version == memRowVersion(row1)); - ASSERT(pSchema2->version == memRowVersion(row2)); +#include "os.h" - if(tsdbMergeBufMakeSureRoom(pBuf, pSchema1, pSchema2) < 0) { - return NULL; - } - return mergeTwoMemRows(*pBuf, row1, row2, pSchema1, pSchema2); +#ifdef __cplusplus +extern "C" { +#endif + +// Forward declaration +typedef struct TDB TDB; +typedef struct TDB_ENV TDB_ENV; + +// SKey +typedef struct { + void * bdata; + uint32_t size; +} TDB_KEY, TDB_VALUE; + +#ifdef __cplusplus } +#endif + +#endif /*_TD_TKV_H_*/ \ No newline at end of file diff --git a/source/libs/tkv/src/inc/tkvBufPool.h b/source/libs/tkv/src/inc/tkvBufPool.h new file mode 100644 index 0000000000..ec8d177a9a --- /dev/null +++ b/source/libs/tkv/src/inc/tkvBufPool.h @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_TKV_BUF_POOL_H_ +#define _TD_TKV_BUF_POOL_H_ + +#include "tkvPage.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct STkvBufPool STkvBufPool; + +int tbpOpen(STkvBufPool **ppTkvBufPool); +int tbpClose(STkvBufPool *pTkvBufPool); +STkvPage *tbpNewPage(STkvBufPool *pTkvBufPool); +int tbpDelPage(STkvBufPool *pTkvBufPool); +STkvPage *tbpFetchPage(STkvBufPool *pTkvBufPool, pgid_t pgid); +int tbpUnpinPage(STkvBufPool *pTkvBufPool, pgid_t pgid); +void tbpFlushPages(STkvBufPool *pTkvBufPool); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_TKV_BUF_POOL_H_*/ \ No newline at end of file diff --git a/source/libs/tkv/src/inc/tkvDB.h b/source/libs/tkv/src/inc/tkvDB.h new file mode 100644 index 0000000000..1a45702540 --- /dev/null +++ b/source/libs/tkv/src/inc/tkvDB.h @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_TKV_DB_H_ +#define _TD_TKV_DB_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +struct TDB { + // TODO +}; + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_TKV_DB_H_*/ \ No newline at end of file diff --git a/source/libs/tkv/inc/tkvDef.h b/source/libs/tkv/src/inc/tkvDef.h similarity index 78% rename from source/libs/tkv/inc/tkvDef.h rename to source/libs/tkv/src/inc/tkvDef.h index 6f1072abd5..cd418019be 100644 --- a/source/libs/tkv/inc/tkvDef.h +++ b/source/libs/tkv/src/inc/tkvDef.h @@ -26,6 +26,15 @@ extern "C" { typedef int32_t pgid_t; #define TKV_IVLD_PGID ((pgid_t)-1) +// framd_id_t +typedef int32_t frame_id_t; + +// pgsize_t +typedef int32_t pgsize_t; +#define TKV_MIN_PGSIZE 512 +#define TKV_MAX_PGSIZE 16384 +#define TKV_IS_PGSIZE_VLD(s) (((s) >= TKV_MIN_PGSIZE) && (TKV_MAX_PGSIZE <= TKV_MAX_PGSIZE)) + #ifdef __cplusplus } #endif diff --git a/source/libs/tkv/inc/tDiskMgr.h b/source/libs/tkv/src/inc/tkvDiskMgr.h similarity index 67% rename from source/libs/tkv/inc/tDiskMgr.h rename to source/libs/tkv/src/inc/tkvDiskMgr.h index 03622284f4..2ebe98ace2 100644 --- a/source/libs/tkv/inc/tDiskMgr.h +++ b/source/libs/tkv/src/inc/tkvDiskMgr.h @@ -24,13 +24,14 @@ extern "C" { #include "tkvDef.h" -typedef struct SDiskMgr SDiskMgr; +typedef struct STkvDiskMgr STkvDiskMgr; -int tdmOpen(SDiskMgr **ppDiskMgr, const char *fname, uint16_t pgsize); -int tdmClose(SDiskMgr *pDiskMgr); -int tdmReadPage(SDiskMgr *pDiskMgr, pgid_t pgid, void *pData); -int tdmWritePage(SDiskMgr *pDiskMgr, pgid_t pgid, const void *pData); -int32_t tdmAllocPage(SDiskMgr *pDiskMgr); +int tdmOpen(STkvDiskMgr **ppDiskMgr, const char *fname, uint16_t pgsize); +int tdmClose(STkvDiskMgr *pDiskMgr); +int tdmReadPage(STkvDiskMgr *pDiskMgr, pgid_t pgid, void *pData); +int tdmWritePage(STkvDiskMgr *pDiskMgr, pgid_t pgid, const void *pData); +int tdmFlush(STkvDiskMgr *pDiskMgr); +pgid_t tdmAllocPage(STkvDiskMgr *pDiskMgr); #ifdef __cplusplus } diff --git a/source/libs/tkv/src/inc/tkvEnv.h b/source/libs/tkv/src/inc/tkvEnv.h new file mode 100644 index 0000000000..eba442e5a5 --- /dev/null +++ b/source/libs/tkv/src/inc/tkvEnv.h @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_TKV_ENV_H_ +#define _TD_TKV_ENV_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +struct TDB_ENV { + char *homeDir; +}; + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_TKV_ENV_H_*/ \ No newline at end of file diff --git a/source/libs/tkv/inc/tPage.h b/source/libs/tkv/src/inc/tkvPage.h similarity index 71% rename from source/libs/tkv/inc/tPage.h rename to source/libs/tkv/src/inc/tkvPage.h index 0546f6184f..d596d215cd 100644 --- a/source/libs/tkv/inc/tPage.h +++ b/source/libs/tkv/src/inc/tkvPage.h @@ -17,22 +17,30 @@ #define _TD_TKV_PAGE_H_ #include "os.h" +#include "tkvDef.h" #ifdef __cplusplus extern "C" { #endif +typedef struct STkvPage { + pgid_t pgid; + int32_t pinCount; + bool idDirty; + char* pData; +} STkvPage; + typedef struct { uint16_t dbver; uint16_t pgsize; uint32_t cksm; -} SPgHdr; +} STkvPgHdr; -typedef struct { - SPgHdr chdr; - uint16_t used; // number of used slots - uint16_t loffset; // the offset of the starting location of the last slot used -} SSlottedPgHdr; +// typedef struct { +// SPgHdr chdr; +// uint16_t used; // number of used slots +// uint16_t loffset; // the offset of the starting location of the last slot used +// } SSlottedPgHdr; #ifdef __cplusplus } diff --git a/source/libs/tkv/src/tDiskMgr.c b/source/libs/tkv/src/tDiskMgr.c index d759171f85..fa8f6062d8 100644 --- a/source/libs/tkv/src/tDiskMgr.c +++ b/source/libs/tkv/src/tDiskMgr.c @@ -13,37 +13,62 @@ * along with this program. If not, see . */ -#include "tDiskMgr.h" +#include "tkvDiskMgr.h" -struct SDiskMgr { - const char *fname; - uint16_t pgsize; - FileFd fd; - int32_t npgid; +struct STkvDiskMgr { + char * fname; + uint16_t pgsize; + FileFd fd; + pgid_t npgid; }; #define PAGE_OFFSET(PGID, PGSIZE) ((PGID) * (PGSIZE)) -int tdmOpen(SDiskMgr **ppDiskMgr, const char *fname, uint16_t pgsize) { - // TODO +int tdmOpen(STkvDiskMgr **ppDiskMgr, const char *fname, uint16_t pgsize) { + STkvDiskMgr *pDiskMgr; + + pDiskMgr = malloc(sizeof(*pDiskMgr)); + if (pDiskMgr == NULL) { + return -1; + } + + pDiskMgr->fname = strdup(fname); + if (pDiskMgr->fname == NULL) { + free(pDiskMgr); + return -1; + } + pDiskMgr->pgsize = pgsize; + pDiskMgr->fd = open(fname, O_CREAT | O_RDWR, 0755); + if (pDiskMgr->fd < 0) { + free(pDiskMgr->fname); + free(pDiskMgr); + return -1; + } + + *ppDiskMgr = pDiskMgr; + return 0; } -int tdmClose(SDiskMgr *pDiskMgr) { - // TODO +int tdmClose(STkvDiskMgr *pDiskMgr) { + close(pDiskMgr->fd); + free(pDiskMgr->fname); + free(pDiskMgr); return 0; } -int tdmReadPage(SDiskMgr *pDiskMgr, pgid_t pgid, void *pData) { +int tdmReadPage(STkvDiskMgr *pDiskMgr, pgid_t pgid, void *pData) { taosLSeekFile(pDiskMgr->fd, PAGE_OFFSET(pgid, pDiskMgr->pgsize), SEEK_SET); taosReadFile(pDiskMgr->fd, pData, pDiskMgr->pgsize); return 0; } -int tdmWritePage(SDiskMgr *pDiskMgr, pgid_t pgid, const void *pData) { +int tdmWritePage(STkvDiskMgr *pDiskMgr, pgid_t pgid, const void *pData) { taosLSeekFile(pDiskMgr->fd, PAGE_OFFSET(pgid, pDiskMgr->pgsize), SEEK_SET); taosWriteFile(pDiskMgr->fd, pData, pDiskMgr->pgsize); return 0; } -int32_t tdmAllocPage(SDiskMgr *pDiskMgr) { return pDiskMgr->npgid++; } \ No newline at end of file +int tdmFlush(STkvDiskMgr *pDiskMgr) { return taosFsyncFile(pDiskMgr->fd); } + +int32_t tdmAllocPage(STkvDiskMgr *pDiskMgr) { return pDiskMgr->npgid++; } \ No newline at end of file diff --git a/source/libs/tkv/src/tkvBufPool.c b/source/libs/tkv/src/tkvBufPool.c new file mode 100644 index 0000000000..86bfa0ba3e --- /dev/null +++ b/source/libs/tkv/src/tkvBufPool.c @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "thash.h" +#include "tlist.h" + +#include "tkvBufPool.h" +#include "tkvDiskMgr.h" +#include "tkvPage.h" + +struct SFrameIdWrapper { + TD_SLIST_NODE(SFrameIdWrapper); + frame_id_t id; +}; + +struct STkvBufPool { + STkvPage* pages; + STkvDiskMgr* pDiskMgr; + SHashObj* pgTb; // page_id_t --> frame_id_t + TD_SLIST(SFrameIdWrapper) freeList; + pthread_mutex_t mutex; +}; + +typedef struct STkvLRUReplacer { +} STkvLRUReplacer; + +typedef struct STkvLFUReplacer { +} STkvLFUReplacer; + +typedef struct STkvCLKReplacer { +} STkvCLKReplacer; + +typedef enum { TKV_LRU_REPLACER = 0, TKV_LFU_REPLACER, TVK_CLK_REPLACER } tkv_replacer_t; + +typedef struct STkvReplacer { + tkv_replacer_t type; + union { + STkvLRUReplacer lruRep; + STkvLFUReplacer lfuRep; + STkvCLKReplacer clkRep; + }; +} STkvReplacer; \ No newline at end of file diff --git a/source/libs/transport/CMakeLists.txt b/source/libs/transport/CMakeLists.txt index 98a380dc8f..c4eeef5df2 100644 --- a/source/libs/transport/CMakeLists.txt +++ b/source/libs/transport/CMakeLists.txt @@ -12,4 +12,19 @@ target_link_libraries( PUBLIC os PUBLIC util PUBLIC common -) \ No newline at end of file +) +if (${BUILD_WITH_UV}) + target_include_directories( + transport + PUBLIC "${CMAKE_SOURCE_DIR}/contrib/libuv/include" + ) + +#LINK_DIRECTORIES("${CMAKE_SOURCE_DIR}/debug/contrib/libuv") + target_link_libraries( + transport + PUBLIC uv_a + ) + add_definitions(-DUSE_UV) +endif(${BUILD_WITH_UV}) + + diff --git a/source/libs/transport/inc/rpcCache.h b/source/libs/transport/inc/rpcCache.h index 3a996aab7c..5148f5e23c 100644 --- a/source/libs/transport/inc/rpcCache.h +++ b/source/libs/transport/inc/rpcCache.h @@ -16,6 +16,8 @@ #ifndef TDENGINE_RPC_CACHE_H #define TDENGINE_RPC_CACHE_H +#include + #ifdef __cplusplus extern "C" { #endif diff --git a/source/libs/transport/inc/rpcHead.h b/source/libs/transport/inc/rpcHead.h index 6e98bbd563..7317d84af1 100644 --- a/source/libs/transport/inc/rpcHead.h +++ b/source/libs/transport/inc/rpcHead.h @@ -16,52 +16,57 @@ #ifndef TDENGINE_RPCHEAD_H #define TDENGINE_RPCHEAD_H +#include #ifdef __cplusplus extern "C" { #endif -#define RPC_CONN_TCP 2 +#ifdef USE_UV + +#else + +#define RPC_CONN_TCP 2 extern int tsRpcOverhead; typedef struct { - void *msg; + void* msg; int msgLen; - uint32_t ip; + uint32_t ip; uint16_t port; int connType; - void *shandle; - void *thandle; - void *chandle; + void* shandle; + void* thandle; + void* chandle; } SRecvInfo; #pragma pack(push, 1) typedef struct { - char version:4; // RPC version - char comp:4; // compression algorithm, 0:no compression 1:lz4 - char resflag:2; // reserved bits - char spi:3; // security parameter index - char encrypt:3; // encrypt algorithm, 0: no encryption - uint16_t tranId; // transcation ID - uint32_t linkUid; // for unique connection ID assigned by client - uint64_t ahandle; // ahandle assigned by client - uint32_t sourceId; // source ID, an index for connection list - uint32_t destId; // destination ID, an index for connection list - uint32_t destIp; // destination IP address, for NAT scenario - char user[TSDB_UNI_LEN]; // user ID - uint16_t port; // for UDP only, port may be changed - char empty[1]; // reserved - uint16_t msgType; // message type - int32_t msgLen; // message length including the header iteslf + char version : 4; // RPC version + char comp : 4; // compression algorithm, 0:no compression 1:lz4 + char resflag : 2; // reserved bits + char spi : 3; // security parameter index + char encrypt : 3; // encrypt algorithm, 0: no encryption + uint16_t tranId; // transcation ID + uint32_t linkUid; // for unique connection ID assigned by client + uint64_t ahandle; // ahandle assigned by client + uint32_t sourceId; // source ID, an index for connection list + uint32_t destId; // destination ID, an index for connection list + uint32_t destIp; // destination IP address, for NAT scenario + char user[TSDB_UNI_LEN]; // user ID + uint16_t port; // for UDP only, port may be changed + char empty[1]; // reserved + uint16_t msgType; // message type + int32_t msgLen; // message length including the header iteslf uint32_t msgVer; - int32_t code; // code in response message - uint8_t content[0]; // message body starts from here + int32_t code; // code in response message + uint8_t content[0]; // message body starts from here } SRpcHead; typedef struct { - int32_t reserved; - int32_t contLen; + int32_t reserved; + int32_t contLen; } SRpcComp; typedef struct { @@ -70,11 +75,10 @@ typedef struct { } SRpcDigest; #pragma pack(pop) - +#endif #ifdef __cplusplus } #endif #endif // TDENGINE_RPCHEAD_H - diff --git a/source/libs/transport/inc/rpcTcp.h b/source/libs/transport/inc/rpcTcp.h index 6ef8fc2d92..5e5c43a1db 100644 --- a/source/libs/transport/inc/rpcTcp.h +++ b/source/libs/transport/inc/rpcTcp.h @@ -15,23 +15,28 @@ #ifndef _rpc_tcp_header_ #define _rpc_tcp_header_ +#include #ifdef __cplusplus extern "C" { #endif +#ifdef USE_UV +#else void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThreads, void *fp, void *shandle); -void taosStopTcpServer(void *param); -void taosCleanUpTcpServer(void *param); +void taosStopTcpServer(void *param); +void taosCleanUpTcpServer(void *param); void *taosInitTcpClient(uint32_t ip, uint16_t port, char *label, int num, void *fp, void *shandle); -void taosStopTcpClient(void *chandle); -void taosCleanUpTcpClient(void *chandle); +void taosStopTcpClient(void *chandle); +void taosCleanUpTcpClient(void *chandle); void *taosOpenTcpClientConnection(void *shandle, void *thandle, uint32_t ip, uint16_t port); void taosCloseTcpConnection(void *chandle); int taosSendTcpData(uint32_t ip, uint16_t port, void *data, int len, void *chandle); +#endif + #ifdef __cplusplus } #endif diff --git a/source/libs/transport/inc/transportInt.h b/source/libs/transport/inc/transportInt.h index 24a2e0ef89..9809f7ee1a 100644 --- a/source/libs/transport/inc/transportInt.h +++ b/source/libs/transport/inc/transportInt.h @@ -20,8 +20,13 @@ extern "C" { #endif +#ifdef USE_UV + +#else + +#endif #ifdef __cplusplus } #endif -#endif /*_TD_TRANSPORT_INT_H_*/ \ No newline at end of file +#endif /*_TD_TRANSPORT_INT_H_*/ diff --git a/source/libs/transport/src/rpcCache.c b/source/libs/transport/src/rpcCache.c index 7de7aa341b..40767d2ba5 100644 --- a/source/libs/transport/src/rpcCache.c +++ b/source/libs/transport/src/rpcCache.c @@ -13,37 +13,40 @@ * along with this program. If not, see . */ +#include "rpcCache.h" #include "os.h" +#include "rpcLog.h" #include "taosdef.h" #include "tglobal.h" #include "tmempool.h" #include "ttimer.h" #include "tutil.h" -#include "rpcLog.h" -#include "rpcCache.h" +#ifdef USE_UV + +#else typedef struct SConnHash { char fqdn[TSDB_FQDN_LEN]; uint16_t port; char connType; struct SConnHash *prev; struct SConnHash *next; - void *data; + void * data; uint64_t time; } SConnHash; typedef struct { - SConnHash **connHashList; + SConnHash ** connHashList; mpool_h connHashMemPool; int maxSessions; int total; int * count; int64_t keepTimer; pthread_mutex_t mutex; - void (*cleanFp)(void *); - void *tmrCtrl; - void *pTimer; - int64_t *lockedBy; + void (*cleanFp)(void *); + void * tmrCtrl; + void * pTimer; + int64_t *lockedBy; } SConnCache; static int rpcHashConn(void *handle, char *fqdn, uint16_t port, int8_t connType); @@ -122,7 +125,7 @@ void rpcAddConnIntoCache(void *handle, void *data, char *fqdn, uint16_t port, in uint64_t time = taosGetTimestampMs(); pCache = (SConnCache *)handle; - assert(pCache); + assert(pCache); assert(data); hash = rpcHashConn(pCache, fqdn, port, connType); @@ -134,7 +137,7 @@ void rpcAddConnIntoCache(void *handle, void *data, char *fqdn, uint16_t port, in pNode->prev = NULL; pNode->time = time; - rpcLockCache(pCache->lockedBy+hash); + rpcLockCache(pCache->lockedBy + hash); pNode->next = pCache->connHashList[hash]; if (pCache->connHashList[hash] != NULL) (pCache->connHashList[hash])->prev = pNode; @@ -143,10 +146,11 @@ void rpcAddConnIntoCache(void *handle, void *data, char *fqdn, uint16_t port, in pCache->count[hash]++; rpcRemoveExpiredNodes(pCache, pNode->next, hash, time); - rpcUnlockCache(pCache->lockedBy+hash); + rpcUnlockCache(pCache->lockedBy + hash); pCache->total++; - // tTrace("%p %s:%hu:%d:%d:%p added into cache, connections:%d", data, fqdn, port, connType, hash, pNode, pCache->count[hash]); + // tTrace("%p %s:%hu:%d:%d:%p added into cache, connections:%d", data, fqdn, port, connType, hash, pNode, + // pCache->count[hash]); return; } @@ -158,12 +162,12 @@ void *rpcGetConnFromCache(void *handle, char *fqdn, uint16_t port, int8_t connTy void * pData = NULL; pCache = (SConnCache *)handle; - assert(pCache); + assert(pCache); uint64_t time = taosGetTimestampMs(); hash = rpcHashConn(pCache, fqdn, port, connType); - rpcLockCache(pCache->lockedBy+hash); + rpcLockCache(pCache->lockedBy + hash); pNode = pCache->connHashList[hash]; while (pNode) { @@ -197,12 +201,14 @@ void *rpcGetConnFromCache(void *handle, char *fqdn, uint16_t port, int8_t connTy pCache->count[hash]--; } - rpcUnlockCache(pCache->lockedBy+hash); + rpcUnlockCache(pCache->lockedBy + hash); if (pData) { - //tTrace("%p %s:%hu:%d:%d:%p retrieved from cache, connections:%d", pData, fqdn, port, connType, hash, pNode, pCache->count[hash]); + // tTrace("%p %s:%hu:%d:%d:%p retrieved from cache, connections:%d", pData, fqdn, port, connType, hash, pNode, + // pCache->count[hash]); } else { - //tTrace("%s:%hu:%d:%d failed to retrieve conn from cache, connections:%d", fqdn, port, connType, hash, pCache->count[hash]); + // tTrace("%s:%hu:%d:%d failed to retrieve conn from cache, connections:%d", fqdn, port, connType, hash, + // pCache->count[hash]); } return pData; @@ -221,10 +227,10 @@ static void rpcCleanConnCache(void *handle, void *tmrId) { uint64_t time = taosGetTimestampMs(); for (hash = 0; hash < pCache->maxSessions; ++hash) { - rpcLockCache(pCache->lockedBy+hash); + rpcLockCache(pCache->lockedBy + hash); pNode = pCache->connHashList[hash]; rpcRemoveExpiredNodes(pCache, pNode, hash, time); - rpcUnlockCache(pCache->lockedBy+hash); + rpcUnlockCache(pCache->lockedBy + hash); } // tTrace("timer, total connections in cache:%d", pCache->total); @@ -233,7 +239,7 @@ static void rpcCleanConnCache(void *handle, void *tmrId) { } static void rpcRemoveExpiredNodes(SConnCache *pCache, SConnHash *pNode, int hash, uint64_t time) { - if (pNode == NULL || (time < pCache->keepTimer + pNode->time) ) return; + if (pNode == NULL || (time < pCache->keepTimer + pNode->time)) return; SConnHash *pPrev = pNode->prev, *pNext; @@ -242,7 +248,8 @@ static void rpcRemoveExpiredNodes(SConnCache *pCache, SConnHash *pNode, int hash pNext = pNode->next; pCache->total--; pCache->count[hash]--; - //tTrace("%p %s:%hu:%d:%d:%p removed from cache, connections:%d", pNode->data, pNode->fqdn, pNode->port, pNode->connType, hash, pNode, + // tTrace("%p %s:%hu:%d:%d:%p removed from cache, connections:%d", pNode->data, pNode->fqdn, pNode->port, + // pNode->connType, hash, pNode, // pCache->count[hash]); taosMemPoolFree(pCache->connHashMemPool, (char *)pNode); pNode = pNext; @@ -257,7 +264,7 @@ static void rpcRemoveExpiredNodes(SConnCache *pCache, SConnHash *pNode, int hash static int rpcHashConn(void *handle, char *fqdn, uint16_t port, int8_t connType) { SConnCache *pCache = (SConnCache *)handle; int hash = 0; - char *temp = fqdn; + char * temp = fqdn; while (*temp) { hash += *temp; @@ -288,4 +295,4 @@ static void rpcUnlockCache(int64_t *lockedBy) { assert(false); } } - +#endif diff --git a/source/libs/transport/src/rpcMain.c b/source/libs/transport/src/rpcMain.c index d58ea63c4f..3095ddb9d2 100644 --- a/source/libs/transport/src/rpcMain.c +++ b/source/libs/transport/src/rpcMain.c @@ -13,117 +13,327 @@ * along with this program. If not, see . */ +#ifdef USE_UV +#include +#endif +#include "lz4.h" #include "os.h" +#include "rpcCache.h" +#include "rpcHead.h" +#include "rpcLog.h" +#include "rpcTcp.h" +#include "rpcUdp.h" +#include "taoserror.h" +#include "tglobal.h" +#include "thash.h" #include "tidpool.h" #include "tmd5.h" #include "tmempool.h" +#include "tmsg.h" +#include "tref.h" +#include "trpc.h" #include "ttimer.h" #include "tutil.h" -#include "lz4.h" -#include "tref.h" -#include "taoserror.h" -#include "tglobal.h" -#include "tmsg.h" -#include "trpc.h" -#include "thash.h" -#include "rpcLog.h" -#include "rpcUdp.h" -#include "rpcCache.h" -#include "rpcTcp.h" -#include "rpcHead.h" -#define RPC_MSG_OVERHEAD (sizeof(SRpcReqContext) + sizeof(SRpcHead) + sizeof(SRpcDigest)) -#define rpcHeadFromCont(cont) ((SRpcHead *) ((char*)cont - sizeof(SRpcHead))) +typedef struct { + int sessions; // number of sessions allowed + int numOfThreads; // number of threads to process incoming messages + int idleTime; // milliseconds; + uint16_t localPort; + int8_t connType; + int index; // for UDP server only, round robin for multiple threads + char label[TSDB_LABEL_LEN]; + + char user[TSDB_UNI_LEN]; // meter ID + char spi; // security parameter index + char encrypt; // encrypt algorithm + char secret[TSDB_PASSWORD_LEN]; // secret for the link + char ckey[TSDB_PASSWORD_LEN]; // ciphering key + + void (*cfp)(void* parent, SRpcMsg*, SEpSet*); + int (*afp)(void* parent, char* user, char* spi, char* encrypt, char* secret, char* ckey); + + int32_t refCount; + void* parent; + void* idPool; // handle to ID pool + void* tmrCtrl; // handle to timer + SHashObj* hash; // handle returned by hash utility + void* tcphandle; // returned handle from TCP initialization + void* udphandle; // returned handle from UDP initialization + void* pCache; // connection cache + pthread_mutex_t mutex; + struct SRpcConn* connList; // connection list +} SRpcInfo; + +#ifdef USE_UV + +#define container_of(ptr, type, member) ((type*)((char*)(ptr)-offsetof(type, member))) + +typedef struct SThreadObj { + pthread_t thread; + uv_pipe_t* pipe; + uv_loop_t* loop; + uv_async_t* workerAsync; // + int fd; +} SThreadObj; + +typedef struct SServerObj { + uv_tcp_t server; + uv_loop_t* loop; + int workerIdx; + int numOfThread; + SThreadObj** pThreadObj; + uv_pipe_t** pipe; +} SServerObj; + +typedef struct SConnCtx { + uv_tcp_t* pClient; + uv_timer_t* pTimer; + uv_async_t* pWorkerAsync; + int ref; +} SConnCtx; + +static void allocBuffer(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf); +static void onTimeout(uv_timer_t* handle); +static void onRead(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf); +static void onWrite(uv_write_t* req, int status); +static void onAccept(uv_stream_t* stream, int status); +void onConnection(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf); +static void workerAsyncCB(uv_async_t* handle); +static void* workerThread(void* arg); + +int32_t rpcInit() { return -1; } +void rpcCleanup() { return; }; +void* rpcOpen(const SRpcInit* pInit) { + SRpcInfo* pRpc = calloc(1, sizeof(SRpcInfo)); + if (pRpc == NULL) { + return NULL; + } + if (pInit->label) { + tstrncpy(pRpc->label, pInit->label, sizeof(pRpc->label)); + } + pRpc->numOfThreads = pInit->numOfThreads > TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS : pInit->numOfThreads; + + SServerObj* srv = calloc(1, sizeof(SServerObj)); + srv->loop = (uv_loop_t*)malloc(sizeof(uv_loop_t)); + srv->numOfThread = pRpc->numOfThreads; + srv->workerIdx = 0; + srv->pThreadObj = (SThreadObj**)calloc(srv->numOfThread, sizeof(SThreadObj*)); + srv->pipe = (uv_pipe_t**)calloc(srv->numOfThread, sizeof(uv_pipe_t*)); + uv_loop_init(srv->loop); + + for (int i = 0; i < srv->numOfThread; i++) { + srv->pThreadObj[i] = (SThreadObj*)calloc(1, sizeof(SThreadObj)); + srv->pipe[i] = (uv_pipe_t*)calloc(2, sizeof(uv_pipe_t)); + int fds[2]; + if (uv_socketpair(AF_UNIX, SOCK_STREAM, fds, UV_NONBLOCK_PIPE, UV_NONBLOCK_PIPE) != 0) { + return NULL; + } + uv_pipe_init(srv->loop, &(srv->pipe[i][0]), 1); + uv_pipe_open(&(srv->pipe[i][0]), fds[1]); // init write + + srv->pThreadObj[i]->fd = fds[0]; + srv->pThreadObj[i]->pipe = &(srv->pipe[i][1]); // init read + int err = pthread_create(&(srv->pThreadObj[i]->thread), NULL, workerThread, (void*)(srv->pThreadObj[i])); + if (err == 0) { + tError("sucess to create worker thread %d", i); + // printf("thread %d create\n", i); + } else { + tError("failed to create worker thread %d", i); + return NULL; + } + } + uv_tcp_init(srv->loop, &srv->server); + struct sockaddr_in bind_addr; + uv_ip4_addr("0.0.0.0", pInit->localPort, &bind_addr); + uv_tcp_bind(&srv->server, (const struct sockaddr*)&bind_addr, 0); + int err = 0; + if ((err = uv_listen((uv_stream_t*)&srv->server, 128, onAccept)) != 0) { + tError("Listen error %s\n", uv_err_name(err)); + return NULL; + } + uv_run(srv->loop, UV_RUN_DEFAULT); + + return pRpc; +} +void rpcClose(void* arg) { return; } +void* rpcMallocCont(int contLen) { return NULL; } +void rpcFreeCont(void* cont) { return; } +void* rpcReallocCont(void* ptr, int contLen) { return NULL; } + +void rpcSendRequest(void* thandle, const SEpSet* pEpSet, SRpcMsg* pMsg, int64_t* rid) { return; } + +void rpcSendResponse(const SRpcMsg* pMsg) {} + +void rpcSendRedirectRsp(void* pConn, const SEpSet* pEpSet) {} +int rpcGetConnInfo(void* thandle, SRpcConnInfo* pInfo) { return -1; } +void rpcSendRecv(void* shandle, SEpSet* pEpSet, SRpcMsg* pReq, SRpcMsg* pRsp) { return; } +int rpcReportProgress(void* pConn, char* pCont, int contLen) { return -1; } +void rpcCancelRequest(int64_t rid) { return; } + +void allocBuffer(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) { + buf->base = malloc(suggested_size); + buf->len = suggested_size; +} + +void onTimeout(uv_timer_t* handle) { + // opt + tDebug("time out"); +} +void onRead(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf) { + // opt + tDebug("data already was read on a stream"); +} + +void onWrite(uv_write_t* req, int status) { + // opt + if (req) tDebug("data already was written on stream"); +} + +void workerAsyncCB(uv_async_t* handle) { + // opt + SThreadObj* pObj = container_of(handle, SThreadObj, workerAsync); +} +void onAccept(uv_stream_t* stream, int status) { + if (status == -1) { + return; + } + SServerObj* pObj = container_of(stream, SServerObj, server); + tDebug("new conntion accepted by main server, dispatch to one worker thread"); + + uv_tcp_t* cli = (uv_tcp_t*)malloc(sizeof(uv_tcp_t)); + uv_tcp_init(pObj->loop, cli); + if (uv_accept(stream, (uv_stream_t*)cli) == 0) { + uv_write_t* wr = (uv_write_t*)malloc(sizeof(uv_write_t)); + + uv_buf_t buf = uv_buf_init("a", 1); + // despatch to worker thread + pObj->workerIdx = (pObj->workerIdx + 1) % pObj->numOfThread; + uv_write2(wr, (uv_stream_t*)&(pObj->pipe[pObj->workerIdx][0]), &buf, 1, (uv_stream_t*)cli, onWrite); + } else { + uv_close((uv_handle_t*)cli, NULL); + } +} +void onConnection(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) { + if (nread < 0) { + if (nread != UV_EOF) { + tError("read error %s", uv_err_name(nread)); + } + // TODO(log other failure reason) + uv_close((uv_handle_t*)q, NULL); + return; + } + SThreadObj* pObj = (SThreadObj*)container_of(q, struct SThreadObj, pipe); + + uv_pipe_t* pipe = (uv_pipe_t*)q; + if (!uv_pipe_pending_count(pipe)) { + tError("No pending count"); + return; + } + uv_handle_type pending = uv_pipe_pending_type(pipe); + assert(pending == UV_TCP); + + SConnCtx* pConn = malloc(sizeof(SConnCtx)); + /* init conn timer*/ + pConn->pTimer = malloc(sizeof(uv_timer_t)); + uv_timer_init(pObj->loop, pConn->pTimer); + + pConn->pClient = (uv_tcp_t*)malloc(sizeof(uv_tcp_t)); + pConn->pWorkerAsync = pObj->workerAsync; // thread safty + uv_tcp_init(pObj->loop, pConn->pClient); + + if (uv_accept(q, (uv_stream_t*)(pConn->pClient)) == 0) { + uv_os_fd_t fd; + uv_fileno((const uv_handle_t*)pConn->pClient, &fd); + tDebug("new connection created: %d", fd); + uv_timer_start(pConn->pTimer, onTimeout, 10, 0); + uv_read_start((uv_stream_t*)(pConn->pClient), allocBuffer, onRead); + } else { + uv_timer_stop(pConn->pTimer); + free(pConn->pTimer); + uv_close((uv_handle_t*)pConn->pClient, NULL); + free(pConn->pClient); + free(pConn); + } +} + +void* workerThread(void* arg) { + SThreadObj* pObj = (SThreadObj*)arg; + int fd = pObj->fd; + pObj->loop = (uv_loop_t*)malloc(sizeof(uv_loop_t)); + uv_loop_init(pObj->loop); + + uv_pipe_init(pObj->loop, pObj->pipe, 1); + uv_pipe_open(pObj->pipe, fd); + + pObj->workerAsync = malloc(sizeof(uv_async_t)); + uv_async_init(pObj->loop, pObj->workerAsync, workerAsyncCB); + uv_read_start((uv_stream_t*)pObj->pipe, allocBuffer, onConnection); +} +#else + +#define RPC_MSG_OVERHEAD (sizeof(SRpcReqContext) + sizeof(SRpcHead) + sizeof(SRpcDigest)) +#define rpcHeadFromCont(cont) ((SRpcHead*)((char*)cont - sizeof(SRpcHead))) #define rpcContFromHead(msg) (msg + sizeof(SRpcHead)) #define rpcMsgLenFromCont(contLen) (contLen + sizeof(SRpcHead)) #define rpcContLenFromMsg(msgLen) (msgLen - sizeof(SRpcHead)) #define rpcIsReq(type) (type & 1U) typedef struct { - int sessions; // number of sessions allowed - int numOfThreads; // number of threads to process incoming messages - int idleTime; // milliseconds; - uint16_t localPort; - int8_t connType; - int index; // for UDP server only, round robin for multiple threads - char label[TSDB_LABEL_LEN]; - - char user[TSDB_UNI_LEN]; // meter ID - char spi; // security parameter index - char encrypt; // encrypt algorithm - char secret[TSDB_PASSWORD_LEN]; // secret for the link - char ckey[TSDB_PASSWORD_LEN]; // ciphering key - - void (*cfp)(void *parent, SRpcMsg *, SEpSet *); - int (*afp)(void *parent, char *user, char *spi, char *encrypt, char *secret, char *ckey); - - int32_t refCount; - void *parent; - void *idPool; // handle to ID pool - void *tmrCtrl; // handle to timer - SHashObj *hash; // handle returned by hash utility - void *tcphandle;// returned handle from TCP initialization - void *udphandle;// returned handle from UDP initialization - void *pCache; // connection cache - pthread_mutex_t mutex; - struct SRpcConn *connList; // connection list -} SRpcInfo; - -typedef struct { - SRpcInfo *pRpc; // associated SRpcInfo - SEpSet epSet; // ip list provided by app - void *ahandle; // handle provided by app - struct SRpcConn *pConn; // pConn allocated - tmsg_t msgType; // message type - uint8_t *pCont; // content provided by app - int32_t contLen; // content length - int32_t code; // error code - int16_t numOfTry; // number of try for different servers - int8_t oldInUse; // server EP inUse passed by app - int8_t redirect; // flag to indicate redirect - int8_t connType; // connection type - int64_t rid; // refId returned by taosAddRef - SRpcMsg *pRsp; // for synchronous API - tsem_t *pSem; // for synchronous API - SEpSet *pSet; // for synchronous API - char msg[0]; // RpcHead starts from here + SRpcInfo * pRpc; // associated SRpcInfo + SEpSet epSet; // ip list provided by app + void * ahandle; // handle provided by app + struct SRpcConn *pConn; // pConn allocated + tmsg_t msgType; // message type + uint8_t * pCont; // content provided by app + int32_t contLen; // content length + int32_t code; // error code + int16_t numOfTry; // number of try for different servers + int8_t oldInUse; // server EP inUse passed by app + int8_t redirect; // flag to indicate redirect + int8_t connType; // connection type + int64_t rid; // refId returned by taosAddRef + SRpcMsg * pRsp; // for synchronous API + tsem_t * pSem; // for synchronous API + SEpSet * pSet; // for synchronous API + char msg[0]; // RpcHead starts from here } SRpcReqContext; typedef struct SRpcConn { - char info[48];// debug info: label + pConn + ahandle - int sid; // session ID - uint32_t ownId; // own link ID - uint32_t peerId; // peer link ID - char user[TSDB_UNI_LEN]; // user ID for the link - char spi; // security parameter index - char encrypt; // encryption, 0:1 - char secret[TSDB_PASSWORD_LEN]; // secret for the link - char ckey[TSDB_PASSWORD_LEN]; // ciphering key - char secured; // if set to 1, no authentication - uint16_t localPort; // for UDP only - uint32_t linkUid; // connection unique ID assigned by client - uint32_t peerIp; // peer IP - uint16_t peerPort; // peer port - char peerFqdn[TSDB_FQDN_LEN]; // peer FQDN or ip string - uint16_t tranId; // outgoing transcation ID, for build message - uint16_t outTranId; // outgoing transcation ID - uint16_t inTranId; // transcation ID for incoming msg - tmsg_t outType; // message type for outgoing request - tmsg_t inType; // message type for incoming request - void *chandle; // handle passed by TCP/UDP connection layer - void *ahandle; // handle provided by upper app layter - int retry; // number of retry for sending request - int tretry; // total retry - void *pTimer; // retry timer to monitor the response - void *pIdleTimer; // idle timer - char *pRspMsg; // response message including header - int rspMsgLen; // response messag length - char *pReqMsg; // request message including header - int reqMsgLen; // request message length - SRpcInfo *pRpc; // the associated SRpcInfo - int8_t connType; // connection type - int64_t lockedBy; // lock for connection - SRpcReqContext *pContext; // request context + char info[48]; // debug info: label + pConn + ahandle + int sid; // session ID + uint32_t ownId; // own link ID + uint32_t peerId; // peer link ID + char user[TSDB_UNI_LEN]; // user ID for the link + char spi; // security parameter index + char encrypt; // encryption, 0:1 + char secret[TSDB_PASSWORD_LEN]; // secret for the link + char ckey[TSDB_PASSWORD_LEN]; // ciphering key + char secured; // if set to 1, no authentication + uint16_t localPort; // for UDP only + uint32_t linkUid; // connection unique ID assigned by client + uint32_t peerIp; // peer IP + uint16_t peerPort; // peer port + char peerFqdn[TSDB_FQDN_LEN]; // peer FQDN or ip string + uint16_t tranId; // outgoing transcation ID, for build message + uint16_t outTranId; // outgoing transcation ID + uint16_t inTranId; // transcation ID for incoming msg + tmsg_t outType; // message type for outgoing request + tmsg_t inType; // message type for incoming request + void * chandle; // handle passed by TCP/UDP connection layer + void * ahandle; // handle provided by upper app layter + int retry; // number of retry for sending request + int tretry; // total retry + void * pTimer; // retry timer to monitor the response + void * pIdleTimer; // idle timer + char * pRspMsg; // response message including header + int rspMsgLen; // response messag length + char * pReqMsg; // request message including header + int reqMsgLen; // request message length + SRpcInfo * pRpc; // the associated SRpcInfo + int8_t connType; // connection type + int64_t lockedBy; // lock for connection + SRpcReqContext *pContext; // request context } SRpcConn; static pthread_once_t tsRpcInitOnce = PTHREAD_ONCE_INIT; @@ -137,41 +347,29 @@ int tsRpcOverhead; static int tsRpcRefId = -1; static int32_t tsRpcNum = 0; -//static pthread_once_t tsRpcInit = PTHREAD_ONCE_INIT; +// static pthread_once_t tsRpcInit = PTHREAD_ONCE_INIT; // server:0 client:1 tcp:2 udp:0 -#define RPC_CONN_UDPS 0 -#define RPC_CONN_UDPC 1 -#define RPC_CONN_TCPS 2 -#define RPC_CONN_TCPC 3 +#define RPC_CONN_UDPS 0 +#define RPC_CONN_UDPC 1 +#define RPC_CONN_TCPS 2 +#define RPC_CONN_TCPC 3 void *(*taosInitConn[])(uint32_t ip, uint16_t port, char *label, int threads, void *fp, void *shandle) = { - taosInitUdpConnection, - taosInitUdpConnection, - taosInitTcpServer, - taosInitTcpClient -}; + taosInitUdpConnection, taosInitUdpConnection, taosInitTcpServer, taosInitTcpClient}; -void (*taosCleanUpConn[])(void *thandle) = { - taosCleanUpUdpConnection, - taosCleanUpUdpConnection, - taosCleanUpTcpServer, - taosCleanUpTcpClient -}; +void (*taosCleanUpConn[])(void *thandle) = {taosCleanUpUdpConnection, taosCleanUpUdpConnection, taosCleanUpTcpServer, + taosCleanUpTcpClient}; void (*taosStopConn[])(void *thandle) = { - taosStopUdpConnection, - taosStopUdpConnection, + taosStopUdpConnection, + taosStopUdpConnection, taosStopTcpServer, taosStopTcpClient, }; int (*taosSendData[])(uint32_t ip, uint16_t port, void *data, int len, void *chandle) = { - taosSendUdpData, - taosSendUdpData, - taosSendTcpData, - taosSendTcpData -}; + taosSendUdpData, taosSendUdpData, taosSendTcpData, taosSendTcpData}; void *(*taosOpenConn[])(void *shandle, void *thandle, uint32_t ip, uint16_t port) = { taosOpenUdpConnection, @@ -180,12 +378,7 @@ void *(*taosOpenConn[])(void *shandle, void *thandle, uint32_t ip, uint16_t port taosOpenTcpClientConnection, }; -void (*taosCloseConn[])(void *chandle) = { - NULL, - NULL, - taosCloseTcpConnection, - taosCloseTcpConnection -}; +void (*taosCloseConn[])(void *chandle) = {NULL, NULL, taosCloseTcpConnection, taosCloseTcpConnection}; static SRpcConn *rpcOpenConn(SRpcInfo *pRpc, char *peerFqdn, uint16_t peerPort, int8_t connType); static void rpcCloseConn(void *thandle); @@ -194,11 +387,11 @@ static SRpcConn *rpcAllocateClientConn(SRpcInfo *pRpc); static SRpcConn *rpcAllocateServerConn(SRpcInfo *pRpc, SRecvInfo *pRecv); static SRpcConn *rpcGetConnObj(SRpcInfo *pRpc, int sid, SRecvInfo *pRecv); -static void rpcSendReqToServer(SRpcInfo *pRpc, SRpcReqContext *pContext); -static void rpcSendQuickRsp(SRpcConn *pConn, int32_t code); -static void rpcSendErrorMsgToPeer(SRecvInfo *pRecv, int32_t code); -static void rpcSendMsgToPeer(SRpcConn *pConn, void *data, int dataLen); -static void rpcSendReqHead(SRpcConn *pConn); +static void rpcSendReqToServer(SRpcInfo *pRpc, SRpcReqContext *pContext); +static void rpcSendQuickRsp(SRpcConn *pConn, int32_t code); +static void rpcSendErrorMsgToPeer(SRecvInfo *pRecv, int32_t code); +static void rpcSendMsgToPeer(SRpcConn *pConn, void *data, int dataLen); +static void rpcSendReqHead(SRpcConn *pConn); static void *rpcProcessMsgFromPeer(SRecvInfo *pRecv); static void rpcProcessIncomingMsg(SRpcConn *pConn, SRpcHead *pHead, SRpcReqContext *pContext); @@ -207,15 +400,15 @@ static void rpcProcessRetryTimer(void *, void *); static void rpcProcessIdleTimer(void *param, void *tmrId); static void rpcProcessProgressTimer(void *param, void *tmrId); -static void rpcFreeMsg(void *msg); -static int32_t rpcCompressRpcMsg(char* pCont, int32_t contLen); +static void rpcFreeMsg(void *msg); +static int32_t rpcCompressRpcMsg(char *pCont, int32_t contLen); static SRpcHead *rpcDecompressRpcMsg(SRpcHead *pHead); -static int rpcAddAuthPart(SRpcConn *pConn, char *msg, int msgLen); -static int rpcCheckAuthentication(SRpcConn *pConn, char *msg, int msgLen); -static void rpcLockConn(SRpcConn *pConn); -static void rpcUnlockConn(SRpcConn *pConn); -static void rpcAddRef(SRpcInfo *pRpc); -static void rpcDecRef(SRpcInfo *pRpc); +static int rpcAddAuthPart(SRpcConn *pConn, char *msg, int msgLen); +static int rpcCheckAuthentication(SRpcConn *pConn, char *msg, int msgLen); +static void rpcLockConn(SRpcConn *pConn); +static void rpcUnlockConn(SRpcConn *pConn); +static void rpcAddRef(SRpcInfo *pRpc); +static void rpcDecRef(SRpcInfo *pRpc); static void rpcFree(void *p) { tTrace("free mem: %p", p); @@ -240,26 +433,26 @@ void rpcCleanup(void) { taosCloseRef(tsRpcRefId); tsRpcRefId = -1; } - + void *rpcOpen(const SRpcInit *pInit) { SRpcInfo *pRpc; - //pthread_once(&tsRpcInit, rpcInit); + // pthread_once(&tsRpcInit, rpcInit); pRpc = (SRpcInfo *)calloc(1, sizeof(SRpcInfo)); if (pRpc == NULL) return NULL; - if(pInit->label) tstrncpy(pRpc->label, pInit->label, sizeof(pRpc->label)); + if (pInit->label) tstrncpy(pRpc->label, pInit->label, sizeof(pRpc->label)); pRpc->connType = pInit->connType; if (pRpc->connType == TAOS_CONN_CLIENT) { pRpc->numOfThreads = pInit->numOfThreads; } else { - pRpc->numOfThreads = pInit->numOfThreads>TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS:pInit->numOfThreads; + pRpc->numOfThreads = pInit->numOfThreads > TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS : pInit->numOfThreads; } pRpc->idleTime = pInit->idleTime; pRpc->localPort = pInit->localPort; pRpc->afp = pInit->afp; - pRpc->sessions = pInit->sessions+1; + pRpc->sessions = pInit->sessions + 1; if (pInit->user) tstrncpy(pRpc->user, pInit->user, sizeof(pRpc->user)); if (pInit->secret) memcpy(pRpc->secret, pInit->secret, sizeof(pRpc->secret)); if (pInit->ckey) tstrncpy(pRpc->ckey, pInit->ckey, sizeof(pRpc->ckey)); @@ -279,14 +472,14 @@ void *rpcOpen(const SRpcInit *pInit) { return NULL; } - pRpc->idPool = taosInitIdPool(pRpc->sessions-1); + pRpc->idPool = taosInitIdPool(pRpc->sessions - 1); if (pRpc->idPool == NULL) { tError("%s failed to init ID pool", pRpc->label); rpcClose(pRpc); return NULL; } - pRpc->tmrCtrl = taosTmrInit(pRpc->sessions*2 + 1, 50, 10000, pRpc->label); + pRpc->tmrCtrl = taosTmrInit(pRpc->sessions * 2 + 1, 50, 10000, pRpc->label); if (pRpc->tmrCtrl == NULL) { tError("%s failed to init timers", pRpc->label); rpcClose(pRpc); @@ -301,8 +494,8 @@ void *rpcOpen(const SRpcInit *pInit) { return NULL; } } else { - pRpc->pCache = rpcOpenConnCache(pRpc->sessions, rpcCloseConn, pRpc->tmrCtrl, pRpc->idleTime * 20); - if ( pRpc->pCache == NULL ) { + pRpc->pCache = rpcOpenConnCache(pRpc->sessions, rpcCloseConn, pRpc->tmrCtrl, pRpc->idleTime * 20); + if (pRpc->pCache == NULL) { tError("%s failed to init connection cache", pRpc->label); rpcClose(pRpc); return NULL; @@ -311,10 +504,10 @@ void *rpcOpen(const SRpcInit *pInit) { pthread_mutex_init(&pRpc->mutex, NULL); - pRpc->tcphandle = (*taosInitConn[pRpc->connType|RPC_CONN_TCP])(0, pRpc->localPort, pRpc->label, - pRpc->numOfThreads, rpcProcessMsgFromPeer, pRpc); - pRpc->udphandle = (*taosInitConn[pRpc->connType])(0, pRpc->localPort, pRpc->label, - pRpc->numOfThreads, rpcProcessMsgFromPeer, pRpc); + pRpc->tcphandle = (*taosInitConn[pRpc->connType | RPC_CONN_TCP])(0, pRpc->localPort, pRpc->label, pRpc->numOfThreads, + rpcProcessMsgFromPeer, pRpc); + pRpc->udphandle = + (*taosInitConn[pRpc->connType])(0, pRpc->localPort, pRpc->label, pRpc->numOfThreads, rpcProcessMsgFromPeer, pRpc); if (pRpc->tcphandle == NULL || pRpc->udphandle == NULL) { tError("%s failed to init network, port:%d", pRpc->label, pRpc->localPort); @@ -334,7 +527,7 @@ void rpcClose(void *param) { (*taosStopConn[pRpc->connType | RPC_CONN_TCP])(pRpc->tcphandle); (*taosStopConn[pRpc->connType])(pRpc->udphandle); - // close all connections + // close all connections for (int i = 0; i < pRpc->sessions; ++i) { if (pRpc->connList && pRpc->connList[i].user[0]) { rpcCloseConn((void *)(pRpc->connList + i)); @@ -375,8 +568,8 @@ void *rpcReallocCont(void *ptr, int contLen) { if (ptr == NULL) return rpcMallocCont(contLen); char *start = ((char *)ptr) - sizeof(SRpcReqContext) - sizeof(SRpcHead); - if (contLen == 0 ) { - free(start); + if (contLen == 0) { + free(start); return NULL; } @@ -385,17 +578,17 @@ void *rpcReallocCont(void *ptr, int contLen) { if (start == NULL) { tError("failed to realloc cont, size:%d", size); return NULL; - } + } return start + sizeof(SRpcReqContext) + sizeof(SRpcHead); } void rpcSendRequest(void *shandle, const SEpSet *pEpSet, SRpcMsg *pMsg, int64_t *pRid) { - SRpcInfo *pRpc = (SRpcInfo *)shandle; + SRpcInfo * pRpc = (SRpcInfo *)shandle; SRpcReqContext *pContext; int contLen = rpcCompressRpcMsg(pMsg->pCont, pMsg->contLen); - pContext = (SRpcReqContext *) ((char*)pMsg->pCont-sizeof(SRpcHead)-sizeof(SRpcReqContext)); + pContext = (SRpcReqContext *)((char *)pMsg->pCont - sizeof(SRpcHead) - sizeof(SRpcReqContext)); pContext->ahandle = pMsg->ahandle; pContext->pRpc = (SRpcInfo *)shandle; pContext->epSet = *pEpSet; @@ -404,16 +597,15 @@ void rpcSendRequest(void *shandle, const SEpSet *pEpSet, SRpcMsg *pMsg, int64_t pContext->msgType = pMsg->msgType; pContext->oldInUse = pEpSet->inUse; - pContext->connType = RPC_CONN_UDPC; - if (contLen > tsRpcMaxUdpSize || tsRpcForceTcp ) pContext->connType = RPC_CONN_TCPC; + pContext->connType = RPC_CONN_UDPC; + if (contLen > tsRpcMaxUdpSize || tsRpcForceTcp) pContext->connType = RPC_CONN_TCPC; - // connection type is application specific. + // connection type is application specific. // for TDengine, all the query, show commands shall have TCP connection tmsg_t type = pMsg->msgType; - if (type == TDMT_VND_QUERY || type == TDMT_MND_SHOW_RETRIEVE - || type == TDMT_VND_FETCH || type == TDMT_MND_VGROUP_LIST - || type == TDMT_VND_TABLES_META || type == TDMT_VND_TABLE_META - || type == TDMT_MND_SHOW || type == TDMT_MND_STATUS || type == TDMT_VND_ALTER_TABLE) + if (type == TDMT_VND_QUERY || type == TDMT_MND_SHOW_RETRIEVE || type == TDMT_VND_FETCH || + type == TDMT_MND_VGROUP_LIST || type == TDMT_VND_TABLES_META || type == TDMT_VND_TABLE_META || + type == TDMT_MND_SHOW || type == TDMT_MND_STATUS || type == TDMT_VND_ALTER_TABLE) pContext->connType = RPC_CONN_TCPC; pContext->rid = taosAddRef(tsRpcRefId, pContext); @@ -425,26 +617,26 @@ void rpcSendRequest(void *shandle, const SEpSet *pEpSet, SRpcMsg *pMsg, int64_t void rpcSendResponse(const SRpcMsg *pRsp) { if (pRsp->handle == NULL) return; - int msgLen = 0; - SRpcConn *pConn = (SRpcConn *)pRsp->handle; - SRpcMsg rpcMsg = *pRsp; - SRpcMsg *pMsg = &rpcMsg; - SRpcInfo *pRpc = pConn->pRpc; + int msgLen = 0; + SRpcConn *pConn = (SRpcConn *)pRsp->handle; + SRpcMsg rpcMsg = *pRsp; + SRpcMsg * pMsg = &rpcMsg; + SRpcInfo *pRpc = pConn->pRpc; - if ( pMsg->pCont == NULL ) { + if (pMsg->pCont == NULL) { pMsg->pCont = rpcMallocCont(0); pMsg->contLen = 0; } - SRpcHead *pHead = rpcHeadFromCont(pMsg->pCont); - char *msg = (char *)pHead; + SRpcHead *pHead = rpcHeadFromCont(pMsg->pCont); + char * msg = (char *)pHead; pMsg->contLen = rpcCompressRpcMsg(pMsg->pCont, pMsg->contLen); msgLen = rpcMsgLenFromCont(pMsg->contLen); rpcLockConn(pConn); - if ( pConn->inType == 0 || pConn->user[0] == 0 ) { + if (pConn->inType == 0 || pConn->user[0] == 0) { tError("%s, connection is already released, rsp wont be sent", pConn->info); rpcUnlockConn(pConn); rpcFreeCont(pMsg->pCont); @@ -454,7 +646,7 @@ void rpcSendResponse(const SRpcMsg *pRsp) { // set msg header pHead->version = 1; - pHead->msgType = pConn->inType+1; + pHead->msgType = pConn->inType + 1; pHead->spi = pConn->spi; pHead->encrypt = pConn->encrypt; pHead->tranId = pConn->inTranId; @@ -463,13 +655,13 @@ void rpcSendResponse(const SRpcMsg *pRsp) { pHead->linkUid = pConn->linkUid; pHead->port = htons(pConn->localPort); pHead->code = htonl(pMsg->code); - pHead->ahandle = (uint64_t) pConn->ahandle; - + pHead->ahandle = (uint64_t)pConn->ahandle; + // set pConn parameters pConn->inType = 0; // response message is released until new response is sent - rpcFreeMsg(pConn->pRspMsg); + rpcFreeMsg(pConn->pRspMsg); pConn->pRspMsg = msg; pConn->rspMsgLen = msgLen; if (pMsg->code == TSDB_CODE_RPC_ACTION_IN_PROGRESS) pConn->inTranId--; @@ -482,23 +674,22 @@ void rpcSendResponse(const SRpcMsg *pRsp) { rpcSendMsgToPeer(pConn, msg, msgLen); // if not set to secured, set it expcet NOT_READY case, since client wont treat it as secured - if (pConn->secured == 0 && pMsg->code != TSDB_CODE_RPC_NOT_READY) - pConn->secured = 1; // connection shall be secured + if (pConn->secured == 0 && pMsg->code != TSDB_CODE_RPC_NOT_READY) pConn->secured = 1; // connection shall be secured if (pConn->pReqMsg) rpcFreeCont(pConn->pReqMsg); pConn->pReqMsg = NULL; pConn->reqMsgLen = 0; rpcUnlockConn(pConn); - rpcDecRef(pRpc); // decrease the referene count + rpcDecRef(pRpc); // decrease the referene count return; } void rpcSendRedirectRsp(void *thandle, const SEpSet *pEpSet) { - SRpcMsg rpcMsg; + SRpcMsg rpcMsg; memset(&rpcMsg, 0, sizeof(rpcMsg)); - + rpcMsg.contLen = sizeof(SEpSet); rpcMsg.pCont = rpcMallocCont(rpcMsg.contLen); if (rpcMsg.pCont == NULL) return; @@ -514,24 +705,24 @@ void rpcSendRedirectRsp(void *thandle, const SEpSet *pEpSet) { } int rpcGetConnInfo(void *thandle, SRpcConnInfo *pInfo) { - SRpcConn *pConn = (SRpcConn *)thandle; + SRpcConn *pConn = (SRpcConn *)thandle; if (pConn->user[0] == 0) return -1; pInfo->clientIp = pConn->peerIp; pInfo->clientPort = pConn->peerPort; // pInfo->serverIp = pConn->destIp; - + tstrncpy(pInfo->user, pConn->user, sizeof(pInfo->user)); return 0; } void rpcSendRecv(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { SRpcReqContext *pContext; - pContext = (SRpcReqContext *) ((char*)pMsg->pCont-sizeof(SRpcHead)-sizeof(SRpcReqContext)); + pContext = (SRpcReqContext *)((char *)pMsg->pCont - sizeof(SRpcHead) - sizeof(SRpcReqContext)); memset(pRsp, 0, sizeof(SRpcMsg)); - - tsem_t sem; + + tsem_t sem; tsem_init(&sem, 0, 0); pContext->pSem = &sem; pContext->pRsp = pRsp; @@ -548,13 +739,13 @@ void rpcSendRecv(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { // this API is used by server app to keep an APP context in case connection is broken int rpcReportProgress(void *handle, char *pCont, int contLen) { SRpcConn *pConn = (SRpcConn *)handle; - int code = 0; + int code = 0; rpcLockConn(pConn); if (pConn->user[0]) { // pReqMsg and reqMsgLen is re-used to store the context from app server - pConn->pReqMsg = pCont; + pConn->pReqMsg = pCont; pConn->reqMsgLen = contLen; } else { tDebug("%s, rpc connection is already released", pConn->info); @@ -567,7 +758,6 @@ int rpcReportProgress(void *handle, char *pCont, int contLen) { } void rpcCancelRequest(int64_t rid) { - SRpcReqContext *pContext = taosAcquireRef(tsRpcRefId, rid); if (pContext == NULL) return; @@ -577,7 +767,7 @@ void rpcCancelRequest(int64_t rid) { } static void rpcFreeMsg(void *msg) { - if ( msg ) { + if (msg) { char *temp = (char *)msg - sizeof(SRpcReqContext); free(temp); tTrace("free mem: %p", temp); @@ -589,14 +779,14 @@ static SRpcConn *rpcOpenConn(SRpcInfo *pRpc, char *peerFqdn, uint16_t peerPort, uint32_t peerIp = taosGetIpv4FromFqdn(peerFqdn); if (peerIp == 0xFFFFFFFF) { - tError("%s, failed to resolve FQDN:%s", pRpc->label, peerFqdn); - terrno = TSDB_CODE_RPC_FQDN_ERROR; + tError("%s, failed to resolve FQDN:%s", pRpc->label, peerFqdn); + terrno = TSDB_CODE_RPC_FQDN_ERROR; return NULL; } - pConn = rpcAllocateClientConn(pRpc); + pConn = rpcAllocateClientConn(pRpc); - if (pConn) { + if (pConn) { tstrncpy(pConn->peerFqdn, peerFqdn, sizeof(pConn->peerFqdn)); pConn->peerIp = peerIp; pConn->peerPort = peerPort; @@ -604,7 +794,7 @@ static SRpcConn *rpcOpenConn(SRpcInfo *pRpc, char *peerFqdn, uint16_t peerPort, pConn->connType = connType; if (taosOpenConn[connType]) { - void *shandle = (connType & RPC_CONN_TCP)? pRpc->tcphandle:pRpc->udphandle; + void *shandle = (connType & RPC_CONN_TCP) ? pRpc->tcphandle : pRpc->udphandle; pConn->chandle = (*taosOpenConn[connType])(shandle, pConn, pConn->peerIp, pConn->peerPort); if (pConn->chandle == NULL) { tError("failed to connect to:%s:%d", taosIpStr(pConn->peerIp), pConn->peerPort); @@ -629,13 +819,14 @@ static void rpcReleaseConn(SRpcConn *pConn) { taosTmrStopA(&pConn->pTimer); taosTmrStopA(&pConn->pIdleTimer); - if ( pRpc->connType == TAOS_CONN_SERVER) { - char hashstr[40] = {0}; - size_t size = snprintf(hashstr, sizeof(hashstr), "%x:%x:%x:%d", pConn->peerIp, pConn->linkUid, pConn->peerId, pConn->connType); + if (pRpc->connType == TAOS_CONN_SERVER) { + char hashstr[40] = {0}; + size_t size = snprintf(hashstr, sizeof(hashstr), "%x:%x:%x:%d", pConn->peerIp, pConn->linkUid, pConn->peerId, + pConn->connType); taosHashRemove(pRpc->hash, hashstr, size); - rpcFreeMsg(pConn->pRspMsg); // it may have a response msg saved, but not request msg + rpcFreeMsg(pConn->pRspMsg); // it may have a response msg saved, but not request msg pConn->pRspMsg = NULL; - + // if server has ever reported progress, free content if (pConn->pReqMsg) rpcFreeCont(pConn->pReqMsg); // do not use rpcFreeMsg } else { @@ -643,17 +834,17 @@ static void rpcReleaseConn(SRpcConn *pConn) { if (pConn->outType && pConn->pReqMsg) { SRpcReqContext *pContext = pConn->pContext; if (pContext) { - if (pContext->pRsp) { - // for synchronous API, post semaphore to unblock app + if (pContext->pRsp) { + // for synchronous API, post semaphore to unblock app pContext->pRsp->code = TSDB_CODE_RPC_APP_ERROR; pContext->pRsp->pCont = NULL; pContext->pRsp->contLen = 0; tsem_post(pContext->pSem); } - pContext->pConn = NULL; + pContext->pConn = NULL; taosRemoveRef(tsRpcRefId, pContext->rid); } else { - assert(0); + assert(0); } } } @@ -682,8 +873,7 @@ static void rpcCloseConn(void *thandle) { rpcLockConn(pConn); - if (pConn->user[0]) - rpcReleaseConn(pConn); + if (pConn->user[0]) rpcReleaseConn(pConn); rpcUnlockConn(pConn); } @@ -717,8 +907,9 @@ static SRpcConn *rpcAllocateServerConn(SRpcInfo *pRpc, SRecvInfo *pRecv) { char hashstr[40] = {0}; SRpcHead *pHead = (SRpcHead *)pRecv->msg; - size_t size = snprintf(hashstr, sizeof(hashstr), "%x:%x:%x:%d", pRecv->ip, pHead->linkUid, pHead->sourceId, pRecv->connType); - + size_t size = + snprintf(hashstr, sizeof(hashstr), "%x:%x:%x:%d", pRecv->ip, pHead->linkUid, pHead->sourceId, pRecv->connType); + // check if it is already allocated SRpcConn **ppConn = (SRpcConn **)(taosHashGet(pRpc->hash, hashstr, size)); if (ppConn) pConn = *ppConn; @@ -767,22 +958,23 @@ static SRpcConn *rpcAllocateServerConn(SRpcInfo *pRpc, SRecvInfo *pRecv) { } taosHashPut(pRpc->hash, hashstr, size, (char *)&pConn, POINTER_BYTES); - tDebug("%s %p server connection is allocated, uid:0x%x sid:%d key:%s", pRpc->label, pConn, pConn->linkUid, sid, hashstr); + tDebug("%s %p server connection is allocated, uid:0x%x sid:%d key:%s", pRpc->label, pConn, pConn->linkUid, sid, + hashstr); } return pConn; } static SRpcConn *rpcGetConnObj(SRpcInfo *pRpc, int sid, SRecvInfo *pRecv) { - SRpcConn *pConn = NULL; + SRpcConn *pConn = NULL; SRpcHead *pHead = (SRpcHead *)pRecv->msg; if (sid) { pConn = pRpc->connList + sid; if (pConn->user[0] == 0) pConn = NULL; - } + } - if (pConn == NULL) { + if (pConn == NULL) { if (pRpc->connType == TAOS_CONN_SERVER) { pConn = rpcAllocateServerConn(pRpc, pRecv); } else { @@ -805,12 +997,13 @@ static SRpcConn *rpcGetConnObj(SRpcInfo *pRpc, int sid, SRecvInfo *pRecv) { static SRpcConn *rpcSetupConnToServer(SRpcReqContext *pContext) { SRpcConn *pConn; SRpcInfo *pRpc = pContext->pRpc; - SEpSet *pEpSet = &pContext->epSet; + SEpSet * pEpSet = &pContext->epSet; - pConn = rpcGetConnFromCache(pRpc->pCache, pEpSet->fqdn[pEpSet->inUse], pEpSet->port[pEpSet->inUse], pContext->connType); - if ( pConn == NULL || pConn->user[0] == 0) { + pConn = + rpcGetConnFromCache(pRpc->pCache, pEpSet->fqdn[pEpSet->inUse], pEpSet->port[pEpSet->inUse], pContext->connType); + if (pConn == NULL || pConn->user[0] == 0) { pConn = rpcOpenConn(pRpc, pEpSet->fqdn[pEpSet->inUse], pEpSet->port[pEpSet->inUse], pContext->connType); - } + } if (pConn) { pConn->tretry = 0; @@ -825,55 +1018,52 @@ static SRpcConn *rpcSetupConnToServer(SRpcReqContext *pContext) { } static int rpcProcessReqHead(SRpcConn *pConn, SRpcHead *pHead) { - - if (pConn->peerId == 0) { - pConn->peerId = pHead->sourceId; - } else { - if (pConn->peerId != pHead->sourceId) { - tDebug("%s, source Id is changed, old:0x%08x new:0x%08x", pConn->info, - pConn->peerId, pHead->sourceId); - return TSDB_CODE_RPC_INVALID_VALUE; - } + if (pConn->peerId == 0) { + pConn->peerId = pHead->sourceId; + } else { + if (pConn->peerId != pHead->sourceId) { + tDebug("%s, source Id is changed, old:0x%08x new:0x%08x", pConn->info, pConn->peerId, pHead->sourceId); + return TSDB_CODE_RPC_INVALID_VALUE; } + } - if (pConn->inTranId == pHead->tranId) { - if (pConn->inType == pHead->msgType) { - if (pHead->code == 0) { - tDebug("%s, %s is retransmitted", pConn->info, TMSG_INFO(pHead->msgType)); - rpcSendQuickRsp(pConn, TSDB_CODE_RPC_ACTION_IN_PROGRESS); - } else { - // do nothing, it is heart beat from client - } - } else if (pConn->inType == 0) { - tDebug("%s, %s is already processed, tranId:%d", pConn->info, TMSG_INFO(pHead->msgType), pConn->inTranId); - rpcSendMsgToPeer(pConn, pConn->pRspMsg, pConn->rspMsgLen); // resend the response + if (pConn->inTranId == pHead->tranId) { + if (pConn->inType == pHead->msgType) { + if (pHead->code == 0) { + tDebug("%s, %s is retransmitted", pConn->info, TMSG_INFO(pHead->msgType)); + rpcSendQuickRsp(pConn, TSDB_CODE_RPC_ACTION_IN_PROGRESS); } else { - tDebug("%s, mismatched message %s and tranId", pConn->info, TMSG_INFO(pHead->msgType)); + // do nothing, it is heart beat from client } - - // do not reply any message - return TSDB_CODE_RPC_ALREADY_PROCESSED; + } else if (pConn->inType == 0) { + tDebug("%s, %s is already processed, tranId:%d", pConn->info, TMSG_INFO(pHead->msgType), pConn->inTranId); + rpcSendMsgToPeer(pConn, pConn->pRspMsg, pConn->rspMsgLen); // resend the response + } else { + tDebug("%s, mismatched message %s and tranId", pConn->info, TMSG_INFO(pHead->msgType)); } - if (pConn->inType != 0) { - tDebug("%s, last session is not finished, inTranId:%d tranId:%d", pConn->info, - pConn->inTranId, pHead->tranId); - return TSDB_CODE_RPC_LAST_SESSION_NOT_FINISHED; - } + // do not reply any message + return TSDB_CODE_RPC_ALREADY_PROCESSED; + } - if (rpcContLenFromMsg(pHead->msgLen) <= 0) { - tDebug("%s, message body is empty, ignore", pConn->info); - return TSDB_CODE_RPC_APP_ERROR; - } + if (pConn->inType != 0) { + tDebug("%s, last session is not finished, inTranId:%d tranId:%d", pConn->info, pConn->inTranId, pHead->tranId); + return TSDB_CODE_RPC_LAST_SESSION_NOT_FINISHED; + } - pConn->inTranId = pHead->tranId; - pConn->inType = pHead->msgType; + if (rpcContLenFromMsg(pHead->msgLen) <= 0) { + tDebug("%s, message body is empty, ignore", pConn->info); + return TSDB_CODE_RPC_APP_ERROR; + } - // start the progress timer to monitor the response from server app - if (pConn->connType != RPC_CONN_TCPS) - pConn->pTimer = taosTmrStart(rpcProcessProgressTimer, tsProgressTimer, pConn, pConn->pRpc->tmrCtrl); - - return 0; + pConn->inTranId = pHead->tranId; + pConn->inType = pHead->msgType; + + // start the progress timer to monitor the response from server app + if (pConn->connType != RPC_CONN_TCPS) + pConn->pTimer = taosTmrStart(rpcProcessProgressTimer, tsProgressTimer, pConn, pConn->pRpc->tmrCtrl); + + return 0; } static int rpcProcessRspHead(SRpcConn *pConn, SRpcHead *pHead) { @@ -898,7 +1088,7 @@ static int rpcProcessRspHead(SRpcConn *pConn, SRpcHead *pHead) { if (pHead->code == TSDB_CODE_RPC_AUTH_REQUIRED && pRpc->spi) { tDebug("%s, authentication shall be restarted", pConn->info); pConn->secured = 0; - rpcSendMsgToPeer(pConn, pConn->pReqMsg, pConn->reqMsgLen); + rpcSendMsgToPeer(pConn, pConn->pReqMsg, pConn->reqMsgLen); if (pConn->connType != RPC_CONN_TCPC) pConn->pTimer = taosTmrStart(rpcProcessRetryTimer, tsRpcTimer, pConn, pRpc->tmrCtrl); return TSDB_CODE_RPC_ALREADY_PROCESSED; @@ -908,7 +1098,7 @@ static int rpcProcessRspHead(SRpcConn *pConn, SRpcHead *pHead) { tDebug("%s, mismatched linkUid, link shall be restarted", pConn->info); pConn->secured = 0; ((SRpcHead *)pConn->pReqMsg)->destId = 0; - rpcSendMsgToPeer(pConn, pConn->pReqMsg, pConn->reqMsgLen); + rpcSendMsgToPeer(pConn, pConn->pReqMsg, pConn->reqMsgLen); if (pConn->connType != RPC_CONN_TCPC) pConn->pTimer = taosTmrStart(rpcProcessRetryTimer, tsRpcTimer, pConn, pRpc->tmrCtrl); return TSDB_CODE_RPC_ALREADY_PROCESSED; @@ -934,25 +1124,25 @@ static int rpcProcessRspHead(SRpcConn *pConn, SRpcHead *pHead) { pConn->reqMsgLen = 0; SRpcReqContext *pContext = pConn->pContext; - if (pHead->code == TSDB_CODE_RPC_REDIRECT) { + if (pHead->code == TSDB_CODE_RPC_REDIRECT) { if (rpcContLenFromMsg(pHead->msgLen) < sizeof(SEpSet)) { // if EpSet is not included in the msg, treat it as NOT_READY - pHead->code = TSDB_CODE_RPC_NOT_READY; + pHead->code = TSDB_CODE_RPC_NOT_READY; } else { pContext->redirect++; if (pContext->redirect > TSDB_MAX_REPLICA) { - pHead->code = TSDB_CODE_RPC_NETWORK_UNAVAIL; + pHead->code = TSDB_CODE_RPC_NETWORK_UNAVAIL; tWarn("%s, too many redirects, quit", pConn->info); } } - } + } return TSDB_CODE_SUCCESS; } static SRpcConn *rpcProcessMsgHead(SRpcInfo *pRpc, SRecvInfo *pRecv, SRpcReqContext **ppContext) { - int32_t sid; - SRpcConn *pConn = NULL; + int32_t sid; + SRpcConn *pConn = NULL; SRpcHead *pHead = (SRpcHead *)pRecv->msg; @@ -961,25 +1151,29 @@ static SRpcConn *rpcProcessMsgHead(SRpcInfo *pRpc, SRecvInfo *pRecv, SRpcReqCont if (TMSG_INDEX(pHead->msgType) >= TDMT_MAX || TMSG_INDEX(pHead->msgType) <= 0) { tDebug("%s sid:%d, invalid message type:%d", pRpc->label, sid, pHead->msgType); - terrno = TSDB_CODE_RPC_INVALID_MSG_TYPE; return NULL; + terrno = TSDB_CODE_RPC_INVALID_MSG_TYPE; + return NULL; } if (sid < 0 || sid >= pRpc->sessions) { - tDebug("%s sid:%d, sid is out of range, max sid:%d, %s discarded", pRpc->label, sid, - pRpc->sessions, TMSG_INFO(pHead->msgType)); - terrno = TSDB_CODE_RPC_INVALID_SESSION_ID; return NULL; + tDebug("%s sid:%d, sid is out of range, max sid:%d, %s discarded", pRpc->label, sid, pRpc->sessions, + TMSG_INFO(pHead->msgType)); + terrno = TSDB_CODE_RPC_INVALID_SESSION_ID; + return NULL; } if (rpcIsReq(pHead->msgType) && htonl(pHead->msgVer) != tsVersion >> 8) { - tDebug("%s sid:%d, invalid client version:%x/%x %s", pRpc->label, sid, htonl(pHead->msgVer), tsVersion, TMSG_INFO(pHead->msgType)); - terrno = TSDB_CODE_RPC_INVALID_VERSION; return NULL; + tDebug("%s sid:%d, invalid client version:%x/%x %s", pRpc->label, sid, htonl(pHead->msgVer), tsVersion, + TMSG_INFO(pHead->msgType)); + terrno = TSDB_CODE_RPC_INVALID_VERSION; + return NULL; } pConn = rpcGetConnObj(pRpc, sid, pRecv); if (pConn == NULL) { - tDebug("%s %p, failed to get connection obj(%s)", pRpc->label, (void *)pHead->ahandle, tstrerror(terrno)); + tDebug("%s %p, failed to get connection obj(%s)", pRpc->label, (void *)pHead->ahandle, tstrerror(terrno)); return NULL; - } + } rpcLockConn(pConn); @@ -990,9 +1184,9 @@ static SRpcConn *rpcProcessMsgHead(SRpcInfo *pRpc, SRecvInfo *pRecv, SRpcReqCont sid = pConn->sid; if (pConn->chandle == NULL) pConn->chandle = pRecv->chandle; - pConn->peerIp = pRecv->ip; + pConn->peerIp = pRecv->ip; pConn->peerPort = pRecv->port; - if (pHead->port) pConn->peerPort = htons(pHead->port); + if (pHead->port) pConn->peerPort = htons(pHead->port); terrno = rpcCheckAuthentication(pConn, (char *)pHead, pRecv->msgLen); @@ -1004,16 +1198,16 @@ static SRpcConn *rpcProcessMsgHead(SRpcInfo *pRpc, SRecvInfo *pRecv, SRpcReqCont // decrypt here } - if ( rpcIsReq(pHead->msgType) ) { + if (rpcIsReq(pHead->msgType)) { pConn->connType = pRecv->connType; terrno = rpcProcessReqHead(pConn, pHead); // stop idle timer - taosTmrStopA(&pConn->pIdleTimer); + taosTmrStopA(&pConn->pIdleTimer); - // client shall send the request within tsRpcTime again for UDP, double it + // client shall send the request within tsRpcTime again for UDP, double it if (pConn->connType != RPC_CONN_TCPS) - pConn->pIdleTimer = taosTmrStart(rpcProcessIdleTimer, tsRpcTimer*2, pConn, pRpc->tmrCtrl); + pConn->pIdleTimer = taosTmrStart(rpcProcessIdleTimer, tsRpcTimer * 2, pConn, pRpc->tmrCtrl); } else { terrno = rpcProcessRspHead(pConn, pHead); *ppContext = pConn->pContext; @@ -1026,9 +1220,9 @@ static SRpcConn *rpcProcessMsgHead(SRpcInfo *pRpc, SRecvInfo *pRecv, SRpcReqCont } static void doRpcReportBrokenLinkToServer(void *param, void *id) { - SRpcMsg *pRpcMsg = (SRpcMsg *)(param); - SRpcConn *pConn = (SRpcConn *)(pRpcMsg->handle); - SRpcInfo *pRpc = pConn->pRpc; + SRpcMsg * pRpcMsg = (SRpcMsg *)(param); + SRpcConn *pConn = (SRpcConn *)(pRpcMsg->handle); + SRpcInfo *pRpc = pConn->pRpc; (*(pRpc->cfp))(pRpc->parent, pRpcMsg, NULL); free(pRpcMsg); } @@ -1041,12 +1235,12 @@ static void rpcReportBrokenLinkToServer(SRpcConn *pConn) { tDebug("%s, notify the server app, connection is gone", pConn->info); SRpcMsg *rpcMsg = malloc(sizeof(SRpcMsg)); - rpcMsg->pCont = pConn->pReqMsg; // pReqMsg is re-used to store the APP context from server - rpcMsg->contLen = pConn->reqMsgLen; // reqMsgLen is re-used to store the APP context length + rpcMsg->pCont = pConn->pReqMsg; // pReqMsg is re-used to store the APP context from server + rpcMsg->contLen = pConn->reqMsgLen; // reqMsgLen is re-used to store the APP context length rpcMsg->ahandle = pConn->ahandle; rpcMsg->handle = pConn; rpcMsg->msgType = pConn->inType; - rpcMsg->code = TSDB_CODE_RPC_NETWORK_UNAVAIL; + rpcMsg->code = TSDB_CODE_RPC_NETWORK_UNAVAIL; pConn->pReqMsg = NULL; pConn->reqMsgLen = 0; if (pRpc->cfp) { @@ -1070,22 +1264,22 @@ static void rpcProcessBrokenLink(SRpcConn *pConn) { pConn->pReqMsg = NULL; taosTmrStart(rpcProcessConnError, 0, pContext, pRpc->tmrCtrl); } - - if (pConn->inType) rpcReportBrokenLinkToServer(pConn); + + if (pConn->inType) rpcReportBrokenLinkToServer(pConn); rpcReleaseConn(pConn); rpcUnlockConn(pConn); } static void *rpcProcessMsgFromPeer(SRecvInfo *pRecv) { - SRpcHead *pHead = (SRpcHead *)pRecv->msg; - SRpcInfo *pRpc = (SRpcInfo *)pRecv->shandle; - SRpcConn *pConn = (SRpcConn *)pRecv->thandle; + SRpcHead *pHead = (SRpcHead *)pRecv->msg; + SRpcInfo *pRpc = (SRpcInfo *)pRecv->shandle; + SRpcConn *pConn = (SRpcConn *)pRecv->thandle; tDump(pRecv->msg, pRecv->msgLen); // underlying UDP layer does not know it is server or client - pRecv->connType = pRecv->connType | pRpc->connType; + pRecv->connType = pRecv->connType | pRpc->connType; if (pRecv->msg == NULL) { rpcProcessBrokenLink(pConn); @@ -1100,62 +1294,62 @@ static void *rpcProcessMsgFromPeer(SRecvInfo *pRecv) { taosIpPort2String(pRecv->ip, pRecv->port, ipstr); if (TMSG_INDEX(pHead->msgType) >= 1 && TMSG_INDEX(pHead->msgType) < TDMT_MAX) { - tDebug("%s %p %p, %s received from %s, parse code:0x%x len:%d sig:0x%08x:0x%08x:%d code:0x%x", pRpc->label, - pConn, (void *)pHead->ahandle, TMSG_INFO(pHead->msgType), ipstr, terrno, pRecv->msgLen, - pHead->sourceId, pHead->destId, pHead->tranId, pHead->code); + tDebug("%s %p %p, %s received from %s, parse code:0x%x len:%d sig:0x%08x:0x%08x:%d code:0x%x", pRpc->label, pConn, + (void *)pHead->ahandle, TMSG_INFO(pHead->msgType), ipstr, terrno, pRecv->msgLen, pHead->sourceId, + pHead->destId, pHead->tranId, pHead->code); } else { - tDebug("%s %p %p, %d received from %s, parse code:0x%x len:%d sig:0x%08x:0x%08x:%d code:0x%x", pRpc->label, - pConn, (void *)pHead->ahandle, pHead->msgType, ipstr, terrno, pRecv->msgLen, - pHead->sourceId, pHead->destId, pHead->tranId, pHead->code); + tDebug("%s %p %p, %d received from %s, parse code:0x%x len:%d sig:0x%08x:0x%08x:%d code:0x%x", pRpc->label, pConn, + (void *)pHead->ahandle, pHead->msgType, ipstr, terrno, pRecv->msgLen, pHead->sourceId, pHead->destId, + pHead->tranId, pHead->code); } int32_t code = terrno; if (code != TSDB_CODE_RPC_ALREADY_PROCESSED) { - if (code != 0) { // parsing error + if (code != 0) { // parsing error if (rpcIsReq(pHead->msgType)) { rpcSendErrorMsgToPeer(pRecv, code); if (code == TSDB_CODE_RPC_INVALID_TIME_STAMP || code == TSDB_CODE_RPC_AUTH_FAILURE) { rpcCloseConn(pConn); } if (TMSG_INDEX(pHead->msgType) + 1 > 1 && TMSG_INDEX(pHead->msgType) + 1 < TDMT_MAX) { - tDebug("%s %p %p, %s is sent with error code:0x%x", pRpc->label, pConn, (void *)pHead->ahandle, TMSG_INFO(pHead->msgType+1), code); + tDebug("%s %p %p, %s is sent with error code:0x%x", pRpc->label, pConn, (void *)pHead->ahandle, + TMSG_INFO(pHead->msgType + 1), code); } else { - tError("%s %p %p, %s is sent with error code:0x%x", pRpc->label, pConn, (void *)pHead->ahandle, TMSG_INFO(pHead->msgType), code); + tError("%s %p %p, %s is sent with error code:0x%x", pRpc->label, pConn, (void *)pHead->ahandle, + TMSG_INFO(pHead->msgType), code); } - } - } else { // msg is passed to app only parsing is ok + } + } else { // msg is passed to app only parsing is ok rpcProcessIncomingMsg(pConn, pHead, pContext); } } - if (code) rpcFreeMsg(pRecv->msg); // parsing failed, msg shall be freed + if (code) rpcFreeMsg(pRecv->msg); // parsing failed, msg shall be freed return pConn; } static void rpcNotifyClient(SRpcReqContext *pContext, SRpcMsg *pMsg) { - SRpcInfo *pRpc = pContext->pRpc; + SRpcInfo *pRpc = pContext->pRpc; pContext->pConn = NULL; - if (pContext->pRsp) { + if (pContext->pRsp) { // for synchronous API memcpy(pContext->pSet, &pContext->epSet, sizeof(SEpSet)); memcpy(pContext->pRsp, pMsg, sizeof(SRpcMsg)); tsem_post(pContext->pSem); } else { - // for asynchronous API + // for asynchronous API SEpSet *pEpSet = NULL; - if (pContext->epSet.inUse != pContext->oldInUse || pContext->redirect) - pEpSet = &pContext->epSet; + if (pContext->epSet.inUse != pContext->oldInUse || pContext->redirect) pEpSet = &pContext->epSet; (*pRpc->cfp)(pRpc->parent, pMsg, pEpSet); } // free the request message - taosRemoveRef(tsRpcRefId, pContext->rid); + taosRemoveRef(tsRpcRefId, pContext->rid); } static void rpcProcessIncomingMsg(SRpcConn *pConn, SRpcHead *pHead, SRpcReqContext *pContext) { - SRpcInfo *pRpc = pConn->pRpc; SRpcMsg rpcMsg; @@ -1180,14 +1374,15 @@ static void rpcProcessIncomingMsg(SRpcConn *pConn, SRpcHead *pHead, SRpcReqConte // for UDP, port may be changed by server, the port in epSet shall be used for cache if (pHead->code != TSDB_CODE_RPC_TOO_SLOW) { - rpcAddConnIntoCache(pRpc->pCache, pConn, pConn->peerFqdn, pContext->epSet.port[pContext->epSet.inUse], pConn->connType); + rpcAddConnIntoCache(pRpc->pCache, pConn, pConn->peerFqdn, pContext->epSet.port[pContext->epSet.inUse], + pConn->connType); } else { rpcCloseConn(pConn); } if (pHead->code == TSDB_CODE_RPC_REDIRECT) { pContext->numOfTry = 0; - SEpSet *pEpSet = (SEpSet*)pHead->content; + SEpSet *pEpSet = (SEpSet *)pHead->content; if (pEpSet->numOfEps > 0) { memcpy(&pContext->epSet, pHead->content, sizeof(pContext->epSet)); tDebug("%s, redirect is received, numOfEps:%d inUse:%d", pConn->info, pContext->epSet.numOfEps, @@ -1200,7 +1395,8 @@ static void rpcProcessIncomingMsg(SRpcConn *pConn, SRpcHead *pHead, SRpcReqConte } rpcSendReqToServer(pRpc, pContext); rpcFreeCont(rpcMsg.pCont); - } else if (pHead->code == TSDB_CODE_RPC_NOT_READY || pHead->code == TSDB_CODE_APP_NOT_READY || pHead->code == TSDB_CODE_DND_OFFLINE) { + } else if (pHead->code == TSDB_CODE_RPC_NOT_READY || pHead->code == TSDB_CODE_APP_NOT_READY || + pHead->code == TSDB_CODE_DND_OFFLINE) { pContext->code = pHead->code; rpcProcessConnError(pContext, NULL); rpcFreeCont(rpcMsg.pCont); @@ -1211,14 +1407,14 @@ static void rpcProcessIncomingMsg(SRpcConn *pConn, SRpcHead *pHead, SRpcReqConte } static void rpcSendQuickRsp(SRpcConn *pConn, int32_t code) { - char msg[RPC_MSG_OVERHEAD]; - SRpcHead *pHead; + char msg[RPC_MSG_OVERHEAD]; + SRpcHead *pHead; // set msg header memset(msg, 0, sizeof(SRpcHead)); pHead = (SRpcHead *)msg; pHead->version = 1; - pHead->msgType = pConn->inType+1; + pHead->msgType = pConn->inType + 1; pHead->spi = pConn->spi; pHead->encrypt = 0; pHead->tranId = pConn->inTranId; @@ -1230,12 +1426,12 @@ static void rpcSendQuickRsp(SRpcConn *pConn, int32_t code) { pHead->code = htonl(code); rpcSendMsgToPeer(pConn, msg, sizeof(SRpcHead)); - pConn->secured = 1; // connection shall be secured + pConn->secured = 1; // connection shall be secured } static void rpcSendReqHead(SRpcConn *pConn) { - char msg[RPC_MSG_OVERHEAD]; - SRpcHead *pHead; + char msg[RPC_MSG_OVERHEAD]; + SRpcHead *pHead; // set msg header memset(msg, 0, sizeof(SRpcHead)); @@ -1257,10 +1453,10 @@ static void rpcSendReqHead(SRpcConn *pConn) { } static void rpcSendErrorMsgToPeer(SRecvInfo *pRecv, int32_t code) { - SRpcHead *pRecvHead, *pReplyHead; - char msg[sizeof(SRpcHead) + sizeof(SRpcDigest) + sizeof(uint32_t) ]; - uint32_t timeStamp; - int msgLen; + SRpcHead *pRecvHead, *pReplyHead; + char msg[sizeof(SRpcHead) + sizeof(SRpcDigest) + sizeof(uint32_t)]; + uint32_t timeStamp; + int msgLen; pRecvHead = (SRpcHead *)pRecv->msg; pReplyHead = (SRpcHead *)msg; @@ -1290,14 +1486,14 @@ static void rpcSendErrorMsgToPeer(SRecvInfo *pRecv, int32_t code) { pReplyHead->msgLen = (int32_t)htonl((uint32_t)msgLen); (*taosSendData[pRecv->connType])(pRecv->ip, pRecv->port, msg, msgLen, pRecv->chandle); - return; + return; } static void rpcSendReqToServer(SRpcInfo *pRpc, SRpcReqContext *pContext) { - SRpcHead *pHead = rpcHeadFromCont(pContext->pCont); - char *msg = (char *)pHead; - int msgLen = rpcMsgLenFromCont(pContext->contLen); - tmsg_t msgType = pContext->msgType; + SRpcHead *pHead = rpcHeadFromCont(pContext->pCont); + char * msg = (char *)pHead; + int msgLen = rpcMsgLenFromCont(pContext->contLen); + tmsg_t msgType = pContext->msgType; pContext->numOfTry++; SRpcConn *pConn = rpcSetupConnToServer(pContext); @@ -1311,13 +1507,13 @@ static void rpcSendReqToServer(SRpcInfo *pRpc, SRpcReqContext *pContext) { pConn->ahandle = pContext->ahandle; rpcLockConn(pConn); - // set the message header + // set the message header pHead->version = 1; pHead->msgVer = htonl(tsVersion >> 8); pHead->msgType = msgType; pHead->encrypt = 0; pConn->tranId++; - if ( pConn->tranId == 0 ) pConn->tranId++; + if (pConn->tranId == 0) pConn->tranId++; pHead->tranId = pConn->tranId; pHead->sourceId = pConn->ownId; pHead->destId = pConn->peerId; @@ -1341,45 +1537,43 @@ static void rpcSendReqToServer(SRpcInfo *pRpc, SRpcReqContext *pContext) { } static void rpcSendMsgToPeer(SRpcConn *pConn, void *msg, int msgLen) { - int writtenLen = 0; - SRpcHead *pHead = (SRpcHead *)msg; + int writtenLen = 0; + SRpcHead *pHead = (SRpcHead *)msg; msgLen = rpcAddAuthPart(pConn, msg, msgLen); - if ( rpcIsReq(pHead->msgType)) { - tDebug("%s, %s is sent to %s:%hu, len:%d sig:0x%08x:0x%08x:%d", - pConn->info, TMSG_INFO(pHead->msgType), pConn->peerFqdn, pConn->peerPort, - msgLen, pHead->sourceId, pHead->destId, pHead->tranId); + if (rpcIsReq(pHead->msgType)) { + tDebug("%s, %s is sent to %s:%hu, len:%d sig:0x%08x:0x%08x:%d", pConn->info, TMSG_INFO(pHead->msgType), + pConn->peerFqdn, pConn->peerPort, msgLen, pHead->sourceId, pHead->destId, pHead->tranId); } else { - if (pHead->code == 0) pConn->secured = 1; // for success response, set link as secured - tDebug("%s, %s is sent to 0x%x:%hu, code:0x%x len:%d sig:0x%08x:0x%08x:%d", - pConn->info, TMSG_INFO(pHead->msgType), pConn->peerIp, pConn->peerPort, - htonl(pHead->code), msgLen, pHead->sourceId, pHead->destId, pHead->tranId); + if (pHead->code == 0) pConn->secured = 1; // for success response, set link as secured + tDebug("%s, %s is sent to 0x%x:%hu, code:0x%x len:%d sig:0x%08x:0x%08x:%d", pConn->info, TMSG_INFO(pHead->msgType), + pConn->peerIp, pConn->peerPort, htonl(pHead->code), msgLen, pHead->sourceId, pHead->destId, pHead->tranId); } - //tTrace("connection type is: %d", pConn->connType); + // tTrace("connection type is: %d", pConn->connType); writtenLen = (*taosSendData[pConn->connType])(pConn->peerIp, pConn->peerPort, pHead, msgLen, pConn->chandle); if (writtenLen != msgLen) { tError("%s, failed to send, msgLen:%d written:%d, reason:%s", pConn->info, msgLen, writtenLen, strerror(errno)); } - + tDump(msg, msgLen); } static void rpcProcessConnError(void *param, void *id) { SRpcReqContext *pContext = (SRpcReqContext *)param; - SRpcInfo *pRpc = pContext->pRpc; + SRpcInfo * pRpc = pContext->pRpc; SRpcMsg rpcMsg; - + if (pRpc == NULL) { return; } - + tDebug("%s %p, connection error happens", pRpc->label, pContext->ahandle); if (pContext->numOfTry >= pContext->epSet.numOfEps || pContext->msgType == TDMT_VND_FETCH) { - rpcMsg.msgType = pContext->msgType+1; + rpcMsg.msgType = pContext->msgType + 1; rpcMsg.ahandle = pContext->ahandle; rpcMsg.code = pContext->code; rpcMsg.pCont = NULL; @@ -1387,7 +1581,7 @@ static void rpcProcessConnError(void *param, void *id) { rpcNotifyClient(pContext, &rpcMsg); } else { - // move to next IP + // move to next IP pContext->epSet.inUse++; pContext->epSet.inUse = pContext->epSet.inUse % pContext->epSet.numOfEps; rpcSendReqToServer(pRpc, pContext); @@ -1407,11 +1601,12 @@ static void rpcProcessRetryTimer(void *param, void *tmrId) { if (pConn->retry < 4) { tDebug("%s, re-send msg:%s to %s:%hu", pConn->info, TMSG_INFO(pConn->outType), pConn->peerFqdn, pConn->peerPort); - rpcSendMsgToPeer(pConn, pConn->pReqMsg, pConn->reqMsgLen); + rpcSendMsgToPeer(pConn, pConn->pReqMsg, pConn->reqMsgLen); pConn->pTimer = taosTmrStart(rpcProcessRetryTimer, tsRpcTimer, pConn, pRpc->tmrCtrl); } else { // close the connection - tDebug("%s, failed to send msg:%s to %s:%hu", pConn->info, TMSG_INFO(pConn->outType), pConn->peerFqdn, pConn->peerPort); + tDebug("%s, failed to send msg:%s to %s:%hu", pConn->info, TMSG_INFO(pConn->outType), pConn->peerFqdn, + pConn->peerPort); if (pConn->pContext) { pConn->pContext->code = TSDB_CODE_RPC_NETWORK_UNAVAIL; pConn->pContext->pConn = NULL; @@ -1434,7 +1629,7 @@ static void rpcProcessIdleTimer(void *param, void *tmrId) { if (pConn->user[0]) { tDebug("%s, close the connection since no activity", pConn->info); - if (pConn->inType) rpcReportBrokenLinkToServer(pConn); + if (pConn->inType) rpcReportBrokenLinkToServer(pConn); rpcReleaseConn(pConn); } else { tDebug("%s, idle timer:%p not processed", pConn->info, tmrId); @@ -1460,34 +1655,34 @@ static void rpcProcessProgressTimer(void *param, void *tmrId) { rpcUnlockConn(pConn); } -static int32_t rpcCompressRpcMsg(char* pCont, int32_t contLen) { - SRpcHead *pHead = rpcHeadFromCont(pCont); - int32_t finalLen = 0; - int overhead = sizeof(SRpcComp); - +static int32_t rpcCompressRpcMsg(char *pCont, int32_t contLen) { + SRpcHead *pHead = rpcHeadFromCont(pCont); + int32_t finalLen = 0; + int overhead = sizeof(SRpcComp); + if (!NEEDTO_COMPRESSS_MSG(contLen)) { return contLen; } - - char *buf = malloc (contLen + overhead + 8); // 8 extra bytes + + char *buf = malloc(contLen + overhead + 8); // 8 extra bytes if (buf == NULL) { tError("failed to allocate memory for rpc msg compression, contLen:%d", contLen); return contLen; } - + int32_t compLen = LZ4_compress_default(pCont, buf, contLen, contLen + overhead); tDebug("compress rpc msg, before:%d, after:%d, overhead:%d", contLen, compLen, overhead); - + /* * only the compressed size is less than the value of contLen - overhead, the compression is applied * The first four bytes is set to 0, the second four bytes are utilized to keep the original length of message */ if (compLen > 0 && compLen < contLen - overhead) { SRpcComp *pComp = (SRpcComp *)pCont; - pComp->reserved = 0; - pComp->contLen = htonl(contLen); + pComp->reserved = 0; + pComp->contLen = htonl(contLen); memcpy(pCont + overhead, buf, compLen); - + pHead->comp = 1; tDebug("compress rpc msg, before:%d, after:%d", contLen, compLen); finalLen = compLen + overhead; @@ -1500,29 +1695,29 @@ static int32_t rpcCompressRpcMsg(char* pCont, int32_t contLen) { } static SRpcHead *rpcDecompressRpcMsg(SRpcHead *pHead) { - int overhead = sizeof(SRpcComp); - SRpcHead *pNewHead = NULL; - uint8_t *pCont = pHead->content; - SRpcComp *pComp = (SRpcComp *)pHead->content; + int overhead = sizeof(SRpcComp); + SRpcHead *pNewHead = NULL; + uint8_t * pCont = pHead->content; + SRpcComp *pComp = (SRpcComp *)pHead->content; if (pHead->comp) { // decompress the content assert(pComp->reserved == 0); int contLen = htonl(pComp->contLen); - + // prepare the temporary buffer to decompress message char *temp = (char *)malloc(contLen + RPC_MSG_OVERHEAD); - pNewHead = (SRpcHead *)(temp + sizeof(SRpcReqContext)); // reserve SRpcReqContext - + pNewHead = (SRpcHead *)(temp + sizeof(SRpcReqContext)); // reserve SRpcReqContext + if (pNewHead) { int compLen = rpcContLenFromMsg(pHead->msgLen) - overhead; - int origLen = LZ4_decompress_safe((char*)(pCont + overhead), (char *)pNewHead->content, compLen, contLen); + int origLen = LZ4_decompress_safe((char *)(pCont + overhead), (char *)pNewHead->content, compLen, contLen); assert(origLen == contLen); - + memcpy(pNewHead, pHead, sizeof(SRpcHead)); pNewHead->msgLen = rpcMsgLenFromCont(origLen); - rpcFreeMsg(pHead); // free the compressed message buffer - pHead = pNewHead; + rpcFreeMsg(pHead); // free the compressed message buffer + pHead = pNewHead; tTrace("decomp malloc mem:%p", temp); } else { tError("failed to allocate memory to decompress msg, contLen:%d", contLen); @@ -1534,7 +1729,7 @@ static SRpcHead *rpcDecompressRpcMsg(SRpcHead *pHead) { static int rpcAuthenticateMsg(void *pMsg, int msgLen, void *pAuth, void *pKey) { T_MD5_CTX context; - int ret = -1; + int ret = -1; tMD5Init(&context); tMD5Update(&context, (uint8_t *)pKey, TSDB_PASSWORD_LEN); @@ -1582,25 +1777,25 @@ static int rpcCheckAuthentication(SRpcConn *pConn, char *msg, int msgLen) { SRpcHead *pHead = (SRpcHead *)msg; int code = 0; - if ((pConn->secured && pHead->spi == 0) || (pHead->spi == 0 && pConn->spi == 0)){ - // secured link, or no authentication + if ((pConn->secured && pHead->spi == 0) || (pHead->spi == 0 && pConn->spi == 0)) { + // secured link, or no authentication pHead->msgLen = (int32_t)htonl((uint32_t)pHead->msgLen); // tTrace("%s, secured link, no auth is required", pConn->info); return 0; } - if ( !rpcIsReq(pHead->msgType) ) { + if (!rpcIsReq(pHead->msgType)) { // for response, if code is auth failure, it shall bypass the auth process code = htonl(pHead->code); if (code == TSDB_CODE_RPC_INVALID_TIME_STAMP || code == TSDB_CODE_RPC_AUTH_FAILURE || - code == TSDB_CODE_RPC_INVALID_VERSION || - code == TSDB_CODE_RPC_AUTH_REQUIRED || code == TSDB_CODE_MND_USER_NOT_EXIST || code == TSDB_CODE_RPC_NOT_READY) { + code == TSDB_CODE_RPC_INVALID_VERSION || code == TSDB_CODE_RPC_AUTH_REQUIRED || + code == TSDB_CODE_MND_USER_NOT_EXIST || code == TSDB_CODE_RPC_NOT_READY) { pHead->msgLen = (int32_t)htonl((uint32_t)pHead->msgLen); // tTrace("%s, dont check authentication since code is:0x%x", pConn->info, code); return 0; } } - + code = 0; if (pHead->spi == pConn->spi) { // authentication @@ -1613,12 +1808,12 @@ static int rpcCheckAuthentication(SRpcConn *pConn, char *msg, int msgLen) { tWarn("%s, time diff:%d is too big, msg discarded", pConn->info, delta); code = TSDB_CODE_RPC_INVALID_TIME_STAMP; } else { - if (rpcAuthenticateMsg(pHead, msgLen-TSDB_AUTH_LEN, pDigest->auth, pConn->secret) < 0) { + if (rpcAuthenticateMsg(pHead, msgLen - TSDB_AUTH_LEN, pDigest->auth, pConn->secret) < 0) { tDebug("%s, authentication failed, msg discarded", pConn->info); code = TSDB_CODE_RPC_AUTH_FAILURE; } else { pHead->msgLen = (int32_t)htonl((uint32_t)pHead->msgLen) - sizeof(SRpcDigest); - if ( !rpcIsReq(pHead->msgType) ) pConn->secured = 1; // link is secured for client + if (!rpcIsReq(pHead->msgType)) pConn->secured = 1; // link is secured for client // tTrace("%s, message is authenticated", pConn->info); } } @@ -1647,13 +1842,9 @@ static void rpcUnlockConn(SRpcConn *pConn) { } } -static void rpcAddRef(SRpcInfo *pRpc) -{ - atomic_add_fetch_32(&pRpc->refCount, 1); -} +static void rpcAddRef(SRpcInfo *pRpc) { atomic_add_fetch_32(&pRpc->refCount, 1); } -static void rpcDecRef(SRpcInfo *pRpc) -{ +static void rpcDecRef(SRpcInfo *pRpc) { if (atomic_sub_fetch_32(&pRpc->refCount, 1) == 0) { rpcCloseConnCache(pRpc->pCache); taosHashCleanup(pRpc->hash); @@ -1668,4 +1859,4 @@ static void rpcDecRef(SRpcInfo *pRpc) atomic_sub_fetch_32(&tsRpcNum, 1); } } - +#endif diff --git a/source/libs/transport/src/rpcTcp.c b/source/libs/transport/src/rpcTcp.c index d0710c883f..9fa51a6fdc 100644 --- a/source/libs/transport/src/rpcTcp.c +++ b/source/libs/transport/src/rpcTcp.c @@ -13,24 +13,30 @@ * along with this program. If not, see . */ +#include "rpcTcp.h" +#ifdef USE_UV +#include +#endif #include "os.h" -#include "tutil.h" +#include "rpcHead.h" +#include "rpcLog.h" #include "taosdef.h" #include "taoserror.h" -#include "rpcLog.h" -#include "rpcHead.h" -#include "rpcTcp.h" +#include "tutil.h" +#ifdef USE_UV + +#else typedef struct SFdObj { - void *signature; - SOCKET fd; // TCP socket FD - void *thandle; // handle from upper layer, like TAOS + void * signature; + SOCKET fd; // TCP socket FD + void * thandle; // handle from upper layer, like TAOS uint32_t ip; uint16_t port; - int16_t closedByApp; // 1: already closed by App + int16_t closedByApp; // 1: already closed by App struct SThreadObj *pThreadObj; - struct SFdObj *prev; - struct SFdObj *next; + struct SFdObj * prev; + struct SFdObj * next; } SFdObj; typedef struct SThreadObj { @@ -43,35 +49,35 @@ typedef struct SThreadObj { int numOfFds; int threadId; char label[TSDB_LABEL_LEN]; - void *shandle; // handle passed by upper layer during server initialization - void *(*processData)(SRecvInfo *pPacket); + void * shandle; // handle passed by upper layer during server initialization + void *(*processData)(SRecvInfo *pPacket); } SThreadObj; typedef struct { - char label[TSDB_LABEL_LEN]; - int32_t index; - int numOfThreads; + char label[TSDB_LABEL_LEN]; + int32_t index; + int numOfThreads; SThreadObj **pThreadObj; } SClientObj; typedef struct { - SOCKET fd; - uint32_t ip; - uint16_t port; - int8_t stop; - int8_t reserve; - char label[TSDB_LABEL_LEN]; - int numOfThreads; - void * shandle; + SOCKET fd; + uint32_t ip; + uint16_t port; + int8_t stop; + int8_t reserve; + char label[TSDB_LABEL_LEN]; + int numOfThreads; + void * shandle; SThreadObj **pThreadObj; - pthread_t thread; + pthread_t thread; } SServerObj; -static void *taosProcessTcpData(void *param); +static void * taosProcessTcpData(void *param); static SFdObj *taosMallocFdObj(SThreadObj *pThreadObj, SOCKET fd); static void taosFreeFdObj(SFdObj *pFdObj); static void taosReportBrokenLink(SFdObj *pFdObj); -static void *taosAcceptTcpConnection(void *arg); +static void * taosAcceptTcpConnection(void *arg); void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThreads, void *fp, void *shandle) { SServerObj *pServerObj; @@ -99,7 +105,7 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread return NULL; } - int code = 0; + int code = 0; pthread_attr_t thattr; pthread_attr_init(&thattr); pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); @@ -110,7 +116,7 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread if (pThreadObj == NULL) { tError("TCP:%s no enough memory", label); terrno = TAOS_SYSTEM_ERROR(errno); - for (int j=0; jpThreadObj[j]); + for (int j = 0; j < i; ++j) free(pServerObj->pThreadObj[j]); free(pServerObj->pThreadObj); free(pServerObj); return NULL; @@ -172,8 +178,10 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread return (void *)pServerObj; } -static void taosStopTcpThread(SThreadObj* pThreadObj) { - if (pThreadObj == NULL) { return;} +static void taosStopTcpThread(SThreadObj *pThreadObj) { + if (pThreadObj == NULL) { + return; + } // save thread into local variable and signal thread to stop pthread_t thread = pThreadObj->thread; if (!taosCheckPthreadValid(thread)) { @@ -194,7 +202,7 @@ void taosStopTcpServer(void *handle) { pServerObj->stop = 1; if (pServerObj->fd >= 0) { - taosShutDownSocketRD(pServerObj->fd); + taosShutDownSocketRD(pServerObj->fd); } if (taosCheckPthreadValid(pServerObj->thread)) { if (taosComparePthread(pServerObj->thread, pthread_self())) { @@ -227,8 +235,8 @@ static void *taosAcceptTcpConnection(void *arg) { SOCKET connFd = -1; struct sockaddr_in caddr; int threadId = 0; - SThreadObj *pThreadObj; - SServerObj *pServerObj; + SThreadObj * pThreadObj; + SServerObj * pServerObj; pServerObj = (SServerObj *)arg; tDebug("%s TCP server is ready, ip:0x%x:%hu", pServerObj->label, pServerObj->ip, pServerObj->port); @@ -253,8 +261,8 @@ static void *taosAcceptTcpConnection(void *arg) { } taosKeepTcpAlive(connFd); - struct timeval to={5, 0}; - int32_t ret = taosSetSockOpt(connFd, SOL_SOCKET, SO_RCVTIMEO, &to, sizeof(to)); + struct timeval to = {5, 0}; + int32_t ret = taosSetSockOpt(connFd, SOL_SOCKET, SO_RCVTIMEO, &to, sizeof(to)); if (ret != 0) { taosCloseSocket(connFd); tError("%s failed to set recv timeout fd(%s)for connection from:%s:%hu", pServerObj->label, strerror(errno), @@ -262,7 +270,6 @@ static void *taosAcceptTcpConnection(void *arg) { continue; } - // pick up the thread to handle this connection pThreadObj = pServerObj->pThreadObj[threadId]; @@ -271,7 +278,7 @@ static void *taosAcceptTcpConnection(void *arg) { pFdObj->ip = caddr.sin_addr.s_addr; pFdObj->port = htons(caddr.sin_port); tDebug("%s new TCP connection from %s:%hu, fd:%d FD:%p numOfFds:%d", pServerObj->label, - taosInetNtoa(caddr.sin_addr), pFdObj->port, connFd, pFdObj, pThreadObj->numOfFds); + taosInetNtoa(caddr.sin_addr), pFdObj->port, connFd, pFdObj, pThreadObj->numOfFds); } else { taosCloseSocket(connFd); tError("%s failed to malloc FdObj(%s) for connection from:%s:%hu", pServerObj->label, strerror(errno), @@ -297,14 +304,14 @@ void *taosInitTcpClient(uint32_t ip, uint16_t port, char *label, int numOfThread tstrncpy(pClientObj->label, label, sizeof(pClientObj->label)); pClientObj->numOfThreads = numOfThreads; - pClientObj->pThreadObj = (SThreadObj **)calloc(numOfThreads, sizeof(SThreadObj*)); + pClientObj->pThreadObj = (SThreadObj **)calloc(numOfThreads, sizeof(SThreadObj *)); if (pClientObj->pThreadObj == NULL) { tError("TCP:%s no enough memory", label); tfree(pClientObj); terrno = TAOS_SYSTEM_ERROR(errno); } - int code = 0; + int code = 0; pthread_attr_t thattr; pthread_attr_init(&thattr); pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); @@ -314,15 +321,15 @@ void *taosInitTcpClient(uint32_t ip, uint16_t port, char *label, int numOfThread if (pThreadObj == NULL) { tError("TCP:%s no enough memory", label); terrno = TAOS_SYSTEM_ERROR(errno); - for (int j=0; jpThreadObj[j]); + for (int j = 0; j < i; ++j) free(pClientObj->pThreadObj[j]); free(pClientObj); pthread_attr_destroy(&thattr); return NULL; } pClientObj->pThreadObj[i] = pThreadObj; taosResetPthread(&pThreadObj->thread); - pThreadObj->ip = ip; - pThreadObj->stop = false; + pThreadObj->ip = ip; + pThreadObj->stop = false; tstrncpy(pThreadObj->label, label, sizeof(pThreadObj->label)); pThreadObj->shandle = shandle; pThreadObj->processData = fp; @@ -364,14 +371,14 @@ void taosStopTcpClient(void *chandle) { if (pClientObj == NULL) return; - tDebug ("%s TCP client is stopped", pClientObj->label); + tDebug("%s TCP client is stopped", pClientObj->label); } void taosCleanUpTcpClient(void *chandle) { SClientObj *pClientObj = chandle; if (pClientObj == NULL) return; for (int i = 0; i < pClientObj->numOfThreads; ++i) { - SThreadObj *pThreadObj= pClientObj->pThreadObj[i]; + SThreadObj *pThreadObj = pClientObj->pThreadObj[i]; taosStopTcpThread(pThreadObj); } @@ -381,9 +388,9 @@ void taosCleanUpTcpClient(void *chandle) { } void *taosOpenTcpClientConnection(void *shandle, void *thandle, uint32_t ip, uint16_t port) { - SClientObj * pClientObj = shandle; - int32_t index = atomic_load_32(&pClientObj->index) % pClientObj->numOfThreads; - atomic_store_32(&pClientObj->index, index + 1); + SClientObj *pClientObj = shandle; + int32_t index = atomic_load_32(&pClientObj->index) % pClientObj->numOfThreads; + atomic_store_32(&pClientObj->index, index + 1); SThreadObj *pThreadObj = pClientObj->pThreadObj[index]; SOCKET fd = taosOpenTcpClientSocket(ip, port, pThreadObj->ip); @@ -394,10 +401,9 @@ void *taosOpenTcpClientConnection(void *shandle, void *thandle, uint32_t ip, uin #endif struct sockaddr_in sin; - uint16_t localPort = 0; - unsigned int addrlen = sizeof(sin); - if (getsockname(fd, (struct sockaddr *)&sin, &addrlen) == 0 && - sin.sin_family == AF_INET && addrlen == sizeof(sin)) { + uint16_t localPort = 0; + unsigned int addrlen = sizeof(sin); + if (getsockname(fd, (struct sockaddr *)&sin, &addrlen) == 0 && sin.sin_family == AF_INET && addrlen == sizeof(sin)) { localPort = (uint16_t)ntohs(sin.sin_port); } @@ -407,8 +413,8 @@ void *taosOpenTcpClientConnection(void *shandle, void *thandle, uint32_t ip, uin pFdObj->thandle = thandle; pFdObj->port = port; pFdObj->ip = ip; - tDebug("%s %p TCP connection to 0x%x:%hu is created, localPort:%hu FD:%p numOfFds:%d", - pThreadObj->label, thandle, ip, port, localPort, pFdObj, pThreadObj->numOfFds); + tDebug("%s %p TCP connection to 0x%x:%hu is created, localPort:%hu FD:%p numOfFds:%d", pThreadObj->label, thandle, + ip, port, localPort, pFdObj, pThreadObj->numOfFds); } else { tError("%s failed to malloc client FdObj(%s)", pThreadObj->label, strerror(errno)); taosCloseSocket(fd); @@ -441,7 +447,6 @@ int taosSendTcpData(uint32_t ip, uint16_t port, void *data, int len, void *chand } static void taosReportBrokenLink(SFdObj *pFdObj) { - SThreadObj *pThreadObj = pFdObj->pThreadObj; // notify the upper layer, so it will clean the associated context @@ -464,9 +469,9 @@ static void taosReportBrokenLink(SFdObj *pFdObj) { } static int taosReadTcpData(SFdObj *pFdObj, SRecvInfo *pInfo) { - SRpcHead rpcHead; - int32_t msgLen, leftLen, retLen, headLen; - char *buffer, *msg; + SRpcHead rpcHead; + int32_t msgLen, leftLen, retLen, headLen; + char * buffer, *msg; SThreadObj *pThreadObj = pFdObj->pThreadObj; @@ -483,7 +488,8 @@ static int taosReadTcpData(SFdObj *pFdObj, SRecvInfo *pInfo) { tError("%s %p TCP malloc(size:%d) fail", pThreadObj->label, pFdObj->thandle, msgLen); return -1; } else { - tTrace("%s %p read data, FD:%p fd:%d TCP malloc mem:%p", pThreadObj->label, pFdObj->thandle, pFdObj, pFdObj->fd, buffer); + tTrace("%s %p read data, FD:%p fd:%d TCP malloc mem:%p", pThreadObj->label, pFdObj->thandle, pFdObj, pFdObj->fd, + buffer); } msg = buffer + tsRpcOverhead; @@ -491,8 +497,7 @@ static int taosReadTcpData(SFdObj *pFdObj, SRecvInfo *pInfo) { retLen = taosReadMsg(pFdObj->fd, msg + headLen, leftLen); if (leftLen != retLen) { - tError("%s %p read error, leftLen:%d retLen:%d FD:%p", - pThreadObj->label, pFdObj->thandle, leftLen, retLen, pFdObj); + tError("%s %p read error, leftLen:%d retLen:%d FD:%p", pThreadObj->label, pFdObj->thandle, leftLen, retLen, pFdObj); free(buffer); return -1; } @@ -519,8 +524,8 @@ static int taosReadTcpData(SFdObj *pFdObj, SRecvInfo *pInfo) { #define maxEvents 10 static void *taosProcessTcpData(void *param) { - SThreadObj *pThreadObj = param; - SFdObj *pFdObj; + SThreadObj * pThreadObj = param; + SFdObj * pFdObj; struct epoll_event events[maxEvents]; SRecvInfo recvInfo; @@ -569,7 +574,7 @@ static void *taosProcessTcpData(void *param) { if (pThreadObj->stop) break; } - if (pThreadObj->pollFd >=0) { + if (pThreadObj->pollFd >= 0) { EpollClose(pThreadObj->pollFd); pThreadObj->pollFd = -1; } @@ -620,7 +625,6 @@ static SFdObj *taosMallocFdObj(SThreadObj *pThreadObj, SOCKET fd) { } static void taosFreeFdObj(SFdObj *pFdObj) { - if (pFdObj == NULL) return; if (pFdObj->signature != pFdObj) return; @@ -638,8 +642,8 @@ static void taosFreeFdObj(SFdObj *pFdObj) { pThreadObj->numOfFds--; if (pThreadObj->numOfFds < 0) - tError("%s %p TCP thread:%d, number of FDs is negative!!!", - pThreadObj->label, pFdObj->thandle, pThreadObj->threadId); + tError("%s %p TCP thread:%d, number of FDs is negative!!!", pThreadObj->label, pFdObj->thandle, + pThreadObj->threadId); if (pFdObj->prev) { (pFdObj->prev)->next = pFdObj->next; @@ -653,8 +657,10 @@ static void taosFreeFdObj(SFdObj *pFdObj) { pthread_mutex_unlock(&pThreadObj->mutex); - tDebug("%s %p TCP connection is closed, FD:%p fd:%d numOfFds:%d", - pThreadObj->label, pFdObj->thandle, pFdObj, pFdObj->fd, pThreadObj->numOfFds); + tDebug("%s %p TCP connection is closed, FD:%p fd:%d numOfFds:%d", pThreadObj->label, pFdObj->thandle, pFdObj, + pFdObj->fd, pThreadObj->numOfFds); tfree(pFdObj); } + +#endif diff --git a/source/libs/transport/src/rpcUdp.c b/source/libs/transport/src/rpcUdp.c index 5bc31c189a..79956cc98d 100644 --- a/source/libs/transport/src/rpcUdp.c +++ b/source/libs/transport/src/rpcUdp.c @@ -13,50 +13,53 @@ * along with this program. If not, see . */ +#include "rpcUdp.h" #include "os.h" -#include "ttimer.h" -#include "tutil.h" +#include "rpcHead.h" +#include "rpcLog.h" #include "taosdef.h" #include "taoserror.h" -#include "rpcLog.h" -#include "rpcUdp.h" -#include "rpcHead.h" +#include "ttimer.h" +#include "tutil.h" +#ifdef USE_UV +// no support upd currently +#else #define RPC_MAX_UDP_CONNS 256 #define RPC_MAX_UDP_PKTS 1000 #define RPC_UDP_BUF_TIME 5 // mseconds #define RPC_MAX_UDP_SIZE 65480 typedef struct { - int index; - SOCKET fd; - uint16_t port; // peer port - uint16_t localPort; // local port - char label[TSDB_LABEL_LEN]; // copy from udpConnSet; - pthread_t thread; - void *hash; - void *shandle; // handle passed by upper layer during server initialization - void *pSet; - void *(*processData)(SRecvInfo *pRecv); - char *buffer; // buffer to receive data + int index; + SOCKET fd; + uint16_t port; // peer port + uint16_t localPort; // local port + char label[TSDB_LABEL_LEN]; // copy from udpConnSet; + pthread_t thread; + void * hash; + void * shandle; // handle passed by upper layer during server initialization + void * pSet; + void *(*processData)(SRecvInfo *pRecv); + char *buffer; // buffer to receive data } SUdpConn; typedef struct { - int index; - int server; - uint32_t ip; // local IP - uint16_t port; // local Port - void *shandle; // handle passed by upper layer during server initialization - int threads; - char label[TSDB_LABEL_LEN]; - void *(*fp)(SRecvInfo *pPacket); - SUdpConn udpConn[]; + int index; + int server; + uint32_t ip; // local IP + uint16_t port; // local Port + void * shandle; // handle passed by upper layer during server initialization + int threads; + char label[TSDB_LABEL_LEN]; + void *(*fp)(SRecvInfo *pPacket); + SUdpConn udpConn[]; } SUdpConnSet; static void *taosRecvUdpData(void *param); void *taosInitUdpConnection(uint32_t ip, uint16_t port, char *label, int threads, void *fp, void *shandle) { - SUdpConn *pConn; + SUdpConn * pConn; SUdpConnSet *pSet; int size = (int)sizeof(SUdpConnSet) + threads * (int)sizeof(SUdpConn); @@ -79,7 +82,7 @@ void *taosInitUdpConnection(uint32_t ip, uint16_t port, char *label, int threads pthread_attr_init(&thAttr); pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE); - int i; + int i; uint16_t ownPort; for (i = 0; i < threads; ++i) { pConn = pSet->udpConn + i; @@ -97,9 +100,9 @@ void *taosInitUdpConnection(uint32_t ip, uint16_t port, char *label, int threads } struct sockaddr_in sin; - unsigned int addrlen = sizeof(sin); - if (getsockname(pConn->fd, (struct sockaddr *)&sin, &addrlen) == 0 && - sin.sin_family == AF_INET && addrlen == sizeof(sin)) { + unsigned int addrlen = sizeof(sin); + if (getsockname(pConn->fd, (struct sockaddr *)&sin, &addrlen) == 0 && sin.sin_family == AF_INET && + addrlen == sizeof(sin)) { pConn->localPort = (uint16_t)ntohs(sin.sin_port); } @@ -118,7 +121,7 @@ void *taosInitUdpConnection(uint32_t ip, uint16_t port, char *label, int threads pthread_attr_destroy(&thAttr); - if (i != threads) { + if (i != threads) { terrno = TAOS_SYSTEM_ERROR(errno); taosCleanUpUdpConnection(pSet); return NULL; @@ -130,14 +133,14 @@ void *taosInitUdpConnection(uint32_t ip, uint16_t port, char *label, int threads void taosStopUdpConnection(void *handle) { SUdpConnSet *pSet = (SUdpConnSet *)handle; - SUdpConn *pConn; + SUdpConn * pConn; if (pSet == NULL) return; for (int i = 0; i < pSet->threads; ++i) { pConn = pSet->udpConn + i; - if (pConn->fd >=0) shutdown(pConn->fd, SHUT_RDWR); - if (pConn->fd >=0) taosCloseSocket(pConn->fd); + if (pConn->fd >= 0) shutdown(pConn->fd, SHUT_RDWR); + if (pConn->fd >= 0) taosCloseSocket(pConn->fd); pConn->fd = -1; } @@ -155,13 +158,13 @@ void taosStopUdpConnection(void *handle) { void taosCleanUpUdpConnection(void *handle) { SUdpConnSet *pSet = (SUdpConnSet *)handle; - SUdpConn *pConn; + SUdpConn * pConn; if (pSet == NULL) return; for (int i = 0; i < pSet->threads; ++i) { pConn = pSet->udpConn + i; - if (pConn->fd >=0) taosCloseSocket(pConn->fd); + if (pConn->fd >= 0) taosCloseSocket(pConn->fd); } tDebug("%s UDP is cleaned up", pSet->label); @@ -182,7 +185,7 @@ void *taosOpenUdpConnection(void *shandle, void *thandle, uint32_t ip, uint16_t } static void *taosRecvUdpData(void *param) { - SUdpConn *pConn = param; + SUdpConn * pConn = param; struct sockaddr_in sourceAdd; ssize_t dataLen; unsigned int addLen; @@ -218,7 +221,7 @@ static void *taosRecvUdpData(void *param) { } int32_t size = dataLen + tsRpcOverhead; - char *tmsg = malloc(size); + char * tmsg = malloc(size); if (NULL == tmsg) { tError("%s failed to allocate memory, size:%" PRId64, pConn->label, (int64_t)dataLen); continue; @@ -257,4 +260,4 @@ int taosSendUdpData(uint32_t ip, uint16_t port, void *data, int dataLen, void *c return ret; } - +#endif diff --git a/source/libs/transport/src/transport.c b/source/libs/transport/src/transport.c index 6dea4a4e57..f2f48bbc8a 100644 --- a/source/libs/transport/src/transport.c +++ b/source/libs/transport/src/transport.c @@ -11,4 +11,4 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . - */ \ No newline at end of file + */ diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 3ea564722b..8294bca959 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -80,6 +80,8 @@ TAOS_DEFINE_ERROR(TSDB_CODE_CHECKSUM_ERROR, "Checksum error") TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_MSG, "Invalid config message") TAOS_DEFINE_ERROR(TSDB_CODE_MSG_NOT_PROCESSED, "Message not processed") TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_PARA, "Invalid parameters") +TAOS_DEFINE_ERROR(TSDB_CODE_REPEAT_INIT, "Repeat initialization") + TAOS_DEFINE_ERROR(TSDB_CODE_REF_NO_MEMORY, "Ref out of memory") TAOS_DEFINE_ERROR(TSDB_CODE_REF_FULL, "too many Ref Objs") TAOS_DEFINE_ERROR(TSDB_CODE_REF_ID_REMOVED, "Ref ID is removed") diff --git a/src/inc/tfs.h b/src/inc/tfs.h index 11e33a3af7..9ad7a8f66e 100644 --- a/src/inc/tfs.h +++ b/src/inc/tfs.h @@ -49,7 +49,7 @@ typedef struct { } STierMeta; int tfsInit(SDiskCfg *pDiskCfg, int ndisk); -void tfsDestroy(); +void tfsCleanup(); void tfsUpdateInfo(SFSMeta *pFSMeta, STierMeta *tierMetas, int8_t numLevels); void tfsGetMeta(SFSMeta *pMeta); void tfsAllocDisk(int expLevel, int *level, int *id); diff --git a/tests/test/c/create_table.c b/tests/test/c/create_table.c index a5a55bbc91..aae4dc7074 100644 --- a/tests/test/c/create_table.c +++ b/tests/test/c/create_table.c @@ -30,6 +30,7 @@ int32_t createTable = 1; int32_t insertData = 0; int32_t batchNum = 100; int32_t numOfVgroups = 2; +int32_t showTablesFlag = 0; typedef struct { int64_t tableBeginIndex; @@ -45,88 +46,9 @@ typedef struct { pthread_t thread; } SThreadInfo; -void parseArgument(int32_t argc, char *argv[]); -void *threadFunc(void *param); -void createDbAndStb(); - -int32_t main(int32_t argc, char *argv[]) { - parseArgument(argc, argv); - createDbAndStb(); - - pPrint("%d threads are spawned to create %d tables", numOfThreads, numOfThreads); - - pthread_attr_t thattr; - pthread_attr_init(&thattr); - pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); - SThreadInfo *pInfo = (SThreadInfo *)calloc(numOfThreads, sizeof(SThreadInfo)); - - //int64_t numOfTablesPerThread = numOfTables / numOfThreads; - //numOfTables = numOfTablesPerThread * numOfThreads; - - - if (numOfThreads < 1) { - numOfThreads = 1; - } - - int64_t a = numOfTables / numOfThreads; - if (a < 1) { - numOfThreads = numOfTables; - a = 1; - } - - int64_t b = 0; - b = numOfTables % numOfThreads; - - int64_t tableFrom = 0; - for (int32_t i = 0; i < numOfThreads; ++i) { - pInfo[i].tableBeginIndex = tableFrom; - pInfo[i].tableEndIndex = i < b ? tableFrom + a : tableFrom + a - 1; - tableFrom = pInfo[i].tableEndIndex + 1; - pInfo[i].threadIndex = i; - pInfo[i].minDelay = INT64_MAX; - strcpy(pInfo[i].dbName, dbName); - strcpy(pInfo[i].stbName, stbName); - pthread_create(&(pInfo[i].thread), &thattr, threadFunc, (void *)(pInfo + i)); - } - - taosMsleep(300); - for (int32_t i = 0; i < numOfThreads; i++) { - pthread_join(pInfo[i].thread, NULL); - } - - int64_t maxDelay = 0; - int64_t minDelay = INT64_MAX; - - float createTableSpeed = 0; - for (int32_t i = 0; i < numOfThreads; ++i) { - createTableSpeed += pInfo[i].createTableSpeed; - - if (pInfo[i].maxDelay > maxDelay) maxDelay = pInfo[i].maxDelay; - if (pInfo[i].minDelay < minDelay) minDelay = pInfo[i].minDelay; - } - - float insertDataSpeed = 0; - for (int32_t i = 0; i < numOfThreads; ++i) { - insertDataSpeed += pInfo[i].insertDataSpeed; - } - - pPrint("%s total %" PRId64 " tables, %.1f tables/second, threads:%d, maxDelay: %" PRId64 "us, minDelay: %" PRId64 "us %s", - GREEN, - numOfTables, - createTableSpeed, - numOfThreads, - maxDelay, - minDelay, - NC); - - if (insertData) { - pPrint("%s total %" PRId64 " tables, %.1f rows/second, threads:%d %s", GREEN, numOfTables, insertDataSpeed, - numOfThreads, NC); - } - - pthread_attr_destroy(&thattr); - free(pInfo); -} +//void parseArgument(int32_t argc, char *argv[]); +//void *threadFunc(void *param); +//void createDbAndStb(); void createDbAndStb() { pPrint("start to create db and stable"); @@ -188,6 +110,62 @@ void printInsertProgress(SThreadInfo *pInfo, int64_t t) { totalTables, seconds, speed); } +static int64_t getResult(TAOS_RES *tres) { + TAOS_ROW row = taos_fetch_row(tres); + if (row == NULL) { + return 0; + } + + int num_fields = taos_num_fields(tres); + TAOS_FIELD *fields = taos_fetch_fields(tres); + int precision = taos_result_precision(tres); + + int64_t numOfRows = 0; + do { + numOfRows++; + row = taos_fetch_row(tres); + } while (row != NULL); + + return numOfRows; +} + + +void showTables() { + pPrint("start to show tables"); + char qstr[32]; + + TAOS *con = taos_connect(NULL, "root", "taosdata", NULL, 0); + if (con == NULL) { + pError("failed to connect to DB, reason:%s", taos_errstr(NULL)); + exit(1); + } + + sprintf(qstr, "use %s", dbName); + TAOS_RES *pRes = taos_query(con, qstr); + int code = taos_errno(pRes); + if (code != 0) { + pError("failed to use db, code:%d reason:%s", taos_errno(pRes), taos_errstr(pRes)); + exit(1); + } + taos_free_result(pRes); + + sprintf(qstr, "show tables"); + pRes = taos_query(con, qstr); + code = taos_errno(pRes); + if (code != 0) { + pError("failed to show tables, code:%d reason:%s", taos_errno(pRes), taos_errstr(pRes)); + exit(0); + } + + int64_t totalTableNum = getResult(pRes); + taos_free_result(pRes); + + pPrint("%s database: %s, total %" PRId64 " tables %s", GREEN, dbName, totalTableNum, NC); + + taos_close(con); +} + + void *threadFunc(void *param) { SThreadInfo *pInfo = (SThreadInfo *)param; char *qstr = malloc(2000 * 1000); @@ -298,6 +276,8 @@ void printHelp() { printf("%s%s%s%d\n", indent, indent, "insertData, default is ", insertData); printf("%s%s\n", indent, "-b"); printf("%s%s%s%d\n", indent, indent, "batchNum, default is ", batchNum); + printf("%s%s\n", indent, "-w"); + printf("%s%s%s%d\n", indent, indent, "showTablesFlag, default is ", showTablesFlag); exit(EXIT_SUCCESS); } @@ -325,6 +305,8 @@ void parseArgument(int32_t argc, char *argv[]) { insertData = atoi(argv[++i]); } else if (strcmp(argv[i], "-b") == 0) { batchNum = atoi(argv[++i]); + } else if (strcmp(argv[i], "-w") == 0) { + showTablesFlag = atoi(argv[++i]); } else { } } @@ -338,6 +320,93 @@ void parseArgument(int32_t argc, char *argv[]) { pPrint("%s createTable:%d %s", GREEN, createTable, NC); pPrint("%s insertData:%d %s", GREEN, insertData, NC); pPrint("%s batchNum:%d %s", GREEN, batchNum, NC); + pPrint("%s showTablesFlag:%d %s", GREEN, showTablesFlag, NC); pPrint("%s start create table performace test %s", GREEN, NC); } + +int32_t main(int32_t argc, char *argv[]) { + parseArgument(argc, argv); + + if (showTablesFlag) { + showTables(); + return 0; + } + + createDbAndStb(); + + pPrint("%d threads are spawned to create %d tables", numOfThreads, numOfThreads); + + pthread_attr_t thattr; + pthread_attr_init(&thattr); + pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); + SThreadInfo *pInfo = (SThreadInfo *)calloc(numOfThreads, sizeof(SThreadInfo)); + + //int64_t numOfTablesPerThread = numOfTables / numOfThreads; + //numOfTables = numOfTablesPerThread * numOfThreads; + + + if (numOfThreads < 1) { + numOfThreads = 1; + } + + int64_t a = numOfTables / numOfThreads; + if (a < 1) { + numOfThreads = numOfTables; + a = 1; + } + + int64_t b = 0; + b = numOfTables % numOfThreads; + + int64_t tableFrom = 0; + for (int32_t i = 0; i < numOfThreads; ++i) { + pInfo[i].tableBeginIndex = tableFrom; + pInfo[i].tableEndIndex = i < b ? tableFrom + a : tableFrom + a - 1; + tableFrom = pInfo[i].tableEndIndex + 1; + pInfo[i].threadIndex = i; + pInfo[i].minDelay = INT64_MAX; + strcpy(pInfo[i].dbName, dbName); + strcpy(pInfo[i].stbName, stbName); + pthread_create(&(pInfo[i].thread), &thattr, threadFunc, (void *)(pInfo + i)); + } + + taosMsleep(300); + for (int32_t i = 0; i < numOfThreads; i++) { + pthread_join(pInfo[i].thread, NULL); + } + + int64_t maxDelay = 0; + int64_t minDelay = INT64_MAX; + + float createTableSpeed = 0; + for (int32_t i = 0; i < numOfThreads; ++i) { + createTableSpeed += pInfo[i].createTableSpeed; + + if (pInfo[i].maxDelay > maxDelay) maxDelay = pInfo[i].maxDelay; + if (pInfo[i].minDelay < minDelay) minDelay = pInfo[i].minDelay; + } + + float insertDataSpeed = 0; + for (int32_t i = 0; i < numOfThreads; ++i) { + insertDataSpeed += pInfo[i].insertDataSpeed; + } + + pPrint("%s total %" PRId64 " tables, %.1f tables/second, threads:%d, maxDelay: %" PRId64 "us, minDelay: %" PRId64 "us %s", + GREEN, + numOfTables, + createTableSpeed, + numOfThreads, + maxDelay, + minDelay, + NC); + + if (insertData) { + pPrint("%s total %" PRId64 " tables, %.1f rows/second, threads:%d %s", GREEN, numOfTables, insertDataSpeed, + numOfThreads, NC); + } + + pthread_attr_destroy(&thattr); + free(pInfo); +} +