diff --git a/example/src/tmq.c b/example/src/tmq.c index 26e5ea82c1..094fd94bfc 100644 --- a/example/src/tmq.c +++ b/example/src/tmq.c @@ -66,7 +66,7 @@ int32_t init_env() { } int32_t create_topic() { - printf("create topic"); + printf("create topic\n"); TAOS_RES* pRes; TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); if (pConn == NULL) { @@ -91,6 +91,10 @@ int32_t create_topic() { return 0; } +void tmq_commit_cb_print(tmq_t* tmq, tmq_resp_err_t resp, tmq_topic_vgroup_list_t* offsets, void* param) { + printf("commit %d\n", resp); +} + tmq_t* build_consumer() { TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); assert(pConn != NULL); @@ -103,6 +107,7 @@ tmq_t* build_consumer() { tmq_conf_t* conf = tmq_conf_new(); tmq_conf_set(conf, "group.id", "tg2"); + tmq_conf_set_offset_commit_cb(conf, tmq_commit_cb_print); tmq_t* tmq = tmq_consumer_new(pConn, conf, NULL, 0); return tmq; } @@ -144,7 +149,7 @@ void basic_consume_loop(tmq_t* tmq, tmq_list_t* topics) { } void sync_consume_loop(tmq_t* tmq, tmq_list_t* topics) { - static const int MIN_COMMIT_COUNT = 1000; + static const int MIN_COMMIT_COUNT = 1; int msg_count = 0; tmq_resp_err_t err; @@ -214,6 +219,6 @@ int main(int argc, char* argv[]) { tmq_t* tmq = build_consumer(); tmq_list_t* topic_list = build_topic_list(); /*perf_loop(tmq, topic_list);*/ - basic_consume_loop(tmq, topic_list); - /*sync_consume_loop(tmq, topic_list);*/ + /*basic_consume_loop(tmq, topic_list);*/ + sync_consume_loop(tmq, topic_list); } diff --git a/include/client/taos.h b/include/client/taos.h index a960d37937..2c8135c8ff 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -198,8 +198,8 @@ DLL_EXPORT TAOS_RES *taos_schemaless_insert(TAOS *taos, char *lines[], int numLi /* --------------------------TMQ INTERFACE------------------------------- */ enum tmq_resp_err_t { + TMQ_RESP_ERR__FAIL = -1, TMQ_RESP_ERR__SUCCESS = 0, - TMQ_RESP_ERR__FAIL = 1, }; typedef enum tmq_resp_err_t tmq_resp_err_t; @@ -226,7 +226,7 @@ DLL_EXPORT const char *tmq_err2str(tmq_resp_err_t); DLL_EXPORT tmq_resp_err_t tmq_subscribe(tmq_t *tmq, tmq_list_t *topic_list); #if 0 DLL_EXPORT tmq_resp_err_t tmq_unsubscribe(tmq_t* tmq); -DLL_EXPORT tmq_resp_err_t tmq_subscription(tmq_t* tmq, tmq_topic_vgroup_list_t** topics); +DLL_EXPORT tmq_resp_err_t tmq_subscription(tmq_t* tmq, tmq_list_t** topics); #endif DLL_EXPORT tmq_message_t *tmq_consumer_poll(tmq_t *tmq, int64_t blocking_time); DLL_EXPORT tmq_resp_err_t tmq_consumer_close(tmq_t *tmq); @@ -238,6 +238,7 @@ DLL_EXPORT tmq_resp_err_t tmq_commit(tmq_t *tmq, const tmq_topic_vgroup_list_t * #if 0 DLL_EXPORT tmq_resp_err_t tmq_commit_message(tmq_t* tmq, const tmq_message_t* tmqmessage, int32_t async); #endif +DLL_EXPORT tmq_resp_err_t tmq_seek(tmq_t *tmq, const tmq_topic_vgroup_t *offset); /* ----------------------TMQ CONFIGURATION INTERFACE---------------------- */ enum tmq_conf_res_t { diff --git a/include/common/common.h b/include/common/common.h index fd5b6717ab..8fa2d03d6d 100644 --- a/include/common/common.h +++ b/include/common/common.h @@ -16,7 +16,6 @@ #ifndef TDENGINE_COMMON_H #define TDENGINE_COMMON_H - #ifdef __cplusplus extern "C" { #endif @@ -43,14 +42,16 @@ extern "C" { // int16_t bytes; // } SSchema; -#define TMQ_REQ_TYPE_COMMIT_ONLY 0 -#define TMQ_REQ_TYPE_CONSUME_ONLY 1 -#define TMQ_REQ_TYPE_CONSUME_AND_COMMIT 2 +enum { + TMQ_CONF__RESET_OFFSET__LATEST = -1, + TMQ_CONF__RESET_OFFSET__EARLIEAST = -2, + TMQ_CONF__RESET_OFFSET__NONE = -3, +}; typedef struct { uint32_t numOfTables; - SArray *pGroupList; - SHashObj *map; // speedup acquire the tableQueryInfo by table uid + SArray* pGroupList; + SHashObj* map; // speedup acquire the tableQueryInfo by table uid } STableGroupInfo; typedef struct SColumnDataAgg { @@ -79,14 +80,14 @@ typedef struct SConstantItem { // info.numOfCols = taosArrayGetSize(pDataBlock) + taosArrayGetSize(pConstantList); typedef struct SSDataBlock { - SColumnDataAgg *pBlockAgg; - SArray *pDataBlock; // SArray - SArray *pConstantList; // SArray, it is a constant/tags value of the corresponding result value. - SDataBlockInfo info; + SColumnDataAgg* pBlockAgg; + SArray* pDataBlock; // SArray + SArray* pConstantList; // SArray, it is a constant/tags value of the corresponding result value. + SDataBlockInfo info; } SSDataBlock; typedef struct SVarColAttr { - int32_t *offset; // start position for each entry in the list + int32_t* offset; // start position for each entry in the list uint32_t length; // used buffer size that contain the valid data uint32_t allocLen; // allocated buffer size } SVarColAttr; @@ -94,11 +95,11 @@ typedef struct SVarColAttr { // pBlockAgg->numOfNull == info.rows, all data are null // pBlockAgg->numOfNull == 0, no data are null. typedef struct SColumnInfoData { - SColumnInfo info; // TODO filter info needs to be removed - bool hasNull;// if current column data has null value. - char *pData; // the corresponding block data in memory + SColumnInfo info; // TODO filter info needs to be removed + bool hasNull; // if current column data has null value. + char* pData; // the corresponding block data in memory union { - char *nullbitmap; // bitmap, one bit for each item in the list + char* nullbitmap; // bitmap, one bit for each item in the list SVarColAttr varmeta; }; } SColumnInfoData; @@ -149,7 +150,6 @@ static FORCE_INLINE int32_t tEncodeSMqConsumeRsp(void** buf, const SMqConsumeRsp int32_t tlen = 0; int32_t sz = 0; tlen += taosEncodeFixedI64(buf, pRsp->consumerId); - tlen += taosEncodeFixedI64(buf, pRsp->committedOffset); tlen += taosEncodeFixedI64(buf, pRsp->reqOffset); tlen += taosEncodeFixedI64(buf, pRsp->rspOffset); tlen += taosEncodeFixedI32(buf, pRsp->skipLogNum); @@ -170,7 +170,6 @@ static FORCE_INLINE int32_t tEncodeSMqConsumeRsp(void** buf, const SMqConsumeRsp static FORCE_INLINE void* tDecodeSMqConsumeRsp(void* buf, SMqConsumeRsp* pRsp) { int32_t sz; buf = taosDecodeFixedI64(buf, &pRsp->consumerId); - buf = taosDecodeFixedI64(buf, &pRsp->committedOffset); buf = taosDecodeFixedI64(buf, &pRsp->reqOffset); buf = taosDecodeFixedI64(buf, &pRsp->rspOffset); buf = taosDecodeFixedI32(buf, &pRsp->skipLogNum); @@ -250,11 +249,11 @@ typedef struct SSqlExpr { char token[TSDB_COL_NAME_LEN]; // original token SSchema resSchema; - int32_t numOfCols; - SColumn* pColumns; // data columns that are required by query - int32_t interBytes; // inter result buffer size - int16_t numOfParams; // argument value of each function - SVariant param[3]; // parameters are not more than 3 + int32_t numOfCols; + SColumn* pColumns; // data columns that are required by query + int32_t interBytes; // inter result buffer size + int16_t numOfParams; // argument value of each function + SVariant param[3]; // parameters are not more than 3 } SSqlExpr; typedef struct SExprInfo { @@ -271,7 +270,7 @@ typedef struct SSessionWindow { SColumn col; } SSessionWindow; -#define QUERY_ASC_FORWARD_STEP 1 +#define QUERY_ASC_FORWARD_STEP 1 #define QUERY_DESC_FORWARD_STEP -1 #define GET_FORWARD_DIRECTION_FACTOR(ord) (((ord) == TSDB_ORDER_ASC) ? QUERY_ASC_FORWARD_STEP : QUERY_DESC_FORWARD_STEP) diff --git a/include/common/tep.h b/include/common/tep.h index 956cf6ec36..6d09d0ce51 100644 --- a/include/common/tep.h +++ b/include/common/tep.h @@ -23,7 +23,7 @@ typedef struct SBlockOrderInfo { // bool hasNull; } SBlockOrderInfo; -int taosGetFqdnPortFromEp(const char *ep, SEp *pEp); +int taosGetFqdnPortFromEp(const char *ep, uint16_t defaultPort, SEp *pEp); void addEpIntoEpSet(SEpSet *pEpSet, const char *fqdn, uint16_t port); bool isEpsetEqual(const SEpSet *s1, const SEpSet *s2); diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 4bc8c9fb86..8bce28d34c 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -21,31 +21,9 @@ extern "C" { #endif #include "tdef.h" -#include "tcfg.h" - -// cluster -extern char tsFirst[]; -extern char tsSecond[]; -extern char tsLocalFqdn[]; -extern char tsLocalEp[]; -extern uint16_t tsServerPort; -extern int32_t tsStatusInterval; -extern int8_t tsEnableTelemetryReporting; -extern int32_t tsNumOfSupportVnodes; // common -extern int tsRpcTimer; -extern int tsRpcMaxTime; -extern int tsRpcForceTcp; // all commands go to tcp protocol if this is enabled -extern int32_t tsMaxConnections; -extern int32_t tsMaxShellConns; -extern int32_t tsShellActivityTimer; -extern uint32_t tsMaxTmrCtrl; -extern float tsNumOfThreadsPerCore; -extern int32_t tsNumOfCommitThreads; -extern float tsRatioOfQueryCores; extern int8_t tsDaylight; -extern int8_t tsEnableCoreFile; extern int32_t tsCompressMsgSize; extern int32_t tsCompressColData; extern int32_t tsMaxNumOfDistinctResults; @@ -78,24 +56,9 @@ extern int32_t tsProjectExecInterval; extern int64_t tsMaxRetentWindow; // system info -extern float tsTotalLogDirGB; -extern float tsTotalTmpDirGB; -extern float tsTotalDataDirGB; -extern float tsAvailLogDirGB; -extern float tsAvailTmpDirectorySpace; -extern float tsAvailDataDirGB; -extern float tsUsedDataDirGB; -extern float tsMinimalLogDirGB; -extern float tsReservedTmpDirectorySpace; -extern float tsMinimalDataDirGB; + extern uint32_t tsVersion; -// build info -extern char version[]; -extern char compatible_version[]; -extern char gitinfo[]; -extern char gitinfoOfInternal[]; -extern char buildinfo[]; // lossy extern char tsLossyColumns[]; @@ -105,13 +68,10 @@ extern uint32_t tsMaxRange; extern uint32_t tsCurRange; extern char tsCompressor[]; -extern int32_t tsDiskCfgNum; -extern SDiskCfg tsDiskCfg[]; #define NEEDTO_COMPRESSS_MSG(size) (tsCompressMsgSize != -1 && (size) > tsCompressMsgSize) void taosInitGlobalCfg(); -int32_t taosCheckAndPrintCfg(); int32_t taosCfgDynamicOptions(char *msg); bool taosCheckBalanceCfgOptions(const char *option, int32_t *vnodeId, int32_t *dnodeId); void taosAddDataDir(int index, char *v1, int level, int primary); diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 5164fb6ccd..ae3586e735 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -422,8 +422,8 @@ typedef struct { } SColumnInfo; typedef struct { - uint64_t uid; - TSKEY key; // last accessed ts, for subscription + int64_t uid; + TSKEY key; // last accessed ts, for subscription } STableIdInfo; typedef struct STimeWindow { @@ -554,8 +554,8 @@ int32_t tSerializeSDropDbReq(void* buf, int32_t bufLen, SDropDbReq* pReq); int32_t tDeserializeSDropDbReq(void* buf, int32_t bufLen, SDropDbReq* pReq); typedef struct { - char db[TSDB_DB_FNAME_LEN]; - uint64_t uid; + char db[TSDB_DB_FNAME_LEN]; + int64_t uid; } SDropDbRsp; int32_t tSerializeSDropDbRsp(void* buf, int32_t bufLen, SDropDbRsp* pRsp); @@ -570,12 +570,12 @@ int32_t tSerializeSUseDbReq(void* buf, int32_t bufLen, SUseDbReq* pReq); int32_t tDeserializeSUseDbReq(void* buf, int32_t bufLen, SUseDbReq* pReq); typedef struct { - char db[TSDB_DB_FNAME_LEN]; - uint64_t uid; - int32_t vgVersion; - int32_t vgNum; - int8_t hashMethod; - SArray* pVgroupInfos; // Array of SVgroupInfo + char db[TSDB_DB_FNAME_LEN]; + int64_t uid; + int32_t vgVersion; + int32_t vgNum; + int8_t hashMethod; + SArray* pVgroupInfos; // Array of SVgroupInfo } SUseDbRsp; int32_t tSerializeSUseDbRsp(void* buf, int32_t bufLen, SUseDbRsp* pRsp); @@ -725,7 +725,7 @@ typedef struct { int32_t vgId; int32_t dnodeId; char db[TSDB_DB_FNAME_LEN]; - uint64_t dbUid; + int64_t dbUid; int32_t vgVersion; int32_t cacheBlockSize; int32_t totalBlocks; @@ -753,10 +753,10 @@ int32_t tSerializeSCreateVnodeReq(void* buf, int32_t bufLen, SCreateVnodeReq* pR int32_t tDeserializeSCreateVnodeReq(void* buf, int32_t bufLen, SCreateVnodeReq* pReq); typedef struct { - int32_t vgId; - int32_t dnodeId; - uint64_t dbUid; - char db[TSDB_DB_FNAME_LEN]; + int32_t vgId; + int32_t dnodeId; + int64_t dbUid; + char db[TSDB_DB_FNAME_LEN]; } SDropVnodeReq, SSyncVnodeReq, SCompactVnodeReq; int32_t tSerializeSDropVnodeReq(void* buf, int32_t bufLen, SDropVnodeReq* pReq); @@ -796,7 +796,7 @@ typedef struct { char tbName[TSDB_TABLE_NAME_LEN]; char stbName[TSDB_TABLE_NAME_LEN]; char dbFName[TSDB_DB_FNAME_LEN]; - uint64_t dbId; + int64_t dbId; int32_t numOfTags; int32_t numOfColumns; int8_t precision; @@ -1268,7 +1268,7 @@ typedef struct { typedef struct { SMsgHead head; char name[TSDB_TABLE_FNAME_LEN]; - uint64_t tuid; + int64_t tuid; int32_t sverson; int32_t execLen; char* executor; @@ -1279,11 +1279,11 @@ typedef struct { typedef struct { SMsgHead head; char name[TSDB_TABLE_FNAME_LEN]; - uint64_t tuid; + int64_t tuid; } SDDropTopicReq; typedef struct SVCreateTbReq { - uint64_t ver; // use a general definition + int64_t ver; // use a general definition char* name; uint32_t ttl; uint32_t keep; @@ -1314,8 +1314,8 @@ int32_t tSerializeSVCreateTbReq(void** buf, SVCreateTbReq* pReq); void* tDeserializeSVCreateTbReq(void* buf, SVCreateTbReq* pReq); typedef struct { - uint64_t ver; // use a general definition - SArray* pArray; + int64_t ver; // use a general definition + SArray* pArray; } SVCreateTbBatchReq; typedef struct { @@ -1325,7 +1325,7 @@ int32_t tSerializeSVCreateTbBatchReq(void** buf, SVCreateTbBatchReq* pReq); void* tDeserializeSVCreateTbBatchReq(void* buf, SVCreateTbBatchReq* pReq); typedef struct { - uint64_t ver; + int64_t ver; char* name; uint8_t type; tb_uid_t suid; @@ -1760,35 +1760,19 @@ typedef struct { char cgroup[TSDB_CONSUMER_GROUP_LEN]; } SMqOffset; -typedef struct { - int32_t vgId; - SArray* offsets; // SArray -} SMqVgOffsets; - typedef struct { int32_t num; SMqOffset* offsets; -} SMqCMResetOffsetReq; +} SMqCMCommitOffsetReq; typedef struct { int32_t reserved; -} SMqCMResetOffsetRsp; - -typedef struct { - int64_t leftForVer; - SMqVgOffsets offsets; -} SMqMVResetOffsetReq; - -typedef struct { - int32_t reserved; -} SMqMVResetOffsetRsp; +} SMqCMCommitOffsetRsp; int32_t tEncodeSMqOffset(SCoder* encoder, const SMqOffset* pOffset); int32_t tDecodeSMqOffset(SCoder* decoder, SMqOffset* pOffset); -int32_t tEncodeSMqCMResetOffsetReq(SCoder* encoder, const SMqCMResetOffsetReq* pReq); -int32_t tDecodeSMqCMResetOffsetReq(SCoder* decoder, SMqCMResetOffsetReq* pReq); -int32_t tEncodeSMqMVResetOffsetReq(SCoder* encoder, const SMqMVResetOffsetReq* pReq); -int32_t tDecodeSMqMVResetOffsetReq(SCoder* decoder, SMqMVResetOffsetReq* pReq); +int32_t tEncodeSMqCMCommitOffsetReq(SCoder* encoder, const SMqCMCommitOffsetReq* pReq); +int32_t tDecodeSMqCMCommitOffsetReq(SCoder* decoder, SMqCMCommitOffsetReq* pReq); typedef struct { uint32_t nCols; @@ -1870,7 +1854,6 @@ typedef struct { typedef struct { int64_t consumerId; SSchemaWrapper* schemas; - int64_t committedOffset; int64_t reqOffset; int64_t rspOffset; int32_t skipLogNum; @@ -1881,22 +1864,18 @@ typedef struct { // one req for one vg+topic typedef struct { SMsgHead head; - // 0: commit only, current offset - // 1: consume only, poll next offset - // 2: commit current and consume next offset - int32_t reqType; - int64_t reqId; int64_t consumerId; int64_t blockingTime; char cgroup[TSDB_CONSUMER_GROUP_LEN]; - int64_t offset; + int64_t currentOffset; char topic[TSDB_TOPIC_FNAME_LEN]; } SMqConsumeReq; typedef struct { int32_t vgId; + int64_t offset; SEpSet epSet; } SMqSubVgEp; @@ -1917,12 +1896,14 @@ static FORCE_INLINE void tDeleteSMqSubTopicEp(SMqSubTopicEp* pSubTopicEp) { taos static FORCE_INLINE int32_t tEncodeSMqSubVgEp(void** buf, const SMqSubVgEp* pVgEp) { int32_t tlen = 0; tlen += taosEncodeFixedI32(buf, pVgEp->vgId); + tlen += taosEncodeFixedI64(buf, pVgEp->offset); tlen += taosEncodeSEpSet(buf, &pVgEp->epSet); return tlen; } static FORCE_INLINE void* tDecodeSMqSubVgEp(void* buf, SMqSubVgEp* pVgEp) { buf = taosDecodeFixedI32(buf, &pVgEp->vgId); + buf = taosDecodeFixedI64(buf, &pVgEp->offset); buf = taosDecodeSEpSet(buf, &pVgEp->epSet); return buf; } diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index 1a63ea73a5..a3e44968f7 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -135,18 +135,19 @@ enum { TD_DEF_MSG_TYPE(TDMT_MND_SHOW, "mnode-show", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_SHOW_RETRIEVE, "mnode-retrieve", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_STATUS, "mnode-status", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_TRANS, "mnode-trans", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_TRANS_TIMER, "mnode-trans-tmr", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_KILL_TRANS, "mnode-kill-trans", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_TELEM_TIMER, "mnode-telem-tmr", SMTimerReq, SMTimerReq) TD_DEF_MSG_TYPE(TDMT_MND_GRANT, "mnode-grant", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_AUTH, "mnode-auth", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_CREATE_TOPIC, "mnode-create-topic", SMCreateTopicReq, SMCreateTopicRsp) TD_DEF_MSG_TYPE(TDMT_MND_ALTER_TOPIC, "mnode-alter-topic", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_DROP_TOPIC, "mnode-drop-topic", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_SUBSCRIBE, "mnode-subscribe", SCMSubscribeReq, SCMSubscribeRsp) - TD_DEF_MSG_TYPE(TDMT_MND_RESET_OFFSET, "mnode-reset-offset", SMqCMResetOffsetReq, SMqCMResetOffsetRsp) TD_DEF_MSG_TYPE(TDMT_MND_GET_SUB_EP, "mnode-get-sub-ep", SMqCMGetSubEpReq, SMqCMGetSubEpRsp) - TD_DEF_MSG_TYPE(TDMT_MND_MQ_TIMER, "mnode-mq-timer", SMTimerReq, SMTimerReq) + TD_DEF_MSG_TYPE(TDMT_MND_MQ_TIMER, "mnode-mq-tmr", SMTimerReq, SMTimerReq) TD_DEF_MSG_TYPE(TDMT_MND_MQ_DO_REBALANCE, "mnode-mq-do-rebalance", SMqDoRebalanceMsg, SMqDoRebalanceMsg) + TD_DEF_MSG_TYPE(TDMT_MND_MQ_COMMIT_OFFSET, "mnode-mq-commit-offset", SMqCMCommitOffsetReq, SMqCMCommitOffsetRsp) // Requests handled by VNODE TD_NEW_MSG_SEG(TDMT_VND_MSG) @@ -194,4 +195,3 @@ enum { TDMT_MAX #endif }; -// clang-format on diff --git a/include/dnode/mgmt/dnode.h b/include/dnode/mgmt/dnode.h index e64d358de4..99fe2457a8 100644 --- a/include/dnode/mgmt/dnode.h +++ b/include/dnode/mgmt/dnode.h @@ -27,15 +27,16 @@ typedef struct SDnode SDnode; /* ------------------------ Environment ------------------ */ typedef struct { - int32_t sver; - int32_t numOfCores; - int16_t numOfCommitThreads; - int8_t enableTelem; - char timezone[TSDB_TIMEZONE_LEN]; - char locale[TSDB_LOCALE_LEN]; - char charset[TSDB_LOCALE_LEN]; - char buildinfo[64]; - char gitinfo[48]; + int32_t sver; + int32_t numOfCores; + uint16_t numOfCommitThreads; + bool enableTelem; + bool printAuth; + char timezone[TSDB_TIMEZONE_LEN]; + char locale[TSDB_LOCALE_LEN]; + char charset[TSDB_LOCALE_LEN]; + char buildinfo[64]; + char gitinfo[48]; } SDnodeEnvCfg; /** @@ -65,6 +66,7 @@ typedef struct { char localEp[TSDB_EP_LEN]; char localFqdn[TSDB_FQDN_LEN]; char firstEp[TSDB_EP_LEN]; + char secondEp[TSDB_EP_LEN]; } SDnodeObjCfg; /** diff --git a/include/dnode/mnode/mnode.h b/include/dnode/mnode/mnode.h index e1306800d3..01855f6d92 100644 --- a/include/dnode/mnode/mnode.h +++ b/include/dnode/mnode/mnode.h @@ -46,7 +46,8 @@ typedef struct SMnodeLoad { typedef struct SMnodeCfg { int32_t sver; - int8_t enableTelem; + bool enableTelem; + bool printAuth; int32_t statusInterval; int32_t shellActivityTimer; char *timezone; diff --git a/include/dnode/mnode/sdb/sdb.h b/include/dnode/mnode/sdb/sdb.h index 1e967a6d2b..bf48a8523c 100644 --- a/include/dnode/mnode/sdb/sdb.h +++ b/include/dnode/mnode/sdb/sdb.h @@ -113,14 +113,15 @@ typedef enum { SDB_USER = 7, SDB_AUTH = 8, SDB_ACCT = 9, - SDB_SUBSCRIBE = 10, - SDB_CONSUMER = 11, - SDB_TOPIC = 12, - SDB_VGROUP = 13, - SDB_STB = 14, - SDB_DB = 15, - SDB_FUNC = 16, - SDB_MAX = 17 + SDB_OFFSET = 10, + SDB_SUBSCRIBE = 11, + SDB_CONSUMER = 12, + SDB_TOPIC = 13, + SDB_VGROUP = 14, + SDB_STB = 15, + SDB_DB = 16, + SDB_FUNC = 17, + SDB_MAX = 18 } ESdbType; typedef struct SSdb SSdb; diff --git a/include/libs/config/config.h b/include/libs/config/config.h new file mode 100644 index 0000000000..3c9ec1e3bf --- /dev/null +++ b/include/libs/config/config.h @@ -0,0 +1,104 @@ + +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_CONFIG_H_ +#define _TD_CONFIG_H_ + +#include "os.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define CFG_NAME_MAX_LEN 128 + +typedef enum { + CFG_STYPE_DEFAULT, + CFG_STYPE_CFG_FILE, + CFG_STYPE_ENV_FILE, + CFG_STYPE_ENV_VAR, + CFG_STYPE_APOLLO_URL, + CFG_STYPE_ARG_LIST, + CFG_STYPE_API_OPTION +} ECfgSrcType; + +typedef enum { + CFG_DTYPE_NONE, + CFG_DTYPE_BOOL, + CFG_DTYPE_INT32, + CFG_DTYPE_INT64, + CFG_DTYPE_FLOAT, + CFG_DTYPE_STRING, + CFG_DTYPE_IPSTR, + CFG_DTYPE_DIR, + CFG_DTYPE_LOCALE, + CFG_DTYPE_CHARSET, + CFG_DTYPE_TIMEZONE +} ECfgDataType; + +typedef struct SConfigItem { + ECfgSrcType stype; + ECfgDataType dtype; + char *name; + union { + bool bval; + float fval; + int32_t i32; + int64_t i64; + char *str; + }; + union { + int64_t imin; + double fmin; + }; + union { + int64_t imax; + double fmax; + }; +} SConfigItem; + +typedef struct SConfig SConfig; + +SConfig *cfgInit(); +int32_t cfgLoad(SConfig *pCfg, ECfgSrcType cfgType, const char *sourceStr); +void cfgCleanup(SConfig *pCfg); + +int32_t cfgGetSize(SConfig *pCfg); +SConfigItem *cfgIterate(SConfig *pCfg, SConfigItem *pIter); +void cfgCancelIterate(SConfig *pCfg, SConfigItem *pIter); +SConfigItem *cfgGetItem(SConfig *pCfg, const char *name); + +int32_t cfgAddBool(SConfig *pCfg, const char *name, bool defaultVal); +int32_t cfgAddInt32(SConfig *pCfg, const char *name, int32_t defaultVal, int64_t minval, int64_t maxval); +int32_t cfgAddInt64(SConfig *pCfg, const char *name, int64_t defaultVal, int64_t minval, int64_t maxval); +int32_t cfgAddFloat(SConfig *pCfg, const char *name, float defaultVal, double minval, double maxval); +int32_t cfgAddString(SConfig *pCfg, const char *name, const char *defaultVal); +int32_t cfgAddIpStr(SConfig *pCfg, const char *name, const char *defaultVa); +int32_t cfgAddDir(SConfig *pCfg, const char *name, const char *defaultVal); +int32_t cfgAddLocale(SConfig *pCfg, const char *name, const char *defaultVal); +int32_t cfgAddCharset(SConfig *pCfg, const char *name, const char *defaultVal); +int32_t cfgAddTimezone(SConfig *pCfg, const char *name, const char *defaultVal); + +const char *cfgStypeStr(ECfgSrcType type); +const char *cfgDtypeStr(ECfgDataType type); + +void cfgDumpCfg(SConfig *pCfg); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_CONFIG_H_*/ diff --git a/include/libs/transport/trpc.h b/include/libs/transport/trpc.h index 538aeb1a0e..f1bab81573 100644 --- a/include/libs/transport/trpc.h +++ b/include/libs/transport/trpc.h @@ -78,6 +78,9 @@ typedef struct SRpcInit { // call back to retrieve the client auth info, for server app only int (*afp)(void *parent, char *tableId, char *spi, char *encrypt, char *secret, char *ckey); + // call back to keep conn or not + bool (*pfp)(void *parent, tmsg_t msgType); + void *parent; } SRpcInit; diff --git a/include/os/osDir.h b/include/os/osDir.h index 35c294af39..01ec24f235 100644 --- a/include/os/osDir.h +++ b/include/os/osDir.h @@ -24,7 +24,7 @@ void taosRemoveDir(const char *dirname); int32_t taosDirExist(char *dirname); int32_t taosMkDir(const char *dirname); void taosRemoveOldFiles(char *dirname, int32_t keepDays); -int32_t taosExpandDir(char *dirname, char *outname, int32_t maxlen); +int32_t taosExpandDir(const char *dirname, char *outname, int32_t maxlen); int32_t taosRealPath(char *dirname, int32_t maxlen); #ifdef __cplusplus diff --git a/include/os/osEnv.h b/include/os/osEnv.h index a7fd86776c..2745520ae3 100644 --- a/include/os/osEnv.h +++ b/include/os/osEnv.h @@ -21,11 +21,24 @@ extern "C" { #endif extern char tsOsName[]; + extern char tsDataDir[]; extern char tsLogDir[]; -extern char tsScriptDir[]; +extern char tsTempDir[]; extern char configDir[]; +extern struct SDiskSpace tsLogSpace; +extern struct SDiskSpace tsTempSpace; +extern struct SDiskSpace tsDataSpace; + +void taosUpdateLogSpace(); +void taosUpdateTempSpace(); +void taosUpdateDataSpace(); +bool taosLogSpaceAvailable(); +bool taosTmpSpaceAvailable(); +bool taosDataSpaceAvailable(); +void taosUpdateAllSpace(); + #ifdef __cplusplus } #endif diff --git a/include/os/osSocket.h b/include/os/osSocket.h index d76b286d65..af99e4b474 100644 --- a/include/os/osSocket.h +++ b/include/os/osSocket.h @@ -55,7 +55,7 @@ void taosSetMaskSIGPIPE(); int32_t taosSetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t optlen); int32_t taosGetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t *optlen); -uint32_t taosInetAddr(char *ipAddr); +uint32_t taosInetAddr(const char *ipAddr); const char *taosInetNtoa(struct in_addr ipInt); #if (defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)) diff --git a/include/os/osSysinfo.h b/include/os/osSysinfo.h index 9dcb075489..6ab2a104df 100644 --- a/include/os/osSysinfo.h +++ b/include/os/osSysinfo.h @@ -25,21 +25,28 @@ extern "C" { #define TSDB_LOCALE_LEN 64 #define TSDB_TIMEZONE_LEN 96 -extern int64_t tsPageSize; -extern int64_t tsOpenMax; -extern int64_t tsStreamMax; -extern int32_t tsNumOfCores; -extern int32_t tsTotalMemoryMB; -extern char tsTimezone[]; -extern char tsLocale[]; -extern char tsCharset[]; // default encode string - typedef struct { int64_t total; int64_t used; int64_t avail; } SDiskSize; +typedef struct SDiskSpace { + int64_t reserved; + SDiskSize size; +} SDiskSpace; + +extern int64_t tsPageSize; +extern int64_t tsOpenMax; +extern int64_t tsStreamMax; +extern int32_t tsNumOfCores; +extern int32_t tsTotalMemoryMB; +extern char tsTimezone[]; +extern char tsLocale[]; +extern char tsCharset[]; // default encode string + + + int32_t taosGetDiskSize(char *dataDir, SDiskSize *diskSize); int32_t taosGetCpuCores(); void taosGetSystemInfo(); diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 92028b0837..2f7d406569 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -20,6 +20,8 @@ extern "C" { #endif +// clang-format off + #define TAOS_DEF_ERROR_CODE(mod, code) ((int32_t)((0x80000000 | ((mod)<<16) | (code)))) #define TAOS_SYSTEM_ERROR(code) (0x80ff0000 | (code)) @@ -71,6 +73,8 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MSG_NOT_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0109) #define TSDB_CODE_INVALID_PARA TAOS_DEF_ERROR_CODE(0, 0x010A) #define TSDB_CODE_REPEAT_INIT TAOS_DEF_ERROR_CODE(0, 0x010B) +#define TSDB_CODE_CFG_NOT_FOUND TAOS_DEF_ERROR_CODE(0, 0x010C) +#define TSDB_CODE_INVALID_CFG TAOS_DEF_ERROR_CODE(0, 0x010D) #define TSDB_CODE_REF_NO_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0110) #define TSDB_CODE_REF_FULL TAOS_DEF_ERROR_CODE(0, 0x0111) @@ -260,6 +264,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_CONSUMER_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03E7) #define TSDB_CODE_MND_UNSUPPORTED_TOPIC TAOS_DEF_ERROR_CODE(0, 0x03E8) #define TSDB_CODE_MND_SUBSCRIBE_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03E9) +#define TSDB_CODE_MND_OFFSET_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03EA) #define TSDB_CODE_MND_MQ_PLACEHOLDER TAOS_DEF_ERROR_CODE(0, 0x03F0) // dnode diff --git a/include/util/tconfig.h b/include/util/tconfig.h deleted file mode 100644 index 7dee01247b..0000000000 --- a/include/util/tconfig.h +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_UTIL_CONFIG_H -#define _TD_UTIL_CONFIG_H - -#ifdef __cplusplus -extern "C" { -#endif - -#define TSDB_CFG_MAX_NUM 115 -#define TSDB_CFG_PRINT_LEN 23 -#define TSDB_CFG_OPTION_LEN 24 -#define TSDB_CFG_VALUE_LEN 41 - -#define TSDB_CFG_CTYPE_B_CONFIG 1U // can be configured from file -#define TSDB_CFG_CTYPE_B_SHOW 2U // can displayed by "show configs" commands -#define TSDB_CFG_CTYPE_B_LOG 4U // is a log type configuration -#define TSDB_CFG_CTYPE_B_CLIENT 8U // can be displayed in the client log -#define TSDB_CFG_CTYPE_B_OPTION 16U // can be configured by taos_options function -#define TSDB_CFG_CTYPE_B_NOT_PRINT 32U // such as password - -#define MAX_FLOAT 100000 -#define MIN_FLOAT 0 - -enum { - TAOS_CFG_CSTATUS_NONE, // not configured - TAOS_CFG_CSTATUS_DEFAULT, // use system default value - TAOS_CFG_CSTATUS_FILE, // configured from file - TAOS_CFG_CSTATUS_OPTION, // configured by taos_options function - TAOS_CFG_CSTATUS_ARG, // configured by program argument -}; - -enum { - TAOS_CFG_VTYPE_INT8, - TAOS_CFG_VTYPE_INT16, - TAOS_CFG_VTYPE_INT32, - TAOS_CFG_VTYPE_UINT16, - TAOS_CFG_VTYPE_FLOAT, - TAOS_CFG_VTYPE_STRING, - TAOS_CFG_VTYPE_IPSTR, - TAOS_CFG_VTYPE_DIRECTORY, - TAOS_CFG_VTYPE_DATA_DIRCTORY, - TAOS_CFG_VTYPE_DOUBLE, -}; - -enum { - TAOS_CFG_UTYPE_NONE, - TAOS_CFG_UTYPE_PERCENT, - TAOS_CFG_UTYPE_GB, - TAOS_CFG_UTYPE_MB, - TAOS_CFG_UTYPE_BYTE, - TAOS_CFG_UTYPE_SECOND, - TAOS_CFG_UTYPE_MS -}; - -typedef struct { - char * option; - void * ptr; - float minValue; - float maxValue; - int8_t cfgType; - int8_t cfgStatus; - int8_t unitType; - int8_t valType; - int32_t ptrLength; -} SGlobalCfg; - -extern SGlobalCfg tsGlobalConfig[]; -extern int32_t tsGlobalConfigNum; -extern char * tsCfgStatusStr[]; - -void taosReadGlobalLogCfg(); -int32_t taosReadCfgFromFile(); -void taosPrintCfg(); -void taosDumpGlobalCfg(); - -void taosAddConfigOption(SGlobalCfg cfg); -SGlobalCfg *taosGetConfigOption(const char *option); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_UTIL_CONFIG_H*/ diff --git a/include/util/tdef.h b/include/util/tdef.h index d7e26e97d8..d0f2b77f1f 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -13,6 +13,8 @@ * along with this program. If not, see . */ +// clang-format off + #ifndef _TD_UTIL_DEF_H #define _TD_UTIL_DEF_H @@ -196,6 +198,7 @@ typedef enum ELogicConditionType { #define TSDB_TOPIC_FNAME_LEN TSDB_TABLE_FNAME_LEN #define TSDB_CONSUMER_GROUP_LEN 192 #define TSDB_SUBSCRIBE_KEY_LEN (TSDB_CONSUMER_GROUP_LEN + TSDB_TOPIC_FNAME_LEN + 2) +#define TSDB_PARTITION_KEY_LEN (TSDB_CONSUMER_GROUP_LEN + TSDB_TOPIC_FNAME_LEN + 2) #define TSDB_COL_NAME_LEN 65 #define TSDB_MAX_SAVED_SQL_LEN TSDB_MAX_COLUMNS * 64 #define TSDB_MAX_SQL_LEN TSDB_PAYLOAD_SIZE @@ -349,9 +352,6 @@ typedef enum ELogicConditionType { #define TSDB_QUERY_TYPE_NON_TYPE 0x00u // none type #define TSDB_QUERY_TYPE_FREE_RESOURCE 0x01u // free qhandle at vnode -#define TSDB_QUERY_TYPE_NON_TYPE 0x00u // none type -#define TSDB_QUERY_TYPE_FREE_RESOURCE 0x01u // free qhandle at vnode - #define TSDB_META_COMPACT_RATIO 0 // disable tsdb meta compact by default diff --git a/include/util/tlog.h b/include/util/tlog.h index 26a5417320..166e186508 100644 --- a/include/util/tlog.h +++ b/include/util/tlog.h @@ -22,8 +22,8 @@ extern "C" { #endif -// log -extern int8_t tsAsyncLog; +extern bool tsLogInited; +extern bool tsAsyncLog; extern int32_t tsNumOfLogLines; extern int32_t tsLogKeepDays; extern int32_t dDebugFlag; @@ -32,9 +32,6 @@ extern int32_t mDebugFlag; extern int32_t cDebugFlag; extern int32_t jniDebugFlag; extern int32_t tmrDebugFlag; -extern int32_t httpDebugFlag; -extern int32_t mqttDebugFlag; -extern int32_t monDebugFlag; extern int32_t uDebugFlag; extern int32_t rpcDebugFlag; extern int32_t qDebugFlag; @@ -42,23 +39,23 @@ extern int32_t wDebugFlag; extern int32_t sDebugFlag; extern int32_t tsdbDebugFlag; extern int32_t tqDebugFlag; -extern int32_t cqDebugFlag; -extern int32_t debugFlag; - -#define DEBUG_FATAL 1U -#define DEBUG_ERROR DEBUG_FATAL -#define DEBUG_WARN 2U -#define DEBUG_INFO DEBUG_WARN -#define DEBUG_DEBUG 4U -#define DEBUG_TRACE 8U -#define DEBUG_DUMP 16U +extern int32_t fsDebugFlag; +#define DEBUG_FATAL 1U +#define DEBUG_ERROR DEBUG_FATAL +#define DEBUG_WARN 2U +#define DEBUG_INFO DEBUG_WARN +#define DEBUG_DEBUG 4U +#define DEBUG_TRACE 8U +#define DEBUG_DUMP 16U #define DEBUG_SCREEN 64U -#define DEBUG_FILE 128U +#define DEBUG_FILE 128U -int32_t taosInitLog(char *logName, int32_t numOfLogLines, int32_t maxFiles); +int32_t taosInitLog(const char *logName, int32_t maxFiles); void taosCloseLog(); void taosResetLog(); +void taosSetAllDebugFlag(int32_t flag); +void taosDumpData(unsigned char *msg, int32_t len); void taosPrintLog(const char *flags, int32_t dflag, const char *format, ...) #ifdef __GNUC__ @@ -72,8 +69,6 @@ void taosPrintLongString(const char *flags, int32_t dflag, const char *format, . #endif ; -void taosDumpData(unsigned char *msg, int32_t len); - #ifdef __cplusplus } #endif diff --git a/include/util/ttimer.h b/include/util/ttimer.h index 89ec6cd8d9..01d70c7d02 100644 --- a/include/util/ttimer.h +++ b/include/util/ttimer.h @@ -26,7 +26,6 @@ typedef void *tmr_h; typedef void (*TAOS_TMR_CALLBACK)(void *, void *); extern int taosTmrThreads; -extern uint32_t tsMaxTmrCtrl; #define MSECONDS_PER_TICK 5 diff --git a/include/util/version.h b/include/util/version.h new file mode 100644 index 0000000000..01efecc59d --- /dev/null +++ b/include/util/version.h @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_UTIL_VERSION_H +#define _TD_UTIL_VERSION_H + +#ifdef __cplusplus +extern "C" { +#endif + +extern char version[]; +extern char compatible_version[]; +extern char gitinfo[]; +extern char gitinfoOfInternal[]; +extern char buildinfo[]; + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_UTIL_VERSION_H*/ diff --git a/source/client/CMakeLists.txt b/source/client/CMakeLists.txt index 3210c0c6de..c793825556 100644 --- a/source/client/CMakeLists.txt +++ b/source/client/CMakeLists.txt @@ -8,7 +8,7 @@ target_include_directories( target_link_libraries( taos INTERFACE api - PRIVATE os util common transport parser planner catalog scheduler function qcom + PRIVATE os util common transport parser planner catalog scheduler function qcom config ) if(${BUILD_TEST}) diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index d7574466cd..60eb0200a7 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -20,17 +20,19 @@ extern "C" { #endif -#include "taos.h" #include "common.h" -#include "tmsg.h" +#include "parser.h" +#include "query.h" +#include "taos.h" #include "tdef.h" #include "tep.h" #include "thash.h" #include "tlist.h" +#include "tmsg.h" #include "tmsgtype.h" #include "trpc.h" -#include "query.h" -#include "parser.h" + +#include "config.h" #define CHECK_CODE_GOTO(expr, label) \ do { \ @@ -46,12 +48,12 @@ extern "C" { typedef struct SAppInstInfo SAppInstInfo; typedef struct SHbConnInfo { - void *param; - SClientHbReq *req; + void* param; + SClientHbReq* req; } SHbConnInfo; typedef struct SAppHbMgr { - char *key; + char* key; // statistics int32_t reportCnt; int32_t connKeyCnt; @@ -62,15 +64,13 @@ typedef struct SAppHbMgr { // connection SAppInstInfo* pAppInstInfo; // info - SHashObj* activeInfo; // hash - SHashObj* connInfo; // hash + SHashObj* activeInfo; // hash + SHashObj* connInfo; // hash } SAppHbMgr; +typedef int32_t (*FHbRspHandle)(struct SAppHbMgr* pAppHbMgr, SClientHbRsp* pRsp); -typedef int32_t (*FHbRspHandle)(struct SAppHbMgr *pAppHbMgr, SClientHbRsp* pRsp); - -typedef int32_t (*FHbReqHandle)(SClientHbKey *connKey, void* param, SClientHbReq *req); - +typedef int32_t (*FHbReqHandle)(SClientHbKey* connKey, void* param, SClientHbReq* req); typedef struct SClientHbMgr { int8_t inited; @@ -83,63 +83,62 @@ typedef struct SClientHbMgr { FHbRspHandle rspHandle[HEARTBEAT_TYPE_MAX]; } SClientHbMgr; - typedef struct SQueryExecMetric { - int64_t start; // start timestamp - int64_t parsed; // start to parse - int64_t send; // start to send to server - int64_t rsp; // receive response from server + int64_t start; // start timestamp + int64_t parsed; // start to parse + int64_t send; // start to send to server + int64_t rsp; // receive response from server } SQueryExecMetric; typedef struct SInstanceSummary { - uint64_t numOfInsertsReq; - uint64_t numOfInsertRows; - uint64_t insertElapsedTime; - uint64_t insertBytes; // submit to tsdb since launched. + uint64_t numOfInsertsReq; + uint64_t numOfInsertRows; + uint64_t insertElapsedTime; + uint64_t insertBytes; // submit to tsdb since launched. - uint64_t fetchBytes; - uint64_t queryElapsedTime; - uint64_t numOfSlowQueries; - uint64_t totalRequests; - uint64_t currentRequests; // the number of SRequestObj + uint64_t fetchBytes; + uint64_t queryElapsedTime; + uint64_t numOfSlowQueries; + uint64_t totalRequests; + uint64_t currentRequests; // the number of SRequestObj } SInstanceSummary; typedef struct SHeartBeatInfo { - void *pTimer; // timer, used to send request msg to mnode + void* pTimer; // timer, used to send request msg to mnode } SHeartBeatInfo; struct SAppInstInfo { - int64_t numOfConns; - SCorEpSet mgmtEp; - SInstanceSummary summary; - SList *pConnList; // STscObj linked list - int64_t clusterId; - void *pTransporter; - struct SAppHbMgr *pAppHbMgr; + int64_t numOfConns; + SCorEpSet mgmtEp; + SInstanceSummary summary; + SList* pConnList; // STscObj linked list + int64_t clusterId; + void* pTransporter; + struct SAppHbMgr* pAppHbMgr; }; typedef struct SAppInfo { int64_t startTime; char appName[TSDB_APP_NAME_LEN]; - char *ep; + char* ep; int32_t pid; int32_t numOfThreads; - SHashObj *pInstMap; + SHashObj* pInstMap; pthread_mutex_t mutex; } SAppInfo; typedef struct STscObj { - char user[TSDB_USER_LEN]; - char pass[TSDB_PASSWORD_LEN]; - char db[TSDB_DB_FNAME_LEN]; - char ver[128]; - int32_t acctId; - uint32_t connId; - int32_t connType; - uint64_t id; // ref ID returned by taosAddRef - pthread_mutex_t mutex; // used to protect the operation on db - int32_t numOfReqs; // number of sqlObj bound to this connection - SAppInstInfo *pAppInfo; + char user[TSDB_USER_LEN]; + char pass[TSDB_PASSWORD_LEN]; + char db[TSDB_DB_FNAME_LEN]; + char ver[128]; + int32_t acctId; + uint32_t connId; + int32_t connType; + uint64_t id; // ref ID returned by taosAddRef + pthread_mutex_t mutex; // used to protect the operation on db + int32_t numOfReqs; // number of sqlObj bound to this connection + SAppInstInfo* pAppInfo; } STscObj; typedef struct SMqConsumer { @@ -147,49 +146,49 @@ typedef struct SMqConsumer { } SMqConsumer; typedef struct SReqResultInfo { - const char *pRspMsg; - const char *pData; - TAOS_FIELD *fields; - uint32_t numOfCols; - int32_t *length; - TAOS_ROW row; - char **pCol; - uint32_t numOfRows; - uint64_t totalRows; - uint32_t current; - bool completed; + const char* pRspMsg; + const char* pData; + TAOS_FIELD* fields; + uint32_t numOfCols; + int32_t* length; + TAOS_ROW row; + char** pCol; + uint32_t numOfRows; + uint64_t totalRows; + uint32_t current; + bool completed; } SReqResultInfo; typedef struct SShowReqInfo { - int64_t execId; // showId/queryId - int32_t vgId; - SArray *pArray; // SArray - int32_t currentIndex; // current accessed vgroup index. + int64_t execId; // showId/queryId + int32_t vgId; + SArray* pArray; // SArray + int32_t currentIndex; // current accessed vgroup index. } SShowReqInfo; typedef struct SRequestSendRecvBody { - tsem_t rspSem; // not used now + tsem_t rspSem; // not used now void* fp; - SShowReqInfo showInfo; // todo this attribute will be removed after the query framework being completed. + SShowReqInfo showInfo; // todo this attribute will be removed after the query framework being completed. SDataBuf requestMsg; - struct SSchJob *pQueryJob; // query job, created according to sql query DAG. - struct SQueryDag *pDag; // the query dag, generated according to the sql statement. + struct SSchJob* pQueryJob; // query job, created according to sql query DAG. + struct SQueryDag* pDag; // the query dag, generated according to the sql statement. SReqResultInfo resInfo; } SRequestSendRecvBody; -#define ERROR_MSG_BUF_DEFAULT_SIZE 512 +#define ERROR_MSG_BUF_DEFAULT_SIZE 512 typedef struct SRequestObj { - uint64_t requestId; - int32_t type; // request type - STscObj *pTscObj; - char *sqlstr; // sql string - int32_t sqlLen; - int64_t self; - char *msgBuf; - void *pInfo; // sql parse info, generated by parser module - int32_t code; - SQueryExecMetric metric; + uint64_t requestId; + int32_t type; // request type + STscObj* pTscObj; + char* sqlstr; // sql string + int32_t sqlLen; + int64_t self; + char* msgBuf; + void* pInfo; // sql parse info, generated by parser module + int32_t code; + SQueryExecMetric metric; SRequestSendRecvBody body; } SRequestObj; @@ -198,51 +197,52 @@ extern int32_t clientReqRefPool; extern int32_t clientConnRefPool; extern int (*handleRequestRspFp[TDMT_MAX])(void*, const SDataBuf* pMsg, int32_t code); -int genericRspCallback(void* param, const SDataBuf* pMsg, int32_t code); +int genericRspCallback(void* param, const SDataBuf* pMsg, int32_t code); SMsgSendInfo* buildMsgInfoImpl(SRequestObj* pReqObj); -int taos_init(); +int taos_init(); -void* createTscObj(const char* user, const char* auth, const char *db, SAppInstInfo* pAppInfo); -void destroyTscObj(void*pObj); +void* createTscObj(const char* user, const char* auth, const char* db, SAppInstInfo* pAppInfo); +void destroyTscObj(void* pObj); uint64_t generateRequestId(); -void *createRequest(STscObj* pObj, __taos_async_fn_t fp, void* param, int32_t type); +void* createRequest(STscObj* pObj, __taos_async_fn_t fp, void* param, int32_t type); void destroyRequest(SRequestObj* pRequest); -char *getDbOfConnection(STscObj* pObj); +char* getDbOfConnection(STscObj* pObj); void setConnectionDB(STscObj* pTscObj, const char* db); void taos_init_imp(void); -int taos_options_imp(TSDB_OPTION option, const char *str); +int taos_options_imp(TSDB_OPTION option, const char* str); -void* openTransporter(const char *user, const char *auth, int32_t numOfThreads); +void* openTransporter(const char* user, const char* auth, int32_t numOfThreads); +bool persistConnForSpecificMsg(void* parenct, tmsg_t msgType); void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet); void initMsgHandleFp(); -TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass, const char *auth, const char *db, uint16_t port); +TAOS* taos_connect_internal(const char* ip, const char* user, const char* pass, const char* auth, const char* db, + uint16_t port); -void *doFetchRow(SRequestObj* pRequest); +void* doFetchRow(SRequestObj* pRequest); -void setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32_t numOfCols, int32_t numOfRows); +void setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32_t numOfCols, int32_t numOfRows); - -int32_t buildRequest(STscObj *pTscObj, const char *sql, int sqlLen, SRequestObj** pRequest); +int32_t buildRequest(STscObj* pTscObj, const char* sql, int sqlLen, SRequestObj** pRequest); int32_t parseSql(SRequestObj* pRequest, SQueryNode** pQuery); -// --- heartbeat +// --- heartbeat // global, called by mgmt int hbMgrInit(); void hbMgrCleanUp(); int hbHandleRsp(SClientHbBatchRsp* hbRsp); // cluster level -SAppHbMgr* appHbMgrInit(SAppInstInfo* pAppInstInfo, char *key); -void appHbMgrCleanup(void); +SAppHbMgr* appHbMgrInit(SAppInstInfo* pAppInstInfo, char* key); +void appHbMgrCleanup(void); // conn level int hbRegisterConn(SAppHbMgr* pAppHbMgr, int32_t connId, int64_t clusterId, int32_t hbType); @@ -254,6 +254,12 @@ int hbAddConnInfo(SAppHbMgr* pAppHbMgr, SClientHbKey connKey, void* key, void* v void hbMgrInitMqHbRspHandle(); +// config +int32_t tscInitLog(const char *cfgDir, const char *envFile, const char *apolloUrl); +int32_t tscInitCfg(const char *cfgDir, const char *envFile, const char *apolloUrl); + +extern SConfig *tscCfg; + #ifdef __cplusplus } #endif diff --git a/source/client/src/clientCfg.c b/source/client/src/clientCfg.c new file mode 100644 index 0000000000..22b1407a15 --- /dev/null +++ b/source/client/src/clientCfg.c @@ -0,0 +1,208 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "clientInt.h" +#include "ulog.h" + +// todo refact +SConfig *tscCfg; + +static int32_t tscLoadCfg(SConfig *pConfig, const char *inputCfgDir, const char *envFile, const char *apolloUrl) { + char cfgDir[PATH_MAX] = {0}; + char cfgFile[PATH_MAX + 100] = {0}; + + taosExpandDir(inputCfgDir, cfgDir, PATH_MAX); + snprintf(cfgFile, sizeof(cfgFile), "%s" TD_DIRSEP "taos.cfg", cfgDir); + + if (cfgLoad(pConfig, CFG_STYPE_APOLLO_URL, apolloUrl) != 0) { + uError("failed to load from apollo url:%s since %s\n", apolloUrl, terrstr()); + return -1; + } + + if (cfgLoad(pConfig, CFG_STYPE_CFG_FILE, cfgFile) != 0) { + if (cfgLoad(pConfig, CFG_STYPE_CFG_FILE, cfgDir) != 0) { + uError("failed to load from config file:%s since %s\n", cfgFile, terrstr()); + return -1; + } + } + + if (cfgLoad(pConfig, CFG_STYPE_ENV_FILE, envFile) != 0) { + uError("failed to load from env file:%s since %s\n", envFile, terrstr()); + return -1; + } + + if (cfgLoad(pConfig, CFG_STYPE_ENV_VAR, NULL) != 0) { + uError("failed to load from global env variables since %s\n", terrstr()); + return -1; + } + + return 0; +} + +static int32_t tscAddLogCfg(SConfig *pCfg) { + if (cfgAddDir(pCfg, "logDir", "/var/log/taos") != 0) return -1; + if (cfgAddBool(pCfg, "asyncLog", 1) != 0) return -1; + if (cfgAddInt32(pCfg, "numOfLogLines", 10000000, 1000, 2000000000) != 0) return -1; + if (cfgAddInt32(pCfg, "logKeepDays", 0, -365000, 365000) != 0) return -1; + if (cfgAddInt32(pCfg, "debugFlag", 0, 0, 255) != 0) return -1; + if (cfgAddInt32(pCfg, "cDebugFlag", 0, 0, 255) != 0) return -1; + if (cfgAddInt32(pCfg, "jniDebugFlag", 0, 0, 255) != 0) return -1; + if (cfgAddInt32(pCfg, "tmrDebugFlag", 0, 0, 255) != 0) return -1; + if (cfgAddInt32(pCfg, "uDebugFlag", 0, 0, 255) != 0) return -1; + if (cfgAddInt32(pCfg, "rpcDebugFlag", 0, 0, 255) != 0) return -1; + return 0; +} + +static int32_t tscSetLogCfg(SConfig *pCfg) { + tstrncpy(tsLogDir, cfgGetItem(pCfg, "logDir")->str, PATH_MAX); + tsAsyncLog = cfgGetItem(pCfg, "asyncLog")->bval; + tsNumOfLogLines = cfgGetItem(pCfg, "numOfLogLines")->i32; + tsLogKeepDays = cfgGetItem(pCfg, "logKeepDays")->i32; + cDebugFlag = cfgGetItem(pCfg, "cDebugFlag")->i32; + jniDebugFlag = cfgGetItem(pCfg, "jniDebugFlag")->i32; + tmrDebugFlag = cfgGetItem(pCfg, "tmrDebugFlag")->i32; + uDebugFlag = cfgGetItem(pCfg, "uDebugFlag")->i32; + rpcDebugFlag = cfgGetItem(pCfg, "rpcDebugFlag")->i32; + + int32_t debugFlag = cfgGetItem(pCfg, "debugFlag")->i32; + taosSetAllDebugFlag(debugFlag); + return 0; +} + +int32_t tscInitLog(const char *cfgDir, const char *envFile, const char *apolloUrl) { + if (tsLogInited) return 0; + + SConfig *pCfg = cfgInit(); + if (pCfg == NULL) return -1; + + if (tscAddLogCfg(pCfg) != 0) { + printf("failed to add log cfg since %s\n", terrstr()); + cfgCleanup(pCfg); + return -1; + } + + if (tscLoadCfg(pCfg, cfgDir, envFile, apolloUrl) != 0) { + printf("failed to load log cfg since %s\n", terrstr()); + cfgCleanup(pCfg); + return -1; + } + + if (tscSetLogCfg(pCfg) != 0) { + printf("failed to set log cfg since %s\n", terrstr()); + cfgCleanup(pCfg); + return -1; + } + + const int32_t maxLogFileNum = 10; + if (taosInitLog("taoslog", maxLogFileNum) != 0) { + printf("failed to init log file since %s\n", terrstr()); + cfgCleanup(pCfg); + return -1; + } + + cfgDumpCfg(pCfg); + cfgCleanup(pCfg); + return 0; +} + +static int32_t tscAddEpCfg(SConfig *pCfg) { + char defaultFqdn[TSDB_FQDN_LEN] = {0}; + if (taosGetFqdn(defaultFqdn) != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + if (cfgAddString(pCfg, "fqdn", defaultFqdn) != 0) return -1; + + int32_t defaultServerPort = 6030; + if (cfgAddInt32(pCfg, "serverPort", defaultServerPort, 1, 65056) != 0) return -1; + + char defaultFirstEp[TSDB_EP_LEN] = {0}; + char defaultSecondEp[TSDB_EP_LEN] = {0}; + snprintf(defaultFirstEp, TSDB_EP_LEN, "%s:%d", defaultFqdn, defaultServerPort); + snprintf(defaultSecondEp, TSDB_EP_LEN, "%s:%d", defaultFqdn, defaultServerPort); + if (cfgAddString(pCfg, "firstEp", defaultFirstEp) != 0) return -1; + if (cfgAddString(pCfg, "secondEp", defaultSecondEp) != 0) return -1; + + return 0; +} + +static int32_t tscAddCfg(SConfig *pCfg) { + if (tscAddEpCfg(pCfg) != 0) return -1; + + + // if (cfgAddString(pCfg, "buildinfo", buildinfo) != 0) return -1; + // if (cfgAddString(pCfg, "gitinfo", gitinfo) != 0) return -1; + // if (cfgAddString(pCfg, "version", version) != 0) return -1; + + // if (cfgAddDir(pCfg, "dataDir", tsDataDir) != 0) return -1; + if (cfgAddTimezone(pCfg, "timezone", "") != 0) return -1; + if (cfgAddLocale(pCfg, "locale", "") != 0) return -1; + if (cfgAddCharset(pCfg, "charset", "") != 0) return -1; + if (cfgAddInt32(pCfg, "numOfCores", 1, 1, 100000) != 0) return -1; + if (cfgAddInt32(pCfg, "numOfCommitThreads", 4, 1, 1000) != 0) return -1; + // if (cfgAddBool(pCfg, "telemetryReporting", 0) != 0) return -1; + if (cfgAddBool(pCfg, "enableCoreFile", 0) != 0) return -1; + // if (cfgAddInt32(pCfg, "supportVnodes", 256, 0, 65536) != 0) return -1; + if (cfgAddInt32(pCfg, "statusInterval", 1, 1, 30) != 0) return -1; + if (cfgAddFloat(pCfg, "numOfThreadsPerCore", 1, 0, 10) != 0) return -1; + if (cfgAddFloat(pCfg, "ratioOfQueryCores", 1, 0, 5) != 0) return -1; + if (cfgAddInt32(pCfg, "shellActivityTimer", 3, 1, 120) != 0) return -1; + + if (cfgAddInt32(pCfg, "maxConnections", 50000, 1, 100000) != 0) return -1; + return 0; +} + +int32_t tscCheckCfg(SConfig *pCfg) { + bool enableCore = cfgGetItem(pCfg, "enableCoreFile")->bval; + taosSetCoreDump(enableCore); + + + + return 0; +} + +SConfig *tscInitCfgImp(const char *cfgDir, const char *envFile, const char *apolloUrl) { + SConfig *pCfg = cfgInit(); + if (pCfg == NULL) return NULL; + + if (tscAddCfg(pCfg) != 0) { + uError("failed to init tsc cfg since %s", terrstr()); + cfgCleanup(pCfg); + return NULL; + } + + if (tscLoadCfg(pCfg, cfgDir, envFile, apolloUrl) != 0) { + printf("failed to load tsc cfg since %s\n", terrstr()); + cfgCleanup(pCfg); + return NULL; + } + + if (tscCheckCfg(pCfg) != 0) { + uError("failed to check cfg since %s", terrstr()); + cfgCleanup(pCfg); + return NULL; + } + + cfgDumpCfg(pCfg); + return pCfg; +} + +int32_t tscInitCfg(const char *cfgDir, const char *envFile, const char *apolloUrl) { + tscCfg = tscInitCfgImp(cfgDir, envFile, apolloUrl); + if (tscCfg == NULL) return -1; + + return 0; +} \ No newline at end of file diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index cb73701bfa..70788ec2c9 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -13,16 +13,15 @@ * along with this program. If not, see . */ -#include "os.h" #include "catalog.h" #include "clientInt.h" #include "clientLog.h" +#include "os.h" #include "query.h" #include "scheduler.h" -#include "tmsg.h" #include "tcache.h" -#include "tconfig.h" #include "tglobal.h" +#include "tmsg.h" #include "tnote.h" #include "tref.h" #include "trpc.h" @@ -30,16 +29,16 @@ #include "ttimezone.h" #define TSC_VAR_NOT_RELEASE 1 -#define TSC_VAR_RELEASED 0 +#define TSC_VAR_RELEASED 0 -SAppInfo appInfo; -int32_t clientReqRefPool = -1; -int32_t clientConnRefPool = -1; +SAppInfo appInfo; +int32_t clientReqRefPool = -1; +int32_t clientConnRefPool = -1; static pthread_once_t tscinit = PTHREAD_ONCE_INIT; -volatile int32_t tscInitRes = 0; +volatile int32_t tscInitRes = 0; -static void registerRequest(SRequestObj* pRequest) { +static void registerRequest(SRequestObj *pRequest) { STscObj *pTscObj = (STscObj *)taosAcquireRef(clientConnRefPool, pRequest->pTscObj->id); assert(pTscObj != NULL); @@ -53,69 +52,58 @@ static void registerRequest(SRequestObj* pRequest) { int32_t total = atomic_add_fetch_32(&pSummary->totalRequests, 1); int32_t currentInst = atomic_add_fetch_32(&pSummary->currentRequests, 1); - tscDebug("0x%" PRIx64 " new Request from connObj:0x%" PRIx64 ", current:%d, app current:%d, total:%d, reqId:0x%"PRIx64, pRequest->self, - pRequest->pTscObj->id, num, currentInst, total, pRequest->requestId); + tscDebug("0x%" PRIx64 " new Request from connObj:0x%" PRIx64 + ", current:%d, app current:%d, total:%d, reqId:0x%" PRIx64, + pRequest->self, pRequest->pTscObj->id, num, currentInst, total, pRequest->requestId); } } -static void deregisterRequest(SRequestObj* pRequest) { +static void deregisterRequest(SRequestObj *pRequest) { assert(pRequest != NULL); - STscObj* pTscObj = pRequest->pTscObj; - SInstanceSummary* pActivity = &pTscObj->pAppInfo->summary; + STscObj * pTscObj = pRequest->pTscObj; + SInstanceSummary *pActivity = &pTscObj->pAppInfo->summary; int32_t currentInst = atomic_sub_fetch_32(&pActivity->currentRequests, 1); int32_t num = atomic_sub_fetch_32(&pTscObj->numOfReqs, 1); int64_t duration = taosGetTimestampMs() - pRequest->metric.start; - tscDebug("0x%"PRIx64" free Request from connObj: 0x%"PRIx64", reqId:0x%"PRIx64" elapsed:%"PRIu64" ms, current:%d, app current:%d", pRequest->self, pTscObj->id, - pRequest->requestId, duration, num, currentInst); + tscDebug("0x%" PRIx64 " free Request from connObj: 0x%" PRIx64 ", reqId:0x%" PRIx64 " elapsed:%" PRIu64 + " ms, current:%d, app current:%d", + pRequest->self, pTscObj->id, pRequest->requestId, duration, num, currentInst); taosReleaseRef(clientConnRefPool, pTscObj->id); } -static void tscInitLogFile() { - taosReadGlobalLogCfg(); - if (mkdir(tsLogDir, 0755) != 0 && errno != EEXIST) { - printf("failed to create log dir:%s\n", tsLogDir); - } - const char *defaultLogFileNamePrefix = "taoslog"; - const int32_t maxLogFileNum = 10; - - char temp[128] = {0}; - sprintf(temp, "%s/%s", tsLogDir, defaultLogFileNamePrefix); - if (taosInitLog(temp, tsNumOfLogLines, maxLogFileNum) < 0) { - printf("failed to open log file in directory:%s\n", tsLogDir); - } -} // todo close the transporter properly -void closeTransporter(STscObj* pTscObj) { +void closeTransporter(STscObj *pTscObj) { if (pTscObj == NULL || pTscObj->pAppInfo->pTransporter == NULL) { return; } - tscDebug("free transporter:%p in connObj: 0x%"PRIx64, pTscObj->pAppInfo->pTransporter, pTscObj->id); + tscDebug("free transporter:%p in connObj: 0x%" PRIx64, pTscObj->pAppInfo->pTransporter, pTscObj->id); rpcClose(pTscObj->pAppInfo->pTransporter); } // TODO refactor -void* openTransporter(const char *user, const char *auth, int32_t numOfThread) { +void *openTransporter(const char *user, const char *auth, int32_t numOfThread) { SRpcInit rpcInit; memset(&rpcInit, 0, sizeof(rpcInit)); rpcInit.localPort = 0; rpcInit.label = "TSC"; rpcInit.numOfThreads = numOfThread; rpcInit.cfp = processMsgFromServer; - rpcInit.sessions = tsMaxConnections; + rpcInit.pfp = persistConnForSpecificMsg; + rpcInit.sessions = cfgGetItem(tscCfg, "maxConnections")->i32; rpcInit.connType = TAOS_CONN_CLIENT; rpcInit.user = (char *)user; - rpcInit.idleTime = tsShellActivityTimer * 1000; + rpcInit.idleTime = cfgGetItem(tscCfg, "shellActivityTimer")->i32 * 1000; rpcInit.ckey = "key"; rpcInit.spi = 1; rpcInit.secret = (char *)auth; - void* pDnodeConn = rpcOpen(&rpcInit); + void *pDnodeConn = rpcOpen(&rpcInit); if (pDnodeConn == NULL) { tscError("failed to init connection to server"); return NULL; @@ -130,12 +118,12 @@ void destroyTscObj(void *pObj) { SClientHbKey connKey = {.connId = pTscObj->connId, .hbType = pTscObj->connType}; hbDeregisterConn(pTscObj->pAppInfo->pAppHbMgr, connKey); atomic_sub_fetch_64(&pTscObj->pAppInfo->numOfConns, 1); - tscDebug("connObj 0x%"PRIx64" destroyed, totalConn:%"PRId64, pTscObj->id, pTscObj->pAppInfo->numOfConns); + tscDebug("connObj 0x%" PRIx64 " destroyed, totalConn:%" PRId64, pTscObj->id, pTscObj->pAppInfo->numOfConns); pthread_mutex_destroy(&pTscObj->mutex); tfree(pTscObj); } -void* createTscObj(const char* user, const char* auth, const char *db, SAppInstInfo* pAppInfo) { +void *createTscObj(const char *user, const char *auth, const char *db, SAppInstInfo *pAppInfo) { STscObj *pObj = (STscObj *)calloc(1, sizeof(STscObj)); if (NULL == pObj) { terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; @@ -153,11 +141,11 @@ void* createTscObj(const char* user, const char* auth, const char *db, SAppInstI pthread_mutex_init(&pObj->mutex, NULL); pObj->id = taosAddRef(clientConnRefPool, pObj); - tscDebug("connObj created, 0x%"PRIx64, pObj->id); + tscDebug("connObj created, 0x%" PRIx64, pObj->id); return pObj; } -void* createRequest(STscObj* pObj, __taos_async_fn_t fp, void* param, int32_t type) { +void *createRequest(STscObj *pObj, __taos_async_fn_t fp, void *param, int32_t type) { assert(pObj != NULL); SRequestObj *pRequest = (SRequestObj *)calloc(1, sizeof(SRequestObj)); @@ -166,20 +154,20 @@ void* createRequest(STscObj* pObj, __taos_async_fn_t fp, void* param, int32_t ty return NULL; } - pRequest->requestId = generateRequestId(); + pRequest->requestId = generateRequestId(); pRequest->metric.start = taosGetTimestampMs(); - pRequest->type = type; - pRequest->pTscObj = pObj; - pRequest->body.fp = fp; // not used it yet - pRequest->msgBuf = calloc(1, ERROR_MSG_BUF_DEFAULT_SIZE); + pRequest->type = type; + pRequest->pTscObj = pObj; + pRequest->body.fp = fp; // not used it yet + pRequest->msgBuf = calloc(1, ERROR_MSG_BUF_DEFAULT_SIZE); tsem_init(&pRequest->body.rspSem, 0, 0); registerRequest(pRequest); return pRequest; } -static void doFreeReqResultInfo(SReqResultInfo* pResInfo) { +static void doFreeReqResultInfo(SReqResultInfo *pResInfo) { tfree(pResInfo->pRspMsg); tfree(pResInfo->length); tfree(pResInfo->row); @@ -187,9 +175,9 @@ static void doFreeReqResultInfo(SReqResultInfo* pResInfo) { tfree(pResInfo->fields); } -static void doDestroyRequest(void* p) { +static void doDestroyRequest(void *p) { assert(p != NULL); - SRequestObj* pRequest = (SRequestObj*)p; + SRequestObj *pRequest = (SRequestObj *)p; assert(RID_VALID(pRequest->self)); @@ -208,7 +196,7 @@ static void doDestroyRequest(void* p) { tfree(pRequest); } -void destroyRequest(SRequestObj* pRequest) { +void destroyRequest(SRequestObj *pRequest) { if (pRequest == NULL) { return; } @@ -225,11 +213,13 @@ void taos_init_imp(void) { srand(taosGetTimestampSec()); deltaToUtcInitOnce(); - taosInitGlobalCfg(); - taosReadCfgFromFile(); - tscInitLogFile(); - if (taosCheckAndPrintCfg()) { + if (tscInitLog(configDir, NULL, NULL) != 0) { + tscInitRes = -1; + return; + } + + if (tscInitCfg(configDir, NULL, NULL) != 0) { tscInitRes = -1; return; } @@ -245,21 +235,21 @@ void taos_init_imp(void) { SSchedulerCfg scfg = {.maxJobNum = 100}; schedulerInit(&scfg); - tscDebug("starting to initialize TAOS driver, local ep: %s", tsLocalEp); + tscDebug("starting to initialize TAOS driver"); taosSetCoreDump(true); initTaskQueue(); clientConnRefPool = taosOpenRef(200, destroyTscObj); - clientReqRefPool = taosOpenRef(40960, doDestroyRequest); + clientReqRefPool = taosOpenRef(40960, doDestroyRequest); taosGetAppName(appInfo.appName, NULL); pthread_mutex_init(&appInfo.mutex, NULL); - appInfo.pid = taosGetPId(); + appInfo.pid = taosGetPId(); appInfo.startTime = taosGetTimestampMs(); - appInfo.pInstMap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + appInfo.pInstMap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); tscDebug("client is initialized successfully"); } @@ -269,6 +259,7 @@ int taos_init() { } int taos_options_imp(TSDB_OPTION option, const char *str) { +#if 0 SGlobalCfg *cfg = NULL; switch (option) { @@ -281,7 +272,8 @@ int taos_options_imp(TSDB_OPTION option, const char *str) { cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION; tscInfo("set config file directory:%s", str); } else { - tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, str, tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr); + tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, str, + tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr); } break; @@ -296,7 +288,8 @@ int taos_options_imp(TSDB_OPTION option, const char *str) { cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION; tscInfo("set shellActivityTimer:%d", tsShellActivityTimer); } else { - tscWarn("config option:%s, input value:%s, is configured by %s, use %d", cfg->option, str, tsCfgStatusStr[cfg->cfgStatus], *(int32_t *)cfg->ptr); + tscWarn("config option:%s, input value:%s, is configured by %s, use %d", cfg->option, str, + tsCfgStatusStr[cfg->cfgStatus], *(int32_t *)cfg->ptr); } break; @@ -313,8 +306,8 @@ int taos_options_imp(TSDB_OPTION option, const char *str) { if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_OPTION) { char sep = '.'; - if (strlen(tsLocale) == 0) { // locale does not set yet - char* defaultLocale = setlocale(LC_CTYPE, ""); + if (strlen(tsLocale) == 0) { // locale does not set yet + char *defaultLocale = setlocale(LC_CTYPE, ""); // The locale of the current OS does not be set correctly, so the default locale cannot be acquired. // The launch of current system will abort soon. @@ -329,10 +322,10 @@ int taos_options_imp(TSDB_OPTION option, const char *str) { // set the user specified locale char *locale = setlocale(LC_CTYPE, str); - if (locale != NULL) { // failed to set the user specified locale + if (locale != NULL) { // failed to set the user specified locale tscInfo("locale set, prev locale:%s, new locale:%s", tsLocale, locale); cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION; - } else { // set the user specified locale failed, use default LC_CTYPE as current locale + } else { // set the user specified locale failed, use default LC_CTYPE as current locale locale = setlocale(LC_CTYPE, tsLocale); tscInfo("failed to set locale:%s, current locale:%s", str, tsLocale); } @@ -360,11 +353,12 @@ int taos_options_imp(TSDB_OPTION option, const char *str) { } free(charset); - } else { // it may be windows system + } else { // it may be windows system tscInfo("charset remains:%s", tsCharset); } } else { - tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, str, tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr); + tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, str, + tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr); } break; } @@ -394,7 +388,8 @@ int taos_options_imp(TSDB_OPTION option, const char *str) { tscInfo("charset:%s not valid", str); } } else { - tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, str, tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr); + tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, str, + tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr); } break; @@ -410,7 +405,8 @@ int taos_options_imp(TSDB_OPTION option, const char *str) { cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION; tscDebug("timezone set:%s, input:%s by taos_options", tsTimezone, str); } else { - tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, str, tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr); + tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, str, + tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr); } break; @@ -419,7 +415,7 @@ int taos_options_imp(TSDB_OPTION option, const char *str) { tscError("Invalid option %d", option); return -1; } - +#endif return 0; } @@ -434,7 +430,7 @@ int taos_options_imp(TSDB_OPTION option, const char *str) { */ uint64_t generateRequestId() { static uint64_t hashId = 0; - static int32_t requestSerialId = 0; + static int32_t requestSerialId = 0; if (hashId == 0) { char uid[64] = {0}; @@ -448,9 +444,9 @@ uint64_t generateRequestId() { } } - int64_t ts = taosGetTimestampMs(); - uint64_t pid = taosGetPId(); - int32_t val = atomic_add_fetch_32(&requestSerialId, 1); + int64_t ts = taosGetTimestampMs(); + uint64_t pid = taosGetPId(); + int32_t val = atomic_add_fetch_32(&requestSerialId, 1); uint64_t id = ((hashId & 0x0FFF) << 52) | ((pid & 0x0FFF) << 40) | ((ts & 0xFFFFFF) << 16) | (val & 0xFFFF); return id; diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 2eff353c39..68f65cebc4 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -12,7 +12,7 @@ #include "tpagedbuf.h" #include "tref.h" -static int32_t initEpSetFromCfg(const char* firstEp, const char* secondEp, SCorEpSet* pEpSet); +static int32_t initEpSetFromCfg(const char* ip, uint16_t port, SCorEpSet* pEpSet); static SMsgSendInfo* buildConnectMsg(SRequestObj* pRequest); static void destroySendMsgInfo(SMsgSendInfo* pMsgBody); static void setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp); @@ -81,19 +81,7 @@ TAOS* taos_connect_internal(const char* ip, const char* user, const char* pass, } SCorEpSet epSet = {0}; - if (ip) { - if (initEpSetFromCfg(ip, NULL, &epSet) < 0) { - return NULL; - } - - if (port) { - epSet.epSet.eps[0].port = port; - } - } else { - if (initEpSetFromCfg(tsFirst, tsSecond, &epSet) < 0) { - return NULL; - } - } + initEpSetFromCfg(ip, port, &epSet); char* key = getClusterKey(user, secretEncrypt, ip, port); SAppInstInfo** pInst = NULL; @@ -282,32 +270,40 @@ _return: return pRequest; } -int initEpSetFromCfg(const char* firstEp, const char* secondEp, SCorEpSet* pEpSet) { - pEpSet->version = 0; +int initEpSetFromCfg(const char* ip, uint16_t port, SCorEpSet* pEpSet) { + SConfigItem* pFirst = cfgGetItem(tscCfg, "firstEp"); + SConfigItem* pSecond = cfgGetItem(tscCfg, "secondEp"); + SConfigItem* pPort = cfgGetItem(tscCfg, "serverPort"); // init mnode ip set SEpSet* mgmtEpSet = &(pEpSet->epSet); mgmtEpSet->numOfEps = 0; mgmtEpSet->inUse = 0; + pEpSet->version = 0; - if (firstEp && firstEp[0] != 0) { - if (strlen(firstEp) >= TSDB_EP_LEN) { - terrno = TSDB_CODE_TSC_INVALID_FQDN; - return -1; - } - - taosGetFqdnPortFromEp(firstEp, &mgmtEpSet->eps[0]); + if (ip != NULL) { + taosGetFqdnPortFromEp(ip, (uint16_t)pPort->i32, &mgmtEpSet->eps[0]); mgmtEpSet->numOfEps++; - } - - if (secondEp && secondEp[0] != 0) { - if (strlen(secondEp) >= TSDB_EP_LEN) { - terrno = TSDB_CODE_TSC_INVALID_FQDN; - return -1; + if (port) { + mgmtEpSet->eps[0].port = port; + } + } else { + if (pFirst->str[0] != 0) { + if (strlen(pFirst->str) >= TSDB_EP_LEN) { + terrno = TSDB_CODE_TSC_INVALID_FQDN; + return -1; + } + taosGetFqdnPortFromEp(pFirst->str, (uint16_t)pPort->i32, &mgmtEpSet->eps[0]); + mgmtEpSet->numOfEps++; + } + if (pSecond->str[0] != 0) { + if (strlen(pSecond->str) >= TSDB_EP_LEN) { + terrno = TSDB_CODE_TSC_INVALID_FQDN; + return -1; + } + taosGetFqdnPortFromEp(pSecond->str, (uint16_t)pPort->i32, &mgmtEpSet->eps[1]); + mgmtEpSet->numOfEps++; } - - taosGetFqdnPortFromEp(secondEp, &mgmtEpSet->eps[mgmtEpSet->numOfEps]); - mgmtEpSet->numOfEps++; } if (mgmtEpSet->numOfEps == 0) { @@ -370,7 +366,6 @@ static SMsgSendInfo* buildConnectMsg(SRequestObj* pRequest) { pMsgSendInfo->fp = handleRequestRspFp[TMSG_INDEX(pMsgSendInfo->msgType)]; pMsgSendInfo->param = pRequest; - SConnectReq connectReq = {0}; STscObj* pObj = pRequest->pTscObj; @@ -398,7 +393,9 @@ static void destroySendMsgInfo(SMsgSendInfo* pMsgBody) { tfree(pMsgBody->msgInfo.pData); tfree(pMsgBody); } - +bool persistConnForSpecificMsg(void* parenct, tmsg_t msgType) { + return msgType == TDMT_VND_QUERY_RSP || msgType == TDMT_VND_FETCH_RSP || msgType == TDMT_VND_RES_READY_RSP; +} void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) { SMsgSendInfo* pSendInfo = (SMsgSendInfo*)pMsg->ahandle; assert(pMsg->ahandle != NULL); diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index b29813be03..5bcc287116 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -7,6 +7,7 @@ #include "tmsg.h" #include "tglobal.h" #include "catalog.h" +#include "version.h" #define TSC_VAR_NOT_RELEASE 1 #define TSC_VAR_RELEASED 0 @@ -56,9 +57,7 @@ void taos_cleanup(void) { } TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port) { - int32_t p = (port != 0) ? port : tsServerPort; - - tscDebug("try to connect to %s:%u, user:%s db:%s", ip, p, user, db); + tscDebug("try to connect to %s:%u, user:%s db:%s", ip, port, user, db); if (user == NULL) { user = TSDB_DEFAULT_USER; } @@ -67,7 +66,7 @@ TAOS *taos_connect(const char *ip, const char *user, const char *pass, const cha pass = TSDB_DEFAULT_PASS; } - return taos_connect_internal(ip, user, pass, NULL, db, p); + return taos_connect_internal(ip, user, pass, NULL, db, port); } void taos_close(TAOS* taos) { diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index 229d3a9ec3..9a1025c4bd 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -45,22 +45,24 @@ struct tmq_topic_vgroup_list_t { }; struct tmq_conf_t { - char clientId[256]; - char groupId[256]; - bool auto_commit; + char clientId[256]; + char groupId[256]; + int8_t auto_commit; + int8_t resetOffset; + tmq_commit_cb* commit_cb; /*char* ip;*/ /*uint16_t port;*/ - tmq_commit_cb* commit_cb; }; struct tmq_t { // conf char groupId[256]; char clientId[256]; - bool autoCommit; + int8_t autoCommit; SRWLatch lock; int64_t consumerId; int32_t epoch; + int32_t resetOffsetCfg; int64_t status; tsem_t rspSem; STscObj* pTscObj; @@ -79,7 +81,6 @@ typedef struct { // statistics int64_t pollCnt; // offset - int64_t committedOffset; int64_t currentOffset; // connection info int32_t vgId; @@ -115,21 +116,17 @@ typedef struct { } SMqConsumeCbParam; typedef struct { - tmq_t* tmq; - SMqClientVg* pVg; - int32_t async; - tsem_t rspSem; -} SMqCommitCbParam; - -typedef struct { - tmq_t* tmq; + tmq_t* tmq; + /*SMqClientVg* pVg;*/ + int32_t async; tsem_t rspSem; tmq_resp_err_t rspErr; -} SMqResetOffsetParam; +} SMqCommitCbParam; tmq_conf_t* tmq_conf_new() { tmq_conf_t* conf = calloc(1, sizeof(tmq_conf_t)); conf->auto_commit = false; + conf->resetOffset = TMQ_CONF__RESET_OFFSET__LATEST; return conf; } @@ -157,6 +154,20 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value return TMQ_CONF_INVALID; } } + if (strcmp(key, "auto.offset.reset") == 0) { + if (strcmp(value, "none") == 0) { + conf->resetOffset = TMQ_CONF__RESET_OFFSET__NONE; + return TMQ_CONF_OK; + } else if (strcmp(value, "earliest") == 0) { + conf->resetOffset = TMQ_CONF__RESET_OFFSET__EARLIEAST; + return TMQ_CONF_OK; + } else if (strcmp(value, "latest") == 0) { + conf->resetOffset = TMQ_CONF__RESET_OFFSET__LATEST; + return TMQ_CONF_OK; + } else { + return TMQ_CONF_INVALID; + } + } return TMQ_CONF_UNKNOWN; } @@ -190,14 +201,12 @@ int32_t tmqCommitCb(void* param, const SDataBuf* pMsg, int32_t code) { if (pParam->tmq->commit_cb) { pParam->tmq->commit_cb(pParam->tmq, rspErr, NULL, NULL); } - if (!pParam->async) tsem_post(&pParam->rspSem); - return 0; -} - -int32_t tmqResetOffsetCb(void* param, const SDataBuf* pMsg, int32_t code) { - SMqResetOffsetParam* pParam = (SMqResetOffsetParam*)param; - pParam->rspErr = code; - tsem_post(&pParam->rspSem); + if (!pParam->async) + tsem_post(&pParam->rspSem); + else { + tsem_destroy(&pParam->rspSem); + free(param); + } return 0; } @@ -216,6 +225,7 @@ tmq_t* tmq_consumer_new(void* conn, tmq_conf_t* conf, char* errstr, int32_t errs strcpy(pTmq->groupId, conf->groupId); pTmq->autoCommit = conf->auto_commit; pTmq->commit_cb = conf->commit_cb; + pTmq->resetOffsetCfg = conf->resetOffset; tsem_init(&pTmq->rspSem, 0, 0); pTmq->consumerId = generateRequestId() & (((uint64_t)-1) >> 1); @@ -223,18 +233,40 @@ tmq_t* tmq_consumer_new(void* conn, tmq_conf_t* conf, char* errstr, int32_t errs return pTmq; } -tmq_resp_err_t tmq_reset_offset(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets) { - SRequestObj* pRequest = NULL; +tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets, int32_t async) { + // TODO: add read write lock + SRequestObj* pRequest = NULL; + tmq_resp_err_t resp = TMQ_RESP_ERR__SUCCESS; // build msg // send to mnode - SMqCMResetOffsetReq req; - req.num = offsets->cnt; - req.offsets = (SMqOffset*)offsets->elems; + SMqCMCommitOffsetReq req; + SArray* pArray = NULL; + + if (offsets == NULL) { + pArray = taosArrayInit(0, sizeof(SMqOffset)); + for (int i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) { + SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i); + for (int j = 0; j < taosArrayGetSize(pTopic->vgs); j++) { + SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j); + SMqOffset offset; + strcpy(offset.topicName, pTopic->topicName); + strcpy(offset.cgroup, tmq->groupId); + offset.vgId = pVg->vgId; + offset.offset = pVg->currentOffset; + taosArrayPush(pArray, &offset); + } + } + req.num = pArray->size; + req.offsets = pArray->pData; + } else { + req.num = offsets->cnt; + req.offsets = (SMqOffset*)offsets->elems; + } SCoder encoder; tCoderInit(&encoder, TD_LITTLE_ENDIAN, NULL, 0, TD_ENCODER); - tEncodeSMqCMResetOffsetReq(&encoder, &req); + tEncodeSMqCMCommitOffsetReq(&encoder, &req); int32_t tlen = encoder.pos; void* buf = malloc(tlen); if (buf == NULL) { @@ -244,32 +276,41 @@ tmq_resp_err_t tmq_reset_offset(tmq_t* tmq, const tmq_topic_vgroup_list_t* offse tCoderClear(&encoder); tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, tlen, TD_ENCODER); - tEncodeSMqCMResetOffsetReq(&encoder, &req); + tEncodeSMqCMCommitOffsetReq(&encoder, &req); tCoderClear(&encoder); - pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_MND_RESET_OFFSET); + pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_MND_MQ_COMMIT_OFFSET); if (pRequest == NULL) { tscError("failed to malloc request"); } - SMqResetOffsetParam param = {0}; - tsem_init(¶m.rspSem, 0, 0); - param.tmq = tmq; + SMqCommitCbParam* pParam = malloc(sizeof(SMqCommitCbParam)); + if (pParam == NULL) { + return -1; + } + pParam->tmq = tmq; + tsem_init(&pParam->rspSem, 0, 0); pRequest->body.requestMsg = (SDataBuf){.pData = buf, .len = tlen}; SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest); - sendInfo->param = ¶m; - sendInfo->fp = tmqResetOffsetCb; + sendInfo->param = pParam; + sendInfo->fp = tmqCommitCb; SEpSet epSet = getEpSet_s(&tmq->pTscObj->pAppInfo->mgmtEp); int64_t transporterId = 0; asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo); - tsem_wait(¶m.rspSem); - tsem_destroy(¶m.rspSem); + if (!async) { + tsem_wait(&pParam->rspSem); + resp = pParam->rspErr; + } - return param.rspErr; + if (pArray) { + taosArrayDestroy(pArray); + } + + return resp; } tmq_resp_err_t tmq_subscribe(tmq_t* tmq, tmq_list_t* topic_list) { @@ -641,8 +682,7 @@ int32_t tmqAskEpCb(void* param, const SDataBuf* pMsg, int32_t code) { // clang-format off SMqClientVg clientVg = { .pollCnt = 0, - .committedOffset = -1, - .currentOffset = -1, + .currentOffset = pVgEp->offset, .vgId = pVgEp->vgId, .epSet = pVgEp->epSet }; @@ -708,23 +748,51 @@ END: return 0; } -SMqConsumeReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blocking_time, int32_t type, SMqClientTopic* pTopic, - SMqClientVg* pVg) { +tmq_resp_err_t tmq_seek(tmq_t* tmq, const tmq_topic_vgroup_t* offset) { + const SMqOffset* pOffset = &offset->offset; + if (strcmp(pOffset->cgroup, tmq->groupId) != 0) { + return TMQ_RESP_ERR__FAIL; + } + int32_t sz = taosArrayGetSize(tmq->clientTopics); + for (int32_t i = 0; i < sz; i++) { + SMqClientTopic* clientTopic = taosArrayGet(tmq->clientTopics, i); + if (strcmp(clientTopic->topicName, pOffset->topicName) == 0) { + int32_t vgSz = taosArrayGetSize(clientTopic->vgs); + for (int32_t j = 0; j < vgSz; j++) { + SMqClientVg* pVg = taosArrayGet(clientTopic->vgs, j); + if (pVg->vgId == pOffset->vgId) { + pVg->currentOffset = pOffset->offset; + return TMQ_RESP_ERR__SUCCESS; + } + } + } + } + return TMQ_RESP_ERR__FAIL; +} + +SMqConsumeReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blocking_time, SMqClientTopic* pTopic, SMqClientVg* pVg) { + int64_t reqOffset; + if (pVg->currentOffset >= 0) { + reqOffset = pVg->currentOffset; + } else { + if (tmq->resetOffsetCfg == TMQ_CONF__RESET_OFFSET__NONE) { + tscError("unable to poll since no committed offset but reset offset is set to none"); + return NULL; + } + reqOffset = tmq->resetOffsetCfg; + } + SMqConsumeReq* pReq = malloc(sizeof(SMqConsumeReq)); if (pReq == NULL) { return NULL; } - pReq->reqType = type; + strcpy(pReq->topic, pTopic->topicName); - pReq->blockingTime = blocking_time; - pReq->consumerId = tmq->consumerId; strcpy(pReq->cgroup, tmq->groupId); - if (type == TMQ_REQ_TYPE_COMMIT_ONLY) { - pReq->offset = pVg->currentOffset; - } else { - pReq->offset = pVg->currentOffset + 1; - } + pReq->blockingTime = blocking_time; + pReq->consumerId = tmq->consumerId; + pReq->currentOffset = reqOffset; pReq->head.vgId = htonl(pVg->vgId); pReq->head.contLen = htonl(sizeof(SMqConsumeReq)); @@ -743,13 +811,13 @@ tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) { if (taosArrayGetSize(tmq->clientTopics) == 0) { tscDebug("consumer:%ld poll but not assigned", tmq->consumerId); - printf("over1\n"); + /*printf("over1\n");*/ usleep(blocking_time * 1000); return NULL; } SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, tmq->nextTopicIdx); if (taosArrayGetSize(pTopic->vgs) == 0) { - printf("over2\n"); + /*printf("over2\n");*/ usleep(blocking_time * 1000); return NULL; } @@ -760,8 +828,7 @@ tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) { pTopic->nextVgIdx = (pTopic->nextVgIdx + 1) % taosArrayGetSize(pTopic->vgs); SMqClientVg* pVg = taosArrayGet(pTopic->vgs, pTopic->nextVgIdx); /*printf("consume vg %d, offset %ld\n", pVg->vgId, pVg->currentOffset);*/ - int32_t reqType = tmq->autoCommit ? TMQ_REQ_TYPE_CONSUME_AND_COMMIT : TMQ_REQ_TYPE_CONSUME_ONLY; - SMqConsumeReq* pReq = tmqBuildConsumeReqImpl(tmq, blocking_time, reqType, pTopic, pVg); + SMqConsumeReq* pReq = tmqBuildConsumeReqImpl(tmq, blocking_time, pTopic, pVg); if (pReq == NULL) { ASSERT(false); usleep(blocking_time * 1000); @@ -821,6 +888,7 @@ tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) { /*return pRequest;*/ } +#if 0 tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* tmq_topic_vgroup_list, int32_t async) { if (tmq_topic_vgroup_list != NULL) { // TODO @@ -831,7 +899,7 @@ tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* tmq_topic_v SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i); for (int j = 0; j < taosArrayGetSize(pTopic->vgs); j++) { SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j); - SMqConsumeReq* pReq = tmqBuildConsumeReqImpl(tmq, 0, TMQ_REQ_TYPE_COMMIT_ONLY, pTopic, pVg); + SMqConsumeReq* pReq = tmqBuildConsumeReqImpl(tmq, 0, pTopic, pVg); SRequestObj* pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_VND_CONSUME); pRequest->body.requestMsg = (SDataBuf){.pData = pReq, .len = sizeof(SMqConsumeReq)}; @@ -858,6 +926,7 @@ tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* tmq_topic_v return 0; } +#endif void tmq_message_destroy(tmq_message_t* tmq_message) { if (tmq_message == NULL) return; diff --git a/source/client/test/CMakeLists.txt b/source/client/test/CMakeLists.txt index ee5109860e..7f6ce81788 100644 --- a/source/client/test/CMakeLists.txt +++ b/source/client/test/CMakeLists.txt @@ -8,13 +8,13 @@ AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) ADD_EXECUTABLE(clientTest clientTests.cpp) TARGET_LINK_LIBRARIES( clientTest - PUBLIC os util common transport parser catalog scheduler function gtest taos qcom + PUBLIC os util common transport parser catalog scheduler function gtest taos qcom config ) ADD_EXECUTABLE(tmqTest tmqTest.cpp) TARGET_LINK_LIBRARIES( tmqTest - PUBLIC os util common transport parser catalog scheduler function gtest taos qcom + PUBLIC os util common transport parser catalog scheduler function gtest taos qcom config ) TARGET_INCLUDE_DIRECTORIES( diff --git a/source/common/src/tep.c b/source/common/src/tep.c index 970b6d954f..3e98cae279 100644 --- a/source/common/src/tep.c +++ b/source/common/src/tep.c @@ -4,7 +4,7 @@ #include "tglobal.h" #include "tlockfree.h" -int taosGetFqdnPortFromEp(const char *ep, SEp* pEp) { +int taosGetFqdnPortFromEp(const char *ep, uint16_t defaultPort, SEp* pEp) { pEp->port = 0; strcpy(pEp->fqdn, ep); @@ -15,7 +15,7 @@ int taosGetFqdnPortFromEp(const char *ep, SEp* pEp) { } if (pEp->port == 0) { - pEp->port = tsServerPort; + pEp->port = defaultPort; return -1; } diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index afba1190ba..72783bf4ab 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -19,7 +19,6 @@ #include "taosdef.h" #include "taoserror.h" #include "tcompare.h" -#include "tconfig.h" #include "tep.h" #include "tglobal.h" #include "tlocale.h" @@ -28,29 +27,9 @@ #include "tutil.h" #include "ulog.h" -// cluster -char tsFirst[TSDB_EP_LEN] = {0}; -char tsSecond[TSDB_EP_LEN] = {0}; -char tsLocalFqdn[TSDB_FQDN_LEN] = {0}; -char tsLocalEp[TSDB_EP_LEN] = {0}; // Local End Point, hostname:port -uint16_t tsServerPort = 6030; -int32_t tsStatusInterval = 1; // second -int8_t tsEnableTelemetryReporting = 0; -char tsEmail[TSDB_FQDN_LEN] = {0}; -int32_t tsNumOfSupportVnodes = 128; // common -int32_t tsRpcTimer = 300; -int32_t tsRpcMaxTime = 600; // seconds; -int32_t tsRpcForceTcp = 1; // disable this, means query, show command use udp protocol as default -int32_t tsMaxShellConns = 50000; -int32_t tsMaxConnections = 50000; -int32_t tsShellActivityTimer = 3; // second -float tsNumOfThreadsPerCore = 1.0f; -int32_t tsNumOfCommitThreads = 4; -float tsRatioOfQueryCores = 1.0f; int8_t tsDaylight = 0; -int8_t tsEnableCoreFile = 0; int32_t tsMaxBinaryDisplayWidth = 30; int8_t tsEnableSlaveQuery = 1; int8_t tsEnableAdjustMaster = 1; @@ -131,14 +110,6 @@ int8_t tsDeadLockKillQuery = 0; // For backward compatibility bool tsdbForceKeepFile = false; -int32_t tsDiskCfgNum = 0; - -#ifndef _STORAGE -SDiskCfg tsDiskCfg[1]; -#else -SDiskCfg tsDiskCfg[TFS_MAX_DISKS]; -#endif - /* * minimum scale for whole system, millisecond by default * for TSDB_TIME_PRECISION_MILLI: 86400000L @@ -148,13 +119,6 @@ SDiskCfg tsDiskCfg[TFS_MAX_DISKS]; int64_t tsTickPerDay[] = {86400000L, 86400000000L, 86400000000000L}; // system info -float tsTotalTmpDirGB = 0; -float tsTotalDataDirGB = 0; -float tsAvailTmpDirectorySpace = 0; -float tsAvailDataDirGB = 0; -float tsUsedDataDirGB = 0; -float tsReservedTmpDirectorySpace = 1.0f; -float tsMinimalDataDirGB = 2.0f; int32_t tsTotalMemoryMB = 0; uint32_t tsVersion = 0; @@ -179,24 +143,9 @@ char *qtypeStr[] = {"rpc", "fwd", "wal", "cq", "query"}; static pthread_once_t tsInitGlobalCfgOnce = PTHREAD_ONCE_INIT; -void taosSetAllDebugFlag() { - if (debugFlag != 0) { - mDebugFlag = debugFlag; - dDebugFlag = debugFlag; - vDebugFlag = debugFlag; - jniDebugFlag = debugFlag; - qDebugFlag = debugFlag; - rpcDebugFlag = debugFlag; - uDebugFlag = debugFlag; - sDebugFlag = debugFlag; - wDebugFlag = debugFlag; - tsdbDebugFlag = debugFlag; - cqDebugFlag = debugFlag; - uInfo("all debug flag are set to %d", debugFlag); - } -} int32_t taosCfgDynamicOptions(char *msg) { + #if 0 char *option, *value; int32_t olen, vlen; int32_t vint = 0; @@ -266,130 +215,43 @@ int32_t taosCfgDynamicOptions(char *msg) { } } +#endif return false; } -void taosAddDataDir(int index, char *v1, int level, int primary) { - tstrncpy(tsDiskCfg[index].dir, v1, TSDB_FILENAME_LEN); - tsDiskCfg[index].level = level; - tsDiskCfg[index].primary = primary; - uTrace("dataDir:%s, level:%d primary:%d is configured", v1, level, primary); -} +// void taosAddDataDir(int index, char *v1, int level, int primary) { +// tstrncpy(tsDiskCfg[index].dir, v1, TSDB_FILENAME_LEN); +// tsDiskCfg[index].level = level; +// tsDiskCfg[index].primary = primary; +// uTrace("dataDir:%s, level:%d primary:%d is configured", v1, level, primary); +// } #ifndef _STORAGE -void taosReadDataDirCfg(char *v1, char *v2, char *v3) { - if (tsDiskCfgNum == 1) { - SDiskCfg *cfg = &tsDiskCfg[0]; - uInfo("dataDir:%s, level:%d primary:%d is replaced by %s", cfg->dir, cfg->level, cfg->primary, v1); - } - taosAddDataDir(0, v1, 0, 1); - tsDiskCfgNum = 1; -} +// void taosReadDataDirCfg(char *v1, char *v2, char *v3) { +// if (tsDiskCfgNum == 1) { +// SDiskCfg *cfg = &tsDiskCfg[0]; +// uInfo("dataDir:%s, level:%d primary:%d is replaced by %s", cfg->dir, cfg->level, cfg->primary, v1); +// } +// taosAddDataDir(0, v1, 0, 1); +// tsDiskCfgNum = 1; +// } -void taosPrintDataDirCfg() { - for (int i = 0; i < tsDiskCfgNum; ++i) { - SDiskCfg *cfg = &tsDiskCfg[i]; - uInfo(" dataDir: %s", cfg->dir); - } -} +// void taosPrintDataDirCfg() { +// for (int i = 0; i < tsDiskCfgNum; ++i) { +// SDiskCfg *cfg = &tsDiskCfg[i]; +// uInfo(" dataDir: %s", cfg->dir); +// } +// } #endif -static void taosCheckDataDirCfg() { - if (tsDiskCfgNum <= 0) { - taosAddDataDir(0, tsDataDir, 0, 1); - tsDiskCfgNum = 1; - uTrace("dataDir:%s, level:0 primary:1 is configured by default", tsDataDir); - } -} + static void doInitGlobalConfig(void) { osInit(); srand(taosSafeRand()); - +#if 0 SGlobalCfg cfg = {0}; - // ip address - cfg.option = "firstEp"; - cfg.ptr = tsFirst; - cfg.valType = TAOS_CFG_VTYPE_STRING; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 0; - cfg.ptrLength = TSDB_EP_LEN; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "secondEp"; - cfg.ptr = tsSecond; - cfg.valType = TAOS_CFG_VTYPE_STRING; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 0; - cfg.ptrLength = TSDB_EP_LEN; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "fqdn"; - cfg.ptr = tsLocalFqdn; - cfg.valType = TAOS_CFG_VTYPE_STRING; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 0; - cfg.ptrLength = TSDB_FQDN_LEN; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - // port - cfg.option = "serverPort"; - cfg.ptr = &tsServerPort; - cfg.valType = TAOS_CFG_VTYPE_UINT16; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 1; - cfg.maxValue = 65056; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "supportVnodes"; - cfg.ptr = &tsNumOfSupportVnodes; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 65536; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - // directory - cfg.option = "configDir"; - cfg.ptr = configDir; - cfg.valType = TAOS_CFG_VTYPE_DIRECTORY; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 0; - cfg.ptrLength = TSDB_FILENAME_LEN; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "logDir"; - cfg.ptr = tsLogDir; - cfg.valType = TAOS_CFG_VTYPE_DIRECTORY; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT | TSDB_CFG_CTYPE_B_LOG; - cfg.minValue = 0; - cfg.maxValue = 0; - cfg.ptrLength = TSDB_FILENAME_LEN; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "scriptDir"; - cfg.ptr = tsScriptDir; - cfg.valType = TAOS_CFG_VTYPE_DIRECTORY; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 0; - cfg.ptrLength = TSDB_FILENAME_LEN; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); cfg.option = "dataDir"; cfg.ptr = tsDataDir; @@ -401,36 +263,6 @@ static void doInitGlobalConfig(void) { cfg.unitType = TAOS_CFG_UTYPE_NONE; taosAddConfigOption(cfg); - // dnode configs - cfg.option = "numOfThreadsPerCore"; - cfg.ptr = &tsNumOfThreadsPerCore; - cfg.valType = TAOS_CFG_VTYPE_FLOAT; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 10; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "numOfCommitThreads"; - cfg.ptr = &tsNumOfCommitThreads; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; - cfg.minValue = 1; - cfg.maxValue = 100; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "ratioOfQueryCores"; - cfg.ptr = &tsRatioOfQueryCores; - cfg.valType = TAOS_CFG_VTYPE_FLOAT; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; - cfg.minValue = 0.0f; - cfg.maxValue = 2.0f; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); cfg.option = "maxNumOfDistinctRes"; cfg.ptr = &tsMaxNumOfDistinctResults; @@ -442,77 +274,6 @@ static void doInitGlobalConfig(void) { cfg.unitType = TAOS_CFG_UTYPE_NONE; taosAddConfigOption(cfg); - cfg.option = "telemetryReporting"; - cfg.ptr = &tsEnableTelemetryReporting; - cfg.valType = TAOS_CFG_VTYPE_INT8; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 0; - cfg.maxValue = 1; - cfg.ptrLength = 1; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - // timer - cfg.option = "maxTmrCtrl"; - cfg.ptr = &tsMaxTmrCtrl; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 8; - cfg.maxValue = 2048; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "rpcTimer"; - cfg.ptr = &tsRpcTimer; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 100; - cfg.maxValue = 3000; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_MS; - taosAddConfigOption(cfg); - - cfg.option = "rpcForceTcp"; - cfg.ptr = &tsRpcForceTcp; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 1; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "rpcMaxTime"; - cfg.ptr = &tsRpcMaxTime; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 100; - cfg.maxValue = 7200; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_SECOND; - taosAddConfigOption(cfg); - - cfg.option = "statusInterval"; - cfg.ptr = &tsStatusInterval; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 1; - cfg.maxValue = 10; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_SECOND; - taosAddConfigOption(cfg); - - cfg.option = "shellActivityTimer"; - cfg.ptr = &tsShellActivityTimer; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 1; - cfg.maxValue = 120; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_SECOND; - taosAddConfigOption(cfg); - cfg.option = "minSlidingTime"; cfg.ptr = &tsMinSlidingTime; cfg.valType = TAOS_CFG_VTYPE_INT32; @@ -653,87 +414,7 @@ static void doInitGlobalConfig(void) { cfg.unitType = TAOS_CFG_UTYPE_NONE; taosAddConfigOption(cfg); - // locale & charset - cfg.option = "timezone"; - cfg.ptr = tsTimezone; - cfg.valType = TAOS_CFG_VTYPE_STRING; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 0; - cfg.ptrLength = TSDB_TIMEZONE_LEN; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - cfg.option = "locale"; - cfg.ptr = tsLocale; - cfg.valType = TAOS_CFG_VTYPE_STRING; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 0; - cfg.ptrLength = TSDB_LOCALE_LEN; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "charset"; - cfg.ptr = tsCharset; - cfg.valType = TAOS_CFG_VTYPE_STRING; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 0; - cfg.ptrLength = TSDB_LOCALE_LEN; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - // connect configs - cfg.option = "maxShellConns"; - cfg.ptr = &tsMaxShellConns; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 10; - cfg.maxValue = 50000000; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "maxConnections"; - cfg.ptr = &tsMaxConnections; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 1; - cfg.maxValue = 100000; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "minimalLogDirGB"; - cfg.ptr = &tsMinimalLogDirGB; - cfg.valType = TAOS_CFG_VTYPE_FLOAT; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 0.001f; - cfg.maxValue = 10000000; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_GB; - taosAddConfigOption(cfg); - - cfg.option = "minimalTmpDirGB"; - cfg.ptr = &tsReservedTmpDirectorySpace; - cfg.valType = TAOS_CFG_VTYPE_FLOAT; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 0.001f; - cfg.maxValue = 10000000; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_GB; - taosAddConfigOption(cfg); - - cfg.option = "minimalDataDirGB"; - cfg.ptr = &tsMinimalDataDirGB; - cfg.valType = TAOS_CFG_VTYPE_FLOAT; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 0.001f; - cfg.maxValue = 10000000; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_GB; - taosAddConfigOption(cfg); cfg.option = "slaveQuery"; cfg.ptr = &tsEnableSlaveQuery; @@ -745,176 +426,7 @@ static void doInitGlobalConfig(void) { cfg.unitType = TAOS_CFG_UTYPE_NONE; taosAddConfigOption(cfg); - // debug flag - cfg.option = "numOfLogLines"; - cfg.ptr = &tsNumOfLogLines; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 1000; - cfg.maxValue = 2000000000; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - cfg.option = "logKeepDays"; - cfg.ptr = &tsLogKeepDays; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = -365000; - cfg.maxValue = 365000; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "asyncLog"; - cfg.ptr = &tsAsyncLog; - cfg.valType = TAOS_CFG_VTYPE_INT8; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 1; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "debugFlag"; - cfg.ptr = &debugFlag; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 255; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "mDebugFlag"; - cfg.ptr = &mDebugFlag; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG; - cfg.minValue = 0; - cfg.maxValue = 255; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "dDebugFlag"; - cfg.ptr = &dDebugFlag; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG; - cfg.minValue = 0; - cfg.maxValue = 255; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "sDebugFlag"; - cfg.ptr = &sDebugFlag; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG; - cfg.minValue = 0; - cfg.maxValue = 255; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "wDebugFlag"; - cfg.ptr = &wDebugFlag; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG; - cfg.minValue = 0; - cfg.maxValue = 255; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "rpcDebugFlag"; - cfg.ptr = &rpcDebugFlag; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 255; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "tmrDebugFlag"; - cfg.ptr = &tmrDebugFlag; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 255; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "cDebugFlag"; - cfg.ptr = &cDebugFlag; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 255; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "jniDebugFlag"; - cfg.ptr = &jniDebugFlag; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 255; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "uDebugFlag"; - cfg.ptr = &uDebugFlag; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 255; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "qDebugFlag"; - cfg.ptr = &qDebugFlag; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 255; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "vDebugFlag"; - cfg.ptr = &vDebugFlag; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG; - cfg.minValue = 0; - cfg.maxValue = 255; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "tsdbDebugFlag"; - cfg.ptr = &tsdbDebugFlag; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG; - cfg.minValue = 0; - cfg.maxValue = 255; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "cqDebugFlag"; - cfg.ptr = &cqDebugFlag; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG; - cfg.minValue = 0; - cfg.maxValue = 255; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); cfg.option = "enableRecordSql"; cfg.ptr = &tsTscEnableRecordSql; @@ -926,57 +438,6 @@ static void doInitGlobalConfig(void) { cfg.unitType = TAOS_CFG_UTYPE_NONE; taosAddConfigOption(cfg); - cfg.option = "enableCoreFile"; - cfg.ptr = &tsEnableCoreFile; - cfg.valType = TAOS_CFG_VTYPE_INT8; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; - cfg.minValue = 0; - cfg.maxValue = 1; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - // version info - cfg.option = "gitinfo"; - cfg.ptr = gitinfo; - cfg.valType = TAOS_CFG_VTYPE_STRING; - cfg.cfgType = TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 0; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "gitinfoOfInternal"; - cfg.ptr = gitinfoOfInternal; - cfg.valType = TAOS_CFG_VTYPE_STRING; - cfg.cfgType = TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 0; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "buildinfo"; - cfg.ptr = buildinfo; - cfg.valType = TAOS_CFG_VTYPE_STRING; - cfg.cfgType = TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 0; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "version"; - cfg.ptr = version; - cfg.valType = TAOS_CFG_VTYPE_STRING; - cfg.cfgType = TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 0; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - cfg.option = "maxBinaryDisplayWidth"; cfg.ptr = &tsMaxBinaryDisplayWidth; cfg.valType = TAOS_CFG_VTYPE_INT32; @@ -1025,7 +486,7 @@ static void doInitGlobalConfig(void) { cfg.valType = TAOS_CFG_VTYPE_DOUBLE; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; cfg.minValue = MIN_FLOAT; - cfg.maxValue = MAX_FLOAT; + cfg.maxValue = 100000; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; @@ -1035,8 +496,8 @@ static void doInitGlobalConfig(void) { cfg.ptr = &dPrecision; cfg.valType = TAOS_CFG_VTYPE_DOUBLE; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; - cfg.minValue = MIN_FLOAT; - cfg.maxValue = MAX_FLOAT; + cfg.minValue = 100000; + cfg.maxValue = 0; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; taosAddConfigOption(cfg); @@ -1064,67 +525,12 @@ static void doInitGlobalConfig(void) { #else // assert(tsGlobalConfigNum == TSDB_CFG_MAX_NUM - 5); #endif + +#endif } void taosInitGlobalCfg() { pthread_once(&tsInitGlobalCfgOnce, doInitGlobalConfig); } -int32_t taosCheckAndPrintCfg() { - SEp ep = {0}; - if (debugFlag & DEBUG_TRACE || debugFlag & DEBUG_DEBUG || debugFlag & DEBUG_DUMP) { - taosSetAllDebugFlag(); - } - - if (tsLocalFqdn[0] == 0) { - taosGetFqdn(tsLocalFqdn); - } - - snprintf(tsLocalEp, sizeof(tsLocalEp), "%s:%u", tsLocalFqdn, tsServerPort); - uInfo("localEp is: %s", tsLocalEp); - - if (tsFirst[0] == 0) { - strcpy(tsFirst, tsLocalEp); - } else { - taosGetFqdnPortFromEp(tsFirst, &ep); - snprintf(tsFirst, sizeof(tsFirst), "%s:%u", ep.fqdn, ep.port); - } - - if (tsSecond[0] == 0) { - strcpy(tsSecond, tsLocalEp); - } else { - taosGetFqdnPortFromEp(tsSecond, &ep); - snprintf(tsSecond, sizeof(tsSecond), "%s:%u", ep.fqdn, ep.port); - } - - taosCheckDataDirCfg(); - - if (taosDirExist(tsTempDir) != 0) { - return -1; - } - - taosGetSystemInfo(); - - tsSetLocale(); - - SGlobalCfg *cfg_timezone = taosGetConfigOption("timezone"); - if (cfg_timezone && cfg_timezone->cfgStatus == TAOS_CFG_CSTATUS_FILE) { - tsSetTimeZone(); - } - - if (tsNumOfCores <= 0) { - tsNumOfCores = 1; - } - - if (tsQueryBufferSize >= 0) { - tsQueryBufferSizeBytes = tsQueryBufferSize * 1048576UL; - } - - uInfo(" check global cfg completed"); - uInfo("=================================="); - taosPrintCfg(); - - return 0; -} - /* * alter dnode 1 balance "vnode:1-dnode:2" */ diff --git a/source/common/src/tlocale.c b/source/common/src/tlocale.c index a98a46b28a..1f2b9cd4b1 100644 --- a/source/common/src/tlocale.c +++ b/source/common/src/tlocale.c @@ -17,7 +17,6 @@ #include "os.h" #include "ulog.h" #include "tglobal.h" -#include "tconfig.h" #include "tutil.h" /** diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 46feab7791..8101c18ebf 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -285,7 +285,7 @@ int32_t tDeserializeSClientHbBatchRsp(void *buf, int32_t bufLen, SClientHbBatchR int32_t tSerializeSVCreateTbReq(void **buf, SVCreateTbReq *pReq) { int32_t tlen = 0; - tlen += taosEncodeFixedU64(buf, pReq->ver); + tlen += taosEncodeFixedI64(buf, pReq->ver); tlen += taosEncodeString(buf, pReq->name); tlen += taosEncodeFixedU32(buf, pReq->ttl); tlen += taosEncodeFixedU32(buf, pReq->keep); @@ -330,7 +330,7 @@ int32_t tSerializeSVCreateTbReq(void **buf, SVCreateTbReq *pReq) { } void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) { - buf = taosDecodeFixedU64(buf, &(pReq->ver)); + buf = taosDecodeFixedI64(buf, &(pReq->ver)); buf = taosDecodeString(buf, &(pReq->name)); buf = taosDecodeFixedU32(buf, &(pReq->ttl)); buf = taosDecodeFixedU32(buf, &(pReq->keep)); @@ -380,7 +380,7 @@ void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) { int32_t tSerializeSVCreateTbBatchReq(void **buf, SVCreateTbBatchReq *pReq) { int32_t tlen = 0; - tlen += taosEncodeFixedU64(buf, pReq->ver); + tlen += taosEncodeFixedI64(buf, pReq->ver); tlen += taosEncodeFixedU32(buf, taosArrayGetSize(pReq->pArray)); for (size_t i = 0; i < taosArrayGetSize(pReq->pArray); i++) { SVCreateTbReq *pCreateTbReq = taosArrayGet(pReq->pArray, i); @@ -393,7 +393,7 @@ int32_t tSerializeSVCreateTbBatchReq(void **buf, SVCreateTbBatchReq *pReq) { void *tDeserializeSVCreateTbBatchReq(void *buf, SVCreateTbBatchReq *pReq) { uint32_t nsize = 0; - buf = taosDecodeFixedU64(buf, &pReq->ver); + buf = taosDecodeFixedI64(buf, &pReq->ver); buf = taosDecodeFixedU32(buf, &nsize); pReq->pArray = taosArrayInit(nsize, sizeof(SVCreateTbReq)); for (size_t i = 0; i < nsize; i++) { @@ -407,14 +407,14 @@ void *tDeserializeSVCreateTbBatchReq(void *buf, SVCreateTbBatchReq *pReq) { int32_t tSerializeSVDropTbReq(void **buf, SVDropTbReq *pReq) { int32_t tlen = 0; - tlen += taosEncodeFixedU64(buf, pReq->ver); + tlen += taosEncodeFixedI64(buf, pReq->ver); tlen += taosEncodeString(buf, pReq->name); tlen += taosEncodeFixedU8(buf, pReq->type); return tlen; } void *tDeserializeSVDropTbReq(void *buf, SVDropTbReq *pReq) { - buf = taosDecodeFixedU64(buf, &pReq->ver); + buf = taosDecodeFixedI64(buf, &pReq->ver); buf = taosDecodeString(buf, &pReq->name); buf = taosDecodeFixedU8(buf, &pReq->type); return buf; @@ -1393,7 +1393,7 @@ int32_t tSerializeSDropDbRsp(void *buf, int32_t bufLen, SDropDbRsp *pRsp) { if (tStartEncode(&encoder) < 0) return -1; if (tEncodeCStr(&encoder, pRsp->db) < 0) return -1; - if (tEncodeU64(&encoder, pRsp->uid) < 0) return -1; + if (tEncodeI64(&encoder, pRsp->uid) < 0) return -1; tEndEncode(&encoder); int32_t tlen = encoder.pos; @@ -1407,7 +1407,7 @@ int32_t tDeserializeSDropDbRsp(void *buf, int32_t bufLen, SDropDbRsp *pRsp) { if (tStartDecode(&decoder) < 0) return -1; if (tDecodeCStrTo(&decoder, pRsp->db) < 0) return -1; - if (tDecodeU64(&decoder, &pRsp->uid) < 0) return -1; + if (tDecodeI64(&decoder, &pRsp->uid) < 0) return -1; tEndDecode(&decoder); tCoderClear(&decoder); @@ -1468,7 +1468,7 @@ int32_t tDeserializeSSyncDbReq(void *buf, int32_t bufLen, SSyncDbReq *pReq) { static int32_t tSerializeSUseDbRspImp(SCoder *pEncoder, SUseDbRsp *pRsp) { if (tEncodeCStr(pEncoder, pRsp->db) < 0) return -1; - if (tEncodeU64(pEncoder, pRsp->uid) < 0) return -1; + if (tEncodeI64(pEncoder, pRsp->uid) < 0) return -1; if (tEncodeI32(pEncoder, pRsp->vgVersion) < 0) return -1; if (tEncodeI32(pEncoder, pRsp->vgNum) < 0) return -1; if (tEncodeI8(pEncoder, pRsp->hashMethod) < 0) return -1; @@ -1518,7 +1518,7 @@ int32_t tSerializeSUseDbBatchRsp(void *buf, int32_t bufLen, SUseDbBatchRsp *pRsp int32_t tDeserializeSUseDbRspImp(SCoder *pDecoder, SUseDbRsp *pRsp) { if (tDecodeCStrTo(pDecoder, pRsp->db) < 0) return -1; - if (tDecodeU64(pDecoder, &pRsp->uid) < 0) return -1; + if (tDecodeI64(pDecoder, &pRsp->uid) < 0) return -1; if (tDecodeI32(pDecoder, &pRsp->vgVersion) < 0) return -1; if (tDecodeI32(pDecoder, &pRsp->vgNum) < 0) return -1; if (tDecodeI8(pDecoder, &pRsp->hashMethod) < 0) return -1; @@ -1661,7 +1661,7 @@ static int32_t tEncodeSTableMetaRsp(SCoder *pEncoder, STableMetaRsp *pRsp) { if (tEncodeCStr(pEncoder, pRsp->tbName) < 0) return -1; if (tEncodeCStr(pEncoder, pRsp->stbName) < 0) return -1; if (tEncodeCStr(pEncoder, pRsp->dbFName) < 0) return -1; - if (tEncodeU64(pEncoder, pRsp->dbId) < 0) return -1; + if (tEncodeI64(pEncoder, pRsp->dbId) < 0) return -1; if (tEncodeI32(pEncoder, pRsp->numOfTags) < 0) return -1; if (tEncodeI32(pEncoder, pRsp->numOfColumns) < 0) return -1; if (tEncodeI8(pEncoder, pRsp->precision) < 0) return -1; @@ -1684,7 +1684,7 @@ static int32_t tDecodeSTableMetaRsp(SCoder *pDecoder, STableMetaRsp *pRsp) { if (tDecodeCStrTo(pDecoder, pRsp->tbName) < 0) return -1; if (tDecodeCStrTo(pDecoder, pRsp->stbName) < 0) return -1; if (tDecodeCStrTo(pDecoder, pRsp->dbFName) < 0) return -1; - if (tDecodeU64(pDecoder, &pRsp->dbId) < 0) return -1; + if (tDecodeI64(pDecoder, &pRsp->dbId) < 0) return -1; if (tDecodeI32(pDecoder, &pRsp->numOfTags) < 0) return -1; if (tDecodeI32(pDecoder, &pRsp->numOfColumns) < 0) return -1; if (tDecodeI8(pDecoder, &pRsp->precision) < 0) return -1; @@ -2093,7 +2093,7 @@ int32_t tSerializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq *pR if (tEncodeI32(&encoder, pReq->vgId) < 0) return -1; if (tEncodeI32(&encoder, pReq->dnodeId) < 0) return -1; if (tEncodeCStr(&encoder, pReq->db) < 0) return -1; - if (tEncodeU64(&encoder, pReq->dbUid) < 0) return -1; + if (tEncodeI64(&encoder, pReq->dbUid) < 0) return -1; if (tEncodeI32(&encoder, pReq->vgVersion) < 0) return -1; if (tEncodeI32(&encoder, pReq->cacheBlockSize) < 0) return -1; if (tEncodeI32(&encoder, pReq->totalBlocks) < 0) return -1; @@ -2133,7 +2133,7 @@ int32_t tDeserializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq * if (tDecodeI32(&decoder, &pReq->vgId) < 0) return -1; if (tDecodeI32(&decoder, &pReq->dnodeId) < 0) return -1; if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1; - if (tDecodeU64(&decoder, &pReq->dbUid) < 0) return -1; + if (tDecodeI64(&decoder, &pReq->dbUid) < 0) return -1; if (tDecodeI32(&decoder, &pReq->vgVersion) < 0) return -1; if (tDecodeI32(&decoder, &pReq->cacheBlockSize) < 0) return -1; if (tDecodeI32(&decoder, &pReq->totalBlocks) < 0) return -1; @@ -2171,7 +2171,7 @@ int32_t tSerializeSDropVnodeReq(void *buf, int32_t bufLen, SDropVnodeReq *pReq) if (tStartEncode(&encoder) < 0) return -1; if (tEncodeI32(&encoder, pReq->vgId) < 0) return -1; if (tEncodeI32(&encoder, pReq->dnodeId) < 0) return -1; - if (tEncodeU64(&encoder, pReq->dbUid) < 0) return -1; + if (tEncodeI64(&encoder, pReq->dbUid) < 0) return -1; if (tEncodeCStr(&encoder, pReq->db) < 0) return -1; tEndEncode(&encoder); @@ -2187,7 +2187,7 @@ int32_t tDeserializeSDropVnodeReq(void *buf, int32_t bufLen, SDropVnodeReq *pReq if (tStartDecode(&decoder) < 0) return -1; if (tDecodeI32(&decoder, &pReq->vgId) < 0) return -1; if (tDecodeI32(&decoder, &pReq->dnodeId) < 0) return -1; - if (tDecodeU64(&decoder, &pReq->dbUid) < 0) return -1; + if (tDecodeI64(&decoder, &pReq->dbUid) < 0) return -1; if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1; tEndDecode(&decoder); @@ -2356,35 +2356,7 @@ int32_t tDecodeSMqOffset(SCoder *decoder, SMqOffset *pOffset) { return 0; } -int32_t tEncodeSMqVgOffsets(SCoder *encoder, const SMqVgOffsets *pOffsets) { - if (tStartEncode(encoder) < 0) return -1; - if (tEncodeI32(encoder, pOffsets->vgId) < 0) return -1; - int32_t sz = taosArrayGetSize(pOffsets->offsets); - if (tEncodeI32(encoder, sz) < 0) return -1; - for (int32_t i = 0; i < sz; i++) { - SMqOffset *offset = taosArrayGet(pOffsets->offsets, i); - if (tEncodeSMqOffset(encoder, offset) < 0) return -1; - } - tEndEncode(encoder); - return encoder->pos; -} - -int32_t tDecodeSMqVgOffsets(SCoder *decoder, SMqVgOffsets *pOffsets) { - int32_t sz; - if (tStartDecode(decoder) < 0) return -1; - if (tDecodeI32(decoder, &pOffsets->vgId) < 0) return -1; - if (tDecodeI32(decoder, &sz) < 0) return -1; - pOffsets->offsets = taosArrayInit(sz, sizeof(SMqOffset)); - for (int32_t i = 0; i < sz; i++) { - SMqOffset offset; - if (tDecodeSMqOffset(decoder, &offset) < 0) return -1; - taosArrayPush(pOffsets->offsets, &offset); - } - tEndDecode(decoder); - return 0; -} - -int32_t tEncodeSMqCMResetOffsetReq(SCoder *encoder, const SMqCMResetOffsetReq *pReq) { +int32_t tEncodeSMqCMCommitOffsetReq(SCoder *encoder, const SMqCMCommitOffsetReq *pReq) { if (tStartEncode(encoder) < 0) return -1; if (tEncodeI32(encoder, pReq->num) < 0) return -1; for (int32_t i = 0; i < pReq->num; i++) { @@ -2394,7 +2366,7 @@ int32_t tEncodeSMqCMResetOffsetReq(SCoder *encoder, const SMqCMResetOffsetReq *p return encoder->pos; } -int32_t tDecodeSMqCMResetOffsetReq(SCoder *decoder, SMqCMResetOffsetReq *pReq) { +int32_t tDecodeSMqCMCommitOffsetReq(SCoder *decoder, SMqCMCommitOffsetReq *pReq) { if (tStartDecode(decoder) < 0) return -1; if (tDecodeI32(decoder, &pReq->num) < 0) return -1; pReq->offsets = TCODER_MALLOC(pReq->num * sizeof(SMqOffset), decoder); @@ -2405,23 +2377,3 @@ int32_t tDecodeSMqCMResetOffsetReq(SCoder *decoder, SMqCMResetOffsetReq *pReq) { tEndDecode(decoder); return 0; } - -#if 0 -int32_t tEncodeSMqMVResetOffsetReq(SCoder *encoder, const SMqMVResetOffsetReq *pReq) { - if (tEncodeI64(encoder, pReq->leftForVer) < 0) return -1; - for (int32_t i = 0; i < pReq->num; i++) { - tEncodeSMqOffset(encoder, &pReq->offsets[i]); - } - return encoder->pos; -} - -int32_t tDecodeSMqMVResetOffsetReq(SCoder *decoder, SMqMVResetOffsetReq *pReq) { - if (tDecodeI32(decoder, &pReq->num) < 0) return -1; - pReq->offsets = TCODER_MALLOC(pReq->num * sizeof(SMqOffset), decoder); - if (pReq->offsets == NULL) return -1; - for (int32_t i = 0; i < pReq->num; i++) { - tDecodeSMqOffset(decoder, &pReq->offsets[i]); - } - return 0; -} -#endif diff --git a/source/dnode/mgmt/daemon/CMakeLists.txt b/source/dnode/mgmt/daemon/CMakeLists.txt index f1ce726d85..a577344761 100644 --- a/source/dnode/mgmt/daemon/CMakeLists.txt +++ b/source/dnode/mgmt/daemon/CMakeLists.txt @@ -1,8 +1,9 @@ aux_source_directory(src DAEMON_SRC) add_executable(taosd ${DAEMON_SRC}) -target_link_libraries( + +target_include_directories( taosd - PUBLIC dnode - PUBLIC util - PUBLIC os + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) + +target_link_libraries(taosd dnode config util os) diff --git a/source/dnode/mgmt/daemon/inc/dmnInt.h b/source/dnode/mgmt/daemon/inc/dmnInt.h new file mode 100644 index 0000000000..19ec147769 --- /dev/null +++ b/source/dnode/mgmt/daemon/inc/dmnInt.h @@ -0,0 +1,47 @@ + +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_DMN_INT_H_ +#define _TD_DMN_INT_H_ + +#include "config.h" +#include "dnode.h" +#include "taoserror.h" +#include "tglobal.h" +#include "ulog.h" +#include "version.h" + +#ifdef __cplusplus +extern "C" { +#endif + +int32_t dmnAddLogCfg(SConfig *pCfg); +int32_t dmnInitLog(const char *cfgDir, const char *envFile, const char *apolloUrl); +int32_t dmnLoadCfg(SConfig *pConfig, const char *inputCfgDir, const char *envFile, const char *apolloUrl); + +SConfig *dmnReadCfg(const char *cfgDir, const char *envFile, const char *apolloUrl); +SDnodeEnvCfg dmnGetEnvCfg(SConfig *pCfg); +SDnodeObjCfg dmnGetObjCfg(SConfig *pCfg); + +void dmnDumpCfg(SConfig *pCfg); +void dmnPrintVersion(); +void dmnGenerateGrant(); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_DMN_INT_H_*/ diff --git a/source/dnode/mgmt/daemon/src/daemon.c b/source/dnode/mgmt/daemon/src/daemon.c deleted file mode 100644 index ce672bf7b8..0000000000 --- a/source/dnode/mgmt/daemon/src/daemon.c +++ /dev/null @@ -1,218 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "dnode.h" -#include "os.h" -#include "tconfig.h" -#include "tglobal.h" -#include "tnote.h" -#include "ulog.h" - -static struct { - bool stop; - bool dumpConfig; - bool generateGrant; - bool printAuth; - bool printVersion; - char configDir[PATH_MAX]; -} dmn = {0}; - -void dmnSigintHandle(int signum, void *info, void *ctx) { - uInfo("singal:%d is received", signum); - dmn.stop = true; -} - -void dmnSetSignalHandle() { - taosSetSignal(SIGTERM, dmnSigintHandle); - taosSetSignal(SIGHUP, dmnSigintHandle); - taosSetSignal(SIGINT, dmnSigintHandle); - taosSetSignal(SIGABRT, dmnSigintHandle); - taosSetSignal(SIGBREAK, dmnSigintHandle); -} - -int dmnParseOption(int argc, char const *argv[]) { - tstrncpy(dmn.configDir, "/etc/taos", PATH_MAX); - - for (int i = 1; i < argc; ++i) { - if (strcmp(argv[i], "-c") == 0) { - if (i < argc - 1) { - if (strlen(argv[++i]) >= PATH_MAX) { - printf("config file path overflow"); - return -1; - } - tstrncpy(dmn.configDir, argv[i], PATH_MAX); - } else { - printf("'-c' requires a parameter, default is %s\n", configDir); - return -1; - } - } else if (strcmp(argv[i], "-C") == 0) { - dmn.dumpConfig = true; - } else if (strcmp(argv[i], "-k") == 0) { - dmn.generateGrant = true; - } else if (strcmp(argv[i], "-A") == 0) { - dmn.printAuth = true; - } else if (strcmp(argv[i], "-V") == 0) { - dmn.printVersion = true; - } else { - } - } - - return 0; -} - -void dmnGenerateGrant() { -#if 0 - grantParseParameter(); -#endif -} - -void dmnPrintVersion() { -#ifdef TD_ENTERPRISE - char *releaseName = "enterprise"; -#else - char *releaseName = "community"; -#endif - printf("%s version: %s compatible_version: %s\n", releaseName, version, compatible_version); - printf("gitinfo: %s\n", gitinfo); - printf("gitinfoI: %s\n", gitinfoOfInternal); - printf("builuInfo: %s\n", buildinfo); -} - -int dmnReadConfig(const char *path) { - tstrncpy(configDir, dmn.configDir, PATH_MAX); - taosInitGlobalCfg(); - taosReadGlobalLogCfg(); - - if (taosMkDir(tsLogDir) != 0) { - printf("failed to create dir: %s, reason: %s\n", tsLogDir, strerror(errno)); - return -1; - } - - char temp[PATH_MAX]; - snprintf(temp, PATH_MAX, "%s/taosdlog", tsLogDir); - if (taosInitLog(temp, tsNumOfLogLines, 1) != 0) { - printf("failed to init log file\n"); - return -1; - } - - if (taosInitNotes() != 0) { - printf("failed to init log file\n"); - return -1; - } - - if (taosReadCfgFromFile() != 0) { - uError("failed to read config"); - return -1; - } - - if (taosCheckAndPrintCfg() != 0) { - uError("failed to check config"); - return -1; - } - - taosSetCoreDump(tsEnableCoreFile); - return 0; -} - -void dmnDumpConfig() { taosDumpGlobalCfg(); } - -void dmnWaitSignal() { - dmnSetSignalHandle(); - while (!dmn.stop) { - taosMsleep(100); - } -} - -void dnmInitEnvCfg(SDnodeEnvCfg *pCfg) { - pCfg->sver = 30000000; // 3.0.0.0 - pCfg->numOfCores = tsNumOfCores; - pCfg->numOfCommitThreads = tsNumOfCommitThreads; - pCfg->enableTelem = 0; - tstrncpy(pCfg->timezone, tsTimezone, TSDB_TIMEZONE_LEN); - tstrncpy(pCfg->locale, tsLocale, TSDB_LOCALE_LEN); - tstrncpy(pCfg->charset, tsCharset, TSDB_LOCALE_LEN); - tstrncpy(pCfg->buildinfo, buildinfo, 64); - tstrncpy(pCfg->gitinfo, gitinfo, 48); -} - -void dmnInitObjCfg(SDnodeObjCfg *pCfg) { - pCfg->numOfSupportVnodes = tsNumOfSupportVnodes; - pCfg->statusInterval = tsStatusInterval; - pCfg->numOfThreadsPerCore = tsNumOfThreadsPerCore; - pCfg->ratioOfQueryCores = tsRatioOfQueryCores; - pCfg->maxShellConns = tsMaxShellConns; - pCfg->shellActivityTimer = tsShellActivityTimer; - pCfg->serverPort = tsServerPort; - tstrncpy(pCfg->dataDir, tsDataDir, TSDB_FILENAME_LEN); - tstrncpy(pCfg->localEp, tsLocalEp, TSDB_EP_LEN); - tstrncpy(pCfg->localFqdn, tsLocalFqdn, TSDB_FQDN_LEN); - tstrncpy(pCfg->firstEp, tsFirst, TSDB_EP_LEN); -} - -int dmnRunDnode() { - SDnodeEnvCfg envCfg = {0}; - SDnodeObjCfg objCfg = {0}; - - dnmInitEnvCfg(&envCfg); - dmnInitObjCfg(&objCfg); - - if (dndInit(&envCfg) != 0) { - uInfo("Failed to start TDengine, please check the log at %s", tsLogDir); - return -1; - } - - SDnode *pDnode = dndCreate(&objCfg); - if (pDnode == NULL) { - uInfo("Failed to start TDengine, please check the log at %s", tsLogDir); - return -1; - } - - uInfo("Started TDengine service successfully."); - dmnWaitSignal(); - uInfo("TDengine is shut down!"); - - dndClose(pDnode); - dndCleanup(); - taosCloseLog(); - return 0; -} - -int main(int argc, char const *argv[]) { - if (dmnParseOption(argc, argv) != 0) { - return -1; - } - - if (dmn.generateGrant) { - dmnGenerateGrant(); - return 0; - } - - if (dmn.printVersion) { - dmnPrintVersion(); - return 0; - } - - if (dmnReadConfig(dmn.configDir) != 0) { - return -1; - } - - if (dmn.dumpConfig) { - dmnDumpConfig(); - return 0; - } - - return dmnRunDnode(); -} diff --git a/source/dnode/mgmt/daemon/src/dmnCfg.c b/source/dnode/mgmt/daemon/src/dmnCfg.c new file mode 100644 index 0000000000..7bf538caed --- /dev/null +++ b/source/dnode/mgmt/daemon/src/dmnCfg.c @@ -0,0 +1,208 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "dmnInt.h" +#include "tlocale.h" +#include "ttimezone.h" + +static int32_t dmnAddEpCfg(SConfig *pCfg) { + char defaultFqdn[TSDB_FQDN_LEN] = {0}; + if (taosGetFqdn(defaultFqdn) != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + if (cfgAddString(pCfg, "fqdn", defaultFqdn) != 0) return -1; + + int32_t defaultServerPort = 6030; + if (cfgAddInt32(pCfg, "serverPort", defaultServerPort, 1, 65056) != 0) return -1; + + char defaultFirstEp[TSDB_EP_LEN] = {0}; + char defaultSecondEp[TSDB_EP_LEN] = {0}; + snprintf(defaultFirstEp, TSDB_EP_LEN, "%s:%d", defaultFqdn, defaultServerPort); + snprintf(defaultSecondEp, TSDB_EP_LEN, "%s:%d", defaultFqdn, defaultServerPort); + if (cfgAddString(pCfg, "firstEp", defaultFirstEp) != 0) return -1; + if (cfgAddString(pCfg, "secondEp", defaultSecondEp) != 0) return -1; + + return 0; +} + +static int32_t dmnAddDirCfg(SConfig *pCfg) { + if (cfgAddDir(pCfg, "dataDir", tsDataDir) != 0) return -1; + if (cfgAddDir(pCfg, "tempDir", tsTempDir) != 0) return -1; + if (cfgAddFloat(pCfg, "minimalDataDirGB", 2.0f, 0.001f, 10000000) != 0) return -1; + if (cfgAddFloat(pCfg, "minimalTempDirGB", 1.0f, 0.001f, 10000000) != 0) return -1; + return 0; +} + +static int32_t dmnCheckDirCfg(SConfig *pCfg) { + tstrncpy(tsDataDir, cfgGetItem(pCfg, "dataDir")->str, PATH_MAX); + tstrncpy(tsTempDir, cfgGetItem(pCfg, "tempDir")->str, PATH_MAX); + tsDataSpace.reserved = cfgGetItem(pCfg, "minimalDataDirGB")->fval; + tsTempSpace.reserved = cfgGetItem(pCfg, "minimalTempDirGB")->fval; + return 0; +} + +static int32_t dmnAddVersionCfg(SConfig *pCfg) { + if (cfgAddString(pCfg, "buildinfo", buildinfo) != 0) return -1; + if (cfgAddString(pCfg, "gitinfo", gitinfo) != 0) return -1; + if (cfgAddString(pCfg, "version", version) != 0) return -1; + return 0; +} + +static int32_t dmnAddDnodeCfg(SConfig *pCfg) { + if (dmnAddEpCfg(pCfg) != 0) return -1; + if (dmnAddDirCfg(pCfg) != 0) return -1; + if (dmnAddVersionCfg(pCfg) != 0) return -1; + + if (cfgAddTimezone(pCfg, "timezone", "") != 0) return -1; + if (cfgAddLocale(pCfg, "locale", "") != 0) return -1; + if (cfgAddCharset(pCfg, "charset", "") != 0) return -1; + if (cfgAddInt32(pCfg, "numOfCores", 2, 1, 100000) != 0) return -1; + if (cfgAddInt32(pCfg, "numOfCommitThreads", 4, 1, 1000) != 0) return -1; + if (cfgAddBool(pCfg, "telemetryReporting", 0) != 0) return -1; + if (cfgAddBool(pCfg, "enableCoreFile", 0) != 0) return -1; + if (cfgAddInt32(pCfg, "supportVnodes", 256, 0, 65536) != 0) return -1; + if (cfgAddInt32(pCfg, "statusInterval", 1, 1, 30) != 0) return -1; + if (cfgAddFloat(pCfg, "numOfThreadsPerCore", 1, 0, 10) != 0) return -1; + if (cfgAddFloat(pCfg, "ratioOfQueryCores", 1, 0, 5) != 0) return -1; + if (cfgAddInt32(pCfg, "maxShellConns", 50000, 10, 50000000) != 0) return -1; + if (cfgAddInt32(pCfg, "shellActivityTimer", 3, 1, 120) != 0) return -1; + return 0; +} + +static int32_t dmnCheckCfg(SConfig *pCfg) { + bool enableCore = cfgGetItem(pCfg, "enableCoreFile")->bval; + taosSetCoreDump(enableCore); + + if (dmnCheckDirCfg(pCfg) != 0) { + return -1; + } + + taosGetSystemInfo(); + + tsSetTimeZone(); + tsSetLocale(); + + if (tsNumOfCores <= 0) { + tsNumOfCores = 1; + } + + if (tsQueryBufferSize >= 0) { + tsQueryBufferSizeBytes = tsQueryBufferSize * 1048576UL; + } + + return 0; +} + +SConfig *dmnReadCfg(const char *cfgDir, const char *envFile, const char *apolloUrl) { + SConfig *pCfg = cfgInit(); + if (pCfg == NULL) return NULL; + + if (dmnAddLogCfg(pCfg) != 0) { + uError("failed to add log cfg since %s", terrstr()); + cfgCleanup(pCfg); + return NULL; + } + + if (dmnAddDnodeCfg(pCfg) != 0) { + uError("failed to init dnode cfg since %s", terrstr()); + cfgCleanup(pCfg); + return NULL; + } + + if (dmnLoadCfg(pCfg, cfgDir, envFile, apolloUrl) != 0) { + uError("failed to load cfg since %s", terrstr()); + cfgCleanup(pCfg); + return NULL; + } + + if (dmnCheckCfg(pCfg) != 0) { + uError("failed to check cfg since %s", terrstr()); + cfgCleanup(pCfg); + return NULL; + } + + cfgDumpCfg(pCfg); + return pCfg; +} + +void dmnDumpCfg(SConfig *pCfg) { + printf("taos global config:\n"); + printf("==================================\n"); + + SConfigItem *pItem = cfgIterate(pCfg, NULL); + while (pItem != NULL) { + switch (pItem->dtype) { + case CFG_DTYPE_BOOL: + printf("cfg:%s, value:%u src:%s\n", pItem->name, pItem->bval, cfgStypeStr(pItem->stype)); + break; + case CFG_DTYPE_INT32: + printf("cfg:%s, value:%d src:%s\n", pItem->name, pItem->i32, cfgStypeStr(pItem->stype)); + break; + case CFG_DTYPE_INT64: + printf("cfg:%s, value:%" PRId64 " src:%s\n", pItem->name, pItem->i64, cfgStypeStr(pItem->stype)); + break; + case CFG_DTYPE_FLOAT: + printf("cfg:%s, value:%f src:%s\n", pItem->name, pItem->fval, cfgStypeStr(pItem->stype)); + break; + case CFG_DTYPE_STRING: + case CFG_DTYPE_IPSTR: + case CFG_DTYPE_DIR: + case CFG_DTYPE_LOCALE: + case CFG_DTYPE_CHARSET: + case CFG_DTYPE_TIMEZONE: + printf("cfg:%s, value:%s src:%s\n", pItem->name, pItem->str, cfgStypeStr(pItem->stype)); + break; + } + pItem = cfgIterate(pCfg, pItem); + } +} + +SDnodeEnvCfg dmnGetEnvCfg(SConfig *pCfg) { + SDnodeEnvCfg envCfg = {0}; + + const char *vstr = cfgGetItem(pCfg, "version")->str; + envCfg.sver = 30000000; + tstrncpy(envCfg.buildinfo, cfgGetItem(pCfg, "buildinfo")->str, sizeof(envCfg.buildinfo)); + tstrncpy(envCfg.gitinfo, cfgGetItem(pCfg, "gitinfo")->str, sizeof(envCfg.gitinfo)); + tstrncpy(envCfg.timezone, cfgGetItem(pCfg, "timezone")->str, sizeof(envCfg.timezone)); + tstrncpy(envCfg.locale, cfgGetItem(pCfg, "locale")->str, sizeof(envCfg.locale)); + tstrncpy(envCfg.charset, cfgGetItem(pCfg, "charset")->str, sizeof(envCfg.charset)); + envCfg.numOfCores = cfgGetItem(pCfg, "numOfCores")->i32; + envCfg.numOfCommitThreads = (uint16_t)cfgGetItem(pCfg, "numOfCommitThreads")->i32; + envCfg.enableTelem = cfgGetItem(pCfg, "telemetryReporting")->bval; + + return envCfg; +} + +SDnodeObjCfg dmnGetObjCfg(SConfig *pCfg) { + SDnodeObjCfg objCfg = {0}; + + objCfg.numOfSupportVnodes = cfgGetItem(pCfg, "supportVnodes")->i32; + objCfg.statusInterval = cfgGetItem(pCfg, "statusInterval")->i32; + objCfg.numOfThreadsPerCore = cfgGetItem(pCfg, "numOfThreadsPerCore")->fval; + objCfg.ratioOfQueryCores = cfgGetItem(pCfg, "ratioOfQueryCores")->fval; + objCfg.maxShellConns = cfgGetItem(pCfg, "maxShellConns")->i32; + objCfg.shellActivityTimer = cfgGetItem(pCfg, "shellActivityTimer")->i32; + tstrncpy(objCfg.dataDir, cfgGetItem(pCfg, "dataDir")->str, sizeof(objCfg.dataDir)); + + tstrncpy(objCfg.firstEp, cfgGetItem(pCfg, "firstEp")->str, sizeof(objCfg.firstEp)); + tstrncpy(objCfg.secondEp, cfgGetItem(pCfg, "secondEp")->str, sizeof(objCfg.firstEp)); + objCfg.serverPort = (uint16_t)cfgGetItem(pCfg, "serverPort")->i32; + tstrncpy(objCfg.localFqdn, cfgGetItem(pCfg, "fqdn")->str, sizeof(objCfg.localFqdn)); + snprintf(objCfg.localEp, sizeof(objCfg.localEp), "%s:%u", objCfg.localFqdn, objCfg.serverPort); + return objCfg; +} \ No newline at end of file diff --git a/source/dnode/mgmt/daemon/src/dmnLog.c b/source/dnode/mgmt/daemon/src/dmnLog.c new file mode 100644 index 0000000000..22a7d7b080 --- /dev/null +++ b/source/dnode/mgmt/daemon/src/dmnLog.c @@ -0,0 +1,132 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "dmnInt.h" + +int32_t dmnAddLogCfg(SConfig *pCfg) { + if (cfgAddDir(pCfg, "logDir", "/var/log/taos") != 0) return -1; + if (cfgAddFloat(pCfg, "minimalLogDirGB", 1.0f, 0.001f, 10000000) != 0) return -1; + if (cfgAddBool(pCfg, "asyncLog", 1) != 0) return -1; + if (cfgAddInt32(pCfg, "numOfLogLines", 10000000, 1000, 2000000000) != 0) return -1; + if (cfgAddInt32(pCfg, "logKeepDays", 0, -365000, 365000) != 0) return -1; + if (cfgAddInt32(pCfg, "debugFlag", 0, 0, 255) != 0) return -1; + if (cfgAddInt32(pCfg, "dDebugFlag", 0, 0, 255) != 0) return -1; + if (cfgAddInt32(pCfg, "vDebugFlag", 0, 0, 255) != 0) return -1; + if (cfgAddInt32(pCfg, "mDebugFlag", 0, 0, 255) != 0) return -1; + if (cfgAddInt32(pCfg, "cDebugFlag", 0, 0, 255) != 0) return -1; + if (cfgAddInt32(pCfg, "jniDebugFlag", 0, 0, 255) != 0) return -1; + if (cfgAddInt32(pCfg, "tmrDebugFlag", 0, 0, 255) != 0) return -1; + if (cfgAddInt32(pCfg, "uDebugFlag", 0, 0, 255) != 0) return -1; + if (cfgAddInt32(pCfg, "rpcDebugFlag", 0, 0, 255) != 0) return -1; + if (cfgAddInt32(pCfg, "qDebugFlag", 0, 0, 255) != 0) return -1; + if (cfgAddInt32(pCfg, "wDebugFlag", 0, 0, 255) != 0) return -1; + if (cfgAddInt32(pCfg, "sDebugFlag", 0, 0, 255) != 0) return -1; + if (cfgAddInt32(pCfg, "tsdbDebugFlag", 0, 0, 255) != 0) return -1; + if (cfgAddInt32(pCfg, "tqDebugFlag", 0, 0, 255) != 0) return -1; + if (cfgAddInt32(pCfg, "fsDebugFlag", 0, 0, 255) != 0) return -1; + return 0; +} + +int32_t dmnSetLogCfg(SConfig *pCfg) { + tstrncpy(tsLogDir, cfgGetItem(pCfg, "logDir")->str, PATH_MAX); + tsLogSpace.reserved = cfgGetItem(pCfg, "minimalLogDirGB")->fval; + tsAsyncLog = cfgGetItem(pCfg, "asyncLog")->bval; + tsNumOfLogLines = cfgGetItem(pCfg, "numOfLogLines")->i32; + tsLogKeepDays = cfgGetItem(pCfg, "logKeepDays")->i32; + dDebugFlag = cfgGetItem(pCfg, "dDebugFlag")->i32; + vDebugFlag = cfgGetItem(pCfg, "vDebugFlag")->i32; + mDebugFlag = cfgGetItem(pCfg, "mDebugFlag")->i32; + cDebugFlag = cfgGetItem(pCfg, "cDebugFlag")->i32; + jniDebugFlag = cfgGetItem(pCfg, "jniDebugFlag")->i32; + tmrDebugFlag = cfgGetItem(pCfg, "tmrDebugFlag")->i32; + uDebugFlag = cfgGetItem(pCfg, "uDebugFlag")->i32; + rpcDebugFlag = cfgGetItem(pCfg, "rpcDebugFlag")->i32; + qDebugFlag = cfgGetItem(pCfg, "qDebugFlag")->i32; + wDebugFlag = cfgGetItem(pCfg, "wDebugFlag")->i32; + sDebugFlag = cfgGetItem(pCfg, "sDebugFlag")->i32; + tsdbDebugFlag = cfgGetItem(pCfg, "tsdbDebugFlag")->i32; + tqDebugFlag = cfgGetItem(pCfg, "tqDebugFlag")->i32; + fsDebugFlag = cfgGetItem(pCfg, "fsDebugFlag")->i32; + + int32_t debugFlag = cfgGetItem(pCfg, "debugFlag")->i32; + taosSetAllDebugFlag(debugFlag); + + return 0; +} + +int32_t dmnInitLog(const char *cfgDir, const char *envFile, const char *apolloUrl) { + SConfig *pCfg = cfgInit(); + if (pCfg == NULL) return -1; + + if (dmnAddLogCfg(pCfg) != 0) { + printf("failed to add log cfg since %s\n", terrstr()); + cfgCleanup(pCfg); + return -1; + } + + if (dmnLoadCfg(pCfg, cfgDir, envFile, apolloUrl) != 0) { + printf("failed to load log cfg since %s\n", terrstr()); + cfgCleanup(pCfg); + return -1; + } + + if (dmnSetLogCfg(pCfg) != 0) { + printf("failed to set log cfg since %s\n", terrstr()); + cfgCleanup(pCfg); + return -1; + } + + if (taosInitLog("taosdlog", 1) != 0) { + printf("failed to init log file since %s\n", terrstr()); + cfgCleanup(pCfg); + return -1; + } + + cfgCleanup(pCfg); + return 0; +} + +int32_t dmnLoadCfg(SConfig *pConfig, const char *inputCfgDir, const char *envFile, const char *apolloUrl) { + char configDir[PATH_MAX] = {0}; + char configFile[PATH_MAX + 100] = {0}; + + taosExpandDir(inputCfgDir, configDir, PATH_MAX); + snprintf(configFile, sizeof(configFile), "%s" TD_DIRSEP "taos.cfg", configDir); + + if (cfgLoad(pConfig, CFG_STYPE_APOLLO_URL, apolloUrl) != 0) { + uError("failed to load from apollo url:%s since %s\n", apolloUrl, terrstr()); + return -1; + } + + if (cfgLoad(pConfig, CFG_STYPE_CFG_FILE, configFile) != 0) { + if (cfgLoad(pConfig, CFG_STYPE_CFG_FILE, configDir) != 0) { + uError("failed to load from config file:%s since %s\n", configFile, terrstr()); + return -1; + } + } + + if (cfgLoad(pConfig, CFG_STYPE_ENV_FILE, envFile) != 0) { + uError("failed to load from env file:%s since %s\n", envFile, terrstr()); + return -1; + } + + if (cfgLoad(pConfig, CFG_STYPE_ENV_VAR, NULL) != 0) { + uError("failed to load from global env variables since %s\n", terrstr()); + return -1; + } + + return 0; +} diff --git a/source/dnode/mgmt/daemon/src/dmnMain.c b/source/dnode/mgmt/daemon/src/dmnMain.c new file mode 100644 index 0000000000..1f1d2d8d05 --- /dev/null +++ b/source/dnode/mgmt/daemon/src/dmnMain.c @@ -0,0 +1,133 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "dmnInt.h" + +static struct { + bool stop; + bool dumpConfig; + bool generateGrant; + bool printAuth; + bool printVersion; + char envFile[PATH_MAX]; + char apolloUrl[PATH_MAX]; +} dmn = {0}; + +static void dmnSigintHandle(int signum, void *info, void *ctx) { + uInfo("singal:%d is received", signum); + dmn.stop = true; +} + +static void dmnSetSignalHandle() { + taosSetSignal(SIGTERM, dmnSigintHandle); + taosSetSignal(SIGHUP, dmnSigintHandle); + taosSetSignal(SIGINT, dmnSigintHandle); + taosSetSignal(SIGABRT, dmnSigintHandle); + taosSetSignal(SIGBREAK, dmnSigintHandle); +} + +static void dmnWaitSignal() { + dmnSetSignalHandle(); + while (!dmn.stop) { + taosMsleep(100); + } +} + +static int32_t dmnParseOption(int32_t argc, char const *argv[]) { + for (int32_t i = 1; i < argc; ++i) { + if (strcmp(argv[i], "-c") == 0) { + if (i < argc - 1) { + if (strlen(argv[++i]) >= PATH_MAX) { + printf("config file path overflow"); + return -1; + } + tstrncpy(configDir, argv[i], PATH_MAX); + } else { + printf("'-c' requires a parameter, default is %s\n", configDir); + return -1; + } + } else if (strcmp(argv[i], "-C") == 0) { + dmn.dumpConfig = true; + } else if (strcmp(argv[i], "-k") == 0) { + dmn.generateGrant = true; + } else if (strcmp(argv[i], "-V") == 0) { + dmn.printVersion = true; + } else { + } + } + + return 0; +} + +int32_t dmnRunDnode(SConfig *pCfg) { + SDnodeEnvCfg envCfg = dmnGetEnvCfg(pCfg); + if (dndInit(&envCfg) != 0) { + uInfo("Failed to start TDengine, please check the log"); + return -1; + } + + SDnodeObjCfg objCfg = dmnGetObjCfg(pCfg); + SDnode *pDnode = dndCreate(&objCfg); + if (pDnode == NULL) { + uInfo("Failed to start TDengine, please check the log"); + return -1; + } + + uInfo("Started TDengine service successfully."); + dmnWaitSignal(); + uInfo("TDengine is shut down!"); + + dndClose(pDnode); + dndCleanup(); + taosCloseLog(); + return 0; +} + +int main(int argc, char const *argv[]) { + if (dmnParseOption(argc, argv) != 0) { + return -1; + } + + if (dmn.generateGrant) { + dmnGenerateGrant(); + return 0; + } + + if (dmn.printVersion) { + dmnPrintVersion(); + return 0; + } + + if (dmnInitLog(configDir, dmn.envFile, dmn.apolloUrl) != 0) { + return -1; + } + + SConfig *pCfg = dmnReadCfg(configDir, dmn.envFile, dmn.apolloUrl); + if (pCfg == NULL) { + uInfo("Failed to start TDengine since read config error"); + return -1; + } + + if (dmn.dumpConfig) { + dmnDumpCfg(pCfg); + cfgCleanup(pCfg); + return 0; + } + + int32_t code = dmnRunDnode(pCfg); + cfgCleanup(pCfg); + return code; +} diff --git a/source/dnode/mgmt/daemon/src/dmnUtil.c b/source/dnode/mgmt/daemon/src/dmnUtil.c new file mode 100644 index 0000000000..608bc7f3e6 --- /dev/null +++ b/source/dnode/mgmt/daemon/src/dmnUtil.c @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "dmnInt.h" + +void dmnGenerateGrant() { +#if 0 + grantParseParameter(); +#endif +} + +void dmnPrintVersion() { +#ifdef TD_ENTERPRISE + char *releaseName = "enterprise"; +#else + char *releaseName = "community"; +#endif + printf("%s version: %s compatible_version: %s\n", releaseName, version, compatible_version); + printf("gitinfo: %s\n", gitinfo); + printf("gitinfoI: %s\n", gitinfoOfInternal); + printf("builuInfo: %s\n", buildinfo); +} diff --git a/source/dnode/mgmt/impl/src/dndEnv.c b/source/dnode/mgmt/impl/src/dndEnv.c index 02dced53c2..58517c8151 100644 --- a/source/dnode/mgmt/impl/src/dndEnv.c +++ b/source/dnode/mgmt/impl/src/dndEnv.c @@ -325,18 +325,6 @@ void taosGetDisk() { SDiskSize diskSize = tfsGetSize(pTfs); tfsUpdateSize(&fsMeta); - tsTotalDataDirGB = (float)(fsMeta.total / unit); - tsUsedDataDirGB = (float)(fsMeta.used / unit); - tsAvailDataDirGB = (float)(fsMeta.avail / unit); - if (taosGetDiskSize(tsLogDir, &diskSize) == 0) { - tsTotalLogDirGB = (float)(diskSize.total / unit); - tsAvailLogDirGB = (float)(diskSize.avail / unit); - } - - if (taosGetDiskSize(tsTempDir, &diskSize) == 0) { - tsTotalTmpDirGB = (float)(diskSize.total / unit); - tsAvailTmpDirectorySpace = (float)(diskSize.avail / unit); - } #endif } \ No newline at end of file diff --git a/source/dnode/mgmt/impl/src/dndMgmt.c b/source/dnode/mgmt/impl/src/dndMgmt.c index 3d149a3e60..1748c52c66 100644 --- a/source/dnode/mgmt/impl/src/dndMgmt.c +++ b/source/dnode/mgmt/impl/src/dndMgmt.c @@ -296,7 +296,7 @@ PRASE_DNODE_OVER: if (taosArrayGetSize(pMgmt->pDnodeEps) == 0) { SDnodeEp dnodeEp = {0}; dnodeEp.isMnode = 1; - taosGetFqdnPortFromEp(pDnode->cfg.firstEp, &dnodeEp.ep); + taosGetFqdnPortFromEp(pDnode->cfg.firstEp, pDnode->cfg.serverPort, &dnodeEp.ep); taosArrayPush(pMgmt->pDnodeEps, &dnodeEp); } diff --git a/source/dnode/mgmt/impl/src/dndTransport.c b/source/dnode/mgmt/impl/src/dndTransport.c index 4e35baf905..0aae145d2f 100644 --- a/source/dnode/mgmt/impl/src/dndTransport.c +++ b/source/dnode/mgmt/impl/src/dndTransport.c @@ -25,8 +25,8 @@ #include "dndMnode.h" #include "dndVnodes.h" -#define INTERNAL_USER "_dnd" -#define INTERNAL_CKEY "_key" +#define INTERNAL_USER "_dnd" +#define INTERNAL_CKEY "_key" #define INTERNAL_SECRET "_pwd" static void dndInitMsgFp(STransMgmt *pMgmt) { @@ -113,6 +113,7 @@ static void dndInitMsgFp(STransMgmt *pMgmt) { pMgmt->msgFp[TMSG_INDEX(TDMT_MND_ALTER_TOPIC)] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_TOPIC)] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_MND_SUBSCRIBE)] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_MND_MQ_COMMIT_OFFSET)] = dndProcessMnodeWriteMsg; /*pMgmt->msgFp[TMSG_INDEX(TDMT_VND_SUBSCRIBE_RSP)] = dndProcessMnodeWriteMsg;*/ pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_SET_CONN_RSP)] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_REB_RSP)] = dndProcessMnodeWriteMsg; @@ -155,7 +156,7 @@ static void dndInitMsgFp(STransMgmt *pMgmt) { } static void dndProcessResponse(void *parent, SRpcMsg *pRsp, SEpSet *pEpSet) { - SDnode * pDnode = parent; + SDnode *pDnode = parent; STransMgmt *pMgmt = &pDnode->tmgmt; tmsg_t msgType = pRsp->msgType; @@ -219,7 +220,7 @@ static void dndCleanupClient(SDnode *pDnode) { } static void dndProcessRequest(void *param, SRpcMsg *pReq, SEpSet *pEpSet) { - SDnode * pDnode = param; + SDnode *pDnode = param; STransMgmt *pMgmt = &pDnode->tmgmt; tmsg_t msgType = pReq->msgType; @@ -313,7 +314,7 @@ static int32_t dndRetrieveUserAuthInfo(void *parent, char *user, char *spi, char SAuthReq authReq = {0}; tstrncpy(authReq.user, user, TSDB_USER_LEN); int32_t contLen = tSerializeSAuthReq(NULL, 0, &authReq); - void * pReq = rpcMallocCont(contLen); + void *pReq = rpcMallocCont(contLen); tSerializeSAuthReq(pReq, contLen, &authReq); SRpcMsg rpcMsg = {.pCont = pReq, .contLen = contLen, .msgType = TDMT_MND_AUTH, .ahandle = (void *)9528}; diff --git a/source/dnode/mgmt/impl/test/sut/inc/sut.h b/source/dnode/mgmt/impl/test/sut/inc/sut.h index 250d563a8b..5e42ee78df 100644 --- a/source/dnode/mgmt/impl/test/sut/inc/sut.h +++ b/source/dnode/mgmt/impl/test/sut/inc/sut.h @@ -20,7 +20,7 @@ #include "os.h" #include "dnode.h" -#include "tconfig.h" +#include "tmsg.h" #include "tdataformat.h" #include "tglobal.h" #include "tmsg.h" diff --git a/source/dnode/mgmt/impl/test/sut/src/sut.cpp b/source/dnode/mgmt/impl/test/sut/src/sut.cpp index 771c5886ef..d32bdf29c6 100644 --- a/source/dnode/mgmt/impl/test/sut/src/sut.cpp +++ b/source/dnode/mgmt/impl/test/sut/src/sut.cpp @@ -28,16 +28,13 @@ void Testbase::InitLog(const char* path) { wDebugFlag = 0; sDebugFlag = 0; tsdbDebugFlag = 0; - cqDebugFlag = 0; tscEmbeddedInUtil = 1; tsAsyncLog = 0; taosRemoveDir(path); taosMkDir(path); - - char temp[PATH_MAX]; - snprintf(temp, PATH_MAX, "%s/taosdlog", path); - if (taosInitLog(temp, tsNumOfLogLines, 1) != 0) { + tstrncpy(tsLogDir, path, PATH_MAX); + if (taosInitLog("taosdlog", 1) != 0) { printf("failed to init log file\n"); } } diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index e64191f715..bb2367ef72 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -123,6 +123,7 @@ typedef enum { TRN_TYPE_DROP_TOPIC = 1015, TRN_TYPE_SUBSCRIBE = 1016, TRN_TYPE_REBALANCE = 1017, + TRN_TYPE_COMMIT_OFFSET = 1018, TRN_TYPE_BASIC_SCOPE_END, TRN_TYPE_GLOBAL_SCOPE = 2000, TRN_TYPE_CREATE_DNODE = 2001, @@ -176,7 +177,7 @@ typedef struct { SArray* undoActions; int64_t createdTime; int64_t lastExecTime; - uint64_t dbUid; + int64_t dbUid; char dbname[TSDB_DB_FNAME_LEN]; char lastError[TSDB_TRANS_ERROR_LEN]; } STrans; @@ -304,16 +305,16 @@ typedef struct { } SDbCfg; typedef struct { - char name[TSDB_DB_FNAME_LEN]; - char acct[TSDB_USER_LEN]; - char createUser[TSDB_USER_LEN]; - int64_t createdTime; - int64_t updateTime; - uint64_t uid; - int32_t cfgVersion; - int32_t vgVersion; - int8_t hashMethod; // default is 1 - SDbCfg cfg; + char name[TSDB_DB_FNAME_LEN]; + char acct[TSDB_USER_LEN]; + char createUser[TSDB_USER_LEN]; + int64_t createdTime; + int64_t updateTime; + int64_t uid; + int32_t cfgVersion; + int32_t vgVersion; + int8_t hashMethod; // default is 1 + SDbCfg cfg; } SDbObj; typedef struct { @@ -346,8 +347,8 @@ typedef struct { char db[TSDB_DB_FNAME_LEN]; int64_t createdTime; int64_t updateTime; - uint64_t uid; - uint64_t dbUid; + int64_t uid; + int64_t dbUid; int32_t version; int32_t nextColId; int32_t numOfColumns; @@ -465,6 +466,24 @@ static FORCE_INLINE void tDeleteSMqSubConsumer(SMqSubConsumer* pSubConsumer) { } } +typedef struct { + char key[TSDB_PARTITION_KEY_LEN]; + int64_t offset; +} SMqOffsetObj; + +static FORCE_INLINE int32_t tEncodeSMqOffsetObj(void** buf, const SMqOffsetObj* pOffset) { + int32_t tlen = 0; + tlen += taosEncodeString(buf, pOffset->key); + tlen += taosEncodeFixedI64(buf, pOffset->offset); + return tlen; +} + +static FORCE_INLINE void* tDecodeSMqOffsetObj(void* buf, SMqOffsetObj* pOffset) { + buf = taosDecodeStringTo(buf, pOffset->key); + buf = taosDecodeFixedI64(buf, &pOffset->offset); + return buf; +} + typedef struct { char key[TSDB_SUBSCRIBE_KEY_LEN]; int32_t status; diff --git a/source/dnode/mnode/impl/inc/mndInt.h b/source/dnode/mnode/impl/inc/mndInt.h index 97cd6caf04..5c32da966b 100644 --- a/source/dnode/mnode/impl/inc/mndInt.h +++ b/source/dnode/mnode/impl/inc/mndInt.h @@ -56,12 +56,9 @@ typedef struct { } SProfileMgmt; typedef struct { - int8_t enable; - pthread_mutex_t lock; - pthread_cond_t cond; - volatile int32_t exit; - pthread_t thread; - char email[TSDB_FQDN_LEN]; + bool enable; + SRWLatch lock; + char email[TSDB_FQDN_LEN]; } STelemMgmt; typedef struct { @@ -81,6 +78,7 @@ typedef struct SMnode { tmr_h timer; tmr_h transTimer; tmr_h mqTimer; + tmr_h telemTimer; char *path; SMnodeCfg cfg; int64_t checkTime; diff --git a/source/dnode/mnode/impl/inc/mndOffset.h b/source/dnode/mnode/impl/inc/mndOffset.h new file mode 100644 index 0000000000..e18eec973f --- /dev/null +++ b/source/dnode/mnode/impl/inc/mndOffset.h @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_MND_OFFSET_H_ +#define _TD_MND_OFFSET_H_ + +#include "mndInt.h" + +#ifdef __cplusplus +extern "C" { +#endif + +int32_t mndInitOffset(SMnode *pMnode); +void mndCleanupOffset(SMnode *pMnode); + +SMqOffsetObj *mndAcquireOffset(SMnode *pMnode, const char *key); +void mndReleaseOffset(SMnode *pMnode, SMqOffsetObj *pOffset); + +SSdbRaw *mndOffsetActionEncode(SMqOffsetObj *pOffset); +SSdbRow *mndOffsetActionDecode(SSdbRaw *pRaw); + +int32_t mndCreateOffset(STrans *pTrans, const char *cgroup, const char *topicName, const SArray *vgs); + +static FORCE_INLINE int32_t mndMakePartitionKey(char *key, const char *cgroup, const char *topicName, int32_t vgId) { + return snprintf(key, TSDB_PARTITION_KEY_LEN, "%d:%s:%s", vgId, cgroup, topicName); +} + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_MND_OFFSET_H_*/ diff --git a/source/dnode/mnode/impl/src/mndCluster.c b/source/dnode/mnode/impl/src/mndCluster.c index a41ccd9896..3410a386da 100644 --- a/source/dnode/mnode/impl/src/mndCluster.c +++ b/source/dnode/mnode/impl/src/mndCluster.c @@ -52,7 +52,7 @@ int32_t mndGetClusterName(SMnode *pMnode, char *clusterName, int32_t len) { SSdb *pSdb = pMnode->pSdb; SClusterObj *pCluster = sdbAcquire(pSdb, SDB_CLUSTER, &pMnode->clusterId); - if (pCluster = NULL) { + if (pCluster == NULL) { return -1; } diff --git a/source/dnode/mnode/impl/src/mndOffset.c b/source/dnode/mnode/impl/src/mndOffset.c new file mode 100644 index 0000000000..ac9e99ebd4 --- /dev/null +++ b/source/dnode/mnode/impl/src/mndOffset.c @@ -0,0 +1,224 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "mndOffset.h" +#include "mndAuth.h" +#include "mndDb.h" +#include "mndDnode.h" +#include "mndMnode.h" +#include "mndShow.h" +#include "mndStb.h" +#include "mndTrans.h" +#include "mndUser.h" +#include "mndVgroup.h" +#include "tname.h" + +#define MND_OFFSET_VER_NUMBER 1 +#define MND_OFFSET_RESERVE_SIZE 64 + +static int32_t mndOffsetActionInsert(SSdb *pSdb, SMqOffsetObj *pOffset); +static int32_t mndOffsetActionDelete(SSdb *pSdb, SMqOffsetObj *pOffset); +static int32_t mndOffsetActionUpdate(SSdb *pSdb, SMqOffsetObj *pOffset, SMqOffsetObj *pNewOffset); +static int32_t mndProcessCommitOffsetReq(SMnodeMsg *pReq); + +int32_t mndInitOffset(SMnode *pMnode) { + SSdbTable table = {.sdbType = SDB_OFFSET, + .keyType = SDB_KEY_BINARY, + .encodeFp = (SdbEncodeFp)mndOffsetActionEncode, + .decodeFp = (SdbDecodeFp)mndOffsetActionDecode, + .insertFp = (SdbInsertFp)mndOffsetActionInsert, + .updateFp = (SdbUpdateFp)mndOffsetActionUpdate, + .deleteFp = (SdbDeleteFp)mndOffsetActionDelete}; + + mndSetMsgHandle(pMnode, TDMT_MND_MQ_COMMIT_OFFSET, mndProcessCommitOffsetReq); + + return sdbSetTable(pMnode->pSdb, table); +} + +void mndCleanupOffset(SMnode *pMnode) {} + +SSdbRaw *mndOffsetActionEncode(SMqOffsetObj *pOffset) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + void *buf = NULL; + int32_t tlen = tEncodeSMqOffsetObj(NULL, pOffset); + int32_t size = sizeof(int32_t) + tlen + MND_OFFSET_RESERVE_SIZE; + + SSdbRaw *pRaw = sdbAllocRaw(SDB_OFFSET, MND_OFFSET_VER_NUMBER, size); + if (pRaw == NULL) goto OFFSET_ENCODE_OVER; + + buf = malloc(tlen); + if (buf == NULL) goto OFFSET_ENCODE_OVER; + + void *abuf = buf; + tEncodeSMqOffsetObj(&abuf, pOffset); + + int32_t dataPos = 0; + SDB_SET_INT32(pRaw, dataPos, tlen, OFFSET_ENCODE_OVER); + SDB_SET_BINARY(pRaw, dataPos, buf, tlen, OFFSET_ENCODE_OVER); + SDB_SET_RESERVE(pRaw, dataPos, MND_OFFSET_RESERVE_SIZE, OFFSET_ENCODE_OVER); + SDB_SET_DATALEN(pRaw, dataPos, OFFSET_ENCODE_OVER); + + terrno = TSDB_CODE_SUCCESS; + +OFFSET_ENCODE_OVER: + tfree(buf); + if (terrno != TSDB_CODE_SUCCESS) { + mError("offset:%s, failed to encode to raw:%p since %s", pOffset->key, pRaw, terrstr()); + sdbFreeRaw(pRaw); + return NULL; + } + + mTrace("offset:%s, encode to raw:%p, row:%p", pOffset->key, pRaw, pOffset); + return pRaw; +} + +SSdbRow *mndOffsetActionDecode(SSdbRaw *pRaw) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + void *buf = NULL; + + int8_t sver = 0; + if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto OFFSET_DECODE_OVER; + + if (sver != MND_OFFSET_VER_NUMBER) { + terrno = TSDB_CODE_SDB_INVALID_DATA_VER; + goto OFFSET_DECODE_OVER; + } + + int32_t size = sizeof(SMqOffsetObj); + SSdbRow *pRow = sdbAllocRow(size); + if (pRow == NULL) goto OFFSET_DECODE_OVER; + + SMqOffsetObj *pOffset = sdbGetRowObj(pRow); + if (pOffset == NULL) goto OFFSET_DECODE_OVER; + + int32_t dataPos = 0; + int32_t tlen; + SDB_GET_INT32(pRaw, dataPos, &tlen, OFFSET_DECODE_OVER); + buf = malloc(tlen + 1); + if (buf == NULL) goto OFFSET_DECODE_OVER; + SDB_GET_BINARY(pRaw, dataPos, buf, tlen, OFFSET_DECODE_OVER); + SDB_GET_RESERVE(pRaw, dataPos, MND_OFFSET_RESERVE_SIZE, OFFSET_DECODE_OVER); + + if (tDecodeSMqOffsetObj(buf, pOffset) == NULL) { + goto OFFSET_DECODE_OVER; + } + + terrno = TSDB_CODE_SUCCESS; + +OFFSET_DECODE_OVER: + tfree(buf); + if (terrno != TSDB_CODE_SUCCESS) { + mError("offset:%s, failed to decode from raw:%p since %s", pOffset->key, pRaw, terrstr()); + tfree(pRow); + return NULL; + } + + mTrace("offset:%s, decode from raw:%p, row:%p", pOffset->key, pRaw, pOffset); + return pRow; +} + +int32_t mndCreateOffset(STrans *pTrans, const char *cgroup, const char *topicName, const SArray *vgs) { + int32_t code = 0; + int32_t sz = taosArrayGetSize(vgs); + for (int32_t i = 0; i < sz; i++) { + SMqConsumerEp *pConsumerEp = taosArrayGet(vgs, i); + SMqOffsetObj offsetObj; + if (mndMakePartitionKey(offsetObj.key, cgroup, topicName, pConsumerEp->vgId) < 0) { + return -1; + } + offsetObj.offset = -1; + SSdbRaw *pOffsetRaw = mndOffsetActionEncode(&offsetObj); + if (pOffsetRaw == NULL) { + return -1; + } + sdbSetRawStatus(pOffsetRaw, SDB_STATUS_READY); + if (mndTransAppendRedolog(pTrans, pOffsetRaw) < 0) { + return -1; + } + } + return 0; +} + +static int32_t mndProcessCommitOffsetReq(SMnodeMsg *pMsg) { + char key[TSDB_PARTITION_KEY_LEN]; + + SMnode *pMnode = pMsg->pMnode; + char *msgStr = pMsg->rpcMsg.pCont; + SMqCMCommitOffsetReq commitOffsetReq; + SCoder decoder; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, msgStr, pMsg->rpcMsg.contLen, TD_DECODER); + + tDecodeSMqCMCommitOffsetReq(&decoder, &commitOffsetReq); + + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_COMMIT_OFFSET, &pMsg->rpcMsg); + + for (int32_t i = 0; i < commitOffsetReq.num; i++) { + SMqOffset *pOffset = &commitOffsetReq.offsets[i]; + if (mndMakePartitionKey(key, pOffset->cgroup, pOffset->topicName, pOffset->vgId) < 0) { + return -1; + } + SMqOffsetObj *pOffsetObj = mndAcquireOffset(pMnode, key); + ASSERT(pOffsetObj); + pOffsetObj->offset = pOffset->offset; + SSdbRaw *pOffsetRaw = mndOffsetActionEncode(pOffsetObj); + sdbSetRawStatus(pOffsetRaw, SDB_STATUS_READY); + mndTransAppendRedolog(pTrans, pOffsetRaw); + mndReleaseOffset(pMnode, pOffsetObj); + } + + if (mndTransPrepare(pMnode, pTrans) != 0) { + mError("mq-commit-offset-trans:%d, failed to prepare since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + + mndTransDrop(pTrans); + return TSDB_CODE_MND_ACTION_IN_PROGRESS; +} + +static int32_t mndOffsetActionInsert(SSdb *pSdb, SMqOffsetObj *pOffset) { + mTrace("offset:%s, perform insert action", pOffset->key); + return 0; +} + +static int32_t mndOffsetActionDelete(SSdb *pSdb, SMqOffsetObj *pOffset) { + mTrace("offset:%s, perform delete action", pOffset->key); + return 0; +} + +static int32_t mndOffsetActionUpdate(SSdb *pSdb, SMqOffsetObj *pOldOffset, SMqOffsetObj *pNewOffset) { + mTrace("offset:%s, perform update action", pOldOffset->key); + return 0; +} + +SMqOffsetObj *mndAcquireOffset(SMnode *pMnode, const char *key) { + SSdb *pSdb = pMnode->pSdb; + SMqOffsetObj *pOffset = sdbAcquire(pSdb, SDB_OFFSET, key); + if (pOffset == NULL && terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) { + terrno = TSDB_CODE_MND_OFFSET_NOT_EXIST; + } + return pOffset; +} + +void mndReleaseOffset(SMnode *pMnode, SMqOffsetObj *pOffset) { + SSdb *pSdb = pMnode->pSdb; + sdbRelease(pSdb, pOffset); +} + +static void mndCancelGetNextOffset(SMnode *pMnode, void *pIter) { + SSdb *pSdb = pMnode->pSdb; + sdbCancelFetch(pSdb, pIter); +} diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index df892e2242..2dfdc8dc7f 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -21,6 +21,7 @@ #include "mndStb.h" #include "mndUser.h" #include "tglobal.h" +#include "version.h" #define QUERY_ID_SIZE 20 #define QUERY_OBJ_ID_SIZE 18 diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index a5c4c41244..2ea157fea4 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -19,6 +19,7 @@ #include "mndDb.h" #include "mndDnode.h" #include "mndMnode.h" +#include "mndOffset.h" #include "mndShow.h" #include "mndStb.h" #include "mndTopic.h" @@ -80,13 +81,13 @@ int32_t mndInitSubscribe(SMnode *pMnode) { return sdbSetTable(pMnode->pSdb, table); } -static SMqSubscribeObj *mndCreateSubscription(SMnode *pMnode, const SMqTopicObj *pTopic, const char *consumerGroup) { +static SMqSubscribeObj *mndCreateSubscription(SMnode *pMnode, const SMqTopicObj *pTopic, const char *cgroup) { SMqSubscribeObj *pSub = tNewSubscribeObj(); if (pSub == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } - char *key = mndMakeSubscribeKey(consumerGroup, pTopic->name); + char *key = mndMakeSubscribeKey(cgroup, pTopic->name); if (key == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; tDeleteSMqSubscribeObj(pSub); @@ -289,9 +290,15 @@ static int32_t mndProcessGetSubEpReq(SMnodeMsg *pMsg) { strcpy(topicEp.topic, topicName); topicEp.vgs = taosArrayInit(vgsz, sizeof(SMqSubVgEp)); for (int32_t k = 0; k < vgsz; k++) { + char offsetKey[TSDB_PARTITION_KEY_LEN]; SMqConsumerEp *pConsumerEp = taosArrayGet(pSubConsumer->vgInfo, k); - - SMqSubVgEp vgEp = {.epSet = pConsumerEp->epSet, .vgId = pConsumerEp->vgId}; + SMqSubVgEp vgEp = {.epSet = pConsumerEp->epSet, .vgId = pConsumerEp->vgId, .offset = -1}; + mndMakePartitionKey(offsetKey, pConsumer->cgroup, topicName, pConsumerEp->vgId); + SMqOffsetObj *pOffsetObj = mndAcquireOffset(pMnode, offsetKey); + if (pOffsetObj != NULL) { + vgEp.offset = pOffsetObj->offset; + mndReleaseOffset(pMnode, pOffsetObj); + } taosArrayPush(topicEp.vgs, &vgEp); } taosArrayPush(rsp.topics, &topicEp); @@ -870,7 +877,7 @@ static SSdbRaw *mndSubActionEncode(SMqSubscribeObj *pSub) { SUB_ENCODE_OVER: tfree(buf); - if (terrno != 0) { + if (terrno != TSDB_CODE_SUCCESS) { mError("subscribe:%s, failed to encode to raw:%p since %s", pSub->key, pRaw, terrstr()); sdbFreeRaw(pRaw); return NULL; @@ -1085,6 +1092,8 @@ static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg) { mDebug("create new subscription by consumer %ld, group: %s, topic %s", consumerId, cgroup, newTopicName); pSub = mndCreateSubscription(pMnode, pTopic, cgroup); createSub = true; + + mndCreateOffset(pTrans, cgroup, newTopicName, pSub->unassignedVg); } SMqSubConsumer mqSubConsumer; diff --git a/source/dnode/mnode/impl/src/mndTelem.c b/source/dnode/mnode/impl/src/mndTelem.c index 5beb1b10e3..5292dd0a41 100644 --- a/source/dnode/mnode/impl/src/mndTelem.c +++ b/source/dnode/mnode/impl/src/mndTelem.c @@ -21,8 +21,8 @@ #include "tversion.h" #define TELEMETRY_SERVER "telemetry.taosdata.com" -#define TELEMETRY_PORT 80 -#define REPORT_INTERVAL 86400 +#define TELEMETRY_PORT 80 +#define REPORT_INTERVAL 86400 static void mndBeginObject(SBufferWriter* bw) { tbufWriteChar(bw, '{'); } @@ -33,25 +33,8 @@ static void mndCloseObject(SBufferWriter* bw) { } else { tbufWriteChar(bw, '}'); } - tbufWriteChar(bw, ','); } -#if 0 -static void beginArray(SBufferWriter* bw) { - tbufWriteChar(bw, '['); -} - -static void closeArray(SBufferWriter* bw) { - size_t len = tbufTell(bw); - if (tbufGetData(bw, false)[len - 1] == ',') { - tbufWriteCharAt(bw, len - 1, ']'); - } else { - tbufWriteChar(bw, ']'); - } - tbufWriteChar(bw, ','); -} -#endif - static void mndWriteString(SBufferWriter* bw, const char* str) { tbufWriteChar(bw, '"'); tbufWrite(bw, str, strlen(str)); @@ -61,7 +44,7 @@ static void mndWriteString(SBufferWriter* bw, const char* str) { static void mndAddIntField(SBufferWriter* bw, const char* k, int64_t v) { mndWriteString(bw, k); tbufWriteChar(bw, ':'); - char buf[32]; + char buf[32] = {0}; sprintf(buf, "%" PRId64, v); tbufWrite(bw, buf, strlen(buf)); tbufWriteChar(bw, ','); @@ -184,24 +167,17 @@ static void mndAddRuntimeInfo(SMnode* pMnode, SBufferWriter* bw) { } static void mndSendTelemetryReport(SMnode* pMnode) { - STelemMgmt* pMgmt = &pMnode->telemMgmt; - - char buf[128] = {0}; - uint32_t ip = taosGetIpv4FromFqdn(TELEMETRY_SERVER); - if (ip == 0xffffffff) { - mDebug("failed to get IP address of " TELEMETRY_SERVER " since :%s", strerror(errno)); - return; - } - SOCKET fd = taosOpenTcpClientSocket(ip, TELEMETRY_PORT, 0); - if (fd < 0) { - mDebug("failed to create socket for telemetry, reason:%s", strerror(errno)); - return; - } + STelemMgmt* pMgmt = &pMnode->telemMgmt; + SBufferWriter bw = tbufInitWriter(NULL, false); + int32_t code = -1; + char buf[128] = {0}; + SOCKET fd = 0; char clusterName[64] = {0}; - mndGetClusterName(pMnode, clusterName, sizeof(clusterName)); + if (mndGetClusterName(pMnode, clusterName, sizeof(clusterName)) != 0) { + goto SEND_OVER; + } - SBufferWriter bw = tbufInitWriter(NULL, false); mndBeginObject(&bw); mndAddStringField(&bw, "instanceId", clusterName); mndAddIntField(&bw, "reportVersion", 1); @@ -212,54 +188,69 @@ static void mndSendTelemetryReport(SMnode* pMnode) { mndAddRuntimeInfo(pMnode, &bw); mndCloseObject(&bw); + uint32_t ip = taosGetIpv4FromFqdn(TELEMETRY_SERVER); + if (ip == 0xffffffff) { + terrno = TAOS_SYSTEM_ERROR(errno); + mError("failed to get ip of %s since :%s", TELEMETRY_SERVER, terrstr()); + goto SEND_OVER; + } + + fd = taosOpenTcpClientSocket(ip, TELEMETRY_PORT, 0); + if (fd < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + mError("failed to create socket to %s:%d since:%s", TELEMETRY_SERVER, TELEMETRY_PORT, terrstr()); + goto SEND_OVER; + } + const char* header = "POST /report HTTP/1.1\n" "Host: " TELEMETRY_SERVER "\n" "Content-Type: application/json\n" "Content-Length: "; + if (taosWriteSocket(fd, (void*)header, (int32_t)strlen(header)) < 0) { + goto SEND_OVER; + } - taosWriteSocket(fd, (void*)header, (int32_t)strlen(header)); - int32_t contLen = (int32_t)(tbufTell(&bw) - 1); + int32_t contLen = (int32_t)(tbufTell(&bw)); sprintf(buf, "%d\n\n", contLen); - taosWriteSocket(fd, buf, (int32_t)strlen(buf)); - taosWriteSocket(fd, tbufGetData(&bw, false), contLen); - tbufCloseWriter(&bw); + if (taosWriteSocket(fd, buf, (int32_t)strlen(buf)) < 0) { + goto SEND_OVER; + } + + const char* pCont = tbufGetData(&bw, false); + if (taosWriteSocket(fd, (void*)pCont, contLen) < 0) { + goto SEND_OVER; + } // read something to avoid nginx error 499 if (taosReadSocket(fd, buf, 10) < 0) { - mDebug("failed to receive response since %s", strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + mError("failed to receive response since %s", terrstr()); + goto SEND_OVER; } + mInfo("send telemetry to %s:%d, len:%d content: %s", TELEMETRY_SERVER, TELEMETRY_PORT, contLen, pCont); + code = 0; + +SEND_OVER: + tbufCloseWriter(&bw); taosCloseSocket(fd); + + if (code != 0) { + mError("failed to send telemetry to %s:%d since %s", TELEMETRY_SERVER, TELEMETRY_PORT, terrstr()); + } } -static void* mndTelemThreadFp(void* param) { - SMnode* pMnode = param; +static int32_t mndProcessTelemTimer(SMnodeMsg* pReq) { + SMnode* pMnode = pReq->pMnode; STelemMgmt* pMgmt = &pMnode->telemMgmt; + if (!pMgmt->enable) return 0; - struct timespec end = {0}; - clock_gettime(CLOCK_REALTIME, &end); - end.tv_sec += 300; // wait 5 minutes before send first report - - setThreadName("mnd-telem"); - - while (!pMgmt->exit) { - int32_t r = 0; - struct timespec ts = end; - pthread_mutex_lock(&pMgmt->lock); - r = pthread_cond_timedwait(&pMgmt->cond, &pMgmt->lock, &ts); - pthread_mutex_unlock(&pMgmt->lock); - if (r == 0) break; - if (r != ETIMEDOUT) continue; - - if (mndIsMaster(pMnode)) { - mndSendTelemetryReport(pMnode); - } - end.tv_sec += REPORT_INTERVAL; - } - - return NULL; + taosWLockLatch(&pMgmt->lock); + mndSendTelemetryReport(pMnode); + taosWUnLockLatch(&pMgmt->lock); + return 0; } static void mndGetEmail(SMnode* pMnode, char* filepath) { @@ -280,43 +271,12 @@ static void mndGetEmail(SMnode* pMnode, char* filepath) { int32_t mndInitTelem(SMnode* pMnode) { STelemMgmt* pMgmt = &pMnode->telemMgmt; pMgmt->enable = pMnode->cfg.enableTelem; - - if (!pMgmt->enable) return 0; - - pMgmt->exit = 0; - pthread_mutex_init(&pMgmt->lock, NULL); - pthread_cond_init(&pMgmt->cond, NULL); - pMgmt->email[0] = 0; - + taosInitRWLatch(&pMgmt->lock); mndGetEmail(pMnode, "/usr/local/taos/email"); - pthread_attr_t attr; - pthread_attr_init(&attr); - pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE); - - int32_t code = pthread_create(&pMgmt->thread, &attr, mndTelemThreadFp, pMnode); - pthread_attr_destroy(&attr); - if (code != 0) { - mDebug("failed to create telemetry thread since :%s", strerror(code)); - } - - mInfo("mnd telemetry is initialized"); + mndSetMsgHandle(pMnode, TDMT_MND_TELEM_TIMER, mndProcessTelemTimer); + mDebug("mnode telemetry is initialized"); return 0; } -void mndCleanupTelem(SMnode* pMnode) { - STelemMgmt* pMgmt = &pMnode->telemMgmt; - if (!pMgmt->enable) return; - - if (taosCheckPthreadValid(pMgmt->thread)) { - pthread_mutex_lock(&pMgmt->lock); - pMgmt->exit = 1; - pthread_cond_signal(&pMgmt->cond); - pthread_mutex_unlock(&pMgmt->lock); - - pthread_join(pMgmt->thread, NULL); - } - - pthread_mutex_destroy(&pMgmt->lock); - pthread_cond_destroy(&pMgmt->cond); -} +void mndCleanupTelem(SMnode* pMnode) {} diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 7d684cd542..a009e01a52 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -71,7 +71,7 @@ int32_t mndInitTrans(SMnode *pMnode) { .updateFp = (SdbUpdateFp)mndTransActionUpdate, .deleteFp = (SdbDeleteFp)mndTransActionDelete}; - mndSetMsgHandle(pMnode, TDMT_MND_TRANS, mndProcessTransReq); + mndSetMsgHandle(pMnode, TDMT_MND_TRANS_TIMER, mndProcessTransReq); mndSetMsgHandle(pMnode, TDMT_MND_KILL_TRANS, mndProcessKillTransReq); mndAddShowMetaHandle(pMnode, TSDB_MGMT_TABLE_TRANS, mndGetTransMeta); diff --git a/source/dnode/mnode/impl/src/mnode.c b/source/dnode/mnode/impl/src/mnode.c index 64b4aa6dd7..299e66a7c0 100644 --- a/source/dnode/mnode/impl/src/mnode.c +++ b/source/dnode/mnode/impl/src/mnode.c @@ -23,6 +23,7 @@ #include "mndDnode.h" #include "mndFunc.h" #include "mndMnode.h" +#include "mndOffset.h" #include "mndProfile.h" #include "mndQnode.h" #include "mndShow.h" @@ -36,6 +37,10 @@ #include "mndUser.h" #include "mndVgroup.h" +#define MQ_TIMER_MS 3000 +#define TRNAS_TIMER_MS 6000 +#define TELEM_TIMER_MS 86400000 + int32_t mndSendReqToDnode(SMnode *pMnode, SEpSet *pEpSet, SRpcMsg *pMsg) { if (pMnode == NULL || pMnode->sendReqToDnodeFp == NULL) { terrno = TSDB_CODE_MND_NOT_READY; @@ -73,46 +78,60 @@ static void *mndBuildTimerMsg(int32_t *pContLen) { return pReq; } -static void mndTransReExecute(void *param, void *tmrId) { +static void mndPullupTrans(void *param, void *tmrId) { SMnode *pMnode = param; if (mndIsMaster(pMnode)) { int32_t contLen = 0; - void * pReq = mndBuildTimerMsg(&contLen); - SRpcMsg rpcMsg = {.msgType = TDMT_MND_TRANS, .pCont = pReq, .contLen = contLen}; + void *pReq = mndBuildTimerMsg(&contLen); + SRpcMsg rpcMsg = {.msgType = TDMT_MND_TRANS_TIMER, .pCont = pReq, .contLen = contLen}; pMnode->putReqToMWriteQFp(pMnode->pDnode, &rpcMsg); } - taosTmrReset(mndTransReExecute, 3000, pMnode, pMnode->timer, &pMnode->transTimer); + taosTmrReset(mndPullupTrans, TRNAS_TIMER_MS, pMnode, pMnode->timer, &pMnode->transTimer); } static void mndCalMqRebalance(void *param, void *tmrId) { SMnode *pMnode = param; if (mndIsMaster(pMnode)) { int32_t contLen = 0; - void * pReq = mndBuildTimerMsg(&contLen); + void *pReq = mndBuildTimerMsg(&contLen); SRpcMsg rpcMsg = {.msgType = TDMT_MND_MQ_TIMER, .pCont = pReq, .contLen = contLen}; pMnode->putReqToMReadQFp(pMnode->pDnode, &rpcMsg); } - taosTmrReset(mndCalMqRebalance, 3000, pMnode, pMnode->timer, &pMnode->mqTimer); + taosTmrReset(mndCalMqRebalance, MQ_TIMER_MS, pMnode, pMnode->timer, &pMnode->mqTimer); +} + +static void mndPullupTelem(void *param, void *tmrId) { + SMnode *pMnode = param; + if (mndIsMaster(pMnode)) { + int32_t contLen = 0; + void *pReq = mndBuildTimerMsg(&contLen); + SRpcMsg rpcMsg = {.msgType = TDMT_MND_TELEM_TIMER, .pCont = pReq, .contLen = contLen}; + pMnode->putReqToMReadQFp(pMnode->pDnode, &rpcMsg); + } + + taosTmrReset(mndPullupTelem, TELEM_TIMER_MS, pMnode, pMnode->timer, &pMnode->telemTimer); } static int32_t mndInitTimer(SMnode *pMnode) { - if (pMnode->timer == NULL) { - pMnode->timer = taosTmrInit(5000, 200, 3600000, "MND"); - } - + pMnode->timer = taosTmrInit(5000, 200, 3600000, "MND"); if (pMnode->timer == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - if (taosTmrReset(mndTransReExecute, 6000, pMnode, pMnode->timer, &pMnode->transTimer)) { + if (taosTmrReset(mndPullupTrans, TRNAS_TIMER_MS, pMnode, pMnode->timer, &pMnode->transTimer)) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - if (taosTmrReset(mndCalMqRebalance, 3000, pMnode, pMnode->timer, &pMnode->mqTimer)) { + if (taosTmrReset(mndCalMqRebalance, MQ_TIMER_MS, pMnode, pMnode->timer, &pMnode->mqTimer)) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + if (taosTmrReset(mndPullupTelem, 60000, pMnode, pMnode->timer, &pMnode->telemTimer)) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } @@ -126,6 +145,8 @@ static void mndCleanupTimer(SMnode *pMnode) { pMnode->transTimer = NULL; taosTmrStop(pMnode->mqTimer); pMnode->mqTimer = NULL; + taosTmrStop(pMnode->telemTimer); + pMnode->telemTimer = NULL; taosTmrCleanUp(pMnode->timer); pMnode->timer = NULL; } @@ -197,6 +218,7 @@ static int32_t mndInitSteps(SMnode *pMnode) { if (mndAllocStep(pMnode, "mnode-topic", mndInitTopic, mndCleanupTopic) != 0) return -1; if (mndAllocStep(pMnode, "mnode-consumer", mndInitConsumer, mndCleanupConsumer) != 0) return -1; if (mndAllocStep(pMnode, "mnode-subscribe", mndInitSubscribe, mndCleanupSubscribe) != 0) return -1; + if (mndAllocStep(pMnode, "mnode-offset", mndInitOffset, mndCleanupOffset) != 0) return -1; if (mndAllocStep(pMnode, "mnode-vgroup", mndInitVgroup, mndCleanupVgroup) != 0) return -1; if (mndAllocStep(pMnode, "mnode-stb", mndInitStb, mndCleanupStb) != 0) return -1; if (mndAllocStep(pMnode, "mnode-db", mndInitDb, mndCleanupDb) != 0) return -1; @@ -404,8 +426,8 @@ SMnodeMsg *mndInitMsg(SMnode *pMnode, SRpcMsg *pRpcMsg) { return NULL; } - if (pRpcMsg->msgType != TDMT_MND_TRANS && pRpcMsg->msgType != TDMT_MND_MQ_TIMER && - pRpcMsg->msgType != TDMT_MND_MQ_DO_REBALANCE) { + if (pRpcMsg->msgType != TDMT_MND_TRANS_TIMER && pRpcMsg->msgType != TDMT_MND_MQ_TIMER && + pRpcMsg->msgType != TDMT_MND_MQ_DO_REBALANCE && pRpcMsg->msgType != TDMT_MND_TELEM_TIMER) { SRpcConnInfo connInfo = {0}; if ((pRpcMsg->msgType & 1U) && rpcGetConnInfo(pRpcMsg->handle, &connInfo) != 0) { taosFreeQitem(pMsg); @@ -440,7 +462,7 @@ void mndProcessMsg(SMnodeMsg *pMsg) { SMnode *pMnode = pMsg->pMnode; int32_t code = 0; tmsg_t msgType = pMsg->rpcMsg.msgType; - void * ahandle = pMsg->rpcMsg.ahandle; + void *ahandle = pMsg->rpcMsg.ahandle; bool isReq = (msgType & 1U); mTrace("msg:%p, type:%s will be processed, app:%p", pMsg, TMSG_INFO(msgType), ahandle); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 4dc46b3798..ac9dde3597 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -207,9 +207,17 @@ int32_t tqDeserializeConsumer(STQ* pTq, const STqSerializedHead* pHead, STqConsu int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) { SMqConsumeReq* pReq = pMsg->pCont; int64_t consumerId = pReq->consumerId; - int64_t fetchOffset = pReq->offset; + int64_t fetchOffset; /*int64_t blockingTime = pReq->blockingTime;*/ + if (pReq->currentOffset == TMQ_CONF__RESET_OFFSET__EARLIEAST) { + fetchOffset = 0; + } else if (pReq->currentOffset == TMQ_CONF__RESET_OFFSET__LATEST) { + fetchOffset = walGetLastVer(pTq->pWal); + } else { + fetchOffset = pReq->currentOffset + 1; + } + SMqConsumeRsp rsp = {.consumerId = consumerId, .numOfTopics = 0, .pBlockData = NULL}; STqConsumer* pConsumer = tqHandleGet(pTq->tqMeta, consumerId); @@ -226,31 +234,9 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) { ASSERT(strcmp(pTopic->topicName, pReq->topic) == 0); ASSERT(pConsumer->consumerId == consumerId); - if (pReq->reqType == TMQ_REQ_TYPE_COMMIT_ONLY) { - pTopic->committedOffset = pReq->offset; - /*printf("offset %ld committed\n", pTopic->committedOffset);*/ - pMsg->pCont = NULL; - pMsg->contLen = 0; - pMsg->code = 0; - rpcSendResponse(pMsg); - return 0; - } - - if (pReq->reqType == TMQ_REQ_TYPE_CONSUME_AND_COMMIT) { - if (pTopic->committedOffset < pReq->offset - 1) { - pTopic->committedOffset = pReq->offset - 1; - /*printf("offset %ld committed\n", pTopic->committedOffset);*/ - } - } - - rsp.committedOffset = pTopic->committedOffset; - rsp.reqOffset = pReq->offset; + rsp.reqOffset = pReq->currentOffset; rsp.skipLogNum = 0; - if (fetchOffset <= pTopic->committedOffset) { - fetchOffset = pTopic->committedOffset + 1; - } - SWalHead* pHead; while (1) { int8_t pos = fetchOffset % TQ_BUFFER_SIZE; diff --git a/source/dnode/vnode/src/vnd/vnodeWrite.c b/source/dnode/vnode/src/vnd/vnodeWrite.c index ade280eecc..c3947da459 100644 --- a/source/dnode/vnode/src/vnd/vnodeWrite.c +++ b/source/dnode/vnode/src/vnd/vnodeWrite.c @@ -22,14 +22,15 @@ int vnodeProcessWMsgs(SVnode *pVnode, SArray *pMsgs) { for (int i = 0; i < taosArrayGetSize(pMsgs); i++) { pMsg = *(SRpcMsg **)taosArrayGet(pMsgs, i); - // ser request version + // set request version void *pBuf = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); int64_t ver = pVnode->state.processed++; taosEncodeFixedI64(&pBuf, ver); if (walWrite(pVnode->pWal, ver, pMsg->msgType, pMsg->pCont, pMsg->contLen) < 0) { - /*ASSERT(false);*/ // TODO: handle error + /*ASSERT(false);*/ + vError("vnode:%d write wal error since %s", pVnode->vgId, terrstr()); } } diff --git a/source/libs/CMakeLists.txt b/source/libs/CMakeLists.txt index a2454e4c5e..da2ab5b425 100644 --- a/source/libs/CMakeLists.txt +++ b/source/libs/CMakeLists.txt @@ -14,4 +14,5 @@ add_subdirectory(qcom) add_subdirectory(qworker) add_subdirectory(tfs) add_subdirectory(nodes) -add_subdirectory(scalar) \ No newline at end of file +add_subdirectory(config) +add_subdirectory(scalar) diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index 1c84f06a22..ebe20fbb7f 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -131,9 +131,7 @@ void ctgTestInitLogFile() { ctgDbgEnableDebug("api"); - char temp[128] = {0}; - sprintf(temp, "%s/%s", tsLogDir, defaultLogFileNamePrefix); - if (taosInitLog(temp, tsNumOfLogLines, maxLogFileNum) < 0) { + if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) { printf("failed to open log file in directory:%s\n", tsLogDir); } } diff --git a/source/libs/config/CMakeLists.txt b/source/libs/config/CMakeLists.txt new file mode 100644 index 0000000000..596cc0b433 --- /dev/null +++ b/source/libs/config/CMakeLists.txt @@ -0,0 +1,13 @@ +aux_source_directory(src CONFIG_SRC) +add_library(config STATIC ${CONFIG_SRC}) +target_include_directories( + config + PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/config" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" +) + +target_link_libraries(config os util common) + +if(${BUILD_TEST}) + ADD_SUBDIRECTORY(test) +endif(${BUILD_TEST}) \ No newline at end of file diff --git a/source/libs/config/inc/cfgInt.h b/source/libs/config/inc/cfgInt.h new file mode 100644 index 0000000000..6497393792 --- /dev/null +++ b/source/libs/config/inc/cfgInt.h @@ -0,0 +1,47 @@ + +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_CFG_INT_H_ +#define _TD_CFG_INT_H_ + +#include "config.h" +#include "taoserror.h" +#include "thash.h" +#include "tutil.h" +#include "ulog.h" +#include "tglobal.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct SConfig { + ECfgSrcType stype; + SHashObj *hash; +} SConfig; + +int32_t cfgLoadFromCfgFile(SConfig *pConfig, const char *filepath); +int32_t cfgLoadFromEnvFile(SConfig *pConfig, const char *filepath); +int32_t cfgLoadFromEnvVar(SConfig *pConfig); +int32_t cfgLoadFromApollUrl(SConfig *pConfig, const char *url); + +int32_t cfgSetItem(SConfig *pConfig, const char *name, const char *value, ECfgSrcType stype); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_CFG_INT_H_*/ diff --git a/source/libs/config/src/cfgApolloUrl.c b/source/libs/config/src/cfgApolloUrl.c new file mode 100644 index 0000000000..f35eca70c3 --- /dev/null +++ b/source/libs/config/src/cfgApolloUrl.c @@ -0,0 +1,22 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "cfgInt.h" + +int32_t cfgLoadFromApollUrl(SConfig *pConfig, const char *url) { + uInfo("load from apoll url %s", url); + return 0; +} \ No newline at end of file diff --git a/source/libs/config/src/cfgCfgFile.c b/source/libs/config/src/cfgCfgFile.c new file mode 100644 index 0000000000..4eb835be7f --- /dev/null +++ b/source/libs/config/src/cfgCfgFile.c @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "cfgInt.h" + +int32_t cfgLoadFromCfgFile(SConfig *pConfig, const char *filepath) { + char *line, *name, *value, *value2, *value3; + int olen, vlen, vlen2, vlen3; + ssize_t _bytes = 0; + size_t len = 1024; + + FILE *fp = fopen(filepath, "r"); + if (fp == NULL) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + line = malloc(len); + + while (!feof(fp)) { + memset(line, 0, len); + + name = value = value2 = value3 = NULL; + olen = vlen = vlen2 = vlen3 = 0; + + _bytes = tgetline(&line, &len, fp); + if (_bytes < 0) { + break; + } + + line[len - 1] = 0; + + paGetToken(line, &name, &olen); + if (olen == 0) continue; + name[olen] = 0; + + paGetToken(name + olen + 1, &value, &vlen); + if (vlen == 0) continue; + value[vlen] = 0; + + paGetToken(value + vlen + 1, &value2, &vlen2); + if (vlen2 != 0) { + value2[vlen2] = 0; + paGetToken(value2 + vlen2 + 1, &value3, &vlen3); + if (vlen3 != 0) value3[vlen3] = 0; + } + + cfgSetItem(pConfig, name, value, CFG_STYPE_CFG_FILE); + // taosReadConfigOption(name, value, value2, value3); + } + + fclose(fp); + tfree(line); + + uInfo("load from cfg file %s success", filepath); + return 0; +} \ No newline at end of file diff --git a/source/libs/config/src/cfgEnvFile.c b/source/libs/config/src/cfgEnvFile.c new file mode 100644 index 0000000000..a12d1cd1d5 --- /dev/null +++ b/source/libs/config/src/cfgEnvFile.c @@ -0,0 +1,22 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "cfgInt.h" + +int32_t cfgLoadFromEnvFile(SConfig *pConfig, const char *filepath) { + uInfo("load from env file %s", filepath); + return 0; +} \ No newline at end of file diff --git a/source/libs/config/src/cfgEnvVar.c b/source/libs/config/src/cfgEnvVar.c new file mode 100644 index 0000000000..c3b3df4047 --- /dev/null +++ b/source/libs/config/src/cfgEnvVar.c @@ -0,0 +1,22 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "cfgInt.h" + +int32_t cfgLoadFromEnvVar(SConfig *pConfig) { + uInfo("load from global env variables"); + return 0; +} \ No newline at end of file diff --git a/source/libs/config/src/config.c b/source/libs/config/src/config.c new file mode 100644 index 0000000000..17a652d2b5 --- /dev/null +++ b/source/libs/config/src/config.c @@ -0,0 +1,635 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "cfgInt.h" +#include "tep.h" +#include "tlocale.h" +#include "tmsg.h" +#include "ttimezone.h" + +#define CFG_NAME_PRINT_LEN 22 +#define CFG_SRC_PRINT_LEN 12 + +SConfig *cfgInit() { + SConfig *pCfg = calloc(1, sizeof(SConfig)); + if (pCfg == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + pCfg->hash = taosHashInit(16, MurmurHash3_32, false, HASH_NO_LOCK); + if (pCfg->hash == NULL) { + free(pCfg); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + return pCfg; +} + +int32_t cfgLoad(SConfig *pCfg, ECfgSrcType cfgType, const char *sourceStr) { + switch (cfgType) { + case CFG_STYPE_CFG_FILE: + return cfgLoadFromCfgFile(pCfg, sourceStr); + case CFG_STYPE_ENV_FILE: + return cfgLoadFromEnvFile(pCfg, sourceStr); + case CFG_STYPE_ENV_VAR: + return cfgLoadFromEnvVar(pCfg); + case CFG_STYPE_APOLLO_URL: + return cfgLoadFromApollUrl(pCfg, sourceStr); + default: + return -1; + } +} + +void cfgCleanup(SConfig *pCfg) { + if (pCfg != NULL) { + if (pCfg->hash != NULL) { + taosHashCleanup(pCfg->hash); + pCfg->hash == NULL; + } + free(pCfg); + } +} + +int32_t cfgGetSize(SConfig *pCfg) { return taosHashGetSize(pCfg->hash); } + +SConfigItem *cfgIterate(SConfig *pCfg, SConfigItem *pIter) { return taosHashIterate(pCfg->hash, pIter); } + +void cfgCancelIterate(SConfig *pCfg, SConfigItem *pIter) { return taosHashCancelIterate(pCfg->hash, pIter); } + +static int32_t cfgCheckAndSetTimezone(SConfigItem *pItem, const char *timezone) { + tfree(pItem->str); + pItem->str = strdup(timezone); + if (pItem->str == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + return 0; +} + +static int32_t cfgCheckAndSetCharset(SConfigItem *pItem, const char *charset) { + tfree(pItem->str); + pItem->str = strdup(charset); + if (pItem->str == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + return 0; +} + +static int32_t cfgCheckAndSetLocale(SConfigItem *pItem, const char *locale) { + tfree(pItem->str); + pItem->str = strdup(locale); + if (pItem->str == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + return 0; +} + +static int32_t cfgCheckAndSetDir(SConfigItem *pItem, const char *inputDir) { + char fullDir[PATH_MAX] = {0}; + if (taosExpandDir(inputDir, fullDir, PATH_MAX) != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + uError("failed to expand dir:%s since %s", inputDir, terrstr()); + return -1; + } + + if (taosRealPath(fullDir, PATH_MAX) != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + uError("failed to get realpath of dir:%s since %s", inputDir, terrstr()); + return -1; + } + + if (taosMkDir(fullDir) != 0) { + uError("failed to create dir:%s realpath:%s since %s", inputDir, fullDir, terrstr()); + return -1; + } + + tfree(pItem->str); + pItem->str = strdup(fullDir); + if (pItem->str == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + return 0; +} + +static int32_t cfgCheckAndSetIpStr(SConfigItem *pItem, const char *ip) { + uint32_t value = taosInetAddr(ip); + if (value == INADDR_NONE) { + uError("ip:%s is not a valid ip address", ip); + return -1; + } + + tfree(pItem->str); + pItem->str = strdup(ip); + if (pItem->str == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + return 0; +} + +static int32_t cfgSetBool(SConfigItem *pItem, const char *value, ECfgSrcType stype) { + bool tmp = false; + if (strcasecmp(value, "true") == 0) { + tmp = true; + } + if (atoi(value) > 0) { + tmp = true; + } + + pItem->bval = tmp; + pItem->stype = stype; + return 0; +} + +static int32_t cfgSetInt32(SConfigItem *pItem, const char *value, ECfgSrcType stype) { + int32_t ival = (int32_t)atoi(value); + if (ival < pItem->imin || ival > pItem->imax) { + uError("cfg:%s, type:%s src:%s value:%d out of range[%" PRId64 ", %" PRId64 "], use last src:%s value:%d", + pItem->name, cfgDtypeStr(pItem->dtype), cfgStypeStr(stype), ival, pItem->imin, pItem->imax, + cfgStypeStr(pItem->stype), pItem->i32); + terrno = TSDB_CODE_OUT_OF_RANGE; + return -1; + } + + pItem->i32 = ival; + pItem->stype = stype; + return 0; +} + +static int32_t cfgSetInt64(SConfigItem *pItem, const char *value, ECfgSrcType stype) { + int64_t ival = (int64_t)atoi(value); + if (ival < pItem->imin || ival > pItem->imax) { + uError("cfg:%s, type:%s src:%s value:%" PRId64 " out of range[%" PRId64 ", %" PRId64 + "], use last src:%s value:%" PRId64, + pItem->name, cfgDtypeStr(pItem->dtype), cfgStypeStr(stype), ival, pItem->imin, pItem->imax, + cfgStypeStr(pItem->stype), pItem->i64); + terrno = TSDB_CODE_OUT_OF_RANGE; + return -1; + } + + pItem->i64 = ival; + pItem->stype = stype; + return 0; +} + +static int32_t cfgSetFloat(SConfigItem *pItem, const char *value, ECfgSrcType stype) { + float fval = (float)atof(value); + if (fval < pItem->fmin || fval > pItem->fmax) { + uError("cfg:%s, type:%s src:%s value:%f out of range[%f, %f], use last src:%s value:%f", pItem->name, + cfgDtypeStr(pItem->dtype), cfgStypeStr(stype), fval, pItem->fmin, pItem->fmax, cfgStypeStr(pItem->stype), + pItem->fval); + terrno = TSDB_CODE_OUT_OF_RANGE; + return -1; + } + + pItem->fval = fval; + pItem->stype = stype; + return 0; +} + +static int32_t cfgSetString(SConfigItem *pItem, const char *value, ECfgSrcType stype) { + char *tmp = strdup(value); + if (tmp == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + uError("cfg:%s, type:%s src:%s value:%s failed to dup since %s, use last src:%s value:%s", pItem->name, + cfgDtypeStr(pItem->dtype), cfgStypeStr(stype), value, terrstr(), cfgStypeStr(pItem->stype), pItem->str); + return -1; + } + + free(pItem->str); + pItem->str = tmp; + pItem->stype = stype; + return 0; +} + +static int32_t cfgSetIpStr(SConfigItem *pItem, const char *value, ECfgSrcType stype) { + char *tmp = strdup(value); + if (tmp == NULL || cfgCheckAndSetIpStr(pItem, value) != 0) { + free(tmp); + terrno = TSDB_CODE_OUT_OF_MEMORY; + uError("cfg:%s, type:%s src:%s value:%s failed to dup since %s, use last src:%s value:%s", pItem->name, + cfgDtypeStr(pItem->dtype), cfgStypeStr(stype), value, terrstr(), cfgStypeStr(pItem->stype), pItem->str); + return -1; + } + + pItem->stype = stype; + return 0; +} + +static int32_t cfgSetDir(SConfigItem *pItem, const char *value, ECfgSrcType stype) { + char *tmp = strdup(value); + if (tmp == NULL || cfgCheckAndSetDir(pItem, value) != 0) { + free(tmp); + terrno = TSDB_CODE_OUT_OF_MEMORY; + uError("cfg:%s, type:%s src:%s value:%s failed to dup since %s, use last src:%s value:%s", pItem->name, + cfgDtypeStr(pItem->dtype), cfgStypeStr(stype), value, terrstr(), cfgStypeStr(pItem->stype), pItem->str); + return -1; + } + + pItem->stype = stype; + return 0; +} + +static int32_t cfgSetLocale(SConfigItem *pItem, const char *value, ECfgSrcType stype) { + char *tmp = strdup(value); + if (tmp == NULL || cfgCheckAndSetLocale(pItem, value) != 0) { + free(tmp); + terrno = TSDB_CODE_OUT_OF_MEMORY; + uError("cfg:%s, type:%s src:%s value:%s failed to dup since %s, use last src:%s value:%s", pItem->name, + cfgDtypeStr(pItem->dtype), cfgStypeStr(stype), value, terrstr(), cfgStypeStr(pItem->stype), pItem->str); + return -1; + } + + pItem->stype = stype; + return 0; +} + +static int32_t cfgSetCharset(SConfigItem *pItem, const char *value, ECfgSrcType stype) { + char *tmp = strdup(value); + if (tmp == NULL || cfgCheckAndSetCharset(pItem, value) != 0) { + free(tmp); + terrno = TSDB_CODE_OUT_OF_MEMORY; + uError("cfg:%s, type:%s src:%s value:%s failed to dup since %s, use last src:%s value:%s", pItem->name, + cfgDtypeStr(pItem->dtype), cfgStypeStr(stype), value, terrstr(), cfgStypeStr(pItem->stype), pItem->str); + return -1; + } + + pItem->stype = stype; + return 0; +} + +static int32_t cfgSetTimezone(SConfigItem *pItem, const char *value, ECfgSrcType stype) { + char *tmp = strdup(value); + if (tmp == NULL || cfgCheckAndSetTimezone(pItem, value) != 0) { + free(tmp); + terrno = TSDB_CODE_OUT_OF_MEMORY; + uError("cfg:%s, type:%s src:%s value:%s failed to dup since %s, use last src:%s value:%s", pItem->name, + cfgDtypeStr(pItem->dtype), cfgStypeStr(stype), value, terrstr(), cfgStypeStr(pItem->stype), pItem->str); + return -1; + } + + pItem->stype = stype; + return 0; +} + +int32_t cfgSetItem(SConfig *pCfg, const char *name, const char *value, ECfgSrcType stype) { + SConfigItem *pItem = cfgGetItem(pCfg, name); + if (pItem == NULL) { + return -1; + } + + switch (pItem->dtype) { + case CFG_DTYPE_BOOL: + return cfgSetBool(pItem, value, stype); + case CFG_DTYPE_INT32: + return cfgSetInt32(pItem, value, stype); + case CFG_DTYPE_INT64: + return cfgSetInt64(pItem, value, stype); + case CFG_DTYPE_FLOAT: + return cfgSetFloat(pItem, value, stype); + case CFG_DTYPE_STRING: + return cfgSetString(pItem, value, stype); + case CFG_DTYPE_IPSTR: + return cfgSetIpStr(pItem, value, stype); + case CFG_DTYPE_DIR: + return cfgSetDir(pItem, value, stype); + case CFG_DTYPE_TIMEZONE: + return cfgSetTimezone(pItem, value, stype); + case CFG_DTYPE_CHARSET: + return cfgSetCharset(pItem, value, stype); + case CFG_DTYPE_LOCALE: + return cfgSetLocale(pItem, value, stype); + case CFG_DTYPE_NONE: + default: + break; + } + + terrno = TSDB_CODE_INVALID_CFG; + return -1; +} + +SConfigItem *cfgGetItem(SConfig *pCfg, const char *name) { + char lowcaseName[CFG_NAME_MAX_LEN + 1] = {0}; + memcpy(lowcaseName, name, CFG_NAME_MAX_LEN); + strntolower(lowcaseName, name, CFG_NAME_MAX_LEN); + + SConfigItem *pItem = taosHashGet(pCfg->hash, lowcaseName, strlen(lowcaseName) + 1); + if (pItem == NULL) { + terrno = TSDB_CODE_CFG_NOT_FOUND; + } + + return pItem; +} + +static int32_t cfgAddItem(SConfig *pCfg, SConfigItem *pItem, const char *name) { + pItem->stype = CFG_STYPE_DEFAULT; + pItem->name = strdup(name); + if (pItem->name == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + char lowcaseName[CFG_NAME_MAX_LEN + 1] = {0}; + memcpy(lowcaseName, name, CFG_NAME_MAX_LEN); + strntolower(lowcaseName, name, CFG_NAME_MAX_LEN); + + if (taosHashPut(pCfg->hash, lowcaseName, strlen(lowcaseName) + 1, pItem, sizeof(SConfigItem)) != 0) { + if (pItem->dtype == CFG_DTYPE_STRING) { + free(pItem->str); + } + free(pItem->name); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + return 0; +} + +int32_t cfgAddBool(SConfig *pCfg, const char *name, bool defaultVal) { + SConfigItem item = {.dtype = CFG_DTYPE_BOOL, .bval = defaultVal}; + return cfgAddItem(pCfg, &item, name); +} + +int32_t cfgAddInt32(SConfig *pCfg, const char *name, int32_t defaultVal, int64_t minval, int64_t maxval) { + if (defaultVal < minval || defaultVal > maxval) { + terrno = TSDB_CODE_OUT_OF_RANGE; + return -1; + } + + SConfigItem item = {.dtype = CFG_DTYPE_INT32, .i32 = defaultVal, .imin = minval, .imax = maxval}; + return cfgAddItem(pCfg, &item, name); +} + +int32_t cfgAddInt64(SConfig *pCfg, const char *name, int64_t defaultVal, int64_t minval, int64_t maxval) { + if (defaultVal < minval || defaultVal > maxval) { + terrno = TSDB_CODE_OUT_OF_RANGE; + return -1; + } + + SConfigItem item = {.dtype = CFG_DTYPE_INT64, .i64 = defaultVal, .imin = minval, .imax = maxval}; + return cfgAddItem(pCfg, &item, name); +} + +int32_t cfgAddFloat(SConfig *pCfg, const char *name, float defaultVal, double minval, double maxval) { + if (defaultVal < minval || defaultVal > maxval) { + terrno = TSDB_CODE_OUT_OF_RANGE; + return -1; + } + + SConfigItem item = {.dtype = CFG_DTYPE_FLOAT, .fval = defaultVal, .fmin = minval, .fmax = maxval}; + return cfgAddItem(pCfg, &item, name); +} + +int32_t cfgAddString(SConfig *pCfg, const char *name, const char *defaultVal) { + SConfigItem item = {.dtype = CFG_DTYPE_STRING}; + item.str = strdup(defaultVal); + if (item.str == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + return cfgAddItem(pCfg, &item, name); +} + +int32_t cfgAddIpStr(SConfig *pCfg, const char *name, const char *defaultVal) { + SConfigItem item = {.dtype = CFG_DTYPE_IPSTR}; + if (cfgCheckAndSetIpStr(&item, defaultVal) != 0) { + return -1; + } + + return cfgAddItem(pCfg, &item, name); +} + +int32_t cfgAddDir(SConfig *pCfg, const char *name, const char *defaultVal) { + SConfigItem item = {.dtype = CFG_DTYPE_DIR}; + if (cfgCheckAndSetDir(&item, defaultVal) != 0) { + return -1; + } + + return cfgAddItem(pCfg, &item, name); +} + +int32_t cfgAddLocale(SConfig *pCfg, const char *name, const char *defaultVal) { + SConfigItem item = {.dtype = CFG_DTYPE_LOCALE}; + if (cfgCheckAndSetLocale(&item, defaultVal) != 0) { + return -1; + } + + return cfgAddItem(pCfg, &item, name); +} + +int32_t cfgAddCharset(SConfig *pCfg, const char *name, const char *defaultVal) { + SConfigItem item = {.dtype = CFG_DTYPE_CHARSET}; + if (cfgCheckAndSetCharset(&item, defaultVal) != 0) { + return -1; + } + + return cfgAddItem(pCfg, &item, name); +} + +int32_t cfgAddTimezone(SConfig *pCfg, const char *name, const char *defaultVal) { + SConfigItem item = {.dtype = CFG_DTYPE_TIMEZONE}; + if (cfgCheckAndSetTimezone(&item, defaultVal) != 0) { + return -1; + } + + return cfgAddItem(pCfg, &item, name); +} + +const char *cfgStypeStr(ECfgSrcType type) { + switch (type) { + case CFG_STYPE_DEFAULT: + return "default"; + case CFG_STYPE_CFG_FILE: + return "cfg_file"; + case CFG_STYPE_ENV_FILE: + return "env_file"; + case CFG_STYPE_ENV_VAR: + return "env_var"; + case CFG_STYPE_APOLLO_URL: + return "apollo_url"; + case CFG_STYPE_ARG_LIST: + return "arg_list"; + case CFG_STYPE_API_OPTION: + return "api_option"; + default: + return "invalid"; + } +} + +const char *cfgDtypeStr(ECfgDataType type) { + switch (type) { + case CFG_DTYPE_NONE: + return "none"; + case CFG_DTYPE_BOOL: + return "bool"; + case CFG_DTYPE_INT32: + return "int32"; + case CFG_DTYPE_INT64: + return "int64"; + case CFG_DTYPE_FLOAT: + return "float"; + case CFG_DTYPE_STRING: + return "string"; + case CFG_DTYPE_IPSTR: + return "ipstr"; + case CFG_DTYPE_DIR: + return "dir"; + case CFG_DTYPE_LOCALE: + return "locale"; + case CFG_DTYPE_CHARSET: + return "charset"; + case CFG_DTYPE_TIMEZONE: + return "timezone"; + default: + return "invalid"; + } +} + +void cfgDumpCfg(SConfig *pCfg) { + uInfo(" global config"); + uInfo("================================================================="); + + char src[CFG_SRC_PRINT_LEN + 1] = {0}; + char name[CFG_NAME_PRINT_LEN + 1] = {0}; + + SConfigItem *pItem = cfgIterate(pCfg, NULL); + while (pItem != NULL) { + tstrncpy(src, cfgStypeStr(pItem->stype), CFG_SRC_PRINT_LEN); + for (int32_t i = 0; i < CFG_SRC_PRINT_LEN; ++i) { + if (src[i] == 0) src[i] = ' '; + } + + tstrncpy(name, pItem->name, CFG_NAME_PRINT_LEN); + for (int32_t i = 0; i < CFG_NAME_PRINT_LEN; ++i) { + if (name[i] == 0) name[i] = ' '; + } + + switch (pItem->dtype) { + case CFG_DTYPE_BOOL: + uInfo("%s %s %u", src, name, pItem->bval); + break; + case CFG_DTYPE_INT32: + uInfo("%s %s %d", src, name, pItem->i32); + break; + case CFG_DTYPE_INT64: + uInfo("%s %s %" PRId64, src, name, pItem->i64); + break; + case CFG_DTYPE_FLOAT: + uInfo("%s %s %f", src, name, pItem->fval); + break; + case CFG_DTYPE_STRING: + case CFG_DTYPE_IPSTR: + case CFG_DTYPE_DIR: + case CFG_DTYPE_LOCALE: + case CFG_DTYPE_CHARSET: + case CFG_DTYPE_TIMEZONE: + uInfo("%s %s %s", src, name, pItem->str); + break; + } + pItem = cfgIterate(pCfg, pItem); + } + + uInfo("================================================================="); +} +#if 0 +// int32_t cfgCheck(SConfig *pCfg) { +// SConfigItem *pItem = NULL; + +// pItem = cfgGetItem(pCfg, "serverPort"); +// if (pItem != NULL) { +// tsServerPort = (uint16_t)pItem->i32; +// } + +// pItem = cfgGetItem(pCfg, "firstEp"); +// if (pItem != NULL) { +// tstrncpy(tsFirst, pItem->str, TSDB_EP_LEN); +// } + +// snprintf(tsLocalEp, TSDB_EP_LEN, "%s:%u", tsLocalFqdn, tsServerPort); +// uInfo("localEp is: %s", tsLocalEp); + +// SEp ep = {0}; +// if (tsFirst[0] == 0) { +// strcpy(tsFirst, tsLocalEp); +// } else { +// taosGetFqdnPortFromEp(tsFirst, &ep); +// snprintf(tsFirst, TSDB_EP_LEN, "%s:%u", ep.fqdn, ep.port); +// } + +// pItem = cfgGetItem(pCfg, "secondEp"); +// if (pItem != NULL) { +// tstrncpy(tsSecond, pItem->str, TSDB_EP_LEN); +// } + +// if (tsSecond[0] == 0) { +// strcpy(tsSecond, tsLocalEp); +// } else { +// taosGetFqdnPortFromEp(tsSecond, &ep); +// snprintf(tsSecond, TSDB_EP_LEN, "%s:%u", ep.fqdn, ep.port); +// } + +// pItem = cfgGetItem(pCfg, "dataDir"); +// if (pItem != NULL) { +// tstrncpy(tsDataDir, pItem->str, PATH_MAX); +// } + +// if (tsDiskCfgNum <= 0) { +// taosAddDataDir(0, tsDataDir, 0, 1); +// tsDiskCfgNum = 1; +// uTrace("dataDir:%s, level:0 primary:1 is configured by default", tsDataDir); +// } + +// if (taosDirExist(tsTempDir) != 0) { +// return -1; +// } + +// taosGetSystemInfo(); + +// tsSetLocale(); + +// // SGlobalCfg *cfg_timezone = taosGetConfigOption("timezone"); +// // if (cfg_timezone && cfg_timezone->cfgStatus == TAOS_CFG_CSTATUS_FILE) { +// tsSetTimeZone(); +// // } + +// pItem = cfgGetItem(pCfg, "numOfCores"); +// if (pItem != NULL) { +// tsNumOfCores = pItem->i32; +// } + +// if (tsNumOfCores <= 0) { +// tsNumOfCores = 1; +// } + +// if (tsQueryBufferSize >= 0) { +// tsQueryBufferSizeBytes = tsQueryBufferSize * 1048576UL; +// } + +// cfgPrintCfg(pCfg); + +// return 0; +// } +#endif \ No newline at end of file diff --git a/source/libs/config/test/CMakeLists.txt b/source/libs/config/test/CMakeLists.txt new file mode 100644 index 0000000000..1e63d5025d --- /dev/null +++ b/source/libs/config/test/CMakeLists.txt @@ -0,0 +1,14 @@ +enable_testing() + +aux_source_directory(. CFG_TEST_SRC) +add_executable(cfg_test ${CFG_TEST_SRC}) +target_link_libraries( + cfg_test + PUBLIC config + PUBLIC gtest_main +) + +add_test( + NAME cfg_test + COMMAND cfg_test +) diff --git a/source/libs/config/test/cfgTest.cpp b/source/libs/config/test/cfgTest.cpp new file mode 100644 index 0000000000..7c70e1a8f8 --- /dev/null +++ b/source/libs/config/test/cfgTest.cpp @@ -0,0 +1,147 @@ +/** + * @file cfgTest.cpp + * @author slguan (slguan@taosdata.com) + * @brief config module tests + * @version 1.0 + * @date 2022-02-20 + * + * @copyright Copyright (c) 2022 + * + */ + +#include +#include "config.h" + +class CfgTest : public ::testing::Test { + protected: + static void SetUpTestSuite() {} + static void TearDownTestSuite() {} + + public: + void SetUp() override {} + void TearDown() override {} + void InitCfg(SConfig *pConfig); + + static const char *pConfig; +}; + +const char *CfgTest::pConfig; + +TEST_F(CfgTest, 01_Str) { + EXPECT_STREQ(cfgStypeStr(CFG_STYPE_DEFAULT), "default"); + EXPECT_STREQ(cfgStypeStr(CFG_STYPE_CFG_FILE), "cfg_file"); + EXPECT_STREQ(cfgStypeStr(CFG_STYPE_ENV_FILE), "env_file"); + EXPECT_STREQ(cfgStypeStr(CFG_STYPE_ENV_VAR), "env_var"); + EXPECT_STREQ(cfgStypeStr(CFG_STYPE_APOLLO_URL), "apollo_url"); + EXPECT_STREQ(cfgStypeStr(CFG_STYPE_ARG_LIST), "arg_list"); + EXPECT_STREQ(cfgStypeStr(CFG_STYPE_API_OPTION), "api_option"); + EXPECT_STREQ(cfgStypeStr(ECfgSrcType(1024)), "invalid"); + + EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_NONE), "none"); + EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_BOOL), "bool"); + EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_INT32), "int32"); + EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_INT64), "int64"); + EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_FLOAT), "float"); + EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_STRING), "string"); + EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_IPSTR), "ipstr"); + EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_DIR), "dir"); + EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_DIR), "dir"); + EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_DIR), "dir"); + EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_DIR), "dir"); + EXPECT_STREQ(cfgDtypeStr(ECfgDataType(1024)), "invalid"); +} + +TEST_F(CfgTest, 02_Basic) { + SConfig *pConfig = cfgInit(); + ASSERT_NE(pConfig, nullptr); + + EXPECT_EQ(cfgAddBool(pConfig, "test_bool", 0), 0); + EXPECT_EQ(cfgAddInt32(pConfig, "test_int32", 1, 0, 16), 0); + EXPECT_EQ(cfgAddInt64(pConfig, "test_int64", 2, 0, 16), 0); + EXPECT_EQ(cfgAddFloat(pConfig, "test_float", 3, 0, 16), 0); + EXPECT_EQ(cfgAddString(pConfig, "test_string", "4"), 0); + EXPECT_EQ(cfgAddIpStr(pConfig, "test_ipstr", "192.168.0.1"), 0); + EXPECT_EQ(cfgAddDir(pConfig, "test_dir", "/tmp"), 0); + + EXPECT_EQ(cfgGetSize(pConfig), 7); + + int32_t size = 0; + SConfigItem *pItem = cfgIterate(pConfig, NULL); + while (pItem != NULL) { + switch (pItem->dtype) { + case CFG_DTYPE_BOOL: + printf("index:%d, cfg:%s value:%d\n", size, pItem->name, pItem->bval); + break; + case CFG_DTYPE_INT32: + printf("index:%d, cfg:%s value:%d\n", size, pItem->name, pItem->i32); + break; + case CFG_DTYPE_INT64: + printf("index:%d, cfg:%s value:%" PRId64 "\n", size, pItem->name, pItem->i64); + break; + case CFG_DTYPE_FLOAT: + printf("index:%d, cfg:%s value:%f\n", size, pItem->name, pItem->fval); + break; + case CFG_DTYPE_STRING: + printf("index:%d, cfg:%s value:%s\n", size, pItem->name, pItem->str); + break; + case CFG_DTYPE_IPSTR: + printf("index:%d, cfg:%s value:%s\n", size, pItem->name, pItem->str); + break; + case CFG_DTYPE_DIR: + printf("index:%d, cfg:%s value:%s\n", size, pItem->name, pItem->str); + break; + default: + printf("index:%d, cfg:%s invalid cfg dtype:%d\n", size, pItem->name, pItem->dtype); + break; + } + size++; + pItem = cfgIterate(pConfig, pItem); + } + cfgCancelIterate(pConfig, pItem); + + EXPECT_EQ(cfgGetSize(pConfig), 7); + + pItem = cfgGetItem(pConfig, "test_bool"); + EXPECT_EQ(pItem->stype, CFG_STYPE_DEFAULT); + EXPECT_EQ(pItem->dtype, CFG_DTYPE_BOOL); + EXPECT_STREQ(pItem->name, "test_bool"); + EXPECT_EQ(pItem->bval, 0); + + pItem = cfgGetItem(pConfig, "test_int32"); + EXPECT_EQ(pItem->stype, CFG_STYPE_DEFAULT); + EXPECT_EQ(pItem->dtype, CFG_DTYPE_INT32); + EXPECT_STREQ(pItem->name, "test_int32"); + EXPECT_EQ(pItem->i32, 1); + + pItem = cfgGetItem(pConfig, "test_int64"); + EXPECT_EQ(pItem->stype, CFG_STYPE_DEFAULT); + EXPECT_EQ(pItem->dtype, CFG_DTYPE_INT64); + EXPECT_STREQ(pItem->name, "test_int64"); + EXPECT_EQ(pItem->i64, 2); + + pItem = cfgGetItem(pConfig, "test_float"); + EXPECT_EQ(pItem->stype, CFG_STYPE_DEFAULT); + EXPECT_EQ(pItem->dtype, CFG_DTYPE_FLOAT); + EXPECT_STREQ(pItem->name, "test_float"); + EXPECT_EQ(pItem->fval, 3); + + pItem = cfgGetItem(pConfig, "test_string"); + EXPECT_EQ(pItem->stype, CFG_STYPE_DEFAULT); + EXPECT_EQ(pItem->dtype, CFG_DTYPE_STRING); + EXPECT_STREQ(pItem->name, "test_string"); + EXPECT_STREQ(pItem->str, "4"); + + pItem = cfgGetItem(pConfig, "test_ipstr"); + EXPECT_EQ(pItem->stype, CFG_STYPE_DEFAULT); + EXPECT_EQ(pItem->dtype, CFG_DTYPE_IPSTR); + EXPECT_STREQ(pItem->name, "test_ipstr"); + EXPECT_STREQ(pItem->str, "192.168.0.1"); + + pItem = cfgGetItem(pConfig, "test_dir"); + EXPECT_EQ(pItem->stype, CFG_STYPE_DEFAULT); + EXPECT_EQ(pItem->dtype, CFG_DTYPE_DIR); + EXPECT_STREQ(pItem->name, "test_dir"); + EXPECT_STREQ(pItem->str, "/tmp"); + + cfgCleanup(pConfig); +} diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 18485249b3..4078ee9291 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -35,7 +35,7 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, char* id) } else { SStreamBlockScanInfo* pInfo = pOperator->info; if (tqReadHandleSetMsg(pInfo->readerHandle, input, 0) < 0) { - qError("submit msg error while set stream msg, %s" PRIx64, id); + qError("submit msg messed up when initing stream block, %s" PRIx64, id); return TSDB_CODE_QRY_APP_ERROR; } return TSDB_CODE_SUCCESS; diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 5508b5ecd7..62c0e3623f 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -132,7 +132,8 @@ do { \ } while (0) int32_t getMaximumIdleDurationSec() { - return tsShellActivityTimer * 2; + // todo + return 6; //tsShellActivityTimer * 2; } static int32_t getExprFunctionId(SExprInfo *pExprInfo) { @@ -5301,10 +5302,12 @@ SOperatorInfo* createExchangeOperatorInfo(const SArray* pSources, const SArray* rpcInit.label = "EX"; rpcInit.numOfThreads = 1; rpcInit.cfp = qProcessFetchRsp; - rpcInit.sessions = tsMaxConnections; + // todo + rpcInit.sessions = 50000; //tsMaxConnections; rpcInit.connType = TAOS_CONN_CLIENT; rpcInit.user = (char *)"root"; - rpcInit.idleTime = tsShellActivityTimer * 1000; + // todo + rpcInit.idleTime = 6; //tsShellActivityTimer * 1000; rpcInit.ckey = "key"; rpcInit.spi = 1; rpcInit.secret = (char *)"dcc5bed04851fec854c035b2e40263b6"; diff --git a/source/libs/index/src/index_fst.c b/source/libs/index/src/index_fst.c index 3664bcfad0..a6cabbd439 100644 --- a/source/libs/index/src/index_fst.c +++ b/source/libs/index/src/index_fst.c @@ -235,6 +235,7 @@ void fstStateCompileForOneTrans(FstCountingWriter* w, CompiledAddr addr, FstTran FstState st = fstStateCreate(OneTrans); fstStateSetCommInput(&st, trn->inp); + bool null = false; uint8_t inp = fstStateCommInput(&st, &null); if (null == true) { @@ -936,6 +937,7 @@ FstLastTransition* fstLastTransitionCreate(uint8_t inp, Output out) { } void fstLastTransitionDestroy(FstLastTransition* trn) { free(trn); } + void fstBuilderNodeUnfinishedLastCompiled(FstBuilderNodeUnfinished* unNode, CompiledAddr addr) { FstLastTransition* trn = unNode->last; if (trn == NULL) { diff --git a/source/libs/index/src/index_fst_automation.c b/source/libs/index/src/index_fst_automation.c index 590ff294bf..ed1ad7a374 100644 --- a/source/libs/index/src/index_fst_automation.c +++ b/source/libs/index/src/index_fst_automation.c @@ -16,24 +16,24 @@ #include "index_fst_automation.h" StartWithStateValue* startWithStateValueCreate(StartWithStateKind kind, ValueType ty, void* val) { - StartWithStateValue* nsv = calloc(1, sizeof(StartWithStateValue)); - if (nsv == NULL) { + StartWithStateValue* sv = calloc(1, sizeof(StartWithStateValue)); + if (sv == NULL) { return NULL; } - nsv->kind = kind; - nsv->type = ty; + sv->kind = kind; + sv->type = ty; if (ty == FST_INT) { - nsv->val = *(int*)val; + sv->val = *(int*)val; } else if (ty == FST_CHAR) { size_t len = strlen((char*)val); - nsv->ptr = (char*)calloc(1, len + 1); - memcpy(nsv->ptr, val, len); + sv->ptr = (char*)calloc(1, len + 1); + memcpy(sv->ptr, val, len); } else if (ty == FST_ARRAY) { // TODO, // nsv->arr = taosArrayFromList() } - return nsv; + return sv; } void startWithStateValueDestroy(void* val) { StartWithStateValue* sv = (StartWithStateValue*)val; @@ -146,11 +146,9 @@ AutomationCtx* automCtxCreate(void* data, AutomationType atype) { if (atype == AUTOMATION_ALWAYS) { int val = 0; sv = startWithStateValueCreate(Running, FST_INT, &val); - ctx->stdata = (void*)sv; } else if (atype == AUTOMATION_PREFIX) { int val = 0; sv = startWithStateValueCreate(Running, FST_INT, &val); - ctx->stdata = (void*)sv; } else if (atype == AUTMMATION_MATCH) { } else { // add more search type @@ -160,9 +158,8 @@ AutomationCtx* automCtxCreate(void* data, AutomationType atype) { if (data != NULL) { char* src = (char*)data; size_t len = strlen(src); - dst = (char*)malloc(len * sizeof(char) + 1); + dst = (char*)calloc(1, len * sizeof(char) + 1); memcpy(dst, src, len); - dst[len] = 0; } ctx->data = dst; diff --git a/source/libs/index/test/fstUT.cc b/source/libs/index/test/fstUT.cc index d10f5f47b8..d59a3428da 100644 --- a/source/libs/index/test/fstUT.cc +++ b/source/libs/index/test/fstUT.cc @@ -30,8 +30,8 @@ static void EnvInit() { taosRemoveDir(path.c_str()); taosMkDir(path.c_str()); // init log file - snprintf(indexlog, PATH_MAX, "%s/tindex.idx", path.c_str()); - if (taosInitLog(indexlog, tsNumOfLogLines, 1) != 0) { + tstrncpy(tsLogDir, path.c_str(), PATH_MAX); + if (taosInitLog("tindex.idx", 1) != 0) { printf("failed to init log"); } // init index file @@ -99,6 +99,7 @@ class FstReadMemory { fstSliceDestroy(&skey); return ok; } + bool GetWithTimeCostUs(const std::string& key, uint64_t* val, uint64_t* elapse) { int64_t s = taosGetTimestampUs(); bool ok = this->Get(key, val); @@ -120,8 +121,6 @@ class FstReadMemory { printf("key: %s, val: %" PRIu64 "\n", key.c_str(), (uint64_t)(rt->out.out)); swsResultDestroy(rt); } - for (size_t i = 0; i < result.size(); i++) { - } std::cout << std::endl; return true; } @@ -137,7 +136,7 @@ class FstReadMemory { fstDestroy(_fst); fstSliceDestroy(&_s); writerCtxDestroy(_wc, false); - tfCleanup(); + // tfCleanup(); } private: @@ -196,6 +195,10 @@ class TFst { } return fr->Get(k, v); } + bool Search(AutomationCtx* ctx, std::vector& result) { + // add more + return fr->Search(ctx, result); + } private: FstWriter* fw; @@ -229,5 +232,9 @@ TEST_F(FstEnv, writeNormal) { assert(fst->Get("a", &val) == false); assert(fst->Get("aa", &val) == true); assert(val == 0); + + std::vector rlt; + AutomationCtx* ctx = automCtxCreate((void*)"ab", AUTOMATION_ALWAYS); + assert(fst->Search(ctx, rlt) == true); } -TEST_F(FstEnv, writeExcpet) {} +TEST_F(FstEnv, WriteMillonrRecord) {} diff --git a/source/libs/qcom/src/queryUtil.c b/source/libs/qcom/src/queryUtil.c index 4cd040c238..b29817a318 100644 --- a/source/libs/qcom/src/queryUtil.c +++ b/source/libs/qcom/src/queryUtil.c @@ -85,9 +85,11 @@ static void* pTaskQueue = NULL; int32_t initTaskQueue() { double factor = 4.0; - int32_t numOfThreads = TMAX((int)(tsNumOfCores * tsNumOfThreadsPerCore / factor), 2); + // todo + // int32_t numOfThreads = TMAX((int)(tsNumOfCores * tsNumOfThreadsPerCore / factor), 2); + int32_t numOfThreads = TMAX((int)(tsNumOfCores * 1.0f / factor), 2); - int32_t queueSize = tsMaxConnections * 2; + int32_t queueSize = 25000; //tsMaxConnections * 2; pTaskQueue = taosInitScheduler(queueSize, numOfThreads, "tsc"); if (NULL == pTaskQueue) { qError("failed to init task queue"); diff --git a/source/libs/qworker/test/qworkerTests.cpp b/source/libs/qworker/test/qworkerTests.cpp index 38890150ee..8ad5a76388 100644 --- a/source/libs/qworker/test/qworkerTests.cpp +++ b/source/libs/qworker/test/qworkerTests.cpp @@ -101,9 +101,7 @@ void qwtInitLogFile() { tsAsyncLog = 0; qDebugFlag = 159; - char temp[128] = {0}; - sprintf(temp, "%s/%s", tsLogDir, defaultLogFileNamePrefix); - if (taosInitLog(temp, tsNumOfLogLines, maxLogFileNum) < 0) { + if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) { printf("failed to open log file in directory:%s\n", tsLogDir); } diff --git a/source/libs/scalar/test/filter/filterTests.cpp b/source/libs/scalar/test/filter/filterTests.cpp index 3d4d94d92a..a159263804 100644 --- a/source/libs/scalar/test/filter/filterTests.cpp +++ b/source/libs/scalar/test/filter/filterTests.cpp @@ -52,9 +52,7 @@ void flttInitLogFile() { tsAsyncLog = 0; qDebugFlag = 159; - char temp[128] = {0}; - sprintf(temp, "%s/%s", tsLogDir, defaultLogFileNamePrefix); - if (taosInitLog(temp, tsNumOfLogLines, maxLogFileNum) < 0) { + if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) { printf("failed to open log file in directory:%s\n", tsLogDir); } } diff --git a/source/libs/scalar/test/scalar/scalarTests.cpp b/source/libs/scalar/test/scalar/scalarTests.cpp index 2f0a60362d..24bc8eaf40 100644 --- a/source/libs/scalar/test/scalar/scalarTests.cpp +++ b/source/libs/scalar/test/scalar/scalarTests.cpp @@ -51,9 +51,7 @@ void scltInitLogFile() { tsAsyncLog = 0; qDebugFlag = 159; - char temp[128] = {0}; - sprintf(temp, "%s/%s", tsLogDir, defaultLogFileNamePrefix); - if (taosInitLog(temp, tsNumOfLogLines, maxLogFileNum) < 0) { + if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) { printf("failed to open log file in directory:%s\n", tsLogDir); } } diff --git a/source/libs/scheduler/test/schedulerTests.cpp b/source/libs/scheduler/test/schedulerTests.cpp index 6bfff0197c..89d365a7e7 100644 --- a/source/libs/scheduler/test/schedulerTests.cpp +++ b/source/libs/scheduler/test/schedulerTests.cpp @@ -66,9 +66,7 @@ void schtInitLogFile() { tsAsyncLog = 0; qDebugFlag = 159; - char temp[128] = {0}; - sprintf(temp, "%s/%s", tsLogDir, defaultLogFileNamePrefix); - if (taosInitLog(temp, tsNumOfLogLines, maxLogFileNum) < 0) { + if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) { printf("failed to open log file in directory:%s\n", tsLogDir); } diff --git a/source/libs/transport/inc/transportInt.h b/source/libs/transport/inc/transportInt.h index a36b671eb4..d080db753d 100644 --- a/source/libs/transport/inc/transportInt.h +++ b/source/libs/transport/inc/transportInt.h @@ -66,6 +66,7 @@ typedef struct { void (*cfp)(void* parent, SRpcMsg*, SEpSet*); int (*afp)(void* parent, char* user, char* spi, char* encrypt, char* secret, char* ckey); + bool (*pfp)(void* parent, tmsg_t msgType); int32_t refCount; void* parent; diff --git a/source/libs/transport/src/rpcMain.c b/source/libs/transport/src/rpcMain.c index 72c1ff6893..44bb6fdb70 100644 --- a/source/libs/transport/src/rpcMain.c +++ b/source/libs/transport/src/rpcMain.c @@ -42,6 +42,8 @@ int tsRpcMaxRetry; int tsRpcHeadSize; int tsRpcOverhead; +int32_t tsRpcForceTcp = 1; // disable this, means query, show command use udp protocol as default + SHashObj *tsFqdnHash; #ifndef USE_UV @@ -143,6 +145,10 @@ typedef struct SRpcConn { static int tsRpcRefId = -1; static int32_t tsRpcNum = 0; + +int32_t tsRpcTimer = 300; +int32_t tsRpcMaxTime = 600; // seconds; + // static pthread_once_t tsRpcInit = PTHREAD_ONCE_INIT; // server:0 client:1 tcp:2 udp:0 diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index 48c15ca286..453b5184d2 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -29,7 +29,12 @@ void* rpcOpen(const SRpcInit* pInit) { if (pInit->label) { tstrncpy(pRpc->label, pInit->label, strlen(pInit->label) + 1); } + + // register callback handle pRpc->cfp = pInit->cfp; + pRpc->afp = pInit->afp; + pRpc->pfp = pInit->pfp; + if (pInit->connType == TAOS_CONN_SERVER) { pRpc->numOfThreads = pInit->numOfThreads > TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS : pInit->numOfThreads; } else { diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 8312c0217c..d5fa4c8c47 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -134,8 +134,7 @@ static void clientHandleResp(SCliConn* conn) { rpcMsg.msgType = pHead->msgType; rpcMsg.ahandle = pCtx->ahandle; - if (rpcMsg.msgType == TDMT_VND_QUERY_RSP || rpcMsg.msgType == TDMT_VND_FETCH_RSP || - rpcMsg.msgType == TDMT_VND_RES_READY_RSP) { + if (pRpc->pfp != NULL && (pRpc->pfp)(pRpc->parent, rpcMsg.msgType)) { rpcMsg.handle = conn; conn->persist = 1; tDebug("client conn %p persist by app", conn); @@ -185,18 +184,13 @@ static void clientHandleExcept(SCliConn* pConn) { clientConnDestroy(pConn, true); return; } - SCliMsg* pMsg = pConn->data; - - tmsg_t msgType = TDMT_MND_CONNECT; - if (pMsg != NULL) { - msgType = pMsg->msg.msgType; - } + SCliMsg* pMsg = pConn->data; STransConnCtx* pCtx = pMsg->ctx; SRpcMsg rpcMsg = {0}; rpcMsg.ahandle = pCtx->ahandle; rpcMsg.code = TSDB_CODE_RPC_NETWORK_UNAVAIL; - rpcMsg.msgType = msgType + 1; + rpcMsg.msgType = pMsg->msg.msgType + 1; if (pConn->push != NULL && pConn->ctnRdCnt != 0) { (*pConn->push->callback)(pConn->push->arg, &rpcMsg); @@ -445,7 +439,7 @@ static void clientConnCb(uv_connect_t* req, int status) { addrlen = sizeof(pConn->locaddr); uv_tcp_getsockname((uv_tcp_t*)pConn->stream, (struct sockaddr*)&pConn->locaddr, &addrlen); - tTrace("client conn %p create", pConn); + tTrace("client conn %p connect to server successfully", pConn); assert(pConn->stream == req->handle); clientWrite(pConn); @@ -524,6 +518,7 @@ static void clientHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd) { struct sockaddr_in addr; uv_ip4_addr(pMsg->ctx->ip, pMsg->ctx->port, &addr); // handle error in callback if fail to connect + tTrace("client conn %p try to connect to %s:%d", conn, pMsg->ctx->ip, pMsg->ctx->port); uv_tcp_connect(&conn->connReq, (uv_tcp_t*)(conn->stream), (const struct sockaddr*)&addr, clientConnCb); } diff --git a/source/libs/transport/src/transSrv.c b/source/libs/transport/src/transSrv.c index 9fca371bf3..f0db054797 100644 --- a/source/libs/transport/src/transSrv.c +++ b/source/libs/transport/src/transSrv.c @@ -413,11 +413,6 @@ void uvWorkerAsyncCb(uv_async_t* handle) { } else { uvStartSendResp(msg); } - // uv_buf_t wb; - // uvPrepareSendData(msg, &wb); - // uv_timer_stop(conn->pTimer); - - // uv_write(conn->pWriter, (uv_stream_t*)conn->pTcp, &wb, 1, uvOnWriteCb); } } static void uvAcceptAsyncCb(uv_async_t* async) { @@ -490,7 +485,6 @@ void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) { pConn->pTimer->data = pConn; pConn->hostThrd = pThrd; - // pConn->pWorkerAsync = pThrd->workerAsync; // thread safty // init client handle pConn->pTcp = (uv_tcp_t*)malloc(sizeof(uv_tcp_t)); @@ -730,14 +724,9 @@ void destroyWorkThrd(SWorkThrdObj* pThrd) { } void sendQuitToWorkThrd(SWorkThrdObj* pThrd) { SSrvMsg* srvMsg = calloc(1, sizeof(SSrvMsg)); - - // pthread_mutex_lock(&pThrd->msgMtx); - // QUEUE_PUSH(&pThrd->msg, &srvMsg->q); - // pthread_mutex_unlock(&pThrd->msgMtx); tDebug("send quit msg to work thread"); transSendAsync(pThrd->asyncPool, &srvMsg->q); - // uv_async_send(pThrd->workerAsync); } void taosCloseServer(void* arg) { @@ -774,19 +763,12 @@ void rpcSendResponse(const SRpcMsg* pMsg) { SSrvMsg* srvMsg = calloc(1, sizeof(SSrvMsg)); srvMsg->pConn = pConn; srvMsg->msg = *pMsg; - - // pthread_mutex_lock(&pThrd->msgMtx); - // QUEUE_PUSH(&pThrd->msg, &srvMsg->q); - // pthread_mutex_unlock(&pThrd->msgMtx); - tTrace("server conn %p start to send resp", pConn); transSendAsync(pThrd->asyncPool, &srvMsg->q); - // uv_async_send(pThrd->workerAsync); } int rpcGetConnInfo(void* thandle, SRpcConnInfo* pInfo) { SSrvConn* pConn = thandle; - // struct sockaddr* pPeerName = &pConn->peername; struct sockaddr_in addr = pConn->addr; pInfo->clientIp = (uint32_t)(addr.sin_addr.s_addr); diff --git a/source/libs/transport/test/pushClient.c b/source/libs/transport/test/pushClient.c index f1aadafacc..4842a0c800 100644 --- a/source/libs/transport/test/pushClient.c +++ b/source/libs/transport/test/pushClient.c @@ -192,7 +192,7 @@ int main(int argc, char *argv[]) { } } - taosInitLog("client.log", 100000, 10); + taosInitLog("client.log", 10); void *pRpc = rpcOpen(&rpcInit); if (pRpc == NULL) { diff --git a/source/libs/transport/test/pushServer.c b/source/libs/transport/test/pushServer.c index 0bcc47383b..0ccfde16ce 100644 --- a/source/libs/transport/test/pushServer.c +++ b/source/libs/transport/test/pushServer.c @@ -169,7 +169,7 @@ int main(int argc, char *argv[]) { tsAsyncLog = 0; rpcInit.connType = TAOS_CONN_SERVER; - taosInitLog("server.log", 100000, 10); + taosInitLog("server.log", 10); void *pRpc = rpcOpen(&rpcInit); if (pRpc == NULL) { diff --git a/source/libs/transport/test/rclient.c b/source/libs/transport/test/rclient.c index cc6a63d3cd..bcdf32bf6a 100644 --- a/source/libs/transport/test/rclient.c +++ b/source/libs/transport/test/rclient.c @@ -169,7 +169,7 @@ int main(int argc, char *argv[]) { } } - taosInitLog("client.log", 100000, 10); + taosInitLog("client.log", 10); void *pRpc = rpcOpen(&rpcInit); if (pRpc == NULL) { diff --git a/source/libs/transport/test/rsclient.c b/source/libs/transport/test/rsclient.c index 65170d4abb..26a02eb05b 100644 --- a/source/libs/transport/test/rsclient.c +++ b/source/libs/transport/test/rsclient.c @@ -97,7 +97,7 @@ int main(int argc, char *argv[]) { rpcInit.label = "APP"; rpcInit.numOfThreads = 1; rpcInit.sessions = 100; - rpcInit.idleTime = tsShellActivityTimer*1000; + rpcInit.idleTime = 3000; //tsShellActivityTimer*1000; rpcInit.user = "michael"; rpcInit.secret = secret; rpcInit.ckey = "key"; @@ -148,7 +148,7 @@ int main(int argc, char *argv[]) { } } - taosInitLog("client.log", 100000, 10); + taosInitLog("client.log", 10); void *pRpc = rpcOpen(&rpcInit); if (pRpc == NULL) { diff --git a/source/libs/transport/test/rserver.c b/source/libs/transport/test/rserver.c index d1a587f4e5..4337cd242c 100644 --- a/source/libs/transport/test/rserver.c +++ b/source/libs/transport/test/rserver.c @@ -158,7 +158,7 @@ int main(int argc, char *argv[]) { tsAsyncLog = 0; rpcInit.connType = TAOS_CONN_SERVER; - taosInitLog("server.log", 100000, 10); + taosInitLog("server.log", 10); void *pRpc = rpcOpen(&rpcInit); if (pRpc == NULL) { diff --git a/source/libs/transport/test/syncClient.c b/source/libs/transport/test/syncClient.c index c5d7f5664a..b7ef296b9d 100644 --- a/source/libs/transport/test/syncClient.c +++ b/source/libs/transport/test/syncClient.c @@ -170,7 +170,7 @@ int main(int argc, char *argv[]) { } } - taosInitLog("client.log", 100000, 10); + taosInitLog("client.log", 10); void *pRpc = rpcOpen(&rpcInit); if (pRpc == NULL) { diff --git a/source/libs/transport/test/transUT.cc b/source/libs/transport/test/transUT.cc index d1fefe2c72..f5b3ed4c32 100644 --- a/source/libs/transport/test/transUT.cc +++ b/source/libs/transport/test/transUT.cc @@ -148,7 +148,6 @@ class TransObj { wDebugFlag = 0; sDebugFlag = 0; tsdbDebugFlag = 0; - cqDebugFlag = 0; tscEmbeddedInUtil = 1; tsAsyncLog = 0; @@ -156,9 +155,8 @@ class TransObj { taosRemoveDir(path.c_str()); taosMkDir(path.c_str()); - char temp[PATH_MAX]; - snprintf(temp, PATH_MAX, "%s/taosdlog", path.c_str()); - if (taosInitLog(temp, tsNumOfLogLines, 1) != 0) { + tstrncpy(tsLogDir, path.c_str(), PATH_MAX); + if (taosInitLog("taosdlog", 1) != 0) { printf("failed to init log file\n"); } cli = new Client; diff --git a/source/os/src/osDir.c b/source/os/src/osDir.c index 8999646f6a..8fee6a6f7d 100644 --- a/source/os/src/osDir.c +++ b/source/os/src/osDir.c @@ -114,7 +114,7 @@ void taosRemoveOldFiles(char *dirname, int32_t keepDays) { rmdir(dirname); } -int32_t taosExpandDir(char *dirname, char *outname, int32_t maxlen) { +int32_t taosExpandDir(const char *dirname, char *outname, int32_t maxlen) { wordexp_t full_path; if (0 != wordexp(dirname, &full_path, 0)) { //printf("failed to expand path:%s since %s", dirname, strerror(errno)); diff --git a/source/os/src/osEnv.c b/source/os/src/osEnv.c index 7bc5e0a7fc..b90d9fbe94 100644 --- a/source/os/src/osEnv.c +++ b/source/os/src/osEnv.c @@ -16,6 +16,30 @@ #define _DEFAULT_SOURCE #include "os.h" +#include "osEnv.h" +#include "osSysinfo.h" + +SDiskSpace tsLogSpace; +SDiskSpace tsTempSpace; +SDiskSpace tsDataSpace; + +void taosUpdateLogSpace() { taosGetDiskSize(tsLogDir, &tsLogSpace.size); } + +void taosUpdateTempSpace() { taosGetDiskSize(tsTempDir, &tsTempSpace.size); } + +void taosUpdateDataSpace() { taosGetDiskSize(tsDataDir, &tsDataSpace.size); } + +bool taosLogSpaceAvailable() { return tsLogSpace.reserved < tsLogSpace.size.avail; } + +bool taosTempSpaceAvailable() { return tsTempSpace.reserved < tsTempSpace.size.avail; } + +bool taosDataSpaceAvailable() { return tsDataSpace.reserved < tsDataSpace.size.avail; } + +void taosUpdateAllSpace() { + taosUpdateLogSpace(); + taosUpdateTempSpace(); + taosUpdateDataSpace(); +} #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) @@ -23,7 +47,6 @@ char tsOsName[10] = "Windows"; char configDir[PATH_MAX] = "C:/TDengine/cfg"; char tsDataDir[PATH_MAX] = "C:/TDengine/data"; char tsLogDir[PATH_MAX] = "C:/TDengine/log"; -char tsScriptDir[PATH_MAX] = "C:/TDengine/script"; char tsTempDir[PATH_MAX] = "C:\\Windows\\Temp"; extern taosWinSocketInit(); @@ -47,7 +70,6 @@ char tsOsName[10] = "Darwin"; char configDir[PATH_MAX] = "/usr/local/etc/taos"; char tsDataDir[PATH_MAX] = "/usr/local/var/lib/taos"; char tsLogDir[PATH_MAX] = "/usr/local/var/log/taos"; -char tsScriptDir[PATH_MAX] = "/usr/local/etc/taos"; char tsTempDir[PATH_MAX] = "/tmp/taosd"; void osInit() {} @@ -58,7 +80,6 @@ char tsOsName[10] = "Linux"; char configDir[PATH_MAX] = "/etc/taos"; char tsDataDir[PATH_MAX] = "/var/lib/taos"; char tsLogDir[PATH_MAX] = "/var/log/taos"; -char tsScriptDir[PATH_MAX] = "/etc/taos"; char tsTempDir[PATH_MAX] = "/tmp/"; void osInit() {} diff --git a/source/os/src/osSocket.c b/source/os/src/osSocket.c index e12f9493b3..8afb4ae8b7 100644 --- a/source/os/src/osSocket.c +++ b/source/os/src/osSocket.c @@ -150,7 +150,7 @@ int32_t taosGetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *op #if !((defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)) && defined(_MSC_VER)) -uint32_t taosInetAddr(char *ipAddr) { return inet_addr(ipAddr); } +uint32_t taosInetAddr(const char *ipAddr) { return inet_addr(ipAddr); } const char *taosInetNtoa(struct in_addr ipInt) { return inet_ntoa(ipInt); } @@ -240,7 +240,7 @@ int32_t taosSetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *op #ifdef _MSC_VER //#if _MSC_VER >= 1900 -uint32_t taosInetAddr(char *ipAddr) { +uint32_t taosInetAddr(const char *ipAddr) { uint32_t value; int32_t ret = inet_pton(AF_INET, ipAddr, &value); if (ret <= 0) { diff --git a/source/common/src/ttimezone.c b/source/os/src/osTimeZone.c similarity index 95% rename from source/common/src/ttimezone.c rename to source/os/src/osTimeZone.c index c45e39c20d..da40e14389 100644 --- a/source/common/src/ttimezone.c +++ b/source/os/src/osTimeZone.c @@ -15,13 +15,13 @@ #define _DEFAULT_SOURCE #include "os.h" -#include "ulog.h" -#include "tglobal.h" -#include "tconfig.h" -#include "tutil.h" +// #include "ulog.h" +// #include "tglobal.h" +// #include "tutil.h" // TODO refactor to set the tz value through parameter void tsSetTimeZone() { +#if 0 SGlobalCfg *cfg_timezone = taosGetConfigOption("timezone"); if (cfg_timezone != NULL) { uInfo("timezone is set to %s by %s", tsTimezone, tsCfgStatusStr[cfg_timezone->cfgStatus]); @@ -64,4 +64,6 @@ void tsSetTimeZone() { tsDaylight = daylight; uInfo("timezone format changed to %s", tsTimezone); + +#endif } diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c deleted file mode 100644 index 469da11d93..0000000000 --- a/source/util/src/tconfig.c +++ /dev/null @@ -1,507 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "tconfig.h" -#include "tutil.h" -#include "ulog.h" - -SGlobalCfg tsGlobalConfig[TSDB_CFG_MAX_NUM] = {{0}}; -int32_t tsGlobalConfigNum = 0; - -static char *tsGlobalUnit[] = { - " ", - "(%)", - "(GB)", - "(Mb)", - "(byte)", - "(s)", - "(ms)" -}; - -char *tsCfgStatusStr[] = { - "none", - "system default", - "config file", - "taos_options", - "program argument list" -}; - -static void taosReadFloatConfig(SGlobalCfg *cfg, char *input_value) { - float value = (float)atof(input_value); - float *option = (float *)cfg->ptr; - if (value < cfg->minValue || value > cfg->maxValue) { - uError("config option:%s, input value:%s, out of range[%f, %f], use default value:%f", - cfg->option, input_value, cfg->minValue, cfg->maxValue, *option); - } else { - if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_FILE) { - *option = value; - cfg->cfgStatus = TAOS_CFG_CSTATUS_FILE; - } else { - uWarn("config option:%s, input value:%s, is configured by %s, use %f", cfg->option, input_value, - tsCfgStatusStr[cfg->cfgStatus], *option); - } - } -} - -static void taosReadDoubleConfig(SGlobalCfg *cfg, char *input_value) { - double value = atof(input_value); - double *option = (double *)cfg->ptr; - if (value < cfg->minValue || value > cfg->maxValue) { - uError("config option:%s, input value:%s, out of range[%f, %f], use default value:%f", - cfg->option, input_value, cfg->minValue, cfg->maxValue, *option); - } else { - if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_FILE) { - *option = value; - cfg->cfgStatus = TAOS_CFG_CSTATUS_FILE; - } else { - uWarn("config option:%s, input value:%s, is configured by %s, use %f", cfg->option, input_value, - tsCfgStatusStr[cfg->cfgStatus], *option); - } - } -} - -static void taosReadInt32Config(SGlobalCfg *cfg, char *input_value) { - int32_t value = atoi(input_value); - int32_t *option = (int32_t *)cfg->ptr; - if (value < cfg->minValue || value > cfg->maxValue) { - uError("config option:%s, input value:%s, out of range[%f, %f], use default value:%d", - cfg->option, input_value, cfg->minValue, cfg->maxValue, *option); - } else { - if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_FILE) { - *option = value; - cfg->cfgStatus = TAOS_CFG_CSTATUS_FILE; - } else { - uWarn("config option:%s, input value:%s, is configured by %s, use %d", cfg->option, input_value, - tsCfgStatusStr[cfg->cfgStatus], *option); - } - } -} - -static void taosReadInt16Config(SGlobalCfg *cfg, char *input_value) { - int32_t value = atoi(input_value); - int16_t *option = (int16_t *)cfg->ptr; - if (value < cfg->minValue || value > cfg->maxValue) { - uError("config option:%s, input value:%s, out of range[%f, %f], use default value:%d", - cfg->option, input_value, cfg->minValue, cfg->maxValue, *option); - } else { - if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_FILE) { - *option = (int16_t)value; - cfg->cfgStatus = TAOS_CFG_CSTATUS_FILE; - } else { - uWarn("config option:%s, input value:%s, is configured by %s, use %d", cfg->option, input_value, - tsCfgStatusStr[cfg->cfgStatus], *option); - } - } -} - -static void taosReadUInt16Config(SGlobalCfg *cfg, char *input_value) { - int32_t value = atoi(input_value); - uint16_t *option = (uint16_t *)cfg->ptr; - if (value < cfg->minValue || value > cfg->maxValue) { - uError("config option:%s, input value:%s, out of range[%f, %f], use default value:%d", - cfg->option, input_value, cfg->minValue, cfg->maxValue, *option); - } else { - if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_FILE) { - *option = (uint16_t)value; - cfg->cfgStatus = TAOS_CFG_CSTATUS_FILE; - } else { - uWarn("config option:%s, input value:%s, is configured by %s, use %d", cfg->option, input_value, - tsCfgStatusStr[cfg->cfgStatus], *option); - } - } -} - -static void taosReadInt8Config(SGlobalCfg *cfg, char *input_value) { - int32_t value = atoi(input_value); - int8_t *option = (int8_t *)cfg->ptr; - if (value < cfg->minValue || value > cfg->maxValue) { - uError("config option:%s, input value:%s, out of range[%f, %f], use default value:%d", - cfg->option, input_value, cfg->minValue, cfg->maxValue, *option); - } else { - if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_FILE) { - *option = (int8_t)value; - cfg->cfgStatus = TAOS_CFG_CSTATUS_FILE; - } else { - uWarn("config option:%s, input value:%s, is configured by %s, use %d", cfg->option, input_value, - tsCfgStatusStr[cfg->cfgStatus], *option); - } - } -} - -static bool taosReadDirectoryConfig(SGlobalCfg *cfg, char *input_value) { - int length = (int)strlen(input_value); - char *option = (char *)cfg->ptr; - if (length <= 0 || length > cfg->ptrLength) { - uError("config option:%s, input value:%s, length out of range[0, %d], use default value:%s", cfg->option, - input_value, cfg->ptrLength, option); - return false; - } else { - if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_FILE) { - taosExpandDir(input_value, option, cfg->ptrLength); - taosRealPath(option, cfg->ptrLength); - - if (taosMkDir(option) != 0) { - uError("config option:%s, input value:%s, directory not exist, create fail:%s", cfg->option, input_value, - strerror(errno)); - return false; - } - cfg->cfgStatus = TAOS_CFG_CSTATUS_FILE; - } else { - uWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, input_value, - tsCfgStatusStr[cfg->cfgStatus], option); - } - } - - return true; -} - -static void taosReadIpStrConfig(SGlobalCfg *cfg, char *input_value) { - uint32_t value = taosInetAddr(input_value); - char * option = (char *)cfg->ptr; - if (value == INADDR_NONE) { - uError("config option:%s, input value:%s, is not a valid ip address, use default value:%s", - cfg->option, input_value, option); - } else { - if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_FILE) { - strncpy(option, input_value, cfg->ptrLength); - cfg->cfgStatus = TAOS_CFG_CSTATUS_FILE; - } else { - uWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, input_value, - tsCfgStatusStr[cfg->cfgStatus], option); - } - } -} - -static void taosReadStringConfig(SGlobalCfg *cfg, char *input_value) { - int length = (int) strlen(input_value); - char *option = (char *)cfg->ptr; - if (length <= 0 || length > cfg->ptrLength) { - uError("config option:%s, input value:%s, length out of range[0, %d], use default value:%s", - cfg->option, input_value, cfg->ptrLength, option); - } else { - if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_FILE) { - strncpy(option, input_value, cfg->ptrLength); - cfg->cfgStatus = TAOS_CFG_CSTATUS_FILE; - } else { - uWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, input_value, - tsCfgStatusStr[cfg->cfgStatus], option); - } - } -} - -static void taosReadLogOption(char *option, char *value) { - for (int i = 0; i < tsGlobalConfigNum; ++i) { - SGlobalCfg *cfg = tsGlobalConfig + i; - if (!(cfg->cfgType & TSDB_CFG_CTYPE_B_CONFIG) || !(cfg->cfgType & TSDB_CFG_CTYPE_B_LOG)) continue; - if (strcasecmp(cfg->option, option) != 0) continue; - - switch (cfg->valType) { - case TAOS_CFG_VTYPE_INT32: - taosReadInt32Config(cfg, value); - // if (strcasecmp(cfg->option, "debugFlag") == 0) { - // taosSetAllDebugFlag(); - // } - break; - case TAOS_CFG_VTYPE_DIRECTORY: - taosReadDirectoryConfig(cfg, value); - break; - default: - break; - } - break; - } -} - -SGlobalCfg *taosGetConfigOption(const char *option) { - for (int i = 0; i < tsGlobalConfigNum; ++i) { - SGlobalCfg *cfg = tsGlobalConfig + i; - if (strcasecmp(cfg->option, option) != 0) continue; - return cfg; - } - return NULL; -} - -static void taosReadConfigOption(const char *option, char *value, char *value2, char *value3) { - for (int i = 0; i < tsGlobalConfigNum; ++i) { - SGlobalCfg *cfg = tsGlobalConfig + i; - if (!(cfg->cfgType & TSDB_CFG_CTYPE_B_CONFIG)) continue; - if (strcasecmp(cfg->option, option) != 0) continue; - - switch (cfg->valType) { - case TAOS_CFG_VTYPE_INT8: - taosReadInt8Config(cfg, value); - break; - case TAOS_CFG_VTYPE_INT16: - taosReadInt16Config(cfg, value); - break; - case TAOS_CFG_VTYPE_INT32: - taosReadInt32Config(cfg, value); - break; - case TAOS_CFG_VTYPE_UINT16: - taosReadUInt16Config(cfg, value); - break; - case TAOS_CFG_VTYPE_FLOAT: - taosReadFloatConfig(cfg, value); - break; - case TAOS_CFG_VTYPE_DOUBLE: - taosReadDoubleConfig(cfg, value); - break; - case TAOS_CFG_VTYPE_STRING: - taosReadStringConfig(cfg, value); - break; - case TAOS_CFG_VTYPE_IPSTR: - taosReadIpStrConfig(cfg, value); - break; - case TAOS_CFG_VTYPE_DIRECTORY: - taosReadDirectoryConfig(cfg, value); - break; - case TAOS_CFG_VTYPE_DATA_DIRCTORY: - if (taosReadDirectoryConfig(cfg, value)) { - // taosReadDataDirCfg(value, value2, value3); - } - break; - default: - uError("config option:%s, input value:%s, can't be recognized", option, value); - break; - } - break; - } -} - -void taosAddConfigOption(SGlobalCfg cfg) { - tsGlobalConfig[tsGlobalConfigNum++] = cfg; -} - -void taosReadGlobalLogCfg() { - FILE * fp; - char * line, *option, *value; - int olen, vlen; - char fileName[PATH_MAX] = {0}; - - taosExpandDir(configDir, configDir, PATH_MAX); - taosReadLogOption("logDir", tsLogDir); - - sprintf(fileName, "%s/taos.cfg", configDir); - fp = fopen(fileName, "r"); - if (fp == NULL) { - printf("\nconfig file:%s not found, all variables are set to default\n", fileName); - return; - } - - ssize_t _bytes = 0; - size_t len = 1024; - line = calloc(1, len); - - while (!feof(fp)) { - memset(line, 0, len); - - option = value = NULL; - olen = vlen = 0; - - _bytes = tgetline(&line, &len, fp); - if (_bytes < 0) - { - break; - } - - line[len - 1] = 0; - - paGetToken(line, &option, &olen); - if (olen == 0) continue; - option[olen] = 0; - - paGetToken(option + olen + 1, &value, &vlen); - if (vlen == 0) continue; - value[vlen] = 0; - - taosReadLogOption(option, value); - } - - tfree(line); - fclose(fp); -} - -int32_t taosReadCfgFromFile() { - char * line, *option, *value, *value2, *value3; - int olen, vlen, vlen2, vlen3; - char fileName[PATH_MAX] = {0}; - - sprintf(fileName, "%s/taos.cfg", configDir); - FILE *fp = fopen(fileName, "r"); - if (fp == NULL) { - fp = fopen(configDir, "r"); - if (fp == NULL) { - return -1; - } - } - - ssize_t _bytes = 0; - size_t len = 1024; - line = calloc(1, len); - - while (!feof(fp)) { - memset(line, 0, len); - - option = value = value2 = value3 = NULL; - olen = vlen = vlen2 = vlen3 = 0; - - _bytes = tgetline(&line, &len, fp); - if (_bytes < 0) - { - break; - } - - line[len - 1] = 0; - - paGetToken(line, &option, &olen); - if (olen == 0) continue; - option[olen] = 0; - - paGetToken(option + olen + 1, &value, &vlen); - if (vlen == 0) continue; - value[vlen] = 0; - - paGetToken(value + vlen + 1, &value2, &vlen2); - if (vlen2 != 0) { - value2[vlen2] = 0; - paGetToken(value2 + vlen2 + 1, &value3, &vlen3); - if (vlen3 != 0) value3[vlen3] = 0; - } - - taosReadConfigOption(option, value, value2, value3); - } - - fclose(fp); - - tfree(line); - - // if (debugFlag & DEBUG_TRACE || debugFlag & DEBUG_DEBUG || debugFlag & DEBUG_DUMP) { - // taosSetAllDebugFlag(); - // } - - return 0; -} - -void taosPrintCfg() { - uInfo(" taos config & system info:"); - uInfo("=================================="); - - for (int i = 0; i < tsGlobalConfigNum; ++i) { - SGlobalCfg *cfg = tsGlobalConfig + i; - if (tscEmbeddedInUtil == 0 && !(cfg->cfgType & TSDB_CFG_CTYPE_B_CLIENT)) continue; - if (cfg->cfgType & TSDB_CFG_CTYPE_B_NOT_PRINT) continue; - - int optionLen = (int)strlen(cfg->option); - int blankLen = TSDB_CFG_PRINT_LEN - optionLen; - blankLen = blankLen < 0 ? 0 : blankLen; - - char blank[TSDB_CFG_PRINT_LEN]; - memset(blank, ' ', TSDB_CFG_PRINT_LEN); - blank[blankLen] = 0; - - switch (cfg->valType) { - case TAOS_CFG_VTYPE_INT8: - uInfo(" %s:%s%d%s", cfg->option, blank, *((int8_t *)cfg->ptr), tsGlobalUnit[cfg->unitType]); - break; - case TAOS_CFG_VTYPE_INT16: - uInfo(" %s:%s%d%s", cfg->option, blank, *((int16_t *)cfg->ptr), tsGlobalUnit[cfg->unitType]); - break; - case TAOS_CFG_VTYPE_INT32: - uInfo(" %s:%s%d%s", cfg->option, blank, *((int32_t *)cfg->ptr), tsGlobalUnit[cfg->unitType]); - break; - case TAOS_CFG_VTYPE_UINT16: - uInfo(" %s:%s%d%s", cfg->option, blank, *((uint16_t *)cfg->ptr), tsGlobalUnit[cfg->unitType]); - break; - case TAOS_CFG_VTYPE_FLOAT: - uInfo(" %s:%s%f%s", cfg->option, blank, *((float *)cfg->ptr), tsGlobalUnit[cfg->unitType]); - break; - case TAOS_CFG_VTYPE_DOUBLE: - uInfo(" %s:%s%f%s", cfg->option, blank, *((double *)cfg->ptr), tsGlobalUnit[cfg->unitType]); - break; - case TAOS_CFG_VTYPE_STRING: - case TAOS_CFG_VTYPE_IPSTR: - case TAOS_CFG_VTYPE_DIRECTORY: - uInfo(" %s:%s%s%s", cfg->option, blank, (char *)cfg->ptr, tsGlobalUnit[cfg->unitType]); - break; - default: - break; - } - } - - taosPrintOsInfo(); - uInfo("=================================="); -} - -static void taosDumpCfg(SGlobalCfg *cfg) { - int optionLen = (int)strlen(cfg->option); - int blankLen = TSDB_CFG_PRINT_LEN - optionLen; - blankLen = blankLen < 0 ? 0 : blankLen; - - char blank[TSDB_CFG_PRINT_LEN]; - memset(blank, ' ', TSDB_CFG_PRINT_LEN); - blank[blankLen] = 0; - - switch (cfg->valType) { - case TAOS_CFG_VTYPE_INT8: - printf(" %s:%s%d%s\n", cfg->option, blank, *((int8_t *)cfg->ptr), tsGlobalUnit[cfg->unitType]); - break; - case TAOS_CFG_VTYPE_INT16: - printf(" %s:%s%d%s\n", cfg->option, blank, *((int16_t *)cfg->ptr), tsGlobalUnit[cfg->unitType]); - break; - case TAOS_CFG_VTYPE_INT32: - printf(" %s:%s%d%s\n", cfg->option, blank, *((int32_t *)cfg->ptr), tsGlobalUnit[cfg->unitType]); - break; - case TAOS_CFG_VTYPE_UINT16: - printf(" %s:%s%d%s\n", cfg->option, blank, *((uint16_t *)cfg->ptr), tsGlobalUnit[cfg->unitType]); - break; - case TAOS_CFG_VTYPE_FLOAT: - printf(" %s:%s%f%s\n", cfg->option, blank, *((float *)cfg->ptr), tsGlobalUnit[cfg->unitType]); - break; - case TAOS_CFG_VTYPE_STRING: - case TAOS_CFG_VTYPE_IPSTR: - case TAOS_CFG_VTYPE_DIRECTORY: - printf(" %s:%s%s%s\n", cfg->option, blank, (char *)cfg->ptr, tsGlobalUnit[cfg->unitType]); - break; - default: - break; - } -} - -void taosDumpGlobalCfg() { - printf("taos global config:\n"); - printf("==================================\n"); - for (int i = 0; i < tsGlobalConfigNum; ++i) { - SGlobalCfg *cfg = tsGlobalConfig + i; - if (tscEmbeddedInUtil == 0 && !(cfg->cfgType & TSDB_CFG_CTYPE_B_CLIENT)) continue; - if (cfg->cfgType & TSDB_CFG_CTYPE_B_NOT_PRINT) continue; - if (!(cfg->cfgType & TSDB_CFG_CTYPE_B_SHOW)) continue; - - taosDumpCfg(cfg); - } - - printf("\ntaos local config:\n"); - printf("==================================\n"); - - for (int i = 0; i < tsGlobalConfigNum; ++i) { - SGlobalCfg *cfg = tsGlobalConfig + i; - if (tscEmbeddedInUtil == 0 && !(cfg->cfgType & TSDB_CFG_CTYPE_B_CLIENT)) continue; - if (cfg->cfgType & TSDB_CFG_CTYPE_B_NOT_PRINT) continue; - if (cfg->cfgType & TSDB_CFG_CTYPE_B_SHOW) continue; - - taosDumpCfg(cfg); - } -} diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 2b53a769ff..e6bef5f8b9 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -81,6 +81,8 @@ TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_MSG, "Invalid message") TAOS_DEFINE_ERROR(TSDB_CODE_MSG_NOT_PROCESSED, "Message not processed") TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_PARA, "Invalid parameters") TAOS_DEFINE_ERROR(TSDB_CODE_REPEAT_INIT, "Repeat initialization") +TAOS_DEFINE_ERROR(TSDB_CODE_CFG_NOT_FOUND, "Config not found") +TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_CFG, "Invalid config option") TAOS_DEFINE_ERROR(TSDB_CODE_REF_NO_MEMORY, "Ref out of memory") TAOS_DEFINE_ERROR(TSDB_CODE_REF_FULL, "too many Ref Objs") diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index 12aa77214f..f5d02df80d 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -70,29 +70,26 @@ typedef struct { int8_t tscEmbeddedInUtil = 0; int32_t tsLogKeepDays = 0; -int8_t tsAsyncLog = 1; -float tsTotalLogDirGB = 0; -float tsAvailLogDirGB = 0; -float tsMinimalLogDirGB = 1.0f; +bool tsAsyncLog = true; +bool tsLogInited = false; int64_t asyncLogLostLines = 0; int32_t writeInterval = DEFAULT_LOG_INTERVAL; // log int32_t tsNumOfLogLines = 10000000; -int32_t mDebugFlag = 131; int32_t dDebugFlag = 135; int32_t vDebugFlag = 135; +int32_t mDebugFlag = 131; int32_t cDebugFlag = 131; int32_t jniDebugFlag = 131; -int32_t qDebugFlag = 131; -int32_t rpcDebugFlag = 131; +int32_t tmrDebugFlag = 131; int32_t uDebugFlag = 131; -int32_t debugFlag = 0; -int32_t sDebugFlag = 135; +int32_t rpcDebugFlag = 131; +int32_t qDebugFlag = 131; int32_t wDebugFlag = 135; +int32_t sDebugFlag = 135; int32_t tsdbDebugFlag = 131; int32_t tqDebugFlag = 135; -int32_t cqDebugFlag = 131; int32_t fsDebugFlag = 135; int64_t dbgEmptyW = 0; @@ -120,11 +117,18 @@ static int32_t taosStartLog() { return 0; } -int32_t taosInitLog(char *logName, int numOfLogLines, int maxFiles) { +int32_t taosInitLog(const char *logName, int maxFiles) { + if (tsLogInited) return 0; + taosUpdateLogSpace(); + + char fullName[PATH_MAX] = {0}; + snprintf(fullName, PATH_MAX, "%s" TD_DIRSEP "%s", tsLogDir, logName); + tsLogObj.logHandle = taosLogBuffNew(TSDB_DEFAULT_LOG_BUF_SIZE); if (tsLogObj.logHandle == NULL) return -1; - if (taosOpenLogFile(logName, numOfLogLines, maxFiles) < 0) return -1; + if (taosOpenLogFile(fullName, tsNumOfLogLines, maxFiles) < 0) return -1; if (taosStartLog() < 0) return -1; + tsLogInited = true; return 0; } @@ -217,7 +221,7 @@ static void *taosThreadToOpenNewFile(void *param) { uInfo(" new log file:%d is opened", tsLogObj.flag); uInfo("=================================="); - taosPrintCfg(); + // taosPrintCfg(); taosKeepOldLog(keepName); return NULL; @@ -377,12 +381,7 @@ static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum) { } void taosPrintLog(const char *flags, int32_t dflag, const char *format, ...) { - if (tsTotalLogDirGB != 0 && tsAvailLogDirGB < tsMinimalLogDirGB) { - printf("server disk:%s space remain %.3f GB, total %.1f GB, stop print log.\n", tsLogDir, tsAvailLogDirGB, - tsTotalLogDirGB); - fflush(stdout); - return; - } + if (!taosLogSpaceAvailable()) return; va_list argpointer; char buffer[MAX_LOGLINE_BUFFER_SIZE] = {0}; @@ -437,12 +436,7 @@ void taosPrintLog(const char *flags, int32_t dflag, const char *format, ...) { } void taosDumpData(unsigned char *msg, int32_t len) { - if (tsTotalLogDirGB != 0 && tsAvailLogDirGB < tsMinimalLogDirGB) { - printf("server disk:%s space remain %.3f GB, total %.1f GB, stop dump log.\n", tsLogDir, tsAvailLogDirGB, - tsTotalLogDirGB); - fflush(stdout); - return; - } + if (!taosLogSpaceAvailable()) return; char temp[256]; int32_t i, pos = 0, c = 0; @@ -465,12 +459,7 @@ void taosDumpData(unsigned char *msg, int32_t len) { } void taosPrintLongString(const char *flags, int32_t dflag, const char *format, ...) { - if (tsTotalLogDirGB != 0 && tsAvailLogDirGB < tsMinimalLogDirGB) { - printf("server disk:%s space remain %.3f GB, total %.1f GB, stop write log.\n", tsLogDir, tsAvailLogDirGB, - tsTotalLogDirGB); - fflush(stdout); - return; - } + if (!taosLogSpaceAvailable()) return; va_list argpointer; char buffer[MAX_LOGLINE_DUMP_BUFFER_SIZE]; @@ -773,3 +762,23 @@ void taosPrintOsInfo() { uInfo(" os version: %s", info.version); uInfo(" os machine: %s", info.machine); } + +void taosSetAllDebugFlag(int32_t flag) { + if (!(flag & DEBUG_TRACE || flag & DEBUG_DEBUG || flag & DEBUG_DUMP)) return; + + dDebugFlag = flag; + vDebugFlag = flag; + mDebugFlag = flag; + cDebugFlag = flag; + jniDebugFlag = flag; + uDebugFlag = flag; + rpcDebugFlag = flag; + qDebugFlag = flag; + wDebugFlag = flag; + sDebugFlag = flag; + tsdbDebugFlag = flag; + tqDebugFlag = flag; + fsDebugFlag = flag; + + uInfo("all debug flag are set to %d", flag); +} \ No newline at end of file diff --git a/source/util/src/ttimer.c b/source/util/src/ttimer.c index 65101a5e07..ca0c6e9a24 100644 --- a/source/util/src/ttimer.c +++ b/source/util/src/ttimer.c @@ -111,7 +111,6 @@ typedef struct time_wheel_t { tmr_obj_t** slots; } time_wheel_t; -int32_t tmrDebugFlag = 131; uint32_t tsMaxTmrCtrl = 512; static pthread_once_t tmrModuleInit = PTHREAD_ONCE_INIT; diff --git a/source/util/test/trefTest.c b/source/util/test/trefTest.c index 54aa6a5f34..1484ed2182 100644 --- a/source/util/test/trefTest.c +++ b/source/util/test/trefTest.c @@ -158,7 +158,7 @@ int main(int argc, char *argv[]) { } } - taosInitLog("tref.log", 5000000, 10); + taosInitLog("tref.log", 10); SRefSpace *pSpaceList = (SRefSpace *) calloc(sizeof(SRefSpace), threads); pthread_t *pThreadList = (pthread_t *) calloc(sizeof(pthread_t), threads); diff --git a/tests b/tests index 4da4bb00c5..498e17e225 160000 --- a/tests +++ b/tests @@ -1 +1 @@ -Subproject commit 4da4bb00c5615f0b1bea04340abef5d9d15a96b4 +Subproject commit 498e17e225c454f2b45ff16eaa6d3eb595b107cf diff --git a/tools/shell/src/shellLinux.c b/tools/shell/src/shellLinux.c index 81d8c0a35b..89b477f08a 100644 --- a/tools/shell/src/shellLinux.c +++ b/tools/shell/src/shellLinux.c @@ -20,6 +20,7 @@ #include "shellCommand.h" #include "tkey.h" #include "ulog.h" +#include "version.h" #include #include diff --git a/tools/shell/src/shellMain.c b/tools/shell/src/shellMain.c index 607dc79257..8a1763c4fc 100644 --- a/tools/shell/src/shellMain.c +++ b/tools/shell/src/shellMain.c @@ -15,7 +15,6 @@ #include "os.h" #include "shell.h" -#include "tconfig.h" #include "tglobal.h" pthread_t pid;