xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/3.0' into feature/qnode

This commit is contained in:
dapan1121 2022-02-24 10:10:02 +08:00
parent 0526e5a675 3d02c75895
commit 38fd1a657c
103 changed files with 3012 additions and 2273 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
example/src/tmq.c
View File

@ -66,7 +66,7 @@ int32_t init_env() {
}
int32_t create_topic() {
printf("create topic");
printf("create topic\n");
TAOS_RES* pRes;
TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
if (pConn == NULL) {
@ -91,6 +91,10 @@ int32_t create_topic() {
return 0;
}
void tmq_commit_cb_print(tmq_t* tmq, tmq_resp_err_t resp, tmq_topic_vgroup_list_t* offsets, void* param) {
printf("commit %d\n", resp);
}
tmq_t* build_consumer() {
TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
assert(pConn != NULL);
@ -103,6 +107,7 @@ tmq_t* build_consumer() {
tmq_conf_t* conf = tmq_conf_new();
tmq_conf_set(conf, "group.id", "tg2");
tmq_conf_set_offset_commit_cb(conf, tmq_commit_cb_print);
tmq_t* tmq = tmq_consumer_new(pConn, conf, NULL, 0);
return tmq;
}
@ -144,7 +149,7 @@ void basic_consume_loop(tmq_t* tmq, tmq_list_t* topics) {
}
void sync_consume_loop(tmq_t* tmq, tmq_list_t* topics) {
static const int MIN_COMMIT_COUNT = 1000;
static const int MIN_COMMIT_COUNT = 1;
int msg_count = 0;
tmq_resp_err_t err;
@ -214,6 +219,6 @@ int main(int argc, char* argv[]) {
tmq_t* tmq = build_consumer();
tmq_list_t* topic_list = build_topic_list();
/*perf_loop(tmq, topic_list);*/
basic_consume_loop(tmq, topic_list);
/*sync_consume_loop(tmq, topic_list);*/
/*basic_consume_loop(tmq, topic_list);*/
sync_consume_loop(tmq, topic_list);
}

5
include/client/taos.h
View File

@ -198,8 +198,8 @@ DLL_EXPORT TAOS_RES *taos_schemaless_insert(TAOS *taos, char *lines[], int numLi
/* --------------------------TMQ INTERFACE------------------------------- */
enum tmq_resp_err_t {
TMQ_RESP_ERR__FAIL = -1,
TMQ_RESP_ERR__SUCCESS = 0,
TMQ_RESP_ERR__FAIL = 1,
};
typedef enum tmq_resp_err_t tmq_resp_err_t;
@ -226,7 +226,7 @@ DLL_EXPORT const char *tmq_err2str(tmq_resp_err_t);
DLL_EXPORT tmq_resp_err_t tmq_subscribe(tmq_t *tmq, tmq_list_t *topic_list);
#if 0
DLL_EXPORT tmq_resp_err_t tmq_unsubscribe(tmq_t* tmq);
DLL_EXPORT tmq_resp_err_t tmq_subscription(tmq_t* tmq, tmq_topic_vgroup_list_t** topics);
DLL_EXPORT tmq_resp_err_t tmq_subscription(tmq_t* tmq, tmq_list_t** topics);
#endif
DLL_EXPORT tmq_message_t *tmq_consumer_poll(tmq_t *tmq, int64_t blocking_time);
DLL_EXPORT tmq_resp_err_t tmq_consumer_close(tmq_t *tmq);
@ -238,6 +238,7 @@ DLL_EXPORT tmq_resp_err_t tmq_commit(tmq_t *tmq, const tmq_topic_vgroup_list_t *
#if 0
DLL_EXPORT tmq_resp_err_t tmq_commit_message(tmq_t* tmq, const tmq_message_t* tmqmessage, int32_t async);
#endif
DLL_EXPORT tmq_resp_err_t tmq_seek(tmq_t *tmq, const tmq_topic_vgroup_t *offset);
/* ----------------------TMQ CONFIGURATION INTERFACE---------------------- */
enum tmq_conf_res_t {

45
include/common/common.h
View File

@ -16,7 +16,6 @@
#ifndef TDENGINE_COMMON_H
#define TDENGINE_COMMON_H
#ifdef __cplusplus
extern "C" {
#endif
@ -43,14 +42,16 @@ extern "C" {
// int16_t bytes;
// } SSchema;
#define TMQ_REQ_TYPE_COMMIT_ONLY 0
#define TMQ_REQ_TYPE_CONSUME_ONLY 1
#define TMQ_REQ_TYPE_CONSUME_AND_COMMIT 2
enum {
TMQ_CONF__RESET_OFFSET__LATEST = -1,
TMQ_CONF__RESET_OFFSET__EARLIEAST = -2,
TMQ_CONF__RESET_OFFSET__NONE = -3,
};
typedef struct {
uint32_t numOfTables;
SArray *pGroupList;
SHashObj *map; // speedup acquire the tableQueryInfo by table uid
SArray* pGroupList;
SHashObj* map; // speedup acquire the tableQueryInfo by table uid
} STableGroupInfo;
typedef struct SColumnDataAgg {
@ -79,14 +80,14 @@ typedef struct SConstantItem {
// info.numOfCols = taosArrayGetSize(pDataBlock) + taosArrayGetSize(pConstantList);
typedef struct SSDataBlock {
SColumnDataAgg *pBlockAgg;
SArray *pDataBlock; // SArray<SColumnInfoData>
SArray *pConstantList; // SArray<SConstantItem>, it is a constant/tags value of the corresponding result value.
SDataBlockInfo info;
SColumnDataAgg* pBlockAgg;
SArray* pDataBlock; // SArray<SColumnInfoData>
SArray* pConstantList; // SArray<SConstantItem>, it is a constant/tags value of the corresponding result value.
SDataBlockInfo info;
} SSDataBlock;
typedef struct SVarColAttr {
int32_t *offset; // start position for each entry in the list
int32_t* offset; // start position for each entry in the list
uint32_t length; // used buffer size that contain the valid data
uint32_t allocLen; // allocated buffer size
} SVarColAttr;
@ -94,11 +95,11 @@ typedef struct SVarColAttr {
// pBlockAgg->numOfNull == info.rows, all data are null
// pBlockAgg->numOfNull == 0, no data are null.
typedef struct SColumnInfoData {
SColumnInfo info; // TODO filter info needs to be removed
bool hasNull;// if current column data has null value.
char *pData; // the corresponding block data in memory
SColumnInfo info; // TODO filter info needs to be removed
bool hasNull; // if current column data has null value.
char* pData; // the corresponding block data in memory
union {
char *nullbitmap; // bitmap, one bit for each item in the list
char* nullbitmap; // bitmap, one bit for each item in the list
SVarColAttr varmeta;
};
} SColumnInfoData;
@ -149,7 +150,6 @@ static FORCE_INLINE int32_t tEncodeSMqConsumeRsp(void** buf, const SMqConsumeRsp
int32_t tlen = 0;
int32_t sz = 0;
tlen += taosEncodeFixedI64(buf, pRsp->consumerId);
tlen += taosEncodeFixedI64(buf, pRsp->committedOffset);
tlen += taosEncodeFixedI64(buf, pRsp->reqOffset);
tlen += taosEncodeFixedI64(buf, pRsp->rspOffset);
tlen += taosEncodeFixedI32(buf, pRsp->skipLogNum);
@ -170,7 +170,6 @@ static FORCE_INLINE int32_t tEncodeSMqConsumeRsp(void** buf, const SMqConsumeRsp
static FORCE_INLINE void* tDecodeSMqConsumeRsp(void* buf, SMqConsumeRsp* pRsp) {
int32_t sz;
buf = taosDecodeFixedI64(buf, &pRsp->consumerId);
buf = taosDecodeFixedI64(buf, &pRsp->committedOffset);
buf = taosDecodeFixedI64(buf, &pRsp->reqOffset);
buf = taosDecodeFixedI64(buf, &pRsp->rspOffset);
buf = taosDecodeFixedI32(buf, &pRsp->skipLogNum);
@ -250,11 +249,11 @@ typedef struct SSqlExpr {
char token[TSDB_COL_NAME_LEN]; // original token
SSchema resSchema;
int32_t numOfCols;
SColumn* pColumns; // data columns that are required by query
int32_t interBytes; // inter result buffer size
int16_t numOfParams; // argument value of each function
SVariant param[3]; // parameters are not more than 3
int32_t numOfCols;
SColumn* pColumns; // data columns that are required by query
int32_t interBytes; // inter result buffer size
int16_t numOfParams; // argument value of each function
SVariant param[3]; // parameters are not more than 3
} SSqlExpr;
typedef struct SExprInfo {
@ -271,7 +270,7 @@ typedef struct SSessionWindow {
SColumn col;
} SSessionWindow;
#define QUERY_ASC_FORWARD_STEP 1
#define QUERY_ASC_FORWARD_STEP 1
#define QUERY_DESC_FORWARD_STEP -1
#define GET_FORWARD_DIRECTION_FACTOR(ord) (((ord) == TSDB_ORDER_ASC) ? QUERY_ASC_FORWARD_STEP : QUERY_DESC_FORWARD_STEP)

2
include/common/tep.h
View File

@ -23,7 +23,7 @@ typedef struct SBlockOrderInfo {
// bool hasNull;
} SBlockOrderInfo;
int taosGetFqdnPortFromEp(const char *ep, SEp *pEp);
int taosGetFqdnPortFromEp(const char *ep, uint16_t defaultPort, SEp *pEp);
void addEpIntoEpSet(SEpSet *pEpSet, const char *fqdn, uint16_t port);
bool isEpsetEqual(const SEpSet *s1, const SEpSet *s2);

42
include/common/tglobal.h
View File

@ -21,31 +21,9 @@ extern "C" {
#endif
#include "tdef.h"
#include "tcfg.h"
// cluster
extern char tsFirst[];
extern char tsSecond[];
extern char tsLocalFqdn[];
extern char tsLocalEp[];
extern uint16_t tsServerPort;
extern int32_t tsStatusInterval;
extern int8_t tsEnableTelemetryReporting;
extern int32_t tsNumOfSupportVnodes;
// common
extern int tsRpcTimer;
extern int tsRpcMaxTime;
extern int tsRpcForceTcp; // all commands go to tcp protocol if this is enabled
extern int32_t tsMaxConnections;
extern int32_t tsMaxShellConns;
extern int32_t tsShellActivityTimer;
extern uint32_t tsMaxTmrCtrl;
extern float tsNumOfThreadsPerCore;
extern int32_t tsNumOfCommitThreads;
extern float tsRatioOfQueryCores;
extern int8_t tsDaylight;
extern int8_t tsEnableCoreFile;
extern int32_t tsCompressMsgSize;
extern int32_t tsCompressColData;
extern int32_t tsMaxNumOfDistinctResults;
@ -78,24 +56,9 @@ extern int32_t tsProjectExecInterval;
extern int64_t tsMaxRetentWindow;
// system info
extern float tsTotalLogDirGB;
extern float tsTotalTmpDirGB;
extern float tsTotalDataDirGB;
extern float tsAvailLogDirGB;
extern float tsAvailTmpDirectorySpace;
extern float tsAvailDataDirGB;
extern float tsUsedDataDirGB;
extern float tsMinimalLogDirGB;
extern float tsReservedTmpDirectorySpace;
extern float tsMinimalDataDirGB;
extern uint32_t tsVersion;
// build info
extern char version[];
extern char compatible_version[];
extern char gitinfo[];
extern char gitinfoOfInternal[];
extern char buildinfo[];
// lossy
extern char tsLossyColumns[];
@ -105,13 +68,10 @@ extern uint32_t tsMaxRange;
extern uint32_t tsCurRange;
extern char tsCompressor[];
extern int32_t tsDiskCfgNum;
extern SDiskCfg tsDiskCfg[];
#define NEEDTO_COMPRESSS_MSG(size) (tsCompressMsgSize != -1 && (size) > tsCompressMsgSize)
void taosInitGlobalCfg();
int32_t taosCheckAndPrintCfg();
int32_t taosCfgDynamicOptions(char *msg);
bool taosCheckBalanceCfgOptions(const char *option, int32_t *vnodeId, int32_t *dnodeId);
void taosAddDataDir(int index, char *v1, int level, int primary);

79
include/common/tmsg.h
View File

@ -422,8 +422,8 @@ typedef struct {
} SColumnInfo;
typedef struct {
uint64_t uid;
TSKEY key; // last accessed ts, for subscription
int64_t uid;
TSKEY key; // last accessed ts, for subscription
} STableIdInfo;
typedef struct STimeWindow {
@ -554,8 +554,8 @@ int32_t tSerializeSDropDbReq(void* buf, int32_t bufLen, SDropDbReq* pReq);
int32_t tDeserializeSDropDbReq(void* buf, int32_t bufLen, SDropDbReq* pReq);
typedef struct {
char db[TSDB_DB_FNAME_LEN];
uint64_t uid;
char db[TSDB_DB_FNAME_LEN];
int64_t uid;
} SDropDbRsp;
int32_t tSerializeSDropDbRsp(void* buf, int32_t bufLen, SDropDbRsp* pRsp);
@ -570,12 +570,12 @@ int32_t tSerializeSUseDbReq(void* buf, int32_t bufLen, SUseDbReq* pReq);
int32_t tDeserializeSUseDbReq(void* buf, int32_t bufLen, SUseDbReq* pReq);
typedef struct {
char db[TSDB_DB_FNAME_LEN];
uint64_t uid;
int32_t vgVersion;
int32_t vgNum;
int8_t hashMethod;
SArray* pVgroupInfos; // Array of SVgroupInfo
char db[TSDB_DB_FNAME_LEN];
int64_t uid;
int32_t vgVersion;
int32_t vgNum;
int8_t hashMethod;
SArray* pVgroupInfos; // Array of SVgroupInfo
} SUseDbRsp;
int32_t tSerializeSUseDbRsp(void* buf, int32_t bufLen, SUseDbRsp* pRsp);
@ -725,7 +725,7 @@ typedef struct {
int32_t vgId;
int32_t dnodeId;
char db[TSDB_DB_FNAME_LEN];
uint64_t dbUid;
int64_t dbUid;
int32_t vgVersion;
int32_t cacheBlockSize;
int32_t totalBlocks;
@ -753,10 +753,10 @@ int32_t tSerializeSCreateVnodeReq(void* buf, int32_t bufLen, SCreateVnodeReq* pR
int32_t tDeserializeSCreateVnodeReq(void* buf, int32_t bufLen, SCreateVnodeReq* pReq);
typedef struct {
int32_t vgId;
int32_t dnodeId;
uint64_t dbUid;
char db[TSDB_DB_FNAME_LEN];
int32_t vgId;
int32_t dnodeId;
int64_t dbUid;
char db[TSDB_DB_FNAME_LEN];
} SDropVnodeReq, SSyncVnodeReq, SCompactVnodeReq;
int32_t tSerializeSDropVnodeReq(void* buf, int32_t bufLen, SDropVnodeReq* pReq);
@ -796,7 +796,7 @@ typedef struct {
char tbName[TSDB_TABLE_NAME_LEN];
char stbName[TSDB_TABLE_NAME_LEN];
char dbFName[TSDB_DB_FNAME_LEN];
uint64_t dbId;
int64_t dbId;
int32_t numOfTags;
int32_t numOfColumns;
int8_t precision;
@ -1268,7 +1268,7 @@ typedef struct {
typedef struct {
SMsgHead head;
char name[TSDB_TABLE_FNAME_LEN];
uint64_t tuid;
int64_t tuid;
int32_t sverson;
int32_t execLen;
char* executor;
@ -1279,11 +1279,11 @@ typedef struct {
typedef struct {
SMsgHead head;
char name[TSDB_TABLE_FNAME_LEN];
uint64_t tuid;
int64_t tuid;
} SDDropTopicReq;
typedef struct SVCreateTbReq {
uint64_t ver; // use a general definition
int64_t ver; // use a general definition
char* name;
uint32_t ttl;
uint32_t keep;
@ -1314,8 +1314,8 @@ int32_t tSerializeSVCreateTbReq(void** buf, SVCreateTbReq* pReq);
void* tDeserializeSVCreateTbReq(void* buf, SVCreateTbReq* pReq);
typedef struct {
uint64_t ver; // use a general definition
SArray* pArray;
int64_t ver; // use a general definition
SArray* pArray;
} SVCreateTbBatchReq;
typedef struct {
@ -1325,7 +1325,7 @@ int32_t tSerializeSVCreateTbBatchReq(void** buf, SVCreateTbBatchReq* pReq);
void* tDeserializeSVCreateTbBatchReq(void* buf, SVCreateTbBatchReq* pReq);
typedef struct {
uint64_t ver;
int64_t ver;
char* name;
uint8_t type;
tb_uid_t suid;
@ -1760,35 +1760,19 @@ typedef struct {
char cgroup[TSDB_CONSUMER_GROUP_LEN];
} SMqOffset;
typedef struct {
int32_t vgId;
SArray* offsets; // SArray<SMqOffset>
} SMqVgOffsets;
typedef struct {
int32_t num;
SMqOffset* offsets;
} SMqCMResetOffsetReq;
} SMqCMCommitOffsetReq;
typedef struct {
int32_t reserved;
} SMqCMResetOffsetRsp;
typedef struct {
int64_t leftForVer;
SMqVgOffsets offsets;
} SMqMVResetOffsetReq;
typedef struct {
int32_t reserved;
} SMqMVResetOffsetRsp;
} SMqCMCommitOffsetRsp;
int32_t tEncodeSMqOffset(SCoder* encoder, const SMqOffset* pOffset);
int32_t tDecodeSMqOffset(SCoder* decoder, SMqOffset* pOffset);
int32_t tEncodeSMqCMResetOffsetReq(SCoder* encoder, const SMqCMResetOffsetReq* pReq);
int32_t tDecodeSMqCMResetOffsetReq(SCoder* decoder, SMqCMResetOffsetReq* pReq);
int32_t tEncodeSMqMVResetOffsetReq(SCoder* encoder, const SMqMVResetOffsetReq* pReq);
int32_t tDecodeSMqMVResetOffsetReq(SCoder* decoder, SMqMVResetOffsetReq* pReq);
int32_t tEncodeSMqCMCommitOffsetReq(SCoder* encoder, const SMqCMCommitOffsetReq* pReq);
int32_t tDecodeSMqCMCommitOffsetReq(SCoder* decoder, SMqCMCommitOffsetReq* pReq);
typedef struct {
uint32_t nCols;
@ -1870,7 +1854,6 @@ typedef struct {
typedef struct {
int64_t consumerId;
SSchemaWrapper* schemas;
int64_t committedOffset;
int64_t reqOffset;
int64_t rspOffset;
int32_t skipLogNum;
@ -1881,22 +1864,18 @@ typedef struct {
// one req for one vg+topic
typedef struct {
SMsgHead head;
// 0: commit only, current offset
// 1: consume only, poll next offset
// 2: commit current and consume next offset
int32_t reqType;
int64_t reqId;
int64_t consumerId;
int64_t blockingTime;
char cgroup[TSDB_CONSUMER_GROUP_LEN];
int64_t offset;
int64_t currentOffset;
char topic[TSDB_TOPIC_FNAME_LEN];
} SMqConsumeReq;
typedef struct {
int32_t vgId;
int64_t offset;
SEpSet epSet;
} SMqSubVgEp;
@ -1917,12 +1896,14 @@ static FORCE_INLINE void tDeleteSMqSubTopicEp(SMqSubTopicEp* pSubTopicEp) { taos
static FORCE_INLINE int32_t tEncodeSMqSubVgEp(void** buf, const SMqSubVgEp* pVgEp) {
int32_t tlen = 0;
tlen += taosEncodeFixedI32(buf, pVgEp->vgId);
tlen += taosEncodeFixedI64(buf, pVgEp->offset);
tlen += taosEncodeSEpSet(buf, &pVgEp->epSet);
return tlen;
}
static FORCE_INLINE void* tDecodeSMqSubVgEp(void* buf, SMqSubVgEp* pVgEp) {
buf = taosDecodeFixedI32(buf, &pVgEp->vgId);
buf = taosDecodeFixedI64(buf, &pVgEp->offset);
buf = taosDecodeSEpSet(buf, &pVgEp->epSet);
return buf;
}

8
include/common/tmsgdef.h
View File

@ -135,18 +135,19 @@ enum {
TD_DEF_MSG_TYPE(TDMT_MND_SHOW, "mnode-show", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_SHOW_RETRIEVE, "mnode-retrieve", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_STATUS, "mnode-status", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_TRANS, "mnode-trans", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_TRANS_TIMER, "mnode-trans-tmr", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_KILL_TRANS, "mnode-kill-trans", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_TELEM_TIMER, "mnode-telem-tmr", SMTimerReq, SMTimerReq)
TD_DEF_MSG_TYPE(TDMT_MND_GRANT, "mnode-grant", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_AUTH, "mnode-auth", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_CREATE_TOPIC, "mnode-create-topic", SMCreateTopicReq, SMCreateTopicRsp)
TD_DEF_MSG_TYPE(TDMT_MND_ALTER_TOPIC, "mnode-alter-topic", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_DROP_TOPIC, "mnode-drop-topic", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_SUBSCRIBE, "mnode-subscribe", SCMSubscribeReq, SCMSubscribeRsp)
TD_DEF_MSG_TYPE(TDMT_MND_RESET_OFFSET, "mnode-reset-offset", SMqCMResetOffsetReq, SMqCMResetOffsetRsp)
TD_DEF_MSG_TYPE(TDMT_MND_GET_SUB_EP, "mnode-get-sub-ep", SMqCMGetSubEpReq, SMqCMGetSubEpRsp)
TD_DEF_MSG_TYPE(TDMT_MND_MQ_TIMER, "mnode-mq-timer", SMTimerReq, SMTimerReq)
TD_DEF_MSG_TYPE(TDMT_MND_MQ_TIMER, "mnode-mq-tmr", SMTimerReq, SMTimerReq)
TD_DEF_MSG_TYPE(TDMT_MND_MQ_DO_REBALANCE, "mnode-mq-do-rebalance", SMqDoRebalanceMsg, SMqDoRebalanceMsg)
TD_DEF_MSG_TYPE(TDMT_MND_MQ_COMMIT_OFFSET, "mnode-mq-commit-offset", SMqCMCommitOffsetReq, SMqCMCommitOffsetRsp)
// Requests handled by VNODE
TD_NEW_MSG_SEG(TDMT_VND_MSG)
@ -194,4 +195,3 @@ enum {
TDMT_MAX
#endif
};
// clang-format on

20
include/dnode/mgmt/dnode.h
View File

@ -27,15 +27,16 @@ typedef struct SDnode SDnode;
/* ------------------------ Environment ------------------ */
typedef struct {
int32_t sver;
int32_t numOfCores;
int16_t numOfCommitThreads;
int8_t enableTelem;
char timezone[TSDB_TIMEZONE_LEN];
char locale[TSDB_LOCALE_LEN];
char charset[TSDB_LOCALE_LEN];
char buildinfo[64];
char gitinfo[48];
int32_t sver;
int32_t numOfCores;
uint16_t numOfCommitThreads;
bool enableTelem;
bool printAuth;
char timezone[TSDB_TIMEZONE_LEN];
char locale[TSDB_LOCALE_LEN];
char charset[TSDB_LOCALE_LEN];
char buildinfo[64];
char gitinfo[48];
} SDnodeEnvCfg;
/**
@ -65,6 +66,7 @@ typedef struct {
char localEp[TSDB_EP_LEN];
char localFqdn[TSDB_FQDN_LEN];
char firstEp[TSDB_EP_LEN];
char secondEp[TSDB_EP_LEN];
} SDnodeObjCfg;
/**

3
include/dnode/mnode/mnode.h
View File

@ -46,7 +46,8 @@ typedef struct SMnodeLoad {
typedef struct SMnodeCfg {
int32_t sver;
int8_t enableTelem;
bool enableTelem;
bool printAuth;
int32_t statusInterval;
int32_t shellActivityTimer;
char *timezone;

17
include/dnode/mnode/sdb/sdb.h
View File

@ -113,14 +113,15 @@ typedef enum {
SDB_USER = 7,
SDB_AUTH = 8,
SDB_ACCT = 9,
SDB_SUBSCRIBE = 10,
SDB_CONSUMER = 11,
SDB_TOPIC = 12,
SDB_VGROUP = 13,
SDB_STB = 14,
SDB_DB = 15,
SDB_FUNC = 16,
SDB_MAX = 17
SDB_OFFSET = 10,
SDB_SUBSCRIBE = 11,
SDB_CONSUMER = 12,
SDB_TOPIC = 13,
SDB_VGROUP = 14,
SDB_STB = 15,
SDB_DB = 16,
SDB_FUNC = 17,
SDB_MAX = 18
} ESdbType;
typedef struct SSdb SSdb;

104
include/libs/config/config.h Normal file
View File

@ -0,0 +1,104 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_CONFIG_H_
#define _TD_CONFIG_H_
#include "os.h"
#ifdef __cplusplus
extern "C" {
#endif
#define CFG_NAME_MAX_LEN 128
typedef enum {
CFG_STYPE_DEFAULT,
CFG_STYPE_CFG_FILE,
CFG_STYPE_ENV_FILE,
CFG_STYPE_ENV_VAR,
CFG_STYPE_APOLLO_URL,
CFG_STYPE_ARG_LIST,
CFG_STYPE_API_OPTION
} ECfgSrcType;
typedef enum {
CFG_DTYPE_NONE,
CFG_DTYPE_BOOL,
CFG_DTYPE_INT32,
CFG_DTYPE_INT64,
CFG_DTYPE_FLOAT,
CFG_DTYPE_STRING,
CFG_DTYPE_IPSTR,
CFG_DTYPE_DIR,
CFG_DTYPE_LOCALE,
CFG_DTYPE_CHARSET,
CFG_DTYPE_TIMEZONE
} ECfgDataType;
typedef struct SConfigItem {
ECfgSrcType stype;
ECfgDataType dtype;
char *name;
union {
bool bval;
float fval;
int32_t i32;
int64_t i64;
char *str;
};
union {
int64_t imin;
double fmin;
};
union {
int64_t imax;
double fmax;
};
} SConfigItem;
typedef struct SConfig SConfig;
SConfig *cfgInit();
int32_t cfgLoad(SConfig *pCfg, ECfgSrcType cfgType, const char *sourceStr);
void cfgCleanup(SConfig *pCfg);
int32_t cfgGetSize(SConfig *pCfg);
SConfigItem *cfgIterate(SConfig *pCfg, SConfigItem *pIter);
void cfgCancelIterate(SConfig *pCfg, SConfigItem *pIter);
SConfigItem *cfgGetItem(SConfig *pCfg, const char *name);
int32_t cfgAddBool(SConfig *pCfg, const char *name, bool defaultVal);
int32_t cfgAddInt32(SConfig *pCfg, const char *name, int32_t defaultVal, int64_t minval, int64_t maxval);
int32_t cfgAddInt64(SConfig *pCfg, const char *name, int64_t defaultVal, int64_t minval, int64_t maxval);
int32_t cfgAddFloat(SConfig *pCfg, const char *name, float defaultVal, double minval, double maxval);
int32_t cfgAddString(SConfig *pCfg, const char *name, const char *defaultVal);
int32_t cfgAddIpStr(SConfig *pCfg, const char *name, const char *defaultVa);
int32_t cfgAddDir(SConfig *pCfg, const char *name, const char *defaultVal);
int32_t cfgAddLocale(SConfig *pCfg, const char *name, const char *defaultVal);
int32_t cfgAddCharset(SConfig *pCfg, const char *name, const char *defaultVal);
int32_t cfgAddTimezone(SConfig *pCfg, const char *name, const char *defaultVal);
const char *cfgStypeStr(ECfgSrcType type);
const char *cfgDtypeStr(ECfgDataType type);
void cfgDumpCfg(SConfig *pCfg);
#ifdef __cplusplus
}
#endif
#endif /*_TD_CONFIG_H_*/

3
include/libs/transport/trpc.h
View File

@ -78,6 +78,9 @@ typedef struct SRpcInit {
// call back to retrieve the client auth info, for server app only
int (*afp)(void *parent, char *tableId, char *spi, char *encrypt, char *secret, char *ckey);
// call back to keep conn or not
bool (*pfp)(void *parent, tmsg_t msgType);
void *parent;
} SRpcInit;

2
include/os/osDir.h
View File

@ -24,7 +24,7 @@ void taosRemoveDir(const char *dirname);
int32_t taosDirExist(char *dirname);
int32_t taosMkDir(const char *dirname);
void taosRemoveOldFiles(char *dirname, int32_t keepDays);
int32_t taosExpandDir(char *dirname, char *outname, int32_t maxlen);
int32_t taosExpandDir(const char *dirname, char *outname, int32_t maxlen);
int32_t taosRealPath(char *dirname, int32_t maxlen);
#ifdef __cplusplus

15
include/os/osEnv.h
View File

@ -21,11 +21,24 @@ extern "C" {
#endif
extern char tsOsName[];
extern char tsDataDir[];
extern char tsLogDir[];
extern char tsScriptDir[];
extern char tsTempDir[];
extern char configDir[];
extern struct SDiskSpace tsLogSpace;
extern struct SDiskSpace tsTempSpace;
extern struct SDiskSpace tsDataSpace;
void taosUpdateLogSpace();
void taosUpdateTempSpace();
void taosUpdateDataSpace();
bool taosLogSpaceAvailable();
bool taosTmpSpaceAvailable();
bool taosDataSpaceAvailable();
void taosUpdateAllSpace();
#ifdef __cplusplus
}
#endif

2
include/os/osSocket.h
View File

@ -55,7 +55,7 @@ void taosSetMaskSIGPIPE();
int32_t taosSetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t optlen);
int32_t taosGetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t *optlen);
uint32_t taosInetAddr(char *ipAddr);
uint32_t taosInetAddr(const char *ipAddr);
const char *taosInetNtoa(struct in_addr ipInt);
#if (defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32))

25
include/os/osSysinfo.h
View File

@ -25,21 +25,28 @@ extern "C" {
#define TSDB_LOCALE_LEN 64
#define TSDB_TIMEZONE_LEN 96
extern int64_t tsPageSize;
extern int64_t tsOpenMax;
extern int64_t tsStreamMax;
extern int32_t tsNumOfCores;
extern int32_t tsTotalMemoryMB;
extern char tsTimezone[];
extern char tsLocale[];
extern char tsCharset[]; // default encode string
typedef struct {
int64_t total;
int64_t used;
int64_t avail;
} SDiskSize;
typedef struct SDiskSpace {
int64_t reserved;
SDiskSize size;
} SDiskSpace;
extern int64_t tsPageSize;
extern int64_t tsOpenMax;
extern int64_t tsStreamMax;
extern int32_t tsNumOfCores;
extern int32_t tsTotalMemoryMB;
extern char tsTimezone[];
extern char tsLocale[];
extern char tsCharset[]; // default encode string
int32_t taosGetDiskSize(char *dataDir, SDiskSize *diskSize);
int32_t taosGetCpuCores();
void taosGetSystemInfo();

5
include/util/taoserror.h
View File

@ -20,6 +20,8 @@
extern "C" {
#endif
// clang-format off
#define TAOS_DEF_ERROR_CODE(mod, code) ((int32_t)((0x80000000 | ((mod)<<16) | (code))))
#define TAOS_SYSTEM_ERROR(code) (0x80ff0000 | (code))
@ -71,6 +73,8 @@ int32_t* taosGetErrno();
#define TSDB_CODE_MSG_NOT_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0109)
#define TSDB_CODE_INVALID_PARA TAOS_DEF_ERROR_CODE(0, 0x010A)
#define TSDB_CODE_REPEAT_INIT TAOS_DEF_ERROR_CODE(0, 0x010B)
#define TSDB_CODE_CFG_NOT_FOUND TAOS_DEF_ERROR_CODE(0, 0x010C)
#define TSDB_CODE_INVALID_CFG TAOS_DEF_ERROR_CODE(0, 0x010D)
#define TSDB_CODE_REF_NO_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0110)
#define TSDB_CODE_REF_FULL TAOS_DEF_ERROR_CODE(0, 0x0111)
@ -260,6 +264,7 @@ int32_t* taosGetErrno();
#define TSDB_CODE_MND_CONSUMER_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03E7)
#define TSDB_CODE_MND_UNSUPPORTED_TOPIC TAOS_DEF_ERROR_CODE(0, 0x03E8)
#define TSDB_CODE_MND_SUBSCRIBE_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03E9)
#define TSDB_CODE_MND_OFFSET_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03EA)
#define TSDB_CODE_MND_MQ_PLACEHOLDER TAOS_DEF_ERROR_CODE(0, 0x03F0)
// dnode

97
include/util/tconfig.h
View File

@ -1,97 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_UTIL_CONFIG_H
#define _TD_UTIL_CONFIG_H
#ifdef __cplusplus
extern "C" {
#endif
#define TSDB_CFG_MAX_NUM 115
#define TSDB_CFG_PRINT_LEN 23
#define TSDB_CFG_OPTION_LEN 24
#define TSDB_CFG_VALUE_LEN 41
#define TSDB_CFG_CTYPE_B_CONFIG 1U // can be configured from file
#define TSDB_CFG_CTYPE_B_SHOW 2U // can displayed by "show configs" commands
#define TSDB_CFG_CTYPE_B_LOG 4U // is a log type configuration
#define TSDB_CFG_CTYPE_B_CLIENT 8U // can be displayed in the client log
#define TSDB_CFG_CTYPE_B_OPTION 16U // can be configured by taos_options function
#define TSDB_CFG_CTYPE_B_NOT_PRINT 32U // such as password
#define MAX_FLOAT 100000
#define MIN_FLOAT 0
enum {
TAOS_CFG_CSTATUS_NONE, // not configured
TAOS_CFG_CSTATUS_DEFAULT, // use system default value
TAOS_CFG_CSTATUS_FILE, // configured from file
TAOS_CFG_CSTATUS_OPTION, // configured by taos_options function
TAOS_CFG_CSTATUS_ARG, // configured by program argument
};
enum {
TAOS_CFG_VTYPE_INT8,
TAOS_CFG_VTYPE_INT16,
TAOS_CFG_VTYPE_INT32,
TAOS_CFG_VTYPE_UINT16,
TAOS_CFG_VTYPE_FLOAT,
TAOS_CFG_VTYPE_STRING,
TAOS_CFG_VTYPE_IPSTR,
TAOS_CFG_VTYPE_DIRECTORY,
TAOS_CFG_VTYPE_DATA_DIRCTORY,
TAOS_CFG_VTYPE_DOUBLE,
};
enum {
TAOS_CFG_UTYPE_NONE,
TAOS_CFG_UTYPE_PERCENT,
TAOS_CFG_UTYPE_GB,
TAOS_CFG_UTYPE_MB,
TAOS_CFG_UTYPE_BYTE,
TAOS_CFG_UTYPE_SECOND,
TAOS_CFG_UTYPE_MS
};
typedef struct {
char * option;
void * ptr;
float minValue;
float maxValue;
int8_t cfgType;
int8_t cfgStatus;
int8_t unitType;
int8_t valType;
int32_t ptrLength;
} SGlobalCfg;
extern SGlobalCfg tsGlobalConfig[];
extern int32_t tsGlobalConfigNum;
extern char * tsCfgStatusStr[];
void taosReadGlobalLogCfg();
int32_t taosReadCfgFromFile();
void taosPrintCfg();
void taosDumpGlobalCfg();
void taosAddConfigOption(SGlobalCfg cfg);
SGlobalCfg *taosGetConfigOption(const char *option);
#ifdef __cplusplus
}
#endif
#endif /*_TD_UTIL_CONFIG_H*/

6
include/util/tdef.h
View File

@ -13,6 +13,8 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
// clang-format off
#ifndef _TD_UTIL_DEF_H
#define _TD_UTIL_DEF_H
@ -196,6 +198,7 @@ typedef enum ELogicConditionType {
#define TSDB_TOPIC_FNAME_LEN TSDB_TABLE_FNAME_LEN
#define TSDB_CONSUMER_GROUP_LEN 192
#define TSDB_SUBSCRIBE_KEY_LEN (TSDB_CONSUMER_GROUP_LEN + TSDB_TOPIC_FNAME_LEN + 2)
#define TSDB_PARTITION_KEY_LEN (TSDB_CONSUMER_GROUP_LEN + TSDB_TOPIC_FNAME_LEN + 2)
#define TSDB_COL_NAME_LEN 65
#define TSDB_MAX_SAVED_SQL_LEN TSDB_MAX_COLUMNS * 64
#define TSDB_MAX_SQL_LEN TSDB_PAYLOAD_SIZE
@ -349,9 +352,6 @@ typedef enum ELogicConditionType {
#define TSDB_QUERY_TYPE_NON_TYPE 0x00u // none type
#define TSDB_QUERY_TYPE_FREE_RESOURCE 0x01u // free qhandle at vnode
#define TSDB_QUERY_TYPE_NON_TYPE 0x00u // none type
#define TSDB_QUERY_TYPE_FREE_RESOURCE 0x01u // free qhandle at vnode
#define TSDB_META_COMPACT_RATIO 0 // disable tsdb meta compact by default

33
include/util/tlog.h
View File

@ -22,8 +22,8 @@
extern "C" {
#endif
// log
extern int8_t tsAsyncLog;
extern bool tsLogInited;
extern bool tsAsyncLog;
extern int32_t tsNumOfLogLines;
extern int32_t tsLogKeepDays;
extern int32_t dDebugFlag;
@ -32,9 +32,6 @@ extern int32_t mDebugFlag;
extern int32_t cDebugFlag;
extern int32_t jniDebugFlag;
extern int32_t tmrDebugFlag;
extern int32_t httpDebugFlag;
extern int32_t mqttDebugFlag;
extern int32_t monDebugFlag;
extern int32_t uDebugFlag;
extern int32_t rpcDebugFlag;
extern int32_t qDebugFlag;
@ -42,23 +39,23 @@ extern int32_t wDebugFlag;
extern int32_t sDebugFlag;
extern int32_t tsdbDebugFlag;
extern int32_t tqDebugFlag;
extern int32_t cqDebugFlag;
extern int32_t debugFlag;
#define DEBUG_FATAL 1U
#define DEBUG_ERROR DEBUG_FATAL
#define DEBUG_WARN 2U
#define DEBUG_INFO DEBUG_WARN
#define DEBUG_DEBUG 4U
#define DEBUG_TRACE 8U
#define DEBUG_DUMP 16U
extern int32_t fsDebugFlag;
#define DEBUG_FATAL 1U
#define DEBUG_ERROR DEBUG_FATAL
#define DEBUG_WARN 2U
#define DEBUG_INFO DEBUG_WARN
#define DEBUG_DEBUG 4U
#define DEBUG_TRACE 8U
#define DEBUG_DUMP 16U
#define DEBUG_SCREEN 64U
#define DEBUG_FILE 128U
#define DEBUG_FILE 128U
int32_t taosInitLog(char *logName, int32_t numOfLogLines, int32_t maxFiles);
int32_t taosInitLog(const char *logName, int32_t maxFiles);
void taosCloseLog();
void taosResetLog();
void taosSetAllDebugFlag(int32_t flag);
void taosDumpData(unsigned char *msg, int32_t len);
void taosPrintLog(const char *flags, int32_t dflag, const char *format, ...)
#ifdef __GNUC__
@ -72,8 +69,6 @@ void taosPrintLongString(const char *flags, int32_t dflag, const char *format, .
#endif
;
void taosDumpData(unsigned char *msg, int32_t len);
#ifdef __cplusplus
}
#endif

1
include/util/ttimer.h
View File

@ -26,7 +26,6 @@ typedef void *tmr_h;
typedef void (*TAOS_TMR_CALLBACK)(void *, void *);
extern int taosTmrThreads;
extern uint32_t tsMaxTmrCtrl;
#define MSECONDS_PER_TICK 5

33
include/util/version.h Normal file
View File

@ -0,0 +1,33 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_UTIL_VERSION_H
#define _TD_UTIL_VERSION_H
#ifdef __cplusplus
extern "C" {
#endif
extern char version[];
extern char compatible_version[];
extern char gitinfo[];
extern char gitinfoOfInternal[];
extern char buildinfo[];
#ifdef __cplusplus
}
#endif
#endif /*_TD_UTIL_VERSION_H*/

2
source/client/CMakeLists.txt
View File

@ -8,7 +8,7 @@ target_include_directories(
target_link_libraries(
taos
INTERFACE api
PRIVATE os util common transport parser planner catalog scheduler function qcom
PRIVATE os util common transport parser planner catalog scheduler function qcom config
)
if(${BUILD_TEST})

194
source/client/inc/clientInt.h
View File

@ -20,17 +20,19 @@
extern "C" {
#endif
#include "taos.h"
#include "common.h"
#include "tmsg.h"
#include "parser.h"
#include "query.h"
#include "taos.h"
#include "tdef.h"
#include "tep.h"
#include "thash.h"
#include "tlist.h"
#include "tmsg.h"
#include "tmsgtype.h"
#include "trpc.h"
#include "query.h"
#include "parser.h"
#include "config.h"
#define CHECK_CODE_GOTO(expr, label) \
do { \
@ -46,12 +48,12 @@ extern "C" {
typedef struct SAppInstInfo SAppInstInfo;
typedef struct SHbConnInfo {
void *param;
SClientHbReq *req;
void* param;
SClientHbReq* req;
} SHbConnInfo;
typedef struct SAppHbMgr {
char *key;
char* key;
// statistics
int32_t reportCnt;
int32_t connKeyCnt;
@ -62,15 +64,13 @@ typedef struct SAppHbMgr {
// connection
SAppInstInfo* pAppInstInfo;
// info
SHashObj* activeInfo; // hash<SClientHbKey, SClientHbReq>
SHashObj* connInfo; // hash<SClientHbKey, SHbConnInfo>
SHashObj* activeInfo; // hash<SClientHbKey, SClientHbReq>
SHashObj* connInfo; // hash<SClientHbKey, SHbConnInfo>
} SAppHbMgr;
typedef int32_t (*FHbRspHandle)(struct SAppHbMgr* pAppHbMgr, SClientHbRsp* pRsp);
typedef int32_t (*FHbRspHandle)(struct SAppHbMgr *pAppHbMgr, SClientHbRsp* pRsp);
typedef int32_t (*FHbReqHandle)(SClientHbKey *connKey, void* param, SClientHbReq *req);
typedef int32_t (*FHbReqHandle)(SClientHbKey* connKey, void* param, SClientHbReq* req);
typedef struct SClientHbMgr {
int8_t inited;
@ -83,63 +83,62 @@ typedef struct SClientHbMgr {
FHbRspHandle rspHandle[HEARTBEAT_TYPE_MAX];
} SClientHbMgr;
typedef struct SQueryExecMetric {
int64_t start; // start timestamp
int64_t parsed; // start to parse
int64_t send; // start to send to server
int64_t rsp; // receive response from server
int64_t start; // start timestamp
int64_t parsed; // start to parse
int64_t send; // start to send to server
int64_t rsp; // receive response from server
} SQueryExecMetric;
typedef struct SInstanceSummary {
uint64_t numOfInsertsReq;
uint64_t numOfInsertRows;
uint64_t insertElapsedTime;
uint64_t insertBytes; // submit to tsdb since launched.
uint64_t numOfInsertsReq;
uint64_t numOfInsertRows;
uint64_t insertElapsedTime;
uint64_t insertBytes; // submit to tsdb since launched.
uint64_t fetchBytes;
uint64_t queryElapsedTime;
uint64_t numOfSlowQueries;
uint64_t totalRequests;
uint64_t currentRequests; // the number of SRequestObj
uint64_t fetchBytes;
uint64_t queryElapsedTime;
uint64_t numOfSlowQueries;
uint64_t totalRequests;
uint64_t currentRequests; // the number of SRequestObj
} SInstanceSummary;
typedef struct SHeartBeatInfo {
void *pTimer; // timer, used to send request msg to mnode
void* pTimer; // timer, used to send request msg to mnode
} SHeartBeatInfo;
struct SAppInstInfo {
int64_t numOfConns;
SCorEpSet mgmtEp;
SInstanceSummary summary;
SList *pConnList; // STscObj linked list
int64_t clusterId;
void *pTransporter;
struct SAppHbMgr *pAppHbMgr;
int64_t numOfConns;
SCorEpSet mgmtEp;
SInstanceSummary summary;
SList* pConnList; // STscObj linked list
int64_t clusterId;
void* pTransporter;
struct SAppHbMgr* pAppHbMgr;
};
typedef struct SAppInfo {
int64_t startTime;
char appName[TSDB_APP_NAME_LEN];
char *ep;
char* ep;
int32_t pid;
int32_t numOfThreads;
SHashObj *pInstMap;
SHashObj* pInstMap;
pthread_mutex_t mutex;
} SAppInfo;
typedef struct STscObj {
char user[TSDB_USER_LEN];
char pass[TSDB_PASSWORD_LEN];
char db[TSDB_DB_FNAME_LEN];
char ver[128];
int32_t acctId;
uint32_t connId;
int32_t connType;
uint64_t id; // ref ID returned by taosAddRef
pthread_mutex_t mutex; // used to protect the operation on db
int32_t numOfReqs; // number of sqlObj bound to this connection
SAppInstInfo *pAppInfo;
char user[TSDB_USER_LEN];
char pass[TSDB_PASSWORD_LEN];
char db[TSDB_DB_FNAME_LEN];
char ver[128];
int32_t acctId;
uint32_t connId;
int32_t connType;
uint64_t id; // ref ID returned by taosAddRef
pthread_mutex_t mutex; // used to protect the operation on db
int32_t numOfReqs; // number of sqlObj bound to this connection
SAppInstInfo* pAppInfo;
} STscObj;
typedef struct SMqConsumer {
@ -147,49 +146,49 @@ typedef struct SMqConsumer {
} SMqConsumer;
typedef struct SReqResultInfo {
const char *pRspMsg;
const char *pData;
TAOS_FIELD *fields;
uint32_t numOfCols;
int32_t *length;
TAOS_ROW row;
char **pCol;
uint32_t numOfRows;
uint64_t totalRows;
uint32_t current;
bool completed;
const char* pRspMsg;
const char* pData;
TAOS_FIELD* fields;
uint32_t numOfCols;
int32_t* length;
TAOS_ROW row;
char** pCol;
uint32_t numOfRows;
uint64_t totalRows;
uint32_t current;
bool completed;
} SReqResultInfo;
typedef struct SShowReqInfo {
int64_t execId; // showId/queryId
int32_t vgId;
SArray *pArray; // SArray<SVgroupInfo>
int32_t currentIndex; // current accessed vgroup index.
int64_t execId; // showId/queryId
int32_t vgId;
SArray* pArray; // SArray<SVgroupInfo>
int32_t currentIndex; // current accessed vgroup index.
} SShowReqInfo;
typedef struct SRequestSendRecvBody {
tsem_t rspSem; // not used now
tsem_t rspSem; // not used now
void* fp;
SShowReqInfo showInfo; // todo this attribute will be removed after the query framework being completed.
SShowReqInfo showInfo; // todo this attribute will be removed after the query framework being completed.
SDataBuf requestMsg;
struct SSchJob *pQueryJob; // query job, created according to sql query DAG.
struct SQueryDag *pDag; // the query dag, generated according to the sql statement.
struct SSchJob* pQueryJob; // query job, created according to sql query DAG.
struct SQueryDag* pDag; // the query dag, generated according to the sql statement.
SReqResultInfo resInfo;
} SRequestSendRecvBody;
#define ERROR_MSG_BUF_DEFAULT_SIZE 512
#define ERROR_MSG_BUF_DEFAULT_SIZE 512
typedef struct SRequestObj {
uint64_t requestId;
int32_t type; // request type
STscObj *pTscObj;
char *sqlstr; // sql string
int32_t sqlLen;
int64_t self;
char *msgBuf;
void *pInfo; // sql parse info, generated by parser module
int32_t code;
SQueryExecMetric metric;
uint64_t requestId;
int32_t type; // request type
STscObj* pTscObj;
char* sqlstr; // sql string
int32_t sqlLen;
int64_t self;
char* msgBuf;
void* pInfo; // sql parse info, generated by parser module
int32_t code;
SQueryExecMetric metric;
SRequestSendRecvBody body;
} SRequestObj;
@ -198,51 +197,52 @@ extern int32_t clientReqRefPool;
extern int32_t clientConnRefPool;
extern int (*handleRequestRspFp[TDMT_MAX])(void*, const SDataBuf* pMsg, int32_t code);
int genericRspCallback(void* param, const SDataBuf* pMsg, int32_t code);
int genericRspCallback(void* param, const SDataBuf* pMsg, int32_t code);
SMsgSendInfo* buildMsgInfoImpl(SRequestObj* pReqObj);
int taos_init();
int taos_init();
void* createTscObj(const char* user, const char* auth, const char *db, SAppInstInfo* pAppInfo);
void destroyTscObj(void*pObj);
void* createTscObj(const char* user, const char* auth, const char* db, SAppInstInfo* pAppInfo);
void destroyTscObj(void* pObj);
uint64_t generateRequestId();
void *createRequest(STscObj* pObj, __taos_async_fn_t fp, void* param, int32_t type);
void* createRequest(STscObj* pObj, __taos_async_fn_t fp, void* param, int32_t type);
void destroyRequest(SRequestObj* pRequest);
char *getDbOfConnection(STscObj* pObj);
char* getDbOfConnection(STscObj* pObj);
void setConnectionDB(STscObj* pTscObj, const char* db);
void taos_init_imp(void);
int taos_options_imp(TSDB_OPTION option, const char *str);
int taos_options_imp(TSDB_OPTION option, const char* str);
void* openTransporter(const char *user, const char *auth, int32_t numOfThreads);
void* openTransporter(const char* user, const char* auth, int32_t numOfThreads);
bool persistConnForSpecificMsg(void* parenct, tmsg_t msgType);
void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet);
void initMsgHandleFp();
TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass, const char *auth, const char *db, uint16_t port);
TAOS* taos_connect_internal(const char* ip, const char* user, const char* pass, const char* auth, const char* db,
uint16_t port);
void *doFetchRow(SRequestObj* pRequest);
void* doFetchRow(SRequestObj* pRequest);
void setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32_t numOfCols, int32_t numOfRows);
void setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32_t numOfCols, int32_t numOfRows);
int32_t buildRequest(STscObj *pTscObj, const char *sql, int sqlLen, SRequestObj** pRequest);
int32_t buildRequest(STscObj* pTscObj, const char* sql, int sqlLen, SRequestObj** pRequest);
int32_t parseSql(SRequestObj* pRequest, SQueryNode** pQuery);
// --- heartbeat
// --- heartbeat
// global, called by mgmt
int hbMgrInit();
void hbMgrCleanUp();
int hbHandleRsp(SClientHbBatchRsp* hbRsp);
// cluster level
SAppHbMgr* appHbMgrInit(SAppInstInfo* pAppInstInfo, char *key);
void appHbMgrCleanup(void);
SAppHbMgr* appHbMgrInit(SAppInstInfo* pAppInstInfo, char* key);
void appHbMgrCleanup(void);
// conn level
int hbRegisterConn(SAppHbMgr* pAppHbMgr, int32_t connId, int64_t clusterId, int32_t hbType);
@ -254,6 +254,12 @@ int hbAddConnInfo(SAppHbMgr* pAppHbMgr, SClientHbKey connKey, void* key, void* v
void hbMgrInitMqHbRspHandle();
// config
int32_t tscInitLog(const char *cfgDir, const char *envFile, const char *apolloUrl);
int32_t tscInitCfg(const char *cfgDir, const char *envFile, const char *apolloUrl);
extern SConfig *tscCfg;
#ifdef __cplusplus
}
#endif

208
source/client/src/clientCfg.c Normal file
View File

@ -0,0 +1,208 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "clientInt.h"
#include "ulog.h"
// todo refact
SConfig *tscCfg;
static int32_t tscLoadCfg(SConfig *pConfig, const char *inputCfgDir, const char *envFile, const char *apolloUrl) {
char cfgDir[PATH_MAX] = {0};
char cfgFile[PATH_MAX + 100] = {0};
taosExpandDir(inputCfgDir, cfgDir, PATH_MAX);
snprintf(cfgFile, sizeof(cfgFile), "%s" TD_DIRSEP "taos.cfg", cfgDir);
if (cfgLoad(pConfig, CFG_STYPE_APOLLO_URL, apolloUrl) != 0) {
uError("failed to load from apollo url:%s since %s\n", apolloUrl, terrstr());
return -1;
}
if (cfgLoad(pConfig, CFG_STYPE_CFG_FILE, cfgFile) != 0) {
if (cfgLoad(pConfig, CFG_STYPE_CFG_FILE, cfgDir) != 0) {
uError("failed to load from config file:%s since %s\n", cfgFile, terrstr());
return -1;
}
}
if (cfgLoad(pConfig, CFG_STYPE_ENV_FILE, envFile) != 0) {
uError("failed to load from env file:%s since %s\n", envFile, terrstr());
return -1;
}
if (cfgLoad(pConfig, CFG_STYPE_ENV_VAR, NULL) != 0) {
uError("failed to load from global env variables since %s\n", terrstr());
return -1;
}
return 0;
}
static int32_t tscAddLogCfg(SConfig *pCfg) {
if (cfgAddDir(pCfg, "logDir", "/var/log/taos") != 0) return -1;
if (cfgAddBool(pCfg, "asyncLog", 1) != 0) return -1;
if (cfgAddInt32(pCfg, "numOfLogLines", 10000000, 1000, 2000000000) != 0) return -1;
if (cfgAddInt32(pCfg, "logKeepDays", 0, -365000, 365000) != 0) return -1;
if (cfgAddInt32(pCfg, "debugFlag", 0, 0, 255) != 0) return -1;
if (cfgAddInt32(pCfg, "cDebugFlag", 0, 0, 255) != 0) return -1;
if (cfgAddInt32(pCfg, "jniDebugFlag", 0, 0, 255) != 0) return -1;
if (cfgAddInt32(pCfg, "tmrDebugFlag", 0, 0, 255) != 0) return -1;
if (cfgAddInt32(pCfg, "uDebugFlag", 0, 0, 255) != 0) return -1;
if (cfgAddInt32(pCfg, "rpcDebugFlag", 0, 0, 255) != 0) return -1;
return 0;
}
static int32_t tscSetLogCfg(SConfig *pCfg) {
tstrncpy(tsLogDir, cfgGetItem(pCfg, "logDir")->str, PATH_MAX);
tsAsyncLog = cfgGetItem(pCfg, "asyncLog")->bval;
tsNumOfLogLines = cfgGetItem(pCfg, "numOfLogLines")->i32;
tsLogKeepDays = cfgGetItem(pCfg, "logKeepDays")->i32;
cDebugFlag = cfgGetItem(pCfg, "cDebugFlag")->i32;
jniDebugFlag = cfgGetItem(pCfg, "jniDebugFlag")->i32;
tmrDebugFlag = cfgGetItem(pCfg, "tmrDebugFlag")->i32;
uDebugFlag = cfgGetItem(pCfg, "uDebugFlag")->i32;
rpcDebugFlag = cfgGetItem(pCfg, "rpcDebugFlag")->i32;
int32_t debugFlag = cfgGetItem(pCfg, "debugFlag")->i32;
taosSetAllDebugFlag(debugFlag);
return 0;
}
int32_t tscInitLog(const char *cfgDir, const char *envFile, const char *apolloUrl) {
if (tsLogInited) return 0;
SConfig *pCfg = cfgInit();
if (pCfg == NULL) return -1;
if (tscAddLogCfg(pCfg) != 0) {
printf("failed to add log cfg since %s\n", terrstr());
cfgCleanup(pCfg);
return -1;
}
if (tscLoadCfg(pCfg, cfgDir, envFile, apolloUrl) != 0) {
printf("failed to load log cfg since %s\n", terrstr());
cfgCleanup(pCfg);
return -1;
}
if (tscSetLogCfg(pCfg) != 0) {
printf("failed to set log cfg since %s\n", terrstr());
cfgCleanup(pCfg);
return -1;
}
const int32_t maxLogFileNum = 10;
if (taosInitLog("taoslog", maxLogFileNum) != 0) {
printf("failed to init log file since %s\n", terrstr());
cfgCleanup(pCfg);
return -1;
}
cfgDumpCfg(pCfg);
cfgCleanup(pCfg);
return 0;
}
static int32_t tscAddEpCfg(SConfig *pCfg) {
char defaultFqdn[TSDB_FQDN_LEN] = {0};
if (taosGetFqdn(defaultFqdn) != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
if (cfgAddString(pCfg, "fqdn", defaultFqdn) != 0) return -1;
int32_t defaultServerPort = 6030;
if (cfgAddInt32(pCfg, "serverPort", defaultServerPort, 1, 65056) != 0) return -1;
char defaultFirstEp[TSDB_EP_LEN] = {0};
char defaultSecondEp[TSDB_EP_LEN] = {0};
snprintf(defaultFirstEp, TSDB_EP_LEN, "%s:%d", defaultFqdn, defaultServerPort);
snprintf(defaultSecondEp, TSDB_EP_LEN, "%s:%d", defaultFqdn, defaultServerPort);
if (cfgAddString(pCfg, "firstEp", defaultFirstEp) != 0) return -1;
if (cfgAddString(pCfg, "secondEp", defaultSecondEp) != 0) return -1;
return 0;
}
static int32_t tscAddCfg(SConfig *pCfg) {
if (tscAddEpCfg(pCfg) != 0) return -1;
// if (cfgAddString(pCfg, "buildinfo", buildinfo) != 0) return -1;
// if (cfgAddString(pCfg, "gitinfo", gitinfo) != 0) return -1;
// if (cfgAddString(pCfg, "version", version) != 0) return -1;
// if (cfgAddDir(pCfg, "dataDir", tsDataDir) != 0) return -1;
if (cfgAddTimezone(pCfg, "timezone", "") != 0) return -1;
if (cfgAddLocale(pCfg, "locale", "") != 0) return -1;
if (cfgAddCharset(pCfg, "charset", "") != 0) return -1;
if (cfgAddInt32(pCfg, "numOfCores", 1, 1, 100000) != 0) return -1;
if (cfgAddInt32(pCfg, "numOfCommitThreads", 4, 1, 1000) != 0) return -1;
// if (cfgAddBool(pCfg, "telemetryReporting", 0) != 0) return -1;
if (cfgAddBool(pCfg, "enableCoreFile", 0) != 0) return -1;
// if (cfgAddInt32(pCfg, "supportVnodes", 256, 0, 65536) != 0) return -1;
if (cfgAddInt32(pCfg, "statusInterval", 1, 1, 30) != 0) return -1;
if (cfgAddFloat(pCfg, "numOfThreadsPerCore", 1, 0, 10) != 0) return -1;
if (cfgAddFloat(pCfg, "ratioOfQueryCores", 1, 0, 5) != 0) return -1;
if (cfgAddInt32(pCfg, "shellActivityTimer", 3, 1, 120) != 0) return -1;
if (cfgAddInt32(pCfg, "maxConnections", 50000, 1, 100000) != 0) return -1;
return 0;
}
int32_t tscCheckCfg(SConfig *pCfg) {
bool enableCore = cfgGetItem(pCfg, "enableCoreFile")->bval;
taosSetCoreDump(enableCore);
return 0;
}
SConfig *tscInitCfgImp(const char *cfgDir, const char *envFile, const char *apolloUrl) {
SConfig *pCfg = cfgInit();
if (pCfg == NULL) return NULL;
if (tscAddCfg(pCfg) != 0) {
uError("failed to init tsc cfg since %s", terrstr());
cfgCleanup(pCfg);
return NULL;
}
if (tscLoadCfg(pCfg, cfgDir, envFile, apolloUrl) != 0) {
printf("failed to load tsc cfg since %s\n", terrstr());
cfgCleanup(pCfg);
return NULL;
}
if (tscCheckCfg(pCfg) != 0) {
uError("failed to check cfg since %s", terrstr());
cfgCleanup(pCfg);
return NULL;
}
cfgDumpCfg(pCfg);
return pCfg;
}
int32_t tscInitCfg(const char *cfgDir, const char *envFile, const char *apolloUrl) {
tscCfg = tscInitCfgImp(cfgDir, envFile, apolloUrl);
if (tscCfg == NULL) return -1;
return 0;
}

140
source/client/src/clientEnv.c
View File

@ -13,16 +13,15 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "os.h"
#include "catalog.h"
#include "clientInt.h"
#include "clientLog.h"
#include "os.h"
#include "query.h"
#include "scheduler.h"
#include "tmsg.h"
#include "tcache.h"
#include "tconfig.h"
#include "tglobal.h"
#include "tmsg.h"
#include "tnote.h"
#include "tref.h"
#include "trpc.h"
@ -30,16 +29,16 @@
#include "ttimezone.h"
#define TSC_VAR_NOT_RELEASE 1
#define TSC_VAR_RELEASED 0
#define TSC_VAR_RELEASED 0
SAppInfo appInfo;
int32_t clientReqRefPool = -1;
int32_t clientConnRefPool = -1;
SAppInfo appInfo;
int32_t clientReqRefPool = -1;
int32_t clientConnRefPool = -1;
static pthread_once_t tscinit = PTHREAD_ONCE_INIT;
volatile int32_t tscInitRes = 0;
volatile int32_t tscInitRes = 0;
static void registerRequest(SRequestObj* pRequest) {
static void registerRequest(SRequestObj *pRequest) {
STscObj *pTscObj = (STscObj *)taosAcquireRef(clientConnRefPool, pRequest->pTscObj->id);
assert(pTscObj != NULL);
@ -53,69 +52,58 @@ static void registerRequest(SRequestObj* pRequest) {
int32_t total = atomic_add_fetch_32(&pSummary->totalRequests, 1);
int32_t currentInst = atomic_add_fetch_32(&pSummary->currentRequests, 1);
tscDebug("0x%" PRIx64 " new Request from connObj:0x%" PRIx64 ", current:%d, app current:%d, total:%d, reqId:0x%"PRIx64, pRequest->self,
pRequest->pTscObj->id, num, currentInst, total, pRequest->requestId);
tscDebug("0x%" PRIx64 " new Request from connObj:0x%" PRIx64
", current:%d, app current:%d, total:%d, reqId:0x%" PRIx64,
pRequest->self, pRequest->pTscObj->id, num, currentInst, total, pRequest->requestId);
}
}
static void deregisterRequest(SRequestObj* pRequest) {
static void deregisterRequest(SRequestObj *pRequest) {
assert(pRequest != NULL);
STscObj* pTscObj = pRequest->pTscObj;
SInstanceSummary* pActivity = &pTscObj->pAppInfo->summary;
STscObj * pTscObj = pRequest->pTscObj;
SInstanceSummary *pActivity = &pTscObj->pAppInfo->summary;
int32_t currentInst = atomic_sub_fetch_32(&pActivity->currentRequests, 1);
int32_t num = atomic_sub_fetch_32(&pTscObj->numOfReqs, 1);
int64_t duration = taosGetTimestampMs() - pRequest->metric.start;
tscDebug("0x%"PRIx64" free Request from connObj: 0x%"PRIx64", reqId:0x%"PRIx64" elapsed:%"PRIu64" ms, current:%d, app current:%d", pRequest->self, pTscObj->id,
pRequest->requestId, duration, num, currentInst);
tscDebug("0x%" PRIx64 " free Request from connObj: 0x%" PRIx64 ", reqId:0x%" PRIx64 " elapsed:%" PRIu64
" ms, current:%d, app current:%d",
pRequest->self, pTscObj->id, pRequest->requestId, duration, num, currentInst);
taosReleaseRef(clientConnRefPool, pTscObj->id);
}
static void tscInitLogFile() {
taosReadGlobalLogCfg();
if (mkdir(tsLogDir, 0755) != 0 && errno != EEXIST) {
printf("failed to create log dir:%s\n", tsLogDir);
}
const char *defaultLogFileNamePrefix = "taoslog";
const int32_t maxLogFileNum = 10;
char temp[128] = {0};
sprintf(temp, "%s/%s", tsLogDir, defaultLogFileNamePrefix);
if (taosInitLog(temp, tsNumOfLogLines, maxLogFileNum) < 0) {
printf("failed to open log file in directory:%s\n", tsLogDir);
}
}
// todo close the transporter properly
void closeTransporter(STscObj* pTscObj) {
void closeTransporter(STscObj *pTscObj) {
if (pTscObj == NULL || pTscObj->pAppInfo->pTransporter == NULL) {
return;
}
tscDebug("free transporter:%p in connObj: 0x%"PRIx64, pTscObj->pAppInfo->pTransporter, pTscObj->id);
tscDebug("free transporter:%p in connObj: 0x%" PRIx64, pTscObj->pAppInfo->pTransporter, pTscObj->id);
rpcClose(pTscObj->pAppInfo->pTransporter);
}
// TODO refactor
void* openTransporter(const char *user, const char *auth, int32_t numOfThread) {
void *openTransporter(const char *user, const char *auth, int32_t numOfThread) {
SRpcInit rpcInit;
memset(&rpcInit, 0, sizeof(rpcInit));
rpcInit.localPort = 0;
rpcInit.label = "TSC";
rpcInit.numOfThreads = numOfThread;
rpcInit.cfp = processMsgFromServer;
rpcInit.sessions = tsMaxConnections;
rpcInit.pfp = persistConnForSpecificMsg;
rpcInit.sessions = cfgGetItem(tscCfg, "maxConnections")->i32;
rpcInit.connType = TAOS_CONN_CLIENT;
rpcInit.user = (char *)user;
rpcInit.idleTime = tsShellActivityTimer * 1000;
rpcInit.idleTime = cfgGetItem(tscCfg, "shellActivityTimer")->i32 * 1000;
rpcInit.ckey = "key";
rpcInit.spi = 1;
rpcInit.secret = (char *)auth;
void* pDnodeConn = rpcOpen(&rpcInit);
void *pDnodeConn = rpcOpen(&rpcInit);
if (pDnodeConn == NULL) {
tscError("failed to init connection to server");
return NULL;
@ -130,12 +118,12 @@ void destroyTscObj(void *pObj) {
SClientHbKey connKey = {.connId = pTscObj->connId, .hbType = pTscObj->connType};
hbDeregisterConn(pTscObj->pAppInfo->pAppHbMgr, connKey);
atomic_sub_fetch_64(&pTscObj->pAppInfo->numOfConns, 1);
tscDebug("connObj 0x%"PRIx64" destroyed, totalConn:%"PRId64, pTscObj->id, pTscObj->pAppInfo->numOfConns);
tscDebug("connObj 0x%" PRIx64 " destroyed, totalConn:%" PRId64, pTscObj->id, pTscObj->pAppInfo->numOfConns);
pthread_mutex_destroy(&pTscObj->mutex);
tfree(pTscObj);
}
void* createTscObj(const char* user, const char* auth, const char *db, SAppInstInfo* pAppInfo) {
void *createTscObj(const char *user, const char *auth, const char *db, SAppInstInfo *pAppInfo) {
STscObj *pObj = (STscObj *)calloc(1, sizeof(STscObj));
if (NULL == pObj) {
terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
@ -153,11 +141,11 @@ void* createTscObj(const char* user, const char* auth, const char *db, SAppInstI
pthread_mutex_init(&pObj->mutex, NULL);
pObj->id = taosAddRef(clientConnRefPool, pObj);
tscDebug("connObj created, 0x%"PRIx64, pObj->id);
tscDebug("connObj created, 0x%" PRIx64, pObj->id);
return pObj;
}
void* createRequest(STscObj* pObj, __taos_async_fn_t fp, void* param, int32_t type) {
void *createRequest(STscObj *pObj, __taos_async_fn_t fp, void *param, int32_t type) {
assert(pObj != NULL);
SRequestObj *pRequest = (SRequestObj *)calloc(1, sizeof(SRequestObj));
@ -166,20 +154,20 @@ void* createRequest(STscObj* pObj, __taos_async_fn_t fp, void* param, int32_t ty
return NULL;
}
pRequest->requestId = generateRequestId();
pRequest->requestId = generateRequestId();
pRequest->metric.start = taosGetTimestampMs();
pRequest->type = type;
pRequest->pTscObj = pObj;
pRequest->body.fp = fp; // not used it yet
pRequest->msgBuf = calloc(1, ERROR_MSG_BUF_DEFAULT_SIZE);
pRequest->type = type;
pRequest->pTscObj = pObj;
pRequest->body.fp = fp; // not used it yet
pRequest->msgBuf = calloc(1, ERROR_MSG_BUF_DEFAULT_SIZE);
tsem_init(&pRequest->body.rspSem, 0, 0);
registerRequest(pRequest);
return pRequest;
}
static void doFreeReqResultInfo(SReqResultInfo* pResInfo) {
static void doFreeReqResultInfo(SReqResultInfo *pResInfo) {
tfree(pResInfo->pRspMsg);
tfree(pResInfo->length);
tfree(pResInfo->row);
@ -187,9 +175,9 @@ static void doFreeReqResultInfo(SReqResultInfo* pResInfo) {
tfree(pResInfo->fields);
}
static void doDestroyRequest(void* p) {
static void doDestroyRequest(void *p) {
assert(p != NULL);
SRequestObj* pRequest = (SRequestObj*)p;
SRequestObj *pRequest = (SRequestObj *)p;
assert(RID_VALID(pRequest->self));
@ -208,7 +196,7 @@ static void doDestroyRequest(void* p) {
tfree(pRequest);
}
void destroyRequest(SRequestObj* pRequest) {
void destroyRequest(SRequestObj *pRequest) {
if (pRequest == NULL) {
return;
}
@ -225,11 +213,13 @@ void taos_init_imp(void) {
srand(taosGetTimestampSec());
deltaToUtcInitOnce();
taosInitGlobalCfg();
taosReadCfgFromFile();
tscInitLogFile();
if (taosCheckAndPrintCfg()) {
if (tscInitLog(configDir, NULL, NULL) != 0) {
tscInitRes = -1;
return;
}
if (tscInitCfg(configDir, NULL, NULL) != 0) {
tscInitRes = -1;
return;
}
@ -245,21 +235,21 @@ void taos_init_imp(void) {
SSchedulerCfg scfg = {.maxJobNum = 100};
schedulerInit(&scfg);
tscDebug("starting to initialize TAOS driver, local ep: %s", tsLocalEp);
tscDebug("starting to initialize TAOS driver");
taosSetCoreDump(true);
initTaskQueue();
clientConnRefPool = taosOpenRef(200, destroyTscObj);
clientReqRefPool = taosOpenRef(40960, doDestroyRequest);
clientReqRefPool = taosOpenRef(40960, doDestroyRequest);
taosGetAppName(appInfo.appName, NULL);
pthread_mutex_init(&appInfo.mutex, NULL);
appInfo.pid = taosGetPId();
appInfo.pid = taosGetPId();
appInfo.startTime = taosGetTimestampMs();
appInfo.pInstMap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
appInfo.pInstMap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
tscDebug("client is initialized successfully");
}
@ -269,6 +259,7 @@ int taos_init() {
}
int taos_options_imp(TSDB_OPTION option, const char *str) {
#if 0
SGlobalCfg *cfg = NULL;
switch (option) {
@ -281,7 +272,8 @@ int taos_options_imp(TSDB_OPTION option, const char *str) {
cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION;
tscInfo("set config file directory:%s", str);
} else {
tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, str, tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr);
tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, str,
tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr);
}
break;
@ -296,7 +288,8 @@ int taos_options_imp(TSDB_OPTION option, const char *str) {
cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION;
tscInfo("set shellActivityTimer:%d", tsShellActivityTimer);
} else {
tscWarn("config option:%s, input value:%s, is configured by %s, use %d", cfg->option, str, tsCfgStatusStr[cfg->cfgStatus], *(int32_t *)cfg->ptr);
tscWarn("config option:%s, input value:%s, is configured by %s, use %d", cfg->option, str,
tsCfgStatusStr[cfg->cfgStatus], *(int32_t *)cfg->ptr);
}
break;
@ -313,8 +306,8 @@ int taos_options_imp(TSDB_OPTION option, const char *str) {
if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_OPTION) {
char sep = '.';
if (strlen(tsLocale) == 0) { // locale does not set yet
char* defaultLocale = setlocale(LC_CTYPE, "");
if (strlen(tsLocale) == 0) { // locale does not set yet
char *defaultLocale = setlocale(LC_CTYPE, "");
// The locale of the current OS does not be set correctly, so the default locale cannot be acquired.
// The launch of current system will abort soon.
@ -329,10 +322,10 @@ int taos_options_imp(TSDB_OPTION option, const char *str) {
// set the user specified locale
char *locale = setlocale(LC_CTYPE, str);
if (locale != NULL) { // failed to set the user specified locale
if (locale != NULL) { // failed to set the user specified locale
tscInfo("locale set, prev locale:%s, new locale:%s", tsLocale, locale);
cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION;
} else { // set the user specified locale failed, use default LC_CTYPE as current locale
} else { // set the user specified locale failed, use default LC_CTYPE as current locale
locale = setlocale(LC_CTYPE, tsLocale);
tscInfo("failed to set locale:%s, current locale:%s", str, tsLocale);
}
@ -360,11 +353,12 @@ int taos_options_imp(TSDB_OPTION option, const char *str) {
}
free(charset);
} else { // it may be windows system
} else { // it may be windows system
tscInfo("charset remains:%s", tsCharset);
}
} else {
tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, str, tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr);
tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, str,
tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr);
}
break;
}
@ -394,7 +388,8 @@ int taos_options_imp(TSDB_OPTION option, const char *str) {
tscInfo("charset:%s not valid", str);
}
} else {
tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, str, tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr);
tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, str,
tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr);
}
break;
@ -410,7 +405,8 @@ int taos_options_imp(TSDB_OPTION option, const char *str) {
cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION;
tscDebug("timezone set:%s, input:%s by taos_options", tsTimezone, str);
} else {
tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, str, tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr);
tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, str,
tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr);
}
break;
@ -419,7 +415,7 @@ int taos_options_imp(TSDB_OPTION option, const char *str) {
tscError("Invalid option %d", option);
return -1;
}
#endif
return 0;
}
@ -434,7 +430,7 @@ int taos_options_imp(TSDB_OPTION option, const char *str) {
*/
uint64_t generateRequestId() {
static uint64_t hashId = 0;
static int32_t requestSerialId = 0;
static int32_t requestSerialId = 0;
if (hashId == 0) {
char uid[64] = {0};
@ -448,9 +444,9 @@ uint64_t generateRequestId() {
}
}
int64_t ts = taosGetTimestampMs();
uint64_t pid = taosGetPId();
int32_t val = atomic_add_fetch_32(&requestSerialId, 1);
int64_t ts = taosGetTimestampMs();
uint64_t pid = taosGetPId();
int32_t val = atomic_add_fetch_32(&requestSerialId, 1);
uint64_t id = ((hashId & 0x0FFF) << 52) | ((pid & 0x0FFF) << 40) | ((ts & 0xFFFFFF) << 16) | (val & 0xFFFF);
return id;

65
source/client/src/clientImpl.c
View File

@ -12,7 +12,7 @@
#include "tpagedbuf.h"
#include "tref.h"
static int32_t initEpSetFromCfg(const char* firstEp, const char* secondEp, SCorEpSet* pEpSet);
static int32_t initEpSetFromCfg(const char* ip, uint16_t port, SCorEpSet* pEpSet);
static SMsgSendInfo* buildConnectMsg(SRequestObj* pRequest);
static void destroySendMsgInfo(SMsgSendInfo* pMsgBody);
static void setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp);
@ -81,19 +81,7 @@ TAOS* taos_connect_internal(const char* ip, const char* user, const char* pass,
}
SCorEpSet epSet = {0};
if (ip) {
if (initEpSetFromCfg(ip, NULL, &epSet) < 0) {
return NULL;
}
if (port) {
epSet.epSet.eps[0].port = port;
}
} else {
if (initEpSetFromCfg(tsFirst, tsSecond, &epSet) < 0) {
return NULL;
}
}
initEpSetFromCfg(ip, port, &epSet);
char* key = getClusterKey(user, secretEncrypt, ip, port);
SAppInstInfo** pInst = NULL;
@ -282,32 +270,40 @@ _return:
return pRequest;
}
int initEpSetFromCfg(const char* firstEp, const char* secondEp, SCorEpSet* pEpSet) {
pEpSet->version = 0;
int initEpSetFromCfg(const char* ip, uint16_t port, SCorEpSet* pEpSet) {
SConfigItem* pFirst = cfgGetItem(tscCfg, "firstEp");
SConfigItem* pSecond = cfgGetItem(tscCfg, "secondEp");
SConfigItem* pPort = cfgGetItem(tscCfg, "serverPort");
// init mnode ip set
SEpSet* mgmtEpSet = &(pEpSet->epSet);
mgmtEpSet->numOfEps = 0;
mgmtEpSet->inUse = 0;
pEpSet->version = 0;
if (firstEp && firstEp[0] != 0) {
if (strlen(firstEp) >= TSDB_EP_LEN) {
terrno = TSDB_CODE_TSC_INVALID_FQDN;
return -1;
}
taosGetFqdnPortFromEp(firstEp, &mgmtEpSet->eps[0]);
if (ip != NULL) {
taosGetFqdnPortFromEp(ip, (uint16_t)pPort->i32, &mgmtEpSet->eps[0]);
mgmtEpSet->numOfEps++;
}
if (secondEp && secondEp[0] != 0) {
if (strlen(secondEp) >= TSDB_EP_LEN) {
terrno = TSDB_CODE_TSC_INVALID_FQDN;
return -1;
if (port) {
mgmtEpSet->eps[0].port = port;
}
} else {
if (pFirst->str[0] != 0) {
if (strlen(pFirst->str) >= TSDB_EP_LEN) {
terrno = TSDB_CODE_TSC_INVALID_FQDN;
return -1;
}
taosGetFqdnPortFromEp(pFirst->str, (uint16_t)pPort->i32, &mgmtEpSet->eps[0]);
mgmtEpSet->numOfEps++;
}
if (pSecond->str[0] != 0) {
if (strlen(pSecond->str) >= TSDB_EP_LEN) {
terrno = TSDB_CODE_TSC_INVALID_FQDN;
return -1;
}
taosGetFqdnPortFromEp(pSecond->str, (uint16_t)pPort->i32, &mgmtEpSet->eps[1]);
mgmtEpSet->numOfEps++;
}
taosGetFqdnPortFromEp(secondEp, &mgmtEpSet->eps[mgmtEpSet->numOfEps]);
mgmtEpSet->numOfEps++;
}
if (mgmtEpSet->numOfEps == 0) {
@ -370,7 +366,6 @@ static SMsgSendInfo* buildConnectMsg(SRequestObj* pRequest) {
pMsgSendInfo->fp = handleRequestRspFp[TMSG_INDEX(pMsgSendInfo->msgType)];
pMsgSendInfo->param = pRequest;
SConnectReq connectReq = {0};
STscObj* pObj = pRequest->pTscObj;
@ -398,7 +393,9 @@ static void destroySendMsgInfo(SMsgSendInfo* pMsgBody) {
tfree(pMsgBody->msgInfo.pData);
tfree(pMsgBody);
}
bool persistConnForSpecificMsg(void* parenct, tmsg_t msgType) {
return msgType == TDMT_VND_QUERY_RSP || msgType == TDMT_VND_FETCH_RSP || msgType == TDMT_VND_RES_READY_RSP;
}
void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) {
SMsgSendInfo* pSendInfo = (SMsgSendInfo*)pMsg->ahandle;
assert(pMsg->ahandle != NULL);

7
source/client/src/clientMain.c
View File

@ -7,6 +7,7 @@
#include "tmsg.h"
#include "tglobal.h"
#include "catalog.h"
#include "version.h"
#define TSC_VAR_NOT_RELEASE 1
#define TSC_VAR_RELEASED 0
@ -56,9 +57,7 @@ void taos_cleanup(void) {
}
TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port) {
int32_t p = (port != 0) ? port : tsServerPort;
tscDebug("try to connect to %s:%u, user:%s db:%s", ip, p, user, db);
tscDebug("try to connect to %s:%u, user:%s db:%s", ip, port, user, db);
if (user == NULL) {
user = TSDB_DEFAULT_USER;
}
@ -67,7 +66,7 @@ TAOS *taos_connect(const char *ip, const char *user, const char *pass, const cha
pass = TSDB_DEFAULT_PASS;
}
return taos_connect_internal(ip, user, pass, NULL, db, p);
return taos_connect_internal(ip, user, pass, NULL, db, port);
}
void taos_close(TAOS* taos) {

181
source/client/src/tmq.c
View File

@ -45,22 +45,24 @@ struct tmq_topic_vgroup_list_t {
};
struct tmq_conf_t {
char clientId[256];
char groupId[256];
bool auto_commit;
char clientId[256];
char groupId[256];
int8_t auto_commit;
int8_t resetOffset;
tmq_commit_cb* commit_cb;
/*char* ip;*/
/*uint16_t port;*/
tmq_commit_cb* commit_cb;
};
struct tmq_t {
// conf
char groupId[256];
char clientId[256];
bool autoCommit;
int8_t autoCommit;
SRWLatch lock;
int64_t consumerId;
int32_t epoch;
int32_t resetOffsetCfg;
int64_t status;
tsem_t rspSem;
STscObj* pTscObj;
@ -79,7 +81,6 @@ typedef struct {
// statistics
int64_t pollCnt;
// offset
int64_t committedOffset;
int64_t currentOffset;
// connection info
int32_t vgId;
@ -115,21 +116,17 @@ typedef struct {
} SMqConsumeCbParam;
typedef struct {
tmq_t* tmq;
SMqClientVg* pVg;
int32_t async;
tsem_t rspSem;
} SMqCommitCbParam;
typedef struct {
tmq_t* tmq;
tmq_t* tmq;
/*SMqClientVg* pVg;*/
int32_t async;
tsem_t rspSem;
tmq_resp_err_t rspErr;
} SMqResetOffsetParam;
} SMqCommitCbParam;
tmq_conf_t* tmq_conf_new() {
tmq_conf_t* conf = calloc(1, sizeof(tmq_conf_t));
conf->auto_commit = false;
conf->resetOffset = TMQ_CONF__RESET_OFFSET__LATEST;
return conf;
}
@ -157,6 +154,20 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value
return TMQ_CONF_INVALID;
}
}
if (strcmp(key, "auto.offset.reset") == 0) {
if (strcmp(value, "none") == 0) {
conf->resetOffset = TMQ_CONF__RESET_OFFSET__NONE;
return TMQ_CONF_OK;
} else if (strcmp(value, "earliest") == 0) {
conf->resetOffset = TMQ_CONF__RESET_OFFSET__EARLIEAST;
return TMQ_CONF_OK;
} else if (strcmp(value, "latest") == 0) {
conf->resetOffset = TMQ_CONF__RESET_OFFSET__LATEST;
return TMQ_CONF_OK;
} else {
return TMQ_CONF_INVALID;
}
}
return TMQ_CONF_UNKNOWN;
}
@ -190,14 +201,12 @@ int32_t tmqCommitCb(void* param, const SDataBuf* pMsg, int32_t code) {
if (pParam->tmq->commit_cb) {
pParam->tmq->commit_cb(pParam->tmq, rspErr, NULL, NULL);
}
if (!pParam->async) tsem_post(&pParam->rspSem);
return 0;
}
int32_t tmqResetOffsetCb(void* param, const SDataBuf* pMsg, int32_t code) {
SMqResetOffsetParam* pParam = (SMqResetOffsetParam*)param;
pParam->rspErr = code;
tsem_post(&pParam->rspSem);
if (!pParam->async)
tsem_post(&pParam->rspSem);
else {
tsem_destroy(&pParam->rspSem);
free(param);
}
return 0;
}
@ -216,6 +225,7 @@ tmq_t* tmq_consumer_new(void* conn, tmq_conf_t* conf, char* errstr, int32_t errs
strcpy(pTmq->groupId, conf->groupId);
pTmq->autoCommit = conf->auto_commit;
pTmq->commit_cb = conf->commit_cb;
pTmq->resetOffsetCfg = conf->resetOffset;
tsem_init(&pTmq->rspSem, 0, 0);
pTmq->consumerId = generateRequestId() & (((uint64_t)-1) >> 1);
@ -223,18 +233,40 @@ tmq_t* tmq_consumer_new(void* conn, tmq_conf_t* conf, char* errstr, int32_t errs
return pTmq;
}
tmq_resp_err_t tmq_reset_offset(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets) {
SRequestObj* pRequest = NULL;
tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets, int32_t async) {
// TODO: add read write lock
SRequestObj* pRequest = NULL;
tmq_resp_err_t resp = TMQ_RESP_ERR__SUCCESS;
// build msg
// send to mnode
SMqCMResetOffsetReq req;
req.num = offsets->cnt;
req.offsets = (SMqOffset*)offsets->elems;
SMqCMCommitOffsetReq req;
SArray* pArray = NULL;
if (offsets == NULL) {
pArray = taosArrayInit(0, sizeof(SMqOffset));
for (int i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) {
SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i);
for (int j = 0; j < taosArrayGetSize(pTopic->vgs); j++) {
SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j);
SMqOffset offset;
strcpy(offset.topicName, pTopic->topicName);
strcpy(offset.cgroup, tmq->groupId);
offset.vgId = pVg->vgId;
offset.offset = pVg->currentOffset;
taosArrayPush(pArray, &offset);
}
}
req.num = pArray->size;
req.offsets = pArray->pData;
} else {
req.num = offsets->cnt;
req.offsets = (SMqOffset*)offsets->elems;
}
SCoder encoder;
tCoderInit(&encoder, TD_LITTLE_ENDIAN, NULL, 0, TD_ENCODER);
tEncodeSMqCMResetOffsetReq(&encoder, &req);
tEncodeSMqCMCommitOffsetReq(&encoder, &req);
int32_t tlen = encoder.pos;
void* buf = malloc(tlen);
if (buf == NULL) {
@ -244,32 +276,41 @@ tmq_resp_err_t tmq_reset_offset(tmq_t* tmq, const tmq_topic_vgroup_list_t* offse
tCoderClear(&encoder);
tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, tlen, TD_ENCODER);
tEncodeSMqCMResetOffsetReq(&encoder, &req);
tEncodeSMqCMCommitOffsetReq(&encoder, &req);
tCoderClear(&encoder);
pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_MND_RESET_OFFSET);
pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_MND_MQ_COMMIT_OFFSET);
if (pRequest == NULL) {
tscError("failed to malloc request");
}
SMqResetOffsetParam param = {0};
tsem_init(&param.rspSem, 0, 0);
param.tmq = tmq;
SMqCommitCbParam* pParam = malloc(sizeof(SMqCommitCbParam));
if (pParam == NULL) {
return -1;
}
pParam->tmq = tmq;
tsem_init(&pParam->rspSem, 0, 0);
pRequest->body.requestMsg = (SDataBuf){.pData = buf, .len = tlen};
SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest);
sendInfo->param = &param;
sendInfo->fp = tmqResetOffsetCb;
sendInfo->param = pParam;
sendInfo->fp = tmqCommitCb;
SEpSet epSet = getEpSet_s(&tmq->pTscObj->pAppInfo->mgmtEp);
int64_t transporterId = 0;
asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo);
tsem_wait(&param.rspSem);
tsem_destroy(&param.rspSem);
if (!async) {
tsem_wait(&pParam->rspSem);
resp = pParam->rspErr;
}
return param.rspErr;
if (pArray) {
taosArrayDestroy(pArray);
}
return resp;
}
tmq_resp_err_t tmq_subscribe(tmq_t* tmq, tmq_list_t* topic_list) {
@ -641,8 +682,7 @@ int32_t tmqAskEpCb(void* param, const SDataBuf* pMsg, int32_t code) {
// clang-format off
SMqClientVg clientVg = {
.pollCnt = 0,
.committedOffset = -1,
.currentOffset = -1,
.currentOffset = pVgEp->offset,
.vgId = pVgEp->vgId,
.epSet = pVgEp->epSet
};
@ -708,23 +748,51 @@ END:
return 0;
}
SMqConsumeReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blocking_time, int32_t type, SMqClientTopic* pTopic,
SMqClientVg* pVg) {
tmq_resp_err_t tmq_seek(tmq_t* tmq, const tmq_topic_vgroup_t* offset) {
const SMqOffset* pOffset = &offset->offset;
if (strcmp(pOffset->cgroup, tmq->groupId) != 0) {
return TMQ_RESP_ERR__FAIL;
}
int32_t sz = taosArrayGetSize(tmq->clientTopics);
for (int32_t i = 0; i < sz; i++) {
SMqClientTopic* clientTopic = taosArrayGet(tmq->clientTopics, i);
if (strcmp(clientTopic->topicName, pOffset->topicName) == 0) {
int32_t vgSz = taosArrayGetSize(clientTopic->vgs);
for (int32_t j = 0; j < vgSz; j++) {
SMqClientVg* pVg = taosArrayGet(clientTopic->vgs, j);
if (pVg->vgId == pOffset->vgId) {
pVg->currentOffset = pOffset->offset;
return TMQ_RESP_ERR__SUCCESS;
}
}
}
}
return TMQ_RESP_ERR__FAIL;
}
SMqConsumeReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blocking_time, SMqClientTopic* pTopic, SMqClientVg* pVg) {
int64_t reqOffset;
if (pVg->currentOffset >= 0) {
reqOffset = pVg->currentOffset;
} else {
if (tmq->resetOffsetCfg == TMQ_CONF__RESET_OFFSET__NONE) {
tscError("unable to poll since no committed offset but reset offset is set to none");
return NULL;
}
reqOffset = tmq->resetOffsetCfg;
}
SMqConsumeReq* pReq = malloc(sizeof(SMqConsumeReq));
if (pReq == NULL) {
return NULL;
}
pReq->reqType = type;
strcpy(pReq->topic, pTopic->topicName);
pReq->blockingTime = blocking_time;
pReq->consumerId = tmq->consumerId;
strcpy(pReq->cgroup, tmq->groupId);
if (type == TMQ_REQ_TYPE_COMMIT_ONLY) {
pReq->offset = pVg->currentOffset;
} else {
pReq->offset = pVg->currentOffset + 1;
}
pReq->blockingTime = blocking_time;
pReq->consumerId = tmq->consumerId;
pReq->currentOffset = reqOffset;
pReq->head.vgId = htonl(pVg->vgId);
pReq->head.contLen = htonl(sizeof(SMqConsumeReq));
@ -743,13 +811,13 @@ tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) {
if (taosArrayGetSize(tmq->clientTopics) == 0) {
tscDebug("consumer:%ld poll but not assigned", tmq->consumerId);
printf("over1\n");
/*printf("over1\n");*/
usleep(blocking_time * 1000);
return NULL;
}
SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, tmq->nextTopicIdx);
if (taosArrayGetSize(pTopic->vgs) == 0) {
printf("over2\n");
/*printf("over2\n");*/
usleep(blocking_time * 1000);
return NULL;
}
@ -760,8 +828,7 @@ tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) {
pTopic->nextVgIdx = (pTopic->nextVgIdx + 1) % taosArrayGetSize(pTopic->vgs);
SMqClientVg* pVg = taosArrayGet(pTopic->vgs, pTopic->nextVgIdx);
/*printf("consume vg %d, offset %ld\n", pVg->vgId, pVg->currentOffset);*/
int32_t reqType = tmq->autoCommit ? TMQ_REQ_TYPE_CONSUME_AND_COMMIT : TMQ_REQ_TYPE_CONSUME_ONLY;
SMqConsumeReq* pReq = tmqBuildConsumeReqImpl(tmq, blocking_time, reqType, pTopic, pVg);
SMqConsumeReq* pReq = tmqBuildConsumeReqImpl(tmq, blocking_time, pTopic, pVg);
if (pReq == NULL) {
ASSERT(false);
usleep(blocking_time * 1000);
@ -821,6 +888,7 @@ tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) {
/*return pRequest;*/
}
#if 0
tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* tmq_topic_vgroup_list, int32_t async) {
if (tmq_topic_vgroup_list != NULL) {
// TODO
@ -831,7 +899,7 @@ tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* tmq_topic_v
SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i);
for (int j = 0; j < taosArrayGetSize(pTopic->vgs); j++) {
SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j);
SMqConsumeReq* pReq = tmqBuildConsumeReqImpl(tmq, 0, TMQ_REQ_TYPE_COMMIT_ONLY, pTopic, pVg);
SMqConsumeReq* pReq = tmqBuildConsumeReqImpl(tmq, 0, pTopic, pVg);
SRequestObj* pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_VND_CONSUME);
pRequest->body.requestMsg = (SDataBuf){.pData = pReq, .len = sizeof(SMqConsumeReq)};
@ -858,6 +926,7 @@ tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* tmq_topic_v
return 0;
}
#endif
void tmq_message_destroy(tmq_message_t* tmq_message) {
if (tmq_message == NULL) return;

4
source/client/test/CMakeLists.txt
View File

@ -8,13 +8,13 @@ AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
ADD_EXECUTABLE(clientTest clientTests.cpp)
TARGET_LINK_LIBRARIES(
clientTest
PUBLIC os util common transport parser catalog scheduler function gtest taos qcom
PUBLIC os util common transport parser catalog scheduler function gtest taos qcom config
)
ADD_EXECUTABLE(tmqTest tmqTest.cpp)
TARGET_LINK_LIBRARIES(
tmqTest
PUBLIC os util common transport parser catalog scheduler function gtest taos qcom
PUBLIC os util common transport parser catalog scheduler function gtest taos qcom config
)
TARGET_INCLUDE_DIRECTORIES(

4
source/common/src/tep.c
View File

@ -4,7 +4,7 @@
#include "tglobal.h"
#include "tlockfree.h"
int taosGetFqdnPortFromEp(const char *ep, SEp* pEp) {
int taosGetFqdnPortFromEp(const char *ep, uint16_t defaultPort, SEp* pEp) {
pEp->port = 0;
strcpy(pEp->fqdn, ep);
@ -15,7 +15,7 @@ int taosGetFqdnPortFromEp(const char *ep, SEp* pEp) {
}
if (pEp->port == 0) {
pEp->port = tsServerPort;
pEp->port = defaultPort;
return -1;
}

652
source/common/src/tglobal.c
View File

@ -19,7 +19,6 @@
#include "taosdef.h"
#include "taoserror.h"
#include "tcompare.h"
#include "tconfig.h"
#include "tep.h"
#include "tglobal.h"
#include "tlocale.h"
@ -28,29 +27,9 @@
#include "tutil.h"
#include "ulog.h"
// cluster
char tsFirst[TSDB_EP_LEN] = {0};
char tsSecond[TSDB_EP_LEN] = {0};
char tsLocalFqdn[TSDB_FQDN_LEN] = {0};
char tsLocalEp[TSDB_EP_LEN] = {0}; // Local End Point, hostname:port
uint16_t tsServerPort = 6030;
int32_t tsStatusInterval = 1; // second
int8_t tsEnableTelemetryReporting = 0;
char tsEmail[TSDB_FQDN_LEN] = {0};
int32_t tsNumOfSupportVnodes = 128;
// common
int32_t tsRpcTimer = 300;
int32_t tsRpcMaxTime = 600; // seconds;
int32_t tsRpcForceTcp = 1; // disable this, means query, show command use udp protocol as default
int32_t tsMaxShellConns = 50000;
int32_t tsMaxConnections = 50000;
int32_t tsShellActivityTimer = 3; // second
float tsNumOfThreadsPerCore = 1.0f;
int32_t tsNumOfCommitThreads = 4;
float tsRatioOfQueryCores = 1.0f;
int8_t tsDaylight = 0;
int8_t tsEnableCoreFile = 0;
int32_t tsMaxBinaryDisplayWidth = 30;
int8_t tsEnableSlaveQuery = 1;
int8_t tsEnableAdjustMaster = 1;
@ -131,14 +110,6 @@ int8_t tsDeadLockKillQuery = 0;
// For backward compatibility
bool tsdbForceKeepFile = false;
int32_t tsDiskCfgNum = 0;
#ifndef _STORAGE
SDiskCfg tsDiskCfg[1];
#else
SDiskCfg tsDiskCfg[TFS_MAX_DISKS];
#endif
/*
* minimum scale for whole system, millisecond by default
* for TSDB_TIME_PRECISION_MILLI: 86400000L
@ -148,13 +119,6 @@ SDiskCfg tsDiskCfg[TFS_MAX_DISKS];
int64_t tsTickPerDay[] = {86400000L, 86400000000L, 86400000000000L};
// system info
float tsTotalTmpDirGB = 0;
float tsTotalDataDirGB = 0;
float tsAvailTmpDirectorySpace = 0;
float tsAvailDataDirGB = 0;
float tsUsedDataDirGB = 0;
float tsReservedTmpDirectorySpace = 1.0f;
float tsMinimalDataDirGB = 2.0f;
int32_t tsTotalMemoryMB = 0;
uint32_t tsVersion = 0;
@ -179,24 +143,9 @@ char *qtypeStr[] = {"rpc", "fwd", "wal", "cq", "query"};
static pthread_once_t tsInitGlobalCfgOnce = PTHREAD_ONCE_INIT;
void taosSetAllDebugFlag() {
if (debugFlag != 0) {
mDebugFlag = debugFlag;
dDebugFlag = debugFlag;
vDebugFlag = debugFlag;
jniDebugFlag = debugFlag;
qDebugFlag = debugFlag;
rpcDebugFlag = debugFlag;
uDebugFlag = debugFlag;
sDebugFlag = debugFlag;
wDebugFlag = debugFlag;
tsdbDebugFlag = debugFlag;
cqDebugFlag = debugFlag;
uInfo("all debug flag are set to %d", debugFlag);
}
}
int32_t taosCfgDynamicOptions(char *msg) {
#if 0
char *option, *value;
int32_t olen, vlen;
int32_t vint = 0;
@ -266,130 +215,43 @@ int32_t taosCfgDynamicOptions(char *msg) {
}
}
#endif
return false;
}
void taosAddDataDir(int index, char *v1, int level, int primary) {
tstrncpy(tsDiskCfg[index].dir, v1, TSDB_FILENAME_LEN);
tsDiskCfg[index].level = level;
tsDiskCfg[index].primary = primary;
uTrace("dataDir:%s, level:%d primary:%d is configured", v1, level, primary);
}
// void taosAddDataDir(int index, char *v1, int level, int primary) {
// tstrncpy(tsDiskCfg[index].dir, v1, TSDB_FILENAME_LEN);
// tsDiskCfg[index].level = level;
// tsDiskCfg[index].primary = primary;
// uTrace("dataDir:%s, level:%d primary:%d is configured", v1, level, primary);
// }
#ifndef _STORAGE
void taosReadDataDirCfg(char *v1, char *v2, char *v3) {
if (tsDiskCfgNum == 1) {
SDiskCfg *cfg = &tsDiskCfg[0];
uInfo("dataDir:%s, level:%d primary:%d is replaced by %s", cfg->dir, cfg->level, cfg->primary, v1);
}
taosAddDataDir(0, v1, 0, 1);
tsDiskCfgNum = 1;
}
// void taosReadDataDirCfg(char *v1, char *v2, char *v3) {
// if (tsDiskCfgNum == 1) {
// SDiskCfg *cfg = &tsDiskCfg[0];
// uInfo("dataDir:%s, level:%d primary:%d is replaced by %s", cfg->dir, cfg->level, cfg->primary, v1);
// }
// taosAddDataDir(0, v1, 0, 1);
// tsDiskCfgNum = 1;
// }
void taosPrintDataDirCfg() {
for (int i = 0; i < tsDiskCfgNum; ++i) {
SDiskCfg *cfg = &tsDiskCfg[i];
uInfo(" dataDir: %s", cfg->dir);
}
}
// void taosPrintDataDirCfg() {
// for (int i = 0; i < tsDiskCfgNum; ++i) {
// SDiskCfg *cfg = &tsDiskCfg[i];
// uInfo(" dataDir: %s", cfg->dir);
// }
// }
#endif
static void taosCheckDataDirCfg() {
if (tsDiskCfgNum <= 0) {
taosAddDataDir(0, tsDataDir, 0, 1);
tsDiskCfgNum = 1;
uTrace("dataDir:%s, level:0 primary:1 is configured by default", tsDataDir);
}
}
static void doInitGlobalConfig(void) {
osInit();
srand(taosSafeRand());
#if 0
SGlobalCfg cfg = {0};
// ip address
cfg.option = "firstEp";
cfg.ptr = tsFirst;
cfg.valType = TAOS_CFG_VTYPE_STRING;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT;
cfg.minValue = 0;
cfg.maxValue = 0;
cfg.ptrLength = TSDB_EP_LEN;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
cfg.option = "secondEp";
cfg.ptr = tsSecond;
cfg.valType = TAOS_CFG_VTYPE_STRING;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT;
cfg.minValue = 0;
cfg.maxValue = 0;
cfg.ptrLength = TSDB_EP_LEN;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
cfg.option = "fqdn";
cfg.ptr = tsLocalFqdn;
cfg.valType = TAOS_CFG_VTYPE_STRING;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT;
cfg.minValue = 0;
cfg.maxValue = 0;
cfg.ptrLength = TSDB_FQDN_LEN;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
// port
cfg.option = "serverPort";
cfg.ptr = &tsServerPort;
cfg.valType = TAOS_CFG_VTYPE_UINT16;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT;
cfg.minValue = 1;
cfg.maxValue = 65056;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
cfg.option = "supportVnodes";
cfg.ptr = &tsNumOfSupportVnodes;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT;
cfg.minValue = 0;
cfg.maxValue = 65536;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
// directory
cfg.option = "configDir";
cfg.ptr = configDir;
cfg.valType = TAOS_CFG_VTYPE_DIRECTORY;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT;
cfg.minValue = 0;
cfg.maxValue = 0;
cfg.ptrLength = TSDB_FILENAME_LEN;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
cfg.option = "logDir";
cfg.ptr = tsLogDir;
cfg.valType = TAOS_CFG_VTYPE_DIRECTORY;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT | TSDB_CFG_CTYPE_B_LOG;
cfg.minValue = 0;
cfg.maxValue = 0;
cfg.ptrLength = TSDB_FILENAME_LEN;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
cfg.option = "scriptDir";
cfg.ptr = tsScriptDir;
cfg.valType = TAOS_CFG_VTYPE_DIRECTORY;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT;
cfg.minValue = 0;
cfg.maxValue = 0;
cfg.ptrLength = TSDB_FILENAME_LEN;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
cfg.option = "dataDir";
cfg.ptr = tsDataDir;
@ -401,36 +263,6 @@ static void doInitGlobalConfig(void) {
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
// dnode configs
cfg.option = "numOfThreadsPerCore";
cfg.ptr = &tsNumOfThreadsPerCore;
cfg.valType = TAOS_CFG_VTYPE_FLOAT;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT;
cfg.minValue = 0;
cfg.maxValue = 10;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
cfg.option = "numOfCommitThreads";
cfg.ptr = &tsNumOfCommitThreads;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG;
cfg.minValue = 1;
cfg.maxValue = 100;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
cfg.option = "ratioOfQueryCores";
cfg.ptr = &tsRatioOfQueryCores;
cfg.valType = TAOS_CFG_VTYPE_FLOAT;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG;
cfg.minValue = 0.0f;
cfg.maxValue = 2.0f;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
cfg.option = "maxNumOfDistinctRes";
cfg.ptr = &tsMaxNumOfDistinctResults;
@ -442,77 +274,6 @@ static void doInitGlobalConfig(void) {
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
cfg.option = "telemetryReporting";
cfg.ptr = &tsEnableTelemetryReporting;
cfg.valType = TAOS_CFG_VTYPE_INT8;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = 0;
cfg.maxValue = 1;
cfg.ptrLength = 1;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
// timer
cfg.option = "maxTmrCtrl";
cfg.ptr = &tsMaxTmrCtrl;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = 8;
cfg.maxValue = 2048;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
cfg.option = "rpcTimer";
cfg.ptr = &tsRpcTimer;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT;
cfg.minValue = 100;
cfg.maxValue = 3000;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_MS;
taosAddConfigOption(cfg);
cfg.option = "rpcForceTcp";
cfg.ptr = &tsRpcForceTcp;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT;
cfg.minValue = 0;
cfg.maxValue = 1;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
cfg.option = "rpcMaxTime";
cfg.ptr = &tsRpcMaxTime;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT;
cfg.minValue = 100;
cfg.maxValue = 7200;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_SECOND;
taosAddConfigOption(cfg);
cfg.option = "statusInterval";
cfg.ptr = &tsStatusInterval;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = 1;
cfg.maxValue = 10;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_SECOND;
taosAddConfigOption(cfg);
cfg.option = "shellActivityTimer";
cfg.ptr = &tsShellActivityTimer;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT;
cfg.minValue = 1;
cfg.maxValue = 120;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_SECOND;
taosAddConfigOption(cfg);
cfg.option = "minSlidingTime";
cfg.ptr = &tsMinSlidingTime;
cfg.valType = TAOS_CFG_VTYPE_INT32;
@ -653,87 +414,7 @@ static void doInitGlobalConfig(void) {
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
// locale & charset
cfg.option = "timezone";
cfg.ptr = tsTimezone;
cfg.valType = TAOS_CFG_VTYPE_STRING;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT;
cfg.minValue = 0;
cfg.maxValue = 0;
cfg.ptrLength = TSDB_TIMEZONE_LEN;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
cfg.option = "locale";
cfg.ptr = tsLocale;
cfg.valType = TAOS_CFG_VTYPE_STRING;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT;
cfg.minValue = 0;
cfg.maxValue = 0;
cfg.ptrLength = TSDB_LOCALE_LEN;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
cfg.option = "charset";
cfg.ptr = tsCharset;
cfg.valType = TAOS_CFG_VTYPE_STRING;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT;
cfg.minValue = 0;
cfg.maxValue = 0;
cfg.ptrLength = TSDB_LOCALE_LEN;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
// connect configs
cfg.option = "maxShellConns";
cfg.ptr = &tsMaxShellConns;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = 10;
cfg.maxValue = 50000000;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
cfg.option = "maxConnections";
cfg.ptr = &tsMaxConnections;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = 1;
cfg.maxValue = 100000;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
cfg.option = "minimalLogDirGB";
cfg.ptr = &tsMinimalLogDirGB;
cfg.valType = TAOS_CFG_VTYPE_FLOAT;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = 0.001f;
cfg.maxValue = 10000000;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_GB;
taosAddConfigOption(cfg);
cfg.option = "minimalTmpDirGB";
cfg.ptr = &tsReservedTmpDirectorySpace;
cfg.valType = TAOS_CFG_VTYPE_FLOAT;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = 0.001f;
cfg.maxValue = 10000000;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_GB;
taosAddConfigOption(cfg);
cfg.option = "minimalDataDirGB";
cfg.ptr = &tsMinimalDataDirGB;
cfg.valType = TAOS_CFG_VTYPE_FLOAT;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = 0.001f;
cfg.maxValue = 10000000;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_GB;
taosAddConfigOption(cfg);
cfg.option = "slaveQuery";
cfg.ptr = &tsEnableSlaveQuery;
@ -745,176 +426,7 @@ static void doInitGlobalConfig(void) {
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
// debug flag
cfg.option = "numOfLogLines";
cfg.ptr = &tsNumOfLogLines;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT;
cfg.minValue = 1000;
cfg.maxValue = 2000000000;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
cfg.option = "logKeepDays";
cfg.ptr = &tsLogKeepDays;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT;
cfg.minValue = -365000;
cfg.maxValue = 365000;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
cfg.option = "asyncLog";
cfg.ptr = &tsAsyncLog;
cfg.valType = TAOS_CFG_VTYPE_INT8;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT;
cfg.minValue = 0;
cfg.maxValue = 1;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
cfg.option = "debugFlag";
cfg.ptr = &debugFlag;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT;
cfg.minValue = 0;
cfg.maxValue = 255;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
cfg.option = "mDebugFlag";
cfg.ptr = &mDebugFlag;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG;
cfg.minValue = 0;
cfg.maxValue = 255;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
cfg.option = "dDebugFlag";
cfg.ptr = &dDebugFlag;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG;
cfg.minValue = 0;
cfg.maxValue = 255;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
cfg.option = "sDebugFlag";
cfg.ptr = &sDebugFlag;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG;
cfg.minValue = 0;
cfg.maxValue = 255;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
cfg.option = "wDebugFlag";
cfg.ptr = &wDebugFlag;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG;
cfg.minValue = 0;
cfg.maxValue = 255;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
cfg.option = "rpcDebugFlag";
cfg.ptr = &rpcDebugFlag;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT;
cfg.minValue = 0;
cfg.maxValue = 255;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
cfg.option = "tmrDebugFlag";
cfg.ptr = &tmrDebugFlag;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT;
cfg.minValue = 0;
cfg.maxValue = 255;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
cfg.option = "cDebugFlag";
cfg.ptr = &cDebugFlag;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT;
cfg.minValue = 0;
cfg.maxValue = 255;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
cfg.option = "jniDebugFlag";
cfg.ptr = &jniDebugFlag;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT;
cfg.minValue = 0;
cfg.maxValue = 255;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
cfg.option = "uDebugFlag";
cfg.ptr = &uDebugFlag;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT;
cfg.minValue = 0;
cfg.maxValue = 255;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
cfg.option = "qDebugFlag";
cfg.ptr = &qDebugFlag;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT;
cfg.minValue = 0;
cfg.maxValue = 255;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
cfg.option = "vDebugFlag";
cfg.ptr = &vDebugFlag;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG;
cfg.minValue = 0;
cfg.maxValue = 255;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
cfg.option = "tsdbDebugFlag";
cfg.ptr = &tsdbDebugFlag;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG;
cfg.minValue = 0;
cfg.maxValue = 255;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
cfg.option = "cqDebugFlag";
cfg.ptr = &cqDebugFlag;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG;
cfg.minValue = 0;
cfg.maxValue = 255;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
cfg.option = "enableRecordSql";
cfg.ptr = &tsTscEnableRecordSql;
@ -926,57 +438,6 @@ static void doInitGlobalConfig(void) {
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
cfg.option = "enableCoreFile";
cfg.ptr = &tsEnableCoreFile;
cfg.valType = TAOS_CFG_VTYPE_INT8;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG;
cfg.minValue = 0;
cfg.maxValue = 1;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
// version info
cfg.option = "gitinfo";
cfg.ptr = gitinfo;
cfg.valType = TAOS_CFG_VTYPE_STRING;
cfg.cfgType = TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT;
cfg.minValue = 0;
cfg.maxValue = 0;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
cfg.option = "gitinfoOfInternal";
cfg.ptr = gitinfoOfInternal;
cfg.valType = TAOS_CFG_VTYPE_STRING;
cfg.cfgType = TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT;
cfg.minValue = 0;
cfg.maxValue = 0;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
cfg.option = "buildinfo";
cfg.ptr = buildinfo;
cfg.valType = TAOS_CFG_VTYPE_STRING;
cfg.cfgType = TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT;
cfg.minValue = 0;
cfg.maxValue = 0;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
cfg.option = "version";
cfg.ptr = version;
cfg.valType = TAOS_CFG_VTYPE_STRING;
cfg.cfgType = TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT;
cfg.minValue = 0;
cfg.maxValue = 0;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
cfg.option = "maxBinaryDisplayWidth";
cfg.ptr = &tsMaxBinaryDisplayWidth;
cfg.valType = TAOS_CFG_VTYPE_INT32;
@ -1025,7 +486,7 @@ static void doInitGlobalConfig(void) {
cfg.valType = TAOS_CFG_VTYPE_DOUBLE;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG;
cfg.minValue = MIN_FLOAT;
cfg.maxValue = MAX_FLOAT;
cfg.maxValue = 100000;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
@ -1035,8 +496,8 @@ static void doInitGlobalConfig(void) {
cfg.ptr = &dPrecision;
cfg.valType = TAOS_CFG_VTYPE_DOUBLE;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG;
cfg.minValue = MIN_FLOAT;
cfg.maxValue = MAX_FLOAT;
cfg.minValue = 100000;
cfg.maxValue = 0;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
@ -1064,67 +525,12 @@ static void doInitGlobalConfig(void) {
#else
// assert(tsGlobalConfigNum == TSDB_CFG_MAX_NUM - 5);
#endif
#endif
}
void taosInitGlobalCfg() { pthread_once(&tsInitGlobalCfgOnce, doInitGlobalConfig); }
int32_t taosCheckAndPrintCfg() {
SEp ep = {0};
if (debugFlag & DEBUG_TRACE || debugFlag & DEBUG_DEBUG || debugFlag & DEBUG_DUMP) {
taosSetAllDebugFlag();
}
if (tsLocalFqdn[0] == 0) {
taosGetFqdn(tsLocalFqdn);
}
snprintf(tsLocalEp, sizeof(tsLocalEp), "%s:%u", tsLocalFqdn, tsServerPort);
uInfo("localEp is: %s", tsLocalEp);
if (tsFirst[0] == 0) {
strcpy(tsFirst, tsLocalEp);
} else {
taosGetFqdnPortFromEp(tsFirst, &ep);
snprintf(tsFirst, sizeof(tsFirst), "%s:%u", ep.fqdn, ep.port);
}
if (tsSecond[0] == 0) {
strcpy(tsSecond, tsLocalEp);
} else {
taosGetFqdnPortFromEp(tsSecond, &ep);
snprintf(tsSecond, sizeof(tsSecond), "%s:%u", ep.fqdn, ep.port);
}
taosCheckDataDirCfg();
if (taosDirExist(tsTempDir) != 0) {
return -1;
}
taosGetSystemInfo();
tsSetLocale();
SGlobalCfg *cfg_timezone = taosGetConfigOption("timezone");
if (cfg_timezone && cfg_timezone->cfgStatus == TAOS_CFG_CSTATUS_FILE) {
tsSetTimeZone();
}
if (tsNumOfCores <= 0) {
tsNumOfCores = 1;
}
if (tsQueryBufferSize >= 0) {
tsQueryBufferSizeBytes = tsQueryBufferSize * 1048576UL;
}
uInfo(" check global cfg completed");
uInfo("==================================");
taosPrintCfg();
return 0;
}
/*
* alter dnode 1 balance "vnode:1-dnode:2"
*/

1
source/common/src/tlocale.c
View File

@ -17,7 +17,6 @@
#include "os.h"
#include "ulog.h"
#include "tglobal.h"
#include "tconfig.h"
#include "tutil.h"
/**

84
source/common/src/tmsg.c
View File

@ -285,7 +285,7 @@ int32_t tDeserializeSClientHbBatchRsp(void *buf, int32_t bufLen, SClientHbBatchR
int32_t tSerializeSVCreateTbReq(void **buf, SVCreateTbReq *pReq) {
int32_t tlen = 0;
tlen += taosEncodeFixedU64(buf, pReq->ver);
tlen += taosEncodeFixedI64(buf, pReq->ver);
tlen += taosEncodeString(buf, pReq->name);
tlen += taosEncodeFixedU32(buf, pReq->ttl);
tlen += taosEncodeFixedU32(buf, pReq->keep);
@ -330,7 +330,7 @@ int32_t tSerializeSVCreateTbReq(void **buf, SVCreateTbReq *pReq) {
}
void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) {
buf = taosDecodeFixedU64(buf, &(pReq->ver));
buf = taosDecodeFixedI64(buf, &(pReq->ver));
buf = taosDecodeString(buf, &(pReq->name));
buf = taosDecodeFixedU32(buf, &(pReq->ttl));
buf = taosDecodeFixedU32(buf, &(pReq->keep));
@ -380,7 +380,7 @@ void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) {
int32_t tSerializeSVCreateTbBatchReq(void **buf, SVCreateTbBatchReq *pReq) {
int32_t tlen = 0;
tlen += taosEncodeFixedU64(buf, pReq->ver);
tlen += taosEncodeFixedI64(buf, pReq->ver);
tlen += taosEncodeFixedU32(buf, taosArrayGetSize(pReq->pArray));
for (size_t i = 0; i < taosArrayGetSize(pReq->pArray); i++) {
SVCreateTbReq *pCreateTbReq = taosArrayGet(pReq->pArray, i);
@ -393,7 +393,7 @@ int32_t tSerializeSVCreateTbBatchReq(void **buf, SVCreateTbBatchReq *pReq) {
void *tDeserializeSVCreateTbBatchReq(void *buf, SVCreateTbBatchReq *pReq) {
uint32_t nsize = 0;
buf = taosDecodeFixedU64(buf, &pReq->ver);
buf = taosDecodeFixedI64(buf, &pReq->ver);
buf = taosDecodeFixedU32(buf, &nsize);
pReq->pArray = taosArrayInit(nsize, sizeof(SVCreateTbReq));
for (size_t i = 0; i < nsize; i++) {
@ -407,14 +407,14 @@ void *tDeserializeSVCreateTbBatchReq(void *buf, SVCreateTbBatchReq *pReq) {
int32_t tSerializeSVDropTbReq(void **buf, SVDropTbReq *pReq) {
int32_t tlen = 0;
tlen += taosEncodeFixedU64(buf, pReq->ver);
tlen += taosEncodeFixedI64(buf, pReq->ver);
tlen += taosEncodeString(buf, pReq->name);
tlen += taosEncodeFixedU8(buf, pReq->type);
return tlen;
}
void *tDeserializeSVDropTbReq(void *buf, SVDropTbReq *pReq) {
buf = taosDecodeFixedU64(buf, &pReq->ver);
buf = taosDecodeFixedI64(buf, &pReq->ver);
buf = taosDecodeString(buf, &pReq->name);
buf = taosDecodeFixedU8(buf, &pReq->type);
return buf;
@ -1393,7 +1393,7 @@ int32_t tSerializeSDropDbRsp(void *buf, int32_t bufLen, SDropDbRsp *pRsp) {
if (tStartEncode(&encoder) < 0) return -1;
if (tEncodeCStr(&encoder, pRsp->db) < 0) return -1;
if (tEncodeU64(&encoder, pRsp->uid) < 0) return -1;
if (tEncodeI64(&encoder, pRsp->uid) < 0) return -1;
tEndEncode(&encoder);
int32_t tlen = encoder.pos;
@ -1407,7 +1407,7 @@ int32_t tDeserializeSDropDbRsp(void *buf, int32_t bufLen, SDropDbRsp *pRsp) {
if (tStartDecode(&decoder) < 0) return -1;
if (tDecodeCStrTo(&decoder, pRsp->db) < 0) return -1;
if (tDecodeU64(&decoder, &pRsp->uid) < 0) return -1;
if (tDecodeI64(&decoder, &pRsp->uid) < 0) return -1;
tEndDecode(&decoder);
tCoderClear(&decoder);
@ -1468,7 +1468,7 @@ int32_t tDeserializeSSyncDbReq(void *buf, int32_t bufLen, SSyncDbReq *pReq) {
static int32_t tSerializeSUseDbRspImp(SCoder *pEncoder, SUseDbRsp *pRsp) {
if (tEncodeCStr(pEncoder, pRsp->db) < 0) return -1;
if (tEncodeU64(pEncoder, pRsp->uid) < 0) return -1;
if (tEncodeI64(pEncoder, pRsp->uid) < 0) return -1;
if (tEncodeI32(pEncoder, pRsp->vgVersion) < 0) return -1;
if (tEncodeI32(pEncoder, pRsp->vgNum) < 0) return -1;
if (tEncodeI8(pEncoder, pRsp->hashMethod) < 0) return -1;
@ -1518,7 +1518,7 @@ int32_t tSerializeSUseDbBatchRsp(void *buf, int32_t bufLen, SUseDbBatchRsp *pRsp
int32_t tDeserializeSUseDbRspImp(SCoder *pDecoder, SUseDbRsp *pRsp) {
if (tDecodeCStrTo(pDecoder, pRsp->db) < 0) return -1;
if (tDecodeU64(pDecoder, &pRsp->uid) < 0) return -1;
if (tDecodeI64(pDecoder, &pRsp->uid) < 0) return -1;
if (tDecodeI32(pDecoder, &pRsp->vgVersion) < 0) return -1;
if (tDecodeI32(pDecoder, &pRsp->vgNum) < 0) return -1;
if (tDecodeI8(pDecoder, &pRsp->hashMethod) < 0) return -1;
@ -1661,7 +1661,7 @@ static int32_t tEncodeSTableMetaRsp(SCoder *pEncoder, STableMetaRsp *pRsp) {
if (tEncodeCStr(pEncoder, pRsp->tbName) < 0) return -1;
if (tEncodeCStr(pEncoder, pRsp->stbName) < 0) return -1;
if (tEncodeCStr(pEncoder, pRsp->dbFName) < 0) return -1;
if (tEncodeU64(pEncoder, pRsp->dbId) < 0) return -1;
if (tEncodeI64(pEncoder, pRsp->dbId) < 0) return -1;
if (tEncodeI32(pEncoder, pRsp->numOfTags) < 0) return -1;
if (tEncodeI32(pEncoder, pRsp->numOfColumns) < 0) return -1;
if (tEncodeI8(pEncoder, pRsp->precision) < 0) return -1;
@ -1684,7 +1684,7 @@ static int32_t tDecodeSTableMetaRsp(SCoder *pDecoder, STableMetaRsp *pRsp) {
if (tDecodeCStrTo(pDecoder, pRsp->tbName) < 0) return -1;
if (tDecodeCStrTo(pDecoder, pRsp->stbName) < 0) return -1;
if (tDecodeCStrTo(pDecoder, pRsp->dbFName) < 0) return -1;
if (tDecodeU64(pDecoder, &pRsp->dbId) < 0) return -1;
if (tDecodeI64(pDecoder, &pRsp->dbId) < 0) return -1;
if (tDecodeI32(pDecoder, &pRsp->numOfTags) < 0) return -1;
if (tDecodeI32(pDecoder, &pRsp->numOfColumns) < 0) return -1;
if (tDecodeI8(pDecoder, &pRsp->precision) < 0) return -1;
@ -2093,7 +2093,7 @@ int32_t tSerializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq *pR
if (tEncodeI32(&encoder, pReq->vgId) < 0) return -1;
if (tEncodeI32(&encoder, pReq->dnodeId) < 0) return -1;
if (tEncodeCStr(&encoder, pReq->db) < 0) return -1;
if (tEncodeU64(&encoder, pReq->dbUid) < 0) return -1;
if (tEncodeI64(&encoder, pReq->dbUid) < 0) return -1;
if (tEncodeI32(&encoder, pReq->vgVersion) < 0) return -1;
if (tEncodeI32(&encoder, pReq->cacheBlockSize) < 0) return -1;
if (tEncodeI32(&encoder, pReq->totalBlocks) < 0) return -1;
@ -2133,7 +2133,7 @@ int32_t tDeserializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq *
if (tDecodeI32(&decoder, &pReq->vgId) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->dnodeId) < 0) return -1;
if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1;
if (tDecodeU64(&decoder, &pReq->dbUid) < 0) return -1;
if (tDecodeI64(&decoder, &pReq->dbUid) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->vgVersion) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->cacheBlockSize) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->totalBlocks) < 0) return -1;
@ -2171,7 +2171,7 @@ int32_t tSerializeSDropVnodeReq(void *buf, int32_t bufLen, SDropVnodeReq *pReq)
if (tStartEncode(&encoder) < 0) return -1;
if (tEncodeI32(&encoder, pReq->vgId) < 0) return -1;
if (tEncodeI32(&encoder, pReq->dnodeId) < 0) return -1;
if (tEncodeU64(&encoder, pReq->dbUid) < 0) return -1;
if (tEncodeI64(&encoder, pReq->dbUid) < 0) return -1;
if (tEncodeCStr(&encoder, pReq->db) < 0) return -1;
tEndEncode(&encoder);
@ -2187,7 +2187,7 @@ int32_t tDeserializeSDropVnodeReq(void *buf, int32_t bufLen, SDropVnodeReq *pReq
if (tStartDecode(&decoder) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->vgId) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->dnodeId) < 0) return -1;
if (tDecodeU64(&decoder, &pReq->dbUid) < 0) return -1;
if (tDecodeI64(&decoder, &pReq->dbUid) < 0) return -1;
if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1;
tEndDecode(&decoder);
@ -2356,35 +2356,7 @@ int32_t tDecodeSMqOffset(SCoder *decoder, SMqOffset *pOffset) {
return 0;
}
int32_t tEncodeSMqVgOffsets(SCoder *encoder, const SMqVgOffsets *pOffsets) {
if (tStartEncode(encoder) < 0) return -1;
if (tEncodeI32(encoder, pOffsets->vgId) < 0) return -1;
int32_t sz = taosArrayGetSize(pOffsets->offsets);
if (tEncodeI32(encoder, sz) < 0) return -1;
for (int32_t i = 0; i < sz; i++) {
SMqOffset *offset = taosArrayGet(pOffsets->offsets, i);
if (tEncodeSMqOffset(encoder, offset) < 0) return -1;
}
tEndEncode(encoder);
return encoder->pos;
}
int32_t tDecodeSMqVgOffsets(SCoder *decoder, SMqVgOffsets *pOffsets) {
int32_t sz;
if (tStartDecode(decoder) < 0) return -1;
if (tDecodeI32(decoder, &pOffsets->vgId) < 0) return -1;
if (tDecodeI32(decoder, &sz) < 0) return -1;
pOffsets->offsets = taosArrayInit(sz, sizeof(SMqOffset));
for (int32_t i = 0; i < sz; i++) {
SMqOffset offset;
if (tDecodeSMqOffset(decoder, &offset) < 0) return -1;
taosArrayPush(pOffsets->offsets, &offset);
}
tEndDecode(decoder);
return 0;
}
int32_t tEncodeSMqCMResetOffsetReq(SCoder *encoder, const SMqCMResetOffsetReq *pReq) {
int32_t tEncodeSMqCMCommitOffsetReq(SCoder *encoder, const SMqCMCommitOffsetReq *pReq) {
if (tStartEncode(encoder) < 0) return -1;
if (tEncodeI32(encoder, pReq->num) < 0) return -1;
for (int32_t i = 0; i < pReq->num; i++) {
@ -2394,7 +2366,7 @@ int32_t tEncodeSMqCMResetOffsetReq(SCoder *encoder, const SMqCMResetOffsetReq *p
return encoder->pos;
}
int32_t tDecodeSMqCMResetOffsetReq(SCoder *decoder, SMqCMResetOffsetReq *pReq) {
int32_t tDecodeSMqCMCommitOffsetReq(SCoder *decoder, SMqCMCommitOffsetReq *pReq) {
if (tStartDecode(decoder) < 0) return -1;
if (tDecodeI32(decoder, &pReq->num) < 0) return -1;
pReq->offsets = TCODER_MALLOC(pReq->num * sizeof(SMqOffset), decoder);
@ -2405,23 +2377,3 @@ int32_t tDecodeSMqCMResetOffsetReq(SCoder *decoder, SMqCMResetOffsetReq *pReq) {
tEndDecode(decoder);
return 0;
}
#if 0
int32_t tEncodeSMqMVResetOffsetReq(SCoder *encoder, const SMqMVResetOffsetReq *pReq) {
if (tEncodeI64(encoder, pReq->leftForVer) < 0) return -1;
for (int32_t i = 0; i < pReq->num; i++) {
tEncodeSMqOffset(encoder, &pReq->offsets[i]);
}
return encoder->pos;
}
int32_t tDecodeSMqMVResetOffsetReq(SCoder *decoder, SMqMVResetOffsetReq *pReq) {
if (tDecodeI32(decoder, &pReq->num) < 0) return -1;
pReq->offsets = TCODER_MALLOC(pReq->num * sizeof(SMqOffset), decoder);
if (pReq->offsets == NULL) return -1;
for (int32_t i = 0; i < pReq->num; i++) {
tDecodeSMqOffset(decoder, &pReq->offsets[i]);
}
return 0;
}
#endif

9
source/dnode/mgmt/daemon/CMakeLists.txt
View File

@ -1,8 +1,9 @@
aux_source_directory(src DAEMON_SRC)
add_executable(taosd ${DAEMON_SRC})
target_link_libraries(
target_include_directories(
taosd
PUBLIC dnode
PUBLIC util
PUBLIC os
PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
)
target_link_libraries(taosd dnode config util os)

47
source/dnode/mgmt/daemon/inc/dmnInt.h Normal file
View File

@ -0,0 +1,47 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_DMN_INT_H_
#define _TD_DMN_INT_H_
#include "config.h"
#include "dnode.h"
#include "taoserror.h"
#include "tglobal.h"
#include "ulog.h"
#include "version.h"
#ifdef __cplusplus
extern "C" {
#endif
int32_t dmnAddLogCfg(SConfig *pCfg);
int32_t dmnInitLog(const char *cfgDir, const char *envFile, const char *apolloUrl);
int32_t dmnLoadCfg(SConfig *pConfig, const char *inputCfgDir, const char *envFile, const char *apolloUrl);
SConfig *dmnReadCfg(const char *cfgDir, const char *envFile, const char *apolloUrl);
SDnodeEnvCfg dmnGetEnvCfg(SConfig *pCfg);
SDnodeObjCfg dmnGetObjCfg(SConfig *pCfg);
void dmnDumpCfg(SConfig *pCfg);
void dmnPrintVersion();
void dmnGenerateGrant();
#ifdef __cplusplus
}
#endif
#endif /*_TD_DMN_INT_H_*/

218
source/dnode/mgmt/daemon/src/daemon.c
View File

@ -1,218 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "dnode.h"
#include "os.h"
#include "tconfig.h"
#include "tglobal.h"
#include "tnote.h"
#include "ulog.h"
static struct {
bool stop;
bool dumpConfig;
bool generateGrant;
bool printAuth;
bool printVersion;
char configDir[PATH_MAX];
} dmn = {0};
void dmnSigintHandle(int signum, void *info, void *ctx) {
uInfo("singal:%d is received", signum);
dmn.stop = true;
}
void dmnSetSignalHandle() {
taosSetSignal(SIGTERM, dmnSigintHandle);
taosSetSignal(SIGHUP, dmnSigintHandle);
taosSetSignal(SIGINT, dmnSigintHandle);
taosSetSignal(SIGABRT, dmnSigintHandle);
taosSetSignal(SIGBREAK, dmnSigintHandle);
}
int dmnParseOption(int argc, char const *argv[]) {
tstrncpy(dmn.configDir, "/etc/taos", PATH_MAX);
for (int i = 1; i < argc; ++i) {
if (strcmp(argv[i], "-c") == 0) {
if (i < argc - 1) {
if (strlen(argv[++i]) >= PATH_MAX) {
printf("config file path overflow");
return -1;
}
tstrncpy(dmn.configDir, argv[i], PATH_MAX);
} else {
printf("'-c' requires a parameter, default is %s\n", configDir);
return -1;
}
} else if (strcmp(argv[i], "-C") == 0) {
dmn.dumpConfig = true;
} else if (strcmp(argv[i], "-k") == 0) {
dmn.generateGrant = true;
} else if (strcmp(argv[i], "-A") == 0) {
dmn.printAuth = true;
} else if (strcmp(argv[i], "-V") == 0) {
dmn.printVersion = true;
} else {
}
}
return 0;
}
void dmnGenerateGrant() {
#if 0
grantParseParameter();
#endif
}
void dmnPrintVersion() {
#ifdef TD_ENTERPRISE
char *releaseName = "enterprise";
#else
char *releaseName = "community";
#endif
printf("%s version: %s compatible_version: %s\n", releaseName, version, compatible_version);
printf("gitinfo: %s\n", gitinfo);
printf("gitinfoI: %s\n", gitinfoOfInternal);
printf("builuInfo: %s\n", buildinfo);
}
int dmnReadConfig(const char *path) {
tstrncpy(configDir, dmn.configDir, PATH_MAX);
taosInitGlobalCfg();
taosReadGlobalLogCfg();
if (taosMkDir(tsLogDir) != 0) {
printf("failed to create dir: %s, reason: %s\n", tsLogDir, strerror(errno));
return -1;
}
char temp[PATH_MAX];
snprintf(temp, PATH_MAX, "%s/taosdlog", tsLogDir);
if (taosInitLog(temp, tsNumOfLogLines, 1) != 0) {
printf("failed to init log file\n");
return -1;
}
if (taosInitNotes() != 0) {
printf("failed to init log file\n");
return -1;
}
if (taosReadCfgFromFile() != 0) {
uError("failed to read config");
return -1;
}
if (taosCheckAndPrintCfg() != 0) {
uError("failed to check config");
return -1;
}
taosSetCoreDump(tsEnableCoreFile);
return 0;
}
void dmnDumpConfig() { taosDumpGlobalCfg(); }
void dmnWaitSignal() {
dmnSetSignalHandle();
while (!dmn.stop) {
taosMsleep(100);
}
}
void dnmInitEnvCfg(SDnodeEnvCfg *pCfg) {
pCfg->sver = 30000000; // 3.0.0.0
pCfg->numOfCores = tsNumOfCores;
pCfg->numOfCommitThreads = tsNumOfCommitThreads;
pCfg->enableTelem = 0;
tstrncpy(pCfg->timezone, tsTimezone, TSDB_TIMEZONE_LEN);
tstrncpy(pCfg->locale, tsLocale, TSDB_LOCALE_LEN);
tstrncpy(pCfg->charset, tsCharset, TSDB_LOCALE_LEN);
tstrncpy(pCfg->buildinfo, buildinfo, 64);
tstrncpy(pCfg->gitinfo, gitinfo, 48);
}
void dmnInitObjCfg(SDnodeObjCfg *pCfg) {
pCfg->numOfSupportVnodes = tsNumOfSupportVnodes;
pCfg->statusInterval = tsStatusInterval;
pCfg->numOfThreadsPerCore = tsNumOfThreadsPerCore;
pCfg->ratioOfQueryCores = tsRatioOfQueryCores;
pCfg->maxShellConns = tsMaxShellConns;
pCfg->shellActivityTimer = tsShellActivityTimer;
pCfg->serverPort = tsServerPort;
tstrncpy(pCfg->dataDir, tsDataDir, TSDB_FILENAME_LEN);
tstrncpy(pCfg->localEp, tsLocalEp, TSDB_EP_LEN);
tstrncpy(pCfg->localFqdn, tsLocalFqdn, TSDB_FQDN_LEN);
tstrncpy(pCfg->firstEp, tsFirst, TSDB_EP_LEN);
}
int dmnRunDnode() {
SDnodeEnvCfg envCfg = {0};
SDnodeObjCfg objCfg = {0};
dnmInitEnvCfg(&envCfg);
dmnInitObjCfg(&objCfg);
if (dndInit(&envCfg) != 0) {
uInfo("Failed to start TDengine, please check the log at %s", tsLogDir);
return -1;
}
SDnode *pDnode = dndCreate(&objCfg);
if (pDnode == NULL) {
uInfo("Failed to start TDengine, please check the log at %s", tsLogDir);
return -1;
}
uInfo("Started TDengine service successfully.");
dmnWaitSignal();
uInfo("TDengine is shut down!");
dndClose(pDnode);
dndCleanup();
taosCloseLog();
return 0;
}
int main(int argc, char const *argv[]) {
if (dmnParseOption(argc, argv) != 0) {
return -1;
}
if (dmn.generateGrant) {
dmnGenerateGrant();
return 0;
}
if (dmn.printVersion) {
dmnPrintVersion();
return 0;
}
if (dmnReadConfig(dmn.configDir) != 0) {
return -1;
}
if (dmn.dumpConfig) {
dmnDumpConfig();
return 0;
}
return dmnRunDnode();
}

208
source/dnode/mgmt/daemon/src/dmnCfg.c Normal file
View File

@ -0,0 +1,208 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "dmnInt.h"
#include "tlocale.h"
#include "ttimezone.h"
static int32_t dmnAddEpCfg(SConfig *pCfg) {
char defaultFqdn[TSDB_FQDN_LEN] = {0};
if (taosGetFqdn(defaultFqdn) != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
if (cfgAddString(pCfg, "fqdn", defaultFqdn) != 0) return -1;
int32_t defaultServerPort = 6030;
if (cfgAddInt32(pCfg, "serverPort", defaultServerPort, 1, 65056) != 0) return -1;
char defaultFirstEp[TSDB_EP_LEN] = {0};
char defaultSecondEp[TSDB_EP_LEN] = {0};
snprintf(defaultFirstEp, TSDB_EP_LEN, "%s:%d", defaultFqdn, defaultServerPort);
snprintf(defaultSecondEp, TSDB_EP_LEN, "%s:%d", defaultFqdn, defaultServerPort);
if (cfgAddString(pCfg, "firstEp", defaultFirstEp) != 0) return -1;
if (cfgAddString(pCfg, "secondEp", defaultSecondEp) != 0) return -1;
return 0;
}
static int32_t dmnAddDirCfg(SConfig *pCfg) {
if (cfgAddDir(pCfg, "dataDir", tsDataDir) != 0) return -1;
if (cfgAddDir(pCfg, "tempDir", tsTempDir) != 0) return -1;
if (cfgAddFloat(pCfg, "minimalDataDirGB", 2.0f, 0.001f, 10000000) != 0) return -1;
if (cfgAddFloat(pCfg, "minimalTempDirGB", 1.0f, 0.001f, 10000000) != 0) return -1;
return 0;
}
static int32_t dmnCheckDirCfg(SConfig *pCfg) {
tstrncpy(tsDataDir, cfgGetItem(pCfg, "dataDir")->str, PATH_MAX);
tstrncpy(tsTempDir, cfgGetItem(pCfg, "tempDir")->str, PATH_MAX);
tsDataSpace.reserved = cfgGetItem(pCfg, "minimalDataDirGB")->fval;
tsTempSpace.reserved = cfgGetItem(pCfg, "minimalTempDirGB")->fval;
return 0;
}
static int32_t dmnAddVersionCfg(SConfig *pCfg) {
if (cfgAddString(pCfg, "buildinfo", buildinfo) != 0) return -1;
if (cfgAddString(pCfg, "gitinfo", gitinfo) != 0) return -1;
if (cfgAddString(pCfg, "version", version) != 0) return -1;
return 0;
}
static int32_t dmnAddDnodeCfg(SConfig *pCfg) {
if (dmnAddEpCfg(pCfg) != 0) return -1;
if (dmnAddDirCfg(pCfg) != 0) return -1;
if (dmnAddVersionCfg(pCfg) != 0) return -1;
if (cfgAddTimezone(pCfg, "timezone", "") != 0) return -1;
if (cfgAddLocale(pCfg, "locale", "") != 0) return -1;
if (cfgAddCharset(pCfg, "charset", "") != 0) return -1;
if (cfgAddInt32(pCfg, "numOfCores", 2, 1, 100000) != 0) return -1;
if (cfgAddInt32(pCfg, "numOfCommitThreads", 4, 1, 1000) != 0) return -1;
if (cfgAddBool(pCfg, "telemetryReporting", 0) != 0) return -1;
if (cfgAddBool(pCfg, "enableCoreFile", 0) != 0) return -1;
if (cfgAddInt32(pCfg, "supportVnodes", 256, 0, 65536) != 0) return -1;
if (cfgAddInt32(pCfg, "statusInterval", 1, 1, 30) != 0) return -1;
if (cfgAddFloat(pCfg, "numOfThreadsPerCore", 1, 0, 10) != 0) return -1;
if (cfgAddFloat(pCfg, "ratioOfQueryCores", 1, 0, 5) != 0) return -1;
if (cfgAddInt32(pCfg, "maxShellConns", 50000, 10, 50000000) != 0) return -1;
if (cfgAddInt32(pCfg, "shellActivityTimer", 3, 1, 120) != 0) return -1;
return 0;
}
static int32_t dmnCheckCfg(SConfig *pCfg) {
bool enableCore = cfgGetItem(pCfg, "enableCoreFile")->bval;
taosSetCoreDump(enableCore);
if (dmnCheckDirCfg(pCfg) != 0) {
return -1;
}
taosGetSystemInfo();
tsSetTimeZone();
tsSetLocale();
if (tsNumOfCores <= 0) {
tsNumOfCores = 1;
}
if (tsQueryBufferSize >= 0) {
tsQueryBufferSizeBytes = tsQueryBufferSize * 1048576UL;
}
return 0;
}
SConfig *dmnReadCfg(const char *cfgDir, const char *envFile, const char *apolloUrl) {
SConfig *pCfg = cfgInit();
if (pCfg == NULL) return NULL;
if (dmnAddLogCfg(pCfg) != 0) {
uError("failed to add log cfg since %s", terrstr());
cfgCleanup(pCfg);
return NULL;
}
if (dmnAddDnodeCfg(pCfg) != 0) {
uError("failed to init dnode cfg since %s", terrstr());
cfgCleanup(pCfg);
return NULL;
}
if (dmnLoadCfg(pCfg, cfgDir, envFile, apolloUrl) != 0) {
uError("failed to load cfg since %s", terrstr());
cfgCleanup(pCfg);
return NULL;
}
if (dmnCheckCfg(pCfg) != 0) {
uError("failed to check cfg since %s", terrstr());
cfgCleanup(pCfg);
return NULL;
}
cfgDumpCfg(pCfg);
return pCfg;
}
void dmnDumpCfg(SConfig *pCfg) {
printf("taos global config:\n");
printf("==================================\n");
SConfigItem *pItem = cfgIterate(pCfg, NULL);
while (pItem != NULL) {
switch (pItem->dtype) {
case CFG_DTYPE_BOOL:
printf("cfg:%s, value:%u src:%s\n", pItem->name, pItem->bval, cfgStypeStr(pItem->stype));
break;
case CFG_DTYPE_INT32:
printf("cfg:%s, value:%d src:%s\n", pItem->name, pItem->i32, cfgStypeStr(pItem->stype));
break;
case CFG_DTYPE_INT64:
printf("cfg:%s, value:%" PRId64 " src:%s\n", pItem->name, pItem->i64, cfgStypeStr(pItem->stype));
break;
case CFG_DTYPE_FLOAT:
printf("cfg:%s, value:%f src:%s\n", pItem->name, pItem->fval, cfgStypeStr(pItem->stype));
break;
case CFG_DTYPE_STRING:
case CFG_DTYPE_IPSTR:
case CFG_DTYPE_DIR:
case CFG_DTYPE_LOCALE:
case CFG_DTYPE_CHARSET:
case CFG_DTYPE_TIMEZONE:
printf("cfg:%s, value:%s src:%s\n", pItem->name, pItem->str, cfgStypeStr(pItem->stype));
break;
}
pItem = cfgIterate(pCfg, pItem);
}
}
SDnodeEnvCfg dmnGetEnvCfg(SConfig *pCfg) {
SDnodeEnvCfg envCfg = {0};
const char *vstr = cfgGetItem(pCfg, "version")->str;
envCfg.sver = 30000000;
tstrncpy(envCfg.buildinfo, cfgGetItem(pCfg, "buildinfo")->str, sizeof(envCfg.buildinfo));
tstrncpy(envCfg.gitinfo, cfgGetItem(pCfg, "gitinfo")->str, sizeof(envCfg.gitinfo));
tstrncpy(envCfg.timezone, cfgGetItem(pCfg, "timezone")->str, sizeof(envCfg.timezone));
tstrncpy(envCfg.locale, cfgGetItem(pCfg, "locale")->str, sizeof(envCfg.locale));
tstrncpy(envCfg.charset, cfgGetItem(pCfg, "charset")->str, sizeof(envCfg.charset));
envCfg.numOfCores = cfgGetItem(pCfg, "numOfCores")->i32;
envCfg.numOfCommitThreads = (uint16_t)cfgGetItem(pCfg, "numOfCommitThreads")->i32;
envCfg.enableTelem = cfgGetItem(pCfg, "telemetryReporting")->bval;
return envCfg;
}
SDnodeObjCfg dmnGetObjCfg(SConfig *pCfg) {
SDnodeObjCfg objCfg = {0};
objCfg.numOfSupportVnodes = cfgGetItem(pCfg, "supportVnodes")->i32;
objCfg.statusInterval = cfgGetItem(pCfg, "statusInterval")->i32;
objCfg.numOfThreadsPerCore = cfgGetItem(pCfg, "numOfThreadsPerCore")->fval;
objCfg.ratioOfQueryCores = cfgGetItem(pCfg, "ratioOfQueryCores")->fval;
objCfg.maxShellConns = cfgGetItem(pCfg, "maxShellConns")->i32;
objCfg.shellActivityTimer = cfgGetItem(pCfg, "shellActivityTimer")->i32;
tstrncpy(objCfg.dataDir, cfgGetItem(pCfg, "dataDir")->str, sizeof(objCfg.dataDir));
tstrncpy(objCfg.firstEp, cfgGetItem(pCfg, "firstEp")->str, sizeof(objCfg.firstEp));
tstrncpy(objCfg.secondEp, cfgGetItem(pCfg, "secondEp")->str, sizeof(objCfg.firstEp));
objCfg.serverPort = (uint16_t)cfgGetItem(pCfg, "serverPort")->i32;
tstrncpy(objCfg.localFqdn, cfgGetItem(pCfg, "fqdn")->str, sizeof(objCfg.localFqdn));
snprintf(objCfg.localEp, sizeof(objCfg.localEp), "%s:%u", objCfg.localFqdn, objCfg.serverPort);
return objCfg;
}

132
source/dnode/mgmt/daemon/src/dmnLog.c Normal file
View File

@ -0,0 +1,132 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "dmnInt.h"
int32_t dmnAddLogCfg(SConfig *pCfg) {
if (cfgAddDir(pCfg, "logDir", "/var/log/taos") != 0) return -1;
if (cfgAddFloat(pCfg, "minimalLogDirGB", 1.0f, 0.001f, 10000000) != 0) return -1;
if (cfgAddBool(pCfg, "asyncLog", 1) != 0) return -1;
if (cfgAddInt32(pCfg, "numOfLogLines", 10000000, 1000, 2000000000) != 0) return -1;
if (cfgAddInt32(pCfg, "logKeepDays", 0, -365000, 365000) != 0) return -1;
if (cfgAddInt32(pCfg, "debugFlag", 0, 0, 255) != 0) return -1;
if (cfgAddInt32(pCfg, "dDebugFlag", 0, 0, 255) != 0) return -1;
if (cfgAddInt32(pCfg, "vDebugFlag", 0, 0, 255) != 0) return -1;
if (cfgAddInt32(pCfg, "mDebugFlag", 0, 0, 255) != 0) return -1;
if (cfgAddInt32(pCfg, "cDebugFlag", 0, 0, 255) != 0) return -1;
if (cfgAddInt32(pCfg, "jniDebugFlag", 0, 0, 255) != 0) return -1;
if (cfgAddInt32(pCfg, "tmrDebugFlag", 0, 0, 255) != 0) return -1;
if (cfgAddInt32(pCfg, "uDebugFlag", 0, 0, 255) != 0) return -1;
if (cfgAddInt32(pCfg, "rpcDebugFlag", 0, 0, 255) != 0) return -1;
if (cfgAddInt32(pCfg, "qDebugFlag", 0, 0, 255) != 0) return -1;
if (cfgAddInt32(pCfg, "wDebugFlag", 0, 0, 255) != 0) return -1;
if (cfgAddInt32(pCfg, "sDebugFlag", 0, 0, 255) != 0) return -1;
if (cfgAddInt32(pCfg, "tsdbDebugFlag", 0, 0, 255) != 0) return -1;
if (cfgAddInt32(pCfg, "tqDebugFlag", 0, 0, 255) != 0) return -1;
if (cfgAddInt32(pCfg, "fsDebugFlag", 0, 0, 255) != 0) return -1;
return 0;
}
int32_t dmnSetLogCfg(SConfig *pCfg) {
tstrncpy(tsLogDir, cfgGetItem(pCfg, "logDir")->str, PATH_MAX);
tsLogSpace.reserved = cfgGetItem(pCfg, "minimalLogDirGB")->fval;
tsAsyncLog = cfgGetItem(pCfg, "asyncLog")->bval;
tsNumOfLogLines = cfgGetItem(pCfg, "numOfLogLines")->i32;
tsLogKeepDays = cfgGetItem(pCfg, "logKeepDays")->i32;
dDebugFlag = cfgGetItem(pCfg, "dDebugFlag")->i32;
vDebugFlag = cfgGetItem(pCfg, "vDebugFlag")->i32;
mDebugFlag = cfgGetItem(pCfg, "mDebugFlag")->i32;
cDebugFlag = cfgGetItem(pCfg, "cDebugFlag")->i32;
jniDebugFlag = cfgGetItem(pCfg, "jniDebugFlag")->i32;
tmrDebugFlag = cfgGetItem(pCfg, "tmrDebugFlag")->i32;
uDebugFlag = cfgGetItem(pCfg, "uDebugFlag")->i32;
rpcDebugFlag = cfgGetItem(pCfg, "rpcDebugFlag")->i32;
qDebugFlag = cfgGetItem(pCfg, "qDebugFlag")->i32;
wDebugFlag = cfgGetItem(pCfg, "wDebugFlag")->i32;
sDebugFlag = cfgGetItem(pCfg, "sDebugFlag")->i32;
tsdbDebugFlag = cfgGetItem(pCfg, "tsdbDebugFlag")->i32;
tqDebugFlag = cfgGetItem(pCfg, "tqDebugFlag")->i32;
fsDebugFlag = cfgGetItem(pCfg, "fsDebugFlag")->i32;
int32_t debugFlag = cfgGetItem(pCfg, "debugFlag")->i32;
taosSetAllDebugFlag(debugFlag);
return 0;
}
int32_t dmnInitLog(const char *cfgDir, const char *envFile, const char *apolloUrl) {
SConfig *pCfg = cfgInit();
if (pCfg == NULL) return -1;
if (dmnAddLogCfg(pCfg) != 0) {
printf("failed to add log cfg since %s\n", terrstr());
cfgCleanup(pCfg);
return -1;
}
if (dmnLoadCfg(pCfg, cfgDir, envFile, apolloUrl) != 0) {
printf("failed to load log cfg since %s\n", terrstr());
cfgCleanup(pCfg);
return -1;
}
if (dmnSetLogCfg(pCfg) != 0) {
printf("failed to set log cfg since %s\n", terrstr());
cfgCleanup(pCfg);
return -1;
}
if (taosInitLog("taosdlog", 1) != 0) {
printf("failed to init log file since %s\n", terrstr());
cfgCleanup(pCfg);
return -1;
}
cfgCleanup(pCfg);
return 0;
}
int32_t dmnLoadCfg(SConfig *pConfig, const char *inputCfgDir, const char *envFile, const char *apolloUrl) {
char configDir[PATH_MAX] = {0};
char configFile[PATH_MAX + 100] = {0};
taosExpandDir(inputCfgDir, configDir, PATH_MAX);
snprintf(configFile, sizeof(configFile), "%s" TD_DIRSEP "taos.cfg", configDir);
if (cfgLoad(pConfig, CFG_STYPE_APOLLO_URL, apolloUrl) != 0) {
uError("failed to load from apollo url:%s since %s\n", apolloUrl, terrstr());
return -1;
}
if (cfgLoad(pConfig, CFG_STYPE_CFG_FILE, configFile) != 0) {
if (cfgLoad(pConfig, CFG_STYPE_CFG_FILE, configDir) != 0) {
uError("failed to load from config file:%s since %s\n", configFile, terrstr());
return -1;
}
}
if (cfgLoad(pConfig, CFG_STYPE_ENV_FILE, envFile) != 0) {
uError("failed to load from env file:%s since %s\n", envFile, terrstr());
return -1;
}
if (cfgLoad(pConfig, CFG_STYPE_ENV_VAR, NULL) != 0) {
uError("failed to load from global env variables since %s\n", terrstr());
return -1;
}
return 0;
}

133
source/dnode/mgmt/daemon/src/dmnMain.c Normal file
View File

@ -0,0 +1,133 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "dmnInt.h"
static struct {
bool stop;
bool dumpConfig;
bool generateGrant;
bool printAuth;
bool printVersion;
char envFile[PATH_MAX];
char apolloUrl[PATH_MAX];
} dmn = {0};
static void dmnSigintHandle(int signum, void *info, void *ctx) {
uInfo("singal:%d is received", signum);
dmn.stop = true;
}
static void dmnSetSignalHandle() {
taosSetSignal(SIGTERM, dmnSigintHandle);
taosSetSignal(SIGHUP, dmnSigintHandle);
taosSetSignal(SIGINT, dmnSigintHandle);
taosSetSignal(SIGABRT, dmnSigintHandle);
taosSetSignal(SIGBREAK, dmnSigintHandle);
}
static void dmnWaitSignal() {
dmnSetSignalHandle();
while (!dmn.stop) {
taosMsleep(100);
}
}
static int32_t dmnParseOption(int32_t argc, char const *argv[]) {
for (int32_t i = 1; i < argc; ++i) {
if (strcmp(argv[i], "-c") == 0) {
if (i < argc - 1) {
if (strlen(argv[++i]) >= PATH_MAX) {
printf("config file path overflow");
return -1;
}
tstrncpy(configDir, argv[i], PATH_MAX);
} else {
printf("'-c' requires a parameter, default is %s\n", configDir);
return -1;
}
} else if (strcmp(argv[i], "-C") == 0) {
dmn.dumpConfig = true;
} else if (strcmp(argv[i], "-k") == 0) {
dmn.generateGrant = true;
} else if (strcmp(argv[i], "-V") == 0) {
dmn.printVersion = true;
} else {
}
}
return 0;
}
int32_t dmnRunDnode(SConfig *pCfg) {
SDnodeEnvCfg envCfg = dmnGetEnvCfg(pCfg);
if (dndInit(&envCfg) != 0) {
uInfo("Failed to start TDengine, please check the log");
return -1;
}
SDnodeObjCfg objCfg = dmnGetObjCfg(pCfg);
SDnode *pDnode = dndCreate(&objCfg);
if (pDnode == NULL) {
uInfo("Failed to start TDengine, please check the log");
return -1;
}
uInfo("Started TDengine service successfully.");
dmnWaitSignal();
uInfo("TDengine is shut down!");
dndClose(pDnode);
dndCleanup();
taosCloseLog();
return 0;
}
int main(int argc, char const *argv[]) {
if (dmnParseOption(argc, argv) != 0) {
return -1;
}
if (dmn.generateGrant) {
dmnGenerateGrant();
return 0;
}
if (dmn.printVersion) {
dmnPrintVersion();
return 0;
}
if (dmnInitLog(configDir, dmn.envFile, dmn.apolloUrl) != 0) {
return -1;
}
SConfig *pCfg = dmnReadCfg(configDir, dmn.envFile, dmn.apolloUrl);
if (pCfg == NULL) {
uInfo("Failed to start TDengine since read config error");
return -1;
}
if (dmn.dumpConfig) {
dmnDumpCfg(pCfg);
cfgCleanup(pCfg);
return 0;
}
int32_t code = dmnRunDnode(pCfg);
cfgCleanup(pCfg);
return code;
}

35
source/dnode/mgmt/daemon/src/dmnUtil.c Normal file
View File

@ -0,0 +1,35 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "dmnInt.h"
void dmnGenerateGrant() {
#if 0
grantParseParameter();
#endif
}
void dmnPrintVersion() {
#ifdef TD_ENTERPRISE
char *releaseName = "enterprise";
#else
char *releaseName = "community";
#endif
printf("%s version: %s compatible_version: %s\n", releaseName, version, compatible_version);
printf("gitinfo: %s\n", gitinfo);
printf("gitinfoI: %s\n", gitinfoOfInternal);
printf("builuInfo: %s\n", buildinfo);
}

12
source/dnode/mgmt/impl/src/dndEnv.c
View File

@ -325,18 +325,6 @@ void taosGetDisk() {
SDiskSize diskSize = tfsGetSize(pTfs);
tfsUpdateSize(&fsMeta);
tsTotalDataDirGB = (float)(fsMeta.total / unit);
tsUsedDataDirGB = (float)(fsMeta.used / unit);
tsAvailDataDirGB = (float)(fsMeta.avail / unit);
if (taosGetDiskSize(tsLogDir, &diskSize) == 0) {
tsTotalLogDirGB = (float)(diskSize.total / unit);
tsAvailLogDirGB = (float)(diskSize.avail / unit);
}
if (taosGetDiskSize(tsTempDir, &diskSize) == 0) {
tsTotalTmpDirGB = (float)(diskSize.total / unit);
tsAvailTmpDirectorySpace = (float)(diskSize.avail / unit);
}
#endif
}

2
source/dnode/mgmt/impl/src/dndMgmt.c
View File

@ -296,7 +296,7 @@ PRASE_DNODE_OVER:
if (taosArrayGetSize(pMgmt->pDnodeEps) == 0) {
SDnodeEp dnodeEp = {0};
dnodeEp.isMnode = 1;
taosGetFqdnPortFromEp(pDnode->cfg.firstEp, &dnodeEp.ep);
taosGetFqdnPortFromEp(pDnode->cfg.firstEp, pDnode->cfg.serverPort, &dnodeEp.ep);
taosArrayPush(pMgmt->pDnodeEps, &dnodeEp);
}

11
source/dnode/mgmt/impl/src/dndTransport.c
View File

@ -25,8 +25,8 @@
#include "dndMnode.h"
#include "dndVnodes.h"
#define INTERNAL_USER "_dnd"
#define INTERNAL_CKEY "_key"
#define INTERNAL_USER "_dnd"
#define INTERNAL_CKEY "_key"
#define INTERNAL_SECRET "_pwd"
static void dndInitMsgFp(STransMgmt *pMgmt) {
@ -113,6 +113,7 @@ static void dndInitMsgFp(STransMgmt *pMgmt) {
pMgmt->msgFp[TMSG_INDEX(TDMT_MND_ALTER_TOPIC)] = dndProcessMnodeWriteMsg;
pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_TOPIC)] = dndProcessMnodeWriteMsg;
pMgmt->msgFp[TMSG_INDEX(TDMT_MND_SUBSCRIBE)] = dndProcessMnodeWriteMsg;
pMgmt->msgFp[TMSG_INDEX(TDMT_MND_MQ_COMMIT_OFFSET)] = dndProcessMnodeWriteMsg;
/*pMgmt->msgFp[TMSG_INDEX(TDMT_VND_SUBSCRIBE_RSP)] = dndProcessMnodeWriteMsg;*/
pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_SET_CONN_RSP)] = dndProcessMnodeWriteMsg;
pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_REB_RSP)] = dndProcessMnodeWriteMsg;
@ -155,7 +156,7 @@ static void dndInitMsgFp(STransMgmt *pMgmt) {
}
static void dndProcessResponse(void *parent, SRpcMsg *pRsp, SEpSet *pEpSet) {
SDnode * pDnode = parent;
SDnode *pDnode = parent;
STransMgmt *pMgmt = &pDnode->tmgmt;
tmsg_t msgType = pRsp->msgType;
@ -219,7 +220,7 @@ static void dndCleanupClient(SDnode *pDnode) {
}
static void dndProcessRequest(void *param, SRpcMsg *pReq, SEpSet *pEpSet) {
SDnode * pDnode = param;
SDnode *pDnode = param;
STransMgmt *pMgmt = &pDnode->tmgmt;
tmsg_t msgType = pReq->msgType;
@ -313,7 +314,7 @@ static int32_t dndRetrieveUserAuthInfo(void *parent, char *user, char *spi, char
SAuthReq authReq = {0};
tstrncpy(authReq.user, user, TSDB_USER_LEN);
int32_t contLen = tSerializeSAuthReq(NULL, 0, &authReq);
void * pReq = rpcMallocCont(contLen);
void *pReq = rpcMallocCont(contLen);
tSerializeSAuthReq(pReq, contLen, &authReq);
SRpcMsg rpcMsg = {.pCont = pReq, .contLen = contLen, .msgType = TDMT_MND_AUTH, .ahandle = (void *)9528};

2
source/dnode/mgmt/impl/test/sut/inc/sut.h
View File

@ -20,7 +20,7 @@
#include "os.h"
#include "dnode.h"
#include "tconfig.h"
#include "tmsg.h"
#include "tdataformat.h"
#include "tglobal.h"
#include "tmsg.h"

7
source/dnode/mgmt/impl/test/sut/src/sut.cpp
View File

@ -28,16 +28,13 @@ void Testbase::InitLog(const char* path) {
wDebugFlag = 0;
sDebugFlag = 0;
tsdbDebugFlag = 0;
cqDebugFlag = 0;
tscEmbeddedInUtil = 1;
tsAsyncLog = 0;
taosRemoveDir(path);
taosMkDir(path);
char temp[PATH_MAX];
snprintf(temp, PATH_MAX, "%s/taosdlog", path);
if (taosInitLog(temp, tsNumOfLogLines, 1) != 0) {
tstrncpy(tsLogDir, path, PATH_MAX);
if (taosInitLog("taosdlog", 1) != 0) {
printf("failed to init log file\n");
}
}

45
source/dnode/mnode/impl/inc/mndDef.h
View File

@ -123,6 +123,7 @@ typedef enum {
TRN_TYPE_DROP_TOPIC = 1015,
TRN_TYPE_SUBSCRIBE = 1016,
TRN_TYPE_REBALANCE = 1017,
TRN_TYPE_COMMIT_OFFSET = 1018,
TRN_TYPE_BASIC_SCOPE_END,
TRN_TYPE_GLOBAL_SCOPE = 2000,
TRN_TYPE_CREATE_DNODE = 2001,
@ -176,7 +177,7 @@ typedef struct {
SArray* undoActions;
int64_t createdTime;
int64_t lastExecTime;
uint64_t dbUid;
int64_t dbUid;
char dbname[TSDB_DB_FNAME_LEN];
char lastError[TSDB_TRANS_ERROR_LEN];
} STrans;
@ -304,16 +305,16 @@ typedef struct {
} SDbCfg;
typedef struct {
char name[TSDB_DB_FNAME_LEN];
char acct[TSDB_USER_LEN];
char createUser[TSDB_USER_LEN];
int64_t createdTime;
int64_t updateTime;
uint64_t uid;
int32_t cfgVersion;
int32_t vgVersion;
int8_t hashMethod; // default is 1
SDbCfg cfg;
char name[TSDB_DB_FNAME_LEN];
char acct[TSDB_USER_LEN];
char createUser[TSDB_USER_LEN];
int64_t createdTime;
int64_t updateTime;
int64_t uid;
int32_t cfgVersion;
int32_t vgVersion;
int8_t hashMethod; // default is 1
SDbCfg cfg;
} SDbObj;
typedef struct {
@ -346,8 +347,8 @@ typedef struct {
char db[TSDB_DB_FNAME_LEN];
int64_t createdTime;
int64_t updateTime;
uint64_t uid;
uint64_t dbUid;
int64_t uid;
int64_t dbUid;
int32_t version;
int32_t nextColId;
int32_t numOfColumns;
@ -465,6 +466,24 @@ static FORCE_INLINE void tDeleteSMqSubConsumer(SMqSubConsumer* pSubConsumer) {
}
}
typedef struct {
char key[TSDB_PARTITION_KEY_LEN];
int64_t offset;
} SMqOffsetObj;
static FORCE_INLINE int32_t tEncodeSMqOffsetObj(void** buf, const SMqOffsetObj* pOffset) {
int32_t tlen = 0;
tlen += taosEncodeString(buf, pOffset->key);
tlen += taosEncodeFixedI64(buf, pOffset->offset);
return tlen;
}
static FORCE_INLINE void* tDecodeSMqOffsetObj(void* buf, SMqOffsetObj* pOffset) {
buf = taosDecodeStringTo(buf, pOffset->key);
buf = taosDecodeFixedI64(buf, &pOffset->offset);
return buf;
}
typedef struct {
char key[TSDB_SUBSCRIBE_KEY_LEN];
int32_t status;

10
source/dnode/mnode/impl/inc/mndInt.h
View File

@ -56,12 +56,9 @@ typedef struct {
} SProfileMgmt;
typedef struct {
int8_t enable;
pthread_mutex_t lock;
pthread_cond_t cond;
volatile int32_t exit;
pthread_t thread;
char email[TSDB_FQDN_LEN];
bool enable;
SRWLatch lock;
char email[TSDB_FQDN_LEN];
} STelemMgmt;
typedef struct {
@ -81,6 +78,7 @@ typedef struct SMnode {
tmr_h timer;
tmr_h transTimer;
tmr_h mqTimer;
tmr_h telemTimer;
char *path;
SMnodeCfg cfg;
int64_t checkTime;

44
source/dnode/mnode/impl/inc/mndOffset.h Normal file
View File

@ -0,0 +1,44 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_MND_OFFSET_H_
#define _TD_MND_OFFSET_H_
#include "mndInt.h"
#ifdef __cplusplus
extern "C" {
#endif
int32_t mndInitOffset(SMnode *pMnode);
void mndCleanupOffset(SMnode *pMnode);
SMqOffsetObj *mndAcquireOffset(SMnode *pMnode, const char *key);
void mndReleaseOffset(SMnode *pMnode, SMqOffsetObj *pOffset);
SSdbRaw *mndOffsetActionEncode(SMqOffsetObj *pOffset);
SSdbRow *mndOffsetActionDecode(SSdbRaw *pRaw);
int32_t mndCreateOffset(STrans *pTrans, const char *cgroup, const char *topicName, const SArray *vgs);
static FORCE_INLINE int32_t mndMakePartitionKey(char *key, const char *cgroup, const char *topicName, int32_t vgId) {
return snprintf(key, TSDB_PARTITION_KEY_LEN, "%d:%s:%s", vgId, cgroup, topicName);
}
#ifdef __cplusplus
}
#endif
#endif /*_TD_MND_OFFSET_H_*/

2
source/dnode/mnode/impl/src/mndCluster.c
View File

@ -52,7 +52,7 @@ int32_t mndGetClusterName(SMnode *pMnode, char *clusterName, int32_t len) {
SSdb *pSdb = pMnode->pSdb;
SClusterObj *pCluster = sdbAcquire(pSdb, SDB_CLUSTER, &pMnode->clusterId);
if (pCluster = NULL) {
if (pCluster == NULL) {
return -1;
}

224
source/dnode/mnode/impl/src/mndOffset.c Normal file
View File

@ -0,0 +1,224 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "mndOffset.h"
#include "mndAuth.h"
#include "mndDb.h"
#include "mndDnode.h"
#include "mndMnode.h"
#include "mndShow.h"
#include "mndStb.h"
#include "mndTrans.h"
#include "mndUser.h"
#include "mndVgroup.h"
#include "tname.h"
#define MND_OFFSET_VER_NUMBER 1
#define MND_OFFSET_RESERVE_SIZE 64
static int32_t mndOffsetActionInsert(SSdb *pSdb, SMqOffsetObj *pOffset);
static int32_t mndOffsetActionDelete(SSdb *pSdb, SMqOffsetObj *pOffset);
static int32_t mndOffsetActionUpdate(SSdb *pSdb, SMqOffsetObj *pOffset, SMqOffsetObj *pNewOffset);
static int32_t mndProcessCommitOffsetReq(SMnodeMsg *pReq);
int32_t mndInitOffset(SMnode *pMnode) {
SSdbTable table = {.sdbType = SDB_OFFSET,
.keyType = SDB_KEY_BINARY,
.encodeFp = (SdbEncodeFp)mndOffsetActionEncode,
.decodeFp = (SdbDecodeFp)mndOffsetActionDecode,
.insertFp = (SdbInsertFp)mndOffsetActionInsert,
.updateFp = (SdbUpdateFp)mndOffsetActionUpdate,
.deleteFp = (SdbDeleteFp)mndOffsetActionDelete};
mndSetMsgHandle(pMnode, TDMT_MND_MQ_COMMIT_OFFSET, mndProcessCommitOffsetReq);
return sdbSetTable(pMnode->pSdb, table);
}
void mndCleanupOffset(SMnode *pMnode) {}
SSdbRaw *mndOffsetActionEncode(SMqOffsetObj *pOffset) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
void *buf = NULL;
int32_t tlen = tEncodeSMqOffsetObj(NULL, pOffset);
int32_t size = sizeof(int32_t) + tlen + MND_OFFSET_RESERVE_SIZE;
SSdbRaw *pRaw = sdbAllocRaw(SDB_OFFSET, MND_OFFSET_VER_NUMBER, size);
if (pRaw == NULL) goto OFFSET_ENCODE_OVER;
buf = malloc(tlen);
if (buf == NULL) goto OFFSET_ENCODE_OVER;
void *abuf = buf;
tEncodeSMqOffsetObj(&abuf, pOffset);
int32_t dataPos = 0;
SDB_SET_INT32(pRaw, dataPos, tlen, OFFSET_ENCODE_OVER);
SDB_SET_BINARY(pRaw, dataPos, buf, tlen, OFFSET_ENCODE_OVER);
SDB_SET_RESERVE(pRaw, dataPos, MND_OFFSET_RESERVE_SIZE, OFFSET_ENCODE_OVER);
SDB_SET_DATALEN(pRaw, dataPos, OFFSET_ENCODE_OVER);
terrno = TSDB_CODE_SUCCESS;
OFFSET_ENCODE_OVER:
tfree(buf);
if (terrno != TSDB_CODE_SUCCESS) {
mError("offset:%s, failed to encode to raw:%p since %s", pOffset->key, pRaw, terrstr());
sdbFreeRaw(pRaw);
return NULL;
}
mTrace("offset:%s, encode to raw:%p, row:%p", pOffset->key, pRaw, pOffset);
return pRaw;
}
SSdbRow *mndOffsetActionDecode(SSdbRaw *pRaw) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
void *buf = NULL;
int8_t sver = 0;
if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto OFFSET_DECODE_OVER;
if (sver != MND_OFFSET_VER_NUMBER) {
terrno = TSDB_CODE_SDB_INVALID_DATA_VER;
goto OFFSET_DECODE_OVER;
}
int32_t size = sizeof(SMqOffsetObj);
SSdbRow *pRow = sdbAllocRow(size);
if (pRow == NULL) goto OFFSET_DECODE_OVER;
SMqOffsetObj *pOffset = sdbGetRowObj(pRow);
if (pOffset == NULL) goto OFFSET_DECODE_OVER;
int32_t dataPos = 0;
int32_t tlen;
SDB_GET_INT32(pRaw, dataPos, &tlen, OFFSET_DECODE_OVER);
buf = malloc(tlen + 1);
if (buf == NULL) goto OFFSET_DECODE_OVER;
SDB_GET_BINARY(pRaw, dataPos, buf, tlen, OFFSET_DECODE_OVER);
SDB_GET_RESERVE(pRaw, dataPos, MND_OFFSET_RESERVE_SIZE, OFFSET_DECODE_OVER);
if (tDecodeSMqOffsetObj(buf, pOffset) == NULL) {
goto OFFSET_DECODE_OVER;
}
terrno = TSDB_CODE_SUCCESS;
OFFSET_DECODE_OVER:
tfree(buf);
if (terrno != TSDB_CODE_SUCCESS) {
mError("offset:%s, failed to decode from raw:%p since %s", pOffset->key, pRaw, terrstr());
tfree(pRow);
return NULL;
}
mTrace("offset:%s, decode from raw:%p, row:%p", pOffset->key, pRaw, pOffset);
return pRow;
}
int32_t mndCreateOffset(STrans *pTrans, const char *cgroup, const char *topicName, const SArray *vgs) {
int32_t code = 0;
int32_t sz = taosArrayGetSize(vgs);
for (int32_t i = 0; i < sz; i++) {
SMqConsumerEp *pConsumerEp = taosArrayGet(vgs, i);
SMqOffsetObj offsetObj;
if (mndMakePartitionKey(offsetObj.key, cgroup, topicName, pConsumerEp->vgId) < 0) {
return -1;
}
offsetObj.offset = -1;
SSdbRaw *pOffsetRaw = mndOffsetActionEncode(&offsetObj);
if (pOffsetRaw == NULL) {
return -1;
}
sdbSetRawStatus(pOffsetRaw, SDB_STATUS_READY);
if (mndTransAppendRedolog(pTrans, pOffsetRaw) < 0) {
return -1;
}
}
return 0;
}
static int32_t mndProcessCommitOffsetReq(SMnodeMsg *pMsg) {
char key[TSDB_PARTITION_KEY_LEN];
SMnode *pMnode = pMsg->pMnode;
char *msgStr = pMsg->rpcMsg.pCont;
SMqCMCommitOffsetReq commitOffsetReq;
SCoder decoder;
tCoderInit(&decoder, TD_LITTLE_ENDIAN, msgStr, pMsg->rpcMsg.contLen, TD_DECODER);
tDecodeSMqCMCommitOffsetReq(&decoder, &commitOffsetReq);
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_COMMIT_OFFSET, &pMsg->rpcMsg);
for (int32_t i = 0; i < commitOffsetReq.num; i++) {
SMqOffset *pOffset = &commitOffsetReq.offsets[i];
if (mndMakePartitionKey(key, pOffset->cgroup, pOffset->topicName, pOffset->vgId) < 0) {
return -1;
}
SMqOffsetObj *pOffsetObj = mndAcquireOffset(pMnode, key);
ASSERT(pOffsetObj);
pOffsetObj->offset = pOffset->offset;
SSdbRaw *pOffsetRaw = mndOffsetActionEncode(pOffsetObj);
sdbSetRawStatus(pOffsetRaw, SDB_STATUS_READY);
mndTransAppendRedolog(pTrans, pOffsetRaw);
mndReleaseOffset(pMnode, pOffsetObj);
}
if (mndTransPrepare(pMnode, pTrans) != 0) {
mError("mq-commit-offset-trans:%d, failed to prepare since %s", pTrans->id, terrstr());
mndTransDrop(pTrans);
return -1;
}
mndTransDrop(pTrans);
return TSDB_CODE_MND_ACTION_IN_PROGRESS;
}
static int32_t mndOffsetActionInsert(SSdb *pSdb, SMqOffsetObj *pOffset) {
mTrace("offset:%s, perform insert action", pOffset->key);
return 0;
}
static int32_t mndOffsetActionDelete(SSdb *pSdb, SMqOffsetObj *pOffset) {
mTrace("offset:%s, perform delete action", pOffset->key);
return 0;
}
static int32_t mndOffsetActionUpdate(SSdb *pSdb, SMqOffsetObj *pOldOffset, SMqOffsetObj *pNewOffset) {
mTrace("offset:%s, perform update action", pOldOffset->key);
return 0;
}
SMqOffsetObj *mndAcquireOffset(SMnode *pMnode, const char *key) {
SSdb *pSdb = pMnode->pSdb;
SMqOffsetObj *pOffset = sdbAcquire(pSdb, SDB_OFFSET, key);
if (pOffset == NULL && terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) {
terrno = TSDB_CODE_MND_OFFSET_NOT_EXIST;
}
return pOffset;
}
void mndReleaseOffset(SMnode *pMnode, SMqOffsetObj *pOffset) {
SSdb *pSdb = pMnode->pSdb;
sdbRelease(pSdb, pOffset);
}
static void mndCancelGetNextOffset(SMnode *pMnode, void *pIter) {
SSdb *pSdb = pMnode->pSdb;
sdbCancelFetch(pSdb, pIter);
}

1
source/dnode/mnode/impl/src/mndProfile.c
View File

@ -21,6 +21,7 @@
#include "mndStb.h"
#include "mndUser.h"
#include "tglobal.h"
#include "version.h"
#define QUERY_ID_SIZE 20
#define QUERY_OBJ_ID_SIZE 18

19
source/dnode/mnode/impl/src/mndSubscribe.c
View File

@ -19,6 +19,7 @@
#include "mndDb.h"
#include "mndDnode.h"
#include "mndMnode.h"
#include "mndOffset.h"
#include "mndShow.h"
#include "mndStb.h"
#include "mndTopic.h"
@ -80,13 +81,13 @@ int32_t mndInitSubscribe(SMnode *pMnode) {
return sdbSetTable(pMnode->pSdb, table);
}
static SMqSubscribeObj *mndCreateSubscription(SMnode *pMnode, const SMqTopicObj *pTopic, const char *consumerGroup) {
static SMqSubscribeObj *mndCreateSubscription(SMnode *pMnode, const SMqTopicObj *pTopic, const char *cgroup) {
SMqSubscribeObj *pSub = tNewSubscribeObj();
if (pSub == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
char *key = mndMakeSubscribeKey(consumerGroup, pTopic->name);
char *key = mndMakeSubscribeKey(cgroup, pTopic->name);
if (key == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
tDeleteSMqSubscribeObj(pSub);
@ -289,9 +290,15 @@ static int32_t mndProcessGetSubEpReq(SMnodeMsg *pMsg) {
strcpy(topicEp.topic, topicName);
topicEp.vgs = taosArrayInit(vgsz, sizeof(SMqSubVgEp));
for (int32_t k = 0; k < vgsz; k++) {
char offsetKey[TSDB_PARTITION_KEY_LEN];
SMqConsumerEp *pConsumerEp = taosArrayGet(pSubConsumer->vgInfo, k);
SMqSubVgEp vgEp = {.epSet = pConsumerEp->epSet, .vgId = pConsumerEp->vgId};
SMqSubVgEp vgEp = {.epSet = pConsumerEp->epSet, .vgId = pConsumerEp->vgId, .offset = -1};
mndMakePartitionKey(offsetKey, pConsumer->cgroup, topicName, pConsumerEp->vgId);
SMqOffsetObj *pOffsetObj = mndAcquireOffset(pMnode, offsetKey);
if (pOffsetObj != NULL) {
vgEp.offset = pOffsetObj->offset;
mndReleaseOffset(pMnode, pOffsetObj);
}
taosArrayPush(topicEp.vgs, &vgEp);
}
taosArrayPush(rsp.topics, &topicEp);
@ -870,7 +877,7 @@ static SSdbRaw *mndSubActionEncode(SMqSubscribeObj *pSub) {
SUB_ENCODE_OVER:
tfree(buf);
if (terrno != 0) {
if (terrno != TSDB_CODE_SUCCESS) {
mError("subscribe:%s, failed to encode to raw:%p since %s", pSub->key, pRaw, terrstr());
sdbFreeRaw(pRaw);
return NULL;
@ -1085,6 +1092,8 @@ static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg) {
mDebug("create new subscription by consumer %ld, group: %s, topic %s", consumerId, cgroup, newTopicName);
pSub = mndCreateSubscription(pMnode, pTopic, cgroup);
createSub = true;
mndCreateOffset(pTrans, cgroup, newTopicName, pSub->unassignedVg);
}
SMqSubConsumer mqSubConsumer;

160
source/dnode/mnode/impl/src/mndTelem.c
View File

@ -21,8 +21,8 @@
#include "tversion.h"
#define TELEMETRY_SERVER "telemetry.taosdata.com"
#define TELEMETRY_PORT 80
#define REPORT_INTERVAL 86400
#define TELEMETRY_PORT 80
#define REPORT_INTERVAL 86400
static void mndBeginObject(SBufferWriter* bw) { tbufWriteChar(bw, '{'); }
@ -33,25 +33,8 @@ static void mndCloseObject(SBufferWriter* bw) {
} else {
tbufWriteChar(bw, '}');
}
tbufWriteChar(bw, ',');
}
#if 0
static void beginArray(SBufferWriter* bw) {
tbufWriteChar(bw, '[');
}
static void closeArray(SBufferWriter* bw) {
size_t len = tbufTell(bw);
if (tbufGetData(bw, false)[len - 1] == ',') {
tbufWriteCharAt(bw, len - 1, ']');
} else {
tbufWriteChar(bw, ']');
}
tbufWriteChar(bw, ',');
}
#endif
static void mndWriteString(SBufferWriter* bw, const char* str) {
tbufWriteChar(bw, '"');
tbufWrite(bw, str, strlen(str));
@ -61,7 +44,7 @@ static void mndWriteString(SBufferWriter* bw, const char* str) {
static void mndAddIntField(SBufferWriter* bw, const char* k, int64_t v) {
mndWriteString(bw, k);
tbufWriteChar(bw, ':');
char buf[32];
char buf[32] = {0};
sprintf(buf, "%" PRId64, v);
tbufWrite(bw, buf, strlen(buf));
tbufWriteChar(bw, ',');
@ -184,24 +167,17 @@ static void mndAddRuntimeInfo(SMnode* pMnode, SBufferWriter* bw) {
}
static void mndSendTelemetryReport(SMnode* pMnode) {
STelemMgmt* pMgmt = &pMnode->telemMgmt;
char buf[128] = {0};
uint32_t ip = taosGetIpv4FromFqdn(TELEMETRY_SERVER);
if (ip == 0xffffffff) {
mDebug("failed to get IP address of " TELEMETRY_SERVER " since :%s", strerror(errno));
return;
}
SOCKET fd = taosOpenTcpClientSocket(ip, TELEMETRY_PORT, 0);
if (fd < 0) {
mDebug("failed to create socket for telemetry, reason:%s", strerror(errno));
return;
}
STelemMgmt* pMgmt = &pMnode->telemMgmt;
SBufferWriter bw = tbufInitWriter(NULL, false);
int32_t code = -1;
char buf[128] = {0};
SOCKET fd = 0;
char clusterName[64] = {0};
mndGetClusterName(pMnode, clusterName, sizeof(clusterName));
if (mndGetClusterName(pMnode, clusterName, sizeof(clusterName)) != 0) {
goto SEND_OVER;
}
SBufferWriter bw = tbufInitWriter(NULL, false);
mndBeginObject(&bw);
mndAddStringField(&bw, "instanceId", clusterName);
mndAddIntField(&bw, "reportVersion", 1);
@ -212,54 +188,69 @@ static void mndSendTelemetryReport(SMnode* pMnode) {
mndAddRuntimeInfo(pMnode, &bw);
mndCloseObject(&bw);
uint32_t ip = taosGetIpv4FromFqdn(TELEMETRY_SERVER);
if (ip == 0xffffffff) {
terrno = TAOS_SYSTEM_ERROR(errno);
mError("failed to get ip of %s since :%s", TELEMETRY_SERVER, terrstr());
goto SEND_OVER;
}
fd = taosOpenTcpClientSocket(ip, TELEMETRY_PORT, 0);
if (fd < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
mError("failed to create socket to %s:%d since:%s", TELEMETRY_SERVER, TELEMETRY_PORT, terrstr());
goto SEND_OVER;
}
const char* header =
"POST /report HTTP/1.1\n"
"Host: " TELEMETRY_SERVER
"\n"
"Content-Type: application/json\n"
"Content-Length: ";
if (taosWriteSocket(fd, (void*)header, (int32_t)strlen(header)) < 0) {
goto SEND_OVER;
}
taosWriteSocket(fd, (void*)header, (int32_t)strlen(header));
int32_t contLen = (int32_t)(tbufTell(&bw) - 1);
int32_t contLen = (int32_t)(tbufTell(&bw));
sprintf(buf, "%d\n\n", contLen);
taosWriteSocket(fd, buf, (int32_t)strlen(buf));
taosWriteSocket(fd, tbufGetData(&bw, false), contLen);
tbufCloseWriter(&bw);
if (taosWriteSocket(fd, buf, (int32_t)strlen(buf)) < 0) {
goto SEND_OVER;
}
const char* pCont = tbufGetData(&bw, false);
if (taosWriteSocket(fd, (void*)pCont, contLen) < 0) {
goto SEND_OVER;
}
// read something to avoid nginx error 499
if (taosReadSocket(fd, buf, 10) < 0) {
mDebug("failed to receive response since %s", strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno);
mError("failed to receive response since %s", terrstr());
goto SEND_OVER;
}
mInfo("send telemetry to %s:%d, len:%d content: %s", TELEMETRY_SERVER, TELEMETRY_PORT, contLen, pCont);
code = 0;
SEND_OVER:
tbufCloseWriter(&bw);
taosCloseSocket(fd);
if (code != 0) {
mError("failed to send telemetry to %s:%d since %s", TELEMETRY_SERVER, TELEMETRY_PORT, terrstr());
}
}
static void* mndTelemThreadFp(void* param) {
SMnode* pMnode = param;
static int32_t mndProcessTelemTimer(SMnodeMsg* pReq) {
SMnode* pMnode = pReq->pMnode;
STelemMgmt* pMgmt = &pMnode->telemMgmt;
if (!pMgmt->enable) return 0;
struct timespec end = {0};
clock_gettime(CLOCK_REALTIME, &end);
end.tv_sec += 300; // wait 5 minutes before send first report
setThreadName("mnd-telem");
while (!pMgmt->exit) {
int32_t r = 0;
struct timespec ts = end;
pthread_mutex_lock(&pMgmt->lock);
r = pthread_cond_timedwait(&pMgmt->cond, &pMgmt->lock, &ts);
pthread_mutex_unlock(&pMgmt->lock);
if (r == 0) break;
if (r != ETIMEDOUT) continue;
if (mndIsMaster(pMnode)) {
mndSendTelemetryReport(pMnode);
}
end.tv_sec += REPORT_INTERVAL;
}
return NULL;
taosWLockLatch(&pMgmt->lock);
mndSendTelemetryReport(pMnode);
taosWUnLockLatch(&pMgmt->lock);
return 0;
}
static void mndGetEmail(SMnode* pMnode, char* filepath) {
@ -280,43 +271,12 @@ static void mndGetEmail(SMnode* pMnode, char* filepath) {
int32_t mndInitTelem(SMnode* pMnode) {
STelemMgmt* pMgmt = &pMnode->telemMgmt;
pMgmt->enable = pMnode->cfg.enableTelem;
if (!pMgmt->enable) return 0;
pMgmt->exit = 0;
pthread_mutex_init(&pMgmt->lock, NULL);
pthread_cond_init(&pMgmt->cond, NULL);
pMgmt->email[0] = 0;
taosInitRWLatch(&pMgmt->lock);
mndGetEmail(pMnode, "/usr/local/taos/email");
pthread_attr_t attr;
pthread_attr_init(&attr);
pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE);
int32_t code = pthread_create(&pMgmt->thread, &attr, mndTelemThreadFp, pMnode);
pthread_attr_destroy(&attr);
if (code != 0) {
mDebug("failed to create telemetry thread since :%s", strerror(code));
}
mInfo("mnd telemetry is initialized");
mndSetMsgHandle(pMnode, TDMT_MND_TELEM_TIMER, mndProcessTelemTimer);
mDebug("mnode telemetry is initialized");
return 0;
}
void mndCleanupTelem(SMnode* pMnode) {
STelemMgmt* pMgmt = &pMnode->telemMgmt;
if (!pMgmt->enable) return;
if (taosCheckPthreadValid(pMgmt->thread)) {
pthread_mutex_lock(&pMgmt->lock);
pMgmt->exit = 1;
pthread_cond_signal(&pMgmt->cond);
pthread_mutex_unlock(&pMgmt->lock);
pthread_join(pMgmt->thread, NULL);
}
pthread_mutex_destroy(&pMgmt->lock);
pthread_cond_destroy(&pMgmt->cond);
}
void mndCleanupTelem(SMnode* pMnode) {}

2
source/dnode/mnode/impl/src/mndTrans.c
View File

@ -71,7 +71,7 @@ int32_t mndInitTrans(SMnode *pMnode) {
.updateFp = (SdbUpdateFp)mndTransActionUpdate,
.deleteFp = (SdbDeleteFp)mndTransActionDelete};
mndSetMsgHandle(pMnode, TDMT_MND_TRANS, mndProcessTransReq);
mndSetMsgHandle(pMnode, TDMT_MND_TRANS_TIMER, mndProcessTransReq);
mndSetMsgHandle(pMnode, TDMT_MND_KILL_TRANS, mndProcessKillTransReq);
mndAddShowMetaHandle(pMnode, TSDB_MGMT_TABLE_TRANS, mndGetTransMeta);

52
source/dnode/mnode/impl/src/mnode.c
View File

@ -23,6 +23,7 @@
#include "mndDnode.h"
#include "mndFunc.h"
#include "mndMnode.h"
#include "mndOffset.h"
#include "mndProfile.h"
#include "mndQnode.h"
#include "mndShow.h"
@ -36,6 +37,10 @@
#include "mndUser.h"
#include "mndVgroup.h"
#define MQ_TIMER_MS 3000
#define TRNAS_TIMER_MS 6000
#define TELEM_TIMER_MS 86400000
int32_t mndSendReqToDnode(SMnode *pMnode, SEpSet *pEpSet, SRpcMsg *pMsg) {
if (pMnode == NULL || pMnode->sendReqToDnodeFp == NULL) {
terrno = TSDB_CODE_MND_NOT_READY;
@ -73,46 +78,60 @@ static void *mndBuildTimerMsg(int32_t *pContLen) {
return pReq;
}
static void mndTransReExecute(void *param, void *tmrId) {
static void mndPullupTrans(void *param, void *tmrId) {
SMnode *pMnode = param;
if (mndIsMaster(pMnode)) {
int32_t contLen = 0;
void * pReq = mndBuildTimerMsg(&contLen);
SRpcMsg rpcMsg = {.msgType = TDMT_MND_TRANS, .pCont = pReq, .contLen = contLen};
void *pReq = mndBuildTimerMsg(&contLen);
SRpcMsg rpcMsg = {.msgType = TDMT_MND_TRANS_TIMER, .pCont = pReq, .contLen = contLen};
pMnode->putReqToMWriteQFp(pMnode->pDnode, &rpcMsg);
}
taosTmrReset(mndTransReExecute, 3000, pMnode, pMnode->timer, &pMnode->transTimer);
taosTmrReset(mndPullupTrans, TRNAS_TIMER_MS, pMnode, pMnode->timer, &pMnode->transTimer);
}
static void mndCalMqRebalance(void *param, void *tmrId) {
SMnode *pMnode = param;
if (mndIsMaster(pMnode)) {
int32_t contLen = 0;
void * pReq = mndBuildTimerMsg(&contLen);
void *pReq = mndBuildTimerMsg(&contLen);
SRpcMsg rpcMsg = {.msgType = TDMT_MND_MQ_TIMER, .pCont = pReq, .contLen = contLen};
pMnode->putReqToMReadQFp(pMnode->pDnode, &rpcMsg);
}
taosTmrReset(mndCalMqRebalance, 3000, pMnode, pMnode->timer, &pMnode->mqTimer);
taosTmrReset(mndCalMqRebalance, MQ_TIMER_MS, pMnode, pMnode->timer, &pMnode->mqTimer);
}
static void mndPullupTelem(void *param, void *tmrId) {
SMnode *pMnode = param;
if (mndIsMaster(pMnode)) {
int32_t contLen = 0;
void *pReq = mndBuildTimerMsg(&contLen);
SRpcMsg rpcMsg = {.msgType = TDMT_MND_TELEM_TIMER, .pCont = pReq, .contLen = contLen};
pMnode->putReqToMReadQFp(pMnode->pDnode, &rpcMsg);
}
taosTmrReset(mndPullupTelem, TELEM_TIMER_MS, pMnode, pMnode->timer, &pMnode->telemTimer);
}
static int32_t mndInitTimer(SMnode *pMnode) {
if (pMnode->timer == NULL) {
pMnode->timer = taosTmrInit(5000, 200, 3600000, "MND");
}
pMnode->timer = taosTmrInit(5000, 200, 3600000, "MND");
if (pMnode->timer == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
if (taosTmrReset(mndTransReExecute, 6000, pMnode, pMnode->timer, &pMnode->transTimer)) {
if (taosTmrReset(mndPullupTrans, TRNAS_TIMER_MS, pMnode, pMnode->timer, &pMnode->transTimer)) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
if (taosTmrReset(mndCalMqRebalance, 3000, pMnode, pMnode->timer, &pMnode->mqTimer)) {
if (taosTmrReset(mndCalMqRebalance, MQ_TIMER_MS, pMnode, pMnode->timer, &pMnode->mqTimer)) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
if (taosTmrReset(mndPullupTelem, 60000, pMnode, pMnode->timer, &pMnode->telemTimer)) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
@ -126,6 +145,8 @@ static void mndCleanupTimer(SMnode *pMnode) {
pMnode->transTimer = NULL;
taosTmrStop(pMnode->mqTimer);
pMnode->mqTimer = NULL;
taosTmrStop(pMnode->telemTimer);
pMnode->telemTimer = NULL;
taosTmrCleanUp(pMnode->timer);
pMnode->timer = NULL;
}
@ -197,6 +218,7 @@ static int32_t mndInitSteps(SMnode *pMnode) {
if (mndAllocStep(pMnode, "mnode-topic", mndInitTopic, mndCleanupTopic) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-consumer", mndInitConsumer, mndCleanupConsumer) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-subscribe", mndInitSubscribe, mndCleanupSubscribe) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-offset", mndInitOffset, mndCleanupOffset) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-vgroup", mndInitVgroup, mndCleanupVgroup) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-stb", mndInitStb, mndCleanupStb) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-db", mndInitDb, mndCleanupDb) != 0) return -1;
@ -404,8 +426,8 @@ SMnodeMsg *mndInitMsg(SMnode *pMnode, SRpcMsg *pRpcMsg) {
return NULL;
}
if (pRpcMsg->msgType != TDMT_MND_TRANS && pRpcMsg->msgType != TDMT_MND_MQ_TIMER &&
pRpcMsg->msgType != TDMT_MND_MQ_DO_REBALANCE) {
if (pRpcMsg->msgType != TDMT_MND_TRANS_TIMER && pRpcMsg->msgType != TDMT_MND_MQ_TIMER &&
pRpcMsg->msgType != TDMT_MND_MQ_DO_REBALANCE && pRpcMsg->msgType != TDMT_MND_TELEM_TIMER) {
SRpcConnInfo connInfo = {0};
if ((pRpcMsg->msgType & 1U) && rpcGetConnInfo(pRpcMsg->handle, &connInfo) != 0) {
taosFreeQitem(pMsg);
@ -440,7 +462,7 @@ void mndProcessMsg(SMnodeMsg *pMsg) {
SMnode *pMnode = pMsg->pMnode;
int32_t code = 0;
tmsg_t msgType = pMsg->rpcMsg.msgType;
void * ahandle = pMsg->rpcMsg.ahandle;
void *ahandle = pMsg->rpcMsg.ahandle;
bool isReq = (msgType & 1U);
mTrace("msg:%p, type:%s will be processed, app:%p", pMsg, TMSG_INFO(msgType), ahandle);

34
source/dnode/vnode/src/tq/tq.c
View File

@ -207,9 +207,17 @@ int32_t tqDeserializeConsumer(STQ* pTq, const STqSerializedHead* pHead, STqConsu
int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) {
SMqConsumeReq* pReq = pMsg->pCont;
int64_t consumerId = pReq->consumerId;
int64_t fetchOffset = pReq->offset;
int64_t fetchOffset;
/*int64_t blockingTime = pReq->blockingTime;*/
if (pReq->currentOffset == TMQ_CONF__RESET_OFFSET__EARLIEAST) {
fetchOffset = 0;
} else if (pReq->currentOffset == TMQ_CONF__RESET_OFFSET__LATEST) {
fetchOffset = walGetLastVer(pTq->pWal);
} else {
fetchOffset = pReq->currentOffset + 1;
}
SMqConsumeRsp rsp = {.consumerId = consumerId, .numOfTopics = 0, .pBlockData = NULL};
STqConsumer* pConsumer = tqHandleGet(pTq->tqMeta, consumerId);
@ -226,31 +234,9 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) {
ASSERT(strcmp(pTopic->topicName, pReq->topic) == 0);
ASSERT(pConsumer->consumerId == consumerId);
if (pReq->reqType == TMQ_REQ_TYPE_COMMIT_ONLY) {
pTopic->committedOffset = pReq->offset;
/*printf("offset %ld committed\n", pTopic->committedOffset);*/
pMsg->pCont = NULL;
pMsg->contLen = 0;
pMsg->code = 0;
rpcSendResponse(pMsg);
return 0;
}
if (pReq->reqType == TMQ_REQ_TYPE_CONSUME_AND_COMMIT) {
if (pTopic->committedOffset < pReq->offset - 1) {
pTopic->committedOffset = pReq->offset - 1;
/*printf("offset %ld committed\n", pTopic->committedOffset);*/
}
}
rsp.committedOffset = pTopic->committedOffset;
rsp.reqOffset = pReq->offset;
rsp.reqOffset = pReq->currentOffset;
rsp.skipLogNum = 0;
if (fetchOffset <= pTopic->committedOffset) {
fetchOffset = pTopic->committedOffset + 1;
}
SWalHead* pHead;
while (1) {
int8_t pos = fetchOffset % TQ_BUFFER_SIZE;

5
source/dnode/vnode/src/vnd/vnodeWrite.c
View File

@ -22,14 +22,15 @@ int vnodeProcessWMsgs(SVnode *pVnode, SArray *pMsgs) {
for (int i = 0; i < taosArrayGetSize(pMsgs); i++) {
pMsg = *(SRpcMsg **)taosArrayGet(pMsgs, i);
// ser request version
// set request version
void *pBuf = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead));
int64_t ver = pVnode->state.processed++;
taosEncodeFixedI64(&pBuf, ver);
if (walWrite(pVnode->pWal, ver, pMsg->msgType, pMsg->pCont, pMsg->contLen) < 0) {
/*ASSERT(false);*/
// TODO: handle error
/*ASSERT(false);*/
vError("vnode:%d write wal error since %s", pVnode->vgId, terrstr());
}
}

3
source/libs/CMakeLists.txt
View File

@ -14,4 +14,5 @@ add_subdirectory(qcom)
add_subdirectory(qworker)
add_subdirectory(tfs)
add_subdirectory(nodes)
add_subdirectory(scalar)
add_subdirectory(config)
add_subdirectory(scalar)

4
source/libs/catalog/test/catalogTests.cpp
View File

@ -131,9 +131,7 @@ void ctgTestInitLogFile() {
ctgDbgEnableDebug("api");
char temp[128] = {0};
sprintf(temp, "%s/%s", tsLogDir, defaultLogFileNamePrefix);
if (taosInitLog(temp, tsNumOfLogLines, maxLogFileNum) < 0) {
if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) {
printf("failed to open log file in directory:%s\n", tsLogDir);
}
}

13
source/libs/config/CMakeLists.txt Normal file
View File

@ -0,0 +1,13 @@
aux_source_directory(src CONFIG_SRC)
add_library(config STATIC ${CONFIG_SRC})
target_include_directories(
config
PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/config"
PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
)
target_link_libraries(config os util common)
if(${BUILD_TEST})
ADD_SUBDIRECTORY(test)
endif(${BUILD_TEST})

47
source/libs/config/inc/cfgInt.h Normal file
View File

@ -0,0 +1,47 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_CFG_INT_H_
#define _TD_CFG_INT_H_
#include "config.h"
#include "taoserror.h"
#include "thash.h"
#include "tutil.h"
#include "ulog.h"
#include "tglobal.h"
#ifdef __cplusplus
extern "C" {
#endif
typedef struct SConfig {
ECfgSrcType stype;
SHashObj *hash;
} SConfig;
int32_t cfgLoadFromCfgFile(SConfig *pConfig, const char *filepath);
int32_t cfgLoadFromEnvFile(SConfig *pConfig, const char *filepath);
int32_t cfgLoadFromEnvVar(SConfig *pConfig);
int32_t cfgLoadFromApollUrl(SConfig *pConfig, const char *url);
int32_t cfgSetItem(SConfig *pConfig, const char *name, const char *value, ECfgSrcType stype);
#ifdef __cplusplus
}
#endif
#endif /*_TD_CFG_INT_H_*/

22
source/libs/config/src/cfgApolloUrl.c Normal file
View File

@ -0,0 +1,22 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "cfgInt.h"
int32_t cfgLoadFromApollUrl(SConfig *pConfig, const char *url) {
uInfo("load from apoll url %s", url);
return 0;
}

70
source/libs/config/src/cfgCfgFile.c Normal file
View File

@ -0,0 +1,70 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "cfgInt.h"
int32_t cfgLoadFromCfgFile(SConfig *pConfig, const char *filepath) {
char *line, *name, *value, *value2, *value3;
int olen, vlen, vlen2, vlen3;
ssize_t _bytes = 0;
size_t len = 1024;
FILE *fp = fopen(filepath, "r");
if (fp == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
line = malloc(len);
while (!feof(fp)) {
memset(line, 0, len);
name = value = value2 = value3 = NULL;
olen = vlen = vlen2 = vlen3 = 0;
_bytes = tgetline(&line, &len, fp);
if (_bytes < 0) {
break;
}
line[len - 1] = 0;
paGetToken(line, &name, &olen);
if (olen == 0) continue;
name[olen] = 0;
paGetToken(name + olen + 1, &value, &vlen);
if (vlen == 0) continue;
value[vlen] = 0;
paGetToken(value + vlen + 1, &value2, &vlen2);
if (vlen2 != 0) {
value2[vlen2] = 0;
paGetToken(value2 + vlen2 + 1, &value3, &vlen3);
if (vlen3 != 0) value3[vlen3] = 0;
}
cfgSetItem(pConfig, name, value, CFG_STYPE_CFG_FILE);
// taosReadConfigOption(name, value, value2, value3);
}
fclose(fp);
tfree(line);
uInfo("load from cfg file %s success", filepath);
return 0;
}

22
source/libs/config/src/cfgEnvFile.c Normal file
View File

@ -0,0 +1,22 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "cfgInt.h"
int32_t cfgLoadFromEnvFile(SConfig *pConfig, const char *filepath) {
uInfo("load from env file %s", filepath);
return 0;
}

22
source/libs/config/src/cfgEnvVar.c Normal file
View File

@ -0,0 +1,22 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "cfgInt.h"
int32_t cfgLoadFromEnvVar(SConfig *pConfig) {
uInfo("load from global env variables");
return 0;
}

635
source/libs/config/src/config.c Normal file
View File

@ -0,0 +1,635 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "cfgInt.h"
#include "tep.h"
#include "tlocale.h"
#include "tmsg.h"
#include "ttimezone.h"
#define CFG_NAME_PRINT_LEN 22
#define CFG_SRC_PRINT_LEN 12
SConfig *cfgInit() {
SConfig *pCfg = calloc(1, sizeof(SConfig));
if (pCfg == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
pCfg->hash = taosHashInit(16, MurmurHash3_32, false, HASH_NO_LOCK);
if (pCfg->hash == NULL) {
free(pCfg);
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
return pCfg;
}
int32_t cfgLoad(SConfig *pCfg, ECfgSrcType cfgType, const char *sourceStr) {
switch (cfgType) {
case CFG_STYPE_CFG_FILE:
return cfgLoadFromCfgFile(pCfg, sourceStr);
case CFG_STYPE_ENV_FILE:
return cfgLoadFromEnvFile(pCfg, sourceStr);
case CFG_STYPE_ENV_VAR:
return cfgLoadFromEnvVar(pCfg);
case CFG_STYPE_APOLLO_URL:
return cfgLoadFromApollUrl(pCfg, sourceStr);
default:
return -1;
}
}
void cfgCleanup(SConfig *pCfg) {
if (pCfg != NULL) {
if (pCfg->hash != NULL) {
taosHashCleanup(pCfg->hash);
pCfg->hash == NULL;
}
free(pCfg);
}
}
int32_t cfgGetSize(SConfig *pCfg) { return taosHashGetSize(pCfg->hash); }
SConfigItem *cfgIterate(SConfig *pCfg, SConfigItem *pIter) { return taosHashIterate(pCfg->hash, pIter); }
void cfgCancelIterate(SConfig *pCfg, SConfigItem *pIter) { return taosHashCancelIterate(pCfg->hash, pIter); }
static int32_t cfgCheckAndSetTimezone(SConfigItem *pItem, const char *timezone) {
tfree(pItem->str);
pItem->str = strdup(timezone);
if (pItem->str == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
return 0;
}
static int32_t cfgCheckAndSetCharset(SConfigItem *pItem, const char *charset) {
tfree(pItem->str);
pItem->str = strdup(charset);
if (pItem->str == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
return 0;
}
static int32_t cfgCheckAndSetLocale(SConfigItem *pItem, const char *locale) {
tfree(pItem->str);
pItem->str = strdup(locale);
if (pItem->str == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
return 0;
}
static int32_t cfgCheckAndSetDir(SConfigItem *pItem, const char *inputDir) {
char fullDir[PATH_MAX] = {0};
if (taosExpandDir(inputDir, fullDir, PATH_MAX) != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
uError("failed to expand dir:%s since %s", inputDir, terrstr());
return -1;
}
if (taosRealPath(fullDir, PATH_MAX) != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
uError("failed to get realpath of dir:%s since %s", inputDir, terrstr());
return -1;
}
if (taosMkDir(fullDir) != 0) {
uError("failed to create dir:%s realpath:%s since %s", inputDir, fullDir, terrstr());
return -1;
}
tfree(pItem->str);
pItem->str = strdup(fullDir);
if (pItem->str == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
return 0;
}
static int32_t cfgCheckAndSetIpStr(SConfigItem *pItem, const char *ip) {
uint32_t value = taosInetAddr(ip);
if (value == INADDR_NONE) {
uError("ip:%s is not a valid ip address", ip);
return -1;
}
tfree(pItem->str);
pItem->str = strdup(ip);
if (pItem->str == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
return 0;
}
static int32_t cfgSetBool(SConfigItem *pItem, const char *value, ECfgSrcType stype) {
bool tmp = false;
if (strcasecmp(value, "true") == 0) {
tmp = true;
}
if (atoi(value) > 0) {
tmp = true;
}
pItem->bval = tmp;
pItem->stype = stype;
return 0;
}
static int32_t cfgSetInt32(SConfigItem *pItem, const char *value, ECfgSrcType stype) {
int32_t ival = (int32_t)atoi(value);
if (ival < pItem->imin || ival > pItem->imax) {
uError("cfg:%s, type:%s src:%s value:%d out of range[%" PRId64 ", %" PRId64 "], use last src:%s value:%d",
pItem->name, cfgDtypeStr(pItem->dtype), cfgStypeStr(stype), ival, pItem->imin, pItem->imax,
cfgStypeStr(pItem->stype), pItem->i32);
terrno = TSDB_CODE_OUT_OF_RANGE;
return -1;
}
pItem->i32 = ival;
pItem->stype = stype;
return 0;
}
static int32_t cfgSetInt64(SConfigItem *pItem, const char *value, ECfgSrcType stype) {
int64_t ival = (int64_t)atoi(value);
if (ival < pItem->imin || ival > pItem->imax) {
uError("cfg:%s, type:%s src:%s value:%" PRId64 " out of range[%" PRId64 ", %" PRId64
"], use last src:%s value:%" PRId64,
pItem->name, cfgDtypeStr(pItem->dtype), cfgStypeStr(stype), ival, pItem->imin, pItem->imax,
cfgStypeStr(pItem->stype), pItem->i64);
terrno = TSDB_CODE_OUT_OF_RANGE;
return -1;
}
pItem->i64 = ival;
pItem->stype = stype;
return 0;
}
static int32_t cfgSetFloat(SConfigItem *pItem, const char *value, ECfgSrcType stype) {
float fval = (float)atof(value);
if (fval < pItem->fmin || fval > pItem->fmax) {
uError("cfg:%s, type:%s src:%s value:%f out of range[%f, %f], use last src:%s value:%f", pItem->name,
cfgDtypeStr(pItem->dtype), cfgStypeStr(stype), fval, pItem->fmin, pItem->fmax, cfgStypeStr(pItem->stype),
pItem->fval);
terrno = TSDB_CODE_OUT_OF_RANGE;
return -1;
}
pItem->fval = fval;
pItem->stype = stype;
return 0;
}
static int32_t cfgSetString(SConfigItem *pItem, const char *value, ECfgSrcType stype) {
char *tmp = strdup(value);
if (tmp == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
uError("cfg:%s, type:%s src:%s value:%s failed to dup since %s, use last src:%s value:%s", pItem->name,
cfgDtypeStr(pItem->dtype), cfgStypeStr(stype), value, terrstr(), cfgStypeStr(pItem->stype), pItem->str);
return -1;
}
free(pItem->str);
pItem->str = tmp;
pItem->stype = stype;
return 0;
}
static int32_t cfgSetIpStr(SConfigItem *pItem, const char *value, ECfgSrcType stype) {
char *tmp = strdup(value);
if (tmp == NULL || cfgCheckAndSetIpStr(pItem, value) != 0) {
free(tmp);
terrno = TSDB_CODE_OUT_OF_MEMORY;
uError("cfg:%s, type:%s src:%s value:%s failed to dup since %s, use last src:%s value:%s", pItem->name,
cfgDtypeStr(pItem->dtype), cfgStypeStr(stype), value, terrstr(), cfgStypeStr(pItem->stype), pItem->str);
return -1;
}
pItem->stype = stype;
return 0;
}
static int32_t cfgSetDir(SConfigItem *pItem, const char *value, ECfgSrcType stype) {
char *tmp = strdup(value);
if (tmp == NULL || cfgCheckAndSetDir(pItem, value) != 0) {
free(tmp);
terrno = TSDB_CODE_OUT_OF_MEMORY;
uError("cfg:%s, type:%s src:%s value:%s failed to dup since %s, use last src:%s value:%s", pItem->name,
cfgDtypeStr(pItem->dtype), cfgStypeStr(stype), value, terrstr(), cfgStypeStr(pItem->stype), pItem->str);
return -1;
}
pItem->stype = stype;
return 0;
}
static int32_t cfgSetLocale(SConfigItem *pItem, const char *value, ECfgSrcType stype) {
char *tmp = strdup(value);
if (tmp == NULL || cfgCheckAndSetLocale(pItem, value) != 0) {
free(tmp);
terrno = TSDB_CODE_OUT_OF_MEMORY;
uError("cfg:%s, type:%s src:%s value:%s failed to dup since %s, use last src:%s value:%s", pItem->name,
cfgDtypeStr(pItem->dtype), cfgStypeStr(stype), value, terrstr(), cfgStypeStr(pItem->stype), pItem->str);
return -1;
}
pItem->stype = stype;
return 0;
}
static int32_t cfgSetCharset(SConfigItem *pItem, const char *value, ECfgSrcType stype) {
char *tmp = strdup(value);
if (tmp == NULL || cfgCheckAndSetCharset(pItem, value) != 0) {
free(tmp);
terrno = TSDB_CODE_OUT_OF_MEMORY;
uError("cfg:%s, type:%s src:%s value:%s failed to dup since %s, use last src:%s value:%s", pItem->name,
cfgDtypeStr(pItem->dtype), cfgStypeStr(stype), value, terrstr(), cfgStypeStr(pItem->stype), pItem->str);
return -1;
}
pItem->stype = stype;
return 0;
}
static int32_t cfgSetTimezone(SConfigItem *pItem, const char *value, ECfgSrcType stype) {
char *tmp = strdup(value);
if (tmp == NULL || cfgCheckAndSetTimezone(pItem, value) != 0) {
free(tmp);
terrno = TSDB_CODE_OUT_OF_MEMORY;
uError("cfg:%s, type:%s src:%s value:%s failed to dup since %s, use last src:%s value:%s", pItem->name,
cfgDtypeStr(pItem->dtype), cfgStypeStr(stype), value, terrstr(), cfgStypeStr(pItem->stype), pItem->str);
return -1;
}
pItem->stype = stype;
return 0;
}
int32_t cfgSetItem(SConfig *pCfg, const char *name, const char *value, ECfgSrcType stype) {
SConfigItem *pItem = cfgGetItem(pCfg, name);
if (pItem == NULL) {
return -1;
}
switch (pItem->dtype) {
case CFG_DTYPE_BOOL:
return cfgSetBool(pItem, value, stype);
case CFG_DTYPE_INT32:
return cfgSetInt32(pItem, value, stype);
case CFG_DTYPE_INT64:
return cfgSetInt64(pItem, value, stype);
case CFG_DTYPE_FLOAT:
return cfgSetFloat(pItem, value, stype);
case CFG_DTYPE_STRING:
return cfgSetString(pItem, value, stype);
case CFG_DTYPE_IPSTR:
return cfgSetIpStr(pItem, value, stype);
case CFG_DTYPE_DIR:
return cfgSetDir(pItem, value, stype);
case CFG_DTYPE_TIMEZONE:
return cfgSetTimezone(pItem, value, stype);
case CFG_DTYPE_CHARSET:
return cfgSetCharset(pItem, value, stype);
case CFG_DTYPE_LOCALE:
return cfgSetLocale(pItem, value, stype);
case CFG_DTYPE_NONE:
default:
break;
}
terrno = TSDB_CODE_INVALID_CFG;
return -1;
}
SConfigItem *cfgGetItem(SConfig *pCfg, const char *name) {
char lowcaseName[CFG_NAME_MAX_LEN + 1] = {0};
memcpy(lowcaseName, name, CFG_NAME_MAX_LEN);
strntolower(lowcaseName, name, CFG_NAME_MAX_LEN);
SConfigItem *pItem = taosHashGet(pCfg->hash, lowcaseName, strlen(lowcaseName) + 1);
if (pItem == NULL) {
terrno = TSDB_CODE_CFG_NOT_FOUND;
}
return pItem;
}
static int32_t cfgAddItem(SConfig *pCfg, SConfigItem *pItem, const char *name) {
pItem->stype = CFG_STYPE_DEFAULT;
pItem->name = strdup(name);
if (pItem->name == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
char lowcaseName[CFG_NAME_MAX_LEN + 1] = {0};
memcpy(lowcaseName, name, CFG_NAME_MAX_LEN);
strntolower(lowcaseName, name, CFG_NAME_MAX_LEN);
if (taosHashPut(pCfg->hash, lowcaseName, strlen(lowcaseName) + 1, pItem, sizeof(SConfigItem)) != 0) {
if (pItem->dtype == CFG_DTYPE_STRING) {
free(pItem->str);
}
free(pItem->name);
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
return 0;
}
int32_t cfgAddBool(SConfig *pCfg, const char *name, bool defaultVal) {
SConfigItem item = {.dtype = CFG_DTYPE_BOOL, .bval = defaultVal};
return cfgAddItem(pCfg, &item, name);
}
int32_t cfgAddInt32(SConfig *pCfg, const char *name, int32_t defaultVal, int64_t minval, int64_t maxval) {
if (defaultVal < minval || defaultVal > maxval) {
terrno = TSDB_CODE_OUT_OF_RANGE;
return -1;
}
SConfigItem item = {.dtype = CFG_DTYPE_INT32, .i32 = defaultVal, .imin = minval, .imax = maxval};
return cfgAddItem(pCfg, &item, name);
}
int32_t cfgAddInt64(SConfig *pCfg, const char *name, int64_t defaultVal, int64_t minval, int64_t maxval) {
if (defaultVal < minval || defaultVal > maxval) {
terrno = TSDB_CODE_OUT_OF_RANGE;
return -1;
}
SConfigItem item = {.dtype = CFG_DTYPE_INT64, .i64 = defaultVal, .imin = minval, .imax = maxval};
return cfgAddItem(pCfg, &item, name);
}
int32_t cfgAddFloat(SConfig *pCfg, const char *name, float defaultVal, double minval, double maxval) {
if (defaultVal < minval || defaultVal > maxval) {
terrno = TSDB_CODE_OUT_OF_RANGE;
return -1;
}
SConfigItem item = {.dtype = CFG_DTYPE_FLOAT, .fval = defaultVal, .fmin = minval, .fmax = maxval};
return cfgAddItem(pCfg, &item, name);
}
int32_t cfgAddString(SConfig *pCfg, const char *name, const char *defaultVal) {
SConfigItem item = {.dtype = CFG_DTYPE_STRING};
item.str = strdup(defaultVal);
if (item.str == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
return cfgAddItem(pCfg, &item, name);
}
int32_t cfgAddIpStr(SConfig *pCfg, const char *name, const char *defaultVal) {
SConfigItem item = {.dtype = CFG_DTYPE_IPSTR};
if (cfgCheckAndSetIpStr(&item, defaultVal) != 0) {
return -1;
}
return cfgAddItem(pCfg, &item, name);
}
int32_t cfgAddDir(SConfig *pCfg, const char *name, const char *defaultVal) {
SConfigItem item = {.dtype = CFG_DTYPE_DIR};
if (cfgCheckAndSetDir(&item, defaultVal) != 0) {
return -1;
}
return cfgAddItem(pCfg, &item, name);
}
int32_t cfgAddLocale(SConfig *pCfg, const char *name, const char *defaultVal) {
SConfigItem item = {.dtype = CFG_DTYPE_LOCALE};
if (cfgCheckAndSetLocale(&item, defaultVal) != 0) {
return -1;
}
return cfgAddItem(pCfg, &item, name);
}
int32_t cfgAddCharset(SConfig *pCfg, const char *name, const char *defaultVal) {
SConfigItem item = {.dtype = CFG_DTYPE_CHARSET};
if (cfgCheckAndSetCharset(&item, defaultVal) != 0) {
return -1;
}
return cfgAddItem(pCfg, &item, name);
}
int32_t cfgAddTimezone(SConfig *pCfg, const char *name, const char *defaultVal) {
SConfigItem item = {.dtype = CFG_DTYPE_TIMEZONE};
if (cfgCheckAndSetTimezone(&item, defaultVal) != 0) {
return -1;
}
return cfgAddItem(pCfg, &item, name);
}
const char *cfgStypeStr(ECfgSrcType type) {
switch (type) {
case CFG_STYPE_DEFAULT:
return "default";
case CFG_STYPE_CFG_FILE:
return "cfg_file";
case CFG_STYPE_ENV_FILE:
return "env_file";
case CFG_STYPE_ENV_VAR:
return "env_var";
case CFG_STYPE_APOLLO_URL:
return "apollo_url";
case CFG_STYPE_ARG_LIST:
return "arg_list";
case CFG_STYPE_API_OPTION:
return "api_option";
default:
return "invalid";
}
}
const char *cfgDtypeStr(ECfgDataType type) {
switch (type) {
case CFG_DTYPE_NONE:
return "none";
case CFG_DTYPE_BOOL:
return "bool";
case CFG_DTYPE_INT32:
return "int32";
case CFG_DTYPE_INT64:
return "int64";
case CFG_DTYPE_FLOAT:
return "float";
case CFG_DTYPE_STRING:
return "string";
case CFG_DTYPE_IPSTR:
return "ipstr";
case CFG_DTYPE_DIR:
return "dir";
case CFG_DTYPE_LOCALE:
return "locale";
case CFG_DTYPE_CHARSET:
return "charset";
case CFG_DTYPE_TIMEZONE:
return "timezone";
default:
return "invalid";
}
}
void cfgDumpCfg(SConfig *pCfg) {
uInfo(" global config");
uInfo("=================================================================");
char src[CFG_SRC_PRINT_LEN + 1] = {0};
char name[CFG_NAME_PRINT_LEN + 1] = {0};
SConfigItem *pItem = cfgIterate(pCfg, NULL);
while (pItem != NULL) {
tstrncpy(src, cfgStypeStr(pItem->stype), CFG_SRC_PRINT_LEN);
for (int32_t i = 0; i < CFG_SRC_PRINT_LEN; ++i) {
if (src[i] == 0) src[i] = ' ';
}
tstrncpy(name, pItem->name, CFG_NAME_PRINT_LEN);
for (int32_t i = 0; i < CFG_NAME_PRINT_LEN; ++i) {
if (name[i] == 0) name[i] = ' ';
}
switch (pItem->dtype) {
case CFG_DTYPE_BOOL:
uInfo("%s %s %u", src, name, pItem->bval);
break;
case CFG_DTYPE_INT32:
uInfo("%s %s %d", src, name, pItem->i32);
break;
case CFG_DTYPE_INT64:
uInfo("%s %s %" PRId64, src, name, pItem->i64);
break;
case CFG_DTYPE_FLOAT:
uInfo("%s %s %f", src, name, pItem->fval);
break;
case CFG_DTYPE_STRING:
case CFG_DTYPE_IPSTR:
case CFG_DTYPE_DIR:
case CFG_DTYPE_LOCALE:
case CFG_DTYPE_CHARSET:
case CFG_DTYPE_TIMEZONE:
uInfo("%s %s %s", src, name, pItem->str);
break;
}
pItem = cfgIterate(pCfg, pItem);
}
uInfo("=================================================================");
}
#if 0
// int32_t cfgCheck(SConfig *pCfg) {
// SConfigItem *pItem = NULL;
// pItem = cfgGetItem(pCfg, "serverPort");
// if (pItem != NULL) {
// tsServerPort = (uint16_t)pItem->i32;
// }
// pItem = cfgGetItem(pCfg, "firstEp");
// if (pItem != NULL) {
// tstrncpy(tsFirst, pItem->str, TSDB_EP_LEN);
// }
// snprintf(tsLocalEp, TSDB_EP_LEN, "%s:%u", tsLocalFqdn, tsServerPort);
// uInfo("localEp is: %s", tsLocalEp);
// SEp ep = {0};
// if (tsFirst[0] == 0) {
// strcpy(tsFirst, tsLocalEp);
// } else {
// taosGetFqdnPortFromEp(tsFirst, &ep);
// snprintf(tsFirst, TSDB_EP_LEN, "%s:%u", ep.fqdn, ep.port);
// }
// pItem = cfgGetItem(pCfg, "secondEp");
// if (pItem != NULL) {
// tstrncpy(tsSecond, pItem->str, TSDB_EP_LEN);
// }
// if (tsSecond[0] == 0) {
// strcpy(tsSecond, tsLocalEp);
// } else {
// taosGetFqdnPortFromEp(tsSecond, &ep);
// snprintf(tsSecond, TSDB_EP_LEN, "%s:%u", ep.fqdn, ep.port);
// }
// pItem = cfgGetItem(pCfg, "dataDir");
// if (pItem != NULL) {
// tstrncpy(tsDataDir, pItem->str, PATH_MAX);
// }
// if (tsDiskCfgNum <= 0) {
// taosAddDataDir(0, tsDataDir, 0, 1);
// tsDiskCfgNum = 1;
// uTrace("dataDir:%s, level:0 primary:1 is configured by default", tsDataDir);
// }
// if (taosDirExist(tsTempDir) != 0) {
// return -1;
// }
// taosGetSystemInfo();
// tsSetLocale();
// // SGlobalCfg *cfg_timezone = taosGetConfigOption("timezone");
// // if (cfg_timezone && cfg_timezone->cfgStatus == TAOS_CFG_CSTATUS_FILE) {
// tsSetTimeZone();
// // }
// pItem = cfgGetItem(pCfg, "numOfCores");
// if (pItem != NULL) {
// tsNumOfCores = pItem->i32;
// }
// if (tsNumOfCores <= 0) {
// tsNumOfCores = 1;
// }
// if (tsQueryBufferSize >= 0) {
// tsQueryBufferSizeBytes = tsQueryBufferSize * 1048576UL;
// }
// cfgPrintCfg(pCfg);
// return 0;
// }
#endif

14
source/libs/config/test/CMakeLists.txt Normal file
View File

@ -0,0 +1,14 @@
enable_testing()
aux_source_directory(. CFG_TEST_SRC)
add_executable(cfg_test ${CFG_TEST_SRC})
target_link_libraries(
cfg_test
PUBLIC config
PUBLIC gtest_main
)
add_test(
NAME cfg_test
COMMAND cfg_test
)

147
source/libs/config/test/cfgTest.cpp Normal file
View File

@ -0,0 +1,147 @@
/**
* @file cfgTest.cpp
* @author slguan (slguan@taosdata.com)
* @brief config module tests
* @version 1.0
* @date 2022-02-20
*
* @copyright Copyright (c) 2022
*
*/
#include <gtest/gtest.h>
#include "config.h"
class CfgTest : public ::testing::Test {
protected:
static void SetUpTestSuite() {}
static void TearDownTestSuite() {}
public:
void SetUp() override {}
void TearDown() override {}
void InitCfg(SConfig *pConfig);
static const char *pConfig;
};
const char *CfgTest::pConfig;
TEST_F(CfgTest, 01_Str) {
EXPECT_STREQ(cfgStypeStr(CFG_STYPE_DEFAULT), "default");
EXPECT_STREQ(cfgStypeStr(CFG_STYPE_CFG_FILE), "cfg_file");
EXPECT_STREQ(cfgStypeStr(CFG_STYPE_ENV_FILE), "env_file");
EXPECT_STREQ(cfgStypeStr(CFG_STYPE_ENV_VAR), "env_var");
EXPECT_STREQ(cfgStypeStr(CFG_STYPE_APOLLO_URL), "apollo_url");
EXPECT_STREQ(cfgStypeStr(CFG_STYPE_ARG_LIST), "arg_list");
EXPECT_STREQ(cfgStypeStr(CFG_STYPE_API_OPTION), "api_option");
EXPECT_STREQ(cfgStypeStr(ECfgSrcType(1024)), "invalid");
EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_NONE), "none");
EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_BOOL), "bool");
EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_INT32), "int32");
EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_INT64), "int64");
EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_FLOAT), "float");
EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_STRING), "string");
EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_IPSTR), "ipstr");
EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_DIR), "dir");
EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_DIR), "dir");
EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_DIR), "dir");
EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_DIR), "dir");
EXPECT_STREQ(cfgDtypeStr(ECfgDataType(1024)), "invalid");
}
TEST_F(CfgTest, 02_Basic) {
SConfig *pConfig = cfgInit();
ASSERT_NE(pConfig, nullptr);
EXPECT_EQ(cfgAddBool(pConfig, "test_bool", 0), 0);
EXPECT_EQ(cfgAddInt32(pConfig, "test_int32", 1, 0, 16), 0);
EXPECT_EQ(cfgAddInt64(pConfig, "test_int64", 2, 0, 16), 0);
EXPECT_EQ(cfgAddFloat(pConfig, "test_float", 3, 0, 16), 0);
EXPECT_EQ(cfgAddString(pConfig, "test_string", "4"), 0);
EXPECT_EQ(cfgAddIpStr(pConfig, "test_ipstr", "192.168.0.1"), 0);
EXPECT_EQ(cfgAddDir(pConfig, "test_dir", "/tmp"), 0);
EXPECT_EQ(cfgGetSize(pConfig), 7);
int32_t size = 0;
SConfigItem *pItem = cfgIterate(pConfig, NULL);
while (pItem != NULL) {
switch (pItem->dtype) {
case CFG_DTYPE_BOOL:
printf("index:%d, cfg:%s value:%d\n", size, pItem->name, pItem->bval);
break;
case CFG_DTYPE_INT32:
printf("index:%d, cfg:%s value:%d\n", size, pItem->name, pItem->i32);
break;
case CFG_DTYPE_INT64:
printf("index:%d, cfg:%s value:%" PRId64 "\n", size, pItem->name, pItem->i64);
break;
case CFG_DTYPE_FLOAT:
printf("index:%d, cfg:%s value:%f\n", size, pItem->name, pItem->fval);
break;
case CFG_DTYPE_STRING:
printf("index:%d, cfg:%s value:%s\n", size, pItem->name, pItem->str);
break;
case CFG_DTYPE_IPSTR:
printf("index:%d, cfg:%s value:%s\n", size, pItem->name, pItem->str);
break;
case CFG_DTYPE_DIR:
printf("index:%d, cfg:%s value:%s\n", size, pItem->name, pItem->str);
break;
default:
printf("index:%d, cfg:%s invalid cfg dtype:%d\n", size, pItem->name, pItem->dtype);
break;
}
size++;
pItem = cfgIterate(pConfig, pItem);
}
cfgCancelIterate(pConfig, pItem);
EXPECT_EQ(cfgGetSize(pConfig), 7);
pItem = cfgGetItem(pConfig, "test_bool");
EXPECT_EQ(pItem->stype, CFG_STYPE_DEFAULT);
EXPECT_EQ(pItem->dtype, CFG_DTYPE_BOOL);
EXPECT_STREQ(pItem->name, "test_bool");
EXPECT_EQ(pItem->bval, 0);
pItem = cfgGetItem(pConfig, "test_int32");
EXPECT_EQ(pItem->stype, CFG_STYPE_DEFAULT);
EXPECT_EQ(pItem->dtype, CFG_DTYPE_INT32);
EXPECT_STREQ(pItem->name, "test_int32");
EXPECT_EQ(pItem->i32, 1);
pItem = cfgGetItem(pConfig, "test_int64");
EXPECT_EQ(pItem->stype, CFG_STYPE_DEFAULT);
EXPECT_EQ(pItem->dtype, CFG_DTYPE_INT64);
EXPECT_STREQ(pItem->name, "test_int64");
EXPECT_EQ(pItem->i64, 2);
pItem = cfgGetItem(pConfig, "test_float");
EXPECT_EQ(pItem->stype, CFG_STYPE_DEFAULT);
EXPECT_EQ(pItem->dtype, CFG_DTYPE_FLOAT);
EXPECT_STREQ(pItem->name, "test_float");
EXPECT_EQ(pItem->fval, 3);
pItem = cfgGetItem(pConfig, "test_string");
EXPECT_EQ(pItem->stype, CFG_STYPE_DEFAULT);
EXPECT_EQ(pItem->dtype, CFG_DTYPE_STRING);
EXPECT_STREQ(pItem->name, "test_string");
EXPECT_STREQ(pItem->str, "4");
pItem = cfgGetItem(pConfig, "test_ipstr");
EXPECT_EQ(pItem->stype, CFG_STYPE_DEFAULT);
EXPECT_EQ(pItem->dtype, CFG_DTYPE_IPSTR);
EXPECT_STREQ(pItem->name, "test_ipstr");
EXPECT_STREQ(pItem->str, "192.168.0.1");
pItem = cfgGetItem(pConfig, "test_dir");
EXPECT_EQ(pItem->stype, CFG_STYPE_DEFAULT);
EXPECT_EQ(pItem->dtype, CFG_DTYPE_DIR);
EXPECT_STREQ(pItem->name, "test_dir");
EXPECT_STREQ(pItem->str, "/tmp");
cfgCleanup(pConfig);
}

2
source/libs/executor/src/executor.c
View File

@ -35,7 +35,7 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, char* id)
} else {
SStreamBlockScanInfo* pInfo = pOperator->info;
if (tqReadHandleSetMsg(pInfo->readerHandle, input, 0) < 0) {
qError("submit msg error while set stream msg, %s" PRIx64, id);
qError("submit msg messed up when initing stream block, %s" PRIx64, id);
return TSDB_CODE_QRY_APP_ERROR;
}
return TSDB_CODE_SUCCESS;

9
source/libs/executor/src/executorimpl.c
View File

@ -132,7 +132,8 @@ do { \
} while (0)
int32_t getMaximumIdleDurationSec() {
return tsShellActivityTimer * 2;
// todo
return 6; //tsShellActivityTimer * 2;
}
static int32_t getExprFunctionId(SExprInfo *pExprInfo) {
@ -5301,10 +5302,12 @@ SOperatorInfo* createExchangeOperatorInfo(const SArray* pSources, const SArray*
rpcInit.label = "EX";
rpcInit.numOfThreads = 1;
rpcInit.cfp = qProcessFetchRsp;
rpcInit.sessions = tsMaxConnections;
// todo
rpcInit.sessions = 50000; //tsMaxConnections;
rpcInit.connType = TAOS_CONN_CLIENT;
rpcInit.user = (char *)"root";
rpcInit.idleTime = tsShellActivityTimer * 1000;
// todo
rpcInit.idleTime = 6; //tsShellActivityTimer * 1000;
rpcInit.ckey = "key";
rpcInit.spi = 1;
rpcInit.secret = (char *)"dcc5bed04851fec854c035b2e40263b6";

2
source/libs/index/src/index_fst.c
View File

@ -235,6 +235,7 @@ void fstStateCompileForOneTrans(FstCountingWriter* w, CompiledAddr addr, FstTran
FstState st = fstStateCreate(OneTrans);
fstStateSetCommInput(&st, trn->inp);
bool null = false;
uint8_t inp = fstStateCommInput(&st, &null);
if (null == true) {
@ -936,6 +937,7 @@ FstLastTransition* fstLastTransitionCreate(uint8_t inp, Output out) {
}
void fstLastTransitionDestroy(FstLastTransition* trn) { free(trn); }
void fstBuilderNodeUnfinishedLastCompiled(FstBuilderNodeUnfinished* unNode, CompiledAddr addr) {
FstLastTransition* trn = unNode->last;
if (trn == NULL) {

21
source/libs/index/src/index_fst_automation.c
View File

@ -16,24 +16,24 @@
#include "index_fst_automation.h"
StartWithStateValue* startWithStateValueCreate(StartWithStateKind kind, ValueType ty, void* val) {
StartWithStateValue* nsv = calloc(1, sizeof(StartWithStateValue));
if (nsv == NULL) {
StartWithStateValue* sv = calloc(1, sizeof(StartWithStateValue));
if (sv == NULL) {
return NULL;
}
nsv->kind = kind;
nsv->type = ty;
sv->kind = kind;
sv->type = ty;
if (ty == FST_INT) {
nsv->val = *(int*)val;
sv->val = *(int*)val;
} else if (ty == FST_CHAR) {
size_t len = strlen((char*)val);
nsv->ptr = (char*)calloc(1, len + 1);
memcpy(nsv->ptr, val, len);
sv->ptr = (char*)calloc(1, len + 1);
memcpy(sv->ptr, val, len);
} else if (ty == FST_ARRAY) {
// TODO,
// nsv->arr = taosArrayFromList()
}
return nsv;
return sv;
}
void startWithStateValueDestroy(void* val) {
StartWithStateValue* sv = (StartWithStateValue*)val;
@ -146,11 +146,9 @@ AutomationCtx* automCtxCreate(void* data, AutomationType atype) {
if (atype == AUTOMATION_ALWAYS) {
int val = 0;
sv = startWithStateValueCreate(Running, FST_INT, &val);
ctx->stdata = (void*)sv;
} else if (atype == AUTOMATION_PREFIX) {
int val = 0;
sv = startWithStateValueCreate(Running, FST_INT, &val);
ctx->stdata = (void*)sv;
} else if (atype == AUTMMATION_MATCH) {
} else {
// add more search type
@ -160,9 +158,8 @@ AutomationCtx* automCtxCreate(void* data, AutomationType atype) {
if (data != NULL) {
char* src = (char*)data;
size_t len = strlen(src);
dst = (char*)malloc(len * sizeof(char) + 1);
dst = (char*)calloc(1, len * sizeof(char) + 1);
memcpy(dst, src, len);
dst[len] = 0;
}
ctx->data = dst;

19
source/libs/index/test/fstUT.cc
View File

@ -30,8 +30,8 @@ static void EnvInit() {
taosRemoveDir(path.c_str());
taosMkDir(path.c_str());
// init log file
snprintf(indexlog, PATH_MAX, "%s/tindex.idx", path.c_str());
if (taosInitLog(indexlog, tsNumOfLogLines, 1) != 0) {
tstrncpy(tsLogDir, path.c_str(), PATH_MAX);
if (taosInitLog("tindex.idx", 1) != 0) {
printf("failed to init log");
}
// init index file
@ -99,6 +99,7 @@ class FstReadMemory {
fstSliceDestroy(&skey);
return ok;
}
bool GetWithTimeCostUs(const std::string& key, uint64_t* val, uint64_t* elapse) {
int64_t s = taosGetTimestampUs();
bool ok = this->Get(key, val);
@ -120,8 +121,6 @@ class FstReadMemory {
printf("key: %s, val: %" PRIu64 "\n", key.c_str(), (uint64_t)(rt->out.out));
swsResultDestroy(rt);
}
for (size_t i = 0; i < result.size(); i++) {
}
std::cout << std::endl;
return true;
}
@ -137,7 +136,7 @@ class FstReadMemory {
fstDestroy(_fst);
fstSliceDestroy(&_s);
writerCtxDestroy(_wc, false);
tfCleanup();
// tfCleanup();
}
private:
@ -196,6 +195,10 @@ class TFst {
}
return fr->Get(k, v);
}
bool Search(AutomationCtx* ctx, std::vector<uint64_t>& result) {
// add more
return fr->Search(ctx, result);
}
private:
FstWriter* fw;
@ -229,5 +232,9 @@ TEST_F(FstEnv, writeNormal) {
assert(fst->Get("a", &val) == false);
assert(fst->Get("aa", &val) == true);
assert(val == 0);
std::vector<uint64_t> rlt;
AutomationCtx* ctx = automCtxCreate((void*)"ab", AUTOMATION_ALWAYS);
assert(fst->Search(ctx, rlt) == true);
}
TEST_F(FstEnv, writeExcpet) {}
TEST_F(FstEnv, WriteMillonrRecord) {}

6
source/libs/qcom/src/queryUtil.c
View File

@ -85,9 +85,11 @@ static void* pTaskQueue = NULL;
int32_t initTaskQueue() {
double factor = 4.0;
int32_t numOfThreads = TMAX((int)(tsNumOfCores * tsNumOfThreadsPerCore / factor), 2);
// todo
// int32_t numOfThreads = TMAX((int)(tsNumOfCores * tsNumOfThreadsPerCore / factor), 2);
int32_t numOfThreads = TMAX((int)(tsNumOfCores * 1.0f / factor), 2);
int32_t queueSize = tsMaxConnections * 2;
int32_t queueSize = 25000; //tsMaxConnections * 2;
pTaskQueue = taosInitScheduler(queueSize, numOfThreads, "tsc");
if (NULL == pTaskQueue) {
qError("failed to init task queue");

4
source/libs/qworker/test/qworkerTests.cpp
View File

@ -101,9 +101,7 @@ void qwtInitLogFile() {
tsAsyncLog = 0;
qDebugFlag = 159;
char temp[128] = {0};
sprintf(temp, "%s/%s", tsLogDir, defaultLogFileNamePrefix);
if (taosInitLog(temp, tsNumOfLogLines, maxLogFileNum) < 0) {
if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) {
printf("failed to open log file in directory:%s\n", tsLogDir);
}

4
source/libs/scalar/test/filter/filterTests.cpp
View File

@ -52,9 +52,7 @@ void flttInitLogFile() {
tsAsyncLog = 0;
qDebugFlag = 159;
char temp[128] = {0};
sprintf(temp, "%s/%s", tsLogDir, defaultLogFileNamePrefix);
if (taosInitLog(temp, tsNumOfLogLines, maxLogFileNum) < 0) {
if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) {
printf("failed to open log file in directory:%s\n", tsLogDir);
}
}

4
source/libs/scalar/test/scalar/scalarTests.cpp
View File

@ -51,9 +51,7 @@ void scltInitLogFile() {
tsAsyncLog = 0;
qDebugFlag = 159;
char temp[128] = {0};
sprintf(temp, "%s/%s", tsLogDir, defaultLogFileNamePrefix);
if (taosInitLog(temp, tsNumOfLogLines, maxLogFileNum) < 0) {
if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) {
printf("failed to open log file in directory:%s\n", tsLogDir);
}
}

4
source/libs/scheduler/test/schedulerTests.cpp
View File

@ -66,9 +66,7 @@ void schtInitLogFile() {
tsAsyncLog = 0;
qDebugFlag = 159;
char temp[128] = {0};
sprintf(temp, "%s/%s", tsLogDir, defaultLogFileNamePrefix);
if (taosInitLog(temp, tsNumOfLogLines, maxLogFileNum) < 0) {
if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) {
printf("failed to open log file in directory:%s\n", tsLogDir);
}

1
source/libs/transport/inc/transportInt.h
View File

@ -66,6 +66,7 @@ typedef struct {
void (*cfp)(void* parent, SRpcMsg*, SEpSet*);
int (*afp)(void* parent, char* user, char* spi, char* encrypt, char* secret, char* ckey);
bool (*pfp)(void* parent, tmsg_t msgType);
int32_t refCount;
void* parent;

6
source/libs/transport/src/rpcMain.c
View File

@ -42,6 +42,8 @@ int tsRpcMaxRetry;
int tsRpcHeadSize;
int tsRpcOverhead;
int32_t tsRpcForceTcp = 1; // disable this, means query, show command use udp protocol as default
SHashObj *tsFqdnHash;
#ifndef USE_UV
@ -143,6 +145,10 @@ typedef struct SRpcConn {
static int tsRpcRefId = -1;
static int32_t tsRpcNum = 0;
int32_t tsRpcTimer = 300;
int32_t tsRpcMaxTime = 600; // seconds;
// static pthread_once_t tsRpcInit = PTHREAD_ONCE_INIT;
// server:0 client:1 tcp:2 udp:0

5
source/libs/transport/src/trans.c
View File

@ -29,7 +29,12 @@ void* rpcOpen(const SRpcInit* pInit) {
if (pInit->label) {
tstrncpy(pRpc->label, pInit->label, strlen(pInit->label) + 1);
}
// register callback handle
pRpc->cfp = pInit->cfp;
pRpc->afp = pInit->afp;
pRpc->pfp = pInit->pfp;
if (pInit->connType == TAOS_CONN_SERVER) {
pRpc->numOfThreads = pInit->numOfThreads > TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS : pInit->numOfThreads;
} else {

15
source/libs/transport/src/transCli.c
View File

@ -134,8 +134,7 @@ static void clientHandleResp(SCliConn* conn) {
rpcMsg.msgType = pHead->msgType;
rpcMsg.ahandle = pCtx->ahandle;
if (rpcMsg.msgType == TDMT_VND_QUERY_RSP || rpcMsg.msgType == TDMT_VND_FETCH_RSP ||
rpcMsg.msgType == TDMT_VND_RES_READY_RSP) {
if (pRpc->pfp != NULL && (pRpc->pfp)(pRpc->parent, rpcMsg.msgType)) {
rpcMsg.handle = conn;
conn->persist = 1;
tDebug("client conn %p persist by app", conn);
@ -185,18 +184,13 @@ static void clientHandleExcept(SCliConn* pConn) {
clientConnDestroy(pConn, true);
return;
}
SCliMsg* pMsg = pConn->data;
tmsg_t msgType = TDMT_MND_CONNECT;
if (pMsg != NULL) {
msgType = pMsg->msg.msgType;
}
SCliMsg* pMsg = pConn->data;
STransConnCtx* pCtx = pMsg->ctx;
SRpcMsg rpcMsg = {0};
rpcMsg.ahandle = pCtx->ahandle;
rpcMsg.code = TSDB_CODE_RPC_NETWORK_UNAVAIL;
rpcMsg.msgType = msgType + 1;
rpcMsg.msgType = pMsg->msg.msgType + 1;
if (pConn->push != NULL && pConn->ctnRdCnt != 0) {
(*pConn->push->callback)(pConn->push->arg, &rpcMsg);
@ -445,7 +439,7 @@ static void clientConnCb(uv_connect_t* req, int status) {
addrlen = sizeof(pConn->locaddr);
uv_tcp_getsockname((uv_tcp_t*)pConn->stream, (struct sockaddr*)&pConn->locaddr, &addrlen);
tTrace("client conn %p create", pConn);
tTrace("client conn %p connect to server successfully", pConn);
assert(pConn->stream == req->handle);
clientWrite(pConn);
@ -524,6 +518,7 @@ static void clientHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd) {
struct sockaddr_in addr;
uv_ip4_addr(pMsg->ctx->ip, pMsg->ctx->port, &addr);
// handle error in callback if fail to connect
tTrace("client conn %p try to connect to %s:%d", conn, pMsg->ctx->ip, pMsg->ctx->port);
uv_tcp_connect(&conn->connReq, (uv_tcp_t*)(conn->stream), (const struct sockaddr*)&addr, clientConnCb);
}

18
source/libs/transport/src/transSrv.c
View File

@ -413,11 +413,6 @@ void uvWorkerAsyncCb(uv_async_t* handle) {
} else {
uvStartSendResp(msg);
}
// uv_buf_t wb;
// uvPrepareSendData(msg, &wb);
// uv_timer_stop(conn->pTimer);
// uv_write(conn->pWriter, (uv_stream_t*)conn->pTcp, &wb, 1, uvOnWriteCb);
}
}
static void uvAcceptAsyncCb(uv_async_t* async) {
@ -490,7 +485,6 @@ void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) {
pConn->pTimer->data = pConn;
pConn->hostThrd = pThrd;
// pConn->pWorkerAsync = pThrd->workerAsync; // thread safty
// init client handle
pConn->pTcp = (uv_tcp_t*)malloc(sizeof(uv_tcp_t));
@ -730,14 +724,9 @@ void destroyWorkThrd(SWorkThrdObj* pThrd) {
}
void sendQuitToWorkThrd(SWorkThrdObj* pThrd) {
SSrvMsg* srvMsg = calloc(1, sizeof(SSrvMsg));
// pthread_mutex_lock(&pThrd->msgMtx);
// QUEUE_PUSH(&pThrd->msg, &srvMsg->q);
// pthread_mutex_unlock(&pThrd->msgMtx);
tDebug("send quit msg to work thread");
transSendAsync(pThrd->asyncPool, &srvMsg->q);
// uv_async_send(pThrd->workerAsync);
}
void taosCloseServer(void* arg) {
@ -774,19 +763,12 @@ void rpcSendResponse(const SRpcMsg* pMsg) {
SSrvMsg* srvMsg = calloc(1, sizeof(SSrvMsg));
srvMsg->pConn = pConn;
srvMsg->msg = *pMsg;
// pthread_mutex_lock(&pThrd->msgMtx);
// QUEUE_PUSH(&pThrd->msg, &srvMsg->q);
// pthread_mutex_unlock(&pThrd->msgMtx);
tTrace("server conn %p start to send resp", pConn);
transSendAsync(pThrd->asyncPool, &srvMsg->q);
// uv_async_send(pThrd->workerAsync);
}
int rpcGetConnInfo(void* thandle, SRpcConnInfo* pInfo) {
SSrvConn* pConn = thandle;
// struct sockaddr* pPeerName = &pConn->peername;
struct sockaddr_in addr = pConn->addr;
pInfo->clientIp = (uint32_t)(addr.sin_addr.s_addr);

2
source/libs/transport/test/pushClient.c
View File

@ -192,7 +192,7 @@ int main(int argc, char *argv[]) {
}
}
taosInitLog("client.log", 100000, 10);
taosInitLog("client.log", 10);
void *pRpc = rpcOpen(&rpcInit);
if (pRpc == NULL) {

2
source/libs/transport/test/pushServer.c
View File

@ -169,7 +169,7 @@ int main(int argc, char *argv[]) {
tsAsyncLog = 0;
rpcInit.connType = TAOS_CONN_SERVER;
taosInitLog("server.log", 100000, 10);
taosInitLog("server.log", 10);
void *pRpc = rpcOpen(&rpcInit);
if (pRpc == NULL) {

2
source/libs/transport/test/rclient.c
View File

@ -169,7 +169,7 @@ int main(int argc, char *argv[]) {
}
}
taosInitLog("client.log", 100000, 10);
taosInitLog("client.log", 10);
void *pRpc = rpcOpen(&rpcInit);
if (pRpc == NULL) {

4
source/libs/transport/test/rsclient.c
View File

@ -97,7 +97,7 @@ int main(int argc, char *argv[]) {
rpcInit.label = "APP";
rpcInit.numOfThreads = 1;
rpcInit.sessions = 100;
rpcInit.idleTime = tsShellActivityTimer*1000;
rpcInit.idleTime = 3000; //tsShellActivityTimer*1000;
rpcInit.user = "michael";
rpcInit.secret = secret;
rpcInit.ckey = "key";
@ -148,7 +148,7 @@ int main(int argc, char *argv[]) {
}
}
taosInitLog("client.log", 100000, 10);
taosInitLog("client.log", 10);
void *pRpc = rpcOpen(&rpcInit);
if (pRpc == NULL) {

2
source/libs/transport/test/rserver.c
View File

@ -158,7 +158,7 @@ int main(int argc, char *argv[]) {
tsAsyncLog = 0;
rpcInit.connType = TAOS_CONN_SERVER;
taosInitLog("server.log", 100000, 10);
taosInitLog("server.log", 10);
void *pRpc = rpcOpen(&rpcInit);
if (pRpc == NULL) {

2
source/libs/transport/test/syncClient.c
View File

@ -170,7 +170,7 @@ int main(int argc, char *argv[]) {
}
}
taosInitLog("client.log", 100000, 10);
taosInitLog("client.log", 10);
void *pRpc = rpcOpen(&rpcInit);
if (pRpc == NULL) {

6
source/libs/transport/test/transUT.cc
View File

@ -148,7 +148,6 @@ class TransObj {
wDebugFlag = 0;
sDebugFlag = 0;
tsdbDebugFlag = 0;
cqDebugFlag = 0;
tscEmbeddedInUtil = 1;
tsAsyncLog = 0;
@ -156,9 +155,8 @@ class TransObj {
taosRemoveDir(path.c_str());
taosMkDir(path.c_str());
char temp[PATH_MAX];
snprintf(temp, PATH_MAX, "%s/taosdlog", path.c_str());
if (taosInitLog(temp, tsNumOfLogLines, 1) != 0) {
tstrncpy(tsLogDir, path.c_str(), PATH_MAX);
if (taosInitLog("taosdlog", 1) != 0) {
printf("failed to init log file\n");
}
cli = new Client;

2
source/os/src/osDir.c
View File

@ -114,7 +114,7 @@ void taosRemoveOldFiles(char *dirname, int32_t keepDays) {
rmdir(dirname);
}
int32_t taosExpandDir(char *dirname, char *outname, int32_t maxlen) {
int32_t taosExpandDir(const char *dirname, char *outname, int32_t maxlen) {
wordexp_t full_path;
if (0 != wordexp(dirname, &full_path, 0)) {
//printf("failed to expand path:%s since %s", dirname, strerror(errno));

27
source/os/src/osEnv.c
View File

@ -16,6 +16,30 @@
#define _DEFAULT_SOURCE
#include "os.h"
#include "osEnv.h"
#include "osSysinfo.h"
SDiskSpace tsLogSpace;
SDiskSpace tsTempSpace;
SDiskSpace tsDataSpace;
void taosUpdateLogSpace() { taosGetDiskSize(tsLogDir, &tsLogSpace.size); }
void taosUpdateTempSpace() { taosGetDiskSize(tsTempDir, &tsTempSpace.size); }
void taosUpdateDataSpace() { taosGetDiskSize(tsDataDir, &tsDataSpace.size); }
bool taosLogSpaceAvailable() { return tsLogSpace.reserved < tsLogSpace.size.avail; }
bool taosTempSpaceAvailable() { return tsTempSpace.reserved < tsTempSpace.size.avail; }
bool taosDataSpaceAvailable() { return tsDataSpace.reserved < tsDataSpace.size.avail; }
void taosUpdateAllSpace() {
taosUpdateLogSpace();
taosUpdateTempSpace();
taosUpdateDataSpace();
}
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
@ -23,7 +47,6 @@ char tsOsName[10] = "Windows";
char configDir[PATH_MAX] = "C:/TDengine/cfg";
char tsDataDir[PATH_MAX] = "C:/TDengine/data";
char tsLogDir[PATH_MAX] = "C:/TDengine/log";
char tsScriptDir[PATH_MAX] = "C:/TDengine/script";
char tsTempDir[PATH_MAX] = "C:\\Windows\\Temp";
extern taosWinSocketInit();
@ -47,7 +70,6 @@ char tsOsName[10] = "Darwin";
char configDir[PATH_MAX] = "/usr/local/etc/taos";
char tsDataDir[PATH_MAX] = "/usr/local/var/lib/taos";
char tsLogDir[PATH_MAX] = "/usr/local/var/log/taos";
char tsScriptDir[PATH_MAX] = "/usr/local/etc/taos";
char tsTempDir[PATH_MAX] = "/tmp/taosd";
void osInit() {}
@ -58,7 +80,6 @@ char tsOsName[10] = "Linux";
char configDir[PATH_MAX] = "/etc/taos";
char tsDataDir[PATH_MAX] = "/var/lib/taos";
char tsLogDir[PATH_MAX] = "/var/log/taos";
char tsScriptDir[PATH_MAX] = "/etc/taos";
char tsTempDir[PATH_MAX] = "/tmp/";
void osInit() {}

4
source/os/src/osSocket.c
View File

@ -150,7 +150,7 @@ int32_t taosGetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *op
#if !((defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)) && defined(_MSC_VER))
uint32_t taosInetAddr(char *ipAddr) { return inet_addr(ipAddr); }
uint32_t taosInetAddr(const char *ipAddr) { return inet_addr(ipAddr); }
const char *taosInetNtoa(struct in_addr ipInt) { return inet_ntoa(ipInt); }
@ -240,7 +240,7 @@ int32_t taosSetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *op
#ifdef _MSC_VER
//#if _MSC_VER >= 1900
uint32_t taosInetAddr(char *ipAddr) {
uint32_t taosInetAddr(const char *ipAddr) {
uint32_t value;
int32_t ret = inet_pton(AF_INET, ipAddr, &value);
if (ret <= 0) {

10
source/common/src/ttimezone.c → source/os/src/osTimeZone.c
View File

@ -15,13 +15,13 @@
#define _DEFAULT_SOURCE
#include "os.h"
#include "ulog.h"
#include "tglobal.h"
#include "tconfig.h"
#include "tutil.h"
// #include "ulog.h"
// #include "tglobal.h"
// #include "tutil.h"
// TODO refactor to set the tz value through parameter
void tsSetTimeZone() {
#if 0
SGlobalCfg *cfg_timezone = taosGetConfigOption("timezone");
if (cfg_timezone != NULL) {
uInfo("timezone is set to %s by %s", tsTimezone, tsCfgStatusStr[cfg_timezone->cfgStatus]);
@ -64,4 +64,6 @@ void tsSetTimeZone() {
tsDaylight = daylight;
uInfo("timezone format changed to %s", tsTimezone);
#endif
}

507
source/util/src/tconfig.c
View File

@ -1,507 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "os.h"
#include "tconfig.h"
#include "tutil.h"
#include "ulog.h"
SGlobalCfg tsGlobalConfig[TSDB_CFG_MAX_NUM] = {{0}};
int32_t tsGlobalConfigNum = 0;
static char *tsGlobalUnit[] = {
" ",
"(%)",
"(GB)",
"(Mb)",
"(byte)",
"(s)",
"(ms)"
};
char *tsCfgStatusStr[] = {
"none",
"system default",
"config file",
"taos_options",
"program argument list"
};
static void taosReadFloatConfig(SGlobalCfg *cfg, char *input_value) {
float value = (float)atof(input_value);
float *option = (float *)cfg->ptr;
if (value < cfg->minValue || value > cfg->maxValue) {
uError("config option:%s, input value:%s, out of range[%f, %f], use default value:%f",
cfg->option, input_value, cfg->minValue, cfg->maxValue, *option);
} else {
if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_FILE) {
*option = value;
cfg->cfgStatus = TAOS_CFG_CSTATUS_FILE;
} else {
uWarn("config option:%s, input value:%s, is configured by %s, use %f", cfg->option, input_value,
tsCfgStatusStr[cfg->cfgStatus], *option);
}
}
}
static void taosReadDoubleConfig(SGlobalCfg *cfg, char *input_value) {
double value = atof(input_value);
double *option = (double *)cfg->ptr;
if (value < cfg->minValue || value > cfg->maxValue) {
uError("config option:%s, input value:%s, out of range[%f, %f], use default value:%f",
cfg->option, input_value, cfg->minValue, cfg->maxValue, *option);
} else {
if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_FILE) {
*option = value;
cfg->cfgStatus = TAOS_CFG_CSTATUS_FILE;
} else {
uWarn("config option:%s, input value:%s, is configured by %s, use %f", cfg->option, input_value,
tsCfgStatusStr[cfg->cfgStatus], *option);
}
}
}
static void taosReadInt32Config(SGlobalCfg *cfg, char *input_value) {
int32_t value = atoi(input_value);
int32_t *option = (int32_t *)cfg->ptr;
if (value < cfg->minValue || value > cfg->maxValue) {
uError("config option:%s, input value:%s, out of range[%f, %f], use default value:%d",
cfg->option, input_value, cfg->minValue, cfg->maxValue, *option);
} else {
if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_FILE) {
*option = value;
cfg->cfgStatus = TAOS_CFG_CSTATUS_FILE;
} else {
uWarn("config option:%s, input value:%s, is configured by %s, use %d", cfg->option, input_value,
tsCfgStatusStr[cfg->cfgStatus], *option);
}
}
}
static void taosReadInt16Config(SGlobalCfg *cfg, char *input_value) {
int32_t value = atoi(input_value);
int16_t *option = (int16_t *)cfg->ptr;
if (value < cfg->minValue || value > cfg->maxValue) {
uError("config option:%s, input value:%s, out of range[%f, %f], use default value:%d",
cfg->option, input_value, cfg->minValue, cfg->maxValue, *option);
} else {
if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_FILE) {
*option = (int16_t)value;
cfg->cfgStatus = TAOS_CFG_CSTATUS_FILE;
} else {
uWarn("config option:%s, input value:%s, is configured by %s, use %d", cfg->option, input_value,
tsCfgStatusStr[cfg->cfgStatus], *option);
}
}
}
static void taosReadUInt16Config(SGlobalCfg *cfg, char *input_value) {
int32_t value = atoi(input_value);
uint16_t *option = (uint16_t *)cfg->ptr;
if (value < cfg->minValue || value > cfg->maxValue) {
uError("config option:%s, input value:%s, out of range[%f, %f], use default value:%d",
cfg->option, input_value, cfg->minValue, cfg->maxValue, *option);
} else {
if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_FILE) {
*option = (uint16_t)value;
cfg->cfgStatus = TAOS_CFG_CSTATUS_FILE;
} else {
uWarn("config option:%s, input value:%s, is configured by %s, use %d", cfg->option, input_value,
tsCfgStatusStr[cfg->cfgStatus], *option);
}
}
}
static void taosReadInt8Config(SGlobalCfg *cfg, char *input_value) {
int32_t value = atoi(input_value);
int8_t *option = (int8_t *)cfg->ptr;
if (value < cfg->minValue || value > cfg->maxValue) {
uError("config option:%s, input value:%s, out of range[%f, %f], use default value:%d",
cfg->option, input_value, cfg->minValue, cfg->maxValue, *option);
} else {
if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_FILE) {
*option = (int8_t)value;
cfg->cfgStatus = TAOS_CFG_CSTATUS_FILE;
} else {
uWarn("config option:%s, input value:%s, is configured by %s, use %d", cfg->option, input_value,
tsCfgStatusStr[cfg->cfgStatus], *option);
}
}
}
static bool taosReadDirectoryConfig(SGlobalCfg *cfg, char *input_value) {
int length = (int)strlen(input_value);
char *option = (char *)cfg->ptr;
if (length <= 0 || length > cfg->ptrLength) {
uError("config option:%s, input value:%s, length out of range[0, %d], use default value:%s", cfg->option,
input_value, cfg->ptrLength, option);
return false;
} else {
if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_FILE) {
taosExpandDir(input_value, option, cfg->ptrLength);
taosRealPath(option, cfg->ptrLength);
if (taosMkDir(option) != 0) {
uError("config option:%s, input value:%s, directory not exist, create fail:%s", cfg->option, input_value,
strerror(errno));
return false;
}
cfg->cfgStatus = TAOS_CFG_CSTATUS_FILE;
} else {
uWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, input_value,
tsCfgStatusStr[cfg->cfgStatus], option);
}
}
return true;
}
static void taosReadIpStrConfig(SGlobalCfg *cfg, char *input_value) {
uint32_t value = taosInetAddr(input_value);
char * option = (char *)cfg->ptr;
if (value == INADDR_NONE) {
uError("config option:%s, input value:%s, is not a valid ip address, use default value:%s",
cfg->option, input_value, option);
} else {
if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_FILE) {
strncpy(option, input_value, cfg->ptrLength);
cfg->cfgStatus = TAOS_CFG_CSTATUS_FILE;
} else {
uWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, input_value,
tsCfgStatusStr[cfg->cfgStatus], option);
}
}
}
static void taosReadStringConfig(SGlobalCfg *cfg, char *input_value) {
int length = (int) strlen(input_value);
char *option = (char *)cfg->ptr;
if (length <= 0 || length > cfg->ptrLength) {
uError("config option:%s, input value:%s, length out of range[0, %d], use default value:%s",
cfg->option, input_value, cfg->ptrLength, option);
} else {
if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_FILE) {
strncpy(option, input_value, cfg->ptrLength);
cfg->cfgStatus = TAOS_CFG_CSTATUS_FILE;
} else {
uWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, input_value,
tsCfgStatusStr[cfg->cfgStatus], option);
}
}
}
static void taosReadLogOption(char *option, char *value) {
for (int i = 0; i < tsGlobalConfigNum; ++i) {
SGlobalCfg *cfg = tsGlobalConfig + i;
if (!(cfg->cfgType & TSDB_CFG_CTYPE_B_CONFIG) || !(cfg->cfgType & TSDB_CFG_CTYPE_B_LOG)) continue;
if (strcasecmp(cfg->option, option) != 0) continue;
switch (cfg->valType) {
case TAOS_CFG_VTYPE_INT32:
taosReadInt32Config(cfg, value);
// if (strcasecmp(cfg->option, "debugFlag") == 0) {
// taosSetAllDebugFlag();
// }
break;
case TAOS_CFG_VTYPE_DIRECTORY:
taosReadDirectoryConfig(cfg, value);
break;
default:
break;
}
break;
}
}
SGlobalCfg *taosGetConfigOption(const char *option) {
for (int i = 0; i < tsGlobalConfigNum; ++i) {
SGlobalCfg *cfg = tsGlobalConfig + i;
if (strcasecmp(cfg->option, option) != 0) continue;
return cfg;
}
return NULL;
}
static void taosReadConfigOption(const char *option, char *value, char *value2, char *value3) {
for (int i = 0; i < tsGlobalConfigNum; ++i) {
SGlobalCfg *cfg = tsGlobalConfig + i;
if (!(cfg->cfgType & TSDB_CFG_CTYPE_B_CONFIG)) continue;
if (strcasecmp(cfg->option, option) != 0) continue;
switch (cfg->valType) {
case TAOS_CFG_VTYPE_INT8:
taosReadInt8Config(cfg, value);
break;
case TAOS_CFG_VTYPE_INT16:
taosReadInt16Config(cfg, value);
break;
case TAOS_CFG_VTYPE_INT32:
taosReadInt32Config(cfg, value);
break;
case TAOS_CFG_VTYPE_UINT16:
taosReadUInt16Config(cfg, value);
break;
case TAOS_CFG_VTYPE_FLOAT:
taosReadFloatConfig(cfg, value);
break;
case TAOS_CFG_VTYPE_DOUBLE:
taosReadDoubleConfig(cfg, value);
break;
case TAOS_CFG_VTYPE_STRING:
taosReadStringConfig(cfg, value);
break;
case TAOS_CFG_VTYPE_IPSTR:
taosReadIpStrConfig(cfg, value);
break;
case TAOS_CFG_VTYPE_DIRECTORY:
taosReadDirectoryConfig(cfg, value);
break;
case TAOS_CFG_VTYPE_DATA_DIRCTORY:
if (taosReadDirectoryConfig(cfg, value)) {
// taosReadDataDirCfg(value, value2, value3);
}
break;
default:
uError("config option:%s, input value:%s, can't be recognized", option, value);
break;
}
break;
}
}
void taosAddConfigOption(SGlobalCfg cfg) {
tsGlobalConfig[tsGlobalConfigNum++] = cfg;
}
void taosReadGlobalLogCfg() {
FILE * fp;
char * line, *option, *value;
int olen, vlen;
char fileName[PATH_MAX] = {0};
taosExpandDir(configDir, configDir, PATH_MAX);
taosReadLogOption("logDir", tsLogDir);
sprintf(fileName, "%s/taos.cfg", configDir);
fp = fopen(fileName, "r");
if (fp == NULL) {
printf("\nconfig file:%s not found, all variables are set to default\n", fileName);
return;
}
ssize_t _bytes = 0;
size_t len = 1024;
line = calloc(1, len);
while (!feof(fp)) {
memset(line, 0, len);
option = value = NULL;
olen = vlen = 0;
_bytes = tgetline(&line, &len, fp);
if (_bytes < 0)
{
break;
}
line[len - 1] = 0;
paGetToken(line, &option, &olen);
if (olen == 0) continue;
option[olen] = 0;
paGetToken(option + olen + 1, &value, &vlen);
if (vlen == 0) continue;
value[vlen] = 0;
taosReadLogOption(option, value);
}
tfree(line);
fclose(fp);
}
int32_t taosReadCfgFromFile() {
char * line, *option, *value, *value2, *value3;
int olen, vlen, vlen2, vlen3;
char fileName[PATH_MAX] = {0};
sprintf(fileName, "%s/taos.cfg", configDir);
FILE *fp = fopen(fileName, "r");
if (fp == NULL) {
fp = fopen(configDir, "r");
if (fp == NULL) {
return -1;
}
}
ssize_t _bytes = 0;
size_t len = 1024;
line = calloc(1, len);
while (!feof(fp)) {
memset(line, 0, len);
option = value = value2 = value3 = NULL;
olen = vlen = vlen2 = vlen3 = 0;
_bytes = tgetline(&line, &len, fp);
if (_bytes < 0)
{
break;
}
line[len - 1] = 0;
paGetToken(line, &option, &olen);
if (olen == 0) continue;
option[olen] = 0;
paGetToken(option + olen + 1, &value, &vlen);
if (vlen == 0) continue;
value[vlen] = 0;
paGetToken(value + vlen + 1, &value2, &vlen2);
if (vlen2 != 0) {
value2[vlen2] = 0;
paGetToken(value2 + vlen2 + 1, &value3, &vlen3);
if (vlen3 != 0) value3[vlen3] = 0;
}
taosReadConfigOption(option, value, value2, value3);
}
fclose(fp);
tfree(line);
// if (debugFlag & DEBUG_TRACE || debugFlag & DEBUG_DEBUG || debugFlag & DEBUG_DUMP) {
// taosSetAllDebugFlag();
// }
return 0;
}
void taosPrintCfg() {
uInfo(" taos config & system info:");
uInfo("==================================");
for (int i = 0; i < tsGlobalConfigNum; ++i) {
SGlobalCfg *cfg = tsGlobalConfig + i;
if (tscEmbeddedInUtil == 0 && !(cfg->cfgType & TSDB_CFG_CTYPE_B_CLIENT)) continue;
if (cfg->cfgType & TSDB_CFG_CTYPE_B_NOT_PRINT) continue;
int optionLen = (int)strlen(cfg->option);
int blankLen = TSDB_CFG_PRINT_LEN - optionLen;
blankLen = blankLen < 0 ? 0 : blankLen;
char blank[TSDB_CFG_PRINT_LEN];
memset(blank, ' ', TSDB_CFG_PRINT_LEN);
blank[blankLen] = 0;
switch (cfg->valType) {
case TAOS_CFG_VTYPE_INT8:
uInfo(" %s:%s%d%s", cfg->option, blank, *((int8_t *)cfg->ptr), tsGlobalUnit[cfg->unitType]);
break;
case TAOS_CFG_VTYPE_INT16:
uInfo(" %s:%s%d%s", cfg->option, blank, *((int16_t *)cfg->ptr), tsGlobalUnit[cfg->unitType]);
break;
case TAOS_CFG_VTYPE_INT32:
uInfo(" %s:%s%d%s", cfg->option, blank, *((int32_t *)cfg->ptr), tsGlobalUnit[cfg->unitType]);
break;
case TAOS_CFG_VTYPE_UINT16:
uInfo(" %s:%s%d%s", cfg->option, blank, *((uint16_t *)cfg->ptr), tsGlobalUnit[cfg->unitType]);
break;
case TAOS_CFG_VTYPE_FLOAT:
uInfo(" %s:%s%f%s", cfg->option, blank, *((float *)cfg->ptr), tsGlobalUnit[cfg->unitType]);
break;
case TAOS_CFG_VTYPE_DOUBLE:
uInfo(" %s:%s%f%s", cfg->option, blank, *((double *)cfg->ptr), tsGlobalUnit[cfg->unitType]);
break;
case TAOS_CFG_VTYPE_STRING:
case TAOS_CFG_VTYPE_IPSTR:
case TAOS_CFG_VTYPE_DIRECTORY:
uInfo(" %s:%s%s%s", cfg->option, blank, (char *)cfg->ptr, tsGlobalUnit[cfg->unitType]);
break;
default:
break;
}
}
taosPrintOsInfo();
uInfo("==================================");
}
static void taosDumpCfg(SGlobalCfg *cfg) {
int optionLen = (int)strlen(cfg->option);
int blankLen = TSDB_CFG_PRINT_LEN - optionLen;
blankLen = blankLen < 0 ? 0 : blankLen;
char blank[TSDB_CFG_PRINT_LEN];
memset(blank, ' ', TSDB_CFG_PRINT_LEN);
blank[blankLen] = 0;
switch (cfg->valType) {
case TAOS_CFG_VTYPE_INT8:
printf(" %s:%s%d%s\n", cfg->option, blank, *((int8_t *)cfg->ptr), tsGlobalUnit[cfg->unitType]);
break;
case TAOS_CFG_VTYPE_INT16:
printf(" %s:%s%d%s\n", cfg->option, blank, *((int16_t *)cfg->ptr), tsGlobalUnit[cfg->unitType]);
break;
case TAOS_CFG_VTYPE_INT32:
printf(" %s:%s%d%s\n", cfg->option, blank, *((int32_t *)cfg->ptr), tsGlobalUnit[cfg->unitType]);
break;
case TAOS_CFG_VTYPE_UINT16:
printf(" %s:%s%d%s\n", cfg->option, blank, *((uint16_t *)cfg->ptr), tsGlobalUnit[cfg->unitType]);
break;
case TAOS_CFG_VTYPE_FLOAT:
printf(" %s:%s%f%s\n", cfg->option, blank, *((float *)cfg->ptr), tsGlobalUnit[cfg->unitType]);
break;
case TAOS_CFG_VTYPE_STRING:
case TAOS_CFG_VTYPE_IPSTR:
case TAOS_CFG_VTYPE_DIRECTORY:
printf(" %s:%s%s%s\n", cfg->option, blank, (char *)cfg->ptr, tsGlobalUnit[cfg->unitType]);
break;
default:
break;
}
}
void taosDumpGlobalCfg() {
printf("taos global config:\n");
printf("==================================\n");
for (int i = 0; i < tsGlobalConfigNum; ++i) {
SGlobalCfg *cfg = tsGlobalConfig + i;
if (tscEmbeddedInUtil == 0 && !(cfg->cfgType & TSDB_CFG_CTYPE_B_CLIENT)) continue;
if (cfg->cfgType & TSDB_CFG_CTYPE_B_NOT_PRINT) continue;
if (!(cfg->cfgType & TSDB_CFG_CTYPE_B_SHOW)) continue;
taosDumpCfg(cfg);
}
printf("\ntaos local config:\n");
printf("==================================\n");
for (int i = 0; i < tsGlobalConfigNum; ++i) {
SGlobalCfg *cfg = tsGlobalConfig + i;
if (tscEmbeddedInUtil == 0 && !(cfg->cfgType & TSDB_CFG_CTYPE_B_CLIENT)) continue;
if (cfg->cfgType & TSDB_CFG_CTYPE_B_NOT_PRINT) continue;
if (cfg->cfgType & TSDB_CFG_CTYPE_B_SHOW) continue;
taosDumpCfg(cfg);
}
}

2
source/util/src/terror.c
View File

@ -81,6 +81,8 @@ TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_MSG, "Invalid message")
TAOS_DEFINE_ERROR(TSDB_CODE_MSG_NOT_PROCESSED, "Message not processed")
TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_PARA, "Invalid parameters")
TAOS_DEFINE_ERROR(TSDB_CODE_REPEAT_INIT, "Repeat initialization")
TAOS_DEFINE_ERROR(TSDB_CODE_CFG_NOT_FOUND, "Config not found")
TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_CFG, "Invalid config option")
TAOS_DEFINE_ERROR(TSDB_CODE_REF_NO_MEMORY, "Ref out of memory")
TAOS_DEFINE_ERROR(TSDB_CODE_REF_FULL, "too many Ref Objs")

71
source/util/src/tlog.c
View File

@ -70,29 +70,26 @@ typedef struct {
int8_t tscEmbeddedInUtil = 0;
int32_t tsLogKeepDays = 0;
int8_t tsAsyncLog = 1;
float tsTotalLogDirGB = 0;
float tsAvailLogDirGB = 0;
float tsMinimalLogDirGB = 1.0f;
bool tsAsyncLog = true;
bool tsLogInited = false;
int64_t asyncLogLostLines = 0;
int32_t writeInterval = DEFAULT_LOG_INTERVAL;
// log
int32_t tsNumOfLogLines = 10000000;
int32_t mDebugFlag = 131;
int32_t dDebugFlag = 135;
int32_t vDebugFlag = 135;
int32_t mDebugFlag = 131;
int32_t cDebugFlag = 131;
int32_t jniDebugFlag = 131;
int32_t qDebugFlag = 131;
int32_t rpcDebugFlag = 131;
int32_t tmrDebugFlag = 131;
int32_t uDebugFlag = 131;
int32_t debugFlag = 0;
int32_t sDebugFlag = 135;
int32_t rpcDebugFlag = 131;
int32_t qDebugFlag = 131;
int32_t wDebugFlag = 135;
int32_t sDebugFlag = 135;
int32_t tsdbDebugFlag = 131;
int32_t tqDebugFlag = 135;
int32_t cqDebugFlag = 131;
int32_t fsDebugFlag = 135;
int64_t dbgEmptyW = 0;
@ -120,11 +117,18 @@ static int32_t taosStartLog() {
return 0;
}
int32_t taosInitLog(char *logName, int numOfLogLines, int maxFiles) {
int32_t taosInitLog(const char *logName, int maxFiles) {
if (tsLogInited) return 0;
taosUpdateLogSpace();
char fullName[PATH_MAX] = {0};
snprintf(fullName, PATH_MAX, "%s" TD_DIRSEP "%s", tsLogDir, logName);
tsLogObj.logHandle = taosLogBuffNew(TSDB_DEFAULT_LOG_BUF_SIZE);
if (tsLogObj.logHandle == NULL) return -1;
if (taosOpenLogFile(logName, numOfLogLines, maxFiles) < 0) return -1;
if (taosOpenLogFile(fullName, tsNumOfLogLines, maxFiles) < 0) return -1;
if (taosStartLog() < 0) return -1;
tsLogInited = true;
return 0;
}
@ -217,7 +221,7 @@ static void *taosThreadToOpenNewFile(void *param) {
uInfo(" new log file:%d is opened", tsLogObj.flag);
uInfo("==================================");
taosPrintCfg();
// taosPrintCfg();
taosKeepOldLog(keepName);
return NULL;
@ -377,12 +381,7 @@ static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum) {
}
void taosPrintLog(const char *flags, int32_t dflag, const char *format, ...) {
if (tsTotalLogDirGB != 0 && tsAvailLogDirGB < tsMinimalLogDirGB) {
printf("server disk:%s space remain %.3f GB, total %.1f GB, stop print log.\n", tsLogDir, tsAvailLogDirGB,
tsTotalLogDirGB);
fflush(stdout);
return;
}
if (!taosLogSpaceAvailable()) return;
va_list argpointer;
char buffer[MAX_LOGLINE_BUFFER_SIZE] = {0};
@ -437,12 +436,7 @@ void taosPrintLog(const char *flags, int32_t dflag, const char *format, ...) {
}
void taosDumpData(unsigned char *msg, int32_t len) {
if (tsTotalLogDirGB != 0 && tsAvailLogDirGB < tsMinimalLogDirGB) {
printf("server disk:%s space remain %.3f GB, total %.1f GB, stop dump log.\n", tsLogDir, tsAvailLogDirGB,
tsTotalLogDirGB);
fflush(stdout);
return;
}
if (!taosLogSpaceAvailable()) return;
char temp[256];
int32_t i, pos = 0, c = 0;
@ -465,12 +459,7 @@ void taosDumpData(unsigned char *msg, int32_t len) {
}
void taosPrintLongString(const char *flags, int32_t dflag, const char *format, ...) {
if (tsTotalLogDirGB != 0 && tsAvailLogDirGB < tsMinimalLogDirGB) {
printf("server disk:%s space remain %.3f GB, total %.1f GB, stop write log.\n", tsLogDir, tsAvailLogDirGB,
tsTotalLogDirGB);
fflush(stdout);
return;
}
if (!taosLogSpaceAvailable()) return;
va_list argpointer;
char buffer[MAX_LOGLINE_DUMP_BUFFER_SIZE];
@ -773,3 +762,23 @@ void taosPrintOsInfo() {
uInfo(" os version: %s", info.version);
uInfo(" os machine: %s", info.machine);
}
void taosSetAllDebugFlag(int32_t flag) {
if (!(flag & DEBUG_TRACE || flag & DEBUG_DEBUG || flag & DEBUG_DUMP)) return;
dDebugFlag = flag;
vDebugFlag = flag;
mDebugFlag = flag;
cDebugFlag = flag;
jniDebugFlag = flag;
uDebugFlag = flag;
rpcDebugFlag = flag;
qDebugFlag = flag;
wDebugFlag = flag;
sDebugFlag = flag;
tsdbDebugFlag = flag;
tqDebugFlag = flag;
fsDebugFlag = flag;
uInfo("all debug flag are set to %d", flag);
}

1
source/util/src/ttimer.c
View File

@ -111,7 +111,6 @@ typedef struct time_wheel_t {
tmr_obj_t** slots;
} time_wheel_t;
int32_t tmrDebugFlag = 131;
uint32_t tsMaxTmrCtrl = 512;
static pthread_once_t tmrModuleInit = PTHREAD_ONCE_INIT;

2
source/util/test/trefTest.c
View File

@ -158,7 +158,7 @@ int main(int argc, char *argv[]) {
}
}
taosInitLog("tref.log", 5000000, 10);
taosInitLog("tref.log", 10);
SRefSpace *pSpaceList = (SRefSpace *) calloc(sizeof(SRefSpace), threads);
pthread_t *pThreadList = (pthread_t *) calloc(sizeof(pthread_t), threads);

Some files were not shown because too many files have changed in this diff Show More