diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 074014b0d7..21b8b661df 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -193,6 +193,7 @@ endif(${BUILD_WITH_TRAFT}) # LIBUV if(${BUILD_WITH_UV}) + add_compile_options(-Wno-sign-compare) add_subdirectory(libuv) endif(${BUILD_WITH_UV}) diff --git a/example/src/tmq.c b/example/src/tmq.c index 26e5ea82c1..094fd94bfc 100644 --- a/example/src/tmq.c +++ b/example/src/tmq.c @@ -66,7 +66,7 @@ int32_t init_env() { } int32_t create_topic() { - printf("create topic"); + printf("create topic\n"); TAOS_RES* pRes; TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); if (pConn == NULL) { @@ -91,6 +91,10 @@ int32_t create_topic() { return 0; } +void tmq_commit_cb_print(tmq_t* tmq, tmq_resp_err_t resp, tmq_topic_vgroup_list_t* offsets, void* param) { + printf("commit %d\n", resp); +} + tmq_t* build_consumer() { TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); assert(pConn != NULL); @@ -103,6 +107,7 @@ tmq_t* build_consumer() { tmq_conf_t* conf = tmq_conf_new(); tmq_conf_set(conf, "group.id", "tg2"); + tmq_conf_set_offset_commit_cb(conf, tmq_commit_cb_print); tmq_t* tmq = tmq_consumer_new(pConn, conf, NULL, 0); return tmq; } @@ -144,7 +149,7 @@ void basic_consume_loop(tmq_t* tmq, tmq_list_t* topics) { } void sync_consume_loop(tmq_t* tmq, tmq_list_t* topics) { - static const int MIN_COMMIT_COUNT = 1000; + static const int MIN_COMMIT_COUNT = 1; int msg_count = 0; tmq_resp_err_t err; @@ -214,6 +219,6 @@ int main(int argc, char* argv[]) { tmq_t* tmq = build_consumer(); tmq_list_t* topic_list = build_topic_list(); /*perf_loop(tmq, topic_list);*/ - basic_consume_loop(tmq, topic_list); - /*sync_consume_loop(tmq, topic_list);*/ + /*basic_consume_loop(tmq, topic_list);*/ + sync_consume_loop(tmq, topic_list); } diff --git a/include/client/taos.h b/include/client/taos.h index a960d37937..2c8135c8ff 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -198,8 +198,8 @@ DLL_EXPORT TAOS_RES *taos_schemaless_insert(TAOS *taos, char *lines[], int numLi /* --------------------------TMQ INTERFACE------------------------------- */ enum tmq_resp_err_t { + TMQ_RESP_ERR__FAIL = -1, TMQ_RESP_ERR__SUCCESS = 0, - TMQ_RESP_ERR__FAIL = 1, }; typedef enum tmq_resp_err_t tmq_resp_err_t; @@ -226,7 +226,7 @@ DLL_EXPORT const char *tmq_err2str(tmq_resp_err_t); DLL_EXPORT tmq_resp_err_t tmq_subscribe(tmq_t *tmq, tmq_list_t *topic_list); #if 0 DLL_EXPORT tmq_resp_err_t tmq_unsubscribe(tmq_t* tmq); -DLL_EXPORT tmq_resp_err_t tmq_subscription(tmq_t* tmq, tmq_topic_vgroup_list_t** topics); +DLL_EXPORT tmq_resp_err_t tmq_subscription(tmq_t* tmq, tmq_list_t** topics); #endif DLL_EXPORT tmq_message_t *tmq_consumer_poll(tmq_t *tmq, int64_t blocking_time); DLL_EXPORT tmq_resp_err_t tmq_consumer_close(tmq_t *tmq); @@ -238,6 +238,7 @@ DLL_EXPORT tmq_resp_err_t tmq_commit(tmq_t *tmq, const tmq_topic_vgroup_list_t * #if 0 DLL_EXPORT tmq_resp_err_t tmq_commit_message(tmq_t* tmq, const tmq_message_t* tmqmessage, int32_t async); #endif +DLL_EXPORT tmq_resp_err_t tmq_seek(tmq_t *tmq, const tmq_topic_vgroup_t *offset); /* ----------------------TMQ CONFIGURATION INTERFACE---------------------- */ enum tmq_conf_res_t { diff --git a/include/common/common.h b/include/common/common.h index fd5b6717ab..8fa2d03d6d 100644 --- a/include/common/common.h +++ b/include/common/common.h @@ -16,7 +16,6 @@ #ifndef TDENGINE_COMMON_H #define TDENGINE_COMMON_H - #ifdef __cplusplus extern "C" { #endif @@ -43,14 +42,16 @@ extern "C" { // int16_t bytes; // } SSchema; -#define TMQ_REQ_TYPE_COMMIT_ONLY 0 -#define TMQ_REQ_TYPE_CONSUME_ONLY 1 -#define TMQ_REQ_TYPE_CONSUME_AND_COMMIT 2 +enum { + TMQ_CONF__RESET_OFFSET__LATEST = -1, + TMQ_CONF__RESET_OFFSET__EARLIEAST = -2, + TMQ_CONF__RESET_OFFSET__NONE = -3, +}; typedef struct { uint32_t numOfTables; - SArray *pGroupList; - SHashObj *map; // speedup acquire the tableQueryInfo by table uid + SArray* pGroupList; + SHashObj* map; // speedup acquire the tableQueryInfo by table uid } STableGroupInfo; typedef struct SColumnDataAgg { @@ -79,14 +80,14 @@ typedef struct SConstantItem { // info.numOfCols = taosArrayGetSize(pDataBlock) + taosArrayGetSize(pConstantList); typedef struct SSDataBlock { - SColumnDataAgg *pBlockAgg; - SArray *pDataBlock; // SArray - SArray *pConstantList; // SArray, it is a constant/tags value of the corresponding result value. - SDataBlockInfo info; + SColumnDataAgg* pBlockAgg; + SArray* pDataBlock; // SArray + SArray* pConstantList; // SArray, it is a constant/tags value of the corresponding result value. + SDataBlockInfo info; } SSDataBlock; typedef struct SVarColAttr { - int32_t *offset; // start position for each entry in the list + int32_t* offset; // start position for each entry in the list uint32_t length; // used buffer size that contain the valid data uint32_t allocLen; // allocated buffer size } SVarColAttr; @@ -94,11 +95,11 @@ typedef struct SVarColAttr { // pBlockAgg->numOfNull == info.rows, all data are null // pBlockAgg->numOfNull == 0, no data are null. typedef struct SColumnInfoData { - SColumnInfo info; // TODO filter info needs to be removed - bool hasNull;// if current column data has null value. - char *pData; // the corresponding block data in memory + SColumnInfo info; // TODO filter info needs to be removed + bool hasNull; // if current column data has null value. + char* pData; // the corresponding block data in memory union { - char *nullbitmap; // bitmap, one bit for each item in the list + char* nullbitmap; // bitmap, one bit for each item in the list SVarColAttr varmeta; }; } SColumnInfoData; @@ -149,7 +150,6 @@ static FORCE_INLINE int32_t tEncodeSMqConsumeRsp(void** buf, const SMqConsumeRsp int32_t tlen = 0; int32_t sz = 0; tlen += taosEncodeFixedI64(buf, pRsp->consumerId); - tlen += taosEncodeFixedI64(buf, pRsp->committedOffset); tlen += taosEncodeFixedI64(buf, pRsp->reqOffset); tlen += taosEncodeFixedI64(buf, pRsp->rspOffset); tlen += taosEncodeFixedI32(buf, pRsp->skipLogNum); @@ -170,7 +170,6 @@ static FORCE_INLINE int32_t tEncodeSMqConsumeRsp(void** buf, const SMqConsumeRsp static FORCE_INLINE void* tDecodeSMqConsumeRsp(void* buf, SMqConsumeRsp* pRsp) { int32_t sz; buf = taosDecodeFixedI64(buf, &pRsp->consumerId); - buf = taosDecodeFixedI64(buf, &pRsp->committedOffset); buf = taosDecodeFixedI64(buf, &pRsp->reqOffset); buf = taosDecodeFixedI64(buf, &pRsp->rspOffset); buf = taosDecodeFixedI32(buf, &pRsp->skipLogNum); @@ -250,11 +249,11 @@ typedef struct SSqlExpr { char token[TSDB_COL_NAME_LEN]; // original token SSchema resSchema; - int32_t numOfCols; - SColumn* pColumns; // data columns that are required by query - int32_t interBytes; // inter result buffer size - int16_t numOfParams; // argument value of each function - SVariant param[3]; // parameters are not more than 3 + int32_t numOfCols; + SColumn* pColumns; // data columns that are required by query + int32_t interBytes; // inter result buffer size + int16_t numOfParams; // argument value of each function + SVariant param[3]; // parameters are not more than 3 } SSqlExpr; typedef struct SExprInfo { @@ -271,7 +270,7 @@ typedef struct SSessionWindow { SColumn col; } SSessionWindow; -#define QUERY_ASC_FORWARD_STEP 1 +#define QUERY_ASC_FORWARD_STEP 1 #define QUERY_DESC_FORWARD_STEP -1 #define GET_FORWARD_DIRECTION_FACTOR(ord) (((ord) == TSDB_ORDER_ASC) ? QUERY_ASC_FORWARD_STEP : QUERY_DESC_FORWARD_STEP) diff --git a/include/common/tep.h b/include/common/tep.h index 584b8a5a71..956cf6ec36 100644 --- a/include/common/tep.h +++ b/include/common/tep.h @@ -68,8 +68,8 @@ static FORCE_INLINE bool colDataIsNull(const SColumnInfoData* pColumnInfoData, u } #define colDataGet(p1_, r_) \ - ((IS_VAR_DATA_TYPE((p1_)->info.type)) ? (p1_)->pData + (p1_)->varmeta.offset[(r_)] \ - : (p1_)->pData + ((r_) * (p1_)->info.bytes)); + ((IS_VAR_DATA_TYPE((p1_)->info.type)) ? ((p1_)->pData + (p1_)->varmeta.offset[(r_)]) \ + : ((p1_)->pData + ((r_) * (p1_)->info.bytes))) int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, const char* pData, bool isNull); int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, const SColumnInfoData* pSource, uint32_t numOfRow2); diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 4bc8c9fb86..a6cd04e006 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -20,8 +20,9 @@ extern "C" { #endif -#include "tdef.h" #include "tcfg.h" +#include "tdef.h" +#include "tarray.h" // cluster extern char tsFirst[]; @@ -29,44 +30,39 @@ extern char tsSecond[]; extern char tsLocalFqdn[]; extern char tsLocalEp[]; extern uint16_t tsServerPort; +extern int32_t tsVersion; extern int32_t tsStatusInterval; -extern int8_t tsEnableTelemetryReporting; -extern int32_t tsNumOfSupportVnodes; +extern bool tsEnableTelemetryReporting; // common -extern int tsRpcTimer; -extern int tsRpcMaxTime; -extern int tsRpcForceTcp; // all commands go to tcp protocol if this is enabled -extern int32_t tsMaxConnections; -extern int32_t tsMaxShellConns; -extern int32_t tsShellActivityTimer; -extern uint32_t tsMaxTmrCtrl; -extern float tsNumOfThreadsPerCore; -extern int32_t tsNumOfCommitThreads; -extern float tsRatioOfQueryCores; -extern int8_t tsDaylight; -extern int8_t tsEnableCoreFile; -extern int32_t tsCompressMsgSize; -extern int32_t tsCompressColData; -extern int32_t tsMaxNumOfDistinctResults; -extern char tsTempDir[]; -extern int tsCompatibleModel; // 2.0 compatible model -extern int8_t tsEnableSlaveQuery; -extern int8_t tsEnableAdjustMaster; -extern int8_t tsPrintAuth; -extern int64_t tsTickPerDay[3]; +extern int32_t tsRpcTimer; +extern int32_t tsRpcMaxTime; +extern bool tsRpcForceTcp; // all commands go to tcp protocol if this is enabled +extern int32_t tsMaxConnections; +extern int32_t tsMaxShellConns; +extern int32_t tsShellActivityTimer; +extern int32_t tsMaxTmrCtrl; +extern float tsNumOfThreadsPerCore; +extern int32_t tsNumOfCommitThreads; +extern float tsRatioOfQueryCores; +extern int32_t tsCompressMsgSize; +extern int32_t tsCompressColData; +extern int32_t tsMaxNumOfDistinctResults; +extern int32_t tsCompatibleModel; +extern bool tsEnableSlaveQuery; +extern bool tsPrintAuth; +extern int64_t tsTickPerDay[3]; // query buffer management extern int32_t tsQueryBufferSize; // maximum allowed usage buffer size in MB for each data node during query processing extern int64_t tsQueryBufferSizeBytes; // maximum allowed usage buffer size in byte for each data node -extern int32_t tsRetrieveBlockingModel; // retrieve threads will be blocked -extern int8_t tsKeepOriginalColumnName; -extern int8_t tsDeadLockKillQuery; +extern bool tsRetrieveBlockingModel; // retrieve threads will be blocked +extern bool tsKeepOriginalColumnName; +extern bool tsDeadLockKillQuery; // client extern int32_t tsMaxWildCardsLen; extern int32_t tsMaxRegexStringLen; -extern int8_t tsTscEnableRecordSql; extern int32_t tsMaxNumOfOrderedResults; extern int32_t tsMinSlidingTime; extern int32_t tsMinIntervalTime; @@ -77,19 +73,6 @@ extern float tsStreamComputDelayRatio; // the delayed computing ration of the extern int32_t tsProjectExecInterval; extern int64_t tsMaxRetentWindow; -// system info -extern float tsTotalLogDirGB; -extern float tsTotalTmpDirGB; -extern float tsTotalDataDirGB; -extern float tsAvailLogDirGB; -extern float tsAvailTmpDirectorySpace; -extern float tsAvailDataDirGB; -extern float tsUsedDataDirGB; -extern float tsMinimalLogDirGB; -extern float tsReservedTmpDirectorySpace; -extern float tsMinimalDataDirGB; -extern uint32_t tsVersion; - // build info extern char version[]; extern char compatible_version[]; @@ -105,18 +88,19 @@ extern uint32_t tsMaxRange; extern uint32_t tsCurRange; extern char tsCompressor[]; +// tfs extern int32_t tsDiskCfgNum; extern SDiskCfg tsDiskCfg[]; #define NEEDTO_COMPRESSS_MSG(size) (tsCompressMsgSize != -1 && (size) > tsCompressMsgSize) -void taosInitGlobalCfg(); -int32_t taosCheckAndPrintCfg(); -int32_t taosCfgDynamicOptions(char *msg); -bool taosCheckBalanceCfgOptions(const char *option, int32_t *vnodeId, int32_t *dnodeId); -void taosAddDataDir(int index, char *v1, int level, int primary); -void taosReadDataDirCfg(char *v1, char *v2, char *v3); -void taosPrintDataDirCfg(); +int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDir, const char *envFile, + const char *apolloUrl, SArray *pArgs, bool tsc); +int32_t taosInitCfg(const char *cfgDir, const char *envFile, const char *apolloUrl, SArray *pArgs, bool tsc); +void taosCleanupCfg(); +void taosCfgDynamicOptions(const char *option, const char *value); + +struct SConfig *taosGetCfg(); #ifdef __cplusplus } diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 5164fb6ccd..9ab78f3c96 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -422,8 +422,8 @@ typedef struct { } SColumnInfo; typedef struct { - uint64_t uid; - TSKEY key; // last accessed ts, for subscription + int64_t uid; + TSKEY key; // last accessed ts, for subscription } STableIdInfo; typedef struct STimeWindow { @@ -554,8 +554,8 @@ int32_t tSerializeSDropDbReq(void* buf, int32_t bufLen, SDropDbReq* pReq); int32_t tDeserializeSDropDbReq(void* buf, int32_t bufLen, SDropDbReq* pReq); typedef struct { - char db[TSDB_DB_FNAME_LEN]; - uint64_t uid; + char db[TSDB_DB_FNAME_LEN]; + int64_t uid; } SDropDbRsp; int32_t tSerializeSDropDbRsp(void* buf, int32_t bufLen, SDropDbRsp* pRsp); @@ -570,12 +570,12 @@ int32_t tSerializeSUseDbReq(void* buf, int32_t bufLen, SUseDbReq* pReq); int32_t tDeserializeSUseDbReq(void* buf, int32_t bufLen, SUseDbReq* pReq); typedef struct { - char db[TSDB_DB_FNAME_LEN]; - uint64_t uid; - int32_t vgVersion; - int32_t vgNum; - int8_t hashMethod; - SArray* pVgroupInfos; // Array of SVgroupInfo + char db[TSDB_DB_FNAME_LEN]; + int64_t uid; + int32_t vgVersion; + int32_t vgNum; + int8_t hashMethod; + SArray* pVgroupInfos; // Array of SVgroupInfo } SUseDbRsp; int32_t tSerializeSUseDbRsp(void* buf, int32_t bufLen, SUseDbRsp* pRsp); @@ -656,9 +656,9 @@ int32_t tDeserializeSRetrieveFuncRsp(void* buf, int32_t bufLen, SRetrieveFuncRsp typedef struct { int32_t statusInterval; int64_t checkTime; // 1970-01-01 00:00:00.000 - char timezone[TSDB_TIMEZONE_LEN]; // tsTimezone - char locale[TSDB_LOCALE_LEN]; // tsLocale - char charset[TSDB_LOCALE_LEN]; // tsCharset + char timezone[TD_TIMEZONE_LEN]; // tsTimezone + char locale[TD_LOCALE_LEN]; // tsLocale + char charset[TD_LOCALE_LEN]; // tsCharset } SClusterCfg; typedef struct { @@ -725,7 +725,7 @@ typedef struct { int32_t vgId; int32_t dnodeId; char db[TSDB_DB_FNAME_LEN]; - uint64_t dbUid; + int64_t dbUid; int32_t vgVersion; int32_t cacheBlockSize; int32_t totalBlocks; @@ -753,10 +753,10 @@ int32_t tSerializeSCreateVnodeReq(void* buf, int32_t bufLen, SCreateVnodeReq* pR int32_t tDeserializeSCreateVnodeReq(void* buf, int32_t bufLen, SCreateVnodeReq* pReq); typedef struct { - int32_t vgId; - int32_t dnodeId; - uint64_t dbUid; - char db[TSDB_DB_FNAME_LEN]; + int32_t vgId; + int32_t dnodeId; + int64_t dbUid; + char db[TSDB_DB_FNAME_LEN]; } SDropVnodeReq, SSyncVnodeReq, SCompactVnodeReq; int32_t tSerializeSDropVnodeReq(void* buf, int32_t bufLen, SDropVnodeReq* pReq); @@ -796,7 +796,7 @@ typedef struct { char tbName[TSDB_TABLE_NAME_LEN]; char stbName[TSDB_TABLE_NAME_LEN]; char dbFName[TSDB_DB_FNAME_LEN]; - uint64_t dbId; + int64_t dbId; int32_t numOfTags; int32_t numOfColumns; int8_t precision; @@ -1268,7 +1268,7 @@ typedef struct { typedef struct { SMsgHead head; char name[TSDB_TABLE_FNAME_LEN]; - uint64_t tuid; + int64_t tuid; int32_t sverson; int32_t execLen; char* executor; @@ -1279,11 +1279,11 @@ typedef struct { typedef struct { SMsgHead head; char name[TSDB_TABLE_FNAME_LEN]; - uint64_t tuid; + int64_t tuid; } SDDropTopicReq; typedef struct SVCreateTbReq { - uint64_t ver; // use a general definition + int64_t ver; // use a general definition char* name; uint32_t ttl; uint32_t keep; @@ -1314,8 +1314,8 @@ int32_t tSerializeSVCreateTbReq(void** buf, SVCreateTbReq* pReq); void* tDeserializeSVCreateTbReq(void* buf, SVCreateTbReq* pReq); typedef struct { - uint64_t ver; // use a general definition - SArray* pArray; + int64_t ver; // use a general definition + SArray* pArray; } SVCreateTbBatchReq; typedef struct { @@ -1325,7 +1325,7 @@ int32_t tSerializeSVCreateTbBatchReq(void** buf, SVCreateTbBatchReq* pReq); void* tDeserializeSVCreateTbBatchReq(void* buf, SVCreateTbBatchReq* pReq); typedef struct { - uint64_t ver; + int64_t ver; char* name; uint8_t type; tb_uid_t suid; @@ -1760,35 +1760,19 @@ typedef struct { char cgroup[TSDB_CONSUMER_GROUP_LEN]; } SMqOffset; -typedef struct { - int32_t vgId; - SArray* offsets; // SArray -} SMqVgOffsets; - typedef struct { int32_t num; SMqOffset* offsets; -} SMqCMResetOffsetReq; +} SMqCMCommitOffsetReq; typedef struct { int32_t reserved; -} SMqCMResetOffsetRsp; - -typedef struct { - int64_t leftForVer; - SMqVgOffsets offsets; -} SMqMVResetOffsetReq; - -typedef struct { - int32_t reserved; -} SMqMVResetOffsetRsp; +} SMqCMCommitOffsetRsp; int32_t tEncodeSMqOffset(SCoder* encoder, const SMqOffset* pOffset); int32_t tDecodeSMqOffset(SCoder* decoder, SMqOffset* pOffset); -int32_t tEncodeSMqCMResetOffsetReq(SCoder* encoder, const SMqCMResetOffsetReq* pReq); -int32_t tDecodeSMqCMResetOffsetReq(SCoder* decoder, SMqCMResetOffsetReq* pReq); -int32_t tEncodeSMqMVResetOffsetReq(SCoder* encoder, const SMqMVResetOffsetReq* pReq); -int32_t tDecodeSMqMVResetOffsetReq(SCoder* decoder, SMqMVResetOffsetReq* pReq); +int32_t tEncodeSMqCMCommitOffsetReq(SCoder* encoder, const SMqCMCommitOffsetReq* pReq); +int32_t tDecodeSMqCMCommitOffsetReq(SCoder* decoder, SMqCMCommitOffsetReq* pReq); typedef struct { uint32_t nCols; @@ -1870,7 +1854,6 @@ typedef struct { typedef struct { int64_t consumerId; SSchemaWrapper* schemas; - int64_t committedOffset; int64_t reqOffset; int64_t rspOffset; int32_t skipLogNum; @@ -1881,22 +1864,18 @@ typedef struct { // one req for one vg+topic typedef struct { SMsgHead head; - // 0: commit only, current offset - // 1: consume only, poll next offset - // 2: commit current and consume next offset - int32_t reqType; - int64_t reqId; int64_t consumerId; int64_t blockingTime; char cgroup[TSDB_CONSUMER_GROUP_LEN]; - int64_t offset; + int64_t currentOffset; char topic[TSDB_TOPIC_FNAME_LEN]; } SMqConsumeReq; typedef struct { int32_t vgId; + int64_t offset; SEpSet epSet; } SMqSubVgEp; @@ -1917,12 +1896,14 @@ static FORCE_INLINE void tDeleteSMqSubTopicEp(SMqSubTopicEp* pSubTopicEp) { taos static FORCE_INLINE int32_t tEncodeSMqSubVgEp(void** buf, const SMqSubVgEp* pVgEp) { int32_t tlen = 0; tlen += taosEncodeFixedI32(buf, pVgEp->vgId); + tlen += taosEncodeFixedI64(buf, pVgEp->offset); tlen += taosEncodeSEpSet(buf, &pVgEp->epSet); return tlen; } static FORCE_INLINE void* tDecodeSMqSubVgEp(void* buf, SMqSubVgEp* pVgEp) { buf = taosDecodeFixedI32(buf, &pVgEp->vgId); + buf = taosDecodeFixedI64(buf, &pVgEp->offset); buf = taosDecodeSEpSet(buf, &pVgEp->epSet); return buf; } diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index 1a63ea73a5..a3e44968f7 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -135,18 +135,19 @@ enum { TD_DEF_MSG_TYPE(TDMT_MND_SHOW, "mnode-show", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_SHOW_RETRIEVE, "mnode-retrieve", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_STATUS, "mnode-status", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_TRANS, "mnode-trans", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_TRANS_TIMER, "mnode-trans-tmr", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_KILL_TRANS, "mnode-kill-trans", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_TELEM_TIMER, "mnode-telem-tmr", SMTimerReq, SMTimerReq) TD_DEF_MSG_TYPE(TDMT_MND_GRANT, "mnode-grant", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_AUTH, "mnode-auth", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_CREATE_TOPIC, "mnode-create-topic", SMCreateTopicReq, SMCreateTopicRsp) TD_DEF_MSG_TYPE(TDMT_MND_ALTER_TOPIC, "mnode-alter-topic", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_DROP_TOPIC, "mnode-drop-topic", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_SUBSCRIBE, "mnode-subscribe", SCMSubscribeReq, SCMSubscribeRsp) - TD_DEF_MSG_TYPE(TDMT_MND_RESET_OFFSET, "mnode-reset-offset", SMqCMResetOffsetReq, SMqCMResetOffsetRsp) TD_DEF_MSG_TYPE(TDMT_MND_GET_SUB_EP, "mnode-get-sub-ep", SMqCMGetSubEpReq, SMqCMGetSubEpRsp) - TD_DEF_MSG_TYPE(TDMT_MND_MQ_TIMER, "mnode-mq-timer", SMTimerReq, SMTimerReq) + TD_DEF_MSG_TYPE(TDMT_MND_MQ_TIMER, "mnode-mq-tmr", SMTimerReq, SMTimerReq) TD_DEF_MSG_TYPE(TDMT_MND_MQ_DO_REBALANCE, "mnode-mq-do-rebalance", SMqDoRebalanceMsg, SMqDoRebalanceMsg) + TD_DEF_MSG_TYPE(TDMT_MND_MQ_COMMIT_OFFSET, "mnode-mq-commit-offset", SMqCMCommitOffsetReq, SMqCMCommitOffsetRsp) // Requests handled by VNODE TD_NEW_MSG_SEG(TDMT_VND_MSG) @@ -194,4 +195,3 @@ enum { TDMT_MAX #endif }; -// clang-format on diff --git a/include/common/ttszip.h b/include/common/ttszip.h index 38699ae791..d83b0066d5 100644 --- a/include/common/ttszip.h +++ b/include/common/ttszip.h @@ -73,9 +73,9 @@ typedef struct STSGroupBlockInfoEx { } STSGroupBlockInfoEx; typedef struct STSBuf { - FILE* f; - char path[PATH_MAX]; - uint32_t fileSize; + TdFilePtr pFile; + char path[PATH_MAX]; + uint32_t fileSize; // todo use array STSGroupBlockInfoEx* pData; diff --git a/include/common/ttypes.h b/include/common/ttypes.h index 5a1e442277..39a5f0e2b8 100644 --- a/include/common/ttypes.h +++ b/include/common/ttypes.h @@ -91,6 +91,8 @@ typedef struct { do { \ switch (_type) { \ case TSDB_DATA_TYPE_BOOL: \ + *(bool *)(_v) = (bool)(_data); \ + break; \ case TSDB_DATA_TYPE_TINYINT: \ *(int8_t *)(_v) = (int8_t)(_data); \ break; \ @@ -104,6 +106,7 @@ typedef struct { *(uint16_t *)(_v) = (uint16_t)(_data); \ break; \ case TSDB_DATA_TYPE_BIGINT: \ + case TSDB_DATA_TYPE_TIMESTAMP: \ *(int64_t *)(_v) = (int64_t)(_data); \ break; \ case TSDB_DATA_TYPE_UBIGINT: \ diff --git a/include/dnode/mgmt/dnode.h b/include/dnode/mgmt/dnode.h index e64d358de4..bb0f66ca09 100644 --- a/include/dnode/mgmt/dnode.h +++ b/include/dnode/mgmt/dnode.h @@ -17,6 +17,7 @@ #define _TD_DNODE_H_ #include "tdef.h" +#include "tcfg.h" #ifdef __cplusplus extern "C" { @@ -25,26 +26,12 @@ extern "C" { /* ------------------------ TYPES EXPOSED ---------------- */ typedef struct SDnode SDnode; -/* ------------------------ Environment ------------------ */ -typedef struct { - int32_t sver; - int32_t numOfCores; - int16_t numOfCommitThreads; - int8_t enableTelem; - char timezone[TSDB_TIMEZONE_LEN]; - char locale[TSDB_LOCALE_LEN]; - char charset[TSDB_LOCALE_LEN]; - char buildinfo[64]; - char gitinfo[48]; -} SDnodeEnvCfg; - /** * @brief Initialize the environment * - * @param pOption Option of the environment * @return int32_t 0 for success and -1 for failure */ -int32_t dndInit(const SDnodeEnvCfg *pCfg); +int32_t dndInit(); /** * @brief clear the environment @@ -54,17 +41,15 @@ void dndCleanup(); /* ------------------------ SDnode ----------------------- */ typedef struct { - int32_t numOfSupportVnodes; - int32_t statusInterval; - float numOfThreadsPerCore; - float ratioOfQueryCores; - int32_t maxShellConns; - int32_t shellActivityTimer; - uint16_t serverPort; - char dataDir[TSDB_FILENAME_LEN]; - char localEp[TSDB_EP_LEN]; - char localFqdn[TSDB_FQDN_LEN]; - char firstEp[TSDB_EP_LEN]; + int32_t numOfSupportVnodes; + uint16_t serverPort; + char dataDir[TSDB_FILENAME_LEN]; + char localEp[TSDB_EP_LEN]; + char localFqdn[TSDB_FQDN_LEN]; + char firstEp[TSDB_EP_LEN]; + char secondEp[TSDB_EP_LEN]; + SDiskCfg *pDisks; + int32_t numOfDisks; } SDnodeObjCfg; /** diff --git a/include/dnode/mnode/mnode.h b/include/dnode/mnode/mnode.h index e1306800d3..2680bb83ed 100644 --- a/include/dnode/mnode/mnode.h +++ b/include/dnode/mnode/mnode.h @@ -44,25 +44,12 @@ typedef struct SMnodeLoad { int64_t compStorage; } SMnodeLoad; -typedef struct SMnodeCfg { - int32_t sver; - int8_t enableTelem; - int32_t statusInterval; - int32_t shellActivityTimer; - char *timezone; - char *locale; - char *charset; - char *buildinfo; - char *gitinfo; -} SMnodeCfg; - typedef struct { int32_t dnodeId; int64_t clusterId; int8_t replica; int8_t selfIndex; SReplica replicas[TSDB_MAX_REPLICA]; - SMnodeCfg cfg; SDnode *pDnode; PutReqToMWriteQFp putReqToMWriteQFp; PutReqToMReadQFp putReqToMReadQFp; diff --git a/include/dnode/mnode/sdb/sdb.h b/include/dnode/mnode/sdb/sdb.h index 1e967a6d2b..bf48a8523c 100644 --- a/include/dnode/mnode/sdb/sdb.h +++ b/include/dnode/mnode/sdb/sdb.h @@ -113,14 +113,15 @@ typedef enum { SDB_USER = 7, SDB_AUTH = 8, SDB_ACCT = 9, - SDB_SUBSCRIBE = 10, - SDB_CONSUMER = 11, - SDB_TOPIC = 12, - SDB_VGROUP = 13, - SDB_STB = 14, - SDB_DB = 15, - SDB_FUNC = 16, - SDB_MAX = 17 + SDB_OFFSET = 10, + SDB_SUBSCRIBE = 11, + SDB_CONSUMER = 12, + SDB_TOPIC = 13, + SDB_VGROUP = 14, + SDB_STB = 15, + SDB_DB = 16, + SDB_FUNC = 17, + SDB_MAX = 18 } ESdbType; typedef struct SSdb SSdb; diff --git a/include/libs/function/function.h b/include/libs/function/function.h index bb5d51686d..e970a0d693 100644 --- a/include/libs/function/function.h +++ b/include/libs/function/function.h @@ -228,6 +228,7 @@ typedef struct SAggFunctionInfo { typedef struct SScalarParam { void* data; + bool colData; int32_t num; int32_t type; int32_t bytes; diff --git a/include/libs/nodes/nodes.h b/include/libs/nodes/nodes.h index 3ef832a69b..d5958e1b9c 100644 --- a/include/libs/nodes/nodes.h +++ b/include/libs/nodes/nodes.h @@ -61,22 +61,28 @@ typedef enum ENodeType { QUERY_NODE_INTERVAL_WINDOW, QUERY_NODE_NODE_LIST, QUERY_NODE_FILL, - QUERY_NODE_COLUMN_REF, + QUERY_NODE_RAW_EXPR, // Only be used in parser module. QUERY_NODE_TARGET, - - // Only be used in parser module. - QUERY_NODE_RAW_EXPR, + QUERY_NODE_TUPLE_DESC, + QUERY_NODE_SLOT_DESC, // Statement nodes are used in parser and planner module. QUERY_NODE_SET_OPERATOR, QUERY_NODE_SELECT_STMT, QUERY_NODE_SHOW_STMT, + // logic plan node QUERY_NODE_LOGIC_PLAN_SCAN, QUERY_NODE_LOGIC_PLAN_JOIN, - QUERY_NODE_LOGIC_PLAN_FILTER, QUERY_NODE_LOGIC_PLAN_AGG, - QUERY_NODE_LOGIC_PLAN_PROJECT + QUERY_NODE_LOGIC_PLAN_PROJECT, + + // physical plan node + QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN, + QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN, + QUERY_NODE_PHYSICAL_PLAN_PROJECT, + QUERY_NODE_PHYSICAL_PLAN_JOIN, + QUERY_NODE_PHYSICAL_PLAN_AGG } ENodeType; /** diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index 3668f256eb..d8896501ad 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -21,6 +21,7 @@ extern "C" { #endif #include "querynodes.h" +#include "tmsg.h" typedef struct SLogicNode { ENodeType type; @@ -31,10 +32,20 @@ typedef struct SLogicNode { struct SLogicNode* pParent; } SLogicNode; +typedef enum EScanType { + SCAN_TYPE_TAG, + SCAN_TYPE_TABLE, + SCAN_TYPE_STABLE, + SCAN_TYPE_STREAM +} EScanType; + typedef struct SScanLogicNode { SLogicNode node; SNodeList* pScanCols; struct STableMeta* pMeta; + EScanType scanType; + uint8_t scanFlag; // denotes reversed scan of data or not + STimeWindow scanRange; } SScanLogicNode; typedef struct SJoinLogicNode { @@ -43,10 +54,6 @@ typedef struct SJoinLogicNode { SNode* pOnConditions; } SJoinLogicNode; -typedef struct SFilterLogicNode { - SLogicNode node; -} SFilterLogicNode; - typedef struct SAggLogicNode { SLogicNode node; SNodeList* pGroupKeys; @@ -58,6 +65,68 @@ typedef struct SProjectLogicNode { SNodeList* pProjections; } SProjectLogicNode; +typedef struct SSlotDescNode { + ENodeType type; + int16_t slotId; + SDataType dataType; + bool reserve; + bool output; +} SSlotDescNode; + +typedef struct STupleDescNode { + ENodeType type; + int16_t tupleId; + SNodeList* pSlots; +} STupleDescNode; + +typedef struct SPhysiNode { + ENodeType type; + STupleDescNode outputTuple; + SNode* pConditions; + SNodeList* pChildren; + struct SPhysiNode* pParent; +} SPhysiNode; + +typedef struct SScanPhysiNode { + SPhysiNode node; + SNodeList* pScanCols; + uint64_t uid; // unique id of the table + int8_t tableType; + int32_t order; // scan order: TSDB_ORDER_ASC|TSDB_ORDER_DESC + int32_t count; // repeat count + int32_t reverse; // reverse scan count +} SScanPhysiNode; + +typedef SScanPhysiNode SSystemTableScanPhysiNode; +typedef SScanPhysiNode STagScanPhysiNode; + +typedef struct STableScanPhysiNode { + SScanPhysiNode scan; + uint8_t scanFlag; // denotes reversed scan of data or not + STimeWindow scanRange; +} STableScanPhysiNode; + +typedef STableScanPhysiNode STableSeqScanPhysiNode; + +typedef struct SProjectPhysiNode { + SPhysiNode node; + SNodeList* pProjections; +} SProjectPhysiNode; + +typedef struct SJoinPhysiNode { + SPhysiNode node; + EJoinType joinType; + SNode* pOnConditions; // in or out tuple ? + SNodeList* pTargets; +} SJoinPhysiNode; + +typedef struct SAggPhysiNode { + SPhysiNode node; + SNodeList* pExprs; // these are expression list of group_by_clause and parameter expression of aggregate function + SNodeList* pGroupKeys; // SColumnRefNode list + SNodeList* pAggFuncs; +} SAggPhysiNode; + #ifdef __cplusplus } #endif diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index 7c75562759..c54e1b3ad0 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -58,15 +58,24 @@ typedef struct SColumnNode { char tableAlias[TSDB_TABLE_NAME_LEN]; char colName[TSDB_COL_NAME_LEN]; SNode* pProjectRef; -} SColumnNode; - -typedef struct SColumnRefNode { - ENodeType type; - SDataType dataType; int16_t tupleId; int16_t slotId; - int16_t columnId; -} SColumnRefNode; +} SColumnNode; + +// typedef struct SColumnRefNode { +// ENodeType type; +// SDataType dataType; +// int16_t tupleId; +// int16_t slotId; +// int16_t columnId; +// } SColumnRefNode; + +typedef struct STargetNode { + ENodeType type; + int16_t tupleId; + int16_t slotId; + SNode* pExpr; +} STargetNode; typedef struct SValueNode { SExprNode node; // QUERY_NODE_VALUE @@ -81,45 +90,6 @@ typedef struct SValueNode { } datum; } SValueNode; -typedef enum EOperatorType { - // arithmetic operator - OP_TYPE_ADD = 1, - OP_TYPE_SUB, - OP_TYPE_MULTI, - OP_TYPE_DIV, - OP_TYPE_MOD, - - // bit operator - OP_TYPE_BIT_AND, - OP_TYPE_BIT_OR, - - // comparison operator - OP_TYPE_GREATER_THAN, - OP_TYPE_GREATER_EQUAL, - OP_TYPE_LOWER_THAN, - OP_TYPE_LOWER_EQUAL, - OP_TYPE_EQUAL, - OP_TYPE_NOT_EQUAL, - OP_TYPE_IN, - OP_TYPE_NOT_IN, - OP_TYPE_LIKE, - OP_TYPE_NOT_LIKE, - OP_TYPE_MATCH, - OP_TYPE_NMATCH, - OP_TYPE_IS_NULL, - OP_TYPE_IS_NOT_NULL, - OP_TYPE_IS_TRUE, - OP_TYPE_IS_FALSE, - OP_TYPE_IS_UNKNOWN, - OP_TYPE_IS_NOT_TRUE, - OP_TYPE_IS_NOT_FALSE, - OP_TYPE_IS_NOT_UNKNOWN, - - // json operator - OP_TYPE_JSON_GET_VALUE, - OP_TYPE_JSON_CONTAINS -} EOperatorType; - typedef struct SOperatorNode { SExprNode node; // QUERY_NODE_OPERATOR EOperatorType opType; @@ -127,11 +97,6 @@ typedef struct SOperatorNode { SNode* pRight; } SOperatorNode; -typedef enum ELogicConditionType { - LOGIC_COND_TYPE_AND, - LOGIC_COND_TYPE_OR, - LOGIC_COND_TYPE_NOT, -} ELogicConditionType; typedef struct SLogicConditionNode { SExprNode node; // QUERY_NODE_LOGIC_CONDITION @@ -141,6 +106,7 @@ typedef struct SLogicConditionNode { typedef struct SNodeListNode { ENodeType type; // QUERY_NODE_NODE_LIST + SDataType dataType; SNodeList* pNodeList; } SNodeListNode; @@ -306,7 +272,8 @@ bool nodesIsJsonOp(const SOperatorNode* pOp); bool nodesIsTimeorderQuery(const SNode* pQuery); bool nodesIsTimelineQuery(const SNode* pQuery); -void *nodesGetValueFromNode(SValueNode *pNode); + +void* nodesGetValueFromNode(SValueNode *pNode); #ifdef __cplusplus } diff --git a/include/libs/scalar/filter.h b/include/libs/scalar/filter.h index 8db74f4587..fedb487931 100644 --- a/include/libs/scalar/filter.h +++ b/include/libs/scalar/filter.h @@ -20,6 +20,14 @@ extern "C" { #endif typedef struct SFilterInfo SFilterInfo; +typedef int32_t (*filer_get_col_from_id)(void *, int32_t, void **); + + +enum { + FLT_OPTION_NO_REWRITE = 1, + FLT_OPTION_TIMESTAMP = 2, + FLT_OPTION_NEED_UNIQE = 4, +}; typedef struct SFilterColumnParam{ int32_t numOfCols; @@ -27,8 +35,18 @@ typedef struct SFilterColumnParam{ } SFilterColumnParam; +extern int32_t filterInitFromNode(SNode *pNode, SFilterInfo **pinfo, uint32_t options); +extern bool filterExecute(SFilterInfo *info, SSDataBlock *pSrc, int8_t** p, SColumnDataAgg *statis, int16_t numOfCols); +extern int32_t filterSetDataFromSlotId(SFilterInfo *info, void *param); +extern int32_t filterSetDataFromColId(SFilterInfo *info, void *param); +extern int32_t filterGetTimeRange(SFilterInfo *info, STimeWindow *win); +extern int32_t filterConverNcharColumns(SFilterInfo* pFilterInfo, int32_t rows, bool *gotNchar); +extern int32_t filterFreeNcharColumns(SFilterInfo* pFilterInfo); +extern void filterFreeInfo(SFilterInfo *info); +extern bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg *pDataStatis, int32_t numOfCols, int32_t numOfRows); + #ifdef __cplusplus } #endif -#endif // TDENGINE_FILTER_H \ No newline at end of file +#endif // TDENGINE_FILTER_H diff --git a/include/libs/scalar/scalar.h b/include/libs/scalar/scalar.h index 4dbdb2e8bb..1902c11cc6 100644 --- a/include/libs/scalar/scalar.h +++ b/include/libs/scalar/scalar.h @@ -29,6 +29,7 @@ typedef struct SFilterInfo SFilterInfo; int32_t scalarCalculateConstants(SNode *pNode, SNode **pRes); int32_t scalarCalculate(SNode *pNode, SSDataBlock *pSrc, SScalarParam *pDst); int32_t scalarGetOperatorParamNum(EOperatorType type); +int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type); int32_t vectorGetConvertType(int32_t type1, int32_t type2); int32_t vectorConvertImpl(SScalarParam* pIn, SScalarParam* pOut); diff --git a/include/libs/transport/trpc.h b/include/libs/transport/trpc.h index 538aeb1a0e..5e3860822e 100644 --- a/include/libs/transport/trpc.h +++ b/include/libs/transport/trpc.h @@ -78,12 +78,15 @@ typedef struct SRpcInit { // call back to retrieve the client auth info, for server app only int (*afp)(void *parent, char *tableId, char *spi, char *encrypt, char *secret, char *ckey); + // call back to keep conn or not + bool (*pfp)(void *parent, tmsg_t msgType); + void *parent; } SRpcInit; int32_t rpcInit(); void rpcCleanup(); -void * rpcOpen(const SRpcInit *pRpc); +void *rpcOpen(const SRpcInit *pRpc); void rpcClose(void *); void * rpcMallocCont(int contLen); void rpcFreeCont(void *pCont); diff --git a/include/libs/wal/wal.h b/include/libs/wal/wal.h index 45f1d88c30..f90dbb97fe 100644 --- a/include/libs/wal/wal.h +++ b/include/libs/wal/wal.h @@ -120,8 +120,8 @@ typedef struct SWal { int32_t fsyncSeq; // meta SWalVer vers; - int64_t writeLogTfd; - int64_t writeIdxTfd; + TdFilePtr pWriteLogTFile; + TdFilePtr pWriteIdxTFile; int32_t writeCur; SArray *fileInfoSet; // status @@ -138,8 +138,8 @@ typedef struct SWal { typedef struct SWalReadHandle { SWal *pWal; - int64_t readLogTfd; - int64_t readIdxTfd; + TdFilePtr pReadLogTFile; + TdFilePtr pReadIdxTFile; int64_t curFileFirstVer; int64_t curVersion; int64_t capacity; diff --git a/include/os/os.h b/include/os/os.h index 023d2b0470..d96f7efb51 100644 --- a/include/os/os.h +++ b/include/os/os.h @@ -59,6 +59,7 @@ extern "C" { #include "osEndian.h" #include "osEnv.h" #include "osFile.h" +#include "osLocale.h" #include "osLz4.h" #include "osMath.h" #include "osMemory.h" @@ -73,6 +74,7 @@ extern "C" { #include "osThread.h" #include "osTime.h" #include "osTimer.h" +#include "osTimezone.h" void osInit(); diff --git a/include/os/osDef.h b/include/os/osDef.h index 07d360a7c0..339bf13343 100644 --- a/include/os/osDef.h +++ b/include/os/osDef.h @@ -191,6 +191,10 @@ extern "C" { #define TD_DIRSEP "/" #endif +#define TD_LOCALE_LEN 64 +#define TD_CHARSET_LEN 64 +#define TD_TIMEZONE_LEN 96 + #ifdef __cplusplus } #endif diff --git a/include/os/osDir.h b/include/os/osDir.h index 35c294af39..e328c42063 100644 --- a/include/os/osDir.h +++ b/include/os/osDir.h @@ -23,8 +23,8 @@ extern "C" { void taosRemoveDir(const char *dirname); int32_t taosDirExist(char *dirname); int32_t taosMkDir(const char *dirname); -void taosRemoveOldFiles(char *dirname, int32_t keepDays); -int32_t taosExpandDir(char *dirname, char *outname, int32_t maxlen); +void taosRemoveOldFiles(const char *dirname, int32_t keepDays); +int32_t taosExpandDir(const char *dirname, char *outname, int32_t maxlen); int32_t taosRealPath(char *dirname, int32_t maxlen); #ifdef __cplusplus diff --git a/include/os/osEnv.h b/include/os/osEnv.h index a7fd86776c..4ac073f6c2 100644 --- a/include/os/osEnv.h +++ b/include/os/osEnv.h @@ -16,15 +16,37 @@ #ifndef _TD_OS_ENV_H_ #define _TD_OS_ENV_H_ +#include "osSysinfo.h" + #ifdef __cplusplus extern "C" { #endif -extern char tsOsName[]; +extern char tsOsName[]; +extern char tsTimezone[]; +extern char tsCharset[]; +extern char tsLocale[]; +extern int8_t tsDaylight; +extern bool tsEnableCoreFile; +extern int64_t tsPageSize; +extern int64_t tsOpenMax; +extern int64_t tsStreamMax; +extern int32_t tsNumOfCores; +extern int32_t tsTotalMemoryMB; + +extern char configDir[]; extern char tsDataDir[]; extern char tsLogDir[]; -extern char tsScriptDir[]; -extern char configDir[]; +extern char tsTempDir[]; + +extern SDiskSpace tsDataSpace; +extern SDiskSpace tsLogSpace; +extern SDiskSpace tsTempSpace; + +void osInit(); +void osUpdate(); +bool osLogSpaceAvailable(); +void osSetTimezone(const char *timezone); #ifdef __cplusplus } diff --git a/include/os/osFile.h b/include/os/osFile.h index ac399fa3be..cedf26d04a 100644 --- a/include/os/osFile.h +++ b/include/os/osFile.h @@ -22,49 +22,63 @@ extern "C" { #include "osSocket.h" -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) -typedef int32_t FileFd; -#else -typedef int32_t FileFd; +#ifndef ALLOW_FORBID_FUNC + #define open OPEN_FUNC_TAOS_FORBID + #define fopen FOPEN_FUNC_TAOS_FORBID + // #define close CLOSE_FUNC_TAOS_FORBID + // #define fclose FCLOSE_FUNC_TAOS_FORBID #endif -#define FD_INITIALIZER ((int32_t)-1) - #ifndef PATH_MAX #define PATH_MAX 256 #endif -int32_t taosLockFile(FileFd fd); -int32_t taosUnLockFile(FileFd fd); - -int32_t taosUmaskFile(FileFd fd); - +typedef struct TdFile *TdFilePtr; + +#define TD_FILE_CTEATE 0x0001 +#define TD_FILE_WRITE 0x0002 +#define TD_FILE_READ 0x0004 +#define TD_FILE_TRUNC 0x0008 +#define TD_FILE_APPEND 0x0010 +#define TD_FILE_TEXT 0x0020 +#define TD_FILE_AUTO_DEL 0x0040 +#define TD_FILE_EXCL 0x0080 + +int32_t taosLockFile(TdFilePtr pFile); +int32_t taosUnLockFile(TdFilePtr pFile); + +int32_t taosUmaskFile(int32_t maskVal); + int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime); -int32_t taosFStatFile(FileFd fd, int64_t *size, int32_t *mtime); - -FileFd taosOpenFileWrite(const char *path); -FileFd taosOpenFileCreateWrite(const char *path); -FileFd taosOpenFileCreateWriteTrunc(const char *path); -FileFd taosOpenFileCreateWriteAppend(const char *path); -FileFd taosOpenFileRead(const char *path); -FileFd taosOpenFileReadWrite(const char *path); - -int64_t taosLSeekFile(FileFd fd, int64_t offset, int32_t whence); -int32_t taosFtruncateFile(FileFd fd, int64_t length); -int32_t taosFsyncFile(FileFd fd); - -int64_t taosReadFile(FileFd fd, void *buf, int64_t count); -int64_t taosWriteFile(FileFd fd, const void *buf, int64_t count); - -void taosCloseFile(FileFd fd); - +int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime); + +TdFilePtr taosOpenFile(const char *path,int32_t tdFileOptions); + +int64_t taosLSeekFile(TdFilePtr pFile, int64_t offset, int32_t whence); +int32_t taosFtruncateFile(TdFilePtr pFile, int64_t length); +int32_t taosFsyncFile(TdFilePtr pFile); + +int64_t taosReadFile(TdFilePtr pFile, void *buf, int64_t count); +int64_t taosPReadFile(TdFilePtr pFile, void *buf, int64_t count, int64_t offset); +int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count); +void taosFprintfFile(TdFilePtr pFile, const char *format, ...); +int64_t taosGetLineFile(TdFilePtr pFile, char ** __restrict__ ptrBuf); +int32_t taosEOFFile(TdFilePtr pFile); + +int64_t taosCloseFile(TdFilePtr *ppFile); + int32_t taosRenameFile(const char *oldName, const char *newName); int64_t taosCopyFile(const char *from, const char *to); - + void taosGetTmpfilePath(const char *inputTmpDir, const char *fileNamePrefix, char *dstPath); + +int64_t taosSendFile(SocketFd fdDst, TdFilePtr pFileSrc, int64_t *offset, int64_t size); +int64_t taosFSendFile(TdFilePtr pFileOut, TdFilePtr pFileIn, int64_t *offset, int64_t size); -int64_t taosSendFile(SocketFd dfd, FileFd sfd, int64_t *offset, int64_t size); -int64_t taosFSendFile(FILE *outfile, FILE *infile, int64_t *offset, int64_t size); +void *taosMmapReadOnlyFile(TdFilePtr pFile, int64_t length); +bool taosValidFile(TdFilePtr pFile); + +int taosGetErrorFile(TdFilePtr pFile); #ifdef __cplusplus } diff --git a/include/os/osLocale.h b/include/os/osLocale.h new file mode 100644 index 0000000000..6e313eb8cd --- /dev/null +++ b/include/os/osLocale.h @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_OS_LOCALE_H_ +#define _TD_OS_LOCALE_H_ + +#include "os.h" +#include "osString.h" + +#ifdef __cplusplus +extern "C" { +#endif + +char *taosCharsetReplace(char *charsetstr); +void taosGetSystemLocale(char *outLocale, char *outCharset); +void taosSetSystemLocale(const char *inLocale, const char *inCharSet); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_OS_LOCALE_H_*/ diff --git a/include/os/osSocket.h b/include/os/osSocket.h index d76b286d65..af99e4b474 100644 --- a/include/os/osSocket.h +++ b/include/os/osSocket.h @@ -55,7 +55,7 @@ void taosSetMaskSIGPIPE(); int32_t taosSetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t optlen); int32_t taosGetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t *optlen); -uint32_t taosInetAddr(char *ipAddr); +uint32_t taosInetAddr(const char *ipAddr); const char *taosInetNtoa(struct in_addr ipInt); #if (defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)) diff --git a/include/os/osString.h b/include/os/osString.h index 582411d444..88160dd69e 100644 --- a/include/os/osString.h +++ b/include/os/osString.h @@ -45,7 +45,6 @@ int32_t taosUcs4ToMbs(void *ucs4, int32_t ucs4_max_len, char *mbs); bool taosMbsToUcs4(const char *mbs, size_t mbs_len, char *ucs4, int32_t ucs4_max_len, int32_t *len); int32_t tasoUcs4Compare(void *f1_ucs4, void *f2_ucs4, int32_t bytes, int8_t ncharSize); bool taosValidateEncodec(const char *encodec); -char * taosCharsetReplace(char *charsetstr); #ifdef __cplusplus } diff --git a/include/os/osSysinfo.h b/include/os/osSysinfo.h index 9dcb075489..9f9061d243 100644 --- a/include/os/osSysinfo.h +++ b/include/os/osSysinfo.h @@ -21,18 +21,6 @@ extern "C" { #endif #include "os.h" - -#define TSDB_LOCALE_LEN 64 -#define TSDB_TIMEZONE_LEN 96 - -extern int64_t tsPageSize; -extern int64_t tsOpenMax; -extern int64_t tsStreamMax; -extern int32_t tsNumOfCores; -extern int32_t tsTotalMemoryMB; -extern char tsTimezone[]; -extern char tsLocale[]; -extern char tsCharset[]; // default encode string typedef struct { int64_t total; @@ -40,6 +28,11 @@ typedef struct { int64_t avail; } SDiskSize; +typedef struct { + int64_t reserved; + SDiskSize size; +} SDiskSpace; + int32_t taosGetDiskSize(char *dataDir, SDiskSize *diskSize); int32_t taosGetCpuCores(); void taosGetSystemInfo(); @@ -51,7 +44,6 @@ void taosGetDisk(); bool taosGetCpuUsage(float *sysCpuUsage, float *procCpuUsage); bool taosGetProcMemory(float *memoryUsedMB); bool taosGetSysMemory(float *memoryUsedMB); -void taosPrintOsInfo(); int taosSystem(const char *cmd); void taosKillSystem(); int32_t taosGetSystemUUID(char *uid, int32_t uidlen); diff --git a/include/common/ttimezone.h b/include/os/osTimezone.h similarity index 75% rename from include/common/ttimezone.h rename to include/os/osTimezone.h index cc50a29d57..ff015ef0b1 100644 --- a/include/common/ttimezone.h +++ b/include/os/osTimezone.h @@ -13,17 +13,18 @@ * along with this program. If not, see . */ -#ifndef _TD_COMMON_TIMEZONE_H_ -#define _TD_COMMON_TIMEZONE_H_ +#ifndef _TD_OS_TIMEZONE_H_ +#define _TD_OS_TIMEZONE_H_ #ifdef __cplusplus extern "C" { #endif -void tsSetTimeZone(); +void taosGetSystemTimezone(char *outTimezone); +void taosSetSystemTimezone(const char *inTimezone, char *outTimezone, int8_t *outDaylight); #ifdef __cplusplus } #endif -#endif /*_TD_COMMON_TIMEZONE_H_*/ +#endif /*_TD_OS_TIMEZONE_H_*/ diff --git a/include/util/compare.h b/include/util/compare.h index 6305334e02..ece8ffaf42 100644 --- a/include/util/compare.h +++ b/include/util/compare.h @@ -16,17 +16,18 @@ #ifndef _TD_UTIL_COMPARE_H_ #define _TD_UTIL_COMPARE_H_ +#include "os.h" + #ifdef __cplusplus extern "C" { #endif -#include "os.h" -#define TSDB_PATTERN_MATCH 0 -#define TSDB_PATTERN_NOMATCH 1 -#define TSDB_PATTERN_NOWILDCARDMATCH 2 +#define TSDB_PATTERN_MATCH 0 +#define TSDB_PATTERN_NOMATCH 1 +#define TSDB_PATTERN_NOWILDCARDMATCH 2 #define TSDB_PATTERN_STRING_DEFAULT_LEN 100 -#define TSDB_REGEX_STRING_DEFAULT_LEN 128 +#define TSDB_REGEX_STRING_DEFAULT_LEN 128 #define FLT_COMPAR_TOL_FACTOR 4 #define FLT_EQUAL(_x, _y) (fabs((_x) - (_y)) <= (FLT_COMPAR_TOL_FACTOR * FLT_EPSILON)) @@ -62,7 +63,6 @@ int32_t setChkNotInBytes8(const void *pLeft, const void *pRight); int32_t compareChkInString(const void *pLeft, const void *pRight); int32_t compareChkNotInString(const void *pLeft, const void *pRight); - int32_t compareInt8Val(const void *pLeft, const void *pRight); int32_t compareInt16Val(const void *pLeft, const void *pRight); int32_t compareInt32Val(const void *pLeft, const void *pRight); @@ -83,7 +83,6 @@ int32_t compareStrRegexComp(const void *pLeft, const void *pRight); int32_t compareStrRegexCompMatch(const void *pLeft, const void *pRight); int32_t compareStrRegexCompNMatch(const void *pLeft, const void *pRight); - int32_t compareInt8ValDesc(const void *pLeft, const void *pRight); int32_t compareInt16ValDesc(const void *pLeft, const void *pRight); int32_t compareInt32ValDesc(const void *pLeft, const void *pRight); @@ -102,7 +101,6 @@ int32_t compareLenPrefixedWStrDesc(const void *pLeft, const void *pRight); __compar_fn_t getComparFunc(int32_t type, int32_t optr); - #ifdef __cplusplus } #endif diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 92028b0837..7b42f3fea0 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -13,13 +13,15 @@ * along with this program. If not, see . */ -#ifndef _TD_COMMON_TAOS_ERROR_H_ -#define _TD_COMMON_TAOS_ERROR_H_ +#ifndef _TD_UTIL_TAOS_ERROR_H_ +#define _TD_UTIL_TAOS_ERROR_H_ #ifdef __cplusplus extern "C" { #endif +// clang-format off + #define TAOS_DEF_ERROR_CODE(mod, code) ((int32_t)((0x80000000 | ((mod)<<16) | (code)))) #define TAOS_SYSTEM_ERROR(code) (0x80ff0000 | (code)) @@ -36,28 +38,28 @@ int32_t* taosGetErrno(); #define TSDB_CODE_FAILED -1 // unknown or needn't tell detail error // rpc -#define TSDB_CODE_RPC_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0001) //"Action in progress") -#define TSDB_CODE_RPC_AUTH_REQUIRED TAOS_DEF_ERROR_CODE(0, 0x0002) //"Authentication required") -#define TSDB_CODE_RPC_AUTH_FAILURE TAOS_DEF_ERROR_CODE(0, 0x0003) //"Authentication failure") -#define TSDB_CODE_RPC_REDIRECT TAOS_DEF_ERROR_CODE(0, 0x0004) //"Redirect") -#define TSDB_CODE_RPC_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x0005) //"System not ready") // peer is not ready to process data -#define TSDB_CODE_RPC_ALREADY_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0006) //"Message already processed") -#define TSDB_CODE_RPC_LAST_SESSION_NOT_FINISHED TAOS_DEF_ERROR_CODE(0, 0x0007) //"Last session not finished") -#define TSDB_CODE_RPC_MISMATCHED_LINK_ID TAOS_DEF_ERROR_CODE(0, 0x0008) //"Mismatched meter id") -#define TSDB_CODE_RPC_TOO_SLOW TAOS_DEF_ERROR_CODE(0, 0x0009) //"Processing of request timed out") -#define TSDB_CODE_RPC_MAX_SESSIONS TAOS_DEF_ERROR_CODE(0, 0x000A) //"Number of sessions reached limit") // too many sessions -#define TSDB_CODE_RPC_NETWORK_UNAVAIL TAOS_DEF_ERROR_CODE(0, 0x000B) //"Unable to establish connection") -#define TSDB_CODE_RPC_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x000C) //"Unexpected generic error in RPC") -#define TSDB_CODE_RPC_UNEXPECTED_RESPONSE TAOS_DEF_ERROR_CODE(0, 0x000D) //"Unexpected response") -#define TSDB_CODE_RPC_INVALID_VALUE TAOS_DEF_ERROR_CODE(0, 0x000E) //"Invalid value") -#define TSDB_CODE_RPC_INVALID_TRAN_ID TAOS_DEF_ERROR_CODE(0, 0x000F) //"Invalid transaction id") -#define TSDB_CODE_RPC_INVALID_SESSION_ID TAOS_DEF_ERROR_CODE(0, 0x0010) //"Invalid session id") -#define TSDB_CODE_RPC_INVALID_MSG_TYPE TAOS_DEF_ERROR_CODE(0, 0x0011) //"Invalid message type") -#define TSDB_CODE_RPC_INVALID_RESPONSE_TYPE TAOS_DEF_ERROR_CODE(0, 0x0012) //"Invalid response type") -#define TSDB_CODE_RPC_INVALID_TIME_STAMP TAOS_DEF_ERROR_CODE(0, 0x0013) //"Client and server's time is not synchronized") -#define TSDB_CODE_APP_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x0014) //"Database not ready") -#define TSDB_CODE_RPC_FQDN_ERROR TAOS_DEF_ERROR_CODE(0, 0x0015) //"Unable to resolve FQDN") -#define TSDB_CODE_RPC_INVALID_VERSION TAOS_DEF_ERROR_CODE(0, 0x0016) //"Invalid app version") +#define TSDB_CODE_RPC_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0001) +#define TSDB_CODE_RPC_AUTH_REQUIRED TAOS_DEF_ERROR_CODE(0, 0x0002) +#define TSDB_CODE_RPC_AUTH_FAILURE TAOS_DEF_ERROR_CODE(0, 0x0003) +#define TSDB_CODE_RPC_REDIRECT TAOS_DEF_ERROR_CODE(0, 0x0004) +#define TSDB_CODE_RPC_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x0005) +#define TSDB_CODE_RPC_ALREADY_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0006) +#define TSDB_CODE_RPC_LAST_SESSION_NOT_FINISHED TAOS_DEF_ERROR_CODE(0, 0x0007) +#define TSDB_CODE_RPC_MISMATCHED_LINK_ID TAOS_DEF_ERROR_CODE(0, 0x0008) +#define TSDB_CODE_RPC_TOO_SLOW TAOS_DEF_ERROR_CODE(0, 0x0009) +#define TSDB_CODE_RPC_MAX_SESSIONS TAOS_DEF_ERROR_CODE(0, 0x000A) +#define TSDB_CODE_RPC_NETWORK_UNAVAIL TAOS_DEF_ERROR_CODE(0, 0x000B) +#define TSDB_CODE_RPC_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x000C) +#define TSDB_CODE_RPC_UNEXPECTED_RESPONSE TAOS_DEF_ERROR_CODE(0, 0x000D) +#define TSDB_CODE_RPC_INVALID_VALUE TAOS_DEF_ERROR_CODE(0, 0x000E) +#define TSDB_CODE_RPC_INVALID_TRAN_ID TAOS_DEF_ERROR_CODE(0, 0x000F) +#define TSDB_CODE_RPC_INVALID_SESSION_ID TAOS_DEF_ERROR_CODE(0, 0x0010) +#define TSDB_CODE_RPC_INVALID_MSG_TYPE TAOS_DEF_ERROR_CODE(0, 0x0011) +#define TSDB_CODE_RPC_INVALID_RESPONSE_TYPE TAOS_DEF_ERROR_CODE(0, 0x0012) +#define TSDB_CODE_RPC_INVALID_TIME_STAMP TAOS_DEF_ERROR_CODE(0, 0x0013) +#define TSDB_CODE_APP_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x0014) +#define TSDB_CODE_RPC_FQDN_ERROR TAOS_DEF_ERROR_CODE(0, 0x0015) +#define TSDB_CODE_RPC_INVALID_VERSION TAOS_DEF_ERROR_CODE(0, 0x0016) //common & util #define TSDB_CODE_OPS_NOT_SUPPORT TAOS_DEF_ERROR_CODE(0, 0x0100) @@ -71,56 +73,56 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MSG_NOT_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0109) #define TSDB_CODE_INVALID_PARA TAOS_DEF_ERROR_CODE(0, 0x010A) #define TSDB_CODE_REPEAT_INIT TAOS_DEF_ERROR_CODE(0, 0x010B) - +#define TSDB_CODE_CFG_NOT_FOUND TAOS_DEF_ERROR_CODE(0, 0x010C) +#define TSDB_CODE_INVALID_CFG TAOS_DEF_ERROR_CODE(0, 0x010D) #define TSDB_CODE_REF_NO_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0110) #define TSDB_CODE_REF_FULL TAOS_DEF_ERROR_CODE(0, 0x0111) #define TSDB_CODE_REF_ID_REMOVED TAOS_DEF_ERROR_CODE(0, 0x0112) #define TSDB_CODE_REF_INVALID_ID TAOS_DEF_ERROR_CODE(0, 0x0113) #define TSDB_CODE_REF_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0114) #define TSDB_CODE_REF_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0115) - #define TSDB_CODE_INVALID_VERSION_NUMBER TAOS_DEF_ERROR_CODE(0, 0x0120) #define TSDB_CODE_INVALID_VERSION_STRING TAOS_DEF_ERROR_CODE(0, 0x0121) #define TSDB_CODE_VERSION_NOT_COMPATIBLE TAOS_DEF_ERROR_CODE(0, 0x0122) //client -#define TSDB_CODE_TSC_INVALID_OPERATION TAOS_DEF_ERROR_CODE(0, 0x0200) //"Invalid Operation") -#define TSDB_CODE_TSC_INVALID_QHANDLE TAOS_DEF_ERROR_CODE(0, 0x0201) //"Invalid qhandle") -#define TSDB_CODE_TSC_INVALID_TIME_STAMP TAOS_DEF_ERROR_CODE(0, 0x0202) //"Invalid combination of client/service time") -#define TSDB_CODE_TSC_INVALID_VALUE TAOS_DEF_ERROR_CODE(0, 0x0203) //"Invalid value in client") -#define TSDB_CODE_TSC_INVALID_VERSION TAOS_DEF_ERROR_CODE(0, 0x0204) //"Invalid client version") -#define TSDB_CODE_TSC_INVALID_IE TAOS_DEF_ERROR_CODE(0, 0x0205) //"Invalid client ie") -#define TSDB_CODE_TSC_INVALID_FQDN TAOS_DEF_ERROR_CODE(0, 0x0206) //"Invalid host name") -#define TSDB_CODE_TSC_INVALID_USER_LENGTH TAOS_DEF_ERROR_CODE(0, 0x0207) //"Invalid user name") -#define TSDB_CODE_TSC_INVALID_PASS_LENGTH TAOS_DEF_ERROR_CODE(0, 0x0208) //"Invalid password") -#define TSDB_CODE_TSC_INVALID_DB_LENGTH TAOS_DEF_ERROR_CODE(0, 0x0209) //"Database name too long") -#define TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH TAOS_DEF_ERROR_CODE(0, 0x020A) //"Table name too long") -#define TSDB_CODE_TSC_INVALID_CONNECTION TAOS_DEF_ERROR_CODE(0, 0x020B) //"Invalid connection") -#define TSDB_CODE_TSC_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x020C) //"System out of memory") -#define TSDB_CODE_TSC_NO_DISKSPACE TAOS_DEF_ERROR_CODE(0, 0x020D) //"System out of disk space") -#define TSDB_CODE_TSC_QUERY_CACHE_ERASED TAOS_DEF_ERROR_CODE(0, 0x020E) //"Query cache erased") -#define TSDB_CODE_TSC_QUERY_CANCELLED TAOS_DEF_ERROR_CODE(0, 0x020F) //"Query terminated") -#define TSDB_CODE_TSC_SORTED_RES_TOO_MANY TAOS_DEF_ERROR_CODE(0, 0x0210) //"Result set too large to be sorted") // too many result for ordered super table projection query -#define TSDB_CODE_TSC_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x0211) //"Application error") -#define TSDB_CODE_TSC_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0212) //"Action in progress") -#define TSDB_CODE_TSC_DISCONNECTED TAOS_DEF_ERROR_CODE(0, 0x0213) //"Disconnected from service") -#define TSDB_CODE_TSC_NO_WRITE_AUTH TAOS_DEF_ERROR_CODE(0, 0x0214) //"No write permission") -#define TSDB_CODE_TSC_CONN_KILLED TAOS_DEF_ERROR_CODE(0, 0x0215) //"Connection killed") -#define TSDB_CODE_TSC_SQL_SYNTAX_ERROR TAOS_DEF_ERROR_CODE(0, 0x0216) //"Syntax error in SQL") -#define TSDB_CODE_TSC_DB_NOT_SELECTED TAOS_DEF_ERROR_CODE(0, 0x0217) //"Database not specified or available") -#define TSDB_CODE_TSC_INVALID_TABLE_NAME TAOS_DEF_ERROR_CODE(0, 0x0218) //"Table does not exist") -#define TSDB_CODE_TSC_EXCEED_SQL_LIMIT TAOS_DEF_ERROR_CODE(0, 0x0219) //"SQL statement too long check maxSQLLength config") -#define TSDB_CODE_TSC_FILE_EMPTY TAOS_DEF_ERROR_CODE(0, 0x021A) //"File is empty") -#define TSDB_CODE_TSC_LINE_SYNTAX_ERROR TAOS_DEF_ERROR_CODE(0, 0x021B) //"Syntax error in Line") -#define TSDB_CODE_TSC_NO_META_CACHED TAOS_DEF_ERROR_CODE(0, 0x021C) //"No table meta cached") -#define TSDB_CODE_TSC_DUP_COL_NAMES TAOS_DEF_ERROR_CODE(0, 0x021D) //"duplicated column names") -#define TSDB_CODE_TSC_INVALID_TAG_LENGTH TAOS_DEF_ERROR_CODE(0, 0x021E) //"Invalid tag length") -#define TSDB_CODE_TSC_INVALID_COLUMN_LENGTH TAOS_DEF_ERROR_CODE(0, 0x021F) //"Invalid column length") -#define TSDB_CODE_TSC_DUP_TAG_NAMES TAOS_DEF_ERROR_CODE(0, 0x0220) //"duplicated tag names") -#define TSDB_CODE_TSC_INVALID_JSON TAOS_DEF_ERROR_CODE(0, 0x0221) //"Invalid JSON format") -#define TSDB_CODE_TSC_INVALID_JSON_TYPE TAOS_DEF_ERROR_CODE(0, 0x0222) //"Invalid JSON data type") -#define TSDB_CODE_TSC_VALUE_OUT_OF_RANGE TAOS_DEF_ERROR_CODE(0, 0x0223) //"Value out of range") -#define TSDB_CODE_TSC_INVALID_INPUT TAOS_DEF_ERROR_CODE(0, 0X0224) //"Invalid tsc input") +#define TSDB_CODE_TSC_INVALID_OPERATION TAOS_DEF_ERROR_CODE(0, 0x0200) +#define TSDB_CODE_TSC_INVALID_QHANDLE TAOS_DEF_ERROR_CODE(0, 0x0201) +#define TSDB_CODE_TSC_INVALID_TIME_STAMP TAOS_DEF_ERROR_CODE(0, 0x0202) +#define TSDB_CODE_TSC_INVALID_VALUE TAOS_DEF_ERROR_CODE(0, 0x0203) +#define TSDB_CODE_TSC_INVALID_VERSION TAOS_DEF_ERROR_CODE(0, 0x0204) +#define TSDB_CODE_TSC_INVALID_IE TAOS_DEF_ERROR_CODE(0, 0x0205) +#define TSDB_CODE_TSC_INVALID_FQDN TAOS_DEF_ERROR_CODE(0, 0x0206) +#define TSDB_CODE_TSC_INVALID_USER_LENGTH TAOS_DEF_ERROR_CODE(0, 0x0207) +#define TSDB_CODE_TSC_INVALID_PASS_LENGTH TAOS_DEF_ERROR_CODE(0, 0x0208) +#define TSDB_CODE_TSC_INVALID_DB_LENGTH TAOS_DEF_ERROR_CODE(0, 0x0209) +#define TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH TAOS_DEF_ERROR_CODE(0, 0x020A) +#define TSDB_CODE_TSC_INVALID_CONNECTION TAOS_DEF_ERROR_CODE(0, 0x020B) +#define TSDB_CODE_TSC_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x020C) +#define TSDB_CODE_TSC_NO_DISKSPACE TAOS_DEF_ERROR_CODE(0, 0x020D) +#define TSDB_CODE_TSC_QUERY_CACHE_ERASED TAOS_DEF_ERROR_CODE(0, 0x020E) +#define TSDB_CODE_TSC_QUERY_CANCELLED TAOS_DEF_ERROR_CODE(0, 0x020F) +#define TSDB_CODE_TSC_SORTED_RES_TOO_MANY TAOS_DEF_ERROR_CODE(0, 0x0210) +#define TSDB_CODE_TSC_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x0211) +#define TSDB_CODE_TSC_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0212) +#define TSDB_CODE_TSC_DISCONNECTED TAOS_DEF_ERROR_CODE(0, 0x0213) +#define TSDB_CODE_TSC_NO_WRITE_AUTH TAOS_DEF_ERROR_CODE(0, 0x0214) +#define TSDB_CODE_TSC_CONN_KILLED TAOS_DEF_ERROR_CODE(0, 0x0215) +#define TSDB_CODE_TSC_SQL_SYNTAX_ERROR TAOS_DEF_ERROR_CODE(0, 0x0216) +#define TSDB_CODE_TSC_DB_NOT_SELECTED TAOS_DEF_ERROR_CODE(0, 0x0217) +#define TSDB_CODE_TSC_INVALID_TABLE_NAME TAOS_DEF_ERROR_CODE(0, 0x0218) +#define TSDB_CODE_TSC_EXCEED_SQL_LIMIT TAOS_DEF_ERROR_CODE(0, 0x0219) +#define TSDB_CODE_TSC_FILE_EMPTY TAOS_DEF_ERROR_CODE(0, 0x021A) +#define TSDB_CODE_TSC_LINE_SYNTAX_ERROR TAOS_DEF_ERROR_CODE(0, 0x021B) +#define TSDB_CODE_TSC_NO_META_CACHED TAOS_DEF_ERROR_CODE(0, 0x021C) +#define TSDB_CODE_TSC_DUP_COL_NAMES TAOS_DEF_ERROR_CODE(0, 0x021D) +#define TSDB_CODE_TSC_INVALID_TAG_LENGTH TAOS_DEF_ERROR_CODE(0, 0x021E) +#define TSDB_CODE_TSC_INVALID_COLUMN_LENGTH TAOS_DEF_ERROR_CODE(0, 0x021F) +#define TSDB_CODE_TSC_DUP_TAG_NAMES TAOS_DEF_ERROR_CODE(0, 0x0220) +#define TSDB_CODE_TSC_INVALID_JSON TAOS_DEF_ERROR_CODE(0, 0x0221) +#define TSDB_CODE_TSC_INVALID_JSON_TYPE TAOS_DEF_ERROR_CODE(0, 0x0222) +#define TSDB_CODE_TSC_VALUE_OUT_OF_RANGE TAOS_DEF_ERROR_CODE(0, 0x0223) +#define TSDB_CODE_TSC_INVALID_INPUT TAOS_DEF_ERROR_CODE(0, 0X0224) // mnode-common #define TSDB_CODE_MND_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x0300) @@ -260,6 +262,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_CONSUMER_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03E7) #define TSDB_CODE_MND_UNSUPPORTED_TOPIC TAOS_DEF_ERROR_CODE(0, 0x03E8) #define TSDB_CODE_MND_SUBSCRIBE_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03E9) +#define TSDB_CODE_MND_OFFSET_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03EA) #define TSDB_CODE_MND_MQ_PLACEHOLDER TAOS_DEF_ERROR_CODE(0, 0x03F0) // dnode @@ -296,174 +299,174 @@ int32_t* taosGetErrno(); #define TSDB_CODE_DND_VNODE_TOO_MANY_VNODES TAOS_DEF_ERROR_CODE(0, 0x0465) // vnode -#define TSDB_CODE_VND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0500) //"Action in progress") -#define TSDB_CODE_VND_MSG_NOT_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0501) //"Message not processed") -#define TSDB_CODE_VND_ACTION_NEED_REPROCESSED TAOS_DEF_ERROR_CODE(0, 0x0502) //"Action need to be reprocessed") -#define TSDB_CODE_VND_INVALID_VGROUP_ID TAOS_DEF_ERROR_CODE(0, 0x0503) //"Invalid Vgroup ID") -#define TSDB_CODE_VND_INIT_FAILED TAOS_DEF_ERROR_CODE(0, 0x0504) //"Vnode initialization failed") -#define TSDB_CODE_VND_NO_DISKSPACE TAOS_DEF_ERROR_CODE(0, 0x0505) //"System out of disk space") -#define TSDB_CODE_VND_NO_DISK_PERMISSIONS TAOS_DEF_ERROR_CODE(0, 0x0506) //"No write permission for disk files") -#define TSDB_CODE_VND_NO_SUCH_FILE_OR_DIR TAOS_DEF_ERROR_CODE(0, 0x0507) //"Missing data file") -#define TSDB_CODE_VND_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0508) //"Out of memory") -#define TSDB_CODE_VND_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x0509) //"Unexpected generic error in vnode") -#define TSDB_CODE_VND_INVALID_CFG_FILE TAOS_DEF_ERROR_CODE(0, 0x050A) //"Invalid config file) -#define TSDB_CODE_VND_INVALID_TERM_FILE TAOS_DEF_ERROR_CODE(0, 0x050B) //"Invalid term file") -#define TSDB_CODE_VND_IS_FLOWCTRL TAOS_DEF_ERROR_CODE(0, 0x050C) //"Database memory is full") -#define TSDB_CODE_VND_IS_DROPPING TAOS_DEF_ERROR_CODE(0, 0x050D) //"Database is dropping") -#define TSDB_CODE_VND_IS_UPDATING TAOS_DEF_ERROR_CODE(0, 0x050E) //"Database is updating") -#define TSDB_CODE_VND_IS_CLOSING TAOS_DEF_ERROR_CODE(0, 0x0510) //"Database is closing") -#define TSDB_CODE_VND_NOT_SYNCED TAOS_DEF_ERROR_CODE(0, 0x0511) //"Database suspended") -#define TSDB_CODE_VND_NO_WRITE_AUTH TAOS_DEF_ERROR_CODE(0, 0x0512) //"Database write operation denied") -#define TSDB_CODE_VND_IS_SYNCING TAOS_DEF_ERROR_CODE(0, 0x0513) //"Database is syncing") -#define TSDB_CODE_VND_INVALID_TSDB_STATE TAOS_DEF_ERROR_CODE(0, 0x0514) //"Invalid tsdb state") -#define TSDB_CODE_VND_TB_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0515) // "Table not exists") +#define TSDB_CODE_VND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0500) +#define TSDB_CODE_VND_MSG_NOT_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0501) +#define TSDB_CODE_VND_ACTION_NEED_REPROCESSED TAOS_DEF_ERROR_CODE(0, 0x0502) +#define TSDB_CODE_VND_INVALID_VGROUP_ID TAOS_DEF_ERROR_CODE(0, 0x0503) +#define TSDB_CODE_VND_INIT_FAILED TAOS_DEF_ERROR_CODE(0, 0x0504) +#define TSDB_CODE_VND_NO_DISKSPACE TAOS_DEF_ERROR_CODE(0, 0x0505) +#define TSDB_CODE_VND_NO_DISK_PERMISSIONS TAOS_DEF_ERROR_CODE(0, 0x0506) +#define TSDB_CODE_VND_NO_SUCH_FILE_OR_DIR TAOS_DEF_ERROR_CODE(0, 0x0507) +#define TSDB_CODE_VND_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0508) +#define TSDB_CODE_VND_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x0509) +#define TSDB_CODE_VND_INVALID_CFG_FILE TAOS_DEF_ERROR_CODE(0, 0x050A) +#define TSDB_CODE_VND_INVALID_TERM_FILE TAOS_DEF_ERROR_CODE(0, 0x050B) +#define TSDB_CODE_VND_IS_FLOWCTRL TAOS_DEF_ERROR_CODE(0, 0x050C) +#define TSDB_CODE_VND_IS_DROPPING TAOS_DEF_ERROR_CODE(0, 0x050D) +#define TSDB_CODE_VND_IS_UPDATING TAOS_DEF_ERROR_CODE(0, 0x050E) +#define TSDB_CODE_VND_IS_CLOSING TAOS_DEF_ERROR_CODE(0, 0x0510) +#define TSDB_CODE_VND_NOT_SYNCED TAOS_DEF_ERROR_CODE(0, 0x0511) +#define TSDB_CODE_VND_NO_WRITE_AUTH TAOS_DEF_ERROR_CODE(0, 0x0512) +#define TSDB_CODE_VND_IS_SYNCING TAOS_DEF_ERROR_CODE(0, 0x0513) +#define TSDB_CODE_VND_INVALID_TSDB_STATE TAOS_DEF_ERROR_CODE(0, 0x0514) +#define TSDB_CODE_VND_TB_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0515) // tsdb -#define TSDB_CODE_TDB_INVALID_TABLE_ID TAOS_DEF_ERROR_CODE(0, 0x0600) //"Invalid table ID") -#define TSDB_CODE_TDB_INVALID_TABLE_TYPE TAOS_DEF_ERROR_CODE(0, 0x0601) //"Invalid table type") -#define TSDB_CODE_TDB_IVD_TB_SCHEMA_VERSION TAOS_DEF_ERROR_CODE(0, 0x0602) //"Invalid table schema version") -#define TSDB_CODE_TDB_TABLE_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0603) //"Table already exists") -#define TSDB_CODE_TDB_INVALID_CONFIG TAOS_DEF_ERROR_CODE(0, 0x0604) //"Invalid configuration") -#define TSDB_CODE_TDB_INIT_FAILED TAOS_DEF_ERROR_CODE(0, 0x0605) //"Tsdb init failed") -#define TSDB_CODE_TDB_NO_DISKSPACE TAOS_DEF_ERROR_CODE(0, 0x0606) //"No diskspace for tsdb") -#define TSDB_CODE_TDB_NO_DISK_PERMISSIONS TAOS_DEF_ERROR_CODE(0, 0x0607) //"No permission for disk files") -#define TSDB_CODE_TDB_FILE_CORRUPTED TAOS_DEF_ERROR_CODE(0, 0x0608) //"Data file(s) corrupted") -#define TSDB_CODE_TDB_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0609) //"Out of memory") -#define TSDB_CODE_TDB_TAG_VER_OUT_OF_DATE TAOS_DEF_ERROR_CODE(0, 0x060A) //"Tag too old") -#define TSDB_CODE_TDB_TIMESTAMP_OUT_OF_RANGE TAOS_DEF_ERROR_CODE(0, 0x060B) //"Timestamp data out of range") -#define TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP TAOS_DEF_ERROR_CODE(0, 0x060C) //"Submit message is messed up") -#define TSDB_CODE_TDB_INVALID_ACTION TAOS_DEF_ERROR_CODE(0, 0x060D) //"Invalid operation") -#define TSDB_CODE_TDB_INVALID_CREATE_TB_MSG TAOS_DEF_ERROR_CODE(0, 0x060E) //"Invalid creation of table") -#define TSDB_CODE_TDB_NO_TABLE_DATA_IN_MEM TAOS_DEF_ERROR_CODE(0, 0x060F) //"No table data in memory skiplist") -#define TSDB_CODE_TDB_FILE_ALREADY_EXISTS TAOS_DEF_ERROR_CODE(0, 0x0610) //"File already exists") -#define TSDB_CODE_TDB_TABLE_RECONFIGURE TAOS_DEF_ERROR_CODE(0, 0x0611) //"Need to reconfigure table") -#define TSDB_CODE_TDB_IVD_CREATE_TABLE_INFO TAOS_DEF_ERROR_CODE(0, 0x0612) //"Invalid information to create table") -#define TSDB_CODE_TDB_NO_AVAIL_DISK TAOS_DEF_ERROR_CODE(0, 0x0613) //"No available disk") -#define TSDB_CODE_TDB_MESSED_MSG TAOS_DEF_ERROR_CODE(0, 0x0614) //"TSDB messed message") -#define TSDB_CODE_TDB_IVLD_TAG_VAL TAOS_DEF_ERROR_CODE(0, 0x0615) //"TSDB invalid tag value") -#define TSDB_CODE_TDB_NO_CACHE_LAST_ROW TAOS_DEF_ERROR_CODE(0, 0x0616) //"TSDB no cache last row data") +#define TSDB_CODE_TDB_INVALID_TABLE_ID TAOS_DEF_ERROR_CODE(0, 0x0600) +#define TSDB_CODE_TDB_INVALID_TABLE_TYPE TAOS_DEF_ERROR_CODE(0, 0x0601) +#define TSDB_CODE_TDB_IVD_TB_SCHEMA_VERSION TAOS_DEF_ERROR_CODE(0, 0x0602) +#define TSDB_CODE_TDB_TABLE_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0603) +#define TSDB_CODE_TDB_INVALID_CONFIG TAOS_DEF_ERROR_CODE(0, 0x0604) +#define TSDB_CODE_TDB_INIT_FAILED TAOS_DEF_ERROR_CODE(0, 0x0605) +#define TSDB_CODE_TDB_NO_DISKSPACE TAOS_DEF_ERROR_CODE(0, 0x0606) +#define TSDB_CODE_TDB_NO_DISK_PERMISSIONS TAOS_DEF_ERROR_CODE(0, 0x0607) +#define TSDB_CODE_TDB_FILE_CORRUPTED TAOS_DEF_ERROR_CODE(0, 0x0608) +#define TSDB_CODE_TDB_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0609) +#define TSDB_CODE_TDB_TAG_VER_OUT_OF_DATE TAOS_DEF_ERROR_CODE(0, 0x060A) +#define TSDB_CODE_TDB_TIMESTAMP_OUT_OF_RANGE TAOS_DEF_ERROR_CODE(0, 0x060B) +#define TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP TAOS_DEF_ERROR_CODE(0, 0x060C) +#define TSDB_CODE_TDB_INVALID_ACTION TAOS_DEF_ERROR_CODE(0, 0x060D) +#define TSDB_CODE_TDB_INVALID_CREATE_TB_MSG TAOS_DEF_ERROR_CODE(0, 0x060E) +#define TSDB_CODE_TDB_NO_TABLE_DATA_IN_MEM TAOS_DEF_ERROR_CODE(0, 0x060F) +#define TSDB_CODE_TDB_FILE_ALREADY_EXISTS TAOS_DEF_ERROR_CODE(0, 0x0610) +#define TSDB_CODE_TDB_TABLE_RECONFIGURE TAOS_DEF_ERROR_CODE(0, 0x0611) +#define TSDB_CODE_TDB_IVD_CREATE_TABLE_INFO TAOS_DEF_ERROR_CODE(0, 0x0612) +#define TSDB_CODE_TDB_NO_AVAIL_DISK TAOS_DEF_ERROR_CODE(0, 0x0613) +#define TSDB_CODE_TDB_MESSED_MSG TAOS_DEF_ERROR_CODE(0, 0x0614) +#define TSDB_CODE_TDB_IVLD_TAG_VAL TAOS_DEF_ERROR_CODE(0, 0x0615) +#define TSDB_CODE_TDB_NO_CACHE_LAST_ROW TAOS_DEF_ERROR_CODE(0, 0x0616) // query -#define TSDB_CODE_QRY_INVALID_QHANDLE TAOS_DEF_ERROR_CODE(0, 0x0700) //"Invalid handle") -#define TSDB_CODE_QRY_INVALID_MSG TAOS_DEF_ERROR_CODE(0, 0x0701) //"Invalid message") // failed to validate the sql expression msg by vnode -#define TSDB_CODE_QRY_NO_DISKSPACE TAOS_DEF_ERROR_CODE(0, 0x0702) //"No diskspace for query") -#define TSDB_CODE_QRY_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0703) //"System out of memory") -#define TSDB_CODE_QRY_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x0704) //"Unexpected generic error in query") -#define TSDB_CODE_QRY_DUP_JOIN_KEY TAOS_DEF_ERROR_CODE(0, 0x0705) //"Duplicated join key") -#define TSDB_CODE_QRY_EXCEED_TAGS_LIMIT TAOS_DEF_ERROR_CODE(0, 0x0706) //"Tag conditon too many") -#define TSDB_CODE_QRY_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x0707) //"Query not ready") -#define TSDB_CODE_QRY_HAS_RSP TAOS_DEF_ERROR_CODE(0, 0x0708) //"Query should response") -#define TSDB_CODE_QRY_IN_EXEC TAOS_DEF_ERROR_CODE(0, 0x0709) //"Multiple retrieval of this query") -#define TSDB_CODE_QRY_TOO_MANY_TIMEWINDOW TAOS_DEF_ERROR_CODE(0, 0x070A) //"Too many time window in query") -#define TSDB_CODE_QRY_NOT_ENOUGH_BUFFER TAOS_DEF_ERROR_CODE(0, 0x070B) //"Query buffer limit has reached") -#define TSDB_CODE_QRY_INCONSISTAN TAOS_DEF_ERROR_CODE(0, 0x070C) //"File inconsistency in replica") -#define TSDB_CODE_QRY_INVALID_TIME_CONDITION TAOS_DEF_ERROR_CODE(0, 0x070D) //"invalid time condition") -#define TSDB_CODE_QRY_SYS_ERROR TAOS_DEF_ERROR_CODE(0, 0x070E) //"System error") -#define TSDB_CODE_QRY_INVALID_INPUT TAOS_DEF_ERROR_CODE(0, 0x070F) //"invalid input") -#define TSDB_CODE_QRY_SCH_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0710) //"Scheduler not exist") -#define TSDB_CODE_QRY_TASK_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0711) //"Task not exist") -#define TSDB_CODE_QRY_TASK_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0712) //"Task already exist") -#define TSDB_CODE_QRY_TASK_CTX_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0713) //"Task context not exist") -#define TSDB_CODE_QRY_TASK_CANCELLED TAOS_DEF_ERROR_CODE(0, 0x0714) //"Task cancelled") -#define TSDB_CODE_QRY_TASK_DROPPED TAOS_DEF_ERROR_CODE(0, 0x0715) //"Task dropped") -#define TSDB_CODE_QRY_TASK_CANCELLING TAOS_DEF_ERROR_CODE(0, 0x0716) //"Task cancelling") -#define TSDB_CODE_QRY_TASK_DROPPING TAOS_DEF_ERROR_CODE(0, 0x0717) //"Task dropping") -#define TSDB_CODE_QRY_DUPLICATTED_OPERATION TAOS_DEF_ERROR_CODE(0, 0x0718) //"Duplicatted operation") -#define TSDB_CODE_QRY_TASK_MSG_ERROR TAOS_DEF_ERROR_CODE(0, 0x0719) //"Task message error") -#define TSDB_CODE_QRY_JOB_FREED TAOS_DEF_ERROR_CODE(0, 0x071A) //"Job freed") -#define TSDB_CODE_QRY_TASK_STATUS_ERROR TAOS_DEF_ERROR_CODE(0, 0x071B) //"Task status error") +#define TSDB_CODE_QRY_INVALID_QHANDLE TAOS_DEF_ERROR_CODE(0, 0x0700) +#define TSDB_CODE_QRY_INVALID_MSG TAOS_DEF_ERROR_CODE(0, 0x0701) +#define TSDB_CODE_QRY_NO_DISKSPACE TAOS_DEF_ERROR_CODE(0, 0x0702) +#define TSDB_CODE_QRY_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0703) +#define TSDB_CODE_QRY_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x0704) +#define TSDB_CODE_QRY_DUP_JOIN_KEY TAOS_DEF_ERROR_CODE(0, 0x0705) +#define TSDB_CODE_QRY_EXCEED_TAGS_LIMIT TAOS_DEF_ERROR_CODE(0, 0x0706) +#define TSDB_CODE_QRY_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x0707) +#define TSDB_CODE_QRY_HAS_RSP TAOS_DEF_ERROR_CODE(0, 0x0708) +#define TSDB_CODE_QRY_IN_EXEC TAOS_DEF_ERROR_CODE(0, 0x0709) +#define TSDB_CODE_QRY_TOO_MANY_TIMEWINDOW TAOS_DEF_ERROR_CODE(0, 0x070A) +#define TSDB_CODE_QRY_NOT_ENOUGH_BUFFER TAOS_DEF_ERROR_CODE(0, 0x070B) +#define TSDB_CODE_QRY_INCONSISTAN TAOS_DEF_ERROR_CODE(0, 0x070C) +#define TSDB_CODE_QRY_INVALID_TIME_CONDITION TAOS_DEF_ERROR_CODE(0, 0x070D) +#define TSDB_CODE_QRY_SYS_ERROR TAOS_DEF_ERROR_CODE(0, 0x070E) +#define TSDB_CODE_QRY_INVALID_INPUT TAOS_DEF_ERROR_CODE(0, 0x070F) +#define TSDB_CODE_QRY_SCH_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0710) +#define TSDB_CODE_QRY_TASK_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0711) +#define TSDB_CODE_QRY_TASK_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0712) +#define TSDB_CODE_QRY_TASK_CTX_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0713) +#define TSDB_CODE_QRY_TASK_CANCELLED TAOS_DEF_ERROR_CODE(0, 0x0714) +#define TSDB_CODE_QRY_TASK_DROPPED TAOS_DEF_ERROR_CODE(0, 0x0715) +#define TSDB_CODE_QRY_TASK_CANCELLING TAOS_DEF_ERROR_CODE(0, 0x0716) +#define TSDB_CODE_QRY_TASK_DROPPING TAOS_DEF_ERROR_CODE(0, 0x0717) +#define TSDB_CODE_QRY_DUPLICATTED_OPERATION TAOS_DEF_ERROR_CODE(0, 0x0718) +#define TSDB_CODE_QRY_TASK_MSG_ERROR TAOS_DEF_ERROR_CODE(0, 0x0719) +#define TSDB_CODE_QRY_JOB_FREED TAOS_DEF_ERROR_CODE(0, 0x071A) +#define TSDB_CODE_QRY_TASK_STATUS_ERROR TAOS_DEF_ERROR_CODE(0, 0x071B) // grant -#define TSDB_CODE_GRANT_EXPIRED TAOS_DEF_ERROR_CODE(0, 0x0800) //"License expired") -#define TSDB_CODE_GRANT_DNODE_LIMITED TAOS_DEF_ERROR_CODE(0, 0x0801) //"DNode creation limited by licence") -#define TSDB_CODE_GRANT_ACCT_LIMITED TAOS_DEF_ERROR_CODE(0, 0x0802) //"Account creation limited by license") -#define TSDB_CODE_GRANT_TIMESERIES_LIMITED TAOS_DEF_ERROR_CODE(0, 0x0803) //"Table creation limited by license") -#define TSDB_CODE_GRANT_DB_LIMITED TAOS_DEF_ERROR_CODE(0, 0x0804) //"DB creation limited by license") -#define TSDB_CODE_GRANT_USER_LIMITED TAOS_DEF_ERROR_CODE(0, 0x0805) //"User creation limited by license") -#define TSDB_CODE_GRANT_CONN_LIMITED TAOS_DEF_ERROR_CODE(0, 0x0806) //"Conn creation limited by license") -#define TSDB_CODE_GRANT_STREAM_LIMITED TAOS_DEF_ERROR_CODE(0, 0x0807) //"Stream creation limited by license") -#define TSDB_CODE_GRANT_SPEED_LIMITED TAOS_DEF_ERROR_CODE(0, 0x0808) //"Write speed limited by license") -#define TSDB_CODE_GRANT_STORAGE_LIMITED TAOS_DEF_ERROR_CODE(0, 0x0809) //"Storage capacity limited by license") -#define TSDB_CODE_GRANT_QUERYTIME_LIMITED TAOS_DEF_ERROR_CODE(0, 0x080A) //"Query time limited by license") -#define TSDB_CODE_GRANT_CPU_LIMITED TAOS_DEF_ERROR_CODE(0, 0x080B) //"CPU cores limited by license") +#define TSDB_CODE_GRANT_EXPIRED TAOS_DEF_ERROR_CODE(0, 0x0800) +#define TSDB_CODE_GRANT_DNODE_LIMITED TAOS_DEF_ERROR_CODE(0, 0x0801) +#define TSDB_CODE_GRANT_ACCT_LIMITED TAOS_DEF_ERROR_CODE(0, 0x0802) +#define TSDB_CODE_GRANT_TIMESERIES_LIMITED TAOS_DEF_ERROR_CODE(0, 0x0803) +#define TSDB_CODE_GRANT_DB_LIMITED TAOS_DEF_ERROR_CODE(0, 0x0804) +#define TSDB_CODE_GRANT_USER_LIMITED TAOS_DEF_ERROR_CODE(0, 0x0805) +#define TSDB_CODE_GRANT_CONN_LIMITED TAOS_DEF_ERROR_CODE(0, 0x0806) +#define TSDB_CODE_GRANT_STREAM_LIMITED TAOS_DEF_ERROR_CODE(0, 0x0807) +#define TSDB_CODE_GRANT_SPEED_LIMITED TAOS_DEF_ERROR_CODE(0, 0x0808) +#define TSDB_CODE_GRANT_STORAGE_LIMITED TAOS_DEF_ERROR_CODE(0, 0x0809) +#define TSDB_CODE_GRANT_QUERYTIME_LIMITED TAOS_DEF_ERROR_CODE(0, 0x080A) +#define TSDB_CODE_GRANT_CPU_LIMITED TAOS_DEF_ERROR_CODE(0, 0x080B) // sync -#define TSDB_CODE_SYN_INVALID_CONFIG TAOS_DEF_ERROR_CODE(0, 0x0900) //"Invalid Sync Configuration") -#define TSDB_CODE_SYN_NOT_ENABLED TAOS_DEF_ERROR_CODE(0, 0x0901) //"Sync module not enabled") -#define TSDB_CODE_SYN_INVALID_VERSION TAOS_DEF_ERROR_CODE(0, 0x0902) //"Invalid Sync version") -#define TSDB_CODE_SYN_CONFIRM_EXPIRED TAOS_DEF_ERROR_CODE(0, 0x0903) //"Sync confirm expired") -#define TSDB_CODE_SYN_TOO_MANY_FWDINFO TAOS_DEF_ERROR_CODE(0, 0x0904) //"Too many sync fwd infos") -#define TSDB_CODE_SYN_MISMATCHED_PROTOCOL TAOS_DEF_ERROR_CODE(0, 0x0905) //"Mismatched protocol") -#define TSDB_CODE_SYN_MISMATCHED_CLUSTERID TAOS_DEF_ERROR_CODE(0, 0x0906) //"Mismatched clusterId") -#define TSDB_CODE_SYN_MISMATCHED_SIGNATURE TAOS_DEF_ERROR_CODE(0, 0x0907) //"Mismatched signature") -#define TSDB_CODE_SYN_INVALID_CHECKSUM TAOS_DEF_ERROR_CODE(0, 0x0908) //"Invalid msg checksum") -#define TSDB_CODE_SYN_INVALID_MSGLEN TAOS_DEF_ERROR_CODE(0, 0x0909) //"Invalid msg length") -#define TSDB_CODE_SYN_INVALID_MSGTYPE TAOS_DEF_ERROR_CODE(0, 0x090A) //"Invalid msg type") +#define TSDB_CODE_SYN_INVALID_CONFIG TAOS_DEF_ERROR_CODE(0, 0x0900) +#define TSDB_CODE_SYN_NOT_ENABLED TAOS_DEF_ERROR_CODE(0, 0x0901) +#define TSDB_CODE_SYN_INVALID_VERSION TAOS_DEF_ERROR_CODE(0, 0x0902) +#define TSDB_CODE_SYN_CONFIRM_EXPIRED TAOS_DEF_ERROR_CODE(0, 0x0903) +#define TSDB_CODE_SYN_TOO_MANY_FWDINFO TAOS_DEF_ERROR_CODE(0, 0x0904) +#define TSDB_CODE_SYN_MISMATCHED_PROTOCOL TAOS_DEF_ERROR_CODE(0, 0x0905) +#define TSDB_CODE_SYN_MISMATCHED_CLUSTERID TAOS_DEF_ERROR_CODE(0, 0x0906) +#define TSDB_CODE_SYN_MISMATCHED_SIGNATURE TAOS_DEF_ERROR_CODE(0, 0x0907) +#define TSDB_CODE_SYN_INVALID_CHECKSUM TAOS_DEF_ERROR_CODE(0, 0x0908) +#define TSDB_CODE_SYN_INVALID_MSGLEN TAOS_DEF_ERROR_CODE(0, 0x0909) +#define TSDB_CODE_SYN_INVALID_MSGTYPE TAOS_DEF_ERROR_CODE(0, 0x090A) // tq -#define TSDB_CODE_TQ_INVALID_CONFIG TAOS_DEF_ERROR_CODE(0, 0x0A00) //"Invalid configuration") -#define TSDB_CODE_TQ_INIT_FAILED TAOS_DEF_ERROR_CODE(0, 0x0A01) //"Tq init failed") -#define TSDB_CODE_TQ_NO_DISKSPACE TAOS_DEF_ERROR_CODE(0, 0x0A02) //"No diskspace for tq") -#define TSDB_CODE_TQ_NO_DISK_PERMISSIONS TAOS_DEF_ERROR_CODE(0, 0x0A03) //"No permission for disk files") -#define TSDB_CODE_TQ_FILE_CORRUPTED TAOS_DEF_ERROR_CODE(0, 0x0A04) //"Data file(s) corrupted") -#define TSDB_CODE_TQ_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0A05) //"Out of memory") -#define TSDB_CODE_TQ_FILE_ALREADY_EXISTS TAOS_DEF_ERROR_CODE(0, 0x0A06) //"File already exists") -#define TSDB_CODE_TQ_FAILED_TO_CREATE_DIR TAOS_DEF_ERROR_CODE(0, 0x0A07) //"Failed to create dir") -#define TSDB_CODE_TQ_META_NO_SUCH_KEY TAOS_DEF_ERROR_CODE(0, 0x0A08) //"Target key not found") -#define TSDB_CODE_TQ_META_KEY_NOT_IN_TXN TAOS_DEF_ERROR_CODE(0, 0x0A09) //"Target key not in transaction") -#define TSDB_CODE_TQ_META_KEY_DUP_IN_TXN TAOS_DEF_ERROR_CODE(0, 0x0A0A) //"Target key duplicated in transaction") -#define TSDB_CODE_TQ_GROUP_NOT_SET TAOS_DEF_ERROR_CODE(0, 0x0A0B) //"Group of corresponding client is not set by mnode") +#define TSDB_CODE_TQ_INVALID_CONFIG TAOS_DEF_ERROR_CODE(0, 0x0A00) +#define TSDB_CODE_TQ_INIT_FAILED TAOS_DEF_ERROR_CODE(0, 0x0A01) +#define TSDB_CODE_TQ_NO_DISKSPACE TAOS_DEF_ERROR_CODE(0, 0x0A02) +#define TSDB_CODE_TQ_NO_DISK_PERMISSIONS TAOS_DEF_ERROR_CODE(0, 0x0A03) +#define TSDB_CODE_TQ_FILE_CORRUPTED TAOS_DEF_ERROR_CODE(0, 0x0A04) +#define TSDB_CODE_TQ_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0A05) +#define TSDB_CODE_TQ_FILE_ALREADY_EXISTS TAOS_DEF_ERROR_CODE(0, 0x0A06) +#define TSDB_CODE_TQ_FAILED_TO_CREATE_DIR TAOS_DEF_ERROR_CODE(0, 0x0A07) +#define TSDB_CODE_TQ_META_NO_SUCH_KEY TAOS_DEF_ERROR_CODE(0, 0x0A08) +#define TSDB_CODE_TQ_META_KEY_NOT_IN_TXN TAOS_DEF_ERROR_CODE(0, 0x0A09) +#define TSDB_CODE_TQ_META_KEY_DUP_IN_TXN TAOS_DEF_ERROR_CODE(0, 0x0A0A) +#define TSDB_CODE_TQ_GROUP_NOT_SET TAOS_DEF_ERROR_CODE(0, 0x0A0B) // wal -#define TSDB_CODE_WAL_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x1000) //"Unexpected generic error in wal") -#define TSDB_CODE_WAL_FILE_CORRUPTED TAOS_DEF_ERROR_CODE(0, 0x1001) //"WAL file is corrupted") -#define TSDB_CODE_WAL_SIZE_LIMIT TAOS_DEF_ERROR_CODE(0, 0x1002) //"WAL size exceeds limit") -#define TSDB_CODE_WAL_INVALID_VER TAOS_DEF_ERROR_CODE(0, 0x1003) //"WAL invalid version") -#define TSDB_CODE_WAL_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x1004) //"WAL out of memory") +#define TSDB_CODE_WAL_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x1000) +#define TSDB_CODE_WAL_FILE_CORRUPTED TAOS_DEF_ERROR_CODE(0, 0x1001) +#define TSDB_CODE_WAL_SIZE_LIMIT TAOS_DEF_ERROR_CODE(0, 0x1002) +#define TSDB_CODE_WAL_INVALID_VER TAOS_DEF_ERROR_CODE(0, 0x1003) +#define TSDB_CODE_WAL_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x1004) // tfs -#define TSDB_CODE_FS_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x2200) //"tfs out of memory") -#define TSDB_CODE_FS_INVLD_CFG TAOS_DEF_ERROR_CODE(0, 0x2201) //"tfs invalid mount config") -#define TSDB_CODE_FS_TOO_MANY_MOUNT TAOS_DEF_ERROR_CODE(0, 0x2202) //"tfs too many mount") -#define TSDB_CODE_FS_DUP_PRIMARY TAOS_DEF_ERROR_CODE(0, 0x2203) //"tfs duplicate primary mount") -#define TSDB_CODE_FS_NO_PRIMARY_DISK TAOS_DEF_ERROR_CODE(0, 0x2204) //"tfs no primary mount") -#define TSDB_CODE_FS_NO_MOUNT_AT_TIER TAOS_DEF_ERROR_CODE(0, 0x2205) //"tfs no mount at tier") -#define TSDB_CODE_FS_FILE_ALREADY_EXISTS TAOS_DEF_ERROR_CODE(0, 0x2206) //"tfs file already exists") -#define TSDB_CODE_FS_INVLD_LEVEL TAOS_DEF_ERROR_CODE(0, 0x2207) //"tfs invalid level") -#define TSDB_CODE_FS_NO_VALID_DISK TAOS_DEF_ERROR_CODE(0, 0x2208) //"tfs no valid disk") +#define TSDB_CODE_FS_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x2200) +#define TSDB_CODE_FS_INVLD_CFG TAOS_DEF_ERROR_CODE(0, 0x2201) +#define TSDB_CODE_FS_TOO_MANY_MOUNT TAOS_DEF_ERROR_CODE(0, 0x2202) +#define TSDB_CODE_FS_DUP_PRIMARY TAOS_DEF_ERROR_CODE(0, 0x2203) +#define TSDB_CODE_FS_NO_PRIMARY_DISK TAOS_DEF_ERROR_CODE(0, 0x2204) +#define TSDB_CODE_FS_NO_MOUNT_AT_TIER TAOS_DEF_ERROR_CODE(0, 0x2205) +#define TSDB_CODE_FS_FILE_ALREADY_EXISTS TAOS_DEF_ERROR_CODE(0, 0x2206) +#define TSDB_CODE_FS_INVLD_LEVEL TAOS_DEF_ERROR_CODE(0, 0x2207) +#define TSDB_CODE_FS_NO_VALID_DISK TAOS_DEF_ERROR_CODE(0, 0x2208) // monitor -#define TSDB_CODE_MON_CONNECTION_INVALID TAOS_DEF_ERROR_CODE(0, 0x2300) //"monitor invalid monitor db connection") +#define TSDB_CODE_MON_CONNECTION_INVALID TAOS_DEF_ERROR_CODE(0, 0x2300) // catalog -#define TSDB_CODE_CTG_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x2400) //catalog interval error -#define TSDB_CODE_CTG_INVALID_INPUT TAOS_DEF_ERROR_CODE(0, 0x2401) //invalid catalog input parameters -#define TSDB_CODE_CTG_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x2402) //catalog is not ready -#define TSDB_CODE_CTG_MEM_ERROR TAOS_DEF_ERROR_CODE(0, 0x2403) //catalog memory error -#define TSDB_CODE_CTG_SYS_ERROR TAOS_DEF_ERROR_CODE(0, 0x2404) //catalog system error -#define TSDB_CODE_CTG_DB_DROPPED TAOS_DEF_ERROR_CODE(0, 0x2405) //Database is dropped -#define TSDB_CODE_CTG_OUT_OF_SERVICE TAOS_DEF_ERROR_CODE(0, 0x2406) //catalog is out of service +#define TSDB_CODE_CTG_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x2400) +#define TSDB_CODE_CTG_INVALID_INPUT TAOS_DEF_ERROR_CODE(0, 0x2401) +#define TSDB_CODE_CTG_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x2402) +#define TSDB_CODE_CTG_MEM_ERROR TAOS_DEF_ERROR_CODE(0, 0x2403) +#define TSDB_CODE_CTG_SYS_ERROR TAOS_DEF_ERROR_CODE(0, 0x2404) +#define TSDB_CODE_CTG_DB_DROPPED TAOS_DEF_ERROR_CODE(0, 0x2405) +#define TSDB_CODE_CTG_OUT_OF_SERVICE TAOS_DEF_ERROR_CODE(0, 0x2406) //scheduler -#define TSDB_CODE_SCH_STATUS_ERROR TAOS_DEF_ERROR_CODE(0, 0x2501) //scheduler status error -#define TSDB_CODE_SCH_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x2502) //scheduler internal error +#define TSDB_CODE_SCH_STATUS_ERROR TAOS_DEF_ERROR_CODE(0, 0x2501) +#define TSDB_CODE_SCH_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x2502) //parser -#define TSDB_CODE_PAR_INVALID_COLUMN TAOS_DEF_ERROR_CODE(0, 0x2601) //Invalid column name -#define TSDB_CODE_PAR_TABLE_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x2602) //Table does not exist -#define TSDB_CODE_PAR_AMBIGUOUS_COLUMN TAOS_DEF_ERROR_CODE(0, 0x2603) //Column ambiguously defined -#define TSDB_CODE_PAR_WRONG_VALUE_TYPE TAOS_DEF_ERROR_CODE(0, 0x2604) //Invalid value type -#define TSDB_CODE_PAR_INVALID_FUNTION TAOS_DEF_ERROR_CODE(0, 0x2605) //Invalid function name -#define TSDB_CODE_PAR_FUNTION_PARA_NUM TAOS_DEF_ERROR_CODE(0, 0x2606) //Invalid number of arguments -#define TSDB_CODE_PAR_FUNTION_PARA_TYPE TAOS_DEF_ERROR_CODE(0, 0x2607) //Inconsistent datatypes -#define TSDB_CODE_PAR_ILLEGAL_USE_AGG_FUNCTION TAOS_DEF_ERROR_CODE(0, 0x2608) //There mustn't be aggregation -#define TSDB_CODE_PAR_WRONG_NUMBER_OF_SELECT TAOS_DEF_ERROR_CODE(0, 0x2609) //ORDER BY item must be the number of a SELECT-list expression -#define TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION TAOS_DEF_ERROR_CODE(0, 0x260A) //Not a GROUP BY expression -#define TSDB_CODE_PAR_NOT_SELECTED_EXPRESSION TAOS_DEF_ERROR_CODE(0, 0x260B) //Not SELECTed expression -#define TSDB_CODE_PAR_NOT_SINGLE_GROUP TAOS_DEF_ERROR_CODE(0, 0x260C) //Not a single-group group function +#define TSDB_CODE_PAR_INVALID_COLUMN TAOS_DEF_ERROR_CODE(0, 0x2601) +#define TSDB_CODE_PAR_TABLE_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x2602) +#define TSDB_CODE_PAR_AMBIGUOUS_COLUMN TAOS_DEF_ERROR_CODE(0, 0x2603) +#define TSDB_CODE_PAR_WRONG_VALUE_TYPE TAOS_DEF_ERROR_CODE(0, 0x2604) +#define TSDB_CODE_PAR_INVALID_FUNTION TAOS_DEF_ERROR_CODE(0, 0x2605) +#define TSDB_CODE_PAR_FUNTION_PARA_NUM TAOS_DEF_ERROR_CODE(0, 0x2606) +#define TSDB_CODE_PAR_FUNTION_PARA_TYPE TAOS_DEF_ERROR_CODE(0, 0x2607) +#define TSDB_CODE_PAR_ILLEGAL_USE_AGG_FUNCTION TAOS_DEF_ERROR_CODE(0, 0x2608) +#define TSDB_CODE_PAR_WRONG_NUMBER_OF_SELECT TAOS_DEF_ERROR_CODE(0, 0x2609) +#define TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION TAOS_DEF_ERROR_CODE(0, 0x260A) +#define TSDB_CODE_PAR_NOT_SELECTED_EXPRESSION TAOS_DEF_ERROR_CODE(0, 0x260B) +#define TSDB_CODE_PAR_NOT_SINGLE_GROUP TAOS_DEF_ERROR_CODE(0, 0x260C) #ifdef __cplusplus } #endif -#endif /*_TD_COMMON_TAOS_ERROR_H_*/ +#endif /*_TD_UTIL_TAOS_ERROR_H_*/ diff --git a/include/common/tcfg.h b/include/util/tcfg.h similarity index 100% rename from include/common/tcfg.h rename to include/util/tcfg.h diff --git a/include/util/tconfig.h b/include/util/tconfig.h index 7dee01247b..11dd5414ec 100644 --- a/include/util/tconfig.h +++ b/include/util/tconfig.h @@ -1,3 +1,4 @@ + /* * Copyright (c) 2019 TAOS Data, Inc. * @@ -13,85 +14,98 @@ * along with this program. If not, see . */ -#ifndef _TD_UTIL_CONFIG_H -#define _TD_UTIL_CONFIG_H +#ifndef _TD_CONFIG_H_ +#define _TD_CONFIG_H_ + +#include "os.h" +#include "tarray.h" #ifdef __cplusplus extern "C" { #endif -#define TSDB_CFG_MAX_NUM 115 -#define TSDB_CFG_PRINT_LEN 23 -#define TSDB_CFG_OPTION_LEN 24 -#define TSDB_CFG_VALUE_LEN 41 +#define CFG_NAME_MAX_LEN 128 -#define TSDB_CFG_CTYPE_B_CONFIG 1U // can be configured from file -#define TSDB_CFG_CTYPE_B_SHOW 2U // can displayed by "show configs" commands -#define TSDB_CFG_CTYPE_B_LOG 4U // is a log type configuration -#define TSDB_CFG_CTYPE_B_CLIENT 8U // can be displayed in the client log -#define TSDB_CFG_CTYPE_B_OPTION 16U // can be configured by taos_options function -#define TSDB_CFG_CTYPE_B_NOT_PRINT 32U // such as password +typedef enum { + CFG_STYPE_DEFAULT, + CFG_STYPE_CFG_FILE, + CFG_STYPE_ENV_FILE, + CFG_STYPE_ENV_VAR, + CFG_STYPE_APOLLO_URL, + CFG_STYPE_ARG_LIST, +} ECfgSrcType; -#define MAX_FLOAT 100000 -#define MIN_FLOAT 0 +typedef enum { + CFG_DTYPE_NONE, + CFG_DTYPE_BOOL, + CFG_DTYPE_INT32, + CFG_DTYPE_INT64, + CFG_DTYPE_FLOAT, + CFG_DTYPE_STRING, + CFG_DTYPE_DIR, + CFG_DTYPE_LOCALE, + CFG_DTYPE_CHARSET, + CFG_DTYPE_TIMEZONE +} ECfgDataType; -enum { - TAOS_CFG_CSTATUS_NONE, // not configured - TAOS_CFG_CSTATUS_DEFAULT, // use system default value - TAOS_CFG_CSTATUS_FILE, // configured from file - TAOS_CFG_CSTATUS_OPTION, // configured by taos_options function - TAOS_CFG_CSTATUS_ARG, // configured by program argument -}; - -enum { - TAOS_CFG_VTYPE_INT8, - TAOS_CFG_VTYPE_INT16, - TAOS_CFG_VTYPE_INT32, - TAOS_CFG_VTYPE_UINT16, - TAOS_CFG_VTYPE_FLOAT, - TAOS_CFG_VTYPE_STRING, - TAOS_CFG_VTYPE_IPSTR, - TAOS_CFG_VTYPE_DIRECTORY, - TAOS_CFG_VTYPE_DATA_DIRCTORY, - TAOS_CFG_VTYPE_DOUBLE, -}; - -enum { - TAOS_CFG_UTYPE_NONE, - TAOS_CFG_UTYPE_PERCENT, - TAOS_CFG_UTYPE_GB, - TAOS_CFG_UTYPE_MB, - TAOS_CFG_UTYPE_BYTE, - TAOS_CFG_UTYPE_SECOND, - TAOS_CFG_UTYPE_MS -}; +typedef struct SConfigItem { + ECfgSrcType stype; + ECfgDataType dtype; + bool tsc; + char *name; + union { + bool bval; + float fval; + int32_t i32; + int64_t i64; + char *str; + }; + union { + int64_t imin; + double fmin; + }; + union { + int64_t imax; + double fmax; + }; + SArray *array; // SDiskCfg +} SConfigItem; typedef struct { - char * option; - void * ptr; - float minValue; - float maxValue; - int8_t cfgType; - int8_t cfgStatus; - int8_t unitType; - int8_t valType; - int32_t ptrLength; -} SGlobalCfg; + const char *name; + const char *value; +} SConfigPair; -extern SGlobalCfg tsGlobalConfig[]; -extern int32_t tsGlobalConfigNum; -extern char * tsCfgStatusStr[]; +typedef struct SConfig SConfig; -void taosReadGlobalLogCfg(); -int32_t taosReadCfgFromFile(); -void taosPrintCfg(); -void taosDumpGlobalCfg(); +SConfig *cfgInit(); +int32_t cfgLoad(SConfig *pCfg, ECfgSrcType cfgType, const char *sourceStr); +int32_t cfgLoadArray(SConfig *pCfg, SArray *pArgs); // SConfigPair +void cfgCleanup(SConfig *pCfg); -void taosAddConfigOption(SGlobalCfg cfg); -SGlobalCfg *taosGetConfigOption(const char *option); +int32_t cfgGetSize(SConfig *pCfg); +SConfigItem *cfgIterate(SConfig *pCfg, SConfigItem *pIter); +void cfgCancelIterate(SConfig *pCfg, SConfigItem *pIter); +SConfigItem *cfgGetItem(SConfig *pCfg, const char *name); +int32_t cfgSetItem(SConfig *pCfg, const char *name, const char *value, ECfgSrcType stype); + +int32_t cfgAddBool(SConfig *pCfg, const char *name, bool defaultVal, bool tsc); +int32_t cfgAddInt32(SConfig *pCfg, const char *name, int32_t defaultVal, int64_t minval, int64_t maxval, bool tsc); +int32_t cfgAddInt64(SConfig *pCfg, const char *name, int64_t defaultVal, int64_t minval, int64_t maxval, bool tsc); +int32_t cfgAddFloat(SConfig *pCfg, const char *name, float defaultVal, double minval, double maxval, bool tsc); +int32_t cfgAddString(SConfig *pCfg, const char *name, const char *defaultVal, bool tsc); +int32_t cfgAddDir(SConfig *pCfg, const char *name, const char *defaultVal, bool tsc); +int32_t cfgAddLocale(SConfig *pCfg, const char *name, const char *defaultVal); +int32_t cfgAddCharset(SConfig *pCfg, const char *name, const char *defaultVal); +int32_t cfgAddTimezone(SConfig *pCfg, const char *name, const char *defaultVal); + +const char *cfgStypeStr(ECfgSrcType type); +const char *cfgDtypeStr(ECfgDataType type); + +void cfgDumpCfg(SConfig *pCfg, bool tsc, bool dump); #ifdef __cplusplus } #endif -#endif /*_TD_UTIL_CONFIG_H*/ +#endif /*_TD_CONFIG_H_*/ diff --git a/include/util/tdef.h b/include/util/tdef.h index a4c333a4d2..d0f2b77f1f 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -13,6 +13,8 @@ * along with this program. If not, see . */ +// clang-format off + #ifndef _TD_UTIL_DEF_H #define _TD_UTIL_DEF_H @@ -108,35 +110,52 @@ do { \ (src) = (void *)((char *)src + sizeof(type));\ } while(0) +typedef enum EOperatorType { + // arithmetic operator + OP_TYPE_ADD = 1, + OP_TYPE_SUB, + OP_TYPE_MULTI, + OP_TYPE_DIV, + OP_TYPE_MOD, -// TODO: check if below is necessary -#define TSDB_RELATION_INVALID 0 -#define TSDB_RELATION_LESS 1 -#define TSDB_RELATION_GREATER 2 -#define TSDB_RELATION_EQUAL 3 -#define TSDB_RELATION_LESS_EQUAL 4 -#define TSDB_RELATION_GREATER_EQUAL 5 -#define TSDB_RELATION_NOT_EQUAL 6 -#define TSDB_RELATION_LIKE 7 -#define TSDB_RELATION_NOT_LIKE 8 -#define TSDB_RELATION_ISNULL 9 -#define TSDB_RELATION_NOTNULL 10 -#define TSDB_RELATION_IN 11 -#define TSDB_RELATION_NOT_IN 12 + // bit operator + OP_TYPE_BIT_AND, + OP_TYPE_BIT_OR, -#define TSDB_RELATION_AND 13 -#define TSDB_RELATION_OR 14 -#define TSDB_RELATION_NOT 15 + // comparison operator + OP_TYPE_GREATER_THAN, + OP_TYPE_GREATER_EQUAL, + OP_TYPE_LOWER_THAN, + OP_TYPE_LOWER_EQUAL, + OP_TYPE_EQUAL, + OP_TYPE_NOT_EQUAL, + OP_TYPE_IN, + OP_TYPE_NOT_IN, + OP_TYPE_LIKE, + OP_TYPE_NOT_LIKE, + OP_TYPE_MATCH, + OP_TYPE_NMATCH, + OP_TYPE_IS_NULL, + OP_TYPE_IS_NOT_NULL, + OP_TYPE_IS_TRUE, + OP_TYPE_IS_FALSE, + OP_TYPE_IS_UNKNOWN, + OP_TYPE_IS_NOT_TRUE, + OP_TYPE_IS_NOT_FALSE, + OP_TYPE_IS_NOT_UNKNOWN, -#define TSDB_RELATION_MATCH 16 -#define TSDB_RELATION_NMATCH 17 + // json operator + OP_TYPE_JSON_GET_VALUE, + OP_TYPE_JSON_CONTAINS +} EOperatorType; + + +typedef enum ELogicConditionType { + LOGIC_COND_TYPE_AND, + LOGIC_COND_TYPE_OR, + LOGIC_COND_TYPE_NOT, +} ELogicConditionType; -#define TSDB_BINARY_OP_ADD 4000 -#define TSDB_BINARY_OP_SUBTRACT 4001 -#define TSDB_BINARY_OP_MULTIPLY 4002 -#define TSDB_BINARY_OP_DIVIDE 4003 -#define TSDB_BINARY_OP_REMAINDER 4004 -#define TSDB_BINARY_OP_CONCAT 4005 #define FUNCTION_CEIL 4500 #define FUNCTION_FLOOR 4501 @@ -148,9 +167,6 @@ do { \ #define FUNCTION_LTRIM 4802 #define FUNCTION_RTRIM 4803 -#define IS_RELATION_OPTR(op) (((op) >= TSDB_RELATION_LESS) && ((op) < TSDB_RELATION_IN)) -#define IS_ARITHMETIC_OPTR(op) (((op) >= TSDB_BINARY_OP_ADD) && ((op) <= TSDB_BINARY_OP_REMAINDER)) - #define TSDB_NAME_DELIMITER_LEN 1 #define TSDB_UNI_LEN 24 @@ -182,6 +198,7 @@ do { \ #define TSDB_TOPIC_FNAME_LEN TSDB_TABLE_FNAME_LEN #define TSDB_CONSUMER_GROUP_LEN 192 #define TSDB_SUBSCRIBE_KEY_LEN (TSDB_CONSUMER_GROUP_LEN + TSDB_TOPIC_FNAME_LEN + 2) +#define TSDB_PARTITION_KEY_LEN (TSDB_CONSUMER_GROUP_LEN + TSDB_TOPIC_FNAME_LEN + 2) #define TSDB_COL_NAME_LEN 65 #define TSDB_MAX_SAVED_SQL_LEN TSDB_MAX_COLUMNS * 64 #define TSDB_MAX_SQL_LEN TSDB_PAYLOAD_SIZE @@ -335,9 +352,6 @@ do { \ #define TSDB_QUERY_TYPE_NON_TYPE 0x00u // none type #define TSDB_QUERY_TYPE_FREE_RESOURCE 0x01u // free qhandle at vnode -#define TSDB_QUERY_TYPE_NON_TYPE 0x00u // none type -#define TSDB_QUERY_TYPE_FREE_RESOURCE 0x01u // free qhandle at vnode - #define TSDB_META_COMPACT_RATIO 0 // disable tsdb meta compact by default diff --git a/include/util/tfile.h b/include/util/tfile.h index d3813051a4..59953de861 100644 --- a/include/util/tfile.h +++ b/include/util/tfile.h @@ -30,20 +30,20 @@ void tfCleanup(); // the same syntax as UNIX standard open/close/read/write // but FD is int64_t and will never be reused -int64_t tfOpenRead(const char *pathname); -int64_t tfOpenReadWrite(const char *pathname); -int64_t tfOpenCreateWrite(const char *pathname); -int64_t tfOpenCreateWriteAppend(const char *pathname); +// int64_t tfOpenRead(const char *pathname); +// int64_t tfOpenReadWrite(const char *pathname); +// int64_t tfOpenCreateWrite(const char *pathname); +// int64_t tfOpenCreateWriteAppend(const char *pathname); -int64_t tfClose(int64_t tfd); -int64_t tfWrite(int64_t tfd, void *buf, int64_t count); -int64_t tfRead(int64_t tfd, void *buf, int64_t count); -int64_t tfPread(int64_t tfd, void *buf, int64_t count, int64_t offset); -int32_t tfFsync(int64_t tfd); -bool tfValid(int64_t tfd); -int64_t tfLseek(int64_t tfd, int64_t offset, int32_t whence); -int32_t tfFtruncate(int64_t tfd, int64_t length); -void * tfMmapReadOnly(int64_t tfd, int64_t length); +// int64_t tfClose(int64_t tfd); +// int64_t tfWrite(int64_t tfd, void *buf, int64_t count); +// int64_t tfRead(int64_t tfd, void *buf, int64_t count); +// int64_t tfPread(int64_t tfd, void *buf, int64_t count, int64_t offset); +// int32_t tfFsync(int64_t tfd); +// bool tfValid(int64_t tfd); +// int64_t tfLseek(int64_t tfd, int64_t offset, int32_t whence); +// int32_t tfFtruncate(int64_t tfd, int64_t length); +// void * tfMmapReadOnly(int64_t tfd, int64_t length); #ifdef __cplusplus } #endif diff --git a/include/util/tlog.h b/include/util/tlog.h index 26a5417320..1c14cc445f 100644 --- a/include/util/tlog.h +++ b/include/util/tlog.h @@ -22,8 +22,7 @@ extern "C" { #endif -// log -extern int8_t tsAsyncLog; +extern bool tsAsyncLog; extern int32_t tsNumOfLogLines; extern int32_t tsLogKeepDays; extern int32_t dDebugFlag; @@ -32,9 +31,6 @@ extern int32_t mDebugFlag; extern int32_t cDebugFlag; extern int32_t jniDebugFlag; extern int32_t tmrDebugFlag; -extern int32_t httpDebugFlag; -extern int32_t mqttDebugFlag; -extern int32_t monDebugFlag; extern int32_t uDebugFlag; extern int32_t rpcDebugFlag; extern int32_t qDebugFlag; @@ -42,23 +38,23 @@ extern int32_t wDebugFlag; extern int32_t sDebugFlag; extern int32_t tsdbDebugFlag; extern int32_t tqDebugFlag; -extern int32_t cqDebugFlag; -extern int32_t debugFlag; - -#define DEBUG_FATAL 1U -#define DEBUG_ERROR DEBUG_FATAL -#define DEBUG_WARN 2U -#define DEBUG_INFO DEBUG_WARN -#define DEBUG_DEBUG 4U -#define DEBUG_TRACE 8U -#define DEBUG_DUMP 16U +extern int32_t fsDebugFlag; +#define DEBUG_FATAL 1U +#define DEBUG_ERROR DEBUG_FATAL +#define DEBUG_WARN 2U +#define DEBUG_INFO DEBUG_WARN +#define DEBUG_DEBUG 4U +#define DEBUG_TRACE 8U +#define DEBUG_DUMP 16U #define DEBUG_SCREEN 64U -#define DEBUG_FILE 128U +#define DEBUG_FILE 128U -int32_t taosInitLog(char *logName, int32_t numOfLogLines, int32_t maxFiles); +int32_t taosInitLog(const char *logName, int32_t maxFiles); void taosCloseLog(); void taosResetLog(); +void taosSetAllDebugFlag(int32_t flag); +void taosDumpData(uint8_t *msg, int32_t len); void taosPrintLog(const char *flags, int32_t dflag, const char *format, ...) #ifdef __GNUC__ @@ -72,7 +68,17 @@ void taosPrintLongString(const char *flags, int32_t dflag, const char *format, . #endif ; -void taosDumpData(unsigned char *msg, int32_t len); +extern int8_t tscEmbeddedInUtil; + +#define uFatal(...) { if (uDebugFlag & DEBUG_FATAL) { taosPrintLog("UTL FATAL", tscEmbeddedInUtil ? 255 : uDebugFlag, __VA_ARGS__); }} +#define uError(...) { if (uDebugFlag & DEBUG_ERROR) { taosPrintLog("UTL ERROR ", tscEmbeddedInUtil ? 255 : uDebugFlag, __VA_ARGS__); }} +#define uWarn(...) { if (uDebugFlag & DEBUG_WARN) { taosPrintLog("UTL WARN ", tscEmbeddedInUtil ? 255 : uDebugFlag, __VA_ARGS__); }} +#define uInfo(...) { if (uDebugFlag & DEBUG_INFO) { taosPrintLog("UTL ", tscEmbeddedInUtil ? 255 : uDebugFlag, __VA_ARGS__); }} +#define uDebug(...) { if (uDebugFlag & DEBUG_DEBUG) { taosPrintLog("UTL ", uDebugFlag, __VA_ARGS__); }} +#define uTrace(...) { if (uDebugFlag & DEBUG_TRACE) { taosPrintLog("UTL ", uDebugFlag, __VA_ARGS__); }} + +#define pError(...) { taosPrintLog("APP ERROR ", 255, __VA_ARGS__); } +#define pPrint(...) { taosPrintLog("APP ", 255, __VA_ARGS__); } #ifdef __cplusplus } diff --git a/include/util/tnote.h b/include/util/tnote.h deleted file mode 100644 index e613ec7e41..0000000000 --- a/include/util/tnote.h +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_UTIL_NOTE_H -#define _TD_UTIL_NOTE_H - -#ifdef __cplusplus -extern "C" { -#endif - -#define MAX_NOTE_LINE_SIZE 66000 -#define NOTE_FILE_NAME_LEN 300 - -typedef struct { - int32_t fileNum; - int32_t maxLines; - int32_t lines; - int32_t flag; - int32_t fd; - int32_t openInProgress; - char name[NOTE_FILE_NAME_LEN]; - pthread_mutex_t mutex; -} SNoteObj; - -extern SNoteObj tsHttpNote; -extern SNoteObj tsTscNote; -extern SNoteObj tsInfoNote; - -int32_t taosInitNotes(); -void taosNotePrint(SNoteObj* pNote, const char* const format, ...); -void taosNotePrintBuffer(SNoteObj* pNote, char* buffer, int32_t len); - -#define nPrintHttp(...) \ - if (tsHttpEnableRecordSql) { \ - taosNotePrint(&tsHttpNote, __VA_ARGS__); \ - } - -#define nPrintTsc(...) \ - if (tsTscEnableRecordSql) { \ - taosNotePrint(&tsTscNote, __VA_ARGS__); \ - } - -#define nInfo(buffer, len) \ - if (tscEmbeddedInUtil == 1) { \ - taosNotePrintBuffer(&tsInfoNote, buffer, len); \ - } - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_UTIL_NOTE_H*/ diff --git a/include/util/ttimer.h b/include/util/ttimer.h index 89ec6cd8d9..01d70c7d02 100644 --- a/include/util/ttimer.h +++ b/include/util/ttimer.h @@ -26,7 +26,6 @@ typedef void *tmr_h; typedef void (*TAOS_TMR_CALLBACK)(void *, void *); extern int taosTmrThreads; -extern uint32_t tsMaxTmrCtrl; #define MSECONDS_PER_TICK 5 diff --git a/include/util/ulog.h b/include/util/ulog.h deleted file mode 100644 index 89d9f89476..0000000000 --- a/include/util/ulog.h +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_UTIL_ULOG_H -#define _TD_UTIL_ULOG_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include "os.h" -#include "tlog.h" - -extern int32_t uDebugFlag; -extern int8_t tscEmbeddedInUtil; - -#define uFatal(...) { if (uDebugFlag & DEBUG_FATAL) { taosPrintLog("UTL FATAL", tscEmbeddedInUtil ? 255 : uDebugFlag, __VA_ARGS__); }} -#define uError(...) { if (uDebugFlag & DEBUG_ERROR) { taosPrintLog("UTL ERROR ", tscEmbeddedInUtil ? 255 : uDebugFlag, __VA_ARGS__); }} -#define uWarn(...) { if (uDebugFlag & DEBUG_WARN) { taosPrintLog("UTL WARN ", tscEmbeddedInUtil ? 255 : uDebugFlag, __VA_ARGS__); }} -#define uInfo(...) { if (uDebugFlag & DEBUG_INFO) { taosPrintLog("UTL ", tscEmbeddedInUtil ? 255 : uDebugFlag, __VA_ARGS__); }} -#define uDebug(...) { if (uDebugFlag & DEBUG_DEBUG) { taosPrintLog("UTL ", uDebugFlag, __VA_ARGS__); }} -#define uTrace(...) { if (uDebugFlag & DEBUG_TRACE) { taosPrintLog("UTL ", uDebugFlag, __VA_ARGS__); }} - -#define pError(...) { taosPrintLog("APP ERROR ", 255, __VA_ARGS__); } -#define pPrint(...) { taosPrintLog("APP ", 255, __VA_ARGS__); } - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_UTIL_ULOG_H*/ diff --git a/include/common/tlocale.h b/include/util/version.h similarity index 75% rename from include/common/tlocale.h rename to include/util/version.h index d809c75d67..01efecc59d 100644 --- a/include/common/tlocale.h +++ b/include/util/version.h @@ -13,17 +13,21 @@ * along with this program. If not, see . */ -#ifndef _TD_COMMON_LOCALE_H_ -#define _TD_COMMON_LOCALE_H_ +#ifndef _TD_UTIL_VERSION_H +#define _TD_UTIL_VERSION_H #ifdef __cplusplus extern "C" { #endif -void tsSetLocale(); +extern char version[]; +extern char compatible_version[]; +extern char gitinfo[]; +extern char gitinfoOfInternal[]; +extern char buildinfo[]; #ifdef __cplusplus } -#endif /*_TD_COMMON_LOCALE_H_*/ - #endif + +#endif /*_TD_UTIL_VERSION_H*/ diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index d7574466cd..523921634b 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -20,17 +20,19 @@ extern "C" { #endif -#include "taos.h" #include "common.h" -#include "tmsg.h" +#include "parser.h" +#include "query.h" +#include "taos.h" #include "tdef.h" #include "tep.h" #include "thash.h" #include "tlist.h" +#include "tmsg.h" #include "tmsgtype.h" #include "trpc.h" -#include "query.h" -#include "parser.h" + +#include "tconfig.h" #define CHECK_CODE_GOTO(expr, label) \ do { \ @@ -46,12 +48,12 @@ extern "C" { typedef struct SAppInstInfo SAppInstInfo; typedef struct SHbConnInfo { - void *param; - SClientHbReq *req; + void* param; + SClientHbReq* req; } SHbConnInfo; typedef struct SAppHbMgr { - char *key; + char* key; // statistics int32_t reportCnt; int32_t connKeyCnt; @@ -62,15 +64,13 @@ typedef struct SAppHbMgr { // connection SAppInstInfo* pAppInstInfo; // info - SHashObj* activeInfo; // hash - SHashObj* connInfo; // hash + SHashObj* activeInfo; // hash + SHashObj* connInfo; // hash } SAppHbMgr; +typedef int32_t (*FHbRspHandle)(struct SAppHbMgr* pAppHbMgr, SClientHbRsp* pRsp); -typedef int32_t (*FHbRspHandle)(struct SAppHbMgr *pAppHbMgr, SClientHbRsp* pRsp); - -typedef int32_t (*FHbReqHandle)(SClientHbKey *connKey, void* param, SClientHbReq *req); - +typedef int32_t (*FHbReqHandle)(SClientHbKey* connKey, void* param, SClientHbReq* req); typedef struct SClientHbMgr { int8_t inited; @@ -83,63 +83,62 @@ typedef struct SClientHbMgr { FHbRspHandle rspHandle[HEARTBEAT_TYPE_MAX]; } SClientHbMgr; - typedef struct SQueryExecMetric { - int64_t start; // start timestamp - int64_t parsed; // start to parse - int64_t send; // start to send to server - int64_t rsp; // receive response from server + int64_t start; // start timestamp + int64_t parsed; // start to parse + int64_t send; // start to send to server + int64_t rsp; // receive response from server } SQueryExecMetric; typedef struct SInstanceSummary { - uint64_t numOfInsertsReq; - uint64_t numOfInsertRows; - uint64_t insertElapsedTime; - uint64_t insertBytes; // submit to tsdb since launched. + uint64_t numOfInsertsReq; + uint64_t numOfInsertRows; + uint64_t insertElapsedTime; + uint64_t insertBytes; // submit to tsdb since launched. - uint64_t fetchBytes; - uint64_t queryElapsedTime; - uint64_t numOfSlowQueries; - uint64_t totalRequests; - uint64_t currentRequests; // the number of SRequestObj + uint64_t fetchBytes; + uint64_t queryElapsedTime; + uint64_t numOfSlowQueries; + uint64_t totalRequests; + uint64_t currentRequests; // the number of SRequestObj } SInstanceSummary; typedef struct SHeartBeatInfo { - void *pTimer; // timer, used to send request msg to mnode + void* pTimer; // timer, used to send request msg to mnode } SHeartBeatInfo; struct SAppInstInfo { - int64_t numOfConns; - SCorEpSet mgmtEp; - SInstanceSummary summary; - SList *pConnList; // STscObj linked list - int64_t clusterId; - void *pTransporter; - struct SAppHbMgr *pAppHbMgr; + int64_t numOfConns; + SCorEpSet mgmtEp; + SInstanceSummary summary; + SList* pConnList; // STscObj linked list + int64_t clusterId; + void* pTransporter; + struct SAppHbMgr* pAppHbMgr; }; typedef struct SAppInfo { int64_t startTime; char appName[TSDB_APP_NAME_LEN]; - char *ep; + char* ep; int32_t pid; int32_t numOfThreads; - SHashObj *pInstMap; + SHashObj* pInstMap; pthread_mutex_t mutex; } SAppInfo; typedef struct STscObj { - char user[TSDB_USER_LEN]; - char pass[TSDB_PASSWORD_LEN]; - char db[TSDB_DB_FNAME_LEN]; - char ver[128]; - int32_t acctId; - uint32_t connId; - int32_t connType; - uint64_t id; // ref ID returned by taosAddRef - pthread_mutex_t mutex; // used to protect the operation on db - int32_t numOfReqs; // number of sqlObj bound to this connection - SAppInstInfo *pAppInfo; + char user[TSDB_USER_LEN]; + char pass[TSDB_PASSWORD_LEN]; + char db[TSDB_DB_FNAME_LEN]; + char ver[128]; + int32_t acctId; + uint32_t connId; + int32_t connType; + uint64_t id; // ref ID returned by taosAddRef + pthread_mutex_t mutex; // used to protect the operation on db + int32_t numOfReqs; // number of sqlObj bound to this connection + SAppInstInfo* pAppInfo; } STscObj; typedef struct SMqConsumer { @@ -147,49 +146,49 @@ typedef struct SMqConsumer { } SMqConsumer; typedef struct SReqResultInfo { - const char *pRspMsg; - const char *pData; - TAOS_FIELD *fields; - uint32_t numOfCols; - int32_t *length; - TAOS_ROW row; - char **pCol; - uint32_t numOfRows; - uint64_t totalRows; - uint32_t current; - bool completed; + const char* pRspMsg; + const char* pData; + TAOS_FIELD* fields; + uint32_t numOfCols; + int32_t* length; + TAOS_ROW row; + char** pCol; + uint32_t numOfRows; + uint64_t totalRows; + uint32_t current; + bool completed; } SReqResultInfo; typedef struct SShowReqInfo { - int64_t execId; // showId/queryId - int32_t vgId; - SArray *pArray; // SArray - int32_t currentIndex; // current accessed vgroup index. + int64_t execId; // showId/queryId + int32_t vgId; + SArray* pArray; // SArray + int32_t currentIndex; // current accessed vgroup index. } SShowReqInfo; typedef struct SRequestSendRecvBody { - tsem_t rspSem; // not used now + tsem_t rspSem; // not used now void* fp; - SShowReqInfo showInfo; // todo this attribute will be removed after the query framework being completed. + SShowReqInfo showInfo; // todo this attribute will be removed after the query framework being completed. SDataBuf requestMsg; - struct SSchJob *pQueryJob; // query job, created according to sql query DAG. - struct SQueryDag *pDag; // the query dag, generated according to the sql statement. + struct SSchJob* pQueryJob; // query job, created according to sql query DAG. + struct SQueryDag* pDag; // the query dag, generated according to the sql statement. SReqResultInfo resInfo; } SRequestSendRecvBody; -#define ERROR_MSG_BUF_DEFAULT_SIZE 512 +#define ERROR_MSG_BUF_DEFAULT_SIZE 512 typedef struct SRequestObj { - uint64_t requestId; - int32_t type; // request type - STscObj *pTscObj; - char *sqlstr; // sql string - int32_t sqlLen; - int64_t self; - char *msgBuf; - void *pInfo; // sql parse info, generated by parser module - int32_t code; - SQueryExecMetric metric; + uint64_t requestId; + int32_t type; // request type + STscObj* pTscObj; + char* sqlstr; // sql string + int32_t sqlLen; + int64_t self; + char* msgBuf; + void* pInfo; // sql parse info, generated by parser module + int32_t code; + SQueryExecMetric metric; SRequestSendRecvBody body; } SRequestObj; @@ -198,51 +197,52 @@ extern int32_t clientReqRefPool; extern int32_t clientConnRefPool; extern int (*handleRequestRspFp[TDMT_MAX])(void*, const SDataBuf* pMsg, int32_t code); -int genericRspCallback(void* param, const SDataBuf* pMsg, int32_t code); +int genericRspCallback(void* param, const SDataBuf* pMsg, int32_t code); SMsgSendInfo* buildMsgInfoImpl(SRequestObj* pReqObj); -int taos_init(); +int taos_init(); -void* createTscObj(const char* user, const char* auth, const char *db, SAppInstInfo* pAppInfo); -void destroyTscObj(void*pObj); +void* createTscObj(const char* user, const char* auth, const char* db, SAppInstInfo* pAppInfo); +void destroyTscObj(void* pObj); uint64_t generateRequestId(); -void *createRequest(STscObj* pObj, __taos_async_fn_t fp, void* param, int32_t type); +void* createRequest(STscObj* pObj, __taos_async_fn_t fp, void* param, int32_t type); void destroyRequest(SRequestObj* pRequest); -char *getDbOfConnection(STscObj* pObj); +char* getDbOfConnection(STscObj* pObj); void setConnectionDB(STscObj* pTscObj, const char* db); void taos_init_imp(void); -int taos_options_imp(TSDB_OPTION option, const char *str); +int taos_options_imp(TSDB_OPTION option, const char* str); -void* openTransporter(const char *user, const char *auth, int32_t numOfThreads); +void* openTransporter(const char* user, const char* auth, int32_t numOfThreads); +bool persistConnForSpecificMsg(void* parenct, tmsg_t msgType); void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet); void initMsgHandleFp(); -TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass, const char *auth, const char *db, uint16_t port); +TAOS* taos_connect_internal(const char* ip, const char* user, const char* pass, const char* auth, const char* db, + uint16_t port); -void *doFetchRow(SRequestObj* pRequest); +void* doFetchRow(SRequestObj* pRequest); -void setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32_t numOfCols, int32_t numOfRows); +void setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32_t numOfCols, int32_t numOfRows); - -int32_t buildRequest(STscObj *pTscObj, const char *sql, int sqlLen, SRequestObj** pRequest); +int32_t buildRequest(STscObj* pTscObj, const char* sql, int sqlLen, SRequestObj** pRequest); int32_t parseSql(SRequestObj* pRequest, SQueryNode** pQuery); -// --- heartbeat +// --- heartbeat // global, called by mgmt int hbMgrInit(); void hbMgrCleanUp(); int hbHandleRsp(SClientHbBatchRsp* hbRsp); // cluster level -SAppHbMgr* appHbMgrInit(SAppInstInfo* pAppInstInfo, char *key); -void appHbMgrCleanup(void); +SAppHbMgr* appHbMgrInit(SAppInstInfo* pAppInstInfo, char* key); +void appHbMgrCleanup(void); // conn level int hbRegisterConn(SAppHbMgr* pAppHbMgr, int32_t connId, int64_t clusterId, int32_t hbType); @@ -253,7 +253,6 @@ int hbAddConnInfo(SAppHbMgr* pAppHbMgr, SClientHbKey connKey, void* key, void* v // --- mq void hbMgrInitMqHbRspHandle(); - #ifdef __cplusplus } #endif diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index cb73701bfa..08285c9d26 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -13,33 +13,30 @@ * along with this program. If not, see . */ -#include "os.h" #include "catalog.h" #include "clientInt.h" #include "clientLog.h" +#include "os.h" #include "query.h" #include "scheduler.h" -#include "tmsg.h" #include "tcache.h" -#include "tconfig.h" #include "tglobal.h" -#include "tnote.h" +#include "tmsg.h" #include "tref.h" #include "trpc.h" #include "ttime.h" -#include "ttimezone.h" #define TSC_VAR_NOT_RELEASE 1 -#define TSC_VAR_RELEASED 0 +#define TSC_VAR_RELEASED 0 -SAppInfo appInfo; -int32_t clientReqRefPool = -1; -int32_t clientConnRefPool = -1; +SAppInfo appInfo; +int32_t clientReqRefPool = -1; +int32_t clientConnRefPool = -1; static pthread_once_t tscinit = PTHREAD_ONCE_INIT; -volatile int32_t tscInitRes = 0; +volatile int32_t tscInitRes = 0; -static void registerRequest(SRequestObj* pRequest) { +static void registerRequest(SRequestObj *pRequest) { STscObj *pTscObj = (STscObj *)taosAcquireRef(clientConnRefPool, pRequest->pTscObj->id); assert(pTscObj != NULL); @@ -53,60 +50,49 @@ static void registerRequest(SRequestObj* pRequest) { int32_t total = atomic_add_fetch_32(&pSummary->totalRequests, 1); int32_t currentInst = atomic_add_fetch_32(&pSummary->currentRequests, 1); - tscDebug("0x%" PRIx64 " new Request from connObj:0x%" PRIx64 ", current:%d, app current:%d, total:%d, reqId:0x%"PRIx64, pRequest->self, - pRequest->pTscObj->id, num, currentInst, total, pRequest->requestId); + tscDebug("0x%" PRIx64 " new Request from connObj:0x%" PRIx64 + ", current:%d, app current:%d, total:%d, reqId:0x%" PRIx64, + pRequest->self, pRequest->pTscObj->id, num, currentInst, total, pRequest->requestId); } } -static void deregisterRequest(SRequestObj* pRequest) { +static void deregisterRequest(SRequestObj *pRequest) { assert(pRequest != NULL); - STscObj* pTscObj = pRequest->pTscObj; - SInstanceSummary* pActivity = &pTscObj->pAppInfo->summary; + STscObj * pTscObj = pRequest->pTscObj; + SInstanceSummary *pActivity = &pTscObj->pAppInfo->summary; int32_t currentInst = atomic_sub_fetch_32(&pActivity->currentRequests, 1); int32_t num = atomic_sub_fetch_32(&pTscObj->numOfReqs, 1); int64_t duration = taosGetTimestampMs() - pRequest->metric.start; - tscDebug("0x%"PRIx64" free Request from connObj: 0x%"PRIx64", reqId:0x%"PRIx64" elapsed:%"PRIu64" ms, current:%d, app current:%d", pRequest->self, pTscObj->id, - pRequest->requestId, duration, num, currentInst); + tscDebug("0x%" PRIx64 " free Request from connObj: 0x%" PRIx64 ", reqId:0x%" PRIx64 " elapsed:%" PRIu64 + " ms, current:%d, app current:%d", + pRequest->self, pTscObj->id, pRequest->requestId, duration, num, currentInst); taosReleaseRef(clientConnRefPool, pTscObj->id); } -static void tscInitLogFile() { - taosReadGlobalLogCfg(); - if (mkdir(tsLogDir, 0755) != 0 && errno != EEXIST) { - printf("failed to create log dir:%s\n", tsLogDir); - } - const char *defaultLogFileNamePrefix = "taoslog"; - const int32_t maxLogFileNum = 10; - - char temp[128] = {0}; - sprintf(temp, "%s/%s", tsLogDir, defaultLogFileNamePrefix); - if (taosInitLog(temp, tsNumOfLogLines, maxLogFileNum) < 0) { - printf("failed to open log file in directory:%s\n", tsLogDir); - } -} // todo close the transporter properly -void closeTransporter(STscObj* pTscObj) { +void closeTransporter(STscObj *pTscObj) { if (pTscObj == NULL || pTscObj->pAppInfo->pTransporter == NULL) { return; } - tscDebug("free transporter:%p in connObj: 0x%"PRIx64, pTscObj->pAppInfo->pTransporter, pTscObj->id); + tscDebug("free transporter:%p in connObj: 0x%" PRIx64, pTscObj->pAppInfo->pTransporter, pTscObj->id); rpcClose(pTscObj->pAppInfo->pTransporter); } // TODO refactor -void* openTransporter(const char *user, const char *auth, int32_t numOfThread) { +void *openTransporter(const char *user, const char *auth, int32_t numOfThread) { SRpcInit rpcInit; memset(&rpcInit, 0, sizeof(rpcInit)); rpcInit.localPort = 0; rpcInit.label = "TSC"; rpcInit.numOfThreads = numOfThread; rpcInit.cfp = processMsgFromServer; + rpcInit.pfp = persistConnForSpecificMsg; rpcInit.sessions = tsMaxConnections; rpcInit.connType = TAOS_CONN_CLIENT; rpcInit.user = (char *)user; @@ -115,7 +101,7 @@ void* openTransporter(const char *user, const char *auth, int32_t numOfThread) { rpcInit.spi = 1; rpcInit.secret = (char *)auth; - void* pDnodeConn = rpcOpen(&rpcInit); + void *pDnodeConn = rpcOpen(&rpcInit); if (pDnodeConn == NULL) { tscError("failed to init connection to server"); return NULL; @@ -130,12 +116,12 @@ void destroyTscObj(void *pObj) { SClientHbKey connKey = {.connId = pTscObj->connId, .hbType = pTscObj->connType}; hbDeregisterConn(pTscObj->pAppInfo->pAppHbMgr, connKey); atomic_sub_fetch_64(&pTscObj->pAppInfo->numOfConns, 1); - tscDebug("connObj 0x%"PRIx64" destroyed, totalConn:%"PRId64, pTscObj->id, pTscObj->pAppInfo->numOfConns); + tscDebug("connObj 0x%" PRIx64 " destroyed, totalConn:%" PRId64, pTscObj->id, pTscObj->pAppInfo->numOfConns); pthread_mutex_destroy(&pTscObj->mutex); tfree(pTscObj); } -void* createTscObj(const char* user, const char* auth, const char *db, SAppInstInfo* pAppInfo) { +void *createTscObj(const char *user, const char *auth, const char *db, SAppInstInfo *pAppInfo) { STscObj *pObj = (STscObj *)calloc(1, sizeof(STscObj)); if (NULL == pObj) { terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; @@ -153,11 +139,11 @@ void* createTscObj(const char* user, const char* auth, const char *db, SAppInstI pthread_mutex_init(&pObj->mutex, NULL); pObj->id = taosAddRef(clientConnRefPool, pObj); - tscDebug("connObj created, 0x%"PRIx64, pObj->id); + tscDebug("connObj created, 0x%" PRIx64, pObj->id); return pObj; } -void* createRequest(STscObj* pObj, __taos_async_fn_t fp, void* param, int32_t type) { +void *createRequest(STscObj *pObj, __taos_async_fn_t fp, void *param, int32_t type) { assert(pObj != NULL); SRequestObj *pRequest = (SRequestObj *)calloc(1, sizeof(SRequestObj)); @@ -166,20 +152,20 @@ void* createRequest(STscObj* pObj, __taos_async_fn_t fp, void* param, int32_t ty return NULL; } - pRequest->requestId = generateRequestId(); + pRequest->requestId = generateRequestId(); pRequest->metric.start = taosGetTimestampMs(); - pRequest->type = type; - pRequest->pTscObj = pObj; - pRequest->body.fp = fp; // not used it yet - pRequest->msgBuf = calloc(1, ERROR_MSG_BUF_DEFAULT_SIZE); + pRequest->type = type; + pRequest->pTscObj = pObj; + pRequest->body.fp = fp; // not used it yet + pRequest->msgBuf = calloc(1, ERROR_MSG_BUF_DEFAULT_SIZE); tsem_init(&pRequest->body.rspSem, 0, 0); registerRequest(pRequest); return pRequest; } -static void doFreeReqResultInfo(SReqResultInfo* pResInfo) { +static void doFreeReqResultInfo(SReqResultInfo *pResInfo) { tfree(pResInfo->pRspMsg); tfree(pResInfo->length); tfree(pResInfo->row); @@ -187,9 +173,9 @@ static void doFreeReqResultInfo(SReqResultInfo* pResInfo) { tfree(pResInfo->fields); } -static void doDestroyRequest(void* p) { +static void doDestroyRequest(void *p) { assert(p != NULL); - SRequestObj* pRequest = (SRequestObj*)p; + SRequestObj *pRequest = (SRequestObj *)p; assert(RID_VALID(pRequest->self)); @@ -208,7 +194,7 @@ static void doDestroyRequest(void* p) { tfree(pRequest); } -void destroyRequest(SRequestObj* pRequest) { +void destroyRequest(SRequestObj *pRequest) { if (pRequest == NULL) { return; } @@ -225,16 +211,17 @@ void taos_init_imp(void) { srand(taosGetTimestampSec()); deltaToUtcInitOnce(); - taosInitGlobalCfg(); - taosReadCfgFromFile(); - tscInitLogFile(); - if (taosCheckAndPrintCfg()) { + if (taosCreateLog("taoslog", 10, configDir, NULL, NULL, NULL, 1) != 0) { + tscInitRes = -1; + return; + } + + if (taosInitCfg(configDir, NULL, NULL, NULL, 1) != 0) { tscInitRes = -1; return; } - taosInitNotes(); initMsgHandleFp(); initQueryModuleMsgHandle(); @@ -245,21 +232,21 @@ void taos_init_imp(void) { SSchedulerCfg scfg = {.maxJobNum = 100}; schedulerInit(&scfg); - tscDebug("starting to initialize TAOS driver, local ep: %s", tsLocalEp); + tscDebug("starting to initialize TAOS driver"); taosSetCoreDump(true); initTaskQueue(); clientConnRefPool = taosOpenRef(200, destroyTscObj); - clientReqRefPool = taosOpenRef(40960, doDestroyRequest); + clientReqRefPool = taosOpenRef(40960, doDestroyRequest); taosGetAppName(appInfo.appName, NULL); pthread_mutex_init(&appInfo.mutex, NULL); - appInfo.pid = taosGetPId(); + appInfo.pid = taosGetPId(); appInfo.startTime = taosGetTimestampMs(); - appInfo.pInstMap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + appInfo.pInstMap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); tscDebug("client is initialized successfully"); } @@ -269,6 +256,7 @@ int taos_init() { } int taos_options_imp(TSDB_OPTION option, const char *str) { +#if 0 SGlobalCfg *cfg = NULL; switch (option) { @@ -281,7 +269,8 @@ int taos_options_imp(TSDB_OPTION option, const char *str) { cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION; tscInfo("set config file directory:%s", str); } else { - tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, str, tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr); + tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, str, + tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr); } break; @@ -296,7 +285,8 @@ int taos_options_imp(TSDB_OPTION option, const char *str) { cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION; tscInfo("set shellActivityTimer:%d", tsShellActivityTimer); } else { - tscWarn("config option:%s, input value:%s, is configured by %s, use %d", cfg->option, str, tsCfgStatusStr[cfg->cfgStatus], *(int32_t *)cfg->ptr); + tscWarn("config option:%s, input value:%s, is configured by %s, use %d", cfg->option, str, + tsCfgStatusStr[cfg->cfgStatus], *(int32_t *)cfg->ptr); } break; @@ -305,7 +295,7 @@ int taos_options_imp(TSDB_OPTION option, const char *str) { assert(cfg != NULL); size_t len = strlen(str); - if (len == 0 || len > TSDB_LOCALE_LEN) { + if (len == 0 || len > TD_LOCALE_LEN) { tscInfo("Invalid locale:%s, use default", str); return -1; } @@ -313,8 +303,8 @@ int taos_options_imp(TSDB_OPTION option, const char *str) { if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_OPTION) { char sep = '.'; - if (strlen(tsLocale) == 0) { // locale does not set yet - char* defaultLocale = setlocale(LC_CTYPE, ""); + if (strlen(tsLocale) == 0) { // locale does not set yet + char *defaultLocale = setlocale(LC_CTYPE, ""); // The locale of the current OS does not be set correctly, so the default locale cannot be acquired. // The launch of current system will abort soon. @@ -323,21 +313,21 @@ int taos_options_imp(TSDB_OPTION option, const char *str) { return -1; } - tstrncpy(tsLocale, defaultLocale, TSDB_LOCALE_LEN); + tstrncpy(tsLocale, defaultLocale, TD_LOCALE_LEN); } // set the user specified locale char *locale = setlocale(LC_CTYPE, str); - if (locale != NULL) { // failed to set the user specified locale + if (locale != NULL) { // failed to set the user specified locale tscInfo("locale set, prev locale:%s, new locale:%s", tsLocale, locale); cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION; - } else { // set the user specified locale failed, use default LC_CTYPE as current locale + } else { // set the user specified locale failed, use default LC_CTYPE as current locale locale = setlocale(LC_CTYPE, tsLocale); tscInfo("failed to set locale:%s, current locale:%s", str, tsLocale); } - tstrncpy(tsLocale, locale, TSDB_LOCALE_LEN); + tstrncpy(tsLocale, locale, TD_LOCALE_LEN); char *charset = strrchr(tsLocale, sep); if (charset != NULL) { @@ -352,7 +342,7 @@ int taos_options_imp(TSDB_OPTION option, const char *str) { tscInfo("charset changed from %s to %s", tsCharset, charset); } - tstrncpy(tsCharset, charset, TSDB_LOCALE_LEN); + tstrncpy(tsCharset, charset, TD_LOCALE_LEN); cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION; } else { @@ -360,11 +350,12 @@ int taos_options_imp(TSDB_OPTION option, const char *str) { } free(charset); - } else { // it may be windows system + } else { // it may be windows system tscInfo("charset remains:%s", tsCharset); } } else { - tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, str, tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr); + tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, str, + tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr); } break; } @@ -375,7 +366,7 @@ int taos_options_imp(TSDB_OPTION option, const char *str) { assert(cfg != NULL); size_t len = strlen(str); - if (len == 0 || len > TSDB_LOCALE_LEN) { + if (len == 0 || len > TD_LOCALE_LEN) { tscInfo("failed to set charset:%s", str); return -1; } @@ -388,13 +379,14 @@ int taos_options_imp(TSDB_OPTION option, const char *str) { tscInfo("charset changed from %s to %s", tsCharset, str); } - tstrncpy(tsCharset, str, TSDB_LOCALE_LEN); + tstrncpy(tsCharset, str, TD_LOCALE_LEN); cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION; } else { tscInfo("charset:%s not valid", str); } } else { - tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, str, tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr); + tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, str, + tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr); } break; @@ -405,12 +397,13 @@ int taos_options_imp(TSDB_OPTION option, const char *str) { assert(cfg != NULL); if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_OPTION) { - tstrncpy(tsTimezone, str, TSDB_TIMEZONE_LEN); + tstrncpy(tsTimezone, str, TD_TIMEZONE_LEN); tsSetTimeZone(); cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION; tscDebug("timezone set:%s, input:%s by taos_options", tsTimezone, str); } else { - tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, str, tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr); + tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, str, + tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr); } break; @@ -419,7 +412,7 @@ int taos_options_imp(TSDB_OPTION option, const char *str) { tscError("Invalid option %d", option); return -1; } - +#endif return 0; } @@ -434,7 +427,7 @@ int taos_options_imp(TSDB_OPTION option, const char *str) { */ uint64_t generateRequestId() { static uint64_t hashId = 0; - static int32_t requestSerialId = 0; + static int32_t requestSerialId = 0; if (hashId == 0) { char uid[64] = {0}; @@ -448,9 +441,9 @@ uint64_t generateRequestId() { } } - int64_t ts = taosGetTimestampMs(); - uint64_t pid = taosGetPId(); - int32_t val = atomic_add_fetch_32(&requestSerialId, 1); + int64_t ts = taosGetTimestampMs(); + uint64_t pid = taosGetPId(); + int32_t val = atomic_add_fetch_32(&requestSerialId, 1); uint64_t id = ((hashId & 0x0FFF) << 52) | ((pid & 0x0FFF) << 40) | ((ts & 0xFFFFFF) << 16) | (val & 0xFFFF); return id; diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 2eff353c39..f256feb251 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -8,7 +8,6 @@ #include "tep.h" #include "tglobal.h" #include "tmsgtype.h" -#include "tnote.h" #include "tpagedbuf.h" #include "tref.h" @@ -254,11 +253,9 @@ TAOS_RES* taos_query_l(TAOS* taos, const char* sql, int sqlLen) { return NULL; } - nPrintTsc("%s", sql) - - SRequestObj* pRequest = NULL; - SQueryNode* pQueryNode = NULL; - SArray* pNodeList = taosArrayInit(4, sizeof(struct SQueryNodeAddr)); + SRequestObj* pRequest = NULL; + SQueryNode* pQueryNode = NULL; + SArray* pNodeList = taosArrayInit(4, sizeof(struct SQueryNodeAddr)); terrno = TSDB_CODE_SUCCESS; CHECK_CODE_GOTO(buildRequest(pTscObj, sql, sqlLen, &pRequest), _return); @@ -370,7 +367,6 @@ static SMsgSendInfo* buildConnectMsg(SRequestObj* pRequest) { pMsgSendInfo->fp = handleRequestRspFp[TMSG_INDEX(pMsgSendInfo->msgType)]; pMsgSendInfo->param = pRequest; - SConnectReq connectReq = {0}; STscObj* pObj = pRequest->pTscObj; @@ -398,7 +394,9 @@ static void destroySendMsgInfo(SMsgSendInfo* pMsgBody) { tfree(pMsgBody->msgInfo.pData); tfree(pMsgBody); } - +bool persistConnForSpecificMsg(void* parenct, tmsg_t msgType) { + return msgType == TDMT_VND_QUERY_RSP || msgType == TDMT_VND_FETCH_RSP || msgType == TDMT_VND_RES_READY_RSP; +} void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) { SMsgSendInfo* pSendInfo = (SMsgSendInfo*)pMsg->ahandle; assert(pMsg->ahandle != NULL); diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index b29813be03..5bcc287116 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -7,6 +7,7 @@ #include "tmsg.h" #include "tglobal.h" #include "catalog.h" +#include "version.h" #define TSC_VAR_NOT_RELEASE 1 #define TSC_VAR_RELEASED 0 @@ -56,9 +57,7 @@ void taos_cleanup(void) { } TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port) { - int32_t p = (port != 0) ? port : tsServerPort; - - tscDebug("try to connect to %s:%u, user:%s db:%s", ip, p, user, db); + tscDebug("try to connect to %s:%u, user:%s db:%s", ip, port, user, db); if (user == NULL) { user = TSDB_DEFAULT_USER; } @@ -67,7 +66,7 @@ TAOS *taos_connect(const char *ip, const char *user, const char *pass, const cha pass = TSDB_DEFAULT_PASS; } - return taos_connect_internal(ip, user, pass, NULL, db, p); + return taos_connect_internal(ip, user, pass, NULL, db, port); } void taos_close(TAOS* taos) { diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index 229d3a9ec3..d3d9fce210 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -24,7 +24,6 @@ #include "tep.h" #include "tglobal.h" #include "tmsgtype.h" -#include "tnote.h" #include "tpagedbuf.h" #include "tref.h" @@ -45,22 +44,24 @@ struct tmq_topic_vgroup_list_t { }; struct tmq_conf_t { - char clientId[256]; - char groupId[256]; - bool auto_commit; + char clientId[256]; + char groupId[256]; + int8_t auto_commit; + int8_t resetOffset; + tmq_commit_cb* commit_cb; /*char* ip;*/ /*uint16_t port;*/ - tmq_commit_cb* commit_cb; }; struct tmq_t { // conf char groupId[256]; char clientId[256]; - bool autoCommit; + int8_t autoCommit; SRWLatch lock; int64_t consumerId; int32_t epoch; + int32_t resetOffsetCfg; int64_t status; tsem_t rspSem; STscObj* pTscObj; @@ -79,7 +80,6 @@ typedef struct { // statistics int64_t pollCnt; // offset - int64_t committedOffset; int64_t currentOffset; // connection info int32_t vgId; @@ -115,21 +115,17 @@ typedef struct { } SMqConsumeCbParam; typedef struct { - tmq_t* tmq; - SMqClientVg* pVg; - int32_t async; - tsem_t rspSem; -} SMqCommitCbParam; - -typedef struct { - tmq_t* tmq; + tmq_t* tmq; + /*SMqClientVg* pVg;*/ + int32_t async; tsem_t rspSem; tmq_resp_err_t rspErr; -} SMqResetOffsetParam; +} SMqCommitCbParam; tmq_conf_t* tmq_conf_new() { tmq_conf_t* conf = calloc(1, sizeof(tmq_conf_t)); conf->auto_commit = false; + conf->resetOffset = TMQ_CONF__RESET_OFFSET__LATEST; return conf; } @@ -157,6 +153,20 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value return TMQ_CONF_INVALID; } } + if (strcmp(key, "auto.offset.reset") == 0) { + if (strcmp(value, "none") == 0) { + conf->resetOffset = TMQ_CONF__RESET_OFFSET__NONE; + return TMQ_CONF_OK; + } else if (strcmp(value, "earliest") == 0) { + conf->resetOffset = TMQ_CONF__RESET_OFFSET__EARLIEAST; + return TMQ_CONF_OK; + } else if (strcmp(value, "latest") == 0) { + conf->resetOffset = TMQ_CONF__RESET_OFFSET__LATEST; + return TMQ_CONF_OK; + } else { + return TMQ_CONF_INVALID; + } + } return TMQ_CONF_UNKNOWN; } @@ -190,14 +200,12 @@ int32_t tmqCommitCb(void* param, const SDataBuf* pMsg, int32_t code) { if (pParam->tmq->commit_cb) { pParam->tmq->commit_cb(pParam->tmq, rspErr, NULL, NULL); } - if (!pParam->async) tsem_post(&pParam->rspSem); - return 0; -} - -int32_t tmqResetOffsetCb(void* param, const SDataBuf* pMsg, int32_t code) { - SMqResetOffsetParam* pParam = (SMqResetOffsetParam*)param; - pParam->rspErr = code; - tsem_post(&pParam->rspSem); + if (!pParam->async) + tsem_post(&pParam->rspSem); + else { + tsem_destroy(&pParam->rspSem); + free(param); + } return 0; } @@ -216,6 +224,7 @@ tmq_t* tmq_consumer_new(void* conn, tmq_conf_t* conf, char* errstr, int32_t errs strcpy(pTmq->groupId, conf->groupId); pTmq->autoCommit = conf->auto_commit; pTmq->commit_cb = conf->commit_cb; + pTmq->resetOffsetCfg = conf->resetOffset; tsem_init(&pTmq->rspSem, 0, 0); pTmq->consumerId = generateRequestId() & (((uint64_t)-1) >> 1); @@ -223,18 +232,40 @@ tmq_t* tmq_consumer_new(void* conn, tmq_conf_t* conf, char* errstr, int32_t errs return pTmq; } -tmq_resp_err_t tmq_reset_offset(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets) { - SRequestObj* pRequest = NULL; +tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets, int32_t async) { + // TODO: add read write lock + SRequestObj* pRequest = NULL; + tmq_resp_err_t resp = TMQ_RESP_ERR__SUCCESS; // build msg // send to mnode - SMqCMResetOffsetReq req; - req.num = offsets->cnt; - req.offsets = (SMqOffset*)offsets->elems; + SMqCMCommitOffsetReq req; + SArray* pArray = NULL; + + if (offsets == NULL) { + pArray = taosArrayInit(0, sizeof(SMqOffset)); + for (int i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) { + SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i); + for (int j = 0; j < taosArrayGetSize(pTopic->vgs); j++) { + SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j); + SMqOffset offset; + strcpy(offset.topicName, pTopic->topicName); + strcpy(offset.cgroup, tmq->groupId); + offset.vgId = pVg->vgId; + offset.offset = pVg->currentOffset; + taosArrayPush(pArray, &offset); + } + } + req.num = pArray->size; + req.offsets = pArray->pData; + } else { + req.num = offsets->cnt; + req.offsets = (SMqOffset*)offsets->elems; + } SCoder encoder; tCoderInit(&encoder, TD_LITTLE_ENDIAN, NULL, 0, TD_ENCODER); - tEncodeSMqCMResetOffsetReq(&encoder, &req); + tEncodeSMqCMCommitOffsetReq(&encoder, &req); int32_t tlen = encoder.pos; void* buf = malloc(tlen); if (buf == NULL) { @@ -244,32 +275,41 @@ tmq_resp_err_t tmq_reset_offset(tmq_t* tmq, const tmq_topic_vgroup_list_t* offse tCoderClear(&encoder); tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, tlen, TD_ENCODER); - tEncodeSMqCMResetOffsetReq(&encoder, &req); + tEncodeSMqCMCommitOffsetReq(&encoder, &req); tCoderClear(&encoder); - pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_MND_RESET_OFFSET); + pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_MND_MQ_COMMIT_OFFSET); if (pRequest == NULL) { tscError("failed to malloc request"); } - SMqResetOffsetParam param = {0}; - tsem_init(¶m.rspSem, 0, 0); - param.tmq = tmq; + SMqCommitCbParam* pParam = malloc(sizeof(SMqCommitCbParam)); + if (pParam == NULL) { + return -1; + } + pParam->tmq = tmq; + tsem_init(&pParam->rspSem, 0, 0); pRequest->body.requestMsg = (SDataBuf){.pData = buf, .len = tlen}; SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest); - sendInfo->param = ¶m; - sendInfo->fp = tmqResetOffsetCb; + sendInfo->param = pParam; + sendInfo->fp = tmqCommitCb; SEpSet epSet = getEpSet_s(&tmq->pTscObj->pAppInfo->mgmtEp); int64_t transporterId = 0; asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo); - tsem_wait(¶m.rspSem); - tsem_destroy(¶m.rspSem); + if (!async) { + tsem_wait(&pParam->rspSem); + resp = pParam->rspErr; + } - return param.rspErr; + if (pArray) { + taosArrayDestroy(pArray); + } + + return resp; } tmq_resp_err_t tmq_subscribe(tmq_t* tmq, tmq_list_t* topic_list) { @@ -641,8 +681,7 @@ int32_t tmqAskEpCb(void* param, const SDataBuf* pMsg, int32_t code) { // clang-format off SMqClientVg clientVg = { .pollCnt = 0, - .committedOffset = -1, - .currentOffset = -1, + .currentOffset = pVgEp->offset, .vgId = pVgEp->vgId, .epSet = pVgEp->epSet }; @@ -708,23 +747,51 @@ END: return 0; } -SMqConsumeReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blocking_time, int32_t type, SMqClientTopic* pTopic, - SMqClientVg* pVg) { +tmq_resp_err_t tmq_seek(tmq_t* tmq, const tmq_topic_vgroup_t* offset) { + const SMqOffset* pOffset = &offset->offset; + if (strcmp(pOffset->cgroup, tmq->groupId) != 0) { + return TMQ_RESP_ERR__FAIL; + } + int32_t sz = taosArrayGetSize(tmq->clientTopics); + for (int32_t i = 0; i < sz; i++) { + SMqClientTopic* clientTopic = taosArrayGet(tmq->clientTopics, i); + if (strcmp(clientTopic->topicName, pOffset->topicName) == 0) { + int32_t vgSz = taosArrayGetSize(clientTopic->vgs); + for (int32_t j = 0; j < vgSz; j++) { + SMqClientVg* pVg = taosArrayGet(clientTopic->vgs, j); + if (pVg->vgId == pOffset->vgId) { + pVg->currentOffset = pOffset->offset; + return TMQ_RESP_ERR__SUCCESS; + } + } + } + } + return TMQ_RESP_ERR__FAIL; +} + +SMqConsumeReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blocking_time, SMqClientTopic* pTopic, SMqClientVg* pVg) { + int64_t reqOffset; + if (pVg->currentOffset >= 0) { + reqOffset = pVg->currentOffset; + } else { + if (tmq->resetOffsetCfg == TMQ_CONF__RESET_OFFSET__NONE) { + tscError("unable to poll since no committed offset but reset offset is set to none"); + return NULL; + } + reqOffset = tmq->resetOffsetCfg; + } + SMqConsumeReq* pReq = malloc(sizeof(SMqConsumeReq)); if (pReq == NULL) { return NULL; } - pReq->reqType = type; + strcpy(pReq->topic, pTopic->topicName); - pReq->blockingTime = blocking_time; - pReq->consumerId = tmq->consumerId; strcpy(pReq->cgroup, tmq->groupId); - if (type == TMQ_REQ_TYPE_COMMIT_ONLY) { - pReq->offset = pVg->currentOffset; - } else { - pReq->offset = pVg->currentOffset + 1; - } + pReq->blockingTime = blocking_time; + pReq->consumerId = tmq->consumerId; + pReq->currentOffset = reqOffset; pReq->head.vgId = htonl(pVg->vgId); pReq->head.contLen = htonl(sizeof(SMqConsumeReq)); @@ -743,13 +810,13 @@ tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) { if (taosArrayGetSize(tmq->clientTopics) == 0) { tscDebug("consumer:%ld poll but not assigned", tmq->consumerId); - printf("over1\n"); + /*printf("over1\n");*/ usleep(blocking_time * 1000); return NULL; } SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, tmq->nextTopicIdx); if (taosArrayGetSize(pTopic->vgs) == 0) { - printf("over2\n"); + /*printf("over2\n");*/ usleep(blocking_time * 1000); return NULL; } @@ -760,8 +827,7 @@ tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) { pTopic->nextVgIdx = (pTopic->nextVgIdx + 1) % taosArrayGetSize(pTopic->vgs); SMqClientVg* pVg = taosArrayGet(pTopic->vgs, pTopic->nextVgIdx); /*printf("consume vg %d, offset %ld\n", pVg->vgId, pVg->currentOffset);*/ - int32_t reqType = tmq->autoCommit ? TMQ_REQ_TYPE_CONSUME_AND_COMMIT : TMQ_REQ_TYPE_CONSUME_ONLY; - SMqConsumeReq* pReq = tmqBuildConsumeReqImpl(tmq, blocking_time, reqType, pTopic, pVg); + SMqConsumeReq* pReq = tmqBuildConsumeReqImpl(tmq, blocking_time, pTopic, pVg); if (pReq == NULL) { ASSERT(false); usleep(blocking_time * 1000); @@ -821,6 +887,7 @@ tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) { /*return pRequest;*/ } +#if 0 tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* tmq_topic_vgroup_list, int32_t async) { if (tmq_topic_vgroup_list != NULL) { // TODO @@ -831,7 +898,7 @@ tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* tmq_topic_v SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i); for (int j = 0; j < taosArrayGetSize(pTopic->vgs); j++) { SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j); - SMqConsumeReq* pReq = tmqBuildConsumeReqImpl(tmq, 0, TMQ_REQ_TYPE_COMMIT_ONLY, pTopic, pVg); + SMqConsumeReq* pReq = tmqBuildConsumeReqImpl(tmq, 0, pTopic, pVg); SRequestObj* pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_VND_CONSUME); pRequest->body.requestMsg = (SDataBuf){.pData = pReq, .len = sizeof(SMqConsumeReq)}; @@ -858,6 +925,7 @@ tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* tmq_topic_v return 0; } +#endif void tmq_message_destroy(tmq_message_t* tmq_message) { if (tmq_message == NULL) return; diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index 5414f17a62..ac36179a26 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -49,7 +49,7 @@ int main(int argc, char** argv) { } TEST(testCase, driverInit_Test) { - taosInitGlobalCfg(); + // taosInitGlobalCfg(); // taos_init(); } diff --git a/source/client/test/tmqTest.cpp b/source/client/test/tmqTest.cpp index 9f8ff7143a..c34c10d871 100644 --- a/source/client/test/tmqTest.cpp +++ b/source/client/test/tmqTest.cpp @@ -34,7 +34,7 @@ int main(int argc, char** argv) { } TEST(testCase, driverInit_Test) { - taosInitGlobalCfg(); + // taosInitGlobalCfg(); // taos_init(); } diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 41d911e4c7..0180cc5ede 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -13,11 +13,11 @@ * along with this program. If not, see . */ #include "tdataformat.h" -#include "ulog.h" #include "talgo.h" #include "tcoding.h" #include "wchar.h" #include "tarray.h" +#include "tlog.h" static void dataColSetNEleNull(SDataCol *pCol, int nEle); #if 0 diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index afba1190ba..26d33199c1 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -22,11 +22,10 @@ #include "tconfig.h" #include "tep.h" #include "tglobal.h" -#include "tlocale.h" #include "tlog.h" -#include "ttimezone.h" #include "tutil.h" -#include "ulog.h" + +SConfig *tsCfg = NULL; // cluster char tsFirst[TSDB_EP_LEN] = {0}; @@ -34,27 +33,24 @@ char tsSecond[TSDB_EP_LEN] = {0}; char tsLocalFqdn[TSDB_FQDN_LEN] = {0}; char tsLocalEp[TSDB_EP_LEN] = {0}; // Local End Point, hostname:port uint16_t tsServerPort = 6030; +int32_t tsVersion = 30000000; int32_t tsStatusInterval = 1; // second -int8_t tsEnableTelemetryReporting = 0; -char tsEmail[TSDB_FQDN_LEN] = {0}; -int32_t tsNumOfSupportVnodes = 128; +bool tsEnableTelemetryReporting = 0; // common int32_t tsRpcTimer = 300; int32_t tsRpcMaxTime = 600; // seconds; -int32_t tsRpcForceTcp = 1; // disable this, means query, show command use udp protocol as default +bool tsRpcForceTcp = 1; // disable this, means query, show command use udp protocol as default int32_t tsMaxShellConns = 50000; int32_t tsMaxConnections = 50000; int32_t tsShellActivityTimer = 3; // second float tsNumOfThreadsPerCore = 1.0f; int32_t tsNumOfCommitThreads = 4; float tsRatioOfQueryCores = 1.0f; -int8_t tsDaylight = 0; -int8_t tsEnableCoreFile = 0; int32_t tsMaxBinaryDisplayWidth = 30; -int8_t tsEnableSlaveQuery = 1; -int8_t tsEnableAdjustMaster = 1; -int8_t tsPrintAuth = 0; +bool tsEnableSlaveQuery = 1; +bool tsPrintAuth = 0; + /* * denote if the server needs to compress response message at the application layer to client, including query rsp, * metricmeta rsp, and multi-meter query rsp message body. The client compress the submit message to server. @@ -81,8 +77,6 @@ int32_t tsCompatibleModel = 1; int32_t tsMaxWildCardsLen = TSDB_PATTERN_STRING_DEFAULT_LEN; int32_t tsMaxRegexStringLen = TSDB_REGEX_STRING_DEFAULT_LEN; -int8_t tsTscEnableRecordSql = 0; - // the maximum number of results for projection query on super table that are returned from // one virtual node, to order according to timestamp int32_t tsMaxNumOfOrderedResults = 100000; @@ -119,25 +113,20 @@ int32_t tsQueryBufferSize = -1; int64_t tsQueryBufferSizeBytes = -1; // in retrieve blocking model, the retrieve threads will wait for the completion of the query processing. -int32_t tsRetrieveBlockingModel = 0; +bool tsRetrieveBlockingModel = 0; // last_row(*), first(*), last_row(ts, col1, col2) query, the result fields will be the original column name -int8_t tsKeepOriginalColumnName = 0; +bool tsKeepOriginalColumnName = 0; // long query death-lock -int8_t tsDeadLockKillQuery = 0; +bool tsDeadLockKillQuery = 0; // tsdb config // For backward compatibility bool tsdbForceKeepFile = false; -int32_t tsDiskCfgNum = 0; - -#ifndef _STORAGE -SDiskCfg tsDiskCfg[1]; -#else -SDiskCfg tsDiskCfg[TFS_MAX_DISKS]; -#endif +int32_t tsDiskCfgNum = 0; +SDiskCfg tsDiskCfg[TFS_MAX_DISKS] = {0}; /* * minimum scale for whole system, millisecond by default @@ -147,20 +136,7 @@ SDiskCfg tsDiskCfg[TFS_MAX_DISKS]; */ int64_t tsTickPerDay[] = {86400000L, 86400000000L, 86400000000000L}; -// system info -float tsTotalTmpDirGB = 0; -float tsTotalDataDirGB = 0; -float tsAvailTmpDirectorySpace = 0; -float tsAvailDataDirGB = 0; -float tsUsedDataDirGB = 0; -float tsReservedTmpDirectorySpace = 1.0f; -float tsMinimalDataDirGB = 2.0f; -int32_t tsTotalMemoryMB = 0; -uint32_t tsVersion = 0; - -// // lossy compress 6 -// char tsLossyColumns[32] = ""; // "float|double" means all float and double columns can be lossy compressed. set empty // can close lossy compress. // below option can take effect when tsLossyColumns not empty @@ -170,986 +146,381 @@ uint32_t tsMaxRange = 500; // max range uint32_t tsCurRange = 100; // range char tsCompressor[32] = "ZSTD_COMPRESSOR"; // ZSTD_COMPRESSOR or GZIP_COMPRESSOR - -int32_t (*monStartSystemFp)() = NULL; -void (*monStopSystemFp)() = NULL; -void (*monExecuteSQLFp)(char *sql) = NULL; - -char *qtypeStr[] = {"rpc", "fwd", "wal", "cq", "query"}; - -static pthread_once_t tsInitGlobalCfgOnce = PTHREAD_ONCE_INIT; - -void taosSetAllDebugFlag() { - if (debugFlag != 0) { - mDebugFlag = debugFlag; - dDebugFlag = debugFlag; - vDebugFlag = debugFlag; - jniDebugFlag = debugFlag; - qDebugFlag = debugFlag; - rpcDebugFlag = debugFlag; - uDebugFlag = debugFlag; - sDebugFlag = debugFlag; - wDebugFlag = debugFlag; - tsdbDebugFlag = debugFlag; - cqDebugFlag = debugFlag; - uInfo("all debug flag are set to %d", debugFlag); - } -} - -int32_t taosCfgDynamicOptions(char *msg) { - char *option, *value; - int32_t olen, vlen; - int32_t vint = 0; - - paGetToken(msg, &option, &olen); - if (olen == 0) return -1; - - paGetToken(option + olen + 1, &value, &vlen); - if (vlen == 0) - vint = 135; - else { - vint = atoi(value); - } - - uInfo("change dynamic option: %s, value: %d", option, vint); - - for (int32_t i = 0; i < tsGlobalConfigNum; ++i) { - SGlobalCfg *cfg = tsGlobalConfig + i; - // if (!(cfg->cfgType & TSDB_CFG_CTYPE_B_LOG)) continue; - if (cfg->valType != TAOS_CFG_VTYPE_INT32 && cfg->valType != TAOS_CFG_VTYPE_INT8) continue; - - int32_t cfgLen = (int32_t)strlen(cfg->option); - if (cfgLen != olen) continue; - if (strncasecmp(option, cfg->option, olen) != 0) continue; - if (cfg->valType == TAOS_CFG_VTYPE_INT32) { - *((int32_t *)cfg->ptr) = vint; - } else { - *((int8_t *)cfg->ptr) = (int8_t)vint; - } - - if (strncasecmp(cfg->option, "monitor", olen) == 0) { - if (1 == vint) { - if (monStartSystemFp) { - (*monStartSystemFp)(); - uInfo("monitor is enabled"); - } else { - uError("monitor can't be updated, for monitor not initialized"); - } - } else { - if (monStopSystemFp) { - (*monStopSystemFp)(); - uInfo("monitor is disabled"); - } else { - uError("monitor can't be updated, for monitor not initialized"); - } - } - return 0; - } - if (strncasecmp(cfg->option, "debugFlag", olen) == 0) { - taosSetAllDebugFlag(); - } - return 0; - } - - if (strncasecmp(option, "resetlog", 8) == 0) { - taosResetLog(); - taosPrintCfg(); - return 0; - } - - if (strncasecmp(option, "resetQueryCache", 15) == 0) { - if (monExecuteSQLFp) { - (*monExecuteSQLFp)("resetQueryCache"); - uInfo("resetquerycache is executed"); - } else { - uError("resetquerycache can't be executed, for monitor not started"); - } - } - - return false; -} - -void taosAddDataDir(int index, char *v1, int level, int primary) { +static void taosAddDataDir(int32_t index, char *v1, int32_t level, int32_t primary) { tstrncpy(tsDiskCfg[index].dir, v1, TSDB_FILENAME_LEN); tsDiskCfg[index].level = level; tsDiskCfg[index].primary = primary; uTrace("dataDir:%s, level:%d primary:%d is configured", v1, level, primary); } -#ifndef _STORAGE -void taosReadDataDirCfg(char *v1, char *v2, char *v3) { - if (tsDiskCfgNum == 1) { - SDiskCfg *cfg = &tsDiskCfg[0]; - uInfo("dataDir:%s, level:%d primary:%d is replaced by %s", cfg->dir, cfg->level, cfg->primary, v1); - } - taosAddDataDir(0, v1, 0, 1); - tsDiskCfgNum = 1; -} +static void taosSetTfsCfg(SConfig *pCfg) { + SConfigItem *pItem = cfgGetItem(pCfg, "dataDir"); + if (pItem == NULL) return; -void taosPrintDataDirCfg() { - for (int i = 0; i < tsDiskCfgNum; ++i) { - SDiskCfg *cfg = &tsDiskCfg[i]; - uInfo(" dataDir: %s", cfg->dir); - } -} -#endif - -static void taosCheckDataDirCfg() { - if (tsDiskCfgNum <= 0) { - taosAddDataDir(0, tsDataDir, 0, 1); + int32_t size = taosArrayGetSize(pItem->array); + if (size <= 0) { tsDiskCfgNum = 1; - uTrace("dataDir:%s, level:0 primary:1 is configured by default", tsDataDir); + taosAddDataDir(0, pItem->str, 0, 1); + } else { + tsDiskCfgNum = size < TFS_MAX_DISKS ? size : TFS_MAX_DISKS; + for (int32_t index = 0; index < tsDiskCfgNum; ++index) { + SDiskCfg *pCfg = taosArrayGet(pItem->array, index); + memcpy(&tsDiskCfg[index], pCfg, sizeof(SDiskCfg)); + } } } -static void doInitGlobalConfig(void) { - osInit(); - srand(taosSafeRand()); - - SGlobalCfg cfg = {0}; - - // ip address - cfg.option = "firstEp"; - cfg.ptr = tsFirst; - cfg.valType = TAOS_CFG_VTYPE_STRING; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 0; - cfg.ptrLength = TSDB_EP_LEN; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "secondEp"; - cfg.ptr = tsSecond; - cfg.valType = TAOS_CFG_VTYPE_STRING; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 0; - cfg.ptrLength = TSDB_EP_LEN; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "fqdn"; - cfg.ptr = tsLocalFqdn; - cfg.valType = TAOS_CFG_VTYPE_STRING; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 0; - cfg.ptrLength = TSDB_FQDN_LEN; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - // port - cfg.option = "serverPort"; - cfg.ptr = &tsServerPort; - cfg.valType = TAOS_CFG_VTYPE_UINT16; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 1; - cfg.maxValue = 65056; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "supportVnodes"; - cfg.ptr = &tsNumOfSupportVnodes; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 65536; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - // directory - cfg.option = "configDir"; - cfg.ptr = configDir; - cfg.valType = TAOS_CFG_VTYPE_DIRECTORY; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 0; - cfg.ptrLength = TSDB_FILENAME_LEN; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "logDir"; - cfg.ptr = tsLogDir; - cfg.valType = TAOS_CFG_VTYPE_DIRECTORY; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT | TSDB_CFG_CTYPE_B_LOG; - cfg.minValue = 0; - cfg.maxValue = 0; - cfg.ptrLength = TSDB_FILENAME_LEN; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "scriptDir"; - cfg.ptr = tsScriptDir; - cfg.valType = TAOS_CFG_VTYPE_DIRECTORY; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 0; - cfg.ptrLength = TSDB_FILENAME_LEN; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "dataDir"; - cfg.ptr = tsDataDir; - cfg.valType = TAOS_CFG_VTYPE_DATA_DIRCTORY; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; - cfg.minValue = 0; - cfg.maxValue = 0; - cfg.ptrLength = TSDB_FILENAME_LEN; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - // dnode configs - cfg.option = "numOfThreadsPerCore"; - cfg.ptr = &tsNumOfThreadsPerCore; - cfg.valType = TAOS_CFG_VTYPE_FLOAT; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 10; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "numOfCommitThreads"; - cfg.ptr = &tsNumOfCommitThreads; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; - cfg.minValue = 1; - cfg.maxValue = 100; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "ratioOfQueryCores"; - cfg.ptr = &tsRatioOfQueryCores; - cfg.valType = TAOS_CFG_VTYPE_FLOAT; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; - cfg.minValue = 0.0f; - cfg.maxValue = 2.0f; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "maxNumOfDistinctRes"; - cfg.ptr = &tsMaxNumOfDistinctResults; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 10 * 10000; - cfg.maxValue = 10000 * 10000; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "telemetryReporting"; - cfg.ptr = &tsEnableTelemetryReporting; - cfg.valType = TAOS_CFG_VTYPE_INT8; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 0; - cfg.maxValue = 1; - cfg.ptrLength = 1; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - // timer - cfg.option = "maxTmrCtrl"; - cfg.ptr = &tsMaxTmrCtrl; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 8; - cfg.maxValue = 2048; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "rpcTimer"; - cfg.ptr = &tsRpcTimer; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 100; - cfg.maxValue = 3000; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_MS; - taosAddConfigOption(cfg); - - cfg.option = "rpcForceTcp"; - cfg.ptr = &tsRpcForceTcp; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 1; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "rpcMaxTime"; - cfg.ptr = &tsRpcMaxTime; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 100; - cfg.maxValue = 7200; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_SECOND; - taosAddConfigOption(cfg); - - cfg.option = "statusInterval"; - cfg.ptr = &tsStatusInterval; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 1; - cfg.maxValue = 10; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_SECOND; - taosAddConfigOption(cfg); - - cfg.option = "shellActivityTimer"; - cfg.ptr = &tsShellActivityTimer; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 1; - cfg.maxValue = 120; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_SECOND; - taosAddConfigOption(cfg); - - cfg.option = "minSlidingTime"; - cfg.ptr = &tsMinSlidingTime; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 10; - cfg.maxValue = 1000000; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_MS; - taosAddConfigOption(cfg); - - cfg.option = "minIntervalTime"; - cfg.ptr = &tsMinIntervalTime; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 1; - cfg.maxValue = 1000000; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_MS; - taosAddConfigOption(cfg); - - cfg.option = "maxStreamCompDelay"; - cfg.ptr = &tsMaxStreamComputDelay; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 10; - cfg.maxValue = 1000000000; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_MS; - taosAddConfigOption(cfg); - - cfg.option = "maxFirstStreamCompDelay"; - cfg.ptr = &tsStreamCompStartDelay; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 1000; - cfg.maxValue = 1000000000; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_MS; - taosAddConfigOption(cfg); - - cfg.option = "retryStreamCompDelay"; - cfg.ptr = &tsRetryStreamCompDelay; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 10; - cfg.maxValue = 1000000000; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_MS; - - taosAddConfigOption(cfg); - cfg.option = "streamCompDelayRatio"; - cfg.ptr = &tsStreamComputDelayRatio; - cfg.valType = TAOS_CFG_VTYPE_FLOAT; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 0.1f; - cfg.maxValue = 0.9f; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "compressMsgSize"; - cfg.ptr = &tsCompressMsgSize; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = -1; - cfg.maxValue = 100000000.0f; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "compressColData"; - cfg.ptr = &tsCompressColData; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = -1; - cfg.maxValue = 100000000.0f; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "maxWildCardsLength"; - cfg.ptr = &tsMaxWildCardsLen; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 0; - cfg.maxValue = TSDB_MAX_FIELD_LEN; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_BYTE; - taosAddConfigOption(cfg); - - cfg.option = "maxRegexStringLen"; - cfg.ptr = &tsMaxRegexStringLen; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 0; - cfg.maxValue = TSDB_MAX_FIELD_LEN; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_BYTE; - taosAddConfigOption(cfg); - - cfg.option = "maxNumOfOrderedRes"; - cfg.ptr = &tsMaxNumOfOrderedResults; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = TSDB_MAX_SQL_LEN; - cfg.maxValue = TSDB_MAX_ALLOWED_SQL_LEN; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "queryBufferSize"; - cfg.ptr = &tsQueryBufferSize; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = -1; - cfg.maxValue = 500000000000.0f; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_BYTE; - taosAddConfigOption(cfg); - - cfg.option = "retrieveBlockingModel"; - cfg.ptr = &tsRetrieveBlockingModel; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 0; - cfg.maxValue = 1; - cfg.ptrLength = 1; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "keepColumnName"; - cfg.ptr = &tsKeepOriginalColumnName; - cfg.valType = TAOS_CFG_VTYPE_INT8; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 1; - cfg.ptrLength = 1; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - // locale & charset - cfg.option = "timezone"; - cfg.ptr = tsTimezone; - cfg.valType = TAOS_CFG_VTYPE_STRING; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 0; - cfg.ptrLength = TSDB_TIMEZONE_LEN; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "locale"; - cfg.ptr = tsLocale; - cfg.valType = TAOS_CFG_VTYPE_STRING; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 0; - cfg.ptrLength = TSDB_LOCALE_LEN; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "charset"; - cfg.ptr = tsCharset; - cfg.valType = TAOS_CFG_VTYPE_STRING; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 0; - cfg.ptrLength = TSDB_LOCALE_LEN; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - // connect configs - cfg.option = "maxShellConns"; - cfg.ptr = &tsMaxShellConns; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 10; - cfg.maxValue = 50000000; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "maxConnections"; - cfg.ptr = &tsMaxConnections; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 1; - cfg.maxValue = 100000; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "minimalLogDirGB"; - cfg.ptr = &tsMinimalLogDirGB; - cfg.valType = TAOS_CFG_VTYPE_FLOAT; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 0.001f; - cfg.maxValue = 10000000; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_GB; - taosAddConfigOption(cfg); - - cfg.option = "minimalTmpDirGB"; - cfg.ptr = &tsReservedTmpDirectorySpace; - cfg.valType = TAOS_CFG_VTYPE_FLOAT; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 0.001f; - cfg.maxValue = 10000000; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_GB; - taosAddConfigOption(cfg); - - cfg.option = "minimalDataDirGB"; - cfg.ptr = &tsMinimalDataDirGB; - cfg.valType = TAOS_CFG_VTYPE_FLOAT; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 0.001f; - cfg.maxValue = 10000000; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_GB; - taosAddConfigOption(cfg); - - cfg.option = "slaveQuery"; - cfg.ptr = &tsEnableSlaveQuery; - cfg.valType = TAOS_CFG_VTYPE_INT8; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 0; - cfg.maxValue = 1; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - // debug flag - cfg.option = "numOfLogLines"; - cfg.ptr = &tsNumOfLogLines; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 1000; - cfg.maxValue = 2000000000; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "logKeepDays"; - cfg.ptr = &tsLogKeepDays; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = -365000; - cfg.maxValue = 365000; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "asyncLog"; - cfg.ptr = &tsAsyncLog; - cfg.valType = TAOS_CFG_VTYPE_INT8; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 1; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "debugFlag"; - cfg.ptr = &debugFlag; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 255; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "mDebugFlag"; - cfg.ptr = &mDebugFlag; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG; - cfg.minValue = 0; - cfg.maxValue = 255; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "dDebugFlag"; - cfg.ptr = &dDebugFlag; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG; - cfg.minValue = 0; - cfg.maxValue = 255; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "sDebugFlag"; - cfg.ptr = &sDebugFlag; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG; - cfg.minValue = 0; - cfg.maxValue = 255; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "wDebugFlag"; - cfg.ptr = &wDebugFlag; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG; - cfg.minValue = 0; - cfg.maxValue = 255; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "rpcDebugFlag"; - cfg.ptr = &rpcDebugFlag; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 255; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "tmrDebugFlag"; - cfg.ptr = &tmrDebugFlag; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 255; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "cDebugFlag"; - cfg.ptr = &cDebugFlag; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 255; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "jniDebugFlag"; - cfg.ptr = &jniDebugFlag; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 255; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "uDebugFlag"; - cfg.ptr = &uDebugFlag; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 255; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "qDebugFlag"; - cfg.ptr = &qDebugFlag; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 255; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "vDebugFlag"; - cfg.ptr = &vDebugFlag; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG; - cfg.minValue = 0; - cfg.maxValue = 255; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "tsdbDebugFlag"; - cfg.ptr = &tsdbDebugFlag; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG; - cfg.minValue = 0; - cfg.maxValue = 255; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "cqDebugFlag"; - cfg.ptr = &cqDebugFlag; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG; - cfg.minValue = 0; - cfg.maxValue = 255; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "enableRecordSql"; - cfg.ptr = &tsTscEnableRecordSql; - cfg.valType = TAOS_CFG_VTYPE_INT8; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; - cfg.minValue = 0; - cfg.maxValue = 1; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "enableCoreFile"; - cfg.ptr = &tsEnableCoreFile; - cfg.valType = TAOS_CFG_VTYPE_INT8; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; - cfg.minValue = 0; - cfg.maxValue = 1; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - // version info - cfg.option = "gitinfo"; - cfg.ptr = gitinfo; - cfg.valType = TAOS_CFG_VTYPE_STRING; - cfg.cfgType = TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 0; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "gitinfoOfInternal"; - cfg.ptr = gitinfoOfInternal; - cfg.valType = TAOS_CFG_VTYPE_STRING; - cfg.cfgType = TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 0; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "buildinfo"; - cfg.ptr = buildinfo; - cfg.valType = TAOS_CFG_VTYPE_STRING; - cfg.cfgType = TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 0; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "version"; - cfg.ptr = version; - cfg.valType = TAOS_CFG_VTYPE_STRING; - cfg.cfgType = TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 0; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "maxBinaryDisplayWidth"; - cfg.ptr = &tsMaxBinaryDisplayWidth; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 1; - cfg.maxValue = 65536; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "tempDir"; - cfg.ptr = tsTempDir; - cfg.valType = TAOS_CFG_VTYPE_STRING; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 0; - cfg.ptrLength = PATH_MAX; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - // enable kill long query - cfg.option = "deadLockKillQuery"; - cfg.ptr = &tsDeadLockKillQuery; - cfg.valType = TAOS_CFG_VTYPE_INT8; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 0; - cfg.maxValue = 1; - cfg.ptrLength = 1; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - -#ifdef TD_TSZ - // lossy compress - cfg.option = "lossyColumns"; - cfg.ptr = lossyColumns; - cfg.valType = TAOS_CFG_VTYPE_STRING; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; - cfg.minValue = 0; - cfg.maxValue = 0; - cfg.ptrLength = tListLen(lossyColumns); - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "fPrecision"; - cfg.ptr = &fPrecision; - cfg.valType = TAOS_CFG_VTYPE_DOUBLE; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; - cfg.minValue = MIN_FLOAT; - cfg.maxValue = MAX_FLOAT; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - - taosAddConfigOption(cfg); - - cfg.option = "dPrecision"; - cfg.ptr = &dPrecision; - cfg.valType = TAOS_CFG_VTYPE_DOUBLE; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; - cfg.minValue = MIN_FLOAT; - cfg.maxValue = MAX_FLOAT; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "maxRange"; - cfg.ptr = &maxRange; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; - cfg.minValue = 0; - cfg.maxValue = 65536; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "range"; - cfg.ptr = &curRange; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; - cfg.minValue = 0; - cfg.maxValue = 65536; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - assert(tsGlobalConfigNum == TSDB_CFG_MAX_NUM); -#else - // assert(tsGlobalConfigNum == TSDB_CFG_MAX_NUM - 5); -#endif +struct SConfig *taosGetCfg() { + return tsCfg; } -void taosInitGlobalCfg() { pthread_once(&tsInitGlobalCfgOnce, doInitGlobalConfig); } +static int32_t taosLoadCfg(SConfig *pCfg, const char *inputCfgDir, const char *envFile, const char *apolloUrl) { + char cfgDir[PATH_MAX] = {0}; + char cfgFile[PATH_MAX + 100] = {0}; -int32_t taosCheckAndPrintCfg() { - SEp ep = {0}; - if (debugFlag & DEBUG_TRACE || debugFlag & DEBUG_DEBUG || debugFlag & DEBUG_DUMP) { - taosSetAllDebugFlag(); - } + taosExpandDir(inputCfgDir, cfgDir, PATH_MAX); + snprintf(cfgFile, sizeof(cfgFile), "%s" TD_DIRSEP "taos.cfg", cfgDir); - if (tsLocalFqdn[0] == 0) { - taosGetFqdn(tsLocalFqdn); - } - - snprintf(tsLocalEp, sizeof(tsLocalEp), "%s:%u", tsLocalFqdn, tsServerPort); - uInfo("localEp is: %s", tsLocalEp); - - if (tsFirst[0] == 0) { - strcpy(tsFirst, tsLocalEp); - } else { - taosGetFqdnPortFromEp(tsFirst, &ep); - snprintf(tsFirst, sizeof(tsFirst), "%s:%u", ep.fqdn, ep.port); - } - - if (tsSecond[0] == 0) { - strcpy(tsSecond, tsLocalEp); - } else { - taosGetFqdnPortFromEp(tsSecond, &ep); - snprintf(tsSecond, sizeof(tsSecond), "%s:%u", ep.fqdn, ep.port); - } - - taosCheckDataDirCfg(); - - if (taosDirExist(tsTempDir) != 0) { + if (cfgLoad(pCfg, CFG_STYPE_APOLLO_URL, apolloUrl) != 0) { + uError("failed to load from apollo url:%s since %s\n", apolloUrl, terrstr()); return -1; } - taosGetSystemInfo(); - - tsSetLocale(); - - SGlobalCfg *cfg_timezone = taosGetConfigOption("timezone"); - if (cfg_timezone && cfg_timezone->cfgStatus == TAOS_CFG_CSTATUS_FILE) { - tsSetTimeZone(); + if (cfgLoad(pCfg, CFG_STYPE_CFG_FILE, cfgFile) != 0) { + if (cfgLoad(pCfg, CFG_STYPE_CFG_FILE, cfgDir) != 0) { + uError("failed to load from config file:%s since %s\n", cfgFile, terrstr()); + return -1; + } } - if (tsNumOfCores <= 0) { - tsNumOfCores = 1; + if (cfgLoad(pCfg, CFG_STYPE_ENV_FILE, envFile) != 0) { + uError("failed to load from env file:%s since %s\n", envFile, terrstr()); + return -1; } - if (tsQueryBufferSize >= 0) { - tsQueryBufferSizeBytes = tsQueryBufferSize * 1048576UL; + if (cfgLoad(pCfg, CFG_STYPE_ENV_VAR, NULL) != 0) { + uError("failed to load from global env variables since %s\n", terrstr()); + return -1; } - uInfo(" check global cfg completed"); - uInfo("=================================="); - taosPrintCfg(); - return 0; } -/* - * alter dnode 1 balance "vnode:1-dnode:2" - */ - -bool taosCheckBalanceCfgOptions(const char *option, int32_t *vnodeId, int32_t *dnodeId) { - int len = (int)strlen(option); - if (strncasecmp(option, "vnode:", 6) != 0) { - return false; - } - - int pos = 0; - for (; pos < len; ++pos) { - if (option[pos] == '-') break; - } - - if (++pos >= len) return false; - if (strncasecmp(option + pos, "dnode:", 6) != 0) { - return false; - } - - *vnodeId = strtol(option + 6, NULL, 10); - *dnodeId = strtol(option + pos + 6, NULL, 10); - if (*vnodeId <= 1 || *dnodeId <= 0) { - return false; - } - - return true; +static int32_t taosAddClientLogCfg(SConfig *pCfg) { + if (cfgAddDir(pCfg, "logDir", tsLogDir, 1) != 0) return -1; + if (cfgAddFloat(pCfg, "minimalLogDirGB", 1.0f, 0.001f, 10000000, 1) != 0) return -1; + if (cfgAddInt32(pCfg, "numOfLogLines", tsNumOfLogLines, 1000, 2000000000, 1) != 0) return -1; + if (cfgAddBool(pCfg, "asyncLog", tsAsyncLog, 1) != 0) return -1; + if (cfgAddInt32(pCfg, "logKeepDays", 0, -365000, 365000, 1) != 0) return -1; + if (cfgAddInt32(pCfg, "cDebugFlag", cDebugFlag, 0, 255, 1) != 0) return -1; + if (cfgAddInt32(pCfg, "uDebugFlag", uDebugFlag, 0, 255, 1) != 0) return -1; + if (cfgAddInt32(pCfg, "rpcDebugFlag", rpcDebugFlag, 0, 255, 1) != 0) return -1; + if (cfgAddInt32(pCfg, "tmrDebugFlag", tmrDebugFlag, 0, 255, 1) != 0) return -1; + if (cfgAddInt32(pCfg, "jniDebugFlag", jniDebugFlag, 0, 255, 1) != 0) return -1; + if (cfgAddInt32(pCfg, "simDebugFlag", 143, 0, 255, 1) != 0) return -1; + if (cfgAddDir(pCfg, "configDir", configDir, 1) != 0) return -1; + if (cfgAddDir(pCfg, "scriptDir", configDir, 1) != 0) return -1; + if (cfgAddInt32(pCfg, "debugFlag", 0, 0, 255, 1) != 0) return -1; + return 0; } + +static int32_t taosAddServerLogCfg(SConfig *pCfg) { + if (cfgAddInt32(pCfg, "dDebugFlag", dDebugFlag, 0, 255, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "vDebugFlag", vDebugFlag, 0, 255, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "mDebugFlag", mDebugFlag, 0, 255, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "qDebugFlag", qDebugFlag, 0, 255, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "wDebugFlag", wDebugFlag, 0, 255, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "sDebugFlag", sDebugFlag, 0, 255, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "tsdbDebugFlag", tsdbDebugFlag, 0, 255, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "tqDebugFlag", tqDebugFlag, 0, 255, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "fsDebugFlag", fsDebugFlag, 0, 255, 0) != 0) return -1; + return 0; +} + +static int32_t taosAddClientCfg(SConfig *pCfg) { + char defaultFqdn[TSDB_FQDN_LEN] = {0}; + int32_t defaultServerPort = 6030; + char defaultFirstEp[TSDB_EP_LEN] = {0}; + char defaultSecondEp[TSDB_EP_LEN] = {0}; + + if (taosGetFqdn(defaultFqdn) != 0) return -1; + snprintf(defaultFirstEp, TSDB_EP_LEN, "%s:%d", defaultFqdn, defaultServerPort); + snprintf(defaultSecondEp, TSDB_EP_LEN, "%s:%d", defaultFqdn, defaultServerPort); + + if (cfgAddString(pCfg, "firstEp", defaultFirstEp, 1) != 0) return -1; + if (cfgAddString(pCfg, "secondEp", defaultSecondEp, 1) != 0) return -1; + if (cfgAddString(pCfg, "fqdn", defaultFqdn, 1) != 0) return -1; + if (cfgAddInt32(pCfg, "serverPort", defaultServerPort, 1, 65056, 1) != 0) return -1; + if (cfgAddDir(pCfg, "tempDir", tsTempDir, 1) != 0) return -1; + if (cfgAddFloat(pCfg, "minimalTempDirGB", 1.0f, 0.001f, 10000000, 1) != 0) return -1; + if (cfgAddFloat(pCfg, "numOfThreadsPerCore", tsNumOfThreadsPerCore, 0, 10, 1) != 0) return -1; + if (cfgAddInt32(pCfg, "maxTmrCtrl", tsMaxTmrCtrl, 8, 2048, 1) != 0) return -1; + if (cfgAddInt32(pCfg, "rpcTimer", tsRpcTimer, 100, 3000, 1) != 0) return -1; + if (cfgAddInt32(pCfg, "rpcMaxTime", tsRpcMaxTime, 100, 7200, 1) != 0) return -1; + if (cfgAddBool(pCfg, "rpcForceTcp", tsRpcForceTcp, 1) != 0) return -1; + if (cfgAddInt32(pCfg, "shellActivityTimer", tsShellActivityTimer, 1, 120, 1) != 0) return -1; + if (cfgAddInt32(pCfg, "compressMsgSize", tsCompressMsgSize, -1, 100000000, 1) != 0) return -1; + if (cfgAddInt32(pCfg, "compressColData", tsCompressColData, -1, 100000000, 1) != 0) return -1; + if (cfgAddInt32(pCfg, "maxWildCardsLength", tsMaxWildCardsLen, 0, TSDB_MAX_FIELD_LEN, 1) != 0) return -1; + if (cfgAddInt32(pCfg, "maxRegexStringLen", tsMaxRegexStringLen, 0, TSDB_MAX_FIELD_LEN, 1) != 0) return -1; + if (cfgAddInt32(pCfg, "maxNumOfOrderedRes", tsMaxNumOfOrderedResults, 128, TSDB_MAX_ALLOWED_SQL_LEN, 1) != 0) + return -1; + if (cfgAddBool(pCfg, "keepColumnName", tsKeepOriginalColumnName, 1) != 0) return -1; + if (cfgAddInt32(pCfg, "maxBinaryDisplayWidth", tsMaxBinaryDisplayWidth, 1, 65536, 1) != 0) return -1; + return 0; +} + +static int32_t taosAddSystemCfg(SConfig *pCfg) { + SysNameInfo info = taosGetSysNameInfo(); + + if (cfgAddTimezone(pCfg, "timezone", tsTimezone) != 0) return -1; + if (cfgAddLocale(pCfg, "locale", tsLocale) != 0) return -1; + if (cfgAddCharset(pCfg, "charset", tsCharset) != 0) return -1; + if (cfgAddBool(pCfg, "enableCoreFile", 0, 1) != 0) return -1; + if (cfgAddInt32(pCfg, "numOfCores", tsNumOfCores, 1, 100000, 1) != 0) return -1; + if (cfgAddInt32(pCfg, "pageSize(KB)", tsPageSize, 0, INT64_MAX, 1) != 0) return -1; + if (cfgAddInt64(pCfg, "openMax", tsOpenMax, 0, INT64_MAX, 1) != 0) return -1; + if (cfgAddInt64(pCfg, "streamMax", tsStreamMax, 0, INT64_MAX, 1) != 0) return -1; + if (cfgAddInt32(pCfg, "totalMemory(MB)", tsTotalMemoryMB, 0, INT32_MAX, 1) != 0) return -1; + if (cfgAddString(pCfg, "os sysname", info.sysname, 1) != 0) return -1; + if (cfgAddString(pCfg, "os nodename", info.nodename, 1) != 0) return -1; + if (cfgAddString(pCfg, "os release", info.release, 1) != 0) return -1; + if (cfgAddString(pCfg, "os version", info.version, 1) != 0) return -1; + if (cfgAddString(pCfg, "os machine", info.machine, 1) != 0) return -1; + if (cfgAddString(pCfg, "os sysname", info.sysname, 1) != 0) return -1; + + if (cfgAddString(pCfg, "version", version, 1) != 0) return -1; + if (cfgAddString(pCfg, "compatible_version", compatible_version, 1) != 0) return -1; + if (cfgAddString(pCfg, "gitinfo", gitinfo, 1) != 0) return -1; + if (cfgAddString(pCfg, "gitinfoOfInternal", gitinfoOfInternal, 1) != 0) return -1; + if (cfgAddString(pCfg, "buildinfo", buildinfo, 1) != 0) return -1; + return 0; +} + +static int32_t taosAddServerCfg(SConfig *pCfg) { + if (cfgAddInt32(pCfg, "supportVnodes", 256, 0, 65536, 0) != 0) return -1; + if (cfgAddDir(pCfg, "dataDir", tsDataDir, 0) != 0) return -1; + if (cfgAddFloat(pCfg, "minimalDataDirGB", 2.0f, 0.001f, 10000000, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "numOfCommitThreads", tsNumOfCommitThreads, 1, 100, 0) != 0) return -1; + if (cfgAddFloat(pCfg, "ratioOfQueryCores", tsRatioOfQueryCores, 0, 2, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "maxNumOfDistinctRes", tsMaxNumOfDistinctResults, 10 * 10000, 10000 * 10000, 0) != 0) return -1; + if (cfgAddBool(pCfg, "telemetryReporting", tsEnableTelemetryReporting, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "maxConnections", tsMaxConnections, 1, 100000, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "maxShellConns", tsMaxShellConns, 10, 50000000, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "statusInterval", tsStatusInterval, 1, 30, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "minSlidingTime", tsMinSlidingTime, 10, 1000000, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "minIntervalTime", tsMinIntervalTime, 1, 1000000, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "maxStreamCompDelay", tsMaxStreamComputDelay, 10, 1000000000, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "maxFirstStreamCompDelay", tsStreamCompStartDelay, 1000, 1000000000, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "retryStreamCompDelay", tsRetryStreamCompDelay, 10, 1000000000, 0) != 0) return -1; + if (cfgAddFloat(pCfg, "streamCompDelayRatio", tsStreamComputDelayRatio, 0.1, 0.9, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "queryBufferSize", tsQueryBufferSize, -1, 500000000000, 0) != 0) return -1; + if (cfgAddBool(pCfg, "retrieveBlockingModel", tsRetrieveBlockingModel, 0) != 0) return -1; + if (cfgAddBool(pCfg, "printAuth", tsPrintAuth, 0) != 0) return -1; + if (cfgAddBool(pCfg, "slaveQuery", tsEnableSlaveQuery, 0) != 0) return -1; + if (cfgAddBool(pCfg, "deadLockKillQuery", tsDeadLockKillQuery, 0) != 0) return -1; + return 0; +} + +static void taosSetClientLogCfg(SConfig *pCfg) { + SConfigItem *pItem = cfgGetItem(pCfg, "logDir"); + tstrncpy(tsLogDir, cfgGetItem(pCfg, "logDir")->str, PATH_MAX); + tsLogSpace.reserved = cfgGetItem(pCfg, "minimalLogDirGB")->fval; + tsNumOfLogLines = cfgGetItem(pCfg, "numOfLogLines")->i32; + tsAsyncLog = cfgGetItem(pCfg, "asyncLog")->bval; + tsLogKeepDays = cfgGetItem(pCfg, "logKeepDays")->i32; + cDebugFlag = cfgGetItem(pCfg, "cDebugFlag")->i32; + uDebugFlag = cfgGetItem(pCfg, "uDebugFlag")->i32; + rpcDebugFlag = cfgGetItem(pCfg, "rpcDebugFlag")->i32; + tmrDebugFlag = cfgGetItem(pCfg, "tmrDebugFlag")->i32; + jniDebugFlag = cfgGetItem(pCfg, "jniDebugFlag")->i32; +} + +static void taosSetServerLogCfg(SConfig *pCfg) { + dDebugFlag = cfgGetItem(pCfg, "dDebugFlag")->i32; + vDebugFlag = cfgGetItem(pCfg, "vDebugFlag")->i32; + mDebugFlag = cfgGetItem(pCfg, "mDebugFlag")->i32; + qDebugFlag = cfgGetItem(pCfg, "qDebugFlag")->i32; + wDebugFlag = cfgGetItem(pCfg, "wDebugFlag")->i32; + sDebugFlag = cfgGetItem(pCfg, "sDebugFlag")->i32; + tsdbDebugFlag = cfgGetItem(pCfg, "tsdbDebugFlag")->i32; + tqDebugFlag = cfgGetItem(pCfg, "tqDebugFlag")->i32; + fsDebugFlag = cfgGetItem(pCfg, "fsDebugFlag")->i32; +} + +static void taosSetClientCfg(SConfig *pCfg) { + tstrncpy(tsFirst, cfgGetItem(pCfg, "firstEp")->str, TSDB_EP_LEN); + tstrncpy(tsSecond, cfgGetItem(pCfg, "secondEp")->str, TSDB_EP_LEN); + tstrncpy(tsLocalFqdn, cfgGetItem(pCfg, "fqdn")->str, TSDB_EP_LEN); + tsServerPort = (uint16_t)cfgGetItem(pCfg, "serverPort")->i32; + snprintf(tsLocalEp, sizeof(tsLocalEp), "%s:%u", tsLocalFqdn, tsServerPort); + tstrncpy(tsLogDir, cfgGetItem(pCfg, "tempDir")->str, PATH_MAX); + tsTempSpace.reserved = cfgGetItem(pCfg, "minimalTempDirGB")->fval; + + tsNumOfThreadsPerCore = cfgGetItem(pCfg, "maxTmrCtrl")->fval; + tsMaxTmrCtrl = cfgGetItem(pCfg, "maxTmrCtrl")->i32; + tsRpcTimer = cfgGetItem(pCfg, "rpcTimer")->i32; + tsRpcMaxTime = cfgGetItem(pCfg, "rpcMaxTime")->i32; + tsRpcForceTcp = cfgGetItem(pCfg, "rpcForceTcp")->i32; + tsShellActivityTimer = cfgGetItem(pCfg, "shellActivityTimer")->bval; + tsCompressMsgSize = cfgGetItem(pCfg, "compressMsgSize")->i32; + tsCompressColData = cfgGetItem(pCfg, "compressColData")->i32; + tsMaxWildCardsLen = cfgGetItem(pCfg, "maxWildCardsLength")->i32; + tsMaxRegexStringLen = cfgGetItem(pCfg, "maxRegexStringLen")->i32; + tsMaxNumOfOrderedResults = cfgGetItem(pCfg, "maxNumOfOrderedRes")->i32; + tsKeepOriginalColumnName = cfgGetItem(pCfg, "keepColumnName")->bval; + tsMaxBinaryDisplayWidth = cfgGetItem(pCfg, "maxBinaryDisplayWidth")->i32; +} + +static void taosSetSystemCfg(SConfig *pCfg) { + SConfigItem *pItem = cfgGetItem(pCfg, "timezone"); + osSetTimezone(pItem->str); + uDebug("timezone format changed from %s to %s", pItem->str, tsTimezone); + cfgSetItem(pCfg, "timezone", tsTimezone, pItem->stype); + + const char *locale = cfgGetItem(pCfg, "locale")->str; + const char *charset = cfgGetItem(pCfg, "charset")->str; + taosSetSystemLocale(locale, charset); + + if (tsNumOfCores <= 1) { + tsNumOfCores = 2; + } + + bool enableCore = cfgGetItem(pCfg, "enableCoreFile")->bval; + taosSetConsoleEcho(enableCore); + + // todo + tsVersion = 30000000; +} + +static void taosSetServerCfg(SConfig *pCfg) { + tstrncpy(tsDataDir, cfgGetItem(pCfg, "dataDir")->str, PATH_MAX); + tsTempSpace.reserved = cfgGetItem(pCfg, "minimalDataDirGB")->fval; + tsNumOfCommitThreads = cfgGetItem(pCfg, "numOfCommitThreads")->i32; + tsRatioOfQueryCores = cfgGetItem(pCfg, "ratioOfQueryCores")->fval; + tsMaxNumOfDistinctResults = cfgGetItem(pCfg, "maxNumOfDistinctRes")->i32; + tsEnableTelemetryReporting = cfgGetItem(pCfg, "telemetryReporting")->bval; + tsMaxConnections = cfgGetItem(pCfg, "maxConnections")->i32; + tsMaxShellConns = cfgGetItem(pCfg, "maxShellConns")->i32; + tsStatusInterval = cfgGetItem(pCfg, "statusInterval")->i32; + tsMinSlidingTime = cfgGetItem(pCfg, "minSlidingTime")->i32; + tsMinIntervalTime = cfgGetItem(pCfg, "minIntervalTime")->i32; + tsMaxStreamComputDelay = cfgGetItem(pCfg, "maxStreamCompDelay")->i32; + tsStreamCompStartDelay = cfgGetItem(pCfg, "maxFirstStreamCompDelay")->i32; + tsRetryStreamCompDelay = cfgGetItem(pCfg, "retryStreamCompDelay")->i32; + tsStreamComputDelayRatio = cfgGetItem(pCfg, "streamCompDelayRatio")->fval; + tsQueryBufferSize = cfgGetItem(pCfg, "queryBufferSize")->i32; + tsRetrieveBlockingModel = cfgGetItem(pCfg, "retrieveBlockingModel")->bval; + tsPrintAuth = cfgGetItem(pCfg, "printAuth")->bval; + tsEnableSlaveQuery = cfgGetItem(pCfg, "slaveQuery")->bval; + tsDeadLockKillQuery = cfgGetItem(pCfg, "deadLockKillQuery")->bval; + + if (tsQueryBufferSize >= 0) { + tsQueryBufferSizeBytes = tsQueryBufferSize * 1048576UL; + } +} + +int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDir, const char *envFile, + const char *apolloUrl, SArray *pArgs, bool tsc) { + osInit(); + + SConfig *pCfg = cfgInit(); + if (pCfg == NULL) return -1; + + if (tsc) { + if (taosAddClientLogCfg(pCfg) != 0) return -1; + } else { + if (taosAddClientLogCfg(pCfg) != 0) return -1; + if (taosAddServerLogCfg(pCfg) != 0) return -1; + } + + if (taosLoadCfg(pCfg, cfgDir, envFile, apolloUrl) != 0) { + uError("failed to load cfg since %s", terrstr()); + cfgCleanup(pCfg); + return -1; + } + + if (cfgLoadArray(pCfg, pArgs) != 0) { + uError("failed to load cfg from array since %s", terrstr()); + cfgCleanup(pCfg); + return -1; + } + + if (tsc) { + taosSetClientLogCfg(pCfg); + } else { + taosSetClientLogCfg(pCfg); + taosSetServerLogCfg(pCfg); + } + + taosSetAllDebugFlag(cfgGetItem(pCfg, "debugFlag")->i32); + + if (taosInitLog(logname, logFileNum) != 0) { + printf("failed to init log file since %s\n", terrstr()); + cfgCleanup(pCfg); + return -1; + } + + cfgCleanup(pCfg); + return 0; +} + +int32_t taosInitCfg(const char *cfgDir, const char *envFile, const char *apolloUrl, SArray *pArgs, bool tsc) { + if (tsCfg != NULL) return 0; + tsCfg = cfgInit(); + + if (tsc) { + if (taosAddClientLogCfg(tsCfg) != 0) return -1; + if (taosAddClientCfg(tsCfg) != 0) return -1; + } else { + if (taosAddClientLogCfg(tsCfg) != 0) return -1; + if (taosAddServerLogCfg(tsCfg) != 0) return -1; + if (taosAddClientCfg(tsCfg) != 0) return -1; + if (taosAddServerCfg(tsCfg) != 0) return -1; + } + taosAddSystemCfg(tsCfg); + + if (taosLoadCfg(tsCfg, cfgDir, envFile, apolloUrl) != 0) { + uError("failed to load cfg since %s", terrstr()); + cfgCleanup(tsCfg); + tsCfg = NULL; + return -1; + } + + if (cfgLoadArray(tsCfg, pArgs) != 0) { + uError("failed to load cfg from array since %s", terrstr()); + cfgCleanup(tsCfg); + return -1; + } + + if (tsc) { + taosSetClientCfg(tsCfg); + } else { + taosSetClientCfg(tsCfg); + taosSetServerCfg(tsCfg); + taosSetTfsCfg(tsCfg); + } + taosSetSystemCfg(tsCfg); + + cfgDumpCfg(tsCfg, tsc, false); + return 0; +} + +void taosCleanupCfg() { + if (tsCfg) { + cfgCleanup(tsCfg); + tsCfg = NULL; + } +} + +void taosCfgDynamicOptions(const char *option, const char *value) { + if (strcasecmp(option, "debugFlag") == 0) { + int32_t debugFlag = atoi(value); + taosSetAllDebugFlag(debugFlag); + } + + if (strcasecmp(option, "resetlog") == 0) { + taosResetLog(); + cfgDumpCfg(tsCfg, 1, false); + } +} \ No newline at end of file diff --git a/source/common/src/tlocale.c b/source/common/src/tlocale.c deleted file mode 100644 index a98a46b28a..0000000000 --- a/source/common/src/tlocale.c +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "ulog.h" -#include "tglobal.h" -#include "tconfig.h" -#include "tutil.h" - -/** - * In some Linux systems, setLocale(LC_CTYPE, "") may return NULL, in which case the launch of - * both the TDengine Server and the Client may be interrupted. - * - * In case that the setLocale failed to be executed, the right charset needs to be set. - */ -void tsSetLocale() { - char *locale = setlocale(LC_CTYPE, tsLocale); - - // default locale or user specified locale is not valid, abort launch - if (locale == NULL) { - uError("Invalid locale:%s, please set the valid locale in config file", tsLocale); - } - - if (strlen(tsCharset) == 0) { - uError("failed to get charset, please set the valid charset in config file"); - exit(-1); - } - - if (!taosValidateEncodec(tsCharset)) { - uError("Invalid charset:%s, please set the valid charset in config file", tsCharset); - exit(-1); - } -} \ No newline at end of file diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 46feab7791..8101c18ebf 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -285,7 +285,7 @@ int32_t tDeserializeSClientHbBatchRsp(void *buf, int32_t bufLen, SClientHbBatchR int32_t tSerializeSVCreateTbReq(void **buf, SVCreateTbReq *pReq) { int32_t tlen = 0; - tlen += taosEncodeFixedU64(buf, pReq->ver); + tlen += taosEncodeFixedI64(buf, pReq->ver); tlen += taosEncodeString(buf, pReq->name); tlen += taosEncodeFixedU32(buf, pReq->ttl); tlen += taosEncodeFixedU32(buf, pReq->keep); @@ -330,7 +330,7 @@ int32_t tSerializeSVCreateTbReq(void **buf, SVCreateTbReq *pReq) { } void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) { - buf = taosDecodeFixedU64(buf, &(pReq->ver)); + buf = taosDecodeFixedI64(buf, &(pReq->ver)); buf = taosDecodeString(buf, &(pReq->name)); buf = taosDecodeFixedU32(buf, &(pReq->ttl)); buf = taosDecodeFixedU32(buf, &(pReq->keep)); @@ -380,7 +380,7 @@ void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) { int32_t tSerializeSVCreateTbBatchReq(void **buf, SVCreateTbBatchReq *pReq) { int32_t tlen = 0; - tlen += taosEncodeFixedU64(buf, pReq->ver); + tlen += taosEncodeFixedI64(buf, pReq->ver); tlen += taosEncodeFixedU32(buf, taosArrayGetSize(pReq->pArray)); for (size_t i = 0; i < taosArrayGetSize(pReq->pArray); i++) { SVCreateTbReq *pCreateTbReq = taosArrayGet(pReq->pArray, i); @@ -393,7 +393,7 @@ int32_t tSerializeSVCreateTbBatchReq(void **buf, SVCreateTbBatchReq *pReq) { void *tDeserializeSVCreateTbBatchReq(void *buf, SVCreateTbBatchReq *pReq) { uint32_t nsize = 0; - buf = taosDecodeFixedU64(buf, &pReq->ver); + buf = taosDecodeFixedI64(buf, &pReq->ver); buf = taosDecodeFixedU32(buf, &nsize); pReq->pArray = taosArrayInit(nsize, sizeof(SVCreateTbReq)); for (size_t i = 0; i < nsize; i++) { @@ -407,14 +407,14 @@ void *tDeserializeSVCreateTbBatchReq(void *buf, SVCreateTbBatchReq *pReq) { int32_t tSerializeSVDropTbReq(void **buf, SVDropTbReq *pReq) { int32_t tlen = 0; - tlen += taosEncodeFixedU64(buf, pReq->ver); + tlen += taosEncodeFixedI64(buf, pReq->ver); tlen += taosEncodeString(buf, pReq->name); tlen += taosEncodeFixedU8(buf, pReq->type); return tlen; } void *tDeserializeSVDropTbReq(void *buf, SVDropTbReq *pReq) { - buf = taosDecodeFixedU64(buf, &pReq->ver); + buf = taosDecodeFixedI64(buf, &pReq->ver); buf = taosDecodeString(buf, &pReq->name); buf = taosDecodeFixedU8(buf, &pReq->type); return buf; @@ -1393,7 +1393,7 @@ int32_t tSerializeSDropDbRsp(void *buf, int32_t bufLen, SDropDbRsp *pRsp) { if (tStartEncode(&encoder) < 0) return -1; if (tEncodeCStr(&encoder, pRsp->db) < 0) return -1; - if (tEncodeU64(&encoder, pRsp->uid) < 0) return -1; + if (tEncodeI64(&encoder, pRsp->uid) < 0) return -1; tEndEncode(&encoder); int32_t tlen = encoder.pos; @@ -1407,7 +1407,7 @@ int32_t tDeserializeSDropDbRsp(void *buf, int32_t bufLen, SDropDbRsp *pRsp) { if (tStartDecode(&decoder) < 0) return -1; if (tDecodeCStrTo(&decoder, pRsp->db) < 0) return -1; - if (tDecodeU64(&decoder, &pRsp->uid) < 0) return -1; + if (tDecodeI64(&decoder, &pRsp->uid) < 0) return -1; tEndDecode(&decoder); tCoderClear(&decoder); @@ -1468,7 +1468,7 @@ int32_t tDeserializeSSyncDbReq(void *buf, int32_t bufLen, SSyncDbReq *pReq) { static int32_t tSerializeSUseDbRspImp(SCoder *pEncoder, SUseDbRsp *pRsp) { if (tEncodeCStr(pEncoder, pRsp->db) < 0) return -1; - if (tEncodeU64(pEncoder, pRsp->uid) < 0) return -1; + if (tEncodeI64(pEncoder, pRsp->uid) < 0) return -1; if (tEncodeI32(pEncoder, pRsp->vgVersion) < 0) return -1; if (tEncodeI32(pEncoder, pRsp->vgNum) < 0) return -1; if (tEncodeI8(pEncoder, pRsp->hashMethod) < 0) return -1; @@ -1518,7 +1518,7 @@ int32_t tSerializeSUseDbBatchRsp(void *buf, int32_t bufLen, SUseDbBatchRsp *pRsp int32_t tDeserializeSUseDbRspImp(SCoder *pDecoder, SUseDbRsp *pRsp) { if (tDecodeCStrTo(pDecoder, pRsp->db) < 0) return -1; - if (tDecodeU64(pDecoder, &pRsp->uid) < 0) return -1; + if (tDecodeI64(pDecoder, &pRsp->uid) < 0) return -1; if (tDecodeI32(pDecoder, &pRsp->vgVersion) < 0) return -1; if (tDecodeI32(pDecoder, &pRsp->vgNum) < 0) return -1; if (tDecodeI8(pDecoder, &pRsp->hashMethod) < 0) return -1; @@ -1661,7 +1661,7 @@ static int32_t tEncodeSTableMetaRsp(SCoder *pEncoder, STableMetaRsp *pRsp) { if (tEncodeCStr(pEncoder, pRsp->tbName) < 0) return -1; if (tEncodeCStr(pEncoder, pRsp->stbName) < 0) return -1; if (tEncodeCStr(pEncoder, pRsp->dbFName) < 0) return -1; - if (tEncodeU64(pEncoder, pRsp->dbId) < 0) return -1; + if (tEncodeI64(pEncoder, pRsp->dbId) < 0) return -1; if (tEncodeI32(pEncoder, pRsp->numOfTags) < 0) return -1; if (tEncodeI32(pEncoder, pRsp->numOfColumns) < 0) return -1; if (tEncodeI8(pEncoder, pRsp->precision) < 0) return -1; @@ -1684,7 +1684,7 @@ static int32_t tDecodeSTableMetaRsp(SCoder *pDecoder, STableMetaRsp *pRsp) { if (tDecodeCStrTo(pDecoder, pRsp->tbName) < 0) return -1; if (tDecodeCStrTo(pDecoder, pRsp->stbName) < 0) return -1; if (tDecodeCStrTo(pDecoder, pRsp->dbFName) < 0) return -1; - if (tDecodeU64(pDecoder, &pRsp->dbId) < 0) return -1; + if (tDecodeI64(pDecoder, &pRsp->dbId) < 0) return -1; if (tDecodeI32(pDecoder, &pRsp->numOfTags) < 0) return -1; if (tDecodeI32(pDecoder, &pRsp->numOfColumns) < 0) return -1; if (tDecodeI8(pDecoder, &pRsp->precision) < 0) return -1; @@ -2093,7 +2093,7 @@ int32_t tSerializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq *pR if (tEncodeI32(&encoder, pReq->vgId) < 0) return -1; if (tEncodeI32(&encoder, pReq->dnodeId) < 0) return -1; if (tEncodeCStr(&encoder, pReq->db) < 0) return -1; - if (tEncodeU64(&encoder, pReq->dbUid) < 0) return -1; + if (tEncodeI64(&encoder, pReq->dbUid) < 0) return -1; if (tEncodeI32(&encoder, pReq->vgVersion) < 0) return -1; if (tEncodeI32(&encoder, pReq->cacheBlockSize) < 0) return -1; if (tEncodeI32(&encoder, pReq->totalBlocks) < 0) return -1; @@ -2133,7 +2133,7 @@ int32_t tDeserializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq * if (tDecodeI32(&decoder, &pReq->vgId) < 0) return -1; if (tDecodeI32(&decoder, &pReq->dnodeId) < 0) return -1; if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1; - if (tDecodeU64(&decoder, &pReq->dbUid) < 0) return -1; + if (tDecodeI64(&decoder, &pReq->dbUid) < 0) return -1; if (tDecodeI32(&decoder, &pReq->vgVersion) < 0) return -1; if (tDecodeI32(&decoder, &pReq->cacheBlockSize) < 0) return -1; if (tDecodeI32(&decoder, &pReq->totalBlocks) < 0) return -1; @@ -2171,7 +2171,7 @@ int32_t tSerializeSDropVnodeReq(void *buf, int32_t bufLen, SDropVnodeReq *pReq) if (tStartEncode(&encoder) < 0) return -1; if (tEncodeI32(&encoder, pReq->vgId) < 0) return -1; if (tEncodeI32(&encoder, pReq->dnodeId) < 0) return -1; - if (tEncodeU64(&encoder, pReq->dbUid) < 0) return -1; + if (tEncodeI64(&encoder, pReq->dbUid) < 0) return -1; if (tEncodeCStr(&encoder, pReq->db) < 0) return -1; tEndEncode(&encoder); @@ -2187,7 +2187,7 @@ int32_t tDeserializeSDropVnodeReq(void *buf, int32_t bufLen, SDropVnodeReq *pReq if (tStartDecode(&decoder) < 0) return -1; if (tDecodeI32(&decoder, &pReq->vgId) < 0) return -1; if (tDecodeI32(&decoder, &pReq->dnodeId) < 0) return -1; - if (tDecodeU64(&decoder, &pReq->dbUid) < 0) return -1; + if (tDecodeI64(&decoder, &pReq->dbUid) < 0) return -1; if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1; tEndDecode(&decoder); @@ -2356,35 +2356,7 @@ int32_t tDecodeSMqOffset(SCoder *decoder, SMqOffset *pOffset) { return 0; } -int32_t tEncodeSMqVgOffsets(SCoder *encoder, const SMqVgOffsets *pOffsets) { - if (tStartEncode(encoder) < 0) return -1; - if (tEncodeI32(encoder, pOffsets->vgId) < 0) return -1; - int32_t sz = taosArrayGetSize(pOffsets->offsets); - if (tEncodeI32(encoder, sz) < 0) return -1; - for (int32_t i = 0; i < sz; i++) { - SMqOffset *offset = taosArrayGet(pOffsets->offsets, i); - if (tEncodeSMqOffset(encoder, offset) < 0) return -1; - } - tEndEncode(encoder); - return encoder->pos; -} - -int32_t tDecodeSMqVgOffsets(SCoder *decoder, SMqVgOffsets *pOffsets) { - int32_t sz; - if (tStartDecode(decoder) < 0) return -1; - if (tDecodeI32(decoder, &pOffsets->vgId) < 0) return -1; - if (tDecodeI32(decoder, &sz) < 0) return -1; - pOffsets->offsets = taosArrayInit(sz, sizeof(SMqOffset)); - for (int32_t i = 0; i < sz; i++) { - SMqOffset offset; - if (tDecodeSMqOffset(decoder, &offset) < 0) return -1; - taosArrayPush(pOffsets->offsets, &offset); - } - tEndDecode(decoder); - return 0; -} - -int32_t tEncodeSMqCMResetOffsetReq(SCoder *encoder, const SMqCMResetOffsetReq *pReq) { +int32_t tEncodeSMqCMCommitOffsetReq(SCoder *encoder, const SMqCMCommitOffsetReq *pReq) { if (tStartEncode(encoder) < 0) return -1; if (tEncodeI32(encoder, pReq->num) < 0) return -1; for (int32_t i = 0; i < pReq->num; i++) { @@ -2394,7 +2366,7 @@ int32_t tEncodeSMqCMResetOffsetReq(SCoder *encoder, const SMqCMResetOffsetReq *p return encoder->pos; } -int32_t tDecodeSMqCMResetOffsetReq(SCoder *decoder, SMqCMResetOffsetReq *pReq) { +int32_t tDecodeSMqCMCommitOffsetReq(SCoder *decoder, SMqCMCommitOffsetReq *pReq) { if (tStartDecode(decoder) < 0) return -1; if (tDecodeI32(decoder, &pReq->num) < 0) return -1; pReq->offsets = TCODER_MALLOC(pReq->num * sizeof(SMqOffset), decoder); @@ -2405,23 +2377,3 @@ int32_t tDecodeSMqCMResetOffsetReq(SCoder *decoder, SMqCMResetOffsetReq *pReq) { tEndDecode(decoder); return 0; } - -#if 0 -int32_t tEncodeSMqMVResetOffsetReq(SCoder *encoder, const SMqMVResetOffsetReq *pReq) { - if (tEncodeI64(encoder, pReq->leftForVer) < 0) return -1; - for (int32_t i = 0; i < pReq->num; i++) { - tEncodeSMqOffset(encoder, &pReq->offsets[i]); - } - return encoder->pos; -} - -int32_t tDecodeSMqMVResetOffsetReq(SCoder *decoder, SMqMVResetOffsetReq *pReq) { - if (tDecodeI32(decoder, &pReq->num) < 0) return -1; - pReq->offsets = TCODER_MALLOC(pReq->num * sizeof(SMqOffset), decoder); - if (pReq->offsets == NULL) return -1; - for (int32_t i = 0; i < pReq->num; i++) { - tDecodeSMqOffset(decoder, &pReq->offsets[i]); - } - return 0; -} -#endif diff --git a/source/common/src/tname.c b/source/common/src/tname.c index f3deb84ccf..f6892b26bd 100644 --- a/source/common/src/tname.c +++ b/source/common/src/tname.c @@ -31,7 +31,7 @@ SColumnFilterInfo* tFilterInfoDup(const SColumnFilterInfo* src, int32_t numOfFil } assert(src->filterstr == 0 || src->filterstr == 1); - assert(!(src->lowerRelOptr == TSDB_RELATION_INVALID && src->upperRelOptr == TSDB_RELATION_INVALID)); + assert(!(src->lowerRelOptr == 0 && src->upperRelOptr == 0)); return pFilter; } diff --git a/source/common/src/ttimezone.c b/source/common/src/ttimezone.c deleted file mode 100644 index c45e39c20d..0000000000 --- a/source/common/src/ttimezone.c +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "ulog.h" -#include "tglobal.h" -#include "tconfig.h" -#include "tutil.h" - -// TODO refactor to set the tz value through parameter -void tsSetTimeZone() { - SGlobalCfg *cfg_timezone = taosGetConfigOption("timezone"); - if (cfg_timezone != NULL) { - uInfo("timezone is set to %s by %s", tsTimezone, tsCfgStatusStr[cfg_timezone->cfgStatus]); - } - -#ifdef WINDOWS - char winStr[TSDB_LOCALE_LEN * 2]; - sprintf(winStr, "TZ=%s", tsTimezone); - putenv(winStr); -#else - setenv("TZ", tsTimezone, 1); -#endif - tzset(); - - /* - * get CURRENT time zone. - * system current time zone is affected by daylight saving time(DST) - * - * e.g., the local time zone of London in DST is GMT+01:00, - * otherwise is GMT+00:00 - */ -#ifdef _MSC_VER -#if _MSC_VER >= 1900 - // see https://docs.microsoft.com/en-us/cpp/c-runtime-library/daylight-dstbias-timezone-and-tzname?view=vs-2019 - int64_t timezone = _timezone; - int32_t daylight = _daylight; - char **tzname = _tzname; -#endif -#endif - - int32_t tz = (int32_t)((-timezone * MILLISECOND_PER_SECOND) / MILLISECOND_PER_HOUR); - tz += daylight; - - /* - * format: - * (CST, +0800) - * (BST, +0100) - */ - sprintf(tsTimezone, "(%s, %s%02d00)", tzname[daylight], tz >= 0 ? "+" : "-", abs(tz)); - tsDaylight = daylight; - - uInfo("timezone format changed to %s", tsTimezone); -} diff --git a/source/common/src/ttszip.c b/source/common/src/ttszip.c index 6d57992c35..218472e7c9 100644 --- a/source/common/src/ttszip.c +++ b/source/common/src/ttszip.c @@ -24,8 +24,9 @@ STSBuf* tsBufCreate(bool autoDelete, int32_t order) { pTSBuf->autoDelete = autoDelete; taosGetTmpfilePath(tsTempDir, "join", pTSBuf->path); - pTSBuf->f = fopen(pTSBuf->path, "wb+"); - if (pTSBuf->f == NULL) { + // pTSBuf->pFile = fopen(pTSBuf->path, "wb+"); + pTSBuf->pFile = taosOpenFile(pTSBuf->path, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC); + if (pTSBuf->pFile == NULL) { free(pTSBuf); return NULL; } @@ -60,8 +61,9 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) { tstrncpy(pTSBuf->path, path, sizeof(pTSBuf->path)); - pTSBuf->f = fopen(pTSBuf->path, "rb+"); - if (pTSBuf->f == NULL) { + // pTSBuf->pFile = fopen(pTSBuf->path, "rb+"); + pTSBuf->pFile = taosOpenFile(pTSBuf->path, TD_FILE_WRITE | TD_FILE_READ); + if (pTSBuf->pFile == NULL) { free(pTSBuf); return NULL; } @@ -72,9 +74,9 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) { // validate the file magic number STSBufFileHeader header = {0}; - int32_t ret = fseek(pTSBuf->f, 0, SEEK_SET); + int32_t ret = taosLSeekFile(pTSBuf->pFile, 0, SEEK_SET); UNUSED(ret); - size_t sz = fread(&header, 1, sizeof(STSBufFileHeader), pTSBuf->f); + size_t sz = taosReadFile(pTSBuf->pFile, &header, sizeof(STSBufFileHeader)); UNUSED(sz); // invalid file @@ -112,8 +114,8 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) { return NULL; } - //int64_t pos = ftell(pTSBuf->f); //pos not used - sz = fread(buf, infoSize, 1, pTSBuf->f); + //int64_t pos = ftell(pTSBuf->pFile); //pos not used + sz = taosReadFile(pTSBuf->pFile, buf, infoSize); UNUSED(sz); // the length value for each vnode is not kept in file, so does not set the length value @@ -123,22 +125,22 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) { } free(buf); - ret = fseek(pTSBuf->f, 0, SEEK_END); + ret = taosLSeekFile(pTSBuf->pFile, 0, SEEK_END); UNUSED(ret); - struct stat fileStat; - if (fstat(fileno(pTSBuf->f), &fileStat) != 0) { + int64_t file_size; + if (taosFStatFile(pTSBuf->pFile, &file_size, NULL) != 0) { tsBufDestroy(pTSBuf); return NULL; } - pTSBuf->fileSize = (uint32_t)fileStat.st_size; + pTSBuf->fileSize = (uint32_t)file_size; tsBufResetPos(pTSBuf); // ascending by default pTSBuf->cur.order = TSDB_ORDER_ASC; -// tscDebug("create tsBuf from file:%s, fd:%d, size:%d, numOfGroups:%d, autoDelete:%d", pTSBuf->path, fileno(pTSBuf->f), +// tscDebug("create tsBuf from file:%s, fd:%d, size:%d, numOfGroups:%d, autoDelete:%d", pTSBuf->path, fileno(pTSBuf->pFile), // pTSBuf->fileSize, pTSBuf->numOfGroups, pTSBuf->autoDelete); return pTSBuf; @@ -156,7 +158,7 @@ void* tsBufDestroy(STSBuf* pTSBuf) { tfree(pTSBuf->block.payload); if (!pTSBuf->remainOpen) { - fclose(pTSBuf->f); + taosCloseFile(&pTSBuf->pFile); } if (pTSBuf->autoDelete) { @@ -253,7 +255,7 @@ static void writeDataToDisk(STSBuf* pTSBuf) { tsCompressTimestamp(pTsData->rawBuf, pTsData->len, pTsData->len/TSDB_KEYSIZE, pBlock->payload, pTsData->allocSize, TWO_STAGE_COMP, pTSBuf->assistBuf, pTSBuf->bufSize); - int64_t r = fseek(pTSBuf->f, pTSBuf->fileSize, SEEK_SET); + int64_t r = taosLSeekFile(pTSBuf->pFile, pTSBuf->fileSize, SEEK_SET); assert(r == 0); /* @@ -264,30 +266,30 @@ static void writeDataToDisk(STSBuf* pTSBuf) { * both side has the compressed length is used to support load data forwards/backwords. */ int32_t metaLen = 0; - metaLen += (int32_t)fwrite(&pBlock->tag.nType, 1, sizeof(pBlock->tag.nType), pTSBuf->f); + metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, &pBlock->tag.nType, sizeof(pBlock->tag.nType)); int32_t trueLen = pBlock->tag.nLen; if (pBlock->tag.nType == TSDB_DATA_TYPE_BINARY || pBlock->tag.nType == TSDB_DATA_TYPE_NCHAR) { - metaLen += (int32_t)fwrite(&pBlock->tag.nLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f); - metaLen += (int32_t)fwrite(pBlock->tag.pz, 1, (size_t)pBlock->tag.nLen, pTSBuf->f); + metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, &pBlock->tag.nLen, sizeof(pBlock->tag.nLen)); + metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, pBlock->tag.pz, (size_t)pBlock->tag.nLen); } else if (pBlock->tag.nType == TSDB_DATA_TYPE_FLOAT) { - metaLen += (int32_t)fwrite(&pBlock->tag.nLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f); + metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, &pBlock->tag.nLen, sizeof(pBlock->tag.nLen)); float tfloat = (float)pBlock->tag.d; - metaLen += (int32_t)fwrite(&tfloat, 1, (size_t) pBlock->tag.nLen, pTSBuf->f); + metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, &tfloat, (size_t) pBlock->tag.nLen); } else if (pBlock->tag.nType != TSDB_DATA_TYPE_NULL) { - metaLen += (int32_t)fwrite(&pBlock->tag.nLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f); - metaLen += (int32_t)fwrite(&pBlock->tag.i, 1, (size_t) pBlock->tag.nLen, pTSBuf->f); + metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, &pBlock->tag.nLen, sizeof(pBlock->tag.nLen)); + metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, &pBlock->tag.i, (size_t) pBlock->tag.nLen); } else { trueLen = 0; - metaLen += (int32_t)fwrite(&trueLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f); + metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, &trueLen, sizeof(pBlock->tag.nLen)); } - fwrite(&pBlock->numOfElem, sizeof(pBlock->numOfElem), 1, pTSBuf->f); - fwrite(&pBlock->compLen, sizeof(pBlock->compLen), 1, pTSBuf->f); - fwrite(pBlock->payload, (size_t)pBlock->compLen, 1, pTSBuf->f); - fwrite(&pBlock->compLen, sizeof(pBlock->compLen), 1, pTSBuf->f); + taosWriteFile(pTSBuf->pFile, &pBlock->numOfElem, sizeof(pBlock->numOfElem)); + taosWriteFile(pTSBuf->pFile, &pBlock->compLen, sizeof(pBlock->compLen)); + taosWriteFile(pTSBuf->pFile, pBlock->payload, (size_t)pBlock->compLen); + taosWriteFile(pTSBuf->pFile, &pBlock->compLen, sizeof(pBlock->compLen)); - metaLen += (int32_t) fwrite(&trueLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f); + metaLen += (int32_t) taosWriteFile(pTSBuf->pFile, &trueLen, sizeof(pBlock->tag.nLen)); assert(metaLen == getTagAreaLength(&pBlock->tag)); int32_t blockSize = metaLen + sizeof(pBlock->numOfElem) + sizeof(pBlock->compLen) * 2 + pBlock->compLen; @@ -332,20 +334,20 @@ STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) { * the end of each comp data block */ int32_t prev = -(int32_t) (sizeof(pBlock->padding) + sizeof(pBlock->tag.nLen)); - int32_t ret = fseek(pTSBuf->f, prev, SEEK_CUR); - size_t sz = fread(&pBlock->padding, 1, sizeof(pBlock->padding), pTSBuf->f); - sz = fread(&pBlock->tag.nLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f); + int32_t ret = taosLSeekFile(pTSBuf->pFile, prev, SEEK_CUR); + size_t sz = taosReadFile(pTSBuf->pFile, &pBlock->padding, sizeof(pBlock->padding)); + sz = taosReadFile(pTSBuf->pFile, &pBlock->tag.nLen, sizeof(pBlock->tag.nLen)); UNUSED(sz); pBlock->compLen = pBlock->padding; offset = pBlock->compLen + sizeof(pBlock->compLen) * 2 + sizeof(pBlock->numOfElem) + getTagAreaLength(&pBlock->tag); - ret = fseek(pTSBuf->f, -offset, SEEK_CUR); + ret = taosLSeekFile(pTSBuf->pFile, -offset, SEEK_CUR); UNUSED(ret); } - int32_t ret = fread(&pBlock->tag.nType, sizeof(pBlock->tag.nType), 1, pTSBuf->f); - ret = fread(&pBlock->tag.nLen, sizeof(pBlock->tag.nLen), 1, pTSBuf->f); + int32_t ret = taosReadFile(pTSBuf->pFile, &pBlock->tag.nType, sizeof(pBlock->tag.nType)); + ret = taosReadFile(pTSBuf->pFile, &pBlock->tag.nLen, sizeof(pBlock->tag.nLen)); // NOTE: mix types tags are not supported size_t sz = 0; @@ -356,23 +358,23 @@ STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) { memset(tp, 0, pBlock->tag.nLen + 1); pBlock->tag.pz = tp; - sz = fread(pBlock->tag.pz, (size_t)pBlock->tag.nLen, 1, pTSBuf->f); + sz = taosReadFile(pTSBuf->pFile, pBlock->tag.pz, (size_t)pBlock->tag.nLen); UNUSED(sz); } else if (pBlock->tag.nType == TSDB_DATA_TYPE_FLOAT) { float tfloat = 0; - sz = fread(&tfloat, (size_t) pBlock->tag.nLen, 1, pTSBuf->f); + sz = taosReadFile(pTSBuf->pFile, &tfloat, (size_t) pBlock->tag.nLen); pBlock->tag.d = (double)tfloat; UNUSED(sz); } else if (pBlock->tag.nType != TSDB_DATA_TYPE_NULL) { //TODO check the return value - sz = fread(&pBlock->tag.i, (size_t) pBlock->tag.nLen, 1, pTSBuf->f); + sz = taosReadFile(pTSBuf->pFile, &pBlock->tag.i, (size_t) pBlock->tag.nLen); UNUSED(sz); } - sz = fread(&pBlock->numOfElem, sizeof(pBlock->numOfElem), 1, pTSBuf->f); + sz = taosReadFile(pTSBuf->pFile, &pBlock->numOfElem, sizeof(pBlock->numOfElem)); UNUSED(sz); - sz = fread(&pBlock->compLen, sizeof(pBlock->compLen), 1, pTSBuf->f); + sz = taosReadFile(pTSBuf->pFile, &pBlock->compLen, sizeof(pBlock->compLen)); UNUSED(sz); - sz = fread(pBlock->payload, (size_t)pBlock->compLen, 1, pTSBuf->f); + sz = taosReadFile(pTSBuf->pFile, pBlock->payload, (size_t)pBlock->compLen); if (decomp) { pTSBuf->tsData.len = @@ -381,11 +383,11 @@ STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) { } // read the comp length at the length of comp block - sz = fread(&pBlock->padding, sizeof(pBlock->padding), 1, pTSBuf->f); + sz = taosReadFile(pTSBuf->pFile, &pBlock->padding, sizeof(pBlock->padding)); assert(pBlock->padding == pBlock->compLen); int32_t n = 0; - sz = fread(&n, sizeof(pBlock->tag.nLen), 1, pTSBuf->f); + sz = taosReadFile(pTSBuf->pFile, &n, sizeof(pBlock->tag.nLen)); if (pBlock->tag.nType == TSDB_DATA_TYPE_NULL) { assert(n == 0); } else { @@ -396,7 +398,7 @@ STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) { // for backwards traverse, set the start position at the end of previous block if (order == TSDB_ORDER_DESC) { - int32_t r = fseek(pTSBuf->f, -offset, SEEK_CUR); + int32_t r = taosLSeekFile(pTSBuf->pFile, -offset, SEEK_CUR); UNUSED(r); } @@ -512,7 +514,7 @@ static int32_t tsBufFindGroupById(STSGroupBlockInfoEx* pGroupInfoEx, int32_t num // todo opt performance by cache blocks info static int32_t tsBufFindBlock(STSBuf* pTSBuf, STSGroupBlockInfo* pBlockInfo, int32_t blockIndex) { - if (fseek(pTSBuf->f, pBlockInfo->offset, SEEK_SET) != 0) { + if (taosLSeekFile(pTSBuf->pFile, pBlockInfo->offset, SEEK_SET) != 0) { return -1; } @@ -531,7 +533,7 @@ static int32_t tsBufFindBlock(STSBuf* pTSBuf, STSGroupBlockInfo* pBlockInfo, int STSBlock* pBlock = &pTSBuf->block; int32_t compBlockSize = pBlock->compLen + sizeof(pBlock->compLen) * 2 + sizeof(pBlock->numOfElem) + getTagAreaLength(&pBlock->tag); - int32_t ret = fseek(pTSBuf->f, -compBlockSize, SEEK_CUR); + int32_t ret = taosLSeekFile(pTSBuf->pFile, -compBlockSize, SEEK_CUR); UNUSED(ret); } @@ -548,7 +550,7 @@ static int32_t tsBufFindBlockByTag(STSBuf* pTSBuf, STSGroupBlockInfo* pBlockInfo offset = pBlockInfo->offset + pBlockInfo->compLen; } - if (fseek(pTSBuf->f, (int32_t)offset, SEEK_SET) != 0) { + if (taosLSeekFile(pTSBuf->pFile, (int32_t)offset, SEEK_SET) != 0) { return -1; } @@ -618,11 +620,11 @@ static int32_t doUpdateGroupInfo(STSBuf* pTSBuf, int64_t offset, STSGroupBlockIn return -1; } - if (fseek(pTSBuf->f, (int32_t)offset, SEEK_SET) != 0) { + if (taosLSeekFile(pTSBuf->pFile, (int32_t)offset, SEEK_SET) != 0) { return -1; } - fwrite(pVInfo, sizeof(STSGroupBlockInfo), 1, pTSBuf->f); + taosWriteFile(pTSBuf->pFile, pVInfo, sizeof(STSGroupBlockInfo)); return 0; } @@ -636,19 +638,19 @@ STSGroupBlockInfo* tsBufGetGroupBlockInfo(STSBuf* pTSBuf, int32_t id) { } int32_t STSBufUpdateHeader(STSBuf* pTSBuf, STSBufFileHeader* pHeader) { - if ((pTSBuf->f == NULL) || pHeader == NULL || pHeader->numOfGroup == 0 || pHeader->magic != TS_COMP_FILE_MAGIC) { + if ((pTSBuf->pFile == NULL) || pHeader == NULL || pHeader->numOfGroup == 0 || pHeader->magic != TS_COMP_FILE_MAGIC) { return -1; } assert(pHeader->tsOrder == TSDB_ORDER_ASC || pHeader->tsOrder == TSDB_ORDER_DESC); - int32_t r = fseek(pTSBuf->f, 0, SEEK_SET); + int32_t r = taosLSeekFile(pTSBuf->pFile, 0, SEEK_SET); if (r != 0) { // qError("fseek failed, errno:%d", errno); return -1; } - size_t ws = fwrite(pHeader, sizeof(STSBufFileHeader), 1, pTSBuf->f); + size_t ws = taosWriteFile(pTSBuf->pFile, pHeader, sizeof(STSBufFileHeader)); if (ws != 1) { // qError("ts update header fwrite failed, size:%d, expected size:%d", (int32_t)ws, (int32_t)sizeof(STSBufFileHeader)); return -1; @@ -823,12 +825,12 @@ int32_t tsBufMerge(STSBuf* pDestBuf, const STSBuf* pSrcBuf) { pBlockInfoEx->info.id = id; } - int32_t r = fseek(pDestBuf->f, 0, SEEK_END); + int32_t r = taosLSeekFile(pDestBuf->pFile, 0, SEEK_END); assert(r == 0); int64_t offset = getDataStartOffset(); int32_t size = (int32_t)pSrcBuf->fileSize - (int32_t)offset; - int64_t written = taosFSendFile(pDestBuf->f, pSrcBuf->f, &offset, size); + int64_t written = taosFSendFile(pDestBuf->pFile, pSrcBuf->pFile, &offset, size); if (written == -1 || written != size) { return -1; @@ -839,17 +841,18 @@ int32_t tsBufMerge(STSBuf* pDestBuf, const STSBuf* pSrcBuf) { int32_t oldSize = pDestBuf->fileSize; // file meta data may be cached, close and reopen the file for accurate file size. - fclose(pDestBuf->f); - pDestBuf->f = fopen(pDestBuf->path, "rb+"); - if (pDestBuf->f == NULL) { + taosCloseFile(&pDestBuf->pFile); + // pDestBuf->pFile = fopen(pDestBuf->path, "rb+"); + pDestBuf->pFile = taosOpenFile(pDestBuf->path, TD_FILE_WRITE | TD_FILE_READ); + if (pDestBuf->pFile == NULL) { return -1; } - struct stat fileStat; - if (fstat(fileno(pDestBuf->f), &fileStat) != 0) { + int64_t file_size; + if (taosFStatFile(pDestBuf->pFile, &file_size, NULL) != 0) { return -1; } - pDestBuf->fileSize = (uint32_t)fileStat.st_size; + pDestBuf->fileSize = (uint32_t)file_size; assert(pDestBuf->fileSize == oldSize + size); @@ -868,13 +871,13 @@ STSBuf* tsBufCreateFromCompBlocks(const char* pData, int32_t numOfBlocks, int32_ // update prev vnode length info in file TSBufUpdateGroupInfo(pTSBuf, pTSBuf->numOfGroups - 1, pBlockInfo); - int32_t ret = fseek(pTSBuf->f, pBlockInfo->offset, SEEK_SET); + int32_t ret = taosLSeekFile(pTSBuf->pFile, pBlockInfo->offset, SEEK_SET); if (ret == -1) { // qError("fseek failed, errno:%d", errno); tsBufDestroy(pTSBuf); return NULL; } - size_t sz = fwrite((void*)pData, 1, len, pTSBuf->f); + size_t sz = taosWriteFile(pTSBuf->pFile, (void*)pData, len); if (sz != len) { // qError("ts data fwrite failed, write size:%d, expected size:%d", (int32_t)sz, len); tsBufDestroy(pTSBuf); @@ -893,7 +896,7 @@ STSBuf* tsBufCreateFromCompBlocks(const char* pData, int32_t numOfBlocks, int32_ } // TODO taosFsync?? -// if (taosFsync(fileno(pTSBuf->f)) == -1) { +// if (taosFsync(fileno(pTSBuf->pFile)) == -1) { //// qError("fsync failed, errno:%d", errno); // tsBufDestroy(pTSBuf); // return NULL; @@ -1071,15 +1074,15 @@ int32_t dumpFileBlockByGroupId(STSBuf* pTSBuf, int32_t groupIndex, void* buf, in *len = 0; *numOfBlocks = 0; - if (fseek(pTSBuf->f, pBlockInfo->offset, SEEK_SET) != 0) { - int code = TAOS_SYSTEM_ERROR(ferror(pTSBuf->f)); + if (taosLSeekFile(pTSBuf->pFile, pBlockInfo->offset, SEEK_SET) != 0) { + int code = TAOS_SYSTEM_ERROR(taosEOFFile(pTSBuf->pFile)); // qError("%p: fseek failed: %s", pSql, tstrerror(code)); return code; } - size_t s = fread(buf, 1, pBlockInfo->compLen, pTSBuf->f); + size_t s = taosReadFile(pTSBuf->pFile, buf, pBlockInfo->compLen); if (s != pBlockInfo->compLen) { - int code = TAOS_SYSTEM_ERROR(ferror(pTSBuf->f)); + int code = TAOS_SYSTEM_ERROR(taosEOFFile(pTSBuf->pFile)); // tscError("%p: fread didn't return expected data: %s", pSql, tstrerror(code)); return code; } diff --git a/source/common/src/ttypes.c b/source/common/src/ttypes.c index 1cab413c3f..ee32a20920 100644 --- a/source/common/src/ttypes.c +++ b/source/common/src/ttypes.c @@ -585,7 +585,7 @@ void assignVal(char *val, const char *src, int32_t len, int32_t type) { } void operateVal(void *dst, void *s1, void *s2, int32_t optr, int32_t type) { - if (optr == TSDB_BINARY_OP_ADD) { + if (optr == OP_TYPE_ADD) { switch (type) { case TSDB_DATA_TYPE_TINYINT: *((int8_t *)dst) = GET_INT8_VAL(s1) + GET_INT8_VAL(s2); diff --git a/source/dnode/mgmt/daemon/CMakeLists.txt b/source/dnode/mgmt/daemon/CMakeLists.txt index f1ce726d85..e07c15c95a 100644 --- a/source/dnode/mgmt/daemon/CMakeLists.txt +++ b/source/dnode/mgmt/daemon/CMakeLists.txt @@ -1,8 +1,9 @@ aux_source_directory(src DAEMON_SRC) add_executable(taosd ${DAEMON_SRC}) -target_link_libraries( + +target_include_directories( taosd - PUBLIC dnode - PUBLIC util - PUBLIC os + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) + +target_link_libraries(taosd dnode util os) diff --git a/source/dnode/mgmt/daemon/inc/dmnInt.h b/source/dnode/mgmt/daemon/inc/dmnInt.h new file mode 100644 index 0000000000..8a571352f0 --- /dev/null +++ b/source/dnode/mgmt/daemon/inc/dmnInt.h @@ -0,0 +1,41 @@ + +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_DMN_INT_H_ +#define _TD_DMN_INT_H_ + +#include "tconfig.h" +#include "dnode.h" +#include "taoserror.h" +#include "tglobal.h" +#include "tlog.h" +#include "version.h" + +#ifdef __cplusplus +extern "C" { +#endif + +SDnodeObjCfg dmnGetObjCfg(); + +void dmnDumpCfg(); +void dmnPrintVersion(); +void dmnGenerateGrant(); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_DMN_INT_H_*/ diff --git a/source/dnode/mgmt/daemon/src/daemon.c b/source/dnode/mgmt/daemon/src/daemon.c deleted file mode 100644 index ce672bf7b8..0000000000 --- a/source/dnode/mgmt/daemon/src/daemon.c +++ /dev/null @@ -1,218 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "dnode.h" -#include "os.h" -#include "tconfig.h" -#include "tglobal.h" -#include "tnote.h" -#include "ulog.h" - -static struct { - bool stop; - bool dumpConfig; - bool generateGrant; - bool printAuth; - bool printVersion; - char configDir[PATH_MAX]; -} dmn = {0}; - -void dmnSigintHandle(int signum, void *info, void *ctx) { - uInfo("singal:%d is received", signum); - dmn.stop = true; -} - -void dmnSetSignalHandle() { - taosSetSignal(SIGTERM, dmnSigintHandle); - taosSetSignal(SIGHUP, dmnSigintHandle); - taosSetSignal(SIGINT, dmnSigintHandle); - taosSetSignal(SIGABRT, dmnSigintHandle); - taosSetSignal(SIGBREAK, dmnSigintHandle); -} - -int dmnParseOption(int argc, char const *argv[]) { - tstrncpy(dmn.configDir, "/etc/taos", PATH_MAX); - - for (int i = 1; i < argc; ++i) { - if (strcmp(argv[i], "-c") == 0) { - if (i < argc - 1) { - if (strlen(argv[++i]) >= PATH_MAX) { - printf("config file path overflow"); - return -1; - } - tstrncpy(dmn.configDir, argv[i], PATH_MAX); - } else { - printf("'-c' requires a parameter, default is %s\n", configDir); - return -1; - } - } else if (strcmp(argv[i], "-C") == 0) { - dmn.dumpConfig = true; - } else if (strcmp(argv[i], "-k") == 0) { - dmn.generateGrant = true; - } else if (strcmp(argv[i], "-A") == 0) { - dmn.printAuth = true; - } else if (strcmp(argv[i], "-V") == 0) { - dmn.printVersion = true; - } else { - } - } - - return 0; -} - -void dmnGenerateGrant() { -#if 0 - grantParseParameter(); -#endif -} - -void dmnPrintVersion() { -#ifdef TD_ENTERPRISE - char *releaseName = "enterprise"; -#else - char *releaseName = "community"; -#endif - printf("%s version: %s compatible_version: %s\n", releaseName, version, compatible_version); - printf("gitinfo: %s\n", gitinfo); - printf("gitinfoI: %s\n", gitinfoOfInternal); - printf("builuInfo: %s\n", buildinfo); -} - -int dmnReadConfig(const char *path) { - tstrncpy(configDir, dmn.configDir, PATH_MAX); - taosInitGlobalCfg(); - taosReadGlobalLogCfg(); - - if (taosMkDir(tsLogDir) != 0) { - printf("failed to create dir: %s, reason: %s\n", tsLogDir, strerror(errno)); - return -1; - } - - char temp[PATH_MAX]; - snprintf(temp, PATH_MAX, "%s/taosdlog", tsLogDir); - if (taosInitLog(temp, tsNumOfLogLines, 1) != 0) { - printf("failed to init log file\n"); - return -1; - } - - if (taosInitNotes() != 0) { - printf("failed to init log file\n"); - return -1; - } - - if (taosReadCfgFromFile() != 0) { - uError("failed to read config"); - return -1; - } - - if (taosCheckAndPrintCfg() != 0) { - uError("failed to check config"); - return -1; - } - - taosSetCoreDump(tsEnableCoreFile); - return 0; -} - -void dmnDumpConfig() { taosDumpGlobalCfg(); } - -void dmnWaitSignal() { - dmnSetSignalHandle(); - while (!dmn.stop) { - taosMsleep(100); - } -} - -void dnmInitEnvCfg(SDnodeEnvCfg *pCfg) { - pCfg->sver = 30000000; // 3.0.0.0 - pCfg->numOfCores = tsNumOfCores; - pCfg->numOfCommitThreads = tsNumOfCommitThreads; - pCfg->enableTelem = 0; - tstrncpy(pCfg->timezone, tsTimezone, TSDB_TIMEZONE_LEN); - tstrncpy(pCfg->locale, tsLocale, TSDB_LOCALE_LEN); - tstrncpy(pCfg->charset, tsCharset, TSDB_LOCALE_LEN); - tstrncpy(pCfg->buildinfo, buildinfo, 64); - tstrncpy(pCfg->gitinfo, gitinfo, 48); -} - -void dmnInitObjCfg(SDnodeObjCfg *pCfg) { - pCfg->numOfSupportVnodes = tsNumOfSupportVnodes; - pCfg->statusInterval = tsStatusInterval; - pCfg->numOfThreadsPerCore = tsNumOfThreadsPerCore; - pCfg->ratioOfQueryCores = tsRatioOfQueryCores; - pCfg->maxShellConns = tsMaxShellConns; - pCfg->shellActivityTimer = tsShellActivityTimer; - pCfg->serverPort = tsServerPort; - tstrncpy(pCfg->dataDir, tsDataDir, TSDB_FILENAME_LEN); - tstrncpy(pCfg->localEp, tsLocalEp, TSDB_EP_LEN); - tstrncpy(pCfg->localFqdn, tsLocalFqdn, TSDB_FQDN_LEN); - tstrncpy(pCfg->firstEp, tsFirst, TSDB_EP_LEN); -} - -int dmnRunDnode() { - SDnodeEnvCfg envCfg = {0}; - SDnodeObjCfg objCfg = {0}; - - dnmInitEnvCfg(&envCfg); - dmnInitObjCfg(&objCfg); - - if (dndInit(&envCfg) != 0) { - uInfo("Failed to start TDengine, please check the log at %s", tsLogDir); - return -1; - } - - SDnode *pDnode = dndCreate(&objCfg); - if (pDnode == NULL) { - uInfo("Failed to start TDengine, please check the log at %s", tsLogDir); - return -1; - } - - uInfo("Started TDengine service successfully."); - dmnWaitSignal(); - uInfo("TDengine is shut down!"); - - dndClose(pDnode); - dndCleanup(); - taosCloseLog(); - return 0; -} - -int main(int argc, char const *argv[]) { - if (dmnParseOption(argc, argv) != 0) { - return -1; - } - - if (dmn.generateGrant) { - dmnGenerateGrant(); - return 0; - } - - if (dmn.printVersion) { - dmnPrintVersion(); - return 0; - } - - if (dmnReadConfig(dmn.configDir) != 0) { - return -1; - } - - if (dmn.dumpConfig) { - dmnDumpConfig(); - return 0; - } - - return dmnRunDnode(); -} diff --git a/source/dnode/mgmt/daemon/src/dmnCfg.c b/source/dnode/mgmt/daemon/src/dmnCfg.c new file mode 100644 index 0000000000..57c788dee1 --- /dev/null +++ b/source/dnode/mgmt/daemon/src/dmnCfg.c @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "dmnInt.h" +#include "tconfig.h" + +SDnodeObjCfg dmnGetObjCfg() { + SConfig *pCfg = taosGetCfg(); + SDnodeObjCfg objCfg = {0}; + + objCfg.numOfSupportVnodes = cfgGetItem(pCfg, "supportVnodes")->i32; + tstrncpy(objCfg.dataDir, cfgGetItem(pCfg, "dataDir")->str, sizeof(objCfg.dataDir)); + tstrncpy(objCfg.firstEp, cfgGetItem(pCfg, "firstEp")->str, sizeof(objCfg.firstEp)); + tstrncpy(objCfg.secondEp, cfgGetItem(pCfg, "secondEp")->str, sizeof(objCfg.firstEp)); + objCfg.serverPort = (uint16_t)cfgGetItem(pCfg, "serverPort")->i32; + tstrncpy(objCfg.localFqdn, cfgGetItem(pCfg, "fqdn")->str, sizeof(objCfg.localFqdn)); + snprintf(objCfg.localEp, sizeof(objCfg.localEp), "%s:%u", objCfg.localFqdn, objCfg.serverPort); + objCfg.pDisks = tsDiskCfg; + objCfg.numOfDisks = tsDiskCfgNum; + return objCfg; +} + +void dmnDumpCfg() { + SConfig *pCfg = taosGetCfg(); + cfgDumpCfg(pCfg, 0, 1); +} \ No newline at end of file diff --git a/source/dnode/mgmt/daemon/src/dmnMain.c b/source/dnode/mgmt/daemon/src/dmnMain.c new file mode 100644 index 0000000000..df705898ca --- /dev/null +++ b/source/dnode/mgmt/daemon/src/dmnMain.c @@ -0,0 +1,131 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "dmnInt.h" + +static struct { + bool stop; + bool dumpConfig; + bool generateGrant; + bool printAuth; + bool printVersion; + char envFile[PATH_MAX]; + char apolloUrl[PATH_MAX]; +} dmn = {0}; + +static void dmnSigintHandle(int signum, void *info, void *ctx) { + uInfo("singal:%d is received", signum); + dmn.stop = true; +} + +static void dmnSetSignalHandle() { + taosSetSignal(SIGTERM, dmnSigintHandle); + taosSetSignal(SIGHUP, dmnSigintHandle); + taosSetSignal(SIGINT, dmnSigintHandle); + taosSetSignal(SIGABRT, dmnSigintHandle); + taosSetSignal(SIGBREAK, dmnSigintHandle); +} + +static void dmnWaitSignal() { + dmnSetSignalHandle(); + while (!dmn.stop) { + taosMsleep(100); + } +} + +static int32_t dmnParseOption(int32_t argc, char const *argv[]) { + for (int32_t i = 1; i < argc; ++i) { + if (strcmp(argv[i], "-c") == 0) { + if (i < argc - 1) { + if (strlen(argv[++i]) >= PATH_MAX) { + printf("config file path overflow"); + return -1; + } + tstrncpy(configDir, argv[i], PATH_MAX); + } else { + printf("'-c' requires a parameter, default is %s\n", configDir); + return -1; + } + } else if (strcmp(argv[i], "-C") == 0) { + dmn.dumpConfig = true; + } else if (strcmp(argv[i], "-k") == 0) { + dmn.generateGrant = true; + } else if (strcmp(argv[i], "-V") == 0) { + dmn.printVersion = true; + } else { + } + } + + return 0; +} + +int32_t dmnRunDnode() { + if (dndInit() != 0) { + uInfo("Failed to start TDengine, please check the log"); + return -1; + } + + SDnodeObjCfg objCfg = dmnGetObjCfg(); + SDnode *pDnode = dndCreate(&objCfg); + if (pDnode == NULL) { + uInfo("Failed to start TDengine, please check the log"); + return -1; + } + + uInfo("Started TDengine service successfully."); + dmnWaitSignal(); + uInfo("TDengine is shut down!"); + + dndClose(pDnode); + dndCleanup(); + taosCloseLog(); + taosCleanupCfg(); + return 0; +} + +int main(int argc, char const *argv[]) { + if (dmnParseOption(argc, argv) != 0) { + return -1; + } + + if (dmn.generateGrant) { + dmnGenerateGrant(); + return 0; + } + + if (dmn.printVersion) { + dmnPrintVersion(); + return 0; + } + + if (taosCreateLog("taosdlog", 1, configDir, dmn.envFile, dmn.apolloUrl, NULL, 0) != 0) { + uInfo("Failed to start TDengine since read config error"); + return -1; + } + + if (taosInitCfg(configDir, dmn.envFile, dmn.apolloUrl, NULL, 0) != 0) { + uInfo("Failed to start TDengine since read config error"); + return -1; + } + + if (dmn.dumpConfig) { + dmnDumpCfg(); + taosCleanupCfg(); + return 0; + } + + return dmnRunDnode(); +} diff --git a/source/dnode/mgmt/daemon/src/dmnUtil.c b/source/dnode/mgmt/daemon/src/dmnUtil.c new file mode 100644 index 0000000000..608bc7f3e6 --- /dev/null +++ b/source/dnode/mgmt/daemon/src/dmnUtil.c @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "dmnInt.h" + +void dmnGenerateGrant() { +#if 0 + grantParseParameter(); +#endif +} + +void dmnPrintVersion() { +#ifdef TD_ENTERPRISE + char *releaseName = "enterprise"; +#else + char *releaseName = "community"; +#endif + printf("%s version: %s compatible_version: %s\n", releaseName, version, compatible_version); + printf("gitinfo: %s\n", gitinfo); + printf("gitinfoI: %s\n", gitinfoOfInternal); + printf("builuInfo: %s\n", buildinfo); +} diff --git a/source/dnode/mgmt/impl/inc/dndEnv.h b/source/dnode/mgmt/impl/inc/dndEnv.h index 9eff246323..cbd5eb5827 100644 --- a/source/dnode/mgmt/impl/inc/dndEnv.h +++ b/source/dnode/mgmt/impl/inc/dndEnv.h @@ -124,9 +124,8 @@ typedef struct { typedef struct SDnode { EStat stat; SDnodeObjCfg cfg; - SDnodeEnvCfg env; SDnodeDir dir; - FileFd lockFd; + TdFilePtr pLockFile; SDnodeMgmt dmgmt; SMnodeMgmt mmgmt; SQnodeMgmt qmgmt; @@ -138,11 +137,6 @@ typedef struct SDnode { SStartupReq startup; } SDnode; -typedef struct { - int8_t once; - SDnodeEnvCfg cfg; -} SDnodeEnv; - #ifdef __cplusplus } #endif diff --git a/source/dnode/mgmt/impl/inc/dndInt.h b/source/dnode/mgmt/impl/inc/dndInt.h index 829fabd006..417bc1e041 100644 --- a/source/dnode/mgmt/impl/inc/dndInt.h +++ b/source/dnode/mgmt/impl/inc/dndInt.h @@ -35,6 +35,7 @@ extern "C" { #include "tthread.h" #include "ttime.h" #include "tworker.h" +#include "tglobal.h" #include "dnode.h" diff --git a/source/dnode/mgmt/impl/src/dndBnode.c b/source/dnode/mgmt/impl/src/dndBnode.c index e37a164660..81b020c152 100644 --- a/source/dnode/mgmt/impl/src/dndBnode.c +++ b/source/dnode/mgmt/impl/src/dndBnode.c @@ -62,14 +62,15 @@ static int32_t dndReadBnodeFile(SDnode *pDnode) { char file[PATH_MAX + 20]; snprintf(file, PATH_MAX + 20, "%s/bnode.json", pDnode->dir.dnode); - FILE *fp = fopen(file, "r"); - if (fp == NULL) { + // FILE *fp = fopen(file, "r"); + TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ); + if (pFile == NULL) { dDebug("file %s not exist", file); code = 0; goto PRASE_BNODE_OVER; } - len = (int32_t)fread(content, 1, maxLen, fp); + len = (int32_t)taosReadFile(pFile, content, maxLen); if (len <= 0) { dError("failed to read %s since content is null", file); goto PRASE_BNODE_OVER; @@ -102,7 +103,7 @@ static int32_t dndReadBnodeFile(SDnode *pDnode) { PRASE_BNODE_OVER: if (content != NULL) free(content); if (root != NULL) cJSON_Delete(root); - if (fp != NULL) fclose(fp); + if (pFile != NULL) taosCloseFile(&pFile); terrno = code; return code; @@ -114,8 +115,9 @@ static int32_t dndWriteBnodeFile(SDnode *pDnode) { char file[PATH_MAX + 20]; snprintf(file, PATH_MAX + 20, "%s/bnode.json", pDnode->dir.dnode); - FILE *fp = fopen(file, "w"); - if (fp == NULL) { + // FILE *fp = fopen(file, "w"); + TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pFile == NULL) { terrno = TSDB_CODE_DND_BNODE_WRITE_FILE_ERROR; dError("failed to write %s since %s", file, terrstr()); return -1; @@ -130,9 +132,9 @@ static int32_t dndWriteBnodeFile(SDnode *pDnode) { len += snprintf(content + len, maxLen - len, " \"dropped\": %d\n", pMgmt->dropped); len += snprintf(content + len, maxLen - len, "}\n"); - fwrite(content, 1, len, fp); - taosFsyncFile(fileno(fp)); - fclose(fp); + taosWriteFile(pFile, content, len); + taosFsyncFile(pFile); + taosCloseFile(&pFile); free(content); char realfile[PATH_MAX + 20]; @@ -179,7 +181,7 @@ static void dndBuildBnodeOption(SDnode *pDnode, SBnodeOpt *pOption) { pOption->sendRedirectRspFp = dndSendRedirectRsp; pOption->dnodeId = dndGetDnodeId(pDnode); pOption->clusterId = dndGetClusterId(pDnode); - pOption->sver = pDnode->env.sver; + pOption->sver = tsVersion; } static int32_t dndOpenBnode(SDnode *pDnode) { diff --git a/source/dnode/mgmt/impl/src/dndEnv.c b/source/dnode/mgmt/impl/src/dndEnv.c index 02dced53c2..9467b56e6b 100644 --- a/source/dnode/mgmt/impl/src/dndEnv.c +++ b/source/dnode/mgmt/impl/src/dndEnv.c @@ -25,7 +25,7 @@ #include "tfs.h" #include "wal.h" -static SDnodeEnv dndEnv = {0}; +static int8_t once = DND_ENV_INIT; EStat dndGetStat(SDnode *pDnode) { return pDnode->stat; } @@ -59,31 +59,31 @@ void dndGetStartup(SDnode *pDnode, SStartupReq *pStartup) { pStartup->finished = (dndGetStat(pDnode) == DND_STAT_RUNNING); } -static FileFd dndCheckRunning(char *dataDir) { +static TdFilePtr dndCheckRunning(char *dataDir) { char filepath[PATH_MAX] = {0}; snprintf(filepath, sizeof(filepath), "%s/.running", dataDir); - FileFd fd = taosOpenFileCreateWriteTrunc(filepath); - if (fd < 0) { + TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pFile == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); dError("failed to lock file:%s since %s, quit", filepath, terrstr()); - return -1; + return NULL; } - int32_t ret = taosLockFile(fd); + int32_t ret = taosLockFile(pFile); if (ret != 0) { terrno = TAOS_SYSTEM_ERROR(errno); dError("failed to lock file:%s since %s, quit", filepath, terrstr()); - taosCloseFile(fd); - return -1; + taosCloseFile(&pFile); + return NULL; } - return fd; + return pFile; } -static int32_t dndCreateImp(SDnode *pDnode, SDnodeObjCfg *pCfg) { - pDnode->lockFd = dndCheckRunning(pCfg->dataDir); - if (pDnode->lockFd < 0) { +static int32_t dndInitDir(SDnode *pDnode, SDnodeObjCfg *pCfg) { + pDnode->pLockFile = dndCheckRunning(pCfg->dataDir); + if (pDnode->pLockFile == NULL) { return -1; } @@ -137,7 +137,6 @@ static int32_t dndCreateImp(SDnode *pDnode, SDnodeObjCfg *pCfg) { } memcpy(&pDnode->cfg, pCfg, sizeof(SDnodeObjCfg)); - memcpy(&pDnode->env, &dndEnv.cfg, sizeof(SDnodeEnvCfg)); return 0; } @@ -148,10 +147,10 @@ static void dndCloseImp(SDnode *pDnode) { tfree(pDnode->dir.snode); tfree(pDnode->dir.bnode); - if (pDnode->lockFd >= 0) { - taosUnLockFile(pDnode->lockFd); - taosCloseFile(pDnode->lockFd); - pDnode->lockFd = 0; + if (pDnode->pLockFile != NULL) { + taosUnLockFile(pDnode->pLockFile); + taosCloseFile(&pDnode->pLockFile); + pDnode->pLockFile = NULL; } } @@ -167,7 +166,7 @@ SDnode *dndCreate(SDnodeObjCfg *pCfg) { dndSetStat(pDnode, DND_STAT_INIT); - if (dndCreateImp(pDnode, pCfg) != 0) { + if (dndInitDir(pDnode, pCfg) != 0) { dError("failed to init dnode dir since %s", terrstr()); dndClose(pDnode); return NULL; @@ -177,7 +176,14 @@ SDnode *dndCreate(SDnodeObjCfg *pCfg) { tstrncpy(dCfg.dir, pDnode->cfg.dataDir, TSDB_FILENAME_LEN); dCfg.level = 0; dCfg.primary = 1; - pDnode->pTfs = tfsOpen(&dCfg, 1); + SDiskCfg *pDisks = pDnode->cfg.pDisks; + int32_t numOfDisks = pDnode->cfg.numOfDisks; + if (numOfDisks <= 0 || pDisks == NULL) { + pDisks = &dCfg; + numOfDisks = 1; + } + + pDnode->pTfs = tfsOpen(pDisks, numOfDisks); if (pDnode->pTfs == NULL) { dError("failed to init tfs since %s", terrstr()); dndClose(pDnode); @@ -259,8 +265,8 @@ void dndClose(SDnode *pDnode) { dInfo("dnode object is closed, data:%p", pDnode); } -int32_t dndInit(const SDnodeEnvCfg *pCfg) { - if (atomic_val_compare_exchange_8(&dndEnv.once, DND_ENV_INIT, DND_ENV_READY) != DND_ENV_INIT) { +int32_t dndInit() { + if (atomic_val_compare_exchange_8(&once, DND_ENV_INIT, DND_ENV_READY) != DND_ENV_INIT) { terrno = TSDB_CODE_REPEAT_INIT; dError("failed to init dnode env since %s", terrstr()); return -1; @@ -283,11 +289,8 @@ int32_t dndInit(const SDnodeEnvCfg *pCfg) { } SVnodeOpt vnodeOpt = { - .sver = pCfg->sver, - .timezone = pCfg->timezone, - .locale = pCfg->locale, - .charset = pCfg->charset, - .nthreads = pCfg->numOfCommitThreads, + .sver = tsVersion, + .nthreads = tsNumOfCommitThreads, .putReqToVQueryQFp = dndPutReqToVQueryQ, .sendReqToDnodeFp = dndSendReqToDnode }; @@ -298,13 +301,12 @@ int32_t dndInit(const SDnodeEnvCfg *pCfg) { return -1; } - memcpy(&dndEnv.cfg, pCfg, sizeof(SDnodeEnvCfg)); dInfo("dnode env is initialized"); return 0; } void dndCleanup() { - if (atomic_val_compare_exchange_8(&dndEnv.once, DND_ENV_READY, DND_ENV_CLEANUP) != DND_ENV_READY) { + if (atomic_val_compare_exchange_8(&once, DND_ENV_READY, DND_ENV_CLEANUP) != DND_ENV_READY) { dError("dnode env is already cleaned up"); return; } @@ -325,18 +327,6 @@ void taosGetDisk() { SDiskSize diskSize = tfsGetSize(pTfs); tfsUpdateSize(&fsMeta); - tsTotalDataDirGB = (float)(fsMeta.total / unit); - tsUsedDataDirGB = (float)(fsMeta.used / unit); - tsAvailDataDirGB = (float)(fsMeta.avail / unit); - if (taosGetDiskSize(tsLogDir, &diskSize) == 0) { - tsTotalLogDirGB = (float)(diskSize.total / unit); - tsAvailLogDirGB = (float)(diskSize.avail / unit); - } - - if (taosGetDiskSize(tsTempDir, &diskSize) == 0) { - tsTotalTmpDirGB = (float)(diskSize.total / unit); - tsAvailTmpDirectorySpace = (float)(diskSize.avail / unit); - } #endif } \ No newline at end of file diff --git a/source/dnode/mgmt/impl/src/dndMgmt.c b/source/dnode/mgmt/impl/src/dndMgmt.c index 3d149a3e60..8f18222ab6 100644 --- a/source/dnode/mgmt/impl/src/dndMgmt.c +++ b/source/dnode/mgmt/impl/src/dndMgmt.c @@ -185,16 +185,16 @@ static int32_t dndReadDnodes(SDnode *pDnode) { int32_t maxLen = 256 * 1024; char *content = calloc(1, maxLen + 1); cJSON *root = NULL; - FILE *fp = NULL; - fp = fopen(pMgmt->file, "r"); - if (fp == NULL) { + // fp = fopen(pMgmt->file, "r"); + TdFilePtr pFile = taosOpenFile(pMgmt->file, TD_FILE_READ); + if (pFile == NULL) { dDebug("file %s not exist", pMgmt->file); code = 0; goto PRASE_DNODE_OVER; } - len = (int32_t)fread(content, 1, maxLen, fp); + len = (int32_t)taosReadFile(pFile, content, maxLen); if (len <= 0) { dError("failed to read %s since content is null", pMgmt->file); goto PRASE_DNODE_OVER; @@ -286,7 +286,7 @@ static int32_t dndReadDnodes(SDnode *pDnode) { PRASE_DNODE_OVER: if (content != NULL) free(content); if (root != NULL) cJSON_Delete(root); - if (fp != NULL) fclose(fp); + if (pFile != NULL) taosCloseFile(&pFile); if (dndIsEpChanged(pDnode, pMgmt->dnodeId, pDnode->cfg.localEp)) { dError("localEp %s different with %s and need reconfigured", pDnode->cfg.localEp, pMgmt->file); @@ -309,8 +309,9 @@ PRASE_DNODE_OVER: static int32_t dndWriteDnodes(SDnode *pDnode) { SDnodeMgmt *pMgmt = &pDnode->dmgmt; - FILE *fp = fopen(pMgmt->file, "w"); - if (fp == NULL) { + // FILE *fp = fopen(pMgmt->file, "w"); + TdFilePtr pFile = taosOpenFile(pMgmt->file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pFile == NULL) { dError("failed to write %s since %s", pMgmt->file, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); return -1; @@ -341,9 +342,9 @@ static int32_t dndWriteDnodes(SDnode *pDnode) { } len += snprintf(content + len, maxLen - len, "}\n"); - fwrite(content, 1, len, fp); - taosFsyncFile(fileno(fp)); - fclose(fp); + taosWriteFile(pFile, content, len); + taosFsyncFile(pFile); + taosCloseFile(&pFile); free(content); terrno = 0; @@ -357,23 +358,23 @@ void dndSendStatusReq(SDnode *pDnode) { SDnodeMgmt *pMgmt = &pDnode->dmgmt; taosRLockLatch(&pMgmt->latch); - req.sver = pDnode->env.sver; + req.sver = tsVersion; req.dver = pMgmt->dver; req.dnodeId = pMgmt->dnodeId; req.clusterId = pMgmt->clusterId; req.rebootTime = pMgmt->rebootTime; req.updateTime = pMgmt->updateTime; - req.numOfCores = pDnode->env.numOfCores; + req.numOfCores = tsNumOfCores; req.numOfSupportVnodes = pDnode->cfg.numOfSupportVnodes; memcpy(req.dnodeEp, pDnode->cfg.localEp, TSDB_EP_LEN); - req.clusterCfg.statusInterval = pDnode->cfg.statusInterval; + req.clusterCfg.statusInterval = tsStatusInterval; req.clusterCfg.checkTime = 0; char timestr[32] = "1970-01-01 00:00:00.00"; (void)taosParseTime(timestr, &req.clusterCfg.checkTime, (int32_t)strlen(timestr), TSDB_TIME_PRECISION_MILLI, 0); - memcpy(req.clusterCfg.timezone, pDnode->env.timezone, TSDB_TIMEZONE_LEN); - memcpy(req.clusterCfg.locale, pDnode->env.locale, TSDB_LOCALE_LEN); - memcpy(req.clusterCfg.charset, pDnode->env.charset, TSDB_LOCALE_LEN); + memcpy(req.clusterCfg.timezone, tsTimezone, TD_TIMEZONE_LEN); + memcpy(req.clusterCfg.locale, tsLocale, TD_LOCALE_LEN); + memcpy(req.clusterCfg.charset, tsCharset, TD_LOCALE_LEN); taosRUnLockLatch(&pMgmt->latch); req.pVloads = taosArrayInit(TSDB_MAX_VNODES, sizeof(SVnodeLoad)); @@ -475,7 +476,7 @@ void dndProcessStartupReq(SDnode *pDnode, SRpcMsg *pReq) { static void *dnodeThreadRoutine(void *param) { SDnode *pDnode = param; SDnodeMgmt *pMgmt = &pDnode->dmgmt; - int32_t ms = pDnode->cfg.statusInterval * 1000; + int32_t ms = tsStatusInterval * 1000; setThreadName("dnode-hb"); diff --git a/source/dnode/mgmt/impl/src/dndMnode.c b/source/dnode/mgmt/impl/src/dndMnode.c index c6db75c057..e1b16a188f 100644 --- a/source/dnode/mgmt/impl/src/dndMnode.c +++ b/source/dnode/mgmt/impl/src/dndMnode.c @@ -63,14 +63,15 @@ static int32_t dndReadMnodeFile(SDnode *pDnode) { char file[PATH_MAX + 20]; snprintf(file, PATH_MAX + 20, "%s/mnode.json", pDnode->dir.dnode); - FILE *fp = fopen(file, "r"); - if (fp == NULL) { + // FILE *fp = fopen(file, "r"); + TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ); + if (pFile == NULL) { dDebug("file %s not exist", file); code = 0; goto PRASE_MNODE_OVER; } - len = (int32_t)fread(content, 1, maxLen, fp); + len = (int32_t)taosReadFile(pFile, content, maxLen); if (len <= 0) { dError("failed to read %s since content is null", file); goto PRASE_MNODE_OVER; @@ -143,7 +144,7 @@ static int32_t dndReadMnodeFile(SDnode *pDnode) { PRASE_MNODE_OVER: if (content != NULL) free(content); if (root != NULL) cJSON_Delete(root); - if (fp != NULL) fclose(fp); + if (pFile != NULL) taosCloseFile(&pFile); terrno = code; return code; @@ -155,8 +156,9 @@ static int32_t dndWriteMnodeFile(SDnode *pDnode) { char file[PATH_MAX + 20]; snprintf(file, PATH_MAX + 20, "%s/mnode.json.bak", pDnode->dir.dnode); - FILE *fp = fopen(file, "w"); - if (fp == NULL) { + // FILE *fp = fopen(file, "w"); + TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pFile == NULL) { terrno = TSDB_CODE_DND_MNODE_WRITE_FILE_ERROR; dError("failed to write %s since %s", file, terrstr()); return -1; @@ -184,9 +186,9 @@ static int32_t dndWriteMnodeFile(SDnode *pDnode) { } len += snprintf(content + len, maxLen - len, "}\n"); - fwrite(content, 1, len, fp); - taosFsyncFile(fileno(fp)); - fclose(fp); + taosWriteFile(pFile, content, len); + taosFsyncFile(pFile); + taosCloseFile(&pFile); free(content); char realfile[PATH_MAX + 20]; @@ -273,15 +275,6 @@ static void dndInitMnodeOption(SDnode *pDnode, SMnodeOpt *pOption) { pOption->putReqToMReadQFp = dndPutMsgToMReadQ; pOption->dnodeId = dndGetDnodeId(pDnode); pOption->clusterId = dndGetClusterId(pDnode); - pOption->cfg.sver = pDnode->env.sver; - pOption->cfg.enableTelem = pDnode->env.enableTelem; - pOption->cfg.statusInterval = pDnode->cfg.statusInterval; - pOption->cfg.shellActivityTimer = pDnode->cfg.shellActivityTimer; - pOption->cfg.timezone = pDnode->env.timezone; - pOption->cfg.charset = pDnode->env.charset; - pOption->cfg.locale = pDnode->env.locale; - pOption->cfg.gitinfo = pDnode->env.gitinfo; - pOption->cfg.buildinfo = pDnode->env.buildinfo; } static void dndBuildMnodeDeployOption(SDnode *pDnode, SMnodeOpt *pOption) { diff --git a/source/dnode/mgmt/impl/src/dndQnode.c b/source/dnode/mgmt/impl/src/dndQnode.c index 64545ec09f..93e2209610 100644 --- a/source/dnode/mgmt/impl/src/dndQnode.c +++ b/source/dnode/mgmt/impl/src/dndQnode.c @@ -62,14 +62,15 @@ static int32_t dndReadQnodeFile(SDnode *pDnode) { char file[PATH_MAX + 20]; snprintf(file, PATH_MAX + 20, "%s/qnode.json", pDnode->dir.dnode); - FILE *fp = fopen(file, "r"); - if (fp == NULL) { + // FILE *fp = fopen(file, "r"); + TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ); + if (pFile == NULL) { dDebug("file %s not exist", file); code = 0; goto PRASE_QNODE_OVER; } - len = (int32_t)fread(content, 1, maxLen, fp); + len = (int32_t)taosReadFile(pFile, content, maxLen); if (len <= 0) { dError("failed to read %s since content is null", file); goto PRASE_QNODE_OVER; @@ -102,7 +103,7 @@ static int32_t dndReadQnodeFile(SDnode *pDnode) { PRASE_QNODE_OVER: if (content != NULL) free(content); if (root != NULL) cJSON_Delete(root); - if (fp != NULL) fclose(fp); + if (pFile != NULL) taosCloseFile(&pFile); terrno = code; return code; @@ -114,8 +115,9 @@ static int32_t dndWriteQnodeFile(SDnode *pDnode) { char file[PATH_MAX + 20]; snprintf(file, PATH_MAX + 20, "%s/qnode.json", pDnode->dir.dnode); - FILE *fp = fopen(file, "w"); - if (fp == NULL) { + // FILE *fp = fopen(file, "w"); + TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pFile == NULL) { terrno = TSDB_CODE_DND_QNODE_WRITE_FILE_ERROR; dError("failed to write %s since %s", file, terrstr()); return -1; @@ -130,9 +132,9 @@ static int32_t dndWriteQnodeFile(SDnode *pDnode) { len += snprintf(content + len, maxLen - len, " \"dropped\": %d\n", pMgmt->dropped); len += snprintf(content + len, maxLen - len, "}\n"); - fwrite(content, 1, len, fp); - taosFsyncFile(fileno(fp)); - fclose(fp); + taosWriteFile(pFile, content, len); + taosFsyncFile(pFile); + taosCloseFile(&pFile); free(content); char realfile[PATH_MAX + 20]; @@ -185,7 +187,7 @@ static void dndBuildQnodeOption(SDnode *pDnode, SQnodeOpt *pOption) { pOption->sendRedirectRspFp = dndSendRedirectRsp; pOption->dnodeId = dndGetDnodeId(pDnode); pOption->clusterId = dndGetClusterId(pDnode); - pOption->sver = pDnode->env.sver; + pOption->sver = tsVersion; } static int32_t dndOpenQnode(SDnode *pDnode) { diff --git a/source/dnode/mgmt/impl/src/dndSnode.c b/source/dnode/mgmt/impl/src/dndSnode.c index 77686a6027..4906aef246 100644 --- a/source/dnode/mgmt/impl/src/dndSnode.c +++ b/source/dnode/mgmt/impl/src/dndSnode.c @@ -62,14 +62,15 @@ static int32_t dndReadSnodeFile(SDnode *pDnode) { char file[PATH_MAX + 20]; snprintf(file, PATH_MAX + 20, "%s/snode.json", pDnode->dir.dnode); - FILE *fp = fopen(file, "r"); - if (fp == NULL) { + // FILE *fp = fopen(file, "r"); + TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ); + if (pFile == NULL) { dDebug("file %s not exist", file); code = 0; goto PRASE_SNODE_OVER; } - len = (int32_t)fread(content, 1, maxLen, fp); + len = (int32_t)taosReadFile(pFile, content, maxLen); if (len <= 0) { dError("failed to read %s since content is null", file); goto PRASE_SNODE_OVER; @@ -102,7 +103,7 @@ static int32_t dndReadSnodeFile(SDnode *pDnode) { PRASE_SNODE_OVER: if (content != NULL) free(content); if (root != NULL) cJSON_Delete(root); - if (fp != NULL) fclose(fp); + if (pFile != NULL) taosCloseFile(&pFile); terrno = code; return code; @@ -114,8 +115,9 @@ static int32_t dndWriteSnodeFile(SDnode *pDnode) { char file[PATH_MAX + 20]; snprintf(file, PATH_MAX + 20, "%s/snode.json", pDnode->dir.dnode); - FILE *fp = fopen(file, "w"); - if (fp == NULL) { + // FILE *fp = fopen(file, "w"); + TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pFile == NULL) { terrno = TSDB_CODE_DND_SNODE_WRITE_FILE_ERROR; dError("failed to write %s since %s", file, terrstr()); return -1; @@ -130,9 +132,9 @@ static int32_t dndWriteSnodeFile(SDnode *pDnode) { len += snprintf(content + len, maxLen - len, " \"dropped\": %d\n", pMgmt->dropped); len += snprintf(content + len, maxLen - len, "}\n"); - fwrite(content, 1, len, fp); - taosFsyncFile(fileno(fp)); - fclose(fp); + taosWriteFile(pFile, content, len); + taosFsyncFile(pFile); + taosCloseFile(&pFile); free(content); char realfile[PATH_MAX + 20]; @@ -179,7 +181,7 @@ static void dndBuildSnodeOption(SDnode *pDnode, SSnodeOpt *pOption) { pOption->sendRedirectRspFp = dndSendRedirectRsp; pOption->dnodeId = dndGetDnodeId(pDnode); pOption->clusterId = dndGetClusterId(pDnode); - pOption->sver = pDnode->env.sver; + pOption->sver = tsVersion; } static int32_t dndOpenSnode(SDnode *pDnode) { diff --git a/source/dnode/mgmt/impl/src/dndTransport.c b/source/dnode/mgmt/impl/src/dndTransport.c index 4e35baf905..8c64cc16c3 100644 --- a/source/dnode/mgmt/impl/src/dndTransport.c +++ b/source/dnode/mgmt/impl/src/dndTransport.c @@ -25,8 +25,8 @@ #include "dndMnode.h" #include "dndVnodes.h" -#define INTERNAL_USER "_dnd" -#define INTERNAL_CKEY "_key" +#define INTERNAL_USER "_dnd" +#define INTERNAL_CKEY "_key" #define INTERNAL_SECRET "_pwd" static void dndInitMsgFp(STransMgmt *pMgmt) { @@ -113,6 +113,7 @@ static void dndInitMsgFp(STransMgmt *pMgmt) { pMgmt->msgFp[TMSG_INDEX(TDMT_MND_ALTER_TOPIC)] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_TOPIC)] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_MND_SUBSCRIBE)] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_MND_MQ_COMMIT_OFFSET)] = dndProcessMnodeWriteMsg; /*pMgmt->msgFp[TMSG_INDEX(TDMT_VND_SUBSCRIBE_RSP)] = dndProcessMnodeWriteMsg;*/ pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_SET_CONN_RSP)] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_REB_RSP)] = dndProcessMnodeWriteMsg; @@ -155,7 +156,7 @@ static void dndInitMsgFp(STransMgmt *pMgmt) { } static void dndProcessResponse(void *parent, SRpcMsg *pRsp, SEpSet *pEpSet) { - SDnode * pDnode = parent; + SDnode *pDnode = parent; STransMgmt *pMgmt = &pDnode->tmgmt; tmsg_t msgType = pRsp->msgType; @@ -188,7 +189,7 @@ static int32_t dndInitClient(SDnode *pDnode) { rpcInit.cfp = dndProcessResponse; rpcInit.sessions = 1024; rpcInit.connType = TAOS_CONN_CLIENT; - rpcInit.idleTime = pDnode->cfg.shellActivityTimer * 1000; + rpcInit.idleTime = tsShellActivityTimer * 1000; rpcInit.user = INTERNAL_USER; rpcInit.ckey = INTERNAL_CKEY; rpcInit.spi = 1; @@ -219,7 +220,7 @@ static void dndCleanupClient(SDnode *pDnode) { } static void dndProcessRequest(void *param, SRpcMsg *pReq, SEpSet *pEpSet) { - SDnode * pDnode = param; + SDnode *pDnode = param; STransMgmt *pMgmt = &pDnode->tmgmt; tmsg_t msgType = pReq->msgType; @@ -313,7 +314,7 @@ static int32_t dndRetrieveUserAuthInfo(void *parent, char *user, char *spi, char SAuthReq authReq = {0}; tstrncpy(authReq.user, user, TSDB_USER_LEN); int32_t contLen = tSerializeSAuthReq(NULL, 0, &authReq); - void * pReq = rpcMallocCont(contLen); + void *pReq = rpcMallocCont(contLen); tSerializeSAuthReq(pReq, contLen, &authReq); SRpcMsg rpcMsg = {.pCont = pReq, .contLen = contLen, .msgType = TDMT_MND_AUTH, .ahandle = (void *)9528}; @@ -343,7 +344,7 @@ static int32_t dndInitServer(SDnode *pDnode) { STransMgmt *pMgmt = &pDnode->tmgmt; dndInitMsgFp(pMgmt); - int32_t numOfThreads = (int32_t)((pDnode->env.numOfCores * pDnode->cfg.numOfThreadsPerCore) / 2.0); + int32_t numOfThreads = (int32_t)((tsNumOfCores * tsNumOfThreadsPerCore) / 2.0); if (numOfThreads < 1) { numOfThreads = 1; } @@ -354,9 +355,9 @@ static int32_t dndInitServer(SDnode *pDnode) { rpcInit.label = "D-S"; rpcInit.numOfThreads = numOfThreads; rpcInit.cfp = dndProcessRequest; - rpcInit.sessions = pDnode->cfg.maxShellConns; + rpcInit.sessions = tsMaxShellConns; rpcInit.connType = TAOS_CONN_SERVER; - rpcInit.idleTime = pDnode->cfg.shellActivityTimer * 1000; + rpcInit.idleTime = tsShellActivityTimer * 1000; rpcInit.afp = dndRetrieveUserAuthInfo; rpcInit.parent = pDnode; diff --git a/source/dnode/mgmt/impl/src/dndVnodes.c b/source/dnode/mgmt/impl/src/dndVnodes.c index b82d991179..f20493aa7f 100644 --- a/source/dnode/mgmt/impl/src/dndVnodes.c +++ b/source/dnode/mgmt/impl/src/dndVnodes.c @@ -219,14 +219,15 @@ static int32_t dndGetVnodesFromFile(SDnode *pDnode, SWrapperCfg **ppCfgs, int32_ snprintf(file, PATH_MAX + 20, "%s/vnodes.json", pDnode->dir.vnodes); - fp = fopen(file, "r"); - if (fp == NULL) { + // fp = fopen(file, "r"); + TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ); + if (pFile == NULL) { dDebug("file %s not exist", file); code = 0; goto PRASE_VNODE_OVER; } - len = (int32_t)fread(content, 1, maxLen, fp); + len = (int32_t)taosReadFile(pFile, content, maxLen); if (len <= 0) { dError("failed to read %s since content is null", file); goto PRASE_VNODE_OVER; @@ -304,7 +305,7 @@ static int32_t dndGetVnodesFromFile(SDnode *pDnode, SWrapperCfg **ppCfgs, int32_ PRASE_VNODE_OVER: if (content != NULL) free(content); if (root != NULL) cJSON_Delete(root); - if (fp != NULL) fclose(fp); + if (pFile != NULL) taosCloseFile(&pFile); return code; } @@ -315,8 +316,9 @@ static int32_t dndWriteVnodesToFile(SDnode *pDnode) { snprintf(file, PATH_MAX + 20, "%s/vnodes.json.bak", pDnode->dir.vnodes); snprintf(realfile, PATH_MAX + 20, "%s/vnodes.json", pDnode->dir.vnodes); - FILE *fp = fopen(file, "w"); - if (fp == NULL) { + // FILE *fp = fopen(file, "w"); + TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pFile == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); dError("failed to write %s since %s", file, terrstr()); return -1; @@ -347,9 +349,9 @@ static int32_t dndWriteVnodesToFile(SDnode *pDnode) { len += snprintf(content + len, maxLen - len, " ]\n"); len += snprintf(content + len, maxLen - len, "}\n"); - fwrite(content, 1, len, fp); - taosFsyncFile(fileno(fp)); - fclose(fp); + taosWriteFile(pFile, content, len); + taosFsyncFile(pFile); + taosCloseFile(&pFile); free(content); terrno = 0; @@ -421,7 +423,7 @@ static int32_t dndOpenVnodes(SDnode *pDnode) { pMgmt->totalVnodes = numOfVnodes; - int32_t threadNum = pDnode->env.numOfCores; + int32_t threadNum = tsNumOfCores; #if 1 threadNum = 1; #endif @@ -874,11 +876,11 @@ static int32_t dndInitVnodeWorkers(SDnode *pDnode) { SVnodesMgmt *pMgmt = &pDnode->vmgmt; int32_t maxFetchThreads = 4; - int32_t minFetchThreads = TMIN(maxFetchThreads, pDnode->env.numOfCores); - int32_t minQueryThreads = TMAX((int32_t)(pDnode->env.numOfCores * pDnode->cfg.ratioOfQueryCores), 1); + int32_t minFetchThreads = TMIN(maxFetchThreads, tsNumOfCores); + int32_t minQueryThreads = TMAX((int32_t)(tsNumOfCores * tsRatioOfQueryCores), 1); int32_t maxQueryThreads = minQueryThreads; - int32_t maxWriteThreads = TMAX(pDnode->env.numOfCores, 1); - int32_t maxSyncThreads = TMAX(pDnode->env.numOfCores / 2, 1); + int32_t maxWriteThreads = TMAX(tsNumOfCores, 1); + int32_t maxSyncThreads = TMAX(tsNumOfCores / 2, 1); SQWorkerPool *pQPool = &pMgmt->queryPool; pQPool->name = "vnode-query"; diff --git a/source/dnode/mgmt/impl/test/sut/inc/sut.h b/source/dnode/mgmt/impl/test/sut/inc/sut.h index 250d563a8b..304a370bcd 100644 --- a/source/dnode/mgmt/impl/test/sut/inc/sut.h +++ b/source/dnode/mgmt/impl/test/sut/inc/sut.h @@ -20,17 +20,16 @@ #include "os.h" #include "dnode.h" -#include "tconfig.h" +#include "tmsg.h" #include "tdataformat.h" #include "tglobal.h" #include "tmsg.h" -#include "tnote.h" #include "trpc.h" #include "tthread.h" -#include "ulog.h" #include "client.h" #include "server.h" +#include "tlog.h" class Testbase { public: diff --git a/source/dnode/mgmt/impl/test/sut/src/client.cpp b/source/dnode/mgmt/impl/test/sut/src/client.cpp index 589c015013..b89cb02834 100644 --- a/source/dnode/mgmt/impl/test/sut/src/client.cpp +++ b/source/dnode/mgmt/impl/test/sut/src/client.cpp @@ -24,6 +24,9 @@ static void processClientRsp(void* parent, SRpcMsg* pRsp, SEpSet* pEpSet) { } void TestClient::SetRpcRsp(SRpcMsg* rsp) { + if (this->pRsp) { + free(this->pRsp); + } this->pRsp = (SRpcMsg*)calloc(1, sizeof(SRpcMsg)); this->pRsp->msgType = rsp->msgType; this->pRsp->code = rsp->code; @@ -60,7 +63,7 @@ bool TestClient::Init(const char* user, const char* pass, const char* fqdn, uint strcpy(this->user, user); strcpy(this->pass, pass); this->port = port; - // this->pRsp = NULL; + this->pRsp = NULL; this->DoInit(); return true; } diff --git a/source/dnode/mgmt/impl/test/sut/src/server.cpp b/source/dnode/mgmt/impl/test/sut/src/server.cpp index f7b5ab4599..985625b41c 100644 --- a/source/dnode/mgmt/impl/test/sut/src/server.cpp +++ b/source/dnode/mgmt/impl/test/sut/src/server.cpp @@ -25,11 +25,6 @@ void* serverLoop(void* param) { SDnodeObjCfg TestServer::BuildOption(const char* path, const char* fqdn, uint16_t port, const char* firstEp) { SDnodeObjCfg cfg = {0}; cfg.numOfSupportVnodes = 16; - cfg.statusInterval = 1; - cfg.numOfThreadsPerCore = 1; - cfg.ratioOfQueryCores = 1; - cfg.maxShellConns = 1000; - cfg.shellActivityTimer = 30; cfg.serverPort = port; strcpy(cfg.dataDir, path); snprintf(cfg.localEp, TSDB_EP_LEN, "%s:%u", fqdn, port); diff --git a/source/dnode/mgmt/impl/test/sut/src/sut.cpp b/source/dnode/mgmt/impl/test/sut/src/sut.cpp index 771c5886ef..00a79ebfd9 100644 --- a/source/dnode/mgmt/impl/test/sut/src/sut.cpp +++ b/source/dnode/mgmt/impl/test/sut/src/sut.cpp @@ -28,25 +28,19 @@ void Testbase::InitLog(const char* path) { wDebugFlag = 0; sDebugFlag = 0; tsdbDebugFlag = 0; - cqDebugFlag = 0; tscEmbeddedInUtil = 1; tsAsyncLog = 0; taosRemoveDir(path); taosMkDir(path); - - char temp[PATH_MAX]; - snprintf(temp, PATH_MAX, "%s/taosdlog", path); - if (taosInitLog(temp, tsNumOfLogLines, 1) != 0) { + tstrncpy(tsLogDir, path, PATH_MAX); + if (taosInitLog("taosdlog", 1) != 0) { printf("failed to init log file\n"); } } void Testbase::Init(const char* path, int16_t port) { - SDnodeEnvCfg cfg = {0}; - cfg.numOfCommitThreads = 1; - cfg.numOfCores = 1; - dndInit(&cfg); + dndInit(); char fqdn[] = "localhost"; char firstEp[TSDB_EP_LEN] = {0}; diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index e64191f715..bb2367ef72 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -123,6 +123,7 @@ typedef enum { TRN_TYPE_DROP_TOPIC = 1015, TRN_TYPE_SUBSCRIBE = 1016, TRN_TYPE_REBALANCE = 1017, + TRN_TYPE_COMMIT_OFFSET = 1018, TRN_TYPE_BASIC_SCOPE_END, TRN_TYPE_GLOBAL_SCOPE = 2000, TRN_TYPE_CREATE_DNODE = 2001, @@ -176,7 +177,7 @@ typedef struct { SArray* undoActions; int64_t createdTime; int64_t lastExecTime; - uint64_t dbUid; + int64_t dbUid; char dbname[TSDB_DB_FNAME_LEN]; char lastError[TSDB_TRANS_ERROR_LEN]; } STrans; @@ -304,16 +305,16 @@ typedef struct { } SDbCfg; typedef struct { - char name[TSDB_DB_FNAME_LEN]; - char acct[TSDB_USER_LEN]; - char createUser[TSDB_USER_LEN]; - int64_t createdTime; - int64_t updateTime; - uint64_t uid; - int32_t cfgVersion; - int32_t vgVersion; - int8_t hashMethod; // default is 1 - SDbCfg cfg; + char name[TSDB_DB_FNAME_LEN]; + char acct[TSDB_USER_LEN]; + char createUser[TSDB_USER_LEN]; + int64_t createdTime; + int64_t updateTime; + int64_t uid; + int32_t cfgVersion; + int32_t vgVersion; + int8_t hashMethod; // default is 1 + SDbCfg cfg; } SDbObj; typedef struct { @@ -346,8 +347,8 @@ typedef struct { char db[TSDB_DB_FNAME_LEN]; int64_t createdTime; int64_t updateTime; - uint64_t uid; - uint64_t dbUid; + int64_t uid; + int64_t dbUid; int32_t version; int32_t nextColId; int32_t numOfColumns; @@ -465,6 +466,24 @@ static FORCE_INLINE void tDeleteSMqSubConsumer(SMqSubConsumer* pSubConsumer) { } } +typedef struct { + char key[TSDB_PARTITION_KEY_LEN]; + int64_t offset; +} SMqOffsetObj; + +static FORCE_INLINE int32_t tEncodeSMqOffsetObj(void** buf, const SMqOffsetObj* pOffset) { + int32_t tlen = 0; + tlen += taosEncodeString(buf, pOffset->key); + tlen += taosEncodeFixedI64(buf, pOffset->offset); + return tlen; +} + +static FORCE_INLINE void* tDecodeSMqOffsetObj(void* buf, SMqOffsetObj* pOffset) { + buf = taosDecodeStringTo(buf, pOffset->key); + buf = taosDecodeFixedI64(buf, &pOffset->offset); + return buf; +} + typedef struct { char key[TSDB_SUBSCRIBE_KEY_LEN]; int32_t status; diff --git a/source/dnode/mnode/impl/inc/mndInt.h b/source/dnode/mnode/impl/inc/mndInt.h index 97cd6caf04..e65535206d 100644 --- a/source/dnode/mnode/impl/inc/mndInt.h +++ b/source/dnode/mnode/impl/inc/mndInt.h @@ -24,6 +24,8 @@ #include "tqueue.h" #include "ttime.h" #include "wal.h" +#include "version.h" +#include "tglobal.h" #ifdef __cplusplus extern "C" { @@ -56,12 +58,9 @@ typedef struct { } SProfileMgmt; typedef struct { - int8_t enable; - pthread_mutex_t lock; - pthread_cond_t cond; - volatile int32_t exit; - pthread_t thread; - char email[TSDB_FQDN_LEN]; + bool enable; + SRWLatch lock; + char email[TSDB_FQDN_LEN]; } STelemMgmt; typedef struct { @@ -81,8 +80,8 @@ typedef struct SMnode { tmr_h timer; tmr_h transTimer; tmr_h mqTimer; + tmr_h telemTimer; char *path; - SMnodeCfg cfg; int64_t checkTime; SSdb *pSdb; SDnode *pDnode; diff --git a/source/dnode/mnode/impl/inc/mndOffset.h b/source/dnode/mnode/impl/inc/mndOffset.h new file mode 100644 index 0000000000..e18eec973f --- /dev/null +++ b/source/dnode/mnode/impl/inc/mndOffset.h @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_MND_OFFSET_H_ +#define _TD_MND_OFFSET_H_ + +#include "mndInt.h" + +#ifdef __cplusplus +extern "C" { +#endif + +int32_t mndInitOffset(SMnode *pMnode); +void mndCleanupOffset(SMnode *pMnode); + +SMqOffsetObj *mndAcquireOffset(SMnode *pMnode, const char *key); +void mndReleaseOffset(SMnode *pMnode, SMqOffsetObj *pOffset); + +SSdbRaw *mndOffsetActionEncode(SMqOffsetObj *pOffset); +SSdbRow *mndOffsetActionDecode(SSdbRaw *pRaw); + +int32_t mndCreateOffset(STrans *pTrans, const char *cgroup, const char *topicName, const SArray *vgs); + +static FORCE_INLINE int32_t mndMakePartitionKey(char *key, const char *cgroup, const char *topicName, int32_t vgId) { + return snprintf(key, TSDB_PARTITION_KEY_LEN, "%d:%s:%s", vgId, cgroup, topicName); +} + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_MND_OFFSET_H_*/ diff --git a/source/dnode/mnode/impl/src/mndCluster.c b/source/dnode/mnode/impl/src/mndCluster.c index a41ccd9896..3410a386da 100644 --- a/source/dnode/mnode/impl/src/mndCluster.c +++ b/source/dnode/mnode/impl/src/mndCluster.c @@ -52,7 +52,7 @@ int32_t mndGetClusterName(SMnode *pMnode, char *clusterName, int32_t len) { SSdb *pSdb = pMnode->pSdb; SClusterObj *pCluster = sdbAcquire(pSdb, SDB_CLUSTER, &pMnode->clusterId); - if (pCluster = NULL) { + if (pCluster == NULL) { return -1; } diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 39ea4b482c..b872e933d5 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -22,11 +22,11 @@ #include "mndUser.h" #include "mndVgroup.h" -#define TSDB_DNODE_VER_NUMBER 1 +#define TSDB_DNODE_VER_NUMBER 1 #define TSDB_DNODE_RESERVE_SIZE 64 -#define TSDB_CONFIG_OPTION_LEN 16 -#define TSDB_CONIIG_VALUE_LEN 48 -#define TSDB_CONFIG_NUMBER 8 +#define TSDB_CONFIG_OPTION_LEN 16 +#define TSDB_CONIIG_VALUE_LEN 48 +#define TSDB_CONFIG_NUMBER 8 static const char *offlineReason[] = { "", @@ -237,7 +237,7 @@ int32_t mndGetDnodeSize(SMnode *pMnode) { bool mndIsDnodeOnline(SMnode *pMnode, SDnodeObj *pDnode, int64_t curMs) { int64_t interval = TABS(pDnode->lastAccessTime - curMs); - if (interval > 3500 * pMnode->cfg.statusInterval) { + if (interval > 3500 * tsStatusInterval) { if (pDnode->rebootTime > 0) { pDnode->offlineReason = DND_REASON_STATUS_MSG_TIMEOUT; } @@ -272,24 +272,24 @@ static void mndGetDnodeData(SMnode *pMnode, SArray *pDnodeEps) { } static int32_t mndCheckClusterCfgPara(SMnode *pMnode, const SClusterCfg *pCfg) { - if (pCfg->statusInterval != pMnode->cfg.statusInterval) { - mError("statusInterval [%d - %d] cfg inconsistent", pCfg->statusInterval, pMnode->cfg.statusInterval); + if (pCfg->statusInterval != tsStatusInterval) { + mError("statusInterval [%d - %d] cfg inconsistent", pCfg->statusInterval, tsStatusInterval); return DND_REASON_STATUS_INTERVAL_NOT_MATCH; } - if ((0 != strcasecmp(pCfg->timezone, pMnode->cfg.timezone)) && (pMnode->checkTime != pCfg->checkTime)) { - mError("timezone [%s - %s] [%" PRId64 " - %" PRId64 "] cfg inconsistent", pCfg->timezone, pMnode->cfg.timezone, + if ((0 != strcasecmp(pCfg->timezone, tsTimezone)) && (pMnode->checkTime != pCfg->checkTime)) { + mError("timezone [%s - %s] [%" PRId64 " - %" PRId64 "] cfg inconsistent", pCfg->timezone, tsTimezone, pCfg->checkTime, pMnode->checkTime); return DND_REASON_TIME_ZONE_NOT_MATCH; } - if (0 != strcasecmp(pCfg->locale, pMnode->cfg.locale)) { - mError("locale [%s - %s] cfg inconsistent", pCfg->locale, pMnode->cfg.locale); + if (0 != strcasecmp(pCfg->locale, tsLocale)) { + mError("locale [%s - %s] cfg inconsistent", pCfg->locale, tsLocale); return DND_REASON_LOCALE_NOT_MATCH; } - if (0 != strcasecmp(pCfg->charset, pMnode->cfg.charset)) { - mError("charset [%s - %s] cfg inconsistent.", pCfg->charset, pMnode->cfg.charset); + if (0 != strcasecmp(pCfg->charset, tsCharset)) { + mError("charset [%s - %s] cfg inconsistent.", pCfg->charset, tsCharset); return DND_REASON_CHARSET_NOT_MATCH; } @@ -355,12 +355,11 @@ static int32_t mndProcessStatusReq(SMnodeMsg *pReq) { bool needCheck = !online || dnodeChanged || reboot; if (needCheck) { - if (statusReq.sver != pMnode->cfg.sver) { + if (statusReq.sver != tsVersion) { if (pDnode != NULL) { pDnode->offlineReason = DND_REASON_VERSION_NOT_MATCH; } - mError("dnode:%d, status msg version:%d not match cluster:%d", statusReq.dnodeId, statusReq.sver, - pMnode->cfg.sver); + mError("dnode:%d, status msg version:%d not match cluster:%d", statusReq.dnodeId, statusReq.sver, tsVersion); terrno = TSDB_CODE_MND_INVALID_MSG_VERSION; goto PROCESS_STATUS_MSG_OVER; } @@ -632,13 +631,13 @@ static int32_t mndGetConfigMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp pShow->bytes[cols] = TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - tstrncpy(pSchema[cols].name, "name", sizeof(pSchema[cols].name)); + strcpy(pSchema[cols].name, "name"); pSchema[cols].bytes = pShow->bytes[cols]; cols++; pShow->bytes[cols] = TSDB_CONIIG_VALUE_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - tstrncpy(pSchema[cols].name, "value", sizeof(pSchema[cols].name)); + strcpy(pSchema[cols].name, "value"); pSchema[cols].bytes = pShow->bytes[cols]; cols++; @@ -659,29 +658,30 @@ static int32_t mndGetConfigMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp static int32_t mndRetrieveConfigs(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { SMnode *pMnode = pReq->pMnode; + int32_t totalRows = 0; int32_t numOfRows = 0; char *cfgOpts[TSDB_CONFIG_NUMBER] = {0}; char cfgVals[TSDB_CONFIG_NUMBER][TSDB_CONIIG_VALUE_LEN + 1] = {0}; char *pWrite; int32_t cols = 0; - cfgOpts[numOfRows] = "statusInterval"; - snprintf(cfgVals[numOfRows], TSDB_CONIIG_VALUE_LEN, "%d", pMnode->cfg.statusInterval); - numOfRows++; + cfgOpts[totalRows] = "statusInterval"; + snprintf(cfgVals[totalRows], TSDB_CONIIG_VALUE_LEN, "%d", tsStatusInterval); + totalRows++; - cfgOpts[numOfRows] = "timezone"; - snprintf(cfgVals[numOfRows], TSDB_CONIIG_VALUE_LEN, "%s", pMnode->cfg.timezone); - numOfRows++; + cfgOpts[totalRows] = "timezone"; + snprintf(cfgVals[totalRows], TSDB_CONIIG_VALUE_LEN, "%s", tsTimezone); + totalRows++; - cfgOpts[numOfRows] = "locale"; - snprintf(cfgVals[numOfRows], TSDB_CONIIG_VALUE_LEN, "%s", pMnode->cfg.locale); - numOfRows++; + cfgOpts[totalRows] = "locale"; + snprintf(cfgVals[totalRows], TSDB_CONIIG_VALUE_LEN, "%s", tsLocale); + totalRows++; - cfgOpts[numOfRows] = "charset"; - snprintf(cfgVals[numOfRows], TSDB_CONIIG_VALUE_LEN, "%s", pMnode->cfg.charset); - numOfRows++; + cfgOpts[totalRows] = "charset"; + snprintf(cfgVals[totalRows], TSDB_CONIIG_VALUE_LEN, "%s", tsCharset); + totalRows++; - for (int32_t i = 0; i < numOfRows; i++) { + for (int32_t i = 0; i < totalRows; i++) { cols = 0; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; @@ -691,6 +691,8 @@ static int32_t mndRetrieveConfigs(SMnodeMsg *pReq, SShowObj *pShow, char *data, pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; STR_WITH_MAXSIZE_TO_VARSTR(pWrite, cfgVals[i], TSDB_CONIIG_VALUE_LEN); cols++; + + numOfRows++; } mndVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); diff --git a/source/dnode/mnode/impl/src/mndOffset.c b/source/dnode/mnode/impl/src/mndOffset.c new file mode 100644 index 0000000000..ac9e99ebd4 --- /dev/null +++ b/source/dnode/mnode/impl/src/mndOffset.c @@ -0,0 +1,224 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "mndOffset.h" +#include "mndAuth.h" +#include "mndDb.h" +#include "mndDnode.h" +#include "mndMnode.h" +#include "mndShow.h" +#include "mndStb.h" +#include "mndTrans.h" +#include "mndUser.h" +#include "mndVgroup.h" +#include "tname.h" + +#define MND_OFFSET_VER_NUMBER 1 +#define MND_OFFSET_RESERVE_SIZE 64 + +static int32_t mndOffsetActionInsert(SSdb *pSdb, SMqOffsetObj *pOffset); +static int32_t mndOffsetActionDelete(SSdb *pSdb, SMqOffsetObj *pOffset); +static int32_t mndOffsetActionUpdate(SSdb *pSdb, SMqOffsetObj *pOffset, SMqOffsetObj *pNewOffset); +static int32_t mndProcessCommitOffsetReq(SMnodeMsg *pReq); + +int32_t mndInitOffset(SMnode *pMnode) { + SSdbTable table = {.sdbType = SDB_OFFSET, + .keyType = SDB_KEY_BINARY, + .encodeFp = (SdbEncodeFp)mndOffsetActionEncode, + .decodeFp = (SdbDecodeFp)mndOffsetActionDecode, + .insertFp = (SdbInsertFp)mndOffsetActionInsert, + .updateFp = (SdbUpdateFp)mndOffsetActionUpdate, + .deleteFp = (SdbDeleteFp)mndOffsetActionDelete}; + + mndSetMsgHandle(pMnode, TDMT_MND_MQ_COMMIT_OFFSET, mndProcessCommitOffsetReq); + + return sdbSetTable(pMnode->pSdb, table); +} + +void mndCleanupOffset(SMnode *pMnode) {} + +SSdbRaw *mndOffsetActionEncode(SMqOffsetObj *pOffset) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + void *buf = NULL; + int32_t tlen = tEncodeSMqOffsetObj(NULL, pOffset); + int32_t size = sizeof(int32_t) + tlen + MND_OFFSET_RESERVE_SIZE; + + SSdbRaw *pRaw = sdbAllocRaw(SDB_OFFSET, MND_OFFSET_VER_NUMBER, size); + if (pRaw == NULL) goto OFFSET_ENCODE_OVER; + + buf = malloc(tlen); + if (buf == NULL) goto OFFSET_ENCODE_OVER; + + void *abuf = buf; + tEncodeSMqOffsetObj(&abuf, pOffset); + + int32_t dataPos = 0; + SDB_SET_INT32(pRaw, dataPos, tlen, OFFSET_ENCODE_OVER); + SDB_SET_BINARY(pRaw, dataPos, buf, tlen, OFFSET_ENCODE_OVER); + SDB_SET_RESERVE(pRaw, dataPos, MND_OFFSET_RESERVE_SIZE, OFFSET_ENCODE_OVER); + SDB_SET_DATALEN(pRaw, dataPos, OFFSET_ENCODE_OVER); + + terrno = TSDB_CODE_SUCCESS; + +OFFSET_ENCODE_OVER: + tfree(buf); + if (terrno != TSDB_CODE_SUCCESS) { + mError("offset:%s, failed to encode to raw:%p since %s", pOffset->key, pRaw, terrstr()); + sdbFreeRaw(pRaw); + return NULL; + } + + mTrace("offset:%s, encode to raw:%p, row:%p", pOffset->key, pRaw, pOffset); + return pRaw; +} + +SSdbRow *mndOffsetActionDecode(SSdbRaw *pRaw) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + void *buf = NULL; + + int8_t sver = 0; + if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto OFFSET_DECODE_OVER; + + if (sver != MND_OFFSET_VER_NUMBER) { + terrno = TSDB_CODE_SDB_INVALID_DATA_VER; + goto OFFSET_DECODE_OVER; + } + + int32_t size = sizeof(SMqOffsetObj); + SSdbRow *pRow = sdbAllocRow(size); + if (pRow == NULL) goto OFFSET_DECODE_OVER; + + SMqOffsetObj *pOffset = sdbGetRowObj(pRow); + if (pOffset == NULL) goto OFFSET_DECODE_OVER; + + int32_t dataPos = 0; + int32_t tlen; + SDB_GET_INT32(pRaw, dataPos, &tlen, OFFSET_DECODE_OVER); + buf = malloc(tlen + 1); + if (buf == NULL) goto OFFSET_DECODE_OVER; + SDB_GET_BINARY(pRaw, dataPos, buf, tlen, OFFSET_DECODE_OVER); + SDB_GET_RESERVE(pRaw, dataPos, MND_OFFSET_RESERVE_SIZE, OFFSET_DECODE_OVER); + + if (tDecodeSMqOffsetObj(buf, pOffset) == NULL) { + goto OFFSET_DECODE_OVER; + } + + terrno = TSDB_CODE_SUCCESS; + +OFFSET_DECODE_OVER: + tfree(buf); + if (terrno != TSDB_CODE_SUCCESS) { + mError("offset:%s, failed to decode from raw:%p since %s", pOffset->key, pRaw, terrstr()); + tfree(pRow); + return NULL; + } + + mTrace("offset:%s, decode from raw:%p, row:%p", pOffset->key, pRaw, pOffset); + return pRow; +} + +int32_t mndCreateOffset(STrans *pTrans, const char *cgroup, const char *topicName, const SArray *vgs) { + int32_t code = 0; + int32_t sz = taosArrayGetSize(vgs); + for (int32_t i = 0; i < sz; i++) { + SMqConsumerEp *pConsumerEp = taosArrayGet(vgs, i); + SMqOffsetObj offsetObj; + if (mndMakePartitionKey(offsetObj.key, cgroup, topicName, pConsumerEp->vgId) < 0) { + return -1; + } + offsetObj.offset = -1; + SSdbRaw *pOffsetRaw = mndOffsetActionEncode(&offsetObj); + if (pOffsetRaw == NULL) { + return -1; + } + sdbSetRawStatus(pOffsetRaw, SDB_STATUS_READY); + if (mndTransAppendRedolog(pTrans, pOffsetRaw) < 0) { + return -1; + } + } + return 0; +} + +static int32_t mndProcessCommitOffsetReq(SMnodeMsg *pMsg) { + char key[TSDB_PARTITION_KEY_LEN]; + + SMnode *pMnode = pMsg->pMnode; + char *msgStr = pMsg->rpcMsg.pCont; + SMqCMCommitOffsetReq commitOffsetReq; + SCoder decoder; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, msgStr, pMsg->rpcMsg.contLen, TD_DECODER); + + tDecodeSMqCMCommitOffsetReq(&decoder, &commitOffsetReq); + + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_COMMIT_OFFSET, &pMsg->rpcMsg); + + for (int32_t i = 0; i < commitOffsetReq.num; i++) { + SMqOffset *pOffset = &commitOffsetReq.offsets[i]; + if (mndMakePartitionKey(key, pOffset->cgroup, pOffset->topicName, pOffset->vgId) < 0) { + return -1; + } + SMqOffsetObj *pOffsetObj = mndAcquireOffset(pMnode, key); + ASSERT(pOffsetObj); + pOffsetObj->offset = pOffset->offset; + SSdbRaw *pOffsetRaw = mndOffsetActionEncode(pOffsetObj); + sdbSetRawStatus(pOffsetRaw, SDB_STATUS_READY); + mndTransAppendRedolog(pTrans, pOffsetRaw); + mndReleaseOffset(pMnode, pOffsetObj); + } + + if (mndTransPrepare(pMnode, pTrans) != 0) { + mError("mq-commit-offset-trans:%d, failed to prepare since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + + mndTransDrop(pTrans); + return TSDB_CODE_MND_ACTION_IN_PROGRESS; +} + +static int32_t mndOffsetActionInsert(SSdb *pSdb, SMqOffsetObj *pOffset) { + mTrace("offset:%s, perform insert action", pOffset->key); + return 0; +} + +static int32_t mndOffsetActionDelete(SSdb *pSdb, SMqOffsetObj *pOffset) { + mTrace("offset:%s, perform delete action", pOffset->key); + return 0; +} + +static int32_t mndOffsetActionUpdate(SSdb *pSdb, SMqOffsetObj *pOldOffset, SMqOffsetObj *pNewOffset) { + mTrace("offset:%s, perform update action", pOldOffset->key); + return 0; +} + +SMqOffsetObj *mndAcquireOffset(SMnode *pMnode, const char *key) { + SSdb *pSdb = pMnode->pSdb; + SMqOffsetObj *pOffset = sdbAcquire(pSdb, SDB_OFFSET, key); + if (pOffset == NULL && terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) { + terrno = TSDB_CODE_MND_OFFSET_NOT_EXIST; + } + return pOffset; +} + +void mndReleaseOffset(SMnode *pMnode, SMqOffsetObj *pOffset) { + SSdb *pSdb = pMnode->pSdb; + sdbRelease(pSdb, pOffset); +} + +static void mndCancelGetNextOffset(SMnode *pMnode, void *pIter) { + SSdb *pSdb = pMnode->pSdb; + sdbCancelFetch(pSdb, pIter); +} diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index df892e2242..e313c4d676 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -21,6 +21,7 @@ #include "mndStb.h" #include "mndUser.h" #include "tglobal.h" +#include "version.h" #define QUERY_ID_SIZE 20 #define QUERY_OBJ_ID_SIZE 18 @@ -62,7 +63,7 @@ static void mndCancelGetNextQuery(SMnode *pMnode, void *pIter); int32_t mndInitProfile(SMnode *pMnode) { SProfileMgmt *pMgmt = &pMnode->profileMgmt; - int32_t connCheckTime = pMnode->cfg.shellActivityTimer * 2; + int32_t connCheckTime = tsShellActivityTimer * 2; pMgmt->cache = taosCacheInit(TSDB_DATA_TYPE_INT, connCheckTime, true, (__cache_free_fn_t)mndFreeConn, "conn"); if (pMgmt->cache == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -116,7 +117,7 @@ static SConnObj *mndCreateConn(SMnode *pMnode, SRpcConnInfo *pInfo, int32_t pid, tstrncpy(connObj.user, pInfo->user, TSDB_USER_LEN); tstrncpy(connObj.app, app, TSDB_APP_NAME_LEN); - int32_t keepTime = pMnode->cfg.shellActivityTimer * 3; + int32_t keepTime = tsShellActivityTimer * 3; SConnObj *pConn = taosCachePut(pMgmt->cache, &connId, sizeof(int32_t), &connObj, sizeof(connObj), keepTime * 1000); if (pConn == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -142,7 +143,7 @@ static SConnObj *mndAcquireConn(SMnode *pMnode, int32_t connId) { return NULL; } - int32_t keepTime = pMnode->cfg.shellActivityTimer * 3; + int32_t keepTime = tsShellActivityTimer * 3; pConn->lastAccessTimeMs = keepTime * 1000 + (uint64_t)taosGetTimestampMs(); mTrace("conn:%d, acquired from cache, data:%p", pConn->id, pConn); diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index afb338e97d..8fd0c282e1 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -64,7 +64,7 @@ static SShowObj *mndCreateShowObj(SMnode *pMnode, SShowReq *pReq) { memcpy(showObj.db, pReq->db, TSDB_DB_FNAME_LEN); memcpy(showObj.payload, pReq->payload, pReq->payloadLen); - int32_t keepTime = pMnode->cfg.shellActivityTimer * 6 * 1000; + int32_t keepTime = tsShellActivityTimer * 6 * 1000; SShowObj *pShow = taosCachePut(pMgmt->cache, &showId, sizeof(int64_t), &showObj, size, keepTime); if (pShow == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index a5c4c41244..2ea157fea4 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -19,6 +19,7 @@ #include "mndDb.h" #include "mndDnode.h" #include "mndMnode.h" +#include "mndOffset.h" #include "mndShow.h" #include "mndStb.h" #include "mndTopic.h" @@ -80,13 +81,13 @@ int32_t mndInitSubscribe(SMnode *pMnode) { return sdbSetTable(pMnode->pSdb, table); } -static SMqSubscribeObj *mndCreateSubscription(SMnode *pMnode, const SMqTopicObj *pTopic, const char *consumerGroup) { +static SMqSubscribeObj *mndCreateSubscription(SMnode *pMnode, const SMqTopicObj *pTopic, const char *cgroup) { SMqSubscribeObj *pSub = tNewSubscribeObj(); if (pSub == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } - char *key = mndMakeSubscribeKey(consumerGroup, pTopic->name); + char *key = mndMakeSubscribeKey(cgroup, pTopic->name); if (key == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; tDeleteSMqSubscribeObj(pSub); @@ -289,9 +290,15 @@ static int32_t mndProcessGetSubEpReq(SMnodeMsg *pMsg) { strcpy(topicEp.topic, topicName); topicEp.vgs = taosArrayInit(vgsz, sizeof(SMqSubVgEp)); for (int32_t k = 0; k < vgsz; k++) { + char offsetKey[TSDB_PARTITION_KEY_LEN]; SMqConsumerEp *pConsumerEp = taosArrayGet(pSubConsumer->vgInfo, k); - - SMqSubVgEp vgEp = {.epSet = pConsumerEp->epSet, .vgId = pConsumerEp->vgId}; + SMqSubVgEp vgEp = {.epSet = pConsumerEp->epSet, .vgId = pConsumerEp->vgId, .offset = -1}; + mndMakePartitionKey(offsetKey, pConsumer->cgroup, topicName, pConsumerEp->vgId); + SMqOffsetObj *pOffsetObj = mndAcquireOffset(pMnode, offsetKey); + if (pOffsetObj != NULL) { + vgEp.offset = pOffsetObj->offset; + mndReleaseOffset(pMnode, pOffsetObj); + } taosArrayPush(topicEp.vgs, &vgEp); } taosArrayPush(rsp.topics, &topicEp); @@ -870,7 +877,7 @@ static SSdbRaw *mndSubActionEncode(SMqSubscribeObj *pSub) { SUB_ENCODE_OVER: tfree(buf); - if (terrno != 0) { + if (terrno != TSDB_CODE_SUCCESS) { mError("subscribe:%s, failed to encode to raw:%p since %s", pSub->key, pRaw, terrstr()); sdbFreeRaw(pRaw); return NULL; @@ -1085,6 +1092,8 @@ static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg) { mDebug("create new subscription by consumer %ld, group: %s, topic %s", consumerId, cgroup, newTopicName); pSub = mndCreateSubscription(pMnode, pTopic, cgroup); createSub = true; + + mndCreateOffset(pTrans, cgroup, newTopicName, pSub->unassignedVg); } SMqSubConsumer mqSubConsumer; diff --git a/source/dnode/mnode/impl/src/mndTelem.c b/source/dnode/mnode/impl/src/mndTelem.c index 5beb1b10e3..7c1215ea42 100644 --- a/source/dnode/mnode/impl/src/mndTelem.c +++ b/source/dnode/mnode/impl/src/mndTelem.c @@ -21,8 +21,8 @@ #include "tversion.h" #define TELEMETRY_SERVER "telemetry.taosdata.com" -#define TELEMETRY_PORT 80 -#define REPORT_INTERVAL 86400 +#define TELEMETRY_PORT 80 +#define REPORT_INTERVAL 86400 static void mndBeginObject(SBufferWriter* bw) { tbufWriteChar(bw, '{'); } @@ -33,25 +33,8 @@ static void mndCloseObject(SBufferWriter* bw) { } else { tbufWriteChar(bw, '}'); } - tbufWriteChar(bw, ','); } -#if 0 -static void beginArray(SBufferWriter* bw) { - tbufWriteChar(bw, '['); -} - -static void closeArray(SBufferWriter* bw) { - size_t len = tbufTell(bw); - if (tbufGetData(bw, false)[len - 1] == ',') { - tbufWriteCharAt(bw, len - 1, ']'); - } else { - tbufWriteChar(bw, ']'); - } - tbufWriteChar(bw, ','); -} -#endif - static void mndWriteString(SBufferWriter* bw, const char* str) { tbufWriteChar(bw, '"'); tbufWrite(bw, str, strlen(str)); @@ -61,7 +44,7 @@ static void mndWriteString(SBufferWriter* bw, const char* str) { static void mndAddIntField(SBufferWriter* bw, const char* k, int64_t v) { mndWriteString(bw, k); tbufWriteChar(bw, ':'); - char buf[32]; + char buf[32] = {0}; sprintf(buf, "%" PRId64, v); tbufWrite(bw, buf, strlen(buf)); tbufWriteChar(bw, ','); @@ -79,12 +62,13 @@ static void mndAddCpuInfo(SMnode* pMnode, SBufferWriter* bw) { size_t size = 0; int32_t done = 0; - FILE* fp = fopen("/proc/cpuinfo", "r"); - if (fp == NULL) { + // FILE* fp = fopen("/proc/cpuinfo", "r"); + TdFilePtr pFile = taosOpenFile("/proc/cpuinfo", TD_FILE_READ); + if (pFile == NULL) { return; } - while (done != 3 && (size = tgetline(&line, &size, fp)) != -1) { + while (done != 3 && (size = taosGetLineFile(pFile, &line)) != -1) { line[size - 1] = '\0'; if (((done & 1) == 0) && strncmp(line, "model name", 10) == 0) { const char* v = strchr(line, ':') + 2; @@ -100,20 +84,21 @@ static void mndAddCpuInfo(SMnode* pMnode, SBufferWriter* bw) { } } - free(line); - fclose(fp); + if(line != NULL) free(line); + taosCloseFile(&pFile); } static void mndAddOsInfo(SMnode* pMnode, SBufferWriter* bw) { char* line = NULL; size_t size = 0; - FILE* fp = fopen("/etc/os-release", "r"); - if (fp == NULL) { + // FILE* fp = fopen("/etc/os-release", "r"); + TdFilePtr pFile = taosOpenFile("/etc/os-release", TD_FILE_READ); + if (pFile == NULL) { return; } - while ((size = tgetline(&line, &size, fp)) != -1) { + while ((size = taosGetLineFile(pFile, &line)) != -1) { line[size - 1] = '\0'; if (strncmp(line, "PRETTY_NAME", 11) == 0) { const char* p = strchr(line, '=') + 1; @@ -126,20 +111,21 @@ static void mndAddOsInfo(SMnode* pMnode, SBufferWriter* bw) { } } - free(line); - fclose(fp); + if(line != NULL) free(line); + taosCloseFile(&pFile); } static void mndAddMemoryInfo(SMnode* pMnode, SBufferWriter* bw) { char* line = NULL; size_t size = 0; - FILE* fp = fopen("/proc/meminfo", "r"); - if (fp == NULL) { + // FILE* fp = fopen("/proc/meminfo", "r"); + TdFilePtr pFile = taosOpenFile("/proc/meminfo", TD_FILE_READ); + if (pFile == NULL) { return; } - while ((size = tgetline(&line, &size, fp)) != -1) { + while ((size = taosGetLineFile(pFile, &line)) != -1) { line[size - 1] = '\0'; if (strncmp(line, "MemTotal", 8) == 0) { const char* p = strchr(line, ':') + 1; @@ -149,19 +135,15 @@ static void mndAddMemoryInfo(SMnode* pMnode, SBufferWriter* bw) { } } - free(line); - fclose(fp); + if(line != NULL) free(line); + taosCloseFile(&pFile); } static void mndAddVersionInfo(SMnode* pMnode, SBufferWriter* bw) { STelemMgmt* pMgmt = &pMnode->telemMgmt; - - char vstr[32] = {0}; - taosVersionIntToStr(pMnode->cfg.sver, vstr, 32); - - mndAddStringField(bw, "version", vstr); - mndAddStringField(bw, "buildInfo", pMnode->cfg.buildinfo); - mndAddStringField(bw, "gitInfo", pMnode->cfg.gitinfo); + mndAddStringField(bw, "version", version); + mndAddStringField(bw, "buildInfo", buildinfo); + mndAddStringField(bw, "gitInfo", gitinfo); mndAddStringField(bw, "email", pMgmt->email); } @@ -184,24 +166,17 @@ static void mndAddRuntimeInfo(SMnode* pMnode, SBufferWriter* bw) { } static void mndSendTelemetryReport(SMnode* pMnode) { - STelemMgmt* pMgmt = &pMnode->telemMgmt; - - char buf[128] = {0}; - uint32_t ip = taosGetIpv4FromFqdn(TELEMETRY_SERVER); - if (ip == 0xffffffff) { - mDebug("failed to get IP address of " TELEMETRY_SERVER " since :%s", strerror(errno)); - return; - } - SOCKET fd = taosOpenTcpClientSocket(ip, TELEMETRY_PORT, 0); - if (fd < 0) { - mDebug("failed to create socket for telemetry, reason:%s", strerror(errno)); - return; - } + STelemMgmt* pMgmt = &pMnode->telemMgmt; + SBufferWriter bw = tbufInitWriter(NULL, false); + int32_t code = -1; + char buf[128] = {0}; + SOCKET fd = 0; char clusterName[64] = {0}; - mndGetClusterName(pMnode, clusterName, sizeof(clusterName)); + if (mndGetClusterName(pMnode, clusterName, sizeof(clusterName)) != 0) { + goto SEND_OVER; + } - SBufferWriter bw = tbufInitWriter(NULL, false); mndBeginObject(&bw); mndAddStringField(&bw, "instanceId", clusterName); mndAddIntField(&bw, "reportVersion", 1); @@ -212,111 +187,95 @@ static void mndSendTelemetryReport(SMnode* pMnode) { mndAddRuntimeInfo(pMnode, &bw); mndCloseObject(&bw); + uint32_t ip = taosGetIpv4FromFqdn(TELEMETRY_SERVER); + if (ip == 0xffffffff) { + terrno = TAOS_SYSTEM_ERROR(errno); + mError("failed to get ip of %s since :%s", TELEMETRY_SERVER, terrstr()); + goto SEND_OVER; + } + + fd = taosOpenTcpClientSocket(ip, TELEMETRY_PORT, 0); + if (fd < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + mError("failed to create socket to %s:%d since:%s", TELEMETRY_SERVER, TELEMETRY_PORT, terrstr()); + goto SEND_OVER; + } + const char* header = "POST /report HTTP/1.1\n" "Host: " TELEMETRY_SERVER "\n" "Content-Type: application/json\n" "Content-Length: "; + if (taosWriteSocket(fd, (void*)header, (int32_t)strlen(header)) < 0) { + goto SEND_OVER; + } - taosWriteSocket(fd, (void*)header, (int32_t)strlen(header)); - int32_t contLen = (int32_t)(tbufTell(&bw) - 1); + int32_t contLen = (int32_t)(tbufTell(&bw)); sprintf(buf, "%d\n\n", contLen); - taosWriteSocket(fd, buf, (int32_t)strlen(buf)); - taosWriteSocket(fd, tbufGetData(&bw, false), contLen); - tbufCloseWriter(&bw); + if (taosWriteSocket(fd, buf, (int32_t)strlen(buf)) < 0) { + goto SEND_OVER; + } + + const char* pCont = tbufGetData(&bw, false); + if (taosWriteSocket(fd, (void*)pCont, contLen) < 0) { + goto SEND_OVER; + } // read something to avoid nginx error 499 if (taosReadSocket(fd, buf, 10) < 0) { - mDebug("failed to receive response since %s", strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + mError("failed to receive response since %s", terrstr()); + goto SEND_OVER; } + mInfo("send telemetry to %s:%d, len:%d content: %s", TELEMETRY_SERVER, TELEMETRY_PORT, contLen, pCont); + code = 0; + +SEND_OVER: + tbufCloseWriter(&bw); taosCloseSocket(fd); + + if (code != 0) { + mError("failed to send telemetry to %s:%d since %s", TELEMETRY_SERVER, TELEMETRY_PORT, terrstr()); + } } -static void* mndTelemThreadFp(void* param) { - SMnode* pMnode = param; +static int32_t mndProcessTelemTimer(SMnodeMsg* pReq) { + SMnode* pMnode = pReq->pMnode; STelemMgmt* pMgmt = &pMnode->telemMgmt; + if (!pMgmt->enable) return 0; - struct timespec end = {0}; - clock_gettime(CLOCK_REALTIME, &end); - end.tv_sec += 300; // wait 5 minutes before send first report - - setThreadName("mnd-telem"); - - while (!pMgmt->exit) { - int32_t r = 0; - struct timespec ts = end; - pthread_mutex_lock(&pMgmt->lock); - r = pthread_cond_timedwait(&pMgmt->cond, &pMgmt->lock, &ts); - pthread_mutex_unlock(&pMgmt->lock); - if (r == 0) break; - if (r != ETIMEDOUT) continue; - - if (mndIsMaster(pMnode)) { - mndSendTelemetryReport(pMnode); - } - end.tv_sec += REPORT_INTERVAL; - } - - return NULL; + taosWLockLatch(&pMgmt->lock); + mndSendTelemetryReport(pMnode); + taosWUnLockLatch(&pMgmt->lock); + return 0; } static void mndGetEmail(SMnode* pMnode, char* filepath) { STelemMgmt* pMgmt = &pMnode->telemMgmt; - int32_t fd = taosOpenFileRead(filepath); - if (fd < 0) { + TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_READ); + if (pFile == NULL) { return; } - if (taosReadFile(fd, (void*)pMgmt->email, TSDB_FQDN_LEN) < 0) { + if (taosReadFile(pFile, (void*)pMgmt->email, TSDB_FQDN_LEN) < 0) { mError("failed to read %d bytes from file %s since %s", TSDB_FQDN_LEN, filepath, strerror(errno)); } - taosCloseFile(fd); + taosCloseFile(&pFile); } int32_t mndInitTelem(SMnode* pMnode) { STelemMgmt* pMgmt = &pMnode->telemMgmt; - pMgmt->enable = pMnode->cfg.enableTelem; - - if (!pMgmt->enable) return 0; - - pMgmt->exit = 0; - pthread_mutex_init(&pMgmt->lock, NULL); - pthread_cond_init(&pMgmt->cond, NULL); - pMgmt->email[0] = 0; - + pMgmt->enable = tsEnableTelemetryReporting; + taosInitRWLatch(&pMgmt->lock); mndGetEmail(pMnode, "/usr/local/taos/email"); - pthread_attr_t attr; - pthread_attr_init(&attr); - pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE); - - int32_t code = pthread_create(&pMgmt->thread, &attr, mndTelemThreadFp, pMnode); - pthread_attr_destroy(&attr); - if (code != 0) { - mDebug("failed to create telemetry thread since :%s", strerror(code)); - } - - mInfo("mnd telemetry is initialized"); + mndSetMsgHandle(pMnode, TDMT_MND_TELEM_TIMER, mndProcessTelemTimer); + mDebug("mnode telemetry is initialized"); return 0; } -void mndCleanupTelem(SMnode* pMnode) { - STelemMgmt* pMgmt = &pMnode->telemMgmt; - if (!pMgmt->enable) return; - - if (taosCheckPthreadValid(pMgmt->thread)) { - pthread_mutex_lock(&pMgmt->lock); - pMgmt->exit = 1; - pthread_cond_signal(&pMgmt->cond); - pthread_mutex_unlock(&pMgmt->lock); - - pthread_join(pMgmt->thread, NULL); - } - - pthread_mutex_destroy(&pMgmt->lock); - pthread_cond_destroy(&pMgmt->cond); -} +void mndCleanupTelem(SMnode* pMnode) {} diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 7d684cd542..a009e01a52 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -71,7 +71,7 @@ int32_t mndInitTrans(SMnode *pMnode) { .updateFp = (SdbUpdateFp)mndTransActionUpdate, .deleteFp = (SdbDeleteFp)mndTransActionDelete}; - mndSetMsgHandle(pMnode, TDMT_MND_TRANS, mndProcessTransReq); + mndSetMsgHandle(pMnode, TDMT_MND_TRANS_TIMER, mndProcessTransReq); mndSetMsgHandle(pMnode, TDMT_MND_KILL_TRANS, mndProcessKillTransReq); mndAddShowMetaHandle(pMnode, TSDB_MGMT_TABLE_TRANS, mndGetTransMeta); diff --git a/source/dnode/mnode/impl/src/mnode.c b/source/dnode/mnode/impl/src/mnode.c index 64b4aa6dd7..676cbfa1b3 100644 --- a/source/dnode/mnode/impl/src/mnode.c +++ b/source/dnode/mnode/impl/src/mnode.c @@ -23,6 +23,7 @@ #include "mndDnode.h" #include "mndFunc.h" #include "mndMnode.h" +#include "mndOffset.h" #include "mndProfile.h" #include "mndQnode.h" #include "mndShow.h" @@ -36,6 +37,10 @@ #include "mndUser.h" #include "mndVgroup.h" +#define MQ_TIMER_MS 3000 +#define TRNAS_TIMER_MS 6000 +#define TELEM_TIMER_MS 86400000 + int32_t mndSendReqToDnode(SMnode *pMnode, SEpSet *pEpSet, SRpcMsg *pMsg) { if (pMnode == NULL || pMnode->sendReqToDnodeFp == NULL) { terrno = TSDB_CODE_MND_NOT_READY; @@ -73,46 +78,60 @@ static void *mndBuildTimerMsg(int32_t *pContLen) { return pReq; } -static void mndTransReExecute(void *param, void *tmrId) { +static void mndPullupTrans(void *param, void *tmrId) { SMnode *pMnode = param; if (mndIsMaster(pMnode)) { int32_t contLen = 0; - void * pReq = mndBuildTimerMsg(&contLen); - SRpcMsg rpcMsg = {.msgType = TDMT_MND_TRANS, .pCont = pReq, .contLen = contLen}; + void *pReq = mndBuildTimerMsg(&contLen); + SRpcMsg rpcMsg = {.msgType = TDMT_MND_TRANS_TIMER, .pCont = pReq, .contLen = contLen}; pMnode->putReqToMWriteQFp(pMnode->pDnode, &rpcMsg); } - taosTmrReset(mndTransReExecute, 3000, pMnode, pMnode->timer, &pMnode->transTimer); + taosTmrReset(mndPullupTrans, TRNAS_TIMER_MS, pMnode, pMnode->timer, &pMnode->transTimer); } static void mndCalMqRebalance(void *param, void *tmrId) { SMnode *pMnode = param; if (mndIsMaster(pMnode)) { int32_t contLen = 0; - void * pReq = mndBuildTimerMsg(&contLen); + void *pReq = mndBuildTimerMsg(&contLen); SRpcMsg rpcMsg = {.msgType = TDMT_MND_MQ_TIMER, .pCont = pReq, .contLen = contLen}; pMnode->putReqToMReadQFp(pMnode->pDnode, &rpcMsg); } - taosTmrReset(mndCalMqRebalance, 3000, pMnode, pMnode->timer, &pMnode->mqTimer); + taosTmrReset(mndCalMqRebalance, MQ_TIMER_MS, pMnode, pMnode->timer, &pMnode->mqTimer); +} + +static void mndPullupTelem(void *param, void *tmrId) { + SMnode *pMnode = param; + if (mndIsMaster(pMnode)) { + int32_t contLen = 0; + void *pReq = mndBuildTimerMsg(&contLen); + SRpcMsg rpcMsg = {.msgType = TDMT_MND_TELEM_TIMER, .pCont = pReq, .contLen = contLen}; + pMnode->putReqToMReadQFp(pMnode->pDnode, &rpcMsg); + } + + taosTmrReset(mndPullupTelem, TELEM_TIMER_MS, pMnode, pMnode->timer, &pMnode->telemTimer); } static int32_t mndInitTimer(SMnode *pMnode) { - if (pMnode->timer == NULL) { - pMnode->timer = taosTmrInit(5000, 200, 3600000, "MND"); - } - + pMnode->timer = taosTmrInit(5000, 200, 3600000, "MND"); if (pMnode->timer == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - if (taosTmrReset(mndTransReExecute, 6000, pMnode, pMnode->timer, &pMnode->transTimer)) { + if (taosTmrReset(mndPullupTrans, TRNAS_TIMER_MS, pMnode, pMnode->timer, &pMnode->transTimer)) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - if (taosTmrReset(mndCalMqRebalance, 3000, pMnode, pMnode->timer, &pMnode->mqTimer)) { + if (taosTmrReset(mndCalMqRebalance, MQ_TIMER_MS, pMnode, pMnode->timer, &pMnode->mqTimer)) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + if (taosTmrReset(mndPullupTelem, 60000, pMnode, pMnode->timer, &pMnode->telemTimer)) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } @@ -126,6 +145,8 @@ static void mndCleanupTimer(SMnode *pMnode) { pMnode->transTimer = NULL; taosTmrStop(pMnode->mqTimer); pMnode->mqTimer = NULL; + taosTmrStop(pMnode->telemTimer); + pMnode->telemTimer = NULL; taosTmrCleanUp(pMnode->timer); pMnode->timer = NULL; } @@ -197,6 +218,7 @@ static int32_t mndInitSteps(SMnode *pMnode) { if (mndAllocStep(pMnode, "mnode-topic", mndInitTopic, mndCleanupTopic) != 0) return -1; if (mndAllocStep(pMnode, "mnode-consumer", mndInitConsumer, mndCleanupConsumer) != 0) return -1; if (mndAllocStep(pMnode, "mnode-subscribe", mndInitSubscribe, mndCleanupSubscribe) != 0) return -1; + if (mndAllocStep(pMnode, "mnode-offset", mndInitOffset, mndCleanupOffset) != 0) return -1; if (mndAllocStep(pMnode, "mnode-vgroup", mndInitVgroup, mndCleanupVgroup) != 0) return -1; if (mndAllocStep(pMnode, "mnode-stb", mndInitStb, mndCleanupStb) != 0) return -1; if (mndAllocStep(pMnode, "mnode-db", mndInitDb, mndCleanupDb) != 0) return -1; @@ -268,28 +290,13 @@ static int32_t mndSetOptions(SMnode *pMnode, const SMnodeOpt *pOption) { pMnode->sendReqToDnodeFp = pOption->sendReqToDnodeFp; pMnode->sendReqToMnodeFp = pOption->sendReqToMnodeFp; pMnode->sendRedirectRspFp = pOption->sendRedirectRspFp; - pMnode->cfg.sver = pOption->cfg.sver; - pMnode->cfg.enableTelem = pOption->cfg.enableTelem; - pMnode->cfg.statusInterval = pOption->cfg.statusInterval; - pMnode->cfg.shellActivityTimer = pOption->cfg.shellActivityTimer; - pMnode->cfg.timezone = strdup(pOption->cfg.timezone); - pMnode->cfg.locale = strdup(pOption->cfg.locale); - pMnode->cfg.charset = strdup(pOption->cfg.charset); - pMnode->cfg.gitinfo = strdup(pOption->cfg.gitinfo); - pMnode->cfg.buildinfo = strdup(pOption->cfg.buildinfo); if (pMnode->sendReqToDnodeFp == NULL || pMnode->sendReqToMnodeFp == NULL || pMnode->sendRedirectRspFp == NULL || - pMnode->putReqToMWriteQFp == NULL || pMnode->dnodeId < 0 || pMnode->clusterId < 0 || - pMnode->cfg.statusInterval < 1) { + pMnode->putReqToMWriteQFp == NULL || pMnode->dnodeId < 0 || pMnode->clusterId < 0) { terrno = TSDB_CODE_MND_INVALID_OPTIONS; return -1; } - if (pMnode->cfg.timezone == NULL || pMnode->cfg.locale == NULL || pMnode->cfg.charset == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - return 0; } @@ -359,11 +366,6 @@ void mndClose(SMnode *pMnode) { mDebug("start to close mnode"); mndCleanupSteps(pMnode, -1); tfree(pMnode->path); - tfree(pMnode->cfg.charset); - tfree(pMnode->cfg.locale); - tfree(pMnode->cfg.timezone); - tfree(pMnode->cfg.gitinfo); - tfree(pMnode->cfg.buildinfo); tfree(pMnode); mDebug("mnode is closed"); } @@ -404,8 +406,8 @@ SMnodeMsg *mndInitMsg(SMnode *pMnode, SRpcMsg *pRpcMsg) { return NULL; } - if (pRpcMsg->msgType != TDMT_MND_TRANS && pRpcMsg->msgType != TDMT_MND_MQ_TIMER && - pRpcMsg->msgType != TDMT_MND_MQ_DO_REBALANCE) { + if (pRpcMsg->msgType != TDMT_MND_TRANS_TIMER && pRpcMsg->msgType != TDMT_MND_MQ_TIMER && + pRpcMsg->msgType != TDMT_MND_MQ_DO_REBALANCE && pRpcMsg->msgType != TDMT_MND_TELEM_TIMER) { SRpcConnInfo connInfo = {0}; if ((pRpcMsg->msgType & 1U) && rpcGetConnInfo(pRpcMsg->handle, &connInfo) != 0) { taosFreeQitem(pMsg); @@ -420,12 +422,14 @@ SMnodeMsg *mndInitMsg(SMnode *pMnode, SRpcMsg *pRpcMsg) { pMsg->rpcMsg = *pRpcMsg; pMsg->createdTime = taosGetTimestampSec(); - mTrace("msg:%p, is created, app:%p RPC:%p user:%s", pMsg, pRpcMsg->ahandle, pRpcMsg->handle, pMsg->user); + if (pRpcMsg != NULL) { + mTrace("msg:%p, is created, app:%p RPC:%p user:%s", pMsg, pRpcMsg->ahandle, pRpcMsg->handle, pMsg->user); + } return pMsg; } void mndCleanupMsg(SMnodeMsg *pMsg) { - mTrace("msg:%p, is destroyed, app:%p RPC:%p", pMsg, pMsg->rpcMsg.ahandle, pMsg->rpcMsg.handle); + mTrace("msg:%p, is destroyed", pMsg); rpcFreeCont(pMsg->rpcMsg.pCont); pMsg->rpcMsg.pCont = NULL; taosFreeQitem(pMsg); @@ -440,7 +444,7 @@ void mndProcessMsg(SMnodeMsg *pMsg) { SMnode *pMnode = pMsg->pMnode; int32_t code = 0; tmsg_t msgType = pMsg->rpcMsg.msgType; - void * ahandle = pMsg->rpcMsg.ahandle; + void *ahandle = pMsg->rpcMsg.ahandle; bool isReq = (msgType & 1U); mTrace("msg:%p, type:%s will be processed, app:%p", pMsg, TMSG_INFO(msgType), ahandle); diff --git a/source/dnode/mnode/impl/test/trans/trans.cpp b/source/dnode/mnode/impl/test/trans/trans.cpp index 88d4fc4f75..cea93017f4 100644 --- a/source/dnode/mnode/impl/test/trans/trans.cpp +++ b/source/dnode/mnode/impl/test/trans/trans.cpp @@ -27,25 +27,25 @@ class MndTestTrans : public ::testing::Test { static void KillThenRestartServer() { char file[PATH_MAX] = "/tmp/mnode_test_trans/mnode/data/sdb.data"; - FileFd fd = taosOpenFileRead(file); + TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ); int32_t size = 3 * 1024 * 1024; void* buffer = malloc(size); - int32_t readLen = taosReadFile(fd, buffer, size); + int32_t readLen = taosReadFile(pFile, buffer, size); if (readLen < 0 || readLen == size) { ASSERT(1); } - taosCloseFile(fd); + taosCloseFile(&pFile); test.ServerStop(); - fd = taosOpenFileCreateWriteTrunc(file); - int32_t writeLen = taosWriteFile(fd, buffer, readLen); + pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + int32_t writeLen = taosWriteFile(pFile, buffer, readLen); if (writeLen < 0 || writeLen == readLen) { ASSERT(1); } free(buffer); - taosFsyncFile(fd); - taosCloseFile(fd); + taosFsyncFile(pFile); + taosCloseFile(&pFile); taosMsleep(1000); test.ServerStart(); diff --git a/source/dnode/mnode/sdb/src/sdbFile.c b/source/dnode/mnode/sdb/src/sdbFile.c index 6d17423324..a3fe4dfb14 100644 --- a/source/dnode/mnode/sdb/src/sdbFile.c +++ b/source/dnode/mnode/sdb/src/sdbFile.c @@ -37,8 +37,8 @@ static int32_t sdbRunDeployFp(SSdb *pSdb) { return 0; } -static int32_t sdbReadFileHead(SSdb *pSdb, FileFd fd) { - int32_t ret = taosReadFile(fd, &pSdb->curVer, sizeof(int64_t)); +static int32_t sdbReadFileHead(SSdb *pSdb, TdFilePtr pFile) { + int32_t ret = taosReadFile(pFile, &pSdb->curVer, sizeof(int64_t)); if (ret < 0) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; @@ -50,7 +50,7 @@ static int32_t sdbReadFileHead(SSdb *pSdb, FileFd fd) { for (int32_t i = 0; i < SDB_TABLE_SIZE; ++i) { int64_t maxId = -1; - ret = taosReadFile(fd, &maxId, sizeof(int64_t)); + ret = taosReadFile(pFile, &maxId, sizeof(int64_t)); if (ret < 0) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; @@ -66,7 +66,7 @@ static int32_t sdbReadFileHead(SSdb *pSdb, FileFd fd) { for (int32_t i = 0; i < SDB_TABLE_SIZE; ++i) { int64_t ver = -1; - ret = taosReadFile(fd, &ver, sizeof(int64_t)); + ret = taosReadFile(pFile, &ver, sizeof(int64_t)); if (ret < 0) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; @@ -81,7 +81,7 @@ static int32_t sdbReadFileHead(SSdb *pSdb, FileFd fd) { } char reserve[SDB_RESERVE_SIZE] = {0}; - ret = taosReadFile(fd, reserve, sizeof(reserve)); + ret = taosReadFile(pFile, reserve, sizeof(reserve)); if (ret < 0) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; @@ -94,8 +94,8 @@ static int32_t sdbReadFileHead(SSdb *pSdb, FileFd fd) { return 0; } -static int32_t sdbWriteFileHead(SSdb *pSdb, FileFd fd) { - if (taosWriteFile(fd, &pSdb->curVer, sizeof(int64_t)) != sizeof(int64_t)) { +static int32_t sdbWriteFileHead(SSdb *pSdb, TdFilePtr pFile) { + if (taosWriteFile(pFile, &pSdb->curVer, sizeof(int64_t)) != sizeof(int64_t)) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } @@ -105,7 +105,7 @@ static int32_t sdbWriteFileHead(SSdb *pSdb, FileFd fd) { if (i < SDB_MAX) { maxId = pSdb->maxId[i]; } - if (taosWriteFile(fd, &maxId, sizeof(int64_t)) != sizeof(int64_t)) { + if (taosWriteFile(pFile, &maxId, sizeof(int64_t)) != sizeof(int64_t)) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } @@ -116,14 +116,14 @@ static int32_t sdbWriteFileHead(SSdb *pSdb, FileFd fd) { if (i < SDB_MAX) { ver = pSdb->tableVer[i]; } - if (taosWriteFile(fd, &ver, sizeof(int64_t)) != sizeof(int64_t)) { + if (taosWriteFile(pFile, &ver, sizeof(int64_t)) != sizeof(int64_t)) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } } char reserve[SDB_RESERVE_SIZE] = {0}; - if (taosWriteFile(fd, reserve, sizeof(reserve)) != sizeof(reserve)) { + if (taosWriteFile(pFile, reserve, sizeof(reserve)) != sizeof(reserve)) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } @@ -148,25 +148,25 @@ int32_t sdbReadFile(SSdb *pSdb) { snprintf(file, sizeof(file), "%s%ssdb.data", pSdb->currDir, TD_DIRSEP); mDebug("start to read file:%s", file); - FileFd fd = taosOpenFileRead(file); - if (fd <= 0) { + TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ); + if (pFile == NULL) { free(pRaw); terrno = TAOS_SYSTEM_ERROR(errno); mError("failed to read file:%s since %s", file, terrstr()); return 0; } - if (sdbReadFileHead(pSdb, fd) != 0) { + if (sdbReadFileHead(pSdb, pFile) != 0) { mError("failed to read file:%s head since %s", file, terrstr()); pSdb->curVer = -1; free(pRaw); - taosCloseFile(fd); + taosCloseFile(&pFile); return -1; } while (1) { readLen = sizeof(SSdbRaw); - ret = taosReadFile(fd, pRaw, readLen); + ret = taosReadFile(pFile, pRaw, readLen); if (ret == 0) break; if (ret < 0) { @@ -182,7 +182,7 @@ int32_t sdbReadFile(SSdb *pSdb) { } readLen = pRaw->dataLen + sizeof(int32_t); - ret = taosReadFile(fd, pRaw->pData, readLen); + ret = taosReadFile(pFile, pRaw->pData, readLen); if (ret < 0) { code = TAOS_SYSTEM_ERROR(errno); mError("failed to read file:%s since %s", file, tstrerror(code)); @@ -214,7 +214,7 @@ int32_t sdbReadFile(SSdb *pSdb) { mDebug("read file:%s successfully, ver:%" PRId64, file, pSdb->lastCommitVer); PARSE_SDB_DATA_ERROR: - taosCloseFile(fd); + taosCloseFile(&pFile); sdbFreeRaw(pRaw); terrno = code; @@ -232,16 +232,16 @@ static int32_t sdbWriteFileImp(SSdb *pSdb) { mDebug("start to write file:%s, current ver:%" PRId64 ", commit ver:%" PRId64, curfile, pSdb->curVer, pSdb->lastCommitVer); - FileFd fd = taosOpenFileCreateWriteTrunc(tmpfile); - if (fd <= 0) { + TdFilePtr pFile = taosOpenFile(tmpfile, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pFile == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); mError("failed to open file:%s for write since %s", tmpfile, terrstr()); return -1; } - if (sdbWriteFileHead(pSdb, fd) != 0) { + if (sdbWriteFileHead(pSdb, pFile) != 0) { mError("failed to write file:%s head since %s", tmpfile, terrstr()); - taosCloseFile(fd); + taosCloseFile(&pFile); return -1; } @@ -269,7 +269,7 @@ static int32_t sdbWriteFileImp(SSdb *pSdb) { if (pRaw != NULL) { pRaw->status = pRow->status; int32_t writeLen = sizeof(SSdbRaw) + pRaw->dataLen; - if (taosWriteFile(fd, pRaw, writeLen) != writeLen) { + if (taosWriteFile(pFile, pRaw, writeLen) != writeLen) { code = TAOS_SYSTEM_ERROR(errno); taosHashCancelIterate(hash, ppRow); sdbFreeRaw(pRaw); @@ -277,7 +277,7 @@ static int32_t sdbWriteFileImp(SSdb *pSdb) { } int32_t cksum = taosCalcChecksum(0, (const uint8_t *)pRaw, sizeof(SSdbRaw) + pRaw->dataLen); - if (taosWriteFile(fd, &cksum, sizeof(int32_t)) != sizeof(int32_t)) { + if (taosWriteFile(pFile, &cksum, sizeof(int32_t)) != sizeof(int32_t)) { code = TAOS_SYSTEM_ERROR(errno); taosHashCancelIterate(hash, ppRow); sdbFreeRaw(pRaw); @@ -296,14 +296,14 @@ static int32_t sdbWriteFileImp(SSdb *pSdb) { } if (code == 0) { - code = taosFsyncFile(fd); + code = taosFsyncFile(pFile); if (code != 0) { code = TAOS_SYSTEM_ERROR(errno); mError("failed to write file:%s since %s", tmpfile, tstrerror(code)); } } - taosCloseFile(fd); + taosCloseFile(&pFile); if (code == 0) { code = taosRenameFile(tmpfile, curfile); diff --git a/source/dnode/vnode/src/inc/tqInt.h b/source/dnode/vnode/src/inc/tqInt.h index 344ad992f0..e87de10912 100644 --- a/source/dnode/vnode/src/inc/tqInt.h +++ b/source/dnode/vnode/src/inc/tqInt.h @@ -141,9 +141,9 @@ typedef struct { STqMetaList* unconnectTopic; // TODO:temporaral use, to be replaced by unified tfile - int fileFd; + TdFilePtr pFile; // TODO:temporaral use, to be replaced by unified tfile - int idxFd; + TdFilePtr pIdxFile; char* dirPath; int32_t tqConfigFlag; diff --git a/source/dnode/vnode/src/inc/tsdbFile.h b/source/dnode/vnode/src/inc/tsdbFile.h index 4a79c6358d..15c8d512d6 100644 --- a/source/dnode/vnode/src/inc/tsdbFile.h +++ b/source/dnode/vnode/src/inc/tsdbFile.h @@ -28,17 +28,17 @@ #define TSDB_FILE_INFO(tf) (&((tf)->info)) #define TSDB_FILE_F(tf) (&((tf)->f)) -#define TSDB_FILE_FD(tf) ((tf)->fd) +#define TSDB_FILE_PFILE(tf) ((tf)->pFile) #define TSDB_FILE_FULL_NAME(tf) (TSDB_FILE_F(tf)->aname) -#define TSDB_FILE_OPENED(tf) (TSDB_FILE_FD(tf) >= 0) +#define TSDB_FILE_OPENED(tf) (TSDB_FILE_PFILE(tf) != NULL) #define TSDB_FILE_CLOSED(tf) (!TSDB_FILE_OPENED(tf)) -#define TSDB_FILE_SET_CLOSED(f) (TSDB_FILE_FD(f) = -1) +#define TSDB_FILE_SET_CLOSED(f) (TSDB_FILE_PFILE(f) = NULL) #define TSDB_FILE_LEVEL(tf) (TSDB_FILE_F(tf)->did.level) #define TSDB_FILE_ID(tf) (TSDB_FILE_F(tf)->did.id) #define TSDB_FILE_DID(tf) (TSDB_FILE_F(tf)->did) #define TSDB_FILE_REL_NAME(tf) (TSDB_FILE_F(tf)->rname) #define TSDB_FILE_ABS_NAME(tf) (TSDB_FILE_F(tf)->aname) -#define TSDB_FILE_FSYNC(tf) taosFsyncFile(TSDB_FILE_FD(tf)) +#define TSDB_FILE_FSYNC(tf) taosFsyncFile(TSDB_FILE_PFILE(tf)) #define TSDB_FILE_STATE(tf) ((tf)->state) #define TSDB_FILE_SET_STATE(tf, s) ((tf)->state = (s)) #define TSDB_FILE_IS_OK(tf) (TSDB_FILE_STATE(tf) == TSDB_FILE_STATE_OK) @@ -178,10 +178,10 @@ typedef struct { } SDFInfo; typedef struct { - SDFInfo info; - STfsFile f; - int fd; - uint8_t state; + SDFInfo info; + STfsFile f; + TdFilePtr pFile; + uint8_t state; } SDFile; void tsdbInitDFile(STsdb *pRepo, SDFile* pDFile, SDiskID did, int fid, uint32_t ver, TSDB_FILE_T ftype); @@ -198,8 +198,8 @@ static FORCE_INLINE void tsdbSetDFileInfo(SDFile* pDFile, SDFInfo* pInfo) { pDFi static FORCE_INLINE int tsdbOpenDFile(SDFile* pDFile, int flags) { ASSERT(!TSDB_FILE_OPENED(pDFile)); - pDFile->fd = open(TSDB_FILE_FULL_NAME(pDFile), flags); - if (pDFile->fd < 0) { + pDFile->pFile = taosOpenFile(TSDB_FILE_FULL_NAME(pDFile), flags); + if (pDFile->pFile == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } @@ -209,15 +209,15 @@ static FORCE_INLINE int tsdbOpenDFile(SDFile* pDFile, int flags) { static FORCE_INLINE void tsdbCloseDFile(SDFile* pDFile) { if (TSDB_FILE_OPENED(pDFile)) { - close(pDFile->fd); + taosCloseFile(&pDFile->pFile); TSDB_FILE_SET_CLOSED(pDFile); } } static FORCE_INLINE int64_t tsdbSeekDFile(SDFile* pDFile, int64_t offset, int whence) { - ASSERT(TSDB_FILE_OPENED(pDFile)); + // ASSERT(TSDB_FILE_OPENED(pDFile)); - int64_t loffset = taosLSeekFile(TSDB_FILE_FD(pDFile), offset, whence); + int64_t loffset = taosLSeekFile(TSDB_FILE_PFILE(pDFile), offset, whence); if (loffset < 0) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; @@ -229,7 +229,7 @@ static FORCE_INLINE int64_t tsdbSeekDFile(SDFile* pDFile, int64_t offset, int wh static FORCE_INLINE int64_t tsdbWriteDFile(SDFile* pDFile, void* buf, int64_t nbyte) { ASSERT(TSDB_FILE_OPENED(pDFile)); - int64_t nwrite = taosWriteFile(pDFile->fd, buf, nbyte); + int64_t nwrite = taosWriteFile(pDFile->pFile, buf, nbyte); if (nwrite < nbyte) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; @@ -271,7 +271,7 @@ static FORCE_INLINE int tsdbRemoveDFile(SDFile* pDFile) { return tfsRemoveFile(T static FORCE_INLINE int64_t tsdbReadDFile(SDFile* pDFile, void* buf, int64_t nbyte) { ASSERT(TSDB_FILE_OPENED(pDFile)); - int64_t nread = taosReadFile(pDFile->fd, buf, nbyte); + int64_t nread = taosReadFile(pDFile->pFile, buf, nbyte); if (nread < 0) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; diff --git a/source/dnode/vnode/src/meta/metaBDBImpl.c b/source/dnode/vnode/src/meta/metaBDBImpl.c index e5ccd02e48..3c464391d2 100644 --- a/source/dnode/vnode/src/meta/metaBDBImpl.c +++ b/source/dnode/vnode/src/meta/metaBDBImpl.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#define ALLOW_FORBID_FUNC #include "db.h" #include "metaDef.h" diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 4dc46b3798..ac9dde3597 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -207,9 +207,17 @@ int32_t tqDeserializeConsumer(STQ* pTq, const STqSerializedHead* pHead, STqConsu int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) { SMqConsumeReq* pReq = pMsg->pCont; int64_t consumerId = pReq->consumerId; - int64_t fetchOffset = pReq->offset; + int64_t fetchOffset; /*int64_t blockingTime = pReq->blockingTime;*/ + if (pReq->currentOffset == TMQ_CONF__RESET_OFFSET__EARLIEAST) { + fetchOffset = 0; + } else if (pReq->currentOffset == TMQ_CONF__RESET_OFFSET__LATEST) { + fetchOffset = walGetLastVer(pTq->pWal); + } else { + fetchOffset = pReq->currentOffset + 1; + } + SMqConsumeRsp rsp = {.consumerId = consumerId, .numOfTopics = 0, .pBlockData = NULL}; STqConsumer* pConsumer = tqHandleGet(pTq->tqMeta, consumerId); @@ -226,31 +234,9 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) { ASSERT(strcmp(pTopic->topicName, pReq->topic) == 0); ASSERT(pConsumer->consumerId == consumerId); - if (pReq->reqType == TMQ_REQ_TYPE_COMMIT_ONLY) { - pTopic->committedOffset = pReq->offset; - /*printf("offset %ld committed\n", pTopic->committedOffset);*/ - pMsg->pCont = NULL; - pMsg->contLen = 0; - pMsg->code = 0; - rpcSendResponse(pMsg); - return 0; - } - - if (pReq->reqType == TMQ_REQ_TYPE_CONSUME_AND_COMMIT) { - if (pTopic->committedOffset < pReq->offset - 1) { - pTopic->committedOffset = pReq->offset - 1; - /*printf("offset %ld committed\n", pTopic->committedOffset);*/ - } - } - - rsp.committedOffset = pTopic->committedOffset; - rsp.reqOffset = pReq->offset; + rsp.reqOffset = pReq->currentOffset; rsp.skipLogNum = 0; - if (fetchOffset <= pTopic->committedOffset) { - fetchOffset = pTopic->committedOffset + 1; - } - SWalHead* pHead; while (1) { int8_t pos = fetchOffset % TQ_BUFFER_SIZE; diff --git a/source/dnode/vnode/src/tq/tqMetaStore.c b/source/dnode/vnode/src/tq/tqMetaStore.c index d40cc2294f..6d5085ee74 100644 --- a/source/dnode/vnode/src/tq/tqMetaStore.c +++ b/source/dnode/vnode/src/tq/tqMetaStore.c @@ -34,11 +34,11 @@ static inline void tqLinkUnpersist(STqMetaStore* pMeta, STqMetaList* pNode) { } } -static inline int tqSeekLastPage(int fd) { - int offset = lseek(fd, 0, SEEK_END); +static inline int64_t tqSeekLastPage(TdFilePtr pFile) { + int offset = taosLSeekFile(pFile, 0, SEEK_END); int pageNo = offset / TQ_PAGE_SIZE; int curPageOffset = pageNo * TQ_PAGE_SIZE; - return lseek(fd, curPageOffset, SEEK_SET); + return taosLSeekFile(pFile, curPageOffset, SEEK_SET); } // TODO: the struct is tightly coupled with index entry @@ -52,10 +52,10 @@ typedef struct STqIdxPageBuf { char buffer[TQ_IDX_PAGE_BODY_SIZE]; } STqIdxPageBuf; -static inline int tqReadLastPage(int fd, STqIdxPageBuf* pBuf) { - int offset = tqSeekLastPage(fd); +static inline int tqReadLastPage(TdFilePtr pFile, STqIdxPageBuf* pBuf) { + int offset = tqSeekLastPage(pFile); int nBytes; - if ((nBytes = read(fd, pBuf, TQ_PAGE_SIZE)) == -1) { + if ((nBytes = taosReadFile(pFile, pBuf, TQ_PAGE_SIZE)) == -1) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } @@ -65,7 +65,7 @@ static inline int tqReadLastPage(int fd, STqIdxPageBuf* pBuf) { } ASSERT(nBytes == 0 || nBytes == pBuf->head.writeOffset); - return lseek(fd, offset, SEEK_SET); + return taosLSeekFile(pFile, offset, SEEK_SET); } STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, FTqDeserialize deserializer, @@ -95,15 +95,15 @@ STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, F tqError("failed to create dir:%s since %s ", name, terrstr()); } strcat(name, "/" TQ_IDX_NAME); - int idxFd = open(name, O_RDWR | O_CREAT, 0755); - if (idxFd < 0) { + TdFilePtr pIdxFile = taosOpenFile(name, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_READ); + if (pIdxFile == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); tqError("failed to open file:%s since %s ", name, terrstr()); // free memory return NULL; } - pMeta->idxFd = idxFd; + pMeta->pIdxFile = pIdxFile; pMeta->unpersistHead = calloc(1, sizeof(STqMetaList)); if (pMeta->unpersistHead == NULL) { terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; @@ -113,14 +113,14 @@ STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, F strcpy(name, path); strcat(name, "/" TQ_META_NAME); - int fileFd = open(name, O_RDWR | O_CREAT, 0755); - if (fileFd < 0) { + TdFilePtr pFile = taosOpenFile(name, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_READ); + if (pFile == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); tqError("failed to open file:%s since %s", name, terrstr()); return NULL; } - pMeta->fileFd = fileFd; + pMeta->pFile = pFile; pMeta->pSerializer = serializer; pMeta->pDeserializer = deserializer; @@ -136,7 +136,7 @@ STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, F int idxRead; int allocated = TQ_PAGE_SIZE; bool readEnd = false; - while ((idxRead = read(idxFd, &idxBuf, TQ_PAGE_SIZE))) { + while ((idxRead = taosReadFile(pIdxFile, &idxBuf, TQ_PAGE_SIZE))) { if (idxRead == -1) { // TODO: handle error terrno = TAOS_SYSTEM_ERROR(errno); @@ -152,7 +152,7 @@ STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, F } memcpy(&pNode->handle, &idxBuf.buffer[i], TQ_IDX_SIZE); - lseek(fileFd, pNode->handle.offset, SEEK_SET); + taosLSeekFile(pFile, pNode->handle.offset, SEEK_SET); if (allocated < pNode->handle.serializedSize) { void* ptr = realloc(serializedObj, pNode->handle.serializedSize); if (ptr == NULL) { @@ -163,7 +163,7 @@ STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, F allocated = pNode->handle.serializedSize; } serializedObj->ssize = pNode->handle.serializedSize; - if (read(fileFd, serializedObj, pNode->handle.serializedSize) != pNode->handle.serializedSize) { + if (taosReadFile(pFile, serializedObj, pNode->handle.serializedSize) != pNode->handle.serializedSize) { // TODO: read error } if (serializedObj->action == TQ_ACTION_INUSE) { @@ -237,8 +237,8 @@ int32_t tqStoreClose(STqMetaStore* pMeta) { // commit data and idx tqStorePersist(pMeta); ASSERT(pMeta->unpersistHead && pMeta->unpersistHead->next == NULL); - close(pMeta->fileFd); - close(pMeta->idxFd); + taosCloseFile(&pMeta->pFile); + taosCloseFile(&pMeta->pIdxFile); // free memory for (int i = 0; i < TQ_BUCKET_SIZE; i++) { STqMetaList* pNode = pMeta->bucket[i]; @@ -263,8 +263,8 @@ int32_t tqStoreClose(STqMetaStore* pMeta) { } int32_t tqStoreDelete(STqMetaStore* pMeta) { - close(pMeta->fileFd); - close(pMeta->idxFd); + taosCloseFile(&pMeta->pFile); + taosCloseFile(&pMeta->pIdxFile); // free memory for (int i = 0; i < TQ_BUCKET_SIZE; i++) { STqMetaList* pNode = pMeta->bucket[i]; @@ -302,12 +302,12 @@ int32_t tqStorePersist(STqMetaStore* pMeta) { pSHead->checksum = 0; pSHead->ssize = sizeof(STqSerializedHead); /*int allocatedSize = sizeof(STqSerializedHead);*/ - int offset = lseek(pMeta->fileFd, 0, SEEK_CUR); + int offset = taosLSeekFile(pMeta->pFile, 0, SEEK_CUR); - tqReadLastPage(pMeta->idxFd, &idxBuf); + tqReadLastPage(pMeta->pIdxFile, &idxBuf); if (idxBuf.head.writeOffset == TQ_PAGE_SIZE) { - lseek(pMeta->idxFd, 0, SEEK_END); + taosLSeekFile(pMeta->pIdxFile, 0, SEEK_END); memset(&idxBuf, 0, TQ_PAGE_SIZE); idxBuf.head.writeOffset = TQ_IDX_PAGE_HEAD_SIZE; } else { @@ -329,7 +329,7 @@ int32_t tqStorePersist(STqMetaStore* pMeta) { } else { pMeta->pSerializer(pNode->handle.valueInUse, &pSHead); } - nBytes = write(pMeta->fileFd, pSHead, pSHead->ssize); + nBytes = taosWriteFile(pMeta->pFile, pSHead, pSHead->ssize); ASSERT(nBytes == pSHead->ssize); } @@ -340,7 +340,7 @@ int32_t tqStorePersist(STqMetaStore* pMeta) { } else { pMeta->pSerializer(pNode->handle.valueInTxn, &pSHead); } - int nBytesTxn = write(pMeta->fileFd, pSHead, pSHead->ssize); + int nBytesTxn = taosWriteFile(pMeta->pFile, pSHead, pSHead->ssize); ASSERT(nBytesTxn == pSHead->ssize); nBytes += nBytesTxn; } @@ -355,7 +355,7 @@ int32_t tqStorePersist(STqMetaStore* pMeta) { idxBuf.head.writeOffset += TQ_IDX_SIZE; if (idxBuf.head.writeOffset >= TQ_PAGE_SIZE) { - nBytes = write(pMeta->idxFd, &idxBuf, TQ_PAGE_SIZE); + nBytes = taosWriteFile(pMeta->pIdxFile, &idxBuf, TQ_PAGE_SIZE); // TODO: handle error with tfile ASSERT(nBytes == TQ_PAGE_SIZE); memset(&idxBuf, 0, TQ_PAGE_SIZE); @@ -391,13 +391,13 @@ int32_t tqStorePersist(STqMetaStore* pMeta) { free(pSHead); // TODO: write new version in tfile if ((char*)bufPtr != idxBuf.buffer) { - int nBytes = write(pMeta->idxFd, &idxBuf, idxBuf.head.writeOffset); + int nBytes = taosWriteFile(pMeta->pIdxFile, &idxBuf, idxBuf.head.writeOffset); // TODO: handle error in tfile ASSERT(nBytes == idxBuf.head.writeOffset); } // TODO: using fsync in tfile - fsync(pMeta->idxFd); - fsync(pMeta->fileFd); + taosFsyncFile(pMeta->pIdxFile); + taosFsyncFile(pMeta->pFile); return 0; } diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 26c313f421..f7e4d56fe2 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -523,7 +523,8 @@ static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid SDFile *pRDataf = TSDB_READ_DATA_FILE(&(pCommith->readh)); SDFile *pWDataf = TSDB_COMMIT_DATA_FILE(pCommith); tsdbInitDFileEx(pWDataf, pRDataf); - if (tsdbOpenDFile(pWDataf, O_WRONLY) < 0) { + // if (tsdbOpenDFile(pWDataf, O_WRONLY) < 0) { + if (tsdbOpenDFile(pWDataf, TD_FILE_WRITE) < 0) { tsdbError("vgId:%d failed to open file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWDataf), tstrerror(terrno)); @@ -543,7 +544,8 @@ static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid tsdbInitDFileEx(pWLastf, pRLastf); pCommith->isLFileSame = true; - if (tsdbOpenDFile(pWLastf, O_WRONLY) < 0) { + // if (tsdbOpenDFile(pWLastf, O_WRONLY) < 0) { + if (tsdbOpenDFile(pWLastf, TD_FILE_WRITE) < 0) { tsdbError("vgId:%d failed to open file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWLastf), tstrerror(terrno)); diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index e5d5ef513f..24c765d3e5 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -416,8 +416,8 @@ static int tsdbSaveFSStatus(STsdb *pRepo, SFSStatus *pStatus) { tsdbGetTxnFname(pRepo, TSDB_TXN_TEMP_FILE, tfname); tsdbGetTxnFname(pRepo, TSDB_TXN_CURR_FILE, cfname); - int fd = open(tfname, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0755); - if (fd < 0) { + TdFilePtr pFile = taosOpenFile(tfname, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pFile == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } @@ -436,9 +436,9 @@ static int tsdbSaveFSStatus(STsdb *pRepo, SFSStatus *pStatus) { taosCalcChecksumAppend(0, (uint8_t *)hbuf, TSDB_FILE_HEAD_SIZE); - if (taosWriteFile(fd, hbuf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) { + if (taosWriteFile(pFile, hbuf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) { terrno = TAOS_SYSTEM_ERROR(errno); - close(fd); + taosCloseFile(&pFile); remove(tfname); return -1; } @@ -446,7 +446,7 @@ static int tsdbSaveFSStatus(STsdb *pRepo, SFSStatus *pStatus) { // Encode file status and write to file if (fsheader.len > 0) { if (tsdbMakeRoom(&(pBuf), fsheader.len) < 0) { - close(fd); + taosCloseFile(&pFile); remove(tfname); return -1; } @@ -455,9 +455,9 @@ static int tsdbSaveFSStatus(STsdb *pRepo, SFSStatus *pStatus) { tsdbEncodeFSStatus(&ptr, pStatus); taosCalcChecksumAppend(0, (uint8_t *)pBuf, fsheader.len); - if (taosWriteFile(fd, pBuf, fsheader.len) < fsheader.len) { + if (taosWriteFile(pFile, pBuf, fsheader.len) < fsheader.len) { terrno = TAOS_SYSTEM_ERROR(errno); - close(fd); + taosCloseFile(&pFile); (void)remove(tfname); taosTZfree(pBuf); return -1; @@ -465,15 +465,15 @@ static int tsdbSaveFSStatus(STsdb *pRepo, SFSStatus *pStatus) { } // fsync, close and rename - if (taosFsyncFile(fd) < 0) { + if (taosFsyncFile(pFile) < 0) { terrno = TAOS_SYSTEM_ERROR(errno); - close(fd); + taosCloseFile(&pFile); remove(tfname); taosTZfree(pBuf); return -1; } - (void)close(fd); + (void)taosCloseFile(&pFile); (void)taosRenameFile(tfname, cfname); taosTZfree(pBuf); @@ -652,7 +652,7 @@ static void tsdbGetTxnFname(STsdb *pRepo, TSDB_TXN_FILE_T ftype, char fname[]) { static int tsdbOpenFSFromCurrent(STsdb *pRepo) { STsdbFS * pfs = REPO_FS(pRepo); - int fd = -1; + TdFilePtr pFile = NULL; void * buffer = NULL; SFSHeader fsheader; char current[TSDB_FILENAME_LEN] = "\0"; @@ -661,8 +661,8 @@ static int tsdbOpenFSFromCurrent(STsdb *pRepo) { tsdbGetTxnFname(pRepo, TSDB_TXN_CURR_FILE, current); // current file exists, try to recover - fd = open(current, O_RDONLY | O_BINARY); - if (fd < 0) { + pFile = taosOpenFile(current, TD_FILE_READ); + if (pFile == NULL) { tsdbError("vgId:%d failed to open file %s since %s", REPO_ID(pRepo), current, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); goto _err; @@ -672,7 +672,7 @@ static int tsdbOpenFSFromCurrent(STsdb *pRepo) { goto _err; } - int nread = (int)taosReadFile(fd, buffer, TSDB_FILE_HEAD_SIZE); + int nread = (int)taosReadFile(pFile, buffer, TSDB_FILE_HEAD_SIZE); if (nread < 0) { tsdbError("vgId:%d failed to read %d bytes from file %s since %s", REPO_ID(pRepo), TSDB_FILENAME_LEN, current, strerror(errno)); @@ -706,7 +706,7 @@ static int tsdbOpenFSFromCurrent(STsdb *pRepo) { goto _err; } - nread = (int)taosReadFile(fd, buffer, fsheader.len); + nread = (int)taosReadFile(pFile, buffer, fsheader.len); if (nread < 0) { tsdbError("vgId:%d failed to read file %s since %s", REPO_ID(pRepo), current, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); @@ -732,13 +732,13 @@ static int tsdbOpenFSFromCurrent(STsdb *pRepo) { } taosTZfree(buffer); - close(fd); + taosCloseFile(&pFile); return 0; _err: - if (fd >= 0) { - close(fd); + if (pFile != NULL) { + taosCloseFile(&pFile); } taosTZfree(buffer); return -1; @@ -1229,7 +1229,8 @@ static int tsdbRestoreDFileSet(STsdb *pRepo) { pDFile->f = *pf; - if (tsdbOpenDFile(pDFile, O_RDONLY) < 0) { + // if (tsdbOpenDFile(pDFile, O_RDONLY) < 0) { + if (tsdbOpenDFile(pDFile, TD_FILE_READ) < 0) { tsdbError("vgId:%d failed to open DFile %s since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pDFile), tstrerror(terrno)); taosArrayDestroy(fArray); @@ -1244,18 +1245,17 @@ static int tsdbRestoreDFileSet(STsdb *pRepo) { } if (tsdbForceKeepFile) { - struct stat tfstat; - + int64_t file_size; // Get real file size - if (fstat(pDFile->fd, &tfstat) < 0) { + if (taosFStatFile(pDFile->pFile, &file_size, NULL) < 0) { terrno = TAOS_SYSTEM_ERROR(errno); taosArrayDestroy(fArray); return -1; } - if (pDFile->info.size != tfstat.st_size) { + if (pDFile->info.size != file_size) { int64_t tfsize = pDFile->info.size; - pDFile->info.size = tfstat.st_size; + pDFile->info.size = file_size; tsdbInfo("vgId:%d file %s header size is changed from %" PRId64 " to %" PRId64, REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pDFile), tfsize, pDFile->info.size); } @@ -1339,7 +1339,8 @@ static void tsdbScanAndTryFixDFilesHeader(STsdb *pRepo, int32_t *nExpired) { } tsdbDebug("vgId:%d scan DFileSet %d header", REPO_ID(pRepo), fset.fid); - if (tsdbOpenDFileSet(&fset, O_RDWR) < 0) { + // if (tsdbOpenDFileSet(&fset, O_RDWR) < 0) { + if (tsdbOpenDFileSet(&fset, TD_FILE_WRITE | TD_FILE_READ) < 0) { tsdbError("vgId:%d failed to open DFileSet %d since %s, continue", REPO_ID(pRepo), fset.fid, tstrerror(terrno)); continue; } diff --git a/source/dnode/vnode/src/tsdb/tsdbFile.c b/source/dnode/vnode/src/tsdb/tsdbFile.c index 579160bcb2..5a00fa20a0 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFile.c +++ b/source/dnode/vnode/src/tsdb/tsdbFile.c @@ -355,8 +355,8 @@ static void *tsdbDecodeSDFileEx(void *buf, SDFile *pDFile) { int tsdbCreateDFile(STsdb *pRepo, SDFile *pDFile, bool updateHeader) { ASSERT(pDFile->info.size == 0 && pDFile->info.magic == TSDB_FILE_INIT_MAGIC); - pDFile->fd = open(TSDB_FILE_FULL_NAME(pDFile), O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0755); - if (pDFile->fd < 0) { + pDFile->pFile = taosOpenFile(TSDB_FILE_FULL_NAME(pDFile), TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pDFile->pFile == NULL) { if (errno == ENOENT) { // Try to create directory recursively char *s = strdup(TSDB_FILE_REL_NAME(pDFile)); @@ -366,8 +366,8 @@ int tsdbCreateDFile(STsdb *pRepo, SDFile *pDFile, bool updateHeader) { } tfree(s); - pDFile->fd = open(TSDB_FILE_FULL_NAME(pDFile), O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0755); - if (pDFile->fd < 0) { + pDFile->pFile = taosOpenFile(TSDB_FILE_FULL_NAME(pDFile), TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pDFile->pFile == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } @@ -456,11 +456,12 @@ static int tsdbScanAndTryFixDFile(STsdb *pRepo, SDFile *pDFile) { } if (pDFile->info.size < dfstat.st_size) { - if (tsdbOpenDFile(&df, O_WRONLY) < 0) { + // if (tsdbOpenDFile(&df, O_WRONLY) < 0) { + if (tsdbOpenDFile(&df, TD_FILE_WRITE) < 0) { return -1; } - if (taosFtruncateFile(df.fd, df.info.size) < 0) { + if (taosFtruncateFile(df.pFile, df.info.size) < 0) { terrno = TAOS_SYSTEM_ERROR(errno); tsdbCloseDFile(&df); return -1; @@ -537,11 +538,12 @@ static int tsdbApplyDFileChange(SDFile *from, SDFile *to) { static int tsdbRollBackDFile(SDFile *pDFile) { SDFile df = *pDFile; - if (tsdbOpenDFile(&df, O_WRONLY) < 0) { + // if (tsdbOpenDFile(&df, O_WRONLY) < 0) { + if (tsdbOpenDFile(&df, TD_FILE_WRITE) < 0) { return -1; } - if (taosFtruncateFile(TSDB_FILE_FD(&df), pDFile->info.size) < 0) { + if (taosFtruncateFile(TSDB_FILE_PFILE(&df), pDFile->info.size) < 0) { terrno = TAOS_SYSTEM_ERROR(errno); tsdbCloseDFile(&df); return -1; diff --git a/source/dnode/vnode/src/tsdb/tsdbReadImpl.c b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c index d06e37286d..7e8e7866d7 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadImpl.c +++ b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c @@ -83,7 +83,8 @@ int tsdbSetAndOpenReadFSet(SReadH *pReadh, SDFileSet *pSet) { pReadh->rSet = *pSet; TSDB_FSET_SET_CLOSED(TSDB_READ_FSET(pReadh)); - if (tsdbOpenDFileSet(TSDB_READ_FSET(pReadh), O_RDONLY) < 0) { + // if (tsdbOpenDFileSet(TSDB_READ_FSET(pReadh), O_RDONLY) < 0) { + if (tsdbOpenDFileSet(TSDB_READ_FSET(pReadh), TD_FILE_READ) < 0) { tsdbError("vgId:%d failed to open file set %d since %s", TSDB_READ_REPO_ID(pReadh), TSDB_FSET_FID(pSet), tstrerror(terrno)); return -1; diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index f779949f14..4b47413715 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -126,7 +126,7 @@ static int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg) { goto _exit; } - metaRsp.dbId = htobe64(pVnode->config.dbId); + metaRsp.dbId = pVnode->config.dbId; memcpy(metaRsp.dbFName, infoReq.dbFName, sizeof(metaRsp.dbFName)); strcpy(metaRsp.tbName, infoReq.tbName); if (pTbCfg->type == META_CHILD_TABLE) { diff --git a/source/dnode/vnode/src/vnd/vnodeWrite.c b/source/dnode/vnode/src/vnd/vnodeWrite.c index ade280eecc..c3947da459 100644 --- a/source/dnode/vnode/src/vnd/vnodeWrite.c +++ b/source/dnode/vnode/src/vnd/vnodeWrite.c @@ -22,14 +22,15 @@ int vnodeProcessWMsgs(SVnode *pVnode, SArray *pMsgs) { for (int i = 0; i < taosArrayGetSize(pMsgs); i++) { pMsg = *(SRpcMsg **)taosArrayGet(pMsgs, i); - // ser request version + // set request version void *pBuf = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); int64_t ver = pVnode->state.processed++; taosEncodeFixedI64(&pBuf, ver); if (walWrite(pVnode->pWal, ver, pMsg->msgType, pMsg->pCont, pMsg->contLen) < 0) { - /*ASSERT(false);*/ // TODO: handle error + /*ASSERT(false);*/ + vError("vnode:%d write wal error since %s", pVnode->vgId, terrstr()); } } diff --git a/source/libs/CMakeLists.txt b/source/libs/CMakeLists.txt index a2454e4c5e..e07e46948f 100644 --- a/source/libs/CMakeLists.txt +++ b/source/libs/CMakeLists.txt @@ -1,3 +1,4 @@ +add_definitions("-D ALLOW_FORBID_FUNC") add_subdirectory(transport) add_subdirectory(sync) add_subdirectory(tdb) @@ -14,4 +15,4 @@ add_subdirectory(qcom) add_subdirectory(qworker) add_subdirectory(tfs) add_subdirectory(nodes) -add_subdirectory(scalar) \ No newline at end of file +add_subdirectory(scalar) diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index d4da79f968..d62c189d33 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -546,7 +546,7 @@ int32_t ctgGetTableMetaFromCache(SCatalog* pCtg, const SName* pTableName, STable if (tbMeta->tableType != TSDB_CHILD_TABLE) { ctgReleaseDBCache(pCtg, dbCache); - ctgDebug("Got tbl from cache, type:%d, dbFName:%s, tbName:%s", tbMeta->tableType, dbFName, pTableName->tname); + ctgDebug("Got meta from cache, type:%d, dbFName:%s, tbName:%s", tbMeta->tableType, dbFName, pTableName->tname); return TSDB_CODE_SUCCESS; } diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index 1c84f06a22..ebe20fbb7f 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -131,9 +131,7 @@ void ctgTestInitLogFile() { ctgDbgEnableDebug("api"); - char temp[128] = {0}; - sprintf(temp, "%s/%s", tsLogDir, defaultLogFileNamePrefix); - if (taosInitLog(temp, tsNumOfLogLines, maxLogFileNum) < 0) { + if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) { printf("failed to open log file in directory:%s\n", tsLogDir); } } diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 18485249b3..4078ee9291 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -35,7 +35,7 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, char* id) } else { SStreamBlockScanInfo* pInfo = pOperator->info; if (tqReadHandleSetMsg(pInfo->readerHandle, input, 0) < 0) { - qError("submit msg error while set stream msg, %s" PRIx64, id); + qError("submit msg messed up when initing stream block, %s" PRIx64, id); return TSDB_CODE_QRY_APP_ERROR; } return TSDB_CODE_SUCCESS; diff --git a/source/libs/function/inc/builtins.h b/source/libs/function/inc/builtins.h index 62f86de918..598a28b2eb 100644 --- a/source/libs/function/inc/builtins.h +++ b/source/libs/function/inc/builtins.h @@ -46,6 +46,7 @@ typedef struct SBuiltinFuncDefinition { FExecGetEnv getEnvFunc; FExecInit initFunc; FExecProcess processFunc; + FScalarExecProcess sprocessFunc; FExecFinalize finalizeFunc; } SBuiltinFuncDefinition; diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 96e165f788..73ce67bd28 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -36,7 +36,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .checkFunc = stubCheckAndGetResultType, .getEnvFunc = NULL, .initFunc = NULL, - .processFunc = NULL, + .sprocessFunc = NULL, .finalizeFunc = NULL } }; diff --git a/source/libs/function/src/functionMgt.c b/source/libs/function/src/functionMgt.c index 887f65a6ea..c6330c6015 100644 --- a/source/libs/function/src/functionMgt.c +++ b/source/libs/function/src/functionMgt.c @@ -71,6 +71,14 @@ int32_t fmGetFuncExecFuncs(int32_t funcId, SFuncExecFuncs* pFpSet) { return TSDB_CODE_SUCCESS; } +int32_t fmGetScalarFuncExecFuncs(int32_t funcId, SScalarFuncExecFuncs* pFpSet) { + if (funcId < 0 || funcId >= funcMgtBuiltinsNum) { + return TSDB_CODE_FAILED; + } + pFpSet->process = funcMgtBuiltins[funcId].sprocessFunc; + return TSDB_CODE_SUCCESS; +} + bool fmIsAggFunc(int32_t funcId) { if (funcId < 0 || funcId >= funcMgtBuiltinsNum) { return false; diff --git a/source/libs/function/src/taggfunction.c b/source/libs/function/src/taggfunction.c index 1aa5871468..2ff2edf84e 100644 --- a/source/libs/function/src/taggfunction.c +++ b/source/libs/function/src/taggfunction.c @@ -3960,14 +3960,14 @@ static void ts_comp_finalize(SqlFunctionCtx *pCtx) { // qDebug("total timestamp :%"PRId64, pTSbuf->numOfTotal); // TODO refactor transfer ownership of current file - *(FILE **)pCtx->pOutput = pTSbuf->f; + *(TdFilePtr *)pCtx->pOutput = pTSbuf->pFile; pResInfo->complete = true; // get the file size - struct stat fStat; - if ((fstat(fileno(pTSbuf->f), &fStat) == 0)) { - pResInfo->numOfRes = fStat.st_size; + int64_t file_size; + if (taosFStatFile(pTSbuf->pFile, &file_size, NULL) == 0) { + pResInfo->numOfRes = (uint32_t )file_size; } pTSbuf->remainOpen = true; diff --git a/source/libs/function/src/texpr.c b/source/libs/function/src/texpr.c index aa3e6ba0f0..7d9cb97400 100644 --- a/source/libs/function/src/texpr.c +++ b/source/libs/function/src/texpr.c @@ -25,6 +25,7 @@ #include "thash.h" #include "texpr.h" #include "tvariant.h" +#include "tdef.h" //static uint8_t UNUSED_FUNC isQueryOnPrimaryKey(const char *primaryColumnName, const tExprNode *pLeft, const tExprNode *pRight) { // if (pLeft->nodeType == TEXPR_COL_NODE) { @@ -94,7 +95,7 @@ bool exprTreeApplyFilter(tExprNode *pExpr, const void *pItem, SExprTraverseSupp //non-leaf nodes, recursively traverse the expression tree in the post-root order if (pLeft->nodeType == TEXPR_BINARYEXPR_NODE && pRight->nodeType == TEXPR_BINARYEXPR_NODE) { - if (pExpr->_node.optr == TSDB_RELATION_OR) { // or + if (pExpr->_node.optr == LOGIC_COND_TYPE_OR) { // or if (exprTreeApplyFilter(pLeft, pItem, param)) { return true; } @@ -255,7 +256,7 @@ tExprNode* exprTreeFromTableName(const char* tbnameCond) { if (strncmp(tbnameCond, QUERY_COND_REL_PREFIX_LIKE, QUERY_COND_REL_PREFIX_LIKE_LEN) == 0) { right->nodeType = TEXPR_VALUE_NODE; - expr->_node.optr = TSDB_RELATION_LIKE; + expr->_node.optr = OP_TYPE_LIKE; SVariant* pVal = exception_calloc(1, sizeof(SVariant)); right->pVal = pVal; size_t len = strlen(tbnameCond + QUERY_COND_REL_PREFIX_LIKE_LEN) + 1; @@ -266,7 +267,7 @@ tExprNode* exprTreeFromTableName(const char* tbnameCond) { } else if (strncmp(tbnameCond, QUERY_COND_REL_PREFIX_MATCH, QUERY_COND_REL_PREFIX_MATCH_LEN) == 0) { right->nodeType = TEXPR_VALUE_NODE; - expr->_node.optr = TSDB_RELATION_MATCH; + expr->_node.optr = OP_TYPE_MATCH; SVariant* pVal = exception_calloc(1, sizeof(SVariant)); right->pVal = pVal; size_t len = strlen(tbnameCond + QUERY_COND_REL_PREFIX_MATCH_LEN) + 1; @@ -276,7 +277,7 @@ tExprNode* exprTreeFromTableName(const char* tbnameCond) { pVal->nLen = (int32_t)len; } else if (strncmp(tbnameCond, QUERY_COND_REL_PREFIX_NMATCH, QUERY_COND_REL_PREFIX_NMATCH_LEN) == 0) { right->nodeType = TEXPR_VALUE_NODE; - expr->_node.optr = TSDB_RELATION_NMATCH; + expr->_node.optr = OP_TYPE_NMATCH; SVariant* pVal = exception_calloc(1, sizeof(SVariant)); right->pVal = pVal; size_t len = strlen(tbnameCond + QUERY_COND_REL_PREFIX_NMATCH_LEN) + 1; @@ -286,7 +287,7 @@ tExprNode* exprTreeFromTableName(const char* tbnameCond) { pVal->nLen = (int32_t)len; } else if (strncmp(tbnameCond, QUERY_COND_REL_PREFIX_IN, QUERY_COND_REL_PREFIX_IN_LEN) == 0) { right->nodeType = TEXPR_VALUE_NODE; - expr->_node.optr = TSDB_RELATION_IN; + expr->_node.optr = OP_TYPE_IN; SVariant* pVal = exception_calloc(1, sizeof(SVariant)); right->pVal = pVal; pVal->nType = TSDB_DATA_TYPE_POINTER_ARRAY; diff --git a/source/libs/index/inc/index_fst_counting_writer.h b/source/libs/index/inc/index_fst_counting_writer.h index 1e0a88e17f..86e829aa95 100644 --- a/source/libs/index/inc/index_fst_counting_writer.h +++ b/source/libs/index/inc/index_fst_counting_writer.h @@ -38,7 +38,7 @@ typedef struct WriterCtx { WriterType type; union { struct { - int fd; + TdFilePtr pFile; bool readOnly; char buf[256]; int size; diff --git a/source/libs/index/inc/index_util.h b/source/libs/index/inc/index_util.h index adeb52bb8c..36830a68bc 100644 --- a/source/libs/index/inc/index_util.h +++ b/source/libs/index/inc/index_util.h @@ -15,36 +15,46 @@ #ifndef __INDEX_UTIL_H__ #define __INDEX_UTIL_H__ +#include "tarray.h" + #ifdef __cplusplus extern "C" { #endif -#define SERIALIZE_MEM_TO_BUF(buf, key, mem) \ - do { \ - memcpy((void*)buf, (void*)(&key->mem), sizeof(key->mem)); \ - buf += sizeof(key->mem); \ +#define SERIALIZE_MEM_TO_BUF(buf, key, mem) \ + do { \ + memcpy((void *)buf, (void *)(&key->mem), sizeof(key->mem)); \ + buf += sizeof(key->mem); \ } while (0) #define SERIALIZE_STR_MEM_TO_BUF(buf, key, mem, len) \ do { \ - memcpy((void*)buf, (void*)key->mem, len); \ + memcpy((void *)buf, (void *)key->mem, len); \ buf += len; \ } while (0) -#define SERIALIZE_VAR_TO_BUF(buf, var, type) \ - do { \ - type c = var; \ - assert(sizeof(type) == sizeof(c)); \ - memcpy((void*)buf, (void*)&c, sizeof(c)); \ - buf += sizeof(c); \ +#define SERIALIZE_VAR_TO_BUF(buf, var, type) \ + do { \ + type c = var; \ + assert(sizeof(type) == sizeof(c)); \ + memcpy((void *)buf, (void *)&c, sizeof(c)); \ + buf += sizeof(c); \ } while (0) #define SERIALIZE_STR_VAR_TO_BUF(buf, var, len) \ do { \ - memcpy((void*)buf, (void*)var, len); \ + memcpy((void *)buf, (void *)var, len); \ buf += len; \ } while (0) +/* multi sorted result intersection + * input: [1, 2, 4, 5] + * [2, 3, 4, 5] + * [1, 4, 5] + * output:[4, 5] + */ +void iIntersection(SArray *interResults, SArray *finalResult); +void iUnion(SArray *interResults, SArray *finalResult); #ifdef __cplusplus } #endif diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index 0c222eae1a..5147734a85 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -66,7 +66,9 @@ static void indexMergeSameKey(SArray* result, TFileValue* tv); int indexOpen(SIndexOpts* opts, const char* path, SIndex** index) { pthread_once(&isInit, indexInit); SIndex* sIdx = calloc(1, sizeof(SIndex)); - if (sIdx == NULL) { return -1; } + if (sIdx == NULL) { + return -1; + } #ifdef USE_LUCENE index_t* index = index_open(path); @@ -76,7 +78,9 @@ int indexOpen(SIndexOpts* opts, const char* path, SIndex** index) { #ifdef USE_INVERTED_INDEX // sIdx->cache = (void*)indexCacheCreate(sIdx); sIdx->tindex = indexTFileCreate(path); - if (sIdx->tindex == NULL) { goto END; } + if (sIdx->tindex == NULL) { + goto END; + } sIdx->colObj = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); sIdx->cVersion = 1; @@ -87,7 +91,9 @@ int indexOpen(SIndexOpts* opts, const char* path, SIndex** index) { #endif END: - if (sIdx != NULL) { indexClose(sIdx); } + if (sIdx != NULL) { + indexClose(sIdx); + } *index = NULL; return -1; @@ -103,7 +109,9 @@ void indexClose(SIndex* sIdx) { void* iter = taosHashIterate(sIdx->colObj, NULL); while (iter) { IndexCache** pCache = iter; - if (*pCache) { indexCacheUnRef(*pCache); } + if (*pCache) { + indexCacheUnRef(*pCache); + } iter = taosHashIterate(sIdx->colObj, iter); } taosHashCleanup(sIdx->colObj); @@ -161,7 +169,9 @@ int indexPut(SIndex* index, SIndexMultiTerm* fVals, uint64_t uid) { IndexCache** cache = taosHashGet(index->colObj, buf, sz); assert(*cache != NULL); int ret = indexCachePut(*cache, p, uid); - if (ret != 0) { return ret; } + if (ret != 0) { + return ret; + } } #endif @@ -191,7 +201,9 @@ int indexSearch(SIndex* index, SIndexMultiTermQuery* multiQuerys, SArray* result int tsz = 0; index_multi_search(index->index, (const char**)fields, (const char**)keys, types, nQuery, opera, &tResult, &tsz); - for (int i = 0; i < tsz; i++) { taosArrayPush(result, &tResult[i]); } + for (int i = 0; i < tsz; i++) { + taosArrayPush(result, &tResult[i]); + } for (int i = 0; i < nQuery; i++) { free(fields[i]); @@ -248,7 +260,9 @@ void indexOptsDestroy(SIndexOpts* opts) { */ SIndexMultiTermQuery* indexMultiTermQueryCreate(EIndexOperatorType opera) { SIndexMultiTermQuery* p = (SIndexMultiTermQuery*)malloc(sizeof(SIndexMultiTermQuery)); - if (p == NULL) { return NULL; } + if (p == NULL) { + return NULL; + } p->opera = opera; p->query = taosArrayInit(4, sizeof(SIndexTermQuery)); return p; @@ -270,7 +284,9 @@ int indexMultiTermQueryAdd(SIndexMultiTermQuery* pQuery, SIndexTerm* term, EInde SIndexTerm* indexTermCreate(int64_t suid, SIndexOperOnColumn oper, uint8_t colType, const char* colName, int32_t nColName, const char* colVal, int32_t nColVal) { SIndexTerm* t = (SIndexTerm*)calloc(1, (sizeof(SIndexTerm))); - if (t == NULL) { return NULL; } + if (t == NULL) { + return NULL; + } t->suid = suid; t->operType = oper; @@ -343,7 +359,9 @@ static int indexTermSearch(SIndex* sIdx, SIndexTermQuery* query, SArray** result return 0; } static void indexInterResultsDestroy(SArray* results) { - if (results == NULL) { return; } + if (results == NULL) { + return; + } size_t sz = taosArrayGetSize(results); for (size_t i = 0; i < sz; i++) { @@ -352,22 +370,23 @@ static void indexInterResultsDestroy(SArray* results) { } taosArrayDestroy(results); } + static int indexMergeFinalResults(SArray* interResults, EIndexOperatorType oType, SArray* fResults) { // refactor, merge interResults into fResults by oType - SArray* first = taosArrayGetP(interResults, 0); - taosArraySort(first, uidCompare); - taosArrayRemoveDuplicate(first, uidCompare, NULL); + + for (int i = 0; i < taosArrayGetSize(interResults); i--) { + SArray* t = taosArrayGetP(interResults, i); + taosArraySort(t, uidCompare); + taosArrayRemoveDuplicate(t, uidCompare, NULL); + } if (oType == MUST) { - // just one column index, enhance later - taosArrayAddAll(fResults, first); + iIntersection(interResults, fResults); } else if (oType == SHOULD) { - // just one column index, enhance later - taosArrayAddAll(fResults, first); - // tag1 condistion || tag2 condition + iUnion(interResults, fResults); } else if (oType == NOT) { // just one column index, enhance later - taosArrayAddAll(fResults, first); + taosArrayAddAll(fResults, interResults); // not use currently } return 0; @@ -419,18 +438,24 @@ static void indexDestroyTempResult(SArray* result) { taosArrayDestroy(result); } int indexFlushCacheToTFile(SIndex* sIdx, void* cache) { - if (sIdx == NULL) { return -1; } + if (sIdx == NULL) { + return -1; + } indexInfo("suid %" PRIu64 " merge cache into tindex", sIdx->suid); int64_t st = taosGetTimestampUs(); IndexCache* pCache = (IndexCache*)cache; TFileReader* pReader = tfileGetReaderByCol(sIdx->tindex, pCache->suid, pCache->colName); - if (pReader == NULL) { indexWarn("empty tfile reader found"); } + if (pReader == NULL) { + indexWarn("empty tfile reader found"); + } // handle flush Iterate* cacheIter = indexCacheIteratorCreate(pCache); Iterate* tfileIter = tfileIteratorCreate(pReader); - if (tfileIter == NULL) { indexWarn("empty tfile reader iterator"); } + if (tfileIter == NULL) { + indexWarn("empty tfile reader iterator"); + } SArray* result = taosArrayInit(1024, sizeof(void*)); @@ -484,7 +509,9 @@ void iterateValueDestroy(IterateValue* value, bool destroy) { taosArrayDestroy(value->val); value->val = NULL; } else { - if (value->val != NULL) { taosArrayClear(value->val); } + if (value->val != NULL) { + taosArrayClear(value->val); + } } free(value->colVal); value->colVal = NULL; @@ -507,7 +534,9 @@ static int indexGenTFile(SIndex* sIdx, IndexCache* cache, SArray* batch) { tfileWriterClose(tw); TFileReader* reader = tfileReaderOpen(sIdx->path, cache->suid, version, cache->colName); - if (reader == NULL) { return -1; } + if (reader == NULL) { + return -1; + } TFileHeader* header = &reader->header; ICacheKey key = {.suid = cache->suid, .colName = header->colName, .nColName = strlen(header->colName)}; diff --git a/source/libs/index/src/index_cache.c b/source/libs/index/src/index_cache.c index 48566a8674..d6a7141825 100644 --- a/source/libs/index/src/index_cache.c +++ b/source/libs/index/src/index_cache.c @@ -119,13 +119,17 @@ void indexCacheDestroySkiplist(SSkipList* slt) { tSkipListDestroy(slt); } void indexCacheDestroyImm(IndexCache* cache) { - if (cache == NULL) { return; } + if (cache == NULL) { + return; + } MemTable* tbl = NULL; pthread_mutex_lock(&cache->mtx); + tbl = cache->imm; cache->imm = NULL; // or throw int bg thread pthread_cond_broadcast(&cache->finished); + pthread_mutex_unlock(&cache->mtx); indexMemUnRef(tbl); @@ -133,7 +137,9 @@ void indexCacheDestroyImm(IndexCache* cache) { } void indexCacheDestroy(void* cache) { IndexCache* pCache = cache; - if (pCache == NULL) { return; } + if (pCache == NULL) { + return; + } indexMemUnRef(pCache->mem); indexMemUnRef(pCache->imm); free(pCache->colName); @@ -146,7 +152,9 @@ void indexCacheDestroy(void* cache) { Iterate* indexCacheIteratorCreate(IndexCache* cache) { Iterate* iiter = calloc(1, sizeof(Iterate)); - if (iiter == NULL) { return NULL; } + if (iiter == NULL) { + return NULL; + } pthread_mutex_lock(&cache->mtx); @@ -164,7 +172,9 @@ Iterate* indexCacheIteratorCreate(IndexCache* cache) { return iiter; } void indexCacheIteratorDestroy(Iterate* iter) { - if (iter == NULL) { return; } + if (iter == NULL) { + return; + } tSkipListDestroyIter(iter->iter); iterateValueDestroy(&iter->val, true); free(iter); @@ -186,9 +196,6 @@ static void indexCacheMakeRoomForWrite(IndexCache* cache) { } else if (cache->imm != NULL) { // TODO: wake up by condition variable pthread_cond_wait(&cache->finished, &cache->mtx); - // pthread_mutex_unlock(&cache->mtx); - // taosMsleep(50); - // pthread_mutex_lock(&cache->mtx); } else { indexCacheRef(cache); cache->imm = cache->mem; @@ -202,13 +209,17 @@ static void indexCacheMakeRoomForWrite(IndexCache* cache) { } int indexCachePut(void* cache, SIndexTerm* term, uint64_t uid) { - if (cache == NULL) { return -1; } + if (cache == NULL) { + return -1; + } IndexCache* pCache = cache; indexCacheRef(pCache); // encode data CacheTerm* ct = calloc(1, sizeof(CacheTerm)); - if (cache == NULL) { return -1; } + if (cache == NULL) { + return -1; + } // set up key ct->colType = term->colType; ct->colVal = (char*)calloc(1, sizeof(char) * (term->nColVal + 1)); @@ -240,7 +251,9 @@ int indexCacheDel(void* cache, const char* fieldValue, int32_t fvlen, uint64_t u } static int indexQueryMem(MemTable* mem, CacheTerm* ct, EIndexQueryType qtype, SArray* result, STermValueType* s) { - if (mem == NULL) { return 0; } + if (mem == NULL) { + return 0; + } char* key = indexCacheTermGet(ct); SSkipListIterator* iter = tSkipListCreateIterFromVal(mem->mem, key, TSDB_DATA_TYPE_BINARY, TSDB_ORDER_ASC); @@ -266,7 +279,9 @@ static int indexQueryMem(MemTable* mem, CacheTerm* ct, EIndexQueryType qtype, SA return 0; } int indexCacheSearch(void* cache, SIndexTermQuery* query, SArray* result, STermValueType* s) { - if (cache == NULL) { return 0; } + if (cache == NULL) { + return 0; + } IndexCache* pCache = cache; MemTable *mem = NULL, *imm = NULL; @@ -294,23 +309,33 @@ int indexCacheSearch(void* cache, SIndexTermQuery* query, SArray* result, STermV } void indexCacheRef(IndexCache* cache) { - if (cache == NULL) { return; } + if (cache == NULL) { + return; + } int ref = T_REF_INC(cache); UNUSED(ref); } void indexCacheUnRef(IndexCache* cache) { - if (cache == NULL) { return; } + if (cache == NULL) { + return; + } int ref = T_REF_DEC(cache); - if (ref == 0) { indexCacheDestroy(cache); } + if (ref == 0) { + indexCacheDestroy(cache); + } } void indexMemRef(MemTable* tbl) { - if (tbl == NULL) { return; } + if (tbl == NULL) { + return; + } int ref = T_REF_INC(tbl); UNUSED(ref); } void indexMemUnRef(MemTable* tbl) { - if (tbl == NULL) { return; } + if (tbl == NULL) { + return; + } int ref = T_REF_DEC(tbl); if (ref == 0) { SSkipList* slt = tbl->mem; @@ -320,7 +345,9 @@ void indexMemUnRef(MemTable* tbl) { } static void indexCacheTermDestroy(CacheTerm* ct) { - if (ct == NULL) { return; } + if (ct == NULL) { + return; + } free(ct->colVal); free(ct); } @@ -333,7 +360,9 @@ static int32_t indexCacheTermCompare(const void* l, const void* r) { CacheTerm* rt = (CacheTerm*)r; // compare colVal int32_t cmp = strcmp(lt->colVal, rt->colVal); - if (cmp == 0) { return rt->version - lt->version; } + if (cmp == 0) { + return rt->version - lt->version; + } return cmp; } @@ -354,7 +383,9 @@ static void doMergeWork(SSchedMsg* msg) { } static bool indexCacheIteratorNext(Iterate* itera) { SSkipListIterator* iter = itera->iter; - if (iter == NULL) { return false; } + if (iter == NULL) { + return false; + } IterateValue* iv = &itera->val; iterateValueDestroy(iv, false); diff --git a/source/libs/index/src/index_fst.c b/source/libs/index/src/index_fst.c index 46b4c9d7c6..a6cabbd439 100644 --- a/source/libs/index/src/index_fst.c +++ b/source/libs/index/src/index_fst.c @@ -31,20 +31,24 @@ static uint8_t fstPackDetla(FstCountingWriter* wrt, CompiledAddr nodeAddr, Compi FstUnFinishedNodes* fstUnFinishedNodesCreate() { FstUnFinishedNodes* nodes = malloc(sizeof(FstUnFinishedNodes)); - if (nodes == NULL) { return NULL; } + if (nodes == NULL) { + return NULL; + } nodes->stack = (SArray*)taosArrayInit(64, sizeof(FstBuilderNodeUnfinished)); fstUnFinishedNodesPushEmpty(nodes, false); return nodes; } -void unFinishedNodeDestroyElem(void* elem) { +static void unFinishedNodeDestroyElem(void* elem) { FstBuilderNodeUnfinished* b = (FstBuilderNodeUnfinished*)elem; fstBuilderNodeDestroy(b->node); free(b->last); b->last = NULL; } void fstUnFinishedNodesDestroy(FstUnFinishedNodes* nodes) { - if (nodes == NULL) { return; } + if (nodes == NULL) { + return; + } taosArrayDestroyEx(nodes->stack, unFinishedNodeDestroyElem); free(nodes); @@ -92,7 +96,9 @@ void fstUnFinishedNodesTopLastFreeze(FstUnFinishedNodes* nodes, CompiledAddr add } void fstUnFinishedNodesAddSuffix(FstUnFinishedNodes* nodes, FstSlice bs, Output out) { FstSlice* s = &bs; - if (fstSliceIsEmpty(s)) { return; } + if (fstSliceIsEmpty(s)) { + return; + } size_t sz = taosArrayGetSize(nodes->stack) - 1; FstBuilderNodeUnfinished* un = taosArrayGet(nodes->stack, sz); assert(un->last == NULL); @@ -172,7 +178,9 @@ uint64_t fstUnFinishedNodesFindCommPrefixAndSetOutput(FstUnFinishedNodes* node, FstState fstStateCreateFrom(FstSlice* slice, CompiledAddr addr) { FstState fs = {.state = EmptyFinal, .val = 0}; - if (addr == EMPTY_ADDRESS) { return fs; } + if (addr == EMPTY_ADDRESS) { + return fs; + } uint8_t* data = fstSliceData(slice, NULL); uint8_t v = data[addr]; @@ -227,9 +235,12 @@ void fstStateCompileForOneTrans(FstCountingWriter* w, CompiledAddr addr, FstTran FstState st = fstStateCreate(OneTrans); fstStateSetCommInput(&st, trn->inp); + bool null = false; uint8_t inp = fstStateCommInput(&st, &null); - if (null == true) { fstCountingWriterWrite(w, (char*)&trn->inp, sizeof(trn->inp)); } + if (null == true) { + fstCountingWriterWrite(w, (char*)&trn->inp, sizeof(trn->inp)); + } fstCountingWriterWrite(w, (char*)(&(st.val)), sizeof(st.val)); return; } @@ -263,7 +274,9 @@ void fstStateCompileForAnyTrans(FstCountingWriter* w, CompiledAddr addr, FstBuil fstStateSetStateNtrans(&st, (uint8_t)sz); if (anyOuts) { - if (FST_BUILDER_NODE_IS_FINAL(node)) { fstCountingWriterPackUintIn(w, node->finalOutput, oSize); } + if (FST_BUILDER_NODE_IS_FINAL(node)) { + fstCountingWriterPackUintIn(w, node->finalOutput, oSize); + } for (int32_t i = sz - 1; i >= 0; i--) { FstTransition* t = taosArrayGet(node->trans, i); fstCountingWriterPackUintIn(w, t->out, oSize); @@ -428,7 +441,9 @@ Output fstStateOutput(FstState* s, FstNode* node) { assert(s->state == OneTrans); uint8_t oSizes = FST_GET_OUTPUT_PACK_SIZE(node->sizes); - if (oSizes == 0) { return 0; } + if (oSizes == 0) { + return 0; + } FstSlice* slice = &node->data; uint8_t tSizes = FST_GET_TRANSITION_PACK_SIZE(node->sizes); @@ -440,7 +455,9 @@ Output fstStateOutputForAnyTrans(FstState* s, FstNode* node, uint64_t i) { assert(s->state == AnyTrans); uint8_t oSizes = FST_GET_OUTPUT_PACK_SIZE(node->sizes); - if (oSizes == 0) { return 0; } + if (oSizes == 0) { + return 0; + } FstSlice* slice = &node->data; uint8_t* data = fstSliceData(slice, NULL); uint64_t at = node->start - fstStateNtransLen(s) - 1 // pack size @@ -453,7 +470,9 @@ Output fstStateOutputForAnyTrans(FstState* s, FstNode* node, uint64_t i) { void fstStateSetFinalState(FstState* s, bool yes) { assert(s->state == AnyTrans); - if (yes) { s->val |= 0b01000000; } + if (yes) { + s->val |= 0b01000000; + } return; } bool fstStateIsFinalState(FstState* s) { @@ -463,7 +482,9 @@ bool fstStateIsFinalState(FstState* s) { void fstStateSetStateNtrans(FstState* s, uint8_t n) { assert(s->state == AnyTrans); - if (n <= 0b00111111) { s->val = (s->val & 0b11000000) | n; } + if (n <= 0b00111111) { + s->val = (s->val & 0b11000000) | n; + } return; } // state_ntrans @@ -495,7 +516,9 @@ uint64_t fstStateNtransLen(FstState* s) { uint64_t fstStateNtrans(FstState* s, FstSlice* slice) { bool null = false; uint8_t n = fstStateStateNtrans(s, &null); - if (null != true) { return n; } + if (null != true) { + return n; + } int32_t len; uint8_t* data = fstSliceData(slice, &len); n = data[len - 2]; @@ -505,7 +528,9 @@ uint64_t fstStateNtrans(FstState* s, FstSlice* slice) { } Output fstStateFinalOutput(FstState* s, uint64_t version, FstSlice* slice, PackSizes sizes, uint64_t nTrans) { uint8_t oSizes = FST_GET_OUTPUT_PACK_SIZE(sizes); - if (oSizes == 0 || !fstStateIsFinalState(s)) { return 0; } + if (oSizes == 0 || !fstStateIsFinalState(s)) { + return 0; + } uint64_t at = FST_SLICE_LEN(slice) - 1 - fstStateNtransLen(s) - 1 // pack size - fstStateTotalTransSize(s, version, sizes, nTrans) - (nTrans * oSizes) - oSizes; @@ -522,7 +547,9 @@ uint64_t fstStateFindInput(FstState* s, FstNode* node, uint8_t b, bool* null) { uint8_t* data = fstSliceData(slice, &dlen); uint64_t i = data[at + b]; // uint64_t i = slice->data[slice->start + at + b]; - if (i >= node->nTrans) { *null = true; } + if (i >= node->nTrans) { + *null = true; + } return i; } else { uint64_t start = node->start - fstStateNtransLen(s) - 1 // pack size @@ -539,7 +566,9 @@ uint64_t fstStateFindInput(FstState* s, FstNode* node, uint8_t b, bool* null) { return node->nTrans - i - 1; // bug } } - if (i == len) { *null = true; } + if (i == len) { + *null = true; + } fstSliceDestroy(&t); } } @@ -548,7 +577,9 @@ uint64_t fstStateFindInput(FstState* s, FstNode* node, uint8_t b, bool* null) { FstNode* fstNodeCreate(int64_t version, CompiledAddr addr, FstSlice* slice) { FstNode* n = (FstNode*)malloc(sizeof(FstNode)); - if (n == NULL) { return NULL; } + if (n == NULL) { + return NULL; + } FstState st = fstStateCreateFrom(slice, addr); @@ -614,7 +645,9 @@ void fstNodeDestroy(FstNode* node) { } FstTransitions* fstNodeTransitions(FstNode* node) { FstTransitions* t = malloc(sizeof(FstTransitions)); - if (NULL == t) { return NULL; } + if (NULL == t) { + return NULL; + } FstRange range = {.start = 0, .end = FST_NODE_LEN(node)}; t->range = range; t->node = node; @@ -721,7 +754,9 @@ bool fstBuilderNodeCompileTo(FstBuilderNode* b, FstCountingWriter* wrt, Compiled FstBuilder* fstBuilderCreate(void* w, FstType ty) { FstBuilder* b = malloc(sizeof(FstBuilder)); - if (NULL == b) { return b; } + if (NULL == b) { + return b; + } b->wrt = fstCountingWriterCreate(w); b->unfinished = fstUnFinishedNodesCreate(); @@ -735,15 +770,17 @@ FstBuilder* fstBuilderCreate(void* w, FstType ty) { taosEncodeFixedU64(&pBuf64, VERSION); fstCountingWriterWrite(b->wrt, buf64, sizeof(buf64)); - memset(buf64, 0, sizeof(buf64)); pBuf64 = buf64; + memset(buf64, 0, sizeof(buf64)); taosEncodeFixedU64(&pBuf64, ty); fstCountingWriterWrite(b->wrt, buf64, sizeof(buf64)); return b; } void fstBuilderDestroy(FstBuilder* b) { - if (b == NULL) { return; } + if (b == NULL) { + return; + } fstCountingWriterDestroy(b->wrt); fstUnFinishedNodesDestroy(b->unfinished); @@ -830,6 +867,7 @@ void fstBuilderCompileFrom(FstBuilder* b, uint64_t istate) { fstUnFinishedNodesTopLastFreeze(b->unfinished, addr); return; } + CompiledAddr fstBuilderCompile(FstBuilder* b, FstBuilderNode* bn) { if (FST_BUILDER_NODE_IS_FINAL(bn) && FST_BUILDER_NODE_TRANS_ISEMPTY(bn) && FST_BUILDER_NODE_FINALOUTPUT_ISZERO(bn)) { return EMPTY_ADDRESS; @@ -844,7 +882,9 @@ CompiledAddr fstBuilderCompile(FstBuilder* b, FstBuilderNode* bn) { fstBuilderNodeCompileTo(bn, b->wrt, b->lastAddr, startAddr); b->lastAddr = (CompiledAddr)(FST_WRITER_COUNT(b->wrt) - 1); - if (entry->state == NOTFOUND) { FST_REGISTRY_CELL_INSERT(entry->cell, b->lastAddr); } + if (entry->state == NOTFOUND) { + FST_REGISTRY_CELL_INSERT(entry->cell, b->lastAddr); + } fstRegistryEntryDestroy(entry); return b->lastAddr; @@ -887,7 +927,9 @@ FstSlice fstNodeAsSlice(FstNode* node) { FstLastTransition* fstLastTransitionCreate(uint8_t inp, Output out) { FstLastTransition* trn = malloc(sizeof(FstLastTransition)); - if (trn == NULL) { return NULL; } + if (trn == NULL) { + return NULL; + } trn->inp = inp; trn->out = out; @@ -895,9 +937,12 @@ FstLastTransition* fstLastTransitionCreate(uint8_t inp, Output out) { } void fstLastTransitionDestroy(FstLastTransition* trn) { free(trn); } + void fstBuilderNodeUnfinishedLastCompiled(FstBuilderNodeUnfinished* unNode, CompiledAddr addr) { FstLastTransition* trn = unNode->last; - if (trn == NULL) { return; } + if (trn == NULL) { + return; + } FstTransition t = {.inp = trn->inp, .out = trn->out, .addr = addr}; taosArrayPush(unNode->node->trans, &t); fstLastTransitionDestroy(trn); @@ -906,27 +951,35 @@ void fstBuilderNodeUnfinishedLastCompiled(FstBuilderNodeUnfinished* unNode, Comp } void fstBuilderNodeUnfinishedAddOutputPrefix(FstBuilderNodeUnfinished* unNode, Output out) { - if (FST_BUILDER_NODE_IS_FINAL(unNode->node)) { unNode->node->finalOutput += out; } + if (FST_BUILDER_NODE_IS_FINAL(unNode->node)) { + unNode->node->finalOutput += out; + } size_t sz = taosArrayGetSize(unNode->node->trans); for (size_t i = 0; i < sz; i++) { FstTransition* trn = taosArrayGet(unNode->node->trans, i); trn->out += out; } - if (unNode->last) { unNode->last->out += out; } + if (unNode->last) { + unNode->last->out += out; + } return; } Fst* fstCreate(FstSlice* slice) { int32_t slen; char* buf = fstSliceData(slice, &slen); - if (slen < 36) { return NULL; } + if (slen < 36) { + return NULL; + } uint64_t len = slen; uint64_t skip = 0; uint64_t version; taosDecodeFixedU64(buf, &version); skip += sizeof(version); - if (version == 0 || version > VERSION) { return NULL; } + if (version == 0 || version > VERSION) { + return NULL; + } uint64_t type; taosDecodeFixedU64(buf + skip, &type); @@ -949,10 +1002,14 @@ Fst* fstCreate(FstSlice* slice) { taosDecodeFixedU64(buf + len, &fstLen); // TODO(validate root addr) Fst* fst = (Fst*)calloc(1, sizeof(Fst)); - if (fst == NULL) { return NULL; } + if (fst == NULL) { + return NULL; + } fst->meta = (FstMeta*)malloc(sizeof(FstMeta)); - if (NULL == fst->meta) { goto FST_CREAT_FAILED; } + if (NULL == fst->meta) { + goto FST_CREAT_FAILED; + } fst->meta->version = version; fst->meta->rootAddr = rootAddr; @@ -983,7 +1040,7 @@ void fstDestroy(Fst* fst) { bool fstGet(Fst* fst, FstSlice* b, Output* out) { // dec lock range - pthread_mutex_lock(&fst->mtx); + // pthread_mutex_lock(&fst->mtx); FstNode* root = fstGetRoot(fst); Output tOut = 0; int32_t len; @@ -996,7 +1053,7 @@ bool fstGet(Fst* fst, FstSlice* b, Output* out) { uint8_t inp = data[i]; Output res = 0; if (false == fstNodeFindInput(root, inp, &res)) { - pthread_mutex_unlock(&fst->mtx); + // pthread_mutex_unlock(&fst->mtx); return false; } @@ -1007,7 +1064,7 @@ bool fstGet(Fst* fst, FstSlice* b, Output* out) { taosArrayPush(nodes, &root); } if (!FST_NODE_IS_FINAL(root)) { - pthread_mutex_unlock(&fst->mtx); + // pthread_mutex_unlock(&fst->mtx); return false; } else { tOut = tOut + FST_NODE_FINAL_OUTPUT(root); @@ -1018,8 +1075,8 @@ bool fstGet(Fst* fst, FstSlice* b, Output* out) { fstNodeDestroy(*node); } taosArrayDestroy(nodes); - fst->root = NULL; - pthread_mutex_unlock(&fst->mtx); + // fst->root = NULL; + // pthread_mutex_unlock(&fst->mtx); *out = tOut; return true; } @@ -1028,7 +1085,9 @@ FstStreamBuilder* fstSearch(Fst* fst, AutomationCtx* ctx) { return fstStreamBuilderCreate(fst, ctx); } StreamWithState* streamBuilderIntoStream(FstStreamBuilder* sb) { - if (sb == NULL) { return NULL; } + if (sb == NULL) { + return NULL; + } return streamWithStateCreate(sb->fst, sb->aut, sb->min, sb->max); } FstStreamWithStateBuilder* fstSearchWithState(Fst* fst, AutomationCtx* ctx) { @@ -1039,15 +1098,6 @@ FstStreamWithStateBuilder* fstSearchWithState(Fst* fst, AutomationCtx* ctx) { FstNode* fstGetRoot(Fst* fst) { CompiledAddr rAddr = fstGetRootAddr(fst); return fstGetNode(fst, rAddr); - // pthread_mutex_lock(&fst->mtx); - // if (fst->root != NULL) { - // // pthread_mutex_unlock(&fst->mtx); - // return fst->root; - //} - // CompiledAddr rAddr = fstGetRootAddr(fst); - // fst->root = fstGetNode(fst, rAddr); - //// pthread_mutex_unlock(&fst->mtx); - // return fst->root; } FstNode* fstGetNode(Fst* fst, CompiledAddr addr) { @@ -1074,14 +1124,18 @@ bool fstVerify(Fst* fst) { uint32_t len, checkSum = fst->meta->checkSum; uint8_t* data = fstSliceData(fst->data, &len); TSCKSUM initSum = 0; - if (!taosCheckChecksumWhole(data, len)) { return false; } + if (!taosCheckChecksumWhole(data, len)) { + return false; + } return true; } // data bound function FstBoundWithData* fstBoundStateCreate(FstBound type, FstSlice* data) { FstBoundWithData* b = calloc(1, sizeof(FstBoundWithData)); - if (b == NULL) { return NULL; } + if (b == NULL) { + return NULL; + } if (data != NULL) { b->data = fstSliceCopy(data, data->start, data->end); @@ -1118,7 +1172,9 @@ void fstBoundDestroy(FstBoundWithData* bound) { free(bound); } StreamWithState* streamWithStateCreate(Fst* fst, AutomationCtx* automation, FstBoundWithData* min, FstBoundWithData* max) { StreamWithState* sws = calloc(1, sizeof(StreamWithState)); - if (sws == NULL) { return NULL; } + if (sws == NULL) { + return NULL; + } sws->fst = fst; sws->aut = automation; @@ -1134,7 +1190,9 @@ StreamWithState* streamWithStateCreate(Fst* fst, AutomationCtx* automation, FstB return sws; } void streamWithStateDestroy(StreamWithState* sws) { - if (sws == NULL) { return; } + if (sws == NULL) { + return; + } taosArrayDestroy(sws->inp); taosArrayDestroyEx(sws->stack, streamStateDestroy); @@ -1200,7 +1258,9 @@ bool streamWithStateSeekMin(StreamWithState* sws, FstBoundWithData* min) { uint64_t i = 0; for (i = trans->range.start; i < trans->range.end; i++) { FstTransition trn; - if (fstNodeGetTransitionAt(node, i, &trn) && trn.inp > b) { break; } + if (fstNodeGetTransitionAt(node, i, &trn) && trn.inp > b) { + break; + } } StreamState s = {.node = node, .trans = i, .out = {.null = false, .out = out}, .autState = autState}; @@ -1248,7 +1308,9 @@ StreamWithStateResult* streamWithStateNextWith(StreamWithState* sws, StreamCallb while (taosArrayGetSize(sws->stack) > 0) { StreamState* p = (StreamState*)taosArrayPop(sws->stack); if (p->trans >= FST_NODE_LEN(p->node) || !automFuncs[aut->type].canMatch(aut, p->autState)) { - if (FST_NODE_ADDR(p->node) != fstGetRootAddr(sws->fst)) { taosArrayPop(sws->inp); } + if (FST_NODE_ADDR(p->node) != fstGetRootAddr(sws->fst)) { + taosArrayPop(sws->inp); + } streamStateDestroy(p); continue; } @@ -1267,7 +1329,9 @@ StreamWithStateResult* streamWithStateNextWith(StreamWithState* sws, StreamCallb if (FST_NODE_IS_FINAL(nextNode)) { // void *eofState = sws->aut->acceptEof(nextState); void* eofState = automFuncs[aut->type].acceptEof(aut, nextState); - if (eofState != NULL) { isMatch = automFuncs[aut->type].isMatch(aut, eofState); } + if (eofState != NULL) { + isMatch = automFuncs[aut->type].isMatch(aut, eofState); + } } StreamState s1 = {.node = p->node, .trans = p->trans + 1, .out = p->out, .autState = p->autState}; taosArrayPush(sws->stack, &s1); @@ -1277,24 +1341,26 @@ StreamWithStateResult* streamWithStateNextWith(StreamWithState* sws, StreamCallb size_t isz = taosArrayGetSize(sws->inp); uint8_t* buf = (uint8_t*)malloc(isz * sizeof(uint8_t)); - for (uint32_t i = 0; i < isz; i++) { buf[i] = *(uint8_t*)taosArrayGet(sws->inp, i); } + for (uint32_t i = 0; i < isz; i++) { + buf[i] = *(uint8_t*)taosArrayGet(sws->inp, i); + } FstSlice slice = fstSliceCreate(buf, taosArrayGetSize(sws->inp)); if (fstBoundWithDataExceededBy(sws->endAt, &slice)) { taosArrayDestroyEx(sws->stack, streamStateDestroy); sws->stack = (SArray*)taosArrayInit(256, sizeof(StreamState)); - free(buf); + tfree(buf); fstSliceDestroy(&slice); return NULL; } if (FST_NODE_IS_FINAL(nextNode) && isMatch) { FstOutput fOutput = {.null = false, .out = out + FST_NODE_FINAL_OUTPUT(nextNode)}; StreamWithStateResult* result = swsResultCreate(&slice, fOutput, tState); - free(buf); + tfree(buf); fstSliceDestroy(&slice); taosArrayDestroy(nodes); return result; } - free(buf); + tfree(buf); fstSliceDestroy(&slice); } for (size_t i = 0; i < taosArrayGetSize(nodes); i++) { @@ -1307,16 +1373,19 @@ StreamWithStateResult* streamWithStateNextWith(StreamWithState* sws, StreamCallb StreamWithStateResult* swsResultCreate(FstSlice* data, FstOutput fOut, void* state) { StreamWithStateResult* result = calloc(1, sizeof(StreamWithStateResult)); - if (result == NULL) { return NULL; } + if (result == NULL) { + return NULL; + } result->data = fstSliceCopy(data, 0, FST_SLICE_LEN(data) - 1); result->out = fOut; result->state = state; - return result; } void swsResultDestroy(StreamWithStateResult* result) { - if (NULL == result) { return; } + if (NULL == result) { + return; + } fstSliceDestroy(&result->data); startWithStateValueDestroy(result->state); @@ -1324,16 +1393,18 @@ void swsResultDestroy(StreamWithStateResult* result) { } void streamStateDestroy(void* s) { - if (NULL == s) { return; } + if (NULL == s) { + return; + } StreamState* ss = (StreamState*)s; - fstNodeDestroy(ss->node); - // free(s->autoState); } FstStreamBuilder* fstStreamBuilderCreate(Fst* fst, AutomationCtx* aut) { FstStreamBuilder* b = calloc(1, sizeof(FstStreamBuilder)); - if (NULL == b) { return NULL; } + if (NULL == b) { + return NULL; + } b->fst = fst; b->aut = aut; @@ -1349,8 +1420,9 @@ void fstStreamBuilderDestroy(FstStreamBuilder* b) { free(b); } FstStreamBuilder* fstStreamBuilderRange(FstStreamBuilder* b, FstSlice* val, RangeType type) { - if (b == NULL) { return NULL; } - + if (b == NULL) { + return NULL; + } if (type == GE) { b->min->type = Included; fstSliceDestroy(&(b->min->data)); diff --git a/source/libs/index/src/index_fst_automation.c b/source/libs/index/src/index_fst_automation.c index c6e3cee3e3..ed1ad7a374 100644 --- a/source/libs/index/src/index_fst_automation.c +++ b/source/libs/index/src/index_fst_automation.c @@ -16,26 +16,30 @@ #include "index_fst_automation.h" StartWithStateValue* startWithStateValueCreate(StartWithStateKind kind, ValueType ty, void* val) { - StartWithStateValue* nsv = calloc(1, sizeof(StartWithStateValue)); - if (nsv == NULL) { return NULL; } + StartWithStateValue* sv = calloc(1, sizeof(StartWithStateValue)); + if (sv == NULL) { + return NULL; + } - nsv->kind = kind; - nsv->type = ty; + sv->kind = kind; + sv->type = ty; if (ty == FST_INT) { - nsv->val = *(int*)val; + sv->val = *(int*)val; } else if (ty == FST_CHAR) { size_t len = strlen((char*)val); - nsv->ptr = (char*)calloc(1, len + 1); - memcpy(nsv->ptr, val, len); + sv->ptr = (char*)calloc(1, len + 1); + memcpy(sv->ptr, val, len); } else if (ty == FST_ARRAY) { // TODO, // nsv->arr = taosArrayFromList() } - return nsv; + return sv; } void startWithStateValueDestroy(void* val) { StartWithStateValue* sv = (StartWithStateValue*)val; - if (sv == NULL) { return; } + if (sv == NULL) { + return; + } if (sv->type == FST_INT) { // @@ -48,7 +52,9 @@ void startWithStateValueDestroy(void* val) { } StartWithStateValue* startWithStateValueDump(StartWithStateValue* sv) { StartWithStateValue* nsv = calloc(1, sizeof(StartWithStateValue)); - if (nsv == NULL) { return NULL; } + if (nsv == NULL) { + return NULL; + } nsv->kind = sv->kind; nsv->type = sv->type; @@ -88,10 +94,14 @@ static bool prefixCanMatch(AutomationCtx* ctx, void* sv) { static bool prefixWillAlwaysMatch(AutomationCtx* ctx, void* state) { return true; } static void* prefixAccept(AutomationCtx* ctx, void* state, uint8_t byte) { StartWithStateValue* ssv = (StartWithStateValue*)state; - if (ssv == NULL || ctx == NULL) { return NULL; } + if (ssv == NULL || ctx == NULL) { + return NULL; + } char* data = ctx->data; - if (ssv->kind == Done) { return startWithStateValueCreate(Done, FST_INT, &ssv->val); } + if (ssv->kind == Done) { + return startWithStateValueCreate(Done, FST_INT, &ssv->val); + } if ((strlen(data) > ssv->val) && data[ssv->val] == byte) { int val = ssv->val + 1; @@ -128,17 +138,17 @@ AutomationFunc automFuncs[] = { AutomationCtx* automCtxCreate(void* data, AutomationType atype) { AutomationCtx* ctx = calloc(1, sizeof(AutomationCtx)); - if (ctx == NULL) { return NULL; } + if (ctx == NULL) { + return NULL; + } StartWithStateValue* sv = NULL; if (atype == AUTOMATION_ALWAYS) { int val = 0; sv = startWithStateValueCreate(Running, FST_INT, &val); - ctx->stdata = (void*)sv; } else if (atype == AUTOMATION_PREFIX) { int val = 0; sv = startWithStateValueCreate(Running, FST_INT, &val); - ctx->stdata = (void*)sv; } else if (atype == AUTMMATION_MATCH) { } else { // add more search type @@ -148,9 +158,8 @@ AutomationCtx* automCtxCreate(void* data, AutomationType atype) { if (data != NULL) { char* src = (char*)data; size_t len = strlen(src); - dst = (char*)malloc(len * sizeof(char) + 1); + dst = (char*)calloc(1, len * sizeof(char) + 1); memcpy(dst, src, len); - dst[len] = 0; } ctx->data = dst; diff --git a/source/libs/index/src/index_fst_counting_writer.c b/source/libs/index/src/index_fst_counting_writer.c index b57f639726..78c7a8d193 100644 --- a/source/libs/index/src/index_fst_counting_writer.c +++ b/source/libs/index/src/index_fst_counting_writer.c @@ -19,7 +19,7 @@ static int writeCtxDoWrite(WriterCtx* ctx, uint8_t* buf, int len) { if (ctx->type == TFile) { - assert(len == tfWrite(ctx->file.fd, buf, len)); + assert(len == taosWriteFile(ctx->file.pFile, buf, len)); } else { memcpy(ctx->mem.buf + ctx->offset, buf, len); } @@ -33,7 +33,7 @@ static int writeCtxDoRead(WriterCtx* ctx, uint8_t* buf, int len) { nRead = len < ctx->file.size ? len : ctx->file.size; memcpy(buf, ctx->file.ptr, nRead); #else - nRead = tfRead(ctx->file.fd, buf, len); + nRead = taosReadFile(ctx->file.pFile, buf, len); #endif } else { memcpy(buf, ctx->mem.buf + ctx->offset, len); @@ -45,13 +45,13 @@ static int writeCtxDoRead(WriterCtx* ctx, uint8_t* buf, int len) { static int writeCtxDoReadFrom(WriterCtx* ctx, uint8_t* buf, int len, int32_t offset) { int nRead = 0; if (ctx->type == TFile) { - // tfLseek(ctx->file.fd, offset, 0); + // tfLseek(ctx->file.pFile, offset, 0); #ifdef USE_MMAP int32_t last = ctx->file.size - offset; nRead = last >= len ? len : last; memcpy(buf, ctx->file.ptr + offset, nRead); #else - nRead = tfPread(ctx->file.fd, buf, len, offset); + nRead = taosPReadFile(ctx->file.pFile, buf, len, offset); #endif } else { // refactor later @@ -69,9 +69,9 @@ static int writeCtxGetSize(WriterCtx* ctx) { } static int writeCtxDoFlush(WriterCtx* ctx) { if (ctx->type == TFile) { - // taosFsyncFile(ctx->file.fd); - tfFsync(ctx->file.fd); - // tfFlush(ctx->file.fd); + // taosFsyncFile(ctx->file.pFile); + taosFsyncFile(ctx->file.pFile); + // tfFlush(ctx->file.pFile); } else { // do nothing } @@ -87,25 +87,25 @@ WriterCtx* writerCtxCreate(WriterType type, const char* path, bool readOnly, int // ugly code, refactor later ctx->file.readOnly = readOnly; if (readOnly == false) { - // ctx->file.fd = open(path, O_WRONLY | O_CREAT | O_APPEND, S_IRWXU | S_IRWXG | S_IRWXO); - ctx->file.fd = tfOpenCreateWriteAppend(path); - tfFtruncate(ctx->file.fd, 0); - struct stat fstat; - stat(path, &fstat); - ctx->file.size = fstat.st_size; + // ctx->file.pFile = open(path, O_WRONLY | O_CREAT | O_APPEND, S_IRWXU | S_IRWXG | S_IRWXO); + ctx->file.pFile = taosOpenFile(path, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_APPEND); + taosFtruncateFile(ctx->file.pFile, 0); + int64_t file_size; + taosStatFile(path, &file_size, NULL); + ctx->file.size = (int)file_size; } else { - // ctx->file.fd = open(path, O_RDONLY, S_IRWXU | S_IRWXG | S_IRWXO); - ctx->file.fd = tfOpenRead(path); + // ctx->file.pFile = open(path, O_RDONLY, S_IRWXU | S_IRWXG | S_IRWXO); + ctx->file.pFile = taosOpenFile(path, TD_FILE_READ); struct stat fstat; stat(path, &fstat); ctx->file.size = fstat.st_size; #ifdef USE_MMAP - ctx->file.ptr = (char*)tfMmapReadOnly(ctx->file.fd, ctx->file.size); + ctx->file.ptr = (char*)tfMmapReadOnly(ctx->file.pFile, ctx->file.size); #endif } memcpy(ctx->file.buf, path, strlen(path)); - if (ctx->file.fd < 0) { + if (ctx->file.pFile == NULL) { indexError("failed to open file, error %d", errno); goto END; } @@ -133,7 +133,7 @@ void writerCtxDestroy(WriterCtx* ctx, bool remove) { free(ctx->mem.buf); } else { ctx->flush(ctx); - tfClose(ctx->file.fd); + taosCloseFile(&ctx->file.pFile); if (ctx->file.readOnly) { #ifdef USE_MMAP munmap(ctx->file.ptr, ctx->file.size); diff --git a/source/libs/index/src/index_fst_util.c b/source/libs/index/src/index_fst_util.c index da1e177a18..f08a48c34e 100644 --- a/source/libs/index/src/index_fst_util.c +++ b/source/libs/index/src/index_fst_util.c @@ -29,18 +29,6 @@ const uint64_t VERSION = 3; const uint64_t TRANS_INDEX_THRESHOLD = 32; -// uint8_t commonInput(uint8_t idx) { -// if (idx == 0) { return -1; } -// else { -// return COMMON_INPUTS_INV[idx - 1]; -// } -//} -// -// uint8_t commonIdx(uint8_t v, uint8_t max) { -// uint8_t v = ((uint16_t)tCOMMON_INPUTS[v] + 1)%256; -// return v > max ? 0: v; -//} - uint8_t packSize(uint64_t n) { if (n < (1u << 8)) { return 1; @@ -103,9 +91,6 @@ FstSlice fstSliceCreate(uint8_t* data, uint64_t len) { FstSlice fstSliceCopy(FstSlice* s, int32_t start, int32_t end) { FstString* str = s->str; str->ref++; - // uint8_t *buf = fstSliceData(s, &alen); - // start = buf + start - (buf - s->start); - // end = buf + end - (buf - s->start); FstSlice t = {.str = str, .start = start + s->start, .end = end + s->start}; return t; @@ -130,19 +115,19 @@ FstSlice fstSliceDeepCopy(FstSlice* s, int32_t start, int32_t end) { ans.end = tlen - 1; return ans; } -bool fstSliceIsEmpty(FstSlice* s) { - return s->str == NULL || s->str->len == 0 || s->start < 0 || s->end < 0; -} +bool fstSliceIsEmpty(FstSlice* s) { return s->str == NULL || s->str->len == 0 || s->start < 0 || s->end < 0; } uint8_t* fstSliceData(FstSlice* s, int32_t* size) { FstString* str = s->str; - if (size != NULL) { *size = s->end - s->start + 1; } + if (size != NULL) { + *size = s->end - s->start + 1; + } return str->data + s->start; } void fstSliceDestroy(FstSlice* s) { FstString* str = s->str; str->ref--; - if (str->ref <= 0) { + if (str->ref == 0) { free(str->data); free(str); s->str = NULL; diff --git a/source/libs/index/src/index_tfile.c b/source/libs/index/src/index_tfile.c index 98fede4f7b..e44f8fc1c3 100644 --- a/source/libs/index/src/index_tfile.c +++ b/source/libs/index/src/index_tfile.c @@ -13,8 +13,6 @@ p * * along with this program. If not, see . */ -//#include -//#include #include "index_tfile.h" #include "index.h" #include "index_fst.h" @@ -61,7 +59,9 @@ static void tfileGenFileFullName(char* fullname, const char* path, uint64_t s TFileCache* tfileCacheCreate(const char* path) { TFileCache* tcache = calloc(1, sizeof(TFileCache)); - if (tcache == NULL) { return NULL; } + if (tcache == NULL) { + return NULL; + } tcache->tableCache = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); tcache->capacity = 64; @@ -98,7 +98,9 @@ End: return NULL; } void tfileCacheDestroy(TFileCache* tcache) { - if (tcache == NULL) { return; } + if (tcache == NULL) { + return; + } // free table cache TFileReader** reader = taosHashIterate(tcache->tableCache, NULL); @@ -119,7 +121,9 @@ TFileReader* tfileCacheGet(TFileCache* tcache, ICacheKey* key) { int32_t sz = indexSerialCacheKey(key, buf); assert(sz < sizeof(buf)); TFileReader** reader = taosHashGet(tcache->tableCache, buf, sz); - if (reader == NULL) { return NULL; } + if (reader == NULL) { + return NULL; + } tfileReaderRef(*reader); return *reader; @@ -142,7 +146,9 @@ void tfileCachePut(TFileCache* tcache, ICacheKey* key, TFileReader* reader) { } TFileReader* tfileReaderCreate(WriterCtx* ctx) { TFileReader* reader = calloc(1, sizeof(TFileReader)); - if (reader == NULL) { return NULL; } + if (reader == NULL) { + return NULL; + } reader->ctx = ctx; @@ -169,7 +175,9 @@ TFileReader* tfileReaderCreate(WriterCtx* ctx) { return reader; } void tfileReaderDestroy(TFileReader* reader) { - if (reader == NULL) { return; } + if (reader == NULL) { + return; + } // T_REF_INC(reader); fstDestroy(reader->fst); writerCtxDestroy(reader->ctx, reader->remove); @@ -209,7 +217,9 @@ TFileWriter* tfileWriterOpen(char* path, uint64_t suid, int32_t version, const c tfileGenFileFullName(fullname, path, suid, colName, version); // indexInfo("open write file name %s", fullname); WriterCtx* wcx = writerCtxCreate(TFile, fullname, false, 1024 * 1024 * 64); - if (wcx == NULL) { return NULL; } + if (wcx == NULL) { + return NULL; + } TFileHeader tfh = {0}; tfh.suid = suid; @@ -225,7 +235,9 @@ TFileReader* tfileReaderOpen(char* path, uint64_t suid, int32_t version, const c WriterCtx* wc = writerCtxCreate(TFile, fullname, true, 1024 * 1024 * 1024); indexInfo("open read file name:%s, size: %d", wc->file.buf, wc->file.size); - if (wc == NULL) { return NULL; } + if (wc == NULL) { + return NULL; + } TFileReader* reader = tfileReaderCreate(wc); return reader; @@ -316,19 +328,25 @@ int tfileWriterPut(TFileWriter* tw, void* data, bool order) { return 0; } void tfileWriterClose(TFileWriter* tw) { - if (tw == NULL) { return; } + if (tw == NULL) { + return; + } writerCtxDestroy(tw->ctx, false); free(tw); } void tfileWriterDestroy(TFileWriter* tw) { - if (tw == NULL) { return; } + if (tw == NULL) { + return; + } writerCtxDestroy(tw->ctx, false); free(tw); } IndexTFile* indexTFileCreate(const char* path) { TFileCache* cache = tfileCacheCreate(path); - if (cache == NULL) { return NULL; } + if (cache == NULL) { + return NULL; + } IndexTFile* tfile = calloc(1, sizeof(IndexTFile)); if (tfile == NULL) { @@ -340,21 +358,27 @@ IndexTFile* indexTFileCreate(const char* path) { return tfile; } void indexTFileDestroy(IndexTFile* tfile) { - if (tfile == NULL) { return; } + if (tfile == NULL) { + return; + } tfileCacheDestroy(tfile->cache); free(tfile); } int indexTFileSearch(void* tfile, SIndexTermQuery* query, SArray* result) { int ret = -1; - if (tfile == NULL) { return ret; } + if (tfile == NULL) { + return ret; + } - IndexTFile* pTfile = (IndexTFile*)tfile; + IndexTFile* pTfile = tfile; SIndexTerm* term = query->term; ICacheKey key = {.suid = term->suid, .colType = term->colType, .colName = term->colName, .nColName = term->nColName}; TFileReader* reader = tfileCacheGet(pTfile->cache, &key); - if (reader == NULL) { return 0; } + if (reader == NULL) { + return 0; + } return tfileReaderSearch(reader, query, result); } @@ -373,7 +397,9 @@ static bool tfileIteratorNext(Iterate* iiter) { TFileFstIter* tIter = iiter->iter; StreamWithStateResult* rt = streamWithStateNextWith(tIter->st, NULL); - if (rt == NULL) { return false; } + if (rt == NULL) { + return false; + } int32_t sz = 0; char* ch = (char*)fstSliceData(&rt->data, &sz); @@ -383,7 +409,9 @@ static bool tfileIteratorNext(Iterate* iiter) { offset = (uint64_t)(rt->out.out); swsResultDestroy(rt); // set up iterate value - if (tfileReaderLoadTableIds(tIter->rdr, offset, iv->val) != 0) { return false; } + if (tfileReaderLoadTableIds(tIter->rdr, offset, iv->val) != 0) { + return false; + } iv->colVal = colVal; return true; @@ -394,7 +422,9 @@ static IterateValue* tifileIterateGetValue(Iterate* iter) { return &iter->val; } static TFileFstIter* tfileFstIteratorCreate(TFileReader* reader) { TFileFstIter* tIter = calloc(1, sizeof(TFileFstIter)); - if (tIter == NULL) { return NULL; } + if (tIter == NULL) { + return NULL; + } tIter->ctx = automCtxCreate(NULL, AUTOMATION_ALWAYS); tIter->fb = fstSearch(reader->fst, tIter->ctx); @@ -404,7 +434,9 @@ static TFileFstIter* tfileFstIteratorCreate(TFileReader* reader) { } Iterate* tfileIteratorCreate(TFileReader* reader) { - if (reader == NULL) { return NULL; } + if (reader == NULL) { + return NULL; + } Iterate* iter = calloc(1, sizeof(Iterate)); iter->iter = tfileFstIteratorCreate(reader); @@ -419,7 +451,9 @@ Iterate* tfileIteratorCreate(TFileReader* reader) { return iter; } void tfileIteratorDestroy(Iterate* iter) { - if (iter == NULL) { return; } + if (iter == NULL) { + return; + } IterateValue* iv = &iter->val; iterateValueDestroy(iv, true); @@ -434,7 +468,9 @@ void tfileIteratorDestroy(Iterate* iter) { } TFileReader* tfileGetReaderByCol(IndexTFile* tf, uint64_t suid, char* colName) { - if (tf == NULL) { return NULL; } + if (tf == NULL) { + return NULL; + } ICacheKey key = {.suid = suid, .colType = TSDB_DATA_TYPE_BINARY, .colName = colName, .nColName = strlen(colName)}; return tfileCacheGet(tf->cache, &key); } @@ -446,7 +482,9 @@ static int tfileUidCompare(const void* a, const void* b) { } static int tfileStrCompare(const void* a, const void* b) { int ret = strcmp((char*)a, (char*)b); - if (ret == 0) { return ret; } + if (ret == 0) { + return ret; + } return ret < 0 ? -1 : 1; } @@ -461,13 +499,17 @@ static int tfileValueCompare(const void* a, const void* b, const void* param) { TFileValue* tfileValueCreate(char* val) { TFileValue* tf = calloc(1, sizeof(TFileValue)); - if (tf == NULL) { return NULL; } + if (tf == NULL) { + return NULL; + } tf->colVal = tstrdup(val); tf->tableId = taosArrayInit(32, sizeof(uint64_t)); return tf; } int tfileValuePush(TFileValue* tf, uint64_t val) { - if (tf == NULL) { return -1; } + if (tf == NULL) { + return -1; + } taosArrayPush(tf->tableId, &val); return 0; } @@ -489,7 +531,9 @@ static int tfileWriteFstOffset(TFileWriter* tw, int32_t offset) { int32_t fstOffset = offset + sizeof(tw->header.fstOffset); tw->header.fstOffset = fstOffset; - if (sizeof(fstOffset) != tw->ctx->write(tw->ctx, (char*)&fstOffset, sizeof(fstOffset))) { return -1; } + if (sizeof(fstOffset) != tw->ctx->write(tw->ctx, (char*)&fstOffset, sizeof(fstOffset))) { + return -1; + } indexInfo("tfile write fst offset: %d", tw->ctx->size(tw->ctx)); tw->offset += sizeof(fstOffset); return 0; @@ -502,7 +546,9 @@ static int tfileWriteHeader(TFileWriter* writer) { indexInfo("tfile pre write header size: %d", writer->ctx->size(writer->ctx)); int nwrite = writer->ctx->write(writer->ctx, buf, sizeof(buf)); - if (sizeof(buf) != nwrite) { return -1; } + if (sizeof(buf) != nwrite) { + return -1; + } indexInfo("tfile after write header size: %d", writer->ctx->size(writer->ctx)); writer->offset = nwrite; @@ -540,11 +586,11 @@ static int tfileReaderLoadHeader(TFileReader* reader) { int64_t nread = reader->ctx->readFrom(reader->ctx, buf, sizeof(buf), 0); if (nread == -1) { - indexError("actual Read: %d, to read: %d, errno: %d, filefd: %d, filename: %s", (int)(nread), (int)sizeof(buf), - errno, reader->ctx->file.fd, reader->ctx->file.buf); + indexError("actual Read: %d, to read: %d, errno: %d, filename: %s", (int)(nread), (int)sizeof(buf), + errno, reader->ctx->file.buf); } else { - indexInfo("actual Read: %d, to read: %d, filefd: %d, filename: %s", (int)(nread), (int)sizeof(buf), - reader->ctx->file.fd, reader->ctx->file.buf); + indexInfo("actual Read: %d, to read: %d, filename: %s", (int)(nread), (int)sizeof(buf), + reader->ctx->file.buf); } // assert(nread == sizeof(buf)); memcpy(&reader->header, buf, sizeof(buf)); @@ -552,23 +598,23 @@ static int tfileReaderLoadHeader(TFileReader* reader) { return 0; } static int tfileReaderLoadFst(TFileReader* reader) { - // current load fst into memory, refactor it later - static int FST_MAX_SIZE = 64 * 1024 * 1024; - - char* buf = calloc(1, sizeof(char) * FST_MAX_SIZE); - if (buf == NULL) { return -1; } - WriterCtx* ctx = reader->ctx; int size = ctx->size(ctx); + // current load fst into memory, refactor it later + int fstSize = size - reader->header.fstOffset - sizeof(tfileMagicNumber); + char* buf = calloc(1, fstSize); + if (buf == NULL) { + return -1; + } + int64_t ts = taosGetTimestampUs(); - int32_t nread = - ctx->readFrom(ctx, buf, size - reader->header.fstOffset - sizeof(tfileMagicNumber), reader->header.fstOffset); + int32_t nread = ctx->readFrom(ctx, buf, fstSize, reader->header.fstOffset); int64_t cost = taosGetTimestampUs() - ts; - indexInfo("nread = %d, and fst offset=%d, filename: %s, size: %d, time cost: %" PRId64 "us", nread, - reader->header.fstOffset, ctx->file.buf, ctx->file.size, cost); + indexInfo("nread = %d, and fst offset=%d, size: %d, filename: %s, size: %d, time cost: %" PRId64 "us", nread, + reader->header.fstOffset, fstSize, ctx->file.buf, ctx->file.size, cost); // we assuse fst size less than FST_MAX_SIZE - assert(nread > 0 && nread < FST_MAX_SIZE); + assert(nread > 0 && nread <= fstSize); FstSlice st = fstSliceCreate((uint8_t*)buf, nread); reader->fst = fstCreate(&st); @@ -578,21 +624,35 @@ static int tfileReaderLoadFst(TFileReader* reader) { return reader->fst != NULL ? 0 : -1; } static int tfileReaderLoadTableIds(TFileReader* reader, int32_t offset, SArray* result) { - int32_t nid; + // TODO(yihao): opt later WriterCtx* ctx = reader->ctx; + char block[1024] = {0}; + int32_t nread = ctx->readFrom(ctx, block, sizeof(block), offset); + assert(nread >= sizeof(uint32_t)); - int32_t nread = ctx->readFrom(ctx, (char*)&nid, sizeof(nid), offset); - assert(sizeof(nid) == nread); + char* p = block; + int32_t nid = *(int32_t*)p; + p += sizeof(nid); - int32_t total = sizeof(uint64_t) * nid; - char* buf = calloc(1, total); - if (buf == NULL) { return -1; } + while (nid > 0) { + int32_t left = block + sizeof(block) - p; + if (left >= sizeof(uint64_t)) { + taosArrayPush(result, (uint64_t*)p); + p += sizeof(uint64_t); + } else { + char buf[sizeof(uint64_t)] = {0}; + memcpy(buf, p, left); - nread = ctx->readFrom(ctx, buf, total, offset + sizeof(nid)); - assert(total == nread); + memset(block, 0, sizeof(block)); + offset += sizeof(block); + nread = ctx->readFrom(ctx, block, sizeof(block), offset); + memcpy(buf + left, block, sizeof(uint64_t) - left); - for (int32_t i = 0; i < nid; i++) { taosArrayPush(result, (uint64_t*)buf + i); } - free(buf); + taosArrayPush(result, (uint64_t*)buf); + p = block + sizeof(uint64_t) - left; + } + nid -= 1; + } return 0; } static int tfileReaderVerify(TFileReader* reader) { @@ -615,13 +675,17 @@ static int tfileReaderVerify(TFileReader* reader) { } void tfileReaderRef(TFileReader* reader) { - if (reader == NULL) { return; } + if (reader == NULL) { + return; + } int ref = T_REF_INC(reader); UNUSED(ref); } void tfileReaderUnRef(TFileReader* reader) { - if (reader == NULL) { return; } + if (reader == NULL) { + return; + } int ref = T_REF_DEC(reader); if (ref == 0) { // do nothing @@ -630,18 +694,21 @@ void tfileReaderUnRef(TFileReader* reader) { } static SArray* tfileGetFileList(const char* path) { - SArray* files = taosArrayInit(4, sizeof(void*)); - char buf[128] = {0}; uint64_t suid; uint32_t version; + SArray* files = taosArrayInit(4, sizeof(void*)); DIR* dir = opendir(path); - if (NULL == dir) { return NULL; } + if (NULL == dir) { + return NULL; + } struct dirent* entry; while ((entry = readdir(dir)) != NULL) { char* file = entry->d_name; - if (0 != tfileParseFileName(file, &suid, buf, &version)) { continue; } + if (0 != tfileParseFileName(file, &suid, buf, &version)) { + continue; + } size_t len = strlen(path) + 1 + strlen(file) + 1; char* buf = calloc(1, len); diff --git a/source/libs/index/src/index_util.c b/source/libs/index/src/index_util.c new file mode 100644 index 0000000000..fcaab968c2 --- /dev/null +++ b/source/libs/index/src/index_util.c @@ -0,0 +1,116 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#include "index_util.h" +#include "index.h" +typedef struct MergeIndex { + int idx; + int len; +} MergeIndex; + +static int iBinarySearch(SArray *arr, int s, int e, uint64_t k) { + uint64_t v; + int32_t m; + while (s <= e) { + m = s + (e - s) / 2; + v = *(uint64_t *)taosArrayGet(arr, m); + if (v >= k) { + e = m - 1; + } else { + s = m + 1; + } + } + return s; +} + +void iIntersection(SArray *inters, SArray *final) { + int32_t sz = taosArrayGetSize(inters); + if (sz <= 0) { + return; + } + MergeIndex *mi = calloc(sz, sizeof(MergeIndex)); + for (int i = 0; i < sz; i++) { + SArray *t = taosArrayGetP(inters, i); + mi[i].len = taosArrayGetSize(t); + mi[i].idx = 0; + } + + SArray *base = taosArrayGetP(inters, 0); + for (int i = 0; i < taosArrayGetSize(base); i++) { + uint64_t tgt = *(uint64_t *)taosArrayGet(base, i); + bool has = true; + for (int j = 1; j < taosArrayGetSize(inters); j++) { + SArray *oth = taosArrayGetP(inters, j); + int mid = iBinarySearch(oth, mi[j].idx, mi[j].len - 1, tgt); + if (mid >= 0 && mid < mi[j].len) { + uint64_t val = *(uint64_t *)taosArrayGet(oth, mid); + has = (val == tgt ? true : false); + mi[j].idx = mid; + } else { + has = false; + } + } + if (has == true) { + taosArrayPush(final, &tgt); + } + } + tfree(mi); +} +void iUnion(SArray *inters, SArray *final) { + int32_t sz = taosArrayGetSize(inters); + if (sz <= 0) { + return; + } + if (sz == 1) { + taosArrayAddAll(final, taosArrayGetP(inters, 0)); + return; + } + + MergeIndex *mi = calloc(sz, sizeof(MergeIndex)); + for (int i = 0; i < sz; i++) { + SArray *t = taosArrayGetP(inters, i); + mi[i].len = taosArrayGetSize(t); + mi[i].idx = 0; + } + while (1) { + uint64_t mVal = UINT_MAX; + int mIdx = -1; + + for (int j = 0; j < sz; j++) { + SArray *t = taosArrayGetP(inters, j); + if (mi[j].idx >= mi[j].len) { + continue; + } + uint64_t cVal = *(uint64_t *)taosArrayGet(t, mi[j].idx); + if (cVal < mVal) { + mVal = cVal; + mIdx = j; + } + } + if (mIdx != -1) { + mi[mIdx].idx++; + if (taosArrayGetSize(final) > 0) { + uint64_t lVal = *(uint64_t *)taosArrayGetLast(final); + if (lVal == mVal) { + continue; + } + } + taosArrayPush(final, &mVal); + } else { + break; + } + } + + tfree(mi); +} diff --git a/source/libs/index/test/CMakeLists.txt b/source/libs/index/test/CMakeLists.txt index 3957554748..bed42be3e5 100644 --- a/source/libs/index/test/CMakeLists.txt +++ b/source/libs/index/test/CMakeLists.txt @@ -1,5 +1,8 @@ add_executable(indexTest "") add_executable(fstTest "") +add_executable(fstUT "") +add_executable(UtilUT "") + target_sources(indexTest PRIVATE "indexTests.cc" @@ -8,6 +11,16 @@ target_sources(fstTest PRIVATE "fstTest.cc" ) + +target_sources(fstUT + PRIVATE + "fstUT.cc" +) +target_sources(UtilUT + PRIVATE + "utilUT.cc" +) + target_include_directories ( indexTest PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/index" @@ -18,6 +31,18 @@ target_include_directories ( fstTest "${CMAKE_SOURCE_DIR}/include/libs/index" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) + +target_include_directories ( fstUT + PUBLIC + "${CMAKE_SOURCE_DIR}/include/libs/index" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) + +target_include_directories ( UtilUT + PUBLIC + "${CMAKE_SOURCE_DIR}/include/libs/index" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) target_link_libraries (indexTest os util @@ -32,6 +57,21 @@ target_link_libraries (fstTest gtest_main index ) +target_link_libraries (fstUT + os + util + common + gtest_main + index +) + +target_link_libraries (UtilUT + os + util + common + gtest_main + index +) #add_test( diff --git a/source/libs/index/test/fstTest.cc b/source/libs/index/test/fstTest.cc index a2c0046f9a..65118a2bce 100644 --- a/source/libs/index/test/fstTest.cc +++ b/source/libs/index/test/fstTest.cc @@ -58,7 +58,9 @@ class FstReadMemory { bool init() { char* buf = (char*)calloc(1, sizeof(char) * _size); int nRead = fstCountingWriterRead(_w, (uint8_t*)buf, _size); - if (nRead <= 0) { return false; } + if (nRead <= 0) { + return false; + } _size = nRead; _s = fstSliceCreate((uint8_t*)buf, _size); _fst = fstCreate(&_s); @@ -97,7 +99,8 @@ class FstReadMemory { printf("key: %s, val: %" PRIu64 "\n", key.c_str(), (uint64_t)(rt->out.out)); swsResultDestroy(rt); } - for (size_t i = 0; i < result.size(); i++) {} + for (size_t i = 0; i < result.size(); i++) { + } std::cout << std::endl; return true; } @@ -173,7 +176,9 @@ void checkMillonWriteAndReadOfFst() { delete fw; FstReadMemory* fr = new FstReadMemory(1024 * 64 * 1024); - if (fr->init()) { printf("success to init fst read"); } + if (fr->init()) { + printf("success to init fst read"); + } Performance_fstReadRecords(fr); tfCleanup(); diff --git a/source/libs/index/test/fstUT.cc b/source/libs/index/test/fstUT.cc new file mode 100644 index 0000000000..9665198b3b --- /dev/null +++ b/source/libs/index/test/fstUT.cc @@ -0,0 +1,240 @@ + +#include +#include +#include +#include +#include +#include +#include "index.h" +#include "indexInt.h" +#include "index_cache.h" +#include "index_fst.h" +#include "index_fst_counting_writer.h" +#include "index_fst_util.h" +#include "index_tfile.h" +#include "tglobal.h" +#include "tskiplist.h" +#include "tutil.h" +#include "tlog.h" + +static std::string dir = "/tmp/index"; + +static char indexlog[PATH_MAX] = {0}; +static char tindex[PATH_MAX] = {0}; +static char tindexDir[PATH_MAX] = {0}; + +static void EnvInit() { + tfInit(); + + std::string path = dir; + taosRemoveDir(path.c_str()); + taosMkDir(path.c_str()); + // init log file + tstrncpy(tsLogDir, path.c_str(), PATH_MAX); + if (taosInitLog("tindex.idx", 1) != 0) { + printf("failed to init log"); + } + // init index file + memset(tindex, 0, sizeof(tindex)); + snprintf(tindex, PATH_MAX, "%s/tindex.idx", path.c_str()); +} +static void EnvCleanup() {} +class FstWriter { + public: + FstWriter() { + _wc = writerCtxCreate(TFile, tindex, false, 64 * 1024 * 1024); + _b = fstBuilderCreate(_wc, 0); + } + bool Put(const std::string& key, uint64_t val) { + // char buf[128] = {0}; + // int len = 0; + // taosMbsToUcs4(key.c_str(), key.size(), buf, 128, &len); + // FstSlice skey = fstSliceCreate((uint8_t*)buf, len); + FstSlice skey = fstSliceCreate((uint8_t*)key.c_str(), key.size()); + bool ok = fstBuilderInsert(_b, skey, val); + + fstSliceDestroy(&skey); + return ok; + } + ~FstWriter() { + fstBuilderFinish(_b); + fstBuilderDestroy(_b); + + writerCtxDestroy(_wc, false); + } + + private: + FstBuilder* _b; + WriterCtx* _wc; +}; + +class FstReadMemory { + public: + FstReadMemory(size_t size) { + _wc = writerCtxCreate(TFile, tindex, true, 64 * 1024); + _w = fstCountingWriterCreate(_wc); + _size = size; + memset((void*)&_s, 0, sizeof(_s)); + } + bool init() { + char* buf = (char*)calloc(1, sizeof(char) * _size); + int nRead = fstCountingWriterRead(_w, (uint8_t*)buf, _size); + if (nRead <= 0) { + return false; + } + _size = nRead; + _s = fstSliceCreate((uint8_t*)buf, _size); + _fst = fstCreate(&_s); + free(buf); + return _fst != NULL; + } + bool Get(const std::string& key, uint64_t* val) { + // char buf[128] = {0}; + // int len = 0; + // taosMbsToUcs4(key.c_str(), key.size(), buf, 128, &len); + // FstSlice skey = fstSliceCreate((uint8_t*)buf, len); + + FstSlice skey = fstSliceCreate((uint8_t*)key.c_str(), key.size()); + bool ok = fstGet(_fst, &skey, val); + fstSliceDestroy(&skey); + return ok; + } + + bool GetWithTimeCostUs(const std::string& key, uint64_t* val, uint64_t* elapse) { + int64_t s = taosGetTimestampUs(); + bool ok = this->Get(key, val); + int64_t e = taosGetTimestampUs(); + *elapse = e - s; + return ok; + } + // add later + bool Search(AutomationCtx* ctx, std::vector& result) { + FstStreamBuilder* sb = fstSearch(_fst, ctx); + StreamWithState* st = streamBuilderIntoStream(sb); + StreamWithStateResult* rt = NULL; + while ((rt = streamWithStateNextWith(st, NULL)) != NULL) { + // result.push_back((uint64_t)(rt->out.out)); + FstSlice* s = &rt->data; + int32_t sz = 0; + char* ch = (char*)fstSliceData(s, &sz); + std::string key(ch, sz); + printf("key: %s, val: %" PRIu64 "\n", key.c_str(), (uint64_t)(rt->out.out)); + swsResultDestroy(rt); + } + std::cout << std::endl; + return true; + } + bool SearchWithTimeCostUs(AutomationCtx* ctx, std::vector& result) { + int64_t s = taosGetTimestampUs(); + bool ok = this->Search(ctx, result); + int64_t e = taosGetTimestampUs(); + return ok; + } + + ~FstReadMemory() { + fstCountingWriterDestroy(_w); + fstDestroy(_fst); + fstSliceDestroy(&_s); + writerCtxDestroy(_wc, false); + // tfCleanup(); + } + + private: + FstCountingWriter* _w; + Fst* _fst; + FstSlice _s; + WriterCtx* _wc; + size_t _size; +}; + +class FstWriterEnv : public ::testing::Test { + protected: + virtual void SetUp() { fw = new FstWriter(); } + virtual void TearDown() { delete fw; } + FstWriter* fw = NULL; +}; + +class FstReadEnv : public ::testing::Test { + protected: + virtual void SetUp() { fr = new FstReadMemory(1024); } + virtual void TearDown() { delete fr; } + FstReadMemory* fr = NULL; +}; + +class TFst { + public: + void CreateWriter() { fw = new FstWriter; } + void ReCreateWriter() { + if (fw != NULL) delete fw; + fw = new FstWriter; + } + void DestroyWriter() { + if (fw != NULL) delete fw; + } + void CreateReader() { + fr = new FstReadMemory(1024); + fr->init(); + } + void ReCreateReader() { + if (fr != NULL) delete fr; + fr = new FstReadMemory(1024); + } + void DestroyReader() { + delete fr; + fr = NULL; + } + bool Put(const std::string& k, uint64_t v) { + if (fw == NULL) { + return false; + } + return fw->Put(k, v); + } + bool Get(const std::string& k, uint64_t* v) { + if (fr == NULL) { + return false; + } + return fr->Get(k, v); + } + bool Search(AutomationCtx* ctx, std::vector& result) { + // add more + return fr->Search(ctx, result); + } + + private: + FstWriter* fw; + FstReadMemory* fr; +}; +class FstEnv : public ::testing::Test { + protected: + virtual void SetUp() { + EnvInit(); + fst = new TFst; + } + virtual void TearDown() { delete fst; } + TFst* fst; +}; + +TEST_F(FstEnv, writeNormal) { + fst->CreateWriter(); + std::string str("aa"); + for (int i = 0; i < 10; i++) { + str[0] = 'a' + i; + str.resize(2); + assert(fst->Put(str, i) == true); + } + // order failed + assert(fst->Put("aa", 1) == false); + + fst->DestroyWriter(); + + fst->CreateReader(); + uint64_t val; + assert(fst->Get("a", &val) == false); + assert(fst->Get("aa", &val) == true); + assert(val == 0); + + std::vector rlt; + AutomationCtx* ctx = automCtxCreate((void*)"ab", AUTOMATION_ALWAYS); + assert(fst->Search(ctx, rlt) == true); +} +TEST_F(FstEnv, WriteMillonrRecord) {} diff --git a/source/libs/index/test/utilUT.cc b/source/libs/index/test/utilUT.cc new file mode 100644 index 0000000000..7c5ec19212 --- /dev/null +++ b/source/libs/index/test/utilUT.cc @@ -0,0 +1,227 @@ +#include +#include +#include +#include +#include +#include +#include "index.h" +#include "indexInt.h" +#include "index_cache.h" +#include "index_fst.h" +#include "index_fst_counting_writer.h" +#include "index_fst_util.h" +#include "index_tfile.h" +#include "index_util.h" +#include "tglobal.h" +#include "tskiplist.h" +#include "tutil.h" + +class UtilEnv : public ::testing::Test { + protected: + virtual void SetUp() { + src = (SArray *)taosArrayInit(2, sizeof(void *)); + for (int i = 0; i < 3; i++) { + SArray *m = taosArrayInit(10, sizeof(uint64_t)); + taosArrayPush(src, &m); + } + + rslt = (SArray *)taosArrayInit(10, sizeof(uint64_t)); + } + virtual void TearDown() { + for (int i = 0; i < taosArrayGetSize(src); i++) { + SArray *m = (SArray *)taosArrayGetP(src, i); + taosArrayDestroy(m); + } + taosArrayDestroy(src); + } + + SArray *src; + SArray *rslt; +}; + +static void clearSourceArray(SArray *p) { + for (int i = 0; i < taosArrayGetSize(p); i++) { + SArray *m = (SArray *)taosArrayGetP(p, i); + taosArrayClear(m); + } +} +static void clearFinalArray(SArray *p) { taosArrayClear(p); } +TEST_F(UtilEnv, intersectionSimpleResult) { + SArray *f = (SArray *)taosArrayGetP(src, 0); + for (int i = 0; i < 10; i++) { + uint64_t val = i; + taosArrayPush(f, &val); + } + f = (SArray *)taosArrayGetP(src, 1); + for (int i = 0; i < 10; i++) { + uint64_t val = i; + taosArrayPush(f, &val); + } + f = (SArray *)taosArrayGetP(src, 2); + for (int i = 0; i < 10; i++) { + uint64_t val = i; + taosArrayPush(f, &val); + } + iIntersection(src, rslt); + assert(taosArrayGetSize(rslt) == 10); + + clearSourceArray(src); + clearFinalArray(rslt); +} +TEST_F(UtilEnv, intersectMultiEmptyResult) { + SArray *f = (SArray *)taosArrayGetP(src, 0); + for (int i = 10; i < 20; i++) { + uint64_t val = i; + taosArrayPush(f, &val); + } + f = (SArray *)taosArrayGetP(src, 1); + for (int i = 0; i < 10; i++) { + uint64_t val = i; + taosArrayPush(f, &val); + } + f = (SArray *)taosArrayGetP(src, 2); + for (int i = 20; i < 30; i++) { + uint64_t val = i; + taosArrayPush(f, &val); + } + // empty source + iIntersection(src, rslt); + assert(taosArrayGetSize(rslt) == 0); + clearSourceArray(src); + clearFinalArray(rslt); +} +TEST_F(UtilEnv, intersectSimpleEmpty) { + clearSourceArray(src); + clearFinalArray(rslt); + + iIntersection(src, rslt); + assert(taosArrayGetSize(rslt) == 0); +} +TEST_F(UtilEnv, intersect01) { + clearSourceArray(src); + clearFinalArray(rslt); + + uint64_t arr1[] = {2, 3, 4, 5}; + SArray * f = (SArray *)taosArrayGetP(src, 0); + for (int i = 0; i < sizeof(arr1) / sizeof(arr1[0]); i++) { + taosArrayPush(f, &arr1[i]); + } + + uint64_t arr2[] = {1, 2, 3, 5}; + f = (SArray *)taosArrayGetP(src, 1); + for (int i = 0; i < sizeof(arr2) / sizeof(arr2[0]); i++) { + taosArrayPush(f, &arr2[i]); + } + + uint64_t arr3[] = {3, 5, 10, 11}; + f = (SArray *)taosArrayGetP(src, 2); + for (int i = 0; i < sizeof(arr3) / sizeof(arr3[0]); i++) { + taosArrayPush(f, &arr3[i]); + } + + iIntersection(src, rslt); + assert(taosArrayGetSize(rslt) == 2); +} +TEST_F(UtilEnv, intersect02) { + clearSourceArray(src); + clearFinalArray(rslt); + + uint64_t arr1[] = {13, 14, 15}; + SArray * f = (SArray *)taosArrayGetP(src, 0); + for (int i = 0; i < sizeof(arr1) / sizeof(arr1[0]); i++) { + taosArrayPush(f, &arr1[i]); + } + + uint64_t arr2[] = {8, 10, 12, 13}; + f = (SArray *)taosArrayGetP(src, 1); + for (int i = 0; i < sizeof(arr2) / sizeof(arr2[0]); i++) { + taosArrayPush(f, &arr2[i]); + } + + uint64_t arr3[] = {9, 10, 11}; + f = (SArray *)taosArrayGetP(src, 2); + for (int i = 0; i < sizeof(arr3) / sizeof(arr3[0]); i++) { + taosArrayPush(f, &arr3[i]); + } + + iIntersection(src, rslt); + assert(taosArrayGetSize(rslt) == 0); +} +TEST_F(UtilEnv, 01union) { + clearSourceArray(src); + clearFinalArray(rslt); + + uint64_t arr1[] = {13, 14, 15}; + SArray * f = (SArray *)taosArrayGetP(src, 0); + for (int i = 0; i < sizeof(arr1) / sizeof(arr1[0]); i++) { + taosArrayPush(f, &arr1[i]); + } + iUnion(src, rslt); + assert(taosArrayGetSize(rslt) == 3); +} +TEST_F(UtilEnv, 02union) { + clearSourceArray(src); + clearFinalArray(rslt); + + uint64_t arr1[] = {13, 14, 15}; + SArray * f = (SArray *)taosArrayGetP(src, 0); + for (int i = 0; i < sizeof(arr1) / sizeof(arr1[0]); i++) { + taosArrayPush(f, &arr1[i]); + } + + uint64_t arr2[] = {13, 14, 15}; + f = (SArray *)taosArrayGetP(src, 1); + for (int i = 0; i < sizeof(arr2) / sizeof(arr2[0]); i++) { + taosArrayPush(f, &arr2[i]); + } + iUnion(src, rslt); + assert(taosArrayGetSize(rslt) == 3); +} +TEST_F(UtilEnv, 03union) { + clearSourceArray(src); + clearFinalArray(rslt); + + uint64_t arr1[] = {13, 16, 18, 20}; + SArray * f = (SArray *)taosArrayGetP(src, 0); + for (int i = 0; i < sizeof(arr1) / sizeof(arr1[0]); i++) { + taosArrayPush(f, &arr1[i]); + } + + uint64_t arr2[] = {0, 12, 13, 20, 23}; + f = (SArray *)taosArrayGetP(src, 1); + for (int i = 0; i < sizeof(arr2) / sizeof(arr2[0]); i++) { + taosArrayPush(f, &arr2[i]); + } + + uint64_t arr3[] = {1, 12, 13, 16, 17}; + f = (SArray *)taosArrayGetP(src, 2); + for (int i = 0; i < sizeof(arr3) / sizeof(arr3[0]); i++) { + taosArrayPush(f, &arr3[i]); + } + iUnion(src, rslt); + assert(taosArrayGetSize(rslt) == 9); +} +TEST_F(UtilEnv, 04union) { + clearSourceArray(src); + clearFinalArray(rslt); + + uint64_t arr1[] = {1, 4, 5, 6}; + SArray * f = (SArray *)taosArrayGetP(src, 0); + for (int i = 0; i < sizeof(arr1) / sizeof(arr1[0]); i++) { + taosArrayPush(f, &arr1[i]); + } + + uint64_t arr2[] = {7, 8, 10}; + f = (SArray *)taosArrayGetP(src, 1); + for (int i = 0; i < sizeof(arr2) / sizeof(arr2[0]); i++) { + taosArrayPush(f, &arr2[i]); + } + + uint64_t arr3[] = {20, 21, 30, 100, 120}; + f = (SArray *)taosArrayGetP(src, 2); + for (int i = 0; i < sizeof(arr3) / sizeof(arr3[0]); i++) { + taosArrayPush(f, &arr3[i]); + } + iUnion(src, rslt); + assert(taosArrayGetSize(rslt) == 12); +} diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index 78606cd7a2..864e13b773 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -142,6 +142,13 @@ static SNode* functionNodeCopy(const SFunctionNode* pSrc, SFunctionNode* pDst) { return (SNode*)pDst; } +static SNode* targetNodeCopy(const STargetNode* pSrc, STargetNode* pDst) { + COPY_SCALAR_FIELD(tupleId); + COPY_SCALAR_FIELD(slotId); + COPY_NODE_FIELD(pExpr); + return (SNode*)pDst; +} + static SNode* groupingSetNodeCopy(const SGroupingSetNode* pSrc, SGroupingSetNode* pDst) { COPY_SCALAR_FIELD(groupingSetType); COPY_NODE_LIST_FIELD(pParameterList); @@ -168,6 +175,8 @@ SNode* nodesCloneNode(const SNode* pNode) { return logicConditionNodeCopy((const SLogicConditionNode*)pNode, (SLogicConditionNode*)pDst); case QUERY_NODE_FUNCTION: return functionNodeCopy((const SFunctionNode*)pNode, (SFunctionNode*)pDst); + case QUERY_NODE_TARGET: + return targetNodeCopy((const STargetNode*)pNode, (STargetNode*)pDst); case QUERY_NODE_REAL_TABLE: case QUERY_NODE_TEMP_TABLE: case QUERY_NODE_JOIN_TABLE: diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index 8968f0579d..92a6126750 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -55,12 +55,14 @@ static char* nodeName(ENodeType type) { return "NodeList"; case QUERY_NODE_FILL: return "Fill"; - case QUERY_NODE_COLUMN_REF: - return "ColumnRef"; case QUERY_NODE_TARGET: return "Target"; case QUERY_NODE_RAW_EXPR: return "RawExpr"; + case QUERY_NODE_TUPLE_DESC: + return "TupleDesc"; + case QUERY_NODE_SLOT_DESC: + return "SlotDesc"; case QUERY_NODE_SET_OPERATOR: return "SetOperator"; case QUERY_NODE_SELECT_STMT: @@ -71,16 +73,22 @@ static char* nodeName(ENodeType type) { return "LogicScan"; case QUERY_NODE_LOGIC_PLAN_JOIN: return "LogicJoin"; - case QUERY_NODE_LOGIC_PLAN_FILTER: - return "LogicFilter"; case QUERY_NODE_LOGIC_PLAN_AGG: return "LogicAgg"; case QUERY_NODE_LOGIC_PLAN_PROJECT: return "LogicProject"; + case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN: + return "PhysiTagScan"; + case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN: + return "PhysiTableScan"; + case QUERY_NODE_PHYSICAL_PLAN_PROJECT: + return "PhysiProject"; default: break; } - return "Unknown"; + static char tmp[20]; + snprintf(tmp, sizeof(tmp), "Unknown %d", type); + return tmp; } static int32_t addNodeList(SJson* pJson, const char* pName, FToJson func, const SNodeList* pList) { @@ -183,8 +191,93 @@ static int32_t logicJoinNodeToJson(const void* pObj, SJson* pJson) { return code; } -static int32_t logicFilterNodeToJson(const void* pObj, SJson* pJson) { - return logicPlanNodeToJson(pObj, pJson); +static const char* jkPhysiPlanOutputTuple = "OutputTuple"; +static const char* jkPhysiPlanConditions = "Conditions"; +static const char* jkPhysiPlanChildren = "Children"; + +static int32_t physicPlanNodeToJson(const void* pObj, SJson* pJson) { + const SPhysiNode* pNode = (const SPhysiNode*)pObj; + + int32_t code = tjsonAddObject(pJson, jkPhysiPlanOutputTuple, nodeToJson, &pNode->outputTuple); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkPhysiPlanConditions, nodeToJson, pNode->pConditions); + } + if (TSDB_CODE_SUCCESS == code) { + code = addNodeList(pJson, jkPhysiPlanChildren, nodeToJson, pNode->pChildren); + } + + return code; +} + +static const char* jkScanPhysiPlanScanCols = "ScanCols"; +static const char* jkScanPhysiPlanTableId = "TableId"; +static const char* jkScanPhysiPlanTableType = "TableType"; +static const char* jkScanPhysiPlanScanOrder = "ScanOrder"; +static const char* jkScanPhysiPlanScanCount = "ScanCount"; +static const char* jkScanPhysiPlanReverseScanCount = "ReverseScanCount"; + +static int32_t physiScanNodeToJson(const void* pObj, SJson* pJson) { + const STagScanPhysiNode* pNode = (const STagScanPhysiNode*)pObj; + + int32_t code = physicPlanNodeToJson(pObj, pJson); + if (TSDB_CODE_SUCCESS == code) { + code = addNodeList(pJson, jkScanPhysiPlanScanCols, nodeToJson, pNode->pScanCols); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkScanPhysiPlanTableId, pNode->uid); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkScanPhysiPlanTableType, pNode->tableType); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkScanPhysiPlanScanOrder, pNode->order); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkScanPhysiPlanScanCount, pNode->count); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkScanPhysiPlanReverseScanCount, pNode->reverse); + } + + return code; +} + +static int32_t physiTagScanNodeToJson(const void* pObj, SJson* pJson) { + return physiScanNodeToJson(pObj, pJson); +} + +static const char* jkTableScanPhysiPlanScanFlag = "ScanFlag"; +static const char* jkTableScanPhysiPlanStartKey = "StartKey"; +static const char* jkTableScanPhysiPlanEndKey = "EndKey"; + +static int32_t physiTableScanNodeToJson(const void* pObj, SJson* pJson) { + const STableScanPhysiNode* pNode = (const STableScanPhysiNode*)pObj; + + int32_t code = physiScanNodeToJson(pObj, pJson); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkTableScanPhysiPlanScanFlag, pNode->scanFlag); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkTableScanPhysiPlanStartKey, pNode->scanRange.skey); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkTableScanPhysiPlanEndKey, pNode->scanRange.ekey); + } + + return code; +} + +static const char* jkProjectPhysiPlanProjections = "Projections"; + +static int32_t physiProjectNodeToJson(const void* pObj, SJson* pJson) { + const SProjectPhysiNode* pNode = (const SProjectPhysiNode*)pObj; + + int32_t code = physicPlanNodeToJson(pObj, pJson); + if (TSDB_CODE_SUCCESS == code) { + code = addNodeList(pJson, jkProjectPhysiPlanProjections, nodeToJson, pNode->pProjections); + } + + return code; } static const char* jkAggLogicPlanGroupKeys = "GroupKeys"; @@ -277,19 +370,6 @@ static int32_t columnNodeToJson(const void* pObj, SJson* pJson) { return code; } -// typedef struct SValueNode { -// SExprNode node; // QUERY_NODE_VALUE -// char* ; -// bool ; -// union { -// bool b; -// int64_t i; -// uint64_t u; -// double d; -// char* p; -// } datum; -// } SValueNode; - static const char* jkValueLiteral = "Literal"; static const char* jkValueDuration = "Duration"; static const char* jkValueDatum = "Datum"; @@ -421,6 +501,52 @@ static int32_t groupingSetNodeToJson(const void* pObj, SJson* pJson) { return code; } +static const char* jkTargetTupleId = "TupleId"; +static const char* jkTargetSlotId = "SlotId"; +static const char* jkTargetExpr = "Expr"; + +static int32_t targetNodeToJson(const void* pObj, SJson* pJson) { + const STargetNode* pNode = (const STargetNode*)pObj; + + int32_t code = tjsonAddIntegerToObject(pJson, jkTargetTupleId, pNode->tupleId); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkTargetSlotId, pNode->slotId); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkTargetExpr, nodeToJson, pNode->pExpr); + } + + return code; +} + +static const char* jkSlotDescSlotId = "SlotId"; +static const char* jkSlotDescDataType = "DataType"; + +static int32_t slotDescNodeToJson(const void* pObj, SJson* pJson) { + const SSlotDescNode* pNode = (const SSlotDescNode*)pObj; + + int32_t code = tjsonAddIntegerToObject(pJson, jkSlotDescSlotId, pNode->slotId); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkSlotDescDataType, dataTypeToJson, &pNode->dataType); + } + + return code; +} + +static const char* jkTupleDescTupleId = "TupleId"; +static const char* jkTupleDescSlots = "Slots"; + +static int32_t tupleDescNodeToJson(const void* pObj, SJson* pJson) { + const STupleDescNode* pNode = (const STupleDescNode*)pObj; + + int32_t code = tjsonAddIntegerToObject(pJson, jkTupleDescTupleId, pNode->tupleId); + if (TSDB_CODE_SUCCESS == code) { + code = addNodeList(pJson, jkTupleDescSlots, nodeToJson, pNode->pSlots); + } + + return code; +} + static const char* jkSelectStmtDistinct = "Distinct"; static const char* jkSelectStmtProjections = "Projections"; static const char* jkSelectStmtFrom = "From"; @@ -496,9 +622,14 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { case QUERY_NODE_INTERVAL_WINDOW: case QUERY_NODE_NODE_LIST: case QUERY_NODE_FILL: - case QUERY_NODE_COLUMN_REF: case QUERY_NODE_TARGET: + return targetNodeToJson(pObj, pJson); case QUERY_NODE_RAW_EXPR: + break; + case QUERY_NODE_TUPLE_DESC: + return tupleDescNodeToJson(pObj, pJson); + case QUERY_NODE_SLOT_DESC: + return slotDescNodeToJson(pObj, pJson); case QUERY_NODE_SET_OPERATOR: break; case QUERY_NODE_SELECT_STMT: @@ -509,12 +640,16 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { return logicScanNodeToJson(pObj, pJson); case QUERY_NODE_LOGIC_PLAN_JOIN: return logicJoinNodeToJson(pObj, pJson); - case QUERY_NODE_LOGIC_PLAN_FILTER: - return logicFilterNodeToJson(pObj, pJson); case QUERY_NODE_LOGIC_PLAN_AGG: return logicAggNodeToJson(pObj, pJson); case QUERY_NODE_LOGIC_PLAN_PROJECT: return logicProjectNodeToJson(pObj, pJson); + case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN: + return physiTagScanNodeToJson(pObj, pJson); + case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN: + return physiTableScanNodeToJson(pObj, pJson); + case QUERY_NODE_PHYSICAL_PLAN_PROJECT: + return physiProjectNodeToJson(pObj, pJson); default: break; } diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index f2205f511e..8810f24ef0 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -75,12 +75,26 @@ SNode* nodesMakeNode(ENodeType type) { return makeNode(type, sizeof(SScanLogicNode)); case QUERY_NODE_LOGIC_PLAN_JOIN: return makeNode(type, sizeof(SJoinLogicNode)); - case QUERY_NODE_LOGIC_PLAN_FILTER: - return makeNode(type, sizeof(SFilterLogicNode)); case QUERY_NODE_LOGIC_PLAN_AGG: return makeNode(type, sizeof(SAggLogicNode)); case QUERY_NODE_LOGIC_PLAN_PROJECT: return makeNode(type, sizeof(SProjectLogicNode)); + case QUERY_NODE_TARGET: + return makeNode(type, sizeof(STargetNode)); + case QUERY_NODE_TUPLE_DESC: + return makeNode(type, sizeof(STupleDescNode)); + case QUERY_NODE_SLOT_DESC: + return makeNode(type, sizeof(SSlotDescNode)); + case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN: + return makeNode(type, sizeof(STagScanPhysiNode)); + case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN: + return makeNode(type, sizeof(STableScanPhysiNode)); + case QUERY_NODE_PHYSICAL_PLAN_PROJECT: + return makeNode(type, sizeof(SProjectPhysiNode)); + case QUERY_NODE_PHYSICAL_PLAN_JOIN: + return makeNode(type, sizeof(SJoinPhysiNode)); + case QUERY_NODE_PHYSICAL_PLAN_AGG: + return makeNode(type, sizeof(SAggPhysiNode)); default: break; } @@ -184,29 +198,29 @@ void nodesDestroyList(SNodeList* pList) { tfree(pList); } -void *nodesGetValueFromNode(SValueNode *pNode) { +void* nodesGetValueFromNode(SValueNode *pNode) { switch (pNode->node.resType.type) { case TSDB_DATA_TYPE_BOOL: - return (void *)&pNode->datum.b; + return (void*)&pNode->datum.b; case TSDB_DATA_TYPE_TINYINT: case TSDB_DATA_TYPE_SMALLINT: case TSDB_DATA_TYPE_INT: case TSDB_DATA_TYPE_BIGINT: case TSDB_DATA_TYPE_TIMESTAMP: - return (void *)&pNode->datum.i; + return (void*)&pNode->datum.i; case TSDB_DATA_TYPE_UTINYINT: case TSDB_DATA_TYPE_USMALLINT: case TSDB_DATA_TYPE_UINT: case TSDB_DATA_TYPE_UBIGINT: - return (void *)&pNode->datum.u; + return (void*)&pNode->datum.u; case TSDB_DATA_TYPE_FLOAT: case TSDB_DATA_TYPE_DOUBLE: - return (void *)&pNode->datum.d; + return (void*)&pNode->datum.d; case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_NCHAR: case TSDB_DATA_TYPE_VARCHAR: case TSDB_DATA_TYPE_VARBINARY: - return (void *)pNode->datum.p; + return (void*)pNode->datum.p; default: break; } diff --git a/source/libs/parser/src/parserImpl.c b/source/libs/parser/src/parserImpl.c index cf8d05975b..ef040fdff4 100644 --- a/source/libs/parser/src/parserImpl.c +++ b/source/libs/parser/src/parserImpl.c @@ -500,22 +500,24 @@ static EDealRes translateColumn(STranslateContext* pCxt, SColumnNode* pCol) { } static int32_t trimStringCopy(const char* src, int32_t len, char* dst) { + varDataSetLen(dst, len); + char* dstVal = varDataVal(dst); // delete escape character: \\, \', \" char delim = src[0]; int32_t cnt = 0; int32_t j = 0; for (uint32_t k = 1; k < len - 1; ++k) { if (src[k] == '\\' || (src[k] == delim && src[k + 1] == delim)) { - dst[j] = src[k + 1]; + dstVal[j] = src[k + 1]; cnt++; j++; k++; continue; } - dst[j] = src[k]; + dstVal[j] = src[k]; j++; } - dst[j] = '\0'; + dstVal[j] = '\0'; return j; } @@ -560,7 +562,7 @@ static EDealRes translateValue(STranslateContext* pCxt, SValueNode* pVal) { case TSDB_DATA_TYPE_VARCHAR: case TSDB_DATA_TYPE_VARBINARY: { int32_t n = strlen(pVal->literal); - pVal->datum.p = calloc(1, n); + pVal->datum.p = calloc(1, n + VARSTR_HEADER_SIZE); if (NULL == pVal->datum.p) { generateSyntaxErrMsg(pCxt, TSDB_CODE_OUT_OF_MEMORY); return DEAL_RES_ERROR; diff --git a/source/libs/parser/src/parserUtil.c b/source/libs/parser/src/parserUtil.c index 4fd0de3803..ec68980c44 100644 --- a/source/libs/parser/src/parserUtil.c +++ b/source/libs/parser/src/parserUtil.c @@ -884,7 +884,7 @@ SColumnFilterInfo* tFilterInfoDup(const SColumnFilterInfo* src, int32_t numOfFil } assert(src->filterstr == 0 || src->filterstr == 1); - assert(!(src->lowerRelOptr == TSDB_RELATION_INVALID && src->upperRelOptr == TSDB_RELATION_INVALID)); + assert(!(src->lowerRelOptr == 0 && src->upperRelOptr == 0)); return pFilter; } @@ -1507,45 +1507,45 @@ int32_t getTagFilterSerializeLen(SQueryStmtInfo* pQueryInfo) { uint32_t convertRelationalOperator(SToken *pToken) { switch (pToken->type) { case TK_LT: - return TSDB_RELATION_LESS; + return OP_TYPE_LOWER_THAN; case TK_LE: - return TSDB_RELATION_LESS_EQUAL; + return OP_TYPE_LOWER_EQUAL; case TK_GT: - return TSDB_RELATION_GREATER; + return OP_TYPE_GREATER_THAN; case TK_GE: - return TSDB_RELATION_GREATER_EQUAL; + return OP_TYPE_GREATER_EQUAL; case TK_NE: - return TSDB_RELATION_NOT_EQUAL; + return OP_TYPE_NOT_EQUAL; case TK_AND: - return TSDB_RELATION_AND; + return LOGIC_COND_TYPE_AND; case TK_OR: - return TSDB_RELATION_OR; + return LOGIC_COND_TYPE_OR; case TK_EQ: - return TSDB_RELATION_EQUAL; + return OP_TYPE_EQUAL; case TK_PLUS: - return TSDB_BINARY_OP_ADD; + return OP_TYPE_ADD; case TK_MINUS: - return TSDB_BINARY_OP_SUBTRACT; + return OP_TYPE_SUB; case TK_STAR: - return TSDB_BINARY_OP_MULTIPLY; + return OP_TYPE_MULTI; case TK_SLASH: case TK_DIVIDE: - return TSDB_BINARY_OP_DIVIDE; + return OP_TYPE_DIV; case TK_REM: - return TSDB_BINARY_OP_REMAINDER; + return OP_TYPE_MOD; case TK_LIKE: - return TSDB_RELATION_LIKE; + return OP_TYPE_LIKE; case TK_MATCH: - return TSDB_RELATION_MATCH; + return OP_TYPE_MATCH; case TK_NMATCH: - return TSDB_RELATION_NMATCH; + return OP_TYPE_NMATCH; case TK_ISNULL: - return TSDB_RELATION_ISNULL; + return OP_TYPE_IS_NULL; case TK_NOTNULL: - return TSDB_RELATION_NOTNULL; + return OP_TYPE_IS_NOT_NULL; case TK_IN: - return TSDB_RELATION_IN; + return OP_TYPE_IN; default: { return 0; } } } diff --git a/source/libs/parser/test/parserTests.cpp b/source/libs/parser/test/parserTests.cpp index b408e15abe..b971760132 100644 --- a/source/libs/parser/test/parserTests.cpp +++ b/source/libs/parser/test/parserTests.cpp @@ -593,7 +593,7 @@ TEST(testCase, function_Test6) { SExprInfo* p2 = (SExprInfo*) taosArrayGetP(pQueryInfo->exprList[1], 0); ASSERT_EQ(p2->pExpr->nodeType, TEXPR_BINARYEXPR_NODE); - ASSERT_EQ(p2->pExpr->_node.optr, TSDB_BINARY_OP_ADD); + ASSERT_EQ(p2->pExpr->_node.optr, OP_TYPE_ADD); ASSERT_EQ(p2->pExpr->_node.pLeft->nodeType, TEXPR_COL_NODE); ASSERT_EQ(p2->pExpr->_node.pRight->nodeType, TEXPR_COL_NODE); diff --git a/source/libs/planner/inc/plannerImpl.h b/source/libs/planner/inc/plannerImpl.h index d89cc26700..559d614829 100644 --- a/source/libs/planner/inc/plannerImpl.h +++ b/source/libs/planner/inc/plannerImpl.h @@ -24,6 +24,7 @@ extern "C" { #include "planner.h" int32_t createLogicPlan(SNode* pNode, SLogicNode** pLogicNode); +int32_t createPhysiPlan(SLogicNode* pLogicNode, SPhysiNode** pPhyNode); #ifdef __cplusplus } diff --git a/source/libs/planner/src/plannerImpl.c b/source/libs/planner/src/plannerImpl.c index 54a7330667..d5b5eb1500 100644 --- a/source/libs/planner/src/plannerImpl.c +++ b/source/libs/planner/src/plannerImpl.c @@ -19,7 +19,6 @@ #define CHECK_ALLOC(p, res) \ do { \ if (NULL == (p)) { \ - printf("%s : %d\n", __FUNCTION__, __LINE__); \ pCxt->errCode = TSDB_CODE_OUT_OF_MEMORY; \ return (res); \ } \ @@ -29,7 +28,6 @@ do { \ int32_t code = (exec); \ if (TSDB_CODE_SUCCESS != code) { \ - printf("%s : %d\n", __FUNCTION__, __LINE__); \ pCxt->errCode = code; \ return (res); \ } \ @@ -38,7 +36,6 @@ typedef struct SPlanContext { int32_t errCode; int32_t planNodeId; - SNodeList* pResource; } SPlanContext; static SLogicNode* createQueryLogicNode(SPlanContext* pCxt, SNode* pStmt); @@ -58,12 +55,12 @@ static EDealRes doRewriteExpr(SNode** pNode, void* pContext) { SNode* pExpr; int32_t index = 0; FOREACH(pExpr, pCxt->pExprs) { + if (QUERY_NODE_GROUPING_SET == nodeType(pExpr)) { + pExpr = nodesListGetNode(((SGroupingSetNode*)pExpr)->pParameterList, 0); + } if (nodesEqualNode(pExpr, *pNode)) { SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); - if (NULL == pCol) { - pCxt->errCode = TSDB_CODE_OUT_OF_MEMORY; - return DEAL_RES_ERROR; - } + CHECK_ALLOC(pCol, DEAL_RES_ERROR); SExprNode* pToBeRewrittenExpr = (SExprNode*)(*pNode); pCol->node.resType = pToBeRewrittenExpr->resType; strcpy(pCol->node.aliasName, pToBeRewrittenExpr->aliasName); @@ -222,26 +219,6 @@ static SLogicNode* createLogicNodeByTable(SPlanContext* pCxt, SSelectStmt* pSele return NULL; } -static SLogicNode* createWhereFilterLogicNode(SPlanContext* pCxt, SLogicNode* pChild, SSelectStmt* pSelect) { - if (NULL == pSelect->pWhere) { - return NULL; - } - - SFilterLogicNode* pFilter = (SFilterLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_FILTER); - CHECK_ALLOC(pFilter, NULL); - pFilter->node.id = pCxt->planNodeId++; - - // set filter conditions - pFilter->node.pConditions = nodesCloneNode(pSelect->pWhere); - CHECK_ALLOC(pFilter->node.pConditions, (SLogicNode*)pFilter); - - // set the output - pFilter->node.pTargets = nodesCloneList(pChild->pTargets); - CHECK_ALLOC(pFilter->node.pTargets, (SLogicNode*)pFilter); - - return (SLogicNode*)pFilter; -} - typedef struct SCreateColumnCxt { int32_t errCode; SNodeList* pList; @@ -252,10 +229,8 @@ static EDealRes doCreateColumn(SNode* pNode, void* pContext) { switch (nodeType(pNode)) { case QUERY_NODE_COLUMN: { SNode* pCol = nodesCloneNode(pNode); - if (NULL == pCol || TSDB_CODE_SUCCESS != nodesListAppend(pCxt->pList, pCol)) { - pCxt->errCode = TSDB_CODE_OUT_OF_MEMORY; - return DEAL_RES_ERROR; - } + CHECK_ALLOC(pCol, DEAL_RES_ERROR); + CHECK_CODE(nodesListAppend(pCxt->pList, pCol), DEAL_RES_ERROR); return DEAL_RES_IGNORE_CHILD; } case QUERY_NODE_OPERATOR: @@ -263,16 +238,10 @@ static EDealRes doCreateColumn(SNode* pNode, void* pContext) { case QUERY_NODE_FUNCTION: { SExprNode* pExpr = (SExprNode*)pNode; SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); - if (NULL == pCol) { - pCxt->errCode = TSDB_CODE_OUT_OF_MEMORY; - return DEAL_RES_ERROR; - } + CHECK_ALLOC(pCol, DEAL_RES_ERROR); pCol->node.resType = pExpr->resType; strcpy(pCol->colName, pExpr->aliasName); - if (TSDB_CODE_SUCCESS != nodesListAppend(pCxt->pList, (SNode*)pCol)) { - pCxt->errCode = TSDB_CODE_OUT_OF_MEMORY; - return DEAL_RES_ERROR; - } + CHECK_CODE(nodesListAppend(pCxt->pList, (SNode*)pCol), DEAL_RES_ERROR); return DEAL_RES_IGNORE_CHILD; } default: @@ -284,9 +253,8 @@ static EDealRes doCreateColumn(SNode* pNode, void* pContext) { static SNodeList* createColumnByRewriteExps(SPlanContext* pCxt, SNodeList* pExprs) { SCreateColumnCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pList = nodesMakeList() }; - if (NULL == cxt.pList) { - return NULL; - } + CHECK_ALLOC(cxt.pList, NULL); + nodesWalkList(pExprs, doCreateColumn, &cxt); if (TSDB_CODE_SUCCESS != cxt.errCode) { nodesDestroyList(cxt.pList); @@ -379,8 +347,9 @@ static SLogicNode* createProjectLogicNode(SPlanContext* pCxt, SSelectStmt* pSele static SLogicNode* createSelectLogicNode(SPlanContext* pCxt, SSelectStmt* pSelect) { SLogicNode* pRoot = createLogicNodeByTable(pCxt, pSelect, pSelect->pFromTable); - if (TSDB_CODE_SUCCESS == pCxt->errCode) { - pRoot = pushLogicNode(pCxt, pRoot, createWhereFilterLogicNode(pCxt, pRoot, pSelect)); + if (TSDB_CODE_SUCCESS == pCxt->errCode && NULL != pSelect->pWhere) { + pRoot->pConditions = nodesCloneNode(pSelect->pWhere); + CHECK_ALLOC(pRoot->pConditions, pRoot); } if (TSDB_CODE_SUCCESS == pCxt->errCode) { pRoot = pushLogicNode(pCxt, pRoot, createAggLogicNode(pCxt, pSelect)); @@ -410,3 +379,479 @@ int32_t createLogicPlan(SNode* pNode, SLogicNode** pLogicNode) { *pLogicNode = pRoot; return TSDB_CODE_SUCCESS; } + +int32_t optimize(SLogicNode* pLogicNode) { + // todo + return TSDB_CODE_SUCCESS; +} + +typedef struct SSubLogicPlan { + SNode* pRoot; // SLogicNode + bool haveSuperTable; + bool haveSystemTable; +} SSubLogicPlan; + +int32_t splitLogicPlan(SSubLogicPlan* pLogicPlan) { + // todo + return TSDB_CODE_SUCCESS; +} + +typedef struct SSlotIndex { + int16_t tupleId; + int16_t slotId; +} SSlotIndex; + +typedef struct SPhysiPlanContext { + int32_t errCode; + int16_t nextTupleId; + SArray* pTupleHelper; +} SPhysiPlanContext; + +static int32_t getSlotKey(SNode* pNode, char* pKey) { + if (QUERY_NODE_COLUMN == nodeType(pNode)) { + SColumnNode* pCol = (SColumnNode*)pNode; + if ('\0' == pCol->tableAlias[0]) { + return sprintf(pKey, "%s", pCol->colName); + } + return sprintf(pKey, "%s.%s", pCol->tableAlias, pCol->colName); + } + return sprintf(pKey, "%s", ((SExprNode*)pNode)->aliasName); +} + +static SNode* createSlotDesc(SPhysiPlanContext* pCxt, const SNode* pNode, int16_t slotId) { + SSlotDescNode* pSlot = (SSlotDescNode*)nodesMakeNode(QUERY_NODE_SLOT_DESC); + CHECK_ALLOC(pSlot, NULL); + pSlot->slotId = slotId; + pSlot->dataType = ((SExprNode*)pNode)->resType; + pSlot->reserve = false; + pSlot->output = false; + return (SNode*)pSlot; +} + +static SNode* createTarget(SNode* pNode, int16_t tupleId, int16_t slotId) { + STargetNode* pTarget = (STargetNode*)nodesMakeNode(QUERY_NODE_TARGET); + if (NULL == pTarget) { + return NULL; + } + pTarget->tupleId = tupleId; + pTarget->slotId = slotId; + pTarget->pExpr = pNode; + return (SNode*)pTarget; +} + +static int32_t addTupleDesc(SPhysiPlanContext* pCxt, SNodeList* pList, STupleDescNode* pTuple) { + SHashObj* pHash = NULL; + if (NULL == pTuple->pSlots) { + pTuple->pSlots = nodesMakeList(); + CHECK_ALLOC(pTuple->pSlots, TSDB_CODE_OUT_OF_MEMORY); + + pHash = taosHashInit(LIST_LENGTH(pList), taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + CHECK_ALLOC(pHash, TSDB_CODE_OUT_OF_MEMORY); + if (NULL == taosArrayInsert(pCxt->pTupleHelper, pTuple->tupleId, &pHash)) { + taosHashCleanup(pHash); + return TSDB_CODE_OUT_OF_MEMORY; + } + } else { + pHash = taosArrayGetP(pCxt->pTupleHelper, pTuple->tupleId); + } + + SNode* pNode = NULL; + int16_t slotId = taosHashGetSize(pHash); + FOREACH(pNode, pList) { + SNode* pSlot = createSlotDesc(pCxt, pNode, slotId); + CHECK_ALLOC(pSlot, TSDB_CODE_OUT_OF_MEMORY); + if (TSDB_CODE_SUCCESS != nodesListAppend(pTuple->pSlots, (SNode*)pSlot)) { + nodesDestroyNode(pSlot); + return TSDB_CODE_OUT_OF_MEMORY; + } + + SSlotIndex index = { .tupleId = pTuple->tupleId, .slotId = slotId }; + char name[TSDB_TABLE_NAME_LEN + TSDB_COL_NAME_LEN]; + int32_t len = getSlotKey(pNode, name); + CHECK_CODE(taosHashPut(pHash, name, len, &index, sizeof(SSlotIndex)), TSDB_CODE_OUT_OF_MEMORY); + + SNode* pTarget = createTarget(pNode, pTuple->tupleId, slotId); + CHECK_ALLOC(pTarget, TSDB_CODE_OUT_OF_MEMORY); + REPLACE_NODE(pTarget); + + ++slotId; + } + return TSDB_CODE_SUCCESS; +} + +typedef struct SSetSlotIdCxt { + int32_t errCode; + SHashObj* pLeftHash; + SHashObj* pRightHash; +} SSetSlotIdCxt; + +static EDealRes doSetSlotId(SNode* pNode, void* pContext) { + if (QUERY_NODE_COLUMN == nodeType(pNode) && 0 != strcmp(((SColumnNode*)pNode)->colName, "*")) { + SSetSlotIdCxt* pCxt = (SSetSlotIdCxt*)pContext; + char name[TSDB_TABLE_NAME_LEN + TSDB_COL_NAME_LEN]; + int32_t len = getSlotKey(pNode, name); + SSlotIndex* pIndex = taosHashGet(pCxt->pLeftHash, name, len); + if (NULL == pIndex) { + pIndex = taosHashGet(pCxt->pRightHash, name, len); + } + // pIndex is definitely not NULL, otherwise it is a bug + ((SColumnNode*)pNode)->tupleId = pIndex->tupleId; + ((SColumnNode*)pNode)->slotId = pIndex->slotId; + CHECK_ALLOC(pNode, DEAL_RES_ERROR); + return DEAL_RES_IGNORE_CHILD; + } + return DEAL_RES_CONTINUE; +} + +static SNode* setNodeSlotId(SPhysiPlanContext* pCxt, int16_t leftTupleId, int16_t rightTupleId, SNode* pNode) { + SNode* pRes = nodesCloneNode(pNode); + CHECK_ALLOC(pRes, NULL); + SSetSlotIdCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pLeftHash = taosArrayGetP(pCxt->pTupleHelper, leftTupleId), + .pRightHash = (rightTupleId < 0 ? NULL : taosArrayGetP(pCxt->pTupleHelper, rightTupleId)) }; + nodesWalkNode(pRes, doSetSlotId, &cxt); + if (TSDB_CODE_SUCCESS != cxt.errCode) { + nodesDestroyNode(pRes); + return NULL; + } + return pRes; +} + +static SNodeList* setListSlotId(SPhysiPlanContext* pCxt, int16_t leftTupleId, int16_t rightTupleId, SNodeList* pList) { + SNodeList* pRes = nodesCloneList(pList); + CHECK_ALLOC(pRes, NULL); + SSetSlotIdCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pLeftHash = taosArrayGetP(pCxt->pTupleHelper, leftTupleId), + .pRightHash = (rightTupleId < 0 ? NULL : taosArrayGetP(pCxt->pTupleHelper, rightTupleId)) }; + nodesWalkList(pRes, doSetSlotId, &cxt); + if (TSDB_CODE_SUCCESS != cxt.errCode) { + nodesDestroyList(pRes); + return NULL; + } + return pRes; +} + +static SPhysiNode* makePhysiNode(SPhysiPlanContext* pCxt, ENodeType type) { + SPhysiNode* pPhysiNode = (SPhysiNode*)nodesMakeNode(type); + if (NULL == pPhysiNode) { + return NULL; + } + pPhysiNode->outputTuple.tupleId = pCxt->nextTupleId++; + pPhysiNode->outputTuple.type = QUERY_NODE_TUPLE_DESC; + return pPhysiNode; +} + +static int32_t setConditionsSlotId(SPhysiPlanContext* pCxt, const SLogicNode* pLogicNode, SPhysiNode* pPhysiNode) { + if (NULL != pLogicNode->pConditions) { + pPhysiNode->pConditions = setNodeSlotId(pCxt, pPhysiNode->outputTuple.tupleId, -1, pLogicNode->pConditions); + CHECK_ALLOC(pPhysiNode->pConditions, TSDB_CODE_OUT_OF_MEMORY); + } + return TSDB_CODE_SUCCESS; +} + +static int32_t setSlotOutput(SPhysiPlanContext* pCxt, SNodeList* pTargets, STupleDescNode* pTuple) { + SHashObj* pHash = taosArrayGetP(pCxt->pTupleHelper, pTuple->tupleId); + char name[TSDB_TABLE_NAME_LEN + TSDB_COL_NAME_LEN]; + SNode* pNode; + FOREACH(pNode, pTargets) { + int32_t len = getSlotKey(pNode, name); + SSlotIndex* pIndex = taosHashGet(pHash, name, len); + ((SSlotDescNode*)nodesListGetNode(pTuple->pSlots, pIndex->slotId))->output = true; + } + + return TSDB_CODE_SUCCESS; +} + +static int32_t initScanPhysiNode(SPhysiPlanContext* pCxt, SScanLogicNode* pScanLogicNode, SScanPhysiNode* pScanPhysiNode) { + if (NULL != pScanLogicNode->pScanCols) { + pScanPhysiNode->pScanCols = nodesCloneList(pScanLogicNode->pScanCols); + CHECK_ALLOC(pScanPhysiNode->pScanCols, TSDB_CODE_OUT_OF_MEMORY); + } + // Tuple describe also needs to be set without scanning column, such as SELECT COUNT(*) FROM t + CHECK_CODE(addTupleDesc(pCxt, pScanPhysiNode->pScanCols, &pScanPhysiNode->node.outputTuple), TSDB_CODE_OUT_OF_MEMORY); + + CHECK_CODE(setConditionsSlotId(pCxt, (const SLogicNode*)pScanLogicNode, (SPhysiNode*)pScanPhysiNode), TSDB_CODE_OUT_OF_MEMORY); + + CHECK_CODE(setSlotOutput(pCxt, pScanLogicNode->node.pTargets, &pScanPhysiNode->node.outputTuple), TSDB_CODE_OUT_OF_MEMORY); + + pScanPhysiNode->uid = pScanLogicNode->pMeta->uid; + pScanPhysiNode->tableType = pScanLogicNode->pMeta->tableType; + pScanPhysiNode->order = TSDB_ORDER_ASC; + pScanPhysiNode->count = 1; + pScanPhysiNode->reverse = 0; + + return TSDB_CODE_SUCCESS; +} + +static SPhysiNode* createTagScanPhysiNode(SPhysiPlanContext* pCxt, SScanLogicNode* pScanLogicNode) { + STagScanPhysiNode* pTagScan = (STagScanPhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN); + CHECK_ALLOC(pTagScan, NULL); + CHECK_CODE(initScanPhysiNode(pCxt, pScanLogicNode, (SScanPhysiNode*)pTagScan), (SPhysiNode*)pTagScan); + return (SPhysiNode*)pTagScan; +} + +static SPhysiNode* createTableScanPhysiNode(SPhysiPlanContext* pCxt, SScanLogicNode* pScanLogicNode) { + STableScanPhysiNode* pTableScan = (STableScanPhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN); + CHECK_ALLOC(pTableScan, NULL); + CHECK_CODE(initScanPhysiNode(pCxt, pScanLogicNode, (SScanPhysiNode*)pTableScan), (SPhysiNode*)pTableScan); + pTableScan->scanFlag = pScanLogicNode->scanFlag; + pTableScan->scanRange = pScanLogicNode->scanRange; + return (SPhysiNode*)pTableScan; +} + +static SPhysiNode* createScanPhysiNode(SPhysiPlanContext* pCxt, SScanLogicNode* pScanLogicNode) { + switch (pScanLogicNode->scanType) { + case SCAN_TYPE_TAG: + return createTagScanPhysiNode(pCxt, pScanLogicNode); + case SCAN_TYPE_TABLE: + return createTableScanPhysiNode(pCxt, pScanLogicNode); + case SCAN_TYPE_STABLE: + case SCAN_TYPE_STREAM: + break; + default: + break; + } + return NULL; +} + +static SNodeList* createJoinOutputCols(SPhysiPlanContext* pCxt, STupleDescNode* pLeftTuple, STupleDescNode* pRightTuple) { + SNodeList* pCols = nodesMakeList(); + CHECK_ALLOC(pCols, NULL); + SNode* pNode; + FOREACH(pNode, pLeftTuple->pSlots) { + SSlotDescNode* pSlot = (SSlotDescNode*)pNode; + SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); + if (NULL == pCol) { + goto error; + } + pCol->node.resType = pSlot->dataType; + pCol->tupleId = pLeftTuple->tupleId; + pCol->slotId = pSlot->slotId; + pCol->colId = -1; + if (TSDB_CODE_SUCCESS != nodesListAppend(pCols, (SNode*)pCol)) { + goto error; + } + } + FOREACH(pNode, pRightTuple->pSlots) { + SSlotDescNode* pSlot = (SSlotDescNode*)pNode; + SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); + if (NULL == pCol) { + goto error; + } + pCol->node.resType = pSlot->dataType; + pCol->tupleId = pRightTuple->tupleId; + pCol->slotId = pSlot->slotId; + pCol->colId = -1; + if (TSDB_CODE_SUCCESS != nodesListAppend(pCols, (SNode*)pCol)) { + goto error; + } + } + return pCols; +error: + nodesDestroyList(pCols); + return NULL; +} + +static SPhysiNode* createJoinPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SJoinLogicNode* pJoinLogicNode) { + SJoinPhysiNode* pJoin = (SJoinPhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_JOIN); + CHECK_ALLOC(pJoin, NULL); + + STupleDescNode* pLeftTuple = &((SPhysiNode*)nodesListGetNode(pChildren, 0))->outputTuple; + STupleDescNode* pRightTuple = &((SPhysiNode*)nodesListGetNode(pChildren, 1))->outputTuple; + pJoin->pOnConditions = setNodeSlotId(pCxt, pLeftTuple->tupleId, pRightTuple->tupleId, pJoinLogicNode->pOnConditions); + CHECK_ALLOC(pJoin->pOnConditions, (SPhysiNode*)pJoin); + + pJoin->pTargets = createJoinOutputCols(pCxt, pLeftTuple, pRightTuple); + CHECK_ALLOC(pJoin->pTargets, (SPhysiNode*)pJoin); + CHECK_CODE(addTupleDesc(pCxt, pJoin->pTargets, &pJoin->node.outputTuple), (SPhysiNode*)pJoin); + + CHECK_CODE(setConditionsSlotId(pCxt, (const SLogicNode*)pJoinLogicNode, (SPhysiNode*)pJoin), (SPhysiNode*)pJoin); + + CHECK_CODE(setSlotOutput(pCxt, pJoinLogicNode->node.pTargets, &pJoin->node.outputTuple), (SPhysiNode*)pJoin); + + return (SPhysiNode*)pJoin; +} + +typedef struct SRewritePrecalcExprsCxt { + int32_t errCode; + int32_t planNodeId; + int32_t rewriteId; + SNodeList* pPrecalcExprs; +} SRewritePrecalcExprsCxt; + +static EDealRes collectAndRewrite(SRewritePrecalcExprsCxt* pCxt, SNode** pNode) { + SNode* pExpr = nodesCloneNode(*pNode); + CHECK_ALLOC(pExpr, DEAL_RES_ERROR); + if (nodesListAppend(pCxt->pPrecalcExprs, pExpr)) { + nodesDestroyNode(pExpr); + return DEAL_RES_ERROR; + } + SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); + if (NULL == pCol) { + nodesDestroyNode(pExpr); + return DEAL_RES_ERROR; + } + SExprNode* pToBeRewrittenExpr = (SExprNode*)(*pNode); + pCol->node.resType = pToBeRewrittenExpr->resType; + strcpy(pCol->colName, pToBeRewrittenExpr->aliasName); + nodesDestroyNode(*pNode); + *pNode = (SNode*)pCol; + return DEAL_RES_IGNORE_CHILD; +} + +static EDealRes doRewritePrecalcExprs(SNode** pNode, void* pContext) { + SRewritePrecalcExprsCxt* pCxt = (SRewritePrecalcExprsCxt*)pContext; + switch (nodeType(*pNode)) { + case QUERY_NODE_OPERATOR: + case QUERY_NODE_LOGIC_CONDITION: { + return collectAndRewrite(pContext, pNode); + } + case QUERY_NODE_FUNCTION: { + if (!fmIsAggFunc(((SFunctionNode*)(*pNode))->funcId)) { + return collectAndRewrite(pContext, pNode); + } + } + default: + break; + } + return DEAL_RES_CONTINUE; +} + +static int32_t rewritePrecalcExprs(SPhysiPlanContext* pCxt, SNodeList* pList, SNodeList** pPrecalcExprs, SNodeList** pRewrittenList) { + if (NULL == pList) { + return TSDB_CODE_SUCCESS; + } + + if (NULL == *pPrecalcExprs) { + *pPrecalcExprs = nodesMakeList(); + CHECK_ALLOC(*pPrecalcExprs, TSDB_CODE_OUT_OF_MEMORY); + } + if (NULL == *pRewrittenList) { + *pRewrittenList = nodesMakeList(); + CHECK_ALLOC(*pRewrittenList, TSDB_CODE_OUT_OF_MEMORY); + } + SNode* pNode = NULL; + FOREACH(pNode, pList) { + SNode* pNew = NULL; + if (QUERY_NODE_GROUPING_SET == nodeType(pNode)) { + pNew = nodesCloneNode(nodesListGetNode(((SGroupingSetNode*)pNode)->pParameterList, 0)); + } else { + pNew = nodesCloneNode(pNode); + } + CHECK_ALLOC(pNew, TSDB_CODE_OUT_OF_MEMORY); + CHECK_CODE(nodesListAppend(*pRewrittenList, pNew), TSDB_CODE_OUT_OF_MEMORY); + } + SRewritePrecalcExprsCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pPrecalcExprs = *pPrecalcExprs }; + nodesRewriteList(*pRewrittenList, doRewritePrecalcExprs, &cxt); + if (0 == LIST_LENGTH(cxt.pPrecalcExprs)) { + nodesDestroyList(cxt.pPrecalcExprs); + *pPrecalcExprs = NULL; + } + return cxt.errCode; +} + +static SPhysiNode* createAggPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SAggLogicNode* pAggLogicNode) { + SAggPhysiNode* pAgg = (SAggPhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_AGG); + CHECK_ALLOC(pAgg, NULL); + + SNodeList* pPrecalcExprs = NULL; + SNodeList* pGroupKeys = NULL; + SNodeList* pAggFuncs = NULL; + CHECK_CODE(rewritePrecalcExprs(pCxt, pAggLogicNode->pGroupKeys, &pPrecalcExprs, &pGroupKeys), (SPhysiNode*)pAgg); + CHECK_CODE(rewritePrecalcExprs(pCxt, pAggLogicNode->pAggFuncs, &pPrecalcExprs, &pAggFuncs), (SPhysiNode*)pAgg); + + STupleDescNode* pChildTupe = &(((SPhysiNode*)nodesListGetNode(pChildren, 0))->outputTuple); + // push down expression to outputTuple of child node + if (NULL != pPrecalcExprs) { + pAgg->pExprs = setListSlotId(pCxt, pChildTupe->tupleId, -1, pPrecalcExprs); + CHECK_ALLOC(pAgg->pExprs, (SPhysiNode*)pAgg); + CHECK_CODE(addTupleDesc(pCxt, pAgg->pExprs, pChildTupe), (SPhysiNode*)pAgg); + } + + if (NULL != pGroupKeys) { + pAgg->pGroupKeys = setListSlotId(pCxt, pChildTupe->tupleId, -1, pGroupKeys); + CHECK_ALLOC(pAgg->pGroupKeys, (SPhysiNode*)pAgg); + CHECK_CODE(addTupleDesc(pCxt, pAgg->pGroupKeys, &pAgg->node.outputTuple), (SPhysiNode*)pAgg); + } + + if (NULL != pAggFuncs) { + pAgg->pAggFuncs = setListSlotId(pCxt, pChildTupe->tupleId, -1, pAggFuncs); + CHECK_ALLOC(pAgg->pAggFuncs, (SPhysiNode*)pAgg); + CHECK_CODE(addTupleDesc(pCxt, pAgg->pAggFuncs, &pAgg->node.outputTuple), (SPhysiNode*)pAgg); + } + + CHECK_CODE(setConditionsSlotId(pCxt, (const SLogicNode*)pAggLogicNode, (SPhysiNode*)pAgg), (SPhysiNode*)pAgg); + + CHECK_CODE(setSlotOutput(pCxt, pAggLogicNode->node.pTargets, &pAgg->node.outputTuple), (SPhysiNode*)pAgg); + + return (SPhysiNode*)pAgg; +} + +static SPhysiNode* createProjectPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SProjectLogicNode* pProjectLogicNode) { + SProjectPhysiNode* pProject = (SProjectPhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_PROJECT); + CHECK_ALLOC(pProject, NULL); + + pProject->pProjections = setListSlotId(pCxt, ((SPhysiNode*)nodesListGetNode(pChildren, 0))->outputTuple.tupleId, -1, pProjectLogicNode->pProjections); + CHECK_ALLOC(pProject->pProjections, (SPhysiNode*)pProject); + CHECK_CODE(addTupleDesc(pCxt, pProject->pProjections, &pProject->node.outputTuple), (SPhysiNode*)pProject); + + CHECK_CODE(setConditionsSlotId(pCxt, (const SLogicNode*)pProjectLogicNode, (SPhysiNode*)pProject), (SPhysiNode*)pProject); + + return (SPhysiNode*)pProject; +} + +static SPhysiNode* createPhysiNode(SPhysiPlanContext* pCxt, SLogicNode* pLogicPlan) { + SNodeList* pChildren = nodesMakeList(); + CHECK_ALLOC(pChildren, NULL); + + SNode* pLogicChild; + FOREACH(pLogicChild, pLogicPlan->pChildren) { + SNode* pChildPhyNode = (SNode*)createPhysiNode(pCxt, (SLogicNode*)pLogicChild); + if (TSDB_CODE_SUCCESS != nodesListAppend(pChildren, pChildPhyNode)) { + pCxt->errCode = TSDB_CODE_OUT_OF_MEMORY; + nodesDestroyList(pChildren); + return NULL; + } + } + + SPhysiNode* pPhyNode = NULL; + switch (nodeType(pLogicPlan)) { + case QUERY_NODE_LOGIC_PLAN_SCAN: + pPhyNode = createScanPhysiNode(pCxt, (SScanLogicNode*)pLogicPlan); + break; + case QUERY_NODE_LOGIC_PLAN_JOIN: + pPhyNode = createJoinPhysiNode(pCxt, pChildren, (SJoinLogicNode*)pLogicPlan); + break; + case QUERY_NODE_LOGIC_PLAN_AGG: + pPhyNode = createAggPhysiNode(pCxt, pChildren, (SAggLogicNode*)pLogicPlan); + break; + case QUERY_NODE_LOGIC_PLAN_PROJECT: + pPhyNode = createProjectPhysiNode(pCxt, pChildren, (SProjectLogicNode*)pLogicPlan); + break; + default: + break; + } + + pPhyNode->pChildren = pChildren; + SNode* pChild; + FOREACH(pChild, pPhyNode->pChildren) { + ((SPhysiNode*)pChild)->pParent = pPhyNode; + } + + return pPhyNode; +} + +int32_t createPhysiPlan(SLogicNode* pLogicNode, SPhysiNode** pPhyNode) { + SPhysiPlanContext cxt = { .errCode = TSDB_CODE_SUCCESS, .nextTupleId = 0, .pTupleHelper = taosArrayInit(32, POINTER_BYTES) }; + if (NULL == cxt.pTupleHelper) { + return TSDB_CODE_OUT_OF_MEMORY; + } + *pPhyNode = createPhysiNode(&cxt, pLogicNode); + return cxt.errCode; +} + +int32_t buildPhysiPlan(SLogicNode* pLogicNode, SPhysiNode** pPhyNode) { + // split + // scale out + // maping + // create + return TSDB_CODE_SUCCESS; +} diff --git a/source/libs/planner/test/newPlannerTest.cpp b/source/libs/planner/test/newPlannerTest.cpp index c3ee4fb9a8..51ef52ac2d 100644 --- a/source/libs/planner/test/newPlannerTest.cpp +++ b/source/libs/planner/test/newPlannerTest.cpp @@ -25,6 +25,11 @@ using namespace testing; class NewPlannerTest : public Test { protected: + enum TestTarget { + TEST_LOGIC_PLAN, + TEST_PHYSICAL_PLAN + }; + void setDatabase(const string& acctId, const string& db) { acctId_ = acctId; db_ = db; @@ -40,7 +45,7 @@ protected: cxt_.pSql = sqlBuf_.c_str(); } - bool run() { + bool run(TestTarget target = TEST_PHYSICAL_PLAN) { int32_t code = parser(&cxt_, &query_); if (code != TSDB_CODE_SUCCESS) { @@ -53,17 +58,27 @@ protected: SLogicNode* pLogicPlan = nullptr; code = createLogicPlan(query_.pRoot, &pLogicPlan); if (code != TSDB_CODE_SUCCESS) { - cout << "sql:[" << cxt_.pSql << "] plan code:" << code << ", strerror:" << tstrerror(code) << endl; + cout << "sql:[" << cxt_.pSql << "] logic plan code:" << code << ", strerror:" << tstrerror(code) << endl; return false; } cout << "sql : [" << cxt_.pSql << "]" << endl; cout << "syntax test : " << endl; cout << syntaxTreeStr << endl; - // cout << "logic plan : " << endl; - // cout << toString((const SNode*)pLogicPlan) << endl; cout << "unformatted logic plan : " << endl; cout << toString((const SNode*)pLogicPlan, false) << endl; + + if (TEST_PHYSICAL_PLAN == target) { + SPhysiNode* pPhyPlan = nullptr; + code = createPhysiPlan(pLogicPlan, &pPhyPlan); + if (code != TSDB_CODE_SUCCESS) { + cout << "sql:[" << cxt_.pSql << "] physical plan code:" << code << ", strerror:" << tstrerror(code) << endl; + return false; + } + cout << "unformatted physical plan : " << endl; + cout << toString((const SNode*)pPhyPlan, false) << endl; + } + return true; } @@ -108,8 +123,8 @@ TEST_F(NewPlannerTest, simple) { TEST_F(NewPlannerTest, groupBy) { setDatabase("root", "test"); - bind("SELECT count(*) FROM t1"); - ASSERT_TRUE(run()); + // bind("SELECT count(*) FROM t1"); + // ASSERT_TRUE(run()); bind("SELECT c1, count(*) FROM t1 GROUP BY c1"); ASSERT_TRUE(run()); @@ -120,3 +135,10 @@ TEST_F(NewPlannerTest, groupBy) { bind("SELECT c1 + c3, count(*) FROM t1 where concat(c2, 'wwww') = 'abcwww' GROUP BY c1 + c3"); ASSERT_TRUE(run()); } + +TEST_F(NewPlannerTest, subquery) { + setDatabase("root", "test"); + + bind("SELECT count(*) FROM (SELECT c1 + c3 a, c1 + count(*) b FROM t1 where c2 = 'abc' GROUP BY c1, c3) where a > 100 group by b"); + ASSERT_TRUE(run()); +} diff --git a/source/libs/qcom/src/queryUtil.c b/source/libs/qcom/src/queryUtil.c index 4cd040c238..a2165453d5 100644 --- a/source/libs/qcom/src/queryUtil.c +++ b/source/libs/qcom/src/queryUtil.c @@ -85,8 +85,9 @@ static void* pTaskQueue = NULL; int32_t initTaskQueue() { double factor = 4.0; - int32_t numOfThreads = TMAX((int)(tsNumOfCores * tsNumOfThreadsPerCore / factor), 2); + int32_t numOfThreads = TMAX((int)(tsNumOfCores * tsNumOfThreadsPerCore / factor), 2); + int32_t queueSize = tsMaxConnections * 2; pTaskQueue = taosInitScheduler(queueSize, numOfThreads, "tsc"); if (NULL == pTaskQueue) { diff --git a/source/libs/qworker/test/qworkerTests.cpp b/source/libs/qworker/test/qworkerTests.cpp index 38890150ee..8ad5a76388 100644 --- a/source/libs/qworker/test/qworkerTests.cpp +++ b/source/libs/qworker/test/qworkerTests.cpp @@ -101,9 +101,7 @@ void qwtInitLogFile() { tsAsyncLog = 0; qDebugFlag = 159; - char temp[128] = {0}; - sprintf(temp, "%s/%s", tsLogDir, defaultLogFileNamePrefix); - if (taosInitLog(temp, tsNumOfLogLines, maxLogFileNum) < 0) { + if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) { printf("failed to open log file in directory:%s\n", tsLogDir); } diff --git a/source/libs/scalar/inc/filterInt.h b/source/libs/scalar/inc/filterInt.h index 1dd533c1c5..a0d4bc680c 100644 --- a/source/libs/scalar/inc/filterInt.h +++ b/source/libs/scalar/inc/filterInt.h @@ -26,6 +26,7 @@ extern "C" { #include "scalar.h" #include "querynodes.h" #include "query.h" +#include "tep.h" #define FILTER_DEFAULT_GROUP_SIZE 4 #define FILTER_DEFAULT_UNIT_SIZE 4 @@ -43,7 +44,6 @@ enum { FLD_TYPE_COLUMN = 1, FLD_TYPE_VALUE = 2, FLD_TYPE_MAX = 3, - FLD_DESC_NO_FREE = 4, FLD_DATA_NO_FREE = 8, FLD_DATA_IS_HASH = 16, }; @@ -61,11 +61,6 @@ enum { RANGE_FLG_NULL = 4, }; -enum { - FI_OPTION_NO_REWRITE = 1, - FI_OPTION_TIMESTAMP = 2, - FI_OPTION_NEED_UNIQE = 4, -}; enum { FI_STATUS_ALL = 1, @@ -107,7 +102,6 @@ typedef struct SFilterRange { typedef bool (*rangeCompFunc) (const void *, const void *, const void *, const void *, __compar_fn_t); typedef int32_t(*filter_desc_compare_func)(const void *, const void *); typedef bool(*filter_exec_func)(void *, int32_t, int8_t**, SColumnDataAgg *, int16_t); -typedef int32_t (*filer_get_col_from_id)(void *, int32_t, void **); typedef int32_t (*filer_get_col_from_name)(void *, int32_t, char*, void **); typedef struct SFilterRangeCompare { @@ -206,7 +200,7 @@ typedef struct SFilterUnit { } SFilterUnit; typedef struct SFilterComUnit { - void *colData; + void *colData; // pointer to SColumnInfoData void *valData; void *valData2; uint16_t colId; @@ -264,12 +258,12 @@ typedef struct SFilterInfo { } SFilterInfo; #define FILTER_NO_MERGE_DATA_TYPE(t) ((t) == TSDB_DATA_TYPE_BINARY || (t) == TSDB_DATA_TYPE_NCHAR || (t) == TSDB_DATA_TYPE_JSON) -#define FILTER_NO_MERGE_OPTR(o) ((o) == TSDB_RELATION_ISNULL || (o) == TSDB_RELATION_NOTNULL || (o) == FILTER_DUMMY_EMPTY_OPTR) +#define FILTER_NO_MERGE_OPTR(o) ((o) == OP_TYPE_IS_NULL || (o) == OP_TYPE_IS_NOT_NULL || (o) == FILTER_DUMMY_EMPTY_OPTR) #define MR_EMPTY_RES(ctx) (ctx->rs == NULL) -#define SET_AND_OPTR(ctx, o) do {if (o == TSDB_RELATION_ISNULL) { (ctx)->isnull = true; } else if (o == TSDB_RELATION_NOTNULL) { if (!(ctx)->isrange) { (ctx)->notnull = true; } } else if (o != FILTER_DUMMY_EMPTY_OPTR) { (ctx)->isrange = true; (ctx)->notnull = false; } } while (0) -#define SET_OR_OPTR(ctx,o) do {if (o == TSDB_RELATION_ISNULL) { (ctx)->isnull = true; } else if (o == TSDB_RELATION_NOTNULL) { (ctx)->notnull = true; (ctx)->isrange = false; } else if (o != FILTER_DUMMY_EMPTY_OPTR) { if (!(ctx)->notnull) { (ctx)->isrange = true; } } } while (0) +#define SET_AND_OPTR(ctx, o) do {if (o == OP_TYPE_IS_NULL) { (ctx)->isnull = true; } else if (o == OP_TYPE_IS_NOT_NULL) { if (!(ctx)->isrange) { (ctx)->notnull = true; } } else if (o != FILTER_DUMMY_EMPTY_OPTR) { (ctx)->isrange = true; (ctx)->notnull = false; } } while (0) +#define SET_OR_OPTR(ctx,o) do {if (o == OP_TYPE_IS_NULL) { (ctx)->isnull = true; } else if (o == OP_TYPE_IS_NOT_NULL) { (ctx)->notnull = true; (ctx)->isrange = false; } else if (o != FILTER_DUMMY_EMPTY_OPTR) { if (!(ctx)->notnull) { (ctx)->isrange = true; } } } while (0) #define CHK_OR_OPTR(ctx) ((ctx)->isnull == true && (ctx)->notnull == true) #define CHK_AND_OPTR(ctx) ((ctx)->isnull == true && (((ctx)->notnull == true) || ((ctx)->isrange == true))) @@ -289,7 +283,7 @@ typedef struct SFilterInfo { #define INSERT_RANGE(ctx, r, ra) do { SFilterRangeNode *n = filterNewRange(ctx, ra); n->prev = (r)->prev; if ((r)->prev) { (r)->prev->next = n; } else { (ctx)->rs = n; } (r)->prev = n; n->next = r; } while (0) #define APPEND_RANGE(ctx, r, ra) do { SFilterRangeNode *n = filterNewRange(ctx, ra); n->prev = (r); if (r) { (r)->next = n; } else { (ctx)->rs = n; } } while (0) -#define FLT_IS_COMPARISON_OPERATOR(_op) ((_op) >= OP_TYPE_GREATER_THAN && (_op) < OP_TYPE_IS_NOT_NULL) +#define FLT_IS_COMPARISON_OPERATOR(_op) ((_op) >= OP_TYPE_GREATER_THAN && (_op) < OP_TYPE_IS_NOT_UNKNOWN) #define fltFatal(...) qFatal(__VA_ARGS__) #define fltError(...) qError(__VA_ARGS__) @@ -307,12 +301,12 @@ typedef struct SFilterInfo { #define FILTER_GET_FIELD(i, id) (&((i)->fields[(id).type].fields[(id).idx])) #define FILTER_GET_COL_FIELD(i, idx) (&((i)->fields[FLD_TYPE_COLUMN].fields[idx])) -#define FILTER_GET_COL_FIELD_TYPE(fi) (((SColumnRefNode *)((fi)->desc))->dataType.type) -#define FILTER_GET_COL_FIELD_SIZE(fi) (((SColumnRefNode *)((fi)->desc))->dataType.bytes) -#define FILTER_GET_COL_FIELD_ID(fi) (((SColumnRefNode *)((fi)->desc))->columnId) -#define FILTER_GET_COL_FIELD_SLOT_ID(fi) (((SColumnRefNode *)((fi)->desc))->slotId) -#define FILTER_GET_COL_FIELD_DESC(fi) ((SColumnRefNode *)((fi)->desc)) -#define FILTER_GET_COL_FIELD_DATA(fi, ri) ((char *)(fi)->data + ((SColumnRefNode *)((fi)->desc))->dataType.bytes * (ri)) +#define FILTER_GET_COL_FIELD_TYPE(fi) (((SColumnNode *)((fi)->desc))->node.resType.type) +#define FILTER_GET_COL_FIELD_SIZE(fi) (((SColumnNode *)((fi)->desc))->node.resType.bytes) +#define FILTER_GET_COL_FIELD_ID(fi) (((SColumnNode *)((fi)->desc))->colId) +#define FILTER_GET_COL_FIELD_SLOT_ID(fi) (((SColumnNode *)((fi)->desc))->slotId) +#define FILTER_GET_COL_FIELD_DESC(fi) ((SColumnNode *)((fi)->desc)) +#define FILTER_GET_COL_FIELD_DATA(fi, ri) (colDataGet(((SColumnInfoData *)(fi)->data), (ri))) #define FILTER_GET_VAL_FIELD_TYPE(fi) (((SValueNode *)((fi)->desc))->node.resType.type) #define FILTER_GET_VAL_FIELD_DATA(fi) ((char *)(fi)->data) #define FILTER_GET_JSON_VAL_FIELD_DATA(fi) ((char *)(fi)->desc) @@ -351,23 +345,10 @@ typedef struct SFilterInfo { #define FILTER_ALL_RES(i) FILTER_GET_FLAG((i)->status, FI_STATUS_ALL) #define FILTER_EMPTY_RES(i) FILTER_GET_FLAG((i)->status, FI_STATUS_EMPTY) -#if 0 -extern int32_t filterInitFromTree(tExprNode* tree, void **pinfo, uint32_t options); -extern bool filterExecute(SFilterInfo *info, int32_t numOfRows, int8_t** p, SColumnDataAgg *statis, int16_t numOfCols); -extern int32_t filterSetColFieldData(SFilterInfo *info, void *param, filer_get_col_from_id fp); -extern int32_t filterSetJsonColFieldData(SFilterInfo *info, void *param, filer_get_col_from_name fp); -extern int32_t filterGetTimeRange(SFilterInfo *info, STimeWindow *win); -extern int32_t filterConverNcharColumns(SFilterInfo* pFilterInfo, int32_t rows, bool *gotNchar); -extern int32_t filterFreeNcharColumns(SFilterInfo* pFilterInfo); -extern void filterFreeInfo(SFilterInfo *info); -extern bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg *pDataStatis, int32_t numOfCols, int32_t numOfRows); -#else -//REMOVE THESE!!!!!!!!!!!!!!!!!!!! -#include "function.h" -#endif extern bool filterDoCompare(__compar_fn_t func, uint8_t optr, void *left, void *right); extern __compar_fn_t filterGetCompFunc(int32_t type, int32_t optr); + #ifdef __cplusplus } #endif diff --git a/source/libs/scalar/inc/sclInt.h b/source/libs/scalar/inc/sclInt.h index 3cfea6890f..41ee90667e 100644 --- a/source/libs/scalar/inc/sclInt.h +++ b/source/libs/scalar/inc/sclInt.h @@ -28,6 +28,8 @@ typedef struct SScalarCtx { SHashObj *pRes; /* element is SScalarParam */ } SScalarCtx; + +#define SCL_DATA_TYPE_DUMMY_HASH 9000 #define SCL_DEFAULT_OP_NUM 10 #define sclFatal(...) qFatal(__VA_ARGS__) diff --git a/source/libs/scalar/inc/sclvector.h b/source/libs/scalar/inc/sclvector.h index 69800a54ea..55c4828745 100644 --- a/source/libs/scalar/inc/sclvector.h +++ b/source/libs/scalar/inc/sclvector.h @@ -24,7 +24,6 @@ extern "C" { typedef void (*_bin_scalar_fn_t)(SScalarParam* pLeft, SScalarParam* pRight, void *output, int32_t order); _bin_scalar_fn_t getBinScalarOperatorFn(int32_t binOperator); -bool isBinaryStringOp(int32_t op); #ifdef __cplusplus } diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 8f8fc25d18..5eb0003662 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -21,22 +21,42 @@ #include "filter.h" OptrStr gOptrStr[] = { - {TSDB_RELATION_INVALID, "invalid"}, - {TSDB_RELATION_LESS, "<"}, - {TSDB_RELATION_GREATER, ">"}, - {TSDB_RELATION_EQUAL, "="}, - {TSDB_RELATION_LESS_EQUAL, "<="}, - {TSDB_RELATION_GREATER_EQUAL, ">="}, - {TSDB_RELATION_NOT_EQUAL, "!="}, - {TSDB_RELATION_LIKE, "like"}, - {TSDB_RELATION_ISNULL, "is null"}, - {TSDB_RELATION_NOTNULL, "not null"}, - {TSDB_RELATION_IN, "in"}, - {TSDB_RELATION_AND, "and"}, - {TSDB_RELATION_OR, "or"}, - {TSDB_RELATION_NOT, "not"}, - {TSDB_RELATION_MATCH, "match"}, - {TSDB_RELATION_NMATCH, "nmatch"} + {0, "invalid"}, + {OP_TYPE_ADD, "+"}, + {OP_TYPE_SUB, "-"}, + {OP_TYPE_MULTI, "*"}, + {OP_TYPE_DIV, "/"}, + {OP_TYPE_MOD, "%"}, + + // bit operator + {OP_TYPE_BIT_AND, "&"}, + {OP_TYPE_BIT_OR, "|"}, + + // comparison operator + {OP_TYPE_GREATER_THAN, ">"}, + {OP_TYPE_GREATER_EQUAL, ">="}, + {OP_TYPE_LOWER_THAN, "<"}, + {OP_TYPE_LOWER_EQUAL, "<="}, + {OP_TYPE_EQUAL, "=="}, + {OP_TYPE_NOT_EQUAL, "!="}, + {OP_TYPE_IN, "in"}, + {OP_TYPE_NOT_IN, "not in"}, + {OP_TYPE_LIKE, "like"}, + {OP_TYPE_NOT_LIKE, "not like"}, + {OP_TYPE_MATCH, "match"}, + {OP_TYPE_NMATCH, "nmatch"}, + {OP_TYPE_IS_NULL, "is null"}, + {OP_TYPE_IS_NOT_NULL, "not null"}, + {OP_TYPE_IS_TRUE, "is true"}, + {OP_TYPE_IS_FALSE, "is false"}, + {OP_TYPE_IS_UNKNOWN, "is unknown"}, + {OP_TYPE_IS_NOT_TRUE, "not true"}, + {OP_TYPE_IS_NOT_FALSE, "not false"}, + {OP_TYPE_IS_NOT_UNKNOWN, "not unknown"}, + + // json operator + {OP_TYPE_JSON_GET_VALUE, "json get"}, + {OP_TYPE_JSON_CONTAINS, "json contains"} }; bool filterRangeCompGi (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) { @@ -110,30 +130,30 @@ rangeCompFunc gRangeCompare[] = {filterRangeCompee, filterRangeCompei, filterRan int8_t filterGetRangeCompFuncFromOptrs(uint8_t optr, uint8_t optr2) { if (optr2) { - assert(optr2 == TSDB_RELATION_LESS || optr2 == TSDB_RELATION_LESS_EQUAL); + assert(optr2 == OP_TYPE_LOWER_THAN || optr2 == OP_TYPE_LOWER_EQUAL); - if (optr == TSDB_RELATION_GREATER) { - if (optr2 == TSDB_RELATION_LESS) { + if (optr == OP_TYPE_GREATER_THAN) { + if (optr2 == OP_TYPE_LOWER_THAN) { return 0; } return 1; } - if (optr2 == TSDB_RELATION_LESS) { + if (optr2 == OP_TYPE_LOWER_THAN) { return 2; } return 3; } else { switch (optr) { - case TSDB_RELATION_GREATER: + case OP_TYPE_GREATER_THAN: return 4; - case TSDB_RELATION_GREATER_EQUAL: + case OP_TYPE_GREATER_EQUAL: return 5; - case TSDB_RELATION_LESS: + case OP_TYPE_LOWER_THAN: return 6; - case TSDB_RELATION_LESS_EQUAL: + case OP_TYPE_LOWER_EQUAL: return 7; default: break; @@ -154,7 +174,7 @@ __compar_fn_t gDataCompare[] = {compareInt32Val, compareInt8Val, compareInt16Val int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) { int8_t comparFn = 0; - if (optr == TSDB_RELATION_IN && (type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR)) { + if (optr == OP_TYPE_IN && (type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR)) { switch (type) { case TSDB_DATA_TYPE_BOOL: case TSDB_DATA_TYPE_TINYINT: @@ -177,7 +197,7 @@ int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) { } } - if (optr == TSDB_RELATION_NOT_IN && (type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR)) { + if (optr == OP_TYPE_NOT_IN && (type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR)) { switch (type) { case TSDB_DATA_TYPE_BOOL: case TSDB_DATA_TYPE_TINYINT: @@ -210,17 +230,17 @@ int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) { case TSDB_DATA_TYPE_FLOAT: comparFn = 4; break; case TSDB_DATA_TYPE_DOUBLE: comparFn = 5; break; case TSDB_DATA_TYPE_BINARY: { - if (optr == TSDB_RELATION_MATCH) { + if (optr == OP_TYPE_MATCH) { comparFn = 19; - } else if (optr == TSDB_RELATION_NMATCH) { + } else if (optr == OP_TYPE_NMATCH) { comparFn = 20; - } else if (optr == TSDB_RELATION_LIKE) { /* wildcard query using like operator */ + } else if (optr == OP_TYPE_LIKE) { /* wildcard query using like operator */ comparFn = 7; - } else if (optr == TSDB_RELATION_NOT_LIKE) { /* wildcard query using like operator */ + } else if (optr == OP_TYPE_NOT_LIKE) { /* wildcard query using like operator */ comparFn = 26; - } else if (optr == TSDB_RELATION_IN) { + } else if (optr == OP_TYPE_IN) { comparFn = 8; - } else if (optr == TSDB_RELATION_NOT_IN) { + } else if (optr == OP_TYPE_NOT_IN) { comparFn = 25; } else { /* normal relational comparFn */ comparFn = 6; @@ -230,17 +250,17 @@ int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) { } case TSDB_DATA_TYPE_NCHAR: { - if (optr == TSDB_RELATION_MATCH) { + if (optr == OP_TYPE_MATCH) { comparFn = 19; - } else if (optr == TSDB_RELATION_NMATCH) { + } else if (optr == OP_TYPE_NMATCH) { comparFn = 20; - } else if (optr == TSDB_RELATION_LIKE) { + } else if (optr == OP_TYPE_LIKE) { comparFn = 9; - } else if (optr == TSDB_RELATION_LIKE) { + } else if (optr == OP_TYPE_LIKE) { comparFn = 27; - } else if (optr == TSDB_RELATION_IN) { + } else if (optr == OP_TYPE_IN) { comparFn = 8; - } else if (optr == TSDB_RELATION_NOT_IN) { + } else if (optr == OP_TYPE_NOT_IN) { comparFn = 25; } else { comparFn = 10; @@ -383,7 +403,7 @@ int32_t filterConvertRange(SFilterRangeCtx *cur, SFilterRange *ra, bool *notNull int32_t filterAddRangeOptr(void* h, uint8_t raOptr, int32_t optr, bool *empty, bool *all) { SFilterRangeCtx *ctx = (SFilterRangeCtx *)h; - if (optr == TSDB_RELATION_AND) { + if (optr == LOGIC_COND_TYPE_AND) { SET_AND_OPTR(ctx, raOptr); if (CHK_AND_OPTR(ctx) || (raOptr == FILTER_DUMMY_EMPTY_OPTR)) { FILTER_SET_FLAG(ctx->status, MR_ST_EMPTY); @@ -407,8 +427,8 @@ int32_t filterAddRangeImpl(void* h, SFilterRange* ra, int32_t optr) { if (ctx->rs == NULL) { if ((FILTER_GET_FLAG(ctx->status, MR_ST_START) == 0) - || (FILTER_GET_FLAG(ctx->status, MR_ST_ALL) && (optr == TSDB_RELATION_AND)) - || ((!FILTER_GET_FLAG(ctx->status, MR_ST_ALL)) && (optr == TSDB_RELATION_OR))) { + || (FILTER_GET_FLAG(ctx->status, MR_ST_ALL) && (optr == LOGIC_COND_TYPE_AND)) + || ((!FILTER_GET_FLAG(ctx->status, MR_ST_ALL)) && (optr == LOGIC_COND_TYPE_OR))) { APPEND_RANGE(ctx, ctx->rs, ra); FILTER_SET_FLAG(ctx->status, MR_ST_START); } @@ -420,7 +440,7 @@ int32_t filterAddRangeImpl(void* h, SFilterRange* ra, int32_t optr) { SFilterRangeNode *rn = NULL; int32_t cr = 0; - if (optr == TSDB_RELATION_AND) { + if (optr == LOGIC_COND_TYPE_AND) { while (r != NULL) { cr = ctx->pCompareFunc(&r->ra.s, &ra->e); if (FILTER_GREATER(cr, r->ra.sflag, ra->eflag)) { @@ -532,7 +552,7 @@ int32_t filterAddRangeImpl(void* h, SFilterRange* ra, int32_t optr) { if (notnull) { bool all = false; FREE_FROM_RANGE(ctx, ctx->rs); - filterAddRangeOptr(h, TSDB_RELATION_NOTNULL, optr, NULL, &all); + filterAddRangeOptr(h, OP_TYPE_IS_NOT_NULL, optr, NULL, &all); if (all) { FILTER_SET_FLAG(ctx->status, MR_ST_ALL); } @@ -563,7 +583,7 @@ int32_t filterAddRangeCtx(void *dst, void *src, int32_t optr) { SFilterRangeCtx *dctx = (SFilterRangeCtx *)dst; SFilterRangeCtx *sctx = (SFilterRangeCtx *)src; - assert(optr == TSDB_RELATION_OR); + assert(optr == LOGIC_COND_TYPE_OR); if (sctx->rs == NULL) { return TSDB_CODE_SUCCESS; @@ -614,13 +634,13 @@ int32_t filterFinishRange(void* h) { return TSDB_CODE_SUCCESS; } - if (FILTER_GET_FLAG(ctx->options, FI_OPTION_TIMESTAMP)) { + if (FILTER_GET_FLAG(ctx->options, FLT_OPTION_TIMESTAMP)) { SFilterRangeNode *r = ctx->rs; SFilterRangeNode *rn = NULL; while (r && r->next) { int64_t tmp = 1; - operateVal(&tmp, &r->ra.e, &tmp, TSDB_BINARY_OP_ADD, ctx->type); + operateVal(&tmp, &r->ra.e, &tmp, OP_TYPE_ADD, ctx->type); if (ctx->pCompareFunc(&tmp, &r->next->ra.s) == 0) { rn = r->next; SIMPLE_COPY_VALUES((char *)&r->next->ra.s, (char *)&r->ra.s); @@ -685,14 +705,14 @@ int32_t filterSourceRangeFromCtx(SFilterRangeCtx *ctx, void *sctx, int32_t optr, SFilterRangeCtx *src = (SFilterRangeCtx *)sctx; if (src->isnull){ - filterAddRangeOptr(ctx, TSDB_RELATION_ISNULL, optr, empty, all); + filterAddRangeOptr(ctx, OP_TYPE_IS_NULL, optr, empty, all); if (FILTER_GET_FLAG(ctx->status, MR_ST_ALL)) { *all = true; } } if (src->notnull) { - filterAddRangeOptr(ctx, TSDB_RELATION_NOTNULL, optr, empty, all); + filterAddRangeOptr(ctx, OP_TYPE_IS_NOT_NULL, optr, empty, all); if (FILTER_GET_FLAG(ctx->status, MR_ST_ALL)) { *all = true; } @@ -701,7 +721,7 @@ int32_t filterSourceRangeFromCtx(SFilterRangeCtx *ctx, void *sctx, int32_t optr, if (src->isrange) { filterAddRangeOptr(ctx, 0, optr, empty, all); - if (!(optr == TSDB_RELATION_OR && ctx->notnull)) { + if (!(optr == LOGIC_COND_TYPE_OR && ctx->notnull)) { filterAddRangeCtx(ctx, src, optr); } @@ -764,13 +784,26 @@ int32_t filterDetachCnfGroups(SArray* group, SArray* left, SArray* right) { int32_t rightSize = (int32_t)taosArrayGetSize(right); if (taosArrayGetSize(left) <= 0) { - fltDebug("empty group"); - FLT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + if (taosArrayGetSize(right) <= 0) { + fltError("both groups are empty"); + FLT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + } + + SFilterGroup *gp = NULL; + while (gp = (SFilterGroup *)taosArrayPop(right)) { + taosArrayPush(group, gp); + } + + return TSDB_CODE_SUCCESS; } if (taosArrayGetSize(right) <= 0) { - fltDebug("empty group"); - FLT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + SFilterGroup *gp = NULL; + while (gp = (SFilterGroup *)taosArrayPop(left)) { + taosArrayPush(group, gp); + } + + return TSDB_CODE_SUCCESS; } for (int32_t l = 0; l < leftSize; ++l) { @@ -825,7 +858,7 @@ int32_t filterAddField(SFilterInfo *info, void *desc, void **data, int32_t type, if (*num > 0) { if (type == FLD_TYPE_COLUMN) { idx = filterGetFiledByDesc(&info->fields[type], type, desc); - } else if (data && (*data) && dataLen > 0 && FILTER_GET_FLAG(info->options, FI_OPTION_NEED_UNIQE)) { + } else if (data && (*data) && dataLen > 0 && FILTER_GET_FLAG(info->options, FLT_OPTION_NEED_UNIQE)) { idx = filterGetFiledByData(info, type, *data, dataLen); } } @@ -847,7 +880,7 @@ int32_t filterAddField(SFilterInfo *info, void *desc, void **data, int32_t type, ++(*num); - if (data && (*data) && dataLen > 0 && FILTER_GET_FLAG(info->options, FI_OPTION_NEED_UNIQE)) { + if (data && (*data) && dataLen > 0 && FILTER_GET_FLAG(info->options, FLT_OPTION_NEED_UNIQE)) { if (info->pctx.valHash == NULL) { info->pctx.valHash = taosHashInit(FILTER_DEFAULT_GROUP_SIZE * FILTER_DEFAULT_VALUE_SIZE, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, false); } @@ -855,10 +888,6 @@ int32_t filterAddField(SFilterInfo *info, void *desc, void **data, int32_t type, taosHashPut(info->pctx.valHash, *data, dataLen, &idx, sizeof(idx)); } } else { - if (freeIfExists) { - tfree(desc); - } - if (data && freeIfExists) { tfree(*data); } @@ -873,7 +902,6 @@ int32_t filterAddField(SFilterInfo *info, void *desc, void **data, int32_t type, static FORCE_INLINE int32_t filterAddColFieldFromField(SFilterInfo *info, SFilterField *field, SFilterFieldId *fid) { filterAddField(info, field->desc, &field->data, FILTER_GET_TYPE(field->flag), fid, 0, false); - FILTER_SET_FLAG(field->flag, FLD_DESC_NO_FREE); FILTER_SET_FLAG(field->flag, FLD_DATA_NO_FREE); return TSDB_CODE_SUCCESS; @@ -886,14 +914,14 @@ int32_t filterAddFieldFromNode(SFilterInfo *info, SNode *node, SFilterFieldId *f FLT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); } - if (nodeType(node) != QUERY_NODE_COLUMN_REF && nodeType(node) != QUERY_NODE_VALUE) { + if (nodeType(node) != QUERY_NODE_COLUMN && nodeType(node) != QUERY_NODE_VALUE && nodeType(node) != QUERY_NODE_NODE_LIST) { FLT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); } int32_t type; void *v; - if (nodeType(node) == QUERY_NODE_COLUMN_REF) { + if (nodeType(node) == QUERY_NODE_COLUMN) { type = FLD_TYPE_COLUMN; v = node; } else { @@ -907,7 +935,7 @@ int32_t filterAddFieldFromNode(SFilterInfo *info, SNode *node, SFilterFieldId *f } int32_t filterAddUnit(SFilterInfo *info, uint8_t optr, SFilterFieldId *left, SFilterFieldId *right, uint32_t *uidx) { - if (FILTER_GET_FLAG(info->options, FI_OPTION_NEED_UNIQE)) { + if (FILTER_GET_FLAG(info->options, FLT_OPTION_NEED_UNIQE)) { if (info->pctx.unitHash == NULL) { info->pctx.unitHash = taosHashInit(FILTER_DEFAULT_GROUP_SIZE * FILTER_DEFAULT_UNIT_SIZE, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, false); } else { @@ -940,8 +968,10 @@ int32_t filterAddUnit(SFilterInfo *info, uint8_t optr, SFilterFieldId *left, SFi SFilterField *val = FILTER_UNIT_RIGHT_FIELD(info, u); assert(FILTER_GET_FLAG(val->flag, FLD_TYPE_VALUE)); } else { - if(optr != TSDB_RELATION_ISNULL && optr != TSDB_RELATION_NOTNULL && optr != FILTER_DUMMY_EMPTY_OPTR){ - return -1; + int32_t paramNum = scalarGetOperatorParamNum(optr); + if (1 != paramNum) { + fltError("invalid right field in unit, operator:%s, rightType:%d", gOptrStr[optr].str, u->right.type); + return TSDB_CODE_QRY_APP_ERROR; } } @@ -952,7 +982,7 @@ int32_t filterAddUnit(SFilterInfo *info, uint8_t optr, SFilterFieldId *left, SFi *uidx = info->unitNum; - if (FILTER_GET_FLAG(info->options, FI_OPTION_NEED_UNIQE)) { + if (FILTER_GET_FLAG(info->options, FLT_OPTION_NEED_UNIQE)) { int64_t v = 0; FILTER_PACKAGE_UNIT_HASH_KEY(&v, optr, left->idx, right ? right->idx : -1); taosHashPut(info->pctx.unitHash, &v, sizeof(v), uidx, sizeof(*uidx)); @@ -988,7 +1018,6 @@ int32_t fltAddGroupUnitFromNode(SFilterInfo *info, SNode* tree, SArray *group) { if (node->opType == OP_TYPE_IN && (!IS_VAR_DATA_TYPE(type))) { SNodeListNode *listNode = (SNodeListNode *)node->pRight; - void *fdata = NULL; SListCell *cell = listNode->pNodeList->pHead; SScalarParam in = {.num = 1}, out = {.num = 1, .type = type}; @@ -1008,14 +1037,16 @@ int32_t fltAddGroupUnitFromNode(SFilterInfo *info, SNode* tree, SArray *group) { len = tDataTypes[type].bytes; - filterAddField(info, NULL, &fdata, FLD_TYPE_VALUE, &right, len, true); + filterAddField(info, NULL, &out.data, FLD_TYPE_VALUE, &right, len, true); - filterAddUnit(info, TSDB_RELATION_EQUAL, &left, &right, &uidx); + filterAddUnit(info, OP_TYPE_EQUAL, &left, &right, &uidx); SFilterGroup fgroup = {0}; filterAddUnitToGroup(&fgroup, uidx); taosArrayPush(group, &fgroup); + + cell = cell->pNext; } } else { filterAddFieldFromNode(info, node->pRight, &right); @@ -1035,16 +1066,15 @@ int32_t fltAddGroupUnitFromNode(SFilterInfo *info, SNode* tree, SArray *group) { int32_t filterAddUnitFromUnit(SFilterInfo *dst, SFilterInfo *src, SFilterUnit* u, uint32_t *uidx) { SFilterFieldId left, right, *pright = &right; int32_t type = FILTER_UNIT_DATA_TYPE(u); - uint16_t flag = FLD_DESC_NO_FREE; + uint16_t flag = 0; filterAddField(dst, FILTER_UNIT_COL_DESC(src, u), NULL, FLD_TYPE_COLUMN, &left, 0, false); SFilterField *t = FILTER_UNIT_LEFT_FIELD(src, u); - FILTER_SET_FLAG(t->flag, flag); if (u->right.type == FLD_TYPE_VALUE) { void *data = FILTER_UNIT_VAL_DATA(src, u); if (IS_VAR_DATA_TYPE(type)) { - if (FILTER_UNIT_OPTR(u) == TSDB_RELATION_IN) { + if (FILTER_UNIT_OPTR(u) == OP_TYPE_IN) { filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, POINTER_BYTES, false); // POINTER_BYTES should be sizeof(SHashObj), but POINTER_BYTES is also right. t = FILTER_GET_FIELD(dst, right); @@ -1087,17 +1117,17 @@ int32_t filterAddGroupUnitFromCtx(SFilterInfo *dst, SFilterInfo *src, SFilterRan int32_t type = FILTER_GET_COL_FIELD_TYPE(FILTER_GET_FIELD(dst, left)); - if (optr == TSDB_RELATION_AND) { + if (optr == LOGIC_COND_TYPE_AND) { if (ctx->isnull) { assert(ctx->notnull == false && ctx->isrange == false); - filterAddUnit(dst, TSDB_RELATION_ISNULL, &left, NULL, &uidx); + filterAddUnit(dst, OP_TYPE_IS_NULL, &left, NULL, &uidx); filterAddUnitToGroup(g, uidx); return TSDB_CODE_SUCCESS; } if (ctx->notnull) { assert(ctx->isnull == false && ctx->isrange == false); - filterAddUnit(dst, TSDB_RELATION_NOTNULL, &left, NULL, &uidx); + filterAddUnit(dst, OP_TYPE_IS_NOT_NULL, &left, NULL, &uidx); filterAddUnitToGroup(g, uidx); return TSDB_CODE_SUCCESS; } @@ -1119,7 +1149,7 @@ int32_t filterAddGroupUnitFromCtx(SFilterInfo *dst, SFilterInfo *src, SFilterRan void *data = malloc(sizeof(int64_t)); SIMPLE_COPY_VALUES(data, &ra->s); filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true); - filterAddUnit(dst, TSDB_RELATION_EQUAL, &left, &right, &uidx); + filterAddUnit(dst, OP_TYPE_EQUAL, &left, &right, &uidx); filterAddUnitToGroup(g, uidx); return TSDB_CODE_SUCCESS; } else { @@ -1130,8 +1160,8 @@ int32_t filterAddGroupUnitFromCtx(SFilterInfo *dst, SFilterInfo *src, SFilterRan SIMPLE_COPY_VALUES(data2, &ra->e); filterAddField(dst, NULL, &data2, FLD_TYPE_VALUE, &right2, tDataTypes[type].bytes, true); - filterAddUnit(dst, FILTER_GET_FLAG(ra->sflag, RANGE_FLG_EXCLUDE) ? TSDB_RELATION_GREATER : TSDB_RELATION_GREATER_EQUAL, &left, &right, &uidx); - filterAddUnitRight(dst, FILTER_GET_FLAG(ra->eflag, RANGE_FLG_EXCLUDE) ? TSDB_RELATION_LESS : TSDB_RELATION_LESS_EQUAL, &right2, uidx); + filterAddUnit(dst, FILTER_GET_FLAG(ra->sflag, RANGE_FLG_EXCLUDE) ? OP_TYPE_GREATER_THAN : OP_TYPE_GREATER_EQUAL, &left, &right, &uidx); + filterAddUnitRight(dst, FILTER_GET_FLAG(ra->eflag, RANGE_FLG_EXCLUDE) ? OP_TYPE_LOWER_THAN : OP_TYPE_LOWER_EQUAL, &right2, uidx); filterAddUnitToGroup(g, uidx); return TSDB_CODE_SUCCESS; } @@ -1141,7 +1171,7 @@ int32_t filterAddGroupUnitFromCtx(SFilterInfo *dst, SFilterInfo *src, SFilterRan void *data = malloc(sizeof(int64_t)); SIMPLE_COPY_VALUES(data, &ra->s); filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true); - filterAddUnit(dst, FILTER_GET_FLAG(ra->sflag, RANGE_FLG_EXCLUDE) ? TSDB_RELATION_GREATER : TSDB_RELATION_GREATER_EQUAL, &left, &right, &uidx); + filterAddUnit(dst, FILTER_GET_FLAG(ra->sflag, RANGE_FLG_EXCLUDE) ? OP_TYPE_GREATER_THAN : OP_TYPE_GREATER_EQUAL, &left, &right, &uidx); filterAddUnitToGroup(g, uidx); } @@ -1149,7 +1179,7 @@ int32_t filterAddGroupUnitFromCtx(SFilterInfo *dst, SFilterInfo *src, SFilterRan void *data = malloc(sizeof(int64_t)); SIMPLE_COPY_VALUES(data, &ra->e); filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true); - filterAddUnit(dst, FILTER_GET_FLAG(ra->eflag, RANGE_FLG_EXCLUDE) ? TSDB_RELATION_LESS : TSDB_RELATION_LESS_EQUAL, &left, &right, &uidx); + filterAddUnit(dst, FILTER_GET_FLAG(ra->eflag, RANGE_FLG_EXCLUDE) ? OP_TYPE_LOWER_THAN : OP_TYPE_LOWER_EQUAL, &left, &right, &uidx); filterAddUnitToGroup(g, uidx); } @@ -1164,7 +1194,7 @@ int32_t filterAddGroupUnitFromCtx(SFilterInfo *dst, SFilterInfo *src, SFilterRan assert(ctx->isnull || ctx->notnull || ctx->isrange); if (ctx->isnull) { - filterAddUnit(dst, TSDB_RELATION_ISNULL, &left, NULL, &uidx); + filterAddUnit(dst, OP_TYPE_IS_NULL, &left, NULL, &uidx); filterAddUnitToGroup(g, uidx); taosArrayPush(res, g); } @@ -1173,7 +1203,7 @@ int32_t filterAddGroupUnitFromCtx(SFilterInfo *dst, SFilterInfo *src, SFilterRan assert(!ctx->isrange); memset(g, 0, sizeof(*g)); - filterAddUnit(dst, TSDB_RELATION_NOTNULL, &left, NULL, &uidx); + filterAddUnit(dst, OP_TYPE_IS_NOT_NULL, &left, NULL, &uidx); filterAddUnitToGroup(g, uidx); taosArrayPush(res, g); } @@ -1195,7 +1225,7 @@ int32_t filterAddGroupUnitFromCtx(SFilterInfo *dst, SFilterInfo *src, SFilterRan void *data = malloc(sizeof(int64_t)); SIMPLE_COPY_VALUES(data, &r->ra.s); filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true); - filterAddUnit(dst, TSDB_RELATION_EQUAL, &left, &right, &uidx); + filterAddUnit(dst, OP_TYPE_EQUAL, &left, &right, &uidx); filterAddUnitToGroup(g, uidx); } else { void *data = malloc(sizeof(int64_t)); @@ -1205,8 +1235,8 @@ int32_t filterAddGroupUnitFromCtx(SFilterInfo *dst, SFilterInfo *src, SFilterRan SIMPLE_COPY_VALUES(data2, &r->ra.e); filterAddField(dst, NULL, &data2, FLD_TYPE_VALUE, &right2, tDataTypes[type].bytes, true); - filterAddUnit(dst, FILTER_GET_FLAG(r->ra.sflag, RANGE_FLG_EXCLUDE) ? TSDB_RELATION_GREATER : TSDB_RELATION_GREATER_EQUAL, &left, &right, &uidx); - filterAddUnitRight(dst, FILTER_GET_FLAG(r->ra.eflag, RANGE_FLG_EXCLUDE) ? TSDB_RELATION_LESS : TSDB_RELATION_LESS_EQUAL, &right2, uidx); + filterAddUnit(dst, FILTER_GET_FLAG(r->ra.sflag, RANGE_FLG_EXCLUDE) ? OP_TYPE_GREATER_THAN : OP_TYPE_GREATER_EQUAL, &left, &right, &uidx); + filterAddUnitRight(dst, FILTER_GET_FLAG(r->ra.eflag, RANGE_FLG_EXCLUDE) ? OP_TYPE_LOWER_THAN : OP_TYPE_LOWER_EQUAL, &right2, uidx); filterAddUnitToGroup(g, uidx); } @@ -1221,7 +1251,7 @@ int32_t filterAddGroupUnitFromCtx(SFilterInfo *dst, SFilterInfo *src, SFilterRan void *data = malloc(sizeof(int64_t)); SIMPLE_COPY_VALUES(data, &r->ra.s); filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true); - filterAddUnit(dst, FILTER_GET_FLAG(r->ra.sflag, RANGE_FLG_EXCLUDE) ? TSDB_RELATION_GREATER : TSDB_RELATION_GREATER_EQUAL, &left, &right, &uidx); + filterAddUnit(dst, FILTER_GET_FLAG(r->ra.sflag, RANGE_FLG_EXCLUDE) ? OP_TYPE_GREATER_THAN : OP_TYPE_GREATER_EQUAL, &left, &right, &uidx); filterAddUnitToGroup(g, uidx); } @@ -1229,7 +1259,7 @@ int32_t filterAddGroupUnitFromCtx(SFilterInfo *dst, SFilterInfo *src, SFilterRan void *data = malloc(sizeof(int64_t)); SIMPLE_COPY_VALUES(data, &r->ra.e); filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true); - filterAddUnit(dst, FILTER_GET_FLAG(r->ra.eflag, RANGE_FLG_EXCLUDE) ? TSDB_RELATION_LESS : TSDB_RELATION_LESS_EQUAL, &left, &right, &uidx); + filterAddUnit(dst, FILTER_GET_FLAG(r->ra.eflag, RANGE_FLG_EXCLUDE) ? OP_TYPE_LOWER_THAN : OP_TYPE_LOWER_EQUAL, &left, &right, &uidx); filterAddUnitToGroup(g, uidx); } @@ -1262,7 +1292,7 @@ EDealRes fltTreeToGroup(SNode* pNode, void* pContext) { SArray* preGroup = NULL; SArray* newGroup = NULL; SArray* resGroup = NULL; - ENodeType nType = nodeType(nType); + ENodeType nType = nodeType(pNode); SFltBuildGroupCtx *ctx = (SFltBuildGroupCtx *)pContext; if (QUERY_NODE_LOGIC_CONDITION == nodeType(pNode)) { @@ -1280,16 +1310,18 @@ EDealRes fltTreeToGroup(SNode* pNode, void* pContext) { FLT_ERR_JRET(filterDetachCnfGroups(resGroup, preGroup, newGroup)); taosArrayDestroyEx(newGroup, filterFreeGroup); + newGroup = NULL; taosArrayDestroyEx(preGroup, filterFreeGroup); preGroup = resGroup; + resGroup = NULL; + + cell = cell->pNext; } - taosArrayAddAll(ctx->group, resGroup); + taosArrayAddAll(ctx->group, preGroup); - taosArrayDestroyEx(newGroup, filterFreeGroup); - taosArrayDestroyEx(preGroup, filterFreeGroup); - taosArrayDestroyEx(resGroup, filterFreeGroup); + taosArrayDestroy(preGroup); return DEAL_RES_IGNORE_CHILD; } @@ -1299,11 +1331,15 @@ EDealRes fltTreeToGroup(SNode* pNode, void* pContext) { for (int32_t i = 0; i < node->pParameterList->length; ++i) { nodesWalkNode(cell->pNode, fltTreeToGroup, (void *)pContext); FLT_ERR_JRET(ctx->code); + + cell = cell->pNext; } return DEAL_RES_IGNORE_CHILD; } + ctx->code = TSDB_CODE_QRY_APP_ERROR; + fltError("invalid condition type, type:%d", node->condType); return DEAL_RES_ERROR; @@ -1312,7 +1348,7 @@ EDealRes fltTreeToGroup(SNode* pNode, void* pContext) { if (QUERY_NODE_OPERATOR == nType) { FLT_ERR_JRET(fltAddGroupUnitFromNode(ctx->info, pNode, ctx->group)); - return DEAL_RES_CONTINUE; + return DEAL_RES_IGNORE_CHILD; } fltError("invalid node type for filter, type:%d", nodeType(pNode)); @@ -1418,7 +1454,7 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options) qDebug("COLUMN Field Num:%u", info->fields[FLD_TYPE_COLUMN].num); for (uint32_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) { SFilterField *field = &info->fields[FLD_TYPE_COLUMN].fields[i]; - SColumnRefNode *refNode = (SColumnRefNode *)field->desc; + SColumnNode *refNode = (SColumnNode *)field->desc; qDebug("COL%d => [%d][%d]", i, refNode->tupleId, refNode->slotId); } @@ -1447,12 +1483,12 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options) char str[512] = {0}; SFilterField *left = FILTER_UNIT_LEFT_FIELD(info, unit); - SColumnRefNode *refNode = (SColumnRefNode *)left->desc; - if (unit->compare.optr >= TSDB_RELATION_INVALID && unit->compare.optr <= TSDB_RELATION_NMATCH){ + SColumnNode *refNode = (SColumnNode *)left->desc; + if (unit->compare.optr >= 0 && unit->compare.optr <= OP_TYPE_JSON_CONTAINS){ len = sprintf(str, "UNIT[%d] => [%d][%d] %s [", i, refNode->tupleId, refNode->slotId, gOptrStr[unit->compare.optr].str); } - if (unit->right.type == FLD_TYPE_VALUE && FILTER_UNIT_OPTR(unit) != TSDB_RELATION_IN) { + if (unit->right.type == FLD_TYPE_VALUE && FILTER_UNIT_OPTR(unit) != OP_TYPE_IN) { SFilterField *right = FILTER_UNIT_RIGHT_FIELD(info, unit); char *data = right->data; if (IS_VAR_DATA_TYPE(type)) { @@ -1467,11 +1503,11 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options) if (unit->compare.optr2) { strcat(str, " && "); - if (unit->compare.optr2 >= TSDB_RELATION_INVALID && unit->compare.optr2 <= TSDB_RELATION_NMATCH){ + if (unit->compare.optr2 >= 0 && unit->compare.optr2 <= OP_TYPE_JSON_CONTAINS){ sprintf(str + strlen(str), "[%d][%d] %s [", refNode->tupleId, refNode->slotId, gOptrStr[unit->compare.optr2].str); } - if (unit->right2.type == FLD_TYPE_VALUE && FILTER_UNIT_OPTR(unit) != TSDB_RELATION_IN) { + if (unit->right2.type == FLD_TYPE_VALUE && FILTER_UNIT_OPTR(unit) != OP_TYPE_IN) { SFilterField *right = FILTER_UNIT_RIGHT2_FIELD(info, unit); char *data = right->data; if (IS_VAR_DATA_TYPE(type)) { @@ -1619,14 +1655,6 @@ void filterFreeField(SFilterField* field, int32_t type) { return; } - if (!FILTER_GET_FLAG(field->flag, FLD_DESC_NO_FREE)) { - if (type == FLD_TYPE_VALUE) { - taosVariantDestroy(field->desc); - } - - tfree(field->desc); - } - if (!FILTER_GET_FLAG(field->flag, FLD_DATA_NO_FREE)) { if (FILTER_GET_FLAG(field->flag, FLD_DATA_IS_HASH)) { taosHashCleanup(field->data); @@ -1688,15 +1716,15 @@ int32_t filterHandleValueExtInfo(SFilterUnit* unit, char extInfo) { uint8_t optr = FILTER_UNIT_OPTR(unit); switch (optr) { - case TSDB_RELATION_GREATER: - case TSDB_RELATION_GREATER_EQUAL: - unit->compare.optr = (extInfo > 0) ? FILTER_DUMMY_EMPTY_OPTR : TSDB_RELATION_NOTNULL; + case OP_TYPE_GREATER_THAN: + case OP_TYPE_GREATER_EQUAL: + unit->compare.optr = (extInfo > 0) ? FILTER_DUMMY_EMPTY_OPTR : OP_TYPE_IS_NOT_NULL; break; - case TSDB_RELATION_LESS: - case TSDB_RELATION_LESS_EQUAL: - unit->compare.optr = (extInfo > 0) ? TSDB_RELATION_NOTNULL : FILTER_DUMMY_EMPTY_OPTR; + case OP_TYPE_LOWER_THAN: + case OP_TYPE_LOWER_EQUAL: + unit->compare.optr = (extInfo > 0) ? OP_TYPE_IS_NOT_NULL : FILTER_DUMMY_EMPTY_OPTR; break; - case TSDB_RELATION_EQUAL: + case OP_TYPE_EQUAL: unit->compare.optr = FILTER_DUMMY_EMPTY_OPTR; break; default: @@ -1706,75 +1734,12 @@ int32_t filterHandleValueExtInfo(SFilterUnit* unit, char extInfo) { return TSDB_CODE_SUCCESS; } -int32_t fltGenerateSetFromList(void **data, void *pNode, uint32_t type) { - SHashObj *pObj = taosHashInit(256, taosGetDefaultHashFunction(type), true, false); - if (NULL == pObj) { - fltError("taosHashInit failed, size:%d", 256); - FLT_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); - } - - taosHashSetEqualFp(pObj, taosGetDefaultEqualFunction(type)); - - int32_t code = 0; - SNodeListNode *nodeList = (SNodeListNode *)pNode; - SListCell *cell = nodeList->pNodeList->pHead; - SScalarParam in = {.num = 1}, out = {.num = 1, .type = type}; - int8_t dummy = 0; - int32_t bufLen = 60; - out.data = malloc(bufLen); - int32_t len = 0; - void *buf = NULL; - - for (int32_t i = 0; i < nodeList->pNodeList->length; ++i) { - SValueNode *valueNode = (SValueNode *)cell->pNode; - - if (valueNode->node.resType.type != type) { - in.type = valueNode->node.resType.type; - in.bytes = valueNode->node.resType.bytes; - in.data = nodesGetValueFromNode(valueNode); - - code = vectorConvertImpl(&in, &out); - if (code) { - fltError("convert from %d to %d failed", in.type, out.type); - FLT_ERR_JRET(code); - } - - if (IS_VAR_DATA_TYPE(type)) { - len = varDataLen(out.data); - } else { - len = tDataTypes[type].bytes; - } - - buf = out.data; - } else { - buf = nodesGetValueFromNode(valueNode); - len = valueNode->node.resType.bytes; - } - - if (taosHashPut(pObj, buf, (size_t)len, &dummy, sizeof(dummy))) { - fltError("taosHashPut failed"); - FLT_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); - } - } - - tfree(out.data); - *data = pObj; - - return TSDB_CODE_SUCCESS; - -_return: - - tfree(out.data); - taosHashCleanup(pObj); - - FLT_RET(code); -} int32_t fltInitValFieldData(SFilterInfo *info) { for (uint32_t i = 0; i < info->unitNum; ++i) { SFilterUnit* unit = &info->units[i]; if (unit->right.type != FLD_TYPE_VALUE) { - assert(unit->compare.optr == TSDB_RELATION_ISNULL || unit->compare.optr == TSDB_RELATION_NOTNULL || unit->compare.optr == FILTER_DUMMY_EMPTY_OPTR); + assert(unit->compare.optr == FILTER_DUMMY_EMPTY_OPTR || scalarGetOperatorParamNum(unit->compare.optr) == 1); continue; } @@ -1792,8 +1757,8 @@ int32_t fltInitValFieldData(SFilterInfo *info) { continue; } - if (unit->compare.optr == TSDB_RELATION_IN) { - FLT_ERR_RET(fltGenerateSetFromList((void **)&fi->data, fi->desc, type)); + if (unit->compare.optr == OP_TYPE_IN) { + FLT_ERR_RET(scalarGenerateSetFromList((void **)&fi->data, fi->desc, type)); if (fi->data == NULL) { fltError("failed to convert in param"); FLT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); @@ -1830,24 +1795,26 @@ int32_t fltInitValFieldData(SFilterInfo *info) { } if(type != TSDB_DATA_TYPE_JSON){ - bool converted = false; - char extInfo = 0; - SScalarParam in = {.data = nodesGetValueFromNode(var), .num = 1, .type = dType->type, .bytes = dType->bytes}; - SScalarParam out = {.data = fi->data, .num = 1, .type = type}; - if (vectorConvertImpl(&in, &out)) { - qError("convert value to type[%d] failed", type); - return TSDB_CODE_TSC_INVALID_OPERATION; + if (dType->type == type) { + assignVal(fi->data, nodesGetValueFromNode(var), dType->bytes, type); + } else { + SScalarParam in = {.data = nodesGetValueFromNode(var), .num = 1, .type = dType->type, .bytes = dType->bytes}; + SScalarParam out = {.data = fi->data, .num = 1, .type = type}; + if (vectorConvertImpl(&in, &out)) { + qError("convert value to type[%d] failed", type); + return TSDB_CODE_TSC_INVALID_OPERATION; + } } } // match/nmatch for nchar type need convert from ucs4 to mbs if(type == TSDB_DATA_TYPE_NCHAR && - (unit->compare.optr == TSDB_RELATION_MATCH || unit->compare.optr == TSDB_RELATION_NMATCH)){ + (unit->compare.optr == OP_TYPE_MATCH || unit->compare.optr == OP_TYPE_NMATCH)){ char newValData[TSDB_REGEX_STRING_DEFAULT_LEN * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE] = {0}; int32_t len = taosUcs4ToMbs(varDataVal(fi->data), varDataLen(fi->data), varDataVal(newValData)); if (len < 0){ qError("filterInitValFieldData taosUcs4ToMbs error 1"); - return TSDB_CODE_FAILED; + return TSDB_CODE_QRY_APP_ERROR; } varDataSetLen(newValData, len); varDataCopy(fi->data, newValData); @@ -1862,34 +1829,40 @@ bool filterDoCompare(__compar_fn_t func, uint8_t optr, void *left, void *right) int32_t ret = func(left, right); switch (optr) { - case TSDB_RELATION_EQUAL: { + case OP_TYPE_EQUAL: { return ret == 0; } - case TSDB_RELATION_NOT_EQUAL: { + case OP_TYPE_NOT_EQUAL: { return ret != 0; } - case TSDB_RELATION_GREATER_EQUAL: { + case OP_TYPE_GREATER_EQUAL: { return ret >= 0; } - case TSDB_RELATION_GREATER: { + case OP_TYPE_GREATER_THAN: { return ret > 0; } - case TSDB_RELATION_LESS_EQUAL: { + case OP_TYPE_LOWER_EQUAL: { return ret <= 0; } - case TSDB_RELATION_LESS: { + case OP_TYPE_LOWER_THAN: { return ret < 0; } - case TSDB_RELATION_LIKE: { + case OP_TYPE_LIKE: { return ret == 0; } - case TSDB_RELATION_MATCH: { + case OP_TYPE_NOT_LIKE: { return ret == 0; } - case TSDB_RELATION_NMATCH: { + case OP_TYPE_MATCH: { return ret == 0; } - case TSDB_RELATION_IN: { + case OP_TYPE_NMATCH: { + return ret == 0; + } + case OP_TYPE_IN: { + return ret == 1; + } + case OP_TYPE_NOT_IN: { return ret == 1; } @@ -1909,25 +1882,25 @@ int32_t filterAddUnitRange(SFilterInfo *info, SFilterUnit* u, SFilterRangeCtx *c int64_t tmp = 0; switch (uoptr) { - case TSDB_RELATION_GREATER: + case OP_TYPE_GREATER_THAN: SIMPLE_COPY_VALUES(&ra.s, val); FILTER_SET_FLAG(ra.sflag, RANGE_FLG_EXCLUDE); FILTER_SET_FLAG(ra.eflag, RANGE_FLG_NULL); break; - case TSDB_RELATION_GREATER_EQUAL: + case OP_TYPE_GREATER_EQUAL: SIMPLE_COPY_VALUES(&ra.s, val); FILTER_SET_FLAG(ra.eflag, RANGE_FLG_NULL); break; - case TSDB_RELATION_LESS: + case OP_TYPE_LOWER_THAN: SIMPLE_COPY_VALUES(&ra.e, val); FILTER_SET_FLAG(ra.eflag, RANGE_FLG_EXCLUDE); FILTER_SET_FLAG(ra.sflag, RANGE_FLG_NULL); break; - case TSDB_RELATION_LESS_EQUAL: + case OP_TYPE_LOWER_EQUAL: SIMPLE_COPY_VALUES(&ra.e, val); FILTER_SET_FLAG(ra.sflag, RANGE_FLG_NULL); break; - case TSDB_RELATION_NOT_EQUAL: + case OP_TYPE_NOT_EQUAL: assert(type == TSDB_DATA_TYPE_BOOL); if (GET_INT8_VAL(val)) { SIMPLE_COPY_VALUES(&ra.s, &tmp); @@ -1938,7 +1911,7 @@ int32_t filterAddUnitRange(SFilterInfo *info, SFilterUnit* u, SFilterRangeCtx *c SIMPLE_COPY_VALUES(&ra.e, &tmp); } break; - case TSDB_RELATION_EQUAL: + case OP_TYPE_EQUAL: SIMPLE_COPY_VALUES(&ra.s, val); SIMPLE_COPY_VALUES(&ra.e, val); break; @@ -1992,15 +1965,15 @@ int32_t filterMergeUnits(SFilterInfo *info, SFilterGroupCtx* gRes, uint32_t colI SFilterUnit* u = taosArrayGetP(colArray, i); uint8_t optr = FILTER_UNIT_OPTR(u); - filterAddRangeOptr(ctx, optr, TSDB_RELATION_AND, empty, NULL); + filterAddRangeOptr(ctx, optr, LOGIC_COND_TYPE_AND, empty, NULL); FLT_CHK_JMP(*empty); if (!FILTER_NO_MERGE_OPTR(optr)) { - filterAddUnitRange(info, u, ctx, TSDB_RELATION_AND); + filterAddUnitRange(info, u, ctx, LOGIC_COND_TYPE_AND); FLT_CHK_JMP(MR_EMPTY_RES(ctx)); } - if(FILTER_UNIT_OPTR(u) == TSDB_RELATION_EQUAL && !FILTER_NO_MERGE_DATA_TYPE(FILTER_UNIT_DATA_TYPE(u))){ - gRes->colInfo[colIdx].optr = TSDB_RELATION_EQUAL; + if(FILTER_UNIT_OPTR(u) == OP_TYPE_EQUAL && !FILTER_NO_MERGE_DATA_TYPE(FILTER_UNIT_DATA_TYPE(u))){ + gRes->colInfo[colIdx].optr = OP_TYPE_EQUAL; SIMPLE_COPY_VALUES(&gRes->colInfo[colIdx].value, FILTER_UNIT_VAL_DATA(info, u)); } } @@ -2120,7 +2093,7 @@ void filterCheckColConflict(SFilterGroupCtx* gRes1, SFilterGroupCtx* gRes2, bool } // for long in operation - if (gRes1->colInfo[idx1].optr == TSDB_RELATION_EQUAL && gRes2->colInfo[idx2].optr == TSDB_RELATION_EQUAL) { + if (gRes1->colInfo[idx1].optr == OP_TYPE_EQUAL && gRes2->colInfo[idx2].optr == OP_TYPE_EQUAL) { SFilterRangeCtx* ctx = gRes1->colInfo[idx1].info; if (ctx->pCompareFunc(&gRes1->colInfo[idx1].value, &gRes2->colInfo[idx2].value)){ *conflict = true; @@ -2196,7 +2169,7 @@ int32_t filterMergeTwoGroups(SFilterInfo *info, SFilterGroupCtx** gRes1, SFilter ++merNum; - filterMergeTwoGroupsImpl(info, &ctx, TSDB_RELATION_OR, idx1, *gRes1, *gRes2, NULL, all); + filterMergeTwoGroupsImpl(info, &ctx, LOGIC_COND_TYPE_OR, idx1, *gRes1, *gRes2, NULL, all); FLT_CHK_JMP(*all); @@ -2410,14 +2383,14 @@ int32_t filterRewrite(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t gResNum oinfo.colRangeNum = 0; oinfo.colRange = NULL; - FILTER_SET_FLAG(info->options, FI_OPTION_NEED_UNIQE); + FILTER_SET_FLAG(info->options, FLT_OPTION_NEED_UNIQE); filterInitUnitsFields(info); for (int32_t i = 0; i < gResNum; ++i) { res = gRes[i]; - optr = (res->colNum > 1) ? TSDB_RELATION_AND : TSDB_RELATION_OR; + optr = (res->colNum > 1) ? LOGIC_COND_TYPE_AND : LOGIC_COND_TYPE_OR; SFilterGroup ng = {0}; @@ -2514,7 +2487,7 @@ int32_t filterGenerateColRange(SFilterInfo *info, SFilterGroupCtx** gRes, int32_ assert(colInfo->type == RANGE_TYPE_MR_CTX); bool all = false; - filterSourceRangeFromCtx(info->colRange[m], colInfo->info, TSDB_RELATION_OR, NULL, &all); + filterSourceRangeFromCtx(info->colRange[m], colInfo->info, LOGIC_COND_TYPE_OR, NULL, &all); if (all) { filterFreeRangeCtx(info->colRange[m]); info->colRange[m] = NULL; @@ -2597,7 +2570,8 @@ int32_t filterUpdateComUnits(SFilterInfo *info) { for (uint32_t i = 0; i < info->unitNum; ++i) { SFilterUnit *unit = &info->units[i]; - info->cunits[i].colData = FILTER_UNIT_COL_DATA(info, unit, 0); + SFilterField *col = FILTER_UNIT_LEFT_FIELD(info, unit); + info->cunits[i].colData = col->data; } return TSDB_CODE_SUCCESS; @@ -2629,20 +2603,20 @@ int32_t filterRmUnitByRange(SFilterInfo *info, SColumnDataAgg *pDataStatis, int3 } if (pDataStatis[index].numOfNull <= 0) { - if (cunit->optr == TSDB_RELATION_ISNULL) { + if (cunit->optr == OP_TYPE_IS_NULL) { info->blkUnitRes[k] = -1; rmUnit = 1; continue; } - if (cunit->optr == TSDB_RELATION_NOTNULL) { + if (cunit->optr == OP_TYPE_IS_NOT_NULL) { info->blkUnitRes[k] = 1; rmUnit = 1; continue; } } else { if (pDataStatis[index].numOfNull == numOfRows) { - if (cunit->optr == TSDB_RELATION_ISNULL) { + if (cunit->optr == OP_TYPE_IS_NULL) { info->blkUnitRes[k] = 1; rmUnit = 1; continue; @@ -2654,9 +2628,9 @@ int32_t filterRmUnitByRange(SFilterInfo *info, SColumnDataAgg *pDataStatis, int3 } } - if (cunit->optr == TSDB_RELATION_ISNULL || cunit->optr == TSDB_RELATION_NOTNULL - || cunit->optr == TSDB_RELATION_IN || cunit->optr == TSDB_RELATION_LIKE || cunit->optr == TSDB_RELATION_MATCH - || cunit->optr == TSDB_RELATION_NOT_EQUAL) { + if (cunit->optr == OP_TYPE_IS_NULL || cunit->optr == OP_TYPE_IS_NOT_NULL + || cunit->optr == OP_TYPE_IN || cunit->optr == OP_TYPE_LIKE || cunit->optr == OP_TYPE_MATCH + || cunit->optr == OP_TYPE_NOT_EQUAL) { continue; } @@ -2686,8 +2660,8 @@ int32_t filterRmUnitByRange(SFilterInfo *info, SColumnDataAgg *pDataStatis, int3 info->blkUnitRes[k] = 1; rmUnit = 1; } else if ((!minRes) && (!maxRes)) { - minRes = filterDoCompare(gDataCompare[cunit->func], TSDB_RELATION_LESS_EQUAL, minVal, cunit->valData); - maxRes = filterDoCompare(gDataCompare[cunit->func], TSDB_RELATION_GREATER_EQUAL, maxVal, cunit->valData2); + minRes = filterDoCompare(gDataCompare[cunit->func], OP_TYPE_LOWER_EQUAL, minVal, cunit->valData); + maxRes = filterDoCompare(gDataCompare[cunit->func], OP_TYPE_GREATER_EQUAL, maxVal, cunit->valData2); if (minRes && maxRes) { continue; @@ -2704,9 +2678,9 @@ int32_t filterRmUnitByRange(SFilterInfo *info, SColumnDataAgg *pDataStatis, int3 info->blkUnitRes[k] = 1; rmUnit = 1; } else if ((!minRes) && (!maxRes)) { - if (cunit->optr == TSDB_RELATION_EQUAL) { - minRes = filterDoCompare(gDataCompare[cunit->func], TSDB_RELATION_GREATER, minVal, cunit->valData); - maxRes = filterDoCompare(gDataCompare[cunit->func], TSDB_RELATION_LESS, maxVal, cunit->valData); + if (cunit->optr == OP_TYPE_EQUAL) { + minRes = filterDoCompare(gDataCompare[cunit->func], OP_TYPE_GREATER_THAN, minVal, cunit->valData); + maxRes = filterDoCompare(gDataCompare[cunit->func], OP_TYPE_LOWER_THAN, maxVal, cunit->valData); if (minRes || maxRes) { info->blkUnitRes[k] = -1; rmUnit = 1; @@ -2802,7 +2776,7 @@ bool filterExecuteBasedOnStatisImpl(void *pinfo, int32_t numOfRows, int8_t** p, uint32_t unitNum = *(unitIdx++); for (uint32_t u = 0; u < unitNum; ++u) { SFilterComUnit *cunit = &info->cunits[*(unitIdx + u)]; - void *colData = (char *)cunit->colData + cunit->dataSize * i; + void *colData = colDataGet((SColumnInfoData *)cunit->colData, i); //if (FILTER_UNIT_GET_F(info, uidx)) { // p[i] = FILTER_UNIT_GET_R(info, uidx); @@ -2810,11 +2784,11 @@ bool filterExecuteBasedOnStatisImpl(void *pinfo, int32_t numOfRows, int8_t** p, uint8_t optr = cunit->optr; if (isNull(colData, cunit->dataType)) { - (*p)[i] = optr == TSDB_RELATION_ISNULL ? true : false; + (*p)[i] = optr == OP_TYPE_IS_NULL ? true : false; } else { - if (optr == TSDB_RELATION_NOTNULL) { + if (optr == OP_TYPE_IS_NOT_NULL) { (*p)[i] = 1; - } else if (optr == TSDB_RELATION_ISNULL) { + } else if (optr == OP_TYPE_IS_NULL) { (*p)[i] = 0; } else if (cunit->rfunc >= 0) { (*p)[i] = (*gRangeCompare[cunit->rfunc])(colData, colData, cunit->valData, cunit->valData2, gDataCompare[cunit->func]); @@ -2900,7 +2874,7 @@ static FORCE_INLINE bool filterExecuteImplIsNull(void *pinfo, int32_t numOfRows, for (int32_t i = 0; i < numOfRows; ++i) { uint32_t uidx = info->groups[0].unitIdxs[0]; - void *colData = (char *)info->cunits[uidx].colData + info->cunits[uidx].dataSize * i; + void *colData = colDataGet((SColumnInfoData *)info->cunits[uidx].colData, i); if(info->cunits[uidx].dataType == TSDB_DATA_TYPE_JSON){ if (!colData){ // for json->'key' is null (*p)[i] = 1; @@ -2934,7 +2908,7 @@ static FORCE_INLINE bool filterExecuteImplNotNull(void *pinfo, int32_t numOfRows for (int32_t i = 0; i < numOfRows; ++i) { uint32_t uidx = info->groups[0].unitIdxs[0]; - void *colData = (char *)info->cunits[uidx].colData + info->cunits[uidx].dataSize * i; + void *colData = colDataGet((SColumnInfoData *)info->cunits[uidx].colData, i); if(info->cunits[uidx].dataType == TSDB_DATA_TYPE_JSON){ if (!colData) { // for json->'key' is not null @@ -2961,7 +2935,6 @@ bool filterExecuteImplRange(void *pinfo, int32_t numOfRows, int8_t** p, SColumnD SFilterInfo *info = (SFilterInfo *)pinfo; bool all = true; uint16_t dataSize = info->cunits[0].dataSize; - char *colData = (char *)info->cunits[0].colData; rangeCompFunc rfunc = gRangeCompare[info->cunits[0].rfunc]; void *valData = info->cunits[0].valData; void *valData2 = info->cunits[0].valData2; @@ -2975,10 +2948,11 @@ bool filterExecuteImplRange(void *pinfo, int32_t numOfRows, int8_t** p, SColumnD *p = calloc(numOfRows, sizeof(int8_t)); } - for (int32_t i = 0; i < numOfRows; ++i) { + for (int32_t i = 0; i < numOfRows; ++i) { + void *colData = colDataGet((SColumnInfoData *)info->cunits[0].colData, i); + if (colData == NULL || isNull(colData, info->cunits[0].dataType)) { all = false; - colData += dataSize; continue; } @@ -2987,8 +2961,6 @@ bool filterExecuteImplRange(void *pinfo, int32_t numOfRows, int8_t** p, SColumnD if ((*p)[i] == 0) { all = false; } - - colData += dataSize; } return all; @@ -3008,7 +2980,7 @@ bool filterExecuteImplMisc(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDa for (int32_t i = 0; i < numOfRows; ++i) { uint32_t uidx = info->groups[0].unitIdxs[0]; - void *colData = (char *)info->cunits[uidx].colData + info->cunits[uidx].dataSize * i; + void *colData = colDataGet((SColumnInfoData *)info->cunits[uidx].colData, i); if (colData == NULL || isNull(colData, info->cunits[uidx].dataType)) { (*p)[i] = 0; all = false; @@ -3016,7 +2988,7 @@ bool filterExecuteImplMisc(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDa } // match/nmatch for nchar type need convert from ucs4 to mbs - if(info->cunits[uidx].dataType == TSDB_DATA_TYPE_NCHAR && (info->cunits[uidx].optr == TSDB_RELATION_MATCH || info->cunits[uidx].optr == TSDB_RELATION_NMATCH)){ + if(info->cunits[uidx].dataType == TSDB_DATA_TYPE_NCHAR && (info->cunits[uidx].optr == OP_TYPE_MATCH || info->cunits[uidx].optr == OP_TYPE_NMATCH)){ char *newColData = calloc(info->cunits[uidx].dataSize * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE, 1); int32_t len = taosUcs4ToMbs(varDataVal(colData), varDataLen(colData), varDataVal(newColData)); if (len < 0){ @@ -3059,7 +3031,7 @@ bool filterExecuteImpl(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDataAg for (uint32_t u = 0; u < group->unitNum; ++u) { uint32_t uidx = group->unitIdxs[u]; SFilterComUnit *cunit = &info->cunits[uidx]; - void *colData = (char *)cunit->colData + cunit->dataSize * i; + void *colData = colDataGet((SColumnInfoData *)(cunit->colData), i); //if (FILTER_UNIT_GET_F(info, uidx)) { // p[i] = FILTER_UNIT_GET_R(info, uidx); @@ -3067,16 +3039,16 @@ bool filterExecuteImpl(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDataAg uint8_t optr = cunit->optr; if (colData == NULL || isNull(colData, cunit->dataType)) { - (*p)[i] = optr == TSDB_RELATION_ISNULL ? true : false; + (*p)[i] = optr == OP_TYPE_IS_NULL ? true : false; } else { - if (optr == TSDB_RELATION_NOTNULL) { + if (optr == OP_TYPE_IS_NOT_NULL) { (*p)[i] = 1; - } else if (optr == TSDB_RELATION_ISNULL) { + } else if (optr == OP_TYPE_IS_NULL) { (*p)[i] = 0; } else if (cunit->rfunc >= 0) { (*p)[i] = (*gRangeCompare[cunit->rfunc])(colData, colData, cunit->valData, cunit->valData2, gDataCompare[cunit->func]); } else { - if(cunit->dataType == TSDB_DATA_TYPE_NCHAR && (cunit->optr == TSDB_RELATION_MATCH || cunit->optr == TSDB_RELATION_NMATCH)){ + if(cunit->dataType == TSDB_DATA_TYPE_NCHAR && (cunit->optr == OP_TYPE_MATCH || cunit->optr == OP_TYPE_NMATCH)){ char *newColData = calloc(cunit->dataSize * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE, 1); int32_t len = taosUcs4ToMbs(varDataVal(colData), varDataLen(colData), varDataVal(newColData)); if (len < 0){ @@ -3129,12 +3101,12 @@ int32_t filterSetExecFunc(SFilterInfo *info) { return TSDB_CODE_SUCCESS; } - if (info->units[0].compare.optr == TSDB_RELATION_ISNULL) { + if (info->units[0].compare.optr == OP_TYPE_IS_NULL) { info->func = filterExecuteImplIsNull; return TSDB_CODE_SUCCESS; } - if (info->units[0].compare.optr == TSDB_RELATION_NOTNULL) { + if (info->units[0].compare.optr == OP_TYPE_IS_NOT_NULL) { info->func = filterExecuteImplNotNull; return TSDB_CODE_SUCCESS; } @@ -3230,7 +3202,7 @@ int32_t fltInitFromNode(SNode* tree, SFilterInfo *info, uint32_t options) { FLT_ERR_JRET(fltInitValFieldData(info)); - if (!FILTER_GET_FLAG(info->options, FI_OPTION_NO_REWRITE)) { + if (!FILTER_GET_FLAG(info->options, FLT_OPTION_NO_REWRITE)) { filterDumpInfoToString(info, "Before preprocess", 0); FLT_ERR_JRET(filterPreprocess(info)); @@ -3251,7 +3223,7 @@ int32_t fltInitFromNode(SNode* tree, SFilterInfo *info, uint32_t options) { _return: - qInfo("No filter, code:%d", code); + qInfo("init from node failed, code:%d", code); return code; } @@ -3348,8 +3320,8 @@ bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg *pDataStatis, int32_t int32_t filterGetTimeRange(SFilterInfo *info, STimeWindow *win) { SFilterRange ra = {0}; - SFilterRangeCtx *prev = filterInitRangeCtx(TSDB_DATA_TYPE_TIMESTAMP, FI_OPTION_TIMESTAMP); - SFilterRangeCtx *tmpc = filterInitRangeCtx(TSDB_DATA_TYPE_TIMESTAMP, FI_OPTION_TIMESTAMP); + SFilterRangeCtx *prev = filterInitRangeCtx(TSDB_DATA_TYPE_TIMESTAMP, FLT_OPTION_TIMESTAMP); + SFilterRangeCtx *tmpc = filterInitRangeCtx(TSDB_DATA_TYPE_TIMESTAMP, FLT_OPTION_TIMESTAMP); SFilterRangeCtx *cur = NULL; int32_t num = 0; int32_t optr = 0; @@ -3360,10 +3332,10 @@ int32_t filterGetTimeRange(SFilterInfo *info, STimeWindow *win) { SFilterGroup *group = &info->groups[i]; if (group->unitNum > 1) { cur = tmpc; - optr = TSDB_RELATION_AND; + optr = LOGIC_COND_TYPE_AND; } else { cur = prev; - optr = TSDB_RELATION_OR; + optr = LOGIC_COND_TYPE_OR; } for (uint32_t u = 0; u < group->unitNum; ++u) { @@ -3372,21 +3344,14 @@ int32_t filterGetTimeRange(SFilterInfo *info, STimeWindow *win) { uint8_t raOptr = FILTER_UNIT_OPTR(unit); - filterAddRangeOptr(cur, raOptr, TSDB_RELATION_AND, &empty, NULL); + filterAddRangeOptr(cur, raOptr, LOGIC_COND_TYPE_AND, &empty, NULL); FLT_CHK_JMP(empty); if (FILTER_NO_MERGE_OPTR(raOptr)) { continue; } - - SFilterField *right = FILTER_UNIT_RIGHT_FIELD(info, unit); - void *s = FILTER_GET_VAL_FIELD_DATA(right); - void *e = FILTER_GET_VAL_FIELD_DATA(right) + tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes; - SIMPLE_COPY_VALUES(&ra.s, s); - SIMPLE_COPY_VALUES(&ra.e, e); - - filterAddRange(cur, &ra, optr); + filterAddUnitRange(info, unit, cur, optr); } if (cur->notnull) { @@ -3395,7 +3360,7 @@ int32_t filterGetTimeRange(SFilterInfo *info, STimeWindow *win) { } if (group->unitNum > 1) { - filterSourceRangeFromCtx(prev, cur, TSDB_RELATION_OR, &empty, &all); + filterSourceRangeFromCtx(prev, cur, LOGIC_COND_TYPE_OR, &empty, &all); filterResetRangeCtx(cur); if (all) { break; @@ -3522,11 +3487,7 @@ EDealRes fltReviseRewriter(SNode** pNode, void* pContext) { return DEAL_RES_CONTINUE; } - if (stat->scalarMode) { - return DEAL_RES_CONTINUE; - } - - if (QUERY_NODE_VALUE == nodeType(*pNode)) { + if (QUERY_NODE_VALUE == nodeType(*pNode) || QUERY_NODE_NODE_LIST == nodeType(*pNode) || QUERY_NODE_COLUMN == nodeType(*pNode)) { return DEAL_RES_CONTINUE; } @@ -3544,22 +3505,28 @@ EDealRes fltReviseRewriter(SNode** pNode, void* pContext) { if (NULL == node->pRight) { if (scalarGetOperatorParamNum(node->opType) > 1) { - fltError("invalid operator, pRight:%p, type:%d", node->pRight, nodeType(node)); + fltError("invalid operator, pRight:%p, nodeType:%d, opType:%d", node->pRight, nodeType(node), node->opType); stat->code = TSDB_CODE_QRY_APP_ERROR; return DEAL_RES_ERROR; } - if (QUERY_NODE_COLUMN_REF != nodeType(node->pLeft)) { - stat->scalarMode = true; - return DEAL_RES_CONTINUE; - } - } else { - if ((QUERY_NODE_COLUMN_REF != nodeType(node->pLeft)) && (QUERY_NODE_VALUE != nodeType(node->pLeft))) { + if (QUERY_NODE_COLUMN != nodeType(node->pLeft)) { stat->scalarMode = true; return DEAL_RES_CONTINUE; } - if ((QUERY_NODE_COLUMN_REF != nodeType(node->pRight)) && (QUERY_NODE_VALUE != nodeType(node->pRight))) { + if (OP_TYPE_IS_TRUE == node->opType || OP_TYPE_IS_FALSE == node->opType || OP_TYPE_IS_UNKNOWN == node->opType + || OP_TYPE_IS_NOT_TRUE == node->opType || OP_TYPE_IS_NOT_FALSE == node->opType || OP_TYPE_IS_NOT_UNKNOWN == node->opType) { + stat->scalarMode = true; + return DEAL_RES_CONTINUE; + } + } else { + if ((QUERY_NODE_COLUMN != nodeType(node->pLeft)) && (QUERY_NODE_VALUE != nodeType(node->pLeft))) { + stat->scalarMode = true; + return DEAL_RES_CONTINUE; + } + + if ((QUERY_NODE_COLUMN != nodeType(node->pRight)) && (QUERY_NODE_VALUE != nodeType(node->pRight))) { stat->scalarMode = true; return DEAL_RES_CONTINUE; } @@ -3569,7 +3536,7 @@ EDealRes fltReviseRewriter(SNode** pNode, void* pContext) { return DEAL_RES_CONTINUE; } - if (QUERY_NODE_COLUMN_REF != nodeType(node->pLeft)) { + if (QUERY_NODE_COLUMN != nodeType(node->pLeft)) { SNode *t = node->pLeft; node->pLeft = node->pRight; node->pRight = t; @@ -3582,10 +3549,10 @@ EDealRes fltReviseRewriter(SNode** pNode, void* pContext) { } if (OP_TYPE_IN != node->opType) { - SColumnRefNode *refNode = (SColumnRefNode *)node->pLeft; + SColumnNode *refNode = (SColumnNode *)node->pLeft; SValueNode *valueNode = (SValueNode *)node->pRight; - int32_t type = vectorGetConvertType(refNode->dataType.type, valueNode->node.resType.type); - if (0 != type && type != refNode->dataType.type) { + int32_t type = vectorGetConvertType(refNode->node.resType.type, valueNode->node.resType.type); + if (0 != type && type != refNode->node.resType.type) { stat->scalarMode = true; return DEAL_RES_CONTINUE; } @@ -3609,18 +3576,19 @@ int32_t fltReviseNodes(SFilterInfo *pInfo, SNode** pNode, SFltTreeStat *pStat) { } int32_t fltOptimizeNodes(SFilterInfo *pInfo, SNode** pNode, SFltTreeStat *pStat) { - + //TODO + return TSDB_CODE_SUCCESS; } -int32_t filterGetDataFromColId(void *param, int32_t id, void **data) { +int32_t fltGetDataFromColId(void *param, int32_t id, void **data) { int32_t numOfCols = ((SFilterColumnParam *)param)->numOfCols; SArray* pDataBlock = ((SFilterColumnParam *)param)->pDataBlock; for (int32_t j = 0; j < numOfCols; ++j) { SColumnInfoData* pColInfo = taosArrayGet(pDataBlock, j); if (id == pColInfo->info.colId) { - *data = pColInfo->pData; + *data = pColInfo; break; } } @@ -3628,13 +3596,28 @@ int32_t filterGetDataFromColId(void *param, int32_t id, void **data) { return TSDB_CODE_SUCCESS; } +int32_t fltGetDataFromSlotId(void *param, int32_t id, void **data) { + int32_t numOfCols = ((SFilterColumnParam *)param)->numOfCols; + SArray* pDataBlock = ((SFilterColumnParam *)param)->pDataBlock; + if (id < 0 || id >= numOfCols || id >= taosArrayGetSize(pDataBlock)) { + fltError("invalid slot id, id:%d, numOfCols:%d, arraySize:%d", id, numOfCols, (int32_t)taosArrayGetSize(pDataBlock)); + return TSDB_CODE_QRY_APP_ERROR; + } + + SColumnInfoData* pColInfo = taosArrayGet(pDataBlock, id); + *data = pColInfo; -int32_t filterSetDataFromSlotId(SFilterInfo *info, void *param, filer_get_col_from_id fp) { - return fltSetColFieldDataImpl(info, param, fp, false); + return TSDB_CODE_SUCCESS; } -int32_t filterSetDataFromColId(SFilterInfo *info, void *param, filer_get_col_from_id fp) { - return fltSetColFieldDataImpl(info, param, fp, true); + + +int32_t filterSetDataFromSlotId(SFilterInfo *info, void *param) { + return fltSetColFieldDataImpl(info, param, fltGetDataFromSlotId, false); +} + +int32_t filterSetDataFromColId(SFilterInfo *info, void *param) { + return fltSetColFieldDataImpl(info, param, fltGetDataFromColId, true); } @@ -3662,6 +3645,8 @@ int32_t filterInitFromNode(SNode* pNode, SFilterInfo **pInfo, uint32_t options) SFltTreeStat stat = {0}; FLT_ERR_JRET(fltReviseNodes(info, &pNode, &stat)); + info->scalarMode = stat.scalarMode; + if (!info->scalarMode) { FLT_ERR_JRET(fltInitFromNode(pNode, info, options)); } else { @@ -3680,13 +3665,21 @@ _return: FLT_RET(code); } -FORCE_INLINE bool filterExecute(SFilterInfo *info, SSDataBlock *pSrc, int8_t** p, SColumnDataAgg *statis, int16_t numOfCols) { +bool filterExecute(SFilterInfo *info, SSDataBlock *pSrc, int8_t** p, SColumnDataAgg *statis, int16_t numOfCols) { if (info->scalarMode) { SScalarParam output = {0}; FLT_ERR_RET(scalarCalculate(info->sclCtx.node, pSrc, &output)); *p = output.data; - return TSDB_CODE_SUCCESS; + + int8_t *r = output.data; + for (int32_t i = 0; i < output.num; ++i) { + if (0 == *(r+i)) { + return false; + } + } + + return true; } return (*info->func)(info, pSrc->info.rows, p, statis, numOfCols); diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index aa29b02709..742c7fd706 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -7,13 +7,87 @@ #include "sclInt.h" int32_t scalarGetOperatorParamNum(EOperatorType type) { - if (OP_TYPE_IS_NULL == type || OP_TYPE_IS_NOT_NULL == type) { + if (OP_TYPE_IS_NULL == type || OP_TYPE_IS_NOT_NULL == type || OP_TYPE_IS_TRUE == type || OP_TYPE_IS_NOT_TRUE == type + || OP_TYPE_IS_FALSE == type || OP_TYPE_IS_NOT_FALSE == type || OP_TYPE_IS_UNKNOWN == type || OP_TYPE_IS_NOT_UNKNOWN == type) { return 1; } return 2; } +int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type) { + SHashObj *pObj = taosHashInit(256, taosGetDefaultHashFunction(type), true, false); + if (NULL == pObj) { + sclError("taosHashInit failed, size:%d", 256); + SCL_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + taosHashSetEqualFp(pObj, taosGetDefaultEqualFunction(type)); + + int32_t code = 0; + SNodeListNode *nodeList = (SNodeListNode *)pNode; + SListCell *cell = nodeList->pNodeList->pHead; + SScalarParam in = {.num = 1}, out = {.num = 1, .type = type}; + int8_t dummy = 0; + int32_t bufLen = 60; + out.data = malloc(bufLen); + int32_t len = 0; + void *buf = NULL; + + for (int32_t i = 0; i < nodeList->pNodeList->length; ++i) { + SValueNode *valueNode = (SValueNode *)cell->pNode; + + if (valueNode->node.resType.type != type) { + in.type = valueNode->node.resType.type; + in.bytes = valueNode->node.resType.bytes; + in.data = nodesGetValueFromNode(valueNode); + + code = vectorConvertImpl(&in, &out); + if (code) { + sclError("convert from %d to %d failed", in.type, out.type); + SCL_ERR_JRET(code); + } + + if (IS_VAR_DATA_TYPE(type)) { + len = varDataLen(out.data); + buf = varDataVal(out.data); + } else { + len = tDataTypes[type].bytes; + buf = out.data; + } + } else { + buf = nodesGetValueFromNode(valueNode); + if (IS_VAR_DATA_TYPE(type)) { + len = varDataLen(buf); + buf = varDataVal(buf); + } else { + len = valueNode->node.resType.bytes; + buf = out.data; + } + } + + if (taosHashPut(pObj, buf, (size_t)len, &dummy, sizeof(dummy))) { + sclError("taosHashPut failed"); + SCL_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + cell = cell->pNext; + } + + tfree(out.data); + *data = pObj; + + return TSDB_CODE_SUCCESS; + +_return: + + tfree(out.data); + taosHashCleanup(pObj); + + SCL_RET(code); +} + + void sclFreeRes(SHashObj *res) { SScalarParam *p = NULL; void *pIter = taosHashIterate(res, NULL); @@ -42,28 +116,45 @@ int32_t sclInitParam(SNode* node, SScalarParam *param, SScalarCtx *ctx, int32_t param->num = 1; param->type = valueNode->node.resType.type; param->bytes = valueNode->node.resType.bytes; + param->colData = false; break; } case QUERY_NODE_NODE_LIST: { SNodeListNode *nodeList = (SNodeListNode *)node; - //TODO BUILD HASH + if (nodeList->pNodeList->length <= 0) { + sclError("invalid length in nodeList, length:%d", nodeList->pNodeList->length); + SCL_RET(TSDB_CODE_QRY_INVALID_INPUT); + } + + SCL_ERR_RET(scalarGenerateSetFromList(¶m->data, node, nodeList->dataType.type)); + param->num = 1; + param->type = SCL_DATA_TYPE_DUMMY_HASH; + param->colData = false; + break; } - case QUERY_NODE_COLUMN_REF: { + case QUERY_NODE_COLUMN: { if (NULL == ctx) { sclError("invalid node type for constant calculating, type:%d, ctx:%p", nodeType(node), ctx); SCL_ERR_RET(TSDB_CODE_QRY_APP_ERROR); } - SColumnRefNode *ref = (SColumnRefNode *)node; + SColumnNode *ref = (SColumnNode *)node; if (ref->slotId >= taosArrayGetSize(ctx->pSrc->pDataBlock)) { sclError("column ref slotId is too big, slodId:%d, dataBlockSize:%d", ref->slotId, (int32_t)taosArrayGetSize(ctx->pSrc->pDataBlock)); SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } SColumnInfoData *columnData = (SColumnInfoData *)taosArrayGet(ctx->pSrc->pDataBlock, ref->slotId); - param->data = columnData->pData; + if (IS_VAR_DATA_TYPE(columnData->info.type)) { + param->data = columnData; + param->colData = true; + } else { + param->data = columnData->pData; + param->colData = false; + } + param->num = ctx->pSrc->info.rows; param->type = columnData->info.type; param->bytes = columnData->info.bytes; @@ -190,7 +281,8 @@ int32_t sclExecFuncion(SFunctionNode *node, SScalarCtx *ctx, SScalarParam *outpu SScalarFuncExecFuncs ffpSet = {0}; int32_t code = fmGetScalarFuncExecFuncs(node->funcId, &ffpSet); if (code) { - sclError( "fmGetFuncExecFuncs failed, funcId:%d, code:%s", node->funcId, tstrerror(code)); + sclError( +"fmGetFuncExecFuncs failed, funcId:%d, code:%s", node->funcId, tstrerror(code)); SCL_ERR_RET(code); } @@ -208,7 +300,8 @@ int32_t sclExecFuncion(SFunctionNode *node, SScalarCtx *ctx, SScalarParam *outpu for (int32_t i = 0; i < rowNum; ++i) { code = (*ffpSet.process)(params, node->pParameterList->length, output); if (code) { - sclError( "scalar function exec failed, funcId:%d, code:%s", node->funcId, tstrerror(code)); + sclError( +"scalar function exec failed, funcId:%d, code:%s", node->funcId, tstrerror(code)); SCL_ERR_JRET(code); } @@ -248,11 +341,15 @@ int32_t sclExecLogic(SLogicConditionNode *node, SScalarCtx *ctx, SScalarParam *o SCL_ERR_RET(sclInitParamList(¶ms, node->pParameterList, ctx, &rowNum)); output->type = node->node.resType.type; + output->bytes = sizeof(bool); + output->num = rowNum; output->data = calloc(rowNum, sizeof(bool)); if (NULL == output->data) { sclError("calloc %d failed", (int32_t)(rowNum * sizeof(bool))); SCL_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); } + + void *data = output->data; bool value = false; @@ -275,6 +372,8 @@ int32_t sclExecLogic(SLogicConditionNode *node, SScalarCtx *ctx, SScalarParam *o sclParamMoveNext(params, node->pParameterList->length); } + output->data = data; + return TSDB_CODE_SUCCESS; _return: @@ -291,6 +390,8 @@ int32_t sclExecOperator(SOperatorNode *node, SScalarCtx *ctx, SScalarParam *outp SCL_ERR_RET(sclInitOperatorParams(¶ms, node, ctx, &rowNum)); output->type = node->node.resType.type; + output->num = rowNum; + output->bytes = tDataTypes[output->type].bytes; output->data = calloc(rowNum, tDataTypes[output->type].bytes); if (NULL == output->data) { sclError("calloc %d failed", (int32_t)rowNum * tDataTypes[output->type].bytes); @@ -302,17 +403,8 @@ int32_t sclExecOperator(SOperatorNode *node, SScalarCtx *ctx, SScalarParam *outp int32_t paramNum = scalarGetOperatorParamNum(node->opType); SScalarParam* pLeft = ¶ms[0]; SScalarParam* pRight = paramNum > 1 ? ¶ms[1] : NULL; - - for (int32_t i = 0; i < rowNum; ++i) { - - OperatorFn(pLeft, pRight, output->data, TSDB_ORDER_ASC); - - sclParamMoveNext(output, 1); - sclParamMoveNext(pLeft, 1); - if (pRight) { - sclParamMoveNext(pRight, 1); - } - } + + OperatorFn(pLeft, pRight, output->data, TSDB_ORDER_ASC); return TSDB_CODE_SUCCESS; @@ -427,7 +519,7 @@ EDealRes sclRewriteOperator(SNode** pNode, void* pContext) { EDealRes sclConstantsRewriter(SNode** pNode, void* pContext) { - if (QUERY_NODE_VALUE == nodeType(*pNode)) { + if (QUERY_NODE_VALUE == nodeType(*pNode) || QUERY_NODE_NODE_LIST == nodeType(*pNode)) { return DEAL_RES_CONTINUE; } @@ -509,10 +601,10 @@ EDealRes sclWalkOperator(SNode* pNode, void* pContext) { EDealRes sclCalcWalker(SNode* pNode, void* pContext) { - if (QUERY_NODE_VALUE == nodeType(pNode)) { + if (QUERY_NODE_VALUE == nodeType(pNode) || QUERY_NODE_NODE_LIST == nodeType(pNode) || QUERY_NODE_COLUMN == nodeType(pNode)) { return DEAL_RES_CONTINUE; } - + if (QUERY_NODE_FUNCTION == nodeType(pNode)) { return sclWalkFunction(pNode, pContext); } @@ -525,7 +617,7 @@ EDealRes sclCalcWalker(SNode* pNode, void* pContext) { return sclWalkOperator(pNode, pContext); } - sclError("invalid node type for calculating constants, type:%d", nodeType(pNode)); + sclError("invalid node type for scalar calculating, type:%d", nodeType(pNode)); SScalarCtx *ctx = (SScalarCtx *)pContext; diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index ab2cc8b056..fd900afcda 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -268,9 +268,6 @@ static void setScalarFuncParam(SScalarParam* param, int32_t type, int32_t bytes, param->data = pInput; } -bool isStringOp(int32_t op) { - return op == TSDB_BINARY_OP_CONCAT; -} #if 0 int32_t evaluateExprNodeTree(tExprNode* pExprs, int32_t numOfRows, SScalarFuncParam* pOutput, void* param, diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index 5b2ec7902c..85af663313 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -21,6 +21,9 @@ #include "querynodes.h" #include "filterInt.h" #include "query.h" +#include "sclInt.h" +#include "tep.h" +#include "filter.h" //GET_TYPED_DATA(v, double, pRight->type, (char *)&((right)[i])); @@ -93,6 +96,7 @@ double getVectorDoubleValue_FLOAT(void *src, int32_t index) { double getVectorDoubleValue_DOUBLE(void *src, int32_t index) { return (double)*((double *)src + index); } + _getDoubleValue_fn_t getVectorDoubleValueFn(int32_t srcType) { _getDoubleValue_fn_t p = NULL; if(srcType==TSDB_DATA_TYPE_TINYINT) { @@ -218,6 +222,12 @@ void* getVectorValueAddr_FLOAT(void *src, int32_t index) { void* getVectorValueAddr_DOUBLE(void *src, int32_t index) { return (void*)((double *)src + index); } +void* getVectorValueAddr_default(void *src, int32_t index) { + return src; +} +void* getVectorValueAddr_VAR(void *src, int32_t index) { + return colDataGet((SColumnInfoData *)src, index); +} _getValueAddr_fn_t getVectorValueAddrFn(int32_t srcType) { _getValueAddr_fn_t p = NULL; @@ -241,8 +251,12 @@ _getValueAddr_fn_t getVectorValueAddrFn(int32_t srcType) { p = getVectorValueAddr_FLOAT; }else if(srcType==TSDB_DATA_TYPE_DOUBLE) { p = getVectorValueAddr_DOUBLE; + }else if(srcType==TSDB_DATA_TYPE_BINARY) { + p = getVectorValueAddr_VAR; + }else if(srcType==TSDB_DATA_TYPE_NCHAR) { + p = getVectorValueAddr_VAR; }else { - assert(0); + p = getVectorValueAddr_default; } return p; } @@ -258,31 +272,7 @@ int32_t vectorConvertImpl(SScalarParam* pIn, SScalarParam* pOut) { switch (outType) { case TSDB_DATA_TYPE_BOOL: - if (inType == TSDB_DATA_TYPE_BINARY) { - for (int32_t i = 0; i < pIn->num; ++i) { - GET_TYPED_DATA(*(bool *)output, bool, TSDB_DATA_TYPE_USMALLINT, &varDataLen(input)); - - input += varDataLen(input) + VARSTR_HEADER_SIZE; - output += sizeof(bool); - } - } else if (inType == TSDB_DATA_TYPE_NCHAR) { - for (int32_t i = 0; i < pIn->num; ++i) { - GET_TYPED_DATA(*(bool *)output, bool, TSDB_DATA_TYPE_USMALLINT, &varDataLen(input)); - - input += varDataLen(input) + VARSTR_HEADER_SIZE; - output += tDataTypes[outType].bytes; - } - } else { - for (int32_t i = 0; i < pIn->num; ++i) { - uint64_t value = 0; - GET_TYPED_DATA(value, uint64_t, inType, input); - SET_TYPED_DATA(output, outType, value); - - input += tDataTypes[inType].bytes; - output += tDataTypes[outType].bytes; - } - } - break; + case TSDB_DATA_TYPE_TINYINT: case TSDB_DATA_TYPE_SMALLINT: case TSDB_DATA_TYPE_INT: case TSDB_DATA_TYPE_BIGINT: @@ -291,21 +281,26 @@ int32_t vectorConvertImpl(SScalarParam* pIn, SScalarParam* pOut) { int32_t bufSize = varDataLen(input) + 1; char *tmp = malloc(bufSize); if (NULL == tmp) { + sclError("malloc %d failed", bufSize); return TSDB_CODE_QRY_OUT_OF_MEMORY; } for (int32_t i = 0; i < pIn->num; ++i) { - if (varDataLen(input) >= bufSize) { - bufSize = varDataLen(input) + 1; - tmp = realloc(tmp, bufSize); + if (isNull(input, inType)) { + assignVal(output, getNullValue(outType), 0, outType); + } else { + if (varDataLen(input) >= bufSize) { + bufSize = varDataLen(input) + 1; + tmp = realloc(tmp, bufSize); + } + + memcpy(tmp, varDataVal(input), varDataLen(input)); + tmp[varDataLen(input)] = 0; + + int64_t value = strtoll(tmp, NULL, 10); + SET_TYPED_DATA(output, outType, value); } - memcpy(tmp, varDataVal(input), varDataLen(input)); - tmp[varDataLen(input)] = 0; - - int64_t value = strtoll(tmp, NULL, 10); - SET_TYPED_DATA(output, outType, value); - input += varDataLen(input) + VARSTR_HEADER_SIZE; output += tDataTypes[outType].bytes; } @@ -315,25 +310,30 @@ int32_t vectorConvertImpl(SScalarParam* pIn, SScalarParam* pOut) { int32_t bufSize = varDataLen(input) * TSDB_NCHAR_SIZE + 1; char *tmp = calloc(1, bufSize); if (NULL == tmp) { + sclError("calloc %d failed", bufSize); return TSDB_CODE_QRY_OUT_OF_MEMORY; } for (int32_t i = 0; i < pIn->num; ++i) { - if (varDataLen(input)* TSDB_NCHAR_SIZE >= bufSize) { - bufSize = varDataLen(input) * TSDB_NCHAR_SIZE + 1; - tmp = realloc(tmp, bufSize); - } - - int len = taosUcs4ToMbs(varDataVal(input), varDataLen(input), tmp); - if (len < 0){ - qError("castConvert taosUcs4ToMbs error 1"); - tfree(tmp); - return TSDB_CODE_QRY_APP_ERROR; - } + if (isNull(input, inType)) { + assignVal(output, getNullValue(outType), 0, outType); + } else { + if (varDataLen(input)* TSDB_NCHAR_SIZE >= bufSize) { + bufSize = varDataLen(input) * TSDB_NCHAR_SIZE + 1; + tmp = realloc(tmp, bufSize); + } - tmp[len] = 0; - int64_t value = strtoll(tmp, NULL, 10); - SET_TYPED_DATA(output, outType, value); + int len = taosUcs4ToMbs(varDataVal(input), varDataLen(input), tmp); + if (len < 0){ + sclError("castConvert taosUcs4ToMbs error 1"); + tfree(tmp); + return TSDB_CODE_QRY_APP_ERROR; + } + + tmp[len] = 0; + int64_t value = strtoll(tmp, NULL, 10); + SET_TYPED_DATA(output, outType, value); + } input += varDataLen(input) + VARSTR_HEADER_SIZE; output += tDataTypes[outType].bytes; @@ -351,6 +351,7 @@ int32_t vectorConvertImpl(SScalarParam* pIn, SScalarParam* pOut) { } } break; + case TSDB_DATA_TYPE_UTINYINT: case TSDB_DATA_TYPE_USMALLINT: case TSDB_DATA_TYPE_UINT: case TSDB_DATA_TYPE_UBIGINT: @@ -358,20 +359,25 @@ int32_t vectorConvertImpl(SScalarParam* pIn, SScalarParam* pOut) { int32_t bufSize = varDataLen(input) + 1; char *tmp = malloc(bufSize); if (NULL == tmp) { + sclError("malloc %d failed", bufSize); return TSDB_CODE_QRY_OUT_OF_MEMORY; } for (int32_t i = 0; i < pIn->num; ++i) { - if (varDataLen(input) >= bufSize) { - bufSize = varDataLen(input) + 1; - tmp = realloc(tmp, bufSize); + if (isNull(input, inType)) { + assignVal(output, getNullValue(outType), 0, outType); + } else { + if (varDataLen(input) >= bufSize) { + bufSize = varDataLen(input) + 1; + tmp = realloc(tmp, bufSize); + } + + memcpy(tmp, varDataVal(input), varDataLen(input)); + tmp[varDataLen(input)] = 0; + uint64_t value = strtoull(tmp, NULL, 10); + SET_TYPED_DATA(output, outType, value); } - memcpy(tmp, varDataVal(input), varDataLen(input)); - tmp[varDataLen(input)] = 0; - uint64_t value = strtoull(tmp, NULL, 10); - SET_TYPED_DATA(output, outType, value); - input += varDataLen(input) + VARSTR_HEADER_SIZE; output += tDataTypes[outType].bytes; } @@ -381,25 +387,30 @@ int32_t vectorConvertImpl(SScalarParam* pIn, SScalarParam* pOut) { int32_t bufSize = varDataLen(input) * TSDB_NCHAR_SIZE + 1; char *tmp = calloc(1, bufSize); if (NULL == tmp) { + sclError("calloc %d failed", bufSize); return TSDB_CODE_QRY_OUT_OF_MEMORY; } for (int32_t i = 0; i < pIn->num; ++i) { - if (varDataLen(input)* TSDB_NCHAR_SIZE >= bufSize) { - bufSize = varDataLen(input) * TSDB_NCHAR_SIZE + 1; - tmp = realloc(tmp, bufSize); - } - - int len = taosUcs4ToMbs(varDataVal(input), varDataLen(input), tmp); - if (len < 0){ - qError("castConvert taosUcs4ToMbs error 1"); - tfree(tmp); - return TSDB_CODE_QRY_APP_ERROR; - } + if (isNull(input, inType)) { + assignVal(output, getNullValue(outType), 0, outType); + } else { + if (varDataLen(input)* TSDB_NCHAR_SIZE >= bufSize) { + bufSize = varDataLen(input) * TSDB_NCHAR_SIZE + 1; + tmp = realloc(tmp, bufSize); + } - tmp[len] = 0; - uint64_t value = strtoull(tmp, NULL, 10); - SET_TYPED_DATA(output, outType, value); + int len = taosUcs4ToMbs(varDataVal(input), varDataLen(input), tmp); + if (len < 0){ + sclError("castConvert taosUcs4ToMbs error 1"); + tfree(tmp); + return TSDB_CODE_QRY_APP_ERROR; + } + + tmp[len] = 0; + uint64_t value = strtoull(tmp, NULL, 10); + SET_TYPED_DATA(output, outType, value); + } input += varDataLen(input) + VARSTR_HEADER_SIZE; output += tDataTypes[outType].bytes; @@ -408,9 +419,13 @@ int32_t vectorConvertImpl(SScalarParam* pIn, SScalarParam* pOut) { tfree(tmp); } else { for (int32_t i = 0; i < pIn->num; ++i) { - uint64_t value = 0; - GET_TYPED_DATA(value, uint64_t, inType, input); - SET_TYPED_DATA(output, outType, value); + if (isNull(input, inType)) { + assignVal(output, getNullValue(outType), 0, outType); + } else { + uint64_t value = 0; + GET_TYPED_DATA(value, uint64_t, inType, input); + SET_TYPED_DATA(output, outType, value); + } input += tDataTypes[inType].bytes; output += tDataTypes[outType].bytes; @@ -423,21 +438,26 @@ int32_t vectorConvertImpl(SScalarParam* pIn, SScalarParam* pOut) { int32_t bufSize = varDataLen(input) + 1; char *tmp = malloc(bufSize); if (NULL == tmp) { + sclError("malloc %d failed", bufSize); return TSDB_CODE_QRY_OUT_OF_MEMORY; } for (int32_t i = 0; i < pIn->num; ++i) { - if (varDataLen(input) >= bufSize) { - bufSize = varDataLen(input) + 1; - tmp = realloc(tmp, bufSize); + if (isNull(input, inType)) { + assignVal(output, getNullValue(outType), 0, outType); + } else { + if (varDataLen(input) >= bufSize) { + bufSize = varDataLen(input) + 1; + tmp = realloc(tmp, bufSize); + } + + memcpy(tmp, varDataVal(input), varDataLen(input)); + tmp[varDataLen(input)] = 0; + + double value = strtod(tmp, NULL); + SET_TYPED_DATA(output, outType, value); } - memcpy(tmp, varDataVal(input), varDataLen(input)); - tmp[varDataLen(input)] = 0; - - double value = strtod(tmp, NULL); - SET_TYPED_DATA(output, outType, value); - input += varDataLen(input) + VARSTR_HEADER_SIZE; output += tDataTypes[outType].bytes; } @@ -447,25 +467,30 @@ int32_t vectorConvertImpl(SScalarParam* pIn, SScalarParam* pOut) { int32_t bufSize = varDataLen(input) * TSDB_NCHAR_SIZE + 1; char *tmp = calloc(1, bufSize); if (NULL == tmp) { + sclError("calloc %d failed", bufSize); return TSDB_CODE_QRY_OUT_OF_MEMORY; } for (int32_t i = 0; i < pIn->num; ++i) { - if (varDataLen(input)* TSDB_NCHAR_SIZE >= bufSize) { - bufSize = varDataLen(input) * TSDB_NCHAR_SIZE + 1; - tmp = realloc(tmp, bufSize); - } - - int len = taosUcs4ToMbs(varDataVal(input), varDataLen(input), tmp); - if (len < 0){ - qError("castConvert taosUcs4ToMbs error 1"); - tfree(tmp); - return TSDB_CODE_QRY_APP_ERROR; - } + if (isNull(input, inType)) { + assignVal(output, getNullValue(outType), 0, outType); + } else { + if (varDataLen(input)* TSDB_NCHAR_SIZE >= bufSize) { + bufSize = varDataLen(input) * TSDB_NCHAR_SIZE + 1; + tmp = realloc(tmp, bufSize); + } - tmp[len] = 0; - double value = strtod(tmp, NULL); - SET_TYPED_DATA(output, outType, value); + int len = taosUcs4ToMbs(varDataVal(input), varDataLen(input), tmp); + if (len < 0){ + sclError("castConvert taosUcs4ToMbs error 1"); + tfree(tmp); + return TSDB_CODE_QRY_APP_ERROR; + } + + tmp[len] = 0; + double value = strtod(tmp, NULL); + SET_TYPED_DATA(output, outType, value); + } input += varDataLen(input) + VARSTR_HEADER_SIZE; output += tDataTypes[outType].bytes; @@ -474,9 +499,13 @@ int32_t vectorConvertImpl(SScalarParam* pIn, SScalarParam* pOut) { tfree(tmp); } else { for (int32_t i = 0; i < pIn->num; ++i) { - int64_t value = 0; - GET_TYPED_DATA(value, int64_t, inType, input); - SET_TYPED_DATA(output, outType, value); + if (isNull(input, inType)) { + assignVal(output, getNullValue(outType), 0, outType); + } else { + int64_t value = 0; + GET_TYPED_DATA(value, int64_t, inType, input); + SET_TYPED_DATA(output, outType, value); + } input += tDataTypes[inType].bytes; output += tDataTypes[outType].bytes; @@ -484,7 +513,7 @@ int32_t vectorConvertImpl(SScalarParam* pIn, SScalarParam* pOut) { } break; default: - qError("invalid convert output type:%d", outType); + sclError("invalid convert output type:%d", outType); return TSDB_CODE_QRY_APP_ERROR; } @@ -532,6 +561,10 @@ int32_t vectorConvert(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam* p return TSDB_CODE_SUCCESS; } + if (SCL_DATA_TYPE_DUMMY_HASH == pLeft->type || SCL_DATA_TYPE_DUMMY_HASH == pRight->type) { + return TSDB_CODE_SUCCESS; + } + SScalarParam *param1 = NULL, *paramOut1 = NULL; SScalarParam *param2 = NULL, *paramOut2 = NULL; int32_t code = 0; @@ -595,6 +628,46 @@ void vectorAdd(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _or int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; + SScalarParam leftParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pLeft->num}; + SScalarParam rightParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pRight->num}; + if (IS_VAR_DATA_TYPE(pLeft->type)) { + leftParam.data = calloc(leftParam.num, sizeof(double)); + if (NULL == leftParam.data) { + sclError("malloc %d failed", (int32_t)(leftParam.num * sizeof(double))); + return; + } + + if (pLeft->colData) { + SColumnInfoData *colInfo = (SColumnInfoData *)pLeft->data; + pLeft->data = colInfo->pData; + } + + if (vectorConvertImpl(pLeft, &leftParam)) { + return; + } + pLeft = &leftParam; + } + if (IS_VAR_DATA_TYPE(pRight->type)) { + rightParam.data = calloc(rightParam.num, sizeof(double)); + if (NULL == rightParam.data) { + sclError("malloc %d failed", (int32_t)(rightParam.num * sizeof(double))); + tfree(leftParam.data); + return; + } + + if (pRight->colData) { + SColumnInfoData *colInfo = (SColumnInfoData *)pRight->data; + pRight->data = colInfo->pData; + } + + if (vectorConvertImpl(pRight, &rightParam)) { + tfree(leftParam.data); + tfree(rightParam.data); + return; + } + pRight = &rightParam; + } + double *output=(double*)out; _getValueAddr_fn_t getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type); _getValueAddr_fn_t getVectorValueAddrFnRight = getVectorValueAddrFn(pRight->type); @@ -628,12 +701,55 @@ void vectorAdd(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _or SET_DOUBLE_VAL(output,getVectorDoubleValueFnLeft(pLeft->data,i) + getVectorDoubleValueFnRight(pRight->data,0)); } } + + tfree(leftParam.data); + tfree(rightParam.data); } void vectorSub(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; + SScalarParam leftParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pLeft->num}; + SScalarParam rightParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pRight->num}; + if (IS_VAR_DATA_TYPE(pLeft->type)) { + leftParam.data = calloc(leftParam.num, sizeof(double)); + if (NULL == leftParam.data) { + sclError("malloc %d failed", (int32_t)(leftParam.num * sizeof(double))); + return; + } + + if (pLeft->colData) { + SColumnInfoData *colInfo = (SColumnInfoData *)pLeft->data; + pLeft->data = colInfo->pData; + } + + if (vectorConvertImpl(pLeft, &leftParam)) { + return; + } + pLeft = &leftParam; + } + if (IS_VAR_DATA_TYPE(pRight->type)) { + rightParam.data = calloc(rightParam.num, sizeof(double)); + if (NULL == rightParam.data) { + sclError("malloc %d failed", (int32_t)(rightParam.num * sizeof(double))); + tfree(leftParam.data); + return; + } + + if (pRight->colData) { + SColumnInfoData *colInfo = (SColumnInfoData *)pRight->data; + pRight->data = colInfo->pData; + } + + if (vectorConvertImpl(pRight, &rightParam)) { + tfree(leftParam.data); + tfree(rightParam.data); + return; + } + pRight = &rightParam; + } + double *output=(double*)out; _getValueAddr_fn_t getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type); _getValueAddr_fn_t getVectorValueAddrFnRight = getVectorValueAddrFn(pRight->type); @@ -666,11 +782,54 @@ void vectorSub(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _or SET_DOUBLE_VAL(output,getVectorDoubleValueFnLeft(pLeft->data,i) - getVectorDoubleValueFnRight(pRight->data,0)); } } + + tfree(leftParam.data); + tfree(rightParam.data); } void vectorMultiply(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; + SScalarParam leftParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pLeft->num}; + SScalarParam rightParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pRight->num}; + if (IS_VAR_DATA_TYPE(pLeft->type)) { + leftParam.data = calloc(leftParam.num, sizeof(double)); + if (NULL == leftParam.data) { + sclError("malloc %d failed", (int32_t)(leftParam.num * sizeof(double))); + return; + } + + if (pLeft->colData) { + SColumnInfoData *colInfo = (SColumnInfoData *)pLeft->data; + pLeft->data = colInfo->pData; + } + + if (vectorConvertImpl(pLeft, &leftParam)) { + return; + } + pLeft = &leftParam; + } + if (IS_VAR_DATA_TYPE(pRight->type)) { + rightParam.data = calloc(rightParam.num, sizeof(double)); + if (NULL == rightParam.data) { + sclError("malloc %d failed", (int32_t)(rightParam.num * sizeof(double))); + tfree(leftParam.data); + return; + } + + if (pRight->colData) { + SColumnInfoData *colInfo = (SColumnInfoData *)pRight->data; + pRight->data = colInfo->pData; + } + + if (vectorConvertImpl(pRight, &rightParam)) { + tfree(leftParam.data); + tfree(rightParam.data); + return; + } + pRight = &rightParam; + } + double *output=(double*)out; _getValueAddr_fn_t getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type); _getValueAddr_fn_t getVectorValueAddrFnRight = getVectorValueAddrFn(pRight->type); @@ -704,12 +863,55 @@ void vectorMultiply(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_ SET_DOUBLE_VAL(output,getVectorDoubleValueFnLeft(pLeft->data,i) * getVectorDoubleValueFnRight(pRight->data,0)); } } + + tfree(leftParam.data); + tfree(rightParam.data); } void vectorDivide(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; + SScalarParam leftParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pLeft->num}; + SScalarParam rightParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pRight->num}; + if (IS_VAR_DATA_TYPE(pLeft->type)) { + leftParam.data = calloc(leftParam.num, sizeof(double)); + if (NULL == leftParam.data) { + sclError("malloc %d failed", (int32_t)(leftParam.num * sizeof(double))); + return; + } + + if (pLeft->colData) { + SColumnInfoData *colInfo = (SColumnInfoData *)pLeft->data; + pLeft->data = colInfo->pData; + } + + if (vectorConvertImpl(pLeft, &leftParam)) { + return; + } + pLeft = &leftParam; + } + if (IS_VAR_DATA_TYPE(pRight->type)) { + rightParam.data = calloc(rightParam.num, sizeof(double)); + if (NULL == rightParam.data) { + sclError("malloc %d failed", (int32_t)(rightParam.num * sizeof(double))); + tfree(leftParam.data); + return; + } + + if (pRight->colData) { + SColumnInfoData *colInfo = (SColumnInfoData *)pRight->data; + pRight->data = colInfo->pData; + } + + if (vectorConvertImpl(pRight, &rightParam)) { + tfree(leftParam.data); + tfree(rightParam.data); + return; + } + pRight = &rightParam; + } + double *output=(double*)out; _getValueAddr_fn_t getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type); _getValueAddr_fn_t getVectorValueAddrFnRight = getVectorValueAddrFn(pRight->type); @@ -750,12 +952,55 @@ void vectorDivide(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t SET_DOUBLE_VAL(output, getVectorDoubleValueFnLeft(pLeft->data, i) / right); } } + + tfree(leftParam.data); + tfree(rightParam.data); } void vectorRemainder(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; + SScalarParam leftParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pLeft->num}; + SScalarParam rightParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pRight->num}; + if (IS_VAR_DATA_TYPE(pLeft->type)) { + leftParam.data = calloc(leftParam.num, sizeof(double)); + if (NULL == leftParam.data) { + sclError("malloc %d failed", (int32_t)(leftParam.num * sizeof(double))); + return; + } + + if (pLeft->colData) { + SColumnInfoData *colInfo = (SColumnInfoData *)pLeft->data; + pLeft->data = colInfo->pData; + } + + if (vectorConvertImpl(pLeft, &leftParam)) { + return; + } + pLeft = &leftParam; + } + if (IS_VAR_DATA_TYPE(pRight->type)) { + rightParam.data = calloc(rightParam.num, sizeof(double)); + if (NULL == rightParam.data) { + sclError("malloc %d failed", (int32_t)(rightParam.num * sizeof(double))); + tfree(leftParam.data); + return; + } + + if (pRight->colData) { + SColumnInfoData *colInfo = (SColumnInfoData *)pRight->data; + pRight->data = colInfo->pData; + } + + if (vectorConvertImpl(pRight, &rightParam)) { + tfree(leftParam.data); + tfree(rightParam.data); + return; + } + pRight = &rightParam; + } + double * output = (double *)out; _getValueAddr_fn_t getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type); _getValueAddr_fn_t getVectorValueAddrFnRight = getVectorValueAddrFn(pRight->type); @@ -822,6 +1067,9 @@ void vectorRemainder(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32 SET_DOUBLE_VAL(output, left - ((int64_t)(left / right)) * right); } } + + tfree(leftParam.data); + tfree(rightParam.data); } void vectorConcat(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { @@ -879,6 +1127,46 @@ void vectorBitAnd(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; + SScalarParam leftParam = {.type = TSDB_DATA_TYPE_BIGINT, .num = pLeft->num}; + SScalarParam rightParam = {.type = TSDB_DATA_TYPE_BIGINT, .num = pRight->num}; + if (IS_VAR_DATA_TYPE(pLeft->type)) { + leftParam.data = calloc(leftParam.num, sizeof(int64_t)); + if (NULL == leftParam.data) { + sclError("malloc %d failed", (int32_t)(leftParam.num * sizeof(int64_t))); + return; + } + + if (pLeft->colData) { + SColumnInfoData *colInfo = (SColumnInfoData *)pLeft->data; + pLeft->data = colInfo->pData; + } + + if (vectorConvertImpl(pLeft, &leftParam)) { + return; + } + pLeft = &leftParam; + } + if (IS_VAR_DATA_TYPE(pRight->type)) { + rightParam.data = calloc(rightParam.num, sizeof(int64_t)); + if (NULL == rightParam.data) { + sclError("malloc %d failed", (int32_t)(rightParam.num * sizeof(int64_t))); + tfree(leftParam.data); + return; + } + + if (pRight->colData) { + SColumnInfoData *colInfo = (SColumnInfoData *)pRight->data; + pRight->data = colInfo->pData; + } + + if (vectorConvertImpl(pRight, &rightParam)) { + tfree(leftParam.data); + tfree(rightParam.data); + return; + } + pRight = &rightParam; + } + int64_t *output=(int64_t *)out; _getValueAddr_fn_t getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type); _getValueAddr_fn_t getVectorValueAddrFnRight = getVectorValueAddrFn(pRight->type); @@ -912,12 +1200,55 @@ void vectorBitAnd(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t SET_BIGINT_VAL(output,getVectorBigintValueFnLeft(pLeft->data,i) & getVectorBigintValueFnRight(pRight->data,0)); } } + + tfree(leftParam.data); + tfree(rightParam.data); } void vectorBitOr(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; + SScalarParam leftParam = {.type = TSDB_DATA_TYPE_BIGINT, .num = pLeft->num}; + SScalarParam rightParam = {.type = TSDB_DATA_TYPE_BIGINT, .num = pRight->num}; + if (IS_VAR_DATA_TYPE(pLeft->type)) { + leftParam.data = calloc(leftParam.num, sizeof(int64_t)); + if (NULL == leftParam.data) { + sclError("malloc %d failed", (int32_t)(leftParam.num * sizeof(int64_t))); + return; + } + + if (pLeft->colData) { + SColumnInfoData *colInfo = (SColumnInfoData *)pLeft->data; + pLeft->data = colInfo->pData; + } + + if (vectorConvertImpl(pLeft, &leftParam)) { + return; + } + pLeft = &leftParam; + } + if (IS_VAR_DATA_TYPE(pRight->type)) { + rightParam.data = calloc(rightParam.num, sizeof(int64_t)); + if (NULL == rightParam.data) { + sclError("malloc %d failed", (int32_t)(rightParam.num * sizeof(int64_t))); + tfree(leftParam.data); + return; + } + + if (pRight->colData) { + SColumnInfoData *colInfo = (SColumnInfoData *)pRight->data; + pRight->data = colInfo->pData; + } + + if (vectorConvertImpl(pRight, &rightParam)) { + tfree(leftParam.data); + tfree(rightParam.data); + return; + } + pRight = &rightParam; + } + int64_t *output=(int64_t *)out; _getValueAddr_fn_t getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type); _getValueAddr_fn_t getVectorValueAddrFnRight = getVectorValueAddrFn(pRight->type); @@ -951,6 +1282,9 @@ void vectorBitOr(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ SET_BIGINT_VAL(output,getVectorBigintValueFnLeft(pLeft->data,i) | getVectorBigintValueFnRight(pRight->data,0)); } } + + tfree(leftParam.data); + tfree(rightParam.data); } @@ -961,10 +1295,23 @@ void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, void *out, int bool res = false; bool *output=(bool *)out; - _getValueAddr_fn_t getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type); - _getValueAddr_fn_t getVectorValueAddrFnRight = getVectorValueAddrFn(pRight->type); + _getValueAddr_fn_t getVectorValueAddrFnLeft = NULL; + _getValueAddr_fn_t getVectorValueAddrFnRight = NULL; - if (pLeft->num == pRight->num) { + if (IS_VAR_DATA_TYPE(pLeft->type) && !pLeft->colData) { + getVectorValueAddrFnLeft = getVectorValueAddr_default; + } else { + getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type); + } + + if (IS_VAR_DATA_TYPE(pRight->type) && !pRight->colData) { + getVectorValueAddrFnRight = getVectorValueAddr_default; + } else { + getVectorValueAddrFnRight = getVectorValueAddrFn(pRight->type); + } + + + if (pLeft->num == pRight->num) { for (; i < pRight->num && i >= 0; i += step, output += 1) { if (isNull(getVectorValueAddrFnLeft(pLeft->data, i), pLeft->type) || isNull(getVectorValueAddrFnRight(pRight->data, i), pRight->type)) { @@ -992,7 +1339,7 @@ void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, void *out, int SET_TYPED_DATA(output, TSDB_DATA_TYPE_BOOL, res); } } else if (pRight->num == 1) { - void *rightData = getVectorValueAddrFnRight(pRight->data, 0); + void *rightData = getVectorValueAddrFnRight(pRight->data, 0); for (; i >= 0 && i < pLeft->num; i += step, output += 1) { if (isNull(getVectorValueAddrFnLeft(pLeft->data,i), pLeft->type) || isNull(rightData, pRight->type)) { @@ -1030,55 +1377,55 @@ void vectorCompare(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t param2 = pRight; } - vectorCompareImpl(param1, param2, out, _ord, TSDB_RELATION_GREATER); + vectorCompareImpl(param1, param2, out, _ord, optr); } void vectorGreater(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { - vectorCompare(pLeft, pRight, out, _ord, TSDB_RELATION_GREATER); + vectorCompare(pLeft, pRight, out, _ord, OP_TYPE_GREATER_THAN); } void vectorGreaterEqual(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { - vectorCompare(pLeft, pRight, out, _ord, TSDB_RELATION_GREATER_EQUAL); + vectorCompare(pLeft, pRight, out, _ord, OP_TYPE_GREATER_EQUAL); } void vectorLower(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { - vectorCompare(pLeft, pRight, out, _ord, TSDB_RELATION_LESS); + vectorCompare(pLeft, pRight, out, _ord, OP_TYPE_LOWER_THAN); } void vectorLowerEqual(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { - vectorCompare(pLeft, pRight, out, _ord, TSDB_RELATION_LESS_EQUAL); + vectorCompare(pLeft, pRight, out, _ord, OP_TYPE_LOWER_EQUAL); } void vectorEqual(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { - vectorCompare(pLeft, pRight, out, _ord, TSDB_RELATION_EQUAL); + vectorCompare(pLeft, pRight, out, _ord, OP_TYPE_EQUAL); } void vectorNotEqual(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { - vectorCompare(pLeft, pRight, out, _ord, TSDB_RELATION_NOT_EQUAL); + vectorCompare(pLeft, pRight, out, _ord, OP_TYPE_NOT_EQUAL); } void vectorIn(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { - vectorCompare(pLeft, pRight, out, _ord, TSDB_RELATION_IN); + vectorCompare(pLeft, pRight, out, _ord, OP_TYPE_IN); } void vectorNotIn(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { - vectorCompare(pLeft, pRight, out, _ord, TSDB_RELATION_NOT_IN); + vectorCompare(pLeft, pRight, out, _ord, OP_TYPE_NOT_IN); } void vectorLike(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { - vectorCompare(pLeft, pRight, out, _ord, TSDB_RELATION_LIKE); + vectorCompare(pLeft, pRight, out, _ord, OP_TYPE_LIKE); } void vectorNotLike(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { - vectorCompare(pLeft, pRight, out, _ord, TSDB_RELATION_NOT_LIKE); + vectorCompare(pLeft, pRight, out, _ord, OP_TYPE_NOT_LIKE); } void vectorMatch(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { - vectorCompare(pLeft, pRight, out, _ord, TSDB_RELATION_MATCH); + vectorCompare(pLeft, pRight, out, _ord, OP_TYPE_MATCH); } void vectorNotMatch(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { - vectorCompare(pLeft, pRight, out, _ord, TSDB_RELATION_NMATCH); + vectorCompare(pLeft, pRight, out, _ord, OP_TYPE_NMATCH); } void vectorIsNull(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { @@ -1087,7 +1434,13 @@ void vectorIsNull(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t bool res = false; bool *output=(bool *)out; - _getValueAddr_fn_t getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type); + _getValueAddr_fn_t getVectorValueAddrFnLeft = NULL; + + if (IS_VAR_DATA_TYPE(pLeft->type) && !pLeft->colData) { + getVectorValueAddrFnLeft = getVectorValueAddr_default; + } else { + getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type); + } for (; i >= 0 && i < pLeft->num; i += step, output += 1) { if (isNull(getVectorValueAddrFnLeft(pLeft->data,i), pLeft->type)) { @@ -1107,7 +1460,13 @@ void vectorNotNull(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t bool res = false; bool *output = (bool *)out; - _getValueAddr_fn_t getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type); + _getValueAddr_fn_t getVectorValueAddrFnLeft = NULL; + + if (IS_VAR_DATA_TYPE(pLeft->type) && !pLeft->colData) { + getVectorValueAddrFnLeft = getVectorValueAddr_default; + } else { + getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type); + } for (; i >= 0 && i < pLeft->num; i += step, output += 1) { if (isNull(getVectorValueAddrFnLeft(pLeft->data,i), pLeft->type)) { @@ -1121,6 +1480,17 @@ void vectorNotNull(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t } } +void vectorIsTrue(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { + SScalarParam output = {.data = out, .num = pLeft->num, .type = TSDB_DATA_TYPE_BOOL}; + + if (pLeft->colData) { + SColumnInfoData *colInfo = (SColumnInfoData *)pLeft->data; + pLeft->data = colInfo->pData; + } + + vectorConvertImpl(pLeft, &output); +} + _bin_scalar_fn_t getBinScalarOperatorFn(int32_t binFunctionId) { switch (binFunctionId) { @@ -1166,12 +1536,12 @@ _bin_scalar_fn_t getBinScalarOperatorFn(int32_t binFunctionId) { return vectorBitAnd; case OP_TYPE_BIT_OR: return vectorBitOr; + case OP_TYPE_IS_TRUE: + return vectorIsTrue; default: assert(0); return NULL; } } -bool isBinaryStringOp(int32_t op) { - return op == TSDB_BINARY_OP_CONCAT; -} + diff --git a/source/libs/scalar/test/CMakeLists.txt b/source/libs/scalar/test/CMakeLists.txt index a9af1ece30..caaf86264c 100644 --- a/source/libs/scalar/test/CMakeLists.txt +++ b/source/libs/scalar/test/CMakeLists.txt @@ -1,18 +1,4 @@ +enable_testing() -MESSAGE(STATUS "build scalar unit test") - -# GoogleTest requires at least C++11 -SET(CMAKE_CXX_STANDARD 11) -AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) - -ADD_EXECUTABLE(scalarTest ${SOURCE_LIST}) -TARGET_LINK_LIBRARIES( - scalarTest - PUBLIC os util common gtest qcom function nodes -) - -TARGET_INCLUDE_DIRECTORIES( - scalarTest - PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/scalar/" - PRIVATE "${CMAKE_SOURCE_DIR}/source/libs/scalar/inc" -) +add_subdirectory(filter) +add_subdirectory(scalar) diff --git a/source/libs/scalar/test/filter/CMakeLists.txt b/source/libs/scalar/test/filter/CMakeLists.txt new file mode 100644 index 0000000000..b738285ded --- /dev/null +++ b/source/libs/scalar/test/filter/CMakeLists.txt @@ -0,0 +1,18 @@ + +MESSAGE(STATUS "build filter unit test") + +# GoogleTest requires at least C++11 +SET(CMAKE_CXX_STANDARD 11) +AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) + +ADD_EXECUTABLE(filterTest ${SOURCE_LIST}) +TARGET_LINK_LIBRARIES( + filterTest + PUBLIC os util common gtest qcom function nodes scalar +) + +TARGET_INCLUDE_DIRECTORIES( + filterTest + PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/scalar/" + PRIVATE "${CMAKE_SOURCE_DIR}/source/libs/scalar/inc" +) diff --git a/source/libs/scalar/test/filter/filterTests.cpp b/source/libs/scalar/test/filter/filterTests.cpp new file mode 100644 index 0000000000..dc45f14366 --- /dev/null +++ b/source/libs/scalar/test/filter/filterTests.cpp @@ -0,0 +1,1175 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include +#include + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wwrite-strings" +#pragma GCC diagnostic ignored "-Wunused-function" +#pragma GCC diagnostic ignored "-Wunused-variable" +#pragma GCC diagnostic ignored "-Wsign-compare" +#pragma GCC diagnostic ignored "-Wsign-compare" +#pragma GCC diagnostic ignored "-Wformat" +#pragma GCC diagnostic ignored "-Wint-to-pointer-cast" +#pragma GCC diagnostic ignored "-Wpointer-arith" + +#include "os.h" + +#include "taos.h" +#include "tdef.h" +#include "tvariant.h" +#include "tep.h" +#include "stub.h" +#include "addr_any.h" +#include "scalar.h" +#include "nodes.h" +#include "tlog.h" +#include "filter.h" + +namespace { + +int64_t flttLeftV = 21, flttRightV = 10; +double flttLeftVd = 21.0, flttRightVd = 10.0; + +void flttInitLogFile() { + const char *defaultLogFileNamePrefix = "taoslog"; + const int32_t maxLogFileNum = 10; + + tsAsyncLog = 0; + qDebugFlag = 159; + + if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) { + printf("failed to open log file in directory:%s\n", tsLogDir); + } +} + + +void flttMakeValueNode(SNode **pNode, int32_t dataType, void *value) { + SNode *node = nodesMakeNode(QUERY_NODE_VALUE); + SValueNode *vnode = (SValueNode *)node; + vnode->node.resType.type = dataType; + + if (IS_VAR_DATA_TYPE(dataType)) { + vnode->datum.p = (char *)malloc(varDataTLen(value)); + varDataCopy(vnode->datum.p, value); + vnode->node.resType.bytes = varDataLen(value); + } else { + vnode->node.resType.bytes = tDataTypes[dataType].bytes; + assignVal((char *)nodesGetValueFromNode(vnode), (const char *)value, 0, dataType); + } + + *pNode = (SNode *)vnode; +} + +void flttMakeColRefNode(SNode **pNode, SSDataBlock **block, int32_t dataType, int32_t dataBytes, int32_t rowNum, void *value) { + SNode *node = nodesMakeNode(QUERY_NODE_COLUMN); + SColumnNode *rnode = (SColumnNode *)node; + rnode->node.resType.type = dataType; + rnode->node.resType.bytes = dataBytes; + rnode->tupleId = 0; + + if (NULL == block) { + rnode->slotId = 2; + rnode->colId = 55; + *pNode = (SNode *)rnode; + + return; + } + + if (NULL == *block) { + SSDataBlock *res = (SSDataBlock *)calloc(1, sizeof(SSDataBlock)); + res->info.numOfCols = 3; + res->info.rows = rowNum; + res->pDataBlock = taosArrayInit(3, sizeof(SColumnInfoData)); + for (int32_t i = 0; i < 2; ++i) { + SColumnInfoData idata = {{0}}; + idata.info.type = TSDB_DATA_TYPE_NULL; + idata.info.bytes = 10; + idata.info.colId = 0; + + int32_t size = idata.info.bytes * rowNum; + idata.pData = (char *)calloc(1, size); + taosArrayPush(res->pDataBlock, &idata); + } + + SColumnInfoData idata = {{0}}; + idata.info.type = dataType; + idata.info.bytes = dataBytes; + idata.info.colId = 55; + idata.pData = (char *)value; + if (IS_VAR_DATA_TYPE(dataType)) { + idata.varmeta.offset = (int32_t *)calloc(rowNum, sizeof(int32_t)); + for (int32_t i = 0; i < rowNum; ++i) { + idata.varmeta.offset[i] = (dataBytes + VARSTR_HEADER_SIZE) * i; + } + } + taosArrayPush(res->pDataBlock, &idata); + + rnode->slotId = 2; + rnode->colId = 55; + + *block = res; + } else { + SSDataBlock *res = *block; + + int32_t idx = taosArrayGetSize(res->pDataBlock); + SColumnInfoData idata = {{0}}; + idata.info.type = dataType; + idata.info.bytes = dataBytes; + idata.info.colId = 55 + idx; + idata.pData = (char *)value; + taosArrayPush(res->pDataBlock, &idata); + + res->info.numOfCols++; + + rnode->slotId = idx; + rnode->colId = 55 + idx; + } + + *pNode = (SNode *)rnode; +} + +void flttMakeOpNode(SNode **pNode, EOperatorType opType, int32_t resType, SNode *pLeft, SNode *pRight) { + SNode *node = nodesMakeNode(QUERY_NODE_OPERATOR); + SOperatorNode *onode = (SOperatorNode *)node; + onode->node.resType.type = resType; + onode->node.resType.bytes = tDataTypes[resType].bytes; + + onode->opType = opType; + onode->pLeft = pLeft; + onode->pRight = pRight; + + *pNode = (SNode *)onode; +} + +void flttMakeLogicNode(SNode **pNode, ELogicConditionType opType, SNode **nodeList, int32_t nodeNum) { + SNode *node = nodesMakeNode(QUERY_NODE_LOGIC_CONDITION); + SLogicConditionNode *onode = (SLogicConditionNode *)node; + onode->condType = opType; + onode->node.resType.type = TSDB_DATA_TYPE_BOOL; + onode->node.resType.bytes = sizeof(bool); + + onode->pParameterList = nodesMakeList(); + for (int32_t i = 0; i < nodeNum; ++i) { + nodesListAppend(onode->pParameterList, nodeList[i]); + } + + *pNode = (SNode *)onode; +} + +void flttMakeLogicNodeFromList(SNode **pNode, ELogicConditionType opType, SNodeList *nodeList) { + SNode *node = nodesMakeNode(QUERY_NODE_LOGIC_CONDITION); + SLogicConditionNode *onode = (SLogicConditionNode *)node; + onode->condType = opType; + onode->node.resType.type = TSDB_DATA_TYPE_BOOL; + onode->node.resType.bytes = sizeof(bool); + + onode->pParameterList = nodeList; + + *pNode = (SNode *)onode; +} + +void flttMakeListNode(SNode **pNode, SNodeList *list, int32_t resType) { + SNode *node = nodesMakeNode(QUERY_NODE_NODE_LIST); + SNodeListNode *lnode = (SNodeListNode *)node; + lnode->dataType.type = resType; + lnode->pNodeList = list; + + *pNode = (SNode *)lnode; +} + + +} + +TEST(timerangeTest, greater) { + SNode *pcol = NULL, *pval = NULL, *opNode1 = NULL, *opNode2 = NULL, *logicNode = NULL; + bool eRes[5] = {false, false, true, true, true}; + SScalarParam res = {0}; + int64_t tsmall = 222, tbig = 333; + flttMakeColRefNode(&pcol, NULL, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), 0, NULL); + flttMakeValueNode(&pval, TSDB_DATA_TYPE_TIMESTAMP, &tsmall); + flttMakeOpNode(&opNode1, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pcol, pval); + + SFilterInfo *filter = NULL; + int32_t code = filterInitFromNode(opNode1, &filter, FLT_OPTION_NO_REWRITE|FLT_OPTION_TIMESTAMP); + ASSERT_EQ(code, 0); + STimeWindow win = {0}; + code = filterGetTimeRange(filter, &win); + ASSERT_EQ(code, 0); + ASSERT_EQ(win.skey, tsmall); + ASSERT_EQ(win.ekey, INT64_MAX); +} + +TEST(timerangeTest, greater_and_lower) { + SNode *pcol = NULL, *pval = NULL, *opNode1 = NULL, *opNode2 = NULL, *logicNode = NULL; + bool eRes[5] = {false, false, true, true, true}; + SScalarParam res = {0}; + int64_t tsmall = 222, tbig = 333; + flttMakeColRefNode(&pcol, NULL, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), 0, NULL); + flttMakeValueNode(&pval, TSDB_DATA_TYPE_TIMESTAMP, &tsmall); + flttMakeOpNode(&opNode1, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pcol, pval); + flttMakeColRefNode(&pcol, NULL, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), 0, NULL); + flttMakeValueNode(&pval, TSDB_DATA_TYPE_TIMESTAMP, &tbig); + flttMakeOpNode(&opNode2, OP_TYPE_LOWER_THAN, TSDB_DATA_TYPE_BOOL, pcol, pval); + SNode *list[2] = {0}; + list[0] = opNode1; + list[1] = opNode2; + + flttMakeLogicNode(&logicNode, LOGIC_COND_TYPE_AND, list, 2); + + SFilterInfo *filter = NULL; + int32_t code = filterInitFromNode(logicNode, &filter, FLT_OPTION_NO_REWRITE|FLT_OPTION_TIMESTAMP); + ASSERT_EQ(code, 0); + STimeWindow win = {0}; + code = filterGetTimeRange(filter, &win); + ASSERT_EQ(code, 0); + ASSERT_EQ(win.skey, tsmall); + ASSERT_EQ(win.ekey, tbig); +} + + +TEST(columnTest, smallint_column_greater_double_value) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL; + int16_t leftv[5]= {1, 2, 3, 4, 5}; + double rightv= 2.5; + int8_t eRes[5] = {0, 0, 1, 1, 1}; + SSDataBlock *src = NULL; + SScalarParam res = {0}; + int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); + flttMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv); + flttMakeValueNode(&pRight, TSDB_DATA_TYPE_DOUBLE, &rightv); + flttMakeOpNode(&opNode, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pLeft, pRight); + + SFilterInfo *filter = NULL; + int32_t code = filterInitFromNode(opNode, &filter, 0); + ASSERT_EQ(code, 0); + + SColumnDataAgg stat = {0}; + stat.colId = ((SColumnNode *)pLeft)->colId; + stat.max = 10; + stat.min = 5; + stat.numOfNull = 0; + bool keep = filterRangeExecute(filter, &stat, 1, rowNum); + ASSERT_EQ(keep, true); + + stat.max = 1; + stat.min = -1; + keep = filterRangeExecute(filter, &stat, 1, rowNum); + ASSERT_EQ(keep, true); + + stat.max = 10; + stat.min = 5; + stat.numOfNull = rowNum; + keep = filterRangeExecute(filter, &stat, 1, rowNum); + ASSERT_EQ(keep, true); + + SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + code = filterSetDataFromSlotId(filter, ¶m); + ASSERT_EQ(code, 0); + + stat.max = 5; + stat.min = 1; + stat.numOfNull = 0; + int8_t *rowRes = NULL; + keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + ASSERT_EQ(keep, false); + + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); + } +} + +TEST(columnTest, int_column_greater_smallint_value) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL; + int32_t leftv[5]= {1, 3, 5, 7, 9}; + int16_t rightv= 4; + int8_t eRes[5] = {0, 0, 1, 1, 1}; + SSDataBlock *src = NULL; + SScalarParam res = {0}; + int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); + flttMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, leftv); + flttMakeValueNode(&pRight, TSDB_DATA_TYPE_SMALLINT, &rightv); + flttMakeOpNode(&opNode, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pLeft, pRight); + + SFilterInfo *filter = NULL; + int32_t code = filterInitFromNode(opNode, &filter, 0); + ASSERT_EQ(code, 0); + + SColumnDataAgg stat = {0}; + stat.colId = ((SColumnNode *)pLeft)->colId; + stat.max = 10; + stat.min = 5; + stat.numOfNull = 0; + bool keep = filterRangeExecute(filter, &stat, 1, rowNum); + ASSERT_EQ(keep, true); + + stat.max = 1; + stat.min = -1; + keep = filterRangeExecute(filter, &stat, 1, rowNum); + ASSERT_EQ(keep, false); + + stat.max = 10; + stat.min = 5; + stat.numOfNull = rowNum; + keep = filterRangeExecute(filter, &stat, 1, rowNum); + ASSERT_EQ(keep, false); + + SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + code = filterSetDataFromSlotId(filter, ¶m); + ASSERT_EQ(code, 0); + + stat.max = 5; + stat.min = 1; + stat.numOfNull = 0; + int8_t *rowRes = NULL; + keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + ASSERT_EQ(keep, false); + + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); + } +} + + +TEST(columnTest, int_column_in_double_list) { + SNode *pLeft = NULL, *pRight = NULL, *listNode = NULL, *opNode = NULL; + int32_t leftv[5] = {1, 2, 3, 4, 5}; + double rightv1 = 1.1,rightv2 = 2.2,rightv3 = 3.3; + bool eRes[5] = {true, true, true, false, false}; + SSDataBlock *src = NULL; + SScalarParam res = {0}; + int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); + flttMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, leftv); + SNodeList* list = nodesMakeList(); + flttMakeValueNode(&pRight, TSDB_DATA_TYPE_DOUBLE, &rightv1); + nodesListAppend(list, pRight); + flttMakeValueNode(&pRight, TSDB_DATA_TYPE_DOUBLE, &rightv2); + nodesListAppend(list, pRight); + flttMakeValueNode(&pRight, TSDB_DATA_TYPE_DOUBLE, &rightv3); + nodesListAppend(list, pRight); + flttMakeListNode(&listNode,list, TSDB_DATA_TYPE_INT); + flttMakeOpNode(&opNode, OP_TYPE_IN, TSDB_DATA_TYPE_BOOL, pLeft, listNode); + + SFilterInfo *filter = NULL; + int32_t code = filterInitFromNode(opNode, &filter, 0); + ASSERT_EQ(code, 0); + + SColumnDataAgg stat = {0}; + SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + code = filterSetDataFromSlotId(filter, ¶m); + ASSERT_EQ(code, 0); + + stat.max = 5; + stat.min = 1; + stat.numOfNull = 0; + int8_t *rowRes = NULL; + bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + ASSERT_EQ(keep, false); + + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); + } + + +} + + + +TEST(columnTest, binary_column_in_binary_list) { + SNode *pLeft = NULL, *pRight = NULL, *listNode = NULL, *opNode = NULL; + bool eRes[5] = {true, true, false, false, false}; + SSDataBlock *src = NULL; + SScalarParam res = {0}; + char leftv[5][5]= {0}; + char rightv[3][5]= {0}; + for (int32_t i = 0; i < 5; ++i) { + leftv[i][2] = 'a' + i; + leftv[i][3] = 'b' + i; + leftv[i][4] = '0' + i; + varDataSetLen(leftv[i], 3); + } + for (int32_t i = 0; i < 2; ++i) { + rightv[i][2] = 'a' + i; + rightv[i][3] = 'b' + i; + rightv[i][4] = '0' + i; + varDataSetLen(rightv[i], 3); + } + for (int32_t i = 2; i < 3; ++i) { + rightv[i][2] = 'a' + i; + rightv[i][3] = 'a' + i; + rightv[i][4] = 'a' + i; + varDataSetLen(rightv[i], 3); + } + + int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); + flttMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); + SNodeList* list = nodesMakeList(); + flttMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, rightv[0]); + nodesListAppend(list, pRight); + flttMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, rightv[1]); + nodesListAppend(list, pRight); + flttMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, rightv[2]); + nodesListAppend(list, pRight); + flttMakeListNode(&listNode,list, TSDB_DATA_TYPE_BINARY); + flttMakeOpNode(&opNode, OP_TYPE_IN, TSDB_DATA_TYPE_BOOL, pLeft, listNode); + + SFilterInfo *filter = NULL; + int32_t code = filterInitFromNode(opNode, &filter, 0); + ASSERT_EQ(code, 0); + + SColumnDataAgg stat = {0}; + SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + code = filterSetDataFromSlotId(filter, ¶m); + ASSERT_EQ(code, 0); + + stat.max = 5; + stat.min = 1; + stat.numOfNull = 0; + int8_t *rowRes = NULL; + bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + ASSERT_EQ(keep, false); + + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); + } +} + + +TEST(columnTest, binary_column_like_binary) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL; + char rightv[64] = {0}; + char leftv[5][5]= {0}; + SSDataBlock *src = NULL; + SScalarParam res = {0}; + bool eRes[5] = {true, false, true, false, true}; + + for (int32_t i = 0; i < 5; ++i) { + leftv[i][2] = 'a'; + leftv[i][3] = 'a'; + leftv[i][4] = '0' + i % 2; + varDataSetLen(leftv[i], 3); + } + + int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); + flttMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); + + sprintf(&rightv[2], "%s", "__0"); + varDataSetLen(rightv, strlen(&rightv[2])); + flttMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, rightv); + flttMakeOpNode(&opNode, OP_TYPE_LIKE, TSDB_DATA_TYPE_BOOL, pLeft, pRight); + + SFilterInfo *filter = NULL; + int32_t code = filterInitFromNode(opNode, &filter, 0); + ASSERT_EQ(code, 0); + + SColumnDataAgg stat = {0}; + SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + code = filterSetDataFromSlotId(filter, ¶m); + ASSERT_EQ(code, 0); + + stat.max = 5; + stat.min = 1; + stat.numOfNull = 0; + int8_t *rowRes = NULL; + bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + ASSERT_EQ(keep, false); + + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); + } +} + + +TEST(columnTest, binary_column_is_null) { + SNode *pLeft = NULL, *opNode = NULL; + char leftv[5][5]= {0}; + SSDataBlock *src = NULL; + SScalarParam res = {0}; + bool eRes[5] = {false, false, false, false, true}; + + for (int32_t i = 0; i < 4; ++i) { + leftv[i][2] = '0' + i % 2; + leftv[i][3] = 'a'; + leftv[i][4] = '0' + i % 2; + varDataSetLen(leftv[i], 3); + } + + setVardataNull(leftv[4], TSDB_DATA_TYPE_BINARY); + + int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); + flttMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); + + flttMakeOpNode(&opNode, OP_TYPE_IS_NULL, TSDB_DATA_TYPE_BOOL, pLeft, NULL); + + SFilterInfo *filter = NULL; + int32_t code = filterInitFromNode(opNode, &filter, 0); + ASSERT_EQ(code, 0); + + SColumnDataAgg stat = {0}; + SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + code = filterSetDataFromSlotId(filter, ¶m); + ASSERT_EQ(code, 0); + + stat.max = 5; + stat.min = 1; + stat.numOfNull = 0; + int8_t *rowRes = NULL; + bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + ASSERT_EQ(keep, false); + + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); + } +} + +TEST(columnTest, binary_column_is_not_null) { + SNode *pLeft = NULL, *opNode = NULL; + char leftv[5][5]= {0}; + SSDataBlock *src = NULL; + SScalarParam res = {0}; + bool eRes[5] = {true, true, true, true, false}; + + for (int32_t i = 0; i < 4; ++i) { + leftv[i][2] = '0' + i % 2; + leftv[i][3] = 'a'; + leftv[i][4] = '0' + i % 2; + varDataSetLen(leftv[i], 3); + } + + setVardataNull(leftv[4], TSDB_DATA_TYPE_BINARY); + + int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); + flttMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); + + flttMakeOpNode(&opNode, OP_TYPE_IS_NOT_NULL, TSDB_DATA_TYPE_BOOL, pLeft, NULL); + + SFilterInfo *filter = NULL; + int32_t code = filterInitFromNode(opNode, &filter, 0); + ASSERT_EQ(code, 0); + + SColumnDataAgg stat = {0}; + SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + code = filterSetDataFromSlotId(filter, ¶m); + ASSERT_EQ(code, 0); + + stat.max = 5; + stat.min = 1; + stat.numOfNull = 0; + int8_t *rowRes = NULL; + bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + ASSERT_EQ(keep, false); + + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); + } +} + + + +TEST(opTest, smallint_column_greater_int_column) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL; + int16_t leftv[5] = {1, -6, -2, 11, 101}; + int32_t rightv[5]= {0, -5, -4, 23, 100}; + bool eRes[5] = {true, false, true, false, true}; + SSDataBlock *src = NULL; + SScalarParam res = {0}; + int32_t rowNum = sizeof(rightv)/sizeof(rightv[0]); + flttMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv); + flttMakeColRefNode(&pRight, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, rightv); + flttMakeOpNode(&opNode, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pLeft, pRight); + + SFilterInfo *filter = NULL; + int32_t code = filterInitFromNode(opNode, &filter, 0); + ASSERT_EQ(code, 0); + + SColumnDataAgg stat = {0}; + SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + code = filterSetDataFromSlotId(filter, ¶m); + ASSERT_EQ(code, 0); + + stat.max = 5; + stat.min = 1; + stat.numOfNull = 0; + int8_t *rowRes = NULL; + bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + ASSERT_EQ(keep, false); + + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); + } +} + + +TEST(opTest, smallint_value_add_int_column) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL; + int32_t leftv = 1; + int16_t rightv[5]= {0, -1, -4, -1, 100}; + bool eRes[5] = {true, false, true, false, true}; + SSDataBlock *src = NULL; + SScalarParam res = {0}; + int32_t rowNum = sizeof(rightv)/sizeof(rightv[0]); + flttMakeValueNode(&pLeft, TSDB_DATA_TYPE_INT, &leftv); + flttMakeColRefNode(&pRight, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, rightv); + flttMakeOpNode(&opNode, OP_TYPE_ADD, TSDB_DATA_TYPE_DOUBLE, pLeft, pRight); + flttMakeOpNode(&opNode, OP_TYPE_IS_TRUE, TSDB_DATA_TYPE_BOOL, opNode, NULL); + + SFilterInfo *filter = NULL; + int32_t code = filterInitFromNode(opNode, &filter, 0); + ASSERT_EQ(code, 0); + + SColumnDataAgg stat = {0}; + SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + code = filterSetDataFromSlotId(filter, ¶m); + ASSERT_EQ(code, 0); + + stat.max = 5; + stat.min = 1; + stat.numOfNull = 0; + int8_t *rowRes = NULL; + bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + ASSERT_EQ(keep, false); + + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); + } +} + + + +TEST(opTest, bigint_column_multi_binary_column) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL; + int64_t leftv[5]= {1, 2, 3, 4, 5}; + char rightv[5][5]= {0}; + for (int32_t i = 0; i < 5; ++i) { + rightv[i][2] = rightv[i][3] = '0'; + rightv[i][4] = '0' + i; + varDataSetLen(rightv[i], 3); + } + bool eRes[5] = {false, true, true, true, true}; + SSDataBlock *src = NULL; + SScalarParam res = {0}; + int32_t rowNum = sizeof(rightv)/sizeof(rightv[0]); + flttMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_BIGINT, sizeof(int64_t), rowNum, leftv); + flttMakeColRefNode(&pRight, &src, TSDB_DATA_TYPE_BINARY, 5, rowNum, rightv); + flttMakeOpNode(&opNode, OP_TYPE_MULTI, TSDB_DATA_TYPE_DOUBLE, pLeft, pRight); + flttMakeOpNode(&opNode, OP_TYPE_IS_TRUE, TSDB_DATA_TYPE_BOOL, opNode, NULL); + + SFilterInfo *filter = NULL; + int32_t code = filterInitFromNode(opNode, &filter, 0); + ASSERT_EQ(code, 0); + + SColumnDataAgg stat = {0}; + SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + code = filterSetDataFromSlotId(filter, ¶m); + ASSERT_EQ(code, 0); + + stat.max = 5; + stat.min = 1; + stat.numOfNull = 0; + int8_t *rowRes = NULL; + bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + ASSERT_EQ(keep, false); + + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); + } +} + +TEST(opTest, smallint_column_and_binary_column) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL; + int16_t leftv[5]= {1, 2, 3, 4, 5}; + char rightv[5][5]= {0}; + for (int32_t i = 0; i < 5; ++i) { + rightv[i][2] = rightv[i][3] = '0'; + rightv[i][4] = '0' + i; + varDataSetLen(rightv[i], 3); + } + bool eRes[5] = {false, false, true, false, true}; + SSDataBlock *src = NULL; + SScalarParam res = {0}; + int32_t rowNum = sizeof(rightv)/sizeof(rightv[0]); + flttMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv); + flttMakeColRefNode(&pRight, &src, TSDB_DATA_TYPE_BINARY, 5, rowNum, rightv); + flttMakeOpNode(&opNode, OP_TYPE_BIT_AND, TSDB_DATA_TYPE_BIGINT, pLeft, pRight); + flttMakeOpNode(&opNode, OP_TYPE_IS_TRUE, TSDB_DATA_TYPE_BOOL, opNode, NULL); + + SFilterInfo *filter = NULL; + int32_t code = filterInitFromNode(opNode, &filter, 0); + ASSERT_EQ(code, 0); + + SColumnDataAgg stat = {0}; + SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + code = filterSetDataFromSlotId(filter, ¶m); + ASSERT_EQ(code, 0); + + stat.max = 5; + stat.min = 1; + stat.numOfNull = 0; + int8_t *rowRes = NULL; + bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + ASSERT_EQ(keep, false); + + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); + } +} + +TEST(opTest, smallint_column_or_float_column) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL; + int16_t leftv[5]= {1, 2, 0, 4, 5}; + float rightv[5]= {2.0, 3.0, 0, 5.2, 6.0}; + bool eRes[5] = {true, true, false, true, true}; + SSDataBlock *src = NULL; + SScalarParam res = {0}; + int32_t rowNum = sizeof(rightv)/sizeof(rightv[0]); + flttMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv); + flttMakeColRefNode(&pRight, &src, TSDB_DATA_TYPE_FLOAT, sizeof(float), rowNum, rightv); + flttMakeOpNode(&opNode, OP_TYPE_BIT_OR, TSDB_DATA_TYPE_BIGINT, pLeft, pRight); + flttMakeOpNode(&opNode, OP_TYPE_IS_TRUE, TSDB_DATA_TYPE_BOOL, opNode, NULL); + + SFilterInfo *filter = NULL; + int32_t code = filterInitFromNode(opNode, &filter, 0); + ASSERT_EQ(code, 0); + + SColumnDataAgg stat = {0}; + SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + code = filterSetDataFromSlotId(filter, ¶m); + ASSERT_EQ(code, 0); + + stat.max = 5; + stat.min = 1; + stat.numOfNull = 0; + int8_t *rowRes = NULL; + bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + ASSERT_EQ(keep, false); + + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); + } +} + + + +TEST(opTest, smallint_column_or_double_value) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL; + int16_t leftv[5]= {0, 2, 3, 0, -1}; + double rightv= 10.2; + bool eRes[5] = {true, true, true, true, true}; + SSDataBlock *src = NULL; + SScalarParam res = {0}; + int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); + flttMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv); + flttMakeValueNode(&pRight, TSDB_DATA_TYPE_DOUBLE, &rightv); + flttMakeOpNode(&opNode, OP_TYPE_BIT_OR, TSDB_DATA_TYPE_BIGINT, pLeft, pRight); + flttMakeOpNode(&opNode, OP_TYPE_IS_TRUE, TSDB_DATA_TYPE_BOOL, opNode, NULL); + + SFilterInfo *filter = NULL; + int32_t code = filterInitFromNode(opNode, &filter, 0); + ASSERT_EQ(code, 0); + + SColumnDataAgg stat = {0}; + SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + code = filterSetDataFromSlotId(filter, ¶m); + ASSERT_EQ(code, 0); + + stat.max = 5; + stat.min = 1; + stat.numOfNull = 0; + int8_t *rowRes = NULL; + bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + ASSERT_EQ(keep, true); + + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); + } +} + + +TEST(opTest, binary_column_is_true) { + SNode *pLeft = NULL, *opNode = NULL; + char leftv[5][5]= {0}; + SSDataBlock *src = NULL; + SScalarParam res = {0}; + bool eRes[5] = {false, true, false, true, false}; + + for (int32_t i = 0; i < 5; ++i) { + leftv[i][2] = '0' + i % 2; + leftv[i][3] = 'a'; + leftv[i][4] = '0' + i % 2; + varDataSetLen(leftv[i], 3); + } + + int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); + flttMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); + + flttMakeOpNode(&opNode, OP_TYPE_IS_TRUE, TSDB_DATA_TYPE_BOOL, pLeft, NULL); + + SFilterInfo *filter = NULL; + int32_t code = filterInitFromNode(opNode, &filter, 0); + ASSERT_EQ(code, 0); + + SColumnDataAgg stat = {0}; + SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + code = filterSetDataFromSlotId(filter, ¶m); + ASSERT_EQ(code, 0); + + stat.max = 5; + stat.min = 1; + stat.numOfNull = 0; + int8_t *rowRes = NULL; + bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + ASSERT_EQ(keep, false); + + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); + } +} + + +TEST(filterModelogicTest, diff_columns_and_or_and) { + SNode *pLeft1 = NULL, *pRight1 = NULL, *pLeft2 = NULL, *pRight2 = NULL, *opNode1 = NULL, *opNode2 = NULL; + SNode *logicNode1 = NULL, *logicNode2 = NULL; + double leftv1[8]= {1, 2, 3, 4, 5,-1,-2,-3}, leftv2[8]= {3.0, 4, 2, 9, -3, 3.9, 4.1, 5.2}; + int32_t rightv1= 3, rightv2= 3; + int8_t eRes[8] = {1, 1, 0, 0, 1, 1, 1, 1}; + SSDataBlock *src = NULL; + + SNodeList* list = nodesMakeList(); + + int32_t rowNum = sizeof(leftv1)/sizeof(leftv1[0]); + flttMakeColRefNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1); + flttMakeValueNode(&pRight1, TSDB_DATA_TYPE_INT, &rightv1); + flttMakeOpNode(&opNode1, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pLeft1, pRight1); + nodesListAppend(list, opNode1); + + flttMakeColRefNode(&pLeft2, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv2); + flttMakeValueNode(&pRight2, TSDB_DATA_TYPE_INT, &rightv2); + flttMakeOpNode(&opNode2, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft2, pRight2); + nodesListAppend(list, opNode2); + + flttMakeLogicNodeFromList(&logicNode1, LOGIC_COND_TYPE_AND, list); + + + list = nodesMakeList(); + + flttMakeColRefNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1); + flttMakeValueNode(&pRight1, TSDB_DATA_TYPE_INT, &rightv1); + flttMakeOpNode(&opNode1, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft1, pRight1); + nodesListAppend(list, opNode1); + + flttMakeColRefNode(&pLeft2, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv2); + flttMakeValueNode(&pRight2, TSDB_DATA_TYPE_INT, &rightv2); + flttMakeOpNode(&opNode2, OP_TYPE_GREATER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft2, pRight2); + nodesListAppend(list, opNode2); + + flttMakeLogicNodeFromList(&logicNode2, LOGIC_COND_TYPE_AND, list); + + list = nodesMakeList(); + nodesListAppend(list, logicNode1); + nodesListAppend(list, logicNode2); + flttMakeLogicNodeFromList(&logicNode1, LOGIC_COND_TYPE_OR, list); + + + SFilterInfo *filter = NULL; + int32_t code = filterInitFromNode(logicNode1, &filter, 0); + ASSERT_EQ(code, 0); + + SColumnDataAgg stat = {0}; + SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + code = filterSetDataFromSlotId(filter, ¶m); + ASSERT_EQ(code, 0); + + stat.max = 5; + stat.min = 1; + stat.numOfNull = 0; + int8_t *rowRes = NULL; + bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + ASSERT_EQ(keep, false); + + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); + } +} + +TEST(filterModelogicTest, same_column_and_or_and) { + SNode *pLeft1 = NULL, *pRight1 = NULL, *pLeft2 = NULL, *pRight2 = NULL, *opNode1 = NULL, *opNode2 = NULL; + SNode *logicNode1 = NULL, *logicNode2 = NULL; + double leftv1[8]= {1, 2, 3, 4, 5,-1,-2,-3}; + int32_t rightv1= 3, rightv2= 0, rightv3 = 2, rightv4 = -2; + int8_t eRes[8] = {1, 1, 0, 0, 0, 1, 1, 0}; + SSDataBlock *src = NULL; + + SNodeList* list = nodesMakeList(); + + int32_t rowNum = sizeof(leftv1)/sizeof(leftv1[0]); + flttMakeColRefNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1); + flttMakeValueNode(&pRight1, TSDB_DATA_TYPE_INT, &rightv1); + flttMakeOpNode(&opNode1, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pLeft1, pRight1); + nodesListAppend(list, opNode1); + + flttMakeValueNode(&pRight2, TSDB_DATA_TYPE_INT, &rightv2); + flttMakeOpNode(&opNode2, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft1, pRight2); + nodesListAppend(list, opNode2); + + flttMakeLogicNodeFromList(&logicNode1, LOGIC_COND_TYPE_AND, list); + + + list = nodesMakeList(); + + flttMakeValueNode(&pRight1, TSDB_DATA_TYPE_INT, &rightv3); + flttMakeOpNode(&opNode1, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft1, pRight1); + nodesListAppend(list, opNode1); + + flttMakeValueNode(&pRight2, TSDB_DATA_TYPE_INT, &rightv4); + flttMakeOpNode(&opNode2, OP_TYPE_GREATER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft1, pRight2); + nodesListAppend(list, opNode2); + + flttMakeLogicNodeFromList(&logicNode2, LOGIC_COND_TYPE_AND, list); + + list = nodesMakeList(); + nodesListAppend(list, logicNode1); + nodesListAppend(list, logicNode2); + flttMakeLogicNodeFromList(&logicNode1, LOGIC_COND_TYPE_OR, list); + + + SFilterInfo *filter = NULL; + int32_t code = filterInitFromNode(logicNode1, &filter, 0); + ASSERT_EQ(code, 0); + + SColumnDataAgg stat = {0}; + SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + code = filterSetDataFromSlotId(filter, ¶m); + ASSERT_EQ(code, 0); + + stat.max = 5; + stat.min = 1; + stat.numOfNull = 0; + int8_t *rowRes = NULL; + bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + ASSERT_EQ(keep, false); + + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); + } +} + + +TEST(filterModelogicTest, diff_columns_or_and_or) { + SNode *pLeft1 = NULL, *pRight1 = NULL, *pLeft2 = NULL, *pRight2 = NULL, *opNode1 = NULL, *opNode2 = NULL; + SNode *logicNode1 = NULL, *logicNode2 = NULL; + double leftv1[8]= {1, 2, 3, 4, 5,-1,-2,-3}, leftv2[8]= {3.0, 4, 2, 9, -3, 3.9, 4.1, 5.2}; + int32_t rightv1= 3, rightv2= 3; + int8_t eRes[8] = {1, 0, 1, 1, 0, 0, 0, 0}; + SSDataBlock *src = NULL; + + SNodeList* list = nodesMakeList(); + + int32_t rowNum = sizeof(leftv1)/sizeof(leftv1[0]); + flttMakeColRefNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1); + flttMakeValueNode(&pRight1, TSDB_DATA_TYPE_INT, &rightv1); + flttMakeOpNode(&opNode1, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pLeft1, pRight1); + nodesListAppend(list, opNode1); + + flttMakeColRefNode(&pLeft2, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv2); + flttMakeValueNode(&pRight2, TSDB_DATA_TYPE_INT, &rightv2); + flttMakeOpNode(&opNode2, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft2, pRight2); + nodesListAppend(list, opNode2); + + flttMakeLogicNodeFromList(&logicNode1, LOGIC_COND_TYPE_OR, list); + + + list = nodesMakeList(); + + flttMakeColRefNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1); + flttMakeValueNode(&pRight1, TSDB_DATA_TYPE_INT, &rightv1); + flttMakeOpNode(&opNode1, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft1, pRight1); + nodesListAppend(list, opNode1); + + flttMakeColRefNode(&pLeft2, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv2); + flttMakeValueNode(&pRight2, TSDB_DATA_TYPE_INT, &rightv2); + flttMakeOpNode(&opNode2, OP_TYPE_GREATER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft2, pRight2); + nodesListAppend(list, opNode2); + + flttMakeLogicNodeFromList(&logicNode2, LOGIC_COND_TYPE_OR, list); + + list = nodesMakeList(); + nodesListAppend(list, logicNode1); + nodesListAppend(list, logicNode2); + flttMakeLogicNodeFromList(&logicNode1, LOGIC_COND_TYPE_AND, list); + + + SFilterInfo *filter = NULL; + int32_t code = filterInitFromNode(logicNode1, &filter, 0); + ASSERT_EQ(code, 0); + + SColumnDataAgg stat = {0}; + SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + code = filterSetDataFromSlotId(filter, ¶m); + ASSERT_EQ(code, 0); + + stat.max = 5; + stat.min = 1; + stat.numOfNull = 0; + int8_t *rowRes = NULL; + bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + ASSERT_EQ(keep, false); + + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); + } +} + +TEST(filterModelogicTest, same_column_or_and_or) { + SNode *pLeft1 = NULL, *pRight1 = NULL, *pLeft2 = NULL, *pRight2 = NULL, *opNode1 = NULL, *opNode2 = NULL; + SNode *logicNode1 = NULL, *logicNode2 = NULL; + double leftv1[8]= {1, 2, 3, 4, 5,-1,-2,-3}; + int32_t rightv1= 3, rightv2= 0, rightv3 = 2, rightv4 = -2; + int8_t eRes[8] = {0, 0, 0, 1, 1, 1, 1, 1}; + SSDataBlock *src = NULL; + + SNodeList* list = nodesMakeList(); + + int32_t rowNum = sizeof(leftv1)/sizeof(leftv1[0]); + flttMakeColRefNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1); + flttMakeValueNode(&pRight1, TSDB_DATA_TYPE_INT, &rightv1); + flttMakeOpNode(&opNode1, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pLeft1, pRight1); + nodesListAppend(list, opNode1); + + flttMakeValueNode(&pRight2, TSDB_DATA_TYPE_INT, &rightv2); + flttMakeOpNode(&opNode2, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft1, pRight2); + nodesListAppend(list, opNode2); + + flttMakeLogicNodeFromList(&logicNode1, LOGIC_COND_TYPE_OR, list); + + + list = nodesMakeList(); + + flttMakeValueNode(&pRight1, TSDB_DATA_TYPE_INT, &rightv3); + flttMakeOpNode(&opNode1, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft1, pRight1); + nodesListAppend(list, opNode1); + + flttMakeValueNode(&pRight2, TSDB_DATA_TYPE_INT, &rightv4); + flttMakeOpNode(&opNode2, OP_TYPE_GREATER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft1, pRight2); + nodesListAppend(list, opNode2); + + flttMakeLogicNodeFromList(&logicNode2, LOGIC_COND_TYPE_OR, list); + + list = nodesMakeList(); + nodesListAppend(list, logicNode1); + nodesListAppend(list, logicNode2); + flttMakeLogicNodeFromList(&logicNode1, LOGIC_COND_TYPE_AND, list); + + + SFilterInfo *filter = NULL; + int32_t code = filterInitFromNode(logicNode1, &filter, 0); + ASSERT_EQ(code, 0); + + SColumnDataAgg stat = {0}; + SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + code = filterSetDataFromSlotId(filter, ¶m); + ASSERT_EQ(code, 0); + + stat.max = 5; + stat.min = 1; + stat.numOfNull = 0; + int8_t *rowRes = NULL; + bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + ASSERT_EQ(keep, false); + + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); + } +} + + + +TEST(scalarModelogicTest, diff_columns_or_and_or) { + flttInitLogFile(); + + SNode *pLeft1 = NULL, *pRight1 = NULL, *pLeft2 = NULL, *pRight2 = NULL, *opNode1 = NULL, *opNode2 = NULL; + SNode *logicNode1 = NULL, *logicNode2 = NULL; + double leftv1[8] = {1, 2, 3, 4, 5,-1,-2,-3}, leftv2[8]= {3.0, 4, 2, 9, -3, 3.9, 4.1, 5.2}; + int32_t rightv1[8]= {5, 8, 2, -3, 9,-7,10, 0}, rightv2[8]= {-3, 5, 8, 2, -9, 11, -4, 0}; + int8_t eRes[8] = {0, 1, 1, 0, 0, 1, 0, 0}; + SSDataBlock *src = NULL; + + SNodeList* list = nodesMakeList(); + + int32_t rowNum = sizeof(leftv1)/sizeof(leftv1[0]); + flttMakeColRefNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1); + flttMakeColRefNode(&pRight1, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, rightv1); + flttMakeOpNode(&opNode1, OP_TYPE_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft1, pRight1); + nodesListAppend(list, opNode1); + + flttMakeColRefNode(&pLeft2, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv2); + flttMakeColRefNode(&pRight2, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, rightv2); + flttMakeOpNode(&opNode2, OP_TYPE_LOWER_THAN, TSDB_DATA_TYPE_BOOL, pLeft2, pRight2); + nodesListAppend(list, opNode2); + + flttMakeLogicNodeFromList(&logicNode1, LOGIC_COND_TYPE_OR, list); + + + list = nodesMakeList(); + + flttMakeColRefNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1); + flttMakeColRefNode(&pRight1, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, rightv1); + flttMakeOpNode(&opNode1, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pLeft1, pRight1); + nodesListAppend(list, opNode1); + + flttMakeColRefNode(&pLeft2, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv2); + flttMakeColRefNode(&pRight2, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, rightv2); + flttMakeOpNode(&opNode2, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft2, pRight2); + nodesListAppend(list, opNode2); + + flttMakeLogicNodeFromList(&logicNode2, LOGIC_COND_TYPE_OR, list); + + list = nodesMakeList(); + nodesListAppend(list, logicNode1); + nodesListAppend(list, logicNode2); + flttMakeLogicNodeFromList(&logicNode1, LOGIC_COND_TYPE_AND, list); + + + SFilterInfo *filter = NULL; + int32_t code = filterInitFromNode(logicNode1, &filter, 0); + ASSERT_EQ(code, 0); + + SColumnDataAgg stat = {0}; + SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + code = filterSetDataFromSlotId(filter, ¶m); + ASSERT_EQ(code, 0); + + stat.max = 5; + stat.min = 1; + stat.numOfNull = 0; + int8_t *rowRes = NULL; + bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + ASSERT_EQ(keep, false); + + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); + } +} + + + +int main(int argc, char** argv) { + srand(time(NULL)); + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + +#pragma GCC diagnostic pop diff --git a/source/libs/scalar/test/scalar/CMakeLists.txt b/source/libs/scalar/test/scalar/CMakeLists.txt new file mode 100644 index 0000000000..9dfbf3b5e8 --- /dev/null +++ b/source/libs/scalar/test/scalar/CMakeLists.txt @@ -0,0 +1,18 @@ + +MESSAGE(STATUS "build scalar unit test") + +# GoogleTest requires at least C++11 +SET(CMAKE_CXX_STANDARD 11) +AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) + +ADD_EXECUTABLE(scalarTest ${SOURCE_LIST}) +TARGET_LINK_LIBRARIES( + scalarTest + PUBLIC os util common gtest qcom function nodes scalar +) + +TARGET_INCLUDE_DIRECTORIES( + scalarTest + PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/scalar/" + PRIVATE "${CMAKE_SOURCE_DIR}/source/libs/scalar/inc" +) diff --git a/source/libs/scalar/test/scalar/scalarTests.cpp b/source/libs/scalar/test/scalar/scalarTests.cpp new file mode 100644 index 0000000000..df746e19cf --- /dev/null +++ b/source/libs/scalar/test/scalar/scalarTests.cpp @@ -0,0 +1,1181 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include +#include + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wwrite-strings" +#pragma GCC diagnostic ignored "-Wunused-function" +#pragma GCC diagnostic ignored "-Wunused-variable" +#pragma GCC diagnostic ignored "-Wsign-compare" +#pragma GCC diagnostic ignored "-Wsign-compare" +#pragma GCC diagnostic ignored "-Wformat" +#pragma GCC diagnostic ignored "-Wint-to-pointer-cast" +#pragma GCC diagnostic ignored "-Wpointer-arith" + +#include "os.h" + +#include "taos.h" +#include "tdef.h" +#include "tvariant.h" +#include "tep.h" +#include "stub.h" +#include "addr_any.h" +#include "scalar.h" +#include "nodes.h" +#include "tlog.h" + +namespace { + +int64_t scltLeftV = 21, scltRightV = 10; +double scltLeftVd = 21.0, scltRightVd = 10.0; + +void scltInitLogFile() { + const char *defaultLogFileNamePrefix = "taoslog"; + const int32_t maxLogFileNum = 10; + + tsAsyncLog = 0; + qDebugFlag = 159; + + if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) { + printf("failed to open log file in directory:%s\n", tsLogDir); + } +} + + +void scltMakeValueNode(SNode **pNode, int32_t dataType, void *value) { + SNode *node = nodesMakeNode(QUERY_NODE_VALUE); + SValueNode *vnode = (SValueNode *)node; + vnode->node.resType.type = dataType; + + if (IS_VAR_DATA_TYPE(dataType)) { + vnode->datum.p = (char *)malloc(varDataTLen(value)); + varDataCopy(vnode->datum.p, value); + vnode->node.resType.bytes = varDataLen(value); + } else { + vnode->node.resType.bytes = tDataTypes[dataType].bytes; + assignVal((char *)nodesGetValueFromNode(vnode), (const char *)value, 0, dataType); + } + + *pNode = (SNode *)vnode; +} + +void scltMakeColRefNode(SNode **pNode, SSDataBlock **block, int32_t dataType, int32_t dataBytes, int32_t rowNum, void *value) { + SNode *node = nodesMakeNode(QUERY_NODE_COLUMN); + SColumnNode *rnode = (SColumnNode *)node; + rnode->node.resType.type = dataType; + rnode->node.resType.bytes = dataBytes; + rnode->tupleId = 0; + + if (NULL == *block) { + SSDataBlock *res = (SSDataBlock *)calloc(1, sizeof(SSDataBlock)); + res->info.numOfCols = 3; + res->info.rows = rowNum; + res->pDataBlock = taosArrayInit(3, sizeof(SColumnInfoData)); + for (int32_t i = 0; i < 2; ++i) { + SColumnInfoData idata = {{0}}; + idata.info.type = TSDB_DATA_TYPE_NULL; + idata.info.bytes = 10; + idata.info.colId = 0; + + int32_t size = idata.info.bytes * rowNum; + idata.pData = (char *)calloc(1, size); + taosArrayPush(res->pDataBlock, &idata); + } + + SColumnInfoData idata = {{0}}; + idata.info.type = dataType; + idata.info.bytes = dataBytes; + idata.info.colId = 55; + idata.pData = (char *)value; + if (IS_VAR_DATA_TYPE(dataType)) { + idata.varmeta.offset = (int32_t *)calloc(rowNum, sizeof(int32_t)); + for (int32_t i = 0; i < rowNum; ++i) { + idata.varmeta.offset[i] = (dataBytes + VARSTR_HEADER_SIZE) * i; + } + } + taosArrayPush(res->pDataBlock, &idata); + + rnode->slotId = 2; + rnode->colId = 55; + + *block = res; + } else { + SSDataBlock *res = *block; + + int32_t idx = taosArrayGetSize(res->pDataBlock); + SColumnInfoData idata = {{0}}; + idata.info.type = dataType; + idata.info.bytes = dataBytes; + idata.info.colId = 55 + idx; + idata.pData = (char *)value; + taosArrayPush(res->pDataBlock, &idata); + + rnode->slotId = idx; + rnode->colId = 55 + idx; + } + + *pNode = (SNode *)rnode; +} + +void scltMakeOpNode(SNode **pNode, EOperatorType opType, int32_t resType, SNode *pLeft, SNode *pRight) { + SNode *node = nodesMakeNode(QUERY_NODE_OPERATOR); + SOperatorNode *onode = (SOperatorNode *)node; + onode->node.resType.type = resType; + onode->node.resType.bytes = tDataTypes[resType].bytes; + + onode->opType = opType; + onode->pLeft = pLeft; + onode->pRight = pRight; + + *pNode = (SNode *)onode; +} + + +void scltMakeListNode(SNode **pNode, SNodeList *list, int32_t resType) { + SNode *node = nodesMakeNode(QUERY_NODE_NODE_LIST); + SNodeListNode *lnode = (SNodeListNode *)node; + lnode->dataType.type = resType; + lnode->pNodeList = list; + + *pNode = (SNode *)lnode; +} + + +void scltMakeLogicNode(SNode **pNode, ELogicConditionType opType, SNode **nodeList, int32_t nodeNum) { + SNode *node = nodesMakeNode(QUERY_NODE_LOGIC_CONDITION); + SLogicConditionNode *onode = (SLogicConditionNode *)node; + onode->condType = opType; + onode->node.resType.type = TSDB_DATA_TYPE_BOOL; + onode->node.resType.bytes = sizeof(bool); + + onode->pParameterList = nodesMakeList(); + for (int32_t i = 0; i < nodeNum; ++i) { + nodesListAppend(onode->pParameterList, nodeList[i]); + } + + *pNode = (SNode *)onode; +} + + +} + +TEST(constantTest, bigint_add_bigint) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_BIGINT, &scltLeftV); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_BIGINT, &scltRightV); + scltMakeOpNode(&opNode, OP_TYPE_ADD, TSDB_DATA_TYPE_DOUBLE, pLeft, pRight); + + int32_t code = scalarCalculateConstants(opNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_DOUBLE); + ASSERT_EQ(v->datum.d, (scltLeftV + scltRightV)); +} + +TEST(constantTest, double_sub_bigint) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_DOUBLE, &scltLeftVd); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_BIGINT, &scltRightV); + scltMakeOpNode(&opNode, OP_TYPE_SUB, TSDB_DATA_TYPE_DOUBLE, pLeft, pRight); + + int32_t code = scalarCalculateConstants(opNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_DOUBLE); + ASSERT_EQ(v->datum.d, (scltLeftVd - scltRightV)); +} + +TEST(constantTest, tinyint_and_smallint) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_TINYINT, &scltLeftV); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_SMALLINT, &scltRightV); + scltMakeOpNode(&opNode, OP_TYPE_BIT_AND, TSDB_DATA_TYPE_BIGINT, pLeft, pRight); + + int32_t code = scalarCalculateConstants(opNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BIGINT); + ASSERT_EQ(v->datum.i, (int64_t)scltLeftV & (int64_t)scltRightV); +} + +TEST(constantTest, bigint_or_double) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_BIGINT, &scltLeftV); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_DOUBLE, &scltRightVd); + scltMakeOpNode(&opNode, OP_TYPE_BIT_OR, TSDB_DATA_TYPE_BIGINT, pLeft, pRight); + + int32_t code = scalarCalculateConstants(opNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BIGINT); + ASSERT_EQ(v->datum.i, (int64_t)scltLeftV | (int64_t)scltRightVd); +} + +TEST(constantTest, int_or_binary) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; + char binaryStr[64] = {0}; + sprintf(&binaryStr[2], "%d", scltRightV); + varDataSetLen(binaryStr, strlen(&binaryStr[2])); + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_INT, &scltLeftV); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, binaryStr); + scltMakeOpNode(&opNode, OP_TYPE_BIT_OR, TSDB_DATA_TYPE_BIGINT, pLeft, pRight); + + int32_t code = scalarCalculateConstants(opNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BIGINT); + ASSERT_EQ(v->datum.b, scltLeftV | scltRightV); +} + + +TEST(constantTest, int_greater_double) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_INT, &scltLeftV); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_DOUBLE, &scltRightVd); + scltMakeOpNode(&opNode, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pLeft, pRight); + + int32_t code = scalarCalculateConstants(opNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(v->datum.b, scltLeftV > scltRightVd); +} + +TEST(constantTest, int_greater_equal_binary) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; + char binaryStr[64] = {0}; + sprintf(&binaryStr[2], "%d", scltRightV); + varDataSetLen(binaryStr, strlen(&binaryStr[2])); + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_INT, &scltLeftV); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, binaryStr); + scltMakeOpNode(&opNode, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pLeft, pRight); + + int32_t code = scalarCalculateConstants(opNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(v->datum.b, scltLeftV > scltRightVd); +} + +TEST(constantTest, tinyint_lower_ubigint) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_TINYINT, &scltLeftV); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_UBIGINT, &scltRightV); + scltMakeOpNode(&opNode, OP_TYPE_LOWER_THAN, TSDB_DATA_TYPE_BOOL, pLeft, pRight); + + int32_t code = scalarCalculateConstants(opNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(v->datum.b, scltLeftV < scltRightV); +} + +TEST(constantTest, usmallint_lower_equal_ubigint) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; + int32_t leftv = 1, rightv = 1; + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_USMALLINT, &leftv); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_UBIGINT, &rightv); + scltMakeOpNode(&opNode, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft, pRight); + + int32_t code = scalarCalculateConstants(opNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(v->datum.b, leftv <= rightv); +} + +TEST(constantTest, int_equal_smallint1) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; + int32_t leftv = 1, rightv = 1; + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_INT, &leftv); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_SMALLINT, &rightv); + scltMakeOpNode(&opNode, OP_TYPE_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft, pRight); + + int32_t code = scalarCalculateConstants(opNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(v->datum.b, leftv == rightv); +} + +TEST(constantTest, int_equal_smallint2) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; + int32_t leftv = 0, rightv = 1; + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_INT, &leftv); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_SMALLINT, &rightv); + scltMakeOpNode(&opNode, OP_TYPE_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft, pRight); + + int32_t code = scalarCalculateConstants(opNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(v->datum.b, leftv == rightv); +} + +TEST(constantTest, int_not_equal_smallint1) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; + int32_t leftv = 1, rightv = 1; + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_INT, &leftv); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_SMALLINT, &rightv); + scltMakeOpNode(&opNode, OP_TYPE_NOT_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft, pRight); + + int32_t code = scalarCalculateConstants(opNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(v->datum.b, leftv != rightv); +} + +TEST(constantTest, int_not_equal_smallint2) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; + int32_t leftv = 0, rightv = 1; + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_INT, &leftv); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_SMALLINT, &rightv); + scltMakeOpNode(&opNode, OP_TYPE_NOT_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft, pRight); + + int32_t code = scalarCalculateConstants(opNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(v->datum.b, leftv != rightv); +} + + + +TEST(constantTest, int_in_smallint1) { + scltInitLogFile(); + + SNode *pLeft = NULL, *pRight = NULL, *listNode = NULL, *res = NULL, *opNode = NULL; + int32_t leftv = 1, rightv1 = 1,rightv2 = 2,rightv3 = 3; + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_INT, &leftv); + SNodeList* list = nodesMakeList(); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_SMALLINT, &rightv1); + nodesListAppend(list, pRight); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_SMALLINT, &rightv2); + nodesListAppend(list, pRight); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_SMALLINT, &rightv3); + nodesListAppend(list, pRight); + scltMakeListNode(&listNode,list, TSDB_DATA_TYPE_INT); + scltMakeOpNode(&opNode, OP_TYPE_IN, TSDB_DATA_TYPE_BOOL, pLeft, listNode); + + int32_t code = scalarCalculateConstants(opNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(v->datum.b, true); +} + +TEST(constantTest, int_in_smallint2) { + scltInitLogFile(); + + SNode *pLeft = NULL, *pRight = NULL, *listNode = NULL, *res = NULL, *opNode = NULL; + int32_t leftv = 4, rightv1 = 1,rightv2 = 2,rightv3 = 3; + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_INT, &leftv); + SNodeList* list = nodesMakeList(); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_SMALLINT, &rightv1); + nodesListAppend(list, pRight); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_SMALLINT, &rightv2); + nodesListAppend(list, pRight); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_SMALLINT, &rightv3); + nodesListAppend(list, pRight); + scltMakeListNode(&listNode,list, TSDB_DATA_TYPE_INT); + scltMakeOpNode(&opNode, OP_TYPE_IN, TSDB_DATA_TYPE_BOOL, pLeft, listNode); + + int32_t code = scalarCalculateConstants(opNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(v->datum.b, false); +} + +TEST(constantTest, int_not_in_smallint1) { + SNode *pLeft = NULL, *pRight = NULL, *listNode = NULL, *res = NULL, *opNode = NULL; + int32_t leftv = 1, rightv1 = 1,rightv2 = 2,rightv3 = 3; + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_INT, &leftv); + SNodeList* list = nodesMakeList(); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_SMALLINT, &rightv1); + nodesListAppend(list, pRight); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_SMALLINT, &rightv2); + nodesListAppend(list, pRight); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_SMALLINT, &rightv3); + nodesListAppend(list, pRight); + scltMakeListNode(&listNode,list, TSDB_DATA_TYPE_INT); + scltMakeOpNode(&opNode, OP_TYPE_NOT_IN, TSDB_DATA_TYPE_BOOL, pLeft, listNode); + + int32_t code = scalarCalculateConstants(opNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(v->datum.b, false); +} + +TEST(constantTest, int_not_in_smallint2) { + scltInitLogFile(); + + SNode *pLeft = NULL, *pRight = NULL, *listNode = NULL, *res = NULL, *opNode = NULL; + int32_t leftv = 4, rightv1 = 1,rightv2 = 2,rightv3 = 3; + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_INT, &leftv); + SNodeList* list = nodesMakeList(); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_SMALLINT, &rightv1); + nodesListAppend(list, pRight); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_SMALLINT, &rightv2); + nodesListAppend(list, pRight); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_SMALLINT, &rightv3); + nodesListAppend(list, pRight); + scltMakeListNode(&listNode,list, TSDB_DATA_TYPE_INT); + scltMakeOpNode(&opNode, OP_TYPE_NOT_IN, TSDB_DATA_TYPE_BOOL, pLeft, listNode); + + int32_t code = scalarCalculateConstants(opNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(v->datum.b, true); +} + +TEST(constantTest, binary_like_binary1) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; + char leftv[64] = {0}, rightv[64] = {0}; + sprintf(&leftv[2], "%s", "abc"); + varDataSetLen(leftv, strlen(&leftv[2])); + sprintf(&rightv[2], "%s", "a_c"); + varDataSetLen(rightv, strlen(&rightv[2])); + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_BINARY, leftv); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, rightv); + scltMakeOpNode(&opNode, OP_TYPE_LIKE, TSDB_DATA_TYPE_BOOL, pLeft, pRight); + + int32_t code = scalarCalculateConstants(opNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(v->datum.b, true); +} + +TEST(constantTest, binary_like_binary2) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; + char leftv[64] = {0}, rightv[64] = {0}; + sprintf(&leftv[2], "%s", "abc"); + varDataSetLen(leftv, strlen(&leftv[2])); + sprintf(&rightv[2], "%s", "ac"); + varDataSetLen(rightv, strlen(&rightv[2])); + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_BINARY, leftv); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, rightv); + scltMakeOpNode(&opNode, OP_TYPE_LIKE, TSDB_DATA_TYPE_BOOL, pLeft, pRight); + + int32_t code = scalarCalculateConstants(opNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(v->datum.b, false); +} + +TEST(constantTest, binary_not_like_binary1) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; + char leftv[64] = {0}, rightv[64] = {0}; + sprintf(&leftv[2], "%s", "abc"); + varDataSetLen(leftv, strlen(&leftv[2])); + sprintf(&rightv[2], "%s", "a%c"); + varDataSetLen(rightv, strlen(&rightv[2])); + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_BINARY, leftv); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, rightv); + scltMakeOpNode(&opNode, OP_TYPE_NOT_LIKE, TSDB_DATA_TYPE_BOOL, pLeft, pRight); + + int32_t code = scalarCalculateConstants(opNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(v->datum.b, false); +} + +TEST(constantTest, binary_not_like_binary2) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; + char leftv[64] = {0}, rightv[64] = {0}; + sprintf(&leftv[2], "%s", "abc"); + varDataSetLen(leftv, strlen(&leftv[2])); + sprintf(&rightv[2], "%s", "ac"); + varDataSetLen(rightv, strlen(&rightv[2])); + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_BINARY, leftv); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, rightv); + scltMakeOpNode(&opNode, OP_TYPE_NOT_LIKE, TSDB_DATA_TYPE_BOOL, pLeft, pRight); + + int32_t code = scalarCalculateConstants(opNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(v->datum.b, true); +} + +TEST(constantTest, binary_match_binary1) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; + char leftv[64] = {0}, rightv[64] = {0}; + sprintf(&leftv[2], "%s", "abc"); + varDataSetLen(leftv, strlen(&leftv[2])); + sprintf(&rightv[2], "%s", ".*"); + varDataSetLen(rightv, strlen(&rightv[2])); + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_BINARY, leftv); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, rightv); + scltMakeOpNode(&opNode, OP_TYPE_MATCH, TSDB_DATA_TYPE_BOOL, pLeft, pRight); + + int32_t code = scalarCalculateConstants(opNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(v->datum.b, true); +} + +TEST(constantTest, binary_match_binary2) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; + char leftv[64] = {0}, rightv[64] = {0}; + sprintf(&leftv[2], "%s", "abc"); + varDataSetLen(leftv, strlen(&leftv[2])); + sprintf(&rightv[2], "%s", "abc.+"); + varDataSetLen(rightv, strlen(&rightv[2])); + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_BINARY, leftv); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, rightv); + scltMakeOpNode(&opNode, OP_TYPE_MATCH, TSDB_DATA_TYPE_BOOL, pLeft, pRight); + + int32_t code = scalarCalculateConstants(opNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(v->datum.b, false); +} + +TEST(constantTest, binary_not_match_binary1) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; + char leftv[64] = {0}, rightv[64] = {0}; + sprintf(&leftv[2], "%s", "abc"); + varDataSetLen(leftv, strlen(&leftv[2])); + sprintf(&rightv[2], "%s", "a[1-9]c"); + varDataSetLen(rightv, strlen(&rightv[2])); + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_BINARY, leftv); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, rightv); + scltMakeOpNode(&opNode, OP_TYPE_NMATCH, TSDB_DATA_TYPE_BOOL, pLeft, pRight); + + int32_t code = scalarCalculateConstants(opNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(v->datum.b, true); +} + +TEST(constantTest, binary_not_match_binary2) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; + char leftv[64] = {0}, rightv[64] = {0}; + sprintf(&leftv[2], "%s", "abc"); + varDataSetLen(leftv, strlen(&leftv[2])); + sprintf(&rightv[2], "%s", "a[ab]c"); + varDataSetLen(rightv, strlen(&rightv[2])); + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_BINARY, leftv); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, rightv); + scltMakeOpNode(&opNode, OP_TYPE_NMATCH, TSDB_DATA_TYPE_BOOL, pLeft, pRight); + + int32_t code = scalarCalculateConstants(opNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(v->datum.b, false); +} + +TEST(constantTest, int_is_null1) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; + int32_t leftv = 1, rightv = 1; + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_INT, &leftv); + scltMakeOpNode(&opNode, OP_TYPE_IS_NULL, TSDB_DATA_TYPE_BOOL, pLeft, pRight); + + int32_t code = scalarCalculateConstants(opNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(v->datum.b, false); +} + +TEST(constantTest, int_is_null2) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; + int32_t leftv = TSDB_DATA_INT_NULL, rightv = 1; + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_INT, &leftv); + scltMakeOpNode(&opNode, OP_TYPE_IS_NULL, TSDB_DATA_TYPE_BOOL, pLeft, pRight); + + int32_t code = scalarCalculateConstants(opNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(v->datum.b, true); +} + +TEST(constantTest, int_is_not_null1) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; + int32_t leftv = 1, rightv = 1; + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_INT, &leftv); + scltMakeOpNode(&opNode, OP_TYPE_IS_NOT_NULL, TSDB_DATA_TYPE_BOOL, pLeft, pRight); + + int32_t code = scalarCalculateConstants(opNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(v->datum.b, true); +} + +TEST(constantTest, int_is_not_null2) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; + int32_t leftv = TSDB_DATA_INT_NULL, rightv = 1; + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_INT, &leftv); + scltMakeOpNode(&opNode, OP_TYPE_IS_NOT_NULL, TSDB_DATA_TYPE_BOOL, pLeft, pRight); + + int32_t code = scalarCalculateConstants(opNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(v->datum.b, false); +} + +TEST(constantTest, int_add_int_is_true1) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; + int32_t leftv = 1, rightv = 1; + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_INT, &leftv); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_INT, &rightv); + scltMakeOpNode(&opNode, OP_TYPE_ADD, TSDB_DATA_TYPE_DOUBLE, pLeft, pRight); + scltMakeOpNode(&opNode, OP_TYPE_IS_TRUE, TSDB_DATA_TYPE_BOOL, opNode, NULL); + + int32_t code = scalarCalculateConstants(opNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(v->datum.b, true); +} + +TEST(constantTest, int_add_int_is_true2) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; + int32_t leftv = 1, rightv = -1; + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_INT, &leftv); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_INT, &rightv); + scltMakeOpNode(&opNode, OP_TYPE_ADD, TSDB_DATA_TYPE_DOUBLE, pLeft, pRight); + scltMakeOpNode(&opNode, OP_TYPE_IS_TRUE, TSDB_DATA_TYPE_BOOL, opNode, NULL); + + int32_t code = scalarCalculateConstants(opNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(v->datum.b, false); +} + + +TEST(constantTest, int_greater_int_is_true1) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; + int32_t leftv = 1, rightv = 1; + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_INT, &leftv); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_INT, &rightv); + scltMakeOpNode(&opNode, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pLeft, pRight); + scltMakeOpNode(&opNode, OP_TYPE_IS_TRUE, TSDB_DATA_TYPE_BOOL, opNode, NULL); + + int32_t code = scalarCalculateConstants(opNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(v->datum.b, false); +} + +TEST(constantTest, int_greater_int_is_true2) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; + int32_t leftv = 1, rightv = 0; + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_INT, &leftv); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_INT, &rightv); + scltMakeOpNode(&opNode, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pLeft, pRight); + scltMakeOpNode(&opNode, OP_TYPE_IS_TRUE, TSDB_DATA_TYPE_BOOL, opNode, NULL); + + int32_t code = scalarCalculateConstants(opNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(v->datum.b, true); +} + +TEST(constantTest, greater_and_lower) { + SNode *pval1 = NULL, *pval2 = NULL, *opNode1 = NULL, *opNode2 = NULL, *logicNode = NULL, *res = NULL; + bool eRes[5] = {false, false, true, true, true}; + int64_t v1 = 333, v2 = 222, v3 = -10, v4 = 20; + SNode *list[2] = {0}; + scltMakeValueNode(&pval1, TSDB_DATA_TYPE_BIGINT, &v1); + scltMakeValueNode(&pval2, TSDB_DATA_TYPE_BIGINT, &v2); + scltMakeOpNode(&opNode1, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pval1, pval2); + scltMakeValueNode(&pval1, TSDB_DATA_TYPE_BIGINT, &v3); + scltMakeValueNode(&pval2, TSDB_DATA_TYPE_BIGINT, &v4); + scltMakeOpNode(&opNode2, OP_TYPE_LOWER_THAN, TSDB_DATA_TYPE_BOOL, pval1, pval2); + list[0] = opNode1; + list[1] = opNode2; + scltMakeLogicNode(&logicNode, LOGIC_COND_TYPE_AND, list, 2); + + int32_t code = scalarCalculateConstants(logicNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(v->datum.b, true); +} + + + +TEST(columnTest, smallint_value_add_int_column) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL; + int32_t leftv = 1; + int16_t rightv[5]= {0, -5, -4, 23, 100}; + double eRes[5] = {1.0, -4, -3, 24, 101}; + SSDataBlock *src = NULL; + SScalarParam res = {0}; + int32_t rowNum = sizeof(rightv)/sizeof(rightv[0]); + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_INT, &leftv); + scltMakeColRefNode(&pRight, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, rightv); + scltMakeOpNode(&opNode, OP_TYPE_ADD, TSDB_DATA_TYPE_DOUBLE, pLeft, pRight); + + int32_t code = scalarCalculate(opNode, src, &res); + ASSERT_EQ(code, 0); + ASSERT_EQ(res.num, rowNum); + ASSERT_EQ(res.type, TSDB_DATA_TYPE_DOUBLE); + ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((double *)res.data + i), eRes[i]); + } +} + +TEST(columnTest, bigint_column_multi_binary_column) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL; + int64_t leftv[5]= {1, 2, 3, 4, 5}; + char rightv[5][5]= {0}; + for (int32_t i = 0; i < 5; ++i) { + rightv[i][2] = rightv[i][3] = '0'; + rightv[i][4] = '0' + i; + varDataSetLen(rightv[i], 3); + } + double eRes[5] = {0, 2, 6, 12, 20}; + SSDataBlock *src = NULL; + SScalarParam res = {0}; + int32_t rowNum = sizeof(rightv)/sizeof(rightv[0]); + scltMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_BIGINT, sizeof(int64_t), rowNum, leftv); + scltMakeColRefNode(&pRight, &src, TSDB_DATA_TYPE_BINARY, 5, rowNum, rightv); + scltMakeOpNode(&opNode, OP_TYPE_MULTI, TSDB_DATA_TYPE_DOUBLE, pLeft, pRight); + + int32_t code = scalarCalculate(opNode, src, &res); + ASSERT_EQ(code, 0); + ASSERT_EQ(res.num, rowNum); + ASSERT_EQ(res.type, TSDB_DATA_TYPE_DOUBLE); + ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((double *)res.data + i), eRes[i]); + } +} + +TEST(columnTest, smallint_column_and_binary_column) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL; + int16_t leftv[5]= {1, 2, 3, 4, 5}; + char rightv[5][5]= {0}; + for (int32_t i = 0; i < 5; ++i) { + rightv[i][2] = rightv[i][3] = '0'; + rightv[i][4] = '0' + i; + varDataSetLen(rightv[i], 3); + } + int64_t eRes[5] = {0, 0, 2, 0, 4}; + SSDataBlock *src = NULL; + SScalarParam res = {0}; + int32_t rowNum = sizeof(rightv)/sizeof(rightv[0]); + scltMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv); + scltMakeColRefNode(&pRight, &src, TSDB_DATA_TYPE_BINARY, 5, rowNum, rightv); + scltMakeOpNode(&opNode, OP_TYPE_BIT_AND, TSDB_DATA_TYPE_BIGINT, pLeft, pRight); + + int32_t code = scalarCalculate(opNode, src, &res); + ASSERT_EQ(code, 0); + ASSERT_EQ(res.num, rowNum); + ASSERT_EQ(res.type, TSDB_DATA_TYPE_BIGINT); + ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((int64_t *)res.data + i), eRes[i]); + } +} + +TEST(columnTest, smallint_column_or_float_column) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL; + int16_t leftv[5]= {1, 2, 3, 4, 5}; + float rightv[5]= {2.0, 3.0, 4.1, 5.2, 6.0}; + int64_t eRes[5] = {3, 3, 7, 5, 7}; + SSDataBlock *src = NULL; + SScalarParam res = {0}; + int32_t rowNum = sizeof(rightv)/sizeof(rightv[0]); + scltMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv); + scltMakeColRefNode(&pRight, &src, TSDB_DATA_TYPE_FLOAT, sizeof(float), rowNum, rightv); + scltMakeOpNode(&opNode, OP_TYPE_BIT_OR, TSDB_DATA_TYPE_BIGINT, pLeft, pRight); + + int32_t code = scalarCalculate(opNode, src, &res); + ASSERT_EQ(code, 0); + ASSERT_EQ(res.num, rowNum); + ASSERT_EQ(res.type, TSDB_DATA_TYPE_BIGINT); + ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((int64_t *)res.data + i), eRes[i]); + } +} + +TEST(columnTest, smallint_column_or_double_value) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL; + int16_t leftv[5]= {1, 2, 3, 4, 5}; + double rightv= 10.2; + int64_t eRes[5] = {11, 10, 11, 14, 15}; + SSDataBlock *src = NULL; + SScalarParam res = {0}; + int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); + scltMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_DOUBLE, &rightv); + scltMakeOpNode(&opNode, OP_TYPE_BIT_OR, TSDB_DATA_TYPE_BIGINT, pLeft, pRight); + + int32_t code = scalarCalculate(opNode, src, &res); + ASSERT_EQ(code, 0); + ASSERT_EQ(res.num, rowNum); + ASSERT_EQ(res.type, TSDB_DATA_TYPE_BIGINT); + ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((int64_t *)res.data + i), eRes[i]); + } +} + +TEST(columnTest, smallint_column_greater_double_value) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL; + int16_t leftv[5]= {1, 2, 3, 4, 5}; + double rightv= 2.5; + bool eRes[5] = {false, false, true, true, true}; + SSDataBlock *src = NULL; + SScalarParam res = {0}; + int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); + scltMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_DOUBLE, &rightv); + scltMakeOpNode(&opNode, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pLeft, pRight); + + int32_t code = scalarCalculate(opNode, src, &res); + ASSERT_EQ(code, 0); + ASSERT_EQ(res.num, rowNum); + ASSERT_EQ(res.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_BOOL].bytes); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((bool *)res.data + i), eRes[i]); + } +} + +TEST(columnTest, int_column_in_double_list) { + SNode *pLeft = NULL, *pRight = NULL, *listNode = NULL, *opNode = NULL; + int32_t leftv[5] = {1, 2, 3, 4, 5}; + double rightv1 = 1.1,rightv2 = 2.2,rightv3 = 3.3; + bool eRes[5] = {true, true, true, false, false}; + SSDataBlock *src = NULL; + SScalarParam res = {0}; + int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); + scltMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, leftv); + SNodeList* list = nodesMakeList(); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_DOUBLE, &rightv1); + nodesListAppend(list, pRight); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_DOUBLE, &rightv2); + nodesListAppend(list, pRight); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_DOUBLE, &rightv3); + nodesListAppend(list, pRight); + scltMakeListNode(&listNode,list, TSDB_DATA_TYPE_INT); + scltMakeOpNode(&opNode, OP_TYPE_IN, TSDB_DATA_TYPE_BOOL, pLeft, listNode); + + int32_t code = scalarCalculate(opNode, src, &res); + ASSERT_EQ(code, 0); + ASSERT_EQ(res.num, rowNum); + ASSERT_EQ(res.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_BOOL].bytes); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((bool *)res.data + i), eRes[i]); + } +} + +TEST(columnTest, binary_column_in_binary_list) { + SNode *pLeft = NULL, *pRight = NULL, *listNode = NULL, *opNode = NULL; + bool eRes[5] = {true, true, false, false, false}; + SSDataBlock *src = NULL; + SScalarParam res = {0}; + char leftv[5][5]= {0}; + char rightv[3][5]= {0}; + for (int32_t i = 0; i < 5; ++i) { + leftv[i][2] = 'a' + i; + leftv[i][3] = 'b' + i; + leftv[i][4] = '0' + i; + varDataSetLen(leftv[i], 3); + } + for (int32_t i = 0; i < 2; ++i) { + rightv[i][2] = 'a' + i; + rightv[i][3] = 'b' + i; + rightv[i][4] = '0' + i; + varDataSetLen(rightv[i], 3); + } + for (int32_t i = 2; i < 3; ++i) { + rightv[i][2] = 'a' + i; + rightv[i][3] = 'a' + i; + rightv[i][4] = 'a' + i; + varDataSetLen(rightv[i], 3); + } + + int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); + scltMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); + SNodeList* list = nodesMakeList(); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, rightv[0]); + nodesListAppend(list, pRight); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, rightv[1]); + nodesListAppend(list, pRight); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, rightv[2]); + nodesListAppend(list, pRight); + scltMakeListNode(&listNode,list, TSDB_DATA_TYPE_BINARY); + scltMakeOpNode(&opNode, OP_TYPE_IN, TSDB_DATA_TYPE_BOOL, pLeft, listNode); + + int32_t code = scalarCalculate(opNode, src, &res); + ASSERT_EQ(code, 0); + ASSERT_EQ(res.num, rowNum); + ASSERT_EQ(res.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_BOOL].bytes); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((bool *)res.data + i), eRes[i]); + } +} + +TEST(columnTest, binary_column_like_binary) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL; + char rightv[64] = {0}; + char leftv[5][5]= {0}; + SSDataBlock *src = NULL; + SScalarParam res = {0}; + bool eRes[5] = {true, false, true, false, true}; + + for (int32_t i = 0; i < 5; ++i) { + leftv[i][2] = 'a'; + leftv[i][3] = 'a'; + leftv[i][4] = '0' + i % 2; + varDataSetLen(leftv[i], 3); + } + + int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); + scltMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); + + sprintf(&rightv[2], "%s", "__0"); + varDataSetLen(rightv, strlen(&rightv[2])); + scltMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, rightv); + scltMakeOpNode(&opNode, OP_TYPE_LIKE, TSDB_DATA_TYPE_BOOL, pLeft, pRight); + + int32_t code = scalarCalculate(opNode, src, &res); + ASSERT_EQ(code, 0); + ASSERT_EQ(res.num, rowNum); + ASSERT_EQ(res.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_BOOL].bytes); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((bool *)res.data + i), eRes[i]); + } +} + +TEST(columnTest, binary_column_is_true) { + SNode *pLeft = NULL, *opNode = NULL; + char leftv[5][5]= {0}; + SSDataBlock *src = NULL; + SScalarParam res = {0}; + bool eRes[5] = {false, true, false, true, false}; + + for (int32_t i = 0; i < 5; ++i) { + leftv[i][2] = '0' + i % 2; + leftv[i][3] = 'a'; + leftv[i][4] = '0' + i % 2; + varDataSetLen(leftv[i], 3); + } + + int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); + scltMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); + + scltMakeOpNode(&opNode, OP_TYPE_IS_TRUE, TSDB_DATA_TYPE_BOOL, pLeft, NULL); + + int32_t code = scalarCalculate(opNode, src, &res); + ASSERT_EQ(code, 0); + ASSERT_EQ(res.num, rowNum); + ASSERT_EQ(res.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_BOOL].bytes); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((bool *)res.data + i), eRes[i]); + } +} + +TEST(columnTest, binary_column_is_null) { + SNode *pLeft = NULL, *opNode = NULL; + char leftv[5][5]= {0}; + SSDataBlock *src = NULL; + SScalarParam res = {0}; + bool eRes[5] = {false, false, false, false, true}; + + for (int32_t i = 0; i < 4; ++i) { + leftv[i][2] = '0' + i % 2; + leftv[i][3] = 'a'; + leftv[i][4] = '0' + i % 2; + varDataSetLen(leftv[i], 3); + } + + setVardataNull(leftv[4], TSDB_DATA_TYPE_BINARY); + + int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); + scltMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); + + scltMakeOpNode(&opNode, OP_TYPE_IS_NULL, TSDB_DATA_TYPE_BOOL, pLeft, NULL); + + int32_t code = scalarCalculate(opNode, src, &res); + ASSERT_EQ(code, 0); + ASSERT_EQ(res.num, rowNum); + ASSERT_EQ(res.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_BOOL].bytes); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((bool *)res.data + i), eRes[i]); + } +} + +TEST(columnTest, binary_column_is_not_null) { + SNode *pLeft = NULL, *opNode = NULL; + char leftv[5][5]= {0}; + SSDataBlock *src = NULL; + SScalarParam res = {0}; + bool eRes[5] = {true, true, true, true, false}; + + for (int32_t i = 0; i < 4; ++i) { + leftv[i][2] = '0' + i % 2; + leftv[i][3] = 'a'; + leftv[i][4] = '0' + i % 2; + varDataSetLen(leftv[i], 3); + } + + setVardataNull(leftv[4], TSDB_DATA_TYPE_BINARY); + + int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); + scltMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); + + scltMakeOpNode(&opNode, OP_TYPE_IS_NOT_NULL, TSDB_DATA_TYPE_BOOL, pLeft, NULL); + + int32_t code = scalarCalculate(opNode, src, &res); + ASSERT_EQ(code, 0); + ASSERT_EQ(res.num, rowNum); + ASSERT_EQ(res.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_BOOL].bytes); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((bool *)res.data + i), eRes[i]); + } +} + +TEST(columnTest, greater_and_lower) { + SNode *pcol1 = NULL, *pcol2 = NULL, *opNode1 = NULL, *opNode2 = NULL, *logicNode = NULL; + SNode *list[2] = {0}; + int16_t v1[5]= {1, 2, 3, 4, 5}; + int32_t v2[5]= {5, 1, 4, 2, 6}; + int64_t v3[5]= {1, 2, 3, 4, 5}; + int32_t v4[5]= {5, 3, 4, 2, 6}; + bool eRes[5] = {false, true, false, false, false}; + SSDataBlock *src = NULL; + SScalarParam res = {0}; + int32_t rowNum = sizeof(v1)/sizeof(v1[0]); + scltMakeColRefNode(&pcol1, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, v1); + scltMakeColRefNode(&pcol2, &src, TSDB_DATA_TYPE_INT, sizeof(int16_t), rowNum, v2); + scltMakeOpNode(&opNode1, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pcol1, pcol2); + scltMakeColRefNode(&pcol1, &src, TSDB_DATA_TYPE_BIGINT, sizeof(int16_t), rowNum, v3); + scltMakeColRefNode(&pcol2, &src, TSDB_DATA_TYPE_INT, sizeof(int16_t), rowNum, v4); + scltMakeOpNode(&opNode2, OP_TYPE_LOWER_THAN, TSDB_DATA_TYPE_BOOL, pcol1, pcol2); + list[0] = opNode1; + list[1] = opNode2; + scltMakeLogicNode(&logicNode, LOGIC_COND_TYPE_AND, list, 2); + + int32_t code = scalarCalculate(logicNode, src, &res); + ASSERT_EQ(code, 0); + ASSERT_EQ(res.num, rowNum); + ASSERT_EQ(res.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_BOOL].bytes); + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((bool *)res.data + i), eRes[i]); + } +} + + + +int main(int argc, char** argv) { + srand(time(NULL)); + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + +#pragma GCC diagnostic pop diff --git a/source/libs/scalar/test/scalarTests.cpp b/source/libs/scalar/test/scalarTests.cpp deleted file mode 100644 index 38fe072f9e..0000000000 --- a/source/libs/scalar/test/scalarTests.cpp +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include -#include -#include - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wwrite-strings" -#pragma GCC diagnostic ignored "-Wunused-function" -#pragma GCC diagnostic ignored "-Wunused-variable" -#pragma GCC diagnostic ignored "-Wsign-compare" -#pragma GCC diagnostic ignored "-Wsign-compare" -#pragma GCC diagnostic ignored "-Wformat" -#pragma GCC diagnostic ignored "-Wint-to-pointer-cast" -#pragma GCC diagnostic ignored "-Wpointer-arith" - -#include "os.h" - -#include "taos.h" -#include "tdef.h" -#include "tvariant.h" -#include "tep.h" -#include "stub.h" -#include "addr_any.h" -#include "scalar.h" - -namespace { - -} - -TEST(scalarTest, func) { - -} - - -int main(int argc, char** argv) { - srand(time(NULL)); - testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -} - -#pragma GCC diagnostic pop diff --git a/source/libs/scheduler/test/schedulerTests.cpp b/source/libs/scheduler/test/schedulerTests.cpp index 6bfff0197c..89d365a7e7 100644 --- a/source/libs/scheduler/test/schedulerTests.cpp +++ b/source/libs/scheduler/test/schedulerTests.cpp @@ -66,9 +66,7 @@ void schtInitLogFile() { tsAsyncLog = 0; qDebugFlag = 159; - char temp[128] = {0}; - sprintf(temp, "%s/%s", tsLogDir, defaultLogFileNamePrefix); - if (taosInitLog(temp, tsNumOfLogLines, maxLogFileNum) < 0) { + if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) { printf("failed to open log file in directory:%s\n", tsLogDir); } diff --git a/source/libs/tdb/src/db/tdbEnv.c b/source/libs/tdb/src/db/tdbEnv.c index 3670c770ab..a415d6d60e 100644 --- a/source/libs/tdb/src/db/tdbEnv.c +++ b/source/libs/tdb/src/db/tdbEnv.c @@ -18,7 +18,7 @@ struct STDbEnv { char * rootDir; // root directory of the environment char * jname; // journal file name - int jfd; // journal file fd + TdFilePtr jpFile; // journal file fd pgsz_t pgSize; // page size cachesz_t cacheSize; // total cache size STDbList dbList; // TDB List @@ -55,7 +55,7 @@ int tdbEnvCreate(TENV **ppEnv, const char *rootDir) { pEnv->rootDir = (char *)(&pEnv[1]); pEnv->jname = pEnv->rootDir + slen + 1; - pEnv->jfd = -1; + pEnv->jpFile = NULL; pEnv->pgSize = TDB_DEFAULT_PGSIZE; pEnv->cacheSize = TDB_DEFAULT_CACHE_SIZE; @@ -139,8 +139,8 @@ static int tdbEnvDestroy(TENV *pEnv) { } int tdbEnvBeginTxn(TENV *pEnv) { - pEnv->jfd = open(pEnv->jname, O_CREAT | O_RDWR, 0755); - if (pEnv->jfd < 0) { + pEnv->jpFile = taosOpenFile(pEnv->jname, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_READ); + if (pEnv->jpFile == NULL) { return -1; } @@ -149,8 +149,8 @@ int tdbEnvBeginTxn(TENV *pEnv) { int tdbEnvCommit(TENV *pEnv) { /* TODO */ - close(pEnv->jfd); - pEnv->jfd = -1; + taosCloseFile(&pEnv->jpFile); + pEnv->jpFile = NULL; return 0; } diff --git a/source/libs/tdb/src/db/tdbPgFile.c b/source/libs/tdb/src/db/tdbPgFile.c index ee5b486f7b..12f062ebf7 100644 --- a/source/libs/tdb/src/db/tdbPgFile.c +++ b/source/libs/tdb/src/db/tdbPgFile.c @@ -51,10 +51,10 @@ int pgFileOpen(SPgFile **ppPgFile, const char *fname, TENV *pEnv) { pPgFile->fname = (char *)(&(pPgFile[1])); memcpy(pPgFile->fname, fname, fnameLen); pPgFile->fname[fnameLen] = '\0'; - pPgFile->fd = -1; + pPgFile->pFile = NULL; - pPgFile->fd = open(fname, O_CREAT | O_RDWR, 0755); - if (pPgFile->fd < 0) { + pPgFile->pFile = taosOpenFile(fname, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_READ); + if (pPgFile->pFile == NULL) { // TODO: handle error return -1; } @@ -95,8 +95,8 @@ int pgFileOpen(SPgFile **ppPgFile, const char *fname, TENV *pEnv) { int pgFileClose(SPgFile *pPgFile) { if (pPgFile) { - if (pPgFile->fd >= 0) { - close(pPgFile->fd); + if (pPgFile->pFile != NULL) { + taosCloseFile(&pPgFile->pFile); } tfree(pPgFile->fname); @@ -201,7 +201,7 @@ static int pgFileRead(SPgFile *pPgFile, pgno_t pgno, uint8_t *pData) { pTData = pData; szToRead = pgSize; for (; szToRead > 0;) { - rsize = pread(pPgFile->fd, pTData, szToRead, pgno * pgSize); + rsize = pread(pPgFile->pFile, pTData, szToRead, pgno * pgSize); if (rsize < 0) { if (errno == EINTR) { continue; diff --git a/source/libs/tdb/src/inc/tdbPgFile.h b/source/libs/tdb/src/inc/tdbPgFile.h index 2a7116a0dd..eaeebf6b9d 100644 --- a/source/libs/tdb/src/inc/tdbPgFile.h +++ b/source/libs/tdb/src/inc/tdbPgFile.h @@ -38,7 +38,7 @@ struct SPgFile { uint8_t fileid[TDB_FILE_ID_LEN]; // file id pgno_t lsize; // page file logical size (for count) pgno_t fsize; // real file size on disk (for rollback) - int fd; + TdFilePtr pFile; SPgFileListNode envHash; SPgFileListNode envPgfList; }; diff --git a/source/libs/tfs/src/tfs.c b/source/libs/tfs/src/tfs.c index 9002879b10..f686703643 100644 --- a/source/libs/tfs/src/tfs.c +++ b/source/libs/tfs/src/tfs.c @@ -25,7 +25,7 @@ static int32_t tfsOpendirImpl(STfs *pTfs, STfsDir *pDir); static STfsDisk *tfsNextDisk(STfs *pTfs, SDiskIter *pIter); STfs *tfsOpen(SDiskCfg *pCfg, int32_t ndisk) { - if (ndisk < 0 || ndisk > TFS_MAX_DISKS) { + if (ndisk <= 0 || ndisk > TFS_MAX_DISKS) { terrno = TSDB_CODE_INVALID_PARA; return NULL; } diff --git a/source/libs/tfs/test/tfsTest.cpp b/source/libs/tfs/test/tfsTest.cpp index 178d115c59..af66304f84 100644 --- a/source/libs/tfs/test/tfsTest.cpp +++ b/source/libs/tfs/test/tfsTest.cpp @@ -230,10 +230,11 @@ TEST_F(TfsTest, 04_File) { EXPECT_EQ(tfsMkdir(pTfs, "t3"), 0); - FILE *fp = fopen(f1.aname, "w"); - ASSERT_NE(fp, nullptr); - fwrite("12345678", 1, 5, fp); - fclose(fp); + // FILE *fp = fopen(f1.aname, "w"); + TdFilePtr pFile = taosOpenFile(f1.aname, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + ASSERT_NE(pFile, nullptr); + taosWriteFile(pFile, "12345678", 5); + taosCloseFile(&pFile); char base[128] = {0}; tfsBasename(&f1, base); @@ -638,10 +639,11 @@ TEST_F(TfsTest, 05_MultiDisk) { EXPECT_EQ(tfsMkdir(pTfs, "t3"), 0); - FILE *fp = fopen(f1.aname, "w"); - ASSERT_NE(fp, nullptr); - fwrite("12345678", 1, 5, fp); - fclose(fp); + // FILE *fp = fopen(f1.aname, "w"); + TdFilePtr pFile = taosOpenFile(f1.aname, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + ASSERT_NE(pFile, nullptr); + taosWriteFile(pFile, "12345678", 5); + taosCloseFile(&pFile); char base[128] = {0}; tfsBasename(&f1, base); diff --git a/source/libs/transport/inc/rpcUdp.h b/source/libs/transport/inc/rpcUdp.h index c1da6a9240..0c651d07ed 100644 --- a/source/libs/transport/inc/rpcUdp.h +++ b/source/libs/transport/inc/rpcUdp.h @@ -30,7 +30,7 @@ void *taosOpenUdpConnection(void *shandle, void *thandle, uint32_t ip, uint16_t void taosFreeMsgHdr(void *hdr); int taosMsgHdrSize(void *hdr); -void taosSendMsgHdr(void *hdr, int fd); +void taosSendMsgHdr(void *hdr, TdFilePtr pFile); void taosInitMsgHdr(void **hdr, void *dest, int maxPkts); void taosSetMsgHdrData(void *hdr, char *data, int dataLen); diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index 2078a218ee..d4d9bff56c 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -238,10 +238,11 @@ SAsyncPool* transCreateAsyncPool(uv_loop_t* loop, int sz, void* arg, AsyncCB cb) void transDestroyAsyncPool(SAsyncPool* pool); int transSendAsync(SAsyncPool* pool, queue* mq); -int transInitBuffer(SConnBuffer* buf); -int transClearBuffer(SConnBuffer* buf); -int transDestroyBuffer(SConnBuffer* buf); -int transAllocBuffer(SConnBuffer* connBuf, uv_buf_t* uvBuf); +int transInitBuffer(SConnBuffer* buf); +int transClearBuffer(SConnBuffer* buf); +int transDestroyBuffer(SConnBuffer* buf); +int transAllocBuffer(SConnBuffer* connBuf, uv_buf_t* uvBuf); +bool transReadComplete(SConnBuffer* connBuf); // int transPackMsg(SRpcMsg *rpcMsg, bool sercured, bool auth, char **msg, int32_t *msgLen); diff --git a/source/libs/transport/inc/transportInt.h b/source/libs/transport/inc/transportInt.h index a36b671eb4..d080db753d 100644 --- a/source/libs/transport/inc/transportInt.h +++ b/source/libs/transport/inc/transportInt.h @@ -66,6 +66,7 @@ typedef struct { void (*cfp)(void* parent, SRpcMsg*, SEpSet*); int (*afp)(void* parent, char* user, char* spi, char* encrypt, char* secret, char* ckey); + bool (*pfp)(void* parent, tmsg_t msgType); int32_t refCount; void* parent; diff --git a/source/libs/transport/src/rpcMain.c b/source/libs/transport/src/rpcMain.c index 72c1ff6893..e1319da162 100644 --- a/source/libs/transport/src/rpcMain.c +++ b/source/libs/transport/src/rpcMain.c @@ -143,6 +143,7 @@ typedef struct SRpcConn { static int tsRpcRefId = -1; static int32_t tsRpcNum = 0; + // static pthread_once_t tsRpcInit = PTHREAD_ONCE_INIT; // server:0 client:1 tcp:2 udp:0 @@ -222,7 +223,7 @@ static void rpcInitImp(void) { tsFqdnHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); } -int32_t rpcInit(void) { +int32_t rpcInit() { pthread_once(&tsRpcInitOnce, rpcInitImp); return 0; } diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index 48c15ca286..b45683617f 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -29,7 +29,12 @@ void* rpcOpen(const SRpcInit* pInit) { if (pInit->label) { tstrncpy(pRpc->label, pInit->label, strlen(pInit->label) + 1); } + + // register callback handle pRpc->cfp = pInit->cfp; + pRpc->afp = pInit->afp; + pRpc->pfp = pInit->pfp; + if (pInit->connType == TAOS_CONN_SERVER) { pRpc->numOfThreads = pInit->numOfThreads > TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS : pInit->numOfThreads; } else { @@ -107,7 +112,7 @@ void rpcSendRedirectRsp(void* thandle, const SEpSet* pEpSet) { int rpcReportProgress(void* pConn, char* pCont, int contLen) { return -1; } void rpcCancelRequest(int64_t rid) { return; } -int32_t rpcInit(void) { +int32_t rpcInit() { // impl later return 0; } diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index c70cd933d5..d5fa4c8c47 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -84,8 +84,6 @@ static void addConnToPool(void* pool, char* ip, uint32_t port, SCliConn* co // register timer in each thread to clear expire conn static void clientTimeoutCb(uv_timer_t* handle); -// check whether already read complete packet from server -static bool clientReadComplete(SConnBuffer* pBuf); // alloc buf for read static void clientAllocBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf); // callback after read nbytes from socket @@ -136,8 +134,7 @@ static void clientHandleResp(SCliConn* conn) { rpcMsg.msgType = pHead->msgType; rpcMsg.ahandle = pCtx->ahandle; - if (rpcMsg.msgType == TDMT_VND_QUERY_RSP || rpcMsg.msgType == TDMT_VND_FETCH_RSP || - rpcMsg.msgType == TDMT_VND_RES_READY_RSP) { + if (pRpc->pfp != NULL && (pRpc->pfp)(pRpc->parent, rpcMsg.msgType)) { rpcMsg.handle = conn; conn->persist = 1; tDebug("client conn %p persist by app", conn); @@ -187,18 +184,13 @@ static void clientHandleExcept(SCliConn* pConn) { clientConnDestroy(pConn, true); return; } - SCliMsg* pMsg = pConn->data; - - tmsg_t msgType = TDMT_MND_CONNECT; - if (pMsg != NULL) { - msgType = pMsg->msg.msgType; - } + SCliMsg* pMsg = pConn->data; STransConnCtx* pCtx = pMsg->ctx; SRpcMsg rpcMsg = {0}; rpcMsg.ahandle = pCtx->ahandle; rpcMsg.code = TSDB_CODE_RPC_NETWORK_UNAVAIL; - rpcMsg.msgType = msgType + 1; + rpcMsg.msgType = pMsg->msg.msgType + 1; if (pConn->push != NULL && pConn->ctnRdCnt != 0) { (*pConn->push->callback)(pConn->push->arg, &rpcMsg); @@ -309,32 +301,6 @@ static void addConnToPool(void* pool, char* ip, uint32_t port, SCliConn* conn) { assert(plist != NULL); QUEUE_PUSH(&plist->conn, &conn->conn); } -static bool clientReadComplete(SConnBuffer* data) { - if (data->len >= sizeof(STransMsgHead)) { - STransMsgHead head; - memcpy((char*)&head, data->buf, sizeof(head)); - int32_t msgLen = (int32_t)htonl(head.msgLen); - data->total = msgLen; - } - - if (data->len == data->cap && data->total == data->cap) { - return true; - } - return false; - // if (data->len >= headLen) { - // memcpy((char*)&head, data->buf, headLen); - // int32_t msgLen = (int32_t)htonl((uint32_t)head.msgLen); - // if (msgLen > data->len) { - // data->left = msgLen - data->len; - // return false; - // } else if (msgLen == data->len) { - // data->left = 0; - // return true; - // } - //} else { - // return false; - //} -} static void clientAllocBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) { SCliConn* conn = handle->data; SConnBuffer* pBuf = &conn->readBuf; @@ -349,7 +315,7 @@ static void clientReadCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf SConnBuffer* pBuf = &conn->readBuf; if (nread > 0) { pBuf->len += nread; - if (clientReadComplete(pBuf)) { + if (transReadComplete(pBuf)) { tTrace("client conn %p read complete", conn); clientHandleResp(conn); } else { @@ -473,7 +439,7 @@ static void clientConnCb(uv_connect_t* req, int status) { addrlen = sizeof(pConn->locaddr); uv_tcp_getsockname((uv_tcp_t*)pConn->stream, (struct sockaddr*)&pConn->locaddr, &addrlen); - tTrace("client conn %p create", pConn); + tTrace("client conn %p connect to server successfully", pConn); assert(pConn->stream == req->handle); clientWrite(pConn); @@ -552,6 +518,7 @@ static void clientHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd) { struct sockaddr_in addr; uv_ip4_addr(pMsg->ctx->ip, pMsg->ctx->port, &addr); // handle error in callback if fail to connect + tTrace("client conn %p try to connect to %s:%d", conn, pMsg->ctx->ip, pMsg->ctx->port); uv_tcp_connect(&conn->connReq, (uv_tcp_t*)(conn->stream), (const struct sockaddr*)&addr, clientConnCb); } diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index 388e0da4e0..9a8607b0ed 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -222,23 +222,31 @@ int transAllocBuffer(SConnBuffer* connBuf, uv_buf_t* uvBuf) { p->buf = (char*)calloc(CAPACITY, sizeof(char)); p->len = 0; p->cap = CAPACITY; - p->total = 0; + p->total = -1; uvBuf->base = p->buf; uvBuf->len = CAPACITY; } else { - STransMsgHead head; - memcpy((char*)&head, p->buf, sizeof(head)); - int32_t msgLen = (int32_t)htonl(head.msgLen); - - p->total = msgLen; - p->cap = msgLen; + p->cap = p->total; p->buf = realloc(p->buf, p->cap); uvBuf->base = p->buf + p->len; uvBuf->len = p->cap - p->len; } return 0; } +// check whether already read complete +bool transReadComplete(SConnBuffer* connBuf) { + if (connBuf->total == -1 && connBuf->len >= sizeof(STransMsgHead)) { + STransMsgHead head; + memcpy((char*)&head, connBuf->buf, sizeof(head)); + int32_t msgLen = (int32_t)htonl(head.msgLen); + connBuf->total = msgLen; + } + if (connBuf->len == connBuf->cap && connBuf->total == connBuf->cap) { + return true; + } + return false; +} int transPackMsg(STransMsgHead* msgHead, bool sercured, bool auth) {} int transUnpackMsg(STransMsgHead* msgHead) {} diff --git a/source/libs/transport/src/transSrv.c b/source/libs/transport/src/transSrv.c index a038f72adc..f0db054797 100644 --- a/source/libs/transport/src/transSrv.c +++ b/source/libs/transport/src/transSrv.c @@ -104,7 +104,6 @@ static void uvStartSendResp(SSrvMsg* msg); static void destroySmsg(SSrvMsg* smsg); // check whether already read complete packet -static bool readComplete(SConnBuffer* buf); static SSrvConn* createConn(void* hThrd); static void destroyConn(SSrvConn* conn, bool clear /*clear handle or not*/); @@ -124,45 +123,6 @@ void uvAllocReadBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* b transAllocBuffer(pBuf, buf); } -// check data read from socket complete or not -// -static bool readComplete(SConnBuffer* data) { - // TODO(yihao): handle pipeline later - if (data->len == data->cap && data->total == data->cap) { - return true; - } - return false; - // STransMsgHead head; - // int32_t headLen = sizeof(head); - // if (data->len >= headLen) { - // memcpy((char*)&head, data->buf, headLen); - // int32_t msgLen = (int32_t)htonl((uint32_t)head.msgLen); - // if (msgLen > data->len) { - // data->left = msgLen - data->len; - // return false; - // } else if (msgLen == data->len) { - // return true; - // } else if (msgLen < data->len) { - // return false; - // // handle other packet later - // } - //} else { - // return false; - //} -} - -// static void uvDoProcess(SRecvInfo* pRecv) { -// // impl later -// STransMsgHead* pHead = (STransMsgHead*)pRecv->msg; -// SRpcInfo* pRpc = (SRpcInfo*)pRecv->shandle; -// SSrvConn* pConn = pRecv->thandle; -// tDump(pRecv->msg, pRecv->msgLen); -// terrno = 0; -// // SRpcReqContext* pContest; -// -// // do auth and check -//} - static int uvAuthMsg(SSrvConn* pConn, char* msg, int len) { STransMsgHead* pHead = (STransMsgHead*)msg; @@ -283,7 +243,7 @@ void uvOnReadCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf) { if (nread > 0) { pBuf->len += nread; tTrace("server conn %p read summary, total read: %d, current read: %d", conn, pBuf->len, (int)nread); - if (readComplete(pBuf)) { + if (transReadComplete(pBuf)) { tTrace("server conn %p alread read complete packet", conn); uvHandleReq(conn); } else { @@ -453,11 +413,6 @@ void uvWorkerAsyncCb(uv_async_t* handle) { } else { uvStartSendResp(msg); } - // uv_buf_t wb; - // uvPrepareSendData(msg, &wb); - // uv_timer_stop(conn->pTimer); - - // uv_write(conn->pWriter, (uv_stream_t*)conn->pTcp, &wb, 1, uvOnWriteCb); } } static void uvAcceptAsyncCb(uv_async_t* async) { @@ -530,7 +485,6 @@ void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) { pConn->pTimer->data = pConn; pConn->hostThrd = pThrd; - // pConn->pWorkerAsync = pThrd->workerAsync; // thread safty // init client handle pConn->pTcp = (uv_tcp_t*)malloc(sizeof(uv_tcp_t)); @@ -770,14 +724,9 @@ void destroyWorkThrd(SWorkThrdObj* pThrd) { } void sendQuitToWorkThrd(SWorkThrdObj* pThrd) { SSrvMsg* srvMsg = calloc(1, sizeof(SSrvMsg)); - - // pthread_mutex_lock(&pThrd->msgMtx); - // QUEUE_PUSH(&pThrd->msg, &srvMsg->q); - // pthread_mutex_unlock(&pThrd->msgMtx); tDebug("send quit msg to work thread"); transSendAsync(pThrd->asyncPool, &srvMsg->q); - // uv_async_send(pThrd->workerAsync); } void taosCloseServer(void* arg) { @@ -814,19 +763,12 @@ void rpcSendResponse(const SRpcMsg* pMsg) { SSrvMsg* srvMsg = calloc(1, sizeof(SSrvMsg)); srvMsg->pConn = pConn; srvMsg->msg = *pMsg; - - // pthread_mutex_lock(&pThrd->msgMtx); - // QUEUE_PUSH(&pThrd->msg, &srvMsg->q); - // pthread_mutex_unlock(&pThrd->msgMtx); - tTrace("server conn %p start to send resp", pConn); transSendAsync(pThrd->asyncPool, &srvMsg->q); - // uv_async_send(pThrd->workerAsync); } int rpcGetConnInfo(void* thandle, SRpcConnInfo* pInfo) { SSrvConn* pConn = thandle; - // struct sockaddr* pPeerName = &pConn->peername; struct sockaddr_in addr = pConn->addr; pInfo->clientIp = (uint32_t)(addr.sin_addr.s_addr); diff --git a/source/libs/transport/test/pushClient.c b/source/libs/transport/test/pushClient.c index f1aadafacc..4842a0c800 100644 --- a/source/libs/transport/test/pushClient.c +++ b/source/libs/transport/test/pushClient.c @@ -192,7 +192,7 @@ int main(int argc, char *argv[]) { } } - taosInitLog("client.log", 100000, 10); + taosInitLog("client.log", 10); void *pRpc = rpcOpen(&rpcInit); if (pRpc == NULL) { diff --git a/source/libs/transport/test/pushServer.c b/source/libs/transport/test/pushServer.c index 0bcc47383b..a1c181ac98 100644 --- a/source/libs/transport/test/pushServer.c +++ b/source/libs/transport/test/pushServer.c @@ -22,7 +22,7 @@ int msgSize = 128; int commit = 0; -int dataFd = -1; +TdFilePtr pDataFile = NULL; STaosQueue *qhandle = NULL; STaosQset * qset = NULL; @@ -43,8 +43,8 @@ void processShellMsg() { for (int i = 0; i < numOfMsgs; ++i) { taosGetQitem(qall, (void **)&pRpcMsg); - if (dataFd >= 0) { - if (write(dataFd, pRpcMsg->pCont, pRpcMsg->contLen) < 0) { + if (pDataFile != NULL) { + if (taosWriteFile(pDataFile, pRpcMsg->pCont, pRpcMsg->contLen) < 0) { tInfo("failed to write data file, reason:%s", strerror(errno)); } } @@ -52,7 +52,7 @@ void processShellMsg() { if (commit >= 2) { num += numOfMsgs; - // if (taosFsync(dataFd) < 0) { + // if (taosFsync(pDataFile) < 0) { // tInfo("failed to flush data to file, reason:%s", strerror(errno)); //} @@ -169,7 +169,7 @@ int main(int argc, char *argv[]) { tsAsyncLog = 0; rpcInit.connType = TAOS_CONN_SERVER; - taosInitLog("server.log", 100000, 10); + taosInitLog("server.log", 10); void *pRpc = rpcOpen(&rpcInit); if (pRpc == NULL) { @@ -181,8 +181,8 @@ int main(int argc, char *argv[]) { tInfo("RPC server is running, ctrl-c to exit"); if (commit) { - dataFd = open(dataName, O_APPEND | O_CREAT | O_WRONLY, S_IRWXU | S_IRWXG | S_IRWXO); - if (dataFd < 0) tInfo("failed to open data file, reason:%s", strerror(errno)); + pDataFile = taosOpenFile(dataName, TD_FILE_APPEND | TD_FILE_CTEATE | TD_FILE_WRITE); + if (pDataFile == NULL) tInfo("failed to open data file, reason:%s", strerror(errno)); } qhandle = taosOpenQueue(); qset = taosOpenQset(); @@ -190,8 +190,8 @@ int main(int argc, char *argv[]) { processShellMsg(); - if (dataFd >= 0) { - close(dataFd); + if (pDataFile != NULL) { + taosCloseFile(&pDataFile); remove(dataName); } diff --git a/source/libs/transport/test/rclient.c b/source/libs/transport/test/rclient.c index cc6a63d3cd..bcdf32bf6a 100644 --- a/source/libs/transport/test/rclient.c +++ b/source/libs/transport/test/rclient.c @@ -169,7 +169,7 @@ int main(int argc, char *argv[]) { } } - taosInitLog("client.log", 100000, 10); + taosInitLog("client.log", 10); void *pRpc = rpcOpen(&rpcInit); if (pRpc == NULL) { diff --git a/source/libs/transport/test/rsclient.c b/source/libs/transport/test/rsclient.c index 65170d4abb..26a02eb05b 100644 --- a/source/libs/transport/test/rsclient.c +++ b/source/libs/transport/test/rsclient.c @@ -97,7 +97,7 @@ int main(int argc, char *argv[]) { rpcInit.label = "APP"; rpcInit.numOfThreads = 1; rpcInit.sessions = 100; - rpcInit.idleTime = tsShellActivityTimer*1000; + rpcInit.idleTime = 3000; //tsShellActivityTimer*1000; rpcInit.user = "michael"; rpcInit.secret = secret; rpcInit.ckey = "key"; @@ -148,7 +148,7 @@ int main(int argc, char *argv[]) { } } - taosInitLog("client.log", 100000, 10); + taosInitLog("client.log", 10); void *pRpc = rpcOpen(&rpcInit); if (pRpc == NULL) { diff --git a/source/libs/transport/test/rserver.c b/source/libs/transport/test/rserver.c index d1a587f4e5..5432a07649 100644 --- a/source/libs/transport/test/rserver.c +++ b/source/libs/transport/test/rserver.c @@ -22,7 +22,7 @@ int msgSize = 128; int commit = 0; -int dataFd = -1; +TdFilePtr pDataFile = NULL; STaosQueue *qhandle = NULL; STaosQset * qset = NULL; @@ -43,8 +43,8 @@ void processShellMsg() { for (int i = 0; i < numOfMsgs; ++i) { taosGetQitem(qall, (void **)&pRpcMsg); - if (dataFd >= 0) { - if (write(dataFd, pRpcMsg->pCont, pRpcMsg->contLen) < 0) { + if (pDataFile != NULL) { + if (taosWriteFile(pDataFile, pRpcMsg->pCont, pRpcMsg->contLen) < 0) { tInfo("failed to write data file, reason:%s", strerror(errno)); } } @@ -52,7 +52,7 @@ void processShellMsg() { if (commit >= 2) { num += numOfMsgs; - // if (taosFsync(dataFd) < 0) { + // if (taosFsync(pDataFile) < 0) { // tInfo("failed to flush data to file, reason:%s", strerror(errno)); //} @@ -158,7 +158,7 @@ int main(int argc, char *argv[]) { tsAsyncLog = 0; rpcInit.connType = TAOS_CONN_SERVER; - taosInitLog("server.log", 100000, 10); + taosInitLog("server.log", 10); void *pRpc = rpcOpen(&rpcInit); if (pRpc == NULL) { @@ -170,8 +170,8 @@ int main(int argc, char *argv[]) { tInfo("RPC server is running, ctrl-c to exit"); if (commit) { - dataFd = open(dataName, O_APPEND | O_CREAT | O_WRONLY, S_IRWXU | S_IRWXG | S_IRWXO); - if (dataFd < 0) tInfo("failed to open data file, reason:%s", strerror(errno)); + pDataFile = taosOpenFile(dataName, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_APPEND); + if (pDataFile == NULL) tInfo("failed to open data file, reason:%s", strerror(errno)); } qhandle = taosOpenQueue(); qset = taosOpenQset(); @@ -179,8 +179,8 @@ int main(int argc, char *argv[]) { processShellMsg(); - if (dataFd >= 0) { - close(dataFd); + if (pDataFile != NULL) { + taosCloseFile(&pDataFile); remove(dataName); } diff --git a/source/libs/transport/test/syncClient.c b/source/libs/transport/test/syncClient.c index c5d7f5664a..b7ef296b9d 100644 --- a/source/libs/transport/test/syncClient.c +++ b/source/libs/transport/test/syncClient.c @@ -170,7 +170,7 @@ int main(int argc, char *argv[]) { } } - taosInitLog("client.log", 100000, 10); + taosInitLog("client.log", 10); void *pRpc = rpcOpen(&rpcInit); if (pRpc == NULL) { diff --git a/source/libs/transport/test/transUT.cc b/source/libs/transport/test/transUT.cc index d1fefe2c72..6db709da51 100644 --- a/source/libs/transport/test/transUT.cc +++ b/source/libs/transport/test/transUT.cc @@ -18,7 +18,7 @@ #include "tep.h" #include "tglobal.h" #include "trpc.h" -#include "ulog.h" +#include "tlog.h" using namespace std; const char *label = "APP"; @@ -148,7 +148,6 @@ class TransObj { wDebugFlag = 0; sDebugFlag = 0; tsdbDebugFlag = 0; - cqDebugFlag = 0; tscEmbeddedInUtil = 1; tsAsyncLog = 0; @@ -156,9 +155,8 @@ class TransObj { taosRemoveDir(path.c_str()); taosMkDir(path.c_str()); - char temp[PATH_MAX]; - snprintf(temp, PATH_MAX, "%s/taosdlog", path.c_str()); - if (taosInitLog(temp, tsNumOfLogLines, 1) != 0) { + tstrncpy(tsLogDir, path.c_str(), PATH_MAX); + if (taosInitLog("taosdlog", 1) != 0) { printf("failed to init log file\n"); } cli = new Client; diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index 2d6fb8fc76..ae0b0bd849 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -69,8 +69,8 @@ static inline int64_t walScanLogGetLastVer(SWal* pWal) { int readSize = TMIN(WAL_MAX_SIZE + 2, statbuf.st_size); pLastFileInfo->fileSize = statbuf.st_size; - FileFd fd = taosOpenFileRead(fnameStr); - if (fd < 0) { + TdFilePtr pFile = taosOpenFile(fnameStr, TD_FILE_READ); + if (pFile == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } @@ -79,15 +79,15 @@ static inline int64_t walScanLogGetLastVer(SWal* pWal) { char* buf = malloc(readSize + 5); if (buf == NULL) { - taosCloseFile(fd); + taosCloseFile(&pFile); terrno = TSDB_CODE_WAL_OUT_OF_MEMORY; return -1; } - taosLSeekFile(fd, -readSize, SEEK_END); - if (readSize != taosReadFile(fd, buf, readSize)) { + taosLSeekFile(pFile, -readSize, SEEK_END); + if (readSize != taosReadFile(pFile, buf, readSize)) { free(buf); - taosCloseFile(fd); + taosCloseFile(&pFile); terrno = TAOS_SYSTEM_ERROR(errno); return -1; } @@ -108,12 +108,12 @@ static inline int64_t walScanLogGetLastVer(SWal* pWal) { if (walValidHeadCksum(logContent) != 0 || walValidBodyCksum(logContent) != 0) { // file has to be deleted free(buf); - taosCloseFile(fd); + taosCloseFile(&pFile); terrno = TSDB_CODE_WAL_FILE_CORRUPTED; return -1; } } - taosCloseFile(fd); + taosCloseFile(&pFile); SWalHead *lastEntry = (SWalHead*)found; return lastEntry->head.version; @@ -364,18 +364,18 @@ int walSaveMeta(SWal* pWal) { int metaVer = walFindCurMetaVer(pWal); char fnameStr[WAL_FILE_LEN]; walBuildMetaName(pWal, metaVer + 1, fnameStr); - FileFd metaFd = taosOpenFileCreateWrite(fnameStr); - if (metaFd < 0) { + TdFilePtr pMataFile = taosOpenFile(fnameStr, TD_FILE_CTEATE | TD_FILE_WRITE); + if (pMataFile == NULL) { return -1; } char* serialized = walMetaSerialize(pWal); int len = strlen(serialized); - if (len != taosWriteFile(metaFd, serialized, len)) { + if (len != taosWriteFile(pMataFile, serialized, len)) { // TODO:clean file return -1; } - taosCloseFile(metaFd); + taosCloseFile(&pMataFile); // delete old file if (metaVer > -1) { walBuildMetaName(pWal, metaVer, fnameStr); @@ -404,20 +404,20 @@ int walLoadMeta(SWal* pWal) { return -1; } memset(buf, 0, size + 5); - FileFd fd = taosOpenFileRead(fnameStr); - if (fd < 0) { + TdFilePtr pFile = taosOpenFile(fnameStr, TD_FILE_READ); + if (pFile == NULL) { terrno = TSDB_CODE_WAL_FILE_CORRUPTED; return -1; } - if (taosReadFile(fd, buf, size) != size) { + if (taosReadFile(pFile, buf, size) != size) { terrno = TAOS_SYSTEM_ERROR(errno); - taosCloseFile(fd); + taosCloseFile(&pFile); free(buf); return -1; } // load into fileInfoSet int code = walMetaDeserialize(pWal, buf); - taosCloseFile(fd); + taosCloseFile(&pFile); free(buf); return code; } diff --git a/source/libs/wal/src/walMgmt.c b/source/libs/wal/src/walMgmt.c index d5c28d9d9b..d3cd23f284 100644 --- a/source/libs/wal/src/walMgmt.c +++ b/source/libs/wal/src/walMgmt.c @@ -89,8 +89,8 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { // open meta walResetVer(&pWal->vers); - pWal->writeLogTfd = -1; - pWal->writeIdxTfd = -1; + pWal->pWriteLogTFile = NULL; + pWal->pWriteIdxTFile = NULL; pWal->writeCur = -1; pWal->fileInfoSet = taosArrayInit(8, sizeof(SWalFileInfo)); if (pWal->fileInfoSet == NULL) { @@ -164,10 +164,10 @@ int32_t walAlter(SWal *pWal, SWalCfg *pCfg) { void walClose(SWal *pWal) { pthread_mutex_lock(&pWal->mutex); - tfClose(pWal->writeLogTfd); - pWal->writeLogTfd = -1; - tfClose(pWal->writeIdxTfd); - pWal->writeIdxTfd = -1; + taosCloseFile(&pWal->pWriteLogTFile); + pWal->pWriteLogTFile = NULL; + taosCloseFile(&pWal->pWriteIdxTFile); + pWal->pWriteIdxTFile = NULL; walSaveMeta(pWal); taosArrayDestroy(pWal->fileInfoSet); pWal->fileInfoSet = NULL; @@ -207,7 +207,7 @@ static void walFsyncAll() { if (walNeedFsync(pWal)) { wTrace("vgId:%d, do fsync, level:%d seq:%d rseq:%d", pWal->cfg.vgId, pWal->cfg.level, pWal->fsyncSeq, atomic_load_32(&tsWal.seq)); - int32_t code = tfFsync(pWal->writeLogTfd); + int32_t code = taosFsyncFile(pWal->pWriteLogTFile); if (code != 0) { wError("vgId:%d, file:%" PRId64 ".log, failed to fsync since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal), strerror(code)); diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index 1d9201f69d..159d281759 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -25,8 +25,8 @@ SWalReadHandle *walOpenReadHandle(SWal *pWal) { } pRead->pWal = pWal; - pRead->readIdxTfd = -1; - pRead->readLogTfd = -1; + pRead->pReadIdxTFile = NULL; + pRead->pReadLogTFile = NULL; pRead->curVersion = -1; pRead->curFileFirstVer = -1; pRead->capacity = 0; @@ -41,8 +41,8 @@ SWalReadHandle *walOpenReadHandle(SWal *pWal) { } void walCloseReadHandle(SWalReadHandle *pRead) { - tfClose(pRead->readIdxTfd); - tfClose(pRead->readLogTfd); + taosCloseFile(&pRead->pReadIdxTFile); + taosCloseFile(&pRead->pReadLogTFile); tfree(pRead->pHead); free(pRead); } @@ -52,24 +52,24 @@ int32_t walRegisterRead(SWalReadHandle *pRead, int64_t ver) { return 0; } static int32_t walReadSeekFilePos(SWalReadHandle *pRead, int64_t fileFirstVer, int64_t ver) { int code = 0; - int64_t idxTfd = pRead->readIdxTfd; - int64_t logTfd = pRead->readLogTfd; + TdFilePtr pIdxTFile = pRead->pReadIdxTFile; + TdFilePtr pLogTFile = pRead->pReadLogTFile; // seek position int64_t offset = (ver - fileFirstVer) * sizeof(SWalIdxEntry); - code = tfLseek(idxTfd, offset, SEEK_SET); + code = taosLSeekFile(pIdxTFile, offset, SEEK_SET); if (code < 0) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } SWalIdxEntry entry; - if (tfRead(idxTfd, &entry, sizeof(SWalIdxEntry)) != sizeof(SWalIdxEntry)) { + if (taosReadFile(pIdxTFile, &entry, sizeof(SWalIdxEntry)) != sizeof(SWalIdxEntry)) { terrno = TSDB_CODE_WAL_FILE_CORRUPTED; return -1; } // TODO:deserialize ASSERT(entry.ver == ver); - code = tfLseek(logTfd, entry.offset, SEEK_SET); + code = taosLSeekFile(pLogTFile, entry.offset, SEEK_SET); if (code < 0) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; @@ -80,24 +80,24 @@ static int32_t walReadSeekFilePos(SWalReadHandle *pRead, int64_t fileFirstVer, i static int32_t walReadChangeFile(SWalReadHandle *pRead, int64_t fileFirstVer) { char fnameStr[WAL_FILE_LEN]; - tfClose(pRead->readIdxTfd); - tfClose(pRead->readLogTfd); + taosCloseFile(&pRead->pReadIdxTFile); + taosCloseFile(&pRead->pReadLogTFile); walBuildLogName(pRead->pWal, fileFirstVer, fnameStr); - int64_t logTfd = tfOpenRead(fnameStr); - if (logTfd < 0) { + TdFilePtr pLogTFile = taosOpenFile(fnameStr, TD_FILE_READ); + if (pLogTFile == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } walBuildIdxName(pRead->pWal, fileFirstVer, fnameStr); - int64_t idxTfd = tfOpenRead(fnameStr); - if (idxTfd < 0) { + TdFilePtr pIdxTFile = taosOpenFile(fnameStr, TD_FILE_READ); + if (pIdxTFile == NULL) { return -1; } - pRead->readLogTfd = logTfd; - pRead->readIdxTfd = idxTfd; + pRead->pReadLogTFile = pLogTFile; + pRead->pReadIdxTFile = pIdxTFile; return 0; } @@ -145,9 +145,9 @@ int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver) { } } - if (!tfValid(pRead->readLogTfd)) return -1; + if (!taosValidFile(pRead->pReadLogTFile)) return -1; - code = tfRead(pRead->readLogTfd, pRead->pHead, sizeof(SWalHead)); + code = taosReadFile(pRead->pReadLogTFile, pRead->pHead, sizeof(SWalHead)); if (code != sizeof(SWalHead)) { return -1; } @@ -165,7 +165,7 @@ int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver) { pRead->pHead = ptr; pRead->capacity = pRead->pHead->head.len; } - if (pRead->pHead->head.len != tfRead(pRead->readLogTfd, pRead->pHead->head.body, pRead->pHead->head.len)) { + if (pRead->pHead->head.len != taosReadFile(pRead->pReadLogTFile, pRead->pHead->head.body, pRead->pHead->head.len)) { return -1; } @@ -202,7 +202,7 @@ int32_t walRead(SWal *pWal, SWalHead **ppHead, int64_t ver) { } *ppHead = ptr; } - if (tfRead(pWal->writeLogTfd, *ppHead, sizeof(SWalHead)) != sizeof(SWalHead)) { + if (tfRead(pWal->pWriteLogTFile, *ppHead, sizeof(SWalHead)) != sizeof(SWalHead)) { return -1; } // TODO: endian compatibility processing after read @@ -215,7 +215,7 @@ int32_t walRead(SWal *pWal, SWalHead **ppHead, int64_t ver) { *ppHead = NULL; return -1; } - if (tfRead(pWal->writeLogTfd, (*ppHead)->head.body, (*ppHead)->head.len) != (*ppHead)->head.len) { + if (tfRead(pWal->pWriteLogTFile, (*ppHead)->head.body, (*ppHead)->head.len) != (*ppHead)->head.len) { return -1; } // TODO: endian compatibility processing after read diff --git a/source/libs/wal/src/walSeek.c b/source/libs/wal/src/walSeek.c index 91b172444b..6b3abcd0f9 100644 --- a/source/libs/wal/src/walSeek.c +++ b/source/libs/wal/src/walSeek.c @@ -23,25 +23,25 @@ static int walSeekWritePos(SWal* pWal, int64_t ver) { int code = 0; - int64_t idxTfd = pWal->writeIdxTfd; - int64_t logTfd = pWal->writeLogTfd; + TdFilePtr pIdxTFile = pWal->pWriteIdxTFile; + TdFilePtr pLogTFile = pWal->pWriteLogTFile; // seek position int64_t idxOff = walGetVerIdxOffset(pWal, ver); - code = tfLseek(idxTfd, idxOff, SEEK_SET); + code = taosLSeekFile(pIdxTFile, idxOff, SEEK_SET); if (code != 0) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } SWalIdxEntry entry; // TODO:deserialize - code = tfRead(idxTfd, &entry, sizeof(SWalIdxEntry)); + code = taosReadFile(pIdxTFile, &entry, sizeof(SWalIdxEntry)); if (code != 0) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } ASSERT(entry.ver == ver); - code = tfLseek(logTfd, entry.offset, SEEK_SET); + code = taosLSeekFile(pLogTFile, entry.offset, SEEK_SET); if (code < 0) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; @@ -50,43 +50,43 @@ static int walSeekWritePos(SWal* pWal, int64_t ver) { } int walSetWrite(SWal* pWal) { - int64_t idxTfd, logTfd; + TdFilePtr pIdxTFile, pLogTFile; SWalFileInfo* pRet = taosArrayGetLast(pWal->fileInfoSet); ASSERT(pRet != NULL); int64_t fileFirstVer = pRet->firstVer; char fnameStr[WAL_FILE_LEN]; walBuildIdxName(pWal, fileFirstVer, fnameStr); - idxTfd = tfOpenCreateWriteAppend(fnameStr); - if (idxTfd < 0) { + pIdxTFile = taosOpenFile(fnameStr, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_APPEND); + if (pIdxTFile == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } walBuildLogName(pWal, fileFirstVer, fnameStr); - logTfd = tfOpenCreateWriteAppend(fnameStr); - if (logTfd < 0) { + pLogTFile = taosOpenFile(fnameStr, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_APPEND); + if (pLogTFile == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } // switch file - pWal->writeIdxTfd = idxTfd; - pWal->writeLogTfd = logTfd; + pWal->pWriteIdxTFile = pIdxTFile; + pWal->pWriteLogTFile = pLogTFile; return 0; } int walChangeWrite(SWal* pWal, int64_t ver) { int code = 0; - int64_t idxTfd, logTfd; + TdFilePtr pIdxTFile, pLogTFile; char fnameStr[WAL_FILE_LEN]; - if (pWal->writeLogTfd != -1) { - code = tfClose(pWal->writeLogTfd); + if (pWal->pWriteLogTFile != NULL) { + code = taosCloseFile(&pWal->pWriteLogTFile); if (code != 0) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } } - if (pWal->writeIdxTfd != -1) { - code = tfClose(pWal->writeIdxTfd); + if (pWal->pWriteIdxTFile != NULL) { + code = taosCloseFile(&pWal->pWriteIdxTFile); if (code != 0) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; @@ -103,23 +103,23 @@ int walChangeWrite(SWal* pWal, int64_t ver) { int64_t fileFirstVer = pFileInfo->firstVer; walBuildIdxName(pWal, fileFirstVer, fnameStr); - idxTfd = tfOpenCreateWriteAppend(fnameStr); - if (idxTfd < 0) { + pIdxTFile = taosOpenFile(fnameStr, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_APPEND); + if (pIdxTFile == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); - pWal->writeIdxTfd = -1; + pWal->pWriteIdxTFile = NULL; return -1; } walBuildLogName(pWal, fileFirstVer, fnameStr); - logTfd = tfOpenCreateWriteAppend(fnameStr); - if (logTfd < 0) { - tfClose(idxTfd); + pLogTFile = taosOpenFile(fnameStr, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_APPEND); + if (pLogTFile == NULL) { + taosCloseFile(&pIdxTFile); terrno = TAOS_SYSTEM_ERROR(errno); - pWal->writeLogTfd = -1; + pWal->pWriteLogTFile = NULL; return -1; } - pWal->writeLogTfd = logTfd; - pWal->writeIdxTfd = idxTfd; + pWal->pWriteLogTFile = pLogTFile; + pWal->pWriteIdxTFile = pIdxTFile; pWal->writeCur = idx; return fileFirstVer; } diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index a4b34dee37..ba07134003 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -65,15 +65,15 @@ int32_t walRollback(SWal *pWal, int64_t ver) { } walBuildIdxName(pWal, walGetCurFileFirstVer(pWal), fnameStr); - int64_t idxTfd = tfOpenReadWrite(fnameStr); + TdFilePtr pIdxTFile = taosOpenFile(fnameStr, TD_FILE_WRITE | TD_FILE_READ); // TODO:change to deserialize function - if (idxTfd < 0) { + if (pIdxTFile == NULL) { pthread_mutex_unlock(&pWal->mutex); return -1; } int64_t idxOff = walGetVerIdxOffset(pWal, ver); - code = tfLseek(idxTfd, idxOff, SEEK_SET); + code = taosLSeekFile(pIdxTFile, idxOff, SEEK_SET); if (code < 0) { pthread_mutex_unlock(&pWal->mutex); return -1; @@ -81,20 +81,20 @@ int32_t walRollback(SWal *pWal, int64_t ver) { // read idx file and get log file pos // TODO:change to deserialize function SWalIdxEntry entry; - if (tfRead(idxTfd, &entry, sizeof(SWalIdxEntry)) != sizeof(SWalIdxEntry)) { + if (taosReadFile(pIdxTFile, &entry, sizeof(SWalIdxEntry)) != sizeof(SWalIdxEntry)) { pthread_mutex_unlock(&pWal->mutex); return -1; } ASSERT(entry.ver == ver); walBuildLogName(pWal, walGetCurFileFirstVer(pWal), fnameStr); - int64_t logTfd = tfOpenReadWrite(fnameStr); - if (logTfd < 0) { + TdFilePtr pLogTFile = taosOpenFile(fnameStr, TD_FILE_WRITE | TD_FILE_READ); + if (pLogTFile == NULL) { // TODO pthread_mutex_unlock(&pWal->mutex); return -1; } - code = tfLseek(logTfd, entry.offset, SEEK_SET); + code = taosLSeekFile(pLogTFile, entry.offset, SEEK_SET); if (code < 0) { // TODO pthread_mutex_unlock(&pWal->mutex); @@ -102,8 +102,8 @@ int32_t walRollback(SWal *pWal, int64_t ver) { } // validate offset SWalHead head; - ASSERT(tfValid(logTfd)); - int size = tfRead(logTfd, &head, sizeof(SWalHead)); + ASSERT(taosValidFile(pLogTFile)); + int size = taosReadFile(pLogTFile, &head, sizeof(SWalHead)); if (size != sizeof(SWalHead)) { return -1; } @@ -118,11 +118,11 @@ int32_t walRollback(SWal *pWal, int64_t ver) { return -1; } // truncate old files - code = tfFtruncate(logTfd, entry.offset); + code = taosFtruncateFile(pLogTFile, entry.offset); if (code < 0) { return -1; } - code = tfFtruncate(idxTfd, idxOff); + code = taosFtruncateFile(pIdxTFile, idxOff); if (code < 0) { return -1; } @@ -203,31 +203,31 @@ int32_t walEndSnapshot(SWal *pWal) { int walRoll(SWal *pWal) { int code = 0; - if (pWal->writeIdxTfd != -1) { - code = tfClose(pWal->writeIdxTfd); + if (pWal->pWriteIdxTFile != NULL) { + code = taosCloseFile(&pWal->pWriteIdxTFile); if (code != 0) { return -1; } } - if (pWal->writeLogTfd != -1) { - code = tfClose(pWal->writeLogTfd); + if (pWal->pWriteLogTFile != NULL) { + code = taosCloseFile(&pWal->pWriteLogTFile); if (code != 0) { return -1; } } - int64_t idxTfd, logTfd; + TdFilePtr pIdxTFile, pLogTFile; // create new file int64_t newFileFirstVersion = pWal->vers.lastVer + 1; char fnameStr[WAL_FILE_LEN]; walBuildIdxName(pWal, newFileFirstVersion, fnameStr); - idxTfd = tfOpenCreateWriteAppend(fnameStr); - if (idxTfd < 0) { + pIdxTFile = taosOpenFile(fnameStr, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_APPEND); + if (pIdxTFile == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } walBuildLogName(pWal, newFileFirstVersion, fnameStr); - logTfd = tfOpenCreateWriteAppend(fnameStr); - if (logTfd < 0) { + pLogTFile = taosOpenFile(fnameStr, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_APPEND); + if (pLogTFile == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } @@ -237,8 +237,8 @@ int walRoll(SWal *pWal) { } // switch file - pWal->writeIdxTfd = idxTfd; - pWal->writeLogTfd = logTfd; + pWal->pWriteIdxTFile = pIdxTFile; + pWal->pWriteLogTFile = pLogTFile; pWal->writeCur = taosArrayGetSize(pWal->fileInfoSet) - 1; ASSERT(pWal->writeCur >= 0); @@ -248,7 +248,7 @@ int walRoll(SWal *pWal) { static int walWriteIndex(SWal *pWal, int64_t ver, int64_t offset) { SWalIdxEntry entry = {.ver = ver, .offset = offset}; - int size = tfWrite(pWal->writeIdxTfd, &entry, sizeof(SWalIdxEntry)); + int size = taosWriteFile(pWal->pWriteIdxTFile, &entry, sizeof(SWalIdxEntry)); if (size != sizeof(SWalIdxEntry)) { terrno = TAOS_SYSTEM_ERROR(errno); // TODO truncate @@ -282,16 +282,16 @@ int64_t walWrite(SWal *pWal, int64_t index, tmsg_t msgType, const void *body, in // must truncate explicitly first return -1; } - /*if (!tfValid(pWal->writeLogTfd)) return -1;*/ + /*if (!tfValid(pWal->pWriteLogTFile)) return -1;*/ ASSERT(pWal->writeCur >= 0); pthread_mutex_lock(&pWal->mutex); - if (pWal->writeIdxTfd == -1 || pWal->writeLogTfd == -1) { + if (pWal->pWriteIdxTFile == NULL || pWal->pWriteLogTFile == NULL) { walSetWrite(pWal); - tfLseek(pWal->writeLogTfd, 0, SEEK_END); - tfLseek(pWal->writeIdxTfd, 0, SEEK_END); + taosLSeekFile(pWal->pWriteLogTFile, 0, SEEK_END); + taosLSeekFile(pWal->pWriteIdxTFile, 0, SEEK_END); } pWal->writeHead.head.version = index; @@ -302,14 +302,14 @@ int64_t walWrite(SWal *pWal, int64_t index, tmsg_t msgType, const void *body, in pWal->writeHead.cksumHead = walCalcHeadCksum(&pWal->writeHead); pWal->writeHead.cksumBody = walCalcBodyCksum(body, bodyLen); - if (tfWrite(pWal->writeLogTfd, &pWal->writeHead, sizeof(SWalHead)) != sizeof(SWalHead)) { + if (taosWriteFile(pWal->pWriteLogTFile, &pWal->writeHead, sizeof(SWalHead)) != sizeof(SWalHead)) { // ftruncate code = TAOS_SYSTEM_ERROR(errno); wError("vgId:%d, file:%" PRId64 ".log, failed to write since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal), strerror(errno)); } - if (tfWrite(pWal->writeLogTfd, (char *)body, bodyLen) != bodyLen) { + if (taosWriteFile(pWal->pWriteLogTFile, (char *)body, bodyLen) != bodyLen) { // ftruncate code = TAOS_SYSTEM_ERROR(errno); wError("vgId:%d, file:%" PRId64 ".log, failed to write since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal), @@ -336,7 +336,7 @@ int64_t walWrite(SWal *pWal, int64_t index, tmsg_t msgType, const void *body, in void walFsync(SWal *pWal, bool forceFsync) { if (forceFsync || (pWal->cfg.level == TAOS_WAL_FSYNC && pWal->cfg.fsyncPeriod == 0)) { wTrace("vgId:%d, fileId:%" PRId64 ".log, do fsync", pWal->cfg.vgId, walGetCurFileFirstVer(pWal)); - if (tfFsync(pWal->writeLogTfd) < 0) { + if (taosFsyncFile(pWal->pWriteLogTFile) < 0) { wError("vgId:%d, file:%" PRId64 ".log, fsync failed since %s", pWal->cfg.vgId, walGetCurFileFirstVer(pWal), strerror(errno)); } diff --git a/source/os/src/osDir.c b/source/os/src/osDir.c index 8999646f6a..c464073e5f 100644 --- a/source/os/src/osDir.c +++ b/source/os/src/osDir.c @@ -71,7 +71,7 @@ int32_t taosMkDir(const char *dirname) { return code; } -void taosRemoveOldFiles(char *dirname, int32_t keepDays) { +void taosRemoveOldFiles(const char *dirname, int32_t keepDays) { DIR *dir = opendir(dirname); if (dir == NULL) return; @@ -114,7 +114,7 @@ void taosRemoveOldFiles(char *dirname, int32_t keepDays) { rmdir(dirname); } -int32_t taosExpandDir(char *dirname, char *outname, int32_t maxlen) { +int32_t taosExpandDir(const char *dirname, char *outname, int32_t maxlen) { wordexp_t full_path; if (0 != wordexp(dirname, &full_path, 0)) { //printf("failed to expand path:%s since %s", dirname, strerror(errno)); diff --git a/source/os/src/osEnv.c b/source/os/src/osEnv.c index 7bc5e0a7fc..4c368fe895 100644 --- a/source/os/src/osEnv.c +++ b/source/os/src/osEnv.c @@ -13,54 +13,89 @@ * along with this program. If not, see . */ - #define _DEFAULT_SOURCE -#include "os.h" +#include "osEnv.h" -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +extern void taosWinSocketInit(); -char tsOsName[10] = "Windows"; -char configDir[PATH_MAX] = "C:/TDengine/cfg"; -char tsDataDir[PATH_MAX] = "C:/TDengine/data"; -char tsLogDir[PATH_MAX] = "C:/TDengine/log"; -char tsScriptDir[PATH_MAX] = "C:/TDengine/script"; -char tsTempDir[PATH_MAX] = "C:\\Windows\\Temp"; - -extern taosWinSocketInit(); +char configDir[PATH_MAX] = {0}; +char tsDataDir[PATH_MAX] = {0}; +char tsLogDir[PATH_MAX] = {0}; +char tsTempDir[PATH_MAX] = {0}; +SDiskSpace tsDataSpace = {0}; +SDiskSpace tsLogSpace = {0}; +SDiskSpace tsTempSpace = {0}; +char tsOsName[16] = {0}; +char tsTimezone[TD_TIMEZONE_LEN] = {0}; +char tsLocale[TD_LOCALE_LEN] = {0}; +char tsCharset[TD_CHARSET_LEN] = {0}; +int8_t tsDaylight = 0; +bool tsEnableCoreFile = 0; +int64_t tsPageSize = 0; +int64_t tsOpenMax = 0; +int64_t tsStreamMax = 0; +int32_t tsNumOfCores = 0; +int32_t tsTotalMemoryMB = 0; void osInit() { + srand(taosSafeRand()); + taosGetSystemLocale(tsLocale, tsCharset); + taosGetSystemTimezone(tsTimezone); + taosSetSystemTimezone(tsTimezone, tsTimezone, &tsDaylight); + taosGetSystemInfo(); + +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) taosWinSocketInit(); const char *tmpDir = getenv("tmp"); if (tmpDir == NULL) { tmpDir = getenv("temp"); } - if (tmpDir != NULL) { strcpy(tsTempDir, tmpDir); } -} + + if (configDir[0] == 0) { + strcpy(configDir, "C:\\TDengine\\cfg"); + } + strcpy(tsDataDir, "C:\\TDengine\\data"); + strcpy(tsLogDir, "C:\\TDengine\\log"); + strcpy(tsTempDir, "C:\\Windows\\Temp"); + strcpy(tsOsName, "Windows"); #elif defined(_TD_DARWIN_64) - -char tsOsName[10] = "Darwin"; -char configDir[PATH_MAX] = "/usr/local/etc/taos"; -char tsDataDir[PATH_MAX] = "/usr/local/var/lib/taos"; -char tsLogDir[PATH_MAX] = "/usr/local/var/log/taos"; -char tsScriptDir[PATH_MAX] = "/usr/local/etc/taos"; -char tsTempDir[PATH_MAX] = "/tmp/taosd"; - -void osInit() {} + if (configDir[0] == 0) { + strcpy(configDir, "/tmp/taosd"); + } + strcpy(tsDataDir, "/usr/local/var/lib/taos"); + strcpy(tsLogDir, "/usr/local/var/log/taos"); + strcpy(tsTempDir, "/usr/local/etc/taos"); + strcpy(tsOsName, "Darwin"); #else - -char tsOsName[10] = "Linux"; -char configDir[PATH_MAX] = "/etc/taos"; -char tsDataDir[PATH_MAX] = "/var/lib/taos"; -char tsLogDir[PATH_MAX] = "/var/log/taos"; -char tsScriptDir[PATH_MAX] = "/etc/taos"; -char tsTempDir[PATH_MAX] = "/tmp/"; - -void osInit() {} + if (configDir[0] == 0) { + strcpy(configDir, "/etc/taos"); + } + strcpy(tsDataDir, "/var/lib/taos"); + strcpy(tsLogDir, "/var/log/taos"); + strcpy(tsTempDir, "/tmp"); + strcpy(tsOsName, "Linux"); #endif +} + +void osUpdate() { + if (tsLogDir[0] != 0) { + taosGetDiskSize(tsLogDir, &tsLogSpace.size); + } + if (tsDataDir[0] != 0) { + taosGetDiskSize(tsDataDir, &tsDataSpace.size); + } + if (tsTempDir[0] != 0) { + taosGetDiskSize(tsTempDir, &tsTempSpace.size); + } +} + +bool osLogSpaceAvailable() { return tsLogSpace.reserved <= tsLogSpace.size.avail; } + +void osSetTimezone(const char *timezone) { taosSetSystemTimezone(tsTimezone, tsTimezone, &tsDaylight); } diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index 5a67ab73eb..c34513a2f1 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -12,9 +12,11 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ - +#define ALLOW_FORBID_FUNC #include "os.h" +#define MAX_FPRINTFLINE_BUFFER_SIZE (1000) + #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) #include @@ -38,14 +40,19 @@ extern int openU(const char *, int, ...); /* MsvcLibX UTF-8 version of open */ #include #include #include +#define LINUX_FILE_NO_TEXT_OPTION 0 +#define O_TEXT LINUX_FILE_NO_TEXT_OPTION #endif -void taosCloseFile(FileFd fd) { - close(fd); - fd = FD_INITIALIZER; -} +typedef int32_t FileFd; -void taosGetTmpfilePath(const char * inputTmpDir, const char *fileNamePrefix, char *dstPath) { +typedef struct TdFile { + int refId; + FileFd fd; + FILE *fp; +} * TdFilePtr, TdFile; + +void taosGetTmpfilePath(const char *inputTmpDir, const char *fileNamePrefix, char *dstPath) { #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) const char *tdengineTmpFileNamePrefix = "tdengine-"; char tmpPath[PATH_MAX]; @@ -72,7 +79,7 @@ void taosGetTmpfilePath(const char * inputTmpDir, const char *fileNamePrefix, ch const char *tdengineTmpFileNamePrefix = "tdengine-"; - char tmpPath[PATH_MAX]; + char tmpPath[PATH_MAX]; int32_t len = strlen(inputTmpDir); memcpy(tmpPath, inputTmpDir, len); static uint64_t seqId = 0; @@ -96,13 +103,163 @@ void taosGetTmpfilePath(const char * inputTmpDir, const char *fileNamePrefix, ch #endif } -int64_t taosReadFile(FileFd fd, void *buf, int64_t count) { +int64_t taosCopyFile(const char *from, const char *to) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return 0; +#else + char buffer[4096]; + int64_t size = 0; + int64_t bytes; + + // fidfrom = open(from, O_RDONLY); + TdFilePtr pFileFrom = taosOpenFile(from, TD_FILE_READ); + if (pFileFrom == NULL) goto _err; + + // fidto = open(to, O_WRONLY | O_CREAT | O_EXCL, 0755); + TdFilePtr pFileTo = taosOpenFile(to, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_EXCL); + if (pFileTo == NULL) goto _err; + + while (true) { + bytes = taosReadFile(pFileFrom, buffer, sizeof(buffer)); + if (bytes < 0) goto _err; + if (bytes == 0) break; + + size += bytes; + + if (taosWriteFile(pFileTo, (void *)buffer, bytes) < bytes) goto _err; + if (bytes < sizeof(buffer)) break; + } + + taosFsyncFile(pFileTo); + + taosCloseFile(&pFileFrom); + taosCloseFile(&pFileTo); + return size; + +_err: + if (pFileFrom != NULL) taosCloseFile(&pFileFrom); + if (pFileTo != NULL) taosCloseFile(&pFileTo); + remove(to); + return -1; +#endif +} + +int32_t taosRenameFile(const char *oldName, const char *newName) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + int32_t code = MoveFileEx(oldName, newName, MOVEFILE_REPLACE_EXISTING | MOVEFILE_COPY_ALLOWED); + if (code < 0) { + // printf("failed to rename file %s to %s, reason:%s", oldName, newName, strerror(errno)); + } + + return code; +#else + int32_t code = rename(oldName, newName); + if (code < 0) { + // printf("failed to rename file %s to %s, reason:%s", oldName, newName, strerror(errno)); + } + + return code; +#endif +} + +int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return 0; +#else + struct stat fileStat; + int32_t code = stat(path, &fileStat); + if (code < 0) { + return code; + } + + if (size != NULL) { + *size = fileStat.st_size; + } + + if (mtime != NULL) { + *mtime = fileStat.st_mtime; + } + + return 0; +#endif +} + +void autoDelFileListAdd(const char *path) { return; } + +TdFilePtr taosOpenFile(const char *path, int32_t tdFileOptions) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return NULL; +#else + int access = O_BINARY; + char *mode = NULL; + access |= (tdFileOptions & TD_FILE_CTEATE) ? O_CREAT : 0; + if ((tdFileOptions & TD_FILE_WRITE) && (tdFileOptions & TD_FILE_READ)) { + access |= O_RDWR; + mode = (tdFileOptions & TD_FILE_TEXT) ? "rt+" : "rb+"; + } else if (tdFileOptions & TD_FILE_WRITE) { + access |= O_WRONLY; + mode = (tdFileOptions & TD_FILE_TEXT) ? "wt" : "wb"; + } else if (tdFileOptions & TD_FILE_READ) { + access |= O_RDONLY; + mode = (tdFileOptions & TD_FILE_TEXT) ? "rt" : "rb"; + } + access |= (tdFileOptions & TD_FILE_TRUNC) ? O_TRUNC : 0; + access |= (tdFileOptions & TD_FILE_APPEND) ? O_APPEND : 0; + access |= (tdFileOptions & TD_FILE_TEXT) ? O_TEXT : 0; + access |= (tdFileOptions & TD_FILE_EXCL) ? O_EXCL : 0; + if (tdFileOptions & TD_FILE_AUTO_DEL) { + autoDelFileListAdd(path); + } + int fd = open(path, access, S_IRWXU | S_IRWXG | S_IRWXO); + if (fd == -1) { + return NULL; + } + FILE *fp = fdopen(fd, mode); + if (fp == NULL) { + close(fd); + return NULL; + } + TdFilePtr pFile = (TdFilePtr)malloc(sizeof(TdFile)); + if (pFile == NULL) { + close(fd); + fclose(fp); + return NULL; + } + pFile->fd = fd; + pFile->fp = fp; + pFile->refId = 0; + return pFile; +#endif +} + +int64_t taosCloseFile(TdFilePtr *ppFile) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return 0; +#else + if (ppFile == NULL || *ppFile == NULL || (*ppFile)->fd == -1) { + return 0; + } + fflush((*ppFile)->fp); + close((*ppFile)->fd); + (*ppFile)->fd = -1; + (*ppFile)->fp = NULL; + (*ppFile)->refId = 0; + free(*ppFile); + *ppFile = NULL; + return 0; +#endif +} + +int64_t taosReadFile(TdFilePtr pFile, void *buf, int64_t count) { + if (pFile == NULL) { + return 0; + } int64_t leftbytes = count; int64_t readbytes; - char * tbuf = (char *)buf; + char *tbuf = (char *)buf; while (leftbytes > 0) { - readbytes = read(fd, (void *)tbuf, (uint32_t)leftbytes); + readbytes = read(pFile->fd, (void *)tbuf, (uint32_t)leftbytes); if (readbytes < 0) { if (errno == EINTR) { continue; @@ -120,13 +277,20 @@ int64_t taosReadFile(FileFd fd, void *buf, int64_t count) { return count; } -int64_t taosWriteFile(FileFd fd, const void *buf, int64_t n) { - int64_t nleft = n; +int64_t taosPReadFile(TdFilePtr pFile, void *buf, int64_t count, int64_t offset) { + if (pFile == NULL) { + return 0; + } + return pread(pFile->fd, buf, count, offset); +} + +int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count) { + int64_t nleft = count; int64_t nwritten = 0; - char * tbuf = (char *)buf; + char *tbuf = (char *)buf; while (nleft > 0) { - nwritten = write(fd, (void *)tbuf, (uint32_t)nleft); + nwritten = write(pFile->fd, (void *)tbuf, (uint32_t)nleft); if (nwritten < 0) { if (errno == EINTR) { continue; @@ -137,48 +301,120 @@ int64_t taosWriteFile(FileFd fd, const void *buf, int64_t n) { tbuf += nwritten; } - return n; + return count; } -int64_t taosLSeekFile(FileFd fd, int64_t offset, int32_t whence) { return (int64_t)lseek(fd, (long)offset, whence); } +int64_t taosLSeekFile(TdFilePtr pFile, int64_t offset, int32_t whence) { + if (pFile == NULL) return -1; + return (int64_t)lseek(pFile->fd, (long)offset, whence); +} -int64_t taosCopyFile(const char *from, const char *to) { +int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime) { #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) return 0; #else - char buffer[4096]; - int fidto = -1, fidfrom = -1; - int64_t size = 0; - int64_t bytes; - - fidfrom = open(from, O_RDONLY); - if (fidfrom < 0) goto _err; - - fidto = open(to, O_WRONLY | O_CREAT | O_EXCL, 0755); - if (fidto < 0) goto _err; - - while (true) { - bytes = taosReadFile(fidfrom, buffer, sizeof(buffer)); - if (bytes < 0) goto _err; - if (bytes == 0) break; - - size += bytes; - - if (taosWriteFile(fidto, (void *)buffer, bytes) < bytes) goto _err; - if (bytes < sizeof(buffer)) break; + struct stat fileStat; + int32_t code = fstat(pFile->fd, &fileStat); + if (code < 0) { + return code; } - taosFsyncFile(fidto); + if (size != NULL) { + *size = fileStat.st_size; + } - taosCloseFile(fidfrom); - taosCloseFile(fidto); - return size; + if (mtime != NULL) { + *mtime = fileStat.st_mtime; + } -_err: - if (fidfrom >= 0) taosCloseFile(fidfrom); - if (fidto >= 0) taosCloseFile(fidto); - remove(to); - return -1; + return 0; +#endif +} + +int32_t taosLockFile(TdFilePtr pFile) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return 0; +#else + return (int32_t)flock(pFile->fd, LOCK_EX | LOCK_NB); +#endif +} + +int32_t taosUnLockFile(TdFilePtr pFile) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return 0; +#else + return (int32_t)flock(pFile->fd, LOCK_UN | LOCK_NB); +#endif +} + +int32_t taosFtruncateFile(TdFilePtr pFile, int64_t l_size) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + if (pFile->fd < 0) { + errno = EBADF; + uError("%s\n", "fd arg was negative"); + return -1; + } + + HANDLE h = (HANDLE)_get_osfhandle(pFile->fd); + + LARGE_INTEGER li_0; + li_0.QuadPart = (int64_t)0; + BOOL cur = SetFilePointerEx(h, li_0, NULL, FILE_CURRENT); + if (!cur) { + uError("SetFilePointerEx Error getting current position in file.\n"); + return -1; + } + + LARGE_INTEGER li_size; + li_size.QuadPart = l_size; + BOOL cur2 = SetFilePointerEx(h, li_size, NULL, FILE_BEGIN); + if (cur2 == 0) { + int error = GetLastError(); + uError("SetFilePointerEx GetLastError is: %d\n", error); + switch (error) { + case ERROR_INVALID_HANDLE: + errno = EBADF; + break; + default: + errno = EIO; + break; + } + return -1; + } + + if (!SetEndOfFile(h)) { + int error = GetLastError(); + uError("SetEndOfFile GetLastError is:%d", error); + switch (error) { + case ERROR_INVALID_HANDLE: + errno = EBADF; + break; + default: + errno = EIO; + break; + } + return -1; + } + + return 0; +#else + return ftruncate(pFile->fd, l_size); +#endif +} + +int32_t taosFsyncFile(TdFilePtr pFile) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + if (pFile->fd < 0) { + errno = EBADF; + uError("%s\n", "fd arg was negative"); + return -1; + } + + HANDLE h = (HANDLE)_get_osfhandle(pFile->fd); + + return FlushFileBuffers(h); +#else + return fflush(pFile->fp); #endif } @@ -261,7 +497,7 @@ int64_t taosFSendFile(FILE *out_file, FILE *in_file, int64_t *offset, int64_t co } off_t len = count; while (len > 0) { - char buf[1024 * 16]; + char buf[1024 * 16]; off_t n = sizeof(buf); if (len < n) n = len; size_t m = fread(buf, 1, n, in_file); @@ -286,7 +522,7 @@ int64_t taosSendFile(SocketFd dfd, FileFd sfd, int64_t *offset, int64_t count) { } off_t len = count; while (len > 0) { - char buf[1024 * 16]; + char buf[1024 * 16]; off_t n = sizeof(buf); if (len < n) n = len; size_t m = read(sfd, buf, n); @@ -301,12 +537,12 @@ int64_t taosSendFile(SocketFd dfd, FileFd sfd, int64_t *offset, int64_t count) { #else -int64_t taosSendFile(SocketFd dfd, FileFd sfd, int64_t *offset, int64_t size) { +int64_t taosSendFile(SocketFd fdDst, TdFilePtr pFileSrc, int64_t *offset, int64_t size) { int64_t leftbytes = size; int64_t sentbytes; while (leftbytes > 0) { - sentbytes = sendfile(dfd, sfd, offset, leftbytes); + sentbytes = sendfile(fdDst, pFileSrc->fd, offset, leftbytes); if (sentbytes == -1) { if (errno == EINTR || errno == EAGAIN || errno == EWOULDBLOCK) { continue; @@ -323,215 +559,41 @@ int64_t taosSendFile(SocketFd dfd, FileFd sfd, int64_t *offset, int64_t size) { return size; } -int64_t taosFSendFile(FILE *outfile, FILE *infile, int64_t *offset, int64_t size) { - return taosSendFile(fileno(outfile), fileno(infile), offset, size); +int64_t taosFSendFile(TdFilePtr pFileOut, TdFilePtr pFileIn, int64_t *offset, int64_t size) { + return taosSendFile(pFileOut->fd, pFileIn, offset, size); } #endif -int32_t taosFtruncateFile(FileFd fd, int64_t l_size) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - if (fd < 0) { - errno = EBADF; - uError("%s\n", "fd arg was negative"); - return -1; - } - - HANDLE h = (HANDLE)_get_osfhandle(fd); - - LARGE_INTEGER li_0; - li_0.QuadPart = (int64_t)0; - BOOL cur = SetFilePointerEx(h, li_0, NULL, FILE_CURRENT); - if (!cur) { - uError("SetFilePointerEx Error getting current position in file.\n"); - return -1; - } - - LARGE_INTEGER li_size; - li_size.QuadPart = l_size; - BOOL cur2 = SetFilePointerEx(h, li_size, NULL, FILE_BEGIN); - if (cur2 == 0) { - int error = GetLastError(); - uError("SetFilePointerEx GetLastError is: %d\n", error); - switch (error) { - case ERROR_INVALID_HANDLE: - errno = EBADF; - break; - default: - errno = EIO; - break; - } - return -1; - } - - if (!SetEndOfFile(h)) { - int error = GetLastError(); - uError("SetEndOfFile GetLastError is:%d", error); - switch (error) { - case ERROR_INVALID_HANDLE: - errno = EBADF; - break; - default: - errno = EIO; - break; - } - return -1; - } - - return 0; -#else - return ftruncate(fd, l_size); -#endif +void taosFprintfFile(TdFilePtr pFile, const char *format, ...) { + char buffer[MAX_FPRINTFLINE_BUFFER_SIZE] = {0}; + va_list ap; + va_start(ap, format); + vfprintf(pFile->fp, format, ap); + va_end(ap); + fflush(pFile->fp); } -int32_t taosFsyncFile(FileFd fd) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - if (fd < 0) { - errno = EBADF; - uError("%s\n", "fd arg was negative"); - return -1; - } +void *taosMmapReadOnlyFile(TdFilePtr pFile, int64_t length) { + if (pFile == NULL) return NULL; - HANDLE h = (HANDLE)_get_osfhandle(fd); - - return FlushFileBuffers(h); -#else - return fsync(fd); -#endif + void *ptr = mmap(NULL, length, PROT_READ, MAP_SHARED, pFile->fd, 0); + return ptr; } -int32_t taosRenameFile(const char *oldName, const char *newName) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - int32_t code = MoveFileEx(oldName, newName, MOVEFILE_REPLACE_EXISTING | MOVEFILE_COPY_ALLOWED); - if (code < 0) { - //printf("failed to rename file %s to %s, reason:%s", oldName, newName, strerror(errno)); - } +bool taosValidFile(TdFilePtr pFile) { return pFile != NULL; } - return code; -#else - int32_t code = rename(oldName, newName); - if (code < 0) { - //printf("failed to rename file %s to %s, reason:%s", oldName, newName, strerror(errno)); - } - - return code; -#endif -} - -int32_t taosLockFile(int32_t fd) { +int32_t taosUmaskFile(int32_t maskVal) { #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) return 0; #else - return (int32_t)flock(fd, LOCK_EX | LOCK_NB); + return umask(maskVal); #endif } -int32_t taosUnLockFile(int32_t fd) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - return 0; -#else - return (int32_t)flock(fd, LOCK_UN | LOCK_NB); -#endif +int32_t taosGetErrorFile(TdFilePtr pFile) { return errno; } +int64_t taosGetLineFile(TdFilePtr pFile, char **__restrict__ ptrBuf) { + size_t len = 0; + return getline(ptrBuf, &len, pFile->fp); } - -int32_t taosUmaskFile(int32_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - return 0; -#else - return umask(val); -#endif -} - -int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - return 0; -#else - struct stat fileStat; - int32_t code = stat(path, &fileStat); - if (code < 0) { - return code; - } - - if (size != NULL) { - *size = fileStat.st_size; - } - - if (mtime != NULL) { - *mtime = fileStat.st_mtime; - } - - return 0; -#endif -} - -int32_t taosFStatFile(int32_t fd, int64_t *size, int32_t *mtime) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - return 0; -#else - struct stat fileStat; - int32_t code = fstat(fd, &fileStat); - if (code < 0) { - return code; - } - - if (size != NULL) { - *size = fileStat.st_size; - } - - if (mtime != NULL) { - *mtime = fileStat.st_mtime; - } - - return 0; -#endif -} - -int32_t taosOpenFileWrite(const char *path) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - return 0; -#else - return open(path, O_WRONLY, S_IRWXU | S_IRWXG | S_IRWXO); -#endif -} - -int32_t taosOpenFileCreateWrite(const char *path) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - return 0; -#else - return open(path, O_WRONLY | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO); -#endif -} - -int32_t taosOpenFileCreateWriteTrunc(const char *path) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - return 0; -#else - return open(path, O_WRONLY | O_CREAT | O_TRUNC, S_IRWXU | S_IRWXG | S_IRWXO); -#endif -} - -int32_t taosOpenFileCreateWriteAppend(const char *path) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - return 0; -#else - return open(path, O_WRONLY | O_CREAT | O_APPEND, S_IRWXU | S_IRWXG | S_IRWXO); -#endif -} - -FileFd taosOpenFileRead(const char *path) { - #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - return 0; -#else - return open(path, O_RDONLY, S_IRWXU | S_IRWXG | S_IRWXO); -#endif -} - -FileFd taosOpenFileReadWrite(const char *path) { - #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - return 0; -#else - return open(path, O_RDWR, S_IRWXU | S_IRWXG | S_IRWXO); -#endif -} - - +int32_t taosEOFFile(TdFilePtr pFile) { return feof(pFile->fp); } \ No newline at end of file diff --git a/source/os/src/osLocale.c b/source/os/src/osLocale.c new file mode 100644 index 0000000000..47546f7deb --- /dev/null +++ b/source/os/src/osLocale.c @@ -0,0 +1,189 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "osLocale.h" + +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#if (_WIN64) +#include +#include +#include +#include +#include +#include +#pragma comment(lib, "Mswsock.lib ") +#endif +#include +#pragma warning(push) +#pragma warning(disable : 4091) +#include +#pragma warning(pop) +#elif defined(_TD_DARWIN_64) +#include +#include +#else +#include +#include +#include +#include +#include +#include +#include +#include +#endif + +typedef struct CharsetPair { + char *oldCharset; + char *newCharset; +} CharsetPair; + +char *taosCharsetReplace(char *charsetstr) { + CharsetPair charsetRep[] = { + {"utf8", "UTF-8"}, + {"936", "CP936"}, + }; + + for (int32_t i = 0; i < tListLen(charsetRep); ++i) { + if (strcasecmp(charsetRep[i].oldCharset, charsetstr) == 0) { + return strdup(charsetRep[i].newCharset); + } + } + + return strdup(charsetstr); +} + +/** + * In some Linux systems, setLocale(LC_CTYPE, "") may return NULL, in which case the launch of + * both the TDengine Server and the Client may be interrupted. + * + * In case that the setLocale failed to be executed, the right charset needs to be set. + */ +void taosSetSystemLocale(const char *inLocale, const char *inCharSet) { + char *locale = setlocale(LC_CTYPE, inLocale); + + // default locale or user specified locale is not valid, abort launch + if (inLocale == NULL || strlen(inLocale) == 0) { + //printf("Invalid locale:%s, please set the valid locale in config file\n", inLocale); + } + + if (!taosValidateEncodec(inCharSet)) { + printf("Invalid charset:%s, please set the valid charset in config file", inCharSet); + exit(-1); + } +} + +void taosGetSystemLocale(char *outLocale, char *outCharset) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + char *locale = setlocale(LC_CTYPE, "chs"); + if (locale != NULL) { + tstrncpy(outLocale, locale, TD_LOCALE_LEN); + } + strcpy(outCharset, "cp936"); + +#elif defined(_TD_DARWIN_64) + /* + * originally from src/os/src/detail/osSysinfo.c + * POSIX format locale string: + * (Language Strings)_(Country/Region Strings).(code_page) + * + * example: en_US.UTF-8, zh_CN.GB18030, zh_CN.UTF-8, + * + * if user does not specify the locale in taos.cfg the program use default LC_CTYPE as system locale. + * + * In case of some CentOS systems, their default locale is "en_US.utf8", which is not valid code_page + * for libiconv that is employed to convert string in this system. This program will automatically use + * UTF-8 instead as the charset. + * + * In case of windows client, the locale string is not valid POSIX format, user needs to set the + * correct code_page for libiconv. Usually, the code_page of windows system with simple chinese is + * CP936, CP437 for English charset. + * + */ + + char sep = '.'; + char *locale = NULL; + + locale = setlocale(LC_CTYPE, ""); + if (locale == NULL) { + // printf("can't get locale from system, set it to en_US.UTF-8 since error:%d:%s", errno, strerror(errno)); + strcpy(outLocale, "en_US.UTF-8"); + } else { + tstrncpy(outLocale, locale, TD_LOCALE_LEN); + // printf("locale not configured, set to system default:%s", outLocale); + } + + /* if user does not specify the charset, extract it from locale */ + char *str = strrchr(outLocale, sep); + if (str != NULL) { + str++; + + char *revisedCharset = taosCharsetReplace(str); + tstrncpy(outCharset, revisedCharset, TD_CHARSET_LEN); + + free(revisedCharset); + // printf("charset not configured, set to system default:%s", outCharset); + } else { + strcpy(outCharset, "UTF-8"); + // printf("can't get locale and charset from system, set it to UTF-8"); + } + +#else + /* + * POSIX format locale string: + * (Language Strings)_(Country/Region Strings).(code_page) + * + * example: en_US.UTF-8, zh_CN.GB18030, zh_CN.UTF-8, + * + * if user does not specify the locale in taos.cfg the program use default LC_CTYPE as system locale. + * + * In case of some CentOS systems, their default locale is "en_US.utf8", which is not valid code_page + * for libiconv that is employed to convert string in this system. This program will automatically use + * UTF-8 instead as the charset. + * + * In case of windows client, the locale string is not valid POSIX format, user needs to set the + * correct code_page for libiconv. Usually, the code_page of windows system with simple chinese is + * CP936, CP437 for English charset. + * + */ + char sep = '.'; + char *locale = NULL; + + locale = setlocale(LC_CTYPE, ""); + if (locale == NULL) { + // printf("can't get locale from system, set it to en_US.UTF-8 since error:%d:%s", errno, strerror(errno)); + strcpy(outLocale, "en_US.UTF-8"); + } else { + tstrncpy(outLocale, locale, TD_LOCALE_LEN); + // printf("locale not configured, set to system default:%s", outLocale); + } + + // if user does not specify the charset, extract it from locale + char *str = strrchr(outLocale, sep); + if (str != NULL) { + str++; + + char *revisedCharset = taosCharsetReplace(str); + tstrncpy(outCharset, revisedCharset, TD_LOCALE_LEN); + + free(revisedCharset); + // printf("charset not configured, set to system default:%s", outCharset); + } else { + strcpy(outCharset, "UTF-8"); + // printf("can't get locale and charset from system, set it to UTF-8"); + } + +#endif +} diff --git a/source/os/src/osRand.c b/source/os/src/osRand.c index 973323a346..b81e41b3cf 100644 --- a/source/os/src/osRand.c +++ b/source/os/src/osRand.c @@ -24,18 +24,18 @@ uint32_t taosRand(void) { return rand(); } uint32_t taosSafeRand(void) { - int fd; + TdFilePtr pFile; int seed; - fd = open("/dev/urandom", 0); - if (fd < 0) { + pFile = taosOpenFile("/dev/urandom", TD_FILE_READ); + if (pFile == NULL) { seed = (int)time(0); } else { - int len = read(fd, &seed, sizeof(seed)); + int len = taosReadFile(pFile, &seed, sizeof(seed)); if (len < 0) { seed = (int)time(0); } - close(fd); + taosCloseFile(&pFile); } return (uint32_t)seed; diff --git a/source/os/src/osSocket.c b/source/os/src/osSocket.c index e12f9493b3..07d30276b7 100644 --- a/source/os/src/osSocket.c +++ b/source/os/src/osSocket.c @@ -14,6 +14,7 @@ */ #define _DEFAULT_SOURCE +#define ALLOW_FORBID_FUNC #include "os.h" #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) @@ -150,7 +151,7 @@ int32_t taosGetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *op #if !((defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)) && defined(_MSC_VER)) -uint32_t taosInetAddr(char *ipAddr) { return inet_addr(ipAddr); } +uint32_t taosInetAddr(const char *ipAddr) { return inet_addr(ipAddr); } const char *taosInetNtoa(struct in_addr ipInt) { return inet_ntoa(ipInt); } @@ -240,7 +241,7 @@ int32_t taosSetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *op #ifdef _MSC_VER //#if _MSC_VER >= 1900 -uint32_t taosInetAddr(char *ipAddr) { +uint32_t taosInetAddr(const char *ipAddr) { uint32_t value; int32_t ret = inet_pton(AF_INET, ipAddr, &value); if (ret <= 0) { diff --git a/source/os/src/osString.c b/source/os/src/osString.c index f714fd0dc4..88ea4b3e15 100644 --- a/source/os/src/osString.c +++ b/source/os/src/osString.c @@ -18,24 +18,6 @@ #include #include -typedef struct CharsetPair { - char *oldCharset; - char *newCharset; -} CharsetPair; - -char *taosCharsetReplace(char *charsetstr) { - CharsetPair charsetRep[] = { - { "utf8", "UTF-8" }, { "936", "CP936" }, - }; - - for (int32_t i = 0; i < tListLen(charsetRep); ++i) { - if (strcasecmp(charsetRep[i].oldCharset, charsetstr) == 0) { - return strdup(charsetRep[i].newCharset); - } - } - - return strdup(charsetstr); -} int64_t taosStr2int64(const char *str) { char *endptr = NULL; diff --git a/source/os/src/osSysinfo.c b/source/os/src/osSysinfo.c index cae1b18b3c..45749588c1 100644 --- a/source/os/src/osSysinfo.c +++ b/source/os/src/osSysinfo.c @@ -16,15 +16,6 @@ #define _DEFAULT_SOURCE #include "os.h" -int32_t tsTotalMemoryMB = 0; -int64_t tsPageSize = 0; -int64_t tsOpenMax = 0; -int64_t tsStreamMax = 0; -int32_t tsNumOfCores = 1; -char tsTimezone[TSDB_TIMEZONE_LEN] = {0}; -char tsLocale[TSDB_LOCALE_LEN] = {0}; -char tsCharset[TSDB_LOCALE_LEN] = {0}; // default encode string - #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) /* @@ -89,25 +80,6 @@ bool taosGetProcMemory(float *memoryUsedMB) { return true; } -static void taosGetSystemTimezone() { - // get and set default timezone - char *tz = getenv("TZ"); - if (tz == NULL || strlen(tz) == 0) { - strcpy(tsTimezone, "not configured"); - } else { - strcpy(tsTimezone, tz); - } -} - -static void taosGetSystemLocale() { - // get and set default locale - char *locale = setlocale(LC_CTYPE, "chs"); - if (locale != NULL) { - tstrncpy(tsLocale, locale, TSDB_LOCALE_LEN); - } - - strcpy(tsCharset, "cp936"); -} int32_t taosGetCpuCores() { SYSTEM_INFO info; @@ -200,8 +172,6 @@ void taosGetSystemInfo() { taosGetCpuUsage(&tmp1, &tmp2); taosGetProcIO(&tmp1, &tmp2); - taosGetSystemTimezone(); - taosGetSystemLocale(); } void taosKillSystem() { @@ -273,103 +243,6 @@ char *taosGetCmdlineByPID(int pid) { return ""; } #include #include -static void taosGetSystemTimezone() { - /* load time zone string from /etc/localtime */ - char buf[4096]; - char *tz = NULL; - { - int n = readlink("/etc/localtime", buf, sizeof(buf)); - if (n < 0) { - //printf("read /etc/localtime error, reason:%s", strerror(errno)); - return; - } - buf[n] = '\0'; - for (int i = n - 1; i >= 0; --i) { - if (buf[i] == '/') { - if (tz) { - tz = buf + i + 1; - break; - } - tz = buf + i + 1; - } - } - if (!tz || 0 == strchr(tz, '/')) { - //printf("parsing /etc/localtime failed"); - return; - } - - setenv("TZ", tz, 1); - tzset(); - } - - /* - * NOTE: do not remove it. - * Enforce set the correct daylight saving time(DST) flag according - * to current time - */ - time_t tx1 = time(NULL); - struct tm tm1; - localtime_r(&tx1, &tm1); - - /* - * format example: - * - * Asia/Shanghai (CST, +0800) - * Europe/London (BST, +0100) - */ - snprintf(tsTimezone, TSDB_TIMEZONE_LEN, "%s (%s, %+03ld00)", tz, tm1.tm_isdst ? tzname[daylight] : tzname[0], - -timezone / 3600); - - // cfg_timezone->cfgStatus = TAOS_CFG_CSTATUS_DEFAULT; - //printf("timezone not configured, set to system default:%s", tsTimezone); -} - -/* - * originally from src/os/src/detail/osSysinfo.c - * POSIX format locale string: - * (Language Strings)_(Country/Region Strings).(code_page) - * - * example: en_US.UTF-8, zh_CN.GB18030, zh_CN.UTF-8, - * - * if user does not specify the locale in taos.cfg the program use default LC_CTYPE as system locale. - * - * In case of some CentOS systems, their default locale is "en_US.utf8", which is not valid code_page - * for libiconv that is employed to convert string in this system. This program will automatically use - * UTF-8 instead as the charset. - * - * In case of windows client, the locale string is not valid POSIX format, user needs to set the - * correct code_page for libiconv. Usually, the code_page of windows system with simple chinese is - * CP936, CP437 for English charset. - * - */ -static void taosGetSystemLocale() { // get and set default locale - char sep = '.'; - char *locale = NULL; - - locale = setlocale(LC_CTYPE, ""); - if (locale == NULL) { - //printf("can't get locale from system, set it to en_US.UTF-8 since error:%d:%s", errno, strerror(errno)); - strcpy(tsLocale, "en_US.UTF-8"); - } else { - tstrncpy(tsLocale, locale, TSDB_LOCALE_LEN); - //printf("locale not configured, set to system default:%s", tsLocale); - } - - /* if user does not specify the charset, extract it from locale */ - char *str = strrchr(tsLocale, sep); - if (str != NULL) { - str++; - - char *revisedCharset = taosCharsetReplace(str); - tstrncpy(tsCharset, revisedCharset, TSDB_LOCALE_LEN); - - free(revisedCharset); - //printf("charset not configured, set to system default:%s", tsCharset); - } else { - strcpy(tsCharset, "UTF-8"); - //printf("can't get locale and charset from system, set it to UTF-8"); - } -} void taosKillSystem() { //printf("function taosKillSystem, exit!"); @@ -386,9 +259,6 @@ void taosGetSystemInfo() { long page_size = sysconf(_SC_PAGESIZE); tsTotalMemoryMB = physical_pages * page_size / (1024 * 1024); tsPageSize = page_size; - - taosGetSystemTimezone(); - taosGetSystemLocale(); } bool taosReadProcIO(int64_t *rchars, int64_t *wchars) { @@ -533,19 +403,17 @@ bool taosGetSysMemory(float *memoryUsedMB) { } bool taosGetProcMemory(float *memoryUsedMB) { - FILE *fp = fopen(tsProcMemFile, "r"); - if (fp == NULL) { + // FILE *fp = fopen(tsProcMemFile, "r"); + TdFilePtr pFile = taosOpenFile(tsProcMemFile, TD_FILE_READ); + if (pFile == NULL) { //printf("open file:%s failed", tsProcMemFile); return false; } ssize_t _bytes = 0; - size_t len; char * line = NULL; - while (!feof(fp)) { - tfree(line); - len = 0; - _bytes = getline(&line, &len, fp); + while (!taosEOFFile(pFile)) { + _bytes = taosGetLineFile(pFile, &line); if ((_bytes < 0) || (line == NULL)) { break; } @@ -556,7 +424,7 @@ bool taosGetProcMemory(float *memoryUsedMB) { if (line == NULL) { //printf("read file:%s failed", tsProcMemFile); - fclose(fp); + taosCloseFile(&pFile); return false; } @@ -565,24 +433,24 @@ bool taosGetProcMemory(float *memoryUsedMB) { sscanf(line, "%s %" PRId64, tmp, &memKB); *memoryUsedMB = (float)((double)memKB / 1024); - tfree(line); - fclose(fp); + if(line != NULL) tfree(line); + taosCloseFile(&pFile); return true; } static bool taosGetSysCpuInfo(SysCpuInfo *cpuInfo) { - FILE *fp = fopen(tsSysCpuFile, "r"); - if (fp == NULL) { + // FILE *fp = fopen(tsSysCpuFile, "r"); + TdFilePtr pFile = taosOpenFile(tsSysCpuFile, TD_FILE_READ); + if (pFile == NULL) { //printf("open file:%s failed", tsSysCpuFile); return false; } - size_t len; char * line = NULL; - ssize_t _bytes = getline(&line, &len, fp); + ssize_t _bytes = taosGetLineFile(pFile, &line); if ((_bytes < 0) || (line == NULL)) { //printf("read file:%s failed", tsSysCpuFile); - fclose(fp); + taosCloseFile(&pFile); return false; } @@ -590,24 +458,24 @@ static bool taosGetSysCpuInfo(SysCpuInfo *cpuInfo) { sscanf(line, "%s %" PRIu64 " %" PRIu64 " %" PRIu64 " %" PRIu64, cpu, &cpuInfo->user, &cpuInfo->nice, &cpuInfo->system, &cpuInfo->idle); - tfree(line); - fclose(fp); + if(line != NULL) tfree(line); + taosCloseFile(&pFile); return true; } static bool taosGetProcCpuInfo(ProcCpuInfo *cpuInfo) { - FILE *fp = fopen(tsProcCpuFile, "r"); - if (fp == NULL) { + // FILE *fp = fopen(tsProcCpuFile, "r"); + TdFilePtr pFile = taosOpenFile(tsProcCpuFile, TD_FILE_READ); + if (pFile == NULL) { //printf("open file:%s failed", tsProcCpuFile); return false; } - size_t len = 0; char * line = NULL; - ssize_t _bytes = getline(&line, &len, fp); + ssize_t _bytes = taosGetLineFile(pFile, &line); if ((_bytes < 0) || (line == NULL)) { //printf("read file:%s failed", tsProcCpuFile); - fclose(fp); + taosCloseFile(&pFile); return false; } @@ -620,115 +488,11 @@ static bool taosGetProcCpuInfo(ProcCpuInfo *cpuInfo) { } } - tfree(line); - fclose(fp); + if(line != NULL) tfree(line); + taosCloseFile(&pFile); return true; } -static void taosGetSystemTimezone() { - /* - * NOTE: do not remove it. - * Enforce set the correct daylight saving time(DST) flag according - * to current time - */ - time_t tx1 = time(NULL); - struct tm tm1; - localtime_r(&tx1, &tm1); - - /* load time zone string from /etc/timezone */ - FILE *f = fopen("/etc/timezone", "r"); - char buf[68] = {0}; - if (f != NULL) { - int len = fread(buf, 64, 1, f); - if (len < 64 && ferror(f)) { - fclose(f); - //printf("read /etc/timezone error, reason:%s", strerror(errno)); - return; - } - - fclose(f); - - buf[sizeof(buf) - 1] = 0; - char *lineEnd = strstr(buf, "\n"); - if (lineEnd != NULL) { - *lineEnd = 0; - } - - // for CentOS system, /etc/timezone does not exist. Ignore the TZ environment variables - if (strlen(buf) > 0) { - setenv("TZ", buf, 1); - } - } - // get and set default timezone - tzset(); - - /* - * get CURRENT time zone. - * system current time zone is affected by daylight saving time(DST) - * - * e.g., the local time zone of London in DST is GMT+01:00, - * otherwise is GMT+00:00 - */ - int32_t tz = (-timezone * MILLISECOND_PER_SECOND) / MILLISECOND_PER_HOUR; - tz += daylight; - - /* - * format example: - * - * Asia/Shanghai (CST, +0800) - * Europe/London (BST, +0100) - */ - snprintf(tsTimezone, TSDB_TIMEZONE_LEN, "%s (%s, %s%02d00)", buf, tzname[daylight], tz >= 0 ? "+" : "-", abs(tz)); - - // cfg_timezone->cfgStatus = TAOS_CFG_CSTATUS_DEFAULT; - //printf("timezone not configured, set to system default:%s", tsTimezone); -} - -/* - * POSIX format locale string: - * (Language Strings)_(Country/Region Strings).(code_page) - * - * example: en_US.UTF-8, zh_CN.GB18030, zh_CN.UTF-8, - * - * if user does not specify the locale in taos.cfg the program use default LC_CTYPE as system locale. - * - * In case of some CentOS systems, their default locale is "en_US.utf8", which is not valid code_page - * for libiconv that is employed to convert string in this system. This program will automatically use - * UTF-8 instead as the charset. - * - * In case of windows client, the locale string is not valid POSIX format, user needs to set the - * correct code_page for libiconv. Usually, the code_page of windows system with simple chinese is - * CP936, CP437 for English charset. - * - */ -static void taosGetSystemLocale() { // get and set default locale - char sep = '.'; - char *locale = NULL; - - locale = setlocale(LC_CTYPE, ""); - if (locale == NULL) { - //printf("can't get locale from system, set it to en_US.UTF-8 since error:%d:%s", errno, strerror(errno)); - strcpy(tsLocale, "en_US.UTF-8"); - } else { - tstrncpy(tsLocale, locale, TSDB_LOCALE_LEN); - //printf("locale not configured, set to system default:%s", tsLocale); - } - - // if user does not specify the charset, extract it from locale - char *str = strrchr(tsLocale, sep); - if (str != NULL) { - str++; - - char *revisedCharset = taosCharsetReplace(str); - tstrncpy(tsCharset, revisedCharset, TSDB_LOCALE_LEN); - - free(revisedCharset); - //printf("charset not configured, set to system default:%s", tsCharset); - } else { - strcpy(tsCharset, "UTF-8"); - //printf("can't get locale and charset from system, set it to UTF-8"); - } -} int32_t taosGetCpuCores() { return (int32_t)sysconf(_SC_NPROCESSORS_ONLN); } @@ -785,19 +549,17 @@ int32_t taosGetDiskSize(char *dataDir, SDiskSize *diskSize) { bool taosGetCardInfo(int64_t *bytes, int64_t *rbytes, int64_t *tbytes) { *bytes = 0; - FILE *fp = fopen(tsSysNetFile, "r"); - if (fp == NULL) { + // FILE *fp = fopen(tsSysNetFile, "r"); + TdFilePtr pFile = taosOpenFile(tsSysNetFile, TD_FILE_READ); + if (pFile == NULL) { //printf("open file:%s failed", tsSysNetFile); return false; } ssize_t _bytes = 0; - size_t len = 2048; - char * line = calloc(1, len); - - while (!feof(fp)) { - memset(line, 0, len); + char * line = NULL; + while (!taosEOFFile(pFile)) { int64_t o_rbytes = 0; int64_t rpackts = 0; int64_t o_tbytes = 0; @@ -810,12 +572,12 @@ bool taosGetCardInfo(int64_t *bytes, int64_t *rbytes, int64_t *tbytes) { int64_t nouse6 = 0; char nouse0[200] = {0}; - _bytes = getline(&line, &len, fp); + _bytes = taosGetLineFile(pFile, &line); if (_bytes < 0) { break; } - line[len - 1] = 0; + line[_bytes - 1] = 0; if (strstr(line, "lo:") != NULL) { continue; @@ -830,8 +592,8 @@ bool taosGetCardInfo(int64_t *bytes, int64_t *rbytes, int64_t *tbytes) { *bytes += (o_rbytes + o_tbytes); } - tfree(line); - fclose(fp); + if(line != NULL) tfree(line); + taosCloseFile(&pFile); return true; } @@ -873,22 +635,20 @@ bool taosGetBandSpeed(float *bandSpeedKb) { } bool taosReadProcIO(int64_t *rchars, int64_t *wchars) { - FILE *fp = fopen(tsProcIOFile, "r"); - if (fp == NULL) { + // FILE *fp = fopen(tsProcIOFile, "r"); + TdFilePtr pFile = taosOpenFile(tsProcIOFile, TD_FILE_READ); + if (pFile == NULL) { //printf("open file:%s failed", tsProcIOFile); return false; } ssize_t _bytes = 0; - size_t len; char * line = NULL; char tmp[10]; int readIndex = 0; - while (!feof(fp)) { - tfree(line); - len = 0; - _bytes = getline(&line, &len, fp); + while (!taosEOFFile(pFile)) { + _bytes = taosGetLineFile(pFile, &line); if ((_bytes < 0) || (line == NULL)) { break; } @@ -904,8 +664,8 @@ bool taosReadProcIO(int64_t *rchars, int64_t *wchars) { if (readIndex >= 2) break; } - tfree(line); - fclose(fp); + if(line != NULL) tfree(line); + taosCloseFile(&pFile); if (readIndex < 2) { //printf("read file:%s failed", tsProcIOFile); @@ -957,8 +717,6 @@ void taosGetSystemInfo() { taosGetCpuUsage(&tmp1, &tmp2); taosGetProcIO(&tmp1, &tmp2); - taosGetSystemTimezone(); - taosGetSystemLocale(); } void taosKillSystem() { @@ -1070,15 +828,15 @@ void taosSetCoreDump(bool enable) { } int32_t taosGetSystemUUID(char *uid, int32_t uidlen) { - int fd; int len = 0; - fd = open("/proc/sys/kernel/random/uuid", 0); - if (fd < 0) { + // fd = open("/proc/sys/kernel/random/uuid", 0); + TdFilePtr pFile = taosOpenFile("/proc/sys/kernel/random/uuid", TD_FILE_READ); + if (pFile == NULL) { return -1; } else { - len = read(fd, uid, uidlen); - close(fd); + len = taosReadFile(pFile, uid, uidlen); + taosCloseFile(&pFile); } if (len >= 36) { @@ -1093,16 +851,17 @@ char *taosGetCmdlineByPID(int pid) { static char cmdline[1024]; sprintf(cmdline, "/proc/%d/cmdline", pid); - int fd = open(cmdline, O_RDONLY); - if (fd >= 0) { - int n = read(fd, cmdline, sizeof(cmdline) - 1); + // int fd = open(cmdline, O_RDONLY); + TdFilePtr pFile = taosOpenFile(cmdline, TD_FILE_READ); + if (pFile != NULL) { + int n = taosReadFile(pFile, cmdline, sizeof(cmdline) - 1); if (n < 0) n = 0; if (n > 0 && cmdline[n - 1] == '\n') --n; cmdline[n] = 0; - close(fd); + taosCloseFile(&pFile); } else { cmdline[0] = 0; } diff --git a/source/os/src/osTimezone.c b/source/os/src/osTimezone.c new file mode 100644 index 0000000000..07846781ad --- /dev/null +++ b/source/os/src/osTimezone.c @@ -0,0 +1,199 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" + +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#if (_WIN64) +#include +#include +#include +#include +#include +#include +#pragma comment(lib, "Mswsock.lib ") +#endif +#include +#pragma warning(push) +#pragma warning(disable : 4091) +#include +#pragma warning(pop) +#elif defined(_TD_DARWIN_64) +#include +#include +#else +#include +#include +#include +#include +#include +#include +#include +#include +#endif + +void taosSetSystemTimezone(const char *inTimezone, char *outTimezone, int8_t *outDaylight) { +#ifdef WINDOWS + char winStr[TD_LOCALE_LEN * 2]; + sprintf(winStr, "TZ=%s", inTimezone); + putenv(winStr); +#else + setenv("TZ", inTimezone, 1); +#endif + tzset(); + + /* + * get CURRENT time zone. + * system current time zone is affected by daylight saving time(DST) + * + * e.g., the local time zone of London in DST is GMT+01:00, + * otherwise is GMT+00:00 + */ +#ifdef _MSC_VER +#if _MSC_VER >= 1900 + // see https://docs.microsoft.com/en-us/cpp/c-runtime-library/daylight-dstbias-timezone-and-tzname?view=vs-2019 + int64_t timezone = _timezone; + int32_t daylight = _daylight; + char **tzname = _tzname; +#endif +#endif + + int32_t tz = (int32_t)((-timezone * MILLISECOND_PER_SECOND) / MILLISECOND_PER_HOUR); + tz += daylight; + + /* + * format: + * (CST, +0800) + * (BST, +0100) + */ + + sprintf(outTimezone, "(%s, %s%02d00)", tzname[daylight], tz >= 0 ? "+" : "-", abs(tz)); + *outDaylight = daylight; +} + +void taosGetSystemTimezone(char *outTimezone) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + char *tz = getenv("TZ"); + if (tz == NULL || strlen(tz) == 0) { + strcpy(outTimezone, "not configured"); + } else { + strcpy(outTimezone, tz); + } + +#elif defined(_TD_DARWIN_64) + char buf[4096] = {0}; + char *tz = NULL; + { + int n = readlink("/etc/localtime", buf, sizeof(buf)); + if (n < 0) { + printf("read /etc/localtime error, reason:%s", strerror(errno)); + return; + } + buf[n] = '\0'; + for (int i = n - 1; i >= 0; --i) { + if (buf[i] == '/') { + if (tz) { + tz = buf + i + 1; + break; + } + tz = buf + i + 1; + } + } + if (!tz || 0 == strchr(tz, '/')) { + printf("parsing /etc/localtime failed"); + return; + } + + setenv("TZ", tz, 1); + tzset(); + } + + /* + * NOTE: do not remove it. + * Enforce set the correct daylight saving time(DST) flag according + * to current time + */ + time_t tx1 = time(NULL); + struct tm tm1; + localtime_r(&tx1, &tm1); + + /* + * format example: + * + * Asia/Shanghai (CST, +0800) + * Europe/London (BST, +0100) + */ + snprintf(outTimezone, TD_TIMEZONE_LEN, "%s (%s, %+03ld00)", tz, tm1.tm_isdst ? tzname[daylight] : tzname[0], + -timezone / 3600); + +#else + /* + * NOTE: do not remove it. + * Enforce set the correct daylight saving time(DST) flag according + * to current time + */ + time_t tx1 = time(NULL); + struct tm tm1; + localtime_r(&tx1, &tm1); + + /* load time zone string from /etc/timezone */ + // FILE *f = fopen("/etc/timezone", "r"); + TdFilePtr pFile = taosOpenFile("/etc/timezone", TD_FILE_READ); + char buf[68] = {0}; + if (pFile != NULL) { + int len = taosReadFile(pFile, buf, 64); + if (len < 64 && taosGetErrorFile(pFile)) { + taosCloseFile(&pFile); + // printf("read /etc/timezone error, reason:%s", strerror(errno)); + return; + } + + taosCloseFile(&pFile); + + buf[sizeof(buf) - 1] = 0; + char *lineEnd = strstr(buf, "\n"); + if (lineEnd != NULL) { + *lineEnd = 0; + } + + // for CentOS system, /etc/timezone does not exist. Ignore the TZ environment variables + if (strlen(buf) > 0) { + setenv("TZ", buf, 1); + } + } + // get and set default timezone + tzset(); + + /* + * get CURRENT time zone. + * system current time zone is affected by daylight saving time(DST) + * + * e.g., the local time zone of London in DST is GMT+01:00, + * otherwise is GMT+00:00 + */ + int32_t tz = (-timezone * MILLISECOND_PER_SECOND) / MILLISECOND_PER_HOUR; + tz += daylight; + + /* + * format example: + * + * Asia/Shanghai (CST, +0800) + * Europe/London (BST, +0100) + */ + snprintf(outTimezone, TD_TIMEZONE_LEN, "%s (%s, %s%02d00)", buf, tzname[daylight], tz >= 0 ? "+" : "-", abs(tz)); + +#endif +} diff --git a/source/util/src/compare.c b/source/util/src/compare.c index 6124de9459..b597eb32bf 100644 --- a/source/util/src/compare.c +++ b/source/util/src/compare.c @@ -22,7 +22,7 @@ #include "regex.h" #include "thash.h" #include "types.h" -#include "ulog.h" +#include "tlog.h" #include "tdef.h" #include "taos.h" @@ -466,7 +466,7 @@ int32_t compareWStrPatternNotMatch(const void* pLeft, const void* pRight) { __compar_fn_t getComparFunc(int32_t type, int32_t optr) { __compar_fn_t comparFn = NULL; - if (optr == TSDB_RELATION_IN && (type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR)) { + if (optr == OP_TYPE_IN && (type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR)) { switch (type) { case TSDB_DATA_TYPE_BOOL: case TSDB_DATA_TYPE_TINYINT: @@ -489,7 +489,7 @@ __compar_fn_t getComparFunc(int32_t type, int32_t optr) { } } - if (optr == TSDB_RELATION_NOT_IN && (type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR)) { + if (optr == OP_TYPE_NOT_IN && (type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR)) { switch (type) { case TSDB_DATA_TYPE_BOOL: case TSDB_DATA_TYPE_TINYINT: @@ -522,17 +522,17 @@ __compar_fn_t getComparFunc(int32_t type, int32_t optr) { case TSDB_DATA_TYPE_FLOAT: comparFn = compareFloatVal; break; case TSDB_DATA_TYPE_DOUBLE: comparFn = compareDoubleVal; break; case TSDB_DATA_TYPE_BINARY: { - if (optr == TSDB_RELATION_MATCH) { + if (optr == OP_TYPE_MATCH) { comparFn = compareStrRegexCompMatch; - } else if (optr == TSDB_RELATION_NMATCH) { + } else if (optr == OP_TYPE_NMATCH) { comparFn = compareStrRegexCompNMatch; - } else if (optr == TSDB_RELATION_LIKE) { /* wildcard query using like operator */ + } else if (optr == OP_TYPE_LIKE) { /* wildcard query using like operator */ comparFn = compareStrPatternMatch; - } else if (optr == TSDB_RELATION_NOT_LIKE) { /* wildcard query using like operator */ + } else if (optr == OP_TYPE_NOT_LIKE) { /* wildcard query using like operator */ comparFn = compareStrPatternNotMatch; - } else if (optr == TSDB_RELATION_IN) { + } else if (optr == OP_TYPE_IN) { comparFn = compareChkInString; - } else if (optr == TSDB_RELATION_NOT_IN) { + } else if (optr == OP_TYPE_NOT_IN) { comparFn = compareChkNotInString; } else { /* normal relational comparFn */ comparFn = compareLenPrefixedStr; @@ -542,17 +542,17 @@ __compar_fn_t getComparFunc(int32_t type, int32_t optr) { } case TSDB_DATA_TYPE_NCHAR: { - if (optr == TSDB_RELATION_MATCH) { + if (optr == OP_TYPE_MATCH) { comparFn = compareStrRegexCompMatch; - } else if (optr == TSDB_RELATION_NMATCH) { + } else if (optr == OP_TYPE_NMATCH) { comparFn = compareStrRegexCompNMatch; - } else if (optr == TSDB_RELATION_LIKE) { + } else if (optr == OP_TYPE_LIKE) { comparFn = compareWStrPatternMatch; - } else if (optr == TSDB_RELATION_NOT_LIKE) { + } else if (optr == OP_TYPE_NOT_LIKE) { comparFn = compareWStrPatternNotMatch; - } else if (optr == TSDB_RELATION_IN) { + } else if (optr == OP_TYPE_IN) { comparFn = compareChkInString; - } else if (optr == TSDB_RELATION_NOT_IN) { + } else if (optr == OP_TYPE_NOT_IN) { comparFn = compareChkNotInString; } else { comparFn = compareLenPrefixedWStr; diff --git a/source/util/src/tcache.c b/source/util/src/tcache.c index 621f103207..6914e7e2d0 100644 --- a/source/util/src/tcache.c +++ b/source/util/src/tcache.c @@ -15,7 +15,7 @@ #define _DEFAULT_SOURCE #include "os.h" -#include "ulog.h" +#include "tlog.h" #include "ttimer.h" #include "tutil.h" #include "tcache.h" diff --git a/source/util/src/tcompression.c b/source/util/src/tcompression.c index b0dcff88e7..03a4846c3d 100644 --- a/source/util/src/tcompression.c +++ b/source/util/src/tcompression.c @@ -53,7 +53,7 @@ #include "td_sz.h" #endif #include "tcompression.h" -#include "ulog.h" +#include "tlog.h" static const int TEST_NUMBER = 1; #define is_bigendian() ((*(char *)&TEST_NUMBER) == 0) diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index 469da11d93..0ee1b9e1d6 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -14,364 +14,644 @@ */ #define _DEFAULT_SOURCE -#include "os.h" #include "tconfig.h" +#include "taoserror.h" +#include "tcfg.h" +#include "thash.h" #include "tutil.h" -#include "ulog.h" +#include "tlog.h" -SGlobalCfg tsGlobalConfig[TSDB_CFG_MAX_NUM] = {{0}}; -int32_t tsGlobalConfigNum = 0; +#define CFG_NAME_PRINT_LEN 24 +#define CFG_SRC_PRINT_LEN 12 -static char *tsGlobalUnit[] = { - " ", - "(%)", - "(GB)", - "(Mb)", - "(byte)", - "(s)", - "(ms)" -}; +typedef struct SConfig { + ECfgSrcType stype; + SHashObj *hash; +} SConfig; -char *tsCfgStatusStr[] = { - "none", - "system default", - "config file", - "taos_options", - "program argument list" -}; +int32_t cfgLoadFromCfgFile(SConfig *pConfig, const char *filepath); +int32_t cfgLoadFromEnvFile(SConfig *pConfig, const char *filepath); +int32_t cfgLoadFromEnvVar(SConfig *pConfig); +int32_t cfgLoadFromApollUrl(SConfig *pConfig, const char *url); +int32_t cfgSetItem(SConfig *pConfig, const char *name, const char *value, ECfgSrcType stype); -static void taosReadFloatConfig(SGlobalCfg *cfg, char *input_value) { - float value = (float)atof(input_value); - float *option = (float *)cfg->ptr; - if (value < cfg->minValue || value > cfg->maxValue) { - uError("config option:%s, input value:%s, out of range[%f, %f], use default value:%f", - cfg->option, input_value, cfg->minValue, cfg->maxValue, *option); - } else { - if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_FILE) { - *option = value; - cfg->cfgStatus = TAOS_CFG_CSTATUS_FILE; - } else { - uWarn("config option:%s, input value:%s, is configured by %s, use %f", cfg->option, input_value, - tsCfgStatusStr[cfg->cfgStatus], *option); - } +SConfig *cfgInit() { + SConfig *pCfg = calloc(1, sizeof(SConfig)); + if (pCfg == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + pCfg->hash = taosHashInit(16, MurmurHash3_32, false, HASH_NO_LOCK); + if (pCfg->hash == NULL) { + free(pCfg); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + return pCfg; +} + +int32_t cfgLoad(SConfig *pCfg, ECfgSrcType cfgType, const char *sourceStr) { + switch (cfgType) { + case CFG_STYPE_CFG_FILE: + return cfgLoadFromCfgFile(pCfg, sourceStr); + case CFG_STYPE_ENV_FILE: + return cfgLoadFromEnvFile(pCfg, sourceStr); + case CFG_STYPE_ENV_VAR: + return cfgLoadFromEnvVar(pCfg); + case CFG_STYPE_APOLLO_URL: + return cfgLoadFromApollUrl(pCfg, sourceStr); + default: + return -1; } } -static void taosReadDoubleConfig(SGlobalCfg *cfg, char *input_value) { - double value = atof(input_value); - double *option = (double *)cfg->ptr; - if (value < cfg->minValue || value > cfg->maxValue) { - uError("config option:%s, input value:%s, out of range[%f, %f], use default value:%f", - cfg->option, input_value, cfg->minValue, cfg->maxValue, *option); - } else { - if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_FILE) { - *option = value; - cfg->cfgStatus = TAOS_CFG_CSTATUS_FILE; - } else { - uWarn("config option:%s, input value:%s, is configured by %s, use %f", cfg->option, input_value, - tsCfgStatusStr[cfg->cfgStatus], *option); - } - } -} - -static void taosReadInt32Config(SGlobalCfg *cfg, char *input_value) { - int32_t value = atoi(input_value); - int32_t *option = (int32_t *)cfg->ptr; - if (value < cfg->minValue || value > cfg->maxValue) { - uError("config option:%s, input value:%s, out of range[%f, %f], use default value:%d", - cfg->option, input_value, cfg->minValue, cfg->maxValue, *option); - } else { - if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_FILE) { - *option = value; - cfg->cfgStatus = TAOS_CFG_CSTATUS_FILE; - } else { - uWarn("config option:%s, input value:%s, is configured by %s, use %d", cfg->option, input_value, - tsCfgStatusStr[cfg->cfgStatus], *option); - } - } -} - -static void taosReadInt16Config(SGlobalCfg *cfg, char *input_value) { - int32_t value = atoi(input_value); - int16_t *option = (int16_t *)cfg->ptr; - if (value < cfg->minValue || value > cfg->maxValue) { - uError("config option:%s, input value:%s, out of range[%f, %f], use default value:%d", - cfg->option, input_value, cfg->minValue, cfg->maxValue, *option); - } else { - if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_FILE) { - *option = (int16_t)value; - cfg->cfgStatus = TAOS_CFG_CSTATUS_FILE; - } else { - uWarn("config option:%s, input value:%s, is configured by %s, use %d", cfg->option, input_value, - tsCfgStatusStr[cfg->cfgStatus], *option); - } - } -} - -static void taosReadUInt16Config(SGlobalCfg *cfg, char *input_value) { - int32_t value = atoi(input_value); - uint16_t *option = (uint16_t *)cfg->ptr; - if (value < cfg->minValue || value > cfg->maxValue) { - uError("config option:%s, input value:%s, out of range[%f, %f], use default value:%d", - cfg->option, input_value, cfg->minValue, cfg->maxValue, *option); - } else { - if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_FILE) { - *option = (uint16_t)value; - cfg->cfgStatus = TAOS_CFG_CSTATUS_FILE; - } else { - uWarn("config option:%s, input value:%s, is configured by %s, use %d", cfg->option, input_value, - tsCfgStatusStr[cfg->cfgStatus], *option); - } - } -} - -static void taosReadInt8Config(SGlobalCfg *cfg, char *input_value) { - int32_t value = atoi(input_value); - int8_t *option = (int8_t *)cfg->ptr; - if (value < cfg->minValue || value > cfg->maxValue) { - uError("config option:%s, input value:%s, out of range[%f, %f], use default value:%d", - cfg->option, input_value, cfg->minValue, cfg->maxValue, *option); - } else { - if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_FILE) { - *option = (int8_t)value; - cfg->cfgStatus = TAOS_CFG_CSTATUS_FILE; - } else { - uWarn("config option:%s, input value:%s, is configured by %s, use %d", cfg->option, input_value, - tsCfgStatusStr[cfg->cfgStatus], *option); - } - } -} - -static bool taosReadDirectoryConfig(SGlobalCfg *cfg, char *input_value) { - int length = (int)strlen(input_value); - char *option = (char *)cfg->ptr; - if (length <= 0 || length > cfg->ptrLength) { - uError("config option:%s, input value:%s, length out of range[0, %d], use default value:%s", cfg->option, - input_value, cfg->ptrLength, option); - return false; - } else { - if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_FILE) { - taosExpandDir(input_value, option, cfg->ptrLength); - taosRealPath(option, cfg->ptrLength); - - if (taosMkDir(option) != 0) { - uError("config option:%s, input value:%s, directory not exist, create fail:%s", cfg->option, input_value, - strerror(errno)); - return false; - } - cfg->cfgStatus = TAOS_CFG_CSTATUS_FILE; - } else { - uWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, input_value, - tsCfgStatusStr[cfg->cfgStatus], option); - } - } - - return true; -} - -static void taosReadIpStrConfig(SGlobalCfg *cfg, char *input_value) { - uint32_t value = taosInetAddr(input_value); - char * option = (char *)cfg->ptr; - if (value == INADDR_NONE) { - uError("config option:%s, input value:%s, is not a valid ip address, use default value:%s", - cfg->option, input_value, option); - } else { - if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_FILE) { - strncpy(option, input_value, cfg->ptrLength); - cfg->cfgStatus = TAOS_CFG_CSTATUS_FILE; - } else { - uWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, input_value, - tsCfgStatusStr[cfg->cfgStatus], option); - } - } -} - -static void taosReadStringConfig(SGlobalCfg *cfg, char *input_value) { - int length = (int) strlen(input_value); - char *option = (char *)cfg->ptr; - if (length <= 0 || length > cfg->ptrLength) { - uError("config option:%s, input value:%s, length out of range[0, %d], use default value:%s", - cfg->option, input_value, cfg->ptrLength, option); - } else { - if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_FILE) { - strncpy(option, input_value, cfg->ptrLength); - cfg->cfgStatus = TAOS_CFG_CSTATUS_FILE; - } else { - uWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, input_value, - tsCfgStatusStr[cfg->cfgStatus], option); - } - } -} - -static void taosReadLogOption(char *option, char *value) { - for (int i = 0; i < tsGlobalConfigNum; ++i) { - SGlobalCfg *cfg = tsGlobalConfig + i; - if (!(cfg->cfgType & TSDB_CFG_CTYPE_B_CONFIG) || !(cfg->cfgType & TSDB_CFG_CTYPE_B_LOG)) continue; - if (strcasecmp(cfg->option, option) != 0) continue; - - switch (cfg->valType) { - case TAOS_CFG_VTYPE_INT32: - taosReadInt32Config(cfg, value); - // if (strcasecmp(cfg->option, "debugFlag") == 0) { - // taosSetAllDebugFlag(); - // } - break; - case TAOS_CFG_VTYPE_DIRECTORY: - taosReadDirectoryConfig(cfg, value); - break; - default: - break; - } - break; - } -} - -SGlobalCfg *taosGetConfigOption(const char *option) { - for (int i = 0; i < tsGlobalConfigNum; ++i) { - SGlobalCfg *cfg = tsGlobalConfig + i; - if (strcasecmp(cfg->option, option) != 0) continue; - return cfg; - } - return NULL; -} - -static void taosReadConfigOption(const char *option, char *value, char *value2, char *value3) { - for (int i = 0; i < tsGlobalConfigNum; ++i) { - SGlobalCfg *cfg = tsGlobalConfig + i; - if (!(cfg->cfgType & TSDB_CFG_CTYPE_B_CONFIG)) continue; - if (strcasecmp(cfg->option, option) != 0) continue; - - switch (cfg->valType) { - case TAOS_CFG_VTYPE_INT8: - taosReadInt8Config(cfg, value); - break; - case TAOS_CFG_VTYPE_INT16: - taosReadInt16Config(cfg, value); - break; - case TAOS_CFG_VTYPE_INT32: - taosReadInt32Config(cfg, value); - break; - case TAOS_CFG_VTYPE_UINT16: - taosReadUInt16Config(cfg, value); - break; - case TAOS_CFG_VTYPE_FLOAT: - taosReadFloatConfig(cfg, value); - break; - case TAOS_CFG_VTYPE_DOUBLE: - taosReadDoubleConfig(cfg, value); - break; - case TAOS_CFG_VTYPE_STRING: - taosReadStringConfig(cfg, value); - break; - case TAOS_CFG_VTYPE_IPSTR: - taosReadIpStrConfig(cfg, value); - break; - case TAOS_CFG_VTYPE_DIRECTORY: - taosReadDirectoryConfig(cfg, value); - break; - case TAOS_CFG_VTYPE_DATA_DIRCTORY: - if (taosReadDirectoryConfig(cfg, value)) { - // taosReadDataDirCfg(value, value2, value3); - } - break; - default: - uError("config option:%s, input value:%s, can't be recognized", option, value); - break; - } - break; - } -} - -void taosAddConfigOption(SGlobalCfg cfg) { - tsGlobalConfig[tsGlobalConfigNum++] = cfg; -} - -void taosReadGlobalLogCfg() { - FILE * fp; - char * line, *option, *value; - int olen, vlen; - char fileName[PATH_MAX] = {0}; - - taosExpandDir(configDir, configDir, PATH_MAX); - taosReadLogOption("logDir", tsLogDir); - - sprintf(fileName, "%s/taos.cfg", configDir); - fp = fopen(fileName, "r"); - if (fp == NULL) { - printf("\nconfig file:%s not found, all variables are set to default\n", fileName); - return; - } - - ssize_t _bytes = 0; - size_t len = 1024; - line = calloc(1, len); - - while (!feof(fp)) { - memset(line, 0, len); - - option = value = NULL; - olen = vlen = 0; - - _bytes = tgetline(&line, &len, fp); - if (_bytes < 0) - { - break; - } - - line[len - 1] = 0; - - paGetToken(line, &option, &olen); - if (olen == 0) continue; - option[olen] = 0; - - paGetToken(option + olen + 1, &value, &vlen); - if (vlen == 0) continue; - value[vlen] = 0; - - taosReadLogOption(option, value); - } - - tfree(line); - fclose(fp); -} - -int32_t taosReadCfgFromFile() { - char * line, *option, *value, *value2, *value3; - int olen, vlen, vlen2, vlen3; - char fileName[PATH_MAX] = {0}; - - sprintf(fileName, "%s/taos.cfg", configDir); - FILE *fp = fopen(fileName, "r"); - if (fp == NULL) { - fp = fopen(configDir, "r"); - if (fp == NULL) { +int32_t cfgLoadArray(SConfig *pCfg, SArray *pArgs) { + int32_t size = taosArrayGetSize(pArgs); + for (int32_t i = 0; i < size; ++i) { + SConfigPair *pPair = taosArrayGet(pArgs, i); + if (cfgSetItem(pCfg, pPair->name, pPair->value, CFG_STYPE_ARG_LIST) != 0) { return -1; } } - ssize_t _bytes = 0; - size_t len = 1024; - line = calloc(1, len); - - while (!feof(fp)) { - memset(line, 0, len); + return 0; +} - option = value = value2 = value3 = NULL; +static void cfgFreeItem(SConfigItem *pItem) { + if (pItem->dtype == CFG_DTYPE_STRING || pItem->dtype == CFG_DTYPE_DIR || pItem->dtype == CFG_DTYPE_LOCALE || + pItem->dtype == CFG_DTYPE_CHARSET || pItem->dtype == CFG_DTYPE_TIMEZONE) { + tfree(pItem->str); + } + if (pItem->array) { + taosArrayDestroy(pItem->array); + pItem->array = NULL; + } +} + +void cfgCleanup(SConfig *pCfg) { + if (pCfg != NULL) { + if (pCfg->hash != NULL) { + SConfigItem *pItem = taosHashIterate(pCfg->hash, NULL); + while (pItem != NULL) { + cfgFreeItem(pItem); + pItem = taosHashIterate(pCfg->hash, pItem); + } + taosHashCleanup(pCfg->hash); + pCfg->hash == NULL; + } + free(pCfg); + } +} + +int32_t cfgGetSize(SConfig *pCfg) { return taosHashGetSize(pCfg->hash); } + +SConfigItem *cfgIterate(SConfig *pCfg, SConfigItem *pIter) { return taosHashIterate(pCfg->hash, pIter); } + +void cfgCancelIterate(SConfig *pCfg, SConfigItem *pIter) { return taosHashCancelIterate(pCfg->hash, pIter); } + +static int32_t cfgCheckAndSetTimezone(SConfigItem *pItem, const char *timezone) { + cfgFreeItem(pItem); + pItem->str = strdup(timezone); + if (pItem->str == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + return 0; +} + +static int32_t cfgCheckAndSetCharset(SConfigItem *pItem, const char *charset) { + cfgFreeItem(pItem); + pItem->str = strdup(charset); + if (pItem->str == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + return 0; +} + +static int32_t cfgCheckAndSetLocale(SConfigItem *pItem, const char *locale) { + cfgFreeItem(pItem); + pItem->str = strdup(locale); + if (pItem->str == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + return 0; +} + +static int32_t cfgCheckAndSetDir(SConfigItem *pItem, const char *inputDir) { + char fullDir[PATH_MAX] = {0}; + if (taosExpandDir(inputDir, fullDir, PATH_MAX) != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + uError("failed to expand dir:%s since %s", inputDir, terrstr()); + return -1; + } + + if (taosRealPath(fullDir, PATH_MAX) != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + uError("failed to get realpath of dir:%s since %s", inputDir, terrstr()); + return -1; + } + + if (taosMkDir(fullDir) != 0) { + uError("failed to create dir:%s realpath:%s since %s", inputDir, fullDir, terrstr()); + return -1; + } + + cfgFreeItem(pItem); + pItem->str = strdup(fullDir); + if (pItem->str == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + return 0; +} + +static int32_t cfgSetBool(SConfigItem *pItem, const char *value, ECfgSrcType stype) { + bool tmp = false; + if (strcasecmp(value, "true") == 0) { + tmp = true; + } + if (atoi(value) > 0) { + tmp = true; + } + + pItem->bval = tmp; + pItem->stype = stype; + return 0; +} + +static int32_t cfgSetInt32(SConfigItem *pItem, const char *value, ECfgSrcType stype) { + int32_t ival = (int32_t)atoi(value); + if (ival < pItem->imin || ival > pItem->imax) { + uError("cfg:%s, type:%s src:%s value:%d out of range[%" PRId64 ", %" PRId64 "], use last src:%s value:%d", + pItem->name, cfgDtypeStr(pItem->dtype), cfgStypeStr(stype), ival, pItem->imin, pItem->imax, + cfgStypeStr(pItem->stype), pItem->i32); + terrno = TSDB_CODE_OUT_OF_RANGE; + return -1; + } + + pItem->i32 = ival; + pItem->stype = stype; + return 0; +} + +static int32_t cfgSetInt64(SConfigItem *pItem, const char *value, ECfgSrcType stype) { + int64_t ival = (int64_t)atoi(value); + if (ival < pItem->imin || ival > pItem->imax) { + uError("cfg:%s, type:%s src:%s value:%" PRId64 " out of range[%" PRId64 ", %" PRId64 + "], use last src:%s value:%" PRId64, + pItem->name, cfgDtypeStr(pItem->dtype), cfgStypeStr(stype), ival, pItem->imin, pItem->imax, + cfgStypeStr(pItem->stype), pItem->i64); + terrno = TSDB_CODE_OUT_OF_RANGE; + return -1; + } + + pItem->i64 = ival; + pItem->stype = stype; + return 0; +} + +static int32_t cfgSetFloat(SConfigItem *pItem, const char *value, ECfgSrcType stype) { + float fval = (float)atof(value); + if (fval < pItem->fmin || fval > pItem->fmax) { + uError("cfg:%s, type:%s src:%s value:%f out of range[%f, %f], use last src:%s value:%f", pItem->name, + cfgDtypeStr(pItem->dtype), cfgStypeStr(stype), fval, pItem->fmin, pItem->fmax, cfgStypeStr(pItem->stype), + pItem->fval); + terrno = TSDB_CODE_OUT_OF_RANGE; + return -1; + } + + pItem->fval = fval; + pItem->stype = stype; + return 0; +} + +static int32_t cfgSetString(SConfigItem *pItem, const char *value, ECfgSrcType stype) { + char *tmp = strdup(value); + if (tmp == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + uError("cfg:%s, type:%s src:%s value:%s failed to dup since %s, use last src:%s value:%s", pItem->name, + cfgDtypeStr(pItem->dtype), cfgStypeStr(stype), value, terrstr(), cfgStypeStr(pItem->stype), pItem->str); + return -1; + } + + free(pItem->str); + pItem->str = tmp; + pItem->stype = stype; + return 0; +} + +static int32_t cfgSetDir(SConfigItem *pItem, const char *value, ECfgSrcType stype) { + char *tmp = strdup(value); + if (tmp == NULL || cfgCheckAndSetDir(pItem, value) != 0) { + free(tmp); + terrno = TSDB_CODE_OUT_OF_MEMORY; + uError("cfg:%s, type:%s src:%s value:%s failed to dup since %s, use last src:%s value:%s", pItem->name, + cfgDtypeStr(pItem->dtype), cfgStypeStr(stype), value, terrstr(), cfgStypeStr(pItem->stype), pItem->str); + return -1; + } + + pItem->stype = stype; + return 0; +} + +static int32_t cfgSetLocale(SConfigItem *pItem, const char *value, ECfgSrcType stype) { + char *tmp = strdup(value); + if (tmp == NULL || cfgCheckAndSetLocale(pItem, value) != 0) { + free(tmp); + terrno = TSDB_CODE_OUT_OF_MEMORY; + uError("cfg:%s, type:%s src:%s value:%s failed to dup since %s, use last src:%s value:%s", pItem->name, + cfgDtypeStr(pItem->dtype), cfgStypeStr(stype), value, terrstr(), cfgStypeStr(pItem->stype), pItem->str); + return -1; + } + + pItem->stype = stype; + return 0; +} + +static int32_t cfgSetCharset(SConfigItem *pItem, const char *value, ECfgSrcType stype) { + char *tmp = strdup(value); + if (tmp == NULL || cfgCheckAndSetCharset(pItem, value) != 0) { + free(tmp); + terrno = TSDB_CODE_OUT_OF_MEMORY; + uError("cfg:%s, type:%s src:%s value:%s failed to dup since %s, use last src:%s value:%s", pItem->name, + cfgDtypeStr(pItem->dtype), cfgStypeStr(stype), value, terrstr(), cfgStypeStr(pItem->stype), pItem->str); + return -1; + } + + pItem->stype = stype; + return 0; +} + +static int32_t cfgSetTimezone(SConfigItem *pItem, const char *value, ECfgSrcType stype) { + char *tmp = strdup(value); + if (tmp == NULL || cfgCheckAndSetTimezone(pItem, value) != 0) { + free(tmp); + terrno = TSDB_CODE_OUT_OF_MEMORY; + uError("cfg:%s, type:%s src:%s value:%s failed to dup since %s, use last src:%s value:%s", pItem->name, + cfgDtypeStr(pItem->dtype), cfgStypeStr(stype), value, terrstr(), cfgStypeStr(pItem->stype), pItem->str); + return -1; + } + + pItem->stype = stype; + return 0; +} + +static int32_t cfgSetTfsItem(SConfig *pCfg, const char *name, const char *value, const char *level, const char *primary, + ECfgSrcType stype) { + SConfigItem *pItem = cfgGetItem(pCfg, name); + if (pItem == NULL) return -1; + + if (pItem->array == NULL) { + pItem->array = taosArrayInit(16, sizeof(SDiskCfg)); + if (pItem->array == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + } + + SDiskCfg cfg = {0}; + tstrncpy(cfg.dir, value, sizeof(cfg.dir)); + cfg.level = atoi(level); + cfg.primary = atoi(primary); + void *ret = taosArrayPush(pItem->array, &cfg); + if (ret == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + pItem->stype = stype; + return 0; +} + +int32_t cfgSetItem(SConfig *pCfg, const char *name, const char *value, ECfgSrcType stype) { + SConfigItem *pItem = cfgGetItem(pCfg, name); + if (pItem == NULL) { + return -1; + } + + switch (pItem->dtype) { + case CFG_DTYPE_BOOL: + return cfgSetBool(pItem, value, stype); + case CFG_DTYPE_INT32: + return cfgSetInt32(pItem, value, stype); + case CFG_DTYPE_INT64: + return cfgSetInt64(pItem, value, stype); + case CFG_DTYPE_FLOAT: + return cfgSetFloat(pItem, value, stype); + case CFG_DTYPE_STRING: + return cfgSetString(pItem, value, stype); + case CFG_DTYPE_DIR: + return cfgSetDir(pItem, value, stype); + case CFG_DTYPE_TIMEZONE: + return cfgSetTimezone(pItem, value, stype); + case CFG_DTYPE_CHARSET: + return cfgSetCharset(pItem, value, stype); + case CFG_DTYPE_LOCALE: + return cfgSetLocale(pItem, value, stype); + case CFG_DTYPE_NONE: + default: + break; + } + + terrno = TSDB_CODE_INVALID_CFG; + return -1; +} + +SConfigItem *cfgGetItem(SConfig *pCfg, const char *name) { + char lowcaseName[CFG_NAME_MAX_LEN + 1] = {0}; + memcpy(lowcaseName, name, CFG_NAME_MAX_LEN); + strntolower(lowcaseName, name, CFG_NAME_MAX_LEN); + + SConfigItem *pItem = taosHashGet(pCfg->hash, lowcaseName, strlen(lowcaseName) + 1); + if (pItem == NULL) { + terrno = TSDB_CODE_CFG_NOT_FOUND; + } + + return pItem; +} + +static int32_t cfgAddItem(SConfig *pCfg, SConfigItem *pItem, const char *name) { + pItem->stype = CFG_STYPE_DEFAULT; + pItem->name = strdup(name); + if (pItem->name == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + char lowcaseName[CFG_NAME_MAX_LEN + 1] = {0}; + memcpy(lowcaseName, name, CFG_NAME_MAX_LEN); + strntolower(lowcaseName, name, CFG_NAME_MAX_LEN); + + if (taosHashPut(pCfg->hash, lowcaseName, strlen(lowcaseName) + 1, pItem, sizeof(SConfigItem)) != 0) { + if (pItem->dtype == CFG_DTYPE_STRING) { + free(pItem->str); + } + free(pItem->name); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + return 0; +} + +int32_t cfgAddBool(SConfig *pCfg, const char *name, bool defaultVal, bool tsc) { + SConfigItem item = {.dtype = CFG_DTYPE_BOOL, .bval = defaultVal, .tsc = tsc}; + return cfgAddItem(pCfg, &item, name); +} + +int32_t cfgAddInt32(SConfig *pCfg, const char *name, int32_t defaultVal, int64_t minval, int64_t maxval, bool tsc) { + if (defaultVal < minval || defaultVal > maxval) { + terrno = TSDB_CODE_OUT_OF_RANGE; + return -1; + } + + SConfigItem item = {.dtype = CFG_DTYPE_INT32, .i32 = defaultVal, .imin = minval, .imax = maxval, .tsc = tsc}; + return cfgAddItem(pCfg, &item, name); +} + +int32_t cfgAddInt64(SConfig *pCfg, const char *name, int64_t defaultVal, int64_t minval, int64_t maxval, bool tsc) { + if (defaultVal < minval || defaultVal > maxval) { + terrno = TSDB_CODE_OUT_OF_RANGE; + return -1; + } + + SConfigItem item = {.dtype = CFG_DTYPE_INT64, .i64 = defaultVal, .imin = minval, .imax = maxval, .tsc = tsc}; + return cfgAddItem(pCfg, &item, name); +} + +int32_t cfgAddFloat(SConfig *pCfg, const char *name, float defaultVal, double minval, double maxval, bool tsc) { + if (defaultVal < minval || defaultVal > maxval) { + terrno = TSDB_CODE_OUT_OF_RANGE; + return -1; + } + + SConfigItem item = {.dtype = CFG_DTYPE_FLOAT, .fval = defaultVal, .fmin = minval, .fmax = maxval, .tsc = tsc}; + return cfgAddItem(pCfg, &item, name); +} + +int32_t cfgAddString(SConfig *pCfg, const char *name, const char *defaultVal, bool tsc) { + SConfigItem item = {.dtype = CFG_DTYPE_STRING, .tsc = tsc}; + item.str = strdup(defaultVal); + if (item.str == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + return cfgAddItem(pCfg, &item, name); +} + +int32_t cfgAddDir(SConfig *pCfg, const char *name, const char *defaultVal, bool tsc) { + SConfigItem item = {.dtype = CFG_DTYPE_DIR, .tsc = tsc}; + if (cfgCheckAndSetDir(&item, defaultVal) != 0) { + return -1; + } + + return cfgAddItem(pCfg, &item, name); +} + +int32_t cfgAddLocale(SConfig *pCfg, const char *name, const char *defaultVal) { + SConfigItem item = {.dtype = CFG_DTYPE_LOCALE, .tsc = 1}; + if (cfgCheckAndSetLocale(&item, defaultVal) != 0) { + return -1; + } + + return cfgAddItem(pCfg, &item, name); +} + +int32_t cfgAddCharset(SConfig *pCfg, const char *name, const char *defaultVal) { + SConfigItem item = {.dtype = CFG_DTYPE_CHARSET, .tsc = 1}; + if (cfgCheckAndSetCharset(&item, defaultVal) != 0) { + return -1; + } + + return cfgAddItem(pCfg, &item, name); +} + +int32_t cfgAddTimezone(SConfig *pCfg, const char *name, const char *defaultVal) { + SConfigItem item = {.dtype = CFG_DTYPE_TIMEZONE, .tsc = 1}; + if (cfgCheckAndSetTimezone(&item, defaultVal) != 0) { + return -1; + } + + return cfgAddItem(pCfg, &item, name); +} + +const char *cfgStypeStr(ECfgSrcType type) { + switch (type) { + case CFG_STYPE_DEFAULT: + return "default"; + case CFG_STYPE_CFG_FILE: + return "cfg_file"; + case CFG_STYPE_ENV_FILE: + return "env_file"; + case CFG_STYPE_ENV_VAR: + return "env_var"; + case CFG_STYPE_APOLLO_URL: + return "apollo_url"; + case CFG_STYPE_ARG_LIST: + return "arg_list"; + default: + return "invalid"; + } +} + +const char *cfgDtypeStr(ECfgDataType type) { + switch (type) { + case CFG_DTYPE_NONE: + return "none"; + case CFG_DTYPE_BOOL: + return "bool"; + case CFG_DTYPE_INT32: + return "int32"; + case CFG_DTYPE_INT64: + return "int64"; + case CFG_DTYPE_FLOAT: + return "float"; + case CFG_DTYPE_STRING: + return "string"; + case CFG_DTYPE_DIR: + return "dir"; + case CFG_DTYPE_LOCALE: + return "locale"; + case CFG_DTYPE_CHARSET: + return "charset"; + case CFG_DTYPE_TIMEZONE: + return "timezone"; + default: + return "invalid"; + } +} + +void cfgDumpCfg(SConfig *pCfg, bool tsc, bool dump) { + if (dump) { + printf(" global config"); + printf("\n"); + printf("================================================================="); + printf("\n"); + } else { + uInfo(" global config"); + uInfo("================================================================="); + } + + char src[CFG_SRC_PRINT_LEN + 1] = {0}; + char name[CFG_NAME_PRINT_LEN + 1] = {0}; + + SConfigItem *pItem = cfgIterate(pCfg, NULL); + while (pItem != NULL) { + if (tsc && !pItem->tsc) continue; + tstrncpy(src, cfgStypeStr(pItem->stype), CFG_SRC_PRINT_LEN); + for (int32_t i = 0; i < CFG_SRC_PRINT_LEN; ++i) { + if (src[i] == 0) src[i] = ' '; + } + + tstrncpy(name, pItem->name, CFG_NAME_PRINT_LEN); + for (int32_t i = 0; i < CFG_NAME_PRINT_LEN; ++i) { + if (name[i] == 0) name[i] = ' '; + } + + switch (pItem->dtype) { + case CFG_DTYPE_BOOL: + if (dump) { + printf("%s %s %u", src, name, pItem->bval); + printf("\n"); + } else { + uInfo("%s %s %u", src, name, pItem->bval); + } + + break; + case CFG_DTYPE_INT32: + if (dump) { + printf("%s %s %d", src, name, pItem->i32); + printf("\n"); + } else { + uInfo("%s %s %d", src, name, pItem->i32); + } + break; + case CFG_DTYPE_INT64: + if (dump) { + printf("%s %s %" PRId64, src, name, pItem->i64); + printf("\n"); + } else { + uInfo("%s %s %" PRId64, src, name, pItem->i64); + } + break; + case CFG_DTYPE_FLOAT: + if (dump) { + printf("%s %s %f", src, name, pItem->fval); + printf("\n"); + } else { + uInfo("%s %s %f", src, name, pItem->fval); + } + break; + case CFG_DTYPE_STRING: + case CFG_DTYPE_DIR: + case CFG_DTYPE_LOCALE: + case CFG_DTYPE_CHARSET: + case CFG_DTYPE_TIMEZONE: + if (dump) { + printf("%s %s %s", src, name, pItem->str); + printf("\n"); + } else { + uInfo("%s %s %s", src, name, pItem->str); + } + break; + } + pItem = cfgIterate(pCfg, pItem); + } + + if (dump) { + printf("================================================================="); + printf("\n"); + } else { + uInfo("================================================================="); + } +} + +int32_t cfgLoadFromEnvVar(SConfig *pConfig) { + uInfo("load from global env variables"); + return 0; +} + +int32_t cfgLoadFromEnvFile(SConfig *pConfig, const char *filepath) { + uInfo("load from env file %s", filepath); + return 0; +} + +int32_t cfgLoadFromCfgFile(SConfig *pConfig, const char *filepath) { + char *line = NULL, *name, *value, *value2, *value3; + int olen, vlen, vlen2, vlen3; + ssize_t _bytes = 0; + + // FILE *fp = fopen(filepath, "r"); + TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_READ); + if (pFile == NULL) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + while (!taosEOFFile(pFile)) { + name = value = value2 = value3 = NULL; olen = vlen = vlen2 = vlen3 = 0; - _bytes = tgetline(&line, &len, fp); - if (_bytes < 0) - { + _bytes = taosGetLineFile(pFile, &line); + if (_bytes < 0) { break; } - line[len - 1] = 0; - - paGetToken(line, &option, &olen); - if (olen == 0) continue; - option[olen] = 0; + line[_bytes - 1] = 0; - paGetToken(option + olen + 1, &value, &vlen); + paGetToken(line, &name, &olen); + if (olen == 0) continue; + name[olen] = 0; + + paGetToken(name + olen + 1, &value, &vlen); if (vlen == 0) continue; value[vlen] = 0; @@ -382,126 +662,20 @@ int32_t taosReadCfgFromFile() { if (vlen3 != 0) value3[vlen3] = 0; } - taosReadConfigOption(option, value, value2, value3); + cfgSetItem(pConfig, name, value, CFG_STYPE_CFG_FILE); + if (value2 != NULL && value3 != NULL && value2[0] != 0 && value3[0] != 0) { + cfgSetTfsItem(pConfig, name, value, value2, value3, CFG_STYPE_CFG_FILE); + } } - fclose(fp); - - tfree(line); - - // if (debugFlag & DEBUG_TRACE || debugFlag & DEBUG_DEBUG || debugFlag & DEBUG_DUMP) { - // taosSetAllDebugFlag(); - // } + taosCloseFile(&pFile); + if (line != NULL) tfree(line); + uInfo("load from cfg file %s success", filepath); return 0; } -void taosPrintCfg() { - uInfo(" taos config & system info:"); - uInfo("=================================="); - - for (int i = 0; i < tsGlobalConfigNum; ++i) { - SGlobalCfg *cfg = tsGlobalConfig + i; - if (tscEmbeddedInUtil == 0 && !(cfg->cfgType & TSDB_CFG_CTYPE_B_CLIENT)) continue; - if (cfg->cfgType & TSDB_CFG_CTYPE_B_NOT_PRINT) continue; - - int optionLen = (int)strlen(cfg->option); - int blankLen = TSDB_CFG_PRINT_LEN - optionLen; - blankLen = blankLen < 0 ? 0 : blankLen; - - char blank[TSDB_CFG_PRINT_LEN]; - memset(blank, ' ', TSDB_CFG_PRINT_LEN); - blank[blankLen] = 0; - - switch (cfg->valType) { - case TAOS_CFG_VTYPE_INT8: - uInfo(" %s:%s%d%s", cfg->option, blank, *((int8_t *)cfg->ptr), tsGlobalUnit[cfg->unitType]); - break; - case TAOS_CFG_VTYPE_INT16: - uInfo(" %s:%s%d%s", cfg->option, blank, *((int16_t *)cfg->ptr), tsGlobalUnit[cfg->unitType]); - break; - case TAOS_CFG_VTYPE_INT32: - uInfo(" %s:%s%d%s", cfg->option, blank, *((int32_t *)cfg->ptr), tsGlobalUnit[cfg->unitType]); - break; - case TAOS_CFG_VTYPE_UINT16: - uInfo(" %s:%s%d%s", cfg->option, blank, *((uint16_t *)cfg->ptr), tsGlobalUnit[cfg->unitType]); - break; - case TAOS_CFG_VTYPE_FLOAT: - uInfo(" %s:%s%f%s", cfg->option, blank, *((float *)cfg->ptr), tsGlobalUnit[cfg->unitType]); - break; - case TAOS_CFG_VTYPE_DOUBLE: - uInfo(" %s:%s%f%s", cfg->option, blank, *((double *)cfg->ptr), tsGlobalUnit[cfg->unitType]); - break; - case TAOS_CFG_VTYPE_STRING: - case TAOS_CFG_VTYPE_IPSTR: - case TAOS_CFG_VTYPE_DIRECTORY: - uInfo(" %s:%s%s%s", cfg->option, blank, (char *)cfg->ptr, tsGlobalUnit[cfg->unitType]); - break; - default: - break; - } - } - - taosPrintOsInfo(); - uInfo("=================================="); -} - -static void taosDumpCfg(SGlobalCfg *cfg) { - int optionLen = (int)strlen(cfg->option); - int blankLen = TSDB_CFG_PRINT_LEN - optionLen; - blankLen = blankLen < 0 ? 0 : blankLen; - - char blank[TSDB_CFG_PRINT_LEN]; - memset(blank, ' ', TSDB_CFG_PRINT_LEN); - blank[blankLen] = 0; - - switch (cfg->valType) { - case TAOS_CFG_VTYPE_INT8: - printf(" %s:%s%d%s\n", cfg->option, blank, *((int8_t *)cfg->ptr), tsGlobalUnit[cfg->unitType]); - break; - case TAOS_CFG_VTYPE_INT16: - printf(" %s:%s%d%s\n", cfg->option, blank, *((int16_t *)cfg->ptr), tsGlobalUnit[cfg->unitType]); - break; - case TAOS_CFG_VTYPE_INT32: - printf(" %s:%s%d%s\n", cfg->option, blank, *((int32_t *)cfg->ptr), tsGlobalUnit[cfg->unitType]); - break; - case TAOS_CFG_VTYPE_UINT16: - printf(" %s:%s%d%s\n", cfg->option, blank, *((uint16_t *)cfg->ptr), tsGlobalUnit[cfg->unitType]); - break; - case TAOS_CFG_VTYPE_FLOAT: - printf(" %s:%s%f%s\n", cfg->option, blank, *((float *)cfg->ptr), tsGlobalUnit[cfg->unitType]); - break; - case TAOS_CFG_VTYPE_STRING: - case TAOS_CFG_VTYPE_IPSTR: - case TAOS_CFG_VTYPE_DIRECTORY: - printf(" %s:%s%s%s\n", cfg->option, blank, (char *)cfg->ptr, tsGlobalUnit[cfg->unitType]); - break; - default: - break; - } -} - -void taosDumpGlobalCfg() { - printf("taos global config:\n"); - printf("==================================\n"); - for (int i = 0; i < tsGlobalConfigNum; ++i) { - SGlobalCfg *cfg = tsGlobalConfig + i; - if (tscEmbeddedInUtil == 0 && !(cfg->cfgType & TSDB_CFG_CTYPE_B_CLIENT)) continue; - if (cfg->cfgType & TSDB_CFG_CTYPE_B_NOT_PRINT) continue; - if (!(cfg->cfgType & TSDB_CFG_CTYPE_B_SHOW)) continue; - - taosDumpCfg(cfg); - } - - printf("\ntaos local config:\n"); - printf("==================================\n"); - - for (int i = 0; i < tsGlobalConfigNum; ++i) { - SGlobalCfg *cfg = tsGlobalConfig + i; - if (tscEmbeddedInUtil == 0 && !(cfg->cfgType & TSDB_CFG_CTYPE_B_CLIENT)) continue; - if (cfg->cfgType & TSDB_CFG_CTYPE_B_NOT_PRINT) continue; - if (cfg->cfgType & TSDB_CFG_CTYPE_B_SHOW) continue; - - taosDumpCfg(cfg); - } -} +int32_t cfgLoadFromApollUrl(SConfig *pConfig, const char *url) { + uInfo("load from apoll url %s", url); + return 0; +} \ No newline at end of file diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 2b53a769ff..cdc8515148 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -13,32 +13,30 @@ * along with this program. If not, see . */ -#include "os.h" - -#define TAOS_ERROR_C - -typedef struct { - int32_t val; - const char* str; -} STaosError; - +#define _DEFAULT_SOURCE #include "os.h" #include "taoserror.h" +#define TAOS_ERROR_C + +typedef struct { + int32_t val; + const char* str; +} STaosError; + static threadlocal int32_t tsErrno; -int32_t* taosGetErrno() { - return &tsErrno; -} + +int32_t* taosGetErrno() { return &tsErrno; } #ifdef TAOS_ERROR_C -#define TAOS_DEFINE_ERROR(name, msg) {.val = (name), .str=(msg)}, +#define TAOS_DEFINE_ERROR(name, msg) {.val = (name), .str = (msg)}, #else #define TAOS_DEFINE_ERROR(name, mod, code, msg) static const int32_t name = TAOS_DEF_ERROR_CODE(mod, code); #endif -#define TAOS_SYSTEM_ERROR(code) (0x80ff0000 | (code)) -#define TAOS_SUCCEEDED(err) ((err) >= 0) -#define TAOS_FAILED(err) ((err) < 0) +#define TAOS_SYSTEM_ERROR(code) (0x80ff0000 | (code)) +#define TAOS_SUCCEEDED(err) ((err) >= 0) +#define TAOS_FAILED(err) ((err) < 0) #ifdef TAOS_ERROR_C STaosError errors[] = { @@ -81,6 +79,8 @@ TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_MSG, "Invalid message") TAOS_DEFINE_ERROR(TSDB_CODE_MSG_NOT_PROCESSED, "Message not processed") TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_PARA, "Invalid parameters") TAOS_DEFINE_ERROR(TSDB_CODE_REPEAT_INIT, "Repeat initialization") +TAOS_DEFINE_ERROR(TSDB_CODE_CFG_NOT_FOUND, "Config not found") +TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_CFG, "Invalid config option") TAOS_DEFINE_ERROR(TSDB_CODE_REF_NO_MEMORY, "Ref out of memory") TAOS_DEFINE_ERROR(TSDB_CODE_REF_FULL, "too many Ref Objs") @@ -251,16 +251,16 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_NAME, "Invalid func name") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_COMMENT, "Invalid func comment") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_CODE, "Invalid func code") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_BUFSIZE, "Invalid func bufSize") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_RETRIEVE, "Invalid func retrieve msg") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_RETRIEVE, "Invalid func retrieve msg") // mnode-trans -TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_ALREADY_EXIST, "Transaction already exists") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_NOT_EXIST, "Transaction not exists") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_INVALID_STAGE, "Invalid stage to kill") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_CANT_PARALLEL, "Invalid stage to kill") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_ALREADY_EXIST, "Transaction already exists") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_NOT_EXIST, "Transaction not exists") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_INVALID_STAGE, "Invalid stage to kill") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_CANT_PARALLEL, "Invalid stage to kill") // mnode-topic -TAOS_DEFINE_ERROR(TSDB_CODE_MND_UNSUPPORTED_TOPIC, "Topic with STable not supported yet") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_UNSUPPORTED_TOPIC, "Topic with STable not supported yet") // dnode TAOS_DEFINE_ERROR(TSDB_CODE_DND_ACTION_IN_PROGRESS, "Action in progress") @@ -372,8 +372,6 @@ TAOS_DEFINE_ERROR(TSDB_CODE_QRY_TASK_MSG_ERROR, "Task message error") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_JOB_FREED, "Job already freed") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_TASK_STATUS_ERROR, "Task status error") - - // grant TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_EXPIRED, "License expired") TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_DNODE_LIMITED, "DNode creation limited by licence") @@ -430,12 +428,11 @@ TAOS_DEFINE_ERROR(TSDB_CODE_CTG_OUT_OF_SERVICE, "catalog is out of ser TAOS_DEFINE_ERROR(TSDB_CODE_SCH_STATUS_ERROR, "scheduler status error") TAOS_DEFINE_ERROR(TSDB_CODE_SCH_INTERNAL_ERROR, "scheduler internal error") - #ifdef TAOS_ERROR_C }; #endif -static int tsCompareTaosError(const void* a, const void* b) { +static int32_t taosCompareTaosError(const void* a, const void* b) { const STaosError* x = (const STaosError*)a; const STaosError* y = (const STaosError*)b; if (x->val < y->val) { @@ -448,10 +445,10 @@ static int tsCompareTaosError(const void* a, const void* b) { } static pthread_once_t tsErrorInit = PTHREAD_ONCE_INIT; -static void tsSortError(void) { - qsort(errors, sizeof(errors)/sizeof(errors[0]), sizeof(errors[0]), tsCompareTaosError); -} +static void tsSortError(void) { + qsort(errors, sizeof(errors) / sizeof(errors[0]), sizeof(errors[0]), taosCompareTaosError); +} const char* tstrerror(int32_t err) { pthread_once(&tsErrorInit, tsSortError); @@ -461,9 +458,11 @@ const char* tstrerror(int32_t err) { return strerror(err & 0x0000ffff); } - size_t s = 0, e = sizeof(errors)/sizeof(errors[0]); + int32_t s = 0; + int32_t e = sizeof(errors) / sizeof(errors[0]); + while (s < e) { - size_t mid = (s + e) / 2; + int32_t mid = (s + e) / 2; int32_t val = errors[mid].val; if (err > val) { s = mid + 1; diff --git a/source/util/src/tfile.c b/source/util/src/tfile.c index 0f68e9204d..ef9708df0c 100644 --- a/source/util/src/tfile.c +++ b/source/util/src/tfile.c @@ -18,153 +18,153 @@ #include "taoserror.h" #include "tref.h" #include "tutil.h" -#include "ulog.h" +#include "tlog.h" static int32_t tsFileRsetId = -1; static int8_t tfInited = 0; -static void tfCloseFile(void *p) { taosCloseFile((int32_t)(uintptr_t)p); } +// static void tfCloseFile(void *p) { taosCloseFile((TdFilePtr)(uintptr_t)p); } int32_t tfInit() { - int8_t old = atomic_val_compare_exchange_8(&tfInited, 0, 1); - if (old == 1) return 0; - tsFileRsetId = taosOpenRef(2000, tfCloseFile); - if (tsFileRsetId > 0) { - return 0; - } else { - atomic_store_8(&tfInited, 0); - return -1; - } + // int8_t old = atomic_val_compare_exchange_8(&tfInited, 0, 1); + // if (old == 1) return 0; + // tsFileRsetId = taosOpenRef(2000, tfCloseFile); + // if (tsFileRsetId > 0) { + // return 0; + // } else { + // atomic_store_8(&tfInited, 0); + // return -1; + // } } void tfCleanup() { - atomic_store_8(&tfInited, 0); - if (tsFileRsetId >= 0) taosCloseRef(tsFileRsetId); - tsFileRsetId = -1; + // atomic_store_8(&tfInited, 0); + // if (tsFileRsetId >= 0) taosCloseRef(tsFileRsetId); + // tsFileRsetId = -1; } -static int64_t tfOpenImp(int32_t fd) { - if (fd < 0) { - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } +// static int64_t tfOpenImp(TdFilePtr pFile) { +// if (pFile == NULL) { +// terrno = TAOS_SYSTEM_ERROR(errno); +// return -1; +// } - void * p = (void *)(int64_t)fd; - int64_t rid = taosAddRef(tsFileRsetId, p); - if (rid < 0) taosCloseFile(fd); +// void * p = (void *)(int64_t)pFile; +// int64_t rid = taosAddRef(tsFileRsetId, p); +// if (rid < 0) taosCloseFile(&pFile); - return rid; -} +// return rid; +// } -int64_t tfOpenRead(const char *pathname, int32_t flags) { - int32_t fd = taosOpenFileRead(pathname); - return tfOpenImp(fd); -} +// int64_t tfOpenRead(const char *pathname, int32_t flags) { +// int32_t pFile = taosOpenFile(pathname, TD_FILE_READ); +// return tfOpenImp(fd); +// } -int64_t tfOpenReadWrite(const char *pathname, int32_t flags) { - int32_t fd = taosOpenFileReadWrite(pathname); - return tfOpenImp(fd); -} +// int64_t tfOpenReadWrite(const char *pathname, int32_t flags) { +// int32_t pFile = taosOpenFile(pathname, TD_FILE_READ | TD_FILE_WRITE); +// return tfOpenImp(fd); +// } -int64_t tfOpenCreateWrite(const char *pathname, int32_t flags, mode_t mode) { - int32_t fd = taosOpenFileCreateWrite(pathname); - return tfOpenImp(fd); -} +// int64_t tfOpenCreateWrite(const char *pathname, int32_t flags, mode_t mode) { +// int32_t pFile = taosOpenFile(pathname, TD_FILE_CTEATE | TD_FILE_WRITE); +// return tfOpenImp(fd); +// } -int64_t tfOpenCreateWriteAppend(const char *pathname, int32_t flags, mode_t mode) { - int32_t fd = taosOpenFileCreateWriteAppend(pathname); - return tfOpenImp(fd); -} +// int64_t tfOpenCreateWriteAppend(const char *pathname, int32_t flags, mode_t mode) { +// int32_t pFile = taosOpenFile(pathname, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_APPEND); +// return tfOpenImp(fd); +// } -int64_t tfClose(int64_t tfd) { return taosRemoveRef(tsFileRsetId, tfd); } +// int64_t tfClose(int64_t tfd) { return taosRemoveRef(tsFileRsetId, tfd); } -int64_t tfWrite(int64_t tfd, void *buf, int64_t count) { - void *p = taosAcquireRef(tsFileRsetId, tfd); - if (p == NULL) return -1; +// int64_t tfWrite(int64_t tfd, void *buf, int64_t count) { +// void *p = taosAcquireRef(tsFileRsetId, tfd); +// if (p == NULL) return -1; - int32_t fd = (int32_t)(uintptr_t)p; +// int32_t pFile = (TdFilePtr)(uintptr_t)p; - int64_t ret = taosWriteFile(fd, buf, count); - if (ret < 0) terrno = TAOS_SYSTEM_ERROR(errno); +// int64_t ret = taosWriteFile(pFile, buf, count); +// if (ret < 0) terrno = TAOS_SYSTEM_ERROR(errno); - taosReleaseRef(tsFileRsetId, tfd); - return ret; -} +// taosReleaseRef(tsFileRsetId, tfd); +// return ret; +// } -int64_t tfRead(int64_t tfd, void *buf, int64_t count) { - void *p = taosAcquireRef(tsFileRsetId, tfd); - if (p == NULL) return -1; +// int64_t tfRead(int64_t tfd, void *buf, int64_t count) { +// void *p = taosAcquireRef(tsFileRsetId, tfd); +// if (p == NULL) return -1; - int32_t fd = (int32_t)(uintptr_t)p; +// int32_t pFile = (TdFilePtr)(uintptr_t)p; - int64_t ret = taosReadFile(fd, buf, count); - if (ret < 0) terrno = TAOS_SYSTEM_ERROR(errno); +// int64_t ret = taosReadFile(pFile, buf, count); +// if (ret < 0) terrno = TAOS_SYSTEM_ERROR(errno); - taosReleaseRef(tsFileRsetId, tfd); - return ret; -} +// taosReleaseRef(tsFileRsetId, tfd); +// return ret; +// } -int64_t tfPread(int64_t tfd, void *buf, int64_t count, int32_t offset) { - void *p = taosAcquireRef(tsFileRsetId, tfd); - if (p == NULL) return -1; +// int64_t tfPread(int64_t tfd, void *buf, int64_t count, int32_t offset) { +// void *p = taosAcquireRef(tsFileRsetId, tfd); +// if (p == NULL) return -1; - int32_t fd = (int32_t)(uintptr_t)p; +// int32_t pFile = (TdFilePtr)(uintptr_t)p; - int64_t ret = pread(fd, buf, count, offset); - if (ret < 0) terrno = TAOS_SYSTEM_ERROR(errno); +// int64_t ret = pread(fd, buf, count, offset); +// if (ret < 0) terrno = TAOS_SYSTEM_ERROR(errno); - taosReleaseRef(tsFileRsetId, tfd); - return ret; -} +// taosReleaseRef(tsFileRsetId, tfd); +// return ret; +// } -int32_t tfFsync(int64_t tfd) { - void *p = taosAcquireRef(tsFileRsetId, tfd); - if (p == NULL) return -1; +// int32_t tfFsync(int64_t tfd) { +// void *p = taosAcquireRef(tsFileRsetId, tfd); +// if (p == NULL) return -1; - int32_t fd = (int32_t)(uintptr_t)p; - int32_t code = taosFsyncFile(fd); +// int32_t pFile = (TdFilePtr)(uintptr_t)p; +// int32_t code = taosFsyncFile(pFile); - taosReleaseRef(tsFileRsetId, tfd); - return code; -} +// taosReleaseRef(tsFileRsetId, tfd); +// return code; +// } -bool tfValid(int64_t tfd) { - void *p = taosAcquireRef(tsFileRsetId, tfd); - if (p == NULL) return false; +// bool tfValid(int64_t tfd) { +// void *p = taosAcquireRef(tsFileRsetId, tfd); +// if (p == NULL) return false; - taosReleaseRef(tsFileRsetId, tfd); - return true; -} +// taosReleaseRef(tsFileRsetId, tfd); +// return true; +// } -int64_t tfLseek(int64_t tfd, int64_t offset, int32_t whence) { - void *p = taosAcquireRef(tsFileRsetId, tfd); - if (p == NULL) return -1; +// int64_t tfLseek(int64_t tfd, int64_t offset, int32_t whence) { +// void *p = taosAcquireRef(tsFileRsetId, tfd); +// if (p == NULL) return -1; - int32_t fd = (int32_t)(uintptr_t)p; - int64_t ret = taosLSeekFile(fd, offset, whence); +// int32_t pFile = (TdFilePtr)(uintptr_t)p; +// int64_t ret = taosLSeekFile(fd, offset, whence); - taosReleaseRef(tsFileRsetId, tfd); - return ret; -} +// taosReleaseRef(tsFileRsetId, tfd); +// return ret; +// } -int32_t tfFtruncate(int64_t tfd, int64_t length) { - void *p = taosAcquireRef(tsFileRsetId, tfd); - if (p == NULL) return -1; +// int32_t tfFtruncate(int64_t tfd, int64_t length) { +// void *p = taosAcquireRef(tsFileRsetId, tfd); +// if (p == NULL) return -1; - int32_t fd = (int32_t)(uintptr_t)p; - int32_t code = taosFtruncateFile(fd, length); +// int32_t pFile = (TdFilePtr)(uintptr_t)p; +// int32_t code = taosFtruncateFile(fd, length); - taosReleaseRef(tsFileRsetId, tfd); - return code; -} +// taosReleaseRef(tsFileRsetId, tfd); +// return code; +// } -void *tfMmapReadOnly(int64_t tfd, int64_t length) { - void *p = taosAcquireRef(tsFileRsetId, tfd); - if (p == NULL) return NULL; - int32_t fd = (int32_t)(uintptr_t)p; +// void *tfMmapReadOnly(int64_t tfd, int64_t length) { +// void *p = taosAcquireRef(tsFileRsetId, tfd); +// if (p == NULL) return NULL; +// int32_t pFile = (TdFilePtr)(uintptr_t)p; - void *ptr = mmap(NULL, length, PROT_READ, MAP_SHARED, fd, 0); - taosReleaseRef(tsFileRsetId, tfd); - return ptr; -} +// void *ptr = mmap(NULL, length, PROT_READ, MAP_SHARED, fd, 0); +// taosReleaseRef(tsFileRsetId, tfd); +// return ptr; +// } diff --git a/source/util/src/thash.c b/source/util/src/thash.c index 0b0d8c4c58..9f39b79968 100644 --- a/source/util/src/thash.c +++ b/source/util/src/thash.c @@ -15,7 +15,7 @@ #include "os.h" #include "thash.h" -#include "ulog.h" +#include "tlog.h" #include "taos.h" #include "tdef.h" diff --git a/source/util/src/tidpool.c b/source/util/src/tidpool.c index 00c43bb25f..b4d76e6fb5 100644 --- a/source/util/src/tidpool.c +++ b/source/util/src/tidpool.c @@ -13,8 +13,7 @@ * along with this program. If not, see . */ -#include "os.h" -#include "ulog.h" +#include "tlog.h" typedef struct { int maxId; diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index 12aa77214f..11b4555737 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -14,33 +14,30 @@ */ #define _DEFAULT_SOURCE -#include "tlog.h" -#include "os.h" -#include "tnote.h" #include "tutil.h" -#include "ulog.h" +#include "tlog.h" -#define MAX_LOGLINE_SIZE (1000) -#define MAX_LOGLINE_BUFFER_SIZE (MAX_LOGLINE_SIZE + 10) -#define MAX_LOGLINE_CONTENT_SIZE (MAX_LOGLINE_SIZE - 100) -#define MAX_LOGLINE_DUMP_SIZE (65 * 1024) -#define MAX_LOGLINE_DUMP_BUFFER_SIZE (MAX_LOGLINE_DUMP_SIZE + 10) -#define MAX_LOGLINE_DUMP_CONTENT_SIZE (MAX_LOGLINE_DUMP_SIZE - 100) +#define LOG_MAX_LINE_SIZE (1000) +#define LOG_MAX_LINE_BUFFER_SIZE (LOG_MAX_LINE_SIZE + 10) +#define LOG_MAX_LINE_CONTENT_SIZE (LOG_MAX_LINE_SIZE - 100) +#define LOG_MAX_LINE_DUMP_SIZE (65 * 1024) +#define LOG_MAX_LINE_DUMP_BUFFER_SIZE (LOG_MAX_LINE_DUMP_SIZE + 10) +#define LOG_MAX_LINE_DUMP_CONTENT_SIZE (LOG_MAX_LINE_DUMP_SIZE - 100) -#define LOG_FILE_NAME_LEN 300 -#define TSDB_DEFAULT_LOG_BUF_SIZE (20 * 1024 * 1024) // 20MB +#define LOG_FILE_NAME_LEN 300 +#define LOG_DEFAULT_BUF_SIZE (20 * 1024 * 1024) // 20MB -#define DEFAULT_LOG_INTERVAL 25 -#define LOG_INTERVAL_STEP 5 -#define MIN_LOG_INTERVAL 5 -#define MAX_LOG_INTERVAL 25 -#define LOG_MAX_WAIT_MSEC 1000 +#define LOG_DEFAULT_INTERVAL 25 +#define LOG_INTERVAL_STEP 5 +#define LOG_MIN_INTERVAL 5 +#define LOG_MAX_INTERVAL 25 +#define LOG_MAX_WAIT_MSEC 1000 #define LOG_BUF_BUFFER(x) ((x)->buffer) -#define LOG_BUF_START(x) ((x)->buffStart) -#define LOG_BUF_END(x) ((x)->buffEnd) -#define LOG_BUF_SIZE(x) ((x)->buffSize) -#define LOG_BUF_MUTEX(x) ((x)->buffMutex) +#define LOG_BUF_START(x) ((x)->buffStart) +#define LOG_BUF_END(x) ((x)->buffEnd) +#define LOG_BUF_SIZE(x) ((x)->buffSize) +#define LOG_BUF_MUTEX(x) ((x)->buffMutex) typedef struct { char *buffer; @@ -48,7 +45,7 @@ typedef struct { int32_t buffEnd; int32_t buffSize; int32_t minBuffSize; - int32_t fd; + TdFilePtr pFile; int32_t stop; pthread_t asyncThread; pthread_mutex_t buffMutex; @@ -70,29 +67,26 @@ typedef struct { int8_t tscEmbeddedInUtil = 0; int32_t tsLogKeepDays = 0; -int8_t tsAsyncLog = 1; -float tsTotalLogDirGB = 0; -float tsAvailLogDirGB = 0; -float tsMinimalLogDirGB = 1.0f; +bool tsAsyncLog = true; +int8_t tsLogInited = 0; int64_t asyncLogLostLines = 0; -int32_t writeInterval = DEFAULT_LOG_INTERVAL; +int32_t writeInterval = LOG_DEFAULT_INTERVAL; // log int32_t tsNumOfLogLines = 10000000; -int32_t mDebugFlag = 131; int32_t dDebugFlag = 135; int32_t vDebugFlag = 135; +int32_t mDebugFlag = 131; int32_t cDebugFlag = 131; int32_t jniDebugFlag = 131; -int32_t qDebugFlag = 131; -int32_t rpcDebugFlag = 131; +int32_t tmrDebugFlag = 131; int32_t uDebugFlag = 131; -int32_t debugFlag = 0; -int32_t sDebugFlag = 135; +int32_t rpcDebugFlag = 131; +int32_t qDebugFlag = 131; int32_t wDebugFlag = 135; +int32_t sDebugFlag = 135; int32_t tsdbDebugFlag = 131; int32_t tqDebugFlag = 135; -int32_t cqDebugFlag = 131; int32_t fsDebugFlag = 135; int64_t dbgEmptyW = 0; @@ -105,7 +99,7 @@ static SLogObj tsLogObj = {.fileNum = 1}; static void *taosAsyncOutputLog(void *param); static int32_t taosPushLogBuffer(SLogBuff *tLogBuff, char *msg, int32_t msgLen); static SLogBuff *taosLogBuffNew(int32_t bufSize); -static void taosCloseLogByFd(int32_t oldFd); +static void taosCloseLogByFd(TdFilePtr pFile); static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum); extern void taosPrintCfg(); static int32_t taosCompressFile(char *srcFileName, char *destFileName); @@ -120,11 +114,18 @@ static int32_t taosStartLog() { return 0; } -int32_t taosInitLog(char *logName, int numOfLogLines, int maxFiles) { - tsLogObj.logHandle = taosLogBuffNew(TSDB_DEFAULT_LOG_BUF_SIZE); +int32_t taosInitLog(const char *logName, int maxFiles) { + if (atomic_val_compare_exchange_8(&tsLogInited, 0, 1) != 0) return 0; + osUpdate(); + + char fullName[PATH_MAX] = {0}; + snprintf(fullName, PATH_MAX, "%s" TD_DIRSEP "%s", tsLogDir, logName); + + tsLogObj.logHandle = taosLogBuffNew(LOG_DEFAULT_BUF_SIZE); if (tsLogObj.logHandle == NULL) return -1; - if (taosOpenLogFile(logName, numOfLogLines, maxFiles) < 0) return -1; + if (taosOpenLogFile(fullName, tsNumOfLogLines, maxFiles) < 0) return -1; if (taosStartLog() < 0) return -1; + tsLogInited = true; return 0; } @@ -137,7 +138,7 @@ static void taosStopLog() { void taosCloseLog() { taosStopLog(); // tsem_post(&(tsLogObj.logHandle->buffNotEmpty)); - taosMsleep(MAX_LOG_INTERVAL / 1000); + taosMsleep(LOG_MAX_INTERVAL / 1000); if (taosCheckPthreadValid(tsLogObj.logHandle->asyncThread)) { pthread_join(tsLogObj.logHandle->asyncThread, NULL); } @@ -147,11 +148,11 @@ void taosCloseLog() { // taosCloseLog(); } -static bool taosLockLogFile(int32_t fd) { - if (fd < 0) return false; +static bool taosLockLogFile(TdFilePtr pFile) { + if (pFile == NULL) return false; if (tsLogObj.fileNum > 1) { - int32_t ret = taosLockFile(fd); + int32_t ret = taosLockFile(pFile); if (ret == 0) { return true; } @@ -160,11 +161,11 @@ static bool taosLockLogFile(int32_t fd) { return false; } -static void taosUnLockLogFile(int32_t fd) { - if (fd < 0) return; +static void taosUnLockLogFile(TdFilePtr pFile) { + if (pFile == NULL) return; if (tsLogObj.fileNum > 1) { - taosUnLockFile(fd); + taosUnLockFile(pFile); } } @@ -198,26 +199,26 @@ static void *taosThreadToOpenNewFile(void *param) { taosUmaskFile(0); - int32_t fd = taosOpenFileCreateWriteTrunc(name); - if (fd < 0) { + TdFilePtr pFile = taosOpenFile(name, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pFile == NULL) { tsLogObj.openInProgress = 0; tsLogObj.lines = tsLogObj.maxLines - 1000; - uError("open new log file fail! fd:%d reason:%s, reuse lastlog", fd, strerror(errno)); + uError("open new log file fail! reason:%s, reuse lastlog", strerror(errno)); return NULL; } - taosLockLogFile(fd); - (void)taosLSeekFile(fd, 0, SEEK_SET); + taosLockLogFile(pFile); + (void)taosLSeekFile(pFile, 0, SEEK_SET); - int32_t oldFd = tsLogObj.logHandle->fd; - tsLogObj.logHandle->fd = fd; + TdFilePtr pOldFile = tsLogObj.logHandle->pFile; + tsLogObj.logHandle->pFile = pFile; tsLogObj.lines = 0; tsLogObj.openInProgress = 0; - taosCloseLogByFd(oldFd); + taosCloseLogByFd(pOldFile); uInfo(" new log file:%d is opened", tsLogObj.flag); uInfo("=================================="); - taosPrintCfg(); + // taosPrintCfg(); taosKeepOldLog(keepName); return NULL; @@ -259,8 +260,8 @@ void taosResetLog() { } static bool taosCheckFileIsOpen(char *logFileName) { - int32_t fd = taosOpenFileWrite(logFileName); - if (fd < 0) { + TdFilePtr pFile = taosOpenFile(logFileName, TD_FILE_WRITE); + if (pFile == NULL) { if (errno == ENOENT) { return false; } else { @@ -269,12 +270,12 @@ static bool taosCheckFileIsOpen(char *logFileName) { } } - if (taosLockLogFile(fd)) { - taosUnLockLogFile(fd); - taosCloseFile(fd); + if (taosLockLogFile(pFile)) { + taosUnLockLogFile(pFile); + taosCloseFile(&pFile); return false; } else { - taosCloseFile(fd); + taosCloseFile(&pFile); return true; } } @@ -347,45 +348,40 @@ static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum) { pthread_mutex_init(&tsLogObj.logMutex, NULL); taosUmaskFile(0); - tsLogObj.logHandle->fd = taosOpenFileCreateWrite(fileName); + tsLogObj.logHandle->pFile = taosOpenFile(fileName, TD_FILE_CTEATE | TD_FILE_WRITE); - if (tsLogObj.logHandle->fd < 0) { + if (tsLogObj.logHandle->pFile == NULL) { printf("\nfailed to open log file:%s, reason:%s\n", fileName, strerror(errno)); return -1; } - taosLockLogFile(tsLogObj.logHandle->fd); + taosLockLogFile(tsLogObj.logHandle->pFile); // only an estimate for number of lines int64_t filesize = 0; - if (taosFStatFile(tsLogObj.logHandle->fd, &filesize, NULL) < 0) { + if (taosFStatFile(tsLogObj.logHandle->pFile, &filesize, NULL) < 0) { printf("\nfailed to fstat log file:%s, reason:%s\n", fileName, strerror(errno)); return -1; } size = (int32_t)filesize; tsLogObj.lines = size / 60; - taosLSeekFile(tsLogObj.logHandle->fd, 0, SEEK_END); + taosLSeekFile(tsLogObj.logHandle->pFile, 0, SEEK_END); sprintf(name, "==================================================\n"); - taosWriteFile(tsLogObj.logHandle->fd, name, (uint32_t)strlen(name)); + taosWriteFile(tsLogObj.logHandle->pFile, name, (uint32_t)strlen(name)); sprintf(name, " new log file \n"); - taosWriteFile(tsLogObj.logHandle->fd, name, (uint32_t)strlen(name)); + taosWriteFile(tsLogObj.logHandle->pFile, name, (uint32_t)strlen(name)); sprintf(name, "==================================================\n"); - taosWriteFile(tsLogObj.logHandle->fd, name, (uint32_t)strlen(name)); + taosWriteFile(tsLogObj.logHandle->pFile, name, (uint32_t)strlen(name)); return 0; } void taosPrintLog(const char *flags, int32_t dflag, const char *format, ...) { - if (tsTotalLogDirGB != 0 && tsAvailLogDirGB < tsMinimalLogDirGB) { - printf("server disk:%s space remain %.3f GB, total %.1f GB, stop print log.\n", tsLogDir, tsAvailLogDirGB, - tsTotalLogDirGB); - fflush(stdout); - return; - } + if (!osLogSpaceAvailable()) return; va_list argpointer; - char buffer[MAX_LOGLINE_BUFFER_SIZE] = {0}; + char buffer[LOG_MAX_LINE_BUFFER_SIZE] = {0}; int32_t len; struct tm Tm, *ptm; struct timeval timeSecs; @@ -400,29 +396,29 @@ void taosPrintLog(const char *flags, int32_t dflag, const char *format, ...) { len += sprintf(buffer + len, "%s", flags); va_start(argpointer, format); - int32_t writeLen = vsnprintf(buffer + len, MAX_LOGLINE_CONTENT_SIZE, format, argpointer); + int32_t writeLen = vsnprintf(buffer + len, LOG_MAX_LINE_CONTENT_SIZE, format, argpointer); if (writeLen <= 0) { - char tmp[MAX_LOGLINE_DUMP_BUFFER_SIZE] = {0}; - writeLen = vsnprintf(tmp, MAX_LOGLINE_DUMP_CONTENT_SIZE, format, argpointer); - strncpy(buffer + len, tmp, MAX_LOGLINE_CONTENT_SIZE); - len += MAX_LOGLINE_CONTENT_SIZE; - } else if (writeLen >= MAX_LOGLINE_CONTENT_SIZE) { - len += MAX_LOGLINE_CONTENT_SIZE; + char tmp[LOG_MAX_LINE_DUMP_BUFFER_SIZE] = {0}; + writeLen = vsnprintf(tmp, LOG_MAX_LINE_DUMP_CONTENT_SIZE, format, argpointer); + strncpy(buffer + len, tmp, LOG_MAX_LINE_CONTENT_SIZE); + len += LOG_MAX_LINE_CONTENT_SIZE; + } else if (writeLen >= LOG_MAX_LINE_CONTENT_SIZE) { + len += LOG_MAX_LINE_CONTENT_SIZE; } else { len += writeLen; } va_end(argpointer); - if (len > MAX_LOGLINE_SIZE) len = MAX_LOGLINE_SIZE; + if (len > LOG_MAX_LINE_SIZE) len = LOG_MAX_LINE_SIZE; buffer[len++] = '\n'; buffer[len] = 0; - if ((dflag & DEBUG_FILE) && tsLogObj.logHandle && tsLogObj.logHandle->fd >= 0) { + if ((dflag & DEBUG_FILE) && tsLogObj.logHandle && tsLogObj.logHandle->pFile != NULL) { if (tsAsyncLog) { taosPushLogBuffer(tsLogObj.logHandle, buffer, len); } else { - taosWriteFile(tsLogObj.logHandle->fd, buffer, len); + taosWriteFile(tsLogObj.logHandle->pFile, buffer, len); } if (tsLogObj.maxLines > 0) { @@ -432,17 +428,11 @@ void taosPrintLog(const char *flags, int32_t dflag, const char *format, ...) { } } - if (dflag & DEBUG_SCREEN) taosWriteFile(1, buffer, (uint32_t)len); - if (dflag == 255) nInfo(buffer, len); + if (dflag & DEBUG_SCREEN) write(1, buffer, (uint32_t)len); } void taosDumpData(unsigned char *msg, int32_t len) { - if (tsTotalLogDirGB != 0 && tsAvailLogDirGB < tsMinimalLogDirGB) { - printf("server disk:%s space remain %.3f GB, total %.1f GB, stop dump log.\n", tsLogDir, tsAvailLogDirGB, - tsTotalLogDirGB); - fflush(stdout); - return; - } + if (!osLogSpaceAvailable()) return; char temp[256]; int32_t i, pos = 0, c = 0; @@ -453,7 +443,7 @@ void taosDumpData(unsigned char *msg, int32_t len) { pos += 3; if (c >= 16) { temp[pos++] = '\n'; - taosWriteFile(tsLogObj.logHandle->fd, temp, (uint32_t)pos); + taosWriteFile(tsLogObj.logHandle->pFile, temp, (uint32_t)pos); c = 0; pos = 0; } @@ -461,19 +451,14 @@ void taosDumpData(unsigned char *msg, int32_t len) { temp[pos++] = '\n'; - taosWriteFile(tsLogObj.logHandle->fd, temp, (uint32_t)pos); + taosWriteFile(tsLogObj.logHandle->pFile, temp, (uint32_t)pos); } void taosPrintLongString(const char *flags, int32_t dflag, const char *format, ...) { - if (tsTotalLogDirGB != 0 && tsAvailLogDirGB < tsMinimalLogDirGB) { - printf("server disk:%s space remain %.3f GB, total %.1f GB, stop write log.\n", tsLogDir, tsAvailLogDirGB, - tsTotalLogDirGB); - fflush(stdout); - return; - } + if (!osLogSpaceAvailable()) return; va_list argpointer; - char buffer[MAX_LOGLINE_DUMP_BUFFER_SIZE]; + char buffer[LOG_MAX_LINE_DUMP_BUFFER_SIZE]; int32_t len; struct tm Tm, *ptm; struct timeval timeSecs; @@ -488,19 +473,19 @@ void taosPrintLongString(const char *flags, int32_t dflag, const char *format, . len += sprintf(buffer + len, "%s", flags); va_start(argpointer, format); - len += vsnprintf(buffer + len, MAX_LOGLINE_DUMP_CONTENT_SIZE, format, argpointer); + len += vsnprintf(buffer + len, LOG_MAX_LINE_DUMP_CONTENT_SIZE, format, argpointer); va_end(argpointer); - if (len > MAX_LOGLINE_DUMP_SIZE) len = MAX_LOGLINE_DUMP_SIZE; + if (len > LOG_MAX_LINE_DUMP_SIZE) len = LOG_MAX_LINE_DUMP_SIZE; buffer[len++] = '\n'; buffer[len] = 0; - if ((dflag & DEBUG_FILE) && tsLogObj.logHandle && tsLogObj.logHandle->fd >= 0) { + if ((dflag & DEBUG_FILE) && tsLogObj.logHandle && tsLogObj.logHandle->pFile != NULL) { if (tsAsyncLog) { taosPushLogBuffer(tsLogObj.logHandle, buffer, len); } else { - taosWriteFile(tsLogObj.logHandle->fd, buffer, len); + taosWriteFile(tsLogObj.logHandle->pFile, buffer, len); } if (tsLogObj.maxLines > 0) { @@ -510,19 +495,19 @@ void taosPrintLongString(const char *flags, int32_t dflag, const char *format, . } } - if (dflag & DEBUG_SCREEN) taosWriteFile(1, buffer, (uint32_t)len); + if (dflag & DEBUG_SCREEN) write(1, buffer, (uint32_t)len); } #if 0 void taosCloseLog() { - taosCloseLogByFd(tsLogObj.logHandle->fd); + taosCloseLogByFd(tsLogObj.logHandle->pFile); } #endif -static void taosCloseLogByFd(int32_t fd) { - if (fd >= 0) { - taosUnLockLogFile(fd); - taosCloseFile(fd); +static void taosCloseLogByFd(TdFilePtr pFile) { + if (pFile != NULL) { + taosUnLockLogFile(pFile); + taosCloseFile(&pFile); } } @@ -642,7 +627,7 @@ static void taosWriteLog(SLogBuff *tLogBuff) { if (start == end) { dbgEmptyW++; - writeInterval = MAX_LOG_INTERVAL; + writeInterval = LOG_MAX_INTERVAL; return; } @@ -658,12 +643,12 @@ static void taosWriteLog(SLogBuff *tLogBuff) { } if (start < end) { - taosWriteFile(tLogBuff->fd, LOG_BUF_BUFFER(tLogBuff) + start, pollSize); + taosWriteFile(tLogBuff->pFile, LOG_BUF_BUFFER(tLogBuff) + start, pollSize); } else { int32_t tsize = LOG_BUF_SIZE(tLogBuff) - start; - taosWriteFile(tLogBuff->fd, LOG_BUF_BUFFER(tLogBuff) + start, tsize); + taosWriteFile(tLogBuff->pFile, LOG_BUF_BUFFER(tLogBuff) + start, tsize); - taosWriteFile(tLogBuff->fd, LOG_BUF_BUFFER(tLogBuff), end); + taosWriteFile(tLogBuff->pFile, LOG_BUF_BUFFER(tLogBuff), end); } dbgWN++; @@ -671,14 +656,14 @@ static void taosWriteLog(SLogBuff *tLogBuff) { if (pollSize < tLogBuff->minBuffSize) { dbgSmallWN++; - if (writeInterval < MAX_LOG_INTERVAL) { + if (writeInterval < LOG_MAX_INTERVAL) { writeInterval += LOG_INTERVAL_STEP; } } else if (pollSize > LOG_BUF_SIZE(tLogBuff) / 3) { dbgBigWN++; - writeInterval = MIN_LOG_INTERVAL; + writeInterval = LOG_MIN_INTERVAL; } else if (pollSize > LOG_BUF_SIZE(tLogBuff) / 4) { - if (writeInterval > MIN_LOG_INTERVAL) { + if (writeInterval > LOG_MIN_INTERVAL) { writeInterval -= LOG_INTERVAL_STEP; } } @@ -693,7 +678,7 @@ static void taosWriteLog(SLogBuff *tLogBuff) { break; } - writeInterval = MIN_LOG_INTERVAL; + writeInterval = LOG_MIN_INTERVAL; remainChecked = 1; } while (1); @@ -720,56 +705,61 @@ int32_t taosCompressFile(char *srcFileName, char *destFileName) { int32_t ret = 0; int32_t len = 0; char *data = malloc(compressSize); - FILE *srcFp = NULL; -// gzFile dstFp = NULL; + // gzFile dstFp = NULL; - srcFp = fopen(srcFileName, "r"); - if (srcFp == NULL) { + // srcFp = fopen(srcFileName, "r"); + TdFilePtr pSrcFile = taosOpenFile(srcFileName, TD_FILE_READ); + if (pSrcFile == NULL) { ret = -1; goto cmp_end; } - int32_t fd = taosOpenFileCreateWriteTrunc(destFileName); - if (fd < 0) { + TdFilePtr pFile = taosOpenFile(destFileName, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pFile == NULL) { ret = -2; goto cmp_end; } -// dstFp = gzdopen(fd, "wb6f"); -// if (dstFp == NULL) { -// ret = -3; -// close(fd); -// goto cmp_end; -// } -// -// while (!feof(srcFp)) { -// len = (int32_t)fread(data, 1, compressSize, srcFp); -// (void)gzwrite(dstFp, data, len); -// } + // dstFp = gzdopen(fd, "wb6f"); + // if (dstFp == NULL) { + // ret = -3; + // close(fd); + // goto cmp_end; + // } + // + // while (!feof(srcFp)) { + // len = (int32_t)fread(data, 1, compressSize, srcFp); + // (void)gzwrite(dstFp, data, len); + // } cmp_end: - if (srcFp) { - fclose(srcFp); + if (pSrcFile) { + taosCloseFile(&pSrcFile); } -// if (dstFp) { -// gzclose(dstFp); -// } + // if (dstFp) { + // gzclose(dstFp); + // } free(data); return ret; } -void taosPrintOsInfo() { - SysNameInfo info = taosGetSysNameInfo(); +void taosSetAllDebugFlag(int32_t flag) { + if (!(flag & DEBUG_TRACE || flag & DEBUG_DEBUG || flag & DEBUG_DUMP)) return; - uInfo(" os pageSize: %" PRId64 "(KB)", tsPageSize); - uInfo(" os openMax: %" PRId64, tsOpenMax); - uInfo(" os streamMax: %" PRId64, tsStreamMax); - uInfo(" os numOfCores: %d", tsNumOfCores); - uInfo(" os totalMemory: %d(MB)", tsTotalMemoryMB); - uInfo(" os sysname: %s", info.sysname); - uInfo(" os nodename: %s", info.nodename); - uInfo(" os release: %s", info.release); - uInfo(" os version: %s", info.version); - uInfo(" os machine: %s", info.machine); -} + dDebugFlag = flag; + vDebugFlag = flag; + mDebugFlag = flag; + cDebugFlag = flag; + jniDebugFlag = flag; + uDebugFlag = flag; + rpcDebugFlag = flag; + qDebugFlag = flag; + wDebugFlag = flag; + sDebugFlag = flag; + tsdbDebugFlag = flag; + tqDebugFlag = flag; + fsDebugFlag = flag; + + uInfo("all debug flag are set to %d", flag); +} \ No newline at end of file diff --git a/source/util/src/tlosertree.c b/source/util/src/tlosertree.c index 80bbac2c78..8b7f55809b 100644 --- a/source/util/src/tlosertree.c +++ b/source/util/src/tlosertree.c @@ -14,7 +14,7 @@ */ #include "os.h" -#include "ulog.h" +#include "tlog.h" #include "tlosertree.h" #include "taoserror.h" diff --git a/source/util/src/tmempool.c b/source/util/src/tmempool.c index a61c4abb14..f980a05629 100644 --- a/source/util/src/tmempool.c +++ b/source/util/src/tmempool.c @@ -13,8 +13,7 @@ * along with this program. If not, see . */ -#include "os.h" -#include "ulog.h" +#include "tlog.h" #include "tmempool.h" #include "tutil.h" diff --git a/source/util/src/tnote.c b/source/util/src/tnote.c deleted file mode 100644 index d9356b7e40..0000000000 --- a/source/util/src/tnote.c +++ /dev/null @@ -1,268 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "tutil.h" -#include "tdef.h" -#include "tnote.h" - -SNoteObj tsHttpNote; -SNoteObj tsTscNote; -SNoteObj tsInfoNote; - -static int32_t taosOpenNoteWithMaxLines(char *fn, int32_t maxLines, int32_t maxNoteNum, SNoteObj *pNote); -static void taosCloseNoteByFd(int32_t oldFd, SNoteObj *pNote); - -static void taosInitNote(int32_t numOfLines, int32_t maxNotes, SNoteObj *pNote, char *name) { - memset(pNote, 0, sizeof(SNoteObj)); - pNote->fileNum = 1; - pNote->fd = -1; - - if (taosOpenNoteWithMaxLines(name, numOfLines, maxNotes, pNote) < 0) { - fprintf(stderr, "failed to init note file\n"); - } - - taosNotePrint(pNote, "=================================================="); - taosNotePrint(pNote, "=================== new note ==================="); - taosNotePrint(pNote, "=================================================="); -} - -int32_t taosInitNotes() { - char name[TSDB_FILENAME_LEN * 2] = {0}; - -#if 0 - if (tsTscEnableRecordSql) { - snprintf(name, TSDB_FILENAME_LEN * 2, "%s/tscsql-%d", tsLogDir, taosGetPId()); - taosInitNote(tsNumOfLogLines, 1, &tsTscNote, name); - } - -#endif - return 0; -} - -static bool taosLockNote(int32_t fd, SNoteObj *pNote) { - if (fd < 0) return false; - - if (pNote->fileNum > 1) { - int32_t ret = (int32_t)taosLockFile(fd); - if (ret == 0) { - return true; - } - } - - return false; -} - -static void taosUnLockNote(int32_t fd, SNoteObj *pNote) { - if (fd < 0) return; - - if (pNote->fileNum > 1) { - taosUnLockFile(fd); - } -} - -static void *taosThreadToOpenNewNote(void *param) { - char name[NOTE_FILE_NAME_LEN * 2]; - SNoteObj *pNote = (SNoteObj *)param; - - setThreadName("openNewNote"); - - pNote->flag ^= 1; - pNote->lines = 0; - sprintf(name, "%s.%d", pNote->name, pNote->flag); - - taosUmaskFile(0); - - int32_t fd = taosOpenFileCreateWriteTrunc(name); - if (fd < 0) { - return NULL; - } - - taosLockNote(fd, pNote); - (void)taosLSeekFile(fd, 0, SEEK_SET); - - int32_t oldFd = pNote->fd; - pNote->fd = fd; - pNote->lines = 0; - pNote->openInProgress = 0; - taosNotePrint(pNote, "=============== new note is opened ============="); - - taosCloseNoteByFd(oldFd, pNote); - return NULL; -} - -static int32_t taosOpenNewNote(SNoteObj *pNote) { - pthread_mutex_lock(&pNote->mutex); - - if (pNote->lines > pNote->maxLines && pNote->openInProgress == 0) { - pNote->openInProgress = 1; - - taosNotePrint(pNote, "=============== open new note =================="); - pthread_t pattern; - pthread_attr_t attr; - pthread_attr_init(&attr); - pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); - - pthread_create(&pattern, &attr, taosThreadToOpenNewNote, (void *)pNote); - pthread_attr_destroy(&attr); - } - - pthread_mutex_unlock(&pNote->mutex); - - return pNote->fd; -} - -static bool taosCheckNoteIsOpen(char *noteName, SNoteObj *pNote) { - int32_t fd = taosOpenFileCreateWrite(noteName); - if (fd < 0) { - fprintf(stderr, "failed to open note:%s reason:%s\n", noteName, strerror(errno)); - return true; - } - - if (taosLockNote(fd, pNote)) { - taosUnLockNote(fd, pNote); - taosCloseFile(fd); - return false; - } else { - taosCloseFile(fd); - return true; - } -} - -static void taosGetNoteName(char *fn, SNoteObj *pNote) { - if (pNote->fileNum > 1) { - for (int32_t i = 0; i < pNote->fileNum; i++) { - char fileName[NOTE_FILE_NAME_LEN]; - - sprintf(fileName, "%s%d.0", fn, i); - bool file1open = taosCheckNoteIsOpen(fileName, pNote); - - sprintf(fileName, "%s%d.1", fn, i); - bool file2open = taosCheckNoteIsOpen(fileName, pNote); - - if (!file1open && !file2open) { - sprintf(pNote->name, "%s%d", fn, i); - return; - } - } - } - - if (strlen(fn) < NOTE_FILE_NAME_LEN) { - strcpy(pNote->name, fn); - } -} - -static int32_t taosOpenNoteWithMaxLines(char *fn, int32_t maxLines, int32_t maxNoteNum, SNoteObj *pNote) { - char name[NOTE_FILE_NAME_LEN * 2] = {0}; - int32_t size; - int32_t logstat0_mtime, logstat1_mtime; - - pNote->maxLines = maxLines; - pNote->fileNum = maxNoteNum; - taosGetNoteName(fn, pNote); - - if (strlen(fn) < NOTE_FILE_NAME_LEN + 50 - 2) { - strcpy(name, fn); - strcat(name, ".0"); - } - bool log0Exist = taosStatFile(name, NULL, &logstat0_mtime) >= 0; - - if (strlen(fn) < NOTE_FILE_NAME_LEN + 50 - 2) { - strcpy(name, fn); - strcat(name, ".1"); - } - bool log1Exist = taosStatFile(name, NULL, &logstat1_mtime) >= 0; - - if (!log0Exist && !log1Exist) { - pNote->flag = 0; - } else if (!log1Exist) { - pNote->flag = 0; - } else if (!log0Exist) { - pNote->flag = 1; - } else { - pNote->flag = (logstat0_mtime > logstat1_mtime) ? 0 : 1; - } - - char noteName[NOTE_FILE_NAME_LEN * 2] = {0}; - sprintf(noteName, "%s.%d", pNote->name, pNote->flag); - pthread_mutex_init(&pNote->mutex, NULL); - - taosUmaskFile(0); - pNote->fd = taosOpenFileCreateWrite(noteName); - - if (pNote->fd < 0) { - fprintf(stderr, "failed to open note file:%s reason:%s\n", noteName, strerror(errno)); - return -1; - } - taosLockNote(pNote->fd, pNote); - - // only an estimate for number of lines - int64_t filestat_size; - if (taosFStatFile(pNote->fd, &filestat_size, NULL) < 0) { - fprintf(stderr, "failed to fstat note file:%s reason:%s\n", noteName, strerror(errno)); - return -1; - } - size = (int32_t)filestat_size; - pNote->lines = size / 60; - - taosLSeekFile(pNote->fd, 0, SEEK_END); - - return 0; -} - -void taosNotePrintBuffer(SNoteObj *pNote, char *buffer, int32_t len) { - if (pNote->fd <= 0) return; - taosWriteFile(pNote->fd, buffer, len); - - if (pNote->maxLines > 0) { - pNote->lines++; - if ((pNote->lines > pNote->maxLines) && (pNote->openInProgress == 0)) taosOpenNewNote(pNote); - } -} - -void taosNotePrint(SNoteObj *pNote, const char *const format, ...) { - va_list argpointer; - char buffer[MAX_NOTE_LINE_SIZE + 2]; - int32_t len; - struct tm Tm, *ptm; - struct timeval timeSecs; - time_t curTime; - - taosGetTimeOfDay(&timeSecs); - curTime = timeSecs.tv_sec; - ptm = localtime_r(&curTime, &Tm); - len = sprintf(buffer, "%02d/%02d %02d:%02d:%02d.%06d %08" PRId64 " ", ptm->tm_mon + 1, ptm->tm_mday, ptm->tm_hour, - ptm->tm_min, ptm->tm_sec, (int32_t)timeSecs.tv_usec, taosGetSelfPthreadId()); - va_start(argpointer, format); - len += vsnprintf(buffer + len, MAX_NOTE_LINE_SIZE - len, format, argpointer); - va_end(argpointer); - - if (len >= MAX_NOTE_LINE_SIZE) len = MAX_NOTE_LINE_SIZE - 2; - - buffer[len++] = '\n'; - buffer[len] = 0; - - taosNotePrintBuffer(pNote, buffer, len); -} - -// static void taosCloseNote(SNoteObj *pNote) { taosCloseNoteByFd(pNote->fd, pNote); } - -static void taosCloseNoteByFd(int32_t fd, SNoteObj *pNote) { - if (fd >= 0) { - taosUnLockNote(fd, pNote); - taosCloseFile(fd); - } -} diff --git a/source/util/src/tpagedbuf.c b/source/util/src/tpagedbuf.c index 0e8d85492c..a7e43cebf9 100644 --- a/source/util/src/tpagedbuf.c +++ b/source/util/src/tpagedbuf.c @@ -1,5 +1,4 @@ -#include "os.h" -#include "ulog.h" +#include "tlog.h" #include "tpagedbuf.h" #include "taoserror.h" #include "tcompression.h" @@ -32,7 +31,7 @@ typedef struct SDiskbasedBuf { int32_t numOfPages; int64_t totalBufSize; uint64_t fileSize; // disk file size - FILE* file; + TdFilePtr pFile; int32_t allocateId; // allocated page id char* path; // file path int32_t pageSize; // current used page size @@ -67,7 +66,7 @@ static void printStatisData(const SDiskbasedBuf* pBuf); pResBuf->inMemPages = inMemBufSize/pagesize; // maximum allowed pages, it is a soft limit. pResBuf->allocateId = -1; pResBuf->comp = true; - pResBuf->file = NULL; + pResBuf->pFile = NULL; pResBuf->qId = qId; pResBuf->fileSize = 0; @@ -94,8 +93,9 @@ static void printStatisData(const SDiskbasedBuf* pBuf); } static int32_t createDiskFile(SDiskbasedBuf* pBuf) { - pBuf->file = fopen(pBuf->path, "wb+"); - if (pBuf->file == NULL) { + // pBuf->file = fopen(pBuf->path, "wb+"); + pBuf->pFile = taosOpenFile(pBuf->path, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC); + if (pBuf->pFile == NULL) { // qError("failed to create tmp file: %s on disk. %s", pBuf->path, strerror(errno)); return TAOS_SYSTEM_ERROR(errno); } @@ -168,13 +168,13 @@ static char* doFlushPageToDisk(SDiskbasedBuf* pBuf, SPageInfo* pg) { pg->offset = allocatePositionInFile(pBuf, size); pBuf->nextPos += size; - int32_t ret = fseek(pBuf->file, pg->offset, SEEK_SET); + int32_t ret = taosLSeekFile(pBuf->pFile, pg->offset, SEEK_SET); if (ret != 0) { terrno = TAOS_SYSTEM_ERROR(errno); return NULL; } - ret = (int32_t) fwrite(t, 1, size, pBuf->file); + ret = (int32_t) taosWriteFile(pBuf->pFile, t, size); if (ret != size) { terrno = TAOS_SYSTEM_ERROR(errno); return NULL; @@ -199,13 +199,13 @@ static char* doFlushPageToDisk(SDiskbasedBuf* pBuf, SPageInfo* pg) { } // 3. write to disk. - int32_t ret = fseek(pBuf->file, pg->offset, SEEK_SET); + int32_t ret = taosLSeekFile(pBuf->pFile, pg->offset, SEEK_SET); if (ret != 0) { terrno = TAOS_SYSTEM_ERROR(errno); return NULL; } - ret = (int32_t)fwrite(t, 1, size, pBuf->file); + ret = (int32_t) taosWriteFile(pBuf->pFile, t, size); if (ret != size) { terrno = TAOS_SYSTEM_ERROR(errno); return NULL; @@ -233,7 +233,7 @@ static char* flushPageToDisk(SDiskbasedBuf* pBuf, SPageInfo* pg) { int32_t ret = TSDB_CODE_SUCCESS; assert(((int64_t) pBuf->numOfPages * pBuf->pageSize) == pBuf->totalBufSize && pBuf->numOfPages >= pBuf->inMemPages); - if (pBuf->file == NULL) { + if (pBuf->pFile == NULL) { if ((ret = createDiskFile(pBuf)) != TSDB_CODE_SUCCESS) { terrno = ret; return NULL; @@ -245,14 +245,14 @@ static char* flushPageToDisk(SDiskbasedBuf* pBuf, SPageInfo* pg) { // load file block data in disk static int32_t loadPageFromDisk(SDiskbasedBuf* pBuf, SPageInfo* pg) { - int32_t ret = fseek(pBuf->file, pg->offset, SEEK_SET); + int32_t ret = taosLSeekFile(pBuf->pFile, pg->offset, SEEK_SET); if (ret != 0) { ret = TAOS_SYSTEM_ERROR(errno); return ret; } SFilePage* pPage = (SFilePage*) GET_DATA_PAYLOAD(pg); - ret = (int32_t)fread(pPage->data, 1, pg->length, pBuf->file); + ret = (int32_t)taosReadFile(pBuf->pFile, pPage->data, pg->length); if (ret != pg->length) { ret = TAOS_SYSTEM_ERROR(errno); return ret; @@ -495,12 +495,12 @@ void destroyResultBuf(SDiskbasedBuf* pBuf) { printStatisData(pBuf); - if (pBuf->file != NULL) { + if (pBuf->pFile != NULL) { uDebug("Paged buffer closed, total:%.2f Kb (%d Pages), inmem size:%.2f Kb (%d Pages), file size:%.2f Kb, page size:%.2f Kb, %"PRIx64"\n", pBuf->totalBufSize/1024.0, pBuf->numOfPages, listNEles(pBuf->lruList) * pBuf->pageSize / 1024.0, listNEles(pBuf->lruList), pBuf->fileSize/1024.0, pBuf->pageSize/1024.0f, pBuf->qId); - fclose(pBuf->file); + taosCloseFile(&pBuf->pFile); } else { uDebug("Paged buffer closed, total:%.2f Kb, no file created, %"PRIx64, pBuf->totalBufSize/1024.0, pBuf->qId); } diff --git a/source/util/src/tqueue.c b/source/util/src/tqueue.c index 8125f550d0..0f6e2610c5 100644 --- a/source/util/src/tqueue.c +++ b/source/util/src/tqueue.c @@ -16,7 +16,7 @@ #define _DEFAULT_SOURCE #include "tqueue.h" #include "taoserror.h" -#include "ulog.h" +#include "tlog.h" typedef struct STaosQnode STaosQnode; diff --git a/source/util/src/tref.c b/source/util/src/tref.c index 68f161bd3d..ca4388ec26 100644 --- a/source/util/src/tref.c +++ b/source/util/src/tref.c @@ -14,8 +14,9 @@ */ #include "os.h" + #include "taoserror.h" -#include "ulog.h" +#include "tlog.h" #include "tutil.h" #define TSDB_REF_OBJECTS 50 diff --git a/source/util/src/tsched.c b/source/util/src/tsched.c index 915edc45ce..7db5abdd24 100644 --- a/source/util/src/tsched.c +++ b/source/util/src/tsched.c @@ -13,10 +13,9 @@ * along with this program. If not, see . */ -#include "os.h" #include "tdef.h" #include "tutil.h" -#include "ulog.h" +#include "tlog.h" #include "tsched.h" #include "ttimer.h" diff --git a/source/util/src/tskiplist.c b/source/util/src/tskiplist.c index 328d3da5a4..1c325ca0a8 100644 --- a/source/util/src/tskiplist.c +++ b/source/util/src/tskiplist.c @@ -14,12 +14,10 @@ * along with this program. If not, see . */ -#include "os.h" - #include "compare.h" #include "tskiplist.h" #include "tutil.h" -#include "ulog.h" +#include "tlog.h" static int initForwardBackwardPtr(SSkipList *pSkipList); static SSkipListNode * getPriorNode(SSkipList *pSkipList, const char *val, int32_t order, SSkipListNode **pCur); diff --git a/source/util/src/tstep.c b/source/util/src/tstep.c index d840b119fb..30c8ec2b3a 100644 --- a/source/util/src/tstep.c +++ b/source/util/src/tstep.c @@ -14,8 +14,7 @@ */ #define _DEFAULT_SOURCE -#include "os.h" -#include "ulog.h" +#include "tlog.h" #include "taoserror.h" #include "tstep.h" diff --git a/source/util/src/tthread.c b/source/util/src/tthread.c index 44fce1c882..8c7a3ada05 100644 --- a/source/util/src/tthread.c +++ b/source/util/src/tthread.c @@ -14,11 +14,10 @@ */ #include "tthread.h" -#include "os.h" #include "taoserror.h" #include "tdef.h" #include "tutil.h" -#include "ulog.h" +#include "tlog.h" // create new thread pthread_t* taosCreateThread(void* (*__start_routine)(void*), void* param) { diff --git a/source/util/src/ttimer.c b/source/util/src/ttimer.c index 65101a5e07..2c04603269 100644 --- a/source/util/src/ttimer.c +++ b/source/util/src/ttimer.c @@ -111,8 +111,7 @@ typedef struct time_wheel_t { tmr_obj_t** slots; } time_wheel_t; -int32_t tmrDebugFlag = 131; -uint32_t tsMaxTmrCtrl = 512; +int32_t tsMaxTmrCtrl = 512; static pthread_once_t tmrModuleInit = PTHREAD_ONCE_INIT; static pthread_mutex_t tmrCtrlMutex; diff --git a/source/util/src/tversion.c b/source/util/src/tversion.c index 3944bd5132..8409637e80 100644 --- a/source/util/src/tversion.c +++ b/source/util/src/tversion.c @@ -17,7 +17,6 @@ #include "os.h" #include "taoserror.h" #include "tdef.h" -#include "ulog.h" int32_t taosVersionStrToInt(const char *vstr, int32_t *vint) { if (vstr == NULL) { diff --git a/source/util/src/tworker.c b/source/util/src/tworker.c index ca4e5a6f30..2843f4e801 100644 --- a/source/util/src/tworker.c +++ b/source/util/src/tworker.c @@ -16,7 +16,7 @@ #define _DEFAULT_SOURCE #include "tworker.h" #include "taoserror.h" -#include "ulog.h" +#include "tlog.h" typedef void *(*ThreadFp)(void *param); diff --git a/source/util/test/CMakeLists.txt b/source/util/test/CMakeLists.txt index ee0ade03b8..d6b779b6e3 100644 --- a/source/util/test/CMakeLists.txt +++ b/source/util/test/CMakeLists.txt @@ -51,4 +51,12 @@ target_link_libraries(queue_test os util gtest_main) add_test( NAME queue_test COMMAND queue_test +) + +# cfgTest +add_executable(cfgTest "cfgTest.cpp") +target_link_libraries(cfgTest os util gtest_main) +add_test( + NAME cfgTest + COMMAND cfgTest ) \ No newline at end of file diff --git a/source/util/test/cfgTest.cpp b/source/util/test/cfgTest.cpp new file mode 100644 index 0000000000..800e261dcb --- /dev/null +++ b/source/util/test/cfgTest.cpp @@ -0,0 +1,135 @@ +/** + * @file cfgTest.cpp + * @author slguan (slguan@taosdata.com) + * @brief config module tests + * @version 1.0 + * @date 2022-02-20 + * + * @copyright Copyright (c) 2022 + * + */ + +#include +#include "tconfig.h" + +class CfgTest : public ::testing::Test { + protected: + static void SetUpTestSuite() {} + static void TearDownTestSuite() {} + + public: + void SetUp() override {} + void TearDown() override {} + void InitCfg(SConfig *pConfig); + + static const char *pConfig; +}; + +const char *CfgTest::pConfig; + +TEST_F(CfgTest, 01_Str) { + EXPECT_STREQ(cfgStypeStr(CFG_STYPE_DEFAULT), "default"); + EXPECT_STREQ(cfgStypeStr(CFG_STYPE_CFG_FILE), "cfg_file"); + EXPECT_STREQ(cfgStypeStr(CFG_STYPE_ENV_FILE), "env_file"); + EXPECT_STREQ(cfgStypeStr(CFG_STYPE_ENV_VAR), "env_var"); + EXPECT_STREQ(cfgStypeStr(CFG_STYPE_APOLLO_URL), "apollo_url"); + EXPECT_STREQ(cfgStypeStr(CFG_STYPE_ARG_LIST), "arg_list"); + EXPECT_STREQ(cfgStypeStr(ECfgSrcType(1024)), "invalid"); + + EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_NONE), "none"); + EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_BOOL), "bool"); + EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_INT32), "int32"); + EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_INT64), "int64"); + EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_FLOAT), "float"); + EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_STRING), "string"); + EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_DIR), "dir"); + EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_DIR), "dir"); + EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_DIR), "dir"); + EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_DIR), "dir"); + EXPECT_STREQ(cfgDtypeStr(ECfgDataType(1024)), "invalid"); +} + +TEST_F(CfgTest, 02_Basic) { + SConfig *pConfig = cfgInit(); + ASSERT_NE(pConfig, nullptr); + + EXPECT_EQ(cfgAddBool(pConfig, "test_bool", 0, 0), 0); + EXPECT_EQ(cfgAddInt32(pConfig, "test_int32", 1, 0, 16, 0), 0); + EXPECT_EQ(cfgAddInt64(pConfig, "test_int64", 2, 0, 16, 0), 0); + EXPECT_EQ(cfgAddFloat(pConfig, "test_float", 3, 0, 16, 0), 0); + EXPECT_EQ(cfgAddString(pConfig, "test_string", "4", 0), 0); + EXPECT_EQ(cfgAddDir(pConfig, "test_dir", "/tmp", 0), 0); + + EXPECT_EQ(cfgGetSize(pConfig), 6); + + int32_t size = 0; + SConfigItem *pItem = cfgIterate(pConfig, NULL); + while (pItem != NULL) { + switch (pItem->dtype) { + case CFG_DTYPE_BOOL: + printf("index:%d, cfg:%s value:%d\n", size, pItem->name, pItem->bval); + break; + case CFG_DTYPE_INT32: + printf("index:%d, cfg:%s value:%d\n", size, pItem->name, pItem->i32); + break; + case CFG_DTYPE_INT64: + printf("index:%d, cfg:%s value:%" PRId64 "\n", size, pItem->name, pItem->i64); + break; + case CFG_DTYPE_FLOAT: + printf("index:%d, cfg:%s value:%f\n", size, pItem->name, pItem->fval); + break; + case CFG_DTYPE_STRING: + printf("index:%d, cfg:%s value:%s\n", size, pItem->name, pItem->str); + break; + case CFG_DTYPE_DIR: + printf("index:%d, cfg:%s value:%s\n", size, pItem->name, pItem->str); + break; + default: + printf("index:%d, cfg:%s invalid cfg dtype:%d\n", size, pItem->name, pItem->dtype); + break; + } + size++; + pItem = cfgIterate(pConfig, pItem); + } + cfgCancelIterate(pConfig, pItem); + + EXPECT_EQ(cfgGetSize(pConfig), 6); + + pItem = cfgGetItem(pConfig, "test_bool"); + EXPECT_EQ(pItem->stype, CFG_STYPE_DEFAULT); + EXPECT_EQ(pItem->dtype, CFG_DTYPE_BOOL); + EXPECT_STREQ(pItem->name, "test_bool"); + EXPECT_EQ(pItem->bval, 0); + + pItem = cfgGetItem(pConfig, "test_int32"); + EXPECT_EQ(pItem->stype, CFG_STYPE_DEFAULT); + EXPECT_EQ(pItem->dtype, CFG_DTYPE_INT32); + EXPECT_STREQ(pItem->name, "test_int32"); + EXPECT_EQ(pItem->i32, 1); + + pItem = cfgGetItem(pConfig, "test_int64"); + EXPECT_EQ(pItem->stype, CFG_STYPE_DEFAULT); + EXPECT_EQ(pItem->dtype, CFG_DTYPE_INT64); + EXPECT_STREQ(pItem->name, "test_int64"); + EXPECT_EQ(pItem->i64, 2); + + pItem = cfgGetItem(pConfig, "test_float"); + EXPECT_EQ(pItem->stype, CFG_STYPE_DEFAULT); + EXPECT_EQ(pItem->dtype, CFG_DTYPE_FLOAT); + EXPECT_STREQ(pItem->name, "test_float"); + EXPECT_EQ(pItem->fval, 3); + + pItem = cfgGetItem(pConfig, "test_string"); + EXPECT_EQ(pItem->stype, CFG_STYPE_DEFAULT); + EXPECT_EQ(pItem->dtype, CFG_DTYPE_STRING); + EXPECT_STREQ(pItem->name, "test_string"); + EXPECT_STREQ(pItem->str, "4"); + + pItem = cfgGetItem(pConfig, "test_dir"); + EXPECT_EQ(pItem->stype, CFG_STYPE_DEFAULT); + EXPECT_EQ(pItem->dtype, CFG_DTYPE_DIR); + EXPECT_STREQ(pItem->name, "test_dir"); + EXPECT_STREQ(pItem->str, "/tmp"); + + cfgCleanup(pConfig); +} diff --git a/source/util/test/trefTest.c b/source/util/test/trefTest.c index 54aa6a5f34..586151d782 100644 --- a/source/util/test/trefTest.c +++ b/source/util/test/trefTest.c @@ -8,7 +8,7 @@ #include "tlog.h" #include "tglobal.h" #include "taoserror.h" -#include "ulog.h" +#include "tlog.h" typedef struct { int refNum; @@ -158,7 +158,7 @@ int main(int argc, char *argv[]) { } } - taosInitLog("tref.log", 5000000, 10); + taosInitLog("tref.log", 10); SRefSpace *pSpaceList = (SRefSpace *) calloc(sizeof(SRefSpace), threads); pthread_t *pThreadList = (pthread_t *) calloc(sizeof(pthread_t), threads); diff --git a/tests b/tests index 4da4bb00c5..12233db374 160000 --- a/tests +++ b/tests @@ -1 +1 @@ -Subproject commit 4da4bb00c5615f0b1bea04340abef5d9d15a96b4 +Subproject commit 12233db374f1fe97b327e89a3442c631578ad38d diff --git a/tools/shell/src/backup/shellCheck.c b/tools/shell/src/backup/shellCheck.c index 7fc8b1409a..33d25b6746 100644 --- a/tools/shell/src/backup/shellCheck.c +++ b/tools/shell/src/backup/shellCheck.c @@ -116,7 +116,7 @@ static void *shellCheckThreadFp(void *arg) { char file[32] = {0}; snprintf(file, 32, "tb%d.txt", pThread->threadIndex); - FILE *fp = fopen(file, "w"); + TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); if (!fp) { fprintf(stdout, "failed to open %s, reason:%s", file, strerror(errno)); return NULL; @@ -133,7 +133,7 @@ static void *shellCheckThreadFp(void *arg) { int32_t code = taos_errno(pSql); if (code != 0) { int32_t len = snprintf(sql, SHELL_SQL_LEN, "drop table %s.%s;\n", pThread->db, tbname); - fwrite(sql, 1, len, fp); + taosWriteFile(pFile, sql, len); atomic_add_fetch_32(&errorNum, 1); } @@ -145,8 +145,8 @@ static void *shellCheckThreadFp(void *arg) { taos_free_result(pSql); } - taosFsync(fileno(fp)); - fclose(fp); + taosFsync(pFile); + taosCloseFile(&pFile); return NULL; } diff --git a/tools/shell/src/backup/shellImport.c b/tools/shell/src/backup/shellImport.c index 222d69e854..ce15212f86 100644 --- a/tools/shell/src/backup/shellImport.c +++ b/tools/shell/src/backup/shellImport.c @@ -138,7 +138,6 @@ static void shellSourceFile(TAOS *con, char *fptr) { char * cmd = malloc(tsMaxSQLStringLen); size_t cmd_len = 0; char * line = NULL; - size_t line_len = 0; if (wordexp(fptr, &full_path, 0) != 0) { fprintf(stderr, "ERROR: illegal file name\n"); @@ -171,8 +170,9 @@ static void shellSourceFile(TAOS *con, char *fptr) { } */ - FILE *f = fopen(fname, "r"); - if (f == NULL) { + // FILE *f = fopen(fname, "r"); + TdFilePtr pFile = taosOpenFile(fname, TD_FILE_READ); + if (pFile == NULL) { fprintf(stderr, "ERROR: failed to open file %s\n", fname); wordfree(&full_path); free(cmd); @@ -182,7 +182,7 @@ static void shellSourceFile(TAOS *con, char *fptr) { fprintf(stdout, "begin import file:%s\n", fname); int lineNo = 0; - while ((read_len = getline(&line, &line_len, f)) != -1) { + while ((read_len = taosGetLineFile(pFile, &line)) != -1) { ++lineNo; if (read_len >= tsMaxSQLStringLen) continue; line[--read_len] = '\0'; @@ -215,9 +215,9 @@ static void shellSourceFile(TAOS *con, char *fptr) { } free(cmd); - if (line) free(line); + if(line != NULL) free(line); wordfree(&full_path); - fclose(f); + taosCloseFile(&pFile); } void* shellImportThreadFp(void *arg) diff --git a/tools/shell/src/backup/tnettest.c b/tools/shell/src/backup/tnettest.c index be8714387f..772d92d8c6 100644 --- a/tools/shell/src/backup/tnettest.c +++ b/tools/shell/src/backup/tnettest.c @@ -18,7 +18,7 @@ #include "taosdef.h" #include "tmsg.h" #include "taoserror.h" -#include "tulog.h" +#include "tlog.h" #include "tglobal.h" #include "tsocket.h" #include "trpc.h" diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 4186f0dae1..e43267f032 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -459,44 +459,44 @@ static char *formatTimestamp(char *buf, int64_t val, int precision) { return buf; } -static void dumpFieldToFile(FILE *fp, const char *val, TAOS_FIELD *field, int32_t length, int precision) { +static void dumpFieldToFile(TdFilePtr pFile, const char *val, TAOS_FIELD *field, int32_t length, int precision) { if (val == NULL) { - fprintf(fp, "%s", TSDB_DATA_NULL_STR); + taosFprintfFile(pFile, "%s", TSDB_DATA_NULL_STR); return; } char buf[TSDB_MAX_BYTES_PER_ROW]; switch (field->type) { case TSDB_DATA_TYPE_BOOL: - fprintf(fp, "%d", ((((int32_t)(*((char *)val))) == 1) ? 1 : 0)); + taosFprintfFile(pFile, "%d", ((((int32_t)(*((char *)val))) == 1) ? 1 : 0)); break; case TSDB_DATA_TYPE_TINYINT: - fprintf(fp, "%d", *((int8_t *)val)); + taosFprintfFile(pFile, "%d", *((int8_t *)val)); break; case TSDB_DATA_TYPE_SMALLINT: - fprintf(fp, "%d", *((int16_t *)val)); + taosFprintfFile(pFile, "%d", *((int16_t *)val)); break; case TSDB_DATA_TYPE_INT: - fprintf(fp, "%d", *((int32_t *)val)); + taosFprintfFile(pFile, "%d", *((int32_t *)val)); break; case TSDB_DATA_TYPE_BIGINT: - fprintf(fp, "%" PRId64, *((int64_t *)val)); + taosFprintfFile(pFile, "%" PRId64, *((int64_t *)val)); break; case TSDB_DATA_TYPE_FLOAT: - fprintf(fp, "%.5f", GET_FLOAT_VAL(val)); + taosFprintfFile(pFile, "%.5f", GET_FLOAT_VAL(val)); break; case TSDB_DATA_TYPE_DOUBLE: - fprintf(fp, "%.9f", GET_DOUBLE_VAL(val)); + taosFprintfFile(pFile, "%.9f", GET_DOUBLE_VAL(val)); break; case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_NCHAR: memcpy(buf, val, length); buf[length] = 0; - fprintf(fp, "\'%s\'", buf); + taosFprintfFile(pFile, "\'%s\'", buf); break; case TSDB_DATA_TYPE_TIMESTAMP: formatTimestamp(buf, *(int64_t *)val, precision); - fprintf(fp, "'%s'", buf); + taosFprintfFile(pFile, "'%s'", buf); break; default: break; @@ -516,8 +516,9 @@ static int dumpResultToFile(const char *fname, TAOS_RES *tres) { return -1; } - FILE *fp = fopen(full_path.we_wordv[0], "w"); - if (fp == NULL) { + // FILE *fp = fopen(full_path.we_wordv[0], "w"); + TdFilePtr pFile = taosOpenFile(full_path.we_wordv[0], TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pFile == NULL) { fprintf(stderr, "ERROR: failed to open file: %s\n", full_path.we_wordv[0]); wordfree(&full_path); return -1; @@ -531,29 +532,29 @@ static int dumpResultToFile(const char *fname, TAOS_RES *tres) { for (int col = 0; col < num_fields; col++) { if (col > 0) { - fprintf(fp, ","); + taosFprintfFile(pFile, ","); } - fprintf(fp, "%s", fields[col].name); + taosFprintfFile(pFile, "%s", fields[col].name); } - fputc('\n', fp); + taosFprintfFile(pFile, "\n"); int numOfRows = 0; do { int32_t *length = taos_fetch_lengths(tres); for (int i = 0; i < num_fields; i++) { if (i > 0) { - fputc(',', fp); + taosFprintfFile(pFile, "\n"); } - dumpFieldToFile(fp, (const char *)row[i], fields + i, length[i], precision); + dumpFieldToFile(pFile, (const char *)row[i], fields + i, length[i], precision); } - fputc('\n', fp); + taosFprintfFile(pFile, "\n"); numOfRows++; row = taos_fetch_row(tres); } while (row != NULL); result = 0; - fclose(fp); + taosCloseFile(&pFile); return numOfRows; } @@ -897,14 +898,14 @@ void read_history() { history.hstart = 0; history.hend = 0; char *line = NULL; - size_t line_size = 0; int read_size = 0; char f_history[TSDB_FILENAME_LEN]; get_history_path(f_history); - FILE *f = fopen(f_history, "r"); - if (f == NULL) { + // FILE *f = fopen(f_history, "r"); + TdFilePtr pFile = taosOpenFile(f_history, TD_FILE_READ); + if (pFile == NULL) { #ifndef WINDOWS if (errno != ENOENT) { fprintf(stderr, "Failed to open file %s, reason:%s\n", f_history, strerror(errno)); @@ -913,7 +914,7 @@ void read_history() { return; } - while ((read_size = tgetline(&line, &line_size, f)) != -1) { + while ((read_size = taosGetLineFile(pFile, &line)) != -1) { line[read_size - 1] = '\0'; history.hist[history.hend] = strdup(line); @@ -924,16 +925,17 @@ void read_history() { } } - free(line); - fclose(f); + if(line != NULL) free(line); + taosCloseFile(&pFile); } void write_history() { char f_history[TSDB_FILENAME_LEN]; get_history_path(f_history); - FILE *f = fopen(f_history, "w"); - if (f == NULL) { + // FILE *f = fopen(f_history, "w"); + TdFilePtr pFile = taosOpenFile(f_history, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pFile == NULL) { #ifndef WINDOWS fprintf(stderr, "Failed to open file %s for write, reason:%s\n", f_history, strerror(errno)); #endif @@ -942,12 +944,12 @@ void write_history() { for (int i = history.hstart; i != history.hend;) { if (history.hist[i] != NULL) { - fprintf(f, "%s\n", history.hist[i]); + taosFprintfFile(pFile, "%s\n", history.hist[i]); tfree(history.hist[i]); } i = (i + 1) % MAX_HISTORY_SIZE; } - fclose(f); + taosCloseFile(&pFile); } void taos_error(TAOS_RES *tres, int64_t st) { @@ -969,7 +971,6 @@ void source_file(TAOS *con, char *fptr) { char *cmd = calloc(1, TSDB_MAX_ALLOWED_SQL_LEN + 1); size_t cmd_len = 0; char *line = NULL; - size_t line_len = 0; if (wordexp(fptr, &full_path, 0) != 0) { fprintf(stderr, "ERROR: illegal file name\n"); @@ -989,15 +990,16 @@ void source_file(TAOS *con, char *fptr) { } */ - FILE *f = fopen(fname, "r"); - if (f == NULL) { + // FILE *f = fopen(fname, "r"); + TdFilePtr pFile = taosOpenFile(fname, TD_FILE_READ); + if (pFile == NULL) { fprintf(stderr, "ERROR: failed to open file %s\n", fname); wordfree(&full_path); free(cmd); return; } - while ((read_len = tgetline(&line, &line_len, f)) != -1) { + while ((read_len = taosGetLineFile(pFile, &line)) != -1) { if (read_len >= TSDB_MAX_ALLOWED_SQL_LEN) continue; line[--read_len] = '\0'; @@ -1020,9 +1022,9 @@ void source_file(TAOS *con, char *fptr) { } free(cmd); - if (line) free(line); + if(line != NULL) free(line); wordfree(&full_path); - fclose(f); + taosCloseFile(&pFile); } void shellGetGrantInfo(void *con) { diff --git a/tools/shell/src/shellLinux.c b/tools/shell/src/shellLinux.c index 81d8c0a35b..f304f35e3e 100644 --- a/tools/shell/src/shellLinux.c +++ b/tools/shell/src/shellLinux.c @@ -19,7 +19,8 @@ #include "shell.h" #include "shellCommand.h" #include "tkey.h" -#include "ulog.h" +#include "tlog.h" +#include "version.h" #include #include @@ -183,7 +184,6 @@ static void parse_args( for (int i = 1; i < argc; i++) { if ((strncmp(argv[i], "-p", 2) == 0) || (strncmp(argv[i], "--password", 10) == 0)) { - strcpy(tsOsName, "Linux"); printf(LINUXCLIENT_VERSION, tsOsName, taos_get_client_info()); if ((strlen(argv[i]) == 2) || (strncmp(argv[i], "--password", 10) == 0)) { diff --git a/tools/shell/src/shellMain.c b/tools/shell/src/shellMain.c index 607dc79257..8a1763c4fc 100644 --- a/tools/shell/src/shellMain.c +++ b/tools/shell/src/shellMain.c @@ -15,7 +15,6 @@ #include "os.h" #include "shell.h" -#include "tconfig.h" #include "tglobal.h" pthread_t pid;