xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/3.0' into feature/qnode

This commit is contained in:
dapan1121 2022-02-25 14:31:37 +08:00
parent bf4efa77cb d1e628187a
commit e9ff317d9b
138 changed files with 3057 additions and 3620 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
contrib/CMakeLists.txt
View File

@ -193,6 +193,7 @@ endif(${BUILD_WITH_TRAFT})
# LIBUV # LIBUV
if(${BUILD_WITH_UV}) if(${BUILD_WITH_UV})
add_compile_options(-Wno-sign-compare)
add_subdirectory(libuv) add_subdirectory(libuv)
endif(${BUILD_WITH_UV}) endif(${BUILD_WITH_UV})

2
include/common/tep.h
View File

@ -23,7 +23,7 @@ typedef struct SBlockOrderInfo {
// bool hasNull; // bool hasNull;
} SBlockOrderInfo; } SBlockOrderInfo;
int taosGetFqdnPortFromEp(const char *ep, uint16_t defaultPort, SEp *pEp); int taosGetFqdnPortFromEp(const char *ep, SEp *pEp);
void addEpIntoEpSet(SEpSet *pEpSet, const char *fqdn, uint16_t port); void addEpIntoEpSet(SEpSet *pEpSet, const char *fqdn, uint16_t port);
bool isEpsetEqual(const SEpSet *s1, const SEpSet *s2); bool isEpsetEqual(const SEpSet *s1, const SEpSet *s2);

63
include/common/tglobal.h
View File

@ -20,31 +20,48 @@
extern "C" { extern "C" {
#endif #endif
#include "tcfg.h"
#include "tdef.h" #include "tdef.h"
// cluster
extern char tsFirst[];
extern char tsSecond[];
extern char tsLocalFqdn[];
extern char tsLocalEp[];
extern uint16_t tsServerPort;
extern int32_t tsVersion;
extern int32_t tsStatusInterval;
extern bool tsEnableTelemetryReporting;
// common // common
extern int8_t tsDaylight; extern int32_t tsRpcTimer;
extern int32_t tsRpcMaxTime;
extern bool tsRpcForceTcp; // all commands go to tcp protocol if this is enabled
extern int32_t tsMaxConnections;
extern int32_t tsMaxShellConns;
extern int32_t tsShellActivityTimer;
extern int32_t tsMaxTmrCtrl;
extern float tsNumOfThreadsPerCore;
extern int32_t tsNumOfCommitThreads;
extern float tsRatioOfQueryCores;
extern int32_t tsCompressMsgSize; extern int32_t tsCompressMsgSize;
extern int32_t tsCompressColData; extern int32_t tsCompressColData;
extern int32_t tsMaxNumOfDistinctResults; extern int32_t tsMaxNumOfDistinctResults;
extern char tsTempDir[]; extern int32_t tsCompatibleModel;
extern int tsCompatibleModel; // 2.0 compatible model extern bool tsEnableSlaveQuery;
extern int8_t tsEnableSlaveQuery; extern bool tsPrintAuth;
extern int8_t tsEnableAdjustMaster;
extern int8_t tsPrintAuth;
extern int64_t tsTickPerDay[3]; extern int64_t tsTickPerDay[3];
// query buffer management // query buffer management
extern int32_t tsQueryBufferSize; // maximum allowed usage buffer size in MB for each data node during query processing extern int32_t tsQueryBufferSize; // maximum allowed usage buffer size in MB for each data node during query processing
extern int64_t tsQueryBufferSizeBytes; // maximum allowed usage buffer size in byte for each data node extern int64_t tsQueryBufferSizeBytes; // maximum allowed usage buffer size in byte for each data node
extern int32_t tsRetrieveBlockingModel; // retrieve threads will be blocked extern bool tsRetrieveBlockingModel; // retrieve threads will be blocked
extern int8_t tsKeepOriginalColumnName; extern bool tsKeepOriginalColumnName;
extern int8_t tsDeadLockKillQuery; extern bool tsDeadLockKillQuery;
// client // client
extern int32_t tsMaxWildCardsLen; extern int32_t tsMaxWildCardsLen;
extern int32_t tsMaxRegexStringLen; extern int32_t tsMaxRegexStringLen;
extern int8_t tsTscEnableRecordSql;
extern int32_t tsMaxNumOfOrderedResults; extern int32_t tsMaxNumOfOrderedResults;
extern int32_t tsMinSlidingTime; extern int32_t tsMinSlidingTime;
extern int32_t tsMinIntervalTime; extern int32_t tsMinIntervalTime;
@ -55,10 +72,12 @@ extern float tsStreamComputDelayRatio; // the delayed computing ration of the
extern int32_t tsProjectExecInterval; extern int32_t tsProjectExecInterval;
extern int64_t tsMaxRetentWindow; extern int64_t tsMaxRetentWindow;
// system info // build info
extern char version[];
extern uint32_t tsVersion; extern char compatible_version[];
extern char gitinfo[];
extern char gitinfoOfInternal[];
extern char buildinfo[];
// lossy // lossy
extern char tsLossyColumns[]; extern char tsLossyColumns[];
@ -68,15 +87,19 @@ extern uint32_t tsMaxRange;
extern uint32_t tsCurRange; extern uint32_t tsCurRange;
extern char tsCompressor[]; extern char tsCompressor[];
// tfs
extern int32_t tsDiskCfgNum;
extern SDiskCfg tsDiskCfg[];
#define NEEDTO_COMPRESSS_MSG(size) (tsCompressMsgSize != -1 && (size) > tsCompressMsgSize) #define NEEDTO_COMPRESSS_MSG(size) (tsCompressMsgSize != -1 && (size) > tsCompressMsgSize)
void taosInitGlobalCfg(); int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDir, const char *envFile,
int32_t taosCfgDynamicOptions(char *msg); const char *apolloUrl, bool tsc);
bool taosCheckBalanceCfgOptions(const char *option, int32_t *vnodeId, int32_t *dnodeId); int32_t taosInitCfg(const char *cfgDir, const char *envFile, const char *apolloUrl, bool tsc);
void taosAddDataDir(int index, char *v1, int level, int primary); void taosCleanupCfg();
void taosReadDataDirCfg(char *v1, char *v2, char *v3); void taosCfgDynamicOptions(const char *option, const char *value);
void taosPrintDataDirCfg();
struct SConfig *taosGetCfg();
#ifdef __cplusplus #ifdef __cplusplus
} }

6
include/common/tmsg.h
View File

@ -656,9 +656,9 @@ int32_t tDeserializeSRetrieveFuncRsp(void* buf, int32_t bufLen, SRetrieveFuncRsp
typedef struct { typedef struct {
int32_t statusInterval; int32_t statusInterval;
int64_t checkTime; // 1970-01-01 00:00:00.000 int64_t checkTime; // 1970-01-01 00:00:00.000
char timezone[TSDB_TIMEZONE_LEN]; // tsTimezone char timezone[TD_TIMEZONE_LEN]; // tsTimezone
char locale[TSDB_LOCALE_LEN]; // tsLocale char locale[TD_LOCALE_LEN]; // tsLocale
char charset[TSDB_LOCALE_LEN]; // tsCharset char charset[TD_LOCALE_LEN]; // tsCharset
} SClusterCfg; } SClusterCfg;
typedef struct { typedef struct {

2
include/common/ttszip.h
View File

@ -73,7 +73,7 @@ typedef struct STSGroupBlockInfoEx {
} STSGroupBlockInfoEx; } STSGroupBlockInfoEx;
typedef struct STSBuf { typedef struct STSBuf {
FILE* f; TdFilePtr pFile;
char path[PATH_MAX]; char path[PATH_MAX];
uint32_t fileSize; uint32_t fileSize;

22
include/dnode/mgmt/dnode.h
View File

@ -25,27 +25,12 @@ extern "C" {
/* ------------------------ TYPES EXPOSED ---------------- */ /* ------------------------ TYPES EXPOSED ---------------- */
typedef struct SDnode SDnode; typedef struct SDnode SDnode;
/* ------------------------ Environment ------------------ */
typedef struct {
int32_t sver;
int32_t numOfCores;
uint16_t numOfCommitThreads;
bool enableTelem;
bool printAuth;
char timezone[TSDB_TIMEZONE_LEN];
char locale[TSDB_LOCALE_LEN];
char charset[TSDB_LOCALE_LEN];
char buildinfo[64];
char gitinfo[48];
} SDnodeEnvCfg;
/** /**
* @brief Initialize the environment * @brief Initialize the environment
* *
* @param pOption Option of the environment
* @return int32_t 0 for success and -1 for failure * @return int32_t 0 for success and -1 for failure
*/ */
int32_t dndInit(const SDnodeEnvCfg *pCfg); int32_t dndInit();
/** /**
* @brief clear the environment * @brief clear the environment
@ -56,11 +41,6 @@ void dndCleanup();
/* ------------------------ SDnode ----------------------- */ /* ------------------------ SDnode ----------------------- */
typedef struct { typedef struct {
int32_t numOfSupportVnodes; int32_t numOfSupportVnodes;
int32_t statusInterval;
float numOfThreadsPerCore;
float ratioOfQueryCores;
int32_t maxShellConns;
int32_t shellActivityTimer;
uint16_t serverPort; uint16_t serverPort;
char dataDir[TSDB_FILENAME_LEN]; char dataDir[TSDB_FILENAME_LEN];
char localEp[TSDB_EP_LEN]; char localEp[TSDB_EP_LEN];

14
include/dnode/mnode/mnode.h
View File

@ -44,26 +44,12 @@ typedef struct SMnodeLoad {
int64_t compStorage; int64_t compStorage;
} SMnodeLoad; } SMnodeLoad;
typedef struct SMnodeCfg {
int32_t sver;
bool enableTelem;
bool printAuth;
int32_t statusInterval;
int32_t shellActivityTimer;
char *timezone;
char *locale;
char *charset;
char *buildinfo;
char *gitinfo;
} SMnodeCfg;
typedef struct { typedef struct {
int32_t dnodeId; int32_t dnodeId;
int64_t clusterId; int64_t clusterId;
int8_t replica; int8_t replica;
int8_t selfIndex; int8_t selfIndex;
SReplica replicas[TSDB_MAX_REPLICA]; SReplica replicas[TSDB_MAX_REPLICA];
SMnodeCfg cfg;
SDnode *pDnode; SDnode *pDnode;
PutReqToMWriteQFp putReqToMWriteQFp; PutReqToMWriteQFp putReqToMWriteQFp;
PutReqToMReadQFp putReqToMReadQFp; PutReqToMReadQFp putReqToMReadQFp;

5
include/libs/nodes/nodes.h
View File

@ -62,7 +62,6 @@ typedef enum ENodeType {
QUERY_NODE_NODE_LIST, QUERY_NODE_NODE_LIST,
QUERY_NODE_FILL, QUERY_NODE_FILL,
QUERY_NODE_RAW_EXPR, // Only be used in parser module. QUERY_NODE_RAW_EXPR, // Only be used in parser module.
QUERY_NODE_COLUMN_REF,
QUERY_NODE_TARGET, QUERY_NODE_TARGET,
QUERY_NODE_TUPLE_DESC, QUERY_NODE_TUPLE_DESC,
QUERY_NODE_SLOT_DESC, QUERY_NODE_SLOT_DESC,
@ -81,7 +80,9 @@ typedef enum ENodeType {
// physical plan node // physical plan node
QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN, QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN,
QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN, QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN,
QUERY_NODE_PHYSICAL_PLAN_PROJECT QUERY_NODE_PHYSICAL_PLAN_PROJECT,
QUERY_NODE_PHYSICAL_PLAN_JOIN,
QUERY_NODE_PHYSICAL_PLAN_AGG
} ENodeType; } ENodeType;
/** /**

16
include/libs/nodes/plannodes.h
View File

@ -69,8 +69,6 @@ typedef struct SSlotDescNode {
ENodeType type; ENodeType type;
int16_t slotId; int16_t slotId;
SDataType dataType; SDataType dataType;
int16_t srcTupleId;
int16_t srcSlotId;
bool reserve; bool reserve;
bool output; bool output;
} SSlotDescNode; } SSlotDescNode;
@ -115,6 +113,20 @@ typedef struct SProjectPhysiNode {
SNodeList* pProjections; SNodeList* pProjections;
} SProjectPhysiNode; } SProjectPhysiNode;
typedef struct SJoinPhysiNode {
SPhysiNode node;
EJoinType joinType;
SNode* pOnConditions; // in or out tuple ?
SNodeList* pTargets;
} SJoinPhysiNode;
typedef struct SAggPhysiNode {
SPhysiNode node;
SNodeList* pExprs; // these are expression list of group_by_clause and parameter expression of aggregate function
SNodeList* pGroupKeys; // SColumnRefNode list
SNodeList* pAggFuncs;
} SAggPhysiNode;
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

16
include/libs/nodes/querynodes.h
View File

@ -58,15 +58,17 @@ typedef struct SColumnNode {
char tableAlias[TSDB_TABLE_NAME_LEN]; char tableAlias[TSDB_TABLE_NAME_LEN];
char colName[TSDB_COL_NAME_LEN]; char colName[TSDB_COL_NAME_LEN];
SNode* pProjectRef; SNode* pProjectRef;
} SColumnNode;
typedef struct SColumnRefNode {
ENodeType type;
SDataType dataType;
int16_t tupleId; int16_t tupleId;
int16_t slotId; int16_t slotId;
int16_t columnId; } SColumnNode;
} SColumnRefNode;
// typedef struct SColumnRefNode {
// ENodeType type;
// SDataType dataType;
// int16_t tupleId;
// int16_t slotId;
// int16_t columnId;
// } SColumnRefNode;
typedef struct STargetNode { typedef struct STargetNode {
ENodeType type; ENodeType type;

8
include/libs/wal/wal.h
View File

@ -120,8 +120,8 @@ typedef struct SWal {
int32_t fsyncSeq; int32_t fsyncSeq;
// meta // meta
SWalVer vers; SWalVer vers;
int64_t writeLogTfd; TdFilePtr pWriteLogTFile;
int64_t writeIdxTfd; TdFilePtr pWriteIdxTFile;
int32_t writeCur; int32_t writeCur;
SArray *fileInfoSet; SArray *fileInfoSet;
// status // status
@ -138,8 +138,8 @@ typedef struct SWal {
typedef struct SWalReadHandle { typedef struct SWalReadHandle {
SWal *pWal; SWal *pWal;
int64_t readLogTfd; TdFilePtr pReadLogTFile;
int64_t readIdxTfd; TdFilePtr pReadIdxTFile;
int64_t curFileFirstVer; int64_t curFileFirstVer;
int64_t curVersion; int64_t curVersion;
int64_t capacity; int64_t capacity;

2
include/os/os.h
View File

@ -59,6 +59,7 @@ extern "C" {
#include "osEndian.h" #include "osEndian.h"
#include "osEnv.h" #include "osEnv.h"
#include "osFile.h" #include "osFile.h"
#include "osLocale.h"
#include "osLz4.h" #include "osLz4.h"
#include "osMath.h" #include "osMath.h"
#include "osMemory.h" #include "osMemory.h"
@ -73,6 +74,7 @@ extern "C" {
#include "osThread.h" #include "osThread.h"
#include "osTime.h" #include "osTime.h"
#include "osTimer.h" #include "osTimer.h"
#include "osTimezone.h"
void osInit(); void osInit();

4
include/os/osDef.h
View File

@ -191,6 +191,10 @@ extern "C" {
#define TD_DIRSEP "/" #define TD_DIRSEP "/"
#endif #endif
#define TD_LOCALE_LEN 64
#define TD_CHARSET_LEN 64
#define TD_TIMEZONE_LEN 96
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

2
include/os/osDir.h
View File

@ -23,7 +23,7 @@ extern "C" {
void taosRemoveDir(const char *dirname); void taosRemoveDir(const char *dirname);
int32_t taosDirExist(char *dirname); int32_t taosDirExist(char *dirname);
int32_t taosMkDir(const char *dirname); int32_t taosMkDir(const char *dirname);
void taosRemoveOldFiles(char *dirname, int32_t keepDays); void taosRemoveOldFiles(const char *dirname, int32_t keepDays);
int32_t taosExpandDir(const char *dirname, char *outname, int32_t maxlen); int32_t taosExpandDir(const char *dirname, char *outname, int32_t maxlen);
int32_t taosRealPath(char *dirname, int32_t maxlen); int32_t taosRealPath(char *dirname, int32_t maxlen);

31
include/os/osEnv.h
View File

@ -16,28 +16,37 @@
#ifndef _TD_OS_ENV_H_ #ifndef _TD_OS_ENV_H_
#define _TD_OS_ENV_H_ #define _TD_OS_ENV_H_
#include "osSysinfo.h"
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
#endif #endif
extern char tsOsName[]; extern char tsOsName[];
extern char tsTimezone[];
extern char tsCharset[];
extern char tsLocale[];
extern int8_t tsDaylight;
extern bool tsEnableCoreFile;
extern int64_t tsPageSize;
extern int64_t tsOpenMax;
extern int64_t tsStreamMax;
extern int32_t tsNumOfCores;
extern int32_t tsTotalMemoryMB;
extern char configDir[];
extern char tsDataDir[]; extern char tsDataDir[];
extern char tsLogDir[]; extern char tsLogDir[];
extern char tsTempDir[]; extern char tsTempDir[];
extern char configDir[];
extern struct SDiskSpace tsLogSpace; extern SDiskSpace tsDataSpace;
extern struct SDiskSpace tsTempSpace; extern SDiskSpace tsLogSpace;
extern struct SDiskSpace tsDataSpace; extern SDiskSpace tsTempSpace;
void taosUpdateLogSpace(); void osInit();
void taosUpdateTempSpace(); void osUpdate();
void taosUpdateDataSpace(); bool osLogSpaceAvailable();
bool taosLogSpaceAvailable(); void osSetTimezone(const char *timezone);
bool taosTmpSpaceAvailable();
bool taosDataSpaceAvailable();
void taosUpdateAllSpace();
#ifdef __cplusplus #ifdef __cplusplus
} }

62
include/os/osFile.h
View File

@ -22,49 +22,63 @@ extern "C" {
#include "osSocket.h" #include "osSocket.h"
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) #ifndef ALLOW_FORBID_FUNC
typedef int32_t FileFd; #define open OPEN_FUNC_TAOS_FORBID
#else #define fopen FOPEN_FUNC_TAOS_FORBID
typedef int32_t FileFd; // #define close CLOSE_FUNC_TAOS_FORBID
// #define fclose FCLOSE_FUNC_TAOS_FORBID
#endif #endif
#define FD_INITIALIZER ((int32_t)-1)
#ifndef PATH_MAX #ifndef PATH_MAX
#define PATH_MAX 256 #define PATH_MAX 256
#endif #endif
int32_t taosLockFile(FileFd fd); typedef struct TdFile *TdFilePtr;
int32_t taosUnLockFile(FileFd fd);
int32_t taosUmaskFile(FileFd fd); #define TD_FILE_CTEATE 0x0001
#define TD_FILE_WRITE 0x0002
#define TD_FILE_READ 0x0004
#define TD_FILE_TRUNC 0x0008
#define TD_FILE_APPEND 0x0010
#define TD_FILE_TEXT 0x0020
#define TD_FILE_AUTO_DEL 0x0040
#define TD_FILE_EXCL 0x0080
int32_t taosLockFile(TdFilePtr pFile);
int32_t taosUnLockFile(TdFilePtr pFile);
int32_t taosUmaskFile(int32_t maskVal);
int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime); int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime);
int32_t taosFStatFile(FileFd fd, int64_t *size, int32_t *mtime); int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime);
FileFd taosOpenFileWrite(const char *path); TdFilePtr taosOpenFile(const char *path,int32_t tdFileOptions);
FileFd taosOpenFileCreateWrite(const char *path);
FileFd taosOpenFileCreateWriteTrunc(const char *path);
FileFd taosOpenFileCreateWriteAppend(const char *path);
FileFd taosOpenFileRead(const char *path);
FileFd taosOpenFileReadWrite(const char *path);
int64_t taosLSeekFile(FileFd fd, int64_t offset, int32_t whence); int64_t taosLSeekFile(TdFilePtr pFile, int64_t offset, int32_t whence);
int32_t taosFtruncateFile(FileFd fd, int64_t length); int32_t taosFtruncateFile(TdFilePtr pFile, int64_t length);
int32_t taosFsyncFile(FileFd fd); int32_t taosFsyncFile(TdFilePtr pFile);
int64_t taosReadFile(FileFd fd, void *buf, int64_t count); int64_t taosReadFile(TdFilePtr pFile, void *buf, int64_t count);
int64_t taosWriteFile(FileFd fd, const void *buf, int64_t count); int64_t taosPReadFile(TdFilePtr pFile, void *buf, int64_t count, int64_t offset);
int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count);
void taosFprintfFile(TdFilePtr pFile, const char *format, ...);
size_t taosGetLineFile(TdFilePtr pFile, char ** __restrict__ ptrBuf);
int32_t taosEOFFile(TdFilePtr pFile);
void taosCloseFile(FileFd fd); int64_t taosCloseFile(TdFilePtr *ppFile);
int32_t taosRenameFile(const char *oldName, const char *newName); int32_t taosRenameFile(const char *oldName, const char *newName);
int64_t taosCopyFile(const char *from, const char *to); int64_t taosCopyFile(const char *from, const char *to);
void taosGetTmpfilePath(const char *inputTmpDir, const char *fileNamePrefix, char *dstPath); void taosGetTmpfilePath(const char *inputTmpDir, const char *fileNamePrefix, char *dstPath);
int64_t taosSendFile(SocketFd dfd, FileFd sfd, int64_t *offset, int64_t size); int64_t taosSendFile(SocketFd fdDst, TdFilePtr pFileSrc, int64_t *offset, int64_t size);
int64_t taosFSendFile(FILE *outfile, FILE *infile, int64_t *offset, int64_t size); int64_t taosFSendFile(TdFilePtr pFileOut, TdFilePtr pFileIn, int64_t *offset, int64_t size);
void *taosMmapReadOnlyFile(TdFilePtr pFile, int64_t length);
bool taosValidFile(TdFilePtr pFile);
int taosGetErrorFile(TdFilePtr pFile);
#ifdef __cplusplus #ifdef __cplusplus
} }

13
include/common/ttimezone.h → include/os/osLocale.h
View File

@ -13,17 +13,22 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
#ifndef _TD_COMMON_TIMEZONE_H_ #ifndef _TD_OS_LOCALE_H_
#define _TD_COMMON_TIMEZONE_H_ #define _TD_OS_LOCALE_H_
#include "os.h"
#include "osString.h"
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
#endif #endif
void tsSetTimeZone(); char *taosCharsetReplace(char *charsetstr);
void taosGetSystemLocale(char *outLocale, char *outCharset);
void taosSetSystemLocale(const char *inLocale, const char *inCharSet);
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif
#endif /*_TD_COMMON_TIMEZONE_H_*/ #endif /*_TD_OS_LOCALE_H_*/

1
include/os/osString.h
View File

@ -45,7 +45,6 @@ int32_t taosUcs4ToMbs(void *ucs4, int32_t ucs4_max_len, char *mbs);
bool taosMbsToUcs4(const char *mbs, size_t mbs_len, char *ucs4, int32_t ucs4_max_len, int32_t *len); bool taosMbsToUcs4(const char *mbs, size_t mbs_len, char *ucs4, int32_t ucs4_max_len, int32_t *len);
int32_t tasoUcs4Compare(void *f1_ucs4, void *f2_ucs4, int32_t bytes, int8_t ncharSize); int32_t tasoUcs4Compare(void *f1_ucs4, void *f2_ucs4, int32_t bytes, int8_t ncharSize);
bool taosValidateEncodec(const char *encodec); bool taosValidateEncodec(const char *encodec);
char * taosCharsetReplace(char *charsetstr);
#ifdef __cplusplus #ifdef __cplusplus
} }

17
include/os/osSysinfo.h
View File

@ -22,31 +22,17 @@ extern "C" {
#include "os.h" #include "os.h"
#define TSDB_LOCALE_LEN 64
#define TSDB_TIMEZONE_LEN 96
typedef struct { typedef struct {
int64_t total; int64_t total;
int64_t used; int64_t used;
int64_t avail; int64_t avail;
} SDiskSize; } SDiskSize;
typedef struct SDiskSpace { typedef struct {
int64_t reserved; int64_t reserved;
SDiskSize size; SDiskSize size;
} SDiskSpace; } SDiskSpace;
extern int64_t tsPageSize;
extern int64_t tsOpenMax;
extern int64_t tsStreamMax;
extern int32_t tsNumOfCores;
extern int32_t tsTotalMemoryMB;
extern char tsTimezone[];
extern char tsLocale[];
extern char tsCharset[]; // default encode string
int32_t taosGetDiskSize(char *dataDir, SDiskSize *diskSize); int32_t taosGetDiskSize(char *dataDir, SDiskSize *diskSize);
int32_t taosGetCpuCores(); int32_t taosGetCpuCores();
void taosGetSystemInfo(); void taosGetSystemInfo();
@ -58,7 +44,6 @@ void taosGetDisk();
bool taosGetCpuUsage(float *sysCpuUsage, float *procCpuUsage); bool taosGetCpuUsage(float *sysCpuUsage, float *procCpuUsage);
bool taosGetProcMemory(float *memoryUsedMB); bool taosGetProcMemory(float *memoryUsedMB);
bool taosGetSysMemory(float *memoryUsedMB); bool taosGetSysMemory(float *memoryUsedMB);
void taosPrintOsInfo();
int taosSystem(const char *cmd); int taosSystem(const char *cmd);
void taosKillSystem(); void taosKillSystem();
int32_t taosGetSystemUUID(char *uid, int32_t uidlen); int32_t taosGetSystemUUID(char *uid, int32_t uidlen);

11
include/common/tlocale.h → include/os/osTimezone.h
View File

@ -13,17 +13,18 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
#ifndef _TD_COMMON_LOCALE_H_ #ifndef _TD_OS_TIMEZONE_H_
#define _TD_COMMON_LOCALE_H_ #define _TD_OS_TIMEZONE_H_
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
#endif #endif
void tsSetLocale(); void taosGetSystemTimezone(char *outTimezone);
void taosSetSystemTimezone(const char *inTimezone, char *outTimezone, int8_t *outDaylight);
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif /*_TD_COMMON_LOCALE_H_*/
#endif #endif
#endif /*_TD_OS_TIMEZONE_H_*/

0
include/common/tcfg.h → include/util/tcfg.h
View File

18
include/libs/config/config.h → include/util/tconfig.h
View File

@ -42,7 +42,6 @@ typedef enum {
CFG_DTYPE_INT64, CFG_DTYPE_INT64,
CFG_DTYPE_FLOAT, CFG_DTYPE_FLOAT,
CFG_DTYPE_STRING, CFG_DTYPE_STRING,
CFG_DTYPE_IPSTR,
CFG_DTYPE_DIR, CFG_DTYPE_DIR,
CFG_DTYPE_LOCALE, CFG_DTYPE_LOCALE,
CFG_DTYPE_CHARSET, CFG_DTYPE_CHARSET,
@ -52,6 +51,7 @@ typedef enum {
typedef struct SConfigItem { typedef struct SConfigItem {
ECfgSrcType stype; ECfgSrcType stype;
ECfgDataType dtype; ECfgDataType dtype;
bool tsc;
char *name; char *name;
union { union {
bool bval; bool bval;
@ -80,14 +80,14 @@ int32_t cfgGetSize(SConfig *pCfg);
SConfigItem *cfgIterate(SConfig *pCfg, SConfigItem *pIter); SConfigItem *cfgIterate(SConfig *pCfg, SConfigItem *pIter);
void cfgCancelIterate(SConfig *pCfg, SConfigItem *pIter); void cfgCancelIterate(SConfig *pCfg, SConfigItem *pIter);
SConfigItem *cfgGetItem(SConfig *pCfg, const char *name); SConfigItem *cfgGetItem(SConfig *pCfg, const char *name);
int32_t cfgSetItem(SConfig *pCfg, const char *name, const char *value, ECfgSrcType stype);
int32_t cfgAddBool(SConfig *pCfg, const char *name, bool defaultVal); int32_t cfgAddBool(SConfig *pCfg, const char *name, bool defaultVal, bool tsc);
int32_t cfgAddInt32(SConfig *pCfg, const char *name, int32_t defaultVal, int64_t minval, int64_t maxval); int32_t cfgAddInt32(SConfig *pCfg, const char *name, int32_t defaultVal, int64_t minval, int64_t maxval, bool tsc);
int32_t cfgAddInt64(SConfig *pCfg, const char *name, int64_t defaultVal, int64_t minval, int64_t maxval); int32_t cfgAddInt64(SConfig *pCfg, const char *name, int64_t defaultVal, int64_t minval, int64_t maxval, bool tsc);
int32_t cfgAddFloat(SConfig *pCfg, const char *name, float defaultVal, double minval, double maxval); int32_t cfgAddFloat(SConfig *pCfg, const char *name, float defaultVal, double minval, double maxval, bool tsc);
int32_t cfgAddString(SConfig *pCfg, const char *name, const char *defaultVal); int32_t cfgAddString(SConfig *pCfg, const char *name, const char *defaultVal, bool tsc);
int32_t cfgAddIpStr(SConfig *pCfg, const char *name, const char *defaultVa); int32_t cfgAddDir(SConfig *pCfg, const char *name, const char *defaultVal, bool tsc);
int32_t cfgAddDir(SConfig *pCfg, const char *name, const char *defaultVal);
int32_t cfgAddLocale(SConfig *pCfg, const char *name, const char *defaultVal); int32_t cfgAddLocale(SConfig *pCfg, const char *name, const char *defaultVal);
int32_t cfgAddCharset(SConfig *pCfg, const char *name, const char *defaultVal); int32_t cfgAddCharset(SConfig *pCfg, const char *name, const char *defaultVal);
int32_t cfgAddTimezone(SConfig *pCfg, const char *name, const char *defaultVal); int32_t cfgAddTimezone(SConfig *pCfg, const char *name, const char *defaultVal);
@ -95,7 +95,7 @@ int32_t cfgAddTimezone(SConfig *pCfg, const char *name, const char *defaultVal);
const char *cfgStypeStr(ECfgSrcType type); const char *cfgStypeStr(ECfgSrcType type);
const char *cfgDtypeStr(ECfgDataType type); const char *cfgDtypeStr(ECfgDataType type);
void cfgDumpCfg(SConfig *pCfg); void cfgDumpCfg(SConfig *pCfg, bool tsc, bool dump);
#ifdef __cplusplus #ifdef __cplusplus
} }

26
include/util/tfile.h
View File

@ -30,20 +30,20 @@ void tfCleanup();
// the same syntax as UNIX standard open/close/read/write // the same syntax as UNIX standard open/close/read/write
// but FD is int64_t and will never be reused // but FD is int64_t and will never be reused
int64_t tfOpenRead(const char *pathname); // int64_t tfOpenRead(const char *pathname);
int64_t tfOpenReadWrite(const char *pathname); // int64_t tfOpenReadWrite(const char *pathname);
int64_t tfOpenCreateWrite(const char *pathname); // int64_t tfOpenCreateWrite(const char *pathname);
int64_t tfOpenCreateWriteAppend(const char *pathname); // int64_t tfOpenCreateWriteAppend(const char *pathname);
int64_t tfClose(int64_t tfd); // int64_t tfClose(int64_t tfd);
int64_t tfWrite(int64_t tfd, void *buf, int64_t count); // int64_t tfWrite(int64_t tfd, void *buf, int64_t count);
int64_t tfRead(int64_t tfd, void *buf, int64_t count); // int64_t tfRead(int64_t tfd, void *buf, int64_t count);
int64_t tfPread(int64_t tfd, void *buf, int64_t count, int64_t offset); // int64_t tfPread(int64_t tfd, void *buf, int64_t count, int64_t offset);
int32_t tfFsync(int64_t tfd); // int32_t tfFsync(int64_t tfd);
bool tfValid(int64_t tfd); // bool tfValid(int64_t tfd);
int64_t tfLseek(int64_t tfd, int64_t offset, int32_t whence); // int64_t tfLseek(int64_t tfd, int64_t offset, int32_t whence);
int32_t tfFtruncate(int64_t tfd, int64_t length); // int32_t tfFtruncate(int64_t tfd, int64_t length);
void * tfMmapReadOnly(int64_t tfd, int64_t length); // void * tfMmapReadOnly(int64_t tfd, int64_t length);
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

64
include/util/tnote.h
View File

@ -1,64 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_UTIL_NOTE_H
#define _TD_UTIL_NOTE_H
#ifdef __cplusplus
extern "C" {
#endif
#define MAX_NOTE_LINE_SIZE 66000
#define NOTE_FILE_NAME_LEN 300
typedef struct {
int32_t fileNum;
int32_t maxLines;
int32_t lines;
int32_t flag;
int32_t fd;
int32_t openInProgress;
char name[NOTE_FILE_NAME_LEN];
pthread_mutex_t mutex;
} SNoteObj;
extern SNoteObj tsHttpNote;
extern SNoteObj tsTscNote;
extern SNoteObj tsInfoNote;
int32_t taosInitNotes();
void taosNotePrint(SNoteObj* pNote, const char* const format, ...);
void taosNotePrintBuffer(SNoteObj* pNote, char* buffer, int32_t len);
#define nPrintHttp(...) \
if (tsHttpEnableRecordSql) { \
taosNotePrint(&tsHttpNote, __VA_ARGS__); \
}
#define nPrintTsc(...) \
if (tsTscEnableRecordSql) { \
taosNotePrint(&tsTscNote, __VA_ARGS__); \
}
#define nInfo(buffer, len) \
if (tscEmbeddedInUtil == 1) { \
taosNotePrintBuffer(&tsInfoNote, buffer, len); \
}
#ifdef __cplusplus
}
#endif
#endif /*_TD_UTIL_NOTE_H*/

2
source/client/CMakeLists.txt
View File

@ -8,7 +8,7 @@ target_include_directories(
target_link_libraries( target_link_libraries(
taos taos
INTERFACE api INTERFACE api
PRIVATE os util common transport parser planner catalog scheduler function qcom config PRIVATE os util common transport parser planner catalog scheduler function qcom
) )
if(${BUILD_TEST}) if(${BUILD_TEST})

9
source/client/inc/clientInt.h
View File

@ -32,7 +32,7 @@ extern "C" {
#include "tmsgtype.h" #include "tmsgtype.h"
#include "trpc.h" #include "trpc.h"
#include "config.h" #include "tconfig.h"
#define CHECK_CODE_GOTO(expr, label) \ #define CHECK_CODE_GOTO(expr, label) \
do { \ do { \
@ -253,13 +253,6 @@ int hbAddConnInfo(SAppHbMgr* pAppHbMgr, SClientHbKey connKey, void* key, void* v
// --- mq // --- mq
void hbMgrInitMqHbRspHandle(); void hbMgrInitMqHbRspHandle();
// config
int32_t tscInitLog(const char *cfgDir, const char *envFile, const char *apolloUrl);
int32_t tscInitCfg(const char *cfgDir, const char *envFile, const char *apolloUrl);
extern SConfig *tscCfg;
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

208
source/client/src/clientCfg.c
View File

@ -1,208 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "clientInt.h"
#include "ulog.h"
// todo refact
SConfig *tscCfg;
static int32_t tscLoadCfg(SConfig *pConfig, const char *inputCfgDir, const char *envFile, const char *apolloUrl) {
char cfgDir[PATH_MAX] = {0};
char cfgFile[PATH_MAX + 100] = {0};
taosExpandDir(inputCfgDir, cfgDir, PATH_MAX);
snprintf(cfgFile, sizeof(cfgFile), "%s" TD_DIRSEP "taos.cfg", cfgDir);
if (cfgLoad(pConfig, CFG_STYPE_APOLLO_URL, apolloUrl) != 0) {
uError("failed to load from apollo url:%s since %s\n", apolloUrl, terrstr());
return -1;
}
if (cfgLoad(pConfig, CFG_STYPE_CFG_FILE, cfgFile) != 0) {
if (cfgLoad(pConfig, CFG_STYPE_CFG_FILE, cfgDir) != 0) {
uError("failed to load from config file:%s since %s\n", cfgFile, terrstr());
return -1;
}
}
if (cfgLoad(pConfig, CFG_STYPE_ENV_FILE, envFile) != 0) {
uError("failed to load from env file:%s since %s\n", envFile, terrstr());
return -1;
}
if (cfgLoad(pConfig, CFG_STYPE_ENV_VAR, NULL) != 0) {
uError("failed to load from global env variables since %s\n", terrstr());
return -1;
}
return 0;
}
static int32_t tscAddLogCfg(SConfig *pCfg) {
if (cfgAddDir(pCfg, "logDir", "/var/log/taos") != 0) return -1;
if (cfgAddBool(pCfg, "asyncLog", 1) != 0) return -1;
if (cfgAddInt32(pCfg, "numOfLogLines", 10000000, 1000, 2000000000) != 0) return -1;
if (cfgAddInt32(pCfg, "logKeepDays", 0, -365000, 365000) != 0) return -1;
if (cfgAddInt32(pCfg, "debugFlag", 0, 0, 255) != 0) return -1;
if (cfgAddInt32(pCfg, "cDebugFlag", 0, 0, 255) != 0) return -1;
if (cfgAddInt32(pCfg, "jniDebugFlag", 0, 0, 255) != 0) return -1;
if (cfgAddInt32(pCfg, "tmrDebugFlag", 0, 0, 255) != 0) return -1;
if (cfgAddInt32(pCfg, "uDebugFlag", 0, 0, 255) != 0) return -1;
if (cfgAddInt32(pCfg, "rpcDebugFlag", 0, 0, 255) != 0) return -1;
return 0;
}
static int32_t tscSetLogCfg(SConfig *pCfg) {
tstrncpy(tsLogDir, cfgGetItem(pCfg, "logDir")->str, PATH_MAX);
tsAsyncLog = cfgGetItem(pCfg, "asyncLog")->bval;
tsNumOfLogLines = cfgGetItem(pCfg, "numOfLogLines")->i32;
tsLogKeepDays = cfgGetItem(pCfg, "logKeepDays")->i32;
cDebugFlag = cfgGetItem(pCfg, "cDebugFlag")->i32;
jniDebugFlag = cfgGetItem(pCfg, "jniDebugFlag")->i32;
tmrDebugFlag = cfgGetItem(pCfg, "tmrDebugFlag")->i32;
uDebugFlag = cfgGetItem(pCfg, "uDebugFlag")->i32;
rpcDebugFlag = cfgGetItem(pCfg, "rpcDebugFlag")->i32;
int32_t debugFlag = cfgGetItem(pCfg, "debugFlag")->i32;
taosSetAllDebugFlag(debugFlag);
return 0;
}
int32_t tscInitLog(const char *cfgDir, const char *envFile, const char *apolloUrl) {
if (tsLogInited) return 0;
SConfig *pCfg = cfgInit();
if (pCfg == NULL) return -1;
if (tscAddLogCfg(pCfg) != 0) {
printf("failed to add log cfg since %s\n", terrstr());
cfgCleanup(pCfg);
return -1;
}
if (tscLoadCfg(pCfg, cfgDir, envFile, apolloUrl) != 0) {
printf("failed to load log cfg since %s\n", terrstr());
cfgCleanup(pCfg);
return -1;
}
if (tscSetLogCfg(pCfg) != 0) {
printf("failed to set log cfg since %s\n", terrstr());
cfgCleanup(pCfg);
return -1;
}
const int32_t maxLogFileNum = 10;
if (taosInitLog("taoslog", maxLogFileNum) != 0) {
printf("failed to init log file since %s\n", terrstr());
cfgCleanup(pCfg);
return -1;
}
cfgDumpCfg(pCfg);
cfgCleanup(pCfg);
return 0;
}
static int32_t tscAddEpCfg(SConfig *pCfg) {
char defaultFqdn[TSDB_FQDN_LEN] = {0};
if (taosGetFqdn(defaultFqdn) != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
if (cfgAddString(pCfg, "fqdn", defaultFqdn) != 0) return -1;
int32_t defaultServerPort = 6030;
if (cfgAddInt32(pCfg, "serverPort", defaultServerPort, 1, 65056) != 0) return -1;
char defaultFirstEp[TSDB_EP_LEN] = {0};
char defaultSecondEp[TSDB_EP_LEN] = {0};
snprintf(defaultFirstEp, TSDB_EP_LEN, "%s:%d", defaultFqdn, defaultServerPort);
snprintf(defaultSecondEp, TSDB_EP_LEN, "%s:%d", defaultFqdn, defaultServerPort);
if (cfgAddString(pCfg, "firstEp", defaultFirstEp) != 0) return -1;
if (cfgAddString(pCfg, "secondEp", defaultSecondEp) != 0) return -1;
return 0;
}
static int32_t tscAddCfg(SConfig *pCfg) {
if (tscAddEpCfg(pCfg) != 0) return -1;
// if (cfgAddString(pCfg, "buildinfo", buildinfo) != 0) return -1;
// if (cfgAddString(pCfg, "gitinfo", gitinfo) != 0) return -1;
// if (cfgAddString(pCfg, "version", version) != 0) return -1;
// if (cfgAddDir(pCfg, "dataDir", tsDataDir) != 0) return -1;
if (cfgAddTimezone(pCfg, "timezone", "") != 0) return -1;
if (cfgAddLocale(pCfg, "locale", "") != 0) return -1;
if (cfgAddCharset(pCfg, "charset", "") != 0) return -1;
if (cfgAddInt32(pCfg, "numOfCores", 1, 1, 100000) != 0) return -1;
if (cfgAddInt32(pCfg, "numOfCommitThreads", 4, 1, 1000) != 0) return -1;
// if (cfgAddBool(pCfg, "telemetryReporting", 0) != 0) return -1;
if (cfgAddBool(pCfg, "enableCoreFile", 0) != 0) return -1;
// if (cfgAddInt32(pCfg, "supportVnodes", 256, 0, 65536) != 0) return -1;
if (cfgAddInt32(pCfg, "statusInterval", 1, 1, 30) != 0) return -1;
if (cfgAddFloat(pCfg, "numOfThreadsPerCore", 1, 0, 10) != 0) return -1;
if (cfgAddFloat(pCfg, "ratioOfQueryCores", 1, 0, 5) != 0) return -1;
if (cfgAddInt32(pCfg, "shellActivityTimer", 3, 1, 120) != 0) return -1;
if (cfgAddInt32(pCfg, "maxConnections", 50000, 1, 100000) != 0) return -1;
return 0;
}
int32_t tscCheckCfg(SConfig *pCfg) {
bool enableCore = cfgGetItem(pCfg, "enableCoreFile")->bval;
taosSetCoreDump(enableCore);
return 0;
}
SConfig *tscInitCfgImp(const char *cfgDir, const char *envFile, const char *apolloUrl) {
SConfig *pCfg = cfgInit();
if (pCfg == NULL) return NULL;
if (tscAddCfg(pCfg) != 0) {
uError("failed to init tsc cfg since %s", terrstr());
cfgCleanup(pCfg);
return NULL;
}
if (tscLoadCfg(pCfg, cfgDir, envFile, apolloUrl) != 0) {
printf("failed to load tsc cfg since %s\n", terrstr());
cfgCleanup(pCfg);
return NULL;
}
if (tscCheckCfg(pCfg) != 0) {
uError("failed to check cfg since %s", terrstr());
cfgCleanup(pCfg);
return NULL;
}
cfgDumpCfg(pCfg);
return pCfg;
}
int32_t tscInitCfg(const char *cfgDir, const char *envFile, const char *apolloUrl) {
tscCfg = tscInitCfgImp(cfgDir, envFile, apolloUrl);
if (tscCfg == NULL) return -1;
return 0;
}

25
source/client/src/clientEnv.c
View File

@ -22,11 +22,9 @@
#include "tcache.h" #include "tcache.h"
#include "tglobal.h" #include "tglobal.h"
#include "tmsg.h" #include "tmsg.h"
#include "tnote.h"
#include "tref.h" #include "tref.h"
#include "trpc.h" #include "trpc.h"
#include "ttime.h" #include "ttime.h"
#include "ttimezone.h"
#define TSC_VAR_NOT_RELEASE 1 #define TSC_VAR_NOT_RELEASE 1
#define TSC_VAR_RELEASED 0 #define TSC_VAR_RELEASED 0
@ -95,10 +93,10 @@ void *openTransporter(const char *user, const char *auth, int32_t numOfThread) {
rpcInit.numOfThreads = numOfThread; rpcInit.numOfThreads = numOfThread;
rpcInit.cfp = processMsgFromServer; rpcInit.cfp = processMsgFromServer;
rpcInit.pfp = persistConnForSpecificMsg; rpcInit.pfp = persistConnForSpecificMsg;
rpcInit.sessions = cfgGetItem(tscCfg, "maxConnections")->i32; rpcInit.sessions = tsMaxConnections;
rpcInit.connType = TAOS_CONN_CLIENT; rpcInit.connType = TAOS_CONN_CLIENT;
rpcInit.user = (char *)user; rpcInit.user = (char *)user;
rpcInit.idleTime = cfgGetItem(tscCfg, "shellActivityTimer")->i32 * 1000; rpcInit.idleTime = tsShellActivityTimer * 1000;
rpcInit.ckey = "key"; rpcInit.ckey = "key";
rpcInit.spi = 1; rpcInit.spi = 1;
rpcInit.secret = (char *)auth; rpcInit.secret = (char *)auth;
@ -214,17 +212,16 @@ void taos_init_imp(void) {
deltaToUtcInitOnce(); deltaToUtcInitOnce();
if (tscInitLog(configDir, NULL, NULL) != 0) { if (taosCreateLog("taoslog", 10, configDir, NULL, NULL, 1) != 0) {
tscInitRes = -1; tscInitRes = -1;
return; return;
} }
if (tscInitCfg(configDir, NULL, NULL) != 0) { if (taosInitCfg(configDir, NULL, NULL, 1) != 0) {
tscInitRes = -1; tscInitRes = -1;
return; return;
} }
taosInitNotes();
initMsgHandleFp(); initMsgHandleFp();
initQueryModuleMsgHandle(); initQueryModuleMsgHandle();
@ -298,7 +295,7 @@ int taos_options_imp(TSDB_OPTION option, const char *str) {
assert(cfg != NULL); assert(cfg != NULL);
size_t len = strlen(str); size_t len = strlen(str);
if (len == 0 || len > TSDB_LOCALE_LEN) { if (len == 0 || len > TD_LOCALE_LEN) {
tscInfo("Invalid locale:%s, use default", str); tscInfo("Invalid locale:%s, use default", str);
return -1; return -1;
} }
@ -316,7 +313,7 @@ int taos_options_imp(TSDB_OPTION option, const char *str) {
return -1; return -1;
} }
tstrncpy(tsLocale, defaultLocale, TSDB_LOCALE_LEN); tstrncpy(tsLocale, defaultLocale, TD_LOCALE_LEN);
} }
// set the user specified locale // set the user specified locale
@ -330,7 +327,7 @@ int taos_options_imp(TSDB_OPTION option, const char *str) {
tscInfo("failed to set locale:%s, current locale:%s", str, tsLocale); tscInfo("failed to set locale:%s, current locale:%s", str, tsLocale);
} }
tstrncpy(tsLocale, locale, TSDB_LOCALE_LEN); tstrncpy(tsLocale, locale, TD_LOCALE_LEN);
char *charset = strrchr(tsLocale, sep); char *charset = strrchr(tsLocale, sep);
if (charset != NULL) { if (charset != NULL) {
@ -345,7 +342,7 @@ int taos_options_imp(TSDB_OPTION option, const char *str) {
tscInfo("charset changed from %s to %s", tsCharset, charset); tscInfo("charset changed from %s to %s", tsCharset, charset);
} }
tstrncpy(tsCharset, charset, TSDB_LOCALE_LEN); tstrncpy(tsCharset, charset, TD_LOCALE_LEN);
cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION; cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION;
} else { } else {
@ -369,7 +366,7 @@ int taos_options_imp(TSDB_OPTION option, const char *str) {
assert(cfg != NULL); assert(cfg != NULL);
size_t len = strlen(str); size_t len = strlen(str);
if (len == 0 || len > TSDB_LOCALE_LEN) { if (len == 0 || len > TD_LOCALE_LEN) {
tscInfo("failed to set charset:%s", str); tscInfo("failed to set charset:%s", str);
return -1; return -1;
} }
@ -382,7 +379,7 @@ int taos_options_imp(TSDB_OPTION option, const char *str) {
tscInfo("charset changed from %s to %s", tsCharset, str); tscInfo("charset changed from %s to %s", tsCharset, str);
} }
tstrncpy(tsCharset, str, TSDB_LOCALE_LEN); tstrncpy(tsCharset, str, TD_LOCALE_LEN);
cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION; cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION;
} else { } else {
tscInfo("charset:%s not valid", str); tscInfo("charset:%s not valid", str);
@ -400,7 +397,7 @@ int taos_options_imp(TSDB_OPTION option, const char *str) {
assert(cfg != NULL); assert(cfg != NULL);
if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_OPTION) { if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_OPTION) {
tstrncpy(tsTimezone, str, TSDB_TIMEZONE_LEN); tstrncpy(tsTimezone, str, TD_TIMEZONE_LEN);
tsSetTimeZone(); tsSetTimeZone();
cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION; cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION;
tscDebug("timezone set:%s, input:%s by taos_options", tsTimezone, str); tscDebug("timezone set:%s, input:%s by taos_options", tsTimezone, str);

49
source/client/src/clientImpl.c
View File

@ -8,11 +8,10 @@
#include "tep.h" #include "tep.h"
#include "tglobal.h" #include "tglobal.h"
#include "tmsgtype.h" #include "tmsgtype.h"
#include "tnote.h"
#include "tpagedbuf.h" #include "tpagedbuf.h"
#include "tref.h" #include "tref.h"
static int32_t initEpSetFromCfg(const char* ip, uint16_t port, SCorEpSet* pEpSet); static int32_t initEpSetFromCfg(const char* firstEp, const char* secondEp, SCorEpSet* pEpSet);
static SMsgSendInfo* buildConnectMsg(SRequestObj* pRequest); static SMsgSendInfo* buildConnectMsg(SRequestObj* pRequest);
static void destroySendMsgInfo(SMsgSendInfo* pMsgBody); static void destroySendMsgInfo(SMsgSendInfo* pMsgBody);
static void setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp); static void setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp);
@ -81,7 +80,19 @@ TAOS* taos_connect_internal(const char* ip, const char* user, const char* pass,
} }
SCorEpSet epSet = {0}; SCorEpSet epSet = {0};
initEpSetFromCfg(ip, port, &epSet); if (ip) {
if (initEpSetFromCfg(ip, NULL, &epSet) < 0) {
return NULL;
}
if (port) {
epSet.epSet.eps[0].port = port;
}
} else {
if (initEpSetFromCfg(tsFirst, tsSecond, &epSet) < 0) {
return NULL;
}
}
char* key = getClusterKey(user, secretEncrypt, ip, port); char* key = getClusterKey(user, secretEncrypt, ip, port);
SAppInstInfo** pInst = NULL; SAppInstInfo** pInst = NULL;
@ -242,8 +253,6 @@ TAOS_RES* taos_query_l(TAOS* taos, const char* sql, int sqlLen) {
return NULL; return NULL;
} }
nPrintTsc("%s", sql)
SRequestObj* pRequest = NULL; SRequestObj* pRequest = NULL;
SQueryNode* pQueryNode = NULL; SQueryNode* pQueryNode = NULL;
SArray* pNodeList = taosArrayInit(4, sizeof(struct SQueryNodeAddr)); SArray* pNodeList = taosArrayInit(4, sizeof(struct SQueryNodeAddr));
@ -270,41 +279,33 @@ _return:
return pRequest; return pRequest;
} }
int initEpSetFromCfg(const char* ip, uint16_t port, SCorEpSet* pEpSet) { int initEpSetFromCfg(const char* firstEp, const char* secondEp, SCorEpSet* pEpSet) {
SConfigItem* pFirst = cfgGetItem(tscCfg, "firstEp"); pEpSet->version = 0;
SConfigItem* pSecond = cfgGetItem(tscCfg, "secondEp");
SConfigItem* pPort = cfgGetItem(tscCfg, "serverPort");
// init mnode ip set // init mnode ip set
SEpSet* mgmtEpSet = &(pEpSet->epSet); SEpSet* mgmtEpSet = &(pEpSet->epSet);
mgmtEpSet->numOfEps = 0; mgmtEpSet->numOfEps = 0;
mgmtEpSet->inUse = 0; mgmtEpSet->inUse = 0;
pEpSet->version = 0;
if (ip != NULL) { if (firstEp && firstEp[0] != 0) {
taosGetFqdnPortFromEp(ip, (uint16_t)pPort->i32, &mgmtEpSet->eps[0]); if (strlen(firstEp) >= TSDB_EP_LEN) {
mgmtEpSet->numOfEps++;
if (port) {
mgmtEpSet->eps[0].port = port;
}
} else {
if (pFirst->str[0] != 0) {
if (strlen(pFirst->str) >= TSDB_EP_LEN) {
terrno = TSDB_CODE_TSC_INVALID_FQDN; terrno = TSDB_CODE_TSC_INVALID_FQDN;
return -1; return -1;
} }
taosGetFqdnPortFromEp(pFirst->str, (uint16_t)pPort->i32, &mgmtEpSet->eps[0]);
taosGetFqdnPortFromEp(firstEp, &mgmtEpSet->eps[0]);
mgmtEpSet->numOfEps++; mgmtEpSet->numOfEps++;
} }
if (pSecond->str[0] != 0) {
if (strlen(pSecond->str) >= TSDB_EP_LEN) { if (secondEp && secondEp[0] != 0) {
if (strlen(secondEp) >= TSDB_EP_LEN) {
terrno = TSDB_CODE_TSC_INVALID_FQDN; terrno = TSDB_CODE_TSC_INVALID_FQDN;
return -1; return -1;
} }
taosGetFqdnPortFromEp(pSecond->str, (uint16_t)pPort->i32, &mgmtEpSet->eps[1]);
taosGetFqdnPortFromEp(secondEp, &mgmtEpSet->eps[mgmtEpSet->numOfEps]);
mgmtEpSet->numOfEps++; mgmtEpSet->numOfEps++;
} }
}
if (mgmtEpSet->numOfEps == 0) { if (mgmtEpSet->numOfEps == 0) {
terrno = TSDB_CODE_TSC_INVALID_FQDN; terrno = TSDB_CODE_TSC_INVALID_FQDN;

1
source/client/src/tmq.c
View File

@ -24,7 +24,6 @@
#include "tep.h" #include "tep.h"
#include "tglobal.h" #include "tglobal.h"
#include "tmsgtype.h" #include "tmsgtype.h"
#include "tnote.h"
#include "tpagedbuf.h" #include "tpagedbuf.h"
#include "tref.h" #include "tref.h"

4
source/client/test/CMakeLists.txt
View File

@ -8,13 +8,13 @@ AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
ADD_EXECUTABLE(clientTest clientTests.cpp) ADD_EXECUTABLE(clientTest clientTests.cpp)
TARGET_LINK_LIBRARIES( TARGET_LINK_LIBRARIES(
clientTest clientTest
PUBLIC os util common transport parser catalog scheduler function gtest taos qcom config PUBLIC os util common transport parser catalog scheduler function gtest taos qcom
) )
ADD_EXECUTABLE(tmqTest tmqTest.cpp) ADD_EXECUTABLE(tmqTest tmqTest.cpp)
TARGET_LINK_LIBRARIES( TARGET_LINK_LIBRARIES(
tmqTest tmqTest
PUBLIC os util common transport parser catalog scheduler function gtest taos qcom config PUBLIC os util common transport parser catalog scheduler function gtest taos qcom
) )
TARGET_INCLUDE_DIRECTORIES( TARGET_INCLUDE_DIRECTORIES(

2
source/client/test/clientTests.cpp
View File

@ -49,7 +49,7 @@ int main(int argc, char** argv) {
} }
TEST(testCase, driverInit_Test) { TEST(testCase, driverInit_Test) {
taosInitGlobalCfg(); // taosInitGlobalCfg();
// taos_init(); // taos_init();
} }

2
source/client/test/tmqTest.cpp
View File

@ -34,7 +34,7 @@ int main(int argc, char** argv) {
} }
TEST(testCase, driverInit_Test) { TEST(testCase, driverInit_Test) {
taosInitGlobalCfg(); // taosInitGlobalCfg();
// taos_init(); // taos_init();
} }

4
source/common/src/tep.c
View File

@ -4,7 +4,7 @@
#include "tglobal.h" #include "tglobal.h"
#include "tlockfree.h" #include "tlockfree.h"
int taosGetFqdnPortFromEp(const char *ep, uint16_t defaultPort, SEp* pEp) { int taosGetFqdnPortFromEp(const char *ep, SEp* pEp) {
pEp->port = 0; pEp->port = 0;
strcpy(pEp->fqdn, ep); strcpy(pEp->fqdn, ep);
@ -15,7 +15,7 @@ int taosGetFqdnPortFromEp(const char *ep, uint16_t defaultPort, SEp* pEp) {
} }
if (pEp->port == 0) { if (pEp->port == 0) {
pEp->port = defaultPort; pEp->port = tsServerPort;
return -1; return -1;
} }

769
source/common/src/tglobal.c
View File

@ -19,21 +19,39 @@
#include "taosdef.h" #include "taosdef.h"
#include "taoserror.h" #include "taoserror.h"
#include "tcompare.h" #include "tcompare.h"
#include "tconfig.h"
#include "tep.h" #include "tep.h"
#include "tglobal.h" #include "tglobal.h"
#include "tlocale.h"
#include "tlog.h" #include "tlog.h"
#include "ttimezone.h"
#include "tutil.h" #include "tutil.h"
#include "ulog.h" #include "ulog.h"
SConfig *tsCfg = NULL;
// cluster
char tsFirst[TSDB_EP_LEN] = {0};
char tsSecond[TSDB_EP_LEN] = {0};
char tsLocalFqdn[TSDB_FQDN_LEN] = {0};
char tsLocalEp[TSDB_EP_LEN] = {0}; // Local End Point, hostname:port
uint16_t tsServerPort = 6030;
int32_t tsVersion = 30000000;
int32_t tsStatusInterval = 1; // second
bool tsEnableTelemetryReporting = 0;
// common // common
int8_t tsDaylight = 0; int32_t tsRpcTimer = 300;
int32_t tsRpcMaxTime = 600; // seconds;
bool tsRpcForceTcp = 1; // disable this, means query, show command use udp protocol as default
int32_t tsMaxShellConns = 50000;
int32_t tsMaxConnections = 50000;
int32_t tsShellActivityTimer = 3; // second
float tsNumOfThreadsPerCore = 1.0f;
int32_t tsNumOfCommitThreads = 4;
float tsRatioOfQueryCores = 1.0f;
int32_t tsMaxBinaryDisplayWidth = 30; int32_t tsMaxBinaryDisplayWidth = 30;
int8_t tsEnableSlaveQuery = 1; bool tsEnableSlaveQuery = 1;
int8_t tsEnableAdjustMaster = 1; bool tsPrintAuth = 0;
int8_t tsPrintAuth = 0;
/* /*
* denote if the server needs to compress response message at the application layer to client, including query rsp, * denote if the server needs to compress response message at the application layer to client, including query rsp,
* metricmeta rsp, and multi-meter query rsp message body. The client compress the submit message to server. * metricmeta rsp, and multi-meter query rsp message body. The client compress the submit message to server.
@ -60,8 +78,6 @@ int32_t tsCompatibleModel = 1;
int32_t tsMaxWildCardsLen = TSDB_PATTERN_STRING_DEFAULT_LEN; int32_t tsMaxWildCardsLen = TSDB_PATTERN_STRING_DEFAULT_LEN;
int32_t tsMaxRegexStringLen = TSDB_REGEX_STRING_DEFAULT_LEN; int32_t tsMaxRegexStringLen = TSDB_REGEX_STRING_DEFAULT_LEN;
int8_t tsTscEnableRecordSql = 0;
// the maximum number of results for projection query on super table that are returned from // the maximum number of results for projection query on super table that are returned from
// one virtual node, to order according to timestamp // one virtual node, to order according to timestamp
int32_t tsMaxNumOfOrderedResults = 100000; int32_t tsMaxNumOfOrderedResults = 100000;
@ -98,18 +114,21 @@ int32_t tsQueryBufferSize = -1;
int64_t tsQueryBufferSizeBytes = -1; int64_t tsQueryBufferSizeBytes = -1;
// in retrieve blocking model, the retrieve threads will wait for the completion of the query processing. // in retrieve blocking model, the retrieve threads will wait for the completion of the query processing.
int32_t tsRetrieveBlockingModel = 0; bool tsRetrieveBlockingModel = 0;
// last_row(*), first(*), last_row(ts, col1, col2) query, the result fields will be the original column name // last_row(*), first(*), last_row(ts, col1, col2) query, the result fields will be the original column name
int8_t tsKeepOriginalColumnName = 0; bool tsKeepOriginalColumnName = 0;
// long query death-lock // long query death-lock
int8_t tsDeadLockKillQuery = 0; bool tsDeadLockKillQuery = 0;
// tsdb config // tsdb config
// For backward compatibility // For backward compatibility
bool tsdbForceKeepFile = false; bool tsdbForceKeepFile = false;
int32_t tsDiskCfgNum = 0;
SDiskCfg tsDiskCfg[TFS_MAX_DISKS] = {0};
/* /*
* minimum scale for whole system, millisecond by default * minimum scale for whole system, millisecond by default
* for TSDB_TIME_PRECISION_MILLI: 86400000L * for TSDB_TIME_PRECISION_MILLI: 86400000L
@ -118,13 +137,7 @@ bool tsdbForceKeepFile = false;
*/ */
int64_t tsTickPerDay[] = {86400000L, 86400000000L, 86400000000000L}; int64_t tsTickPerDay[] = {86400000L, 86400000000L, 86400000000000L};
// system info
int32_t tsTotalMemoryMB = 0;
uint32_t tsVersion = 0;
//
// lossy compress 6 // lossy compress 6
//
char tsLossyColumns[32] = ""; // "float|double" means all float and double columns can be lossy compressed. set empty char tsLossyColumns[32] = ""; // "float|double" means all float and double columns can be lossy compressed. set empty
// can close lossy compress. // can close lossy compress.
// below option can take effect when tsLossyColumns not empty // below option can take effect when tsLossyColumns not empty
@ -134,428 +147,360 @@ uint32_t tsMaxRange = 500; // max range
uint32_t tsCurRange = 100; // range uint32_t tsCurRange = 100; // range
char tsCompressor[32] = "ZSTD_COMPRESSOR"; // ZSTD_COMPRESSOR or GZIP_COMPRESSOR char tsCompressor[32] = "ZSTD_COMPRESSOR"; // ZSTD_COMPRESSOR or GZIP_COMPRESSOR
static void taosAddDataDir(int32_t index, char *v1, int32_t level, int32_t primary) {
int32_t (*monStartSystemFp)() = NULL; tstrncpy(tsDiskCfg[index].dir, v1, TSDB_FILENAME_LEN);
void (*monStopSystemFp)() = NULL; tsDiskCfg[index].level = level;
void (*monExecuteSQLFp)(char *sql) = NULL; tsDiskCfg[index].primary = primary;
uTrace("dataDir:%s, level:%d primary:%d is configured", v1, level, primary);
char *qtypeStr[] = {"rpc", "fwd", "wal", "cq", "query"};
static pthread_once_t tsInitGlobalCfgOnce = PTHREAD_ONCE_INIT;
int32_t taosCfgDynamicOptions(char *msg) {
#if 0
char *option, *value;
int32_t olen, vlen;
int32_t vint = 0;
paGetToken(msg, &option, &olen);
if (olen == 0) return -1;
paGetToken(option + olen + 1, &value, &vlen);
if (vlen == 0)
vint = 135;
else {
vint = atoi(value);
} }
uInfo("change dynamic option: %s, value: %d", option, vint); static void taosReadDataDirCfg(char *v1, char *v2, char *v3) {
if (tsDiskCfgNum == 1) {
for (int32_t i = 0; i < tsGlobalConfigNum; ++i) { SDiskCfg *cfg = &tsDiskCfg[0];
SGlobalCfg *cfg = tsGlobalConfig + i; uInfo("dataDir:%s, level:%d primary:%d is replaced by %s", cfg->dir, cfg->level, cfg->primary, v1);
// if (!(cfg->cfgType & TSDB_CFG_CTYPE_B_LOG)) continue; }
if (cfg->valType != TAOS_CFG_VTYPE_INT32 && cfg->valType != TAOS_CFG_VTYPE_INT8) continue; taosAddDataDir(0, v1, 0, 1);
tsDiskCfgNum = 1;
int32_t cfgLen = (int32_t)strlen(cfg->option);
if (cfgLen != olen) continue;
if (strncasecmp(option, cfg->option, olen) != 0) continue;
if (cfg->valType == TAOS_CFG_VTYPE_INT32) {
*((int32_t *)cfg->ptr) = vint;
} else {
*((int8_t *)cfg->ptr) = (int8_t)vint;
} }
if (strncasecmp(cfg->option, "monitor", olen) == 0) { static void taosPrintDataDirCfg() {
if (1 == vint) { for (int32_t i = 0; i < tsDiskCfgNum; ++i) {
if (monStartSystemFp) { SDiskCfg *cfg = &tsDiskCfg[i];
(*monStartSystemFp)(); uInfo(" dataDir: %s", cfg->dir);
uInfo("monitor is enabled");
} else {
uError("monitor can't be updated, for monitor not initialized");
}
} else {
if (monStopSystemFp) {
(*monStopSystemFp)();
uInfo("monitor is disabled");
} else {
uError("monitor can't be updated, for monitor not initialized");
} }
} }
return 0;
struct SConfig *taosGetCfg() {
return tsCfg;
} }
if (strncasecmp(cfg->option, "debugFlag", olen) == 0) {
taosSetAllDebugFlag(); static int32_t taosLoadCfg(SConfig *pCfg, const char *inputCfgDir, const char *envFile, const char *apolloUrl) {
char cfgDir[PATH_MAX] = {0};
char cfgFile[PATH_MAX + 100] = {0};
taosExpandDir(inputCfgDir, cfgDir, PATH_MAX);
snprintf(cfgFile, sizeof(cfgFile), "%s" TD_DIRSEP "taos.cfg", cfgDir);
if (cfgLoad(pCfg, CFG_STYPE_APOLLO_URL, apolloUrl) != 0) {
uError("failed to load from apollo url:%s since %s\n", apolloUrl, terrstr());
return -1;
} }
if (cfgLoad(pCfg, CFG_STYPE_CFG_FILE, cfgFile) != 0) {
if (cfgLoad(pCfg, CFG_STYPE_CFG_FILE, cfgDir) != 0) {
uError("failed to load from config file:%s since %s\n", cfgFile, terrstr());
return -1;
}
}
if (cfgLoad(pCfg, CFG_STYPE_ENV_FILE, envFile) != 0) {
uError("failed to load from env file:%s since %s\n", envFile, terrstr());
return -1;
}
if (cfgLoad(pCfg, CFG_STYPE_ENV_VAR, NULL) != 0) {
uError("failed to load from global env variables since %s\n", terrstr());
return -1;
}
return 0; return 0;
} }
if (strncasecmp(option, "resetlog", 8) == 0) { static void taosAddClientLogCfg(SConfig *pCfg) {
taosResetLog(); cfgAddDir(pCfg, "logDir", tsLogDir, 1);
taosPrintCfg(); cfgAddFloat(pCfg, "minimalLogDirGB", 1.0f, 0.001f, 10000000, 1);
return 0; cfgAddInt32(pCfg, "numOfLogLines", tsNumOfLogLines, 1000, 2000000000, 1);
cfgAddBool(pCfg, "asyncLog", tsAsyncLog, 1);
cfgAddInt32(pCfg, "logKeepDays", 0, -365000, 365000, 1);
cfgAddInt32(pCfg, "cDebugFlag", cDebugFlag, 0, 255, 1);
cfgAddInt32(pCfg, "uDebugFlag", uDebugFlag, 0, 255, 1);
cfgAddInt32(pCfg, "rpcDebugFlag", rpcDebugFlag, 0, 255, 1);
cfgAddInt32(pCfg, "tmrDebugFlag", tmrDebugFlag, 0, 255, 1);
cfgAddInt32(pCfg, "jniDebugFlag", jniDebugFlag, 0, 255, 1);
cfgAddInt32(pCfg, "simDebugFlag", 143, 0, 255, 1);
cfgAddDir(pCfg, "configDir", configDir, 1);
cfgAddDir(pCfg, "scriptDir", configDir, 1);
cfgAddInt32(pCfg, "debugFlag", 0, 0, 255, 1);
} }
if (strncasecmp(option, "resetQueryCache", 15) == 0) { static void taosAddServerLogCfg(SConfig *pCfg) {
if (monExecuteSQLFp) { cfgAddInt32(pCfg, "dDebugFlag", dDebugFlag, 0, 255, 0);
(*monExecuteSQLFp)("resetQueryCache"); cfgAddInt32(pCfg, "vDebugFlag", vDebugFlag, 0, 255, 0);
uInfo("resetquerycache is executed"); cfgAddInt32(pCfg, "mDebugFlag", mDebugFlag, 0, 255, 0);
} else { cfgAddInt32(pCfg, "qDebugFlag", qDebugFlag, 0, 255, 0);
uError("resetquerycache can't be executed, for monitor not started"); cfgAddInt32(pCfg, "wDebugFlag", wDebugFlag, 0, 255, 0);
cfgAddInt32(pCfg, "sDebugFlag", sDebugFlag, 0, 255, 0);
cfgAddInt32(pCfg, "tsdbDebugFlag", tsdbDebugFlag, 0, 255, 0);
cfgAddInt32(pCfg, "tqDebugFlag", tqDebugFlag, 0, 255, 0);
cfgAddInt32(pCfg, "fsDebugFlag", fsDebugFlag, 0, 255, 0);
}
static void taosAddClientCfg(SConfig *pCfg) {
char defaultFqdn[TSDB_FQDN_LEN] = {0};
int32_t defaultServerPort = 6030;
char defaultFirstEp[TSDB_EP_LEN] = {0};
char defaultSecondEp[TSDB_EP_LEN] = {0};
taosGetFqdn(defaultFqdn);
snprintf(defaultFirstEp, TSDB_EP_LEN, "%s:%d", defaultFqdn, defaultServerPort);
snprintf(defaultSecondEp, TSDB_EP_LEN, "%s:%d", defaultFqdn, defaultServerPort);
cfgAddString(pCfg, "firstEp", defaultFirstEp, 1);
cfgAddString(pCfg, "secondEp", defaultSecondEp, 1);
cfgAddString(pCfg, "fqdn", defaultFqdn, 1);
cfgAddInt32(pCfg, "serverPort", defaultServerPort, 1, 65056, 1);
cfgAddDir(pCfg, "tempDir", tsTempDir, 1);
cfgAddFloat(pCfg, "minimalTempDirGB", 1.0f, 0.001f, 10000000, 1);
cfgAddFloat(pCfg, "numOfThreadsPerCore", tsNumOfThreadsPerCore, 0, 10, 1);
cfgAddInt32(pCfg, "maxTmrCtrl", tsMaxTmrCtrl, 8, 2048, 1);
cfgAddInt32(pCfg, "rpcTimer", tsRpcTimer, 100, 3000, 1);
cfgAddInt32(pCfg, "rpcMaxTime", tsRpcMaxTime, 100, 7200, 1);
cfgAddBool(pCfg, "rpcForceTcp", tsRpcForceTcp, 1);
cfgAddInt32(pCfg, "shellActivityTimer", tsShellActivityTimer, 1, 120, 1);
cfgAddInt32(pCfg, "compressMsgSize", tsCompressMsgSize, -1, 100000000, 1);
cfgAddInt32(pCfg, "compressColData", tsCompressColData, -1, 100000000, 1);
cfgAddInt32(pCfg, "maxWildCardsLength", tsMaxWildCardsLen, 0, TSDB_MAX_FIELD_LEN, 1);
cfgAddInt32(pCfg, "maxRegexStringLen", tsMaxRegexStringLen, 0, TSDB_MAX_FIELD_LEN, 1);
cfgAddInt32(pCfg, "maxNumOfOrderedRes", tsMaxNumOfOrderedResults, 128, TSDB_MAX_ALLOWED_SQL_LEN, 1);
cfgAddBool(pCfg, "keepColumnName", tsKeepOriginalColumnName, 1);
cfgAddInt32(pCfg, "maxBinaryDisplayWidth", tsMaxBinaryDisplayWidth, 1, 65536, 1);
}
static void taosAddSystemInfo(SConfig *pCfg) {
SysNameInfo info = taosGetSysNameInfo();
cfgAddTimezone(pCfg, "timezone", tsTimezone);
cfgAddLocale(pCfg, "locale", tsLocale);
cfgAddCharset(pCfg, "charset", tsCharset);
cfgAddBool(pCfg, "enableCoreFile", 0, 1);
cfgAddInt32(pCfg, "numOfCores", tsNumOfCores, 1, 100000, 1);
cfgAddInt32(pCfg, "pageSize(KB)", tsPageSize, 0, INT64_MAX, 1);
cfgAddInt64(pCfg, "openMax", tsOpenMax, 0, INT64_MAX, 1);
cfgAddInt64(pCfg, "streamMax", tsStreamMax, 0, INT64_MAX, 1);
cfgAddInt32(pCfg, "totalMemory(MB)", tsTotalMemoryMB, 0, INT32_MAX, 1);
cfgAddString(pCfg, "os sysname", info.sysname, 1);
cfgAddString(pCfg, "os nodename", info.nodename, 1);
cfgAddString(pCfg, "os release", info.release, 1);
cfgAddString(pCfg, "os version", info.version, 1);
cfgAddString(pCfg, "os machine", info.machine, 1);
cfgAddString(pCfg, "os sysname", info.sysname, 1);
cfgAddString(pCfg, "version", version, 1);
cfgAddString(pCfg, "compatible_version", compatible_version, 1);
cfgAddString(pCfg, "gitinfo", gitinfo, 1);
cfgAddString(pCfg, "gitinfoOfInternal", gitinfoOfInternal, 1);
cfgAddString(pCfg, "buildinfo", buildinfo, 1);
}
static void taosAddServerCfg(SConfig *pCfg) {
cfgAddInt32(pCfg, "supportVnodes", 256, 0, 65536, 0);
cfgAddDir(pCfg, "dataDir", tsDataDir, 0);
cfgAddFloat(pCfg, "minimalDataDirGB", 2.0f, 0.001f, 10000000, 0);
cfgAddInt32(pCfg, "numOfCommitThreads", tsNumOfCommitThreads, 1, 100, 0);
cfgAddFloat(pCfg, "ratioOfQueryCores", tsRatioOfQueryCores, 0, 2, 0);
cfgAddInt32(pCfg, "maxNumOfDistinctRes", tsMaxNumOfDistinctResults, 10 * 10000, 10000 * 10000, 0);
cfgAddBool(pCfg, "telemetryReporting", tsEnableTelemetryReporting, 0);
cfgAddInt32(pCfg, "maxConnections", tsMaxConnections, 1, 100000, 0);
cfgAddInt32(pCfg, "maxShellConns", tsMaxShellConns, 10, 50000000, 0);
cfgAddInt32(pCfg, "statusInterval", tsStatusInterval, 1, 30, 0);
cfgAddInt32(pCfg, "minSlidingTime", tsMinSlidingTime, 10, 1000000, 0);
cfgAddInt32(pCfg, "minIntervalTime", tsMinIntervalTime, 1, 1000000, 0);
cfgAddInt32(pCfg, "maxStreamCompDelay", tsMaxStreamComputDelay, 10, 1000000000, 0);
cfgAddInt32(pCfg, "maxFirstStreamCompDelay", tsStreamCompStartDelay, 1000, 1000000000, 0);
cfgAddInt32(pCfg, "retryStreamCompDelay", tsRetryStreamCompDelay, 10, 1000000000, 0);
cfgAddFloat(pCfg, "streamCompDelayRatio", tsStreamComputDelayRatio, 0.1, 0.9, 0);
cfgAddInt32(pCfg, "queryBufferSize", tsQueryBufferSize, -1, 500000000000, 0);
cfgAddBool(pCfg, "retrieveBlockingModel", tsRetrieveBlockingModel, 0);
cfgAddBool(pCfg, "printAuth", tsPrintAuth, 0);
cfgAddBool(pCfg, "slaveQuery", tsEnableSlaveQuery, 0);
cfgAddBool(pCfg, "deadLockKillQuery", tsDeadLockKillQuery, 0);
}
static void taosSetClientLogCfg(SConfig *pCfg) {
SConfigItem *pItem = cfgGetItem(pCfg, "logDir");
tstrncpy(tsLogDir, cfgGetItem(pCfg, "logDir")->str, PATH_MAX);
tsLogSpace.reserved = cfgGetItem(pCfg, "minimalLogDirGB")->fval;
tsNumOfLogLines = cfgGetItem(pCfg, "numOfLogLines")->i32;
tsAsyncLog = cfgGetItem(pCfg, "asyncLog")->bval;
tsLogKeepDays = cfgGetItem(pCfg, "logKeepDays")->i32;
cDebugFlag = cfgGetItem(pCfg, "cDebugFlag")->i32;
uDebugFlag = cfgGetItem(pCfg, "uDebugFlag")->i32;
rpcDebugFlag = cfgGetItem(pCfg, "rpcDebugFlag")->i32;
tmrDebugFlag = cfgGetItem(pCfg, "tmrDebugFlag")->i32;
jniDebugFlag = cfgGetItem(pCfg, "jniDebugFlag")->i32;
}
static void taosSetServerLogCfg(SConfig *pCfg) {
dDebugFlag = cfgGetItem(pCfg, "dDebugFlag")->i32;
vDebugFlag = cfgGetItem(pCfg, "vDebugFlag")->i32;
mDebugFlag = cfgGetItem(pCfg, "mDebugFlag")->i32;
qDebugFlag = cfgGetItem(pCfg, "qDebugFlag")->i32;
wDebugFlag = cfgGetItem(pCfg, "wDebugFlag")->i32;
sDebugFlag = cfgGetItem(pCfg, "sDebugFlag")->i32;
tsdbDebugFlag = cfgGetItem(pCfg, "tsdbDebugFlag")->i32;
tqDebugFlag = cfgGetItem(pCfg, "tqDebugFlag")->i32;
fsDebugFlag = cfgGetItem(pCfg, "fsDebugFlag")->i32;
}
static void taosSetClientCfg(SConfig *pCfg) {
tstrncpy(tsFirst, cfgGetItem(pCfg, "firstEp")->str, TSDB_EP_LEN);
tstrncpy(tsSecond, cfgGetItem(pCfg, "secondEp")->str, TSDB_EP_LEN);
tstrncpy(tsLocalFqdn, cfgGetItem(pCfg, "fqdn")->str, TSDB_EP_LEN);
tsServerPort = (uint16_t)cfgGetItem(pCfg, "serverPort")->i32;
snprintf(tsLocalEp, sizeof(tsLocalEp), "%s:%u", tsLocalFqdn, tsServerPort);
tstrncpy(tsLogDir, cfgGetItem(pCfg, "tempDir")->str, PATH_MAX);
tsTempSpace.reserved = cfgGetItem(pCfg, "minimalTempDirGB")->fval;
tsNumOfThreadsPerCore = cfgGetItem(pCfg, "maxTmrCtrl")->fval;
tsMaxTmrCtrl = cfgGetItem(pCfg, "maxTmrCtrl")->i32;
tsRpcTimer = cfgGetItem(pCfg, "rpcTimer")->i32;
tsRpcMaxTime = cfgGetItem(pCfg, "rpcMaxTime")->i32;
tsRpcForceTcp = cfgGetItem(pCfg, "rpcForceTcp")->i32;
tsShellActivityTimer = cfgGetItem(pCfg, "shellActivityTimer")->bval;
tsCompressMsgSize = cfgGetItem(pCfg, "compressMsgSize")->i32;
tsCompressColData = cfgGetItem(pCfg, "compressColData")->i32;
tsMaxWildCardsLen = cfgGetItem(pCfg, "maxWildCardsLength")->i32;
tsMaxRegexStringLen = cfgGetItem(pCfg, "maxRegexStringLen")->i32;
tsMaxNumOfOrderedResults = cfgGetItem(pCfg, "maxNumOfOrderedRes")->i32;
tsKeepOriginalColumnName = cfgGetItem(pCfg, "keepColumnName")->bval;
tsMaxBinaryDisplayWidth = cfgGetItem(pCfg, "maxBinaryDisplayWidth")->i32;
}
static void taosSetSystemCfg(SConfig *pCfg) {
SConfigItem *pItem = cfgGetItem(pCfg, "timezone");
osSetTimezone(pItem->str);
uDebug("timezone format changed from %s to %s", pItem->str, tsTimezone);
cfgSetItem(pCfg, "timezone", tsTimezone, pItem->stype);
const char *locale = cfgGetItem(pCfg, "locale")->str;
const char *charset = cfgGetItem(pCfg, "charset")->str;
taosSetSystemLocale(locale, charset);
if (tsNumOfCores <= 1) {
tsNumOfCores = 2;
}
bool enableCore = cfgGetItem(pCfg, "enableCoreFile")->bval;
taosSetConsoleEcho(enableCore);
// todo
tsVersion = 30000000;
}
static void taosSetServerCfg(SConfig *pCfg) {
tstrncpy(tsDataDir, cfgGetItem(pCfg, "dataDir")->str, PATH_MAX);
tsTempSpace.reserved = cfgGetItem(pCfg, "minimalDataDirGB")->fval;
tsNumOfCommitThreads = cfgGetItem(pCfg, "numOfCommitThreads")->i32;
tsRatioOfQueryCores = cfgGetItem(pCfg, "ratioOfQueryCores")->fval;
tsMaxNumOfDistinctResults = cfgGetItem(pCfg, "maxNumOfDistinctRes")->i32;
tsEnableTelemetryReporting = cfgGetItem(pCfg, "telemetryReporting")->bval;
tsMaxConnections = cfgGetItem(pCfg, "maxConnections")->i32;
tsMaxShellConns = cfgGetItem(pCfg, "maxShellConns")->i32;
tsStatusInterval = cfgGetItem(pCfg, "statusInterval")->i32;
tsMinSlidingTime = cfgGetItem(pCfg, "minSlidingTime")->i32;
tsMinIntervalTime = cfgGetItem(pCfg, "minIntervalTime")->i32;
tsMaxStreamComputDelay = cfgGetItem(pCfg, "maxStreamCompDelay")->i32;
tsStreamCompStartDelay = cfgGetItem(pCfg, "maxFirstStreamCompDelay")->i32;
tsRetryStreamCompDelay = cfgGetItem(pCfg, "retryStreamCompDelay")->i32;
tsStreamComputDelayRatio = cfgGetItem(pCfg, "streamCompDelayRatio")->fval;
tsQueryBufferSize = cfgGetItem(pCfg, "queryBufferSize")->i32;
tsRetrieveBlockingModel = cfgGetItem(pCfg, "retrieveBlockingModel")->bval;
tsPrintAuth = cfgGetItem(pCfg, "printAuth")->bval;
tsEnableSlaveQuery = cfgGetItem(pCfg, "slaveQuery")->bval;
tsDeadLockKillQuery = cfgGetItem(pCfg, "deadLockKillQuery")->bval;
if (tsQueryBufferSize >= 0) {
tsQueryBufferSizeBytes = tsQueryBufferSize * 1048576UL;
} }
} }
#endif int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDir, const char *envFile,
return false; const char *apolloUrl, bool tsc) {
}
// void taosAddDataDir(int index, char *v1, int level, int primary) {
// tstrncpy(tsDiskCfg[index].dir, v1, TSDB_FILENAME_LEN);
// tsDiskCfg[index].level = level;
// tsDiskCfg[index].primary = primary;
// uTrace("dataDir:%s, level:%d primary:%d is configured", v1, level, primary);
// }
#ifndef _STORAGE
// void taosReadDataDirCfg(char *v1, char *v2, char *v3) {
// if (tsDiskCfgNum == 1) {
// SDiskCfg *cfg = &tsDiskCfg[0];
// uInfo("dataDir:%s, level:%d primary:%d is replaced by %s", cfg->dir, cfg->level, cfg->primary, v1);
// }
// taosAddDataDir(0, v1, 0, 1);
// tsDiskCfgNum = 1;
// }
// void taosPrintDataDirCfg() {
// for (int i = 0; i < tsDiskCfgNum; ++i) {
// SDiskCfg *cfg = &tsDiskCfg[i];
// uInfo(" dataDir: %s", cfg->dir);
// }
// }
#endif
static void doInitGlobalConfig(void) {
osInit(); osInit();
srand(taosSafeRand());
#if 0
SGlobalCfg cfg = {0};
SConfig *pCfg = cfgInit();
if (pCfg == NULL) return -1;
cfg.option = "dataDir"; if (tsc) {
cfg.ptr = tsDataDir; taosAddClientLogCfg(pCfg);
cfg.valType = TAOS_CFG_VTYPE_DATA_DIRCTORY; } else {
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; taosAddClientLogCfg(pCfg);
cfg.minValue = 0; taosAddServerLogCfg(pCfg);
cfg.maxValue = 0;
cfg.ptrLength = TSDB_FILENAME_LEN;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
cfg.option = "maxNumOfDistinctRes";
cfg.ptr = &tsMaxNumOfDistinctResults;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT;
cfg.minValue = 10 * 10000;
cfg.maxValue = 10000 * 10000;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
cfg.option = "minSlidingTime";
cfg.ptr = &tsMinSlidingTime;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = 10;
cfg.maxValue = 1000000;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_MS;
taosAddConfigOption(cfg);
cfg.option = "minIntervalTime";
cfg.ptr = &tsMinIntervalTime;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = 1;
cfg.maxValue = 1000000;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_MS;
taosAddConfigOption(cfg);
cfg.option = "maxStreamCompDelay";
cfg.ptr = &tsMaxStreamComputDelay;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = 10;
cfg.maxValue = 1000000000;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_MS;
taosAddConfigOption(cfg);
cfg.option = "maxFirstStreamCompDelay";
cfg.ptr = &tsStreamCompStartDelay;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = 1000;
cfg.maxValue = 1000000000;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_MS;
taosAddConfigOption(cfg);
cfg.option = "retryStreamCompDelay";
cfg.ptr = &tsRetryStreamCompDelay;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = 10;
cfg.maxValue = 1000000000;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_MS;
taosAddConfigOption(cfg);
cfg.option = "streamCompDelayRatio";
cfg.ptr = &tsStreamComputDelayRatio;
cfg.valType = TAOS_CFG_VTYPE_FLOAT;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = 0.1f;
cfg.maxValue = 0.9f;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
cfg.option = "compressMsgSize";
cfg.ptr = &tsCompressMsgSize;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = -1;
cfg.maxValue = 100000000.0f;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
cfg.option = "compressColData";
cfg.ptr = &tsCompressColData;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = -1;
cfg.maxValue = 100000000.0f;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
cfg.option = "maxWildCardsLength";
cfg.ptr = &tsMaxWildCardsLen;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = 0;
cfg.maxValue = TSDB_MAX_FIELD_LEN;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_BYTE;
taosAddConfigOption(cfg);
cfg.option = "maxRegexStringLen";
cfg.ptr = &tsMaxRegexStringLen;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = 0;
cfg.maxValue = TSDB_MAX_FIELD_LEN;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_BYTE;
taosAddConfigOption(cfg);
cfg.option = "maxNumOfOrderedRes";
cfg.ptr = &tsMaxNumOfOrderedResults;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = TSDB_MAX_SQL_LEN;
cfg.maxValue = TSDB_MAX_ALLOWED_SQL_LEN;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
cfg.option = "queryBufferSize";
cfg.ptr = &tsQueryBufferSize;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = -1;
cfg.maxValue = 500000000000.0f;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_BYTE;
taosAddConfigOption(cfg);
cfg.option = "retrieveBlockingModel";
cfg.ptr = &tsRetrieveBlockingModel;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = 0;
cfg.maxValue = 1;
cfg.ptrLength = 1;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
cfg.option = "keepColumnName";
cfg.ptr = &tsKeepOriginalColumnName;
cfg.valType = TAOS_CFG_VTYPE_INT8;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT;
cfg.minValue = 0;
cfg.maxValue = 1;
cfg.ptrLength = 1;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
cfg.option = "slaveQuery";
cfg.ptr = &tsEnableSlaveQuery;
cfg.valType = TAOS_CFG_VTYPE_INT8;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = 0;
cfg.maxValue = 1;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
cfg.option = "enableRecordSql";
cfg.ptr = &tsTscEnableRecordSql;
cfg.valType = TAOS_CFG_VTYPE_INT8;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG;
cfg.minValue = 0;
cfg.maxValue = 1;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
cfg.option = "maxBinaryDisplayWidth";
cfg.ptr = &tsMaxBinaryDisplayWidth;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT;
cfg.minValue = 1;
cfg.maxValue = 65536;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
cfg.option = "tempDir";
cfg.ptr = tsTempDir;
cfg.valType = TAOS_CFG_VTYPE_STRING;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT;
cfg.minValue = 0;
cfg.maxValue = 0;
cfg.ptrLength = PATH_MAX;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
// enable kill long query
cfg.option = "deadLockKillQuery";
cfg.ptr = &tsDeadLockKillQuery;
cfg.valType = TAOS_CFG_VTYPE_INT8;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = 0;
cfg.maxValue = 1;
cfg.ptrLength = 1;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
#ifdef TD_TSZ
// lossy compress
cfg.option = "lossyColumns";
cfg.ptr = lossyColumns;
cfg.valType = TAOS_CFG_VTYPE_STRING;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG;
cfg.minValue = 0;
cfg.maxValue = 0;
cfg.ptrLength = tListLen(lossyColumns);
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
cfg.option = "fPrecision";
cfg.ptr = &fPrecision;
cfg.valType = TAOS_CFG_VTYPE_DOUBLE;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG;
cfg.minValue = MIN_FLOAT;
cfg.maxValue = 100000;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
cfg.option = "dPrecision";
cfg.ptr = &dPrecision;
cfg.valType = TAOS_CFG_VTYPE_DOUBLE;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG;
cfg.minValue = 100000;
cfg.maxValue = 0;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
cfg.option = "maxRange";
cfg.ptr = &maxRange;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG;
cfg.minValue = 0;
cfg.maxValue = 65536;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
cfg.option = "range";
cfg.ptr = &curRange;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG;
cfg.minValue = 0;
cfg.maxValue = 65536;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosAddConfigOption(cfg);
assert(tsGlobalConfigNum == TSDB_CFG_MAX_NUM);
#else
// assert(tsGlobalConfigNum == TSDB_CFG_MAX_NUM - 5);
#endif
#endif
} }
void taosInitGlobalCfg() { pthread_once(&tsInitGlobalCfgOnce, doInitGlobalConfig); } if (taosLoadCfg(pCfg, cfgDir, envFile, apolloUrl) != 0) {
uError("failed to load cfg since %s", terrstr());
/* cfgCleanup(pCfg);
* alter dnode 1 balance "vnode:1-dnode:2" return -1;
*/
bool taosCheckBalanceCfgOptions(const char *option, int32_t *vnodeId, int32_t *dnodeId) {
int len = (int)strlen(option);
if (strncasecmp(option, "vnode:", 6) != 0) {
return false;
} }
int pos = 0; if (tsc) {
for (; pos < len; ++pos) { taosSetClientLogCfg(pCfg);
if (option[pos] == '-') break; taosSetAllDebugFlag(cfgGetItem(pCfg, "debugFlag")->i32);
} else {
taosSetClientLogCfg(pCfg);
taosSetServerLogCfg(pCfg);
taosSetAllDebugFlag(cfgGetItem(pCfg, "debugFlag")->i32);
} }
if (++pos >= len) return false; if (taosInitLog(logname, logFileNum) != 0) {
if (strncasecmp(option + pos, "dnode:", 6) != 0) { printf("failed to init log file since %s\n", terrstr());
return false; cfgCleanup(pCfg);
return -1;
} }
*vnodeId = strtol(option + 6, NULL, 10); cfgCleanup(pCfg);
*dnodeId = strtol(option + pos + 6, NULL, 10); return 0;
if (*vnodeId <= 1 || *dnodeId <= 0) {
return false;
} }
return true; int32_t taosInitCfg(const char *cfgDir, const char *envFile, const char *apolloUrl, bool tsc) {
if (tsCfg != NULL) return 0;
tsCfg = cfgInit();
if (tsc) {
taosAddClientLogCfg(tsCfg);
taosAddClientCfg(tsCfg);
} else {
taosAddClientLogCfg(tsCfg);
taosAddServerLogCfg(tsCfg);
taosAddClientCfg(tsCfg);
taosAddServerCfg(tsCfg);
}
taosAddSystemInfo(tsCfg);
if (taosLoadCfg(tsCfg, cfgDir, envFile, apolloUrl) != 0) {
uError("failed to load cfg since %s", terrstr());
cfgCleanup(tsCfg);
tsCfg = NULL;
return -1;
}
if (tsc) {
taosSetClientCfg(tsCfg);
} else {
taosSetClientCfg(tsCfg);
taosSetServerCfg(tsCfg);
}
taosSetSystemCfg(tsCfg);
cfgDumpCfg(tsCfg, tsc, false);
return 0;
}
void taosCleanupCfg() {
if (tsCfg) {
cfgCleanup(tsCfg);
tsCfg = NULL;
}
}
void taosCfgDynamicOptions(const char *option, const char *value) {
if (strcasecmp(option, "debugFlag") == 0) {
int32_t debugFlag = atoi(value);
taosSetAllDebugFlag(debugFlag);
}
if (strcasecmp(option, "resetlog") == 0) {
taosResetLog();
cfgDumpCfg(tsCfg, 1, false);
}
} }

45
source/common/src/tlocale.c
View File

@ -1,45 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "os.h"
#include "ulog.h"
#include "tglobal.h"
#include "tutil.h"
/**
* In some Linux systems, setLocale(LC_CTYPE, "") may return NULL, in which case the launch of
* both the TDengine Server and the Client may be interrupted.
*
* In case that the setLocale failed to be executed, the right charset needs to be set.
*/
void tsSetLocale() {
char *locale = setlocale(LC_CTYPE, tsLocale);
// default locale or user specified locale is not valid, abort launch
if (locale == NULL) {
uError("Invalid locale:%s, please set the valid locale in config file", tsLocale);
}
if (strlen(tsCharset) == 0) {
uError("failed to get charset, please set the valid charset in config file");
exit(-1);
}
if (!taosValidateEncodec(tsCharset)) {
uError("Invalid charset:%s, please set the valid charset in config file", tsCharset);
exit(-1);
}
}

135
source/common/src/ttszip.c
View File

@ -24,8 +24,9 @@ STSBuf* tsBufCreate(bool autoDelete, int32_t order) {
pTSBuf->autoDelete = autoDelete; pTSBuf->autoDelete = autoDelete;
taosGetTmpfilePath(tsTempDir, "join", pTSBuf->path); taosGetTmpfilePath(tsTempDir, "join", pTSBuf->path);
pTSBuf->f = fopen(pTSBuf->path, "wb+"); // pTSBuf->pFile = fopen(pTSBuf->path, "wb+");
if (pTSBuf->f == NULL) { pTSBuf->pFile = taosOpenFile(pTSBuf->path, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC);
if (pTSBuf->pFile == NULL) {
free(pTSBuf); free(pTSBuf);
return NULL; return NULL;
} }
@ -60,8 +61,9 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) {
tstrncpy(pTSBuf->path, path, sizeof(pTSBuf->path)); tstrncpy(pTSBuf->path, path, sizeof(pTSBuf->path));
pTSBuf->f = fopen(pTSBuf->path, "rb+"); // pTSBuf->pFile = fopen(pTSBuf->path, "rb+");
if (pTSBuf->f == NULL) { pTSBuf->pFile = taosOpenFile(pTSBuf->path, TD_FILE_WRITE | TD_FILE_READ);
if (pTSBuf->pFile == NULL) {
free(pTSBuf); free(pTSBuf);
return NULL; return NULL;
} }
@ -72,9 +74,9 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) {
// validate the file magic number // validate the file magic number
STSBufFileHeader header = {0}; STSBufFileHeader header = {0};
int32_t ret = fseek(pTSBuf->f, 0, SEEK_SET); int32_t ret = taosLSeekFile(pTSBuf->pFile, 0, SEEK_SET);
UNUSED(ret); UNUSED(ret);
size_t sz = fread(&header, 1, sizeof(STSBufFileHeader), pTSBuf->f); size_t sz = taosReadFile(pTSBuf->pFile, &header, sizeof(STSBufFileHeader));
UNUSED(sz); UNUSED(sz);
// invalid file // invalid file
@ -112,8 +114,8 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) {
return NULL; return NULL;
} }
//int64_t pos = ftell(pTSBuf->f); //pos not used //int64_t pos = ftell(pTSBuf->pFile); //pos not used
sz = fread(buf, infoSize, 1, pTSBuf->f); sz = taosReadFile(pTSBuf->pFile, buf, infoSize);
UNUSED(sz); UNUSED(sz);
// the length value for each vnode is not kept in file, so does not set the length value // the length value for each vnode is not kept in file, so does not set the length value
@ -123,22 +125,22 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) {
} }
free(buf); free(buf);
ret = fseek(pTSBuf->f, 0, SEEK_END); ret = taosLSeekFile(pTSBuf->pFile, 0, SEEK_END);
UNUSED(ret); UNUSED(ret);
struct stat fileStat; int64_t file_size;
if (fstat(fileno(pTSBuf->f), &fileStat) != 0) { if (taosFStatFile(pTSBuf->pFile, &file_size, NULL) != 0) {
tsBufDestroy(pTSBuf); tsBufDestroy(pTSBuf);
return NULL; return NULL;
} }
pTSBuf->fileSize = (uint32_t)fileStat.st_size; pTSBuf->fileSize = (uint32_t)file_size;
tsBufResetPos(pTSBuf); tsBufResetPos(pTSBuf);
// ascending by default // ascending by default
pTSBuf->cur.order = TSDB_ORDER_ASC; pTSBuf->cur.order = TSDB_ORDER_ASC;
// tscDebug("create tsBuf from file:%s, fd:%d, size:%d, numOfGroups:%d, autoDelete:%d", pTSBuf->path, fileno(pTSBuf->f), // tscDebug("create tsBuf from file:%s, fd:%d, size:%d, numOfGroups:%d, autoDelete:%d", pTSBuf->path, fileno(pTSBuf->pFile),
// pTSBuf->fileSize, pTSBuf->numOfGroups, pTSBuf->autoDelete); // pTSBuf->fileSize, pTSBuf->numOfGroups, pTSBuf->autoDelete);
return pTSBuf; return pTSBuf;
@ -156,7 +158,7 @@ void* tsBufDestroy(STSBuf* pTSBuf) {
tfree(pTSBuf->block.payload); tfree(pTSBuf->block.payload);
if (!pTSBuf->remainOpen) { if (!pTSBuf->remainOpen) {
fclose(pTSBuf->f); taosCloseFile(&pTSBuf->pFile);
} }
if (pTSBuf->autoDelete) { if (pTSBuf->autoDelete) {
@ -253,7 +255,7 @@ static void writeDataToDisk(STSBuf* pTSBuf) {
tsCompressTimestamp(pTsData->rawBuf, pTsData->len, pTsData->len/TSDB_KEYSIZE, pBlock->payload, pTsData->allocSize, tsCompressTimestamp(pTsData->rawBuf, pTsData->len, pTsData->len/TSDB_KEYSIZE, pBlock->payload, pTsData->allocSize,
TWO_STAGE_COMP, pTSBuf->assistBuf, pTSBuf->bufSize); TWO_STAGE_COMP, pTSBuf->assistBuf, pTSBuf->bufSize);
int64_t r = fseek(pTSBuf->f, pTSBuf->fileSize, SEEK_SET); int64_t r = taosLSeekFile(pTSBuf->pFile, pTSBuf->fileSize, SEEK_SET);
assert(r == 0); assert(r == 0);
/* /*
@ -264,30 +266,30 @@ static void writeDataToDisk(STSBuf* pTSBuf) {
* both side has the compressed length is used to support load data forwards/backwords. * both side has the compressed length is used to support load data forwards/backwords.
*/ */
int32_t metaLen = 0; int32_t metaLen = 0;
metaLen += (int32_t)fwrite(&pBlock->tag.nType, 1, sizeof(pBlock->tag.nType), pTSBuf->f); metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, &pBlock->tag.nType, sizeof(pBlock->tag.nType));
int32_t trueLen = pBlock->tag.nLen; int32_t trueLen = pBlock->tag.nLen;
if (pBlock->tag.nType == TSDB_DATA_TYPE_BINARY || pBlock->tag.nType == TSDB_DATA_TYPE_NCHAR) { if (pBlock->tag.nType == TSDB_DATA_TYPE_BINARY || pBlock->tag.nType == TSDB_DATA_TYPE_NCHAR) {
metaLen += (int32_t)fwrite(&pBlock->tag.nLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f); metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, &pBlock->tag.nLen, sizeof(pBlock->tag.nLen));
metaLen += (int32_t)fwrite(pBlock->tag.pz, 1, (size_t)pBlock->tag.nLen, pTSBuf->f); metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, pBlock->tag.pz, (size_t)pBlock->tag.nLen);
} else if (pBlock->tag.nType == TSDB_DATA_TYPE_FLOAT) { } else if (pBlock->tag.nType == TSDB_DATA_TYPE_FLOAT) {
metaLen += (int32_t)fwrite(&pBlock->tag.nLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f); metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, &pBlock->tag.nLen, sizeof(pBlock->tag.nLen));
float tfloat = (float)pBlock->tag.d; float tfloat = (float)pBlock->tag.d;
metaLen += (int32_t)fwrite(&tfloat, 1, (size_t) pBlock->tag.nLen, pTSBuf->f); metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, &tfloat, (size_t) pBlock->tag.nLen);
} else if (pBlock->tag.nType != TSDB_DATA_TYPE_NULL) { } else if (pBlock->tag.nType != TSDB_DATA_TYPE_NULL) {
metaLen += (int32_t)fwrite(&pBlock->tag.nLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f); metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, &pBlock->tag.nLen, sizeof(pBlock->tag.nLen));
metaLen += (int32_t)fwrite(&pBlock->tag.i, 1, (size_t) pBlock->tag.nLen, pTSBuf->f); metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, &pBlock->tag.i, (size_t) pBlock->tag.nLen);
} else { } else {
trueLen = 0; trueLen = 0;
metaLen += (int32_t)fwrite(&trueLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f); metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, &trueLen, sizeof(pBlock->tag.nLen));
} }
fwrite(&pBlock->numOfElem, sizeof(pBlock->numOfElem), 1, pTSBuf->f); taosWriteFile(pTSBuf->pFile, &pBlock->numOfElem, sizeof(pBlock->numOfElem));
fwrite(&pBlock->compLen, sizeof(pBlock->compLen), 1, pTSBuf->f); taosWriteFile(pTSBuf->pFile, &pBlock->compLen, sizeof(pBlock->compLen));
fwrite(pBlock->payload, (size_t)pBlock->compLen, 1, pTSBuf->f); taosWriteFile(pTSBuf->pFile, pBlock->payload, (size_t)pBlock->compLen);
fwrite(&pBlock->compLen, sizeof(pBlock->compLen), 1, pTSBuf->f); taosWriteFile(pTSBuf->pFile, &pBlock->compLen, sizeof(pBlock->compLen));
metaLen += (int32_t) fwrite(&trueLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f); metaLen += (int32_t) taosWriteFile(pTSBuf->pFile, &trueLen, sizeof(pBlock->tag.nLen));
assert(metaLen == getTagAreaLength(&pBlock->tag)); assert(metaLen == getTagAreaLength(&pBlock->tag));
int32_t blockSize = metaLen + sizeof(pBlock->numOfElem) + sizeof(pBlock->compLen) * 2 + pBlock->compLen; int32_t blockSize = metaLen + sizeof(pBlock->numOfElem) + sizeof(pBlock->compLen) * 2 + pBlock->compLen;
@ -332,20 +334,20 @@ STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) {
* the end of each comp data block * the end of each comp data block
*/ */
int32_t prev = -(int32_t) (sizeof(pBlock->padding) + sizeof(pBlock->tag.nLen)); int32_t prev = -(int32_t) (sizeof(pBlock->padding) + sizeof(pBlock->tag.nLen));
int32_t ret = fseek(pTSBuf->f, prev, SEEK_CUR); int32_t ret = taosLSeekFile(pTSBuf->pFile, prev, SEEK_CUR);
size_t sz = fread(&pBlock->padding, 1, sizeof(pBlock->padding), pTSBuf->f); size_t sz = taosReadFile(pTSBuf->pFile, &pBlock->padding, sizeof(pBlock->padding));
sz = fread(&pBlock->tag.nLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f); sz = taosReadFile(pTSBuf->pFile, &pBlock->tag.nLen, sizeof(pBlock->tag.nLen));
UNUSED(sz); UNUSED(sz);
pBlock->compLen = pBlock->padding; pBlock->compLen = pBlock->padding;
offset = pBlock->compLen + sizeof(pBlock->compLen) * 2 + sizeof(pBlock->numOfElem) + getTagAreaLength(&pBlock->tag); offset = pBlock->compLen + sizeof(pBlock->compLen) * 2 + sizeof(pBlock->numOfElem) + getTagAreaLength(&pBlock->tag);
ret = fseek(pTSBuf->f, -offset, SEEK_CUR); ret = taosLSeekFile(pTSBuf->pFile, -offset, SEEK_CUR);
UNUSED(ret); UNUSED(ret);
} }
int32_t ret = fread(&pBlock->tag.nType, sizeof(pBlock->tag.nType), 1, pTSBuf->f); int32_t ret = taosReadFile(pTSBuf->pFile, &pBlock->tag.nType, sizeof(pBlock->tag.nType));
ret = fread(&pBlock->tag.nLen, sizeof(pBlock->tag.nLen), 1, pTSBuf->f); ret = taosReadFile(pTSBuf->pFile, &pBlock->tag.nLen, sizeof(pBlock->tag.nLen));
// NOTE: mix types tags are not supported // NOTE: mix types tags are not supported
size_t sz = 0; size_t sz = 0;
@ -356,23 +358,23 @@ STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) {
memset(tp, 0, pBlock->tag.nLen + 1); memset(tp, 0, pBlock->tag.nLen + 1);
pBlock->tag.pz = tp; pBlock->tag.pz = tp;
sz = fread(pBlock->tag.pz, (size_t)pBlock->tag.nLen, 1, pTSBuf->f); sz = taosReadFile(pTSBuf->pFile, pBlock->tag.pz, (size_t)pBlock->tag.nLen);
UNUSED(sz); UNUSED(sz);
} else if (pBlock->tag.nType == TSDB_DATA_TYPE_FLOAT) { } else if (pBlock->tag.nType == TSDB_DATA_TYPE_FLOAT) {
float tfloat = 0; float tfloat = 0;
sz = fread(&tfloat, (size_t) pBlock->tag.nLen, 1, pTSBuf->f); sz = taosReadFile(pTSBuf->pFile, &tfloat, (size_t) pBlock->tag.nLen);
pBlock->tag.d = (double)tfloat; pBlock->tag.d = (double)tfloat;
UNUSED(sz); UNUSED(sz);
} else if (pBlock->tag.nType != TSDB_DATA_TYPE_NULL) { //TODO check the return value } else if (pBlock->tag.nType != TSDB_DATA_TYPE_NULL) { //TODO check the return value
sz = fread(&pBlock->tag.i, (size_t) pBlock->tag.nLen, 1, pTSBuf->f); sz = taosReadFile(pTSBuf->pFile, &pBlock->tag.i, (size_t) pBlock->tag.nLen);
UNUSED(sz); UNUSED(sz);
} }
sz = fread(&pBlock->numOfElem, sizeof(pBlock->numOfElem), 1, pTSBuf->f); sz = taosReadFile(pTSBuf->pFile, &pBlock->numOfElem, sizeof(pBlock->numOfElem));
UNUSED(sz); UNUSED(sz);
sz = fread(&pBlock->compLen, sizeof(pBlock->compLen), 1, pTSBuf->f); sz = taosReadFile(pTSBuf->pFile, &pBlock->compLen, sizeof(pBlock->compLen));
UNUSED(sz); UNUSED(sz);
sz = fread(pBlock->payload, (size_t)pBlock->compLen, 1, pTSBuf->f); sz = taosReadFile(pTSBuf->pFile, pBlock->payload, (size_t)pBlock->compLen);
if (decomp) { if (decomp) {
pTSBuf->tsData.len = pTSBuf->tsData.len =
@ -381,11 +383,11 @@ STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) {
} }
// read the comp length at the length of comp block // read the comp length at the length of comp block
sz = fread(&pBlock->padding, sizeof(pBlock->padding), 1, pTSBuf->f); sz = taosReadFile(pTSBuf->pFile, &pBlock->padding, sizeof(pBlock->padding));
assert(pBlock->padding == pBlock->compLen); assert(pBlock->padding == pBlock->compLen);
int32_t n = 0; int32_t n = 0;
sz = fread(&n, sizeof(pBlock->tag.nLen), 1, pTSBuf->f); sz = taosReadFile(pTSBuf->pFile, &n, sizeof(pBlock->tag.nLen));
if (pBlock->tag.nType == TSDB_DATA_TYPE_NULL) { if (pBlock->tag.nType == TSDB_DATA_TYPE_NULL) {
assert(n == 0); assert(n == 0);
} else { } else {
@ -396,7 +398,7 @@ STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) {
// for backwards traverse, set the start position at the end of previous block // for backwards traverse, set the start position at the end of previous block
if (order == TSDB_ORDER_DESC) { if (order == TSDB_ORDER_DESC) {
int32_t r = fseek(pTSBuf->f, -offset, SEEK_CUR); int32_t r = taosLSeekFile(pTSBuf->pFile, -offset, SEEK_CUR);
UNUSED(r); UNUSED(r);
} }
@ -512,7 +514,7 @@ static int32_t tsBufFindGroupById(STSGroupBlockInfoEx* pGroupInfoEx, int32_t num
// todo opt performance by cache blocks info // todo opt performance by cache blocks info
static int32_t tsBufFindBlock(STSBuf* pTSBuf, STSGroupBlockInfo* pBlockInfo, int32_t blockIndex) { static int32_t tsBufFindBlock(STSBuf* pTSBuf, STSGroupBlockInfo* pBlockInfo, int32_t blockIndex) {
if (fseek(pTSBuf->f, pBlockInfo->offset, SEEK_SET) != 0) { if (taosLSeekFile(pTSBuf->pFile, pBlockInfo->offset, SEEK_SET) != 0) {
return -1; return -1;
} }
@ -531,7 +533,7 @@ static int32_t tsBufFindBlock(STSBuf* pTSBuf, STSGroupBlockInfo* pBlockInfo, int
STSBlock* pBlock = &pTSBuf->block; STSBlock* pBlock = &pTSBuf->block;
int32_t compBlockSize = int32_t compBlockSize =
pBlock->compLen + sizeof(pBlock->compLen) * 2 + sizeof(pBlock->numOfElem) + getTagAreaLength(&pBlock->tag); pBlock->compLen + sizeof(pBlock->compLen) * 2 + sizeof(pBlock->numOfElem) + getTagAreaLength(&pBlock->tag);
int32_t ret = fseek(pTSBuf->f, -compBlockSize, SEEK_CUR); int32_t ret = taosLSeekFile(pTSBuf->pFile, -compBlockSize, SEEK_CUR);
UNUSED(ret); UNUSED(ret);
} }
@ -548,7 +550,7 @@ static int32_t tsBufFindBlockByTag(STSBuf* pTSBuf, STSGroupBlockInfo* pBlockInfo
offset = pBlockInfo->offset + pBlockInfo->compLen; offset = pBlockInfo->offset + pBlockInfo->compLen;
} }
if (fseek(pTSBuf->f, (int32_t)offset, SEEK_SET) != 0) { if (taosLSeekFile(pTSBuf->pFile, (int32_t)offset, SEEK_SET) != 0) {
return -1; return -1;
} }
@ -618,11 +620,11 @@ static int32_t doUpdateGroupInfo(STSBuf* pTSBuf, int64_t offset, STSGroupBlockIn
return -1; return -1;
} }
if (fseek(pTSBuf->f, (int32_t)offset, SEEK_SET) != 0) { if (taosLSeekFile(pTSBuf->pFile, (int32_t)offset, SEEK_SET) != 0) {
return -1; return -1;
} }
fwrite(pVInfo, sizeof(STSGroupBlockInfo), 1, pTSBuf->f); taosWriteFile(pTSBuf->pFile, pVInfo, sizeof(STSGroupBlockInfo));
return 0; return 0;
} }
@ -636,19 +638,19 @@ STSGroupBlockInfo* tsBufGetGroupBlockInfo(STSBuf* pTSBuf, int32_t id) {
} }
int32_t STSBufUpdateHeader(STSBuf* pTSBuf, STSBufFileHeader* pHeader) { int32_t STSBufUpdateHeader(STSBuf* pTSBuf, STSBufFileHeader* pHeader) {
if ((pTSBuf->f == NULL) || pHeader == NULL || pHeader->numOfGroup == 0 || pHeader->magic != TS_COMP_FILE_MAGIC) { if ((pTSBuf->pFile == NULL) || pHeader == NULL || pHeader->numOfGroup == 0 || pHeader->magic != TS_COMP_FILE_MAGIC) {
return -1; return -1;
} }
assert(pHeader->tsOrder == TSDB_ORDER_ASC || pHeader->tsOrder == TSDB_ORDER_DESC); assert(pHeader->tsOrder == TSDB_ORDER_ASC || pHeader->tsOrder == TSDB_ORDER_DESC);
int32_t r = fseek(pTSBuf->f, 0, SEEK_SET); int32_t r = taosLSeekFile(pTSBuf->pFile, 0, SEEK_SET);
if (r != 0) { if (r != 0) {
// qError("fseek failed, errno:%d", errno); // qError("fseek failed, errno:%d", errno);
return -1; return -1;
} }
size_t ws = fwrite(pHeader, sizeof(STSBufFileHeader), 1, pTSBuf->f); size_t ws = taosWriteFile(pTSBuf->pFile, pHeader, sizeof(STSBufFileHeader));
if (ws != 1) { if (ws != 1) {
// qError("ts update header fwrite failed, size:%d, expected size:%d", (int32_t)ws, (int32_t)sizeof(STSBufFileHeader)); // qError("ts update header fwrite failed, size:%d, expected size:%d", (int32_t)ws, (int32_t)sizeof(STSBufFileHeader));
return -1; return -1;
@ -823,12 +825,12 @@ int32_t tsBufMerge(STSBuf* pDestBuf, const STSBuf* pSrcBuf) {
pBlockInfoEx->info.id = id; pBlockInfoEx->info.id = id;
} }
int32_t r = fseek(pDestBuf->f, 0, SEEK_END); int32_t r = taosLSeekFile(pDestBuf->pFile, 0, SEEK_END);
assert(r == 0); assert(r == 0);
int64_t offset = getDataStartOffset(); int64_t offset = getDataStartOffset();
int32_t size = (int32_t)pSrcBuf->fileSize - (int32_t)offset; int32_t size = (int32_t)pSrcBuf->fileSize - (int32_t)offset;
int64_t written = taosFSendFile(pDestBuf->f, pSrcBuf->f, &offset, size); int64_t written = taosFSendFile(pDestBuf->pFile, pSrcBuf->pFile, &offset, size);
if (written == -1 || written != size) { if (written == -1 || written != size) {
return -1; return -1;
@ -839,17 +841,18 @@ int32_t tsBufMerge(STSBuf* pDestBuf, const STSBuf* pSrcBuf) {
int32_t oldSize = pDestBuf->fileSize; int32_t oldSize = pDestBuf->fileSize;
// file meta data may be cached, close and reopen the file for accurate file size. // file meta data may be cached, close and reopen the file for accurate file size.
fclose(pDestBuf->f); taosCloseFile(&pDestBuf->pFile);
pDestBuf->f = fopen(pDestBuf->path, "rb+"); // pDestBuf->pFile = fopen(pDestBuf->path, "rb+");
if (pDestBuf->f == NULL) { pDestBuf->pFile = taosOpenFile(pDestBuf->path, TD_FILE_WRITE | TD_FILE_READ);
if (pDestBuf->pFile == NULL) {
return -1; return -1;
} }
struct stat fileStat; int64_t file_size;
if (fstat(fileno(pDestBuf->f), &fileStat) != 0) { if (taosFStatFile(pDestBuf->pFile, &file_size, NULL) != 0) {
return -1; return -1;
} }
pDestBuf->fileSize = (uint32_t)fileStat.st_size; pDestBuf->fileSize = (uint32_t)file_size;
assert(pDestBuf->fileSize == oldSize + size); assert(pDestBuf->fileSize == oldSize + size);
@ -868,13 +871,13 @@ STSBuf* tsBufCreateFromCompBlocks(const char* pData, int32_t numOfBlocks, int32_
// update prev vnode length info in file // update prev vnode length info in file
TSBufUpdateGroupInfo(pTSBuf, pTSBuf->numOfGroups - 1, pBlockInfo); TSBufUpdateGroupInfo(pTSBuf, pTSBuf->numOfGroups - 1, pBlockInfo);
int32_t ret = fseek(pTSBuf->f, pBlockInfo->offset, SEEK_SET); int32_t ret = taosLSeekFile(pTSBuf->pFile, pBlockInfo->offset, SEEK_SET);
if (ret == -1) { if (ret == -1) {
// qError("fseek failed, errno:%d", errno); // qError("fseek failed, errno:%d", errno);
tsBufDestroy(pTSBuf); tsBufDestroy(pTSBuf);
return NULL; return NULL;
} }
size_t sz = fwrite((void*)pData, 1, len, pTSBuf->f); size_t sz = taosWriteFile(pTSBuf->pFile, (void*)pData, len);
if (sz != len) { if (sz != len) {
// qError("ts data fwrite failed, write size:%d, expected size:%d", (int32_t)sz, len); // qError("ts data fwrite failed, write size:%d, expected size:%d", (int32_t)sz, len);
tsBufDestroy(pTSBuf); tsBufDestroy(pTSBuf);
@ -893,7 +896,7 @@ STSBuf* tsBufCreateFromCompBlocks(const char* pData, int32_t numOfBlocks, int32_
} }
// TODO taosFsync?? // TODO taosFsync??
// if (taosFsync(fileno(pTSBuf->f)) == -1) { // if (taosFsync(fileno(pTSBuf->pFile)) == -1) {
//// qError("fsync failed, errno:%d", errno); //// qError("fsync failed, errno:%d", errno);
// tsBufDestroy(pTSBuf); // tsBufDestroy(pTSBuf);
// return NULL; // return NULL;
@ -1071,15 +1074,15 @@ int32_t dumpFileBlockByGroupId(STSBuf* pTSBuf, int32_t groupIndex, void* buf, in
*len = 0; *len = 0;
*numOfBlocks = 0; *numOfBlocks = 0;
if (fseek(pTSBuf->f, pBlockInfo->offset, SEEK_SET) != 0) { if (taosLSeekFile(pTSBuf->pFile, pBlockInfo->offset, SEEK_SET) != 0) {
int code = TAOS_SYSTEM_ERROR(ferror(pTSBuf->f)); int code = TAOS_SYSTEM_ERROR(taosEOFFile(pTSBuf->pFile));
// qError("%p: fseek failed: %s", pSql, tstrerror(code)); // qError("%p: fseek failed: %s", pSql, tstrerror(code));
return code; return code;
} }
size_t s = fread(buf, 1, pBlockInfo->compLen, pTSBuf->f); size_t s = taosReadFile(pTSBuf->pFile, buf, pBlockInfo->compLen);
if (s != pBlockInfo->compLen) { if (s != pBlockInfo->compLen) {
int code = TAOS_SYSTEM_ERROR(ferror(pTSBuf->f)); int code = TAOS_SYSTEM_ERROR(taosEOFFile(pTSBuf->pFile));
// tscError("%p: fread didn't return expected data: %s", pSql, tstrerror(code)); // tscError("%p: fread didn't return expected data: %s", pSql, tstrerror(code));
return code; return code;
} }

2
source/dnode/mgmt/daemon/CMakeLists.txt
View File

@ -6,4 +6,4 @@ target_include_directories(
PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
) )
target_link_libraries(taosd dnode config util os) target_link_libraries(taosd dnode util os)

12
source/dnode/mgmt/daemon/inc/dmnInt.h
View File

@ -17,7 +17,7 @@
#ifndef _TD_DMN_INT_H_ #ifndef _TD_DMN_INT_H_
#define _TD_DMN_INT_H_ #define _TD_DMN_INT_H_
#include "config.h" #include "tconfig.h"
#include "dnode.h" #include "dnode.h"
#include "taoserror.h" #include "taoserror.h"
#include "tglobal.h" #include "tglobal.h"
@ -28,15 +28,9 @@
extern "C" { extern "C" {
#endif #endif
int32_t dmnAddLogCfg(SConfig *pCfg); SDnodeObjCfg dmnGetObjCfg();
int32_t dmnInitLog(const char *cfgDir, const char *envFile, const char *apolloUrl);
int32_t dmnLoadCfg(SConfig *pConfig, const char *inputCfgDir, const char *envFile, const char *apolloUrl);
SConfig *dmnReadCfg(const char *cfgDir, const char *envFile, const char *apolloUrl); void dmnDumpCfg();
SDnodeEnvCfg dmnGetEnvCfg(SConfig *pCfg);
SDnodeObjCfg dmnGetObjCfg(SConfig *pCfg);
void dmnDumpCfg(SConfig *pCfg);
void dmnPrintVersion(); void dmnPrintVersion();
void dmnGenerateGrant(); void dmnGenerateGrant();

187
source/dnode/mgmt/daemon/src/dmnCfg.c
View File

@ -15,190 +15,14 @@
#define _DEFAULT_SOURCE #define _DEFAULT_SOURCE
#include "dmnInt.h" #include "dmnInt.h"
#include "tlocale.h" #include "tconfig.h"
#include "ttimezone.h"
static int32_t dmnAddEpCfg(SConfig *pCfg) { SDnodeObjCfg dmnGetObjCfg() {
char defaultFqdn[TSDB_FQDN_LEN] = {0}; SConfig *pCfg = taosGetCfg();
if (taosGetFqdn(defaultFqdn) != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
if (cfgAddString(pCfg, "fqdn", defaultFqdn) != 0) return -1;
int32_t defaultServerPort = 6030;
if (cfgAddInt32(pCfg, "serverPort", defaultServerPort, 1, 65056) != 0) return -1;
char defaultFirstEp[TSDB_EP_LEN] = {0};
char defaultSecondEp[TSDB_EP_LEN] = {0};
snprintf(defaultFirstEp, TSDB_EP_LEN, "%s:%d", defaultFqdn, defaultServerPort);
snprintf(defaultSecondEp, TSDB_EP_LEN, "%s:%d", defaultFqdn, defaultServerPort);
if (cfgAddString(pCfg, "firstEp", defaultFirstEp) != 0) return -1;
if (cfgAddString(pCfg, "secondEp", defaultSecondEp) != 0) return -1;
return 0;
}
static int32_t dmnAddDirCfg(SConfig *pCfg) {
if (cfgAddDir(pCfg, "dataDir", tsDataDir) != 0) return -1;
if (cfgAddDir(pCfg, "tempDir", tsTempDir) != 0) return -1;
if (cfgAddFloat(pCfg, "minimalDataDirGB", 2.0f, 0.001f, 10000000) != 0) return -1;
if (cfgAddFloat(pCfg, "minimalTempDirGB", 1.0f, 0.001f, 10000000) != 0) return -1;
return 0;
}
static int32_t dmnCheckDirCfg(SConfig *pCfg) {
tstrncpy(tsDataDir, cfgGetItem(pCfg, "dataDir")->str, PATH_MAX);
tstrncpy(tsTempDir, cfgGetItem(pCfg, "tempDir")->str, PATH_MAX);
tsDataSpace.reserved = cfgGetItem(pCfg, "minimalDataDirGB")->fval;
tsTempSpace.reserved = cfgGetItem(pCfg, "minimalTempDirGB")->fval;
return 0;
}
static int32_t dmnAddVersionCfg(SConfig *pCfg) {
if (cfgAddString(pCfg, "buildinfo", buildinfo) != 0) return -1;
if (cfgAddString(pCfg, "gitinfo", gitinfo) != 0) return -1;
if (cfgAddString(pCfg, "version", version) != 0) return -1;
return 0;
}
static int32_t dmnAddDnodeCfg(SConfig *pCfg) {
if (dmnAddEpCfg(pCfg) != 0) return -1;
if (dmnAddDirCfg(pCfg) != 0) return -1;
if (dmnAddVersionCfg(pCfg) != 0) return -1;
if (cfgAddTimezone(pCfg, "timezone", "") != 0) return -1;
if (cfgAddLocale(pCfg, "locale", "") != 0) return -1;
if (cfgAddCharset(pCfg, "charset", "") != 0) return -1;
if (cfgAddInt32(pCfg, "numOfCores", 2, 1, 100000) != 0) return -1;
if (cfgAddInt32(pCfg, "numOfCommitThreads", 4, 1, 1000) != 0) return -1;
if (cfgAddBool(pCfg, "telemetryReporting", 0) != 0) return -1;
if (cfgAddBool(pCfg, "enableCoreFile", 0) != 0) return -1;
if (cfgAddInt32(pCfg, "supportVnodes", 256, 0, 65536) != 0) return -1;
if (cfgAddInt32(pCfg, "statusInterval", 1, 1, 30) != 0) return -1;
if (cfgAddFloat(pCfg, "numOfThreadsPerCore", 1, 0, 10) != 0) return -1;
if (cfgAddFloat(pCfg, "ratioOfQueryCores", 1, 0, 5) != 0) return -1;
if (cfgAddInt32(pCfg, "maxShellConns", 50000, 10, 50000000) != 0) return -1;
if (cfgAddInt32(pCfg, "shellActivityTimer", 3, 1, 120) != 0) return -1;
return 0;
}
static int32_t dmnCheckCfg(SConfig *pCfg) {
bool enableCore = cfgGetItem(pCfg, "enableCoreFile")->bval;
taosSetCoreDump(enableCore);
if (dmnCheckDirCfg(pCfg) != 0) {
return -1;
}
taosGetSystemInfo();
tsSetTimeZone();
tsSetLocale();
if (tsNumOfCores <= 0) {
tsNumOfCores = 1;
}
if (tsQueryBufferSize >= 0) {
tsQueryBufferSizeBytes = tsQueryBufferSize * 1048576UL;
}
return 0;
}
SConfig *dmnReadCfg(const char *cfgDir, const char *envFile, const char *apolloUrl) {
SConfig *pCfg = cfgInit();
if (pCfg == NULL) return NULL;
if (dmnAddLogCfg(pCfg) != 0) {
uError("failed to add log cfg since %s", terrstr());
cfgCleanup(pCfg);
return NULL;
}
if (dmnAddDnodeCfg(pCfg) != 0) {
uError("failed to init dnode cfg since %s", terrstr());
cfgCleanup(pCfg);
return NULL;
}
if (dmnLoadCfg(pCfg, cfgDir, envFile, apolloUrl) != 0) {
uError("failed to load cfg since %s", terrstr());
cfgCleanup(pCfg);
return NULL;
}
if (dmnCheckCfg(pCfg) != 0) {
uError("failed to check cfg since %s", terrstr());
cfgCleanup(pCfg);
return NULL;
}
cfgDumpCfg(pCfg);
return pCfg;
}
void dmnDumpCfg(SConfig *pCfg) {
printf("taos global config:\n");
printf("==================================\n");
SConfigItem *pItem = cfgIterate(pCfg, NULL);
while (pItem != NULL) {
switch (pItem->dtype) {
case CFG_DTYPE_BOOL:
printf("cfg:%s, value:%u src:%s\n", pItem->name, pItem->bval, cfgStypeStr(pItem->stype));
break;
case CFG_DTYPE_INT32:
printf("cfg:%s, value:%d src:%s\n", pItem->name, pItem->i32, cfgStypeStr(pItem->stype));
break;
case CFG_DTYPE_INT64:
printf("cfg:%s, value:%" PRId64 " src:%s\n", pItem->name, pItem->i64, cfgStypeStr(pItem->stype));
break;
case CFG_DTYPE_FLOAT:
printf("cfg:%s, value:%f src:%s\n", pItem->name, pItem->fval, cfgStypeStr(pItem->stype));
break;
case CFG_DTYPE_STRING:
case CFG_DTYPE_IPSTR:
case CFG_DTYPE_DIR:
case CFG_DTYPE_LOCALE:
case CFG_DTYPE_CHARSET:
case CFG_DTYPE_TIMEZONE:
printf("cfg:%s, value:%s src:%s\n", pItem->name, pItem->str, cfgStypeStr(pItem->stype));
break;
}
pItem = cfgIterate(pCfg, pItem);
}
}
SDnodeEnvCfg dmnGetEnvCfg(SConfig *pCfg) {
SDnodeEnvCfg envCfg = {0};
const char *vstr = cfgGetItem(pCfg, "version")->str;
envCfg.sver = 30000000;
tstrncpy(envCfg.buildinfo, cfgGetItem(pCfg, "buildinfo")->str, sizeof(envCfg.buildinfo));
tstrncpy(envCfg.gitinfo, cfgGetItem(pCfg, "gitinfo")->str, sizeof(envCfg.gitinfo));
tstrncpy(envCfg.timezone, cfgGetItem(pCfg, "timezone")->str, sizeof(envCfg.timezone));
tstrncpy(envCfg.locale, cfgGetItem(pCfg, "locale")->str, sizeof(envCfg.locale));
tstrncpy(envCfg.charset, cfgGetItem(pCfg, "charset")->str, sizeof(envCfg.charset));
envCfg.numOfCores = cfgGetItem(pCfg, "numOfCores")->i32;
envCfg.numOfCommitThreads = (uint16_t)cfgGetItem(pCfg, "numOfCommitThreads")->i32;
envCfg.enableTelem = cfgGetItem(pCfg, "telemetryReporting")->bval;
return envCfg;
}
SDnodeObjCfg dmnGetObjCfg(SConfig *pCfg) {
SDnodeObjCfg objCfg = {0}; SDnodeObjCfg objCfg = {0};
objCfg.numOfSupportVnodes = cfgGetItem(pCfg, "supportVnodes")->i32; objCfg.numOfSupportVnodes = cfgGetItem(pCfg, "supportVnodes")->i32;
objCfg.statusInterval = cfgGetItem(pCfg, "statusInterval")->i32;
objCfg.numOfThreadsPerCore = cfgGetItem(pCfg, "numOfThreadsPerCore")->fval;
objCfg.ratioOfQueryCores = cfgGetItem(pCfg, "ratioOfQueryCores")->fval;
objCfg.maxShellConns = cfgGetItem(pCfg, "maxShellConns")->i32;
objCfg.shellActivityTimer = cfgGetItem(pCfg, "shellActivityTimer")->i32;
tstrncpy(objCfg.dataDir, cfgGetItem(pCfg, "dataDir")->str, sizeof(objCfg.dataDir)); tstrncpy(objCfg.dataDir, cfgGetItem(pCfg, "dataDir")->str, sizeof(objCfg.dataDir));
tstrncpy(objCfg.firstEp, cfgGetItem(pCfg, "firstEp")->str, sizeof(objCfg.firstEp)); tstrncpy(objCfg.firstEp, cfgGetItem(pCfg, "firstEp")->str, sizeof(objCfg.firstEp));
tstrncpy(objCfg.secondEp, cfgGetItem(pCfg, "secondEp")->str, sizeof(objCfg.firstEp)); tstrncpy(objCfg.secondEp, cfgGetItem(pCfg, "secondEp")->str, sizeof(objCfg.firstEp));
objCfg.serverPort = (uint16_t)cfgGetItem(pCfg, "serverPort")->i32; objCfg.serverPort = (uint16_t)cfgGetItem(pCfg, "serverPort")->i32;
@ -206,3 +30,8 @@ SDnodeObjCfg dmnGetObjCfg(SConfig *pCfg) {
snprintf(objCfg.localEp, sizeof(objCfg.localEp), "%s:%u", objCfg.localFqdn, objCfg.serverPort); snprintf(objCfg.localEp, sizeof(objCfg.localEp), "%s:%u", objCfg.localFqdn, objCfg.serverPort);
return objCfg; return objCfg;
} }
void dmnDumpCfg() {
SConfig *pCfg = taosGetCfg();
cfgDumpCfg(pCfg, 0, 1);
}

132
source/dnode/mgmt/daemon/src/dmnLog.c
View File

@ -1,132 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "dmnInt.h"
int32_t dmnAddLogCfg(SConfig *pCfg) {
if (cfgAddDir(pCfg, "logDir", "/var/log/taos") != 0) return -1;
if (cfgAddFloat(pCfg, "minimalLogDirGB", 1.0f, 0.001f, 10000000) != 0) return -1;
if (cfgAddBool(pCfg, "asyncLog", 1) != 0) return -1;
if (cfgAddInt32(pCfg, "numOfLogLines", 10000000, 1000, 2000000000) != 0) return -1;
if (cfgAddInt32(pCfg, "logKeepDays", 0, -365000, 365000) != 0) return -1;
if (cfgAddInt32(pCfg, "debugFlag", 0, 0, 255) != 0) return -1;
if (cfgAddInt32(pCfg, "dDebugFlag", 0, 0, 255) != 0) return -1;
if (cfgAddInt32(pCfg, "vDebugFlag", 0, 0, 255) != 0) return -1;
if (cfgAddInt32(pCfg, "mDebugFlag", 0, 0, 255) != 0) return -1;
if (cfgAddInt32(pCfg, "cDebugFlag", 0, 0, 255) != 0) return -1;
if (cfgAddInt32(pCfg, "jniDebugFlag", 0, 0, 255) != 0) return -1;
if (cfgAddInt32(pCfg, "tmrDebugFlag", 0, 0, 255) != 0) return -1;
if (cfgAddInt32(pCfg, "uDebugFlag", 0, 0, 255) != 0) return -1;
if (cfgAddInt32(pCfg, "rpcDebugFlag", 0, 0, 255) != 0) return -1;
if (cfgAddInt32(pCfg, "qDebugFlag", 0, 0, 255) != 0) return -1;
if (cfgAddInt32(pCfg, "wDebugFlag", 0, 0, 255) != 0) return -1;
if (cfgAddInt32(pCfg, "sDebugFlag", 0, 0, 255) != 0) return -1;
if (cfgAddInt32(pCfg, "tsdbDebugFlag", 0, 0, 255) != 0) return -1;
if (cfgAddInt32(pCfg, "tqDebugFlag", 0, 0, 255) != 0) return -1;
if (cfgAddInt32(pCfg, "fsDebugFlag", 0, 0, 255) != 0) return -1;
return 0;
}
int32_t dmnSetLogCfg(SConfig *pCfg) {
tstrncpy(tsLogDir, cfgGetItem(pCfg, "logDir")->str, PATH_MAX);
tsLogSpace.reserved = cfgGetItem(pCfg, "minimalLogDirGB")->fval;
tsAsyncLog = cfgGetItem(pCfg, "asyncLog")->bval;
tsNumOfLogLines = cfgGetItem(pCfg, "numOfLogLines")->i32;
tsLogKeepDays = cfgGetItem(pCfg, "logKeepDays")->i32;
dDebugFlag = cfgGetItem(pCfg, "dDebugFlag")->i32;
vDebugFlag = cfgGetItem(pCfg, "vDebugFlag")->i32;
mDebugFlag = cfgGetItem(pCfg, "mDebugFlag")->i32;
cDebugFlag = cfgGetItem(pCfg, "cDebugFlag")->i32;
jniDebugFlag = cfgGetItem(pCfg, "jniDebugFlag")->i32;
tmrDebugFlag = cfgGetItem(pCfg, "tmrDebugFlag")->i32;
uDebugFlag = cfgGetItem(pCfg, "uDebugFlag")->i32;
rpcDebugFlag = cfgGetItem(pCfg, "rpcDebugFlag")->i32;
qDebugFlag = cfgGetItem(pCfg, "qDebugFlag")->i32;
wDebugFlag = cfgGetItem(pCfg, "wDebugFlag")->i32;
sDebugFlag = cfgGetItem(pCfg, "sDebugFlag")->i32;
tsdbDebugFlag = cfgGetItem(pCfg, "tsdbDebugFlag")->i32;
tqDebugFlag = cfgGetItem(pCfg, "tqDebugFlag")->i32;
fsDebugFlag = cfgGetItem(pCfg, "fsDebugFlag")->i32;
int32_t debugFlag = cfgGetItem(pCfg, "debugFlag")->i32;
taosSetAllDebugFlag(debugFlag);
return 0;
}
int32_t dmnInitLog(const char *cfgDir, const char *envFile, const char *apolloUrl) {
SConfig *pCfg = cfgInit();
if (pCfg == NULL) return -1;
if (dmnAddLogCfg(pCfg) != 0) {
printf("failed to add log cfg since %s\n", terrstr());
cfgCleanup(pCfg);
return -1;
}
if (dmnLoadCfg(pCfg, cfgDir, envFile, apolloUrl) != 0) {
printf("failed to load log cfg since %s\n", terrstr());
cfgCleanup(pCfg);
return -1;
}
if (dmnSetLogCfg(pCfg) != 0) {
printf("failed to set log cfg since %s\n", terrstr());
cfgCleanup(pCfg);
return -1;
}
if (taosInitLog("taosdlog", 1) != 0) {
printf("failed to init log file since %s\n", terrstr());
cfgCleanup(pCfg);
return -1;
}
cfgCleanup(pCfg);
return 0;
}
int32_t dmnLoadCfg(SConfig *pConfig, const char *inputCfgDir, const char *envFile, const char *apolloUrl) {
char configDir[PATH_MAX] = {0};
char configFile[PATH_MAX + 100] = {0};
taosExpandDir(inputCfgDir, configDir, PATH_MAX);
snprintf(configFile, sizeof(configFile), "%s" TD_DIRSEP "taos.cfg", configDir);
if (cfgLoad(pConfig, CFG_STYPE_APOLLO_URL, apolloUrl) != 0) {
uError("failed to load from apollo url:%s since %s\n", apolloUrl, terrstr());
return -1;
}
if (cfgLoad(pConfig, CFG_STYPE_CFG_FILE, configFile) != 0) {
if (cfgLoad(pConfig, CFG_STYPE_CFG_FILE, configDir) != 0) {
uError("failed to load from config file:%s since %s\n", configFile, terrstr());
return -1;
}
}
if (cfgLoad(pConfig, CFG_STYPE_ENV_FILE, envFile) != 0) {
uError("failed to load from env file:%s since %s\n", envFile, terrstr());
return -1;
}
if (cfgLoad(pConfig, CFG_STYPE_ENV_VAR, NULL) != 0) {
uError("failed to load from global env variables since %s\n", terrstr());
return -1;
}
return 0;
}

22
source/dnode/mgmt/daemon/src/dmnMain.c
View File

@ -72,14 +72,13 @@ static int32_t dmnParseOption(int32_t argc, char const *argv[]) {
return 0; return 0;
} }
int32_t dmnRunDnode(SConfig *pCfg) { int32_t dmnRunDnode() {
SDnodeEnvCfg envCfg = dmnGetEnvCfg(pCfg); if (dndInit() != 0) {
if (dndInit(&envCfg) != 0) {
uInfo("Failed to start TDengine, please check the log"); uInfo("Failed to start TDengine, please check the log");
return -1; return -1;
} }
SDnodeObjCfg objCfg = dmnGetObjCfg(pCfg); SDnodeObjCfg objCfg = dmnGetObjCfg();
SDnode *pDnode = dndCreate(&objCfg); SDnode *pDnode = dndCreate(&objCfg);
if (pDnode == NULL) { if (pDnode == NULL) {
uInfo("Failed to start TDengine, please check the log"); uInfo("Failed to start TDengine, please check the log");
@ -93,6 +92,7 @@ int32_t dmnRunDnode(SConfig *pCfg) {
dndClose(pDnode); dndClose(pDnode);
dndCleanup(); dndCleanup();
taosCloseLog(); taosCloseLog();
taosCleanupCfg();
return 0; return 0;
} }
@ -111,23 +111,21 @@ int main(int argc, char const *argv[]) {
return 0; return 0;
} }
if (dmnInitLog(configDir, dmn.envFile, dmn.apolloUrl) != 0) { if (taosCreateLog("taosdlog", 1, configDir, dmn.envFile, dmn.apolloUrl, 0) != 0) {
uInfo("Failed to start TDengine since read config error");
return -1; return -1;
} }
SConfig *pCfg = dmnReadCfg(configDir, dmn.envFile, dmn.apolloUrl); if (taosInitCfg(configDir, dmn.envFile, dmn.apolloUrl, 0) != 0) {
if (pCfg == NULL) {
uInfo("Failed to start TDengine since read config error"); uInfo("Failed to start TDengine since read config error");
return -1; return -1;
} }
if (dmn.dumpConfig) { if (dmn.dumpConfig) {
dmnDumpCfg(pCfg); dmnDumpCfg();
cfgCleanup(pCfg); taosCleanupCfg();
return 0; return 0;
} }
int32_t code = dmnRunDnode(pCfg); return dmnRunDnode();
cfgCleanup(pCfg);
return code;
} }

8
source/dnode/mgmt/impl/inc/dndEnv.h
View File

@ -124,9 +124,8 @@ typedef struct {
typedef struct SDnode { typedef struct SDnode {
EStat stat; EStat stat;
SDnodeObjCfg cfg; SDnodeObjCfg cfg;
SDnodeEnvCfg env;
SDnodeDir dir; SDnodeDir dir;
FileFd lockFd; TdFilePtr pLockFile;
SDnodeMgmt dmgmt; SDnodeMgmt dmgmt;
SMnodeMgmt mmgmt; SMnodeMgmt mmgmt;
SQnodeMgmt qmgmt; SQnodeMgmt qmgmt;
@ -138,11 +137,6 @@ typedef struct SDnode {
SStartupReq startup; SStartupReq startup;
} SDnode; } SDnode;
typedef struct {
int8_t once;
SDnodeEnvCfg cfg;
} SDnodeEnv;
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

1
source/dnode/mgmt/impl/inc/dndInt.h
View File

@ -35,6 +35,7 @@ extern "C" {
#include "tthread.h" #include "tthread.h"
#include "ttime.h" #include "ttime.h"
#include "tworker.h" #include "tworker.h"
#include "tglobal.h"
#include "dnode.h" #include "dnode.h"

22
source/dnode/mgmt/impl/src/dndBnode.c
View File

@ -62,14 +62,15 @@ static int32_t dndReadBnodeFile(SDnode *pDnode) {
char file[PATH_MAX + 20]; char file[PATH_MAX + 20];
snprintf(file, PATH_MAX + 20, "%s/bnode.json", pDnode->dir.dnode); snprintf(file, PATH_MAX + 20, "%s/bnode.json", pDnode->dir.dnode);
FILE *fp = fopen(file, "r"); // FILE *fp = fopen(file, "r");
if (fp == NULL) { TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ);
if (pFile == NULL) {
dDebug("file %s not exist", file); dDebug("file %s not exist", file);
code = 0; code = 0;
goto PRASE_BNODE_OVER; goto PRASE_BNODE_OVER;
} }
len = (int32_t)fread(content, 1, maxLen, fp); len = (int32_t)taosReadFile(pFile, content, maxLen);
if (len <= 0) { if (len <= 0) {
dError("failed to read %s since content is null", file); dError("failed to read %s since content is null", file);
goto PRASE_BNODE_OVER; goto PRASE_BNODE_OVER;
@ -102,7 +103,7 @@ static int32_t dndReadBnodeFile(SDnode *pDnode) {
PRASE_BNODE_OVER: PRASE_BNODE_OVER:
if (content != NULL) free(content); if (content != NULL) free(content);
if (root != NULL) cJSON_Delete(root); if (root != NULL) cJSON_Delete(root);
if (fp != NULL) fclose(fp); if (pFile != NULL) taosCloseFile(&pFile);
terrno = code; terrno = code;
return code; return code;
@ -114,8 +115,9 @@ static int32_t dndWriteBnodeFile(SDnode *pDnode) {
char file[PATH_MAX + 20]; char file[PATH_MAX + 20];
snprintf(file, PATH_MAX + 20, "%s/bnode.json", pDnode->dir.dnode); snprintf(file, PATH_MAX + 20, "%s/bnode.json", pDnode->dir.dnode);
FILE *fp = fopen(file, "w"); // FILE *fp = fopen(file, "w");
if (fp == NULL) { TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
if (pFile == NULL) {
terrno = TSDB_CODE_DND_BNODE_WRITE_FILE_ERROR; terrno = TSDB_CODE_DND_BNODE_WRITE_FILE_ERROR;
dError("failed to write %s since %s", file, terrstr()); dError("failed to write %s since %s", file, terrstr());
return -1; return -1;
@ -130,9 +132,9 @@ static int32_t dndWriteBnodeFile(SDnode *pDnode) {
len += snprintf(content + len, maxLen - len, " \"dropped\": %d\n", pMgmt->dropped); len += snprintf(content + len, maxLen - len, " \"dropped\": %d\n", pMgmt->dropped);
len += snprintf(content + len, maxLen - len, "}\n"); len += snprintf(content + len, maxLen - len, "}\n");
fwrite(content, 1, len, fp); taosWriteFile(pFile, content, len);
taosFsyncFile(fileno(fp)); taosFsyncFile(pFile);
fclose(fp); taosCloseFile(&pFile);
free(content); free(content);
char realfile[PATH_MAX + 20]; char realfile[PATH_MAX + 20];
@ -179,7 +181,7 @@ static void dndBuildBnodeOption(SDnode *pDnode, SBnodeOpt *pOption) {
pOption->sendRedirectRspFp = dndSendRedirectRsp; pOption->sendRedirectRspFp = dndSendRedirectRsp;
pOption->dnodeId = dndGetDnodeId(pDnode); pOption->dnodeId = dndGetDnodeId(pDnode);
pOption->clusterId = dndGetClusterId(pDnode); pOption->clusterId = dndGetClusterId(pDnode);
pOption->sver = pDnode->env.sver; pOption->sver = tsVersion;
} }
static int32_t dndOpenBnode(SDnode *pDnode) { static int32_t dndOpenBnode(SDnode *pDnode) {

45
source/dnode/mgmt/impl/src/dndEnv.c
View File

@ -25,7 +25,7 @@
#include "tfs.h" #include "tfs.h"
#include "wal.h" #include "wal.h"
static SDnodeEnv dndEnv = {0}; static int8_t once = DND_ENV_INIT;
EStat dndGetStat(SDnode *pDnode) { return pDnode->stat; } EStat dndGetStat(SDnode *pDnode) { return pDnode->stat; }
@ -59,31 +59,31 @@ void dndGetStartup(SDnode *pDnode, SStartupReq *pStartup) {
pStartup->finished = (dndGetStat(pDnode) == DND_STAT_RUNNING); pStartup->finished = (dndGetStat(pDnode) == DND_STAT_RUNNING);
} }
static FileFd dndCheckRunning(char *dataDir) { static TdFilePtr dndCheckRunning(char *dataDir) {
char filepath[PATH_MAX] = {0}; char filepath[PATH_MAX] = {0};
snprintf(filepath, sizeof(filepath), "%s/.running", dataDir); snprintf(filepath, sizeof(filepath), "%s/.running", dataDir);
FileFd fd = taosOpenFileCreateWriteTrunc(filepath); TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
if (fd < 0) { if (pFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
dError("failed to lock file:%s since %s, quit", filepath, terrstr()); dError("failed to lock file:%s since %s, quit", filepath, terrstr());
return -1; return NULL;
} }
int32_t ret = taosLockFile(fd); int32_t ret = taosLockFile(pFile);
if (ret != 0) { if (ret != 0) {
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
dError("failed to lock file:%s since %s, quit", filepath, terrstr()); dError("failed to lock file:%s since %s, quit", filepath, terrstr());
taosCloseFile(fd); taosCloseFile(&pFile);
return -1; return NULL;
} }
return fd; return pFile;
} }
static int32_t dndCreateImp(SDnode *pDnode, SDnodeObjCfg *pCfg) { static int32_t dndCreateImp(SDnode *pDnode, SDnodeObjCfg *pCfg) {
pDnode->lockFd = dndCheckRunning(pCfg->dataDir); pDnode->pLockFile = dndCheckRunning(pCfg->dataDir);
if (pDnode->lockFd < 0) { if (pDnode->pLockFile == NULL) {
return -1; return -1;
} }
@ -137,7 +137,6 @@ static int32_t dndCreateImp(SDnode *pDnode, SDnodeObjCfg *pCfg) {
} }
memcpy(&pDnode->cfg, pCfg, sizeof(SDnodeObjCfg)); memcpy(&pDnode->cfg, pCfg, sizeof(SDnodeObjCfg));
memcpy(&pDnode->env, &dndEnv.cfg, sizeof(SDnodeEnvCfg));
return 0; return 0;
} }
@ -148,10 +147,10 @@ static void dndCloseImp(SDnode *pDnode) {
tfree(pDnode->dir.snode); tfree(pDnode->dir.snode);
tfree(pDnode->dir.bnode); tfree(pDnode->dir.bnode);
if (pDnode->lockFd >= 0) { if (pDnode->pLockFile != NULL) {
taosUnLockFile(pDnode->lockFd); taosUnLockFile(pDnode->pLockFile);
taosCloseFile(pDnode->lockFd); taosCloseFile(&pDnode->pLockFile);
pDnode->lockFd = 0; pDnode->pLockFile = NULL;
} }
} }
@ -259,8 +258,8 @@ void dndClose(SDnode *pDnode) {
dInfo("dnode object is closed, data:%p", pDnode); dInfo("dnode object is closed, data:%p", pDnode);
} }
int32_t dndInit(const SDnodeEnvCfg *pCfg) { int32_t dndInit() {
if (atomic_val_compare_exchange_8(&dndEnv.once, DND_ENV_INIT, DND_ENV_READY) != DND_ENV_INIT) { if (atomic_val_compare_exchange_8(&once, DND_ENV_INIT, DND_ENV_READY) != DND_ENV_INIT) {
terrno = TSDB_CODE_REPEAT_INIT; terrno = TSDB_CODE_REPEAT_INIT;
dError("failed to init dnode env since %s", terrstr()); dError("failed to init dnode env since %s", terrstr());
return -1; return -1;
@ -283,11 +282,8 @@ int32_t dndInit(const SDnodeEnvCfg *pCfg) {
} }
SVnodeOpt vnodeOpt = { SVnodeOpt vnodeOpt = {
.sver = pCfg->sver, .sver = tsVersion,
.timezone = pCfg->timezone, .nthreads = tsNumOfCommitThreads,
.locale = pCfg->locale,
.charset = pCfg->charset,
.nthreads = pCfg->numOfCommitThreads,
.putReqToVQueryQFp = dndPutReqToVQueryQ, .putReqToVQueryQFp = dndPutReqToVQueryQ,
.sendReqToDnodeFp = dndSendReqToDnode .sendReqToDnodeFp = dndSendReqToDnode
}; };
@ -298,13 +294,12 @@ int32_t dndInit(const SDnodeEnvCfg *pCfg) {
return -1; return -1;
} }
memcpy(&dndEnv.cfg, pCfg, sizeof(SDnodeEnvCfg));
dInfo("dnode env is initialized"); dInfo("dnode env is initialized");
return 0; return 0;
} }
void dndCleanup() { void dndCleanup() {
if (atomic_val_compare_exchange_8(&dndEnv.once, DND_ENV_READY, DND_ENV_CLEANUP) != DND_ENV_READY) { if (atomic_val_compare_exchange_8(&once, DND_ENV_READY, DND_ENV_CLEANUP) != DND_ENV_READY) {
dError("dnode env is already cleaned up"); dError("dnode env is already cleaned up");
return; return;
} }

37
source/dnode/mgmt/impl/src/dndMgmt.c
View File

@ -185,16 +185,16 @@ static int32_t dndReadDnodes(SDnode *pDnode) {
int32_t maxLen = 256 * 1024; int32_t maxLen = 256 * 1024;
char *content = calloc(1, maxLen + 1); char *content = calloc(1, maxLen + 1);
cJSON *root = NULL; cJSON *root = NULL;
FILE *fp = NULL;
fp = fopen(pMgmt->file, "r"); // fp = fopen(pMgmt->file, "r");
if (fp == NULL) { TdFilePtr pFile = taosOpenFile(pMgmt->file, TD_FILE_READ);
if (pFile == NULL) {
dDebug("file %s not exist", pMgmt->file); dDebug("file %s not exist", pMgmt->file);
code = 0; code = 0;
goto PRASE_DNODE_OVER; goto PRASE_DNODE_OVER;
} }
len = (int32_t)fread(content, 1, maxLen, fp); len = (int32_t)taosReadFile(pFile, content, maxLen);
if (len <= 0) { if (len <= 0) {
dError("failed to read %s since content is null", pMgmt->file); dError("failed to read %s since content is null", pMgmt->file);
goto PRASE_DNODE_OVER; goto PRASE_DNODE_OVER;
@ -286,7 +286,7 @@ static int32_t dndReadDnodes(SDnode *pDnode) {
PRASE_DNODE_OVER: PRASE_DNODE_OVER:
if (content != NULL) free(content); if (content != NULL) free(content);
if (root != NULL) cJSON_Delete(root); if (root != NULL) cJSON_Delete(root);
if (fp != NULL) fclose(fp); if (pFile != NULL) taosCloseFile(&pFile);
if (dndIsEpChanged(pDnode, pMgmt->dnodeId, pDnode->cfg.localEp)) { if (dndIsEpChanged(pDnode, pMgmt->dnodeId, pDnode->cfg.localEp)) {
dError("localEp %s different with %s and need reconfigured", pDnode->cfg.localEp, pMgmt->file); dError("localEp %s different with %s and need reconfigured", pDnode->cfg.localEp, pMgmt->file);
@ -296,7 +296,7 @@ PRASE_DNODE_OVER:
if (taosArrayGetSize(pMgmt->pDnodeEps) == 0) { if (taosArrayGetSize(pMgmt->pDnodeEps) == 0) {
SDnodeEp dnodeEp = {0}; SDnodeEp dnodeEp = {0};
dnodeEp.isMnode = 1; dnodeEp.isMnode = 1;
taosGetFqdnPortFromEp(pDnode->cfg.firstEp, pDnode->cfg.serverPort, &dnodeEp.ep); taosGetFqdnPortFromEp(pDnode->cfg.firstEp, &dnodeEp.ep);
taosArrayPush(pMgmt->pDnodeEps, &dnodeEp); taosArrayPush(pMgmt->pDnodeEps, &dnodeEp);
} }
@ -309,8 +309,9 @@ PRASE_DNODE_OVER:
static int32_t dndWriteDnodes(SDnode *pDnode) { static int32_t dndWriteDnodes(SDnode *pDnode) {
SDnodeMgmt *pMgmt = &pDnode->dmgmt; SDnodeMgmt *pMgmt = &pDnode->dmgmt;
FILE *fp = fopen(pMgmt->file, "w"); // FILE *fp = fopen(pMgmt->file, "w");
if (fp == NULL) { TdFilePtr pFile = taosOpenFile(pMgmt->file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
if (pFile == NULL) {
dError("failed to write %s since %s", pMgmt->file, strerror(errno)); dError("failed to write %s since %s", pMgmt->file, strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
return -1; return -1;
@ -341,9 +342,9 @@ static int32_t dndWriteDnodes(SDnode *pDnode) {
} }
len += snprintf(content + len, maxLen - len, "}\n"); len += snprintf(content + len, maxLen - len, "}\n");
fwrite(content, 1, len, fp); taosWriteFile(pFile, content, len);
taosFsyncFile(fileno(fp)); taosFsyncFile(pFile);
fclose(fp); taosCloseFile(&pFile);
free(content); free(content);
terrno = 0; terrno = 0;
@ -357,23 +358,23 @@ void dndSendStatusReq(SDnode *pDnode) {
SDnodeMgmt *pMgmt = &pDnode->dmgmt; SDnodeMgmt *pMgmt = &pDnode->dmgmt;
taosRLockLatch(&pMgmt->latch); taosRLockLatch(&pMgmt->latch);
req.sver = pDnode->env.sver; req.sver = tsVersion;
req.dver = pMgmt->dver; req.dver = pMgmt->dver;
req.dnodeId = pMgmt->dnodeId; req.dnodeId = pMgmt->dnodeId;
req.clusterId = pMgmt->clusterId; req.clusterId = pMgmt->clusterId;
req.rebootTime = pMgmt->rebootTime; req.rebootTime = pMgmt->rebootTime;
req.updateTime = pMgmt->updateTime; req.updateTime = pMgmt->updateTime;
req.numOfCores = pDnode->env.numOfCores; req.numOfCores = tsNumOfCores;
req.numOfSupportVnodes = pDnode->cfg.numOfSupportVnodes; req.numOfSupportVnodes = pDnode->cfg.numOfSupportVnodes;
memcpy(req.dnodeEp, pDnode->cfg.localEp, TSDB_EP_LEN); memcpy(req.dnodeEp, pDnode->cfg.localEp, TSDB_EP_LEN);
req.clusterCfg.statusInterval = pDnode->cfg.statusInterval; req.clusterCfg.statusInterval = tsStatusInterval;
req.clusterCfg.checkTime = 0; req.clusterCfg.checkTime = 0;
char timestr[32] = "1970-01-01 00:00:00.00"; char timestr[32] = "1970-01-01 00:00:00.00";
(void)taosParseTime(timestr, &req.clusterCfg.checkTime, (int32_t)strlen(timestr), TSDB_TIME_PRECISION_MILLI, 0); (void)taosParseTime(timestr, &req.clusterCfg.checkTime, (int32_t)strlen(timestr), TSDB_TIME_PRECISION_MILLI, 0);
memcpy(req.clusterCfg.timezone, pDnode->env.timezone, TSDB_TIMEZONE_LEN); memcpy(req.clusterCfg.timezone, tsTimezone, TD_TIMEZONE_LEN);
memcpy(req.clusterCfg.locale, pDnode->env.locale, TSDB_LOCALE_LEN); memcpy(req.clusterCfg.locale, tsLocale, TD_LOCALE_LEN);
memcpy(req.clusterCfg.charset, pDnode->env.charset, TSDB_LOCALE_LEN); memcpy(req.clusterCfg.charset, tsCharset, TD_LOCALE_LEN);
taosRUnLockLatch(&pMgmt->latch); taosRUnLockLatch(&pMgmt->latch);
req.pVloads = taosArrayInit(TSDB_MAX_VNODES, sizeof(SVnodeLoad)); req.pVloads = taosArrayInit(TSDB_MAX_VNODES, sizeof(SVnodeLoad));
@ -475,7 +476,7 @@ void dndProcessStartupReq(SDnode *pDnode, SRpcMsg *pReq) {
static void *dnodeThreadRoutine(void *param) { static void *dnodeThreadRoutine(void *param) {
SDnode *pDnode = param; SDnode *pDnode = param;
SDnodeMgmt *pMgmt = &pDnode->dmgmt; SDnodeMgmt *pMgmt = &pDnode->dmgmt;
int32_t ms = pDnode->cfg.statusInterval * 1000; int32_t ms = tsStatusInterval * 1000;
setThreadName("dnode-hb"); setThreadName("dnode-hb");

29
source/dnode/mgmt/impl/src/dndMnode.c
View File

@ -63,14 +63,15 @@ static int32_t dndReadMnodeFile(SDnode *pDnode) {
char file[PATH_MAX + 20]; char file[PATH_MAX + 20];
snprintf(file, PATH_MAX + 20, "%s/mnode.json", pDnode->dir.dnode); snprintf(file, PATH_MAX + 20, "%s/mnode.json", pDnode->dir.dnode);
FILE *fp = fopen(file, "r"); // FILE *fp = fopen(file, "r");
if (fp == NULL) { TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ);
if (pFile == NULL) {
dDebug("file %s not exist", file); dDebug("file %s not exist", file);
code = 0; code = 0;
goto PRASE_MNODE_OVER; goto PRASE_MNODE_OVER;
} }
len = (int32_t)fread(content, 1, maxLen, fp); len = (int32_t)taosReadFile(pFile, content, maxLen);
if (len <= 0) { if (len <= 0) {
dError("failed to read %s since content is null", file); dError("failed to read %s since content is null", file);
goto PRASE_MNODE_OVER; goto PRASE_MNODE_OVER;
@ -143,7 +144,7 @@ static int32_t dndReadMnodeFile(SDnode *pDnode) {
PRASE_MNODE_OVER: PRASE_MNODE_OVER:
if (content != NULL) free(content); if (content != NULL) free(content);
if (root != NULL) cJSON_Delete(root); if (root != NULL) cJSON_Delete(root);
if (fp != NULL) fclose(fp); if (pFile != NULL) taosCloseFile(&pFile);
terrno = code; terrno = code;
return code; return code;
@ -155,8 +156,9 @@ static int32_t dndWriteMnodeFile(SDnode *pDnode) {
char file[PATH_MAX + 20]; char file[PATH_MAX + 20];
snprintf(file, PATH_MAX + 20, "%s/mnode.json.bak", pDnode->dir.dnode); snprintf(file, PATH_MAX + 20, "%s/mnode.json.bak", pDnode->dir.dnode);
FILE *fp = fopen(file, "w"); // FILE *fp = fopen(file, "w");
if (fp == NULL) { TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
if (pFile == NULL) {
terrno = TSDB_CODE_DND_MNODE_WRITE_FILE_ERROR; terrno = TSDB_CODE_DND_MNODE_WRITE_FILE_ERROR;
dError("failed to write %s since %s", file, terrstr()); dError("failed to write %s since %s", file, terrstr());
return -1; return -1;
@ -184,9 +186,9 @@ static int32_t dndWriteMnodeFile(SDnode *pDnode) {
} }
len += snprintf(content + len, maxLen - len, "}\n"); len += snprintf(content + len, maxLen - len, "}\n");
fwrite(content, 1, len, fp); taosWriteFile(pFile, content, len);
taosFsyncFile(fileno(fp)); taosFsyncFile(pFile);
fclose(fp); taosCloseFile(&pFile);
free(content); free(content);
char realfile[PATH_MAX + 20]; char realfile[PATH_MAX + 20];
@ -273,15 +275,6 @@ static void dndInitMnodeOption(SDnode *pDnode, SMnodeOpt *pOption) {
pOption->putReqToMReadQFp = dndPutMsgToMReadQ; pOption->putReqToMReadQFp = dndPutMsgToMReadQ;
pOption->dnodeId = dndGetDnodeId(pDnode); pOption->dnodeId = dndGetDnodeId(pDnode);
pOption->clusterId = dndGetClusterId(pDnode); pOption->clusterId = dndGetClusterId(pDnode);
pOption->cfg.sver = pDnode->env.sver;
pOption->cfg.enableTelem = pDnode->env.enableTelem;
pOption->cfg.statusInterval = pDnode->cfg.statusInterval;
pOption->cfg.shellActivityTimer = pDnode->cfg.shellActivityTimer;
pOption->cfg.timezone = pDnode->env.timezone;
pOption->cfg.charset = pDnode->env.charset;
pOption->cfg.locale = pDnode->env.locale;
pOption->cfg.gitinfo = pDnode->env.gitinfo;
pOption->cfg.buildinfo = pDnode->env.buildinfo;
} }
static void dndBuildMnodeDeployOption(SDnode *pDnode, SMnodeOpt *pOption) { static void dndBuildMnodeDeployOption(SDnode *pDnode, SMnodeOpt *pOption) {

22
source/dnode/mgmt/impl/src/dndQnode.c
View File

@ -62,14 +62,15 @@ static int32_t dndReadQnodeFile(SDnode *pDnode) {
char file[PATH_MAX + 20]; char file[PATH_MAX + 20];
snprintf(file, PATH_MAX + 20, "%s/qnode.json", pDnode->dir.dnode); snprintf(file, PATH_MAX + 20, "%s/qnode.json", pDnode->dir.dnode);
FILE *fp = fopen(file, "r"); // FILE *fp = fopen(file, "r");
if (fp == NULL) { TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ);
if (pFile == NULL) {
dDebug("file %s not exist", file); dDebug("file %s not exist", file);
code = 0; code = 0;
goto PRASE_QNODE_OVER; goto PRASE_QNODE_OVER;
} }
len = (int32_t)fread(content, 1, maxLen, fp); len = (int32_t)taosReadFile(pFile, content, maxLen);
if (len <= 0) { if (len <= 0) {
dError("failed to read %s since content is null", file); dError("failed to read %s since content is null", file);
goto PRASE_QNODE_OVER; goto PRASE_QNODE_OVER;
@ -102,7 +103,7 @@ static int32_t dndReadQnodeFile(SDnode *pDnode) {
PRASE_QNODE_OVER: PRASE_QNODE_OVER:
if (content != NULL) free(content); if (content != NULL) free(content);
if (root != NULL) cJSON_Delete(root); if (root != NULL) cJSON_Delete(root);
if (fp != NULL) fclose(fp); if (pFile != NULL) taosCloseFile(&pFile);
terrno = code; terrno = code;
return code; return code;
@ -114,8 +115,9 @@ static int32_t dndWriteQnodeFile(SDnode *pDnode) {
char file[PATH_MAX + 20]; char file[PATH_MAX + 20];
snprintf(file, PATH_MAX + 20, "%s/qnode.json", pDnode->dir.dnode); snprintf(file, PATH_MAX + 20, "%s/qnode.json", pDnode->dir.dnode);
FILE *fp = fopen(file, "w"); // FILE *fp = fopen(file, "w");
if (fp == NULL) { TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
if (pFile == NULL) {
terrno = TSDB_CODE_DND_QNODE_WRITE_FILE_ERROR; terrno = TSDB_CODE_DND_QNODE_WRITE_FILE_ERROR;
dError("failed to write %s since %s", file, terrstr()); dError("failed to write %s since %s", file, terrstr());
return -1; return -1;
@ -130,9 +132,9 @@ static int32_t dndWriteQnodeFile(SDnode *pDnode) {
len += snprintf(content + len, maxLen - len, " \"dropped\": %d\n", pMgmt->dropped); len += snprintf(content + len, maxLen - len, " \"dropped\": %d\n", pMgmt->dropped);
len += snprintf(content + len, maxLen - len, "}\n"); len += snprintf(content + len, maxLen - len, "}\n");
fwrite(content, 1, len, fp); taosWriteFile(pFile, content, len);
taosFsyncFile(fileno(fp)); taosFsyncFile(pFile);
fclose(fp); taosCloseFile(&pFile);
free(content); free(content);
char realfile[PATH_MAX + 20]; char realfile[PATH_MAX + 20];
@ -185,7 +187,7 @@ static void dndBuildQnodeOption(SDnode *pDnode, SQnodeOpt *pOption) {
pOption->sendRedirectRspFp = dndSendRedirectRsp; pOption->sendRedirectRspFp = dndSendRedirectRsp;
pOption->dnodeId = dndGetDnodeId(pDnode); pOption->dnodeId = dndGetDnodeId(pDnode);
pOption->clusterId = dndGetClusterId(pDnode); pOption->clusterId = dndGetClusterId(pDnode);
pOption->sver = pDnode->env.sver; pOption->sver = tsVersion;
} }
static int32_t dndOpenQnode(SDnode *pDnode) { static int32_t dndOpenQnode(SDnode *pDnode) {

22
source/dnode/mgmt/impl/src/dndSnode.c
View File

@ -62,14 +62,15 @@ static int32_t dndReadSnodeFile(SDnode *pDnode) {
char file[PATH_MAX + 20]; char file[PATH_MAX + 20];
snprintf(file, PATH_MAX + 20, "%s/snode.json", pDnode->dir.dnode); snprintf(file, PATH_MAX + 20, "%s/snode.json", pDnode->dir.dnode);
FILE *fp = fopen(file, "r"); // FILE *fp = fopen(file, "r");
if (fp == NULL) { TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ);
if (pFile == NULL) {
dDebug("file %s not exist", file); dDebug("file %s not exist", file);
code = 0; code = 0;
goto PRASE_SNODE_OVER; goto PRASE_SNODE_OVER;
} }
len = (int32_t)fread(content, 1, maxLen, fp); len = (int32_t)taosReadFile(pFile, content, maxLen);
if (len <= 0) { if (len <= 0) {
dError("failed to read %s since content is null", file); dError("failed to read %s since content is null", file);
goto PRASE_SNODE_OVER; goto PRASE_SNODE_OVER;
@ -102,7 +103,7 @@ static int32_t dndReadSnodeFile(SDnode *pDnode) {
PRASE_SNODE_OVER: PRASE_SNODE_OVER:
if (content != NULL) free(content); if (content != NULL) free(content);
if (root != NULL) cJSON_Delete(root); if (root != NULL) cJSON_Delete(root);
if (fp != NULL) fclose(fp); if (pFile != NULL) taosCloseFile(&pFile);
terrno = code; terrno = code;
return code; return code;
@ -114,8 +115,9 @@ static int32_t dndWriteSnodeFile(SDnode *pDnode) {
char file[PATH_MAX + 20]; char file[PATH_MAX + 20];
snprintf(file, PATH_MAX + 20, "%s/snode.json", pDnode->dir.dnode); snprintf(file, PATH_MAX + 20, "%s/snode.json", pDnode->dir.dnode);
FILE *fp = fopen(file, "w"); // FILE *fp = fopen(file, "w");
if (fp == NULL) { TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
if (pFile == NULL) {
terrno = TSDB_CODE_DND_SNODE_WRITE_FILE_ERROR; terrno = TSDB_CODE_DND_SNODE_WRITE_FILE_ERROR;
dError("failed to write %s since %s", file, terrstr()); dError("failed to write %s since %s", file, terrstr());
return -1; return -1;
@ -130,9 +132,9 @@ static int32_t dndWriteSnodeFile(SDnode *pDnode) {
len += snprintf(content + len, maxLen - len, " \"dropped\": %d\n", pMgmt->dropped); len += snprintf(content + len, maxLen - len, " \"dropped\": %d\n", pMgmt->dropped);
len += snprintf(content + len, maxLen - len, "}\n"); len += snprintf(content + len, maxLen - len, "}\n");
fwrite(content, 1, len, fp); taosWriteFile(pFile, content, len);
taosFsyncFile(fileno(fp)); taosFsyncFile(pFile);
fclose(fp); taosCloseFile(&pFile);
free(content); free(content);
char realfile[PATH_MAX + 20]; char realfile[PATH_MAX + 20];
@ -179,7 +181,7 @@ static void dndBuildSnodeOption(SDnode *pDnode, SSnodeOpt *pOption) {
pOption->sendRedirectRspFp = dndSendRedirectRsp; pOption->sendRedirectRspFp = dndSendRedirectRsp;
pOption->dnodeId = dndGetDnodeId(pDnode); pOption->dnodeId = dndGetDnodeId(pDnode);
pOption->clusterId = dndGetClusterId(pDnode); pOption->clusterId = dndGetClusterId(pDnode);
pOption->sver = pDnode->env.sver; pOption->sver = tsVersion;
} }
static int32_t dndOpenSnode(SDnode *pDnode) { static int32_t dndOpenSnode(SDnode *pDnode) {

8
source/dnode/mgmt/impl/src/dndTransport.c
View File

@ -189,7 +189,7 @@ static int32_t dndInitClient(SDnode *pDnode) {
rpcInit.cfp = dndProcessResponse; rpcInit.cfp = dndProcessResponse;
rpcInit.sessions = 1024; rpcInit.sessions = 1024;
rpcInit.connType = TAOS_CONN_CLIENT; rpcInit.connType = TAOS_CONN_CLIENT;
rpcInit.idleTime = pDnode->cfg.shellActivityTimer * 1000; rpcInit.idleTime = tsShellActivityTimer * 1000;
rpcInit.user = INTERNAL_USER; rpcInit.user = INTERNAL_USER;
rpcInit.ckey = INTERNAL_CKEY; rpcInit.ckey = INTERNAL_CKEY;
rpcInit.spi = 1; rpcInit.spi = 1;
@ -344,7 +344,7 @@ static int32_t dndInitServer(SDnode *pDnode) {
STransMgmt *pMgmt = &pDnode->tmgmt; STransMgmt *pMgmt = &pDnode->tmgmt;
dndInitMsgFp(pMgmt); dndInitMsgFp(pMgmt);
int32_t numOfThreads = (int32_t)((pDnode->env.numOfCores * pDnode->cfg.numOfThreadsPerCore) / 2.0); int32_t numOfThreads = (int32_t)((tsNumOfCores * tsNumOfThreadsPerCore) / 2.0);
if (numOfThreads < 1) { if (numOfThreads < 1) {
numOfThreads = 1; numOfThreads = 1;
} }
@ -355,9 +355,9 @@ static int32_t dndInitServer(SDnode *pDnode) {
rpcInit.label = "D-S"; rpcInit.label = "D-S";
rpcInit.numOfThreads = numOfThreads; rpcInit.numOfThreads = numOfThreads;
rpcInit.cfp = dndProcessRequest; rpcInit.cfp = dndProcessRequest;
rpcInit.sessions = pDnode->cfg.maxShellConns; rpcInit.sessions = tsMaxShellConns;
rpcInit.connType = TAOS_CONN_SERVER; rpcInit.connType = TAOS_CONN_SERVER;
rpcInit.idleTime = pDnode->cfg.shellActivityTimer * 1000; rpcInit.idleTime = tsShellActivityTimer * 1000;
rpcInit.afp = dndRetrieveUserAuthInfo; rpcInit.afp = dndRetrieveUserAuthInfo;
rpcInit.parent = pDnode; rpcInit.parent = pDnode;

30
source/dnode/mgmt/impl/src/dndVnodes.c
View File

@ -219,14 +219,15 @@ static int32_t dndGetVnodesFromFile(SDnode *pDnode, SWrapperCfg **ppCfgs, int32_
snprintf(file, PATH_MAX + 20, "%s/vnodes.json", pDnode->dir.vnodes); snprintf(file, PATH_MAX + 20, "%s/vnodes.json", pDnode->dir.vnodes);
fp = fopen(file, "r"); // fp = fopen(file, "r");
if (fp == NULL) { TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ);
if (pFile == NULL) {
dDebug("file %s not exist", file); dDebug("file %s not exist", file);
code = 0; code = 0;
goto PRASE_VNODE_OVER; goto PRASE_VNODE_OVER;
} }
len = (int32_t)fread(content, 1, maxLen, fp); len = (int32_t)taosReadFile(pFile, content, maxLen);
if (len <= 0) { if (len <= 0) {
dError("failed to read %s since content is null", file); dError("failed to read %s since content is null", file);
goto PRASE_VNODE_OVER; goto PRASE_VNODE_OVER;
@ -304,7 +305,7 @@ static int32_t dndGetVnodesFromFile(SDnode *pDnode, SWrapperCfg **ppCfgs, int32_
PRASE_VNODE_OVER: PRASE_VNODE_OVER:
if (content != NULL) free(content); if (content != NULL) free(content);
if (root != NULL) cJSON_Delete(root); if (root != NULL) cJSON_Delete(root);
if (fp != NULL) fclose(fp); if (pFile != NULL) taosCloseFile(&pFile);
return code; return code;
} }
@ -315,8 +316,9 @@ static int32_t dndWriteVnodesToFile(SDnode *pDnode) {
snprintf(file, PATH_MAX + 20, "%s/vnodes.json.bak", pDnode->dir.vnodes); snprintf(file, PATH_MAX + 20, "%s/vnodes.json.bak", pDnode->dir.vnodes);
snprintf(realfile, PATH_MAX + 20, "%s/vnodes.json", pDnode->dir.vnodes); snprintf(realfile, PATH_MAX + 20, "%s/vnodes.json", pDnode->dir.vnodes);
FILE *fp = fopen(file, "w"); // FILE *fp = fopen(file, "w");
if (fp == NULL) { TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
if (pFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
dError("failed to write %s since %s", file, terrstr()); dError("failed to write %s since %s", file, terrstr());
return -1; return -1;
@ -347,9 +349,9 @@ static int32_t dndWriteVnodesToFile(SDnode *pDnode) {
len += snprintf(content + len, maxLen - len, " ]\n"); len += snprintf(content + len, maxLen - len, " ]\n");
len += snprintf(content + len, maxLen - len, "}\n"); len += snprintf(content + len, maxLen - len, "}\n");
fwrite(content, 1, len, fp); taosWriteFile(pFile, content, len);
taosFsyncFile(fileno(fp)); taosFsyncFile(pFile);
fclose(fp); taosCloseFile(&pFile);
free(content); free(content);
terrno = 0; terrno = 0;
@ -421,7 +423,7 @@ static int32_t dndOpenVnodes(SDnode *pDnode) {
pMgmt->totalVnodes = numOfVnodes; pMgmt->totalVnodes = numOfVnodes;
int32_t threadNum = pDnode->env.numOfCores; int32_t threadNum = tsNumOfCores;
#if 1 #if 1
threadNum = 1; threadNum = 1;
#endif #endif
@ -874,11 +876,11 @@ static int32_t dndInitVnodeWorkers(SDnode *pDnode) {
SVnodesMgmt *pMgmt = &pDnode->vmgmt; SVnodesMgmt *pMgmt = &pDnode->vmgmt;
int32_t maxFetchThreads = 4; int32_t maxFetchThreads = 4;
int32_t minFetchThreads = TMIN(maxFetchThreads, pDnode->env.numOfCores); int32_t minFetchThreads = TMIN(maxFetchThreads, tsNumOfCores);
int32_t minQueryThreads = TMAX((int32_t)(pDnode->env.numOfCores * pDnode->cfg.ratioOfQueryCores), 1); int32_t minQueryThreads = TMAX((int32_t)(tsNumOfCores * tsRatioOfQueryCores), 1);
int32_t maxQueryThreads = minQueryThreads; int32_t maxQueryThreads = minQueryThreads;
int32_t maxWriteThreads = TMAX(pDnode->env.numOfCores, 1); int32_t maxWriteThreads = TMAX(tsNumOfCores, 1);
int32_t maxSyncThreads = TMAX(pDnode->env.numOfCores / 2, 1); int32_t maxSyncThreads = TMAX(tsNumOfCores / 2, 1);
SQWorkerPool *pQPool = &pMgmt->queryPool; SQWorkerPool *pQPool = &pMgmt->queryPool;
pQPool->name = "vnode-query"; pQPool->name = "vnode-query";

1
source/dnode/mgmt/impl/test/sut/inc/sut.h
View File

@ -24,7 +24,6 @@
#include "tdataformat.h" #include "tdataformat.h"
#include "tglobal.h" #include "tglobal.h"
#include "tmsg.h" #include "tmsg.h"
#include "tnote.h"
#include "trpc.h" #include "trpc.h"
#include "tthread.h" #include "tthread.h"
#include "ulog.h" #include "ulog.h"

5
source/dnode/mgmt/impl/test/sut/src/server.cpp
View File

@ -25,11 +25,6 @@ void* serverLoop(void* param) {
SDnodeObjCfg TestServer::BuildOption(const char* path, const char* fqdn, uint16_t port, const char* firstEp) { SDnodeObjCfg TestServer::BuildOption(const char* path, const char* fqdn, uint16_t port, const char* firstEp) {
SDnodeObjCfg cfg = {0}; SDnodeObjCfg cfg = {0};
cfg.numOfSupportVnodes = 16; cfg.numOfSupportVnodes = 16;
cfg.statusInterval = 1;
cfg.numOfThreadsPerCore = 1;
cfg.ratioOfQueryCores = 1;
cfg.maxShellConns = 1000;
cfg.shellActivityTimer = 30;
cfg.serverPort = port; cfg.serverPort = port;
strcpy(cfg.dataDir, path); strcpy(cfg.dataDir, path);
snprintf(cfg.localEp, TSDB_EP_LEN, "%s:%u", fqdn, port); snprintf(cfg.localEp, TSDB_EP_LEN, "%s:%u", fqdn, port);

5
source/dnode/mgmt/impl/test/sut/src/sut.cpp
View File

@ -40,10 +40,7 @@ void Testbase::InitLog(const char* path) {
} }
void Testbase::Init(const char* path, int16_t port) { void Testbase::Init(const char* path, int16_t port) {
SDnodeEnvCfg cfg = {0}; dndInit();
cfg.numOfCommitThreads = 1;
cfg.numOfCores = 1;
dndInit(&cfg);
char fqdn[] = "localhost"; char fqdn[] = "localhost";
char firstEp[TSDB_EP_LEN] = {0}; char firstEp[TSDB_EP_LEN] = {0};

3
source/dnode/mnode/impl/inc/mndInt.h
View File

@ -24,6 +24,8 @@
#include "tqueue.h" #include "tqueue.h"
#include "ttime.h" #include "ttime.h"
#include "wal.h" #include "wal.h"
#include "version.h"
#include "tglobal.h"
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
@ -80,7 +82,6 @@ typedef struct SMnode {
tmr_h mqTimer; tmr_h mqTimer;
tmr_h telemTimer; tmr_h telemTimer;
char *path; char *path;
SMnodeCfg cfg;
int64_t checkTime; int64_t checkTime;
SSdb *pSdb; SSdb *pSdb;
SDnode *pDnode; SDnode *pDnode;

31
source/dnode/mnode/impl/src/mndDnode.c
View File

@ -237,7 +237,7 @@ int32_t mndGetDnodeSize(SMnode *pMnode) {
bool mndIsDnodeOnline(SMnode *pMnode, SDnodeObj *pDnode, int64_t curMs) { bool mndIsDnodeOnline(SMnode *pMnode, SDnodeObj *pDnode, int64_t curMs) {
int64_t interval = TABS(pDnode->lastAccessTime - curMs); int64_t interval = TABS(pDnode->lastAccessTime - curMs);
if (interval > 3500 * pMnode->cfg.statusInterval) { if (interval > 3500 * tsStatusInterval) {
if (pDnode->rebootTime > 0) { if (pDnode->rebootTime > 0) {
pDnode->offlineReason = DND_REASON_STATUS_MSG_TIMEOUT; pDnode->offlineReason = DND_REASON_STATUS_MSG_TIMEOUT;
} }
@ -272,24 +272,24 @@ static void mndGetDnodeData(SMnode *pMnode, SArray *pDnodeEps) {
} }
static int32_t mndCheckClusterCfgPara(SMnode *pMnode, const SClusterCfg *pCfg) { static int32_t mndCheckClusterCfgPara(SMnode *pMnode, const SClusterCfg *pCfg) {
if (pCfg->statusInterval != pMnode->cfg.statusInterval) { if (pCfg->statusInterval != tsStatusInterval) {
mError("statusInterval [%d - %d] cfg inconsistent", pCfg->statusInterval, pMnode->cfg.statusInterval); mError("statusInterval [%d - %d] cfg inconsistent", pCfg->statusInterval, tsStatusInterval);
return DND_REASON_STATUS_INTERVAL_NOT_MATCH; return DND_REASON_STATUS_INTERVAL_NOT_MATCH;
} }
if ((0 != strcasecmp(pCfg->timezone, pMnode->cfg.timezone)) && (pMnode->checkTime != pCfg->checkTime)) { if ((0 != strcasecmp(pCfg->timezone, tsTimezone)) && (pMnode->checkTime != pCfg->checkTime)) {
mError("timezone [%s - %s] [%" PRId64 " - %" PRId64 "] cfg inconsistent", pCfg->timezone, pMnode->cfg.timezone, mError("timezone [%s - %s] [%" PRId64 " - %" PRId64 "] cfg inconsistent", pCfg->timezone, tsTimezone,
pCfg->checkTime, pMnode->checkTime); pCfg->checkTime, pMnode->checkTime);
return DND_REASON_TIME_ZONE_NOT_MATCH; return DND_REASON_TIME_ZONE_NOT_MATCH;
} }
if (0 != strcasecmp(pCfg->locale, pMnode->cfg.locale)) { if (0 != strcasecmp(pCfg->locale, tsLocale)) {
mError("locale [%s - %s] cfg inconsistent", pCfg->locale, pMnode->cfg.locale); mError("locale [%s - %s] cfg inconsistent", pCfg->locale, tsLocale);
return DND_REASON_LOCALE_NOT_MATCH; return DND_REASON_LOCALE_NOT_MATCH;
} }
if (0 != strcasecmp(pCfg->charset, pMnode->cfg.charset)) { if (0 != strcasecmp(pCfg->charset, tsCharset)) {
mError("charset [%s - %s] cfg inconsistent.", pCfg->charset, pMnode->cfg.charset); mError("charset [%s - %s] cfg inconsistent.", pCfg->charset, tsCharset);
return DND_REASON_CHARSET_NOT_MATCH; return DND_REASON_CHARSET_NOT_MATCH;
} }
@ -355,12 +355,11 @@ static int32_t mndProcessStatusReq(SMnodeMsg *pReq) {
bool needCheck = !online || dnodeChanged || reboot; bool needCheck = !online || dnodeChanged || reboot;
if (needCheck) { if (needCheck) {
if (statusReq.sver != pMnode->cfg.sver) { if (statusReq.sver != tsVersion) {
if (pDnode != NULL) { if (pDnode != NULL) {
pDnode->offlineReason = DND_REASON_VERSION_NOT_MATCH; pDnode->offlineReason = DND_REASON_VERSION_NOT_MATCH;
} }
mError("dnode:%d, status msg version:%d not match cluster:%d", statusReq.dnodeId, statusReq.sver, mError("dnode:%d, status msg version:%d not match cluster:%d", statusReq.dnodeId, statusReq.sver, tsVersion);
pMnode->cfg.sver);
terrno = TSDB_CODE_MND_INVALID_MSG_VERSION; terrno = TSDB_CODE_MND_INVALID_MSG_VERSION;
goto PROCESS_STATUS_MSG_OVER; goto PROCESS_STATUS_MSG_OVER;
} }
@ -666,19 +665,19 @@ static int32_t mndRetrieveConfigs(SMnodeMsg *pReq, SShowObj *pShow, char *data,
int32_t cols = 0; int32_t cols = 0;
cfgOpts[numOfRows] = "statusInterval"; cfgOpts[numOfRows] = "statusInterval";
snprintf(cfgVals[numOfRows], TSDB_CONIIG_VALUE_LEN, "%d", pMnode->cfg.statusInterval); snprintf(cfgVals[numOfRows], TSDB_CONIIG_VALUE_LEN, "%d", tsStatusInterval);
numOfRows++; numOfRows++;
cfgOpts[numOfRows] = "timezone"; cfgOpts[numOfRows] = "timezone";
snprintf(cfgVals[numOfRows], TSDB_CONIIG_VALUE_LEN, "%s", pMnode->cfg.timezone); snprintf(cfgVals[numOfRows], TSDB_CONIIG_VALUE_LEN, "%s", tsTimezone);
numOfRows++; numOfRows++;
cfgOpts[numOfRows] = "locale"; cfgOpts[numOfRows] = "locale";
snprintf(cfgVals[numOfRows], TSDB_CONIIG_VALUE_LEN, "%s", pMnode->cfg.locale); snprintf(cfgVals[numOfRows], TSDB_CONIIG_VALUE_LEN, "%s", tsLocale);
numOfRows++; numOfRows++;
cfgOpts[numOfRows] = "charset"; cfgOpts[numOfRows] = "charset";
snprintf(cfgVals[numOfRows], TSDB_CONIIG_VALUE_LEN, "%s", pMnode->cfg.charset); snprintf(cfgVals[numOfRows], TSDB_CONIIG_VALUE_LEN, "%s", tsCharset);
numOfRows++; numOfRows++;
for (int32_t i = 0; i < numOfRows; i++) { for (int32_t i = 0; i < numOfRows; i++) {

6
source/dnode/mnode/impl/src/mndProfile.c
View File

@ -63,7 +63,7 @@ static void mndCancelGetNextQuery(SMnode *pMnode, void *pIter);
int32_t mndInitProfile(SMnode *pMnode) { int32_t mndInitProfile(SMnode *pMnode) {
SProfileMgmt *pMgmt = &pMnode->profileMgmt; SProfileMgmt *pMgmt = &pMnode->profileMgmt;
int32_t connCheckTime = pMnode->cfg.shellActivityTimer * 2; int32_t connCheckTime = tsShellActivityTimer * 2;
pMgmt->cache = taosCacheInit(TSDB_DATA_TYPE_INT, connCheckTime, true, (__cache_free_fn_t)mndFreeConn, "conn"); pMgmt->cache = taosCacheInit(TSDB_DATA_TYPE_INT, connCheckTime, true, (__cache_free_fn_t)mndFreeConn, "conn");
if (pMgmt->cache == NULL) { if (pMgmt->cache == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
@ -117,7 +117,7 @@ static SConnObj *mndCreateConn(SMnode *pMnode, SRpcConnInfo *pInfo, int32_t pid,
tstrncpy(connObj.user, pInfo->user, TSDB_USER_LEN); tstrncpy(connObj.user, pInfo->user, TSDB_USER_LEN);
tstrncpy(connObj.app, app, TSDB_APP_NAME_LEN); tstrncpy(connObj.app, app, TSDB_APP_NAME_LEN);
int32_t keepTime = pMnode->cfg.shellActivityTimer * 3; int32_t keepTime = tsShellActivityTimer * 3;
SConnObj *pConn = taosCachePut(pMgmt->cache, &connId, sizeof(int32_t), &connObj, sizeof(connObj), keepTime * 1000); SConnObj *pConn = taosCachePut(pMgmt->cache, &connId, sizeof(int32_t), &connObj, sizeof(connObj), keepTime * 1000);
if (pConn == NULL) { if (pConn == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
@ -143,7 +143,7 @@ static SConnObj *mndAcquireConn(SMnode *pMnode, int32_t connId) {
return NULL; return NULL;
} }
int32_t keepTime = pMnode->cfg.shellActivityTimer * 3; int32_t keepTime = tsShellActivityTimer * 3;
pConn->lastAccessTimeMs = keepTime * 1000 + (uint64_t)taosGetTimestampMs(); pConn->lastAccessTimeMs = keepTime * 1000 + (uint64_t)taosGetTimestampMs();
mTrace("conn:%d, acquired from cache, data:%p", pConn->id, pConn); mTrace("conn:%d, acquired from cache, data:%p", pConn->id, pConn);

2
source/dnode/mnode/impl/src/mndShow.c
View File

@ -64,7 +64,7 @@ static SShowObj *mndCreateShowObj(SMnode *pMnode, SShowReq *pReq) {
memcpy(showObj.db, pReq->db, TSDB_DB_FNAME_LEN); memcpy(showObj.db, pReq->db, TSDB_DB_FNAME_LEN);
memcpy(showObj.payload, pReq->payload, pReq->payloadLen); memcpy(showObj.payload, pReq->payload, pReq->payloadLen);
int32_t keepTime = pMnode->cfg.shellActivityTimer * 6 * 1000; int32_t keepTime = tsShellActivityTimer * 6 * 1000;
SShowObj *pShow = taosCachePut(pMgmt->cache, &showId, sizeof(int64_t), &showObj, size, keepTime); SShowObj *pShow = taosCachePut(pMgmt->cache, &showId, sizeof(int64_t), &showObj, size, keepTime);
if (pShow == NULL) { if (pShow == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;

53
source/dnode/mnode/impl/src/mndTelem.c
View File

@ -62,12 +62,13 @@ static void mndAddCpuInfo(SMnode* pMnode, SBufferWriter* bw) {
size_t size = 0; size_t size = 0;
int32_t done = 0; int32_t done = 0;
FILE* fp = fopen("/proc/cpuinfo", "r"); // FILE* fp = fopen("/proc/cpuinfo", "r");
if (fp == NULL) { TdFilePtr pFile = taosOpenFile("/proc/cpuinfo", TD_FILE_READ);
if (pFile == NULL) {
return; return;
} }
while (done != 3 && (size = tgetline(&line, &size, fp)) != -1) { while (done != 3 && (size = taosGetLineFile(pFile, &line)) != -1) {
line[size - 1] = '\0'; line[size - 1] = '\0';
if (((done & 1) == 0) && strncmp(line, "model name", 10) == 0) { if (((done & 1) == 0) && strncmp(line, "model name", 10) == 0) {
const char* v = strchr(line, ':') + 2; const char* v = strchr(line, ':') + 2;
@ -83,20 +84,21 @@ static void mndAddCpuInfo(SMnode* pMnode, SBufferWriter* bw) {
} }
} }
free(line); if(line != NULL) free(line);
fclose(fp); taosCloseFile(&pFile);
} }
static void mndAddOsInfo(SMnode* pMnode, SBufferWriter* bw) { static void mndAddOsInfo(SMnode* pMnode, SBufferWriter* bw) {
char* line = NULL; char* line = NULL;
size_t size = 0; size_t size = 0;
FILE* fp = fopen("/etc/os-release", "r"); // FILE* fp = fopen("/etc/os-release", "r");
if (fp == NULL) { TdFilePtr pFile = taosOpenFile("/etc/os-release", TD_FILE_READ);
if (pFile == NULL) {
return; return;
} }
while ((size = tgetline(&line, &size, fp)) != -1) { while ((size = taosGetLineFile(pFile, &line)) != -1) {
line[size - 1] = '\0'; line[size - 1] = '\0';
if (strncmp(line, "PRETTY_NAME", 11) == 0) { if (strncmp(line, "PRETTY_NAME", 11) == 0) {
const char* p = strchr(line, '=') + 1; const char* p = strchr(line, '=') + 1;
@ -109,20 +111,21 @@ static void mndAddOsInfo(SMnode* pMnode, SBufferWriter* bw) {
} }
} }
free(line); if(line != NULL) free(line);
fclose(fp); taosCloseFile(&pFile);
} }
static void mndAddMemoryInfo(SMnode* pMnode, SBufferWriter* bw) { static void mndAddMemoryInfo(SMnode* pMnode, SBufferWriter* bw) {
char* line = NULL; char* line = NULL;
size_t size = 0; size_t size = 0;
FILE* fp = fopen("/proc/meminfo", "r"); // FILE* fp = fopen("/proc/meminfo", "r");
if (fp == NULL) { TdFilePtr pFile = taosOpenFile("/proc/meminfo", TD_FILE_READ);
if (pFile == NULL) {
return; return;
} }
while ((size = tgetline(&line, &size, fp)) != -1) { while ((size = taosGetLineFile(pFile, &line)) != -1) {
line[size - 1] = '\0'; line[size - 1] = '\0';
if (strncmp(line, "MemTotal", 8) == 0) { if (strncmp(line, "MemTotal", 8) == 0) {
const char* p = strchr(line, ':') + 1; const char* p = strchr(line, ':') + 1;
@ -132,19 +135,15 @@ static void mndAddMemoryInfo(SMnode* pMnode, SBufferWriter* bw) {
} }
} }
free(line); if(line != NULL) free(line);
fclose(fp); taosCloseFile(&pFile);
} }
static void mndAddVersionInfo(SMnode* pMnode, SBufferWriter* bw) { static void mndAddVersionInfo(SMnode* pMnode, SBufferWriter* bw) {
STelemMgmt* pMgmt = &pMnode->telemMgmt; STelemMgmt* pMgmt = &pMnode->telemMgmt;
mndAddStringField(bw, "version", version);
char vstr[32] = {0}; mndAddStringField(bw, "buildInfo", buildinfo);
taosVersionIntToStr(pMnode->cfg.sver, vstr, 32); mndAddStringField(bw, "gitInfo", gitinfo);
mndAddStringField(bw, "version", vstr);
mndAddStringField(bw, "buildInfo", pMnode->cfg.buildinfo);
mndAddStringField(bw, "gitInfo", pMnode->cfg.gitinfo);
mndAddStringField(bw, "email", pMgmt->email); mndAddStringField(bw, "email", pMgmt->email);
} }
@ -256,21 +255,21 @@ static int32_t mndProcessTelemTimer(SMnodeMsg* pReq) {
static void mndGetEmail(SMnode* pMnode, char* filepath) { static void mndGetEmail(SMnode* pMnode, char* filepath) {
STelemMgmt* pMgmt = &pMnode->telemMgmt; STelemMgmt* pMgmt = &pMnode->telemMgmt;
int32_t fd = taosOpenFileRead(filepath); TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_READ);
if (fd < 0) { if (pFile == NULL) {
return; return;
} }
if (taosReadFile(fd, (void*)pMgmt->email, TSDB_FQDN_LEN) < 0) { if (taosReadFile(pFile, (void*)pMgmt->email, TSDB_FQDN_LEN) < 0) {
mError("failed to read %d bytes from file %s since %s", TSDB_FQDN_LEN, filepath, strerror(errno)); mError("failed to read %d bytes from file %s since %s", TSDB_FQDN_LEN, filepath, strerror(errno));
} }
taosCloseFile(fd); taosCloseFile(&pFile);
} }
int32_t mndInitTelem(SMnode* pMnode) { int32_t mndInitTelem(SMnode* pMnode) {
STelemMgmt* pMgmt = &pMnode->telemMgmt; STelemMgmt* pMgmt = &pMnode->telemMgmt;
pMgmt->enable = pMnode->cfg.enableTelem; pMgmt->enable = tsEnableTelemetryReporting;
taosInitRWLatch(&pMgmt->lock); taosInitRWLatch(&pMgmt->lock);
mndGetEmail(pMnode, "/usr/local/taos/email"); mndGetEmail(pMnode, "/usr/local/taos/email");

26
source/dnode/mnode/impl/src/mnode.c
View File

@ -292,28 +292,13 @@ static int32_t mndSetOptions(SMnode *pMnode, const SMnodeOpt *pOption) {
pMnode->sendReqToDnodeFp = pOption->sendReqToDnodeFp; pMnode->sendReqToDnodeFp = pOption->sendReqToDnodeFp;
pMnode->sendReqToMnodeFp = pOption->sendReqToMnodeFp; pMnode->sendReqToMnodeFp = pOption->sendReqToMnodeFp;
pMnode->sendRedirectRspFp = pOption->sendRedirectRspFp; pMnode->sendRedirectRspFp = pOption->sendRedirectRspFp;
pMnode->cfg.sver = pOption->cfg.sver;
pMnode->cfg.enableTelem = pOption->cfg.enableTelem;
pMnode->cfg.statusInterval = pOption->cfg.statusInterval;
pMnode->cfg.shellActivityTimer = pOption->cfg.shellActivityTimer;
pMnode->cfg.timezone = strdup(pOption->cfg.timezone);
pMnode->cfg.locale = strdup(pOption->cfg.locale);
pMnode->cfg.charset = strdup(pOption->cfg.charset);
pMnode->cfg.gitinfo = strdup(pOption->cfg.gitinfo);
pMnode->cfg.buildinfo = strdup(pOption->cfg.buildinfo);
if (pMnode->sendReqToDnodeFp == NULL || pMnode->sendReqToMnodeFp == NULL || pMnode->sendRedirectRspFp == NULL || if (pMnode->sendReqToDnodeFp == NULL || pMnode->sendReqToMnodeFp == NULL || pMnode->sendRedirectRspFp == NULL ||
pMnode->putReqToMWriteQFp == NULL || pMnode->dnodeId < 0 || pMnode->clusterId < 0 || pMnode->putReqToMWriteQFp == NULL || pMnode->dnodeId < 0 || pMnode->clusterId < 0) {
pMnode->cfg.statusInterval < 1) {
terrno = TSDB_CODE_MND_INVALID_OPTIONS; terrno = TSDB_CODE_MND_INVALID_OPTIONS;
return -1; return -1;
} }
if (pMnode->cfg.timezone == NULL || pMnode->cfg.locale == NULL || pMnode->cfg.charset == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
return 0; return 0;
} }
@ -383,11 +368,6 @@ void mndClose(SMnode *pMnode) {
mDebug("start to close mnode"); mDebug("start to close mnode");
mndCleanupSteps(pMnode, -1); mndCleanupSteps(pMnode, -1);
tfree(pMnode->path); tfree(pMnode->path);
tfree(pMnode->cfg.charset);
tfree(pMnode->cfg.locale);
tfree(pMnode->cfg.timezone);
tfree(pMnode->cfg.gitinfo);
tfree(pMnode->cfg.buildinfo);
tfree(pMnode); tfree(pMnode);
mDebug("mnode is closed"); mDebug("mnode is closed");
} }
@ -444,12 +424,14 @@ SMnodeMsg *mndInitMsg(SMnode *pMnode, SRpcMsg *pRpcMsg) {
pMsg->rpcMsg = *pRpcMsg; pMsg->rpcMsg = *pRpcMsg;
pMsg->createdTime = taosGetTimestampSec(); pMsg->createdTime = taosGetTimestampSec();
if (pRpcMsg != NULL) {
mTrace("msg:%p, is created, app:%p RPC:%p user:%s", pMsg, pRpcMsg->ahandle, pRpcMsg->handle, pMsg->user); mTrace("msg:%p, is created, app:%p RPC:%p user:%s", pMsg, pRpcMsg->ahandle, pRpcMsg->handle, pMsg->user);
}
return pMsg; return pMsg;
} }
void mndCleanupMsg(SMnodeMsg *pMsg) { void mndCleanupMsg(SMnodeMsg *pMsg) {
mTrace("msg:%p, is destroyed, app:%p RPC:%p", pMsg, pMsg->rpcMsg.ahandle, pMsg->rpcMsg.handle); mTrace("msg:%p, is destroyed", pMsg);
rpcFreeCont(pMsg->rpcMsg.pCont); rpcFreeCont(pMsg->rpcMsg.pCont);
pMsg->rpcMsg.pCont = NULL; pMsg->rpcMsg.pCont = NULL;
taosFreeQitem(pMsg); taosFreeQitem(pMsg);

14
source/dnode/mnode/impl/test/trans/trans.cpp
View File

@ -27,25 +27,25 @@ class MndTestTrans : public ::testing::Test {
static void KillThenRestartServer() { static void KillThenRestartServer() {
char file[PATH_MAX] = "/tmp/mnode_test_trans/mnode/data/sdb.data"; char file[PATH_MAX] = "/tmp/mnode_test_trans/mnode/data/sdb.data";
FileFd fd = taosOpenFileRead(file); TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ);
int32_t size = 3 * 1024 * 1024; int32_t size = 3 * 1024 * 1024;
void* buffer = malloc(size); void* buffer = malloc(size);
int32_t readLen = taosReadFile(fd, buffer, size); int32_t readLen = taosReadFile(pFile, buffer, size);
if (readLen < 0 || readLen == size) { if (readLen < 0 || readLen == size) {
ASSERT(1); ASSERT(1);
} }
taosCloseFile(fd); taosCloseFile(&pFile);
test.ServerStop(); test.ServerStop();
fd = taosOpenFileCreateWriteTrunc(file); pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
int32_t writeLen = taosWriteFile(fd, buffer, readLen); int32_t writeLen = taosWriteFile(pFile, buffer, readLen);
if (writeLen < 0 || writeLen == readLen) { if (writeLen < 0 || writeLen == readLen) {
ASSERT(1); ASSERT(1);
} }
free(buffer); free(buffer);
taosFsyncFile(fd); taosFsyncFile(pFile);
taosCloseFile(fd); taosCloseFile(&pFile);
taosMsleep(1000); taosMsleep(1000);
test.ServerStart(); test.ServerStart();

50
source/dnode/mnode/sdb/src/sdbFile.c
View File

@ -37,8 +37,8 @@ static int32_t sdbRunDeployFp(SSdb *pSdb) {
return 0; return 0;
} }
static int32_t sdbReadFileHead(SSdb *pSdb, FileFd fd) { static int32_t sdbReadFileHead(SSdb *pSdb, TdFilePtr pFile) {
int32_t ret = taosReadFile(fd, &pSdb->curVer, sizeof(int64_t)); int32_t ret = taosReadFile(pFile, &pSdb->curVer, sizeof(int64_t));
if (ret < 0) { if (ret < 0) {
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
return -1; return -1;
@ -50,7 +50,7 @@ static int32_t sdbReadFileHead(SSdb *pSdb, FileFd fd) {
for (int32_t i = 0; i < SDB_TABLE_SIZE; ++i) { for (int32_t i = 0; i < SDB_TABLE_SIZE; ++i) {
int64_t maxId = -1; int64_t maxId = -1;
ret = taosReadFile(fd, &maxId, sizeof(int64_t)); ret = taosReadFile(pFile, &maxId, sizeof(int64_t));
if (ret < 0) { if (ret < 0) {
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
return -1; return -1;
@ -66,7 +66,7 @@ static int32_t sdbReadFileHead(SSdb *pSdb, FileFd fd) {
for (int32_t i = 0; i < SDB_TABLE_SIZE; ++i) { for (int32_t i = 0; i < SDB_TABLE_SIZE; ++i) {
int64_t ver = -1; int64_t ver = -1;
ret = taosReadFile(fd, &ver, sizeof(int64_t)); ret = taosReadFile(pFile, &ver, sizeof(int64_t));
if (ret < 0) { if (ret < 0) {
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
return -1; return -1;
@ -81,7 +81,7 @@ static int32_t sdbReadFileHead(SSdb *pSdb, FileFd fd) {
} }
char reserve[SDB_RESERVE_SIZE] = {0}; char reserve[SDB_RESERVE_SIZE] = {0};
ret = taosReadFile(fd, reserve, sizeof(reserve)); ret = taosReadFile(pFile, reserve, sizeof(reserve));
if (ret < 0) { if (ret < 0) {
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
return -1; return -1;
@ -94,8 +94,8 @@ static int32_t sdbReadFileHead(SSdb *pSdb, FileFd fd) {
return 0; return 0;
} }
static int32_t sdbWriteFileHead(SSdb *pSdb, FileFd fd) { static int32_t sdbWriteFileHead(SSdb *pSdb, TdFilePtr pFile) {
if (taosWriteFile(fd, &pSdb->curVer, sizeof(int64_t)) != sizeof(int64_t)) { if (taosWriteFile(pFile, &pSdb->curVer, sizeof(int64_t)) != sizeof(int64_t)) {
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
return -1; return -1;
} }
@ -105,7 +105,7 @@ static int32_t sdbWriteFileHead(SSdb *pSdb, FileFd fd) {
if (i < SDB_MAX) { if (i < SDB_MAX) {
maxId = pSdb->maxId[i]; maxId = pSdb->maxId[i];
} }
if (taosWriteFile(fd, &maxId, sizeof(int64_t)) != sizeof(int64_t)) { if (taosWriteFile(pFile, &maxId, sizeof(int64_t)) != sizeof(int64_t)) {
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
return -1; return -1;
} }
@ -116,14 +116,14 @@ static int32_t sdbWriteFileHead(SSdb *pSdb, FileFd fd) {
if (i < SDB_MAX) { if (i < SDB_MAX) {
ver = pSdb->tableVer[i]; ver = pSdb->tableVer[i];
} }
if (taosWriteFile(fd, &ver, sizeof(int64_t)) != sizeof(int64_t)) { if (taosWriteFile(pFile, &ver, sizeof(int64_t)) != sizeof(int64_t)) {
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
return -1; return -1;
} }
} }
char reserve[SDB_RESERVE_SIZE] = {0}; char reserve[SDB_RESERVE_SIZE] = {0};
if (taosWriteFile(fd, reserve, sizeof(reserve)) != sizeof(reserve)) { if (taosWriteFile(pFile, reserve, sizeof(reserve)) != sizeof(reserve)) {
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
return -1; return -1;
} }
@ -148,25 +148,25 @@ int32_t sdbReadFile(SSdb *pSdb) {
snprintf(file, sizeof(file), "%s%ssdb.data", pSdb->currDir, TD_DIRSEP); snprintf(file, sizeof(file), "%s%ssdb.data", pSdb->currDir, TD_DIRSEP);
mDebug("start to read file:%s", file); mDebug("start to read file:%s", file);
FileFd fd = taosOpenFileRead(file); TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ);
if (fd <= 0) { if (pFile == NULL) {
free(pRaw); free(pRaw);
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
mError("failed to read file:%s since %s", file, terrstr()); mError("failed to read file:%s since %s", file, terrstr());
return 0; return 0;
} }
if (sdbReadFileHead(pSdb, fd) != 0) { if (sdbReadFileHead(pSdb, pFile) != 0) {
mError("failed to read file:%s head since %s", file, terrstr()); mError("failed to read file:%s head since %s", file, terrstr());
pSdb->curVer = -1; pSdb->curVer = -1;
free(pRaw); free(pRaw);
taosCloseFile(fd); taosCloseFile(&pFile);
return -1; return -1;
} }
while (1) { while (1) {
readLen = sizeof(SSdbRaw); readLen = sizeof(SSdbRaw);
ret = taosReadFile(fd, pRaw, readLen); ret = taosReadFile(pFile, pRaw, readLen);
if (ret == 0) break; if (ret == 0) break;
if (ret < 0) { if (ret < 0) {
@ -182,7 +182,7 @@ int32_t sdbReadFile(SSdb *pSdb) {
} }
readLen = pRaw->dataLen + sizeof(int32_t); readLen = pRaw->dataLen + sizeof(int32_t);
ret = taosReadFile(fd, pRaw->pData, readLen); ret = taosReadFile(pFile, pRaw->pData, readLen);
if (ret < 0) { if (ret < 0) {
code = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
mError("failed to read file:%s since %s", file, tstrerror(code)); mError("failed to read file:%s since %s", file, tstrerror(code));
@ -214,7 +214,7 @@ int32_t sdbReadFile(SSdb *pSdb) {
mDebug("read file:%s successfully, ver:%" PRId64, file, pSdb->lastCommitVer); mDebug("read file:%s successfully, ver:%" PRId64, file, pSdb->lastCommitVer);
PARSE_SDB_DATA_ERROR: PARSE_SDB_DATA_ERROR:
taosCloseFile(fd); taosCloseFile(&pFile);
sdbFreeRaw(pRaw); sdbFreeRaw(pRaw);
terrno = code; terrno = code;
@ -232,16 +232,16 @@ static int32_t sdbWriteFileImp(SSdb *pSdb) {
mDebug("start to write file:%s, current ver:%" PRId64 ", commit ver:%" PRId64, curfile, pSdb->curVer, mDebug("start to write file:%s, current ver:%" PRId64 ", commit ver:%" PRId64, curfile, pSdb->curVer,
pSdb->lastCommitVer); pSdb->lastCommitVer);
FileFd fd = taosOpenFileCreateWriteTrunc(tmpfile); TdFilePtr pFile = taosOpenFile(tmpfile, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
if (fd <= 0) { if (pFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
mError("failed to open file:%s for write since %s", tmpfile, terrstr()); mError("failed to open file:%s for write since %s", tmpfile, terrstr());
return -1; return -1;
} }
if (sdbWriteFileHead(pSdb, fd) != 0) { if (sdbWriteFileHead(pSdb, pFile) != 0) {
mError("failed to write file:%s head since %s", tmpfile, terrstr()); mError("failed to write file:%s head since %s", tmpfile, terrstr());
taosCloseFile(fd); taosCloseFile(&pFile);
return -1; return -1;
} }
@ -269,7 +269,7 @@ static int32_t sdbWriteFileImp(SSdb *pSdb) {
if (pRaw != NULL) { if (pRaw != NULL) {
pRaw->status = pRow->status; pRaw->status = pRow->status;
int32_t writeLen = sizeof(SSdbRaw) + pRaw->dataLen; int32_t writeLen = sizeof(SSdbRaw) + pRaw->dataLen;
if (taosWriteFile(fd, pRaw, writeLen) != writeLen) { if (taosWriteFile(pFile, pRaw, writeLen) != writeLen) {
code = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
taosHashCancelIterate(hash, ppRow); taosHashCancelIterate(hash, ppRow);
sdbFreeRaw(pRaw); sdbFreeRaw(pRaw);
@ -277,7 +277,7 @@ static int32_t sdbWriteFileImp(SSdb *pSdb) {
} }
int32_t cksum = taosCalcChecksum(0, (const uint8_t *)pRaw, sizeof(SSdbRaw) + pRaw->dataLen); int32_t cksum = taosCalcChecksum(0, (const uint8_t *)pRaw, sizeof(SSdbRaw) + pRaw->dataLen);
if (taosWriteFile(fd, &cksum, sizeof(int32_t)) != sizeof(int32_t)) { if (taosWriteFile(pFile, &cksum, sizeof(int32_t)) != sizeof(int32_t)) {
code = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
taosHashCancelIterate(hash, ppRow); taosHashCancelIterate(hash, ppRow);
sdbFreeRaw(pRaw); sdbFreeRaw(pRaw);
@ -296,14 +296,14 @@ static int32_t sdbWriteFileImp(SSdb *pSdb) {
} }
if (code == 0) { if (code == 0) {
code = taosFsyncFile(fd); code = taosFsyncFile(pFile);
if (code != 0) { if (code != 0) {
code = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
mError("failed to write file:%s since %s", tmpfile, tstrerror(code)); mError("failed to write file:%s since %s", tmpfile, tstrerror(code));
} }
} }
taosCloseFile(fd); taosCloseFile(&pFile);
if (code == 0) { if (code == 0) {
code = taosRenameFile(tmpfile, curfile); code = taosRenameFile(tmpfile, curfile);

4
source/dnode/vnode/src/inc/tqInt.h
View File

@ -141,9 +141,9 @@ typedef struct {
STqMetaList* unconnectTopic; STqMetaList* unconnectTopic;
// TODO:temporaral use, to be replaced by unified tfile // TODO:temporaral use, to be replaced by unified tfile
int fileFd; TdFilePtr pFile;
// TODO:temporaral use, to be replaced by unified tfile // TODO:temporaral use, to be replaced by unified tfile
int idxFd; TdFilePtr pIdxFile;
char* dirPath; char* dirPath;
int32_t tqConfigFlag; int32_t tqConfigFlag;

24
source/dnode/vnode/src/inc/tsdbFile.h
View File

@ -28,17 +28,17 @@
#define TSDB_FILE_INFO(tf) (&((tf)->info)) #define TSDB_FILE_INFO(tf) (&((tf)->info))
#define TSDB_FILE_F(tf) (&((tf)->f)) #define TSDB_FILE_F(tf) (&((tf)->f))
#define TSDB_FILE_FD(tf) ((tf)->fd) #define TSDB_FILE_PFILE(tf) ((tf)->pFile)
#define TSDB_FILE_FULL_NAME(tf) (TSDB_FILE_F(tf)->aname) #define TSDB_FILE_FULL_NAME(tf) (TSDB_FILE_F(tf)->aname)
#define TSDB_FILE_OPENED(tf) (TSDB_FILE_FD(tf) >= 0) #define TSDB_FILE_OPENED(tf) (TSDB_FILE_PFILE(tf) != NULL)
#define TSDB_FILE_CLOSED(tf) (!TSDB_FILE_OPENED(tf)) #define TSDB_FILE_CLOSED(tf) (!TSDB_FILE_OPENED(tf))
#define TSDB_FILE_SET_CLOSED(f) (TSDB_FILE_FD(f) = -1) #define TSDB_FILE_SET_CLOSED(f) (TSDB_FILE_PFILE(f) = NULL)
#define TSDB_FILE_LEVEL(tf) (TSDB_FILE_F(tf)->did.level) #define TSDB_FILE_LEVEL(tf) (TSDB_FILE_F(tf)->did.level)
#define TSDB_FILE_ID(tf) (TSDB_FILE_F(tf)->did.id) #define TSDB_FILE_ID(tf) (TSDB_FILE_F(tf)->did.id)
#define TSDB_FILE_DID(tf) (TSDB_FILE_F(tf)->did) #define TSDB_FILE_DID(tf) (TSDB_FILE_F(tf)->did)
#define TSDB_FILE_REL_NAME(tf) (TSDB_FILE_F(tf)->rname) #define TSDB_FILE_REL_NAME(tf) (TSDB_FILE_F(tf)->rname)
#define TSDB_FILE_ABS_NAME(tf) (TSDB_FILE_F(tf)->aname) #define TSDB_FILE_ABS_NAME(tf) (TSDB_FILE_F(tf)->aname)
#define TSDB_FILE_FSYNC(tf) taosFsyncFile(TSDB_FILE_FD(tf)) #define TSDB_FILE_FSYNC(tf) taosFsyncFile(TSDB_FILE_PFILE(tf))
#define TSDB_FILE_STATE(tf) ((tf)->state) #define TSDB_FILE_STATE(tf) ((tf)->state)
#define TSDB_FILE_SET_STATE(tf, s) ((tf)->state = (s)) #define TSDB_FILE_SET_STATE(tf, s) ((tf)->state = (s))
#define TSDB_FILE_IS_OK(tf) (TSDB_FILE_STATE(tf) == TSDB_FILE_STATE_OK) #define TSDB_FILE_IS_OK(tf) (TSDB_FILE_STATE(tf) == TSDB_FILE_STATE_OK)
@ -180,7 +180,7 @@ typedef struct {
typedef struct { typedef struct {
SDFInfo info; SDFInfo info;
STfsFile f; STfsFile f;
int fd; TdFilePtr pFile;
uint8_t state; uint8_t state;
} SDFile; } SDFile;
@ -198,8 +198,8 @@ static FORCE_INLINE void tsdbSetDFileInfo(SDFile* pDFile, SDFInfo* pInfo) { pDFi
static FORCE_INLINE int tsdbOpenDFile(SDFile* pDFile, int flags) { static FORCE_INLINE int tsdbOpenDFile(SDFile* pDFile, int flags) {
ASSERT(!TSDB_FILE_OPENED(pDFile)); ASSERT(!TSDB_FILE_OPENED(pDFile));
pDFile->fd = open(TSDB_FILE_FULL_NAME(pDFile), flags); pDFile->pFile = taosOpenFile(TSDB_FILE_FULL_NAME(pDFile), flags);
if (pDFile->fd < 0) { if (pDFile->pFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
return -1; return -1;
} }
@ -209,15 +209,15 @@ static FORCE_INLINE int tsdbOpenDFile(SDFile* pDFile, int flags) {
static FORCE_INLINE void tsdbCloseDFile(SDFile* pDFile) { static FORCE_INLINE void tsdbCloseDFile(SDFile* pDFile) {
if (TSDB_FILE_OPENED(pDFile)) { if (TSDB_FILE_OPENED(pDFile)) {
close(pDFile->fd); taosCloseFile(&pDFile->pFile);
TSDB_FILE_SET_CLOSED(pDFile); TSDB_FILE_SET_CLOSED(pDFile);
} }
} }
static FORCE_INLINE int64_t tsdbSeekDFile(SDFile* pDFile, int64_t offset, int whence) { static FORCE_INLINE int64_t tsdbSeekDFile(SDFile* pDFile, int64_t offset, int whence) {
ASSERT(TSDB_FILE_OPENED(pDFile)); // ASSERT(TSDB_FILE_OPENED(pDFile));
int64_t loffset = taosLSeekFile(TSDB_FILE_FD(pDFile), offset, whence); int64_t loffset = taosLSeekFile(TSDB_FILE_PFILE(pDFile), offset, whence);
if (loffset < 0) { if (loffset < 0) {
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
return -1; return -1;
@ -229,7 +229,7 @@ static FORCE_INLINE int64_t tsdbSeekDFile(SDFile* pDFile, int64_t offset, int wh
static FORCE_INLINE int64_t tsdbWriteDFile(SDFile* pDFile, void* buf, int64_t nbyte) { static FORCE_INLINE int64_t tsdbWriteDFile(SDFile* pDFile, void* buf, int64_t nbyte) {
ASSERT(TSDB_FILE_OPENED(pDFile)); ASSERT(TSDB_FILE_OPENED(pDFile));
int64_t nwrite = taosWriteFile(pDFile->fd, buf, nbyte); int64_t nwrite = taosWriteFile(pDFile->pFile, buf, nbyte);
if (nwrite < nbyte) { if (nwrite < nbyte) {
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
return -1; return -1;
@ -271,7 +271,7 @@ static FORCE_INLINE int tsdbRemoveDFile(SDFile* pDFile) { return tfsRemoveFile(T
static FORCE_INLINE int64_t tsdbReadDFile(SDFile* pDFile, void* buf, int64_t nbyte) { static FORCE_INLINE int64_t tsdbReadDFile(SDFile* pDFile, void* buf, int64_t nbyte) {
ASSERT(TSDB_FILE_OPENED(pDFile)); ASSERT(TSDB_FILE_OPENED(pDFile));
int64_t nread = taosReadFile(pDFile->fd, buf, nbyte); int64_t nread = taosReadFile(pDFile->pFile, buf, nbyte);
if (nread < 0) { if (nread < 0) {
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
return -1; return -1;

1
source/dnode/vnode/src/meta/metaBDBImpl.c
View File

@ -13,6 +13,7 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
#define ALLOW_FORBID_FUNC
#include "db.h" #include "db.h"
#include "metaDef.h" #include "metaDef.h"

58
source/dnode/vnode/src/tq/tqMetaStore.c
View File

@ -34,11 +34,11 @@ static inline void tqLinkUnpersist(STqMetaStore* pMeta, STqMetaList* pNode) {
} }
} }
static inline int tqSeekLastPage(int fd) { static inline int64_t tqSeekLastPage(TdFilePtr pFile) {
int offset = lseek(fd, 0, SEEK_END); int offset = taosLSeekFile(pFile, 0, SEEK_END);
int pageNo = offset / TQ_PAGE_SIZE; int pageNo = offset / TQ_PAGE_SIZE;
int curPageOffset = pageNo * TQ_PAGE_SIZE; int curPageOffset = pageNo * TQ_PAGE_SIZE;
return lseek(fd, curPageOffset, SEEK_SET); return taosLSeekFile(pFile, curPageOffset, SEEK_SET);
} }
// TODO: the struct is tightly coupled with index entry // TODO: the struct is tightly coupled with index entry
@ -52,10 +52,10 @@ typedef struct STqIdxPageBuf {
char buffer[TQ_IDX_PAGE_BODY_SIZE]; char buffer[TQ_IDX_PAGE_BODY_SIZE];
} STqIdxPageBuf; } STqIdxPageBuf;
static inline int tqReadLastPage(int fd, STqIdxPageBuf* pBuf) { static inline int tqReadLastPage(TdFilePtr pFile, STqIdxPageBuf* pBuf) {
int offset = tqSeekLastPage(fd); int offset = tqSeekLastPage(pFile);
int nBytes; int nBytes;
if ((nBytes = read(fd, pBuf, TQ_PAGE_SIZE)) == -1) { if ((nBytes = taosReadFile(pFile, pBuf, TQ_PAGE_SIZE)) == -1) {
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
return -1; return -1;
} }
@ -65,7 +65,7 @@ static inline int tqReadLastPage(int fd, STqIdxPageBuf* pBuf) {
} }
ASSERT(nBytes == 0 || nBytes == pBuf->head.writeOffset); ASSERT(nBytes == 0 || nBytes == pBuf->head.writeOffset);
return lseek(fd, offset, SEEK_SET); return taosLSeekFile(pFile, offset, SEEK_SET);
} }
STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, FTqDeserialize deserializer, STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, FTqDeserialize deserializer,
@ -95,15 +95,15 @@ STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, F
tqError("failed to create dir:%s since %s ", name, terrstr()); tqError("failed to create dir:%s since %s ", name, terrstr());
} }
strcat(name, "/" TQ_IDX_NAME); strcat(name, "/" TQ_IDX_NAME);
int idxFd = open(name, O_RDWR | O_CREAT, 0755); TdFilePtr pIdxFile = taosOpenFile(name, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_READ);
if (idxFd < 0) { if (pIdxFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
tqError("failed to open file:%s since %s ", name, terrstr()); tqError("failed to open file:%s since %s ", name, terrstr());
// free memory // free memory
return NULL; return NULL;
} }
pMeta->idxFd = idxFd; pMeta->pIdxFile = pIdxFile;
pMeta->unpersistHead = calloc(1, sizeof(STqMetaList)); pMeta->unpersistHead = calloc(1, sizeof(STqMetaList));
if (pMeta->unpersistHead == NULL) { if (pMeta->unpersistHead == NULL) {
terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; terrno = TSDB_CODE_TQ_OUT_OF_MEMORY;
@ -113,14 +113,14 @@ STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, F
strcpy(name, path); strcpy(name, path);
strcat(name, "/" TQ_META_NAME); strcat(name, "/" TQ_META_NAME);
int fileFd = open(name, O_RDWR | O_CREAT, 0755); TdFilePtr pFile = taosOpenFile(name, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_READ);
if (fileFd < 0) { if (pFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
tqError("failed to open file:%s since %s", name, terrstr()); tqError("failed to open file:%s since %s", name, terrstr());
return NULL; return NULL;
} }
pMeta->fileFd = fileFd; pMeta->pFile = pFile;
pMeta->pSerializer = serializer; pMeta->pSerializer = serializer;
pMeta->pDeserializer = deserializer; pMeta->pDeserializer = deserializer;
@ -136,7 +136,7 @@ STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, F
int idxRead; int idxRead;
int allocated = TQ_PAGE_SIZE; int allocated = TQ_PAGE_SIZE;
bool readEnd = false; bool readEnd = false;
while ((idxRead = read(idxFd, &idxBuf, TQ_PAGE_SIZE))) { while ((idxRead = taosReadFile(pIdxFile, &idxBuf, TQ_PAGE_SIZE))) {
if (idxRead == -1) { if (idxRead == -1) {
// TODO: handle error // TODO: handle error
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
@ -152,7 +152,7 @@ STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, F
} }
memcpy(&pNode->handle, &idxBuf.buffer[i], TQ_IDX_SIZE); memcpy(&pNode->handle, &idxBuf.buffer[i], TQ_IDX_SIZE);
lseek(fileFd, pNode->handle.offset, SEEK_SET); taosLSeekFile(pFile, pNode->handle.offset, SEEK_SET);
if (allocated < pNode->handle.serializedSize) { if (allocated < pNode->handle.serializedSize) {
void* ptr = realloc(serializedObj, pNode->handle.serializedSize); void* ptr = realloc(serializedObj, pNode->handle.serializedSize);
if (ptr == NULL) { if (ptr == NULL) {
@ -163,7 +163,7 @@ STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, F
allocated = pNode->handle.serializedSize; allocated = pNode->handle.serializedSize;
} }
serializedObj->ssize = pNode->handle.serializedSize; serializedObj->ssize = pNode->handle.serializedSize;
if (read(fileFd, serializedObj, pNode->handle.serializedSize) != pNode->handle.serializedSize) { if (taosReadFile(pFile, serializedObj, pNode->handle.serializedSize) != pNode->handle.serializedSize) {
// TODO: read error // TODO: read error
} }
if (serializedObj->action == TQ_ACTION_INUSE) { if (serializedObj->action == TQ_ACTION_INUSE) {
@ -237,8 +237,8 @@ int32_t tqStoreClose(STqMetaStore* pMeta) {
// commit data and idx // commit data and idx
tqStorePersist(pMeta); tqStorePersist(pMeta);
ASSERT(pMeta->unpersistHead && pMeta->unpersistHead->next == NULL); ASSERT(pMeta->unpersistHead && pMeta->unpersistHead->next == NULL);
close(pMeta->fileFd); taosCloseFile(&pMeta->pFile);
close(pMeta->idxFd); taosCloseFile(&pMeta->pIdxFile);
// free memory // free memory
for (int i = 0; i < TQ_BUCKET_SIZE; i++) { for (int i = 0; i < TQ_BUCKET_SIZE; i++) {
STqMetaList* pNode = pMeta->bucket[i]; STqMetaList* pNode = pMeta->bucket[i];
@ -263,8 +263,8 @@ int32_t tqStoreClose(STqMetaStore* pMeta) {
} }
int32_t tqStoreDelete(STqMetaStore* pMeta) { int32_t tqStoreDelete(STqMetaStore* pMeta) {
close(pMeta->fileFd); taosCloseFile(&pMeta->pFile);
close(pMeta->idxFd); taosCloseFile(&pMeta->pIdxFile);
// free memory // free memory
for (int i = 0; i < TQ_BUCKET_SIZE; i++) { for (int i = 0; i < TQ_BUCKET_SIZE; i++) {
STqMetaList* pNode = pMeta->bucket[i]; STqMetaList* pNode = pMeta->bucket[i];
@ -302,12 +302,12 @@ int32_t tqStorePersist(STqMetaStore* pMeta) {
pSHead->checksum = 0; pSHead->checksum = 0;
pSHead->ssize = sizeof(STqSerializedHead); pSHead->ssize = sizeof(STqSerializedHead);
/*int allocatedSize = sizeof(STqSerializedHead);*/ /*int allocatedSize = sizeof(STqSerializedHead);*/
int offset = lseek(pMeta->fileFd, 0, SEEK_CUR); int offset = taosLSeekFile(pMeta->pFile, 0, SEEK_CUR);
tqReadLastPage(pMeta->idxFd, &idxBuf); tqReadLastPage(pMeta->pIdxFile, &idxBuf);
if (idxBuf.head.writeOffset == TQ_PAGE_SIZE) { if (idxBuf.head.writeOffset == TQ_PAGE_SIZE) {
lseek(pMeta->idxFd, 0, SEEK_END); taosLSeekFile(pMeta->pIdxFile, 0, SEEK_END);
memset(&idxBuf, 0, TQ_PAGE_SIZE); memset(&idxBuf, 0, TQ_PAGE_SIZE);
idxBuf.head.writeOffset = TQ_IDX_PAGE_HEAD_SIZE; idxBuf.head.writeOffset = TQ_IDX_PAGE_HEAD_SIZE;
} else { } else {
@ -329,7 +329,7 @@ int32_t tqStorePersist(STqMetaStore* pMeta) {
} else { } else {
pMeta->pSerializer(pNode->handle.valueInUse, &pSHead); pMeta->pSerializer(pNode->handle.valueInUse, &pSHead);
} }
nBytes = write(pMeta->fileFd, pSHead, pSHead->ssize); nBytes = taosWriteFile(pMeta->pFile, pSHead, pSHead->ssize);
ASSERT(nBytes == pSHead->ssize); ASSERT(nBytes == pSHead->ssize);
} }
@ -340,7 +340,7 @@ int32_t tqStorePersist(STqMetaStore* pMeta) {
} else { } else {
pMeta->pSerializer(pNode->handle.valueInTxn, &pSHead); pMeta->pSerializer(pNode->handle.valueInTxn, &pSHead);
} }
int nBytesTxn = write(pMeta->fileFd, pSHead, pSHead->ssize); int nBytesTxn = taosWriteFile(pMeta->pFile, pSHead, pSHead->ssize);
ASSERT(nBytesTxn == pSHead->ssize); ASSERT(nBytesTxn == pSHead->ssize);
nBytes += nBytesTxn; nBytes += nBytesTxn;
} }
@ -355,7 +355,7 @@ int32_t tqStorePersist(STqMetaStore* pMeta) {
idxBuf.head.writeOffset += TQ_IDX_SIZE; idxBuf.head.writeOffset += TQ_IDX_SIZE;
if (idxBuf.head.writeOffset >= TQ_PAGE_SIZE) { if (idxBuf.head.writeOffset >= TQ_PAGE_SIZE) {
nBytes = write(pMeta->idxFd, &idxBuf, TQ_PAGE_SIZE); nBytes = taosWriteFile(pMeta->pIdxFile, &idxBuf, TQ_PAGE_SIZE);
// TODO: handle error with tfile // TODO: handle error with tfile
ASSERT(nBytes == TQ_PAGE_SIZE); ASSERT(nBytes == TQ_PAGE_SIZE);
memset(&idxBuf, 0, TQ_PAGE_SIZE); memset(&idxBuf, 0, TQ_PAGE_SIZE);
@ -391,13 +391,13 @@ int32_t tqStorePersist(STqMetaStore* pMeta) {
free(pSHead); free(pSHead);
// TODO: write new version in tfile // TODO: write new version in tfile
if ((char*)bufPtr != idxBuf.buffer) { if ((char*)bufPtr != idxBuf.buffer) {
int nBytes = write(pMeta->idxFd, &idxBuf, idxBuf.head.writeOffset); int nBytes = taosWriteFile(pMeta->pIdxFile, &idxBuf, idxBuf.head.writeOffset);
// TODO: handle error in tfile // TODO: handle error in tfile
ASSERT(nBytes == idxBuf.head.writeOffset); ASSERT(nBytes == idxBuf.head.writeOffset);
} }
// TODO: using fsync in tfile // TODO: using fsync in tfile
fsync(pMeta->idxFd); taosFsyncFile(pMeta->pIdxFile);
fsync(pMeta->fileFd); taosFsyncFile(pMeta->pFile);
return 0; return 0;
} }

6
source/dnode/vnode/src/tsdb/tsdbCommit.c
View File

@ -523,7 +523,8 @@ static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid
SDFile *pRDataf = TSDB_READ_DATA_FILE(&(pCommith->readh)); SDFile *pRDataf = TSDB_READ_DATA_FILE(&(pCommith->readh));
SDFile *pWDataf = TSDB_COMMIT_DATA_FILE(pCommith); SDFile *pWDataf = TSDB_COMMIT_DATA_FILE(pCommith);
tsdbInitDFileEx(pWDataf, pRDataf); tsdbInitDFileEx(pWDataf, pRDataf);
if (tsdbOpenDFile(pWDataf, O_WRONLY) < 0) { // if (tsdbOpenDFile(pWDataf, O_WRONLY) < 0) {
if (tsdbOpenDFile(pWDataf, TD_FILE_WRITE) < 0) {
tsdbError("vgId:%d failed to open file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWDataf), tsdbError("vgId:%d failed to open file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWDataf),
tstrerror(terrno)); tstrerror(terrno));
@ -543,7 +544,8 @@ static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid
tsdbInitDFileEx(pWLastf, pRLastf); tsdbInitDFileEx(pWLastf, pRLastf);
pCommith->isLFileSame = true; pCommith->isLFileSame = true;
if (tsdbOpenDFile(pWLastf, O_WRONLY) < 0) { // if (tsdbOpenDFile(pWLastf, O_WRONLY) < 0) {
if (tsdbOpenDFile(pWLastf, TD_FILE_WRITE) < 0) {
tsdbError("vgId:%d failed to open file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWLastf), tsdbError("vgId:%d failed to open file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWLastf),
tstrerror(terrno)); tstrerror(terrno));

51
source/dnode/vnode/src/tsdb/tsdbFS.c
View File

@ -416,8 +416,8 @@ static int tsdbSaveFSStatus(STsdb *pRepo, SFSStatus *pStatus) {
tsdbGetTxnFname(pRepo, TSDB_TXN_TEMP_FILE, tfname); tsdbGetTxnFname(pRepo, TSDB_TXN_TEMP_FILE, tfname);
tsdbGetTxnFname(pRepo, TSDB_TXN_CURR_FILE, cfname); tsdbGetTxnFname(pRepo, TSDB_TXN_CURR_FILE, cfname);
int fd = open(tfname, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0755); TdFilePtr pFile = taosOpenFile(tfname, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
if (fd < 0) { if (pFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
return -1; return -1;
} }
@ -436,9 +436,9 @@ static int tsdbSaveFSStatus(STsdb *pRepo, SFSStatus *pStatus) {
taosCalcChecksumAppend(0, (uint8_t *)hbuf, TSDB_FILE_HEAD_SIZE); taosCalcChecksumAppend(0, (uint8_t *)hbuf, TSDB_FILE_HEAD_SIZE);
if (taosWriteFile(fd, hbuf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) { if (taosWriteFile(pFile, hbuf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) {
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
close(fd); taosCloseFile(&pFile);
remove(tfname); remove(tfname);
return -1; return -1;
} }
@ -446,7 +446,7 @@ static int tsdbSaveFSStatus(STsdb *pRepo, SFSStatus *pStatus) {
// Encode file status and write to file // Encode file status and write to file
if (fsheader.len > 0) { if (fsheader.len > 0) {
if (tsdbMakeRoom(&(pBuf), fsheader.len) < 0) { if (tsdbMakeRoom(&(pBuf), fsheader.len) < 0) {
close(fd); taosCloseFile(&pFile);
remove(tfname); remove(tfname);
return -1; return -1;
} }
@ -455,9 +455,9 @@ static int tsdbSaveFSStatus(STsdb *pRepo, SFSStatus *pStatus) {
tsdbEncodeFSStatus(&ptr, pStatus); tsdbEncodeFSStatus(&ptr, pStatus);
taosCalcChecksumAppend(0, (uint8_t *)pBuf, fsheader.len); taosCalcChecksumAppend(0, (uint8_t *)pBuf, fsheader.len);
if (taosWriteFile(fd, pBuf, fsheader.len) < fsheader.len) { if (taosWriteFile(pFile, pBuf, fsheader.len) < fsheader.len) {
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
close(fd); taosCloseFile(&pFile);
(void)remove(tfname); (void)remove(tfname);
taosTZfree(pBuf); taosTZfree(pBuf);
return -1; return -1;
@ -465,15 +465,15 @@ static int tsdbSaveFSStatus(STsdb *pRepo, SFSStatus *pStatus) {
} }
// fsync, close and rename // fsync, close and rename
if (taosFsyncFile(fd) < 0) { if (taosFsyncFile(pFile) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
close(fd); taosCloseFile(&pFile);
remove(tfname); remove(tfname);
taosTZfree(pBuf); taosTZfree(pBuf);
return -1; return -1;
} }
(void)close(fd); (void)taosCloseFile(&pFile);
(void)taosRenameFile(tfname, cfname); (void)taosRenameFile(tfname, cfname);
taosTZfree(pBuf); taosTZfree(pBuf);
@ -652,7 +652,7 @@ static void tsdbGetTxnFname(STsdb *pRepo, TSDB_TXN_FILE_T ftype, char fname[]) {
static int tsdbOpenFSFromCurrent(STsdb *pRepo) { static int tsdbOpenFSFromCurrent(STsdb *pRepo) {
STsdbFS * pfs = REPO_FS(pRepo); STsdbFS * pfs = REPO_FS(pRepo);
int fd = -1; TdFilePtr pFile = NULL;
void * buffer = NULL; void * buffer = NULL;
SFSHeader fsheader; SFSHeader fsheader;
char current[TSDB_FILENAME_LEN] = "\0"; char current[TSDB_FILENAME_LEN] = "\0";
@ -661,8 +661,8 @@ static int tsdbOpenFSFromCurrent(STsdb *pRepo) {
tsdbGetTxnFname(pRepo, TSDB_TXN_CURR_FILE, current); tsdbGetTxnFname(pRepo, TSDB_TXN_CURR_FILE, current);
// current file exists, try to recover // current file exists, try to recover
fd = open(current, O_RDONLY | O_BINARY); pFile = taosOpenFile(current, TD_FILE_READ);
if (fd < 0) { if (pFile == NULL) {
tsdbError("vgId:%d failed to open file %s since %s", REPO_ID(pRepo), current, strerror(errno)); tsdbError("vgId:%d failed to open file %s since %s", REPO_ID(pRepo), current, strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
goto _err; goto _err;
@ -672,7 +672,7 @@ static int tsdbOpenFSFromCurrent(STsdb *pRepo) {
goto _err; goto _err;
} }
int nread = (int)taosReadFile(fd, buffer, TSDB_FILE_HEAD_SIZE); int nread = (int)taosReadFile(pFile, buffer, TSDB_FILE_HEAD_SIZE);
if (nread < 0) { if (nread < 0) {
tsdbError("vgId:%d failed to read %d bytes from file %s since %s", REPO_ID(pRepo), TSDB_FILENAME_LEN, current, tsdbError("vgId:%d failed to read %d bytes from file %s since %s", REPO_ID(pRepo), TSDB_FILENAME_LEN, current,
strerror(errno)); strerror(errno));
@ -706,7 +706,7 @@ static int tsdbOpenFSFromCurrent(STsdb *pRepo) {
goto _err; goto _err;
} }
nread = (int)taosReadFile(fd, buffer, fsheader.len); nread = (int)taosReadFile(pFile, buffer, fsheader.len);
if (nread < 0) { if (nread < 0) {
tsdbError("vgId:%d failed to read file %s since %s", REPO_ID(pRepo), current, strerror(errno)); tsdbError("vgId:%d failed to read file %s since %s", REPO_ID(pRepo), current, strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
@ -732,13 +732,13 @@ static int tsdbOpenFSFromCurrent(STsdb *pRepo) {
} }
taosTZfree(buffer); taosTZfree(buffer);
close(fd); taosCloseFile(&pFile);
return 0; return 0;
_err: _err:
if (fd >= 0) { if (pFile != NULL) {
close(fd); taosCloseFile(&pFile);
} }
taosTZfree(buffer); taosTZfree(buffer);
return -1; return -1;
@ -1229,7 +1229,8 @@ static int tsdbRestoreDFileSet(STsdb *pRepo) {
pDFile->f = *pf; pDFile->f = *pf;
if (tsdbOpenDFile(pDFile, O_RDONLY) < 0) { // if (tsdbOpenDFile(pDFile, O_RDONLY) < 0) {
if (tsdbOpenDFile(pDFile, TD_FILE_READ) < 0) {
tsdbError("vgId:%d failed to open DFile %s since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pDFile), tsdbError("vgId:%d failed to open DFile %s since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pDFile),
tstrerror(terrno)); tstrerror(terrno));
taosArrayDestroy(fArray); taosArrayDestroy(fArray);
@ -1244,18 +1245,17 @@ static int tsdbRestoreDFileSet(STsdb *pRepo) {
} }
if (tsdbForceKeepFile) { if (tsdbForceKeepFile) {
struct stat tfstat; int64_t file_size;
// Get real file size // Get real file size
if (fstat(pDFile->fd, &tfstat) < 0) { if (taosFStatFile(pDFile->pFile, &file_size, NULL) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
taosArrayDestroy(fArray); taosArrayDestroy(fArray);
return -1; return -1;
} }
if (pDFile->info.size != tfstat.st_size) { if (pDFile->info.size != file_size) {
int64_t tfsize = pDFile->info.size; int64_t tfsize = pDFile->info.size;
pDFile->info.size = tfstat.st_size; pDFile->info.size = file_size;
tsdbInfo("vgId:%d file %s header size is changed from %" PRId64 " to %" PRId64, REPO_ID(pRepo), tsdbInfo("vgId:%d file %s header size is changed from %" PRId64 " to %" PRId64, REPO_ID(pRepo),
TSDB_FILE_FULL_NAME(pDFile), tfsize, pDFile->info.size); TSDB_FILE_FULL_NAME(pDFile), tfsize, pDFile->info.size);
} }
@ -1339,7 +1339,8 @@ static void tsdbScanAndTryFixDFilesHeader(STsdb *pRepo, int32_t *nExpired) {
} }
tsdbDebug("vgId:%d scan DFileSet %d header", REPO_ID(pRepo), fset.fid); tsdbDebug("vgId:%d scan DFileSet %d header", REPO_ID(pRepo), fset.fid);
if (tsdbOpenDFileSet(&fset, O_RDWR) < 0) { // if (tsdbOpenDFileSet(&fset, O_RDWR) < 0) {
if (tsdbOpenDFileSet(&fset, TD_FILE_WRITE | TD_FILE_READ) < 0) {
tsdbError("vgId:%d failed to open DFileSet %d since %s, continue", REPO_ID(pRepo), fset.fid, tstrerror(terrno)); tsdbError("vgId:%d failed to open DFileSet %d since %s, continue", REPO_ID(pRepo), fset.fid, tstrerror(terrno));
continue; continue;
} }

18
source/dnode/vnode/src/tsdb/tsdbFile.c
View File

@ -355,8 +355,8 @@ static void *tsdbDecodeSDFileEx(void *buf, SDFile *pDFile) {
int tsdbCreateDFile(STsdb *pRepo, SDFile *pDFile, bool updateHeader) { int tsdbCreateDFile(STsdb *pRepo, SDFile *pDFile, bool updateHeader) {
ASSERT(pDFile->info.size == 0 && pDFile->info.magic == TSDB_FILE_INIT_MAGIC); ASSERT(pDFile->info.size == 0 && pDFile->info.magic == TSDB_FILE_INIT_MAGIC);
pDFile->fd = open(TSDB_FILE_FULL_NAME(pDFile), O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0755); pDFile->pFile = taosOpenFile(TSDB_FILE_FULL_NAME(pDFile), TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
if (pDFile->fd < 0) { if (pDFile->pFile < 0) {
if (errno == ENOENT) { if (errno == ENOENT) {
// Try to create directory recursively // Try to create directory recursively
char *s = strdup(TSDB_FILE_REL_NAME(pDFile)); char *s = strdup(TSDB_FILE_REL_NAME(pDFile));
@ -366,8 +366,8 @@ int tsdbCreateDFile(STsdb *pRepo, SDFile *pDFile, bool updateHeader) {
} }
tfree(s); tfree(s);
pDFile->fd = open(TSDB_FILE_FULL_NAME(pDFile), O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0755); pDFile->pFile = taosOpenFile(TSDB_FILE_FULL_NAME(pDFile), TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
if (pDFile->fd < 0) { if (pDFile->pFile < 0) {
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
return -1; return -1;
} }
@ -456,11 +456,12 @@ static int tsdbScanAndTryFixDFile(STsdb *pRepo, SDFile *pDFile) {
} }
if (pDFile->info.size < dfstat.st_size) { if (pDFile->info.size < dfstat.st_size) {
if (tsdbOpenDFile(&df, O_WRONLY) < 0) { // if (tsdbOpenDFile(&df, O_WRONLY) < 0) {
if (tsdbOpenDFile(&df, TD_FILE_WRITE) < 0) {
return -1; return -1;
} }
if (taosFtruncateFile(df.fd, df.info.size) < 0) { if (taosFtruncateFile(df.pFile, df.info.size) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
tsdbCloseDFile(&df); tsdbCloseDFile(&df);
return -1; return -1;
@ -537,11 +538,12 @@ static int tsdbApplyDFileChange(SDFile *from, SDFile *to) {
static int tsdbRollBackDFile(SDFile *pDFile) { static int tsdbRollBackDFile(SDFile *pDFile) {
SDFile df = *pDFile; SDFile df = *pDFile;
if (tsdbOpenDFile(&df, O_WRONLY) < 0) { // if (tsdbOpenDFile(&df, O_WRONLY) < 0) {
if (tsdbOpenDFile(&df, TD_FILE_WRITE) < 0) {
return -1; return -1;
} }
if (taosFtruncateFile(TSDB_FILE_FD(&df), pDFile->info.size) < 0) { if (taosFtruncateFile(TSDB_FILE_PFILE(&df), pDFile->info.size) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
tsdbCloseDFile(&df); tsdbCloseDFile(&df);
return -1; return -1;

3
source/dnode/vnode/src/tsdb/tsdbReadImpl.c
View File

@ -83,7 +83,8 @@ int tsdbSetAndOpenReadFSet(SReadH *pReadh, SDFileSet *pSet) {
pReadh->rSet = *pSet; pReadh->rSet = *pSet;
TSDB_FSET_SET_CLOSED(TSDB_READ_FSET(pReadh)); TSDB_FSET_SET_CLOSED(TSDB_READ_FSET(pReadh));
if (tsdbOpenDFileSet(TSDB_READ_FSET(pReadh), O_RDONLY) < 0) { // if (tsdbOpenDFileSet(TSDB_READ_FSET(pReadh), O_RDONLY) < 0) {
if (tsdbOpenDFileSet(TSDB_READ_FSET(pReadh), TD_FILE_READ) < 0) {
tsdbError("vgId:%d failed to open file set %d since %s", TSDB_READ_REPO_ID(pReadh), TSDB_FSET_FID(pSet), tsdbError("vgId:%d failed to open file set %d since %s", TSDB_READ_REPO_ID(pReadh), TSDB_FSET_FID(pSet),
tstrerror(terrno)); tstrerror(terrno));
return -1; return -1;

2
source/libs/CMakeLists.txt
View File

@ -1,3 +1,4 @@
add_definitions("-D ALLOW_FORBID_FUNC")
add_subdirectory(transport) add_subdirectory(transport)
add_subdirectory(sync) add_subdirectory(sync)
add_subdirectory(tdb) add_subdirectory(tdb)
@ -14,5 +15,4 @@ add_subdirectory(qcom)
add_subdirectory(qworker) add_subdirectory(qworker)
add_subdirectory(tfs) add_subdirectory(tfs)
add_subdirectory(nodes) add_subdirectory(nodes)
add_subdirectory(config)
add_subdirectory(scalar) add_subdirectory(scalar)

13
source/libs/config/CMakeLists.txt
View File

@ -1,13 +0,0 @@
aux_source_directory(src CONFIG_SRC)
add_library(config STATIC ${CONFIG_SRC})
target_include_directories(
config
PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/config"
PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
)
target_link_libraries(config os util common)
if(${BUILD_TEST})
ADD_SUBDIRECTORY(test)
endif(${BUILD_TEST})

47
source/libs/config/inc/cfgInt.h
View File

@ -1,47 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_CFG_INT_H_
#define _TD_CFG_INT_H_
#include "config.h"
#include "taoserror.h"
#include "thash.h"
#include "tutil.h"
#include "ulog.h"
#include "tglobal.h"
#ifdef __cplusplus
extern "C" {
#endif
typedef struct SConfig {
ECfgSrcType stype;
SHashObj *hash;
} SConfig;
int32_t cfgLoadFromCfgFile(SConfig *pConfig, const char *filepath);
int32_t cfgLoadFromEnvFile(SConfig *pConfig, const char *filepath);
int32_t cfgLoadFromEnvVar(SConfig *pConfig);
int32_t cfgLoadFromApollUrl(SConfig *pConfig, const char *url);
int32_t cfgSetItem(SConfig *pConfig, const char *name, const char *value, ECfgSrcType stype);
#ifdef __cplusplus
}
#endif
#endif /*_TD_CFG_INT_H_*/

22
source/libs/config/src/cfgApolloUrl.c
View File

@ -1,22 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "cfgInt.h"
int32_t cfgLoadFromApollUrl(SConfig *pConfig, const char *url) {
uInfo("load from apoll url %s", url);
return 0;
}

70
source/libs/config/src/cfgCfgFile.c
View File

@ -1,70 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "cfgInt.h"
int32_t cfgLoadFromCfgFile(SConfig *pConfig, const char *filepath) {
char *line, *name, *value, *value2, *value3;
int olen, vlen, vlen2, vlen3;
ssize_t _bytes = 0;
size_t len = 1024;
FILE *fp = fopen(filepath, "r");
if (fp == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
line = malloc(len);
while (!feof(fp)) {
memset(line, 0, len);
name = value = value2 = value3 = NULL;
olen = vlen = vlen2 = vlen3 = 0;
_bytes = tgetline(&line, &len, fp);
if (_bytes < 0) {
break;
}
line[len - 1] = 0;
paGetToken(line, &name, &olen);
if (olen == 0) continue;
name[olen] = 0;
paGetToken(name + olen + 1, &value, &vlen);
if (vlen == 0) continue;
value[vlen] = 0;
paGetToken(value + vlen + 1, &value2, &vlen2);
if (vlen2 != 0) {
value2[vlen2] = 0;
paGetToken(value2 + vlen2 + 1, &value3, &vlen3);
if (vlen3 != 0) value3[vlen3] = 0;
}
cfgSetItem(pConfig, name, value, CFG_STYPE_CFG_FILE);
// taosReadConfigOption(name, value, value2, value3);
}
fclose(fp);
tfree(line);
uInfo("load from cfg file %s success", filepath);
return 0;
}

22
source/libs/config/src/cfgEnvFile.c
View File

@ -1,22 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "cfgInt.h"
int32_t cfgLoadFromEnvFile(SConfig *pConfig, const char *filepath) {
uInfo("load from env file %s", filepath);
return 0;
}

22
source/libs/config/src/cfgEnvVar.c
View File

@ -1,22 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "cfgInt.h"
int32_t cfgLoadFromEnvVar(SConfig *pConfig) {
uInfo("load from global env variables");
return 0;
}

14
source/libs/config/test/CMakeLists.txt
View File

@ -1,14 +0,0 @@
enable_testing()
aux_source_directory(. CFG_TEST_SRC)
add_executable(cfg_test ${CFG_TEST_SRC})
target_link_libraries(
cfg_test
PUBLIC config
PUBLIC gtest_main
)
add_test(
NAME cfg_test
COMMAND cfg_test
)

9
source/libs/executor/src/executorimpl.c
View File

@ -132,8 +132,7 @@ do { \
} while (0) } while (0)
int32_t getMaximumIdleDurationSec() { int32_t getMaximumIdleDurationSec() {
// todo return tsShellActivityTimer * 2;
return 6; //tsShellActivityTimer * 2;
} }
static int32_t getExprFunctionId(SExprInfo *pExprInfo) { static int32_t getExprFunctionId(SExprInfo *pExprInfo) {
@ -5302,12 +5301,10 @@ SOperatorInfo* createExchangeOperatorInfo(const SArray* pSources, const SArray*
rpcInit.label = "EX"; rpcInit.label = "EX";
rpcInit.numOfThreads = 1; rpcInit.numOfThreads = 1;
rpcInit.cfp = qProcessFetchRsp; rpcInit.cfp = qProcessFetchRsp;
// todo rpcInit.sessions = tsMaxConnections;
rpcInit.sessions = 50000; //tsMaxConnections;
rpcInit.connType = TAOS_CONN_CLIENT; rpcInit.connType = TAOS_CONN_CLIENT;
rpcInit.user = (char *)"root"; rpcInit.user = (char *)"root";
// todo rpcInit.idleTime = tsShellActivityTimer * 1000;
rpcInit.idleTime = 6; //tsShellActivityTimer * 1000;
rpcInit.ckey = "key"; rpcInit.ckey = "key";
rpcInit.spi = 1; rpcInit.spi = 1;
rpcInit.secret = (char *)"dcc5bed04851fec854c035b2e40263b6"; rpcInit.secret = (char *)"dcc5bed04851fec854c035b2e40263b6";

8
source/libs/function/src/taggfunction.c
View File

@ -3960,14 +3960,14 @@ static void ts_comp_finalize(SqlFunctionCtx *pCtx) {
// qDebug("total timestamp :%"PRId64, pTSbuf->numOfTotal); // qDebug("total timestamp :%"PRId64, pTSbuf->numOfTotal);
// TODO refactor transfer ownership of current file // TODO refactor transfer ownership of current file
*(FILE **)pCtx->pOutput = pTSbuf->f; *(TdFilePtr *)pCtx->pOutput = pTSbuf->pFile;
pResInfo->complete = true; pResInfo->complete = true;
// get the file size // get the file size
struct stat fStat; int64_t file_size;
if ((fstat(fileno(pTSbuf->f), &fStat) == 0)) { if (taosFStatFile(pTSbuf->pFile, &file_size, NULL) == 0) {
pResInfo->numOfRes = fStat.st_size; pResInfo->numOfRes = (uint32_t )file_size;
} }
pTSbuf->remainOpen = true; pTSbuf->remainOpen = true;

2
source/libs/index/inc/index_fst_counting_writer.h
View File

@ -38,7 +38,7 @@ typedef struct WriterCtx {
WriterType type; WriterType type;
union { union {
struct { struct {
int fd; TdFilePtr pFile;
bool readOnly; bool readOnly;
char buf[256]; char buf[256];
int size; int size;

10
source/libs/index/inc/index_util.h
View File

@ -15,6 +15,8 @@
#ifndef __INDEX_UTIL_H__ #ifndef __INDEX_UTIL_H__
#define __INDEX_UTIL_H__ #define __INDEX_UTIL_H__
#include "tarray.h"
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
#endif #endif
@ -45,6 +47,14 @@ extern "C" {
buf += len; \ buf += len; \
} while (0) } while (0)
/* multi sorted result intersection
* input: [1, 2, 4, 5]
* [2, 3, 4, 5]
* [1, 4, 5]
* output:[4, 5]
*/
void iIntersection(SArray *interResults, SArray *finalResult);
void iUnion(SArray *interResults, SArray *finalResult);
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

19
source/libs/index/src/index.c
View File

@ -370,22 +370,23 @@ static void indexInterResultsDestroy(SArray* results) {
} }
taosArrayDestroy(results); taosArrayDestroy(results);
} }
static int indexMergeFinalResults(SArray* interResults, EIndexOperatorType oType, SArray* fResults) { static int indexMergeFinalResults(SArray* interResults, EIndexOperatorType oType, SArray* fResults) {
// refactor, merge interResults into fResults by oType // refactor, merge interResults into fResults by oType
SArray* first = taosArrayGetP(interResults, 0);
taosArraySort(first, uidCompare); for (int i = 0; i < taosArrayGetSize(interResults); i--) {
taosArrayRemoveDuplicate(first, uidCompare, NULL); SArray* t = taosArrayGetP(interResults, i);
taosArraySort(t, uidCompare);
taosArrayRemoveDuplicate(t, uidCompare, NULL);
}
if (oType == MUST) { if (oType == MUST) {
// just one column index, enhance later iIntersection(interResults, fResults);
taosArrayAddAll(fResults, first);
} else if (oType == SHOULD) { } else if (oType == SHOULD) {
// just one column index, enhance later iUnion(interResults, fResults);
taosArrayAddAll(fResults, first);
// tag1 condistion || tag2 condition
} else if (oType == NOT) { } else if (oType == NOT) {
// just one column index, enhance later // just one column index, enhance later
taosArrayAddAll(fResults, first); taosArrayAddAll(fResults, interResults);
// not use currently // not use currently
} }
return 0; return 0;

36
source/libs/index/src/index_fst_counting_writer.c
View File

@ -19,7 +19,7 @@
static int writeCtxDoWrite(WriterCtx* ctx, uint8_t* buf, int len) { static int writeCtxDoWrite(WriterCtx* ctx, uint8_t* buf, int len) {
if (ctx->type == TFile) { if (ctx->type == TFile) {
assert(len == tfWrite(ctx->file.fd, buf, len)); assert(len == taosWriteFile(ctx->file.pFile, buf, len));
} else { } else {
memcpy(ctx->mem.buf + ctx->offset, buf, len); memcpy(ctx->mem.buf + ctx->offset, buf, len);
} }
@ -33,7 +33,7 @@ static int writeCtxDoRead(WriterCtx* ctx, uint8_t* buf, int len) {
nRead = len < ctx->file.size ? len : ctx->file.size; nRead = len < ctx->file.size ? len : ctx->file.size;
memcpy(buf, ctx->file.ptr, nRead); memcpy(buf, ctx->file.ptr, nRead);
#else #else
nRead = tfRead(ctx->file.fd, buf, len); nRead = taosReadFile(ctx->file.pFile, buf, len);
#endif #endif
} else { } else {
memcpy(buf, ctx->mem.buf + ctx->offset, len); memcpy(buf, ctx->mem.buf + ctx->offset, len);
@ -45,13 +45,13 @@ static int writeCtxDoRead(WriterCtx* ctx, uint8_t* buf, int len) {
static int writeCtxDoReadFrom(WriterCtx* ctx, uint8_t* buf, int len, int32_t offset) { static int writeCtxDoReadFrom(WriterCtx* ctx, uint8_t* buf, int len, int32_t offset) {
int nRead = 0; int nRead = 0;
if (ctx->type == TFile) { if (ctx->type == TFile) {
// tfLseek(ctx->file.fd, offset, 0); // tfLseek(ctx->file.pFile, offset, 0);
#ifdef USE_MMAP #ifdef USE_MMAP
int32_t last = ctx->file.size - offset; int32_t last = ctx->file.size - offset;
nRead = last >= len ? len : last; nRead = last >= len ? len : last;
memcpy(buf, ctx->file.ptr + offset, nRead); memcpy(buf, ctx->file.ptr + offset, nRead);
#else #else
nRead = tfPread(ctx->file.fd, buf, len, offset); nRead = taosPReadFile(ctx->file.pFile, buf, len, offset);
#endif #endif
} else { } else {
// refactor later // refactor later
@ -69,9 +69,9 @@ static int writeCtxGetSize(WriterCtx* ctx) {
} }
static int writeCtxDoFlush(WriterCtx* ctx) { static int writeCtxDoFlush(WriterCtx* ctx) {
if (ctx->type == TFile) { if (ctx->type == TFile) {
// taosFsyncFile(ctx->file.fd); // taosFsyncFile(ctx->file.pFile);
tfFsync(ctx->file.fd); taosFsyncFile(ctx->file.pFile);
// tfFlush(ctx->file.fd); // tfFlush(ctx->file.pFile);
} else { } else {
// do nothing // do nothing
} }
@ -87,25 +87,25 @@ WriterCtx* writerCtxCreate(WriterType type, const char* path, bool readOnly, int
// ugly code, refactor later // ugly code, refactor later
ctx->file.readOnly = readOnly; ctx->file.readOnly = readOnly;
if (readOnly == false) { if (readOnly == false) {
// ctx->file.fd = open(path, O_WRONLY | O_CREAT | O_APPEND, S_IRWXU | S_IRWXG | S_IRWXO); // ctx->file.pFile = open(path, O_WRONLY | O_CREAT | O_APPEND, S_IRWXU | S_IRWXG | S_IRWXO);
ctx->file.fd = tfOpenCreateWriteAppend(path); ctx->file.pFile = taosOpenFile(path, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_APPEND);
tfFtruncate(ctx->file.fd, 0); taosFtruncateFile(ctx->file.pFile, 0);
struct stat fstat; int64_t file_size;
stat(path, &fstat); taosStatFile(path, &file_size, NULL);
ctx->file.size = fstat.st_size; ctx->file.size = (int)file_size;
} else { } else {
// ctx->file.fd = open(path, O_RDONLY, S_IRWXU | S_IRWXG | S_IRWXO); // ctx->file.pFile = open(path, O_RDONLY, S_IRWXU | S_IRWXG | S_IRWXO);
ctx->file.fd = tfOpenRead(path); ctx->file.pFile = taosOpenFile(path, TD_FILE_READ);
struct stat fstat; struct stat fstat;
stat(path, &fstat); stat(path, &fstat);
ctx->file.size = fstat.st_size; ctx->file.size = fstat.st_size;
#ifdef USE_MMAP #ifdef USE_MMAP
ctx->file.ptr = (char*)tfMmapReadOnly(ctx->file.fd, ctx->file.size); ctx->file.ptr = (char*)tfMmapReadOnly(ctx->file.pFile, ctx->file.size);
#endif #endif
} }
memcpy(ctx->file.buf, path, strlen(path)); memcpy(ctx->file.buf, path, strlen(path));
if (ctx->file.fd < 0) { if (ctx->file.pFile < 0) {
indexError("failed to open file, error %d", errno); indexError("failed to open file, error %d", errno);
goto END; goto END;
} }
@ -133,7 +133,7 @@ void writerCtxDestroy(WriterCtx* ctx, bool remove) {
free(ctx->mem.buf); free(ctx->mem.buf);
} else { } else {
ctx->flush(ctx); ctx->flush(ctx);
tfClose(ctx->file.fd); taosCloseFile(&ctx->file.pFile);
if (ctx->file.readOnly) { if (ctx->file.readOnly) {
#ifdef USE_MMAP #ifdef USE_MMAP
munmap(ctx->file.ptr, ctx->file.size); munmap(ctx->file.ptr, ctx->file.size);

10
source/libs/index/src/index_tfile.c
View File

@ -371,7 +371,7 @@ int indexTFileSearch(void* tfile, SIndexTermQuery* query, SArray* result) {
return ret; return ret;
} }
IndexTFile* pTfile = (IndexTFile*)tfile; IndexTFile* pTfile = tfile;
SIndexTerm* term = query->term; SIndexTerm* term = query->term;
ICacheKey key = {.suid = term->suid, .colType = term->colType, .colName = term->colName, .nColName = term->nColName}; ICacheKey key = {.suid = term->suid, .colType = term->colType, .colName = term->colName, .nColName = term->nColName};
@ -586,11 +586,11 @@ static int tfileReaderLoadHeader(TFileReader* reader) {
int64_t nread = reader->ctx->readFrom(reader->ctx, buf, sizeof(buf), 0); int64_t nread = reader->ctx->readFrom(reader->ctx, buf, sizeof(buf), 0);
if (nread == -1) { if (nread == -1) {
indexError("actual Read: %d, to read: %d, errno: %d, filefd: %d, filename: %s", (int)(nread), (int)sizeof(buf), indexError("actual Read: %d, to read: %d, errno: %d, filename: %s", (int)(nread), (int)sizeof(buf),
errno, reader->ctx->file.fd, reader->ctx->file.buf); errno, reader->ctx->file.buf);
} else { } else {
indexInfo("actual Read: %d, to read: %d, filefd: %d, filename: %s", (int)(nread), (int)sizeof(buf), indexInfo("actual Read: %d, to read: %d, filename: %s", (int)(nread), (int)sizeof(buf),
reader->ctx->file.fd, reader->ctx->file.buf); reader->ctx->file.buf);
} }
// assert(nread == sizeof(buf)); // assert(nread == sizeof(buf));
memcpy(&reader->header, buf, sizeof(buf)); memcpy(&reader->header, buf, sizeof(buf));

111
source/libs/index/src/index_util.c Normal file
View File

@ -0,0 +1,111 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "index_util.h"
#include "index.h"
typedef struct MergeIndex {
int idx;
int len;
} MergeIndex;
static int iBinarySearch(SArray *arr, int s, int e, uint64_t k) {
uint64_t v;
int32_t m;
while (s <= e) {
m = s + (e - s) / 2;
v = *(uint64_t *)taosArrayGet(arr, m);
if (v >= k) {
e = m - 1;
} else {
s = m + 1;
}
}
return s;
}
void iIntersection(SArray *inters, SArray *final) {
int32_t sz = taosArrayGetSize(inters);
if (sz <= 0) {
return;
}
MergeIndex *mi = calloc(sz, sizeof(MergeIndex));
for (int i = 0; i < sz; i++) {
SArray *t = taosArrayGetP(inters, i);
mi[i].len = taosArrayGetSize(t);
mi[i].idx = 0;
}
SArray *base = taosArrayGetP(inters, 0);
for (int i = 0; i < taosArrayGetSize(base); i++) {
uint64_t tgt = *(uint64_t *)taosArrayGet(base, i);
bool has = true;
for (int j = 1; j < taosArrayGetSize(inters); j++) {
SArray *oth = taosArrayGetP(inters, j);
int mid = iBinarySearch(oth, mi[j].idx, mi[j].len - 1, tgt);
if (mid >= 0 && mid < mi[j].len) {
uint64_t val = *(uint64_t *)taosArrayGet(oth, mid);
has = (val == tgt ? true : false);
mi[j].idx = mid;
} else {
has = false;
}
}
if (has == true) {
taosArrayPush(final, &tgt);
}
}
tfree(mi);
}
void iUnion(SArray *inters, SArray *final) {
int32_t sz = taosArrayGetSize(inters);
if (sz <= 0) {
return;
}
MergeIndex *mi = calloc(sz, sizeof(MergeIndex));
for (int i = 0; i < sz; i++) {
SArray *t = taosArrayGetP(inters, i);
mi[i].len = taosArrayGetSize(t);
mi[i].idx = 0;
}
while (1) {
uint64_t mVal = UINT_MAX;
int mIdx = -1;
for (int j = 0; j < sz; j++) {
SArray *t = taosArrayGetP(inters, j);
if (mi[j].idx >= mi[j].len) {
continue;
}
uint64_t cVal = *(uint64_t *)taosArrayGet(t, mi[j].idx);
if (cVal < mVal) {
mVal = cVal;
mIdx = j;
}
}
if (mIdx != -1) {
mi[mIdx].idx++;
if (taosArrayGetSize(final) > 0) {
uint64_t lVal = *(uint64_t *)taosArrayGetLast(final);
if (lVal == mVal) {
continue;
}
}
taosArrayPush(final, &mVal);
} else {
break;
}
}
tfree(mi);
}

20
source/libs/index/test/CMakeLists.txt
View File

@ -1,6 +1,7 @@
add_executable(indexTest "") add_executable(indexTest "")
add_executable(fstTest "") add_executable(fstTest "")
add_executable(fstUT "") add_executable(fstUT "")
add_executable(UtilUT "")
target_sources(indexTest target_sources(indexTest
PRIVATE PRIVATE
@ -15,6 +16,11 @@ target_sources(fstUT
PRIVATE PRIVATE
"fstUT.cc" "fstUT.cc"
) )
target_sources(UtilUT
PRIVATE
"utilUT.cc"
)
target_include_directories ( indexTest target_include_directories ( indexTest
PUBLIC PUBLIC
"${CMAKE_SOURCE_DIR}/include/libs/index" "${CMAKE_SOURCE_DIR}/include/libs/index"
@ -31,6 +37,12 @@ target_include_directories ( fstUT
"${CMAKE_SOURCE_DIR}/include/libs/index" "${CMAKE_SOURCE_DIR}/include/libs/index"
"${CMAKE_CURRENT_SOURCE_DIR}/../inc" "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
) )
target_include_directories ( UtilUT
PUBLIC
"${CMAKE_SOURCE_DIR}/include/libs/index"
"${CMAKE_CURRENT_SOURCE_DIR}/../inc"
)
target_link_libraries (indexTest target_link_libraries (indexTest
os os
util util
@ -53,6 +65,14 @@ target_link_libraries (fstUT
index index
) )
target_link_libraries (UtilUT
os
util
common
gtest_main
index
)
#add_test( #add_test(
# NAME index_test # NAME index_test

203
source/libs/index/test/utilUT.cc Normal file
View File

@ -0,0 +1,203 @@
#include <gtest/gtest.h>
#include <algorithm>
#include <iostream>
#include <string>
#include <thread>
#include <vector>
#include "index.h"
#include "indexInt.h"
#include "index_cache.h"
#include "index_fst.h"
#include "index_fst_counting_writer.h"
#include "index_fst_util.h"
#include "index_tfile.h"
#include "index_util.h"
#include "tglobal.h"
#include "tskiplist.h"
#include "tutil.h"
class UtilEnv : public ::testing::Test {
protected:
virtual void SetUp() {
src = (SArray *)taosArrayInit(2, sizeof(void *));
for (int i = 0; i < 3; i++) {
SArray *m = taosArrayInit(10, sizeof(uint64_t));
taosArrayPush(src, &m);
}
rslt = (SArray *)taosArrayInit(10, sizeof(uint64_t));
}
virtual void TearDown() {
for (int i = 0; i < taosArrayGetSize(src); i++) {
SArray *m = (SArray *)taosArrayGetP(src, i);
taosArrayDestroy(m);
}
taosArrayDestroy(src);
}
SArray *src;
SArray *rslt;
};
static void clearSourceArray(SArray *p) {
for (int i = 0; i < taosArrayGetSize(p); i++) {
SArray *m = (SArray *)taosArrayGetP(p, i);
taosArrayClear(m);
}
}
static void clearFinalArray(SArray *p) { taosArrayClear(p); }
TEST_F(UtilEnv, intersectionSimpleResult) {
SArray *f = (SArray *)taosArrayGetP(src, 0);
for (int i = 0; i < 10; i++) {
uint64_t val = i;
taosArrayPush(f, &val);
}
f = (SArray *)taosArrayGetP(src, 1);
for (int i = 0; i < 10; i++) {
uint64_t val = i;
taosArrayPush(f, &val);
}
f = (SArray *)taosArrayGetP(src, 2);
for (int i = 0; i < 10; i++) {
uint64_t val = i;
taosArrayPush(f, &val);
}
iIntersection(src, rslt);
assert(taosArrayGetSize(rslt) == 10);
clearSourceArray(src);
clearFinalArray(rslt);
}
TEST_F(UtilEnv, intersectMultiEmptyResult) {
SArray *f = (SArray *)taosArrayGetP(src, 0);
for (int i = 10; i < 20; i++) {
uint64_t val = i;
taosArrayPush(f, &val);
}
f = (SArray *)taosArrayGetP(src, 1);
for (int i = 0; i < 10; i++) {
uint64_t val = i;
taosArrayPush(f, &val);
}
f = (SArray *)taosArrayGetP(src, 2);
for (int i = 20; i < 30; i++) {
uint64_t val = i;
taosArrayPush(f, &val);
}
// empty source
iIntersection(src, rslt);
assert(taosArrayGetSize(rslt) == 0);
clearSourceArray(src);
clearFinalArray(rslt);
}
TEST_F(UtilEnv, intersectSimpleEmpty) {
clearSourceArray(src);
clearFinalArray(rslt);
iIntersection(src, rslt);
assert(taosArrayGetSize(rslt) == 0);
}
TEST_F(UtilEnv, intersect01) {
clearSourceArray(src);
clearFinalArray(rslt);
uint64_t arr1[] = {2, 3, 4, 5};
SArray * f = (SArray *)taosArrayGetP(src, 0);
for (int i = 0; i < sizeof(arr1) / sizeof(arr1[0]); i++) {
taosArrayPush(f, &arr1[i]);
}
uint64_t arr2[] = {1, 2, 3, 5};
f = (SArray *)taosArrayGetP(src, 1);
for (int i = 0; i < sizeof(arr2) / sizeof(arr2[0]); i++) {
taosArrayPush(f, &arr2[i]);
}
uint64_t arr3[] = {3, 5, 10, 11};
f = (SArray *)taosArrayGetP(src, 2);
for (int i = 0; i < sizeof(arr3) / sizeof(arr3[0]); i++) {
taosArrayPush(f, &arr3[i]);
}
iIntersection(src, rslt);
assert(taosArrayGetSize(rslt) == 2);
}
TEST_F(UtilEnv, intersect02) {
clearSourceArray(src);
clearFinalArray(rslt);
uint64_t arr1[] = {13, 14, 15};
SArray * f = (SArray *)taosArrayGetP(src, 0);
for (int i = 0; i < sizeof(arr1) / sizeof(arr1[0]); i++) {
taosArrayPush(f, &arr1[i]);
}
uint64_t arr2[] = {8, 10, 12, 13};
f = (SArray *)taosArrayGetP(src, 1);
for (int i = 0; i < sizeof(arr2) / sizeof(arr2[0]); i++) {
taosArrayPush(f, &arr2[i]);
}
uint64_t arr3[] = {9, 10, 11};
f = (SArray *)taosArrayGetP(src, 2);
for (int i = 0; i < sizeof(arr3) / sizeof(arr3[0]); i++) {
taosArrayPush(f, &arr3[i]);
}
iIntersection(src, rslt);
assert(taosArrayGetSize(rslt) == 0);
}
TEST_F(UtilEnv, 01union) {
clearSourceArray(src);
clearFinalArray(rslt);
uint64_t arr1[] = {13, 14, 15};
SArray * f = (SArray *)taosArrayGetP(src, 0);
for (int i = 0; i < sizeof(arr1) / sizeof(arr1[0]); i++) {
taosArrayPush(f, &arr1[i]);
}
iUnion(src, rslt);
assert(taosArrayGetSize(rslt) == 3);
}
TEST_F(UtilEnv, 02union) {
clearSourceArray(src);
clearFinalArray(rslt);
uint64_t arr1[] = {13, 14, 15};
SArray * f = (SArray *)taosArrayGetP(src, 0);
for (int i = 0; i < sizeof(arr1) / sizeof(arr1[0]); i++) {
taosArrayPush(f, &arr1[i]);
}
uint64_t arr2[] = {13, 14, 15};
f = (SArray *)taosArrayGetP(src, 1);
for (int i = 0; i < sizeof(arr2) / sizeof(arr2[0]); i++) {
taosArrayPush(f, &arr2[i]);
}
iUnion(src, rslt);
assert(taosArrayGetSize(rslt) == 3);
}
TEST_F(UtilEnv, 03union) {
clearSourceArray(src);
clearFinalArray(rslt);
uint64_t arr1[] = {13, 16, 18, 20};
SArray * f = (SArray *)taosArrayGetP(src, 0);
for (int i = 0; i < sizeof(arr1) / sizeof(arr1[0]); i++) {
taosArrayPush(f, &arr1[i]);
}
uint64_t arr2[] = {0, 12, 13, 20, 23};
f = (SArray *)taosArrayGetP(src, 1);
for (int i = 0; i < sizeof(arr2) / sizeof(arr2[0]); i++) {
taosArrayPush(f, &arr2[i]);
}
uint64_t arr3[] = {1, 12, 13, 16, 17};
f = (SArray *)taosArrayGetP(src, 2);
for (int i = 0; i < sizeof(arr3) / sizeof(arr3[0]); i++) {
taosArrayPush(f, &arr3[i]);
}
iUnion(src, rslt);
assert(taosArrayGetSize(rslt) == 9);
}

10
source/libs/nodes/src/nodesCloneFuncs.c
View File

@ -142,14 +142,6 @@ static SNode* functionNodeCopy(const SFunctionNode* pSrc, SFunctionNode* pDst) {
return (SNode*)pDst; return (SNode*)pDst;
} }
static SNode* columnRefNodeCopy(const SColumnRefNode* pSrc, SColumnRefNode* pDst) {
dataTypeCopy(&pSrc->dataType, &pDst->dataType);
COPY_SCALAR_FIELD(tupleId);
COPY_SCALAR_FIELD(slotId);
COPY_SCALAR_FIELD(columnId);
return (SNode*)pDst;
}
static SNode* targetNodeCopy(const STargetNode* pSrc, STargetNode* pDst) { static SNode* targetNodeCopy(const STargetNode* pSrc, STargetNode* pDst) {
COPY_SCALAR_FIELD(tupleId); COPY_SCALAR_FIELD(tupleId);
COPY_SCALAR_FIELD(slotId); COPY_SCALAR_FIELD(slotId);
@ -183,8 +175,6 @@ SNode* nodesCloneNode(const SNode* pNode) {
return logicConditionNodeCopy((const SLogicConditionNode*)pNode, (SLogicConditionNode*)pDst); return logicConditionNodeCopy((const SLogicConditionNode*)pNode, (SLogicConditionNode*)pDst);
case QUERY_NODE_FUNCTION: case QUERY_NODE_FUNCTION:
return functionNodeCopy((const SFunctionNode*)pNode, (SFunctionNode*)pDst); return functionNodeCopy((const SFunctionNode*)pNode, (SFunctionNode*)pDst);
case QUERY_NODE_COLUMN_REF:
return columnRefNodeCopy((const SColumnRefNode*)pNode, (SColumnRefNode*)pDst);
case QUERY_NODE_TARGET: case QUERY_NODE_TARGET:
return targetNodeCopy((const STargetNode*)pNode, (STargetNode*)pDst); return targetNodeCopy((const STargetNode*)pNode, (STargetNode*)pDst);
case QUERY_NODE_REAL_TABLE: case QUERY_NODE_REAL_TABLE:

26
source/libs/nodes/src/nodesCodeFuncs.c
View File

@ -55,8 +55,6 @@ static char* nodeName(ENodeType type) {
return "NodeList"; return "NodeList";
case QUERY_NODE_FILL: case QUERY_NODE_FILL:
return "Fill"; return "Fill";
case QUERY_NODE_COLUMN_REF:
return "ColumnRef";
case QUERY_NODE_TARGET: case QUERY_NODE_TARGET:
return "Target"; return "Target";
case QUERY_NODE_RAW_EXPR: case QUERY_NODE_RAW_EXPR:
@ -503,28 +501,6 @@ static int32_t groupingSetNodeToJson(const void* pObj, SJson* pJson) {
return code; return code;
} }
static const char* jkColumnRefDataType = "DataType";
static const char* jkColumnRefTupleId = "TupleId";
static const char* jkColumnRefSlotId = "SlotId";
static const char* jkColumnRefColumnId = "ColumnId";
static int32_t columnRefNodeToJson(const void* pObj, SJson* pJson) {
const SColumnRefNode* pNode = (const SColumnRefNode*)pObj;
int32_t code = tjsonAddObject(pJson, jkColumnRefDataType, dataTypeToJson, &pNode->dataType);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkColumnRefTupleId, pNode->tupleId);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkColumnRefSlotId, pNode->slotId);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkColumnRefColumnId, pNode->columnId);
}
return code;
}
static const char* jkTargetTupleId = "TupleId"; static const char* jkTargetTupleId = "TupleId";
static const char* jkTargetSlotId = "SlotId"; static const char* jkTargetSlotId = "SlotId";
static const char* jkTargetExpr = "Expr"; static const char* jkTargetExpr = "Expr";
@ -646,8 +622,6 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) {
case QUERY_NODE_INTERVAL_WINDOW: case QUERY_NODE_INTERVAL_WINDOW:
case QUERY_NODE_NODE_LIST: case QUERY_NODE_NODE_LIST:
case QUERY_NODE_FILL: case QUERY_NODE_FILL:
case QUERY_NODE_COLUMN_REF:
return columnRefNodeToJson(pObj, pJson);
case QUERY_NODE_TARGET: case QUERY_NODE_TARGET:
return targetNodeToJson(pObj, pJson); return targetNodeToJson(pObj, pJson);
case QUERY_NODE_RAW_EXPR: case QUERY_NODE_RAW_EXPR:

6
source/libs/nodes/src/nodesUtilFuncs.c
View File

@ -63,8 +63,6 @@ SNode* nodesMakeNode(ENodeType type) {
return makeNode(type, sizeof(SNodeListNode)); return makeNode(type, sizeof(SNodeListNode));
case QUERY_NODE_FILL: case QUERY_NODE_FILL:
return makeNode(type, sizeof(SFillNode)); return makeNode(type, sizeof(SFillNode));
case QUERY_NODE_COLUMN_REF:
return makeNode(type, sizeof(SColumnRefNode));
case QUERY_NODE_RAW_EXPR: case QUERY_NODE_RAW_EXPR:
return makeNode(type, sizeof(SRawExprNode)); return makeNode(type, sizeof(SRawExprNode));
case QUERY_NODE_SET_OPERATOR: case QUERY_NODE_SET_OPERATOR:
@ -93,6 +91,10 @@ SNode* nodesMakeNode(ENodeType type) {
return makeNode(type, sizeof(STableScanPhysiNode)); return makeNode(type, sizeof(STableScanPhysiNode));
case QUERY_NODE_PHYSICAL_PLAN_PROJECT: case QUERY_NODE_PHYSICAL_PLAN_PROJECT:
return makeNode(type, sizeof(SProjectPhysiNode)); return makeNode(type, sizeof(SProjectPhysiNode));
case QUERY_NODE_PHYSICAL_PLAN_JOIN:
return makeNode(type, sizeof(SJoinPhysiNode));
case QUERY_NODE_PHYSICAL_PLAN_AGG:
return makeNode(type, sizeof(SAggPhysiNode));
default: default:
break; break;
} }

346
source/libs/planner/src/plannerImpl.c
View File

@ -55,6 +55,9 @@ static EDealRes doRewriteExpr(SNode** pNode, void* pContext) {
SNode* pExpr; SNode* pExpr;
int32_t index = 0; int32_t index = 0;
FOREACH(pExpr, pCxt->pExprs) { FOREACH(pExpr, pCxt->pExprs) {
if (QUERY_NODE_GROUPING_SET == nodeType(pExpr)) {
pExpr = nodesListGetNode(((SGroupingSetNode*)pExpr)->pParameterList, 0);
}
if (nodesEqualNode(pExpr, *pNode)) { if (nodesEqualNode(pExpr, *pNode)) {
SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN);
CHECK_ALLOC(pCol, DEAL_RES_ERROR); CHECK_ALLOC(pCol, DEAL_RES_ERROR);
@ -406,33 +409,22 @@ typedef struct SPhysiPlanContext {
static int32_t getSlotKey(SNode* pNode, char* pKey) { static int32_t getSlotKey(SNode* pNode, char* pKey) {
if (QUERY_NODE_COLUMN == nodeType(pNode)) { if (QUERY_NODE_COLUMN == nodeType(pNode)) {
return sprintf(pKey, "%s.%s", ((SColumnNode*)pNode)->tableAlias, ((SColumnNode*)pNode)->colName); SColumnNode* pCol = (SColumnNode*)pNode;
} else { if ('\0' == pCol->tableAlias[0]) {
return sprintf(pKey, "%s", pCol->colName);
}
return sprintf(pKey, "%s.%s", pCol->tableAlias, pCol->colName);
}
return sprintf(pKey, "%s", ((SExprNode*)pNode)->aliasName); return sprintf(pKey, "%s", ((SExprNode*)pNode)->aliasName);
} }
}
static SNode* createColumnRef(SNode* pNode, int16_t tupleId, int16_t slotId) {
SColumnRefNode* pCol = (SColumnRefNode*)nodesMakeNode(QUERY_NODE_COLUMN_REF);
if (NULL == pCol) {
return NULL;
}
pCol->dataType = ((SExprNode*)pNode)->resType;
pCol->tupleId = tupleId;
pCol->slotId = slotId;
pCol->columnId = (QUERY_NODE_COLUMN == nodeType(pNode) ? ((SColumnNode*)pNode)->colId : -1);
return (SNode*)pCol;
}
static SNode* createSlotDesc(SPhysiPlanContext* pCxt, const SNode* pNode, int16_t slotId) { static SNode* createSlotDesc(SPhysiPlanContext* pCxt, const SNode* pNode, int16_t slotId) {
SSlotDescNode* pSlot = (SSlotDescNode*)nodesMakeNode(QUERY_NODE_SLOT_DESC); SSlotDescNode* pSlot = (SSlotDescNode*)nodesMakeNode(QUERY_NODE_SLOT_DESC);
CHECK_ALLOC(pSlot, NULL); CHECK_ALLOC(pSlot, NULL);
pSlot->slotId = slotId; pSlot->slotId = slotId;
pSlot->dataType = ((SExprNode*)pNode)->resType; pSlot->dataType = ((SExprNode*)pNode)->resType;
pSlot->srcTupleId = -1;
pSlot->srcSlotId = -1;
pSlot->reserve = false; pSlot->reserve = false;
pSlot->output = true; pSlot->output = false;
return (SNode*)pSlot; return (SNode*)pSlot;
} }
@ -443,17 +435,11 @@ static SNode* createTarget(SNode* pNode, int16_t tupleId, int16_t slotId) {
} }
pTarget->tupleId = tupleId; pTarget->tupleId = tupleId;
pTarget->slotId = slotId; pTarget->slotId = slotId;
pTarget->pExpr = nodesCloneNode(pNode); pTarget->pExpr = pNode;
if (NULL == pTarget->pExpr) {
nodesDestroyNode((SNode*)pTarget);
return NULL;
}
return (SNode*)pTarget; return (SNode*)pTarget;
} }
static int32_t addTupleDesc(SPhysiPlanContext* pCxt, SNodeList* pList, STupleDescNode* pTuple, SNodeList** pOutput) { static int32_t addTupleDesc(SPhysiPlanContext* pCxt, SNodeList* pList, STupleDescNode* pTuple) {
pTuple->tupleId = pCxt->nextTupleId++;
SHashObj* pHash = NULL; SHashObj* pHash = NULL;
if (NULL == pTuple->pSlots) { if (NULL == pTuple->pSlots) {
pTuple->pSlots = nodesMakeList(); pTuple->pSlots = nodesMakeList();
@ -469,11 +455,8 @@ static int32_t addTupleDesc(SPhysiPlanContext* pCxt, SNodeList* pList, STupleDes
pHash = taosArrayGetP(pCxt->pTupleHelper, pTuple->tupleId); pHash = taosArrayGetP(pCxt->pTupleHelper, pTuple->tupleId);
} }
*pOutput = nodesMakeList();
CHECK_ALLOC(*pOutput, TSDB_CODE_OUT_OF_MEMORY);
SNode* pNode = NULL; SNode* pNode = NULL;
int16_t slotId = 0; int16_t slotId = taosHashGetSize(pHash);
FOREACH(pNode, pList) { FOREACH(pNode, pList) {
SNode* pSlot = createSlotDesc(pCxt, pNode, slotId); SNode* pSlot = createSlotDesc(pCxt, pNode, slotId);
CHECK_ALLOC(pSlot, TSDB_CODE_OUT_OF_MEMORY); CHECK_ALLOC(pSlot, TSDB_CODE_OUT_OF_MEMORY);
@ -482,48 +465,50 @@ static int32_t addTupleDesc(SPhysiPlanContext* pCxt, SNodeList* pList, STupleDes
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
} }
SNode* pTarget = createTarget(pNode, pTuple->tupleId, slotId);
CHECK_ALLOC(pTarget, TSDB_CODE_OUT_OF_MEMORY);
if (TSDB_CODE_SUCCESS != nodesListAppend(*pOutput, pTarget)) {
nodesDestroyNode(pTarget);
return TSDB_CODE_OUT_OF_MEMORY;
}
SSlotIndex index = { .tupleId = pTuple->tupleId, .slotId = slotId }; SSlotIndex index = { .tupleId = pTuple->tupleId, .slotId = slotId };
char name[TSDB_TABLE_NAME_LEN + TSDB_COL_NAME_LEN]; char name[TSDB_TABLE_NAME_LEN + TSDB_COL_NAME_LEN];
int32_t len = getSlotKey(pNode, name); int32_t len = getSlotKey(pNode, name);
CHECK_CODE(taosHashPut(pHash, name, len, &index, sizeof(SSlotIndex)), TSDB_CODE_OUT_OF_MEMORY); CHECK_CODE(taosHashPut(pHash, name, len, &index, sizeof(SSlotIndex)), TSDB_CODE_OUT_OF_MEMORY);
SNode* pTarget = createTarget(pNode, pTuple->tupleId, slotId);
CHECK_ALLOC(pTarget, TSDB_CODE_OUT_OF_MEMORY);
REPLACE_NODE(pTarget);
++slotId; ++slotId;
} }
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
typedef struct STransformCxt { typedef struct SSetSlotIdCxt {
int32_t errCode; int32_t errCode;
SHashObj* pHash; SHashObj* pLeftHash;
} STransformCxt; SHashObj* pRightHash;
} SSetSlotIdCxt;
static EDealRes doTransform(SNode** pNode, void* pContext) { static EDealRes doSetSlotId(SNode* pNode, void* pContext) {
if (QUERY_NODE_COLUMN == nodeType(*pNode)) { if (QUERY_NODE_COLUMN == nodeType(pNode) && 0 != strcmp(((SColumnNode*)pNode)->colName, "*")) {
STransformCxt* pCxt = (STransformCxt*)pContext; SSetSlotIdCxt* pCxt = (SSetSlotIdCxt*)pContext;
char name[TSDB_TABLE_NAME_LEN + TSDB_COL_NAME_LEN]; char name[TSDB_TABLE_NAME_LEN + TSDB_COL_NAME_LEN];
int32_t len = getSlotKey(*pNode, name); int32_t len = getSlotKey(pNode, name);
SSlotIndex* pIndex = taosHashGet(pCxt->pHash, name, len); SSlotIndex* pIndex = taosHashGet(pCxt->pLeftHash, name, len);
if (NULL != pIndex) { if (NULL == pIndex) {
*pNode = createColumnRef(*pNode, pIndex->tupleId, pIndex->slotId); pIndex = taosHashGet(pCxt->pRightHash, name, len);
CHECK_ALLOC(*pNode, DEAL_RES_ERROR);
return DEAL_RES_IGNORE_CHILD;
} }
// pIndex is definitely not NULL, otherwise it is a bug
((SColumnNode*)pNode)->tupleId = pIndex->tupleId;
((SColumnNode*)pNode)->slotId = pIndex->slotId;
CHECK_ALLOC(pNode, DEAL_RES_ERROR);
return DEAL_RES_IGNORE_CHILD;
} }
return DEAL_RES_CONTINUE; return DEAL_RES_CONTINUE;
} }
static SNode* transformForPhysiPlan(SPhysiPlanContext* pCxt, int16_t tupleId, SNode* pNode) { static SNode* setNodeSlotId(SPhysiPlanContext* pCxt, int16_t leftTupleId, int16_t rightTupleId, SNode* pNode) {
SNode* pRes = nodesCloneNode(pNode); SNode* pRes = nodesCloneNode(pNode);
CHECK_ALLOC(pRes, NULL); CHECK_ALLOC(pRes, NULL);
STransformCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pHash = taosArrayGetP(pCxt->pTupleHelper, tupleId) }; SSetSlotIdCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pLeftHash = taosArrayGetP(pCxt->pTupleHelper, leftTupleId),
nodesRewriteNode(&pRes, doTransform, &cxt); .pRightHash = (rightTupleId < 0 ? NULL : taosArrayGetP(pCxt->pTupleHelper, rightTupleId)) };
nodesWalkNode(pRes, doSetSlotId, &cxt);
if (TSDB_CODE_SUCCESS != cxt.errCode) { if (TSDB_CODE_SUCCESS != cxt.errCode) {
nodesDestroyNode(pRes); nodesDestroyNode(pRes);
return NULL; return NULL;
@ -531,11 +516,12 @@ static SNode* transformForPhysiPlan(SPhysiPlanContext* pCxt, int16_t tupleId, SN
return pRes; return pRes;
} }
static SNodeList* transformListForPhysiPlan(SPhysiPlanContext* pCxt, int16_t tupleId, SNodeList* pList) { static SNodeList* setListSlotId(SPhysiPlanContext* pCxt, int16_t leftTupleId, int16_t rightTupleId, SNodeList* pList) {
SNodeList* pRes = nodesCloneList(pList); SNodeList* pRes = nodesCloneList(pList);
CHECK_ALLOC(pRes, NULL); CHECK_ALLOC(pRes, NULL);
STransformCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pHash = taosArrayGetP(pCxt->pTupleHelper, tupleId) }; SSetSlotIdCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pLeftHash = taosArrayGetP(pCxt->pTupleHelper, leftTupleId),
nodesRewriteList(pRes, doTransform, &cxt); .pRightHash = (rightTupleId < 0 ? NULL : taosArrayGetP(pCxt->pTupleHelper, rightTupleId)) };
nodesWalkList(pRes, doSetSlotId, &cxt);
if (TSDB_CODE_SUCCESS != cxt.errCode) { if (TSDB_CODE_SUCCESS != cxt.errCode) {
nodesDestroyList(pRes); nodesDestroyList(pRes);
return NULL; return NULL;
@ -543,22 +529,48 @@ static SNodeList* transformListForPhysiPlan(SPhysiPlanContext* pCxt, int16_t tup
return pRes; return pRes;
} }
static SPhysiNode* makePhysiNode(ENodeType type) { static SPhysiNode* makePhysiNode(SPhysiPlanContext* pCxt, ENodeType type) {
SPhysiNode* pPhysiNode = (SPhysiNode*)nodesMakeNode(type); SPhysiNode* pPhysiNode = (SPhysiNode*)nodesMakeNode(type);
if (NULL == pPhysiNode) { if (NULL == pPhysiNode) {
return NULL; return NULL;
} }
pPhysiNode->outputTuple.tupleId = pCxt->nextTupleId++;
pPhysiNode->outputTuple.type = QUERY_NODE_TUPLE_DESC; pPhysiNode->outputTuple.type = QUERY_NODE_TUPLE_DESC;
return pPhysiNode; return pPhysiNode;
} }
static int32_t initScanPhysiNode(SPhysiPlanContext* pCxt, SScanLogicNode* pScanLogicNode, SScanPhysiNode* pScanPhysiNode) { static int32_t setConditionsSlotId(SPhysiPlanContext* pCxt, const SLogicNode* pLogicNode, SPhysiNode* pPhysiNode) {
CHECK_CODE(addTupleDesc(pCxt, pScanLogicNode->pScanCols, &pScanPhysiNode->node.outputTuple, &pScanPhysiNode->pScanCols), TSDB_CODE_OUT_OF_MEMORY); if (NULL != pLogicNode->pConditions) {
pPhysiNode->pConditions = setNodeSlotId(pCxt, pPhysiNode->outputTuple.tupleId, -1, pLogicNode->pConditions);
if (NULL != pScanLogicNode->node.pConditions) { CHECK_ALLOC(pPhysiNode->pConditions, TSDB_CODE_OUT_OF_MEMORY);
pScanPhysiNode->node.pConditions = transformForPhysiPlan(pCxt, pScanPhysiNode->node.outputTuple.tupleId, pScanLogicNode->node.pConditions);
CHECK_ALLOC(pScanPhysiNode->node.pConditions, TSDB_CODE_OUT_OF_MEMORY);
} }
return TSDB_CODE_SUCCESS;
}
static int32_t setSlotOutput(SPhysiPlanContext* pCxt, SNodeList* pTargets, STupleDescNode* pTuple) {
SHashObj* pHash = taosArrayGetP(pCxt->pTupleHelper, pTuple->tupleId);
char name[TSDB_TABLE_NAME_LEN + TSDB_COL_NAME_LEN];
SNode* pNode;
FOREACH(pNode, pTargets) {
int32_t len = getSlotKey(pNode, name);
SSlotIndex* pIndex = taosHashGet(pHash, name, len);
((SSlotDescNode*)nodesListGetNode(pTuple->pSlots, pIndex->slotId))->output = true;
}
return TSDB_CODE_SUCCESS;
}
static int32_t initScanPhysiNode(SPhysiPlanContext* pCxt, SScanLogicNode* pScanLogicNode, SScanPhysiNode* pScanPhysiNode) {
if (NULL != pScanLogicNode->pScanCols) {
pScanPhysiNode->pScanCols = nodesCloneList(pScanLogicNode->pScanCols);
CHECK_ALLOC(pScanPhysiNode->pScanCols, TSDB_CODE_OUT_OF_MEMORY);
}
// Tuple describe also needs to be set without scanning column, such as SELECT COUNT(*) FROM t
CHECK_CODE(addTupleDesc(pCxt, pScanPhysiNode->pScanCols, &pScanPhysiNode->node.outputTuple), TSDB_CODE_OUT_OF_MEMORY);
CHECK_CODE(setConditionsSlotId(pCxt, (const SLogicNode*)pScanLogicNode, (SPhysiNode*)pScanPhysiNode), TSDB_CODE_OUT_OF_MEMORY);
CHECK_CODE(setSlotOutput(pCxt, pScanLogicNode->node.pTargets, &pScanPhysiNode->node.outputTuple), TSDB_CODE_OUT_OF_MEMORY);
pScanPhysiNode->uid = pScanLogicNode->pMeta->uid; pScanPhysiNode->uid = pScanLogicNode->pMeta->uid;
pScanPhysiNode->tableType = pScanLogicNode->pMeta->tableType; pScanPhysiNode->tableType = pScanLogicNode->pMeta->tableType;
@ -570,14 +582,14 @@ static int32_t initScanPhysiNode(SPhysiPlanContext* pCxt, SScanLogicNode* pScanL
} }
static SPhysiNode* createTagScanPhysiNode(SPhysiPlanContext* pCxt, SScanLogicNode* pScanLogicNode) { static SPhysiNode* createTagScanPhysiNode(SPhysiPlanContext* pCxt, SScanLogicNode* pScanLogicNode) {
STagScanPhysiNode* pTagScan = (STagScanPhysiNode*)makePhysiNode(QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN); STagScanPhysiNode* pTagScan = (STagScanPhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN);
CHECK_ALLOC(pTagScan, NULL); CHECK_ALLOC(pTagScan, NULL);
CHECK_CODE(initScanPhysiNode(pCxt, pScanLogicNode, (SScanPhysiNode*)pTagScan), (SPhysiNode*)pTagScan); CHECK_CODE(initScanPhysiNode(pCxt, pScanLogicNode, (SScanPhysiNode*)pTagScan), (SPhysiNode*)pTagScan);
return (SPhysiNode*)pTagScan; return (SPhysiNode*)pTagScan;
} }
static SPhysiNode* createTableScanPhysiNode(SPhysiPlanContext* pCxt, SScanLogicNode* pScanLogicNode) { static SPhysiNode* createTableScanPhysiNode(SPhysiPlanContext* pCxt, SScanLogicNode* pScanLogicNode) {
STableScanPhysiNode* pTableScan = (STableScanPhysiNode*)makePhysiNode(QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN); STableScanPhysiNode* pTableScan = (STableScanPhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN);
CHECK_ALLOC(pTableScan, NULL); CHECK_ALLOC(pTableScan, NULL);
CHECK_CODE(initScanPhysiNode(pCxt, pScanLogicNode, (SScanPhysiNode*)pTableScan), (SPhysiNode*)pTableScan); CHECK_CODE(initScanPhysiNode(pCxt, pScanLogicNode, (SScanPhysiNode*)pTableScan), (SPhysiNode*)pTableScan);
pTableScan->scanFlag = pScanLogicNode->scanFlag; pTableScan->scanFlag = pScanLogicNode->scanFlag;
@ -597,35 +609,205 @@ static SPhysiNode* createScanPhysiNode(SPhysiPlanContext* pCxt, SScanLogicNode*
default: default:
break; break;
} }
return NULL;
} }
static SPhysiNode* createProjectPhysiNode(SPhysiPlanContext* pCxt, SProjectLogicNode* pProjectLogicNode) { static SNodeList* createJoinOutputCols(SPhysiPlanContext* pCxt, STupleDescNode* pLeftTuple, STupleDescNode* pRightTuple) {
SProjectPhysiNode* pProject = (SProjectPhysiNode*)makePhysiNode(QUERY_NODE_PHYSICAL_PLAN_PROJECT); SNodeList* pCols = nodesMakeList();
CHECK_ALLOC(pCols, NULL);
SNode* pNode;
FOREACH(pNode, pLeftTuple->pSlots) {
SSlotDescNode* pSlot = (SSlotDescNode*)pNode;
SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN);
if (NULL == pCol) {
goto error;
}
pCol->node.resType = pSlot->dataType;
pCol->tupleId = pLeftTuple->tupleId;
pCol->slotId = pSlot->slotId;
pCol->colId = -1;
if (TSDB_CODE_SUCCESS != nodesListAppend(pCols, (SNode*)pCol)) {
goto error;
}
}
FOREACH(pNode, pRightTuple->pSlots) {
SSlotDescNode* pSlot = (SSlotDescNode*)pNode;
SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN);
if (NULL == pCol) {
goto error;
}
pCol->node.resType = pSlot->dataType;
pCol->tupleId = pRightTuple->tupleId;
pCol->slotId = pSlot->slotId;
pCol->colId = -1;
if (TSDB_CODE_SUCCESS != nodesListAppend(pCols, (SNode*)pCol)) {
goto error;
}
}
return pCols;
error:
nodesDestroyList(pCols);
return NULL;
}
static SPhysiNode* createJoinPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SJoinLogicNode* pJoinLogicNode) {
SJoinPhysiNode* pJoin = (SJoinPhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_JOIN);
CHECK_ALLOC(pJoin, NULL);
STupleDescNode* pLeftTuple = &((SPhysiNode*)nodesListGetNode(pChildren, 0))->outputTuple;
STupleDescNode* pRightTuple = &((SPhysiNode*)nodesListGetNode(pChildren, 1))->outputTuple;
pJoin->pOnConditions = setNodeSlotId(pCxt, pLeftTuple->tupleId, pRightTuple->tupleId, pJoinLogicNode->pOnConditions);
CHECK_ALLOC(pJoin->pOnConditions, (SPhysiNode*)pJoin);
pJoin->pTargets = createJoinOutputCols(pCxt, pLeftTuple, pRightTuple);
CHECK_ALLOC(pJoin->pTargets, (SPhysiNode*)pJoin);
CHECK_CODE(addTupleDesc(pCxt, pJoin->pTargets, &pJoin->node.outputTuple), (SPhysiNode*)pJoin);
CHECK_CODE(setConditionsSlotId(pCxt, (const SLogicNode*)pJoinLogicNode, (SPhysiNode*)pJoin), (SPhysiNode*)pJoin);
CHECK_CODE(setSlotOutput(pCxt, pJoinLogicNode->node.pTargets, &pJoin->node.outputTuple), (SPhysiNode*)pJoin);
return (SPhysiNode*)pJoin;
}
typedef struct SRewritePrecalcExprsCxt {
int32_t errCode;
int32_t planNodeId;
int32_t rewriteId;
SNodeList* pPrecalcExprs;
} SRewritePrecalcExprsCxt;
static EDealRes collectAndRewrite(SRewritePrecalcExprsCxt* pCxt, SNode** pNode) {
SNode* pExpr = nodesCloneNode(*pNode);
CHECK_ALLOC(pExpr, DEAL_RES_ERROR);
if (nodesListAppend(pCxt->pPrecalcExprs, pExpr)) {
nodesDestroyNode(pExpr);
return DEAL_RES_ERROR;
}
SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN);
if (NULL == pCol) {
nodesDestroyNode(pExpr);
return DEAL_RES_ERROR;
}
SExprNode* pToBeRewrittenExpr = (SExprNode*)(*pNode);
pCol->node.resType = pToBeRewrittenExpr->resType;
strcpy(pCol->colName, pToBeRewrittenExpr->aliasName);
nodesDestroyNode(*pNode);
*pNode = (SNode*)pCol;
return DEAL_RES_IGNORE_CHILD;
}
static EDealRes doRewritePrecalcExprs(SNode** pNode, void* pContext) {
SRewritePrecalcExprsCxt* pCxt = (SRewritePrecalcExprsCxt*)pContext;
switch (nodeType(*pNode)) {
case QUERY_NODE_OPERATOR:
case QUERY_NODE_LOGIC_CONDITION: {
return collectAndRewrite(pContext, pNode);
}
case QUERY_NODE_FUNCTION: {
if (!fmIsAggFunc(((SFunctionNode*)(*pNode))->funcId)) {
return collectAndRewrite(pContext, pNode);
}
}
default:
break;
}
return DEAL_RES_CONTINUE;
}
static int32_t rewritePrecalcExprs(SPhysiPlanContext* pCxt, SNodeList* pList, SNodeList** pPrecalcExprs, SNodeList** pRewrittenList) {
if (NULL == pList) {
return TSDB_CODE_SUCCESS;
}
if (NULL == *pPrecalcExprs) {
*pPrecalcExprs = nodesMakeList();
CHECK_ALLOC(*pPrecalcExprs, TSDB_CODE_OUT_OF_MEMORY);
}
if (NULL == *pRewrittenList) {
*pRewrittenList = nodesMakeList();
CHECK_ALLOC(*pRewrittenList, TSDB_CODE_OUT_OF_MEMORY);
}
SNode* pNode = NULL;
FOREACH(pNode, pList) {
SNode* pNew = NULL;
if (QUERY_NODE_GROUPING_SET == nodeType(pNode)) {
pNew = nodesCloneNode(nodesListGetNode(((SGroupingSetNode*)pNode)->pParameterList, 0));
} else {
pNew = nodesCloneNode(pNode);
}
CHECK_ALLOC(pNew, TSDB_CODE_OUT_OF_MEMORY);
CHECK_CODE(nodesListAppend(*pRewrittenList, pNew), TSDB_CODE_OUT_OF_MEMORY);
}
SRewritePrecalcExprsCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pPrecalcExprs = *pPrecalcExprs };
nodesRewriteList(*pRewrittenList, doRewritePrecalcExprs, &cxt);
if (0 == LIST_LENGTH(cxt.pPrecalcExprs)) {
nodesDestroyList(cxt.pPrecalcExprs);
*pPrecalcExprs = NULL;
}
return cxt.errCode;
}
static SPhysiNode* createAggPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SAggLogicNode* pAggLogicNode) {
SAggPhysiNode* pAgg = (SAggPhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_AGG);
CHECK_ALLOC(pAgg, NULL);
SNodeList* pPrecalcExprs = NULL;
SNodeList* pGroupKeys = NULL;
SNodeList* pAggFuncs = NULL;
CHECK_CODE(rewritePrecalcExprs(pCxt, pAggLogicNode->pGroupKeys, &pPrecalcExprs, &pGroupKeys), (SPhysiNode*)pAgg);
CHECK_CODE(rewritePrecalcExprs(pCxt, pAggLogicNode->pAggFuncs, &pPrecalcExprs, &pAggFuncs), (SPhysiNode*)pAgg);
STupleDescNode* pChildTupe = &(((SPhysiNode*)nodesListGetNode(pChildren, 0))->outputTuple);
// push down expression to outputTuple of child node
if (NULL != pPrecalcExprs) {
pAgg->pExprs = setListSlotId(pCxt, pChildTupe->tupleId, -1, pPrecalcExprs);
CHECK_ALLOC(pAgg->pExprs, (SPhysiNode*)pAgg);
CHECK_CODE(addTupleDesc(pCxt, pAgg->pExprs, pChildTupe), (SPhysiNode*)pAgg);
}
if (NULL != pGroupKeys) {
pAgg->pGroupKeys = setListSlotId(pCxt, pChildTupe->tupleId, -1, pGroupKeys);
CHECK_ALLOC(pAgg->pGroupKeys, (SPhysiNode*)pAgg);
CHECK_CODE(addTupleDesc(pCxt, pAgg->pGroupKeys, &pAgg->node.outputTuple), (SPhysiNode*)pAgg);
}
if (NULL != pAggFuncs) {
pAgg->pAggFuncs = setListSlotId(pCxt, pChildTupe->tupleId, -1, pAggFuncs);
CHECK_ALLOC(pAgg->pAggFuncs, (SPhysiNode*)pAgg);
CHECK_CODE(addTupleDesc(pCxt, pAgg->pAggFuncs, &pAgg->node.outputTuple), (SPhysiNode*)pAgg);
}
CHECK_CODE(setConditionsSlotId(pCxt, (const SLogicNode*)pAggLogicNode, (SPhysiNode*)pAgg), (SPhysiNode*)pAgg);
CHECK_CODE(setSlotOutput(pCxt, pAggLogicNode->node.pTargets, &pAgg->node.outputTuple), (SPhysiNode*)pAgg);
return (SPhysiNode*)pAgg;
}
static SPhysiNode* createProjectPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SProjectLogicNode* pProjectLogicNode) {
SProjectPhysiNode* pProject = (SProjectPhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_PROJECT);
CHECK_ALLOC(pProject, NULL); CHECK_ALLOC(pProject, NULL);
SNodeList* pProjections = transformListForPhysiPlan(pCxt, pProject->node.outputTuple.tupleId, pProjectLogicNode->pProjections); pProject->pProjections = setListSlotId(pCxt, ((SPhysiNode*)nodesListGetNode(pChildren, 0))->outputTuple.tupleId, -1, pProjectLogicNode->pProjections);
CHECK_ALLOC(pProjections, (SPhysiNode*)pProject); CHECK_ALLOC(pProject->pProjections, (SPhysiNode*)pProject);
CHECK_CODE(addTupleDesc(pCxt, pProjections, &pProject->node.outputTuple, &pProject->pProjections), (SPhysiNode*)pProject); CHECK_CODE(addTupleDesc(pCxt, pProject->pProjections, &pProject->node.outputTuple), (SPhysiNode*)pProject);
nodesDestroyList(pProjections);
if (NULL != pProjectLogicNode->node.pConditions) { CHECK_CODE(setConditionsSlotId(pCxt, (const SLogicNode*)pProjectLogicNode, (SPhysiNode*)pProject), (SPhysiNode*)pProject);
pProject->node.pConditions = transformForPhysiPlan(pCxt, pProject->node.outputTuple.tupleId, pProjectLogicNode->node.pConditions);
CHECK_ALLOC(pProject->node.pConditions, (SPhysiNode*)pProject);
}
return (SPhysiNode*)pProject; return (SPhysiNode*)pProject;
} }
static SPhysiNode* createPhysiNode(SPhysiPlanContext* pCxt, SLogicNode* pLogicPlan) { static SPhysiNode* createPhysiNode(SPhysiPlanContext* pCxt, SLogicNode* pLogicPlan) {
SNodeList* pChildern = nodesMakeList(); SNodeList* pChildren = nodesMakeList();
CHECK_ALLOC(pChildern, NULL); CHECK_ALLOC(pChildren, NULL);
SNode* pLogicChild; SNode* pLogicChild;
FOREACH(pLogicChild, pLogicPlan->pChildren) { FOREACH(pLogicChild, pLogicPlan->pChildren) {
SNode* pChildPhyNode = (SNode*)createPhysiNode(pCxt, (SLogicNode*)pLogicChild); SNode* pChildPhyNode = (SNode*)createPhysiNode(pCxt, (SLogicNode*)pLogicChild);
if (TSDB_CODE_SUCCESS != nodesListAppend(pChildern, pChildPhyNode)) { if (TSDB_CODE_SUCCESS != nodesListAppend(pChildren, pChildPhyNode)) {
pCxt->errCode = TSDB_CODE_OUT_OF_MEMORY; pCxt->errCode = TSDB_CODE_OUT_OF_MEMORY;
nodesDestroyList(pChildern); nodesDestroyList(pChildren);
return NULL; return NULL;
} }
} }
@ -636,23 +818,23 @@ static SPhysiNode* createPhysiNode(SPhysiPlanContext* pCxt, SLogicNode* pLogicPl
pPhyNode = createScanPhysiNode(pCxt, (SScanLogicNode*)pLogicPlan); pPhyNode = createScanPhysiNode(pCxt, (SScanLogicNode*)pLogicPlan);
break; break;
case QUERY_NODE_LOGIC_PLAN_JOIN: case QUERY_NODE_LOGIC_PLAN_JOIN:
pPhyNode = createJoinPhysiNode(pCxt, pChildren, (SJoinLogicNode*)pLogicPlan);
break; break;
case QUERY_NODE_LOGIC_PLAN_AGG: case QUERY_NODE_LOGIC_PLAN_AGG:
pPhyNode = createAggPhysiNode(pCxt, pChildren, (SAggLogicNode*)pLogicPlan);
break; break;
case QUERY_NODE_LOGIC_PLAN_PROJECT: case QUERY_NODE_LOGIC_PLAN_PROJECT:
pPhyNode = createProjectPhysiNode(pCxt, (SProjectLogicNode*)pLogicPlan); pPhyNode = createProjectPhysiNode(pCxt, pChildren, (SProjectLogicNode*)pLogicPlan);
break; break;
default: default:
break; break;
} }
if (NULL != pPhyNode) { pPhyNode->pChildren = pChildren;
pPhyNode->pChildren = pChildern;
SNode* pChild; SNode* pChild;
FOREACH(pChild, pPhyNode->pChildren) { FOREACH(pChild, pPhyNode->pChildren) {
((SPhysiNode*)pChild)->pParent = pPhyNode; ((SPhysiNode*)pChild)->pParent = pPhyNode;
} }
}
return pPhyNode; return pPhyNode;
} }

4
source/libs/planner/test/newPlannerTest.cpp
View File

@ -123,8 +123,8 @@ TEST_F(NewPlannerTest, simple) {
TEST_F(NewPlannerTest, groupBy) { TEST_F(NewPlannerTest, groupBy) {
setDatabase("root", "test"); setDatabase("root", "test");
bind("SELECT count(*) FROM t1"); // bind("SELECT count(*) FROM t1");
ASSERT_TRUE(run()); // ASSERT_TRUE(run());
bind("SELECT c1, count(*) FROM t1 GROUP BY c1"); bind("SELECT c1, count(*) FROM t1 GROUP BY c1");
ASSERT_TRUE(run()); ASSERT_TRUE(run());

7
source/libs/qcom/src/queryUtil.c
View File

@ -85,11 +85,10 @@ static void* pTaskQueue = NULL;
int32_t initTaskQueue() { int32_t initTaskQueue() {
double factor = 4.0; double factor = 4.0;
// todo
// int32_t numOfThreads = TMAX((int)(tsNumOfCores * tsNumOfThreadsPerCore / factor), 2);
int32_t numOfThreads = TMAX((int)(tsNumOfCores * 1.0f / factor), 2);
int32_t queueSize = 25000; //tsMaxConnections * 2; int32_t numOfThreads = TMAX((int)(tsNumOfCores * tsNumOfThreadsPerCore / factor), 2);
int32_t queueSize = tsMaxConnections * 2;
pTaskQueue = taosInitScheduler(queueSize, numOfThreads, "tsc"); pTaskQueue = taosInitScheduler(queueSize, numOfThreads, "tsc");
if (NULL == pTaskQueue) { if (NULL == pTaskQueue) {
qError("failed to init task queue"); qError("failed to init task queue");

10
source/libs/scalar/inc/filterInt.h
View File

@ -301,11 +301,11 @@ typedef struct SFilterInfo {
#define FILTER_GET_FIELD(i, id) (&((i)->fields[(id).type].fields[(id).idx])) #define FILTER_GET_FIELD(i, id) (&((i)->fields[(id).type].fields[(id).idx]))
#define FILTER_GET_COL_FIELD(i, idx) (&((i)->fields[FLD_TYPE_COLUMN].fields[idx])) #define FILTER_GET_COL_FIELD(i, idx) (&((i)->fields[FLD_TYPE_COLUMN].fields[idx]))
#define FILTER_GET_COL_FIELD_TYPE(fi) (((SColumnRefNode *)((fi)->desc))->dataType.type) #define FILTER_GET_COL_FIELD_TYPE(fi) (((SColumnNode *)((fi)->desc))->node.resType.type)
#define FILTER_GET_COL_FIELD_SIZE(fi) (((SColumnRefNode *)((fi)->desc))->dataType.bytes) #define FILTER_GET_COL_FIELD_SIZE(fi) (((SColumnNode *)((fi)->desc))->node.resType.bytes)
#define FILTER_GET_COL_FIELD_ID(fi) (((SColumnRefNode *)((fi)->desc))->columnId) #define FILTER_GET_COL_FIELD_ID(fi) (((SColumnNode *)((fi)->desc))->colId)
#define FILTER_GET_COL_FIELD_SLOT_ID(fi) (((SColumnRefNode *)((fi)->desc))->slotId) #define FILTER_GET_COL_FIELD_SLOT_ID(fi) (((SColumnNode *)((fi)->desc))->slotId)
#define FILTER_GET_COL_FIELD_DESC(fi) ((SColumnRefNode *)((fi)->desc)) #define FILTER_GET_COL_FIELD_DESC(fi) ((SColumnNode *)((fi)->desc))
#define FILTER_GET_COL_FIELD_DATA(fi, ri) (colDataGet(((SColumnInfoData *)(fi)->data), (ri))) #define FILTER_GET_COL_FIELD_DATA(fi, ri) (colDataGet(((SColumnInfoData *)(fi)->data), (ri)))
#define FILTER_GET_VAL_FIELD_TYPE(fi) (((SValueNode *)((fi)->desc))->node.resType.type) #define FILTER_GET_VAL_FIELD_TYPE(fi) (((SValueNode *)((fi)->desc))->node.resType.type)
#define FILTER_GET_VAL_FIELD_DATA(fi) ((char *)(fi)->data) #define FILTER_GET_VAL_FIELD_DATA(fi) ((char *)(fi)->data)

24
source/libs/scalar/src/filter.c
View File

@ -914,14 +914,14 @@ int32_t filterAddFieldFromNode(SFilterInfo *info, SNode *node, SFilterFieldId *f
FLT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); FLT_ERR_RET(TSDB_CODE_QRY_APP_ERROR);
} }
if (nodeType(node) != QUERY_NODE_COLUMN_REF && nodeType(node) != QUERY_NODE_VALUE && nodeType(node) != QUERY_NODE_NODE_LIST) { if (nodeType(node) != QUERY_NODE_COLUMN && nodeType(node) != QUERY_NODE_VALUE && nodeType(node) != QUERY_NODE_NODE_LIST) {
FLT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); FLT_ERR_RET(TSDB_CODE_QRY_APP_ERROR);
} }
int32_t type; int32_t type;
void *v; void *v;
if (nodeType(node) == QUERY_NODE_COLUMN_REF) { if (nodeType(node) == QUERY_NODE_COLUMN) {
type = FLD_TYPE_COLUMN; type = FLD_TYPE_COLUMN;
v = node; v = node;
} else { } else {
@ -1454,7 +1454,7 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options)
qDebug("COLUMN Field Num:%u", info->fields[FLD_TYPE_COLUMN].num); qDebug("COLUMN Field Num:%u", info->fields[FLD_TYPE_COLUMN].num);
for (uint32_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) { for (uint32_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) {
SFilterField *field = &info->fields[FLD_TYPE_COLUMN].fields[i]; SFilterField *field = &info->fields[FLD_TYPE_COLUMN].fields[i];
SColumnRefNode *refNode = (SColumnRefNode *)field->desc; SColumnNode *refNode = (SColumnNode *)field->desc;
qDebug("COL%d => [%d][%d]", i, refNode->tupleId, refNode->slotId); qDebug("COL%d => [%d][%d]", i, refNode->tupleId, refNode->slotId);
} }
@ -1483,7 +1483,7 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options)
char str[512] = {0}; char str[512] = {0};
SFilterField *left = FILTER_UNIT_LEFT_FIELD(info, unit); SFilterField *left = FILTER_UNIT_LEFT_FIELD(info, unit);
SColumnRefNode *refNode = (SColumnRefNode *)left->desc; SColumnNode *refNode = (SColumnNode *)left->desc;
if (unit->compare.optr >= 0 && unit->compare.optr <= OP_TYPE_JSON_CONTAINS){ if (unit->compare.optr >= 0 && unit->compare.optr <= OP_TYPE_JSON_CONTAINS){
len = sprintf(str, "UNIT[%d] => [%d][%d] %s [", i, refNode->tupleId, refNode->slotId, gOptrStr[unit->compare.optr].str); len = sprintf(str, "UNIT[%d] => [%d][%d] %s [", i, refNode->tupleId, refNode->slotId, gOptrStr[unit->compare.optr].str);
} }
@ -3487,7 +3487,7 @@ EDealRes fltReviseRewriter(SNode** pNode, void* pContext) {
return DEAL_RES_CONTINUE; return DEAL_RES_CONTINUE;
} }
if (QUERY_NODE_VALUE == nodeType(*pNode) || QUERY_NODE_NODE_LIST == nodeType(*pNode) || QUERY_NODE_COLUMN_REF == nodeType(*pNode)) { if (QUERY_NODE_VALUE == nodeType(*pNode) || QUERY_NODE_NODE_LIST == nodeType(*pNode) || QUERY_NODE_COLUMN == nodeType(*pNode)) {
return DEAL_RES_CONTINUE; return DEAL_RES_CONTINUE;
} }
@ -3510,7 +3510,7 @@ EDealRes fltReviseRewriter(SNode** pNode, void* pContext) {
return DEAL_RES_ERROR; return DEAL_RES_ERROR;
} }
if (QUERY_NODE_COLUMN_REF != nodeType(node->pLeft)) { if (QUERY_NODE_COLUMN != nodeType(node->pLeft)) {
stat->scalarMode = true; stat->scalarMode = true;
return DEAL_RES_CONTINUE; return DEAL_RES_CONTINUE;
} }
@ -3521,12 +3521,12 @@ EDealRes fltReviseRewriter(SNode** pNode, void* pContext) {
return DEAL_RES_CONTINUE; return DEAL_RES_CONTINUE;
} }
} else { } else {
if ((QUERY_NODE_COLUMN_REF != nodeType(node->pLeft)) && (QUERY_NODE_VALUE != nodeType(node->pLeft))) { if ((QUERY_NODE_COLUMN != nodeType(node->pLeft)) && (QUERY_NODE_VALUE != nodeType(node->pLeft))) {
stat->scalarMode = true; stat->scalarMode = true;
return DEAL_RES_CONTINUE; return DEAL_RES_CONTINUE;
} }
if ((QUERY_NODE_COLUMN_REF != nodeType(node->pRight)) && (QUERY_NODE_VALUE != nodeType(node->pRight))) { if ((QUERY_NODE_COLUMN != nodeType(node->pRight)) && (QUERY_NODE_VALUE != nodeType(node->pRight))) {
stat->scalarMode = true; stat->scalarMode = true;
return DEAL_RES_CONTINUE; return DEAL_RES_CONTINUE;
} }
@ -3536,7 +3536,7 @@ EDealRes fltReviseRewriter(SNode** pNode, void* pContext) {
return DEAL_RES_CONTINUE; return DEAL_RES_CONTINUE;
} }
if (QUERY_NODE_COLUMN_REF != nodeType(node->pLeft)) { if (QUERY_NODE_COLUMN != nodeType(node->pLeft)) {
SNode *t = node->pLeft; SNode *t = node->pLeft;
node->pLeft = node->pRight; node->pLeft = node->pRight;
node->pRight = t; node->pRight = t;
@ -3549,10 +3549,10 @@ EDealRes fltReviseRewriter(SNode** pNode, void* pContext) {
} }
if (OP_TYPE_IN != node->opType) { if (OP_TYPE_IN != node->opType) {
SColumnRefNode *refNode = (SColumnRefNode *)node->pLeft; SColumnNode *refNode = (SColumnNode *)node->pLeft;
SValueNode *valueNode = (SValueNode *)node->pRight; SValueNode *valueNode = (SValueNode *)node->pRight;
int32_t type = vectorGetConvertType(refNode->dataType.type, valueNode->node.resType.type); int32_t type = vectorGetConvertType(refNode->node.resType.type, valueNode->node.resType.type);
if (0 != type && type != refNode->dataType.type) { if (0 != type && type != refNode->node.resType.type) {
stat->scalarMode = true; stat->scalarMode = true;
return DEAL_RES_CONTINUE; return DEAL_RES_CONTINUE;
} }

12
source/libs/scalar/src/scalar.c
View File

@ -134,13 +134,13 @@ int32_t sclInitParam(SNode* node, SScalarParam *param, SScalarCtx *ctx, int32_t
break; break;
} }
case QUERY_NODE_COLUMN_REF: { case QUERY_NODE_COLUMN: {
if (NULL == ctx) { if (NULL == ctx) {
sclError("invalid node type for constant calculating, type:%d, ctx:%p", nodeType(node), ctx); sclError("invalid node type for constant calculating, type:%d, ctx:%p", nodeType(node), ctx);
SCL_ERR_RET(TSDB_CODE_QRY_APP_ERROR); SCL_ERR_RET(TSDB_CODE_QRY_APP_ERROR);
} }
SColumnRefNode *ref = (SColumnRefNode *)node; SColumnNode *ref = (SColumnNode *)node;
if (ref->slotId >= taosArrayGetSize(ctx->pSrc->pDataBlock)) { if (ref->slotId >= taosArrayGetSize(ctx->pSrc->pDataBlock)) {
sclError("column ref slotId is too big, slodId:%d, dataBlockSize:%d", ref->slotId, (int32_t)taosArrayGetSize(ctx->pSrc->pDataBlock)); sclError("column ref slotId is too big, slodId:%d, dataBlockSize:%d", ref->slotId, (int32_t)taosArrayGetSize(ctx->pSrc->pDataBlock));
SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
@ -281,7 +281,8 @@ int32_t sclExecFuncion(SFunctionNode *node, SScalarCtx *ctx, SScalarParam *outpu
SScalarFuncExecFuncs ffpSet = {0}; SScalarFuncExecFuncs ffpSet = {0};
int32_t code = fmGetScalarFuncExecFuncs(node->funcId, &ffpSet); int32_t code = fmGetScalarFuncExecFuncs(node->funcId, &ffpSet);
if (code) { if (code) {
sclError( "fmGetFuncExecFuncs failed, funcId:%d, code:%s", node->funcId, tstrerror(code)); sclError(
"fmGetFuncExecFuncs failed, funcId:%d, code:%s", node->funcId, tstrerror(code));
SCL_ERR_RET(code); SCL_ERR_RET(code);
} }
@ -299,7 +300,8 @@ int32_t sclExecFuncion(SFunctionNode *node, SScalarCtx *ctx, SScalarParam *outpu
for (int32_t i = 0; i < rowNum; ++i) { for (int32_t i = 0; i < rowNum; ++i) {
code = (*ffpSet.process)(params, node->pParameterList->length, output); code = (*ffpSet.process)(params, node->pParameterList->length, output);
if (code) { if (code) {
sclError( "scalar function exec failed, funcId:%d, code:%s", node->funcId, tstrerror(code)); sclError(
"scalar function exec failed, funcId:%d, code:%s", node->funcId, tstrerror(code));
SCL_ERR_JRET(code); SCL_ERR_JRET(code);
} }
@ -599,7 +601,7 @@ EDealRes sclWalkOperator(SNode* pNode, void* pContext) {
EDealRes sclCalcWalker(SNode* pNode, void* pContext) { EDealRes sclCalcWalker(SNode* pNode, void* pContext) {
if (QUERY_NODE_VALUE == nodeType(pNode) || QUERY_NODE_NODE_LIST == nodeType(pNode) || QUERY_NODE_COLUMN_REF == nodeType(pNode)) { if (QUERY_NODE_VALUE == nodeType(pNode) || QUERY_NODE_NODE_LIST == nodeType(pNode) || QUERY_NODE_COLUMN == nodeType(pNode)) {
return DEAL_RES_CONTINUE; return DEAL_RES_CONTINUE;
} }

18
source/libs/scalar/test/filter/filterTests.cpp
View File

@ -76,15 +76,15 @@ void flttMakeValueNode(SNode **pNode, int32_t dataType, void *value) {
} }
void flttMakeColRefNode(SNode **pNode, SSDataBlock **block, int32_t dataType, int32_t dataBytes, int32_t rowNum, void *value) { void flttMakeColRefNode(SNode **pNode, SSDataBlock **block, int32_t dataType, int32_t dataBytes, int32_t rowNum, void *value) {
SNode *node = nodesMakeNode(QUERY_NODE_COLUMN_REF); SNode *node = nodesMakeNode(QUERY_NODE_COLUMN);
SColumnRefNode *rnode = (SColumnRefNode *)node; SColumnNode *rnode = (SColumnNode *)node;
rnode->dataType.type = dataType; rnode->node.resType.type = dataType;
rnode->dataType.bytes = dataBytes; rnode->node.resType.bytes = dataBytes;
rnode->tupleId = 0; rnode->tupleId = 0;
if (NULL == block) { if (NULL == block) {
rnode->slotId = 2; rnode->slotId = 2;
rnode->columnId = 55; rnode->colId = 55;
*pNode = (SNode *)rnode; *pNode = (SNode *)rnode;
return; return;
@ -120,7 +120,7 @@ void flttMakeColRefNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in
taosArrayPush(res->pDataBlock, &idata); taosArrayPush(res->pDataBlock, &idata);
rnode->slotId = 2; rnode->slotId = 2;
rnode->columnId = 55; rnode->colId = 55;
*block = res; *block = res;
} else { } else {
@ -137,7 +137,7 @@ void flttMakeColRefNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in
res->info.numOfCols++; res->info.numOfCols++;
rnode->slotId = idx; rnode->slotId = idx;
rnode->columnId = 55 + idx; rnode->colId = 55 + idx;
} }
*pNode = (SNode *)rnode; *pNode = (SNode *)rnode;
@ -259,7 +259,7 @@ TEST(columnTest, smallint_column_greater_double_value) {
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
SColumnDataAgg stat = {0}; SColumnDataAgg stat = {0};
stat.colId = ((SColumnRefNode *)pLeft)->columnId; stat.colId = ((SColumnNode *)pLeft)->colId;
stat.max = 10; stat.max = 10;
stat.min = 5; stat.min = 5;
stat.numOfNull = 0; stat.numOfNull = 0;
@ -310,7 +310,7 @@ TEST(columnTest, int_column_greater_smallint_value) {
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
SColumnDataAgg stat = {0}; SColumnDataAgg stat = {0};
stat.colId = ((SColumnRefNode *)pLeft)->columnId; stat.colId = ((SColumnNode *)pLeft)->colId;
stat.max = 10; stat.max = 10;
stat.min = 5; stat.min = 5;
stat.numOfNull = 0; stat.numOfNull = 0;

12
source/libs/scalar/test/scalar/scalarTests.cpp
View File

@ -75,10 +75,10 @@ void scltMakeValueNode(SNode **pNode, int32_t dataType, void *value) {
} }
void scltMakeColRefNode(SNode **pNode, SSDataBlock **block, int32_t dataType, int32_t dataBytes, int32_t rowNum, void *value) { void scltMakeColRefNode(SNode **pNode, SSDataBlock **block, int32_t dataType, int32_t dataBytes, int32_t rowNum, void *value) {
SNode *node = nodesMakeNode(QUERY_NODE_COLUMN_REF); SNode *node = nodesMakeNode(QUERY_NODE_COLUMN);
SColumnRefNode *rnode = (SColumnRefNode *)node; SColumnNode *rnode = (SColumnNode *)node;
rnode->dataType.type = dataType; rnode->node.resType.type = dataType;
rnode->dataType.bytes = dataBytes; rnode->node.resType.bytes = dataBytes;
rnode->tupleId = 0; rnode->tupleId = 0;
if (NULL == *block) { if (NULL == *block) {
@ -111,7 +111,7 @@ void scltMakeColRefNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in
taosArrayPush(res->pDataBlock, &idata); taosArrayPush(res->pDataBlock, &idata);
rnode->slotId = 2; rnode->slotId = 2;
rnode->columnId = 55; rnode->colId = 55;
*block = res; *block = res;
} else { } else {
@ -126,7 +126,7 @@ void scltMakeColRefNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in
taosArrayPush(res->pDataBlock, &idata); taosArrayPush(res->pDataBlock, &idata);
rnode->slotId = idx; rnode->slotId = idx;
rnode->columnId = 55 + idx; rnode->colId = 55 + idx;
} }
*pNode = (SNode *)rnode; *pNode = (SNode *)rnode;

Some files were not shown because too many files have changed in this diff Show More