diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 074014b0d7..21b8b661df 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -193,6 +193,7 @@ endif(${BUILD_WITH_TRAFT}) # LIBUV if(${BUILD_WITH_UV}) + add_compile_options(-Wno-sign-compare) add_subdirectory(libuv) endif(${BUILD_WITH_UV}) diff --git a/include/common/tep.h b/include/common/tep.h index 6d09d0ce51..956cf6ec36 100644 --- a/include/common/tep.h +++ b/include/common/tep.h @@ -23,7 +23,7 @@ typedef struct SBlockOrderInfo { // bool hasNull; } SBlockOrderInfo; -int taosGetFqdnPortFromEp(const char *ep, uint16_t defaultPort, SEp *pEp); +int taosGetFqdnPortFromEp(const char *ep, SEp *pEp); void addEpIntoEpSet(SEpSet *pEpSet, const char *fqdn, uint16_t port); bool isEpsetEqual(const SEpSet *s1, const SEpSet *s2); diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 8bce28d34c..b54058d489 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -20,31 +20,48 @@ extern "C" { #endif +#include "tcfg.h" #include "tdef.h" +// cluster +extern char tsFirst[]; +extern char tsSecond[]; +extern char tsLocalFqdn[]; +extern char tsLocalEp[]; +extern uint16_t tsServerPort; +extern int32_t tsVersion; +extern int32_t tsStatusInterval; +extern bool tsEnableTelemetryReporting; + // common -extern int8_t tsDaylight; -extern int32_t tsCompressMsgSize; -extern int32_t tsCompressColData; -extern int32_t tsMaxNumOfDistinctResults; -extern char tsTempDir[]; -extern int tsCompatibleModel; // 2.0 compatible model -extern int8_t tsEnableSlaveQuery; -extern int8_t tsEnableAdjustMaster; -extern int8_t tsPrintAuth; -extern int64_t tsTickPerDay[3]; +extern int32_t tsRpcTimer; +extern int32_t tsRpcMaxTime; +extern bool tsRpcForceTcp; // all commands go to tcp protocol if this is enabled +extern int32_t tsMaxConnections; +extern int32_t tsMaxShellConns; +extern int32_t tsShellActivityTimer; +extern int32_t tsMaxTmrCtrl; +extern float tsNumOfThreadsPerCore; +extern int32_t tsNumOfCommitThreads; +extern float tsRatioOfQueryCores; +extern int32_t tsCompressMsgSize; +extern int32_t tsCompressColData; +extern int32_t tsMaxNumOfDistinctResults; +extern int32_t tsCompatibleModel; +extern bool tsEnableSlaveQuery; +extern bool tsPrintAuth; +extern int64_t tsTickPerDay[3]; // query buffer management extern int32_t tsQueryBufferSize; // maximum allowed usage buffer size in MB for each data node during query processing extern int64_t tsQueryBufferSizeBytes; // maximum allowed usage buffer size in byte for each data node -extern int32_t tsRetrieveBlockingModel; // retrieve threads will be blocked -extern int8_t tsKeepOriginalColumnName; -extern int8_t tsDeadLockKillQuery; +extern bool tsRetrieveBlockingModel; // retrieve threads will be blocked +extern bool tsKeepOriginalColumnName; +extern bool tsDeadLockKillQuery; // client extern int32_t tsMaxWildCardsLen; extern int32_t tsMaxRegexStringLen; -extern int8_t tsTscEnableRecordSql; extern int32_t tsMaxNumOfOrderedResults; extern int32_t tsMinSlidingTime; extern int32_t tsMinIntervalTime; @@ -55,10 +72,12 @@ extern float tsStreamComputDelayRatio; // the delayed computing ration of the extern int32_t tsProjectExecInterval; extern int64_t tsMaxRetentWindow; -// system info - -extern uint32_t tsVersion; - +// build info +extern char version[]; +extern char compatible_version[]; +extern char gitinfo[]; +extern char gitinfoOfInternal[]; +extern char buildinfo[]; // lossy extern char tsLossyColumns[]; @@ -68,15 +87,19 @@ extern uint32_t tsMaxRange; extern uint32_t tsCurRange; extern char tsCompressor[]; +// tfs +extern int32_t tsDiskCfgNum; +extern SDiskCfg tsDiskCfg[]; #define NEEDTO_COMPRESSS_MSG(size) (tsCompressMsgSize != -1 && (size) > tsCompressMsgSize) -void taosInitGlobalCfg(); -int32_t taosCfgDynamicOptions(char *msg); -bool taosCheckBalanceCfgOptions(const char *option, int32_t *vnodeId, int32_t *dnodeId); -void taosAddDataDir(int index, char *v1, int level, int primary); -void taosReadDataDirCfg(char *v1, char *v2, char *v3); -void taosPrintDataDirCfg(); +int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDir, const char *envFile, + const char *apolloUrl, bool tsc); +int32_t taosInitCfg(const char *cfgDir, const char *envFile, const char *apolloUrl, bool tsc); +void taosCleanupCfg(); +void taosCfgDynamicOptions(const char *option, const char *value); + +struct SConfig *taosGetCfg(); #ifdef __cplusplus } diff --git a/include/common/tmsg.h b/include/common/tmsg.h index ae3586e735..9ab78f3c96 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -656,9 +656,9 @@ int32_t tDeserializeSRetrieveFuncRsp(void* buf, int32_t bufLen, SRetrieveFuncRsp typedef struct { int32_t statusInterval; int64_t checkTime; // 1970-01-01 00:00:00.000 - char timezone[TSDB_TIMEZONE_LEN]; // tsTimezone - char locale[TSDB_LOCALE_LEN]; // tsLocale - char charset[TSDB_LOCALE_LEN]; // tsCharset + char timezone[TD_TIMEZONE_LEN]; // tsTimezone + char locale[TD_LOCALE_LEN]; // tsLocale + char charset[TD_LOCALE_LEN]; // tsCharset } SClusterCfg; typedef struct { diff --git a/include/common/ttszip.h b/include/common/ttszip.h index 38699ae791..d83b0066d5 100644 --- a/include/common/ttszip.h +++ b/include/common/ttszip.h @@ -73,9 +73,9 @@ typedef struct STSGroupBlockInfoEx { } STSGroupBlockInfoEx; typedef struct STSBuf { - FILE* f; - char path[PATH_MAX]; - uint32_t fileSize; + TdFilePtr pFile; + char path[PATH_MAX]; + uint32_t fileSize; // todo use array STSGroupBlockInfoEx* pData; diff --git a/include/dnode/mgmt/dnode.h b/include/dnode/mgmt/dnode.h index 99fe2457a8..a929f7a3fb 100644 --- a/include/dnode/mgmt/dnode.h +++ b/include/dnode/mgmt/dnode.h @@ -25,27 +25,12 @@ extern "C" { /* ------------------------ TYPES EXPOSED ---------------- */ typedef struct SDnode SDnode; -/* ------------------------ Environment ------------------ */ -typedef struct { - int32_t sver; - int32_t numOfCores; - uint16_t numOfCommitThreads; - bool enableTelem; - bool printAuth; - char timezone[TSDB_TIMEZONE_LEN]; - char locale[TSDB_LOCALE_LEN]; - char charset[TSDB_LOCALE_LEN]; - char buildinfo[64]; - char gitinfo[48]; -} SDnodeEnvCfg; - /** * @brief Initialize the environment * - * @param pOption Option of the environment * @return int32_t 0 for success and -1 for failure */ -int32_t dndInit(const SDnodeEnvCfg *pCfg); +int32_t dndInit(); /** * @brief clear the environment @@ -56,11 +41,6 @@ void dndCleanup(); /* ------------------------ SDnode ----------------------- */ typedef struct { int32_t numOfSupportVnodes; - int32_t statusInterval; - float numOfThreadsPerCore; - float ratioOfQueryCores; - int32_t maxShellConns; - int32_t shellActivityTimer; uint16_t serverPort; char dataDir[TSDB_FILENAME_LEN]; char localEp[TSDB_EP_LEN]; diff --git a/include/dnode/mnode/mnode.h b/include/dnode/mnode/mnode.h index 01855f6d92..2680bb83ed 100644 --- a/include/dnode/mnode/mnode.h +++ b/include/dnode/mnode/mnode.h @@ -44,26 +44,12 @@ typedef struct SMnodeLoad { int64_t compStorage; } SMnodeLoad; -typedef struct SMnodeCfg { - int32_t sver; - bool enableTelem; - bool printAuth; - int32_t statusInterval; - int32_t shellActivityTimer; - char *timezone; - char *locale; - char *charset; - char *buildinfo; - char *gitinfo; -} SMnodeCfg; - typedef struct { int32_t dnodeId; int64_t clusterId; int8_t replica; int8_t selfIndex; SReplica replicas[TSDB_MAX_REPLICA]; - SMnodeCfg cfg; SDnode *pDnode; PutReqToMWriteQFp putReqToMWriteQFp; PutReqToMReadQFp putReqToMReadQFp; diff --git a/include/libs/nodes/nodes.h b/include/libs/nodes/nodes.h index f09ffa8926..d5958e1b9c 100644 --- a/include/libs/nodes/nodes.h +++ b/include/libs/nodes/nodes.h @@ -62,7 +62,6 @@ typedef enum ENodeType { QUERY_NODE_NODE_LIST, QUERY_NODE_FILL, QUERY_NODE_RAW_EXPR, // Only be used in parser module. - QUERY_NODE_COLUMN_REF, QUERY_NODE_TARGET, QUERY_NODE_TUPLE_DESC, QUERY_NODE_SLOT_DESC, @@ -81,7 +80,9 @@ typedef enum ENodeType { // physical plan node QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN, QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN, - QUERY_NODE_PHYSICAL_PLAN_PROJECT + QUERY_NODE_PHYSICAL_PLAN_PROJECT, + QUERY_NODE_PHYSICAL_PLAN_JOIN, + QUERY_NODE_PHYSICAL_PLAN_AGG } ENodeType; /** diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index 40684f2b26..d8896501ad 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -69,8 +69,6 @@ typedef struct SSlotDescNode { ENodeType type; int16_t slotId; SDataType dataType; - int16_t srcTupleId; - int16_t srcSlotId; bool reserve; bool output; } SSlotDescNode; @@ -115,6 +113,20 @@ typedef struct SProjectPhysiNode { SNodeList* pProjections; } SProjectPhysiNode; +typedef struct SJoinPhysiNode { + SPhysiNode node; + EJoinType joinType; + SNode* pOnConditions; // in or out tuple ? + SNodeList* pTargets; +} SJoinPhysiNode; + +typedef struct SAggPhysiNode { + SPhysiNode node; + SNodeList* pExprs; // these are expression list of group_by_clause and parameter expression of aggregate function + SNodeList* pGroupKeys; // SColumnRefNode list + SNodeList* pAggFuncs; +} SAggPhysiNode; + #ifdef __cplusplus } #endif diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index 7bc8617db1..c54e1b3ad0 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -58,15 +58,17 @@ typedef struct SColumnNode { char tableAlias[TSDB_TABLE_NAME_LEN]; char colName[TSDB_COL_NAME_LEN]; SNode* pProjectRef; -} SColumnNode; - -typedef struct SColumnRefNode { - ENodeType type; - SDataType dataType; int16_t tupleId; int16_t slotId; - int16_t columnId; -} SColumnRefNode; +} SColumnNode; + +// typedef struct SColumnRefNode { +// ENodeType type; +// SDataType dataType; +// int16_t tupleId; +// int16_t slotId; +// int16_t columnId; +// } SColumnRefNode; typedef struct STargetNode { ENodeType type; diff --git a/include/libs/transport/trpc.h b/include/libs/transport/trpc.h index f1bab81573..5e3860822e 100644 --- a/include/libs/transport/trpc.h +++ b/include/libs/transport/trpc.h @@ -86,7 +86,7 @@ typedef struct SRpcInit { int32_t rpcInit(); void rpcCleanup(); -void * rpcOpen(const SRpcInit *pRpc); +void *rpcOpen(const SRpcInit *pRpc); void rpcClose(void *); void * rpcMallocCont(int contLen); void rpcFreeCont(void *pCont); diff --git a/include/libs/wal/wal.h b/include/libs/wal/wal.h index 45f1d88c30..f90dbb97fe 100644 --- a/include/libs/wal/wal.h +++ b/include/libs/wal/wal.h @@ -120,8 +120,8 @@ typedef struct SWal { int32_t fsyncSeq; // meta SWalVer vers; - int64_t writeLogTfd; - int64_t writeIdxTfd; + TdFilePtr pWriteLogTFile; + TdFilePtr pWriteIdxTFile; int32_t writeCur; SArray *fileInfoSet; // status @@ -138,8 +138,8 @@ typedef struct SWal { typedef struct SWalReadHandle { SWal *pWal; - int64_t readLogTfd; - int64_t readIdxTfd; + TdFilePtr pReadLogTFile; + TdFilePtr pReadIdxTFile; int64_t curFileFirstVer; int64_t curVersion; int64_t capacity; diff --git a/include/os/os.h b/include/os/os.h index 023d2b0470..d96f7efb51 100644 --- a/include/os/os.h +++ b/include/os/os.h @@ -59,6 +59,7 @@ extern "C" { #include "osEndian.h" #include "osEnv.h" #include "osFile.h" +#include "osLocale.h" #include "osLz4.h" #include "osMath.h" #include "osMemory.h" @@ -73,6 +74,7 @@ extern "C" { #include "osThread.h" #include "osTime.h" #include "osTimer.h" +#include "osTimezone.h" void osInit(); diff --git a/include/os/osDef.h b/include/os/osDef.h index 07d360a7c0..339bf13343 100644 --- a/include/os/osDef.h +++ b/include/os/osDef.h @@ -191,6 +191,10 @@ extern "C" { #define TD_DIRSEP "/" #endif +#define TD_LOCALE_LEN 64 +#define TD_CHARSET_LEN 64 +#define TD_TIMEZONE_LEN 96 + #ifdef __cplusplus } #endif diff --git a/include/os/osDir.h b/include/os/osDir.h index 01ec24f235..e328c42063 100644 --- a/include/os/osDir.h +++ b/include/os/osDir.h @@ -23,7 +23,7 @@ extern "C" { void taosRemoveDir(const char *dirname); int32_t taosDirExist(char *dirname); int32_t taosMkDir(const char *dirname); -void taosRemoveOldFiles(char *dirname, int32_t keepDays); +void taosRemoveOldFiles(const char *dirname, int32_t keepDays); int32_t taosExpandDir(const char *dirname, char *outname, int32_t maxlen); int32_t taosRealPath(char *dirname, int32_t maxlen); diff --git a/include/os/osEnv.h b/include/os/osEnv.h index 2745520ae3..4ac073f6c2 100644 --- a/include/os/osEnv.h +++ b/include/os/osEnv.h @@ -16,28 +16,37 @@ #ifndef _TD_OS_ENV_H_ #define _TD_OS_ENV_H_ +#include "osSysinfo.h" + #ifdef __cplusplus extern "C" { #endif -extern char tsOsName[]; +extern char tsOsName[]; +extern char tsTimezone[]; +extern char tsCharset[]; +extern char tsLocale[]; +extern int8_t tsDaylight; +extern bool tsEnableCoreFile; +extern int64_t tsPageSize; +extern int64_t tsOpenMax; +extern int64_t tsStreamMax; +extern int32_t tsNumOfCores; +extern int32_t tsTotalMemoryMB; +extern char configDir[]; extern char tsDataDir[]; extern char tsLogDir[]; extern char tsTempDir[]; -extern char configDir[]; -extern struct SDiskSpace tsLogSpace; -extern struct SDiskSpace tsTempSpace; -extern struct SDiskSpace tsDataSpace; +extern SDiskSpace tsDataSpace; +extern SDiskSpace tsLogSpace; +extern SDiskSpace tsTempSpace; -void taosUpdateLogSpace(); -void taosUpdateTempSpace(); -void taosUpdateDataSpace(); -bool taosLogSpaceAvailable(); -bool taosTmpSpaceAvailable(); -bool taosDataSpaceAvailable(); -void taosUpdateAllSpace(); +void osInit(); +void osUpdate(); +bool osLogSpaceAvailable(); +void osSetTimezone(const char *timezone); #ifdef __cplusplus } diff --git a/include/os/osFile.h b/include/os/osFile.h index ac399fa3be..59492f6694 100644 --- a/include/os/osFile.h +++ b/include/os/osFile.h @@ -22,49 +22,63 @@ extern "C" { #include "osSocket.h" -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) -typedef int32_t FileFd; -#else -typedef int32_t FileFd; +#ifndef ALLOW_FORBID_FUNC + #define open OPEN_FUNC_TAOS_FORBID + #define fopen FOPEN_FUNC_TAOS_FORBID + // #define close CLOSE_FUNC_TAOS_FORBID + // #define fclose FCLOSE_FUNC_TAOS_FORBID #endif -#define FD_INITIALIZER ((int32_t)-1) - #ifndef PATH_MAX #define PATH_MAX 256 #endif -int32_t taosLockFile(FileFd fd); -int32_t taosUnLockFile(FileFd fd); - -int32_t taosUmaskFile(FileFd fd); - +typedef struct TdFile *TdFilePtr; + +#define TD_FILE_CTEATE 0x0001 +#define TD_FILE_WRITE 0x0002 +#define TD_FILE_READ 0x0004 +#define TD_FILE_TRUNC 0x0008 +#define TD_FILE_APPEND 0x0010 +#define TD_FILE_TEXT 0x0020 +#define TD_FILE_AUTO_DEL 0x0040 +#define TD_FILE_EXCL 0x0080 + +int32_t taosLockFile(TdFilePtr pFile); +int32_t taosUnLockFile(TdFilePtr pFile); + +int32_t taosUmaskFile(int32_t maskVal); + int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime); -int32_t taosFStatFile(FileFd fd, int64_t *size, int32_t *mtime); - -FileFd taosOpenFileWrite(const char *path); -FileFd taosOpenFileCreateWrite(const char *path); -FileFd taosOpenFileCreateWriteTrunc(const char *path); -FileFd taosOpenFileCreateWriteAppend(const char *path); -FileFd taosOpenFileRead(const char *path); -FileFd taosOpenFileReadWrite(const char *path); - -int64_t taosLSeekFile(FileFd fd, int64_t offset, int32_t whence); -int32_t taosFtruncateFile(FileFd fd, int64_t length); -int32_t taosFsyncFile(FileFd fd); - -int64_t taosReadFile(FileFd fd, void *buf, int64_t count); -int64_t taosWriteFile(FileFd fd, const void *buf, int64_t count); - -void taosCloseFile(FileFd fd); - +int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime); + +TdFilePtr taosOpenFile(const char *path,int32_t tdFileOptions); + +int64_t taosLSeekFile(TdFilePtr pFile, int64_t offset, int32_t whence); +int32_t taosFtruncateFile(TdFilePtr pFile, int64_t length); +int32_t taosFsyncFile(TdFilePtr pFile); + +int64_t taosReadFile(TdFilePtr pFile, void *buf, int64_t count); +int64_t taosPReadFile(TdFilePtr pFile, void *buf, int64_t count, int64_t offset); +int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count); +void taosFprintfFile(TdFilePtr pFile, const char *format, ...); +size_t taosGetLineFile(TdFilePtr pFile, char ** __restrict__ ptrBuf); +int32_t taosEOFFile(TdFilePtr pFile); + +int64_t taosCloseFile(TdFilePtr *ppFile); + int32_t taosRenameFile(const char *oldName, const char *newName); int64_t taosCopyFile(const char *from, const char *to); - + void taosGetTmpfilePath(const char *inputTmpDir, const char *fileNamePrefix, char *dstPath); + +int64_t taosSendFile(SocketFd fdDst, TdFilePtr pFileSrc, int64_t *offset, int64_t size); +int64_t taosFSendFile(TdFilePtr pFileOut, TdFilePtr pFileIn, int64_t *offset, int64_t size); -int64_t taosSendFile(SocketFd dfd, FileFd sfd, int64_t *offset, int64_t size); -int64_t taosFSendFile(FILE *outfile, FILE *infile, int64_t *offset, int64_t size); +void *taosMmapReadOnlyFile(TdFilePtr pFile, int64_t length); +bool taosValidFile(TdFilePtr pFile); + +int taosGetErrorFile(TdFilePtr pFile); #ifdef __cplusplus } diff --git a/include/common/ttimezone.h b/include/os/osLocale.h similarity index 70% rename from include/common/ttimezone.h rename to include/os/osLocale.h index cc50a29d57..6e313eb8cd 100644 --- a/include/common/ttimezone.h +++ b/include/os/osLocale.h @@ -13,17 +13,22 @@ * along with this program. If not, see . */ -#ifndef _TD_COMMON_TIMEZONE_H_ -#define _TD_COMMON_TIMEZONE_H_ +#ifndef _TD_OS_LOCALE_H_ +#define _TD_OS_LOCALE_H_ + +#include "os.h" +#include "osString.h" #ifdef __cplusplus extern "C" { #endif -void tsSetTimeZone(); +char *taosCharsetReplace(char *charsetstr); +void taosGetSystemLocale(char *outLocale, char *outCharset); +void taosSetSystemLocale(const char *inLocale, const char *inCharSet); #ifdef __cplusplus } #endif -#endif /*_TD_COMMON_TIMEZONE_H_*/ +#endif /*_TD_OS_LOCALE_H_*/ diff --git a/include/os/osString.h b/include/os/osString.h index 582411d444..88160dd69e 100644 --- a/include/os/osString.h +++ b/include/os/osString.h @@ -45,7 +45,6 @@ int32_t taosUcs4ToMbs(void *ucs4, int32_t ucs4_max_len, char *mbs); bool taosMbsToUcs4(const char *mbs, size_t mbs_len, char *ucs4, int32_t ucs4_max_len, int32_t *len); int32_t tasoUcs4Compare(void *f1_ucs4, void *f2_ucs4, int32_t bytes, int8_t ncharSize); bool taosValidateEncodec(const char *encodec); -char * taosCharsetReplace(char *charsetstr); #ifdef __cplusplus } diff --git a/include/os/osSysinfo.h b/include/os/osSysinfo.h index 6ab2a104df..9f9061d243 100644 --- a/include/os/osSysinfo.h +++ b/include/os/osSysinfo.h @@ -21,9 +21,6 @@ extern "C" { #endif #include "os.h" - -#define TSDB_LOCALE_LEN 64 -#define TSDB_TIMEZONE_LEN 96 typedef struct { int64_t total; @@ -31,22 +28,11 @@ typedef struct { int64_t avail; } SDiskSize; -typedef struct SDiskSpace { +typedef struct { int64_t reserved; SDiskSize size; } SDiskSpace; -extern int64_t tsPageSize; -extern int64_t tsOpenMax; -extern int64_t tsStreamMax; -extern int32_t tsNumOfCores; -extern int32_t tsTotalMemoryMB; -extern char tsTimezone[]; -extern char tsLocale[]; -extern char tsCharset[]; // default encode string - - - int32_t taosGetDiskSize(char *dataDir, SDiskSize *diskSize); int32_t taosGetCpuCores(); void taosGetSystemInfo(); @@ -58,7 +44,6 @@ void taosGetDisk(); bool taosGetCpuUsage(float *sysCpuUsage, float *procCpuUsage); bool taosGetProcMemory(float *memoryUsedMB); bool taosGetSysMemory(float *memoryUsedMB); -void taosPrintOsInfo(); int taosSystem(const char *cmd); void taosKillSystem(); int32_t taosGetSystemUUID(char *uid, int32_t uidlen); diff --git a/include/common/tlocale.h b/include/os/osTimezone.h similarity index 75% rename from include/common/tlocale.h rename to include/os/osTimezone.h index d809c75d67..ff015ef0b1 100644 --- a/include/common/tlocale.h +++ b/include/os/osTimezone.h @@ -13,17 +13,18 @@ * along with this program. If not, see . */ -#ifndef _TD_COMMON_LOCALE_H_ -#define _TD_COMMON_LOCALE_H_ +#ifndef _TD_OS_TIMEZONE_H_ +#define _TD_OS_TIMEZONE_H_ #ifdef __cplusplus extern "C" { #endif -void tsSetLocale(); +void taosGetSystemTimezone(char *outTimezone); +void taosSetSystemTimezone(const char *inTimezone, char *outTimezone, int8_t *outDaylight); #ifdef __cplusplus } -#endif /*_TD_COMMON_LOCALE_H_*/ - #endif + +#endif /*_TD_OS_TIMEZONE_H_*/ diff --git a/include/common/tcfg.h b/include/util/tcfg.h similarity index 100% rename from include/common/tcfg.h rename to include/util/tcfg.h diff --git a/include/libs/config/config.h b/include/util/tconfig.h similarity index 87% rename from include/libs/config/config.h rename to include/util/tconfig.h index 3c9ec1e3bf..2eb98a3ba8 100644 --- a/include/libs/config/config.h +++ b/include/util/tconfig.h @@ -42,7 +42,6 @@ typedef enum { CFG_DTYPE_INT64, CFG_DTYPE_FLOAT, CFG_DTYPE_STRING, - CFG_DTYPE_IPSTR, CFG_DTYPE_DIR, CFG_DTYPE_LOCALE, CFG_DTYPE_CHARSET, @@ -52,6 +51,7 @@ typedef enum { typedef struct SConfigItem { ECfgSrcType stype; ECfgDataType dtype; + bool tsc; char *name; union { bool bval; @@ -80,14 +80,14 @@ int32_t cfgGetSize(SConfig *pCfg); SConfigItem *cfgIterate(SConfig *pCfg, SConfigItem *pIter); void cfgCancelIterate(SConfig *pCfg, SConfigItem *pIter); SConfigItem *cfgGetItem(SConfig *pCfg, const char *name); +int32_t cfgSetItem(SConfig *pCfg, const char *name, const char *value, ECfgSrcType stype); -int32_t cfgAddBool(SConfig *pCfg, const char *name, bool defaultVal); -int32_t cfgAddInt32(SConfig *pCfg, const char *name, int32_t defaultVal, int64_t minval, int64_t maxval); -int32_t cfgAddInt64(SConfig *pCfg, const char *name, int64_t defaultVal, int64_t minval, int64_t maxval); -int32_t cfgAddFloat(SConfig *pCfg, const char *name, float defaultVal, double minval, double maxval); -int32_t cfgAddString(SConfig *pCfg, const char *name, const char *defaultVal); -int32_t cfgAddIpStr(SConfig *pCfg, const char *name, const char *defaultVa); -int32_t cfgAddDir(SConfig *pCfg, const char *name, const char *defaultVal); +int32_t cfgAddBool(SConfig *pCfg, const char *name, bool defaultVal, bool tsc); +int32_t cfgAddInt32(SConfig *pCfg, const char *name, int32_t defaultVal, int64_t minval, int64_t maxval, bool tsc); +int32_t cfgAddInt64(SConfig *pCfg, const char *name, int64_t defaultVal, int64_t minval, int64_t maxval, bool tsc); +int32_t cfgAddFloat(SConfig *pCfg, const char *name, float defaultVal, double minval, double maxval, bool tsc); +int32_t cfgAddString(SConfig *pCfg, const char *name, const char *defaultVal, bool tsc); +int32_t cfgAddDir(SConfig *pCfg, const char *name, const char *defaultVal, bool tsc); int32_t cfgAddLocale(SConfig *pCfg, const char *name, const char *defaultVal); int32_t cfgAddCharset(SConfig *pCfg, const char *name, const char *defaultVal); int32_t cfgAddTimezone(SConfig *pCfg, const char *name, const char *defaultVal); @@ -95,7 +95,7 @@ int32_t cfgAddTimezone(SConfig *pCfg, const char *name, const char *defaultVal); const char *cfgStypeStr(ECfgSrcType type); const char *cfgDtypeStr(ECfgDataType type); -void cfgDumpCfg(SConfig *pCfg); +void cfgDumpCfg(SConfig *pCfg, bool tsc, bool dump); #ifdef __cplusplus } diff --git a/include/util/tfile.h b/include/util/tfile.h index d3813051a4..59953de861 100644 --- a/include/util/tfile.h +++ b/include/util/tfile.h @@ -30,20 +30,20 @@ void tfCleanup(); // the same syntax as UNIX standard open/close/read/write // but FD is int64_t and will never be reused -int64_t tfOpenRead(const char *pathname); -int64_t tfOpenReadWrite(const char *pathname); -int64_t tfOpenCreateWrite(const char *pathname); -int64_t tfOpenCreateWriteAppend(const char *pathname); +// int64_t tfOpenRead(const char *pathname); +// int64_t tfOpenReadWrite(const char *pathname); +// int64_t tfOpenCreateWrite(const char *pathname); +// int64_t tfOpenCreateWriteAppend(const char *pathname); -int64_t tfClose(int64_t tfd); -int64_t tfWrite(int64_t tfd, void *buf, int64_t count); -int64_t tfRead(int64_t tfd, void *buf, int64_t count); -int64_t tfPread(int64_t tfd, void *buf, int64_t count, int64_t offset); -int32_t tfFsync(int64_t tfd); -bool tfValid(int64_t tfd); -int64_t tfLseek(int64_t tfd, int64_t offset, int32_t whence); -int32_t tfFtruncate(int64_t tfd, int64_t length); -void * tfMmapReadOnly(int64_t tfd, int64_t length); +// int64_t tfClose(int64_t tfd); +// int64_t tfWrite(int64_t tfd, void *buf, int64_t count); +// int64_t tfRead(int64_t tfd, void *buf, int64_t count); +// int64_t tfPread(int64_t tfd, void *buf, int64_t count, int64_t offset); +// int32_t tfFsync(int64_t tfd); +// bool tfValid(int64_t tfd); +// int64_t tfLseek(int64_t tfd, int64_t offset, int32_t whence); +// int32_t tfFtruncate(int64_t tfd, int64_t length); +// void * tfMmapReadOnly(int64_t tfd, int64_t length); #ifdef __cplusplus } #endif diff --git a/include/util/tnote.h b/include/util/tnote.h deleted file mode 100644 index e613ec7e41..0000000000 --- a/include/util/tnote.h +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_UTIL_NOTE_H -#define _TD_UTIL_NOTE_H - -#ifdef __cplusplus -extern "C" { -#endif - -#define MAX_NOTE_LINE_SIZE 66000 -#define NOTE_FILE_NAME_LEN 300 - -typedef struct { - int32_t fileNum; - int32_t maxLines; - int32_t lines; - int32_t flag; - int32_t fd; - int32_t openInProgress; - char name[NOTE_FILE_NAME_LEN]; - pthread_mutex_t mutex; -} SNoteObj; - -extern SNoteObj tsHttpNote; -extern SNoteObj tsTscNote; -extern SNoteObj tsInfoNote; - -int32_t taosInitNotes(); -void taosNotePrint(SNoteObj* pNote, const char* const format, ...); -void taosNotePrintBuffer(SNoteObj* pNote, char* buffer, int32_t len); - -#define nPrintHttp(...) \ - if (tsHttpEnableRecordSql) { \ - taosNotePrint(&tsHttpNote, __VA_ARGS__); \ - } - -#define nPrintTsc(...) \ - if (tsTscEnableRecordSql) { \ - taosNotePrint(&tsTscNote, __VA_ARGS__); \ - } - -#define nInfo(buffer, len) \ - if (tscEmbeddedInUtil == 1) { \ - taosNotePrintBuffer(&tsInfoNote, buffer, len); \ - } - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_UTIL_NOTE_H*/ diff --git a/source/client/CMakeLists.txt b/source/client/CMakeLists.txt index c793825556..3210c0c6de 100644 --- a/source/client/CMakeLists.txt +++ b/source/client/CMakeLists.txt @@ -8,7 +8,7 @@ target_include_directories( target_link_libraries( taos INTERFACE api - PRIVATE os util common transport parser planner catalog scheduler function qcom config + PRIVATE os util common transport parser planner catalog scheduler function qcom ) if(${BUILD_TEST}) diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index 60eb0200a7..523921634b 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -32,7 +32,7 @@ extern "C" { #include "tmsgtype.h" #include "trpc.h" -#include "config.h" +#include "tconfig.h" #define CHECK_CODE_GOTO(expr, label) \ do { \ @@ -253,13 +253,6 @@ int hbAddConnInfo(SAppHbMgr* pAppHbMgr, SClientHbKey connKey, void* key, void* v // --- mq void hbMgrInitMqHbRspHandle(); - -// config -int32_t tscInitLog(const char *cfgDir, const char *envFile, const char *apolloUrl); -int32_t tscInitCfg(const char *cfgDir, const char *envFile, const char *apolloUrl); - -extern SConfig *tscCfg; - #ifdef __cplusplus } #endif diff --git a/source/client/src/clientCfg.c b/source/client/src/clientCfg.c deleted file mode 100644 index 22b1407a15..0000000000 --- a/source/client/src/clientCfg.c +++ /dev/null @@ -1,208 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "clientInt.h" -#include "ulog.h" - -// todo refact -SConfig *tscCfg; - -static int32_t tscLoadCfg(SConfig *pConfig, const char *inputCfgDir, const char *envFile, const char *apolloUrl) { - char cfgDir[PATH_MAX] = {0}; - char cfgFile[PATH_MAX + 100] = {0}; - - taosExpandDir(inputCfgDir, cfgDir, PATH_MAX); - snprintf(cfgFile, sizeof(cfgFile), "%s" TD_DIRSEP "taos.cfg", cfgDir); - - if (cfgLoad(pConfig, CFG_STYPE_APOLLO_URL, apolloUrl) != 0) { - uError("failed to load from apollo url:%s since %s\n", apolloUrl, terrstr()); - return -1; - } - - if (cfgLoad(pConfig, CFG_STYPE_CFG_FILE, cfgFile) != 0) { - if (cfgLoad(pConfig, CFG_STYPE_CFG_FILE, cfgDir) != 0) { - uError("failed to load from config file:%s since %s\n", cfgFile, terrstr()); - return -1; - } - } - - if (cfgLoad(pConfig, CFG_STYPE_ENV_FILE, envFile) != 0) { - uError("failed to load from env file:%s since %s\n", envFile, terrstr()); - return -1; - } - - if (cfgLoad(pConfig, CFG_STYPE_ENV_VAR, NULL) != 0) { - uError("failed to load from global env variables since %s\n", terrstr()); - return -1; - } - - return 0; -} - -static int32_t tscAddLogCfg(SConfig *pCfg) { - if (cfgAddDir(pCfg, "logDir", "/var/log/taos") != 0) return -1; - if (cfgAddBool(pCfg, "asyncLog", 1) != 0) return -1; - if (cfgAddInt32(pCfg, "numOfLogLines", 10000000, 1000, 2000000000) != 0) return -1; - if (cfgAddInt32(pCfg, "logKeepDays", 0, -365000, 365000) != 0) return -1; - if (cfgAddInt32(pCfg, "debugFlag", 0, 0, 255) != 0) return -1; - if (cfgAddInt32(pCfg, "cDebugFlag", 0, 0, 255) != 0) return -1; - if (cfgAddInt32(pCfg, "jniDebugFlag", 0, 0, 255) != 0) return -1; - if (cfgAddInt32(pCfg, "tmrDebugFlag", 0, 0, 255) != 0) return -1; - if (cfgAddInt32(pCfg, "uDebugFlag", 0, 0, 255) != 0) return -1; - if (cfgAddInt32(pCfg, "rpcDebugFlag", 0, 0, 255) != 0) return -1; - return 0; -} - -static int32_t tscSetLogCfg(SConfig *pCfg) { - tstrncpy(tsLogDir, cfgGetItem(pCfg, "logDir")->str, PATH_MAX); - tsAsyncLog = cfgGetItem(pCfg, "asyncLog")->bval; - tsNumOfLogLines = cfgGetItem(pCfg, "numOfLogLines")->i32; - tsLogKeepDays = cfgGetItem(pCfg, "logKeepDays")->i32; - cDebugFlag = cfgGetItem(pCfg, "cDebugFlag")->i32; - jniDebugFlag = cfgGetItem(pCfg, "jniDebugFlag")->i32; - tmrDebugFlag = cfgGetItem(pCfg, "tmrDebugFlag")->i32; - uDebugFlag = cfgGetItem(pCfg, "uDebugFlag")->i32; - rpcDebugFlag = cfgGetItem(pCfg, "rpcDebugFlag")->i32; - - int32_t debugFlag = cfgGetItem(pCfg, "debugFlag")->i32; - taosSetAllDebugFlag(debugFlag); - return 0; -} - -int32_t tscInitLog(const char *cfgDir, const char *envFile, const char *apolloUrl) { - if (tsLogInited) return 0; - - SConfig *pCfg = cfgInit(); - if (pCfg == NULL) return -1; - - if (tscAddLogCfg(pCfg) != 0) { - printf("failed to add log cfg since %s\n", terrstr()); - cfgCleanup(pCfg); - return -1; - } - - if (tscLoadCfg(pCfg, cfgDir, envFile, apolloUrl) != 0) { - printf("failed to load log cfg since %s\n", terrstr()); - cfgCleanup(pCfg); - return -1; - } - - if (tscSetLogCfg(pCfg) != 0) { - printf("failed to set log cfg since %s\n", terrstr()); - cfgCleanup(pCfg); - return -1; - } - - const int32_t maxLogFileNum = 10; - if (taosInitLog("taoslog", maxLogFileNum) != 0) { - printf("failed to init log file since %s\n", terrstr()); - cfgCleanup(pCfg); - return -1; - } - - cfgDumpCfg(pCfg); - cfgCleanup(pCfg); - return 0; -} - -static int32_t tscAddEpCfg(SConfig *pCfg) { - char defaultFqdn[TSDB_FQDN_LEN] = {0}; - if (taosGetFqdn(defaultFqdn) != 0) { - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - if (cfgAddString(pCfg, "fqdn", defaultFqdn) != 0) return -1; - - int32_t defaultServerPort = 6030; - if (cfgAddInt32(pCfg, "serverPort", defaultServerPort, 1, 65056) != 0) return -1; - - char defaultFirstEp[TSDB_EP_LEN] = {0}; - char defaultSecondEp[TSDB_EP_LEN] = {0}; - snprintf(defaultFirstEp, TSDB_EP_LEN, "%s:%d", defaultFqdn, defaultServerPort); - snprintf(defaultSecondEp, TSDB_EP_LEN, "%s:%d", defaultFqdn, defaultServerPort); - if (cfgAddString(pCfg, "firstEp", defaultFirstEp) != 0) return -1; - if (cfgAddString(pCfg, "secondEp", defaultSecondEp) != 0) return -1; - - return 0; -} - -static int32_t tscAddCfg(SConfig *pCfg) { - if (tscAddEpCfg(pCfg) != 0) return -1; - - - // if (cfgAddString(pCfg, "buildinfo", buildinfo) != 0) return -1; - // if (cfgAddString(pCfg, "gitinfo", gitinfo) != 0) return -1; - // if (cfgAddString(pCfg, "version", version) != 0) return -1; - - // if (cfgAddDir(pCfg, "dataDir", tsDataDir) != 0) return -1; - if (cfgAddTimezone(pCfg, "timezone", "") != 0) return -1; - if (cfgAddLocale(pCfg, "locale", "") != 0) return -1; - if (cfgAddCharset(pCfg, "charset", "") != 0) return -1; - if (cfgAddInt32(pCfg, "numOfCores", 1, 1, 100000) != 0) return -1; - if (cfgAddInt32(pCfg, "numOfCommitThreads", 4, 1, 1000) != 0) return -1; - // if (cfgAddBool(pCfg, "telemetryReporting", 0) != 0) return -1; - if (cfgAddBool(pCfg, "enableCoreFile", 0) != 0) return -1; - // if (cfgAddInt32(pCfg, "supportVnodes", 256, 0, 65536) != 0) return -1; - if (cfgAddInt32(pCfg, "statusInterval", 1, 1, 30) != 0) return -1; - if (cfgAddFloat(pCfg, "numOfThreadsPerCore", 1, 0, 10) != 0) return -1; - if (cfgAddFloat(pCfg, "ratioOfQueryCores", 1, 0, 5) != 0) return -1; - if (cfgAddInt32(pCfg, "shellActivityTimer", 3, 1, 120) != 0) return -1; - - if (cfgAddInt32(pCfg, "maxConnections", 50000, 1, 100000) != 0) return -1; - return 0; -} - -int32_t tscCheckCfg(SConfig *pCfg) { - bool enableCore = cfgGetItem(pCfg, "enableCoreFile")->bval; - taosSetCoreDump(enableCore); - - - - return 0; -} - -SConfig *tscInitCfgImp(const char *cfgDir, const char *envFile, const char *apolloUrl) { - SConfig *pCfg = cfgInit(); - if (pCfg == NULL) return NULL; - - if (tscAddCfg(pCfg) != 0) { - uError("failed to init tsc cfg since %s", terrstr()); - cfgCleanup(pCfg); - return NULL; - } - - if (tscLoadCfg(pCfg, cfgDir, envFile, apolloUrl) != 0) { - printf("failed to load tsc cfg since %s\n", terrstr()); - cfgCleanup(pCfg); - return NULL; - } - - if (tscCheckCfg(pCfg) != 0) { - uError("failed to check cfg since %s", terrstr()); - cfgCleanup(pCfg); - return NULL; - } - - cfgDumpCfg(pCfg); - return pCfg; -} - -int32_t tscInitCfg(const char *cfgDir, const char *envFile, const char *apolloUrl) { - tscCfg = tscInitCfgImp(cfgDir, envFile, apolloUrl); - if (tscCfg == NULL) return -1; - - return 0; -} \ No newline at end of file diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 70788ec2c9..7ffb5dcd44 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -22,11 +22,9 @@ #include "tcache.h" #include "tglobal.h" #include "tmsg.h" -#include "tnote.h" #include "tref.h" #include "trpc.h" #include "ttime.h" -#include "ttimezone.h" #define TSC_VAR_NOT_RELEASE 1 #define TSC_VAR_RELEASED 0 @@ -95,10 +93,10 @@ void *openTransporter(const char *user, const char *auth, int32_t numOfThread) { rpcInit.numOfThreads = numOfThread; rpcInit.cfp = processMsgFromServer; rpcInit.pfp = persistConnForSpecificMsg; - rpcInit.sessions = cfgGetItem(tscCfg, "maxConnections")->i32; + rpcInit.sessions = tsMaxConnections; rpcInit.connType = TAOS_CONN_CLIENT; rpcInit.user = (char *)user; - rpcInit.idleTime = cfgGetItem(tscCfg, "shellActivityTimer")->i32 * 1000; + rpcInit.idleTime = tsShellActivityTimer * 1000; rpcInit.ckey = "key"; rpcInit.spi = 1; rpcInit.secret = (char *)auth; @@ -214,17 +212,16 @@ void taos_init_imp(void) { deltaToUtcInitOnce(); - if (tscInitLog(configDir, NULL, NULL) != 0) { + if (taosCreateLog("taoslog", 10, configDir, NULL, NULL, 1) != 0) { tscInitRes = -1; return; } - if (tscInitCfg(configDir, NULL, NULL) != 0) { + if (taosInitCfg(configDir, NULL, NULL, 1) != 0) { tscInitRes = -1; return; } - taosInitNotes(); initMsgHandleFp(); initQueryModuleMsgHandle(); @@ -298,7 +295,7 @@ int taos_options_imp(TSDB_OPTION option, const char *str) { assert(cfg != NULL); size_t len = strlen(str); - if (len == 0 || len > TSDB_LOCALE_LEN) { + if (len == 0 || len > TD_LOCALE_LEN) { tscInfo("Invalid locale:%s, use default", str); return -1; } @@ -316,7 +313,7 @@ int taos_options_imp(TSDB_OPTION option, const char *str) { return -1; } - tstrncpy(tsLocale, defaultLocale, TSDB_LOCALE_LEN); + tstrncpy(tsLocale, defaultLocale, TD_LOCALE_LEN); } // set the user specified locale @@ -330,7 +327,7 @@ int taos_options_imp(TSDB_OPTION option, const char *str) { tscInfo("failed to set locale:%s, current locale:%s", str, tsLocale); } - tstrncpy(tsLocale, locale, TSDB_LOCALE_LEN); + tstrncpy(tsLocale, locale, TD_LOCALE_LEN); char *charset = strrchr(tsLocale, sep); if (charset != NULL) { @@ -345,7 +342,7 @@ int taos_options_imp(TSDB_OPTION option, const char *str) { tscInfo("charset changed from %s to %s", tsCharset, charset); } - tstrncpy(tsCharset, charset, TSDB_LOCALE_LEN); + tstrncpy(tsCharset, charset, TD_LOCALE_LEN); cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION; } else { @@ -369,7 +366,7 @@ int taos_options_imp(TSDB_OPTION option, const char *str) { assert(cfg != NULL); size_t len = strlen(str); - if (len == 0 || len > TSDB_LOCALE_LEN) { + if (len == 0 || len > TD_LOCALE_LEN) { tscInfo("failed to set charset:%s", str); return -1; } @@ -382,7 +379,7 @@ int taos_options_imp(TSDB_OPTION option, const char *str) { tscInfo("charset changed from %s to %s", tsCharset, str); } - tstrncpy(tsCharset, str, TSDB_LOCALE_LEN); + tstrncpy(tsCharset, str, TD_LOCALE_LEN); cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION; } else { tscInfo("charset:%s not valid", str); @@ -400,7 +397,7 @@ int taos_options_imp(TSDB_OPTION option, const char *str) { assert(cfg != NULL); if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_OPTION) { - tstrncpy(tsTimezone, str, TSDB_TIMEZONE_LEN); + tstrncpy(tsTimezone, str, TD_TIMEZONE_LEN); tsSetTimeZone(); cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION; tscDebug("timezone set:%s, input:%s by taos_options", tsTimezone, str); diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 68f65cebc4..f256feb251 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -8,11 +8,10 @@ #include "tep.h" #include "tglobal.h" #include "tmsgtype.h" -#include "tnote.h" #include "tpagedbuf.h" #include "tref.h" -static int32_t initEpSetFromCfg(const char* ip, uint16_t port, SCorEpSet* pEpSet); +static int32_t initEpSetFromCfg(const char* firstEp, const char* secondEp, SCorEpSet* pEpSet); static SMsgSendInfo* buildConnectMsg(SRequestObj* pRequest); static void destroySendMsgInfo(SMsgSendInfo* pMsgBody); static void setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp); @@ -81,7 +80,19 @@ TAOS* taos_connect_internal(const char* ip, const char* user, const char* pass, } SCorEpSet epSet = {0}; - initEpSetFromCfg(ip, port, &epSet); + if (ip) { + if (initEpSetFromCfg(ip, NULL, &epSet) < 0) { + return NULL; + } + + if (port) { + epSet.epSet.eps[0].port = port; + } + } else { + if (initEpSetFromCfg(tsFirst, tsSecond, &epSet) < 0) { + return NULL; + } + } char* key = getClusterKey(user, secretEncrypt, ip, port); SAppInstInfo** pInst = NULL; @@ -242,11 +253,9 @@ TAOS_RES* taos_query_l(TAOS* taos, const char* sql, int sqlLen) { return NULL; } - nPrintTsc("%s", sql) - - SRequestObj* pRequest = NULL; - SQueryNode* pQueryNode = NULL; - SArray* pNodeList = taosArrayInit(4, sizeof(struct SQueryNodeAddr)); + SRequestObj* pRequest = NULL; + SQueryNode* pQueryNode = NULL; + SArray* pNodeList = taosArrayInit(4, sizeof(struct SQueryNodeAddr)); terrno = TSDB_CODE_SUCCESS; CHECK_CODE_GOTO(buildRequest(pTscObj, sql, sqlLen, &pRequest), _return); @@ -270,40 +279,32 @@ _return: return pRequest; } -int initEpSetFromCfg(const char* ip, uint16_t port, SCorEpSet* pEpSet) { - SConfigItem* pFirst = cfgGetItem(tscCfg, "firstEp"); - SConfigItem* pSecond = cfgGetItem(tscCfg, "secondEp"); - SConfigItem* pPort = cfgGetItem(tscCfg, "serverPort"); +int initEpSetFromCfg(const char* firstEp, const char* secondEp, SCorEpSet* pEpSet) { + pEpSet->version = 0; // init mnode ip set SEpSet* mgmtEpSet = &(pEpSet->epSet); mgmtEpSet->numOfEps = 0; mgmtEpSet->inUse = 0; - pEpSet->version = 0; - if (ip != NULL) { - taosGetFqdnPortFromEp(ip, (uint16_t)pPort->i32, &mgmtEpSet->eps[0]); + if (firstEp && firstEp[0] != 0) { + if (strlen(firstEp) >= TSDB_EP_LEN) { + terrno = TSDB_CODE_TSC_INVALID_FQDN; + return -1; + } + + taosGetFqdnPortFromEp(firstEp, &mgmtEpSet->eps[0]); mgmtEpSet->numOfEps++; - if (port) { - mgmtEpSet->eps[0].port = port; - } - } else { - if (pFirst->str[0] != 0) { - if (strlen(pFirst->str) >= TSDB_EP_LEN) { - terrno = TSDB_CODE_TSC_INVALID_FQDN; - return -1; - } - taosGetFqdnPortFromEp(pFirst->str, (uint16_t)pPort->i32, &mgmtEpSet->eps[0]); - mgmtEpSet->numOfEps++; - } - if (pSecond->str[0] != 0) { - if (strlen(pSecond->str) >= TSDB_EP_LEN) { - terrno = TSDB_CODE_TSC_INVALID_FQDN; - return -1; - } - taosGetFqdnPortFromEp(pSecond->str, (uint16_t)pPort->i32, &mgmtEpSet->eps[1]); - mgmtEpSet->numOfEps++; + } + + if (secondEp && secondEp[0] != 0) { + if (strlen(secondEp) >= TSDB_EP_LEN) { + terrno = TSDB_CODE_TSC_INVALID_FQDN; + return -1; } + + taosGetFqdnPortFromEp(secondEp, &mgmtEpSet->eps[mgmtEpSet->numOfEps]); + mgmtEpSet->numOfEps++; } if (mgmtEpSet->numOfEps == 0) { diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index 9a1025c4bd..d3d9fce210 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -24,7 +24,6 @@ #include "tep.h" #include "tglobal.h" #include "tmsgtype.h" -#include "tnote.h" #include "tpagedbuf.h" #include "tref.h" diff --git a/source/client/test/CMakeLists.txt b/source/client/test/CMakeLists.txt index 7f6ce81788..ee5109860e 100644 --- a/source/client/test/CMakeLists.txt +++ b/source/client/test/CMakeLists.txt @@ -8,13 +8,13 @@ AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) ADD_EXECUTABLE(clientTest clientTests.cpp) TARGET_LINK_LIBRARIES( clientTest - PUBLIC os util common transport parser catalog scheduler function gtest taos qcom config + PUBLIC os util common transport parser catalog scheduler function gtest taos qcom ) ADD_EXECUTABLE(tmqTest tmqTest.cpp) TARGET_LINK_LIBRARIES( tmqTest - PUBLIC os util common transport parser catalog scheduler function gtest taos qcom config + PUBLIC os util common transport parser catalog scheduler function gtest taos qcom ) TARGET_INCLUDE_DIRECTORIES( diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index 5414f17a62..ac36179a26 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -49,7 +49,7 @@ int main(int argc, char** argv) { } TEST(testCase, driverInit_Test) { - taosInitGlobalCfg(); + // taosInitGlobalCfg(); // taos_init(); } diff --git a/source/client/test/tmqTest.cpp b/source/client/test/tmqTest.cpp index 9f8ff7143a..c34c10d871 100644 --- a/source/client/test/tmqTest.cpp +++ b/source/client/test/tmqTest.cpp @@ -34,7 +34,7 @@ int main(int argc, char** argv) { } TEST(testCase, driverInit_Test) { - taosInitGlobalCfg(); + // taosInitGlobalCfg(); // taos_init(); } diff --git a/source/common/src/tep.c b/source/common/src/tep.c index 3e98cae279..970b6d954f 100644 --- a/source/common/src/tep.c +++ b/source/common/src/tep.c @@ -4,7 +4,7 @@ #include "tglobal.h" #include "tlockfree.h" -int taosGetFqdnPortFromEp(const char *ep, uint16_t defaultPort, SEp* pEp) { +int taosGetFqdnPortFromEp(const char *ep, SEp* pEp) { pEp->port = 0; strcpy(pEp->fqdn, ep); @@ -15,7 +15,7 @@ int taosGetFqdnPortFromEp(const char *ep, uint16_t defaultPort, SEp* pEp) { } if (pEp->port == 0) { - pEp->port = defaultPort; + pEp->port = tsServerPort; return -1; } diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 72783bf4ab..f76e8380e4 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -19,21 +19,39 @@ #include "taosdef.h" #include "taoserror.h" #include "tcompare.h" +#include "tconfig.h" #include "tep.h" #include "tglobal.h" -#include "tlocale.h" #include "tlog.h" -#include "ttimezone.h" #include "tutil.h" #include "ulog.h" +SConfig *tsCfg = NULL; + +// cluster +char tsFirst[TSDB_EP_LEN] = {0}; +char tsSecond[TSDB_EP_LEN] = {0}; +char tsLocalFqdn[TSDB_FQDN_LEN] = {0}; +char tsLocalEp[TSDB_EP_LEN] = {0}; // Local End Point, hostname:port +uint16_t tsServerPort = 6030; +int32_t tsVersion = 30000000; +int32_t tsStatusInterval = 1; // second +bool tsEnableTelemetryReporting = 0; // common -int8_t tsDaylight = 0; +int32_t tsRpcTimer = 300; +int32_t tsRpcMaxTime = 600; // seconds; +bool tsRpcForceTcp = 1; // disable this, means query, show command use udp protocol as default +int32_t tsMaxShellConns = 50000; +int32_t tsMaxConnections = 50000; +int32_t tsShellActivityTimer = 3; // second +float tsNumOfThreadsPerCore = 1.0f; +int32_t tsNumOfCommitThreads = 4; +float tsRatioOfQueryCores = 1.0f; int32_t tsMaxBinaryDisplayWidth = 30; -int8_t tsEnableSlaveQuery = 1; -int8_t tsEnableAdjustMaster = 1; -int8_t tsPrintAuth = 0; +bool tsEnableSlaveQuery = 1; +bool tsPrintAuth = 0; + /* * denote if the server needs to compress response message at the application layer to client, including query rsp, * metricmeta rsp, and multi-meter query rsp message body. The client compress the submit message to server. @@ -60,8 +78,6 @@ int32_t tsCompatibleModel = 1; int32_t tsMaxWildCardsLen = TSDB_PATTERN_STRING_DEFAULT_LEN; int32_t tsMaxRegexStringLen = TSDB_REGEX_STRING_DEFAULT_LEN; -int8_t tsTscEnableRecordSql = 0; - // the maximum number of results for projection query on super table that are returned from // one virtual node, to order according to timestamp int32_t tsMaxNumOfOrderedResults = 100000; @@ -98,18 +114,21 @@ int32_t tsQueryBufferSize = -1; int64_t tsQueryBufferSizeBytes = -1; // in retrieve blocking model, the retrieve threads will wait for the completion of the query processing. -int32_t tsRetrieveBlockingModel = 0; +bool tsRetrieveBlockingModel = 0; // last_row(*), first(*), last_row(ts, col1, col2) query, the result fields will be the original column name -int8_t tsKeepOriginalColumnName = 0; +bool tsKeepOriginalColumnName = 0; // long query death-lock -int8_t tsDeadLockKillQuery = 0; +bool tsDeadLockKillQuery = 0; // tsdb config // For backward compatibility bool tsdbForceKeepFile = false; +int32_t tsDiskCfgNum = 0; +SDiskCfg tsDiskCfg[TFS_MAX_DISKS] = {0}; + /* * minimum scale for whole system, millisecond by default * for TSDB_TIME_PRECISION_MILLI: 86400000L @@ -118,13 +137,7 @@ bool tsdbForceKeepFile = false; */ int64_t tsTickPerDay[] = {86400000L, 86400000000L, 86400000000000L}; -// system info -int32_t tsTotalMemoryMB = 0; -uint32_t tsVersion = 0; - -// // lossy compress 6 -// char tsLossyColumns[32] = ""; // "float|double" means all float and double columns can be lossy compressed. set empty // can close lossy compress. // below option can take effect when tsLossyColumns not empty @@ -134,428 +147,360 @@ uint32_t tsMaxRange = 500; // max range uint32_t tsCurRange = 100; // range char tsCompressor[32] = "ZSTD_COMPRESSOR"; // ZSTD_COMPRESSOR or GZIP_COMPRESSOR - -int32_t (*monStartSystemFp)() = NULL; -void (*monStopSystemFp)() = NULL; -void (*monExecuteSQLFp)(char *sql) = NULL; - -char *qtypeStr[] = {"rpc", "fwd", "wal", "cq", "query"}; - -static pthread_once_t tsInitGlobalCfgOnce = PTHREAD_ONCE_INIT; - - -int32_t taosCfgDynamicOptions(char *msg) { - #if 0 - char *option, *value; - int32_t olen, vlen; - int32_t vint = 0; - - paGetToken(msg, &option, &olen); - if (olen == 0) return -1; - - paGetToken(option + olen + 1, &value, &vlen); - if (vlen == 0) - vint = 135; - else { - vint = atoi(value); - } - - uInfo("change dynamic option: %s, value: %d", option, vint); - - for (int32_t i = 0; i < tsGlobalConfigNum; ++i) { - SGlobalCfg *cfg = tsGlobalConfig + i; - // if (!(cfg->cfgType & TSDB_CFG_CTYPE_B_LOG)) continue; - if (cfg->valType != TAOS_CFG_VTYPE_INT32 && cfg->valType != TAOS_CFG_VTYPE_INT8) continue; - - int32_t cfgLen = (int32_t)strlen(cfg->option); - if (cfgLen != olen) continue; - if (strncasecmp(option, cfg->option, olen) != 0) continue; - if (cfg->valType == TAOS_CFG_VTYPE_INT32) { - *((int32_t *)cfg->ptr) = vint; - } else { - *((int8_t *)cfg->ptr) = (int8_t)vint; - } - - if (strncasecmp(cfg->option, "monitor", olen) == 0) { - if (1 == vint) { - if (monStartSystemFp) { - (*monStartSystemFp)(); - uInfo("monitor is enabled"); - } else { - uError("monitor can't be updated, for monitor not initialized"); - } - } else { - if (monStopSystemFp) { - (*monStopSystemFp)(); - uInfo("monitor is disabled"); - } else { - uError("monitor can't be updated, for monitor not initialized"); - } - } - return 0; - } - if (strncasecmp(cfg->option, "debugFlag", olen) == 0) { - taosSetAllDebugFlag(); - } - return 0; - } - - if (strncasecmp(option, "resetlog", 8) == 0) { - taosResetLog(); - taosPrintCfg(); - return 0; - } - - if (strncasecmp(option, "resetQueryCache", 15) == 0) { - if (monExecuteSQLFp) { - (*monExecuteSQLFp)("resetQueryCache"); - uInfo("resetquerycache is executed"); - } else { - uError("resetquerycache can't be executed, for monitor not started"); - } - } - -#endif - return false; +static void taosAddDataDir(int32_t index, char *v1, int32_t level, int32_t primary) { + tstrncpy(tsDiskCfg[index].dir, v1, TSDB_FILENAME_LEN); + tsDiskCfg[index].level = level; + tsDiskCfg[index].primary = primary; + uTrace("dataDir:%s, level:%d primary:%d is configured", v1, level, primary); } -// void taosAddDataDir(int index, char *v1, int level, int primary) { -// tstrncpy(tsDiskCfg[index].dir, v1, TSDB_FILENAME_LEN); -// tsDiskCfg[index].level = level; -// tsDiskCfg[index].primary = primary; -// uTrace("dataDir:%s, level:%d primary:%d is configured", v1, level, primary); -// } +static void taosReadDataDirCfg(char *v1, char *v2, char *v3) { + if (tsDiskCfgNum == 1) { + SDiskCfg *cfg = &tsDiskCfg[0]; + uInfo("dataDir:%s, level:%d primary:%d is replaced by %s", cfg->dir, cfg->level, cfg->primary, v1); + } + taosAddDataDir(0, v1, 0, 1); + tsDiskCfgNum = 1; +} -#ifndef _STORAGE -// void taosReadDataDirCfg(char *v1, char *v2, char *v3) { -// if (tsDiskCfgNum == 1) { -// SDiskCfg *cfg = &tsDiskCfg[0]; -// uInfo("dataDir:%s, level:%d primary:%d is replaced by %s", cfg->dir, cfg->level, cfg->primary, v1); -// } -// taosAddDataDir(0, v1, 0, 1); -// tsDiskCfgNum = 1; -// } +static void taosPrintDataDirCfg() { + for (int32_t i = 0; i < tsDiskCfgNum; ++i) { + SDiskCfg *cfg = &tsDiskCfg[i]; + uInfo(" dataDir: %s", cfg->dir); + } +} -// void taosPrintDataDirCfg() { -// for (int i = 0; i < tsDiskCfgNum; ++i) { -// SDiskCfg *cfg = &tsDiskCfg[i]; -// uInfo(" dataDir: %s", cfg->dir); -// } -// } -#endif +struct SConfig *taosGetCfg() { + return tsCfg; +} +static int32_t taosLoadCfg(SConfig *pCfg, const char *inputCfgDir, const char *envFile, const char *apolloUrl) { + char cfgDir[PATH_MAX] = {0}; + char cfgFile[PATH_MAX + 100] = {0}; + taosExpandDir(inputCfgDir, cfgDir, PATH_MAX); + snprintf(cfgFile, sizeof(cfgFile), "%s" TD_DIRSEP "taos.cfg", cfgDir); -static void doInitGlobalConfig(void) { + if (cfgLoad(pCfg, CFG_STYPE_APOLLO_URL, apolloUrl) != 0) { + uError("failed to load from apollo url:%s since %s\n", apolloUrl, terrstr()); + return -1; + } + + if (cfgLoad(pCfg, CFG_STYPE_CFG_FILE, cfgFile) != 0) { + if (cfgLoad(pCfg, CFG_STYPE_CFG_FILE, cfgDir) != 0) { + uError("failed to load from config file:%s since %s\n", cfgFile, terrstr()); + return -1; + } + } + + if (cfgLoad(pCfg, CFG_STYPE_ENV_FILE, envFile) != 0) { + uError("failed to load from env file:%s since %s\n", envFile, terrstr()); + return -1; + } + + if (cfgLoad(pCfg, CFG_STYPE_ENV_VAR, NULL) != 0) { + uError("failed to load from global env variables since %s\n", terrstr()); + return -1; + } + + return 0; +} + +static void taosAddClientLogCfg(SConfig *pCfg) { + cfgAddDir(pCfg, "logDir", tsLogDir, 1); + cfgAddFloat(pCfg, "minimalLogDirGB", 1.0f, 0.001f, 10000000, 1); + cfgAddInt32(pCfg, "numOfLogLines", tsNumOfLogLines, 1000, 2000000000, 1); + cfgAddBool(pCfg, "asyncLog", tsAsyncLog, 1); + cfgAddInt32(pCfg, "logKeepDays", 0, -365000, 365000, 1); + cfgAddInt32(pCfg, "cDebugFlag", cDebugFlag, 0, 255, 1); + cfgAddInt32(pCfg, "uDebugFlag", uDebugFlag, 0, 255, 1); + cfgAddInt32(pCfg, "rpcDebugFlag", rpcDebugFlag, 0, 255, 1); + cfgAddInt32(pCfg, "tmrDebugFlag", tmrDebugFlag, 0, 255, 1); + cfgAddInt32(pCfg, "jniDebugFlag", jniDebugFlag, 0, 255, 1); + cfgAddInt32(pCfg, "simDebugFlag", 143, 0, 255, 1); + cfgAddDir(pCfg, "configDir", configDir, 1); + cfgAddDir(pCfg, "scriptDir", configDir, 1); + cfgAddInt32(pCfg, "debugFlag", 0, 0, 255, 1); +} + +static void taosAddServerLogCfg(SConfig *pCfg) { + cfgAddInt32(pCfg, "dDebugFlag", dDebugFlag, 0, 255, 0); + cfgAddInt32(pCfg, "vDebugFlag", vDebugFlag, 0, 255, 0); + cfgAddInt32(pCfg, "mDebugFlag", mDebugFlag, 0, 255, 0); + cfgAddInt32(pCfg, "qDebugFlag", qDebugFlag, 0, 255, 0); + cfgAddInt32(pCfg, "wDebugFlag", wDebugFlag, 0, 255, 0); + cfgAddInt32(pCfg, "sDebugFlag", sDebugFlag, 0, 255, 0); + cfgAddInt32(pCfg, "tsdbDebugFlag", tsdbDebugFlag, 0, 255, 0); + cfgAddInt32(pCfg, "tqDebugFlag", tqDebugFlag, 0, 255, 0); + cfgAddInt32(pCfg, "fsDebugFlag", fsDebugFlag, 0, 255, 0); +} + +static void taosAddClientCfg(SConfig *pCfg) { + char defaultFqdn[TSDB_FQDN_LEN] = {0}; + int32_t defaultServerPort = 6030; + char defaultFirstEp[TSDB_EP_LEN] = {0}; + char defaultSecondEp[TSDB_EP_LEN] = {0}; + taosGetFqdn(defaultFqdn); + snprintf(defaultFirstEp, TSDB_EP_LEN, "%s:%d", defaultFqdn, defaultServerPort); + snprintf(defaultSecondEp, TSDB_EP_LEN, "%s:%d", defaultFqdn, defaultServerPort); + + cfgAddString(pCfg, "firstEp", defaultFirstEp, 1); + cfgAddString(pCfg, "secondEp", defaultSecondEp, 1); + cfgAddString(pCfg, "fqdn", defaultFqdn, 1); + cfgAddInt32(pCfg, "serverPort", defaultServerPort, 1, 65056, 1); + cfgAddDir(pCfg, "tempDir", tsTempDir, 1); + cfgAddFloat(pCfg, "minimalTempDirGB", 1.0f, 0.001f, 10000000, 1); + cfgAddFloat(pCfg, "numOfThreadsPerCore", tsNumOfThreadsPerCore, 0, 10, 1); + cfgAddInt32(pCfg, "maxTmrCtrl", tsMaxTmrCtrl, 8, 2048, 1); + cfgAddInt32(pCfg, "rpcTimer", tsRpcTimer, 100, 3000, 1); + cfgAddInt32(pCfg, "rpcMaxTime", tsRpcMaxTime, 100, 7200, 1); + cfgAddBool(pCfg, "rpcForceTcp", tsRpcForceTcp, 1); + cfgAddInt32(pCfg, "shellActivityTimer", tsShellActivityTimer, 1, 120, 1); + cfgAddInt32(pCfg, "compressMsgSize", tsCompressMsgSize, -1, 100000000, 1); + cfgAddInt32(pCfg, "compressColData", tsCompressColData, -1, 100000000, 1); + cfgAddInt32(pCfg, "maxWildCardsLength", tsMaxWildCardsLen, 0, TSDB_MAX_FIELD_LEN, 1); + cfgAddInt32(pCfg, "maxRegexStringLen", tsMaxRegexStringLen, 0, TSDB_MAX_FIELD_LEN, 1); + cfgAddInt32(pCfg, "maxNumOfOrderedRes", tsMaxNumOfOrderedResults, 128, TSDB_MAX_ALLOWED_SQL_LEN, 1); + cfgAddBool(pCfg, "keepColumnName", tsKeepOriginalColumnName, 1); + cfgAddInt32(pCfg, "maxBinaryDisplayWidth", tsMaxBinaryDisplayWidth, 1, 65536, 1); +} + +static void taosAddSystemInfo(SConfig *pCfg) { + SysNameInfo info = taosGetSysNameInfo(); + + cfgAddTimezone(pCfg, "timezone", tsTimezone); + cfgAddLocale(pCfg, "locale", tsLocale); + cfgAddCharset(pCfg, "charset", tsCharset); + cfgAddBool(pCfg, "enableCoreFile", 0, 1); + cfgAddInt32(pCfg, "numOfCores", tsNumOfCores, 1, 100000, 1); + cfgAddInt32(pCfg, "pageSize(KB)", tsPageSize, 0, INT64_MAX, 1); + cfgAddInt64(pCfg, "openMax", tsOpenMax, 0, INT64_MAX, 1); + cfgAddInt64(pCfg, "streamMax", tsStreamMax, 0, INT64_MAX, 1); + cfgAddInt32(pCfg, "totalMemory(MB)", tsTotalMemoryMB, 0, INT32_MAX, 1); + cfgAddString(pCfg, "os sysname", info.sysname, 1); + cfgAddString(pCfg, "os nodename", info.nodename, 1); + cfgAddString(pCfg, "os release", info.release, 1); + cfgAddString(pCfg, "os version", info.version, 1); + cfgAddString(pCfg, "os machine", info.machine, 1); + cfgAddString(pCfg, "os sysname", info.sysname, 1); + + cfgAddString(pCfg, "version", version, 1); + cfgAddString(pCfg, "compatible_version", compatible_version, 1); + cfgAddString(pCfg, "gitinfo", gitinfo, 1); + cfgAddString(pCfg, "gitinfoOfInternal", gitinfoOfInternal, 1); + cfgAddString(pCfg, "buildinfo", buildinfo, 1); +} + +static void taosAddServerCfg(SConfig *pCfg) { + cfgAddInt32(pCfg, "supportVnodes", 256, 0, 65536, 0); + cfgAddDir(pCfg, "dataDir", tsDataDir, 0); + cfgAddFloat(pCfg, "minimalDataDirGB", 2.0f, 0.001f, 10000000, 0); + cfgAddInt32(pCfg, "numOfCommitThreads", tsNumOfCommitThreads, 1, 100, 0); + cfgAddFloat(pCfg, "ratioOfQueryCores", tsRatioOfQueryCores, 0, 2, 0); + cfgAddInt32(pCfg, "maxNumOfDistinctRes", tsMaxNumOfDistinctResults, 10 * 10000, 10000 * 10000, 0); + cfgAddBool(pCfg, "telemetryReporting", tsEnableTelemetryReporting, 0); + cfgAddInt32(pCfg, "maxConnections", tsMaxConnections, 1, 100000, 0); + cfgAddInt32(pCfg, "maxShellConns", tsMaxShellConns, 10, 50000000, 0); + cfgAddInt32(pCfg, "statusInterval", tsStatusInterval, 1, 30, 0); + cfgAddInt32(pCfg, "minSlidingTime", tsMinSlidingTime, 10, 1000000, 0); + cfgAddInt32(pCfg, "minIntervalTime", tsMinIntervalTime, 1, 1000000, 0); + cfgAddInt32(pCfg, "maxStreamCompDelay", tsMaxStreamComputDelay, 10, 1000000000, 0); + cfgAddInt32(pCfg, "maxFirstStreamCompDelay", tsStreamCompStartDelay, 1000, 1000000000, 0); + cfgAddInt32(pCfg, "retryStreamCompDelay", tsRetryStreamCompDelay, 10, 1000000000, 0); + cfgAddFloat(pCfg, "streamCompDelayRatio", tsStreamComputDelayRatio, 0.1, 0.9, 0); + cfgAddInt32(pCfg, "queryBufferSize", tsQueryBufferSize, -1, 500000000000, 0); + cfgAddBool(pCfg, "retrieveBlockingModel", tsRetrieveBlockingModel, 0); + cfgAddBool(pCfg, "printAuth", tsPrintAuth, 0); + cfgAddBool(pCfg, "slaveQuery", tsEnableSlaveQuery, 0); + cfgAddBool(pCfg, "deadLockKillQuery", tsDeadLockKillQuery, 0); +} + +static void taosSetClientLogCfg(SConfig *pCfg) { + SConfigItem *pItem = cfgGetItem(pCfg, "logDir"); + tstrncpy(tsLogDir, cfgGetItem(pCfg, "logDir")->str, PATH_MAX); + tsLogSpace.reserved = cfgGetItem(pCfg, "minimalLogDirGB")->fval; + tsNumOfLogLines = cfgGetItem(pCfg, "numOfLogLines")->i32; + tsAsyncLog = cfgGetItem(pCfg, "asyncLog")->bval; + tsLogKeepDays = cfgGetItem(pCfg, "logKeepDays")->i32; + cDebugFlag = cfgGetItem(pCfg, "cDebugFlag")->i32; + uDebugFlag = cfgGetItem(pCfg, "uDebugFlag")->i32; + rpcDebugFlag = cfgGetItem(pCfg, "rpcDebugFlag")->i32; + tmrDebugFlag = cfgGetItem(pCfg, "tmrDebugFlag")->i32; + jniDebugFlag = cfgGetItem(pCfg, "jniDebugFlag")->i32; +} + +static void taosSetServerLogCfg(SConfig *pCfg) { + dDebugFlag = cfgGetItem(pCfg, "dDebugFlag")->i32; + vDebugFlag = cfgGetItem(pCfg, "vDebugFlag")->i32; + mDebugFlag = cfgGetItem(pCfg, "mDebugFlag")->i32; + qDebugFlag = cfgGetItem(pCfg, "qDebugFlag")->i32; + wDebugFlag = cfgGetItem(pCfg, "wDebugFlag")->i32; + sDebugFlag = cfgGetItem(pCfg, "sDebugFlag")->i32; + tsdbDebugFlag = cfgGetItem(pCfg, "tsdbDebugFlag")->i32; + tqDebugFlag = cfgGetItem(pCfg, "tqDebugFlag")->i32; + fsDebugFlag = cfgGetItem(pCfg, "fsDebugFlag")->i32; +} + +static void taosSetClientCfg(SConfig *pCfg) { + tstrncpy(tsFirst, cfgGetItem(pCfg, "firstEp")->str, TSDB_EP_LEN); + tstrncpy(tsSecond, cfgGetItem(pCfg, "secondEp")->str, TSDB_EP_LEN); + tstrncpy(tsLocalFqdn, cfgGetItem(pCfg, "fqdn")->str, TSDB_EP_LEN); + tsServerPort = (uint16_t)cfgGetItem(pCfg, "serverPort")->i32; + snprintf(tsLocalEp, sizeof(tsLocalEp), "%s:%u", tsLocalFqdn, tsServerPort); + tstrncpy(tsLogDir, cfgGetItem(pCfg, "tempDir")->str, PATH_MAX); + tsTempSpace.reserved = cfgGetItem(pCfg, "minimalTempDirGB")->fval; + + tsNumOfThreadsPerCore = cfgGetItem(pCfg, "maxTmrCtrl")->fval; + tsMaxTmrCtrl = cfgGetItem(pCfg, "maxTmrCtrl")->i32; + tsRpcTimer = cfgGetItem(pCfg, "rpcTimer")->i32; + tsRpcMaxTime = cfgGetItem(pCfg, "rpcMaxTime")->i32; + tsRpcForceTcp = cfgGetItem(pCfg, "rpcForceTcp")->i32; + tsShellActivityTimer = cfgGetItem(pCfg, "shellActivityTimer")->bval; + tsCompressMsgSize = cfgGetItem(pCfg, "compressMsgSize")->i32; + tsCompressColData = cfgGetItem(pCfg, "compressColData")->i32; + tsMaxWildCardsLen = cfgGetItem(pCfg, "maxWildCardsLength")->i32; + tsMaxRegexStringLen = cfgGetItem(pCfg, "maxRegexStringLen")->i32; + tsMaxNumOfOrderedResults = cfgGetItem(pCfg, "maxNumOfOrderedRes")->i32; + tsKeepOriginalColumnName = cfgGetItem(pCfg, "keepColumnName")->bval; + tsMaxBinaryDisplayWidth = cfgGetItem(pCfg, "maxBinaryDisplayWidth")->i32; +} + +static void taosSetSystemCfg(SConfig *pCfg) { + SConfigItem *pItem = cfgGetItem(pCfg, "timezone"); + osSetTimezone(pItem->str); + uDebug("timezone format changed from %s to %s", pItem->str, tsTimezone); + cfgSetItem(pCfg, "timezone", tsTimezone, pItem->stype); + + const char *locale = cfgGetItem(pCfg, "locale")->str; + const char *charset = cfgGetItem(pCfg, "charset")->str; + taosSetSystemLocale(locale, charset); + + if (tsNumOfCores <= 1) { + tsNumOfCores = 2; + } + + bool enableCore = cfgGetItem(pCfg, "enableCoreFile")->bval; + taosSetConsoleEcho(enableCore); + + // todo + tsVersion = 30000000; +} + +static void taosSetServerCfg(SConfig *pCfg) { + tstrncpy(tsDataDir, cfgGetItem(pCfg, "dataDir")->str, PATH_MAX); + tsTempSpace.reserved = cfgGetItem(pCfg, "minimalDataDirGB")->fval; + tsNumOfCommitThreads = cfgGetItem(pCfg, "numOfCommitThreads")->i32; + tsRatioOfQueryCores = cfgGetItem(pCfg, "ratioOfQueryCores")->fval; + tsMaxNumOfDistinctResults = cfgGetItem(pCfg, "maxNumOfDistinctRes")->i32; + tsEnableTelemetryReporting = cfgGetItem(pCfg, "telemetryReporting")->bval; + tsMaxConnections = cfgGetItem(pCfg, "maxConnections")->i32; + tsMaxShellConns = cfgGetItem(pCfg, "maxShellConns")->i32; + tsStatusInterval = cfgGetItem(pCfg, "statusInterval")->i32; + tsMinSlidingTime = cfgGetItem(pCfg, "minSlidingTime")->i32; + tsMinIntervalTime = cfgGetItem(pCfg, "minIntervalTime")->i32; + tsMaxStreamComputDelay = cfgGetItem(pCfg, "maxStreamCompDelay")->i32; + tsStreamCompStartDelay = cfgGetItem(pCfg, "maxFirstStreamCompDelay")->i32; + tsRetryStreamCompDelay = cfgGetItem(pCfg, "retryStreamCompDelay")->i32; + tsStreamComputDelayRatio = cfgGetItem(pCfg, "streamCompDelayRatio")->fval; + tsQueryBufferSize = cfgGetItem(pCfg, "queryBufferSize")->i32; + tsRetrieveBlockingModel = cfgGetItem(pCfg, "retrieveBlockingModel")->bval; + tsPrintAuth = cfgGetItem(pCfg, "printAuth")->bval; + tsEnableSlaveQuery = cfgGetItem(pCfg, "slaveQuery")->bval; + tsDeadLockKillQuery = cfgGetItem(pCfg, "deadLockKillQuery")->bval; + + if (tsQueryBufferSize >= 0) { + tsQueryBufferSizeBytes = tsQueryBufferSize * 1048576UL; + } +} + +int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDir, const char *envFile, + const char *apolloUrl, bool tsc) { osInit(); - srand(taosSafeRand()); -#if 0 - SGlobalCfg cfg = {0}; + SConfig *pCfg = cfgInit(); + if (pCfg == NULL) return -1; - cfg.option = "dataDir"; - cfg.ptr = tsDataDir; - cfg.valType = TAOS_CFG_VTYPE_DATA_DIRCTORY; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; - cfg.minValue = 0; - cfg.maxValue = 0; - cfg.ptrLength = TSDB_FILENAME_LEN; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); + if (tsc) { + taosAddClientLogCfg(pCfg); + } else { + taosAddClientLogCfg(pCfg); + taosAddServerLogCfg(pCfg); + } + if (taosLoadCfg(pCfg, cfgDir, envFile, apolloUrl) != 0) { + uError("failed to load cfg since %s", terrstr()); + cfgCleanup(pCfg); + return -1; + } - cfg.option = "maxNumOfDistinctRes"; - cfg.ptr = &tsMaxNumOfDistinctResults; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 10 * 10000; - cfg.maxValue = 10000 * 10000; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); + if (tsc) { + taosSetClientLogCfg(pCfg); + taosSetAllDebugFlag(cfgGetItem(pCfg, "debugFlag")->i32); + } else { + taosSetClientLogCfg(pCfg); + taosSetServerLogCfg(pCfg); + taosSetAllDebugFlag(cfgGetItem(pCfg, "debugFlag")->i32); + } - cfg.option = "minSlidingTime"; - cfg.ptr = &tsMinSlidingTime; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 10; - cfg.maxValue = 1000000; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_MS; - taosAddConfigOption(cfg); + if (taosInitLog(logname, logFileNum) != 0) { + printf("failed to init log file since %s\n", terrstr()); + cfgCleanup(pCfg); + return -1; + } - cfg.option = "minIntervalTime"; - cfg.ptr = &tsMinIntervalTime; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 1; - cfg.maxValue = 1000000; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_MS; - taosAddConfigOption(cfg); - - cfg.option = "maxStreamCompDelay"; - cfg.ptr = &tsMaxStreamComputDelay; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 10; - cfg.maxValue = 1000000000; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_MS; - taosAddConfigOption(cfg); - - cfg.option = "maxFirstStreamCompDelay"; - cfg.ptr = &tsStreamCompStartDelay; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 1000; - cfg.maxValue = 1000000000; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_MS; - taosAddConfigOption(cfg); - - cfg.option = "retryStreamCompDelay"; - cfg.ptr = &tsRetryStreamCompDelay; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 10; - cfg.maxValue = 1000000000; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_MS; - - taosAddConfigOption(cfg); - cfg.option = "streamCompDelayRatio"; - cfg.ptr = &tsStreamComputDelayRatio; - cfg.valType = TAOS_CFG_VTYPE_FLOAT; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 0.1f; - cfg.maxValue = 0.9f; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "compressMsgSize"; - cfg.ptr = &tsCompressMsgSize; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = -1; - cfg.maxValue = 100000000.0f; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "compressColData"; - cfg.ptr = &tsCompressColData; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = -1; - cfg.maxValue = 100000000.0f; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "maxWildCardsLength"; - cfg.ptr = &tsMaxWildCardsLen; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 0; - cfg.maxValue = TSDB_MAX_FIELD_LEN; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_BYTE; - taosAddConfigOption(cfg); - - cfg.option = "maxRegexStringLen"; - cfg.ptr = &tsMaxRegexStringLen; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 0; - cfg.maxValue = TSDB_MAX_FIELD_LEN; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_BYTE; - taosAddConfigOption(cfg); - - cfg.option = "maxNumOfOrderedRes"; - cfg.ptr = &tsMaxNumOfOrderedResults; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = TSDB_MAX_SQL_LEN; - cfg.maxValue = TSDB_MAX_ALLOWED_SQL_LEN; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "queryBufferSize"; - cfg.ptr = &tsQueryBufferSize; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = -1; - cfg.maxValue = 500000000000.0f; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_BYTE; - taosAddConfigOption(cfg); - - cfg.option = "retrieveBlockingModel"; - cfg.ptr = &tsRetrieveBlockingModel; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 0; - cfg.maxValue = 1; - cfg.ptrLength = 1; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "keepColumnName"; - cfg.ptr = &tsKeepOriginalColumnName; - cfg.valType = TAOS_CFG_VTYPE_INT8; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 1; - cfg.ptrLength = 1; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - - - cfg.option = "slaveQuery"; - cfg.ptr = &tsEnableSlaveQuery; - cfg.valType = TAOS_CFG_VTYPE_INT8; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 0; - cfg.maxValue = 1; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - - - cfg.option = "enableRecordSql"; - cfg.ptr = &tsTscEnableRecordSql; - cfg.valType = TAOS_CFG_VTYPE_INT8; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; - cfg.minValue = 0; - cfg.maxValue = 1; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "maxBinaryDisplayWidth"; - cfg.ptr = &tsMaxBinaryDisplayWidth; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 1; - cfg.maxValue = 65536; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "tempDir"; - cfg.ptr = tsTempDir; - cfg.valType = TAOS_CFG_VTYPE_STRING; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 0; - cfg.ptrLength = PATH_MAX; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - // enable kill long query - cfg.option = "deadLockKillQuery"; - cfg.ptr = &tsDeadLockKillQuery; - cfg.valType = TAOS_CFG_VTYPE_INT8; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 0; - cfg.maxValue = 1; - cfg.ptrLength = 1; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - -#ifdef TD_TSZ - // lossy compress - cfg.option = "lossyColumns"; - cfg.ptr = lossyColumns; - cfg.valType = TAOS_CFG_VTYPE_STRING; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; - cfg.minValue = 0; - cfg.maxValue = 0; - cfg.ptrLength = tListLen(lossyColumns); - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "fPrecision"; - cfg.ptr = &fPrecision; - cfg.valType = TAOS_CFG_VTYPE_DOUBLE; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; - cfg.minValue = MIN_FLOAT; - cfg.maxValue = 100000; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - - taosAddConfigOption(cfg); - - cfg.option = "dPrecision"; - cfg.ptr = &dPrecision; - cfg.valType = TAOS_CFG_VTYPE_DOUBLE; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; - cfg.minValue = 100000; - cfg.maxValue = 0; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "maxRange"; - cfg.ptr = &maxRange; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; - cfg.minValue = 0; - cfg.maxValue = 65536; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "range"; - cfg.ptr = &curRange; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; - cfg.minValue = 0; - cfg.maxValue = 65536; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - assert(tsGlobalConfigNum == TSDB_CFG_MAX_NUM); -#else - // assert(tsGlobalConfigNum == TSDB_CFG_MAX_NUM - 5); -#endif - -#endif + cfgCleanup(pCfg); + return 0; } -void taosInitGlobalCfg() { pthread_once(&tsInitGlobalCfgOnce, doInitGlobalConfig); } +int32_t taosInitCfg(const char *cfgDir, const char *envFile, const char *apolloUrl, bool tsc) { + if (tsCfg != NULL) return 0; + tsCfg = cfgInit(); -/* - * alter dnode 1 balance "vnode:1-dnode:2" - */ + if (tsc) { + taosAddClientLogCfg(tsCfg); + taosAddClientCfg(tsCfg); + } else { + taosAddClientLogCfg(tsCfg); + taosAddServerLogCfg(tsCfg); + taosAddClientCfg(tsCfg); + taosAddServerCfg(tsCfg); + } + taosAddSystemInfo(tsCfg); -bool taosCheckBalanceCfgOptions(const char *option, int32_t *vnodeId, int32_t *dnodeId) { - int len = (int)strlen(option); - if (strncasecmp(option, "vnode:", 6) != 0) { - return false; + if (taosLoadCfg(tsCfg, cfgDir, envFile, apolloUrl) != 0) { + uError("failed to load cfg since %s", terrstr()); + cfgCleanup(tsCfg); + tsCfg = NULL; + return -1; } - int pos = 0; - for (; pos < len; ++pos) { - if (option[pos] == '-') break; + if (tsc) { + taosSetClientCfg(tsCfg); + } else { + taosSetClientCfg(tsCfg); + taosSetServerCfg(tsCfg); } + taosSetSystemCfg(tsCfg); - if (++pos >= len) return false; - if (strncasecmp(option + pos, "dnode:", 6) != 0) { - return false; - } - - *vnodeId = strtol(option + 6, NULL, 10); - *dnodeId = strtol(option + pos + 6, NULL, 10); - if (*vnodeId <= 1 || *dnodeId <= 0) { - return false; - } - - return true; + cfgDumpCfg(tsCfg, tsc, false); + return 0; } + +void taosCleanupCfg() { + if (tsCfg) { + cfgCleanup(tsCfg); + tsCfg = NULL; + } +} + +void taosCfgDynamicOptions(const char *option, const char *value) { + if (strcasecmp(option, "debugFlag") == 0) { + int32_t debugFlag = atoi(value); + taosSetAllDebugFlag(debugFlag); + } + + if (strcasecmp(option, "resetlog") == 0) { + taosResetLog(); + cfgDumpCfg(tsCfg, 1, false); + } +} \ No newline at end of file diff --git a/source/common/src/tlocale.c b/source/common/src/tlocale.c deleted file mode 100644 index 1f2b9cd4b1..0000000000 --- a/source/common/src/tlocale.c +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "ulog.h" -#include "tglobal.h" -#include "tutil.h" - -/** - * In some Linux systems, setLocale(LC_CTYPE, "") may return NULL, in which case the launch of - * both the TDengine Server and the Client may be interrupted. - * - * In case that the setLocale failed to be executed, the right charset needs to be set. - */ -void tsSetLocale() { - char *locale = setlocale(LC_CTYPE, tsLocale); - - // default locale or user specified locale is not valid, abort launch - if (locale == NULL) { - uError("Invalid locale:%s, please set the valid locale in config file", tsLocale); - } - - if (strlen(tsCharset) == 0) { - uError("failed to get charset, please set the valid charset in config file"); - exit(-1); - } - - if (!taosValidateEncodec(tsCharset)) { - uError("Invalid charset:%s, please set the valid charset in config file", tsCharset); - exit(-1); - } -} \ No newline at end of file diff --git a/source/common/src/ttszip.c b/source/common/src/ttszip.c index 6d57992c35..218472e7c9 100644 --- a/source/common/src/ttszip.c +++ b/source/common/src/ttszip.c @@ -24,8 +24,9 @@ STSBuf* tsBufCreate(bool autoDelete, int32_t order) { pTSBuf->autoDelete = autoDelete; taosGetTmpfilePath(tsTempDir, "join", pTSBuf->path); - pTSBuf->f = fopen(pTSBuf->path, "wb+"); - if (pTSBuf->f == NULL) { + // pTSBuf->pFile = fopen(pTSBuf->path, "wb+"); + pTSBuf->pFile = taosOpenFile(pTSBuf->path, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC); + if (pTSBuf->pFile == NULL) { free(pTSBuf); return NULL; } @@ -60,8 +61,9 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) { tstrncpy(pTSBuf->path, path, sizeof(pTSBuf->path)); - pTSBuf->f = fopen(pTSBuf->path, "rb+"); - if (pTSBuf->f == NULL) { + // pTSBuf->pFile = fopen(pTSBuf->path, "rb+"); + pTSBuf->pFile = taosOpenFile(pTSBuf->path, TD_FILE_WRITE | TD_FILE_READ); + if (pTSBuf->pFile == NULL) { free(pTSBuf); return NULL; } @@ -72,9 +74,9 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) { // validate the file magic number STSBufFileHeader header = {0}; - int32_t ret = fseek(pTSBuf->f, 0, SEEK_SET); + int32_t ret = taosLSeekFile(pTSBuf->pFile, 0, SEEK_SET); UNUSED(ret); - size_t sz = fread(&header, 1, sizeof(STSBufFileHeader), pTSBuf->f); + size_t sz = taosReadFile(pTSBuf->pFile, &header, sizeof(STSBufFileHeader)); UNUSED(sz); // invalid file @@ -112,8 +114,8 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) { return NULL; } - //int64_t pos = ftell(pTSBuf->f); //pos not used - sz = fread(buf, infoSize, 1, pTSBuf->f); + //int64_t pos = ftell(pTSBuf->pFile); //pos not used + sz = taosReadFile(pTSBuf->pFile, buf, infoSize); UNUSED(sz); // the length value for each vnode is not kept in file, so does not set the length value @@ -123,22 +125,22 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) { } free(buf); - ret = fseek(pTSBuf->f, 0, SEEK_END); + ret = taosLSeekFile(pTSBuf->pFile, 0, SEEK_END); UNUSED(ret); - struct stat fileStat; - if (fstat(fileno(pTSBuf->f), &fileStat) != 0) { + int64_t file_size; + if (taosFStatFile(pTSBuf->pFile, &file_size, NULL) != 0) { tsBufDestroy(pTSBuf); return NULL; } - pTSBuf->fileSize = (uint32_t)fileStat.st_size; + pTSBuf->fileSize = (uint32_t)file_size; tsBufResetPos(pTSBuf); // ascending by default pTSBuf->cur.order = TSDB_ORDER_ASC; -// tscDebug("create tsBuf from file:%s, fd:%d, size:%d, numOfGroups:%d, autoDelete:%d", pTSBuf->path, fileno(pTSBuf->f), +// tscDebug("create tsBuf from file:%s, fd:%d, size:%d, numOfGroups:%d, autoDelete:%d", pTSBuf->path, fileno(pTSBuf->pFile), // pTSBuf->fileSize, pTSBuf->numOfGroups, pTSBuf->autoDelete); return pTSBuf; @@ -156,7 +158,7 @@ void* tsBufDestroy(STSBuf* pTSBuf) { tfree(pTSBuf->block.payload); if (!pTSBuf->remainOpen) { - fclose(pTSBuf->f); + taosCloseFile(&pTSBuf->pFile); } if (pTSBuf->autoDelete) { @@ -253,7 +255,7 @@ static void writeDataToDisk(STSBuf* pTSBuf) { tsCompressTimestamp(pTsData->rawBuf, pTsData->len, pTsData->len/TSDB_KEYSIZE, pBlock->payload, pTsData->allocSize, TWO_STAGE_COMP, pTSBuf->assistBuf, pTSBuf->bufSize); - int64_t r = fseek(pTSBuf->f, pTSBuf->fileSize, SEEK_SET); + int64_t r = taosLSeekFile(pTSBuf->pFile, pTSBuf->fileSize, SEEK_SET); assert(r == 0); /* @@ -264,30 +266,30 @@ static void writeDataToDisk(STSBuf* pTSBuf) { * both side has the compressed length is used to support load data forwards/backwords. */ int32_t metaLen = 0; - metaLen += (int32_t)fwrite(&pBlock->tag.nType, 1, sizeof(pBlock->tag.nType), pTSBuf->f); + metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, &pBlock->tag.nType, sizeof(pBlock->tag.nType)); int32_t trueLen = pBlock->tag.nLen; if (pBlock->tag.nType == TSDB_DATA_TYPE_BINARY || pBlock->tag.nType == TSDB_DATA_TYPE_NCHAR) { - metaLen += (int32_t)fwrite(&pBlock->tag.nLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f); - metaLen += (int32_t)fwrite(pBlock->tag.pz, 1, (size_t)pBlock->tag.nLen, pTSBuf->f); + metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, &pBlock->tag.nLen, sizeof(pBlock->tag.nLen)); + metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, pBlock->tag.pz, (size_t)pBlock->tag.nLen); } else if (pBlock->tag.nType == TSDB_DATA_TYPE_FLOAT) { - metaLen += (int32_t)fwrite(&pBlock->tag.nLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f); + metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, &pBlock->tag.nLen, sizeof(pBlock->tag.nLen)); float tfloat = (float)pBlock->tag.d; - metaLen += (int32_t)fwrite(&tfloat, 1, (size_t) pBlock->tag.nLen, pTSBuf->f); + metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, &tfloat, (size_t) pBlock->tag.nLen); } else if (pBlock->tag.nType != TSDB_DATA_TYPE_NULL) { - metaLen += (int32_t)fwrite(&pBlock->tag.nLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f); - metaLen += (int32_t)fwrite(&pBlock->tag.i, 1, (size_t) pBlock->tag.nLen, pTSBuf->f); + metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, &pBlock->tag.nLen, sizeof(pBlock->tag.nLen)); + metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, &pBlock->tag.i, (size_t) pBlock->tag.nLen); } else { trueLen = 0; - metaLen += (int32_t)fwrite(&trueLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f); + metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, &trueLen, sizeof(pBlock->tag.nLen)); } - fwrite(&pBlock->numOfElem, sizeof(pBlock->numOfElem), 1, pTSBuf->f); - fwrite(&pBlock->compLen, sizeof(pBlock->compLen), 1, pTSBuf->f); - fwrite(pBlock->payload, (size_t)pBlock->compLen, 1, pTSBuf->f); - fwrite(&pBlock->compLen, sizeof(pBlock->compLen), 1, pTSBuf->f); + taosWriteFile(pTSBuf->pFile, &pBlock->numOfElem, sizeof(pBlock->numOfElem)); + taosWriteFile(pTSBuf->pFile, &pBlock->compLen, sizeof(pBlock->compLen)); + taosWriteFile(pTSBuf->pFile, pBlock->payload, (size_t)pBlock->compLen); + taosWriteFile(pTSBuf->pFile, &pBlock->compLen, sizeof(pBlock->compLen)); - metaLen += (int32_t) fwrite(&trueLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f); + metaLen += (int32_t) taosWriteFile(pTSBuf->pFile, &trueLen, sizeof(pBlock->tag.nLen)); assert(metaLen == getTagAreaLength(&pBlock->tag)); int32_t blockSize = metaLen + sizeof(pBlock->numOfElem) + sizeof(pBlock->compLen) * 2 + pBlock->compLen; @@ -332,20 +334,20 @@ STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) { * the end of each comp data block */ int32_t prev = -(int32_t) (sizeof(pBlock->padding) + sizeof(pBlock->tag.nLen)); - int32_t ret = fseek(pTSBuf->f, prev, SEEK_CUR); - size_t sz = fread(&pBlock->padding, 1, sizeof(pBlock->padding), pTSBuf->f); - sz = fread(&pBlock->tag.nLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f); + int32_t ret = taosLSeekFile(pTSBuf->pFile, prev, SEEK_CUR); + size_t sz = taosReadFile(pTSBuf->pFile, &pBlock->padding, sizeof(pBlock->padding)); + sz = taosReadFile(pTSBuf->pFile, &pBlock->tag.nLen, sizeof(pBlock->tag.nLen)); UNUSED(sz); pBlock->compLen = pBlock->padding; offset = pBlock->compLen + sizeof(pBlock->compLen) * 2 + sizeof(pBlock->numOfElem) + getTagAreaLength(&pBlock->tag); - ret = fseek(pTSBuf->f, -offset, SEEK_CUR); + ret = taosLSeekFile(pTSBuf->pFile, -offset, SEEK_CUR); UNUSED(ret); } - int32_t ret = fread(&pBlock->tag.nType, sizeof(pBlock->tag.nType), 1, pTSBuf->f); - ret = fread(&pBlock->tag.nLen, sizeof(pBlock->tag.nLen), 1, pTSBuf->f); + int32_t ret = taosReadFile(pTSBuf->pFile, &pBlock->tag.nType, sizeof(pBlock->tag.nType)); + ret = taosReadFile(pTSBuf->pFile, &pBlock->tag.nLen, sizeof(pBlock->tag.nLen)); // NOTE: mix types tags are not supported size_t sz = 0; @@ -356,23 +358,23 @@ STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) { memset(tp, 0, pBlock->tag.nLen + 1); pBlock->tag.pz = tp; - sz = fread(pBlock->tag.pz, (size_t)pBlock->tag.nLen, 1, pTSBuf->f); + sz = taosReadFile(pTSBuf->pFile, pBlock->tag.pz, (size_t)pBlock->tag.nLen); UNUSED(sz); } else if (pBlock->tag.nType == TSDB_DATA_TYPE_FLOAT) { float tfloat = 0; - sz = fread(&tfloat, (size_t) pBlock->tag.nLen, 1, pTSBuf->f); + sz = taosReadFile(pTSBuf->pFile, &tfloat, (size_t) pBlock->tag.nLen); pBlock->tag.d = (double)tfloat; UNUSED(sz); } else if (pBlock->tag.nType != TSDB_DATA_TYPE_NULL) { //TODO check the return value - sz = fread(&pBlock->tag.i, (size_t) pBlock->tag.nLen, 1, pTSBuf->f); + sz = taosReadFile(pTSBuf->pFile, &pBlock->tag.i, (size_t) pBlock->tag.nLen); UNUSED(sz); } - sz = fread(&pBlock->numOfElem, sizeof(pBlock->numOfElem), 1, pTSBuf->f); + sz = taosReadFile(pTSBuf->pFile, &pBlock->numOfElem, sizeof(pBlock->numOfElem)); UNUSED(sz); - sz = fread(&pBlock->compLen, sizeof(pBlock->compLen), 1, pTSBuf->f); + sz = taosReadFile(pTSBuf->pFile, &pBlock->compLen, sizeof(pBlock->compLen)); UNUSED(sz); - sz = fread(pBlock->payload, (size_t)pBlock->compLen, 1, pTSBuf->f); + sz = taosReadFile(pTSBuf->pFile, pBlock->payload, (size_t)pBlock->compLen); if (decomp) { pTSBuf->tsData.len = @@ -381,11 +383,11 @@ STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) { } // read the comp length at the length of comp block - sz = fread(&pBlock->padding, sizeof(pBlock->padding), 1, pTSBuf->f); + sz = taosReadFile(pTSBuf->pFile, &pBlock->padding, sizeof(pBlock->padding)); assert(pBlock->padding == pBlock->compLen); int32_t n = 0; - sz = fread(&n, sizeof(pBlock->tag.nLen), 1, pTSBuf->f); + sz = taosReadFile(pTSBuf->pFile, &n, sizeof(pBlock->tag.nLen)); if (pBlock->tag.nType == TSDB_DATA_TYPE_NULL) { assert(n == 0); } else { @@ -396,7 +398,7 @@ STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) { // for backwards traverse, set the start position at the end of previous block if (order == TSDB_ORDER_DESC) { - int32_t r = fseek(pTSBuf->f, -offset, SEEK_CUR); + int32_t r = taosLSeekFile(pTSBuf->pFile, -offset, SEEK_CUR); UNUSED(r); } @@ -512,7 +514,7 @@ static int32_t tsBufFindGroupById(STSGroupBlockInfoEx* pGroupInfoEx, int32_t num // todo opt performance by cache blocks info static int32_t tsBufFindBlock(STSBuf* pTSBuf, STSGroupBlockInfo* pBlockInfo, int32_t blockIndex) { - if (fseek(pTSBuf->f, pBlockInfo->offset, SEEK_SET) != 0) { + if (taosLSeekFile(pTSBuf->pFile, pBlockInfo->offset, SEEK_SET) != 0) { return -1; } @@ -531,7 +533,7 @@ static int32_t tsBufFindBlock(STSBuf* pTSBuf, STSGroupBlockInfo* pBlockInfo, int STSBlock* pBlock = &pTSBuf->block; int32_t compBlockSize = pBlock->compLen + sizeof(pBlock->compLen) * 2 + sizeof(pBlock->numOfElem) + getTagAreaLength(&pBlock->tag); - int32_t ret = fseek(pTSBuf->f, -compBlockSize, SEEK_CUR); + int32_t ret = taosLSeekFile(pTSBuf->pFile, -compBlockSize, SEEK_CUR); UNUSED(ret); } @@ -548,7 +550,7 @@ static int32_t tsBufFindBlockByTag(STSBuf* pTSBuf, STSGroupBlockInfo* pBlockInfo offset = pBlockInfo->offset + pBlockInfo->compLen; } - if (fseek(pTSBuf->f, (int32_t)offset, SEEK_SET) != 0) { + if (taosLSeekFile(pTSBuf->pFile, (int32_t)offset, SEEK_SET) != 0) { return -1; } @@ -618,11 +620,11 @@ static int32_t doUpdateGroupInfo(STSBuf* pTSBuf, int64_t offset, STSGroupBlockIn return -1; } - if (fseek(pTSBuf->f, (int32_t)offset, SEEK_SET) != 0) { + if (taosLSeekFile(pTSBuf->pFile, (int32_t)offset, SEEK_SET) != 0) { return -1; } - fwrite(pVInfo, sizeof(STSGroupBlockInfo), 1, pTSBuf->f); + taosWriteFile(pTSBuf->pFile, pVInfo, sizeof(STSGroupBlockInfo)); return 0; } @@ -636,19 +638,19 @@ STSGroupBlockInfo* tsBufGetGroupBlockInfo(STSBuf* pTSBuf, int32_t id) { } int32_t STSBufUpdateHeader(STSBuf* pTSBuf, STSBufFileHeader* pHeader) { - if ((pTSBuf->f == NULL) || pHeader == NULL || pHeader->numOfGroup == 0 || pHeader->magic != TS_COMP_FILE_MAGIC) { + if ((pTSBuf->pFile == NULL) || pHeader == NULL || pHeader->numOfGroup == 0 || pHeader->magic != TS_COMP_FILE_MAGIC) { return -1; } assert(pHeader->tsOrder == TSDB_ORDER_ASC || pHeader->tsOrder == TSDB_ORDER_DESC); - int32_t r = fseek(pTSBuf->f, 0, SEEK_SET); + int32_t r = taosLSeekFile(pTSBuf->pFile, 0, SEEK_SET); if (r != 0) { // qError("fseek failed, errno:%d", errno); return -1; } - size_t ws = fwrite(pHeader, sizeof(STSBufFileHeader), 1, pTSBuf->f); + size_t ws = taosWriteFile(pTSBuf->pFile, pHeader, sizeof(STSBufFileHeader)); if (ws != 1) { // qError("ts update header fwrite failed, size:%d, expected size:%d", (int32_t)ws, (int32_t)sizeof(STSBufFileHeader)); return -1; @@ -823,12 +825,12 @@ int32_t tsBufMerge(STSBuf* pDestBuf, const STSBuf* pSrcBuf) { pBlockInfoEx->info.id = id; } - int32_t r = fseek(pDestBuf->f, 0, SEEK_END); + int32_t r = taosLSeekFile(pDestBuf->pFile, 0, SEEK_END); assert(r == 0); int64_t offset = getDataStartOffset(); int32_t size = (int32_t)pSrcBuf->fileSize - (int32_t)offset; - int64_t written = taosFSendFile(pDestBuf->f, pSrcBuf->f, &offset, size); + int64_t written = taosFSendFile(pDestBuf->pFile, pSrcBuf->pFile, &offset, size); if (written == -1 || written != size) { return -1; @@ -839,17 +841,18 @@ int32_t tsBufMerge(STSBuf* pDestBuf, const STSBuf* pSrcBuf) { int32_t oldSize = pDestBuf->fileSize; // file meta data may be cached, close and reopen the file for accurate file size. - fclose(pDestBuf->f); - pDestBuf->f = fopen(pDestBuf->path, "rb+"); - if (pDestBuf->f == NULL) { + taosCloseFile(&pDestBuf->pFile); + // pDestBuf->pFile = fopen(pDestBuf->path, "rb+"); + pDestBuf->pFile = taosOpenFile(pDestBuf->path, TD_FILE_WRITE | TD_FILE_READ); + if (pDestBuf->pFile == NULL) { return -1; } - struct stat fileStat; - if (fstat(fileno(pDestBuf->f), &fileStat) != 0) { + int64_t file_size; + if (taosFStatFile(pDestBuf->pFile, &file_size, NULL) != 0) { return -1; } - pDestBuf->fileSize = (uint32_t)fileStat.st_size; + pDestBuf->fileSize = (uint32_t)file_size; assert(pDestBuf->fileSize == oldSize + size); @@ -868,13 +871,13 @@ STSBuf* tsBufCreateFromCompBlocks(const char* pData, int32_t numOfBlocks, int32_ // update prev vnode length info in file TSBufUpdateGroupInfo(pTSBuf, pTSBuf->numOfGroups - 1, pBlockInfo); - int32_t ret = fseek(pTSBuf->f, pBlockInfo->offset, SEEK_SET); + int32_t ret = taosLSeekFile(pTSBuf->pFile, pBlockInfo->offset, SEEK_SET); if (ret == -1) { // qError("fseek failed, errno:%d", errno); tsBufDestroy(pTSBuf); return NULL; } - size_t sz = fwrite((void*)pData, 1, len, pTSBuf->f); + size_t sz = taosWriteFile(pTSBuf->pFile, (void*)pData, len); if (sz != len) { // qError("ts data fwrite failed, write size:%d, expected size:%d", (int32_t)sz, len); tsBufDestroy(pTSBuf); @@ -893,7 +896,7 @@ STSBuf* tsBufCreateFromCompBlocks(const char* pData, int32_t numOfBlocks, int32_ } // TODO taosFsync?? -// if (taosFsync(fileno(pTSBuf->f)) == -1) { +// if (taosFsync(fileno(pTSBuf->pFile)) == -1) { //// qError("fsync failed, errno:%d", errno); // tsBufDestroy(pTSBuf); // return NULL; @@ -1071,15 +1074,15 @@ int32_t dumpFileBlockByGroupId(STSBuf* pTSBuf, int32_t groupIndex, void* buf, in *len = 0; *numOfBlocks = 0; - if (fseek(pTSBuf->f, pBlockInfo->offset, SEEK_SET) != 0) { - int code = TAOS_SYSTEM_ERROR(ferror(pTSBuf->f)); + if (taosLSeekFile(pTSBuf->pFile, pBlockInfo->offset, SEEK_SET) != 0) { + int code = TAOS_SYSTEM_ERROR(taosEOFFile(pTSBuf->pFile)); // qError("%p: fseek failed: %s", pSql, tstrerror(code)); return code; } - size_t s = fread(buf, 1, pBlockInfo->compLen, pTSBuf->f); + size_t s = taosReadFile(pTSBuf->pFile, buf, pBlockInfo->compLen); if (s != pBlockInfo->compLen) { - int code = TAOS_SYSTEM_ERROR(ferror(pTSBuf->f)); + int code = TAOS_SYSTEM_ERROR(taosEOFFile(pTSBuf->pFile)); // tscError("%p: fread didn't return expected data: %s", pSql, tstrerror(code)); return code; } diff --git a/source/dnode/mgmt/daemon/CMakeLists.txt b/source/dnode/mgmt/daemon/CMakeLists.txt index a577344761..e07c15c95a 100644 --- a/source/dnode/mgmt/daemon/CMakeLists.txt +++ b/source/dnode/mgmt/daemon/CMakeLists.txt @@ -6,4 +6,4 @@ target_include_directories( PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) -target_link_libraries(taosd dnode config util os) +target_link_libraries(taosd dnode util os) diff --git a/source/dnode/mgmt/daemon/inc/dmnInt.h b/source/dnode/mgmt/daemon/inc/dmnInt.h index 19ec147769..82e95782cb 100644 --- a/source/dnode/mgmt/daemon/inc/dmnInt.h +++ b/source/dnode/mgmt/daemon/inc/dmnInt.h @@ -17,7 +17,7 @@ #ifndef _TD_DMN_INT_H_ #define _TD_DMN_INT_H_ -#include "config.h" +#include "tconfig.h" #include "dnode.h" #include "taoserror.h" #include "tglobal.h" @@ -28,15 +28,9 @@ extern "C" { #endif -int32_t dmnAddLogCfg(SConfig *pCfg); -int32_t dmnInitLog(const char *cfgDir, const char *envFile, const char *apolloUrl); -int32_t dmnLoadCfg(SConfig *pConfig, const char *inputCfgDir, const char *envFile, const char *apolloUrl); +SDnodeObjCfg dmnGetObjCfg(); -SConfig *dmnReadCfg(const char *cfgDir, const char *envFile, const char *apolloUrl); -SDnodeEnvCfg dmnGetEnvCfg(SConfig *pCfg); -SDnodeObjCfg dmnGetObjCfg(SConfig *pCfg); - -void dmnDumpCfg(SConfig *pCfg); +void dmnDumpCfg(); void dmnPrintVersion(); void dmnGenerateGrant(); diff --git a/source/dnode/mgmt/daemon/src/dmnCfg.c b/source/dnode/mgmt/daemon/src/dmnCfg.c index 7bf538caed..1e7d1d11ea 100644 --- a/source/dnode/mgmt/daemon/src/dmnCfg.c +++ b/source/dnode/mgmt/daemon/src/dmnCfg.c @@ -15,194 +15,23 @@ #define _DEFAULT_SOURCE #include "dmnInt.h" -#include "tlocale.h" -#include "ttimezone.h" +#include "tconfig.h" -static int32_t dmnAddEpCfg(SConfig *pCfg) { - char defaultFqdn[TSDB_FQDN_LEN] = {0}; - if (taosGetFqdn(defaultFqdn) != 0) { - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - if (cfgAddString(pCfg, "fqdn", defaultFqdn) != 0) return -1; - - int32_t defaultServerPort = 6030; - if (cfgAddInt32(pCfg, "serverPort", defaultServerPort, 1, 65056) != 0) return -1; - - char defaultFirstEp[TSDB_EP_LEN] = {0}; - char defaultSecondEp[TSDB_EP_LEN] = {0}; - snprintf(defaultFirstEp, TSDB_EP_LEN, "%s:%d", defaultFqdn, defaultServerPort); - snprintf(defaultSecondEp, TSDB_EP_LEN, "%s:%d", defaultFqdn, defaultServerPort); - if (cfgAddString(pCfg, "firstEp", defaultFirstEp) != 0) return -1; - if (cfgAddString(pCfg, "secondEp", defaultSecondEp) != 0) return -1; - - return 0; -} - -static int32_t dmnAddDirCfg(SConfig *pCfg) { - if (cfgAddDir(pCfg, "dataDir", tsDataDir) != 0) return -1; - if (cfgAddDir(pCfg, "tempDir", tsTempDir) != 0) return -1; - if (cfgAddFloat(pCfg, "minimalDataDirGB", 2.0f, 0.001f, 10000000) != 0) return -1; - if (cfgAddFloat(pCfg, "minimalTempDirGB", 1.0f, 0.001f, 10000000) != 0) return -1; - return 0; -} - -static int32_t dmnCheckDirCfg(SConfig *pCfg) { - tstrncpy(tsDataDir, cfgGetItem(pCfg, "dataDir")->str, PATH_MAX); - tstrncpy(tsTempDir, cfgGetItem(pCfg, "tempDir")->str, PATH_MAX); - tsDataSpace.reserved = cfgGetItem(pCfg, "minimalDataDirGB")->fval; - tsTempSpace.reserved = cfgGetItem(pCfg, "minimalTempDirGB")->fval; - return 0; -} - -static int32_t dmnAddVersionCfg(SConfig *pCfg) { - if (cfgAddString(pCfg, "buildinfo", buildinfo) != 0) return -1; - if (cfgAddString(pCfg, "gitinfo", gitinfo) != 0) return -1; - if (cfgAddString(pCfg, "version", version) != 0) return -1; - return 0; -} - -static int32_t dmnAddDnodeCfg(SConfig *pCfg) { - if (dmnAddEpCfg(pCfg) != 0) return -1; - if (dmnAddDirCfg(pCfg) != 0) return -1; - if (dmnAddVersionCfg(pCfg) != 0) return -1; - - if (cfgAddTimezone(pCfg, "timezone", "") != 0) return -1; - if (cfgAddLocale(pCfg, "locale", "") != 0) return -1; - if (cfgAddCharset(pCfg, "charset", "") != 0) return -1; - if (cfgAddInt32(pCfg, "numOfCores", 2, 1, 100000) != 0) return -1; - if (cfgAddInt32(pCfg, "numOfCommitThreads", 4, 1, 1000) != 0) return -1; - if (cfgAddBool(pCfg, "telemetryReporting", 0) != 0) return -1; - if (cfgAddBool(pCfg, "enableCoreFile", 0) != 0) return -1; - if (cfgAddInt32(pCfg, "supportVnodes", 256, 0, 65536) != 0) return -1; - if (cfgAddInt32(pCfg, "statusInterval", 1, 1, 30) != 0) return -1; - if (cfgAddFloat(pCfg, "numOfThreadsPerCore", 1, 0, 10) != 0) return -1; - if (cfgAddFloat(pCfg, "ratioOfQueryCores", 1, 0, 5) != 0) return -1; - if (cfgAddInt32(pCfg, "maxShellConns", 50000, 10, 50000000) != 0) return -1; - if (cfgAddInt32(pCfg, "shellActivityTimer", 3, 1, 120) != 0) return -1; - return 0; -} - -static int32_t dmnCheckCfg(SConfig *pCfg) { - bool enableCore = cfgGetItem(pCfg, "enableCoreFile")->bval; - taosSetCoreDump(enableCore); - - if (dmnCheckDirCfg(pCfg) != 0) { - return -1; - } - - taosGetSystemInfo(); - - tsSetTimeZone(); - tsSetLocale(); - - if (tsNumOfCores <= 0) { - tsNumOfCores = 1; - } - - if (tsQueryBufferSize >= 0) { - tsQueryBufferSizeBytes = tsQueryBufferSize * 1048576UL; - } - - return 0; -} - -SConfig *dmnReadCfg(const char *cfgDir, const char *envFile, const char *apolloUrl) { - SConfig *pCfg = cfgInit(); - if (pCfg == NULL) return NULL; - - if (dmnAddLogCfg(pCfg) != 0) { - uError("failed to add log cfg since %s", terrstr()); - cfgCleanup(pCfg); - return NULL; - } - - if (dmnAddDnodeCfg(pCfg) != 0) { - uError("failed to init dnode cfg since %s", terrstr()); - cfgCleanup(pCfg); - return NULL; - } - - if (dmnLoadCfg(pCfg, cfgDir, envFile, apolloUrl) != 0) { - uError("failed to load cfg since %s", terrstr()); - cfgCleanup(pCfg); - return NULL; - } - - if (dmnCheckCfg(pCfg) != 0) { - uError("failed to check cfg since %s", terrstr()); - cfgCleanup(pCfg); - return NULL; - } - - cfgDumpCfg(pCfg); - return pCfg; -} - -void dmnDumpCfg(SConfig *pCfg) { - printf("taos global config:\n"); - printf("==================================\n"); - - SConfigItem *pItem = cfgIterate(pCfg, NULL); - while (pItem != NULL) { - switch (pItem->dtype) { - case CFG_DTYPE_BOOL: - printf("cfg:%s, value:%u src:%s\n", pItem->name, pItem->bval, cfgStypeStr(pItem->stype)); - break; - case CFG_DTYPE_INT32: - printf("cfg:%s, value:%d src:%s\n", pItem->name, pItem->i32, cfgStypeStr(pItem->stype)); - break; - case CFG_DTYPE_INT64: - printf("cfg:%s, value:%" PRId64 " src:%s\n", pItem->name, pItem->i64, cfgStypeStr(pItem->stype)); - break; - case CFG_DTYPE_FLOAT: - printf("cfg:%s, value:%f src:%s\n", pItem->name, pItem->fval, cfgStypeStr(pItem->stype)); - break; - case CFG_DTYPE_STRING: - case CFG_DTYPE_IPSTR: - case CFG_DTYPE_DIR: - case CFG_DTYPE_LOCALE: - case CFG_DTYPE_CHARSET: - case CFG_DTYPE_TIMEZONE: - printf("cfg:%s, value:%s src:%s\n", pItem->name, pItem->str, cfgStypeStr(pItem->stype)); - break; - } - pItem = cfgIterate(pCfg, pItem); - } -} - -SDnodeEnvCfg dmnGetEnvCfg(SConfig *pCfg) { - SDnodeEnvCfg envCfg = {0}; - - const char *vstr = cfgGetItem(pCfg, "version")->str; - envCfg.sver = 30000000; - tstrncpy(envCfg.buildinfo, cfgGetItem(pCfg, "buildinfo")->str, sizeof(envCfg.buildinfo)); - tstrncpy(envCfg.gitinfo, cfgGetItem(pCfg, "gitinfo")->str, sizeof(envCfg.gitinfo)); - tstrncpy(envCfg.timezone, cfgGetItem(pCfg, "timezone")->str, sizeof(envCfg.timezone)); - tstrncpy(envCfg.locale, cfgGetItem(pCfg, "locale")->str, sizeof(envCfg.locale)); - tstrncpy(envCfg.charset, cfgGetItem(pCfg, "charset")->str, sizeof(envCfg.charset)); - envCfg.numOfCores = cfgGetItem(pCfg, "numOfCores")->i32; - envCfg.numOfCommitThreads = (uint16_t)cfgGetItem(pCfg, "numOfCommitThreads")->i32; - envCfg.enableTelem = cfgGetItem(pCfg, "telemetryReporting")->bval; - - return envCfg; -} - -SDnodeObjCfg dmnGetObjCfg(SConfig *pCfg) { +SDnodeObjCfg dmnGetObjCfg() { + SConfig *pCfg = taosGetCfg(); SDnodeObjCfg objCfg = {0}; objCfg.numOfSupportVnodes = cfgGetItem(pCfg, "supportVnodes")->i32; - objCfg.statusInterval = cfgGetItem(pCfg, "statusInterval")->i32; - objCfg.numOfThreadsPerCore = cfgGetItem(pCfg, "numOfThreadsPerCore")->fval; - objCfg.ratioOfQueryCores = cfgGetItem(pCfg, "ratioOfQueryCores")->fval; - objCfg.maxShellConns = cfgGetItem(pCfg, "maxShellConns")->i32; - objCfg.shellActivityTimer = cfgGetItem(pCfg, "shellActivityTimer")->i32; tstrncpy(objCfg.dataDir, cfgGetItem(pCfg, "dataDir")->str, sizeof(objCfg.dataDir)); - tstrncpy(objCfg.firstEp, cfgGetItem(pCfg, "firstEp")->str, sizeof(objCfg.firstEp)); tstrncpy(objCfg.secondEp, cfgGetItem(pCfg, "secondEp")->str, sizeof(objCfg.firstEp)); objCfg.serverPort = (uint16_t)cfgGetItem(pCfg, "serverPort")->i32; tstrncpy(objCfg.localFqdn, cfgGetItem(pCfg, "fqdn")->str, sizeof(objCfg.localFqdn)); snprintf(objCfg.localEp, sizeof(objCfg.localEp), "%s:%u", objCfg.localFqdn, objCfg.serverPort); return objCfg; +} + +void dmnDumpCfg() { + SConfig *pCfg = taosGetCfg(); + cfgDumpCfg(pCfg, 0, 1); } \ No newline at end of file diff --git a/source/dnode/mgmt/daemon/src/dmnLog.c b/source/dnode/mgmt/daemon/src/dmnLog.c deleted file mode 100644 index 22a7d7b080..0000000000 --- a/source/dnode/mgmt/daemon/src/dmnLog.c +++ /dev/null @@ -1,132 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "dmnInt.h" - -int32_t dmnAddLogCfg(SConfig *pCfg) { - if (cfgAddDir(pCfg, "logDir", "/var/log/taos") != 0) return -1; - if (cfgAddFloat(pCfg, "minimalLogDirGB", 1.0f, 0.001f, 10000000) != 0) return -1; - if (cfgAddBool(pCfg, "asyncLog", 1) != 0) return -1; - if (cfgAddInt32(pCfg, "numOfLogLines", 10000000, 1000, 2000000000) != 0) return -1; - if (cfgAddInt32(pCfg, "logKeepDays", 0, -365000, 365000) != 0) return -1; - if (cfgAddInt32(pCfg, "debugFlag", 0, 0, 255) != 0) return -1; - if (cfgAddInt32(pCfg, "dDebugFlag", 0, 0, 255) != 0) return -1; - if (cfgAddInt32(pCfg, "vDebugFlag", 0, 0, 255) != 0) return -1; - if (cfgAddInt32(pCfg, "mDebugFlag", 0, 0, 255) != 0) return -1; - if (cfgAddInt32(pCfg, "cDebugFlag", 0, 0, 255) != 0) return -1; - if (cfgAddInt32(pCfg, "jniDebugFlag", 0, 0, 255) != 0) return -1; - if (cfgAddInt32(pCfg, "tmrDebugFlag", 0, 0, 255) != 0) return -1; - if (cfgAddInt32(pCfg, "uDebugFlag", 0, 0, 255) != 0) return -1; - if (cfgAddInt32(pCfg, "rpcDebugFlag", 0, 0, 255) != 0) return -1; - if (cfgAddInt32(pCfg, "qDebugFlag", 0, 0, 255) != 0) return -1; - if (cfgAddInt32(pCfg, "wDebugFlag", 0, 0, 255) != 0) return -1; - if (cfgAddInt32(pCfg, "sDebugFlag", 0, 0, 255) != 0) return -1; - if (cfgAddInt32(pCfg, "tsdbDebugFlag", 0, 0, 255) != 0) return -1; - if (cfgAddInt32(pCfg, "tqDebugFlag", 0, 0, 255) != 0) return -1; - if (cfgAddInt32(pCfg, "fsDebugFlag", 0, 0, 255) != 0) return -1; - return 0; -} - -int32_t dmnSetLogCfg(SConfig *pCfg) { - tstrncpy(tsLogDir, cfgGetItem(pCfg, "logDir")->str, PATH_MAX); - tsLogSpace.reserved = cfgGetItem(pCfg, "minimalLogDirGB")->fval; - tsAsyncLog = cfgGetItem(pCfg, "asyncLog")->bval; - tsNumOfLogLines = cfgGetItem(pCfg, "numOfLogLines")->i32; - tsLogKeepDays = cfgGetItem(pCfg, "logKeepDays")->i32; - dDebugFlag = cfgGetItem(pCfg, "dDebugFlag")->i32; - vDebugFlag = cfgGetItem(pCfg, "vDebugFlag")->i32; - mDebugFlag = cfgGetItem(pCfg, "mDebugFlag")->i32; - cDebugFlag = cfgGetItem(pCfg, "cDebugFlag")->i32; - jniDebugFlag = cfgGetItem(pCfg, "jniDebugFlag")->i32; - tmrDebugFlag = cfgGetItem(pCfg, "tmrDebugFlag")->i32; - uDebugFlag = cfgGetItem(pCfg, "uDebugFlag")->i32; - rpcDebugFlag = cfgGetItem(pCfg, "rpcDebugFlag")->i32; - qDebugFlag = cfgGetItem(pCfg, "qDebugFlag")->i32; - wDebugFlag = cfgGetItem(pCfg, "wDebugFlag")->i32; - sDebugFlag = cfgGetItem(pCfg, "sDebugFlag")->i32; - tsdbDebugFlag = cfgGetItem(pCfg, "tsdbDebugFlag")->i32; - tqDebugFlag = cfgGetItem(pCfg, "tqDebugFlag")->i32; - fsDebugFlag = cfgGetItem(pCfg, "fsDebugFlag")->i32; - - int32_t debugFlag = cfgGetItem(pCfg, "debugFlag")->i32; - taosSetAllDebugFlag(debugFlag); - - return 0; -} - -int32_t dmnInitLog(const char *cfgDir, const char *envFile, const char *apolloUrl) { - SConfig *pCfg = cfgInit(); - if (pCfg == NULL) return -1; - - if (dmnAddLogCfg(pCfg) != 0) { - printf("failed to add log cfg since %s\n", terrstr()); - cfgCleanup(pCfg); - return -1; - } - - if (dmnLoadCfg(pCfg, cfgDir, envFile, apolloUrl) != 0) { - printf("failed to load log cfg since %s\n", terrstr()); - cfgCleanup(pCfg); - return -1; - } - - if (dmnSetLogCfg(pCfg) != 0) { - printf("failed to set log cfg since %s\n", terrstr()); - cfgCleanup(pCfg); - return -1; - } - - if (taosInitLog("taosdlog", 1) != 0) { - printf("failed to init log file since %s\n", terrstr()); - cfgCleanup(pCfg); - return -1; - } - - cfgCleanup(pCfg); - return 0; -} - -int32_t dmnLoadCfg(SConfig *pConfig, const char *inputCfgDir, const char *envFile, const char *apolloUrl) { - char configDir[PATH_MAX] = {0}; - char configFile[PATH_MAX + 100] = {0}; - - taosExpandDir(inputCfgDir, configDir, PATH_MAX); - snprintf(configFile, sizeof(configFile), "%s" TD_DIRSEP "taos.cfg", configDir); - - if (cfgLoad(pConfig, CFG_STYPE_APOLLO_URL, apolloUrl) != 0) { - uError("failed to load from apollo url:%s since %s\n", apolloUrl, terrstr()); - return -1; - } - - if (cfgLoad(pConfig, CFG_STYPE_CFG_FILE, configFile) != 0) { - if (cfgLoad(pConfig, CFG_STYPE_CFG_FILE, configDir) != 0) { - uError("failed to load from config file:%s since %s\n", configFile, terrstr()); - return -1; - } - } - - if (cfgLoad(pConfig, CFG_STYPE_ENV_FILE, envFile) != 0) { - uError("failed to load from env file:%s since %s\n", envFile, terrstr()); - return -1; - } - - if (cfgLoad(pConfig, CFG_STYPE_ENV_VAR, NULL) != 0) { - uError("failed to load from global env variables since %s\n", terrstr()); - return -1; - } - - return 0; -} diff --git a/source/dnode/mgmt/daemon/src/dmnMain.c b/source/dnode/mgmt/daemon/src/dmnMain.c index 1f1d2d8d05..9ce09c0a5d 100644 --- a/source/dnode/mgmt/daemon/src/dmnMain.c +++ b/source/dnode/mgmt/daemon/src/dmnMain.c @@ -72,15 +72,14 @@ static int32_t dmnParseOption(int32_t argc, char const *argv[]) { return 0; } -int32_t dmnRunDnode(SConfig *pCfg) { - SDnodeEnvCfg envCfg = dmnGetEnvCfg(pCfg); - if (dndInit(&envCfg) != 0) { +int32_t dmnRunDnode() { + if (dndInit() != 0) { uInfo("Failed to start TDengine, please check the log"); return -1; } - SDnodeObjCfg objCfg = dmnGetObjCfg(pCfg); - SDnode *pDnode = dndCreate(&objCfg); + SDnodeObjCfg objCfg = dmnGetObjCfg(); + SDnode *pDnode = dndCreate(&objCfg); if (pDnode == NULL) { uInfo("Failed to start TDengine, please check the log"); return -1; @@ -93,6 +92,7 @@ int32_t dmnRunDnode(SConfig *pCfg) { dndClose(pDnode); dndCleanup(); taosCloseLog(); + taosCleanupCfg(); return 0; } @@ -111,23 +111,21 @@ int main(int argc, char const *argv[]) { return 0; } - if (dmnInitLog(configDir, dmn.envFile, dmn.apolloUrl) != 0) { + if (taosCreateLog("taosdlog", 1, configDir, dmn.envFile, dmn.apolloUrl, 0) != 0) { + uInfo("Failed to start TDengine since read config error"); return -1; } - SConfig *pCfg = dmnReadCfg(configDir, dmn.envFile, dmn.apolloUrl); - if (pCfg == NULL) { + if (taosInitCfg(configDir, dmn.envFile, dmn.apolloUrl, 0) != 0) { uInfo("Failed to start TDengine since read config error"); return -1; } if (dmn.dumpConfig) { - dmnDumpCfg(pCfg); - cfgCleanup(pCfg); + dmnDumpCfg(); + taosCleanupCfg(); return 0; } - int32_t code = dmnRunDnode(pCfg); - cfgCleanup(pCfg); - return code; + return dmnRunDnode(); } diff --git a/source/dnode/mgmt/impl/inc/dndEnv.h b/source/dnode/mgmt/impl/inc/dndEnv.h index 9eff246323..cbd5eb5827 100644 --- a/source/dnode/mgmt/impl/inc/dndEnv.h +++ b/source/dnode/mgmt/impl/inc/dndEnv.h @@ -124,9 +124,8 @@ typedef struct { typedef struct SDnode { EStat stat; SDnodeObjCfg cfg; - SDnodeEnvCfg env; SDnodeDir dir; - FileFd lockFd; + TdFilePtr pLockFile; SDnodeMgmt dmgmt; SMnodeMgmt mmgmt; SQnodeMgmt qmgmt; @@ -138,11 +137,6 @@ typedef struct SDnode { SStartupReq startup; } SDnode; -typedef struct { - int8_t once; - SDnodeEnvCfg cfg; -} SDnodeEnv; - #ifdef __cplusplus } #endif diff --git a/source/dnode/mgmt/impl/inc/dndInt.h b/source/dnode/mgmt/impl/inc/dndInt.h index 829fabd006..417bc1e041 100644 --- a/source/dnode/mgmt/impl/inc/dndInt.h +++ b/source/dnode/mgmt/impl/inc/dndInt.h @@ -35,6 +35,7 @@ extern "C" { #include "tthread.h" #include "ttime.h" #include "tworker.h" +#include "tglobal.h" #include "dnode.h" diff --git a/source/dnode/mgmt/impl/src/dndBnode.c b/source/dnode/mgmt/impl/src/dndBnode.c index e37a164660..81b020c152 100644 --- a/source/dnode/mgmt/impl/src/dndBnode.c +++ b/source/dnode/mgmt/impl/src/dndBnode.c @@ -62,14 +62,15 @@ static int32_t dndReadBnodeFile(SDnode *pDnode) { char file[PATH_MAX + 20]; snprintf(file, PATH_MAX + 20, "%s/bnode.json", pDnode->dir.dnode); - FILE *fp = fopen(file, "r"); - if (fp == NULL) { + // FILE *fp = fopen(file, "r"); + TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ); + if (pFile == NULL) { dDebug("file %s not exist", file); code = 0; goto PRASE_BNODE_OVER; } - len = (int32_t)fread(content, 1, maxLen, fp); + len = (int32_t)taosReadFile(pFile, content, maxLen); if (len <= 0) { dError("failed to read %s since content is null", file); goto PRASE_BNODE_OVER; @@ -102,7 +103,7 @@ static int32_t dndReadBnodeFile(SDnode *pDnode) { PRASE_BNODE_OVER: if (content != NULL) free(content); if (root != NULL) cJSON_Delete(root); - if (fp != NULL) fclose(fp); + if (pFile != NULL) taosCloseFile(&pFile); terrno = code; return code; @@ -114,8 +115,9 @@ static int32_t dndWriteBnodeFile(SDnode *pDnode) { char file[PATH_MAX + 20]; snprintf(file, PATH_MAX + 20, "%s/bnode.json", pDnode->dir.dnode); - FILE *fp = fopen(file, "w"); - if (fp == NULL) { + // FILE *fp = fopen(file, "w"); + TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pFile == NULL) { terrno = TSDB_CODE_DND_BNODE_WRITE_FILE_ERROR; dError("failed to write %s since %s", file, terrstr()); return -1; @@ -130,9 +132,9 @@ static int32_t dndWriteBnodeFile(SDnode *pDnode) { len += snprintf(content + len, maxLen - len, " \"dropped\": %d\n", pMgmt->dropped); len += snprintf(content + len, maxLen - len, "}\n"); - fwrite(content, 1, len, fp); - taosFsyncFile(fileno(fp)); - fclose(fp); + taosWriteFile(pFile, content, len); + taosFsyncFile(pFile); + taosCloseFile(&pFile); free(content); char realfile[PATH_MAX + 20]; @@ -179,7 +181,7 @@ static void dndBuildBnodeOption(SDnode *pDnode, SBnodeOpt *pOption) { pOption->sendRedirectRspFp = dndSendRedirectRsp; pOption->dnodeId = dndGetDnodeId(pDnode); pOption->clusterId = dndGetClusterId(pDnode); - pOption->sver = pDnode->env.sver; + pOption->sver = tsVersion; } static int32_t dndOpenBnode(SDnode *pDnode) { diff --git a/source/dnode/mgmt/impl/src/dndEnv.c b/source/dnode/mgmt/impl/src/dndEnv.c index 58517c8151..247f52f958 100644 --- a/source/dnode/mgmt/impl/src/dndEnv.c +++ b/source/dnode/mgmt/impl/src/dndEnv.c @@ -25,7 +25,7 @@ #include "tfs.h" #include "wal.h" -static SDnodeEnv dndEnv = {0}; +static int8_t once = DND_ENV_INIT; EStat dndGetStat(SDnode *pDnode) { return pDnode->stat; } @@ -59,31 +59,31 @@ void dndGetStartup(SDnode *pDnode, SStartupReq *pStartup) { pStartup->finished = (dndGetStat(pDnode) == DND_STAT_RUNNING); } -static FileFd dndCheckRunning(char *dataDir) { +static TdFilePtr dndCheckRunning(char *dataDir) { char filepath[PATH_MAX] = {0}; snprintf(filepath, sizeof(filepath), "%s/.running", dataDir); - FileFd fd = taosOpenFileCreateWriteTrunc(filepath); - if (fd < 0) { + TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pFile == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); dError("failed to lock file:%s since %s, quit", filepath, terrstr()); - return -1; + return NULL; } - int32_t ret = taosLockFile(fd); + int32_t ret = taosLockFile(pFile); if (ret != 0) { terrno = TAOS_SYSTEM_ERROR(errno); dError("failed to lock file:%s since %s, quit", filepath, terrstr()); - taosCloseFile(fd); - return -1; + taosCloseFile(&pFile); + return NULL; } - return fd; + return pFile; } static int32_t dndCreateImp(SDnode *pDnode, SDnodeObjCfg *pCfg) { - pDnode->lockFd = dndCheckRunning(pCfg->dataDir); - if (pDnode->lockFd < 0) { + pDnode->pLockFile = dndCheckRunning(pCfg->dataDir); + if (pDnode->pLockFile == NULL) { return -1; } @@ -137,7 +137,6 @@ static int32_t dndCreateImp(SDnode *pDnode, SDnodeObjCfg *pCfg) { } memcpy(&pDnode->cfg, pCfg, sizeof(SDnodeObjCfg)); - memcpy(&pDnode->env, &dndEnv.cfg, sizeof(SDnodeEnvCfg)); return 0; } @@ -148,10 +147,10 @@ static void dndCloseImp(SDnode *pDnode) { tfree(pDnode->dir.snode); tfree(pDnode->dir.bnode); - if (pDnode->lockFd >= 0) { - taosUnLockFile(pDnode->lockFd); - taosCloseFile(pDnode->lockFd); - pDnode->lockFd = 0; + if (pDnode->pLockFile != NULL) { + taosUnLockFile(pDnode->pLockFile); + taosCloseFile(&pDnode->pLockFile); + pDnode->pLockFile = NULL; } } @@ -259,8 +258,8 @@ void dndClose(SDnode *pDnode) { dInfo("dnode object is closed, data:%p", pDnode); } -int32_t dndInit(const SDnodeEnvCfg *pCfg) { - if (atomic_val_compare_exchange_8(&dndEnv.once, DND_ENV_INIT, DND_ENV_READY) != DND_ENV_INIT) { +int32_t dndInit() { + if (atomic_val_compare_exchange_8(&once, DND_ENV_INIT, DND_ENV_READY) != DND_ENV_INIT) { terrno = TSDB_CODE_REPEAT_INIT; dError("failed to init dnode env since %s", terrstr()); return -1; @@ -283,11 +282,8 @@ int32_t dndInit(const SDnodeEnvCfg *pCfg) { } SVnodeOpt vnodeOpt = { - .sver = pCfg->sver, - .timezone = pCfg->timezone, - .locale = pCfg->locale, - .charset = pCfg->charset, - .nthreads = pCfg->numOfCommitThreads, + .sver = tsVersion, + .nthreads = tsNumOfCommitThreads, .putReqToVQueryQFp = dndPutReqToVQueryQ, .sendReqToDnodeFp = dndSendReqToDnode }; @@ -298,13 +294,12 @@ int32_t dndInit(const SDnodeEnvCfg *pCfg) { return -1; } - memcpy(&dndEnv.cfg, pCfg, sizeof(SDnodeEnvCfg)); dInfo("dnode env is initialized"); return 0; } void dndCleanup() { - if (atomic_val_compare_exchange_8(&dndEnv.once, DND_ENV_READY, DND_ENV_CLEANUP) != DND_ENV_READY) { + if (atomic_val_compare_exchange_8(&once, DND_ENV_READY, DND_ENV_CLEANUP) != DND_ENV_READY) { dError("dnode env is already cleaned up"); return; } diff --git a/source/dnode/mgmt/impl/src/dndMgmt.c b/source/dnode/mgmt/impl/src/dndMgmt.c index 1748c52c66..8f18222ab6 100644 --- a/source/dnode/mgmt/impl/src/dndMgmt.c +++ b/source/dnode/mgmt/impl/src/dndMgmt.c @@ -185,16 +185,16 @@ static int32_t dndReadDnodes(SDnode *pDnode) { int32_t maxLen = 256 * 1024; char *content = calloc(1, maxLen + 1); cJSON *root = NULL; - FILE *fp = NULL; - fp = fopen(pMgmt->file, "r"); - if (fp == NULL) { + // fp = fopen(pMgmt->file, "r"); + TdFilePtr pFile = taosOpenFile(pMgmt->file, TD_FILE_READ); + if (pFile == NULL) { dDebug("file %s not exist", pMgmt->file); code = 0; goto PRASE_DNODE_OVER; } - len = (int32_t)fread(content, 1, maxLen, fp); + len = (int32_t)taosReadFile(pFile, content, maxLen); if (len <= 0) { dError("failed to read %s since content is null", pMgmt->file); goto PRASE_DNODE_OVER; @@ -286,7 +286,7 @@ static int32_t dndReadDnodes(SDnode *pDnode) { PRASE_DNODE_OVER: if (content != NULL) free(content); if (root != NULL) cJSON_Delete(root); - if (fp != NULL) fclose(fp); + if (pFile != NULL) taosCloseFile(&pFile); if (dndIsEpChanged(pDnode, pMgmt->dnodeId, pDnode->cfg.localEp)) { dError("localEp %s different with %s and need reconfigured", pDnode->cfg.localEp, pMgmt->file); @@ -296,7 +296,7 @@ PRASE_DNODE_OVER: if (taosArrayGetSize(pMgmt->pDnodeEps) == 0) { SDnodeEp dnodeEp = {0}; dnodeEp.isMnode = 1; - taosGetFqdnPortFromEp(pDnode->cfg.firstEp, pDnode->cfg.serverPort, &dnodeEp.ep); + taosGetFqdnPortFromEp(pDnode->cfg.firstEp, &dnodeEp.ep); taosArrayPush(pMgmt->pDnodeEps, &dnodeEp); } @@ -309,8 +309,9 @@ PRASE_DNODE_OVER: static int32_t dndWriteDnodes(SDnode *pDnode) { SDnodeMgmt *pMgmt = &pDnode->dmgmt; - FILE *fp = fopen(pMgmt->file, "w"); - if (fp == NULL) { + // FILE *fp = fopen(pMgmt->file, "w"); + TdFilePtr pFile = taosOpenFile(pMgmt->file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pFile == NULL) { dError("failed to write %s since %s", pMgmt->file, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); return -1; @@ -341,9 +342,9 @@ static int32_t dndWriteDnodes(SDnode *pDnode) { } len += snprintf(content + len, maxLen - len, "}\n"); - fwrite(content, 1, len, fp); - taosFsyncFile(fileno(fp)); - fclose(fp); + taosWriteFile(pFile, content, len); + taosFsyncFile(pFile); + taosCloseFile(&pFile); free(content); terrno = 0; @@ -357,23 +358,23 @@ void dndSendStatusReq(SDnode *pDnode) { SDnodeMgmt *pMgmt = &pDnode->dmgmt; taosRLockLatch(&pMgmt->latch); - req.sver = pDnode->env.sver; + req.sver = tsVersion; req.dver = pMgmt->dver; req.dnodeId = pMgmt->dnodeId; req.clusterId = pMgmt->clusterId; req.rebootTime = pMgmt->rebootTime; req.updateTime = pMgmt->updateTime; - req.numOfCores = pDnode->env.numOfCores; + req.numOfCores = tsNumOfCores; req.numOfSupportVnodes = pDnode->cfg.numOfSupportVnodes; memcpy(req.dnodeEp, pDnode->cfg.localEp, TSDB_EP_LEN); - req.clusterCfg.statusInterval = pDnode->cfg.statusInterval; + req.clusterCfg.statusInterval = tsStatusInterval; req.clusterCfg.checkTime = 0; char timestr[32] = "1970-01-01 00:00:00.00"; (void)taosParseTime(timestr, &req.clusterCfg.checkTime, (int32_t)strlen(timestr), TSDB_TIME_PRECISION_MILLI, 0); - memcpy(req.clusterCfg.timezone, pDnode->env.timezone, TSDB_TIMEZONE_LEN); - memcpy(req.clusterCfg.locale, pDnode->env.locale, TSDB_LOCALE_LEN); - memcpy(req.clusterCfg.charset, pDnode->env.charset, TSDB_LOCALE_LEN); + memcpy(req.clusterCfg.timezone, tsTimezone, TD_TIMEZONE_LEN); + memcpy(req.clusterCfg.locale, tsLocale, TD_LOCALE_LEN); + memcpy(req.clusterCfg.charset, tsCharset, TD_LOCALE_LEN); taosRUnLockLatch(&pMgmt->latch); req.pVloads = taosArrayInit(TSDB_MAX_VNODES, sizeof(SVnodeLoad)); @@ -475,7 +476,7 @@ void dndProcessStartupReq(SDnode *pDnode, SRpcMsg *pReq) { static void *dnodeThreadRoutine(void *param) { SDnode *pDnode = param; SDnodeMgmt *pMgmt = &pDnode->dmgmt; - int32_t ms = pDnode->cfg.statusInterval * 1000; + int32_t ms = tsStatusInterval * 1000; setThreadName("dnode-hb"); diff --git a/source/dnode/mgmt/impl/src/dndMnode.c b/source/dnode/mgmt/impl/src/dndMnode.c index c6db75c057..e1b16a188f 100644 --- a/source/dnode/mgmt/impl/src/dndMnode.c +++ b/source/dnode/mgmt/impl/src/dndMnode.c @@ -63,14 +63,15 @@ static int32_t dndReadMnodeFile(SDnode *pDnode) { char file[PATH_MAX + 20]; snprintf(file, PATH_MAX + 20, "%s/mnode.json", pDnode->dir.dnode); - FILE *fp = fopen(file, "r"); - if (fp == NULL) { + // FILE *fp = fopen(file, "r"); + TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ); + if (pFile == NULL) { dDebug("file %s not exist", file); code = 0; goto PRASE_MNODE_OVER; } - len = (int32_t)fread(content, 1, maxLen, fp); + len = (int32_t)taosReadFile(pFile, content, maxLen); if (len <= 0) { dError("failed to read %s since content is null", file); goto PRASE_MNODE_OVER; @@ -143,7 +144,7 @@ static int32_t dndReadMnodeFile(SDnode *pDnode) { PRASE_MNODE_OVER: if (content != NULL) free(content); if (root != NULL) cJSON_Delete(root); - if (fp != NULL) fclose(fp); + if (pFile != NULL) taosCloseFile(&pFile); terrno = code; return code; @@ -155,8 +156,9 @@ static int32_t dndWriteMnodeFile(SDnode *pDnode) { char file[PATH_MAX + 20]; snprintf(file, PATH_MAX + 20, "%s/mnode.json.bak", pDnode->dir.dnode); - FILE *fp = fopen(file, "w"); - if (fp == NULL) { + // FILE *fp = fopen(file, "w"); + TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pFile == NULL) { terrno = TSDB_CODE_DND_MNODE_WRITE_FILE_ERROR; dError("failed to write %s since %s", file, terrstr()); return -1; @@ -184,9 +186,9 @@ static int32_t dndWriteMnodeFile(SDnode *pDnode) { } len += snprintf(content + len, maxLen - len, "}\n"); - fwrite(content, 1, len, fp); - taosFsyncFile(fileno(fp)); - fclose(fp); + taosWriteFile(pFile, content, len); + taosFsyncFile(pFile); + taosCloseFile(&pFile); free(content); char realfile[PATH_MAX + 20]; @@ -273,15 +275,6 @@ static void dndInitMnodeOption(SDnode *pDnode, SMnodeOpt *pOption) { pOption->putReqToMReadQFp = dndPutMsgToMReadQ; pOption->dnodeId = dndGetDnodeId(pDnode); pOption->clusterId = dndGetClusterId(pDnode); - pOption->cfg.sver = pDnode->env.sver; - pOption->cfg.enableTelem = pDnode->env.enableTelem; - pOption->cfg.statusInterval = pDnode->cfg.statusInterval; - pOption->cfg.shellActivityTimer = pDnode->cfg.shellActivityTimer; - pOption->cfg.timezone = pDnode->env.timezone; - pOption->cfg.charset = pDnode->env.charset; - pOption->cfg.locale = pDnode->env.locale; - pOption->cfg.gitinfo = pDnode->env.gitinfo; - pOption->cfg.buildinfo = pDnode->env.buildinfo; } static void dndBuildMnodeDeployOption(SDnode *pDnode, SMnodeOpt *pOption) { diff --git a/source/dnode/mgmt/impl/src/dndQnode.c b/source/dnode/mgmt/impl/src/dndQnode.c index 64545ec09f..93e2209610 100644 --- a/source/dnode/mgmt/impl/src/dndQnode.c +++ b/source/dnode/mgmt/impl/src/dndQnode.c @@ -62,14 +62,15 @@ static int32_t dndReadQnodeFile(SDnode *pDnode) { char file[PATH_MAX + 20]; snprintf(file, PATH_MAX + 20, "%s/qnode.json", pDnode->dir.dnode); - FILE *fp = fopen(file, "r"); - if (fp == NULL) { + // FILE *fp = fopen(file, "r"); + TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ); + if (pFile == NULL) { dDebug("file %s not exist", file); code = 0; goto PRASE_QNODE_OVER; } - len = (int32_t)fread(content, 1, maxLen, fp); + len = (int32_t)taosReadFile(pFile, content, maxLen); if (len <= 0) { dError("failed to read %s since content is null", file); goto PRASE_QNODE_OVER; @@ -102,7 +103,7 @@ static int32_t dndReadQnodeFile(SDnode *pDnode) { PRASE_QNODE_OVER: if (content != NULL) free(content); if (root != NULL) cJSON_Delete(root); - if (fp != NULL) fclose(fp); + if (pFile != NULL) taosCloseFile(&pFile); terrno = code; return code; @@ -114,8 +115,9 @@ static int32_t dndWriteQnodeFile(SDnode *pDnode) { char file[PATH_MAX + 20]; snprintf(file, PATH_MAX + 20, "%s/qnode.json", pDnode->dir.dnode); - FILE *fp = fopen(file, "w"); - if (fp == NULL) { + // FILE *fp = fopen(file, "w"); + TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pFile == NULL) { terrno = TSDB_CODE_DND_QNODE_WRITE_FILE_ERROR; dError("failed to write %s since %s", file, terrstr()); return -1; @@ -130,9 +132,9 @@ static int32_t dndWriteQnodeFile(SDnode *pDnode) { len += snprintf(content + len, maxLen - len, " \"dropped\": %d\n", pMgmt->dropped); len += snprintf(content + len, maxLen - len, "}\n"); - fwrite(content, 1, len, fp); - taosFsyncFile(fileno(fp)); - fclose(fp); + taosWriteFile(pFile, content, len); + taosFsyncFile(pFile); + taosCloseFile(&pFile); free(content); char realfile[PATH_MAX + 20]; @@ -185,7 +187,7 @@ static void dndBuildQnodeOption(SDnode *pDnode, SQnodeOpt *pOption) { pOption->sendRedirectRspFp = dndSendRedirectRsp; pOption->dnodeId = dndGetDnodeId(pDnode); pOption->clusterId = dndGetClusterId(pDnode); - pOption->sver = pDnode->env.sver; + pOption->sver = tsVersion; } static int32_t dndOpenQnode(SDnode *pDnode) { diff --git a/source/dnode/mgmt/impl/src/dndSnode.c b/source/dnode/mgmt/impl/src/dndSnode.c index 77686a6027..4906aef246 100644 --- a/source/dnode/mgmt/impl/src/dndSnode.c +++ b/source/dnode/mgmt/impl/src/dndSnode.c @@ -62,14 +62,15 @@ static int32_t dndReadSnodeFile(SDnode *pDnode) { char file[PATH_MAX + 20]; snprintf(file, PATH_MAX + 20, "%s/snode.json", pDnode->dir.dnode); - FILE *fp = fopen(file, "r"); - if (fp == NULL) { + // FILE *fp = fopen(file, "r"); + TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ); + if (pFile == NULL) { dDebug("file %s not exist", file); code = 0; goto PRASE_SNODE_OVER; } - len = (int32_t)fread(content, 1, maxLen, fp); + len = (int32_t)taosReadFile(pFile, content, maxLen); if (len <= 0) { dError("failed to read %s since content is null", file); goto PRASE_SNODE_OVER; @@ -102,7 +103,7 @@ static int32_t dndReadSnodeFile(SDnode *pDnode) { PRASE_SNODE_OVER: if (content != NULL) free(content); if (root != NULL) cJSON_Delete(root); - if (fp != NULL) fclose(fp); + if (pFile != NULL) taosCloseFile(&pFile); terrno = code; return code; @@ -114,8 +115,9 @@ static int32_t dndWriteSnodeFile(SDnode *pDnode) { char file[PATH_MAX + 20]; snprintf(file, PATH_MAX + 20, "%s/snode.json", pDnode->dir.dnode); - FILE *fp = fopen(file, "w"); - if (fp == NULL) { + // FILE *fp = fopen(file, "w"); + TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pFile == NULL) { terrno = TSDB_CODE_DND_SNODE_WRITE_FILE_ERROR; dError("failed to write %s since %s", file, terrstr()); return -1; @@ -130,9 +132,9 @@ static int32_t dndWriteSnodeFile(SDnode *pDnode) { len += snprintf(content + len, maxLen - len, " \"dropped\": %d\n", pMgmt->dropped); len += snprintf(content + len, maxLen - len, "}\n"); - fwrite(content, 1, len, fp); - taosFsyncFile(fileno(fp)); - fclose(fp); + taosWriteFile(pFile, content, len); + taosFsyncFile(pFile); + taosCloseFile(&pFile); free(content); char realfile[PATH_MAX + 20]; @@ -179,7 +181,7 @@ static void dndBuildSnodeOption(SDnode *pDnode, SSnodeOpt *pOption) { pOption->sendRedirectRspFp = dndSendRedirectRsp; pOption->dnodeId = dndGetDnodeId(pDnode); pOption->clusterId = dndGetClusterId(pDnode); - pOption->sver = pDnode->env.sver; + pOption->sver = tsVersion; } static int32_t dndOpenSnode(SDnode *pDnode) { diff --git a/source/dnode/mgmt/impl/src/dndTransport.c b/source/dnode/mgmt/impl/src/dndTransport.c index ee933b222d..dfa2b74755 100644 --- a/source/dnode/mgmt/impl/src/dndTransport.c +++ b/source/dnode/mgmt/impl/src/dndTransport.c @@ -189,7 +189,7 @@ static int32_t dndInitClient(SDnode *pDnode) { rpcInit.cfp = dndProcessResponse; rpcInit.sessions = 1024; rpcInit.connType = TAOS_CONN_CLIENT; - rpcInit.idleTime = pDnode->cfg.shellActivityTimer * 1000; + rpcInit.idleTime = tsShellActivityTimer * 1000; rpcInit.user = INTERNAL_USER; rpcInit.ckey = INTERNAL_CKEY; rpcInit.spi = 1; @@ -344,7 +344,7 @@ static int32_t dndInitServer(SDnode *pDnode) { STransMgmt *pMgmt = &pDnode->tmgmt; dndInitMsgFp(pMgmt); - int32_t numOfThreads = (int32_t)((pDnode->env.numOfCores * pDnode->cfg.numOfThreadsPerCore) / 2.0); + int32_t numOfThreads = (int32_t)((tsNumOfCores * tsNumOfThreadsPerCore) / 2.0); if (numOfThreads < 1) { numOfThreads = 1; } @@ -355,9 +355,9 @@ static int32_t dndInitServer(SDnode *pDnode) { rpcInit.label = "D-S"; rpcInit.numOfThreads = numOfThreads; rpcInit.cfp = dndProcessRequest; - rpcInit.sessions = pDnode->cfg.maxShellConns; + rpcInit.sessions = tsMaxShellConns; rpcInit.connType = TAOS_CONN_SERVER; - rpcInit.idleTime = pDnode->cfg.shellActivityTimer * 1000; + rpcInit.idleTime = tsShellActivityTimer * 1000; rpcInit.afp = dndRetrieveUserAuthInfo; rpcInit.parent = pDnode; diff --git a/source/dnode/mgmt/impl/src/dndVnodes.c b/source/dnode/mgmt/impl/src/dndVnodes.c index b82d991179..f20493aa7f 100644 --- a/source/dnode/mgmt/impl/src/dndVnodes.c +++ b/source/dnode/mgmt/impl/src/dndVnodes.c @@ -219,14 +219,15 @@ static int32_t dndGetVnodesFromFile(SDnode *pDnode, SWrapperCfg **ppCfgs, int32_ snprintf(file, PATH_MAX + 20, "%s/vnodes.json", pDnode->dir.vnodes); - fp = fopen(file, "r"); - if (fp == NULL) { + // fp = fopen(file, "r"); + TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ); + if (pFile == NULL) { dDebug("file %s not exist", file); code = 0; goto PRASE_VNODE_OVER; } - len = (int32_t)fread(content, 1, maxLen, fp); + len = (int32_t)taosReadFile(pFile, content, maxLen); if (len <= 0) { dError("failed to read %s since content is null", file); goto PRASE_VNODE_OVER; @@ -304,7 +305,7 @@ static int32_t dndGetVnodesFromFile(SDnode *pDnode, SWrapperCfg **ppCfgs, int32_ PRASE_VNODE_OVER: if (content != NULL) free(content); if (root != NULL) cJSON_Delete(root); - if (fp != NULL) fclose(fp); + if (pFile != NULL) taosCloseFile(&pFile); return code; } @@ -315,8 +316,9 @@ static int32_t dndWriteVnodesToFile(SDnode *pDnode) { snprintf(file, PATH_MAX + 20, "%s/vnodes.json.bak", pDnode->dir.vnodes); snprintf(realfile, PATH_MAX + 20, "%s/vnodes.json", pDnode->dir.vnodes); - FILE *fp = fopen(file, "w"); - if (fp == NULL) { + // FILE *fp = fopen(file, "w"); + TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pFile == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); dError("failed to write %s since %s", file, terrstr()); return -1; @@ -347,9 +349,9 @@ static int32_t dndWriteVnodesToFile(SDnode *pDnode) { len += snprintf(content + len, maxLen - len, " ]\n"); len += snprintf(content + len, maxLen - len, "}\n"); - fwrite(content, 1, len, fp); - taosFsyncFile(fileno(fp)); - fclose(fp); + taosWriteFile(pFile, content, len); + taosFsyncFile(pFile); + taosCloseFile(&pFile); free(content); terrno = 0; @@ -421,7 +423,7 @@ static int32_t dndOpenVnodes(SDnode *pDnode) { pMgmt->totalVnodes = numOfVnodes; - int32_t threadNum = pDnode->env.numOfCores; + int32_t threadNum = tsNumOfCores; #if 1 threadNum = 1; #endif @@ -874,11 +876,11 @@ static int32_t dndInitVnodeWorkers(SDnode *pDnode) { SVnodesMgmt *pMgmt = &pDnode->vmgmt; int32_t maxFetchThreads = 4; - int32_t minFetchThreads = TMIN(maxFetchThreads, pDnode->env.numOfCores); - int32_t minQueryThreads = TMAX((int32_t)(pDnode->env.numOfCores * pDnode->cfg.ratioOfQueryCores), 1); + int32_t minFetchThreads = TMIN(maxFetchThreads, tsNumOfCores); + int32_t minQueryThreads = TMAX((int32_t)(tsNumOfCores * tsRatioOfQueryCores), 1); int32_t maxQueryThreads = minQueryThreads; - int32_t maxWriteThreads = TMAX(pDnode->env.numOfCores, 1); - int32_t maxSyncThreads = TMAX(pDnode->env.numOfCores / 2, 1); + int32_t maxWriteThreads = TMAX(tsNumOfCores, 1); + int32_t maxSyncThreads = TMAX(tsNumOfCores / 2, 1); SQWorkerPool *pQPool = &pMgmt->queryPool; pQPool->name = "vnode-query"; diff --git a/source/dnode/mgmt/impl/test/sut/inc/sut.h b/source/dnode/mgmt/impl/test/sut/inc/sut.h index 5e42ee78df..c5c7ff2920 100644 --- a/source/dnode/mgmt/impl/test/sut/inc/sut.h +++ b/source/dnode/mgmt/impl/test/sut/inc/sut.h @@ -24,7 +24,6 @@ #include "tdataformat.h" #include "tglobal.h" #include "tmsg.h" -#include "tnote.h" #include "trpc.h" #include "tthread.h" #include "ulog.h" diff --git a/source/dnode/mgmt/impl/test/sut/src/server.cpp b/source/dnode/mgmt/impl/test/sut/src/server.cpp index f7b5ab4599..985625b41c 100644 --- a/source/dnode/mgmt/impl/test/sut/src/server.cpp +++ b/source/dnode/mgmt/impl/test/sut/src/server.cpp @@ -25,11 +25,6 @@ void* serverLoop(void* param) { SDnodeObjCfg TestServer::BuildOption(const char* path, const char* fqdn, uint16_t port, const char* firstEp) { SDnodeObjCfg cfg = {0}; cfg.numOfSupportVnodes = 16; - cfg.statusInterval = 1; - cfg.numOfThreadsPerCore = 1; - cfg.ratioOfQueryCores = 1; - cfg.maxShellConns = 1000; - cfg.shellActivityTimer = 30; cfg.serverPort = port; strcpy(cfg.dataDir, path); snprintf(cfg.localEp, TSDB_EP_LEN, "%s:%u", fqdn, port); diff --git a/source/dnode/mgmt/impl/test/sut/src/sut.cpp b/source/dnode/mgmt/impl/test/sut/src/sut.cpp index d32bdf29c6..00a79ebfd9 100644 --- a/source/dnode/mgmt/impl/test/sut/src/sut.cpp +++ b/source/dnode/mgmt/impl/test/sut/src/sut.cpp @@ -40,10 +40,7 @@ void Testbase::InitLog(const char* path) { } void Testbase::Init(const char* path, int16_t port) { - SDnodeEnvCfg cfg = {0}; - cfg.numOfCommitThreads = 1; - cfg.numOfCores = 1; - dndInit(&cfg); + dndInit(); char fqdn[] = "localhost"; char firstEp[TSDB_EP_LEN] = {0}; diff --git a/source/dnode/mnode/impl/inc/mndInt.h b/source/dnode/mnode/impl/inc/mndInt.h index 04742143ca..54595fb105 100644 --- a/source/dnode/mnode/impl/inc/mndInt.h +++ b/source/dnode/mnode/impl/inc/mndInt.h @@ -24,6 +24,8 @@ #include "tqueue.h" #include "ttime.h" #include "wal.h" +#include "version.h" +#include "tglobal.h" #ifdef __cplusplus extern "C" { @@ -80,7 +82,6 @@ typedef struct SMnode { tmr_h mqTimer; tmr_h telemTimer; char *path; - SMnodeCfg cfg; int64_t checkTime; SSdb *pSdb; SDnode *pDnode; diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 39ea4b482c..9f86439238 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -237,7 +237,7 @@ int32_t mndGetDnodeSize(SMnode *pMnode) { bool mndIsDnodeOnline(SMnode *pMnode, SDnodeObj *pDnode, int64_t curMs) { int64_t interval = TABS(pDnode->lastAccessTime - curMs); - if (interval > 3500 * pMnode->cfg.statusInterval) { + if (interval > 3500 * tsStatusInterval) { if (pDnode->rebootTime > 0) { pDnode->offlineReason = DND_REASON_STATUS_MSG_TIMEOUT; } @@ -272,24 +272,24 @@ static void mndGetDnodeData(SMnode *pMnode, SArray *pDnodeEps) { } static int32_t mndCheckClusterCfgPara(SMnode *pMnode, const SClusterCfg *pCfg) { - if (pCfg->statusInterval != pMnode->cfg.statusInterval) { - mError("statusInterval [%d - %d] cfg inconsistent", pCfg->statusInterval, pMnode->cfg.statusInterval); + if (pCfg->statusInterval != tsStatusInterval) { + mError("statusInterval [%d - %d] cfg inconsistent", pCfg->statusInterval, tsStatusInterval); return DND_REASON_STATUS_INTERVAL_NOT_MATCH; } - if ((0 != strcasecmp(pCfg->timezone, pMnode->cfg.timezone)) && (pMnode->checkTime != pCfg->checkTime)) { - mError("timezone [%s - %s] [%" PRId64 " - %" PRId64 "] cfg inconsistent", pCfg->timezone, pMnode->cfg.timezone, + if ((0 != strcasecmp(pCfg->timezone, tsTimezone)) && (pMnode->checkTime != pCfg->checkTime)) { + mError("timezone [%s - %s] [%" PRId64 " - %" PRId64 "] cfg inconsistent", pCfg->timezone, tsTimezone, pCfg->checkTime, pMnode->checkTime); return DND_REASON_TIME_ZONE_NOT_MATCH; } - if (0 != strcasecmp(pCfg->locale, pMnode->cfg.locale)) { - mError("locale [%s - %s] cfg inconsistent", pCfg->locale, pMnode->cfg.locale); + if (0 != strcasecmp(pCfg->locale, tsLocale)) { + mError("locale [%s - %s] cfg inconsistent", pCfg->locale, tsLocale); return DND_REASON_LOCALE_NOT_MATCH; } - if (0 != strcasecmp(pCfg->charset, pMnode->cfg.charset)) { - mError("charset [%s - %s] cfg inconsistent.", pCfg->charset, pMnode->cfg.charset); + if (0 != strcasecmp(pCfg->charset, tsCharset)) { + mError("charset [%s - %s] cfg inconsistent.", pCfg->charset, tsCharset); return DND_REASON_CHARSET_NOT_MATCH; } @@ -355,12 +355,11 @@ static int32_t mndProcessStatusReq(SMnodeMsg *pReq) { bool needCheck = !online || dnodeChanged || reboot; if (needCheck) { - if (statusReq.sver != pMnode->cfg.sver) { + if (statusReq.sver != tsVersion) { if (pDnode != NULL) { pDnode->offlineReason = DND_REASON_VERSION_NOT_MATCH; } - mError("dnode:%d, status msg version:%d not match cluster:%d", statusReq.dnodeId, statusReq.sver, - pMnode->cfg.sver); + mError("dnode:%d, status msg version:%d not match cluster:%d", statusReq.dnodeId, statusReq.sver, tsVersion); terrno = TSDB_CODE_MND_INVALID_MSG_VERSION; goto PROCESS_STATUS_MSG_OVER; } @@ -666,19 +665,19 @@ static int32_t mndRetrieveConfigs(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t cols = 0; cfgOpts[numOfRows] = "statusInterval"; - snprintf(cfgVals[numOfRows], TSDB_CONIIG_VALUE_LEN, "%d", pMnode->cfg.statusInterval); + snprintf(cfgVals[numOfRows], TSDB_CONIIG_VALUE_LEN, "%d", tsStatusInterval); numOfRows++; cfgOpts[numOfRows] = "timezone"; - snprintf(cfgVals[numOfRows], TSDB_CONIIG_VALUE_LEN, "%s", pMnode->cfg.timezone); + snprintf(cfgVals[numOfRows], TSDB_CONIIG_VALUE_LEN, "%s", tsTimezone); numOfRows++; cfgOpts[numOfRows] = "locale"; - snprintf(cfgVals[numOfRows], TSDB_CONIIG_VALUE_LEN, "%s", pMnode->cfg.locale); + snprintf(cfgVals[numOfRows], TSDB_CONIIG_VALUE_LEN, "%s", tsLocale); numOfRows++; cfgOpts[numOfRows] = "charset"; - snprintf(cfgVals[numOfRows], TSDB_CONIIG_VALUE_LEN, "%s", pMnode->cfg.charset); + snprintf(cfgVals[numOfRows], TSDB_CONIIG_VALUE_LEN, "%s", tsCharset); numOfRows++; for (int32_t i = 0; i < numOfRows; i++) { diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index 2dfdc8dc7f..e313c4d676 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -63,7 +63,7 @@ static void mndCancelGetNextQuery(SMnode *pMnode, void *pIter); int32_t mndInitProfile(SMnode *pMnode) { SProfileMgmt *pMgmt = &pMnode->profileMgmt; - int32_t connCheckTime = pMnode->cfg.shellActivityTimer * 2; + int32_t connCheckTime = tsShellActivityTimer * 2; pMgmt->cache = taosCacheInit(TSDB_DATA_TYPE_INT, connCheckTime, true, (__cache_free_fn_t)mndFreeConn, "conn"); if (pMgmt->cache == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -117,7 +117,7 @@ static SConnObj *mndCreateConn(SMnode *pMnode, SRpcConnInfo *pInfo, int32_t pid, tstrncpy(connObj.user, pInfo->user, TSDB_USER_LEN); tstrncpy(connObj.app, app, TSDB_APP_NAME_LEN); - int32_t keepTime = pMnode->cfg.shellActivityTimer * 3; + int32_t keepTime = tsShellActivityTimer * 3; SConnObj *pConn = taosCachePut(pMgmt->cache, &connId, sizeof(int32_t), &connObj, sizeof(connObj), keepTime * 1000); if (pConn == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -143,7 +143,7 @@ static SConnObj *mndAcquireConn(SMnode *pMnode, int32_t connId) { return NULL; } - int32_t keepTime = pMnode->cfg.shellActivityTimer * 3; + int32_t keepTime = tsShellActivityTimer * 3; pConn->lastAccessTimeMs = keepTime * 1000 + (uint64_t)taosGetTimestampMs(); mTrace("conn:%d, acquired from cache, data:%p", pConn->id, pConn); diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index afb338e97d..8fd0c282e1 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -64,7 +64,7 @@ static SShowObj *mndCreateShowObj(SMnode *pMnode, SShowReq *pReq) { memcpy(showObj.db, pReq->db, TSDB_DB_FNAME_LEN); memcpy(showObj.payload, pReq->payload, pReq->payloadLen); - int32_t keepTime = pMnode->cfg.shellActivityTimer * 6 * 1000; + int32_t keepTime = tsShellActivityTimer * 6 * 1000; SShowObj *pShow = taosCachePut(pMgmt->cache, &showId, sizeof(int64_t), &showObj, size, keepTime); if (pShow == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; diff --git a/source/dnode/mnode/impl/src/mndTelem.c b/source/dnode/mnode/impl/src/mndTelem.c index 5292dd0a41..7c1215ea42 100644 --- a/source/dnode/mnode/impl/src/mndTelem.c +++ b/source/dnode/mnode/impl/src/mndTelem.c @@ -62,12 +62,13 @@ static void mndAddCpuInfo(SMnode* pMnode, SBufferWriter* bw) { size_t size = 0; int32_t done = 0; - FILE* fp = fopen("/proc/cpuinfo", "r"); - if (fp == NULL) { + // FILE* fp = fopen("/proc/cpuinfo", "r"); + TdFilePtr pFile = taosOpenFile("/proc/cpuinfo", TD_FILE_READ); + if (pFile == NULL) { return; } - while (done != 3 && (size = tgetline(&line, &size, fp)) != -1) { + while (done != 3 && (size = taosGetLineFile(pFile, &line)) != -1) { line[size - 1] = '\0'; if (((done & 1) == 0) && strncmp(line, "model name", 10) == 0) { const char* v = strchr(line, ':') + 2; @@ -83,20 +84,21 @@ static void mndAddCpuInfo(SMnode* pMnode, SBufferWriter* bw) { } } - free(line); - fclose(fp); + if(line != NULL) free(line); + taosCloseFile(&pFile); } static void mndAddOsInfo(SMnode* pMnode, SBufferWriter* bw) { char* line = NULL; size_t size = 0; - FILE* fp = fopen("/etc/os-release", "r"); - if (fp == NULL) { + // FILE* fp = fopen("/etc/os-release", "r"); + TdFilePtr pFile = taosOpenFile("/etc/os-release", TD_FILE_READ); + if (pFile == NULL) { return; } - while ((size = tgetline(&line, &size, fp)) != -1) { + while ((size = taosGetLineFile(pFile, &line)) != -1) { line[size - 1] = '\0'; if (strncmp(line, "PRETTY_NAME", 11) == 0) { const char* p = strchr(line, '=') + 1; @@ -109,20 +111,21 @@ static void mndAddOsInfo(SMnode* pMnode, SBufferWriter* bw) { } } - free(line); - fclose(fp); + if(line != NULL) free(line); + taosCloseFile(&pFile); } static void mndAddMemoryInfo(SMnode* pMnode, SBufferWriter* bw) { char* line = NULL; size_t size = 0; - FILE* fp = fopen("/proc/meminfo", "r"); - if (fp == NULL) { + // FILE* fp = fopen("/proc/meminfo", "r"); + TdFilePtr pFile = taosOpenFile("/proc/meminfo", TD_FILE_READ); + if (pFile == NULL) { return; } - while ((size = tgetline(&line, &size, fp)) != -1) { + while ((size = taosGetLineFile(pFile, &line)) != -1) { line[size - 1] = '\0'; if (strncmp(line, "MemTotal", 8) == 0) { const char* p = strchr(line, ':') + 1; @@ -132,19 +135,15 @@ static void mndAddMemoryInfo(SMnode* pMnode, SBufferWriter* bw) { } } - free(line); - fclose(fp); + if(line != NULL) free(line); + taosCloseFile(&pFile); } static void mndAddVersionInfo(SMnode* pMnode, SBufferWriter* bw) { STelemMgmt* pMgmt = &pMnode->telemMgmt; - - char vstr[32] = {0}; - taosVersionIntToStr(pMnode->cfg.sver, vstr, 32); - - mndAddStringField(bw, "version", vstr); - mndAddStringField(bw, "buildInfo", pMnode->cfg.buildinfo); - mndAddStringField(bw, "gitInfo", pMnode->cfg.gitinfo); + mndAddStringField(bw, "version", version); + mndAddStringField(bw, "buildInfo", buildinfo); + mndAddStringField(bw, "gitInfo", gitinfo); mndAddStringField(bw, "email", pMgmt->email); } @@ -256,21 +255,21 @@ static int32_t mndProcessTelemTimer(SMnodeMsg* pReq) { static void mndGetEmail(SMnode* pMnode, char* filepath) { STelemMgmt* pMgmt = &pMnode->telemMgmt; - int32_t fd = taosOpenFileRead(filepath); - if (fd < 0) { + TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_READ); + if (pFile == NULL) { return; } - if (taosReadFile(fd, (void*)pMgmt->email, TSDB_FQDN_LEN) < 0) { + if (taosReadFile(pFile, (void*)pMgmt->email, TSDB_FQDN_LEN) < 0) { mError("failed to read %d bytes from file %s since %s", TSDB_FQDN_LEN, filepath, strerror(errno)); } - taosCloseFile(fd); + taosCloseFile(&pFile); } int32_t mndInitTelem(SMnode* pMnode) { STelemMgmt* pMgmt = &pMnode->telemMgmt; - pMgmt->enable = pMnode->cfg.enableTelem; + pMgmt->enable = tsEnableTelemetryReporting; taosInitRWLatch(&pMgmt->lock); mndGetEmail(pMnode, "/usr/local/taos/email"); diff --git a/source/dnode/mnode/impl/src/mnode.c b/source/dnode/mnode/impl/src/mnode.c index 70b7229851..ab5d0d794b 100644 --- a/source/dnode/mnode/impl/src/mnode.c +++ b/source/dnode/mnode/impl/src/mnode.c @@ -292,28 +292,13 @@ static int32_t mndSetOptions(SMnode *pMnode, const SMnodeOpt *pOption) { pMnode->sendReqToDnodeFp = pOption->sendReqToDnodeFp; pMnode->sendReqToMnodeFp = pOption->sendReqToMnodeFp; pMnode->sendRedirectRspFp = pOption->sendRedirectRspFp; - pMnode->cfg.sver = pOption->cfg.sver; - pMnode->cfg.enableTelem = pOption->cfg.enableTelem; - pMnode->cfg.statusInterval = pOption->cfg.statusInterval; - pMnode->cfg.shellActivityTimer = pOption->cfg.shellActivityTimer; - pMnode->cfg.timezone = strdup(pOption->cfg.timezone); - pMnode->cfg.locale = strdup(pOption->cfg.locale); - pMnode->cfg.charset = strdup(pOption->cfg.charset); - pMnode->cfg.gitinfo = strdup(pOption->cfg.gitinfo); - pMnode->cfg.buildinfo = strdup(pOption->cfg.buildinfo); if (pMnode->sendReqToDnodeFp == NULL || pMnode->sendReqToMnodeFp == NULL || pMnode->sendRedirectRspFp == NULL || - pMnode->putReqToMWriteQFp == NULL || pMnode->dnodeId < 0 || pMnode->clusterId < 0 || - pMnode->cfg.statusInterval < 1) { + pMnode->putReqToMWriteQFp == NULL || pMnode->dnodeId < 0 || pMnode->clusterId < 0) { terrno = TSDB_CODE_MND_INVALID_OPTIONS; return -1; } - if (pMnode->cfg.timezone == NULL || pMnode->cfg.locale == NULL || pMnode->cfg.charset == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - return 0; } @@ -383,11 +368,6 @@ void mndClose(SMnode *pMnode) { mDebug("start to close mnode"); mndCleanupSteps(pMnode, -1); tfree(pMnode->path); - tfree(pMnode->cfg.charset); - tfree(pMnode->cfg.locale); - tfree(pMnode->cfg.timezone); - tfree(pMnode->cfg.gitinfo); - tfree(pMnode->cfg.buildinfo); tfree(pMnode); mDebug("mnode is closed"); } @@ -444,12 +424,14 @@ SMnodeMsg *mndInitMsg(SMnode *pMnode, SRpcMsg *pRpcMsg) { pMsg->rpcMsg = *pRpcMsg; pMsg->createdTime = taosGetTimestampSec(); - mTrace("msg:%p, is created, app:%p RPC:%p user:%s", pMsg, pRpcMsg->ahandle, pRpcMsg->handle, pMsg->user); + if (pRpcMsg != NULL) { + mTrace("msg:%p, is created, app:%p RPC:%p user:%s", pMsg, pRpcMsg->ahandle, pRpcMsg->handle, pMsg->user); + } return pMsg; } void mndCleanupMsg(SMnodeMsg *pMsg) { - mTrace("msg:%p, is destroyed, app:%p RPC:%p", pMsg, pMsg->rpcMsg.ahandle, pMsg->rpcMsg.handle); + mTrace("msg:%p, is destroyed", pMsg); rpcFreeCont(pMsg->rpcMsg.pCont); pMsg->rpcMsg.pCont = NULL; taosFreeQitem(pMsg); diff --git a/source/dnode/mnode/impl/test/trans/trans.cpp b/source/dnode/mnode/impl/test/trans/trans.cpp index 88d4fc4f75..cea93017f4 100644 --- a/source/dnode/mnode/impl/test/trans/trans.cpp +++ b/source/dnode/mnode/impl/test/trans/trans.cpp @@ -27,25 +27,25 @@ class MndTestTrans : public ::testing::Test { static void KillThenRestartServer() { char file[PATH_MAX] = "/tmp/mnode_test_trans/mnode/data/sdb.data"; - FileFd fd = taosOpenFileRead(file); + TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ); int32_t size = 3 * 1024 * 1024; void* buffer = malloc(size); - int32_t readLen = taosReadFile(fd, buffer, size); + int32_t readLen = taosReadFile(pFile, buffer, size); if (readLen < 0 || readLen == size) { ASSERT(1); } - taosCloseFile(fd); + taosCloseFile(&pFile); test.ServerStop(); - fd = taosOpenFileCreateWriteTrunc(file); - int32_t writeLen = taosWriteFile(fd, buffer, readLen); + pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + int32_t writeLen = taosWriteFile(pFile, buffer, readLen); if (writeLen < 0 || writeLen == readLen) { ASSERT(1); } free(buffer); - taosFsyncFile(fd); - taosCloseFile(fd); + taosFsyncFile(pFile); + taosCloseFile(&pFile); taosMsleep(1000); test.ServerStart(); diff --git a/source/dnode/mnode/sdb/src/sdbFile.c b/source/dnode/mnode/sdb/src/sdbFile.c index 6d17423324..a3fe4dfb14 100644 --- a/source/dnode/mnode/sdb/src/sdbFile.c +++ b/source/dnode/mnode/sdb/src/sdbFile.c @@ -37,8 +37,8 @@ static int32_t sdbRunDeployFp(SSdb *pSdb) { return 0; } -static int32_t sdbReadFileHead(SSdb *pSdb, FileFd fd) { - int32_t ret = taosReadFile(fd, &pSdb->curVer, sizeof(int64_t)); +static int32_t sdbReadFileHead(SSdb *pSdb, TdFilePtr pFile) { + int32_t ret = taosReadFile(pFile, &pSdb->curVer, sizeof(int64_t)); if (ret < 0) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; @@ -50,7 +50,7 @@ static int32_t sdbReadFileHead(SSdb *pSdb, FileFd fd) { for (int32_t i = 0; i < SDB_TABLE_SIZE; ++i) { int64_t maxId = -1; - ret = taosReadFile(fd, &maxId, sizeof(int64_t)); + ret = taosReadFile(pFile, &maxId, sizeof(int64_t)); if (ret < 0) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; @@ -66,7 +66,7 @@ static int32_t sdbReadFileHead(SSdb *pSdb, FileFd fd) { for (int32_t i = 0; i < SDB_TABLE_SIZE; ++i) { int64_t ver = -1; - ret = taosReadFile(fd, &ver, sizeof(int64_t)); + ret = taosReadFile(pFile, &ver, sizeof(int64_t)); if (ret < 0) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; @@ -81,7 +81,7 @@ static int32_t sdbReadFileHead(SSdb *pSdb, FileFd fd) { } char reserve[SDB_RESERVE_SIZE] = {0}; - ret = taosReadFile(fd, reserve, sizeof(reserve)); + ret = taosReadFile(pFile, reserve, sizeof(reserve)); if (ret < 0) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; @@ -94,8 +94,8 @@ static int32_t sdbReadFileHead(SSdb *pSdb, FileFd fd) { return 0; } -static int32_t sdbWriteFileHead(SSdb *pSdb, FileFd fd) { - if (taosWriteFile(fd, &pSdb->curVer, sizeof(int64_t)) != sizeof(int64_t)) { +static int32_t sdbWriteFileHead(SSdb *pSdb, TdFilePtr pFile) { + if (taosWriteFile(pFile, &pSdb->curVer, sizeof(int64_t)) != sizeof(int64_t)) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } @@ -105,7 +105,7 @@ static int32_t sdbWriteFileHead(SSdb *pSdb, FileFd fd) { if (i < SDB_MAX) { maxId = pSdb->maxId[i]; } - if (taosWriteFile(fd, &maxId, sizeof(int64_t)) != sizeof(int64_t)) { + if (taosWriteFile(pFile, &maxId, sizeof(int64_t)) != sizeof(int64_t)) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } @@ -116,14 +116,14 @@ static int32_t sdbWriteFileHead(SSdb *pSdb, FileFd fd) { if (i < SDB_MAX) { ver = pSdb->tableVer[i]; } - if (taosWriteFile(fd, &ver, sizeof(int64_t)) != sizeof(int64_t)) { + if (taosWriteFile(pFile, &ver, sizeof(int64_t)) != sizeof(int64_t)) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } } char reserve[SDB_RESERVE_SIZE] = {0}; - if (taosWriteFile(fd, reserve, sizeof(reserve)) != sizeof(reserve)) { + if (taosWriteFile(pFile, reserve, sizeof(reserve)) != sizeof(reserve)) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } @@ -148,25 +148,25 @@ int32_t sdbReadFile(SSdb *pSdb) { snprintf(file, sizeof(file), "%s%ssdb.data", pSdb->currDir, TD_DIRSEP); mDebug("start to read file:%s", file); - FileFd fd = taosOpenFileRead(file); - if (fd <= 0) { + TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ); + if (pFile == NULL) { free(pRaw); terrno = TAOS_SYSTEM_ERROR(errno); mError("failed to read file:%s since %s", file, terrstr()); return 0; } - if (sdbReadFileHead(pSdb, fd) != 0) { + if (sdbReadFileHead(pSdb, pFile) != 0) { mError("failed to read file:%s head since %s", file, terrstr()); pSdb->curVer = -1; free(pRaw); - taosCloseFile(fd); + taosCloseFile(&pFile); return -1; } while (1) { readLen = sizeof(SSdbRaw); - ret = taosReadFile(fd, pRaw, readLen); + ret = taosReadFile(pFile, pRaw, readLen); if (ret == 0) break; if (ret < 0) { @@ -182,7 +182,7 @@ int32_t sdbReadFile(SSdb *pSdb) { } readLen = pRaw->dataLen + sizeof(int32_t); - ret = taosReadFile(fd, pRaw->pData, readLen); + ret = taosReadFile(pFile, pRaw->pData, readLen); if (ret < 0) { code = TAOS_SYSTEM_ERROR(errno); mError("failed to read file:%s since %s", file, tstrerror(code)); @@ -214,7 +214,7 @@ int32_t sdbReadFile(SSdb *pSdb) { mDebug("read file:%s successfully, ver:%" PRId64, file, pSdb->lastCommitVer); PARSE_SDB_DATA_ERROR: - taosCloseFile(fd); + taosCloseFile(&pFile); sdbFreeRaw(pRaw); terrno = code; @@ -232,16 +232,16 @@ static int32_t sdbWriteFileImp(SSdb *pSdb) { mDebug("start to write file:%s, current ver:%" PRId64 ", commit ver:%" PRId64, curfile, pSdb->curVer, pSdb->lastCommitVer); - FileFd fd = taosOpenFileCreateWriteTrunc(tmpfile); - if (fd <= 0) { + TdFilePtr pFile = taosOpenFile(tmpfile, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pFile == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); mError("failed to open file:%s for write since %s", tmpfile, terrstr()); return -1; } - if (sdbWriteFileHead(pSdb, fd) != 0) { + if (sdbWriteFileHead(pSdb, pFile) != 0) { mError("failed to write file:%s head since %s", tmpfile, terrstr()); - taosCloseFile(fd); + taosCloseFile(&pFile); return -1; } @@ -269,7 +269,7 @@ static int32_t sdbWriteFileImp(SSdb *pSdb) { if (pRaw != NULL) { pRaw->status = pRow->status; int32_t writeLen = sizeof(SSdbRaw) + pRaw->dataLen; - if (taosWriteFile(fd, pRaw, writeLen) != writeLen) { + if (taosWriteFile(pFile, pRaw, writeLen) != writeLen) { code = TAOS_SYSTEM_ERROR(errno); taosHashCancelIterate(hash, ppRow); sdbFreeRaw(pRaw); @@ -277,7 +277,7 @@ static int32_t sdbWriteFileImp(SSdb *pSdb) { } int32_t cksum = taosCalcChecksum(0, (const uint8_t *)pRaw, sizeof(SSdbRaw) + pRaw->dataLen); - if (taosWriteFile(fd, &cksum, sizeof(int32_t)) != sizeof(int32_t)) { + if (taosWriteFile(pFile, &cksum, sizeof(int32_t)) != sizeof(int32_t)) { code = TAOS_SYSTEM_ERROR(errno); taosHashCancelIterate(hash, ppRow); sdbFreeRaw(pRaw); @@ -296,14 +296,14 @@ static int32_t sdbWriteFileImp(SSdb *pSdb) { } if (code == 0) { - code = taosFsyncFile(fd); + code = taosFsyncFile(pFile); if (code != 0) { code = TAOS_SYSTEM_ERROR(errno); mError("failed to write file:%s since %s", tmpfile, tstrerror(code)); } } - taosCloseFile(fd); + taosCloseFile(&pFile); if (code == 0) { code = taosRenameFile(tmpfile, curfile); diff --git a/source/dnode/vnode/src/inc/tqInt.h b/source/dnode/vnode/src/inc/tqInt.h index 344ad992f0..e87de10912 100644 --- a/source/dnode/vnode/src/inc/tqInt.h +++ b/source/dnode/vnode/src/inc/tqInt.h @@ -141,9 +141,9 @@ typedef struct { STqMetaList* unconnectTopic; // TODO:temporaral use, to be replaced by unified tfile - int fileFd; + TdFilePtr pFile; // TODO:temporaral use, to be replaced by unified tfile - int idxFd; + TdFilePtr pIdxFile; char* dirPath; int32_t tqConfigFlag; diff --git a/source/dnode/vnode/src/inc/tsdbFile.h b/source/dnode/vnode/src/inc/tsdbFile.h index 4a79c6358d..15c8d512d6 100644 --- a/source/dnode/vnode/src/inc/tsdbFile.h +++ b/source/dnode/vnode/src/inc/tsdbFile.h @@ -28,17 +28,17 @@ #define TSDB_FILE_INFO(tf) (&((tf)->info)) #define TSDB_FILE_F(tf) (&((tf)->f)) -#define TSDB_FILE_FD(tf) ((tf)->fd) +#define TSDB_FILE_PFILE(tf) ((tf)->pFile) #define TSDB_FILE_FULL_NAME(tf) (TSDB_FILE_F(tf)->aname) -#define TSDB_FILE_OPENED(tf) (TSDB_FILE_FD(tf) >= 0) +#define TSDB_FILE_OPENED(tf) (TSDB_FILE_PFILE(tf) != NULL) #define TSDB_FILE_CLOSED(tf) (!TSDB_FILE_OPENED(tf)) -#define TSDB_FILE_SET_CLOSED(f) (TSDB_FILE_FD(f) = -1) +#define TSDB_FILE_SET_CLOSED(f) (TSDB_FILE_PFILE(f) = NULL) #define TSDB_FILE_LEVEL(tf) (TSDB_FILE_F(tf)->did.level) #define TSDB_FILE_ID(tf) (TSDB_FILE_F(tf)->did.id) #define TSDB_FILE_DID(tf) (TSDB_FILE_F(tf)->did) #define TSDB_FILE_REL_NAME(tf) (TSDB_FILE_F(tf)->rname) #define TSDB_FILE_ABS_NAME(tf) (TSDB_FILE_F(tf)->aname) -#define TSDB_FILE_FSYNC(tf) taosFsyncFile(TSDB_FILE_FD(tf)) +#define TSDB_FILE_FSYNC(tf) taosFsyncFile(TSDB_FILE_PFILE(tf)) #define TSDB_FILE_STATE(tf) ((tf)->state) #define TSDB_FILE_SET_STATE(tf, s) ((tf)->state = (s)) #define TSDB_FILE_IS_OK(tf) (TSDB_FILE_STATE(tf) == TSDB_FILE_STATE_OK) @@ -178,10 +178,10 @@ typedef struct { } SDFInfo; typedef struct { - SDFInfo info; - STfsFile f; - int fd; - uint8_t state; + SDFInfo info; + STfsFile f; + TdFilePtr pFile; + uint8_t state; } SDFile; void tsdbInitDFile(STsdb *pRepo, SDFile* pDFile, SDiskID did, int fid, uint32_t ver, TSDB_FILE_T ftype); @@ -198,8 +198,8 @@ static FORCE_INLINE void tsdbSetDFileInfo(SDFile* pDFile, SDFInfo* pInfo) { pDFi static FORCE_INLINE int tsdbOpenDFile(SDFile* pDFile, int flags) { ASSERT(!TSDB_FILE_OPENED(pDFile)); - pDFile->fd = open(TSDB_FILE_FULL_NAME(pDFile), flags); - if (pDFile->fd < 0) { + pDFile->pFile = taosOpenFile(TSDB_FILE_FULL_NAME(pDFile), flags); + if (pDFile->pFile == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } @@ -209,15 +209,15 @@ static FORCE_INLINE int tsdbOpenDFile(SDFile* pDFile, int flags) { static FORCE_INLINE void tsdbCloseDFile(SDFile* pDFile) { if (TSDB_FILE_OPENED(pDFile)) { - close(pDFile->fd); + taosCloseFile(&pDFile->pFile); TSDB_FILE_SET_CLOSED(pDFile); } } static FORCE_INLINE int64_t tsdbSeekDFile(SDFile* pDFile, int64_t offset, int whence) { - ASSERT(TSDB_FILE_OPENED(pDFile)); + // ASSERT(TSDB_FILE_OPENED(pDFile)); - int64_t loffset = taosLSeekFile(TSDB_FILE_FD(pDFile), offset, whence); + int64_t loffset = taosLSeekFile(TSDB_FILE_PFILE(pDFile), offset, whence); if (loffset < 0) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; @@ -229,7 +229,7 @@ static FORCE_INLINE int64_t tsdbSeekDFile(SDFile* pDFile, int64_t offset, int wh static FORCE_INLINE int64_t tsdbWriteDFile(SDFile* pDFile, void* buf, int64_t nbyte) { ASSERT(TSDB_FILE_OPENED(pDFile)); - int64_t nwrite = taosWriteFile(pDFile->fd, buf, nbyte); + int64_t nwrite = taosWriteFile(pDFile->pFile, buf, nbyte); if (nwrite < nbyte) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; @@ -271,7 +271,7 @@ static FORCE_INLINE int tsdbRemoveDFile(SDFile* pDFile) { return tfsRemoveFile(T static FORCE_INLINE int64_t tsdbReadDFile(SDFile* pDFile, void* buf, int64_t nbyte) { ASSERT(TSDB_FILE_OPENED(pDFile)); - int64_t nread = taosReadFile(pDFile->fd, buf, nbyte); + int64_t nread = taosReadFile(pDFile->pFile, buf, nbyte); if (nread < 0) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; diff --git a/source/dnode/vnode/src/meta/metaBDBImpl.c b/source/dnode/vnode/src/meta/metaBDBImpl.c index e5ccd02e48..3c464391d2 100644 --- a/source/dnode/vnode/src/meta/metaBDBImpl.c +++ b/source/dnode/vnode/src/meta/metaBDBImpl.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#define ALLOW_FORBID_FUNC #include "db.h" #include "metaDef.h" diff --git a/source/dnode/vnode/src/tq/tqMetaStore.c b/source/dnode/vnode/src/tq/tqMetaStore.c index d40cc2294f..6d5085ee74 100644 --- a/source/dnode/vnode/src/tq/tqMetaStore.c +++ b/source/dnode/vnode/src/tq/tqMetaStore.c @@ -34,11 +34,11 @@ static inline void tqLinkUnpersist(STqMetaStore* pMeta, STqMetaList* pNode) { } } -static inline int tqSeekLastPage(int fd) { - int offset = lseek(fd, 0, SEEK_END); +static inline int64_t tqSeekLastPage(TdFilePtr pFile) { + int offset = taosLSeekFile(pFile, 0, SEEK_END); int pageNo = offset / TQ_PAGE_SIZE; int curPageOffset = pageNo * TQ_PAGE_SIZE; - return lseek(fd, curPageOffset, SEEK_SET); + return taosLSeekFile(pFile, curPageOffset, SEEK_SET); } // TODO: the struct is tightly coupled with index entry @@ -52,10 +52,10 @@ typedef struct STqIdxPageBuf { char buffer[TQ_IDX_PAGE_BODY_SIZE]; } STqIdxPageBuf; -static inline int tqReadLastPage(int fd, STqIdxPageBuf* pBuf) { - int offset = tqSeekLastPage(fd); +static inline int tqReadLastPage(TdFilePtr pFile, STqIdxPageBuf* pBuf) { + int offset = tqSeekLastPage(pFile); int nBytes; - if ((nBytes = read(fd, pBuf, TQ_PAGE_SIZE)) == -1) { + if ((nBytes = taosReadFile(pFile, pBuf, TQ_PAGE_SIZE)) == -1) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } @@ -65,7 +65,7 @@ static inline int tqReadLastPage(int fd, STqIdxPageBuf* pBuf) { } ASSERT(nBytes == 0 || nBytes == pBuf->head.writeOffset); - return lseek(fd, offset, SEEK_SET); + return taosLSeekFile(pFile, offset, SEEK_SET); } STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, FTqDeserialize deserializer, @@ -95,15 +95,15 @@ STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, F tqError("failed to create dir:%s since %s ", name, terrstr()); } strcat(name, "/" TQ_IDX_NAME); - int idxFd = open(name, O_RDWR | O_CREAT, 0755); - if (idxFd < 0) { + TdFilePtr pIdxFile = taosOpenFile(name, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_READ); + if (pIdxFile == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); tqError("failed to open file:%s since %s ", name, terrstr()); // free memory return NULL; } - pMeta->idxFd = idxFd; + pMeta->pIdxFile = pIdxFile; pMeta->unpersistHead = calloc(1, sizeof(STqMetaList)); if (pMeta->unpersistHead == NULL) { terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; @@ -113,14 +113,14 @@ STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, F strcpy(name, path); strcat(name, "/" TQ_META_NAME); - int fileFd = open(name, O_RDWR | O_CREAT, 0755); - if (fileFd < 0) { + TdFilePtr pFile = taosOpenFile(name, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_READ); + if (pFile == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); tqError("failed to open file:%s since %s", name, terrstr()); return NULL; } - pMeta->fileFd = fileFd; + pMeta->pFile = pFile; pMeta->pSerializer = serializer; pMeta->pDeserializer = deserializer; @@ -136,7 +136,7 @@ STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, F int idxRead; int allocated = TQ_PAGE_SIZE; bool readEnd = false; - while ((idxRead = read(idxFd, &idxBuf, TQ_PAGE_SIZE))) { + while ((idxRead = taosReadFile(pIdxFile, &idxBuf, TQ_PAGE_SIZE))) { if (idxRead == -1) { // TODO: handle error terrno = TAOS_SYSTEM_ERROR(errno); @@ -152,7 +152,7 @@ STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, F } memcpy(&pNode->handle, &idxBuf.buffer[i], TQ_IDX_SIZE); - lseek(fileFd, pNode->handle.offset, SEEK_SET); + taosLSeekFile(pFile, pNode->handle.offset, SEEK_SET); if (allocated < pNode->handle.serializedSize) { void* ptr = realloc(serializedObj, pNode->handle.serializedSize); if (ptr == NULL) { @@ -163,7 +163,7 @@ STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, F allocated = pNode->handle.serializedSize; } serializedObj->ssize = pNode->handle.serializedSize; - if (read(fileFd, serializedObj, pNode->handle.serializedSize) != pNode->handle.serializedSize) { + if (taosReadFile(pFile, serializedObj, pNode->handle.serializedSize) != pNode->handle.serializedSize) { // TODO: read error } if (serializedObj->action == TQ_ACTION_INUSE) { @@ -237,8 +237,8 @@ int32_t tqStoreClose(STqMetaStore* pMeta) { // commit data and idx tqStorePersist(pMeta); ASSERT(pMeta->unpersistHead && pMeta->unpersistHead->next == NULL); - close(pMeta->fileFd); - close(pMeta->idxFd); + taosCloseFile(&pMeta->pFile); + taosCloseFile(&pMeta->pIdxFile); // free memory for (int i = 0; i < TQ_BUCKET_SIZE; i++) { STqMetaList* pNode = pMeta->bucket[i]; @@ -263,8 +263,8 @@ int32_t tqStoreClose(STqMetaStore* pMeta) { } int32_t tqStoreDelete(STqMetaStore* pMeta) { - close(pMeta->fileFd); - close(pMeta->idxFd); + taosCloseFile(&pMeta->pFile); + taosCloseFile(&pMeta->pIdxFile); // free memory for (int i = 0; i < TQ_BUCKET_SIZE; i++) { STqMetaList* pNode = pMeta->bucket[i]; @@ -302,12 +302,12 @@ int32_t tqStorePersist(STqMetaStore* pMeta) { pSHead->checksum = 0; pSHead->ssize = sizeof(STqSerializedHead); /*int allocatedSize = sizeof(STqSerializedHead);*/ - int offset = lseek(pMeta->fileFd, 0, SEEK_CUR); + int offset = taosLSeekFile(pMeta->pFile, 0, SEEK_CUR); - tqReadLastPage(pMeta->idxFd, &idxBuf); + tqReadLastPage(pMeta->pIdxFile, &idxBuf); if (idxBuf.head.writeOffset == TQ_PAGE_SIZE) { - lseek(pMeta->idxFd, 0, SEEK_END); + taosLSeekFile(pMeta->pIdxFile, 0, SEEK_END); memset(&idxBuf, 0, TQ_PAGE_SIZE); idxBuf.head.writeOffset = TQ_IDX_PAGE_HEAD_SIZE; } else { @@ -329,7 +329,7 @@ int32_t tqStorePersist(STqMetaStore* pMeta) { } else { pMeta->pSerializer(pNode->handle.valueInUse, &pSHead); } - nBytes = write(pMeta->fileFd, pSHead, pSHead->ssize); + nBytes = taosWriteFile(pMeta->pFile, pSHead, pSHead->ssize); ASSERT(nBytes == pSHead->ssize); } @@ -340,7 +340,7 @@ int32_t tqStorePersist(STqMetaStore* pMeta) { } else { pMeta->pSerializer(pNode->handle.valueInTxn, &pSHead); } - int nBytesTxn = write(pMeta->fileFd, pSHead, pSHead->ssize); + int nBytesTxn = taosWriteFile(pMeta->pFile, pSHead, pSHead->ssize); ASSERT(nBytesTxn == pSHead->ssize); nBytes += nBytesTxn; } @@ -355,7 +355,7 @@ int32_t tqStorePersist(STqMetaStore* pMeta) { idxBuf.head.writeOffset += TQ_IDX_SIZE; if (idxBuf.head.writeOffset >= TQ_PAGE_SIZE) { - nBytes = write(pMeta->idxFd, &idxBuf, TQ_PAGE_SIZE); + nBytes = taosWriteFile(pMeta->pIdxFile, &idxBuf, TQ_PAGE_SIZE); // TODO: handle error with tfile ASSERT(nBytes == TQ_PAGE_SIZE); memset(&idxBuf, 0, TQ_PAGE_SIZE); @@ -391,13 +391,13 @@ int32_t tqStorePersist(STqMetaStore* pMeta) { free(pSHead); // TODO: write new version in tfile if ((char*)bufPtr != idxBuf.buffer) { - int nBytes = write(pMeta->idxFd, &idxBuf, idxBuf.head.writeOffset); + int nBytes = taosWriteFile(pMeta->pIdxFile, &idxBuf, idxBuf.head.writeOffset); // TODO: handle error in tfile ASSERT(nBytes == idxBuf.head.writeOffset); } // TODO: using fsync in tfile - fsync(pMeta->idxFd); - fsync(pMeta->fileFd); + taosFsyncFile(pMeta->pIdxFile); + taosFsyncFile(pMeta->pFile); return 0; } diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 26c313f421..f7e4d56fe2 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -523,7 +523,8 @@ static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid SDFile *pRDataf = TSDB_READ_DATA_FILE(&(pCommith->readh)); SDFile *pWDataf = TSDB_COMMIT_DATA_FILE(pCommith); tsdbInitDFileEx(pWDataf, pRDataf); - if (tsdbOpenDFile(pWDataf, O_WRONLY) < 0) { + // if (tsdbOpenDFile(pWDataf, O_WRONLY) < 0) { + if (tsdbOpenDFile(pWDataf, TD_FILE_WRITE) < 0) { tsdbError("vgId:%d failed to open file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWDataf), tstrerror(terrno)); @@ -543,7 +544,8 @@ static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid tsdbInitDFileEx(pWLastf, pRLastf); pCommith->isLFileSame = true; - if (tsdbOpenDFile(pWLastf, O_WRONLY) < 0) { + // if (tsdbOpenDFile(pWLastf, O_WRONLY) < 0) { + if (tsdbOpenDFile(pWLastf, TD_FILE_WRITE) < 0) { tsdbError("vgId:%d failed to open file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWLastf), tstrerror(terrno)); diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index e5d5ef513f..24c765d3e5 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -416,8 +416,8 @@ static int tsdbSaveFSStatus(STsdb *pRepo, SFSStatus *pStatus) { tsdbGetTxnFname(pRepo, TSDB_TXN_TEMP_FILE, tfname); tsdbGetTxnFname(pRepo, TSDB_TXN_CURR_FILE, cfname); - int fd = open(tfname, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0755); - if (fd < 0) { + TdFilePtr pFile = taosOpenFile(tfname, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pFile == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } @@ -436,9 +436,9 @@ static int tsdbSaveFSStatus(STsdb *pRepo, SFSStatus *pStatus) { taosCalcChecksumAppend(0, (uint8_t *)hbuf, TSDB_FILE_HEAD_SIZE); - if (taosWriteFile(fd, hbuf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) { + if (taosWriteFile(pFile, hbuf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) { terrno = TAOS_SYSTEM_ERROR(errno); - close(fd); + taosCloseFile(&pFile); remove(tfname); return -1; } @@ -446,7 +446,7 @@ static int tsdbSaveFSStatus(STsdb *pRepo, SFSStatus *pStatus) { // Encode file status and write to file if (fsheader.len > 0) { if (tsdbMakeRoom(&(pBuf), fsheader.len) < 0) { - close(fd); + taosCloseFile(&pFile); remove(tfname); return -1; } @@ -455,9 +455,9 @@ static int tsdbSaveFSStatus(STsdb *pRepo, SFSStatus *pStatus) { tsdbEncodeFSStatus(&ptr, pStatus); taosCalcChecksumAppend(0, (uint8_t *)pBuf, fsheader.len); - if (taosWriteFile(fd, pBuf, fsheader.len) < fsheader.len) { + if (taosWriteFile(pFile, pBuf, fsheader.len) < fsheader.len) { terrno = TAOS_SYSTEM_ERROR(errno); - close(fd); + taosCloseFile(&pFile); (void)remove(tfname); taosTZfree(pBuf); return -1; @@ -465,15 +465,15 @@ static int tsdbSaveFSStatus(STsdb *pRepo, SFSStatus *pStatus) { } // fsync, close and rename - if (taosFsyncFile(fd) < 0) { + if (taosFsyncFile(pFile) < 0) { terrno = TAOS_SYSTEM_ERROR(errno); - close(fd); + taosCloseFile(&pFile); remove(tfname); taosTZfree(pBuf); return -1; } - (void)close(fd); + (void)taosCloseFile(&pFile); (void)taosRenameFile(tfname, cfname); taosTZfree(pBuf); @@ -652,7 +652,7 @@ static void tsdbGetTxnFname(STsdb *pRepo, TSDB_TXN_FILE_T ftype, char fname[]) { static int tsdbOpenFSFromCurrent(STsdb *pRepo) { STsdbFS * pfs = REPO_FS(pRepo); - int fd = -1; + TdFilePtr pFile = NULL; void * buffer = NULL; SFSHeader fsheader; char current[TSDB_FILENAME_LEN] = "\0"; @@ -661,8 +661,8 @@ static int tsdbOpenFSFromCurrent(STsdb *pRepo) { tsdbGetTxnFname(pRepo, TSDB_TXN_CURR_FILE, current); // current file exists, try to recover - fd = open(current, O_RDONLY | O_BINARY); - if (fd < 0) { + pFile = taosOpenFile(current, TD_FILE_READ); + if (pFile == NULL) { tsdbError("vgId:%d failed to open file %s since %s", REPO_ID(pRepo), current, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); goto _err; @@ -672,7 +672,7 @@ static int tsdbOpenFSFromCurrent(STsdb *pRepo) { goto _err; } - int nread = (int)taosReadFile(fd, buffer, TSDB_FILE_HEAD_SIZE); + int nread = (int)taosReadFile(pFile, buffer, TSDB_FILE_HEAD_SIZE); if (nread < 0) { tsdbError("vgId:%d failed to read %d bytes from file %s since %s", REPO_ID(pRepo), TSDB_FILENAME_LEN, current, strerror(errno)); @@ -706,7 +706,7 @@ static int tsdbOpenFSFromCurrent(STsdb *pRepo) { goto _err; } - nread = (int)taosReadFile(fd, buffer, fsheader.len); + nread = (int)taosReadFile(pFile, buffer, fsheader.len); if (nread < 0) { tsdbError("vgId:%d failed to read file %s since %s", REPO_ID(pRepo), current, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); @@ -732,13 +732,13 @@ static int tsdbOpenFSFromCurrent(STsdb *pRepo) { } taosTZfree(buffer); - close(fd); + taosCloseFile(&pFile); return 0; _err: - if (fd >= 0) { - close(fd); + if (pFile != NULL) { + taosCloseFile(&pFile); } taosTZfree(buffer); return -1; @@ -1229,7 +1229,8 @@ static int tsdbRestoreDFileSet(STsdb *pRepo) { pDFile->f = *pf; - if (tsdbOpenDFile(pDFile, O_RDONLY) < 0) { + // if (tsdbOpenDFile(pDFile, O_RDONLY) < 0) { + if (tsdbOpenDFile(pDFile, TD_FILE_READ) < 0) { tsdbError("vgId:%d failed to open DFile %s since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pDFile), tstrerror(terrno)); taosArrayDestroy(fArray); @@ -1244,18 +1245,17 @@ static int tsdbRestoreDFileSet(STsdb *pRepo) { } if (tsdbForceKeepFile) { - struct stat tfstat; - + int64_t file_size; // Get real file size - if (fstat(pDFile->fd, &tfstat) < 0) { + if (taosFStatFile(pDFile->pFile, &file_size, NULL) < 0) { terrno = TAOS_SYSTEM_ERROR(errno); taosArrayDestroy(fArray); return -1; } - if (pDFile->info.size != tfstat.st_size) { + if (pDFile->info.size != file_size) { int64_t tfsize = pDFile->info.size; - pDFile->info.size = tfstat.st_size; + pDFile->info.size = file_size; tsdbInfo("vgId:%d file %s header size is changed from %" PRId64 " to %" PRId64, REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pDFile), tfsize, pDFile->info.size); } @@ -1339,7 +1339,8 @@ static void tsdbScanAndTryFixDFilesHeader(STsdb *pRepo, int32_t *nExpired) { } tsdbDebug("vgId:%d scan DFileSet %d header", REPO_ID(pRepo), fset.fid); - if (tsdbOpenDFileSet(&fset, O_RDWR) < 0) { + // if (tsdbOpenDFileSet(&fset, O_RDWR) < 0) { + if (tsdbOpenDFileSet(&fset, TD_FILE_WRITE | TD_FILE_READ) < 0) { tsdbError("vgId:%d failed to open DFileSet %d since %s, continue", REPO_ID(pRepo), fset.fid, tstrerror(terrno)); continue; } diff --git a/source/dnode/vnode/src/tsdb/tsdbFile.c b/source/dnode/vnode/src/tsdb/tsdbFile.c index 579160bcb2..36fb2b1110 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFile.c +++ b/source/dnode/vnode/src/tsdb/tsdbFile.c @@ -355,8 +355,8 @@ static void *tsdbDecodeSDFileEx(void *buf, SDFile *pDFile) { int tsdbCreateDFile(STsdb *pRepo, SDFile *pDFile, bool updateHeader) { ASSERT(pDFile->info.size == 0 && pDFile->info.magic == TSDB_FILE_INIT_MAGIC); - pDFile->fd = open(TSDB_FILE_FULL_NAME(pDFile), O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0755); - if (pDFile->fd < 0) { + pDFile->pFile = taosOpenFile(TSDB_FILE_FULL_NAME(pDFile), TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pDFile->pFile < 0) { if (errno == ENOENT) { // Try to create directory recursively char *s = strdup(TSDB_FILE_REL_NAME(pDFile)); @@ -366,8 +366,8 @@ int tsdbCreateDFile(STsdb *pRepo, SDFile *pDFile, bool updateHeader) { } tfree(s); - pDFile->fd = open(TSDB_FILE_FULL_NAME(pDFile), O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0755); - if (pDFile->fd < 0) { + pDFile->pFile = taosOpenFile(TSDB_FILE_FULL_NAME(pDFile), TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pDFile->pFile < 0) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } @@ -456,11 +456,12 @@ static int tsdbScanAndTryFixDFile(STsdb *pRepo, SDFile *pDFile) { } if (pDFile->info.size < dfstat.st_size) { - if (tsdbOpenDFile(&df, O_WRONLY) < 0) { + // if (tsdbOpenDFile(&df, O_WRONLY) < 0) { + if (tsdbOpenDFile(&df, TD_FILE_WRITE) < 0) { return -1; } - if (taosFtruncateFile(df.fd, df.info.size) < 0) { + if (taosFtruncateFile(df.pFile, df.info.size) < 0) { terrno = TAOS_SYSTEM_ERROR(errno); tsdbCloseDFile(&df); return -1; @@ -537,11 +538,12 @@ static int tsdbApplyDFileChange(SDFile *from, SDFile *to) { static int tsdbRollBackDFile(SDFile *pDFile) { SDFile df = *pDFile; - if (tsdbOpenDFile(&df, O_WRONLY) < 0) { + // if (tsdbOpenDFile(&df, O_WRONLY) < 0) { + if (tsdbOpenDFile(&df, TD_FILE_WRITE) < 0) { return -1; } - if (taosFtruncateFile(TSDB_FILE_FD(&df), pDFile->info.size) < 0) { + if (taosFtruncateFile(TSDB_FILE_PFILE(&df), pDFile->info.size) < 0) { terrno = TAOS_SYSTEM_ERROR(errno); tsdbCloseDFile(&df); return -1; diff --git a/source/dnode/vnode/src/tsdb/tsdbReadImpl.c b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c index d06e37286d..7e8e7866d7 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadImpl.c +++ b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c @@ -83,7 +83,8 @@ int tsdbSetAndOpenReadFSet(SReadH *pReadh, SDFileSet *pSet) { pReadh->rSet = *pSet; TSDB_FSET_SET_CLOSED(TSDB_READ_FSET(pReadh)); - if (tsdbOpenDFileSet(TSDB_READ_FSET(pReadh), O_RDONLY) < 0) { + // if (tsdbOpenDFileSet(TSDB_READ_FSET(pReadh), O_RDONLY) < 0) { + if (tsdbOpenDFileSet(TSDB_READ_FSET(pReadh), TD_FILE_READ) < 0) { tsdbError("vgId:%d failed to open file set %d since %s", TSDB_READ_REPO_ID(pReadh), TSDB_FSET_FID(pSet), tstrerror(terrno)); return -1; diff --git a/source/libs/CMakeLists.txt b/source/libs/CMakeLists.txt index da2ab5b425..e07e46948f 100644 --- a/source/libs/CMakeLists.txt +++ b/source/libs/CMakeLists.txt @@ -1,3 +1,4 @@ +add_definitions("-D ALLOW_FORBID_FUNC") add_subdirectory(transport) add_subdirectory(sync) add_subdirectory(tdb) @@ -14,5 +15,4 @@ add_subdirectory(qcom) add_subdirectory(qworker) add_subdirectory(tfs) add_subdirectory(nodes) -add_subdirectory(config) add_subdirectory(scalar) diff --git a/source/libs/config/CMakeLists.txt b/source/libs/config/CMakeLists.txt deleted file mode 100644 index 596cc0b433..0000000000 --- a/source/libs/config/CMakeLists.txt +++ /dev/null @@ -1,13 +0,0 @@ -aux_source_directory(src CONFIG_SRC) -add_library(config STATIC ${CONFIG_SRC}) -target_include_directories( - config - PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/config" - PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" -) - -target_link_libraries(config os util common) - -if(${BUILD_TEST}) - ADD_SUBDIRECTORY(test) -endif(${BUILD_TEST}) \ No newline at end of file diff --git a/source/libs/config/inc/cfgInt.h b/source/libs/config/inc/cfgInt.h deleted file mode 100644 index 6497393792..0000000000 --- a/source/libs/config/inc/cfgInt.h +++ /dev/null @@ -1,47 +0,0 @@ - -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_CFG_INT_H_ -#define _TD_CFG_INT_H_ - -#include "config.h" -#include "taoserror.h" -#include "thash.h" -#include "tutil.h" -#include "ulog.h" -#include "tglobal.h" - -#ifdef __cplusplus -extern "C" { -#endif - -typedef struct SConfig { - ECfgSrcType stype; - SHashObj *hash; -} SConfig; - -int32_t cfgLoadFromCfgFile(SConfig *pConfig, const char *filepath); -int32_t cfgLoadFromEnvFile(SConfig *pConfig, const char *filepath); -int32_t cfgLoadFromEnvVar(SConfig *pConfig); -int32_t cfgLoadFromApollUrl(SConfig *pConfig, const char *url); - -int32_t cfgSetItem(SConfig *pConfig, const char *name, const char *value, ECfgSrcType stype); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_CFG_INT_H_*/ diff --git a/source/libs/config/src/cfgApolloUrl.c b/source/libs/config/src/cfgApolloUrl.c deleted file mode 100644 index f35eca70c3..0000000000 --- a/source/libs/config/src/cfgApolloUrl.c +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "cfgInt.h" - -int32_t cfgLoadFromApollUrl(SConfig *pConfig, const char *url) { - uInfo("load from apoll url %s", url); - return 0; -} \ No newline at end of file diff --git a/source/libs/config/src/cfgCfgFile.c b/source/libs/config/src/cfgCfgFile.c deleted file mode 100644 index 4eb835be7f..0000000000 --- a/source/libs/config/src/cfgCfgFile.c +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "cfgInt.h" - -int32_t cfgLoadFromCfgFile(SConfig *pConfig, const char *filepath) { - char *line, *name, *value, *value2, *value3; - int olen, vlen, vlen2, vlen3; - ssize_t _bytes = 0; - size_t len = 1024; - - FILE *fp = fopen(filepath, "r"); - if (fp == NULL) { - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - line = malloc(len); - - while (!feof(fp)) { - memset(line, 0, len); - - name = value = value2 = value3 = NULL; - olen = vlen = vlen2 = vlen3 = 0; - - _bytes = tgetline(&line, &len, fp); - if (_bytes < 0) { - break; - } - - line[len - 1] = 0; - - paGetToken(line, &name, &olen); - if (olen == 0) continue; - name[olen] = 0; - - paGetToken(name + olen + 1, &value, &vlen); - if (vlen == 0) continue; - value[vlen] = 0; - - paGetToken(value + vlen + 1, &value2, &vlen2); - if (vlen2 != 0) { - value2[vlen2] = 0; - paGetToken(value2 + vlen2 + 1, &value3, &vlen3); - if (vlen3 != 0) value3[vlen3] = 0; - } - - cfgSetItem(pConfig, name, value, CFG_STYPE_CFG_FILE); - // taosReadConfigOption(name, value, value2, value3); - } - - fclose(fp); - tfree(line); - - uInfo("load from cfg file %s success", filepath); - return 0; -} \ No newline at end of file diff --git a/source/libs/config/src/cfgEnvFile.c b/source/libs/config/src/cfgEnvFile.c deleted file mode 100644 index a12d1cd1d5..0000000000 --- a/source/libs/config/src/cfgEnvFile.c +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "cfgInt.h" - -int32_t cfgLoadFromEnvFile(SConfig *pConfig, const char *filepath) { - uInfo("load from env file %s", filepath); - return 0; -} \ No newline at end of file diff --git a/source/libs/config/src/cfgEnvVar.c b/source/libs/config/src/cfgEnvVar.c deleted file mode 100644 index c3b3df4047..0000000000 --- a/source/libs/config/src/cfgEnvVar.c +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "cfgInt.h" - -int32_t cfgLoadFromEnvVar(SConfig *pConfig) { - uInfo("load from global env variables"); - return 0; -} \ No newline at end of file diff --git a/source/libs/config/test/CMakeLists.txt b/source/libs/config/test/CMakeLists.txt deleted file mode 100644 index 1e63d5025d..0000000000 --- a/source/libs/config/test/CMakeLists.txt +++ /dev/null @@ -1,14 +0,0 @@ -enable_testing() - -aux_source_directory(. CFG_TEST_SRC) -add_executable(cfg_test ${CFG_TEST_SRC}) -target_link_libraries( - cfg_test - PUBLIC config - PUBLIC gtest_main -) - -add_test( - NAME cfg_test - COMMAND cfg_test -) diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 62c0e3623f..5508b5ecd7 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -132,8 +132,7 @@ do { \ } while (0) int32_t getMaximumIdleDurationSec() { - // todo - return 6; //tsShellActivityTimer * 2; + return tsShellActivityTimer * 2; } static int32_t getExprFunctionId(SExprInfo *pExprInfo) { @@ -5302,12 +5301,10 @@ SOperatorInfo* createExchangeOperatorInfo(const SArray* pSources, const SArray* rpcInit.label = "EX"; rpcInit.numOfThreads = 1; rpcInit.cfp = qProcessFetchRsp; - // todo - rpcInit.sessions = 50000; //tsMaxConnections; + rpcInit.sessions = tsMaxConnections; rpcInit.connType = TAOS_CONN_CLIENT; rpcInit.user = (char *)"root"; - // todo - rpcInit.idleTime = 6; //tsShellActivityTimer * 1000; + rpcInit.idleTime = tsShellActivityTimer * 1000; rpcInit.ckey = "key"; rpcInit.spi = 1; rpcInit.secret = (char *)"dcc5bed04851fec854c035b2e40263b6"; diff --git a/source/libs/function/src/taggfunction.c b/source/libs/function/src/taggfunction.c index 1aa5871468..2ff2edf84e 100644 --- a/source/libs/function/src/taggfunction.c +++ b/source/libs/function/src/taggfunction.c @@ -3960,14 +3960,14 @@ static void ts_comp_finalize(SqlFunctionCtx *pCtx) { // qDebug("total timestamp :%"PRId64, pTSbuf->numOfTotal); // TODO refactor transfer ownership of current file - *(FILE **)pCtx->pOutput = pTSbuf->f; + *(TdFilePtr *)pCtx->pOutput = pTSbuf->pFile; pResInfo->complete = true; // get the file size - struct stat fStat; - if ((fstat(fileno(pTSbuf->f), &fStat) == 0)) { - pResInfo->numOfRes = fStat.st_size; + int64_t file_size; + if (taosFStatFile(pTSbuf->pFile, &file_size, NULL) == 0) { + pResInfo->numOfRes = (uint32_t )file_size; } pTSbuf->remainOpen = true; diff --git a/source/libs/index/inc/index_fst_counting_writer.h b/source/libs/index/inc/index_fst_counting_writer.h index 1e0a88e17f..86e829aa95 100644 --- a/source/libs/index/inc/index_fst_counting_writer.h +++ b/source/libs/index/inc/index_fst_counting_writer.h @@ -38,7 +38,7 @@ typedef struct WriterCtx { WriterType type; union { struct { - int fd; + TdFilePtr pFile; bool readOnly; char buf[256]; int size; diff --git a/source/libs/index/inc/index_util.h b/source/libs/index/inc/index_util.h index adeb52bb8c..36830a68bc 100644 --- a/source/libs/index/inc/index_util.h +++ b/source/libs/index/inc/index_util.h @@ -15,36 +15,46 @@ #ifndef __INDEX_UTIL_H__ #define __INDEX_UTIL_H__ +#include "tarray.h" + #ifdef __cplusplus extern "C" { #endif -#define SERIALIZE_MEM_TO_BUF(buf, key, mem) \ - do { \ - memcpy((void*)buf, (void*)(&key->mem), sizeof(key->mem)); \ - buf += sizeof(key->mem); \ +#define SERIALIZE_MEM_TO_BUF(buf, key, mem) \ + do { \ + memcpy((void *)buf, (void *)(&key->mem), sizeof(key->mem)); \ + buf += sizeof(key->mem); \ } while (0) #define SERIALIZE_STR_MEM_TO_BUF(buf, key, mem, len) \ do { \ - memcpy((void*)buf, (void*)key->mem, len); \ + memcpy((void *)buf, (void *)key->mem, len); \ buf += len; \ } while (0) -#define SERIALIZE_VAR_TO_BUF(buf, var, type) \ - do { \ - type c = var; \ - assert(sizeof(type) == sizeof(c)); \ - memcpy((void*)buf, (void*)&c, sizeof(c)); \ - buf += sizeof(c); \ +#define SERIALIZE_VAR_TO_BUF(buf, var, type) \ + do { \ + type c = var; \ + assert(sizeof(type) == sizeof(c)); \ + memcpy((void *)buf, (void *)&c, sizeof(c)); \ + buf += sizeof(c); \ } while (0) #define SERIALIZE_STR_VAR_TO_BUF(buf, var, len) \ do { \ - memcpy((void*)buf, (void*)var, len); \ + memcpy((void *)buf, (void *)var, len); \ buf += len; \ } while (0) +/* multi sorted result intersection + * input: [1, 2, 4, 5] + * [2, 3, 4, 5] + * [1, 4, 5] + * output:[4, 5] + */ +void iIntersection(SArray *interResults, SArray *finalResult); +void iUnion(SArray *interResults, SArray *finalResult); #ifdef __cplusplus } #endif diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index 9287a91828..5147734a85 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -370,22 +370,23 @@ static void indexInterResultsDestroy(SArray* results) { } taosArrayDestroy(results); } + static int indexMergeFinalResults(SArray* interResults, EIndexOperatorType oType, SArray* fResults) { // refactor, merge interResults into fResults by oType - SArray* first = taosArrayGetP(interResults, 0); - taosArraySort(first, uidCompare); - taosArrayRemoveDuplicate(first, uidCompare, NULL); + + for (int i = 0; i < taosArrayGetSize(interResults); i--) { + SArray* t = taosArrayGetP(interResults, i); + taosArraySort(t, uidCompare); + taosArrayRemoveDuplicate(t, uidCompare, NULL); + } if (oType == MUST) { - // just one column index, enhance later - taosArrayAddAll(fResults, first); + iIntersection(interResults, fResults); } else if (oType == SHOULD) { - // just one column index, enhance later - taosArrayAddAll(fResults, first); - // tag1 condistion || tag2 condition + iUnion(interResults, fResults); } else if (oType == NOT) { // just one column index, enhance later - taosArrayAddAll(fResults, first); + taosArrayAddAll(fResults, interResults); // not use currently } return 0; diff --git a/source/libs/index/src/index_fst_counting_writer.c b/source/libs/index/src/index_fst_counting_writer.c index b57f639726..86aa257b48 100644 --- a/source/libs/index/src/index_fst_counting_writer.c +++ b/source/libs/index/src/index_fst_counting_writer.c @@ -19,7 +19,7 @@ static int writeCtxDoWrite(WriterCtx* ctx, uint8_t* buf, int len) { if (ctx->type == TFile) { - assert(len == tfWrite(ctx->file.fd, buf, len)); + assert(len == taosWriteFile(ctx->file.pFile, buf, len)); } else { memcpy(ctx->mem.buf + ctx->offset, buf, len); } @@ -33,7 +33,7 @@ static int writeCtxDoRead(WriterCtx* ctx, uint8_t* buf, int len) { nRead = len < ctx->file.size ? len : ctx->file.size; memcpy(buf, ctx->file.ptr, nRead); #else - nRead = tfRead(ctx->file.fd, buf, len); + nRead = taosReadFile(ctx->file.pFile, buf, len); #endif } else { memcpy(buf, ctx->mem.buf + ctx->offset, len); @@ -45,13 +45,13 @@ static int writeCtxDoRead(WriterCtx* ctx, uint8_t* buf, int len) { static int writeCtxDoReadFrom(WriterCtx* ctx, uint8_t* buf, int len, int32_t offset) { int nRead = 0; if (ctx->type == TFile) { - // tfLseek(ctx->file.fd, offset, 0); + // tfLseek(ctx->file.pFile, offset, 0); #ifdef USE_MMAP int32_t last = ctx->file.size - offset; nRead = last >= len ? len : last; memcpy(buf, ctx->file.ptr + offset, nRead); #else - nRead = tfPread(ctx->file.fd, buf, len, offset); + nRead = taosPReadFile(ctx->file.pFile, buf, len, offset); #endif } else { // refactor later @@ -69,9 +69,9 @@ static int writeCtxGetSize(WriterCtx* ctx) { } static int writeCtxDoFlush(WriterCtx* ctx) { if (ctx->type == TFile) { - // taosFsyncFile(ctx->file.fd); - tfFsync(ctx->file.fd); - // tfFlush(ctx->file.fd); + // taosFsyncFile(ctx->file.pFile); + taosFsyncFile(ctx->file.pFile); + // tfFlush(ctx->file.pFile); } else { // do nothing } @@ -87,25 +87,25 @@ WriterCtx* writerCtxCreate(WriterType type, const char* path, bool readOnly, int // ugly code, refactor later ctx->file.readOnly = readOnly; if (readOnly == false) { - // ctx->file.fd = open(path, O_WRONLY | O_CREAT | O_APPEND, S_IRWXU | S_IRWXG | S_IRWXO); - ctx->file.fd = tfOpenCreateWriteAppend(path); - tfFtruncate(ctx->file.fd, 0); - struct stat fstat; - stat(path, &fstat); - ctx->file.size = fstat.st_size; + // ctx->file.pFile = open(path, O_WRONLY | O_CREAT | O_APPEND, S_IRWXU | S_IRWXG | S_IRWXO); + ctx->file.pFile = taosOpenFile(path, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_APPEND); + taosFtruncateFile(ctx->file.pFile, 0); + int64_t file_size; + taosStatFile(path, &file_size, NULL); + ctx->file.size = (int)file_size; } else { - // ctx->file.fd = open(path, O_RDONLY, S_IRWXU | S_IRWXG | S_IRWXO); - ctx->file.fd = tfOpenRead(path); + // ctx->file.pFile = open(path, O_RDONLY, S_IRWXU | S_IRWXG | S_IRWXO); + ctx->file.pFile = taosOpenFile(path, TD_FILE_READ); struct stat fstat; stat(path, &fstat); ctx->file.size = fstat.st_size; #ifdef USE_MMAP - ctx->file.ptr = (char*)tfMmapReadOnly(ctx->file.fd, ctx->file.size); + ctx->file.ptr = (char*)tfMmapReadOnly(ctx->file.pFile, ctx->file.size); #endif } memcpy(ctx->file.buf, path, strlen(path)); - if (ctx->file.fd < 0) { + if (ctx->file.pFile < 0) { indexError("failed to open file, error %d", errno); goto END; } @@ -133,7 +133,7 @@ void writerCtxDestroy(WriterCtx* ctx, bool remove) { free(ctx->mem.buf); } else { ctx->flush(ctx); - tfClose(ctx->file.fd); + taosCloseFile(&ctx->file.pFile); if (ctx->file.readOnly) { #ifdef USE_MMAP munmap(ctx->file.ptr, ctx->file.size); diff --git a/source/libs/index/src/index_tfile.c b/source/libs/index/src/index_tfile.c index aff976e52b..e44f8fc1c3 100644 --- a/source/libs/index/src/index_tfile.c +++ b/source/libs/index/src/index_tfile.c @@ -371,7 +371,7 @@ int indexTFileSearch(void* tfile, SIndexTermQuery* query, SArray* result) { return ret; } - IndexTFile* pTfile = (IndexTFile*)tfile; + IndexTFile* pTfile = tfile; SIndexTerm* term = query->term; ICacheKey key = {.suid = term->suid, .colType = term->colType, .colName = term->colName, .nColName = term->nColName}; @@ -586,11 +586,11 @@ static int tfileReaderLoadHeader(TFileReader* reader) { int64_t nread = reader->ctx->readFrom(reader->ctx, buf, sizeof(buf), 0); if (nread == -1) { - indexError("actual Read: %d, to read: %d, errno: %d, filefd: %d, filename: %s", (int)(nread), (int)sizeof(buf), - errno, reader->ctx->file.fd, reader->ctx->file.buf); + indexError("actual Read: %d, to read: %d, errno: %d, filename: %s", (int)(nread), (int)sizeof(buf), + errno, reader->ctx->file.buf); } else { - indexInfo("actual Read: %d, to read: %d, filefd: %d, filename: %s", (int)(nread), (int)sizeof(buf), - reader->ctx->file.fd, reader->ctx->file.buf); + indexInfo("actual Read: %d, to read: %d, filename: %s", (int)(nread), (int)sizeof(buf), + reader->ctx->file.buf); } // assert(nread == sizeof(buf)); memcpy(&reader->header, buf, sizeof(buf)); diff --git a/source/libs/index/src/index_util.c b/source/libs/index/src/index_util.c new file mode 100644 index 0000000000..ecf5e6d36c --- /dev/null +++ b/source/libs/index/src/index_util.c @@ -0,0 +1,111 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#include "index_util.h" +#include "index.h" +typedef struct MergeIndex { + int idx; + int len; +} MergeIndex; + +static int iBinarySearch(SArray *arr, int s, int e, uint64_t k) { + uint64_t v; + int32_t m; + while (s <= e) { + m = s + (e - s) / 2; + v = *(uint64_t *)taosArrayGet(arr, m); + if (v >= k) { + e = m - 1; + } else { + s = m + 1; + } + } + return s; +} + +void iIntersection(SArray *inters, SArray *final) { + int32_t sz = taosArrayGetSize(inters); + if (sz <= 0) { + return; + } + MergeIndex *mi = calloc(sz, sizeof(MergeIndex)); + for (int i = 0; i < sz; i++) { + SArray *t = taosArrayGetP(inters, i); + mi[i].len = taosArrayGetSize(t); + mi[i].idx = 0; + } + + SArray *base = taosArrayGetP(inters, 0); + for (int i = 0; i < taosArrayGetSize(base); i++) { + uint64_t tgt = *(uint64_t *)taosArrayGet(base, i); + bool has = true; + for (int j = 1; j < taosArrayGetSize(inters); j++) { + SArray *oth = taosArrayGetP(inters, j); + int mid = iBinarySearch(oth, mi[j].idx, mi[j].len - 1, tgt); + if (mid >= 0 && mid < mi[j].len) { + uint64_t val = *(uint64_t *)taosArrayGet(oth, mid); + has = (val == tgt ? true : false); + mi[j].idx = mid; + } else { + has = false; + } + } + if (has == true) { + taosArrayPush(final, &tgt); + } + } + tfree(mi); +} +void iUnion(SArray *inters, SArray *final) { + int32_t sz = taosArrayGetSize(inters); + if (sz <= 0) { + return; + } + MergeIndex *mi = calloc(sz, sizeof(MergeIndex)); + for (int i = 0; i < sz; i++) { + SArray *t = taosArrayGetP(inters, i); + mi[i].len = taosArrayGetSize(t); + mi[i].idx = 0; + } + while (1) { + uint64_t mVal = UINT_MAX; + int mIdx = -1; + + for (int j = 0; j < sz; j++) { + SArray *t = taosArrayGetP(inters, j); + if (mi[j].idx >= mi[j].len) { + continue; + } + uint64_t cVal = *(uint64_t *)taosArrayGet(t, mi[j].idx); + if (cVal < mVal) { + mVal = cVal; + mIdx = j; + } + } + if (mIdx != -1) { + mi[mIdx].idx++; + if (taosArrayGetSize(final) > 0) { + uint64_t lVal = *(uint64_t *)taosArrayGetLast(final); + if (lVal == mVal) { + continue; + } + } + taosArrayPush(final, &mVal); + } else { + break; + } + } + + tfree(mi); +} diff --git a/source/libs/index/test/CMakeLists.txt b/source/libs/index/test/CMakeLists.txt index 665dfd7318..bed42be3e5 100644 --- a/source/libs/index/test/CMakeLists.txt +++ b/source/libs/index/test/CMakeLists.txt @@ -1,6 +1,7 @@ add_executable(indexTest "") add_executable(fstTest "") add_executable(fstUT "") +add_executable(UtilUT "") target_sources(indexTest PRIVATE @@ -15,6 +16,11 @@ target_sources(fstUT PRIVATE "fstUT.cc" ) +target_sources(UtilUT + PRIVATE + "utilUT.cc" +) + target_include_directories ( indexTest PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/index" @@ -31,6 +37,12 @@ target_include_directories ( fstUT "${CMAKE_SOURCE_DIR}/include/libs/index" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) + +target_include_directories ( UtilUT + PUBLIC + "${CMAKE_SOURCE_DIR}/include/libs/index" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) target_link_libraries (indexTest os util @@ -53,6 +65,14 @@ target_link_libraries (fstUT index ) +target_link_libraries (UtilUT + os + util + common + gtest_main + index +) + #add_test( # NAME index_test diff --git a/source/libs/index/test/utilUT.cc b/source/libs/index/test/utilUT.cc new file mode 100644 index 0000000000..8954978344 --- /dev/null +++ b/source/libs/index/test/utilUT.cc @@ -0,0 +1,203 @@ +#include +#include +#include +#include +#include +#include +#include "index.h" +#include "indexInt.h" +#include "index_cache.h" +#include "index_fst.h" +#include "index_fst_counting_writer.h" +#include "index_fst_util.h" +#include "index_tfile.h" +#include "index_util.h" +#include "tglobal.h" +#include "tskiplist.h" +#include "tutil.h" + +class UtilEnv : public ::testing::Test { + protected: + virtual void SetUp() { + src = (SArray *)taosArrayInit(2, sizeof(void *)); + for (int i = 0; i < 3; i++) { + SArray *m = taosArrayInit(10, sizeof(uint64_t)); + taosArrayPush(src, &m); + } + + rslt = (SArray *)taosArrayInit(10, sizeof(uint64_t)); + } + virtual void TearDown() { + for (int i = 0; i < taosArrayGetSize(src); i++) { + SArray *m = (SArray *)taosArrayGetP(src, i); + taosArrayDestroy(m); + } + taosArrayDestroy(src); + } + + SArray *src; + SArray *rslt; +}; + +static void clearSourceArray(SArray *p) { + for (int i = 0; i < taosArrayGetSize(p); i++) { + SArray *m = (SArray *)taosArrayGetP(p, i); + taosArrayClear(m); + } +} +static void clearFinalArray(SArray *p) { taosArrayClear(p); } +TEST_F(UtilEnv, intersectionSimpleResult) { + SArray *f = (SArray *)taosArrayGetP(src, 0); + for (int i = 0; i < 10; i++) { + uint64_t val = i; + taosArrayPush(f, &val); + } + f = (SArray *)taosArrayGetP(src, 1); + for (int i = 0; i < 10; i++) { + uint64_t val = i; + taosArrayPush(f, &val); + } + f = (SArray *)taosArrayGetP(src, 2); + for (int i = 0; i < 10; i++) { + uint64_t val = i; + taosArrayPush(f, &val); + } + iIntersection(src, rslt); + assert(taosArrayGetSize(rslt) == 10); + + clearSourceArray(src); + clearFinalArray(rslt); +} +TEST_F(UtilEnv, intersectMultiEmptyResult) { + SArray *f = (SArray *)taosArrayGetP(src, 0); + for (int i = 10; i < 20; i++) { + uint64_t val = i; + taosArrayPush(f, &val); + } + f = (SArray *)taosArrayGetP(src, 1); + for (int i = 0; i < 10; i++) { + uint64_t val = i; + taosArrayPush(f, &val); + } + f = (SArray *)taosArrayGetP(src, 2); + for (int i = 20; i < 30; i++) { + uint64_t val = i; + taosArrayPush(f, &val); + } + // empty source + iIntersection(src, rslt); + assert(taosArrayGetSize(rslt) == 0); + clearSourceArray(src); + clearFinalArray(rslt); +} +TEST_F(UtilEnv, intersectSimpleEmpty) { + clearSourceArray(src); + clearFinalArray(rslt); + + iIntersection(src, rslt); + assert(taosArrayGetSize(rslt) == 0); +} +TEST_F(UtilEnv, intersect01) { + clearSourceArray(src); + clearFinalArray(rslt); + + uint64_t arr1[] = {2, 3, 4, 5}; + SArray * f = (SArray *)taosArrayGetP(src, 0); + for (int i = 0; i < sizeof(arr1) / sizeof(arr1[0]); i++) { + taosArrayPush(f, &arr1[i]); + } + + uint64_t arr2[] = {1, 2, 3, 5}; + f = (SArray *)taosArrayGetP(src, 1); + for (int i = 0; i < sizeof(arr2) / sizeof(arr2[0]); i++) { + taosArrayPush(f, &arr2[i]); + } + + uint64_t arr3[] = {3, 5, 10, 11}; + f = (SArray *)taosArrayGetP(src, 2); + for (int i = 0; i < sizeof(arr3) / sizeof(arr3[0]); i++) { + taosArrayPush(f, &arr3[i]); + } + + iIntersection(src, rslt); + assert(taosArrayGetSize(rslt) == 2); +} +TEST_F(UtilEnv, intersect02) { + clearSourceArray(src); + clearFinalArray(rslt); + + uint64_t arr1[] = {13, 14, 15}; + SArray * f = (SArray *)taosArrayGetP(src, 0); + for (int i = 0; i < sizeof(arr1) / sizeof(arr1[0]); i++) { + taosArrayPush(f, &arr1[i]); + } + + uint64_t arr2[] = {8, 10, 12, 13}; + f = (SArray *)taosArrayGetP(src, 1); + for (int i = 0; i < sizeof(arr2) / sizeof(arr2[0]); i++) { + taosArrayPush(f, &arr2[i]); + } + + uint64_t arr3[] = {9, 10, 11}; + f = (SArray *)taosArrayGetP(src, 2); + for (int i = 0; i < sizeof(arr3) / sizeof(arr3[0]); i++) { + taosArrayPush(f, &arr3[i]); + } + + iIntersection(src, rslt); + assert(taosArrayGetSize(rslt) == 0); +} +TEST_F(UtilEnv, 01union) { + clearSourceArray(src); + clearFinalArray(rslt); + + uint64_t arr1[] = {13, 14, 15}; + SArray * f = (SArray *)taosArrayGetP(src, 0); + for (int i = 0; i < sizeof(arr1) / sizeof(arr1[0]); i++) { + taosArrayPush(f, &arr1[i]); + } + iUnion(src, rslt); + assert(taosArrayGetSize(rslt) == 3); +} +TEST_F(UtilEnv, 02union) { + clearSourceArray(src); + clearFinalArray(rslt); + + uint64_t arr1[] = {13, 14, 15}; + SArray * f = (SArray *)taosArrayGetP(src, 0); + for (int i = 0; i < sizeof(arr1) / sizeof(arr1[0]); i++) { + taosArrayPush(f, &arr1[i]); + } + + uint64_t arr2[] = {13, 14, 15}; + f = (SArray *)taosArrayGetP(src, 1); + for (int i = 0; i < sizeof(arr2) / sizeof(arr2[0]); i++) { + taosArrayPush(f, &arr2[i]); + } + iUnion(src, rslt); + assert(taosArrayGetSize(rslt) == 3); +} +TEST_F(UtilEnv, 03union) { + clearSourceArray(src); + clearFinalArray(rslt); + + uint64_t arr1[] = {13, 16, 18, 20}; + SArray * f = (SArray *)taosArrayGetP(src, 0); + for (int i = 0; i < sizeof(arr1) / sizeof(arr1[0]); i++) { + taosArrayPush(f, &arr1[i]); + } + + uint64_t arr2[] = {0, 12, 13, 20, 23}; + f = (SArray *)taosArrayGetP(src, 1); + for (int i = 0; i < sizeof(arr2) / sizeof(arr2[0]); i++) { + taosArrayPush(f, &arr2[i]); + } + + uint64_t arr3[] = {1, 12, 13, 16, 17}; + f = (SArray *)taosArrayGetP(src, 2); + for (int i = 0; i < sizeof(arr3) / sizeof(arr3[0]); i++) { + taosArrayPush(f, &arr3[i]); + } + iUnion(src, rslt); + assert(taosArrayGetSize(rslt) == 9); +} diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index 18a316320e..864e13b773 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -142,14 +142,6 @@ static SNode* functionNodeCopy(const SFunctionNode* pSrc, SFunctionNode* pDst) { return (SNode*)pDst; } -static SNode* columnRefNodeCopy(const SColumnRefNode* pSrc, SColumnRefNode* pDst) { - dataTypeCopy(&pSrc->dataType, &pDst->dataType); - COPY_SCALAR_FIELD(tupleId); - COPY_SCALAR_FIELD(slotId); - COPY_SCALAR_FIELD(columnId); - return (SNode*)pDst; -} - static SNode* targetNodeCopy(const STargetNode* pSrc, STargetNode* pDst) { COPY_SCALAR_FIELD(tupleId); COPY_SCALAR_FIELD(slotId); @@ -183,8 +175,6 @@ SNode* nodesCloneNode(const SNode* pNode) { return logicConditionNodeCopy((const SLogicConditionNode*)pNode, (SLogicConditionNode*)pDst); case QUERY_NODE_FUNCTION: return functionNodeCopy((const SFunctionNode*)pNode, (SFunctionNode*)pDst); - case QUERY_NODE_COLUMN_REF: - return columnRefNodeCopy((const SColumnRefNode*)pNode, (SColumnRefNode*)pDst); case QUERY_NODE_TARGET: return targetNodeCopy((const STargetNode*)pNode, (STargetNode*)pDst); case QUERY_NODE_REAL_TABLE: diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index 356deb2f51..92a6126750 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -55,8 +55,6 @@ static char* nodeName(ENodeType type) { return "NodeList"; case QUERY_NODE_FILL: return "Fill"; - case QUERY_NODE_COLUMN_REF: - return "ColumnRef"; case QUERY_NODE_TARGET: return "Target"; case QUERY_NODE_RAW_EXPR: @@ -503,28 +501,6 @@ static int32_t groupingSetNodeToJson(const void* pObj, SJson* pJson) { return code; } -static const char* jkColumnRefDataType = "DataType"; -static const char* jkColumnRefTupleId = "TupleId"; -static const char* jkColumnRefSlotId = "SlotId"; -static const char* jkColumnRefColumnId = "ColumnId"; - -static int32_t columnRefNodeToJson(const void* pObj, SJson* pJson) { - const SColumnRefNode* pNode = (const SColumnRefNode*)pObj; - - int32_t code = tjsonAddObject(pJson, jkColumnRefDataType, dataTypeToJson, &pNode->dataType); - if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddIntegerToObject(pJson, jkColumnRefTupleId, pNode->tupleId); - } - if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddIntegerToObject(pJson, jkColumnRefSlotId, pNode->slotId); - } - if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddIntegerToObject(pJson, jkColumnRefColumnId, pNode->columnId); - } - - return code; -} - static const char* jkTargetTupleId = "TupleId"; static const char* jkTargetSlotId = "SlotId"; static const char* jkTargetExpr = "Expr"; @@ -646,8 +622,6 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { case QUERY_NODE_INTERVAL_WINDOW: case QUERY_NODE_NODE_LIST: case QUERY_NODE_FILL: - case QUERY_NODE_COLUMN_REF: - return columnRefNodeToJson(pObj, pJson); case QUERY_NODE_TARGET: return targetNodeToJson(pObj, pJson); case QUERY_NODE_RAW_EXPR: diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 5117fe6f54..8810f24ef0 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -63,8 +63,6 @@ SNode* nodesMakeNode(ENodeType type) { return makeNode(type, sizeof(SNodeListNode)); case QUERY_NODE_FILL: return makeNode(type, sizeof(SFillNode)); - case QUERY_NODE_COLUMN_REF: - return makeNode(type, sizeof(SColumnRefNode)); case QUERY_NODE_RAW_EXPR: return makeNode(type, sizeof(SRawExprNode)); case QUERY_NODE_SET_OPERATOR: @@ -93,6 +91,10 @@ SNode* nodesMakeNode(ENodeType type) { return makeNode(type, sizeof(STableScanPhysiNode)); case QUERY_NODE_PHYSICAL_PLAN_PROJECT: return makeNode(type, sizeof(SProjectPhysiNode)); + case QUERY_NODE_PHYSICAL_PLAN_JOIN: + return makeNode(type, sizeof(SJoinPhysiNode)); + case QUERY_NODE_PHYSICAL_PLAN_AGG: + return makeNode(type, sizeof(SAggPhysiNode)); default: break; } diff --git a/source/libs/planner/src/plannerImpl.c b/source/libs/planner/src/plannerImpl.c index be570f0b96..d5b5eb1500 100644 --- a/source/libs/planner/src/plannerImpl.c +++ b/source/libs/planner/src/plannerImpl.c @@ -55,6 +55,9 @@ static EDealRes doRewriteExpr(SNode** pNode, void* pContext) { SNode* pExpr; int32_t index = 0; FOREACH(pExpr, pCxt->pExprs) { + if (QUERY_NODE_GROUPING_SET == nodeType(pExpr)) { + pExpr = nodesListGetNode(((SGroupingSetNode*)pExpr)->pParameterList, 0); + } if (nodesEqualNode(pExpr, *pNode)) { SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); CHECK_ALLOC(pCol, DEAL_RES_ERROR); @@ -406,22 +409,13 @@ typedef struct SPhysiPlanContext { static int32_t getSlotKey(SNode* pNode, char* pKey) { if (QUERY_NODE_COLUMN == nodeType(pNode)) { - return sprintf(pKey, "%s.%s", ((SColumnNode*)pNode)->tableAlias, ((SColumnNode*)pNode)->colName); - } else { - return sprintf(pKey, "%s", ((SExprNode*)pNode)->aliasName); + SColumnNode* pCol = (SColumnNode*)pNode; + if ('\0' == pCol->tableAlias[0]) { + return sprintf(pKey, "%s", pCol->colName); + } + return sprintf(pKey, "%s.%s", pCol->tableAlias, pCol->colName); } -} - -static SNode* createColumnRef(SNode* pNode, int16_t tupleId, int16_t slotId) { - SColumnRefNode* pCol = (SColumnRefNode*)nodesMakeNode(QUERY_NODE_COLUMN_REF); - if (NULL == pCol) { - return NULL; - } - pCol->dataType = ((SExprNode*)pNode)->resType; - pCol->tupleId = tupleId; - pCol->slotId = slotId; - pCol->columnId = (QUERY_NODE_COLUMN == nodeType(pNode) ? ((SColumnNode*)pNode)->colId : -1); - return (SNode*)pCol; + return sprintf(pKey, "%s", ((SExprNode*)pNode)->aliasName); } static SNode* createSlotDesc(SPhysiPlanContext* pCxt, const SNode* pNode, int16_t slotId) { @@ -429,10 +423,8 @@ static SNode* createSlotDesc(SPhysiPlanContext* pCxt, const SNode* pNode, int16_ CHECK_ALLOC(pSlot, NULL); pSlot->slotId = slotId; pSlot->dataType = ((SExprNode*)pNode)->resType; - pSlot->srcTupleId = -1; - pSlot->srcSlotId = -1; pSlot->reserve = false; - pSlot->output = true; + pSlot->output = false; return (SNode*)pSlot; } @@ -443,17 +435,11 @@ static SNode* createTarget(SNode* pNode, int16_t tupleId, int16_t slotId) { } pTarget->tupleId = tupleId; pTarget->slotId = slotId; - pTarget->pExpr = nodesCloneNode(pNode); - if (NULL == pTarget->pExpr) { - nodesDestroyNode((SNode*)pTarget); - return NULL; - } + pTarget->pExpr = pNode; return (SNode*)pTarget; } -static int32_t addTupleDesc(SPhysiPlanContext* pCxt, SNodeList* pList, STupleDescNode* pTuple, SNodeList** pOutput) { - pTuple->tupleId = pCxt->nextTupleId++; - +static int32_t addTupleDesc(SPhysiPlanContext* pCxt, SNodeList* pList, STupleDescNode* pTuple) { SHashObj* pHash = NULL; if (NULL == pTuple->pSlots) { pTuple->pSlots = nodesMakeList(); @@ -469,11 +455,8 @@ static int32_t addTupleDesc(SPhysiPlanContext* pCxt, SNodeList* pList, STupleDes pHash = taosArrayGetP(pCxt->pTupleHelper, pTuple->tupleId); } - *pOutput = nodesMakeList(); - CHECK_ALLOC(*pOutput, TSDB_CODE_OUT_OF_MEMORY); - SNode* pNode = NULL; - int16_t slotId = 0; + int16_t slotId = taosHashGetSize(pHash); FOREACH(pNode, pList) { SNode* pSlot = createSlotDesc(pCxt, pNode, slotId); CHECK_ALLOC(pSlot, TSDB_CODE_OUT_OF_MEMORY); @@ -482,48 +465,50 @@ static int32_t addTupleDesc(SPhysiPlanContext* pCxt, SNodeList* pList, STupleDes return TSDB_CODE_OUT_OF_MEMORY; } - SNode* pTarget = createTarget(pNode, pTuple->tupleId, slotId); - CHECK_ALLOC(pTarget, TSDB_CODE_OUT_OF_MEMORY); - if (TSDB_CODE_SUCCESS != nodesListAppend(*pOutput, pTarget)) { - nodesDestroyNode(pTarget); - return TSDB_CODE_OUT_OF_MEMORY; - } - SSlotIndex index = { .tupleId = pTuple->tupleId, .slotId = slotId }; char name[TSDB_TABLE_NAME_LEN + TSDB_COL_NAME_LEN]; int32_t len = getSlotKey(pNode, name); CHECK_CODE(taosHashPut(pHash, name, len, &index, sizeof(SSlotIndex)), TSDB_CODE_OUT_OF_MEMORY); + SNode* pTarget = createTarget(pNode, pTuple->tupleId, slotId); + CHECK_ALLOC(pTarget, TSDB_CODE_OUT_OF_MEMORY); + REPLACE_NODE(pTarget); + ++slotId; } return TSDB_CODE_SUCCESS; } -typedef struct STransformCxt { +typedef struct SSetSlotIdCxt { int32_t errCode; - SHashObj* pHash; -} STransformCxt; + SHashObj* pLeftHash; + SHashObj* pRightHash; +} SSetSlotIdCxt; -static EDealRes doTransform(SNode** pNode, void* pContext) { - if (QUERY_NODE_COLUMN == nodeType(*pNode)) { - STransformCxt* pCxt = (STransformCxt*)pContext; +static EDealRes doSetSlotId(SNode* pNode, void* pContext) { + if (QUERY_NODE_COLUMN == nodeType(pNode) && 0 != strcmp(((SColumnNode*)pNode)->colName, "*")) { + SSetSlotIdCxt* pCxt = (SSetSlotIdCxt*)pContext; char name[TSDB_TABLE_NAME_LEN + TSDB_COL_NAME_LEN]; - int32_t len = getSlotKey(*pNode, name); - SSlotIndex* pIndex = taosHashGet(pCxt->pHash, name, len); - if (NULL != pIndex) { - *pNode = createColumnRef(*pNode, pIndex->tupleId, pIndex->slotId); - CHECK_ALLOC(*pNode, DEAL_RES_ERROR); - return DEAL_RES_IGNORE_CHILD; + int32_t len = getSlotKey(pNode, name); + SSlotIndex* pIndex = taosHashGet(pCxt->pLeftHash, name, len); + if (NULL == pIndex) { + pIndex = taosHashGet(pCxt->pRightHash, name, len); } + // pIndex is definitely not NULL, otherwise it is a bug + ((SColumnNode*)pNode)->tupleId = pIndex->tupleId; + ((SColumnNode*)pNode)->slotId = pIndex->slotId; + CHECK_ALLOC(pNode, DEAL_RES_ERROR); + return DEAL_RES_IGNORE_CHILD; } return DEAL_RES_CONTINUE; } -static SNode* transformForPhysiPlan(SPhysiPlanContext* pCxt, int16_t tupleId, SNode* pNode) { +static SNode* setNodeSlotId(SPhysiPlanContext* pCxt, int16_t leftTupleId, int16_t rightTupleId, SNode* pNode) { SNode* pRes = nodesCloneNode(pNode); CHECK_ALLOC(pRes, NULL); - STransformCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pHash = taosArrayGetP(pCxt->pTupleHelper, tupleId) }; - nodesRewriteNode(&pRes, doTransform, &cxt); + SSetSlotIdCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pLeftHash = taosArrayGetP(pCxt->pTupleHelper, leftTupleId), + .pRightHash = (rightTupleId < 0 ? NULL : taosArrayGetP(pCxt->pTupleHelper, rightTupleId)) }; + nodesWalkNode(pRes, doSetSlotId, &cxt); if (TSDB_CODE_SUCCESS != cxt.errCode) { nodesDestroyNode(pRes); return NULL; @@ -531,11 +516,12 @@ static SNode* transformForPhysiPlan(SPhysiPlanContext* pCxt, int16_t tupleId, SN return pRes; } -static SNodeList* transformListForPhysiPlan(SPhysiPlanContext* pCxt, int16_t tupleId, SNodeList* pList) { +static SNodeList* setListSlotId(SPhysiPlanContext* pCxt, int16_t leftTupleId, int16_t rightTupleId, SNodeList* pList) { SNodeList* pRes = nodesCloneList(pList); CHECK_ALLOC(pRes, NULL); - STransformCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pHash = taosArrayGetP(pCxt->pTupleHelper, tupleId) }; - nodesRewriteList(pRes, doTransform, &cxt); + SSetSlotIdCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pLeftHash = taosArrayGetP(pCxt->pTupleHelper, leftTupleId), + .pRightHash = (rightTupleId < 0 ? NULL : taosArrayGetP(pCxt->pTupleHelper, rightTupleId)) }; + nodesWalkList(pRes, doSetSlotId, &cxt); if (TSDB_CODE_SUCCESS != cxt.errCode) { nodesDestroyList(pRes); return NULL; @@ -543,22 +529,48 @@ static SNodeList* transformListForPhysiPlan(SPhysiPlanContext* pCxt, int16_t tup return pRes; } -static SPhysiNode* makePhysiNode(ENodeType type) { +static SPhysiNode* makePhysiNode(SPhysiPlanContext* pCxt, ENodeType type) { SPhysiNode* pPhysiNode = (SPhysiNode*)nodesMakeNode(type); if (NULL == pPhysiNode) { return NULL; } + pPhysiNode->outputTuple.tupleId = pCxt->nextTupleId++; pPhysiNode->outputTuple.type = QUERY_NODE_TUPLE_DESC; return pPhysiNode; } -static int32_t initScanPhysiNode(SPhysiPlanContext* pCxt, SScanLogicNode* pScanLogicNode, SScanPhysiNode* pScanPhysiNode) { - CHECK_CODE(addTupleDesc(pCxt, pScanLogicNode->pScanCols, &pScanPhysiNode->node.outputTuple, &pScanPhysiNode->pScanCols), TSDB_CODE_OUT_OF_MEMORY); - - if (NULL != pScanLogicNode->node.pConditions) { - pScanPhysiNode->node.pConditions = transformForPhysiPlan(pCxt, pScanPhysiNode->node.outputTuple.tupleId, pScanLogicNode->node.pConditions); - CHECK_ALLOC(pScanPhysiNode->node.pConditions, TSDB_CODE_OUT_OF_MEMORY); +static int32_t setConditionsSlotId(SPhysiPlanContext* pCxt, const SLogicNode* pLogicNode, SPhysiNode* pPhysiNode) { + if (NULL != pLogicNode->pConditions) { + pPhysiNode->pConditions = setNodeSlotId(pCxt, pPhysiNode->outputTuple.tupleId, -1, pLogicNode->pConditions); + CHECK_ALLOC(pPhysiNode->pConditions, TSDB_CODE_OUT_OF_MEMORY); } + return TSDB_CODE_SUCCESS; +} + +static int32_t setSlotOutput(SPhysiPlanContext* pCxt, SNodeList* pTargets, STupleDescNode* pTuple) { + SHashObj* pHash = taosArrayGetP(pCxt->pTupleHelper, pTuple->tupleId); + char name[TSDB_TABLE_NAME_LEN + TSDB_COL_NAME_LEN]; + SNode* pNode; + FOREACH(pNode, pTargets) { + int32_t len = getSlotKey(pNode, name); + SSlotIndex* pIndex = taosHashGet(pHash, name, len); + ((SSlotDescNode*)nodesListGetNode(pTuple->pSlots, pIndex->slotId))->output = true; + } + + return TSDB_CODE_SUCCESS; +} + +static int32_t initScanPhysiNode(SPhysiPlanContext* pCxt, SScanLogicNode* pScanLogicNode, SScanPhysiNode* pScanPhysiNode) { + if (NULL != pScanLogicNode->pScanCols) { + pScanPhysiNode->pScanCols = nodesCloneList(pScanLogicNode->pScanCols); + CHECK_ALLOC(pScanPhysiNode->pScanCols, TSDB_CODE_OUT_OF_MEMORY); + } + // Tuple describe also needs to be set without scanning column, such as SELECT COUNT(*) FROM t + CHECK_CODE(addTupleDesc(pCxt, pScanPhysiNode->pScanCols, &pScanPhysiNode->node.outputTuple), TSDB_CODE_OUT_OF_MEMORY); + + CHECK_CODE(setConditionsSlotId(pCxt, (const SLogicNode*)pScanLogicNode, (SPhysiNode*)pScanPhysiNode), TSDB_CODE_OUT_OF_MEMORY); + + CHECK_CODE(setSlotOutput(pCxt, pScanLogicNode->node.pTargets, &pScanPhysiNode->node.outputTuple), TSDB_CODE_OUT_OF_MEMORY); pScanPhysiNode->uid = pScanLogicNode->pMeta->uid; pScanPhysiNode->tableType = pScanLogicNode->pMeta->tableType; @@ -570,14 +582,14 @@ static int32_t initScanPhysiNode(SPhysiPlanContext* pCxt, SScanLogicNode* pScanL } static SPhysiNode* createTagScanPhysiNode(SPhysiPlanContext* pCxt, SScanLogicNode* pScanLogicNode) { - STagScanPhysiNode* pTagScan = (STagScanPhysiNode*)makePhysiNode(QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN); + STagScanPhysiNode* pTagScan = (STagScanPhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN); CHECK_ALLOC(pTagScan, NULL); CHECK_CODE(initScanPhysiNode(pCxt, pScanLogicNode, (SScanPhysiNode*)pTagScan), (SPhysiNode*)pTagScan); return (SPhysiNode*)pTagScan; } static SPhysiNode* createTableScanPhysiNode(SPhysiPlanContext* pCxt, SScanLogicNode* pScanLogicNode) { - STableScanPhysiNode* pTableScan = (STableScanPhysiNode*)makePhysiNode(QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN); + STableScanPhysiNode* pTableScan = (STableScanPhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN); CHECK_ALLOC(pTableScan, NULL); CHECK_CODE(initScanPhysiNode(pCxt, pScanLogicNode, (SScanPhysiNode*)pTableScan), (SPhysiNode*)pTableScan); pTableScan->scanFlag = pScanLogicNode->scanFlag; @@ -597,35 +609,205 @@ static SPhysiNode* createScanPhysiNode(SPhysiPlanContext* pCxt, SScanLogicNode* default: break; } + return NULL; } -static SPhysiNode* createProjectPhysiNode(SPhysiPlanContext* pCxt, SProjectLogicNode* pProjectLogicNode) { - SProjectPhysiNode* pProject = (SProjectPhysiNode*)makePhysiNode(QUERY_NODE_PHYSICAL_PLAN_PROJECT); +static SNodeList* createJoinOutputCols(SPhysiPlanContext* pCxt, STupleDescNode* pLeftTuple, STupleDescNode* pRightTuple) { + SNodeList* pCols = nodesMakeList(); + CHECK_ALLOC(pCols, NULL); + SNode* pNode; + FOREACH(pNode, pLeftTuple->pSlots) { + SSlotDescNode* pSlot = (SSlotDescNode*)pNode; + SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); + if (NULL == pCol) { + goto error; + } + pCol->node.resType = pSlot->dataType; + pCol->tupleId = pLeftTuple->tupleId; + pCol->slotId = pSlot->slotId; + pCol->colId = -1; + if (TSDB_CODE_SUCCESS != nodesListAppend(pCols, (SNode*)pCol)) { + goto error; + } + } + FOREACH(pNode, pRightTuple->pSlots) { + SSlotDescNode* pSlot = (SSlotDescNode*)pNode; + SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); + if (NULL == pCol) { + goto error; + } + pCol->node.resType = pSlot->dataType; + pCol->tupleId = pRightTuple->tupleId; + pCol->slotId = pSlot->slotId; + pCol->colId = -1; + if (TSDB_CODE_SUCCESS != nodesListAppend(pCols, (SNode*)pCol)) { + goto error; + } + } + return pCols; +error: + nodesDestroyList(pCols); + return NULL; +} + +static SPhysiNode* createJoinPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SJoinLogicNode* pJoinLogicNode) { + SJoinPhysiNode* pJoin = (SJoinPhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_JOIN); + CHECK_ALLOC(pJoin, NULL); + + STupleDescNode* pLeftTuple = &((SPhysiNode*)nodesListGetNode(pChildren, 0))->outputTuple; + STupleDescNode* pRightTuple = &((SPhysiNode*)nodesListGetNode(pChildren, 1))->outputTuple; + pJoin->pOnConditions = setNodeSlotId(pCxt, pLeftTuple->tupleId, pRightTuple->tupleId, pJoinLogicNode->pOnConditions); + CHECK_ALLOC(pJoin->pOnConditions, (SPhysiNode*)pJoin); + + pJoin->pTargets = createJoinOutputCols(pCxt, pLeftTuple, pRightTuple); + CHECK_ALLOC(pJoin->pTargets, (SPhysiNode*)pJoin); + CHECK_CODE(addTupleDesc(pCxt, pJoin->pTargets, &pJoin->node.outputTuple), (SPhysiNode*)pJoin); + + CHECK_CODE(setConditionsSlotId(pCxt, (const SLogicNode*)pJoinLogicNode, (SPhysiNode*)pJoin), (SPhysiNode*)pJoin); + + CHECK_CODE(setSlotOutput(pCxt, pJoinLogicNode->node.pTargets, &pJoin->node.outputTuple), (SPhysiNode*)pJoin); + + return (SPhysiNode*)pJoin; +} + +typedef struct SRewritePrecalcExprsCxt { + int32_t errCode; + int32_t planNodeId; + int32_t rewriteId; + SNodeList* pPrecalcExprs; +} SRewritePrecalcExprsCxt; + +static EDealRes collectAndRewrite(SRewritePrecalcExprsCxt* pCxt, SNode** pNode) { + SNode* pExpr = nodesCloneNode(*pNode); + CHECK_ALLOC(pExpr, DEAL_RES_ERROR); + if (nodesListAppend(pCxt->pPrecalcExprs, pExpr)) { + nodesDestroyNode(pExpr); + return DEAL_RES_ERROR; + } + SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); + if (NULL == pCol) { + nodesDestroyNode(pExpr); + return DEAL_RES_ERROR; + } + SExprNode* pToBeRewrittenExpr = (SExprNode*)(*pNode); + pCol->node.resType = pToBeRewrittenExpr->resType; + strcpy(pCol->colName, pToBeRewrittenExpr->aliasName); + nodesDestroyNode(*pNode); + *pNode = (SNode*)pCol; + return DEAL_RES_IGNORE_CHILD; +} + +static EDealRes doRewritePrecalcExprs(SNode** pNode, void* pContext) { + SRewritePrecalcExprsCxt* pCxt = (SRewritePrecalcExprsCxt*)pContext; + switch (nodeType(*pNode)) { + case QUERY_NODE_OPERATOR: + case QUERY_NODE_LOGIC_CONDITION: { + return collectAndRewrite(pContext, pNode); + } + case QUERY_NODE_FUNCTION: { + if (!fmIsAggFunc(((SFunctionNode*)(*pNode))->funcId)) { + return collectAndRewrite(pContext, pNode); + } + } + default: + break; + } + return DEAL_RES_CONTINUE; +} + +static int32_t rewritePrecalcExprs(SPhysiPlanContext* pCxt, SNodeList* pList, SNodeList** pPrecalcExprs, SNodeList** pRewrittenList) { + if (NULL == pList) { + return TSDB_CODE_SUCCESS; + } + + if (NULL == *pPrecalcExprs) { + *pPrecalcExprs = nodesMakeList(); + CHECK_ALLOC(*pPrecalcExprs, TSDB_CODE_OUT_OF_MEMORY); + } + if (NULL == *pRewrittenList) { + *pRewrittenList = nodesMakeList(); + CHECK_ALLOC(*pRewrittenList, TSDB_CODE_OUT_OF_MEMORY); + } + SNode* pNode = NULL; + FOREACH(pNode, pList) { + SNode* pNew = NULL; + if (QUERY_NODE_GROUPING_SET == nodeType(pNode)) { + pNew = nodesCloneNode(nodesListGetNode(((SGroupingSetNode*)pNode)->pParameterList, 0)); + } else { + pNew = nodesCloneNode(pNode); + } + CHECK_ALLOC(pNew, TSDB_CODE_OUT_OF_MEMORY); + CHECK_CODE(nodesListAppend(*pRewrittenList, pNew), TSDB_CODE_OUT_OF_MEMORY); + } + SRewritePrecalcExprsCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pPrecalcExprs = *pPrecalcExprs }; + nodesRewriteList(*pRewrittenList, doRewritePrecalcExprs, &cxt); + if (0 == LIST_LENGTH(cxt.pPrecalcExprs)) { + nodesDestroyList(cxt.pPrecalcExprs); + *pPrecalcExprs = NULL; + } + return cxt.errCode; +} + +static SPhysiNode* createAggPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SAggLogicNode* pAggLogicNode) { + SAggPhysiNode* pAgg = (SAggPhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_AGG); + CHECK_ALLOC(pAgg, NULL); + + SNodeList* pPrecalcExprs = NULL; + SNodeList* pGroupKeys = NULL; + SNodeList* pAggFuncs = NULL; + CHECK_CODE(rewritePrecalcExprs(pCxt, pAggLogicNode->pGroupKeys, &pPrecalcExprs, &pGroupKeys), (SPhysiNode*)pAgg); + CHECK_CODE(rewritePrecalcExprs(pCxt, pAggLogicNode->pAggFuncs, &pPrecalcExprs, &pAggFuncs), (SPhysiNode*)pAgg); + + STupleDescNode* pChildTupe = &(((SPhysiNode*)nodesListGetNode(pChildren, 0))->outputTuple); + // push down expression to outputTuple of child node + if (NULL != pPrecalcExprs) { + pAgg->pExprs = setListSlotId(pCxt, pChildTupe->tupleId, -1, pPrecalcExprs); + CHECK_ALLOC(pAgg->pExprs, (SPhysiNode*)pAgg); + CHECK_CODE(addTupleDesc(pCxt, pAgg->pExprs, pChildTupe), (SPhysiNode*)pAgg); + } + + if (NULL != pGroupKeys) { + pAgg->pGroupKeys = setListSlotId(pCxt, pChildTupe->tupleId, -1, pGroupKeys); + CHECK_ALLOC(pAgg->pGroupKeys, (SPhysiNode*)pAgg); + CHECK_CODE(addTupleDesc(pCxt, pAgg->pGroupKeys, &pAgg->node.outputTuple), (SPhysiNode*)pAgg); + } + + if (NULL != pAggFuncs) { + pAgg->pAggFuncs = setListSlotId(pCxt, pChildTupe->tupleId, -1, pAggFuncs); + CHECK_ALLOC(pAgg->pAggFuncs, (SPhysiNode*)pAgg); + CHECK_CODE(addTupleDesc(pCxt, pAgg->pAggFuncs, &pAgg->node.outputTuple), (SPhysiNode*)pAgg); + } + + CHECK_CODE(setConditionsSlotId(pCxt, (const SLogicNode*)pAggLogicNode, (SPhysiNode*)pAgg), (SPhysiNode*)pAgg); + + CHECK_CODE(setSlotOutput(pCxt, pAggLogicNode->node.pTargets, &pAgg->node.outputTuple), (SPhysiNode*)pAgg); + + return (SPhysiNode*)pAgg; +} + +static SPhysiNode* createProjectPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SProjectLogicNode* pProjectLogicNode) { + SProjectPhysiNode* pProject = (SProjectPhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_PROJECT); CHECK_ALLOC(pProject, NULL); - SNodeList* pProjections = transformListForPhysiPlan(pCxt, pProject->node.outputTuple.tupleId, pProjectLogicNode->pProjections); - CHECK_ALLOC(pProjections, (SPhysiNode*)pProject); - CHECK_CODE(addTupleDesc(pCxt, pProjections, &pProject->node.outputTuple, &pProject->pProjections), (SPhysiNode*)pProject); - nodesDestroyList(pProjections); + pProject->pProjections = setListSlotId(pCxt, ((SPhysiNode*)nodesListGetNode(pChildren, 0))->outputTuple.tupleId, -1, pProjectLogicNode->pProjections); + CHECK_ALLOC(pProject->pProjections, (SPhysiNode*)pProject); + CHECK_CODE(addTupleDesc(pCxt, pProject->pProjections, &pProject->node.outputTuple), (SPhysiNode*)pProject); - if (NULL != pProjectLogicNode->node.pConditions) { - pProject->node.pConditions = transformForPhysiPlan(pCxt, pProject->node.outputTuple.tupleId, pProjectLogicNode->node.pConditions); - CHECK_ALLOC(pProject->node.pConditions, (SPhysiNode*)pProject); - } + CHECK_CODE(setConditionsSlotId(pCxt, (const SLogicNode*)pProjectLogicNode, (SPhysiNode*)pProject), (SPhysiNode*)pProject); return (SPhysiNode*)pProject; } static SPhysiNode* createPhysiNode(SPhysiPlanContext* pCxt, SLogicNode* pLogicPlan) { - SNodeList* pChildern = nodesMakeList(); - CHECK_ALLOC(pChildern, NULL); + SNodeList* pChildren = nodesMakeList(); + CHECK_ALLOC(pChildren, NULL); SNode* pLogicChild; FOREACH(pLogicChild, pLogicPlan->pChildren) { SNode* pChildPhyNode = (SNode*)createPhysiNode(pCxt, (SLogicNode*)pLogicChild); - if (TSDB_CODE_SUCCESS != nodesListAppend(pChildern, pChildPhyNode)) { + if (TSDB_CODE_SUCCESS != nodesListAppend(pChildren, pChildPhyNode)) { pCxt->errCode = TSDB_CODE_OUT_OF_MEMORY; - nodesDestroyList(pChildern); + nodesDestroyList(pChildren); return NULL; } } @@ -636,22 +818,22 @@ static SPhysiNode* createPhysiNode(SPhysiPlanContext* pCxt, SLogicNode* pLogicPl pPhyNode = createScanPhysiNode(pCxt, (SScanLogicNode*)pLogicPlan); break; case QUERY_NODE_LOGIC_PLAN_JOIN: + pPhyNode = createJoinPhysiNode(pCxt, pChildren, (SJoinLogicNode*)pLogicPlan); break; case QUERY_NODE_LOGIC_PLAN_AGG: + pPhyNode = createAggPhysiNode(pCxt, pChildren, (SAggLogicNode*)pLogicPlan); break; case QUERY_NODE_LOGIC_PLAN_PROJECT: - pPhyNode = createProjectPhysiNode(pCxt, (SProjectLogicNode*)pLogicPlan); + pPhyNode = createProjectPhysiNode(pCxt, pChildren, (SProjectLogicNode*)pLogicPlan); break; default: break; } - if (NULL != pPhyNode) { - pPhyNode->pChildren = pChildern; - SNode* pChild; - FOREACH(pChild, pPhyNode->pChildren) { - ((SPhysiNode*)pChild)->pParent = pPhyNode; - } + pPhyNode->pChildren = pChildren; + SNode* pChild; + FOREACH(pChild, pPhyNode->pChildren) { + ((SPhysiNode*)pChild)->pParent = pPhyNode; } return pPhyNode; diff --git a/source/libs/planner/test/newPlannerTest.cpp b/source/libs/planner/test/newPlannerTest.cpp index e99f0c150c..51ef52ac2d 100644 --- a/source/libs/planner/test/newPlannerTest.cpp +++ b/source/libs/planner/test/newPlannerTest.cpp @@ -123,8 +123,8 @@ TEST_F(NewPlannerTest, simple) { TEST_F(NewPlannerTest, groupBy) { setDatabase("root", "test"); - bind("SELECT count(*) FROM t1"); - ASSERT_TRUE(run()); + // bind("SELECT count(*) FROM t1"); + // ASSERT_TRUE(run()); bind("SELECT c1, count(*) FROM t1 GROUP BY c1"); ASSERT_TRUE(run()); diff --git a/source/libs/qcom/src/queryUtil.c b/source/libs/qcom/src/queryUtil.c index b29817a318..a2165453d5 100644 --- a/source/libs/qcom/src/queryUtil.c +++ b/source/libs/qcom/src/queryUtil.c @@ -85,11 +85,10 @@ static void* pTaskQueue = NULL; int32_t initTaskQueue() { double factor = 4.0; - // todo - // int32_t numOfThreads = TMAX((int)(tsNumOfCores * tsNumOfThreadsPerCore / factor), 2); - int32_t numOfThreads = TMAX((int)(tsNumOfCores * 1.0f / factor), 2); - int32_t queueSize = 25000; //tsMaxConnections * 2; + int32_t numOfThreads = TMAX((int)(tsNumOfCores * tsNumOfThreadsPerCore / factor), 2); + + int32_t queueSize = tsMaxConnections * 2; pTaskQueue = taosInitScheduler(queueSize, numOfThreads, "tsc"); if (NULL == pTaskQueue) { qError("failed to init task queue"); diff --git a/source/libs/scalar/inc/filterInt.h b/source/libs/scalar/inc/filterInt.h index f9aa994283..a0d4bc680c 100644 --- a/source/libs/scalar/inc/filterInt.h +++ b/source/libs/scalar/inc/filterInt.h @@ -301,11 +301,11 @@ typedef struct SFilterInfo { #define FILTER_GET_FIELD(i, id) (&((i)->fields[(id).type].fields[(id).idx])) #define FILTER_GET_COL_FIELD(i, idx) (&((i)->fields[FLD_TYPE_COLUMN].fields[idx])) -#define FILTER_GET_COL_FIELD_TYPE(fi) (((SColumnRefNode *)((fi)->desc))->dataType.type) -#define FILTER_GET_COL_FIELD_SIZE(fi) (((SColumnRefNode *)((fi)->desc))->dataType.bytes) -#define FILTER_GET_COL_FIELD_ID(fi) (((SColumnRefNode *)((fi)->desc))->columnId) -#define FILTER_GET_COL_FIELD_SLOT_ID(fi) (((SColumnRefNode *)((fi)->desc))->slotId) -#define FILTER_GET_COL_FIELD_DESC(fi) ((SColumnRefNode *)((fi)->desc)) +#define FILTER_GET_COL_FIELD_TYPE(fi) (((SColumnNode *)((fi)->desc))->node.resType.type) +#define FILTER_GET_COL_FIELD_SIZE(fi) (((SColumnNode *)((fi)->desc))->node.resType.bytes) +#define FILTER_GET_COL_FIELD_ID(fi) (((SColumnNode *)((fi)->desc))->colId) +#define FILTER_GET_COL_FIELD_SLOT_ID(fi) (((SColumnNode *)((fi)->desc))->slotId) +#define FILTER_GET_COL_FIELD_DESC(fi) ((SColumnNode *)((fi)->desc)) #define FILTER_GET_COL_FIELD_DATA(fi, ri) (colDataGet(((SColumnInfoData *)(fi)->data), (ri))) #define FILTER_GET_VAL_FIELD_TYPE(fi) (((SValueNode *)((fi)->desc))->node.resType.type) #define FILTER_GET_VAL_FIELD_DATA(fi) ((char *)(fi)->data) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index c8f94d2d97..5eb0003662 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -914,14 +914,14 @@ int32_t filterAddFieldFromNode(SFilterInfo *info, SNode *node, SFilterFieldId *f FLT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); } - if (nodeType(node) != QUERY_NODE_COLUMN_REF && nodeType(node) != QUERY_NODE_VALUE && nodeType(node) != QUERY_NODE_NODE_LIST) { + if (nodeType(node) != QUERY_NODE_COLUMN && nodeType(node) != QUERY_NODE_VALUE && nodeType(node) != QUERY_NODE_NODE_LIST) { FLT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); } int32_t type; void *v; - if (nodeType(node) == QUERY_NODE_COLUMN_REF) { + if (nodeType(node) == QUERY_NODE_COLUMN) { type = FLD_TYPE_COLUMN; v = node; } else { @@ -1454,7 +1454,7 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options) qDebug("COLUMN Field Num:%u", info->fields[FLD_TYPE_COLUMN].num); for (uint32_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) { SFilterField *field = &info->fields[FLD_TYPE_COLUMN].fields[i]; - SColumnRefNode *refNode = (SColumnRefNode *)field->desc; + SColumnNode *refNode = (SColumnNode *)field->desc; qDebug("COL%d => [%d][%d]", i, refNode->tupleId, refNode->slotId); } @@ -1483,7 +1483,7 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options) char str[512] = {0}; SFilterField *left = FILTER_UNIT_LEFT_FIELD(info, unit); - SColumnRefNode *refNode = (SColumnRefNode *)left->desc; + SColumnNode *refNode = (SColumnNode *)left->desc; if (unit->compare.optr >= 0 && unit->compare.optr <= OP_TYPE_JSON_CONTAINS){ len = sprintf(str, "UNIT[%d] => [%d][%d] %s [", i, refNode->tupleId, refNode->slotId, gOptrStr[unit->compare.optr].str); } @@ -3487,7 +3487,7 @@ EDealRes fltReviseRewriter(SNode** pNode, void* pContext) { return DEAL_RES_CONTINUE; } - if (QUERY_NODE_VALUE == nodeType(*pNode) || QUERY_NODE_NODE_LIST == nodeType(*pNode) || QUERY_NODE_COLUMN_REF == nodeType(*pNode)) { + if (QUERY_NODE_VALUE == nodeType(*pNode) || QUERY_NODE_NODE_LIST == nodeType(*pNode) || QUERY_NODE_COLUMN == nodeType(*pNode)) { return DEAL_RES_CONTINUE; } @@ -3510,7 +3510,7 @@ EDealRes fltReviseRewriter(SNode** pNode, void* pContext) { return DEAL_RES_ERROR; } - if (QUERY_NODE_COLUMN_REF != nodeType(node->pLeft)) { + if (QUERY_NODE_COLUMN != nodeType(node->pLeft)) { stat->scalarMode = true; return DEAL_RES_CONTINUE; } @@ -3521,12 +3521,12 @@ EDealRes fltReviseRewriter(SNode** pNode, void* pContext) { return DEAL_RES_CONTINUE; } } else { - if ((QUERY_NODE_COLUMN_REF != nodeType(node->pLeft)) && (QUERY_NODE_VALUE != nodeType(node->pLeft))) { + if ((QUERY_NODE_COLUMN != nodeType(node->pLeft)) && (QUERY_NODE_VALUE != nodeType(node->pLeft))) { stat->scalarMode = true; return DEAL_RES_CONTINUE; } - if ((QUERY_NODE_COLUMN_REF != nodeType(node->pRight)) && (QUERY_NODE_VALUE != nodeType(node->pRight))) { + if ((QUERY_NODE_COLUMN != nodeType(node->pRight)) && (QUERY_NODE_VALUE != nodeType(node->pRight))) { stat->scalarMode = true; return DEAL_RES_CONTINUE; } @@ -3536,7 +3536,7 @@ EDealRes fltReviseRewriter(SNode** pNode, void* pContext) { return DEAL_RES_CONTINUE; } - if (QUERY_NODE_COLUMN_REF != nodeType(node->pLeft)) { + if (QUERY_NODE_COLUMN != nodeType(node->pLeft)) { SNode *t = node->pLeft; node->pLeft = node->pRight; node->pRight = t; @@ -3549,10 +3549,10 @@ EDealRes fltReviseRewriter(SNode** pNode, void* pContext) { } if (OP_TYPE_IN != node->opType) { - SColumnRefNode *refNode = (SColumnRefNode *)node->pLeft; + SColumnNode *refNode = (SColumnNode *)node->pLeft; SValueNode *valueNode = (SValueNode *)node->pRight; - int32_t type = vectorGetConvertType(refNode->dataType.type, valueNode->node.resType.type); - if (0 != type && type != refNode->dataType.type) { + int32_t type = vectorGetConvertType(refNode->node.resType.type, valueNode->node.resType.type); + if (0 != type && type != refNode->node.resType.type) { stat->scalarMode = true; return DEAL_RES_CONTINUE; } diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index 336ef8f4f8..742c7fd706 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -134,13 +134,13 @@ int32_t sclInitParam(SNode* node, SScalarParam *param, SScalarCtx *ctx, int32_t break; } - case QUERY_NODE_COLUMN_REF: { + case QUERY_NODE_COLUMN: { if (NULL == ctx) { sclError("invalid node type for constant calculating, type:%d, ctx:%p", nodeType(node), ctx); SCL_ERR_RET(TSDB_CODE_QRY_APP_ERROR); } - SColumnRefNode *ref = (SColumnRefNode *)node; + SColumnNode *ref = (SColumnNode *)node; if (ref->slotId >= taosArrayGetSize(ctx->pSrc->pDataBlock)) { sclError("column ref slotId is too big, slodId:%d, dataBlockSize:%d", ref->slotId, (int32_t)taosArrayGetSize(ctx->pSrc->pDataBlock)); SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); @@ -281,7 +281,8 @@ int32_t sclExecFuncion(SFunctionNode *node, SScalarCtx *ctx, SScalarParam *outpu SScalarFuncExecFuncs ffpSet = {0}; int32_t code = fmGetScalarFuncExecFuncs(node->funcId, &ffpSet); if (code) { - sclError( "fmGetFuncExecFuncs failed, funcId:%d, code:%s", node->funcId, tstrerror(code)); + sclError( +"fmGetFuncExecFuncs failed, funcId:%d, code:%s", node->funcId, tstrerror(code)); SCL_ERR_RET(code); } @@ -299,7 +300,8 @@ int32_t sclExecFuncion(SFunctionNode *node, SScalarCtx *ctx, SScalarParam *outpu for (int32_t i = 0; i < rowNum; ++i) { code = (*ffpSet.process)(params, node->pParameterList->length, output); if (code) { - sclError( "scalar function exec failed, funcId:%d, code:%s", node->funcId, tstrerror(code)); + sclError( +"scalar function exec failed, funcId:%d, code:%s", node->funcId, tstrerror(code)); SCL_ERR_JRET(code); } @@ -599,7 +601,7 @@ EDealRes sclWalkOperator(SNode* pNode, void* pContext) { EDealRes sclCalcWalker(SNode* pNode, void* pContext) { - if (QUERY_NODE_VALUE == nodeType(pNode) || QUERY_NODE_NODE_LIST == nodeType(pNode) || QUERY_NODE_COLUMN_REF == nodeType(pNode)) { + if (QUERY_NODE_VALUE == nodeType(pNode) || QUERY_NODE_NODE_LIST == nodeType(pNode) || QUERY_NODE_COLUMN == nodeType(pNode)) { return DEAL_RES_CONTINUE; } diff --git a/source/libs/scalar/test/filter/filterTests.cpp b/source/libs/scalar/test/filter/filterTests.cpp index a159263804..dc45f14366 100644 --- a/source/libs/scalar/test/filter/filterTests.cpp +++ b/source/libs/scalar/test/filter/filterTests.cpp @@ -76,15 +76,15 @@ void flttMakeValueNode(SNode **pNode, int32_t dataType, void *value) { } void flttMakeColRefNode(SNode **pNode, SSDataBlock **block, int32_t dataType, int32_t dataBytes, int32_t rowNum, void *value) { - SNode *node = nodesMakeNode(QUERY_NODE_COLUMN_REF); - SColumnRefNode *rnode = (SColumnRefNode *)node; - rnode->dataType.type = dataType; - rnode->dataType.bytes = dataBytes; + SNode *node = nodesMakeNode(QUERY_NODE_COLUMN); + SColumnNode *rnode = (SColumnNode *)node; + rnode->node.resType.type = dataType; + rnode->node.resType.bytes = dataBytes; rnode->tupleId = 0; if (NULL == block) { rnode->slotId = 2; - rnode->columnId = 55; + rnode->colId = 55; *pNode = (SNode *)rnode; return; @@ -120,7 +120,7 @@ void flttMakeColRefNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in taosArrayPush(res->pDataBlock, &idata); rnode->slotId = 2; - rnode->columnId = 55; + rnode->colId = 55; *block = res; } else { @@ -137,7 +137,7 @@ void flttMakeColRefNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in res->info.numOfCols++; rnode->slotId = idx; - rnode->columnId = 55 + idx; + rnode->colId = 55 + idx; } *pNode = (SNode *)rnode; @@ -259,7 +259,7 @@ TEST(columnTest, smallint_column_greater_double_value) { ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - stat.colId = ((SColumnRefNode *)pLeft)->columnId; + stat.colId = ((SColumnNode *)pLeft)->colId; stat.max = 10; stat.min = 5; stat.numOfNull = 0; @@ -310,7 +310,7 @@ TEST(columnTest, int_column_greater_smallint_value) { ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - stat.colId = ((SColumnRefNode *)pLeft)->columnId; + stat.colId = ((SColumnNode *)pLeft)->colId; stat.max = 10; stat.min = 5; stat.numOfNull = 0; diff --git a/source/libs/scalar/test/scalar/scalarTests.cpp b/source/libs/scalar/test/scalar/scalarTests.cpp index 24bc8eaf40..df746e19cf 100644 --- a/source/libs/scalar/test/scalar/scalarTests.cpp +++ b/source/libs/scalar/test/scalar/scalarTests.cpp @@ -75,10 +75,10 @@ void scltMakeValueNode(SNode **pNode, int32_t dataType, void *value) { } void scltMakeColRefNode(SNode **pNode, SSDataBlock **block, int32_t dataType, int32_t dataBytes, int32_t rowNum, void *value) { - SNode *node = nodesMakeNode(QUERY_NODE_COLUMN_REF); - SColumnRefNode *rnode = (SColumnRefNode *)node; - rnode->dataType.type = dataType; - rnode->dataType.bytes = dataBytes; + SNode *node = nodesMakeNode(QUERY_NODE_COLUMN); + SColumnNode *rnode = (SColumnNode *)node; + rnode->node.resType.type = dataType; + rnode->node.resType.bytes = dataBytes; rnode->tupleId = 0; if (NULL == *block) { @@ -111,7 +111,7 @@ void scltMakeColRefNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in taosArrayPush(res->pDataBlock, &idata); rnode->slotId = 2; - rnode->columnId = 55; + rnode->colId = 55; *block = res; } else { @@ -126,7 +126,7 @@ void scltMakeColRefNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in taosArrayPush(res->pDataBlock, &idata); rnode->slotId = idx; - rnode->columnId = 55 + idx; + rnode->colId = 55 + idx; } *pNode = (SNode *)rnode; diff --git a/source/libs/tdb/src/db/tdbEnv.c b/source/libs/tdb/src/db/tdbEnv.c index 3670c770ab..2afce30828 100644 --- a/source/libs/tdb/src/db/tdbEnv.c +++ b/source/libs/tdb/src/db/tdbEnv.c @@ -18,7 +18,7 @@ struct STDbEnv { char * rootDir; // root directory of the environment char * jname; // journal file name - int jfd; // journal file fd + TdFilePtr jpFile; // journal file fd pgsz_t pgSize; // page size cachesz_t cacheSize; // total cache size STDbList dbList; // TDB List @@ -55,7 +55,7 @@ int tdbEnvCreate(TENV **ppEnv, const char *rootDir) { pEnv->rootDir = (char *)(&pEnv[1]); pEnv->jname = pEnv->rootDir + slen + 1; - pEnv->jfd = -1; + pEnv->jpFile = NULL; pEnv->pgSize = TDB_DEFAULT_PGSIZE; pEnv->cacheSize = TDB_DEFAULT_CACHE_SIZE; @@ -139,8 +139,8 @@ static int tdbEnvDestroy(TENV *pEnv) { } int tdbEnvBeginTxn(TENV *pEnv) { - pEnv->jfd = open(pEnv->jname, O_CREAT | O_RDWR, 0755); - if (pEnv->jfd < 0) { + pEnv->jpFile = taosOpenFile(pEnv->jname, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_READ); + if (pEnv->jpFile < 0) { return -1; } @@ -149,8 +149,8 @@ int tdbEnvBeginTxn(TENV *pEnv) { int tdbEnvCommit(TENV *pEnv) { /* TODO */ - close(pEnv->jfd); - pEnv->jfd = -1; + taosCloseFile(&pEnv->jpFile); + pEnv->jpFile = NULL; return 0; } diff --git a/source/libs/tdb/src/db/tdbPgFile.c b/source/libs/tdb/src/db/tdbPgFile.c index ee5b486f7b..48f8ed1792 100644 --- a/source/libs/tdb/src/db/tdbPgFile.c +++ b/source/libs/tdb/src/db/tdbPgFile.c @@ -51,10 +51,10 @@ int pgFileOpen(SPgFile **ppPgFile, const char *fname, TENV *pEnv) { pPgFile->fname = (char *)(&(pPgFile[1])); memcpy(pPgFile->fname, fname, fnameLen); pPgFile->fname[fnameLen] = '\0'; - pPgFile->fd = -1; + pPgFile->pFile = NULL; - pPgFile->fd = open(fname, O_CREAT | O_RDWR, 0755); - if (pPgFile->fd < 0) { + pPgFile->pFile = taosOpenFile(fname, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_READ); + if (pPgFile->pFile == NULL) { // TODO: handle error return -1; } @@ -95,8 +95,8 @@ int pgFileOpen(SPgFile **ppPgFile, const char *fname, TENV *pEnv) { int pgFileClose(SPgFile *pPgFile) { if (pPgFile) { - if (pPgFile->fd >= 0) { - close(pPgFile->fd); + if (pPgFile->pFile >= 0) { + taosCloseFile(&pPgFile->pFile); } tfree(pPgFile->fname); @@ -201,7 +201,7 @@ static int pgFileRead(SPgFile *pPgFile, pgno_t pgno, uint8_t *pData) { pTData = pData; szToRead = pgSize; for (; szToRead > 0;) { - rsize = pread(pPgFile->fd, pTData, szToRead, pgno * pgSize); + rsize = pread(pPgFile->pFile, pTData, szToRead, pgno * pgSize); if (rsize < 0) { if (errno == EINTR) { continue; diff --git a/source/libs/tdb/src/inc/tdbPgFile.h b/source/libs/tdb/src/inc/tdbPgFile.h index 2a7116a0dd..eaeebf6b9d 100644 --- a/source/libs/tdb/src/inc/tdbPgFile.h +++ b/source/libs/tdb/src/inc/tdbPgFile.h @@ -38,7 +38,7 @@ struct SPgFile { uint8_t fileid[TDB_FILE_ID_LEN]; // file id pgno_t lsize; // page file logical size (for count) pgno_t fsize; // real file size on disk (for rollback) - int fd; + TdFilePtr pFile; SPgFileListNode envHash; SPgFileListNode envPgfList; }; diff --git a/source/libs/tfs/test/tfsTest.cpp b/source/libs/tfs/test/tfsTest.cpp index 178d115c59..af66304f84 100644 --- a/source/libs/tfs/test/tfsTest.cpp +++ b/source/libs/tfs/test/tfsTest.cpp @@ -230,10 +230,11 @@ TEST_F(TfsTest, 04_File) { EXPECT_EQ(tfsMkdir(pTfs, "t3"), 0); - FILE *fp = fopen(f1.aname, "w"); - ASSERT_NE(fp, nullptr); - fwrite("12345678", 1, 5, fp); - fclose(fp); + // FILE *fp = fopen(f1.aname, "w"); + TdFilePtr pFile = taosOpenFile(f1.aname, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + ASSERT_NE(pFile, nullptr); + taosWriteFile(pFile, "12345678", 5); + taosCloseFile(&pFile); char base[128] = {0}; tfsBasename(&f1, base); @@ -638,10 +639,11 @@ TEST_F(TfsTest, 05_MultiDisk) { EXPECT_EQ(tfsMkdir(pTfs, "t3"), 0); - FILE *fp = fopen(f1.aname, "w"); - ASSERT_NE(fp, nullptr); - fwrite("12345678", 1, 5, fp); - fclose(fp); + // FILE *fp = fopen(f1.aname, "w"); + TdFilePtr pFile = taosOpenFile(f1.aname, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + ASSERT_NE(pFile, nullptr); + taosWriteFile(pFile, "12345678", 5); + taosCloseFile(&pFile); char base[128] = {0}; tfsBasename(&f1, base); diff --git a/source/libs/transport/inc/rpcUdp.h b/source/libs/transport/inc/rpcUdp.h index c1da6a9240..0c651d07ed 100644 --- a/source/libs/transport/inc/rpcUdp.h +++ b/source/libs/transport/inc/rpcUdp.h @@ -30,7 +30,7 @@ void *taosOpenUdpConnection(void *shandle, void *thandle, uint32_t ip, uint16_t void taosFreeMsgHdr(void *hdr); int taosMsgHdrSize(void *hdr); -void taosSendMsgHdr(void *hdr, int fd); +void taosSendMsgHdr(void *hdr, TdFilePtr pFile); void taosInitMsgHdr(void **hdr, void *dest, int maxPkts); void taosSetMsgHdrData(void *hdr, char *data, int dataLen); diff --git a/source/libs/transport/src/rpcMain.c b/source/libs/transport/src/rpcMain.c index 44bb6fdb70..e1319da162 100644 --- a/source/libs/transport/src/rpcMain.c +++ b/source/libs/transport/src/rpcMain.c @@ -42,8 +42,6 @@ int tsRpcMaxRetry; int tsRpcHeadSize; int tsRpcOverhead; -int32_t tsRpcForceTcp = 1; // disable this, means query, show command use udp protocol as default - SHashObj *tsFqdnHash; #ifndef USE_UV @@ -146,9 +144,6 @@ typedef struct SRpcConn { static int tsRpcRefId = -1; static int32_t tsRpcNum = 0; -int32_t tsRpcTimer = 300; -int32_t tsRpcMaxTime = 600; // seconds; - // static pthread_once_t tsRpcInit = PTHREAD_ONCE_INIT; // server:0 client:1 tcp:2 udp:0 @@ -228,7 +223,7 @@ static void rpcInitImp(void) { tsFqdnHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); } -int32_t rpcInit(void) { +int32_t rpcInit() { pthread_once(&tsRpcInitOnce, rpcInitImp); return 0; } diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index 453b5184d2..b45683617f 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -112,7 +112,7 @@ void rpcSendRedirectRsp(void* thandle, const SEpSet* pEpSet) { int rpcReportProgress(void* pConn, char* pCont, int contLen) { return -1; } void rpcCancelRequest(int64_t rid) { return; } -int32_t rpcInit(void) { +int32_t rpcInit() { // impl later return 0; } diff --git a/source/libs/transport/test/pushServer.c b/source/libs/transport/test/pushServer.c index 0ccfde16ce..a1c181ac98 100644 --- a/source/libs/transport/test/pushServer.c +++ b/source/libs/transport/test/pushServer.c @@ -22,7 +22,7 @@ int msgSize = 128; int commit = 0; -int dataFd = -1; +TdFilePtr pDataFile = NULL; STaosQueue *qhandle = NULL; STaosQset * qset = NULL; @@ -43,8 +43,8 @@ void processShellMsg() { for (int i = 0; i < numOfMsgs; ++i) { taosGetQitem(qall, (void **)&pRpcMsg); - if (dataFd >= 0) { - if (write(dataFd, pRpcMsg->pCont, pRpcMsg->contLen) < 0) { + if (pDataFile != NULL) { + if (taosWriteFile(pDataFile, pRpcMsg->pCont, pRpcMsg->contLen) < 0) { tInfo("failed to write data file, reason:%s", strerror(errno)); } } @@ -52,7 +52,7 @@ void processShellMsg() { if (commit >= 2) { num += numOfMsgs; - // if (taosFsync(dataFd) < 0) { + // if (taosFsync(pDataFile) < 0) { // tInfo("failed to flush data to file, reason:%s", strerror(errno)); //} @@ -181,8 +181,8 @@ int main(int argc, char *argv[]) { tInfo("RPC server is running, ctrl-c to exit"); if (commit) { - dataFd = open(dataName, O_APPEND | O_CREAT | O_WRONLY, S_IRWXU | S_IRWXG | S_IRWXO); - if (dataFd < 0) tInfo("failed to open data file, reason:%s", strerror(errno)); + pDataFile = taosOpenFile(dataName, TD_FILE_APPEND | TD_FILE_CTEATE | TD_FILE_WRITE); + if (pDataFile == NULL) tInfo("failed to open data file, reason:%s", strerror(errno)); } qhandle = taosOpenQueue(); qset = taosOpenQset(); @@ -190,8 +190,8 @@ int main(int argc, char *argv[]) { processShellMsg(); - if (dataFd >= 0) { - close(dataFd); + if (pDataFile != NULL) { + taosCloseFile(&pDataFile); remove(dataName); } diff --git a/source/libs/transport/test/rserver.c b/source/libs/transport/test/rserver.c index 4337cd242c..794497500c 100644 --- a/source/libs/transport/test/rserver.c +++ b/source/libs/transport/test/rserver.c @@ -22,7 +22,7 @@ int msgSize = 128; int commit = 0; -int dataFd = -1; +TdFilePtr pDataFile = NULL; STaosQueue *qhandle = NULL; STaosQset * qset = NULL; @@ -43,8 +43,8 @@ void processShellMsg() { for (int i = 0; i < numOfMsgs; ++i) { taosGetQitem(qall, (void **)&pRpcMsg); - if (dataFd >= 0) { - if (write(dataFd, pRpcMsg->pCont, pRpcMsg->contLen) < 0) { + if (pDataFile >= 0) { + if (taosWriteFile(pDataFile, pRpcMsg->pCont, pRpcMsg->contLen) < 0) { tInfo("failed to write data file, reason:%s", strerror(errno)); } } @@ -52,7 +52,7 @@ void processShellMsg() { if (commit >= 2) { num += numOfMsgs; - // if (taosFsync(dataFd) < 0) { + // if (taosFsync(pDataFile) < 0) { // tInfo("failed to flush data to file, reason:%s", strerror(errno)); //} @@ -170,8 +170,8 @@ int main(int argc, char *argv[]) { tInfo("RPC server is running, ctrl-c to exit"); if (commit) { - dataFd = open(dataName, O_APPEND | O_CREAT | O_WRONLY, S_IRWXU | S_IRWXG | S_IRWXO); - if (dataFd < 0) tInfo("failed to open data file, reason:%s", strerror(errno)); + pDataFile = taosOpenFile(dataName, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_APPEND); + if (pDataFile == NULL) tInfo("failed to open data file, reason:%s", strerror(errno)); } qhandle = taosOpenQueue(); qset = taosOpenQset(); @@ -179,8 +179,8 @@ int main(int argc, char *argv[]) { processShellMsg(); - if (dataFd >= 0) { - close(dataFd); + if (pDataFile != NULL) { + taosCloseFile(&pDataFile); remove(dataName); } diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index 2d6fb8fc76..ae0b0bd849 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -69,8 +69,8 @@ static inline int64_t walScanLogGetLastVer(SWal* pWal) { int readSize = TMIN(WAL_MAX_SIZE + 2, statbuf.st_size); pLastFileInfo->fileSize = statbuf.st_size; - FileFd fd = taosOpenFileRead(fnameStr); - if (fd < 0) { + TdFilePtr pFile = taosOpenFile(fnameStr, TD_FILE_READ); + if (pFile == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } @@ -79,15 +79,15 @@ static inline int64_t walScanLogGetLastVer(SWal* pWal) { char* buf = malloc(readSize + 5); if (buf == NULL) { - taosCloseFile(fd); + taosCloseFile(&pFile); terrno = TSDB_CODE_WAL_OUT_OF_MEMORY; return -1; } - taosLSeekFile(fd, -readSize, SEEK_END); - if (readSize != taosReadFile(fd, buf, readSize)) { + taosLSeekFile(pFile, -readSize, SEEK_END); + if (readSize != taosReadFile(pFile, buf, readSize)) { free(buf); - taosCloseFile(fd); + taosCloseFile(&pFile); terrno = TAOS_SYSTEM_ERROR(errno); return -1; } @@ -108,12 +108,12 @@ static inline int64_t walScanLogGetLastVer(SWal* pWal) { if (walValidHeadCksum(logContent) != 0 || walValidBodyCksum(logContent) != 0) { // file has to be deleted free(buf); - taosCloseFile(fd); + taosCloseFile(&pFile); terrno = TSDB_CODE_WAL_FILE_CORRUPTED; return -1; } } - taosCloseFile(fd); + taosCloseFile(&pFile); SWalHead *lastEntry = (SWalHead*)found; return lastEntry->head.version; @@ -364,18 +364,18 @@ int walSaveMeta(SWal* pWal) { int metaVer = walFindCurMetaVer(pWal); char fnameStr[WAL_FILE_LEN]; walBuildMetaName(pWal, metaVer + 1, fnameStr); - FileFd metaFd = taosOpenFileCreateWrite(fnameStr); - if (metaFd < 0) { + TdFilePtr pMataFile = taosOpenFile(fnameStr, TD_FILE_CTEATE | TD_FILE_WRITE); + if (pMataFile == NULL) { return -1; } char* serialized = walMetaSerialize(pWal); int len = strlen(serialized); - if (len != taosWriteFile(metaFd, serialized, len)) { + if (len != taosWriteFile(pMataFile, serialized, len)) { // TODO:clean file return -1; } - taosCloseFile(metaFd); + taosCloseFile(&pMataFile); // delete old file if (metaVer > -1) { walBuildMetaName(pWal, metaVer, fnameStr); @@ -404,20 +404,20 @@ int walLoadMeta(SWal* pWal) { return -1; } memset(buf, 0, size + 5); - FileFd fd = taosOpenFileRead(fnameStr); - if (fd < 0) { + TdFilePtr pFile = taosOpenFile(fnameStr, TD_FILE_READ); + if (pFile == NULL) { terrno = TSDB_CODE_WAL_FILE_CORRUPTED; return -1; } - if (taosReadFile(fd, buf, size) != size) { + if (taosReadFile(pFile, buf, size) != size) { terrno = TAOS_SYSTEM_ERROR(errno); - taosCloseFile(fd); + taosCloseFile(&pFile); free(buf); return -1; } // load into fileInfoSet int code = walMetaDeserialize(pWal, buf); - taosCloseFile(fd); + taosCloseFile(&pFile); free(buf); return code; } diff --git a/source/libs/wal/src/walMgmt.c b/source/libs/wal/src/walMgmt.c index d5c28d9d9b..d3cd23f284 100644 --- a/source/libs/wal/src/walMgmt.c +++ b/source/libs/wal/src/walMgmt.c @@ -89,8 +89,8 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { // open meta walResetVer(&pWal->vers); - pWal->writeLogTfd = -1; - pWal->writeIdxTfd = -1; + pWal->pWriteLogTFile = NULL; + pWal->pWriteIdxTFile = NULL; pWal->writeCur = -1; pWal->fileInfoSet = taosArrayInit(8, sizeof(SWalFileInfo)); if (pWal->fileInfoSet == NULL) { @@ -164,10 +164,10 @@ int32_t walAlter(SWal *pWal, SWalCfg *pCfg) { void walClose(SWal *pWal) { pthread_mutex_lock(&pWal->mutex); - tfClose(pWal->writeLogTfd); - pWal->writeLogTfd = -1; - tfClose(pWal->writeIdxTfd); - pWal->writeIdxTfd = -1; + taosCloseFile(&pWal->pWriteLogTFile); + pWal->pWriteLogTFile = NULL; + taosCloseFile(&pWal->pWriteIdxTFile); + pWal->pWriteIdxTFile = NULL; walSaveMeta(pWal); taosArrayDestroy(pWal->fileInfoSet); pWal->fileInfoSet = NULL; @@ -207,7 +207,7 @@ static void walFsyncAll() { if (walNeedFsync(pWal)) { wTrace("vgId:%d, do fsync, level:%d seq:%d rseq:%d", pWal->cfg.vgId, pWal->cfg.level, pWal->fsyncSeq, atomic_load_32(&tsWal.seq)); - int32_t code = tfFsync(pWal->writeLogTfd); + int32_t code = taosFsyncFile(pWal->pWriteLogTFile); if (code != 0) { wError("vgId:%d, file:%" PRId64 ".log, failed to fsync since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal), strerror(code)); diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index 1d9201f69d..159d281759 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -25,8 +25,8 @@ SWalReadHandle *walOpenReadHandle(SWal *pWal) { } pRead->pWal = pWal; - pRead->readIdxTfd = -1; - pRead->readLogTfd = -1; + pRead->pReadIdxTFile = NULL; + pRead->pReadLogTFile = NULL; pRead->curVersion = -1; pRead->curFileFirstVer = -1; pRead->capacity = 0; @@ -41,8 +41,8 @@ SWalReadHandle *walOpenReadHandle(SWal *pWal) { } void walCloseReadHandle(SWalReadHandle *pRead) { - tfClose(pRead->readIdxTfd); - tfClose(pRead->readLogTfd); + taosCloseFile(&pRead->pReadIdxTFile); + taosCloseFile(&pRead->pReadLogTFile); tfree(pRead->pHead); free(pRead); } @@ -52,24 +52,24 @@ int32_t walRegisterRead(SWalReadHandle *pRead, int64_t ver) { return 0; } static int32_t walReadSeekFilePos(SWalReadHandle *pRead, int64_t fileFirstVer, int64_t ver) { int code = 0; - int64_t idxTfd = pRead->readIdxTfd; - int64_t logTfd = pRead->readLogTfd; + TdFilePtr pIdxTFile = pRead->pReadIdxTFile; + TdFilePtr pLogTFile = pRead->pReadLogTFile; // seek position int64_t offset = (ver - fileFirstVer) * sizeof(SWalIdxEntry); - code = tfLseek(idxTfd, offset, SEEK_SET); + code = taosLSeekFile(pIdxTFile, offset, SEEK_SET); if (code < 0) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } SWalIdxEntry entry; - if (tfRead(idxTfd, &entry, sizeof(SWalIdxEntry)) != sizeof(SWalIdxEntry)) { + if (taosReadFile(pIdxTFile, &entry, sizeof(SWalIdxEntry)) != sizeof(SWalIdxEntry)) { terrno = TSDB_CODE_WAL_FILE_CORRUPTED; return -1; } // TODO:deserialize ASSERT(entry.ver == ver); - code = tfLseek(logTfd, entry.offset, SEEK_SET); + code = taosLSeekFile(pLogTFile, entry.offset, SEEK_SET); if (code < 0) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; @@ -80,24 +80,24 @@ static int32_t walReadSeekFilePos(SWalReadHandle *pRead, int64_t fileFirstVer, i static int32_t walReadChangeFile(SWalReadHandle *pRead, int64_t fileFirstVer) { char fnameStr[WAL_FILE_LEN]; - tfClose(pRead->readIdxTfd); - tfClose(pRead->readLogTfd); + taosCloseFile(&pRead->pReadIdxTFile); + taosCloseFile(&pRead->pReadLogTFile); walBuildLogName(pRead->pWal, fileFirstVer, fnameStr); - int64_t logTfd = tfOpenRead(fnameStr); - if (logTfd < 0) { + TdFilePtr pLogTFile = taosOpenFile(fnameStr, TD_FILE_READ); + if (pLogTFile == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } walBuildIdxName(pRead->pWal, fileFirstVer, fnameStr); - int64_t idxTfd = tfOpenRead(fnameStr); - if (idxTfd < 0) { + TdFilePtr pIdxTFile = taosOpenFile(fnameStr, TD_FILE_READ); + if (pIdxTFile == NULL) { return -1; } - pRead->readLogTfd = logTfd; - pRead->readIdxTfd = idxTfd; + pRead->pReadLogTFile = pLogTFile; + pRead->pReadIdxTFile = pIdxTFile; return 0; } @@ -145,9 +145,9 @@ int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver) { } } - if (!tfValid(pRead->readLogTfd)) return -1; + if (!taosValidFile(pRead->pReadLogTFile)) return -1; - code = tfRead(pRead->readLogTfd, pRead->pHead, sizeof(SWalHead)); + code = taosReadFile(pRead->pReadLogTFile, pRead->pHead, sizeof(SWalHead)); if (code != sizeof(SWalHead)) { return -1; } @@ -165,7 +165,7 @@ int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver) { pRead->pHead = ptr; pRead->capacity = pRead->pHead->head.len; } - if (pRead->pHead->head.len != tfRead(pRead->readLogTfd, pRead->pHead->head.body, pRead->pHead->head.len)) { + if (pRead->pHead->head.len != taosReadFile(pRead->pReadLogTFile, pRead->pHead->head.body, pRead->pHead->head.len)) { return -1; } @@ -202,7 +202,7 @@ int32_t walRead(SWal *pWal, SWalHead **ppHead, int64_t ver) { } *ppHead = ptr; } - if (tfRead(pWal->writeLogTfd, *ppHead, sizeof(SWalHead)) != sizeof(SWalHead)) { + if (tfRead(pWal->pWriteLogTFile, *ppHead, sizeof(SWalHead)) != sizeof(SWalHead)) { return -1; } // TODO: endian compatibility processing after read @@ -215,7 +215,7 @@ int32_t walRead(SWal *pWal, SWalHead **ppHead, int64_t ver) { *ppHead = NULL; return -1; } - if (tfRead(pWal->writeLogTfd, (*ppHead)->head.body, (*ppHead)->head.len) != (*ppHead)->head.len) { + if (tfRead(pWal->pWriteLogTFile, (*ppHead)->head.body, (*ppHead)->head.len) != (*ppHead)->head.len) { return -1; } // TODO: endian compatibility processing after read diff --git a/source/libs/wal/src/walSeek.c b/source/libs/wal/src/walSeek.c index 91b172444b..7aae9138c1 100644 --- a/source/libs/wal/src/walSeek.c +++ b/source/libs/wal/src/walSeek.c @@ -23,25 +23,25 @@ static int walSeekWritePos(SWal* pWal, int64_t ver) { int code = 0; - int64_t idxTfd = pWal->writeIdxTfd; - int64_t logTfd = pWal->writeLogTfd; + TdFilePtr pIdxTFile = pWal->pWriteIdxTFile; + TdFilePtr pLogTFile = pWal->pWriteLogTFile; // seek position int64_t idxOff = walGetVerIdxOffset(pWal, ver); - code = tfLseek(idxTfd, idxOff, SEEK_SET); + code = taosLSeekFile(pIdxTFile, idxOff, SEEK_SET); if (code != 0) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } SWalIdxEntry entry; // TODO:deserialize - code = tfRead(idxTfd, &entry, sizeof(SWalIdxEntry)); + code = taosReadFile(pIdxTFile, &entry, sizeof(SWalIdxEntry)); if (code != 0) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } ASSERT(entry.ver == ver); - code = tfLseek(logTfd, entry.offset, SEEK_SET); + code = taosLSeekFile(pLogTFile, entry.offset, SEEK_SET); if (code < 0) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; @@ -50,43 +50,43 @@ static int walSeekWritePos(SWal* pWal, int64_t ver) { } int walSetWrite(SWal* pWal) { - int64_t idxTfd, logTfd; + TdFilePtr pIdxTFile, pLogTFile; SWalFileInfo* pRet = taosArrayGetLast(pWal->fileInfoSet); ASSERT(pRet != NULL); int64_t fileFirstVer = pRet->firstVer; char fnameStr[WAL_FILE_LEN]; walBuildIdxName(pWal, fileFirstVer, fnameStr); - idxTfd = tfOpenCreateWriteAppend(fnameStr); - if (idxTfd < 0) { + pIdxTFile = taosOpenFile(fnameStr, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_APPEND); + if (pIdxTFile < 0) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } walBuildLogName(pWal, fileFirstVer, fnameStr); - logTfd = tfOpenCreateWriteAppend(fnameStr); - if (logTfd < 0) { + pLogTFile = taosOpenFile(fnameStr, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_APPEND); + if (pLogTFile < 0) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } // switch file - pWal->writeIdxTfd = idxTfd; - pWal->writeLogTfd = logTfd; + pWal->pWriteIdxTFile = pIdxTFile; + pWal->pWriteLogTFile = pLogTFile; return 0; } int walChangeWrite(SWal* pWal, int64_t ver) { int code = 0; - int64_t idxTfd, logTfd; + TdFilePtr pIdxTFile, pLogTFile; char fnameStr[WAL_FILE_LEN]; - if (pWal->writeLogTfd != -1) { - code = tfClose(pWal->writeLogTfd); + if (pWal->pWriteLogTFile != NULL) { + code = taosCloseFile(&pWal->pWriteLogTFile); if (code != 0) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } } - if (pWal->writeIdxTfd != -1) { - code = tfClose(pWal->writeIdxTfd); + if (pWal->pWriteIdxTFile != NULL) { + code = taosCloseFile(&pWal->pWriteIdxTFile); if (code != 0) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; @@ -103,23 +103,23 @@ int walChangeWrite(SWal* pWal, int64_t ver) { int64_t fileFirstVer = pFileInfo->firstVer; walBuildIdxName(pWal, fileFirstVer, fnameStr); - idxTfd = tfOpenCreateWriteAppend(fnameStr); - if (idxTfd < 0) { + pIdxTFile = taosOpenFile(fnameStr, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_APPEND); + if (pIdxTFile < 0) { terrno = TAOS_SYSTEM_ERROR(errno); - pWal->writeIdxTfd = -1; + pWal->pWriteIdxTFile = NULL; return -1; } walBuildLogName(pWal, fileFirstVer, fnameStr); - logTfd = tfOpenCreateWriteAppend(fnameStr); - if (logTfd < 0) { - tfClose(idxTfd); + pLogTFile = taosOpenFile(fnameStr, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_APPEND); + if (pLogTFile < 0) { + taosCloseFile(&pIdxTFile); terrno = TAOS_SYSTEM_ERROR(errno); - pWal->writeLogTfd = -1; + pWal->pWriteLogTFile = NULL; return -1; } - pWal->writeLogTfd = logTfd; - pWal->writeIdxTfd = idxTfd; + pWal->pWriteLogTFile = pLogTFile; + pWal->pWriteIdxTFile = pIdxTFile; pWal->writeCur = idx; return fileFirstVer; } diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index a4b34dee37..f9bb168234 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -65,15 +65,15 @@ int32_t walRollback(SWal *pWal, int64_t ver) { } walBuildIdxName(pWal, walGetCurFileFirstVer(pWal), fnameStr); - int64_t idxTfd = tfOpenReadWrite(fnameStr); + TdFilePtr pIdxTFile = taosOpenFile(fnameStr, TD_FILE_WRITE | TD_FILE_READ); // TODO:change to deserialize function - if (idxTfd < 0) { + if (pIdxTFile == NULL) { pthread_mutex_unlock(&pWal->mutex); return -1; } int64_t idxOff = walGetVerIdxOffset(pWal, ver); - code = tfLseek(idxTfd, idxOff, SEEK_SET); + code = taosLSeekFile(pIdxTFile, idxOff, SEEK_SET); if (code < 0) { pthread_mutex_unlock(&pWal->mutex); return -1; @@ -81,20 +81,20 @@ int32_t walRollback(SWal *pWal, int64_t ver) { // read idx file and get log file pos // TODO:change to deserialize function SWalIdxEntry entry; - if (tfRead(idxTfd, &entry, sizeof(SWalIdxEntry)) != sizeof(SWalIdxEntry)) { + if (taosReadFile(pIdxTFile, &entry, sizeof(SWalIdxEntry)) != sizeof(SWalIdxEntry)) { pthread_mutex_unlock(&pWal->mutex); return -1; } ASSERT(entry.ver == ver); walBuildLogName(pWal, walGetCurFileFirstVer(pWal), fnameStr); - int64_t logTfd = tfOpenReadWrite(fnameStr); - if (logTfd < 0) { + TdFilePtr pLogTFile = taosOpenFile(fnameStr, TD_FILE_WRITE | TD_FILE_READ); + if (pLogTFile < 0) { // TODO pthread_mutex_unlock(&pWal->mutex); return -1; } - code = tfLseek(logTfd, entry.offset, SEEK_SET); + code = taosLSeekFile(pLogTFile, entry.offset, SEEK_SET); if (code < 0) { // TODO pthread_mutex_unlock(&pWal->mutex); @@ -102,8 +102,8 @@ int32_t walRollback(SWal *pWal, int64_t ver) { } // validate offset SWalHead head; - ASSERT(tfValid(logTfd)); - int size = tfRead(logTfd, &head, sizeof(SWalHead)); + ASSERT(taosValidFile(pLogTFile)); + int size = taosReadFile(pLogTFile, &head, sizeof(SWalHead)); if (size != sizeof(SWalHead)) { return -1; } @@ -118,11 +118,11 @@ int32_t walRollback(SWal *pWal, int64_t ver) { return -1; } // truncate old files - code = tfFtruncate(logTfd, entry.offset); + code = taosFtruncateFile(pLogTFile, entry.offset); if (code < 0) { return -1; } - code = tfFtruncate(idxTfd, idxOff); + code = taosFtruncateFile(pIdxTFile, idxOff); if (code < 0) { return -1; } @@ -203,31 +203,31 @@ int32_t walEndSnapshot(SWal *pWal) { int walRoll(SWal *pWal) { int code = 0; - if (pWal->writeIdxTfd != -1) { - code = tfClose(pWal->writeIdxTfd); + if (pWal->pWriteIdxTFile != NULL) { + code = taosCloseFile(&pWal->pWriteIdxTFile); if (code != 0) { return -1; } } - if (pWal->writeLogTfd != -1) { - code = tfClose(pWal->writeLogTfd); + if (pWal->pWriteLogTFile != NULL) { + code = taosCloseFile(&pWal->pWriteLogTFile); if (code != 0) { return -1; } } - int64_t idxTfd, logTfd; + TdFilePtr pIdxTFile, pLogTFile; // create new file int64_t newFileFirstVersion = pWal->vers.lastVer + 1; char fnameStr[WAL_FILE_LEN]; walBuildIdxName(pWal, newFileFirstVersion, fnameStr); - idxTfd = tfOpenCreateWriteAppend(fnameStr); - if (idxTfd < 0) { + pIdxTFile = taosOpenFile(fnameStr, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_APPEND); + if (pIdxTFile < 0) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } walBuildLogName(pWal, newFileFirstVersion, fnameStr); - logTfd = tfOpenCreateWriteAppend(fnameStr); - if (logTfd < 0) { + pLogTFile = taosOpenFile(fnameStr, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_APPEND); + if (pLogTFile < 0) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } @@ -237,8 +237,8 @@ int walRoll(SWal *pWal) { } // switch file - pWal->writeIdxTfd = idxTfd; - pWal->writeLogTfd = logTfd; + pWal->pWriteIdxTFile = pIdxTFile; + pWal->pWriteLogTFile = pLogTFile; pWal->writeCur = taosArrayGetSize(pWal->fileInfoSet) - 1; ASSERT(pWal->writeCur >= 0); @@ -248,7 +248,7 @@ int walRoll(SWal *pWal) { static int walWriteIndex(SWal *pWal, int64_t ver, int64_t offset) { SWalIdxEntry entry = {.ver = ver, .offset = offset}; - int size = tfWrite(pWal->writeIdxTfd, &entry, sizeof(SWalIdxEntry)); + int size = taosWriteFile(pWal->pWriteIdxTFile, &entry, sizeof(SWalIdxEntry)); if (size != sizeof(SWalIdxEntry)) { terrno = TAOS_SYSTEM_ERROR(errno); // TODO truncate @@ -282,16 +282,16 @@ int64_t walWrite(SWal *pWal, int64_t index, tmsg_t msgType, const void *body, in // must truncate explicitly first return -1; } - /*if (!tfValid(pWal->writeLogTfd)) return -1;*/ + /*if (!tfValid(pWal->pWriteLogTFile)) return -1;*/ ASSERT(pWal->writeCur >= 0); pthread_mutex_lock(&pWal->mutex); - if (pWal->writeIdxTfd == -1 || pWal->writeLogTfd == -1) { + if (pWal->pWriteIdxTFile == NULL || pWal->pWriteLogTFile == NULL) { walSetWrite(pWal); - tfLseek(pWal->writeLogTfd, 0, SEEK_END); - tfLseek(pWal->writeIdxTfd, 0, SEEK_END); + taosLSeekFile(pWal->pWriteLogTFile, 0, SEEK_END); + taosLSeekFile(pWal->pWriteIdxTFile, 0, SEEK_END); } pWal->writeHead.head.version = index; @@ -302,14 +302,14 @@ int64_t walWrite(SWal *pWal, int64_t index, tmsg_t msgType, const void *body, in pWal->writeHead.cksumHead = walCalcHeadCksum(&pWal->writeHead); pWal->writeHead.cksumBody = walCalcBodyCksum(body, bodyLen); - if (tfWrite(pWal->writeLogTfd, &pWal->writeHead, sizeof(SWalHead)) != sizeof(SWalHead)) { + if (taosWriteFile(pWal->pWriteLogTFile, &pWal->writeHead, sizeof(SWalHead)) != sizeof(SWalHead)) { // ftruncate code = TAOS_SYSTEM_ERROR(errno); wError("vgId:%d, file:%" PRId64 ".log, failed to write since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal), strerror(errno)); } - if (tfWrite(pWal->writeLogTfd, (char *)body, bodyLen) != bodyLen) { + if (taosWriteFile(pWal->pWriteLogTFile, (char *)body, bodyLen) != bodyLen) { // ftruncate code = TAOS_SYSTEM_ERROR(errno); wError("vgId:%d, file:%" PRId64 ".log, failed to write since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal), @@ -336,7 +336,7 @@ int64_t walWrite(SWal *pWal, int64_t index, tmsg_t msgType, const void *body, in void walFsync(SWal *pWal, bool forceFsync) { if (forceFsync || (pWal->cfg.level == TAOS_WAL_FSYNC && pWal->cfg.fsyncPeriod == 0)) { wTrace("vgId:%d, fileId:%" PRId64 ".log, do fsync", pWal->cfg.vgId, walGetCurFileFirstVer(pWal)); - if (tfFsync(pWal->writeLogTfd) < 0) { + if (taosFsyncFile(pWal->pWriteLogTFile) < 0) { wError("vgId:%d, file:%" PRId64 ".log, fsync failed since %s", pWal->cfg.vgId, walGetCurFileFirstVer(pWal), strerror(errno)); } diff --git a/source/os/src/osDir.c b/source/os/src/osDir.c index 8fee6a6f7d..c464073e5f 100644 --- a/source/os/src/osDir.c +++ b/source/os/src/osDir.c @@ -71,7 +71,7 @@ int32_t taosMkDir(const char *dirname) { return code; } -void taosRemoveOldFiles(char *dirname, int32_t keepDays) { +void taosRemoveOldFiles(const char *dirname, int32_t keepDays) { DIR *dir = opendir(dirname); if (dir == NULL) return; diff --git a/source/os/src/osEnv.c b/source/os/src/osEnv.c index b90d9fbe94..4c368fe895 100644 --- a/source/os/src/osEnv.c +++ b/source/os/src/osEnv.c @@ -13,75 +13,89 @@ * along with this program. If not, see . */ - #define _DEFAULT_SOURCE -#include "os.h" #include "osEnv.h" -#include "osSysinfo.h" -SDiskSpace tsLogSpace; -SDiskSpace tsTempSpace; -SDiskSpace tsDataSpace; +extern void taosWinSocketInit(); -void taosUpdateLogSpace() { taosGetDiskSize(tsLogDir, &tsLogSpace.size); } - -void taosUpdateTempSpace() { taosGetDiskSize(tsTempDir, &tsTempSpace.size); } - -void taosUpdateDataSpace() { taosGetDiskSize(tsDataDir, &tsDataSpace.size); } - -bool taosLogSpaceAvailable() { return tsLogSpace.reserved < tsLogSpace.size.avail; } - -bool taosTempSpaceAvailable() { return tsTempSpace.reserved < tsTempSpace.size.avail; } - -bool taosDataSpaceAvailable() { return tsDataSpace.reserved < tsDataSpace.size.avail; } - -void taosUpdateAllSpace() { - taosUpdateLogSpace(); - taosUpdateTempSpace(); - taosUpdateDataSpace(); -} - -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - -char tsOsName[10] = "Windows"; -char configDir[PATH_MAX] = "C:/TDengine/cfg"; -char tsDataDir[PATH_MAX] = "C:/TDengine/data"; -char tsLogDir[PATH_MAX] = "C:/TDengine/log"; -char tsTempDir[PATH_MAX] = "C:\\Windows\\Temp"; - -extern taosWinSocketInit(); +char configDir[PATH_MAX] = {0}; +char tsDataDir[PATH_MAX] = {0}; +char tsLogDir[PATH_MAX] = {0}; +char tsTempDir[PATH_MAX] = {0}; +SDiskSpace tsDataSpace = {0}; +SDiskSpace tsLogSpace = {0}; +SDiskSpace tsTempSpace = {0}; +char tsOsName[16] = {0}; +char tsTimezone[TD_TIMEZONE_LEN] = {0}; +char tsLocale[TD_LOCALE_LEN] = {0}; +char tsCharset[TD_CHARSET_LEN] = {0}; +int8_t tsDaylight = 0; +bool tsEnableCoreFile = 0; +int64_t tsPageSize = 0; +int64_t tsOpenMax = 0; +int64_t tsStreamMax = 0; +int32_t tsNumOfCores = 0; +int32_t tsTotalMemoryMB = 0; void osInit() { + srand(taosSafeRand()); + taosGetSystemLocale(tsLocale, tsCharset); + taosGetSystemTimezone(tsTimezone); + taosSetSystemTimezone(tsTimezone, tsTimezone, &tsDaylight); + taosGetSystemInfo(); + +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) taosWinSocketInit(); const char *tmpDir = getenv("tmp"); if (tmpDir == NULL) { tmpDir = getenv("temp"); } - if (tmpDir != NULL) { strcpy(tsTempDir, tmpDir); } -} + + if (configDir[0] == 0) { + strcpy(configDir, "C:\\TDengine\\cfg"); + } + strcpy(tsDataDir, "C:\\TDengine\\data"); + strcpy(tsLogDir, "C:\\TDengine\\log"); + strcpy(tsTempDir, "C:\\Windows\\Temp"); + strcpy(tsOsName, "Windows"); #elif defined(_TD_DARWIN_64) - -char tsOsName[10] = "Darwin"; -char configDir[PATH_MAX] = "/usr/local/etc/taos"; -char tsDataDir[PATH_MAX] = "/usr/local/var/lib/taos"; -char tsLogDir[PATH_MAX] = "/usr/local/var/log/taos"; -char tsTempDir[PATH_MAX] = "/tmp/taosd"; - -void osInit() {} + if (configDir[0] == 0) { + strcpy(configDir, "/tmp/taosd"); + } + strcpy(tsDataDir, "/usr/local/var/lib/taos"); + strcpy(tsLogDir, "/usr/local/var/log/taos"); + strcpy(tsTempDir, "/usr/local/etc/taos"); + strcpy(tsOsName, "Darwin"); #else - -char tsOsName[10] = "Linux"; -char configDir[PATH_MAX] = "/etc/taos"; -char tsDataDir[PATH_MAX] = "/var/lib/taos"; -char tsLogDir[PATH_MAX] = "/var/log/taos"; -char tsTempDir[PATH_MAX] = "/tmp/"; - -void osInit() {} + if (configDir[0] == 0) { + strcpy(configDir, "/etc/taos"); + } + strcpy(tsDataDir, "/var/lib/taos"); + strcpy(tsLogDir, "/var/log/taos"); + strcpy(tsTempDir, "/tmp"); + strcpy(tsOsName, "Linux"); #endif +} + +void osUpdate() { + if (tsLogDir[0] != 0) { + taosGetDiskSize(tsLogDir, &tsLogSpace.size); + } + if (tsDataDir[0] != 0) { + taosGetDiskSize(tsDataDir, &tsDataSpace.size); + } + if (tsTempDir[0] != 0) { + taosGetDiskSize(tsTempDir, &tsTempSpace.size); + } +} + +bool osLogSpaceAvailable() { return tsLogSpace.reserved <= tsLogSpace.size.avail; } + +void osSetTimezone(const char *timezone) { taosSetSystemTimezone(tsTimezone, tsTimezone, &tsDaylight); } diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index 5a67ab73eb..238bc6e372 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -12,38 +12,45 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ - +#define ALLOW_FORBID_FUNC #include "os.h" #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) -#include + #include -#if defined(_MSDOS) -#define open _open -#endif + #if defined(_MSDOS) + #define open _open + #endif -#if defined(_WIN32) -extern int openA(const char *, int, ...); /* MsvcLibX ANSI version of open */ -extern int openU(const char *, int, ...); /* MsvcLibX UTF-8 version of open */ -#if defined(_UTF8_SOURCE) || defined(_BSD_SOURCE) || defined(_GNU_SOURCE) -#define open openU -#else /* _ANSI_SOURCE */ -#define open openA -#endif /* defined(_UTF8_SOURCE) */ -#endif /* defined(_WIN32) */ + #if defined(_WIN32) + extern int openA(const char *, int, ...); /* MsvcLibX ANSI version of open */ + extern int openU(const char *, int, ...); /* MsvcLibX UTF-8 version of open */ + #if defined(_UTF8_SOURCE) || defined(_BSD_SOURCE) || defined(_GNU_SOURCE) + #define open openU + #else /* _ANSI_SOURCE */ + #define open openA + #endif /* defined(_UTF8_SOURCE) */ + #endif /* defined(_WIN32) */ #else -#include -#include -#include -#include -#include + #include + #include + #include + #include + #include + #define LINUX_FILE_NO_TEXT_OPTION 0 + #define O_TEXT LINUX_FILE_NO_TEXT_OPTION #endif -void taosCloseFile(FileFd fd) { - close(fd); - fd = FD_INITIALIZER; -} +typedef int32_t FileFd; + +typedef struct TdFile { + int refId; + FileFd fd; + FILE *fp; +}*TdFilePtr,TdFile; + + void taosGetTmpfilePath(const char * inputTmpDir, const char *fileNamePrefix, char *dstPath) { #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) @@ -96,13 +103,166 @@ void taosGetTmpfilePath(const char * inputTmpDir, const char *fileNamePrefix, ch #endif } -int64_t taosReadFile(FileFd fd, void *buf, int64_t count) { +int64_t taosCopyFile(const char *from, const char *to) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return 0; +#else + char buffer[4096]; + int64_t size = 0; + int64_t bytes; + + // fidfrom = open(from, O_RDONLY); + TdFilePtr pFileFrom = taosOpenFile(from,TD_FILE_READ); + if (pFileFrom == NULL) goto _err; + + // fidto = open(to, O_WRONLY | O_CREAT | O_EXCL, 0755); + TdFilePtr pFileTo = taosOpenFile(to,TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_EXCL); + if (pFileTo == NULL) goto _err; + + while (true) { + bytes = taosReadFile(pFileFrom, buffer, sizeof(buffer)); + if (bytes < 0) goto _err; + if (bytes == 0) break; + + size += bytes; + + if (taosWriteFile(pFileTo, (void *)buffer, bytes) < bytes) goto _err; + if (bytes < sizeof(buffer)) break; + } + + taosFsyncFile(pFileTo); + + taosCloseFile(&pFileFrom); + taosCloseFile(&pFileTo); + return size; + +_err: + if (pFileFrom != NULL) taosCloseFile(&pFileFrom); + if (pFileTo != NULL) taosCloseFile(&pFileTo); + remove(to); + return -1; +#endif +} + +int32_t taosRenameFile(const char *oldName, const char *newName) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + int32_t code = MoveFileEx(oldName, newName, MOVEFILE_REPLACE_EXISTING | MOVEFILE_COPY_ALLOWED); + if (code < 0) { + //printf("failed to rename file %s to %s, reason:%s", oldName, newName, strerror(errno)); + } + + return code; +#else + int32_t code = rename(oldName, newName); + if (code < 0) { + //printf("failed to rename file %s to %s, reason:%s", oldName, newName, strerror(errno)); + } + + return code; +#endif +} + +int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return 0; +#else + struct stat fileStat; + int32_t code = stat(path, &fileStat); + if (code < 0) { + return code; + } + + if (size != NULL) { + *size = fileStat.st_size; + } + + if (mtime != NULL) { + *mtime = fileStat.st_mtime; + } + + return 0; +#endif +} + +void autoDelFileListAdd(const char *path) { + return; +} + +TdFilePtr taosOpenFile(const char *path,int32_t tdFileOptions) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return NULL; +#else + int access = O_BINARY; + char *mode = NULL; + access |= (tdFileOptions & TD_FILE_CTEATE) ? O_CREAT : 0; + if ((tdFileOptions & TD_FILE_WRITE) && (tdFileOptions & TD_FILE_READ)) + { + access |= O_RDWR; + mode = (tdFileOptions & TD_FILE_TEXT) ? "rt+" : "rb+"; + }else if(tdFileOptions & TD_FILE_WRITE) { + access |= O_WRONLY; + mode = (tdFileOptions & TD_FILE_TEXT) ? "wt" : "wb"; + }else if(tdFileOptions & TD_FILE_READ) { + access |= O_RDONLY; + mode = (tdFileOptions & TD_FILE_TEXT) ? "rt" : "rb"; + } + access |= (tdFileOptions & TD_FILE_TRUNC) ? O_TRUNC : 0; + access |= (tdFileOptions & TD_FILE_APPEND) ? O_APPEND : 0; + access |= (tdFileOptions & TD_FILE_TEXT) ? O_TEXT : 0; + access |= (tdFileOptions & TD_FILE_EXCL) ? O_EXCL : 0; + if(tdFileOptions & TD_FILE_AUTO_DEL) { + autoDelFileListAdd(path); + } + int fd = open(path, access, S_IRWXU | S_IRWXG | S_IRWXO); + if(fd == -1) { + return NULL; + } + FILE* fp = fdopen(fd, mode); + if (fp == NULL) { + close(fd); + return NULL; + } + TdFilePtr pFile = (TdFilePtr)malloc(sizeof(TdFile)); + if (pFile == NULL) { + close(fd); + fclose(fp); + return NULL; + } + pFile->fd = fd; + pFile->fp = fp; + pFile->refId = 0; + return pFile; +#endif +} + +int64_t taosCloseFile(TdFilePtr *ppFile) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return 0; +#else + if(ppFile == NULL || *ppFile == NULL || (*ppFile)->fd == -1) { + return 0; + } + fsync((*ppFile)->fd); + close((*ppFile)->fd); + (*ppFile)->fd = -1; + (*ppFile)->fp = NULL; + (*ppFile)->refId = 0; + free(*ppFile); + *ppFile = NULL; + return 0; +#endif +} + +int64_t taosReadFile(TdFilePtr pFile, void *buf, int64_t count) { + if(pFile == NULL) { + return 0; + } int64_t leftbytes = count; int64_t readbytes; char * tbuf = (char *)buf; while (leftbytes > 0) { - readbytes = read(fd, (void *)tbuf, (uint32_t)leftbytes); + readbytes = read(pFile->fd, (void *)tbuf, (uint32_t)leftbytes); if (readbytes < 0) { if (errno == EINTR) { continue; @@ -120,13 +280,20 @@ int64_t taosReadFile(FileFd fd, void *buf, int64_t count) { return count; } -int64_t taosWriteFile(FileFd fd, const void *buf, int64_t n) { - int64_t nleft = n; +int64_t taosPReadFile(TdFilePtr pFile, void *buf, int64_t count, int64_t offset) { + if(pFile == NULL) { + return 0; + } + return pread(pFile->fd, buf, count, offset); +} + +int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count) { + int64_t nleft = count; int64_t nwritten = 0; char * tbuf = (char *)buf; while (nleft > 0) { - nwritten = write(fd, (void *)tbuf, (uint32_t)nleft); + nwritten = write(pFile->fd, (void *)tbuf, (uint32_t)nleft); if (nwritten < 0) { if (errno == EINTR) { continue; @@ -137,48 +304,120 @@ int64_t taosWriteFile(FileFd fd, const void *buf, int64_t n) { tbuf += nwritten; } - return n; + return count; } -int64_t taosLSeekFile(FileFd fd, int64_t offset, int32_t whence) { return (int64_t)lseek(fd, (long)offset, whence); } +int64_t taosLSeekFile(TdFilePtr pFile, int64_t offset, int32_t whence) { + if (pFile == NULL) return -1; + return (int64_t)lseek(pFile->fd, (long)offset, whence); +} -int64_t taosCopyFile(const char *from, const char *to) { +int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime) { #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) return 0; #else - char buffer[4096]; - int fidto = -1, fidfrom = -1; - int64_t size = 0; - int64_t bytes; - - fidfrom = open(from, O_RDONLY); - if (fidfrom < 0) goto _err; - - fidto = open(to, O_WRONLY | O_CREAT | O_EXCL, 0755); - if (fidto < 0) goto _err; - - while (true) { - bytes = taosReadFile(fidfrom, buffer, sizeof(buffer)); - if (bytes < 0) goto _err; - if (bytes == 0) break; - - size += bytes; - - if (taosWriteFile(fidto, (void *)buffer, bytes) < bytes) goto _err; - if (bytes < sizeof(buffer)) break; + struct stat fileStat; + int32_t code = fstat(pFile->fd, &fileStat); + if (code < 0) { + return code; } - taosFsyncFile(fidto); + if (size != NULL) { + *size = fileStat.st_size; + } - taosCloseFile(fidfrom); - taosCloseFile(fidto); - return size; + if (mtime != NULL) { + *mtime = fileStat.st_mtime; + } -_err: - if (fidfrom >= 0) taosCloseFile(fidfrom); - if (fidto >= 0) taosCloseFile(fidto); - remove(to); - return -1; + return 0; +#endif +} + +int32_t taosLockFile(TdFilePtr pFile) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return 0; +#else + return (int32_t)flock(pFile->fd, LOCK_EX | LOCK_NB); +#endif +} + +int32_t taosUnLockFile(TdFilePtr pFile) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return 0; +#else + return (int32_t)flock(pFile->fd, LOCK_UN | LOCK_NB); +#endif +} + +int32_t taosFtruncateFile(TdFilePtr pFile, int64_t l_size) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + if (pFile->fd < 0) { + errno = EBADF; + uError("%s\n", "fd arg was negative"); + return -1; + } + + HANDLE h = (HANDLE)_get_osfhandle(pFile->fd); + + LARGE_INTEGER li_0; + li_0.QuadPart = (int64_t)0; + BOOL cur = SetFilePointerEx(h, li_0, NULL, FILE_CURRENT); + if (!cur) { + uError("SetFilePointerEx Error getting current position in file.\n"); + return -1; + } + + LARGE_INTEGER li_size; + li_size.QuadPart = l_size; + BOOL cur2 = SetFilePointerEx(h, li_size, NULL, FILE_BEGIN); + if (cur2 == 0) { + int error = GetLastError(); + uError("SetFilePointerEx GetLastError is: %d\n", error); + switch (error) { + case ERROR_INVALID_HANDLE: + errno = EBADF; + break; + default: + errno = EIO; + break; + } + return -1; + } + + if (!SetEndOfFile(h)) { + int error = GetLastError(); + uError("SetEndOfFile GetLastError is:%d", error); + switch (error) { + case ERROR_INVALID_HANDLE: + errno = EBADF; + break; + default: + errno = EIO; + break; + } + return -1; + } + + return 0; +#else + return ftruncate(pFile->fd, l_size); +#endif +} + +int32_t taosFsyncFile(TdFilePtr pFile) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + if (pFile->fd < 0) { + errno = EBADF; + uError("%s\n", "fd arg was negative"); + return -1; + } + + HANDLE h = (HANDLE)_get_osfhandle(pFile->fd); + + return FlushFileBuffers(h); +#else + return fsync(pFile->fd); #endif } @@ -301,12 +540,12 @@ int64_t taosSendFile(SocketFd dfd, FileFd sfd, int64_t *offset, int64_t count) { #else -int64_t taosSendFile(SocketFd dfd, FileFd sfd, int64_t *offset, int64_t size) { +int64_t taosSendFile(SocketFd fdDst, TdFilePtr pFileSrc, int64_t *offset, int64_t size) { int64_t leftbytes = size; int64_t sentbytes; while (leftbytes > 0) { - sentbytes = sendfile(dfd, sfd, offset, leftbytes); + sentbytes = sendfile(fdDst, pFileSrc->fd, offset, leftbytes); if (sentbytes == -1) { if (errno == EINTR || errno == EAGAIN || errno == EWOULDBLOCK) { continue; @@ -323,215 +562,50 @@ int64_t taosSendFile(SocketFd dfd, FileFd sfd, int64_t *offset, int64_t size) { return size; } -int64_t taosFSendFile(FILE *outfile, FILE *infile, int64_t *offset, int64_t size) { - return taosSendFile(fileno(outfile), fileno(infile), offset, size); +int64_t taosFSendFile(TdFilePtr pFileOut, TdFilePtr pFileIn, int64_t *offset, int64_t size) { + return taosSendFile(pFileOut->fd, pFileIn, offset, size); } #endif -int32_t taosFtruncateFile(FileFd fd, int64_t l_size) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - if (fd < 0) { - errno = EBADF; - uError("%s\n", "fd arg was negative"); - return -1; - } - HANDLE h = (HANDLE)_get_osfhandle(fd); - - LARGE_INTEGER li_0; - li_0.QuadPart = (int64_t)0; - BOOL cur = SetFilePointerEx(h, li_0, NULL, FILE_CURRENT); - if (!cur) { - uError("SetFilePointerEx Error getting current position in file.\n"); - return -1; - } - - LARGE_INTEGER li_size; - li_size.QuadPart = l_size; - BOOL cur2 = SetFilePointerEx(h, li_size, NULL, FILE_BEGIN); - if (cur2 == 0) { - int error = GetLastError(); - uError("SetFilePointerEx GetLastError is: %d\n", error); - switch (error) { - case ERROR_INVALID_HANDLE: - errno = EBADF; - break; - default: - errno = EIO; - break; - } - return -1; - } - - if (!SetEndOfFile(h)) { - int error = GetLastError(); - uError("SetEndOfFile GetLastError is:%d", error); - switch (error) { - case ERROR_INVALID_HANDLE: - errno = EBADF; - break; - default: - errno = EIO; - break; - } - return -1; - } - - return 0; -#else - return ftruncate(fd, l_size); +#ifdef __GNUC__ + __attribute__((format(printf, 2, 3))) #endif +void taosFprintfFile(TdFilePtr pFile, const char *format, ...) { + va_list ap; + va_start(ap, format); + fprintf(pFile->fp, format, ap); + va_end(ap); + fflush(pFile->fp); } -int32_t taosFsyncFile(FileFd fd) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - if (fd < 0) { - errno = EBADF; - uError("%s\n", "fd arg was negative"); - return -1; - } +void *taosMmapReadOnlyFile(TdFilePtr pFile, int64_t length) { + if (pFile == NULL) return NULL; - HANDLE h = (HANDLE)_get_osfhandle(fd); - - return FlushFileBuffers(h); -#else - return fsync(fd); -#endif + void *ptr = mmap(NULL, length, PROT_READ, MAP_SHARED, pFile->fd, 0); + return ptr; } -int32_t taosRenameFile(const char *oldName, const char *newName) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - int32_t code = MoveFileEx(oldName, newName, MOVEFILE_REPLACE_EXISTING | MOVEFILE_COPY_ALLOWED); - if (code < 0) { - //printf("failed to rename file %s to %s, reason:%s", oldName, newName, strerror(errno)); - } - - return code; -#else - int32_t code = rename(oldName, newName); - if (code < 0) { - //printf("failed to rename file %s to %s, reason:%s", oldName, newName, strerror(errno)); - } - - return code; -#endif +bool taosValidFile(TdFilePtr pFile) { + return pFile != NULL; } -int32_t taosLockFile(int32_t fd) { +int32_t taosUmaskFile(int32_t maskVal) { #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) return 0; #else - return (int32_t)flock(fd, LOCK_EX | LOCK_NB); + return umask(maskVal); #endif } -int32_t taosUnLockFile(int32_t fd) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - return 0; -#else - return (int32_t)flock(fd, LOCK_UN | LOCK_NB); -#endif +int taosGetErrorFile(TdFilePtr pFile) { + return errno; } - -int32_t taosUmaskFile(int32_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - return 0; -#else - return umask(val); -#endif +size_t taosGetLineFile(TdFilePtr pFile, char ** __restrict__ ptrBuf) { + size_t len = 0; + return getline(ptrBuf, &len, pFile->fp); } - -int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - return 0; -#else - struct stat fileStat; - int32_t code = stat(path, &fileStat); - if (code < 0) { - return code; - } - - if (size != NULL) { - *size = fileStat.st_size; - } - - if (mtime != NULL) { - *mtime = fileStat.st_mtime; - } - - return 0; -#endif -} - -int32_t taosFStatFile(int32_t fd, int64_t *size, int32_t *mtime) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - return 0; -#else - struct stat fileStat; - int32_t code = fstat(fd, &fileStat); - if (code < 0) { - return code; - } - - if (size != NULL) { - *size = fileStat.st_size; - } - - if (mtime != NULL) { - *mtime = fileStat.st_mtime; - } - - return 0; -#endif -} - -int32_t taosOpenFileWrite(const char *path) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - return 0; -#else - return open(path, O_WRONLY, S_IRWXU | S_IRWXG | S_IRWXO); -#endif -} - -int32_t taosOpenFileCreateWrite(const char *path) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - return 0; -#else - return open(path, O_WRONLY | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO); -#endif -} - -int32_t taosOpenFileCreateWriteTrunc(const char *path) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - return 0; -#else - return open(path, O_WRONLY | O_CREAT | O_TRUNC, S_IRWXU | S_IRWXG | S_IRWXO); -#endif -} - -int32_t taosOpenFileCreateWriteAppend(const char *path) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - return 0; -#else - return open(path, O_WRONLY | O_CREAT | O_APPEND, S_IRWXU | S_IRWXG | S_IRWXO); -#endif -} - -FileFd taosOpenFileRead(const char *path) { - #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - return 0; -#else - return open(path, O_RDONLY, S_IRWXU | S_IRWXG | S_IRWXO); -#endif -} - -FileFd taosOpenFileReadWrite(const char *path) { - #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - return 0; -#else - return open(path, O_RDWR, S_IRWXU | S_IRWXG | S_IRWXO); -#endif -} - - +int32_t taosEOFFile(TdFilePtr pFile) { + return feof(pFile->fp); +} \ No newline at end of file diff --git a/source/os/src/osLocale.c b/source/os/src/osLocale.c new file mode 100644 index 0000000000..47546f7deb --- /dev/null +++ b/source/os/src/osLocale.c @@ -0,0 +1,189 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "osLocale.h" + +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#if (_WIN64) +#include +#include +#include +#include +#include +#include +#pragma comment(lib, "Mswsock.lib ") +#endif +#include +#pragma warning(push) +#pragma warning(disable : 4091) +#include +#pragma warning(pop) +#elif defined(_TD_DARWIN_64) +#include +#include +#else +#include +#include +#include +#include +#include +#include +#include +#include +#endif + +typedef struct CharsetPair { + char *oldCharset; + char *newCharset; +} CharsetPair; + +char *taosCharsetReplace(char *charsetstr) { + CharsetPair charsetRep[] = { + {"utf8", "UTF-8"}, + {"936", "CP936"}, + }; + + for (int32_t i = 0; i < tListLen(charsetRep); ++i) { + if (strcasecmp(charsetRep[i].oldCharset, charsetstr) == 0) { + return strdup(charsetRep[i].newCharset); + } + } + + return strdup(charsetstr); +} + +/** + * In some Linux systems, setLocale(LC_CTYPE, "") may return NULL, in which case the launch of + * both the TDengine Server and the Client may be interrupted. + * + * In case that the setLocale failed to be executed, the right charset needs to be set. + */ +void taosSetSystemLocale(const char *inLocale, const char *inCharSet) { + char *locale = setlocale(LC_CTYPE, inLocale); + + // default locale or user specified locale is not valid, abort launch + if (inLocale == NULL || strlen(inLocale) == 0) { + //printf("Invalid locale:%s, please set the valid locale in config file\n", inLocale); + } + + if (!taosValidateEncodec(inCharSet)) { + printf("Invalid charset:%s, please set the valid charset in config file", inCharSet); + exit(-1); + } +} + +void taosGetSystemLocale(char *outLocale, char *outCharset) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + char *locale = setlocale(LC_CTYPE, "chs"); + if (locale != NULL) { + tstrncpy(outLocale, locale, TD_LOCALE_LEN); + } + strcpy(outCharset, "cp936"); + +#elif defined(_TD_DARWIN_64) + /* + * originally from src/os/src/detail/osSysinfo.c + * POSIX format locale string: + * (Language Strings)_(Country/Region Strings).(code_page) + * + * example: en_US.UTF-8, zh_CN.GB18030, zh_CN.UTF-8, + * + * if user does not specify the locale in taos.cfg the program use default LC_CTYPE as system locale. + * + * In case of some CentOS systems, their default locale is "en_US.utf8", which is not valid code_page + * for libiconv that is employed to convert string in this system. This program will automatically use + * UTF-8 instead as the charset. + * + * In case of windows client, the locale string is not valid POSIX format, user needs to set the + * correct code_page for libiconv. Usually, the code_page of windows system with simple chinese is + * CP936, CP437 for English charset. + * + */ + + char sep = '.'; + char *locale = NULL; + + locale = setlocale(LC_CTYPE, ""); + if (locale == NULL) { + // printf("can't get locale from system, set it to en_US.UTF-8 since error:%d:%s", errno, strerror(errno)); + strcpy(outLocale, "en_US.UTF-8"); + } else { + tstrncpy(outLocale, locale, TD_LOCALE_LEN); + // printf("locale not configured, set to system default:%s", outLocale); + } + + /* if user does not specify the charset, extract it from locale */ + char *str = strrchr(outLocale, sep); + if (str != NULL) { + str++; + + char *revisedCharset = taosCharsetReplace(str); + tstrncpy(outCharset, revisedCharset, TD_CHARSET_LEN); + + free(revisedCharset); + // printf("charset not configured, set to system default:%s", outCharset); + } else { + strcpy(outCharset, "UTF-8"); + // printf("can't get locale and charset from system, set it to UTF-8"); + } + +#else + /* + * POSIX format locale string: + * (Language Strings)_(Country/Region Strings).(code_page) + * + * example: en_US.UTF-8, zh_CN.GB18030, zh_CN.UTF-8, + * + * if user does not specify the locale in taos.cfg the program use default LC_CTYPE as system locale. + * + * In case of some CentOS systems, their default locale is "en_US.utf8", which is not valid code_page + * for libiconv that is employed to convert string in this system. This program will automatically use + * UTF-8 instead as the charset. + * + * In case of windows client, the locale string is not valid POSIX format, user needs to set the + * correct code_page for libiconv. Usually, the code_page of windows system with simple chinese is + * CP936, CP437 for English charset. + * + */ + char sep = '.'; + char *locale = NULL; + + locale = setlocale(LC_CTYPE, ""); + if (locale == NULL) { + // printf("can't get locale from system, set it to en_US.UTF-8 since error:%d:%s", errno, strerror(errno)); + strcpy(outLocale, "en_US.UTF-8"); + } else { + tstrncpy(outLocale, locale, TD_LOCALE_LEN); + // printf("locale not configured, set to system default:%s", outLocale); + } + + // if user does not specify the charset, extract it from locale + char *str = strrchr(outLocale, sep); + if (str != NULL) { + str++; + + char *revisedCharset = taosCharsetReplace(str); + tstrncpy(outCharset, revisedCharset, TD_LOCALE_LEN); + + free(revisedCharset); + // printf("charset not configured, set to system default:%s", outCharset); + } else { + strcpy(outCharset, "UTF-8"); + // printf("can't get locale and charset from system, set it to UTF-8"); + } + +#endif +} diff --git a/source/os/src/osRand.c b/source/os/src/osRand.c index 973323a346..b81e41b3cf 100644 --- a/source/os/src/osRand.c +++ b/source/os/src/osRand.c @@ -24,18 +24,18 @@ uint32_t taosRand(void) { return rand(); } uint32_t taosSafeRand(void) { - int fd; + TdFilePtr pFile; int seed; - fd = open("/dev/urandom", 0); - if (fd < 0) { + pFile = taosOpenFile("/dev/urandom", TD_FILE_READ); + if (pFile == NULL) { seed = (int)time(0); } else { - int len = read(fd, &seed, sizeof(seed)); + int len = taosReadFile(pFile, &seed, sizeof(seed)); if (len < 0) { seed = (int)time(0); } - close(fd); + taosCloseFile(&pFile); } return (uint32_t)seed; diff --git a/source/os/src/osSocket.c b/source/os/src/osSocket.c index 8afb4ae8b7..07d30276b7 100644 --- a/source/os/src/osSocket.c +++ b/source/os/src/osSocket.c @@ -14,6 +14,7 @@ */ #define _DEFAULT_SOURCE +#define ALLOW_FORBID_FUNC #include "os.h" #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) diff --git a/source/os/src/osString.c b/source/os/src/osString.c index f714fd0dc4..88ea4b3e15 100644 --- a/source/os/src/osString.c +++ b/source/os/src/osString.c @@ -18,24 +18,6 @@ #include #include -typedef struct CharsetPair { - char *oldCharset; - char *newCharset; -} CharsetPair; - -char *taosCharsetReplace(char *charsetstr) { - CharsetPair charsetRep[] = { - { "utf8", "UTF-8" }, { "936", "CP936" }, - }; - - for (int32_t i = 0; i < tListLen(charsetRep); ++i) { - if (strcasecmp(charsetRep[i].oldCharset, charsetstr) == 0) { - return strdup(charsetRep[i].newCharset); - } - } - - return strdup(charsetstr); -} int64_t taosStr2int64(const char *str) { char *endptr = NULL; diff --git a/source/os/src/osSysinfo.c b/source/os/src/osSysinfo.c index cae1b18b3c..45749588c1 100644 --- a/source/os/src/osSysinfo.c +++ b/source/os/src/osSysinfo.c @@ -16,15 +16,6 @@ #define _DEFAULT_SOURCE #include "os.h" -int32_t tsTotalMemoryMB = 0; -int64_t tsPageSize = 0; -int64_t tsOpenMax = 0; -int64_t tsStreamMax = 0; -int32_t tsNumOfCores = 1; -char tsTimezone[TSDB_TIMEZONE_LEN] = {0}; -char tsLocale[TSDB_LOCALE_LEN] = {0}; -char tsCharset[TSDB_LOCALE_LEN] = {0}; // default encode string - #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) /* @@ -89,25 +80,6 @@ bool taosGetProcMemory(float *memoryUsedMB) { return true; } -static void taosGetSystemTimezone() { - // get and set default timezone - char *tz = getenv("TZ"); - if (tz == NULL || strlen(tz) == 0) { - strcpy(tsTimezone, "not configured"); - } else { - strcpy(tsTimezone, tz); - } -} - -static void taosGetSystemLocale() { - // get and set default locale - char *locale = setlocale(LC_CTYPE, "chs"); - if (locale != NULL) { - tstrncpy(tsLocale, locale, TSDB_LOCALE_LEN); - } - - strcpy(tsCharset, "cp936"); -} int32_t taosGetCpuCores() { SYSTEM_INFO info; @@ -200,8 +172,6 @@ void taosGetSystemInfo() { taosGetCpuUsage(&tmp1, &tmp2); taosGetProcIO(&tmp1, &tmp2); - taosGetSystemTimezone(); - taosGetSystemLocale(); } void taosKillSystem() { @@ -273,103 +243,6 @@ char *taosGetCmdlineByPID(int pid) { return ""; } #include #include -static void taosGetSystemTimezone() { - /* load time zone string from /etc/localtime */ - char buf[4096]; - char *tz = NULL; - { - int n = readlink("/etc/localtime", buf, sizeof(buf)); - if (n < 0) { - //printf("read /etc/localtime error, reason:%s", strerror(errno)); - return; - } - buf[n] = '\0'; - for (int i = n - 1; i >= 0; --i) { - if (buf[i] == '/') { - if (tz) { - tz = buf + i + 1; - break; - } - tz = buf + i + 1; - } - } - if (!tz || 0 == strchr(tz, '/')) { - //printf("parsing /etc/localtime failed"); - return; - } - - setenv("TZ", tz, 1); - tzset(); - } - - /* - * NOTE: do not remove it. - * Enforce set the correct daylight saving time(DST) flag according - * to current time - */ - time_t tx1 = time(NULL); - struct tm tm1; - localtime_r(&tx1, &tm1); - - /* - * format example: - * - * Asia/Shanghai (CST, +0800) - * Europe/London (BST, +0100) - */ - snprintf(tsTimezone, TSDB_TIMEZONE_LEN, "%s (%s, %+03ld00)", tz, tm1.tm_isdst ? tzname[daylight] : tzname[0], - -timezone / 3600); - - // cfg_timezone->cfgStatus = TAOS_CFG_CSTATUS_DEFAULT; - //printf("timezone not configured, set to system default:%s", tsTimezone); -} - -/* - * originally from src/os/src/detail/osSysinfo.c - * POSIX format locale string: - * (Language Strings)_(Country/Region Strings).(code_page) - * - * example: en_US.UTF-8, zh_CN.GB18030, zh_CN.UTF-8, - * - * if user does not specify the locale in taos.cfg the program use default LC_CTYPE as system locale. - * - * In case of some CentOS systems, their default locale is "en_US.utf8", which is not valid code_page - * for libiconv that is employed to convert string in this system. This program will automatically use - * UTF-8 instead as the charset. - * - * In case of windows client, the locale string is not valid POSIX format, user needs to set the - * correct code_page for libiconv. Usually, the code_page of windows system with simple chinese is - * CP936, CP437 for English charset. - * - */ -static void taosGetSystemLocale() { // get and set default locale - char sep = '.'; - char *locale = NULL; - - locale = setlocale(LC_CTYPE, ""); - if (locale == NULL) { - //printf("can't get locale from system, set it to en_US.UTF-8 since error:%d:%s", errno, strerror(errno)); - strcpy(tsLocale, "en_US.UTF-8"); - } else { - tstrncpy(tsLocale, locale, TSDB_LOCALE_LEN); - //printf("locale not configured, set to system default:%s", tsLocale); - } - - /* if user does not specify the charset, extract it from locale */ - char *str = strrchr(tsLocale, sep); - if (str != NULL) { - str++; - - char *revisedCharset = taosCharsetReplace(str); - tstrncpy(tsCharset, revisedCharset, TSDB_LOCALE_LEN); - - free(revisedCharset); - //printf("charset not configured, set to system default:%s", tsCharset); - } else { - strcpy(tsCharset, "UTF-8"); - //printf("can't get locale and charset from system, set it to UTF-8"); - } -} void taosKillSystem() { //printf("function taosKillSystem, exit!"); @@ -386,9 +259,6 @@ void taosGetSystemInfo() { long page_size = sysconf(_SC_PAGESIZE); tsTotalMemoryMB = physical_pages * page_size / (1024 * 1024); tsPageSize = page_size; - - taosGetSystemTimezone(); - taosGetSystemLocale(); } bool taosReadProcIO(int64_t *rchars, int64_t *wchars) { @@ -533,19 +403,17 @@ bool taosGetSysMemory(float *memoryUsedMB) { } bool taosGetProcMemory(float *memoryUsedMB) { - FILE *fp = fopen(tsProcMemFile, "r"); - if (fp == NULL) { + // FILE *fp = fopen(tsProcMemFile, "r"); + TdFilePtr pFile = taosOpenFile(tsProcMemFile, TD_FILE_READ); + if (pFile == NULL) { //printf("open file:%s failed", tsProcMemFile); return false; } ssize_t _bytes = 0; - size_t len; char * line = NULL; - while (!feof(fp)) { - tfree(line); - len = 0; - _bytes = getline(&line, &len, fp); + while (!taosEOFFile(pFile)) { + _bytes = taosGetLineFile(pFile, &line); if ((_bytes < 0) || (line == NULL)) { break; } @@ -556,7 +424,7 @@ bool taosGetProcMemory(float *memoryUsedMB) { if (line == NULL) { //printf("read file:%s failed", tsProcMemFile); - fclose(fp); + taosCloseFile(&pFile); return false; } @@ -565,24 +433,24 @@ bool taosGetProcMemory(float *memoryUsedMB) { sscanf(line, "%s %" PRId64, tmp, &memKB); *memoryUsedMB = (float)((double)memKB / 1024); - tfree(line); - fclose(fp); + if(line != NULL) tfree(line); + taosCloseFile(&pFile); return true; } static bool taosGetSysCpuInfo(SysCpuInfo *cpuInfo) { - FILE *fp = fopen(tsSysCpuFile, "r"); - if (fp == NULL) { + // FILE *fp = fopen(tsSysCpuFile, "r"); + TdFilePtr pFile = taosOpenFile(tsSysCpuFile, TD_FILE_READ); + if (pFile == NULL) { //printf("open file:%s failed", tsSysCpuFile); return false; } - size_t len; char * line = NULL; - ssize_t _bytes = getline(&line, &len, fp); + ssize_t _bytes = taosGetLineFile(pFile, &line); if ((_bytes < 0) || (line == NULL)) { //printf("read file:%s failed", tsSysCpuFile); - fclose(fp); + taosCloseFile(&pFile); return false; } @@ -590,24 +458,24 @@ static bool taosGetSysCpuInfo(SysCpuInfo *cpuInfo) { sscanf(line, "%s %" PRIu64 " %" PRIu64 " %" PRIu64 " %" PRIu64, cpu, &cpuInfo->user, &cpuInfo->nice, &cpuInfo->system, &cpuInfo->idle); - tfree(line); - fclose(fp); + if(line != NULL) tfree(line); + taosCloseFile(&pFile); return true; } static bool taosGetProcCpuInfo(ProcCpuInfo *cpuInfo) { - FILE *fp = fopen(tsProcCpuFile, "r"); - if (fp == NULL) { + // FILE *fp = fopen(tsProcCpuFile, "r"); + TdFilePtr pFile = taosOpenFile(tsProcCpuFile, TD_FILE_READ); + if (pFile == NULL) { //printf("open file:%s failed", tsProcCpuFile); return false; } - size_t len = 0; char * line = NULL; - ssize_t _bytes = getline(&line, &len, fp); + ssize_t _bytes = taosGetLineFile(pFile, &line); if ((_bytes < 0) || (line == NULL)) { //printf("read file:%s failed", tsProcCpuFile); - fclose(fp); + taosCloseFile(&pFile); return false; } @@ -620,115 +488,11 @@ static bool taosGetProcCpuInfo(ProcCpuInfo *cpuInfo) { } } - tfree(line); - fclose(fp); + if(line != NULL) tfree(line); + taosCloseFile(&pFile); return true; } -static void taosGetSystemTimezone() { - /* - * NOTE: do not remove it. - * Enforce set the correct daylight saving time(DST) flag according - * to current time - */ - time_t tx1 = time(NULL); - struct tm tm1; - localtime_r(&tx1, &tm1); - - /* load time zone string from /etc/timezone */ - FILE *f = fopen("/etc/timezone", "r"); - char buf[68] = {0}; - if (f != NULL) { - int len = fread(buf, 64, 1, f); - if (len < 64 && ferror(f)) { - fclose(f); - //printf("read /etc/timezone error, reason:%s", strerror(errno)); - return; - } - - fclose(f); - - buf[sizeof(buf) - 1] = 0; - char *lineEnd = strstr(buf, "\n"); - if (lineEnd != NULL) { - *lineEnd = 0; - } - - // for CentOS system, /etc/timezone does not exist. Ignore the TZ environment variables - if (strlen(buf) > 0) { - setenv("TZ", buf, 1); - } - } - // get and set default timezone - tzset(); - - /* - * get CURRENT time zone. - * system current time zone is affected by daylight saving time(DST) - * - * e.g., the local time zone of London in DST is GMT+01:00, - * otherwise is GMT+00:00 - */ - int32_t tz = (-timezone * MILLISECOND_PER_SECOND) / MILLISECOND_PER_HOUR; - tz += daylight; - - /* - * format example: - * - * Asia/Shanghai (CST, +0800) - * Europe/London (BST, +0100) - */ - snprintf(tsTimezone, TSDB_TIMEZONE_LEN, "%s (%s, %s%02d00)", buf, tzname[daylight], tz >= 0 ? "+" : "-", abs(tz)); - - // cfg_timezone->cfgStatus = TAOS_CFG_CSTATUS_DEFAULT; - //printf("timezone not configured, set to system default:%s", tsTimezone); -} - -/* - * POSIX format locale string: - * (Language Strings)_(Country/Region Strings).(code_page) - * - * example: en_US.UTF-8, zh_CN.GB18030, zh_CN.UTF-8, - * - * if user does not specify the locale in taos.cfg the program use default LC_CTYPE as system locale. - * - * In case of some CentOS systems, their default locale is "en_US.utf8", which is not valid code_page - * for libiconv that is employed to convert string in this system. This program will automatically use - * UTF-8 instead as the charset. - * - * In case of windows client, the locale string is not valid POSIX format, user needs to set the - * correct code_page for libiconv. Usually, the code_page of windows system with simple chinese is - * CP936, CP437 for English charset. - * - */ -static void taosGetSystemLocale() { // get and set default locale - char sep = '.'; - char *locale = NULL; - - locale = setlocale(LC_CTYPE, ""); - if (locale == NULL) { - //printf("can't get locale from system, set it to en_US.UTF-8 since error:%d:%s", errno, strerror(errno)); - strcpy(tsLocale, "en_US.UTF-8"); - } else { - tstrncpy(tsLocale, locale, TSDB_LOCALE_LEN); - //printf("locale not configured, set to system default:%s", tsLocale); - } - - // if user does not specify the charset, extract it from locale - char *str = strrchr(tsLocale, sep); - if (str != NULL) { - str++; - - char *revisedCharset = taosCharsetReplace(str); - tstrncpy(tsCharset, revisedCharset, TSDB_LOCALE_LEN); - - free(revisedCharset); - //printf("charset not configured, set to system default:%s", tsCharset); - } else { - strcpy(tsCharset, "UTF-8"); - //printf("can't get locale and charset from system, set it to UTF-8"); - } -} int32_t taosGetCpuCores() { return (int32_t)sysconf(_SC_NPROCESSORS_ONLN); } @@ -785,19 +549,17 @@ int32_t taosGetDiskSize(char *dataDir, SDiskSize *diskSize) { bool taosGetCardInfo(int64_t *bytes, int64_t *rbytes, int64_t *tbytes) { *bytes = 0; - FILE *fp = fopen(tsSysNetFile, "r"); - if (fp == NULL) { + // FILE *fp = fopen(tsSysNetFile, "r"); + TdFilePtr pFile = taosOpenFile(tsSysNetFile, TD_FILE_READ); + if (pFile == NULL) { //printf("open file:%s failed", tsSysNetFile); return false; } ssize_t _bytes = 0; - size_t len = 2048; - char * line = calloc(1, len); - - while (!feof(fp)) { - memset(line, 0, len); + char * line = NULL; + while (!taosEOFFile(pFile)) { int64_t o_rbytes = 0; int64_t rpackts = 0; int64_t o_tbytes = 0; @@ -810,12 +572,12 @@ bool taosGetCardInfo(int64_t *bytes, int64_t *rbytes, int64_t *tbytes) { int64_t nouse6 = 0; char nouse0[200] = {0}; - _bytes = getline(&line, &len, fp); + _bytes = taosGetLineFile(pFile, &line); if (_bytes < 0) { break; } - line[len - 1] = 0; + line[_bytes - 1] = 0; if (strstr(line, "lo:") != NULL) { continue; @@ -830,8 +592,8 @@ bool taosGetCardInfo(int64_t *bytes, int64_t *rbytes, int64_t *tbytes) { *bytes += (o_rbytes + o_tbytes); } - tfree(line); - fclose(fp); + if(line != NULL) tfree(line); + taosCloseFile(&pFile); return true; } @@ -873,22 +635,20 @@ bool taosGetBandSpeed(float *bandSpeedKb) { } bool taosReadProcIO(int64_t *rchars, int64_t *wchars) { - FILE *fp = fopen(tsProcIOFile, "r"); - if (fp == NULL) { + // FILE *fp = fopen(tsProcIOFile, "r"); + TdFilePtr pFile = taosOpenFile(tsProcIOFile, TD_FILE_READ); + if (pFile == NULL) { //printf("open file:%s failed", tsProcIOFile); return false; } ssize_t _bytes = 0; - size_t len; char * line = NULL; char tmp[10]; int readIndex = 0; - while (!feof(fp)) { - tfree(line); - len = 0; - _bytes = getline(&line, &len, fp); + while (!taosEOFFile(pFile)) { + _bytes = taosGetLineFile(pFile, &line); if ((_bytes < 0) || (line == NULL)) { break; } @@ -904,8 +664,8 @@ bool taosReadProcIO(int64_t *rchars, int64_t *wchars) { if (readIndex >= 2) break; } - tfree(line); - fclose(fp); + if(line != NULL) tfree(line); + taosCloseFile(&pFile); if (readIndex < 2) { //printf("read file:%s failed", tsProcIOFile); @@ -957,8 +717,6 @@ void taosGetSystemInfo() { taosGetCpuUsage(&tmp1, &tmp2); taosGetProcIO(&tmp1, &tmp2); - taosGetSystemTimezone(); - taosGetSystemLocale(); } void taosKillSystem() { @@ -1070,15 +828,15 @@ void taosSetCoreDump(bool enable) { } int32_t taosGetSystemUUID(char *uid, int32_t uidlen) { - int fd; int len = 0; - fd = open("/proc/sys/kernel/random/uuid", 0); - if (fd < 0) { + // fd = open("/proc/sys/kernel/random/uuid", 0); + TdFilePtr pFile = taosOpenFile("/proc/sys/kernel/random/uuid", TD_FILE_READ); + if (pFile == NULL) { return -1; } else { - len = read(fd, uid, uidlen); - close(fd); + len = taosReadFile(pFile, uid, uidlen); + taosCloseFile(&pFile); } if (len >= 36) { @@ -1093,16 +851,17 @@ char *taosGetCmdlineByPID(int pid) { static char cmdline[1024]; sprintf(cmdline, "/proc/%d/cmdline", pid); - int fd = open(cmdline, O_RDONLY); - if (fd >= 0) { - int n = read(fd, cmdline, sizeof(cmdline) - 1); + // int fd = open(cmdline, O_RDONLY); + TdFilePtr pFile = taosOpenFile(cmdline, TD_FILE_READ); + if (pFile != NULL) { + int n = taosReadFile(pFile, cmdline, sizeof(cmdline) - 1); if (n < 0) n = 0; if (n > 0 && cmdline[n - 1] == '\n') --n; cmdline[n] = 0; - close(fd); + taosCloseFile(&pFile); } else { cmdline[0] = 0; } diff --git a/source/os/src/osTimeZone.c b/source/os/src/osTimeZone.c deleted file mode 100644 index da40e14389..0000000000 --- a/source/os/src/osTimeZone.c +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -// #include "ulog.h" -// #include "tglobal.h" -// #include "tutil.h" - -// TODO refactor to set the tz value through parameter -void tsSetTimeZone() { -#if 0 - SGlobalCfg *cfg_timezone = taosGetConfigOption("timezone"); - if (cfg_timezone != NULL) { - uInfo("timezone is set to %s by %s", tsTimezone, tsCfgStatusStr[cfg_timezone->cfgStatus]); - } - -#ifdef WINDOWS - char winStr[TSDB_LOCALE_LEN * 2]; - sprintf(winStr, "TZ=%s", tsTimezone); - putenv(winStr); -#else - setenv("TZ", tsTimezone, 1); -#endif - tzset(); - - /* - * get CURRENT time zone. - * system current time zone is affected by daylight saving time(DST) - * - * e.g., the local time zone of London in DST is GMT+01:00, - * otherwise is GMT+00:00 - */ -#ifdef _MSC_VER -#if _MSC_VER >= 1900 - // see https://docs.microsoft.com/en-us/cpp/c-runtime-library/daylight-dstbias-timezone-and-tzname?view=vs-2019 - int64_t timezone = _timezone; - int32_t daylight = _daylight; - char **tzname = _tzname; -#endif -#endif - - int32_t tz = (int32_t)((-timezone * MILLISECOND_PER_SECOND) / MILLISECOND_PER_HOUR); - tz += daylight; - - /* - * format: - * (CST, +0800) - * (BST, +0100) - */ - sprintf(tsTimezone, "(%s, %s%02d00)", tzname[daylight], tz >= 0 ? "+" : "-", abs(tz)); - tsDaylight = daylight; - - uInfo("timezone format changed to %s", tsTimezone); - -#endif -} diff --git a/source/os/src/osTimezone.c b/source/os/src/osTimezone.c new file mode 100644 index 0000000000..07846781ad --- /dev/null +++ b/source/os/src/osTimezone.c @@ -0,0 +1,199 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" + +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#if (_WIN64) +#include +#include +#include +#include +#include +#include +#pragma comment(lib, "Mswsock.lib ") +#endif +#include +#pragma warning(push) +#pragma warning(disable : 4091) +#include +#pragma warning(pop) +#elif defined(_TD_DARWIN_64) +#include +#include +#else +#include +#include +#include +#include +#include +#include +#include +#include +#endif + +void taosSetSystemTimezone(const char *inTimezone, char *outTimezone, int8_t *outDaylight) { +#ifdef WINDOWS + char winStr[TD_LOCALE_LEN * 2]; + sprintf(winStr, "TZ=%s", inTimezone); + putenv(winStr); +#else + setenv("TZ", inTimezone, 1); +#endif + tzset(); + + /* + * get CURRENT time zone. + * system current time zone is affected by daylight saving time(DST) + * + * e.g., the local time zone of London in DST is GMT+01:00, + * otherwise is GMT+00:00 + */ +#ifdef _MSC_VER +#if _MSC_VER >= 1900 + // see https://docs.microsoft.com/en-us/cpp/c-runtime-library/daylight-dstbias-timezone-and-tzname?view=vs-2019 + int64_t timezone = _timezone; + int32_t daylight = _daylight; + char **tzname = _tzname; +#endif +#endif + + int32_t tz = (int32_t)((-timezone * MILLISECOND_PER_SECOND) / MILLISECOND_PER_HOUR); + tz += daylight; + + /* + * format: + * (CST, +0800) + * (BST, +0100) + */ + + sprintf(outTimezone, "(%s, %s%02d00)", tzname[daylight], tz >= 0 ? "+" : "-", abs(tz)); + *outDaylight = daylight; +} + +void taosGetSystemTimezone(char *outTimezone) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + char *tz = getenv("TZ"); + if (tz == NULL || strlen(tz) == 0) { + strcpy(outTimezone, "not configured"); + } else { + strcpy(outTimezone, tz); + } + +#elif defined(_TD_DARWIN_64) + char buf[4096] = {0}; + char *tz = NULL; + { + int n = readlink("/etc/localtime", buf, sizeof(buf)); + if (n < 0) { + printf("read /etc/localtime error, reason:%s", strerror(errno)); + return; + } + buf[n] = '\0'; + for (int i = n - 1; i >= 0; --i) { + if (buf[i] == '/') { + if (tz) { + tz = buf + i + 1; + break; + } + tz = buf + i + 1; + } + } + if (!tz || 0 == strchr(tz, '/')) { + printf("parsing /etc/localtime failed"); + return; + } + + setenv("TZ", tz, 1); + tzset(); + } + + /* + * NOTE: do not remove it. + * Enforce set the correct daylight saving time(DST) flag according + * to current time + */ + time_t tx1 = time(NULL); + struct tm tm1; + localtime_r(&tx1, &tm1); + + /* + * format example: + * + * Asia/Shanghai (CST, +0800) + * Europe/London (BST, +0100) + */ + snprintf(outTimezone, TD_TIMEZONE_LEN, "%s (%s, %+03ld00)", tz, tm1.tm_isdst ? tzname[daylight] : tzname[0], + -timezone / 3600); + +#else + /* + * NOTE: do not remove it. + * Enforce set the correct daylight saving time(DST) flag according + * to current time + */ + time_t tx1 = time(NULL); + struct tm tm1; + localtime_r(&tx1, &tm1); + + /* load time zone string from /etc/timezone */ + // FILE *f = fopen("/etc/timezone", "r"); + TdFilePtr pFile = taosOpenFile("/etc/timezone", TD_FILE_READ); + char buf[68] = {0}; + if (pFile != NULL) { + int len = taosReadFile(pFile, buf, 64); + if (len < 64 && taosGetErrorFile(pFile)) { + taosCloseFile(&pFile); + // printf("read /etc/timezone error, reason:%s", strerror(errno)); + return; + } + + taosCloseFile(&pFile); + + buf[sizeof(buf) - 1] = 0; + char *lineEnd = strstr(buf, "\n"); + if (lineEnd != NULL) { + *lineEnd = 0; + } + + // for CentOS system, /etc/timezone does not exist. Ignore the TZ environment variables + if (strlen(buf) > 0) { + setenv("TZ", buf, 1); + } + } + // get and set default timezone + tzset(); + + /* + * get CURRENT time zone. + * system current time zone is affected by daylight saving time(DST) + * + * e.g., the local time zone of London in DST is GMT+01:00, + * otherwise is GMT+00:00 + */ + int32_t tz = (-timezone * MILLISECOND_PER_SECOND) / MILLISECOND_PER_HOUR; + tz += daylight; + + /* + * format example: + * + * Asia/Shanghai (CST, +0800) + * Europe/London (BST, +0100) + */ + snprintf(outTimezone, TD_TIMEZONE_LEN, "%s (%s, %s%02d00)", buf, tzname[daylight], tz >= 0 ? "+" : "-", abs(tz)); + +#endif +} diff --git a/source/libs/config/src/config.c b/source/util/src/tconfig.c similarity index 76% rename from source/libs/config/src/config.c rename to source/util/src/tconfig.c index 17a652d2b5..7ad08e44d5 100644 --- a/source/libs/config/src/config.c +++ b/source/util/src/tconfig.c @@ -14,14 +14,25 @@ */ #define _DEFAULT_SOURCE -#include "cfgInt.h" -#include "tep.h" -#include "tlocale.h" -#include "tmsg.h" -#include "ttimezone.h" +#include "tconfig.h" +#include "taoserror.h" +#include "thash.h" +#include "tutil.h" +#include "ulog.h" -#define CFG_NAME_PRINT_LEN 22 -#define CFG_SRC_PRINT_LEN 12 +#define CFG_NAME_PRINT_LEN 24 +#define CFG_SRC_PRINT_LEN 12 + +typedef struct SConfig { + ECfgSrcType stype; + SHashObj *hash; +} SConfig; + +int32_t cfgLoadFromCfgFile(SConfig *pConfig, const char *filepath); +int32_t cfgLoadFromEnvFile(SConfig *pConfig, const char *filepath); +int32_t cfgLoadFromEnvVar(SConfig *pConfig); +int32_t cfgLoadFromApollUrl(SConfig *pConfig, const char *url); +int32_t cfgSetItem(SConfig *pConfig, const char *name, const char *value, ECfgSrcType stype); SConfig *cfgInit() { SConfig *pCfg = calloc(1, sizeof(SConfig)); @@ -133,23 +144,6 @@ static int32_t cfgCheckAndSetDir(SConfigItem *pItem, const char *inputDir) { return 0; } -static int32_t cfgCheckAndSetIpStr(SConfigItem *pItem, const char *ip) { - uint32_t value = taosInetAddr(ip); - if (value == INADDR_NONE) { - uError("ip:%s is not a valid ip address", ip); - return -1; - } - - tfree(pItem->str); - pItem->str = strdup(ip); - if (pItem->str == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - return 0; -} - static int32_t cfgSetBool(SConfigItem *pItem, const char *value, ECfgSrcType stype) { bool tmp = false; if (strcasecmp(value, "true") == 0) { @@ -225,20 +219,6 @@ static int32_t cfgSetString(SConfigItem *pItem, const char *value, ECfgSrcType s return 0; } -static int32_t cfgSetIpStr(SConfigItem *pItem, const char *value, ECfgSrcType stype) { - char *tmp = strdup(value); - if (tmp == NULL || cfgCheckAndSetIpStr(pItem, value) != 0) { - free(tmp); - terrno = TSDB_CODE_OUT_OF_MEMORY; - uError("cfg:%s, type:%s src:%s value:%s failed to dup since %s, use last src:%s value:%s", pItem->name, - cfgDtypeStr(pItem->dtype), cfgStypeStr(stype), value, terrstr(), cfgStypeStr(pItem->stype), pItem->str); - return -1; - } - - pItem->stype = stype; - return 0; -} - static int32_t cfgSetDir(SConfigItem *pItem, const char *value, ECfgSrcType stype) { char *tmp = strdup(value); if (tmp == NULL || cfgCheckAndSetDir(pItem, value) != 0) { @@ -312,8 +292,6 @@ int32_t cfgSetItem(SConfig *pCfg, const char *name, const char *value, ECfgSrcTy return cfgSetFloat(pItem, value, stype); case CFG_DTYPE_STRING: return cfgSetString(pItem, value, stype); - case CFG_DTYPE_IPSTR: - return cfgSetIpStr(pItem, value, stype); case CFG_DTYPE_DIR: return cfgSetDir(pItem, value, stype); case CFG_DTYPE_TIMEZONE: @@ -368,43 +346,43 @@ static int32_t cfgAddItem(SConfig *pCfg, SConfigItem *pItem, const char *name) { return 0; } -int32_t cfgAddBool(SConfig *pCfg, const char *name, bool defaultVal) { - SConfigItem item = {.dtype = CFG_DTYPE_BOOL, .bval = defaultVal}; +int32_t cfgAddBool(SConfig *pCfg, const char *name, bool defaultVal, bool tsc) { + SConfigItem item = {.dtype = CFG_DTYPE_BOOL, .bval = defaultVal, .tsc = tsc}; return cfgAddItem(pCfg, &item, name); } -int32_t cfgAddInt32(SConfig *pCfg, const char *name, int32_t defaultVal, int64_t minval, int64_t maxval) { +int32_t cfgAddInt32(SConfig *pCfg, const char *name, int32_t defaultVal, int64_t minval, int64_t maxval, bool tsc) { if (defaultVal < minval || defaultVal > maxval) { terrno = TSDB_CODE_OUT_OF_RANGE; return -1; } - SConfigItem item = {.dtype = CFG_DTYPE_INT32, .i32 = defaultVal, .imin = minval, .imax = maxval}; + SConfigItem item = {.dtype = CFG_DTYPE_INT32, .i32 = defaultVal, .imin = minval, .imax = maxval, .tsc = tsc}; return cfgAddItem(pCfg, &item, name); } -int32_t cfgAddInt64(SConfig *pCfg, const char *name, int64_t defaultVal, int64_t minval, int64_t maxval) { +int32_t cfgAddInt64(SConfig *pCfg, const char *name, int64_t defaultVal, int64_t minval, int64_t maxval, bool tsc) { if (defaultVal < minval || defaultVal > maxval) { terrno = TSDB_CODE_OUT_OF_RANGE; return -1; } - SConfigItem item = {.dtype = CFG_DTYPE_INT64, .i64 = defaultVal, .imin = minval, .imax = maxval}; + SConfigItem item = {.dtype = CFG_DTYPE_INT64, .i64 = defaultVal, .imin = minval, .imax = maxval, .tsc = tsc}; return cfgAddItem(pCfg, &item, name); } -int32_t cfgAddFloat(SConfig *pCfg, const char *name, float defaultVal, double minval, double maxval) { +int32_t cfgAddFloat(SConfig *pCfg, const char *name, float defaultVal, double minval, double maxval, bool tsc) { if (defaultVal < minval || defaultVal > maxval) { terrno = TSDB_CODE_OUT_OF_RANGE; return -1; } - SConfigItem item = {.dtype = CFG_DTYPE_FLOAT, .fval = defaultVal, .fmin = minval, .fmax = maxval}; + SConfigItem item = {.dtype = CFG_DTYPE_FLOAT, .fval = defaultVal, .fmin = minval, .fmax = maxval, .tsc = tsc}; return cfgAddItem(pCfg, &item, name); } -int32_t cfgAddString(SConfig *pCfg, const char *name, const char *defaultVal) { - SConfigItem item = {.dtype = CFG_DTYPE_STRING}; +int32_t cfgAddString(SConfig *pCfg, const char *name, const char *defaultVal, bool tsc) { + SConfigItem item = {.dtype = CFG_DTYPE_STRING, .tsc = tsc}; item.str = strdup(defaultVal); if (item.str == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -413,17 +391,8 @@ int32_t cfgAddString(SConfig *pCfg, const char *name, const char *defaultVal) { return cfgAddItem(pCfg, &item, name); } -int32_t cfgAddIpStr(SConfig *pCfg, const char *name, const char *defaultVal) { - SConfigItem item = {.dtype = CFG_DTYPE_IPSTR}; - if (cfgCheckAndSetIpStr(&item, defaultVal) != 0) { - return -1; - } - - return cfgAddItem(pCfg, &item, name); -} - -int32_t cfgAddDir(SConfig *pCfg, const char *name, const char *defaultVal) { - SConfigItem item = {.dtype = CFG_DTYPE_DIR}; +int32_t cfgAddDir(SConfig *pCfg, const char *name, const char *defaultVal, bool tsc) { + SConfigItem item = {.dtype = CFG_DTYPE_DIR, .tsc = tsc}; if (cfgCheckAndSetDir(&item, defaultVal) != 0) { return -1; } @@ -432,7 +401,7 @@ int32_t cfgAddDir(SConfig *pCfg, const char *name, const char *defaultVal) { } int32_t cfgAddLocale(SConfig *pCfg, const char *name, const char *defaultVal) { - SConfigItem item = {.dtype = CFG_DTYPE_LOCALE}; + SConfigItem item = {.dtype = CFG_DTYPE_LOCALE, .tsc = 1}; if (cfgCheckAndSetLocale(&item, defaultVal) != 0) { return -1; } @@ -441,7 +410,7 @@ int32_t cfgAddLocale(SConfig *pCfg, const char *name, const char *defaultVal) { } int32_t cfgAddCharset(SConfig *pCfg, const char *name, const char *defaultVal) { - SConfigItem item = {.dtype = CFG_DTYPE_CHARSET}; + SConfigItem item = {.dtype = CFG_DTYPE_CHARSET, .tsc = 1}; if (cfgCheckAndSetCharset(&item, defaultVal) != 0) { return -1; } @@ -450,7 +419,7 @@ int32_t cfgAddCharset(SConfig *pCfg, const char *name, const char *defaultVal) { } int32_t cfgAddTimezone(SConfig *pCfg, const char *name, const char *defaultVal) { - SConfigItem item = {.dtype = CFG_DTYPE_TIMEZONE}; + SConfigItem item = {.dtype = CFG_DTYPE_TIMEZONE, .tsc = 1}; if (cfgCheckAndSetTimezone(&item, defaultVal) != 0) { return -1; } @@ -493,8 +462,6 @@ const char *cfgDtypeStr(ECfgDataType type) { return "float"; case CFG_DTYPE_STRING: return "string"; - case CFG_DTYPE_IPSTR: - return "ipstr"; case CFG_DTYPE_DIR: return "dir"; case CFG_DTYPE_LOCALE: @@ -508,15 +475,23 @@ const char *cfgDtypeStr(ECfgDataType type) { } } -void cfgDumpCfg(SConfig *pCfg) { - uInfo(" global config"); - uInfo("================================================================="); +void cfgDumpCfg(SConfig *pCfg, bool tsc, bool dump) { + if (dump) { + printf(" global config"); + printf("\n"); + printf("================================================================="); + printf("\n"); + } else { + uInfo(" global config"); + uInfo("================================================================="); + } char src[CFG_SRC_PRINT_LEN + 1] = {0}; char name[CFG_NAME_PRINT_LEN + 1] = {0}; SConfigItem *pItem = cfgIterate(pCfg, NULL); while (pItem != NULL) { + if (tsc && !pItem->tsc) continue; tstrncpy(src, cfgStypeStr(pItem->stype), CFG_SRC_PRINT_LEN); for (int32_t i = 0; i < CFG_SRC_PRINT_LEN; ++i) { if (src[i] == 0) src[i] = ' '; @@ -529,107 +504,123 @@ void cfgDumpCfg(SConfig *pCfg) { switch (pItem->dtype) { case CFG_DTYPE_BOOL: - uInfo("%s %s %u", src, name, pItem->bval); + if (dump) { + printf("%s %s %u", src, name, pItem->bval); + printf("\n"); + } else { + uInfo("%s %s %u", src, name, pItem->bval); + } + break; case CFG_DTYPE_INT32: - uInfo("%s %s %d", src, name, pItem->i32); + if (dump) { + printf("%s %s %d", src, name, pItem->i32); + printf("\n"); + } else { + uInfo("%s %s %d", src, name, pItem->i32); + } break; case CFG_DTYPE_INT64: - uInfo("%s %s %" PRId64, src, name, pItem->i64); + if (dump) { + printf("%s %s %" PRId64, src, name, pItem->i64); + printf("\n"); + } else { + uInfo("%s %s %" PRId64, src, name, pItem->i64); + } break; case CFG_DTYPE_FLOAT: - uInfo("%s %s %f", src, name, pItem->fval); + if (dump) { + printf("%s %s %f", src, name, pItem->fval); + printf("\n"); + } else { + uInfo("%s %s %f", src, name, pItem->fval); + } break; case CFG_DTYPE_STRING: - case CFG_DTYPE_IPSTR: case CFG_DTYPE_DIR: case CFG_DTYPE_LOCALE: case CFG_DTYPE_CHARSET: case CFG_DTYPE_TIMEZONE: - uInfo("%s %s %s", src, name, pItem->str); + if (dump) { + printf("%s %s %s", src, name, pItem->str); + printf("\n"); + } else { + uInfo("%s %s %s", src, name, pItem->str); + } break; } pItem = cfgIterate(pCfg, pItem); } - uInfo("================================================================="); + if (dump) { + printf("================================================================="); + printf("\n"); + } else { + uInfo("================================================================="); + } } -#if 0 -// int32_t cfgCheck(SConfig *pCfg) { -// SConfigItem *pItem = NULL; -// pItem = cfgGetItem(pCfg, "serverPort"); -// if (pItem != NULL) { -// tsServerPort = (uint16_t)pItem->i32; -// } +int32_t cfgLoadFromEnvVar(SConfig *pConfig) { + uInfo("load from global env variables"); + return 0; +} -// pItem = cfgGetItem(pCfg, "firstEp"); -// if (pItem != NULL) { -// tstrncpy(tsFirst, pItem->str, TSDB_EP_LEN); -// } +int32_t cfgLoadFromEnvFile(SConfig *pConfig, const char *filepath) { + uInfo("load from env file %s", filepath); + return 0; +} -// snprintf(tsLocalEp, TSDB_EP_LEN, "%s:%u", tsLocalFqdn, tsServerPort); -// uInfo("localEp is: %s", tsLocalEp); +int32_t cfgLoadFromCfgFile(SConfig *pConfig, const char *filepath) { + char *line = NULL, *name, *value, *value2, *value3; + int olen, vlen, vlen2, vlen3; + ssize_t _bytes = 0; -// SEp ep = {0}; -// if (tsFirst[0] == 0) { -// strcpy(tsFirst, tsLocalEp); -// } else { -// taosGetFqdnPortFromEp(tsFirst, &ep); -// snprintf(tsFirst, TSDB_EP_LEN, "%s:%u", ep.fqdn, ep.port); -// } + // FILE *fp = fopen(filepath, "r"); + TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_READ); + if (pFile == NULL) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } -// pItem = cfgGetItem(pCfg, "secondEp"); -// if (pItem != NULL) { -// tstrncpy(tsSecond, pItem->str, TSDB_EP_LEN); -// } + while (!taosEOFFile(pFile)) { -// if (tsSecond[0] == 0) { -// strcpy(tsSecond, tsLocalEp); -// } else { -// taosGetFqdnPortFromEp(tsSecond, &ep); -// snprintf(tsSecond, TSDB_EP_LEN, "%s:%u", ep.fqdn, ep.port); -// } + name = value = value2 = value3 = NULL; + olen = vlen = vlen2 = vlen3 = 0; -// pItem = cfgGetItem(pCfg, "dataDir"); -// if (pItem != NULL) { -// tstrncpy(tsDataDir, pItem->str, PATH_MAX); -// } + _bytes = taosGetLineFile(pFile, &line); + if (_bytes < 0) { + break; + } -// if (tsDiskCfgNum <= 0) { -// taosAddDataDir(0, tsDataDir, 0, 1); -// tsDiskCfgNum = 1; -// uTrace("dataDir:%s, level:0 primary:1 is configured by default", tsDataDir); -// } + line[_bytes - 1] = 0; -// if (taosDirExist(tsTempDir) != 0) { -// return -1; -// } + paGetToken(line, &name, &olen); + if (olen == 0) continue; + name[olen] = 0; -// taosGetSystemInfo(); + paGetToken(name + olen + 1, &value, &vlen); + if (vlen == 0) continue; + value[vlen] = 0; -// tsSetLocale(); + paGetToken(value + vlen + 1, &value2, &vlen2); + if (vlen2 != 0) { + value2[vlen2] = 0; + paGetToken(value2 + vlen2 + 1, &value3, &vlen3); + if (vlen3 != 0) value3[vlen3] = 0; + } -// // SGlobalCfg *cfg_timezone = taosGetConfigOption("timezone"); -// // if (cfg_timezone && cfg_timezone->cfgStatus == TAOS_CFG_CSTATUS_FILE) { -// tsSetTimeZone(); -// // } + cfgSetItem(pConfig, name, value, CFG_STYPE_CFG_FILE); + // taosReadConfigOption(name, value, value2, value3); + } -// pItem = cfgGetItem(pCfg, "numOfCores"); -// if (pItem != NULL) { -// tsNumOfCores = pItem->i32; -// } + taosCloseFile(&pFile); + if(line != NULL) tfree(line); -// if (tsNumOfCores <= 0) { -// tsNumOfCores = 1; -// } + uInfo("load from cfg file %s success", filepath); + return 0; +} -// if (tsQueryBufferSize >= 0) { -// tsQueryBufferSizeBytes = tsQueryBufferSize * 1048576UL; -// } - -// cfgPrintCfg(pCfg); - -// return 0; -// } -#endif \ No newline at end of file +int32_t cfgLoadFromApollUrl(SConfig *pConfig, const char *url) { + uInfo("load from apoll url %s", url); + return 0; +} \ No newline at end of file diff --git a/source/util/src/tfile.c b/source/util/src/tfile.c index 0f68e9204d..d10ea6a934 100644 --- a/source/util/src/tfile.c +++ b/source/util/src/tfile.c @@ -24,147 +24,147 @@ static int32_t tsFileRsetId = -1; static int8_t tfInited = 0; -static void tfCloseFile(void *p) { taosCloseFile((int32_t)(uintptr_t)p); } +// static void tfCloseFile(void *p) { taosCloseFile((TdFilePtr)(uintptr_t)p); } int32_t tfInit() { - int8_t old = atomic_val_compare_exchange_8(&tfInited, 0, 1); - if (old == 1) return 0; - tsFileRsetId = taosOpenRef(2000, tfCloseFile); - if (tsFileRsetId > 0) { - return 0; - } else { - atomic_store_8(&tfInited, 0); - return -1; - } + // int8_t old = atomic_val_compare_exchange_8(&tfInited, 0, 1); + // if (old == 1) return 0; + // tsFileRsetId = taosOpenRef(2000, tfCloseFile); + // if (tsFileRsetId > 0) { + // return 0; + // } else { + // atomic_store_8(&tfInited, 0); + // return -1; + // } } void tfCleanup() { - atomic_store_8(&tfInited, 0); - if (tsFileRsetId >= 0) taosCloseRef(tsFileRsetId); - tsFileRsetId = -1; + // atomic_store_8(&tfInited, 0); + // if (tsFileRsetId >= 0) taosCloseRef(tsFileRsetId); + // tsFileRsetId = -1; } -static int64_t tfOpenImp(int32_t fd) { - if (fd < 0) { - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } +// static int64_t tfOpenImp(TdFilePtr pFile) { +// if (pFile == NULL) { +// terrno = TAOS_SYSTEM_ERROR(errno); +// return -1; +// } - void * p = (void *)(int64_t)fd; - int64_t rid = taosAddRef(tsFileRsetId, p); - if (rid < 0) taosCloseFile(fd); +// void * p = (void *)(int64_t)pFile; +// int64_t rid = taosAddRef(tsFileRsetId, p); +// if (rid < 0) taosCloseFile(&pFile); - return rid; -} +// return rid; +// } -int64_t tfOpenRead(const char *pathname, int32_t flags) { - int32_t fd = taosOpenFileRead(pathname); - return tfOpenImp(fd); -} +// int64_t tfOpenRead(const char *pathname, int32_t flags) { +// int32_t pFile = taosOpenFile(pathname, TD_FILE_READ); +// return tfOpenImp(fd); +// } -int64_t tfOpenReadWrite(const char *pathname, int32_t flags) { - int32_t fd = taosOpenFileReadWrite(pathname); - return tfOpenImp(fd); -} +// int64_t tfOpenReadWrite(const char *pathname, int32_t flags) { +// int32_t pFile = taosOpenFile(pathname, TD_FILE_READ | TD_FILE_WRITE); +// return tfOpenImp(fd); +// } -int64_t tfOpenCreateWrite(const char *pathname, int32_t flags, mode_t mode) { - int32_t fd = taosOpenFileCreateWrite(pathname); - return tfOpenImp(fd); -} +// int64_t tfOpenCreateWrite(const char *pathname, int32_t flags, mode_t mode) { +// int32_t pFile = taosOpenFile(pathname, TD_FILE_CTEATE | TD_FILE_WRITE); +// return tfOpenImp(fd); +// } -int64_t tfOpenCreateWriteAppend(const char *pathname, int32_t flags, mode_t mode) { - int32_t fd = taosOpenFileCreateWriteAppend(pathname); - return tfOpenImp(fd); -} +// int64_t tfOpenCreateWriteAppend(const char *pathname, int32_t flags, mode_t mode) { +// int32_t pFile = taosOpenFile(pathname, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_APPEND); +// return tfOpenImp(fd); +// } -int64_t tfClose(int64_t tfd) { return taosRemoveRef(tsFileRsetId, tfd); } +// int64_t tfClose(int64_t tfd) { return taosRemoveRef(tsFileRsetId, tfd); } -int64_t tfWrite(int64_t tfd, void *buf, int64_t count) { - void *p = taosAcquireRef(tsFileRsetId, tfd); - if (p == NULL) return -1; +// int64_t tfWrite(int64_t tfd, void *buf, int64_t count) { +// void *p = taosAcquireRef(tsFileRsetId, tfd); +// if (p == NULL) return -1; - int32_t fd = (int32_t)(uintptr_t)p; +// int32_t pFile = (TdFilePtr)(uintptr_t)p; - int64_t ret = taosWriteFile(fd, buf, count); - if (ret < 0) terrno = TAOS_SYSTEM_ERROR(errno); +// int64_t ret = taosWriteFile(pFile, buf, count); +// if (ret < 0) terrno = TAOS_SYSTEM_ERROR(errno); - taosReleaseRef(tsFileRsetId, tfd); - return ret; -} +// taosReleaseRef(tsFileRsetId, tfd); +// return ret; +// } -int64_t tfRead(int64_t tfd, void *buf, int64_t count) { - void *p = taosAcquireRef(tsFileRsetId, tfd); - if (p == NULL) return -1; +// int64_t tfRead(int64_t tfd, void *buf, int64_t count) { +// void *p = taosAcquireRef(tsFileRsetId, tfd); +// if (p == NULL) return -1; - int32_t fd = (int32_t)(uintptr_t)p; +// int32_t pFile = (TdFilePtr)(uintptr_t)p; - int64_t ret = taosReadFile(fd, buf, count); - if (ret < 0) terrno = TAOS_SYSTEM_ERROR(errno); +// int64_t ret = taosReadFile(pFile, buf, count); +// if (ret < 0) terrno = TAOS_SYSTEM_ERROR(errno); - taosReleaseRef(tsFileRsetId, tfd); - return ret; -} +// taosReleaseRef(tsFileRsetId, tfd); +// return ret; +// } -int64_t tfPread(int64_t tfd, void *buf, int64_t count, int32_t offset) { - void *p = taosAcquireRef(tsFileRsetId, tfd); - if (p == NULL) return -1; +// int64_t tfPread(int64_t tfd, void *buf, int64_t count, int32_t offset) { +// void *p = taosAcquireRef(tsFileRsetId, tfd); +// if (p == NULL) return -1; - int32_t fd = (int32_t)(uintptr_t)p; +// int32_t pFile = (TdFilePtr)(uintptr_t)p; - int64_t ret = pread(fd, buf, count, offset); - if (ret < 0) terrno = TAOS_SYSTEM_ERROR(errno); +// int64_t ret = pread(fd, buf, count, offset); +// if (ret < 0) terrno = TAOS_SYSTEM_ERROR(errno); - taosReleaseRef(tsFileRsetId, tfd); - return ret; -} +// taosReleaseRef(tsFileRsetId, tfd); +// return ret; +// } -int32_t tfFsync(int64_t tfd) { - void *p = taosAcquireRef(tsFileRsetId, tfd); - if (p == NULL) return -1; +// int32_t tfFsync(int64_t tfd) { +// void *p = taosAcquireRef(tsFileRsetId, tfd); +// if (p == NULL) return -1; - int32_t fd = (int32_t)(uintptr_t)p; - int32_t code = taosFsyncFile(fd); +// int32_t pFile = (TdFilePtr)(uintptr_t)p; +// int32_t code = taosFsyncFile(pFile); - taosReleaseRef(tsFileRsetId, tfd); - return code; -} +// taosReleaseRef(tsFileRsetId, tfd); +// return code; +// } -bool tfValid(int64_t tfd) { - void *p = taosAcquireRef(tsFileRsetId, tfd); - if (p == NULL) return false; +// bool tfValid(int64_t tfd) { +// void *p = taosAcquireRef(tsFileRsetId, tfd); +// if (p == NULL) return false; - taosReleaseRef(tsFileRsetId, tfd); - return true; -} +// taosReleaseRef(tsFileRsetId, tfd); +// return true; +// } -int64_t tfLseek(int64_t tfd, int64_t offset, int32_t whence) { - void *p = taosAcquireRef(tsFileRsetId, tfd); - if (p == NULL) return -1; +// int64_t tfLseek(int64_t tfd, int64_t offset, int32_t whence) { +// void *p = taosAcquireRef(tsFileRsetId, tfd); +// if (p == NULL) return -1; - int32_t fd = (int32_t)(uintptr_t)p; - int64_t ret = taosLSeekFile(fd, offset, whence); +// int32_t pFile = (TdFilePtr)(uintptr_t)p; +// int64_t ret = taosLSeekFile(fd, offset, whence); - taosReleaseRef(tsFileRsetId, tfd); - return ret; -} +// taosReleaseRef(tsFileRsetId, tfd); +// return ret; +// } -int32_t tfFtruncate(int64_t tfd, int64_t length) { - void *p = taosAcquireRef(tsFileRsetId, tfd); - if (p == NULL) return -1; +// int32_t tfFtruncate(int64_t tfd, int64_t length) { +// void *p = taosAcquireRef(tsFileRsetId, tfd); +// if (p == NULL) return -1; - int32_t fd = (int32_t)(uintptr_t)p; - int32_t code = taosFtruncateFile(fd, length); +// int32_t pFile = (TdFilePtr)(uintptr_t)p; +// int32_t code = taosFtruncateFile(fd, length); - taosReleaseRef(tsFileRsetId, tfd); - return code; -} +// taosReleaseRef(tsFileRsetId, tfd); +// return code; +// } -void *tfMmapReadOnly(int64_t tfd, int64_t length) { - void *p = taosAcquireRef(tsFileRsetId, tfd); - if (p == NULL) return NULL; - int32_t fd = (int32_t)(uintptr_t)p; +// void *tfMmapReadOnly(int64_t tfd, int64_t length) { +// void *p = taosAcquireRef(tsFileRsetId, tfd); +// if (p == NULL) return NULL; +// int32_t pFile = (TdFilePtr)(uintptr_t)p; - void *ptr = mmap(NULL, length, PROT_READ, MAP_SHARED, fd, 0); - taosReleaseRef(tsFileRsetId, tfd); - return ptr; -} +// void *ptr = mmap(NULL, length, PROT_READ, MAP_SHARED, fd, 0); +// taosReleaseRef(tsFileRsetId, tfd); +// return ptr; +// } diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index f5d02df80d..52593741b4 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -16,7 +16,6 @@ #define _DEFAULT_SOURCE #include "tlog.h" #include "os.h" -#include "tnote.h" #include "tutil.h" #include "ulog.h" @@ -48,7 +47,7 @@ typedef struct { int32_t buffEnd; int32_t buffSize; int32_t minBuffSize; - int32_t fd; + TdFilePtr pFile; int32_t stop; pthread_t asyncThread; pthread_mutex_t buffMutex; @@ -102,7 +101,7 @@ static SLogObj tsLogObj = {.fileNum = 1}; static void *taosAsyncOutputLog(void *param); static int32_t taosPushLogBuffer(SLogBuff *tLogBuff, char *msg, int32_t msgLen); static SLogBuff *taosLogBuffNew(int32_t bufSize); -static void taosCloseLogByFd(int32_t oldFd); +static void taosCloseLogByFd(TdFilePtr pFile); static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum); extern void taosPrintCfg(); static int32_t taosCompressFile(char *srcFileName, char *destFileName); @@ -119,7 +118,7 @@ static int32_t taosStartLog() { int32_t taosInitLog(const char *logName, int maxFiles) { if (tsLogInited) return 0; - taosUpdateLogSpace(); + osUpdate(); char fullName[PATH_MAX] = {0}; snprintf(fullName, PATH_MAX, "%s" TD_DIRSEP "%s", tsLogDir, logName); @@ -151,11 +150,11 @@ void taosCloseLog() { // taosCloseLog(); } -static bool taosLockLogFile(int32_t fd) { - if (fd < 0) return false; +static bool taosLockLogFile(TdFilePtr pFile) { + if (pFile == NULL) return false; if (tsLogObj.fileNum > 1) { - int32_t ret = taosLockFile(fd); + int32_t ret = taosLockFile(pFile); if (ret == 0) { return true; } @@ -164,11 +163,11 @@ static bool taosLockLogFile(int32_t fd) { return false; } -static void taosUnLockLogFile(int32_t fd) { - if (fd < 0) return; +static void taosUnLockLogFile(TdFilePtr pFile) { + if (pFile == NULL) return; if (tsLogObj.fileNum > 1) { - taosUnLockFile(fd); + taosUnLockFile(pFile); } } @@ -202,22 +201,22 @@ static void *taosThreadToOpenNewFile(void *param) { taosUmaskFile(0); - int32_t fd = taosOpenFileCreateWriteTrunc(name); - if (fd < 0) { + TdFilePtr pFile = taosOpenFile(name, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pFile == NULL) { tsLogObj.openInProgress = 0; tsLogObj.lines = tsLogObj.maxLines - 1000; - uError("open new log file fail! fd:%d reason:%s, reuse lastlog", fd, strerror(errno)); + uError("open new log file fail! reason:%s, reuse lastlog", strerror(errno)); return NULL; } - taosLockLogFile(fd); - (void)taosLSeekFile(fd, 0, SEEK_SET); + taosLockLogFile(pFile); + (void)taosLSeekFile(pFile, 0, SEEK_SET); - int32_t oldFd = tsLogObj.logHandle->fd; - tsLogObj.logHandle->fd = fd; + TdFilePtr pOldFile = tsLogObj.logHandle->pFile; + tsLogObj.logHandle->pFile = pFile; tsLogObj.lines = 0; tsLogObj.openInProgress = 0; - taosCloseLogByFd(oldFd); + taosCloseLogByFd(pOldFile); uInfo(" new log file:%d is opened", tsLogObj.flag); uInfo("=================================="); @@ -263,8 +262,8 @@ void taosResetLog() { } static bool taosCheckFileIsOpen(char *logFileName) { - int32_t fd = taosOpenFileWrite(logFileName); - if (fd < 0) { + TdFilePtr pFile = taosOpenFile(logFileName, TD_FILE_WRITE); + if (pFile == NULL) { if (errno == ENOENT) { return false; } else { @@ -273,12 +272,12 @@ static bool taosCheckFileIsOpen(char *logFileName) { } } - if (taosLockLogFile(fd)) { - taosUnLockLogFile(fd); - taosCloseFile(fd); + if (taosLockLogFile(pFile)) { + taosUnLockLogFile(pFile); + taosCloseFile(&pFile); return false; } else { - taosCloseFile(fd); + taosCloseFile(&pFile); return true; } } @@ -351,37 +350,37 @@ static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum) { pthread_mutex_init(&tsLogObj.logMutex, NULL); taosUmaskFile(0); - tsLogObj.logHandle->fd = taosOpenFileCreateWrite(fileName); + tsLogObj.logHandle->pFile = taosOpenFile(fileName, TD_FILE_CTEATE | TD_FILE_WRITE); - if (tsLogObj.logHandle->fd < 0) { + if (tsLogObj.logHandle->pFile == NULL) { printf("\nfailed to open log file:%s, reason:%s\n", fileName, strerror(errno)); return -1; } - taosLockLogFile(tsLogObj.logHandle->fd); + taosLockLogFile(tsLogObj.logHandle->pFile); // only an estimate for number of lines int64_t filesize = 0; - if (taosFStatFile(tsLogObj.logHandle->fd, &filesize, NULL) < 0) { + if (taosFStatFile(tsLogObj.logHandle->pFile, &filesize, NULL) < 0) { printf("\nfailed to fstat log file:%s, reason:%s\n", fileName, strerror(errno)); return -1; } size = (int32_t)filesize; tsLogObj.lines = size / 60; - taosLSeekFile(tsLogObj.logHandle->fd, 0, SEEK_END); + taosLSeekFile(tsLogObj.logHandle->pFile, 0, SEEK_END); sprintf(name, "==================================================\n"); - taosWriteFile(tsLogObj.logHandle->fd, name, (uint32_t)strlen(name)); + taosWriteFile(tsLogObj.logHandle->pFile, name, (uint32_t)strlen(name)); sprintf(name, " new log file \n"); - taosWriteFile(tsLogObj.logHandle->fd, name, (uint32_t)strlen(name)); + taosWriteFile(tsLogObj.logHandle->pFile, name, (uint32_t)strlen(name)); sprintf(name, "==================================================\n"); - taosWriteFile(tsLogObj.logHandle->fd, name, (uint32_t)strlen(name)); + taosWriteFile(tsLogObj.logHandle->pFile, name, (uint32_t)strlen(name)); return 0; } void taosPrintLog(const char *flags, int32_t dflag, const char *format, ...) { - if (!taosLogSpaceAvailable()) return; + if (!osLogSpaceAvailable()) return; va_list argpointer; char buffer[MAX_LOGLINE_BUFFER_SIZE] = {0}; @@ -417,11 +416,11 @@ void taosPrintLog(const char *flags, int32_t dflag, const char *format, ...) { buffer[len++] = '\n'; buffer[len] = 0; - if ((dflag & DEBUG_FILE) && tsLogObj.logHandle && tsLogObj.logHandle->fd >= 0) { + if ((dflag & DEBUG_FILE) && tsLogObj.logHandle && tsLogObj.logHandle->pFile >= 0) { if (tsAsyncLog) { taosPushLogBuffer(tsLogObj.logHandle, buffer, len); } else { - taosWriteFile(tsLogObj.logHandle->fd, buffer, len); + taosWriteFile(tsLogObj.logHandle->pFile, buffer, len); } if (tsLogObj.maxLines > 0) { @@ -431,12 +430,11 @@ void taosPrintLog(const char *flags, int32_t dflag, const char *format, ...) { } } - if (dflag & DEBUG_SCREEN) taosWriteFile(1, buffer, (uint32_t)len); - if (dflag == 255) nInfo(buffer, len); + if (dflag & DEBUG_SCREEN) write(1, buffer, (uint32_t)len); } void taosDumpData(unsigned char *msg, int32_t len) { - if (!taosLogSpaceAvailable()) return; + if (!osLogSpaceAvailable()) return; char temp[256]; int32_t i, pos = 0, c = 0; @@ -447,7 +445,7 @@ void taosDumpData(unsigned char *msg, int32_t len) { pos += 3; if (c >= 16) { temp[pos++] = '\n'; - taosWriteFile(tsLogObj.logHandle->fd, temp, (uint32_t)pos); + taosWriteFile(tsLogObj.logHandle->pFile, temp, (uint32_t)pos); c = 0; pos = 0; } @@ -455,11 +453,11 @@ void taosDumpData(unsigned char *msg, int32_t len) { temp[pos++] = '\n'; - taosWriteFile(tsLogObj.logHandle->fd, temp, (uint32_t)pos); + taosWriteFile(tsLogObj.logHandle->pFile, temp, (uint32_t)pos); } void taosPrintLongString(const char *flags, int32_t dflag, const char *format, ...) { - if (!taosLogSpaceAvailable()) return; + if (!osLogSpaceAvailable()) return; va_list argpointer; char buffer[MAX_LOGLINE_DUMP_BUFFER_SIZE]; @@ -485,11 +483,11 @@ void taosPrintLongString(const char *flags, int32_t dflag, const char *format, . buffer[len++] = '\n'; buffer[len] = 0; - if ((dflag & DEBUG_FILE) && tsLogObj.logHandle && tsLogObj.logHandle->fd >= 0) { + if ((dflag & DEBUG_FILE) && tsLogObj.logHandle && tsLogObj.logHandle->pFile >= 0) { if (tsAsyncLog) { taosPushLogBuffer(tsLogObj.logHandle, buffer, len); } else { - taosWriteFile(tsLogObj.logHandle->fd, buffer, len); + taosWriteFile(tsLogObj.logHandle->pFile, buffer, len); } if (tsLogObj.maxLines > 0) { @@ -499,19 +497,19 @@ void taosPrintLongString(const char *flags, int32_t dflag, const char *format, . } } - if (dflag & DEBUG_SCREEN) taosWriteFile(1, buffer, (uint32_t)len); + if (dflag & DEBUG_SCREEN) write(1, buffer, (uint32_t)len); } #if 0 void taosCloseLog() { - taosCloseLogByFd(tsLogObj.logHandle->fd); + taosCloseLogByFd(tsLogObj.logHandle->pFile); } #endif -static void taosCloseLogByFd(int32_t fd) { - if (fd >= 0) { - taosUnLockLogFile(fd); - taosCloseFile(fd); +static void taosCloseLogByFd(TdFilePtr pFile) { + if (pFile != NULL) { + taosUnLockLogFile(pFile); + taosCloseFile(&pFile); } } @@ -647,12 +645,12 @@ static void taosWriteLog(SLogBuff *tLogBuff) { } if (start < end) { - taosWriteFile(tLogBuff->fd, LOG_BUF_BUFFER(tLogBuff) + start, pollSize); + taosWriteFile(tLogBuff->pFile, LOG_BUF_BUFFER(tLogBuff) + start, pollSize); } else { int32_t tsize = LOG_BUF_SIZE(tLogBuff) - start; - taosWriteFile(tLogBuff->fd, LOG_BUF_BUFFER(tLogBuff) + start, tsize); + taosWriteFile(tLogBuff->pFile, LOG_BUF_BUFFER(tLogBuff) + start, tsize); - taosWriteFile(tLogBuff->fd, LOG_BUF_BUFFER(tLogBuff), end); + taosWriteFile(tLogBuff->pFile, LOG_BUF_BUFFER(tLogBuff), end); } dbgWN++; @@ -709,17 +707,17 @@ int32_t taosCompressFile(char *srcFileName, char *destFileName) { int32_t ret = 0; int32_t len = 0; char *data = malloc(compressSize); - FILE *srcFp = NULL; // gzFile dstFp = NULL; - srcFp = fopen(srcFileName, "r"); - if (srcFp == NULL) { + // srcFp = fopen(srcFileName, "r"); + TdFilePtr pSrcFile = taosOpenFile(srcFileName, TD_FILE_READ); + if (pSrcFile == NULL) { ret = -1; goto cmp_end; } - int32_t fd = taosOpenFileCreateWriteTrunc(destFileName); - if (fd < 0) { + TdFilePtr pFile = taosOpenFile(destFileName, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pFile == NULL) { ret = -2; goto cmp_end; } @@ -737,8 +735,8 @@ int32_t taosCompressFile(char *srcFileName, char *destFileName) { // } cmp_end: - if (srcFp) { - fclose(srcFp); + if (pSrcFile) { + taosCloseFile(&pSrcFile); } // if (dstFp) { // gzclose(dstFp); @@ -748,21 +746,6 @@ cmp_end: return ret; } -void taosPrintOsInfo() { - SysNameInfo info = taosGetSysNameInfo(); - - uInfo(" os pageSize: %" PRId64 "(KB)", tsPageSize); - uInfo(" os openMax: %" PRId64, tsOpenMax); - uInfo(" os streamMax: %" PRId64, tsStreamMax); - uInfo(" os numOfCores: %d", tsNumOfCores); - uInfo(" os totalMemory: %d(MB)", tsTotalMemoryMB); - uInfo(" os sysname: %s", info.sysname); - uInfo(" os nodename: %s", info.nodename); - uInfo(" os release: %s", info.release); - uInfo(" os version: %s", info.version); - uInfo(" os machine: %s", info.machine); -} - void taosSetAllDebugFlag(int32_t flag) { if (!(flag & DEBUG_TRACE || flag & DEBUG_DEBUG || flag & DEBUG_DUMP)) return; diff --git a/source/util/src/tnote.c b/source/util/src/tnote.c deleted file mode 100644 index d9356b7e40..0000000000 --- a/source/util/src/tnote.c +++ /dev/null @@ -1,268 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "tutil.h" -#include "tdef.h" -#include "tnote.h" - -SNoteObj tsHttpNote; -SNoteObj tsTscNote; -SNoteObj tsInfoNote; - -static int32_t taosOpenNoteWithMaxLines(char *fn, int32_t maxLines, int32_t maxNoteNum, SNoteObj *pNote); -static void taosCloseNoteByFd(int32_t oldFd, SNoteObj *pNote); - -static void taosInitNote(int32_t numOfLines, int32_t maxNotes, SNoteObj *pNote, char *name) { - memset(pNote, 0, sizeof(SNoteObj)); - pNote->fileNum = 1; - pNote->fd = -1; - - if (taosOpenNoteWithMaxLines(name, numOfLines, maxNotes, pNote) < 0) { - fprintf(stderr, "failed to init note file\n"); - } - - taosNotePrint(pNote, "=================================================="); - taosNotePrint(pNote, "=================== new note ==================="); - taosNotePrint(pNote, "=================================================="); -} - -int32_t taosInitNotes() { - char name[TSDB_FILENAME_LEN * 2] = {0}; - -#if 0 - if (tsTscEnableRecordSql) { - snprintf(name, TSDB_FILENAME_LEN * 2, "%s/tscsql-%d", tsLogDir, taosGetPId()); - taosInitNote(tsNumOfLogLines, 1, &tsTscNote, name); - } - -#endif - return 0; -} - -static bool taosLockNote(int32_t fd, SNoteObj *pNote) { - if (fd < 0) return false; - - if (pNote->fileNum > 1) { - int32_t ret = (int32_t)taosLockFile(fd); - if (ret == 0) { - return true; - } - } - - return false; -} - -static void taosUnLockNote(int32_t fd, SNoteObj *pNote) { - if (fd < 0) return; - - if (pNote->fileNum > 1) { - taosUnLockFile(fd); - } -} - -static void *taosThreadToOpenNewNote(void *param) { - char name[NOTE_FILE_NAME_LEN * 2]; - SNoteObj *pNote = (SNoteObj *)param; - - setThreadName("openNewNote"); - - pNote->flag ^= 1; - pNote->lines = 0; - sprintf(name, "%s.%d", pNote->name, pNote->flag); - - taosUmaskFile(0); - - int32_t fd = taosOpenFileCreateWriteTrunc(name); - if (fd < 0) { - return NULL; - } - - taosLockNote(fd, pNote); - (void)taosLSeekFile(fd, 0, SEEK_SET); - - int32_t oldFd = pNote->fd; - pNote->fd = fd; - pNote->lines = 0; - pNote->openInProgress = 0; - taosNotePrint(pNote, "=============== new note is opened ============="); - - taosCloseNoteByFd(oldFd, pNote); - return NULL; -} - -static int32_t taosOpenNewNote(SNoteObj *pNote) { - pthread_mutex_lock(&pNote->mutex); - - if (pNote->lines > pNote->maxLines && pNote->openInProgress == 0) { - pNote->openInProgress = 1; - - taosNotePrint(pNote, "=============== open new note =================="); - pthread_t pattern; - pthread_attr_t attr; - pthread_attr_init(&attr); - pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); - - pthread_create(&pattern, &attr, taosThreadToOpenNewNote, (void *)pNote); - pthread_attr_destroy(&attr); - } - - pthread_mutex_unlock(&pNote->mutex); - - return pNote->fd; -} - -static bool taosCheckNoteIsOpen(char *noteName, SNoteObj *pNote) { - int32_t fd = taosOpenFileCreateWrite(noteName); - if (fd < 0) { - fprintf(stderr, "failed to open note:%s reason:%s\n", noteName, strerror(errno)); - return true; - } - - if (taosLockNote(fd, pNote)) { - taosUnLockNote(fd, pNote); - taosCloseFile(fd); - return false; - } else { - taosCloseFile(fd); - return true; - } -} - -static void taosGetNoteName(char *fn, SNoteObj *pNote) { - if (pNote->fileNum > 1) { - for (int32_t i = 0; i < pNote->fileNum; i++) { - char fileName[NOTE_FILE_NAME_LEN]; - - sprintf(fileName, "%s%d.0", fn, i); - bool file1open = taosCheckNoteIsOpen(fileName, pNote); - - sprintf(fileName, "%s%d.1", fn, i); - bool file2open = taosCheckNoteIsOpen(fileName, pNote); - - if (!file1open && !file2open) { - sprintf(pNote->name, "%s%d", fn, i); - return; - } - } - } - - if (strlen(fn) < NOTE_FILE_NAME_LEN) { - strcpy(pNote->name, fn); - } -} - -static int32_t taosOpenNoteWithMaxLines(char *fn, int32_t maxLines, int32_t maxNoteNum, SNoteObj *pNote) { - char name[NOTE_FILE_NAME_LEN * 2] = {0}; - int32_t size; - int32_t logstat0_mtime, logstat1_mtime; - - pNote->maxLines = maxLines; - pNote->fileNum = maxNoteNum; - taosGetNoteName(fn, pNote); - - if (strlen(fn) < NOTE_FILE_NAME_LEN + 50 - 2) { - strcpy(name, fn); - strcat(name, ".0"); - } - bool log0Exist = taosStatFile(name, NULL, &logstat0_mtime) >= 0; - - if (strlen(fn) < NOTE_FILE_NAME_LEN + 50 - 2) { - strcpy(name, fn); - strcat(name, ".1"); - } - bool log1Exist = taosStatFile(name, NULL, &logstat1_mtime) >= 0; - - if (!log0Exist && !log1Exist) { - pNote->flag = 0; - } else if (!log1Exist) { - pNote->flag = 0; - } else if (!log0Exist) { - pNote->flag = 1; - } else { - pNote->flag = (logstat0_mtime > logstat1_mtime) ? 0 : 1; - } - - char noteName[NOTE_FILE_NAME_LEN * 2] = {0}; - sprintf(noteName, "%s.%d", pNote->name, pNote->flag); - pthread_mutex_init(&pNote->mutex, NULL); - - taosUmaskFile(0); - pNote->fd = taosOpenFileCreateWrite(noteName); - - if (pNote->fd < 0) { - fprintf(stderr, "failed to open note file:%s reason:%s\n", noteName, strerror(errno)); - return -1; - } - taosLockNote(pNote->fd, pNote); - - // only an estimate for number of lines - int64_t filestat_size; - if (taosFStatFile(pNote->fd, &filestat_size, NULL) < 0) { - fprintf(stderr, "failed to fstat note file:%s reason:%s\n", noteName, strerror(errno)); - return -1; - } - size = (int32_t)filestat_size; - pNote->lines = size / 60; - - taosLSeekFile(pNote->fd, 0, SEEK_END); - - return 0; -} - -void taosNotePrintBuffer(SNoteObj *pNote, char *buffer, int32_t len) { - if (pNote->fd <= 0) return; - taosWriteFile(pNote->fd, buffer, len); - - if (pNote->maxLines > 0) { - pNote->lines++; - if ((pNote->lines > pNote->maxLines) && (pNote->openInProgress == 0)) taosOpenNewNote(pNote); - } -} - -void taosNotePrint(SNoteObj *pNote, const char *const format, ...) { - va_list argpointer; - char buffer[MAX_NOTE_LINE_SIZE + 2]; - int32_t len; - struct tm Tm, *ptm; - struct timeval timeSecs; - time_t curTime; - - taosGetTimeOfDay(&timeSecs); - curTime = timeSecs.tv_sec; - ptm = localtime_r(&curTime, &Tm); - len = sprintf(buffer, "%02d/%02d %02d:%02d:%02d.%06d %08" PRId64 " ", ptm->tm_mon + 1, ptm->tm_mday, ptm->tm_hour, - ptm->tm_min, ptm->tm_sec, (int32_t)timeSecs.tv_usec, taosGetSelfPthreadId()); - va_start(argpointer, format); - len += vsnprintf(buffer + len, MAX_NOTE_LINE_SIZE - len, format, argpointer); - va_end(argpointer); - - if (len >= MAX_NOTE_LINE_SIZE) len = MAX_NOTE_LINE_SIZE - 2; - - buffer[len++] = '\n'; - buffer[len] = 0; - - taosNotePrintBuffer(pNote, buffer, len); -} - -// static void taosCloseNote(SNoteObj *pNote) { taosCloseNoteByFd(pNote->fd, pNote); } - -static void taosCloseNoteByFd(int32_t fd, SNoteObj *pNote) { - if (fd >= 0) { - taosUnLockNote(fd, pNote); - taosCloseFile(fd); - } -} diff --git a/source/util/src/tpagedbuf.c b/source/util/src/tpagedbuf.c index 0e8d85492c..ca260f0df5 100644 --- a/source/util/src/tpagedbuf.c +++ b/source/util/src/tpagedbuf.c @@ -32,7 +32,7 @@ typedef struct SDiskbasedBuf { int32_t numOfPages; int64_t totalBufSize; uint64_t fileSize; // disk file size - FILE* file; + TdFilePtr pFile; int32_t allocateId; // allocated page id char* path; // file path int32_t pageSize; // current used page size @@ -67,7 +67,7 @@ static void printStatisData(const SDiskbasedBuf* pBuf); pResBuf->inMemPages = inMemBufSize/pagesize; // maximum allowed pages, it is a soft limit. pResBuf->allocateId = -1; pResBuf->comp = true; - pResBuf->file = NULL; + pResBuf->pFile = NULL; pResBuf->qId = qId; pResBuf->fileSize = 0; @@ -94,8 +94,9 @@ static void printStatisData(const SDiskbasedBuf* pBuf); } static int32_t createDiskFile(SDiskbasedBuf* pBuf) { - pBuf->file = fopen(pBuf->path, "wb+"); - if (pBuf->file == NULL) { + // pBuf->file = fopen(pBuf->path, "wb+"); + pBuf->pFile = taosOpenFile(pBuf->path, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC); + if (pBuf->pFile == NULL) { // qError("failed to create tmp file: %s on disk. %s", pBuf->path, strerror(errno)); return TAOS_SYSTEM_ERROR(errno); } @@ -168,13 +169,13 @@ static char* doFlushPageToDisk(SDiskbasedBuf* pBuf, SPageInfo* pg) { pg->offset = allocatePositionInFile(pBuf, size); pBuf->nextPos += size; - int32_t ret = fseek(pBuf->file, pg->offset, SEEK_SET); + int32_t ret = taosLSeekFile(pBuf->pFile, pg->offset, SEEK_SET); if (ret != 0) { terrno = TAOS_SYSTEM_ERROR(errno); return NULL; } - ret = (int32_t) fwrite(t, 1, size, pBuf->file); + ret = (int32_t) taosWriteFile(pBuf->pFile, t, size); if (ret != size) { terrno = TAOS_SYSTEM_ERROR(errno); return NULL; @@ -199,13 +200,13 @@ static char* doFlushPageToDisk(SDiskbasedBuf* pBuf, SPageInfo* pg) { } // 3. write to disk. - int32_t ret = fseek(pBuf->file, pg->offset, SEEK_SET); + int32_t ret = taosLSeekFile(pBuf->pFile, pg->offset, SEEK_SET); if (ret != 0) { terrno = TAOS_SYSTEM_ERROR(errno); return NULL; } - ret = (int32_t)fwrite(t, 1, size, pBuf->file); + ret = (int32_t) taosWriteFile(pBuf->pFile, t, size); if (ret != size) { terrno = TAOS_SYSTEM_ERROR(errno); return NULL; @@ -233,7 +234,7 @@ static char* flushPageToDisk(SDiskbasedBuf* pBuf, SPageInfo* pg) { int32_t ret = TSDB_CODE_SUCCESS; assert(((int64_t) pBuf->numOfPages * pBuf->pageSize) == pBuf->totalBufSize && pBuf->numOfPages >= pBuf->inMemPages); - if (pBuf->file == NULL) { + if (pBuf->pFile == NULL) { if ((ret = createDiskFile(pBuf)) != TSDB_CODE_SUCCESS) { terrno = ret; return NULL; @@ -245,14 +246,14 @@ static char* flushPageToDisk(SDiskbasedBuf* pBuf, SPageInfo* pg) { // load file block data in disk static int32_t loadPageFromDisk(SDiskbasedBuf* pBuf, SPageInfo* pg) { - int32_t ret = fseek(pBuf->file, pg->offset, SEEK_SET); + int32_t ret = taosLSeekFile(pBuf->pFile, pg->offset, SEEK_SET); if (ret != 0) { ret = TAOS_SYSTEM_ERROR(errno); return ret; } SFilePage* pPage = (SFilePage*) GET_DATA_PAYLOAD(pg); - ret = (int32_t)fread(pPage->data, 1, pg->length, pBuf->file); + ret = (int32_t)taosReadFile(pBuf->pFile, pPage->data, pg->length); if (ret != pg->length) { ret = TAOS_SYSTEM_ERROR(errno); return ret; @@ -495,12 +496,12 @@ void destroyResultBuf(SDiskbasedBuf* pBuf) { printStatisData(pBuf); - if (pBuf->file != NULL) { + if (pBuf->pFile != NULL) { uDebug("Paged buffer closed, total:%.2f Kb (%d Pages), inmem size:%.2f Kb (%d Pages), file size:%.2f Kb, page size:%.2f Kb, %"PRIx64"\n", pBuf->totalBufSize/1024.0, pBuf->numOfPages, listNEles(pBuf->lruList) * pBuf->pageSize / 1024.0, listNEles(pBuf->lruList), pBuf->fileSize/1024.0, pBuf->pageSize/1024.0f, pBuf->qId); - fclose(pBuf->file); + taosCloseFile(&pBuf->pFile); } else { uDebug("Paged buffer closed, total:%.2f Kb, no file created, %"PRIx64, pBuf->totalBufSize/1024.0, pBuf->qId); } diff --git a/source/util/src/ttimer.c b/source/util/src/ttimer.c index ca0c6e9a24..2c04603269 100644 --- a/source/util/src/ttimer.c +++ b/source/util/src/ttimer.c @@ -111,7 +111,7 @@ typedef struct time_wheel_t { tmr_obj_t** slots; } time_wheel_t; -uint32_t tsMaxTmrCtrl = 512; +int32_t tsMaxTmrCtrl = 512; static pthread_once_t tmrModuleInit = PTHREAD_ONCE_INIT; static pthread_mutex_t tmrCtrlMutex; diff --git a/source/util/test/CMakeLists.txt b/source/util/test/CMakeLists.txt index ee0ade03b8..d6b779b6e3 100644 --- a/source/util/test/CMakeLists.txt +++ b/source/util/test/CMakeLists.txt @@ -51,4 +51,12 @@ target_link_libraries(queue_test os util gtest_main) add_test( NAME queue_test COMMAND queue_test +) + +# cfgTest +add_executable(cfgTest "cfgTest.cpp") +target_link_libraries(cfgTest os util gtest_main) +add_test( + NAME cfgTest + COMMAND cfgTest ) \ No newline at end of file diff --git a/source/libs/config/test/cfgTest.cpp b/source/util/test/cfgTest.cpp similarity index 81% rename from source/libs/config/test/cfgTest.cpp rename to source/util/test/cfgTest.cpp index 7c70e1a8f8..c352a4c21b 100644 --- a/source/libs/config/test/cfgTest.cpp +++ b/source/util/test/cfgTest.cpp @@ -10,7 +10,7 @@ */ #include -#include "config.h" +#include "tconfig.h" class CfgTest : public ::testing::Test { protected: @@ -43,7 +43,6 @@ TEST_F(CfgTest, 01_Str) { EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_INT64), "int64"); EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_FLOAT), "float"); EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_STRING), "string"); - EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_IPSTR), "ipstr"); EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_DIR), "dir"); EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_DIR), "dir"); EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_DIR), "dir"); @@ -55,15 +54,14 @@ TEST_F(CfgTest, 02_Basic) { SConfig *pConfig = cfgInit(); ASSERT_NE(pConfig, nullptr); - EXPECT_EQ(cfgAddBool(pConfig, "test_bool", 0), 0); - EXPECT_EQ(cfgAddInt32(pConfig, "test_int32", 1, 0, 16), 0); - EXPECT_EQ(cfgAddInt64(pConfig, "test_int64", 2, 0, 16), 0); - EXPECT_EQ(cfgAddFloat(pConfig, "test_float", 3, 0, 16), 0); - EXPECT_EQ(cfgAddString(pConfig, "test_string", "4"), 0); - EXPECT_EQ(cfgAddIpStr(pConfig, "test_ipstr", "192.168.0.1"), 0); - EXPECT_EQ(cfgAddDir(pConfig, "test_dir", "/tmp"), 0); + EXPECT_EQ(cfgAddBool(pConfig, "test_bool", 0, 0), 0); + EXPECT_EQ(cfgAddInt32(pConfig, "test_int32", 1, 0, 16, 0), 0); + EXPECT_EQ(cfgAddInt64(pConfig, "test_int64", 2, 0, 16, 0), 0); + EXPECT_EQ(cfgAddFloat(pConfig, "test_float", 3, 0, 16, 0), 0); + EXPECT_EQ(cfgAddString(pConfig, "test_string", "4", 0), 0); + EXPECT_EQ(cfgAddDir(pConfig, "test_dir", "/tmp", 0), 0); - EXPECT_EQ(cfgGetSize(pConfig), 7); + EXPECT_EQ(cfgGetSize(pConfig), 6); int32_t size = 0; SConfigItem *pItem = cfgIterate(pConfig, NULL); @@ -84,9 +82,6 @@ TEST_F(CfgTest, 02_Basic) { case CFG_DTYPE_STRING: printf("index:%d, cfg:%s value:%s\n", size, pItem->name, pItem->str); break; - case CFG_DTYPE_IPSTR: - printf("index:%d, cfg:%s value:%s\n", size, pItem->name, pItem->str); - break; case CFG_DTYPE_DIR: printf("index:%d, cfg:%s value:%s\n", size, pItem->name, pItem->str); break; @@ -99,7 +94,7 @@ TEST_F(CfgTest, 02_Basic) { } cfgCancelIterate(pConfig, pItem); - EXPECT_EQ(cfgGetSize(pConfig), 7); + EXPECT_EQ(cfgGetSize(pConfig), 6); pItem = cfgGetItem(pConfig, "test_bool"); EXPECT_EQ(pItem->stype, CFG_STYPE_DEFAULT); @@ -131,12 +126,6 @@ TEST_F(CfgTest, 02_Basic) { EXPECT_STREQ(pItem->name, "test_string"); EXPECT_STREQ(pItem->str, "4"); - pItem = cfgGetItem(pConfig, "test_ipstr"); - EXPECT_EQ(pItem->stype, CFG_STYPE_DEFAULT); - EXPECT_EQ(pItem->dtype, CFG_DTYPE_IPSTR); - EXPECT_STREQ(pItem->name, "test_ipstr"); - EXPECT_STREQ(pItem->str, "192.168.0.1"); - pItem = cfgGetItem(pConfig, "test_dir"); EXPECT_EQ(pItem->stype, CFG_STYPE_DEFAULT); EXPECT_EQ(pItem->dtype, CFG_DTYPE_DIR); diff --git a/tests b/tests index 498e17e225..12233db374 160000 --- a/tests +++ b/tests @@ -1 +1 @@ -Subproject commit 498e17e225c454f2b45ff16eaa6d3eb595b107cf +Subproject commit 12233db374f1fe97b327e89a3442c631578ad38d diff --git a/tools/shell/src/backup/shellCheck.c b/tools/shell/src/backup/shellCheck.c index 7fc8b1409a..33d25b6746 100644 --- a/tools/shell/src/backup/shellCheck.c +++ b/tools/shell/src/backup/shellCheck.c @@ -116,7 +116,7 @@ static void *shellCheckThreadFp(void *arg) { char file[32] = {0}; snprintf(file, 32, "tb%d.txt", pThread->threadIndex); - FILE *fp = fopen(file, "w"); + TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); if (!fp) { fprintf(stdout, "failed to open %s, reason:%s", file, strerror(errno)); return NULL; @@ -133,7 +133,7 @@ static void *shellCheckThreadFp(void *arg) { int32_t code = taos_errno(pSql); if (code != 0) { int32_t len = snprintf(sql, SHELL_SQL_LEN, "drop table %s.%s;\n", pThread->db, tbname); - fwrite(sql, 1, len, fp); + taosWriteFile(pFile, sql, len); atomic_add_fetch_32(&errorNum, 1); } @@ -145,8 +145,8 @@ static void *shellCheckThreadFp(void *arg) { taos_free_result(pSql); } - taosFsync(fileno(fp)); - fclose(fp); + taosFsync(pFile); + taosCloseFile(&pFile); return NULL; } diff --git a/tools/shell/src/backup/shellImport.c b/tools/shell/src/backup/shellImport.c index 222d69e854..ce15212f86 100644 --- a/tools/shell/src/backup/shellImport.c +++ b/tools/shell/src/backup/shellImport.c @@ -138,7 +138,6 @@ static void shellSourceFile(TAOS *con, char *fptr) { char * cmd = malloc(tsMaxSQLStringLen); size_t cmd_len = 0; char * line = NULL; - size_t line_len = 0; if (wordexp(fptr, &full_path, 0) != 0) { fprintf(stderr, "ERROR: illegal file name\n"); @@ -171,8 +170,9 @@ static void shellSourceFile(TAOS *con, char *fptr) { } */ - FILE *f = fopen(fname, "r"); - if (f == NULL) { + // FILE *f = fopen(fname, "r"); + TdFilePtr pFile = taosOpenFile(fname, TD_FILE_READ); + if (pFile == NULL) { fprintf(stderr, "ERROR: failed to open file %s\n", fname); wordfree(&full_path); free(cmd); @@ -182,7 +182,7 @@ static void shellSourceFile(TAOS *con, char *fptr) { fprintf(stdout, "begin import file:%s\n", fname); int lineNo = 0; - while ((read_len = getline(&line, &line_len, f)) != -1) { + while ((read_len = taosGetLineFile(pFile, &line)) != -1) { ++lineNo; if (read_len >= tsMaxSQLStringLen) continue; line[--read_len] = '\0'; @@ -215,9 +215,9 @@ static void shellSourceFile(TAOS *con, char *fptr) { } free(cmd); - if (line) free(line); + if(line != NULL) free(line); wordfree(&full_path); - fclose(f); + taosCloseFile(&pFile); } void* shellImportThreadFp(void *arg) diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 4186f0dae1..e43267f032 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -459,44 +459,44 @@ static char *formatTimestamp(char *buf, int64_t val, int precision) { return buf; } -static void dumpFieldToFile(FILE *fp, const char *val, TAOS_FIELD *field, int32_t length, int precision) { +static void dumpFieldToFile(TdFilePtr pFile, const char *val, TAOS_FIELD *field, int32_t length, int precision) { if (val == NULL) { - fprintf(fp, "%s", TSDB_DATA_NULL_STR); + taosFprintfFile(pFile, "%s", TSDB_DATA_NULL_STR); return; } char buf[TSDB_MAX_BYTES_PER_ROW]; switch (field->type) { case TSDB_DATA_TYPE_BOOL: - fprintf(fp, "%d", ((((int32_t)(*((char *)val))) == 1) ? 1 : 0)); + taosFprintfFile(pFile, "%d", ((((int32_t)(*((char *)val))) == 1) ? 1 : 0)); break; case TSDB_DATA_TYPE_TINYINT: - fprintf(fp, "%d", *((int8_t *)val)); + taosFprintfFile(pFile, "%d", *((int8_t *)val)); break; case TSDB_DATA_TYPE_SMALLINT: - fprintf(fp, "%d", *((int16_t *)val)); + taosFprintfFile(pFile, "%d", *((int16_t *)val)); break; case TSDB_DATA_TYPE_INT: - fprintf(fp, "%d", *((int32_t *)val)); + taosFprintfFile(pFile, "%d", *((int32_t *)val)); break; case TSDB_DATA_TYPE_BIGINT: - fprintf(fp, "%" PRId64, *((int64_t *)val)); + taosFprintfFile(pFile, "%" PRId64, *((int64_t *)val)); break; case TSDB_DATA_TYPE_FLOAT: - fprintf(fp, "%.5f", GET_FLOAT_VAL(val)); + taosFprintfFile(pFile, "%.5f", GET_FLOAT_VAL(val)); break; case TSDB_DATA_TYPE_DOUBLE: - fprintf(fp, "%.9f", GET_DOUBLE_VAL(val)); + taosFprintfFile(pFile, "%.9f", GET_DOUBLE_VAL(val)); break; case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_NCHAR: memcpy(buf, val, length); buf[length] = 0; - fprintf(fp, "\'%s\'", buf); + taosFprintfFile(pFile, "\'%s\'", buf); break; case TSDB_DATA_TYPE_TIMESTAMP: formatTimestamp(buf, *(int64_t *)val, precision); - fprintf(fp, "'%s'", buf); + taosFprintfFile(pFile, "'%s'", buf); break; default: break; @@ -516,8 +516,9 @@ static int dumpResultToFile(const char *fname, TAOS_RES *tres) { return -1; } - FILE *fp = fopen(full_path.we_wordv[0], "w"); - if (fp == NULL) { + // FILE *fp = fopen(full_path.we_wordv[0], "w"); + TdFilePtr pFile = taosOpenFile(full_path.we_wordv[0], TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pFile == NULL) { fprintf(stderr, "ERROR: failed to open file: %s\n", full_path.we_wordv[0]); wordfree(&full_path); return -1; @@ -531,29 +532,29 @@ static int dumpResultToFile(const char *fname, TAOS_RES *tres) { for (int col = 0; col < num_fields; col++) { if (col > 0) { - fprintf(fp, ","); + taosFprintfFile(pFile, ","); } - fprintf(fp, "%s", fields[col].name); + taosFprintfFile(pFile, "%s", fields[col].name); } - fputc('\n', fp); + taosFprintfFile(pFile, "\n"); int numOfRows = 0; do { int32_t *length = taos_fetch_lengths(tres); for (int i = 0; i < num_fields; i++) { if (i > 0) { - fputc(',', fp); + taosFprintfFile(pFile, "\n"); } - dumpFieldToFile(fp, (const char *)row[i], fields + i, length[i], precision); + dumpFieldToFile(pFile, (const char *)row[i], fields + i, length[i], precision); } - fputc('\n', fp); + taosFprintfFile(pFile, "\n"); numOfRows++; row = taos_fetch_row(tres); } while (row != NULL); result = 0; - fclose(fp); + taosCloseFile(&pFile); return numOfRows; } @@ -897,14 +898,14 @@ void read_history() { history.hstart = 0; history.hend = 0; char *line = NULL; - size_t line_size = 0; int read_size = 0; char f_history[TSDB_FILENAME_LEN]; get_history_path(f_history); - FILE *f = fopen(f_history, "r"); - if (f == NULL) { + // FILE *f = fopen(f_history, "r"); + TdFilePtr pFile = taosOpenFile(f_history, TD_FILE_READ); + if (pFile == NULL) { #ifndef WINDOWS if (errno != ENOENT) { fprintf(stderr, "Failed to open file %s, reason:%s\n", f_history, strerror(errno)); @@ -913,7 +914,7 @@ void read_history() { return; } - while ((read_size = tgetline(&line, &line_size, f)) != -1) { + while ((read_size = taosGetLineFile(pFile, &line)) != -1) { line[read_size - 1] = '\0'; history.hist[history.hend] = strdup(line); @@ -924,16 +925,17 @@ void read_history() { } } - free(line); - fclose(f); + if(line != NULL) free(line); + taosCloseFile(&pFile); } void write_history() { char f_history[TSDB_FILENAME_LEN]; get_history_path(f_history); - FILE *f = fopen(f_history, "w"); - if (f == NULL) { + // FILE *f = fopen(f_history, "w"); + TdFilePtr pFile = taosOpenFile(f_history, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pFile == NULL) { #ifndef WINDOWS fprintf(stderr, "Failed to open file %s for write, reason:%s\n", f_history, strerror(errno)); #endif @@ -942,12 +944,12 @@ void write_history() { for (int i = history.hstart; i != history.hend;) { if (history.hist[i] != NULL) { - fprintf(f, "%s\n", history.hist[i]); + taosFprintfFile(pFile, "%s\n", history.hist[i]); tfree(history.hist[i]); } i = (i + 1) % MAX_HISTORY_SIZE; } - fclose(f); + taosCloseFile(&pFile); } void taos_error(TAOS_RES *tres, int64_t st) { @@ -969,7 +971,6 @@ void source_file(TAOS *con, char *fptr) { char *cmd = calloc(1, TSDB_MAX_ALLOWED_SQL_LEN + 1); size_t cmd_len = 0; char *line = NULL; - size_t line_len = 0; if (wordexp(fptr, &full_path, 0) != 0) { fprintf(stderr, "ERROR: illegal file name\n"); @@ -989,15 +990,16 @@ void source_file(TAOS *con, char *fptr) { } */ - FILE *f = fopen(fname, "r"); - if (f == NULL) { + // FILE *f = fopen(fname, "r"); + TdFilePtr pFile = taosOpenFile(fname, TD_FILE_READ); + if (pFile == NULL) { fprintf(stderr, "ERROR: failed to open file %s\n", fname); wordfree(&full_path); free(cmd); return; } - while ((read_len = tgetline(&line, &line_len, f)) != -1) { + while ((read_len = taosGetLineFile(pFile, &line)) != -1) { if (read_len >= TSDB_MAX_ALLOWED_SQL_LEN) continue; line[--read_len] = '\0'; @@ -1020,9 +1022,9 @@ void source_file(TAOS *con, char *fptr) { } free(cmd); - if (line) free(line); + if(line != NULL) free(line); wordfree(&full_path); - fclose(f); + taosCloseFile(&pFile); } void shellGetGrantInfo(void *con) { diff --git a/tools/shell/src/shellLinux.c b/tools/shell/src/shellLinux.c index 89b477f08a..b06109184e 100644 --- a/tools/shell/src/shellLinux.c +++ b/tools/shell/src/shellLinux.c @@ -184,7 +184,6 @@ static void parse_args( for (int i = 1; i < argc; i++) { if ((strncmp(argv[i], "-p", 2) == 0) || (strncmp(argv[i], "--password", 10) == 0)) { - strcpy(tsOsName, "Linux"); printf(LINUXCLIENT_VERSION, tsOsName, taos_get_client_info()); if ((strlen(argv[i]) == 2) || (strncmp(argv[i], "--password", 10) == 0)) {