xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

locale

This commit is contained in:
Shengliang Guan 2022-02-24 13:51:12 +08:00
parent 9aaa9d205b
commit 7778830966
15 changed files with 217 additions and 191 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
include/os/os.h
View File

@ -59,6 +59,7 @@ extern "C" {
#include "osEndian.h"
#include "osEnv.h"
#include "osFile.h"
#include "osLocale.h"
#include "osLz4.h"
#include "osMath.h"
#include "osMemory.h"

2
include/os/osEnv.h
View File

@ -48,6 +48,8 @@ char *osDataDir();
char *osName();
char *osTimezone();
int8_t osDaylight();
char *osLocale();
char *osCharset();
void osSetTimezone(const char*timezone);

15
include/common/tlocale.h → include/os/osLocale.h
View File

@ -13,17 +13,22 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_COMMON_LOCALE_H_
#define _TD_COMMON_LOCALE_H_
#ifndef _TD_OS_LOCALE_H_
#define _TD_OS_LOCALE_H_
#include "os.h"
#include "osString.h"
#ifdef __cplusplus
extern "C" {
#endif
void tsSetLocale();
char *taosCharsetReplace(char *charsetstr);
void taosGetSystemLocale(char *outLocale, char *outCharset);
void taosSetSystemLocale(const char *inLocale, const char *inCharSet);
#ifdef __cplusplus
}
#endif /*_TD_COMMON_LOCALE_H_*/
#endif
#endif /*_TD_OS_LOCALE_H_*/

1
include/os/osString.h
View File

@ -45,7 +45,6 @@ int32_t taosUcs4ToMbs(void *ucs4, int32_t ucs4_max_len, char *mbs);
bool taosMbsToUcs4(const char *mbs, size_t mbs_len, char *ucs4, int32_t ucs4_max_len, int32_t *len);
int32_t tasoUcs4Compare(void *f1_ucs4, void *f2_ucs4, int32_t bytes, int8_t ncharSize);
bool taosValidateEncodec(const char *encodec);
char * taosCharsetReplace(char *charsetstr);
#ifdef __cplusplus
}

2
include/os/osSysinfo.h
View File

@ -42,8 +42,6 @@ extern int64_t tsOpenMax;
extern int64_t tsStreamMax;
extern int32_t tsNumOfCores;
extern int32_t tsTotalMemoryMB;
extern char tsLocale[];
extern char tsCharset[]; // default encode string

6
include/os/osTimezone.h
View File

@ -20,11 +20,11 @@
extern "C" {
#endif
void osGetSystemTimezone(char *outTimezone);
void osSetSystemTimezone(const char *inTimezone, char *outTimezone, int8_t *outDaylight);
void taosGetSystemTimezone(char *outTimezone);
void taosSetSystemTimezone(const char *inTimezone, char *outTimezone, int8_t *outDaylight);
#ifdef __cplusplus
}
#endif
#endif /*_TD_OS_SOCKET_H_*/
#endif /*_TD_OS_TIMEZONE_H_*/

1
source/common/src/tglobal.c
View File

@ -21,7 +21,6 @@
#include "tcompare.h"
#include "tep.h"
#include "tglobal.h"
#include "tlocale.h"
#include "tlog.h"
#include "tutil.h"
#include "ulog.h"

45
source/common/src/tlocale.c
View File

@ -1,45 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "os.h"
#include "ulog.h"
#include "tglobal.h"
#include "tutil.h"
/**
* In some Linux systems, setLocale(LC_CTYPE, "") may return NULL, in which case the launch of
* both the TDengine Server and the Client may be interrupted.
*
* In case that the setLocale failed to be executed, the right charset needs to be set.
*/
void tsSetLocale() {
char *locale = setlocale(LC_CTYPE, tsLocale);
// default locale or user specified locale is not valid, abort launch
if (locale == NULL) {
uError("Invalid locale:%s, please set the valid locale in config file", tsLocale);
}
if (strlen(tsCharset) == 0) {
uError("failed to get charset, please set the valid charset in config file");
exit(-1);
}
if (!taosValidateEncodec(tsCharset)) {
uError("Invalid charset:%s, please set the valid charset in config file", tsCharset);
exit(-1);
}
}

2
source/dnode/mgmt/daemon/src/dmnCfg.c
View File

@ -15,7 +15,6 @@
#define _DEFAULT_SOURCE
#include "dmnInt.h"
#include "tlocale.h"
static int32_t dmnAddEpCfg(SConfig *pCfg) {
char defaultFqdn[TSDB_FQDN_LEN] = {0};
@ -105,7 +104,6 @@ static int32_t dmnCheckCfg(SConfig *pCfg) {
taosGetSystemInfo();
tsSetLocale();
if (tsNumOfCores <= 0) {
tsNumOfCores = 1;

1
source/libs/config/src/config.c
View File

@ -16,7 +16,6 @@
#define _DEFAULT_SOURCE
#include "cfgInt.h"
#include "tep.h"
#include "tlocale.h"
#include "tmsg.h"
#define CFG_NAME_PRINT_LEN 22

7
source/os/src/osEnv.c
View File

@ -22,7 +22,8 @@ char configDir[PATH_MAX] = {0};
SOsEnv *osEnv() { return &env; }
void osInitImp() {
osGetSystemTimezone(env.timezone);
taosGetSystemLocale(env.locale, env.charset);
taosGetSystemTimezone(env.timezone);
osSetTimezone(env.timezone);
}
@ -45,10 +46,12 @@ char *osTempDir() { return env.tempDir; }
char *osDataDir() { return env.dataDir; }
char *osName() { return env.osName; }
char *osTimezone() { return env.timezone; }
char *osLocale() { return env.locale; }
char *osCharset() { return env.charset; }
int8_t osDaylight() { return env.daylight; }
void osSetTimezone(const char *timezone) { osSetSystemTimezone(timezone, env.timezone, &env.daylight); }
void osSetTimezone(const char *timezone) { taosSetSystemTimezone(timezone, env.timezone, &env.daylight); }
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)

194
source/os/src/osLocale.c Normal file
View File

@ -0,0 +1,194 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "osLocale.h"
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
#if (_WIN64)
#include <iphlpapi.h>
#include <mswsock.h>
#include <psapi.h>
#include <stdio.h>
#include <windows.h>
#include <ws2tcpip.h>
#pragma comment(lib, "Mswsock.lib ")
#endif
#include <objbase.h>
#pragma warning(push)
#pragma warning(disable : 4091)
#include <DbgHelp.h>
#pragma warning(pop)
#elif defined(_TD_DARWIN_64)
#include <errno.h>
#include <libproc.h>
#else
#include <argp.h>
#include <linux/sysctl.h>
#include <sys/file.h>
#include <sys/resource.h>
#include <sys/statvfs.h>
#include <sys/syscall.h>
#include <sys/utsname.h>
#include <unistd.h>
#endif
typedef struct CharsetPair {
char *oldCharset;
char *newCharset;
} CharsetPair;
char *taosCharsetReplace(char *charsetstr) {
CharsetPair charsetRep[] = {
{"utf8", "UTF-8"},
{"936", "CP936"},
};
for (int32_t i = 0; i < tListLen(charsetRep); ++i) {
if (strcasecmp(charsetRep[i].oldCharset, charsetstr) == 0) {
return strdup(charsetRep[i].newCharset);
}
}
return strdup(charsetstr);
}
/**
* In some Linux systems, setLocale(LC_CTYPE, "") may return NULL, in which case the launch of
* both the TDengine Server and the Client may be interrupted.
*
* In case that the setLocale failed to be executed, the right charset needs to be set.
*/
void taosSetSystemLocale(const char *inLocale, const char *inCharSet) {
char *locale = setlocale(LC_CTYPE, inLocale);
// default locale or user specified locale is not valid, abort launch
if (inLocale == NULL) {
printf("Invalid locale:%s, please set the valid locale in config file\n", inLocale);
}
if (strlen(inCharSet) == 0) {
printf("failed to get charset, please set the valid charset in config file\n");
exit(-1);
}
if (!taosValidateEncodec(inCharSet)) {
printf("Invalid charset:%s, please set the valid charset in config file", inCharSet);
exit(-1);
}
}
void taosGetSystemLocale(char *outLocale, char *outCharset) {
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
char *locale = setlocale(LC_CTYPE, "chs");
if (locale != NULL) {
tstrncpy(outLocale, locale, TD_LOCALE_LEN);
}
strcpy(outCharset, "cp936");
#elif defined(_TD_DARWIN_64)
/*
* originally from src/os/src/detail/osSysinfo.c
* POSIX format locale string:
* (Language Strings)_(Country/Region Strings).(code_page)
*
* example: en_US.UTF-8, zh_CN.GB18030, zh_CN.UTF-8,
*
* if user does not specify the locale in taos.cfg the program use default LC_CTYPE as system locale.
*
* In case of some CentOS systems, their default locale is "en_US.utf8", which is not valid code_page
* for libiconv that is employed to convert string in this system. This program will automatically use
* UTF-8 instead as the charset.
*
* In case of windows client, the locale string is not valid POSIX format, user needs to set the
* correct code_page for libiconv. Usually, the code_page of windows system with simple chinese is
* CP936, CP437 for English charset.
*
*/
char sep = '.';
char *locale = NULL;
locale = setlocale(LC_CTYPE, "");
if (locale == NULL) {
// printf("can't get locale from system, set it to en_US.UTF-8 since error:%d:%s", errno, strerror(errno));
strcpy(outLocale, "en_US.UTF-8");
} else {
tstrncpy(outLocale, locale, TD_LOCALE_LEN);
// printf("locale not configured, set to system default:%s", outLocale);
}
/* if user does not specify the charset, extract it from locale */
char *str = strrchr(outLocale, sep);
if (str != NULL) {
str++;
char *revisedCharset = taosCharsetReplace(str);
tstrncpy(outCharset, revisedCharset, TD_CHARSET_LEN);
free(revisedCharset);
// printf("charset not configured, set to system default:%s", outCharset);
} else {
strcpy(outCharset, "UTF-8");
// printf("can't get locale and charset from system, set it to UTF-8");
}
#else
/*
* POSIX format locale string:
* (Language Strings)_(Country/Region Strings).(code_page)
*
* example: en_US.UTF-8, zh_CN.GB18030, zh_CN.UTF-8,
*
* if user does not specify the locale in taos.cfg the program use default LC_CTYPE as system locale.
*
* In case of some CentOS systems, their default locale is "en_US.utf8", which is not valid code_page
* for libiconv that is employed to convert string in this system. This program will automatically use
* UTF-8 instead as the charset.
*
* In case of windows client, the locale string is not valid POSIX format, user needs to set the
* correct code_page for libiconv. Usually, the code_page of windows system with simple chinese is
* CP936, CP437 for English charset.
*
*/
char sep = '.';
char *locale = NULL;
locale = setlocale(LC_CTYPE, "");
if (locale == NULL) {
// printf("can't get locale from system, set it to en_US.UTF-8 since error:%d:%s", errno, strerror(errno));
strcpy(outLocale, "en_US.UTF-8");
} else {
tstrncpy(outLocale, locale, TD_LOCALE_LEN);
// printf("locale not configured, set to system default:%s", outLocale);
}
// if user does not specify the charset, extract it from locale
char *str = strrchr(outLocale, sep);
if (str != NULL) {
str++;
char *revisedCharset = taosCharsetReplace(str);
tstrncpy(outCharset, revisedCharset, TD_LOCALE_LEN);
free(revisedCharset);
// printf("charset not configured, set to system default:%s", outCharset);
} else {
strcpy(outCharset, "UTF-8");
// printf("can't get locale and charset from system, set it to UTF-8");
}
#endif
}

18
source/os/src/osString.c
View File

@ -18,24 +18,6 @@
#include <wchar.h>
#include <wctype.h>
typedef struct CharsetPair {
char *oldCharset;
char *newCharset;
} CharsetPair;
char *taosCharsetReplace(char *charsetstr) {
CharsetPair charsetRep[] = {
{ "utf8", "UTF-8" }, { "936", "CP936" },
};
for (int32_t i = 0; i < tListLen(charsetRep); ++i) {
if (strcasecmp(charsetRep[i].oldCharset, charsetstr) == 0) {
return strdup(charsetRep[i].newCharset);
}
}
return strdup(charsetstr);
}
int64_t taosStr2int64(const char *str) {
char *endptr = NULL;

109
source/os/src/osSysinfo.c
View File

@ -21,9 +21,6 @@ int64_t tsPageSize = 0;
int64_t tsOpenMax = 0;
int64_t tsStreamMax = 0;
int32_t tsNumOfCores = 1;
char tsLocale[TD_LOCALE_LEN] = {0};
char tsCharset[TD_LOCALE_LEN] = {0}; // default encode string
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
/*
@ -89,17 +86,6 @@ bool taosGetProcMemory(float *memoryUsedMB) {
}
static void taosGetSystemLocale() {
// get and set default locale
char *locale = setlocale(LC_CTYPE, "chs");
if (locale != NULL) {
tstrncpy(tsLocale, locale, TD_LOCALE_LEN);
}
strcpy(tsCharset, "cp936");
}
int32_t taosGetCpuCores() {
SYSTEM_INFO info;
GetSystemInfo(&info);
@ -191,7 +177,6 @@ void taosGetSystemInfo() {
taosGetCpuUsage(&tmp1, &tmp2);
taosGetProcIO(&tmp1, &tmp2);
taosGetSystemLocale();
}
void taosKillSystem() {
@ -263,52 +248,6 @@ char *taosGetCmdlineByPID(int pid) { return ""; }
#include <errno.h>
#include <libproc.h>
/*
* originally from src/os/src/detail/osSysinfo.c
* POSIX format locale string:
* (Language Strings)_(Country/Region Strings).(code_page)
*
* example: en_US.UTF-8, zh_CN.GB18030, zh_CN.UTF-8,
*
* if user does not specify the locale in taos.cfg the program use default LC_CTYPE as system locale.
*
* In case of some CentOS systems, their default locale is "en_US.utf8", which is not valid code_page
* for libiconv that is employed to convert string in this system. This program will automatically use
* UTF-8 instead as the charset.
*
* In case of windows client, the locale string is not valid POSIX format, user needs to set the
* correct code_page for libiconv. Usually, the code_page of windows system with simple chinese is
* CP936, CP437 for English charset.
*
*/
static void taosGetSystemLocale() { // get and set default locale
char sep = '.';
char *locale = NULL;
locale = setlocale(LC_CTYPE, "");
if (locale == NULL) {
//printf("can't get locale from system, set it to en_US.UTF-8 since error:%d:%s", errno, strerror(errno));
strcpy(tsLocale, "en_US.UTF-8");
} else {
tstrncpy(tsLocale, locale, TD_LOCALE_LEN);
//printf("locale not configured, set to system default:%s", tsLocale);
}
/* if user does not specify the charset, extract it from locale */
char *str = strrchr(tsLocale, sep);
if (str != NULL) {
str++;
char *revisedCharset = taosCharsetReplace(str);
tstrncpy(tsCharset, revisedCharset, TD_LOCALE_LEN);
free(revisedCharset);
//printf("charset not configured, set to system default:%s", tsCharset);
} else {
strcpy(tsCharset, "UTF-8");
//printf("can't get locale and charset from system, set it to UTF-8");
}
}
void taosKillSystem() {
//printf("function taosKillSystem, exit!");
@ -325,8 +264,6 @@ void taosGetSystemInfo() {
long page_size = sysconf(_SC_PAGESIZE);
tsTotalMemoryMB = physical_pages * page_size / (1024 * 1024);
tsPageSize = page_size;
taosGetSystemLocale();
}
bool taosReadProcIO(int64_t *rchars, int64_t *wchars) {
@ -563,51 +500,6 @@ static bool taosGetProcCpuInfo(ProcCpuInfo *cpuInfo) {
return true;
}
/*
* POSIX format locale string:
* (Language Strings)_(Country/Region Strings).(code_page)
*
* example: en_US.UTF-8, zh_CN.GB18030, zh_CN.UTF-8,
*
* if user does not specify the locale in taos.cfg the program use default LC_CTYPE as system locale.
*
* In case of some CentOS systems, their default locale is "en_US.utf8", which is not valid code_page
* for libiconv that is employed to convert string in this system. This program will automatically use
* UTF-8 instead as the charset.
*
* In case of windows client, the locale string is not valid POSIX format, user needs to set the
* correct code_page for libiconv. Usually, the code_page of windows system with simple chinese is
* CP936, CP437 for English charset.
*
*/
static void taosGetSystemLocale() { // get and set default locale
char sep = '.';
char *locale = NULL;
locale = setlocale(LC_CTYPE, "");
if (locale == NULL) {
//printf("can't get locale from system, set it to en_US.UTF-8 since error:%d:%s", errno, strerror(errno));
strcpy(tsLocale, "en_US.UTF-8");
} else {
tstrncpy(tsLocale, locale, TD_LOCALE_LEN);
//printf("locale not configured, set to system default:%s", tsLocale);
}
// if user does not specify the charset, extract it from locale
char *str = strrchr(tsLocale, sep);
if (str != NULL) {
str++;
char *revisedCharset = taosCharsetReplace(str);
tstrncpy(tsCharset, revisedCharset, TD_LOCALE_LEN);
free(revisedCharset);
//printf("charset not configured, set to system default:%s", tsCharset);
} else {
strcpy(tsCharset, "UTF-8");
//printf("can't get locale and charset from system, set it to UTF-8");
}
}
int32_t taosGetCpuCores() { return (int32_t)sysconf(_SC_NPROCESSORS_ONLN); }
@ -836,7 +728,6 @@ void taosGetSystemInfo() {
taosGetCpuUsage(&tmp1, &tmp2);
taosGetProcIO(&tmp1, &tmp2);
taosGetSystemLocale();
}
void taosKillSystem() {

4
source/os/src/osTimeZone1.c → source/os/src/osTimezone.c
View File

@ -45,7 +45,7 @@
#include <unistd.h>
#endif
void osSetSystemTimezone(const char *inTimezone, char *outTimezone, int8_t *outDaylight) {
void taosSetSystemTimezone(const char *inTimezone, char *outTimezone, int8_t *outDaylight) {
#ifdef WINDOWS
char winStr[TD_LOCALE_LEN * 2];
sprintf(winStr, "TZ=%s", inTimezone);
@ -84,7 +84,7 @@ void osSetSystemTimezone(const char *inTimezone, char *outTimezone, int8_t *outD
*outDaylight = daylight;
}
void osGetSystemTimezone(char *outTimezone) {
void taosGetSystemTimezone(char *outTimezone) {
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
char *tz = getenv("TZ");
if (tz == NULL || strlen(tz) == 0) {