xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

feat:[TD-32642] add charset for connection support

This commit is contained in:
wangmm0220 2024-11-27 14:25:34 +08:00
parent 2ca9f9c8ce
commit 0e2a16d70f
61 changed files with 598 additions and 487 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
include/common/tdataformat.h
View File

@ -154,7 +154,7 @@ int32_t tEncodeTag(SEncoder *pEncoder, const STag *pTag);
int32_t tDecodeTag(SDecoder *pDecoder, STag **ppTag);
int32_t tTagToValArray(const STag *pTag, SArray **ppArray);
void debugPrintSTag(STag *pTag, const char *tag, int32_t ln); // TODO: remove
int32_t parseJsontoTagData(const char *json, SArray *pTagVals, STag **ppTag, void *pMsgBuf);
int32_t parseJsontoTagData(const char *json, SArray *pTagVals, STag **ppTag, void *pMsgBuf, void *charsetCxt);
// SColData ================================
typedef struct {

2
include/common/tmsg.h
View File

@ -3842,7 +3842,7 @@ typedef struct {
int8_t igExists;
int8_t intervalUnit;
int8_t slidingUnit;
int8_t timezone;
int8_t timezone; // int8_t is not enough, timezone is unit of second
int32_t dstVgId; // for stream
int64_t interval;
int64_t offset;

2
include/common/ttime.h
View File

@ -74,7 +74,7 @@ char getPrecisionUnit(int32_t precision);
int64_t convertTimePrecision(int64_t ts, int32_t fromPrecision, int32_t toPrecision);
int32_t convertTimeFromPrecisionToUnit(int64_t time, int32_t fromPrecision, char toUnit, int64_t* pRes);
int32_t convertStringToTimestamp(int16_t type, char* inputData, int64_t timePrec, int64_t* timeVal, timezone_t tz);
int32_t convertStringToTimestamp(int16_t type, char* inputData, int64_t timePrec, int64_t* timeVal, timezone_t tz, void* charsetCxt);
int32_t getDuration(int64_t val, char unit, int64_t* result, int32_t timePrecision);
int32_t taosFormatUtcTime(char* buf, int32_t bufLen, int64_t ts, int32_t precision);

1
include/libs/function/function.h
View File

@ -293,6 +293,7 @@ struct SScalarParam {
int32_t numOfRows;
int32_t numOfQualified; // number of qualified elements in the final results
timezone_t tz;
void *charsetCxt;
};
#define cleanupResultRowEntry(p) p->initialized = false

4
include/libs/nodes/querynodes.h
View File

@ -132,6 +132,7 @@ typedef struct SValueNode {
int64_t typeData;
int8_t unit;
timezone_t tz;
void *charsetCxt;
} SValueNode;
typedef struct SLeftValueNode {
@ -161,6 +162,7 @@ typedef struct SOperatorNode {
SNode* pLeft;
SNode* pRight;
timezone_t tz;
void* charsetCxt;
} SOperatorNode;
typedef struct SLogicConditionNode {
@ -195,6 +197,7 @@ typedef struct SFunctionNode {
bool dual; // whether select stmt without from stmt, true for without.
// char timezone[TD_TIMEZONE_LEN];
timezone_t tz;
void *charsetCxt;
} SFunctionNode;
typedef struct STableNode {
@ -404,6 +407,7 @@ typedef struct SCaseWhenNode {
SNode* pElse;
SNodeList* pWhenThenList;
timezone_t tz;
void* charsetCxt;
} SCaseWhenNode;
typedef struct SWindowOffsetNode {

25
include/libs/parser/parser.h
View File

@ -102,6 +102,7 @@ typedef struct SParseContext {
SArray* pSubMetaList;
setQueryFn setQueryFp;
timezone_t timezone;
void *charsetCxt;
} SParseContext;
int32_t qParseSql(SParseContext* pCxt, SQuery** pQuery);
@ -140,27 +141,27 @@ void qDestroyStmtDataBlock(STableDataCxt* pBlock);
STableMeta* qGetTableMetaInDataBlock(STableDataCxt* pDataBlock);
int32_t qCloneCurrentTbData(STableDataCxt* pDataBlock, SSubmitTbData** pData);
int32_t qStmtBindParams(SQuery* pQuery, TAOS_MULTI_BIND* pParams, int32_t colIdx);
int32_t qStmtBindParams(SQuery* pQuery, TAOS_MULTI_BIND* pParams, int32_t colIdx, void *charsetCxt);
int32_t qStmtParseQuerySql(SParseContext* pCxt, SQuery* pQuery);
int32_t qBindStmtStbColsValue(void* pBlock, SArray* pCols, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen,
STSchema** pTSchema, SBindInfo* pBindInfos);
int32_t qBindStmtColsValue(void* pBlock, SArray* pCols, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen);
STSchema** pTSchema, SBindInfo* pBindInfos, void* charsetCxt);
int32_t qBindStmtColsValue(void* pBlock, SArray* pCols, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen, void* charsetCxt);
int32_t qBindStmtSingleColValue(void* pBlock, SArray* pCols, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen,
int32_t colIdx, int32_t rowNum);
int32_t colIdx, int32_t rowNum, void* charsetCxt);
int32_t qBuildStmtColFields(void* pDataBlock, int32_t* fieldNum, TAOS_FIELD_E** fields);
int32_t qBuildStmtStbColFields(void* pBlock, int32_t* fieldNum, TAOS_FIELD_STB** fields);
int32_t qBuildStmtTagFields(void* pBlock, void* boundTags, int32_t* fieldNum, TAOS_FIELD_E** fields);
int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, const char* sTableName, char* tName,
TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen);
TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen, void* charsetCxt);
int32_t qStmtBindParams2(SQuery* pQuery, TAOS_STMT2_BIND* pParams, int32_t colIdx);
int32_t qStmtBindParams2(SQuery* pQuery, TAOS_STMT2_BIND* pParams, int32_t colIdx, void* charsetCxt);
int32_t qBindStmtStbColsValue2(void* pBlock, SArray* pCols, TAOS_STMT2_BIND* bind, char* msgBuf, int32_t msgBufLen,
STSchema** pTSchema, SBindInfo2* pBindInfos);
int32_t qBindStmtColsValue2(void* pBlock, SArray* pCols, TAOS_STMT2_BIND* bind, char* msgBuf, int32_t msgBufLen);
STSchema** pTSchema, SBindInfo2* pBindInfos, void *charsetCxt);
int32_t qBindStmtColsValue2(void* pBlock, SArray* pCols, TAOS_STMT2_BIND* bind, char* msgBuf, int32_t msgBufLen, void *charsetCxt);
int32_t qBindStmtSingleColValue2(void* pBlock, SArray* pCols, TAOS_STMT2_BIND* bind, char* msgBuf, int32_t msgBufLen,
int32_t colIdx, int32_t rowNum);
int32_t colIdx, int32_t rowNum, void *charsetCxt);
int32_t qBindStmtTagsValue2(void* pBlock, void* boundTags, int64_t suid, const char* sTableName, char* tName,
TAOS_STMT2_BIND* bind, char* msgBuf, int32_t msgBufLen);
TAOS_STMT2_BIND* bind, char* msgBuf, int32_t msgBufLen, void *charsetCxt);
void destroyBoundColumnInfo(void* pBoundInfo);
int32_t qCreateSName(SName* pName, const char* pTableName, int32_t acctId, char* dbName, char* msgBuf,
@ -170,13 +171,13 @@ void qDestroyBoundColInfo(void* pInfo);
int32_t smlInitHandle(SQuery** query);
int32_t smlBuildRow(STableDataCxt* pTableCxt);
int32_t smlBuildCol(STableDataCxt* pTableCxt, SSchema* schema, void* kv, int32_t index);
int32_t smlBuildCol(STableDataCxt* pTableCxt, SSchema* schema, void* kv, int32_t index, void* charsetCxt);
int32_t smlInitTableDataCtx(SQuery* query, STableMeta* pTableMeta, STableDataCxt** cxt);
void clearColValArraySml(SArray* pCols);
int32_t smlBindData(SQuery* handle, bool dataFormat, SArray* tags, SArray* colsSchema, SArray* cols,
STableMeta* pTableMeta, char* tableName, const char* sTableName, int32_t sTableNameLen, int32_t ttl,
char* msgBuf, int32_t msgBufLen);
char* msgBuf, int32_t msgBufLen, void* charsetCxt);
int32_t smlBuildOutput(SQuery* handle, SHashObj* pVgHash);
int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreateTbReq* pCreateTb, void* fields,
int numFields, bool needChangeLength, char* errstr, int32_t errstrLen, bool raw);

2
include/libs/qcom/query.h
View File

@ -337,7 +337,7 @@ SSchema createSchema(int8_t type, int32_t bytes, col_id_t colId, const char* nam
void destroyQueryExecRes(SExecResult* pRes);
int32_t dataConverToStr(char* str, int64_t capacity, int type, void* buf, int32_t bufSize, int32_t* len);
void parseTagDatatoJson(void* p, char** jsonStr);
void parseTagDatatoJson(void* p, char** jsonStr, void *charsetCxt);
int32_t cloneTableMeta(STableMeta* pSrc, STableMeta** pDst);
void getColumnTypeFromMeta(STableMeta* pMeta, char* pName, ETableColumnType* pType);
int32_t cloneDbVgInfo(SDBVgInfo* pSrc, SDBVgInfo** pDst);

1
include/libs/scalar/scalar.h
View File

@ -105,7 +105,6 @@ int32_t timeTruncateFunction(SScalarParam *pInput, int32_t inputNum, SScalarPara
int32_t timeDiffFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t nowFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t todayFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t timeZoneStrLen();
int32_t timezoneFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t weekdayFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t dayofweekFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);

6
include/os/osEnv.h
View File

@ -25,10 +25,9 @@ extern "C" {
extern char tsOsName[];
extern char tsTimezoneStr[];
extern enum TdTimezone tsTimezone;
extern char tsCharset[];
extern void *tsCharsetCxt;
extern char tsLocale[];
extern int8_t tsDaylight;
extern bool tsEnableCoreFile;
extern int64_t tsPageSizeKB;
extern int64_t tsOpenMax;
@ -67,8 +66,7 @@ bool osDataSpaceSufficient();
bool osTempSpaceSufficient();
int32_t osSetTimezone(const char *timezone);
void osSetSystemLocale(const char *inLocale, const char *inCharSet);
void osSetProcPath(int32_t argc, char **argv);
void osSetProcPath(int32_t argc, char **argv);
#ifdef __cplusplus
}

2
include/os/osLocale.h
View File

@ -30,7 +30,7 @@ extern "C" {
char *taosCharsetReplace(char *charsetstr);
void taosGetSystemLocale(char *outLocale, char *outCharset);
int32_t taosSetSystemLocale(const char *inLocale, const char *inCharSet);
int32_t taosSetSystemLocale(const char *inLocale);
#ifdef __cplusplus
}

24
include/os/osString.h
View File

@ -27,7 +27,19 @@ typedef int32_t TdUcs4;
#else
typedef void *iconv_t;
#endif
typedef enum { M2C = 0, C2M } ConvType;
typedef enum { M2C = 0, C2M, CM_NUM } ConvType;
typedef struct {
iconv_t conv;
int8_t inUse;
} SConv;
typedef struct {
SConv *gConv[CM_NUM];
int32_t convUsed[CM_NUM];
int32_t gConvMaxNum[CM_NUM];
char charset[TD_CHARSET_LEN];
} SConvInfo;
// If the error is in a third-party library, place this header file under the third-party library header file.
// When you want to use this feature, you should find or add the same function in the following section.
@ -78,13 +90,11 @@ int32_t taosStr2int16(const char *str, int16_t *val);
int32_t taosStr2int32(const char *str, int32_t *val);
int32_t taosStr2int8(const char *str, int8_t *val);
int32_t taosConvInit(void);
void taosConvDestroy();
iconv_t taosAcquireConv(int32_t *idx, ConvType type);
void taosReleaseConv(int32_t idx, iconv_t conv, ConvType type);
int32_t taosUcs4ToMbs(TdUcs4 *ucs4, int32_t ucs4_max_len, char *mbs);
iconv_t taosAcquireConv(int32_t *idx, ConvType type, void* charsetCxt);
void taosReleaseConv(int32_t idx, iconv_t conv, ConvType type, void* charsetCxt);
int32_t taosUcs4ToMbs(TdUcs4 *ucs4, int32_t ucs4_max_len, char *mbs, void* charsetCxt);
int32_t taosUcs4ToMbsEx(TdUcs4 *ucs4, int32_t ucs4_max_len, char *mbs, iconv_t conv);
bool taosMbsToUcs4(const char *mbs, size_t mbs_len, TdUcs4 *ucs4, int32_t ucs4_max_len, int32_t *len);
bool taosMbsToUcs4(const char *mbs, size_t mbs_len, TdUcs4 *ucs4, int32_t ucs4_max_len, int32_t *len, void* charsetCxt);
int32_t tasoUcs4Compare(TdUcs4 *f1_ucs4, TdUcs4 *f2_ucs4, int32_t bytes);
int32_t tasoUcs4Copy(TdUcs4 *target_ucs4, TdUcs4 *source_ucs4, int32_t len_ucs4);
bool taosValidateEncodec(const char *encodec);

29
include/os/osTimezone.h
View File

@ -20,33 +20,7 @@
extern "C" {
#endif
enum TdTimezone {
TdWestZone12 = -12,
TdWestZone11,
TdWestZone10,
TdWestZone9,
TdWestZone8,
TdWestZone7,
TdWestZone6,
TdWestZone5,
TdWestZone4,
TdWestZone3,
TdWestZone2,
TdWestZone1,
TdZeroZone,
TdEastZone1,
TdEastZone2,
TdEastZone3,
TdEastZone4,
TdEastZone5,
TdEastZone6,
TdEastZone7,
TdEastZone8,
TdEastZone9,
TdEastZone10,
TdEastZone11,
TdEastZone12
};
#define TdEastZone8 8*60*60
typedef struct state *timezone_t;
struct tm *localtime_rz(timezone_t , time_t const *, struct tm *);
@ -55,6 +29,7 @@ timezone_t tzalloc(char const *);
void tzfree(timezone_t);
void getTimezoneStr(char *tz);
int32_t taosGetLocalTimezoneOffset();
int32_t taosGetSystemTimezone(char *outTimezone);
int32_t taosSetGlobalTimezone(const char *tz);
int32_t taosFormatTimezoneStr(time_t t, const char* tzStr, timezone_t sp, char *outTimezoneStr);

41
include/util/tconv.h Normal file
View File

@ -0,0 +1,41 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_TCONV_H
#define TDENGINE_TCONV_H
#ifdef __cplusplus
extern "C" {
#endif
//#include "osString.h"
//
//bool taosValidateEncodec(const char *encodec);
//int32_t taosUcs4len(TdUcs4 *ucs4);
void* taosConvInit(const char* charset);
void taosConvDestroy();
//iconv_t taosAcquireConv(int32_t *idx, ConvType type, void* charsetCxt);
//void taosReleaseConv(int32_t idx, iconv_t conv, ConvType type, void* charsetCxt);
//int32_t taosUcs4ToMbs(TdUcs4 *ucs4, int32_t ucs4_max_len, char *mbs, void* charsetCxt);
//int32_t taosUcs4ToMbsEx(TdUcs4 *ucs4, int32_t ucs4_max_len, char *mbs, iconv_t conv);
//bool taosMbsToUcs4(const char *mbs, size_t mbs_len, TdUcs4 *ucs4, int32_t ucs4_max_len, int32_t *len, void* charsetCxt);
//int32_t tasoUcs4Compare(TdUcs4 *f1_ucs4, TdUcs4 *f2_ucs4, int32_t bytes);
//int32_t tasoUcs4Copy(TdUcs4 *target_ucs4, TdUcs4 *source_ucs4, int32_t len_ucs4);
#ifdef __cplusplus
}
#endif
#endif // TDENGINE_TCONV_H

3
source/client/inc/clientInt.h
View File

@ -155,7 +155,7 @@ typedef struct {
typedef struct {
timezone_t timezone;
char charset[TD_CHARSET_LEN];
void *charsetCxt;
char app[TSDB_APP_NAME_LEN];
uint32_t ip;
}SOptionInfo;
@ -219,6 +219,7 @@ typedef struct SReqResultInfo {
int32_t precision;
int32_t payloadLen;
char* convertJson;
void* charsetCxt;
} SReqResultInfo;
typedef struct SRequestSendRecvBody {

9
source/client/src/clientEnv.c
View File

@ -36,6 +36,7 @@
#include "tsched.h"
#include "ttime.h"
#include "tversion.h"
#include "tconv.h"
#if defined(CUS_NAME) || defined(CUS_PROMPT) || defined(CUS_EMAIL)
#include "cus_name.h"
@ -556,6 +557,7 @@ int32_t createRequest(uint64_t connId, int32_t type, int64_t reqid, SRequestObj
(*pRequest)->metric.start = taosGetTimestampUs();
(*pRequest)->body.resInfo.convertUcs4 = true; // convert ucs4 by default
(*pRequest)->body.resInfo.charsetCxt = pTscObj->optionInfo.charsetCxt;
(*pRequest)->type = type;
(*pRequest)->allocatorRefId = -1;
@ -969,7 +971,12 @@ void taos_init_imp(void) {
ENV_ERR_RET(taosInitCfg(configDir, NULL, NULL, NULL, NULL, 1), "failed to init cfg");
initQueryModuleMsgHandle();
ENV_ERR_RET(taosConvInit(), "failed to init conv");
if ((tsCharsetCxt = taosConvInit(tsCharset)) == NULL){
tscInitRes = terrno;
tscError("failed to init conv");
return;
}
ENV_ERR_RET(monitorInit(), "failed to init monitor");
ENV_ERR_RET(rpcInit(), "failed to init rpc");

15
source/client/src/clientImpl.c
View File

@ -301,7 +301,8 @@ int32_t parseSql(SRequestObj* pRequest, bool topicQuery, SQuery** pQuery, SStmtC
.nodeOffline = (pTscObj->pAppInfo->onlineDnodes < pTscObj->pAppInfo->totalDnodes),
.isStmtBind = pRequest->isStmtBind,
.setQueryFp = setQueryRequest,
.timezone = pTscObj->optionInfo.timezone,};
.timezone = pTscObj->optionInfo.timezone,
.charsetCxt = pTscObj->optionInfo.charsetCxt,};
cxt.mgmtEpSet = getEpSet_s(&pTscObj->pAppInfo->mgmtEp);
int32_t code = catalogGetHandle(pTscObj->pAppInfo->clusterId, &cxt.pCatalog);
@ -2087,7 +2088,7 @@ static int32_t doPrepareResPtr(SReqResultInfo* pResInfo) {
static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t* colLength) {
int32_t idx = -1;
iconv_t conv = taosAcquireConv(&idx, C2M);
iconv_t conv = taosAcquireConv(&idx, C2M, pResultInfo->charsetCxt);
if (conv == (iconv_t)-1) return TSDB_CODE_TSC_INTERNAL_ERROR;
for (int32_t i = 0; i < pResultInfo->numOfCols; ++i) {
@ -2097,7 +2098,7 @@ static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t* colLength) {
if (type == TSDB_DATA_TYPE_NCHAR && colLength[i] > 0) {
char* p = taosMemoryRealloc(pResultInfo->convertBuf[i], colLength[i]);
if (p == NULL) {
taosReleaseConv(idx, conv, C2M);
taosReleaseConv(idx, conv, C2M, pResultInfo->charsetCxt);
return terrno;
}
@ -2114,7 +2115,7 @@ static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t* colLength) {
"doConvertUCS4 error, invalid data. len:%d, bytes:%d, (p + len):%p, (pResultInfo->convertBuf[i] + "
"colLength[i]):%p",
len, bytes, (p + len), (pResultInfo->convertBuf[i] + colLength[i]));
taosReleaseConv(idx, conv, C2M);
taosReleaseConv(idx, conv, C2M, pResultInfo->charsetCxt);
return TSDB_CODE_TSC_INTERNAL_ERROR;
}
@ -2128,7 +2129,7 @@ static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t* colLength) {
pResultInfo->row[i] = pResultInfo->pCol[i].pData;
}
}
taosReleaseConv(idx, conv, C2M);
taosReleaseConv(idx, conv, C2M, pResultInfo->charsetCxt);
return TSDB_CODE_SUCCESS;
}
@ -2293,7 +2294,7 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo) {
varDataSetLen(dst, strlen(varDataVal(dst)));
} else if (tTagIsJson(data)) {
char* jsonString = NULL;
parseTagDatatoJson(data, &jsonString);
parseTagDatatoJson(data, &jsonString, pResultInfo->charsetCxt);
if (jsonString == NULL) {
tscError("doConvertJson error: parseTagDatatoJson failed");
return terrno;
@ -2303,7 +2304,7 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo) {
} else if (jsonInnerType == TSDB_DATA_TYPE_NCHAR) { // value -> "value"
*(char*)varDataVal(dst) = '\"';
int32_t length = taosUcs4ToMbs((TdUcs4*)varDataVal(jsonInnerData), varDataLen(jsonInnerData),
varDataVal(dst) + CHAR_BYTES);
varDataVal(dst) + CHAR_BYTES, pResultInfo->charsetCxt);
if (length <= 0) {
tscError("charset:%s to %s. convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset);
length = 0;

96
source/client/src/clientMain.c
View File

@ -30,6 +30,7 @@
#include "tref.h"
#include "trpc.h"
#include "version.h"
#include "tconv.h"
#define TSC_VAR_NOT_RELEASE 1
#define TSC_VAR_RELEASED 0
@ -52,6 +53,49 @@ int taos_options(TSDB_OPTION option, const void *arg, ...) {
return ret;
}
static timezone_t setConnnectionTz(const char* val){
timezone_t tz = NULL;
static int32_t lock_c = 0;
for (int i = 1; atomic_val_compare_exchange_32(&lock_c, 0, 1) != 0; ++i) {
if (i % 1000 == 0) {
tscInfo("haven't acquire lock after spin %d times.", i);
(void)sched_yield();
}
}
if (pTimezoneMap == NULL){
pTimezoneMap = taosHashInit(0, MurmurHash3_32, false, HASH_ENTRY_LOCK);
if (pTimezoneMap == NULL) {
atomic_store_32(&lock_c, 0);
goto END;
}
taosHashSetFreeFp(pTimezoneMap, (_hash_free_fn_t)tzfree);
}
timezone_t *tmp = taosHashGet(pTimezoneMap, val, strlen(val));
if (tmp != NULL && *tmp != NULL){
tz = *tmp;
goto END;
}
tscDebug("set timezone to %s", val);
tz = tzalloc(val);
if (tz == NULL) {
tscError("%s unknown timezone %s", __func__, val);
terrno = TAOS_SYSTEM_ERROR(errno);
goto END;
}
int32_t code = taosHashPut(pTimezoneMap, val, strlen(val), &tz, sizeof(timezone_t));
if (code != 0){
tzfree(tz);
tz = NULL;
}
END:
atomic_store_32(&lock_c, 0);
return tz;
}
static int32_t setConnectionOption(TAOS *taos, TSDB_OPTION_CONNECTION option, const char* val){
if (taos == NULL) {
return TSDB_CODE_INVALID_PARA;
@ -74,32 +118,24 @@ static int32_t setConnectionOption(TAOS *taos, TSDB_OPTION_CONNECTION option, co
code = terrno;
goto END;
}
tstrncpy(pObj->optionInfo.charset, val, TD_CHARSET_LEN);
}else{
pObj->optionInfo.charset[0] = 0;
val = tsCharset;
}
void *tmp = taosConvInit(val);
if (tmp == NULL) {
code = terrno;
goto END;
}
pObj->optionInfo.charsetCxt = tmp;
} else if (option == TSDB_OPTION_CONNECTION_TIMEZONE) {
if (val != NULL){
if (strlen(val) == 0){
code = TSDB_CODE_INVALID_PARA;
goto END;
}
timezone_t *tmp = taosHashGet(pTimezoneMap, val, strlen(val));
if (tmp && *tmp){
pObj->optionInfo.timezone = *tmp;
goto END;
}
tscDebug("set timezone to %s", val);
timezone_t tz = tzalloc(val);
if (!tz) {
tscError("%s unknown timezone %s", __func__, val);
code = TAOS_SYSTEM_ERROR(errno);
goto END;
}
code = taosHashPut(pTimezoneMap, val, strlen(val), &tz, sizeof(timezone_t));
if (code != 0){
tzfree(tz);
timezone_t tz = setConnnectionTz(val);
if (tz == NULL){
code = terrno;
goto END;
}
pObj->optionInfo.timezone = tz;
@ -126,26 +162,7 @@ END:
}
int taos_options_connection(TAOS *taos, TSDB_OPTION_CONNECTION option, const void *arg, ...){
static int32_t lock_c = 0;
for (int i = 1; atomic_val_compare_exchange_32(&lock_c, 0, 1) != 0; ++i) {
if (i % 1000 == 0) {
tscInfo("haven't acquire lock after spin %d times.", i);
(void)sched_yield();
}
}
if (pTimezoneMap == NULL){
pTimezoneMap = taosHashInit(0, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK);
if (pTimezoneMap == NULL) {
atomic_store_32(&lock_c, 0);
return terrno;
}
taosHashSetFreeFp(pTimezoneMap, (_hash_free_fn_t)tzfree);
}
int ret = setConnectionOption(taos, option, (const char *)arg);
atomic_store_32(&lock_c, 0);
return ret;
return setConnectionOption(taos, option, (const char *)arg);
}
// this function may be called by user or system, or by both simultaneously.
@ -1342,7 +1359,8 @@ int32_t createParseContext(const SRequestObj *pRequest, SParseContext **pCxt, SS
.parseSqlFp = clientParseSql,
.parseSqlParam = pWrapper,
.setQueryFp = setQueryRequest,
.timezone = pTscObj->optionInfo.timezone};
.timezone = pTscObj->optionInfo.timezone,
.charsetCxt = pTscObj->optionInfo.charsetCxt};
int8_t biMode = atomic_load_8(&((STscObj *)pTscObj)->biMode);
(*pCxt)->biMode = biMode;
return TSDB_CODE_SUCCESS;

4
source/client/src/clientRawBlockWrite.c
View File

@ -414,7 +414,7 @@ static void buildChildElement(cJSON* json, SVCreateTbReq* pCreateReq) {
goto end;
}
char* pJson = NULL;
parseTagDatatoJson(pTag, &pJson);
parseTagDatatoJson(pTag, &pJson, NULL);
if (pJson == NULL) {
uError("parseTagDatatoJson failed, pJson == NULL");
goto end;
@ -730,7 +730,7 @@ static void processAlterTable(SMqMetaRsp* metaRsp, cJSON** pJson) {
uError("processAlterTable isJson false");
goto end;
}
parseTagDatatoJson(vAlterTbReq.pTagVal, &buf);
parseTagDatatoJson(vAlterTbReq.pTagVal, &buf, NULL);
if (buf == NULL) {
uError("parseTagDatatoJson failed, buf == NULL");
goto end;

10
source/client/src/clientSml.c
View File

@ -267,7 +267,7 @@ bool isSmlColAligned(SSmlHandle *info, int cnt, SSmlKv *kv) {
goto END;
}
// bind data
int32_t ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, kv, cnt + 1);
int32_t ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, kv, cnt + 1, info->taos->optionInfo.charsetCxt);
if (unlikely(ret != TSDB_CODE_SUCCESS)) {
uDebug("smlBuildCol error, retry");
goto END;
@ -411,8 +411,8 @@ int32_t smlParseEndTelnetJsonFormat(SSmlHandle *info, SSmlLineInfo *elements, SS
int32_t code = 0;
int32_t lino = 0;
uDebug("SML:0x%" PRIx64 " %s format true, ts:%" PRId64, info->id, __FUNCTION__ , kvTs->i);
SML_CHECK_CODE(smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, kvTs, 0));
SML_CHECK_CODE(smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, kv, 1));
SML_CHECK_CODE(smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, kvTs, 0, info->taos->optionInfo.charsetCxt));
SML_CHECK_CODE(smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, kv, 1, info->taos->optionInfo.charsetCxt));
SML_CHECK_CODE(smlBuildRow(info->currTableDataCtx));
END:
@ -438,7 +438,7 @@ END:
int32_t smlParseEndLine(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs) {
if (info->dataFormat) {
uDebug("SML:0x%" PRIx64 " %s format true, ts:%" PRId64, info->id, __FUNCTION__, kvTs->i);
int32_t ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, kvTs, 0);
int32_t ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, kvTs, 0, info->taos->optionInfo.charsetCxt);
if (ret == TSDB_CODE_SUCCESS) {
ret = smlBuildRow(info->currTableDataCtx);
}
@ -1482,7 +1482,7 @@ static int32_t smlInsertData(SSmlHandle *info) {
SML_CHECK_CODE(smlBindData(info->pQuery, info->dataFormat, tableData->tags, (*pMeta)->cols, tableData->cols,
(*pMeta)->tableMeta, tableData->childTableName, measure, measureLen, info->ttl, info->msgBuf.buf,
info->msgBuf.len));
info->msgBuf.len, info->taos->optionInfo.charsetCxt));
taosMemoryFreeClear(measure);
oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, oneTable);
}

10
source/client/src/clientStmt.c
View File

@ -1071,7 +1071,7 @@ int stmtSetTbTags(TAOS_STMT* stmt, TAOS_MULTI_BIND* tags) {
tscDebug("start to bind stmt tag values");
STMT_ERR_RET(qBindStmtTagsValue(*pDataBlock, pStmt->bInfo.boundTags, pStmt->bInfo.tbSuid, pStmt->bInfo.stbFName,
pStmt->bInfo.sname.tname, tags, pStmt->exec.pRequest->msgBuf,
pStmt->exec.pRequest->msgBufLen));
pStmt->exec.pRequest->msgBufLen, pStmt->taos->optionInfo.charsetCxt));
return TSDB_CODE_SUCCESS;
}
@ -1239,7 +1239,7 @@ int stmtBindBatch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind, int32_t colIdx) {
}
if (STMT_TYPE_QUERY == pStmt->sql.type) {
STMT_ERR_RET(qStmtBindParams(pStmt->sql.pQuery, bind, colIdx));
STMT_ERR_RET(qStmtBindParams(pStmt->sql.pQuery, bind, colIdx, pStmt->taos->optionInfo.charsetCxt));
SParseContext ctx = {.requestId = pStmt->exec.pRequest->requestId,
.acctId = pStmt->taos->acctId,
@ -1325,10 +1325,10 @@ int stmtBindBatch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind, int32_t colIdx) {
if (pStmt->sql.stbInterlaceMode) {
(*pDataBlock)->pData->flags = 0;
code = qBindStmtStbColsValue(*pDataBlock, pCols, bind, pStmt->exec.pRequest->msgBuf,
pStmt->exec.pRequest->msgBufLen, &pStmt->sql.siInfo.pTSchema, pStmt->sql.pBindInfo);
pStmt->exec.pRequest->msgBufLen, &pStmt->sql.siInfo.pTSchema, pStmt->sql.pBindInfo, pStmt->taos->optionInfo.charsetCxt);
} else {
code =
qBindStmtColsValue(*pDataBlock, pCols, bind, pStmt->exec.pRequest->msgBuf, pStmt->exec.pRequest->msgBufLen);
qBindStmtColsValue(*pDataBlock, pCols, bind, pStmt->exec.pRequest->msgBuf, pStmt->exec.pRequest->msgBufLen, pStmt->taos->optionInfo.charsetCxt);
}
if (code) {
@ -1353,7 +1353,7 @@ int stmtBindBatch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind, int32_t colIdx) {
}
code = qBindStmtSingleColValue(*pDataBlock, pCols, bind, pStmt->exec.pRequest->msgBuf,
pStmt->exec.pRequest->msgBufLen, colIdx, pStmt->bInfo.sBindRowNum);
pStmt->exec.pRequest->msgBufLen, colIdx, pStmt->bInfo.sBindRowNum, pStmt->taos->optionInfo.charsetCxt);
if (code) {
tscError("qBindStmtSingleColValue failed, error:%s", tstrerror(code));
STMT_ERR_RET(code);

10
source/client/src/clientStmt2.c
View File

@ -1014,7 +1014,7 @@ int stmtSetTbTags2(TAOS_STMT2* stmt, TAOS_STMT2_BIND* tags) {
tscDebug("start to bind stmt tag values");
STMT_ERR_RET(qBindStmtTagsValue2(*pDataBlock, pStmt->bInfo.boundTags, pStmt->bInfo.tbSuid, pStmt->bInfo.stbFName,
pStmt->bInfo.sname.tname, tags, pStmt->exec.pRequest->msgBuf,
pStmt->exec.pRequest->msgBufLen));
pStmt->exec.pRequest->msgBufLen, pStmt->taos->optionInfo.charsetCxt));
return TSDB_CODE_SUCCESS;
}
@ -1323,7 +1323,7 @@ int stmtBindBatch2(TAOS_STMT2* stmt, TAOS_STMT2_BIND* bind, int32_t colIdx) {
}
if (STMT_TYPE_QUERY == pStmt->sql.type) {
STMT_ERR_RET(qStmtBindParams2(pStmt->sql.pQuery, bind, colIdx));
STMT_ERR_RET(qStmtBindParams2(pStmt->sql.pQuery, bind, colIdx, pStmt->taos->optionInfo.charsetCxt));
SParseContext ctx = {.requestId = pStmt->exec.pRequest->requestId,
.acctId = pStmt->taos->acctId,
@ -1407,10 +1407,10 @@ int stmtBindBatch2(TAOS_STMT2* stmt, TAOS_STMT2_BIND* bind, int32_t colIdx) {
if (pStmt->sql.stbInterlaceMode) {
(*pDataBlock)->pData->flags = 0;
code = qBindStmtStbColsValue2(*pDataBlock, pCols, bind, pStmt->exec.pRequest->msgBuf,
pStmt->exec.pRequest->msgBufLen, &pStmt->sql.siInfo.pTSchema, pStmt->sql.pBindInfo);
pStmt->exec.pRequest->msgBufLen, &pStmt->sql.siInfo.pTSchema, pStmt->sql.pBindInfo, pStmt->taos->optionInfo.charsetCxt);
} else {
code =
qBindStmtColsValue2(*pDataBlock, pCols, bind, pStmt->exec.pRequest->msgBuf, pStmt->exec.pRequest->msgBufLen);
qBindStmtColsValue2(*pDataBlock, pCols, bind, pStmt->exec.pRequest->msgBuf, pStmt->exec.pRequest->msgBufLen, pStmt->taos->optionInfo.charsetCxt);
}
if (code) {
@ -1435,7 +1435,7 @@ int stmtBindBatch2(TAOS_STMT2* stmt, TAOS_STMT2_BIND* bind, int32_t colIdx) {
}
code = qBindStmtSingleColValue2(*pDataBlock, pCols, bind, pStmt->exec.pRequest->msgBuf,
pStmt->exec.pRequest->msgBufLen, colIdx, pStmt->bInfo.sBindRowNum);
pStmt->exec.pRequest->msgBufLen, colIdx, pStmt->bInfo.sBindRowNum, pStmt->taos->optionInfo.charsetCxt);
if (code) {
tscError("qBindStmtSingleColValue failed, error:%s", tstrerror(code));
STMT_ERR_RET(code);

2
source/common/src/tdatablock.c
View File

@ -2641,7 +2641,7 @@ int32_t dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** pDataBuf
char* pData = colDataGetVarData(pColInfoData, j);
int32_t dataSize = TMIN(sizeof(pBuf), varDataLen(pData));
memset(pBuf, 0, sizeof(pBuf));
code = taosUcs4ToMbs((TdUcs4*)varDataVal(pData), dataSize, pBuf);
code = taosUcs4ToMbs((TdUcs4*)varDataVal(pData), dataSize, pBuf, NULL);
if (code < 0) {
uError("func %s failed to convert to ucs charset since %s", __func__, tstrerror(code));
lino = __LINE__;

47
source/common/src/tglobal.c
View File

@ -654,7 +654,6 @@ static int32_t taosAddClientCfg(SConfig *pCfg) {
static int32_t taosAddSystemCfg(SConfig *pCfg) {
SysNameInfo info = taosGetSysNameInfo();
(void)taosGetSystemTimezone(tsTimezoneStr);
TAOS_CHECK_RETURN(cfgAddTimezone(pCfg, "timezone", tsTimezoneStr, CFG_SCOPE_BOTH, CFG_DYN_CLIENT));
TAOS_CHECK_RETURN(cfgAddLocale(pCfg, "locale", tsLocale, CFG_SCOPE_BOTH, CFG_DYN_CLIENT));
TAOS_CHECK_RETURN(cfgAddCharset(pCfg, "charset", tsCharset, CFG_SCOPE_BOTH, CFG_DYN_NONE));
@ -1317,25 +1316,6 @@ static int32_t taosSetClientCfg(SConfig *pCfg) {
static int32_t taosSetSystemCfg(SConfig *pCfg) {
SConfigItem *pItem = NULL;
TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "locale");
const char *locale = pItem->str;
TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "charset");
const char *charset = pItem->str;
int32_t code = taosSetSystemLocale(locale, charset);
if (TSDB_CODE_SUCCESS != code) {
uError("failed to set locale:%s, since: %s", locale, tstrerror(code));
char curLocale[TD_LOCALE_LEN] = {0};
char curCharset[TD_CHARSET_LEN] = {0};
taosGetSystemLocale(curLocale, curCharset);
if (0 != strlen(curLocale) && 0 != strlen(curCharset)) {
uInfo("current locale: %s, charset: %s", curLocale, curCharset);
}
}
osSetSystemLocale(locale, charset);
TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "enableCoreFile");
tsEnableCoreFile = pItem->bval;
taosSetCoreDump(tsEnableCoreFile);
@ -2062,6 +2042,10 @@ static int32_t taosCfgDynamicOptionsForClient(SConfig *pCfg, const char *name) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
if (strcasecmp("locale", name) == 0 || strcasecmp("charset", name) == 0
|| strcasecmp("timezone", name) == 0) {
goto _out;
}
cfgLock(pCfg);
SConfigItem *pItem = cfgGetItem(pCfg, name);
@ -2150,25 +2134,6 @@ static int32_t taosCfgDynamicOptionsForClient(SConfig *pCfg, const char *name) {
}
break;
}
case 'l': {
if (strcasecmp("locale", name) == 0) {
SConfigItem *pLocaleItem = cfgGetItem(pCfg, "locale");
SConfigItem *pCharsetItem = cfgGetItem(pCfg, "charset");
if (pLocaleItem == NULL || pCharsetItem == NULL) {
uError("failed to get locale or charset from cfg");
code = TSDB_CODE_CFG_NOT_FOUND;
goto _out;
}
const char *locale = pLocaleItem->str;
const char *charset = pCharsetItem->str;
TAOS_CHECK_GOTO(taosSetSystemLocale(locale, charset), &lino, _out);
osSetSystemLocale(locale, charset);
uInfo("locale set to '%s', charset set to '%s'", locale, charset);
matched = true;
}
break;
}
case 'm': {
if (strcasecmp("metaCacheMaxSize", name) == 0) {
atomic_store_32(&tsMetaCacheMaxSize, pItem->i32);
@ -2239,9 +2204,7 @@ static int32_t taosCfgDynamicOptionsForClient(SConfig *pCfg, const char *name) {
break;
}
case 't': {
if (strcasecmp("timezone", name) == 0) {
matched = true;
} else if (strcasecmp("tempDir", name) == 0) {
if (strcasecmp("tempDir", name) == 0) {
uInfo("%s set from %s to %s", name, tsTempDir, pItem->str);
tstrncpy(tsTempDir, pItem->str, PATH_MAX);
TAOS_CHECK_GOTO(taosExpandDir(tsTempDir, tsTempDir, PATH_MAX), &lino, _out);

10
source/common/src/ttime.c
View File

@ -480,7 +480,7 @@ int32_t convertTimeFromPrecisionToUnit(int64_t time, int32_t fromPrecision, char
TAOS_RETURN(TSDB_CODE_SUCCESS);
}
int32_t convertStringToTimestamp(int16_t type, char* inputData, int64_t timePrec, int64_t* timeVal, timezone_t tz) {
int32_t convertStringToTimestamp(int16_t type, char* inputData, int64_t timePrec, int64_t* timeVal, timezone_t tz, void* charsetCxt) {
int32_t charLen = varDataLen(inputData);
char* newColData;
if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_VARBINARY) {
@ -500,7 +500,7 @@ int32_t convertStringToTimestamp(int16_t type, char* inputData, int64_t timePrec
if (NULL == newColData) {
TAOS_RETURN(terrno);
}
int len = taosUcs4ToMbs((TdUcs4*)varDataVal(inputData), charLen, newColData);
int len = taosUcs4ToMbs((TdUcs4*)varDataVal(inputData), charLen, newColData, charsetCxt);
if (len < 0) {
taosMemoryFree(newColData);
TAOS_RETURN(TSDB_CODE_FAILED);
@ -2011,7 +2011,11 @@ int64_t taosGetTimestampToday(int32_t precision, timezone_t tz) {
: (precision == TSDB_TIME_PRECISION_MILLI) ? 1000
: (precision == TSDB_TIME_PRECISION_MICRO) ? 1000000
: 1000000000;
time_t t = taosTime(NULL);
time_t t;
int32_t code = taosTime(&t);
if (code != 0) {
return -1;
}
struct tm tm;
if (taosLocalTime(&t, &tm, NULL, 0, tz) == NULL){
uError("%s failed to get local time, code:%d", __FUNCTION__, errno);

6
source/common/test/commonTests.cpp
View File

@ -477,7 +477,7 @@ void test_ts2char(int64_t ts, const char* format, int32_t precison, const char*
TEST(timeTest, ts2char) {
osDefaultInit();
if (tsTimezone != TdEastZone8) GTEST_SKIP();
if (taosGetLocalTimezoneOffset() != TdEastZone8) GTEST_SKIP();
int64_t ts;
const char* format = "YYYY-MM-DD";
ts = 0;
@ -529,7 +529,7 @@ TEST(timeTest, ts2char) {
TEST(timeTest, char2ts) {
osDefaultInit();
if (tsTimezone != TdEastZone8) GTEST_SKIP();
if (taosGetLocalTimezoneOffset() != TdEastZone8) GTEST_SKIP();
int64_t ts;
int32_t code =
TEST_char2ts("YYYY-DD-MM HH12:MI:SS:MSPM", &ts, TSDB_TIME_PRECISION_MILLI, "2023-10-10 12:00:00.000AM");
@ -630,7 +630,7 @@ TEST(timeTest, char2ts) {
// default to 1970-1-1 00:00:00+08 -> 1969-12-31 16:00:00+00
ASSERT_EQ(0, TEST_char2ts("YYYY", &ts, TSDB_TIME_PRECISION_SECONDS, "1970"));
ASSERT_EQ(ts, -1 * tsTimezone * 60 * 60);
ASSERT_EQ(ts, -1 * taosGetLocalTimezoneOffset());
ASSERT_EQ(0, TEST_char2ts("yyyyMM1/dd ", &ts, TSDB_TIME_PRECISION_MICRO, "210001/2"));
ASSERT_EQ(ts, 4102502400000000LL);

3
source/dnode/mgmt/exe/dmMain.c
View File

@ -20,6 +20,7 @@
#include "tconfig.h"
#include "tglobal.h"
#include "version.h"
#include "tconv.h"
#ifdef TD_JEMALLOC_ENABLED
#include "jemalloc/jemalloc.h"
#endif
@ -444,7 +445,7 @@ int mainWindows(int argc, char **argv) {
return code;
}
if ((code = taosConvInit()) != 0) {
if ((tsCharsetCxt = taosConvInit(tsCharset)) == NULL) {
dError("failed to init conv");
taosCloseLog();
taosCleanupArgs();

1
source/dnode/mgmt/node_mgmt/src/dmMgmt.c
View File

@ -22,6 +22,7 @@
#include "tglobal.h"
#include "tgrant.h"
#include "tstream.h"
#include "tconv.h"
static bool dmRequireNode(SDnode *pDnode, SMgmtWrapper *pWrapper) {
SMgmtInputOpt input = dmBuildMgmtInputOpt(pWrapper);

2
source/dnode/mnode/impl/inc/mndDef.h
View File

@ -489,7 +489,7 @@ typedef struct {
int64_t dstTbUid;
int8_t intervalUnit;
int8_t slidingUnit;
int8_t timezone;
int8_t timezone; // int8_t is not enough, timezone is unit of second
int32_t dstVgId; // for stream
int64_t interval;
int64_t offset;

8
source/dnode/mnode/impl/src/mndSma.c
View File

@ -316,7 +316,7 @@ static void *mndBuildVCreateSmaReq(SMnode *pMnode, SVgObj *pVgroup, SSmaObj *pSm
req.version = 0;
req.intervalUnit = pSma->intervalUnit;
req.slidingUnit = pSma->slidingUnit;
req.timezoneInt = pSma->timezone;
// req.timezoneInt = pSma->timezone;
tstrncpy(req.indexName, (char *)tNameGetTableName(&name), TSDB_INDEX_NAME_LEN);
req.exprLen = pSma->exprLen;
req.tagsFilterLen = pSma->tagsFilterLen;
@ -617,9 +617,9 @@ static int32_t mndCreateSma(SMnode *pMnode, SRpcMsg *pReq, SMCreateSmaReq *pCrea
smaObj.intervalUnit = pCreate->intervalUnit;
smaObj.slidingUnit = pCreate->slidingUnit;
#if 0
smaObj.timezone = pCreate->timezone;
// smaObj.timezone = pCreate->timezone;
#endif
smaObj.timezone = tsTimezone; // use timezone of server
// smaObj.timezone = taosGetLocalTimezoneOffset(); // use timezone of server
smaObj.interval = pCreate->interval;
smaObj.offset = pCreate->offset;
smaObj.sliding = pCreate->sliding;
@ -1554,7 +1554,7 @@ static void initSMAObj(SCreateTSMACxt* pCxt) {
pCxt->pSma->dbUid = pCxt->pDb->uid;
pCxt->pSma->interval = pCxt->pCreateSmaReq->interval;
pCxt->pSma->intervalUnit = pCxt->pCreateSmaReq->intervalUnit;
pCxt->pSma->timezone = tsTimezone;
// pCxt->pSma->timezone = taosGetLocalTimezoneOffset();
pCxt->pSma->version = 1;
pCxt->pSma->exprLen = pCxt->pCreateSmaReq->exprLen;

2
source/dnode/vnode/src/meta/metaQuery.c
View File

@ -1328,7 +1328,7 @@ int32_t metaFilterTableIds(void *pVnode, SMetaFltParam *arg, SArray *pUids) {
TAOS_CHECK_GOTO(terrno, NULL, END);
}
if (false == taosMbsToUcs4(tagData, nTagData, (TdUcs4 *)buf, maxSize, &maxSize)) {
if (false == taosMbsToUcs4(tagData, nTagData, (TdUcs4 *)buf, maxSize, &maxSize, NULL)) {
TAOS_CHECK_GOTO(terrno, NULL, END);
}

4
source/dnode/vnode/src/meta/metaTable.c
View File

@ -153,7 +153,7 @@ static int metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const
if (val == NULL) {
TAOS_CHECK_GOTO(terrno, NULL, _exception);
}
int32_t len = taosUcs4ToMbs((TdUcs4 *)pTagVal->pData, pTagVal->nData, val + VARSTR_HEADER_SIZE);
int32_t len = taosUcs4ToMbs((TdUcs4 *)pTagVal->pData, pTagVal->nData, val + VARSTR_HEADER_SIZE, NULL);
if (len < 0) {
TAOS_CHECK_GOTO(len, NULL, _exception);
}
@ -237,7 +237,7 @@ int metaDelJsonVarFromIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSche
if (val == NULL) {
TAOS_CHECK_GOTO(terrno, NULL, _exception);
}
int32_t len = taosUcs4ToMbs((TdUcs4 *)pTagVal->pData, pTagVal->nData, val + VARSTR_HEADER_SIZE);
int32_t len = taosUcs4ToMbs((TdUcs4 *)pTagVal->pData, pTagVal->nData, val + VARSTR_HEADER_SIZE, NULL);
if (len < 0) {
TAOS_CHECK_GOTO(len, NULL, _exception);
}

2
source/libs/catalog/src/catalog.c
View File

@ -447,7 +447,7 @@ int32_t ctgGetTbTag(SCatalog* pCtg, SRequestConnInfo* pConn, SName* pTableName,
}
char* pJson = NULL;
parseTagDatatoJson(pTag, &pJson);
parseTagDatatoJson(pTag, &pJson, NULL);
if(NULL == pJson) {
taosArrayDestroy(pTagVals);
CTG_ERR_JRET(terrno);

2
source/libs/catalog/src/ctgAsync.c
View File

@ -2340,7 +2340,7 @@ int32_t ctgHandleGetTbTagRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf*
}
char* pJson = NULL;
parseTagDatatoJson(pTag, &pJson);
parseTagDatatoJson(pTag, &pJson, NULL);
if (NULL == pJson) {
taosArrayDestroy(pTagVals);
CTG_ERR_JRET(terrno);

2
source/libs/command/src/command.c
View File

@ -585,7 +585,7 @@ static int32_t appendTagValues(char* buf, int32_t* len, STableCfg* pCfg) {
if (tTagIsJson(pTag)) {
char* pJson = NULL;
parseTagDatatoJson(pTag, &pJson);
parseTagDatatoJson(pTag, &pJson, NULL);
if (NULL == pJson) {
qError("failed to parse tag to json, pJson is NULL");
return terrno;

4
source/libs/executor/src/sysscanoperator.c
View File

@ -966,7 +966,7 @@ int32_t convertTagDataToStr(char* str, int32_t strBuffLen, int type, void* buf,
return TSDB_CODE_TSC_INVALID_VALUE;
}
int32_t length = taosUcs4ToMbs((TdUcs4*)buf, bufSize, str);
int32_t length = taosUcs4ToMbs((TdUcs4*)buf, bufSize, str, NULL);
if (length <= 0) {
return TSDB_CODE_TSC_INVALID_VALUE;
}
@ -1117,7 +1117,7 @@ static int32_t sysTableUserTagsFillOneTableTags(const SSysTableScanInfo* pInfo,
if (tagData != NULL) {
if (tagType == TSDB_DATA_TYPE_JSON) {
char* tagJson = NULL;
parseTagDatatoJson(tagData, &tagJson);
parseTagDatatoJson(tagData, &tagJson, NULL);
if (tagJson == NULL) {
code = terrno;
goto _end;

2
source/libs/executor/test/queryPlanTests.cpp
View File

@ -592,7 +592,7 @@ void qptGetRandValue(uint8_t* pType, int32_t* pLen, void** ppVal) {
memset(pTmp, 'A' + taosRand() % 26, *pLen);
*ppVal = taosMemoryCalloc(1, *pLen * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE);
assert(*ppVal);
assert(taosMbsToUcs4(pTmp, *pLen, (TdUcs4 *)varDataVal(*ppVal), *pLen * TSDB_NCHAR_SIZE, NULL));
assert(taosMbsToUcs4(pTmp, *pLen, (TdUcs4 *)varDataVal(*ppVal), *pLen * TSDB_NCHAR_SIZE, NULL, NULL));
*pLen *= TSDB_NCHAR_SIZE;
varDataSetLen(*ppVal, *pLen);
taosMemoryFree(pTmp);

4
source/libs/executor/test/sortTests.cpp
View File

@ -93,7 +93,7 @@ SSDataBlock* getSingleColDummyBlock(void* param) {
char strOri[128] = {0};
taosRandStr(strOri, size);
int32_t len = 0;
bool ret = taosMbsToUcs4(strOri, size, (TdUcs4*)varDataVal(str), size * TSDB_NCHAR_SIZE, &len);
bool ret = taosMbsToUcs4(strOri, size, (TdUcs4*)varDataVal(str), size * TSDB_NCHAR_SIZE, &len, NULL);
if (!ret) {
(void) printf("error\n");
return NULL;
@ -331,7 +331,7 @@ TEST(testCase, external_mem_sort_Test) {
if(pInfo[i].type == TSDB_DATA_TYPE_NCHAR){
char buf[128] = {0};
int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(v), varDataLen(v), buf);
int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(v), varDataLen(v), buf, NULL);
printf("%d: %s\n", row++, buf);
}else if(pInfo[i].type == TSDB_DATA_TYPE_BINARY || pInfo[i]->type == TSDB_DATA_TYPE_GEOMETRY){
char buf[128] = {0};

2
source/libs/function/src/builtins.c
View File

@ -1664,7 +1664,7 @@ static int32_t translateOutVarchar(SFunctionNode* pFunc, char* pErrBuf, int32_t
bytes = TSDB_TABLE_FNAME_LEN - 1 + VARSTR_HEADER_SIZE;
break;
case FUNCTION_TYPE_TIMEZONE:
bytes = timeZoneStrLen();
bytes = TD_TIMEZONE_LEN + VARSTR_HEADER_SIZE;
break;
case FUNCTION_TYPE_IRATE_PARTIAL:
bytes = getIrateInfoSize((pFunc->hasPk) ? pFunc->pkBytes : 0) + VARSTR_HEADER_SIZE;

2
source/libs/parser/inc/parUtil.h
View File

@ -136,7 +136,7 @@ int32_t trimString(const char* src, int32_t len, char* dst, int32_t dlen);
int32_t getVnodeSysTableTargetName(int32_t acctId, SNode* pWhere, SName* pName);
int32_t checkAndTrimValue(SToken* pToken, char* tmpTokenBuf, SMsgBuf* pMsgBuf, int8_t type);
int32_t parseTagValue(SMsgBuf* pMsgBuf, const char** pSql, uint8_t precision, SSchema* pTagSchema, SToken* pToken,
SArray* pTagName, SArray* pTagVals, STag** pTag, timezone_t tz);
SArray* pTagName, SArray* pTagVals, STag** pTag, timezone_t tz, void *charsetCxt);
int32_t parseTbnameToken(SMsgBuf* pMsgBuf, char* tname, SToken* pToken, bool* pFoundCtbName);
int32_t buildCatalogReq(const SParseMetaCache* pMetaCache, SCatalogReq* pCatalogReq);

9
source/libs/parser/src/parAstCreater.c
View File

@ -421,6 +421,7 @@ SNode* createValueNode(SAstCreateContext* pCxt, int32_t dataType, const SToken*
}
val->translate = false;
val->tz = pCxt->pQueryCxt->timezone;
val->charsetCxt = pCxt->pQueryCxt->charsetCxt;
return (SNode*)val;
_err:
return NULL;
@ -964,6 +965,8 @@ SNode* createOperatorNode(SAstCreateContext* pCxt, EOperatorType type, SNode* pL
op->opType = type;
op->pLeft = pLeft;
op->pRight = pRight;
op->tz = pCxt->pQueryCxt->timezone;
op->charsetCxt = pCxt->pQueryCxt->charsetCxt;
return (SNode*)op;
_err:
nodesDestroyNode(pLeft);
@ -1043,6 +1046,7 @@ SNode* createFunctionNode(SAstCreateContext* pCxt, const SToken* pFuncName, SNod
COPY_STRING_FORM_ID_TOKEN(func->functionName, pFuncName);
func->pParameterList = pParameterList;
func->tz = pCxt->pQueryCxt->timezone;
func->charsetCxt = pCxt->pQueryCxt->charsetCxt;
return (SNode*)func;
_err:
nodesDestroyList(pParameterList);
@ -1064,6 +1068,8 @@ SNode* createCastFunctionNode(SAstCreateContext* pCxt, SNode* pExpr, SDataType d
pCxt->errCode = nodesListMakeAppend(&func->pParameterList, pExpr);
CHECK_PARSER_STATUS(pCxt);
func->tz = pCxt->pQueryCxt->timezone;
func->charsetCxt = pCxt->pQueryCxt->charsetCxt;
return (SNode*)func;
_err:
nodesDestroyNode((SNode*)func);
@ -1098,6 +1104,7 @@ SNode* createTrimFunctionNode(SAstCreateContext* pCxt, SNode* pExpr, ETrimType t
func->trimType = type;
pCxt->errCode = nodesListMakeAppend(&func->pParameterList, pExpr);
CHECK_PARSER_STATUS(pCxt);
func->charsetCxt = pCxt->pQueryCxt->charsetCxt;
return (SNode*)func;
_err:
nodesDestroyNode((SNode*)func);
@ -1116,6 +1123,7 @@ SNode* createTrimFunctionNodeExt(SAstCreateContext* pCxt, SNode* pExpr, SNode* p
CHECK_PARSER_STATUS(pCxt);
pCxt->errCode = nodesListMakeAppend(&func->pParameterList, pExpr2);
CHECK_PARSER_STATUS(pCxt);
func->charsetCxt = pCxt->pQueryCxt->charsetCxt;
return (SNode*)func;
_err:
nodesDestroyNode((SNode*)func);
@ -1501,6 +1509,7 @@ SNode* createCaseWhenNode(SAstCreateContext* pCxt, SNode* pCase, SNodeList* pWhe
pCaseWhen->pWhenThenList = pWhenThenList;
pCaseWhen->pElse = pElse;
pCaseWhen->tz = pCxt->pQueryCxt->timezone;
pCaseWhen->charsetCxt = pCxt->pQueryCxt->charsetCxt;
return (SNode*)pCaseWhen;
_err:
nodesDestroyNode(pCase);

14
source/libs/parser/src/parInsertSml.c
View File

@ -98,7 +98,7 @@ end:
* @return int32_t
*/
static int32_t smlBuildTagRow(SArray* cols, SBoundColInfo* tags, SSchema* pSchema, STag** ppTag, SArray** tagName,
SMsgBuf* msg) {
SMsgBuf* msg, void* charsetCxt) {
SArray* pTagArray = taosArrayInit(tags->numOfBound, sizeof(STagVal));
if (!pTagArray) {
return terrno;
@ -142,7 +142,7 @@ static int32_t smlBuildTagRow(SArray* cols, SBoundColInfo* tags, SSchema* pSchem
code = terrno;
goto end;
}
if (!taosMbsToUcs4(kv->value, kv->length, (TdUcs4*)(p), kv->length * TSDB_NCHAR_SIZE, &output)) {
if (!taosMbsToUcs4(kv->value, kv->length, (TdUcs4*)(p), kv->length * TSDB_NCHAR_SIZE, &output, charsetCxt)) {
if (terrno == TAOS_SYSTEM_ERROR(E2BIG)) {
taosMemoryFree(p);
code = generateSyntaxErrMsg(msg, TSDB_CODE_PAR_VALUE_TOO_LONG, pTagSchema->name);
@ -221,7 +221,7 @@ int32_t smlBuildRow(STableDataCxt* pTableCxt) {
return TSDB_CODE_SUCCESS;
}
int32_t smlBuildCol(STableDataCxt* pTableCxt, SSchema* schema, void* data, int32_t index) {
int32_t smlBuildCol(STableDataCxt* pTableCxt, SSchema* schema, void* data, int32_t index, void* charsetCxt) {
int ret = TSDB_CODE_SUCCESS;
SSchema* pColSchema = schema + index;
SColVal* pVal = taosArrayGet(pTableCxt->pValues, index);
@ -256,7 +256,7 @@ int32_t smlBuildCol(STableDataCxt* pTableCxt, SSchema* schema, void* data, int32
ret = terrno;
goto end;
}
if (!taosMbsToUcs4(kv->value, kv->length, (TdUcs4*)pUcs4, size, &len)) {
if (!taosMbsToUcs4(kv->value, kv->length, (TdUcs4*)pUcs4, size, &len, charsetCxt)) {
if (terrno == TAOS_SYSTEM_ERROR(E2BIG)) {
taosMemoryFree(pUcs4);
ret = TSDB_CODE_PAR_VALUE_TOO_LONG;
@ -291,7 +291,7 @@ end:
int32_t smlBindData(SQuery* query, bool dataFormat, SArray* tags, SArray* colsSchema, SArray* cols,
STableMeta* pTableMeta, char* tableName, const char* sTableName, int32_t sTableNameLen, int32_t ttl,
char* msgBuf, int32_t msgBufLen) {
char* msgBuf, int32_t msgBufLen, void* charsetCxt) {
SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen};
SSchema* pTagsSchema = getTableTagSchema(pTableMeta);
@ -313,7 +313,7 @@ int32_t smlBindData(SQuery* query, bool dataFormat, SArray* tags, SArray* colsSc
STag* pTag = NULL;
ret = smlBuildTagRow(tags, &bindTags, pTagsSchema, &pTag, &tagName, &pBuf);
ret = smlBuildTagRow(tags, &bindTags, pTagsSchema, &pTag, &tagName, &pBuf, charsetCxt);
if (ret != TSDB_CODE_SUCCESS) {
goto end;
}
@ -411,7 +411,7 @@ int32_t smlBindData(SQuery* query, bool dataFormat, SArray* tags, SArray* colsSc
ret = terrno;
goto end;
}
if (!taosMbsToUcs4(kv->value, kv->length, (TdUcs4*)pUcs4, pColSchema->bytes - VARSTR_HEADER_SIZE, &len)) {
if (!taosMbsToUcs4(kv->value, kv->length, (TdUcs4*)pUcs4, pColSchema->bytes - VARSTR_HEADER_SIZE, &len, charsetCxt)) {
if (terrno == TAOS_SYSTEM_ERROR(E2BIG)) {
uError("sml bind taosMbsToUcs4 error, kv length:%d, bytes:%d, kv->value:%s", (int)kv->length,
pColSchema->bytes, kv->value);

18
source/libs/parser/src/parInsertSql.c
View File

@ -477,7 +477,7 @@ static int32_t parseVarbinary(SToken* pToken, uint8_t** pData, uint32_t* nData,
}
static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema, int16_t timePrec, STagVal* val,
SMsgBuf* pMsgBuf, timezone_t tz) {
SMsgBuf* pMsgBuf, timezone_t tz, void *charsetCxt) {
int64_t iv;
uint64_t uv;
char* endptr = NULL;
@ -685,7 +685,7 @@ static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema,
if (p == NULL) {
return terrno;
}
if (!taosMbsToUcs4(pToken->z, pToken->n, (TdUcs4*)(p), realLen, &output)) {
if (!taosMbsToUcs4(pToken->z, pToken->n, (TdUcs4*)(p), realLen, &output, charsetCxt)) {
if (terrno == TAOS_SYSTEM_ERROR(E2BIG)) {
taosMemoryFree(p);
return generateSyntaxErrMsg(pMsgBuf, TSDB_CODE_PAR_VALUE_TOO_LONG, pSchema->name);
@ -732,7 +732,7 @@ static int32_t parseBoundTagsClause(SInsertParseContext* pCxt, SVnodeModifyOpStm
}
int32_t parseTagValue(SMsgBuf* pMsgBuf, const char** pSql, uint8_t precision, SSchema* pTagSchema, SToken* pToken,
SArray* pTagName, SArray* pTagVals, STag** pTag, timezone_t tz) {
SArray* pTagName, SArray* pTagVals, STag** pTag, timezone_t tz, void *charsetCxt) {
bool isNull = isNullValue(pTagSchema->type, pToken);
if (!isNull && pTagName) {
if (NULL == taosArrayPush(pTagName, pTagSchema->name)) {
@ -748,14 +748,14 @@ int32_t parseTagValue(SMsgBuf* pMsgBuf, const char** pSql, uint8_t precision, SS
if (isNull) {
return tTagNew(pTagVals, 1, true, pTag);
} else {
return parseJsontoTagData(pToken->z, pTagVals, pTag, pMsgBuf);
return parseJsontoTagData(pToken->z, pTagVals, pTag, pMsgBuf, charsetCxt);
}
}
if (isNull) return 0;
STagVal val = {0};
int32_t code = parseTagToken(pSql, pToken, pTagSchema, precision, &val, pMsgBuf, tz);
int32_t code = parseTagToken(pSql, pToken, pTagSchema, precision, &val, pMsgBuf, tz, charsetCxt);
if (TSDB_CODE_SUCCESS == code) {
if (NULL == taosArrayPush(pTagVals, &val)) {
code = terrno;
@ -978,7 +978,7 @@ static int32_t parseTagsClauseImpl(SInsertParseContext* pCxt, SVnodeModifyOpStmt
code = buildSyntaxErrMsg(&pCxt->msg, "not expected tags values ", token.z);
}
if (TSDB_CODE_SUCCESS == code) {
code = parseTagValue(&pCxt->msg, &pStmt->pSql, precision, pTagSchema, &token, pTagName, pTagVals, &pTag, pCxt->pComCxt->timezone);
code = parseTagValue(&pCxt->msg, &pStmt->pSql, precision, pTagSchema, &token, pTagName, pTagVals, &pTag, pCxt->pComCxt->timezone, pCxt->pComCxt->charsetCxt);
}
}
@ -1620,7 +1620,7 @@ static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql,
if (NULL == pUcs4) {
return terrno;
}
if (!taosMbsToUcs4(pToken->z, pToken->n, (TdUcs4*)pUcs4, realLen, &len)) {
if (!taosMbsToUcs4(pToken->z, pToken->n, (TdUcs4*)pUcs4, realLen, &len, pCxt->pComCxt->charsetCxt)) {
taosMemoryFree(pUcs4);
if (terrno == TAOS_SYSTEM_ERROR(E2BIG)) {
return generateSyntaxErrMsg(&pCxt->msg, TSDB_CODE_PAR_VALUE_TOO_LONG, pSchema->name);
@ -1788,7 +1788,7 @@ static int32_t processCtbTagsAfterCtbName(SInsertParseContext* pCxt, SVnodeModif
if (code == TSDB_CODE_SUCCESS) {
code = parseTagValue(&pCxt->msg, NULL, precision, pTagSchema, pTagToken, pStbRowsCxt->aTagNames,
pStbRowsCxt->aTagVals, &pStbRowsCxt->pTag, pCxt->pComCxt->timezone);
pStbRowsCxt->aTagVals, &pStbRowsCxt->pTag, pCxt->pComCxt->timezone, pCxt->pComCxt->charsetCxt);
}
}
if (code == TSDB_CODE_SUCCESS && !pStbRowsCxt->isJsonTag) {
@ -1858,7 +1858,7 @@ static int32_t doGetStbRowValues(SInsertParseContext* pCxt, SVnodeModifyOpStmt*
}
if (code == TSDB_CODE_SUCCESS) {
code = parseTagValue(&pCxt->msg, ppSql, precision, (SSchema*)pTagSchema, pToken, pTagNames, pTagVals,
&pStbRowsCxt->pTag, pCxt->pComCxt->timezone);
&pStbRowsCxt->pTag, pCxt->pComCxt->timezone, pCxt->pComCxt->charsetCxt);
}
}
} else if (pCols->pColIndex[i] == tbnameIdx) {

48
source/libs/parser/src/parInsertStmt.c
View File

@ -123,7 +123,7 @@ int32_t qBuildStmtOutput(SQuery* pQuery, SHashObj* pVgHash, SHashObj* pBlockHash
}
int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, const char* sTableName, char* tName,
TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen) {
TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen, void* charsetCxt) {
STableDataCxt* pDataBlock = (STableDataCxt*)pBlock;
SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen};
int32_t code = TSDB_CODE_SUCCESS;
@ -183,7 +183,7 @@ int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, const ch
goto end;
}
memcpy(tmp, bind[c].buffer, colLen);
code = parseJsontoTagData(tmp, pTagArray, &pTag, &pBuf);
code = parseJsontoTagData(tmp, pTagArray, &pTag, &pBuf, charsetCxt);
taosMemoryFree(tmp);
if (code != TSDB_CODE_SUCCESS) {
goto end;
@ -208,7 +208,7 @@ int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, const ch
code = terrno;
goto end;
}
if (!taosMbsToUcs4(bind[c].buffer, colLen, (TdUcs4*)(p), colLen * TSDB_NCHAR_SIZE, &output)) {
if (!taosMbsToUcs4(bind[c].buffer, colLen, (TdUcs4*)(p), colLen * TSDB_NCHAR_SIZE, &output, charsetCxt)) {
if (terrno == TAOS_SYSTEM_ERROR(E2BIG)) {
taosMemoryFree(p);
code = generateSyntaxErrMsg(&pBuf, TSDB_CODE_PAR_VALUE_TOO_LONG, pTagSchema->name);
@ -266,7 +266,7 @@ end:
return code;
}
int32_t convertStmtNcharCol(SMsgBuf* pMsgBuf, SSchema* pSchema, TAOS_MULTI_BIND* src, TAOS_MULTI_BIND* dst) {
int32_t convertStmtNcharCol(SMsgBuf* pMsgBuf, SSchema* pSchema, TAOS_MULTI_BIND* src, TAOS_MULTI_BIND* dst, void* charsetCxt) {
int32_t output = 0;
int32_t newBuflen = (pSchema->bytes - VARSTR_HEADER_SIZE) * src->num;
if (dst->buffer_length < newBuflen) {
@ -292,7 +292,7 @@ int32_t convertStmtNcharCol(SMsgBuf* pMsgBuf, SSchema* pSchema, TAOS_MULTI_BIND*
}
if (!taosMbsToUcs4(((char*)src->buffer) + src->buffer_length * i, src->length[i],
(TdUcs4*)(((char*)dst->buffer) + dst->buffer_length * i), dst->buffer_length, &output)) {
(TdUcs4*)(((char*)dst->buffer) + dst->buffer_length * i), dst->buffer_length, &output, charsetCxt)) {
if (terrno == TAOS_SYSTEM_ERROR(E2BIG)) {
return generateSyntaxErrMsg(pMsgBuf, TSDB_CODE_PAR_VALUE_TOO_LONG, pSchema->name);
}
@ -312,7 +312,7 @@ int32_t convertStmtNcharCol(SMsgBuf* pMsgBuf, SSchema* pSchema, TAOS_MULTI_BIND*
}
int32_t qBindStmtStbColsValue(void* pBlock, SArray* pCols, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen,
STSchema** pTSchema, SBindInfo* pBindInfos) {
STSchema** pTSchema, SBindInfo* pBindInfos, void* charsetCxt) {
STableDataCxt* pDataBlock = (STableDataCxt*)pBlock;
SSchema* pSchema = getTableColumnSchema(pDataBlock->pMeta);
SBoundColInfo* boundInfo = &pDataBlock->boundColsInfo;
@ -349,7 +349,7 @@ int32_t qBindStmtStbColsValue(void* pBlock, SArray* pCols, TAOS_MULTI_BIND* bind
}
if (TSDB_DATA_TYPE_NCHAR == pColSchema->type) {
code = convertStmtNcharCol(&pBuf, pColSchema, bind + c, &ncharBind);
code = convertStmtNcharCol(&pBuf, pColSchema, bind + c, &ncharBind, charsetCxt);
if (code) {
goto _return;
}
@ -380,7 +380,7 @@ _return:
return code;
}
int32_t qBindStmtColsValue(void* pBlock, SArray* pCols, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen) {
int32_t qBindStmtColsValue(void* pBlock, SArray* pCols, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen, void* charsetCxt) {
STableDataCxt* pDataBlock = (STableDataCxt*)pBlock;
SSchema* pSchema = getTableColumnSchema(pDataBlock->pMeta);
SBoundColInfo* boundInfo = &pDataBlock->boundColsInfo;
@ -406,7 +406,7 @@ int32_t qBindStmtColsValue(void* pBlock, SArray* pCols, TAOS_MULTI_BIND* bind, c
}
if (TSDB_DATA_TYPE_NCHAR == pColSchema->type) {
code = convertStmtNcharCol(&pBuf, pColSchema, bind + c, &ncharBind);
code = convertStmtNcharCol(&pBuf, pColSchema, bind + c, &ncharBind, charsetCxt);
if (code) {
goto _return;
}
@ -434,7 +434,7 @@ _return:
}
int32_t qBindStmtSingleColValue(void* pBlock, SArray* pCols, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen,
int32_t colIdx, int32_t rowNum) {
int32_t colIdx, int32_t rowNum, void* charsetCxt) {
STableDataCxt* pDataBlock = (STableDataCxt*)pBlock;
SSchema* pSchema = getTableColumnSchema(pDataBlock->pMeta);
SBoundColInfo* boundInfo = &pDataBlock->boundColsInfo;
@ -459,7 +459,7 @@ int32_t qBindStmtSingleColValue(void* pBlock, SArray* pCols, TAOS_MULTI_BIND* bi
}
if (TSDB_DATA_TYPE_NCHAR == pColSchema->type) {
code = convertStmtNcharCol(&pBuf, pColSchema, bind, &ncharBind);
code = convertStmtNcharCol(&pBuf, pColSchema, bind, &ncharBind, charsetCxt);
if (code) {
goto _return;
}
@ -483,7 +483,7 @@ _return:
}
int32_t qBindStmtTagsValue2(void* pBlock, void* boundTags, int64_t suid, const char* sTableName, char* tName,
TAOS_STMT2_BIND* bind, char* msgBuf, int32_t msgBufLen) {
TAOS_STMT2_BIND* bind, char* msgBuf, int32_t msgBufLen, void* charsetCxt) {
STableDataCxt* pDataBlock = (STableDataCxt*)pBlock;
SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen};
int32_t code = TSDB_CODE_SUCCESS;
@ -543,7 +543,7 @@ int32_t qBindStmtTagsValue2(void* pBlock, void* boundTags, int64_t suid, const c
goto end;
}
memcpy(tmp, bind[c].buffer, colLen);
code = parseJsontoTagData(tmp, pTagArray, &pTag, &pBuf);
code = parseJsontoTagData(tmp, pTagArray, &pTag, &pBuf, charsetCxt);
taosMemoryFree(tmp);
if (code != TSDB_CODE_SUCCESS) {
goto end;
@ -568,7 +568,7 @@ int32_t qBindStmtTagsValue2(void* pBlock, void* boundTags, int64_t suid, const c
code = terrno;
goto end;
}
if (!taosMbsToUcs4(bind[c].buffer, colLen, (TdUcs4*)(p), colLen * TSDB_NCHAR_SIZE, &output)) {
if (!taosMbsToUcs4(bind[c].buffer, colLen, (TdUcs4*)(p), colLen * TSDB_NCHAR_SIZE, &output, charsetCxt)) {
if (terrno == TAOS_SYSTEM_ERROR(E2BIG)) {
taosMemoryFree(p);
code = generateSyntaxErrMsg(&pBuf, TSDB_CODE_PAR_VALUE_TOO_LONG, pTagSchema->name);
@ -626,7 +626,7 @@ end:
return code;
}
static int32_t convertStmtStbNcharCol2(SMsgBuf* pMsgBuf, SSchema* pSchema, TAOS_STMT2_BIND* src, TAOS_STMT2_BIND* dst) {
static int32_t convertStmtStbNcharCol2(SMsgBuf* pMsgBuf, SSchema* pSchema, TAOS_STMT2_BIND* src, TAOS_STMT2_BIND* dst, void *charsetCxt) {
int32_t output = 0;
const int32_t max_buf_len = pSchema->bytes - VARSTR_HEADER_SIZE;
@ -648,7 +648,7 @@ static int32_t convertStmtStbNcharCol2(SMsgBuf* pMsgBuf, SSchema* pSchema, TAOS_
continue;
}
if (!taosMbsToUcs4(src_buf, src->length[i], (TdUcs4*)dst_buf, max_buf_len, &output)) {
if (!taosMbsToUcs4(src_buf, src->length[i], (TdUcs4*)dst_buf, max_buf_len, &output, charsetCxt)) {
if (terrno == TAOS_SYSTEM_ERROR(E2BIG)) {
return generateSyntaxErrMsg(pMsgBuf, TSDB_CODE_PAR_VALUE_TOO_LONG, pSchema->name);
}
@ -670,7 +670,7 @@ static int32_t convertStmtStbNcharCol2(SMsgBuf* pMsgBuf, SSchema* pSchema, TAOS_
}
int32_t qBindStmtStbColsValue2(void* pBlock, SArray* pCols, TAOS_STMT2_BIND* bind, char* msgBuf, int32_t msgBufLen,
STSchema** pTSchema, SBindInfo2* pBindInfos) {
STSchema** pTSchema, SBindInfo2* pBindInfos, void *charsetCxt) {
STableDataCxt* pDataBlock = (STableDataCxt*)pBlock;
SSchema* pSchema = getTableColumnSchema(pDataBlock->pMeta);
SBoundColInfo* boundInfo = &pDataBlock->boundColsInfo;
@ -720,7 +720,7 @@ int32_t qBindStmtStbColsValue2(void* pBlock, SArray* pCols, TAOS_STMT2_BIND* bin
}
if (TSDB_DATA_TYPE_NCHAR == pColSchema->type) {
code = convertStmtStbNcharCol2(&pBuf, pColSchema, bind + c, &ncharBind);
code = convertStmtStbNcharCol2(&pBuf, pColSchema, bind + c, &ncharBind, charsetCxt);
if (code) {
goto _return;
}
@ -755,7 +755,7 @@ _return:
return code;
}
static int32_t convertStmtNcharCol2(SMsgBuf* pMsgBuf, SSchema* pSchema, TAOS_STMT2_BIND* src, TAOS_STMT2_BIND* dst) {
static int32_t convertStmtNcharCol2(SMsgBuf* pMsgBuf, SSchema* pSchema, TAOS_STMT2_BIND* src, TAOS_STMT2_BIND* dst, void *charsetCxt) {
int32_t output = 0;
const int32_t max_buf_len = pSchema->bytes - VARSTR_HEADER_SIZE;
@ -785,7 +785,7 @@ static int32_t convertStmtNcharCol2(SMsgBuf* pMsgBuf, SSchema* pSchema, TAOS_STM
/*if (!taosMbsToUcs4(((char*)src->buffer) + src->buffer_length * i, src->length[i],
(TdUcs4*)(((char*)dst->buffer) + dst->buffer_length * i), dst->buffer_length, &output)) {*/
if (!taosMbsToUcs4(src_buf, src->length[i], (TdUcs4*)dst_buf, max_buf_len, &output)) {
if (!taosMbsToUcs4(src_buf, src->length[i], (TdUcs4*)dst_buf, max_buf_len, &output, charsetCxt)) {
if (terrno == TAOS_SYSTEM_ERROR(E2BIG)) {
return generateSyntaxErrMsg(pMsgBuf, TSDB_CODE_PAR_VALUE_TOO_LONG, pSchema->name);
}
@ -806,7 +806,7 @@ static int32_t convertStmtNcharCol2(SMsgBuf* pMsgBuf, SSchema* pSchema, TAOS_STM
return TSDB_CODE_SUCCESS;
}
int32_t qBindStmtColsValue2(void* pBlock, SArray* pCols, TAOS_STMT2_BIND* bind, char* msgBuf, int32_t msgBufLen) {
int32_t qBindStmtColsValue2(void* pBlock, SArray* pCols, TAOS_STMT2_BIND* bind, char* msgBuf, int32_t msgBufLen, void *charsetCxt) {
STableDataCxt* pDataBlock = (STableDataCxt*)pBlock;
SSchema* pSchema = getTableColumnSchema(pDataBlock->pMeta);
SBoundColInfo* boundInfo = &pDataBlock->boundColsInfo;
@ -836,7 +836,7 @@ int32_t qBindStmtColsValue2(void* pBlock, SArray* pCols, TAOS_STMT2_BIND* bind,
}
if (TSDB_DATA_TYPE_NCHAR == pColSchema->type) {
code = convertStmtNcharCol2(&pBuf, pColSchema, bind + c, &ncharBind);
code = convertStmtNcharCol2(&pBuf, pColSchema, bind + c, &ncharBind, charsetCxt);
if (code) {
goto _return;
}
@ -864,7 +864,7 @@ _return:
}
int32_t qBindStmtSingleColValue2(void* pBlock, SArray* pCols, TAOS_STMT2_BIND* bind, char* msgBuf, int32_t msgBufLen,
int32_t colIdx, int32_t rowNum) {
int32_t colIdx, int32_t rowNum, void *charsetCxt) {
STableDataCxt* pDataBlock = (STableDataCxt*)pBlock;
SSchema* pSchema = getTableColumnSchema(pDataBlock->pMeta);
SBoundColInfo* boundInfo = &pDataBlock->boundColsInfo;
@ -889,7 +889,7 @@ int32_t qBindStmtSingleColValue2(void* pBlock, SArray* pCols, TAOS_STMT2_BIND* b
}
if (TSDB_DATA_TYPE_NCHAR == pColSchema->type) {
code = convertStmtNcharCol2(&pBuf, pColSchema, bind, &ncharBind);
code = convertStmtNcharCol2(&pBuf, pColSchema, bind, &ncharBind, charsetCxt);
if (code) {
goto _return;
}

14
source/libs/parser/src/parTranslater.c
View File

@ -2148,7 +2148,7 @@ static EDealRes translateNormalValue(STranslateContext* pCxt, SValueNode* pVal,
int32_t len = 0;
if (!taosMbsToUcs4(pVal->literal, strlen(pVal->literal), (TdUcs4*)varDataVal(pVal->datum.p),
targetDt.bytes - VARSTR_HEADER_SIZE, &len)) {
targetDt.bytes - VARSTR_HEADER_SIZE, &len, pCxt->pParseCxt->charsetCxt)) {
return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, pVal->literal);
}
varDataSetLen(pVal->datum.p, len);
@ -14169,7 +14169,7 @@ static int32_t buildKVRowForBindTags(STranslateContext* pCxt, SCreateSubTableCla
if (pSchema->type == TSDB_DATA_TYPE_JSON) {
isJson = true;
}
code = parseTagValue(&pCxt->msgBuf, &tagStr, precision, pSchema, &token, tagName, pTagArray, ppTag, pCxt->pParseCxt->timezone);
code = parseTagValue(&pCxt->msgBuf, &tagStr, precision, pSchema, &token, tagName, pTagArray, ppTag, pCxt->pParseCxt->timezone, pCxt->pParseCxt->charsetCxt);
}
if (TSDB_CODE_SUCCESS == code) {
@ -14230,7 +14230,7 @@ static int32_t buildKVRowForAllTags(STranslateContext* pCxt, SCreateSubTableClau
if (pTagSchema->type == TSDB_DATA_TYPE_JSON) {
isJson = true;
}
code = parseTagValue(&pCxt->msgBuf, &tagStr, precision, pTagSchema, &token, tagName, pTagArray, ppTag, pCxt->pParseCxt->timezone);
code = parseTagValue(&pCxt->msgBuf, &tagStr, precision, pTagSchema, &token, tagName, pTagArray, ppTag, pCxt->pParseCxt->timezone, pCxt->pParseCxt->charsetCxt);
}
if (TSDB_CODE_SUCCESS == code) {
@ -14426,7 +14426,7 @@ static int32_t fillVgroupInfo(SParseContext* pParseCxt, const SName* pName, SVgr
return code;
}
static int32_t parseOneStbRow(SMsgBuf* pMsgBuf, SParseFileContext* pParFileCxt, timezone_t tz) {
static int32_t parseOneStbRow(SMsgBuf* pMsgBuf, SParseFileContext* pParFileCxt, timezone_t tz, void *charsetCxt) {
int32_t code = TSDB_CODE_SUCCESS;
int sz = taosArrayGetSize(pParFileCxt->aTagIndexs);
int32_t numOfTags = getNumOfTags(pParFileCxt->pStbMeta);
@ -14458,7 +14458,7 @@ static int32_t parseOneStbRow(SMsgBuf* pMsgBuf, SParseFileContext* pParFileCxt,
if (TSDB_CODE_SUCCESS == code) {
SArray* aTagNames = pParFileCxt->tagNameFilled ? NULL : pParFileCxt->aTagNames;
code = parseTagValue(pMsgBuf, &pParFileCxt->pSql, precision, (SSchema*)pTagSchema, &token, aTagNames,
pParFileCxt->aTagVals, &pParFileCxt->pTag, tz);
pParFileCxt->aTagVals, &pParFileCxt->pTag, tz, charsetCxt);
}
} else {
// parse tbname
@ -14524,7 +14524,7 @@ static int32_t parseCsvFile(SMsgBuf* pMsgBuf, SParseContext* pParseCxt, SParseFi
(void)strtolower(pLine, pLine);
pParFileCxt->pSql = pLine;
code = parseOneStbRow(pMsgBuf, pParFileCxt, pParseCxt->timezone);
code = parseOneStbRow(pMsgBuf, pParFileCxt, pParseCxt->timezone, pParseCxt->charsetCxt);
if (TSDB_CODE_SUCCESS == code) {
code = fillVgroupInfo(pParseCxt, &pParFileCxt->ctbName, &pParFileCxt->vg);
@ -15225,7 +15225,7 @@ static int32_t buildUpdateTagValReq(STranslateContext* pCxt, SAlterTableStmt* pS
if (TSDB_CODE_SUCCESS == code) {
code = parseTagValue(&pCxt->msgBuf, &tagStr, pTableMeta->tableInfo.precision, pSchema, &token, NULL,
pReq->pTagArray, &pTag, pCxt->pParseCxt->timezone);
pReq->pTagArray, &pTag, pCxt->pParseCxt->timezone, pCxt->pParseCxt->charsetCxt);
if (pSchema->type == TSDB_DATA_TYPE_JSON && token.type == TK_NULL && code == TSDB_CODE_SUCCESS) {
pReq->tagFree = true;
}

4
source/libs/parser/src/parUtil.c
View File

@ -402,7 +402,7 @@ static bool isValidateTag(char* input) {
return true;
}
int32_t parseJsontoTagData(const char* json, SArray* pTagVals, STag** ppTag, void* pMsgBuf) {
int32_t parseJsontoTagData(const char* json, SArray* pTagVals, STag** ppTag, void* pMsgBuf, void *charsetCxt) {
int32_t retCode = TSDB_CODE_SUCCESS;
cJSON* root = NULL;
SHashObj* keyHash = NULL;
@ -471,7 +471,7 @@ int32_t parseJsontoTagData(const char* json, SArray* pTagVals, STag** ppTag, voi
goto end;
}
val.type = TSDB_DATA_TYPE_NCHAR;
if (valLen > 0 && !taosMbsToUcs4(jsonValue, valLen, (TdUcs4*)tmp, (int32_t)(valLen * TSDB_NCHAR_SIZE), &valLen)) {
if (valLen > 0 && !taosMbsToUcs4(jsonValue, valLen, (TdUcs4*)tmp, (int32_t)(valLen * TSDB_NCHAR_SIZE), &valLen, charsetCxt)) {
uError("charset:%s to %s. val:%s, errno:%s, convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, jsonValue,
strerror(terrno));
retCode = buildSyntaxErrMsg(pMsgBuf, "charset convert json error", jsonValue);

20
source/libs/parser/src/parser.c
View File

@ -153,7 +153,7 @@ static int32_t parseSqlSyntax(SParseContext* pCxt, SQuery** pQuery, SParseMetaCa
return code;
}
static int32_t setValueByBindParam(SValueNode* pVal, TAOS_MULTI_BIND* pParam) {
static int32_t setValueByBindParam(SValueNode* pVal, TAOS_MULTI_BIND* pParam, void *charsetCxt) {
if (!pParam || IS_NULL_TYPE(pParam->buffer_type)) {
return TSDB_CODE_APP_ERROR;
}
@ -200,7 +200,7 @@ static int32_t setValueByBindParam(SValueNode* pVal, TAOS_MULTI_BIND* pParam) {
int32_t output = 0;
if (!taosMbsToUcs4(pParam->buffer, inputSize, (TdUcs4*)varDataVal(pVal->datum.p), pVal->node.resType.bytes,
&output)) {
&output, charsetCxt)) {
return terrno;
}
varDataSetLen(pVal->datum.p, output);
@ -417,19 +417,19 @@ int32_t qInitKeywordsTable() { return taosInitKeywordsTable(); }
void qCleanupKeywordsTable() { taosCleanupKeywordsTable(); }
int32_t qStmtBindParams(SQuery* pQuery, TAOS_MULTI_BIND* pParams, int32_t colIdx) {
int32_t qStmtBindParams(SQuery* pQuery, TAOS_MULTI_BIND* pParams, int32_t colIdx, void *charsetCxt) {
int32_t code = TSDB_CODE_SUCCESS;
if (colIdx < 0) {
int32_t size = taosArrayGetSize(pQuery->pPlaceholderValues);
for (int32_t i = 0; i < size; ++i) {
code = setValueByBindParam((SValueNode*)taosArrayGetP(pQuery->pPlaceholderValues, i), pParams + i);
code = setValueByBindParam((SValueNode*)taosArrayGetP(pQuery->pPlaceholderValues, i), pParams + i, charsetCxt);
if (TSDB_CODE_SUCCESS != code) {
return code;
}
}
} else {
code = setValueByBindParam((SValueNode*)taosArrayGetP(pQuery->pPlaceholderValues, colIdx), pParams);
code = setValueByBindParam((SValueNode*)taosArrayGetP(pQuery->pPlaceholderValues, colIdx), pParams, charsetCxt);
}
if (TSDB_CODE_SUCCESS == code && (colIdx < 0 || colIdx + 1 == pQuery->placeholderNum)) {
@ -443,7 +443,7 @@ int32_t qStmtBindParams(SQuery* pQuery, TAOS_MULTI_BIND* pParams, int32_t colIdx
return code;
}
static int32_t setValueByBindParam2(SValueNode* pVal, TAOS_STMT2_BIND* pParam) {
static int32_t setValueByBindParam2(SValueNode* pVal, TAOS_STMT2_BIND* pParam, void* charsetCxt) {
if (!pParam || IS_NULL_TYPE(pParam->buffer_type)) {
return TSDB_CODE_APP_ERROR;
}
@ -490,7 +490,7 @@ static int32_t setValueByBindParam2(SValueNode* pVal, TAOS_STMT2_BIND* pParam) {
int32_t output = 0;
if (!taosMbsToUcs4(pParam->buffer, inputSize, (TdUcs4*)varDataVal(pVal->datum.p), pVal->node.resType.bytes,
&output)) {
&output, charsetCxt)) {
return terrno;
}
varDataSetLen(pVal->datum.p, output);
@ -509,19 +509,19 @@ static int32_t setValueByBindParam2(SValueNode* pVal, TAOS_STMT2_BIND* pParam) {
return TSDB_CODE_SUCCESS;
}
int32_t qStmtBindParams2(SQuery* pQuery, TAOS_STMT2_BIND* pParams, int32_t colIdx) {
int32_t qStmtBindParams2(SQuery* pQuery, TAOS_STMT2_BIND* pParams, int32_t colIdx, void* charsetCxt) {
int32_t code = TSDB_CODE_SUCCESS;
if (colIdx < 0) {
int32_t size = taosArrayGetSize(pQuery->pPlaceholderValues);
for (int32_t i = 0; i < size; ++i) {
code = setValueByBindParam2((SValueNode*)taosArrayGetP(pQuery->pPlaceholderValues, i), pParams + i);
code = setValueByBindParam2((SValueNode*)taosArrayGetP(pQuery->pPlaceholderValues, i), pParams + i, charsetCxt);
if (TSDB_CODE_SUCCESS != code) {
return code;
}
}
} else {
code = setValueByBindParam2((SValueNode*)taosArrayGetP(pQuery->pPlaceholderValues, colIdx), pParams);
code = setValueByBindParam2((SValueNode*)taosArrayGetP(pQuery->pPlaceholderValues, colIdx), pParams, charsetCxt);
}
if (TSDB_CODE_SUCCESS == code && (colIdx < 0 || colIdx + 1 == pQuery->placeholderNum)) {

2
source/libs/planner/src/planOptimizer.c
View File

@ -6725,7 +6725,7 @@ static int32_t fillTSMAOptCtx(STSMAOptCtx* pTsmaOptCtx, SScanLogicNode* pScan) {
pTsmaOptCtx->queryInterval->sliding = pWindow->sliding;
pTsmaOptCtx->queryInterval->slidingUnit = pWindow->slidingUnit;
pTsmaOptCtx->queryInterval->precision = pWindow->node.precision;
pTsmaOptCtx->queryInterval->tz = tsTimezone;
pTsmaOptCtx->queryInterval->tz = taosGetLocalTimezoneOffset();
pTsmaOptCtx->pAggFuncs = pWindow->pFuncs;
pTsmaOptCtx->ppParentTsmaSubplans = &pWindow->pTsmaSubplans;
} else {

2
source/libs/planner/test/planTestUtil.cpp
View File

@ -377,7 +377,7 @@ class PlannerTestBaseImpl {
}
void doBindParams(SQuery* pQuery, TAOS_MULTI_BIND* pParams, int32_t colIdx) {
DO_WITH_THROW(qStmtBindParams, pQuery, pParams, colIdx);
DO_WITH_THROW(qStmtBindParams, pQuery, pParams, colIdx, NULL);
if (colIdx < 0 || pQuery->placeholderNum == colIdx + 1) {
res_.boundAst_ = toString(pQuery->pRoot);
}

6
source/libs/qcom/src/queryUtil.c
View File

@ -397,7 +397,7 @@ int32_t dataConverToStr(char* str, int64_t capacity, int type, void* buf, int32_
}
*str = '"';
int32_t length = taosUcs4ToMbs((TdUcs4*)buf, bufSize, str + 1);
int32_t length = taosUcs4ToMbs((TdUcs4*)buf, bufSize, str + 1, NULL);
if (length <= 0) {
return TSDB_CODE_TSC_INVALID_VALUE;
}
@ -430,7 +430,7 @@ int32_t dataConverToStr(char* str, int64_t capacity, int type, void* buf, int32_
return TSDB_CODE_SUCCESS;
}
void parseTagDatatoJson(void* p, char** jsonStr) {
void parseTagDatatoJson(void* p, char** jsonStr, void *charsetCxt) {
if (!p || !jsonStr) {
qError("parseTagDatatoJson invalid input, line:%d", __LINE__);
return;
@ -475,7 +475,7 @@ void parseTagDatatoJson(void* p, char** jsonStr) {
if (tagJsonValue == NULL) {
goto end;
}
int32_t length = taosUcs4ToMbs((TdUcs4*)pTagVal->pData, pTagVal->nData, tagJsonValue);
int32_t length = taosUcs4ToMbs((TdUcs4*)pTagVal->pData, pTagVal->nData, tagJsonValue, charsetCxt);
if (length < 0) {
qError("charset:%s to %s. val:%s convert json value failed.", DEFAULT_UNICODE_ENCODEC, tsCharset,
pTagVal->pData);

8
source/libs/scalar/src/filter.c
View File

@ -2266,7 +2266,7 @@ int32_t fltInitValFieldData(SFilterInfo *info) {
// match/nmatch for nchar type need convert from ucs4 to mbs
if (type == TSDB_DATA_TYPE_NCHAR && (unit->compare.optr == OP_TYPE_MATCH || unit->compare.optr == OP_TYPE_NMATCH)) {
char newValData[TSDB_REGEX_STRING_DEFAULT_LEN * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE] = {0};
int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(fi->data), varDataLen(fi->data), varDataVal(newValData));
int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(fi->data), varDataLen(fi->data), varDataVal(newValData), NULL);
if (len < 0) {
qError("filterInitValFieldData taosUcs4ToMbs error 1");
return TSDB_CODE_SCALAR_CONVERT_ERROR;
@ -3603,7 +3603,7 @@ int32_t filterExecuteImplMisc(void *pinfo, int32_t numOfRows, SColumnInfoData *p
if (newColData == NULL) {
FLT_ERR_RET(terrno);
}
int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(colData), varDataLen(colData), varDataVal(newColData));
int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(colData), varDataLen(colData), varDataVal(newColData), NULL);
if (len < 0) {
qError("castConvert1 taosUcs4ToMbs error");
taosMemoryFreeClear(newColData);
@ -3678,7 +3678,7 @@ int32_t filterExecuteImpl(void *pinfo, int32_t numOfRows, SColumnInfoData *pRes,
if (newColData == NULL) {
FLT_ERR_RET(terrno);
}
int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(colData), varDataLen(colData), varDataVal(newColData));
int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(colData), varDataLen(colData), varDataVal(newColData), NULL);
if (len < 0) {
qError("castConvert1 taosUcs4ToMbs error");
taosMemoryFreeClear(newColData);
@ -4614,7 +4614,7 @@ int32_t filterConverNcharColumns(SFilterInfo *info, int32_t rows, bool *gotNchar
varDataCopy(dst, src);
continue;
}
bool ret = taosMbsToUcs4(varDataVal(src), varDataLen(src), (TdUcs4 *)varDataVal(dst), bufSize, &len);
bool ret = taosMbsToUcs4(varDataVal(src), varDataLen(src), (TdUcs4 *)varDataVal(dst), bufSize, &len, NULL);
if (!ret) {
qError("filterConverNcharColumns taosMbsToUcs4 error");
return TSDB_CODE_SCALAR_CONVERT_ERROR;

7
source/libs/scalar/src/scalar.c
View File

@ -24,7 +24,7 @@ int32_t scalarGetOperatorParamNum(EOperatorType type) {
int32_t sclConvertToTsValueNode(int8_t precision, SValueNode *valueNode) {
char *timeStr = valueNode->datum.p;
int64_t value = 0;
int32_t code = convertStringToTimestamp(valueNode->node.resType.type, valueNode->datum.p, precision, &value, valueNode->tz); //todo tz
int32_t code = convertStringToTimestamp(valueNode->node.resType.type, valueNode->datum.p, precision, &value, valueNode->tz, valueNode->charsetCxt); //todo tz
if (code != TSDB_CODE_SUCCESS) {
return code;
}
@ -82,7 +82,9 @@ int32_t sclConvertValueToSclParam(SValueNode *pValueNode, SScalarParam *out, int
}
in.tz = pValueNode->tz;
in.charsetCxt = pValueNode->charsetCxt;
out->tz = pValueNode->tz;
out->charsetCxt = pValueNode->charsetCxt;
code = vectorConvertSingleColImpl(&in, out, overflow, -1, -1);
_exit:
@ -589,6 +591,7 @@ int32_t sclInitOperatorParams(SScalarParam **pParams, SOperatorNode *node, SScal
SCL_ERR_JRET(sclInitParam(node->pLeft, &paramList[0], ctx, rowNum));
paramList[0].tz = node->tz;
paramList[0].charsetCxt = node->charsetCxt;
if (paramNum > 1) {
TSWAP(ctx->type.selfType, ctx->type.peerType);
SCL_ERR_JRET(sclInitParam(node->pRight, &paramList[1], ctx, rowNum));
@ -760,6 +763,7 @@ int32_t sclExecFunction(SFunctionNode *node, SScalarCtx *ctx, SScalarParam *outp
int32_t code = 0;
SCL_ERR_RET(sclInitParamList(&params, node->pParameterList, ctx, &paramNum, &rowNum));
params->tz = node->tz;
params->charsetCxt = node->charsetCxt;
if (fmIsUserDefinedFunc(node->funcId)) {
code = callUdfScalarFunc(node->functionName, params, paramNum, output);
@ -947,6 +951,7 @@ int32_t sclExecCaseWhen(SCaseWhenNode *node, SScalarCtx *ctx, SScalarParam *outp
SCL_ERR_JRET(sclGetNodeRes(node->pCase, ctx, &pCase));
SCL_ERR_JRET(sclGetNodeRes(node->pElse, ctx, &pElse));
pCase->tz = node->tz;
pCase->charsetCxt = node->charsetCxt;
SDataType compType = {0};
compType.type = TSDB_DATA_TYPE_BOOL;
compType.bytes = tDataTypes[compType.type].bytes;

112
source/libs/scalar/src/sclfunc.c
View File

@ -12,8 +12,8 @@ typedef float (*_float_fn_2)(float, float);
typedef double (*_double_fn)(double);
typedef double (*_double_fn_2)(double, double);
typedef int (*_conv_fn)(int);
typedef void (*_trim_space_fn)(char *, char *, int32_t, int32_t);
typedef int32_t (*_trim_fn)(char *, char *, char *, int32_t, int32_t);
typedef void (*_trim_space_fn)(char *, char *, int32_t, int32_t, void*);
typedef int32_t (*_trim_fn)(char *, char *, char *, int32_t, int32_t, void*);
typedef int32_t (*_len_fn)(char *, int32_t, VarDataLenT *);
/** Math functions **/
@ -500,7 +500,7 @@ static int32_t tcharlength(char *input, int32_t type, VarDataLenT *len) {
return TSDB_CODE_SUCCESS;
}
static void tltrimspace(char *input, char *output, int32_t type, int32_t charLen) {
static void tltrimspace(char *input, char *output, int32_t type, int32_t charLen, void* charsetCxt) {
int32_t numOfSpaces = 0;
if (type == TSDB_DATA_TYPE_VARCHAR) {
for (int32_t i = 0; i < charLen; ++i) {
@ -530,7 +530,7 @@ static void tltrimspace(char *input, char *output, int32_t type, int32_t charLen
varDataSetLen(output, resLen);
}
static void tlrtrimspace(char *input, char *output, int32_t type, int32_t charLen) {
static void tlrtrimspace(char *input, char *output, int32_t type, int32_t charLen, void* charsetCxt) {
int32_t numOfLeftSpaces = 0;
int32_t numOfRightSpaces = 0;
if (type == TSDB_DATA_TYPE_VARCHAR) {
@ -615,12 +615,12 @@ static int32_t trimHelper(char *orgStr, char* remStr, int32_t orgLen, int32_t re
}
}
static int32_t convVarcharToNchar(char *input, char **output, int32_t inputLen, int32_t *outputLen) {
static int32_t convVarcharToNchar(char *input, char **output, int32_t inputLen, int32_t *outputLen, void* charsetCxt) {
*output = taosMemoryCalloc(inputLen * TSDB_NCHAR_SIZE, 1);
if (NULL == *output) {
return terrno;
}
bool ret = taosMbsToUcs4(input, inputLen, (TdUcs4 *)*output, inputLen * TSDB_NCHAR_SIZE, outputLen);
bool ret = taosMbsToUcs4(input, inputLen, (TdUcs4 *)*output, inputLen * TSDB_NCHAR_SIZE, outputLen, charsetCxt);
if (!ret) {
taosMemoryFreeClear(*output);
return TSDB_CODE_SCALAR_CONVERT_ERROR;
@ -628,12 +628,12 @@ static int32_t convVarcharToNchar(char *input, char **output, int32_t inputLen,
return TSDB_CODE_SUCCESS;
}
static int32_t convNcharToVarchar(char *input, char **output, int32_t inputLen, int32_t *outputLen) {
static int32_t convNcharToVarchar(char *input, char **output, int32_t inputLen, int32_t *outputLen, void* charsetCxt) {
*output = taosMemoryCalloc(inputLen, 1);
if (NULL == *output) {
return terrno;
}
*outputLen = taosUcs4ToMbs((TdUcs4 *)input, inputLen, *output);
*outputLen = taosUcs4ToMbs((TdUcs4 *)input, inputLen, *output, charsetCxt);
if (*outputLen < 0) {
taosMemoryFree(*output);
return TSDB_CODE_SCALAR_CONVERT_ERROR;
@ -641,14 +641,14 @@ static int32_t convNcharToVarchar(char *input, char **output, int32_t inputLen,
return TSDB_CODE_SUCCESS;
}
static int32_t convBetweenNcharAndVarchar(char *input, char **output, int32_t inputLen, int32_t *outputLen, int32_t wantType) {
static int32_t convBetweenNcharAndVarchar(char *input, char **output, int32_t inputLen, int32_t *outputLen, int32_t wantType, void* charsetCxt) {
if (wantType == TSDB_DATA_TYPE_NCHAR) {
return convVarcharToNchar(input, output, inputLen, outputLen);
return convVarcharToNchar(input, output, inputLen, outputLen, charsetCxt);
} else {
return convNcharToVarchar(input, output, inputLen, outputLen);
return convNcharToVarchar(input, output, inputLen, outputLen, charsetCxt);
}
}
static int32_t tltrim(char *input, char *remInput, char *output, int32_t inputType, int32_t remType) {
static int32_t tltrim(char *input, char *remInput, char *output, int32_t inputType, int32_t remType, void* charsetCxt) {
int32_t orgLen = varDataLen(input);
char *orgStr = varDataVal(input);
int32_t remLen = varDataLen(remInput);
@ -661,7 +661,7 @@ static int32_t tltrim(char *input, char *remInput, char *output, int32_t inputTy
bool needFree = false;
if (inputType != remType) {
SCL_ERR_RET(convBetweenNcharAndVarchar(varDataVal(remInput), &remStr, varDataLen(remInput), &remLen, inputType));
SCL_ERR_RET(convBetweenNcharAndVarchar(varDataVal(remInput), &remStr, varDataLen(remInput), &remLen, inputType, charsetCxt));
needFree = true;
}
@ -683,7 +683,7 @@ static int32_t tltrim(char *input, char *remInput, char *output, int32_t inputTy
return TSDB_CODE_SUCCESS;
}
static void trtrimspace(char *input, char *output, int32_t type, int32_t charLen) {
static void trtrimspace(char *input, char *output, int32_t type, int32_t charLen, void *charsetCxt) {
int32_t numOfSpaces = 0;
if (type == TSDB_DATA_TYPE_VARCHAR) {
for (int32_t i = charLen - 1; i >= 0; --i) {
@ -712,7 +712,7 @@ static void trtrimspace(char *input, char *output, int32_t type, int32_t charLen
varDataSetLen(output, resLen);
}
static int32_t trtrim(char *input, char *remInput, char *output, int32_t inputType, int32_t remType) {
static int32_t trtrim(char *input, char *remInput, char *output, int32_t inputType, int32_t remType, void* charsetCxt) {
int32_t orgLen = varDataLen(input);
char *orgStr = varDataVal(input);
int32_t remLen = varDataLen(remInput);
@ -725,7 +725,7 @@ static int32_t trtrim(char *input, char *remInput, char *output, int32_t inputTy
bool needFree = false;
if (inputType != remType) {
SCL_ERR_RET(convBetweenNcharAndVarchar(varDataVal(remInput), &remStr, varDataLen(remInput), &remLen, inputType));
SCL_ERR_RET(convBetweenNcharAndVarchar(varDataVal(remInput), &remStr, varDataLen(remInput), &remLen, inputType, charsetCxt));
needFree = true;
}
@ -746,7 +746,7 @@ static int32_t trtrim(char *input, char *remInput, char *output, int32_t inputTy
return TSDB_CODE_SUCCESS;
}
static int32_t tlrtrim(char *input, char *remInput, char *output, int32_t inputType, int32_t remType) {
static int32_t tlrtrim(char *input, char *remInput, char *output, int32_t inputType, int32_t remType, void *charsetCxt) {
int32_t orgLen = varDataLen(input);
char *orgStr = varDataVal(input);
int32_t remLen = varDataLen(remInput);
@ -759,7 +759,7 @@ static int32_t tlrtrim(char *input, char *remInput, char *output, int32_t inputT
bool needFree = false;
if (inputType != remType) {
SCL_ERR_RET(convBetweenNcharAndVarchar(varDataVal(remInput), &remStr, varDataLen(remInput), &remLen, inputType));
SCL_ERR_RET(convBetweenNcharAndVarchar(varDataVal(remInput), &remStr, varDataLen(remInput), &remLen, inputType, charsetCxt));
needFree = true;
}
@ -810,14 +810,14 @@ static int32_t doLengthFunction(SScalarParam *pInput, int32_t inputNum, SScalarP
return TSDB_CODE_SUCCESS;
}
static int32_t concatCopyHelper(const char *input, char *output, bool hasNchar, int32_t type, VarDataLenT *dataLen) {
static int32_t concatCopyHelper(const char *input, char *output, bool hasNchar, int32_t type, VarDataLenT *dataLen, void* charsetCxt) {
if (hasNchar && type == TSDB_DATA_TYPE_VARCHAR) {
TdUcs4 *newBuf = taosMemoryCalloc((varDataLen(input) + 1) * TSDB_NCHAR_SIZE, 1);
if (NULL == newBuf) {
return terrno;
}
int32_t len = varDataLen(input);
bool ret = taosMbsToUcs4(varDataVal(input), len, newBuf, (varDataLen(input) + 1) * TSDB_NCHAR_SIZE, &len);
bool ret = taosMbsToUcs4(varDataVal(input), len, newBuf, (varDataLen(input) + 1) * TSDB_NCHAR_SIZE, &len, charsetCxt);
if (!ret) {
taosMemoryFree(newBuf);
return TSDB_CODE_SCALAR_CONVERT_ERROR;
@ -905,7 +905,7 @@ int32_t concatFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu
int32_t rowIdx = (pInput[i].numOfRows == 1) ? 0 : k;
input[i] = colDataGetData(pInputData[i], rowIdx);
SCL_ERR_JRET(concatCopyHelper(input[i], output, hasNchar, GET_PARAM_TYPE(&pInput[i]), &dataLen));
SCL_ERR_JRET(concatCopyHelper(input[i], output, hasNchar, GET_PARAM_TYPE(&pInput[i]), &dataLen, pInput->charsetCxt));
}
varDataSetLen(output, dataLen);
SCL_ERR_JRET(colDataSetVal(pOutputData, k, outputBuf, false));
@ -980,12 +980,12 @@ int32_t concatWsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p
int32_t rowIdx = (pInput[i].numOfRows == 1) ? 0 : k;
SCL_ERR_JRET(concatCopyHelper(colDataGetData(pInputData[i], rowIdx), output, hasNchar, GET_PARAM_TYPE(&pInput[i]), &dataLen));
SCL_ERR_JRET(concatCopyHelper(colDataGetData(pInputData[i], rowIdx), output, hasNchar, GET_PARAM_TYPE(&pInput[i]), &dataLen, pInput->charsetCxt));
if (i < inputNum - 1) {
// insert the separator
char *sep = (pInput[0].numOfRows == 1) ? colDataGetData(pInputData[0], 0) : colDataGetData(pInputData[0], k);
SCL_ERR_JRET(concatCopyHelper(sep, output, hasNchar, GET_PARAM_TYPE(&pInput[0]), &dataLen));
SCL_ERR_JRET(concatCopyHelper(sep, output, hasNchar, GET_PARAM_TYPE(&pInput[0]), &dataLen, pInput->charsetCxt));
}
}
@ -1092,7 +1092,7 @@ static int32_t doTrimFunction(SScalarParam *pInput, int32_t inputNum, SScalarPar
continue;
}
SCL_ERR_JRET(trimFn(colDataGetData(pInputData[1], colIdx2), colDataGetData(pInputData[0], colIdx1),
output, GET_PARAM_TYPE(&pInput[1]), GET_PARAM_TYPE(&pInput[0])));
output, GET_PARAM_TYPE(&pInput[1]), GET_PARAM_TYPE(&pInput[0]), pInput->charsetCxt));
SCL_ERR_JRET(colDataSetVal(pOutputData, i, output, false));
}
} else {
@ -1105,7 +1105,7 @@ static int32_t doTrimFunction(SScalarParam *pInput, int32_t inputNum, SScalarPar
char *input = colDataGetData(pInputData[0], i);
int32_t len = varDataLen(input);
int32_t charLen = (type == TSDB_DATA_TYPE_VARCHAR) ? len : len / TSDB_NCHAR_SIZE;
trimSpaceFn(input, output, type, charLen);
trimSpaceFn(input, output, type, charLen, pInput->charsetCxt);
SCL_ERR_JRET(colDataSetVal(pOutputData, i, output, false));
}
}
@ -1343,7 +1343,7 @@ int32_t charFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
if (convBuf == NULL) {
SCL_ERR_RET(terrno);
}
int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(colDataGetData(pInput[j].columnData, colIdx)), varDataLen(colDataGetData(pInput[j].columnData, colIdx)), convBuf);
int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(colDataGetData(pInput[j].columnData, colIdx)), varDataLen(colDataGetData(pInput[j].columnData, colIdx)), convBuf, pInput->charsetCxt);
if (len < 0) {
taosMemoryFree(convBuf);
code = TSDB_CODE_SCALAR_CONVERT_ERROR;
@ -1386,7 +1386,7 @@ int32_t asciiFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOut
int32_t inLen = varDataLen(colDataGetData(pInputData, i));
SCL_ERR_RET(convBetweenNcharAndVarchar(varDataVal(colDataGetData(pInputData, i)), &in,
varDataLen(colDataGetData(pInputData, i)), &inLen,
TSDB_DATA_TYPE_VARBINARY));
TSDB_DATA_TYPE_VARBINARY, pInput->charsetCxt));
out[i] = (uint8_t)(in)[0];
taosMemoryFree(in);
} else {
@ -1454,7 +1454,9 @@ int32_t positionFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p
int32_t orgLen = varDataLen(colDataGetData(pInputData[1], colIdx2));
bool needFreeSub = false;
if (GET_PARAM_TYPE(&pInput[1]) != GET_PARAM_TYPE(&pInput[0])) {
SCL_ERR_RET(convBetweenNcharAndVarchar(varDataVal(colDataGetData(pInputData[0], colIdx1)), &substr, varDataLen(colDataGetData(pInputData[0], colIdx1)), &subLen, GET_PARAM_TYPE(&pInput[1])));
SCL_ERR_RET(convBetweenNcharAndVarchar(varDataVal(colDataGetData(pInputData[0], colIdx1)), &substr,
varDataLen(colDataGetData(pInputData[0], colIdx1)), &subLen,
GET_PARAM_TYPE(&pInput[1]), pInput->charsetCxt));
needFreeSub = true;
}
@ -1562,13 +1564,13 @@ int32_t replaceFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pO
if (GET_PARAM_TYPE(&pInput[1]) != GET_PARAM_TYPE(&pInput[0])) {
SCL_ERR_JRET(convBetweenNcharAndVarchar(varDataVal(colDataGetData(pInputData[1], colIdx2)), &fromStr,
varDataLen(colDataGetData(pInputData[1], colIdx2)), &fromLen,
GET_PARAM_TYPE(&pInput[0])));
GET_PARAM_TYPE(&pInput[0]), pInput->charsetCxt));
needFreeFrom = true;
}
if (GET_PARAM_TYPE(&pInput[2]) != GET_PARAM_TYPE(&pInput[0])) {
code = convBetweenNcharAndVarchar(varDataVal(colDataGetData(pInputData[2], colIdx3)), &toStr,
varDataLen(colDataGetData(pInputData[2], colIdx3)), &toLen,
GET_PARAM_TYPE(&pInput[0]));
GET_PARAM_TYPE(&pInput[0]), pInput->charsetCxt);
if (TSDB_CODE_SUCCESS != code) {
if (needFreeFrom) {
taosMemoryFree(fromStr);
@ -1674,7 +1676,7 @@ int32_t substrIdxFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *
if (GET_PARAM_TYPE(&pInput[0]) != GET_PARAM_TYPE(&pInput[1])) {
SCL_ERR_JRET(convBetweenNcharAndVarchar(varDataVal(colDataGetData(pInputData[1], colIdx2)), &delimStr,
varDataLen(colDataGetData(pInputData[1], colIdx2)), &delimLen,
GET_PARAM_TYPE(&pInput[0])));
GET_PARAM_TYPE(&pInput[0]), pInput->charsetCxt));
needFreeDelim = true;
}
@ -1853,7 +1855,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
buf[varDataLen(input)] = 0;
*(int8_t *)output = taosStr2Int8(buf, NULL, 10);
} else if (inputType == TSDB_DATA_TYPE_NCHAR) {
int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf);
int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf, pInput->charsetCxt);
if (len < 0) {
code = TSDB_CODE_SCALAR_CONVERT_ERROR;
goto _end;
@ -1872,7 +1874,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
buf[varDataLen(input)] = 0;
*(int16_t *)output = taosStr2Int16(buf, NULL, 10);
} else if (inputType == TSDB_DATA_TYPE_NCHAR) {
int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf);
int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf, pInput->charsetCxt);
if (len < 0) {
code = TSDB_CODE_SCALAR_CONVERT_ERROR;
goto _end;
@ -1890,7 +1892,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
buf[varDataLen(input)] = 0;
*(int32_t *)output = taosStr2Int32(buf, NULL, 10);
} else if (inputType == TSDB_DATA_TYPE_NCHAR) {
int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf);
int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf, pInput->charsetCxt);
if (len < 0) {
code = TSDB_CODE_SCALAR_CONVERT_ERROR;
goto _end;
@ -1909,7 +1911,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
buf[varDataLen(input)] = 0;
*(int64_t *)output = taosStr2Int64(buf, NULL, 10);
} else if (inputType == TSDB_DATA_TYPE_NCHAR) {
int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf);
int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf, pInput->charsetCxt);
if (len < 0) {
code = TSDB_CODE_SCALAR_CONVERT_ERROR;
goto _end;
@ -1927,7 +1929,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
buf[varDataLen(input)] = 0;
*(uint8_t *)output = taosStr2UInt8(buf, NULL, 10);
} else if (inputType == TSDB_DATA_TYPE_NCHAR) {
int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf);
int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf, pInput->charsetCxt);
if (len < 0) {
code = TSDB_CODE_SCALAR_CONVERT_ERROR;
goto _end;
@ -1945,7 +1947,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
buf[varDataLen(input)] = 0;
*(uint16_t *)output = taosStr2UInt16(buf, NULL, 10);
} else if (inputType == TSDB_DATA_TYPE_NCHAR) {
int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf);
int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf, pInput->charsetCxt);
if (len < 0) {
code = TSDB_CODE_SCALAR_CONVERT_ERROR;
goto _end;
@ -1963,7 +1965,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
buf[varDataLen(input)] = 0;
*(uint32_t *)output = taosStr2UInt32(buf, NULL, 10);
} else if (inputType == TSDB_DATA_TYPE_NCHAR) {
int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf);
int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf, pInput->charsetCxt);
if (len < 0) {
code = TSDB_CODE_SCALAR_CONVERT_ERROR;
goto _end;
@ -1981,7 +1983,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
buf[varDataLen(input)] = 0;
*(uint64_t *)output = taosStr2UInt64(buf, NULL, 10);
} else if (inputType == TSDB_DATA_TYPE_NCHAR) {
int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf);
int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf, pInput->charsetCxt);
if (len < 0) {
code = TSDB_CODE_SCALAR_CONVERT_ERROR;
goto _end;
@ -2000,7 +2002,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
buf[varDataLen(input)] = 0;
*(float *)output = taosStr2Float(buf, NULL);
} else if (inputType == TSDB_DATA_TYPE_NCHAR) {
int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf);
int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf, pInput->charsetCxt);
if (len < 0) {
code = TSDB_CODE_SCALAR_CONVERT_ERROR;
goto _end;
@ -2018,7 +2020,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
buf[varDataLen(input)] = 0;
*(double *)output = taosStr2Double(buf, NULL);
} else if (inputType == TSDB_DATA_TYPE_NCHAR) {
int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf);
int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf, pInput->charsetCxt);
if (len < 0) {
code = TSDB_CODE_SCALAR_CONVERT_ERROR;
goto _end;
@ -2036,7 +2038,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
buf[varDataLen(input)] = 0;
*(bool *)output = taosStr2Int8(buf, NULL, 10);
} else if (inputType == TSDB_DATA_TYPE_NCHAR) {
int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf);
int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf, pInput->charsetCxt);
if (len < 0) {
code = TSDB_CODE_SCALAR_CONVERT_ERROR;
goto _end;
@ -2053,7 +2055,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
if (inputType == TSDB_DATA_TYPE_BINARY || inputType == TSDB_DATA_TYPE_NCHAR) {
int64_t timePrec;
GET_TYPED_DATA(timePrec, int64_t, GET_PARAM_TYPE(&pInput[1]), pInput[1].columnData->pData);
int32_t ret = convertStringToTimestamp(inputType, input, timePrec, &timeVal, pInput->tz);
int32_t ret = convertStringToTimestamp(inputType, input, timePrec, &timeVal, pInput->tz, pInput->charsetCxt);
if (ret != TSDB_CODE_SUCCESS) {
*(int64_t *)output = 0;
} else {
@ -2076,7 +2078,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
(void)memcpy(varDataVal(output), varDataVal(input), len);
varDataSetLen(output, len);
} else if (inputType == TSDB_DATA_TYPE_NCHAR) {
int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf);
int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf, pInput->charsetCxt);
if (len < 0) {
code = TSDB_CODE_SCALAR_CONVERT_ERROR;
goto _end;
@ -2111,7 +2113,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
if (inputType == TSDB_DATA_TYPE_BOOL) {
char tmp[8] = {0};
len = tsnprintf(tmp, sizeof(tmp), "%.*s", outputCharLen, *(int8_t *)input ? "true" : "false");
bool ret = taosMbsToUcs4(tmp, len, (TdUcs4 *)varDataVal(output), outputLen - VARSTR_HEADER_SIZE, &len);
bool ret = taosMbsToUcs4(tmp, len, (TdUcs4 *)varDataVal(output), outputLen - VARSTR_HEADER_SIZE, &len, pInput->charsetCxt);
if (!ret) {
code = TSDB_CODE_SCALAR_CONVERT_ERROR;
goto _end;
@ -2121,7 +2123,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
} else if (inputType == TSDB_DATA_TYPE_BINARY) {
len = outputCharLen > varDataLen(input) ? varDataLen(input) : outputCharLen;
bool ret = taosMbsToUcs4(input + VARSTR_HEADER_SIZE, len, (TdUcs4 *)varDataVal(output),
outputLen - VARSTR_HEADER_SIZE, &len);
outputLen - VARSTR_HEADER_SIZE, &len, pInput->charsetCxt);
if (!ret) {
code = TSDB_CODE_SCALAR_CONVERT_ERROR;
goto _end;
@ -2135,7 +2137,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
NUM_TO_STRING(inputType, input, bufSize, buf);
len = (int32_t)strlen(buf);
len = outputCharLen > len ? len : outputCharLen;
bool ret = taosMbsToUcs4(buf, len, (TdUcs4 *)varDataVal(output), outputLen - VARSTR_HEADER_SIZE, &len);
bool ret = taosMbsToUcs4(buf, len, (TdUcs4 *)varDataVal(output), outputLen - VARSTR_HEADER_SIZE, &len, pInput->charsetCxt);
if (!ret) {
code = TSDB_CODE_SCALAR_CONVERT_ERROR;
goto _end;
@ -2283,7 +2285,7 @@ int32_t toUnixtimestampFunction(SScalarParam *pInput, int32_t inputNum, SScalarP
char *input = colDataGetData(pInput[0].columnData, i);
int64_t timeVal = 0;
int32_t ret = convertStringToTimestamp(type, input, timePrec, &timeVal, pInput->tz);
int32_t ret = convertStringToTimestamp(type, input, timePrec, &timeVal, pInput->tz, pInput->charsetCxt);
if (ret != TSDB_CODE_SUCCESS) {
colDataSetNULL(pOutput->columnData, i);
} else {
@ -2323,7 +2325,7 @@ int32_t toJsonFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu
}
(void)memcpy(tmp, varDataVal(input), varDataLen(input));
tmp[varDataLen(input)] = 0;
if (parseJsontoTagData(tmp, pTagVals, &pTag, NULL)) {
if (parseJsontoTagData(tmp, pTagVals, &pTag, NULL, pInput->charsetCxt)) {
code = tTagNew(pTagVals, 1, true, &pTag);
if (TSDB_CODE_SUCCESS != code) {
tTagFree(pTag);
@ -2473,7 +2475,7 @@ int32_t timeTruncateFunction(SScalarParam *pInput, int32_t inputNum, SScalarPara
char *input = colDataGetData(pInput[0].columnData, i);
if (IS_VAR_DATA_TYPE(type)) { /* datetime format strings */
int32_t ret = convertStringToTimestamp(type, input, timePrec, &timeVal, pInput->tz);
int32_t ret = convertStringToTimestamp(type, input, timePrec, &timeVal, pInput->tz, pInput->charsetCxt);
if (ret != TSDB_CODE_SUCCESS) {
colDataSetNULL(pOutput->columnData, i);
continue;
@ -2533,7 +2535,7 @@ int32_t timeDiffFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p
int32_t type = GET_PARAM_TYPE(&pInput[k]);
if (IS_VAR_DATA_TYPE(type)) { /* datetime format strings */
int32_t ret = convertStringToTimestamp(type, input[k], TSDB_TIME_PRECISION_NANO, &timeVal[k], pInput->tz);
int32_t ret = convertStringToTimestamp(type, input[k], TSDB_TIME_PRECISION_NANO, &timeVal[k], pInput->tz, pInput->charsetCxt);
if (ret != TSDB_CODE_SUCCESS) {
hasNull = true;
break;
@ -2660,10 +2662,6 @@ int32_t todayFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOut
return TSDB_CODE_SUCCESS;
}
int32_t timeZoneStrLen() {
return sizeof(VarDataLenT) + strlen(tsTimezoneStr);
}
int32_t timezoneFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
char output[TD_TIMEZONE_LEN + VARSTR_HEADER_SIZE] = {0};
// pInput->tz todo tz
@ -2699,7 +2697,7 @@ int32_t weekdayFunctionImpl(SScalarParam *pInput, int32_t inputNum, SScalarParam
char *input = colDataGetData(pInput[0].columnData, i);
if (IS_VAR_DATA_TYPE(type)) { /* datetime format strings */
int32_t ret = convertStringToTimestamp(type, input, timePrec, &timeVal, pInput->tz);
int32_t ret = convertStringToTimestamp(type, input, timePrec, &timeVal, pInput->tz, pInput->charsetCxt);
if (ret != TSDB_CODE_SUCCESS) {
colDataSetNULL(pOutput->columnData, i);
continue;
@ -2809,7 +2807,7 @@ int32_t weekFunctionImpl(SScalarParam *pInput, int32_t inputNum, SScalarParam *p
char *input = colDataGetData(pInput[0].columnData, i);
if (IS_VAR_DATA_TYPE(type)) { /* datetime format strings */
int32_t ret = convertStringToTimestamp(type, input, prec, &timeVal, pInput->tz);
int32_t ret = convertStringToTimestamp(type, input, prec, &timeVal, pInput->tz, pInput->charsetCxt);
if (ret != TSDB_CODE_SUCCESS) {
colDataSetNULL(pOutput->columnData, i);
continue;

26
source/libs/scalar/src/sclvector.c
View File

@ -103,7 +103,7 @@ int32_t convertNcharToDouble(const void *inData, void *outData) {
if (NULL == tmp) {
SCL_ERR_RET(terrno);
}
int len = taosUcs4ToMbs((TdUcs4 *)varDataVal(inData), varDataLen(inData), tmp);
int len = taosUcs4ToMbs((TdUcs4 *)varDataVal(inData), varDataLen(inData), tmp, NULL);
if (len < 0) {
sclError("castConvert taosUcs4ToMbs error 1");
SCL_ERR_JRET(TSDB_CODE_SCALAR_CONVERT_ERROR);
@ -404,7 +404,7 @@ static FORCE_INLINE int32_t varToNchar(char *buf, SScalarParam *pOut, int32_t ro
SCL_ERR_RET(terrno);
}
int32_t ret =
taosMbsToUcs4(varDataVal(buf), inputLen, (TdUcs4 *)varDataVal(t), outputMaxLen - VARSTR_HEADER_SIZE, &len);
taosMbsToUcs4(varDataVal(buf), inputLen, (TdUcs4 *)varDataVal(t), outputMaxLen - VARSTR_HEADER_SIZE, &len, pOut->charsetCxt);
if (!ret) {
sclError("failed to convert to NCHAR");
SCL_ERR_JRET(TSDB_CODE_SCALAR_CONVERT_ERROR);
@ -426,7 +426,7 @@ static FORCE_INLINE int32_t ncharToVar(char *buf, SScalarParam *pOut, int32_t ro
if (NULL == t) {
SCL_ERR_RET(terrno);
}
int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(buf), varDataLen(buf), varDataVal(t));
int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(buf), varDataLen(buf), varDataVal(t), pOut->charsetCxt);
if (len < 0) {
SCL_ERR_JRET(TSDB_CODE_SCALAR_CONVERT_ERROR);
}
@ -557,7 +557,7 @@ int32_t vectorConvertFromVarData(SSclVectorConvCtx *pCtx, int32_t *overflow) {
SCL_ERR_JRET(TSDB_CODE_APP_ERROR);
}
int len = taosUcs4ToMbs((TdUcs4 *)varDataVal(data), varDataLen(data), tmp);
int len = taosUcs4ToMbs((TdUcs4 *)varDataVal(data), varDataLen(data), tmp, pCtx->pIn->charsetCxt);
if (len < 0) {
sclError("castConvert taosUcs4ToMbs error 1");
SCL_ERR_JRET(TSDB_CODE_SCALAR_CONVERT_ERROR);
@ -592,7 +592,7 @@ int32_t getVectorDoubleValue_JSON(void *src, int32_t index, double *out) {
SCL_RET(TSDB_CODE_SUCCESS);
}
int32_t ncharTobinary(void *buf, void **out) { // todo need to remove , if tobinary is nchar
int32_t ncharTobinary(void *buf, void **out, void* charsetCxt) { // todo need to remove , if tobinary is nchar
int32_t inputLen = varDataTLen(buf);
*out = taosMemoryCalloc(1, inputLen);
@ -601,7 +601,7 @@ int32_t ncharTobinary(void *buf, void **out) { // todo need to remove , if tobi
DEFAULT_UNICODE_ENCODEC, tsCharset, (char *)varDataVal(buf));
SCL_ERR_RET(terrno);
}
int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(buf), varDataLen(buf), varDataVal(*out));
int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(buf), varDataLen(buf), varDataVal(*out), charsetCxt);
if (len < 0) {
sclError("charset:%s to %s. val:%s convert ncharTobinary failed.", DEFAULT_UNICODE_ENCODEC, tsCharset,
(char *)varDataVal(buf));
@ -614,7 +614,7 @@ int32_t ncharTobinary(void *buf, void **out) { // todo need to remove , if tobi
int32_t convertJsonValue(__compar_fn_t *fp, int32_t optr, int8_t typeLeft, int8_t typeRight, char **pLeftData,
char **pRightData, void *pLeftOut, void *pRightOut, bool *isNull, bool *freeLeft,
bool *freeRight, bool *result) {
bool *freeRight, bool *result, void* charsetCxt) {
*result = false;
if (optr == OP_TYPE_JSON_CONTAINS) {
*result = true;
@ -700,13 +700,13 @@ int32_t convertJsonValue(__compar_fn_t *fp, int32_t optr, int8_t typeLeft, int8_
type == TSDB_DATA_TYPE_GEOMETRY) {
if (typeLeft == TSDB_DATA_TYPE_NCHAR) {
char *tmpLeft = NULL;
SCL_ERR_RET(ncharTobinary(*pLeftData, (void *)&tmpLeft));
SCL_ERR_RET(ncharTobinary(*pLeftData, (void *)&tmpLeft, charsetCxt));
*pLeftData = tmpLeft;
*freeLeft = true;
}
if (typeRight == TSDB_DATA_TYPE_NCHAR) {
char *tmpRight = NULL;
SCL_ERR_RET(ncharTobinary(*pRightData, (void *)&tmpRight));
SCL_ERR_RET(ncharTobinary(*pRightData, (void *)&tmpRight, charsetCxt));
*pRightData = tmpRight;
*freeRight = true;
}
@ -1958,7 +1958,7 @@ int32_t doVectorCompareImpl(SScalarParam *pLeft, SScalarParam *pRight, SScalarPa
bool result = false;
SCL_ERR_RET(convertJsonValue(&fp, optr, GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), &pLeftData, &pRightData,
&leftOut, &rightOut, &isJsonnull, &freeLeft, &freeRight, &result));
&leftOut, &rightOut, &isJsonnull, &freeLeft, &freeRight, &result, pLeft->charsetCxt));
if (isJsonnull) {
sclError("doVectorCompareImpl: invalid json null value");
@ -2043,8 +2043,11 @@ int32_t vectorCompareImpl(SScalarParam *pLeft, SScalarParam *pRight, SScalarPara
SScalarParam *param2 = NULL;
int32_t code = TSDB_CODE_SUCCESS;
pRight->tz = pLeft->tz;
pRight->charsetCxt = pLeft->charsetCxt;
pLeftOut.tz = pLeft->tz;
pLeftOut.charsetCxt = pLeft->charsetCxt;
pRightOut.tz = pRight->tz;
pRightOut.charsetCxt = pRight->charsetCxt;
if (noConvertBeforeCompare(GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), optr)) {
param1 = pLeft;
param2 = pRight;
@ -2130,6 +2133,7 @@ int32_t vectorIsNull(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pO
int32_t vectorNotNull(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOut, int32_t _ord) {
if (pRight != NULL) {
pRight->tz = pLeft->tz;
pRight->charsetCxt = pLeft->charsetCxt;
}
for (int32_t i = 0; i < pLeft->numOfRows; ++i) {
int8_t v = IS_HELPER_NULL(pLeft->columnData, i) ? 0 : 1;
@ -2146,6 +2150,8 @@ int32_t vectorNotNull(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *p
int32_t vectorIsTrue(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOut, int32_t _ord) {
if (pRight != NULL) {
pRight->tz = pLeft->tz;
pRight->charsetCxt = pLeft->charsetCxt;
}
SCL_ERR_RET(vectorConvertSingleColImpl(pLeft, pOut, NULL, -1, -1));
for (int32_t i = 0; i < pOut->numOfRows; ++i) {

6
source/libs/scalar/test/scalar/scalarTests.cpp
View File

@ -1450,7 +1450,7 @@ TEST(columnTest, json_column_arith_op) {
SArray *tags = taosArrayInit(1, sizeof(STagVal));
ASSERT_NE(tags, nullptr);
STag *row = NULL;
int32_t code = parseJsontoTagData(rightv, tags, &row, NULL);
int32_t code = parseJsontoTagData(rightv, tags, &row, NULL, NULL);
ASSERT_EQ(code, TSDB_CODE_SUCCESS);
const int32_t len = 8;
@ -1606,7 +1606,7 @@ void *prepareNchar(char *rightData) {
int32_t inputLen = strlen(rightData);
char *t = (char *)taosMemoryCalloc(1, (inputLen + 1) * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE);
taosMbsToUcs4(rightData, inputLen, (TdUcs4 *)varDataVal(t), inputLen * TSDB_NCHAR_SIZE, &len);
taosMbsToUcs4(rightData, inputLen, (TdUcs4 *)varDataVal(t), inputLen * TSDB_NCHAR_SIZE, &len, NULL);
varDataSetLen(t, len);
return t;
}
@ -1623,7 +1623,7 @@ TEST(columnTest, json_column_logic_op) {
SArray *tags = taosArrayInit(1, sizeof(STagVal));
ASSERT_NE(tags, nullptr);
STag *row = NULL;
code = parseJsontoTagData(rightv, tags, &row, NULL);
code = parseJsontoTagData(rightv, tags, &row, NULL, NULL);
ASSERT_EQ(code, TSDB_CODE_SUCCESS);
const int32_t len0 = 6;

8
source/os/src/osEnv.c
View File

@ -25,9 +25,9 @@ SDiskSpace tsLogSpace = {0};
SDiskSpace tsTempSpace = {0};
char tsOsName[16] = {0};
char tsTimezoneStr[TD_TIMEZONE_LEN] = {0};
enum TdTimezone tsTimezone = TdZeroZone;
char tsLocale[TD_LOCALE_LEN] = {0};
char tsCharset[TD_CHARSET_LEN] = {0};
void *tsCharsetCxt = NULL;
bool tsEnableCoreFile = 1;
int64_t tsPageSizeKB = 0;
int64_t tsOpenMax = 0;
@ -48,6 +48,7 @@ int32_t osDefaultInit() {
taosSeedRand(taosSafeRand());
taosGetSystemLocale(tsLocale, tsCharset);
(void)taosGetSystemTimezone(tsTimezoneStr);
taosGetSystemInfo();
@ -117,11 +118,6 @@ bool osTempSpaceSufficient() { return tsTempSpace.size.avail > tsTempSpace.reser
int32_t osSetTimezone(const char *tz) { return taosSetGlobalTimezone(tz); }
void osSetSystemLocale(const char *inLocale, const char *inCharSet) {
if (inLocale) (void)memcpy(tsLocale, inLocale, strlen(inLocale) + 1);
if (inCharSet) (void)memcpy(tsCharset, inCharSet, strlen(inCharSet) + 1);
}
void osSetProcPath(int32_t argc, char **argv) {
if (argv == NULL || argc < 1) {
return; // no command line arguments

63
source/os/src/osLocale.c
View File

@ -78,19 +78,17 @@ char *taosCharsetReplace(char *charsetstr) {
*
* In case that the setLocale failed to be executed, the right charset needs to be set.
*/
int32_t taosSetSystemLocale(const char *inLocale, const char *inCharSet) {
OS_PARAM_CHECK(inLocale);
OS_PARAM_CHECK(inCharSet);
if (!taosValidateEncodec(inCharSet)) {
return terrno;
}
int32_t taosSetSystemLocale(const char *inLocale) {
char *locale = setlocale(LC_CTYPE, inLocale);
if (NULL == locale) {
terrno = TSDB_CODE_INVALID_PARA;
uError("failed to set locale:%s", inLocale);
return terrno;
}
(void)memcpy(tsLocale, inLocale, strlen(inLocale) + 1);
return 0;
}
@ -102,59 +100,6 @@ void taosGetSystemLocale(char *outLocale, char *outCharset) {
tstrncpy(outLocale, locale, TD_LOCALE_LEN);
}
tstrncpy(outCharset, "UTF-8", TD_CHARSET_LEN);
#elif defined(_TD_DARWIN_64)
/*
* originally from src/os/src/detail/osSysinfo.c
* POSIX format locale string:
* (Language Strings)_(Country/Region Strings).(code_page)
*
* example: en_US.UTF-8, zh_CN.GB18030, zh_CN.UTF-8,
*
* if user does not specify the locale in taos.cfg the program use default LC_CTYPE as system locale.
*
* In case of some CentOS systems, their default locale is "en_US.utf8", which is not valid code_page
* for libiconv that is employed to convert string in this system. This program will automatically use
* UTF-8 instead as the charset.
*
* In case of windows client, the locale string is not valid POSIX format, user needs to set the
* correct code_page for libiconv. Usually, the code_page of windows system with simple chinese is
* CP936, CP437 for English charset.
*
*/
char sep = '.';
char *locale = NULL;
locale = setlocale(LC_CTYPE, "");
if (locale == NULL) {
// printf("can't get locale from system, set it to en_US.UTF-8 since error:%d:%s", errno, strerror(errno));
tstrncpy(outLocale, "en_US.UTF-8", TD_LOCALE_LEN);
} else {
tstrncpy(outLocale, locale, TD_LOCALE_LEN);
// printf("locale not configured, set to system default:%s", outLocale);
}
/* if user does not specify the charset, extract it from locale */
char *str = strrchr(outLocale, sep);
if (str != NULL) {
str++;
char *revisedCharset = taosCharsetReplace(str);
if (NULL == revisedCharset) {
tstrncpy(outCharset, "UTF-8", TD_CHARSET_LEN);
} else {
tstrncpy(outCharset, revisedCharset, TD_CHARSET_LEN);
taosMemoryFree(revisedCharset);
}
// printf("charset not configured, set to system default:%s", outCharset);
} else {
tstrncpy(outCharset, "UTF-8", TD_CHARSET_LEN);
// printf("can't get locale and charset from system, set it to UTF-8");
}
#else
/*
* POSIX format locale string:

126
source/os/src/osString.c
View File

@ -229,80 +229,25 @@ int32_t tasoUcs4Copy(TdUcs4 *target_ucs4, TdUcs4 *source_ucs4, int32_t len_ucs4)
return TSDB_CODE_SUCCESS;
}
typedef struct {
iconv_t conv;
int8_t inUse;
} SConv;
// 0: Mbs --> Ucs4
// 1: Ucs4--> Mbs
SConv *gConv[2] = {NULL, NULL};
int32_t convUsed[2] = {0, 0};
int32_t gConvMaxNum[2] = {0, 0};
int32_t taosConvInit(void) {
int8_t M2C = 0;
gConvMaxNum[M2C] = 512;
gConvMaxNum[1 - M2C] = 512;
gConv[M2C] = taosMemoryCalloc(gConvMaxNum[M2C], sizeof(SConv));
if (gConv[M2C] == NULL) {
return terrno;
}
gConv[1 - M2C] = taosMemoryCalloc(gConvMaxNum[1 - M2C], sizeof(SConv));
if (gConv[1 - M2C] == NULL) {
taosMemoryFree(gConv[M2C]);
return terrno;
}
for (int32_t i = 0; i < gConvMaxNum[M2C]; ++i) {
gConv[M2C][i].conv = iconv_open(DEFAULT_UNICODE_ENCODEC, tsCharset);
if ((iconv_t)-1 == gConv[M2C][i].conv) {
terrno = TAOS_SYSTEM_ERROR(errno);
return terrno;
}
}
for (int32_t i = 0; i < gConvMaxNum[1 - M2C]; ++i) {
gConv[1 - M2C][i].conv = iconv_open(tsCharset, DEFAULT_UNICODE_ENCODEC);
if ((iconv_t)-1 == gConv[1 - M2C][i].conv) {
terrno = TAOS_SYSTEM_ERROR(errno);
return terrno;
}
}
return 0;
}
void taosConvDestroy() {
int8_t M2C = 0;
for (int32_t i = 0; i < gConvMaxNum[M2C]; ++i) {
(void)iconv_close(gConv[M2C][i].conv);
}
for (int32_t i = 0; i < gConvMaxNum[1 - M2C]; ++i) {
(void)iconv_close(gConv[1 - M2C][i].conv);
}
taosMemoryFreeClear(gConv[M2C]);
taosMemoryFreeClear(gConv[1 - M2C]);
gConvMaxNum[M2C] = -1;
gConvMaxNum[1 - M2C] = -1;
}
iconv_t taosAcquireConv(int32_t *idx, ConvType type) {
iconv_t taosAcquireConv(int32_t *idx, ConvType type, void* charsetCxt) {
if(idx == NULL) {
terrno = TSDB_CODE_INVALID_PARA;
return (iconv_t)-1;
}
if (gConvMaxNum[type] <= 0) {
if (charsetCxt == NULL){
charsetCxt = tsCharsetCxt;
}
SConvInfo *info = (SConvInfo *)charsetCxt;
if (info->gConvMaxNum[type] <= 0) {
*idx = -1;
if (type == M2C) {
iconv_t c = iconv_open(DEFAULT_UNICODE_ENCODEC, tsCharset);
iconv_t c = iconv_open(DEFAULT_UNICODE_ENCODEC, info->charset);
if ((iconv_t)-1 == c) {
terrno = TAOS_SYSTEM_ERROR(errno);
}
return c;
} else {
iconv_t c = iconv_open(tsCharset, DEFAULT_UNICODE_ENCODEC);
iconv_t c = iconv_open(info->charset, DEFAULT_UNICODE_ENCODEC);
if ((iconv_t)-1 == c) {
terrno = TAOS_SYSTEM_ERROR(errno);
}
@ -311,9 +256,9 @@ iconv_t taosAcquireConv(int32_t *idx, ConvType type) {
}
while (true) {
int32_t used = atomic_add_fetch_32(&convUsed[type], 1);
if (used > gConvMaxNum[type]) {
used = atomic_sub_fetch_32(&convUsed[type], 1);
int32_t used = atomic_add_fetch_32(&info->convUsed[type], 1);
if (used > info->gConvMaxNum[type]) {
(void)atomic_sub_fetch_32(&info->convUsed[type], 1);
(void)sched_yield();
continue;
}
@ -321,38 +266,43 @@ iconv_t taosAcquireConv(int32_t *idx, ConvType type) {
break;
}
int32_t startId = taosGetSelfPthreadId() % gConvMaxNum[type];
int32_t startId = taosGetSelfPthreadId() % info->gConvMaxNum[type];
while (true) {
if (gConv[type][startId].inUse) {
startId = (startId + 1) % gConvMaxNum[type];
if (info->gConv[type][startId].inUse) {
startId = (startId + 1) % info->gConvMaxNum[type];
continue;
}
int8_t old = atomic_val_compare_exchange_8(&gConv[type][startId].inUse, 0, 1);
int8_t old = atomic_val_compare_exchange_8(&info->gConv[type][startId].inUse, 0, 1);
if (0 == old) {
break;
}
}
*idx = startId;
if ((iconv_t)0 == gConv[type][startId].conv) {
if ((iconv_t)0 == info->gConv[type][startId].conv) {
return (iconv_t)-1;
} else {
return gConv[type][startId].conv;
return info->gConv[type][startId].conv;
}
}
void taosReleaseConv(int32_t idx, iconv_t conv, ConvType type) {
void taosReleaseConv(int32_t idx, iconv_t conv, ConvType type, void* charsetCxt) {
if (idx < 0) {
(void)iconv_close(conv);
return;
}
atomic_store_8(&gConv[type][idx].inUse, 0);
(void)atomic_sub_fetch_32(&convUsed[type], 1);
if (charsetCxt == NULL){
charsetCxt = tsCharsetCxt;
}
SConvInfo *info = (SConvInfo *)charsetCxt;
atomic_store_8(&info->gConv[type][idx].inUse, 0);
(void)atomic_sub_fetch_32(&info->convUsed[type], 1);
}
bool taosMbsToUcs4(const char *mbs, size_t mbsLength, TdUcs4 *ucs4, int32_t ucs4_max_len, int32_t *len) {
bool taosMbsToUcs4(const char *mbs, size_t mbsLength, TdUcs4 *ucs4, int32_t ucs4_max_len, int32_t *len, void* charsetCxt) {
if (ucs4_max_len == 0) {
return true;
}
@ -368,20 +318,20 @@ bool taosMbsToUcs4(const char *mbs, size_t mbsLength, TdUcs4 *ucs4, int32_t ucs4
(void)memset(ucs4, 0, ucs4_max_len);
int32_t idx = -1;
iconv_t conv = taosAcquireConv(&idx, M2C);
iconv_t conv = taosAcquireConv(&idx, M2C, charsetCxt);
if ((iconv_t)-1 == conv) {
return false;
}
size_t ucs4_input_len = mbsLength;
size_t outLeft = ucs4_max_len;
if (iconv(conv, (char **)&mbs, &ucs4_input_len, (char **)&ucs4, &outLeft) == -1) {
terrno = TAOS_SYSTEM_ERROR(errno);
taosReleaseConv(idx, conv, M2C);
taosReleaseConv(idx, conv, M2C, charsetCxt);
return false;
}
taosReleaseConv(idx, conv, M2C);
taosReleaseConv(idx, conv, M2C, charsetCxt);
if (len != NULL) {
*len = (int32_t)(ucs4_max_len - outLeft);
if (*len < 0) {
@ -397,7 +347,7 @@ bool taosMbsToUcs4(const char *mbs, size_t mbsLength, TdUcs4 *ucs4, int32_t ucs4
// if success, return the number of bytes written to mbs ( >= 0)
// otherwise return error code ( < 0)
int32_t taosUcs4ToMbs(TdUcs4 *ucs4, int32_t ucs4_max_len, char *mbs) {
int32_t taosUcs4ToMbs(TdUcs4 *ucs4, int32_t ucs4_max_len, char *mbs, void* charsetCxt) {
if (ucs4_max_len == 0) {
return 0;
}
@ -413,22 +363,22 @@ int32_t taosUcs4ToMbs(TdUcs4 *ucs4, int32_t ucs4_max_len, char *mbs) {
int32_t idx = -1;
int32_t code = 0;
iconv_t conv = taosAcquireConv(&idx, C2M);
iconv_t conv = taosAcquireConv(&idx, C2M, charsetCxt);
if ((iconv_t)-1 == conv) {
return terrno;
}
size_t ucs4_input_len = ucs4_max_len;
size_t outLen = ucs4_max_len;
if (iconv(conv, (char **)&ucs4, &ucs4_input_len, &mbs, &outLen) == -1) {
code = TAOS_SYSTEM_ERROR(errno);
taosReleaseConv(idx, conv, C2M);
terrno = code;
taosReleaseConv(idx, conv, C2M, charsetCxt);
terrno = code;
return code;
}
taosReleaseConv(idx, conv, C2M);
taosReleaseConv(idx, conv, C2M, charsetCxt);
return (int32_t)(ucs4_max_len - outLen);
#endif
}

10
source/os/src/osTimezone.c
View File

@ -828,6 +828,16 @@ int32_t taosSetGlobalTimezone(const char *tz) {
}
int32_t taosGetLocalTimezoneOffset() {
time_t tx1 = taosGetTimestampSec();
struct tm tm1;
if (taosLocalTime(&tx1, &tm1, NULL, 0, NULL) == NULL) {
uError("%s failed to get local time: code:%d", __FUNCTION__, errno);
return TSDB_CODE_TIME_ERROR;
}
return (int32_t)(tm1.tm_gmtoff);
}
int32_t taosFormatTimezoneStr(time_t t, const char* tz, timezone_t sp, char *outTimezoneStr){
struct tm tm1;
if (taosLocalTime(&t, &tm1, NULL, 0, sp) == NULL) {

4
source/util/src/tcompare.c
View File

@ -1508,7 +1508,7 @@ int32_t comparewcsRegexMatch(const void *pString, const void *pPattern) {
return 1; // terrno has been set
}
int convertLen = taosUcs4ToMbs((TdUcs4 *)varDataVal(pPattern), len, pattern);
int convertLen = taosUcs4ToMbs((TdUcs4 *)varDataVal(pPattern), len, pattern, NULL);
if (convertLen < 0) {
taosMemoryFree(pattern);
return 1; // terrno has been set
@ -1523,7 +1523,7 @@ int32_t comparewcsRegexMatch(const void *pString, const void *pPattern) {
return 1; // terrno has been set
}
convertLen = taosUcs4ToMbs((TdUcs4 *)varDataVal(pString), len, str);
convertLen = taosUcs4ToMbs((TdUcs4 *)varDataVal(pString), len, str, NULL);
if (convertLen < 0) {
taosMemoryFree(str);
taosMemoryFree(pattern);

52
source/util/src/tconfig.c
View File

@ -24,6 +24,7 @@
#include "tlog.h"
#include "tunit.h"
#include "tutil.h"
#include "tconv.h"
#define CFG_NAME_PRINT_LEN 32
#define CFG_SRC_PRINT_LEN 12
@ -254,10 +255,10 @@ static int32_t cfgSetTimezone(SConfigItem *pItem, const char *value, ECfgSrcType
uError("invalid timezone:%s", value);
TAOS_RETURN(TSDB_CODE_INVALID_TIMEZONE);
}
if (strlen(value) == 0) {
if (value == NULL || strlen(value) == 0) {
uError("cfg:%s, type:%s src:%s, value:%s, skip to set timezone", pItem->name, cfgDtypeStr(pItem->dtype),
cfgStypeStr(stype), value);
TAOS_RETURN(TSDB_CODE_SUCCESS);
TAOS_RETURN(TSDB_CODE_INVALID_CFG);
}
TAOS_CHECK_RETURN(osSetTimezone(value));
@ -266,6 +267,49 @@ static int32_t cfgSetTimezone(SConfigItem *pItem, const char *value, ECfgSrcType
TAOS_RETURN(TSDB_CODE_SUCCESS);
}
static int32_t cfgSetCharset(SConfigItem *pItem, const char *value, ECfgSrcType stype) {
if (stype == CFG_STYPE_ALTER_SERVER_CMD || stype == CFG_STYPE_ALTER_CLIENT_CMD){
uError("failed to config charset, not support");
TAOS_RETURN(TSDB_CODE_INVALID_CFG);
}
if (value == NULL || strlen(value) == 0) {
uError("cfg:%s, type:%s src:%s, value:%s, skip to set charset", pItem->name, cfgDtypeStr(pItem->dtype),
cfgStypeStr(stype), value);
TAOS_RETURN(TSDB_CODE_INVALID_CFG);
}
if (!taosValidateEncodec(value)) {
uError("invalid charset:%s", value);
TAOS_RETURN(terrno);
}
if ((tsCharsetCxt = taosConvInit(value)) == NULL) {
TAOS_RETURN(terrno);
}
(void)memcpy(tsCharset, value, strlen(value) + 1);
TAOS_CHECK_RETURN(doSetConf(pItem, value, stype));
TAOS_RETURN(TSDB_CODE_SUCCESS);
}
static int32_t cfgSetLocale(SConfigItem *pItem, const char *value, ECfgSrcType stype) {
if (stype == CFG_STYPE_ALTER_SERVER_CMD || (pItem->dynScope & CFG_DYN_CLIENT) == 0){
uError("failed to config locale, not support");
TAOS_RETURN(TSDB_CODE_INVALID_CFG);
}
if (value == NULL || strlen(value) == 0 || taosSetSystemLocale(value) != 0) {
uError("cfg:%s, type:%s src:%s, value:%s, skip to set locale", pItem->name, cfgDtypeStr(pItem->dtype),
cfgStypeStr(stype), value);
TAOS_RETURN(TSDB_CODE_INVALID_CFG);
}
TAOS_CHECK_RETURN(doSetConf(pItem, value, stype));
TAOS_RETURN(TSDB_CODE_SUCCESS);
}
static int32_t cfgSetTfsItem(SConfig *pCfg, const char *name, const char *value, const char *level, const char *primary,
const char *disable, ECfgSrcType stype) {
(void)taosThreadMutexLock(&pCfg->lock);
@ -378,11 +422,11 @@ int32_t cfgSetItem(SConfig *pCfg, const char *name, const char *value, ECfgSrcTy
break;
}
case CFG_DTYPE_CHARSET: {
code = doSetConf(pItem, value, stype);
code = cfgSetCharset(pItem, value, stype);
break;
}
case CFG_DTYPE_LOCALE: {
code = doSetConf(pItem, value, stype);
code = cfgSetLocale(pItem, value, stype);
break;
}
case CFG_DTYPE_NONE:

123
source/util/src/tconv.c Normal file
View File

@ -0,0 +1,123 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define ALLOW_FORBID_FUNC
#include "tconv.h"
#include "thash.h"
#include "osString.h"
#define CONV_MAX_NUM 32
SHashObj *gConvInfo = NULL;
// M2C: Mbs --> Ucs4
// C2M: Ucs4--> Mbs
static void taosConvDestroyInner(void *arg) {
SConvInfo *info = (SConvInfo *)arg;
if (info == NULL) {
return;
}
for (int32_t i = 0; i < info->gConvMaxNum[M2C]; ++i) {
(void)iconv_close(info->gConv[M2C][i].conv);
}
for (int32_t i = 0; i < info->gConvMaxNum[C2M]; ++i) {
(void)iconv_close(info->gConv[C2M][i].conv);
}
taosMemoryFreeClear(info->gConv[M2C]);
taosMemoryFreeClear(info->gConv[C2M]);
info->gConvMaxNum[M2C] = -1;
info->gConvMaxNum[C2M] = -1;
}
void* taosConvInit(const char* charset) {
if (charset == NULL){
terrno = TSDB_CODE_INVALID_PARA;
return NULL;
}
void* conv = NULL;
static int32_t lock_c = 0;
for (int i = 1; atomic_val_compare_exchange_32(&lock_c, 0, 1) != 0; ++i) {
if (i % 1000 == 0) {
uInfo("haven't acquire lock after spin %d times.", i);
(void)sched_yield();
}
}
if (gConvInfo == NULL){
gConvInfo = taosHashInit(0, MurmurHash3_32, false, HASH_ENTRY_LOCK);
if (gConvInfo == NULL) {
atomic_store_32(&lock_c, 0);
goto END;
}
taosHashSetFreeFp(gConvInfo, taosConvDestroyInner);
}
conv = taosHashGet(gConvInfo, charset, strlen(charset));
if (conv != NULL){
goto END;
}
SConvInfo info = {0};
info.gConvMaxNum[M2C] = CONV_MAX_NUM;
info.gConvMaxNum[C2M] = CONV_MAX_NUM;
tstrncpy(info.charset, charset, sizeof(info.charset));
info.gConv[M2C] = taosMemoryCalloc(info.gConvMaxNum[M2C], sizeof(SConv));
if (info.gConv[M2C] == NULL) {
goto FAILED;
}
info.gConv[C2M] = taosMemoryCalloc(info.gConvMaxNum[C2M], sizeof(SConv));
if (info.gConv[C2M] == NULL) {
goto FAILED;
}
for (int32_t i = 0; i < info.gConvMaxNum[M2C]; ++i) {
info.gConv[M2C][i].conv = iconv_open(DEFAULT_UNICODE_ENCODEC, charset);
if ((iconv_t)-1 == info.gConv[M2C][i].conv) {
terrno = TAOS_SYSTEM_ERROR(errno);
goto FAILED;
}
}
for (int32_t i = 0; i < info.gConvMaxNum[C2M]; ++i) {
info.gConv[C2M][i].conv = iconv_open(charset, DEFAULT_UNICODE_ENCODEC);
if ((iconv_t)-1 == info.gConv[C2M][i].conv) {
terrno = TAOS_SYSTEM_ERROR(errno);
goto FAILED;
}
}
int32_t code = taosHashPut(gConvInfo, charset, strlen(charset), &info, sizeof(info));
if (code != 0){
goto FAILED;
}
conv = taosHashGet(gConvInfo, charset, strlen(charset));
goto END;
FAILED:
taosConvDestroyInner(&info);
END:
atomic_store_32(&lock_c, 0);
return conv;
}
void taosConvDestroy() {
taosHashCleanup(gConvInfo);
}