xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

feat:[TD-32642] add charset for connection support

This commit is contained in:
wangmm0220 2024-11-27 14:25:34 +08:00
parent 2ca9f9c8ce
commit 0e2a16d70f
61 changed files with 598 additions and 487 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
include/common/tdataformat.h
View File

@ -154,7 +154,7 @@ int32_t tEncodeTag(SEncoder *pEncoder, const STag *pTag);
int32_t tDecodeTag(SDecoder *pDecoder, STag **ppTag); int32_t tDecodeTag(SDecoder *pDecoder, STag **ppTag);
int32_t tTagToValArray(const STag *pTag, SArray **ppArray); int32_t tTagToValArray(const STag *pTag, SArray **ppArray);
void debugPrintSTag(STag *pTag, const char *tag, int32_t ln); // TODO: remove void debugPrintSTag(STag *pTag, const char *tag, int32_t ln); // TODO: remove
int32_t parseJsontoTagData(const char *json, SArray *pTagVals, STag **ppTag, void *pMsgBuf); int32_t parseJsontoTagData(const char *json, SArray *pTagVals, STag **ppTag, void *pMsgBuf, void *charsetCxt);
// SColData ================================ // SColData ================================
typedef struct { typedef struct {

2
include/common/tmsg.h
View File

@ -3842,7 +3842,7 @@ typedef struct {
int8_t igExists; int8_t igExists;
int8_t intervalUnit; int8_t intervalUnit;
int8_t slidingUnit; int8_t slidingUnit;
int8_t timezone; int8_t timezone; // int8_t is not enough, timezone is unit of second
int32_t dstVgId; // for stream int32_t dstVgId; // for stream
int64_t interval; int64_t interval;
int64_t offset; int64_t offset;

2
include/common/ttime.h
View File

@ -74,7 +74,7 @@ char getPrecisionUnit(int32_t precision);
int64_t convertTimePrecision(int64_t ts, int32_t fromPrecision, int32_t toPrecision); int64_t convertTimePrecision(int64_t ts, int32_t fromPrecision, int32_t toPrecision);
int32_t convertTimeFromPrecisionToUnit(int64_t time, int32_t fromPrecision, char toUnit, int64_t* pRes); int32_t convertTimeFromPrecisionToUnit(int64_t time, int32_t fromPrecision, char toUnit, int64_t* pRes);
int32_t convertStringToTimestamp(int16_t type, char* inputData, int64_t timePrec, int64_t* timeVal, timezone_t tz); int32_t convertStringToTimestamp(int16_t type, char* inputData, int64_t timePrec, int64_t* timeVal, timezone_t tz, void* charsetCxt);
int32_t getDuration(int64_t val, char unit, int64_t* result, int32_t timePrecision); int32_t getDuration(int64_t val, char unit, int64_t* result, int32_t timePrecision);
int32_t taosFormatUtcTime(char* buf, int32_t bufLen, int64_t ts, int32_t precision); int32_t taosFormatUtcTime(char* buf, int32_t bufLen, int64_t ts, int32_t precision);

1
include/libs/function/function.h
View File

@ -293,6 +293,7 @@ struct SScalarParam {
int32_t numOfRows; int32_t numOfRows;
int32_t numOfQualified; // number of qualified elements in the final results int32_t numOfQualified; // number of qualified elements in the final results
timezone_t tz; timezone_t tz;
void *charsetCxt;
}; };
#define cleanupResultRowEntry(p) p->initialized = false #define cleanupResultRowEntry(p) p->initialized = false

4
include/libs/nodes/querynodes.h
View File

@ -132,6 +132,7 @@ typedef struct SValueNode {
int64_t typeData; int64_t typeData;
int8_t unit; int8_t unit;
timezone_t tz; timezone_t tz;
void *charsetCxt;
} SValueNode; } SValueNode;
typedef struct SLeftValueNode { typedef struct SLeftValueNode {
@ -161,6 +162,7 @@ typedef struct SOperatorNode {
SNode* pLeft; SNode* pLeft;
SNode* pRight; SNode* pRight;
timezone_t tz; timezone_t tz;
void* charsetCxt;
} SOperatorNode; } SOperatorNode;
typedef struct SLogicConditionNode { typedef struct SLogicConditionNode {
@ -195,6 +197,7 @@ typedef struct SFunctionNode {
bool dual; // whether select stmt without from stmt, true for without. bool dual; // whether select stmt without from stmt, true for without.
// char timezone[TD_TIMEZONE_LEN]; // char timezone[TD_TIMEZONE_LEN];
timezone_t tz; timezone_t tz;
void *charsetCxt;
} SFunctionNode; } SFunctionNode;
typedef struct STableNode { typedef struct STableNode {
@ -404,6 +407,7 @@ typedef struct SCaseWhenNode {
SNode* pElse; SNode* pElse;
SNodeList* pWhenThenList; SNodeList* pWhenThenList;
timezone_t tz; timezone_t tz;
void* charsetCxt;
} SCaseWhenNode; } SCaseWhenNode;
typedef struct SWindowOffsetNode { typedef struct SWindowOffsetNode {

25
include/libs/parser/parser.h
View File

@ -102,6 +102,7 @@ typedef struct SParseContext {
SArray* pSubMetaList; SArray* pSubMetaList;
setQueryFn setQueryFp; setQueryFn setQueryFp;
timezone_t timezone; timezone_t timezone;
void *charsetCxt;
} SParseContext; } SParseContext;
int32_t qParseSql(SParseContext* pCxt, SQuery** pQuery); int32_t qParseSql(SParseContext* pCxt, SQuery** pQuery);
@ -140,27 +141,27 @@ void qDestroyStmtDataBlock(STableDataCxt* pBlock);
STableMeta* qGetTableMetaInDataBlock(STableDataCxt* pDataBlock); STableMeta* qGetTableMetaInDataBlock(STableDataCxt* pDataBlock);
int32_t qCloneCurrentTbData(STableDataCxt* pDataBlock, SSubmitTbData** pData); int32_t qCloneCurrentTbData(STableDataCxt* pDataBlock, SSubmitTbData** pData);
int32_t qStmtBindParams(SQuery* pQuery, TAOS_MULTI_BIND* pParams, int32_t colIdx); int32_t qStmtBindParams(SQuery* pQuery, TAOS_MULTI_BIND* pParams, int32_t colIdx, void *charsetCxt);
int32_t qStmtParseQuerySql(SParseContext* pCxt, SQuery* pQuery); int32_t qStmtParseQuerySql(SParseContext* pCxt, SQuery* pQuery);
int32_t qBindStmtStbColsValue(void* pBlock, SArray* pCols, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen, int32_t qBindStmtStbColsValue(void* pBlock, SArray* pCols, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen,
STSchema** pTSchema, SBindInfo* pBindInfos); STSchema** pTSchema, SBindInfo* pBindInfos, void* charsetCxt);
int32_t qBindStmtColsValue(void* pBlock, SArray* pCols, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen); int32_t qBindStmtColsValue(void* pBlock, SArray* pCols, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen, void* charsetCxt);
int32_t qBindStmtSingleColValue(void* pBlock, SArray* pCols, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen, int32_t qBindStmtSingleColValue(void* pBlock, SArray* pCols, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen,
int32_t colIdx, int32_t rowNum); int32_t colIdx, int32_t rowNum, void* charsetCxt);
int32_t qBuildStmtColFields(void* pDataBlock, int32_t* fieldNum, TAOS_FIELD_E** fields); int32_t qBuildStmtColFields(void* pDataBlock, int32_t* fieldNum, TAOS_FIELD_E** fields);
int32_t qBuildStmtStbColFields(void* pBlock, int32_t* fieldNum, TAOS_FIELD_STB** fields); int32_t qBuildStmtStbColFields(void* pBlock, int32_t* fieldNum, TAOS_FIELD_STB** fields);
int32_t qBuildStmtTagFields(void* pBlock, void* boundTags, int32_t* fieldNum, TAOS_FIELD_E** fields); int32_t qBuildStmtTagFields(void* pBlock, void* boundTags, int32_t* fieldNum, TAOS_FIELD_E** fields);
int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, const char* sTableName, char* tName, int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, const char* sTableName, char* tName,
TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen); TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen, void* charsetCxt);
int32_t qStmtBindParams2(SQuery* pQuery, TAOS_STMT2_BIND* pParams, int32_t colIdx); int32_t qStmtBindParams2(SQuery* pQuery, TAOS_STMT2_BIND* pParams, int32_t colIdx, void* charsetCxt);
int32_t qBindStmtStbColsValue2(void* pBlock, SArray* pCols, TAOS_STMT2_BIND* bind, char* msgBuf, int32_t msgBufLen, int32_t qBindStmtStbColsValue2(void* pBlock, SArray* pCols, TAOS_STMT2_BIND* bind, char* msgBuf, int32_t msgBufLen,
STSchema** pTSchema, SBindInfo2* pBindInfos); STSchema** pTSchema, SBindInfo2* pBindInfos, void *charsetCxt);
int32_t qBindStmtColsValue2(void* pBlock, SArray* pCols, TAOS_STMT2_BIND* bind, char* msgBuf, int32_t msgBufLen); int32_t qBindStmtColsValue2(void* pBlock, SArray* pCols, TAOS_STMT2_BIND* bind, char* msgBuf, int32_t msgBufLen, void *charsetCxt);
int32_t qBindStmtSingleColValue2(void* pBlock, SArray* pCols, TAOS_STMT2_BIND* bind, char* msgBuf, int32_t msgBufLen, int32_t qBindStmtSingleColValue2(void* pBlock, SArray* pCols, TAOS_STMT2_BIND* bind, char* msgBuf, int32_t msgBufLen,
int32_t colIdx, int32_t rowNum); int32_t colIdx, int32_t rowNum, void *charsetCxt);
int32_t qBindStmtTagsValue2(void* pBlock, void* boundTags, int64_t suid, const char* sTableName, char* tName, int32_t qBindStmtTagsValue2(void* pBlock, void* boundTags, int64_t suid, const char* sTableName, char* tName,
TAOS_STMT2_BIND* bind, char* msgBuf, int32_t msgBufLen); TAOS_STMT2_BIND* bind, char* msgBuf, int32_t msgBufLen, void *charsetCxt);
void destroyBoundColumnInfo(void* pBoundInfo); void destroyBoundColumnInfo(void* pBoundInfo);
int32_t qCreateSName(SName* pName, const char* pTableName, int32_t acctId, char* dbName, char* msgBuf, int32_t qCreateSName(SName* pName, const char* pTableName, int32_t acctId, char* dbName, char* msgBuf,
@ -170,13 +171,13 @@ void qDestroyBoundColInfo(void* pInfo);
int32_t smlInitHandle(SQuery** query); int32_t smlInitHandle(SQuery** query);
int32_t smlBuildRow(STableDataCxt* pTableCxt); int32_t smlBuildRow(STableDataCxt* pTableCxt);
int32_t smlBuildCol(STableDataCxt* pTableCxt, SSchema* schema, void* kv, int32_t index); int32_t smlBuildCol(STableDataCxt* pTableCxt, SSchema* schema, void* kv, int32_t index, void* charsetCxt);
int32_t smlInitTableDataCtx(SQuery* query, STableMeta* pTableMeta, STableDataCxt** cxt); int32_t smlInitTableDataCtx(SQuery* query, STableMeta* pTableMeta, STableDataCxt** cxt);
void clearColValArraySml(SArray* pCols); void clearColValArraySml(SArray* pCols);
int32_t smlBindData(SQuery* handle, bool dataFormat, SArray* tags, SArray* colsSchema, SArray* cols, int32_t smlBindData(SQuery* handle, bool dataFormat, SArray* tags, SArray* colsSchema, SArray* cols,
STableMeta* pTableMeta, char* tableName, const char* sTableName, int32_t sTableNameLen, int32_t ttl, STableMeta* pTableMeta, char* tableName, const char* sTableName, int32_t sTableNameLen, int32_t ttl,
char* msgBuf, int32_t msgBufLen); char* msgBuf, int32_t msgBufLen, void* charsetCxt);
int32_t smlBuildOutput(SQuery* handle, SHashObj* pVgHash); int32_t smlBuildOutput(SQuery* handle, SHashObj* pVgHash);
int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreateTbReq* pCreateTb, void* fields, int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreateTbReq* pCreateTb, void* fields,
int numFields, bool needChangeLength, char* errstr, int32_t errstrLen, bool raw); int numFields, bool needChangeLength, char* errstr, int32_t errstrLen, bool raw);

2
include/libs/qcom/query.h
View File

@ -337,7 +337,7 @@ SSchema createSchema(int8_t type, int32_t bytes, col_id_t colId, const char* nam
void destroyQueryExecRes(SExecResult* pRes); void destroyQueryExecRes(SExecResult* pRes);
int32_t dataConverToStr(char* str, int64_t capacity, int type, void* buf, int32_t bufSize, int32_t* len); int32_t dataConverToStr(char* str, int64_t capacity, int type, void* buf, int32_t bufSize, int32_t* len);
void parseTagDatatoJson(void* p, char** jsonStr); void parseTagDatatoJson(void* p, char** jsonStr, void *charsetCxt);
int32_t cloneTableMeta(STableMeta* pSrc, STableMeta** pDst); int32_t cloneTableMeta(STableMeta* pSrc, STableMeta** pDst);
void getColumnTypeFromMeta(STableMeta* pMeta, char* pName, ETableColumnType* pType); void getColumnTypeFromMeta(STableMeta* pMeta, char* pName, ETableColumnType* pType);
int32_t cloneDbVgInfo(SDBVgInfo* pSrc, SDBVgInfo** pDst); int32_t cloneDbVgInfo(SDBVgInfo* pSrc, SDBVgInfo** pDst);

1
include/libs/scalar/scalar.h
View File

@ -105,7 +105,6 @@ int32_t timeTruncateFunction(SScalarParam *pInput, int32_t inputNum, SScalarPara
int32_t timeDiffFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t timeDiffFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t nowFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t nowFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t todayFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t todayFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t timeZoneStrLen();
int32_t timezoneFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t timezoneFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t weekdayFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t weekdayFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t dayofweekFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t dayofweekFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);

6
include/os/osEnv.h
View File

@ -25,10 +25,9 @@ extern "C" {
extern char tsOsName[]; extern char tsOsName[];
extern char tsTimezoneStr[]; extern char tsTimezoneStr[];
extern enum TdTimezone tsTimezone;
extern char tsCharset[]; extern char tsCharset[];
extern void *tsCharsetCxt;
extern char tsLocale[]; extern char tsLocale[];
extern int8_t tsDaylight;
extern bool tsEnableCoreFile; extern bool tsEnableCoreFile;
extern int64_t tsPageSizeKB; extern int64_t tsPageSizeKB;
extern int64_t tsOpenMax; extern int64_t tsOpenMax;
@ -67,8 +66,7 @@ bool osDataSpaceSufficient();
bool osTempSpaceSufficient(); bool osTempSpaceSufficient();
int32_t osSetTimezone(const char *timezone); int32_t osSetTimezone(const char *timezone);
void osSetSystemLocale(const char *inLocale, const char *inCharSet); void osSetProcPath(int32_t argc, char **argv);
void osSetProcPath(int32_t argc, char **argv);
#ifdef __cplusplus #ifdef __cplusplus
} }

2
include/os/osLocale.h
View File

@ -30,7 +30,7 @@ extern "C" {
char *taosCharsetReplace(char *charsetstr); char *taosCharsetReplace(char *charsetstr);
void taosGetSystemLocale(char *outLocale, char *outCharset); void taosGetSystemLocale(char *outLocale, char *outCharset);
int32_t taosSetSystemLocale(const char *inLocale, const char *inCharSet); int32_t taosSetSystemLocale(const char *inLocale);
#ifdef __cplusplus #ifdef __cplusplus
} }

24
include/os/osString.h
View File

@ -27,7 +27,19 @@ typedef int32_t TdUcs4;
#else #else
typedef void *iconv_t; typedef void *iconv_t;
#endif #endif
typedef enum { M2C = 0, C2M } ConvType; typedef enum { M2C = 0, C2M, CM_NUM } ConvType;
typedef struct {
iconv_t conv;
int8_t inUse;
} SConv;
typedef struct {
SConv *gConv[CM_NUM];
int32_t convUsed[CM_NUM];
int32_t gConvMaxNum[CM_NUM];
char charset[TD_CHARSET_LEN];
} SConvInfo;
// If the error is in a third-party library, place this header file under the third-party library header file. // If the error is in a third-party library, place this header file under the third-party library header file.
// When you want to use this feature, you should find or add the same function in the following section. // When you want to use this feature, you should find or add the same function in the following section.
@ -78,13 +90,11 @@ int32_t taosStr2int16(const char *str, int16_t *val);
int32_t taosStr2int32(const char *str, int32_t *val); int32_t taosStr2int32(const char *str, int32_t *val);
int32_t taosStr2int8(const char *str, int8_t *val); int32_t taosStr2int8(const char *str, int8_t *val);
int32_t taosConvInit(void); iconv_t taosAcquireConv(int32_t *idx, ConvType type, void* charsetCxt);
void taosConvDestroy(); void taosReleaseConv(int32_t idx, iconv_t conv, ConvType type, void* charsetCxt);
iconv_t taosAcquireConv(int32_t *idx, ConvType type); int32_t taosUcs4ToMbs(TdUcs4 *ucs4, int32_t ucs4_max_len, char *mbs, void* charsetCxt);
void taosReleaseConv(int32_t idx, iconv_t conv, ConvType type);
int32_t taosUcs4ToMbs(TdUcs4 *ucs4, int32_t ucs4_max_len, char *mbs);
int32_t taosUcs4ToMbsEx(TdUcs4 *ucs4, int32_t ucs4_max_len, char *mbs, iconv_t conv); int32_t taosUcs4ToMbsEx(TdUcs4 *ucs4, int32_t ucs4_max_len, char *mbs, iconv_t conv);
bool taosMbsToUcs4(const char *mbs, size_t mbs_len, TdUcs4 *ucs4, int32_t ucs4_max_len, int32_t *len); bool taosMbsToUcs4(const char *mbs, size_t mbs_len, TdUcs4 *ucs4, int32_t ucs4_max_len, int32_t *len, void* charsetCxt);
int32_t tasoUcs4Compare(TdUcs4 *f1_ucs4, TdUcs4 *f2_ucs4, int32_t bytes); int32_t tasoUcs4Compare(TdUcs4 *f1_ucs4, TdUcs4 *f2_ucs4, int32_t bytes);
int32_t tasoUcs4Copy(TdUcs4 *target_ucs4, TdUcs4 *source_ucs4, int32_t len_ucs4); int32_t tasoUcs4Copy(TdUcs4 *target_ucs4, TdUcs4 *source_ucs4, int32_t len_ucs4);
bool taosValidateEncodec(const char *encodec); bool taosValidateEncodec(const char *encodec);

29
include/os/osTimezone.h
View File

@ -20,33 +20,7 @@
extern "C" { extern "C" {
#endif #endif
enum TdTimezone { #define TdEastZone8 8*60*60
TdWestZone12 = -12,
TdWestZone11,
TdWestZone10,
TdWestZone9,
TdWestZone8,
TdWestZone7,
TdWestZone6,
TdWestZone5,
TdWestZone4,
TdWestZone3,
TdWestZone2,
TdWestZone1,
TdZeroZone,
TdEastZone1,
TdEastZone2,
TdEastZone3,
TdEastZone4,
TdEastZone5,
TdEastZone6,
TdEastZone7,
TdEastZone8,
TdEastZone9,
TdEastZone10,
TdEastZone11,
TdEastZone12
};
typedef struct state *timezone_t; typedef struct state *timezone_t;
struct tm *localtime_rz(timezone_t , time_t const *, struct tm *); struct tm *localtime_rz(timezone_t , time_t const *, struct tm *);
@ -55,6 +29,7 @@ timezone_t tzalloc(char const *);
void tzfree(timezone_t); void tzfree(timezone_t);
void getTimezoneStr(char *tz); void getTimezoneStr(char *tz);
int32_t taosGetLocalTimezoneOffset();
int32_t taosGetSystemTimezone(char *outTimezone); int32_t taosGetSystemTimezone(char *outTimezone);
int32_t taosSetGlobalTimezone(const char *tz); int32_t taosSetGlobalTimezone(const char *tz);
int32_t taosFormatTimezoneStr(time_t t, const char* tzStr, timezone_t sp, char *outTimezoneStr); int32_t taosFormatTimezoneStr(time_t t, const char* tzStr, timezone_t sp, char *outTimezoneStr);

41
include/util/tconv.h Normal file
View File

@ -0,0 +1,41 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_TCONV_H
#define TDENGINE_TCONV_H
#ifdef __cplusplus
extern "C" {
#endif
//#include "osString.h"
//
//bool taosValidateEncodec(const char *encodec);
//int32_t taosUcs4len(TdUcs4 *ucs4);
void* taosConvInit(const char* charset);
void taosConvDestroy();
//iconv_t taosAcquireConv(int32_t *idx, ConvType type, void* charsetCxt);
//void taosReleaseConv(int32_t idx, iconv_t conv, ConvType type, void* charsetCxt);
//int32_t taosUcs4ToMbs(TdUcs4 *ucs4, int32_t ucs4_max_len, char *mbs, void* charsetCxt);
//int32_t taosUcs4ToMbsEx(TdUcs4 *ucs4, int32_t ucs4_max_len, char *mbs, iconv_t conv);
//bool taosMbsToUcs4(const char *mbs, size_t mbs_len, TdUcs4 *ucs4, int32_t ucs4_max_len, int32_t *len, void* charsetCxt);
//int32_t tasoUcs4Compare(TdUcs4 *f1_ucs4, TdUcs4 *f2_ucs4, int32_t bytes);
//int32_t tasoUcs4Copy(TdUcs4 *target_ucs4, TdUcs4 *source_ucs4, int32_t len_ucs4);
#ifdef __cplusplus
}
#endif
#endif // TDENGINE_TCONV_H

3
source/client/inc/clientInt.h
View File

@ -155,7 +155,7 @@ typedef struct {
typedef struct { typedef struct {
timezone_t timezone; timezone_t timezone;
char charset[TD_CHARSET_LEN]; void *charsetCxt;
char app[TSDB_APP_NAME_LEN]; char app[TSDB_APP_NAME_LEN];
uint32_t ip; uint32_t ip;
}SOptionInfo; }SOptionInfo;
@ -219,6 +219,7 @@ typedef struct SReqResultInfo {
int32_t precision; int32_t precision;
int32_t payloadLen; int32_t payloadLen;
char* convertJson; char* convertJson;
void* charsetCxt;
} SReqResultInfo; } SReqResultInfo;
typedef struct SRequestSendRecvBody { typedef struct SRequestSendRecvBody {

9
source/client/src/clientEnv.c
View File

@ -36,6 +36,7 @@
#include "tsched.h" #include "tsched.h"
#include "ttime.h" #include "ttime.h"
#include "tversion.h" #include "tversion.h"
#include "tconv.h"
#if defined(CUS_NAME) || defined(CUS_PROMPT) || defined(CUS_EMAIL) #if defined(CUS_NAME) || defined(CUS_PROMPT) || defined(CUS_EMAIL)
#include "cus_name.h" #include "cus_name.h"
@ -556,6 +557,7 @@ int32_t createRequest(uint64_t connId, int32_t type, int64_t reqid, SRequestObj
(*pRequest)->metric.start = taosGetTimestampUs(); (*pRequest)->metric.start = taosGetTimestampUs();
(*pRequest)->body.resInfo.convertUcs4 = true; // convert ucs4 by default (*pRequest)->body.resInfo.convertUcs4 = true; // convert ucs4 by default
(*pRequest)->body.resInfo.charsetCxt = pTscObj->optionInfo.charsetCxt;
(*pRequest)->type = type; (*pRequest)->type = type;
(*pRequest)->allocatorRefId = -1; (*pRequest)->allocatorRefId = -1;
@ -969,7 +971,12 @@ void taos_init_imp(void) {
ENV_ERR_RET(taosInitCfg(configDir, NULL, NULL, NULL, NULL, 1), "failed to init cfg"); ENV_ERR_RET(taosInitCfg(configDir, NULL, NULL, NULL, NULL, 1), "failed to init cfg");
initQueryModuleMsgHandle(); initQueryModuleMsgHandle();
ENV_ERR_RET(taosConvInit(), "failed to init conv"); if ((tsCharsetCxt = taosConvInit(tsCharset)) == NULL){
tscInitRes = terrno;
tscError("failed to init conv");
return;
}
ENV_ERR_RET(monitorInit(), "failed to init monitor"); ENV_ERR_RET(monitorInit(), "failed to init monitor");
ENV_ERR_RET(rpcInit(), "failed to init rpc"); ENV_ERR_RET(rpcInit(), "failed to init rpc");

15
source/client/src/clientImpl.c
View File

@ -301,7 +301,8 @@ int32_t parseSql(SRequestObj* pRequest, bool topicQuery, SQuery** pQuery, SStmtC
.nodeOffline = (pTscObj->pAppInfo->onlineDnodes < pTscObj->pAppInfo->totalDnodes), .nodeOffline = (pTscObj->pAppInfo->onlineDnodes < pTscObj->pAppInfo->totalDnodes),
.isStmtBind = pRequest->isStmtBind, .isStmtBind = pRequest->isStmtBind,
.setQueryFp = setQueryRequest, .setQueryFp = setQueryRequest,
.timezone = pTscObj->optionInfo.timezone,}; .timezone = pTscObj->optionInfo.timezone,
.charsetCxt = pTscObj->optionInfo.charsetCxt,};
cxt.mgmtEpSet = getEpSet_s(&pTscObj->pAppInfo->mgmtEp); cxt.mgmtEpSet = getEpSet_s(&pTscObj->pAppInfo->mgmtEp);
int32_t code = catalogGetHandle(pTscObj->pAppInfo->clusterId, &cxt.pCatalog); int32_t code = catalogGetHandle(pTscObj->pAppInfo->clusterId, &cxt.pCatalog);
@ -2087,7 +2088,7 @@ static int32_t doPrepareResPtr(SReqResultInfo* pResInfo) {
static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t* colLength) { static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t* colLength) {
int32_t idx = -1; int32_t idx = -1;
iconv_t conv = taosAcquireConv(&idx, C2M); iconv_t conv = taosAcquireConv(&idx, C2M, pResultInfo->charsetCxt);
if (conv == (iconv_t)-1) return TSDB_CODE_TSC_INTERNAL_ERROR; if (conv == (iconv_t)-1) return TSDB_CODE_TSC_INTERNAL_ERROR;
for (int32_t i = 0; i < pResultInfo->numOfCols; ++i) { for (int32_t i = 0; i < pResultInfo->numOfCols; ++i) {
@ -2097,7 +2098,7 @@ static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t* colLength) {
if (type == TSDB_DATA_TYPE_NCHAR && colLength[i] > 0) { if (type == TSDB_DATA_TYPE_NCHAR && colLength[i] > 0) {
char* p = taosMemoryRealloc(pResultInfo->convertBuf[i], colLength[i]); char* p = taosMemoryRealloc(pResultInfo->convertBuf[i], colLength[i]);
if (p == NULL) { if (p == NULL) {
taosReleaseConv(idx, conv, C2M); taosReleaseConv(idx, conv, C2M, pResultInfo->charsetCxt);
return terrno; return terrno;
} }
@ -2114,7 +2115,7 @@ static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t* colLength) {
"doConvertUCS4 error, invalid data. len:%d, bytes:%d, (p + len):%p, (pResultInfo->convertBuf[i] + " "doConvertUCS4 error, invalid data. len:%d, bytes:%d, (p + len):%p, (pResultInfo->convertBuf[i] + "
"colLength[i]):%p", "colLength[i]):%p",
len, bytes, (p + len), (pResultInfo->convertBuf[i] + colLength[i])); len, bytes, (p + len), (pResultInfo->convertBuf[i] + colLength[i]));
taosReleaseConv(idx, conv, C2M); taosReleaseConv(idx, conv, C2M, pResultInfo->charsetCxt);
return TSDB_CODE_TSC_INTERNAL_ERROR; return TSDB_CODE_TSC_INTERNAL_ERROR;
} }
@ -2128,7 +2129,7 @@ static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t* colLength) {
pResultInfo->row[i] = pResultInfo->pCol[i].pData; pResultInfo->row[i] = pResultInfo->pCol[i].pData;
} }
} }
taosReleaseConv(idx, conv, C2M); taosReleaseConv(idx, conv, C2M, pResultInfo->charsetCxt);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -2293,7 +2294,7 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo) {
varDataSetLen(dst, strlen(varDataVal(dst))); varDataSetLen(dst, strlen(varDataVal(dst)));
} else if (tTagIsJson(data)) { } else if (tTagIsJson(data)) {
char* jsonString = NULL; char* jsonString = NULL;
parseTagDatatoJson(data, &jsonString); parseTagDatatoJson(data, &jsonString, pResultInfo->charsetCxt);
if (jsonString == NULL) { if (jsonString == NULL) {
tscError("doConvertJson error: parseTagDatatoJson failed"); tscError("doConvertJson error: parseTagDatatoJson failed");
return terrno; return terrno;
@ -2303,7 +2304,7 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo) {
} else if (jsonInnerType == TSDB_DATA_TYPE_NCHAR) { // value -> "value" } else if (jsonInnerType == TSDB_DATA_TYPE_NCHAR) { // value -> "value"
*(char*)varDataVal(dst) = '\"'; *(char*)varDataVal(dst) = '\"';
int32_t length = taosUcs4ToMbs((TdUcs4*)varDataVal(jsonInnerData), varDataLen(jsonInnerData), int32_t length = taosUcs4ToMbs((TdUcs4*)varDataVal(jsonInnerData), varDataLen(jsonInnerData),
varDataVal(dst) + CHAR_BYTES); varDataVal(dst) + CHAR_BYTES, pResultInfo->charsetCxt);
if (length <= 0) { if (length <= 0) {
tscError("charset:%s to %s. convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset); tscError("charset:%s to %s. convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset);
length = 0; length = 0;

96
source/client/src/clientMain.c
View File

@ -30,6 +30,7 @@
#include "tref.h" #include "tref.h"
#include "trpc.h" #include "trpc.h"
#include "version.h" #include "version.h"
#include "tconv.h"
#define TSC_VAR_NOT_RELEASE 1 #define TSC_VAR_NOT_RELEASE 1
#define TSC_VAR_RELEASED 0 #define TSC_VAR_RELEASED 0
@ -52,6 +53,49 @@ int taos_options(TSDB_OPTION option, const void *arg, ...) {
return ret; return ret;
} }
static timezone_t setConnnectionTz(const char* val){
timezone_t tz = NULL;
static int32_t lock_c = 0;
for (int i = 1; atomic_val_compare_exchange_32(&lock_c, 0, 1) != 0; ++i) {
if (i % 1000 == 0) {
tscInfo("haven't acquire lock after spin %d times.", i);
(void)sched_yield();
}
}
if (pTimezoneMap == NULL){
pTimezoneMap = taosHashInit(0, MurmurHash3_32, false, HASH_ENTRY_LOCK);
if (pTimezoneMap == NULL) {
atomic_store_32(&lock_c, 0);
goto END;
}
taosHashSetFreeFp(pTimezoneMap, (_hash_free_fn_t)tzfree);
}
timezone_t *tmp = taosHashGet(pTimezoneMap, val, strlen(val));
if (tmp != NULL && *tmp != NULL){
tz = *tmp;
goto END;
}
tscDebug("set timezone to %s", val);
tz = tzalloc(val);
if (tz == NULL) {
tscError("%s unknown timezone %s", __func__, val);
terrno = TAOS_SYSTEM_ERROR(errno);
goto END;
}
int32_t code = taosHashPut(pTimezoneMap, val, strlen(val), &tz, sizeof(timezone_t));
if (code != 0){
tzfree(tz);
tz = NULL;
}
END:
atomic_store_32(&lock_c, 0);
return tz;
}
static int32_t setConnectionOption(TAOS *taos, TSDB_OPTION_CONNECTION option, const char* val){ static int32_t setConnectionOption(TAOS *taos, TSDB_OPTION_CONNECTION option, const char* val){
if (taos == NULL) { if (taos == NULL) {
return TSDB_CODE_INVALID_PARA; return TSDB_CODE_INVALID_PARA;
@ -74,32 +118,24 @@ static int32_t setConnectionOption(TAOS *taos, TSDB_OPTION_CONNECTION option, co
code = terrno; code = terrno;
goto END; goto END;
} }
tstrncpy(pObj->optionInfo.charset, val, TD_CHARSET_LEN);
}else{ }else{
pObj->optionInfo.charset[0] = 0; val = tsCharset;
} }
void *tmp = taosConvInit(val);
if (tmp == NULL) {
code = terrno;
goto END;
}
pObj->optionInfo.charsetCxt = tmp;
} else if (option == TSDB_OPTION_CONNECTION_TIMEZONE) { } else if (option == TSDB_OPTION_CONNECTION_TIMEZONE) {
if (val != NULL){ if (val != NULL){
if (strlen(val) == 0){ if (strlen(val) == 0){
code = TSDB_CODE_INVALID_PARA; code = TSDB_CODE_INVALID_PARA;
goto END; goto END;
} }
timezone_t *tmp = taosHashGet(pTimezoneMap, val, strlen(val)); timezone_t tz = setConnnectionTz(val);
if (tmp && *tmp){ if (tz == NULL){
pObj->optionInfo.timezone = *tmp; code = terrno;
goto END;
}
tscDebug("set timezone to %s", val);
timezone_t tz = tzalloc(val);
if (!tz) {
tscError("%s unknown timezone %s", __func__, val);
code = TAOS_SYSTEM_ERROR(errno);
goto END;
}
code = taosHashPut(pTimezoneMap, val, strlen(val), &tz, sizeof(timezone_t));
if (code != 0){
tzfree(tz);
goto END; goto END;
} }
pObj->optionInfo.timezone = tz; pObj->optionInfo.timezone = tz;
@ -126,26 +162,7 @@ END:
} }
int taos_options_connection(TAOS *taos, TSDB_OPTION_CONNECTION option, const void *arg, ...){ int taos_options_connection(TAOS *taos, TSDB_OPTION_CONNECTION option, const void *arg, ...){
static int32_t lock_c = 0; return setConnectionOption(taos, option, (const char *)arg);
for (int i = 1; atomic_val_compare_exchange_32(&lock_c, 0, 1) != 0; ++i) {
if (i % 1000 == 0) {
tscInfo("haven't acquire lock after spin %d times.", i);
(void)sched_yield();
}
}
if (pTimezoneMap == NULL){
pTimezoneMap = taosHashInit(0, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK);
if (pTimezoneMap == NULL) {
atomic_store_32(&lock_c, 0);
return terrno;
}
taosHashSetFreeFp(pTimezoneMap, (_hash_free_fn_t)tzfree);
}
int ret = setConnectionOption(taos, option, (const char *)arg);
atomic_store_32(&lock_c, 0);
return ret;
} }
// this function may be called by user or system, or by both simultaneously. // this function may be called by user or system, or by both simultaneously.
@ -1342,7 +1359,8 @@ int32_t createParseContext(const SRequestObj *pRequest, SParseContext **pCxt, SS
.parseSqlFp = clientParseSql, .parseSqlFp = clientParseSql,
.parseSqlParam = pWrapper, .parseSqlParam = pWrapper,
.setQueryFp = setQueryRequest, .setQueryFp = setQueryRequest,
.timezone = pTscObj->optionInfo.timezone}; .timezone = pTscObj->optionInfo.timezone,
.charsetCxt = pTscObj->optionInfo.charsetCxt};
int8_t biMode = atomic_load_8(&((STscObj *)pTscObj)->biMode); int8_t biMode = atomic_load_8(&((STscObj *)pTscObj)->biMode);
(*pCxt)->biMode = biMode; (*pCxt)->biMode = biMode;
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;

4
source/client/src/clientRawBlockWrite.c
View File

@ -414,7 +414,7 @@ static void buildChildElement(cJSON* json, SVCreateTbReq* pCreateReq) {
goto end; goto end;
} }
char* pJson = NULL; char* pJson = NULL;
parseTagDatatoJson(pTag, &pJson); parseTagDatatoJson(pTag, &pJson, NULL);
if (pJson == NULL) { if (pJson == NULL) {
uError("parseTagDatatoJson failed, pJson == NULL"); uError("parseTagDatatoJson failed, pJson == NULL");
goto end; goto end;
@ -730,7 +730,7 @@ static void processAlterTable(SMqMetaRsp* metaRsp, cJSON** pJson) {
uError("processAlterTable isJson false"); uError("processAlterTable isJson false");
goto end; goto end;
} }
parseTagDatatoJson(vAlterTbReq.pTagVal, &buf); parseTagDatatoJson(vAlterTbReq.pTagVal, &buf, NULL);
if (buf == NULL) { if (buf == NULL) {
uError("parseTagDatatoJson failed, buf == NULL"); uError("parseTagDatatoJson failed, buf == NULL");
goto end; goto end;

10
source/client/src/clientSml.c
View File

@ -267,7 +267,7 @@ bool isSmlColAligned(SSmlHandle *info, int cnt, SSmlKv *kv) {
goto END; goto END;
} }
// bind data // bind data
int32_t ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, kv, cnt + 1); int32_t ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, kv, cnt + 1, info->taos->optionInfo.charsetCxt);
if (unlikely(ret != TSDB_CODE_SUCCESS)) { if (unlikely(ret != TSDB_CODE_SUCCESS)) {
uDebug("smlBuildCol error, retry"); uDebug("smlBuildCol error, retry");
goto END; goto END;
@ -411,8 +411,8 @@ int32_t smlParseEndTelnetJsonFormat(SSmlHandle *info, SSmlLineInfo *elements, SS
int32_t code = 0; int32_t code = 0;
int32_t lino = 0; int32_t lino = 0;
uDebug("SML:0x%" PRIx64 " %s format true, ts:%" PRId64, info->id, __FUNCTION__ , kvTs->i); uDebug("SML:0x%" PRIx64 " %s format true, ts:%" PRId64, info->id, __FUNCTION__ , kvTs->i);
SML_CHECK_CODE(smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, kvTs, 0)); SML_CHECK_CODE(smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, kvTs, 0, info->taos->optionInfo.charsetCxt));
SML_CHECK_CODE(smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, kv, 1)); SML_CHECK_CODE(smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, kv, 1, info->taos->optionInfo.charsetCxt));
SML_CHECK_CODE(smlBuildRow(info->currTableDataCtx)); SML_CHECK_CODE(smlBuildRow(info->currTableDataCtx));
END: END:
@ -438,7 +438,7 @@ END:
int32_t smlParseEndLine(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs) { int32_t smlParseEndLine(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs) {
if (info->dataFormat) { if (info->dataFormat) {
uDebug("SML:0x%" PRIx64 " %s format true, ts:%" PRId64, info->id, __FUNCTION__, kvTs->i); uDebug("SML:0x%" PRIx64 " %s format true, ts:%" PRId64, info->id, __FUNCTION__, kvTs->i);
int32_t ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, kvTs, 0); int32_t ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, kvTs, 0, info->taos->optionInfo.charsetCxt);
if (ret == TSDB_CODE_SUCCESS) { if (ret == TSDB_CODE_SUCCESS) {
ret = smlBuildRow(info->currTableDataCtx); ret = smlBuildRow(info->currTableDataCtx);
} }
@ -1482,7 +1482,7 @@ static int32_t smlInsertData(SSmlHandle *info) {
SML_CHECK_CODE(smlBindData(info->pQuery, info->dataFormat, tableData->tags, (*pMeta)->cols, tableData->cols, SML_CHECK_CODE(smlBindData(info->pQuery, info->dataFormat, tableData->tags, (*pMeta)->cols, tableData->cols,
(*pMeta)->tableMeta, tableData->childTableName, measure, measureLen, info->ttl, info->msgBuf.buf, (*pMeta)->tableMeta, tableData->childTableName, measure, measureLen, info->ttl, info->msgBuf.buf,
info->msgBuf.len)); info->msgBuf.len, info->taos->optionInfo.charsetCxt));
taosMemoryFreeClear(measure); taosMemoryFreeClear(measure);
oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, oneTable); oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, oneTable);
} }

10
source/client/src/clientStmt.c
View File

@ -1071,7 +1071,7 @@ int stmtSetTbTags(TAOS_STMT* stmt, TAOS_MULTI_BIND* tags) {
tscDebug("start to bind stmt tag values"); tscDebug("start to bind stmt tag values");
STMT_ERR_RET(qBindStmtTagsValue(*pDataBlock, pStmt->bInfo.boundTags, pStmt->bInfo.tbSuid, pStmt->bInfo.stbFName, STMT_ERR_RET(qBindStmtTagsValue(*pDataBlock, pStmt->bInfo.boundTags, pStmt->bInfo.tbSuid, pStmt->bInfo.stbFName,
pStmt->bInfo.sname.tname, tags, pStmt->exec.pRequest->msgBuf, pStmt->bInfo.sname.tname, tags, pStmt->exec.pRequest->msgBuf,
pStmt->exec.pRequest->msgBufLen)); pStmt->exec.pRequest->msgBufLen, pStmt->taos->optionInfo.charsetCxt));
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -1239,7 +1239,7 @@ int stmtBindBatch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind, int32_t colIdx) {
} }
if (STMT_TYPE_QUERY == pStmt->sql.type) { if (STMT_TYPE_QUERY == pStmt->sql.type) {
STMT_ERR_RET(qStmtBindParams(pStmt->sql.pQuery, bind, colIdx)); STMT_ERR_RET(qStmtBindParams(pStmt->sql.pQuery, bind, colIdx, pStmt->taos->optionInfo.charsetCxt));
SParseContext ctx = {.requestId = pStmt->exec.pRequest->requestId, SParseContext ctx = {.requestId = pStmt->exec.pRequest->requestId,
.acctId = pStmt->taos->acctId, .acctId = pStmt->taos->acctId,
@ -1325,10 +1325,10 @@ int stmtBindBatch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind, int32_t colIdx) {
if (pStmt->sql.stbInterlaceMode) { if (pStmt->sql.stbInterlaceMode) {
(*pDataBlock)->pData->flags = 0; (*pDataBlock)->pData->flags = 0;
code = qBindStmtStbColsValue(*pDataBlock, pCols, bind, pStmt->exec.pRequest->msgBuf, code = qBindStmtStbColsValue(*pDataBlock, pCols, bind, pStmt->exec.pRequest->msgBuf,
pStmt->exec.pRequest->msgBufLen, &pStmt->sql.siInfo.pTSchema, pStmt->sql.pBindInfo); pStmt->exec.pRequest->msgBufLen, &pStmt->sql.siInfo.pTSchema, pStmt->sql.pBindInfo, pStmt->taos->optionInfo.charsetCxt);
} else { } else {
code = code =
qBindStmtColsValue(*pDataBlock, pCols, bind, pStmt->exec.pRequest->msgBuf, pStmt->exec.pRequest->msgBufLen); qBindStmtColsValue(*pDataBlock, pCols, bind, pStmt->exec.pRequest->msgBuf, pStmt->exec.pRequest->msgBufLen, pStmt->taos->optionInfo.charsetCxt);
} }
if (code) { if (code) {
@ -1353,7 +1353,7 @@ int stmtBindBatch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind, int32_t colIdx) {
} }
code = qBindStmtSingleColValue(*pDataBlock, pCols, bind, pStmt->exec.pRequest->msgBuf, code = qBindStmtSingleColValue(*pDataBlock, pCols, bind, pStmt->exec.pRequest->msgBuf,
pStmt->exec.pRequest->msgBufLen, colIdx, pStmt->bInfo.sBindRowNum); pStmt->exec.pRequest->msgBufLen, colIdx, pStmt->bInfo.sBindRowNum, pStmt->taos->optionInfo.charsetCxt);
if (code) { if (code) {
tscError("qBindStmtSingleColValue failed, error:%s", tstrerror(code)); tscError("qBindStmtSingleColValue failed, error:%s", tstrerror(code));
STMT_ERR_RET(code); STMT_ERR_RET(code);

10
source/client/src/clientStmt2.c
View File

@ -1014,7 +1014,7 @@ int stmtSetTbTags2(TAOS_STMT2* stmt, TAOS_STMT2_BIND* tags) {
tscDebug("start to bind stmt tag values"); tscDebug("start to bind stmt tag values");
STMT_ERR_RET(qBindStmtTagsValue2(*pDataBlock, pStmt->bInfo.boundTags, pStmt->bInfo.tbSuid, pStmt->bInfo.stbFName, STMT_ERR_RET(qBindStmtTagsValue2(*pDataBlock, pStmt->bInfo.boundTags, pStmt->bInfo.tbSuid, pStmt->bInfo.stbFName,
pStmt->bInfo.sname.tname, tags, pStmt->exec.pRequest->msgBuf, pStmt->bInfo.sname.tname, tags, pStmt->exec.pRequest->msgBuf,
pStmt->exec.pRequest->msgBufLen)); pStmt->exec.pRequest->msgBufLen, pStmt->taos->optionInfo.charsetCxt));
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -1323,7 +1323,7 @@ int stmtBindBatch2(TAOS_STMT2* stmt, TAOS_STMT2_BIND* bind, int32_t colIdx) {
} }
if (STMT_TYPE_QUERY == pStmt->sql.type) { if (STMT_TYPE_QUERY == pStmt->sql.type) {
STMT_ERR_RET(qStmtBindParams2(pStmt->sql.pQuery, bind, colIdx)); STMT_ERR_RET(qStmtBindParams2(pStmt->sql.pQuery, bind, colIdx, pStmt->taos->optionInfo.charsetCxt));
SParseContext ctx = {.requestId = pStmt->exec.pRequest->requestId, SParseContext ctx = {.requestId = pStmt->exec.pRequest->requestId,
.acctId = pStmt->taos->acctId, .acctId = pStmt->taos->acctId,
@ -1407,10 +1407,10 @@ int stmtBindBatch2(TAOS_STMT2* stmt, TAOS_STMT2_BIND* bind, int32_t colIdx) {
if (pStmt->sql.stbInterlaceMode) { if (pStmt->sql.stbInterlaceMode) {
(*pDataBlock)->pData->flags = 0; (*pDataBlock)->pData->flags = 0;
code = qBindStmtStbColsValue2(*pDataBlock, pCols, bind, pStmt->exec.pRequest->msgBuf, code = qBindStmtStbColsValue2(*pDataBlock, pCols, bind, pStmt->exec.pRequest->msgBuf,
pStmt->exec.pRequest->msgBufLen, &pStmt->sql.siInfo.pTSchema, pStmt->sql.pBindInfo); pStmt->exec.pRequest->msgBufLen, &pStmt->sql.siInfo.pTSchema, pStmt->sql.pBindInfo, pStmt->taos->optionInfo.charsetCxt);
} else { } else {
code = code =
qBindStmtColsValue2(*pDataBlock, pCols, bind, pStmt->exec.pRequest->msgBuf, pStmt->exec.pRequest->msgBufLen); qBindStmtColsValue2(*pDataBlock, pCols, bind, pStmt->exec.pRequest->msgBuf, pStmt->exec.pRequest->msgBufLen, pStmt->taos->optionInfo.charsetCxt);
} }
if (code) { if (code) {
@ -1435,7 +1435,7 @@ int stmtBindBatch2(TAOS_STMT2* stmt, TAOS_STMT2_BIND* bind, int32_t colIdx) {
} }
code = qBindStmtSingleColValue2(*pDataBlock, pCols, bind, pStmt->exec.pRequest->msgBuf, code = qBindStmtSingleColValue2(*pDataBlock, pCols, bind, pStmt->exec.pRequest->msgBuf,
pStmt->exec.pRequest->msgBufLen, colIdx, pStmt->bInfo.sBindRowNum); pStmt->exec.pRequest->msgBufLen, colIdx, pStmt->bInfo.sBindRowNum, pStmt->taos->optionInfo.charsetCxt);
if (code) { if (code) {
tscError("qBindStmtSingleColValue failed, error:%s", tstrerror(code)); tscError("qBindStmtSingleColValue failed, error:%s", tstrerror(code));
STMT_ERR_RET(code); STMT_ERR_RET(code);

2
source/common/src/tdatablock.c
View File

@ -2641,7 +2641,7 @@ int32_t dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** pDataBuf
char* pData = colDataGetVarData(pColInfoData, j); char* pData = colDataGetVarData(pColInfoData, j);
int32_t dataSize = TMIN(sizeof(pBuf), varDataLen(pData)); int32_t dataSize = TMIN(sizeof(pBuf), varDataLen(pData));
memset(pBuf, 0, sizeof(pBuf)); memset(pBuf, 0, sizeof(pBuf));
code = taosUcs4ToMbs((TdUcs4*)varDataVal(pData), dataSize, pBuf); code = taosUcs4ToMbs((TdUcs4*)varDataVal(pData), dataSize, pBuf, NULL);
if (code < 0) { if (code < 0) {
uError("func %s failed to convert to ucs charset since %s", __func__, tstrerror(code)); uError("func %s failed to convert to ucs charset since %s", __func__, tstrerror(code));
lino = __LINE__; lino = __LINE__;

47
source/common/src/tglobal.c
View File

@ -654,7 +654,6 @@ static int32_t taosAddClientCfg(SConfig *pCfg) {
static int32_t taosAddSystemCfg(SConfig *pCfg) { static int32_t taosAddSystemCfg(SConfig *pCfg) {
SysNameInfo info = taosGetSysNameInfo(); SysNameInfo info = taosGetSysNameInfo();
(void)taosGetSystemTimezone(tsTimezoneStr);
TAOS_CHECK_RETURN(cfgAddTimezone(pCfg, "timezone", tsTimezoneStr, CFG_SCOPE_BOTH, CFG_DYN_CLIENT)); TAOS_CHECK_RETURN(cfgAddTimezone(pCfg, "timezone", tsTimezoneStr, CFG_SCOPE_BOTH, CFG_DYN_CLIENT));
TAOS_CHECK_RETURN(cfgAddLocale(pCfg, "locale", tsLocale, CFG_SCOPE_BOTH, CFG_DYN_CLIENT)); TAOS_CHECK_RETURN(cfgAddLocale(pCfg, "locale", tsLocale, CFG_SCOPE_BOTH, CFG_DYN_CLIENT));
TAOS_CHECK_RETURN(cfgAddCharset(pCfg, "charset", tsCharset, CFG_SCOPE_BOTH, CFG_DYN_NONE)); TAOS_CHECK_RETURN(cfgAddCharset(pCfg, "charset", tsCharset, CFG_SCOPE_BOTH, CFG_DYN_NONE));
@ -1317,25 +1316,6 @@ static int32_t taosSetClientCfg(SConfig *pCfg) {
static int32_t taosSetSystemCfg(SConfig *pCfg) { static int32_t taosSetSystemCfg(SConfig *pCfg) {
SConfigItem *pItem = NULL; SConfigItem *pItem = NULL;
TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "locale");
const char *locale = pItem->str;
TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "charset");
const char *charset = pItem->str;
int32_t code = taosSetSystemLocale(locale, charset);
if (TSDB_CODE_SUCCESS != code) {
uError("failed to set locale:%s, since: %s", locale, tstrerror(code));
char curLocale[TD_LOCALE_LEN] = {0};
char curCharset[TD_CHARSET_LEN] = {0};
taosGetSystemLocale(curLocale, curCharset);
if (0 != strlen(curLocale) && 0 != strlen(curCharset)) {
uInfo("current locale: %s, charset: %s", curLocale, curCharset);
}
}
osSetSystemLocale(locale, charset);
TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "enableCoreFile"); TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "enableCoreFile");
tsEnableCoreFile = pItem->bval; tsEnableCoreFile = pItem->bval;
taosSetCoreDump(tsEnableCoreFile); taosSetCoreDump(tsEnableCoreFile);
@ -2062,6 +2042,10 @@ static int32_t taosCfgDynamicOptionsForClient(SConfig *pCfg, const char *name) {
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0; int32_t lino = 0;
if (strcasecmp("locale", name) == 0 || strcasecmp("charset", name) == 0
|| strcasecmp("timezone", name) == 0) {
goto _out;
}
cfgLock(pCfg); cfgLock(pCfg);
SConfigItem *pItem = cfgGetItem(pCfg, name); SConfigItem *pItem = cfgGetItem(pCfg, name);
@ -2150,25 +2134,6 @@ static int32_t taosCfgDynamicOptionsForClient(SConfig *pCfg, const char *name) {
} }
break; break;
} }
case 'l': {
if (strcasecmp("locale", name) == 0) {
SConfigItem *pLocaleItem = cfgGetItem(pCfg, "locale");
SConfigItem *pCharsetItem = cfgGetItem(pCfg, "charset");
if (pLocaleItem == NULL || pCharsetItem == NULL) {
uError("failed to get locale or charset from cfg");
code = TSDB_CODE_CFG_NOT_FOUND;
goto _out;
}
const char *locale = pLocaleItem->str;
const char *charset = pCharsetItem->str;
TAOS_CHECK_GOTO(taosSetSystemLocale(locale, charset), &lino, _out);
osSetSystemLocale(locale, charset);
uInfo("locale set to '%s', charset set to '%s'", locale, charset);
matched = true;
}
break;
}
case 'm': { case 'm': {
if (strcasecmp("metaCacheMaxSize", name) == 0) { if (strcasecmp("metaCacheMaxSize", name) == 0) {
atomic_store_32(&tsMetaCacheMaxSize, pItem->i32); atomic_store_32(&tsMetaCacheMaxSize, pItem->i32);
@ -2239,9 +2204,7 @@ static int32_t taosCfgDynamicOptionsForClient(SConfig *pCfg, const char *name) {
break; break;
} }
case 't': { case 't': {
if (strcasecmp("timezone", name) == 0) { if (strcasecmp("tempDir", name) == 0) {
matched = true;
} else if (strcasecmp("tempDir", name) == 0) {
uInfo("%s set from %s to %s", name, tsTempDir, pItem->str); uInfo("%s set from %s to %s", name, tsTempDir, pItem->str);
tstrncpy(tsTempDir, pItem->str, PATH_MAX); tstrncpy(tsTempDir, pItem->str, PATH_MAX);
TAOS_CHECK_GOTO(taosExpandDir(tsTempDir, tsTempDir, PATH_MAX), &lino, _out); TAOS_CHECK_GOTO(taosExpandDir(tsTempDir, tsTempDir, PATH_MAX), &lino, _out);

10
source/common/src/ttime.c
View File

@ -480,7 +480,7 @@ int32_t convertTimeFromPrecisionToUnit(int64_t time, int32_t fromPrecision, char
TAOS_RETURN(TSDB_CODE_SUCCESS); TAOS_RETURN(TSDB_CODE_SUCCESS);
} }
int32_t convertStringToTimestamp(int16_t type, char* inputData, int64_t timePrec, int64_t* timeVal, timezone_t tz) { int32_t convertStringToTimestamp(int16_t type, char* inputData, int64_t timePrec, int64_t* timeVal, timezone_t tz, void* charsetCxt) {
int32_t charLen = varDataLen(inputData); int32_t charLen = varDataLen(inputData);
char* newColData; char* newColData;
if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_VARBINARY) { if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_VARBINARY) {
@ -500,7 +500,7 @@ int32_t convertStringToTimestamp(int16_t type, char* inputData, int64_t timePrec
if (NULL == newColData) { if (NULL == newColData) {
TAOS_RETURN(terrno); TAOS_RETURN(terrno);
} }
int len = taosUcs4ToMbs((TdUcs4*)varDataVal(inputData), charLen, newColData); int len = taosUcs4ToMbs((TdUcs4*)varDataVal(inputData), charLen, newColData, charsetCxt);
if (len < 0) { if (len < 0) {
taosMemoryFree(newColData); taosMemoryFree(newColData);
TAOS_RETURN(TSDB_CODE_FAILED); TAOS_RETURN(TSDB_CODE_FAILED);
@ -2011,7 +2011,11 @@ int64_t taosGetTimestampToday(int32_t precision, timezone_t tz) {
: (precision == TSDB_TIME_PRECISION_MILLI) ? 1000 : (precision == TSDB_TIME_PRECISION_MILLI) ? 1000
: (precision == TSDB_TIME_PRECISION_MICRO) ? 1000000 : (precision == TSDB_TIME_PRECISION_MICRO) ? 1000000
: 1000000000; : 1000000000;
time_t t = taosTime(NULL); time_t t;
int32_t code = taosTime(&t);
if (code != 0) {
return -1;
}
struct tm tm; struct tm tm;
if (taosLocalTime(&t, &tm, NULL, 0, tz) == NULL){ if (taosLocalTime(&t, &tm, NULL, 0, tz) == NULL){
uError("%s failed to get local time, code:%d", __FUNCTION__, errno); uError("%s failed to get local time, code:%d", __FUNCTION__, errno);

6
source/common/test/commonTests.cpp
View File

@ -477,7 +477,7 @@ void test_ts2char(int64_t ts, const char* format, int32_t precison, const char*
TEST(timeTest, ts2char) { TEST(timeTest, ts2char) {
osDefaultInit(); osDefaultInit();
if (tsTimezone != TdEastZone8) GTEST_SKIP(); if (taosGetLocalTimezoneOffset() != TdEastZone8) GTEST_SKIP();
int64_t ts; int64_t ts;
const char* format = "YYYY-MM-DD"; const char* format = "YYYY-MM-DD";
ts = 0; ts = 0;
@ -529,7 +529,7 @@ TEST(timeTest, ts2char) {
TEST(timeTest, char2ts) { TEST(timeTest, char2ts) {
osDefaultInit(); osDefaultInit();
if (tsTimezone != TdEastZone8) GTEST_SKIP(); if (taosGetLocalTimezoneOffset() != TdEastZone8) GTEST_SKIP();
int64_t ts; int64_t ts;
int32_t code = int32_t code =
TEST_char2ts("YYYY-DD-MM HH12:MI:SS:MSPM", &ts, TSDB_TIME_PRECISION_MILLI, "2023-10-10 12:00:00.000AM"); TEST_char2ts("YYYY-DD-MM HH12:MI:SS:MSPM", &ts, TSDB_TIME_PRECISION_MILLI, "2023-10-10 12:00:00.000AM");
@ -630,7 +630,7 @@ TEST(timeTest, char2ts) {
// default to 1970-1-1 00:00:00+08 -> 1969-12-31 16:00:00+00 // default to 1970-1-1 00:00:00+08 -> 1969-12-31 16:00:00+00
ASSERT_EQ(0, TEST_char2ts("YYYY", &ts, TSDB_TIME_PRECISION_SECONDS, "1970")); ASSERT_EQ(0, TEST_char2ts("YYYY", &ts, TSDB_TIME_PRECISION_SECONDS, "1970"));
ASSERT_EQ(ts, -1 * tsTimezone * 60 * 60); ASSERT_EQ(ts, -1 * taosGetLocalTimezoneOffset());
ASSERT_EQ(0, TEST_char2ts("yyyyMM1/dd ", &ts, TSDB_TIME_PRECISION_MICRO, "210001/2")); ASSERT_EQ(0, TEST_char2ts("yyyyMM1/dd ", &ts, TSDB_TIME_PRECISION_MICRO, "210001/2"));
ASSERT_EQ(ts, 4102502400000000LL); ASSERT_EQ(ts, 4102502400000000LL);

3
source/dnode/mgmt/exe/dmMain.c
View File

@ -20,6 +20,7 @@
#include "tconfig.h" #include "tconfig.h"
#include "tglobal.h" #include "tglobal.h"
#include "version.h" #include "version.h"
#include "tconv.h"
#ifdef TD_JEMALLOC_ENABLED #ifdef TD_JEMALLOC_ENABLED
#include "jemalloc/jemalloc.h" #include "jemalloc/jemalloc.h"
#endif #endif
@ -444,7 +445,7 @@ int mainWindows(int argc, char **argv) {
return code; return code;
} }
if ((code = taosConvInit()) != 0) { if ((tsCharsetCxt = taosConvInit(tsCharset)) == NULL) {
dError("failed to init conv"); dError("failed to init conv");
taosCloseLog(); taosCloseLog();
taosCleanupArgs(); taosCleanupArgs();

1
source/dnode/mgmt/node_mgmt/src/dmMgmt.c
View File

@ -22,6 +22,7 @@
#include "tglobal.h" #include "tglobal.h"
#include "tgrant.h" #include "tgrant.h"
#include "tstream.h" #include "tstream.h"
#include "tconv.h"
static bool dmRequireNode(SDnode *pDnode, SMgmtWrapper *pWrapper) { static bool dmRequireNode(SDnode *pDnode, SMgmtWrapper *pWrapper) {
SMgmtInputOpt input = dmBuildMgmtInputOpt(pWrapper); SMgmtInputOpt input = dmBuildMgmtInputOpt(pWrapper);

2
source/dnode/mnode/impl/inc/mndDef.h
View File

@ -489,7 +489,7 @@ typedef struct {
int64_t dstTbUid; int64_t dstTbUid;
int8_t intervalUnit; int8_t intervalUnit;
int8_t slidingUnit; int8_t slidingUnit;
int8_t timezone; int8_t timezone; // int8_t is not enough, timezone is unit of second
int32_t dstVgId; // for stream int32_t dstVgId; // for stream
int64_t interval; int64_t interval;
int64_t offset; int64_t offset;

8
source/dnode/mnode/impl/src/mndSma.c
View File

@ -316,7 +316,7 @@ static void *mndBuildVCreateSmaReq(SMnode *pMnode, SVgObj *pVgroup, SSmaObj *pSm
req.version = 0; req.version = 0;
req.intervalUnit = pSma->intervalUnit; req.intervalUnit = pSma->intervalUnit;
req.slidingUnit = pSma->slidingUnit; req.slidingUnit = pSma->slidingUnit;
req.timezoneInt = pSma->timezone; // req.timezoneInt = pSma->timezone;
tstrncpy(req.indexName, (char *)tNameGetTableName(&name), TSDB_INDEX_NAME_LEN); tstrncpy(req.indexName, (char *)tNameGetTableName(&name), TSDB_INDEX_NAME_LEN);
req.exprLen = pSma->exprLen; req.exprLen = pSma->exprLen;
req.tagsFilterLen = pSma->tagsFilterLen; req.tagsFilterLen = pSma->tagsFilterLen;
@ -617,9 +617,9 @@ static int32_t mndCreateSma(SMnode *pMnode, SRpcMsg *pReq, SMCreateSmaReq *pCrea
smaObj.intervalUnit = pCreate->intervalUnit; smaObj.intervalUnit = pCreate->intervalUnit;
smaObj.slidingUnit = pCreate->slidingUnit; smaObj.slidingUnit = pCreate->slidingUnit;
#if 0 #if 0
smaObj.timezone = pCreate->timezone; // smaObj.timezone = pCreate->timezone;
#endif #endif
smaObj.timezone = tsTimezone; // use timezone of server // smaObj.timezone = taosGetLocalTimezoneOffset(); // use timezone of server
smaObj.interval = pCreate->interval; smaObj.interval = pCreate->interval;
smaObj.offset = pCreate->offset; smaObj.offset = pCreate->offset;
smaObj.sliding = pCreate->sliding; smaObj.sliding = pCreate->sliding;
@ -1554,7 +1554,7 @@ static void initSMAObj(SCreateTSMACxt* pCxt) {
pCxt->pSma->dbUid = pCxt->pDb->uid; pCxt->pSma->dbUid = pCxt->pDb->uid;
pCxt->pSma->interval = pCxt->pCreateSmaReq->interval; pCxt->pSma->interval = pCxt->pCreateSmaReq->interval;
pCxt->pSma->intervalUnit = pCxt->pCreateSmaReq->intervalUnit; pCxt->pSma->intervalUnit = pCxt->pCreateSmaReq->intervalUnit;
pCxt->pSma->timezone = tsTimezone; // pCxt->pSma->timezone = taosGetLocalTimezoneOffset();
pCxt->pSma->version = 1; pCxt->pSma->version = 1;
pCxt->pSma->exprLen = pCxt->pCreateSmaReq->exprLen; pCxt->pSma->exprLen = pCxt->pCreateSmaReq->exprLen;

2
source/dnode/vnode/src/meta/metaQuery.c
View File

@ -1328,7 +1328,7 @@ int32_t metaFilterTableIds(void *pVnode, SMetaFltParam *arg, SArray *pUids) {
TAOS_CHECK_GOTO(terrno, NULL, END); TAOS_CHECK_GOTO(terrno, NULL, END);
} }
if (false == taosMbsToUcs4(tagData, nTagData, (TdUcs4 *)buf, maxSize, &maxSize)) { if (false == taosMbsToUcs4(tagData, nTagData, (TdUcs4 *)buf, maxSize, &maxSize, NULL)) {
TAOS_CHECK_GOTO(terrno, NULL, END); TAOS_CHECK_GOTO(terrno, NULL, END);
} }

4
source/dnode/vnode/src/meta/metaTable.c
View File

@ -153,7 +153,7 @@ static int metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const
if (val == NULL) { if (val == NULL) {
TAOS_CHECK_GOTO(terrno, NULL, _exception); TAOS_CHECK_GOTO(terrno, NULL, _exception);
} }
int32_t len = taosUcs4ToMbs((TdUcs4 *)pTagVal->pData, pTagVal->nData, val + VARSTR_HEADER_SIZE); int32_t len = taosUcs4ToMbs((TdUcs4 *)pTagVal->pData, pTagVal->nData, val + VARSTR_HEADER_SIZE, NULL);
if (len < 0) { if (len < 0) {
TAOS_CHECK_GOTO(len, NULL, _exception); TAOS_CHECK_GOTO(len, NULL, _exception);
} }
@ -237,7 +237,7 @@ int metaDelJsonVarFromIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSche
if (val == NULL) { if (val == NULL) {
TAOS_CHECK_GOTO(terrno, NULL, _exception); TAOS_CHECK_GOTO(terrno, NULL, _exception);
} }
int32_t len = taosUcs4ToMbs((TdUcs4 *)pTagVal->pData, pTagVal->nData, val + VARSTR_HEADER_SIZE); int32_t len = taosUcs4ToMbs((TdUcs4 *)pTagVal->pData, pTagVal->nData, val + VARSTR_HEADER_SIZE, NULL);
if (len < 0) { if (len < 0) {
TAOS_CHECK_GOTO(len, NULL, _exception); TAOS_CHECK_GOTO(len, NULL, _exception);
} }

2
source/libs/catalog/src/catalog.c
View File

@ -447,7 +447,7 @@ int32_t ctgGetTbTag(SCatalog* pCtg, SRequestConnInfo* pConn, SName* pTableName,
} }
char* pJson = NULL; char* pJson = NULL;
parseTagDatatoJson(pTag, &pJson); parseTagDatatoJson(pTag, &pJson, NULL);
if(NULL == pJson) { if(NULL == pJson) {
taosArrayDestroy(pTagVals); taosArrayDestroy(pTagVals);
CTG_ERR_JRET(terrno); CTG_ERR_JRET(terrno);

2
source/libs/catalog/src/ctgAsync.c
View File

@ -2340,7 +2340,7 @@ int32_t ctgHandleGetTbTagRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf*
} }
char* pJson = NULL; char* pJson = NULL;
parseTagDatatoJson(pTag, &pJson); parseTagDatatoJson(pTag, &pJson, NULL);
if (NULL == pJson) { if (NULL == pJson) {
taosArrayDestroy(pTagVals); taosArrayDestroy(pTagVals);
CTG_ERR_JRET(terrno); CTG_ERR_JRET(terrno);

2
source/libs/command/src/command.c
View File

@ -585,7 +585,7 @@ static int32_t appendTagValues(char* buf, int32_t* len, STableCfg* pCfg) {
if (tTagIsJson(pTag)) { if (tTagIsJson(pTag)) {
char* pJson = NULL; char* pJson = NULL;
parseTagDatatoJson(pTag, &pJson); parseTagDatatoJson(pTag, &pJson, NULL);
if (NULL == pJson) { if (NULL == pJson) {
qError("failed to parse tag to json, pJson is NULL"); qError("failed to parse tag to json, pJson is NULL");
return terrno; return terrno;

4
source/libs/executor/src/sysscanoperator.c
View File

@ -966,7 +966,7 @@ int32_t convertTagDataToStr(char* str, int32_t strBuffLen, int type, void* buf,
return TSDB_CODE_TSC_INVALID_VALUE; return TSDB_CODE_TSC_INVALID_VALUE;
} }
int32_t length = taosUcs4ToMbs((TdUcs4*)buf, bufSize, str); int32_t length = taosUcs4ToMbs((TdUcs4*)buf, bufSize, str, NULL);
if (length <= 0) { if (length <= 0) {
return TSDB_CODE_TSC_INVALID_VALUE; return TSDB_CODE_TSC_INVALID_VALUE;
} }
@ -1117,7 +1117,7 @@ static int32_t sysTableUserTagsFillOneTableTags(const SSysTableScanInfo* pInfo,
if (tagData != NULL) { if (tagData != NULL) {
if (tagType == TSDB_DATA_TYPE_JSON) { if (tagType == TSDB_DATA_TYPE_JSON) {
char* tagJson = NULL; char* tagJson = NULL;
parseTagDatatoJson(tagData, &tagJson); parseTagDatatoJson(tagData, &tagJson, NULL);
if (tagJson == NULL) { if (tagJson == NULL) {
code = terrno; code = terrno;
goto _end; goto _end;

2
source/libs/executor/test/queryPlanTests.cpp
View File

@ -592,7 +592,7 @@ void qptGetRandValue(uint8_t* pType, int32_t* pLen, void** ppVal) {
memset(pTmp, 'A' + taosRand() % 26, *pLen); memset(pTmp, 'A' + taosRand() % 26, *pLen);
*ppVal = taosMemoryCalloc(1, *pLen * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE); *ppVal = taosMemoryCalloc(1, *pLen * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE);
assert(*ppVal); assert(*ppVal);
assert(taosMbsToUcs4(pTmp, *pLen, (TdUcs4 *)varDataVal(*ppVal), *pLen * TSDB_NCHAR_SIZE, NULL)); assert(taosMbsToUcs4(pTmp, *pLen, (TdUcs4 *)varDataVal(*ppVal), *pLen * TSDB_NCHAR_SIZE, NULL, NULL));
*pLen *= TSDB_NCHAR_SIZE; *pLen *= TSDB_NCHAR_SIZE;
varDataSetLen(*ppVal, *pLen); varDataSetLen(*ppVal, *pLen);
taosMemoryFree(pTmp); taosMemoryFree(pTmp);

4
source/libs/executor/test/sortTests.cpp
View File

@ -93,7 +93,7 @@ SSDataBlock* getSingleColDummyBlock(void* param) {
char strOri[128] = {0}; char strOri[128] = {0};
taosRandStr(strOri, size); taosRandStr(strOri, size);
int32_t len = 0; int32_t len = 0;
bool ret = taosMbsToUcs4(strOri, size, (TdUcs4*)varDataVal(str), size * TSDB_NCHAR_SIZE, &len); bool ret = taosMbsToUcs4(strOri, size, (TdUcs4*)varDataVal(str), size * TSDB_NCHAR_SIZE, &len, NULL);
if (!ret) { if (!ret) {
(void) printf("error\n"); (void) printf("error\n");
return NULL; return NULL;
@ -331,7 +331,7 @@ TEST(testCase, external_mem_sort_Test) {
if(pInfo[i].type == TSDB_DATA_TYPE_NCHAR){ if(pInfo[i].type == TSDB_DATA_TYPE_NCHAR){
char buf[128] = {0}; char buf[128] = {0};
int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(v), varDataLen(v), buf); int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(v), varDataLen(v), buf, NULL);
printf("%d: %s\n", row++, buf); printf("%d: %s\n", row++, buf);
}else if(pInfo[i].type == TSDB_DATA_TYPE_BINARY || pInfo[i]->type == TSDB_DATA_TYPE_GEOMETRY){ }else if(pInfo[i].type == TSDB_DATA_TYPE_BINARY || pInfo[i]->type == TSDB_DATA_TYPE_GEOMETRY){
char buf[128] = {0}; char buf[128] = {0};

2
source/libs/function/src/builtins.c
View File

@ -1664,7 +1664,7 @@ static int32_t translateOutVarchar(SFunctionNode* pFunc, char* pErrBuf, int32_t
bytes = TSDB_TABLE_FNAME_LEN - 1 + VARSTR_HEADER_SIZE; bytes = TSDB_TABLE_FNAME_LEN - 1 + VARSTR_HEADER_SIZE;
break; break;
case FUNCTION_TYPE_TIMEZONE: case FUNCTION_TYPE_TIMEZONE:
bytes = timeZoneStrLen(); bytes = TD_TIMEZONE_LEN + VARSTR_HEADER_SIZE;
break; break;
case FUNCTION_TYPE_IRATE_PARTIAL: case FUNCTION_TYPE_IRATE_PARTIAL:
bytes = getIrateInfoSize((pFunc->hasPk) ? pFunc->pkBytes : 0) + VARSTR_HEADER_SIZE; bytes = getIrateInfoSize((pFunc->hasPk) ? pFunc->pkBytes : 0) + VARSTR_HEADER_SIZE;

2
source/libs/parser/inc/parUtil.h
View File

@ -136,7 +136,7 @@ int32_t trimString(const char* src, int32_t len, char* dst, int32_t dlen);
int32_t getVnodeSysTableTargetName(int32_t acctId, SNode* pWhere, SName* pName); int32_t getVnodeSysTableTargetName(int32_t acctId, SNode* pWhere, SName* pName);
int32_t checkAndTrimValue(SToken* pToken, char* tmpTokenBuf, SMsgBuf* pMsgBuf, int8_t type); int32_t checkAndTrimValue(SToken* pToken, char* tmpTokenBuf, SMsgBuf* pMsgBuf, int8_t type);
int32_t parseTagValue(SMsgBuf* pMsgBuf, const char** pSql, uint8_t precision, SSchema* pTagSchema, SToken* pToken, int32_t parseTagValue(SMsgBuf* pMsgBuf, const char** pSql, uint8_t precision, SSchema* pTagSchema, SToken* pToken,
SArray* pTagName, SArray* pTagVals, STag** pTag, timezone_t tz); SArray* pTagName, SArray* pTagVals, STag** pTag, timezone_t tz, void *charsetCxt);
int32_t parseTbnameToken(SMsgBuf* pMsgBuf, char* tname, SToken* pToken, bool* pFoundCtbName); int32_t parseTbnameToken(SMsgBuf* pMsgBuf, char* tname, SToken* pToken, bool* pFoundCtbName);
int32_t buildCatalogReq(const SParseMetaCache* pMetaCache, SCatalogReq* pCatalogReq); int32_t buildCatalogReq(const SParseMetaCache* pMetaCache, SCatalogReq* pCatalogReq);

9
source/libs/parser/src/parAstCreater.c
View File

@ -421,6 +421,7 @@ SNode* createValueNode(SAstCreateContext* pCxt, int32_t dataType, const SToken*
} }
val->translate = false; val->translate = false;
val->tz = pCxt->pQueryCxt->timezone; val->tz = pCxt->pQueryCxt->timezone;
val->charsetCxt = pCxt->pQueryCxt->charsetCxt;
return (SNode*)val; return (SNode*)val;
_err: _err:
return NULL; return NULL;
@ -964,6 +965,8 @@ SNode* createOperatorNode(SAstCreateContext* pCxt, EOperatorType type, SNode* pL
op->opType = type; op->opType = type;
op->pLeft = pLeft; op->pLeft = pLeft;
op->pRight = pRight; op->pRight = pRight;
op->tz = pCxt->pQueryCxt->timezone;
op->charsetCxt = pCxt->pQueryCxt->charsetCxt;
return (SNode*)op; return (SNode*)op;
_err: _err:
nodesDestroyNode(pLeft); nodesDestroyNode(pLeft);
@ -1043,6 +1046,7 @@ SNode* createFunctionNode(SAstCreateContext* pCxt, const SToken* pFuncName, SNod
COPY_STRING_FORM_ID_TOKEN(func->functionName, pFuncName); COPY_STRING_FORM_ID_TOKEN(func->functionName, pFuncName);
func->pParameterList = pParameterList; func->pParameterList = pParameterList;
func->tz = pCxt->pQueryCxt->timezone; func->tz = pCxt->pQueryCxt->timezone;
func->charsetCxt = pCxt->pQueryCxt->charsetCxt;
return (SNode*)func; return (SNode*)func;
_err: _err:
nodesDestroyList(pParameterList); nodesDestroyList(pParameterList);
@ -1064,6 +1068,8 @@ SNode* createCastFunctionNode(SAstCreateContext* pCxt, SNode* pExpr, SDataType d
pCxt->errCode = nodesListMakeAppend(&func->pParameterList, pExpr); pCxt->errCode = nodesListMakeAppend(&func->pParameterList, pExpr);
CHECK_PARSER_STATUS(pCxt); CHECK_PARSER_STATUS(pCxt);
func->tz = pCxt->pQueryCxt->timezone; func->tz = pCxt->pQueryCxt->timezone;
func->charsetCxt = pCxt->pQueryCxt->charsetCxt;
return (SNode*)func; return (SNode*)func;
_err: _err:
nodesDestroyNode((SNode*)func); nodesDestroyNode((SNode*)func);
@ -1098,6 +1104,7 @@ SNode* createTrimFunctionNode(SAstCreateContext* pCxt, SNode* pExpr, ETrimType t
func->trimType = type; func->trimType = type;
pCxt->errCode = nodesListMakeAppend(&func->pParameterList, pExpr); pCxt->errCode = nodesListMakeAppend(&func->pParameterList, pExpr);
CHECK_PARSER_STATUS(pCxt); CHECK_PARSER_STATUS(pCxt);
func->charsetCxt = pCxt->pQueryCxt->charsetCxt;
return (SNode*)func; return (SNode*)func;
_err: _err:
nodesDestroyNode((SNode*)func); nodesDestroyNode((SNode*)func);
@ -1116,6 +1123,7 @@ SNode* createTrimFunctionNodeExt(SAstCreateContext* pCxt, SNode* pExpr, SNode* p
CHECK_PARSER_STATUS(pCxt); CHECK_PARSER_STATUS(pCxt);
pCxt->errCode = nodesListMakeAppend(&func->pParameterList, pExpr2); pCxt->errCode = nodesListMakeAppend(&func->pParameterList, pExpr2);
CHECK_PARSER_STATUS(pCxt); CHECK_PARSER_STATUS(pCxt);
func->charsetCxt = pCxt->pQueryCxt->charsetCxt;
return (SNode*)func; return (SNode*)func;
_err: _err:
nodesDestroyNode((SNode*)func); nodesDestroyNode((SNode*)func);
@ -1501,6 +1509,7 @@ SNode* createCaseWhenNode(SAstCreateContext* pCxt, SNode* pCase, SNodeList* pWhe
pCaseWhen->pWhenThenList = pWhenThenList; pCaseWhen->pWhenThenList = pWhenThenList;
pCaseWhen->pElse = pElse; pCaseWhen->pElse = pElse;
pCaseWhen->tz = pCxt->pQueryCxt->timezone; pCaseWhen->tz = pCxt->pQueryCxt->timezone;
pCaseWhen->charsetCxt = pCxt->pQueryCxt->charsetCxt;
return (SNode*)pCaseWhen; return (SNode*)pCaseWhen;
_err: _err:
nodesDestroyNode(pCase); nodesDestroyNode(pCase);

14
source/libs/parser/src/parInsertSml.c
View File

@ -98,7 +98,7 @@ end:
* @return int32_t * @return int32_t
*/ */
static int32_t smlBuildTagRow(SArray* cols, SBoundColInfo* tags, SSchema* pSchema, STag** ppTag, SArray** tagName, static int32_t smlBuildTagRow(SArray* cols, SBoundColInfo* tags, SSchema* pSchema, STag** ppTag, SArray** tagName,
SMsgBuf* msg) { SMsgBuf* msg, void* charsetCxt) {
SArray* pTagArray = taosArrayInit(tags->numOfBound, sizeof(STagVal)); SArray* pTagArray = taosArrayInit(tags->numOfBound, sizeof(STagVal));
if (!pTagArray) { if (!pTagArray) {
return terrno; return terrno;
@ -142,7 +142,7 @@ static int32_t smlBuildTagRow(SArray* cols, SBoundColInfo* tags, SSchema* pSchem
code = terrno; code = terrno;
goto end; goto end;
} }
if (!taosMbsToUcs4(kv->value, kv->length, (TdUcs4*)(p), kv->length * TSDB_NCHAR_SIZE, &output)) { if (!taosMbsToUcs4(kv->value, kv->length, (TdUcs4*)(p), kv->length * TSDB_NCHAR_SIZE, &output, charsetCxt)) {
if (terrno == TAOS_SYSTEM_ERROR(E2BIG)) { if (terrno == TAOS_SYSTEM_ERROR(E2BIG)) {
taosMemoryFree(p); taosMemoryFree(p);
code = generateSyntaxErrMsg(msg, TSDB_CODE_PAR_VALUE_TOO_LONG, pTagSchema->name); code = generateSyntaxErrMsg(msg, TSDB_CODE_PAR_VALUE_TOO_LONG, pTagSchema->name);
@ -221,7 +221,7 @@ int32_t smlBuildRow(STableDataCxt* pTableCxt) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t smlBuildCol(STableDataCxt* pTableCxt, SSchema* schema, void* data, int32_t index) { int32_t smlBuildCol(STableDataCxt* pTableCxt, SSchema* schema, void* data, int32_t index, void* charsetCxt) {
int ret = TSDB_CODE_SUCCESS; int ret = TSDB_CODE_SUCCESS;
SSchema* pColSchema = schema + index; SSchema* pColSchema = schema + index;
SColVal* pVal = taosArrayGet(pTableCxt->pValues, index); SColVal* pVal = taosArrayGet(pTableCxt->pValues, index);
@ -256,7 +256,7 @@ int32_t smlBuildCol(STableDataCxt* pTableCxt, SSchema* schema, void* data, int32
ret = terrno; ret = terrno;
goto end; goto end;
} }
if (!taosMbsToUcs4(kv->value, kv->length, (TdUcs4*)pUcs4, size, &len)) { if (!taosMbsToUcs4(kv->value, kv->length, (TdUcs4*)pUcs4, size, &len, charsetCxt)) {
if (terrno == TAOS_SYSTEM_ERROR(E2BIG)) { if (terrno == TAOS_SYSTEM_ERROR(E2BIG)) {
taosMemoryFree(pUcs4); taosMemoryFree(pUcs4);
ret = TSDB_CODE_PAR_VALUE_TOO_LONG; ret = TSDB_CODE_PAR_VALUE_TOO_LONG;
@ -291,7 +291,7 @@ end:
int32_t smlBindData(SQuery* query, bool dataFormat, SArray* tags, SArray* colsSchema, SArray* cols, int32_t smlBindData(SQuery* query, bool dataFormat, SArray* tags, SArray* colsSchema, SArray* cols,
STableMeta* pTableMeta, char* tableName, const char* sTableName, int32_t sTableNameLen, int32_t ttl, STableMeta* pTableMeta, char* tableName, const char* sTableName, int32_t sTableNameLen, int32_t ttl,
char* msgBuf, int32_t msgBufLen) { char* msgBuf, int32_t msgBufLen, void* charsetCxt) {
SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen};
SSchema* pTagsSchema = getTableTagSchema(pTableMeta); SSchema* pTagsSchema = getTableTagSchema(pTableMeta);
@ -313,7 +313,7 @@ int32_t smlBindData(SQuery* query, bool dataFormat, SArray* tags, SArray* colsSc
STag* pTag = NULL; STag* pTag = NULL;
ret = smlBuildTagRow(tags, &bindTags, pTagsSchema, &pTag, &tagName, &pBuf); ret = smlBuildTagRow(tags, &bindTags, pTagsSchema, &pTag, &tagName, &pBuf, charsetCxt);
if (ret != TSDB_CODE_SUCCESS) { if (ret != TSDB_CODE_SUCCESS) {
goto end; goto end;
} }
@ -411,7 +411,7 @@ int32_t smlBindData(SQuery* query, bool dataFormat, SArray* tags, SArray* colsSc
ret = terrno; ret = terrno;
goto end; goto end;
} }
if (!taosMbsToUcs4(kv->value, kv->length, (TdUcs4*)pUcs4, pColSchema->bytes - VARSTR_HEADER_SIZE, &len)) { if (!taosMbsToUcs4(kv->value, kv->length, (TdUcs4*)pUcs4, pColSchema->bytes - VARSTR_HEADER_SIZE, &len, charsetCxt)) {
if (terrno == TAOS_SYSTEM_ERROR(E2BIG)) { if (terrno == TAOS_SYSTEM_ERROR(E2BIG)) {
uError("sml bind taosMbsToUcs4 error, kv length:%d, bytes:%d, kv->value:%s", (int)kv->length, uError("sml bind taosMbsToUcs4 error, kv length:%d, bytes:%d, kv->value:%s", (int)kv->length,
pColSchema->bytes, kv->value); pColSchema->bytes, kv->value);

18
source/libs/parser/src/parInsertSql.c
View File

@ -477,7 +477,7 @@ static int32_t parseVarbinary(SToken* pToken, uint8_t** pData, uint32_t* nData,
} }
static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema, int16_t timePrec, STagVal* val, static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema, int16_t timePrec, STagVal* val,
SMsgBuf* pMsgBuf, timezone_t tz) { SMsgBuf* pMsgBuf, timezone_t tz, void *charsetCxt) {
int64_t iv; int64_t iv;
uint64_t uv; uint64_t uv;
char* endptr = NULL; char* endptr = NULL;
@ -685,7 +685,7 @@ static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema,
if (p == NULL) { if (p == NULL) {
return terrno; return terrno;
} }
if (!taosMbsToUcs4(pToken->z, pToken->n, (TdUcs4*)(p), realLen, &output)) { if (!taosMbsToUcs4(pToken->z, pToken->n, (TdUcs4*)(p), realLen, &output, charsetCxt)) {
if (terrno == TAOS_SYSTEM_ERROR(E2BIG)) { if (terrno == TAOS_SYSTEM_ERROR(E2BIG)) {
taosMemoryFree(p); taosMemoryFree(p);
return generateSyntaxErrMsg(pMsgBuf, TSDB_CODE_PAR_VALUE_TOO_LONG, pSchema->name); return generateSyntaxErrMsg(pMsgBuf, TSDB_CODE_PAR_VALUE_TOO_LONG, pSchema->name);
@ -732,7 +732,7 @@ static int32_t parseBoundTagsClause(SInsertParseContext* pCxt, SVnodeModifyOpStm
} }
int32_t parseTagValue(SMsgBuf* pMsgBuf, const char** pSql, uint8_t precision, SSchema* pTagSchema, SToken* pToken, int32_t parseTagValue(SMsgBuf* pMsgBuf, const char** pSql, uint8_t precision, SSchema* pTagSchema, SToken* pToken,
SArray* pTagName, SArray* pTagVals, STag** pTag, timezone_t tz) { SArray* pTagName, SArray* pTagVals, STag** pTag, timezone_t tz, void *charsetCxt) {
bool isNull = isNullValue(pTagSchema->type, pToken); bool isNull = isNullValue(pTagSchema->type, pToken);
if (!isNull && pTagName) { if (!isNull && pTagName) {
if (NULL == taosArrayPush(pTagName, pTagSchema->name)) { if (NULL == taosArrayPush(pTagName, pTagSchema->name)) {
@ -748,14 +748,14 @@ int32_t parseTagValue(SMsgBuf* pMsgBuf, const char** pSql, uint8_t precision, SS
if (isNull) { if (isNull) {
return tTagNew(pTagVals, 1, true, pTag); return tTagNew(pTagVals, 1, true, pTag);
} else { } else {
return parseJsontoTagData(pToken->z, pTagVals, pTag, pMsgBuf); return parseJsontoTagData(pToken->z, pTagVals, pTag, pMsgBuf, charsetCxt);
} }
} }
if (isNull) return 0; if (isNull) return 0;
STagVal val = {0}; STagVal val = {0};
int32_t code = parseTagToken(pSql, pToken, pTagSchema, precision, &val, pMsgBuf, tz); int32_t code = parseTagToken(pSql, pToken, pTagSchema, precision, &val, pMsgBuf, tz, charsetCxt);
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
if (NULL == taosArrayPush(pTagVals, &val)) { if (NULL == taosArrayPush(pTagVals, &val)) {
code = terrno; code = terrno;
@ -978,7 +978,7 @@ static int32_t parseTagsClauseImpl(SInsertParseContext* pCxt, SVnodeModifyOpStmt
code = buildSyntaxErrMsg(&pCxt->msg, "not expected tags values ", token.z); code = buildSyntaxErrMsg(&pCxt->msg, "not expected tags values ", token.z);
} }
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = parseTagValue(&pCxt->msg, &pStmt->pSql, precision, pTagSchema, &token, pTagName, pTagVals, &pTag, pCxt->pComCxt->timezone); code = parseTagValue(&pCxt->msg, &pStmt->pSql, precision, pTagSchema, &token, pTagName, pTagVals, &pTag, pCxt->pComCxt->timezone, pCxt->pComCxt->charsetCxt);
} }
} }
@ -1620,7 +1620,7 @@ static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql,
if (NULL == pUcs4) { if (NULL == pUcs4) {
return terrno; return terrno;
} }
if (!taosMbsToUcs4(pToken->z, pToken->n, (TdUcs4*)pUcs4, realLen, &len)) { if (!taosMbsToUcs4(pToken->z, pToken->n, (TdUcs4*)pUcs4, realLen, &len, pCxt->pComCxt->charsetCxt)) {
taosMemoryFree(pUcs4); taosMemoryFree(pUcs4);
if (terrno == TAOS_SYSTEM_ERROR(E2BIG)) { if (terrno == TAOS_SYSTEM_ERROR(E2BIG)) {
return generateSyntaxErrMsg(&pCxt->msg, TSDB_CODE_PAR_VALUE_TOO_LONG, pSchema->name); return generateSyntaxErrMsg(&pCxt->msg, TSDB_CODE_PAR_VALUE_TOO_LONG, pSchema->name);
@ -1788,7 +1788,7 @@ static int32_t processCtbTagsAfterCtbName(SInsertParseContext* pCxt, SVnodeModif
if (code == TSDB_CODE_SUCCESS) { if (code == TSDB_CODE_SUCCESS) {
code = parseTagValue(&pCxt->msg, NULL, precision, pTagSchema, pTagToken, pStbRowsCxt->aTagNames, code = parseTagValue(&pCxt->msg, NULL, precision, pTagSchema, pTagToken, pStbRowsCxt->aTagNames,
pStbRowsCxt->aTagVals, &pStbRowsCxt->pTag, pCxt->pComCxt->timezone); pStbRowsCxt->aTagVals, &pStbRowsCxt->pTag, pCxt->pComCxt->timezone, pCxt->pComCxt->charsetCxt);
} }
} }
if (code == TSDB_CODE_SUCCESS && !pStbRowsCxt->isJsonTag) { if (code == TSDB_CODE_SUCCESS && !pStbRowsCxt->isJsonTag) {
@ -1858,7 +1858,7 @@ static int32_t doGetStbRowValues(SInsertParseContext* pCxt, SVnodeModifyOpStmt*
} }
if (code == TSDB_CODE_SUCCESS) { if (code == TSDB_CODE_SUCCESS) {
code = parseTagValue(&pCxt->msg, ppSql, precision, (SSchema*)pTagSchema, pToken, pTagNames, pTagVals, code = parseTagValue(&pCxt->msg, ppSql, precision, (SSchema*)pTagSchema, pToken, pTagNames, pTagVals,
&pStbRowsCxt->pTag, pCxt->pComCxt->timezone); &pStbRowsCxt->pTag, pCxt->pComCxt->timezone, pCxt->pComCxt->charsetCxt);
} }
} }
} else if (pCols->pColIndex[i] == tbnameIdx) { } else if (pCols->pColIndex[i] == tbnameIdx) {

48
source/libs/parser/src/parInsertStmt.c
View File

@ -123,7 +123,7 @@ int32_t qBuildStmtOutput(SQuery* pQuery, SHashObj* pVgHash, SHashObj* pBlockHash
} }
int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, const char* sTableName, char* tName, int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, const char* sTableName, char* tName,
TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen) { TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen, void* charsetCxt) {
STableDataCxt* pDataBlock = (STableDataCxt*)pBlock; STableDataCxt* pDataBlock = (STableDataCxt*)pBlock;
SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen};
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
@ -183,7 +183,7 @@ int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, const ch
goto end; goto end;
} }
memcpy(tmp, bind[c].buffer, colLen); memcpy(tmp, bind[c].buffer, colLen);
code = parseJsontoTagData(tmp, pTagArray, &pTag, &pBuf); code = parseJsontoTagData(tmp, pTagArray, &pTag, &pBuf, charsetCxt);
taosMemoryFree(tmp); taosMemoryFree(tmp);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
goto end; goto end;
@ -208,7 +208,7 @@ int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, const ch
code = terrno; code = terrno;
goto end; goto end;
} }
if (!taosMbsToUcs4(bind[c].buffer, colLen, (TdUcs4*)(p), colLen * TSDB_NCHAR_SIZE, &output)) { if (!taosMbsToUcs4(bind[c].buffer, colLen, (TdUcs4*)(p), colLen * TSDB_NCHAR_SIZE, &output, charsetCxt)) {
if (terrno == TAOS_SYSTEM_ERROR(E2BIG)) { if (terrno == TAOS_SYSTEM_ERROR(E2BIG)) {
taosMemoryFree(p); taosMemoryFree(p);
code = generateSyntaxErrMsg(&pBuf, TSDB_CODE_PAR_VALUE_TOO_LONG, pTagSchema->name); code = generateSyntaxErrMsg(&pBuf, TSDB_CODE_PAR_VALUE_TOO_LONG, pTagSchema->name);
@ -266,7 +266,7 @@ end:
return code; return code;
} }
int32_t convertStmtNcharCol(SMsgBuf* pMsgBuf, SSchema* pSchema, TAOS_MULTI_BIND* src, TAOS_MULTI_BIND* dst) { int32_t convertStmtNcharCol(SMsgBuf* pMsgBuf, SSchema* pSchema, TAOS_MULTI_BIND* src, TAOS_MULTI_BIND* dst, void* charsetCxt) {
int32_t output = 0; int32_t output = 0;
int32_t newBuflen = (pSchema->bytes - VARSTR_HEADER_SIZE) * src->num; int32_t newBuflen = (pSchema->bytes - VARSTR_HEADER_SIZE) * src->num;
if (dst->buffer_length < newBuflen) { if (dst->buffer_length < newBuflen) {
@ -292,7 +292,7 @@ int32_t convertStmtNcharCol(SMsgBuf* pMsgBuf, SSchema* pSchema, TAOS_MULTI_BIND*
} }
if (!taosMbsToUcs4(((char*)src->buffer) + src->buffer_length * i, src->length[i], if (!taosMbsToUcs4(((char*)src->buffer) + src->buffer_length * i, src->length[i],
(TdUcs4*)(((char*)dst->buffer) + dst->buffer_length * i), dst->buffer_length, &output)) { (TdUcs4*)(((char*)dst->buffer) + dst->buffer_length * i), dst->buffer_length, &output, charsetCxt)) {
if (terrno == TAOS_SYSTEM_ERROR(E2BIG)) { if (terrno == TAOS_SYSTEM_ERROR(E2BIG)) {
return generateSyntaxErrMsg(pMsgBuf, TSDB_CODE_PAR_VALUE_TOO_LONG, pSchema->name); return generateSyntaxErrMsg(pMsgBuf, TSDB_CODE_PAR_VALUE_TOO_LONG, pSchema->name);
} }
@ -312,7 +312,7 @@ int32_t convertStmtNcharCol(SMsgBuf* pMsgBuf, SSchema* pSchema, TAOS_MULTI_BIND*
} }
int32_t qBindStmtStbColsValue(void* pBlock, SArray* pCols, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen, int32_t qBindStmtStbColsValue(void* pBlock, SArray* pCols, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen,
STSchema** pTSchema, SBindInfo* pBindInfos) { STSchema** pTSchema, SBindInfo* pBindInfos, void* charsetCxt) {
STableDataCxt* pDataBlock = (STableDataCxt*)pBlock; STableDataCxt* pDataBlock = (STableDataCxt*)pBlock;
SSchema* pSchema = getTableColumnSchema(pDataBlock->pMeta); SSchema* pSchema = getTableColumnSchema(pDataBlock->pMeta);
SBoundColInfo* boundInfo = &pDataBlock->boundColsInfo; SBoundColInfo* boundInfo = &pDataBlock->boundColsInfo;
@ -349,7 +349,7 @@ int32_t qBindStmtStbColsValue(void* pBlock, SArray* pCols, TAOS_MULTI_BIND* bind
} }
if (TSDB_DATA_TYPE_NCHAR == pColSchema->type) { if (TSDB_DATA_TYPE_NCHAR == pColSchema->type) {
code = convertStmtNcharCol(&pBuf, pColSchema, bind + c, &ncharBind); code = convertStmtNcharCol(&pBuf, pColSchema, bind + c, &ncharBind, charsetCxt);
if (code) { if (code) {
goto _return; goto _return;
} }
@ -380,7 +380,7 @@ _return:
return code; return code;
} }
int32_t qBindStmtColsValue(void* pBlock, SArray* pCols, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen) { int32_t qBindStmtColsValue(void* pBlock, SArray* pCols, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen, void* charsetCxt) {
STableDataCxt* pDataBlock = (STableDataCxt*)pBlock; STableDataCxt* pDataBlock = (STableDataCxt*)pBlock;
SSchema* pSchema = getTableColumnSchema(pDataBlock->pMeta); SSchema* pSchema = getTableColumnSchema(pDataBlock->pMeta);
SBoundColInfo* boundInfo = &pDataBlock->boundColsInfo; SBoundColInfo* boundInfo = &pDataBlock->boundColsInfo;
@ -406,7 +406,7 @@ int32_t qBindStmtColsValue(void* pBlock, SArray* pCols, TAOS_MULTI_BIND* bind, c
} }
if (TSDB_DATA_TYPE_NCHAR == pColSchema->type) { if (TSDB_DATA_TYPE_NCHAR == pColSchema->type) {
code = convertStmtNcharCol(&pBuf, pColSchema, bind + c, &ncharBind); code = convertStmtNcharCol(&pBuf, pColSchema, bind + c, &ncharBind, charsetCxt);
if (code) { if (code) {
goto _return; goto _return;
} }
@ -434,7 +434,7 @@ _return:
} }
int32_t qBindStmtSingleColValue(void* pBlock, SArray* pCols, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen, int32_t qBindStmtSingleColValue(void* pBlock, SArray* pCols, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen,
int32_t colIdx, int32_t rowNum) { int32_t colIdx, int32_t rowNum, void* charsetCxt) {
STableDataCxt* pDataBlock = (STableDataCxt*)pBlock; STableDataCxt* pDataBlock = (STableDataCxt*)pBlock;
SSchema* pSchema = getTableColumnSchema(pDataBlock->pMeta); SSchema* pSchema = getTableColumnSchema(pDataBlock->pMeta);
SBoundColInfo* boundInfo = &pDataBlock->boundColsInfo; SBoundColInfo* boundInfo = &pDataBlock->boundColsInfo;
@ -459,7 +459,7 @@ int32_t qBindStmtSingleColValue(void* pBlock, SArray* pCols, TAOS_MULTI_BIND* bi
} }
if (TSDB_DATA_TYPE_NCHAR == pColSchema->type) { if (TSDB_DATA_TYPE_NCHAR == pColSchema->type) {
code = convertStmtNcharCol(&pBuf, pColSchema, bind, &ncharBind); code = convertStmtNcharCol(&pBuf, pColSchema, bind, &ncharBind, charsetCxt);
if (code) { if (code) {
goto _return; goto _return;
} }
@ -483,7 +483,7 @@ _return:
} }
int32_t qBindStmtTagsValue2(void* pBlock, void* boundTags, int64_t suid, const char* sTableName, char* tName, int32_t qBindStmtTagsValue2(void* pBlock, void* boundTags, int64_t suid, const char* sTableName, char* tName,
TAOS_STMT2_BIND* bind, char* msgBuf, int32_t msgBufLen) { TAOS_STMT2_BIND* bind, char* msgBuf, int32_t msgBufLen, void* charsetCxt) {
STableDataCxt* pDataBlock = (STableDataCxt*)pBlock; STableDataCxt* pDataBlock = (STableDataCxt*)pBlock;
SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen};
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
@ -543,7 +543,7 @@ int32_t qBindStmtTagsValue2(void* pBlock, void* boundTags, int64_t suid, const c
goto end; goto end;
} }
memcpy(tmp, bind[c].buffer, colLen); memcpy(tmp, bind[c].buffer, colLen);
code = parseJsontoTagData(tmp, pTagArray, &pTag, &pBuf); code = parseJsontoTagData(tmp, pTagArray, &pTag, &pBuf, charsetCxt);
taosMemoryFree(tmp); taosMemoryFree(tmp);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
goto end; goto end;
@ -568,7 +568,7 @@ int32_t qBindStmtTagsValue2(void* pBlock, void* boundTags, int64_t suid, const c
code = terrno; code = terrno;
goto end; goto end;
} }
if (!taosMbsToUcs4(bind[c].buffer, colLen, (TdUcs4*)(p), colLen * TSDB_NCHAR_SIZE, &output)) { if (!taosMbsToUcs4(bind[c].buffer, colLen, (TdUcs4*)(p), colLen * TSDB_NCHAR_SIZE, &output, charsetCxt)) {
if (terrno == TAOS_SYSTEM_ERROR(E2BIG)) { if (terrno == TAOS_SYSTEM_ERROR(E2BIG)) {
taosMemoryFree(p); taosMemoryFree(p);
code = generateSyntaxErrMsg(&pBuf, TSDB_CODE_PAR_VALUE_TOO_LONG, pTagSchema->name); code = generateSyntaxErrMsg(&pBuf, TSDB_CODE_PAR_VALUE_TOO_LONG, pTagSchema->name);
@ -626,7 +626,7 @@ end:
return code; return code;
} }
static int32_t convertStmtStbNcharCol2(SMsgBuf* pMsgBuf, SSchema* pSchema, TAOS_STMT2_BIND* src, TAOS_STMT2_BIND* dst) { static int32_t convertStmtStbNcharCol2(SMsgBuf* pMsgBuf, SSchema* pSchema, TAOS_STMT2_BIND* src, TAOS_STMT2_BIND* dst, void *charsetCxt) {
int32_t output = 0; int32_t output = 0;
const int32_t max_buf_len = pSchema->bytes - VARSTR_HEADER_SIZE; const int32_t max_buf_len = pSchema->bytes - VARSTR_HEADER_SIZE;
@ -648,7 +648,7 @@ static int32_t convertStmtStbNcharCol2(SMsgBuf* pMsgBuf, SSchema* pSchema, TAOS_
continue; continue;
} }
if (!taosMbsToUcs4(src_buf, src->length[i], (TdUcs4*)dst_buf, max_buf_len, &output)) { if (!taosMbsToUcs4(src_buf, src->length[i], (TdUcs4*)dst_buf, max_buf_len, &output, charsetCxt)) {
if (terrno == TAOS_SYSTEM_ERROR(E2BIG)) { if (terrno == TAOS_SYSTEM_ERROR(E2BIG)) {
return generateSyntaxErrMsg(pMsgBuf, TSDB_CODE_PAR_VALUE_TOO_LONG, pSchema->name); return generateSyntaxErrMsg(pMsgBuf, TSDB_CODE_PAR_VALUE_TOO_LONG, pSchema->name);
} }
@ -670,7 +670,7 @@ static int32_t convertStmtStbNcharCol2(SMsgBuf* pMsgBuf, SSchema* pSchema, TAOS_
} }
int32_t qBindStmtStbColsValue2(void* pBlock, SArray* pCols, TAOS_STMT2_BIND* bind, char* msgBuf, int32_t msgBufLen, int32_t qBindStmtStbColsValue2(void* pBlock, SArray* pCols, TAOS_STMT2_BIND* bind, char* msgBuf, int32_t msgBufLen,
STSchema** pTSchema, SBindInfo2* pBindInfos) { STSchema** pTSchema, SBindInfo2* pBindInfos, void *charsetCxt) {
STableDataCxt* pDataBlock = (STableDataCxt*)pBlock; STableDataCxt* pDataBlock = (STableDataCxt*)pBlock;
SSchema* pSchema = getTableColumnSchema(pDataBlock->pMeta); SSchema* pSchema = getTableColumnSchema(pDataBlock->pMeta);
SBoundColInfo* boundInfo = &pDataBlock->boundColsInfo; SBoundColInfo* boundInfo = &pDataBlock->boundColsInfo;
@ -720,7 +720,7 @@ int32_t qBindStmtStbColsValue2(void* pBlock, SArray* pCols, TAOS_STMT2_BIND* bin
} }
if (TSDB_DATA_TYPE_NCHAR == pColSchema->type) { if (TSDB_DATA_TYPE_NCHAR == pColSchema->type) {
code = convertStmtStbNcharCol2(&pBuf, pColSchema, bind + c, &ncharBind); code = convertStmtStbNcharCol2(&pBuf, pColSchema, bind + c, &ncharBind, charsetCxt);
if (code) { if (code) {
goto _return; goto _return;
} }
@ -755,7 +755,7 @@ _return:
return code; return code;
} }
static int32_t convertStmtNcharCol2(SMsgBuf* pMsgBuf, SSchema* pSchema, TAOS_STMT2_BIND* src, TAOS_STMT2_BIND* dst) { static int32_t convertStmtNcharCol2(SMsgBuf* pMsgBuf, SSchema* pSchema, TAOS_STMT2_BIND* src, TAOS_STMT2_BIND* dst, void *charsetCxt) {
int32_t output = 0; int32_t output = 0;
const int32_t max_buf_len = pSchema->bytes - VARSTR_HEADER_SIZE; const int32_t max_buf_len = pSchema->bytes - VARSTR_HEADER_SIZE;
@ -785,7 +785,7 @@ static int32_t convertStmtNcharCol2(SMsgBuf* pMsgBuf, SSchema* pSchema, TAOS_STM
/*if (!taosMbsToUcs4(((char*)src->buffer) + src->buffer_length * i, src->length[i], /*if (!taosMbsToUcs4(((char*)src->buffer) + src->buffer_length * i, src->length[i],
(TdUcs4*)(((char*)dst->buffer) + dst->buffer_length * i), dst->buffer_length, &output)) {*/ (TdUcs4*)(((char*)dst->buffer) + dst->buffer_length * i), dst->buffer_length, &output)) {*/
if (!taosMbsToUcs4(src_buf, src->length[i], (TdUcs4*)dst_buf, max_buf_len, &output)) { if (!taosMbsToUcs4(src_buf, src->length[i], (TdUcs4*)dst_buf, max_buf_len, &output, charsetCxt)) {
if (terrno == TAOS_SYSTEM_ERROR(E2BIG)) { if (terrno == TAOS_SYSTEM_ERROR(E2BIG)) {
return generateSyntaxErrMsg(pMsgBuf, TSDB_CODE_PAR_VALUE_TOO_LONG, pSchema->name); return generateSyntaxErrMsg(pMsgBuf, TSDB_CODE_PAR_VALUE_TOO_LONG, pSchema->name);
} }
@ -806,7 +806,7 @@ static int32_t convertStmtNcharCol2(SMsgBuf* pMsgBuf, SSchema* pSchema, TAOS_STM
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t qBindStmtColsValue2(void* pBlock, SArray* pCols, TAOS_STMT2_BIND* bind, char* msgBuf, int32_t msgBufLen) { int32_t qBindStmtColsValue2(void* pBlock, SArray* pCols, TAOS_STMT2_BIND* bind, char* msgBuf, int32_t msgBufLen, void *charsetCxt) {
STableDataCxt* pDataBlock = (STableDataCxt*)pBlock; STableDataCxt* pDataBlock = (STableDataCxt*)pBlock;
SSchema* pSchema = getTableColumnSchema(pDataBlock->pMeta); SSchema* pSchema = getTableColumnSchema(pDataBlock->pMeta);
SBoundColInfo* boundInfo = &pDataBlock->boundColsInfo; SBoundColInfo* boundInfo = &pDataBlock->boundColsInfo;
@ -836,7 +836,7 @@ int32_t qBindStmtColsValue2(void* pBlock, SArray* pCols, TAOS_STMT2_BIND* bind,
} }
if (TSDB_DATA_TYPE_NCHAR == pColSchema->type) { if (TSDB_DATA_TYPE_NCHAR == pColSchema->type) {
code = convertStmtNcharCol2(&pBuf, pColSchema, bind + c, &ncharBind); code = convertStmtNcharCol2(&pBuf, pColSchema, bind + c, &ncharBind, charsetCxt);
if (code) { if (code) {
goto _return; goto _return;
} }
@ -864,7 +864,7 @@ _return:
} }
int32_t qBindStmtSingleColValue2(void* pBlock, SArray* pCols, TAOS_STMT2_BIND* bind, char* msgBuf, int32_t msgBufLen, int32_t qBindStmtSingleColValue2(void* pBlock, SArray* pCols, TAOS_STMT2_BIND* bind, char* msgBuf, int32_t msgBufLen,
int32_t colIdx, int32_t rowNum) { int32_t colIdx, int32_t rowNum, void *charsetCxt) {
STableDataCxt* pDataBlock = (STableDataCxt*)pBlock; STableDataCxt* pDataBlock = (STableDataCxt*)pBlock;
SSchema* pSchema = getTableColumnSchema(pDataBlock->pMeta); SSchema* pSchema = getTableColumnSchema(pDataBlock->pMeta);
SBoundColInfo* boundInfo = &pDataBlock->boundColsInfo; SBoundColInfo* boundInfo = &pDataBlock->boundColsInfo;
@ -889,7 +889,7 @@ int32_t qBindStmtSingleColValue2(void* pBlock, SArray* pCols, TAOS_STMT2_BIND* b
} }
if (TSDB_DATA_TYPE_NCHAR == pColSchema->type) { if (TSDB_DATA_TYPE_NCHAR == pColSchema->type) {
code = convertStmtNcharCol2(&pBuf, pColSchema, bind, &ncharBind); code = convertStmtNcharCol2(&pBuf, pColSchema, bind, &ncharBind, charsetCxt);
if (code) { if (code) {
goto _return; goto _return;
} }

14
source/libs/parser/src/parTranslater.c
View File

@ -2148,7 +2148,7 @@ static EDealRes translateNormalValue(STranslateContext* pCxt, SValueNode* pVal,
int32_t len = 0; int32_t len = 0;
if (!taosMbsToUcs4(pVal->literal, strlen(pVal->literal), (TdUcs4*)varDataVal(pVal->datum.p), if (!taosMbsToUcs4(pVal->literal, strlen(pVal->literal), (TdUcs4*)varDataVal(pVal->datum.p),
targetDt.bytes - VARSTR_HEADER_SIZE, &len)) { targetDt.bytes - VARSTR_HEADER_SIZE, &len, pCxt->pParseCxt->charsetCxt)) {
return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, pVal->literal); return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, pVal->literal);
} }
varDataSetLen(pVal->datum.p, len); varDataSetLen(pVal->datum.p, len);
@ -14169,7 +14169,7 @@ static int32_t buildKVRowForBindTags(STranslateContext* pCxt, SCreateSubTableCla
if (pSchema->type == TSDB_DATA_TYPE_JSON) { if (pSchema->type == TSDB_DATA_TYPE_JSON) {
isJson = true; isJson = true;
} }
code = parseTagValue(&pCxt->msgBuf, &tagStr, precision, pSchema, &token, tagName, pTagArray, ppTag, pCxt->pParseCxt->timezone); code = parseTagValue(&pCxt->msgBuf, &tagStr, precision, pSchema, &token, tagName, pTagArray, ppTag, pCxt->pParseCxt->timezone, pCxt->pParseCxt->charsetCxt);
} }
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
@ -14230,7 +14230,7 @@ static int32_t buildKVRowForAllTags(STranslateContext* pCxt, SCreateSubTableClau
if (pTagSchema->type == TSDB_DATA_TYPE_JSON) { if (pTagSchema->type == TSDB_DATA_TYPE_JSON) {
isJson = true; isJson = true;
} }
code = parseTagValue(&pCxt->msgBuf, &tagStr, precision, pTagSchema, &token, tagName, pTagArray, ppTag, pCxt->pParseCxt->timezone); code = parseTagValue(&pCxt->msgBuf, &tagStr, precision, pTagSchema, &token, tagName, pTagArray, ppTag, pCxt->pParseCxt->timezone, pCxt->pParseCxt->charsetCxt);
} }
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
@ -14426,7 +14426,7 @@ static int32_t fillVgroupInfo(SParseContext* pParseCxt, const SName* pName, SVgr
return code; return code;
} }
static int32_t parseOneStbRow(SMsgBuf* pMsgBuf, SParseFileContext* pParFileCxt, timezone_t tz) { static int32_t parseOneStbRow(SMsgBuf* pMsgBuf, SParseFileContext* pParFileCxt, timezone_t tz, void *charsetCxt) {
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
int sz = taosArrayGetSize(pParFileCxt->aTagIndexs); int sz = taosArrayGetSize(pParFileCxt->aTagIndexs);
int32_t numOfTags = getNumOfTags(pParFileCxt->pStbMeta); int32_t numOfTags = getNumOfTags(pParFileCxt->pStbMeta);
@ -14458,7 +14458,7 @@ static int32_t parseOneStbRow(SMsgBuf* pMsgBuf, SParseFileContext* pParFileCxt,
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
SArray* aTagNames = pParFileCxt->tagNameFilled ? NULL : pParFileCxt->aTagNames; SArray* aTagNames = pParFileCxt->tagNameFilled ? NULL : pParFileCxt->aTagNames;
code = parseTagValue(pMsgBuf, &pParFileCxt->pSql, precision, (SSchema*)pTagSchema, &token, aTagNames, code = parseTagValue(pMsgBuf, &pParFileCxt->pSql, precision, (SSchema*)pTagSchema, &token, aTagNames,
pParFileCxt->aTagVals, &pParFileCxt->pTag, tz); pParFileCxt->aTagVals, &pParFileCxt->pTag, tz, charsetCxt);
} }
} else { } else {
// parse tbname // parse tbname
@ -14524,7 +14524,7 @@ static int32_t parseCsvFile(SMsgBuf* pMsgBuf, SParseContext* pParseCxt, SParseFi
(void)strtolower(pLine, pLine); (void)strtolower(pLine, pLine);
pParFileCxt->pSql = pLine; pParFileCxt->pSql = pLine;
code = parseOneStbRow(pMsgBuf, pParFileCxt, pParseCxt->timezone); code = parseOneStbRow(pMsgBuf, pParFileCxt, pParseCxt->timezone, pParseCxt->charsetCxt);
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = fillVgroupInfo(pParseCxt, &pParFileCxt->ctbName, &pParFileCxt->vg); code = fillVgroupInfo(pParseCxt, &pParFileCxt->ctbName, &pParFileCxt->vg);
@ -15225,7 +15225,7 @@ static int32_t buildUpdateTagValReq(STranslateContext* pCxt, SAlterTableStmt* pS
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = parseTagValue(&pCxt->msgBuf, &tagStr, pTableMeta->tableInfo.precision, pSchema, &token, NULL, code = parseTagValue(&pCxt->msgBuf, &tagStr, pTableMeta->tableInfo.precision, pSchema, &token, NULL,
pReq->pTagArray, &pTag, pCxt->pParseCxt->timezone); pReq->pTagArray, &pTag, pCxt->pParseCxt->timezone, pCxt->pParseCxt->charsetCxt);
if (pSchema->type == TSDB_DATA_TYPE_JSON && token.type == TK_NULL && code == TSDB_CODE_SUCCESS) { if (pSchema->type == TSDB_DATA_TYPE_JSON && token.type == TK_NULL && code == TSDB_CODE_SUCCESS) {
pReq->tagFree = true; pReq->tagFree = true;
} }

4
source/libs/parser/src/parUtil.c
View File

@ -402,7 +402,7 @@ static bool isValidateTag(char* input) {
return true; return true;
} }
int32_t parseJsontoTagData(const char* json, SArray* pTagVals, STag** ppTag, void* pMsgBuf) { int32_t parseJsontoTagData(const char* json, SArray* pTagVals, STag** ppTag, void* pMsgBuf, void *charsetCxt) {
int32_t retCode = TSDB_CODE_SUCCESS; int32_t retCode = TSDB_CODE_SUCCESS;
cJSON* root = NULL; cJSON* root = NULL;
SHashObj* keyHash = NULL; SHashObj* keyHash = NULL;
@ -471,7 +471,7 @@ int32_t parseJsontoTagData(const char* json, SArray* pTagVals, STag** ppTag, voi
goto end; goto end;
} }
val.type = TSDB_DATA_TYPE_NCHAR; val.type = TSDB_DATA_TYPE_NCHAR;
if (valLen > 0 && !taosMbsToUcs4(jsonValue, valLen, (TdUcs4*)tmp, (int32_t)(valLen * TSDB_NCHAR_SIZE), &valLen)) { if (valLen > 0 && !taosMbsToUcs4(jsonValue, valLen, (TdUcs4*)tmp, (int32_t)(valLen * TSDB_NCHAR_SIZE), &valLen, charsetCxt)) {
uError("charset:%s to %s. val:%s, errno:%s, convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, jsonValue, uError("charset:%s to %s. val:%s, errno:%s, convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, jsonValue,
strerror(terrno)); strerror(terrno));
retCode = buildSyntaxErrMsg(pMsgBuf, "charset convert json error", jsonValue); retCode = buildSyntaxErrMsg(pMsgBuf, "charset convert json error", jsonValue);

20
source/libs/parser/src/parser.c
View File

@ -153,7 +153,7 @@ static int32_t parseSqlSyntax(SParseContext* pCxt, SQuery** pQuery, SParseMetaCa
return code; return code;
} }
static int32_t setValueByBindParam(SValueNode* pVal, TAOS_MULTI_BIND* pParam) { static int32_t setValueByBindParam(SValueNode* pVal, TAOS_MULTI_BIND* pParam, void *charsetCxt) {
if (!pParam || IS_NULL_TYPE(pParam->buffer_type)) { if (!pParam || IS_NULL_TYPE(pParam->buffer_type)) {
return TSDB_CODE_APP_ERROR; return TSDB_CODE_APP_ERROR;
} }
@ -200,7 +200,7 @@ static int32_t setValueByBindParam(SValueNode* pVal, TAOS_MULTI_BIND* pParam) {
int32_t output = 0; int32_t output = 0;
if (!taosMbsToUcs4(pParam->buffer, inputSize, (TdUcs4*)varDataVal(pVal->datum.p), pVal->node.resType.bytes, if (!taosMbsToUcs4(pParam->buffer, inputSize, (TdUcs4*)varDataVal(pVal->datum.p), pVal->node.resType.bytes,
&output)) { &output, charsetCxt)) {
return terrno; return terrno;
} }
varDataSetLen(pVal->datum.p, output); varDataSetLen(pVal->datum.p, output);
@ -417,19 +417,19 @@ int32_t qInitKeywordsTable() { return taosInitKeywordsTable(); }
void qCleanupKeywordsTable() { taosCleanupKeywordsTable(); } void qCleanupKeywordsTable() { taosCleanupKeywordsTable(); }
int32_t qStmtBindParams(SQuery* pQuery, TAOS_MULTI_BIND* pParams, int32_t colIdx) { int32_t qStmtBindParams(SQuery* pQuery, TAOS_MULTI_BIND* pParams, int32_t colIdx, void *charsetCxt) {
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
if (colIdx < 0) { if (colIdx < 0) {
int32_t size = taosArrayGetSize(pQuery->pPlaceholderValues); int32_t size = taosArrayGetSize(pQuery->pPlaceholderValues);
for (int32_t i = 0; i < size; ++i) { for (int32_t i = 0; i < size; ++i) {
code = setValueByBindParam((SValueNode*)taosArrayGetP(pQuery->pPlaceholderValues, i), pParams + i); code = setValueByBindParam((SValueNode*)taosArrayGetP(pQuery->pPlaceholderValues, i), pParams + i, charsetCxt);
if (TSDB_CODE_SUCCESS != code) { if (TSDB_CODE_SUCCESS != code) {
return code; return code;
} }
} }
} else { } else {
code = setValueByBindParam((SValueNode*)taosArrayGetP(pQuery->pPlaceholderValues, colIdx), pParams); code = setValueByBindParam((SValueNode*)taosArrayGetP(pQuery->pPlaceholderValues, colIdx), pParams, charsetCxt);
} }
if (TSDB_CODE_SUCCESS == code && (colIdx < 0 || colIdx + 1 == pQuery->placeholderNum)) { if (TSDB_CODE_SUCCESS == code && (colIdx < 0 || colIdx + 1 == pQuery->placeholderNum)) {
@ -443,7 +443,7 @@ int32_t qStmtBindParams(SQuery* pQuery, TAOS_MULTI_BIND* pParams, int32_t colIdx
return code; return code;
} }
static int32_t setValueByBindParam2(SValueNode* pVal, TAOS_STMT2_BIND* pParam) { static int32_t setValueByBindParam2(SValueNode* pVal, TAOS_STMT2_BIND* pParam, void* charsetCxt) {
if (!pParam || IS_NULL_TYPE(pParam->buffer_type)) { if (!pParam || IS_NULL_TYPE(pParam->buffer_type)) {
return TSDB_CODE_APP_ERROR; return TSDB_CODE_APP_ERROR;
} }
@ -490,7 +490,7 @@ static int32_t setValueByBindParam2(SValueNode* pVal, TAOS_STMT2_BIND* pParam) {
int32_t output = 0; int32_t output = 0;
if (!taosMbsToUcs4(pParam->buffer, inputSize, (TdUcs4*)varDataVal(pVal->datum.p), pVal->node.resType.bytes, if (!taosMbsToUcs4(pParam->buffer, inputSize, (TdUcs4*)varDataVal(pVal->datum.p), pVal->node.resType.bytes,
&output)) { &output, charsetCxt)) {
return terrno; return terrno;
} }
varDataSetLen(pVal->datum.p, output); varDataSetLen(pVal->datum.p, output);
@ -509,19 +509,19 @@ static int32_t setValueByBindParam2(SValueNode* pVal, TAOS_STMT2_BIND* pParam) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t qStmtBindParams2(SQuery* pQuery, TAOS_STMT2_BIND* pParams, int32_t colIdx) { int32_t qStmtBindParams2(SQuery* pQuery, TAOS_STMT2_BIND* pParams, int32_t colIdx, void* charsetCxt) {
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
if (colIdx < 0) { if (colIdx < 0) {
int32_t size = taosArrayGetSize(pQuery->pPlaceholderValues); int32_t size = taosArrayGetSize(pQuery->pPlaceholderValues);
for (int32_t i = 0; i < size; ++i) { for (int32_t i = 0; i < size; ++i) {
code = setValueByBindParam2((SValueNode*)taosArrayGetP(pQuery->pPlaceholderValues, i), pParams + i); code = setValueByBindParam2((SValueNode*)taosArrayGetP(pQuery->pPlaceholderValues, i), pParams + i, charsetCxt);
if (TSDB_CODE_SUCCESS != code) { if (TSDB_CODE_SUCCESS != code) {
return code; return code;
} }
} }
} else { } else {
code = setValueByBindParam2((SValueNode*)taosArrayGetP(pQuery->pPlaceholderValues, colIdx), pParams); code = setValueByBindParam2((SValueNode*)taosArrayGetP(pQuery->pPlaceholderValues, colIdx), pParams, charsetCxt);
} }
if (TSDB_CODE_SUCCESS == code && (colIdx < 0 || colIdx + 1 == pQuery->placeholderNum)) { if (TSDB_CODE_SUCCESS == code && (colIdx < 0 || colIdx + 1 == pQuery->placeholderNum)) {

2
source/libs/planner/src/planOptimizer.c
View File

@ -6725,7 +6725,7 @@ static int32_t fillTSMAOptCtx(STSMAOptCtx* pTsmaOptCtx, SScanLogicNode* pScan) {
pTsmaOptCtx->queryInterval->sliding = pWindow->sliding; pTsmaOptCtx->queryInterval->sliding = pWindow->sliding;
pTsmaOptCtx->queryInterval->slidingUnit = pWindow->slidingUnit; pTsmaOptCtx->queryInterval->slidingUnit = pWindow->slidingUnit;
pTsmaOptCtx->queryInterval->precision = pWindow->node.precision; pTsmaOptCtx->queryInterval->precision = pWindow->node.precision;
pTsmaOptCtx->queryInterval->tz = tsTimezone; pTsmaOptCtx->queryInterval->tz = taosGetLocalTimezoneOffset();
pTsmaOptCtx->pAggFuncs = pWindow->pFuncs; pTsmaOptCtx->pAggFuncs = pWindow->pFuncs;
pTsmaOptCtx->ppParentTsmaSubplans = &pWindow->pTsmaSubplans; pTsmaOptCtx->ppParentTsmaSubplans = &pWindow->pTsmaSubplans;
} else { } else {

2
source/libs/planner/test/planTestUtil.cpp
View File

@ -377,7 +377,7 @@ class PlannerTestBaseImpl {
} }
void doBindParams(SQuery* pQuery, TAOS_MULTI_BIND* pParams, int32_t colIdx) { void doBindParams(SQuery* pQuery, TAOS_MULTI_BIND* pParams, int32_t colIdx) {
DO_WITH_THROW(qStmtBindParams, pQuery, pParams, colIdx); DO_WITH_THROW(qStmtBindParams, pQuery, pParams, colIdx, NULL);
if (colIdx < 0 || pQuery->placeholderNum == colIdx + 1) { if (colIdx < 0 || pQuery->placeholderNum == colIdx + 1) {
res_.boundAst_ = toString(pQuery->pRoot); res_.boundAst_ = toString(pQuery->pRoot);
} }

6
source/libs/qcom/src/queryUtil.c
View File

@ -397,7 +397,7 @@ int32_t dataConverToStr(char* str, int64_t capacity, int type, void* buf, int32_
} }
*str = '"'; *str = '"';
int32_t length = taosUcs4ToMbs((TdUcs4*)buf, bufSize, str + 1); int32_t length = taosUcs4ToMbs((TdUcs4*)buf, bufSize, str + 1, NULL);
if (length <= 0) { if (length <= 0) {
return TSDB_CODE_TSC_INVALID_VALUE; return TSDB_CODE_TSC_INVALID_VALUE;
} }
@ -430,7 +430,7 @@ int32_t dataConverToStr(char* str, int64_t capacity, int type, void* buf, int32_
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
void parseTagDatatoJson(void* p, char** jsonStr) { void parseTagDatatoJson(void* p, char** jsonStr, void *charsetCxt) {
if (!p || !jsonStr) { if (!p || !jsonStr) {
qError("parseTagDatatoJson invalid input, line:%d", __LINE__); qError("parseTagDatatoJson invalid input, line:%d", __LINE__);
return; return;
@ -475,7 +475,7 @@ void parseTagDatatoJson(void* p, char** jsonStr) {
if (tagJsonValue == NULL) { if (tagJsonValue == NULL) {
goto end; goto end;
} }
int32_t length = taosUcs4ToMbs((TdUcs4*)pTagVal->pData, pTagVal->nData, tagJsonValue); int32_t length = taosUcs4ToMbs((TdUcs4*)pTagVal->pData, pTagVal->nData, tagJsonValue, charsetCxt);
if (length < 0) { if (length < 0) {
qError("charset:%s to %s. val:%s convert json value failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, qError("charset:%s to %s. val:%s convert json value failed.", DEFAULT_UNICODE_ENCODEC, tsCharset,
pTagVal->pData); pTagVal->pData);

8
source/libs/scalar/src/filter.c
View File

@ -2266,7 +2266,7 @@ int32_t fltInitValFieldData(SFilterInfo *info) {
// match/nmatch for nchar type need convert from ucs4 to mbs // match/nmatch for nchar type need convert from ucs4 to mbs
if (type == TSDB_DATA_TYPE_NCHAR && (unit->compare.optr == OP_TYPE_MATCH || unit->compare.optr == OP_TYPE_NMATCH)) { if (type == TSDB_DATA_TYPE_NCHAR && (unit->compare.optr == OP_TYPE_MATCH || unit->compare.optr == OP_TYPE_NMATCH)) {
char newValData[TSDB_REGEX_STRING_DEFAULT_LEN * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE] = {0}; char newValData[TSDB_REGEX_STRING_DEFAULT_LEN * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE] = {0};
int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(fi->data), varDataLen(fi->data), varDataVal(newValData)); int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(fi->data), varDataLen(fi->data), varDataVal(newValData), NULL);
if (len < 0) { if (len < 0) {
qError("filterInitValFieldData taosUcs4ToMbs error 1"); qError("filterInitValFieldData taosUcs4ToMbs error 1");
return TSDB_CODE_SCALAR_CONVERT_ERROR; return TSDB_CODE_SCALAR_CONVERT_ERROR;
@ -3603,7 +3603,7 @@ int32_t filterExecuteImplMisc(void *pinfo, int32_t numOfRows, SColumnInfoData *p
if (newColData == NULL) { if (newColData == NULL) {
FLT_ERR_RET(terrno); FLT_ERR_RET(terrno);
} }
int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(colData), varDataLen(colData), varDataVal(newColData)); int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(colData), varDataLen(colData), varDataVal(newColData), NULL);
if (len < 0) { if (len < 0) {
qError("castConvert1 taosUcs4ToMbs error"); qError("castConvert1 taosUcs4ToMbs error");
taosMemoryFreeClear(newColData); taosMemoryFreeClear(newColData);
@ -3678,7 +3678,7 @@ int32_t filterExecuteImpl(void *pinfo, int32_t numOfRows, SColumnInfoData *pRes,
if (newColData == NULL) { if (newColData == NULL) {
FLT_ERR_RET(terrno); FLT_ERR_RET(terrno);
} }
int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(colData), varDataLen(colData), varDataVal(newColData)); int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(colData), varDataLen(colData), varDataVal(newColData), NULL);
if (len < 0) { if (len < 0) {
qError("castConvert1 taosUcs4ToMbs error"); qError("castConvert1 taosUcs4ToMbs error");
taosMemoryFreeClear(newColData); taosMemoryFreeClear(newColData);
@ -4614,7 +4614,7 @@ int32_t filterConverNcharColumns(SFilterInfo *info, int32_t rows, bool *gotNchar
varDataCopy(dst, src); varDataCopy(dst, src);
continue; continue;
} }
bool ret = taosMbsToUcs4(varDataVal(src), varDataLen(src), (TdUcs4 *)varDataVal(dst), bufSize, &len); bool ret = taosMbsToUcs4(varDataVal(src), varDataLen(src), (TdUcs4 *)varDataVal(dst), bufSize, &len, NULL);
if (!ret) { if (!ret) {
qError("filterConverNcharColumns taosMbsToUcs4 error"); qError("filterConverNcharColumns taosMbsToUcs4 error");
return TSDB_CODE_SCALAR_CONVERT_ERROR; return TSDB_CODE_SCALAR_CONVERT_ERROR;

7
source/libs/scalar/src/scalar.c
View File

@ -24,7 +24,7 @@ int32_t scalarGetOperatorParamNum(EOperatorType type) {
int32_t sclConvertToTsValueNode(int8_t precision, SValueNode *valueNode) { int32_t sclConvertToTsValueNode(int8_t precision, SValueNode *valueNode) {
char *timeStr = valueNode->datum.p; char *timeStr = valueNode->datum.p;
int64_t value = 0; int64_t value = 0;
int32_t code = convertStringToTimestamp(valueNode->node.resType.type, valueNode->datum.p, precision, &value, valueNode->tz); //todo tz int32_t code = convertStringToTimestamp(valueNode->node.resType.type, valueNode->datum.p, precision, &value, valueNode->tz, valueNode->charsetCxt); //todo tz
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
return code; return code;
} }
@ -82,7 +82,9 @@ int32_t sclConvertValueToSclParam(SValueNode *pValueNode, SScalarParam *out, int
} }
in.tz = pValueNode->tz; in.tz = pValueNode->tz;
in.charsetCxt = pValueNode->charsetCxt;
out->tz = pValueNode->tz; out->tz = pValueNode->tz;
out->charsetCxt = pValueNode->charsetCxt;
code = vectorConvertSingleColImpl(&in, out, overflow, -1, -1); code = vectorConvertSingleColImpl(&in, out, overflow, -1, -1);
_exit: _exit:
@ -589,6 +591,7 @@ int32_t sclInitOperatorParams(SScalarParam **pParams, SOperatorNode *node, SScal
SCL_ERR_JRET(sclInitParam(node->pLeft, &paramList[0], ctx, rowNum)); SCL_ERR_JRET(sclInitParam(node->pLeft, &paramList[0], ctx, rowNum));
paramList[0].tz = node->tz; paramList[0].tz = node->tz;
paramList[0].charsetCxt = node->charsetCxt;
if (paramNum > 1) { if (paramNum > 1) {
TSWAP(ctx->type.selfType, ctx->type.peerType); TSWAP(ctx->type.selfType, ctx->type.peerType);
SCL_ERR_JRET(sclInitParam(node->pRight, &paramList[1], ctx, rowNum)); SCL_ERR_JRET(sclInitParam(node->pRight, &paramList[1], ctx, rowNum));
@ -760,6 +763,7 @@ int32_t sclExecFunction(SFunctionNode *node, SScalarCtx *ctx, SScalarParam *outp
int32_t code = 0; int32_t code = 0;
SCL_ERR_RET(sclInitParamList(&params, node->pParameterList, ctx, &paramNum, &rowNum)); SCL_ERR_RET(sclInitParamList(&params, node->pParameterList, ctx, &paramNum, &rowNum));
params->tz = node->tz; params->tz = node->tz;
params->charsetCxt = node->charsetCxt;
if (fmIsUserDefinedFunc(node->funcId)) { if (fmIsUserDefinedFunc(node->funcId)) {
code = callUdfScalarFunc(node->functionName, params, paramNum, output); code = callUdfScalarFunc(node->functionName, params, paramNum, output);
@ -947,6 +951,7 @@ int32_t sclExecCaseWhen(SCaseWhenNode *node, SScalarCtx *ctx, SScalarParam *outp
SCL_ERR_JRET(sclGetNodeRes(node->pCase, ctx, &pCase)); SCL_ERR_JRET(sclGetNodeRes(node->pCase, ctx, &pCase));
SCL_ERR_JRET(sclGetNodeRes(node->pElse, ctx, &pElse)); SCL_ERR_JRET(sclGetNodeRes(node->pElse, ctx, &pElse));
pCase->tz = node->tz; pCase->tz = node->tz;
pCase->charsetCxt = node->charsetCxt;
SDataType compType = {0}; SDataType compType = {0};
compType.type = TSDB_DATA_TYPE_BOOL; compType.type = TSDB_DATA_TYPE_BOOL;
compType.bytes = tDataTypes[compType.type].bytes; compType.bytes = tDataTypes[compType.type].bytes;

112
source/libs/scalar/src/sclfunc.c
View File

@ -12,8 +12,8 @@ typedef float (*_float_fn_2)(float, float);
typedef double (*_double_fn)(double); typedef double (*_double_fn)(double);
typedef double (*_double_fn_2)(double, double); typedef double (*_double_fn_2)(double, double);
typedef int (*_conv_fn)(int); typedef int (*_conv_fn)(int);
typedef void (*_trim_space_fn)(char *, char *, int32_t, int32_t); typedef void (*_trim_space_fn)(char *, char *, int32_t, int32_t, void*);
typedef int32_t (*_trim_fn)(char *, char *, char *, int32_t, int32_t); typedef int32_t (*_trim_fn)(char *, char *, char *, int32_t, int32_t, void*);
typedef int32_t (*_len_fn)(char *, int32_t, VarDataLenT *); typedef int32_t (*_len_fn)(char *, int32_t, VarDataLenT *);
/** Math functions **/ /** Math functions **/
@ -500,7 +500,7 @@ static int32_t tcharlength(char *input, int32_t type, VarDataLenT *len) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static void tltrimspace(char *input, char *output, int32_t type, int32_t charLen) { static void tltrimspace(char *input, char *output, int32_t type, int32_t charLen, void* charsetCxt) {
int32_t numOfSpaces = 0; int32_t numOfSpaces = 0;
if (type == TSDB_DATA_TYPE_VARCHAR) { if (type == TSDB_DATA_TYPE_VARCHAR) {
for (int32_t i = 0; i < charLen; ++i) { for (int32_t i = 0; i < charLen; ++i) {
@ -530,7 +530,7 @@ static void tltrimspace(char *input, char *output, int32_t type, int32_t charLen
varDataSetLen(output, resLen); varDataSetLen(output, resLen);
} }
static void tlrtrimspace(char *input, char *output, int32_t type, int32_t charLen) { static void tlrtrimspace(char *input, char *output, int32_t type, int32_t charLen, void* charsetCxt) {
int32_t numOfLeftSpaces = 0; int32_t numOfLeftSpaces = 0;
int32_t numOfRightSpaces = 0; int32_t numOfRightSpaces = 0;
if (type == TSDB_DATA_TYPE_VARCHAR) { if (type == TSDB_DATA_TYPE_VARCHAR) {
@ -615,12 +615,12 @@ static int32_t trimHelper(char *orgStr, char* remStr, int32_t orgLen, int32_t re
} }
} }
static int32_t convVarcharToNchar(char *input, char **output, int32_t inputLen, int32_t *outputLen) { static int32_t convVarcharToNchar(char *input, char **output, int32_t inputLen, int32_t *outputLen, void* charsetCxt) {
*output = taosMemoryCalloc(inputLen * TSDB_NCHAR_SIZE, 1); *output = taosMemoryCalloc(inputLen * TSDB_NCHAR_SIZE, 1);
if (NULL == *output) { if (NULL == *output) {
return terrno; return terrno;
} }
bool ret = taosMbsToUcs4(input, inputLen, (TdUcs4 *)*output, inputLen * TSDB_NCHAR_SIZE, outputLen); bool ret = taosMbsToUcs4(input, inputLen, (TdUcs4 *)*output, inputLen * TSDB_NCHAR_SIZE, outputLen, charsetCxt);
if (!ret) { if (!ret) {
taosMemoryFreeClear(*output); taosMemoryFreeClear(*output);
return TSDB_CODE_SCALAR_CONVERT_ERROR; return TSDB_CODE_SCALAR_CONVERT_ERROR;
@ -628,12 +628,12 @@ static int32_t convVarcharToNchar(char *input, char **output, int32_t inputLen,
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static int32_t convNcharToVarchar(char *input, char **output, int32_t inputLen, int32_t *outputLen) { static int32_t convNcharToVarchar(char *input, char **output, int32_t inputLen, int32_t *outputLen, void* charsetCxt) {
*output = taosMemoryCalloc(inputLen, 1); *output = taosMemoryCalloc(inputLen, 1);
if (NULL == *output) { if (NULL == *output) {
return terrno; return terrno;
} }
*outputLen = taosUcs4ToMbs((TdUcs4 *)input, inputLen, *output); *outputLen = taosUcs4ToMbs((TdUcs4 *)input, inputLen, *output, charsetCxt);
if (*outputLen < 0) { if (*outputLen < 0) {
taosMemoryFree(*output); taosMemoryFree(*output);
return TSDB_CODE_SCALAR_CONVERT_ERROR; return TSDB_CODE_SCALAR_CONVERT_ERROR;
@ -641,14 +641,14 @@ static int32_t convNcharToVarchar(char *input, char **output, int32_t inputLen,
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static int32_t convBetweenNcharAndVarchar(char *input, char **output, int32_t inputLen, int32_t *outputLen, int32_t wantType) { static int32_t convBetweenNcharAndVarchar(char *input, char **output, int32_t inputLen, int32_t *outputLen, int32_t wantType, void* charsetCxt) {
if (wantType == TSDB_DATA_TYPE_NCHAR) { if (wantType == TSDB_DATA_TYPE_NCHAR) {
return convVarcharToNchar(input, output, inputLen, outputLen); return convVarcharToNchar(input, output, inputLen, outputLen, charsetCxt);
} else { } else {
return convNcharToVarchar(input, output, inputLen, outputLen); return convNcharToVarchar(input, output, inputLen, outputLen, charsetCxt);
} }
} }
static int32_t tltrim(char *input, char *remInput, char *output, int32_t inputType, int32_t remType) { static int32_t tltrim(char *input, char *remInput, char *output, int32_t inputType, int32_t remType, void* charsetCxt) {
int32_t orgLen = varDataLen(input); int32_t orgLen = varDataLen(input);
char *orgStr = varDataVal(input); char *orgStr = varDataVal(input);
int32_t remLen = varDataLen(remInput); int32_t remLen = varDataLen(remInput);
@ -661,7 +661,7 @@ static int32_t tltrim(char *input, char *remInput, char *output, int32_t inputTy
bool needFree = false; bool needFree = false;
if (inputType != remType) { if (inputType != remType) {
SCL_ERR_RET(convBetweenNcharAndVarchar(varDataVal(remInput), &remStr, varDataLen(remInput), &remLen, inputType)); SCL_ERR_RET(convBetweenNcharAndVarchar(varDataVal(remInput), &remStr, varDataLen(remInput), &remLen, inputType, charsetCxt));
needFree = true; needFree = true;
} }
@ -683,7 +683,7 @@ static int32_t tltrim(char *input, char *remInput, char *output, int32_t inputTy
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static void trtrimspace(char *input, char *output, int32_t type, int32_t charLen) { static void trtrimspace(char *input, char *output, int32_t type, int32_t charLen, void *charsetCxt) {
int32_t numOfSpaces = 0; int32_t numOfSpaces = 0;
if (type == TSDB_DATA_TYPE_VARCHAR) { if (type == TSDB_DATA_TYPE_VARCHAR) {
for (int32_t i = charLen - 1; i >= 0; --i) { for (int32_t i = charLen - 1; i >= 0; --i) {
@ -712,7 +712,7 @@ static void trtrimspace(char *input, char *output, int32_t type, int32_t charLen
varDataSetLen(output, resLen); varDataSetLen(output, resLen);
} }
static int32_t trtrim(char *input, char *remInput, char *output, int32_t inputType, int32_t remType) { static int32_t trtrim(char *input, char *remInput, char *output, int32_t inputType, int32_t remType, void* charsetCxt) {
int32_t orgLen = varDataLen(input); int32_t orgLen = varDataLen(input);
char *orgStr = varDataVal(input); char *orgStr = varDataVal(input);
int32_t remLen = varDataLen(remInput); int32_t remLen = varDataLen(remInput);
@ -725,7 +725,7 @@ static int32_t trtrim(char *input, char *remInput, char *output, int32_t inputTy
bool needFree = false; bool needFree = false;
if (inputType != remType) { if (inputType != remType) {
SCL_ERR_RET(convBetweenNcharAndVarchar(varDataVal(remInput), &remStr, varDataLen(remInput), &remLen, inputType)); SCL_ERR_RET(convBetweenNcharAndVarchar(varDataVal(remInput), &remStr, varDataLen(remInput), &remLen, inputType, charsetCxt));
needFree = true; needFree = true;
} }
@ -746,7 +746,7 @@ static int32_t trtrim(char *input, char *remInput, char *output, int32_t inputTy
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static int32_t tlrtrim(char *input, char *remInput, char *output, int32_t inputType, int32_t remType) { static int32_t tlrtrim(char *input, char *remInput, char *output, int32_t inputType, int32_t remType, void *charsetCxt) {
int32_t orgLen = varDataLen(input); int32_t orgLen = varDataLen(input);
char *orgStr = varDataVal(input); char *orgStr = varDataVal(input);
int32_t remLen = varDataLen(remInput); int32_t remLen = varDataLen(remInput);
@ -759,7 +759,7 @@ static int32_t tlrtrim(char *input, char *remInput, char *output, int32_t inputT
bool needFree = false; bool needFree = false;
if (inputType != remType) { if (inputType != remType) {
SCL_ERR_RET(convBetweenNcharAndVarchar(varDataVal(remInput), &remStr, varDataLen(remInput), &remLen, inputType)); SCL_ERR_RET(convBetweenNcharAndVarchar(varDataVal(remInput), &remStr, varDataLen(remInput), &remLen, inputType, charsetCxt));
needFree = true; needFree = true;
} }
@ -810,14 +810,14 @@ static int32_t doLengthFunction(SScalarParam *pInput, int32_t inputNum, SScalarP
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static int32_t concatCopyHelper(const char *input, char *output, bool hasNchar, int32_t type, VarDataLenT *dataLen) { static int32_t concatCopyHelper(const char *input, char *output, bool hasNchar, int32_t type, VarDataLenT *dataLen, void* charsetCxt) {
if (hasNchar && type == TSDB_DATA_TYPE_VARCHAR) { if (hasNchar && type == TSDB_DATA_TYPE_VARCHAR) {
TdUcs4 *newBuf = taosMemoryCalloc((varDataLen(input) + 1) * TSDB_NCHAR_SIZE, 1); TdUcs4 *newBuf = taosMemoryCalloc((varDataLen(input) + 1) * TSDB_NCHAR_SIZE, 1);
if (NULL == newBuf) { if (NULL == newBuf) {
return terrno; return terrno;
} }
int32_t len = varDataLen(input); int32_t len = varDataLen(input);
bool ret = taosMbsToUcs4(varDataVal(input), len, newBuf, (varDataLen(input) + 1) * TSDB_NCHAR_SIZE, &len); bool ret = taosMbsToUcs4(varDataVal(input), len, newBuf, (varDataLen(input) + 1) * TSDB_NCHAR_SIZE, &len, charsetCxt);
if (!ret) { if (!ret) {
taosMemoryFree(newBuf); taosMemoryFree(newBuf);
return TSDB_CODE_SCALAR_CONVERT_ERROR; return TSDB_CODE_SCALAR_CONVERT_ERROR;
@ -905,7 +905,7 @@ int32_t concatFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu
int32_t rowIdx = (pInput[i].numOfRows == 1) ? 0 : k; int32_t rowIdx = (pInput[i].numOfRows == 1) ? 0 : k;
input[i] = colDataGetData(pInputData[i], rowIdx); input[i] = colDataGetData(pInputData[i], rowIdx);
SCL_ERR_JRET(concatCopyHelper(input[i], output, hasNchar, GET_PARAM_TYPE(&pInput[i]), &dataLen)); SCL_ERR_JRET(concatCopyHelper(input[i], output, hasNchar, GET_PARAM_TYPE(&pInput[i]), &dataLen, pInput->charsetCxt));
} }
varDataSetLen(output, dataLen); varDataSetLen(output, dataLen);
SCL_ERR_JRET(colDataSetVal(pOutputData, k, outputBuf, false)); SCL_ERR_JRET(colDataSetVal(pOutputData, k, outputBuf, false));
@ -980,12 +980,12 @@ int32_t concatWsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p
int32_t rowIdx = (pInput[i].numOfRows == 1) ? 0 : k; int32_t rowIdx = (pInput[i].numOfRows == 1) ? 0 : k;
SCL_ERR_JRET(concatCopyHelper(colDataGetData(pInputData[i], rowIdx), output, hasNchar, GET_PARAM_TYPE(&pInput[i]), &dataLen)); SCL_ERR_JRET(concatCopyHelper(colDataGetData(pInputData[i], rowIdx), output, hasNchar, GET_PARAM_TYPE(&pInput[i]), &dataLen, pInput->charsetCxt));
if (i < inputNum - 1) { if (i < inputNum - 1) {
// insert the separator // insert the separator
char *sep = (pInput[0].numOfRows == 1) ? colDataGetData(pInputData[0], 0) : colDataGetData(pInputData[0], k); char *sep = (pInput[0].numOfRows == 1) ? colDataGetData(pInputData[0], 0) : colDataGetData(pInputData[0], k);
SCL_ERR_JRET(concatCopyHelper(sep, output, hasNchar, GET_PARAM_TYPE(&pInput[0]), &dataLen)); SCL_ERR_JRET(concatCopyHelper(sep, output, hasNchar, GET_PARAM_TYPE(&pInput[0]), &dataLen, pInput->charsetCxt));
} }
} }
@ -1092,7 +1092,7 @@ static int32_t doTrimFunction(SScalarParam *pInput, int32_t inputNum, SScalarPar
continue; continue;
} }
SCL_ERR_JRET(trimFn(colDataGetData(pInputData[1], colIdx2), colDataGetData(pInputData[0], colIdx1), SCL_ERR_JRET(trimFn(colDataGetData(pInputData[1], colIdx2), colDataGetData(pInputData[0], colIdx1),
output, GET_PARAM_TYPE(&pInput[1]), GET_PARAM_TYPE(&pInput[0]))); output, GET_PARAM_TYPE(&pInput[1]), GET_PARAM_TYPE(&pInput[0]), pInput->charsetCxt));
SCL_ERR_JRET(colDataSetVal(pOutputData, i, output, false)); SCL_ERR_JRET(colDataSetVal(pOutputData, i, output, false));
} }
} else { } else {
@ -1105,7 +1105,7 @@ static int32_t doTrimFunction(SScalarParam *pInput, int32_t inputNum, SScalarPar
char *input = colDataGetData(pInputData[0], i); char *input = colDataGetData(pInputData[0], i);
int32_t len = varDataLen(input); int32_t len = varDataLen(input);
int32_t charLen = (type == TSDB_DATA_TYPE_VARCHAR) ? len : len / TSDB_NCHAR_SIZE; int32_t charLen = (type == TSDB_DATA_TYPE_VARCHAR) ? len : len / TSDB_NCHAR_SIZE;
trimSpaceFn(input, output, type, charLen); trimSpaceFn(input, output, type, charLen, pInput->charsetCxt);
SCL_ERR_JRET(colDataSetVal(pOutputData, i, output, false)); SCL_ERR_JRET(colDataSetVal(pOutputData, i, output, false));
} }
} }
@ -1343,7 +1343,7 @@ int32_t charFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
if (convBuf == NULL) { if (convBuf == NULL) {
SCL_ERR_RET(terrno); SCL_ERR_RET(terrno);
} }
int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(colDataGetData(pInput[j].columnData, colIdx)), varDataLen(colDataGetData(pInput[j].columnData, colIdx)), convBuf); int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(colDataGetData(pInput[j].columnData, colIdx)), varDataLen(colDataGetData(pInput[j].columnData, colIdx)), convBuf, pInput->charsetCxt);
if (len < 0) { if (len < 0) {
taosMemoryFree(convBuf); taosMemoryFree(convBuf);
code = TSDB_CODE_SCALAR_CONVERT_ERROR; code = TSDB_CODE_SCALAR_CONVERT_ERROR;
@ -1386,7 +1386,7 @@ int32_t asciiFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOut
int32_t inLen = varDataLen(colDataGetData(pInputData, i)); int32_t inLen = varDataLen(colDataGetData(pInputData, i));
SCL_ERR_RET(convBetweenNcharAndVarchar(varDataVal(colDataGetData(pInputData, i)), &in, SCL_ERR_RET(convBetweenNcharAndVarchar(varDataVal(colDataGetData(pInputData, i)), &in,
varDataLen(colDataGetData(pInputData, i)), &inLen, varDataLen(colDataGetData(pInputData, i)), &inLen,
TSDB_DATA_TYPE_VARBINARY)); TSDB_DATA_TYPE_VARBINARY, pInput->charsetCxt));
out[i] = (uint8_t)(in)[0]; out[i] = (uint8_t)(in)[0];
taosMemoryFree(in); taosMemoryFree(in);
} else { } else {
@ -1454,7 +1454,9 @@ int32_t positionFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p
int32_t orgLen = varDataLen(colDataGetData(pInputData[1], colIdx2)); int32_t orgLen = varDataLen(colDataGetData(pInputData[1], colIdx2));
bool needFreeSub = false; bool needFreeSub = false;
if (GET_PARAM_TYPE(&pInput[1]) != GET_PARAM_TYPE(&pInput[0])) { if (GET_PARAM_TYPE(&pInput[1]) != GET_PARAM_TYPE(&pInput[0])) {
SCL_ERR_RET(convBetweenNcharAndVarchar(varDataVal(colDataGetData(pInputData[0], colIdx1)), &substr, varDataLen(colDataGetData(pInputData[0], colIdx1)), &subLen, GET_PARAM_TYPE(&pInput[1]))); SCL_ERR_RET(convBetweenNcharAndVarchar(varDataVal(colDataGetData(pInputData[0], colIdx1)), &substr,
varDataLen(colDataGetData(pInputData[0], colIdx1)), &subLen,
GET_PARAM_TYPE(&pInput[1]), pInput->charsetCxt));
needFreeSub = true; needFreeSub = true;
} }
@ -1562,13 +1564,13 @@ int32_t replaceFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pO
if (GET_PARAM_TYPE(&pInput[1]) != GET_PARAM_TYPE(&pInput[0])) { if (GET_PARAM_TYPE(&pInput[1]) != GET_PARAM_TYPE(&pInput[0])) {
SCL_ERR_JRET(convBetweenNcharAndVarchar(varDataVal(colDataGetData(pInputData[1], colIdx2)), &fromStr, SCL_ERR_JRET(convBetweenNcharAndVarchar(varDataVal(colDataGetData(pInputData[1], colIdx2)), &fromStr,
varDataLen(colDataGetData(pInputData[1], colIdx2)), &fromLen, varDataLen(colDataGetData(pInputData[1], colIdx2)), &fromLen,
GET_PARAM_TYPE(&pInput[0]))); GET_PARAM_TYPE(&pInput[0]), pInput->charsetCxt));
needFreeFrom = true; needFreeFrom = true;
} }
if (GET_PARAM_TYPE(&pInput[2]) != GET_PARAM_TYPE(&pInput[0])) { if (GET_PARAM_TYPE(&pInput[2]) != GET_PARAM_TYPE(&pInput[0])) {
code = convBetweenNcharAndVarchar(varDataVal(colDataGetData(pInputData[2], colIdx3)), &toStr, code = convBetweenNcharAndVarchar(varDataVal(colDataGetData(pInputData[2], colIdx3)), &toStr,
varDataLen(colDataGetData(pInputData[2], colIdx3)), &toLen, varDataLen(colDataGetData(pInputData[2], colIdx3)), &toLen,
GET_PARAM_TYPE(&pInput[0])); GET_PARAM_TYPE(&pInput[0]), pInput->charsetCxt);
if (TSDB_CODE_SUCCESS != code) { if (TSDB_CODE_SUCCESS != code) {
if (needFreeFrom) { if (needFreeFrom) {
taosMemoryFree(fromStr); taosMemoryFree(fromStr);
@ -1674,7 +1676,7 @@ int32_t substrIdxFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *
if (GET_PARAM_TYPE(&pInput[0]) != GET_PARAM_TYPE(&pInput[1])) { if (GET_PARAM_TYPE(&pInput[0]) != GET_PARAM_TYPE(&pInput[1])) {
SCL_ERR_JRET(convBetweenNcharAndVarchar(varDataVal(colDataGetData(pInputData[1], colIdx2)), &delimStr, SCL_ERR_JRET(convBetweenNcharAndVarchar(varDataVal(colDataGetData(pInputData[1], colIdx2)), &delimStr,
varDataLen(colDataGetData(pInputData[1], colIdx2)), &delimLen, varDataLen(colDataGetData(pInputData[1], colIdx2)), &delimLen,
GET_PARAM_TYPE(&pInput[0]))); GET_PARAM_TYPE(&pInput[0]), pInput->charsetCxt));
needFreeDelim = true; needFreeDelim = true;
} }
@ -1853,7 +1855,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
buf[varDataLen(input)] = 0; buf[varDataLen(input)] = 0;
*(int8_t *)output = taosStr2Int8(buf, NULL, 10); *(int8_t *)output = taosStr2Int8(buf, NULL, 10);
} else if (inputType == TSDB_DATA_TYPE_NCHAR) { } else if (inputType == TSDB_DATA_TYPE_NCHAR) {
int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf); int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf, pInput->charsetCxt);
if (len < 0) { if (len < 0) {
code = TSDB_CODE_SCALAR_CONVERT_ERROR; code = TSDB_CODE_SCALAR_CONVERT_ERROR;
goto _end; goto _end;
@ -1872,7 +1874,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
buf[varDataLen(input)] = 0; buf[varDataLen(input)] = 0;
*(int16_t *)output = taosStr2Int16(buf, NULL, 10); *(int16_t *)output = taosStr2Int16(buf, NULL, 10);
} else if (inputType == TSDB_DATA_TYPE_NCHAR) { } else if (inputType == TSDB_DATA_TYPE_NCHAR) {
int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf); int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf, pInput->charsetCxt);
if (len < 0) { if (len < 0) {
code = TSDB_CODE_SCALAR_CONVERT_ERROR; code = TSDB_CODE_SCALAR_CONVERT_ERROR;
goto _end; goto _end;
@ -1890,7 +1892,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
buf[varDataLen(input)] = 0; buf[varDataLen(input)] = 0;
*(int32_t *)output = taosStr2Int32(buf, NULL, 10); *(int32_t *)output = taosStr2Int32(buf, NULL, 10);
} else if (inputType == TSDB_DATA_TYPE_NCHAR) { } else if (inputType == TSDB_DATA_TYPE_NCHAR) {
int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf); int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf, pInput->charsetCxt);
if (len < 0) { if (len < 0) {
code = TSDB_CODE_SCALAR_CONVERT_ERROR; code = TSDB_CODE_SCALAR_CONVERT_ERROR;
goto _end; goto _end;
@ -1909,7 +1911,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
buf[varDataLen(input)] = 0; buf[varDataLen(input)] = 0;
*(int64_t *)output = taosStr2Int64(buf, NULL, 10); *(int64_t *)output = taosStr2Int64(buf, NULL, 10);
} else if (inputType == TSDB_DATA_TYPE_NCHAR) { } else if (inputType == TSDB_DATA_TYPE_NCHAR) {
int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf); int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf, pInput->charsetCxt);
if (len < 0) { if (len < 0) {
code = TSDB_CODE_SCALAR_CONVERT_ERROR; code = TSDB_CODE_SCALAR_CONVERT_ERROR;
goto _end; goto _end;
@ -1927,7 +1929,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
buf[varDataLen(input)] = 0; buf[varDataLen(input)] = 0;
*(uint8_t *)output = taosStr2UInt8(buf, NULL, 10); *(uint8_t *)output = taosStr2UInt8(buf, NULL, 10);
} else if (inputType == TSDB_DATA_TYPE_NCHAR) { } else if (inputType == TSDB_DATA_TYPE_NCHAR) {
int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf); int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf, pInput->charsetCxt);
if (len < 0) { if (len < 0) {
code = TSDB_CODE_SCALAR_CONVERT_ERROR; code = TSDB_CODE_SCALAR_CONVERT_ERROR;
goto _end; goto _end;
@ -1945,7 +1947,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
buf[varDataLen(input)] = 0; buf[varDataLen(input)] = 0;
*(uint16_t *)output = taosStr2UInt16(buf, NULL, 10); *(uint16_t *)output = taosStr2UInt16(buf, NULL, 10);
} else if (inputType == TSDB_DATA_TYPE_NCHAR) { } else if (inputType == TSDB_DATA_TYPE_NCHAR) {
int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf); int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf, pInput->charsetCxt);
if (len < 0) { if (len < 0) {
code = TSDB_CODE_SCALAR_CONVERT_ERROR; code = TSDB_CODE_SCALAR_CONVERT_ERROR;
goto _end; goto _end;
@ -1963,7 +1965,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
buf[varDataLen(input)] = 0; buf[varDataLen(input)] = 0;
*(uint32_t *)output = taosStr2UInt32(buf, NULL, 10); *(uint32_t *)output = taosStr2UInt32(buf, NULL, 10);
} else if (inputType == TSDB_DATA_TYPE_NCHAR) { } else if (inputType == TSDB_DATA_TYPE_NCHAR) {
int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf); int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf, pInput->charsetCxt);
if (len < 0) { if (len < 0) {
code = TSDB_CODE_SCALAR_CONVERT_ERROR; code = TSDB_CODE_SCALAR_CONVERT_ERROR;
goto _end; goto _end;
@ -1981,7 +1983,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
buf[varDataLen(input)] = 0; buf[varDataLen(input)] = 0;
*(uint64_t *)output = taosStr2UInt64(buf, NULL, 10); *(uint64_t *)output = taosStr2UInt64(buf, NULL, 10);
} else if (inputType == TSDB_DATA_TYPE_NCHAR) { } else if (inputType == TSDB_DATA_TYPE_NCHAR) {
int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf); int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf, pInput->charsetCxt);
if (len < 0) { if (len < 0) {
code = TSDB_CODE_SCALAR_CONVERT_ERROR; code = TSDB_CODE_SCALAR_CONVERT_ERROR;
goto _end; goto _end;
@ -2000,7 +2002,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
buf[varDataLen(input)] = 0; buf[varDataLen(input)] = 0;
*(float *)output = taosStr2Float(buf, NULL); *(float *)output = taosStr2Float(buf, NULL);
} else if (inputType == TSDB_DATA_TYPE_NCHAR) { } else if (inputType == TSDB_DATA_TYPE_NCHAR) {
int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf); int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf, pInput->charsetCxt);
if (len < 0) { if (len < 0) {
code = TSDB_CODE_SCALAR_CONVERT_ERROR; code = TSDB_CODE_SCALAR_CONVERT_ERROR;
goto _end; goto _end;
@ -2018,7 +2020,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
buf[varDataLen(input)] = 0; buf[varDataLen(input)] = 0;
*(double *)output = taosStr2Double(buf, NULL); *(double *)output = taosStr2Double(buf, NULL);
} else if (inputType == TSDB_DATA_TYPE_NCHAR) { } else if (inputType == TSDB_DATA_TYPE_NCHAR) {
int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf); int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf, pInput->charsetCxt);
if (len < 0) { if (len < 0) {
code = TSDB_CODE_SCALAR_CONVERT_ERROR; code = TSDB_CODE_SCALAR_CONVERT_ERROR;
goto _end; goto _end;
@ -2036,7 +2038,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
buf[varDataLen(input)] = 0; buf[varDataLen(input)] = 0;
*(bool *)output = taosStr2Int8(buf, NULL, 10); *(bool *)output = taosStr2Int8(buf, NULL, 10);
} else if (inputType == TSDB_DATA_TYPE_NCHAR) { } else if (inputType == TSDB_DATA_TYPE_NCHAR) {
int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf); int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf, pInput->charsetCxt);
if (len < 0) { if (len < 0) {
code = TSDB_CODE_SCALAR_CONVERT_ERROR; code = TSDB_CODE_SCALAR_CONVERT_ERROR;
goto _end; goto _end;
@ -2053,7 +2055,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
if (inputType == TSDB_DATA_TYPE_BINARY || inputType == TSDB_DATA_TYPE_NCHAR) { if (inputType == TSDB_DATA_TYPE_BINARY || inputType == TSDB_DATA_TYPE_NCHAR) {
int64_t timePrec; int64_t timePrec;
GET_TYPED_DATA(timePrec, int64_t, GET_PARAM_TYPE(&pInput[1]), pInput[1].columnData->pData); GET_TYPED_DATA(timePrec, int64_t, GET_PARAM_TYPE(&pInput[1]), pInput[1].columnData->pData);
int32_t ret = convertStringToTimestamp(inputType, input, timePrec, &timeVal, pInput->tz); int32_t ret = convertStringToTimestamp(inputType, input, timePrec, &timeVal, pInput->tz, pInput->charsetCxt);
if (ret != TSDB_CODE_SUCCESS) { if (ret != TSDB_CODE_SUCCESS) {
*(int64_t *)output = 0; *(int64_t *)output = 0;
} else { } else {
@ -2076,7 +2078,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
(void)memcpy(varDataVal(output), varDataVal(input), len); (void)memcpy(varDataVal(output), varDataVal(input), len);
varDataSetLen(output, len); varDataSetLen(output, len);
} else if (inputType == TSDB_DATA_TYPE_NCHAR) { } else if (inputType == TSDB_DATA_TYPE_NCHAR) {
int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf); int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf, pInput->charsetCxt);
if (len < 0) { if (len < 0) {
code = TSDB_CODE_SCALAR_CONVERT_ERROR; code = TSDB_CODE_SCALAR_CONVERT_ERROR;
goto _end; goto _end;
@ -2111,7 +2113,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
if (inputType == TSDB_DATA_TYPE_BOOL) { if (inputType == TSDB_DATA_TYPE_BOOL) {
char tmp[8] = {0}; char tmp[8] = {0};
len = tsnprintf(tmp, sizeof(tmp), "%.*s", outputCharLen, *(int8_t *)input ? "true" : "false"); len = tsnprintf(tmp, sizeof(tmp), "%.*s", outputCharLen, *(int8_t *)input ? "true" : "false");
bool ret = taosMbsToUcs4(tmp, len, (TdUcs4 *)varDataVal(output), outputLen - VARSTR_HEADER_SIZE, &len); bool ret = taosMbsToUcs4(tmp, len, (TdUcs4 *)varDataVal(output), outputLen - VARSTR_HEADER_SIZE, &len, pInput->charsetCxt);
if (!ret) { if (!ret) {
code = TSDB_CODE_SCALAR_CONVERT_ERROR; code = TSDB_CODE_SCALAR_CONVERT_ERROR;
goto _end; goto _end;
@ -2121,7 +2123,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
} else if (inputType == TSDB_DATA_TYPE_BINARY) { } else if (inputType == TSDB_DATA_TYPE_BINARY) {
len = outputCharLen > varDataLen(input) ? varDataLen(input) : outputCharLen; len = outputCharLen > varDataLen(input) ? varDataLen(input) : outputCharLen;
bool ret = taosMbsToUcs4(input + VARSTR_HEADER_SIZE, len, (TdUcs4 *)varDataVal(output), bool ret = taosMbsToUcs4(input + VARSTR_HEADER_SIZE, len, (TdUcs4 *)varDataVal(output),
outputLen - VARSTR_HEADER_SIZE, &len); outputLen - VARSTR_HEADER_SIZE, &len, pInput->charsetCxt);
if (!ret) { if (!ret) {
code = TSDB_CODE_SCALAR_CONVERT_ERROR; code = TSDB_CODE_SCALAR_CONVERT_ERROR;
goto _end; goto _end;
@ -2135,7 +2137,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
NUM_TO_STRING(inputType, input, bufSize, buf); NUM_TO_STRING(inputType, input, bufSize, buf);
len = (int32_t)strlen(buf); len = (int32_t)strlen(buf);
len = outputCharLen > len ? len : outputCharLen; len = outputCharLen > len ? len : outputCharLen;
bool ret = taosMbsToUcs4(buf, len, (TdUcs4 *)varDataVal(output), outputLen - VARSTR_HEADER_SIZE, &len); bool ret = taosMbsToUcs4(buf, len, (TdUcs4 *)varDataVal(output), outputLen - VARSTR_HEADER_SIZE, &len, pInput->charsetCxt);
if (!ret) { if (!ret) {
code = TSDB_CODE_SCALAR_CONVERT_ERROR; code = TSDB_CODE_SCALAR_CONVERT_ERROR;
goto _end; goto _end;
@ -2283,7 +2285,7 @@ int32_t toUnixtimestampFunction(SScalarParam *pInput, int32_t inputNum, SScalarP
char *input = colDataGetData(pInput[0].columnData, i); char *input = colDataGetData(pInput[0].columnData, i);
int64_t timeVal = 0; int64_t timeVal = 0;
int32_t ret = convertStringToTimestamp(type, input, timePrec, &timeVal, pInput->tz); int32_t ret = convertStringToTimestamp(type, input, timePrec, &timeVal, pInput->tz, pInput->charsetCxt);
if (ret != TSDB_CODE_SUCCESS) { if (ret != TSDB_CODE_SUCCESS) {
colDataSetNULL(pOutput->columnData, i); colDataSetNULL(pOutput->columnData, i);
} else { } else {
@ -2323,7 +2325,7 @@ int32_t toJsonFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu
} }
(void)memcpy(tmp, varDataVal(input), varDataLen(input)); (void)memcpy(tmp, varDataVal(input), varDataLen(input));
tmp[varDataLen(input)] = 0; tmp[varDataLen(input)] = 0;
if (parseJsontoTagData(tmp, pTagVals, &pTag, NULL)) { if (parseJsontoTagData(tmp, pTagVals, &pTag, NULL, pInput->charsetCxt)) {
code = tTagNew(pTagVals, 1, true, &pTag); code = tTagNew(pTagVals, 1, true, &pTag);
if (TSDB_CODE_SUCCESS != code) { if (TSDB_CODE_SUCCESS != code) {
tTagFree(pTag); tTagFree(pTag);
@ -2473,7 +2475,7 @@ int32_t timeTruncateFunction(SScalarParam *pInput, int32_t inputNum, SScalarPara
char *input = colDataGetData(pInput[0].columnData, i); char *input = colDataGetData(pInput[0].columnData, i);
if (IS_VAR_DATA_TYPE(type)) { /* datetime format strings */ if (IS_VAR_DATA_TYPE(type)) { /* datetime format strings */
int32_t ret = convertStringToTimestamp(type, input, timePrec, &timeVal, pInput->tz); int32_t ret = convertStringToTimestamp(type, input, timePrec, &timeVal, pInput->tz, pInput->charsetCxt);
if (ret != TSDB_CODE_SUCCESS) { if (ret != TSDB_CODE_SUCCESS) {
colDataSetNULL(pOutput->columnData, i); colDataSetNULL(pOutput->columnData, i);
continue; continue;
@ -2533,7 +2535,7 @@ int32_t timeDiffFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p
int32_t type = GET_PARAM_TYPE(&pInput[k]); int32_t type = GET_PARAM_TYPE(&pInput[k]);
if (IS_VAR_DATA_TYPE(type)) { /* datetime format strings */ if (IS_VAR_DATA_TYPE(type)) { /* datetime format strings */
int32_t ret = convertStringToTimestamp(type, input[k], TSDB_TIME_PRECISION_NANO, &timeVal[k], pInput->tz); int32_t ret = convertStringToTimestamp(type, input[k], TSDB_TIME_PRECISION_NANO, &timeVal[k], pInput->tz, pInput->charsetCxt);
if (ret != TSDB_CODE_SUCCESS) { if (ret != TSDB_CODE_SUCCESS) {
hasNull = true; hasNull = true;
break; break;
@ -2660,10 +2662,6 @@ int32_t todayFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOut
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t timeZoneStrLen() {
return sizeof(VarDataLenT) + strlen(tsTimezoneStr);
}
int32_t timezoneFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { int32_t timezoneFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
char output[TD_TIMEZONE_LEN + VARSTR_HEADER_SIZE] = {0}; char output[TD_TIMEZONE_LEN + VARSTR_HEADER_SIZE] = {0};
// pInput->tz todo tz // pInput->tz todo tz
@ -2699,7 +2697,7 @@ int32_t weekdayFunctionImpl(SScalarParam *pInput, int32_t inputNum, SScalarParam
char *input = colDataGetData(pInput[0].columnData, i); char *input = colDataGetData(pInput[0].columnData, i);
if (IS_VAR_DATA_TYPE(type)) { /* datetime format strings */ if (IS_VAR_DATA_TYPE(type)) { /* datetime format strings */
int32_t ret = convertStringToTimestamp(type, input, timePrec, &timeVal, pInput->tz); int32_t ret = convertStringToTimestamp(type, input, timePrec, &timeVal, pInput->tz, pInput->charsetCxt);
if (ret != TSDB_CODE_SUCCESS) { if (ret != TSDB_CODE_SUCCESS) {
colDataSetNULL(pOutput->columnData, i); colDataSetNULL(pOutput->columnData, i);
continue; continue;
@ -2809,7 +2807,7 @@ int32_t weekFunctionImpl(SScalarParam *pInput, int32_t inputNum, SScalarParam *p
char *input = colDataGetData(pInput[0].columnData, i); char *input = colDataGetData(pInput[0].columnData, i);
if (IS_VAR_DATA_TYPE(type)) { /* datetime format strings */ if (IS_VAR_DATA_TYPE(type)) { /* datetime format strings */
int32_t ret = convertStringToTimestamp(type, input, prec, &timeVal, pInput->tz); int32_t ret = convertStringToTimestamp(type, input, prec, &timeVal, pInput->tz, pInput->charsetCxt);
if (ret != TSDB_CODE_SUCCESS) { if (ret != TSDB_CODE_SUCCESS) {
colDataSetNULL(pOutput->columnData, i); colDataSetNULL(pOutput->columnData, i);
continue; continue;

26
source/libs/scalar/src/sclvector.c
View File

@ -103,7 +103,7 @@ int32_t convertNcharToDouble(const void *inData, void *outData) {
if (NULL == tmp) { if (NULL == tmp) {
SCL_ERR_RET(terrno); SCL_ERR_RET(terrno);
} }
int len = taosUcs4ToMbs((TdUcs4 *)varDataVal(inData), varDataLen(inData), tmp); int len = taosUcs4ToMbs((TdUcs4 *)varDataVal(inData), varDataLen(inData), tmp, NULL);
if (len < 0) { if (len < 0) {
sclError("castConvert taosUcs4ToMbs error 1"); sclError("castConvert taosUcs4ToMbs error 1");
SCL_ERR_JRET(TSDB_CODE_SCALAR_CONVERT_ERROR); SCL_ERR_JRET(TSDB_CODE_SCALAR_CONVERT_ERROR);
@ -404,7 +404,7 @@ static FORCE_INLINE int32_t varToNchar(char *buf, SScalarParam *pOut, int32_t ro
SCL_ERR_RET(terrno); SCL_ERR_RET(terrno);
} }
int32_t ret = int32_t ret =
taosMbsToUcs4(varDataVal(buf), inputLen, (TdUcs4 *)varDataVal(t), outputMaxLen - VARSTR_HEADER_SIZE, &len); taosMbsToUcs4(varDataVal(buf), inputLen, (TdUcs4 *)varDataVal(t), outputMaxLen - VARSTR_HEADER_SIZE, &len, pOut->charsetCxt);
if (!ret) { if (!ret) {
sclError("failed to convert to NCHAR"); sclError("failed to convert to NCHAR");
SCL_ERR_JRET(TSDB_CODE_SCALAR_CONVERT_ERROR); SCL_ERR_JRET(TSDB_CODE_SCALAR_CONVERT_ERROR);
@ -426,7 +426,7 @@ static FORCE_INLINE int32_t ncharToVar(char *buf, SScalarParam *pOut, int32_t ro
if (NULL == t) { if (NULL == t) {
SCL_ERR_RET(terrno); SCL_ERR_RET(terrno);
} }
int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(buf), varDataLen(buf), varDataVal(t)); int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(buf), varDataLen(buf), varDataVal(t), pOut->charsetCxt);
if (len < 0) { if (len < 0) {
SCL_ERR_JRET(TSDB_CODE_SCALAR_CONVERT_ERROR); SCL_ERR_JRET(TSDB_CODE_SCALAR_CONVERT_ERROR);
} }
@ -557,7 +557,7 @@ int32_t vectorConvertFromVarData(SSclVectorConvCtx *pCtx, int32_t *overflow) {
SCL_ERR_JRET(TSDB_CODE_APP_ERROR); SCL_ERR_JRET(TSDB_CODE_APP_ERROR);
} }
int len = taosUcs4ToMbs((TdUcs4 *)varDataVal(data), varDataLen(data), tmp); int len = taosUcs4ToMbs((TdUcs4 *)varDataVal(data), varDataLen(data), tmp, pCtx->pIn->charsetCxt);
if (len < 0) { if (len < 0) {
sclError("castConvert taosUcs4ToMbs error 1"); sclError("castConvert taosUcs4ToMbs error 1");
SCL_ERR_JRET(TSDB_CODE_SCALAR_CONVERT_ERROR); SCL_ERR_JRET(TSDB_CODE_SCALAR_CONVERT_ERROR);
@ -592,7 +592,7 @@ int32_t getVectorDoubleValue_JSON(void *src, int32_t index, double *out) {
SCL_RET(TSDB_CODE_SUCCESS); SCL_RET(TSDB_CODE_SUCCESS);
} }
int32_t ncharTobinary(void *buf, void **out) { // todo need to remove , if tobinary is nchar int32_t ncharTobinary(void *buf, void **out, void* charsetCxt) { // todo need to remove , if tobinary is nchar
int32_t inputLen = varDataTLen(buf); int32_t inputLen = varDataTLen(buf);
*out = taosMemoryCalloc(1, inputLen); *out = taosMemoryCalloc(1, inputLen);
@ -601,7 +601,7 @@ int32_t ncharTobinary(void *buf, void **out) { // todo need to remove , if tobi
DEFAULT_UNICODE_ENCODEC, tsCharset, (char *)varDataVal(buf)); DEFAULT_UNICODE_ENCODEC, tsCharset, (char *)varDataVal(buf));
SCL_ERR_RET(terrno); SCL_ERR_RET(terrno);
} }
int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(buf), varDataLen(buf), varDataVal(*out)); int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(buf), varDataLen(buf), varDataVal(*out), charsetCxt);
if (len < 0) { if (len < 0) {
sclError("charset:%s to %s. val:%s convert ncharTobinary failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, sclError("charset:%s to %s. val:%s convert ncharTobinary failed.", DEFAULT_UNICODE_ENCODEC, tsCharset,
(char *)varDataVal(buf)); (char *)varDataVal(buf));
@ -614,7 +614,7 @@ int32_t ncharTobinary(void *buf, void **out) { // todo need to remove , if tobi
int32_t convertJsonValue(__compar_fn_t *fp, int32_t optr, int8_t typeLeft, int8_t typeRight, char **pLeftData, int32_t convertJsonValue(__compar_fn_t *fp, int32_t optr, int8_t typeLeft, int8_t typeRight, char **pLeftData,
char **pRightData, void *pLeftOut, void *pRightOut, bool *isNull, bool *freeLeft, char **pRightData, void *pLeftOut, void *pRightOut, bool *isNull, bool *freeLeft,
bool *freeRight, bool *result) { bool *freeRight, bool *result, void* charsetCxt) {
*result = false; *result = false;
if (optr == OP_TYPE_JSON_CONTAINS) { if (optr == OP_TYPE_JSON_CONTAINS) {
*result = true; *result = true;
@ -700,13 +700,13 @@ int32_t convertJsonValue(__compar_fn_t *fp, int32_t optr, int8_t typeLeft, int8_
type == TSDB_DATA_TYPE_GEOMETRY) { type == TSDB_DATA_TYPE_GEOMETRY) {
if (typeLeft == TSDB_DATA_TYPE_NCHAR) { if (typeLeft == TSDB_DATA_TYPE_NCHAR) {
char *tmpLeft = NULL; char *tmpLeft = NULL;
SCL_ERR_RET(ncharTobinary(*pLeftData, (void *)&tmpLeft)); SCL_ERR_RET(ncharTobinary(*pLeftData, (void *)&tmpLeft, charsetCxt));
*pLeftData = tmpLeft; *pLeftData = tmpLeft;
*freeLeft = true; *freeLeft = true;
} }
if (typeRight == TSDB_DATA_TYPE_NCHAR) { if (typeRight == TSDB_DATA_TYPE_NCHAR) {
char *tmpRight = NULL; char *tmpRight = NULL;
SCL_ERR_RET(ncharTobinary(*pRightData, (void *)&tmpRight)); SCL_ERR_RET(ncharTobinary(*pRightData, (void *)&tmpRight, charsetCxt));
*pRightData = tmpRight; *pRightData = tmpRight;
*freeRight = true; *freeRight = true;
} }
@ -1958,7 +1958,7 @@ int32_t doVectorCompareImpl(SScalarParam *pLeft, SScalarParam *pRight, SScalarPa
bool result = false; bool result = false;
SCL_ERR_RET(convertJsonValue(&fp, optr, GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), &pLeftData, &pRightData, SCL_ERR_RET(convertJsonValue(&fp, optr, GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), &pLeftData, &pRightData,
&leftOut, &rightOut, &isJsonnull, &freeLeft, &freeRight, &result)); &leftOut, &rightOut, &isJsonnull, &freeLeft, &freeRight, &result, pLeft->charsetCxt));
if (isJsonnull) { if (isJsonnull) {
sclError("doVectorCompareImpl: invalid json null value"); sclError("doVectorCompareImpl: invalid json null value");
@ -2043,8 +2043,11 @@ int32_t vectorCompareImpl(SScalarParam *pLeft, SScalarParam *pRight, SScalarPara
SScalarParam *param2 = NULL; SScalarParam *param2 = NULL;
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
pRight->tz = pLeft->tz; pRight->tz = pLeft->tz;
pRight->charsetCxt = pLeft->charsetCxt;
pLeftOut.tz = pLeft->tz; pLeftOut.tz = pLeft->tz;
pLeftOut.charsetCxt = pLeft->charsetCxt;
pRightOut.tz = pRight->tz; pRightOut.tz = pRight->tz;
pRightOut.charsetCxt = pRight->charsetCxt;
if (noConvertBeforeCompare(GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), optr)) { if (noConvertBeforeCompare(GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), optr)) {
param1 = pLeft; param1 = pLeft;
param2 = pRight; param2 = pRight;
@ -2130,6 +2133,7 @@ int32_t vectorIsNull(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pO
int32_t vectorNotNull(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOut, int32_t _ord) { int32_t vectorNotNull(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOut, int32_t _ord) {
if (pRight != NULL) { if (pRight != NULL) {
pRight->tz = pLeft->tz; pRight->tz = pLeft->tz;
pRight->charsetCxt = pLeft->charsetCxt;
} }
for (int32_t i = 0; i < pLeft->numOfRows; ++i) { for (int32_t i = 0; i < pLeft->numOfRows; ++i) {
int8_t v = IS_HELPER_NULL(pLeft->columnData, i) ? 0 : 1; int8_t v = IS_HELPER_NULL(pLeft->columnData, i) ? 0 : 1;
@ -2146,6 +2150,8 @@ int32_t vectorNotNull(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *p
int32_t vectorIsTrue(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOut, int32_t _ord) { int32_t vectorIsTrue(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOut, int32_t _ord) {
if (pRight != NULL) { if (pRight != NULL) {
pRight->tz = pLeft->tz; pRight->tz = pLeft->tz;
pRight->charsetCxt = pLeft->charsetCxt;
} }
SCL_ERR_RET(vectorConvertSingleColImpl(pLeft, pOut, NULL, -1, -1)); SCL_ERR_RET(vectorConvertSingleColImpl(pLeft, pOut, NULL, -1, -1));
for (int32_t i = 0; i < pOut->numOfRows; ++i) { for (int32_t i = 0; i < pOut->numOfRows; ++i) {

6
source/libs/scalar/test/scalar/scalarTests.cpp
View File

@ -1450,7 +1450,7 @@ TEST(columnTest, json_column_arith_op) {
SArray *tags = taosArrayInit(1, sizeof(STagVal)); SArray *tags = taosArrayInit(1, sizeof(STagVal));
ASSERT_NE(tags, nullptr); ASSERT_NE(tags, nullptr);
STag *row = NULL; STag *row = NULL;
int32_t code = parseJsontoTagData(rightv, tags, &row, NULL); int32_t code = parseJsontoTagData(rightv, tags, &row, NULL, NULL);
ASSERT_EQ(code, TSDB_CODE_SUCCESS); ASSERT_EQ(code, TSDB_CODE_SUCCESS);
const int32_t len = 8; const int32_t len = 8;
@ -1606,7 +1606,7 @@ void *prepareNchar(char *rightData) {
int32_t inputLen = strlen(rightData); int32_t inputLen = strlen(rightData);
char *t = (char *)taosMemoryCalloc(1, (inputLen + 1) * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE); char *t = (char *)taosMemoryCalloc(1, (inputLen + 1) * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE);
taosMbsToUcs4(rightData, inputLen, (TdUcs4 *)varDataVal(t), inputLen * TSDB_NCHAR_SIZE, &len); taosMbsToUcs4(rightData, inputLen, (TdUcs4 *)varDataVal(t), inputLen * TSDB_NCHAR_SIZE, &len, NULL);
varDataSetLen(t, len); varDataSetLen(t, len);
return t; return t;
} }
@ -1623,7 +1623,7 @@ TEST(columnTest, json_column_logic_op) {
SArray *tags = taosArrayInit(1, sizeof(STagVal)); SArray *tags = taosArrayInit(1, sizeof(STagVal));
ASSERT_NE(tags, nullptr); ASSERT_NE(tags, nullptr);
STag *row = NULL; STag *row = NULL;
code = parseJsontoTagData(rightv, tags, &row, NULL); code = parseJsontoTagData(rightv, tags, &row, NULL, NULL);
ASSERT_EQ(code, TSDB_CODE_SUCCESS); ASSERT_EQ(code, TSDB_CODE_SUCCESS);
const int32_t len0 = 6; const int32_t len0 = 6;

8
source/os/src/osEnv.c
View File

@ -25,9 +25,9 @@ SDiskSpace tsLogSpace = {0};
SDiskSpace tsTempSpace = {0}; SDiskSpace tsTempSpace = {0};
char tsOsName[16] = {0}; char tsOsName[16] = {0};
char tsTimezoneStr[TD_TIMEZONE_LEN] = {0}; char tsTimezoneStr[TD_TIMEZONE_LEN] = {0};
enum TdTimezone tsTimezone = TdZeroZone;
char tsLocale[TD_LOCALE_LEN] = {0}; char tsLocale[TD_LOCALE_LEN] = {0};
char tsCharset[TD_CHARSET_LEN] = {0}; char tsCharset[TD_CHARSET_LEN] = {0};
void *tsCharsetCxt = NULL;
bool tsEnableCoreFile = 1; bool tsEnableCoreFile = 1;
int64_t tsPageSizeKB = 0; int64_t tsPageSizeKB = 0;
int64_t tsOpenMax = 0; int64_t tsOpenMax = 0;
@ -48,6 +48,7 @@ int32_t osDefaultInit() {
taosSeedRand(taosSafeRand()); taosSeedRand(taosSafeRand());
taosGetSystemLocale(tsLocale, tsCharset); taosGetSystemLocale(tsLocale, tsCharset);
(void)taosGetSystemTimezone(tsTimezoneStr);
taosGetSystemInfo(); taosGetSystemInfo();
@ -117,11 +118,6 @@ bool osTempSpaceSufficient() { return tsTempSpace.size.avail > tsTempSpace.reser
int32_t osSetTimezone(const char *tz) { return taosSetGlobalTimezone(tz); } int32_t osSetTimezone(const char *tz) { return taosSetGlobalTimezone(tz); }
void osSetSystemLocale(const char *inLocale, const char *inCharSet) {
if (inLocale) (void)memcpy(tsLocale, inLocale, strlen(inLocale) + 1);
if (inCharSet) (void)memcpy(tsCharset, inCharSet, strlen(inCharSet) + 1);
}
void osSetProcPath(int32_t argc, char **argv) { void osSetProcPath(int32_t argc, char **argv) {
if (argv == NULL || argc < 1) { if (argv == NULL || argc < 1) {
return; // no command line arguments return; // no command line arguments

63
source/os/src/osLocale.c
View File

@ -78,19 +78,17 @@ char *taosCharsetReplace(char *charsetstr) {
* *
* In case that the setLocale failed to be executed, the right charset needs to be set. * In case that the setLocale failed to be executed, the right charset needs to be set.
*/ */
int32_t taosSetSystemLocale(const char *inLocale, const char *inCharSet) { int32_t taosSetSystemLocale(const char *inLocale) {
OS_PARAM_CHECK(inLocale);
OS_PARAM_CHECK(inCharSet);
if (!taosValidateEncodec(inCharSet)) {
return terrno;
}
char *locale = setlocale(LC_CTYPE, inLocale); char *locale = setlocale(LC_CTYPE, inLocale);
if (NULL == locale) { if (NULL == locale) {
terrno = TSDB_CODE_INVALID_PARA; terrno = TSDB_CODE_INVALID_PARA;
uError("failed to set locale:%s", inLocale);
return terrno; return terrno;
} }
(void)memcpy(tsLocale, inLocale, strlen(inLocale) + 1);
return 0; return 0;
} }
@ -102,59 +100,6 @@ void taosGetSystemLocale(char *outLocale, char *outCharset) {
tstrncpy(outLocale, locale, TD_LOCALE_LEN); tstrncpy(outLocale, locale, TD_LOCALE_LEN);
} }
tstrncpy(outCharset, "UTF-8", TD_CHARSET_LEN); tstrncpy(outCharset, "UTF-8", TD_CHARSET_LEN);
#elif defined(_TD_DARWIN_64)
/*
* originally from src/os/src/detail/osSysinfo.c
* POSIX format locale string:
* (Language Strings)_(Country/Region Strings).(code_page)
*
* example: en_US.UTF-8, zh_CN.GB18030, zh_CN.UTF-8,
*
* if user does not specify the locale in taos.cfg the program use default LC_CTYPE as system locale.
*
* In case of some CentOS systems, their default locale is "en_US.utf8", which is not valid code_page
* for libiconv that is employed to convert string in this system. This program will automatically use
* UTF-8 instead as the charset.
*
* In case of windows client, the locale string is not valid POSIX format, user needs to set the
* correct code_page for libiconv. Usually, the code_page of windows system with simple chinese is
* CP936, CP437 for English charset.
*
*/
char sep = '.';
char *locale = NULL;
locale = setlocale(LC_CTYPE, "");
if (locale == NULL) {
// printf("can't get locale from system, set it to en_US.UTF-8 since error:%d:%s", errno, strerror(errno));
tstrncpy(outLocale, "en_US.UTF-8", TD_LOCALE_LEN);
} else {
tstrncpy(outLocale, locale, TD_LOCALE_LEN);
// printf("locale not configured, set to system default:%s", outLocale);
}
/* if user does not specify the charset, extract it from locale */
char *str = strrchr(outLocale, sep);
if (str != NULL) {
str++;
char *revisedCharset = taosCharsetReplace(str);
if (NULL == revisedCharset) {
tstrncpy(outCharset, "UTF-8", TD_CHARSET_LEN);
} else {
tstrncpy(outCharset, revisedCharset, TD_CHARSET_LEN);
taosMemoryFree(revisedCharset);
}
// printf("charset not configured, set to system default:%s", outCharset);
} else {
tstrncpy(outCharset, "UTF-8", TD_CHARSET_LEN);
// printf("can't get locale and charset from system, set it to UTF-8");
}
#else #else
/* /*
* POSIX format locale string: * POSIX format locale string:

126
source/os/src/osString.c
View File

@ -229,80 +229,25 @@ int32_t tasoUcs4Copy(TdUcs4 *target_ucs4, TdUcs4 *source_ucs4, int32_t len_ucs4)
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
typedef struct { iconv_t taosAcquireConv(int32_t *idx, ConvType type, void* charsetCxt) {
iconv_t conv;
int8_t inUse;
} SConv;
// 0: Mbs --> Ucs4
// 1: Ucs4--> Mbs
SConv *gConv[2] = {NULL, NULL};
int32_t convUsed[2] = {0, 0};
int32_t gConvMaxNum[2] = {0, 0};
int32_t taosConvInit(void) {
int8_t M2C = 0;
gConvMaxNum[M2C] = 512;
gConvMaxNum[1 - M2C] = 512;
gConv[M2C] = taosMemoryCalloc(gConvMaxNum[M2C], sizeof(SConv));
if (gConv[M2C] == NULL) {
return terrno;
}
gConv[1 - M2C] = taosMemoryCalloc(gConvMaxNum[1 - M2C], sizeof(SConv));
if (gConv[1 - M2C] == NULL) {
taosMemoryFree(gConv[M2C]);
return terrno;
}
for (int32_t i = 0; i < gConvMaxNum[M2C]; ++i) {
gConv[M2C][i].conv = iconv_open(DEFAULT_UNICODE_ENCODEC, tsCharset);
if ((iconv_t)-1 == gConv[M2C][i].conv) {
terrno = TAOS_SYSTEM_ERROR(errno);
return terrno;
}
}
for (int32_t i = 0; i < gConvMaxNum[1 - M2C]; ++i) {
gConv[1 - M2C][i].conv = iconv_open(tsCharset, DEFAULT_UNICODE_ENCODEC);
if ((iconv_t)-1 == gConv[1 - M2C][i].conv) {
terrno = TAOS_SYSTEM_ERROR(errno);
return terrno;
}
}
return 0;
}
void taosConvDestroy() {
int8_t M2C = 0;
for (int32_t i = 0; i < gConvMaxNum[M2C]; ++i) {
(void)iconv_close(gConv[M2C][i].conv);
}
for (int32_t i = 0; i < gConvMaxNum[1 - M2C]; ++i) {
(void)iconv_close(gConv[1 - M2C][i].conv);
}
taosMemoryFreeClear(gConv[M2C]);
taosMemoryFreeClear(gConv[1 - M2C]);
gConvMaxNum[M2C] = -1;
gConvMaxNum[1 - M2C] = -1;
}
iconv_t taosAcquireConv(int32_t *idx, ConvType type) {
if(idx == NULL) { if(idx == NULL) {
terrno = TSDB_CODE_INVALID_PARA; terrno = TSDB_CODE_INVALID_PARA;
return (iconv_t)-1; return (iconv_t)-1;
} }
if (gConvMaxNum[type] <= 0) { if (charsetCxt == NULL){
charsetCxt = tsCharsetCxt;
}
SConvInfo *info = (SConvInfo *)charsetCxt;
if (info->gConvMaxNum[type] <= 0) {
*idx = -1; *idx = -1;
if (type == M2C) { if (type == M2C) {
iconv_t c = iconv_open(DEFAULT_UNICODE_ENCODEC, tsCharset); iconv_t c = iconv_open(DEFAULT_UNICODE_ENCODEC, info->charset);
if ((iconv_t)-1 == c) { if ((iconv_t)-1 == c) {
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
} }
return c; return c;
} else { } else {
iconv_t c = iconv_open(tsCharset, DEFAULT_UNICODE_ENCODEC); iconv_t c = iconv_open(info->charset, DEFAULT_UNICODE_ENCODEC);
if ((iconv_t)-1 == c) { if ((iconv_t)-1 == c) {
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
} }
@ -311,9 +256,9 @@ iconv_t taosAcquireConv(int32_t *idx, ConvType type) {
} }
while (true) { while (true) {
int32_t used = atomic_add_fetch_32(&convUsed[type], 1); int32_t used = atomic_add_fetch_32(&info->convUsed[type], 1);
if (used > gConvMaxNum[type]) { if (used > info->gConvMaxNum[type]) {
used = atomic_sub_fetch_32(&convUsed[type], 1); (void)atomic_sub_fetch_32(&info->convUsed[type], 1);
(void)sched_yield(); (void)sched_yield();
continue; continue;
} }
@ -321,38 +266,43 @@ iconv_t taosAcquireConv(int32_t *idx, ConvType type) {
break; break;
} }
int32_t startId = taosGetSelfPthreadId() % gConvMaxNum[type]; int32_t startId = taosGetSelfPthreadId() % info->gConvMaxNum[type];
while (true) { while (true) {
if (gConv[type][startId].inUse) { if (info->gConv[type][startId].inUse) {
startId = (startId + 1) % gConvMaxNum[type]; startId = (startId + 1) % info->gConvMaxNum[type];
continue; continue;
} }
int8_t old = atomic_val_compare_exchange_8(&gConv[type][startId].inUse, 0, 1); int8_t old = atomic_val_compare_exchange_8(&info->gConv[type][startId].inUse, 0, 1);
if (0 == old) { if (0 == old) {
break; break;
} }
} }
*idx = startId; *idx = startId;
if ((iconv_t)0 == gConv[type][startId].conv) { if ((iconv_t)0 == info->gConv[type][startId].conv) {
return (iconv_t)-1; return (iconv_t)-1;
} else { } else {
return gConv[type][startId].conv; return info->gConv[type][startId].conv;
} }
} }
void taosReleaseConv(int32_t idx, iconv_t conv, ConvType type) { void taosReleaseConv(int32_t idx, iconv_t conv, ConvType type, void* charsetCxt) {
if (idx < 0) { if (idx < 0) {
(void)iconv_close(conv); (void)iconv_close(conv);
return; return;
} }
atomic_store_8(&gConv[type][idx].inUse, 0); if (charsetCxt == NULL){
(void)atomic_sub_fetch_32(&convUsed[type], 1); charsetCxt = tsCharsetCxt;
}
SConvInfo *info = (SConvInfo *)charsetCxt;
atomic_store_8(&info->gConv[type][idx].inUse, 0);
(void)atomic_sub_fetch_32(&info->convUsed[type], 1);
} }
bool taosMbsToUcs4(const char *mbs, size_t mbsLength, TdUcs4 *ucs4, int32_t ucs4_max_len, int32_t *len) { bool taosMbsToUcs4(const char *mbs, size_t mbsLength, TdUcs4 *ucs4, int32_t ucs4_max_len, int32_t *len, void* charsetCxt) {
if (ucs4_max_len == 0) { if (ucs4_max_len == 0) {
return true; return true;
} }
@ -368,20 +318,20 @@ bool taosMbsToUcs4(const char *mbs, size_t mbsLength, TdUcs4 *ucs4, int32_t ucs4
(void)memset(ucs4, 0, ucs4_max_len); (void)memset(ucs4, 0, ucs4_max_len);
int32_t idx = -1; int32_t idx = -1;
iconv_t conv = taosAcquireConv(&idx, M2C); iconv_t conv = taosAcquireConv(&idx, M2C, charsetCxt);
if ((iconv_t)-1 == conv) { if ((iconv_t)-1 == conv) {
return false; return false;
} }
size_t ucs4_input_len = mbsLength; size_t ucs4_input_len = mbsLength;
size_t outLeft = ucs4_max_len; size_t outLeft = ucs4_max_len;
if (iconv(conv, (char **)&mbs, &ucs4_input_len, (char **)&ucs4, &outLeft) == -1) { if (iconv(conv, (char **)&mbs, &ucs4_input_len, (char **)&ucs4, &outLeft) == -1) {
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
taosReleaseConv(idx, conv, M2C); taosReleaseConv(idx, conv, M2C, charsetCxt);
return false; return false;
} }
taosReleaseConv(idx, conv, M2C); taosReleaseConv(idx, conv, M2C, charsetCxt);
if (len != NULL) { if (len != NULL) {
*len = (int32_t)(ucs4_max_len - outLeft); *len = (int32_t)(ucs4_max_len - outLeft);
if (*len < 0) { if (*len < 0) {
@ -397,7 +347,7 @@ bool taosMbsToUcs4(const char *mbs, size_t mbsLength, TdUcs4 *ucs4, int32_t ucs4
// if success, return the number of bytes written to mbs ( >= 0) // if success, return the number of bytes written to mbs ( >= 0)
// otherwise return error code ( < 0) // otherwise return error code ( < 0)
int32_t taosUcs4ToMbs(TdUcs4 *ucs4, int32_t ucs4_max_len, char *mbs) { int32_t taosUcs4ToMbs(TdUcs4 *ucs4, int32_t ucs4_max_len, char *mbs, void* charsetCxt) {
if (ucs4_max_len == 0) { if (ucs4_max_len == 0) {
return 0; return 0;
} }
@ -413,22 +363,22 @@ int32_t taosUcs4ToMbs(TdUcs4 *ucs4, int32_t ucs4_max_len, char *mbs) {
int32_t idx = -1; int32_t idx = -1;
int32_t code = 0; int32_t code = 0;
iconv_t conv = taosAcquireConv(&idx, C2M); iconv_t conv = taosAcquireConv(&idx, C2M, charsetCxt);
if ((iconv_t)-1 == conv) { if ((iconv_t)-1 == conv) {
return terrno; return terrno;
} }
size_t ucs4_input_len = ucs4_max_len; size_t ucs4_input_len = ucs4_max_len;
size_t outLen = ucs4_max_len; size_t outLen = ucs4_max_len;
if (iconv(conv, (char **)&ucs4, &ucs4_input_len, &mbs, &outLen) == -1) { if (iconv(conv, (char **)&ucs4, &ucs4_input_len, &mbs, &outLen) == -1) {
code = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
taosReleaseConv(idx, conv, C2M); taosReleaseConv(idx, conv, C2M, charsetCxt);
terrno = code; terrno = code;
return code; return code;
} }
taosReleaseConv(idx, conv, C2M); taosReleaseConv(idx, conv, C2M, charsetCxt);
return (int32_t)(ucs4_max_len - outLen); return (int32_t)(ucs4_max_len - outLen);
#endif #endif
} }

10
source/os/src/osTimezone.c
View File

@ -828,6 +828,16 @@ int32_t taosSetGlobalTimezone(const char *tz) {
} }
int32_t taosGetLocalTimezoneOffset() {
time_t tx1 = taosGetTimestampSec();
struct tm tm1;
if (taosLocalTime(&tx1, &tm1, NULL, 0, NULL) == NULL) {
uError("%s failed to get local time: code:%d", __FUNCTION__, errno);
return TSDB_CODE_TIME_ERROR;
}
return (int32_t)(tm1.tm_gmtoff);
}
int32_t taosFormatTimezoneStr(time_t t, const char* tz, timezone_t sp, char *outTimezoneStr){ int32_t taosFormatTimezoneStr(time_t t, const char* tz, timezone_t sp, char *outTimezoneStr){
struct tm tm1; struct tm tm1;
if (taosLocalTime(&t, &tm1, NULL, 0, sp) == NULL) { if (taosLocalTime(&t, &tm1, NULL, 0, sp) == NULL) {

4
source/util/src/tcompare.c
View File

@ -1508,7 +1508,7 @@ int32_t comparewcsRegexMatch(const void *pString, const void *pPattern) {
return 1; // terrno has been set return 1; // terrno has been set
} }
int convertLen = taosUcs4ToMbs((TdUcs4 *)varDataVal(pPattern), len, pattern); int convertLen = taosUcs4ToMbs((TdUcs4 *)varDataVal(pPattern), len, pattern, NULL);
if (convertLen < 0) { if (convertLen < 0) {
taosMemoryFree(pattern); taosMemoryFree(pattern);
return 1; // terrno has been set return 1; // terrno has been set
@ -1523,7 +1523,7 @@ int32_t comparewcsRegexMatch(const void *pString, const void *pPattern) {
return 1; // terrno has been set return 1; // terrno has been set
} }
convertLen = taosUcs4ToMbs((TdUcs4 *)varDataVal(pString), len, str); convertLen = taosUcs4ToMbs((TdUcs4 *)varDataVal(pString), len, str, NULL);
if (convertLen < 0) { if (convertLen < 0) {
taosMemoryFree(str); taosMemoryFree(str);
taosMemoryFree(pattern); taosMemoryFree(pattern);

52
source/util/src/tconfig.c
View File

@ -24,6 +24,7 @@
#include "tlog.h" #include "tlog.h"
#include "tunit.h" #include "tunit.h"
#include "tutil.h" #include "tutil.h"
#include "tconv.h"
#define CFG_NAME_PRINT_LEN 32 #define CFG_NAME_PRINT_LEN 32
#define CFG_SRC_PRINT_LEN 12 #define CFG_SRC_PRINT_LEN 12
@ -254,10 +255,10 @@ static int32_t cfgSetTimezone(SConfigItem *pItem, const char *value, ECfgSrcType
uError("invalid timezone:%s", value); uError("invalid timezone:%s", value);
TAOS_RETURN(TSDB_CODE_INVALID_TIMEZONE); TAOS_RETURN(TSDB_CODE_INVALID_TIMEZONE);
} }
if (strlen(value) == 0) { if (value == NULL || strlen(value) == 0) {
uError("cfg:%s, type:%s src:%s, value:%s, skip to set timezone", pItem->name, cfgDtypeStr(pItem->dtype), uError("cfg:%s, type:%s src:%s, value:%s, skip to set timezone", pItem->name, cfgDtypeStr(pItem->dtype),
cfgStypeStr(stype), value); cfgStypeStr(stype), value);
TAOS_RETURN(TSDB_CODE_SUCCESS); TAOS_RETURN(TSDB_CODE_INVALID_CFG);
} }
TAOS_CHECK_RETURN(osSetTimezone(value)); TAOS_CHECK_RETURN(osSetTimezone(value));
@ -266,6 +267,49 @@ static int32_t cfgSetTimezone(SConfigItem *pItem, const char *value, ECfgSrcType
TAOS_RETURN(TSDB_CODE_SUCCESS); TAOS_RETURN(TSDB_CODE_SUCCESS);
} }
static int32_t cfgSetCharset(SConfigItem *pItem, const char *value, ECfgSrcType stype) {
if (stype == CFG_STYPE_ALTER_SERVER_CMD || stype == CFG_STYPE_ALTER_CLIENT_CMD){
uError("failed to config charset, not support");
TAOS_RETURN(TSDB_CODE_INVALID_CFG);
}
if (value == NULL || strlen(value) == 0) {
uError("cfg:%s, type:%s src:%s, value:%s, skip to set charset", pItem->name, cfgDtypeStr(pItem->dtype),
cfgStypeStr(stype), value);
TAOS_RETURN(TSDB_CODE_INVALID_CFG);
}
if (!taosValidateEncodec(value)) {
uError("invalid charset:%s", value);
TAOS_RETURN(terrno);
}
if ((tsCharsetCxt = taosConvInit(value)) == NULL) {
TAOS_RETURN(terrno);
}
(void)memcpy(tsCharset, value, strlen(value) + 1);
TAOS_CHECK_RETURN(doSetConf(pItem, value, stype));
TAOS_RETURN(TSDB_CODE_SUCCESS);
}
static int32_t cfgSetLocale(SConfigItem *pItem, const char *value, ECfgSrcType stype) {
if (stype == CFG_STYPE_ALTER_SERVER_CMD || (pItem->dynScope & CFG_DYN_CLIENT) == 0){
uError("failed to config locale, not support");
TAOS_RETURN(TSDB_CODE_INVALID_CFG);
}
if (value == NULL || strlen(value) == 0 || taosSetSystemLocale(value) != 0) {
uError("cfg:%s, type:%s src:%s, value:%s, skip to set locale", pItem->name, cfgDtypeStr(pItem->dtype),
cfgStypeStr(stype), value);
TAOS_RETURN(TSDB_CODE_INVALID_CFG);
}
TAOS_CHECK_RETURN(doSetConf(pItem, value, stype));
TAOS_RETURN(TSDB_CODE_SUCCESS);
}
static int32_t cfgSetTfsItem(SConfig *pCfg, const char *name, const char *value, const char *level, const char *primary, static int32_t cfgSetTfsItem(SConfig *pCfg, const char *name, const char *value, const char *level, const char *primary,
const char *disable, ECfgSrcType stype) { const char *disable, ECfgSrcType stype) {
(void)taosThreadMutexLock(&pCfg->lock); (void)taosThreadMutexLock(&pCfg->lock);
@ -378,11 +422,11 @@ int32_t cfgSetItem(SConfig *pCfg, const char *name, const char *value, ECfgSrcTy
break; break;
} }
case CFG_DTYPE_CHARSET: { case CFG_DTYPE_CHARSET: {
code = doSetConf(pItem, value, stype); code = cfgSetCharset(pItem, value, stype);
break; break;
} }
case CFG_DTYPE_LOCALE: { case CFG_DTYPE_LOCALE: {
code = doSetConf(pItem, value, stype); code = cfgSetLocale(pItem, value, stype);
break; break;
} }
case CFG_DTYPE_NONE: case CFG_DTYPE_NONE:

123
source/util/src/tconv.c Normal file
View File

@ -0,0 +1,123 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define ALLOW_FORBID_FUNC
#include "tconv.h"
#include "thash.h"
#include "osString.h"
#define CONV_MAX_NUM 32
SHashObj *gConvInfo = NULL;
// M2C: Mbs --> Ucs4
// C2M: Ucs4--> Mbs
static void taosConvDestroyInner(void *arg) {
SConvInfo *info = (SConvInfo *)arg;
if (info == NULL) {
return;
}
for (int32_t i = 0; i < info->gConvMaxNum[M2C]; ++i) {
(void)iconv_close(info->gConv[M2C][i].conv);
}
for (int32_t i = 0; i < info->gConvMaxNum[C2M]; ++i) {
(void)iconv_close(info->gConv[C2M][i].conv);
}
taosMemoryFreeClear(info->gConv[M2C]);
taosMemoryFreeClear(info->gConv[C2M]);
info->gConvMaxNum[M2C] = -1;
info->gConvMaxNum[C2M] = -1;
}
void* taosConvInit(const char* charset) {
if (charset == NULL){
terrno = TSDB_CODE_INVALID_PARA;
return NULL;
}
void* conv = NULL;
static int32_t lock_c = 0;
for (int i = 1; atomic_val_compare_exchange_32(&lock_c, 0, 1) != 0; ++i) {
if (i % 1000 == 0) {
uInfo("haven't acquire lock after spin %d times.", i);
(void)sched_yield();
}
}
if (gConvInfo == NULL){
gConvInfo = taosHashInit(0, MurmurHash3_32, false, HASH_ENTRY_LOCK);
if (gConvInfo == NULL) {
atomic_store_32(&lock_c, 0);
goto END;
}
taosHashSetFreeFp(gConvInfo, taosConvDestroyInner);
}
conv = taosHashGet(gConvInfo, charset, strlen(charset));
if (conv != NULL){
goto END;
}
SConvInfo info = {0};
info.gConvMaxNum[M2C] = CONV_MAX_NUM;
info.gConvMaxNum[C2M] = CONV_MAX_NUM;
tstrncpy(info.charset, charset, sizeof(info.charset));
info.gConv[M2C] = taosMemoryCalloc(info.gConvMaxNum[M2C], sizeof(SConv));
if (info.gConv[M2C] == NULL) {
goto FAILED;
}
info.gConv[C2M] = taosMemoryCalloc(info.gConvMaxNum[C2M], sizeof(SConv));
if (info.gConv[C2M] == NULL) {
goto FAILED;
}
for (int32_t i = 0; i < info.gConvMaxNum[M2C]; ++i) {
info.gConv[M2C][i].conv = iconv_open(DEFAULT_UNICODE_ENCODEC, charset);
if ((iconv_t)-1 == info.gConv[M2C][i].conv) {
terrno = TAOS_SYSTEM_ERROR(errno);
goto FAILED;
}
}
for (int32_t i = 0; i < info.gConvMaxNum[C2M]; ++i) {
info.gConv[C2M][i].conv = iconv_open(charset, DEFAULT_UNICODE_ENCODEC);
if ((iconv_t)-1 == info.gConv[C2M][i].conv) {
terrno = TAOS_SYSTEM_ERROR(errno);
goto FAILED;
}
}
int32_t code = taosHashPut(gConvInfo, charset, strlen(charset), &info, sizeof(info));
if (code != 0){
goto FAILED;
}
conv = taosHashGet(gConvInfo, charset, strlen(charset));
goto END;
FAILED:
taosConvDestroyInner(&info);
END:
atomic_store_32(&lock_c, 0);
return conv;
}
void taosConvDestroy() {
taosHashCleanup(gConvInfo);
}