diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index f899fc5589..cb05f98f45 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -154,7 +154,7 @@ int32_t tEncodeTag(SEncoder *pEncoder, const STag *pTag); int32_t tDecodeTag(SDecoder *pDecoder, STag **ppTag); int32_t tTagToValArray(const STag *pTag, SArray **ppArray); void debugPrintSTag(STag *pTag, const char *tag, int32_t ln); // TODO: remove -int32_t parseJsontoTagData(const char *json, SArray *pTagVals, STag **ppTag, void *pMsgBuf); +int32_t parseJsontoTagData(const char *json, SArray *pTagVals, STag **ppTag, void *pMsgBuf, void *charsetCxt); // SColData ================================ typedef struct { diff --git a/include/common/tmsg.h b/include/common/tmsg.h index a110e10351..42301773e9 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -3842,7 +3842,7 @@ typedef struct { int8_t igExists; int8_t intervalUnit; int8_t slidingUnit; - int8_t timezone; + int8_t timezone; // int8_t is not enough, timezone is unit of second int32_t dstVgId; // for stream int64_t interval; int64_t offset; diff --git a/include/common/ttime.h b/include/common/ttime.h index 41dd0bf6be..85ca5ae1e6 100644 --- a/include/common/ttime.h +++ b/include/common/ttime.h @@ -74,7 +74,7 @@ char getPrecisionUnit(int32_t precision); int64_t convertTimePrecision(int64_t ts, int32_t fromPrecision, int32_t toPrecision); int32_t convertTimeFromPrecisionToUnit(int64_t time, int32_t fromPrecision, char toUnit, int64_t* pRes); -int32_t convertStringToTimestamp(int16_t type, char* inputData, int64_t timePrec, int64_t* timeVal, timezone_t tz); +int32_t convertStringToTimestamp(int16_t type, char* inputData, int64_t timePrec, int64_t* timeVal, timezone_t tz, void* charsetCxt); int32_t getDuration(int64_t val, char unit, int64_t* result, int32_t timePrecision); int32_t taosFormatUtcTime(char* buf, int32_t bufLen, int64_t ts, int32_t precision); diff --git a/include/libs/function/function.h b/include/libs/function/function.h index cd3ad7597e..824b9cdada 100644 --- a/include/libs/function/function.h +++ b/include/libs/function/function.h @@ -293,6 +293,7 @@ struct SScalarParam { int32_t numOfRows; int32_t numOfQualified; // number of qualified elements in the final results timezone_t tz; + void *charsetCxt; }; #define cleanupResultRowEntry(p) p->initialized = false diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index e559d136e1..9da6117a0e 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -132,6 +132,7 @@ typedef struct SValueNode { int64_t typeData; int8_t unit; timezone_t tz; + void *charsetCxt; } SValueNode; typedef struct SLeftValueNode { @@ -161,6 +162,7 @@ typedef struct SOperatorNode { SNode* pLeft; SNode* pRight; timezone_t tz; + void* charsetCxt; } SOperatorNode; typedef struct SLogicConditionNode { @@ -195,6 +197,7 @@ typedef struct SFunctionNode { bool dual; // whether select stmt without from stmt, true for without. // char timezone[TD_TIMEZONE_LEN]; timezone_t tz; + void *charsetCxt; } SFunctionNode; typedef struct STableNode { @@ -404,6 +407,7 @@ typedef struct SCaseWhenNode { SNode* pElse; SNodeList* pWhenThenList; timezone_t tz; + void* charsetCxt; } SCaseWhenNode; typedef struct SWindowOffsetNode { diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index de390c6dd2..ebf84cff57 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -102,6 +102,7 @@ typedef struct SParseContext { SArray* pSubMetaList; setQueryFn setQueryFp; timezone_t timezone; + void *charsetCxt; } SParseContext; int32_t qParseSql(SParseContext* pCxt, SQuery** pQuery); @@ -140,27 +141,27 @@ void qDestroyStmtDataBlock(STableDataCxt* pBlock); STableMeta* qGetTableMetaInDataBlock(STableDataCxt* pDataBlock); int32_t qCloneCurrentTbData(STableDataCxt* pDataBlock, SSubmitTbData** pData); -int32_t qStmtBindParams(SQuery* pQuery, TAOS_MULTI_BIND* pParams, int32_t colIdx); +int32_t qStmtBindParams(SQuery* pQuery, TAOS_MULTI_BIND* pParams, int32_t colIdx, void *charsetCxt); int32_t qStmtParseQuerySql(SParseContext* pCxt, SQuery* pQuery); int32_t qBindStmtStbColsValue(void* pBlock, SArray* pCols, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen, - STSchema** pTSchema, SBindInfo* pBindInfos); -int32_t qBindStmtColsValue(void* pBlock, SArray* pCols, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen); + STSchema** pTSchema, SBindInfo* pBindInfos, void* charsetCxt); +int32_t qBindStmtColsValue(void* pBlock, SArray* pCols, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen, void* charsetCxt); int32_t qBindStmtSingleColValue(void* pBlock, SArray* pCols, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen, - int32_t colIdx, int32_t rowNum); + int32_t colIdx, int32_t rowNum, void* charsetCxt); int32_t qBuildStmtColFields(void* pDataBlock, int32_t* fieldNum, TAOS_FIELD_E** fields); int32_t qBuildStmtStbColFields(void* pBlock, int32_t* fieldNum, TAOS_FIELD_STB** fields); int32_t qBuildStmtTagFields(void* pBlock, void* boundTags, int32_t* fieldNum, TAOS_FIELD_E** fields); int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, const char* sTableName, char* tName, - TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen); + TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen, void* charsetCxt); -int32_t qStmtBindParams2(SQuery* pQuery, TAOS_STMT2_BIND* pParams, int32_t colIdx); +int32_t qStmtBindParams2(SQuery* pQuery, TAOS_STMT2_BIND* pParams, int32_t colIdx, void* charsetCxt); int32_t qBindStmtStbColsValue2(void* pBlock, SArray* pCols, TAOS_STMT2_BIND* bind, char* msgBuf, int32_t msgBufLen, - STSchema** pTSchema, SBindInfo2* pBindInfos); -int32_t qBindStmtColsValue2(void* pBlock, SArray* pCols, TAOS_STMT2_BIND* bind, char* msgBuf, int32_t msgBufLen); + STSchema** pTSchema, SBindInfo2* pBindInfos, void *charsetCxt); +int32_t qBindStmtColsValue2(void* pBlock, SArray* pCols, TAOS_STMT2_BIND* bind, char* msgBuf, int32_t msgBufLen, void *charsetCxt); int32_t qBindStmtSingleColValue2(void* pBlock, SArray* pCols, TAOS_STMT2_BIND* bind, char* msgBuf, int32_t msgBufLen, - int32_t colIdx, int32_t rowNum); + int32_t colIdx, int32_t rowNum, void *charsetCxt); int32_t qBindStmtTagsValue2(void* pBlock, void* boundTags, int64_t suid, const char* sTableName, char* tName, - TAOS_STMT2_BIND* bind, char* msgBuf, int32_t msgBufLen); + TAOS_STMT2_BIND* bind, char* msgBuf, int32_t msgBufLen, void *charsetCxt); void destroyBoundColumnInfo(void* pBoundInfo); int32_t qCreateSName(SName* pName, const char* pTableName, int32_t acctId, char* dbName, char* msgBuf, @@ -170,13 +171,13 @@ void qDestroyBoundColInfo(void* pInfo); int32_t smlInitHandle(SQuery** query); int32_t smlBuildRow(STableDataCxt* pTableCxt); -int32_t smlBuildCol(STableDataCxt* pTableCxt, SSchema* schema, void* kv, int32_t index); +int32_t smlBuildCol(STableDataCxt* pTableCxt, SSchema* schema, void* kv, int32_t index, void* charsetCxt); int32_t smlInitTableDataCtx(SQuery* query, STableMeta* pTableMeta, STableDataCxt** cxt); void clearColValArraySml(SArray* pCols); int32_t smlBindData(SQuery* handle, bool dataFormat, SArray* tags, SArray* colsSchema, SArray* cols, STableMeta* pTableMeta, char* tableName, const char* sTableName, int32_t sTableNameLen, int32_t ttl, - char* msgBuf, int32_t msgBufLen); + char* msgBuf, int32_t msgBufLen, void* charsetCxt); int32_t smlBuildOutput(SQuery* handle, SHashObj* pVgHash); int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreateTbReq* pCreateTb, void* fields, int numFields, bool needChangeLength, char* errstr, int32_t errstrLen, bool raw); diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index d2f714f400..de2588af0c 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -337,7 +337,7 @@ SSchema createSchema(int8_t type, int32_t bytes, col_id_t colId, const char* nam void destroyQueryExecRes(SExecResult* pRes); int32_t dataConverToStr(char* str, int64_t capacity, int type, void* buf, int32_t bufSize, int32_t* len); -void parseTagDatatoJson(void* p, char** jsonStr); +void parseTagDatatoJson(void* p, char** jsonStr, void *charsetCxt); int32_t cloneTableMeta(STableMeta* pSrc, STableMeta** pDst); void getColumnTypeFromMeta(STableMeta* pMeta, char* pName, ETableColumnType* pType); int32_t cloneDbVgInfo(SDBVgInfo* pSrc, SDBVgInfo** pDst); diff --git a/include/libs/scalar/scalar.h b/include/libs/scalar/scalar.h index 4b89a6a439..fd936dd087 100644 --- a/include/libs/scalar/scalar.h +++ b/include/libs/scalar/scalar.h @@ -105,7 +105,6 @@ int32_t timeTruncateFunction(SScalarParam *pInput, int32_t inputNum, SScalarPara int32_t timeDiffFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t nowFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t todayFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); -int32_t timeZoneStrLen(); int32_t timezoneFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t weekdayFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t dayofweekFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); diff --git a/include/os/osEnv.h b/include/os/osEnv.h index f235d3f235..85fdb7c38c 100644 --- a/include/os/osEnv.h +++ b/include/os/osEnv.h @@ -25,10 +25,9 @@ extern "C" { extern char tsOsName[]; extern char tsTimezoneStr[]; -extern enum TdTimezone tsTimezone; extern char tsCharset[]; +extern void *tsCharsetCxt; extern char tsLocale[]; -extern int8_t tsDaylight; extern bool tsEnableCoreFile; extern int64_t tsPageSizeKB; extern int64_t tsOpenMax; @@ -67,8 +66,7 @@ bool osDataSpaceSufficient(); bool osTempSpaceSufficient(); int32_t osSetTimezone(const char *timezone); -void osSetSystemLocale(const char *inLocale, const char *inCharSet); -void osSetProcPath(int32_t argc, char **argv); +void osSetProcPath(int32_t argc, char **argv); #ifdef __cplusplus } diff --git a/include/os/osLocale.h b/include/os/osLocale.h index 5f2a1c35de..d7a3540887 100644 --- a/include/os/osLocale.h +++ b/include/os/osLocale.h @@ -30,7 +30,7 @@ extern "C" { char *taosCharsetReplace(char *charsetstr); void taosGetSystemLocale(char *outLocale, char *outCharset); -int32_t taosSetSystemLocale(const char *inLocale, const char *inCharSet); +int32_t taosSetSystemLocale(const char *inLocale); #ifdef __cplusplus } diff --git a/include/os/osString.h b/include/os/osString.h index 995aa4a591..810c59131e 100644 --- a/include/os/osString.h +++ b/include/os/osString.h @@ -27,7 +27,19 @@ typedef int32_t TdUcs4; #else typedef void *iconv_t; #endif -typedef enum { M2C = 0, C2M } ConvType; +typedef enum { M2C = 0, C2M, CM_NUM } ConvType; + +typedef struct { + iconv_t conv; + int8_t inUse; +} SConv; + +typedef struct { + SConv *gConv[CM_NUM]; + int32_t convUsed[CM_NUM]; + int32_t gConvMaxNum[CM_NUM]; + char charset[TD_CHARSET_LEN]; +} SConvInfo; // If the error is in a third-party library, place this header file under the third-party library header file. // When you want to use this feature, you should find or add the same function in the following section. @@ -78,13 +90,11 @@ int32_t taosStr2int16(const char *str, int16_t *val); int32_t taosStr2int32(const char *str, int32_t *val); int32_t taosStr2int8(const char *str, int8_t *val); -int32_t taosConvInit(void); -void taosConvDestroy(); -iconv_t taosAcquireConv(int32_t *idx, ConvType type); -void taosReleaseConv(int32_t idx, iconv_t conv, ConvType type); -int32_t taosUcs4ToMbs(TdUcs4 *ucs4, int32_t ucs4_max_len, char *mbs); +iconv_t taosAcquireConv(int32_t *idx, ConvType type, void* charsetCxt); +void taosReleaseConv(int32_t idx, iconv_t conv, ConvType type, void* charsetCxt); +int32_t taosUcs4ToMbs(TdUcs4 *ucs4, int32_t ucs4_max_len, char *mbs, void* charsetCxt); int32_t taosUcs4ToMbsEx(TdUcs4 *ucs4, int32_t ucs4_max_len, char *mbs, iconv_t conv); -bool taosMbsToUcs4(const char *mbs, size_t mbs_len, TdUcs4 *ucs4, int32_t ucs4_max_len, int32_t *len); +bool taosMbsToUcs4(const char *mbs, size_t mbs_len, TdUcs4 *ucs4, int32_t ucs4_max_len, int32_t *len, void* charsetCxt); int32_t tasoUcs4Compare(TdUcs4 *f1_ucs4, TdUcs4 *f2_ucs4, int32_t bytes); int32_t tasoUcs4Copy(TdUcs4 *target_ucs4, TdUcs4 *source_ucs4, int32_t len_ucs4); bool taosValidateEncodec(const char *encodec); diff --git a/include/os/osTimezone.h b/include/os/osTimezone.h index 2e3d27212f..7057ced35a 100644 --- a/include/os/osTimezone.h +++ b/include/os/osTimezone.h @@ -20,33 +20,7 @@ extern "C" { #endif -enum TdTimezone { - TdWestZone12 = -12, - TdWestZone11, - TdWestZone10, - TdWestZone9, - TdWestZone8, - TdWestZone7, - TdWestZone6, - TdWestZone5, - TdWestZone4, - TdWestZone3, - TdWestZone2, - TdWestZone1, - TdZeroZone, - TdEastZone1, - TdEastZone2, - TdEastZone3, - TdEastZone4, - TdEastZone5, - TdEastZone6, - TdEastZone7, - TdEastZone8, - TdEastZone9, - TdEastZone10, - TdEastZone11, - TdEastZone12 -}; +#define TdEastZone8 8*60*60 typedef struct state *timezone_t; struct tm *localtime_rz(timezone_t , time_t const *, struct tm *); @@ -55,6 +29,7 @@ timezone_t tzalloc(char const *); void tzfree(timezone_t); void getTimezoneStr(char *tz); +int32_t taosGetLocalTimezoneOffset(); int32_t taosGetSystemTimezone(char *outTimezone); int32_t taosSetGlobalTimezone(const char *tz); int32_t taosFormatTimezoneStr(time_t t, const char* tzStr, timezone_t sp, char *outTimezoneStr); diff --git a/include/util/tconv.h b/include/util/tconv.h new file mode 100644 index 0000000000..c886b38f51 --- /dev/null +++ b/include/util/tconv.h @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_TCONV_H +#define TDENGINE_TCONV_H + +#ifdef __cplusplus +extern "C" { +#endif + +//#include "osString.h" +// +//bool taosValidateEncodec(const char *encodec); +//int32_t taosUcs4len(TdUcs4 *ucs4); +void* taosConvInit(const char* charset); +void taosConvDestroy(); +//iconv_t taosAcquireConv(int32_t *idx, ConvType type, void* charsetCxt); +//void taosReleaseConv(int32_t idx, iconv_t conv, ConvType type, void* charsetCxt); +//int32_t taosUcs4ToMbs(TdUcs4 *ucs4, int32_t ucs4_max_len, char *mbs, void* charsetCxt); +//int32_t taosUcs4ToMbsEx(TdUcs4 *ucs4, int32_t ucs4_max_len, char *mbs, iconv_t conv); +//bool taosMbsToUcs4(const char *mbs, size_t mbs_len, TdUcs4 *ucs4, int32_t ucs4_max_len, int32_t *len, void* charsetCxt); +//int32_t tasoUcs4Compare(TdUcs4 *f1_ucs4, TdUcs4 *f2_ucs4, int32_t bytes); +//int32_t tasoUcs4Copy(TdUcs4 *target_ucs4, TdUcs4 *source_ucs4, int32_t len_ucs4); + +#ifdef __cplusplus +} +#endif + +#endif // TDENGINE_TCONV_H diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index 46cc8a869c..a6ab0f8489 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -155,7 +155,7 @@ typedef struct { typedef struct { timezone_t timezone; - char charset[TD_CHARSET_LEN]; + void *charsetCxt; char app[TSDB_APP_NAME_LEN]; uint32_t ip; }SOptionInfo; @@ -219,6 +219,7 @@ typedef struct SReqResultInfo { int32_t precision; int32_t payloadLen; char* convertJson; + void* charsetCxt; } SReqResultInfo; typedef struct SRequestSendRecvBody { diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index dbc623214e..b67d0daf73 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -36,6 +36,7 @@ #include "tsched.h" #include "ttime.h" #include "tversion.h" +#include "tconv.h" #if defined(CUS_NAME) || defined(CUS_PROMPT) || defined(CUS_EMAIL) #include "cus_name.h" @@ -556,6 +557,7 @@ int32_t createRequest(uint64_t connId, int32_t type, int64_t reqid, SRequestObj (*pRequest)->metric.start = taosGetTimestampUs(); (*pRequest)->body.resInfo.convertUcs4 = true; // convert ucs4 by default + (*pRequest)->body.resInfo.charsetCxt = pTscObj->optionInfo.charsetCxt; (*pRequest)->type = type; (*pRequest)->allocatorRefId = -1; @@ -969,7 +971,12 @@ void taos_init_imp(void) { ENV_ERR_RET(taosInitCfg(configDir, NULL, NULL, NULL, NULL, 1), "failed to init cfg"); initQueryModuleMsgHandle(); - ENV_ERR_RET(taosConvInit(), "failed to init conv"); + if ((tsCharsetCxt = taosConvInit(tsCharset)) == NULL){ + tscInitRes = terrno; + tscError("failed to init conv"); + return; + } + ENV_ERR_RET(monitorInit(), "failed to init monitor"); ENV_ERR_RET(rpcInit(), "failed to init rpc"); diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 0620145bcc..26bef9301e 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -301,7 +301,8 @@ int32_t parseSql(SRequestObj* pRequest, bool topicQuery, SQuery** pQuery, SStmtC .nodeOffline = (pTscObj->pAppInfo->onlineDnodes < pTscObj->pAppInfo->totalDnodes), .isStmtBind = pRequest->isStmtBind, .setQueryFp = setQueryRequest, - .timezone = pTscObj->optionInfo.timezone,}; + .timezone = pTscObj->optionInfo.timezone, + .charsetCxt = pTscObj->optionInfo.charsetCxt,}; cxt.mgmtEpSet = getEpSet_s(&pTscObj->pAppInfo->mgmtEp); int32_t code = catalogGetHandle(pTscObj->pAppInfo->clusterId, &cxt.pCatalog); @@ -2087,7 +2088,7 @@ static int32_t doPrepareResPtr(SReqResultInfo* pResInfo) { static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t* colLength) { int32_t idx = -1; - iconv_t conv = taosAcquireConv(&idx, C2M); + iconv_t conv = taosAcquireConv(&idx, C2M, pResultInfo->charsetCxt); if (conv == (iconv_t)-1) return TSDB_CODE_TSC_INTERNAL_ERROR; for (int32_t i = 0; i < pResultInfo->numOfCols; ++i) { @@ -2097,7 +2098,7 @@ static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t* colLength) { if (type == TSDB_DATA_TYPE_NCHAR && colLength[i] > 0) { char* p = taosMemoryRealloc(pResultInfo->convertBuf[i], colLength[i]); if (p == NULL) { - taosReleaseConv(idx, conv, C2M); + taosReleaseConv(idx, conv, C2M, pResultInfo->charsetCxt); return terrno; } @@ -2114,7 +2115,7 @@ static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t* colLength) { "doConvertUCS4 error, invalid data. len:%d, bytes:%d, (p + len):%p, (pResultInfo->convertBuf[i] + " "colLength[i]):%p", len, bytes, (p + len), (pResultInfo->convertBuf[i] + colLength[i])); - taosReleaseConv(idx, conv, C2M); + taosReleaseConv(idx, conv, C2M, pResultInfo->charsetCxt); return TSDB_CODE_TSC_INTERNAL_ERROR; } @@ -2128,7 +2129,7 @@ static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t* colLength) { pResultInfo->row[i] = pResultInfo->pCol[i].pData; } } - taosReleaseConv(idx, conv, C2M); + taosReleaseConv(idx, conv, C2M, pResultInfo->charsetCxt); return TSDB_CODE_SUCCESS; } @@ -2293,7 +2294,7 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo) { varDataSetLen(dst, strlen(varDataVal(dst))); } else if (tTagIsJson(data)) { char* jsonString = NULL; - parseTagDatatoJson(data, &jsonString); + parseTagDatatoJson(data, &jsonString, pResultInfo->charsetCxt); if (jsonString == NULL) { tscError("doConvertJson error: parseTagDatatoJson failed"); return terrno; @@ -2303,7 +2304,7 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo) { } else if (jsonInnerType == TSDB_DATA_TYPE_NCHAR) { // value -> "value" *(char*)varDataVal(dst) = '\"'; int32_t length = taosUcs4ToMbs((TdUcs4*)varDataVal(jsonInnerData), varDataLen(jsonInnerData), - varDataVal(dst) + CHAR_BYTES); + varDataVal(dst) + CHAR_BYTES, pResultInfo->charsetCxt); if (length <= 0) { tscError("charset:%s to %s. convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset); length = 0; diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 10c337fb74..d368fb9a36 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -30,6 +30,7 @@ #include "tref.h" #include "trpc.h" #include "version.h" +#include "tconv.h" #define TSC_VAR_NOT_RELEASE 1 #define TSC_VAR_RELEASED 0 @@ -52,6 +53,49 @@ int taos_options(TSDB_OPTION option, const void *arg, ...) { return ret; } +static timezone_t setConnnectionTz(const char* val){ + timezone_t tz = NULL; + static int32_t lock_c = 0; + + for (int i = 1; atomic_val_compare_exchange_32(&lock_c, 0, 1) != 0; ++i) { + if (i % 1000 == 0) { + tscInfo("haven't acquire lock after spin %d times.", i); + (void)sched_yield(); + } + } + + if (pTimezoneMap == NULL){ + pTimezoneMap = taosHashInit(0, MurmurHash3_32, false, HASH_ENTRY_LOCK); + if (pTimezoneMap == NULL) { + atomic_store_32(&lock_c, 0); + goto END; + } + taosHashSetFreeFp(pTimezoneMap, (_hash_free_fn_t)tzfree); + } + + timezone_t *tmp = taosHashGet(pTimezoneMap, val, strlen(val)); + if (tmp != NULL && *tmp != NULL){ + tz = *tmp; + goto END; + } + + tscDebug("set timezone to %s", val); + tz = tzalloc(val); + if (tz == NULL) { + tscError("%s unknown timezone %s", __func__, val); + terrno = TAOS_SYSTEM_ERROR(errno); + goto END; + } + int32_t code = taosHashPut(pTimezoneMap, val, strlen(val), &tz, sizeof(timezone_t)); + if (code != 0){ + tzfree(tz); + tz = NULL; + } + +END: + atomic_store_32(&lock_c, 0); + return tz; +} static int32_t setConnectionOption(TAOS *taos, TSDB_OPTION_CONNECTION option, const char* val){ if (taos == NULL) { return TSDB_CODE_INVALID_PARA; @@ -74,32 +118,24 @@ static int32_t setConnectionOption(TAOS *taos, TSDB_OPTION_CONNECTION option, co code = terrno; goto END; } - tstrncpy(pObj->optionInfo.charset, val, TD_CHARSET_LEN); }else{ - pObj->optionInfo.charset[0] = 0; + val = tsCharset; } + void *tmp = taosConvInit(val); + if (tmp == NULL) { + code = terrno; + goto END; + } + pObj->optionInfo.charsetCxt = tmp; } else if (option == TSDB_OPTION_CONNECTION_TIMEZONE) { if (val != NULL){ if (strlen(val) == 0){ code = TSDB_CODE_INVALID_PARA; goto END; } - timezone_t *tmp = taosHashGet(pTimezoneMap, val, strlen(val)); - if (tmp && *tmp){ - pObj->optionInfo.timezone = *tmp; - goto END; - } - - tscDebug("set timezone to %s", val); - timezone_t tz = tzalloc(val); - if (!tz) { - tscError("%s unknown timezone %s", __func__, val); - code = TAOS_SYSTEM_ERROR(errno); - goto END; - } - code = taosHashPut(pTimezoneMap, val, strlen(val), &tz, sizeof(timezone_t)); - if (code != 0){ - tzfree(tz); + timezone_t tz = setConnnectionTz(val); + if (tz == NULL){ + code = terrno; goto END; } pObj->optionInfo.timezone = tz; @@ -126,26 +162,7 @@ END: } int taos_options_connection(TAOS *taos, TSDB_OPTION_CONNECTION option, const void *arg, ...){ - static int32_t lock_c = 0; - - for (int i = 1; atomic_val_compare_exchange_32(&lock_c, 0, 1) != 0; ++i) { - if (i % 1000 == 0) { - tscInfo("haven't acquire lock after spin %d times.", i); - (void)sched_yield(); - } - } - - if (pTimezoneMap == NULL){ - pTimezoneMap = taosHashInit(0, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); - if (pTimezoneMap == NULL) { - atomic_store_32(&lock_c, 0); - return terrno; - } - taosHashSetFreeFp(pTimezoneMap, (_hash_free_fn_t)tzfree); - } - int ret = setConnectionOption(taos, option, (const char *)arg); - atomic_store_32(&lock_c, 0); - return ret; + return setConnectionOption(taos, option, (const char *)arg); } // this function may be called by user or system, or by both simultaneously. @@ -1342,7 +1359,8 @@ int32_t createParseContext(const SRequestObj *pRequest, SParseContext **pCxt, SS .parseSqlFp = clientParseSql, .parseSqlParam = pWrapper, .setQueryFp = setQueryRequest, - .timezone = pTscObj->optionInfo.timezone}; + .timezone = pTscObj->optionInfo.timezone, + .charsetCxt = pTscObj->optionInfo.charsetCxt}; int8_t biMode = atomic_load_8(&((STscObj *)pTscObj)->biMode); (*pCxt)->biMode = biMode; return TSDB_CODE_SUCCESS; diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c index 1799f29eb4..fe6ae95b7f 100644 --- a/source/client/src/clientRawBlockWrite.c +++ b/source/client/src/clientRawBlockWrite.c @@ -414,7 +414,7 @@ static void buildChildElement(cJSON* json, SVCreateTbReq* pCreateReq) { goto end; } char* pJson = NULL; - parseTagDatatoJson(pTag, &pJson); + parseTagDatatoJson(pTag, &pJson, NULL); if (pJson == NULL) { uError("parseTagDatatoJson failed, pJson == NULL"); goto end; @@ -730,7 +730,7 @@ static void processAlterTable(SMqMetaRsp* metaRsp, cJSON** pJson) { uError("processAlterTable isJson false"); goto end; } - parseTagDatatoJson(vAlterTbReq.pTagVal, &buf); + parseTagDatatoJson(vAlterTbReq.pTagVal, &buf, NULL); if (buf == NULL) { uError("parseTagDatatoJson failed, buf == NULL"); goto end; diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 8602421ed0..e2f278400d 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -267,7 +267,7 @@ bool isSmlColAligned(SSmlHandle *info, int cnt, SSmlKv *kv) { goto END; } // bind data - int32_t ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, kv, cnt + 1); + int32_t ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, kv, cnt + 1, info->taos->optionInfo.charsetCxt); if (unlikely(ret != TSDB_CODE_SUCCESS)) { uDebug("smlBuildCol error, retry"); goto END; @@ -411,8 +411,8 @@ int32_t smlParseEndTelnetJsonFormat(SSmlHandle *info, SSmlLineInfo *elements, SS int32_t code = 0; int32_t lino = 0; uDebug("SML:0x%" PRIx64 " %s format true, ts:%" PRId64, info->id, __FUNCTION__ , kvTs->i); - SML_CHECK_CODE(smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, kvTs, 0)); - SML_CHECK_CODE(smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, kv, 1)); + SML_CHECK_CODE(smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, kvTs, 0, info->taos->optionInfo.charsetCxt)); + SML_CHECK_CODE(smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, kv, 1, info->taos->optionInfo.charsetCxt)); SML_CHECK_CODE(smlBuildRow(info->currTableDataCtx)); END: @@ -438,7 +438,7 @@ END: int32_t smlParseEndLine(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs) { if (info->dataFormat) { uDebug("SML:0x%" PRIx64 " %s format true, ts:%" PRId64, info->id, __FUNCTION__, kvTs->i); - int32_t ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, kvTs, 0); + int32_t ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, kvTs, 0, info->taos->optionInfo.charsetCxt); if (ret == TSDB_CODE_SUCCESS) { ret = smlBuildRow(info->currTableDataCtx); } @@ -1482,7 +1482,7 @@ static int32_t smlInsertData(SSmlHandle *info) { SML_CHECK_CODE(smlBindData(info->pQuery, info->dataFormat, tableData->tags, (*pMeta)->cols, tableData->cols, (*pMeta)->tableMeta, tableData->childTableName, measure, measureLen, info->ttl, info->msgBuf.buf, - info->msgBuf.len)); + info->msgBuf.len, info->taos->optionInfo.charsetCxt)); taosMemoryFreeClear(measure); oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, oneTable); } diff --git a/source/client/src/clientStmt.c b/source/client/src/clientStmt.c index e56d4cc4f6..fc16242b79 100644 --- a/source/client/src/clientStmt.c +++ b/source/client/src/clientStmt.c @@ -1071,7 +1071,7 @@ int stmtSetTbTags(TAOS_STMT* stmt, TAOS_MULTI_BIND* tags) { tscDebug("start to bind stmt tag values"); STMT_ERR_RET(qBindStmtTagsValue(*pDataBlock, pStmt->bInfo.boundTags, pStmt->bInfo.tbSuid, pStmt->bInfo.stbFName, pStmt->bInfo.sname.tname, tags, pStmt->exec.pRequest->msgBuf, - pStmt->exec.pRequest->msgBufLen)); + pStmt->exec.pRequest->msgBufLen, pStmt->taos->optionInfo.charsetCxt)); return TSDB_CODE_SUCCESS; } @@ -1239,7 +1239,7 @@ int stmtBindBatch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind, int32_t colIdx) { } if (STMT_TYPE_QUERY == pStmt->sql.type) { - STMT_ERR_RET(qStmtBindParams(pStmt->sql.pQuery, bind, colIdx)); + STMT_ERR_RET(qStmtBindParams(pStmt->sql.pQuery, bind, colIdx, pStmt->taos->optionInfo.charsetCxt)); SParseContext ctx = {.requestId = pStmt->exec.pRequest->requestId, .acctId = pStmt->taos->acctId, @@ -1325,10 +1325,10 @@ int stmtBindBatch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind, int32_t colIdx) { if (pStmt->sql.stbInterlaceMode) { (*pDataBlock)->pData->flags = 0; code = qBindStmtStbColsValue(*pDataBlock, pCols, bind, pStmt->exec.pRequest->msgBuf, - pStmt->exec.pRequest->msgBufLen, &pStmt->sql.siInfo.pTSchema, pStmt->sql.pBindInfo); + pStmt->exec.pRequest->msgBufLen, &pStmt->sql.siInfo.pTSchema, pStmt->sql.pBindInfo, pStmt->taos->optionInfo.charsetCxt); } else { code = - qBindStmtColsValue(*pDataBlock, pCols, bind, pStmt->exec.pRequest->msgBuf, pStmt->exec.pRequest->msgBufLen); + qBindStmtColsValue(*pDataBlock, pCols, bind, pStmt->exec.pRequest->msgBuf, pStmt->exec.pRequest->msgBufLen, pStmt->taos->optionInfo.charsetCxt); } if (code) { @@ -1353,7 +1353,7 @@ int stmtBindBatch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind, int32_t colIdx) { } code = qBindStmtSingleColValue(*pDataBlock, pCols, bind, pStmt->exec.pRequest->msgBuf, - pStmt->exec.pRequest->msgBufLen, colIdx, pStmt->bInfo.sBindRowNum); + pStmt->exec.pRequest->msgBufLen, colIdx, pStmt->bInfo.sBindRowNum, pStmt->taos->optionInfo.charsetCxt); if (code) { tscError("qBindStmtSingleColValue failed, error:%s", tstrerror(code)); STMT_ERR_RET(code); diff --git a/source/client/src/clientStmt2.c b/source/client/src/clientStmt2.c index 4bbfc6afaa..9dec5a481f 100644 --- a/source/client/src/clientStmt2.c +++ b/source/client/src/clientStmt2.c @@ -1014,7 +1014,7 @@ int stmtSetTbTags2(TAOS_STMT2* stmt, TAOS_STMT2_BIND* tags) { tscDebug("start to bind stmt tag values"); STMT_ERR_RET(qBindStmtTagsValue2(*pDataBlock, pStmt->bInfo.boundTags, pStmt->bInfo.tbSuid, pStmt->bInfo.stbFName, pStmt->bInfo.sname.tname, tags, pStmt->exec.pRequest->msgBuf, - pStmt->exec.pRequest->msgBufLen)); + pStmt->exec.pRequest->msgBufLen, pStmt->taos->optionInfo.charsetCxt)); return TSDB_CODE_SUCCESS; } @@ -1323,7 +1323,7 @@ int stmtBindBatch2(TAOS_STMT2* stmt, TAOS_STMT2_BIND* bind, int32_t colIdx) { } if (STMT_TYPE_QUERY == pStmt->sql.type) { - STMT_ERR_RET(qStmtBindParams2(pStmt->sql.pQuery, bind, colIdx)); + STMT_ERR_RET(qStmtBindParams2(pStmt->sql.pQuery, bind, colIdx, pStmt->taos->optionInfo.charsetCxt)); SParseContext ctx = {.requestId = pStmt->exec.pRequest->requestId, .acctId = pStmt->taos->acctId, @@ -1407,10 +1407,10 @@ int stmtBindBatch2(TAOS_STMT2* stmt, TAOS_STMT2_BIND* bind, int32_t colIdx) { if (pStmt->sql.stbInterlaceMode) { (*pDataBlock)->pData->flags = 0; code = qBindStmtStbColsValue2(*pDataBlock, pCols, bind, pStmt->exec.pRequest->msgBuf, - pStmt->exec.pRequest->msgBufLen, &pStmt->sql.siInfo.pTSchema, pStmt->sql.pBindInfo); + pStmt->exec.pRequest->msgBufLen, &pStmt->sql.siInfo.pTSchema, pStmt->sql.pBindInfo, pStmt->taos->optionInfo.charsetCxt); } else { code = - qBindStmtColsValue2(*pDataBlock, pCols, bind, pStmt->exec.pRequest->msgBuf, pStmt->exec.pRequest->msgBufLen); + qBindStmtColsValue2(*pDataBlock, pCols, bind, pStmt->exec.pRequest->msgBuf, pStmt->exec.pRequest->msgBufLen, pStmt->taos->optionInfo.charsetCxt); } if (code) { @@ -1435,7 +1435,7 @@ int stmtBindBatch2(TAOS_STMT2* stmt, TAOS_STMT2_BIND* bind, int32_t colIdx) { } code = qBindStmtSingleColValue2(*pDataBlock, pCols, bind, pStmt->exec.pRequest->msgBuf, - pStmt->exec.pRequest->msgBufLen, colIdx, pStmt->bInfo.sBindRowNum); + pStmt->exec.pRequest->msgBufLen, colIdx, pStmt->bInfo.sBindRowNum, pStmt->taos->optionInfo.charsetCxt); if (code) { tscError("qBindStmtSingleColValue failed, error:%s", tstrerror(code)); STMT_ERR_RET(code); diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 452c9f38ef..3b3c4b72d9 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -2641,7 +2641,7 @@ int32_t dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** pDataBuf char* pData = colDataGetVarData(pColInfoData, j); int32_t dataSize = TMIN(sizeof(pBuf), varDataLen(pData)); memset(pBuf, 0, sizeof(pBuf)); - code = taosUcs4ToMbs((TdUcs4*)varDataVal(pData), dataSize, pBuf); + code = taosUcs4ToMbs((TdUcs4*)varDataVal(pData), dataSize, pBuf, NULL); if (code < 0) { uError("func %s failed to convert to ucs charset since %s", __func__, tstrerror(code)); lino = __LINE__; diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 32faaf2bc2..b31c86bd82 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -654,7 +654,6 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { static int32_t taosAddSystemCfg(SConfig *pCfg) { SysNameInfo info = taosGetSysNameInfo(); - (void)taosGetSystemTimezone(tsTimezoneStr); TAOS_CHECK_RETURN(cfgAddTimezone(pCfg, "timezone", tsTimezoneStr, CFG_SCOPE_BOTH, CFG_DYN_CLIENT)); TAOS_CHECK_RETURN(cfgAddLocale(pCfg, "locale", tsLocale, CFG_SCOPE_BOTH, CFG_DYN_CLIENT)); TAOS_CHECK_RETURN(cfgAddCharset(pCfg, "charset", tsCharset, CFG_SCOPE_BOTH, CFG_DYN_NONE)); @@ -1317,25 +1316,6 @@ static int32_t taosSetClientCfg(SConfig *pCfg) { static int32_t taosSetSystemCfg(SConfig *pCfg) { SConfigItem *pItem = NULL; - TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "locale"); - const char *locale = pItem->str; - - TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "charset"); - const char *charset = pItem->str; - - int32_t code = taosSetSystemLocale(locale, charset); - if (TSDB_CODE_SUCCESS != code) { - uError("failed to set locale:%s, since: %s", locale, tstrerror(code)); - char curLocale[TD_LOCALE_LEN] = {0}; - char curCharset[TD_CHARSET_LEN] = {0}; - taosGetSystemLocale(curLocale, curCharset); - if (0 != strlen(curLocale) && 0 != strlen(curCharset)) { - uInfo("current locale: %s, charset: %s", curLocale, curCharset); - } - } - - osSetSystemLocale(locale, charset); - TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "enableCoreFile"); tsEnableCoreFile = pItem->bval; taosSetCoreDump(tsEnableCoreFile); @@ -2062,6 +2042,10 @@ static int32_t taosCfgDynamicOptionsForClient(SConfig *pCfg, const char *name) { int32_t code = TSDB_CODE_SUCCESS; int32_t lino = 0; + if (strcasecmp("locale", name) == 0 || strcasecmp("charset", name) == 0 + || strcasecmp("timezone", name) == 0) { + goto _out; + } cfgLock(pCfg); SConfigItem *pItem = cfgGetItem(pCfg, name); @@ -2150,25 +2134,6 @@ static int32_t taosCfgDynamicOptionsForClient(SConfig *pCfg, const char *name) { } break; } - case 'l': { - if (strcasecmp("locale", name) == 0) { - SConfigItem *pLocaleItem = cfgGetItem(pCfg, "locale"); - SConfigItem *pCharsetItem = cfgGetItem(pCfg, "charset"); - if (pLocaleItem == NULL || pCharsetItem == NULL) { - uError("failed to get locale or charset from cfg"); - code = TSDB_CODE_CFG_NOT_FOUND; - goto _out; - } - - const char *locale = pLocaleItem->str; - const char *charset = pCharsetItem->str; - TAOS_CHECK_GOTO(taosSetSystemLocale(locale, charset), &lino, _out); - osSetSystemLocale(locale, charset); - uInfo("locale set to '%s', charset set to '%s'", locale, charset); - matched = true; - } - break; - } case 'm': { if (strcasecmp("metaCacheMaxSize", name) == 0) { atomic_store_32(&tsMetaCacheMaxSize, pItem->i32); @@ -2239,9 +2204,7 @@ static int32_t taosCfgDynamicOptionsForClient(SConfig *pCfg, const char *name) { break; } case 't': { - if (strcasecmp("timezone", name) == 0) { - matched = true; - } else if (strcasecmp("tempDir", name) == 0) { + if (strcasecmp("tempDir", name) == 0) { uInfo("%s set from %s to %s", name, tsTempDir, pItem->str); tstrncpy(tsTempDir, pItem->str, PATH_MAX); TAOS_CHECK_GOTO(taosExpandDir(tsTempDir, tsTempDir, PATH_MAX), &lino, _out); diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c index 574b2d2fce..893c03335e 100644 --- a/source/common/src/ttime.c +++ b/source/common/src/ttime.c @@ -480,7 +480,7 @@ int32_t convertTimeFromPrecisionToUnit(int64_t time, int32_t fromPrecision, char TAOS_RETURN(TSDB_CODE_SUCCESS); } -int32_t convertStringToTimestamp(int16_t type, char* inputData, int64_t timePrec, int64_t* timeVal, timezone_t tz) { +int32_t convertStringToTimestamp(int16_t type, char* inputData, int64_t timePrec, int64_t* timeVal, timezone_t tz, void* charsetCxt) { int32_t charLen = varDataLen(inputData); char* newColData; if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_VARBINARY) { @@ -500,7 +500,7 @@ int32_t convertStringToTimestamp(int16_t type, char* inputData, int64_t timePrec if (NULL == newColData) { TAOS_RETURN(terrno); } - int len = taosUcs4ToMbs((TdUcs4*)varDataVal(inputData), charLen, newColData); + int len = taosUcs4ToMbs((TdUcs4*)varDataVal(inputData), charLen, newColData, charsetCxt); if (len < 0) { taosMemoryFree(newColData); TAOS_RETURN(TSDB_CODE_FAILED); @@ -2011,7 +2011,11 @@ int64_t taosGetTimestampToday(int32_t precision, timezone_t tz) { : (precision == TSDB_TIME_PRECISION_MILLI) ? 1000 : (precision == TSDB_TIME_PRECISION_MICRO) ? 1000000 : 1000000000; - time_t t = taosTime(NULL); + time_t t; + int32_t code = taosTime(&t); + if (code != 0) { + return -1; + } struct tm tm; if (taosLocalTime(&t, &tm, NULL, 0, tz) == NULL){ uError("%s failed to get local time, code:%d", __FUNCTION__, errno); diff --git a/source/common/test/commonTests.cpp b/source/common/test/commonTests.cpp index 19fe6b00d3..a6ffe2cd40 100644 --- a/source/common/test/commonTests.cpp +++ b/source/common/test/commonTests.cpp @@ -477,7 +477,7 @@ void test_ts2char(int64_t ts, const char* format, int32_t precison, const char* TEST(timeTest, ts2char) { osDefaultInit(); - if (tsTimezone != TdEastZone8) GTEST_SKIP(); + if (taosGetLocalTimezoneOffset() != TdEastZone8) GTEST_SKIP(); int64_t ts; const char* format = "YYYY-MM-DD"; ts = 0; @@ -529,7 +529,7 @@ TEST(timeTest, ts2char) { TEST(timeTest, char2ts) { osDefaultInit(); - if (tsTimezone != TdEastZone8) GTEST_SKIP(); + if (taosGetLocalTimezoneOffset() != TdEastZone8) GTEST_SKIP(); int64_t ts; int32_t code = TEST_char2ts("YYYY-DD-MM HH12:MI:SS:MSPM", &ts, TSDB_TIME_PRECISION_MILLI, "2023-10-10 12:00:00.000AM"); @@ -630,7 +630,7 @@ TEST(timeTest, char2ts) { // default to 1970-1-1 00:00:00+08 -> 1969-12-31 16:00:00+00 ASSERT_EQ(0, TEST_char2ts("YYYY", &ts, TSDB_TIME_PRECISION_SECONDS, "1970")); - ASSERT_EQ(ts, -1 * tsTimezone * 60 * 60); + ASSERT_EQ(ts, -1 * taosGetLocalTimezoneOffset()); ASSERT_EQ(0, TEST_char2ts("yyyyMM1/dd ", &ts, TSDB_TIME_PRECISION_MICRO, "210001/2")); ASSERT_EQ(ts, 4102502400000000LL); diff --git a/source/dnode/mgmt/exe/dmMain.c b/source/dnode/mgmt/exe/dmMain.c index ade5e16894..e9911bb25d 100644 --- a/source/dnode/mgmt/exe/dmMain.c +++ b/source/dnode/mgmt/exe/dmMain.c @@ -20,6 +20,7 @@ #include "tconfig.h" #include "tglobal.h" #include "version.h" +#include "tconv.h" #ifdef TD_JEMALLOC_ENABLED #include "jemalloc/jemalloc.h" #endif @@ -444,7 +445,7 @@ int mainWindows(int argc, char **argv) { return code; } - if ((code = taosConvInit()) != 0) { + if ((tsCharsetCxt = taosConvInit(tsCharset)) == NULL) { dError("failed to init conv"); taosCloseLog(); taosCleanupArgs(); diff --git a/source/dnode/mgmt/node_mgmt/src/dmMgmt.c b/source/dnode/mgmt/node_mgmt/src/dmMgmt.c index 9f1c292a90..f11e1508a8 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmMgmt.c +++ b/source/dnode/mgmt/node_mgmt/src/dmMgmt.c @@ -22,6 +22,7 @@ #include "tglobal.h" #include "tgrant.h" #include "tstream.h" +#include "tconv.h" static bool dmRequireNode(SDnode *pDnode, SMgmtWrapper *pWrapper) { SMgmtInputOpt input = dmBuildMgmtInputOpt(pWrapper); diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index d2d9b2e8eb..65708628b3 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -489,7 +489,7 @@ typedef struct { int64_t dstTbUid; int8_t intervalUnit; int8_t slidingUnit; - int8_t timezone; + int8_t timezone; // int8_t is not enough, timezone is unit of second int32_t dstVgId; // for stream int64_t interval; int64_t offset; diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index a54c7f1b14..d0872009fd 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -316,7 +316,7 @@ static void *mndBuildVCreateSmaReq(SMnode *pMnode, SVgObj *pVgroup, SSmaObj *pSm req.version = 0; req.intervalUnit = pSma->intervalUnit; req.slidingUnit = pSma->slidingUnit; - req.timezoneInt = pSma->timezone; +// req.timezoneInt = pSma->timezone; tstrncpy(req.indexName, (char *)tNameGetTableName(&name), TSDB_INDEX_NAME_LEN); req.exprLen = pSma->exprLen; req.tagsFilterLen = pSma->tagsFilterLen; @@ -617,9 +617,9 @@ static int32_t mndCreateSma(SMnode *pMnode, SRpcMsg *pReq, SMCreateSmaReq *pCrea smaObj.intervalUnit = pCreate->intervalUnit; smaObj.slidingUnit = pCreate->slidingUnit; #if 0 - smaObj.timezone = pCreate->timezone; +// smaObj.timezone = pCreate->timezone; #endif - smaObj.timezone = tsTimezone; // use timezone of server +// smaObj.timezone = taosGetLocalTimezoneOffset(); // use timezone of server smaObj.interval = pCreate->interval; smaObj.offset = pCreate->offset; smaObj.sliding = pCreate->sliding; @@ -1554,7 +1554,7 @@ static void initSMAObj(SCreateTSMACxt* pCxt) { pCxt->pSma->dbUid = pCxt->pDb->uid; pCxt->pSma->interval = pCxt->pCreateSmaReq->interval; pCxt->pSma->intervalUnit = pCxt->pCreateSmaReq->intervalUnit; - pCxt->pSma->timezone = tsTimezone; +// pCxt->pSma->timezone = taosGetLocalTimezoneOffset(); pCxt->pSma->version = 1; pCxt->pSma->exprLen = pCxt->pCreateSmaReq->exprLen; diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index e2ba8d9ccb..87beb8842b 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -1328,7 +1328,7 @@ int32_t metaFilterTableIds(void *pVnode, SMetaFltParam *arg, SArray *pUids) { TAOS_CHECK_GOTO(terrno, NULL, END); } - if (false == taosMbsToUcs4(tagData, nTagData, (TdUcs4 *)buf, maxSize, &maxSize)) { + if (false == taosMbsToUcs4(tagData, nTagData, (TdUcs4 *)buf, maxSize, &maxSize, NULL)) { TAOS_CHECK_GOTO(terrno, NULL, END); } diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 5c3516a962..3414d18590 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -153,7 +153,7 @@ static int metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const if (val == NULL) { TAOS_CHECK_GOTO(terrno, NULL, _exception); } - int32_t len = taosUcs4ToMbs((TdUcs4 *)pTagVal->pData, pTagVal->nData, val + VARSTR_HEADER_SIZE); + int32_t len = taosUcs4ToMbs((TdUcs4 *)pTagVal->pData, pTagVal->nData, val + VARSTR_HEADER_SIZE, NULL); if (len < 0) { TAOS_CHECK_GOTO(len, NULL, _exception); } @@ -237,7 +237,7 @@ int metaDelJsonVarFromIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSche if (val == NULL) { TAOS_CHECK_GOTO(terrno, NULL, _exception); } - int32_t len = taosUcs4ToMbs((TdUcs4 *)pTagVal->pData, pTagVal->nData, val + VARSTR_HEADER_SIZE); + int32_t len = taosUcs4ToMbs((TdUcs4 *)pTagVal->pData, pTagVal->nData, val + VARSTR_HEADER_SIZE, NULL); if (len < 0) { TAOS_CHECK_GOTO(len, NULL, _exception); } diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 0ea1d98312..c194b51b07 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -447,7 +447,7 @@ int32_t ctgGetTbTag(SCatalog* pCtg, SRequestConnInfo* pConn, SName* pTableName, } char* pJson = NULL; - parseTagDatatoJson(pTag, &pJson); + parseTagDatatoJson(pTag, &pJson, NULL); if(NULL == pJson) { taosArrayDestroy(pTagVals); CTG_ERR_JRET(terrno); diff --git a/source/libs/catalog/src/ctgAsync.c b/source/libs/catalog/src/ctgAsync.c index 9bfb4102aa..e3f7ad4811 100644 --- a/source/libs/catalog/src/ctgAsync.c +++ b/source/libs/catalog/src/ctgAsync.c @@ -2340,7 +2340,7 @@ int32_t ctgHandleGetTbTagRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* } char* pJson = NULL; - parseTagDatatoJson(pTag, &pJson); + parseTagDatatoJson(pTag, &pJson, NULL); if (NULL == pJson) { taosArrayDestroy(pTagVals); CTG_ERR_JRET(terrno); diff --git a/source/libs/command/src/command.c b/source/libs/command/src/command.c index 94c000991a..b86621738d 100644 --- a/source/libs/command/src/command.c +++ b/source/libs/command/src/command.c @@ -585,7 +585,7 @@ static int32_t appendTagValues(char* buf, int32_t* len, STableCfg* pCfg) { if (tTagIsJson(pTag)) { char* pJson = NULL; - parseTagDatatoJson(pTag, &pJson); + parseTagDatatoJson(pTag, &pJson, NULL); if (NULL == pJson) { qError("failed to parse tag to json, pJson is NULL"); return terrno; diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c index af5313297e..00b4de4625 100644 --- a/source/libs/executor/src/sysscanoperator.c +++ b/source/libs/executor/src/sysscanoperator.c @@ -966,7 +966,7 @@ int32_t convertTagDataToStr(char* str, int32_t strBuffLen, int type, void* buf, return TSDB_CODE_TSC_INVALID_VALUE; } - int32_t length = taosUcs4ToMbs((TdUcs4*)buf, bufSize, str); + int32_t length = taosUcs4ToMbs((TdUcs4*)buf, bufSize, str, NULL); if (length <= 0) { return TSDB_CODE_TSC_INVALID_VALUE; } @@ -1117,7 +1117,7 @@ static int32_t sysTableUserTagsFillOneTableTags(const SSysTableScanInfo* pInfo, if (tagData != NULL) { if (tagType == TSDB_DATA_TYPE_JSON) { char* tagJson = NULL; - parseTagDatatoJson(tagData, &tagJson); + parseTagDatatoJson(tagData, &tagJson, NULL); if (tagJson == NULL) { code = terrno; goto _end; diff --git a/source/libs/executor/test/queryPlanTests.cpp b/source/libs/executor/test/queryPlanTests.cpp index 6710435aba..cb6020ad29 100755 --- a/source/libs/executor/test/queryPlanTests.cpp +++ b/source/libs/executor/test/queryPlanTests.cpp @@ -592,7 +592,7 @@ void qptGetRandValue(uint8_t* pType, int32_t* pLen, void** ppVal) { memset(pTmp, 'A' + taosRand() % 26, *pLen); *ppVal = taosMemoryCalloc(1, *pLen * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE); assert(*ppVal); - assert(taosMbsToUcs4(pTmp, *pLen, (TdUcs4 *)varDataVal(*ppVal), *pLen * TSDB_NCHAR_SIZE, NULL)); + assert(taosMbsToUcs4(pTmp, *pLen, (TdUcs4 *)varDataVal(*ppVal), *pLen * TSDB_NCHAR_SIZE, NULL, NULL)); *pLen *= TSDB_NCHAR_SIZE; varDataSetLen(*ppVal, *pLen); taosMemoryFree(pTmp); diff --git a/source/libs/executor/test/sortTests.cpp b/source/libs/executor/test/sortTests.cpp index b2313f35a1..ce2da01b66 100644 --- a/source/libs/executor/test/sortTests.cpp +++ b/source/libs/executor/test/sortTests.cpp @@ -93,7 +93,7 @@ SSDataBlock* getSingleColDummyBlock(void* param) { char strOri[128] = {0}; taosRandStr(strOri, size); int32_t len = 0; - bool ret = taosMbsToUcs4(strOri, size, (TdUcs4*)varDataVal(str), size * TSDB_NCHAR_SIZE, &len); + bool ret = taosMbsToUcs4(strOri, size, (TdUcs4*)varDataVal(str), size * TSDB_NCHAR_SIZE, &len, NULL); if (!ret) { (void) printf("error\n"); return NULL; @@ -331,7 +331,7 @@ TEST(testCase, external_mem_sort_Test) { if(pInfo[i].type == TSDB_DATA_TYPE_NCHAR){ char buf[128] = {0}; - int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(v), varDataLen(v), buf); + int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(v), varDataLen(v), buf, NULL); printf("%d: %s\n", row++, buf); }else if(pInfo[i].type == TSDB_DATA_TYPE_BINARY || pInfo[i]->type == TSDB_DATA_TYPE_GEOMETRY){ char buf[128] = {0}; diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index fe570eb212..acfdf87d33 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -1664,7 +1664,7 @@ static int32_t translateOutVarchar(SFunctionNode* pFunc, char* pErrBuf, int32_t bytes = TSDB_TABLE_FNAME_LEN - 1 + VARSTR_HEADER_SIZE; break; case FUNCTION_TYPE_TIMEZONE: - bytes = timeZoneStrLen(); + bytes = TD_TIMEZONE_LEN + VARSTR_HEADER_SIZE; break; case FUNCTION_TYPE_IRATE_PARTIAL: bytes = getIrateInfoSize((pFunc->hasPk) ? pFunc->pkBytes : 0) + VARSTR_HEADER_SIZE; diff --git a/source/libs/parser/inc/parUtil.h b/source/libs/parser/inc/parUtil.h index db9395ea7f..aec5fb4cd6 100644 --- a/source/libs/parser/inc/parUtil.h +++ b/source/libs/parser/inc/parUtil.h @@ -136,7 +136,7 @@ int32_t trimString(const char* src, int32_t len, char* dst, int32_t dlen); int32_t getVnodeSysTableTargetName(int32_t acctId, SNode* pWhere, SName* pName); int32_t checkAndTrimValue(SToken* pToken, char* tmpTokenBuf, SMsgBuf* pMsgBuf, int8_t type); int32_t parseTagValue(SMsgBuf* pMsgBuf, const char** pSql, uint8_t precision, SSchema* pTagSchema, SToken* pToken, - SArray* pTagName, SArray* pTagVals, STag** pTag, timezone_t tz); + SArray* pTagName, SArray* pTagVals, STag** pTag, timezone_t tz, void *charsetCxt); int32_t parseTbnameToken(SMsgBuf* pMsgBuf, char* tname, SToken* pToken, bool* pFoundCtbName); int32_t buildCatalogReq(const SParseMetaCache* pMetaCache, SCatalogReq* pCatalogReq); diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index 5888bb0021..1204c116a6 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -421,6 +421,7 @@ SNode* createValueNode(SAstCreateContext* pCxt, int32_t dataType, const SToken* } val->translate = false; val->tz = pCxt->pQueryCxt->timezone; + val->charsetCxt = pCxt->pQueryCxt->charsetCxt; return (SNode*)val; _err: return NULL; @@ -964,6 +965,8 @@ SNode* createOperatorNode(SAstCreateContext* pCxt, EOperatorType type, SNode* pL op->opType = type; op->pLeft = pLeft; op->pRight = pRight; + op->tz = pCxt->pQueryCxt->timezone; + op->charsetCxt = pCxt->pQueryCxt->charsetCxt; return (SNode*)op; _err: nodesDestroyNode(pLeft); @@ -1043,6 +1046,7 @@ SNode* createFunctionNode(SAstCreateContext* pCxt, const SToken* pFuncName, SNod COPY_STRING_FORM_ID_TOKEN(func->functionName, pFuncName); func->pParameterList = pParameterList; func->tz = pCxt->pQueryCxt->timezone; + func->charsetCxt = pCxt->pQueryCxt->charsetCxt; return (SNode*)func; _err: nodesDestroyList(pParameterList); @@ -1064,6 +1068,8 @@ SNode* createCastFunctionNode(SAstCreateContext* pCxt, SNode* pExpr, SDataType d pCxt->errCode = nodesListMakeAppend(&func->pParameterList, pExpr); CHECK_PARSER_STATUS(pCxt); func->tz = pCxt->pQueryCxt->timezone; + func->charsetCxt = pCxt->pQueryCxt->charsetCxt; + return (SNode*)func; _err: nodesDestroyNode((SNode*)func); @@ -1098,6 +1104,7 @@ SNode* createTrimFunctionNode(SAstCreateContext* pCxt, SNode* pExpr, ETrimType t func->trimType = type; pCxt->errCode = nodesListMakeAppend(&func->pParameterList, pExpr); CHECK_PARSER_STATUS(pCxt); + func->charsetCxt = pCxt->pQueryCxt->charsetCxt; return (SNode*)func; _err: nodesDestroyNode((SNode*)func); @@ -1116,6 +1123,7 @@ SNode* createTrimFunctionNodeExt(SAstCreateContext* pCxt, SNode* pExpr, SNode* p CHECK_PARSER_STATUS(pCxt); pCxt->errCode = nodesListMakeAppend(&func->pParameterList, pExpr2); CHECK_PARSER_STATUS(pCxt); + func->charsetCxt = pCxt->pQueryCxt->charsetCxt; return (SNode*)func; _err: nodesDestroyNode((SNode*)func); @@ -1501,6 +1509,7 @@ SNode* createCaseWhenNode(SAstCreateContext* pCxt, SNode* pCase, SNodeList* pWhe pCaseWhen->pWhenThenList = pWhenThenList; pCaseWhen->pElse = pElse; pCaseWhen->tz = pCxt->pQueryCxt->timezone; + pCaseWhen->charsetCxt = pCxt->pQueryCxt->charsetCxt; return (SNode*)pCaseWhen; _err: nodesDestroyNode(pCase); diff --git a/source/libs/parser/src/parInsertSml.c b/source/libs/parser/src/parInsertSml.c index b5cdf1e4ee..22d1f7edda 100644 --- a/source/libs/parser/src/parInsertSml.c +++ b/source/libs/parser/src/parInsertSml.c @@ -98,7 +98,7 @@ end: * @return int32_t */ static int32_t smlBuildTagRow(SArray* cols, SBoundColInfo* tags, SSchema* pSchema, STag** ppTag, SArray** tagName, - SMsgBuf* msg) { + SMsgBuf* msg, void* charsetCxt) { SArray* pTagArray = taosArrayInit(tags->numOfBound, sizeof(STagVal)); if (!pTagArray) { return terrno; @@ -142,7 +142,7 @@ static int32_t smlBuildTagRow(SArray* cols, SBoundColInfo* tags, SSchema* pSchem code = terrno; goto end; } - if (!taosMbsToUcs4(kv->value, kv->length, (TdUcs4*)(p), kv->length * TSDB_NCHAR_SIZE, &output)) { + if (!taosMbsToUcs4(kv->value, kv->length, (TdUcs4*)(p), kv->length * TSDB_NCHAR_SIZE, &output, charsetCxt)) { if (terrno == TAOS_SYSTEM_ERROR(E2BIG)) { taosMemoryFree(p); code = generateSyntaxErrMsg(msg, TSDB_CODE_PAR_VALUE_TOO_LONG, pTagSchema->name); @@ -221,7 +221,7 @@ int32_t smlBuildRow(STableDataCxt* pTableCxt) { return TSDB_CODE_SUCCESS; } -int32_t smlBuildCol(STableDataCxt* pTableCxt, SSchema* schema, void* data, int32_t index) { +int32_t smlBuildCol(STableDataCxt* pTableCxt, SSchema* schema, void* data, int32_t index, void* charsetCxt) { int ret = TSDB_CODE_SUCCESS; SSchema* pColSchema = schema + index; SColVal* pVal = taosArrayGet(pTableCxt->pValues, index); @@ -256,7 +256,7 @@ int32_t smlBuildCol(STableDataCxt* pTableCxt, SSchema* schema, void* data, int32 ret = terrno; goto end; } - if (!taosMbsToUcs4(kv->value, kv->length, (TdUcs4*)pUcs4, size, &len)) { + if (!taosMbsToUcs4(kv->value, kv->length, (TdUcs4*)pUcs4, size, &len, charsetCxt)) { if (terrno == TAOS_SYSTEM_ERROR(E2BIG)) { taosMemoryFree(pUcs4); ret = TSDB_CODE_PAR_VALUE_TOO_LONG; @@ -291,7 +291,7 @@ end: int32_t smlBindData(SQuery* query, bool dataFormat, SArray* tags, SArray* colsSchema, SArray* cols, STableMeta* pTableMeta, char* tableName, const char* sTableName, int32_t sTableNameLen, int32_t ttl, - char* msgBuf, int32_t msgBufLen) { + char* msgBuf, int32_t msgBufLen, void* charsetCxt) { SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; SSchema* pTagsSchema = getTableTagSchema(pTableMeta); @@ -313,7 +313,7 @@ int32_t smlBindData(SQuery* query, bool dataFormat, SArray* tags, SArray* colsSc STag* pTag = NULL; - ret = smlBuildTagRow(tags, &bindTags, pTagsSchema, &pTag, &tagName, &pBuf); + ret = smlBuildTagRow(tags, &bindTags, pTagsSchema, &pTag, &tagName, &pBuf, charsetCxt); if (ret != TSDB_CODE_SUCCESS) { goto end; } @@ -411,7 +411,7 @@ int32_t smlBindData(SQuery* query, bool dataFormat, SArray* tags, SArray* colsSc ret = terrno; goto end; } - if (!taosMbsToUcs4(kv->value, kv->length, (TdUcs4*)pUcs4, pColSchema->bytes - VARSTR_HEADER_SIZE, &len)) { + if (!taosMbsToUcs4(kv->value, kv->length, (TdUcs4*)pUcs4, pColSchema->bytes - VARSTR_HEADER_SIZE, &len, charsetCxt)) { if (terrno == TAOS_SYSTEM_ERROR(E2BIG)) { uError("sml bind taosMbsToUcs4 error, kv length:%d, bytes:%d, kv->value:%s", (int)kv->length, pColSchema->bytes, kv->value); diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index d3e53205e7..a7d1351aa0 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -477,7 +477,7 @@ static int32_t parseVarbinary(SToken* pToken, uint8_t** pData, uint32_t* nData, } static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema, int16_t timePrec, STagVal* val, - SMsgBuf* pMsgBuf, timezone_t tz) { + SMsgBuf* pMsgBuf, timezone_t tz, void *charsetCxt) { int64_t iv; uint64_t uv; char* endptr = NULL; @@ -685,7 +685,7 @@ static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema, if (p == NULL) { return terrno; } - if (!taosMbsToUcs4(pToken->z, pToken->n, (TdUcs4*)(p), realLen, &output)) { + if (!taosMbsToUcs4(pToken->z, pToken->n, (TdUcs4*)(p), realLen, &output, charsetCxt)) { if (terrno == TAOS_SYSTEM_ERROR(E2BIG)) { taosMemoryFree(p); return generateSyntaxErrMsg(pMsgBuf, TSDB_CODE_PAR_VALUE_TOO_LONG, pSchema->name); @@ -732,7 +732,7 @@ static int32_t parseBoundTagsClause(SInsertParseContext* pCxt, SVnodeModifyOpStm } int32_t parseTagValue(SMsgBuf* pMsgBuf, const char** pSql, uint8_t precision, SSchema* pTagSchema, SToken* pToken, - SArray* pTagName, SArray* pTagVals, STag** pTag, timezone_t tz) { + SArray* pTagName, SArray* pTagVals, STag** pTag, timezone_t tz, void *charsetCxt) { bool isNull = isNullValue(pTagSchema->type, pToken); if (!isNull && pTagName) { if (NULL == taosArrayPush(pTagName, pTagSchema->name)) { @@ -748,14 +748,14 @@ int32_t parseTagValue(SMsgBuf* pMsgBuf, const char** pSql, uint8_t precision, SS if (isNull) { return tTagNew(pTagVals, 1, true, pTag); } else { - return parseJsontoTagData(pToken->z, pTagVals, pTag, pMsgBuf); + return parseJsontoTagData(pToken->z, pTagVals, pTag, pMsgBuf, charsetCxt); } } if (isNull) return 0; STagVal val = {0}; - int32_t code = parseTagToken(pSql, pToken, pTagSchema, precision, &val, pMsgBuf, tz); + int32_t code = parseTagToken(pSql, pToken, pTagSchema, precision, &val, pMsgBuf, tz, charsetCxt); if (TSDB_CODE_SUCCESS == code) { if (NULL == taosArrayPush(pTagVals, &val)) { code = terrno; @@ -978,7 +978,7 @@ static int32_t parseTagsClauseImpl(SInsertParseContext* pCxt, SVnodeModifyOpStmt code = buildSyntaxErrMsg(&pCxt->msg, "not expected tags values ", token.z); } if (TSDB_CODE_SUCCESS == code) { - code = parseTagValue(&pCxt->msg, &pStmt->pSql, precision, pTagSchema, &token, pTagName, pTagVals, &pTag, pCxt->pComCxt->timezone); + code = parseTagValue(&pCxt->msg, &pStmt->pSql, precision, pTagSchema, &token, pTagName, pTagVals, &pTag, pCxt->pComCxt->timezone, pCxt->pComCxt->charsetCxt); } } @@ -1620,7 +1620,7 @@ static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql, if (NULL == pUcs4) { return terrno; } - if (!taosMbsToUcs4(pToken->z, pToken->n, (TdUcs4*)pUcs4, realLen, &len)) { + if (!taosMbsToUcs4(pToken->z, pToken->n, (TdUcs4*)pUcs4, realLen, &len, pCxt->pComCxt->charsetCxt)) { taosMemoryFree(pUcs4); if (terrno == TAOS_SYSTEM_ERROR(E2BIG)) { return generateSyntaxErrMsg(&pCxt->msg, TSDB_CODE_PAR_VALUE_TOO_LONG, pSchema->name); @@ -1788,7 +1788,7 @@ static int32_t processCtbTagsAfterCtbName(SInsertParseContext* pCxt, SVnodeModif if (code == TSDB_CODE_SUCCESS) { code = parseTagValue(&pCxt->msg, NULL, precision, pTagSchema, pTagToken, pStbRowsCxt->aTagNames, - pStbRowsCxt->aTagVals, &pStbRowsCxt->pTag, pCxt->pComCxt->timezone); + pStbRowsCxt->aTagVals, &pStbRowsCxt->pTag, pCxt->pComCxt->timezone, pCxt->pComCxt->charsetCxt); } } if (code == TSDB_CODE_SUCCESS && !pStbRowsCxt->isJsonTag) { @@ -1858,7 +1858,7 @@ static int32_t doGetStbRowValues(SInsertParseContext* pCxt, SVnodeModifyOpStmt* } if (code == TSDB_CODE_SUCCESS) { code = parseTagValue(&pCxt->msg, ppSql, precision, (SSchema*)pTagSchema, pToken, pTagNames, pTagVals, - &pStbRowsCxt->pTag, pCxt->pComCxt->timezone); + &pStbRowsCxt->pTag, pCxt->pComCxt->timezone, pCxt->pComCxt->charsetCxt); } } } else if (pCols->pColIndex[i] == tbnameIdx) { diff --git a/source/libs/parser/src/parInsertStmt.c b/source/libs/parser/src/parInsertStmt.c index c6951d229d..6e648a406f 100644 --- a/source/libs/parser/src/parInsertStmt.c +++ b/source/libs/parser/src/parInsertStmt.c @@ -123,7 +123,7 @@ int32_t qBuildStmtOutput(SQuery* pQuery, SHashObj* pVgHash, SHashObj* pBlockHash } int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, const char* sTableName, char* tName, - TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen) { + TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen, void* charsetCxt) { STableDataCxt* pDataBlock = (STableDataCxt*)pBlock; SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; int32_t code = TSDB_CODE_SUCCESS; @@ -183,7 +183,7 @@ int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, const ch goto end; } memcpy(tmp, bind[c].buffer, colLen); - code = parseJsontoTagData(tmp, pTagArray, &pTag, &pBuf); + code = parseJsontoTagData(tmp, pTagArray, &pTag, &pBuf, charsetCxt); taosMemoryFree(tmp); if (code != TSDB_CODE_SUCCESS) { goto end; @@ -208,7 +208,7 @@ int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, const ch code = terrno; goto end; } - if (!taosMbsToUcs4(bind[c].buffer, colLen, (TdUcs4*)(p), colLen * TSDB_NCHAR_SIZE, &output)) { + if (!taosMbsToUcs4(bind[c].buffer, colLen, (TdUcs4*)(p), colLen * TSDB_NCHAR_SIZE, &output, charsetCxt)) { if (terrno == TAOS_SYSTEM_ERROR(E2BIG)) { taosMemoryFree(p); code = generateSyntaxErrMsg(&pBuf, TSDB_CODE_PAR_VALUE_TOO_LONG, pTagSchema->name); @@ -266,7 +266,7 @@ end: return code; } -int32_t convertStmtNcharCol(SMsgBuf* pMsgBuf, SSchema* pSchema, TAOS_MULTI_BIND* src, TAOS_MULTI_BIND* dst) { +int32_t convertStmtNcharCol(SMsgBuf* pMsgBuf, SSchema* pSchema, TAOS_MULTI_BIND* src, TAOS_MULTI_BIND* dst, void* charsetCxt) { int32_t output = 0; int32_t newBuflen = (pSchema->bytes - VARSTR_HEADER_SIZE) * src->num; if (dst->buffer_length < newBuflen) { @@ -292,7 +292,7 @@ int32_t convertStmtNcharCol(SMsgBuf* pMsgBuf, SSchema* pSchema, TAOS_MULTI_BIND* } if (!taosMbsToUcs4(((char*)src->buffer) + src->buffer_length * i, src->length[i], - (TdUcs4*)(((char*)dst->buffer) + dst->buffer_length * i), dst->buffer_length, &output)) { + (TdUcs4*)(((char*)dst->buffer) + dst->buffer_length * i), dst->buffer_length, &output, charsetCxt)) { if (terrno == TAOS_SYSTEM_ERROR(E2BIG)) { return generateSyntaxErrMsg(pMsgBuf, TSDB_CODE_PAR_VALUE_TOO_LONG, pSchema->name); } @@ -312,7 +312,7 @@ int32_t convertStmtNcharCol(SMsgBuf* pMsgBuf, SSchema* pSchema, TAOS_MULTI_BIND* } int32_t qBindStmtStbColsValue(void* pBlock, SArray* pCols, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen, - STSchema** pTSchema, SBindInfo* pBindInfos) { + STSchema** pTSchema, SBindInfo* pBindInfos, void* charsetCxt) { STableDataCxt* pDataBlock = (STableDataCxt*)pBlock; SSchema* pSchema = getTableColumnSchema(pDataBlock->pMeta); SBoundColInfo* boundInfo = &pDataBlock->boundColsInfo; @@ -349,7 +349,7 @@ int32_t qBindStmtStbColsValue(void* pBlock, SArray* pCols, TAOS_MULTI_BIND* bind } if (TSDB_DATA_TYPE_NCHAR == pColSchema->type) { - code = convertStmtNcharCol(&pBuf, pColSchema, bind + c, &ncharBind); + code = convertStmtNcharCol(&pBuf, pColSchema, bind + c, &ncharBind, charsetCxt); if (code) { goto _return; } @@ -380,7 +380,7 @@ _return: return code; } -int32_t qBindStmtColsValue(void* pBlock, SArray* pCols, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen) { +int32_t qBindStmtColsValue(void* pBlock, SArray* pCols, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen, void* charsetCxt) { STableDataCxt* pDataBlock = (STableDataCxt*)pBlock; SSchema* pSchema = getTableColumnSchema(pDataBlock->pMeta); SBoundColInfo* boundInfo = &pDataBlock->boundColsInfo; @@ -406,7 +406,7 @@ int32_t qBindStmtColsValue(void* pBlock, SArray* pCols, TAOS_MULTI_BIND* bind, c } if (TSDB_DATA_TYPE_NCHAR == pColSchema->type) { - code = convertStmtNcharCol(&pBuf, pColSchema, bind + c, &ncharBind); + code = convertStmtNcharCol(&pBuf, pColSchema, bind + c, &ncharBind, charsetCxt); if (code) { goto _return; } @@ -434,7 +434,7 @@ _return: } int32_t qBindStmtSingleColValue(void* pBlock, SArray* pCols, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen, - int32_t colIdx, int32_t rowNum) { + int32_t colIdx, int32_t rowNum, void* charsetCxt) { STableDataCxt* pDataBlock = (STableDataCxt*)pBlock; SSchema* pSchema = getTableColumnSchema(pDataBlock->pMeta); SBoundColInfo* boundInfo = &pDataBlock->boundColsInfo; @@ -459,7 +459,7 @@ int32_t qBindStmtSingleColValue(void* pBlock, SArray* pCols, TAOS_MULTI_BIND* bi } if (TSDB_DATA_TYPE_NCHAR == pColSchema->type) { - code = convertStmtNcharCol(&pBuf, pColSchema, bind, &ncharBind); + code = convertStmtNcharCol(&pBuf, pColSchema, bind, &ncharBind, charsetCxt); if (code) { goto _return; } @@ -483,7 +483,7 @@ _return: } int32_t qBindStmtTagsValue2(void* pBlock, void* boundTags, int64_t suid, const char* sTableName, char* tName, - TAOS_STMT2_BIND* bind, char* msgBuf, int32_t msgBufLen) { + TAOS_STMT2_BIND* bind, char* msgBuf, int32_t msgBufLen, void* charsetCxt) { STableDataCxt* pDataBlock = (STableDataCxt*)pBlock; SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; int32_t code = TSDB_CODE_SUCCESS; @@ -543,7 +543,7 @@ int32_t qBindStmtTagsValue2(void* pBlock, void* boundTags, int64_t suid, const c goto end; } memcpy(tmp, bind[c].buffer, colLen); - code = parseJsontoTagData(tmp, pTagArray, &pTag, &pBuf); + code = parseJsontoTagData(tmp, pTagArray, &pTag, &pBuf, charsetCxt); taosMemoryFree(tmp); if (code != TSDB_CODE_SUCCESS) { goto end; @@ -568,7 +568,7 @@ int32_t qBindStmtTagsValue2(void* pBlock, void* boundTags, int64_t suid, const c code = terrno; goto end; } - if (!taosMbsToUcs4(bind[c].buffer, colLen, (TdUcs4*)(p), colLen * TSDB_NCHAR_SIZE, &output)) { + if (!taosMbsToUcs4(bind[c].buffer, colLen, (TdUcs4*)(p), colLen * TSDB_NCHAR_SIZE, &output, charsetCxt)) { if (terrno == TAOS_SYSTEM_ERROR(E2BIG)) { taosMemoryFree(p); code = generateSyntaxErrMsg(&pBuf, TSDB_CODE_PAR_VALUE_TOO_LONG, pTagSchema->name); @@ -626,7 +626,7 @@ end: return code; } -static int32_t convertStmtStbNcharCol2(SMsgBuf* pMsgBuf, SSchema* pSchema, TAOS_STMT2_BIND* src, TAOS_STMT2_BIND* dst) { +static int32_t convertStmtStbNcharCol2(SMsgBuf* pMsgBuf, SSchema* pSchema, TAOS_STMT2_BIND* src, TAOS_STMT2_BIND* dst, void *charsetCxt) { int32_t output = 0; const int32_t max_buf_len = pSchema->bytes - VARSTR_HEADER_SIZE; @@ -648,7 +648,7 @@ static int32_t convertStmtStbNcharCol2(SMsgBuf* pMsgBuf, SSchema* pSchema, TAOS_ continue; } - if (!taosMbsToUcs4(src_buf, src->length[i], (TdUcs4*)dst_buf, max_buf_len, &output)) { + if (!taosMbsToUcs4(src_buf, src->length[i], (TdUcs4*)dst_buf, max_buf_len, &output, charsetCxt)) { if (terrno == TAOS_SYSTEM_ERROR(E2BIG)) { return generateSyntaxErrMsg(pMsgBuf, TSDB_CODE_PAR_VALUE_TOO_LONG, pSchema->name); } @@ -670,7 +670,7 @@ static int32_t convertStmtStbNcharCol2(SMsgBuf* pMsgBuf, SSchema* pSchema, TAOS_ } int32_t qBindStmtStbColsValue2(void* pBlock, SArray* pCols, TAOS_STMT2_BIND* bind, char* msgBuf, int32_t msgBufLen, - STSchema** pTSchema, SBindInfo2* pBindInfos) { + STSchema** pTSchema, SBindInfo2* pBindInfos, void *charsetCxt) { STableDataCxt* pDataBlock = (STableDataCxt*)pBlock; SSchema* pSchema = getTableColumnSchema(pDataBlock->pMeta); SBoundColInfo* boundInfo = &pDataBlock->boundColsInfo; @@ -720,7 +720,7 @@ int32_t qBindStmtStbColsValue2(void* pBlock, SArray* pCols, TAOS_STMT2_BIND* bin } if (TSDB_DATA_TYPE_NCHAR == pColSchema->type) { - code = convertStmtStbNcharCol2(&pBuf, pColSchema, bind + c, &ncharBind); + code = convertStmtStbNcharCol2(&pBuf, pColSchema, bind + c, &ncharBind, charsetCxt); if (code) { goto _return; } @@ -755,7 +755,7 @@ _return: return code; } -static int32_t convertStmtNcharCol2(SMsgBuf* pMsgBuf, SSchema* pSchema, TAOS_STMT2_BIND* src, TAOS_STMT2_BIND* dst) { +static int32_t convertStmtNcharCol2(SMsgBuf* pMsgBuf, SSchema* pSchema, TAOS_STMT2_BIND* src, TAOS_STMT2_BIND* dst, void *charsetCxt) { int32_t output = 0; const int32_t max_buf_len = pSchema->bytes - VARSTR_HEADER_SIZE; @@ -785,7 +785,7 @@ static int32_t convertStmtNcharCol2(SMsgBuf* pMsgBuf, SSchema* pSchema, TAOS_STM /*if (!taosMbsToUcs4(((char*)src->buffer) + src->buffer_length * i, src->length[i], (TdUcs4*)(((char*)dst->buffer) + dst->buffer_length * i), dst->buffer_length, &output)) {*/ - if (!taosMbsToUcs4(src_buf, src->length[i], (TdUcs4*)dst_buf, max_buf_len, &output)) { + if (!taosMbsToUcs4(src_buf, src->length[i], (TdUcs4*)dst_buf, max_buf_len, &output, charsetCxt)) { if (terrno == TAOS_SYSTEM_ERROR(E2BIG)) { return generateSyntaxErrMsg(pMsgBuf, TSDB_CODE_PAR_VALUE_TOO_LONG, pSchema->name); } @@ -806,7 +806,7 @@ static int32_t convertStmtNcharCol2(SMsgBuf* pMsgBuf, SSchema* pSchema, TAOS_STM return TSDB_CODE_SUCCESS; } -int32_t qBindStmtColsValue2(void* pBlock, SArray* pCols, TAOS_STMT2_BIND* bind, char* msgBuf, int32_t msgBufLen) { +int32_t qBindStmtColsValue2(void* pBlock, SArray* pCols, TAOS_STMT2_BIND* bind, char* msgBuf, int32_t msgBufLen, void *charsetCxt) { STableDataCxt* pDataBlock = (STableDataCxt*)pBlock; SSchema* pSchema = getTableColumnSchema(pDataBlock->pMeta); SBoundColInfo* boundInfo = &pDataBlock->boundColsInfo; @@ -836,7 +836,7 @@ int32_t qBindStmtColsValue2(void* pBlock, SArray* pCols, TAOS_STMT2_BIND* bind, } if (TSDB_DATA_TYPE_NCHAR == pColSchema->type) { - code = convertStmtNcharCol2(&pBuf, pColSchema, bind + c, &ncharBind); + code = convertStmtNcharCol2(&pBuf, pColSchema, bind + c, &ncharBind, charsetCxt); if (code) { goto _return; } @@ -864,7 +864,7 @@ _return: } int32_t qBindStmtSingleColValue2(void* pBlock, SArray* pCols, TAOS_STMT2_BIND* bind, char* msgBuf, int32_t msgBufLen, - int32_t colIdx, int32_t rowNum) { + int32_t colIdx, int32_t rowNum, void *charsetCxt) { STableDataCxt* pDataBlock = (STableDataCxt*)pBlock; SSchema* pSchema = getTableColumnSchema(pDataBlock->pMeta); SBoundColInfo* boundInfo = &pDataBlock->boundColsInfo; @@ -889,7 +889,7 @@ int32_t qBindStmtSingleColValue2(void* pBlock, SArray* pCols, TAOS_STMT2_BIND* b } if (TSDB_DATA_TYPE_NCHAR == pColSchema->type) { - code = convertStmtNcharCol2(&pBuf, pColSchema, bind, &ncharBind); + code = convertStmtNcharCol2(&pBuf, pColSchema, bind, &ncharBind, charsetCxt); if (code) { goto _return; } diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index c13686dddc..3d75682251 100755 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -2148,7 +2148,7 @@ static EDealRes translateNormalValue(STranslateContext* pCxt, SValueNode* pVal, int32_t len = 0; if (!taosMbsToUcs4(pVal->literal, strlen(pVal->literal), (TdUcs4*)varDataVal(pVal->datum.p), - targetDt.bytes - VARSTR_HEADER_SIZE, &len)) { + targetDt.bytes - VARSTR_HEADER_SIZE, &len, pCxt->pParseCxt->charsetCxt)) { return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, pVal->literal); } varDataSetLen(pVal->datum.p, len); @@ -14169,7 +14169,7 @@ static int32_t buildKVRowForBindTags(STranslateContext* pCxt, SCreateSubTableCla if (pSchema->type == TSDB_DATA_TYPE_JSON) { isJson = true; } - code = parseTagValue(&pCxt->msgBuf, &tagStr, precision, pSchema, &token, tagName, pTagArray, ppTag, pCxt->pParseCxt->timezone); + code = parseTagValue(&pCxt->msgBuf, &tagStr, precision, pSchema, &token, tagName, pTagArray, ppTag, pCxt->pParseCxt->timezone, pCxt->pParseCxt->charsetCxt); } if (TSDB_CODE_SUCCESS == code) { @@ -14230,7 +14230,7 @@ static int32_t buildKVRowForAllTags(STranslateContext* pCxt, SCreateSubTableClau if (pTagSchema->type == TSDB_DATA_TYPE_JSON) { isJson = true; } - code = parseTagValue(&pCxt->msgBuf, &tagStr, precision, pTagSchema, &token, tagName, pTagArray, ppTag, pCxt->pParseCxt->timezone); + code = parseTagValue(&pCxt->msgBuf, &tagStr, precision, pTagSchema, &token, tagName, pTagArray, ppTag, pCxt->pParseCxt->timezone, pCxt->pParseCxt->charsetCxt); } if (TSDB_CODE_SUCCESS == code) { @@ -14426,7 +14426,7 @@ static int32_t fillVgroupInfo(SParseContext* pParseCxt, const SName* pName, SVgr return code; } -static int32_t parseOneStbRow(SMsgBuf* pMsgBuf, SParseFileContext* pParFileCxt, timezone_t tz) { +static int32_t parseOneStbRow(SMsgBuf* pMsgBuf, SParseFileContext* pParFileCxt, timezone_t tz, void *charsetCxt) { int32_t code = TSDB_CODE_SUCCESS; int sz = taosArrayGetSize(pParFileCxt->aTagIndexs); int32_t numOfTags = getNumOfTags(pParFileCxt->pStbMeta); @@ -14458,7 +14458,7 @@ static int32_t parseOneStbRow(SMsgBuf* pMsgBuf, SParseFileContext* pParFileCxt, if (TSDB_CODE_SUCCESS == code) { SArray* aTagNames = pParFileCxt->tagNameFilled ? NULL : pParFileCxt->aTagNames; code = parseTagValue(pMsgBuf, &pParFileCxt->pSql, precision, (SSchema*)pTagSchema, &token, aTagNames, - pParFileCxt->aTagVals, &pParFileCxt->pTag, tz); + pParFileCxt->aTagVals, &pParFileCxt->pTag, tz, charsetCxt); } } else { // parse tbname @@ -14524,7 +14524,7 @@ static int32_t parseCsvFile(SMsgBuf* pMsgBuf, SParseContext* pParseCxt, SParseFi (void)strtolower(pLine, pLine); pParFileCxt->pSql = pLine; - code = parseOneStbRow(pMsgBuf, pParFileCxt, pParseCxt->timezone); + code = parseOneStbRow(pMsgBuf, pParFileCxt, pParseCxt->timezone, pParseCxt->charsetCxt); if (TSDB_CODE_SUCCESS == code) { code = fillVgroupInfo(pParseCxt, &pParFileCxt->ctbName, &pParFileCxt->vg); @@ -15225,7 +15225,7 @@ static int32_t buildUpdateTagValReq(STranslateContext* pCxt, SAlterTableStmt* pS if (TSDB_CODE_SUCCESS == code) { code = parseTagValue(&pCxt->msgBuf, &tagStr, pTableMeta->tableInfo.precision, pSchema, &token, NULL, - pReq->pTagArray, &pTag, pCxt->pParseCxt->timezone); + pReq->pTagArray, &pTag, pCxt->pParseCxt->timezone, pCxt->pParseCxt->charsetCxt); if (pSchema->type == TSDB_DATA_TYPE_JSON && token.type == TK_NULL && code == TSDB_CODE_SUCCESS) { pReq->tagFree = true; } diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index 44e44982a3..07ea54c44f 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -402,7 +402,7 @@ static bool isValidateTag(char* input) { return true; } -int32_t parseJsontoTagData(const char* json, SArray* pTagVals, STag** ppTag, void* pMsgBuf) { +int32_t parseJsontoTagData(const char* json, SArray* pTagVals, STag** ppTag, void* pMsgBuf, void *charsetCxt) { int32_t retCode = TSDB_CODE_SUCCESS; cJSON* root = NULL; SHashObj* keyHash = NULL; @@ -471,7 +471,7 @@ int32_t parseJsontoTagData(const char* json, SArray* pTagVals, STag** ppTag, voi goto end; } val.type = TSDB_DATA_TYPE_NCHAR; - if (valLen > 0 && !taosMbsToUcs4(jsonValue, valLen, (TdUcs4*)tmp, (int32_t)(valLen * TSDB_NCHAR_SIZE), &valLen)) { + if (valLen > 0 && !taosMbsToUcs4(jsonValue, valLen, (TdUcs4*)tmp, (int32_t)(valLen * TSDB_NCHAR_SIZE), &valLen, charsetCxt)) { uError("charset:%s to %s. val:%s, errno:%s, convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, jsonValue, strerror(terrno)); retCode = buildSyntaxErrMsg(pMsgBuf, "charset convert json error", jsonValue); diff --git a/source/libs/parser/src/parser.c b/source/libs/parser/src/parser.c index e2135bfd63..d9783fb9a3 100644 --- a/source/libs/parser/src/parser.c +++ b/source/libs/parser/src/parser.c @@ -153,7 +153,7 @@ static int32_t parseSqlSyntax(SParseContext* pCxt, SQuery** pQuery, SParseMetaCa return code; } -static int32_t setValueByBindParam(SValueNode* pVal, TAOS_MULTI_BIND* pParam) { +static int32_t setValueByBindParam(SValueNode* pVal, TAOS_MULTI_BIND* pParam, void *charsetCxt) { if (!pParam || IS_NULL_TYPE(pParam->buffer_type)) { return TSDB_CODE_APP_ERROR; } @@ -200,7 +200,7 @@ static int32_t setValueByBindParam(SValueNode* pVal, TAOS_MULTI_BIND* pParam) { int32_t output = 0; if (!taosMbsToUcs4(pParam->buffer, inputSize, (TdUcs4*)varDataVal(pVal->datum.p), pVal->node.resType.bytes, - &output)) { + &output, charsetCxt)) { return terrno; } varDataSetLen(pVal->datum.p, output); @@ -417,19 +417,19 @@ int32_t qInitKeywordsTable() { return taosInitKeywordsTable(); } void qCleanupKeywordsTable() { taosCleanupKeywordsTable(); } -int32_t qStmtBindParams(SQuery* pQuery, TAOS_MULTI_BIND* pParams, int32_t colIdx) { +int32_t qStmtBindParams(SQuery* pQuery, TAOS_MULTI_BIND* pParams, int32_t colIdx, void *charsetCxt) { int32_t code = TSDB_CODE_SUCCESS; if (colIdx < 0) { int32_t size = taosArrayGetSize(pQuery->pPlaceholderValues); for (int32_t i = 0; i < size; ++i) { - code = setValueByBindParam((SValueNode*)taosArrayGetP(pQuery->pPlaceholderValues, i), pParams + i); + code = setValueByBindParam((SValueNode*)taosArrayGetP(pQuery->pPlaceholderValues, i), pParams + i, charsetCxt); if (TSDB_CODE_SUCCESS != code) { return code; } } } else { - code = setValueByBindParam((SValueNode*)taosArrayGetP(pQuery->pPlaceholderValues, colIdx), pParams); + code = setValueByBindParam((SValueNode*)taosArrayGetP(pQuery->pPlaceholderValues, colIdx), pParams, charsetCxt); } if (TSDB_CODE_SUCCESS == code && (colIdx < 0 || colIdx + 1 == pQuery->placeholderNum)) { @@ -443,7 +443,7 @@ int32_t qStmtBindParams(SQuery* pQuery, TAOS_MULTI_BIND* pParams, int32_t colIdx return code; } -static int32_t setValueByBindParam2(SValueNode* pVal, TAOS_STMT2_BIND* pParam) { +static int32_t setValueByBindParam2(SValueNode* pVal, TAOS_STMT2_BIND* pParam, void* charsetCxt) { if (!pParam || IS_NULL_TYPE(pParam->buffer_type)) { return TSDB_CODE_APP_ERROR; } @@ -490,7 +490,7 @@ static int32_t setValueByBindParam2(SValueNode* pVal, TAOS_STMT2_BIND* pParam) { int32_t output = 0; if (!taosMbsToUcs4(pParam->buffer, inputSize, (TdUcs4*)varDataVal(pVal->datum.p), pVal->node.resType.bytes, - &output)) { + &output, charsetCxt)) { return terrno; } varDataSetLen(pVal->datum.p, output); @@ -509,19 +509,19 @@ static int32_t setValueByBindParam2(SValueNode* pVal, TAOS_STMT2_BIND* pParam) { return TSDB_CODE_SUCCESS; } -int32_t qStmtBindParams2(SQuery* pQuery, TAOS_STMT2_BIND* pParams, int32_t colIdx) { +int32_t qStmtBindParams2(SQuery* pQuery, TAOS_STMT2_BIND* pParams, int32_t colIdx, void* charsetCxt) { int32_t code = TSDB_CODE_SUCCESS; if (colIdx < 0) { int32_t size = taosArrayGetSize(pQuery->pPlaceholderValues); for (int32_t i = 0; i < size; ++i) { - code = setValueByBindParam2((SValueNode*)taosArrayGetP(pQuery->pPlaceholderValues, i), pParams + i); + code = setValueByBindParam2((SValueNode*)taosArrayGetP(pQuery->pPlaceholderValues, i), pParams + i, charsetCxt); if (TSDB_CODE_SUCCESS != code) { return code; } } } else { - code = setValueByBindParam2((SValueNode*)taosArrayGetP(pQuery->pPlaceholderValues, colIdx), pParams); + code = setValueByBindParam2((SValueNode*)taosArrayGetP(pQuery->pPlaceholderValues, colIdx), pParams, charsetCxt); } if (TSDB_CODE_SUCCESS == code && (colIdx < 0 || colIdx + 1 == pQuery->placeholderNum)) { diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index a1809ff137..2d060fe9bc 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -6725,7 +6725,7 @@ static int32_t fillTSMAOptCtx(STSMAOptCtx* pTsmaOptCtx, SScanLogicNode* pScan) { pTsmaOptCtx->queryInterval->sliding = pWindow->sliding; pTsmaOptCtx->queryInterval->slidingUnit = pWindow->slidingUnit; pTsmaOptCtx->queryInterval->precision = pWindow->node.precision; - pTsmaOptCtx->queryInterval->tz = tsTimezone; + pTsmaOptCtx->queryInterval->tz = taosGetLocalTimezoneOffset(); pTsmaOptCtx->pAggFuncs = pWindow->pFuncs; pTsmaOptCtx->ppParentTsmaSubplans = &pWindow->pTsmaSubplans; } else { diff --git a/source/libs/planner/test/planTestUtil.cpp b/source/libs/planner/test/planTestUtil.cpp index da38a72953..902e254465 100644 --- a/source/libs/planner/test/planTestUtil.cpp +++ b/source/libs/planner/test/planTestUtil.cpp @@ -377,7 +377,7 @@ class PlannerTestBaseImpl { } void doBindParams(SQuery* pQuery, TAOS_MULTI_BIND* pParams, int32_t colIdx) { - DO_WITH_THROW(qStmtBindParams, pQuery, pParams, colIdx); + DO_WITH_THROW(qStmtBindParams, pQuery, pParams, colIdx, NULL); if (colIdx < 0 || pQuery->placeholderNum == colIdx + 1) { res_.boundAst_ = toString(pQuery->pRoot); } diff --git a/source/libs/qcom/src/queryUtil.c b/source/libs/qcom/src/queryUtil.c index 54e92c6a1b..9b59e321a3 100644 --- a/source/libs/qcom/src/queryUtil.c +++ b/source/libs/qcom/src/queryUtil.c @@ -397,7 +397,7 @@ int32_t dataConverToStr(char* str, int64_t capacity, int type, void* buf, int32_ } *str = '"'; - int32_t length = taosUcs4ToMbs((TdUcs4*)buf, bufSize, str + 1); + int32_t length = taosUcs4ToMbs((TdUcs4*)buf, bufSize, str + 1, NULL); if (length <= 0) { return TSDB_CODE_TSC_INVALID_VALUE; } @@ -430,7 +430,7 @@ int32_t dataConverToStr(char* str, int64_t capacity, int type, void* buf, int32_ return TSDB_CODE_SUCCESS; } -void parseTagDatatoJson(void* p, char** jsonStr) { +void parseTagDatatoJson(void* p, char** jsonStr, void *charsetCxt) { if (!p || !jsonStr) { qError("parseTagDatatoJson invalid input, line:%d", __LINE__); return; @@ -475,7 +475,7 @@ void parseTagDatatoJson(void* p, char** jsonStr) { if (tagJsonValue == NULL) { goto end; } - int32_t length = taosUcs4ToMbs((TdUcs4*)pTagVal->pData, pTagVal->nData, tagJsonValue); + int32_t length = taosUcs4ToMbs((TdUcs4*)pTagVal->pData, pTagVal->nData, tagJsonValue, charsetCxt); if (length < 0) { qError("charset:%s to %s. val:%s convert json value failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, pTagVal->pData); diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index cb08851196..f0325dd174 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -2266,7 +2266,7 @@ int32_t fltInitValFieldData(SFilterInfo *info) { // match/nmatch for nchar type need convert from ucs4 to mbs if (type == TSDB_DATA_TYPE_NCHAR && (unit->compare.optr == OP_TYPE_MATCH || unit->compare.optr == OP_TYPE_NMATCH)) { char newValData[TSDB_REGEX_STRING_DEFAULT_LEN * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE] = {0}; - int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(fi->data), varDataLen(fi->data), varDataVal(newValData)); + int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(fi->data), varDataLen(fi->data), varDataVal(newValData), NULL); if (len < 0) { qError("filterInitValFieldData taosUcs4ToMbs error 1"); return TSDB_CODE_SCALAR_CONVERT_ERROR; @@ -3603,7 +3603,7 @@ int32_t filterExecuteImplMisc(void *pinfo, int32_t numOfRows, SColumnInfoData *p if (newColData == NULL) { FLT_ERR_RET(terrno); } - int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(colData), varDataLen(colData), varDataVal(newColData)); + int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(colData), varDataLen(colData), varDataVal(newColData), NULL); if (len < 0) { qError("castConvert1 taosUcs4ToMbs error"); taosMemoryFreeClear(newColData); @@ -3678,7 +3678,7 @@ int32_t filterExecuteImpl(void *pinfo, int32_t numOfRows, SColumnInfoData *pRes, if (newColData == NULL) { FLT_ERR_RET(terrno); } - int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(colData), varDataLen(colData), varDataVal(newColData)); + int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(colData), varDataLen(colData), varDataVal(newColData), NULL); if (len < 0) { qError("castConvert1 taosUcs4ToMbs error"); taosMemoryFreeClear(newColData); @@ -4614,7 +4614,7 @@ int32_t filterConverNcharColumns(SFilterInfo *info, int32_t rows, bool *gotNchar varDataCopy(dst, src); continue; } - bool ret = taosMbsToUcs4(varDataVal(src), varDataLen(src), (TdUcs4 *)varDataVal(dst), bufSize, &len); + bool ret = taosMbsToUcs4(varDataVal(src), varDataLen(src), (TdUcs4 *)varDataVal(dst), bufSize, &len, NULL); if (!ret) { qError("filterConverNcharColumns taosMbsToUcs4 error"); return TSDB_CODE_SCALAR_CONVERT_ERROR; diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index d533eba74e..e871ab031b 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -24,7 +24,7 @@ int32_t scalarGetOperatorParamNum(EOperatorType type) { int32_t sclConvertToTsValueNode(int8_t precision, SValueNode *valueNode) { char *timeStr = valueNode->datum.p; int64_t value = 0; - int32_t code = convertStringToTimestamp(valueNode->node.resType.type, valueNode->datum.p, precision, &value, valueNode->tz); //todo tz + int32_t code = convertStringToTimestamp(valueNode->node.resType.type, valueNode->datum.p, precision, &value, valueNode->tz, valueNode->charsetCxt); //todo tz if (code != TSDB_CODE_SUCCESS) { return code; } @@ -82,7 +82,9 @@ int32_t sclConvertValueToSclParam(SValueNode *pValueNode, SScalarParam *out, int } in.tz = pValueNode->tz; + in.charsetCxt = pValueNode->charsetCxt; out->tz = pValueNode->tz; + out->charsetCxt = pValueNode->charsetCxt; code = vectorConvertSingleColImpl(&in, out, overflow, -1, -1); _exit: @@ -589,6 +591,7 @@ int32_t sclInitOperatorParams(SScalarParam **pParams, SOperatorNode *node, SScal SCL_ERR_JRET(sclInitParam(node->pLeft, ¶mList[0], ctx, rowNum)); paramList[0].tz = node->tz; + paramList[0].charsetCxt = node->charsetCxt; if (paramNum > 1) { TSWAP(ctx->type.selfType, ctx->type.peerType); SCL_ERR_JRET(sclInitParam(node->pRight, ¶mList[1], ctx, rowNum)); @@ -760,6 +763,7 @@ int32_t sclExecFunction(SFunctionNode *node, SScalarCtx *ctx, SScalarParam *outp int32_t code = 0; SCL_ERR_RET(sclInitParamList(¶ms, node->pParameterList, ctx, ¶mNum, &rowNum)); params->tz = node->tz; + params->charsetCxt = node->charsetCxt; if (fmIsUserDefinedFunc(node->funcId)) { code = callUdfScalarFunc(node->functionName, params, paramNum, output); @@ -947,6 +951,7 @@ int32_t sclExecCaseWhen(SCaseWhenNode *node, SScalarCtx *ctx, SScalarParam *outp SCL_ERR_JRET(sclGetNodeRes(node->pCase, ctx, &pCase)); SCL_ERR_JRET(sclGetNodeRes(node->pElse, ctx, &pElse)); pCase->tz = node->tz; + pCase->charsetCxt = node->charsetCxt; SDataType compType = {0}; compType.type = TSDB_DATA_TYPE_BOOL; compType.bytes = tDataTypes[compType.type].bytes; diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index e51ab1c04b..fb4f0db631 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -12,8 +12,8 @@ typedef float (*_float_fn_2)(float, float); typedef double (*_double_fn)(double); typedef double (*_double_fn_2)(double, double); typedef int (*_conv_fn)(int); -typedef void (*_trim_space_fn)(char *, char *, int32_t, int32_t); -typedef int32_t (*_trim_fn)(char *, char *, char *, int32_t, int32_t); +typedef void (*_trim_space_fn)(char *, char *, int32_t, int32_t, void*); +typedef int32_t (*_trim_fn)(char *, char *, char *, int32_t, int32_t, void*); typedef int32_t (*_len_fn)(char *, int32_t, VarDataLenT *); /** Math functions **/ @@ -500,7 +500,7 @@ static int32_t tcharlength(char *input, int32_t type, VarDataLenT *len) { return TSDB_CODE_SUCCESS; } -static void tltrimspace(char *input, char *output, int32_t type, int32_t charLen) { +static void tltrimspace(char *input, char *output, int32_t type, int32_t charLen, void* charsetCxt) { int32_t numOfSpaces = 0; if (type == TSDB_DATA_TYPE_VARCHAR) { for (int32_t i = 0; i < charLen; ++i) { @@ -530,7 +530,7 @@ static void tltrimspace(char *input, char *output, int32_t type, int32_t charLen varDataSetLen(output, resLen); } -static void tlrtrimspace(char *input, char *output, int32_t type, int32_t charLen) { +static void tlrtrimspace(char *input, char *output, int32_t type, int32_t charLen, void* charsetCxt) { int32_t numOfLeftSpaces = 0; int32_t numOfRightSpaces = 0; if (type == TSDB_DATA_TYPE_VARCHAR) { @@ -615,12 +615,12 @@ static int32_t trimHelper(char *orgStr, char* remStr, int32_t orgLen, int32_t re } } -static int32_t convVarcharToNchar(char *input, char **output, int32_t inputLen, int32_t *outputLen) { +static int32_t convVarcharToNchar(char *input, char **output, int32_t inputLen, int32_t *outputLen, void* charsetCxt) { *output = taosMemoryCalloc(inputLen * TSDB_NCHAR_SIZE, 1); if (NULL == *output) { return terrno; } - bool ret = taosMbsToUcs4(input, inputLen, (TdUcs4 *)*output, inputLen * TSDB_NCHAR_SIZE, outputLen); + bool ret = taosMbsToUcs4(input, inputLen, (TdUcs4 *)*output, inputLen * TSDB_NCHAR_SIZE, outputLen, charsetCxt); if (!ret) { taosMemoryFreeClear(*output); return TSDB_CODE_SCALAR_CONVERT_ERROR; @@ -628,12 +628,12 @@ static int32_t convVarcharToNchar(char *input, char **output, int32_t inputLen, return TSDB_CODE_SUCCESS; } -static int32_t convNcharToVarchar(char *input, char **output, int32_t inputLen, int32_t *outputLen) { +static int32_t convNcharToVarchar(char *input, char **output, int32_t inputLen, int32_t *outputLen, void* charsetCxt) { *output = taosMemoryCalloc(inputLen, 1); if (NULL == *output) { return terrno; } - *outputLen = taosUcs4ToMbs((TdUcs4 *)input, inputLen, *output); + *outputLen = taosUcs4ToMbs((TdUcs4 *)input, inputLen, *output, charsetCxt); if (*outputLen < 0) { taosMemoryFree(*output); return TSDB_CODE_SCALAR_CONVERT_ERROR; @@ -641,14 +641,14 @@ static int32_t convNcharToVarchar(char *input, char **output, int32_t inputLen, return TSDB_CODE_SUCCESS; } -static int32_t convBetweenNcharAndVarchar(char *input, char **output, int32_t inputLen, int32_t *outputLen, int32_t wantType) { +static int32_t convBetweenNcharAndVarchar(char *input, char **output, int32_t inputLen, int32_t *outputLen, int32_t wantType, void* charsetCxt) { if (wantType == TSDB_DATA_TYPE_NCHAR) { - return convVarcharToNchar(input, output, inputLen, outputLen); + return convVarcharToNchar(input, output, inputLen, outputLen, charsetCxt); } else { - return convNcharToVarchar(input, output, inputLen, outputLen); + return convNcharToVarchar(input, output, inputLen, outputLen, charsetCxt); } } -static int32_t tltrim(char *input, char *remInput, char *output, int32_t inputType, int32_t remType) { +static int32_t tltrim(char *input, char *remInput, char *output, int32_t inputType, int32_t remType, void* charsetCxt) { int32_t orgLen = varDataLen(input); char *orgStr = varDataVal(input); int32_t remLen = varDataLen(remInput); @@ -661,7 +661,7 @@ static int32_t tltrim(char *input, char *remInput, char *output, int32_t inputTy bool needFree = false; if (inputType != remType) { - SCL_ERR_RET(convBetweenNcharAndVarchar(varDataVal(remInput), &remStr, varDataLen(remInput), &remLen, inputType)); + SCL_ERR_RET(convBetweenNcharAndVarchar(varDataVal(remInput), &remStr, varDataLen(remInput), &remLen, inputType, charsetCxt)); needFree = true; } @@ -683,7 +683,7 @@ static int32_t tltrim(char *input, char *remInput, char *output, int32_t inputTy return TSDB_CODE_SUCCESS; } -static void trtrimspace(char *input, char *output, int32_t type, int32_t charLen) { +static void trtrimspace(char *input, char *output, int32_t type, int32_t charLen, void *charsetCxt) { int32_t numOfSpaces = 0; if (type == TSDB_DATA_TYPE_VARCHAR) { for (int32_t i = charLen - 1; i >= 0; --i) { @@ -712,7 +712,7 @@ static void trtrimspace(char *input, char *output, int32_t type, int32_t charLen varDataSetLen(output, resLen); } -static int32_t trtrim(char *input, char *remInput, char *output, int32_t inputType, int32_t remType) { +static int32_t trtrim(char *input, char *remInput, char *output, int32_t inputType, int32_t remType, void* charsetCxt) { int32_t orgLen = varDataLen(input); char *orgStr = varDataVal(input); int32_t remLen = varDataLen(remInput); @@ -725,7 +725,7 @@ static int32_t trtrim(char *input, char *remInput, char *output, int32_t inputTy bool needFree = false; if (inputType != remType) { - SCL_ERR_RET(convBetweenNcharAndVarchar(varDataVal(remInput), &remStr, varDataLen(remInput), &remLen, inputType)); + SCL_ERR_RET(convBetweenNcharAndVarchar(varDataVal(remInput), &remStr, varDataLen(remInput), &remLen, inputType, charsetCxt)); needFree = true; } @@ -746,7 +746,7 @@ static int32_t trtrim(char *input, char *remInput, char *output, int32_t inputTy return TSDB_CODE_SUCCESS; } -static int32_t tlrtrim(char *input, char *remInput, char *output, int32_t inputType, int32_t remType) { +static int32_t tlrtrim(char *input, char *remInput, char *output, int32_t inputType, int32_t remType, void *charsetCxt) { int32_t orgLen = varDataLen(input); char *orgStr = varDataVal(input); int32_t remLen = varDataLen(remInput); @@ -759,7 +759,7 @@ static int32_t tlrtrim(char *input, char *remInput, char *output, int32_t inputT bool needFree = false; if (inputType != remType) { - SCL_ERR_RET(convBetweenNcharAndVarchar(varDataVal(remInput), &remStr, varDataLen(remInput), &remLen, inputType)); + SCL_ERR_RET(convBetweenNcharAndVarchar(varDataVal(remInput), &remStr, varDataLen(remInput), &remLen, inputType, charsetCxt)); needFree = true; } @@ -810,14 +810,14 @@ static int32_t doLengthFunction(SScalarParam *pInput, int32_t inputNum, SScalarP return TSDB_CODE_SUCCESS; } -static int32_t concatCopyHelper(const char *input, char *output, bool hasNchar, int32_t type, VarDataLenT *dataLen) { +static int32_t concatCopyHelper(const char *input, char *output, bool hasNchar, int32_t type, VarDataLenT *dataLen, void* charsetCxt) { if (hasNchar && type == TSDB_DATA_TYPE_VARCHAR) { TdUcs4 *newBuf = taosMemoryCalloc((varDataLen(input) + 1) * TSDB_NCHAR_SIZE, 1); if (NULL == newBuf) { return terrno; } int32_t len = varDataLen(input); - bool ret = taosMbsToUcs4(varDataVal(input), len, newBuf, (varDataLen(input) + 1) * TSDB_NCHAR_SIZE, &len); + bool ret = taosMbsToUcs4(varDataVal(input), len, newBuf, (varDataLen(input) + 1) * TSDB_NCHAR_SIZE, &len, charsetCxt); if (!ret) { taosMemoryFree(newBuf); return TSDB_CODE_SCALAR_CONVERT_ERROR; @@ -905,7 +905,7 @@ int32_t concatFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu int32_t rowIdx = (pInput[i].numOfRows == 1) ? 0 : k; input[i] = colDataGetData(pInputData[i], rowIdx); - SCL_ERR_JRET(concatCopyHelper(input[i], output, hasNchar, GET_PARAM_TYPE(&pInput[i]), &dataLen)); + SCL_ERR_JRET(concatCopyHelper(input[i], output, hasNchar, GET_PARAM_TYPE(&pInput[i]), &dataLen, pInput->charsetCxt)); } varDataSetLen(output, dataLen); SCL_ERR_JRET(colDataSetVal(pOutputData, k, outputBuf, false)); @@ -980,12 +980,12 @@ int32_t concatWsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p int32_t rowIdx = (pInput[i].numOfRows == 1) ? 0 : k; - SCL_ERR_JRET(concatCopyHelper(colDataGetData(pInputData[i], rowIdx), output, hasNchar, GET_PARAM_TYPE(&pInput[i]), &dataLen)); + SCL_ERR_JRET(concatCopyHelper(colDataGetData(pInputData[i], rowIdx), output, hasNchar, GET_PARAM_TYPE(&pInput[i]), &dataLen, pInput->charsetCxt)); if (i < inputNum - 1) { // insert the separator char *sep = (pInput[0].numOfRows == 1) ? colDataGetData(pInputData[0], 0) : colDataGetData(pInputData[0], k); - SCL_ERR_JRET(concatCopyHelper(sep, output, hasNchar, GET_PARAM_TYPE(&pInput[0]), &dataLen)); + SCL_ERR_JRET(concatCopyHelper(sep, output, hasNchar, GET_PARAM_TYPE(&pInput[0]), &dataLen, pInput->charsetCxt)); } } @@ -1092,7 +1092,7 @@ static int32_t doTrimFunction(SScalarParam *pInput, int32_t inputNum, SScalarPar continue; } SCL_ERR_JRET(trimFn(colDataGetData(pInputData[1], colIdx2), colDataGetData(pInputData[0], colIdx1), - output, GET_PARAM_TYPE(&pInput[1]), GET_PARAM_TYPE(&pInput[0]))); + output, GET_PARAM_TYPE(&pInput[1]), GET_PARAM_TYPE(&pInput[0]), pInput->charsetCxt)); SCL_ERR_JRET(colDataSetVal(pOutputData, i, output, false)); } } else { @@ -1105,7 +1105,7 @@ static int32_t doTrimFunction(SScalarParam *pInput, int32_t inputNum, SScalarPar char *input = colDataGetData(pInputData[0], i); int32_t len = varDataLen(input); int32_t charLen = (type == TSDB_DATA_TYPE_VARCHAR) ? len : len / TSDB_NCHAR_SIZE; - trimSpaceFn(input, output, type, charLen); + trimSpaceFn(input, output, type, charLen, pInput->charsetCxt); SCL_ERR_JRET(colDataSetVal(pOutputData, i, output, false)); } } @@ -1343,7 +1343,7 @@ int32_t charFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp if (convBuf == NULL) { SCL_ERR_RET(terrno); } - int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(colDataGetData(pInput[j].columnData, colIdx)), varDataLen(colDataGetData(pInput[j].columnData, colIdx)), convBuf); + int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(colDataGetData(pInput[j].columnData, colIdx)), varDataLen(colDataGetData(pInput[j].columnData, colIdx)), convBuf, pInput->charsetCxt); if (len < 0) { taosMemoryFree(convBuf); code = TSDB_CODE_SCALAR_CONVERT_ERROR; @@ -1386,7 +1386,7 @@ int32_t asciiFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOut int32_t inLen = varDataLen(colDataGetData(pInputData, i)); SCL_ERR_RET(convBetweenNcharAndVarchar(varDataVal(colDataGetData(pInputData, i)), &in, varDataLen(colDataGetData(pInputData, i)), &inLen, - TSDB_DATA_TYPE_VARBINARY)); + TSDB_DATA_TYPE_VARBINARY, pInput->charsetCxt)); out[i] = (uint8_t)(in)[0]; taosMemoryFree(in); } else { @@ -1454,7 +1454,9 @@ int32_t positionFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p int32_t orgLen = varDataLen(colDataGetData(pInputData[1], colIdx2)); bool needFreeSub = false; if (GET_PARAM_TYPE(&pInput[1]) != GET_PARAM_TYPE(&pInput[0])) { - SCL_ERR_RET(convBetweenNcharAndVarchar(varDataVal(colDataGetData(pInputData[0], colIdx1)), &substr, varDataLen(colDataGetData(pInputData[0], colIdx1)), &subLen, GET_PARAM_TYPE(&pInput[1]))); + SCL_ERR_RET(convBetweenNcharAndVarchar(varDataVal(colDataGetData(pInputData[0], colIdx1)), &substr, + varDataLen(colDataGetData(pInputData[0], colIdx1)), &subLen, + GET_PARAM_TYPE(&pInput[1]), pInput->charsetCxt)); needFreeSub = true; } @@ -1562,13 +1564,13 @@ int32_t replaceFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pO if (GET_PARAM_TYPE(&pInput[1]) != GET_PARAM_TYPE(&pInput[0])) { SCL_ERR_JRET(convBetweenNcharAndVarchar(varDataVal(colDataGetData(pInputData[1], colIdx2)), &fromStr, varDataLen(colDataGetData(pInputData[1], colIdx2)), &fromLen, - GET_PARAM_TYPE(&pInput[0]))); + GET_PARAM_TYPE(&pInput[0]), pInput->charsetCxt)); needFreeFrom = true; } if (GET_PARAM_TYPE(&pInput[2]) != GET_PARAM_TYPE(&pInput[0])) { code = convBetweenNcharAndVarchar(varDataVal(colDataGetData(pInputData[2], colIdx3)), &toStr, varDataLen(colDataGetData(pInputData[2], colIdx3)), &toLen, - GET_PARAM_TYPE(&pInput[0])); + GET_PARAM_TYPE(&pInput[0]), pInput->charsetCxt); if (TSDB_CODE_SUCCESS != code) { if (needFreeFrom) { taosMemoryFree(fromStr); @@ -1674,7 +1676,7 @@ int32_t substrIdxFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam * if (GET_PARAM_TYPE(&pInput[0]) != GET_PARAM_TYPE(&pInput[1])) { SCL_ERR_JRET(convBetweenNcharAndVarchar(varDataVal(colDataGetData(pInputData[1], colIdx2)), &delimStr, varDataLen(colDataGetData(pInputData[1], colIdx2)), &delimLen, - GET_PARAM_TYPE(&pInput[0]))); + GET_PARAM_TYPE(&pInput[0]), pInput->charsetCxt)); needFreeDelim = true; } @@ -1853,7 +1855,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp buf[varDataLen(input)] = 0; *(int8_t *)output = taosStr2Int8(buf, NULL, 10); } else if (inputType == TSDB_DATA_TYPE_NCHAR) { - int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf); + int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf, pInput->charsetCxt); if (len < 0) { code = TSDB_CODE_SCALAR_CONVERT_ERROR; goto _end; @@ -1872,7 +1874,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp buf[varDataLen(input)] = 0; *(int16_t *)output = taosStr2Int16(buf, NULL, 10); } else if (inputType == TSDB_DATA_TYPE_NCHAR) { - int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf); + int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf, pInput->charsetCxt); if (len < 0) { code = TSDB_CODE_SCALAR_CONVERT_ERROR; goto _end; @@ -1890,7 +1892,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp buf[varDataLen(input)] = 0; *(int32_t *)output = taosStr2Int32(buf, NULL, 10); } else if (inputType == TSDB_DATA_TYPE_NCHAR) { - int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf); + int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf, pInput->charsetCxt); if (len < 0) { code = TSDB_CODE_SCALAR_CONVERT_ERROR; goto _end; @@ -1909,7 +1911,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp buf[varDataLen(input)] = 0; *(int64_t *)output = taosStr2Int64(buf, NULL, 10); } else if (inputType == TSDB_DATA_TYPE_NCHAR) { - int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf); + int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf, pInput->charsetCxt); if (len < 0) { code = TSDB_CODE_SCALAR_CONVERT_ERROR; goto _end; @@ -1927,7 +1929,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp buf[varDataLen(input)] = 0; *(uint8_t *)output = taosStr2UInt8(buf, NULL, 10); } else if (inputType == TSDB_DATA_TYPE_NCHAR) { - int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf); + int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf, pInput->charsetCxt); if (len < 0) { code = TSDB_CODE_SCALAR_CONVERT_ERROR; goto _end; @@ -1945,7 +1947,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp buf[varDataLen(input)] = 0; *(uint16_t *)output = taosStr2UInt16(buf, NULL, 10); } else if (inputType == TSDB_DATA_TYPE_NCHAR) { - int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf); + int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf, pInput->charsetCxt); if (len < 0) { code = TSDB_CODE_SCALAR_CONVERT_ERROR; goto _end; @@ -1963,7 +1965,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp buf[varDataLen(input)] = 0; *(uint32_t *)output = taosStr2UInt32(buf, NULL, 10); } else if (inputType == TSDB_DATA_TYPE_NCHAR) { - int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf); + int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf, pInput->charsetCxt); if (len < 0) { code = TSDB_CODE_SCALAR_CONVERT_ERROR; goto _end; @@ -1981,7 +1983,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp buf[varDataLen(input)] = 0; *(uint64_t *)output = taosStr2UInt64(buf, NULL, 10); } else if (inputType == TSDB_DATA_TYPE_NCHAR) { - int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf); + int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf, pInput->charsetCxt); if (len < 0) { code = TSDB_CODE_SCALAR_CONVERT_ERROR; goto _end; @@ -2000,7 +2002,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp buf[varDataLen(input)] = 0; *(float *)output = taosStr2Float(buf, NULL); } else if (inputType == TSDB_DATA_TYPE_NCHAR) { - int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf); + int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf, pInput->charsetCxt); if (len < 0) { code = TSDB_CODE_SCALAR_CONVERT_ERROR; goto _end; @@ -2018,7 +2020,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp buf[varDataLen(input)] = 0; *(double *)output = taosStr2Double(buf, NULL); } else if (inputType == TSDB_DATA_TYPE_NCHAR) { - int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf); + int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf, pInput->charsetCxt); if (len < 0) { code = TSDB_CODE_SCALAR_CONVERT_ERROR; goto _end; @@ -2036,7 +2038,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp buf[varDataLen(input)] = 0; *(bool *)output = taosStr2Int8(buf, NULL, 10); } else if (inputType == TSDB_DATA_TYPE_NCHAR) { - int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf); + int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf, pInput->charsetCxt); if (len < 0) { code = TSDB_CODE_SCALAR_CONVERT_ERROR; goto _end; @@ -2053,7 +2055,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp if (inputType == TSDB_DATA_TYPE_BINARY || inputType == TSDB_DATA_TYPE_NCHAR) { int64_t timePrec; GET_TYPED_DATA(timePrec, int64_t, GET_PARAM_TYPE(&pInput[1]), pInput[1].columnData->pData); - int32_t ret = convertStringToTimestamp(inputType, input, timePrec, &timeVal, pInput->tz); + int32_t ret = convertStringToTimestamp(inputType, input, timePrec, &timeVal, pInput->tz, pInput->charsetCxt); if (ret != TSDB_CODE_SUCCESS) { *(int64_t *)output = 0; } else { @@ -2076,7 +2078,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp (void)memcpy(varDataVal(output), varDataVal(input), len); varDataSetLen(output, len); } else if (inputType == TSDB_DATA_TYPE_NCHAR) { - int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf); + int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), convBuf, pInput->charsetCxt); if (len < 0) { code = TSDB_CODE_SCALAR_CONVERT_ERROR; goto _end; @@ -2111,7 +2113,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp if (inputType == TSDB_DATA_TYPE_BOOL) { char tmp[8] = {0}; len = tsnprintf(tmp, sizeof(tmp), "%.*s", outputCharLen, *(int8_t *)input ? "true" : "false"); - bool ret = taosMbsToUcs4(tmp, len, (TdUcs4 *)varDataVal(output), outputLen - VARSTR_HEADER_SIZE, &len); + bool ret = taosMbsToUcs4(tmp, len, (TdUcs4 *)varDataVal(output), outputLen - VARSTR_HEADER_SIZE, &len, pInput->charsetCxt); if (!ret) { code = TSDB_CODE_SCALAR_CONVERT_ERROR; goto _end; @@ -2121,7 +2123,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp } else if (inputType == TSDB_DATA_TYPE_BINARY) { len = outputCharLen > varDataLen(input) ? varDataLen(input) : outputCharLen; bool ret = taosMbsToUcs4(input + VARSTR_HEADER_SIZE, len, (TdUcs4 *)varDataVal(output), - outputLen - VARSTR_HEADER_SIZE, &len); + outputLen - VARSTR_HEADER_SIZE, &len, pInput->charsetCxt); if (!ret) { code = TSDB_CODE_SCALAR_CONVERT_ERROR; goto _end; @@ -2135,7 +2137,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp NUM_TO_STRING(inputType, input, bufSize, buf); len = (int32_t)strlen(buf); len = outputCharLen > len ? len : outputCharLen; - bool ret = taosMbsToUcs4(buf, len, (TdUcs4 *)varDataVal(output), outputLen - VARSTR_HEADER_SIZE, &len); + bool ret = taosMbsToUcs4(buf, len, (TdUcs4 *)varDataVal(output), outputLen - VARSTR_HEADER_SIZE, &len, pInput->charsetCxt); if (!ret) { code = TSDB_CODE_SCALAR_CONVERT_ERROR; goto _end; @@ -2283,7 +2285,7 @@ int32_t toUnixtimestampFunction(SScalarParam *pInput, int32_t inputNum, SScalarP char *input = colDataGetData(pInput[0].columnData, i); int64_t timeVal = 0; - int32_t ret = convertStringToTimestamp(type, input, timePrec, &timeVal, pInput->tz); + int32_t ret = convertStringToTimestamp(type, input, timePrec, &timeVal, pInput->tz, pInput->charsetCxt); if (ret != TSDB_CODE_SUCCESS) { colDataSetNULL(pOutput->columnData, i); } else { @@ -2323,7 +2325,7 @@ int32_t toJsonFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu } (void)memcpy(tmp, varDataVal(input), varDataLen(input)); tmp[varDataLen(input)] = 0; - if (parseJsontoTagData(tmp, pTagVals, &pTag, NULL)) { + if (parseJsontoTagData(tmp, pTagVals, &pTag, NULL, pInput->charsetCxt)) { code = tTagNew(pTagVals, 1, true, &pTag); if (TSDB_CODE_SUCCESS != code) { tTagFree(pTag); @@ -2473,7 +2475,7 @@ int32_t timeTruncateFunction(SScalarParam *pInput, int32_t inputNum, SScalarPara char *input = colDataGetData(pInput[0].columnData, i); if (IS_VAR_DATA_TYPE(type)) { /* datetime format strings */ - int32_t ret = convertStringToTimestamp(type, input, timePrec, &timeVal, pInput->tz); + int32_t ret = convertStringToTimestamp(type, input, timePrec, &timeVal, pInput->tz, pInput->charsetCxt); if (ret != TSDB_CODE_SUCCESS) { colDataSetNULL(pOutput->columnData, i); continue; @@ -2533,7 +2535,7 @@ int32_t timeDiffFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p int32_t type = GET_PARAM_TYPE(&pInput[k]); if (IS_VAR_DATA_TYPE(type)) { /* datetime format strings */ - int32_t ret = convertStringToTimestamp(type, input[k], TSDB_TIME_PRECISION_NANO, &timeVal[k], pInput->tz); + int32_t ret = convertStringToTimestamp(type, input[k], TSDB_TIME_PRECISION_NANO, &timeVal[k], pInput->tz, pInput->charsetCxt); if (ret != TSDB_CODE_SUCCESS) { hasNull = true; break; @@ -2660,10 +2662,6 @@ int32_t todayFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOut return TSDB_CODE_SUCCESS; } -int32_t timeZoneStrLen() { - return sizeof(VarDataLenT) + strlen(tsTimezoneStr); -} - int32_t timezoneFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { char output[TD_TIMEZONE_LEN + VARSTR_HEADER_SIZE] = {0}; // pInput->tz todo tz @@ -2699,7 +2697,7 @@ int32_t weekdayFunctionImpl(SScalarParam *pInput, int32_t inputNum, SScalarParam char *input = colDataGetData(pInput[0].columnData, i); if (IS_VAR_DATA_TYPE(type)) { /* datetime format strings */ - int32_t ret = convertStringToTimestamp(type, input, timePrec, &timeVal, pInput->tz); + int32_t ret = convertStringToTimestamp(type, input, timePrec, &timeVal, pInput->tz, pInput->charsetCxt); if (ret != TSDB_CODE_SUCCESS) { colDataSetNULL(pOutput->columnData, i); continue; @@ -2809,7 +2807,7 @@ int32_t weekFunctionImpl(SScalarParam *pInput, int32_t inputNum, SScalarParam *p char *input = colDataGetData(pInput[0].columnData, i); if (IS_VAR_DATA_TYPE(type)) { /* datetime format strings */ - int32_t ret = convertStringToTimestamp(type, input, prec, &timeVal, pInput->tz); + int32_t ret = convertStringToTimestamp(type, input, prec, &timeVal, pInput->tz, pInput->charsetCxt); if (ret != TSDB_CODE_SUCCESS) { colDataSetNULL(pOutput->columnData, i); continue; diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index c69501d50b..85bbd40573 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -103,7 +103,7 @@ int32_t convertNcharToDouble(const void *inData, void *outData) { if (NULL == tmp) { SCL_ERR_RET(terrno); } - int len = taosUcs4ToMbs((TdUcs4 *)varDataVal(inData), varDataLen(inData), tmp); + int len = taosUcs4ToMbs((TdUcs4 *)varDataVal(inData), varDataLen(inData), tmp, NULL); if (len < 0) { sclError("castConvert taosUcs4ToMbs error 1"); SCL_ERR_JRET(TSDB_CODE_SCALAR_CONVERT_ERROR); @@ -404,7 +404,7 @@ static FORCE_INLINE int32_t varToNchar(char *buf, SScalarParam *pOut, int32_t ro SCL_ERR_RET(terrno); } int32_t ret = - taosMbsToUcs4(varDataVal(buf), inputLen, (TdUcs4 *)varDataVal(t), outputMaxLen - VARSTR_HEADER_SIZE, &len); + taosMbsToUcs4(varDataVal(buf), inputLen, (TdUcs4 *)varDataVal(t), outputMaxLen - VARSTR_HEADER_SIZE, &len, pOut->charsetCxt); if (!ret) { sclError("failed to convert to NCHAR"); SCL_ERR_JRET(TSDB_CODE_SCALAR_CONVERT_ERROR); @@ -426,7 +426,7 @@ static FORCE_INLINE int32_t ncharToVar(char *buf, SScalarParam *pOut, int32_t ro if (NULL == t) { SCL_ERR_RET(terrno); } - int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(buf), varDataLen(buf), varDataVal(t)); + int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(buf), varDataLen(buf), varDataVal(t), pOut->charsetCxt); if (len < 0) { SCL_ERR_JRET(TSDB_CODE_SCALAR_CONVERT_ERROR); } @@ -557,7 +557,7 @@ int32_t vectorConvertFromVarData(SSclVectorConvCtx *pCtx, int32_t *overflow) { SCL_ERR_JRET(TSDB_CODE_APP_ERROR); } - int len = taosUcs4ToMbs((TdUcs4 *)varDataVal(data), varDataLen(data), tmp); + int len = taosUcs4ToMbs((TdUcs4 *)varDataVal(data), varDataLen(data), tmp, pCtx->pIn->charsetCxt); if (len < 0) { sclError("castConvert taosUcs4ToMbs error 1"); SCL_ERR_JRET(TSDB_CODE_SCALAR_CONVERT_ERROR); @@ -592,7 +592,7 @@ int32_t getVectorDoubleValue_JSON(void *src, int32_t index, double *out) { SCL_RET(TSDB_CODE_SUCCESS); } -int32_t ncharTobinary(void *buf, void **out) { // todo need to remove , if tobinary is nchar +int32_t ncharTobinary(void *buf, void **out, void* charsetCxt) { // todo need to remove , if tobinary is nchar int32_t inputLen = varDataTLen(buf); *out = taosMemoryCalloc(1, inputLen); @@ -601,7 +601,7 @@ int32_t ncharTobinary(void *buf, void **out) { // todo need to remove , if tobi DEFAULT_UNICODE_ENCODEC, tsCharset, (char *)varDataVal(buf)); SCL_ERR_RET(terrno); } - int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(buf), varDataLen(buf), varDataVal(*out)); + int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(buf), varDataLen(buf), varDataVal(*out), charsetCxt); if (len < 0) { sclError("charset:%s to %s. val:%s convert ncharTobinary failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, (char *)varDataVal(buf)); @@ -614,7 +614,7 @@ int32_t ncharTobinary(void *buf, void **out) { // todo need to remove , if tobi int32_t convertJsonValue(__compar_fn_t *fp, int32_t optr, int8_t typeLeft, int8_t typeRight, char **pLeftData, char **pRightData, void *pLeftOut, void *pRightOut, bool *isNull, bool *freeLeft, - bool *freeRight, bool *result) { + bool *freeRight, bool *result, void* charsetCxt) { *result = false; if (optr == OP_TYPE_JSON_CONTAINS) { *result = true; @@ -700,13 +700,13 @@ int32_t convertJsonValue(__compar_fn_t *fp, int32_t optr, int8_t typeLeft, int8_ type == TSDB_DATA_TYPE_GEOMETRY) { if (typeLeft == TSDB_DATA_TYPE_NCHAR) { char *tmpLeft = NULL; - SCL_ERR_RET(ncharTobinary(*pLeftData, (void *)&tmpLeft)); + SCL_ERR_RET(ncharTobinary(*pLeftData, (void *)&tmpLeft, charsetCxt)); *pLeftData = tmpLeft; *freeLeft = true; } if (typeRight == TSDB_DATA_TYPE_NCHAR) { char *tmpRight = NULL; - SCL_ERR_RET(ncharTobinary(*pRightData, (void *)&tmpRight)); + SCL_ERR_RET(ncharTobinary(*pRightData, (void *)&tmpRight, charsetCxt)); *pRightData = tmpRight; *freeRight = true; } @@ -1958,7 +1958,7 @@ int32_t doVectorCompareImpl(SScalarParam *pLeft, SScalarParam *pRight, SScalarPa bool result = false; SCL_ERR_RET(convertJsonValue(&fp, optr, GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), &pLeftData, &pRightData, - &leftOut, &rightOut, &isJsonnull, &freeLeft, &freeRight, &result)); + &leftOut, &rightOut, &isJsonnull, &freeLeft, &freeRight, &result, pLeft->charsetCxt)); if (isJsonnull) { sclError("doVectorCompareImpl: invalid json null value"); @@ -2043,8 +2043,11 @@ int32_t vectorCompareImpl(SScalarParam *pLeft, SScalarParam *pRight, SScalarPara SScalarParam *param2 = NULL; int32_t code = TSDB_CODE_SUCCESS; pRight->tz = pLeft->tz; + pRight->charsetCxt = pLeft->charsetCxt; pLeftOut.tz = pLeft->tz; + pLeftOut.charsetCxt = pLeft->charsetCxt; pRightOut.tz = pRight->tz; + pRightOut.charsetCxt = pRight->charsetCxt; if (noConvertBeforeCompare(GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), optr)) { param1 = pLeft; param2 = pRight; @@ -2130,6 +2133,7 @@ int32_t vectorIsNull(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pO int32_t vectorNotNull(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOut, int32_t _ord) { if (pRight != NULL) { pRight->tz = pLeft->tz; + pRight->charsetCxt = pLeft->charsetCxt; } for (int32_t i = 0; i < pLeft->numOfRows; ++i) { int8_t v = IS_HELPER_NULL(pLeft->columnData, i) ? 0 : 1; @@ -2146,6 +2150,8 @@ int32_t vectorNotNull(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *p int32_t vectorIsTrue(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOut, int32_t _ord) { if (pRight != NULL) { pRight->tz = pLeft->tz; + pRight->charsetCxt = pLeft->charsetCxt; + } SCL_ERR_RET(vectorConvertSingleColImpl(pLeft, pOut, NULL, -1, -1)); for (int32_t i = 0; i < pOut->numOfRows; ++i) { diff --git a/source/libs/scalar/test/scalar/scalarTests.cpp b/source/libs/scalar/test/scalar/scalarTests.cpp index 4cab644582..6a5188f208 100644 --- a/source/libs/scalar/test/scalar/scalarTests.cpp +++ b/source/libs/scalar/test/scalar/scalarTests.cpp @@ -1450,7 +1450,7 @@ TEST(columnTest, json_column_arith_op) { SArray *tags = taosArrayInit(1, sizeof(STagVal)); ASSERT_NE(tags, nullptr); STag *row = NULL; - int32_t code = parseJsontoTagData(rightv, tags, &row, NULL); + int32_t code = parseJsontoTagData(rightv, tags, &row, NULL, NULL); ASSERT_EQ(code, TSDB_CODE_SUCCESS); const int32_t len = 8; @@ -1606,7 +1606,7 @@ void *prepareNchar(char *rightData) { int32_t inputLen = strlen(rightData); char *t = (char *)taosMemoryCalloc(1, (inputLen + 1) * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE); - taosMbsToUcs4(rightData, inputLen, (TdUcs4 *)varDataVal(t), inputLen * TSDB_NCHAR_SIZE, &len); + taosMbsToUcs4(rightData, inputLen, (TdUcs4 *)varDataVal(t), inputLen * TSDB_NCHAR_SIZE, &len, NULL); varDataSetLen(t, len); return t; } @@ -1623,7 +1623,7 @@ TEST(columnTest, json_column_logic_op) { SArray *tags = taosArrayInit(1, sizeof(STagVal)); ASSERT_NE(tags, nullptr); STag *row = NULL; - code = parseJsontoTagData(rightv, tags, &row, NULL); + code = parseJsontoTagData(rightv, tags, &row, NULL, NULL); ASSERT_EQ(code, TSDB_CODE_SUCCESS); const int32_t len0 = 6; diff --git a/source/os/src/osEnv.c b/source/os/src/osEnv.c index eb13560929..ac303092b3 100644 --- a/source/os/src/osEnv.c +++ b/source/os/src/osEnv.c @@ -25,9 +25,9 @@ SDiskSpace tsLogSpace = {0}; SDiskSpace tsTempSpace = {0}; char tsOsName[16] = {0}; char tsTimezoneStr[TD_TIMEZONE_LEN] = {0}; -enum TdTimezone tsTimezone = TdZeroZone; char tsLocale[TD_LOCALE_LEN] = {0}; char tsCharset[TD_CHARSET_LEN] = {0}; +void *tsCharsetCxt = NULL; bool tsEnableCoreFile = 1; int64_t tsPageSizeKB = 0; int64_t tsOpenMax = 0; @@ -48,6 +48,7 @@ int32_t osDefaultInit() { taosSeedRand(taosSafeRand()); taosGetSystemLocale(tsLocale, tsCharset); + (void)taosGetSystemTimezone(tsTimezoneStr); taosGetSystemInfo(); @@ -117,11 +118,6 @@ bool osTempSpaceSufficient() { return tsTempSpace.size.avail > tsTempSpace.reser int32_t osSetTimezone(const char *tz) { return taosSetGlobalTimezone(tz); } -void osSetSystemLocale(const char *inLocale, const char *inCharSet) { - if (inLocale) (void)memcpy(tsLocale, inLocale, strlen(inLocale) + 1); - if (inCharSet) (void)memcpy(tsCharset, inCharSet, strlen(inCharSet) + 1); -} - void osSetProcPath(int32_t argc, char **argv) { if (argv == NULL || argc < 1) { return; // no command line arguments diff --git a/source/os/src/osLocale.c b/source/os/src/osLocale.c index 21f781c7e4..cb0a2646fd 100644 --- a/source/os/src/osLocale.c +++ b/source/os/src/osLocale.c @@ -78,19 +78,17 @@ char *taosCharsetReplace(char *charsetstr) { * * In case that the setLocale failed to be executed, the right charset needs to be set. */ -int32_t taosSetSystemLocale(const char *inLocale, const char *inCharSet) { - OS_PARAM_CHECK(inLocale); - OS_PARAM_CHECK(inCharSet); - if (!taosValidateEncodec(inCharSet)) { - return terrno; - } +int32_t taosSetSystemLocale(const char *inLocale) { char *locale = setlocale(LC_CTYPE, inLocale); if (NULL == locale) { terrno = TSDB_CODE_INVALID_PARA; + uError("failed to set locale:%s", inLocale); return terrno; } + (void)memcpy(tsLocale, inLocale, strlen(inLocale) + 1); + return 0; } @@ -102,59 +100,6 @@ void taosGetSystemLocale(char *outLocale, char *outCharset) { tstrncpy(outLocale, locale, TD_LOCALE_LEN); } tstrncpy(outCharset, "UTF-8", TD_CHARSET_LEN); - -#elif defined(_TD_DARWIN_64) - /* - * originally from src/os/src/detail/osSysinfo.c - * POSIX format locale string: - * (Language Strings)_(Country/Region Strings).(code_page) - * - * example: en_US.UTF-8, zh_CN.GB18030, zh_CN.UTF-8, - * - * if user does not specify the locale in taos.cfg the program use default LC_CTYPE as system locale. - * - * In case of some CentOS systems, their default locale is "en_US.utf8", which is not valid code_page - * for libiconv that is employed to convert string in this system. This program will automatically use - * UTF-8 instead as the charset. - * - * In case of windows client, the locale string is not valid POSIX format, user needs to set the - * correct code_page for libiconv. Usually, the code_page of windows system with simple chinese is - * CP936, CP437 for English charset. - * - */ - - char sep = '.'; - char *locale = NULL; - - locale = setlocale(LC_CTYPE, ""); - if (locale == NULL) { - // printf("can't get locale from system, set it to en_US.UTF-8 since error:%d:%s", errno, strerror(errno)); - tstrncpy(outLocale, "en_US.UTF-8", TD_LOCALE_LEN); - } else { - tstrncpy(outLocale, locale, TD_LOCALE_LEN); - // printf("locale not configured, set to system default:%s", outLocale); - } - - /* if user does not specify the charset, extract it from locale */ - char *str = strrchr(outLocale, sep); - if (str != NULL) { - str++; - - char *revisedCharset = taosCharsetReplace(str); - - if (NULL == revisedCharset) { - tstrncpy(outCharset, "UTF-8", TD_CHARSET_LEN); - } else { - tstrncpy(outCharset, revisedCharset, TD_CHARSET_LEN); - - taosMemoryFree(revisedCharset); - } - // printf("charset not configured, set to system default:%s", outCharset); - } else { - tstrncpy(outCharset, "UTF-8", TD_CHARSET_LEN); - // printf("can't get locale and charset from system, set it to UTF-8"); - } - #else /* * POSIX format locale string: diff --git a/source/os/src/osString.c b/source/os/src/osString.c index 0ee4f1c496..4506248eec 100644 --- a/source/os/src/osString.c +++ b/source/os/src/osString.c @@ -229,80 +229,25 @@ int32_t tasoUcs4Copy(TdUcs4 *target_ucs4, TdUcs4 *source_ucs4, int32_t len_ucs4) return TSDB_CODE_SUCCESS; } -typedef struct { - iconv_t conv; - int8_t inUse; -} SConv; - -// 0: Mbs --> Ucs4 -// 1: Ucs4--> Mbs -SConv *gConv[2] = {NULL, NULL}; -int32_t convUsed[2] = {0, 0}; -int32_t gConvMaxNum[2] = {0, 0}; - -int32_t taosConvInit(void) { - int8_t M2C = 0; - gConvMaxNum[M2C] = 512; - gConvMaxNum[1 - M2C] = 512; - - gConv[M2C] = taosMemoryCalloc(gConvMaxNum[M2C], sizeof(SConv)); - if (gConv[M2C] == NULL) { - return terrno; - } - - gConv[1 - M2C] = taosMemoryCalloc(gConvMaxNum[1 - M2C], sizeof(SConv)); - if (gConv[1 - M2C] == NULL) { - taosMemoryFree(gConv[M2C]); - return terrno; - } - - for (int32_t i = 0; i < gConvMaxNum[M2C]; ++i) { - gConv[M2C][i].conv = iconv_open(DEFAULT_UNICODE_ENCODEC, tsCharset); - if ((iconv_t)-1 == gConv[M2C][i].conv) { - terrno = TAOS_SYSTEM_ERROR(errno); - return terrno; - } - } - for (int32_t i = 0; i < gConvMaxNum[1 - M2C]; ++i) { - gConv[1 - M2C][i].conv = iconv_open(tsCharset, DEFAULT_UNICODE_ENCODEC); - if ((iconv_t)-1 == gConv[1 - M2C][i].conv) { - terrno = TAOS_SYSTEM_ERROR(errno); - return terrno; - } - } - - return 0; -} - -void taosConvDestroy() { - int8_t M2C = 0; - for (int32_t i = 0; i < gConvMaxNum[M2C]; ++i) { - (void)iconv_close(gConv[M2C][i].conv); - } - for (int32_t i = 0; i < gConvMaxNum[1 - M2C]; ++i) { - (void)iconv_close(gConv[1 - M2C][i].conv); - } - taosMemoryFreeClear(gConv[M2C]); - taosMemoryFreeClear(gConv[1 - M2C]); - gConvMaxNum[M2C] = -1; - gConvMaxNum[1 - M2C] = -1; -} - -iconv_t taosAcquireConv(int32_t *idx, ConvType type) { +iconv_t taosAcquireConv(int32_t *idx, ConvType type, void* charsetCxt) { if(idx == NULL) { terrno = TSDB_CODE_INVALID_PARA; return (iconv_t)-1; } - if (gConvMaxNum[type] <= 0) { + if (charsetCxt == NULL){ + charsetCxt = tsCharsetCxt; + } + SConvInfo *info = (SConvInfo *)charsetCxt; + if (info->gConvMaxNum[type] <= 0) { *idx = -1; if (type == M2C) { - iconv_t c = iconv_open(DEFAULT_UNICODE_ENCODEC, tsCharset); + iconv_t c = iconv_open(DEFAULT_UNICODE_ENCODEC, info->charset); if ((iconv_t)-1 == c) { terrno = TAOS_SYSTEM_ERROR(errno); } return c; } else { - iconv_t c = iconv_open(tsCharset, DEFAULT_UNICODE_ENCODEC); + iconv_t c = iconv_open(info->charset, DEFAULT_UNICODE_ENCODEC); if ((iconv_t)-1 == c) { terrno = TAOS_SYSTEM_ERROR(errno); } @@ -311,9 +256,9 @@ iconv_t taosAcquireConv(int32_t *idx, ConvType type) { } while (true) { - int32_t used = atomic_add_fetch_32(&convUsed[type], 1); - if (used > gConvMaxNum[type]) { - used = atomic_sub_fetch_32(&convUsed[type], 1); + int32_t used = atomic_add_fetch_32(&info->convUsed[type], 1); + if (used > info->gConvMaxNum[type]) { + (void)atomic_sub_fetch_32(&info->convUsed[type], 1); (void)sched_yield(); continue; } @@ -321,38 +266,43 @@ iconv_t taosAcquireConv(int32_t *idx, ConvType type) { break; } - int32_t startId = taosGetSelfPthreadId() % gConvMaxNum[type]; + int32_t startId = taosGetSelfPthreadId() % info->gConvMaxNum[type]; while (true) { - if (gConv[type][startId].inUse) { - startId = (startId + 1) % gConvMaxNum[type]; + if (info->gConv[type][startId].inUse) { + startId = (startId + 1) % info->gConvMaxNum[type]; continue; } - int8_t old = atomic_val_compare_exchange_8(&gConv[type][startId].inUse, 0, 1); + int8_t old = atomic_val_compare_exchange_8(&info->gConv[type][startId].inUse, 0, 1); if (0 == old) { break; } } *idx = startId; - if ((iconv_t)0 == gConv[type][startId].conv) { + if ((iconv_t)0 == info->gConv[type][startId].conv) { return (iconv_t)-1; } else { - return gConv[type][startId].conv; + return info->gConv[type][startId].conv; } } -void taosReleaseConv(int32_t idx, iconv_t conv, ConvType type) { +void taosReleaseConv(int32_t idx, iconv_t conv, ConvType type, void* charsetCxt) { if (idx < 0) { (void)iconv_close(conv); return; } - atomic_store_8(&gConv[type][idx].inUse, 0); - (void)atomic_sub_fetch_32(&convUsed[type], 1); + if (charsetCxt == NULL){ + charsetCxt = tsCharsetCxt; + } + SConvInfo *info = (SConvInfo *)charsetCxt; + + atomic_store_8(&info->gConv[type][idx].inUse, 0); + (void)atomic_sub_fetch_32(&info->convUsed[type], 1); } -bool taosMbsToUcs4(const char *mbs, size_t mbsLength, TdUcs4 *ucs4, int32_t ucs4_max_len, int32_t *len) { +bool taosMbsToUcs4(const char *mbs, size_t mbsLength, TdUcs4 *ucs4, int32_t ucs4_max_len, int32_t *len, void* charsetCxt) { if (ucs4_max_len == 0) { return true; } @@ -368,20 +318,20 @@ bool taosMbsToUcs4(const char *mbs, size_t mbsLength, TdUcs4 *ucs4, int32_t ucs4 (void)memset(ucs4, 0, ucs4_max_len); int32_t idx = -1; - iconv_t conv = taosAcquireConv(&idx, M2C); + iconv_t conv = taosAcquireConv(&idx, M2C, charsetCxt); if ((iconv_t)-1 == conv) { return false; } - + size_t ucs4_input_len = mbsLength; size_t outLeft = ucs4_max_len; if (iconv(conv, (char **)&mbs, &ucs4_input_len, (char **)&ucs4, &outLeft) == -1) { terrno = TAOS_SYSTEM_ERROR(errno); - taosReleaseConv(idx, conv, M2C); + taosReleaseConv(idx, conv, M2C, charsetCxt); return false; } - taosReleaseConv(idx, conv, M2C); + taosReleaseConv(idx, conv, M2C, charsetCxt); if (len != NULL) { *len = (int32_t)(ucs4_max_len - outLeft); if (*len < 0) { @@ -397,7 +347,7 @@ bool taosMbsToUcs4(const char *mbs, size_t mbsLength, TdUcs4 *ucs4, int32_t ucs4 // if success, return the number of bytes written to mbs ( >= 0) // otherwise return error code ( < 0) -int32_t taosUcs4ToMbs(TdUcs4 *ucs4, int32_t ucs4_max_len, char *mbs) { +int32_t taosUcs4ToMbs(TdUcs4 *ucs4, int32_t ucs4_max_len, char *mbs, void* charsetCxt) { if (ucs4_max_len == 0) { return 0; } @@ -413,22 +363,22 @@ int32_t taosUcs4ToMbs(TdUcs4 *ucs4, int32_t ucs4_max_len, char *mbs) { int32_t idx = -1; int32_t code = 0; - iconv_t conv = taosAcquireConv(&idx, C2M); + iconv_t conv = taosAcquireConv(&idx, C2M, charsetCxt); if ((iconv_t)-1 == conv) { return terrno; } - + size_t ucs4_input_len = ucs4_max_len; size_t outLen = ucs4_max_len; if (iconv(conv, (char **)&ucs4, &ucs4_input_len, &mbs, &outLen) == -1) { code = TAOS_SYSTEM_ERROR(errno); - taosReleaseConv(idx, conv, C2M); - terrno = code; + taosReleaseConv(idx, conv, C2M, charsetCxt); + terrno = code; return code; } - - taosReleaseConv(idx, conv, C2M); - + + taosReleaseConv(idx, conv, C2M, charsetCxt); + return (int32_t)(ucs4_max_len - outLen); #endif } diff --git a/source/os/src/osTimezone.c b/source/os/src/osTimezone.c index 1f2210fbe1..183b319dcb 100644 --- a/source/os/src/osTimezone.c +++ b/source/os/src/osTimezone.c @@ -828,6 +828,16 @@ int32_t taosSetGlobalTimezone(const char *tz) { } +int32_t taosGetLocalTimezoneOffset() { + time_t tx1 = taosGetTimestampSec(); + struct tm tm1; + if (taosLocalTime(&tx1, &tm1, NULL, 0, NULL) == NULL) { + uError("%s failed to get local time: code:%d", __FUNCTION__, errno); + return TSDB_CODE_TIME_ERROR; + } + return (int32_t)(tm1.tm_gmtoff); +} + int32_t taosFormatTimezoneStr(time_t t, const char* tz, timezone_t sp, char *outTimezoneStr){ struct tm tm1; if (taosLocalTime(&t, &tm1, NULL, 0, sp) == NULL) { diff --git a/source/util/src/tcompare.c b/source/util/src/tcompare.c index b1f4ed0ed3..e3a526b2b4 100644 --- a/source/util/src/tcompare.c +++ b/source/util/src/tcompare.c @@ -1508,7 +1508,7 @@ int32_t comparewcsRegexMatch(const void *pString, const void *pPattern) { return 1; // terrno has been set } - int convertLen = taosUcs4ToMbs((TdUcs4 *)varDataVal(pPattern), len, pattern); + int convertLen = taosUcs4ToMbs((TdUcs4 *)varDataVal(pPattern), len, pattern, NULL); if (convertLen < 0) { taosMemoryFree(pattern); return 1; // terrno has been set @@ -1523,7 +1523,7 @@ int32_t comparewcsRegexMatch(const void *pString, const void *pPattern) { return 1; // terrno has been set } - convertLen = taosUcs4ToMbs((TdUcs4 *)varDataVal(pString), len, str); + convertLen = taosUcs4ToMbs((TdUcs4 *)varDataVal(pString), len, str, NULL); if (convertLen < 0) { taosMemoryFree(str); taosMemoryFree(pattern); diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index 126addc29f..57895d0cca 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -24,6 +24,7 @@ #include "tlog.h" #include "tunit.h" #include "tutil.h" +#include "tconv.h" #define CFG_NAME_PRINT_LEN 32 #define CFG_SRC_PRINT_LEN 12 @@ -254,10 +255,10 @@ static int32_t cfgSetTimezone(SConfigItem *pItem, const char *value, ECfgSrcType uError("invalid timezone:%s", value); TAOS_RETURN(TSDB_CODE_INVALID_TIMEZONE); } - if (strlen(value) == 0) { + if (value == NULL || strlen(value) == 0) { uError("cfg:%s, type:%s src:%s, value:%s, skip to set timezone", pItem->name, cfgDtypeStr(pItem->dtype), cfgStypeStr(stype), value); - TAOS_RETURN(TSDB_CODE_SUCCESS); + TAOS_RETURN(TSDB_CODE_INVALID_CFG); } TAOS_CHECK_RETURN(osSetTimezone(value)); @@ -266,6 +267,49 @@ static int32_t cfgSetTimezone(SConfigItem *pItem, const char *value, ECfgSrcType TAOS_RETURN(TSDB_CODE_SUCCESS); } +static int32_t cfgSetCharset(SConfigItem *pItem, const char *value, ECfgSrcType stype) { + if (stype == CFG_STYPE_ALTER_SERVER_CMD || stype == CFG_STYPE_ALTER_CLIENT_CMD){ + uError("failed to config charset, not support"); + TAOS_RETURN(TSDB_CODE_INVALID_CFG); + } + + if (value == NULL || strlen(value) == 0) { + uError("cfg:%s, type:%s src:%s, value:%s, skip to set charset", pItem->name, cfgDtypeStr(pItem->dtype), + cfgStypeStr(stype), value); + TAOS_RETURN(TSDB_CODE_INVALID_CFG); + } + + if (!taosValidateEncodec(value)) { + uError("invalid charset:%s", value); + TAOS_RETURN(terrno); + } + + if ((tsCharsetCxt = taosConvInit(value)) == NULL) { + TAOS_RETURN(terrno); + } + (void)memcpy(tsCharset, value, strlen(value) + 1); + TAOS_CHECK_RETURN(doSetConf(pItem, value, stype)); + + TAOS_RETURN(TSDB_CODE_SUCCESS); +} + +static int32_t cfgSetLocale(SConfigItem *pItem, const char *value, ECfgSrcType stype) { + if (stype == CFG_STYPE_ALTER_SERVER_CMD || (pItem->dynScope & CFG_DYN_CLIENT) == 0){ + uError("failed to config locale, not support"); + TAOS_RETURN(TSDB_CODE_INVALID_CFG); + } + + if (value == NULL || strlen(value) == 0 || taosSetSystemLocale(value) != 0) { + uError("cfg:%s, type:%s src:%s, value:%s, skip to set locale", pItem->name, cfgDtypeStr(pItem->dtype), + cfgStypeStr(stype), value); + TAOS_RETURN(TSDB_CODE_INVALID_CFG); + } + + TAOS_CHECK_RETURN(doSetConf(pItem, value, stype)); + + TAOS_RETURN(TSDB_CODE_SUCCESS); +} + static int32_t cfgSetTfsItem(SConfig *pCfg, const char *name, const char *value, const char *level, const char *primary, const char *disable, ECfgSrcType stype) { (void)taosThreadMutexLock(&pCfg->lock); @@ -378,11 +422,11 @@ int32_t cfgSetItem(SConfig *pCfg, const char *name, const char *value, ECfgSrcTy break; } case CFG_DTYPE_CHARSET: { - code = doSetConf(pItem, value, stype); + code = cfgSetCharset(pItem, value, stype); break; } case CFG_DTYPE_LOCALE: { - code = doSetConf(pItem, value, stype); + code = cfgSetLocale(pItem, value, stype); break; } case CFG_DTYPE_NONE: diff --git a/source/util/src/tconv.c b/source/util/src/tconv.c new file mode 100644 index 0000000000..d9932489f8 --- /dev/null +++ b/source/util/src/tconv.c @@ -0,0 +1,123 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define ALLOW_FORBID_FUNC + +#include "tconv.h" +#include "thash.h" +#include "osString.h" + +#define CONV_MAX_NUM 32 +SHashObj *gConvInfo = NULL; + +// M2C: Mbs --> Ucs4 +// C2M: Ucs4--> Mbs + +static void taosConvDestroyInner(void *arg) { + SConvInfo *info = (SConvInfo *)arg; + if (info == NULL) { + return; + } + for (int32_t i = 0; i < info->gConvMaxNum[M2C]; ++i) { + (void)iconv_close(info->gConv[M2C][i].conv); + } + for (int32_t i = 0; i < info->gConvMaxNum[C2M]; ++i) { + (void)iconv_close(info->gConv[C2M][i].conv); + } + taosMemoryFreeClear(info->gConv[M2C]); + taosMemoryFreeClear(info->gConv[C2M]); + + info->gConvMaxNum[M2C] = -1; + info->gConvMaxNum[C2M] = -1; +} + +void* taosConvInit(const char* charset) { + if (charset == NULL){ + terrno = TSDB_CODE_INVALID_PARA; + return NULL; + } + + void* conv = NULL; + static int32_t lock_c = 0; + + for (int i = 1; atomic_val_compare_exchange_32(&lock_c, 0, 1) != 0; ++i) { + if (i % 1000 == 0) { + uInfo("haven't acquire lock after spin %d times.", i); + (void)sched_yield(); + } + } + + if (gConvInfo == NULL){ + gConvInfo = taosHashInit(0, MurmurHash3_32, false, HASH_ENTRY_LOCK); + if (gConvInfo == NULL) { + atomic_store_32(&lock_c, 0); + goto END; + } + taosHashSetFreeFp(gConvInfo, taosConvDestroyInner); + } + + conv = taosHashGet(gConvInfo, charset, strlen(charset)); + if (conv != NULL){ + goto END; + } + + SConvInfo info = {0}; + info.gConvMaxNum[M2C] = CONV_MAX_NUM; + info.gConvMaxNum[C2M] = CONV_MAX_NUM; + tstrncpy(info.charset, charset, sizeof(info.charset)); + + info.gConv[M2C] = taosMemoryCalloc(info.gConvMaxNum[M2C], sizeof(SConv)); + if (info.gConv[M2C] == NULL) { + goto FAILED; + } + + info.gConv[C2M] = taosMemoryCalloc(info.gConvMaxNum[C2M], sizeof(SConv)); + if (info.gConv[C2M] == NULL) { + goto FAILED; + } + + for (int32_t i = 0; i < info.gConvMaxNum[M2C]; ++i) { + info.gConv[M2C][i].conv = iconv_open(DEFAULT_UNICODE_ENCODEC, charset); + if ((iconv_t)-1 == info.gConv[M2C][i].conv) { + terrno = TAOS_SYSTEM_ERROR(errno); + goto FAILED; + } + } + for (int32_t i = 0; i < info.gConvMaxNum[C2M]; ++i) { + info.gConv[C2M][i].conv = iconv_open(charset, DEFAULT_UNICODE_ENCODEC); + if ((iconv_t)-1 == info.gConv[C2M][i].conv) { + terrno = TAOS_SYSTEM_ERROR(errno); + goto FAILED; + } + } + + int32_t code = taosHashPut(gConvInfo, charset, strlen(charset), &info, sizeof(info)); + if (code != 0){ + goto FAILED; + } + conv = taosHashGet(gConvInfo, charset, strlen(charset)); + goto END; + +FAILED: + taosConvDestroyInner(&info); + +END: + atomic_store_32(&lock_c, 0); + return conv; +} + +void taosConvDestroy() { + taosHashCleanup(gConvInfo); +}