diff --git a/include/common/ttypes.h b/include/common/ttypes.h index 559983185c..279799b172 100644 --- a/include/common/ttypes.h +++ b/include/common/ttypes.h @@ -269,7 +269,7 @@ typedef struct { (IS_NUMERIC_TYPE(_t) || (_t) == (TSDB_DATA_TYPE_BOOL) || (_t) == (TSDB_DATA_TYPE_TIMESTAMP)) #define IS_VAR_DATA_TYPE(t) \ - (((t) == TSDB_DATA_TYPE_VARCHAR) || (t) == TSDB_DATA_TYPE_VARBINARY || ((t) == TSDB_DATA_TYPE_NCHAR) || ((t) == TSDB_DATA_TYPE_JSON) || ((t) == TSDB_DATA_TYPE_GEOMETRY)) + (((t) == TSDB_DATA_TYPE_VARCHAR) || ((t) == TSDB_DATA_TYPE_VARBINARY) || ((t) == TSDB_DATA_TYPE_NCHAR) || ((t) == TSDB_DATA_TYPE_JSON) || ((t) == TSDB_DATA_TYPE_GEOMETRY)) #define IS_STR_DATA_TYPE(t) (((t) == TSDB_DATA_TYPE_VARCHAR) || ((t) == TSDB_DATA_TYPE_VARBINARY) || ((t) == TSDB_DATA_TYPE_NCHAR)) #define IS_VALID_TINYINT(_t) ((_t) >= INT8_MIN && (_t) <= INT8_MAX) diff --git a/include/os/osString.h b/include/os/osString.h index b609c9d351..ed64cbfe3f 100644 --- a/include/os/osString.h +++ b/include/os/osString.h @@ -90,6 +90,9 @@ int8_t taosStr2Int8(const char *str, char **pEnd, int32_t radix); uint8_t taosStr2UInt8(const char *str, char **pEnd, int32_t radix); double taosStr2Double(const char *str, char **pEnd); float taosStr2Float(const char *str, char **pEnd); +int32_t taosHex2Ascii(const char *z, uint32_t n, void** data, uint32_t* size); +int32_t taosAscii2Hex(const char *z, uint32_t n, void** data, uint32_t* size); +int32_t taosBin2Ascii(const char *z, uint32_t n, void** data, uint32_t* size); #ifdef __cplusplus } diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index d448dd1edf..c78ba4c4a0 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -503,7 +503,7 @@ void setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t pResInfo->userFields[i].bytes = pSchema[i].bytes; pResInfo->userFields[i].type = pSchema[i].type; - if (pSchema[i].type == TSDB_DATA_TYPE_VARCHAR || pSchema[i].type == TSDB_DATA_TYPE_GEOMETRY) { + if (pSchema[i].type == TSDB_DATA_TYPE_VARCHAR || pSchema[i].type == TSDB_DATA_TYPE_VARBINARY || pSchema[i].type == TSDB_DATA_TYPE_GEOMETRY) { pResInfo->userFields[i].bytes -= VARSTR_HEADER_SIZE; } else if (pSchema[i].type == TSDB_DATA_TYPE_NCHAR || pSchema[i].type == TSDB_DATA_TYPE_JSON) { pResInfo->userFields[i].bytes = (pResInfo->userFields[i].bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE; diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c index 0b28949400..af6b3bcc8c 100644 --- a/source/client/src/clientRawBlockWrite.c +++ b/source/client/src/clientRawBlockWrite.c @@ -292,7 +292,13 @@ static void buildChildElement(cJSON* json, SVCreateTbReq* pCreateReq) { cJSON* tvalue = NULL; if (IS_VAR_DATA_TYPE(pTagVal->type)) { - char* buf = taosMemoryCalloc(pTagVal->nData + 3, 1); + char* buf = NULL; + if(pTagVal->type == TSDB_DATA_TYPE_VARBINARY){ + buf = taosMemoryCalloc(pTagVal->nData*2 + 2 + 3, 1); + }else{ + buf = taosMemoryCalloc(pTagVal->nData + 3, 1); + } + if (!buf) goto end; dataConverToStr(buf, pTagVal->type, pTagVal->pData, pTagVal->nData, NULL); tvalue = cJSON_CreateString(buf); @@ -515,7 +521,11 @@ static char* processAlterTable(SMqMetaRsp* metaRsp) { } buf = parseTagDatatoJson(vAlterTbReq.pTagVal); } else { - buf = taosMemoryCalloc(vAlterTbReq.nTagVal + 1, 1); + if(vAlterTbReq.tagType == TSDB_DATA_TYPE_VARBINARY){ + buf = taosMemoryCalloc(vAlterTbReq.nTagVal + 1, 1); + }else{ + buf = taosMemoryCalloc(vAlterTbReq.nTagVal + 1, 1); + } dataConverToStr(buf, vAlterTbReq.tagType, vAlterTbReq.pTagVal, vAlterTbReq.nTagVal, NULL); } diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 343cb9d1a3..266368e9bc 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -596,7 +596,7 @@ static int32_t smlGenerateSchemaAction(SSchema *colField, SHashObj *colHash, SSm return TSDB_CODE_SML_INVALID_DATA; } - if (((colField[*index].type == TSDB_DATA_TYPE_VARCHAR || colField[*index].type == TSDB_DATA_TYPE_GEOMETRY) && + if (((colField[*index].type == TSDB_DATA_TYPE_VARCHAR || colField[*index].type == TSDB_DATA_TYPE_VARCHAR || colField[*index].type == TSDB_DATA_TYPE_GEOMETRY) && (colField[*index].bytes - VARSTR_HEADER_SIZE) < kv->length) || (colField[*index].type == TSDB_DATA_TYPE_NCHAR && ((colField[*index].bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE < kv->length))) { diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 5188b1e27c..efbfa6e4e6 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1852,6 +1852,7 @@ char* dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** pDataBuf) if (len >= size - 1) return dumpBuf; break; case TSDB_DATA_TYPE_VARCHAR: + case TSDB_DATA_TYPE_VARBINARY: case TSDB_DATA_TYPE_GEOMETRY: { memset(pBuf, 0, sizeof(pBuf)); char* pData = colDataGetVarData(pColInfoData, j); @@ -1951,6 +1952,7 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq2** ppReq, const SSDataBlock* pDat } break; case TSDB_DATA_TYPE_NCHAR: + case TSDB_DATA_TYPE_VARBINARY: case TSDB_DATA_TYPE_VARCHAR: { // TSDB_DATA_TYPE_BINARY ASSERT(pColInfoData->info.type == pCol->type); if (colDataIsNull_s(pColInfoData, j)) { @@ -1964,7 +1966,6 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq2** ppReq, const SSDataBlock* pDat } break; } - case TSDB_DATA_TYPE_VARBINARY: case TSDB_DATA_TYPE_DECIMAL: case TSDB_DATA_TYPE_BLOB: case TSDB_DATA_TYPE_JSON: diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 7c6939635a..7b8f0e67fb 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -1146,6 +1146,7 @@ static int tTagValJsonCmprFn(const void *p1, const void *p2) { static void debugPrintTagVal(int8_t type, const void *val, int32_t vlen, const char *tag, int32_t ln) { switch (type) { + case TSDB_DATA_TYPE_VARBINARY: case TSDB_DATA_TYPE_JSON: case TSDB_DATA_TYPE_VARCHAR: case TSDB_DATA_TYPE_NCHAR: diff --git a/source/common/src/trow.c b/source/common/src/trow.c index 039f436505..e2da4d166e 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -356,7 +356,7 @@ int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow) { } } else { varDataLen += sizeof(VarDataLenT); - if (pTColumn->type == TSDB_DATA_TYPE_VARCHAR || pTColumn->type == TSDB_DATA_TYPE_GEOMETRY) { + if (pTColumn->type == TSDB_DATA_TYPE_VARCHAR || pTColumn->type == TSDB_DATA_TYPE_VARBINARY || pTColumn->type == TSDB_DATA_TYPE_GEOMETRY) { varDataLen += CHAR_BYTES; if (maxVarDataLen < CHAR_BYTES + sizeof(VarDataLenT)) { maxVarDataLen = CHAR_BYTES + sizeof(VarDataLenT); diff --git a/source/common/src/ttypes.c b/source/common/src/ttypes.c index 6acd2d087a..8827ddc811 100644 --- a/source/common/src/ttypes.c +++ b/source/common/src/ttypes.c @@ -61,7 +61,7 @@ tDataTypeDescriptor tDataTypes[TSDB_DATA_TYPE_MAX] = { {TSDB_DATA_TYPE_UINT, 12, INT_BYTES, "INT UNSIGNED", 0, UINT32_MAX, tsCompressInt, tsDecompressInt}, {TSDB_DATA_TYPE_UBIGINT, 15, LONG_BYTES, "BIGINT UNSIGNED", 0, UINT64_MAX, tsCompressBigint, tsDecompressBigint}, {TSDB_DATA_TYPE_JSON, 4, TSDB_MAX_JSON_TAG_LEN, "JSON", 0, 0, tsCompressString, tsDecompressString}, - {TSDB_DATA_TYPE_VARBINARY, 9, 1, "VARBINARY", 0, 0, NULL, NULL}, // placeholder, not implemented + {TSDB_DATA_TYPE_VARBINARY, 9, 1, "VARBINARY", 0, 0, tsCompressString, tsDecompressString}, // placeholder, not implemented {TSDB_DATA_TYPE_DECIMAL, 7, 1, "DECIMAL", 0, 0, NULL, NULL}, // placeholder, not implemented {TSDB_DATA_TYPE_BLOB, 4, 1, "BLOB", 0, 0, NULL, NULL}, // placeholder, not implemented {TSDB_DATA_TYPE_MEDIUMBLOB, 10, 1, "MEDIUMBLOB", 0, 0, NULL, NULL}, // placeholder, not implemented diff --git a/source/libs/command/src/command.c b/source/libs/command/src/command.c index 98aab26b0d..444da63a56 100644 --- a/source/libs/command/src/command.c +++ b/source/libs/command/src/command.c @@ -458,7 +458,7 @@ void appendColumnFields(char* buf, int32_t* len, STableCfg* pCfg) { SSchema* pSchema = pCfg->pSchemas + i; char type[32]; sprintf(type, "%s", tDataTypes[pSchema->type].name); - if (TSDB_DATA_TYPE_VARCHAR == pSchema->type || TSDB_DATA_TYPE_GEOMETRY == pSchema->type) { + if (TSDB_DATA_TYPE_VARCHAR == pSchema->type || TSDB_DATA_TYPE_VARBINARY == pSchema->type || TSDB_DATA_TYPE_GEOMETRY == pSchema->type) { sprintf(type + strlen(type), "(%d)", (int32_t)(pSchema->bytes - VARSTR_HEADER_SIZE)); } else if (TSDB_DATA_TYPE_NCHAR == pSchema->type) { sprintf(type + strlen(type), "(%d)", (int32_t)((pSchema->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)); @@ -473,7 +473,7 @@ void appendTagFields(char* buf, int32_t* len, STableCfg* pCfg) { SSchema* pSchema = pCfg->pSchemas + pCfg->numOfColumns + i; char type[32]; sprintf(type, "%s", tDataTypes[pSchema->type].name); - if (TSDB_DATA_TYPE_VARCHAR == pSchema->type || TSDB_DATA_TYPE_GEOMETRY == pSchema->type) { + if (TSDB_DATA_TYPE_VARCHAR == pSchema->type || TSDB_DATA_TYPE_VARBINARY == pSchema->type || TSDB_DATA_TYPE_GEOMETRY == pSchema->type) { sprintf(type + strlen(type), "(%d)", (int32_t)(pSchema->bytes - VARSTR_HEADER_SIZE)); } else if (TSDB_DATA_TYPE_NCHAR == pSchema->type) { sprintf(type + strlen(type), "(%d)", (int32_t)((pSchema->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)); diff --git a/source/libs/executor/src/dataInserter.c b/source/libs/executor/src/dataInserter.c index 646964ebf4..c65dfd97d1 100644 --- a/source/libs/executor/src/dataInserter.c +++ b/source/libs/executor/src/dataInserter.c @@ -213,6 +213,7 @@ int32_t buildSubmitReqFromBlock(SDataInserterHandle* pInserter, SSubmitReq2** pp switch (pColInfoData->info.type) { case TSDB_DATA_TYPE_NCHAR: + case TSDB_DATA_TYPE_VARBINARY: case TSDB_DATA_TYPE_VARCHAR: { // TSDB_DATA_TYPE_BINARY ASSERT(pColInfoData->info.type == pCol->type); if (colDataIsNull_s(pColInfoData, j)) { @@ -226,7 +227,6 @@ int32_t buildSubmitReqFromBlock(SDataInserterHandle* pInserter, SSubmitReq2** pp } break; } - case TSDB_DATA_TYPE_VARBINARY: case TSDB_DATA_TYPE_DECIMAL: case TSDB_DATA_TYPE_BLOB: case TSDB_DATA_TYPE_JSON: diff --git a/source/libs/index/src/indexComm.c b/source/libs/index/src/indexComm.c index 9f8c2139eb..1313221952 100644 --- a/source/libs/index/src/indexComm.c +++ b/source/libs/index/src/indexComm.c @@ -357,17 +357,13 @@ int32_t idxConvertData(void* src, int8_t type, void** dst) { break; } case TSDB_DATA_TYPE_VARCHAR: // TSDB_DATA_TYPE_BINARY + case TSDB_DATA_TYPE_VARBINARY: case TSDB_DATA_TYPE_GEOMETRY: { tlen = taosEncodeBinary(NULL, src, strlen(src)); *dst = taosMemoryCalloc(1, tlen + 1); tlen = taosEncodeBinary(dst, src, strlen(src)); break; } - case TSDB_DATA_TYPE_VARBINARY: - tlen = taosEncodeBinary(NULL, src, strlen(src)); - *dst = taosMemoryCalloc(1, tlen + 1); - tlen = taosEncodeBinary(dst, src, strlen(src)); - break; default: ASSERTS(0, "index invalid input type"); break; @@ -448,6 +444,7 @@ int32_t idxConvertDataToStr(void* src, int8_t type, void** dst) { break; } case TSDB_DATA_TYPE_VARCHAR: // TSDB_DATA_TYPE_BINARY + case TSDB_DATA_TYPE_VARBINARY: case TSDB_DATA_TYPE_GEOMETRY: { tlen = taosEncodeBinary(NULL, varDataVal(src), varDataLen(src)); *dst = taosMemoryCalloc(1, tlen + 1); @@ -455,12 +452,6 @@ int32_t idxConvertDataToStr(void* src, int8_t type, void** dst) { *dst = (char*)*dst - tlen; break; } - case TSDB_DATA_TYPE_VARBINARY: - tlen = taosEncodeBinary(NULL, varDataVal(src), varDataLen(src)); - *dst = taosMemoryCalloc(1, tlen + 1); - tlen = taosEncodeBinary(dst, varDataVal(src), varDataLen(src)); - *dst = (char*)*dst - tlen; - break; default: ASSERTS(0, "index invalid input type"); break; diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index 1000d670d8..c58647e540 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -479,11 +479,32 @@ static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema, } case TSDB_DATA_TYPE_VARBINARY: { // Too long values will raise the invalid sql error message - if (pToken->n + VARSTR_HEADER_SIZE > pSchema->bytes) { + // Too long values will raise the invalid sql error message + void* data = NULL; + uint32_t size = 0; + if (pToken->type == TK_NK_HEX){ + if(taosHex2Ascii(pToken->z, pToken->n, &data, &size) < 0){ + return TSDB_CODE_OUT_OF_MEMORY; + } + }else if(pToken->type == TK_NK_BIN){ + if(taosBin2Ascii(pToken->z, pToken->n, &data, &size) < 0){ + return TSDB_CODE_OUT_OF_MEMORY; + } + }else{ + size = pToken->n; + } + if (size + VARSTR_HEADER_SIZE > pSchema->bytes) { + if(pToken->type == TK_NK_HEX || pToken->type == TK_NK_BIN){ + taosMemoryFree(data); + } return generateSyntaxErrMsg(pMsgBuf, TSDB_CODE_PAR_VALUE_TOO_LONG, pSchema->name); } - val->pData = taosStrdup(pToken->z) - val->nData = pToken->n; + if(pToken->type == TK_NK_HEX || pToken->type == TK_NK_BIN){ + val->pData = data; + }else{ + val->pData = taosStrdup(pToken->z); + } + val->nData = size; break; } case TSDB_DATA_TYPE_GEOMETRY: { @@ -667,6 +688,7 @@ static int32_t rewriteTagCondColumnImpl(STagVal* pVal, SNode** pNode) { *(double*)&pValue->typeData = pValue->datum.d; break; case TSDB_DATA_TYPE_VARCHAR: + case TSDB_DATA_TYPE_VARBINARY: case TSDB_DATA_TYPE_NCHAR: pValue->datum.p = taosMemoryCalloc(1, pVal->nData + VARSTR_HEADER_SIZE); if (NULL == pValue->datum.p) { @@ -696,7 +718,6 @@ static int32_t rewriteTagCondColumnImpl(STagVal* pVal, SNode** pNode) { *(uint64_t*)&pValue->typeData = pValue->datum.i; break; case TSDB_DATA_TYPE_JSON: - case TSDB_DATA_TYPE_VARBINARY: case TSDB_DATA_TYPE_DECIMAL: case TSDB_DATA_TYPE_BLOB: case TSDB_DATA_TYPE_MEDIUMBLOB: @@ -1374,15 +1395,31 @@ static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql, } case TSDB_DATA_TYPE_VARBINARY: { // Too long values will raise the invalid sql error message - if (pToken->n + VARSTR_HEADER_SIZE > pSchema->bytes) { + void* data = NULL; + uint32_t size = 0; + if (pToken->type == TK_NK_HEX){ + if(taosHex2Ascii(pToken->z, pToken->n, &data, &size) < 0){ + return TSDB_CODE_OUT_OF_MEMORY; + } + }else if(pToken->type == TK_NK_BIN){ + if(taosBin2Ascii(pToken->z, pToken->n, &data, &size) < 0){ + return TSDB_CODE_OUT_OF_MEMORY; + } + }else{ + size = pToken->n; + } + if (size + VARSTR_HEADER_SIZE > pSchema->bytes) { + if(pToken->type == TK_NK_HEX || pToken->type == TK_NK_BIN){ + taosMemoryFree(data); + } return generateSyntaxErrMsg(&pCxt->msg, TSDB_CODE_PAR_VALUE_TOO_LONG, pSchema->name); } - pVal->value.pData = taosMemoryMalloc(pToken->n); - if (NULL == pVal->value.pData) { - return TSDB_CODE_OUT_OF_MEMORY; + if(pToken->type == TK_NK_HEX || pToken->type == TK_NK_BIN){ + pVal->value.pData = data; + }else{ + pVal->value.pData = taosStrdup(pToken->z); } - memcpy(pVal->value.pData, pToken->z, pToken->n); - pVal->value.nData = pToken->n + pVal->value.nData = size; break; } case TSDB_DATA_TYPE_NCHAR: { diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 0ca5ae01b5..72d8799a85 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -4784,6 +4784,7 @@ static int32_t checkTableTagsSchema(STranslateContext* pCxt, SHashObj* pHash, SN } if (TSDB_CODE_SUCCESS == code) { if ((TSDB_DATA_TYPE_VARCHAR == pTag->dataType.type && calcTypeBytes(pTag->dataType) > TSDB_MAX_TAGS_LEN) || + (TSDB_DATA_TYPE_VARBINARY == pTag->dataType.type && calcTypeBytes(pTag->dataType) > TSDB_MAX_TAGS_LEN) || (TSDB_DATA_TYPE_NCHAR == pTag->dataType.type && calcTypeBytes(pTag->dataType) > TSDB_MAX_TAGS_LEN) || (TSDB_DATA_TYPE_GEOMETRY == pTag->dataType.type && calcTypeBytes(pTag->dataType) > TSDB_MAX_TAGS_LEN)) { code = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN); @@ -4836,6 +4837,7 @@ static int32_t checkTableColsSchema(STranslateContext* pCxt, SHashObj* pHash, in } if (TSDB_CODE_SUCCESS == code) { if ((TSDB_DATA_TYPE_VARCHAR == pCol->dataType.type && calcTypeBytes(pCol->dataType) > TSDB_MAX_BINARY_LEN) || + (TSDB_DATA_TYPE_VARBINARY == pCol->dataType.type && calcTypeBytes(pCol->dataType) > TSDB_MAX_BINARY_LEN) || (TSDB_DATA_TYPE_NCHAR == pCol->dataType.type && calcTypeBytes(pCol->dataType) > TSDB_MAX_NCHAR_LEN) || (TSDB_DATA_TYPE_GEOMETRY == pCol->dataType.type && calcTypeBytes(pCol->dataType) > TSDB_MAX_GEOMETRY_LEN)) { code = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN); @@ -5532,6 +5534,7 @@ static int32_t checkAlterSuperTableBySchema(STranslateContext* pCxt, SAlterTable if (TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES == pStmt->alterType) { if ((TSDB_DATA_TYPE_VARCHAR == pStmt->dataType.type && calcTypeBytes(pStmt->dataType) > TSDB_MAX_BINARY_LEN) || + (TSDB_DATA_TYPE_VARBINARY == pStmt->dataType.type && calcTypeBytes(pStmt->dataType) > TSDB_MAX_BINARY_LEN) || (TSDB_DATA_TYPE_NCHAR == pStmt->dataType.type && calcTypeBytes(pStmt->dataType) > TSDB_MAX_NCHAR_LEN)) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN); } @@ -5558,6 +5561,7 @@ static int32_t checkAlterSuperTableBySchema(STranslateContext* pCxt, SAlterTable } if ((TSDB_DATA_TYPE_VARCHAR == pStmt->dataType.type && calcTypeBytes(pStmt->dataType) > TSDB_MAX_BINARY_LEN) || + (TSDB_DATA_TYPE_VARBINARY == pStmt->dataType.type && calcTypeBytes(pStmt->dataType) > TSDB_MAX_BINARY_LEN) || (TSDB_DATA_TYPE_NCHAR == pStmt->dataType.type && calcTypeBytes(pStmt->dataType) > TSDB_MAX_NCHAR_LEN)) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN); } @@ -8841,6 +8845,7 @@ static int32_t buildAddColReq(STranslateContext* pCxt, SAlterTableStmt* pStmt, S } if ((TSDB_DATA_TYPE_VARCHAR == pStmt->dataType.type && calcTypeBytes(pStmt->dataType) > TSDB_MAX_BINARY_LEN) || + (TSDB_DATA_TYPE_VARBINARY == pStmt->dataType.type && calcTypeBytes(pStmt->dataType) > TSDB_MAX_BINARY_LEN) || (TSDB_DATA_TYPE_NCHAR == pStmt->dataType.type && calcTypeBytes(pStmt->dataType) > TSDB_MAX_NCHAR_LEN)) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN); } @@ -8898,6 +8903,7 @@ static int32_t buildUpdateColReq(STranslateContext* pCxt, SAlterTableStmt* pStmt } if ((TSDB_DATA_TYPE_VARCHAR == pStmt->dataType.type && calcTypeBytes(pStmt->dataType) > TSDB_MAX_BINARY_LEN) || + (TSDB_DATA_TYPE_VARBINARY == pStmt->dataType.type && calcTypeBytes(pStmt->dataType) > TSDB_MAX_BINARY_LEN) || (TSDB_DATA_TYPE_NCHAR == pStmt->dataType.type && calcTypeBytes(pStmt->dataType) > TSDB_MAX_NCHAR_LEN)) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN); } diff --git a/source/libs/qcom/src/queryUtil.c b/source/libs/qcom/src/queryUtil.c index e245b6f273..5c9c3da5f4 100644 --- a/source/libs/qcom/src/queryUtil.c +++ b/source/libs/qcom/src/queryUtil.c @@ -300,8 +300,23 @@ int32_t dataConverToStr(char* str, int type, void* buf, int32_t bufSize, int32_t n = sprintf(str, "%e", GET_DOUBLE_VAL(buf)); break; - case TSDB_DATA_TYPE_VARBINARY: - sd; + case TSDB_DATA_TYPE_VARBINARY:{ + if (bufSize < 0) { + // tscError("invalid buf size"); + return TSDB_CODE_TSC_INVALID_VALUE; + } + void* data = NULL; + uint32_t size = 0; + if(taosAscii2Hex(buf, bufSize, &data, &size) < 0){ + return TSDB_CODE_OUT_OF_MEMORY; + } + *str = '"'; + memcpy(str + 1, data, size); + *(str + size + 1) = '"'; + n = size + 2; + taosMemoryFree(data); + break; + } case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_GEOMETRY: if (bufSize < 0) { diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index e887bb8888..0efe335c3d 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -207,7 +207,7 @@ int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) { } } - if (optr == OP_TYPE_NOT_IN && (type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR && type != TSDB_DATA_TYPE_GEOMETRY)) { + if (optr == OP_TYPE_NOT_IN && (type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_VARBINARY && type != TSDB_DATA_TYPE_NCHAR && type != TSDB_DATA_TYPE_GEOMETRY)) { switch (type) { case TSDB_DATA_TYPE_BOOL: case TSDB_DATA_TYPE_TINYINT: @@ -258,7 +258,15 @@ int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) { case TSDB_DATA_TYPE_DOUBLE: comparFn = 5; break; - case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_VARBINARY:{ + if (optr == OP_TYPE_IN) { + comparFn = 8; + } else if (optr == OP_TYPE_NOT_IN) { + comparFn = 25; + } else { /* normal relational comparFn */ + comparFn = 30; + } + } case TSDB_DATA_TYPE_BINARY: { if (optr == OP_TYPE_MATCH) { comparFn = 19; diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index 4eb0f0e1bc..b3f67f2c9d 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -1594,6 +1594,8 @@ static int32_t sclGetMathOperatorResType(SOperatorNode *pOp) { SDataType rdt = ((SExprNode *)(pOp->pRight))->resType; if ((TSDB_DATA_TYPE_TIMESTAMP == ldt.type && TSDB_DATA_TYPE_TIMESTAMP == rdt.type) || + TSDB_DATA_TYPE_VARBINARY == ldt.type || + TSDB_DATA_TYPE_VARBINARY == rdt.type || (TSDB_DATA_TYPE_TIMESTAMP == ldt.type && (IS_VAR_DATA_TYPE(rdt.type) || IS_FLOAT_TYPE(rdt.type))) || (TSDB_DATA_TYPE_TIMESTAMP == rdt.type && (IS_VAR_DATA_TYPE(ldt.type) || IS_FLOAT_TYPE(ldt.type)))) { return TSDB_CODE_TSC_INVALID_OPERATION; @@ -1629,7 +1631,7 @@ static int32_t sclGetCompOperatorResType(SOperatorNode *pOp) { return TSDB_CODE_TSC_INVALID_OPERATION; } SDataType rdt = ((SExprNode *)(pOp->pRight))->resType; - if (!IS_VAR_DATA_TYPE(ldt.type) || QUERY_NODE_VALUE != nodeType(pOp->pRight) || + if (ldt.type == TSDB_DATA_TYPE_VARBINARY || !IS_VAR_DATA_TYPE(ldt.type) || QUERY_NODE_VALUE != nodeType(pOp->pRight) || (!IS_STR_DATA_TYPE(rdt.type) && (rdt.type != TSDB_DATA_TYPE_NULL))) { return TSDB_CODE_TSC_INVALID_OPERATION; } @@ -1660,6 +1662,11 @@ static int32_t sclGetJsonOperatorResType(SOperatorNode *pOp) { } static int32_t sclGetBitwiseOperatorResType(SOperatorNode *pOp) { + SDataType ldt = ((SExprNode *)(pOp->pLeft))->resType; + SDataType rdt = ((SExprNode *)(pOp->pRight))->resType; + if(TSDB_DATA_TYPE_VARBINARY == ldt.type || TSDB_DATA_TYPE_VARBINARY == rdt.type){ + return TSDB_CODE_TSC_INVALID_OPERATION; + } pOp->node.resType.type = TSDB_DATA_TYPE_BIGINT; pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes; return TSDB_CODE_SUCCESS; diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index 96a18e56d2..ea637fd730 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -588,7 +588,6 @@ bool convertJsonValue(__compar_fn_t *fp, int32_t optr, int8_t typeLeft, int8_t t if (typeRight == TSDB_DATA_TYPE_NCHAR || typeRight == TSDB_DATA_TYPE_VARCHAR || - typeRight == TSDB_DATA_TYPE_VARBINARY || typeRight == TSDB_DATA_TYPE_GEOMETRY) { return false; } else if (typeRight != type) { @@ -596,7 +595,6 @@ bool convertJsonValue(__compar_fn_t *fp, int32_t optr, int8_t typeLeft, int8_t t *pRightData = pRightOut; } } else if (type == TSDB_DATA_TYPE_BINARY || - type == TSDB_DATA_TYPE_VARBINARY || type == TSDB_DATA_TYPE_GEOMETRY) { if (typeLeft == TSDB_DATA_TYPE_NCHAR) { *pLeftData = ncharTobinary(*pLeftData); @@ -1078,7 +1076,7 @@ static SColumnInfoData *vectorConvertVarToDouble(SScalarParam *pInput, int32_t * SScalarParam output = {0}; SColumnInfoData *pCol = pInput->columnData; - if (IS_VAR_DATA_TYPE(pCol->info.type) && pCol->info.type != TSDB_DATA_TYPE_JSON) { + if (IS_VAR_DATA_TYPE(pCol->info.type) && pCol->info.type != TSDB_DATA_TYPE_JSON && pCol->info.type != TSDB_DATA_TYPE_VARBINARY) { int32_t code = vectorConvertSingleCol(pInput, &output, TSDB_DATA_TYPE_DOUBLE, -1, -1); if (code != TSDB_CODE_SUCCESS) { terrno = code; diff --git a/source/os/src/osString.c b/source/os/src/osString.c index 0f459b58cd..cb01332525 100644 --- a/source/os/src/osString.c +++ b/source/os/src/osString.c @@ -468,3 +468,95 @@ float taosStr2Float(const char *str, char **pEnd) { #endif return tmp; } + +int32_t taosHex2Ascii(const char *z, uint32_t n, void** data, uint32_t* size){ + n -= 2; // remove 0x + z += 2; + *size = n%2 == 0 ? n/2 : n/2 + 1; + uint8_t* tmp = (uint8_t*)taosMemoryCalloc(*size, 1); + if(tmp == NULL) return -1; + int8_t num = 0; + uint8_t *byte = tmp + *size - 1; + + for (int i = n - 1; i >= 0; i--) { + if (z[i] >= 'a') { + *byte |= ((uint8_t)(10 + (z[i] - 'a')) << (num * 4)); + } else if (z[i] >= 'A') { + *byte |= ((uint8_t)(10 + (z[i] - 'A')) << (num * 4)); + } else { + *byte |= ((uint8_t)(z[i] - '0') << (num * 4)); + } + if (num == 1) { + byte--; + num = 0; + } else { + num++; + } + } + *data = tmp; + return 0; +} + +int32_t taosBin2Ascii(const char *z, uint32_t n, void** data, uint32_t* size){ + n -= 2; // remove 0b + z += 2; + *size = n%8 == 0 ? n/8 : n/8 + 1; + uint8_t* tmp = (uint8_t*)taosMemoryCalloc(*size, 1); + if(tmp == NULL) return -1; + int8_t num = 0; + uint8_t *byte = tmp + *size - 1; + + for (int i = n - 1; i >= 0; i--) { + *byte |= ((uint8_t)(z[i] - '0') << num); + if (num == 8) { + byte--; + num = 0; + } else { + num++; + } + } + *data = tmp; + return 0; +} + +static char valueOf(uint8_t symbol) +{ + switch(symbol) + { + case 0: return '0'; + case 1: return '1'; + case 2: return '2'; + case 3: return '3'; + case 4: return '4'; + case 5: return '5'; + case 6: return '6'; + case 7: return '7'; + case 8: return '8'; + case 9: return '9'; + case 10: return 'A'; + case 11: return 'B'; + case 12: return 'C'; + case 13: return 'D'; + case 14: return 'E'; + case 15: return 'F'; + default: + { + return -1; + } + } +} + +int32_t taosAscii2Hex(const char *z, uint32_t n, void** data, uint32_t* size){ + *size = n * 2 + 2; + uint8_t* tmp = (uint8_t*)taosMemoryCalloc(*size, 1); + if(tmp == NULL) return -1; + *data = tmp; + *(tmp++) = '0'; + *(tmp++) = 'X'; + for(int i = 0; i < n; i ++){ + uint8_t val = z[i]; + tmp[i*2] = valueOf(val >> 4); + tmp[i*2 + 1] = valueOf(val & 0x0F); + } + return 0; +} diff --git a/source/util/src/tcompare.c b/source/util/src/tcompare.c index 7ebe32dfaf..9e2fc74366 100644 --- a/source/util/src/tcompare.c +++ b/source/util/src/tcompare.c @@ -1370,8 +1370,16 @@ __compar_fn_t getComparFunc(int32_t type, int32_t optr) { case TSDB_DATA_TYPE_DOUBLE: comparFn = compareDoubleVal; break; + case TSDB_DATA_TYPE_VARBINARY: + if (optr == OP_TYPE_IN) { + comparFn = compareChkInString; + } else if (optr == OP_TYPE_NOT_IN) { + comparFn = compareChkNotInString; + } else { /* normal relational comparFn */ + comparFn = compareLenBinaryVal; + } + break; case TSDB_DATA_TYPE_BINARY: - case TSDB_DATA_TYPE_VARBINARY case TSDB_DATA_TYPE_GEOMETRY: { if (optr == OP_TYPE_MATCH) { comparFn = comparestrRegexMatch; @@ -1457,7 +1465,6 @@ __compar_fn_t getKeyComparFunc(int32_t keyType, int32_t order) { case TSDB_DATA_TYPE_UBIGINT: return (order == TSDB_ORDER_ASC) ? compareUint64Val : compareUint64ValDesc; case TSDB_DATA_TYPE_BINARY: - case TSDB_DATA_TYPE_VARBINARY case TSDB_DATA_TYPE_GEOMETRY: return (order == TSDB_ORDER_ASC) ? compareLenPrefixedStr : compareLenPrefixedStrDesc; case TSDB_DATA_TYPE_NCHAR: diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 12a8807ed5..6a42c208c7 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -581,8 +581,9 @@ void shellPrintField(const char *val, TAOS_FIELD *field, int32_t width, int32_t return; } - int n; - char buf[TSDB_MAX_BYTES_PER_ROW]; + int n = 0; +#define LENGTH 64 + char buf[LENGTH] = {0}; switch (field->type) { case TSDB_DATA_TYPE_BOOL: shellPrintString(((((int32_t)(*((char *)val))) == 1) ? "true" : "false"), width); @@ -615,7 +616,7 @@ void shellPrintField(const char *val, TAOS_FIELD *field, int32_t width, int32_t if (tsEnableScience) { printf("%*.7e",width,GET_FLOAT_VAL(val)); } else { - n = snprintf(buf, TSDB_MAX_BYTES_PER_ROW, "%*.7f", width, GET_FLOAT_VAL(val)); + n = snprintf(buf, LENGTH, "%*.7f", width, GET_FLOAT_VAL(val)); if (n > SHELL_FLOAT_WIDTH) { printf("%*.7e", width,GET_FLOAT_VAL(val)); @@ -626,10 +627,10 @@ void shellPrintField(const char *val, TAOS_FIELD *field, int32_t width, int32_t break; case TSDB_DATA_TYPE_DOUBLE: if (tsEnableScience) { - snprintf(buf, TSDB_MAX_BYTES_PER_ROW, "%*.15e", width,GET_DOUBLE_VAL(val)); + snprintf(buf, LENGTH, "%*.15e", width,GET_DOUBLE_VAL(val)); printf("%s", buf); } else { - n = snprintf(buf, TSDB_MAX_BYTES_PER_ROW, "%*.15f", width, GET_DOUBLE_VAL(val)); + n = snprintf(buf, LENGTH, "%*.15f", width, GET_DOUBLE_VAL(val)); if (n > SHELL_DOUBLE_WIDTH) { printf("%*.15e", width, GET_DOUBLE_VAL(val)); } else { @@ -637,8 +638,17 @@ void shellPrintField(const char *val, TAOS_FIELD *field, int32_t width, int32_t } } break; + case TSDB_DATA_TYPE_VARBINARY:{ + void* data = NULL; + uint32_t size = 0; + if(taosAscii2Hex(val, length, &data, &size) < 0){ + break; + } + shellPrintNChar(data, size, width); + taosMemoryFree(data); + break; + } case TSDB_DATA_TYPE_BINARY: - case TSDB_DATA_TYPE_VARBINARY case TSDB_DATA_TYPE_NCHAR: case TSDB_DATA_TYPE_JSON: shellPrintNChar(val, length, width);