xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

feat:[TD-18789]support varbinary type

This commit is contained in:
wangmm0220 2023-08-25 15:43:45 +08:00
parent dbc1b35d44
commit 26e025d78d
30 changed files with 740 additions and 274 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
include/libs/parser/parser.h
View File

@ -119,6 +119,7 @@ int32_t smlBindData(SQuery* handle, bool dataFormat, SArray* tags, SArray* colsS
char* msgBuf, int32_t msgBufLen);
int32_t smlBuildOutput(SQuery* handle, SHashObj* pVgHash);
int rawBlockBindData(SQuery *query, STableMeta* pTableMeta, void* data, SVCreateTbReq* pCreateTb, TAOS_FIELD *fields, int numFields, bool needChangeLength);
void clearColValArray(SArray* pCols);
int32_t rewriteToVnodeModifyOpStmt(SQuery* pQuery, SArray* pBufArray);
SArray* serializeVgroupsCreateTableBatch(SHashObj* pVgroupHashmap);

4
include/os/osString.h
View File

@ -92,7 +92,9 @@ double taosStr2Double(const char *str, char **pEnd);
float taosStr2Float(const char *str, char **pEnd);
int32_t taosHex2Ascii(const char *z, uint32_t n, void** data, uint32_t* size);
int32_t taosAscii2Hex(const char *z, uint32_t n, void** data, uint32_t* size);
int32_t taosBin2Ascii(const char *z, uint32_t n, void** data, uint32_t* size);
//int32_t taosBin2Ascii(const char *z, uint32_t n, void** data, uint32_t* size);
bool isHex(const char* z, uint32_t n);
bool isValidateHex(const char* z, uint32_t n);
#ifdef __cplusplus
}

1
include/util/taoserror.h
View File

@ -708,6 +708,7 @@ int32_t* taosGetErrno();
#define TSDB_CODE_PAR_INVALID_OPTR_USAGE TAOS_DEF_ERROR_CODE(0, 0x2667)
#define TSDB_CODE_PAR_SYSTABLE_NOT_ALLOWED_FUNC TAOS_DEF_ERROR_CODE(0, 0x2668)
#define TSDB_CODE_PAR_SYSTABLE_NOT_ALLOWED TAOS_DEF_ERROR_CODE(0, 0x2669)
#define TSDB_CODE_PAR_INVALID_VARBINARY TAOS_DEF_ERROR_CODE(0, 0x266A)
#define TSDB_CODE_PAR_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x26FF)
//planner

2
include/util/tcompare.h
View File

@ -102,6 +102,7 @@ int32_t compareUint64ValDesc(const void *pLeft, const void *pRight);
int32_t compareLenPrefixedStrDesc(const void *pLeft, const void *pRight);
int32_t compareLenPrefixedWStrDesc(const void *pLeft, const void *pRight);
int32_t compareLenBinaryValDesc(const void *pLeft, const void *pRight);
int32_t comparestrPatternMatch(const void *pLeft, const void *pRight);
int32_t comparestrPatternNMatch(const void *pLeft, const void *pRight);
@ -202,7 +203,6 @@ int32_t compareUint64Uint32(const void *pLeft, const void *pRight);
__compar_fn_t getComparFunc(int32_t type, int32_t optr);
__compar_fn_t getKeyComparFunc(int32_t keyType, int32_t order);
int32_t doCompare(const char *a, const char *b, int32_t type, size_t size);
#ifdef __cplusplus
}

7
include/util/tdef.h
View File

@ -416,9 +416,10 @@ typedef enum ELogicConditionType {
#define TSDB_EXPLAIN_RESULT_COLUMN_NAME "QUERY_PLAN"
#define TSDB_MAX_FIELD_LEN 65519 // 16384:65519
#define TSDB_MAX_BINARY_LEN TSDB_MAX_FIELD_LEN // 16384-8:65519
#define TSDB_MAX_NCHAR_LEN TSDB_MAX_FIELD_LEN // 16384-8:65519
#define TSDB_MAX_GEOMETRY_LEN TSDB_MAX_FIELD_LEN // 16384-8:65519
#define TSDB_MAX_BINARY_LEN TSDB_MAX_FIELD_LEN // 16384-8:65519
#define TSDB_MAX_NCHAR_LEN TSDB_MAX_FIELD_LEN // 16384-8:65519
#define TSDB_MAX_GEOMETRY_LEN TSDB_MAX_FIELD_LEN // 16384-8:65519
#define TSDB_MAX_VARBINARY_LEN TSDB_MAX_FIELD_LEN // 16384-8:65519
#define PRIMARYKEY_TIMESTAMP_COL_ID 1
#define COL_REACH_END(colId, maxColId) ((colId) > (maxColId))

1
source/client/inc/clientSml.h
View File

@ -251,7 +251,6 @@ int64_t smlParseOpenTsdbTime(SSmlHandle *info, const char *data, int32
int32_t smlClearForRerun(SSmlHandle *info);
int32_t smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg);
uint8_t smlGetTimestampLen(int64_t num);
void clearColValArray(SArray* pCols);
void smlDestroyTableInfo(void *para);
void freeSSmlKv(void* data);

17
source/client/src/clientSml.c
View File

@ -596,7 +596,7 @@ static int32_t smlGenerateSchemaAction(SSchema *colField, SHashObj *colHash, SSm
return TSDB_CODE_SML_INVALID_DATA;
}
if (((colField[*index].type == TSDB_DATA_TYPE_VARCHAR || colField[*index].type == TSDB_DATA_TYPE_VARCHAR || colField[*index].type == TSDB_DATA_TYPE_GEOMETRY) &&
if (((colField[*index].type == TSDB_DATA_TYPE_VARCHAR || colField[*index].type == TSDB_DATA_TYPE_VARBINARY || colField[*index].type == TSDB_DATA_TYPE_GEOMETRY) &&
(colField[*index].bytes - VARSTR_HEADER_SIZE) < kv->length) ||
(colField[*index].type == TSDB_DATA_TYPE_NCHAR &&
((colField[*index].bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE < kv->length))) {
@ -1178,21 +1178,12 @@ void smlDestroyTableInfo(void *para) {
taosMemoryFree(tag);
}
void clearColValArray(SArray *pCols) {
int32_t num = taosArrayGetSize(pCols);
for (int32_t i = 0; i < num; ++i) {
SColVal *pCol = taosArrayGet(pCols, i);
if (TSDB_DATA_TYPE_NCHAR == pCol->type) {
taosMemoryFreeClear(pCol->value.pData);
}
}
}
void freeSSmlKv(void *data) {
SSmlKv *kv = (SSmlKv *)data;
if (kv->keyEscaped) taosMemoryFree((void *)(kv->key));
if (kv->valueEscaped) taosMemoryFree((void *)(kv->value));
if (kv->keyEscaped) taosMemoryFreeClear(kv->key);
if (kv->valueEscaped) taosMemoryFreeClear(kv->value);
if (kv->type == TSDB_DATA_TYPE_GEOMETRY) geosFreeBuffer((void *)(kv->value));
if (kv->type == TSDB_DATA_TYPE_VARBINARY) taosMemoryFreeClear(kv->value);
}
void smlDestroyInfo(SSmlHandle *info) {

44
source/client/src/clientSmlLine.c
View File

@ -109,7 +109,7 @@ int32_t smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg) {
return code;
}
char* tmp = taosMemoryCalloc(pVal->length, 1);
memcpy(tmp, pVal->value + 2, pVal->length - 3);
memcpy(tmp, pVal->value + NCHAR_ADD_LEN - 1, pVal->length - NCHAR_ADD_LEN);
code = doGeomFromText(tmp, (unsigned char **)&pVal->value, &pVal->length);
taosMemoryFree(tmp);
if (code != TSDB_CODE_SUCCESS) {
@ -126,6 +126,44 @@ int32_t smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg) {
return TSDB_CODE_TSC_INVALID_VALUE;
}
if (pVal->value[0] == 'b' || pVal->value[0] == 'B') { // varbinary
if (pVal->value[1] == '"' && pVal->value[pVal->length - 1] == '"' && pVal->length >= 3) {
pVal->type = TSDB_DATA_TYPE_VARBINARY;
if(isHex(pVal->value + NCHAR_ADD_LEN - 1, pVal->length - NCHAR_ADD_LEN)){
if(!isValidateHex(pVal->value + NCHAR_ADD_LEN - 1, pVal->length - NCHAR_ADD_LEN)){
return TSDB_CODE_PAR_INVALID_VARBINARY;
}
void* data = NULL;
uint32_t size = 0;
if(taosHex2Ascii(pVal->value + NCHAR_ADD_LEN - 1, pVal->length - NCHAR_ADD_LEN, &data, &size) < 0){
return TSDB_CODE_OUT_OF_MEMORY;
}
if (size + VARSTR_HEADER_SIZE > TSDB_MAX_VARBINARY_LEN) {
taosMemoryFree(data);
return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN;
}
pVal->value = data;
pVal->length = size;
}else{
pVal->length -= NCHAR_ADD_LEN;
if (pVal->length > TSDB_MAX_VARBINARY_LEN - VARSTR_HEADER_SIZE) {
return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN;
}
void *data = taosMemoryMalloc(pVal->length);
if(data == NULL){
return TSDB_CODE_OUT_OF_MEMORY;
}
memcpy(data, pVal->value + (NCHAR_ADD_LEN - 1), pVal->length);
pVal->value = data;
}
return TSDB_CODE_SUCCESS;
}
return TSDB_CODE_TSC_INVALID_VALUE;
}
if (pVal->value[0] == 't' || pVal->value[0] == 'T') {
if (pVal->length == 1 ||
(pVal->length == 4 && (pVal->value[1] == 'r' || pVal->value[1] == 'R') &&
@ -515,6 +553,10 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
char *tmp = (char *)taosMemoryMalloc(kv.length);
memcpy(tmp, kv.value, kv.length);
PROCESS_SLASH_IN_FIELD_VALUE(tmp, kv.length);
ASSERT(kv.type != TSDB_DATA_TYPE_GEOMETRY);
if(kv.type == TSDB_DATA_TYPE_VARBINARY){
taosMemoryFree((void*)kv.value);
}
kv.value = tmp;
kv.valueEscaped = valueEscaped;
}

4
source/client/src/clientTmq.c
View File

@ -985,6 +985,10 @@ int32_t tmq_subscription(tmq_t* tmq, tmq_list_t** topics) {
int32_t tmq_unsubscribe(tmq_t* tmq) {
if(tmq == NULL) return TSDB_CODE_INVALID_PARA;
if (tmq->status != TMQ_CONSUMER_STATUS__READY) {
tscInfo("consumer:0x%" PRIx64 " not in ready state, close it directly", tmq->consumerId);
return 0;
}
if (tmq->autoCommit) {
int32_t rsp = tmq_commit_sync(tmq, NULL);
if (rsp != 0) {

2
source/dnode/mnode/impl/src/mndTopic.c
View File

@ -4,7 +4,7 @@
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
*f
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.

20
source/libs/parser/src/parInsertSml.c
View File

@ -18,18 +18,6 @@
#include "parToken.h"
#include "ttime.h"
static void clearColValArray(SArray* pCols) {
int32_t num = taosArrayGetSize(pCols);
for (int32_t i = 0; i < num; ++i) {
SColVal* pCol = taosArrayGet(pCols, i);
if (TSDB_DATA_TYPE_NCHAR == pCol->type || TSDB_DATA_TYPE_GEOMETRY == pCol->type) {
taosMemoryFreeClear(pCol->value.pData);
}
pCol->flag = CV_FLAG_NONE;
pCol->value.val = 0;
}
}
int32_t qCreateSName(SName* pName, const char* pTableName, int32_t acctId, char* dbName, char* msgBuf,
int32_t msgBufLen) {
SMsgBuf msg = {.buf = msgBuf, .len = msgBufLen};
@ -237,10 +225,10 @@ int32_t smlBuildCol(STableDataCxt* pTableCxt, SSchema* schema, void* data, int32
}
pVal->value.pData = pUcs4;
pVal->value.nData = len;
} else if (kv->type == TSDB_DATA_TYPE_BINARY || kv->type == TSDB_DATA_TYPE_VARBINARY || kv->type == TSDB_DATA_TYPE_GEOMETRY) {
} else if (kv->type == TSDB_DATA_TYPE_BINARY) {
pVal->value.nData = kv->length;
pVal->value.pData = (uint8_t*)kv->value;
} else if (kv->type == TSDB_DATA_TYPE_GEOMETRY) {
} else if (kv->type == TSDB_DATA_TYPE_GEOMETRY || kv->type == TSDB_DATA_TYPE_VARBINARY) {
pVal->value.nData = kv->length;
pVal->value.pData = taosMemoryMalloc(kv->length);
memcpy(pVal->value.pData, (uint8_t*)kv->value, kv->length);
@ -368,10 +356,10 @@ int32_t smlBindData(SQuery* query, bool dataFormat, SArray* tags, SArray* colsSc
}
pVal->value.pData = pUcs4;
pVal->value.nData = len;
} else if (kv->type == TSDB_DATA_TYPE_BINARY || kv->type == TSDB_DATA_TYPE_VARBINARY || kv->type == TSDB_DATA_TYPE_GEOMETRY) {
} else if (kv->type == TSDB_DATA_TYPE_BINARY) {
pVal->value.nData = kv->length;
pVal->value.pData = (uint8_t*)kv->value;
} else if (kv->type == TSDB_DATA_TYPE_GEOMETRY) {
} else if (kv->type == TSDB_DATA_TYPE_GEOMETRY || kv->type == TSDB_DATA_TYPE_VARBINARY) {
pVal->value.nData = kv->length;
pVal->value.pData = taosMemoryMalloc(kv->length);
memcpy(pVal->value.pData, (uint8_t*)kv->value, kv->length);

95
source/libs/parser/src/parInsertSql.c
View File

@ -326,6 +326,38 @@ static int parseGeometry(SToken *pToken, unsigned char **output, size_t *size) {
return code;
}
static int32_t parseVarbinary(SToken* pToken, uint8_t **pData, uint32_t *nData, int32_t bytes){
if(pToken->type != TK_NK_STRING){
return TSDB_CODE_PAR_INVALID_VARBINARY;
}
if(isHex(pToken->z, pToken->n)){
if(!isValidateHex(pToken->z, pToken->n)){
return TSDB_CODE_PAR_INVALID_VARBINARY;
}
void* data = NULL;
uint32_t size = 0;
if(taosHex2Ascii(pToken->z, pToken->n, &data, &size) < 0){
return TSDB_CODE_OUT_OF_MEMORY;
}
if (size + VARSTR_HEADER_SIZE > bytes) {
taosMemoryFree(data);
return TSDB_CODE_PAR_VALUE_TOO_LONG;
}
*pData = data;
*nData = size;
}else{
if (pToken->n + VARSTR_HEADER_SIZE > bytes) {
return TSDB_CODE_PAR_VALUE_TOO_LONG;
}
*pData = taosStrdup(pToken->z);
*nData = pToken->n;
}
return TSDB_CODE_SUCCESS;
}
static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema, int16_t timePrec, STagVal* val,
SMsgBuf* pMsgBuf) {
int64_t iv;
@ -478,33 +510,10 @@ static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema,
break;
}
case TSDB_DATA_TYPE_VARBINARY: {
// Too long values will raise the invalid sql error message
// Too long values will raise the invalid sql error message
void* data = NULL;
uint32_t size = 0;
if (pToken->type == TK_NK_HEX){
if(taosHex2Ascii(pToken->z, pToken->n, &data, &size) < 0){
return TSDB_CODE_OUT_OF_MEMORY;
}
}else if(pToken->type == TK_NK_BIN){
if(taosBin2Ascii(pToken->z, pToken->n, &data, &size) < 0){
return TSDB_CODE_OUT_OF_MEMORY;
}
}else{
size = pToken->n;
code = parseVarbinary(pToken, &val->pData, &val->nData, pSchema->bytes);
if(code != TSDB_CODE_SUCCESS){
return generateSyntaxErrMsg(pMsgBuf, code, pSchema->name);
}
if (size + VARSTR_HEADER_SIZE > pSchema->bytes) {
if(pToken->type == TK_NK_HEX || pToken->type == TK_NK_BIN){
taosMemoryFree(data);
}
return generateSyntaxErrMsg(pMsgBuf, TSDB_CODE_PAR_VALUE_TOO_LONG, pSchema->name);
}
if(pToken->type == TK_NK_HEX || pToken->type == TK_NK_BIN){
val->pData = data;
}else{
val->pData = taosStrdup(pToken->z);
}
val->nData = size;
break;
}
case TSDB_DATA_TYPE_GEOMETRY: {
@ -1394,32 +1403,10 @@ static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql,
break;
}
case TSDB_DATA_TYPE_VARBINARY: {
// Too long values will raise the invalid sql error message
void* data = NULL;
uint32_t size = 0;
if (pToken->type == TK_NK_HEX){
if(taosHex2Ascii(pToken->z, pToken->n, &data, &size) < 0){
return TSDB_CODE_OUT_OF_MEMORY;
}
}else if(pToken->type == TK_NK_BIN){
if(taosBin2Ascii(pToken->z, pToken->n, &data, &size) < 0){
return TSDB_CODE_OUT_OF_MEMORY;
}
}else{
size = pToken->n;
int32_t code = parseVarbinary(pToken, &pVal->value.pData, &pVal->value.nData, pSchema->bytes);
if(code != TSDB_CODE_SUCCESS){
return generateSyntaxErrMsg(&pCxt->msg, code, pSchema->name);
}
if (size + VARSTR_HEADER_SIZE > pSchema->bytes) {
if(pToken->type == TK_NK_HEX || pToken->type == TK_NK_BIN){
taosMemoryFree(data);
}
return generateSyntaxErrMsg(&pCxt->msg, TSDB_CODE_PAR_VALUE_TOO_LONG, pSchema->name);
}
if(pToken->type == TK_NK_HEX || pToken->type == TK_NK_BIN){
pVal->value.pData = data;
}else{
pVal->value.pData = taosStrdup(pToken->z);
}
pVal->value.nData = size;
break;
}
case TSDB_DATA_TYPE_NCHAR: {
@ -1521,13 +1508,15 @@ static int32_t parseValueToken(SInsertParseContext* pCxt, const char** pSql, STo
return code;
}
static void clearColValArray(SArray* pCols) {
void clearColValArray(SArray* pCols) {
int32_t num = taosArrayGetSize(pCols);
for (int32_t i = 0; i < num; ++i) {
SColVal* pCol = taosArrayGet(pCols, i);
if (IS_VAR_DATA_TYPE(pCol->type)) {
if (TSDB_DATA_TYPE_NCHAR == pCol->type || TSDB_DATA_TYPE_GEOMETRY == pCol->type) {
taosMemoryFreeClear(pCol->value.pData);
}
pCol->flag = CV_FLAG_NONE;
pCol->value.val = 0;
}
}

4
source/libs/parser/src/parInsertUtil.c
View File

@ -333,8 +333,8 @@ int32_t insGetTableDataCxt(SHashObj* pHash, void* id, int32_t idLen, STableMeta*
static void destroyColVal(void* p) {
SColVal* pVal = p;
if (TSDB_DATA_TYPE_NCHAR == pVal->type || TSDB_DATA_TYPE_GEOMETRY == pVal->type) {
taosMemoryFree(pVal->value.pData);
if (TSDB_DATA_TYPE_NCHAR == pVal->type || TSDB_DATA_TYPE_GEOMETRY == pVal->type || TSDB_DATA_TYPE_VARBINARY == pVal->type) {
taosMemoryFreeClear(pVal->value.pData);
}
}

2
source/libs/parser/src/parTokenizer.c
View File

@ -545,7 +545,7 @@ uint32_t tGetToken(const char* z, uint32_t* tokenId) {
return i;
} else if (next == 'x') { // hex number
*tokenId = TK_NK_HEX;
for (i = 2; isdigit(z[i]) || (z[i] >= 'a' && z[i] <= 'f') || (z[i] >= 'A' && z[i] <= 'F'); ++i) {
for (i = 2; isxdigit(z[i]) != 0; ++i) {
}
if (i == 2) {

39
source/libs/parser/src/parTranslater.c
View File

@ -1231,8 +1231,41 @@ static EDealRes translateNormalValue(STranslateContext* pCxt, SValueNode* pVal,
*(double*)&pVal->typeData = pVal->datum.d;
break;
}
case TSDB_DATA_TYPE_VARBINARY: {
if (pVal->node.resType.type != TSDB_DATA_TYPE_BINARY){
return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, pVal->literal);
}
void* data = NULL;
uint32_t size = 0;
bool isHexChar = isHex(pVal->literal, strlen(pVal->literal));
if(isHexChar){
if(!isValidateHex(pVal->literal, strlen(pVal->literal))){
return TSDB_CODE_PAR_INVALID_VARBINARY;
}
if(taosHex2Ascii(pVal->literal, strlen(pVal->literal), &data, &size) < 0){
return TSDB_CODE_OUT_OF_MEMORY;
}
}else{
size = pVal->node.resType.bytes;
data = pVal->literal;
}
if (size + VARSTR_HEADER_SIZE > targetDt.bytes) {
if(isHexChar) taosMemoryFree(data);
return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_VALUE_TOO_LONG, pVal->literal);
}
pVal->datum.p = taosMemoryCalloc(1, size + VARSTR_HEADER_SIZE);
if (NULL == pVal->datum.p) {
if(isHexChar) taosMemoryFree(data);
return generateDealNodeErrMsg(pCxt, TSDB_CODE_OUT_OF_MEMORY);
}
varDataSetLen(pVal->datum.p, size + VARSTR_HEADER_SIZE);
memcpy(varDataVal(pVal->datum.p), data, size);
if(isHexChar) taosMemoryFree(data);
break;
}
case TSDB_DATA_TYPE_VARCHAR:
case TSDB_DATA_TYPE_VARBINARY:
case TSDB_DATA_TYPE_GEOMETRY: {
if (strict && (pVal->node.resType.bytes > targetDt.bytes - VARSTR_HEADER_SIZE)) {
return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, pVal->literal);
@ -1296,7 +1329,9 @@ static EDealRes translateValueImpl(STranslateContext* pCxt, SValueNode* pVal, SD
res = translateNormalValue(pCxt, pVal, targetDt, strict);
}
pVal->node.resType.type = targetDt.type;
pVal->node.resType.bytes = targetDt.bytes;
if( targetDt.type == TSDB_DATA_TYPE_VARBINARY || !strict){
pVal->node.resType.bytes = targetDt.bytes;
}
pVal->node.resType.scale = pVal->unit;
pVal->translate = true;
if (!strict && TSDB_DATA_TYPE_UBIGINT == pVal->node.resType.type && pVal->datum.u <= INT64_MAX) {

2
source/libs/parser/src/parUtil.c
View File

@ -308,7 +308,7 @@ int32_t trimString(const char* src, int32_t len, char* dst, int32_t dlen) {
dst[j] = '\'';
} else if (src[k + 1] == '"') {
dst[j] = '"';
} else if (src[k + 1] == '%' || src[k + 1] == '_') {
} else if (src[k + 1] == '%' || src[k + 1] == '_' || src[k + 1] == 'x') {
dst[j++] = src[k];
dst[j] = src[k + 1];
} else {

3
source/libs/scalar/src/filter.c
View File

@ -266,6 +266,7 @@ int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) {
} else { /* normal relational comparFn */
comparFn = 30;
}
break;
}
case TSDB_DATA_TYPE_BINARY: {
if (optr == OP_TYPE_MATCH) {
@ -1595,7 +1596,7 @@ int32_t fltConverToStr(char *str, int type, void *buf, int32_t bufSize, int32_t
break;
case TSDB_DATA_TYPE_BINARY:
// case TSDB_DATA_TYPE_BINARY:// todovar
case TSDB_DATA_TYPE_VARBINARY:
case TSDB_DATA_TYPE_NCHAR:
case TSDB_DATA_TYPE_GEOMETRY:
if (bufSize < 0) {

90
source/libs/scalar/src/sclvector.c
View File

@ -309,6 +309,47 @@ static FORCE_INLINE void varToBool(char *buf, SScalarParam *pOut, int32_t rowInd
}
// todo remove this malloc
static FORCE_INLINE void varToVarbinary(char *buf, SScalarParam *pOut, int32_t rowIndex, int32_t *overflow) {
terrno = TSDB_CODE_SUCCESS;
if(isHex(varDataVal(buf), varDataLen(buf))){
if(!isValidateHex(varDataVal(buf), varDataLen(buf))){
terrno = TSDB_CODE_PAR_INVALID_VARBINARY;
return;
}
void* data = NULL;
uint32_t size = 0;
if(taosHex2Ascii(varDataVal(buf), varDataLen(buf), &data, &size) < 0){
terrno = TSDB_CODE_OUT_OF_MEMORY;
return;
}
int32_t inputLen = size + VARSTR_HEADER_SIZE;
char *t = taosMemoryCalloc(1, inputLen);
if (t == NULL) {
sclError("Out of memory");
terrno = TSDB_CODE_OUT_OF_MEMORY;
return;
}
varDataSetLen(t, size);
memcpy(varDataVal(t), data, size);
colDataSetVal(pOut->columnData, rowIndex, t, false);
taosMemoryFree(t);
taosMemoryFree(data);
}else{
int32_t inputLen = varDataTLen(buf);
char *t = taosMemoryCalloc(1, inputLen);
if (t == NULL) {
sclError("Out of memory");
terrno = TSDB_CODE_OUT_OF_MEMORY;
return;
}
memcpy(t, buf, inputLen);
colDataSetVal(pOut->columnData, rowIndex, t, false);
taosMemoryFree(t);
}
}
static FORCE_INLINE void varToNchar(char *buf, SScalarParam *pOut, int32_t rowIndex, int32_t *overflow) {
terrno = TSDB_CODE_SUCCESS;
@ -398,18 +439,21 @@ int32_t vectorConvertFromVarData(SSclVectorConvCtx *pCtx, int32_t *overflow) {
func = varToUnsigned;
} else if (IS_FLOAT_TYPE(pCtx->outType)) {
func = varToFloat;
} else if (pCtx->outType == TSDB_DATA_TYPE_VARCHAR &&
} else if ((pCtx->outType == TSDB_DATA_TYPE_VARCHAR || pCtx->outType == TSDB_DATA_TYPE_VARBINARY) &&
pCtx->inType == TSDB_DATA_TYPE_NCHAR) { // nchar -> binary
func = ncharToVar;
vton = true;
} else if (pCtx->outType == TSDB_DATA_TYPE_NCHAR &&
pCtx->inType == TSDB_DATA_TYPE_VARCHAR) { // binary -> nchar
(pCtx->inType == TSDB_DATA_TYPE_VARCHAR || pCtx->inType == TSDB_DATA_TYPE_VARBINARY)) { // binary -> nchar
func = varToNchar;
vton = true;
} else if (TSDB_DATA_TYPE_TIMESTAMP == pCtx->outType) {
func = varToTimestamp;
} else if (TSDB_DATA_TYPE_GEOMETRY == pCtx->outType) {
func = varToGeometry;
} else if (TSDB_DATA_TYPE_VARBINARY == pCtx->outType) {
func = varToVarbinary;
vton = true;
} else {
sclError("invalid convert outType:%d, inType:%d", pCtx->outType, pCtx->inType);
terrno = TSDB_CODE_APP_ERROR;
@ -885,7 +929,7 @@ int32_t vectorConvertSingleColImpl(const SScalarParam *pIn, SScalarParam *pOut,
break;
}
case TSDB_DATA_TYPE_BINARY:
// case TSDB_DATA_TYPE_VARBINARY //todovar
case TSDB_DATA_TYPE_VARBINARY:
case TSDB_DATA_TYPE_NCHAR:
case TSDB_DATA_TYPE_GEOMETRY: {
return vectorConvertToVarData(&cCtx);
@ -901,26 +945,26 @@ int32_t vectorConvertSingleColImpl(const SScalarParam *pIn, SScalarParam *pOut,
int8_t gConvertTypes[TSDB_DATA_TYPE_MAX][TSDB_DATA_TYPE_MAX] = {
/* NULL BOOL TINY SMAL INT BIG FLOA DOUB VARC TIME NCHA UTIN USMA UINT UBIG JSON VARB DECI BLOB MEDB GEOM*/
/*NULL*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
/*BOOL*/ 0, 0, 2, 3, 4, 5, 6, 7, 5, 9, 7, 11, 12, 13, 14, 0, 7, 0, 0, 0, -1,
/*TINY*/ 0, 0, 0, 3, 4, 5, 6, 7, 5, 9, 7, 3, 4, 5, 7, 0, 7, 0, 0, 0, -1,
/*SMAL*/ 0, 0, 0, 0, 4, 5, 6, 7, 5, 9, 7, 3, 4, 5, 7, 0, 7, 0, 0, 0, -1,
/*INT */ 0, 0, 0, 0, 0, 5, 6, 7, 5, 9, 7, 4, 4, 5, 7, 0, 7, 0, 0, 0, -1,
/*BIGI*/ 0, 0, 0, 0, 0, 0, 6, 7, 5, 9, 7, 5, 5, 5, 7, 0, 7, 0, 0, 0, -1,
/*FLOA*/ 0, 0, 0, 0, 0, 0, 0, 7, 7, 6, 7, 6, 6, 6, 6, 0, 7, 0, 0, 0, -1,
/*DOUB*/ 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 7, 7, 7, 7, 7, 0, 7, 0, 0, 0, -1,
/*VARC*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 8, 7, 7, 7, 7, 0, 0, 0, 0, 0, 20,
/*TIME*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 7, 0, 7, 0, 0, 0, -1,
/*NCHA*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 7, 7, 0, 0, 0, 0, 0, -1,
/*UTIN*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 13, 14, 0, 7, 0, 0, 0, -1,
/*USMA*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 14, 0, 7, 0, 0, 0, -1,
/*UINT*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 7, 0, 0, 0, -1,
/*UBIG*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, -1,
/*JSON*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1,
/*VARB*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1,
/*DECI*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1,
/*BLOB*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1,
/*MEDB*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1,
/*GEOM*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
/*BOOL*/ 0, 0, 2, 3, 4, 5, 6, 7, 5, 9, 7, 11, 12, 13, 14, 0, -1, 0, 0, 0, -1,
/*TINY*/ 0, 0, 0, 3, 4, 5, 6, 7, 5, 9, 7, 3, 4, 5, 7, 0, -1, 0, 0, 0, -1,
/*SMAL*/ 0, 0, 0, 0, 4, 5, 6, 7, 5, 9, 7, 3, 4, 5, 7, 0, -1, 0, 0, 0, -1,
/*INT */ 0, 0, 0, 0, 0, 5, 6, 7, 5, 9, 7, 4, 4, 5, 7, 0, -1, 0, 0, 0, -1,
/*BIGI*/ 0, 0, 0, 0, 0, 0, 6, 7, 5, 9, 7, 5, 5, 5, 7, 0, -1, 0, 0, 0, -1,
/*FLOA*/ 0, 0, 0, 0, 0, 0, 0, 7, 7, 6, 7, 6, 6, 6, 6, 0, -1, 0, 0, 0, -1,
/*DOUB*/ 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 7, 7, 7, 7, 7, 0, -1, 0, 0, 0, -1,
/*VARC*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 8, 7, 7, 7, 7, 0, 16, 0, 0, 0, 20,
/*TIME*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 7, 0, -1, 0, 0, 0, -1,
/*NCHA*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 7, 7, 0, 16, 0, 0, 0, -1,
/*UTIN*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 13, 14, 0, -1, 0, 0, 0, -1,
/*USMA*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 14, 0, -1, 0, 0, 0, -1,
/*UINT*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, -1, 0, 0, 0, -1,
/*UBIG*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, 0, 0, 0, -1,
/*JSON*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, 0, 0, 0, -1,
/*VARB*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, -1,-1, -1,
/*DECI*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, 0, 0, 0, -1,
/*BLOB*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, 0, 0, 0, -1,
/*MEDB*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, 0, 0, 0, -1,
/*GEOM*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, 0, 0, 0, 0};
int32_t vectorGetConvertType(int32_t type1, int32_t type2) {
if (type1 == type2) {

46
source/libs/scalar/test/filter/filterTests.cpp
View File

@ -1415,52 +1415,6 @@ void doCompareWithValueRange_OnlyLeftType(__compar_fn_t fp, int32_t rType) {
}
}
void doCompare(const std::vector<int32_t> &lTypes, const std::vector<int32_t> &rTypes, int32_t oper) {
for (int i = 0; i < lTypes.size(); ++i) {
for (int j = 0; j < rTypes.size(); ++j) {
auto fp = filterGetCompFuncEx(lTypes[i], rTypes[j], oper);
switch (lTypes[i]) {
case TSDB_DATA_TYPE_TINYINT:
doCompareWithValueRange_OnlyLeftType<int8_t>(fp, rTypes[j]);
break;
case TSDB_DATA_TYPE_SMALLINT:
doCompareWithValueRange_OnlyLeftType<int16_t>(fp, rTypes[j]);
break;
case TSDB_DATA_TYPE_INT:
doCompareWithValueRange_OnlyLeftType<int32_t>(fp, rTypes[j]);
break;
case TSDB_DATA_TYPE_BIGINT:
doCompareWithValueRange_OnlyLeftType<int64_t>(fp, rTypes[j]);
break;
case TSDB_DATA_TYPE_UTINYINT:
doCompareWithValueRange_OnlyLeftType<uint8_t>(fp, rTypes[j]);
break;
case TSDB_DATA_TYPE_USMALLINT:
doCompareWithValueRange_OnlyLeftType<uint16_t>(fp, rTypes[j]);
break;
case TSDB_DATA_TYPE_UINT:
doCompareWithValueRange_OnlyLeftType<uint32_t>(fp, rTypes[j]);
break;
case TSDB_DATA_TYPE_UBIGINT:
doCompareWithValueRange_OnlyLeftType<uint64_t>(fp, rTypes[j]);
break;
default:
FAIL();
}
}
}
}
TEST(dataCompareTest, signed_and_unsigned_int) {
std::vector<int32_t> lType = {TSDB_DATA_TYPE_TINYINT, TSDB_DATA_TYPE_SMALLINT, TSDB_DATA_TYPE_INT,
TSDB_DATA_TYPE_BIGINT};
std::vector<int32_t> rType = {TSDB_DATA_TYPE_UTINYINT, TSDB_DATA_TYPE_USMALLINT, TSDB_DATA_TYPE_UINT,
TSDB_DATA_TYPE_UBIGINT};
doCompare(lType, rType, OP_TYPE_GREATER_THAN);
doCompare(rType, lType, OP_TYPE_GREATER_THAN);
}
int main(int argc, char **argv) {
taosSeedRand(taosGetTimestampSec());
testing::InitGoogleTest(&argc, argv);

78
source/os/src/osString.c
View File

@ -469,10 +469,28 @@ float taosStr2Float(const char *str, char **pEnd) {
return tmp;
}
#define HEX_PREFIX_LEN 2 // \x
bool isHex(const char* z, uint32_t n){
if(n < HEX_PREFIX_LEN) return false;
if(z[0] == '\\' && z[1] == 'x') return true;
return false;
}
bool isValidateHex(const char* z, uint32_t n){
if(n % 2 != 0) return false;
for(size_t i = HEX_PREFIX_LEN; i < n; i++){
if(isxdigit(z[i]) == 0){
return false;
}
}
return true;
}
int32_t taosHex2Ascii(const char *z, uint32_t n, void** data, uint32_t* size){
n -= 2; // remove 0x
z += 2;
*size = n%2 == 0 ? n/2 : n/2 + 1;
n -= HEX_PREFIX_LEN; // remove 0x
z += HEX_PREFIX_LEN;
*size = n / HEX_PREFIX_LEN;
if(*size == 0) return 0;
uint8_t* tmp = (uint8_t*)taosMemoryCalloc(*size, 1);
if(tmp == NULL) return -1;
int8_t num = 0;
@ -497,27 +515,31 @@ int32_t taosHex2Ascii(const char *z, uint32_t n, void** data, uint32_t* size){
return 0;
}
int32_t taosBin2Ascii(const char *z, uint32_t n, void** data, uint32_t* size){
n -= 2; // remove 0b
z += 2;
*size = n%8 == 0 ? n/8 : n/8 + 1;
uint8_t* tmp = (uint8_t*)taosMemoryCalloc(*size, 1);
if(tmp == NULL) return -1;
int8_t num = 0;
uint8_t *byte = tmp + *size - 1;
for (int i = n - 1; i >= 0; i--) {
*byte |= ((uint8_t)(z[i] - '0') << num);
if (num == 8) {
byte--;
num = 0;
} else {
num++;
}
}
*data = tmp;
return 0;
}
//int32_t taosBin2Ascii(const char *z, uint32_t n, void** data, uint32_t* size){
//
// for (i = 2; isdigit(z[i]) || (z[i] >= 'a' && z[i] <= 'f') || (z[i] >= 'A' && z[i] <= 'F'); ++i) {
// }
//
// n -= 2; // remove 0b
// z += 2;
// *size = n%8 == 0 ? n/8 : n/8 + 1;
// uint8_t* tmp = (uint8_t*)taosMemoryCalloc(*size, 1);
// if(tmp == NULL) return -1;
// int8_t num = 0;
// uint8_t *byte = tmp + *size - 1;
//
// for (int i = n - 1; i >= 0; i--) {
// *byte |= ((uint8_t)(z[i] - '0') << num);
// if (num == 7) {
// byte--;
// num = 0;
// } else {
// num++;
// }
// }
// *data = tmp;
// return 0;
//}
static char valueOf(uint8_t symbol)
{
@ -547,12 +569,12 @@ static char valueOf(uint8_t symbol)
}
int32_t taosAscii2Hex(const char *z, uint32_t n, void** data, uint32_t* size){
*size = n * 2 + 2;
uint8_t* tmp = (uint8_t*)taosMemoryCalloc(*size, 1);
*size = n * 2 + HEX_PREFIX_LEN;
uint8_t* tmp = (uint8_t*)taosMemoryCalloc(*size + 1, 1);
if(tmp == NULL) return -1;
*data = tmp;
*(tmp++) = '0';
*(tmp++) = 'X';
*(tmp++) = '\\';
*(tmp++) = 'x';
for(int i = 0; i < n; i ++){
uint8_t val = z[i];
tmp[i*2] = valueOf(val >> 4);

60
source/util/src/tcompare.c
View File

@ -242,6 +242,10 @@ int32_t compareLenBinaryVal(const void *pLeft, const void *pRight) {
}
}
int32_t compareLenBinaryValDesc(const void *pLeft, const void *pRight) {
return compareLenBinaryVal(pRight, pLeft);
}
// string > number > bool > null
// ref: https://dev.mysql.com/doc/refman/8.0/en/json.html#json-comparison
int32_t compareJsonVal(const void *pLeft, const void *pRight) {
@ -1464,6 +1468,8 @@ __compar_fn_t getKeyComparFunc(int32_t keyType, int32_t order) {
return (order == TSDB_ORDER_ASC) ? compareUint32Val : compareUint32ValDesc;
case TSDB_DATA_TYPE_UBIGINT:
return (order == TSDB_ORDER_ASC) ? compareUint64Val : compareUint64ValDesc;
case TSDB_DATA_TYPE_VARBINARY:
return (order == TSDB_ORDER_ASC) ? compareLenBinaryVal : compareLenBinaryValDesc;
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_GEOMETRY:
return (order == TSDB_ORDER_ASC) ? compareLenPrefixedStr : compareLenPrefixedStrDesc;
@ -1475,57 +1481,3 @@ __compar_fn_t getKeyComparFunc(int32_t keyType, int32_t order) {
return (order == TSDB_ORDER_ASC) ? compareInt32Val : compareInt32ValDesc;
}
}
int32_t doCompare(const char *f1, const char *f2, int32_t type, size_t size) {
switch (type) {
case TSDB_DATA_TYPE_INT:
DEFAULT_COMP(GET_INT32_VAL(f1), GET_INT32_VAL(f2));
case TSDB_DATA_TYPE_DOUBLE:
DEFAULT_DOUBLE_COMP(GET_DOUBLE_VAL(f1), GET_DOUBLE_VAL(f2));
case TSDB_DATA_TYPE_FLOAT:
DEFAULT_FLOAT_COMP(GET_FLOAT_VAL(f1), GET_FLOAT_VAL(f2));
case TSDB_DATA_TYPE_BIGINT:
DEFAULT_COMP(GET_INT64_VAL(f1), GET_INT64_VAL(f2));
case TSDB_DATA_TYPE_SMALLINT:
DEFAULT_COMP(GET_INT16_VAL(f1), GET_INT16_VAL(f2));
case TSDB_DATA_TYPE_TINYINT:
case TSDB_DATA_TYPE_BOOL:
DEFAULT_COMP(GET_INT8_VAL(f1), GET_INT8_VAL(f2));
case TSDB_DATA_TYPE_UTINYINT:
DEFAULT_COMP(GET_UINT8_VAL(f1), GET_UINT8_VAL(f2));
case TSDB_DATA_TYPE_USMALLINT:
DEFAULT_COMP(GET_UINT16_VAL(f1), GET_UINT16_VAL(f2));
case TSDB_DATA_TYPE_UINT:
DEFAULT_COMP(GET_UINT32_VAL(f1), GET_UINT32_VAL(f2));
case TSDB_DATA_TYPE_UBIGINT:
DEFAULT_COMP(GET_UINT64_VAL(f1), GET_UINT64_VAL(f2));
case TSDB_DATA_TYPE_NCHAR: {
tstr *t1 = (tstr *)f1;
tstr *t2 = (tstr *)f2;
if (t1->len != t2->len) {
return t1->len > t2->len ? 1 : -1;
}
int32_t ret = memcmp((TdUcs4 *)t1, (TdUcs4 *)t2, t2->len);
if (ret == 0) {
return ret;
}
return (ret < 0) ? -1 : 1;
}
default: { // todo refactor
tstr *t1 = (tstr *)f1;
tstr *t2 = (tstr *)f2;
if (t1->len != t2->len) {
return t1->len > t2->len ? 1 : -1;
} else {
int32_t ret = strncmp(t1->data, t2->data, t1->len);
if (ret == 0) {
return 0;
} else {
return ret < 0 ? -1 : 1;
}
}
}
}
}

1
source/util/src/terror.c
View File

@ -555,6 +555,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_PAR_NOT_ALLOWED_WIN_QUERY, "Window query not su
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_DROP_COL, "No columns can be dropped")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_COL_JSON, "Only tag can be json type")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_VALUE_TOO_LONG, "Value too long for column/tag")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_VARBINARY, "Invalidate varbinary type")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_DELETE_WHERE, "The DELETE statement must have a definite time window range")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_REDISTRIBUTE_VG, "The REDISTRIBUTE VGROUP statement only support 1 to 3 dnodes")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_FILL_NOT_ALLOWED_FUNC, "Fill not allowed")

1
tests/parallel_test/cases.task
View File

@ -351,6 +351,7 @@
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/smaTest.py -R
,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/sma_index.py
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sml_TS-3724.py
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/varbinary.py
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sml.py
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sml.py -R
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/spread.py

8
tests/system-test/2-query/sml.py
View File

@ -1,18 +1,12 @@
import taos
import sys
import time
import socket
import os
import threading
from util.log import *
from util.sql import *
from util.cases import *
from util.dnodes import *
from util.common import *
sys.path.append("./7-tmq")
from tmqCommon import *
class TDTestCase:
updatecfgDict = {'clientCfg': {'smlChildTableName': 'dataModelName', 'fqdn': 'localhost', 'smlDot2Underline': 0}, 'fqdn': 'localhost'}
@ -32,7 +26,7 @@ class TDTestCase:
tdLog.info(cmdStr)
ret = os.system(cmdStr)
if ret != 0:
tdLog.info("sml_test ret != 0")
tdLog.exit("sml_test ret != 0")
tdSql.query(f"select * from ts3303.stb2")
tdSql.query(f"select * from ts3303.meters")

80
tests/system-test/2-query/varbinary.py Normal file
View File

@ -0,0 +1,80 @@
import sys
import os
from util.log import *
from util.sql import *
from util.cases import *
from util.common import *
class TDTestCase:
def init(self, conn, logSql, replicaVar=1):
self.replicaVar = int(replicaVar)
tdLog.debug(f"start to excute {__file__}")
tdSql.init(conn.cursor(), False)
def test(self):
tdLog.info(" test")
buildPath = tdCom.getBuildPath()
cmdStr = '%s/build/bin/varbinary_test'%(buildPath)
print("cmdStr:", cmdStr)
tdLog.info(cmdStr)
ret = os.system(cmdStr)
if ret != 0:
tdLog.exit("varbinary_test ret != 0")
tdSql.execute(f" create database test")
tdSql.execute(f" use test ")
tdSql.execute(f" create stable stb (ts timestamp, c1 nchar(32), c2 varbinary(16), c3 float) tags (t1 int, t2 binary(8), t3 varbinary(8))")
tdSql.query(f"desc stb")
tdSql.checkRows(7)
tdSql.checkData(2, 1, 'VARBINARY')
tdSql.checkData(2, 2, 16)
tdSql.checkData(6, 1, 'VARBINARY')
tdSql.checkData(6, 2, 8)
# tdSql.execute(f" insert into tb1 using stb tags (1, 'tb1_bin1', 'vart1') values (now, 'nchar1', 'varc1', 0.3)")
# tdSql.execute(f" insert into tb1 values (now + 1s, 'nchar2', null, 0.4)")
#
# tdSql.execute(f" insert into tb2 using stb tags (2, 'tb2_bin1', 093) values (now + 2s, 'nchar1', 892, 0.3)")
# tdSql.execute(f" insert into tb3 using stb tags (3, 'tb3_bin1', 0x7f829) values (now + 3s, 'nchar1', 0x7f829, 0.3)")
# tdSql.execute(f" insert into tb4 using stb tags (4, 'tb4_bin1', 0b100000010) values (now + 4s, 'nchar1', 0b110000001, 0.3)")
# tdSql.execute(f" insert into tb4 values (now + 5s, 'nchar1', 0b11000000100000000, 0.3)")
# tdSql.execute(f" insert into tb5 using stb tags (4, 'tb5_bin1', NULL) values (now + 6s, 'nchar1', 0b10100000011000000110000001, 0.3)")
# basic query
# tdSql.query(f"select c2,t3 from stb order by ts")
# tdSql.checkRows(6)
# tdSql.checkData(0, 0, '0x7661726331')
# tdSql.checkData(0, 1, '0x7661727431')
# tdSql.checkData(1, 0, None)
# tdSql.checkData(1, 1, '0x7661727431')
# tdSql.checkData(2, 0, '0x383932')
# tdSql.checkData(2, 1, '0x303933')
# tdSql.checkData(3, 0, '0x07f829')
# tdSql.checkData(3, 1, '0x07f829')
# tdSql.checkData(4, 0, '0x0181')
# tdSql.checkData(4, 1, '0x0102')
# tdSql.checkData(5, 0, '0x02818181')
# tdSql.checkData(5, 1, None)
# tdSql.query(f"select ts,c2 from stb order by c2")
#
# tdSql.query(f"select c2,t3 from stb where c2 >= 0 order by ts")
#
# tdSql.query(f"select c2,t3 from stb where c2 >= 0x0181 order by ts")
def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring
tdSql.prepare()
self.test()
def stop(self):
tdSql.close()
tdLog.success(f"{__file__} successfully executed")
tdCases.addLinux(__file__, TDTestCase())
tdCases.addWindows(__file__, TDTestCase())

57
tools/shell/src/shellEngine.c
View File

@ -323,13 +323,12 @@ void shellDumpFieldToFile(TdFilePtr pFile, const char *val, TAOS_FIELD *field, i
return;
}
char quotationStr[2];
quotationStr[0] = '\"';
quotationStr[1] = 0;
char quotationStr[2] ={'"', 0};
int32_t width;
int n;
char buf[TSDB_MAX_BYTES_PER_ROW];
int n = 0;
#define LENGTH 64
char buf[LENGTH] = {0};
switch (field->type) {
case TSDB_DATA_TYPE_BOOL:
taosFprintfFile(pFile, "%d", ((((int32_t)(*((char *)val))) == 1) ? 1 : 0));
@ -363,7 +362,7 @@ void shellDumpFieldToFile(TdFilePtr pFile, const char *val, TAOS_FIELD *field, i
if (tsEnableScience) {
taosFprintfFile(pFile, "%*.7e", width, GET_FLOAT_VAL(val));
} else {
n = snprintf(buf, TSDB_MAX_BYTES_PER_ROW, "%*.7f", width, GET_FLOAT_VAL(val));
n = snprintf(buf, LENGTH, "%*.7f", width, GET_FLOAT_VAL(val));
if (n > SHELL_FLOAT_WIDTH) {
taosFprintfFile(pFile, "%*.7e", width, GET_FLOAT_VAL(val));
} else {
@ -374,10 +373,10 @@ void shellDumpFieldToFile(TdFilePtr pFile, const char *val, TAOS_FIELD *field, i
case TSDB_DATA_TYPE_DOUBLE:
width = SHELL_DOUBLE_WIDTH;
if (tsEnableScience) {
snprintf(buf, TSDB_MAX_BYTES_PER_ROW, "%*.15e", width, GET_DOUBLE_VAL(val));
snprintf(buf, LENGTH, "%*.15e", width, GET_DOUBLE_VAL(val));
taosFprintfFile(pFile, "%s", buf);
} else {
n = snprintf(buf, TSDB_MAX_BYTES_PER_ROW, "%*.15f", width, GET_DOUBLE_VAL(val));
n = snprintf(buf, LENGTH, "%*.15f", width, GET_DOUBLE_VAL(val));
if (n > SHELL_DOUBLE_WIDTH) {
taosFprintfFile(pFile, "%*.15e", width, GET_DOUBLE_VAL(val));
} else {
@ -386,26 +385,42 @@ void shellDumpFieldToFile(TdFilePtr pFile, const char *val, TAOS_FIELD *field, i
}
break;
case TSDB_DATA_TYPE_BINARY:
// case TSDB_DATA_TYPE_VARBINARY todovar
case TSDB_DATA_TYPE_NCHAR:
case TSDB_DATA_TYPE_JSON: {
int32_t bufIndex = 0;
char* tmp = (char*)taosMemoryCalloc(length * 2 + 1, 1);
if(tmp == NULL) break;
for (int32_t i = 0; i < length; i++) {
buf[bufIndex] = val[i];
tmp[bufIndex] = val[i];
bufIndex++;
if (val[i] == '\"') {
buf[bufIndex] = val[i];
tmp[bufIndex] = val[i];
bufIndex++;
}
}
buf[bufIndex] = 0;
tmp[bufIndex] = 0;
taosFprintfFile(pFile, "%s%s%s", quotationStr, buf, quotationStr);
taosFprintfFile(pFile, "%s%s%s", quotationStr, tmp, quotationStr);
taosMemoryFree(tmp);
} break;
case TSDB_DATA_TYPE_GEOMETRY:
shellDumpHexValue(buf, val, length);
taosFprintfFile(pFile, "%s", buf);
case TSDB_DATA_TYPE_VARBINARY:{
void* tmp = NULL;
uint32_t size = 0;
if(taosAscii2Hex(val, length, &tmp, &size) < 0){
break;
}
taosFprintfFile(pFile, "%s", tmp);
taosMemoryFree(tmp);
break;
}
case TSDB_DATA_TYPE_GEOMETRY:{
char* tmp = (char*)taosMemoryCalloc(length * 2 + 1, 1);
if(tmp == NULL) break;
shellDumpHexValue(tmp, val, length);
taosFprintfFile(pFile, "%s", buf);
taosMemoryFree(tmp);
break;
}
case TSDB_DATA_TYPE_TIMESTAMP:
shellFormatTimestamp(buf, *(int64_t *)val, precision);
taosFprintfFile(pFile, "%s%s%s", quotationStr, buf, quotationStr);
@ -779,14 +794,20 @@ int32_t shellCalcColWidth(TAOS_FIELD *field, int32_t precision) {
return TMAX(SHELL_DOUBLE_WIDTH, width);
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_VARBINARY:
case TSDB_DATA_TYPE_GEOMETRY:
if (field->bytes > shell.args.displayWidth) {
return TMAX(shell.args.displayWidth, width);
} else {
return TMAX(field->bytes + 2, width);
}
case TSDB_DATA_TYPE_VARBINARY:{
int32_t bytes = field->bytes * 2 + 2;
if (bytes > shell.args.displayWidth) {
return TMAX(shell.args.displayWidth, width);
} else {
return TMAX(bytes + 2, width);
}
}
case TSDB_DATA_TYPE_NCHAR:
case TSDB_DATA_TYPE_JSON: {
uint16_t bytes = field->bytes * TSDB_NCHAR_SIZE;

11
utils/test/c/CMakeLists.txt
View File

@ -8,6 +8,7 @@ add_executable(sml_test sml_test.c)
add_executable(get_db_name_test get_db_name_test.c)
add_executable(tmq_offset tmqOffset.c)
add_executable(tmq_offset_test tmq_offset_test.c)
add_executable(varbinary_test varbinary_test.c)
target_link_libraries(
tmq_offset
PUBLIC taos
@ -65,6 +66,8 @@ target_link_libraries(
PUBLIC util
PUBLIC common
PUBLIC os
PUBLIC geometry
)
target_link_libraries(
@ -74,3 +77,11 @@ target_link_libraries(
PUBLIC common
PUBLIC os
)
target_link_libraries(
varbinary_test
PUBLIC taos
PUBLIC util
PUBLIC common
PUBLIC os
)

80
utils/test/c/sml_test.c
View File

@ -21,6 +21,7 @@
#include "taos.h"
#include "tlog.h"
#include "types.h"
#include "geosWrapper.h"
int smlProcess_influx_Test() {
TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0);
@ -1578,6 +1579,83 @@ int sml_td24559_Test() {
printf("%s result0:%s\n", __FUNCTION__, taos_errstr(pRes));
taos_free_result(pRes);
TAOS_ROW row = NULL;
pRes = taos_query(taos, "select * from stb order by _ts;");
int rowIndex = 0;
while ((row = taos_fetch_row(pRes)) != NULL) {
int32_t* length = taos_fetch_lengths(pRes);
code = initCtxAsText();
ASSERT (code == TSDB_CODE_SUCCESS);
char *outputWKT = NULL;
code = doAsText(row[2], length[2], &outputWKT);
ASSERT (code == TSDB_CODE_SUCCESS);
if (rowIndex == 0) {
ASSERT(strcmp("POINT (4.343000 89.342000)", outputWKT) == 0);
}
if (rowIndex == 3) {
ASSERT(strcmp( "GEOMETRYCOLLECTION (MULTIPOINT ((0.000000 0.000000), (1.000000 1.000000)), POINT (3.000000 4.000000), LINESTRING (2.000000 3.000000, 3.000000 4.000000))", outputWKT) == 0);
}
geosFreeBuffer(outputWKT);
rowIndex++;
}
taos_free_result(pRes);
taos_close(taos);
return code;
}
int sml_td18789_Test() {
TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0);
TAOS_RES *pRes = taos_query(taos, "drop database if exists td18789");
taos_free_result(pRes);
pRes = taos_query(taos, "create database if not exists td18789");
taos_free_result(pRes);
const char *sql[] = {
"vbin,t1=1 f1=283i32,f2=b\"hello\" 1632299372000",
"vbin,t1=1 f2=B\"\\x98f46e\",f1=106i32 1632299373000",
};
pRes = taos_query(taos, "use td18789");
taos_free_result(pRes);
pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_LINE_PROTOCOL,
TSDB_SML_TIMESTAMP_MILLI_SECONDS);
int code = taos_errno(pRes);
printf("%s result0:%s\n", __FUNCTION__, taos_errstr(pRes));
taos_free_result(pRes);
TAOS_ROW row = NULL;
pRes = taos_query(taos, "select *,tbname from vbin order by _ts");
int rowIndex = 0;
while ((row = taos_fetch_row(pRes)) != NULL) {
int32_t* length = taos_fetch_lengths(pRes);
void* data = NULL;
uint32_t size = 0;
if(taosAscii2Hex(row[2], length[2], &data, &size) < 0){
ASSERT(0);
}
if (rowIndex == 0) {
ASSERT(memcmp(data, "\\x68656C6C6F", size) == 0);
}
if (rowIndex == 1) {
ASSERT(memcmp(data, "\\x98F46E", size) == 0);
}
taosMemoryFree(data);
rowIndex++;
}
taos_free_result(pRes);
taos_close(taos);
return code;
@ -1591,6 +1669,8 @@ int main(int argc, char *argv[]) {
int ret = 0;
ret = sml_td24559_Test();
ASSERT(!ret);
ret = sml_td18789_Test();
ASSERT(!ret);
ret = sml_td24070_Test();
ASSERT(!ret);
ret = sml_td23881_Test();

253
utils/test/c/varbinary_test.c Normal file
View File

@ -0,0 +1,253 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <assert.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
#include "taos.h"
#include "types.h"
#include "tlog.h"
#define GET_ROW_NUM \
numRows = 0;\
while(1){\
row = taos_fetch_row(pRes);\
if (row != NULL){\
numRows++;\
}else{\
break;\
}\
}
int varbinary_test() {
TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0);
TAOS_RES *pRes = taos_query(taos, "drop database if exists varbinary_db");
taos_free_result(pRes);
pRes = taos_query(taos, "create database if not exists varbinary_db");
taos_free_result(pRes);
pRes = taos_query(taos, "use varbinary_db");
int code = taos_errno(pRes);
taos_free_result(pRes);
ASSERT(code == 0);
pRes = taos_query(taos, "create stable stb (ts timestamp, c1 nchar(32), c2 varbinary(16), c3 float) tags (t1 int, t2 binary(8), t3 varbinary(8))");
taos_free_result(pRes);
pRes = taos_query(taos, "desc stb");
TAOS_ROW row = NULL;
int32_t rowIndex = 0;
while ((row = taos_fetch_row(pRes)) != NULL) {
char* type = row[1];
int32_t length = *(int32_t *)row[2];
if (rowIndex == 2) {
ASSERT(strncmp(type, "VARBINARY", sizeof("VARBINARY") - 1) == 0);
ASSERT(length == 16);
}
if (rowIndex == 6) {
ASSERT(strncmp(type, "VARBINARY", sizeof("VARBINARY") - 1) == 0);
ASSERT(length == 8);
}
rowIndex++;
}
pRes = taos_query(taos, "insert into tb1 using stb tags (1, 'tb1_bin1', 'vart1') values (now, 'nchar1', 'varc1', 0.3)");
taos_free_result(pRes);
pRes = taos_query(taos, "insert into tb1 values (now + 1s, 'nchar2', null, 0.4);");
taos_free_result(pRes);
pRes = taos_query(taos, "insert into tb3 using stb tags (3, 'tb3_bin1', '\\x7f8290') values (now + 2s, 'nchar1', '\\x7f8290', 0.3)");
taos_free_result(pRes);
pRes = taos_query(taos, "insert into tb3 values (now + 3s, 'nchar1', '\\x7f829000', 0.3)");
taos_free_result(pRes);
pRes = taos_query(taos, "insert into tb2 using stb tags (2, 'tb2_bin1', '\\x') values (now + 4s, 'nchar1', '\\x', 0.3)");
taos_free_result(pRes);
pRes = taos_query(taos, "insert into tb2 values (now + 5s, 'nchar1', '\\x00000000', 0.3)");
taos_free_result(pRes);
// test insert
pRes = taos_query(taos, "insert into tb2 using stb tags (2, 'tb2_bin1', 093) values (now + 2s, 'nchar1', 892, 0.3)");
ASSERT(taos_errno(pRes) != 0);
pRes = taos_query(taos, "insert into tb3 using stb tags (3, 'tb3_bin1', 0x7f829) values (now + 3s, 'nchar1', 0x7f829, 0.3)");
ASSERT(taos_errno(pRes) != 0);
pRes = taos_query(taos, "insert into tb3 using stb tags (3, 'tb3_bin1', '\\x7f829') values (now + 3s, 'nchar1', '\\x7f829', 0.3)");
ASSERT(taos_errno(pRes) != 0);
pRes = taos_query(taos, "insert into tb4 using stb tags (4, 'tb4_bin1', 0b100000010) values (now + 4s, 'nchar1', 0b110000001, 0.3)");
ASSERT(taos_errno(pRes) != 0);
taos_free_result(pRes);
// test error
pRes = taos_query(taos, "select * from tb1 where c2 >= 0x8de6");
ASSERT(taos_errno(pRes) != 0);
taos_free_result(pRes);
pRes = taos_query(taos, "select * from tb1 where c2 >= 0");
ASSERT(taos_errno(pRes) != 0);
taos_free_result(pRes);
pRes = taos_query(taos, "select * from stb where c2 > '\\x7F82900'");
ASSERT(taos_errno(pRes) != 0);
taos_free_result(pRes);
pRes = taos_query(taos, "select c2+2 from stb");
ASSERT(taos_errno(pRes) != 0);
taos_free_result(pRes);
pRes = taos_query(taos, "select c2|2 from stb");
ASSERT(taos_errno(pRes) != 0);
taos_free_result(pRes);
pRes = taos_query(taos, "select * from stb where c2 not like 's%'");
ASSERT(taos_errno(pRes) != 0);
taos_free_result(pRes);
pRes = taos_query(taos, "select * from stb where c2 like 's%'");
ASSERT(taos_errno(pRes) != 0);
taos_free_result(pRes);
pRes = taos_query(taos, "select * from stb where c2 match 'ssd'");
ASSERT(taos_errno(pRes) != 0);
taos_free_result(pRes);
pRes = taos_query(taos, "select * from stb where c2 nmatch 'ssd'");
ASSERT(taos_errno(pRes) != 0);
taos_free_result(pRes);
pRes = taos_query(taos, "select * from stb where c2->'ssd' = 1");
ASSERT(taos_errno(pRes) != 0);
taos_free_result(pRes);
pRes = taos_query(taos, "select * from stb where c2 contains 'ssd'");
ASSERT(taos_errno(pRes) != 0);
taos_free_result(pRes);
pRes = taos_query(taos, "select * from stb where sum(c2) = 2");
ASSERT(taos_errno(pRes) != 0);
taos_free_result(pRes);
// pRes = taos_query(taos, "select * from stb where c2 contains 'ssd'");
// ASSERT(taos_errno(pRes) != 0);
// taos_free_result(pRes);
//
// pRes = taos_query(taos, "select * from stb where c2 contains 'ssd'");
// ASSERT(taos_errno(pRes) != 0);
// taos_free_result(pRes);
int numRows = 0;
pRes = taos_query(taos, "select * from stb where c2 > 'varc1'");
GET_ROW_NUM
ASSERT(numRows == 2);
taos_free_result(pRes);
pRes = taos_query(taos, "select * from stb where c2 > '\\x7F8290'");
GET_ROW_NUM
ASSERT(numRows == 1);
taos_free_result(pRes);
pRes = taos_query(taos, "select * from stb where c2 in ('\\x7F829000','\\x00000000')");
GET_ROW_NUM
ASSERT(numRows == 2);
taos_free_result(pRes);
pRes = taos_query(taos, "select * from stb where c2 not in ('\\x00000000')");
GET_ROW_NUM
ASSERT(numRows == 4);
taos_free_result(pRes);
pRes = taos_query(taos, "select * from stb where c2 is null");
GET_ROW_NUM
ASSERT(numRows == 1);
taos_free_result(pRes);
pRes = taos_query(taos, "select * from stb where c2 is not null");
GET_ROW_NUM
ASSERT(numRows == 5);
taos_free_result(pRes);
pRes = taos_query(taos, "select * from stb where c2 between '\\x3e' and '\\x7F8290'");
GET_ROW_NUM
ASSERT(numRows == 2);
taos_free_result(pRes);
pRes = taos_query(taos, "select * from stb where c2 not between '\\x3e' and '\\x7F8290'");
GET_ROW_NUM
ASSERT(numRows == 3);
taos_free_result(pRes);
pRes = taos_query(taos, "select ts,c2 from stb order by c2");
rowIndex = 0;
while ((row = taos_fetch_row(pRes)) != NULL) {
// int64_t ts = *(int64_t *)row[0];
if (rowIndex == 0) {
ASSERT(row[1] == NULL);
rowIndex++;
continue;
}
int32_t* length = taos_fetch_lengths(pRes);
void* data = NULL;
uint32_t size = 0;
if(taosAscii2Hex(row[1], length[1], &data, &size) < 0){
ASSERT(0);
}
if (rowIndex == 1) {
// ASSERT(ts == 1661943960000);
ASSERT(memcmp(data, "\\x", size) == 0);
}
if (rowIndex == 2) {
// ASSERT(ts == 1661943960000);
ASSERT(memcmp(data, "\\x00000000", size) == 0);
}
if (rowIndex == 3) {
// ASSERT(ts == 1661943960000);
ASSERT(memcmp(data, "\\x7661726331", size) == 0);
}
if (rowIndex == 4) {
// ASSERT(ts == 1661943960000);
ASSERT(memcmp(data, "\\x7F8290", size) == 0);
}
if (rowIndex == 5) {
// ASSERT(ts == 1661943960000);
ASSERT(memcmp(data, "\\x7F829000", size) == 0);
}
taosMemoryFree(data);
rowIndex++;
}
printf("%s result1:%s\n", __FUNCTION__, taos_errstr(pRes));
taos_free_result(pRes);
taos_close(taos);
return code;
}
int main(int argc, char *argv[]) {
int ret = 0;
ret = varbinary_test();
ASSERT(!ret);
return ret;
}

1
utils/tsim/src/simExe.c
View File

@ -751,7 +751,6 @@ bool simExecuteNativeSqlCommand(SScript *script, char *rest, bool isSlow) {
sprintf(value, "%.9lf", GET_DOUBLE_VAL(row[i]));
break;
case TSDB_DATA_TYPE_BINARY:
// case TSDB_DATA_TYPE_VARBINARY todovar
case TSDB_DATA_TYPE_NCHAR:
case TSDB_DATA_TYPE_GEOMETRY:
if (length[i] < 0 || length[i] > 1 << 20) {