From 7441801e9d76029e44095c700999b3de7fc071c9 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 15 Aug 2023 13:57:05 +0800 Subject: [PATCH 01/13] feat:support varbinary type --- docs/examples/c/async_query_example.c | 1 + docs/examples/c/query_example.c | 1 + docs/zh/08-connector/45-php.mdx | 1 + include/common/ttypes.h | 4 +- source/client/src/clientJniConnector.c | 1 + source/client/src/clientMain.c | 3 +- source/client/src/clientRawBlockWrite.c | 12 ++-- source/client/src/clientSml.c | 6 +- source/client/src/clientSmlJson.c | 4 +- source/client/src/clientSmlLine.c | 2 +- source/common/src/ttszip.c | 6 +- source/common/src/ttypes.c | 1 + source/common/src/tvariant.c | 13 +++-- source/dnode/mnode/impl/src/mndFunc.c | 3 +- source/dnode/mnode/impl/src/mndStb.c | 6 +- source/dnode/mnode/impl/src/mndTopic.c | 3 +- source/libs/catalog/src/ctgAsync.c | 1 + source/libs/command/src/command.c | 1 + source/libs/executor/src/sysscanoperator.c | 3 +- source/libs/executor/src/timewindowoperator.c | 2 +- source/libs/index/src/indexComm.c | 3 +- source/libs/index/src/indexJson.c | 4 +- source/libs/index/src/indexTfile.c | 4 +- source/libs/parser/src/parInsertSml.c | 6 +- source/libs/parser/src/parInsertSql.c | 23 +++++++- source/libs/parser/src/parInsertStmt.c | 2 +- source/libs/parser/src/parTokenizer.c | 1 + source/libs/parser/src/parTranslater.c | 2 +- source/libs/qcom/src/queryUtil.c | 4 +- source/libs/scalar/inc/filterInt.h | 2 +- source/libs/scalar/src/filter.c | 7 ++- source/libs/scalar/src/sclvector.c | 58 +++---------------- source/util/src/tcompare.c | 8 ++- source/util/src/thashutil.c | 1 + source/util/src/tskiplist.c | 1 + tests/script/api/batchprepare.c | 5 +- tests/script/api/passwdTest.c | 1 + tests/script/api/stmt_function.c | 1 + tools/shell/src/shellEngine.c | 3 + utils/test/c/tmqSim.c | 1 + utils/tsim/src/simExe.c | 1 + 41 files changed, 115 insertions(+), 97 deletions(-) diff --git a/docs/examples/c/async_query_example.c b/docs/examples/c/async_query_example.c index 0618c09f36..9b4b6c943e 100644 --- a/docs/examples/c/async_query_example.c +++ b/docs/examples/c/async_query_example.c @@ -78,6 +78,7 @@ int printRow(char *str, TAOS_ROW row, TAOS_FIELD *fields, int numFields) { } break; case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_VARBINARY: case TSDB_DATA_TYPE_NCHAR: case TSDB_DATA_TYPE_GEOMETRY: { int32_t charLen = varDataLen((char *)row[i] - VARSTR_HEADER_SIZE); diff --git a/docs/examples/c/query_example.c b/docs/examples/c/query_example.c index 88c031abc6..f483371a97 100644 --- a/docs/examples/c/query_example.c +++ b/docs/examples/c/query_example.c @@ -76,6 +76,7 @@ int printRow(char *str, TAOS_ROW row, TAOS_FIELD *fields, int numFields) { } break; case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_VARBINARY: case TSDB_DATA_TYPE_NCHAR: case TSDB_DATA_TYPE_GEOMETRY: { int32_t charLen = varDataLen((char *)row[i] - VARSTR_HEADER_SIZE); diff --git a/docs/zh/08-connector/45-php.mdx b/docs/zh/08-connector/45-php.mdx index a5c3a1a400..0f32b29bf7 100644 --- a/docs/zh/08-connector/45-php.mdx +++ b/docs/zh/08-connector/45-php.mdx @@ -143,6 +143,7 @@ phpize && ./configure --enable-swoole && make -j && make install | `TDengine\TSDB_DATA_TYPE_FLOAT` | float | | `TDengine\TSDB_DATA_TYPE_DOUBLE` | double | | `TDengine\TSDB_DATA_TYPE_BINARY` | binary | +| `TDengine\TSDB_DATA_TYPE_VARBINARY` | varbinary | | `TDengine\TSDB_DATA_TYPE_TIMESTAMP` | timestamp | | `TDengine\TSDB_DATA_TYPE_NCHAR` | nchar | | `TDengine\TSDB_DATA_TYPE_UTINYINT` | utinyint | diff --git a/include/common/ttypes.h b/include/common/ttypes.h index dd8033eb43..559983185c 100644 --- a/include/common/ttypes.h +++ b/include/common/ttypes.h @@ -269,8 +269,8 @@ typedef struct { (IS_NUMERIC_TYPE(_t) || (_t) == (TSDB_DATA_TYPE_BOOL) || (_t) == (TSDB_DATA_TYPE_TIMESTAMP)) #define IS_VAR_DATA_TYPE(t) \ - (((t) == TSDB_DATA_TYPE_VARCHAR) || ((t) == TSDB_DATA_TYPE_NCHAR) || ((t) == TSDB_DATA_TYPE_JSON) || ((t) == TSDB_DATA_TYPE_GEOMETRY)) -#define IS_STR_DATA_TYPE(t) (((t) == TSDB_DATA_TYPE_VARCHAR) || ((t) == TSDB_DATA_TYPE_NCHAR)) + (((t) == TSDB_DATA_TYPE_VARCHAR) || (t) == TSDB_DATA_TYPE_VARBINARY || ((t) == TSDB_DATA_TYPE_NCHAR) || ((t) == TSDB_DATA_TYPE_JSON) || ((t) == TSDB_DATA_TYPE_GEOMETRY)) +#define IS_STR_DATA_TYPE(t) (((t) == TSDB_DATA_TYPE_VARCHAR) || ((t) == TSDB_DATA_TYPE_VARBINARY) || ((t) == TSDB_DATA_TYPE_NCHAR)) #define IS_VALID_TINYINT(_t) ((_t) >= INT8_MIN && (_t) <= INT8_MAX) #define IS_VALID_SMALLINT(_t) ((_t) >= INT16_MIN && (_t) <= INT16_MAX) diff --git a/source/client/src/clientJniConnector.c b/source/client/src/clientJniConnector.c index 4ad2d4d51f..edbc06b12b 100644 --- a/source/client/src/clientJniConnector.c +++ b/source/client/src/clientJniConnector.c @@ -580,6 +580,7 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_fetchRowImp(JNIEn (*env)->CallVoidMethod(env, rowobj, g_rowdataSetDoubleFp, i, (jdouble)dv); } break; case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_VARBINARY: case TSDB_DATA_TYPE_GEOMETRY: { memcpy(tmp, row[i], length[i]); // handle the case that terminated does not exist (*env)->CallVoidMethod(env, rowobj, g_rowdataSetStringFp, i, (*env)->NewStringUTF(env, tmp)); diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index e262ee04b9..64bcd6d898 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -388,10 +388,11 @@ int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields) } break; case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_VARBINARY: case TSDB_DATA_TYPE_NCHAR: case TSDB_DATA_TYPE_GEOMETRY: { int32_t charLen = varDataLen((char *)row[i] - VARSTR_HEADER_SIZE); - if (fields[i].type == TSDB_DATA_TYPE_BINARY || fields[i].type == TSDB_DATA_TYPE_GEOMETRY) { + if (fields[i].type == TSDB_DATA_TYPE_BINARY || fields[i].type == TSDB_DATA_TYPE_VARBINARY || fields[i].type == TSDB_DATA_TYPE_GEOMETRY) { if (ASSERT(charLen <= fields[i].bytes && charLen >= 0)) { tscError("taos_print_row error binary. charLen:%d, fields[i].bytes:%d", charLen, fields[i].bytes); } diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c index dd311db126..0b28949400 100644 --- a/source/client/src/clientRawBlockWrite.c +++ b/source/client/src/clientRawBlockWrite.c @@ -56,7 +56,7 @@ static char* buildCreateTableJson(SSchemaWrapper* schemaRow, SSchemaWrapper* sch cJSON_AddItemToObject(column, "name", cname); cJSON* ctype = cJSON_CreateNumber(s->type); cJSON_AddItemToObject(column, "type", ctype); - if (s->type == TSDB_DATA_TYPE_BINARY || s->type == TSDB_DATA_TYPE_GEOMETRY) { + if (s->type == TSDB_DATA_TYPE_BINARY || s->type == TSDB_DATA_TYPE_VARBINARY|| s->type == TSDB_DATA_TYPE_GEOMETRY) { int32_t length = s->bytes - VARSTR_HEADER_SIZE; cJSON* cbytes = cJSON_CreateNumber(length); cJSON_AddItemToObject(column, "length", cbytes); @@ -77,7 +77,7 @@ static char* buildCreateTableJson(SSchemaWrapper* schemaRow, SSchemaWrapper* sch cJSON_AddItemToObject(tag, "name", tname); cJSON* ttype = cJSON_CreateNumber(s->type); cJSON_AddItemToObject(tag, "type", ttype); - if (s->type == TSDB_DATA_TYPE_BINARY || s->type == TSDB_DATA_TYPE_GEOMETRY) { + if (s->type == TSDB_DATA_TYPE_BINARY || s->type == TSDB_DATA_TYPE_VARBINARY || s->type == TSDB_DATA_TYPE_GEOMETRY) { int32_t length = s->bytes - VARSTR_HEADER_SIZE; cJSON* cbytes = cJSON_CreateNumber(length); cJSON_AddItemToObject(tag, "length", cbytes); @@ -130,7 +130,7 @@ static char* buildAlterSTableJson(void* alterData, int32_t alterDataLen) { cJSON* colType = cJSON_CreateNumber(field->type); cJSON_AddItemToObject(json, "colType", colType); - if (field->type == TSDB_DATA_TYPE_BINARY || field->type == TSDB_DATA_TYPE_GEOMETRY) { + if (field->type == TSDB_DATA_TYPE_BINARY || field->type == TSDB_DATA_TYPE_VARBINARY || field->type == TSDB_DATA_TYPE_GEOMETRY) { int32_t length = field->bytes - VARSTR_HEADER_SIZE; cJSON* cbytes = cJSON_CreateNumber(length); cJSON_AddItemToObject(json, "colLength", cbytes); @@ -155,7 +155,7 @@ static char* buildAlterSTableJson(void* alterData, int32_t alterDataLen) { cJSON_AddItemToObject(json, "colName", colName); cJSON* colType = cJSON_CreateNumber(field->type); cJSON_AddItemToObject(json, "colType", colType); - if (field->type == TSDB_DATA_TYPE_BINARY || field->type == TSDB_DATA_TYPE_GEOMETRY) { + if (field->type == TSDB_DATA_TYPE_BINARY || field->type == TSDB_DATA_TYPE_VARBINARY || field->type == TSDB_DATA_TYPE_GEOMETRY) { int32_t length = field->bytes - VARSTR_HEADER_SIZE; cJSON* cbytes = cJSON_CreateNumber(length); cJSON_AddItemToObject(json, "colLength", cbytes); @@ -457,7 +457,7 @@ static char* processAlterTable(SMqMetaRsp* metaRsp) { cJSON* colType = cJSON_CreateNumber(vAlterTbReq.type); cJSON_AddItemToObject(json, "colType", colType); - if (vAlterTbReq.type == TSDB_DATA_TYPE_BINARY || vAlterTbReq.type == TSDB_DATA_TYPE_GEOMETRY) { + if (vAlterTbReq.type == TSDB_DATA_TYPE_BINARY || vAlterTbReq.type == TSDB_DATA_TYPE_VARBINARY || vAlterTbReq.type == TSDB_DATA_TYPE_GEOMETRY) { int32_t length = vAlterTbReq.bytes - VARSTR_HEADER_SIZE; cJSON* cbytes = cJSON_CreateNumber(length); cJSON_AddItemToObject(json, "colLength", cbytes); @@ -478,7 +478,7 @@ static char* processAlterTable(SMqMetaRsp* metaRsp) { cJSON_AddItemToObject(json, "colName", colName); cJSON* colType = cJSON_CreateNumber(vAlterTbReq.colModType); cJSON_AddItemToObject(json, "colType", colType); - if (vAlterTbReq.colModType == TSDB_DATA_TYPE_BINARY || vAlterTbReq.colModType == TSDB_DATA_TYPE_GEOMETRY) { + if (vAlterTbReq.colModType == TSDB_DATA_TYPE_BINARY || vAlterTbReq.colModType == TSDB_DATA_TYPE_VARBINARY || vAlterTbReq.colModType == TSDB_DATA_TYPE_GEOMETRY) { int32_t length = vAlterTbReq.colModBytes - VARSTR_HEADER_SIZE; cJSON* cbytes = cJSON_CreateNumber(length); cJSON_AddItemToObject(json, "colLength", cbytes); diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index ffff3df5d0..343cb9d1a3 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -627,7 +627,7 @@ static int32_t smlFindNearestPowerOf2(int32_t length, uint8_t type) { } } - if ((type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_GEOMETRY) && result > TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) { + if ((type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_VARBINARY || type == TSDB_DATA_TYPE_GEOMETRY) && result > TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) { result = TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE; } else if (type == TSDB_DATA_TYPE_NCHAR && result > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { result = (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE; @@ -635,7 +635,7 @@ static int32_t smlFindNearestPowerOf2(int32_t length, uint8_t type) { if (type == TSDB_DATA_TYPE_NCHAR) { result = result * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE; - } else if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_GEOMETRY) { + } else if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_VARBINARY || type == TSDB_DATA_TYPE_GEOMETRY) { result = result + VARSTR_HEADER_SIZE; } return result; @@ -679,7 +679,7 @@ static int32_t smlCheckMeta(SSchema *schema, int32_t length, SArray *cols, bool } static int32_t getBytes(uint8_t type, int32_t length) { - if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_GEOMETRY) { + if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_VARBINARY || type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_GEOMETRY) { return smlFindNearestPowerOf2(length, type); } else { return tDataTypes[type].bytes; diff --git a/source/client/src/clientSmlJson.c b/source/client/src/clientSmlJson.c index 76794fd187..8c53d4f68a 100644 --- a/source/client/src/clientSmlJson.c +++ b/source/client/src/clientSmlJson.c @@ -569,6 +569,8 @@ static int32_t smlConvertJSONNumber(SSmlKv *pVal, char *typeStr, cJSON *value) { static int32_t smlConvertJSONString(SSmlKv *pVal, char *typeStr, cJSON *value) { if (strcasecmp(typeStr, "binary") == 0) { pVal->type = TSDB_DATA_TYPE_BINARY; + } else if (strcasecmp(typeStr, "varbinary") == 0) { + pVal->type = TSDB_DATA_TYPE_VARBINARY; } else if (strcasecmp(typeStr, "nchar") == 0) { pVal->type = TSDB_DATA_TYPE_NCHAR; } else { @@ -577,7 +579,7 @@ static int32_t smlConvertJSONString(SSmlKv *pVal, char *typeStr, cJSON *value) { } pVal->length = strlen(value->valuestring); - if (pVal->type == TSDB_DATA_TYPE_BINARY && pVal->length > TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) { + if ((pVal->type == TSDB_DATA_TYPE_BINARY || pVal->type == TSDB_DATA_TYPE_VARBINARY) && pVal->length > TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) { return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; } if (pVal->type == TSDB_DATA_TYPE_NCHAR && diff --git a/source/client/src/clientSmlLine.c b/source/client/src/clientSmlLine.c index 1ee2cfbedf..90869d6148 100644 --- a/source/client/src/clientSmlLine.c +++ b/source/client/src/clientSmlLine.c @@ -390,7 +390,7 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin SSmlKv kv = {.key = tag->name, .keyLen = strlen(tag->name), .type = tag->type}; if (tag->type == TSDB_DATA_TYPE_NCHAR) { kv.length = (tag->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE; - } else if (tag->type == TSDB_DATA_TYPE_BINARY) { + } else if (tag->type == TSDB_DATA_TYPE_BINARY || tag->type == TSDB_DATA_TYPE_VARBINARY) { kv.length = tag->bytes - VARSTR_HEADER_SIZE; } taosArrayPush((*tmp)->cols, &kv); diff --git a/source/common/src/ttszip.c b/source/common/src/ttszip.c index aabbff6f04..38659eea44 100644 --- a/source/common/src/ttszip.c +++ b/source/common/src/ttszip.c @@ -296,7 +296,8 @@ static void writeDataToDisk(STSBuf* pTSBuf) { metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, &pBlock->tag.nType, sizeof(pBlock->tag.nType)); int32_t trueLen = pBlock->tag.nLen; - if (pBlock->tag.nType == TSDB_DATA_TYPE_BINARY || pBlock->tag.nType == TSDB_DATA_TYPE_NCHAR || pBlock->tag.nType == TSDB_DATA_TYPE_GEOMETRY) { + if (pBlock->tag.nType == TSDB_DATA_TYPE_BINARY || pBlock->tag.nType == TSDB_DATA_TYPE_VARBINARY || + pBlock->tag.nType == TSDB_DATA_TYPE_NCHAR || pBlock->tag.nType == TSDB_DATA_TYPE_GEOMETRY) { metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, &pBlock->tag.nLen, sizeof(pBlock->tag.nLen)); metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, pBlock->tag.pz, (size_t)pBlock->tag.nLen); } else if (pBlock->tag.nType == TSDB_DATA_TYPE_FLOAT) { @@ -378,7 +379,8 @@ STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) { // NOTE: mix types tags are not supported size_t sz = 0; - if (pBlock->tag.nType == TSDB_DATA_TYPE_BINARY || pBlock->tag.nType == TSDB_DATA_TYPE_NCHAR || pBlock->tag.nType == TSDB_DATA_TYPE_GEOMETRY) { + if (pBlock->tag.nType == TSDB_DATA_TYPE_BINARY || pBlock->tag.nType == TSDB_DATA_TYPE_VARBINARY || + pBlock->tag.nType == TSDB_DATA_TYPE_NCHAR || pBlock->tag.nType == TSDB_DATA_TYPE_GEOMETRY) { char* tp = taosMemoryRealloc(pBlock->tag.pz, pBlock->tag.nLen + 1); ASSERT(tp != NULL); diff --git a/source/common/src/ttypes.c b/source/common/src/ttypes.c index 39255cff3a..6acd2d087a 100644 --- a/source/common/src/ttypes.c +++ b/source/common/src/ttypes.c @@ -135,6 +135,7 @@ void assignVal(char *val, const char *src, int32_t len, int32_t type) { *((int64_t *)val) = GET_INT64_VAL(src); break; case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_VARBINARY: case TSDB_DATA_TYPE_GEOMETRY: varDataCopy(val, src); break; diff --git a/source/common/src/tvariant.c b/source/common/src/tvariant.c index cb4a42eb10..5f2796260c 100644 --- a/source/common/src/tvariant.c +++ b/source/common/src/tvariant.c @@ -122,6 +122,7 @@ void taosVariantCreateFromBinary(SVariant *pVar, const char *pz, size_t len, uin break; } case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_VARBINARY: case TSDB_DATA_TYPE_GEOMETRY: { // todo refactor, extract a method pVar->pz = taosMemoryCalloc(len + 1, sizeof(char)); memcpy(pVar->pz, pz, len); @@ -141,7 +142,8 @@ void taosVariantDestroy(SVariant *pVar) { if (pVar == NULL) return; if (pVar->nType == TSDB_DATA_TYPE_BINARY || pVar->nType == TSDB_DATA_TYPE_NCHAR || - pVar->nType == TSDB_DATA_TYPE_JSON || pVar->nType == TSDB_DATA_TYPE_GEOMETRY) { + pVar->nType == TSDB_DATA_TYPE_JSON || pVar->nType == TSDB_DATA_TYPE_GEOMETRY || + pVar->nType == TSDB_DATA_TYPE_VARBINARY) { taosMemoryFreeClear(pVar->pz); pVar->nLen = 0; } @@ -152,8 +154,9 @@ void taosVariantAssign(SVariant *pDst, const SVariant *pSrc) { if (pSrc == NULL || pDst == NULL) return; pDst->nType = pSrc->nType; - if (pSrc->nType == TSDB_DATA_TYPE_BINARY || pSrc->nType == TSDB_DATA_TYPE_NCHAR || - pSrc->nType == TSDB_DATA_TYPE_JSON || pSrc->nType == TSDB_DATA_TYPE_GEOMETRY) { + if (pSrc->nType == TSDB_DATA_TYPE_BINARY ||pSrc->nType == TSDB_DATA_TYPE_VARBINARY || + pSrc->nType == TSDB_DATA_TYPE_NCHAR || pSrc->nType == TSDB_DATA_TYPE_JSON || + pSrc->nType == TSDB_DATA_TYPE_GEOMETRY) { int32_t len = pSrc->nLen + TSDB_NCHAR_SIZE; char *p = taosMemoryRealloc(pDst->pz, len); ASSERT(p); @@ -185,7 +188,8 @@ int32_t taosVariantCompare(const SVariant *p1, const SVariant *p2) { return 1; } - if (p1->nType == TSDB_DATA_TYPE_BINARY || p1->nType == TSDB_DATA_TYPE_NCHAR || p1->nType == TSDB_DATA_TYPE_GEOMETRY) { + if (p1->nType == TSDB_DATA_TYPE_BINARY || p1->nType == TSDB_DATA_TYPE_VARBINARY || + p1->nType == TSDB_DATA_TYPE_NCHAR || p1->nType == TSDB_DATA_TYPE_GEOMETRY) { if (p1->nLen == p2->nLen) { return memcmp(p1->pz, p2->pz, p1->nLen); } else { @@ -237,6 +241,7 @@ char *taosVariantGet(SVariant *pVar, int32_t type) { case TSDB_DATA_TYPE_FLOAT: return (char *)&pVar->f; case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_VARBINARY: case TSDB_DATA_TYPE_JSON: case TSDB_DATA_TYPE_GEOMETRY: return (char *)pVar->pz; diff --git a/source/dnode/mnode/impl/src/mndFunc.c b/source/dnode/mnode/impl/src/mndFunc.c index 4ffc7a20c2..a1e5bf895a 100644 --- a/source/dnode/mnode/impl/src/mndFunc.c +++ b/source/dnode/mnode/impl/src/mndFunc.c @@ -562,7 +562,8 @@ static void *mnodeGenTypeStr(char *buf, int32_t buflen, uint8_t type, int32_t le return msg; } - if (type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_GEOMETRY) { + if (type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_VARBINARY || + type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_GEOMETRY) { int32_t bytes = len > 0 ? (int32_t)(len - VARSTR_HEADER_SIZE) : len; snprintf(buf, buflen - 1, "%s(%d)", tDataTypes[type].name, type == TSDB_DATA_TYPE_NCHAR ? bytes / 4 : bytes); diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 70fd74afc0..df4d4a3be7 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -1478,7 +1478,8 @@ static int32_t mndAlterStbTagBytes(SMnode *pMnode, const SStbObj *pOld, SStbObj SSchema *pTag = pNew->pTags + tag; - if (!(pTag->type == TSDB_DATA_TYPE_BINARY || pTag->type == TSDB_DATA_TYPE_NCHAR || pTag->type == TSDB_DATA_TYPE_GEOMETRY)) { + if (!(pTag->type == TSDB_DATA_TYPE_BINARY || pTag->type == TSDB_DATA_TYPE_VARBINARY || + pTag->type == TSDB_DATA_TYPE_NCHAR || pTag->type == TSDB_DATA_TYPE_GEOMETRY)) { terrno = TSDB_CODE_MND_INVALID_STB_OPTION; return -1; } @@ -1598,7 +1599,8 @@ static int32_t mndAlterStbColumnBytes(SMnode *pMnode, const SStbObj *pOld, SStbO } SSchema *pCol = pNew->pColumns + col; - if (!(pCol->type == TSDB_DATA_TYPE_BINARY || pCol->type == TSDB_DATA_TYPE_NCHAR || pCol->type == TSDB_DATA_TYPE_GEOMETRY)) { + if (!(pCol->type == TSDB_DATA_TYPE_BINARY || pCol->type == TSDB_DATA_TYPE_VARBINARY || + pCol->type == TSDB_DATA_TYPE_NCHAR || pCol->type == TSDB_DATA_TYPE_GEOMETRY)) { terrno = TSDB_CODE_MND_INVALID_STB_OPTION; return -1; } diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index 621a80338d..0c8ebd83c9 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -860,7 +860,8 @@ static void schemaToJson(SSchema *schema, int32_t nCols, char *schemaJson){ cJSON* ctype = cJSON_CreateString(tDataTypes[s->type].name); cJSON_AddItemToObject(column, "type", ctype); int32_t length = 0; - if (s->type == TSDB_DATA_TYPE_BINARY) { + if (s->type == TSDB_DATA_TYPE_BINARY || s->type == TSDB_DATA_TYPE_VARBINARY || + s->type == TSDB_DATA_TYPE_GEOMETRY) { length = s->bytes - VARSTR_HEADER_SIZE; } else if (s->type == TSDB_DATA_TYPE_NCHAR || s->type == TSDB_DATA_TYPE_JSON) { length = (s->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE; diff --git a/source/libs/catalog/src/ctgAsync.c b/source/libs/catalog/src/ctgAsync.c index fb5ecf7ad2..5b0d1b4446 100644 --- a/source/libs/catalog/src/ctgAsync.c +++ b/source/libs/catalog/src/ctgAsync.c @@ -1051,6 +1051,7 @@ void ctgUpdateJobErrCode(SCtgJob* pJob, int32_t errCode) { atomic_store_32(&pJob->jobResCode, errCode); qDebug("QID:0x%" PRIx64 " ctg job errCode updated to %s", pJob->queryId, tstrerror(errCode)); + ASSERT(0); return; } diff --git a/source/libs/command/src/command.c b/source/libs/command/src/command.c index 8ddf730d5a..98aab26b0d 100644 --- a/source/libs/command/src/command.c +++ b/source/libs/command/src/command.c @@ -48,6 +48,7 @@ static int32_t buildRetrieveTableRsp(SSDataBlock* pBlock, int32_t numOfCols, SRe static int32_t getSchemaBytes(const SSchema* pSchema) { switch (pSchema->type) { case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_VARBINARY: case TSDB_DATA_TYPE_GEOMETRY: return (pSchema->bytes - VARSTR_HEADER_SIZE); case TSDB_DATA_TYPE_NCHAR: diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c index a1f83dda2f..019a9f4d2c 100644 --- a/source/libs/executor/src/sysscanoperator.c +++ b/source/libs/executor/src/sysscanoperator.c @@ -400,8 +400,7 @@ static bool sysTableIsOperatorCondOnOneTable(SNode* pCond, char* condTable) { strcasecmp(nodesGetNameFromColumnNode(node->pLeft), "table_name") == 0 && nodeType(node->pRight) == QUERY_NODE_VALUE) { SValueNode* pValue = (SValueNode*)node->pRight; - if (pValue->node.resType.type == TSDB_DATA_TYPE_NCHAR || pValue->node.resType.type == TSDB_DATA_TYPE_VARCHAR || - pValue->node.resType.type == TSDB_DATA_TYPE_BINARY) { + if (pValue->node.resType.type == TSDB_DATA_TYPE_NCHAR || pValue->node.resType.type == TSDB_DATA_TYPE_VARCHAR) { char* value = nodesGetValueFromNode(pValue); strncpy(condTable, varDataVal(value), TSDB_TABLE_NAME_LEN); return true; diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 3abb4f44f4..6884855ece 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -308,7 +308,7 @@ void doTimeWindowInterpolation(SArray* pPrevValues, SArray* pDataBlock, TSKEY pr pCtx[k].end.key = curTs; pCtx[k].end.val = v2; - if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR || + if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_VARBINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR || pColInfo->info.type == TSDB_DATA_TYPE_GEOMETRY) { if (prevRowIndex == -1) { // pCtx[k].start.ptr = (char*)pRuntimeEnv->prevRow[index]; diff --git a/source/libs/index/src/indexComm.c b/source/libs/index/src/indexComm.c index a0310a9f0e..9f8c2139eb 100644 --- a/source/libs/index/src/indexComm.c +++ b/source/libs/index/src/indexComm.c @@ -76,7 +76,8 @@ char* idxInt2str(int64_t val, char* dst, int radix) { return dst - 1; } __compar_fn_t idxGetCompar(int8_t type) { - if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_GEOMETRY) { + if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_VARBINARY || + type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_GEOMETRY) { return (__compar_fn_t)strcmp; } return getComparFunc(type, 0); diff --git a/source/libs/index/src/indexJson.c b/source/libs/index/src/indexJson.c index 1b4f847919..09c1320667 100644 --- a/source/libs/index/src/indexJson.c +++ b/source/libs/index/src/indexJson.c @@ -24,7 +24,7 @@ int indexJsonPut(SIndexJson *index, SIndexJsonMultiTerm *terms, uint64_t uid) { SIndexJsonTerm *p = taosArrayGetP(terms, i); if (p->colType == TSDB_DATA_TYPE_BOOL) { p->colType = TSDB_DATA_TYPE_INT; - } else if (p->colType == TSDB_DATA_TYPE_VARCHAR || p->colType == TSDB_DATA_TYPE_NCHAR || + } else if (p->colType == TSDB_DATA_TYPE_VARBINARY || p->colType == TSDB_DATA_TYPE_NCHAR || p->colType == TSDB_DATA_TYPE_BINARY || p->colType == TSDB_DATA_TYPE_GEOMETRY) { // p->colType = TSDB_DATA_TYPE_NCHAR; } else { @@ -42,7 +42,7 @@ int indexJsonSearch(SIndexJson *index, SIndexJsonMultiTermQuery *tq, SArray *res SIndexJsonTerm *p = taosArrayGetP(terms, i); if (p->colType == TSDB_DATA_TYPE_BOOL) { p->colType = TSDB_DATA_TYPE_INT; - } else if (p->colType == TSDB_DATA_TYPE_VARCHAR || p->colType == TSDB_DATA_TYPE_NCHAR || + } else if (p->colType == TSDB_DATA_TYPE_VARBINARY || p->colType == TSDB_DATA_TYPE_NCHAR || p->colType == TSDB_DATA_TYPE_BINARY || p->colType == TSDB_DATA_TYPE_GEOMETRY) { // p->colType = TSDB_DATA_TYPE_NCHAR; } else { diff --git a/source/libs/index/src/indexTfile.c b/source/libs/index/src/indexTfile.c index 9b6ba40318..8f8b5c7768 100644 --- a/source/libs/index/src/indexTfile.c +++ b/source/libs/index/src/indexTfile.c @@ -545,8 +545,8 @@ int tfileWriterPut(TFileWriter* tw, void* data, bool order) { int8_t colType = tw->header.colType; colType = IDX_TYPE_GET_TYPE(colType); - if (colType == TSDB_DATA_TYPE_BINARY || colType == TSDB_DATA_TYPE_NCHAR || - colType == TSDB_DATA_TYPE_GEOMETRY) { + if (colType == TSDB_DATA_TYPE_BINARY || colType == TSDB_DATA_TYPE_VARBINARY || + colType == TSDB_DATA_TYPE_NCHAR || colType == TSDB_DATA_TYPE_GEOMETRY) { fn = tfileStrCompare; } else { fn = getComparFunc(colType, 0); diff --git a/source/libs/parser/src/parInsertSml.c b/source/libs/parser/src/parInsertSml.c index 78b05b6df5..9bb559d441 100644 --- a/source/libs/parser/src/parInsertSml.c +++ b/source/libs/parser/src/parInsertSml.c @@ -134,7 +134,7 @@ static int32_t smlBuildTagRow(SArray* cols, SBoundColInfo* tags, SSchema* pSchem taosArrayPush(*tagName, pTagSchema->name); STagVal val = {.cid = pTagSchema->colId, .type = pTagSchema->type}; // strcpy(val.colName, pTagSchema->name); - if (pTagSchema->type == TSDB_DATA_TYPE_BINARY || + if (pTagSchema->type == TSDB_DATA_TYPE_BINARY || pTagSchema->type == TSDB_DATA_TYPE_VARBINARY || pTagSchema->type == TSDB_DATA_TYPE_GEOMETRY) { val.pData = (uint8_t*)kv->value; val.nData = kv->length; @@ -237,7 +237,7 @@ int32_t smlBuildCol(STableDataCxt* pTableCxt, SSchema* schema, void* data, int32 } pVal->value.pData = pUcs4; pVal->value.nData = len; - } else if (kv->type == TSDB_DATA_TYPE_BINARY || kv->type == TSDB_DATA_TYPE_GEOMETRY) { + } else if (kv->type == TSDB_DATA_TYPE_BINARY || kv->type == TSDB_DATA_TYPE_VARBINARY || kv->type == TSDB_DATA_TYPE_GEOMETRY) { pVal->value.nData = kv->length; pVal->value.pData = (uint8_t*)kv->value; } else { @@ -364,7 +364,7 @@ int32_t smlBindData(SQuery* query, bool dataFormat, SArray* tags, SArray* colsSc } pVal->value.pData = pUcs4; pVal->value.nData = len; - } else if (kv->type == TSDB_DATA_TYPE_BINARY || kv->type == TSDB_DATA_TYPE_GEOMETRY) { + } else if (kv->type == TSDB_DATA_TYPE_BINARY || kv->type == TSDB_DATA_TYPE_VARBINARY || kv->type == TSDB_DATA_TYPE_GEOMETRY) { pVal->value.nData = kv->length; pVal->value.pData = (uint8_t*)kv->value; } else { diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index 8d35674949..1000d670d8 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -477,7 +477,15 @@ static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema, val->nData = pToken->n; break; } - + case TSDB_DATA_TYPE_VARBINARY: { + // Too long values will raise the invalid sql error message + if (pToken->n + VARSTR_HEADER_SIZE > pSchema->bytes) { + return generateSyntaxErrMsg(pMsgBuf, TSDB_CODE_PAR_VALUE_TOO_LONG, pSchema->name); + } + val->pData = taosStrdup(pToken->z) + val->nData = pToken->n; + break; + } case TSDB_DATA_TYPE_GEOMETRY: { unsigned char* output = NULL; size_t size = 0; @@ -1364,6 +1372,19 @@ static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql, pVal->value.nData = pToken->n; break; } + case TSDB_DATA_TYPE_VARBINARY: { + // Too long values will raise the invalid sql error message + if (pToken->n + VARSTR_HEADER_SIZE > pSchema->bytes) { + return generateSyntaxErrMsg(&pCxt->msg, TSDB_CODE_PAR_VALUE_TOO_LONG, pSchema->name); + } + pVal->value.pData = taosMemoryMalloc(pToken->n); + if (NULL == pVal->value.pData) { + return TSDB_CODE_OUT_OF_MEMORY; + } + memcpy(pVal->value.pData, pToken->z, pToken->n); + pVal->value.nData = pToken->n + break; + } case TSDB_DATA_TYPE_NCHAR: { // if the converted output len is over than pColumnModel->bytes, return error: 'Argument list too long' int32_t len = 0; diff --git a/source/libs/parser/src/parInsertStmt.c b/source/libs/parser/src/parInsertStmt.c index bcbea6cd2e..5137deca2e 100644 --- a/source/libs/parser/src/parInsertStmt.c +++ b/source/libs/parser/src/parInsertStmt.c @@ -128,7 +128,7 @@ int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, const ch } else { STagVal val = {.cid = pTagSchema->colId, .type = pTagSchema->type}; // strcpy(val.colName, pTagSchema->name); - if (pTagSchema->type == TSDB_DATA_TYPE_BINARY || + if (pTagSchema->type == TSDB_DATA_TYPE_BINARY || pTagSchema->type == TSDB_DATA_TYPE_VARBINARY || pTagSchema->type == TSDB_DATA_TYPE_GEOMETRY) { val.pData = (uint8_t*)bind[c].buffer; val.nData = colLen; diff --git a/source/libs/parser/src/parTokenizer.c b/source/libs/parser/src/parTokenizer.c index ca7ac1a0b6..41f6239455 100644 --- a/source/libs/parser/src/parTokenizer.c +++ b/source/libs/parser/src/parTokenizer.c @@ -285,6 +285,7 @@ static SKeyword keywordTable[] = { {"_WEND", TK_WEND}, {"_WSTART", TK_WSTART}, {"ALIVE", TK_ALIVE}, + {"VARBINARY", TK_VARBINARY}, }; // clang-format on diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 38118c03f8..0ca5ae01b5 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1302,7 +1302,7 @@ static EDealRes translateValueImpl(STranslateContext* pCxt, SValueNode* pVal, SD } static int32_t calcTypeBytes(SDataType dt) { - if (TSDB_DATA_TYPE_BINARY == dt.type || TSDB_DATA_TYPE_GEOMETRY == dt.type) { + if (TSDB_DATA_TYPE_BINARY == dt.type || TSDB_DATA_TYPE_VARBINARY == dt.type || TSDB_DATA_TYPE_GEOMETRY == dt.type) { return dt.bytes + VARSTR_HEADER_SIZE; } else if (TSDB_DATA_TYPE_NCHAR == dt.type) { return dt.bytes * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE; diff --git a/source/libs/qcom/src/queryUtil.c b/source/libs/qcom/src/queryUtil.c index 38d7c9da3b..e245b6f273 100644 --- a/source/libs/qcom/src/queryUtil.c +++ b/source/libs/qcom/src/queryUtil.c @@ -44,7 +44,7 @@ static bool doValidateSchema(SSchema* pSchema, int32_t numOfCols, int32_t maxLen } // 2. valid length for each type - if (pSchema[i].type == TSDB_DATA_TYPE_BINARY) { + if (pSchema[i].type == TSDB_DATA_TYPE_BINARY || pSchema[i].type == TSDB_DATA_TYPE_VARBINARY) { if (pSchema[i].bytes > TSDB_MAX_BINARY_LEN) { return false; } @@ -300,6 +300,8 @@ int32_t dataConverToStr(char* str, int type, void* buf, int32_t bufSize, int32_t n = sprintf(str, "%e", GET_DOUBLE_VAL(buf)); break; + case TSDB_DATA_TYPE_VARBINARY: + sd; case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_GEOMETRY: if (bufSize < 0) { diff --git a/source/libs/scalar/inc/filterInt.h b/source/libs/scalar/inc/filterInt.h index 5fb7b0e90c..4296abb588 100644 --- a/source/libs/scalar/inc/filterInt.h +++ b/source/libs/scalar/inc/filterInt.h @@ -272,7 +272,7 @@ struct SFilterInfo { }; #define FILTER_NO_MERGE_DATA_TYPE(t) \ - ((t) == TSDB_DATA_TYPE_BINARY || (t) == TSDB_DATA_TYPE_NCHAR || (t) == TSDB_DATA_TYPE_JSON || \ + ((t) == TSDB_DATA_TYPE_BINARY || (t) == TSDB_DATA_TYPE_VARBINARY || (t) == TSDB_DATA_TYPE_NCHAR || (t) == TSDB_DATA_TYPE_JSON || \ (t) == TSDB_DATA_TYPE_GEOMETRY) #define FILTER_NO_MERGE_OPTR(o) ((o) == OP_TYPE_IS_NULL || (o) == OP_TYPE_IS_NOT_NULL || (o) == FILTER_DUMMY_EMPTY_OPTR) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 892fd588b6..e887bb8888 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -180,7 +180,8 @@ __compar_fn_t gUint64UsignCompare[] = {compareUint64Uint8, compareUint64Uint16, int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) { int8_t comparFn = 0; - if (optr == OP_TYPE_IN && (type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR && type != TSDB_DATA_TYPE_GEOMETRY)) { + if (optr == OP_TYPE_IN && (type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_VARBINARY && + type != TSDB_DATA_TYPE_NCHAR && type != TSDB_DATA_TYPE_GEOMETRY)) { switch (type) { case TSDB_DATA_TYPE_BOOL: case TSDB_DATA_TYPE_TINYINT: @@ -257,6 +258,7 @@ int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) { case TSDB_DATA_TYPE_DOUBLE: comparFn = 5; break; + case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_BINARY: { if (optr == OP_TYPE_MATCH) { comparFn = 19; @@ -466,7 +468,7 @@ static FORCE_INLINE SFilterRangeNode *filterNewRange(SFilterRangeCtx *ctx, SFilt void *filterInitRangeCtx(int32_t type, int32_t options) { if (type > TSDB_DATA_TYPE_UBIGINT || type < TSDB_DATA_TYPE_BOOL || - type == TSDB_DATA_TYPE_BINARY || + type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_VARBINARY || type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_GEOMETRY) { qError("not supported range type:%d", type); return NULL; @@ -1584,6 +1586,7 @@ int32_t fltConverToStr(char *str, int type, void *buf, int32_t bufSize, int32_t n = sprintf(str, "%e", GET_DOUBLE_VAL(buf)); break; + case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_NCHAR: case TSDB_DATA_TYPE_GEOMETRY: diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index 0246724c5b..96a18e56d2 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -190,55 +190,6 @@ _getBigintValue_fn_t getVectorBigintValueFn(int32_t srcType) { return p; } -typedef void *(*_getValueAddr_fn_t)(void *src, int32_t index); - -void *getVectorValueAddr_TINYINT(void *src, int32_t index) { return (void *)((int8_t *)src + index); } -void *getVectorValueAddr_UTINYINT(void *src, int32_t index) { return (void *)((uint8_t *)src + index); } -void *getVectorValueAddr_SMALLINT(void *src, int32_t index) { return (void *)((int16_t *)src + index); } -void *getVectorValueAddr_USMALLINT(void *src, int32_t index) { return (void *)((uint16_t *)src + index); } -void *getVectorValueAddr_INT(void *src, int32_t index) { return (void *)((int32_t *)src + index); } -void *getVectorValueAddr_UINT(void *src, int32_t index) { return (void *)((uint32_t *)src + index); } -void *getVectorValueAddr_BIGINT(void *src, int32_t index) { return (void *)((int64_t *)src + index); } -void *getVectorValueAddr_UBIGINT(void *src, int32_t index) { return (void *)((uint64_t *)src + index); } -void *getVectorValueAddr_FLOAT(void *src, int32_t index) { return (void *)((float *)src + index); } -void *getVectorValueAddr_DOUBLE(void *src, int32_t index) { return (void *)((double *)src + index); } -void *getVectorValueAddr_default(void *src, int32_t index) { return src; } -void *getVectorValueAddr_VAR(void *src, int32_t index) { return colDataGetData((SColumnInfoData *)src, index); } - -_getValueAddr_fn_t getVectorValueAddrFn(int32_t srcType) { - _getValueAddr_fn_t p = NULL; - if (srcType == TSDB_DATA_TYPE_TINYINT) { - p = getVectorValueAddr_TINYINT; - } else if (srcType == TSDB_DATA_TYPE_UTINYINT) { - p = getVectorValueAddr_UTINYINT; - } else if (srcType == TSDB_DATA_TYPE_SMALLINT) { - p = getVectorValueAddr_SMALLINT; - } else if (srcType == TSDB_DATA_TYPE_USMALLINT) { - p = getVectorValueAddr_USMALLINT; - } else if (srcType == TSDB_DATA_TYPE_INT) { - p = getVectorValueAddr_INT; - } else if (srcType == TSDB_DATA_TYPE_UINT) { - p = getVectorValueAddr_UINT; - } else if (srcType == TSDB_DATA_TYPE_BIGINT) { - p = getVectorValueAddr_BIGINT; - } else if (srcType == TSDB_DATA_TYPE_UBIGINT) { - p = getVectorValueAddr_UBIGINT; - } else if (srcType == TSDB_DATA_TYPE_FLOAT) { - p = getVectorValueAddr_FLOAT; - } else if (srcType == TSDB_DATA_TYPE_DOUBLE) { - p = getVectorValueAddr_DOUBLE; - } else if (srcType == TSDB_DATA_TYPE_BINARY) { - p = getVectorValueAddr_VAR; - } else if (srcType == TSDB_DATA_TYPE_NCHAR) { - p = getVectorValueAddr_VAR; - }else if(srcType == TSDB_DATA_TYPE_GEOMETRY) { - p = getVectorValueAddr_VAR; - } else { - p = getVectorValueAddr_default; - } - return p; -} - static FORCE_INLINE void varToTimestamp(char *buf, SScalarParam *pOut, int32_t rowIndex, int32_t *overflow) { terrno = TSDB_CODE_SUCCESS; @@ -597,7 +548,8 @@ bool convertJsonValue(__compar_fn_t *fp, int32_t optr, int8_t typeLeft, int8_t t } if (optr == OP_TYPE_LIKE || optr == OP_TYPE_NOT_LIKE || optr == OP_TYPE_MATCH || optr == OP_TYPE_NMATCH) { - if (typeLeft != TSDB_DATA_TYPE_NCHAR && typeLeft != TSDB_DATA_TYPE_BINARY && typeLeft != TSDB_DATA_TYPE_GEOMETRY) { + if (typeLeft != TSDB_DATA_TYPE_NCHAR && typeLeft != TSDB_DATA_TYPE_BINARY && + typeLeft != TSDB_DATA_TYPE_GEOMETRY && typeLeft != TSDB_DATA_TYPE_VARBINARY) { return false; } } @@ -636,13 +588,16 @@ bool convertJsonValue(__compar_fn_t *fp, int32_t optr, int8_t typeLeft, int8_t t if (typeRight == TSDB_DATA_TYPE_NCHAR || typeRight == TSDB_DATA_TYPE_VARCHAR || + typeRight == TSDB_DATA_TYPE_VARBINARY || typeRight == TSDB_DATA_TYPE_GEOMETRY) { return false; } else if (typeRight != type) { convertNumberToNumber(*pRightData, pRightOut, typeRight, type); *pRightData = pRightOut; } - } else if (type == TSDB_DATA_TYPE_BINARY || typeLeft == TSDB_DATA_TYPE_GEOMETRY) { + } else if (type == TSDB_DATA_TYPE_BINARY || + type == TSDB_DATA_TYPE_VARBINARY || + type == TSDB_DATA_TYPE_GEOMETRY) { if (typeLeft == TSDB_DATA_TYPE_NCHAR) { *pLeftData = ncharTobinary(*pLeftData); *freeLeft = true; @@ -932,6 +887,7 @@ int32_t vectorConvertSingleColImpl(const SScalarParam *pIn, SScalarParam *pOut, break; } case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_VARBINARY case TSDB_DATA_TYPE_NCHAR: case TSDB_DATA_TYPE_GEOMETRY: { return vectorConvertToVarData(&cCtx); diff --git a/source/util/src/tcompare.c b/source/util/src/tcompare.c index 843f9c56dc..7ebe32dfaf 100644 --- a/source/util/src/tcompare.c +++ b/source/util/src/tcompare.c @@ -1301,7 +1301,8 @@ int32_t comparewcsPatternNMatch(const void *pLeft, const void *pRight) { __compar_fn_t getComparFunc(int32_t type, int32_t optr) { __compar_fn_t comparFn = NULL; - if (optr == OP_TYPE_IN && (type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR && type != TSDB_DATA_TYPE_GEOMETRY)) { + if (optr == OP_TYPE_IN && (type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_VARBINARY && + type != TSDB_DATA_TYPE_NCHAR && type != TSDB_DATA_TYPE_GEOMETRY)) { switch (type) { case TSDB_DATA_TYPE_BOOL: case TSDB_DATA_TYPE_TINYINT: @@ -1324,7 +1325,8 @@ __compar_fn_t getComparFunc(int32_t type, int32_t optr) { } } - if (optr == OP_TYPE_NOT_IN && (type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR && type != TSDB_DATA_TYPE_GEOMETRY)) { + if (optr == OP_TYPE_NOT_IN && (type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_VARBINARY && + type != TSDB_DATA_TYPE_NCHAR && type != TSDB_DATA_TYPE_GEOMETRY)) { switch (type) { case TSDB_DATA_TYPE_BOOL: case TSDB_DATA_TYPE_TINYINT: @@ -1369,6 +1371,7 @@ __compar_fn_t getComparFunc(int32_t type, int32_t optr) { comparFn = compareDoubleVal; break; case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_VARBINARY case TSDB_DATA_TYPE_GEOMETRY: { if (optr == OP_TYPE_MATCH) { comparFn = comparestrRegexMatch; @@ -1454,6 +1457,7 @@ __compar_fn_t getKeyComparFunc(int32_t keyType, int32_t order) { case TSDB_DATA_TYPE_UBIGINT: return (order == TSDB_ORDER_ASC) ? compareUint64Val : compareUint64ValDesc; case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_VARBINARY case TSDB_DATA_TYPE_GEOMETRY: return (order == TSDB_ORDER_ASC) ? compareLenPrefixedStr : compareLenPrefixedStrDesc; case TSDB_DATA_TYPE_NCHAR: diff --git a/source/util/src/thashutil.c b/source/util/src/thashutil.c index 891c09cfb5..9d65977ff1 100644 --- a/source/util/src/thashutil.c +++ b/source/util/src/thashutil.c @@ -193,6 +193,7 @@ _hash_fn_t taosGetDefaultHashFunction(int32_t type) { fn = taosIntHash_64; break; case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_VARBINARY: case TSDB_DATA_TYPE_NCHAR: case TSDB_DATA_TYPE_GEOMETRY: fn = MurmurHash3_32; diff --git a/source/util/src/tskiplist.c b/source/util/src/tskiplist.c index 6344af523f..eadd9a2413 100644 --- a/source/util/src/tskiplist.c +++ b/source/util/src/tskiplist.c @@ -376,6 +376,7 @@ void tSkipListPrint(SSkipList *pSkipList, int16_t nlevel) { fprintf(stdout, "%d: %" PRId64 " \n", id++, *(int64_t *)key); break; case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_VARBINARY: case TSDB_DATA_TYPE_GEOMETRY: fprintf(stdout, "%d: %s \n", id++, key); break; diff --git a/tests/script/api/batchprepare.c b/tests/script/api/batchprepare.c index 604d6ade89..0e3b50974a 100644 --- a/tests/script/api/batchprepare.c +++ b/tests/script/api/batchprepare.c @@ -1222,6 +1222,7 @@ int32_t bpAppendValueString(char *buf, int type, void *value, int32_t valueLen, break; case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_VARBINARY: case TSDB_DATA_TYPE_NCHAR: case TSDB_DATA_TYPE_GEOMETRY: buf[*len] = '\''; @@ -1369,7 +1370,7 @@ void bpCheckColTagFields(TAOS_STMT *stmt, int32_t fieldNum, TAOS_FIELD_E* pField exit(1); } - if (pFields[i].type == TSDB_DATA_TYPE_BINARY || pFields[i].type == TSDB_DATA_TYPE_GEOMETRY) { + if (pFields[i].type == TSDB_DATA_TYPE_BINARY || pFields[i].type == TSDB_DATA_TYPE_VARBINARY || pFields[i].type == TSDB_DATA_TYPE_GEOMETRY) { if (pFields[i].bytes != (pBind[i].buffer_length + 2)) { printf("!!!%s %dth field len %d mis-match expect len %d\n", BP_BIND_TYPE_STR(type), i, pFields[i].bytes, (pBind[i].buffer_length + 2)); exit(1); @@ -2256,7 +2257,7 @@ int insertNonExistsTb(TAOS_STMT *stmt, TAOS *taos) { void bpAddWrongVarBuffLen(TAOS_MULTI_BIND* pBind) { for (int32_t i = 0; i < gCurCase->bindColNum; ++i) { - if (pBind[i].buffer_type == TSDB_DATA_TYPE_BINARY || pBind[i].buffer_type == TSDB_DATA_TYPE_NCHAR) { + if (pBind[i].buffer_type == TSDB_DATA_TYPE_BINARY || pBind[i].buffer_type == TSDB_DATA_TYPE_VARBINARY || pBind[i].buffer_type == TSDB_DATA_TYPE_NCHAR) { *pBind[i].length += 100; } } diff --git a/tests/script/api/passwdTest.c b/tests/script/api/passwdTest.c index d9cb2128ef..06d2d3455f 100644 --- a/tests/script/api/passwdTest.c +++ b/tests/script/api/passwdTest.c @@ -145,6 +145,7 @@ int printRow(char *str, TAOS_ROW row, TAOS_FIELD *fields, int numFields) { len += sprintf(str + len, "%lf", dv); } break; case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_VARBINARY: case TSDB_DATA_TYPE_NCHAR: case TSDB_DATA_TYPE_GEOMETRY: { int32_t charLen = varDataLen((char *)row[i] - VARSTR_HEADER_SIZE); diff --git a/tests/script/api/stmt_function.c b/tests/script/api/stmt_function.c index 55ab3f5631..d7d6f7d6f0 100644 --- a/tests/script/api/stmt_function.c +++ b/tests/script/api/stmt_function.c @@ -400,6 +400,7 @@ void taos_stmt_use_result_query(void *taos, char *col, int type) { params->length = ¶ms->buffer_length; break; case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_VARBINARY: case TSDB_DATA_TYPE_GEOMETRY: params->buffer_length = sizeof(v.c6); params->buffer = &v.c6; diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index e9dd067ac4..12a8807ed5 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -386,6 +386,7 @@ void shellDumpFieldToFile(TdFilePtr pFile, const char *val, TAOS_FIELD *field, i } break; case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_VARBINARY case TSDB_DATA_TYPE_NCHAR: case TSDB_DATA_TYPE_JSON: { @@ -637,6 +638,7 @@ void shellPrintField(const char *val, TAOS_FIELD *field, int32_t width, int32_t } break; case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_VARBINARY case TSDB_DATA_TYPE_NCHAR: case TSDB_DATA_TYPE_JSON: shellPrintNChar(val, length, width); @@ -772,6 +774,7 @@ int32_t shellCalcColWidth(TAOS_FIELD *field, int32_t precision) { return TMAX(SHELL_DOUBLE_WIDTH, width); case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_VARBINARY case TSDB_DATA_TYPE_GEOMETRY: if (field->bytes > shell.args.displayWidth) { return TMAX(shell.args.displayWidth, width); diff --git a/utils/test/c/tmqSim.c b/utils/test/c/tmqSim.c index dd6875185f..fbfacd9eda 100644 --- a/utils/test/c/tmqSim.c +++ b/utils/test/c/tmqSim.c @@ -541,6 +541,7 @@ static void shellDumpFieldToFile(TdFilePtr pFile, const char* val, TAOS_FIELD* f } break; case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_VARBINARY: case TSDB_DATA_TYPE_NCHAR: case TSDB_DATA_TYPE_JSON: case TSDB_DATA_TYPE_GEOMETRY: { diff --git a/utils/tsim/src/simExe.c b/utils/tsim/src/simExe.c index 9a0a156717..200d3e30f2 100644 --- a/utils/tsim/src/simExe.c +++ b/utils/tsim/src/simExe.c @@ -751,6 +751,7 @@ bool simExecuteNativeSqlCommand(SScript *script, char *rest, bool isSlow) { sprintf(value, "%.9lf", GET_DOUBLE_VAL(row[i])); break; case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_VARBINARY case TSDB_DATA_TYPE_NCHAR: case TSDB_DATA_TYPE_GEOMETRY: if (length[i] < 0 || length[i] > 1 << 20) { From f266d5176456e02f1959d799e7568a956f9de448 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 16 Aug 2023 19:05:07 +0800 Subject: [PATCH 02/13] feat:support varbinary type --- include/common/ttypes.h | 2 +- include/os/osString.h | 3 + source/client/src/clientImpl.c | 2 +- source/client/src/clientRawBlockWrite.c | 14 +++- source/client/src/clientSml.c | 2 +- source/common/src/tdatablock.c | 3 +- source/common/src/tdataformat.c | 1 + source/common/src/trow.c | 2 +- source/common/src/ttypes.c | 2 +- source/libs/command/src/command.c | 4 +- source/libs/executor/src/dataInserter.c | 2 +- source/libs/index/src/indexComm.c | 13 +--- source/libs/parser/src/parInsertSql.c | 57 ++++++++++++--- source/libs/parser/src/parTranslater.c | 6 ++ source/libs/qcom/src/queryUtil.c | 19 ++++- source/libs/scalar/src/filter.c | 12 +++- source/libs/scalar/src/scalar.c | 9 ++- source/libs/scalar/src/sclvector.c | 4 +- source/os/src/osString.c | 92 +++++++++++++++++++++++++ source/util/src/tcompare.c | 11 ++- tools/shell/src/shellEngine.c | 22 ++++-- 21 files changed, 234 insertions(+), 48 deletions(-) diff --git a/include/common/ttypes.h b/include/common/ttypes.h index 559983185c..279799b172 100644 --- a/include/common/ttypes.h +++ b/include/common/ttypes.h @@ -269,7 +269,7 @@ typedef struct { (IS_NUMERIC_TYPE(_t) || (_t) == (TSDB_DATA_TYPE_BOOL) || (_t) == (TSDB_DATA_TYPE_TIMESTAMP)) #define IS_VAR_DATA_TYPE(t) \ - (((t) == TSDB_DATA_TYPE_VARCHAR) || (t) == TSDB_DATA_TYPE_VARBINARY || ((t) == TSDB_DATA_TYPE_NCHAR) || ((t) == TSDB_DATA_TYPE_JSON) || ((t) == TSDB_DATA_TYPE_GEOMETRY)) + (((t) == TSDB_DATA_TYPE_VARCHAR) || ((t) == TSDB_DATA_TYPE_VARBINARY) || ((t) == TSDB_DATA_TYPE_NCHAR) || ((t) == TSDB_DATA_TYPE_JSON) || ((t) == TSDB_DATA_TYPE_GEOMETRY)) #define IS_STR_DATA_TYPE(t) (((t) == TSDB_DATA_TYPE_VARCHAR) || ((t) == TSDB_DATA_TYPE_VARBINARY) || ((t) == TSDB_DATA_TYPE_NCHAR)) #define IS_VALID_TINYINT(_t) ((_t) >= INT8_MIN && (_t) <= INT8_MAX) diff --git a/include/os/osString.h b/include/os/osString.h index b609c9d351..ed64cbfe3f 100644 --- a/include/os/osString.h +++ b/include/os/osString.h @@ -90,6 +90,9 @@ int8_t taosStr2Int8(const char *str, char **pEnd, int32_t radix); uint8_t taosStr2UInt8(const char *str, char **pEnd, int32_t radix); double taosStr2Double(const char *str, char **pEnd); float taosStr2Float(const char *str, char **pEnd); +int32_t taosHex2Ascii(const char *z, uint32_t n, void** data, uint32_t* size); +int32_t taosAscii2Hex(const char *z, uint32_t n, void** data, uint32_t* size); +int32_t taosBin2Ascii(const char *z, uint32_t n, void** data, uint32_t* size); #ifdef __cplusplus } diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index d448dd1edf..c78ba4c4a0 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -503,7 +503,7 @@ void setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t pResInfo->userFields[i].bytes = pSchema[i].bytes; pResInfo->userFields[i].type = pSchema[i].type; - if (pSchema[i].type == TSDB_DATA_TYPE_VARCHAR || pSchema[i].type == TSDB_DATA_TYPE_GEOMETRY) { + if (pSchema[i].type == TSDB_DATA_TYPE_VARCHAR || pSchema[i].type == TSDB_DATA_TYPE_VARBINARY || pSchema[i].type == TSDB_DATA_TYPE_GEOMETRY) { pResInfo->userFields[i].bytes -= VARSTR_HEADER_SIZE; } else if (pSchema[i].type == TSDB_DATA_TYPE_NCHAR || pSchema[i].type == TSDB_DATA_TYPE_JSON) { pResInfo->userFields[i].bytes = (pResInfo->userFields[i].bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE; diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c index 0b28949400..af6b3bcc8c 100644 --- a/source/client/src/clientRawBlockWrite.c +++ b/source/client/src/clientRawBlockWrite.c @@ -292,7 +292,13 @@ static void buildChildElement(cJSON* json, SVCreateTbReq* pCreateReq) { cJSON* tvalue = NULL; if (IS_VAR_DATA_TYPE(pTagVal->type)) { - char* buf = taosMemoryCalloc(pTagVal->nData + 3, 1); + char* buf = NULL; + if(pTagVal->type == TSDB_DATA_TYPE_VARBINARY){ + buf = taosMemoryCalloc(pTagVal->nData*2 + 2 + 3, 1); + }else{ + buf = taosMemoryCalloc(pTagVal->nData + 3, 1); + } + if (!buf) goto end; dataConverToStr(buf, pTagVal->type, pTagVal->pData, pTagVal->nData, NULL); tvalue = cJSON_CreateString(buf); @@ -515,7 +521,11 @@ static char* processAlterTable(SMqMetaRsp* metaRsp) { } buf = parseTagDatatoJson(vAlterTbReq.pTagVal); } else { - buf = taosMemoryCalloc(vAlterTbReq.nTagVal + 1, 1); + if(vAlterTbReq.tagType == TSDB_DATA_TYPE_VARBINARY){ + buf = taosMemoryCalloc(vAlterTbReq.nTagVal + 1, 1); + }else{ + buf = taosMemoryCalloc(vAlterTbReq.nTagVal + 1, 1); + } dataConverToStr(buf, vAlterTbReq.tagType, vAlterTbReq.pTagVal, vAlterTbReq.nTagVal, NULL); } diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 343cb9d1a3..266368e9bc 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -596,7 +596,7 @@ static int32_t smlGenerateSchemaAction(SSchema *colField, SHashObj *colHash, SSm return TSDB_CODE_SML_INVALID_DATA; } - if (((colField[*index].type == TSDB_DATA_TYPE_VARCHAR || colField[*index].type == TSDB_DATA_TYPE_GEOMETRY) && + if (((colField[*index].type == TSDB_DATA_TYPE_VARCHAR || colField[*index].type == TSDB_DATA_TYPE_VARCHAR || colField[*index].type == TSDB_DATA_TYPE_GEOMETRY) && (colField[*index].bytes - VARSTR_HEADER_SIZE) < kv->length) || (colField[*index].type == TSDB_DATA_TYPE_NCHAR && ((colField[*index].bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE < kv->length))) { diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 5188b1e27c..efbfa6e4e6 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1852,6 +1852,7 @@ char* dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** pDataBuf) if (len >= size - 1) return dumpBuf; break; case TSDB_DATA_TYPE_VARCHAR: + case TSDB_DATA_TYPE_VARBINARY: case TSDB_DATA_TYPE_GEOMETRY: { memset(pBuf, 0, sizeof(pBuf)); char* pData = colDataGetVarData(pColInfoData, j); @@ -1951,6 +1952,7 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq2** ppReq, const SSDataBlock* pDat } break; case TSDB_DATA_TYPE_NCHAR: + case TSDB_DATA_TYPE_VARBINARY: case TSDB_DATA_TYPE_VARCHAR: { // TSDB_DATA_TYPE_BINARY ASSERT(pColInfoData->info.type == pCol->type); if (colDataIsNull_s(pColInfoData, j)) { @@ -1964,7 +1966,6 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq2** ppReq, const SSDataBlock* pDat } break; } - case TSDB_DATA_TYPE_VARBINARY: case TSDB_DATA_TYPE_DECIMAL: case TSDB_DATA_TYPE_BLOB: case TSDB_DATA_TYPE_JSON: diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 7c6939635a..7b8f0e67fb 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -1146,6 +1146,7 @@ static int tTagValJsonCmprFn(const void *p1, const void *p2) { static void debugPrintTagVal(int8_t type, const void *val, int32_t vlen, const char *tag, int32_t ln) { switch (type) { + case TSDB_DATA_TYPE_VARBINARY: case TSDB_DATA_TYPE_JSON: case TSDB_DATA_TYPE_VARCHAR: case TSDB_DATA_TYPE_NCHAR: diff --git a/source/common/src/trow.c b/source/common/src/trow.c index 039f436505..e2da4d166e 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -356,7 +356,7 @@ int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow) { } } else { varDataLen += sizeof(VarDataLenT); - if (pTColumn->type == TSDB_DATA_TYPE_VARCHAR || pTColumn->type == TSDB_DATA_TYPE_GEOMETRY) { + if (pTColumn->type == TSDB_DATA_TYPE_VARCHAR || pTColumn->type == TSDB_DATA_TYPE_VARBINARY || pTColumn->type == TSDB_DATA_TYPE_GEOMETRY) { varDataLen += CHAR_BYTES; if (maxVarDataLen < CHAR_BYTES + sizeof(VarDataLenT)) { maxVarDataLen = CHAR_BYTES + sizeof(VarDataLenT); diff --git a/source/common/src/ttypes.c b/source/common/src/ttypes.c index 6acd2d087a..8827ddc811 100644 --- a/source/common/src/ttypes.c +++ b/source/common/src/ttypes.c @@ -61,7 +61,7 @@ tDataTypeDescriptor tDataTypes[TSDB_DATA_TYPE_MAX] = { {TSDB_DATA_TYPE_UINT, 12, INT_BYTES, "INT UNSIGNED", 0, UINT32_MAX, tsCompressInt, tsDecompressInt}, {TSDB_DATA_TYPE_UBIGINT, 15, LONG_BYTES, "BIGINT UNSIGNED", 0, UINT64_MAX, tsCompressBigint, tsDecompressBigint}, {TSDB_DATA_TYPE_JSON, 4, TSDB_MAX_JSON_TAG_LEN, "JSON", 0, 0, tsCompressString, tsDecompressString}, - {TSDB_DATA_TYPE_VARBINARY, 9, 1, "VARBINARY", 0, 0, NULL, NULL}, // placeholder, not implemented + {TSDB_DATA_TYPE_VARBINARY, 9, 1, "VARBINARY", 0, 0, tsCompressString, tsDecompressString}, // placeholder, not implemented {TSDB_DATA_TYPE_DECIMAL, 7, 1, "DECIMAL", 0, 0, NULL, NULL}, // placeholder, not implemented {TSDB_DATA_TYPE_BLOB, 4, 1, "BLOB", 0, 0, NULL, NULL}, // placeholder, not implemented {TSDB_DATA_TYPE_MEDIUMBLOB, 10, 1, "MEDIUMBLOB", 0, 0, NULL, NULL}, // placeholder, not implemented diff --git a/source/libs/command/src/command.c b/source/libs/command/src/command.c index 98aab26b0d..444da63a56 100644 --- a/source/libs/command/src/command.c +++ b/source/libs/command/src/command.c @@ -458,7 +458,7 @@ void appendColumnFields(char* buf, int32_t* len, STableCfg* pCfg) { SSchema* pSchema = pCfg->pSchemas + i; char type[32]; sprintf(type, "%s", tDataTypes[pSchema->type].name); - if (TSDB_DATA_TYPE_VARCHAR == pSchema->type || TSDB_DATA_TYPE_GEOMETRY == pSchema->type) { + if (TSDB_DATA_TYPE_VARCHAR == pSchema->type || TSDB_DATA_TYPE_VARBINARY == pSchema->type || TSDB_DATA_TYPE_GEOMETRY == pSchema->type) { sprintf(type + strlen(type), "(%d)", (int32_t)(pSchema->bytes - VARSTR_HEADER_SIZE)); } else if (TSDB_DATA_TYPE_NCHAR == pSchema->type) { sprintf(type + strlen(type), "(%d)", (int32_t)((pSchema->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)); @@ -473,7 +473,7 @@ void appendTagFields(char* buf, int32_t* len, STableCfg* pCfg) { SSchema* pSchema = pCfg->pSchemas + pCfg->numOfColumns + i; char type[32]; sprintf(type, "%s", tDataTypes[pSchema->type].name); - if (TSDB_DATA_TYPE_VARCHAR == pSchema->type || TSDB_DATA_TYPE_GEOMETRY == pSchema->type) { + if (TSDB_DATA_TYPE_VARCHAR == pSchema->type || TSDB_DATA_TYPE_VARBINARY == pSchema->type || TSDB_DATA_TYPE_GEOMETRY == pSchema->type) { sprintf(type + strlen(type), "(%d)", (int32_t)(pSchema->bytes - VARSTR_HEADER_SIZE)); } else if (TSDB_DATA_TYPE_NCHAR == pSchema->type) { sprintf(type + strlen(type), "(%d)", (int32_t)((pSchema->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)); diff --git a/source/libs/executor/src/dataInserter.c b/source/libs/executor/src/dataInserter.c index 646964ebf4..c65dfd97d1 100644 --- a/source/libs/executor/src/dataInserter.c +++ b/source/libs/executor/src/dataInserter.c @@ -213,6 +213,7 @@ int32_t buildSubmitReqFromBlock(SDataInserterHandle* pInserter, SSubmitReq2** pp switch (pColInfoData->info.type) { case TSDB_DATA_TYPE_NCHAR: + case TSDB_DATA_TYPE_VARBINARY: case TSDB_DATA_TYPE_VARCHAR: { // TSDB_DATA_TYPE_BINARY ASSERT(pColInfoData->info.type == pCol->type); if (colDataIsNull_s(pColInfoData, j)) { @@ -226,7 +227,6 @@ int32_t buildSubmitReqFromBlock(SDataInserterHandle* pInserter, SSubmitReq2** pp } break; } - case TSDB_DATA_TYPE_VARBINARY: case TSDB_DATA_TYPE_DECIMAL: case TSDB_DATA_TYPE_BLOB: case TSDB_DATA_TYPE_JSON: diff --git a/source/libs/index/src/indexComm.c b/source/libs/index/src/indexComm.c index 9f8c2139eb..1313221952 100644 --- a/source/libs/index/src/indexComm.c +++ b/source/libs/index/src/indexComm.c @@ -357,17 +357,13 @@ int32_t idxConvertData(void* src, int8_t type, void** dst) { break; } case TSDB_DATA_TYPE_VARCHAR: // TSDB_DATA_TYPE_BINARY + case TSDB_DATA_TYPE_VARBINARY: case TSDB_DATA_TYPE_GEOMETRY: { tlen = taosEncodeBinary(NULL, src, strlen(src)); *dst = taosMemoryCalloc(1, tlen + 1); tlen = taosEncodeBinary(dst, src, strlen(src)); break; } - case TSDB_DATA_TYPE_VARBINARY: - tlen = taosEncodeBinary(NULL, src, strlen(src)); - *dst = taosMemoryCalloc(1, tlen + 1); - tlen = taosEncodeBinary(dst, src, strlen(src)); - break; default: ASSERTS(0, "index invalid input type"); break; @@ -448,6 +444,7 @@ int32_t idxConvertDataToStr(void* src, int8_t type, void** dst) { break; } case TSDB_DATA_TYPE_VARCHAR: // TSDB_DATA_TYPE_BINARY + case TSDB_DATA_TYPE_VARBINARY: case TSDB_DATA_TYPE_GEOMETRY: { tlen = taosEncodeBinary(NULL, varDataVal(src), varDataLen(src)); *dst = taosMemoryCalloc(1, tlen + 1); @@ -455,12 +452,6 @@ int32_t idxConvertDataToStr(void* src, int8_t type, void** dst) { *dst = (char*)*dst - tlen; break; } - case TSDB_DATA_TYPE_VARBINARY: - tlen = taosEncodeBinary(NULL, varDataVal(src), varDataLen(src)); - *dst = taosMemoryCalloc(1, tlen + 1); - tlen = taosEncodeBinary(dst, varDataVal(src), varDataLen(src)); - *dst = (char*)*dst - tlen; - break; default: ASSERTS(0, "index invalid input type"); break; diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index 1000d670d8..c58647e540 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -479,11 +479,32 @@ static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema, } case TSDB_DATA_TYPE_VARBINARY: { // Too long values will raise the invalid sql error message - if (pToken->n + VARSTR_HEADER_SIZE > pSchema->bytes) { + // Too long values will raise the invalid sql error message + void* data = NULL; + uint32_t size = 0; + if (pToken->type == TK_NK_HEX){ + if(taosHex2Ascii(pToken->z, pToken->n, &data, &size) < 0){ + return TSDB_CODE_OUT_OF_MEMORY; + } + }else if(pToken->type == TK_NK_BIN){ + if(taosBin2Ascii(pToken->z, pToken->n, &data, &size) < 0){ + return TSDB_CODE_OUT_OF_MEMORY; + } + }else{ + size = pToken->n; + } + if (size + VARSTR_HEADER_SIZE > pSchema->bytes) { + if(pToken->type == TK_NK_HEX || pToken->type == TK_NK_BIN){ + taosMemoryFree(data); + } return generateSyntaxErrMsg(pMsgBuf, TSDB_CODE_PAR_VALUE_TOO_LONG, pSchema->name); } - val->pData = taosStrdup(pToken->z) - val->nData = pToken->n; + if(pToken->type == TK_NK_HEX || pToken->type == TK_NK_BIN){ + val->pData = data; + }else{ + val->pData = taosStrdup(pToken->z); + } + val->nData = size; break; } case TSDB_DATA_TYPE_GEOMETRY: { @@ -667,6 +688,7 @@ static int32_t rewriteTagCondColumnImpl(STagVal* pVal, SNode** pNode) { *(double*)&pValue->typeData = pValue->datum.d; break; case TSDB_DATA_TYPE_VARCHAR: + case TSDB_DATA_TYPE_VARBINARY: case TSDB_DATA_TYPE_NCHAR: pValue->datum.p = taosMemoryCalloc(1, pVal->nData + VARSTR_HEADER_SIZE); if (NULL == pValue->datum.p) { @@ -696,7 +718,6 @@ static int32_t rewriteTagCondColumnImpl(STagVal* pVal, SNode** pNode) { *(uint64_t*)&pValue->typeData = pValue->datum.i; break; case TSDB_DATA_TYPE_JSON: - case TSDB_DATA_TYPE_VARBINARY: case TSDB_DATA_TYPE_DECIMAL: case TSDB_DATA_TYPE_BLOB: case TSDB_DATA_TYPE_MEDIUMBLOB: @@ -1374,15 +1395,31 @@ static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql, } case TSDB_DATA_TYPE_VARBINARY: { // Too long values will raise the invalid sql error message - if (pToken->n + VARSTR_HEADER_SIZE > pSchema->bytes) { + void* data = NULL; + uint32_t size = 0; + if (pToken->type == TK_NK_HEX){ + if(taosHex2Ascii(pToken->z, pToken->n, &data, &size) < 0){ + return TSDB_CODE_OUT_OF_MEMORY; + } + }else if(pToken->type == TK_NK_BIN){ + if(taosBin2Ascii(pToken->z, pToken->n, &data, &size) < 0){ + return TSDB_CODE_OUT_OF_MEMORY; + } + }else{ + size = pToken->n; + } + if (size + VARSTR_HEADER_SIZE > pSchema->bytes) { + if(pToken->type == TK_NK_HEX || pToken->type == TK_NK_BIN){ + taosMemoryFree(data); + } return generateSyntaxErrMsg(&pCxt->msg, TSDB_CODE_PAR_VALUE_TOO_LONG, pSchema->name); } - pVal->value.pData = taosMemoryMalloc(pToken->n); - if (NULL == pVal->value.pData) { - return TSDB_CODE_OUT_OF_MEMORY; + if(pToken->type == TK_NK_HEX || pToken->type == TK_NK_BIN){ + pVal->value.pData = data; + }else{ + pVal->value.pData = taosStrdup(pToken->z); } - memcpy(pVal->value.pData, pToken->z, pToken->n); - pVal->value.nData = pToken->n + pVal->value.nData = size; break; } case TSDB_DATA_TYPE_NCHAR: { diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 0ca5ae01b5..72d8799a85 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -4784,6 +4784,7 @@ static int32_t checkTableTagsSchema(STranslateContext* pCxt, SHashObj* pHash, SN } if (TSDB_CODE_SUCCESS == code) { if ((TSDB_DATA_TYPE_VARCHAR == pTag->dataType.type && calcTypeBytes(pTag->dataType) > TSDB_MAX_TAGS_LEN) || + (TSDB_DATA_TYPE_VARBINARY == pTag->dataType.type && calcTypeBytes(pTag->dataType) > TSDB_MAX_TAGS_LEN) || (TSDB_DATA_TYPE_NCHAR == pTag->dataType.type && calcTypeBytes(pTag->dataType) > TSDB_MAX_TAGS_LEN) || (TSDB_DATA_TYPE_GEOMETRY == pTag->dataType.type && calcTypeBytes(pTag->dataType) > TSDB_MAX_TAGS_LEN)) { code = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN); @@ -4836,6 +4837,7 @@ static int32_t checkTableColsSchema(STranslateContext* pCxt, SHashObj* pHash, in } if (TSDB_CODE_SUCCESS == code) { if ((TSDB_DATA_TYPE_VARCHAR == pCol->dataType.type && calcTypeBytes(pCol->dataType) > TSDB_MAX_BINARY_LEN) || + (TSDB_DATA_TYPE_VARBINARY == pCol->dataType.type && calcTypeBytes(pCol->dataType) > TSDB_MAX_BINARY_LEN) || (TSDB_DATA_TYPE_NCHAR == pCol->dataType.type && calcTypeBytes(pCol->dataType) > TSDB_MAX_NCHAR_LEN) || (TSDB_DATA_TYPE_GEOMETRY == pCol->dataType.type && calcTypeBytes(pCol->dataType) > TSDB_MAX_GEOMETRY_LEN)) { code = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN); @@ -5532,6 +5534,7 @@ static int32_t checkAlterSuperTableBySchema(STranslateContext* pCxt, SAlterTable if (TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES == pStmt->alterType) { if ((TSDB_DATA_TYPE_VARCHAR == pStmt->dataType.type && calcTypeBytes(pStmt->dataType) > TSDB_MAX_BINARY_LEN) || + (TSDB_DATA_TYPE_VARBINARY == pStmt->dataType.type && calcTypeBytes(pStmt->dataType) > TSDB_MAX_BINARY_LEN) || (TSDB_DATA_TYPE_NCHAR == pStmt->dataType.type && calcTypeBytes(pStmt->dataType) > TSDB_MAX_NCHAR_LEN)) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN); } @@ -5558,6 +5561,7 @@ static int32_t checkAlterSuperTableBySchema(STranslateContext* pCxt, SAlterTable } if ((TSDB_DATA_TYPE_VARCHAR == pStmt->dataType.type && calcTypeBytes(pStmt->dataType) > TSDB_MAX_BINARY_LEN) || + (TSDB_DATA_TYPE_VARBINARY == pStmt->dataType.type && calcTypeBytes(pStmt->dataType) > TSDB_MAX_BINARY_LEN) || (TSDB_DATA_TYPE_NCHAR == pStmt->dataType.type && calcTypeBytes(pStmt->dataType) > TSDB_MAX_NCHAR_LEN)) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN); } @@ -8841,6 +8845,7 @@ static int32_t buildAddColReq(STranslateContext* pCxt, SAlterTableStmt* pStmt, S } if ((TSDB_DATA_TYPE_VARCHAR == pStmt->dataType.type && calcTypeBytes(pStmt->dataType) > TSDB_MAX_BINARY_LEN) || + (TSDB_DATA_TYPE_VARBINARY == pStmt->dataType.type && calcTypeBytes(pStmt->dataType) > TSDB_MAX_BINARY_LEN) || (TSDB_DATA_TYPE_NCHAR == pStmt->dataType.type && calcTypeBytes(pStmt->dataType) > TSDB_MAX_NCHAR_LEN)) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN); } @@ -8898,6 +8903,7 @@ static int32_t buildUpdateColReq(STranslateContext* pCxt, SAlterTableStmt* pStmt } if ((TSDB_DATA_TYPE_VARCHAR == pStmt->dataType.type && calcTypeBytes(pStmt->dataType) > TSDB_MAX_BINARY_LEN) || + (TSDB_DATA_TYPE_VARBINARY == pStmt->dataType.type && calcTypeBytes(pStmt->dataType) > TSDB_MAX_BINARY_LEN) || (TSDB_DATA_TYPE_NCHAR == pStmt->dataType.type && calcTypeBytes(pStmt->dataType) > TSDB_MAX_NCHAR_LEN)) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN); } diff --git a/source/libs/qcom/src/queryUtil.c b/source/libs/qcom/src/queryUtil.c index e245b6f273..5c9c3da5f4 100644 --- a/source/libs/qcom/src/queryUtil.c +++ b/source/libs/qcom/src/queryUtil.c @@ -300,8 +300,23 @@ int32_t dataConverToStr(char* str, int type, void* buf, int32_t bufSize, int32_t n = sprintf(str, "%e", GET_DOUBLE_VAL(buf)); break; - case TSDB_DATA_TYPE_VARBINARY: - sd; + case TSDB_DATA_TYPE_VARBINARY:{ + if (bufSize < 0) { + // tscError("invalid buf size"); + return TSDB_CODE_TSC_INVALID_VALUE; + } + void* data = NULL; + uint32_t size = 0; + if(taosAscii2Hex(buf, bufSize, &data, &size) < 0){ + return TSDB_CODE_OUT_OF_MEMORY; + } + *str = '"'; + memcpy(str + 1, data, size); + *(str + size + 1) = '"'; + n = size + 2; + taosMemoryFree(data); + break; + } case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_GEOMETRY: if (bufSize < 0) { diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index e887bb8888..0efe335c3d 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -207,7 +207,7 @@ int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) { } } - if (optr == OP_TYPE_NOT_IN && (type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR && type != TSDB_DATA_TYPE_GEOMETRY)) { + if (optr == OP_TYPE_NOT_IN && (type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_VARBINARY && type != TSDB_DATA_TYPE_NCHAR && type != TSDB_DATA_TYPE_GEOMETRY)) { switch (type) { case TSDB_DATA_TYPE_BOOL: case TSDB_DATA_TYPE_TINYINT: @@ -258,7 +258,15 @@ int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) { case TSDB_DATA_TYPE_DOUBLE: comparFn = 5; break; - case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_VARBINARY:{ + if (optr == OP_TYPE_IN) { + comparFn = 8; + } else if (optr == OP_TYPE_NOT_IN) { + comparFn = 25; + } else { /* normal relational comparFn */ + comparFn = 30; + } + } case TSDB_DATA_TYPE_BINARY: { if (optr == OP_TYPE_MATCH) { comparFn = 19; diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index 4eb0f0e1bc..b3f67f2c9d 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -1594,6 +1594,8 @@ static int32_t sclGetMathOperatorResType(SOperatorNode *pOp) { SDataType rdt = ((SExprNode *)(pOp->pRight))->resType; if ((TSDB_DATA_TYPE_TIMESTAMP == ldt.type && TSDB_DATA_TYPE_TIMESTAMP == rdt.type) || + TSDB_DATA_TYPE_VARBINARY == ldt.type || + TSDB_DATA_TYPE_VARBINARY == rdt.type || (TSDB_DATA_TYPE_TIMESTAMP == ldt.type && (IS_VAR_DATA_TYPE(rdt.type) || IS_FLOAT_TYPE(rdt.type))) || (TSDB_DATA_TYPE_TIMESTAMP == rdt.type && (IS_VAR_DATA_TYPE(ldt.type) || IS_FLOAT_TYPE(ldt.type)))) { return TSDB_CODE_TSC_INVALID_OPERATION; @@ -1629,7 +1631,7 @@ static int32_t sclGetCompOperatorResType(SOperatorNode *pOp) { return TSDB_CODE_TSC_INVALID_OPERATION; } SDataType rdt = ((SExprNode *)(pOp->pRight))->resType; - if (!IS_VAR_DATA_TYPE(ldt.type) || QUERY_NODE_VALUE != nodeType(pOp->pRight) || + if (ldt.type == TSDB_DATA_TYPE_VARBINARY || !IS_VAR_DATA_TYPE(ldt.type) || QUERY_NODE_VALUE != nodeType(pOp->pRight) || (!IS_STR_DATA_TYPE(rdt.type) && (rdt.type != TSDB_DATA_TYPE_NULL))) { return TSDB_CODE_TSC_INVALID_OPERATION; } @@ -1660,6 +1662,11 @@ static int32_t sclGetJsonOperatorResType(SOperatorNode *pOp) { } static int32_t sclGetBitwiseOperatorResType(SOperatorNode *pOp) { + SDataType ldt = ((SExprNode *)(pOp->pLeft))->resType; + SDataType rdt = ((SExprNode *)(pOp->pRight))->resType; + if(TSDB_DATA_TYPE_VARBINARY == ldt.type || TSDB_DATA_TYPE_VARBINARY == rdt.type){ + return TSDB_CODE_TSC_INVALID_OPERATION; + } pOp->node.resType.type = TSDB_DATA_TYPE_BIGINT; pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes; return TSDB_CODE_SUCCESS; diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index 96a18e56d2..ea637fd730 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -588,7 +588,6 @@ bool convertJsonValue(__compar_fn_t *fp, int32_t optr, int8_t typeLeft, int8_t t if (typeRight == TSDB_DATA_TYPE_NCHAR || typeRight == TSDB_DATA_TYPE_VARCHAR || - typeRight == TSDB_DATA_TYPE_VARBINARY || typeRight == TSDB_DATA_TYPE_GEOMETRY) { return false; } else if (typeRight != type) { @@ -596,7 +595,6 @@ bool convertJsonValue(__compar_fn_t *fp, int32_t optr, int8_t typeLeft, int8_t t *pRightData = pRightOut; } } else if (type == TSDB_DATA_TYPE_BINARY || - type == TSDB_DATA_TYPE_VARBINARY || type == TSDB_DATA_TYPE_GEOMETRY) { if (typeLeft == TSDB_DATA_TYPE_NCHAR) { *pLeftData = ncharTobinary(*pLeftData); @@ -1078,7 +1076,7 @@ static SColumnInfoData *vectorConvertVarToDouble(SScalarParam *pInput, int32_t * SScalarParam output = {0}; SColumnInfoData *pCol = pInput->columnData; - if (IS_VAR_DATA_TYPE(pCol->info.type) && pCol->info.type != TSDB_DATA_TYPE_JSON) { + if (IS_VAR_DATA_TYPE(pCol->info.type) && pCol->info.type != TSDB_DATA_TYPE_JSON && pCol->info.type != TSDB_DATA_TYPE_VARBINARY) { int32_t code = vectorConvertSingleCol(pInput, &output, TSDB_DATA_TYPE_DOUBLE, -1, -1); if (code != TSDB_CODE_SUCCESS) { terrno = code; diff --git a/source/os/src/osString.c b/source/os/src/osString.c index 0f459b58cd..cb01332525 100644 --- a/source/os/src/osString.c +++ b/source/os/src/osString.c @@ -468,3 +468,95 @@ float taosStr2Float(const char *str, char **pEnd) { #endif return tmp; } + +int32_t taosHex2Ascii(const char *z, uint32_t n, void** data, uint32_t* size){ + n -= 2; // remove 0x + z += 2; + *size = n%2 == 0 ? n/2 : n/2 + 1; + uint8_t* tmp = (uint8_t*)taosMemoryCalloc(*size, 1); + if(tmp == NULL) return -1; + int8_t num = 0; + uint8_t *byte = tmp + *size - 1; + + for (int i = n - 1; i >= 0; i--) { + if (z[i] >= 'a') { + *byte |= ((uint8_t)(10 + (z[i] - 'a')) << (num * 4)); + } else if (z[i] >= 'A') { + *byte |= ((uint8_t)(10 + (z[i] - 'A')) << (num * 4)); + } else { + *byte |= ((uint8_t)(z[i] - '0') << (num * 4)); + } + if (num == 1) { + byte--; + num = 0; + } else { + num++; + } + } + *data = tmp; + return 0; +} + +int32_t taosBin2Ascii(const char *z, uint32_t n, void** data, uint32_t* size){ + n -= 2; // remove 0b + z += 2; + *size = n%8 == 0 ? n/8 : n/8 + 1; + uint8_t* tmp = (uint8_t*)taosMemoryCalloc(*size, 1); + if(tmp == NULL) return -1; + int8_t num = 0; + uint8_t *byte = tmp + *size - 1; + + for (int i = n - 1; i >= 0; i--) { + *byte |= ((uint8_t)(z[i] - '0') << num); + if (num == 8) { + byte--; + num = 0; + } else { + num++; + } + } + *data = tmp; + return 0; +} + +static char valueOf(uint8_t symbol) +{ + switch(symbol) + { + case 0: return '0'; + case 1: return '1'; + case 2: return '2'; + case 3: return '3'; + case 4: return '4'; + case 5: return '5'; + case 6: return '6'; + case 7: return '7'; + case 8: return '8'; + case 9: return '9'; + case 10: return 'A'; + case 11: return 'B'; + case 12: return 'C'; + case 13: return 'D'; + case 14: return 'E'; + case 15: return 'F'; + default: + { + return -1; + } + } +} + +int32_t taosAscii2Hex(const char *z, uint32_t n, void** data, uint32_t* size){ + *size = n * 2 + 2; + uint8_t* tmp = (uint8_t*)taosMemoryCalloc(*size, 1); + if(tmp == NULL) return -1; + *data = tmp; + *(tmp++) = '0'; + *(tmp++) = 'X'; + for(int i = 0; i < n; i ++){ + uint8_t val = z[i]; + tmp[i*2] = valueOf(val >> 4); + tmp[i*2 + 1] = valueOf(val & 0x0F); + } + return 0; +} diff --git a/source/util/src/tcompare.c b/source/util/src/tcompare.c index 7ebe32dfaf..9e2fc74366 100644 --- a/source/util/src/tcompare.c +++ b/source/util/src/tcompare.c @@ -1370,8 +1370,16 @@ __compar_fn_t getComparFunc(int32_t type, int32_t optr) { case TSDB_DATA_TYPE_DOUBLE: comparFn = compareDoubleVal; break; + case TSDB_DATA_TYPE_VARBINARY: + if (optr == OP_TYPE_IN) { + comparFn = compareChkInString; + } else if (optr == OP_TYPE_NOT_IN) { + comparFn = compareChkNotInString; + } else { /* normal relational comparFn */ + comparFn = compareLenBinaryVal; + } + break; case TSDB_DATA_TYPE_BINARY: - case TSDB_DATA_TYPE_VARBINARY case TSDB_DATA_TYPE_GEOMETRY: { if (optr == OP_TYPE_MATCH) { comparFn = comparestrRegexMatch; @@ -1457,7 +1465,6 @@ __compar_fn_t getKeyComparFunc(int32_t keyType, int32_t order) { case TSDB_DATA_TYPE_UBIGINT: return (order == TSDB_ORDER_ASC) ? compareUint64Val : compareUint64ValDesc; case TSDB_DATA_TYPE_BINARY: - case TSDB_DATA_TYPE_VARBINARY case TSDB_DATA_TYPE_GEOMETRY: return (order == TSDB_ORDER_ASC) ? compareLenPrefixedStr : compareLenPrefixedStrDesc; case TSDB_DATA_TYPE_NCHAR: diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 12a8807ed5..6a42c208c7 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -581,8 +581,9 @@ void shellPrintField(const char *val, TAOS_FIELD *field, int32_t width, int32_t return; } - int n; - char buf[TSDB_MAX_BYTES_PER_ROW]; + int n = 0; +#define LENGTH 64 + char buf[LENGTH] = {0}; switch (field->type) { case TSDB_DATA_TYPE_BOOL: shellPrintString(((((int32_t)(*((char *)val))) == 1) ? "true" : "false"), width); @@ -615,7 +616,7 @@ void shellPrintField(const char *val, TAOS_FIELD *field, int32_t width, int32_t if (tsEnableScience) { printf("%*.7e",width,GET_FLOAT_VAL(val)); } else { - n = snprintf(buf, TSDB_MAX_BYTES_PER_ROW, "%*.7f", width, GET_FLOAT_VAL(val)); + n = snprintf(buf, LENGTH, "%*.7f", width, GET_FLOAT_VAL(val)); if (n > SHELL_FLOAT_WIDTH) { printf("%*.7e", width,GET_FLOAT_VAL(val)); @@ -626,10 +627,10 @@ void shellPrintField(const char *val, TAOS_FIELD *field, int32_t width, int32_t break; case TSDB_DATA_TYPE_DOUBLE: if (tsEnableScience) { - snprintf(buf, TSDB_MAX_BYTES_PER_ROW, "%*.15e", width,GET_DOUBLE_VAL(val)); + snprintf(buf, LENGTH, "%*.15e", width,GET_DOUBLE_VAL(val)); printf("%s", buf); } else { - n = snprintf(buf, TSDB_MAX_BYTES_PER_ROW, "%*.15f", width, GET_DOUBLE_VAL(val)); + n = snprintf(buf, LENGTH, "%*.15f", width, GET_DOUBLE_VAL(val)); if (n > SHELL_DOUBLE_WIDTH) { printf("%*.15e", width, GET_DOUBLE_VAL(val)); } else { @@ -637,8 +638,17 @@ void shellPrintField(const char *val, TAOS_FIELD *field, int32_t width, int32_t } } break; + case TSDB_DATA_TYPE_VARBINARY:{ + void* data = NULL; + uint32_t size = 0; + if(taosAscii2Hex(val, length, &data, &size) < 0){ + break; + } + shellPrintNChar(data, size, width); + taosMemoryFree(data); + break; + } case TSDB_DATA_TYPE_BINARY: - case TSDB_DATA_TYPE_VARBINARY case TSDB_DATA_TYPE_NCHAR: case TSDB_DATA_TYPE_JSON: shellPrintNChar(val, length, width); From dbc1b35d442d22fc67089c92909c0e080792b400 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 16 Aug 2023 19:57:01 +0800 Subject: [PATCH 03/13] feat:support varbinary type --- include/libs/function/taosudf.h | 4 ++-- source/libs/catalog/src/ctgAsync.c | 1 - source/libs/scalar/src/filter.c | 2 +- source/libs/scalar/src/sclvector.c | 2 +- tools/shell/src/shellEngine.c | 4 ++-- utils/tsim/src/simExe.c | 2 +- 6 files changed, 7 insertions(+), 8 deletions(-) diff --git a/include/libs/function/taosudf.h b/include/libs/function/taosudf.h index 995198bc9e..935f9f9b03 100644 --- a/include/libs/function/taosudf.h +++ b/include/libs/function/taosudf.h @@ -109,8 +109,8 @@ typedef uint16_t VarDataLenT; // maxVarDataLen: 65535 #define varDataLenByData(v) (*(VarDataLenT *)(((char *)(v)) - VARSTR_HEADER_SIZE)) #define varDataSetLen(v, _len) (((VarDataLenT *)(v))[0] = (VarDataLenT)(_len)) #define IS_VAR_DATA_TYPE(t) \ - (((t) == TSDB_DATA_TYPE_VARCHAR) || ((t) == TSDB_DATA_TYPE_NCHAR) || ((t) == TSDB_DATA_TYPE_JSON) || ((t) == TSDB_DATA_TYPE_GEOMETRY)) -#define IS_STR_DATA_TYPE(t) (((t) == TSDB_DATA_TYPE_VARCHAR) || ((t) == TSDB_DATA_TYPE_NCHAR)) + (((t) == TSDB_DATA_TYPE_VARCHAR) || ((t) == TSDB_DATA_TYPE_VARBINARY) || ((t) == TSDB_DATA_TYPE_NCHAR) || ((t) == TSDB_DATA_TYPE_JSON) || ((t) == TSDB_DATA_TYPE_GEOMETRY)) +#define IS_STR_DATA_TYPE(t) (((t) == TSDB_DATA_TYPE_VARCHAR) || ((t) == TSDB_DATA_TYPE_VARBINARY) || ((t) == TSDB_DATA_TYPE_NCHAR)) static FORCE_INLINE char *udfColDataGetData(const SUdfColumn *pColumn, int32_t row) { if (IS_VAR_DATA_TYPE(pColumn->colMeta.type)) { diff --git a/source/libs/catalog/src/ctgAsync.c b/source/libs/catalog/src/ctgAsync.c index 5b0d1b4446..fb5ecf7ad2 100644 --- a/source/libs/catalog/src/ctgAsync.c +++ b/source/libs/catalog/src/ctgAsync.c @@ -1051,7 +1051,6 @@ void ctgUpdateJobErrCode(SCtgJob* pJob, int32_t errCode) { atomic_store_32(&pJob->jobResCode, errCode); qDebug("QID:0x%" PRIx64 " ctg job errCode updated to %s", pJob->queryId, tstrerror(errCode)); - ASSERT(0); return; } diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 0efe335c3d..18dbb8d056 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -1595,7 +1595,7 @@ int32_t fltConverToStr(char *str, int type, void *buf, int32_t bufSize, int32_t break; case TSDB_DATA_TYPE_BINARY: - case TSDB_DATA_TYPE_BINARY: +// case TSDB_DATA_TYPE_BINARY:// todovar case TSDB_DATA_TYPE_NCHAR: case TSDB_DATA_TYPE_GEOMETRY: if (bufSize < 0) { diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index ea637fd730..a11a9f9394 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -885,7 +885,7 @@ int32_t vectorConvertSingleColImpl(const SScalarParam *pIn, SScalarParam *pOut, break; } case TSDB_DATA_TYPE_BINARY: - case TSDB_DATA_TYPE_VARBINARY +// case TSDB_DATA_TYPE_VARBINARY //todovar case TSDB_DATA_TYPE_NCHAR: case TSDB_DATA_TYPE_GEOMETRY: { return vectorConvertToVarData(&cCtx); diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 8705d7ae38..f1a9a1d2bf 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -386,7 +386,7 @@ void shellDumpFieldToFile(TdFilePtr pFile, const char *val, TAOS_FIELD *field, i } break; case TSDB_DATA_TYPE_BINARY: - case TSDB_DATA_TYPE_VARBINARY +// case TSDB_DATA_TYPE_VARBINARY todovar case TSDB_DATA_TYPE_NCHAR: case TSDB_DATA_TYPE_JSON: { int32_t bufIndex = 0; @@ -779,7 +779,7 @@ int32_t shellCalcColWidth(TAOS_FIELD *field, int32_t precision) { return TMAX(SHELL_DOUBLE_WIDTH, width); case TSDB_DATA_TYPE_BINARY: - case TSDB_DATA_TYPE_VARBINARY + case TSDB_DATA_TYPE_VARBINARY: case TSDB_DATA_TYPE_GEOMETRY: if (field->bytes > shell.args.displayWidth) { return TMAX(shell.args.displayWidth, width); diff --git a/utils/tsim/src/simExe.c b/utils/tsim/src/simExe.c index 200d3e30f2..72b8eff27d 100644 --- a/utils/tsim/src/simExe.c +++ b/utils/tsim/src/simExe.c @@ -751,7 +751,7 @@ bool simExecuteNativeSqlCommand(SScript *script, char *rest, bool isSlow) { sprintf(value, "%.9lf", GET_DOUBLE_VAL(row[i])); break; case TSDB_DATA_TYPE_BINARY: - case TSDB_DATA_TYPE_VARBINARY +// case TSDB_DATA_TYPE_VARBINARY todovar case TSDB_DATA_TYPE_NCHAR: case TSDB_DATA_TYPE_GEOMETRY: if (length[i] < 0 || length[i] > 1 << 20) { From e24498b0fdf48b4ce69291bf136659c639423761 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 25 Aug 2023 14:59:34 +0800 Subject: [PATCH 04/13] enh: add task notify --- include/common/tmsg.h | 21 ++++- include/common/tmsgdef.h | 1 + include/libs/qworker/qworker.h | 3 + source/client/src/clientEnv.c | 3 +- source/common/src/tmsg.c | 60 ++++++++++++++ source/dnode/mgmt/mgmt_mnode/src/mmHandle.c | 1 + source/dnode/mgmt/mgmt_qnode/src/qmHandle.c | 1 + source/dnode/mgmt/mgmt_vnode/src/vmHandle.c | 1 + source/dnode/mgmt/node_mgmt/src/dmTransport.c | 2 +- source/dnode/mnode/impl/src/mndMain.c | 3 +- source/dnode/mnode/impl/src/mndQuery.c | 4 + source/dnode/qnode/src/qnode.c | 4 + source/dnode/vnode/src/vnd/vnodeSvr.c | 2 + source/libs/function/src/udfd.c | 2 +- source/libs/qworker/inc/qwInt.h | 3 + source/libs/qworker/inc/qwMsg.h | 1 + source/libs/qworker/src/qwMsg.c | 35 ++++++++ source/libs/qworker/src/qwUtil.c | 39 +++++++++ source/libs/qworker/src/qworker.c | 80 ++++++++++++------- source/libs/qworker/test/qworkerTests.cpp | 3 + source/libs/scheduler/inc/schInt.h | 3 +- source/libs/scheduler/src/schJob.c | 4 + source/libs/scheduler/src/schRemote.c | 48 ++++++++++- source/libs/scheduler/src/schTask.c | 54 ++++++++++++- 24 files changed, 339 insertions(+), 39 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index b6684cdd0e..8deec53470 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1418,6 +1418,7 @@ typedef struct { int64_t numOfProcessedCQuery; int64_t numOfProcessedFetch; int64_t numOfProcessedDrop; + int64_t numOfProcessedNotify; int64_t numOfProcessedHb; int64_t numOfProcessedDelete; int64_t cacheDataSize; @@ -2159,8 +2160,24 @@ typedef struct { int32_t tSerializeSTaskDropReq(void* buf, int32_t bufLen, STaskDropReq* pReq); int32_t tDeserializeSTaskDropReq(void* buf, int32_t bufLen, STaskDropReq* pReq); -int32_t tSerializeSTaskDropReq(void* buf, int32_t bufLen, STaskDropReq* pReq); -int32_t tDeserializeSTaskDropReq(void* buf, int32_t bufLen, STaskDropReq* pReq); + + +typedef enum { + TASK_NOTIFY_FINISHED = 1, +} ETaskNotifyType; + +typedef struct { + SMsgHead header; + uint64_t sId; + uint64_t queryId; + uint64_t taskId; + int64_t refId; + int32_t execId; + ETaskNotifyType type; +} STaskNotifyReq; + +int32_t tSerializeSTaskNotifyReq(void* buf, int32_t bufLen, STaskNotifyReq* pReq); +int32_t tDeserializeSTaskNotifyReq(void* buf, int32_t bufLen, STaskNotifyReq* pReq); int32_t tSerializeSQueryTableRsp(void* buf, int32_t bufLen, SQueryTableRsp* pRsp); int32_t tDeserializeSQueryTableRsp(void* buf, int32_t bufLen, SQueryTableRsp* pRsp); diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index 323b2de6ba..29f0667dac 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -244,6 +244,7 @@ enum { // WARN: new msg should be appended to segment tail TD_DEF_MSG_TYPE(TDMT_SCH_DROP_TASK, "drop-task", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_SCH_EXPLAIN, "explain", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_SCH_LINK_BROKEN, "link-broken", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_SCH_TASK_NOTIFY, "task-notify", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_SCH_MAX_MSG, "sch-max", NULL, NULL) diff --git a/include/libs/qworker/qworker.h b/include/libs/qworker/qworker.h index 60ed94d4de..bbd2d76b59 100644 --- a/include/libs/qworker/qworker.h +++ b/include/libs/qworker/qworker.h @@ -45,6 +45,7 @@ typedef struct { uint64_t cqueryProcessed; uint64_t fetchProcessed; uint64_t dropProcessed; + uint64_t notifyProcessed; uint64_t hbProcessed; uint64_t deleteProcessed; @@ -90,6 +91,8 @@ int32_t qWorkerProcessCancelMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, in int32_t qWorkerProcessDropMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int64_t ts); +int32_t qWorkerProcessNotifyMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int64_t ts); + int32_t qWorkerProcessHbMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int64_t ts); int32_t qWorkerProcessDeleteMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, SDeleteRes *pRes); diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 40c27bf164..b555f4e683 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -133,7 +133,8 @@ void closeTransporter(SAppInstInfo *pAppInfo) { static bool clientRpcRfp(int32_t code, tmsg_t msgType) { if (NEED_REDIRECT_ERROR(code)) { if (msgType == TDMT_SCH_QUERY || msgType == TDMT_SCH_MERGE_QUERY || msgType == TDMT_SCH_FETCH || - msgType == TDMT_SCH_MERGE_FETCH || msgType == TDMT_SCH_QUERY_HEARTBEAT || msgType == TDMT_SCH_DROP_TASK) { + msgType == TDMT_SCH_MERGE_FETCH || msgType == TDMT_SCH_QUERY_HEARTBEAT || msgType == TDMT_SCH_DROP_TASK || + msgType == TDMT_SCH_TASK_NOTIFY) { return false; } return true; diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 1431cd049a..9a21563abe 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -1092,6 +1092,7 @@ int32_t tSerializeSStatusReq(void *buf, int32_t bufLen, SStatusReq *pReq) { if (tEncodeI64(&encoder, pReq->qload.numOfProcessedCQuery) < 0) return -1; if (tEncodeI64(&encoder, pReq->qload.numOfProcessedFetch) < 0) return -1; if (tEncodeI64(&encoder, pReq->qload.numOfProcessedDrop) < 0) return -1; + if (tEncodeI64(&encoder, pReq->qload.numOfProcessedNotify) < 0) return -1; if (tEncodeI64(&encoder, pReq->qload.numOfProcessedHb) < 0) return -1; if (tEncodeI64(&encoder, pReq->qload.numOfProcessedDelete) < 0) return -1; if (tEncodeI64(&encoder, pReq->qload.cacheDataSize) < 0) return -1; @@ -1189,6 +1190,7 @@ int32_t tDeserializeSStatusReq(void *buf, int32_t bufLen, SStatusReq *pReq) { if (tDecodeI64(&decoder, &pReq->qload.numOfProcessedCQuery) < 0) return -1; if (tDecodeI64(&decoder, &pReq->qload.numOfProcessedFetch) < 0) return -1; if (tDecodeI64(&decoder, &pReq->qload.numOfProcessedDrop) < 0) return -1; + if (tDecodeI64(&decoder, &pReq->qload.numOfProcessedNotify) < 0) return -1; if (tDecodeI64(&decoder, &pReq->qload.numOfProcessedHb) < 0) return -1; if (tDecodeI64(&decoder, &pReq->qload.numOfProcessedDelete) < 0) return -1; if (tDecodeI64(&decoder, &pReq->qload.cacheDataSize) < 0) return -1; @@ -5874,6 +5876,64 @@ int32_t tDeserializeSTaskDropReq(void *buf, int32_t bufLen, STaskDropReq *pReq) return 0; } +int32_t tSerializeSTaskNotifyReq(void *buf, int32_t bufLen, STaskNotifyReq *pReq) { + int32_t headLen = sizeof(SMsgHead); + if (buf != NULL) { + buf = (char *)buf + headLen; + bufLen -= headLen; + } + + SEncoder encoder = {0}; + tEncoderInit(&encoder, buf, bufLen); + if (tStartEncode(&encoder) < 0) return -1; + + if (tEncodeU64(&encoder, pReq->sId) < 0) return -1; + if (tEncodeU64(&encoder, pReq->queryId) < 0) return -1; + if (tEncodeU64(&encoder, pReq->taskId) < 0) return -1; + if (tEncodeI64(&encoder, pReq->refId) < 0) return -1; + if (tEncodeI32(&encoder, pReq->execId) < 0) return -1; + if (tEncodeI32(&encoder, pReq->type) < 0) return -1; + + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tEncoderClear(&encoder); + + if (buf != NULL) { + SMsgHead *pHead = (SMsgHead *)((char *)buf - headLen); + pHead->vgId = htonl(pReq->header.vgId); + pHead->contLen = htonl(tlen + headLen); + } + + return tlen + headLen; +} + +int32_t tDeserializeSTaskNotifyReq(void *buf, int32_t bufLen, STaskNotifyReq *pReq) { + int32_t headLen = sizeof(SMsgHead); + + SMsgHead *pHead = buf; + pHead->vgId = pReq->header.vgId; + pHead->contLen = pReq->header.contLen; + + SDecoder decoder = {0}; + tDecoderInit(&decoder, (char *)buf + headLen, bufLen - headLen); + + if (tStartDecode(&decoder) < 0) return -1; + + if (tDecodeU64(&decoder, &pReq->sId) < 0) return -1; + if (tDecodeU64(&decoder, &pReq->queryId) < 0) return -1; + if (tDecodeU64(&decoder, &pReq->taskId) < 0) return -1; + if (tDecodeI64(&decoder, &pReq->refId) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->execId) < 0) return -1; + if (tDecodeI32(&decoder, (int32_t*)&pReq->type) < 0) return -1; + + tEndDecode(&decoder); + + tDecoderClear(&decoder); + return 0; +} + + int32_t tSerializeSQueryTableRsp(void *buf, int32_t bufLen, SQueryTableRsp *pRsp) { SEncoder encoder = {0}; tEncoderInit(&encoder, buf, bufLen); diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c index ae1b46a21d..eaa80ba775 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c @@ -187,6 +187,7 @@ SArray *mmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_SCH_QUERY_HEARTBEAT, mmPutMsgToFetchQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SCH_FETCH, mmPutMsgToFetchQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SCH_MERGE_FETCH, mmPutMsgToFetchQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SCH_TASK_NOTIFY, mmPutMsgToFetchQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_CREATE_STB_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_STB_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_STB_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; diff --git a/source/dnode/mgmt/mgmt_qnode/src/qmHandle.c b/source/dnode/mgmt/mgmt_qnode/src/qmHandle.c index 5017ad7b74..86bc11c616 100644 --- a/source/dnode/mgmt/mgmt_qnode/src/qmHandle.c +++ b/source/dnode/mgmt/mgmt_qnode/src/qmHandle.c @@ -89,6 +89,7 @@ SArray *qmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_SCH_CANCEL_TASK, qmPutNodeMsgToFetchQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SCH_DROP_TASK, qmPutNodeMsgToFetchQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SCH_TASK_NOTIFY, qmPutNodeMsgToFetchQueue, 1) == NULL) goto _OVER; code = 0; _OVER: diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index e50a75d33a..f43e1f5537 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -755,6 +755,7 @@ SArray *vmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_VND_TABLES_META, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SCH_CANCEL_TASK, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SCH_DROP_TASK, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SCH_TASK_NOTIFY, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_CREATE_STB, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_TTL_TABLE, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_STB, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; diff --git a/source/dnode/mgmt/node_mgmt/src/dmTransport.c b/source/dnode/mgmt/node_mgmt/src/dmTransport.c index e0f7da3ac4..665f86034d 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmTransport.c +++ b/source/dnode/mgmt/node_mgmt/src/dmTransport.c @@ -272,7 +272,7 @@ static bool rpcRfp(int32_t code, tmsg_t msgType) { code == TSDB_CODE_SYN_RESTORING || code == TSDB_CODE_VND_STOPPED || code == TSDB_CODE_APP_IS_STARTING || code == TSDB_CODE_APP_IS_STOPPING) { if (msgType == TDMT_SCH_QUERY || msgType == TDMT_SCH_MERGE_QUERY || msgType == TDMT_SCH_FETCH || - msgType == TDMT_SCH_MERGE_FETCH) { + msgType == TDMT_SCH_MERGE_FETCH || msgType == TDMT_SCH_TASK_NOTIFY) { return false; } return true; diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index fd4ebf549f..12e28969c9 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -629,7 +629,8 @@ static int32_t mndCheckMnodeState(SRpcMsg *pMsg) { if (!IsReq(pMsg)) return 0; if (pMsg->msgType == TDMT_SCH_QUERY || pMsg->msgType == TDMT_SCH_MERGE_QUERY || pMsg->msgType == TDMT_SCH_QUERY_CONTINUE || pMsg->msgType == TDMT_SCH_QUERY_HEARTBEAT || - pMsg->msgType == TDMT_SCH_FETCH || pMsg->msgType == TDMT_SCH_MERGE_FETCH || pMsg->msgType == TDMT_SCH_DROP_TASK) { + pMsg->msgType == TDMT_SCH_FETCH || pMsg->msgType == TDMT_SCH_MERGE_FETCH || pMsg->msgType == TDMT_SCH_DROP_TASK || + pMsg->msgType == TDMT_SCH_TASK_NOTIFY) { return 0; } diff --git a/source/dnode/mnode/impl/src/mndQuery.c b/source/dnode/mnode/impl/src/mndQuery.c index 8e95fa3d6d..c03b02c17f 100644 --- a/source/dnode/mnode/impl/src/mndQuery.c +++ b/source/dnode/mnode/impl/src/mndQuery.c @@ -55,6 +55,9 @@ int32_t mndProcessQueryMsg(SRpcMsg *pMsg) { case TDMT_SCH_QUERY_HEARTBEAT: code = qWorkerProcessHbMsg(pMnode, pMnode->pQuery, pMsg, 0); break; + case TDMT_SCH_TASK_NOTIFY: + code = qWorkerProcessNotifyMsg(pMnode, pMnode->pQuery, pMsg, 0); + break; default: terrno = TSDB_CODE_APP_ERROR; mError("unknown msg type:%d in query queue", pMsg->msgType); @@ -175,6 +178,7 @@ int32_t mndInitQuery(SMnode *pMnode) { mndSetMsgHandle(pMnode, TDMT_SCH_QUERY_CONTINUE, mndProcessQueryMsg); mndSetMsgHandle(pMnode, TDMT_SCH_FETCH, mndProcessQueryMsg); mndSetMsgHandle(pMnode, TDMT_SCH_MERGE_FETCH, mndProcessQueryMsg); + mndSetMsgHandle(pMnode, TDMT_SCH_TASK_NOTIFY, mndProcessQueryMsg); mndSetMsgHandle(pMnode, TDMT_SCH_DROP_TASK, mndProcessQueryMsg); mndSetMsgHandle(pMnode, TDMT_SCH_QUERY_HEARTBEAT, mndProcessQueryMsg); mndSetMsgHandle(pMnode, TDMT_MND_BATCH_META, mndProcessBatchMetaMsg); diff --git a/source/dnode/qnode/src/qnode.c b/source/dnode/qnode/src/qnode.c index 3482355512..9937debb13 100644 --- a/source/dnode/qnode/src/qnode.c +++ b/source/dnode/qnode/src/qnode.c @@ -57,6 +57,7 @@ int32_t qndGetLoad(SQnode *pQnode, SQnodeLoad *pLoad) { pLoad->numOfProcessedCQuery = stat.cqueryProcessed; pLoad->numOfProcessedFetch = stat.fetchProcessed; pLoad->numOfProcessedDrop = stat.dropProcessed; + pLoad->numOfProcessedNotify = stat.notifyProcessed; pLoad->numOfProcessedHb = stat.hbProcessed; pLoad->numOfProcessedDelete = stat.deleteProcessed; @@ -100,6 +101,9 @@ int32_t qndProcessQueryMsg(SQnode *pQnode, int64_t ts, SRpcMsg *pMsg) { case TDMT_SCH_QUERY_HEARTBEAT: code = qWorkerProcessHbMsg(pQnode, pQnode->pQuery, pMsg, ts); break; + case TDMT_SCH_TASK_NOTIFY: + code = qWorkerProcessNotifyMsg(pQnode, pQnode->pQuery, pMsg, ts); + break; default: qError("unknown msg type:%d in qnode queue", pMsg->msgType); terrno = TSDB_CODE_APP_ERROR; diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index a9399e4db1..737fd03d6f 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -702,6 +702,8 @@ int32_t vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) { // return qWorkerProcessCancelMsg(pVnode, pVnode->pQuery, pMsg, 0); case TDMT_SCH_DROP_TASK: return qWorkerProcessDropMsg(pVnode, pVnode->pQuery, pMsg, 0); + case TDMT_SCH_TASK_NOTIFY: + return qWorkerProcessNotifyMsg(pVnode, pVnode->pQuery, pMsg, 0); case TDMT_SCH_QUERY_HEARTBEAT: return qWorkerProcessHbMsg(pVnode, pVnode->pQuery, pMsg, 0); case TDMT_VND_TABLE_META: diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c index 575bce09bb..bd459af9f5 100644 --- a/source/libs/function/src/udfd.c +++ b/source/libs/function/src/udfd.c @@ -971,7 +971,7 @@ static bool udfdRpcRfp(int32_t code, tmsg_t msgType) { code == TSDB_CODE_RPC_SOMENODE_NOT_CONNECTED || code == TSDB_CODE_SYN_RESTORING || code == TSDB_CODE_MNODE_NOT_FOUND || code == TSDB_CODE_APP_IS_STARTING || code == TSDB_CODE_APP_IS_STOPPING) { if (msgType == TDMT_SCH_QUERY || msgType == TDMT_SCH_MERGE_QUERY || msgType == TDMT_SCH_FETCH || - msgType == TDMT_SCH_MERGE_FETCH) { + msgType == TDMT_SCH_MERGE_FETCH || msgType == TDMT_SCH_TASK_NOTIFY) { return false; } return true; diff --git a/source/libs/qworker/inc/qwInt.h b/source/libs/qworker/inc/qwInt.h index 8797a8cf6b..b4bd1943c5 100644 --- a/source/libs/qworker/inc/qwInt.h +++ b/source/libs/qworker/inc/qwInt.h @@ -133,6 +133,7 @@ typedef struct SQWTaskCtx { bool queryContinue; bool queryExecDone; bool queryInQueue; + bool explainRsped; int32_t rspCode; int64_t affectedRows; // for insert ...select stmt @@ -169,6 +170,7 @@ typedef struct SQWMsgStat { uint64_t rspProcessed; uint64_t cancelProcessed; uint64_t dropProcessed; + uint64_t notifyProcessed; uint64_t hbProcessed; uint64_t deleteProcessed; } SQWMsgStat; @@ -406,6 +408,7 @@ int32_t qwAddTaskCtx(QW_FPARAMS_DEF); void qwDbgSimulateRedirect(SQWMsg *qwMsg, SQWTaskCtx *ctx, bool *rsped); void qwDbgSimulateSleep(void); void qwDbgSimulateDead(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *rsped); +int32_t qwSendExplainResponse(QW_FPARAMS_DEF, SQWTaskCtx *ctx); #ifdef __cplusplus } diff --git a/source/libs/qworker/inc/qwMsg.h b/source/libs/qworker/inc/qwMsg.h index f226b223f7..ae68f69802 100644 --- a/source/libs/qworker/inc/qwMsg.h +++ b/source/libs/qworker/inc/qwMsg.h @@ -30,6 +30,7 @@ int32_t qwProcessCQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg); int32_t qwProcessReady(QW_FPARAMS_DEF, SQWMsg *qwMsg); int32_t qwProcessFetch(QW_FPARAMS_DEF, SQWMsg *qwMsg); int32_t qwProcessDrop(QW_FPARAMS_DEF, SQWMsg *qwMsg); +int32_t qwProcessNotify(QW_FPARAMS_DEF, SQWMsg *qwMsg); int32_t qwProcessHb(SQWorker *mgmt, SQWMsg *qwMsg, SSchedulerHbReq *req); int32_t qwProcessDelete(QW_FPARAMS_DEF, SQWMsg *qwMsg, SDeleteRes *pRes); diff --git a/source/libs/qworker/src/qwMsg.c b/source/libs/qworker/src/qwMsg.c index 0cbcf44ed4..9a1c309ab0 100644 --- a/source/libs/qworker/src/qwMsg.c +++ b/source/libs/qworker/src/qwMsg.c @@ -610,6 +610,41 @@ int32_t qWorkerProcessDropMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int6 return TSDB_CODE_SUCCESS; } +int32_t qWorkerProcessNotifyMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int64_t ts) { + if (NULL == node || NULL == qWorkerMgmt || NULL == pMsg) { + return TSDB_CODE_QRY_INVALID_INPUT; + } + + int32_t code = 0; + SQWorker *mgmt = (SQWorker *)qWorkerMgmt; + + qwUpdateTimeInQueue(mgmt, ts, FETCH_QUEUE); + QW_STAT_INC(mgmt->stat.msgStat.notifyProcessed, 1); + + STaskNotifyReq msg = {0}; + if (tDeserializeSTaskNotifyReq(pMsg->pCont, pMsg->contLen, &msg) < 0) { + QW_ELOG("tDeserializeSTaskNotifyReq failed, contLen:%d", pMsg->contLen); + QW_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); + } + + uint64_t sId = msg.sId; + uint64_t qId = msg.queryId; + uint64_t tId = msg.taskId; + int64_t rId = msg.refId; + int32_t eId = msg.execId; + + SQWMsg qwMsg = {.node = node, .msg = NULL, .msgLen = 0, .code = pMsg->code, .connInfo = pMsg->info, .msgType = msg.type}; + + QW_SCH_TASK_DLOG("processNotify start, node:%p, handle:%p", node, pMsg->info.handle); + + QW_ERR_RET(qwProcessNotify(QW_FPARAMS(), &qwMsg)); + + QW_SCH_TASK_DLOG("processNotify end, node:%p", node); + + return TSDB_CODE_SUCCESS; +} + + int32_t qWorkerProcessHbMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int64_t ts) { if (NULL == node || NULL == qWorkerMgmt || NULL == pMsg) { return TSDB_CODE_QRY_INVALID_INPUT; diff --git a/source/libs/qworker/src/qwUtil.c b/source/libs/qworker/src/qwUtil.c index 3b127ee780..f00c4aef30 100644 --- a/source/libs/qworker/src/qwUtil.c +++ b/source/libs/qworker/src/qwUtil.c @@ -316,6 +316,45 @@ void qwFreeTaskCtx(SQWTaskCtx *ctx) { taosArrayDestroy(ctx->tbInfo); } +static void freeExplainExecItem(void *param) { + SExplainExecInfo *pInfo = param; + taosMemoryFree(pInfo->verboseInfo); +} + + +int32_t qwSendExplainResponse(QW_FPARAMS_DEF, SQWTaskCtx *ctx) { + qTaskInfo_t taskHandle = ctx->taskHandle; + + ctx->explainRsped = true; + + SArray *execInfoList = taosArrayInit(4, sizeof(SExplainExecInfo)); + QW_ERR_RET(qGetExplainExecInfo(taskHandle, execInfoList)); + + if (ctx->localExec) { + SExplainLocalRsp localRsp = {0}; + localRsp.rsp.numOfPlans = taosArrayGetSize(execInfoList); + SExplainExecInfo *pExec = taosMemoryCalloc(localRsp.rsp.numOfPlans, sizeof(SExplainExecInfo)); + memcpy(pExec, taosArrayGet(execInfoList, 0), localRsp.rsp.numOfPlans * sizeof(SExplainExecInfo)); + localRsp.rsp.subplanInfo = pExec; + localRsp.qId = qId; + localRsp.tId = tId; + localRsp.rId = rId; + localRsp.eId = eId; + taosArrayPush(ctx->explainRes, &localRsp); + taosArrayDestroy(execInfoList); + } else { + SRpcHandleInfo connInfo = ctx->ctrlConnInfo; + connInfo.ahandle = NULL; + int32_t code = qwBuildAndSendExplainRsp(&connInfo, execInfoList); + taosArrayDestroyEx(execInfoList, freeExplainExecItem); + QW_ERR_RET(code); + } + + return TSDB_CODE_SUCCESS; +} + + + int32_t qwDropTaskCtx(QW_FPARAMS_DEF) { char id[sizeof(qId) + sizeof(tId) + sizeof(eId)] = {0}; QW_SET_QTID(id, qId, tId, eId); diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 1759cc89f5..afce4a496a 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -90,11 +90,6 @@ int32_t qwProcessHbLinkBroken(SQWorker *mgmt, SQWMsg *qwMsg, SSchedulerHbReq *re QW_RET(TSDB_CODE_SUCCESS); } -static void freeItem(void *param) { - SExplainExecInfo *pInfo = param; - taosMemoryFree(pInfo->verboseInfo); -} - int32_t qwHandleTaskComplete(QW_FPARAMS_DEF, SQWTaskCtx *ctx) { qTaskInfo_t taskHandle = ctx->taskHandle; @@ -102,29 +97,8 @@ int32_t qwHandleTaskComplete(QW_FPARAMS_DEF, SQWTaskCtx *ctx) { ctx->queryExecDone = true; if (TASK_TYPE_TEMP == ctx->taskType && taskHandle) { - if (ctx->explain) { - SArray *execInfoList = taosArrayInit(4, sizeof(SExplainExecInfo)); - QW_ERR_RET(qGetExplainExecInfo(taskHandle, execInfoList)); - - if (ctx->localExec) { - SExplainLocalRsp localRsp = {0}; - localRsp.rsp.numOfPlans = taosArrayGetSize(execInfoList); - SExplainExecInfo *pExec = taosMemoryCalloc(localRsp.rsp.numOfPlans, sizeof(SExplainExecInfo)); - memcpy(pExec, taosArrayGet(execInfoList, 0), localRsp.rsp.numOfPlans * sizeof(SExplainExecInfo)); - localRsp.rsp.subplanInfo = pExec; - localRsp.qId = qId; - localRsp.tId = tId; - localRsp.rId = rId; - localRsp.eId = eId; - taosArrayPush(ctx->explainRes, &localRsp); - taosArrayDestroy(execInfoList); - } else { - SRpcHandleInfo connInfo = ctx->ctrlConnInfo; - connInfo.ahandle = NULL; - int32_t code = qwBuildAndSendExplainRsp(&connInfo, execInfoList); - taosArrayDestroyEx(execInfoList, freeItem); - QW_ERR_RET(code); - } + if (ctx->explain && !ctx->explainRsped) { + QW_ERR_RET(qwSendExplainResponse(QW_FPARAMS(), ctx)); } if (!ctx->needFetch) { @@ -1030,6 +1004,55 @@ _return: QW_RET(TSDB_CODE_SUCCESS); } +int32_t qwProcessNotify(QW_FPARAMS_DEF, SQWMsg *qwMsg) { + int32_t code = 0; + SQWTaskCtx *ctx = NULL; + bool locked = false; + + QW_ERR_JRET(qwAcquireTaskCtx(QW_FPARAMS(), &ctx)); + + QW_LOCK(QW_WRITE, &ctx->lock); + + locked = true; + + if (QW_QUERY_RUNNING(ctx)) { + QW_ERR_JRET(qwKillTaskHandle(ctx, TSDB_CODE_TSC_QUERY_CANCELLED)); + qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_SUCC, ctx->dynamicTask); + } + + switch (qwMsg->msgType) { + case TASK_NOTIFY_FINISHED: + if (ctx->explain && !ctx->explainRsped) { + QW_ERR_RET(qwSendExplainResponse(QW_FPARAMS(), ctx)); + } + break; + default: + QW_ELOG("Invalid task notify type %d", qwMsg->msgType); + QW_ERR_JRET(TSDB_CODE_INVALID_MSG); + break; + } + +_return: + + if (code) { + if (ctx) { + QW_UPDATE_RSP_CODE(ctx, code); + qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_FAIL, ctx->dynamicTask); + } + } + + if (locked) { + QW_UNLOCK(QW_WRITE, &ctx->lock); + } + + if (ctx) { + qwReleaseTaskCtx(mgmt, ctx); + } + + QW_RET(TSDB_CODE_SUCCESS); +} + + int32_t qwProcessHb(SQWorker *mgmt, SQWMsg *qwMsg, SSchedulerHbReq *req) { int32_t code = 0; SSchedulerHbRsp rsp = {0}; @@ -1354,6 +1377,7 @@ int32_t qWorkerGetStat(SReadHandle *handle, void *qWorkerMgmt, SQWorkerStat *pSt pStat->cqueryProcessed = QW_STAT_GET(mgmt->stat.msgStat.cqueryProcessed); pStat->fetchProcessed = QW_STAT_GET(mgmt->stat.msgStat.fetchProcessed); pStat->dropProcessed = QW_STAT_GET(mgmt->stat.msgStat.dropProcessed); + pStat->notifyProcessed = QW_STAT_GET(mgmt->stat.msgStat.notifyProcessed); pStat->hbProcessed = QW_STAT_GET(mgmt->stat.msgStat.hbProcessed); pStat->deleteProcessed = QW_STAT_GET(mgmt->stat.msgStat.deleteProcessed); diff --git a/source/libs/qworker/test/qworkerTests.cpp b/source/libs/qworker/test/qworkerTests.cpp index 02b341e28c..4a0d74a6e3 100644 --- a/source/libs/qworker/test/qworkerTests.cpp +++ b/source/libs/qworker/test/qworkerTests.cpp @@ -818,6 +818,9 @@ void *fetchQueueThread(void *param) { case TDMT_SCH_DROP_TASK: qWorkerProcessDropMsg(mockPointer, mgmt, fetchRpc, 0); break; + case TDMT_SCH_TASK_NOTIFY: + qWorkerProcessNotifyMsg(mockPointer, mgmt, fetchRpc, 0); + break; default: printf("unknown msg type:%d in fetch queue", fetchRpc->msgType); assert(0); diff --git a/source/libs/scheduler/inc/schInt.h b/source/libs/scheduler/inc/schInt.h index aecf3d5d91..63efa6bba4 100644 --- a/source/libs/scheduler/inc/schInt.h +++ b/source/libs/scheduler/inc/schInt.h @@ -531,7 +531,7 @@ void schDeregisterTaskHb(SSchJob *pJob, SSchTask *pTask); void schCleanClusterHb(void *pTrans); int32_t schLaunchTask(SSchJob *job, SSchTask *task); int32_t schDelayLaunchTask(SSchJob *pJob, SSchTask *pTask); -int32_t schBuildAndSendMsg(SSchJob *job, SSchTask *task, SQueryNodeAddr *addr, int32_t msgType); +int32_t schBuildAndSendMsg(SSchJob *job, SSchTask *task, SQueryNodeAddr *addr, int32_t msgType, void* param); SSchJob *schAcquireJob(int64_t refId); int32_t schReleaseJob(int64_t refId); void schFreeFlowCtrl(SSchJob *pJob); @@ -598,6 +598,7 @@ int32_t schProcessOnJobFailure(SSchJob *pJob, int32_t errCode); int32_t schProcessOnJobPartialSuccess(SSchJob *pJob); void schFreeTask(SSchJob *pJob, SSchTask *pTask); void schDropTaskInHashList(SSchJob *pJob, SHashObj *list); +int32_t schNotifyTaskInHashList(SSchJob *pJob, SHashObj *list, ETaskNotifyType type); int32_t schLaunchLevelTasks(SSchJob *pJob, SSchLevel *level); int32_t schGetTaskFromList(SHashObj *pTaskList, uint64_t taskId, SSchTask **pTask); int32_t schInitTask(SSchJob *pJob, SSchTask *pTask, SSubplan *pPlan, SSchLevel *pLevel); diff --git a/source/libs/scheduler/src/schJob.c b/source/libs/scheduler/src/schJob.c index d2ed26d405..87370e8993 100644 --- a/source/libs/scheduler/src/schJob.c +++ b/source/libs/scheduler/src/schJob.c @@ -638,6 +638,10 @@ void schDropJobAllTasks(SSchJob *pJob) { // schDropTaskInHashList(pJob, pJob->failTasks); } +int32_t schNotifyJobAllTasks(SSchJob *pJob, ETaskNotifyType type) { + SCH_RET(schNotifyTaskInHashList(pJob, pJob->execTasks, type)); +} + void schFreeJobImpl(void *job) { if (NULL == job) { return; diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c index 01b4e7e9e6..10e25cae36 100644 --- a/source/libs/scheduler/src/schRemote.c +++ b/source/libs/scheduler/src/schRemote.c @@ -481,6 +481,18 @@ int32_t schHandleDropCallback(void *param, SDataBuf *pMsg, int32_t code) { return TSDB_CODE_SUCCESS; } +int32_t schHandleNotifyCallback(void *param, SDataBuf *pMsg, int32_t code) { + SSchTaskCallbackParam *pParam = (SSchTaskCallbackParam *)param; + qDebug("QID:0x%" PRIx64 ",TID:0x%" PRIx64 " task notify rsp received, code:0x%x", pParam->queryId, pParam->taskId, + code); + if (pMsg) { + taosMemoryFree(pMsg->pData); + taosMemoryFree(pMsg->pEpSet); + } + return TSDB_CODE_SUCCESS; +} + + int32_t schHandleLinkBrokenCallback(void *param, SDataBuf *pMsg, int32_t code) { SSchCallbackParamHeader *head = (SSchCallbackParamHeader *)param; rpcReleaseHandle(pMsg->handle, TAOS_CONN_CLIENT); @@ -646,6 +658,9 @@ int32_t schGetCallbackFp(int32_t msgType, __async_send_cb_fn_t *fp) { case TDMT_SCH_DROP_TASK: *fp = schHandleDropCallback; break; + case TDMT_SCH_TASK_NOTIFY: + *fp = schHandleNotifyCallback; + break; case TDMT_SCH_QUERY_HEARTBEAT: *fp = schHandleHbCallback; break; @@ -1027,7 +1042,7 @@ _return: SCH_RET(code); } -int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, int32_t msgType) { +int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, int32_t msgType, void* param) { int32_t msgSize = 0; void *msg = NULL; int32_t code = 0; @@ -1205,6 +1220,37 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, persistHandle = true; break; } + case TDMT_SCH_TASK_NOTIFY: { + ETaskNotifyType* pType = param; + STaskNotifyReq qMsg; + qMsg.header.vgId = addr->nodeId; + qMsg.header.contLen = 0; + qMsg.sId = schMgmt.sId; + qMsg.queryId = pJob->queryId; + qMsg.taskId = pTask->taskId; + qMsg.refId = pJob->refId; + qMsg.execId = pTask->execId; + qMsg.type = *pType; + + msgSize = tSerializeSTaskNotifyReq(NULL, 0, &qMsg); + if (msgSize < 0) { + SCH_TASK_ELOG("tSerializeSTaskNotifyReq get size, msgSize:%d", msgSize); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + + msg = taosMemoryCalloc(1, msgSize); + if (NULL == msg) { + SCH_TASK_ELOG("calloc %d failed", msgSize); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + + if (tSerializeSTaskNotifyReq(msg, msgSize, &qMsg) < 0) { + SCH_TASK_ELOG("tSerializeSTaskNotifyReq failed, msgSize:%d", msgSize); + taosMemoryFree(msg); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + break; + } default: SCH_TASK_ELOG("unknown msg type to send, msgType:%d", msgType); SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); diff --git a/source/libs/scheduler/src/schTask.c b/source/libs/scheduler/src/schTask.c index 9985e7d6a1..b284edf39a 100644 --- a/source/libs/scheduler/src/schTask.c +++ b/source/libs/scheduler/src/schTask.c @@ -862,7 +862,7 @@ void schDropTaskOnExecNode(SSchJob *pJob, SSchTask *pTask) { while (nodeInfo) { if (nodeInfo->handle) { SCH_SET_TASK_HANDLE(pTask, nodeInfo->handle); - schBuildAndSendMsg(pJob, pTask, &nodeInfo->addr, TDMT_SCH_DROP_TASK); + schBuildAndSendMsg(pJob, pTask, &nodeInfo->addr, TDMT_SCH_DROP_TASK, NULL); SCH_TASK_DLOG("start to drop task's %dth execNode", i); } else { SCH_TASK_DLOG("no need to drop task %dth execNode", i); @@ -875,6 +875,33 @@ void schDropTaskOnExecNode(SSchJob *pJob, SSchTask *pTask) { SCH_TASK_DLOG("task has been dropped on %d exec nodes", size); } +int32_t schNotifyTaskOnExecNode(SSchJob *pJob, SSchTask *pTask, ETaskNotifyType type) { + int32_t size = (int32_t)taosHashGetSize(pTask->execNodes); + if (size <= 0) { + SCH_TASK_DLOG("task no exec address to notify, status:%s", SCH_GET_TASK_STATUS_STR(pTask)); + return TSDB_CODE_SUCCESS; + } + + int32_t i = 0; + SSchNodeInfo *nodeInfo = taosHashIterate(pTask->execNodes, NULL); + while (nodeInfo) { + if (nodeInfo->handle) { + SCH_SET_TASK_HANDLE(pTask, nodeInfo->handle); + SCH_ERR_RET(schBuildAndSendMsg(pJob, pTask, &nodeInfo->addr, TDMT_SCH_TASK_NOTIFY, &type)); + SCH_TASK_DLOG("start to notify %d to task's %dth execNode", type, i); + } else { + SCH_TASK_DLOG("no need to notify %d to task %dth execNode", type, i); + } + + ++i; + nodeInfo = taosHashIterate(pTask->execNodes, nodeInfo); + } + + SCH_TASK_DLOG("task has been notified %d on %d exec nodes", type, size); + return TSDB_CODE_SUCCESS; +} + + int32_t schProcessOnTaskStatusRsp(SQueryNodeEpId *pEpId, SArray *pStatusList) { int32_t taskNum = (int32_t)taosArrayGetSize(pStatusList); SSchTask *pTask = NULL; @@ -1001,7 +1028,7 @@ int32_t schLaunchRemoteTask(SSchJob *pJob, SSchTask *pTask) { SCH_ERR_RET(schEnsureHbConnection(pJob, pTask)); } - SCH_RET(schBuildAndSendMsg(pJob, pTask, NULL, plan->msgType)); + SCH_RET(schBuildAndSendMsg(pJob, pTask, NULL, plan->msgType, NULL)); } int32_t schLaunchLocalTask(SSchJob *pJob, SSchTask *pTask) { @@ -1238,8 +1265,29 @@ void schDropTaskInHashList(SSchJob *pJob, SHashObj *list) { } } +int32_t schNotifyTaskInHashList(SSchJob *pJob, SHashObj *list, ETaskNotifyType type) { + int32_t code = TSDB_CODE_SUCCESS; + void *pIter = taosHashIterate(list, NULL); + while (pIter) { + SSchTask *pTask = *(SSchTask **)pIter; + + SCH_LOCK_TASK(pTask); + code = schNotifyTaskOnExecNode(pJob, pTask, type); + SCH_UNLOCK_TASK(pTask); + + if (TSDB_CODE_SUCCESS != code) { + break; + } + + pIter = taosHashIterate(list, pIter); + } + + SCH_RET(code); +} + + int32_t schExecRemoteFetch(SSchJob *pJob, SSchTask *pTask) { - SCH_RET(schBuildAndSendMsg(pJob, pJob->fetchTask, &pJob->resNode, SCH_FETCH_TYPE(pJob->fetchTask))); + SCH_RET(schBuildAndSendMsg(pJob, pJob->fetchTask, &pJob->resNode, SCH_FETCH_TYPE(pJob->fetchTask), NULL)); } int32_t schExecLocalFetch(SSchJob *pJob, SSchTask *pTask) { From e0cc4e7ed7fccc325ca8dce43055b04c1501e6a7 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 25 Aug 2023 15:12:28 +0800 Subject: [PATCH 05/13] fix: notify task --- source/libs/scheduler/inc/schInt.h | 1 + source/libs/scheduler/src/schRemote.c | 2 ++ 2 files changed, 3 insertions(+) diff --git a/source/libs/scheduler/inc/schInt.h b/source/libs/scheduler/inc/schInt.h index 63efa6bba4..c649e645a0 100644 --- a/source/libs/scheduler/inc/schInt.h +++ b/source/libs/scheduler/inc/schInt.h @@ -613,6 +613,7 @@ int32_t schHandleJobRetry(SSchJob *pJob, SSchTask *pTask, SDataBuf *pMsg, int32 int32_t schChkResetJobRetry(SSchJob *pJob, int32_t rspCode); void schResetTaskForRetry(SSchJob *pJob, SSchTask *pTask); int32_t schChkUpdateRedirectCtx(SSchJob *pJob, SSchTask *pTask, SEpSet *pEpSet, int32_t rspCode); +int32_t schNotifyJobAllTasks(SSchJob *pJob, ETaskNotifyType type); extern SSchDebug gSCHDebug; diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c index 10e25cae36..291a383393 100644 --- a/source/libs/scheduler/src/schRemote.c +++ b/source/libs/scheduler/src/schRemote.c @@ -88,6 +88,8 @@ int32_t schProcessFetchRsp(SSchJob *pJob, SSchTask *pTask, char *msg, int32_t rs SCH_ERR_JRET(qExecExplainEnd(pJob->explainCtx, &pRsp)); if (pRsp) { SCH_ERR_JRET(schProcessOnExplainDone(pJob, pTask, pRsp)); + } else { + SCH_ERR_JRET(schNotifyJobAllTasks(pJob, TASK_NOTIFY_FINISHED)); } taosMemoryFreeClear(msg); From cf50665c209ede9d33f02887544dbcc313a00728 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 25 Aug 2023 15:20:12 +0800 Subject: [PATCH 06/13] fix: dead lock issue --- source/libs/scheduler/inc/schInt.h | 4 ++-- source/libs/scheduler/src/schJob.c | 4 ++-- source/libs/scheduler/src/schRemote.c | 2 +- source/libs/scheduler/src/schTask.c | 20 ++++++++++++-------- 4 files changed, 17 insertions(+), 13 deletions(-) diff --git a/source/libs/scheduler/inc/schInt.h b/source/libs/scheduler/inc/schInt.h index c649e645a0..3b7a76bfc7 100644 --- a/source/libs/scheduler/inc/schInt.h +++ b/source/libs/scheduler/inc/schInt.h @@ -598,7 +598,7 @@ int32_t schProcessOnJobFailure(SSchJob *pJob, int32_t errCode); int32_t schProcessOnJobPartialSuccess(SSchJob *pJob); void schFreeTask(SSchJob *pJob, SSchTask *pTask); void schDropTaskInHashList(SSchJob *pJob, SHashObj *list); -int32_t schNotifyTaskInHashList(SSchJob *pJob, SHashObj *list, ETaskNotifyType type); +int32_t schNotifyTaskInHashList(SSchJob *pJob, SHashObj *list, ETaskNotifyType type, SSchTask *pTask); int32_t schLaunchLevelTasks(SSchJob *pJob, SSchLevel *level); int32_t schGetTaskFromList(SHashObj *pTaskList, uint64_t taskId, SSchTask **pTask); int32_t schInitTask(SSchJob *pJob, SSchTask *pTask, SSubplan *pPlan, SSchLevel *pLevel); @@ -613,7 +613,7 @@ int32_t schHandleJobRetry(SSchJob *pJob, SSchTask *pTask, SDataBuf *pMsg, int32 int32_t schChkResetJobRetry(SSchJob *pJob, int32_t rspCode); void schResetTaskForRetry(SSchJob *pJob, SSchTask *pTask); int32_t schChkUpdateRedirectCtx(SSchJob *pJob, SSchTask *pTask, SEpSet *pEpSet, int32_t rspCode); -int32_t schNotifyJobAllTasks(SSchJob *pJob, ETaskNotifyType type); +int32_t schNotifyJobAllTasks(SSchJob *pJob, SSchTask *pTask, ETaskNotifyType type); extern SSchDebug gSCHDebug; diff --git a/source/libs/scheduler/src/schJob.c b/source/libs/scheduler/src/schJob.c index 87370e8993..b565619e75 100644 --- a/source/libs/scheduler/src/schJob.c +++ b/source/libs/scheduler/src/schJob.c @@ -638,8 +638,8 @@ void schDropJobAllTasks(SSchJob *pJob) { // schDropTaskInHashList(pJob, pJob->failTasks); } -int32_t schNotifyJobAllTasks(SSchJob *pJob, ETaskNotifyType type) { - SCH_RET(schNotifyTaskInHashList(pJob, pJob->execTasks, type)); +int32_t schNotifyJobAllTasks(SSchJob *pJob, SSchTask *pTask, ETaskNotifyType type) { + SCH_RET(schNotifyTaskInHashList(pJob, pJob->execTasks, type, pTask)); } void schFreeJobImpl(void *job) { diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c index 291a383393..7b8decc007 100644 --- a/source/libs/scheduler/src/schRemote.c +++ b/source/libs/scheduler/src/schRemote.c @@ -89,7 +89,7 @@ int32_t schProcessFetchRsp(SSchJob *pJob, SSchTask *pTask, char *msg, int32_t rs if (pRsp) { SCH_ERR_JRET(schProcessOnExplainDone(pJob, pTask, pRsp)); } else { - SCH_ERR_JRET(schNotifyJobAllTasks(pJob, TASK_NOTIFY_FINISHED)); + SCH_ERR_JRET(schNotifyJobAllTasks(pJob, pTask, TASK_NOTIFY_FINISHED)); } taosMemoryFreeClear(msg); diff --git a/source/libs/scheduler/src/schTask.c b/source/libs/scheduler/src/schTask.c index b284edf39a..d96c01fc76 100644 --- a/source/libs/scheduler/src/schTask.c +++ b/source/libs/scheduler/src/schTask.c @@ -1265,18 +1265,22 @@ void schDropTaskInHashList(SSchJob *pJob, SHashObj *list) { } } -int32_t schNotifyTaskInHashList(SSchJob *pJob, SHashObj *list, ETaskNotifyType type) { +int32_t schNotifyTaskInHashList(SSchJob *pJob, SHashObj *list, ETaskNotifyType type, SSchTask *pCurrTask) { int32_t code = TSDB_CODE_SUCCESS; + + SCH_ERR_RET(schNotifyTaskOnExecNode(pJob, pCurrTask, type)); + void *pIter = taosHashIterate(list, NULL); while (pIter) { SSchTask *pTask = *(SSchTask **)pIter; - - SCH_LOCK_TASK(pTask); - code = schNotifyTaskOnExecNode(pJob, pTask, type); - SCH_UNLOCK_TASK(pTask); - - if (TSDB_CODE_SUCCESS != code) { - break; + if (pTask != pCurrTask) { + SCH_LOCK_TASK(pTask); + code = schNotifyTaskOnExecNode(pJob, pTask, type); + SCH_UNLOCK_TASK(pTask); + + if (TSDB_CODE_SUCCESS != code) { + break; + } } pIter = taosHashIterate(list, pIter); From 26e025d78d77bce62c1a792c8197585f38a32194 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 25 Aug 2023 15:43:45 +0800 Subject: [PATCH 07/13] feat:[TD-18789]support varbinary type --- include/libs/parser/parser.h | 1 + include/os/osString.h | 4 +- include/util/taoserror.h | 1 + include/util/tcompare.h | 2 +- include/util/tdef.h | 7 +- source/client/inc/clientSml.h | 1 - source/client/src/clientSml.c | 17 +- source/client/src/clientSmlLine.c | 44 ++- source/client/src/clientTmq.c | 4 + source/dnode/mnode/impl/src/mndTopic.c | 2 +- source/libs/parser/src/parInsertSml.c | 20 +- source/libs/parser/src/parInsertSql.c | 95 +++---- source/libs/parser/src/parInsertUtil.c | 4 +- source/libs/parser/src/parTokenizer.c | 2 +- source/libs/parser/src/parTranslater.c | 39 ++- source/libs/parser/src/parUtil.c | 2 +- source/libs/scalar/src/filter.c | 3 +- source/libs/scalar/src/sclvector.c | 90 +++++-- .../libs/scalar/test/filter/filterTests.cpp | 46 ---- source/os/src/osString.c | 78 ++++-- source/util/src/tcompare.c | 60 +---- source/util/src/terror.c | 1 + tests/parallel_test/cases.task | 1 + tests/system-test/2-query/sml.py | 8 +- tests/system-test/2-query/varbinary.py | 80 ++++++ tools/shell/src/shellEngine.c | 57 ++-- utils/test/c/CMakeLists.txt | 11 + utils/test/c/sml_test.c | 80 ++++++ utils/test/c/varbinary_test.c | 253 ++++++++++++++++++ utils/tsim/src/simExe.c | 1 - 30 files changed, 740 insertions(+), 274 deletions(-) create mode 100644 tests/system-test/2-query/varbinary.py create mode 100644 utils/test/c/varbinary_test.c diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index 58bdb77df3..e6a67e68f0 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -119,6 +119,7 @@ int32_t smlBindData(SQuery* handle, bool dataFormat, SArray* tags, SArray* colsS char* msgBuf, int32_t msgBufLen); int32_t smlBuildOutput(SQuery* handle, SHashObj* pVgHash); int rawBlockBindData(SQuery *query, STableMeta* pTableMeta, void* data, SVCreateTbReq* pCreateTb, TAOS_FIELD *fields, int numFields, bool needChangeLength); +void clearColValArray(SArray* pCols); int32_t rewriteToVnodeModifyOpStmt(SQuery* pQuery, SArray* pBufArray); SArray* serializeVgroupsCreateTableBatch(SHashObj* pVgroupHashmap); diff --git a/include/os/osString.h b/include/os/osString.h index ed64cbfe3f..4982ac8bfd 100644 --- a/include/os/osString.h +++ b/include/os/osString.h @@ -92,7 +92,9 @@ double taosStr2Double(const char *str, char **pEnd); float taosStr2Float(const char *str, char **pEnd); int32_t taosHex2Ascii(const char *z, uint32_t n, void** data, uint32_t* size); int32_t taosAscii2Hex(const char *z, uint32_t n, void** data, uint32_t* size); -int32_t taosBin2Ascii(const char *z, uint32_t n, void** data, uint32_t* size); +//int32_t taosBin2Ascii(const char *z, uint32_t n, void** data, uint32_t* size); +bool isHex(const char* z, uint32_t n); +bool isValidateHex(const char* z, uint32_t n); #ifdef __cplusplus } diff --git a/include/util/taoserror.h b/include/util/taoserror.h index a5081f2c7d..dbfad8bf8a 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -708,6 +708,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_PAR_INVALID_OPTR_USAGE TAOS_DEF_ERROR_CODE(0, 0x2667) #define TSDB_CODE_PAR_SYSTABLE_NOT_ALLOWED_FUNC TAOS_DEF_ERROR_CODE(0, 0x2668) #define TSDB_CODE_PAR_SYSTABLE_NOT_ALLOWED TAOS_DEF_ERROR_CODE(0, 0x2669) +#define TSDB_CODE_PAR_INVALID_VARBINARY TAOS_DEF_ERROR_CODE(0, 0x266A) #define TSDB_CODE_PAR_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x26FF) //planner diff --git a/include/util/tcompare.h b/include/util/tcompare.h index 2fa736f4df..65457b287a 100644 --- a/include/util/tcompare.h +++ b/include/util/tcompare.h @@ -102,6 +102,7 @@ int32_t compareUint64ValDesc(const void *pLeft, const void *pRight); int32_t compareLenPrefixedStrDesc(const void *pLeft, const void *pRight); int32_t compareLenPrefixedWStrDesc(const void *pLeft, const void *pRight); +int32_t compareLenBinaryValDesc(const void *pLeft, const void *pRight); int32_t comparestrPatternMatch(const void *pLeft, const void *pRight); int32_t comparestrPatternNMatch(const void *pLeft, const void *pRight); @@ -202,7 +203,6 @@ int32_t compareUint64Uint32(const void *pLeft, const void *pRight); __compar_fn_t getComparFunc(int32_t type, int32_t optr); __compar_fn_t getKeyComparFunc(int32_t keyType, int32_t order); -int32_t doCompare(const char *a, const char *b, int32_t type, size_t size); #ifdef __cplusplus } diff --git a/include/util/tdef.h b/include/util/tdef.h index e4af88bf10..6343540724 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -416,9 +416,10 @@ typedef enum ELogicConditionType { #define TSDB_EXPLAIN_RESULT_COLUMN_NAME "QUERY_PLAN" #define TSDB_MAX_FIELD_LEN 65519 // 16384:65519 -#define TSDB_MAX_BINARY_LEN TSDB_MAX_FIELD_LEN // 16384-8:65519 -#define TSDB_MAX_NCHAR_LEN TSDB_MAX_FIELD_LEN // 16384-8:65519 -#define TSDB_MAX_GEOMETRY_LEN TSDB_MAX_FIELD_LEN // 16384-8:65519 +#define TSDB_MAX_BINARY_LEN TSDB_MAX_FIELD_LEN // 16384-8:65519 +#define TSDB_MAX_NCHAR_LEN TSDB_MAX_FIELD_LEN // 16384-8:65519 +#define TSDB_MAX_GEOMETRY_LEN TSDB_MAX_FIELD_LEN // 16384-8:65519 +#define TSDB_MAX_VARBINARY_LEN TSDB_MAX_FIELD_LEN // 16384-8:65519 #define PRIMARYKEY_TIMESTAMP_COL_ID 1 #define COL_REACH_END(colId, maxColId) ((colId) > (maxColId)) diff --git a/source/client/inc/clientSml.h b/source/client/inc/clientSml.h index 1839c14894..30d4792279 100644 --- a/source/client/inc/clientSml.h +++ b/source/client/inc/clientSml.h @@ -251,7 +251,6 @@ int64_t smlParseOpenTsdbTime(SSmlHandle *info, const char *data, int32 int32_t smlClearForRerun(SSmlHandle *info); int32_t smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg); uint8_t smlGetTimestampLen(int64_t num); -void clearColValArray(SArray* pCols); void smlDestroyTableInfo(void *para); void freeSSmlKv(void* data); diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index a876ff9455..9b2f805675 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -596,7 +596,7 @@ static int32_t smlGenerateSchemaAction(SSchema *colField, SHashObj *colHash, SSm return TSDB_CODE_SML_INVALID_DATA; } - if (((colField[*index].type == TSDB_DATA_TYPE_VARCHAR || colField[*index].type == TSDB_DATA_TYPE_VARCHAR || colField[*index].type == TSDB_DATA_TYPE_GEOMETRY) && + if (((colField[*index].type == TSDB_DATA_TYPE_VARCHAR || colField[*index].type == TSDB_DATA_TYPE_VARBINARY || colField[*index].type == TSDB_DATA_TYPE_GEOMETRY) && (colField[*index].bytes - VARSTR_HEADER_SIZE) < kv->length) || (colField[*index].type == TSDB_DATA_TYPE_NCHAR && ((colField[*index].bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE < kv->length))) { @@ -1178,21 +1178,12 @@ void smlDestroyTableInfo(void *para) { taosMemoryFree(tag); } -void clearColValArray(SArray *pCols) { - int32_t num = taosArrayGetSize(pCols); - for (int32_t i = 0; i < num; ++i) { - SColVal *pCol = taosArrayGet(pCols, i); - if (TSDB_DATA_TYPE_NCHAR == pCol->type) { - taosMemoryFreeClear(pCol->value.pData); - } - } -} - void freeSSmlKv(void *data) { SSmlKv *kv = (SSmlKv *)data; - if (kv->keyEscaped) taosMemoryFree((void *)(kv->key)); - if (kv->valueEscaped) taosMemoryFree((void *)(kv->value)); + if (kv->keyEscaped) taosMemoryFreeClear(kv->key); + if (kv->valueEscaped) taosMemoryFreeClear(kv->value); if (kv->type == TSDB_DATA_TYPE_GEOMETRY) geosFreeBuffer((void *)(kv->value)); + if (kv->type == TSDB_DATA_TYPE_VARBINARY) taosMemoryFreeClear(kv->value); } void smlDestroyInfo(SSmlHandle *info) { diff --git a/source/client/src/clientSmlLine.c b/source/client/src/clientSmlLine.c index 73622a11cc..e62e018caa 100644 --- a/source/client/src/clientSmlLine.c +++ b/source/client/src/clientSmlLine.c @@ -109,7 +109,7 @@ int32_t smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg) { return code; } char* tmp = taosMemoryCalloc(pVal->length, 1); - memcpy(tmp, pVal->value + 2, pVal->length - 3); + memcpy(tmp, pVal->value + NCHAR_ADD_LEN - 1, pVal->length - NCHAR_ADD_LEN); code = doGeomFromText(tmp, (unsigned char **)&pVal->value, &pVal->length); taosMemoryFree(tmp); if (code != TSDB_CODE_SUCCESS) { @@ -126,6 +126,44 @@ int32_t smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg) { return TSDB_CODE_TSC_INVALID_VALUE; } + if (pVal->value[0] == 'b' || pVal->value[0] == 'B') { // varbinary + if (pVal->value[1] == '"' && pVal->value[pVal->length - 1] == '"' && pVal->length >= 3) { + pVal->type = TSDB_DATA_TYPE_VARBINARY; + if(isHex(pVal->value + NCHAR_ADD_LEN - 1, pVal->length - NCHAR_ADD_LEN)){ + if(!isValidateHex(pVal->value + NCHAR_ADD_LEN - 1, pVal->length - NCHAR_ADD_LEN)){ + return TSDB_CODE_PAR_INVALID_VARBINARY; + } + + void* data = NULL; + uint32_t size = 0; + if(taosHex2Ascii(pVal->value + NCHAR_ADD_LEN - 1, pVal->length - NCHAR_ADD_LEN, &data, &size) < 0){ + return TSDB_CODE_OUT_OF_MEMORY; + } + + if (size + VARSTR_HEADER_SIZE > TSDB_MAX_VARBINARY_LEN) { + taosMemoryFree(data); + return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; + } + pVal->value = data; + pVal->length = size; + }else{ + pVal->length -= NCHAR_ADD_LEN; + if (pVal->length > TSDB_MAX_VARBINARY_LEN - VARSTR_HEADER_SIZE) { + return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; + } + void *data = taosMemoryMalloc(pVal->length); + if(data == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } + memcpy(data, pVal->value + (NCHAR_ADD_LEN - 1), pVal->length); + pVal->value = data; + } + + return TSDB_CODE_SUCCESS; + } + return TSDB_CODE_TSC_INVALID_VALUE; + } + if (pVal->value[0] == 't' || pVal->value[0] == 'T') { if (pVal->length == 1 || (pVal->length == 4 && (pVal->value[1] == 'r' || pVal->value[1] == 'R') && @@ -515,6 +553,10 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin char *tmp = (char *)taosMemoryMalloc(kv.length); memcpy(tmp, kv.value, kv.length); PROCESS_SLASH_IN_FIELD_VALUE(tmp, kv.length); + ASSERT(kv.type != TSDB_DATA_TYPE_GEOMETRY); + if(kv.type == TSDB_DATA_TYPE_VARBINARY){ + taosMemoryFree((void*)kv.value); + } kv.value = tmp; kv.valueEscaped = valueEscaped; } diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index b4168046f4..619f144bf9 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -985,6 +985,10 @@ int32_t tmq_subscription(tmq_t* tmq, tmq_list_t** topics) { int32_t tmq_unsubscribe(tmq_t* tmq) { if(tmq == NULL) return TSDB_CODE_INVALID_PARA; + if (tmq->status != TMQ_CONSUMER_STATUS__READY) { + tscInfo("consumer:0x%" PRIx64 " not in ready state, close it directly", tmq->consumerId); + return 0; + } if (tmq->autoCommit) { int32_t rsp = tmq_commit_sync(tmq, NULL); if (rsp != 0) { diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index 0c8ebd83c9..6250a3dd71 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -4,7 +4,7 @@ * This program is free software: you can use, redistribute, and/or modify * it under the terms of the GNU Affero General Public License, version 3 * or later ("AGPL"), as published by the Free Software Foundation. - * + *f * This program is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. diff --git a/source/libs/parser/src/parInsertSml.c b/source/libs/parser/src/parInsertSml.c index 917848e121..aa10711e3c 100644 --- a/source/libs/parser/src/parInsertSml.c +++ b/source/libs/parser/src/parInsertSml.c @@ -18,18 +18,6 @@ #include "parToken.h" #include "ttime.h" -static void clearColValArray(SArray* pCols) { - int32_t num = taosArrayGetSize(pCols); - for (int32_t i = 0; i < num; ++i) { - SColVal* pCol = taosArrayGet(pCols, i); - if (TSDB_DATA_TYPE_NCHAR == pCol->type || TSDB_DATA_TYPE_GEOMETRY == pCol->type) { - taosMemoryFreeClear(pCol->value.pData); - } - pCol->flag = CV_FLAG_NONE; - pCol->value.val = 0; - } -} - int32_t qCreateSName(SName* pName, const char* pTableName, int32_t acctId, char* dbName, char* msgBuf, int32_t msgBufLen) { SMsgBuf msg = {.buf = msgBuf, .len = msgBufLen}; @@ -237,10 +225,10 @@ int32_t smlBuildCol(STableDataCxt* pTableCxt, SSchema* schema, void* data, int32 } pVal->value.pData = pUcs4; pVal->value.nData = len; - } else if (kv->type == TSDB_DATA_TYPE_BINARY || kv->type == TSDB_DATA_TYPE_VARBINARY || kv->type == TSDB_DATA_TYPE_GEOMETRY) { + } else if (kv->type == TSDB_DATA_TYPE_BINARY) { pVal->value.nData = kv->length; pVal->value.pData = (uint8_t*)kv->value; - } else if (kv->type == TSDB_DATA_TYPE_GEOMETRY) { + } else if (kv->type == TSDB_DATA_TYPE_GEOMETRY || kv->type == TSDB_DATA_TYPE_VARBINARY) { pVal->value.nData = kv->length; pVal->value.pData = taosMemoryMalloc(kv->length); memcpy(pVal->value.pData, (uint8_t*)kv->value, kv->length); @@ -368,10 +356,10 @@ int32_t smlBindData(SQuery* query, bool dataFormat, SArray* tags, SArray* colsSc } pVal->value.pData = pUcs4; pVal->value.nData = len; - } else if (kv->type == TSDB_DATA_TYPE_BINARY || kv->type == TSDB_DATA_TYPE_VARBINARY || kv->type == TSDB_DATA_TYPE_GEOMETRY) { + } else if (kv->type == TSDB_DATA_TYPE_BINARY) { pVal->value.nData = kv->length; pVal->value.pData = (uint8_t*)kv->value; - } else if (kv->type == TSDB_DATA_TYPE_GEOMETRY) { + } else if (kv->type == TSDB_DATA_TYPE_GEOMETRY || kv->type == TSDB_DATA_TYPE_VARBINARY) { pVal->value.nData = kv->length; pVal->value.pData = taosMemoryMalloc(kv->length); memcpy(pVal->value.pData, (uint8_t*)kv->value, kv->length); diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index c58647e540..38afb6ddae 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -326,6 +326,38 @@ static int parseGeometry(SToken *pToken, unsigned char **output, size_t *size) { return code; } +static int32_t parseVarbinary(SToken* pToken, uint8_t **pData, uint32_t *nData, int32_t bytes){ + if(pToken->type != TK_NK_STRING){ + return TSDB_CODE_PAR_INVALID_VARBINARY; + } + + if(isHex(pToken->z, pToken->n)){ + if(!isValidateHex(pToken->z, pToken->n)){ + return TSDB_CODE_PAR_INVALID_VARBINARY; + } + + void* data = NULL; + uint32_t size = 0; + if(taosHex2Ascii(pToken->z, pToken->n, &data, &size) < 0){ + return TSDB_CODE_OUT_OF_MEMORY; + } + + if (size + VARSTR_HEADER_SIZE > bytes) { + taosMemoryFree(data); + return TSDB_CODE_PAR_VALUE_TOO_LONG; + } + *pData = data; + *nData = size; + }else{ + if (pToken->n + VARSTR_HEADER_SIZE > bytes) { + return TSDB_CODE_PAR_VALUE_TOO_LONG; + } + *pData = taosStrdup(pToken->z); + *nData = pToken->n; + } + return TSDB_CODE_SUCCESS; +} + static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema, int16_t timePrec, STagVal* val, SMsgBuf* pMsgBuf) { int64_t iv; @@ -478,33 +510,10 @@ static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema, break; } case TSDB_DATA_TYPE_VARBINARY: { - // Too long values will raise the invalid sql error message - // Too long values will raise the invalid sql error message - void* data = NULL; - uint32_t size = 0; - if (pToken->type == TK_NK_HEX){ - if(taosHex2Ascii(pToken->z, pToken->n, &data, &size) < 0){ - return TSDB_CODE_OUT_OF_MEMORY; - } - }else if(pToken->type == TK_NK_BIN){ - if(taosBin2Ascii(pToken->z, pToken->n, &data, &size) < 0){ - return TSDB_CODE_OUT_OF_MEMORY; - } - }else{ - size = pToken->n; + code = parseVarbinary(pToken, &val->pData, &val->nData, pSchema->bytes); + if(code != TSDB_CODE_SUCCESS){ + return generateSyntaxErrMsg(pMsgBuf, code, pSchema->name); } - if (size + VARSTR_HEADER_SIZE > pSchema->bytes) { - if(pToken->type == TK_NK_HEX || pToken->type == TK_NK_BIN){ - taosMemoryFree(data); - } - return generateSyntaxErrMsg(pMsgBuf, TSDB_CODE_PAR_VALUE_TOO_LONG, pSchema->name); - } - if(pToken->type == TK_NK_HEX || pToken->type == TK_NK_BIN){ - val->pData = data; - }else{ - val->pData = taosStrdup(pToken->z); - } - val->nData = size; break; } case TSDB_DATA_TYPE_GEOMETRY: { @@ -1394,32 +1403,10 @@ static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql, break; } case TSDB_DATA_TYPE_VARBINARY: { - // Too long values will raise the invalid sql error message - void* data = NULL; - uint32_t size = 0; - if (pToken->type == TK_NK_HEX){ - if(taosHex2Ascii(pToken->z, pToken->n, &data, &size) < 0){ - return TSDB_CODE_OUT_OF_MEMORY; - } - }else if(pToken->type == TK_NK_BIN){ - if(taosBin2Ascii(pToken->z, pToken->n, &data, &size) < 0){ - return TSDB_CODE_OUT_OF_MEMORY; - } - }else{ - size = pToken->n; + int32_t code = parseVarbinary(pToken, &pVal->value.pData, &pVal->value.nData, pSchema->bytes); + if(code != TSDB_CODE_SUCCESS){ + return generateSyntaxErrMsg(&pCxt->msg, code, pSchema->name); } - if (size + VARSTR_HEADER_SIZE > pSchema->bytes) { - if(pToken->type == TK_NK_HEX || pToken->type == TK_NK_BIN){ - taosMemoryFree(data); - } - return generateSyntaxErrMsg(&pCxt->msg, TSDB_CODE_PAR_VALUE_TOO_LONG, pSchema->name); - } - if(pToken->type == TK_NK_HEX || pToken->type == TK_NK_BIN){ - pVal->value.pData = data; - }else{ - pVal->value.pData = taosStrdup(pToken->z); - } - pVal->value.nData = size; break; } case TSDB_DATA_TYPE_NCHAR: { @@ -1521,13 +1508,15 @@ static int32_t parseValueToken(SInsertParseContext* pCxt, const char** pSql, STo return code; } -static void clearColValArray(SArray* pCols) { +void clearColValArray(SArray* pCols) { int32_t num = taosArrayGetSize(pCols); for (int32_t i = 0; i < num; ++i) { SColVal* pCol = taosArrayGet(pCols, i); - if (IS_VAR_DATA_TYPE(pCol->type)) { + if (TSDB_DATA_TYPE_NCHAR == pCol->type || TSDB_DATA_TYPE_GEOMETRY == pCol->type) { taosMemoryFreeClear(pCol->value.pData); } + pCol->flag = CV_FLAG_NONE; + pCol->value.val = 0; } } diff --git a/source/libs/parser/src/parInsertUtil.c b/source/libs/parser/src/parInsertUtil.c index 33699ed857..5ae2cf12c9 100644 --- a/source/libs/parser/src/parInsertUtil.c +++ b/source/libs/parser/src/parInsertUtil.c @@ -333,8 +333,8 @@ int32_t insGetTableDataCxt(SHashObj* pHash, void* id, int32_t idLen, STableMeta* static void destroyColVal(void* p) { SColVal* pVal = p; - if (TSDB_DATA_TYPE_NCHAR == pVal->type || TSDB_DATA_TYPE_GEOMETRY == pVal->type) { - taosMemoryFree(pVal->value.pData); + if (TSDB_DATA_TYPE_NCHAR == pVal->type || TSDB_DATA_TYPE_GEOMETRY == pVal->type || TSDB_DATA_TYPE_VARBINARY == pVal->type) { + taosMemoryFreeClear(pVal->value.pData); } } diff --git a/source/libs/parser/src/parTokenizer.c b/source/libs/parser/src/parTokenizer.c index 41f6239455..c06615c083 100644 --- a/source/libs/parser/src/parTokenizer.c +++ b/source/libs/parser/src/parTokenizer.c @@ -545,7 +545,7 @@ uint32_t tGetToken(const char* z, uint32_t* tokenId) { return i; } else if (next == 'x') { // hex number *tokenId = TK_NK_HEX; - for (i = 2; isdigit(z[i]) || (z[i] >= 'a' && z[i] <= 'f') || (z[i] >= 'A' && z[i] <= 'F'); ++i) { + for (i = 2; isxdigit(z[i]) != 0; ++i) { } if (i == 2) { diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 0e8f29e051..fe15e62f9b 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1231,8 +1231,41 @@ static EDealRes translateNormalValue(STranslateContext* pCxt, SValueNode* pVal, *(double*)&pVal->typeData = pVal->datum.d; break; } + case TSDB_DATA_TYPE_VARBINARY: { + if (pVal->node.resType.type != TSDB_DATA_TYPE_BINARY){ + return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, pVal->literal); + } + + void* data = NULL; + uint32_t size = 0; + bool isHexChar = isHex(pVal->literal, strlen(pVal->literal)); + if(isHexChar){ + if(!isValidateHex(pVal->literal, strlen(pVal->literal))){ + return TSDB_CODE_PAR_INVALID_VARBINARY; + } + if(taosHex2Ascii(pVal->literal, strlen(pVal->literal), &data, &size) < 0){ + return TSDB_CODE_OUT_OF_MEMORY; + } + }else{ + size = pVal->node.resType.bytes; + data = pVal->literal; + } + + if (size + VARSTR_HEADER_SIZE > targetDt.bytes) { + if(isHexChar) taosMemoryFree(data); + return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_VALUE_TOO_LONG, pVal->literal); + } + pVal->datum.p = taosMemoryCalloc(1, size + VARSTR_HEADER_SIZE); + if (NULL == pVal->datum.p) { + if(isHexChar) taosMemoryFree(data); + return generateDealNodeErrMsg(pCxt, TSDB_CODE_OUT_OF_MEMORY); + } + varDataSetLen(pVal->datum.p, size + VARSTR_HEADER_SIZE); + memcpy(varDataVal(pVal->datum.p), data, size); + if(isHexChar) taosMemoryFree(data); + break; + } case TSDB_DATA_TYPE_VARCHAR: - case TSDB_DATA_TYPE_VARBINARY: case TSDB_DATA_TYPE_GEOMETRY: { if (strict && (pVal->node.resType.bytes > targetDt.bytes - VARSTR_HEADER_SIZE)) { return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, pVal->literal); @@ -1296,7 +1329,9 @@ static EDealRes translateValueImpl(STranslateContext* pCxt, SValueNode* pVal, SD res = translateNormalValue(pCxt, pVal, targetDt, strict); } pVal->node.resType.type = targetDt.type; - pVal->node.resType.bytes = targetDt.bytes; + if( targetDt.type == TSDB_DATA_TYPE_VARBINARY || !strict){ + pVal->node.resType.bytes = targetDt.bytes; + } pVal->node.resType.scale = pVal->unit; pVal->translate = true; if (!strict && TSDB_DATA_TYPE_UBIGINT == pVal->node.resType.type && pVal->datum.u <= INT64_MAX) { diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index 1c292b1ec4..b3ec6a6ef6 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -308,7 +308,7 @@ int32_t trimString(const char* src, int32_t len, char* dst, int32_t dlen) { dst[j] = '\''; } else if (src[k + 1] == '"') { dst[j] = '"'; - } else if (src[k + 1] == '%' || src[k + 1] == '_') { + } else if (src[k + 1] == '%' || src[k + 1] == '_' || src[k + 1] == 'x') { dst[j++] = src[k]; dst[j] = src[k + 1]; } else { diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 18dbb8d056..6cc235bd06 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -266,6 +266,7 @@ int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) { } else { /* normal relational comparFn */ comparFn = 30; } + break; } case TSDB_DATA_TYPE_BINARY: { if (optr == OP_TYPE_MATCH) { @@ -1595,7 +1596,7 @@ int32_t fltConverToStr(char *str, int type, void *buf, int32_t bufSize, int32_t break; case TSDB_DATA_TYPE_BINARY: -// case TSDB_DATA_TYPE_BINARY:// todovar + case TSDB_DATA_TYPE_VARBINARY: case TSDB_DATA_TYPE_NCHAR: case TSDB_DATA_TYPE_GEOMETRY: if (bufSize < 0) { diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index a11a9f9394..e12c62ad87 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -309,6 +309,47 @@ static FORCE_INLINE void varToBool(char *buf, SScalarParam *pOut, int32_t rowInd } // todo remove this malloc +static FORCE_INLINE void varToVarbinary(char *buf, SScalarParam *pOut, int32_t rowIndex, int32_t *overflow) { + terrno = TSDB_CODE_SUCCESS; + + if(isHex(varDataVal(buf), varDataLen(buf))){ + if(!isValidateHex(varDataVal(buf), varDataLen(buf))){ + terrno = TSDB_CODE_PAR_INVALID_VARBINARY; + return; + } + + void* data = NULL; + uint32_t size = 0; + if(taosHex2Ascii(varDataVal(buf), varDataLen(buf), &data, &size) < 0){ + terrno = TSDB_CODE_OUT_OF_MEMORY; + return; + } + int32_t inputLen = size + VARSTR_HEADER_SIZE; + char *t = taosMemoryCalloc(1, inputLen); + if (t == NULL) { + sclError("Out of memory"); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return; + } + varDataSetLen(t, size); + memcpy(varDataVal(t), data, size); + colDataSetVal(pOut->columnData, rowIndex, t, false); + taosMemoryFree(t); + taosMemoryFree(data); + }else{ + int32_t inputLen = varDataTLen(buf); + char *t = taosMemoryCalloc(1, inputLen); + if (t == NULL) { + sclError("Out of memory"); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return; + } + memcpy(t, buf, inputLen); + colDataSetVal(pOut->columnData, rowIndex, t, false); + taosMemoryFree(t); + } +} + static FORCE_INLINE void varToNchar(char *buf, SScalarParam *pOut, int32_t rowIndex, int32_t *overflow) { terrno = TSDB_CODE_SUCCESS; @@ -398,18 +439,21 @@ int32_t vectorConvertFromVarData(SSclVectorConvCtx *pCtx, int32_t *overflow) { func = varToUnsigned; } else if (IS_FLOAT_TYPE(pCtx->outType)) { func = varToFloat; - } else if (pCtx->outType == TSDB_DATA_TYPE_VARCHAR && + } else if ((pCtx->outType == TSDB_DATA_TYPE_VARCHAR || pCtx->outType == TSDB_DATA_TYPE_VARBINARY) && pCtx->inType == TSDB_DATA_TYPE_NCHAR) { // nchar -> binary func = ncharToVar; vton = true; } else if (pCtx->outType == TSDB_DATA_TYPE_NCHAR && - pCtx->inType == TSDB_DATA_TYPE_VARCHAR) { // binary -> nchar + (pCtx->inType == TSDB_DATA_TYPE_VARCHAR || pCtx->inType == TSDB_DATA_TYPE_VARBINARY)) { // binary -> nchar func = varToNchar; vton = true; } else if (TSDB_DATA_TYPE_TIMESTAMP == pCtx->outType) { func = varToTimestamp; } else if (TSDB_DATA_TYPE_GEOMETRY == pCtx->outType) { func = varToGeometry; + } else if (TSDB_DATA_TYPE_VARBINARY == pCtx->outType) { + func = varToVarbinary; + vton = true; } else { sclError("invalid convert outType:%d, inType:%d", pCtx->outType, pCtx->inType); terrno = TSDB_CODE_APP_ERROR; @@ -885,7 +929,7 @@ int32_t vectorConvertSingleColImpl(const SScalarParam *pIn, SScalarParam *pOut, break; } case TSDB_DATA_TYPE_BINARY: -// case TSDB_DATA_TYPE_VARBINARY //todovar + case TSDB_DATA_TYPE_VARBINARY: case TSDB_DATA_TYPE_NCHAR: case TSDB_DATA_TYPE_GEOMETRY: { return vectorConvertToVarData(&cCtx); @@ -901,26 +945,26 @@ int32_t vectorConvertSingleColImpl(const SScalarParam *pIn, SScalarParam *pOut, int8_t gConvertTypes[TSDB_DATA_TYPE_MAX][TSDB_DATA_TYPE_MAX] = { /* NULL BOOL TINY SMAL INT BIG FLOA DOUB VARC TIME NCHA UTIN USMA UINT UBIG JSON VARB DECI BLOB MEDB GEOM*/ /*NULL*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - /*BOOL*/ 0, 0, 2, 3, 4, 5, 6, 7, 5, 9, 7, 11, 12, 13, 14, 0, 7, 0, 0, 0, -1, - /*TINY*/ 0, 0, 0, 3, 4, 5, 6, 7, 5, 9, 7, 3, 4, 5, 7, 0, 7, 0, 0, 0, -1, - /*SMAL*/ 0, 0, 0, 0, 4, 5, 6, 7, 5, 9, 7, 3, 4, 5, 7, 0, 7, 0, 0, 0, -1, - /*INT */ 0, 0, 0, 0, 0, 5, 6, 7, 5, 9, 7, 4, 4, 5, 7, 0, 7, 0, 0, 0, -1, - /*BIGI*/ 0, 0, 0, 0, 0, 0, 6, 7, 5, 9, 7, 5, 5, 5, 7, 0, 7, 0, 0, 0, -1, - /*FLOA*/ 0, 0, 0, 0, 0, 0, 0, 7, 7, 6, 7, 6, 6, 6, 6, 0, 7, 0, 0, 0, -1, - /*DOUB*/ 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 7, 7, 7, 7, 7, 0, 7, 0, 0, 0, -1, - /*VARC*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 8, 7, 7, 7, 7, 0, 0, 0, 0, 0, 20, - /*TIME*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 7, 0, 7, 0, 0, 0, -1, - /*NCHA*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 7, 7, 0, 0, 0, 0, 0, -1, - /*UTIN*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 13, 14, 0, 7, 0, 0, 0, -1, - /*USMA*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 14, 0, 7, 0, 0, 0, -1, - /*UINT*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 7, 0, 0, 0, -1, - /*UBIG*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, -1, - /*JSON*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, - /*VARB*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, - /*DECI*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, - /*BLOB*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, - /*MEDB*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, - /*GEOM*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; + /*BOOL*/ 0, 0, 2, 3, 4, 5, 6, 7, 5, 9, 7, 11, 12, 13, 14, 0, -1, 0, 0, 0, -1, + /*TINY*/ 0, 0, 0, 3, 4, 5, 6, 7, 5, 9, 7, 3, 4, 5, 7, 0, -1, 0, 0, 0, -1, + /*SMAL*/ 0, 0, 0, 0, 4, 5, 6, 7, 5, 9, 7, 3, 4, 5, 7, 0, -1, 0, 0, 0, -1, + /*INT */ 0, 0, 0, 0, 0, 5, 6, 7, 5, 9, 7, 4, 4, 5, 7, 0, -1, 0, 0, 0, -1, + /*BIGI*/ 0, 0, 0, 0, 0, 0, 6, 7, 5, 9, 7, 5, 5, 5, 7, 0, -1, 0, 0, 0, -1, + /*FLOA*/ 0, 0, 0, 0, 0, 0, 0, 7, 7, 6, 7, 6, 6, 6, 6, 0, -1, 0, 0, 0, -1, + /*DOUB*/ 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 7, 7, 7, 7, 7, 0, -1, 0, 0, 0, -1, + /*VARC*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 8, 7, 7, 7, 7, 0, 16, 0, 0, 0, 20, + /*TIME*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 7, 0, -1, 0, 0, 0, -1, + /*NCHA*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 7, 7, 0, 16, 0, 0, 0, -1, + /*UTIN*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 13, 14, 0, -1, 0, 0, 0, -1, + /*USMA*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 14, 0, -1, 0, 0, 0, -1, + /*UINT*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, -1, 0, 0, 0, -1, + /*UBIG*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, 0, 0, 0, -1, + /*JSON*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, 0, 0, 0, -1, + /*VARB*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, -1,-1, -1, + /*DECI*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, 0, 0, 0, -1, + /*BLOB*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, 0, 0, 0, -1, + /*MEDB*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, 0, 0, 0, -1, + /*GEOM*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, 0, 0, 0, 0}; int32_t vectorGetConvertType(int32_t type1, int32_t type2) { if (type1 == type2) { diff --git a/source/libs/scalar/test/filter/filterTests.cpp b/source/libs/scalar/test/filter/filterTests.cpp index 51ee9b6570..cd3d681f58 100644 --- a/source/libs/scalar/test/filter/filterTests.cpp +++ b/source/libs/scalar/test/filter/filterTests.cpp @@ -1415,52 +1415,6 @@ void doCompareWithValueRange_OnlyLeftType(__compar_fn_t fp, int32_t rType) { } } -void doCompare(const std::vector &lTypes, const std::vector &rTypes, int32_t oper) { - for (int i = 0; i < lTypes.size(); ++i) { - for (int j = 0; j < rTypes.size(); ++j) { - auto fp = filterGetCompFuncEx(lTypes[i], rTypes[j], oper); - switch (lTypes[i]) { - case TSDB_DATA_TYPE_TINYINT: - doCompareWithValueRange_OnlyLeftType(fp, rTypes[j]); - break; - case TSDB_DATA_TYPE_SMALLINT: - doCompareWithValueRange_OnlyLeftType(fp, rTypes[j]); - break; - case TSDB_DATA_TYPE_INT: - doCompareWithValueRange_OnlyLeftType(fp, rTypes[j]); - break; - case TSDB_DATA_TYPE_BIGINT: - doCompareWithValueRange_OnlyLeftType(fp, rTypes[j]); - break; - case TSDB_DATA_TYPE_UTINYINT: - doCompareWithValueRange_OnlyLeftType(fp, rTypes[j]); - break; - case TSDB_DATA_TYPE_USMALLINT: - doCompareWithValueRange_OnlyLeftType(fp, rTypes[j]); - break; - case TSDB_DATA_TYPE_UINT: - doCompareWithValueRange_OnlyLeftType(fp, rTypes[j]); - break; - case TSDB_DATA_TYPE_UBIGINT: - doCompareWithValueRange_OnlyLeftType(fp, rTypes[j]); - break; - default: - FAIL(); - } - } - } -} - -TEST(dataCompareTest, signed_and_unsigned_int) { - std::vector lType = {TSDB_DATA_TYPE_TINYINT, TSDB_DATA_TYPE_SMALLINT, TSDB_DATA_TYPE_INT, - TSDB_DATA_TYPE_BIGINT}; - std::vector rType = {TSDB_DATA_TYPE_UTINYINT, TSDB_DATA_TYPE_USMALLINT, TSDB_DATA_TYPE_UINT, - TSDB_DATA_TYPE_UBIGINT}; - - doCompare(lType, rType, OP_TYPE_GREATER_THAN); - doCompare(rType, lType, OP_TYPE_GREATER_THAN); -} - int main(int argc, char **argv) { taosSeedRand(taosGetTimestampSec()); testing::InitGoogleTest(&argc, argv); diff --git a/source/os/src/osString.c b/source/os/src/osString.c index cb01332525..8aac606473 100644 --- a/source/os/src/osString.c +++ b/source/os/src/osString.c @@ -469,10 +469,28 @@ float taosStr2Float(const char *str, char **pEnd) { return tmp; } +#define HEX_PREFIX_LEN 2 // \x +bool isHex(const char* z, uint32_t n){ + if(n < HEX_PREFIX_LEN) return false; + if(z[0] == '\\' && z[1] == 'x') return true; + return false; +} + +bool isValidateHex(const char* z, uint32_t n){ + if(n % 2 != 0) return false; + for(size_t i = HEX_PREFIX_LEN; i < n; i++){ + if(isxdigit(z[i]) == 0){ + return false; + } + } + return true; +} + int32_t taosHex2Ascii(const char *z, uint32_t n, void** data, uint32_t* size){ - n -= 2; // remove 0x - z += 2; - *size = n%2 == 0 ? n/2 : n/2 + 1; + n -= HEX_PREFIX_LEN; // remove 0x + z += HEX_PREFIX_LEN; + *size = n / HEX_PREFIX_LEN; + if(*size == 0) return 0; uint8_t* tmp = (uint8_t*)taosMemoryCalloc(*size, 1); if(tmp == NULL) return -1; int8_t num = 0; @@ -497,27 +515,31 @@ int32_t taosHex2Ascii(const char *z, uint32_t n, void** data, uint32_t* size){ return 0; } -int32_t taosBin2Ascii(const char *z, uint32_t n, void** data, uint32_t* size){ - n -= 2; // remove 0b - z += 2; - *size = n%8 == 0 ? n/8 : n/8 + 1; - uint8_t* tmp = (uint8_t*)taosMemoryCalloc(*size, 1); - if(tmp == NULL) return -1; - int8_t num = 0; - uint8_t *byte = tmp + *size - 1; - - for (int i = n - 1; i >= 0; i--) { - *byte |= ((uint8_t)(z[i] - '0') << num); - if (num == 8) { - byte--; - num = 0; - } else { - num++; - } - } - *data = tmp; - return 0; -} +//int32_t taosBin2Ascii(const char *z, uint32_t n, void** data, uint32_t* size){ +// +// for (i = 2; isdigit(z[i]) || (z[i] >= 'a' && z[i] <= 'f') || (z[i] >= 'A' && z[i] <= 'F'); ++i) { +// } +// +// n -= 2; // remove 0b +// z += 2; +// *size = n%8 == 0 ? n/8 : n/8 + 1; +// uint8_t* tmp = (uint8_t*)taosMemoryCalloc(*size, 1); +// if(tmp == NULL) return -1; +// int8_t num = 0; +// uint8_t *byte = tmp + *size - 1; +// +// for (int i = n - 1; i >= 0; i--) { +// *byte |= ((uint8_t)(z[i] - '0') << num); +// if (num == 7) { +// byte--; +// num = 0; +// } else { +// num++; +// } +// } +// *data = tmp; +// return 0; +//} static char valueOf(uint8_t symbol) { @@ -547,12 +569,12 @@ static char valueOf(uint8_t symbol) } int32_t taosAscii2Hex(const char *z, uint32_t n, void** data, uint32_t* size){ - *size = n * 2 + 2; - uint8_t* tmp = (uint8_t*)taosMemoryCalloc(*size, 1); + *size = n * 2 + HEX_PREFIX_LEN; + uint8_t* tmp = (uint8_t*)taosMemoryCalloc(*size + 1, 1); if(tmp == NULL) return -1; *data = tmp; - *(tmp++) = '0'; - *(tmp++) = 'X'; + *(tmp++) = '\\'; + *(tmp++) = 'x'; for(int i = 0; i < n; i ++){ uint8_t val = z[i]; tmp[i*2] = valueOf(val >> 4); diff --git a/source/util/src/tcompare.c b/source/util/src/tcompare.c index 9e2fc74366..4bacda48d2 100644 --- a/source/util/src/tcompare.c +++ b/source/util/src/tcompare.c @@ -242,6 +242,10 @@ int32_t compareLenBinaryVal(const void *pLeft, const void *pRight) { } } +int32_t compareLenBinaryValDesc(const void *pLeft, const void *pRight) { + return compareLenBinaryVal(pRight, pLeft); +} + // string > number > bool > null // ref: https://dev.mysql.com/doc/refman/8.0/en/json.html#json-comparison int32_t compareJsonVal(const void *pLeft, const void *pRight) { @@ -1464,6 +1468,8 @@ __compar_fn_t getKeyComparFunc(int32_t keyType, int32_t order) { return (order == TSDB_ORDER_ASC) ? compareUint32Val : compareUint32ValDesc; case TSDB_DATA_TYPE_UBIGINT: return (order == TSDB_ORDER_ASC) ? compareUint64Val : compareUint64ValDesc; + case TSDB_DATA_TYPE_VARBINARY: + return (order == TSDB_ORDER_ASC) ? compareLenBinaryVal : compareLenBinaryValDesc; case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_GEOMETRY: return (order == TSDB_ORDER_ASC) ? compareLenPrefixedStr : compareLenPrefixedStrDesc; @@ -1475,57 +1481,3 @@ __compar_fn_t getKeyComparFunc(int32_t keyType, int32_t order) { return (order == TSDB_ORDER_ASC) ? compareInt32Val : compareInt32ValDesc; } } - -int32_t doCompare(const char *f1, const char *f2, int32_t type, size_t size) { - switch (type) { - case TSDB_DATA_TYPE_INT: - DEFAULT_COMP(GET_INT32_VAL(f1), GET_INT32_VAL(f2)); - case TSDB_DATA_TYPE_DOUBLE: - DEFAULT_DOUBLE_COMP(GET_DOUBLE_VAL(f1), GET_DOUBLE_VAL(f2)); - case TSDB_DATA_TYPE_FLOAT: - DEFAULT_FLOAT_COMP(GET_FLOAT_VAL(f1), GET_FLOAT_VAL(f2)); - case TSDB_DATA_TYPE_BIGINT: - DEFAULT_COMP(GET_INT64_VAL(f1), GET_INT64_VAL(f2)); - case TSDB_DATA_TYPE_SMALLINT: - DEFAULT_COMP(GET_INT16_VAL(f1), GET_INT16_VAL(f2)); - case TSDB_DATA_TYPE_TINYINT: - case TSDB_DATA_TYPE_BOOL: - DEFAULT_COMP(GET_INT8_VAL(f1), GET_INT8_VAL(f2)); - case TSDB_DATA_TYPE_UTINYINT: - DEFAULT_COMP(GET_UINT8_VAL(f1), GET_UINT8_VAL(f2)); - case TSDB_DATA_TYPE_USMALLINT: - DEFAULT_COMP(GET_UINT16_VAL(f1), GET_UINT16_VAL(f2)); - case TSDB_DATA_TYPE_UINT: - DEFAULT_COMP(GET_UINT32_VAL(f1), GET_UINT32_VAL(f2)); - case TSDB_DATA_TYPE_UBIGINT: - DEFAULT_COMP(GET_UINT64_VAL(f1), GET_UINT64_VAL(f2)); - case TSDB_DATA_TYPE_NCHAR: { - tstr *t1 = (tstr *)f1; - tstr *t2 = (tstr *)f2; - - if (t1->len != t2->len) { - return t1->len > t2->len ? 1 : -1; - } - int32_t ret = memcmp((TdUcs4 *)t1, (TdUcs4 *)t2, t2->len); - if (ret == 0) { - return ret; - } - return (ret < 0) ? -1 : 1; - } - default: { // todo refactor - tstr *t1 = (tstr *)f1; - tstr *t2 = (tstr *)f2; - - if (t1->len != t2->len) { - return t1->len > t2->len ? 1 : -1; - } else { - int32_t ret = strncmp(t1->data, t2->data, t1->len); - if (ret == 0) { - return 0; - } else { - return ret < 0 ? -1 : 1; - } - } - } - } -} diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 466b9985e7..998993bf65 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -555,6 +555,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_PAR_NOT_ALLOWED_WIN_QUERY, "Window query not su TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_DROP_COL, "No columns can be dropped") TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_COL_JSON, "Only tag can be json type") TAOS_DEFINE_ERROR(TSDB_CODE_PAR_VALUE_TOO_LONG, "Value too long for column/tag") +TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_VARBINARY, "Invalidate varbinary type") TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_DELETE_WHERE, "The DELETE statement must have a definite time window range") TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_REDISTRIBUTE_VG, "The REDISTRIBUTE VGROUP statement only support 1 to 3 dnodes") TAOS_DEFINE_ERROR(TSDB_CODE_PAR_FILL_NOT_ALLOWED_FUNC, "Fill not allowed") diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 741226f101..e4303b2e15 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -351,6 +351,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/smaTest.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/sma_index.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sml_TS-3724.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/varbinary.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sml.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sml.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/spread.py diff --git a/tests/system-test/2-query/sml.py b/tests/system-test/2-query/sml.py index cae012ece1..53ac85bd8e 100644 --- a/tests/system-test/2-query/sml.py +++ b/tests/system-test/2-query/sml.py @@ -1,18 +1,12 @@ -import taos import sys -import time -import socket import os -import threading from util.log import * from util.sql import * from util.cases import * from util.dnodes import * from util.common import * -sys.path.append("./7-tmq") -from tmqCommon import * class TDTestCase: updatecfgDict = {'clientCfg': {'smlChildTableName': 'dataModelName', 'fqdn': 'localhost', 'smlDot2Underline': 0}, 'fqdn': 'localhost'} @@ -32,7 +26,7 @@ class TDTestCase: tdLog.info(cmdStr) ret = os.system(cmdStr) if ret != 0: - tdLog.info("sml_test ret != 0") + tdLog.exit("sml_test ret != 0") tdSql.query(f"select * from ts3303.stb2") tdSql.query(f"select * from ts3303.meters") diff --git a/tests/system-test/2-query/varbinary.py b/tests/system-test/2-query/varbinary.py new file mode 100644 index 0000000000..8f7032cdc3 --- /dev/null +++ b/tests/system-test/2-query/varbinary.py @@ -0,0 +1,80 @@ +import sys +import os + +from util.log import * +from util.sql import * +from util.cases import * +from util.common import * + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), False) + + def test(self): + tdLog.info(" test") + + buildPath = tdCom.getBuildPath() + cmdStr = '%s/build/bin/varbinary_test'%(buildPath) + print("cmdStr:", cmdStr) + tdLog.info(cmdStr) + ret = os.system(cmdStr) + if ret != 0: + tdLog.exit("varbinary_test ret != 0") + + tdSql.execute(f" create database test") + tdSql.execute(f" use test ") + tdSql.execute(f" create stable stb (ts timestamp, c1 nchar(32), c2 varbinary(16), c3 float) tags (t1 int, t2 binary(8), t3 varbinary(8))") + + tdSql.query(f"desc stb") + tdSql.checkRows(7) + tdSql.checkData(2, 1, 'VARBINARY') + tdSql.checkData(2, 2, 16) + tdSql.checkData(6, 1, 'VARBINARY') + tdSql.checkData(6, 2, 8) + + # tdSql.execute(f" insert into tb1 using stb tags (1, 'tb1_bin1', 'vart1') values (now, 'nchar1', 'varc1', 0.3)") + # tdSql.execute(f" insert into tb1 values (now + 1s, 'nchar2', null, 0.4)") + # + # tdSql.execute(f" insert into tb2 using stb tags (2, 'tb2_bin1', 093) values (now + 2s, 'nchar1', 892, 0.3)") + # tdSql.execute(f" insert into tb3 using stb tags (3, 'tb3_bin1', 0x7f829) values (now + 3s, 'nchar1', 0x7f829, 0.3)") + # tdSql.execute(f" insert into tb4 using stb tags (4, 'tb4_bin1', 0b100000010) values (now + 4s, 'nchar1', 0b110000001, 0.3)") + # tdSql.execute(f" insert into tb4 values (now + 5s, 'nchar1', 0b11000000100000000, 0.3)") + # tdSql.execute(f" insert into tb5 using stb tags (4, 'tb5_bin1', NULL) values (now + 6s, 'nchar1', 0b10100000011000000110000001, 0.3)") + + # basic query + # tdSql.query(f"select c2,t3 from stb order by ts") + # tdSql.checkRows(6) + # tdSql.checkData(0, 0, '0x7661726331') + # tdSql.checkData(0, 1, '0x7661727431') + # tdSql.checkData(1, 0, None) + # tdSql.checkData(1, 1, '0x7661727431') + # tdSql.checkData(2, 0, '0x383932') + # tdSql.checkData(2, 1, '0x303933') + # tdSql.checkData(3, 0, '0x07f829') + # tdSql.checkData(3, 1, '0x07f829') + # tdSql.checkData(4, 0, '0x0181') + # tdSql.checkData(4, 1, '0x0102') + # tdSql.checkData(5, 0, '0x02818181') + # tdSql.checkData(5, 1, None) + + # tdSql.query(f"select ts,c2 from stb order by c2") + # + # tdSql.query(f"select c2,t3 from stb where c2 >= 0 order by ts") + # + # tdSql.query(f"select c2,t3 from stb where c2 >= 0x0181 order by ts") + + + + def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring + tdSql.prepare() + self.test() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index f1a9a1d2bf..35ab86269d 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -323,13 +323,12 @@ void shellDumpFieldToFile(TdFilePtr pFile, const char *val, TAOS_FIELD *field, i return; } - char quotationStr[2]; - quotationStr[0] = '\"'; - quotationStr[1] = 0; + char quotationStr[2] ={'"', 0}; int32_t width; - int n; - char buf[TSDB_MAX_BYTES_PER_ROW]; + int n = 0; +#define LENGTH 64 + char buf[LENGTH] = {0}; switch (field->type) { case TSDB_DATA_TYPE_BOOL: taosFprintfFile(pFile, "%d", ((((int32_t)(*((char *)val))) == 1) ? 1 : 0)); @@ -363,7 +362,7 @@ void shellDumpFieldToFile(TdFilePtr pFile, const char *val, TAOS_FIELD *field, i if (tsEnableScience) { taosFprintfFile(pFile, "%*.7e", width, GET_FLOAT_VAL(val)); } else { - n = snprintf(buf, TSDB_MAX_BYTES_PER_ROW, "%*.7f", width, GET_FLOAT_VAL(val)); + n = snprintf(buf, LENGTH, "%*.7f", width, GET_FLOAT_VAL(val)); if (n > SHELL_FLOAT_WIDTH) { taosFprintfFile(pFile, "%*.7e", width, GET_FLOAT_VAL(val)); } else { @@ -374,10 +373,10 @@ void shellDumpFieldToFile(TdFilePtr pFile, const char *val, TAOS_FIELD *field, i case TSDB_DATA_TYPE_DOUBLE: width = SHELL_DOUBLE_WIDTH; if (tsEnableScience) { - snprintf(buf, TSDB_MAX_BYTES_PER_ROW, "%*.15e", width, GET_DOUBLE_VAL(val)); + snprintf(buf, LENGTH, "%*.15e", width, GET_DOUBLE_VAL(val)); taosFprintfFile(pFile, "%s", buf); } else { - n = snprintf(buf, TSDB_MAX_BYTES_PER_ROW, "%*.15f", width, GET_DOUBLE_VAL(val)); + n = snprintf(buf, LENGTH, "%*.15f", width, GET_DOUBLE_VAL(val)); if (n > SHELL_DOUBLE_WIDTH) { taosFprintfFile(pFile, "%*.15e", width, GET_DOUBLE_VAL(val)); } else { @@ -386,26 +385,42 @@ void shellDumpFieldToFile(TdFilePtr pFile, const char *val, TAOS_FIELD *field, i } break; case TSDB_DATA_TYPE_BINARY: -// case TSDB_DATA_TYPE_VARBINARY todovar case TSDB_DATA_TYPE_NCHAR: case TSDB_DATA_TYPE_JSON: { int32_t bufIndex = 0; + char* tmp = (char*)taosMemoryCalloc(length * 2 + 1, 1); + if(tmp == NULL) break; for (int32_t i = 0; i < length; i++) { - buf[bufIndex] = val[i]; + tmp[bufIndex] = val[i]; bufIndex++; if (val[i] == '\"') { - buf[bufIndex] = val[i]; + tmp[bufIndex] = val[i]; bufIndex++; } } - buf[bufIndex] = 0; + tmp[bufIndex] = 0; - taosFprintfFile(pFile, "%s%s%s", quotationStr, buf, quotationStr); + taosFprintfFile(pFile, "%s%s%s", quotationStr, tmp, quotationStr); + taosMemoryFree(tmp); } break; - case TSDB_DATA_TYPE_GEOMETRY: - shellDumpHexValue(buf, val, length); - taosFprintfFile(pFile, "%s", buf); + case TSDB_DATA_TYPE_VARBINARY:{ + void* tmp = NULL; + uint32_t size = 0; + if(taosAscii2Hex(val, length, &tmp, &size) < 0){ + break; + } + taosFprintfFile(pFile, "%s", tmp); + taosMemoryFree(tmp); break; + } + case TSDB_DATA_TYPE_GEOMETRY:{ + char* tmp = (char*)taosMemoryCalloc(length * 2 + 1, 1); + if(tmp == NULL) break; + shellDumpHexValue(tmp, val, length); + taosFprintfFile(pFile, "%s", buf); + taosMemoryFree(tmp); + break; + } case TSDB_DATA_TYPE_TIMESTAMP: shellFormatTimestamp(buf, *(int64_t *)val, precision); taosFprintfFile(pFile, "%s%s%s", quotationStr, buf, quotationStr); @@ -779,14 +794,20 @@ int32_t shellCalcColWidth(TAOS_FIELD *field, int32_t precision) { return TMAX(SHELL_DOUBLE_WIDTH, width); case TSDB_DATA_TYPE_BINARY: - case TSDB_DATA_TYPE_VARBINARY: case TSDB_DATA_TYPE_GEOMETRY: if (field->bytes > shell.args.displayWidth) { return TMAX(shell.args.displayWidth, width); } else { return TMAX(field->bytes + 2, width); } - + case TSDB_DATA_TYPE_VARBINARY:{ + int32_t bytes = field->bytes * 2 + 2; + if (bytes > shell.args.displayWidth) { + return TMAX(shell.args.displayWidth, width); + } else { + return TMAX(bytes + 2, width); + } + } case TSDB_DATA_TYPE_NCHAR: case TSDB_DATA_TYPE_JSON: { uint16_t bytes = field->bytes * TSDB_NCHAR_SIZE; diff --git a/utils/test/c/CMakeLists.txt b/utils/test/c/CMakeLists.txt index b96814c13b..43e97a3467 100644 --- a/utils/test/c/CMakeLists.txt +++ b/utils/test/c/CMakeLists.txt @@ -8,6 +8,7 @@ add_executable(sml_test sml_test.c) add_executable(get_db_name_test get_db_name_test.c) add_executable(tmq_offset tmqOffset.c) add_executable(tmq_offset_test tmq_offset_test.c) +add_executable(varbinary_test varbinary_test.c) target_link_libraries( tmq_offset PUBLIC taos @@ -65,6 +66,8 @@ target_link_libraries( PUBLIC util PUBLIC common PUBLIC os + PUBLIC geometry + ) target_link_libraries( @@ -74,3 +77,11 @@ target_link_libraries( PUBLIC common PUBLIC os ) + +target_link_libraries( + varbinary_test + PUBLIC taos + PUBLIC util + PUBLIC common + PUBLIC os +) diff --git a/utils/test/c/sml_test.c b/utils/test/c/sml_test.c index 237bfc5092..105fdd2ca4 100644 --- a/utils/test/c/sml_test.c +++ b/utils/test/c/sml_test.c @@ -21,6 +21,7 @@ #include "taos.h" #include "tlog.h" #include "types.h" +#include "geosWrapper.h" int smlProcess_influx_Test() { TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); @@ -1578,6 +1579,83 @@ int sml_td24559_Test() { printf("%s result0:%s\n", __FUNCTION__, taos_errstr(pRes)); taos_free_result(pRes); + TAOS_ROW row = NULL; + pRes = taos_query(taos, "select * from stb order by _ts;"); + int rowIndex = 0; + while ((row = taos_fetch_row(pRes)) != NULL) { + + int32_t* length = taos_fetch_lengths(pRes); + + code = initCtxAsText(); + ASSERT (code == TSDB_CODE_SUCCESS); + char *outputWKT = NULL; + code = doAsText(row[2], length[2], &outputWKT); + ASSERT (code == TSDB_CODE_SUCCESS); + + if (rowIndex == 0) { + ASSERT(strcmp("POINT (4.343000 89.342000)", outputWKT) == 0); + } + if (rowIndex == 3) { + ASSERT(strcmp( "GEOMETRYCOLLECTION (MULTIPOINT ((0.000000 0.000000), (1.000000 1.000000)), POINT (3.000000 4.000000), LINESTRING (2.000000 3.000000, 3.000000 4.000000))", outputWKT) == 0); + } + geosFreeBuffer(outputWKT); + + rowIndex++; + } + taos_free_result(pRes); + + taos_close(taos); + + return code; +} + +int sml_td18789_Test() { + TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + + TAOS_RES *pRes = taos_query(taos, "drop database if exists td18789"); + taos_free_result(pRes); + + pRes = taos_query(taos, "create database if not exists td18789"); + taos_free_result(pRes); + + const char *sql[] = { + "vbin,t1=1 f1=283i32,f2=b\"hello\" 1632299372000", + "vbin,t1=1 f2=B\"\\x98f46e\",f1=106i32 1632299373000", + }; + + pRes = taos_query(taos, "use td18789"); + taos_free_result(pRes); + + pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_LINE_PROTOCOL, + TSDB_SML_TIMESTAMP_MILLI_SECONDS); + + int code = taos_errno(pRes); + printf("%s result0:%s\n", __FUNCTION__, taos_errstr(pRes)); + taos_free_result(pRes); + + TAOS_ROW row = NULL; + pRes = taos_query(taos, "select *,tbname from vbin order by _ts"); + int rowIndex = 0; + while ((row = taos_fetch_row(pRes)) != NULL) { + int32_t* length = taos_fetch_lengths(pRes); + void* data = NULL; + uint32_t size = 0; + if(taosAscii2Hex(row[2], length[2], &data, &size) < 0){ + ASSERT(0); + } + + if (rowIndex == 0) { + ASSERT(memcmp(data, "\\x68656C6C6F", size) == 0); + } + if (rowIndex == 1) { + ASSERT(memcmp(data, "\\x98F46E", size) == 0); + } + taosMemoryFree(data); + + rowIndex++; + } + taos_free_result(pRes); + taos_close(taos); return code; @@ -1591,6 +1669,8 @@ int main(int argc, char *argv[]) { int ret = 0; ret = sml_td24559_Test(); ASSERT(!ret); + ret = sml_td18789_Test(); + ASSERT(!ret); ret = sml_td24070_Test(); ASSERT(!ret); ret = sml_td23881_Test(); diff --git a/utils/test/c/varbinary_test.c b/utils/test/c/varbinary_test.c new file mode 100644 index 0000000000..b2fccee63b --- /dev/null +++ b/utils/test/c/varbinary_test.c @@ -0,0 +1,253 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include +#include +#include +#include +#include "taos.h" +#include "types.h" +#include "tlog.h" + +#define GET_ROW_NUM \ + numRows = 0;\ + while(1){\ + row = taos_fetch_row(pRes);\ + if (row != NULL){\ + numRows++;\ + }else{\ + break;\ + }\ + } +int varbinary_test() { + TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + + TAOS_RES *pRes = taos_query(taos, "drop database if exists varbinary_db"); + taos_free_result(pRes); + + pRes = taos_query(taos, "create database if not exists varbinary_db"); + taos_free_result(pRes); + + pRes = taos_query(taos, "use varbinary_db"); + int code = taos_errno(pRes); + taos_free_result(pRes); + ASSERT(code == 0); + + pRes = taos_query(taos, "create stable stb (ts timestamp, c1 nchar(32), c2 varbinary(16), c3 float) tags (t1 int, t2 binary(8), t3 varbinary(8))"); + taos_free_result(pRes); + + pRes = taos_query(taos, "desc stb"); + + TAOS_ROW row = NULL; + int32_t rowIndex = 0; + while ((row = taos_fetch_row(pRes)) != NULL) { + char* type = row[1]; + int32_t length = *(int32_t *)row[2]; + + if (rowIndex == 2) { + ASSERT(strncmp(type, "VARBINARY", sizeof("VARBINARY") - 1) == 0); + ASSERT(length == 16); + } + + if (rowIndex == 6) { + ASSERT(strncmp(type, "VARBINARY", sizeof("VARBINARY") - 1) == 0); + ASSERT(length == 8); + } + rowIndex++; + } + + pRes = taos_query(taos, "insert into tb1 using stb tags (1, 'tb1_bin1', 'vart1') values (now, 'nchar1', 'varc1', 0.3)"); + taos_free_result(pRes); + pRes = taos_query(taos, "insert into tb1 values (now + 1s, 'nchar2', null, 0.4);"); + taos_free_result(pRes); + pRes = taos_query(taos, "insert into tb3 using stb tags (3, 'tb3_bin1', '\\x7f8290') values (now + 2s, 'nchar1', '\\x7f8290', 0.3)"); + taos_free_result(pRes); + pRes = taos_query(taos, "insert into tb3 values (now + 3s, 'nchar1', '\\x7f829000', 0.3)"); + taos_free_result(pRes); + pRes = taos_query(taos, "insert into tb2 using stb tags (2, 'tb2_bin1', '\\x') values (now + 4s, 'nchar1', '\\x', 0.3)"); + taos_free_result(pRes); + pRes = taos_query(taos, "insert into tb2 values (now + 5s, 'nchar1', '\\x00000000', 0.3)"); + taos_free_result(pRes); + + // test insert + pRes = taos_query(taos, "insert into tb2 using stb tags (2, 'tb2_bin1', 093) values (now + 2s, 'nchar1', 892, 0.3)"); + ASSERT(taos_errno(pRes) != 0); + + pRes = taos_query(taos, "insert into tb3 using stb tags (3, 'tb3_bin1', 0x7f829) values (now + 3s, 'nchar1', 0x7f829, 0.3)"); + ASSERT(taos_errno(pRes) != 0); + + pRes = taos_query(taos, "insert into tb3 using stb tags (3, 'tb3_bin1', '\\x7f829') values (now + 3s, 'nchar1', '\\x7f829', 0.3)"); + ASSERT(taos_errno(pRes) != 0); + + pRes = taos_query(taos, "insert into tb4 using stb tags (4, 'tb4_bin1', 0b100000010) values (now + 4s, 'nchar1', 0b110000001, 0.3)"); + ASSERT(taos_errno(pRes) != 0); + taos_free_result(pRes); + + + // test error + pRes = taos_query(taos, "select * from tb1 where c2 >= 0x8de6"); + ASSERT(taos_errno(pRes) != 0); + taos_free_result(pRes); + + pRes = taos_query(taos, "select * from tb1 where c2 >= 0"); + ASSERT(taos_errno(pRes) != 0); + taos_free_result(pRes); + + pRes = taos_query(taos, "select * from stb where c2 > '\\x7F82900'"); + ASSERT(taos_errno(pRes) != 0); + taos_free_result(pRes); + + pRes = taos_query(taos, "select c2+2 from stb"); + ASSERT(taos_errno(pRes) != 0); + taos_free_result(pRes); + + pRes = taos_query(taos, "select c2|2 from stb"); + ASSERT(taos_errno(pRes) != 0); + taos_free_result(pRes); + + pRes = taos_query(taos, "select * from stb where c2 not like 's%'"); + ASSERT(taos_errno(pRes) != 0); + taos_free_result(pRes); + + pRes = taos_query(taos, "select * from stb where c2 like 's%'"); + ASSERT(taos_errno(pRes) != 0); + taos_free_result(pRes); + + pRes = taos_query(taos, "select * from stb where c2 match 'ssd'"); + ASSERT(taos_errno(pRes) != 0); + taos_free_result(pRes); + + pRes = taos_query(taos, "select * from stb where c2 nmatch 'ssd'"); + ASSERT(taos_errno(pRes) != 0); + taos_free_result(pRes); + + pRes = taos_query(taos, "select * from stb where c2->'ssd' = 1"); + ASSERT(taos_errno(pRes) != 0); + taos_free_result(pRes); + + pRes = taos_query(taos, "select * from stb where c2 contains 'ssd'"); + ASSERT(taos_errno(pRes) != 0); + taos_free_result(pRes); + + pRes = taos_query(taos, "select * from stb where sum(c2) = 2"); + ASSERT(taos_errno(pRes) != 0); + taos_free_result(pRes); + +// pRes = taos_query(taos, "select * from stb where c2 contains 'ssd'"); +// ASSERT(taos_errno(pRes) != 0); +// taos_free_result(pRes); +// +// pRes = taos_query(taos, "select * from stb where c2 contains 'ssd'"); +// ASSERT(taos_errno(pRes) != 0); +// taos_free_result(pRes); + + int numRows = 0; + + pRes = taos_query(taos, "select * from stb where c2 > 'varc1'"); + GET_ROW_NUM + ASSERT(numRows == 2); + taos_free_result(pRes); + + pRes = taos_query(taos, "select * from stb where c2 > '\\x7F8290'"); + GET_ROW_NUM + ASSERT(numRows == 1); + taos_free_result(pRes); + + pRes = taos_query(taos, "select * from stb where c2 in ('\\x7F829000','\\x00000000')"); + GET_ROW_NUM + ASSERT(numRows == 2); + taos_free_result(pRes); + + pRes = taos_query(taos, "select * from stb where c2 not in ('\\x00000000')"); + GET_ROW_NUM + ASSERT(numRows == 4); + taos_free_result(pRes); + + pRes = taos_query(taos, "select * from stb where c2 is null"); + GET_ROW_NUM + ASSERT(numRows == 1); + taos_free_result(pRes); + + pRes = taos_query(taos, "select * from stb where c2 is not null"); + GET_ROW_NUM + ASSERT(numRows == 5); + taos_free_result(pRes); + + pRes = taos_query(taos, "select * from stb where c2 between '\\x3e' and '\\x7F8290'"); + GET_ROW_NUM + ASSERT(numRows == 2); + taos_free_result(pRes); + + + pRes = taos_query(taos, "select * from stb where c2 not between '\\x3e' and '\\x7F8290'"); + GET_ROW_NUM + ASSERT(numRows == 3); + taos_free_result(pRes); + + pRes = taos_query(taos, "select ts,c2 from stb order by c2"); + rowIndex = 0; + while ((row = taos_fetch_row(pRes)) != NULL) { +// int64_t ts = *(int64_t *)row[0]; + if (rowIndex == 0) { + ASSERT(row[1] == NULL); + rowIndex++; + continue; + } + int32_t* length = taos_fetch_lengths(pRes); + void* data = NULL; + uint32_t size = 0; + if(taosAscii2Hex(row[1], length[1], &data, &size) < 0){ + ASSERT(0); + } + + if (rowIndex == 1) { +// ASSERT(ts == 1661943960000); + ASSERT(memcmp(data, "\\x", size) == 0); + } + if (rowIndex == 2) { +// ASSERT(ts == 1661943960000); + ASSERT(memcmp(data, "\\x00000000", size) == 0); + } + if (rowIndex == 3) { +// ASSERT(ts == 1661943960000); + ASSERT(memcmp(data, "\\x7661726331", size) == 0); + } + if (rowIndex == 4) { +// ASSERT(ts == 1661943960000); + ASSERT(memcmp(data, "\\x7F8290", size) == 0); + } + if (rowIndex == 5) { +// ASSERT(ts == 1661943960000); + ASSERT(memcmp(data, "\\x7F829000", size) == 0); + } + taosMemoryFree(data); + + rowIndex++; + } + printf("%s result1:%s\n", __FUNCTION__, taos_errstr(pRes)); + taos_free_result(pRes); + + taos_close(taos); + + return code; +} + +int main(int argc, char *argv[]) { + int ret = 0; + ret = varbinary_test(); + ASSERT(!ret); + return ret; +} diff --git a/utils/tsim/src/simExe.c b/utils/tsim/src/simExe.c index 72b8eff27d..9a0a156717 100644 --- a/utils/tsim/src/simExe.c +++ b/utils/tsim/src/simExe.c @@ -751,7 +751,6 @@ bool simExecuteNativeSqlCommand(SScript *script, char *rest, bool isSlow) { sprintf(value, "%.9lf", GET_DOUBLE_VAL(row[i])); break; case TSDB_DATA_TYPE_BINARY: -// case TSDB_DATA_TYPE_VARBINARY todovar case TSDB_DATA_TYPE_NCHAR: case TSDB_DATA_TYPE_GEOMETRY: if (length[i] < 0 || length[i] > 1 << 20) { From 9cd8df12c23d104b943558b9c2fa240d732985a2 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 25 Aug 2023 16:32:01 +0800 Subject: [PATCH 08/13] fix:memory leak --- include/libs/parser/parser.h | 2 +- source/client/src/clientSmlJson.c | 4 ++-- source/client/src/clientSmlLine.c | 2 +- source/client/src/clientSmlTelnet.c | 2 +- source/libs/parser/src/parInsertSml.c | 14 +++++++++++++- source/libs/parser/src/parInsertSql.c | 6 ++---- 6 files changed, 20 insertions(+), 10 deletions(-) diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index e6a67e68f0..2dc5c8f112 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -114,12 +114,12 @@ int32_t smlBuildRow(STableDataCxt* pTableCxt); int32_t smlBuildCol(STableDataCxt* pTableCxt, SSchema* schema, void* kv, int32_t index); STableDataCxt* smlInitTableDataCtx(SQuery* query, STableMeta* pTableMeta); +void clearColValArraySml(SArray* pCols); int32_t smlBindData(SQuery* handle, bool dataFormat, SArray* tags, SArray* colsSchema, SArray* cols, STableMeta* pTableMeta, char* tableName, const char* sTableName, int32_t sTableNameLen, int32_t ttl, char* msgBuf, int32_t msgBufLen); int32_t smlBuildOutput(SQuery* handle, SHashObj* pVgHash); int rawBlockBindData(SQuery *query, STableMeta* pTableMeta, void* data, SVCreateTbReq* pCreateTb, TAOS_FIELD *fields, int numFields, bool needChangeLength); -void clearColValArray(SArray* pCols); int32_t rewriteToVnodeModifyOpStmt(SQuery* pQuery, SArray* pBufArray); SArray* serializeVgroupsCreateTableBatch(SHashObj* pVgroupHashmap); diff --git a/source/client/src/clientSmlJson.c b/source/client/src/clientSmlJson.c index 8c53d4f68a..f9076112c4 100644 --- a/source/client/src/clientSmlJson.c +++ b/source/client/src/clientSmlJson.c @@ -1012,7 +1012,7 @@ static int32_t smlParseJSONStringExt(SSmlHandle *info, cJSON *root, SSmlLineInfo if (ret == TSDB_CODE_SUCCESS) { ret = smlBuildRow(info->currTableDataCtx); } - clearColValArray(info->currTableDataCtx->pValues); + clearColValArraySml(info->currTableDataCtx->pValues); if (unlikely(ret != TSDB_CODE_SUCCESS)) { smlBuildInvalidDataMsg(&info->msgBuf, "smlBuildCol error", NULL); return ret; @@ -1216,7 +1216,7 @@ static int32_t smlParseJSONString(SSmlHandle *info, char **start, SSmlLineInfo * if (ret == TSDB_CODE_SUCCESS) { ret = smlBuildRow(info->currTableDataCtx); } - clearColValArray(info->currTableDataCtx->pValues); + clearColValArraySml(info->currTableDataCtx->pValues); if (unlikely(ret != TSDB_CODE_SUCCESS)) { smlBuildInvalidDataMsg(&info->msgBuf, "smlBuildCol error", NULL); return ret; diff --git a/source/client/src/clientSmlLine.c b/source/client/src/clientSmlLine.c index e62e018caa..a565fb1a21 100644 --- a/source/client/src/clientSmlLine.c +++ b/source/client/src/clientSmlLine.c @@ -733,7 +733,7 @@ int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine ret = smlBuildRow(info->currTableDataCtx); } - clearColValArray(info->currTableDataCtx->pValues); + clearColValArraySml(info->currTableDataCtx->pValues); if (unlikely(ret != TSDB_CODE_SUCCESS)) { smlBuildInvalidDataMsg(&info->msgBuf, "smlBuildCol error", NULL); return ret; diff --git a/source/client/src/clientSmlTelnet.c b/source/client/src/clientSmlTelnet.c index c378f9b1c3..f9f8602c5a 100644 --- a/source/client/src/clientSmlTelnet.c +++ b/source/client/src/clientSmlTelnet.c @@ -307,7 +307,7 @@ int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine if (ret == TSDB_CODE_SUCCESS) { ret = smlBuildRow(info->currTableDataCtx); } - clearColValArray(info->currTableDataCtx->pValues); + clearColValArraySml(info->currTableDataCtx->pValues); if (unlikely(ret != TSDB_CODE_SUCCESS)) { smlBuildInvalidDataMsg(&info->msgBuf, "smlBuildCol error", NULL); return ret; diff --git a/source/libs/parser/src/parInsertSml.c b/source/libs/parser/src/parInsertSml.c index aa10711e3c..517bc630c3 100644 --- a/source/libs/parser/src/parInsertSml.c +++ b/source/libs/parser/src/parInsertSml.c @@ -181,6 +181,18 @@ STableDataCxt* smlInitTableDataCtx(SQuery* query, STableMeta* pTableMeta) { return pTableCxt; } +void clearColValArraySml(SArray* pCols) { + int32_t num = taosArrayGetSize(pCols); + for (int32_t i = 0; i < num; ++i) { + SColVal* pCol = taosArrayGet(pCols, i); + if (TSDB_DATA_TYPE_NCHAR == pCol->type || TSDB_DATA_TYPE_GEOMETRY == pCol->type || TSDB_DATA_TYPE_VARBINARY == pCol->type) { + taosMemoryFreeClear(pCol->value.pData); + } + pCol->flag = CV_FLAG_NONE; + pCol->value.val = 0; + } +} + int32_t smlBuildRow(STableDataCxt* pTableCxt) { SRow** pRow = taosArrayReserve(pTableCxt->pData->aRowP, 1); int ret = tRowBuild(pTableCxt->pValues, pTableCxt->pSchema, pRow); @@ -376,7 +388,7 @@ int32_t smlBindData(SQuery* query, bool dataFormat, SArray* tags, SArray* colsSc goto end; } insCheckTableDataOrder(pTableCxt, TD_ROW_KEY(*pRow)); - clearColValArray(pTableCxt->pValues); + clearColValArraySml(pTableCxt->pValues); } end: diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index 38afb6ddae..0e007e127e 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -1508,15 +1508,13 @@ static int32_t parseValueToken(SInsertParseContext* pCxt, const char** pSql, STo return code; } -void clearColValArray(SArray* pCols) { +static void clearColValArray(SArray* pCols) { int32_t num = taosArrayGetSize(pCols); for (int32_t i = 0; i < num; ++i) { SColVal* pCol = taosArrayGet(pCols, i); - if (TSDB_DATA_TYPE_NCHAR == pCol->type || TSDB_DATA_TYPE_GEOMETRY == pCol->type) { + if (IS_VAR_DATA_TYPE(pCol->type)) { taosMemoryFreeClear(pCol->value.pData); } - pCol->flag = CV_FLAG_NONE; - pCol->value.val = 0; } } From f0fdbaa3f3dbfcb588ce1303fe6505f6528be9d8 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 25 Aug 2023 16:22:45 +0800 Subject: [PATCH 09/13] fix(tsdb/close): fix uaf with file closing --- source/dnode/vnode/src/tsdb/tsdbReaderWriter.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index cd0bd47d47..dcf3770cbd 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -35,12 +35,12 @@ static int32_t tsdbOpenFileImpl(STsdbFD *pFD) { if (pFD->pFD == NULL) { code = TAOS_SYSTEM_ERROR(ENOENT); - taosMemoryFree(pFD); + // taosMemoryFree(pFD); goto _exit; } } else { code = TAOS_SYSTEM_ERROR(errsv); - taosMemoryFree(pFD); + // taosMemoryFree(pFD); goto _exit; } } @@ -48,8 +48,8 @@ static int32_t tsdbOpenFileImpl(STsdbFD *pFD) { pFD->pBuf = taosMemoryCalloc(1, szPage); if (pFD->pBuf == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - taosCloseFile(&pFD->pFD); - taosMemoryFree(pFD); + // taosCloseFile(&pFD->pFD); + // taosMemoryFree(pFD); goto _exit; } @@ -57,9 +57,9 @@ static int32_t tsdbOpenFileImpl(STsdbFD *pFD) { if (flag != TD_FILE_READ) { if (taosStatFile(path, &pFD->szFile, NULL, NULL) < 0) { code = TAOS_SYSTEM_ERROR(errno); - taosMemoryFree(pFD->pBuf); - taosCloseFile(&pFD->pFD); - taosMemoryFree(pFD); + // taosMemoryFree(pFD->pBuf); + // taosCloseFile(&pFD->pFD); + // taosMemoryFree(pFD); goto _exit; } From 533a7c43f8a221f0f148eb4963247dbd4d8f8390 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Fri, 25 Aug 2023 17:28:33 +0800 Subject: [PATCH 10/13] release 3.1.0.3 --- docs/en/28-releases/01-tdengine.md | 4 ++++ docs/zh/28-releases/01-tdengine.md | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/docs/en/28-releases/01-tdengine.md b/docs/en/28-releases/01-tdengine.md index ff6a36440f..c02b3227ca 100644 --- a/docs/en/28-releases/01-tdengine.md +++ b/docs/en/28-releases/01-tdengine.md @@ -10,6 +10,10 @@ For TDengine 2.x installation packages by version, please visit [here](https://t import Release from "/components/ReleaseV3"; +## 3.1.0.3 + + + ## 3.1.0.2 diff --git a/docs/zh/28-releases/01-tdengine.md b/docs/zh/28-releases/01-tdengine.md index d316b3ab68..d4e4b116b7 100644 --- a/docs/zh/28-releases/01-tdengine.md +++ b/docs/zh/28-releases/01-tdengine.md @@ -10,6 +10,10 @@ TDengine 2.x 各版本安装包请访问[这里](https://www.taosdata.com/all-do import Release from "/components/ReleaseV3"; +## 3.1.0.3 + + + ## 3.1.0.2 From e59536181c9b5111f49cae82e2964c560778a494 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 25 Aug 2023 22:21:33 +0800 Subject: [PATCH 11/13] fix:bytes error --- source/libs/parser/src/parTranslater.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index fe15e62f9b..ad3cfe32c1 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1329,9 +1329,7 @@ static EDealRes translateValueImpl(STranslateContext* pCxt, SValueNode* pVal, SD res = translateNormalValue(pCxt, pVal, targetDt, strict); } pVal->node.resType.type = targetDt.type; - if( targetDt.type == TSDB_DATA_TYPE_VARBINARY || !strict){ - pVal->node.resType.bytes = targetDt.bytes; - } + pVal->node.resType.bytes = targetDt.bytes; pVal->node.resType.scale = pVal->unit; pVal->translate = true; if (!strict && TSDB_DATA_TYPE_UBIGINT == pVal->node.resType.type && pVal->datum.u <= INT64_MAX) { From 438b5a521411d2dc5ec59996f365810ece1e8ab0 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Sat, 26 Aug 2023 00:44:42 +0800 Subject: [PATCH 12/13] fix:disable varbianry test cases --- tests/parallel_test/cases.task | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index e4303b2e15..53b2e7acd9 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -351,7 +351,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/smaTest.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/sma_index.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sml_TS-3724.py -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/varbinary.py +#,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/varbinary.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sml.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sml.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/spread.py From adaf7fbdbd3c2da83ab883465639b77039437bbe Mon Sep 17 00:00:00 2001 From: huolibo Date: Mon, 28 Aug 2023 10:11:47 +0800 Subject: [PATCH 13/13] release(driver): jdbc release 3.2.5 version --- docs/en/07-develop/07-tmq.mdx | 12 ++++++++++++ docs/en/14-reference/03-connector/04-java.mdx | 18 ++++++++++++++++++ docs/zh/07-develop/07-tmq.mdx | 12 ++++++++++++ docs/zh/08-connector/14-java.mdx | 18 ++++++++++++++++++ 4 files changed, 60 insertions(+) diff --git a/docs/en/07-develop/07-tmq.mdx b/docs/en/07-develop/07-tmq.mdx index 5af3897afd..42a9f3af5a 100644 --- a/docs/en/07-develop/07-tmq.mdx +++ b/docs/en/07-develop/07-tmq.mdx @@ -113,7 +113,19 @@ Set subscription() throws SQLException; ConsumerRecords poll(Duration timeout) throws SQLException; +Set assignment() throws SQLException; +long position(TopicPartition partition) throws SQLException; +Map position(String topic) throws SQLException; +Map beginningOffsets(String topic) throws SQLException; +Map endOffsets(String topic) throws SQLException; +Map committed(Set partitions) throws SQLException; + +void seek(TopicPartition partition, long offset) throws SQLException; +void seekToBeginning(Collection partitions) throws SQLException; +void seekToEnd(Collection partitions) throws SQLException; + void commitSync() throws SQLException; +void commitSync(Map offsets) throws SQLException; void close() throws SQLException; ``` diff --git a/docs/en/14-reference/03-connector/04-java.mdx b/docs/en/14-reference/03-connector/04-java.mdx index b12ef38ea8..7e580a52d4 100644 --- a/docs/en/14-reference/03-connector/04-java.mdx +++ b/docs/en/14-reference/03-connector/04-java.mdx @@ -36,6 +36,7 @@ REST connection supports all platforms that can run Java. | taos-jdbcdriver version | major changes | TDengine version | | :---------------------: | :------------------------------------------------------------------------------------------------------------------------------------------------: | :--------------: | +| 3.2.5 | Subscription add committed() and assignment() method | 3.1.0.3 or later | | 3.2.4 | Subscription add the enable.auto.commit parameter and the unsubscribe() method in the WebSocket connection | - | | 3.2.3 | Fixed resultSet data parsing failure in some cases | - | | 3.2.2 | Subscription add seek function | 3.0.5.0 or later | @@ -1019,14 +1020,19 @@ while(true) { #### Assignment subscription Offset ```java +// get topicPartition +Set assignment() throws SQLException; // get offset long position(TopicPartition partition) throws SQLException; Map position(String topic) throws SQLException; Map beginningOffsets(String topic) throws SQLException; Map endOffsets(String topic) throws SQLException; +Map committed(Set partitions) throws SQLException; // Overrides the fetch offsets that the consumer will use on the next poll(timeout). void seek(TopicPartition partition, long offset) throws SQLException; +void seekToBeginning(Collection partitions) throws SQLException; +void seekToEnd(Collection partitions) throws SQLException; ``` Example usage is as follows. @@ -1052,6 +1058,18 @@ try (TaosConsumer consumer = new TaosConsumer<>(properties)) { } ``` +#### Commit offset + +If `enable.auto.commit` is false, offset can be submitted manually. + +```java +void commitSync() throws SQLException; +void commitSync(Map offsets) throws SQLException; +// async commit only support jni connection +void commitAsync(OffsetCommitCallback callback) throws SQLException; +void commitAsync(Map offsets, OffsetCommitCallback callback) throws SQLException; +``` + #### Close subscriptions ```java diff --git a/docs/zh/07-develop/07-tmq.mdx b/docs/zh/07-develop/07-tmq.mdx index 6852d5551e..a73d43cd04 100644 --- a/docs/zh/07-develop/07-tmq.mdx +++ b/docs/zh/07-develop/07-tmq.mdx @@ -115,7 +115,19 @@ Set subscription() throws SQLException; ConsumerRecords poll(Duration timeout) throws SQLException; +Set assignment() throws SQLException; +long position(TopicPartition partition) throws SQLException; +Map position(String topic) throws SQLException; +Map beginningOffsets(String topic) throws SQLException; +Map endOffsets(String topic) throws SQLException; +Map committed(Set partitions) throws SQLException; + +void seek(TopicPartition partition, long offset) throws SQLException; +void seekToBeginning(Collection partitions) throws SQLException; +void seekToEnd(Collection partitions) throws SQLException; + void commitSync() throws SQLException; +void commitSync(Map offsets) throws SQLException; void close() throws SQLException; ``` diff --git a/docs/zh/08-connector/14-java.mdx b/docs/zh/08-connector/14-java.mdx index 36eacd26a4..0ff00d1710 100644 --- a/docs/zh/08-connector/14-java.mdx +++ b/docs/zh/08-connector/14-java.mdx @@ -36,6 +36,7 @@ REST 连接支持所有能运行 Java 的平台。 | taos-jdbcdriver 版本 | 主要变化 | TDengine 版本 | | :------------------: | :----------------------------------------------------------------------------------------------------------------------------------------------------: | :----------------: | +| 3.2.5 | 数据订阅增加 committed()、assignment() 方法 | 3.1.0.3 及更高版本 | | 3.2.4 | 数据订阅在 WebSocket 连接下增加 enable.auto.commit 参数，以及 unsubscribe() 方法。 | - | | 3.2.3 | 修复 ResultSet 在一些情况数据解析失败 | - | | 3.2.2 | 新增功能：数据订阅支持 seek 功能。 | 3.0.5.0 及更高版本 | @@ -1022,14 +1023,19 @@ while(true) { #### 指定订阅 Offset ```java +// 获取订阅的 topicPartition +Set assignment() throws SQLException; // 获取 offset long position(TopicPartition partition) throws SQLException; Map position(String topic) throws SQLException; Map beginningOffsets(String topic) throws SQLException; Map endOffsets(String topic) throws SQLException; +Map committed(Set partitions) throws SQLException; // 指定下一次 poll 中使用的 offset void seek(TopicPartition partition, long offset) throws SQLException; +void seekToBeginning(Collection partitions) throws SQLException; +void seekToEnd(Collection partitions) throws SQLException; ``` 示例代码： @@ -1055,6 +1061,18 @@ try (TaosConsumer consumer = new TaosConsumer<>(properties)) { } ``` +#### 提交 Offset + +当`enable.auto.commit`为 false 时，可以手动提交 offset。 + +```java +void commitSync() throws SQLException; +void commitSync(Map offsets) throws SQLException; +// 异步提交仅在 native 连接下有效 +void commitAsync(OffsetCommitCallback callback) throws SQLException; +void commitAsync(Map offsets, OffsetCommitCallback callback) throws SQLException; +``` + #### 关闭订阅 ```java