xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch 'enh/3.0' into enh/triggerCheckPoint2

This commit is contained in:
yihaoDeng 2023-08-28 10:46:00 +08:00
parent b75ae217f6 5821041336
commit 20b037d9cc
96 changed files with 1413 additions and 363 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
docs/en/07-develop/07-tmq.mdx
View File

@ -113,7 +113,19 @@ Set<String> subscription() throws SQLException;
ConsumerRecords<V> poll(Duration timeout) throws SQLException;
Set<TopicPartition> assignment() throws SQLException;
long position(TopicPartition partition) throws SQLException;
Map<TopicPartition, Long> position(String topic) throws SQLException;
Map<TopicPartition, Long> beginningOffsets(String topic) throws SQLException;
Map<TopicPartition, Long> endOffsets(String topic) throws SQLException;
Map<TopicPartition, OffsetAndMetadata> committed(Set<TopicPartition> partitions) throws SQLException;
void seek(TopicPartition partition, long offset) throws SQLException;
void seekToBeginning(Collection<TopicPartition> partitions) throws SQLException;
void seekToEnd(Collection<TopicPartition> partitions) throws SQLException;
void commitSync() throws SQLException;
void commitSync(Map<TopicPartition, OffsetAndMetadata> offsets) throws SQLException;
void close() throws SQLException;
```

18
docs/en/14-reference/03-connector/04-java.mdx
View File

@ -36,6 +36,7 @@ REST connection supports all platforms that can run Java.
| taos-jdbcdriver version | major changes | TDengine version |
| :---------------------: | :------------------------------------------------------------------------------------------------------------------------------------------------: | :--------------: |
| 3.2.5 | Subscription add committed() and assignment() method | 3.1.0.3 or later |
| 3.2.4 | Subscription add the enable.auto.commit parameter and the unsubscribe() method in the WebSocket connection | - |
| 3.2.3 | Fixed resultSet data parsing failure in some cases | - |
| 3.2.2 | Subscription add seek function | 3.0.5.0 or later |
@ -1019,14 +1020,19 @@ while(true) {
#### Assignment subscription Offset
```java
// get topicPartition
Set<TopicPartition> assignment() throws SQLException;
// get offset
long position(TopicPartition partition) throws SQLException;
Map<TopicPartition, Long> position(String topic) throws SQLException;
Map<TopicPartition, Long> beginningOffsets(String topic) throws SQLException;
Map<TopicPartition, Long> endOffsets(String topic) throws SQLException;
Map<TopicPartition, OffsetAndMetadata> committed(Set<TopicPartition> partitions) throws SQLException;
// Overrides the fetch offsets that the consumer will use on the next poll(timeout).
void seek(TopicPartition partition, long offset) throws SQLException;
void seekToBeginning(Collection<TopicPartition> partitions) throws SQLException;
void seekToEnd(Collection<TopicPartition> partitions) throws SQLException;
```
Example usage is as follows.
@ -1052,6 +1058,18 @@ try (TaosConsumer<ResultBean> consumer = new TaosConsumer<>(properties)) {
}
```
#### Commit offset
If `enable.auto.commit` is false, offset can be submitted manually.
```java
void commitSync() throws SQLException;
void commitSync(Map<TopicPartition, OffsetAndMetadata> offsets) throws SQLException;
// async commit only support jni connection
void commitAsync(OffsetCommitCallback<V> callback) throws SQLException;
void commitAsync(Map<TopicPartition, OffsetAndMetadata> offsets, OffsetCommitCallback<V> callback) throws SQLException;
```
#### Close subscriptions
```java

4
docs/en/28-releases/01-tdengine.md
View File

@ -10,6 +10,10 @@ For TDengine 2.x installation packages by version, please visit [here](https://t
import Release from "/components/ReleaseV3";
## 3.1.0.3
<Release type="tdengine" version="3.1.0.3" />
## 3.1.0.2
<Release type="tdengine" version="3.1.0.2" />

1
docs/examples/c/async_query_example.c
View File

@ -78,6 +78,7 @@ int printRow(char *str, TAOS_ROW row, TAOS_FIELD *fields, int numFields) {
} break;
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_VARBINARY:
case TSDB_DATA_TYPE_NCHAR:
case TSDB_DATA_TYPE_GEOMETRY: {
int32_t charLen = varDataLen((char *)row[i] - VARSTR_HEADER_SIZE);

1
docs/examples/c/query_example.c
View File

@ -76,6 +76,7 @@ int printRow(char *str, TAOS_ROW row, TAOS_FIELD *fields, int numFields) {
} break;
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_VARBINARY:
case TSDB_DATA_TYPE_NCHAR:
case TSDB_DATA_TYPE_GEOMETRY: {
int32_t charLen = varDataLen((char *)row[i] - VARSTR_HEADER_SIZE);

12
docs/zh/07-develop/07-tmq.mdx
View File

@ -115,7 +115,19 @@ Set<String> subscription() throws SQLException;
ConsumerRecords<V> poll(Duration timeout) throws SQLException;
Set<TopicPartition> assignment() throws SQLException;
long position(TopicPartition partition) throws SQLException;
Map<TopicPartition, Long> position(String topic) throws SQLException;
Map<TopicPartition, Long> beginningOffsets(String topic) throws SQLException;
Map<TopicPartition, Long> endOffsets(String topic) throws SQLException;
Map<TopicPartition, OffsetAndMetadata> committed(Set<TopicPartition> partitions) throws SQLException;
void seek(TopicPartition partition, long offset) throws SQLException;
void seekToBeginning(Collection<TopicPartition> partitions) throws SQLException;
void seekToEnd(Collection<TopicPartition> partitions) throws SQLException;
void commitSync() throws SQLException;
void commitSync(Map<TopicPartition, OffsetAndMetadata> offsets) throws SQLException;
void close() throws SQLException;
```

18
docs/zh/08-connector/14-java.mdx
View File

@ -36,6 +36,7 @@ REST 连接支持所有能运行 Java 的平台。
| taos-jdbcdriver 版本 | 主要变化 | TDengine 版本 |
| :------------------: | :----------------------------------------------------------------------------------------------------------------------------------------------------: | :----------------: |
| 3.2.5 | 数据订阅增加 committed()、assignment() 方法 | 3.1.0.3 及更高版本 |
| 3.2.4 | 数据订阅在 WebSocket 连接下增加 enable.auto.commit 参数，以及 unsubscribe() 方法。 | - |
| 3.2.3 | 修复 ResultSet 在一些情况数据解析失败 | - |
| 3.2.2 | 新增功能：数据订阅支持 seek 功能。 | 3.0.5.0 及更高版本 |
@ -1022,14 +1023,19 @@ while(true) {
#### 指定订阅 Offset
```java
// 获取订阅的 topicPartition
Set<TopicPartition> assignment() throws SQLException;
// 获取 offset
long position(TopicPartition partition) throws SQLException;
Map<TopicPartition, Long> position(String topic) throws SQLException;
Map<TopicPartition, Long> beginningOffsets(String topic) throws SQLException;
Map<TopicPartition, Long> endOffsets(String topic) throws SQLException;
Map<TopicPartition, OffsetAndMetadata> committed(Set<TopicPartition> partitions) throws SQLException;
// 指定下一次 poll 中使用的 offset
void seek(TopicPartition partition, long offset) throws SQLException;
void seekToBeginning(Collection<TopicPartition> partitions) throws SQLException;
void seekToEnd(Collection<TopicPartition> partitions) throws SQLException;
```
示例代码：
@ -1055,6 +1061,18 @@ try (TaosConsumer<ResultBean> consumer = new TaosConsumer<>(properties)) {
}
```
#### 提交 Offset
当`enable.auto.commit`为 false 时，可以手动提交 offset。
```java
void commitSync() throws SQLException;
void commitSync(Map<TopicPartition, OffsetAndMetadata> offsets) throws SQLException;
// 异步提交仅在 native 连接下有效
void commitAsync(OffsetCommitCallback<V> callback) throws SQLException;
void commitAsync(Map<TopicPartition, OffsetAndMetadata> offsets, OffsetCommitCallback<V> callback) throws SQLException;
```
#### 关闭订阅
```java

1
docs/zh/08-connector/45-php.mdx
View File

@ -143,6 +143,7 @@ phpize && ./configure --enable-swoole && make -j && make install
| `TDengine\TSDB_DATA_TYPE_FLOAT` | float |
| `TDengine\TSDB_DATA_TYPE_DOUBLE` | double |
| `TDengine\TSDB_DATA_TYPE_BINARY` | binary |
| `TDengine\TSDB_DATA_TYPE_VARBINARY` | varbinary |
| `TDengine\TSDB_DATA_TYPE_TIMESTAMP` | timestamp |
| `TDengine\TSDB_DATA_TYPE_NCHAR` | nchar |
| `TDengine\TSDB_DATA_TYPE_UTINYINT` | utinyint |

4
docs/zh/28-releases/01-tdengine.md
View File

@ -10,6 +10,10 @@ TDengine 2.x 各版本安装包请访问[这里](https://www.taosdata.com/all-do
import Release from "/components/ReleaseV3";
## 3.1.0.3
<Release type="tdengine" version="3.1.0.3" />
## 3.1.0.2
<Release type="tdengine" version="3.1.0.2" />

21
include/common/tmsg.h
View File

@ -1418,6 +1418,7 @@ typedef struct {
int64_t numOfProcessedCQuery;
int64_t numOfProcessedFetch;
int64_t numOfProcessedDrop;
int64_t numOfProcessedNotify;
int64_t numOfProcessedHb;
int64_t numOfProcessedDelete;
int64_t cacheDataSize;
@ -2159,8 +2160,24 @@ typedef struct {
int32_t tSerializeSTaskDropReq(void* buf, int32_t bufLen, STaskDropReq* pReq);
int32_t tDeserializeSTaskDropReq(void* buf, int32_t bufLen, STaskDropReq* pReq);
int32_t tSerializeSTaskDropReq(void* buf, int32_t bufLen, STaskDropReq* pReq);
int32_t tDeserializeSTaskDropReq(void* buf, int32_t bufLen, STaskDropReq* pReq);
typedef enum {
TASK_NOTIFY_FINISHED = 1,
} ETaskNotifyType;
typedef struct {
SMsgHead header;
uint64_t sId;
uint64_t queryId;
uint64_t taskId;
int64_t refId;
int32_t execId;
ETaskNotifyType type;
} STaskNotifyReq;
int32_t tSerializeSTaskNotifyReq(void* buf, int32_t bufLen, STaskNotifyReq* pReq);
int32_t tDeserializeSTaskNotifyReq(void* buf, int32_t bufLen, STaskNotifyReq* pReq);
int32_t tSerializeSQueryTableRsp(void* buf, int32_t bufLen, SQueryTableRsp* pRsp);
int32_t tDeserializeSQueryTableRsp(void* buf, int32_t bufLen, SQueryTableRsp* pRsp);

1
include/common/tmsgdef.h
View File

@ -248,6 +248,7 @@ enum { // WARN: new msg should be appended to segment tail
TD_DEF_MSG_TYPE(TDMT_SCH_DROP_TASK, "drop-task", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_SCH_EXPLAIN, "explain", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_SCH_LINK_BROKEN, "link-broken", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_SCH_TASK_NOTIFY, "task-notify", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_SCH_MAX_MSG, "sch-max", NULL, NULL)

4
include/common/ttypes.h
View File

@ -269,8 +269,8 @@ typedef struct {
(IS_NUMERIC_TYPE(_t) || (_t) == (TSDB_DATA_TYPE_BOOL) || (_t) == (TSDB_DATA_TYPE_TIMESTAMP))
#define IS_VAR_DATA_TYPE(t) \
(((t) == TSDB_DATA_TYPE_VARCHAR) || ((t) == TSDB_DATA_TYPE_NCHAR) || ((t) == TSDB_DATA_TYPE_JSON) || ((t) == TSDB_DATA_TYPE_GEOMETRY))
#define IS_STR_DATA_TYPE(t) (((t) == TSDB_DATA_TYPE_VARCHAR) || ((t) == TSDB_DATA_TYPE_NCHAR))
(((t) == TSDB_DATA_TYPE_VARCHAR) || ((t) == TSDB_DATA_TYPE_VARBINARY) || ((t) == TSDB_DATA_TYPE_NCHAR) || ((t) == TSDB_DATA_TYPE_JSON) || ((t) == TSDB_DATA_TYPE_GEOMETRY))
#define IS_STR_DATA_TYPE(t) (((t) == TSDB_DATA_TYPE_VARCHAR) || ((t) == TSDB_DATA_TYPE_VARBINARY) || ((t) == TSDB_DATA_TYPE_NCHAR))
#define IS_VALID_TINYINT(_t) ((_t) >= INT8_MIN && (_t) <= INT8_MAX)
#define IS_VALID_SMALLINT(_t) ((_t) >= INT16_MIN && (_t) <= INT16_MAX)

4
include/libs/function/taosudf.h
View File

@ -109,8 +109,8 @@ typedef uint16_t VarDataLenT; // maxVarDataLen: 65535
#define varDataLenByData(v) (*(VarDataLenT *)(((char *)(v)) - VARSTR_HEADER_SIZE))
#define varDataSetLen(v, _len) (((VarDataLenT *)(v))[0] = (VarDataLenT)(_len))
#define IS_VAR_DATA_TYPE(t) \
(((t) == TSDB_DATA_TYPE_VARCHAR) || ((t) == TSDB_DATA_TYPE_NCHAR) || ((t) == TSDB_DATA_TYPE_JSON) || ((t) == TSDB_DATA_TYPE_GEOMETRY))
#define IS_STR_DATA_TYPE(t) (((t) == TSDB_DATA_TYPE_VARCHAR) || ((t) == TSDB_DATA_TYPE_NCHAR))
(((t) == TSDB_DATA_TYPE_VARCHAR) || ((t) == TSDB_DATA_TYPE_VARBINARY) || ((t) == TSDB_DATA_TYPE_NCHAR) || ((t) == TSDB_DATA_TYPE_JSON) || ((t) == TSDB_DATA_TYPE_GEOMETRY))
#define IS_STR_DATA_TYPE(t) (((t) == TSDB_DATA_TYPE_VARCHAR) || ((t) == TSDB_DATA_TYPE_VARBINARY) || ((t) == TSDB_DATA_TYPE_NCHAR))
static FORCE_INLINE char *udfColDataGetData(const SUdfColumn *pColumn, int32_t row) {
if (IS_VAR_DATA_TYPE(pColumn->colMeta.type)) {

1
include/libs/parser/parser.h
View File

@ -114,6 +114,7 @@ int32_t smlBuildRow(STableDataCxt* pTableCxt);
int32_t smlBuildCol(STableDataCxt* pTableCxt, SSchema* schema, void* kv, int32_t index);
STableDataCxt* smlInitTableDataCtx(SQuery* query, STableMeta* pTableMeta);
void clearColValArraySml(SArray* pCols);
int32_t smlBindData(SQuery* handle, bool dataFormat, SArray* tags, SArray* colsSchema, SArray* cols,
STableMeta* pTableMeta, char* tableName, const char* sTableName, int32_t sTableNameLen, int32_t ttl,
char* msgBuf, int32_t msgBufLen);

3
include/libs/qworker/qworker.h
View File

@ -45,6 +45,7 @@ typedef struct {
uint64_t cqueryProcessed;
uint64_t fetchProcessed;
uint64_t dropProcessed;
uint64_t notifyProcessed;
uint64_t hbProcessed;
uint64_t deleteProcessed;
@ -90,6 +91,8 @@ int32_t qWorkerProcessCancelMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, in
int32_t qWorkerProcessDropMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int64_t ts);
int32_t qWorkerProcessNotifyMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int64_t ts);
int32_t qWorkerProcessHbMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int64_t ts);
int32_t qWorkerProcessDeleteMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, SDeleteRes *pRes);

5
include/os/osString.h
View File

@ -90,6 +90,11 @@ int8_t taosStr2Int8(const char *str, char **pEnd, int32_t radix);
uint8_t taosStr2UInt8(const char *str, char **pEnd, int32_t radix);
double taosStr2Double(const char *str, char **pEnd);
float taosStr2Float(const char *str, char **pEnd);
int32_t taosHex2Ascii(const char *z, uint32_t n, void** data, uint32_t* size);
int32_t taosAscii2Hex(const char *z, uint32_t n, void** data, uint32_t* size);
//int32_t taosBin2Ascii(const char *z, uint32_t n, void** data, uint32_t* size);
bool isHex(const char* z, uint32_t n);
bool isValidateHex(const char* z, uint32_t n);
#ifdef __cplusplus
}

1
include/util/taoserror.h
View File

@ -709,6 +709,7 @@ int32_t* taosGetErrno();
#define TSDB_CODE_PAR_INVALID_OPTR_USAGE TAOS_DEF_ERROR_CODE(0, 0x2667)
#define TSDB_CODE_PAR_SYSTABLE_NOT_ALLOWED_FUNC TAOS_DEF_ERROR_CODE(0, 0x2668)
#define TSDB_CODE_PAR_SYSTABLE_NOT_ALLOWED TAOS_DEF_ERROR_CODE(0, 0x2669)
#define TSDB_CODE_PAR_INVALID_VARBINARY TAOS_DEF_ERROR_CODE(0, 0x266A)
#define TSDB_CODE_PAR_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x26FF)
//planner

2
include/util/tcompare.h
View File

@ -102,6 +102,7 @@ int32_t compareUint64ValDesc(const void *pLeft, const void *pRight);
int32_t compareLenPrefixedStrDesc(const void *pLeft, const void *pRight);
int32_t compareLenPrefixedWStrDesc(const void *pLeft, const void *pRight);
int32_t compareLenBinaryValDesc(const void *pLeft, const void *pRight);
int32_t comparestrPatternMatch(const void *pLeft, const void *pRight);
int32_t comparestrPatternNMatch(const void *pLeft, const void *pRight);
@ -202,7 +203,6 @@ int32_t compareUint64Uint32(const void *pLeft, const void *pRight);
__compar_fn_t getComparFunc(int32_t type, int32_t optr);
__compar_fn_t getKeyComparFunc(int32_t keyType, int32_t order);
int32_t doCompare(const char *a, const char *b, int32_t type, size_t size);
#ifdef __cplusplus
}

7
include/util/tdef.h
View File

@ -416,9 +416,10 @@ typedef enum ELogicConditionType {
#define TSDB_EXPLAIN_RESULT_COLUMN_NAME "QUERY_PLAN"
#define TSDB_MAX_FIELD_LEN 65519 // 16384:65519
#define TSDB_MAX_BINARY_LEN TSDB_MAX_FIELD_LEN // 16384-8:65519
#define TSDB_MAX_NCHAR_LEN TSDB_MAX_FIELD_LEN // 16384-8:65519
#define TSDB_MAX_GEOMETRY_LEN TSDB_MAX_FIELD_LEN // 16384-8:65519
#define TSDB_MAX_BINARY_LEN TSDB_MAX_FIELD_LEN // 16384-8:65519
#define TSDB_MAX_NCHAR_LEN TSDB_MAX_FIELD_LEN // 16384-8:65519
#define TSDB_MAX_GEOMETRY_LEN TSDB_MAX_FIELD_LEN // 16384-8:65519
#define TSDB_MAX_VARBINARY_LEN TSDB_MAX_FIELD_LEN // 16384-8:65519
#define PRIMARYKEY_TIMESTAMP_COL_ID 1
#define COL_REACH_END(colId, maxColId) ((colId) > (maxColId))

1
source/client/inc/clientSml.h
View File

@ -251,7 +251,6 @@ int64_t smlParseOpenTsdbTime(SSmlHandle *info, const char *data, int32
int32_t smlClearForRerun(SSmlHandle *info);
int32_t smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg);
uint8_t smlGetTimestampLen(int64_t num);
void clearColValArray(SArray* pCols);
void smlDestroyTableInfo(void *para);
void freeSSmlKv(void* data);

3
source/client/src/clientEnv.c
View File

@ -133,7 +133,8 @@ void closeTransporter(SAppInstInfo *pAppInfo) {
static bool clientRpcRfp(int32_t code, tmsg_t msgType) {
if (NEED_REDIRECT_ERROR(code)) {
if (msgType == TDMT_SCH_QUERY || msgType == TDMT_SCH_MERGE_QUERY || msgType == TDMT_SCH_FETCH ||
msgType == TDMT_SCH_MERGE_FETCH || msgType == TDMT_SCH_QUERY_HEARTBEAT || msgType == TDMT_SCH_DROP_TASK) {
msgType == TDMT_SCH_MERGE_FETCH || msgType == TDMT_SCH_QUERY_HEARTBEAT || msgType == TDMT_SCH_DROP_TASK ||
msgType == TDMT_SCH_TASK_NOTIFY) {
return false;
}
return true;

2
source/client/src/clientImpl.c
View File

@ -503,7 +503,7 @@ void setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t
pResInfo->userFields[i].bytes = pSchema[i].bytes;
pResInfo->userFields[i].type = pSchema[i].type;
if (pSchema[i].type == TSDB_DATA_TYPE_VARCHAR || pSchema[i].type == TSDB_DATA_TYPE_GEOMETRY) {
if (pSchema[i].type == TSDB_DATA_TYPE_VARCHAR || pSchema[i].type == TSDB_DATA_TYPE_VARBINARY || pSchema[i].type == TSDB_DATA_TYPE_GEOMETRY) {
pResInfo->userFields[i].bytes -= VARSTR_HEADER_SIZE;
} else if (pSchema[i].type == TSDB_DATA_TYPE_NCHAR || pSchema[i].type == TSDB_DATA_TYPE_JSON) {
pResInfo->userFields[i].bytes = (pResInfo->userFields[i].bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE;

1
source/client/src/clientJniConnector.c
View File

@ -580,6 +580,7 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_fetchRowImp(JNIEn
(*env)->CallVoidMethod(env, rowobj, g_rowdataSetDoubleFp, i, (jdouble)dv);
} break;
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_VARBINARY:
case TSDB_DATA_TYPE_GEOMETRY: {
memcpy(tmp, row[i], length[i]); // handle the case that terminated does not exist
(*env)->CallVoidMethod(env, rowobj, g_rowdataSetStringFp, i, (*env)->NewStringUTF(env, tmp));

3
source/client/src/clientMain.c
View File

@ -388,10 +388,11 @@ int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields)
} break;
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_VARBINARY:
case TSDB_DATA_TYPE_NCHAR:
case TSDB_DATA_TYPE_GEOMETRY: {
int32_t charLen = varDataLen((char *)row[i] - VARSTR_HEADER_SIZE);
if (fields[i].type == TSDB_DATA_TYPE_BINARY || fields[i].type == TSDB_DATA_TYPE_GEOMETRY) {
if (fields[i].type == TSDB_DATA_TYPE_BINARY || fields[i].type == TSDB_DATA_TYPE_VARBINARY || fields[i].type == TSDB_DATA_TYPE_GEOMETRY) {
if (ASSERT(charLen <= fields[i].bytes && charLen >= 0)) {
tscError("taos_print_row error binary. charLen:%d, fields[i].bytes:%d", charLen, fields[i].bytes);
}

26
source/client/src/clientRawBlockWrite.c
View File

@ -56,7 +56,7 @@ static char* buildCreateTableJson(SSchemaWrapper* schemaRow, SSchemaWrapper* sch
cJSON_AddItemToObject(column, "name", cname);
cJSON* ctype = cJSON_CreateNumber(s->type);
cJSON_AddItemToObject(column, "type", ctype);
if (s->type == TSDB_DATA_TYPE_BINARY || s->type == TSDB_DATA_TYPE_GEOMETRY) {
if (s->type == TSDB_DATA_TYPE_BINARY || s->type == TSDB_DATA_TYPE_VARBINARY|| s->type == TSDB_DATA_TYPE_GEOMETRY) {
int32_t length = s->bytes - VARSTR_HEADER_SIZE;
cJSON* cbytes = cJSON_CreateNumber(length);
cJSON_AddItemToObject(column, "length", cbytes);
@ -77,7 +77,7 @@ static char* buildCreateTableJson(SSchemaWrapper* schemaRow, SSchemaWrapper* sch
cJSON_AddItemToObject(tag, "name", tname);
cJSON* ttype = cJSON_CreateNumber(s->type);
cJSON_AddItemToObject(tag, "type", ttype);
if (s->type == TSDB_DATA_TYPE_BINARY || s->type == TSDB_DATA_TYPE_GEOMETRY) {
if (s->type == TSDB_DATA_TYPE_BINARY || s->type == TSDB_DATA_TYPE_VARBINARY || s->type == TSDB_DATA_TYPE_GEOMETRY) {
int32_t length = s->bytes - VARSTR_HEADER_SIZE;
cJSON* cbytes = cJSON_CreateNumber(length);
cJSON_AddItemToObject(tag, "length", cbytes);
@ -130,7 +130,7 @@ static char* buildAlterSTableJson(void* alterData, int32_t alterDataLen) {
cJSON* colType = cJSON_CreateNumber(field->type);
cJSON_AddItemToObject(json, "colType", colType);
if (field->type == TSDB_DATA_TYPE_BINARY || field->type == TSDB_DATA_TYPE_GEOMETRY) {
if (field->type == TSDB_DATA_TYPE_BINARY || field->type == TSDB_DATA_TYPE_VARBINARY || field->type == TSDB_DATA_TYPE_GEOMETRY) {
int32_t length = field->bytes - VARSTR_HEADER_SIZE;
cJSON* cbytes = cJSON_CreateNumber(length);
cJSON_AddItemToObject(json, "colLength", cbytes);
@ -155,7 +155,7 @@ static char* buildAlterSTableJson(void* alterData, int32_t alterDataLen) {
cJSON_AddItemToObject(json, "colName", colName);
cJSON* colType = cJSON_CreateNumber(field->type);
cJSON_AddItemToObject(json, "colType", colType);
if (field->type == TSDB_DATA_TYPE_BINARY || field->type == TSDB_DATA_TYPE_GEOMETRY) {
if (field->type == TSDB_DATA_TYPE_BINARY || field->type == TSDB_DATA_TYPE_VARBINARY || field->type == TSDB_DATA_TYPE_GEOMETRY) {
int32_t length = field->bytes - VARSTR_HEADER_SIZE;
cJSON* cbytes = cJSON_CreateNumber(length);
cJSON_AddItemToObject(json, "colLength", cbytes);
@ -292,7 +292,13 @@ static void buildChildElement(cJSON* json, SVCreateTbReq* pCreateReq) {
cJSON* tvalue = NULL;
if (IS_VAR_DATA_TYPE(pTagVal->type)) {
char* buf = taosMemoryCalloc(pTagVal->nData + 3, 1);
char* buf = NULL;
if(pTagVal->type == TSDB_DATA_TYPE_VARBINARY){
buf = taosMemoryCalloc(pTagVal->nData*2 + 2 + 3, 1);
}else{
buf = taosMemoryCalloc(pTagVal->nData + 3, 1);
}
if (!buf) goto end;
dataConverToStr(buf, pTagVal->type, pTagVal->pData, pTagVal->nData, NULL);
tvalue = cJSON_CreateString(buf);
@ -457,7 +463,7 @@ static char* processAlterTable(SMqMetaRsp* metaRsp) {
cJSON* colType = cJSON_CreateNumber(vAlterTbReq.type);
cJSON_AddItemToObject(json, "colType", colType);
if (vAlterTbReq.type == TSDB_DATA_TYPE_BINARY || vAlterTbReq.type == TSDB_DATA_TYPE_GEOMETRY) {
if (vAlterTbReq.type == TSDB_DATA_TYPE_BINARY || vAlterTbReq.type == TSDB_DATA_TYPE_VARBINARY || vAlterTbReq.type == TSDB_DATA_TYPE_GEOMETRY) {
int32_t length = vAlterTbReq.bytes - VARSTR_HEADER_SIZE;
cJSON* cbytes = cJSON_CreateNumber(length);
cJSON_AddItemToObject(json, "colLength", cbytes);
@ -478,7 +484,7 @@ static char* processAlterTable(SMqMetaRsp* metaRsp) {
cJSON_AddItemToObject(json, "colName", colName);
cJSON* colType = cJSON_CreateNumber(vAlterTbReq.colModType);
cJSON_AddItemToObject(json, "colType", colType);
if (vAlterTbReq.colModType == TSDB_DATA_TYPE_BINARY || vAlterTbReq.colModType == TSDB_DATA_TYPE_GEOMETRY) {
if (vAlterTbReq.colModType == TSDB_DATA_TYPE_BINARY || vAlterTbReq.colModType == TSDB_DATA_TYPE_VARBINARY || vAlterTbReq.colModType == TSDB_DATA_TYPE_GEOMETRY) {
int32_t length = vAlterTbReq.colModBytes - VARSTR_HEADER_SIZE;
cJSON* cbytes = cJSON_CreateNumber(length);
cJSON_AddItemToObject(json, "colLength", cbytes);
@ -515,7 +521,11 @@ static char* processAlterTable(SMqMetaRsp* metaRsp) {
}
buf = parseTagDatatoJson(vAlterTbReq.pTagVal);
} else {
buf = taosMemoryCalloc(vAlterTbReq.nTagVal + 1, 1);
if(vAlterTbReq.tagType == TSDB_DATA_TYPE_VARBINARY){
buf = taosMemoryCalloc(vAlterTbReq.nTagVal + 1, 1);
}else{
buf = taosMemoryCalloc(vAlterTbReq.nTagVal + 1, 1);
}
dataConverToStr(buf, vAlterTbReq.tagType, vAlterTbReq.pTagVal, vAlterTbReq.nTagVal, NULL);
}

23
source/client/src/clientSml.c
View File

@ -596,7 +596,7 @@ static int32_t smlGenerateSchemaAction(SSchema *colField, SHashObj *colHash, SSm
return TSDB_CODE_SML_INVALID_DATA;
}
if (((colField[*index].type == TSDB_DATA_TYPE_VARCHAR || colField[*index].type == TSDB_DATA_TYPE_GEOMETRY) &&
if (((colField[*index].type == TSDB_DATA_TYPE_VARCHAR || colField[*index].type == TSDB_DATA_TYPE_VARBINARY || colField[*index].type == TSDB_DATA_TYPE_GEOMETRY) &&
(colField[*index].bytes - VARSTR_HEADER_SIZE) < kv->length) ||
(colField[*index].type == TSDB_DATA_TYPE_NCHAR &&
((colField[*index].bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE < kv->length))) {
@ -627,7 +627,7 @@ static int32_t smlFindNearestPowerOf2(int32_t length, uint8_t type) {
}
}
if ((type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_GEOMETRY) && result > TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) {
if ((type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_VARBINARY || type == TSDB_DATA_TYPE_GEOMETRY) && result > TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) {
result = TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE;
} else if (type == TSDB_DATA_TYPE_NCHAR && result > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) {
result = (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE;
@ -635,7 +635,7 @@ static int32_t smlFindNearestPowerOf2(int32_t length, uint8_t type) {
if (type == TSDB_DATA_TYPE_NCHAR) {
result = result * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE;
} else if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_GEOMETRY) {
} else if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_VARBINARY || type == TSDB_DATA_TYPE_GEOMETRY) {
result = result + VARSTR_HEADER_SIZE;
}
return result;
@ -679,7 +679,7 @@ static int32_t smlCheckMeta(SSchema *schema, int32_t length, SArray *cols, bool
}
static int32_t getBytes(uint8_t type, int32_t length) {
if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_GEOMETRY) {
if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_VARBINARY || type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_GEOMETRY) {
return smlFindNearestPowerOf2(length, type);
} else {
return tDataTypes[type].bytes;
@ -1178,21 +1178,12 @@ void smlDestroyTableInfo(void *para) {
taosMemoryFree(tag);
}
void clearColValArray(SArray *pCols) {
int32_t num = taosArrayGetSize(pCols);
for (int32_t i = 0; i < num; ++i) {
SColVal *pCol = taosArrayGet(pCols, i);
if (TSDB_DATA_TYPE_NCHAR == pCol->type) {
taosMemoryFreeClear(pCol->value.pData);
}
}
}
void freeSSmlKv(void *data) {
SSmlKv *kv = (SSmlKv *)data;
if (kv->keyEscaped) taosMemoryFree((void *)(kv->key));
if (kv->valueEscaped) taosMemoryFree((void *)(kv->value));
if (kv->keyEscaped) taosMemoryFreeClear(kv->key);
if (kv->valueEscaped) taosMemoryFreeClear(kv->value);
if (kv->type == TSDB_DATA_TYPE_GEOMETRY) geosFreeBuffer((void *)(kv->value));
if (kv->type == TSDB_DATA_TYPE_VARBINARY) taosMemoryFreeClear(kv->value);
}
void smlDestroyInfo(SSmlHandle *info) {

8
source/client/src/clientSmlJson.c
View File

@ -569,6 +569,8 @@ static int32_t smlConvertJSONNumber(SSmlKv *pVal, char *typeStr, cJSON *value) {
static int32_t smlConvertJSONString(SSmlKv *pVal, char *typeStr, cJSON *value) {
if (strcasecmp(typeStr, "binary") == 0) {
pVal->type = TSDB_DATA_TYPE_BINARY;
} else if (strcasecmp(typeStr, "varbinary") == 0) {
pVal->type = TSDB_DATA_TYPE_VARBINARY;
} else if (strcasecmp(typeStr, "nchar") == 0) {
pVal->type = TSDB_DATA_TYPE_NCHAR;
} else {
@ -577,7 +579,7 @@ static int32_t smlConvertJSONString(SSmlKv *pVal, char *typeStr, cJSON *value) {
}
pVal->length = strlen(value->valuestring);
if (pVal->type == TSDB_DATA_TYPE_BINARY && pVal->length > TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) {
if ((pVal->type == TSDB_DATA_TYPE_BINARY || pVal->type == TSDB_DATA_TYPE_VARBINARY) && pVal->length > TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) {
return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN;
}
if (pVal->type == TSDB_DATA_TYPE_NCHAR &&
@ -1010,7 +1012,7 @@ static int32_t smlParseJSONStringExt(SSmlHandle *info, cJSON *root, SSmlLineInfo
if (ret == TSDB_CODE_SUCCESS) {
ret = smlBuildRow(info->currTableDataCtx);
}
clearColValArray(info->currTableDataCtx->pValues);
clearColValArraySml(info->currTableDataCtx->pValues);
if (unlikely(ret != TSDB_CODE_SUCCESS)) {
smlBuildInvalidDataMsg(&info->msgBuf, "smlBuildCol error", NULL);
return ret;
@ -1214,7 +1216,7 @@ static int32_t smlParseJSONString(SSmlHandle *info, char **start, SSmlLineInfo *
if (ret == TSDB_CODE_SUCCESS) {
ret = smlBuildRow(info->currTableDataCtx);
}
clearColValArray(info->currTableDataCtx->pValues);
clearColValArraySml(info->currTableDataCtx->pValues);
if (unlikely(ret != TSDB_CODE_SUCCESS)) {
smlBuildInvalidDataMsg(&info->msgBuf, "smlBuildCol error", NULL);
return ret;

48
source/client/src/clientSmlLine.c
View File

@ -109,7 +109,7 @@ int32_t smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg) {
return code;
}
char* tmp = taosMemoryCalloc(pVal->length, 1);
memcpy(tmp, pVal->value + 2, pVal->length - 3);
memcpy(tmp, pVal->value + NCHAR_ADD_LEN - 1, pVal->length - NCHAR_ADD_LEN);
code = doGeomFromText(tmp, (unsigned char **)&pVal->value, &pVal->length);
taosMemoryFree(tmp);
if (code != TSDB_CODE_SUCCESS) {
@ -126,6 +126,44 @@ int32_t smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg) {
return TSDB_CODE_TSC_INVALID_VALUE;
}
if (pVal->value[0] == 'b' || pVal->value[0] == 'B') { // varbinary
if (pVal->value[1] == '"' && pVal->value[pVal->length - 1] == '"' && pVal->length >= 3) {
pVal->type = TSDB_DATA_TYPE_VARBINARY;
if(isHex(pVal->value + NCHAR_ADD_LEN - 1, pVal->length - NCHAR_ADD_LEN)){
if(!isValidateHex(pVal->value + NCHAR_ADD_LEN - 1, pVal->length - NCHAR_ADD_LEN)){
return TSDB_CODE_PAR_INVALID_VARBINARY;
}
void* data = NULL;
uint32_t size = 0;
if(taosHex2Ascii(pVal->value + NCHAR_ADD_LEN - 1, pVal->length - NCHAR_ADD_LEN, &data, &size) < 0){
return TSDB_CODE_OUT_OF_MEMORY;
}
if (size + VARSTR_HEADER_SIZE > TSDB_MAX_VARBINARY_LEN) {
taosMemoryFree(data);
return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN;
}
pVal->value = data;
pVal->length = size;
}else{
pVal->length -= NCHAR_ADD_LEN;
if (pVal->length > TSDB_MAX_VARBINARY_LEN - VARSTR_HEADER_SIZE) {
return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN;
}
void *data = taosMemoryMalloc(pVal->length);
if(data == NULL){
return TSDB_CODE_OUT_OF_MEMORY;
}
memcpy(data, pVal->value + (NCHAR_ADD_LEN - 1), pVal->length);
pVal->value = data;
}
return TSDB_CODE_SUCCESS;
}
return TSDB_CODE_TSC_INVALID_VALUE;
}
if (pVal->value[0] == 't' || pVal->value[0] == 'T') {
if (pVal->length == 1 ||
(pVal->length == 4 && (pVal->value[1] == 'r' || pVal->value[1] == 'R') &&
@ -414,7 +452,7 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
SSmlKv kv = {.key = tag->name, .keyLen = strlen(tag->name), .type = tag->type};
if (tag->type == TSDB_DATA_TYPE_NCHAR) {
kv.length = (tag->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE;
} else if (tag->type == TSDB_DATA_TYPE_BINARY || tag->type == TSDB_DATA_TYPE_GEOMETRY) {
} else if (tag->type == TSDB_DATA_TYPE_BINARY || tag->type == TSDB_DATA_TYPE_GEOMETRY || tag->type == TSDB_DATA_TYPE_VARBINARY) {
kv.length = tag->bytes - VARSTR_HEADER_SIZE;
}
taosArrayPush((*tmp)->cols, &kv);
@ -515,6 +553,10 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
char *tmp = (char *)taosMemoryMalloc(kv.length);
memcpy(tmp, kv.value, kv.length);
PROCESS_SLASH_IN_FIELD_VALUE(tmp, kv.length);
ASSERT(kv.type != TSDB_DATA_TYPE_GEOMETRY);
if(kv.type == TSDB_DATA_TYPE_VARBINARY){
taosMemoryFree((void*)kv.value);
}
kv.value = tmp;
kv.valueEscaped = valueEscaped;
}
@ -691,7 +733,7 @@ int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine
ret = smlBuildRow(info->currTableDataCtx);
}
clearColValArray(info->currTableDataCtx->pValues);
clearColValArraySml(info->currTableDataCtx->pValues);
if (unlikely(ret != TSDB_CODE_SUCCESS)) {
smlBuildInvalidDataMsg(&info->msgBuf, "smlBuildCol error", NULL);
return ret;

2
source/client/src/clientSmlTelnet.c
View File

@ -307,7 +307,7 @@ int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine
if (ret == TSDB_CODE_SUCCESS) {
ret = smlBuildRow(info->currTableDataCtx);
}
clearColValArray(info->currTableDataCtx->pValues);
clearColValArraySml(info->currTableDataCtx->pValues);
if (unlikely(ret != TSDB_CODE_SUCCESS)) {
smlBuildInvalidDataMsg(&info->msgBuf, "smlBuildCol error", NULL);
return ret;

4
source/client/src/clientTmq.c
View File

@ -985,6 +985,10 @@ int32_t tmq_subscription(tmq_t* tmq, tmq_list_t** topics) {
int32_t tmq_unsubscribe(tmq_t* tmq) {
if(tmq == NULL) return TSDB_CODE_INVALID_PARA;
if (tmq->status != TMQ_CONSUMER_STATUS__READY) {
tscInfo("consumer:0x%" PRIx64 " not in ready state, close it directly", tmq->consumerId);
return 0;
}
if (tmq->autoCommit) {
int32_t rsp = tmq_commit_sync(tmq, NULL);
if (rsp != 0) {

3
source/common/src/tdatablock.c
View File

@ -1920,6 +1920,7 @@ char* dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** pDataBuf,
if (len >= size - 1) return dumpBuf;
break;
case TSDB_DATA_TYPE_VARCHAR:
case TSDB_DATA_TYPE_VARBINARY:
case TSDB_DATA_TYPE_GEOMETRY: {
memset(pBuf, 0, sizeof(pBuf));
char* pData = colDataGetVarData(pColInfoData, j);
@ -2019,6 +2020,7 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq2** ppReq, const SSDataBlock* pDat
}
break;
case TSDB_DATA_TYPE_NCHAR:
case TSDB_DATA_TYPE_VARBINARY:
case TSDB_DATA_TYPE_VARCHAR: { // TSDB_DATA_TYPE_BINARY
ASSERT(pColInfoData->info.type == pCol->type);
if (colDataIsNull_s(pColInfoData, j)) {
@ -2032,7 +2034,6 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq2** ppReq, const SSDataBlock* pDat
}
break;
}
case TSDB_DATA_TYPE_VARBINARY:
case TSDB_DATA_TYPE_DECIMAL:
case TSDB_DATA_TYPE_BLOB:
case TSDB_DATA_TYPE_JSON:

1
source/common/src/tdataformat.c
View File

@ -1146,6 +1146,7 @@ static int tTagValJsonCmprFn(const void *p1, const void *p2) {
static void debugPrintTagVal(int8_t type, const void *val, int32_t vlen, const char *tag, int32_t ln) {
switch (type) {
case TSDB_DATA_TYPE_VARBINARY:
case TSDB_DATA_TYPE_JSON:
case TSDB_DATA_TYPE_VARCHAR:
case TSDB_DATA_TYPE_NCHAR:

60
source/common/src/tmsg.c
View File

@ -1092,6 +1092,7 @@ int32_t tSerializeSStatusReq(void *buf, int32_t bufLen, SStatusReq *pReq) {
if (tEncodeI64(&encoder, pReq->qload.numOfProcessedCQuery) < 0) return -1;
if (tEncodeI64(&encoder, pReq->qload.numOfProcessedFetch) < 0) return -1;
if (tEncodeI64(&encoder, pReq->qload.numOfProcessedDrop) < 0) return -1;
if (tEncodeI64(&encoder, pReq->qload.numOfProcessedNotify) < 0) return -1;
if (tEncodeI64(&encoder, pReq->qload.numOfProcessedHb) < 0) return -1;
if (tEncodeI64(&encoder, pReq->qload.numOfProcessedDelete) < 0) return -1;
if (tEncodeI64(&encoder, pReq->qload.cacheDataSize) < 0) return -1;
@ -1189,6 +1190,7 @@ int32_t tDeserializeSStatusReq(void *buf, int32_t bufLen, SStatusReq *pReq) {
if (tDecodeI64(&decoder, &pReq->qload.numOfProcessedCQuery) < 0) return -1;
if (tDecodeI64(&decoder, &pReq->qload.numOfProcessedFetch) < 0) return -1;
if (tDecodeI64(&decoder, &pReq->qload.numOfProcessedDrop) < 0) return -1;
if (tDecodeI64(&decoder, &pReq->qload.numOfProcessedNotify) < 0) return -1;
if (tDecodeI64(&decoder, &pReq->qload.numOfProcessedHb) < 0) return -1;
if (tDecodeI64(&decoder, &pReq->qload.numOfProcessedDelete) < 0) return -1;
if (tDecodeI64(&decoder, &pReq->qload.cacheDataSize) < 0) return -1;
@ -5874,6 +5876,64 @@ int32_t tDeserializeSTaskDropReq(void *buf, int32_t bufLen, STaskDropReq *pReq)
return 0;
}
int32_t tSerializeSTaskNotifyReq(void *buf, int32_t bufLen, STaskNotifyReq *pReq) {
int32_t headLen = sizeof(SMsgHead);
if (buf != NULL) {
buf = (char *)buf + headLen;
bufLen -= headLen;
}
SEncoder encoder = {0};
tEncoderInit(&encoder, buf, bufLen);
if (tStartEncode(&encoder) < 0) return -1;
if (tEncodeU64(&encoder, pReq->sId) < 0) return -1;
if (tEncodeU64(&encoder, pReq->queryId) < 0) return -1;
if (tEncodeU64(&encoder, pReq->taskId) < 0) return -1;
if (tEncodeI64(&encoder, pReq->refId) < 0) return -1;
if (tEncodeI32(&encoder, pReq->execId) < 0) return -1;
if (tEncodeI32(&encoder, pReq->type) < 0) return -1;
tEndEncode(&encoder);
int32_t tlen = encoder.pos;
tEncoderClear(&encoder);
if (buf != NULL) {
SMsgHead *pHead = (SMsgHead *)((char *)buf - headLen);
pHead->vgId = htonl(pReq->header.vgId);
pHead->contLen = htonl(tlen + headLen);
}
return tlen + headLen;
}
int32_t tDeserializeSTaskNotifyReq(void *buf, int32_t bufLen, STaskNotifyReq *pReq) {
int32_t headLen = sizeof(SMsgHead);
SMsgHead *pHead = buf;
pHead->vgId = pReq->header.vgId;
pHead->contLen = pReq->header.contLen;
SDecoder decoder = {0};
tDecoderInit(&decoder, (char *)buf + headLen, bufLen - headLen);
if (tStartDecode(&decoder) < 0) return -1;
if (tDecodeU64(&decoder, &pReq->sId) < 0) return -1;
if (tDecodeU64(&decoder, &pReq->queryId) < 0) return -1;
if (tDecodeU64(&decoder, &pReq->taskId) < 0) return -1;
if (tDecodeI64(&decoder, &pReq->refId) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->execId) < 0) return -1;
if (tDecodeI32(&decoder, (int32_t*)&pReq->type) < 0) return -1;
tEndDecode(&decoder);
tDecoderClear(&decoder);
return 0;
}
int32_t tSerializeSQueryTableRsp(void *buf, int32_t bufLen, SQueryTableRsp *pRsp) {
SEncoder encoder = {0};
tEncoderInit(&encoder, buf, bufLen);

2
source/common/src/trow.c
View File

@ -356,7 +356,7 @@ int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow) {
}
} else {
varDataLen += sizeof(VarDataLenT);
if (pTColumn->type == TSDB_DATA_TYPE_VARCHAR || pTColumn->type == TSDB_DATA_TYPE_GEOMETRY) {
if (pTColumn->type == TSDB_DATA_TYPE_VARCHAR || pTColumn->type == TSDB_DATA_TYPE_VARBINARY || pTColumn->type == TSDB_DATA_TYPE_GEOMETRY) {
varDataLen += CHAR_BYTES;
if (maxVarDataLen < CHAR_BYTES + sizeof(VarDataLenT)) {
maxVarDataLen = CHAR_BYTES + sizeof(VarDataLenT);

6
source/common/src/ttszip.c
View File

@ -296,7 +296,8 @@ static void writeDataToDisk(STSBuf* pTSBuf) {
metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, &pBlock->tag.nType, sizeof(pBlock->tag.nType));
int32_t trueLen = pBlock->tag.nLen;
if (pBlock->tag.nType == TSDB_DATA_TYPE_BINARY || pBlock->tag.nType == TSDB_DATA_TYPE_NCHAR || pBlock->tag.nType == TSDB_DATA_TYPE_GEOMETRY) {
if (pBlock->tag.nType == TSDB_DATA_TYPE_BINARY || pBlock->tag.nType == TSDB_DATA_TYPE_VARBINARY ||
pBlock->tag.nType == TSDB_DATA_TYPE_NCHAR || pBlock->tag.nType == TSDB_DATA_TYPE_GEOMETRY) {
metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, &pBlock->tag.nLen, sizeof(pBlock->tag.nLen));
metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, pBlock->tag.pz, (size_t)pBlock->tag.nLen);
} else if (pBlock->tag.nType == TSDB_DATA_TYPE_FLOAT) {
@ -378,7 +379,8 @@ STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) {
// NOTE: mix types tags are not supported
size_t sz = 0;
if (pBlock->tag.nType == TSDB_DATA_TYPE_BINARY || pBlock->tag.nType == TSDB_DATA_TYPE_NCHAR || pBlock->tag.nType == TSDB_DATA_TYPE_GEOMETRY) {
if (pBlock->tag.nType == TSDB_DATA_TYPE_BINARY || pBlock->tag.nType == TSDB_DATA_TYPE_VARBINARY ||
pBlock->tag.nType == TSDB_DATA_TYPE_NCHAR || pBlock->tag.nType == TSDB_DATA_TYPE_GEOMETRY) {
char* tp = taosMemoryRealloc(pBlock->tag.pz, pBlock->tag.nLen + 1);
ASSERT(tp != NULL);

3
source/common/src/ttypes.c
View File

@ -61,7 +61,7 @@ tDataTypeDescriptor tDataTypes[TSDB_DATA_TYPE_MAX] = {
{TSDB_DATA_TYPE_UINT, 12, INT_BYTES, "INT UNSIGNED", 0, UINT32_MAX, tsCompressInt, tsDecompressInt},
{TSDB_DATA_TYPE_UBIGINT, 15, LONG_BYTES, "BIGINT UNSIGNED", 0, UINT64_MAX, tsCompressBigint, tsDecompressBigint},
{TSDB_DATA_TYPE_JSON, 4, TSDB_MAX_JSON_TAG_LEN, "JSON", 0, 0, tsCompressString, tsDecompressString},
{TSDB_DATA_TYPE_VARBINARY, 9, 1, "VARBINARY", 0, 0, NULL, NULL}, // placeholder, not implemented
{TSDB_DATA_TYPE_VARBINARY, 9, 1, "VARBINARY", 0, 0, tsCompressString, tsDecompressString}, // placeholder, not implemented
{TSDB_DATA_TYPE_DECIMAL, 7, 1, "DECIMAL", 0, 0, NULL, NULL}, // placeholder, not implemented
{TSDB_DATA_TYPE_BLOB, 4, 1, "BLOB", 0, 0, NULL, NULL}, // placeholder, not implemented
{TSDB_DATA_TYPE_MEDIUMBLOB, 10, 1, "MEDIUMBLOB", 0, 0, NULL, NULL}, // placeholder, not implemented
@ -135,6 +135,7 @@ void assignVal(char *val, const char *src, int32_t len, int32_t type) {
*((int64_t *)val) = GET_INT64_VAL(src);
break;
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_VARBINARY:
case TSDB_DATA_TYPE_GEOMETRY:
varDataCopy(val, src);
break;

13
source/common/src/tvariant.c
View File

@ -122,6 +122,7 @@ void taosVariantCreateFromBinary(SVariant *pVar, const char *pz, size_t len, uin
break;
}
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_VARBINARY:
case TSDB_DATA_TYPE_GEOMETRY: { // todo refactor, extract a method
pVar->pz = taosMemoryCalloc(len + 1, sizeof(char));
memcpy(pVar->pz, pz, len);
@ -141,7 +142,8 @@ void taosVariantDestroy(SVariant *pVar) {
if (pVar == NULL) return;
if (pVar->nType == TSDB_DATA_TYPE_BINARY || pVar->nType == TSDB_DATA_TYPE_NCHAR ||
pVar->nType == TSDB_DATA_TYPE_JSON || pVar->nType == TSDB_DATA_TYPE_GEOMETRY) {
pVar->nType == TSDB_DATA_TYPE_JSON || pVar->nType == TSDB_DATA_TYPE_GEOMETRY ||
pVar->nType == TSDB_DATA_TYPE_VARBINARY) {
taosMemoryFreeClear(pVar->pz);
pVar->nLen = 0;
}
@ -152,8 +154,9 @@ void taosVariantAssign(SVariant *pDst, const SVariant *pSrc) {
if (pSrc == NULL || pDst == NULL) return;
pDst->nType = pSrc->nType;
if (pSrc->nType == TSDB_DATA_TYPE_BINARY || pSrc->nType == TSDB_DATA_TYPE_NCHAR ||
pSrc->nType == TSDB_DATA_TYPE_JSON || pSrc->nType == TSDB_DATA_TYPE_GEOMETRY) {
if (pSrc->nType == TSDB_DATA_TYPE_BINARY ||pSrc->nType == TSDB_DATA_TYPE_VARBINARY ||
pSrc->nType == TSDB_DATA_TYPE_NCHAR || pSrc->nType == TSDB_DATA_TYPE_JSON ||
pSrc->nType == TSDB_DATA_TYPE_GEOMETRY) {
int32_t len = pSrc->nLen + TSDB_NCHAR_SIZE;
char *p = taosMemoryRealloc(pDst->pz, len);
ASSERT(p);
@ -185,7 +188,8 @@ int32_t taosVariantCompare(const SVariant *p1, const SVariant *p2) {
return 1;
}
if (p1->nType == TSDB_DATA_TYPE_BINARY || p1->nType == TSDB_DATA_TYPE_NCHAR || p1->nType == TSDB_DATA_TYPE_GEOMETRY) {
if (p1->nType == TSDB_DATA_TYPE_BINARY || p1->nType == TSDB_DATA_TYPE_VARBINARY ||
p1->nType == TSDB_DATA_TYPE_NCHAR || p1->nType == TSDB_DATA_TYPE_GEOMETRY) {
if (p1->nLen == p2->nLen) {
return memcmp(p1->pz, p2->pz, p1->nLen);
} else {
@ -237,6 +241,7 @@ char *taosVariantGet(SVariant *pVar, int32_t type) {
case TSDB_DATA_TYPE_FLOAT:
return (char *)&pVar->f;
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_VARBINARY:
case TSDB_DATA_TYPE_JSON:
case TSDB_DATA_TYPE_GEOMETRY:
return (char *)pVar->pz;

1
source/dnode/mgmt/mgmt_mnode/src/mmHandle.c
View File

@ -189,6 +189,7 @@ SArray *mmGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_SCH_QUERY_HEARTBEAT, mmPutMsgToFetchQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_SCH_FETCH, mmPutMsgToFetchQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_SCH_MERGE_FETCH, mmPutMsgToFetchQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_SCH_TASK_NOTIFY, mmPutMsgToFetchQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_CREATE_STB_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_STB_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_STB_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;

1
source/dnode/mgmt/mgmt_qnode/src/qmHandle.c
View File

@ -89,6 +89,7 @@ SArray *qmGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_SCH_CANCEL_TASK, qmPutNodeMsgToFetchQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_SCH_DROP_TASK, qmPutNodeMsgToFetchQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_SCH_TASK_NOTIFY, qmPutNodeMsgToFetchQueue, 1) == NULL) goto _OVER;
code = 0;
_OVER:

1
source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
View File

@ -755,6 +755,7 @@ SArray *vmGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_VND_TABLES_META, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_SCH_CANCEL_TASK, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_SCH_DROP_TASK, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_SCH_TASK_NOTIFY, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_CREATE_STB, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_TTL_TABLE, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_STB, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;

2
source/dnode/mgmt/node_mgmt/src/dmTransport.c
View File

@ -272,7 +272,7 @@ static bool rpcRfp(int32_t code, tmsg_t msgType) {
code == TSDB_CODE_SYN_RESTORING || code == TSDB_CODE_VND_STOPPED || code == TSDB_CODE_APP_IS_STARTING ||
code == TSDB_CODE_APP_IS_STOPPING) {
if (msgType == TDMT_SCH_QUERY || msgType == TDMT_SCH_MERGE_QUERY || msgType == TDMT_SCH_FETCH ||
msgType == TDMT_SCH_MERGE_FETCH) {
msgType == TDMT_SCH_MERGE_FETCH || msgType == TDMT_SCH_TASK_NOTIFY) {
return false;
}
return true;

3
source/dnode/mnode/impl/src/mndFunc.c
View File

@ -565,7 +565,8 @@ static void *mnodeGenTypeStr(char *buf, int32_t buflen, uint8_t type, int32_t le
return msg;
}
if (type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_GEOMETRY) {
if (type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_VARBINARY ||
type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_GEOMETRY) {
int32_t bytes = len > 0 ? (int32_t)(len - VARSTR_HEADER_SIZE) : len;
snprintf(buf, buflen - 1, "%s(%d)", tDataTypes[type].name, type == TSDB_DATA_TYPE_NCHAR ? bytes / 4 : bytes);

3
source/dnode/mnode/impl/src/mndMain.c
View File

@ -633,7 +633,8 @@ static int32_t mndCheckMnodeState(SRpcMsg *pMsg) {
if (!IsReq(pMsg)) return 0;
if (pMsg->msgType == TDMT_SCH_QUERY || pMsg->msgType == TDMT_SCH_MERGE_QUERY ||
pMsg->msgType == TDMT_SCH_QUERY_CONTINUE || pMsg->msgType == TDMT_SCH_QUERY_HEARTBEAT ||
pMsg->msgType == TDMT_SCH_FETCH || pMsg->msgType == TDMT_SCH_MERGE_FETCH || pMsg->msgType == TDMT_SCH_DROP_TASK) {
pMsg->msgType == TDMT_SCH_FETCH || pMsg->msgType == TDMT_SCH_MERGE_FETCH || pMsg->msgType == TDMT_SCH_DROP_TASK ||
pMsg->msgType == TDMT_SCH_TASK_NOTIFY) {
return 0;
}

4
source/dnode/mnode/impl/src/mndQuery.c
View File

@ -55,6 +55,9 @@ int32_t mndProcessQueryMsg(SRpcMsg *pMsg) {
case TDMT_SCH_QUERY_HEARTBEAT:
code = qWorkerProcessHbMsg(pMnode, pMnode->pQuery, pMsg, 0);
break;
case TDMT_SCH_TASK_NOTIFY:
code = qWorkerProcessNotifyMsg(pMnode, pMnode->pQuery, pMsg, 0);
break;
default:
terrno = TSDB_CODE_APP_ERROR;
mError("unknown msg type:%d in query queue", pMsg->msgType);
@ -175,6 +178,7 @@ int32_t mndInitQuery(SMnode *pMnode) {
mndSetMsgHandle(pMnode, TDMT_SCH_QUERY_CONTINUE, mndProcessQueryMsg);
mndSetMsgHandle(pMnode, TDMT_SCH_FETCH, mndProcessQueryMsg);
mndSetMsgHandle(pMnode, TDMT_SCH_MERGE_FETCH, mndProcessQueryMsg);
mndSetMsgHandle(pMnode, TDMT_SCH_TASK_NOTIFY, mndProcessQueryMsg);
mndSetMsgHandle(pMnode, TDMT_SCH_DROP_TASK, mndProcessQueryMsg);
mndSetMsgHandle(pMnode, TDMT_SCH_QUERY_HEARTBEAT, mndProcessQueryMsg);
mndSetMsgHandle(pMnode, TDMT_MND_BATCH_META, mndProcessBatchMetaMsg);

8
source/dnode/mnode/impl/src/mndStb.c
View File

@ -1549,8 +1549,8 @@ static int32_t mndAlterStbTagBytes(SMnode *pMnode, const SStbObj *pOld, SStbObj
SSchema *pTag = pNew->pTags + tag;
if (!(pTag->type == TSDB_DATA_TYPE_BINARY || pTag->type == TSDB_DATA_TYPE_NCHAR ||
pTag->type == TSDB_DATA_TYPE_GEOMETRY)) {
if (!(pTag->type == TSDB_DATA_TYPE_BINARY || pTag->type == TSDB_DATA_TYPE_VARBINARY ||
pTag->type == TSDB_DATA_TYPE_NCHAR || pTag->type == TSDB_DATA_TYPE_GEOMETRY)) {
terrno = TSDB_CODE_MND_INVALID_STB_OPTION;
return -1;
}
@ -1671,8 +1671,8 @@ static int32_t mndAlterStbColumnBytes(SMnode *pMnode, const SStbObj *pOld, SStbO
}
SSchema *pCol = pNew->pColumns + col;
if (!(pCol->type == TSDB_DATA_TYPE_BINARY || pCol->type == TSDB_DATA_TYPE_NCHAR ||
pCol->type == TSDB_DATA_TYPE_GEOMETRY)) {
if (!(pCol->type == TSDB_DATA_TYPE_BINARY || pCol->type == TSDB_DATA_TYPE_VARBINARY ||
pCol->type == TSDB_DATA_TYPE_NCHAR || pCol->type == TSDB_DATA_TYPE_GEOMETRY)) {
terrno = TSDB_CODE_MND_INVALID_STB_OPTION;
return -1;
}

5
source/dnode/mnode/impl/src/mndTopic.c
View File

@ -4,7 +4,7 @@
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
*f
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
@ -860,7 +860,8 @@ static void schemaToJson(SSchema *schema, int32_t nCols, char *schemaJson){
cJSON* ctype = cJSON_CreateString(tDataTypes[s->type].name);
cJSON_AddItemToObject(column, "type", ctype);
int32_t length = 0;
if (s->type == TSDB_DATA_TYPE_BINARY) {
if (s->type == TSDB_DATA_TYPE_BINARY || s->type == TSDB_DATA_TYPE_VARBINARY ||
s->type == TSDB_DATA_TYPE_GEOMETRY) {
length = s->bytes - VARSTR_HEADER_SIZE;
} else if (s->type == TSDB_DATA_TYPE_NCHAR || s->type == TSDB_DATA_TYPE_JSON) {
length = (s->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE;

4
source/dnode/qnode/src/qnode.c
View File

@ -57,6 +57,7 @@ int32_t qndGetLoad(SQnode *pQnode, SQnodeLoad *pLoad) {
pLoad->numOfProcessedCQuery = stat.cqueryProcessed;
pLoad->numOfProcessedFetch = stat.fetchProcessed;
pLoad->numOfProcessedDrop = stat.dropProcessed;
pLoad->numOfProcessedNotify = stat.notifyProcessed;
pLoad->numOfProcessedHb = stat.hbProcessed;
pLoad->numOfProcessedDelete = stat.deleteProcessed;
@ -100,6 +101,9 @@ int32_t qndProcessQueryMsg(SQnode *pQnode, int64_t ts, SRpcMsg *pMsg) {
case TDMT_SCH_QUERY_HEARTBEAT:
code = qWorkerProcessHbMsg(pQnode, pQnode->pQuery, pMsg, ts);
break;
case TDMT_SCH_TASK_NOTIFY:
code = qWorkerProcessNotifyMsg(pQnode, pQnode->pQuery, pMsg, ts);
break;
default:
qError("unknown msg type:%d in qnode queue", pMsg->msgType);
terrno = TSDB_CODE_APP_ERROR;

14
source/dnode/vnode/src/tsdb/tsdbReaderWriter.c
View File

@ -35,12 +35,12 @@ static int32_t tsdbOpenFileImpl(STsdbFD *pFD) {
if (pFD->pFD == NULL) {
code = TAOS_SYSTEM_ERROR(ENOENT);
taosMemoryFree(pFD);
// taosMemoryFree(pFD);
goto _exit;
}
} else {
code = TAOS_SYSTEM_ERROR(errsv);
taosMemoryFree(pFD);
// taosMemoryFree(pFD);
goto _exit;
}
}
@ -48,8 +48,8 @@ static int32_t tsdbOpenFileImpl(STsdbFD *pFD) {
pFD->pBuf = taosMemoryCalloc(1, szPage);
if (pFD->pBuf == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
taosCloseFile(&pFD->pFD);
taosMemoryFree(pFD);
// taosCloseFile(&pFD->pFD);
// taosMemoryFree(pFD);
goto _exit;
}
@ -57,9 +57,9 @@ static int32_t tsdbOpenFileImpl(STsdbFD *pFD) {
if (flag != TD_FILE_READ) {
if (taosStatFile(path, &pFD->szFile, NULL, NULL) < 0) {
code = TAOS_SYSTEM_ERROR(errno);
taosMemoryFree(pFD->pBuf);
taosCloseFile(&pFD->pFD);
taosMemoryFree(pFD);
// taosMemoryFree(pFD->pBuf);
// taosCloseFile(&pFD->pFD);
// taosMemoryFree(pFD);
goto _exit;
}

2
source/dnode/vnode/src/vnd/vnodeSvr.c
View File

@ -711,6 +711,8 @@ int32_t vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) {
// return qWorkerProcessCancelMsg(pVnode, pVnode->pQuery, pMsg, 0);
case TDMT_SCH_DROP_TASK:
return qWorkerProcessDropMsg(pVnode, pVnode->pQuery, pMsg, 0);
case TDMT_SCH_TASK_NOTIFY:
return qWorkerProcessNotifyMsg(pVnode, pVnode->pQuery, pMsg, 0);
case TDMT_SCH_QUERY_HEARTBEAT:
return qWorkerProcessHbMsg(pVnode, pVnode->pQuery, pMsg, 0);
case TDMT_VND_TABLE_META:

5
source/libs/command/src/command.c
View File

@ -48,6 +48,7 @@ static int32_t buildRetrieveTableRsp(SSDataBlock* pBlock, int32_t numOfCols, SRe
static int32_t getSchemaBytes(const SSchema* pSchema) {
switch (pSchema->type) {
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_VARBINARY:
case TSDB_DATA_TYPE_GEOMETRY:
return (pSchema->bytes - VARSTR_HEADER_SIZE);
case TSDB_DATA_TYPE_NCHAR:
@ -457,7 +458,7 @@ void appendColumnFields(char* buf, int32_t* len, STableCfg* pCfg) {
SSchema* pSchema = pCfg->pSchemas + i;
char type[32];
sprintf(type, "%s", tDataTypes[pSchema->type].name);
if (TSDB_DATA_TYPE_VARCHAR == pSchema->type || TSDB_DATA_TYPE_GEOMETRY == pSchema->type) {
if (TSDB_DATA_TYPE_VARCHAR == pSchema->type || TSDB_DATA_TYPE_VARBINARY == pSchema->type || TSDB_DATA_TYPE_GEOMETRY == pSchema->type) {
sprintf(type + strlen(type), "(%d)", (int32_t)(pSchema->bytes - VARSTR_HEADER_SIZE));
} else if (TSDB_DATA_TYPE_NCHAR == pSchema->type) {
sprintf(type + strlen(type), "(%d)", (int32_t)((pSchema->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE));
@ -472,7 +473,7 @@ void appendTagFields(char* buf, int32_t* len, STableCfg* pCfg) {
SSchema* pSchema = pCfg->pSchemas + pCfg->numOfColumns + i;
char type[32];
sprintf(type, "%s", tDataTypes[pSchema->type].name);
if (TSDB_DATA_TYPE_VARCHAR == pSchema->type || TSDB_DATA_TYPE_GEOMETRY == pSchema->type) {
if (TSDB_DATA_TYPE_VARCHAR == pSchema->type || TSDB_DATA_TYPE_VARBINARY == pSchema->type || TSDB_DATA_TYPE_GEOMETRY == pSchema->type) {
sprintf(type + strlen(type), "(%d)", (int32_t)(pSchema->bytes - VARSTR_HEADER_SIZE));
} else if (TSDB_DATA_TYPE_NCHAR == pSchema->type) {
sprintf(type + strlen(type), "(%d)", (int32_t)((pSchema->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE));

2
source/libs/executor/src/dataInserter.c
View File

@ -213,6 +213,7 @@ int32_t buildSubmitReqFromBlock(SDataInserterHandle* pInserter, SSubmitReq2** pp
switch (pColInfoData->info.type) {
case TSDB_DATA_TYPE_NCHAR:
case TSDB_DATA_TYPE_VARBINARY:
case TSDB_DATA_TYPE_VARCHAR: { // TSDB_DATA_TYPE_BINARY
ASSERT(pColInfoData->info.type == pCol->type);
if (colDataIsNull_s(pColInfoData, j)) {
@ -226,7 +227,6 @@ int32_t buildSubmitReqFromBlock(SDataInserterHandle* pInserter, SSubmitReq2** pp
}
break;
}
case TSDB_DATA_TYPE_VARBINARY:
case TSDB_DATA_TYPE_DECIMAL:
case TSDB_DATA_TYPE_BLOB:
case TSDB_DATA_TYPE_JSON:

3
source/libs/executor/src/sysscanoperator.c
View File

@ -400,8 +400,7 @@ static bool sysTableIsOperatorCondOnOneTable(SNode* pCond, char* condTable) {
strcasecmp(nodesGetNameFromColumnNode(node->pLeft), "table_name") == 0 &&
nodeType(node->pRight) == QUERY_NODE_VALUE) {
SValueNode* pValue = (SValueNode*)node->pRight;
if (pValue->node.resType.type == TSDB_DATA_TYPE_NCHAR || pValue->node.resType.type == TSDB_DATA_TYPE_VARCHAR ||
pValue->node.resType.type == TSDB_DATA_TYPE_BINARY) {
if (pValue->node.resType.type == TSDB_DATA_TYPE_NCHAR || pValue->node.resType.type == TSDB_DATA_TYPE_VARCHAR) {
char* value = nodesGetValueFromNode(pValue);
strncpy(condTable, varDataVal(value), TSDB_TABLE_NAME_LEN);
return true;

2
source/libs/executor/src/timewindowoperator.c
View File

@ -281,7 +281,7 @@ void doTimeWindowInterpolation(SArray* pPrevValues, SArray* pDataBlock, TSKEY pr
pCtx[k].end.key = curTs;
pCtx[k].end.val = v2;
if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR ||
if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_VARBINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR ||
pColInfo->info.type == TSDB_DATA_TYPE_GEOMETRY) {
if (prevRowIndex == -1) {
// pCtx[k].start.ptr = (char*)pRuntimeEnv->prevRow[index];

2
source/libs/function/src/udfd.c
View File

@ -971,7 +971,7 @@ static bool udfdRpcRfp(int32_t code, tmsg_t msgType) {
code == TSDB_CODE_RPC_SOMENODE_NOT_CONNECTED || code == TSDB_CODE_SYN_RESTORING ||
code == TSDB_CODE_MNODE_NOT_FOUND || code == TSDB_CODE_APP_IS_STARTING || code == TSDB_CODE_APP_IS_STOPPING) {
if (msgType == TDMT_SCH_QUERY || msgType == TDMT_SCH_MERGE_QUERY || msgType == TDMT_SCH_FETCH ||
msgType == TDMT_SCH_MERGE_FETCH) {
msgType == TDMT_SCH_MERGE_FETCH || msgType == TDMT_SCH_TASK_NOTIFY) {
return false;
}
return true;

16
source/libs/index/src/indexComm.c
View File

@ -76,7 +76,8 @@ char* idxInt2str(int64_t val, char* dst, int radix) {
return dst - 1;
}
__compar_fn_t idxGetCompar(int8_t type) {
if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_GEOMETRY) {
if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_VARBINARY ||
type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_GEOMETRY) {
return (__compar_fn_t)strcmp;
}
return getComparFunc(type, 0);
@ -356,17 +357,13 @@ int32_t idxConvertData(void* src, int8_t type, void** dst) {
break;
}
case TSDB_DATA_TYPE_VARCHAR: // TSDB_DATA_TYPE_BINARY
case TSDB_DATA_TYPE_VARBINARY:
case TSDB_DATA_TYPE_GEOMETRY: {
tlen = taosEncodeBinary(NULL, src, strlen(src));
*dst = taosMemoryCalloc(1, tlen + 1);
tlen = taosEncodeBinary(dst, src, strlen(src));
break;
}
case TSDB_DATA_TYPE_VARBINARY:
tlen = taosEncodeBinary(NULL, src, strlen(src));
*dst = taosMemoryCalloc(1, tlen + 1);
tlen = taosEncodeBinary(dst, src, strlen(src));
break;
default:
ASSERTS(0, "index invalid input type");
break;
@ -447,6 +444,7 @@ int32_t idxConvertDataToStr(void* src, int8_t type, void** dst) {
break;
}
case TSDB_DATA_TYPE_VARCHAR: // TSDB_DATA_TYPE_BINARY
case TSDB_DATA_TYPE_VARBINARY:
case TSDB_DATA_TYPE_GEOMETRY: {
tlen = taosEncodeBinary(NULL, varDataVal(src), varDataLen(src));
*dst = taosMemoryCalloc(1, tlen + 1);
@ -454,12 +452,6 @@ int32_t idxConvertDataToStr(void* src, int8_t type, void** dst) {
*dst = (char*)*dst - tlen;
break;
}
case TSDB_DATA_TYPE_VARBINARY:
tlen = taosEncodeBinary(NULL, varDataVal(src), varDataLen(src));
*dst = taosMemoryCalloc(1, tlen + 1);
tlen = taosEncodeBinary(dst, varDataVal(src), varDataLen(src));
*dst = (char*)*dst - tlen;
break;
default:
ASSERTS(0, "index invalid input type");
break;

4
source/libs/index/src/indexJson.c
View File

@ -24,7 +24,7 @@ int indexJsonPut(SIndexJson *index, SIndexJsonMultiTerm *terms, uint64_t uid) {
SIndexJsonTerm *p = taosArrayGetP(terms, i);
if (p->colType == TSDB_DATA_TYPE_BOOL) {
p->colType = TSDB_DATA_TYPE_INT;
} else if (p->colType == TSDB_DATA_TYPE_VARCHAR || p->colType == TSDB_DATA_TYPE_NCHAR ||
} else if (p->colType == TSDB_DATA_TYPE_VARBINARY || p->colType == TSDB_DATA_TYPE_NCHAR ||
p->colType == TSDB_DATA_TYPE_BINARY || p->colType == TSDB_DATA_TYPE_GEOMETRY) {
// p->colType = TSDB_DATA_TYPE_NCHAR;
} else {
@ -42,7 +42,7 @@ int indexJsonSearch(SIndexJson *index, SIndexJsonMultiTermQuery *tq, SArray *res
SIndexJsonTerm *p = taosArrayGetP(terms, i);
if (p->colType == TSDB_DATA_TYPE_BOOL) {
p->colType = TSDB_DATA_TYPE_INT;
} else if (p->colType == TSDB_DATA_TYPE_VARCHAR || p->colType == TSDB_DATA_TYPE_NCHAR ||
} else if (p->colType == TSDB_DATA_TYPE_VARBINARY || p->colType == TSDB_DATA_TYPE_NCHAR ||
p->colType == TSDB_DATA_TYPE_BINARY || p->colType == TSDB_DATA_TYPE_GEOMETRY) {
// p->colType = TSDB_DATA_TYPE_NCHAR;
} else {

4
source/libs/index/src/indexTfile.c
View File

@ -545,8 +545,8 @@ int tfileWriterPut(TFileWriter* tw, void* data, bool order) {
int8_t colType = tw->header.colType;
colType = IDX_TYPE_GET_TYPE(colType);
if (colType == TSDB_DATA_TYPE_BINARY || colType == TSDB_DATA_TYPE_NCHAR ||
colType == TSDB_DATA_TYPE_GEOMETRY) {
if (colType == TSDB_DATA_TYPE_BINARY || colType == TSDB_DATA_TYPE_VARBINARY ||
colType == TSDB_DATA_TYPE_NCHAR || colType == TSDB_DATA_TYPE_GEOMETRY) {
fn = tfileStrCompare;
} else {
fn = getComparFunc(colType, 0);

32
source/libs/parser/src/parInsertSml.c
View File

@ -18,18 +18,6 @@
#include "parToken.h"
#include "ttime.h"
static void clearColValArray(SArray* pCols) {
int32_t num = taosArrayGetSize(pCols);
for (int32_t i = 0; i < num; ++i) {
SColVal* pCol = taosArrayGet(pCols, i);
if (TSDB_DATA_TYPE_NCHAR == pCol->type || TSDB_DATA_TYPE_GEOMETRY == pCol->type) {
taosMemoryFreeClear(pCol->value.pData);
}
pCol->flag = CV_FLAG_NONE;
pCol->value.val = 0;
}
}
int32_t qCreateSName(SName* pName, const char* pTableName, int32_t acctId, char* dbName, char* msgBuf,
int32_t msgBufLen) {
SMsgBuf msg = {.buf = msgBuf, .len = msgBufLen};
@ -134,7 +122,7 @@ static int32_t smlBuildTagRow(SArray* cols, SBoundColInfo* tags, SSchema* pSchem
taosArrayPush(*tagName, pTagSchema->name);
STagVal val = {.cid = pTagSchema->colId, .type = pTagSchema->type};
// strcpy(val.colName, pTagSchema->name);
if (pTagSchema->type == TSDB_DATA_TYPE_BINARY ||
if (pTagSchema->type == TSDB_DATA_TYPE_BINARY || pTagSchema->type == TSDB_DATA_TYPE_VARBINARY ||
pTagSchema->type == TSDB_DATA_TYPE_GEOMETRY) {
val.pData = (uint8_t*)kv->value;
val.nData = kv->length;
@ -193,6 +181,18 @@ STableDataCxt* smlInitTableDataCtx(SQuery* query, STableMeta* pTableMeta) {
return pTableCxt;
}
void clearColValArraySml(SArray* pCols) {
int32_t num = taosArrayGetSize(pCols);
for (int32_t i = 0; i < num; ++i) {
SColVal* pCol = taosArrayGet(pCols, i);
if (TSDB_DATA_TYPE_NCHAR == pCol->type || TSDB_DATA_TYPE_GEOMETRY == pCol->type || TSDB_DATA_TYPE_VARBINARY == pCol->type) {
taosMemoryFreeClear(pCol->value.pData);
}
pCol->flag = CV_FLAG_NONE;
pCol->value.val = 0;
}
}
int32_t smlBuildRow(STableDataCxt* pTableCxt) {
SRow** pRow = taosArrayReserve(pTableCxt->pData->aRowP, 1);
int ret = tRowBuild(pTableCxt->pValues, pTableCxt->pSchema, pRow);
@ -240,7 +240,7 @@ int32_t smlBuildCol(STableDataCxt* pTableCxt, SSchema* schema, void* data, int32
} else if (kv->type == TSDB_DATA_TYPE_BINARY) {
pVal->value.nData = kv->length;
pVal->value.pData = (uint8_t*)kv->value;
} else if (kv->type == TSDB_DATA_TYPE_GEOMETRY) {
} else if (kv->type == TSDB_DATA_TYPE_GEOMETRY || kv->type == TSDB_DATA_TYPE_VARBINARY) {
pVal->value.nData = kv->length;
pVal->value.pData = taosMemoryMalloc(kv->length);
memcpy(pVal->value.pData, (uint8_t*)kv->value, kv->length);
@ -371,7 +371,7 @@ int32_t smlBindData(SQuery* query, bool dataFormat, SArray* tags, SArray* colsSc
} else if (kv->type == TSDB_DATA_TYPE_BINARY) {
pVal->value.nData = kv->length;
pVal->value.pData = (uint8_t*)kv->value;
} else if (kv->type == TSDB_DATA_TYPE_GEOMETRY) {
} else if (kv->type == TSDB_DATA_TYPE_GEOMETRY || kv->type == TSDB_DATA_TYPE_VARBINARY) {
pVal->value.nData = kv->length;
pVal->value.pData = taosMemoryMalloc(kv->length);
memcpy(pVal->value.pData, (uint8_t*)kv->value, kv->length);
@ -388,7 +388,7 @@ int32_t smlBindData(SQuery* query, bool dataFormat, SArray* tags, SArray* colsSc
goto end;
}
insCheckTableDataOrder(pTableCxt, TD_ROW_KEY(*pRow));
clearColValArray(pTableCxt->pValues);
clearColValArraySml(pTableCxt->pValues);
}
end:

49
source/libs/parser/src/parInsertSql.c
View File

@ -326,6 +326,38 @@ static int parseGeometry(SToken *pToken, unsigned char **output, size_t *size) {
return code;
}
static int32_t parseVarbinary(SToken* pToken, uint8_t **pData, uint32_t *nData, int32_t bytes){
if(pToken->type != TK_NK_STRING){
return TSDB_CODE_PAR_INVALID_VARBINARY;
}
if(isHex(pToken->z, pToken->n)){
if(!isValidateHex(pToken->z, pToken->n)){
return TSDB_CODE_PAR_INVALID_VARBINARY;
}
void* data = NULL;
uint32_t size = 0;
if(taosHex2Ascii(pToken->z, pToken->n, &data, &size) < 0){
return TSDB_CODE_OUT_OF_MEMORY;
}
if (size + VARSTR_HEADER_SIZE > bytes) {
taosMemoryFree(data);
return TSDB_CODE_PAR_VALUE_TOO_LONG;
}
*pData = data;
*nData = size;
}else{
if (pToken->n + VARSTR_HEADER_SIZE > bytes) {
return TSDB_CODE_PAR_VALUE_TOO_LONG;
}
*pData = taosStrdup(pToken->z);
*nData = pToken->n;
}
return TSDB_CODE_SUCCESS;
}
static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema, int16_t timePrec, STagVal* val,
SMsgBuf* pMsgBuf) {
int64_t iv;
@ -477,7 +509,13 @@ static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema,
val->nData = pToken->n;
break;
}
case TSDB_DATA_TYPE_VARBINARY: {
code = parseVarbinary(pToken, &val->pData, &val->nData, pSchema->bytes);
if(code != TSDB_CODE_SUCCESS){
return generateSyntaxErrMsg(pMsgBuf, code, pSchema->name);
}
break;
}
case TSDB_DATA_TYPE_GEOMETRY: {
unsigned char* output = NULL;
size_t size = 0;
@ -659,6 +697,7 @@ static int32_t rewriteTagCondColumnImpl(STagVal* pVal, SNode** pNode) {
*(double*)&pValue->typeData = pValue->datum.d;
break;
case TSDB_DATA_TYPE_VARCHAR:
case TSDB_DATA_TYPE_VARBINARY:
case TSDB_DATA_TYPE_NCHAR:
pValue->datum.p = taosMemoryCalloc(1, pVal->nData + VARSTR_HEADER_SIZE);
if (NULL == pValue->datum.p) {
@ -688,7 +727,6 @@ static int32_t rewriteTagCondColumnImpl(STagVal* pVal, SNode** pNode) {
*(uint64_t*)&pValue->typeData = pValue->datum.i;
break;
case TSDB_DATA_TYPE_JSON:
case TSDB_DATA_TYPE_VARBINARY:
case TSDB_DATA_TYPE_DECIMAL:
case TSDB_DATA_TYPE_BLOB:
case TSDB_DATA_TYPE_MEDIUMBLOB:
@ -1364,6 +1402,13 @@ static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql,
pVal->value.nData = pToken->n;
break;
}
case TSDB_DATA_TYPE_VARBINARY: {
int32_t code = parseVarbinary(pToken, &pVal->value.pData, &pVal->value.nData, pSchema->bytes);
if(code != TSDB_CODE_SUCCESS){
return generateSyntaxErrMsg(&pCxt->msg, code, pSchema->name);
}
break;
}
case TSDB_DATA_TYPE_NCHAR: {
// if the converted output len is over than pColumnModel->bytes, return error: 'Argument list too long'
int32_t len = 0;

2
source/libs/parser/src/parInsertStmt.c
View File

@ -128,7 +128,7 @@ int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, const ch
} else {
STagVal val = {.cid = pTagSchema->colId, .type = pTagSchema->type};
// strcpy(val.colName, pTagSchema->name);
if (pTagSchema->type == TSDB_DATA_TYPE_BINARY ||
if (pTagSchema->type == TSDB_DATA_TYPE_BINARY || pTagSchema->type == TSDB_DATA_TYPE_VARBINARY ||
pTagSchema->type == TSDB_DATA_TYPE_GEOMETRY) {
val.pData = (uint8_t*)bind[c].buffer;
val.nData = colLen;

4
source/libs/parser/src/parInsertUtil.c
View File

@ -333,8 +333,8 @@ int32_t insGetTableDataCxt(SHashObj* pHash, void* id, int32_t idLen, STableMeta*
static void destroyColVal(void* p) {
SColVal* pVal = p;
if (TSDB_DATA_TYPE_NCHAR == pVal->type || TSDB_DATA_TYPE_GEOMETRY == pVal->type) {
taosMemoryFree(pVal->value.pData);
if (TSDB_DATA_TYPE_NCHAR == pVal->type || TSDB_DATA_TYPE_GEOMETRY == pVal->type || TSDB_DATA_TYPE_VARBINARY == pVal->type) {
taosMemoryFreeClear(pVal->value.pData);
}
}

3
source/libs/parser/src/parTokenizer.c
View File

@ -287,6 +287,7 @@ static SKeyword keywordTable[] = {
{"_WEND", TK_WEND},
{"_WSTART", TK_WSTART},
{"ALIVE", TK_ALIVE},
{"VARBINARY", TK_VARBINARY},
};
// clang-format on
@ -550,7 +551,7 @@ uint32_t tGetToken(const char* z, uint32_t* tokenId) {
return i;
} else if (next == 'x') { // hex number
*tokenId = TK_NK_HEX;
for (i = 2; isdigit(z[i]) || (z[i] >= 'a' && z[i] <= 'f') || (z[i] >= 'A' && z[i] <= 'F'); ++i) {
for (i = 2; isxdigit(z[i]) != 0; ++i) {
}
if (i == 2) {

43
source/libs/parser/src/parTranslater.c
View File

@ -1229,8 +1229,41 @@ static EDealRes translateNormalValue(STranslateContext* pCxt, SValueNode* pVal,
*(double*)&pVal->typeData = pVal->datum.d;
break;
}
case TSDB_DATA_TYPE_VARBINARY: {
if (pVal->node.resType.type != TSDB_DATA_TYPE_BINARY){
return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, pVal->literal);
}
void* data = NULL;
uint32_t size = 0;
bool isHexChar = isHex(pVal->literal, strlen(pVal->literal));
if(isHexChar){
if(!isValidateHex(pVal->literal, strlen(pVal->literal))){
return TSDB_CODE_PAR_INVALID_VARBINARY;
}
if(taosHex2Ascii(pVal->literal, strlen(pVal->literal), &data, &size) < 0){
return TSDB_CODE_OUT_OF_MEMORY;
}
}else{
size = pVal->node.resType.bytes;
data = pVal->literal;
}
if (size + VARSTR_HEADER_SIZE > targetDt.bytes) {
if(isHexChar) taosMemoryFree(data);
return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_VALUE_TOO_LONG, pVal->literal);
}
pVal->datum.p = taosMemoryCalloc(1, size + VARSTR_HEADER_SIZE);
if (NULL == pVal->datum.p) {
if(isHexChar) taosMemoryFree(data);
return generateDealNodeErrMsg(pCxt, TSDB_CODE_OUT_OF_MEMORY);
}
varDataSetLen(pVal->datum.p, size + VARSTR_HEADER_SIZE);
memcpy(varDataVal(pVal->datum.p), data, size);
if(isHexChar) taosMemoryFree(data);
break;
}
case TSDB_DATA_TYPE_VARCHAR:
case TSDB_DATA_TYPE_VARBINARY:
case TSDB_DATA_TYPE_GEOMETRY: {
if (strict && (pVal->node.resType.bytes > targetDt.bytes - VARSTR_HEADER_SIZE)) {
return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, pVal->literal);
@ -1304,7 +1337,7 @@ static EDealRes translateValueImpl(STranslateContext* pCxt, SValueNode* pVal, SD
}
static int32_t calcTypeBytes(SDataType dt) {
if (TSDB_DATA_TYPE_BINARY == dt.type || TSDB_DATA_TYPE_GEOMETRY == dt.type) {
if (TSDB_DATA_TYPE_BINARY == dt.type || TSDB_DATA_TYPE_VARBINARY == dt.type || TSDB_DATA_TYPE_GEOMETRY == dt.type) {
return dt.bytes + VARSTR_HEADER_SIZE;
} else if (TSDB_DATA_TYPE_NCHAR == dt.type) {
return dt.bytes * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE;
@ -4798,6 +4831,7 @@ static int32_t checkTableTagsSchema(STranslateContext* pCxt, SHashObj* pHash, SN
}
if (TSDB_CODE_SUCCESS == code) {
if ((TSDB_DATA_TYPE_VARCHAR == pTag->dataType.type && calcTypeBytes(pTag->dataType) > TSDB_MAX_TAGS_LEN) ||
(TSDB_DATA_TYPE_VARBINARY == pTag->dataType.type && calcTypeBytes(pTag->dataType) > TSDB_MAX_TAGS_LEN) ||
(TSDB_DATA_TYPE_NCHAR == pTag->dataType.type && calcTypeBytes(pTag->dataType) > TSDB_MAX_TAGS_LEN) ||
(TSDB_DATA_TYPE_GEOMETRY == pTag->dataType.type && calcTypeBytes(pTag->dataType) > TSDB_MAX_TAGS_LEN)) {
code = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN);
@ -4850,6 +4884,7 @@ static int32_t checkTableColsSchema(STranslateContext* pCxt, SHashObj* pHash, in
}
if (TSDB_CODE_SUCCESS == code) {
if ((TSDB_DATA_TYPE_VARCHAR == pCol->dataType.type && calcTypeBytes(pCol->dataType) > TSDB_MAX_BINARY_LEN) ||
(TSDB_DATA_TYPE_VARBINARY == pCol->dataType.type && calcTypeBytes(pCol->dataType) > TSDB_MAX_BINARY_LEN) ||
(TSDB_DATA_TYPE_NCHAR == pCol->dataType.type && calcTypeBytes(pCol->dataType) > TSDB_MAX_NCHAR_LEN) ||
(TSDB_DATA_TYPE_GEOMETRY == pCol->dataType.type && calcTypeBytes(pCol->dataType) > TSDB_MAX_GEOMETRY_LEN)) {
code = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN);
@ -5546,6 +5581,7 @@ static int32_t checkAlterSuperTableBySchema(STranslateContext* pCxt, SAlterTable
if (TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES == pStmt->alterType) {
if ((TSDB_DATA_TYPE_VARCHAR == pStmt->dataType.type && calcTypeBytes(pStmt->dataType) > TSDB_MAX_BINARY_LEN) ||
(TSDB_DATA_TYPE_VARBINARY == pStmt->dataType.type && calcTypeBytes(pStmt->dataType) > TSDB_MAX_BINARY_LEN) ||
(TSDB_DATA_TYPE_NCHAR == pStmt->dataType.type && calcTypeBytes(pStmt->dataType) > TSDB_MAX_NCHAR_LEN)) {
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN);
}
@ -5572,6 +5608,7 @@ static int32_t checkAlterSuperTableBySchema(STranslateContext* pCxt, SAlterTable
}
if ((TSDB_DATA_TYPE_VARCHAR == pStmt->dataType.type && calcTypeBytes(pStmt->dataType) > TSDB_MAX_BINARY_LEN) ||
(TSDB_DATA_TYPE_VARBINARY == pStmt->dataType.type && calcTypeBytes(pStmt->dataType) > TSDB_MAX_BINARY_LEN) ||
(TSDB_DATA_TYPE_NCHAR == pStmt->dataType.type && calcTypeBytes(pStmt->dataType) > TSDB_MAX_NCHAR_LEN)) {
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN);
}
@ -8857,6 +8894,7 @@ static int32_t buildAddColReq(STranslateContext* pCxt, SAlterTableStmt* pStmt, S
}
if ((TSDB_DATA_TYPE_VARCHAR == pStmt->dataType.type && calcTypeBytes(pStmt->dataType) > TSDB_MAX_BINARY_LEN) ||
(TSDB_DATA_TYPE_VARBINARY == pStmt->dataType.type && calcTypeBytes(pStmt->dataType) > TSDB_MAX_BINARY_LEN) ||
(TSDB_DATA_TYPE_NCHAR == pStmt->dataType.type && calcTypeBytes(pStmt->dataType) > TSDB_MAX_NCHAR_LEN)) {
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN);
}
@ -8914,6 +8952,7 @@ static int32_t buildUpdateColReq(STranslateContext* pCxt, SAlterTableStmt* pStmt
}
if ((TSDB_DATA_TYPE_VARCHAR == pStmt->dataType.type && calcTypeBytes(pStmt->dataType) > TSDB_MAX_BINARY_LEN) ||
(TSDB_DATA_TYPE_VARBINARY == pStmt->dataType.type && calcTypeBytes(pStmt->dataType) > TSDB_MAX_BINARY_LEN) ||
(TSDB_DATA_TYPE_NCHAR == pStmt->dataType.type && calcTypeBytes(pStmt->dataType) > TSDB_MAX_NCHAR_LEN)) {
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN);
}

2
source/libs/parser/src/parUtil.c
View File

@ -308,7 +308,7 @@ int32_t trimString(const char* src, int32_t len, char* dst, int32_t dlen) {
dst[j] = '\'';
} else if (src[k + 1] == '"') {
dst[j] = '"';
} else if (src[k + 1] == '%' || src[k + 1] == '_') {
} else if (src[k + 1] == '%' || src[k + 1] == '_' || src[k + 1] == 'x') {
dst[j++] = src[k];
dst[j] = src[k + 1];
} else {

19
source/libs/qcom/src/queryUtil.c
View File

@ -44,7 +44,7 @@ static bool doValidateSchema(SSchema* pSchema, int32_t numOfCols, int32_t maxLen
}
// 2. valid length for each type
if (pSchema[i].type == TSDB_DATA_TYPE_BINARY) {
if (pSchema[i].type == TSDB_DATA_TYPE_BINARY || pSchema[i].type == TSDB_DATA_TYPE_VARBINARY) {
if (pSchema[i].bytes > TSDB_MAX_BINARY_LEN) {
return false;
}
@ -300,6 +300,23 @@ int32_t dataConverToStr(char* str, int type, void* buf, int32_t bufSize, int32_t
n = sprintf(str, "%e", GET_DOUBLE_VAL(buf));
break;
case TSDB_DATA_TYPE_VARBINARY:{
if (bufSize < 0) {
// tscError("invalid buf size");
return TSDB_CODE_TSC_INVALID_VALUE;
}
void* data = NULL;
uint32_t size = 0;
if(taosAscii2Hex(buf, bufSize, &data, &size) < 0){
return TSDB_CODE_OUT_OF_MEMORY;
}
*str = '"';
memcpy(str + 1, data, size);
*(str + size + 1) = '"';
n = size + 2;
taosMemoryFree(data);
break;
}
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_GEOMETRY:
if (bufSize < 0) {

3
source/libs/qworker/inc/qwInt.h
View File

@ -133,6 +133,7 @@ typedef struct SQWTaskCtx {
bool queryContinue;
bool queryExecDone;
bool queryInQueue;
bool explainRsped;
int32_t rspCode;
int64_t affectedRows; // for insert ...select stmt
@ -169,6 +170,7 @@ typedef struct SQWMsgStat {
uint64_t rspProcessed;
uint64_t cancelProcessed;
uint64_t dropProcessed;
uint64_t notifyProcessed;
uint64_t hbProcessed;
uint64_t deleteProcessed;
} SQWMsgStat;
@ -406,6 +408,7 @@ int32_t qwAddTaskCtx(QW_FPARAMS_DEF);
void qwDbgSimulateRedirect(SQWMsg *qwMsg, SQWTaskCtx *ctx, bool *rsped);
void qwDbgSimulateSleep(void);
void qwDbgSimulateDead(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *rsped);
int32_t qwSendExplainResponse(QW_FPARAMS_DEF, SQWTaskCtx *ctx);
#ifdef __cplusplus
}

1
source/libs/qworker/inc/qwMsg.h
View File

@ -30,6 +30,7 @@ int32_t qwProcessCQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg);
int32_t qwProcessReady(QW_FPARAMS_DEF, SQWMsg *qwMsg);
int32_t qwProcessFetch(QW_FPARAMS_DEF, SQWMsg *qwMsg);
int32_t qwProcessDrop(QW_FPARAMS_DEF, SQWMsg *qwMsg);
int32_t qwProcessNotify(QW_FPARAMS_DEF, SQWMsg *qwMsg);
int32_t qwProcessHb(SQWorker *mgmt, SQWMsg *qwMsg, SSchedulerHbReq *req);
int32_t qwProcessDelete(QW_FPARAMS_DEF, SQWMsg *qwMsg, SDeleteRes *pRes);

35
source/libs/qworker/src/qwMsg.c
View File

@ -610,6 +610,41 @@ int32_t qWorkerProcessDropMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int6
return TSDB_CODE_SUCCESS;
}
int32_t qWorkerProcessNotifyMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int64_t ts) {
if (NULL == node || NULL == qWorkerMgmt || NULL == pMsg) {
return TSDB_CODE_QRY_INVALID_INPUT;
}
int32_t code = 0;
SQWorker *mgmt = (SQWorker *)qWorkerMgmt;
qwUpdateTimeInQueue(mgmt, ts, FETCH_QUEUE);
QW_STAT_INC(mgmt->stat.msgStat.notifyProcessed, 1);
STaskNotifyReq msg = {0};
if (tDeserializeSTaskNotifyReq(pMsg->pCont, pMsg->contLen, &msg) < 0) {
QW_ELOG("tDeserializeSTaskNotifyReq failed, contLen:%d", pMsg->contLen);
QW_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
}
uint64_t sId = msg.sId;
uint64_t qId = msg.queryId;
uint64_t tId = msg.taskId;
int64_t rId = msg.refId;
int32_t eId = msg.execId;
SQWMsg qwMsg = {.node = node, .msg = NULL, .msgLen = 0, .code = pMsg->code, .connInfo = pMsg->info, .msgType = msg.type};
QW_SCH_TASK_DLOG("processNotify start, node:%p, handle:%p", node, pMsg->info.handle);
QW_ERR_RET(qwProcessNotify(QW_FPARAMS(), &qwMsg));
QW_SCH_TASK_DLOG("processNotify end, node:%p", node);
return TSDB_CODE_SUCCESS;
}
int32_t qWorkerProcessHbMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int64_t ts) {
if (NULL == node || NULL == qWorkerMgmt || NULL == pMsg) {
return TSDB_CODE_QRY_INVALID_INPUT;

39
source/libs/qworker/src/qwUtil.c
View File

@ -316,6 +316,45 @@ void qwFreeTaskCtx(SQWTaskCtx *ctx) {
taosArrayDestroy(ctx->tbInfo);
}
static void freeExplainExecItem(void *param) {
SExplainExecInfo *pInfo = param;
taosMemoryFree(pInfo->verboseInfo);
}
int32_t qwSendExplainResponse(QW_FPARAMS_DEF, SQWTaskCtx *ctx) {
qTaskInfo_t taskHandle = ctx->taskHandle;
ctx->explainRsped = true;
SArray *execInfoList = taosArrayInit(4, sizeof(SExplainExecInfo));
QW_ERR_RET(qGetExplainExecInfo(taskHandle, execInfoList));
if (ctx->localExec) {
SExplainLocalRsp localRsp = {0};
localRsp.rsp.numOfPlans = taosArrayGetSize(execInfoList);
SExplainExecInfo *pExec = taosMemoryCalloc(localRsp.rsp.numOfPlans, sizeof(SExplainExecInfo));
memcpy(pExec, taosArrayGet(execInfoList, 0), localRsp.rsp.numOfPlans * sizeof(SExplainExecInfo));
localRsp.rsp.subplanInfo = pExec;
localRsp.qId = qId;
localRsp.tId = tId;
localRsp.rId = rId;
localRsp.eId = eId;
taosArrayPush(ctx->explainRes, &localRsp);
taosArrayDestroy(execInfoList);
} else {
SRpcHandleInfo connInfo = ctx->ctrlConnInfo;
connInfo.ahandle = NULL;
int32_t code = qwBuildAndSendExplainRsp(&connInfo, execInfoList);
taosArrayDestroyEx(execInfoList, freeExplainExecItem);
QW_ERR_RET(code);
}
return TSDB_CODE_SUCCESS;
}
int32_t qwDropTaskCtx(QW_FPARAMS_DEF) {
char id[sizeof(qId) + sizeof(tId) + sizeof(eId)] = {0};
QW_SET_QTID(id, qId, tId, eId);

80
source/libs/qworker/src/qworker.c
View File

@ -90,11 +90,6 @@ int32_t qwProcessHbLinkBroken(SQWorker *mgmt, SQWMsg *qwMsg, SSchedulerHbReq *re
QW_RET(TSDB_CODE_SUCCESS);
}
static void freeItem(void *param) {
SExplainExecInfo *pInfo = param;
taosMemoryFree(pInfo->verboseInfo);
}
int32_t qwHandleTaskComplete(QW_FPARAMS_DEF, SQWTaskCtx *ctx) {
qTaskInfo_t taskHandle = ctx->taskHandle;
@ -102,29 +97,8 @@ int32_t qwHandleTaskComplete(QW_FPARAMS_DEF, SQWTaskCtx *ctx) {
ctx->queryExecDone = true;
if (TASK_TYPE_TEMP == ctx->taskType && taskHandle) {
if (ctx->explain) {
SArray *execInfoList = taosArrayInit(4, sizeof(SExplainExecInfo));
QW_ERR_RET(qGetExplainExecInfo(taskHandle, execInfoList));
if (ctx->localExec) {
SExplainLocalRsp localRsp = {0};
localRsp.rsp.numOfPlans = taosArrayGetSize(execInfoList);
SExplainExecInfo *pExec = taosMemoryCalloc(localRsp.rsp.numOfPlans, sizeof(SExplainExecInfo));
memcpy(pExec, taosArrayGet(execInfoList, 0), localRsp.rsp.numOfPlans * sizeof(SExplainExecInfo));
localRsp.rsp.subplanInfo = pExec;
localRsp.qId = qId;
localRsp.tId = tId;
localRsp.rId = rId;
localRsp.eId = eId;
taosArrayPush(ctx->explainRes, &localRsp);
taosArrayDestroy(execInfoList);
} else {
SRpcHandleInfo connInfo = ctx->ctrlConnInfo;
connInfo.ahandle = NULL;
int32_t code = qwBuildAndSendExplainRsp(&connInfo, execInfoList);
taosArrayDestroyEx(execInfoList, freeItem);
QW_ERR_RET(code);
}
if (ctx->explain && !ctx->explainRsped) {
QW_ERR_RET(qwSendExplainResponse(QW_FPARAMS(), ctx));
}
if (!ctx->needFetch) {
@ -1030,6 +1004,55 @@ _return:
QW_RET(TSDB_CODE_SUCCESS);
}
int32_t qwProcessNotify(QW_FPARAMS_DEF, SQWMsg *qwMsg) {
int32_t code = 0;
SQWTaskCtx *ctx = NULL;
bool locked = false;
QW_ERR_JRET(qwAcquireTaskCtx(QW_FPARAMS(), &ctx));
QW_LOCK(QW_WRITE, &ctx->lock);
locked = true;
if (QW_QUERY_RUNNING(ctx)) {
QW_ERR_JRET(qwKillTaskHandle(ctx, TSDB_CODE_TSC_QUERY_CANCELLED));
qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_SUCC, ctx->dynamicTask);
}
switch (qwMsg->msgType) {
case TASK_NOTIFY_FINISHED:
if (ctx->explain && !ctx->explainRsped) {
QW_ERR_RET(qwSendExplainResponse(QW_FPARAMS(), ctx));
}
break;
default:
QW_ELOG("Invalid task notify type %d", qwMsg->msgType);
QW_ERR_JRET(TSDB_CODE_INVALID_MSG);
break;
}
_return:
if (code) {
if (ctx) {
QW_UPDATE_RSP_CODE(ctx, code);
qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_FAIL, ctx->dynamicTask);
}
}
if (locked) {
QW_UNLOCK(QW_WRITE, &ctx->lock);
}
if (ctx) {
qwReleaseTaskCtx(mgmt, ctx);
}
QW_RET(TSDB_CODE_SUCCESS);
}
int32_t qwProcessHb(SQWorker *mgmt, SQWMsg *qwMsg, SSchedulerHbReq *req) {
int32_t code = 0;
SSchedulerHbRsp rsp = {0};
@ -1354,6 +1377,7 @@ int32_t qWorkerGetStat(SReadHandle *handle, void *qWorkerMgmt, SQWorkerStat *pSt
pStat->cqueryProcessed = QW_STAT_GET(mgmt->stat.msgStat.cqueryProcessed);
pStat->fetchProcessed = QW_STAT_GET(mgmt->stat.msgStat.fetchProcessed);
pStat->dropProcessed = QW_STAT_GET(mgmt->stat.msgStat.dropProcessed);
pStat->notifyProcessed = QW_STAT_GET(mgmt->stat.msgStat.notifyProcessed);
pStat->hbProcessed = QW_STAT_GET(mgmt->stat.msgStat.hbProcessed);
pStat->deleteProcessed = QW_STAT_GET(mgmt->stat.msgStat.deleteProcessed);

3
source/libs/qworker/test/qworkerTests.cpp
View File

@ -818,6 +818,9 @@ void *fetchQueueThread(void *param) {
case TDMT_SCH_DROP_TASK:
qWorkerProcessDropMsg(mockPointer, mgmt, fetchRpc, 0);
break;
case TDMT_SCH_TASK_NOTIFY:
qWorkerProcessNotifyMsg(mockPointer, mgmt, fetchRpc, 0);
break;
default:
printf("unknown msg type:%d in fetch queue", fetchRpc->msgType);
assert(0);

2
source/libs/scalar/inc/filterInt.h
View File

@ -272,7 +272,7 @@ struct SFilterInfo {
};
#define FILTER_NO_MERGE_DATA_TYPE(t) \
((t) == TSDB_DATA_TYPE_BINARY || (t) == TSDB_DATA_TYPE_NCHAR || (t) == TSDB_DATA_TYPE_JSON || \
((t) == TSDB_DATA_TYPE_BINARY || (t) == TSDB_DATA_TYPE_VARBINARY || (t) == TSDB_DATA_TYPE_NCHAR || (t) == TSDB_DATA_TYPE_JSON || \
(t) == TSDB_DATA_TYPE_GEOMETRY)
#define FILTER_NO_MERGE_OPTR(o) ((o) == OP_TYPE_IS_NULL || (o) == OP_TYPE_IS_NOT_NULL || (o) == FILTER_DUMMY_EMPTY_OPTR)

18
source/libs/scalar/src/filter.c
View File

@ -180,7 +180,8 @@ __compar_fn_t gUint64UsignCompare[] = {compareUint64Uint8, compareUint64Uint16,
int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) {
int8_t comparFn = 0;
if (optr == OP_TYPE_IN && (type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR && type != TSDB_DATA_TYPE_GEOMETRY)) {
if (optr == OP_TYPE_IN && (type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_VARBINARY &&
type != TSDB_DATA_TYPE_NCHAR && type != TSDB_DATA_TYPE_GEOMETRY)) {
switch (type) {
case TSDB_DATA_TYPE_BOOL:
case TSDB_DATA_TYPE_TINYINT:
@ -206,7 +207,7 @@ int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) {
}
}
if (optr == OP_TYPE_NOT_IN && (type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR && type != TSDB_DATA_TYPE_GEOMETRY)) {
if (optr == OP_TYPE_NOT_IN && (type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_VARBINARY && type != TSDB_DATA_TYPE_NCHAR && type != TSDB_DATA_TYPE_GEOMETRY)) {
switch (type) {
case TSDB_DATA_TYPE_BOOL:
case TSDB_DATA_TYPE_TINYINT:
@ -257,6 +258,16 @@ int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) {
case TSDB_DATA_TYPE_DOUBLE:
comparFn = 5;
break;
case TSDB_DATA_TYPE_VARBINARY:{
if (optr == OP_TYPE_IN) {
comparFn = 8;
} else if (optr == OP_TYPE_NOT_IN) {
comparFn = 25;
} else { /* normal relational comparFn */
comparFn = 30;
}
break;
}
case TSDB_DATA_TYPE_BINARY: {
if (optr == OP_TYPE_MATCH) {
comparFn = 19;
@ -466,7 +477,7 @@ static FORCE_INLINE SFilterRangeNode *filterNewRange(SFilterRangeCtx *ctx, SFilt
void *filterInitRangeCtx(int32_t type, int32_t options) {
if (type > TSDB_DATA_TYPE_UBIGINT || type < TSDB_DATA_TYPE_BOOL ||
type == TSDB_DATA_TYPE_BINARY ||
type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_VARBINARY ||
type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_GEOMETRY) {
qError("not supported range type:%d", type);
return NULL;
@ -1585,6 +1596,7 @@ int32_t fltConverToStr(char *str, int type, void *buf, int32_t bufSize, int32_t
break;
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_VARBINARY:
case TSDB_DATA_TYPE_NCHAR:
case TSDB_DATA_TYPE_GEOMETRY:
if (bufSize < 0) {

9
source/libs/scalar/src/scalar.c
View File

@ -1594,6 +1594,8 @@ static int32_t sclGetMathOperatorResType(SOperatorNode *pOp) {
SDataType rdt = ((SExprNode *)(pOp->pRight))->resType;
if ((TSDB_DATA_TYPE_TIMESTAMP == ldt.type && TSDB_DATA_TYPE_TIMESTAMP == rdt.type) ||
TSDB_DATA_TYPE_VARBINARY == ldt.type ||
TSDB_DATA_TYPE_VARBINARY == rdt.type ||
(TSDB_DATA_TYPE_TIMESTAMP == ldt.type && (IS_VAR_DATA_TYPE(rdt.type) || IS_FLOAT_TYPE(rdt.type))) ||
(TSDB_DATA_TYPE_TIMESTAMP == rdt.type && (IS_VAR_DATA_TYPE(ldt.type) || IS_FLOAT_TYPE(ldt.type)))) {
return TSDB_CODE_TSC_INVALID_OPERATION;
@ -1629,7 +1631,7 @@ static int32_t sclGetCompOperatorResType(SOperatorNode *pOp) {
return TSDB_CODE_TSC_INVALID_OPERATION;
}
SDataType rdt = ((SExprNode *)(pOp->pRight))->resType;
if (!IS_VAR_DATA_TYPE(ldt.type) || QUERY_NODE_VALUE != nodeType(pOp->pRight) ||
if (ldt.type == TSDB_DATA_TYPE_VARBINARY || !IS_VAR_DATA_TYPE(ldt.type) || QUERY_NODE_VALUE != nodeType(pOp->pRight) ||
(!IS_STR_DATA_TYPE(rdt.type) && (rdt.type != TSDB_DATA_TYPE_NULL))) {
return TSDB_CODE_TSC_INVALID_OPERATION;
}
@ -1660,6 +1662,11 @@ static int32_t sclGetJsonOperatorResType(SOperatorNode *pOp) {
}
static int32_t sclGetBitwiseOperatorResType(SOperatorNode *pOp) {
SDataType ldt = ((SExprNode *)(pOp->pLeft))->resType;
SDataType rdt = ((SExprNode *)(pOp->pRight))->resType;
if(TSDB_DATA_TYPE_VARBINARY == ldt.type || TSDB_DATA_TYPE_VARBINARY == rdt.type){
return TSDB_CODE_TSC_INVALID_OPERATION;
}
pOp->node.resType.type = TSDB_DATA_TYPE_BIGINT;
pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes;
return TSDB_CODE_SUCCESS;

146
source/libs/scalar/src/sclvector.c
View File

@ -190,55 +190,6 @@ _getBigintValue_fn_t getVectorBigintValueFn(int32_t srcType) {
return p;
}
typedef void *(*_getValueAddr_fn_t)(void *src, int32_t index);
void *getVectorValueAddr_TINYINT(void *src, int32_t index) { return (void *)((int8_t *)src + index); }
void *getVectorValueAddr_UTINYINT(void *src, int32_t index) { return (void *)((uint8_t *)src + index); }
void *getVectorValueAddr_SMALLINT(void *src, int32_t index) { return (void *)((int16_t *)src + index); }
void *getVectorValueAddr_USMALLINT(void *src, int32_t index) { return (void *)((uint16_t *)src + index); }
void *getVectorValueAddr_INT(void *src, int32_t index) { return (void *)((int32_t *)src + index); }
void *getVectorValueAddr_UINT(void *src, int32_t index) { return (void *)((uint32_t *)src + index); }
void *getVectorValueAddr_BIGINT(void *src, int32_t index) { return (void *)((int64_t *)src + index); }
void *getVectorValueAddr_UBIGINT(void *src, int32_t index) { return (void *)((uint64_t *)src + index); }
void *getVectorValueAddr_FLOAT(void *src, int32_t index) { return (void *)((float *)src + index); }
void *getVectorValueAddr_DOUBLE(void *src, int32_t index) { return (void *)((double *)src + index); }
void *getVectorValueAddr_default(void *src, int32_t index) { return src; }
void *getVectorValueAddr_VAR(void *src, int32_t index) { return colDataGetData((SColumnInfoData *)src, index); }
_getValueAddr_fn_t getVectorValueAddrFn(int32_t srcType) {
_getValueAddr_fn_t p = NULL;
if (srcType == TSDB_DATA_TYPE_TINYINT) {
p = getVectorValueAddr_TINYINT;
} else if (srcType == TSDB_DATA_TYPE_UTINYINT) {
p = getVectorValueAddr_UTINYINT;
} else if (srcType == TSDB_DATA_TYPE_SMALLINT) {
p = getVectorValueAddr_SMALLINT;
} else if (srcType == TSDB_DATA_TYPE_USMALLINT) {
p = getVectorValueAddr_USMALLINT;
} else if (srcType == TSDB_DATA_TYPE_INT) {
p = getVectorValueAddr_INT;
} else if (srcType == TSDB_DATA_TYPE_UINT) {
p = getVectorValueAddr_UINT;
} else if (srcType == TSDB_DATA_TYPE_BIGINT) {
p = getVectorValueAddr_BIGINT;
} else if (srcType == TSDB_DATA_TYPE_UBIGINT) {
p = getVectorValueAddr_UBIGINT;
} else if (srcType == TSDB_DATA_TYPE_FLOAT) {
p = getVectorValueAddr_FLOAT;
} else if (srcType == TSDB_DATA_TYPE_DOUBLE) {
p = getVectorValueAddr_DOUBLE;
} else if (srcType == TSDB_DATA_TYPE_BINARY) {
p = getVectorValueAddr_VAR;
} else if (srcType == TSDB_DATA_TYPE_NCHAR) {
p = getVectorValueAddr_VAR;
}else if(srcType == TSDB_DATA_TYPE_GEOMETRY) {
p = getVectorValueAddr_VAR;
} else {
p = getVectorValueAddr_default;
}
return p;
}
static FORCE_INLINE void varToTimestamp(char *buf, SScalarParam *pOut, int32_t rowIndex, int32_t *overflow) {
terrno = TSDB_CODE_SUCCESS;
@ -358,6 +309,47 @@ static FORCE_INLINE void varToBool(char *buf, SScalarParam *pOut, int32_t rowInd
}
// todo remove this malloc
static FORCE_INLINE void varToVarbinary(char *buf, SScalarParam *pOut, int32_t rowIndex, int32_t *overflow) {
terrno = TSDB_CODE_SUCCESS;
if(isHex(varDataVal(buf), varDataLen(buf))){
if(!isValidateHex(varDataVal(buf), varDataLen(buf))){
terrno = TSDB_CODE_PAR_INVALID_VARBINARY;
return;
}
void* data = NULL;
uint32_t size = 0;
if(taosHex2Ascii(varDataVal(buf), varDataLen(buf), &data, &size) < 0){
terrno = TSDB_CODE_OUT_OF_MEMORY;
return;
}
int32_t inputLen = size + VARSTR_HEADER_SIZE;
char *t = taosMemoryCalloc(1, inputLen);
if (t == NULL) {
sclError("Out of memory");
terrno = TSDB_CODE_OUT_OF_MEMORY;
return;
}
varDataSetLen(t, size);
memcpy(varDataVal(t), data, size);
colDataSetVal(pOut->columnData, rowIndex, t, false);
taosMemoryFree(t);
taosMemoryFree(data);
}else{
int32_t inputLen = varDataTLen(buf);
char *t = taosMemoryCalloc(1, inputLen);
if (t == NULL) {
sclError("Out of memory");
terrno = TSDB_CODE_OUT_OF_MEMORY;
return;
}
memcpy(t, buf, inputLen);
colDataSetVal(pOut->columnData, rowIndex, t, false);
taosMemoryFree(t);
}
}
static FORCE_INLINE void varToNchar(char *buf, SScalarParam *pOut, int32_t rowIndex, int32_t *overflow) {
terrno = TSDB_CODE_SUCCESS;
@ -447,18 +439,21 @@ int32_t vectorConvertFromVarData(SSclVectorConvCtx *pCtx, int32_t *overflow) {
func = varToUnsigned;
} else if (IS_FLOAT_TYPE(pCtx->outType)) {
func = varToFloat;
} else if (pCtx->outType == TSDB_DATA_TYPE_VARCHAR &&
} else if ((pCtx->outType == TSDB_DATA_TYPE_VARCHAR || pCtx->outType == TSDB_DATA_TYPE_VARBINARY) &&
pCtx->inType == TSDB_DATA_TYPE_NCHAR) { // nchar -> binary
func = ncharToVar;
vton = true;
} else if (pCtx->outType == TSDB_DATA_TYPE_NCHAR &&
pCtx->inType == TSDB_DATA_TYPE_VARCHAR) { // binary -> nchar
(pCtx->inType == TSDB_DATA_TYPE_VARCHAR || pCtx->inType == TSDB_DATA_TYPE_VARBINARY)) { // binary -> nchar
func = varToNchar;
vton = true;
} else if (TSDB_DATA_TYPE_TIMESTAMP == pCtx->outType) {
func = varToTimestamp;
} else if (TSDB_DATA_TYPE_GEOMETRY == pCtx->outType) {
func = varToGeometry;
} else if (TSDB_DATA_TYPE_VARBINARY == pCtx->outType) {
func = varToVarbinary;
vton = true;
} else {
sclError("invalid convert outType:%d, inType:%d", pCtx->outType, pCtx->inType);
terrno = TSDB_CODE_APP_ERROR;
@ -597,7 +592,8 @@ bool convertJsonValue(__compar_fn_t *fp, int32_t optr, int8_t typeLeft, int8_t t
}
if (optr == OP_TYPE_LIKE || optr == OP_TYPE_NOT_LIKE || optr == OP_TYPE_MATCH || optr == OP_TYPE_NMATCH) {
if (typeLeft != TSDB_DATA_TYPE_NCHAR && typeLeft != TSDB_DATA_TYPE_BINARY && typeLeft != TSDB_DATA_TYPE_GEOMETRY) {
if (typeLeft != TSDB_DATA_TYPE_NCHAR && typeLeft != TSDB_DATA_TYPE_BINARY &&
typeLeft != TSDB_DATA_TYPE_GEOMETRY && typeLeft != TSDB_DATA_TYPE_VARBINARY) {
return false;
}
}
@ -642,7 +638,8 @@ bool convertJsonValue(__compar_fn_t *fp, int32_t optr, int8_t typeLeft, int8_t t
convertNumberToNumber(*pRightData, pRightOut, typeRight, type);
*pRightData = pRightOut;
}
} else if (type == TSDB_DATA_TYPE_BINARY || typeLeft == TSDB_DATA_TYPE_GEOMETRY) {
} else if (type == TSDB_DATA_TYPE_BINARY ||
type == TSDB_DATA_TYPE_GEOMETRY) {
if (typeLeft == TSDB_DATA_TYPE_NCHAR) {
*pLeftData = ncharTobinary(*pLeftData);
*freeLeft = true;
@ -932,6 +929,7 @@ int32_t vectorConvertSingleColImpl(const SScalarParam *pIn, SScalarParam *pOut,
break;
}
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_VARBINARY:
case TSDB_DATA_TYPE_NCHAR:
case TSDB_DATA_TYPE_GEOMETRY: {
return vectorConvertToVarData(&cCtx);
@ -947,26 +945,26 @@ int32_t vectorConvertSingleColImpl(const SScalarParam *pIn, SScalarParam *pOut,
int8_t gConvertTypes[TSDB_DATA_TYPE_MAX][TSDB_DATA_TYPE_MAX] = {
/* NULL BOOL TINY SMAL INT BIG FLOA DOUB VARC TIME NCHA UTIN USMA UINT UBIG JSON VARB DECI BLOB MEDB GEOM*/
/*NULL*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
/*BOOL*/ 0, 0, 2, 3, 4, 5, 6, 7, 5, 9, 7, 11, 12, 13, 14, 0, 7, 0, 0, 0, -1,
/*TINY*/ 0, 0, 0, 3, 4, 5, 6, 7, 5, 9, 7, 3, 4, 5, 7, 0, 7, 0, 0, 0, -1,
/*SMAL*/ 0, 0, 0, 0, 4, 5, 6, 7, 5, 9, 7, 3, 4, 5, 7, 0, 7, 0, 0, 0, -1,
/*INT */ 0, 0, 0, 0, 0, 5, 6, 7, 5, 9, 7, 4, 4, 5, 7, 0, 7, 0, 0, 0, -1,
/*BIGI*/ 0, 0, 0, 0, 0, 0, 6, 7, 5, 9, 7, 5, 5, 5, 7, 0, 7, 0, 0, 0, -1,
/*FLOA*/ 0, 0, 0, 0, 0, 0, 0, 7, 7, 6, 7, 6, 6, 6, 6, 0, 7, 0, 0, 0, -1,
/*DOUB*/ 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 7, 7, 7, 7, 7, 0, 7, 0, 0, 0, -1,
/*VARC*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 8, 7, 7, 7, 7, 0, 0, 0, 0, 0, 20,
/*TIME*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 7, 0, 7, 0, 0, 0, -1,
/*NCHA*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 7, 7, 0, 0, 0, 0, 0, -1,
/*UTIN*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 13, 14, 0, 7, 0, 0, 0, -1,
/*USMA*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 14, 0, 7, 0, 0, 0, -1,
/*UINT*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 7, 0, 0, 0, -1,
/*UBIG*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, -1,
/*JSON*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1,
/*VARB*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1,
/*DECI*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1,
/*BLOB*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1,
/*MEDB*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1,
/*GEOM*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
/*BOOL*/ 0, 0, 2, 3, 4, 5, 6, 7, 5, 9, 7, 11, 12, 13, 14, 0, -1, 0, 0, 0, -1,
/*TINY*/ 0, 0, 0, 3, 4, 5, 6, 7, 5, 9, 7, 3, 4, 5, 7, 0, -1, 0, 0, 0, -1,
/*SMAL*/ 0, 0, 0, 0, 4, 5, 6, 7, 5, 9, 7, 3, 4, 5, 7, 0, -1, 0, 0, 0, -1,
/*INT */ 0, 0, 0, 0, 0, 5, 6, 7, 5, 9, 7, 4, 4, 5, 7, 0, -1, 0, 0, 0, -1,
/*BIGI*/ 0, 0, 0, 0, 0, 0, 6, 7, 5, 9, 7, 5, 5, 5, 7, 0, -1, 0, 0, 0, -1,
/*FLOA*/ 0, 0, 0, 0, 0, 0, 0, 7, 7, 6, 7, 6, 6, 6, 6, 0, -1, 0, 0, 0, -1,
/*DOUB*/ 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 7, 7, 7, 7, 7, 0, -1, 0, 0, 0, -1,
/*VARC*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 8, 7, 7, 7, 7, 0, 16, 0, 0, 0, 20,
/*TIME*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 7, 0, -1, 0, 0, 0, -1,
/*NCHA*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 7, 7, 0, 16, 0, 0, 0, -1,
/*UTIN*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 13, 14, 0, -1, 0, 0, 0, -1,
/*USMA*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 14, 0, -1, 0, 0, 0, -1,
/*UINT*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, -1, 0, 0, 0, -1,
/*UBIG*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, 0, 0, 0, -1,
/*JSON*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, 0, 0, 0, -1,
/*VARB*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, -1,-1, -1,
/*DECI*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, 0, 0, 0, -1,
/*BLOB*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, 0, 0, 0, -1,
/*MEDB*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, 0, 0, 0, -1,
/*GEOM*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, 0, 0, 0, 0};
int32_t vectorGetConvertType(int32_t type1, int32_t type2) {
if (type1 == type2) {
@ -1122,7 +1120,7 @@ static SColumnInfoData *vectorConvertVarToDouble(SScalarParam *pInput, int32_t *
SScalarParam output = {0};
SColumnInfoData *pCol = pInput->columnData;
if (IS_VAR_DATA_TYPE(pCol->info.type) && pCol->info.type != TSDB_DATA_TYPE_JSON) {
if (IS_VAR_DATA_TYPE(pCol->info.type) && pCol->info.type != TSDB_DATA_TYPE_JSON && pCol->info.type != TSDB_DATA_TYPE_VARBINARY) {
int32_t code = vectorConvertSingleCol(pInput, &output, TSDB_DATA_TYPE_DOUBLE, -1, -1);
if (code != TSDB_CODE_SUCCESS) {
terrno = code;

46
source/libs/scalar/test/filter/filterTests.cpp
View File

@ -1415,52 +1415,6 @@ void doCompareWithValueRange_OnlyLeftType(__compar_fn_t fp, int32_t rType) {
}
}
void doCompare(const std::vector<int32_t> &lTypes, const std::vector<int32_t> &rTypes, int32_t oper) {
for (int i = 0; i < lTypes.size(); ++i) {
for (int j = 0; j < rTypes.size(); ++j) {
auto fp = filterGetCompFuncEx(lTypes[i], rTypes[j], oper);
switch (lTypes[i]) {
case TSDB_DATA_TYPE_TINYINT:
doCompareWithValueRange_OnlyLeftType<int8_t>(fp, rTypes[j]);
break;
case TSDB_DATA_TYPE_SMALLINT:
doCompareWithValueRange_OnlyLeftType<int16_t>(fp, rTypes[j]);
break;
case TSDB_DATA_TYPE_INT:
doCompareWithValueRange_OnlyLeftType<int32_t>(fp, rTypes[j]);
break;
case TSDB_DATA_TYPE_BIGINT:
doCompareWithValueRange_OnlyLeftType<int64_t>(fp, rTypes[j]);
break;
case TSDB_DATA_TYPE_UTINYINT:
doCompareWithValueRange_OnlyLeftType<uint8_t>(fp, rTypes[j]);
break;
case TSDB_DATA_TYPE_USMALLINT:
doCompareWithValueRange_OnlyLeftType<uint16_t>(fp, rTypes[j]);
break;
case TSDB_DATA_TYPE_UINT:
doCompareWithValueRange_OnlyLeftType<uint32_t>(fp, rTypes[j]);
break;
case TSDB_DATA_TYPE_UBIGINT:
doCompareWithValueRange_OnlyLeftType<uint64_t>(fp, rTypes[j]);
break;
default:
FAIL();
}
}
}
}
TEST(dataCompareTest, signed_and_unsigned_int) {
std::vector<int32_t> lType = {TSDB_DATA_TYPE_TINYINT, TSDB_DATA_TYPE_SMALLINT, TSDB_DATA_TYPE_INT,
TSDB_DATA_TYPE_BIGINT};
std::vector<int32_t> rType = {TSDB_DATA_TYPE_UTINYINT, TSDB_DATA_TYPE_USMALLINT, TSDB_DATA_TYPE_UINT,
TSDB_DATA_TYPE_UBIGINT};
doCompare(lType, rType, OP_TYPE_GREATER_THAN);
doCompare(rType, lType, OP_TYPE_GREATER_THAN);
}
int main(int argc, char **argv) {
taosSeedRand(taosGetTimestampSec());
testing::InitGoogleTest(&argc, argv);

4
source/libs/scheduler/inc/schInt.h
View File

@ -531,7 +531,7 @@ void schDeregisterTaskHb(SSchJob *pJob, SSchTask *pTask);
void schCleanClusterHb(void *pTrans);
int32_t schLaunchTask(SSchJob *job, SSchTask *task);
int32_t schDelayLaunchTask(SSchJob *pJob, SSchTask *pTask);
int32_t schBuildAndSendMsg(SSchJob *job, SSchTask *task, SQueryNodeAddr *addr, int32_t msgType);
int32_t schBuildAndSendMsg(SSchJob *job, SSchTask *task, SQueryNodeAddr *addr, int32_t msgType, void* param);
SSchJob *schAcquireJob(int64_t refId);
int32_t schReleaseJob(int64_t refId);
void schFreeFlowCtrl(SSchJob *pJob);
@ -598,6 +598,7 @@ int32_t schProcessOnJobFailure(SSchJob *pJob, int32_t errCode);
int32_t schProcessOnJobPartialSuccess(SSchJob *pJob);
void schFreeTask(SSchJob *pJob, SSchTask *pTask);
void schDropTaskInHashList(SSchJob *pJob, SHashObj *list);
int32_t schNotifyTaskInHashList(SSchJob *pJob, SHashObj *list, ETaskNotifyType type, SSchTask *pTask);
int32_t schLaunchLevelTasks(SSchJob *pJob, SSchLevel *level);
int32_t schGetTaskFromList(SHashObj *pTaskList, uint64_t taskId, SSchTask **pTask);
int32_t schInitTask(SSchJob *pJob, SSchTask *pTask, SSubplan *pPlan, SSchLevel *pLevel);
@ -612,6 +613,7 @@ int32_t schHandleJobRetry(SSchJob *pJob, SSchTask *pTask, SDataBuf *pMsg, int32
int32_t schChkResetJobRetry(SSchJob *pJob, int32_t rspCode);
void schResetTaskForRetry(SSchJob *pJob, SSchTask *pTask);
int32_t schChkUpdateRedirectCtx(SSchJob *pJob, SSchTask *pTask, SEpSet *pEpSet, int32_t rspCode);
int32_t schNotifyJobAllTasks(SSchJob *pJob, SSchTask *pTask, ETaskNotifyType type);
extern SSchDebug gSCHDebug;

4
source/libs/scheduler/src/schJob.c
View File

@ -638,6 +638,10 @@ void schDropJobAllTasks(SSchJob *pJob) {
// schDropTaskInHashList(pJob, pJob->failTasks);
}
int32_t schNotifyJobAllTasks(SSchJob *pJob, SSchTask *pTask, ETaskNotifyType type) {
SCH_RET(schNotifyTaskInHashList(pJob, pJob->execTasks, type, pTask));
}
void schFreeJobImpl(void *job) {
if (NULL == job) {
return;

50
source/libs/scheduler/src/schRemote.c
View File

@ -88,6 +88,8 @@ int32_t schProcessFetchRsp(SSchJob *pJob, SSchTask *pTask, char *msg, int32_t rs
SCH_ERR_JRET(qExecExplainEnd(pJob->explainCtx, &pRsp));
if (pRsp) {
SCH_ERR_JRET(schProcessOnExplainDone(pJob, pTask, pRsp));
} else {
SCH_ERR_JRET(schNotifyJobAllTasks(pJob, pTask, TASK_NOTIFY_FINISHED));
}
taosMemoryFreeClear(msg);
@ -481,6 +483,18 @@ int32_t schHandleDropCallback(void *param, SDataBuf *pMsg, int32_t code) {
return TSDB_CODE_SUCCESS;
}
int32_t schHandleNotifyCallback(void *param, SDataBuf *pMsg, int32_t code) {
SSchTaskCallbackParam *pParam = (SSchTaskCallbackParam *)param;
qDebug("QID:0x%" PRIx64 ",TID:0x%" PRIx64 " task notify rsp received, code:0x%x", pParam->queryId, pParam->taskId,
code);
if (pMsg) {
taosMemoryFree(pMsg->pData);
taosMemoryFree(pMsg->pEpSet);
}
return TSDB_CODE_SUCCESS;
}
int32_t schHandleLinkBrokenCallback(void *param, SDataBuf *pMsg, int32_t code) {
SSchCallbackParamHeader *head = (SSchCallbackParamHeader *)param;
rpcReleaseHandle(pMsg->handle, TAOS_CONN_CLIENT);
@ -646,6 +660,9 @@ int32_t schGetCallbackFp(int32_t msgType, __async_send_cb_fn_t *fp) {
case TDMT_SCH_DROP_TASK:
*fp = schHandleDropCallback;
break;
case TDMT_SCH_TASK_NOTIFY:
*fp = schHandleNotifyCallback;
break;
case TDMT_SCH_QUERY_HEARTBEAT:
*fp = schHandleHbCallback;
break;
@ -1027,7 +1044,7 @@ _return:
SCH_RET(code);
}
int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, int32_t msgType) {
int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, int32_t msgType, void* param) {
int32_t msgSize = 0;
void *msg = NULL;
int32_t code = 0;
@ -1205,6 +1222,37 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr,
persistHandle = true;
break;
}
case TDMT_SCH_TASK_NOTIFY: {
ETaskNotifyType* pType = param;
STaskNotifyReq qMsg;
qMsg.header.vgId = addr->nodeId;
qMsg.header.contLen = 0;
qMsg.sId = schMgmt.sId;
qMsg.queryId = pJob->queryId;
qMsg.taskId = pTask->taskId;
qMsg.refId = pJob->refId;
qMsg.execId = pTask->execId;
qMsg.type = *pType;
msgSize = tSerializeSTaskNotifyReq(NULL, 0, &qMsg);
if (msgSize < 0) {
SCH_TASK_ELOG("tSerializeSTaskNotifyReq get size, msgSize:%d", msgSize);
SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
msg = taosMemoryCalloc(1, msgSize);
if (NULL == msg) {
SCH_TASK_ELOG("calloc %d failed", msgSize);
SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
if (tSerializeSTaskNotifyReq(msg, msgSize, &qMsg) < 0) {
SCH_TASK_ELOG("tSerializeSTaskNotifyReq failed, msgSize:%d", msgSize);
taosMemoryFree(msg);
SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
break;
}
default:
SCH_TASK_ELOG("unknown msg type to send, msgType:%d", msgType);
SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR);

58
source/libs/scheduler/src/schTask.c
View File

@ -862,7 +862,7 @@ void schDropTaskOnExecNode(SSchJob *pJob, SSchTask *pTask) {
while (nodeInfo) {
if (nodeInfo->handle) {
SCH_SET_TASK_HANDLE(pTask, nodeInfo->handle);
schBuildAndSendMsg(pJob, pTask, &nodeInfo->addr, TDMT_SCH_DROP_TASK);
schBuildAndSendMsg(pJob, pTask, &nodeInfo->addr, TDMT_SCH_DROP_TASK, NULL);
SCH_TASK_DLOG("start to drop task's %dth execNode", i);
} else {
SCH_TASK_DLOG("no need to drop task %dth execNode", i);
@ -875,6 +875,33 @@ void schDropTaskOnExecNode(SSchJob *pJob, SSchTask *pTask) {
SCH_TASK_DLOG("task has been dropped on %d exec nodes", size);
}
int32_t schNotifyTaskOnExecNode(SSchJob *pJob, SSchTask *pTask, ETaskNotifyType type) {
int32_t size = (int32_t)taosHashGetSize(pTask->execNodes);
if (size <= 0) {
SCH_TASK_DLOG("task no exec address to notify, status:%s", SCH_GET_TASK_STATUS_STR(pTask));
return TSDB_CODE_SUCCESS;
}
int32_t i = 0;
SSchNodeInfo *nodeInfo = taosHashIterate(pTask->execNodes, NULL);
while (nodeInfo) {
if (nodeInfo->handle) {
SCH_SET_TASK_HANDLE(pTask, nodeInfo->handle);
SCH_ERR_RET(schBuildAndSendMsg(pJob, pTask, &nodeInfo->addr, TDMT_SCH_TASK_NOTIFY, &type));
SCH_TASK_DLOG("start to notify %d to task's %dth execNode", type, i);
} else {
SCH_TASK_DLOG("no need to notify %d to task %dth execNode", type, i);
}
++i;
nodeInfo = taosHashIterate(pTask->execNodes, nodeInfo);
}
SCH_TASK_DLOG("task has been notified %d on %d exec nodes", type, size);
return TSDB_CODE_SUCCESS;
}
int32_t schProcessOnTaskStatusRsp(SQueryNodeEpId *pEpId, SArray *pStatusList) {
int32_t taskNum = (int32_t)taosArrayGetSize(pStatusList);
SSchTask *pTask = NULL;
@ -1001,7 +1028,7 @@ int32_t schLaunchRemoteTask(SSchJob *pJob, SSchTask *pTask) {
SCH_ERR_RET(schEnsureHbConnection(pJob, pTask));
}
SCH_RET(schBuildAndSendMsg(pJob, pTask, NULL, plan->msgType));
SCH_RET(schBuildAndSendMsg(pJob, pTask, NULL, plan->msgType, NULL));
}
int32_t schLaunchLocalTask(SSchJob *pJob, SSchTask *pTask) {
@ -1238,8 +1265,33 @@ void schDropTaskInHashList(SSchJob *pJob, SHashObj *list) {
}
}
int32_t schNotifyTaskInHashList(SSchJob *pJob, SHashObj *list, ETaskNotifyType type, SSchTask *pCurrTask) {
int32_t code = TSDB_CODE_SUCCESS;
SCH_ERR_RET(schNotifyTaskOnExecNode(pJob, pCurrTask, type));
void *pIter = taosHashIterate(list, NULL);
while (pIter) {
SSchTask *pTask = *(SSchTask **)pIter;
if (pTask != pCurrTask) {
SCH_LOCK_TASK(pTask);
code = schNotifyTaskOnExecNode(pJob, pTask, type);
SCH_UNLOCK_TASK(pTask);
if (TSDB_CODE_SUCCESS != code) {
break;
}
}
pIter = taosHashIterate(list, pIter);
}
SCH_RET(code);
}
int32_t schExecRemoteFetch(SSchJob *pJob, SSchTask *pTask) {
SCH_RET(schBuildAndSendMsg(pJob, pJob->fetchTask, &pJob->resNode, SCH_FETCH_TYPE(pJob->fetchTask)));
SCH_RET(schBuildAndSendMsg(pJob, pJob->fetchTask, &pJob->resNode, SCH_FETCH_TYPE(pJob->fetchTask), NULL));
}
int32_t schExecLocalFetch(SSchJob *pJob, SSchTask *pTask) {

114
source/os/src/osString.c
View File

@ -468,3 +468,117 @@ float taosStr2Float(const char *str, char **pEnd) {
#endif
return tmp;
}
#define HEX_PREFIX_LEN 2 // \x
bool isHex(const char* z, uint32_t n){
if(n < HEX_PREFIX_LEN) return false;
if(z[0] == '\\' && z[1] == 'x') return true;
return false;
}
bool isValidateHex(const char* z, uint32_t n){
if(n % 2 != 0) return false;
for(size_t i = HEX_PREFIX_LEN; i < n; i++){
if(isxdigit(z[i]) == 0){
return false;
}
}
return true;
}
int32_t taosHex2Ascii(const char *z, uint32_t n, void** data, uint32_t* size){
n -= HEX_PREFIX_LEN; // remove 0x
z += HEX_PREFIX_LEN;
*size = n / HEX_PREFIX_LEN;
if(*size == 0) return 0;
uint8_t* tmp = (uint8_t*)taosMemoryCalloc(*size, 1);
if(tmp == NULL) return -1;
int8_t num = 0;
uint8_t *byte = tmp + *size - 1;
for (int i = n - 1; i >= 0; i--) {
if (z[i] >= 'a') {
*byte |= ((uint8_t)(10 + (z[i] - 'a')) << (num * 4));
} else if (z[i] >= 'A') {
*byte |= ((uint8_t)(10 + (z[i] - 'A')) << (num * 4));
} else {
*byte |= ((uint8_t)(z[i] - '0') << (num * 4));
}
if (num == 1) {
byte--;
num = 0;
} else {
num++;
}
}
*data = tmp;
return 0;
}
//int32_t taosBin2Ascii(const char *z, uint32_t n, void** data, uint32_t* size){
//
// for (i = 2; isdigit(z[i]) || (z[i] >= 'a' && z[i] <= 'f') || (z[i] >= 'A' && z[i] <= 'F'); ++i) {
// }
//
// n -= 2; // remove 0b
// z += 2;
// *size = n%8 == 0 ? n/8 : n/8 + 1;
// uint8_t* tmp = (uint8_t*)taosMemoryCalloc(*size, 1);
// if(tmp == NULL) return -1;
// int8_t num = 0;
// uint8_t *byte = tmp + *size - 1;
//
// for (int i = n - 1; i >= 0; i--) {
// *byte |= ((uint8_t)(z[i] - '0') << num);
// if (num == 7) {
// byte--;
// num = 0;
// } else {
// num++;
// }
// }
// *data = tmp;
// return 0;
//}
static char valueOf(uint8_t symbol)
{
switch(symbol)
{
case 0: return '0';
case 1: return '1';
case 2: return '2';
case 3: return '3';
case 4: return '4';
case 5: return '5';
case 6: return '6';
case 7: return '7';
case 8: return '8';
case 9: return '9';
case 10: return 'A';
case 11: return 'B';
case 12: return 'C';
case 13: return 'D';
case 14: return 'E';
case 15: return 'F';
default:
{
return -1;
}
}
}
int32_t taosAscii2Hex(const char *z, uint32_t n, void** data, uint32_t* size){
*size = n * 2 + HEX_PREFIX_LEN;
uint8_t* tmp = (uint8_t*)taosMemoryCalloc(*size + 1, 1);
if(tmp == NULL) return -1;
*data = tmp;
*(tmp++) = '\\';
*(tmp++) = 'x';
for(int i = 0; i < n; i ++){
uint8_t val = z[i];
tmp[i*2] = valueOf(val >> 4);
tmp[i*2 + 1] = valueOf(val & 0x0F);
}
return 0;
}

75
source/util/src/tcompare.c
View File

@ -242,6 +242,10 @@ int32_t compareLenBinaryVal(const void *pLeft, const void *pRight) {
}
}
int32_t compareLenBinaryValDesc(const void *pLeft, const void *pRight) {
return compareLenBinaryVal(pRight, pLeft);
}
// string > number > bool > null
// ref: https://dev.mysql.com/doc/refman/8.0/en/json.html#json-comparison
int32_t compareJsonVal(const void *pLeft, const void *pRight) {
@ -1301,7 +1305,8 @@ int32_t comparewcsPatternNMatch(const void *pLeft, const void *pRight) {
__compar_fn_t getComparFunc(int32_t type, int32_t optr) {
__compar_fn_t comparFn = NULL;
if (optr == OP_TYPE_IN && (type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR && type != TSDB_DATA_TYPE_GEOMETRY)) {
if (optr == OP_TYPE_IN && (type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_VARBINARY &&
type != TSDB_DATA_TYPE_NCHAR && type != TSDB_DATA_TYPE_GEOMETRY)) {
switch (type) {
case TSDB_DATA_TYPE_BOOL:
case TSDB_DATA_TYPE_TINYINT:
@ -1324,7 +1329,8 @@ __compar_fn_t getComparFunc(int32_t type, int32_t optr) {
}
}
if (optr == OP_TYPE_NOT_IN && (type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR && type != TSDB_DATA_TYPE_GEOMETRY)) {
if (optr == OP_TYPE_NOT_IN && (type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_VARBINARY &&
type != TSDB_DATA_TYPE_NCHAR && type != TSDB_DATA_TYPE_GEOMETRY)) {
switch (type) {
case TSDB_DATA_TYPE_BOOL:
case TSDB_DATA_TYPE_TINYINT:
@ -1368,6 +1374,15 @@ __compar_fn_t getComparFunc(int32_t type, int32_t optr) {
case TSDB_DATA_TYPE_DOUBLE:
comparFn = compareDoubleVal;
break;
case TSDB_DATA_TYPE_VARBINARY:
if (optr == OP_TYPE_IN) {
comparFn = compareChkInString;
} else if (optr == OP_TYPE_NOT_IN) {
comparFn = compareChkNotInString;
} else { /* normal relational comparFn */
comparFn = compareLenBinaryVal;
}
break;
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_GEOMETRY: {
if (optr == OP_TYPE_MATCH) {
@ -1453,6 +1468,8 @@ __compar_fn_t getKeyComparFunc(int32_t keyType, int32_t order) {
return (order == TSDB_ORDER_ASC) ? compareUint32Val : compareUint32ValDesc;
case TSDB_DATA_TYPE_UBIGINT:
return (order == TSDB_ORDER_ASC) ? compareUint64Val : compareUint64ValDesc;
case TSDB_DATA_TYPE_VARBINARY:
return (order == TSDB_ORDER_ASC) ? compareLenBinaryVal : compareLenBinaryValDesc;
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_GEOMETRY:
return (order == TSDB_ORDER_ASC) ? compareLenPrefixedStr : compareLenPrefixedStrDesc;
@ -1464,57 +1481,3 @@ __compar_fn_t getKeyComparFunc(int32_t keyType, int32_t order) {
return (order == TSDB_ORDER_ASC) ? compareInt32Val : compareInt32ValDesc;
}
}
int32_t doCompare(const char *f1, const char *f2, int32_t type, size_t size) {
switch (type) {
case TSDB_DATA_TYPE_INT:
DEFAULT_COMP(GET_INT32_VAL(f1), GET_INT32_VAL(f2));
case TSDB_DATA_TYPE_DOUBLE:
DEFAULT_DOUBLE_COMP(GET_DOUBLE_VAL(f1), GET_DOUBLE_VAL(f2));
case TSDB_DATA_TYPE_FLOAT:
DEFAULT_FLOAT_COMP(GET_FLOAT_VAL(f1), GET_FLOAT_VAL(f2));
case TSDB_DATA_TYPE_BIGINT:
DEFAULT_COMP(GET_INT64_VAL(f1), GET_INT64_VAL(f2));
case TSDB_DATA_TYPE_SMALLINT:
DEFAULT_COMP(GET_INT16_VAL(f1), GET_INT16_VAL(f2));
case TSDB_DATA_TYPE_TINYINT:
case TSDB_DATA_TYPE_BOOL:
DEFAULT_COMP(GET_INT8_VAL(f1), GET_INT8_VAL(f2));
case TSDB_DATA_TYPE_UTINYINT:
DEFAULT_COMP(GET_UINT8_VAL(f1), GET_UINT8_VAL(f2));
case TSDB_DATA_TYPE_USMALLINT:
DEFAULT_COMP(GET_UINT16_VAL(f1), GET_UINT16_VAL(f2));
case TSDB_DATA_TYPE_UINT:
DEFAULT_COMP(GET_UINT32_VAL(f1), GET_UINT32_VAL(f2));
case TSDB_DATA_TYPE_UBIGINT:
DEFAULT_COMP(GET_UINT64_VAL(f1), GET_UINT64_VAL(f2));
case TSDB_DATA_TYPE_NCHAR: {
tstr *t1 = (tstr *)f1;
tstr *t2 = (tstr *)f2;
if (t1->len != t2->len) {
return t1->len > t2->len ? 1 : -1;
}
int32_t ret = memcmp((TdUcs4 *)t1, (TdUcs4 *)t2, t2->len);
if (ret == 0) {
return ret;
}
return (ret < 0) ? -1 : 1;
}
default: { // todo refactor
tstr *t1 = (tstr *)f1;
tstr *t2 = (tstr *)f2;
if (t1->len != t2->len) {
return t1->len > t2->len ? 1 : -1;
} else {
int32_t ret = strncmp(t1->data, t2->data, t1->len);
if (ret == 0) {
return 0;
} else {
return ret < 0 ? -1 : 1;
}
}
}
}
}

1
source/util/src/terror.c
View File

@ -556,6 +556,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_PAR_NOT_ALLOWED_WIN_QUERY, "Window query not su
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_DROP_COL, "No columns can be dropped")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_COL_JSON, "Only tag can be json type")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_VALUE_TOO_LONG, "Value too long for column/tag")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_VARBINARY, "Invalidate varbinary type")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_DELETE_WHERE, "The DELETE statement must have a definite time window range")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_REDISTRIBUTE_VG, "The REDISTRIBUTE VGROUP statement only support 1 to 3 dnodes")
TAOS_DEFINE_ERROR(TSDB_CODE_PAR_FILL_NOT_ALLOWED_FUNC, "Fill not allowed")

1
source/util/src/thashutil.c
View File

@ -193,6 +193,7 @@ _hash_fn_t taosGetDefaultHashFunction(int32_t type) {
fn = taosIntHash_64;
break;
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_VARBINARY:
case TSDB_DATA_TYPE_NCHAR:
case TSDB_DATA_TYPE_GEOMETRY:
fn = MurmurHash3_32;

1
source/util/src/tskiplist.c
View File

@ -376,6 +376,7 @@ void tSkipListPrint(SSkipList *pSkipList, int16_t nlevel) {
fprintf(stdout, "%d: %" PRId64 " \n", id++, *(int64_t *)key);
break;
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_VARBINARY:
case TSDB_DATA_TYPE_GEOMETRY:
fprintf(stdout, "%d: %s \n", id++, key);
break;

1
tests/parallel_test/cases.task
View File

@ -360,6 +360,7 @@
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/smaTest.py -R
,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/sma_index.py
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sml_TS-3724.py
#,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/varbinary.py
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sml.py
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sml.py -R
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/spread.py

5
tests/script/api/batchprepare.c
View File

@ -1222,6 +1222,7 @@ int32_t bpAppendValueString(char *buf, int type, void *value, int32_t valueLen,
break;
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_VARBINARY:
case TSDB_DATA_TYPE_NCHAR:
case TSDB_DATA_TYPE_GEOMETRY:
buf[*len] = '\'';
@ -1369,7 +1370,7 @@ void bpCheckColTagFields(TAOS_STMT *stmt, int32_t fieldNum, TAOS_FIELD_E* pField
exit(1);
}
if (pFields[i].type == TSDB_DATA_TYPE_BINARY || pFields[i].type == TSDB_DATA_TYPE_GEOMETRY) {
if (pFields[i].type == TSDB_DATA_TYPE_BINARY || pFields[i].type == TSDB_DATA_TYPE_VARBINARY || pFields[i].type == TSDB_DATA_TYPE_GEOMETRY) {
if (pFields[i].bytes != (pBind[i].buffer_length + 2)) {
printf("!!!%s %dth field len %d mis-match expect len %d\n", BP_BIND_TYPE_STR(type), i, pFields[i].bytes, (pBind[i].buffer_length + 2));
exit(1);
@ -2256,7 +2257,7 @@ int insertNonExistsTb(TAOS_STMT *stmt, TAOS *taos) {
void bpAddWrongVarBuffLen(TAOS_MULTI_BIND* pBind) {
for (int32_t i = 0; i < gCurCase->bindColNum; ++i) {
if (pBind[i].buffer_type == TSDB_DATA_TYPE_BINARY || pBind[i].buffer_type == TSDB_DATA_TYPE_NCHAR) {
if (pBind[i].buffer_type == TSDB_DATA_TYPE_BINARY || pBind[i].buffer_type == TSDB_DATA_TYPE_VARBINARY || pBind[i].buffer_type == TSDB_DATA_TYPE_NCHAR) {
*pBind[i].length += 100;
}
}

1
tests/script/api/passwdTest.c
View File

@ -145,6 +145,7 @@ int printRow(char *str, TAOS_ROW row, TAOS_FIELD *fields, int numFields) {
len += sprintf(str + len, "%lf", dv);
} break;
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_VARBINARY:
case TSDB_DATA_TYPE_NCHAR:
case TSDB_DATA_TYPE_GEOMETRY: {
int32_t charLen = varDataLen((char *)row[i] - VARSTR_HEADER_SIZE);

1
tests/script/api/stmt_function.c
View File

@ -400,6 +400,7 @@ void taos_stmt_use_result_query(void *taos, char *col, int type) {
params->length = &params->buffer_length;
break;
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_VARBINARY:
case TSDB_DATA_TYPE_GEOMETRY:
params->buffer_length = sizeof(v.c6);
params->buffer = &v.c6;

8
tests/system-test/2-query/sml.py
View File

@ -1,18 +1,12 @@
import taos
import sys
import time
import socket
import os
import threading
from util.log import *
from util.sql import *
from util.cases import *
from util.dnodes import *
from util.common import *
sys.path.append("./7-tmq")
from tmqCommon import *
class TDTestCase:
updatecfgDict = {'clientCfg': {'smlChildTableName': 'dataModelName', 'fqdn': 'localhost', 'smlDot2Underline': 0}, 'fqdn': 'localhost'}
@ -32,7 +26,7 @@ class TDTestCase:
tdLog.info(cmdStr)
ret = os.system(cmdStr)
if ret != 0:
tdLog.info("sml_test ret != 0")
tdLog.exit("sml_test ret != 0")
tdSql.query(f"select * from ts3303.stb2")
tdSql.query(f"select * from ts3303.meters")

80
tests/system-test/2-query/varbinary.py Normal file
View File

@ -0,0 +1,80 @@
import sys
import os
from util.log import *
from util.sql import *
from util.cases import *
from util.common import *
class TDTestCase:
def init(self, conn, logSql, replicaVar=1):
self.replicaVar = int(replicaVar)
tdLog.debug(f"start to excute {__file__}")
tdSql.init(conn.cursor(), False)
def test(self):
tdLog.info(" test")
buildPath = tdCom.getBuildPath()
cmdStr = '%s/build/bin/varbinary_test'%(buildPath)
print("cmdStr:", cmdStr)
tdLog.info(cmdStr)
ret = os.system(cmdStr)
if ret != 0:
tdLog.exit("varbinary_test ret != 0")
tdSql.execute(f" create database test")
tdSql.execute(f" use test ")
tdSql.execute(f" create stable stb (ts timestamp, c1 nchar(32), c2 varbinary(16), c3 float) tags (t1 int, t2 binary(8), t3 varbinary(8))")
tdSql.query(f"desc stb")
tdSql.checkRows(7)
tdSql.checkData(2, 1, 'VARBINARY')
tdSql.checkData(2, 2, 16)
tdSql.checkData(6, 1, 'VARBINARY')
tdSql.checkData(6, 2, 8)
# tdSql.execute(f" insert into tb1 using stb tags (1, 'tb1_bin1', 'vart1') values (now, 'nchar1', 'varc1', 0.3)")
# tdSql.execute(f" insert into tb1 values (now + 1s, 'nchar2', null, 0.4)")
#
# tdSql.execute(f" insert into tb2 using stb tags (2, 'tb2_bin1', 093) values (now + 2s, 'nchar1', 892, 0.3)")
# tdSql.execute(f" insert into tb3 using stb tags (3, 'tb3_bin1', 0x7f829) values (now + 3s, 'nchar1', 0x7f829, 0.3)")
# tdSql.execute(f" insert into tb4 using stb tags (4, 'tb4_bin1', 0b100000010) values (now + 4s, 'nchar1', 0b110000001, 0.3)")
# tdSql.execute(f" insert into tb4 values (now + 5s, 'nchar1', 0b11000000100000000, 0.3)")
# tdSql.execute(f" insert into tb5 using stb tags (4, 'tb5_bin1', NULL) values (now + 6s, 'nchar1', 0b10100000011000000110000001, 0.3)")
# basic query
# tdSql.query(f"select c2,t3 from stb order by ts")
# tdSql.checkRows(6)
# tdSql.checkData(0, 0, '0x7661726331')
# tdSql.checkData(0, 1, '0x7661727431')
# tdSql.checkData(1, 0, None)
# tdSql.checkData(1, 1, '0x7661727431')
# tdSql.checkData(2, 0, '0x383932')
# tdSql.checkData(2, 1, '0x303933')
# tdSql.checkData(3, 0, '0x07f829')
# tdSql.checkData(3, 1, '0x07f829')
# tdSql.checkData(4, 0, '0x0181')
# tdSql.checkData(4, 1, '0x0102')
# tdSql.checkData(5, 0, '0x02818181')
# tdSql.checkData(5, 1, None)
# tdSql.query(f"select ts,c2 from stb order by c2")
#
# tdSql.query(f"select c2,t3 from stb where c2 >= 0 order by ts")
#
# tdSql.query(f"select c2,t3 from stb where c2 >= 0x0181 order by ts")
def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring
tdSql.prepare()
self.test()
def stop(self):
tdSql.close()
tdLog.success(f"{__file__} successfully executed")
tdCases.addLinux(__file__, TDTestCase())
tdCases.addWindows(__file__, TDTestCase())

76
tools/shell/src/shellEngine.c
View File

@ -323,13 +323,12 @@ void shellDumpFieldToFile(TdFilePtr pFile, const char *val, TAOS_FIELD *field, i
return;
}
char quotationStr[2];
quotationStr[0] = '\"';
quotationStr[1] = 0;
char quotationStr[2] ={'"', 0};
int32_t width;
int n;
char buf[TSDB_MAX_BYTES_PER_ROW];
int n = 0;
#define LENGTH 64
char buf[LENGTH] = {0};
switch (field->type) {
case TSDB_DATA_TYPE_BOOL:
taosFprintfFile(pFile, "%d", ((((int32_t)(*((char *)val))) == 1) ? 1 : 0));
@ -363,7 +362,7 @@ void shellDumpFieldToFile(TdFilePtr pFile, const char *val, TAOS_FIELD *field, i
if (tsEnableScience) {
taosFprintfFile(pFile, "%*.7e", width, GET_FLOAT_VAL(val));
} else {
n = snprintf(buf, TSDB_MAX_BYTES_PER_ROW, "%*.7f", width, GET_FLOAT_VAL(val));
n = snprintf(buf, LENGTH, "%*.7f", width, GET_FLOAT_VAL(val));
if (n > SHELL_FLOAT_WIDTH) {
taosFprintfFile(pFile, "%*.7e", width, GET_FLOAT_VAL(val));
} else {
@ -374,10 +373,10 @@ void shellDumpFieldToFile(TdFilePtr pFile, const char *val, TAOS_FIELD *field, i
case TSDB_DATA_TYPE_DOUBLE:
width = SHELL_DOUBLE_WIDTH;
if (tsEnableScience) {
snprintf(buf, TSDB_MAX_BYTES_PER_ROW, "%*.15e", width, GET_DOUBLE_VAL(val));
snprintf(buf, LENGTH, "%*.15e", width, GET_DOUBLE_VAL(val));
taosFprintfFile(pFile, "%s", buf);
} else {
n = snprintf(buf, TSDB_MAX_BYTES_PER_ROW, "%*.15f", width, GET_DOUBLE_VAL(val));
n = snprintf(buf, LENGTH, "%*.15f", width, GET_DOUBLE_VAL(val));
if (n > SHELL_DOUBLE_WIDTH) {
taosFprintfFile(pFile, "%*.15e", width, GET_DOUBLE_VAL(val));
} else {
@ -389,22 +388,39 @@ void shellDumpFieldToFile(TdFilePtr pFile, const char *val, TAOS_FIELD *field, i
case TSDB_DATA_TYPE_NCHAR:
case TSDB_DATA_TYPE_JSON: {
int32_t bufIndex = 0;
char* tmp = (char*)taosMemoryCalloc(length * 2 + 1, 1);
if(tmp == NULL) break;
for (int32_t i = 0; i < length; i++) {
buf[bufIndex] = val[i];
tmp[bufIndex] = val[i];
bufIndex++;
if (val[i] == '\"') {
buf[bufIndex] = val[i];
tmp[bufIndex] = val[i];
bufIndex++;
}
}
buf[bufIndex] = 0;
tmp[bufIndex] = 0;
taosFprintfFile(pFile, "%s%s%s", quotationStr, buf, quotationStr);
taosFprintfFile(pFile, "%s%s%s", quotationStr, tmp, quotationStr);
taosMemoryFree(tmp);
} break;
case TSDB_DATA_TYPE_GEOMETRY:
shellDumpHexValue(buf, val, length);
taosFprintfFile(pFile, "%s", buf);
case TSDB_DATA_TYPE_VARBINARY:{
void* tmp = NULL;
uint32_t size = 0;
if(taosAscii2Hex(val, length, &tmp, &size) < 0){
break;
}
taosFprintfFile(pFile, "%s", tmp);
taosMemoryFree(tmp);
break;
}
case TSDB_DATA_TYPE_GEOMETRY:{
char* tmp = (char*)taosMemoryCalloc(length * 2 + 1, 1);
if(tmp == NULL) break;
shellDumpHexValue(tmp, val, length);
taosFprintfFile(pFile, "%s", buf);
taosMemoryFree(tmp);
break;
}
case TSDB_DATA_TYPE_TIMESTAMP:
shellFormatTimestamp(buf, *(int64_t *)val, precision);
taosFprintfFile(pFile, "%s%s%s", quotationStr, buf, quotationStr);
@ -576,8 +592,9 @@ void shellPrintField(const char *val, TAOS_FIELD *field, int32_t width, int32_t
return;
}
int n;
char buf[TSDB_MAX_BYTES_PER_ROW];
int n = 0;
#define LENGTH 64
char buf[LENGTH] = {0};
switch (field->type) {
case TSDB_DATA_TYPE_BOOL:
shellPrintString(((((int32_t)(*((char *)val))) == 1) ? "true" : "false"), width);
@ -610,7 +627,7 @@ void shellPrintField(const char *val, TAOS_FIELD *field, int32_t width, int32_t
if (tsEnableScience) {
printf("%*.7e", width, GET_FLOAT_VAL(val));
} else {
n = snprintf(buf, TSDB_MAX_BYTES_PER_ROW, "%*.7f", width, GET_FLOAT_VAL(val));
n = snprintf(buf, LENGTH, "%*.7f", width, GET_FLOAT_VAL(val));
if (n > SHELL_FLOAT_WIDTH) {
printf("%*.7e", width, GET_FLOAT_VAL(val));
} else {
@ -620,10 +637,10 @@ void shellPrintField(const char *val, TAOS_FIELD *field, int32_t width, int32_t
break;
case TSDB_DATA_TYPE_DOUBLE:
if (tsEnableScience) {
snprintf(buf, TSDB_MAX_BYTES_PER_ROW, "%*.15e", width, GET_DOUBLE_VAL(val));
snprintf(buf, LENGTH, "%*.15e", width,GET_DOUBLE_VAL(val));
printf("%s", buf);
} else {
n = snprintf(buf, TSDB_MAX_BYTES_PER_ROW, "%*.15f", width, GET_DOUBLE_VAL(val));
n = snprintf(buf, LENGTH, "%*.15f", width, GET_DOUBLE_VAL(val));
if (n > SHELL_DOUBLE_WIDTH) {
printf("%*.15e", width, GET_DOUBLE_VAL(val));
} else {
@ -631,6 +648,16 @@ void shellPrintField(const char *val, TAOS_FIELD *field, int32_t width, int32_t
}
}
break;
case TSDB_DATA_TYPE_VARBINARY:{
void* data = NULL;
uint32_t size = 0;
if(taosAscii2Hex(val, length, &data, &size) < 0){
break;
}
shellPrintNChar(data, size, width);
taosMemoryFree(data);
break;
}
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_NCHAR:
case TSDB_DATA_TYPE_JSON:
@ -773,7 +800,14 @@ int32_t shellCalcColWidth(TAOS_FIELD *field, int32_t precision) {
} else {
return TMAX(field->bytes + 2, width);
}
case TSDB_DATA_TYPE_VARBINARY:{
int32_t bytes = field->bytes * 2 + 2;
if (bytes > shell.args.displayWidth) {
return TMAX(shell.args.displayWidth, width);
} else {
return TMAX(bytes + 2, width);
}
}
case TSDB_DATA_TYPE_NCHAR:
case TSDB_DATA_TYPE_JSON: {
uint16_t bytes = field->bytes * TSDB_NCHAR_SIZE;

11
utils/test/c/CMakeLists.txt
View File

@ -8,6 +8,7 @@ add_executable(sml_test sml_test.c)
add_executable(get_db_name_test get_db_name_test.c)
add_executable(tmq_offset tmqOffset.c)
add_executable(tmq_offset_test tmq_offset_test.c)
add_executable(varbinary_test varbinary_test.c)
target_link_libraries(
tmq_offset
PUBLIC taos
@ -65,6 +66,8 @@ target_link_libraries(
PUBLIC util
PUBLIC common
PUBLIC os
PUBLIC geometry
)
target_link_libraries(
@ -74,3 +77,11 @@ target_link_libraries(
PUBLIC common
PUBLIC os
)
target_link_libraries(
varbinary_test
PUBLIC taos
PUBLIC util
PUBLIC common
PUBLIC os
)

80
utils/test/c/sml_test.c
View File

@ -21,6 +21,7 @@
#include "taos.h"
#include "tlog.h"
#include "types.h"
#include "geosWrapper.h"
int smlProcess_influx_Test() {
TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0);
@ -1578,6 +1579,83 @@ int sml_td24559_Test() {
printf("%s result0:%s\n", __FUNCTION__, taos_errstr(pRes));
taos_free_result(pRes);
TAOS_ROW row = NULL;
pRes = taos_query(taos, "select * from stb order by _ts;");
int rowIndex = 0;
while ((row = taos_fetch_row(pRes)) != NULL) {
int32_t* length = taos_fetch_lengths(pRes);
code = initCtxAsText();
ASSERT (code == TSDB_CODE_SUCCESS);
char *outputWKT = NULL;
code = doAsText(row[2], length[2], &outputWKT);
ASSERT (code == TSDB_CODE_SUCCESS);
if (rowIndex == 0) {
ASSERT(strcmp("POINT (4.343000 89.342000)", outputWKT) == 0);
}
if (rowIndex == 3) {
ASSERT(strcmp( "GEOMETRYCOLLECTION (MULTIPOINT ((0.000000 0.000000), (1.000000 1.000000)), POINT (3.000000 4.000000), LINESTRING (2.000000 3.000000, 3.000000 4.000000))", outputWKT) == 0);
}
geosFreeBuffer(outputWKT);
rowIndex++;
}
taos_free_result(pRes);
taos_close(taos);
return code;
}
int sml_td18789_Test() {
TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0);
TAOS_RES *pRes = taos_query(taos, "drop database if exists td18789");
taos_free_result(pRes);
pRes = taos_query(taos, "create database if not exists td18789");
taos_free_result(pRes);
const char *sql[] = {
"vbin,t1=1 f1=283i32,f2=b\"hello\" 1632299372000",
"vbin,t1=1 f2=B\"\\x98f46e\",f1=106i32 1632299373000",
};
pRes = taos_query(taos, "use td18789");
taos_free_result(pRes);
pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_LINE_PROTOCOL,
TSDB_SML_TIMESTAMP_MILLI_SECONDS);
int code = taos_errno(pRes);
printf("%s result0:%s\n", __FUNCTION__, taos_errstr(pRes));
taos_free_result(pRes);
TAOS_ROW row = NULL;
pRes = taos_query(taos, "select *,tbname from vbin order by _ts");
int rowIndex = 0;
while ((row = taos_fetch_row(pRes)) != NULL) {
int32_t* length = taos_fetch_lengths(pRes);
void* data = NULL;
uint32_t size = 0;
if(taosAscii2Hex(row[2], length[2], &data, &size) < 0){
ASSERT(0);
}
if (rowIndex == 0) {
ASSERT(memcmp(data, "\\x68656C6C6F", size) == 0);
}
if (rowIndex == 1) {
ASSERT(memcmp(data, "\\x98F46E", size) == 0);
}
taosMemoryFree(data);
rowIndex++;
}
taos_free_result(pRes);
taos_close(taos);
return code;
@ -1591,6 +1669,8 @@ int main(int argc, char *argv[]) {
int ret = 0;
ret = sml_td24559_Test();
ASSERT(!ret);
ret = sml_td18789_Test();
ASSERT(!ret);
ret = sml_td24070_Test();
ASSERT(!ret);
ret = sml_td23881_Test();

1
utils/test/c/tmqSim.c
View File

@ -541,6 +541,7 @@ static void shellDumpFieldToFile(TdFilePtr pFile, const char* val, TAOS_FIELD* f
}
break;
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_VARBINARY:
case TSDB_DATA_TYPE_NCHAR:
case TSDB_DATA_TYPE_JSON:
case TSDB_DATA_TYPE_GEOMETRY: {

253
utils/test/c/varbinary_test.c Normal file
View File

@ -0,0 +1,253 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <assert.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
#include "taos.h"
#include "types.h"
#include "tlog.h"
#define GET_ROW_NUM \
numRows = 0;\
while(1){\
row = taos_fetch_row(pRes);\
if (row != NULL){\
numRows++;\
}else{\
break;\
}\
}
int varbinary_test() {
TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0);
TAOS_RES *pRes = taos_query(taos, "drop database if exists varbinary_db");
taos_free_result(pRes);
pRes = taos_query(taos, "create database if not exists varbinary_db");
taos_free_result(pRes);
pRes = taos_query(taos, "use varbinary_db");
int code = taos_errno(pRes);
taos_free_result(pRes);
ASSERT(code == 0);
pRes = taos_query(taos, "create stable stb (ts timestamp, c1 nchar(32), c2 varbinary(16), c3 float) tags (t1 int, t2 binary(8), t3 varbinary(8))");
taos_free_result(pRes);
pRes = taos_query(taos, "desc stb");
TAOS_ROW row = NULL;
int32_t rowIndex = 0;
while ((row = taos_fetch_row(pRes)) != NULL) {
char* type = row[1];
int32_t length = *(int32_t *)row[2];
if (rowIndex == 2) {
ASSERT(strncmp(type, "VARBINARY", sizeof("VARBINARY") - 1) == 0);
ASSERT(length == 16);
}
if (rowIndex == 6) {
ASSERT(strncmp(type, "VARBINARY", sizeof("VARBINARY") - 1) == 0);
ASSERT(length == 8);
}
rowIndex++;
}
pRes = taos_query(taos, "insert into tb1 using stb tags (1, 'tb1_bin1', 'vart1') values (now, 'nchar1', 'varc1', 0.3)");
taos_free_result(pRes);
pRes = taos_query(taos, "insert into tb1 values (now + 1s, 'nchar2', null, 0.4);");
taos_free_result(pRes);
pRes = taos_query(taos, "insert into tb3 using stb tags (3, 'tb3_bin1', '\\x7f8290') values (now + 2s, 'nchar1', '\\x7f8290', 0.3)");
taos_free_result(pRes);
pRes = taos_query(taos, "insert into tb3 values (now + 3s, 'nchar1', '\\x7f829000', 0.3)");
taos_free_result(pRes);
pRes = taos_query(taos, "insert into tb2 using stb tags (2, 'tb2_bin1', '\\x') values (now + 4s, 'nchar1', '\\x', 0.3)");
taos_free_result(pRes);
pRes = taos_query(taos, "insert into tb2 values (now + 5s, 'nchar1', '\\x00000000', 0.3)");
taos_free_result(pRes);
// test insert
pRes = taos_query(taos, "insert into tb2 using stb tags (2, 'tb2_bin1', 093) values (now + 2s, 'nchar1', 892, 0.3)");
ASSERT(taos_errno(pRes) != 0);
pRes = taos_query(taos, "insert into tb3 using stb tags (3, 'tb3_bin1', 0x7f829) values (now + 3s, 'nchar1', 0x7f829, 0.3)");
ASSERT(taos_errno(pRes) != 0);
pRes = taos_query(taos, "insert into tb3 using stb tags (3, 'tb3_bin1', '\\x7f829') values (now + 3s, 'nchar1', '\\x7f829', 0.3)");
ASSERT(taos_errno(pRes) != 0);
pRes = taos_query(taos, "insert into tb4 using stb tags (4, 'tb4_bin1', 0b100000010) values (now + 4s, 'nchar1', 0b110000001, 0.3)");
ASSERT(taos_errno(pRes) != 0);
taos_free_result(pRes);
// test error
pRes = taos_query(taos, "select * from tb1 where c2 >= 0x8de6");
ASSERT(taos_errno(pRes) != 0);
taos_free_result(pRes);
pRes = taos_query(taos, "select * from tb1 where c2 >= 0");
ASSERT(taos_errno(pRes) != 0);
taos_free_result(pRes);
pRes = taos_query(taos, "select * from stb where c2 > '\\x7F82900'");
ASSERT(taos_errno(pRes) != 0);
taos_free_result(pRes);
pRes = taos_query(taos, "select c2+2 from stb");
ASSERT(taos_errno(pRes) != 0);
taos_free_result(pRes);
pRes = taos_query(taos, "select c2|2 from stb");
ASSERT(taos_errno(pRes) != 0);
taos_free_result(pRes);
pRes = taos_query(taos, "select * from stb where c2 not like 's%'");
ASSERT(taos_errno(pRes) != 0);
taos_free_result(pRes);
pRes = taos_query(taos, "select * from stb where c2 like 's%'");
ASSERT(taos_errno(pRes) != 0);
taos_free_result(pRes);
pRes = taos_query(taos, "select * from stb where c2 match 'ssd'");
ASSERT(taos_errno(pRes) != 0);
taos_free_result(pRes);
pRes = taos_query(taos, "select * from stb where c2 nmatch 'ssd'");
ASSERT(taos_errno(pRes) != 0);
taos_free_result(pRes);
pRes = taos_query(taos, "select * from stb where c2->'ssd' = 1");
ASSERT(taos_errno(pRes) != 0);
taos_free_result(pRes);
pRes = taos_query(taos, "select * from stb where c2 contains 'ssd'");
ASSERT(taos_errno(pRes) != 0);
taos_free_result(pRes);
pRes = taos_query(taos, "select * from stb where sum(c2) = 2");
ASSERT(taos_errno(pRes) != 0);
taos_free_result(pRes);
// pRes = taos_query(taos, "select * from stb where c2 contains 'ssd'");
// ASSERT(taos_errno(pRes) != 0);
// taos_free_result(pRes);
//
// pRes = taos_query(taos, "select * from stb where c2 contains 'ssd'");
// ASSERT(taos_errno(pRes) != 0);
// taos_free_result(pRes);
int numRows = 0;
pRes = taos_query(taos, "select * from stb where c2 > 'varc1'");
GET_ROW_NUM
ASSERT(numRows == 2);
taos_free_result(pRes);
pRes = taos_query(taos, "select * from stb where c2 > '\\x7F8290'");
GET_ROW_NUM
ASSERT(numRows == 1);
taos_free_result(pRes);
pRes = taos_query(taos, "select * from stb where c2 in ('\\x7F829000','\\x00000000')");
GET_ROW_NUM
ASSERT(numRows == 2);
taos_free_result(pRes);
pRes = taos_query(taos, "select * from stb where c2 not in ('\\x00000000')");
GET_ROW_NUM
ASSERT(numRows == 4);
taos_free_result(pRes);
pRes = taos_query(taos, "select * from stb where c2 is null");
GET_ROW_NUM
ASSERT(numRows == 1);
taos_free_result(pRes);
pRes = taos_query(taos, "select * from stb where c2 is not null");
GET_ROW_NUM
ASSERT(numRows == 5);
taos_free_result(pRes);
pRes = taos_query(taos, "select * from stb where c2 between '\\x3e' and '\\x7F8290'");
GET_ROW_NUM
ASSERT(numRows == 2);
taos_free_result(pRes);
pRes = taos_query(taos, "select * from stb where c2 not between '\\x3e' and '\\x7F8290'");
GET_ROW_NUM
ASSERT(numRows == 3);
taos_free_result(pRes);
pRes = taos_query(taos, "select ts,c2 from stb order by c2");
rowIndex = 0;
while ((row = taos_fetch_row(pRes)) != NULL) {
// int64_t ts = *(int64_t *)row[0];
if (rowIndex == 0) {
ASSERT(row[1] == NULL);
rowIndex++;
continue;
}
int32_t* length = taos_fetch_lengths(pRes);
void* data = NULL;
uint32_t size = 0;
if(taosAscii2Hex(row[1], length[1], &data, &size) < 0){
ASSERT(0);
}
if (rowIndex == 1) {
// ASSERT(ts == 1661943960000);
ASSERT(memcmp(data, "\\x", size) == 0);
}
if (rowIndex == 2) {
// ASSERT(ts == 1661943960000);
ASSERT(memcmp(data, "\\x00000000", size) == 0);
}
if (rowIndex == 3) {
// ASSERT(ts == 1661943960000);
ASSERT(memcmp(data, "\\x7661726331", size) == 0);
}
if (rowIndex == 4) {
// ASSERT(ts == 1661943960000);
ASSERT(memcmp(data, "\\x7F8290", size) == 0);
}
if (rowIndex == 5) {
// ASSERT(ts == 1661943960000);
ASSERT(memcmp(data, "\\x7F829000", size) == 0);
}
taosMemoryFree(data);
rowIndex++;
}
printf("%s result1:%s\n", __FUNCTION__, taos_errstr(pRes));
taos_free_result(pRes);
taos_close(taos);
return code;
}
int main(int argc, char *argv[]) {
int ret = 0;
ret = varbinary_test();
ASSERT(!ret);
return ret;
}