From 717651dd5fdd53fa3c7f64d32acefacf4675edca Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 6 Sep 2021 12:32:49 +0800 Subject: [PATCH 01/39] [TD-6443]: Support OpenTSDB HTTP JSON data import format --- src/client/src/tscParseOpenTSDB.c | 51 +++++++++++++++++++++++++++++++ src/inc/taoserror.h | 1 + 2 files changed, 52 insertions(+) diff --git a/src/client/src/tscParseOpenTSDB.c b/src/client/src/tscParseOpenTSDB.c index 309bbb919d..d7f39fddcc 100644 --- a/src/client/src/tscParseOpenTSDB.c +++ b/src/client/src/tscParseOpenTSDB.c @@ -421,3 +421,54 @@ int taos_telnet_insert(TAOS* taos, TAOS_SML_DATA_POINT* points, int numPoint) { tfree(info); return code; } + + +/* telnet style API parser */ +int32_t parseMetricFromJSON(cJSON *root, TAOS_SML_DATA_POINT* pSml, SSmlLinesInfo* info) { + int32_t ret = TSDB_CODE_SUCCESS; + + cJSON *metric = cJSON_GetObjectItem(root, "metric"); + if (metric == NULL || metric->type != cJSON_String) { + tscError("OTD:0x%"PRIx64" failed to parse metric from JSON Payload %s", info->id); + return TSDB_CODE_TSC_INVALID_JSON; + } + + int32_t stableLen = strlen(metric->valuestring); + if (stableLen > TSDB_TABLE_NAME_LEN) { + tscError("OTD:0x%"PRIx64" Metric cannot exceeds 193 characters", info->id); + return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; + } + + pSml->stableName = tcalloc(stableLen + 1, sizeof(char)); + if (pSml->stableName == NULL){ + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + if (isdigit(metric->valuestring[0])) { + tscError("OTD:0x%"PRIx64" Metric cannnot start with digit", info->id); + tfree(pSml->stableName); + return TSDB_CODE_TSC_INVALID_JSON; + } + + tstrncpy(pSml->stableName, metric->valuestring, stableLen); + + return TSDB_CODE_SUCCESS; + +} + +int32_t tscParseJSONPayload(const char* payload, TAOS_SML_DATA_POINT* smlData, SSmlLinesInfo* info) { + int32_t ret = TSDB_CODE_SUCCESS; + + if (payload == NULL) { + tscError("OTD:0x%"PRIx64" empty JSON Payload %s", info->id); + return TSDB_CODE_TSC_INVALID_JSON; + } + + cJSON *root = cJSON_Parse(payload); + if (root == NULL) { + tscError("OTD:0x%"PRIx64" parsing JSON Payload error%s", info->id); + return TSDB_CODE_TSC_INVALID_JSON; + } + + return ret; +} diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index 882aca2b52..ec55713030 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -107,6 +107,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_TSC_INVALID_TAG_LENGTH TAOS_DEF_ERROR_CODE(0, 0x021E) //"Invalid tag length") #define TSDB_CODE_TSC_INVALID_COLUMN_LENGTH TAOS_DEF_ERROR_CODE(0, 0x021F) //"Invalid column length") #define TSDB_CODE_TSC_DUP_TAG_NAMES TAOS_DEF_ERROR_CODE(0, 0x0220) //"duplicated tag names") +#define TSDB_CODE_TSC_INVALID_JSON TAOS_DEF_ERROR_CODE(0, 0x0221) //"Invalid json format") // mnode #define TSDB_CODE_MND_MSG_NOT_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0300) //"Message not processed") From 737d2eff81b2e9c3002a9194bc7e7a45f222af14 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 6 Sep 2021 12:32:49 +0800 Subject: [PATCH 02/39] [TD-6443]: Support OpenTSDB HTTP JSON data import format --- src/client/src/tscParseOpenTSDB.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/client/src/tscParseOpenTSDB.c b/src/client/src/tscParseOpenTSDB.c index d7f39fddcc..8fb1749eb9 100644 --- a/src/client/src/tscParseOpenTSDB.c +++ b/src/client/src/tscParseOpenTSDB.c @@ -3,6 +3,7 @@ #include #include +#include "cJSON.h" #include "hash.h" #include "taos.h" @@ -425,11 +426,9 @@ int taos_telnet_insert(TAOS* taos, TAOS_SML_DATA_POINT* points, int numPoint) { /* telnet style API parser */ int32_t parseMetricFromJSON(cJSON *root, TAOS_SML_DATA_POINT* pSml, SSmlLinesInfo* info) { - int32_t ret = TSDB_CODE_SUCCESS; - cJSON *metric = cJSON_GetObjectItem(root, "metric"); if (metric == NULL || metric->type != cJSON_String) { - tscError("OTD:0x%"PRIx64" failed to parse metric from JSON Payload %s", info->id); + tscError("OTD:0x%"PRIx64" failed to parse metric from JSON Payload", info->id); return TSDB_CODE_TSC_INVALID_JSON; } @@ -460,13 +459,13 @@ int32_t tscParseJSONPayload(const char* payload, TAOS_SML_DATA_POINT* smlData, S int32_t ret = TSDB_CODE_SUCCESS; if (payload == NULL) { - tscError("OTD:0x%"PRIx64" empty JSON Payload %s", info->id); + tscError("OTD:0x%"PRIx64" empty JSON Payload", info->id); return TSDB_CODE_TSC_INVALID_JSON; } cJSON *root = cJSON_Parse(payload); if (root == NULL) { - tscError("OTD:0x%"PRIx64" parsing JSON Payload error%s", info->id); + tscError("OTD:0x%"PRIx64" parsing JSON Payload error", info->id); return TSDB_CODE_TSC_INVALID_JSON; } From 42d2715355ac10d20b923a7c369f640062f84bde Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 6 Sep 2021 12:32:49 +0800 Subject: [PATCH 03/39] [TD-6443]: Support OpenTSDB HTTP JSON data import format --- src/client/CMakeLists.txt | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/client/CMakeLists.txt b/src/client/CMakeLists.txt index 0d06e5d39c..df0cf15fe0 100644 --- a/src/client/CMakeLists.txt +++ b/src/client/CMakeLists.txt @@ -13,13 +13,13 @@ IF (TD_LINUX) # set the static lib name ADD_LIBRARY(taos_static STATIC ${SRC}) - TARGET_LINK_LIBRARIES(taos_static common query trpc tutil pthread m rt ${VAR_TSZ}) + TARGET_LINK_LIBRARIES(taos_static common query trpc tutil pthread m rt cJson ${VAR_TSZ}) SET_TARGET_PROPERTIES(taos_static PROPERTIES OUTPUT_NAME "taos_static") SET_TARGET_PROPERTIES(taos_static PROPERTIES CLEAN_DIRECT_OUTPUT 1) # generate dynamic library (*.so) ADD_LIBRARY(taos SHARED ${SRC}) - TARGET_LINK_LIBRARIES(taos common query trpc tutil pthread m rt) + TARGET_LINK_LIBRARIES(taos common query trpc tutil pthread m rt cJson) IF (TD_LINUX_64) TARGET_LINK_LIBRARIES(taos lua) ENDIF () @@ -39,13 +39,13 @@ ELSEIF (TD_DARWIN) # set the static lib name ADD_LIBRARY(taos_static STATIC ${SRC}) - TARGET_LINK_LIBRARIES(taos_static common query trpc tutil pthread m lua) + TARGET_LINK_LIBRARIES(taos_static common query trpc tutil pthread m lua cJson) SET_TARGET_PROPERTIES(taos_static PROPERTIES OUTPUT_NAME "taos_static") SET_TARGET_PROPERTIES(taos_static PROPERTIES CLEAN_DIRECT_OUTPUT 1) # generate dynamic library (*.dylib) ADD_LIBRARY(taos SHARED ${SRC}) - TARGET_LINK_LIBRARIES(taos common query trpc tutil pthread m lua) + TARGET_LINK_LIBRARIES(taos common query trpc tutil pthread m lua cJson) SET_TARGET_PROPERTIES(taos PROPERTIES CLEAN_DIRECT_OUTPUT 1) #set version of .dylib @@ -63,26 +63,26 @@ ELSEIF (TD_WINDOWS) CONFIGURE_FILE("${TD_COMMUNITY_DIR}/src/client/src/taos.rc.in" "${TD_COMMUNITY_DIR}/src/client/src/taos.rc") ADD_LIBRARY(taos_static STATIC ${SRC}) - TARGET_LINK_LIBRARIES(taos_static trpc tutil query) + TARGET_LINK_LIBRARIES(taos_static trpc tutil query cJson) # generate dynamic library (*.dll) ADD_LIBRARY(taos SHARED ${SRC} ${TD_COMMUNITY_DIR}/src/client/src/taos.rc) IF (NOT TD_GODLL) SET_TARGET_PROPERTIES(taos PROPERTIES LINK_FLAGS /DEF:${TD_COMMUNITY_DIR}/src/client/src/taos.def) ENDIF () - TARGET_LINK_LIBRARIES(taos trpc tutil query lua) + TARGET_LINK_LIBRARIES(taos trpc tutil query lua cJson) ELSEIF (TD_DARWIN) SET(CMAKE_MACOSX_RPATH 1) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/jni/linux) ADD_LIBRARY(taos_static STATIC ${SRC}) - TARGET_LINK_LIBRARIES(taos_static query trpc tutil pthread m lua) + TARGET_LINK_LIBRARIES(taos_static query trpc tutil pthread m lua cJson) SET_TARGET_PROPERTIES(taos_static PROPERTIES OUTPUT_NAME "taos_static") # generate dynamic library (*.dylib) ADD_LIBRARY(taos SHARED ${SRC}) - TARGET_LINK_LIBRARIES(taos query trpc tutil pthread m lua) + TARGET_LINK_LIBRARIES(taos query trpc tutil pthread m lua cJson) SET_TARGET_PROPERTIES(taos PROPERTIES CLEAN_DIRECT_OUTPUT 1) From 691bab64bab73242903cff5d2e7d95eb58dde0c2 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 6 Sep 2021 12:32:49 +0800 Subject: [PATCH 04/39] [TD-6443]: Support OpenTSDB HTTP JSON data import format --- src/client/src/tscParseOpenTSDB.c | 48 ++++++++++++++++++++++++++++--- 1 file changed, 44 insertions(+), 4 deletions(-) diff --git a/src/client/src/tscParseOpenTSDB.c b/src/client/src/tscParseOpenTSDB.c index 8fb1749eb9..0e2f8ae476 100644 --- a/src/client/src/tscParseOpenTSDB.c +++ b/src/client/src/tscParseOpenTSDB.c @@ -13,7 +13,7 @@ #include "tscParseLine.h" -#define MAX_TELNET_FILEDS_NUM 2 +#define MAX_FILEDS_NUM 2 #define OTS_TIMESTAMP_COLUMN_NAME "ts" #define OTS_METRIC_VALUE_COLUMN_NAME "value" @@ -83,7 +83,7 @@ static int32_t parseTelnetTimeStamp(TAOS_SML_KV **pTS, int *num_kvs, const char start = cur = *index; //allocate fields for timestamp and value - *pTS = tcalloc(MAX_TELNET_FILEDS_NUM, sizeof(TAOS_SML_KV)); + *pTS = tcalloc(MAX_FILEDS_NUM, sizeof(TAOS_SML_KV)); while(*cur != '\0') { if (*cur == ' ') { @@ -434,7 +434,7 @@ int32_t parseMetricFromJSON(cJSON *root, TAOS_SML_DATA_POINT* pSml, SSmlLinesInf int32_t stableLen = strlen(metric->valuestring); if (stableLen > TSDB_TABLE_NAME_LEN) { - tscError("OTD:0x%"PRIx64" Metric cannot exceeds 193 characters", info->id); + tscError("OTD:0x%"PRIx64" Metric cannot exceeds 193 characters in JSON", info->id); return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; } @@ -444,7 +444,7 @@ int32_t parseMetricFromJSON(cJSON *root, TAOS_SML_DATA_POINT* pSml, SSmlLinesInf } if (isdigit(metric->valuestring[0])) { - tscError("OTD:0x%"PRIx64" Metric cannnot start with digit", info->id); + tscError("OTD:0x%"PRIx64" Metric cannnot start with digit in JSON", info->id); tfree(pSml->stableName); return TSDB_CODE_TSC_INVALID_JSON; } @@ -455,6 +455,46 @@ int32_t parseMetricFromJSON(cJSON *root, TAOS_SML_DATA_POINT* pSml, SSmlLinesInf } +int32_t parseTimestampFromJSON(cJSON *root, TAOS_SML_KV **pTS, int *num_kvs, SSmlLinesInfo* info) { + //Timestamp must be the first KV to parse + assert(*num_kvs == 0); + int64_t tsVal; + char key[] = OTS_TIMESTAMP_COLUMN_NAME; + + cJSON *timestamp = cJSON_GetObjectItem(root, "timestamp"); + if (timestamp == NULL || timestamp->type != cJSON_Number) { + tscError("OTD:0x%"PRIx64" failed to parse timestamp from JSON Payload", info->id); + return TSDB_CODE_TSC_INVALID_JSON; + } + + //allocate fields for timestamp and value + *pTS = tcalloc(MAX_FILEDS_NUM, sizeof(TAOS_SML_KV)); + + tsVal = convertTimePrecision(timestamp->valueint, TSDB_TIME_PRECISION_MICRO, TSDB_TIME_PRECISION_NANO); + + (*pTS)->key = tcalloc(sizeof(key), 1); + if ((*pTS)->key == NULL){ + tfree(*pTS); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + memcpy((*pTS)->key, key, sizeof(key)); + + (*pTS)->type = TSDB_DATA_TYPE_TIMESTAMP; + (*pTS)->length = (int16_t)tDataTypes[(*pTS)->type].bytes; + + (*pTS)->value = tcalloc((*pTS)->length, 1); + if ((*pTS)->value == NULL){ + tfree((*pTS)->key); + tfree(*pTS); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + memcpy((*pTS)->value, &tsVal, (*pTS)->length); + + *num_kvs += 1; + return TSDB_CODE_SUCCESS; + +} + int32_t tscParseJSONPayload(const char* payload, TAOS_SML_DATA_POINT* smlData, SSmlLinesInfo* info) { int32_t ret = TSDB_CODE_SUCCESS; From 4bd405f64b44b91bda6f91603d36de51d29c8f5c Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 6 Sep 2021 12:32:49 +0800 Subject: [PATCH 05/39] [TD-6443]: Support OpenTSDB HTTP JSON data import format --- src/client/src/tscParseOpenTSDB.c | 54 +++++++++++++++++++++++++++++-- 1 file changed, 51 insertions(+), 3 deletions(-) diff --git a/src/client/src/tscParseOpenTSDB.c b/src/client/src/tscParseOpenTSDB.c index 0e2f8ae476..d0891a3e1d 100644 --- a/src/client/src/tscParseOpenTSDB.c +++ b/src/client/src/tscParseOpenTSDB.c @@ -464,7 +464,7 @@ int32_t parseTimestampFromJSON(cJSON *root, TAOS_SML_KV **pTS, int *num_kvs, SSm cJSON *timestamp = cJSON_GetObjectItem(root, "timestamp"); if (timestamp == NULL || timestamp->type != cJSON_Number) { tscError("OTD:0x%"PRIx64" failed to parse timestamp from JSON Payload", info->id); - return TSDB_CODE_TSC_INVALID_JSON; + return TSDB_CODE_TSC_INVALID_JSON; } //allocate fields for timestamp and value @@ -473,7 +473,7 @@ int32_t parseTimestampFromJSON(cJSON *root, TAOS_SML_KV **pTS, int *num_kvs, SSm tsVal = convertTimePrecision(timestamp->valueint, TSDB_TIME_PRECISION_MICRO, TSDB_TIME_PRECISION_NANO); (*pTS)->key = tcalloc(sizeof(key), 1); - if ((*pTS)->key == NULL){ + if ((*pTS)->key == NULL) { tfree(*pTS); return TSDB_CODE_TSC_OUT_OF_MEMORY; } @@ -483,7 +483,7 @@ int32_t parseTimestampFromJSON(cJSON *root, TAOS_SML_KV **pTS, int *num_kvs, SSm (*pTS)->length = (int16_t)tDataTypes[(*pTS)->type].bytes; (*pTS)->value = tcalloc((*pTS)->length, 1); - if ((*pTS)->value == NULL){ + if ((*pTS)->value == NULL) { tfree((*pTS)->key); tfree(*pTS); return TSDB_CODE_TSC_OUT_OF_MEMORY; @@ -495,6 +495,54 @@ int32_t parseTimestampFromJSON(cJSON *root, TAOS_SML_KV **pTS, int *num_kvs, SSm } +int32_t parseMetricValueFromJSON(cJSON *root, TAOS_SML_KV **pKVs, int *num_kvs, SSmlLinesInfo* info) { + //skip timestamp + TAOS_SML_KV *pVal = *pKVs + 1; + char key[] = OTS_METRIC_VALUE_COLUMN_NAME; + + cJSON *metricVal = cJSON_GetObjectItem(root, "value"); + if (metricVal == NULL) { + tscError("OTD:0x%"PRIx64" failed to parse metric value from JSON Payload", info->id); + return TSDB_CODE_TSC_INVALID_JSON; + } + + switch (metricVal->type) { + case cJSON_True: + case cJSON_False: { + pVal->type = TSDB_DATA_TYPE_BOOL; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + pVal->value = tcalloc(pVal->length, 1); + *(bool *)(pVal->value) = metricVal->type ? true : false; + break; + } + case cJSON_Number: { + //convert default JSON Number type to float + pVal->type = TSDB_DATA_TYPE_FLOAT; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + pVal->value = tcalloc(pVal->length, 1); + *(float *)(pVal->value) = (float)(metricVal->valuedouble); + break; + } + case cJSON_String: { + //convert default JSON String type to nchar + pVal->type = TSDB_DATA_TYPE_NCHAR; + pVal->length = wcslen((wchar_t *)metricVal->string) * TSDB_NCHAR_SIZE; + pVal->value = tcalloc(pVal->length + 1, 1); + memcpy(pVal->value, metricVal->string, pVal->length); + break; + } + default: + return TSDB_CODE_TSC_INVALID_JSON; + } + + pVal->key = tcalloc(sizeof(key), 1); + memcpy(pVal->key, key, sizeof(key)); + + *num_kvs += 1; + return TSDB_CODE_SUCCESS; + +} + int32_t tscParseJSONPayload(const char* payload, TAOS_SML_DATA_POINT* smlData, SSmlLinesInfo* info) { int32_t ret = TSDB_CODE_SUCCESS; From 859e4719aae664d0649960ee730175fc0695c52f Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 6 Sep 2021 12:32:49 +0800 Subject: [PATCH 06/39] [TD-6443]: Support OpenTSDB HTTP JSON data import format --- src/client/src/tscParseOpenTSDB.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/client/src/tscParseOpenTSDB.c b/src/client/src/tscParseOpenTSDB.c index d0891a3e1d..7bca2d0af3 100644 --- a/src/client/src/tscParseOpenTSDB.c +++ b/src/client/src/tscParseOpenTSDB.c @@ -526,9 +526,9 @@ int32_t parseMetricValueFromJSON(cJSON *root, TAOS_SML_KV **pKVs, int *num_kvs, case cJSON_String: { //convert default JSON String type to nchar pVal->type = TSDB_DATA_TYPE_NCHAR; - pVal->length = wcslen((wchar_t *)metricVal->string) * TSDB_NCHAR_SIZE; + pVal->length = wcslen((wchar_t *)metricVal->valuestring) * TSDB_NCHAR_SIZE; pVal->value = tcalloc(pVal->length + 1, 1); - memcpy(pVal->value, metricVal->string, pVal->length); + memcpy(pVal->value, metricVal->valuestring, pVal->length); break; } default: From c0a554e019c5ea548024a23ddd8984f40a36f8ad Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 6 Sep 2021 12:32:49 +0800 Subject: [PATCH 07/39] [TD-6443]: Support OpenTSDB HTTP JSON data import format --- src/client/src/tscParseOpenTSDB.c | 73 +++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/src/client/src/tscParseOpenTSDB.c b/src/client/src/tscParseOpenTSDB.c index 7bca2d0af3..42c6f2d39a 100644 --- a/src/client/src/tscParseOpenTSDB.c +++ b/src/client/src/tscParseOpenTSDB.c @@ -543,6 +543,79 @@ int32_t parseMetricValueFromJSON(cJSON *root, TAOS_SML_KV **pKVs, int *num_kvs, } +int32_t parseTagsFromJSON(cJSON *root, TAOS_SML_KV **pKVs, int *num_kvs, char **childTableName, SSmlLinesInfo* info) { + + cJSON *tags = cJSON_GetObjectItem(root, "tags"); + if (tags == NULL || tags->type != cJSON_Object) { + return TSDB_CODE_TSC_INVALID_JSON; + } + + + cJSON *id = cJSON_GetObjectItem(root, "ID"); + if (id != NULL) { + int32_t idLen = strlen(id->string); + int32_t ret = isValidChildTableName(id->string, idLen); + if (ret) { + return ret; + } + *childTableName = tcalloc(idLen + 1, sizeof(char)); + memcpy(*childTableName, id->string, idLen); + //remove ID from tags list no case sensitive + cJSON_DeleteItemFromObject(tags, "ID"); + } + + int32_t tagNum = cJSON_GetArraySize(tags); + //at least one tag pair required + if (tagNum <= 0) { + return TSDB_CODE_TSC_INVALID_JSON; + } + + //allocate memory for tags + *pKVs = tcalloc(tagNum, sizeof(TAOS_SML_KV)); + TAOS_SML_KV *pkv = *pKVs; + + for (int32_t i = 0; i < tagNum; ++i) { + cJSON *tag = cJSON_GetArrayItem(tags, i); + //key + pkv->key = tcalloc(strlen(tag->string) + 1, sizeof(char)); + strncpy(pkv->key, tag->string, strlen(tag->string)); + //value + switch (tag->type) { + case cJSON_True: + case cJSON_False: { + pkv->type = TSDB_DATA_TYPE_BOOL; + pkv->length = (int16_t)tDataTypes[pkv->type].bytes; + pkv->value = tcalloc(pkv->length, 1); + *(bool *)(pkv->value) = tag->type ? true : false; + break; + } + case cJSON_Number: { + //convert default JSON Number type to float + pkv->type = TSDB_DATA_TYPE_FLOAT; + pkv->length = (int16_t)tDataTypes[pkv->type].bytes; + pkv->value = tcalloc(pkv->length, 1); + *(float *)(pkv->value) = (float)(tag->valuedouble); + break; + } + case cJSON_String: { + //convert default JSON String type to nchar + pkv->type = TSDB_DATA_TYPE_NCHAR; + pkv->length = wcslen((wchar_t *)tag->valuestring) * TSDB_NCHAR_SIZE; + pkv->value = tcalloc(pkv->length + 1, 1); + memcpy(pkv->value, tag->valuestring, pkv->length); + break; + } + default: + tfree(pkv->key); + return TSDB_CODE_TSC_INVALID_JSON; + } + *num_kvs += 1; + } + + return TSDB_CODE_SUCCESS; + +} + int32_t tscParseJSONPayload(const char* payload, TAOS_SML_DATA_POINT* smlData, SSmlLinesInfo* info) { int32_t ret = TSDB_CODE_SUCCESS; From 4b9b44d0eb00b5f4acf0040d43cb1882d7bd947a Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 6 Sep 2021 12:32:49 +0800 Subject: [PATCH 08/39] [TD-6443]: Support OpenTSDB HTTP JSON data import format --- src/client/src/tscParseOpenTSDB.c | 110 ++++++++++++++---------------- 1 file changed, 50 insertions(+), 60 deletions(-) diff --git a/src/client/src/tscParseOpenTSDB.c b/src/client/src/tscParseOpenTSDB.c index 42c6f2d39a..69d2316fb1 100644 --- a/src/client/src/tscParseOpenTSDB.c +++ b/src/client/src/tscParseOpenTSDB.c @@ -495,6 +495,42 @@ int32_t parseTimestampFromJSON(cJSON *root, TAOS_SML_KV **pTS, int *num_kvs, SSm } + +int32_t parseValueFromJSON(cJSON *root, TAOS_SML_KV *pVal) { + int type = root->type; + + switch (type) { + case cJSON_True: + case cJSON_False: { + pVal->type = TSDB_DATA_TYPE_BOOL; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + pVal->value = tcalloc(pVal->length, 1); + *(bool *)(pVal->value) = type ? true : false; + break; + } + case cJSON_Number: { + //convert default JSON Number type to float + pVal->type = TSDB_DATA_TYPE_FLOAT; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + pVal->value = tcalloc(pVal->length, 1); + *(float *)(pVal->value) = (float)(root->valuedouble); + break; + } + case cJSON_String: { + //convert default JSON String type to nchar + pVal->type = TSDB_DATA_TYPE_NCHAR; + pVal->length = wcslen((wchar_t *)root->valuestring) * TSDB_NCHAR_SIZE; + pVal->value = tcalloc(pVal->length + 1, 1); + memcpy(pVal->value, root->valuestring, pVal->length); + break; + } + default: + return TSDB_CODE_TSC_INVALID_JSON; + } + + return TSDB_CODE_SUCCESS; +} + int32_t parseMetricValueFromJSON(cJSON *root, TAOS_SML_KV **pKVs, int *num_kvs, SSmlLinesInfo* info) { //skip timestamp TAOS_SML_KV *pVal = *pKVs + 1; @@ -503,36 +539,12 @@ int32_t parseMetricValueFromJSON(cJSON *root, TAOS_SML_KV **pKVs, int *num_kvs, cJSON *metricVal = cJSON_GetObjectItem(root, "value"); if (metricVal == NULL) { tscError("OTD:0x%"PRIx64" failed to parse metric value from JSON Payload", info->id); - return TSDB_CODE_TSC_INVALID_JSON; + return TSDB_CODE_TSC_INVALID_JSON; } - switch (metricVal->type) { - case cJSON_True: - case cJSON_False: { - pVal->type = TSDB_DATA_TYPE_BOOL; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - pVal->value = tcalloc(pVal->length, 1); - *(bool *)(pVal->value) = metricVal->type ? true : false; - break; - } - case cJSON_Number: { - //convert default JSON Number type to float - pVal->type = TSDB_DATA_TYPE_FLOAT; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - pVal->value = tcalloc(pVal->length, 1); - *(float *)(pVal->value) = (float)(metricVal->valuedouble); - break; - } - case cJSON_String: { - //convert default JSON String type to nchar - pVal->type = TSDB_DATA_TYPE_NCHAR; - pVal->length = wcslen((wchar_t *)metricVal->valuestring) * TSDB_NCHAR_SIZE; - pVal->value = tcalloc(pVal->length + 1, 1); - memcpy(pVal->value, metricVal->valuestring, pVal->length); - break; - } - default: - return TSDB_CODE_TSC_INVALID_JSON; + int32_t ret = parseValueFromJSON(metricVal, pVal); + if (ret != TSDB_CODE_SUCCESS) { + return ret; } pVal->key = tcalloc(sizeof(key), 1); @@ -544,6 +556,7 @@ int32_t parseMetricValueFromJSON(cJSON *root, TAOS_SML_KV **pKVs, int *num_kvs, } int32_t parseTagsFromJSON(cJSON *root, TAOS_SML_KV **pKVs, int *num_kvs, char **childTableName, SSmlLinesInfo* info) { + int32_t ret; cJSON *tags = cJSON_GetObjectItem(root, "tags"); if (tags == NULL || tags->type != cJSON_Object) { @@ -554,8 +567,8 @@ int32_t parseTagsFromJSON(cJSON *root, TAOS_SML_KV **pKVs, int *num_kvs, char ** cJSON *id = cJSON_GetObjectItem(root, "ID"); if (id != NULL) { int32_t idLen = strlen(id->string); - int32_t ret = isValidChildTableName(id->string, idLen); - if (ret) { + ret = isValidChildTableName(id->string, idLen); + if (ret != TSDB_CODE_SUCCESS) { return ret; } *childTableName = tcalloc(idLen + 1, sizeof(char)); @@ -577,39 +590,16 @@ int32_t parseTagsFromJSON(cJSON *root, TAOS_SML_KV **pKVs, int *num_kvs, char ** for (int32_t i = 0; i < tagNum; ++i) { cJSON *tag = cJSON_GetArrayItem(tags, i); //key - pkv->key = tcalloc(strlen(tag->string) + 1, sizeof(char)); - strncpy(pkv->key, tag->string, strlen(tag->string)); + int32_t keyLen = strlen(tag->string); + pkv->key = tcalloc(keyLen + 1, sizeof(char)); + strncpy(pkv->key, tag->string, keyLen); //value - switch (tag->type) { - case cJSON_True: - case cJSON_False: { - pkv->type = TSDB_DATA_TYPE_BOOL; - pkv->length = (int16_t)tDataTypes[pkv->type].bytes; - pkv->value = tcalloc(pkv->length, 1); - *(bool *)(pkv->value) = tag->type ? true : false; - break; - } - case cJSON_Number: { - //convert default JSON Number type to float - pkv->type = TSDB_DATA_TYPE_FLOAT; - pkv->length = (int16_t)tDataTypes[pkv->type].bytes; - pkv->value = tcalloc(pkv->length, 1); - *(float *)(pkv->value) = (float)(tag->valuedouble); - break; - } - case cJSON_String: { - //convert default JSON String type to nchar - pkv->type = TSDB_DATA_TYPE_NCHAR; - pkv->length = wcslen((wchar_t *)tag->valuestring) * TSDB_NCHAR_SIZE; - pkv->value = tcalloc(pkv->length + 1, 1); - memcpy(pkv->value, tag->valuestring, pkv->length); - break; - } - default: - tfree(pkv->key); - return TSDB_CODE_TSC_INVALID_JSON; + ret = parseValueFromJSON(tag, pkv); + if (ret != TSDB_CODE_SUCCESS) { + return ret; } *num_kvs += 1; + pkv++; } return TSDB_CODE_SUCCESS; From 438fc24eb760371aadba1c5d1f85d58b966bafb5 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 6 Sep 2021 12:32:49 +0800 Subject: [PATCH 09/39] [TD-6443]: Support OpenTSDB HTTP JSON data import format --- src/client/src/tscParseOpenTSDB.c | 45 +++++++++++++++++++++++++++++-- 1 file changed, 43 insertions(+), 2 deletions(-) diff --git a/src/client/src/tscParseOpenTSDB.c b/src/client/src/tscParseOpenTSDB.c index 69d2316fb1..6c074872a0 100644 --- a/src/client/src/tscParseOpenTSDB.c +++ b/src/client/src/tscParseOpenTSDB.c @@ -14,6 +14,8 @@ #include "tscParseLine.h" #define MAX_FILEDS_NUM 2 +#define JSON_FIELDS_NUM 4 + #define OTS_TIMESTAMP_COLUMN_NAME "ts" #define OTS_METRIC_VALUE_COLUMN_NAME "value" @@ -606,7 +608,7 @@ int32_t parseTagsFromJSON(cJSON *root, TAOS_SML_KV **pKVs, int *num_kvs, char ** } -int32_t tscParseJSONPayload(const char* payload, TAOS_SML_DATA_POINT* smlData, SSmlLinesInfo* info) { +int32_t tscParseJSONPayload(const char* payload, TAOS_SML_DATA_POINT* pSml, SSmlLinesInfo* info) { int32_t ret = TSDB_CODE_SUCCESS; if (payload == NULL) { @@ -620,5 +622,44 @@ int32_t tscParseJSONPayload(const char* payload, TAOS_SML_DATA_POINT* smlData, S return TSDB_CODE_TSC_INVALID_JSON; } - return ret; + int32_t size = cJSON_GetArraySize(root); + //outmost json fields has to be exactly 4 + if (size != JSON_FIELDS_NUM) { + tscError("OTD:0x%"PRIx64" Invalid num of JSON fields in payload %d", info->id, size); + return TSDB_CODE_TSC_INVALID_JSON; + } + + //Parse metric + ret = parseMetricFromJSON(root, pSml, info); + if (ret != TSDB_CODE_SUCCESS) { + tscError("OTD:0x%"PRIx64" Unable to parse metric from JSON payload", info->id); + return ret; + } + tscDebug("OTD:0x%"PRIx64" Parse metric from JSON payload finished", info->id); + + //Parse timestamp + ret = parseTimestampFromJSON(root, &pSml->fields, &pSml->fieldNum, info); + if (ret) { + tscError("OTD:0x%"PRIx64" Unable to parse timestamp from JSON payload", info->id); + return ret; + } + tscDebug("OTD:0x%"PRIx64" Parse timestamp from JSON payload finished", info->id); + + //Parse metric value + ret = parseMetricValueFromJSON(root, &pSml->fields, &pSml->fieldNum, info); + if (ret) { + tscError("OTD:0x%"PRIx64" Unable to parse metric value from JSON payload", info->id); + return ret; + } + tscDebug("OTD:0x%"PRIx64" Parse metric value from JSON payload finished", info->id); + + //Parse tags + ret = parseTagsFromJSON(root, &pSml->tags, &pSml->tagNum, &pSml->childTableName, info); + if (ret) { + tscError("OTD:0x%"PRIx64" Unable to parse tags from JSON payload", info->id); + return ret; + } + tscDebug("OTD:0x%"PRIx64" Parse tags from JSON payload finished", info->id); + + return TSDB_CODE_SUCCESS; } From 5176237ec504aa511c1ce5868aa79362a36e2537 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 6 Sep 2021 12:32:49 +0800 Subject: [PATCH 10/39] [TD-6443]: Support OpenTSDB HTTP JSON data import format --- src/client/src/tscParseOpenTSDB.c | 63 +++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/src/client/src/tscParseOpenTSDB.c b/src/client/src/tscParseOpenTSDB.c index 6c074872a0..067f4dd595 100644 --- a/src/client/src/tscParseOpenTSDB.c +++ b/src/client/src/tscParseOpenTSDB.c @@ -663,3 +663,66 @@ int32_t tscParseJSONPayload(const char* payload, TAOS_SML_DATA_POINT* pSml, SSml return TSDB_CODE_SUCCESS; } + +int32_t tscParseMultiJSONPayload(char* payload, SArray* points, SSmlLinesInfo* info) { + TAOS_SML_DATA_POINT point = {0}; + int32_t code = tscParseJSONPayload(payload, &point, info); + if (code != TSDB_CODE_SUCCESS) { + tscError("OTD:0x%"PRIx64" data point line parse failed", info->id); + destroySmlDataPoint(&point); + return code; + } else { + tscDebug("OTD:0x%"PRIx64" data point line parse success", info->id); + } + + taosArrayPush(points, &point); + + return TSDB_CODE_SUCCESS; +} + +int taos_insert_json_payload(TAOS* taos, char* payload) { + int32_t code = 0; + + SSmlLinesInfo* info = tcalloc(1, sizeof(SSmlLinesInfo)); + info->id = genUID(); + + if (payload == NULL) { + tscError("OTD:0x%"PRIx64" taos_insert_json_payload payload is NULL", info->id); + code = TSDB_CODE_TSC_APP_ERROR; + return code; + } + + SArray* lpPoints = taosArrayInit(1, sizeof(TAOS_SML_DATA_POINT)); + if (lpPoints == NULL) { + tscError("OTD:0x%"PRIx64" taos_insert_json_payload failed to allocate memory", info->id); + tfree(info); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + tscDebug("OTD:0x%"PRIx64" taos_insert_telnet_lines begin inserting %d points", info->id, 1); + code = tscParseMultiJSONPayload(payload, lpPoints, info); + size_t numPoints = taosArrayGetSize(lpPoints); + + if (code != 0) { + goto cleanup; + } + + TAOS_SML_DATA_POINT* points = TARRAY_GET_START(lpPoints); + code = tscSmlInsert(taos, points, (int)numPoints, info); + if (code != 0) { + tscError("OTD:0x%"PRIx64" taos_insert_json_payload error: %s", info->id, tstrerror((code))); + } + +cleanup: + tscDebug("OTD:0x%"PRIx64" taos_insert_json_payload finish inserting 1 Point. code: %d", info->id, code); + points = TARRAY_GET_START(lpPoints); + numPoints = taosArrayGetSize(lpPoints); + for (int i=0; i Date: Mon, 6 Sep 2021 12:32:49 +0800 Subject: [PATCH 11/39] [TD-6443]: Support OpenTSDB HTTP JSON data import format --- src/inc/taos.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/inc/taos.h b/src/inc/taos.h index a71e4bf50c..edb1552b81 100644 --- a/src/inc/taos.h +++ b/src/inc/taos.h @@ -174,6 +174,8 @@ DLL_EXPORT int taos_insert_lines(TAOS* taos, char* lines[], int numLines); DLL_EXPORT int taos_insert_telnet_lines(TAOS* taos, char* lines[], int numLines); +DLL_EXPORT int taos_insert_json_payload(TAOS* taos, char* payload); + #ifdef __cplusplus } #endif From 80bb505a23a0029a167010c22240c7761e119d2c Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 6 Sep 2021 12:32:49 +0800 Subject: [PATCH 12/39] [TD-6443]: Support OpenTSDB HTTP JSON data import format --- src/client/src/tscParseOpenTSDB.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/client/src/tscParseOpenTSDB.c b/src/client/src/tscParseOpenTSDB.c index 067f4dd595..a35d289647 100644 --- a/src/client/src/tscParseOpenTSDB.c +++ b/src/client/src/tscParseOpenTSDB.c @@ -13,7 +13,7 @@ #include "tscParseLine.h" -#define MAX_FILEDS_NUM 2 +#define MAX_FIELDS_NUM 2 #define JSON_FIELDS_NUM 4 #define OTS_TIMESTAMP_COLUMN_NAME "ts" @@ -85,7 +85,7 @@ static int32_t parseTelnetTimeStamp(TAOS_SML_KV **pTS, int *num_kvs, const char start = cur = *index; //allocate fields for timestamp and value - *pTS = tcalloc(MAX_FILEDS_NUM, sizeof(TAOS_SML_KV)); + *pTS = tcalloc(MAX_FIELDS_NUM, sizeof(TAOS_SML_KV)); while(*cur != '\0') { if (*cur == ' ') { @@ -451,7 +451,7 @@ int32_t parseMetricFromJSON(cJSON *root, TAOS_SML_DATA_POINT* pSml, SSmlLinesInf return TSDB_CODE_TSC_INVALID_JSON; } - tstrncpy(pSml->stableName, metric->valuestring, stableLen); + tstrncpy(pSml->stableName, metric->valuestring, stableLen + 1); return TSDB_CODE_SUCCESS; @@ -470,7 +470,7 @@ int32_t parseTimestampFromJSON(cJSON *root, TAOS_SML_KV **pTS, int *num_kvs, SSm } //allocate fields for timestamp and value - *pTS = tcalloc(MAX_FILEDS_NUM, sizeof(TAOS_SML_KV)); + *pTS = tcalloc(MAX_FIELDS_NUM, sizeof(TAOS_SML_KV)); tsVal = convertTimePrecision(timestamp->valueint, TSDB_TIME_PRECISION_MICRO, TSDB_TIME_PRECISION_NANO); From 0075ec27fb93b8be28285ef479c429ed7b5fe51c Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 6 Sep 2021 12:32:49 +0800 Subject: [PATCH 13/39] [TD-6443]: Support OpenTSDB HTTP JSON data import format --- src/client/src/tscParseOpenTSDB.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/client/src/tscParseOpenTSDB.c b/src/client/src/tscParseOpenTSDB.c index a35d289647..c2a58e1315 100644 --- a/src/client/src/tscParseOpenTSDB.c +++ b/src/client/src/tscParseOpenTSDB.c @@ -521,7 +521,8 @@ int32_t parseValueFromJSON(cJSON *root, TAOS_SML_KV *pVal) { case cJSON_String: { //convert default JSON String type to nchar pVal->type = TSDB_DATA_TYPE_NCHAR; - pVal->length = wcslen((wchar_t *)root->valuestring) * TSDB_NCHAR_SIZE; + //pVal->length = wcslen((wchar_t *)root->valuestring) * TSDB_NCHAR_SIZE; + pVal->length = strlen(root->valuestring); pVal->value = tcalloc(pVal->length + 1, 1); memcpy(pVal->value, root->valuestring, pVal->length); break; From 51e37a9d825f10d8feeb5f4c3133ec8a02e025a4 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 6 Sep 2021 12:32:49 +0800 Subject: [PATCH 14/39] [TD-6443]: Support OpenTSDB HTTP JSON data import format --- src/client/src/tscParseOpenTSDB.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/client/src/tscParseOpenTSDB.c b/src/client/src/tscParseOpenTSDB.c index c2a58e1315..bae8b3acb1 100644 --- a/src/client/src/tscParseOpenTSDB.c +++ b/src/client/src/tscParseOpenTSDB.c @@ -559,7 +559,7 @@ int32_t parseMetricValueFromJSON(cJSON *root, TAOS_SML_KV **pKVs, int *num_kvs, } int32_t parseTagsFromJSON(cJSON *root, TAOS_SML_KV **pKVs, int *num_kvs, char **childTableName, SSmlLinesInfo* info) { - int32_t ret; + int32_t ret = TSDB_CODE_SUCCESS; cJSON *tags = cJSON_GetObjectItem(root, "tags"); if (tags == NULL || tags->type != cJSON_Object) { @@ -567,15 +567,15 @@ int32_t parseTagsFromJSON(cJSON *root, TAOS_SML_KV **pKVs, int *num_kvs, char ** } - cJSON *id = cJSON_GetObjectItem(root, "ID"); + cJSON *id = cJSON_GetObjectItem(tags, "ID"); if (id != NULL) { int32_t idLen = strlen(id->string); - ret = isValidChildTableName(id->string, idLen); + ret = isValidChildTableName(id->valuestring, idLen); if (ret != TSDB_CODE_SUCCESS) { return ret; } *childTableName = tcalloc(idLen + 1, sizeof(char)); - memcpy(*childTableName, id->string, idLen); + memcpy(*childTableName, id->valuestring, idLen); //remove ID from tags list no case sensitive cJSON_DeleteItemFromObject(tags, "ID"); } @@ -605,7 +605,7 @@ int32_t parseTagsFromJSON(cJSON *root, TAOS_SML_KV **pKVs, int *num_kvs, char ** pkv++; } - return TSDB_CODE_SUCCESS; + return ret; } From b60bab7abdf02ec25c45418bba33a631c319d7f7 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 6 Sep 2021 12:32:49 +0800 Subject: [PATCH 15/39] [TD-6443]: Support OpenTSDB HTTP JSON data import format --- src/client/src/tscParseOpenTSDB.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/src/tscParseOpenTSDB.c b/src/client/src/tscParseOpenTSDB.c index bae8b3acb1..c5ea4b8084 100644 --- a/src/client/src/tscParseOpenTSDB.c +++ b/src/client/src/tscParseOpenTSDB.c @@ -569,7 +569,7 @@ int32_t parseTagsFromJSON(cJSON *root, TAOS_SML_KV **pKVs, int *num_kvs, char ** cJSON *id = cJSON_GetObjectItem(tags, "ID"); if (id != NULL) { - int32_t idLen = strlen(id->string); + int32_t idLen = strlen(id->valuestring); ret = isValidChildTableName(id->valuestring, idLen); if (ret != TSDB_CODE_SUCCESS) { return ret; From 7ceb5fa94ef7dc5f8a87d8f83f8c28f3f9b88088 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 6 Sep 2021 12:32:49 +0800 Subject: [PATCH 16/39] [TD-6443]: Support OpenTSDB HTTP JSON data import format --- src/client/src/tscParseOpenTSDB.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/client/src/tscParseOpenTSDB.c b/src/client/src/tscParseOpenTSDB.c index c5ea4b8084..637382455f 100644 --- a/src/client/src/tscParseOpenTSDB.c +++ b/src/client/src/tscParseOpenTSDB.c @@ -567,6 +567,7 @@ int32_t parseTagsFromJSON(cJSON *root, TAOS_SML_KV **pKVs, int *num_kvs, char ** } + //only pick up the first ID value as child table name cJSON *id = cJSON_GetObjectItem(tags, "ID"); if (id != NULL) { int32_t idLen = strlen(id->valuestring); @@ -576,8 +577,11 @@ int32_t parseTagsFromJSON(cJSON *root, TAOS_SML_KV **pKVs, int *num_kvs, char ** } *childTableName = tcalloc(idLen + 1, sizeof(char)); memcpy(*childTableName, id->valuestring, idLen); - //remove ID from tags list no case sensitive - cJSON_DeleteItemFromObject(tags, "ID"); + //remove all ID fields from tags list no case sensitive + while (id != NULL) { + cJSON_DeleteItemFromObject(tags, "ID"); + id = cJSON_GetObjectItem(tags, "ID"); + } } int32_t tagNum = cJSON_GetArraySize(tags); From b39688108e16b88226f11d6d4ce74c8057815336 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 6 Sep 2021 12:32:49 +0800 Subject: [PATCH 17/39] [TD-6443]: Support OpenTSDB HTTP JSON data import format --- src/client/src/tscParseOpenTSDB.c | 53 ++++++++++++++++++++----------- 1 file changed, 34 insertions(+), 19 deletions(-) diff --git a/src/client/src/tscParseOpenTSDB.c b/src/client/src/tscParseOpenTSDB.c index 637382455f..6d0a741bde 100644 --- a/src/client/src/tscParseOpenTSDB.c +++ b/src/client/src/tscParseOpenTSDB.c @@ -613,24 +613,18 @@ int32_t parseTagsFromJSON(cJSON *root, TAOS_SML_KV **pKVs, int *num_kvs, char ** } -int32_t tscParseJSONPayload(const char* payload, TAOS_SML_DATA_POINT* pSml, SSmlLinesInfo* info) { +int32_t tscParseJSONPayload(cJSON *root, TAOS_SML_DATA_POINT* pSml, SSmlLinesInfo* info) { int32_t ret = TSDB_CODE_SUCCESS; - if (payload == NULL) { - tscError("OTD:0x%"PRIx64" empty JSON Payload", info->id); - return TSDB_CODE_TSC_INVALID_JSON; - } - - cJSON *root = cJSON_Parse(payload); - if (root == NULL) { - tscError("OTD:0x%"PRIx64" parsing JSON Payload error", info->id); + if (cJSON_IsObject(root)) { + tscError("OTD:0x%"PRIx64" data point needs to be JSON object", info->id); return TSDB_CODE_TSC_INVALID_JSON; } int32_t size = cJSON_GetArraySize(root); //outmost json fields has to be exactly 4 if (size != JSON_FIELDS_NUM) { - tscError("OTD:0x%"PRIx64" Invalid num of JSON fields in payload %d", info->id, size); + tscError("OTD:0x%"PRIx64" Invalid number of JSON fields in data point %d", info->id, size); return TSDB_CODE_TSC_INVALID_JSON; } @@ -670,17 +664,38 @@ int32_t tscParseJSONPayload(const char* payload, TAOS_SML_DATA_POINT* pSml, SSml } int32_t tscParseMultiJSONPayload(char* payload, SArray* points, SSmlLinesInfo* info) { - TAOS_SML_DATA_POINT point = {0}; - int32_t code = tscParseJSONPayload(payload, &point, info); - if (code != TSDB_CODE_SUCCESS) { - tscError("OTD:0x%"PRIx64" data point line parse failed", info->id); - destroySmlDataPoint(&point); - return code; - } else { - tscDebug("OTD:0x%"PRIx64" data point line parse success", info->id); + int32_t payloadNum, ret; + + if (payload == NULL) { + tscError("OTD:0x%"PRIx64" empty JSON Payload", info->id); + return TSDB_CODE_TSC_INVALID_JSON; } - taosArrayPush(points, &point); + cJSON *root = cJSON_Parse(payload); + //multiple data points must be sent in JSON array + if (cJSON_IsObject(root)) { + payloadNum = 1; + } else if (cJSON_IsArray(root)) { + payloadNum = cJSON_GetArraySize(root); + } else { + tscError("OTD:0x%"PRIx64" invalid JSON Payload", info->id); + return TSDB_CODE_TSC_INVALID_JSON; + } + + for (int32_t i = 0; i < payloadNum; ++i) { + TAOS_SML_DATA_POINT point = {0}; + cJSON *dataPoint = (payloadNum == 1) ? root : cJSON_GetArrayItem(root, i); + + ret = tscParseJSONPayload(dataPoint, &point, info); + if (ret != TSDB_CODE_SUCCESS) { + tscError("OTD:0x%"PRIx64" data point line parse failed", info->id); + destroySmlDataPoint(&point); + return ret; + } else { + tscDebug("OTD:0x%"PRIx64" data point line parse success", info->id); + } + taosArrayPush(points, &point); + } return TSDB_CODE_SUCCESS; } From b5455d3cdd1b04ea4ee65bd41a8c630d433e6683 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 6 Sep 2021 12:32:49 +0800 Subject: [PATCH 18/39] [TD-6443]: Support OpenTSDB HTTP JSON data import format --- src/client/src/tscParseOpenTSDB.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/src/tscParseOpenTSDB.c b/src/client/src/tscParseOpenTSDB.c index 6d0a741bde..66357dbac5 100644 --- a/src/client/src/tscParseOpenTSDB.c +++ b/src/client/src/tscParseOpenTSDB.c @@ -616,7 +616,7 @@ int32_t parseTagsFromJSON(cJSON *root, TAOS_SML_KV **pKVs, int *num_kvs, char ** int32_t tscParseJSONPayload(cJSON *root, TAOS_SML_DATA_POINT* pSml, SSmlLinesInfo* info) { int32_t ret = TSDB_CODE_SUCCESS; - if (cJSON_IsObject(root)) { + if (!cJSON_IsObject(root)) { tscError("OTD:0x%"PRIx64" data point needs to be JSON object", info->id); return TSDB_CODE_TSC_INVALID_JSON; } From 7d9a9f919e6483e298c379a7f64409253839d279 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 6 Sep 2021 12:32:49 +0800 Subject: [PATCH 19/39] [TD-6443]: Support OpenTSDB HTTP JSON data import format --- src/client/src/tscParseOpenTSDB.c | 78 ++++++++++++++++++++++++------- 1 file changed, 61 insertions(+), 17 deletions(-) diff --git a/src/client/src/tscParseOpenTSDB.c b/src/client/src/tscParseOpenTSDB.c index 66357dbac5..73aaebe6b7 100644 --- a/src/client/src/tscParseOpenTSDB.c +++ b/src/client/src/tscParseOpenTSDB.c @@ -14,6 +14,7 @@ #include "tscParseLine.h" #define MAX_FIELDS_NUM 2 +#define JSON_SUB_FIELDS_NUM 2 #define JSON_FIELDS_NUM 4 #define OTS_TIMESTAMP_COLUMN_NAME "ts" @@ -429,8 +430,7 @@ int taos_telnet_insert(TAOS* taos, TAOS_SML_DATA_POINT* points, int numPoint) { /* telnet style API parser */ int32_t parseMetricFromJSON(cJSON *root, TAOS_SML_DATA_POINT* pSml, SSmlLinesInfo* info) { cJSON *metric = cJSON_GetObjectItem(root, "metric"); - if (metric == NULL || metric->type != cJSON_String) { - tscError("OTD:0x%"PRIx64" failed to parse metric from JSON Payload", info->id); + if (cJSON_IsString(metric)) { return TSDB_CODE_TSC_INVALID_JSON; } @@ -457,6 +457,55 @@ int32_t parseMetricFromJSON(cJSON *root, TAOS_SML_DATA_POINT* pSml, SSmlLinesInf } +int32_t parseTimestampFromJSONObj(cJSON *root, int64_t *tsVal, SSmlLinesInfo* info) { + int32_t size = cJSON_GetArraySize(root); + if (size != JSON_SUB_FIELDS_NUM) { + return TSDB_CODE_TSC_INVALID_JSON; + } + + cJSON *value = cJSON_GetObjectItem(root, "value"); + if (!cJSON_IsNumber(value)) { + return TSDB_CODE_TSC_INVALID_JSON; + } + + cJSON *type = cJSON_GetObjectItem(root, "type"); + if (!cJSON_IsString(type)) { + return TSDB_CODE_TSC_INVALID_JSON; + } + + *tsVal = value->valueint; + //if timestamp value is 0 use current system time + if (*tsVal == 0) { + *tsVal = taosGetTimestampNs(); + return TSDB_CODE_SUCCESS; + } + + int32_t typeLen = strlen(type->valuestring); + if (typeLen == 1 && type->valuestring[0] == 's') { + //seconds + *tsVal = *tsVal * 1e9; + } else if (typeLen == 2 && type->valuestring[1] == 's') { + switch (type->valuestring[0]) { + case 'm': + //milliseconds + *tsVal = convertTimePrecision(*tsVal, TSDB_TIME_PRECISION_MILLI, TSDB_TIME_PRECISION_NANO); + break; + case 'u': + //microseconds + *tsVal = convertTimePrecision(*tsVal, TSDB_TIME_PRECISION_MICRO, TSDB_TIME_PRECISION_NANO); + break; + case 'n': + //nanoseconds + *tsVal = *tsVal * 1; + break; + default: + return TSDB_CODE_TSC_INVALID_JSON; + } + } + + return TSDB_CODE_SUCCESS; +} + int32_t parseTimestampFromJSON(cJSON *root, TAOS_SML_KV **pTS, int *num_kvs, SSmlLinesInfo* info) { //Timestamp must be the first KV to parse assert(*num_kvs == 0); @@ -464,32 +513,29 @@ int32_t parseTimestampFromJSON(cJSON *root, TAOS_SML_KV **pTS, int *num_kvs, SSm char key[] = OTS_TIMESTAMP_COLUMN_NAME; cJSON *timestamp = cJSON_GetObjectItem(root, "timestamp"); - if (timestamp == NULL || timestamp->type != cJSON_Number) { - tscError("OTD:0x%"PRIx64" failed to parse timestamp from JSON Payload", info->id); + if (cJSON_IsNumber(timestamp)) { + //timestamp value 0 indicates current system time + if (timestamp->valueint == 0) { + tsVal = taosGetTimestampNs(); + } else { + tsVal = convertTimePrecision(timestamp->valueint, TSDB_TIME_PRECISION_MICRO, TSDB_TIME_PRECISION_NANO); + } + } else if (cJSON_IsObject(timestamp)) { + parseTimestampFromJSONObj(root, &tsVal, info); + } else { return TSDB_CODE_TSC_INVALID_JSON; } //allocate fields for timestamp and value *pTS = tcalloc(MAX_FIELDS_NUM, sizeof(TAOS_SML_KV)); - tsVal = convertTimePrecision(timestamp->valueint, TSDB_TIME_PRECISION_MICRO, TSDB_TIME_PRECISION_NANO); (*pTS)->key = tcalloc(sizeof(key), 1); - if ((*pTS)->key == NULL) { - tfree(*pTS); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } memcpy((*pTS)->key, key, sizeof(key)); (*pTS)->type = TSDB_DATA_TYPE_TIMESTAMP; (*pTS)->length = (int16_t)tDataTypes[(*pTS)->type].bytes; - (*pTS)->value = tcalloc((*pTS)->length, 1); - if ((*pTS)->value == NULL) { - tfree((*pTS)->key); - tfree(*pTS); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } memcpy((*pTS)->value, &tsVal, (*pTS)->length); *num_kvs += 1; @@ -541,7 +587,6 @@ int32_t parseMetricValueFromJSON(cJSON *root, TAOS_SML_KV **pKVs, int *num_kvs, cJSON *metricVal = cJSON_GetObjectItem(root, "value"); if (metricVal == NULL) { - tscError("OTD:0x%"PRIx64" failed to parse metric value from JSON Payload", info->id); return TSDB_CODE_TSC_INVALID_JSON; } @@ -566,7 +611,6 @@ int32_t parseTagsFromJSON(cJSON *root, TAOS_SML_KV **pKVs, int *num_kvs, char ** return TSDB_CODE_TSC_INVALID_JSON; } - //only pick up the first ID value as child table name cJSON *id = cJSON_GetObjectItem(tags, "ID"); if (id != NULL) { From 480c621f7b27913dde2746c5ab4c2f34806603cc Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 6 Sep 2021 12:32:49 +0800 Subject: [PATCH 20/39] [TD-6443]: Support OpenTSDB HTTP JSON data import format --- src/client/src/tscParseOpenTSDB.c | 152 +++++++++++++++++++++++++++++- 1 file changed, 149 insertions(+), 3 deletions(-) diff --git a/src/client/src/tscParseOpenTSDB.c b/src/client/src/tscParseOpenTSDB.c index 73aaebe6b7..379694ede5 100644 --- a/src/client/src/tscParseOpenTSDB.c +++ b/src/client/src/tscParseOpenTSDB.c @@ -543,8 +543,147 @@ int32_t parseTimestampFromJSON(cJSON *root, TAOS_SML_KV **pTS, int *num_kvs, SSm } +int32_t convertJSONBoolValue(TAOS_SML_KV *pVal, char* typeStr, int64_t valueInt, SSmlLinesInfo* info) { + if (strcasecmp(typeStr, "bool") != 0) { + tscError("OTD:0x%"PRIx64" invalid type(%s) in JSON payload", info->id, typeStr); + return TSDB_CODE_TSC_INVALID_JSON_TYPE; + } + pVal->type = TSDB_DATA_TYPE_BOOL; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + pVal->value = tcalloc(pVal->length, 1); + *(bool *)(pVal->value) = valueInt ? true : false; -int32_t parseValueFromJSON(cJSON *root, TAOS_SML_KV *pVal) { + return TSDB_CODE_SUCCESS; +} + +int32_t parseValueFromJSONObj(cJSON *root, TAOS_SML_KV *pVal, SSmlLinesInfo* info) { + int32_t ret = TSDB_CODE_SUCCESS; + int32_t size = cJSON_GetArraySize(root); + + if (size != JSON_SUB_FIELDS_NUM) { + return TSDB_CODE_TSC_INVALID_JSON; + } + + cJSON *value = cJSON_GetObjectItem(root, "value"); + if (value == NULL) { + return TSDB_CODE_TSC_INVALID_JSON; + } + + cJSON *type = cJSON_GetObjectItem(root, "type"); + if (!cJSON_IsString(type)) { + return TSDB_CODE_TSC_INVALID_JSON; + } + + switch (value->type) { + case cJSON_True: + case cJSON_False: { + ret = convertJSONBoolValue(pVal, type->valuestring, value->valueint, info); + if (ret != TSDB_CODE_SUCCESS) { + return ret; + } + break; + } + case cJSON_Number: { + //tinyint + if (strcasecmp(type->valuestring, "i8") == 0) { + if (!IS_VALID_TINYINT(value->valueint)) { + tscError("OTD:0x%"PRIx64" JSON value(%ld) cannot fit in type(tinyint)", info->id, value->valueint); + return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; + } + pVal->type = TSDB_DATA_TYPE_TINYINT; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + pVal->value = tcalloc(pVal->length, 1); + *(int8_t *)(pVal->value) = (int8_t)(value->valueint); + return TSDB_CODE_SUCCESS; + } + //smallint + if (strcasecmp(type->valuestring, "i16") == 0) { + if (!IS_VALID_SMALLINT(value->valueint)) { + tscError("OTD:0x%"PRIx64" JSON value(%ld) cannot fit in type(smallint)", info->id, value->valueint); + return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; + } + pVal->type = TSDB_DATA_TYPE_SMALLINT; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + pVal->value = tcalloc(pVal->length, 1); + *(int16_t *)(pVal->value) = (int16_t)(value->valueint); + return TSDB_CODE_SUCCESS; + } + //int + if (strcasecmp(type->valuestring, "i32") == 0) { + if (!IS_VALID_INT(value->valueint)) { + tscError("OTD:0x%"PRIx64" JSON value(%ld) cannot fit in type(int)", info->id, value->valueint); + return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; + } + pVal->type = TSDB_DATA_TYPE_INT; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + pVal->value = tcalloc(pVal->length, 1); + *(int32_t *)(pVal->value) = (int32_t)(value->valueint); + return TSDB_CODE_SUCCESS; + } + //bigint + if (strcasecmp(type->valuestring, "i64") == 0) { + if (!IS_VALID_BIGINT(value->valueint)) { + tscError("OTD:0x%"PRIx64" JSON value(%ld) cannot fit in type(bigint)", info->id, value->valueint); + return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; + } + pVal->type = TSDB_DATA_TYPE_BIGINT; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + pVal->value = tcalloc(pVal->length, 1); + *(int64_t *)(pVal->value) = (int64_t)(value->valueint); + return TSDB_CODE_SUCCESS; + } + //float + if (strcasecmp(type->valuestring, "f32") == 0) { + if (!IS_VALID_FLOAT(value->valuedouble)) { + tscError("OTD:0x%"PRIx64" JSON value(%f) cannot fit in type(float)", info->id, value->valuedouble); + return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; + } + pVal->type = TSDB_DATA_TYPE_FLOAT; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + pVal->value = tcalloc(pVal->length, 1); + *(float *)(pVal->value) = (float)(value->valuedouble); + return TSDB_CODE_SUCCESS; + } + //double + if (strcasecmp(type->valuestring, "f64") == 0) { + if (!IS_VALID_DOUBLE(value->valuedouble)) { + tscError("OTD:0x%"PRIx64" JSON value(%f) cannot fit in type(double)", info->id, value->valuedouble); + return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; + } + pVal->type = TSDB_DATA_TYPE_DOUBLE; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + pVal->value = tcalloc(pVal->length, 1); + *(double *)(pVal->value) = (double)(value->valuedouble); + return TSDB_CODE_SUCCESS; + } + + //if reach here means type string is not supported + tscError("OTD:0x%"PRIx64" invalid type(%s) in JSON payload", info->id, type->valuestring); + return TSDB_CODE_TSC_INVALID_JSON_TYPE; + } + case cJSON_String: { + if (strcasecmp(type->valuestring, "binary") == 0) { + pVal->type = TSDB_DATA_TYPE_BINARY; + } else if (strcasecmp(type->valuestring, "nchar") == 0) { + pVal->type = TSDB_DATA_TYPE_NCHAR; + } else { + tscError("OTD:0x%"PRIx64" invalid type(%s) in JSON payload", info->id, type->valuestring); + return TSDB_CODE_TSC_INVALID_JSON_TYPE; + } + pVal->length = strlen(root->valuestring); + pVal->value = tcalloc(pVal->length + 1, 1); + memcpy(pVal->value, root->valuestring, pVal->length); + return TSDB_CODE_SUCCESS; + } + + default: + return TSDB_CODE_TSC_INVALID_JSON; + } + + return TSDB_CODE_SUCCESS; +} + +int32_t parseValueFromJSON(cJSON *root, TAOS_SML_KV *pVal, SSmlLinesInfo* info) { int type = root->type; switch (type) { @@ -573,6 +712,10 @@ int32_t parseValueFromJSON(cJSON *root, TAOS_SML_KV *pVal) { memcpy(pVal->value, root->valuestring, pVal->length); break; } + case cJSON_Object: { + parseValueFromJSONObj(root, pVal, info); + break; + } default: return TSDB_CODE_TSC_INVALID_JSON; } @@ -590,7 +733,7 @@ int32_t parseMetricValueFromJSON(cJSON *root, TAOS_SML_KV **pKVs, int *num_kvs, return TSDB_CODE_TSC_INVALID_JSON; } - int32_t ret = parseValueFromJSON(metricVal, pVal); + int32_t ret = parseValueFromJSON(metricVal, pVal, info); if (ret != TSDB_CODE_SUCCESS) { return ret; } @@ -640,12 +783,15 @@ int32_t parseTagsFromJSON(cJSON *root, TAOS_SML_KV **pKVs, int *num_kvs, char ** for (int32_t i = 0; i < tagNum; ++i) { cJSON *tag = cJSON_GetArrayItem(tags, i); + if (tag == NULL) { + return TSDB_CODE_TSC_INVALID_JSON; + } //key int32_t keyLen = strlen(tag->string); pkv->key = tcalloc(keyLen + 1, sizeof(char)); strncpy(pkv->key, tag->string, keyLen); //value - ret = parseValueFromJSON(tag, pkv); + ret = parseValueFromJSON(tag, pkv, info); if (ret != TSDB_CODE_SUCCESS) { return ret; } From 534f64f128e23b11d84008ae1bbba81c14bcac82 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 6 Sep 2021 12:32:49 +0800 Subject: [PATCH 21/39] [TD-6443]: Support OpenTSDB HTTP JSON data import format --- src/inc/taoserror.h | 3 ++- src/util/src/terror.c | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index ec55713030..18b008df4e 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -107,7 +107,8 @@ int32_t* taosGetErrno(); #define TSDB_CODE_TSC_INVALID_TAG_LENGTH TAOS_DEF_ERROR_CODE(0, 0x021E) //"Invalid tag length") #define TSDB_CODE_TSC_INVALID_COLUMN_LENGTH TAOS_DEF_ERROR_CODE(0, 0x021F) //"Invalid column length") #define TSDB_CODE_TSC_DUP_TAG_NAMES TAOS_DEF_ERROR_CODE(0, 0x0220) //"duplicated tag names") -#define TSDB_CODE_TSC_INVALID_JSON TAOS_DEF_ERROR_CODE(0, 0x0221) //"Invalid json format") +#define TSDB_CODE_TSC_INVALID_JSON TAOS_DEF_ERROR_CODE(0, 0x0221) //"Invalid JSON format") +#define TSDB_CODE_TSC_INVALID_JSON_TYPE TAOS_DEF_ERROR_CODE(0, 0x0222) //"Invalid JSON data type") // mnode #define TSDB_CODE_MND_MSG_NOT_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0300) //"Message not processed") diff --git a/src/util/src/terror.c b/src/util/src/terror.c index 8fb39cd170..e10609d97f 100644 --- a/src/util/src/terror.c +++ b/src/util/src/terror.c @@ -116,6 +116,8 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TSC_DUP_COL_NAMES, "duplicated column nam TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_TAG_LENGTH, "Invalid tag length") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_COLUMN_LENGTH, "Invalid column length") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_DUP_TAG_NAMES, "duplicated tag names") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_JSON, "Invalid JSON format") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_JSON_TYPE, "Invalid JSON data type") // mnode TAOS_DEFINE_ERROR(TSDB_CODE_MND_MSG_NOT_PROCESSED, "Message not processed") From 31b3e489c1a35291b498dfb44ec881740076c4ac Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 6 Sep 2021 12:32:49 +0800 Subject: [PATCH 22/39] [TD-6443]: Support OpenTSDB HTTP JSON data import format --- src/inc/taoserror.h | 1 + src/util/src/terror.c | 1 + 2 files changed, 2 insertions(+) diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index 18b008df4e..319aad76e9 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -109,6 +109,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_TSC_DUP_TAG_NAMES TAOS_DEF_ERROR_CODE(0, 0x0220) //"duplicated tag names") #define TSDB_CODE_TSC_INVALID_JSON TAOS_DEF_ERROR_CODE(0, 0x0221) //"Invalid JSON format") #define TSDB_CODE_TSC_INVALID_JSON_TYPE TAOS_DEF_ERROR_CODE(0, 0x0222) //"Invalid JSON data type") +#define TSDB_CODE_TSC_VALUE_OUT_OF_RANGE TAOS_DEF_ERROR_CODE(0, 0x0223) //"Value out of range") // mnode #define TSDB_CODE_MND_MSG_NOT_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0300) //"Message not processed") diff --git a/src/util/src/terror.c b/src/util/src/terror.c index e10609d97f..e3d022a6b0 100644 --- a/src/util/src/terror.c +++ b/src/util/src/terror.c @@ -118,6 +118,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_COLUMN_LENGTH, "Invalid column length TAOS_DEFINE_ERROR(TSDB_CODE_TSC_DUP_TAG_NAMES, "duplicated tag names") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_JSON, "Invalid JSON format") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_JSON_TYPE, "Invalid JSON data type") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_VALUE_OUT_OF_RANGE, "Value out of range") // mnode TAOS_DEFINE_ERROR(TSDB_CODE_MND_MSG_NOT_PROCESSED, "Message not processed") From f0e4dea1cad1a42693ebc86dd82ea598bd3b10a6 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 6 Sep 2021 12:32:49 +0800 Subject: [PATCH 23/39] [TD-6443]: Support OpenTSDB HTTP JSON data import format --- src/client/src/tscParseOpenTSDB.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/src/tscParseOpenTSDB.c b/src/client/src/tscParseOpenTSDB.c index 379694ede5..5686756dc6 100644 --- a/src/client/src/tscParseOpenTSDB.c +++ b/src/client/src/tscParseOpenTSDB.c @@ -692,7 +692,7 @@ int32_t parseValueFromJSON(cJSON *root, TAOS_SML_KV *pVal, SSmlLinesInfo* info) pVal->type = TSDB_DATA_TYPE_BOOL; pVal->length = (int16_t)tDataTypes[pVal->type].bytes; pVal->value = tcalloc(pVal->length, 1); - *(bool *)(pVal->value) = type ? true : false; + *(bool *)(pVal->value) = root->valueint ? true : false; break; } case cJSON_Number: { From 336e82b4f67f4af05032ae3900c63fd79e3068da Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 6 Sep 2021 12:32:49 +0800 Subject: [PATCH 24/39] [TD-6443]: Support OpenTSDB HTTP JSON data import format --- src/client/src/tscParseOpenTSDB.c | 180 ++++++++++++++++-------------- 1 file changed, 94 insertions(+), 86 deletions(-) diff --git a/src/client/src/tscParseOpenTSDB.c b/src/client/src/tscParseOpenTSDB.c index 5686756dc6..7d05b0fca4 100644 --- a/src/client/src/tscParseOpenTSDB.c +++ b/src/client/src/tscParseOpenTSDB.c @@ -543,17 +543,96 @@ int32_t parseTimestampFromJSON(cJSON *root, TAOS_SML_KV **pTS, int *num_kvs, SSm } -int32_t convertJSONBoolValue(TAOS_SML_KV *pVal, char* typeStr, int64_t valueInt, SSmlLinesInfo* info) { - if (strcasecmp(typeStr, "bool") != 0) { - tscError("OTD:0x%"PRIx64" invalid type(%s) in JSON payload", info->id, typeStr); - return TSDB_CODE_TSC_INVALID_JSON_TYPE; - } - pVal->type = TSDB_DATA_TYPE_BOOL; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - pVal->value = tcalloc(pVal->length, 1); - *(bool *)(pVal->value) = valueInt ? true : false; +int32_t convertJSONBool(TAOS_SML_KV *pVal, char* typeStr, int64_t valueInt, SSmlLinesInfo* info) { + if (strcasecmp(typeStr, "bool") != 0) { + tscError("OTD:0x%"PRIx64" invalid type(%s) for JSON Bool", info->id, typeStr); + return TSDB_CODE_TSC_INVALID_JSON_TYPE; + } + pVal->type = TSDB_DATA_TYPE_BOOL; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + pVal->value = tcalloc(pVal->length, 1); + *(bool *)(pVal->value) = valueInt ? true : false; - return TSDB_CODE_SUCCESS; + return TSDB_CODE_SUCCESS; +} + +int32_t convertJSONNumber(TAOS_SML_KV *pVal, char* typeStr, cJSON *value, SSmlLinesInfo* info) { + //tinyint + if (strcasecmp(typeStr, "i8") == 0) { + if (!IS_VALID_TINYINT(value->valueint)) { + tscError("OTD:0x%"PRIx64" JSON value(%ld) cannot fit in type(tinyint)", info->id, value->valueint); + return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; + } + pVal->type = TSDB_DATA_TYPE_TINYINT; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + pVal->value = tcalloc(pVal->length, 1); + *(int8_t *)(pVal->value) = (int8_t)(value->valueint); + return TSDB_CODE_SUCCESS; + } + //smallint + if (strcasecmp(typeStr, "i16") == 0) { + if (!IS_VALID_SMALLINT(value->valueint)) { + tscError("OTD:0x%"PRIx64" JSON value(%ld) cannot fit in type(smallint)", info->id, value->valueint); + return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; + } + pVal->type = TSDB_DATA_TYPE_SMALLINT; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + pVal->value = tcalloc(pVal->length, 1); + *(int16_t *)(pVal->value) = (int16_t)(value->valueint); + return TSDB_CODE_SUCCESS; + } + //int + if (strcasecmp(typeStr, "i32") == 0) { + if (!IS_VALID_INT(value->valueint)) { + tscError("OTD:0x%"PRIx64" JSON value(%ld) cannot fit in type(int)", info->id, value->valueint); + return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; + } + pVal->type = TSDB_DATA_TYPE_INT; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + pVal->value = tcalloc(pVal->length, 1); + *(int32_t *)(pVal->value) = (int32_t)(value->valueint); + return TSDB_CODE_SUCCESS; + } + //bigint + if (strcasecmp(typeStr, "i64") == 0) { + if (!IS_VALID_BIGINT(value->valueint)) { + tscError("OTD:0x%"PRIx64" JSON value(%ld) cannot fit in type(bigint)", info->id, value->valueint); + return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; + } + pVal->type = TSDB_DATA_TYPE_BIGINT; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + pVal->value = tcalloc(pVal->length, 1); + *(int64_t *)(pVal->value) = (int64_t)(value->valueint); + return TSDB_CODE_SUCCESS; + } + //float + if (strcasecmp(typeStr, "f32") == 0) { + if (!IS_VALID_FLOAT(value->valuedouble)) { + tscError("OTD:0x%"PRIx64" JSON value(%f) cannot fit in type(float)", info->id, value->valuedouble); + return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; + } + pVal->type = TSDB_DATA_TYPE_FLOAT; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + pVal->value = tcalloc(pVal->length, 1); + *(float *)(pVal->value) = (float)(value->valuedouble); + return TSDB_CODE_SUCCESS; + } + //double + if (strcasecmp(typeStr, "f64") == 0) { + if (!IS_VALID_DOUBLE(value->valuedouble)) { + tscError("OTD:0x%"PRIx64" JSON value(%f) cannot fit in type(double)", info->id, value->valuedouble); + return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; + } + pVal->type = TSDB_DATA_TYPE_DOUBLE; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + pVal->value = tcalloc(pVal->length, 1); + *(double *)(pVal->value) = (double)(value->valuedouble); + return TSDB_CODE_SUCCESS; + } + + //if reach here means type is unsupported + tscError("OTD:0x%"PRIx64" invalid type(%s) for JSON number", info->id, typeStr); + return TSDB_CODE_TSC_INVALID_JSON_TYPE; } int32_t parseValueFromJSONObj(cJSON *root, TAOS_SML_KV *pVal, SSmlLinesInfo* info) { @@ -577,89 +656,18 @@ int32_t parseValueFromJSONObj(cJSON *root, TAOS_SML_KV *pVal, SSmlLinesInfo* inf switch (value->type) { case cJSON_True: case cJSON_False: { - ret = convertJSONBoolValue(pVal, type->valuestring, value->valueint, info); + ret = convertJSONBool(pVal, type->valuestring, value->valueint, info); if (ret != TSDB_CODE_SUCCESS) { return ret; } break; } case cJSON_Number: { - //tinyint - if (strcasecmp(type->valuestring, "i8") == 0) { - if (!IS_VALID_TINYINT(value->valueint)) { - tscError("OTD:0x%"PRIx64" JSON value(%ld) cannot fit in type(tinyint)", info->id, value->valueint); - return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; - } - pVal->type = TSDB_DATA_TYPE_TINYINT; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - pVal->value = tcalloc(pVal->length, 1); - *(int8_t *)(pVal->value) = (int8_t)(value->valueint); - return TSDB_CODE_SUCCESS; + ret = convertJSONNumber(pVal, type->valuestring, value, info); + if (ret != TSDB_CODE_SUCCESS) { + return ret; } - //smallint - if (strcasecmp(type->valuestring, "i16") == 0) { - if (!IS_VALID_SMALLINT(value->valueint)) { - tscError("OTD:0x%"PRIx64" JSON value(%ld) cannot fit in type(smallint)", info->id, value->valueint); - return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; - } - pVal->type = TSDB_DATA_TYPE_SMALLINT; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - pVal->value = tcalloc(pVal->length, 1); - *(int16_t *)(pVal->value) = (int16_t)(value->valueint); - return TSDB_CODE_SUCCESS; - } - //int - if (strcasecmp(type->valuestring, "i32") == 0) { - if (!IS_VALID_INT(value->valueint)) { - tscError("OTD:0x%"PRIx64" JSON value(%ld) cannot fit in type(int)", info->id, value->valueint); - return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; - } - pVal->type = TSDB_DATA_TYPE_INT; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - pVal->value = tcalloc(pVal->length, 1); - *(int32_t *)(pVal->value) = (int32_t)(value->valueint); - return TSDB_CODE_SUCCESS; - } - //bigint - if (strcasecmp(type->valuestring, "i64") == 0) { - if (!IS_VALID_BIGINT(value->valueint)) { - tscError("OTD:0x%"PRIx64" JSON value(%ld) cannot fit in type(bigint)", info->id, value->valueint); - return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; - } - pVal->type = TSDB_DATA_TYPE_BIGINT; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - pVal->value = tcalloc(pVal->length, 1); - *(int64_t *)(pVal->value) = (int64_t)(value->valueint); - return TSDB_CODE_SUCCESS; - } - //float - if (strcasecmp(type->valuestring, "f32") == 0) { - if (!IS_VALID_FLOAT(value->valuedouble)) { - tscError("OTD:0x%"PRIx64" JSON value(%f) cannot fit in type(float)", info->id, value->valuedouble); - return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; - } - pVal->type = TSDB_DATA_TYPE_FLOAT; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - pVal->value = tcalloc(pVal->length, 1); - *(float *)(pVal->value) = (float)(value->valuedouble); - return TSDB_CODE_SUCCESS; - } - //double - if (strcasecmp(type->valuestring, "f64") == 0) { - if (!IS_VALID_DOUBLE(value->valuedouble)) { - tscError("OTD:0x%"PRIx64" JSON value(%f) cannot fit in type(double)", info->id, value->valuedouble); - return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; - } - pVal->type = TSDB_DATA_TYPE_DOUBLE; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - pVal->value = tcalloc(pVal->length, 1); - *(double *)(pVal->value) = (double)(value->valuedouble); - return TSDB_CODE_SUCCESS; - } - - //if reach here means type string is not supported - tscError("OTD:0x%"PRIx64" invalid type(%s) in JSON payload", info->id, type->valuestring); - return TSDB_CODE_TSC_INVALID_JSON_TYPE; + break; } case cJSON_String: { if (strcasecmp(type->valuestring, "binary") == 0) { From 5b90d7d2a516dcb8821188af85445e721761f955 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 6 Sep 2021 12:32:49 +0800 Subject: [PATCH 25/39] [TD-6443]: Support OpenTSDB HTTP JSON data import format --- src/client/src/tscParseOpenTSDB.c | 35 ++++++++++++++++++------------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/src/client/src/tscParseOpenTSDB.c b/src/client/src/tscParseOpenTSDB.c index 7d05b0fca4..2d2587960f 100644 --- a/src/client/src/tscParseOpenTSDB.c +++ b/src/client/src/tscParseOpenTSDB.c @@ -631,10 +631,25 @@ int32_t convertJSONNumber(TAOS_SML_KV *pVal, char* typeStr, cJSON *value, SSmlLi } //if reach here means type is unsupported - tscError("OTD:0x%"PRIx64" invalid type(%s) for JSON number", info->id, typeStr); + tscError("OTD:0x%"PRIx64" invalid type(%s) for JSON Number", info->id, typeStr); return TSDB_CODE_TSC_INVALID_JSON_TYPE; } +int32_t convertJSONString(TAOS_SML_KV *pVal, char* typeStr, cJSON *value, SSmlLinesInfo* info) { + if (strcasecmp(typeStr, "binary") == 0) { + pVal->type = TSDB_DATA_TYPE_BINARY; + } else if (strcasecmp(typeStr, "nchar") == 0) { + pVal->type = TSDB_DATA_TYPE_NCHAR; + } else { + tscError("OTD:0x%"PRIx64" invalid type(%s) for JSON String", info->id, typeStr); + return TSDB_CODE_TSC_INVALID_JSON_TYPE; + } + pVal->length = strlen(value->valuestring); + pVal->value = tcalloc(pVal->length + 1, 1); + memcpy(pVal->value, value->valuestring, pVal->length); + return TSDB_CODE_SUCCESS; +} + int32_t parseValueFromJSONObj(cJSON *root, TAOS_SML_KV *pVal, SSmlLinesInfo* info) { int32_t ret = TSDB_CODE_SUCCESS; int32_t size = cJSON_GetArraySize(root); @@ -670,22 +685,14 @@ int32_t parseValueFromJSONObj(cJSON *root, TAOS_SML_KV *pVal, SSmlLinesInfo* inf break; } case cJSON_String: { - if (strcasecmp(type->valuestring, "binary") == 0) { - pVal->type = TSDB_DATA_TYPE_BINARY; - } else if (strcasecmp(type->valuestring, "nchar") == 0) { - pVal->type = TSDB_DATA_TYPE_NCHAR; - } else { - tscError("OTD:0x%"PRIx64" invalid type(%s) in JSON payload", info->id, type->valuestring); - return TSDB_CODE_TSC_INVALID_JSON_TYPE; + ret = convertJSONString(pVal, type->valuestring, value, info); + if (ret != TSDB_CODE_SUCCESS) { + return ret; } - pVal->length = strlen(root->valuestring); - pVal->value = tcalloc(pVal->length + 1, 1); - memcpy(pVal->value, root->valuestring, pVal->length); - return TSDB_CODE_SUCCESS; + break; } - default: - return TSDB_CODE_TSC_INVALID_JSON; + return TSDB_CODE_TSC_INVALID_JSON_TYPE; } return TSDB_CODE_SUCCESS; From 65013096a3aa654a7bdfe30134fbac88ebc5dbe7 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 6 Sep 2021 12:32:49 +0800 Subject: [PATCH 26/39] [TD-6443]: Support OpenTSDB HTTP JSON data import format --- src/client/src/tscParseLineProtocol.c | 9 +++++---- src/client/src/tscParseOpenTSDB.c | 5 +++-- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/client/src/tscParseLineProtocol.c b/src/client/src/tscParseLineProtocol.c index c9ad7361ef..e26e439492 100644 --- a/src/client/src/tscParseLineProtocol.c +++ b/src/client/src/tscParseLineProtocol.c @@ -2128,11 +2128,12 @@ int32_t tscParseLines(char* lines[], int numLines, SArray* points, SArray* faile int taos_insert_lines(TAOS* taos, char* lines[], int numLines) { int32_t code = 0; - SSmlLinesInfo* info = calloc(1, sizeof(SSmlLinesInfo)); + SSmlLinesInfo* info = tcalloc(1, sizeof(SSmlLinesInfo)); info->id = genLinesSmlId(); if (numLines <= 0 || numLines > 65536) { tscError("SML:0x%"PRIx64" taos_insert_lines numLines should be between 1 and 65536. numLines: %d", info->id, numLines); + tfree(info); code = TSDB_CODE_TSC_APP_ERROR; return code; } @@ -2140,7 +2141,7 @@ int taos_insert_lines(TAOS* taos, char* lines[], int numLines) { for (int i = 0; i < numLines; ++i) { if (lines[i] == NULL) { tscError("SML:0x%"PRIx64" taos_insert_lines line %d is NULL", info->id, i); - free(info); + tfree(info); code = TSDB_CODE_TSC_APP_ERROR; return code; } @@ -2149,7 +2150,7 @@ int taos_insert_lines(TAOS* taos, char* lines[], int numLines) { SArray* lpPoints = taosArrayInit(numLines, sizeof(TAOS_SML_DATA_POINT)); if (lpPoints == NULL) { tscError("SML:0x%"PRIx64" taos_insert_lines failed to allocate memory", info->id); - free(info); + tfree(info); return TSDB_CODE_TSC_OUT_OF_MEMORY; } @@ -2177,7 +2178,7 @@ cleanup: taosArrayDestroy(lpPoints); - free(info); + tfree(info); return code; } diff --git a/src/client/src/tscParseOpenTSDB.c b/src/client/src/tscParseOpenTSDB.c index 2d2587960f..7683ac0a56 100644 --- a/src/client/src/tscParseOpenTSDB.c +++ b/src/client/src/tscParseOpenTSDB.c @@ -893,11 +893,11 @@ int32_t tscParseMultiJSONPayload(char* payload, SArray* points, SSmlLinesInfo* i ret = tscParseJSONPayload(dataPoint, &point, info); if (ret != TSDB_CODE_SUCCESS) { - tscError("OTD:0x%"PRIx64" data point line parse failed", info->id); + tscError("OTD:0x%"PRIx64" JSON data point parse failed", info->id); destroySmlDataPoint(&point); return ret; } else { - tscDebug("OTD:0x%"PRIx64" data point line parse success", info->id); + tscDebug("OTD:0x%"PRIx64" JSON data point parse success", info->id); } taosArrayPush(points, &point); } @@ -913,6 +913,7 @@ int taos_insert_json_payload(TAOS* taos, char* payload) { if (payload == NULL) { tscError("OTD:0x%"PRIx64" taos_insert_json_payload payload is NULL", info->id); + tfree(info); code = TSDB_CODE_TSC_APP_ERROR; return code; } From 9202a382016dd0aefbf2869e1bfb0e6c4b2e1efb Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 6 Sep 2021 12:32:49 +0800 Subject: [PATCH 27/39] [TD-6443]: Support OpenTSDB HTTP JSON data import format --- src/client/src/tscParseOpenTSDB.c | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/client/src/tscParseOpenTSDB.c b/src/client/src/tscParseOpenTSDB.c index 7683ac0a56..891af02b4f 100644 --- a/src/client/src/tscParseOpenTSDB.c +++ b/src/client/src/tscParseOpenTSDB.c @@ -430,7 +430,7 @@ int taos_telnet_insert(TAOS* taos, TAOS_SML_DATA_POINT* points, int numPoint) { /* telnet style API parser */ int32_t parseMetricFromJSON(cJSON *root, TAOS_SML_DATA_POINT* pSml, SSmlLinesInfo* info) { cJSON *metric = cJSON_GetObjectItem(root, "metric"); - if (cJSON_IsString(metric)) { + if (!cJSON_IsString(metric)) { return TSDB_CODE_TSC_INVALID_JSON; } @@ -521,7 +521,11 @@ int32_t parseTimestampFromJSON(cJSON *root, TAOS_SML_KV **pTS, int *num_kvs, SSm tsVal = convertTimePrecision(timestamp->valueint, TSDB_TIME_PRECISION_MICRO, TSDB_TIME_PRECISION_NANO); } } else if (cJSON_IsObject(timestamp)) { - parseTimestampFromJSONObj(root, &tsVal, info); + int32_t ret = parseTimestampFromJSONObj(root, &tsVal, info); + if (ret != TSDB_CODE_SUCCESS) { + tscError("OTD:0x%"PRIx64" Failed to parse timestamp from JSON Obj", info->id); + return ret; + } } else { return TSDB_CODE_TSC_INVALID_JSON; } @@ -728,7 +732,11 @@ int32_t parseValueFromJSON(cJSON *root, TAOS_SML_KV *pVal, SSmlLinesInfo* info) break; } case cJSON_Object: { - parseValueFromJSONObj(root, pVal, info); + int32_t ret = parseValueFromJSONObj(root, pVal, info); + if (ret != TSDB_CODE_SUCCESS) { + tscError("OTD:0x%"PRIx64" Failed to parse timestamp from JSON Obj", info->id); + return ret; + } break; } default: @@ -883,7 +891,7 @@ int32_t tscParseMultiJSONPayload(char* payload, SArray* points, SSmlLinesInfo* i } else if (cJSON_IsArray(root)) { payloadNum = cJSON_GetArraySize(root); } else { - tscError("OTD:0x%"PRIx64" invalid JSON Payload", info->id); + tscError("OTD:0x%"PRIx64" Invalid JSON Payload", info->id); return TSDB_CODE_TSC_INVALID_JSON; } From a329b9fc99e68a4909727562e3b2023194e96e8d Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 6 Sep 2021 12:32:49 +0800 Subject: [PATCH 28/39] [TD-6443]: Support OpenTSDB HTTP JSON data import format --- src/client/src/tscParseOpenTSDB.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/src/tscParseOpenTSDB.c b/src/client/src/tscParseOpenTSDB.c index 891af02b4f..2b9390a01c 100644 --- a/src/client/src/tscParseOpenTSDB.c +++ b/src/client/src/tscParseOpenTSDB.c @@ -521,7 +521,7 @@ int32_t parseTimestampFromJSON(cJSON *root, TAOS_SML_KV **pTS, int *num_kvs, SSm tsVal = convertTimePrecision(timestamp->valueint, TSDB_TIME_PRECISION_MICRO, TSDB_TIME_PRECISION_NANO); } } else if (cJSON_IsObject(timestamp)) { - int32_t ret = parseTimestampFromJSONObj(root, &tsVal, info); + int32_t ret = parseTimestampFromJSONObj(timestamp, &tsVal, info); if (ret != TSDB_CODE_SUCCESS) { tscError("OTD:0x%"PRIx64" Failed to parse timestamp from JSON Obj", info->id); return ret; From 7745b6b5739ddd97cea1a2a1561997c7b54573af Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 6 Sep 2021 12:32:49 +0800 Subject: [PATCH 29/39] [TD-6443]: Support OpenTSDB HTTP JSON data import format --- src/client/src/tscParseOpenTSDB.c | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/src/client/src/tscParseOpenTSDB.c b/src/client/src/tscParseOpenTSDB.c index 2b9390a01c..8c70b83958 100644 --- a/src/client/src/tscParseOpenTSDB.c +++ b/src/client/src/tscParseOpenTSDB.c @@ -562,7 +562,8 @@ int32_t convertJSONBool(TAOS_SML_KV *pVal, char* typeStr, int64_t valueInt, SSml int32_t convertJSONNumber(TAOS_SML_KV *pVal, char* typeStr, cJSON *value, SSmlLinesInfo* info) { //tinyint - if (strcasecmp(typeStr, "i8") == 0) { + if (strcasecmp(typeStr, "i8") == 0 || + strcasecmp(typeStr, "tinyint") == 0) { if (!IS_VALID_TINYINT(value->valueint)) { tscError("OTD:0x%"PRIx64" JSON value(%ld) cannot fit in type(tinyint)", info->id, value->valueint); return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; @@ -574,7 +575,8 @@ int32_t convertJSONNumber(TAOS_SML_KV *pVal, char* typeStr, cJSON *value, SSmlLi return TSDB_CODE_SUCCESS; } //smallint - if (strcasecmp(typeStr, "i16") == 0) { + if (strcasecmp(typeStr, "i16") == 0 || + strcasecmp(typeStr, "smallint") == 0) { if (!IS_VALID_SMALLINT(value->valueint)) { tscError("OTD:0x%"PRIx64" JSON value(%ld) cannot fit in type(smallint)", info->id, value->valueint); return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; @@ -586,7 +588,8 @@ int32_t convertJSONNumber(TAOS_SML_KV *pVal, char* typeStr, cJSON *value, SSmlLi return TSDB_CODE_SUCCESS; } //int - if (strcasecmp(typeStr, "i32") == 0) { + if (strcasecmp(typeStr, "i32") == 0 || + strcasecmp(typeStr, "int") == 0) { if (!IS_VALID_INT(value->valueint)) { tscError("OTD:0x%"PRIx64" JSON value(%ld) cannot fit in type(int)", info->id, value->valueint); return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; @@ -598,7 +601,8 @@ int32_t convertJSONNumber(TAOS_SML_KV *pVal, char* typeStr, cJSON *value, SSmlLi return TSDB_CODE_SUCCESS; } //bigint - if (strcasecmp(typeStr, "i64") == 0) { + if (strcasecmp(typeStr, "i64") == 0 || + strcasecmp(typeStr, "bigint") == 0) { if (!IS_VALID_BIGINT(value->valueint)) { tscError("OTD:0x%"PRIx64" JSON value(%ld) cannot fit in type(bigint)", info->id, value->valueint); return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; @@ -610,7 +614,8 @@ int32_t convertJSONNumber(TAOS_SML_KV *pVal, char* typeStr, cJSON *value, SSmlLi return TSDB_CODE_SUCCESS; } //float - if (strcasecmp(typeStr, "f32") == 0) { + if (strcasecmp(typeStr, "f32") == 0 || + strcasecmp(typeStr, "float") == 0) { if (!IS_VALID_FLOAT(value->valuedouble)) { tscError("OTD:0x%"PRIx64" JSON value(%f) cannot fit in type(float)", info->id, value->valuedouble); return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; @@ -622,7 +627,8 @@ int32_t convertJSONNumber(TAOS_SML_KV *pVal, char* typeStr, cJSON *value, SSmlLi return TSDB_CODE_SUCCESS; } //double - if (strcasecmp(typeStr, "f64") == 0) { + if (strcasecmp(typeStr, "f64") == 0 || + strcasecmp(typeStr, "double") == 0) { if (!IS_VALID_DOUBLE(value->valuedouble)) { tscError("OTD:0x%"PRIx64" JSON value(%f) cannot fit in type(double)", info->id, value->valuedouble); return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; From 4a72753452e06d2eb5a511a844b5e561db8cd62c Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 6 Sep 2021 12:32:49 +0800 Subject: [PATCH 30/39] [TD-6443]: Support OpenTSDB HTTP JSON data import format --- src/client/src/tscParseOpenTSDB.c | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/src/client/src/tscParseOpenTSDB.c b/src/client/src/tscParseOpenTSDB.c index 8c70b83958..ab3fa8e86e 100644 --- a/src/client/src/tscParseOpenTSDB.c +++ b/src/client/src/tscParseOpenTSDB.c @@ -13,12 +13,12 @@ #include "tscParseLine.h" -#define MAX_FIELDS_NUM 2 -#define JSON_SUB_FIELDS_NUM 2 -#define JSON_FIELDS_NUM 4 +#define OTD_MAX_FIELDS_NUM 2 +#define OTD_JSON_SUB_FIELDS_NUM 2 +#define OTD_JSON_FIELDS_NUM 4 -#define OTS_TIMESTAMP_COLUMN_NAME "ts" -#define OTS_METRIC_VALUE_COLUMN_NAME "value" +#define OTD_TIMESTAMP_COLUMN_NAME "ts" +#define OTD_METRIC_VALUE_COLUMN_NAME "value" /* telnet style API parser */ static uint64_t HandleId = 0; @@ -81,12 +81,12 @@ static int32_t parseTelnetTimeStamp(TAOS_SML_KV **pTS, int *num_kvs, const char const char *start, *cur; int32_t ret = TSDB_CODE_SUCCESS; int len = 0; - char key[] = OTS_TIMESTAMP_COLUMN_NAME; + char key[] = OTD_TIMESTAMP_COLUMN_NAME; char *value = NULL; start = cur = *index; //allocate fields for timestamp and value - *pTS = tcalloc(MAX_FIELDS_NUM, sizeof(TAOS_SML_KV)); + *pTS = tcalloc(OTD_MAX_FIELDS_NUM, sizeof(TAOS_SML_KV)); while(*cur != '\0') { if (*cur == ' ') { @@ -127,7 +127,7 @@ static int32_t parseTelnetMetricValue(TAOS_SML_KV **pKVs, int *num_kvs, const ch const char *start, *cur; int32_t ret = TSDB_CODE_SUCCESS; int len = 0; - char key[] = OTS_METRIC_VALUE_COLUMN_NAME; + char key[] = OTD_METRIC_VALUE_COLUMN_NAME; char *value = NULL; start = cur = *index; @@ -459,7 +459,7 @@ int32_t parseMetricFromJSON(cJSON *root, TAOS_SML_DATA_POINT* pSml, SSmlLinesInf int32_t parseTimestampFromJSONObj(cJSON *root, int64_t *tsVal, SSmlLinesInfo* info) { int32_t size = cJSON_GetArraySize(root); - if (size != JSON_SUB_FIELDS_NUM) { + if (size != OTD_JSON_SUB_FIELDS_NUM) { return TSDB_CODE_TSC_INVALID_JSON; } @@ -510,7 +510,7 @@ int32_t parseTimestampFromJSON(cJSON *root, TAOS_SML_KV **pTS, int *num_kvs, SSm //Timestamp must be the first KV to parse assert(*num_kvs == 0); int64_t tsVal; - char key[] = OTS_TIMESTAMP_COLUMN_NAME; + char key[] = OTD_TIMESTAMP_COLUMN_NAME; cJSON *timestamp = cJSON_GetObjectItem(root, "timestamp"); if (cJSON_IsNumber(timestamp)) { @@ -531,7 +531,7 @@ int32_t parseTimestampFromJSON(cJSON *root, TAOS_SML_KV **pTS, int *num_kvs, SSm } //allocate fields for timestamp and value - *pTS = tcalloc(MAX_FIELDS_NUM, sizeof(TAOS_SML_KV)); + *pTS = tcalloc(OTD_MAX_FIELDS_NUM, sizeof(TAOS_SML_KV)); (*pTS)->key = tcalloc(sizeof(key), 1); @@ -664,7 +664,7 @@ int32_t parseValueFromJSONObj(cJSON *root, TAOS_SML_KV *pVal, SSmlLinesInfo* inf int32_t ret = TSDB_CODE_SUCCESS; int32_t size = cJSON_GetArraySize(root); - if (size != JSON_SUB_FIELDS_NUM) { + if (size != OTD_JSON_SUB_FIELDS_NUM) { return TSDB_CODE_TSC_INVALID_JSON; } @@ -755,7 +755,7 @@ int32_t parseValueFromJSON(cJSON *root, TAOS_SML_KV *pVal, SSmlLinesInfo* info) int32_t parseMetricValueFromJSON(cJSON *root, TAOS_SML_KV **pKVs, int *num_kvs, SSmlLinesInfo* info) { //skip timestamp TAOS_SML_KV *pVal = *pKVs + 1; - char key[] = OTS_METRIC_VALUE_COLUMN_NAME; + char key[] = OTD_METRIC_VALUE_COLUMN_NAME; cJSON *metricVal = cJSON_GetObjectItem(root, "value"); if (metricVal == NULL) { @@ -842,7 +842,7 @@ int32_t tscParseJSONPayload(cJSON *root, TAOS_SML_DATA_POINT* pSml, SSmlLinesInf int32_t size = cJSON_GetArraySize(root); //outmost json fields has to be exactly 4 - if (size != JSON_FIELDS_NUM) { + if (size != OTD_JSON_FIELDS_NUM) { tscError("OTD:0x%"PRIx64" Invalid number of JSON fields in data point %d", info->id, size); return TSDB_CODE_TSC_INVALID_JSON; } From 0601017b6405ce9b9e4243076d2912a0d945a080 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 6 Sep 2021 12:32:49 +0800 Subject: [PATCH 31/39] [TD-6443]: Support OpenTSDB HTTP JSON data import format --- src/client/src/tscParseOpenTSDB.c | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/client/src/tscParseOpenTSDB.c b/src/client/src/tscParseOpenTSDB.c index ab3fa8e86e..f010a99bdc 100644 --- a/src/client/src/tscParseOpenTSDB.c +++ b/src/client/src/tscParseOpenTSDB.c @@ -408,7 +408,7 @@ cleanup: tscDebug("OTD:0x%"PRIx64" taos_insert_telnet_lines finish inserting %d lines. code: %d", info->id, numLines, code); points = TARRAY_GET_START(lpPoints); numPoints = taosArrayGetSize(lpPoints); - for (int i=0; iid); @@ -898,7 +899,8 @@ int32_t tscParseMultiJSONPayload(char* payload, SArray* points, SSmlLinesInfo* i payloadNum = cJSON_GetArraySize(root); } else { tscError("OTD:0x%"PRIx64" Invalid JSON Payload", info->id); - return TSDB_CODE_TSC_INVALID_JSON; + ret = TSDB_CODE_TSC_INVALID_JSON; + goto PARSE_JSON_OVER; } for (int32_t i = 0; i < payloadNum; ++i) { @@ -909,14 +911,16 @@ int32_t tscParseMultiJSONPayload(char* payload, SArray* points, SSmlLinesInfo* i if (ret != TSDB_CODE_SUCCESS) { tscError("OTD:0x%"PRIx64" JSON data point parse failed", info->id); destroySmlDataPoint(&point); - return ret; + goto PARSE_JSON_OVER; } else { tscDebug("OTD:0x%"PRIx64" JSON data point parse success", info->id); } taosArrayPush(points, &point); } - return TSDB_CODE_SUCCESS; +PARSE_JSON_OVER: + cJSON_Delete(root); + return ret; } int taos_insert_json_payload(TAOS* taos, char* payload) { @@ -957,7 +961,7 @@ cleanup: tscDebug("OTD:0x%"PRIx64" taos_insert_json_payload finish inserting 1 Point. code: %d", info->id, code); points = TARRAY_GET_START(lpPoints); numPoints = taosArrayGetSize(lpPoints); - for (int i=0; i Date: Mon, 6 Sep 2021 12:32:49 +0800 Subject: [PATCH 32/39] [TD-6443]: Support OpenTSDB HTTP JSON data import format --- src/client/src/tscParseOpenTSDB.c | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/client/src/tscParseOpenTSDB.c b/src/client/src/tscParseOpenTSDB.c index f010a99bdc..e97882156c 100644 --- a/src/client/src/tscParseOpenTSDB.c +++ b/src/client/src/tscParseOpenTSDB.c @@ -434,7 +434,7 @@ int32_t parseMetricFromJSON(cJSON *root, TAOS_SML_DATA_POINT* pSml, SSmlLinesInf return TSDB_CODE_TSC_INVALID_JSON; } - int32_t stableLen = strlen(metric->valuestring); + uint64_t stableLen = strlen(metric->valuestring); if (stableLen > TSDB_TABLE_NAME_LEN) { tscError("OTD:0x%"PRIx64" Metric cannot exceeds 193 characters in JSON", info->id); return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; @@ -480,10 +480,10 @@ int32_t parseTimestampFromJSONObj(cJSON *root, int64_t *tsVal, SSmlLinesInfo* in return TSDB_CODE_SUCCESS; } - int32_t typeLen = strlen(type->valuestring); + uint64_t typeLen = strlen(type->valuestring); if (typeLen == 1 && type->valuestring[0] == 's') { //seconds - *tsVal = *tsVal * 1e9; + *tsVal = (int64_t)(*tsVal * 1e9); } else if (typeLen == 2 && type->valuestring[1] == 's') { switch (type->valuestring[0]) { case 'm': @@ -565,7 +565,7 @@ int32_t convertJSONNumber(TAOS_SML_KV *pVal, char* typeStr, cJSON *value, SSmlLi if (strcasecmp(typeStr, "i8") == 0 || strcasecmp(typeStr, "tinyint") == 0) { if (!IS_VALID_TINYINT(value->valueint)) { - tscError("OTD:0x%"PRIx64" JSON value(%ld) cannot fit in type(tinyint)", info->id, value->valueint); + tscError("OTD:0x%"PRIx64" JSON value(%"PRId64") cannot fit in type(tinyint)", info->id, value->valueint); return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; } pVal->type = TSDB_DATA_TYPE_TINYINT; @@ -578,7 +578,7 @@ int32_t convertJSONNumber(TAOS_SML_KV *pVal, char* typeStr, cJSON *value, SSmlLi if (strcasecmp(typeStr, "i16") == 0 || strcasecmp(typeStr, "smallint") == 0) { if (!IS_VALID_SMALLINT(value->valueint)) { - tscError("OTD:0x%"PRIx64" JSON value(%ld) cannot fit in type(smallint)", info->id, value->valueint); + tscError("OTD:0x%"PRIx64" JSON value(%"PRId64") cannot fit in type(smallint)", info->id, value->valueint); return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; } pVal->type = TSDB_DATA_TYPE_SMALLINT; @@ -591,7 +591,7 @@ int32_t convertJSONNumber(TAOS_SML_KV *pVal, char* typeStr, cJSON *value, SSmlLi if (strcasecmp(typeStr, "i32") == 0 || strcasecmp(typeStr, "int") == 0) { if (!IS_VALID_INT(value->valueint)) { - tscError("OTD:0x%"PRIx64" JSON value(%ld) cannot fit in type(int)", info->id, value->valueint); + tscError("OTD:0x%"PRIx64" JSON value(%"PRId64") cannot fit in type(int)", info->id, value->valueint); return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; } pVal->type = TSDB_DATA_TYPE_INT; @@ -604,7 +604,7 @@ int32_t convertJSONNumber(TAOS_SML_KV *pVal, char* typeStr, cJSON *value, SSmlLi if (strcasecmp(typeStr, "i64") == 0 || strcasecmp(typeStr, "bigint") == 0) { if (!IS_VALID_BIGINT(value->valueint)) { - tscError("OTD:0x%"PRIx64" JSON value(%ld) cannot fit in type(bigint)", info->id, value->valueint); + tscError("OTD:0x%"PRIx64" JSON value(%"PRId64") cannot fit in type(bigint)", info->id, value->valueint); return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; } pVal->type = TSDB_DATA_TYPE_BIGINT; @@ -654,7 +654,7 @@ int32_t convertJSONString(TAOS_SML_KV *pVal, char* typeStr, cJSON *value, SSmlLi tscError("OTD:0x%"PRIx64" invalid type(%s) for JSON String", info->id, typeStr); return TSDB_CODE_TSC_INVALID_JSON_TYPE; } - pVal->length = strlen(value->valuestring); + pVal->length = (int16_t)strlen(value->valuestring); pVal->value = tcalloc(pVal->length + 1, 1); memcpy(pVal->value, value->valuestring, pVal->length); return TSDB_CODE_SUCCESS; @@ -732,7 +732,7 @@ int32_t parseValueFromJSON(cJSON *root, TAOS_SML_KV *pVal, SSmlLinesInfo* info) //convert default JSON String type to nchar pVal->type = TSDB_DATA_TYPE_NCHAR; //pVal->length = wcslen((wchar_t *)root->valuestring) * TSDB_NCHAR_SIZE; - pVal->length = strlen(root->valuestring); + pVal->length = (int16_t)strlen(root->valuestring); pVal->value = tcalloc(pVal->length + 1, 1); memcpy(pVal->value, root->valuestring, pVal->length); break; @@ -786,7 +786,7 @@ int32_t parseTagsFromJSON(cJSON *root, TAOS_SML_KV **pKVs, int *num_kvs, char ** //only pick up the first ID value as child table name cJSON *id = cJSON_GetObjectItem(tags, "ID"); if (id != NULL) { - int32_t idLen = strlen(id->valuestring); + uint64_t idLen = strlen(id->valuestring); ret = isValidChildTableName(id->valuestring, idLen); if (ret != TSDB_CODE_SUCCESS) { return ret; @@ -816,7 +816,7 @@ int32_t parseTagsFromJSON(cJSON *root, TAOS_SML_KV **pKVs, int *num_kvs, char ** return TSDB_CODE_TSC_INVALID_JSON; } //key - int32_t keyLen = strlen(tag->string); + uint64_t keyLen = strlen(tag->string); pkv->key = tcalloc(keyLen + 1, sizeof(char)); strncpy(pkv->key, tag->string, keyLen); //value From 200de3058ef7b8c21e3352c5364225d095dd6356 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 6 Sep 2021 12:32:49 +0800 Subject: [PATCH 33/39] [TD-6443]: Support OpenTSDB HTTP JSON data import format --- src/client/src/tscParseOpenTSDB.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/client/src/tscParseOpenTSDB.c b/src/client/src/tscParseOpenTSDB.c index e97882156c..bb18b54a7c 100644 --- a/src/client/src/tscParseOpenTSDB.c +++ b/src/client/src/tscParseOpenTSDB.c @@ -434,7 +434,7 @@ int32_t parseMetricFromJSON(cJSON *root, TAOS_SML_DATA_POINT* pSml, SSmlLinesInf return TSDB_CODE_TSC_INVALID_JSON; } - uint64_t stableLen = strlen(metric->valuestring); + size_t stableLen = strlen(metric->valuestring); if (stableLen > TSDB_TABLE_NAME_LEN) { tscError("OTD:0x%"PRIx64" Metric cannot exceeds 193 characters in JSON", info->id); return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; @@ -480,7 +480,7 @@ int32_t parseTimestampFromJSONObj(cJSON *root, int64_t *tsVal, SSmlLinesInfo* in return TSDB_CODE_SUCCESS; } - uint64_t typeLen = strlen(type->valuestring); + size_t typeLen = strlen(type->valuestring); if (typeLen == 1 && type->valuestring[0] == 's') { //seconds *tsVal = (int64_t)(*tsVal * 1e9); @@ -786,8 +786,8 @@ int32_t parseTagsFromJSON(cJSON *root, TAOS_SML_KV **pKVs, int *num_kvs, char ** //only pick up the first ID value as child table name cJSON *id = cJSON_GetObjectItem(tags, "ID"); if (id != NULL) { - uint64_t idLen = strlen(id->valuestring); - ret = isValidChildTableName(id->valuestring, idLen); + size_t idLen = strlen(id->valuestring); + ret = isValidChildTableName(id->valuestring, (int16_t)idLen); if (ret != TSDB_CODE_SUCCESS) { return ret; } @@ -816,7 +816,7 @@ int32_t parseTagsFromJSON(cJSON *root, TAOS_SML_KV **pKVs, int *num_kvs, char ** return TSDB_CODE_TSC_INVALID_JSON; } //key - uint64_t keyLen = strlen(tag->string); + size_t keyLen = strlen(tag->string); pkv->key = tcalloc(keyLen + 1, sizeof(char)); strncpy(pkv->key, tag->string, keyLen); //value From 0df9e2e17ecad90f5516da1810a767c6e84a24bc Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Thu, 9 Sep 2021 09:22:13 +0800 Subject: [PATCH 34/39] [TD-6560]: Add unit test cases for parsing OpenTSDB HTTP JSON data import format --- tests/examples/c/apitest.c | 739 ++++++++++++++++++++++++++++++++++++- 1 file changed, 738 insertions(+), 1 deletion(-) diff --git a/tests/examples/c/apitest.c b/tests/examples/c/apitest.c index 621950a834..b227ae46fb 100644 --- a/tests/examples/c/apitest.c +++ b/tests/examples/c/apitest.c @@ -2,6 +2,7 @@ // to compile: gcc -o apitest apitest.c -ltaos #include "taoserror.h" +#include "cJSON.h" #include #include @@ -1020,7 +1021,7 @@ int32_t verify_schema_less(TAOS* taos) { void verify_telnet_insert(TAOS* taos) { TAOS_RES *result; - result = taos_query(taos, "drop database if exists test;"); + result = taos_query(taos, "drop database if exists db;"); taos_free_result(result); usleep(100000); result = taos_query(taos, "create database db precision 'ms';"); @@ -1197,6 +1198,739 @@ void verify_telnet_insert(TAOS* taos) { return; } +void verify_json_insert(TAOS* taos) { + TAOS_RES *result; + + result = taos_query(taos, "drop database if exists db;"); + taos_free_result(result); + usleep(100000); + result = taos_query(taos, "create database db precision 'ms';"); + taos_free_result(result); + usleep(100000); + + (void)taos_select_db(taos, "db"); + int32_t code = 0; + + char *message = + "{ \ + \"metric\":\"cpu_load_0\", \ + \"timestamp\": 1626006833610123, \ + \"value\": 55.5, \ + \"tags\": \ + { \ + \"host\": \"ubuntu\", \ + \"interface1\": \"eth0\", \ + \"Id\": \"tb0\" \ + } \ + }"; + + code = taos_insert_json_payload(taos, message); + if (code) { + printf("code: %d, %s.\n", code, tstrerror(code)); + } + + char *message1 = + "[ \ + { \ + \"metric\":\"cpu_load_1\", \ + \"timestamp\": 1626006833610123, \ + \"value\": 55.5, \ + \"tags\": \ + { \ + \"host\": \"ubuntu\", \ + \"interface\": \"eth1\", \ + \"Id\": \"tb1\" \ + } \ + }, \ + { \ + \"metric\":\"cpu_load_2\", \ + \"timestamp\": 1626006833610123, \ + \"value\": 55.5, \ + \"tags\": \ + { \ + \"host\": \"ubuntu\", \ + \"interface\": \"eth2\", \ + \"Id\": \"tb2\" \ + } \ + } \ + ]"; + + code = taos_insert_json_payload(taos, message1); + if (code) { + printf("code: %d, %s.\n", code, tstrerror(code)); + } + + char *message2 = + "[ \ + { \ + \"metric\":\"cpu_load_3\", \ + \"timestamp\": \ + { \ + \"value\": 1626006833610123, \ + \"type\": \"us\" \ + }, \ + \"value\": \ + { \ + \"value\": 55, \ + \"type\": \"int\" \ + }, \ + \"tags\": \ + { \ + \"host\": \ + { \ + \"value\": \"ubuntu\", \ + \"type\": \"binary\" \ + }, \ + \"interface\": \ + { \ + \"value\": \"eth3\", \ + \"type\": \"nchar\" \ + }, \ + \"ID\": \"tb3\", \ + \"port\": \ + { \ + \"value\": 4040, \ + \"type\": \"int\" \ + } \ + } \ + }, \ + { \ + \"metric\":\"cpu_load_4\", \ + \"timestamp\": 1626006833610123, \ + \"value\": 66.6, \ + \"tags\": \ + { \ + \"host\": \"ubuntu\", \ + \"interface\": \"eth4\", \ + \"Id\": \"tb4\" \ + } \ + } \ + ]"; + code = taos_insert_json_payload(taos, message2); + if (code) { + printf("code: %d, %s.\n", code, tstrerror(code)); + } + + + cJSON *payload, *tags; + char *payload_str; + + /* Default format */ + //number + payload = cJSON_CreateObject(); + cJSON_AddStringToObject(payload, "metric", "stb0_0"); + cJSON_AddNumberToObject(payload, "timestamp", 1626006833610123); + cJSON_AddNumberToObject(payload, "value", 10); + tags = cJSON_CreateObject(); + cJSON_AddTrueToObject(tags, "t1"); + cJSON_AddFalseToObject(tags, "t2"); + cJSON_AddNumberToObject(tags, "t3", 10); + cJSON_AddStringToObject(tags, "t4", "123_abc_.!@#$%^&*:;,./?|+-=()[]{}<>"); + cJSON_AddItemToObject(payload, "tags", tags); + payload_str = cJSON_Print(payload); + //printf("%s\n", payload_str); + + code = taos_insert_json_payload(taos, payload_str); + if (code) { + printf("code: %d, %s.\n", code, tstrerror(code)); + } + free(payload_str); + cJSON_Delete(payload); + + //true + payload = cJSON_CreateObject(); + cJSON_AddStringToObject(payload, "metric", "stb0_1"); + cJSON_AddNumberToObject(payload, "timestamp", 1626006833610123); + cJSON_AddTrueToObject(payload, "value"); + tags = cJSON_CreateObject(); + cJSON_AddTrueToObject(tags, "t1"); + cJSON_AddFalseToObject(tags, "t2"); + cJSON_AddNumberToObject(tags, "t3", 10); + cJSON_AddStringToObject(tags, "t4", "123_abc_.!@#$%^&*:;,./?|+-=()[]{}<>"); + cJSON_AddItemToObject(payload, "tags", tags); + payload_str = cJSON_Print(payload); + //printf("%s\n", payload_str); + + code = taos_insert_json_payload(taos, payload_str); + if (code) { + printf("code: %d, %s.\n", code, tstrerror(code)); + } + free(payload_str); + cJSON_Delete(payload); + + //false + payload = cJSON_CreateObject(); + cJSON_AddStringToObject(payload, "metric", "stb0_2"); + cJSON_AddNumberToObject(payload, "timestamp", 1626006833610123); + cJSON_AddFalseToObject(payload, "value"); + tags = cJSON_CreateObject(); + cJSON_AddTrueToObject(tags, "t1"); + cJSON_AddFalseToObject(tags, "t2"); + cJSON_AddNumberToObject(tags, "t3", 10); + cJSON_AddStringToObject(tags, "t4", "123_abc_.!@#$%^&*:;,./?|+-=()[]{}<>"); + cJSON_AddItemToObject(payload, "tags", tags); + payload_str = cJSON_Print(payload); + //printf("%s\n", payload_str); + + code = taos_insert_json_payload(taos, payload_str); + if (code) { + printf("code: %d, %s.\n", code, tstrerror(code)); + } + free(payload_str); + cJSON_Delete(payload); + + //string + payload = cJSON_CreateObject(); + cJSON_AddStringToObject(payload, "metric", "stb0_3"); + cJSON_AddNumberToObject(payload, "timestamp", 1626006833610123); + cJSON_AddStringToObject(payload, "value", "123_abc_.!@#$%^&*:;,./?|+-=()[]{}<>"); + tags = cJSON_CreateObject(); + cJSON_AddTrueToObject(tags, "t1"); + cJSON_AddFalseToObject(tags, "t2"); + cJSON_AddNumberToObject(tags, "t3", 10); + cJSON_AddStringToObject(tags, "t4", "123_abc_.!@#$%^&*:;,./?|+-=()[]{}<>"); + cJSON_AddItemToObject(payload, "tags", tags); + payload_str = cJSON_Print(payload); + //printf("%s\n", payload_str); + + code = taos_insert_json_payload(taos, payload_str); + if (code) { + printf("code: %d, %s.\n", code, tstrerror(code)); + } + free(payload_str); + cJSON_Delete(payload); + + //timestamp 0 -> current time + payload = cJSON_CreateObject(); + cJSON_AddStringToObject(payload, "metric", "stb0_4"); + cJSON_AddNumberToObject(payload, "timestamp", 0); + cJSON_AddNumberToObject(payload, "value", 123); + tags = cJSON_CreateObject(); + cJSON_AddTrueToObject(tags, "t1"); + cJSON_AddFalseToObject(tags, "t2"); + cJSON_AddNumberToObject(tags, "t3", 10); + cJSON_AddStringToObject(tags, "t4", "123_abc_.!@#$%^&*:;,./?|+-=()[]{}<>"); + cJSON_AddItemToObject(payload, "tags", tags); + payload_str = cJSON_Print(payload); + //printf("%s\n", payload_str); + + code = taos_insert_json_payload(taos, payload_str); + if (code) { + printf("code: %d, %s.\n", code, tstrerror(code)); + } + free(payload_str); + cJSON_Delete(payload); + + //ID + payload = cJSON_CreateObject(); + cJSON_AddStringToObject(payload, "metric", "stb0_5"); + cJSON_AddNumberToObject(payload, "timestamp", 0); + cJSON_AddNumberToObject(payload, "value", 123); + tags = cJSON_CreateObject(); + cJSON_AddStringToObject(tags, "ID", "tb0_5"); + cJSON_AddTrueToObject(tags, "t1"); + cJSON_AddStringToObject(tags, "iD", "tb000"); + cJSON_AddFalseToObject(tags, "t2"); + cJSON_AddNumberToObject(tags, "t3", 10); + cJSON_AddStringToObject(tags, "t4", "123_abc_.!@#$%^&*:;,./?|+-=()[]{}<>"); + cJSON_AddStringToObject(tags, "id", "tb555"); + cJSON_AddItemToObject(payload, "tags", tags); + payload_str = cJSON_Print(payload); + //printf("%s\n", payload_str); + + code = taos_insert_json_payload(taos, payload_str); + if (code) { + printf("code: %d, %s.\n", code, tstrerror(code)); + } + free(payload_str); + cJSON_Delete(payload); + + /* Nested format */ + //timestamp + cJSON *timestamp; + //seconds + payload = cJSON_CreateObject(); + cJSON_AddStringToObject(payload, "metric", "stb1_0"); + + timestamp = cJSON_CreateObject(); + cJSON_AddNumberToObject(timestamp, "value", 1626006833); + cJSON_AddStringToObject(timestamp, "type", "s"); + cJSON_AddItemToObject(payload, "timestamp", timestamp); + + cJSON_AddNumberToObject(payload, "value", 10); + tags = cJSON_CreateObject(); + cJSON_AddTrueToObject(tags, "t1"); + cJSON_AddFalseToObject(tags, "t2"); + cJSON_AddNumberToObject(tags, "t3", 10); + cJSON_AddStringToObject(tags, "t4", "123_abc_.!@#$%^&*:;,./?|+-=()[]{}<>"); + cJSON_AddItemToObject(payload, "tags", tags); + payload_str = cJSON_Print(payload); + //printf("%s\n", payload_str); + + code = taos_insert_json_payload(taos, payload_str); + if (code) { + printf("code: %d, %s.\n", code, tstrerror(code)); + } + free(payload_str); + cJSON_Delete(payload); + + //milleseconds + payload = cJSON_CreateObject(); + cJSON_AddStringToObject(payload, "metric", "stb1_1"); + + timestamp = cJSON_CreateObject(); + cJSON_AddNumberToObject(timestamp, "value", 1626006833610); + cJSON_AddStringToObject(timestamp, "type", "ms"); + cJSON_AddItemToObject(payload, "timestamp", timestamp); + + cJSON_AddNumberToObject(payload, "value", 10); + tags = cJSON_CreateObject(); + cJSON_AddTrueToObject(tags, "t1"); + cJSON_AddFalseToObject(tags, "t2"); + cJSON_AddNumberToObject(tags, "t3", 10); + cJSON_AddStringToObject(tags, "t4", "123_abc_.!@#$%^&*:;,./?|+-=()[]{}<>"); + cJSON_AddItemToObject(payload, "tags", tags); + payload_str = cJSON_Print(payload); + //printf("%s\n", payload_str); + + code = taos_insert_json_payload(taos, payload_str); + if (code) { + printf("code: %d, %s.\n", code, tstrerror(code)); + } + free(payload_str); + cJSON_Delete(payload); + + //microseconds + payload = cJSON_CreateObject(); + cJSON_AddStringToObject(payload, "metric", "stb1_2"); + + timestamp = cJSON_CreateObject(); + cJSON_AddNumberToObject(timestamp, "value", 1626006833610123); + cJSON_AddStringToObject(timestamp, "type", "us"); + cJSON_AddItemToObject(payload, "timestamp", timestamp); + + cJSON_AddNumberToObject(payload, "value", 10); + tags = cJSON_CreateObject(); + cJSON_AddTrueToObject(tags, "t1"); + cJSON_AddFalseToObject(tags, "t2"); + cJSON_AddNumberToObject(tags, "t3", 10); + cJSON_AddStringToObject(tags, "t4", "123_abc_.!@#$%^&*:;,./?|+-=()[]{}<>"); + cJSON_AddItemToObject(payload, "tags", tags); + payload_str = cJSON_Print(payload); + //printf("%s\n", payload_str); + + code = taos_insert_json_payload(taos, payload_str); + if (code) { + printf("code: %d, %s.\n", code, tstrerror(code)); + } + free(payload_str); + cJSON_Delete(payload); + + //nanoseconds + payload = cJSON_CreateObject(); + cJSON_AddStringToObject(payload, "metric", "stb1_3"); + + timestamp = cJSON_CreateObject(); + cJSON_AddNumberToObject(timestamp, "value", 1626006833610123321); + cJSON_AddStringToObject(timestamp, "type", "ns"); + cJSON_AddItemToObject(payload, "timestamp", timestamp); + + cJSON_AddNumberToObject(payload, "value", 10); + tags = cJSON_CreateObject(); + cJSON_AddTrueToObject(tags, "t1"); + cJSON_AddFalseToObject(tags, "t2"); + cJSON_AddNumberToObject(tags, "t3", 10); + cJSON_AddStringToObject(tags, "t4", "123_abc_.!@#$%^&*:;,./?|+-=()[]{}<>"); + cJSON_AddItemToObject(payload, "tags", tags); + payload_str = cJSON_Print(payload); + //printf("%s\n", payload_str); + + code = taos_insert_json_payload(taos, payload_str); + if (code) { + printf("code: %d, %s.\n", code, tstrerror(code)); + } + free(payload_str); + cJSON_Delete(payload); + + //now + payload = cJSON_CreateObject(); + cJSON_AddStringToObject(payload, "metric", "stb1_4"); + + timestamp = cJSON_CreateObject(); + cJSON_AddNumberToObject(timestamp, "value", 0); + cJSON_AddStringToObject(timestamp, "type", "ns"); + cJSON_AddItemToObject(payload, "timestamp", timestamp); + + cJSON_AddNumberToObject(payload, "value", 10); + tags = cJSON_CreateObject(); + cJSON_AddTrueToObject(tags, "t1"); + cJSON_AddFalseToObject(tags, "t2"); + cJSON_AddNumberToObject(tags, "t3", 10); + cJSON_AddStringToObject(tags, "t4", "123_abc_.!@#$%^&*:;,./?|+-=()[]{}<>"); + cJSON_AddItemToObject(payload, "tags", tags); + payload_str = cJSON_Print(payload); + //printf("%s\n", payload_str); + + code = taos_insert_json_payload(taos, payload_str); + if (code) { + printf("code: %d, %s.\n", code, tstrerror(code)); + } + free(payload_str); + cJSON_Delete(payload); + + //metric value + cJSON *metric_val; + //bool + payload = cJSON_CreateObject(); + cJSON_AddStringToObject(payload, "metric", "stb2_0"); + + timestamp = cJSON_CreateObject(); + cJSON_AddNumberToObject(timestamp, "value", 1626006833); + cJSON_AddStringToObject(timestamp, "type", "s"); + cJSON_AddItemToObject(payload, "timestamp", timestamp); + + metric_val = cJSON_CreateObject(); + cJSON_AddTrueToObject(metric_val, "value"); + cJSON_AddStringToObject(metric_val, "type", "bool"); + cJSON_AddItemToObject(payload, "value", metric_val); + + tags = cJSON_CreateObject(); + cJSON_AddTrueToObject(tags, "t1"); + cJSON_AddFalseToObject(tags, "t2"); + cJSON_AddNumberToObject(tags, "t3", 10); + cJSON_AddStringToObject(tags, "t4", "123_abc_.!@#$%^&*:;,./?|+-=()[]{}<>"); + cJSON_AddItemToObject(payload, "tags", tags); + payload_str = cJSON_Print(payload); + //printf("%s\n", payload_str); + + code = taos_insert_json_payload(taos, payload_str); + if (code) { + printf("code: %d, %s.\n", code, tstrerror(code)); + } + free(payload_str); + cJSON_Delete(payload); + + //tinyint + payload = cJSON_CreateObject(); + cJSON_AddStringToObject(payload, "metric", "stb2_1"); + + timestamp = cJSON_CreateObject(); + cJSON_AddNumberToObject(timestamp, "value", 1626006833); + cJSON_AddStringToObject(timestamp, "type", "s"); + cJSON_AddItemToObject(payload, "timestamp", timestamp); + + metric_val = cJSON_CreateObject(); + cJSON_AddNumberToObject(metric_val, "value", 127); + cJSON_AddStringToObject(metric_val, "type", "tinyint"); + cJSON_AddItemToObject(payload, "value", metric_val); + + tags = cJSON_CreateObject(); + cJSON_AddTrueToObject(tags, "t1"); + cJSON_AddFalseToObject(tags, "t2"); + cJSON_AddNumberToObject(tags, "t3", 10); + cJSON_AddStringToObject(tags, "t4", "123_abc_.!@#$%^&*:;,./?|+-=()[]{}<>"); + cJSON_AddItemToObject(payload, "tags", tags); + payload_str = cJSON_Print(payload); + //printf("%s\n", payload_str); + + code = taos_insert_json_payload(taos, payload_str); + if (code) { + printf("code: %d, %s.\n", code, tstrerror(code)); + } + free(payload_str); + cJSON_Delete(payload); + + //smallint + payload = cJSON_CreateObject(); + cJSON_AddStringToObject(payload, "metric", "stb2_2"); + + timestamp = cJSON_CreateObject(); + cJSON_AddNumberToObject(timestamp, "value", 1626006833); + cJSON_AddStringToObject(timestamp, "type", "s"); + cJSON_AddItemToObject(payload, "timestamp", timestamp); + + metric_val = cJSON_CreateObject(); + cJSON_AddNumberToObject(metric_val, "value", 32767); + cJSON_AddStringToObject(metric_val, "type", "smallint"); + cJSON_AddItemToObject(payload, "value", metric_val); + + tags = cJSON_CreateObject(); + cJSON_AddTrueToObject(tags, "t1"); + cJSON_AddFalseToObject(tags, "t2"); + cJSON_AddNumberToObject(tags, "t3", 10); + cJSON_AddStringToObject(tags, "t4", "123_abc_.!@#$%^&*:;,./?|+-=()[]{}<>"); + cJSON_AddItemToObject(payload, "tags", tags); + payload_str = cJSON_Print(payload); + //printf("%s\n", payload_str); + + code = taos_insert_json_payload(taos, payload_str); + if (code) { + printf("code: %d, %s.\n", code, tstrerror(code)); + } + free(payload_str); + cJSON_Delete(payload); + + //int + payload = cJSON_CreateObject(); + cJSON_AddStringToObject(payload, "metric", "stb2_3"); + + timestamp = cJSON_CreateObject(); + cJSON_AddNumberToObject(timestamp, "value", 1626006833); + cJSON_AddStringToObject(timestamp, "type", "s"); + cJSON_AddItemToObject(payload, "timestamp", timestamp); + + metric_val = cJSON_CreateObject(); + cJSON_AddNumberToObject(metric_val, "value", 2147483647); + cJSON_AddStringToObject(metric_val, "type", "int"); + cJSON_AddItemToObject(payload, "value", metric_val); + + tags = cJSON_CreateObject(); + cJSON_AddTrueToObject(tags, "t1"); + cJSON_AddFalseToObject(tags, "t2"); + cJSON_AddNumberToObject(tags, "t3", 10); + cJSON_AddStringToObject(tags, "t4", "123_abc_.!@#$%^&*:;,./?|+-=()[]{}<>"); + cJSON_AddItemToObject(payload, "tags", tags); + payload_str = cJSON_Print(payload); + //printf("%s\n", payload_str); + + code = taos_insert_json_payload(taos, payload_str); + if (code) { + printf("code: %d, %s.\n", code, tstrerror(code)); + } + free(payload_str); + cJSON_Delete(payload); + + //bigint + payload = cJSON_CreateObject(); + cJSON_AddStringToObject(payload, "metric", "stb2_4"); + + timestamp = cJSON_CreateObject(); + cJSON_AddNumberToObject(timestamp, "value", 1626006833); + cJSON_AddStringToObject(timestamp, "type", "s"); + cJSON_AddItemToObject(payload, "timestamp", timestamp); + + metric_val = cJSON_CreateObject(); + cJSON_AddNumberToObject(metric_val, "value", 9223372036854775807); + cJSON_AddStringToObject(metric_val, "type", "bigint"); + cJSON_AddItemToObject(payload, "value", metric_val); + + tags = cJSON_CreateObject(); + cJSON_AddTrueToObject(tags, "t1"); + cJSON_AddFalseToObject(tags, "t2"); + cJSON_AddNumberToObject(tags, "t3", 10); + cJSON_AddStringToObject(tags, "t4", "123_abc_.!@#$%^&*:;,./?|+-=()[]{}<>"); + cJSON_AddItemToObject(payload, "tags", tags); + payload_str = cJSON_Print(payload); + //printf("%s\n", payload_str); + + code = taos_insert_json_payload(taos, payload_str); + if (code) { + printf("code: %d, %s.\n", code, tstrerror(code)); + } + free(payload_str); + cJSON_Delete(payload); + + //float + payload = cJSON_CreateObject(); + cJSON_AddStringToObject(payload, "metric", "stb2_5"); + + timestamp = cJSON_CreateObject(); + cJSON_AddNumberToObject(timestamp, "value", 1626006833); + cJSON_AddStringToObject(timestamp, "type", "s"); + cJSON_AddItemToObject(payload, "timestamp", timestamp); + + metric_val = cJSON_CreateObject(); + cJSON_AddNumberToObject(metric_val, "value", 11.12345); + cJSON_AddStringToObject(metric_val, "type", "float"); + cJSON_AddItemToObject(payload, "value", metric_val); + + tags = cJSON_CreateObject(); + cJSON_AddTrueToObject(tags, "t1"); + cJSON_AddFalseToObject(tags, "t2"); + cJSON_AddNumberToObject(tags, "t3", 10); + cJSON_AddStringToObject(tags, "t4", "123_abc_.!@#$%^&*:;,./?|+-=()[]{}<>"); + cJSON_AddItemToObject(payload, "tags", tags); + payload_str = cJSON_Print(payload); + //printf("%s\n", payload_str); + + code = taos_insert_json_payload(taos, payload_str); + if (code) { + printf("code: %d, %s.\n", code, tstrerror(code)); + } + free(payload_str); + cJSON_Delete(payload); + + //double + payload = cJSON_CreateObject(); + cJSON_AddStringToObject(payload, "metric", "stb2_6"); + + timestamp = cJSON_CreateObject(); + cJSON_AddNumberToObject(timestamp, "value", 1626006833); + cJSON_AddStringToObject(timestamp, "type", "s"); + cJSON_AddItemToObject(payload, "timestamp", timestamp); + + metric_val = cJSON_CreateObject(); + cJSON_AddNumberToObject(metric_val, "value", 22.123456789); + cJSON_AddStringToObject(metric_val, "type", "double"); + cJSON_AddItemToObject(payload, "value", metric_val); + + tags = cJSON_CreateObject(); + cJSON_AddTrueToObject(tags, "t1"); + cJSON_AddFalseToObject(tags, "t2"); + cJSON_AddNumberToObject(tags, "t3", 10); + cJSON_AddStringToObject(tags, "t4", "123_abc_.!@#$%^&*:;,./?|+-=()[]{}<>"); + cJSON_AddItemToObject(payload, "tags", tags); + payload_str = cJSON_Print(payload); + //printf("%s\n", payload_str); + + code = taos_insert_json_payload(taos, payload_str); + if (code) { + printf("code: %d, %s.\n", code, tstrerror(code)); + } + free(payload_str); + cJSON_Delete(payload); + + //binary + payload = cJSON_CreateObject(); + cJSON_AddStringToObject(payload, "metric", "stb2_7"); + + timestamp = cJSON_CreateObject(); + cJSON_AddNumberToObject(timestamp, "value", 1626006833); + cJSON_AddStringToObject(timestamp, "type", "s"); + cJSON_AddItemToObject(payload, "timestamp", timestamp); + + metric_val = cJSON_CreateObject(); + cJSON_AddStringToObject(metric_val, "value", "123_abc_.!@#$%^&*:;,./?|+-=()[]{}<>"); + cJSON_AddStringToObject(metric_val, "type", "binary"); + cJSON_AddItemToObject(payload, "value", metric_val); + + tags = cJSON_CreateObject(); + cJSON_AddTrueToObject(tags, "t1"); + cJSON_AddFalseToObject(tags, "t2"); + cJSON_AddNumberToObject(tags, "t3", 10); + cJSON_AddStringToObject(tags, "t4", "123_abc_.!@#$%^&*:;,./?|+-=()[]{}<>"); + cJSON_AddItemToObject(payload, "tags", tags); + payload_str = cJSON_Print(payload); + //printf("%s\n", payload_str); + + code = taos_insert_json_payload(taos, payload_str); + if (code) { + printf("code: %d, %s.\n", code, tstrerror(code)); + } + free(payload_str); + cJSON_Delete(payload); + + //nchar + payload = cJSON_CreateObject(); + cJSON_AddStringToObject(payload, "metric", "stb2_8"); + + timestamp = cJSON_CreateObject(); + cJSON_AddNumberToObject(timestamp, "value", 1626006833); + cJSON_AddStringToObject(timestamp, "type", "s"); + cJSON_AddItemToObject(payload, "timestamp", timestamp); + + metric_val = cJSON_CreateObject(); + cJSON_AddStringToObject(metric_val, "value", "你好"); + cJSON_AddStringToObject(metric_val, "type", "nchar"); + cJSON_AddItemToObject(payload, "value", metric_val); + + tags = cJSON_CreateObject(); + cJSON_AddTrueToObject(tags, "t1"); + cJSON_AddFalseToObject(tags, "t2"); + cJSON_AddNumberToObject(tags, "t3", 10); + cJSON_AddStringToObject(tags, "t4", "123_abc_.!@#$%^&*:;,./?|+-=()[]{}<>"); + cJSON_AddItemToObject(payload, "tags", tags); + payload_str = cJSON_Print(payload); + //printf("%s\n", payload_str); + + code = taos_insert_json_payload(taos, payload_str); + if (code) { + printf("code: %d, %s.\n", code, tstrerror(code)); + } + free(payload_str); + cJSON_Delete(payload); + + //tag value + cJSON *tag; + + payload = cJSON_CreateObject(); + cJSON_AddStringToObject(payload, "metric", "stb3_0"); + + timestamp = cJSON_CreateObject(); + cJSON_AddNumberToObject(timestamp, "value", 1626006833); + cJSON_AddStringToObject(timestamp, "type", "s"); + cJSON_AddItemToObject(payload, "timestamp", timestamp); + + metric_val = cJSON_CreateObject(); + cJSON_AddStringToObject(metric_val, "value", "hello"); + cJSON_AddStringToObject(metric_val, "type", "nchar"); + cJSON_AddItemToObject(payload, "value", metric_val); + + tags = cJSON_CreateObject(); + + tag = cJSON_CreateObject(); + cJSON_AddTrueToObject(tag, "value"); + cJSON_AddStringToObject(tag, "type", "bool"); + cJSON_AddItemToObject(tags, "t1", tag); + + tag = cJSON_CreateObject(); + cJSON_AddFalseToObject(tag, "value"); + cJSON_AddStringToObject(tag, "type", "bool"); + cJSON_AddItemToObject(tags, "t2", tag); + + tag = cJSON_CreateObject(); + cJSON_AddNumberToObject(tag, "value", 127); + cJSON_AddStringToObject(tag, "type", "tinyint"); + cJSON_AddItemToObject(tags, "t3", tag); + + tag = cJSON_CreateObject(); + cJSON_AddNumberToObject(tag, "value", 32767); + cJSON_AddStringToObject(tag, "type", "smallint"); + cJSON_AddItemToObject(tags, "t4", tag); + + tag = cJSON_CreateObject(); + cJSON_AddNumberToObject(tag, "value", 2147483647); + cJSON_AddStringToObject(tag, "type", "int"); + cJSON_AddItemToObject(tags, "t5", tag); + + tag = cJSON_CreateObject(); + cJSON_AddNumberToObject(tag, "value", 9223372036854775807); + cJSON_AddStringToObject(tag, "type", "bigint"); + cJSON_AddItemToObject(tags, "t6", tag); + + tag = cJSON_CreateObject(); + cJSON_AddNumberToObject(tag, "value", 11.12345); + cJSON_AddStringToObject(tag, "type", "float"); + cJSON_AddItemToObject(tags, "t7", tag); + + tag = cJSON_CreateObject(); + cJSON_AddNumberToObject(tag, "value", 22.1234567890); + cJSON_AddStringToObject(tag, "type", "double"); + cJSON_AddItemToObject(tags, "t8", tag); + + tag = cJSON_CreateObject(); + cJSON_AddStringToObject(tag, "value", "binary_val"); + cJSON_AddStringToObject(tag, "type", "binary"); + cJSON_AddItemToObject(tags, "t9", tag); + + tag = cJSON_CreateObject(); + cJSON_AddStringToObject(tag, "value", "你好"); + cJSON_AddStringToObject(tag, "type", "nchar"); + cJSON_AddItemToObject(tags, "t10", tag); + + cJSON_AddItemToObject(payload, "tags", tags); + + payload_str = cJSON_Print(payload); + //printf("%s\n", payload_str); + + code = taos_insert_json_payload(taos, payload_str); + if (code) { + printf("code: %d, %s.\n", code, tstrerror(code)); + } + free(payload_str); + cJSON_Delete(payload); +} + int main(int argc, char *argv[]) { const char* host = "127.0.0.1"; const char* user = "root"; @@ -1220,6 +1954,9 @@ int main(int argc, char *argv[]) { printf("************ verify telnet-insert *************\n"); verify_telnet_insert(taos); + printf("************ verify json-insert *************\n"); + verify_json_insert(taos); + printf("************ verify query *************\n"); verify_query(taos); From ca4ffac45aaa8c5275efd1e18fb851531ddafbbe Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Thu, 9 Sep 2021 09:22:13 +0800 Subject: [PATCH 35/39] [TD-6560]: Add unit test cases for parsing OpenTSDB HTTP JSON data import format --- tests/examples/c/apitest.c | 74 +++++++++++++++++++------------------- 1 file changed, 37 insertions(+), 37 deletions(-) diff --git a/tests/examples/c/apitest.c b/tests/examples/c/apitest.c index b227ae46fb..28e8d3f012 100644 --- a/tests/examples/c/apitest.c +++ b/tests/examples/c/apitest.c @@ -1039,7 +1039,7 @@ void verify_telnet_insert(TAOS* taos) { }; code = taos_insert_telnet_lines(taos, lines0, 3); if (code) { - printf("code: %d, %s.\n", code, tstrerror(code)); + printf("lines0 code: %d, %s.\n", code, tstrerror(code)); } /* timestamp */ @@ -1053,7 +1053,7 @@ void verify_telnet_insert(TAOS* taos) { }; code = taos_insert_telnet_lines(taos, lines1, 6); if (code) { - printf("code: %d, %s.\n", code, tstrerror(code)); + printf("lines1 code: %d, %s.\n", code, tstrerror(code)); } /* metric value */ @@ -1064,7 +1064,7 @@ void verify_telnet_insert(TAOS* taos) { }; code = taos_insert_telnet_lines(taos, lines2_0, 2); if (code) { - printf("code: %d, %s.\n", code, tstrerror(code)); + printf("lines2_0 code: %d, %s.\n", code, tstrerror(code)); } //smallint @@ -1074,7 +1074,7 @@ void verify_telnet_insert(TAOS* taos) { }; code = taos_insert_telnet_lines(taos, lines2_1, 2); if (code) { - printf("code: %d, %s.\n", code, tstrerror(code)); + printf("lines2_1 code: %d, %s.\n", code, tstrerror(code)); } //int @@ -1084,7 +1084,7 @@ void verify_telnet_insert(TAOS* taos) { }; code = taos_insert_telnet_lines(taos, lines2_2, 2); if (code) { - printf("code: %d, %s.\n", code, tstrerror(code)); + printf("lines2_2 code: %d, %s.\n", code, tstrerror(code)); } //bigint @@ -1094,7 +1094,7 @@ void verify_telnet_insert(TAOS* taos) { }; code = taos_insert_telnet_lines(taos, lines2_3, 2); if (code) { - printf("code: %d, %s.\n", code, tstrerror(code)); + printf("lines2_3 code: %d, %s.\n", code, tstrerror(code)); } //float @@ -1113,7 +1113,7 @@ void verify_telnet_insert(TAOS* taos) { }; code = taos_insert_telnet_lines(taos, lines2_4, 11); if (code) { - printf("code: %d, %s.\n", code, tstrerror(code)); + printf("lines2_4 code: %d, %s.\n", code, tstrerror(code)); } //double @@ -1131,7 +1131,7 @@ void verify_telnet_insert(TAOS* taos) { }; code = taos_insert_telnet_lines(taos, lines2_5, 10); if (code) { - printf("code: %d, %s.\n", code, tstrerror(code)); + printf("lines2_5 code: %d, %s.\n", code, tstrerror(code)); } //bool @@ -1149,7 +1149,7 @@ void verify_telnet_insert(TAOS* taos) { }; code = taos_insert_telnet_lines(taos, lines2_6, 10); if (code) { - printf("code: %d, %s.\n", code, tstrerror(code)); + printf("lines2_6 code: %d, %s.\n", code, tstrerror(code)); } //binary @@ -1160,7 +1160,7 @@ void verify_telnet_insert(TAOS* taos) { }; code = taos_insert_telnet_lines(taos, lines2_7, 3); if (code) { - printf("code: %d, %s.\n", code, tstrerror(code)); + printf("lines2_7 code: %d, %s.\n", code, tstrerror(code)); } //nchar @@ -1170,7 +1170,7 @@ void verify_telnet_insert(TAOS* taos) { }; code = taos_insert_telnet_lines(taos, lines2_8, 2); if (code) { - printf("code: %d, %s.\n", code, tstrerror(code)); + printf("lines2_8 code: %d, %s.\n", code, tstrerror(code)); } /* tags */ @@ -1181,7 +1181,7 @@ void verify_telnet_insert(TAOS* taos) { }; code = taos_insert_telnet_lines(taos, lines3_0, 2); if (code) { - printf("code: %d, %s.\n", code, tstrerror(code)); + printf("lines3_0 code: %d, %s.\n", code, tstrerror(code)); } //tag ID as child table name @@ -1192,7 +1192,7 @@ void verify_telnet_insert(TAOS* taos) { }; code = taos_insert_telnet_lines(taos, lines3_1, 3); if (code) { - printf("code: %d, %s.\n", code, tstrerror(code)); + printf("lines3_1 code: %d, %s.\n", code, tstrerror(code)); } return; @@ -1226,7 +1226,7 @@ void verify_json_insert(TAOS* taos) { code = taos_insert_json_payload(taos, message); if (code) { - printf("code: %d, %s.\n", code, tstrerror(code)); + printf("payload_0 code: %d, %s.\n", code, tstrerror(code)); } char *message1 = @@ -1257,7 +1257,7 @@ void verify_json_insert(TAOS* taos) { code = taos_insert_json_payload(taos, message1); if (code) { - printf("code: %d, %s.\n", code, tstrerror(code)); + printf("payload_1 code: %d, %s.\n", code, tstrerror(code)); } char *message2 = @@ -1308,7 +1308,7 @@ void verify_json_insert(TAOS* taos) { ]"; code = taos_insert_json_payload(taos, message2); if (code) { - printf("code: %d, %s.\n", code, tstrerror(code)); + printf("payload_2 code: %d, %s.\n", code, tstrerror(code)); } @@ -1332,7 +1332,7 @@ void verify_json_insert(TAOS* taos) { code = taos_insert_json_payload(taos, payload_str); if (code) { - printf("code: %d, %s.\n", code, tstrerror(code)); + printf("payload0_0 code: %d, %s.\n", code, tstrerror(code)); } free(payload_str); cJSON_Delete(payload); @@ -1353,7 +1353,7 @@ void verify_json_insert(TAOS* taos) { code = taos_insert_json_payload(taos, payload_str); if (code) { - printf("code: %d, %s.\n", code, tstrerror(code)); + printf("payload0_1 code: %d, %s.\n", code, tstrerror(code)); } free(payload_str); cJSON_Delete(payload); @@ -1374,7 +1374,7 @@ void verify_json_insert(TAOS* taos) { code = taos_insert_json_payload(taos, payload_str); if (code) { - printf("code: %d, %s.\n", code, tstrerror(code)); + printf("payload0_2 code: %d, %s.\n", code, tstrerror(code)); } free(payload_str); cJSON_Delete(payload); @@ -1395,7 +1395,7 @@ void verify_json_insert(TAOS* taos) { code = taos_insert_json_payload(taos, payload_str); if (code) { - printf("code: %d, %s.\n", code, tstrerror(code)); + printf("payload0_3 code: %d, %s.\n", code, tstrerror(code)); } free(payload_str); cJSON_Delete(payload); @@ -1416,7 +1416,7 @@ void verify_json_insert(TAOS* taos) { code = taos_insert_json_payload(taos, payload_str); if (code) { - printf("code: %d, %s.\n", code, tstrerror(code)); + printf("payload0_4 code: %d, %s.\n", code, tstrerror(code)); } free(payload_str); cJSON_Delete(payload); @@ -1440,7 +1440,7 @@ void verify_json_insert(TAOS* taos) { code = taos_insert_json_payload(taos, payload_str); if (code) { - printf("code: %d, %s.\n", code, tstrerror(code)); + printf("payload0_5 code: %d, %s.\n", code, tstrerror(code)); } free(payload_str); cJSON_Delete(payload); @@ -1469,7 +1469,7 @@ void verify_json_insert(TAOS* taos) { code = taos_insert_json_payload(taos, payload_str); if (code) { - printf("code: %d, %s.\n", code, tstrerror(code)); + printf("payload1_0 code: %d, %s.\n", code, tstrerror(code)); } free(payload_str); cJSON_Delete(payload); @@ -1495,7 +1495,7 @@ void verify_json_insert(TAOS* taos) { code = taos_insert_json_payload(taos, payload_str); if (code) { - printf("code: %d, %s.\n", code, tstrerror(code)); + printf("payload1_1 code: %d, %s.\n", code, tstrerror(code)); } free(payload_str); cJSON_Delete(payload); @@ -1521,7 +1521,7 @@ void verify_json_insert(TAOS* taos) { code = taos_insert_json_payload(taos, payload_str); if (code) { - printf("code: %d, %s.\n", code, tstrerror(code)); + printf("payload1_2 code: %d, %s.\n", code, tstrerror(code)); } free(payload_str); cJSON_Delete(payload); @@ -1547,7 +1547,7 @@ void verify_json_insert(TAOS* taos) { code = taos_insert_json_payload(taos, payload_str); if (code) { - printf("code: %d, %s.\n", code, tstrerror(code)); + printf("payload1_3 code: %d, %s.\n", code, tstrerror(code)); } free(payload_str); cJSON_Delete(payload); @@ -1573,7 +1573,7 @@ void verify_json_insert(TAOS* taos) { code = taos_insert_json_payload(taos, payload_str); if (code) { - printf("code: %d, %s.\n", code, tstrerror(code)); + printf("payload1_4 code: %d, %s.\n", code, tstrerror(code)); } free(payload_str); cJSON_Delete(payload); @@ -1605,7 +1605,7 @@ void verify_json_insert(TAOS* taos) { code = taos_insert_json_payload(taos, payload_str); if (code) { - printf("code: %d, %s.\n", code, tstrerror(code)); + printf("payload2_0 code: %d, %s.\n", code, tstrerror(code)); } free(payload_str); cJSON_Delete(payload); @@ -1635,7 +1635,7 @@ void verify_json_insert(TAOS* taos) { code = taos_insert_json_payload(taos, payload_str); if (code) { - printf("code: %d, %s.\n", code, tstrerror(code)); + printf("payload2_1 code: %d, %s.\n", code, tstrerror(code)); } free(payload_str); cJSON_Delete(payload); @@ -1665,7 +1665,7 @@ void verify_json_insert(TAOS* taos) { code = taos_insert_json_payload(taos, payload_str); if (code) { - printf("code: %d, %s.\n", code, tstrerror(code)); + printf("payload2_2 code: %d, %s.\n", code, tstrerror(code)); } free(payload_str); cJSON_Delete(payload); @@ -1695,7 +1695,7 @@ void verify_json_insert(TAOS* taos) { code = taos_insert_json_payload(taos, payload_str); if (code) { - printf("code: %d, %s.\n", code, tstrerror(code)); + printf("payload2_3 code: %d, %s.\n", code, tstrerror(code)); } free(payload_str); cJSON_Delete(payload); @@ -1725,7 +1725,7 @@ void verify_json_insert(TAOS* taos) { code = taos_insert_json_payload(taos, payload_str); if (code) { - printf("code: %d, %s.\n", code, tstrerror(code)); + printf("payload2_4 code: %d, %s.\n", code, tstrerror(code)); } free(payload_str); cJSON_Delete(payload); @@ -1755,7 +1755,7 @@ void verify_json_insert(TAOS* taos) { code = taos_insert_json_payload(taos, payload_str); if (code) { - printf("code: %d, %s.\n", code, tstrerror(code)); + printf("payload2_5 code: %d, %s.\n", code, tstrerror(code)); } free(payload_str); cJSON_Delete(payload); @@ -1785,7 +1785,7 @@ void verify_json_insert(TAOS* taos) { code = taos_insert_json_payload(taos, payload_str); if (code) { - printf("code: %d, %s.\n", code, tstrerror(code)); + printf("payload2_6 code: %d, %s.\n", code, tstrerror(code)); } free(payload_str); cJSON_Delete(payload); @@ -1815,7 +1815,7 @@ void verify_json_insert(TAOS* taos) { code = taos_insert_json_payload(taos, payload_str); if (code) { - printf("code: %d, %s.\n", code, tstrerror(code)); + printf("payload2_7 code: %d, %s.\n", code, tstrerror(code)); } free(payload_str); cJSON_Delete(payload); @@ -1845,7 +1845,7 @@ void verify_json_insert(TAOS* taos) { code = taos_insert_json_payload(taos, payload_str); if (code) { - printf("code: %d, %s.\n", code, tstrerror(code)); + printf("payload2_8 code: %d, %s.\n", code, tstrerror(code)); } free(payload_str); cJSON_Delete(payload); @@ -1925,7 +1925,7 @@ void verify_json_insert(TAOS* taos) { code = taos_insert_json_payload(taos, payload_str); if (code) { - printf("code: %d, %s.\n", code, tstrerror(code)); + printf("payload3_0 code: %d, %s.\n", code, tstrerror(code)); } free(payload_str); cJSON_Delete(payload); From b27bfa1dce3865c5312741dfd0be16bb77fe5911 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Thu, 9 Sep 2021 09:22:13 +0800 Subject: [PATCH 36/39] [TD-6560]: Add unit test cases for parsing OpenTSDB HTTP JSON data import format --- src/connector/python/taos/cinterface.py | 8 +++++++- src/connector/python/taos/connection.py | 19 +++++++++++++++++++ src/connector/python/taos/error.py | 12 +++++++++++- 3 files changed, 37 insertions(+), 2 deletions(-) diff --git a/src/connector/python/taos/cinterface.py b/src/connector/python/taos/cinterface.py index fadef7c397..7977a6bc02 100644 --- a/src/connector/python/taos/cinterface.py +++ b/src/connector/python/taos/cinterface.py @@ -838,8 +838,14 @@ def taos_insert_telnet_lines(connection, lines): p_lines = lines_type(*lines) errno = _libtaos.taos_insert_telnet_lines(connection, p_lines, num_of_lines) if errno != 0: - raise LinesError("insert telnet lines error", errno) + raise TelnetLinesError("insert telnet lines error", errno) +def taos_insert_json_payload(connection, payload): + # type: (c_void_p, list[str] | tuple(str)) -> None + payload = payload.encode("utf-8") + errno = _libtaos.taos_insert_json_payload(connection, payload) + if errno != 0: + raise JsonPayloadError("insert json payload error", errno) class CTaosInterface(object): def __init__(self, config=None): diff --git a/src/connector/python/taos/connection.py b/src/connector/python/taos/connection.py index a8a71ecc3a..35aca1fb26 100644 --- a/src/connector/python/taos/connection.py +++ b/src/connector/python/taos/connection.py @@ -154,6 +154,25 @@ class TaosConnection(object): """ return taos_insert_telnet_lines(self._conn, lines) + def insert_json_payload(self, payload): + """OpenTSDB HTTP JSON format support + + ## Example + "{ + "metric": "cpu_load_0", + "timestamp": 1626006833610123, + "value": 55.5, + "tags": + { + "host": "ubuntu", + "interface": "eth0", + "Id": "tb0" + } + }" + + """ + return taos_insert_json_payload(self._conn, payload) + def cursor(self): # type: () -> TaosCursor """Return a new Cursor object using the connection.""" diff --git a/src/connector/python/taos/error.py b/src/connector/python/taos/error.py index a30adbb162..f6a9d41f56 100644 --- a/src/connector/python/taos/error.py +++ b/src/connector/python/taos/error.py @@ -83,4 +83,14 @@ class ResultError(DatabaseError): class LinesError(DatabaseError): """taos_insert_lines errors.""" - pass \ No newline at end of file + pass + +class TelnetLinesError(DatabaseError): + """taos_insert_telnet_lines errors.""" + + pass + +class JsonPayloadError(DatabaseError): + """taos_insert_json_payload errors.""" + + pass From a99abf36ab5c32349f0b801e74da79e52671a21c Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Thu, 9 Sep 2021 09:22:13 +0800 Subject: [PATCH 37/39] [TD-6560]: Add unit test cases for parsing OpenTSDB HTTP JSON data import format --- tests/pytest/insert/insertJSONPayload.py | 568 +++++++++++++++++++++++ 1 file changed, 568 insertions(+) create mode 100644 tests/pytest/insert/insertJSONPayload.py diff --git a/tests/pytest/insert/insertJSONPayload.py b/tests/pytest/insert/insertJSONPayload.py new file mode 100644 index 0000000000..30f34446a9 --- /dev/null +++ b/tests/pytest/insert/insertJSONPayload.py @@ -0,0 +1,568 @@ +################################################################### +# Copyright (c) 2021 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + self._conn = conn + + def run(self): + print("running {}".format(__file__)) + tdSql.execute("drop database if exists test") + tdSql.execute("create database if not exists test precision 'us'") + tdSql.execute('use test') + + + ### Default format ### + ### metric value ### + print("============= step1 : test metric value types ================") + payload = ''' + { + "metric": "stb0_0", + "timestamp": 1626006833610123, + "value": 10, + "tags": { + "t1": true, + "t2": false, + "t3": 10, + "t4": "123_abc_.!@#$%^&*:;,./?|+-=()[]{}<>" + } + } + ''' + code = self._conn.insert_json_payload(payload) + print("insert_json_payload result {}".format(code)) + + tdSql.query("describe stb0_0") + tdSql.checkData(1, 1, "FLOAT") + + payload = ''' + { + "metric": "stb0_1", + "timestamp": 1626006833610123, + "value": true, + "tags": { + "t1": true, + "t2": false, + "t3": 10, + "t4": "123_abc_.!@#$%^&*:;,./?|+-=()[]{}<>" + } + } + ''' + code = self._conn.insert_json_payload(payload) + print("insert_json_payload result {}".format(code)) + + tdSql.query("describe stb0_1") + tdSql.checkData(1, 1, "BOOL") + + payload = ''' + { + "metric": "stb0_2", + "timestamp": 1626006833610123, + "value": false, + "tags": { + "t1": true, + "t2": false, + "t3": 10, + "t4": "123_abc_.!@#$%^&*:;,./?|+-=()[]{}<>" + } + } + ''' + code = self._conn.insert_json_payload(payload) + print("insert_json_payload result {}".format(code)) + + tdSql.query("describe stb0_2") + tdSql.checkData(1, 1, "BOOL") + + payload = ''' + { + "metric": "stb0_3", + "timestamp": 1626006833610123, + "value": "123_abc_.!@#$%^&*:;,./?|+-=()[]{}<>", + "tags": { + "t1": true, + "t2": false, + "t3": 10, + "t4": "123_abc_.!@#$%^&*:;,./?|+-=()[]{}<>" + } + } + ''' + code = self._conn.insert_json_payload(payload) + print("insert_json_payload result {}".format(code)) + + tdSql.query("describe stb0_3") + tdSql.checkData(1, 1, "NCHAR") + + ### timestamp 0 ### + payload = ''' + { + "metric": "stb0_4", + "timestamp": 0, + "value": 123, + "tags": { + "t1": true, + "t2": false, + "t3": 10, + "t4": "123_abc_.!@#$%^&*:;,./?|+-=()[]{}<>" + } + } + ''' + code = self._conn.insert_json_payload(payload) + print("insert_json_payload result {}".format(code)) + + + ### ID ### + payload = ''' + { + "metric": "stb0_5", + "timestamp": 0, + "value": 123, + "tags": { + "ID": "tb0_5", + "t1": true, + "iD": "tb000", + "t2": false, + "t3": 10, + "t4": "123_abc_.!@#$%^&*:;,./?|+-=()[]{}<>", + "id": "tb555" + } + } + ''' + code = self._conn.insert_json_payload(payload) + print("insert_json_payload result {}".format(code)) + + tdSql.query("select tbname from stb0_5") + tdSql.checkData(0, 0, "tb0_5") + + ### Nested format ### + ### timestamp ### + #seconds + payload = ''' + { + "metric": "stb1_0", + "timestamp": { + "value": 1626006833, + "type": "s" + }, + "value": 10, + "tags": { + "t1": true, + "t2": false, + "t3": 10, + "t4": "123_abc_.!@#$%^&*:;,./?|+-=()[]{}<>" + } + } + ''' + code = self._conn.insert_json_payload(payload) + print("insert_json_payload result {}".format(code)) + + tdSql.query("select ts from stb1_0") + tdSql.checkData(0, 0, "2021-07-11 20:33:53.000000") + + #milliseconds + payload = ''' + { + "metric": "stb1_1", + "timestamp": { + "value": 1626006833610, + "type": "ms" + }, + "value": 10, + "tags": { + "t1": true, + "t2": false, + "t3": 10, + "t4": "123_abc_.!@#$%^&*:;,./?|+-=()[]{}<>" + } + } + ''' + code = self._conn.insert_json_payload(payload) + print("insert_json_payload result {}".format(code)) + + tdSql.query("select ts from stb1_1") + tdSql.checkData(0, 0, "2021-07-11 20:33:53.610000") + + #microseconds + payload = ''' + { + "metric": "stb1_2", + "timestamp": { + "value": 1626006833610123, + "type": "us" + }, + "value": 10, + "tags": { + "t1": true, + "t2": false, + "t3": 10, + "t4": "123_abc_.!@#$%^&*:;,./?|+-=()[]{}<>" + } + } + ''' + code = self._conn.insert_json_payload(payload) + print("insert_json_payload result {}".format(code)) + + tdSql.query("select ts from stb1_2") + tdSql.checkData(0, 0, "2021-07-11 20:33:53.610123") + + #nanoseconds + payload = ''' + { + "metric": "stb1_3", + "timestamp": { + "value": 1.6260068336101233e+18, + "type": "ns" + }, + "value": 10, + "tags": { + "t1": true, + "t2": false, + "t3": 10, + "t4": "123_abc_.!@#$%^&*:;,./?|+-=()[]{}<>" + } + } + ''' + code = self._conn.insert_json_payload(payload) + print("insert_json_payload result {}".format(code)) + + tdSql.query("select ts from stb1_3") + tdSql.checkData(0, 0, "2021-07-11 20:33:53.610123") + + #now + tdSql.execute('use test') + payload = ''' + { + "metric": "stb1_4", + "timestamp": { + "value": 0, + "type": "ns" + }, + "value": 10, + "tags": { + "t1": true, + "t2": false, + "t3": 10, + "t4": "123_abc_.!@#$%^&*:;,./?|+-=()[]{}<>" + } + } + ''' + code = self._conn.insert_json_payload(payload) + print("insert_json_payload result {}".format(code)) + + ### metric value ### + payload = ''' + { + "metric": "stb2_0", + "timestamp": { + "value": 1626006833, + "type": "s" + }, + "value": { + "value": true, + "type": "bool" + }, + "tags": { + "t1": true, + "t2": false, + "t3": 10, + "t4": "123_abc_.!@#$%^&*:;,./?|+-=()[]{}<>" + } + } + ''' + code = self._conn.insert_json_payload(payload) + print("insert_json_payload result {}".format(code)) + + tdSql.query("describe stb2_0") + tdSql.checkData(1, 1, "BOOL") + + payload = ''' + { + "metric": "stb2_1", + "timestamp": { + "value": 1626006833, + "type": "s" + }, + "value": { + "value": 127, + "type": "tinyint" + }, + "tags": { + "t1": true, + "t2": false, + "t3": 10, + "t4": "123_abc_.!@#$%^&*:;,./?|+-=()[]{}<>" + } + } + ''' + code = self._conn.insert_json_payload(payload) + print("insert_json_payload result {}".format(code)) + + tdSql.query("describe stb2_1") + tdSql.checkData(1, 1, "TINYINT") + + payload = ''' + { + "metric": "stb2_2", + "timestamp": { + "value": 1626006833, + "type": "s" + }, + "value": { + "value": 32767, + "type": "smallint" + }, + "tags": { + "t1": true, + "t2": false, + "t3": 10, + "t4": "123_abc_.!@#$%^&*:;,./?|+-=()[]{}<>" + } + } + ''' + code = self._conn.insert_json_payload(payload) + print("insert_json_payload result {}".format(code)) + + tdSql.query("describe stb2_2") + tdSql.checkData(1, 1, "SMALLINT") + + payload = ''' + { + "metric": "stb2_3", + "timestamp": { + "value": 1626006833, + "type": "s" + }, + "value": { + "value": 2147483647, + "type": "int" + }, + "tags": { + "t1": true, + "t2": false, + "t3": 10, + "t4": "123_abc_.!@#$%^&*:;,./?|+-=()[]{}<>" + } + } + ''' + code = self._conn.insert_json_payload(payload) + print("insert_json_payload result {}".format(code)) + + tdSql.query("describe stb2_3") + tdSql.checkData(1, 1, "INT") + + payload = ''' + { + "metric": "stb2_4", + "timestamp": { + "value": 1626006833, + "type": "s" + }, + "value": { + "value": 9.2233720368547758e+18, + "type": "bigint" + }, + "tags": { + "t1": true, + "t2": false, + "t3": 10, + "t4": "123_abc_.!@#$%^&*:;,./?|+-=()[]{}<>" + } + } + ''' + code = self._conn.insert_json_payload(payload) + print("insert_json_payload result {}".format(code)) + + tdSql.query("describe stb2_4") + tdSql.checkData(1, 1, "BIGINT") + + payload = ''' + { + "metric": "stb2_5", + "timestamp": { + "value": 1626006833, + "type": "s" + }, + "value": { + "value": 11.12345, + "type": "float" + }, + "tags": { + "t1": true, + "t2": false, + "t3": 10, + "t4": "123_abc_.!@#$%^&*:;,./?|+-=()[]{}<>" + } + } + ''' + code = self._conn.insert_json_payload(payload) + print("insert_json_payload result {}".format(code)) + + tdSql.query("describe stb2_5") + tdSql.checkData(1, 1, "FLOAT") + + payload = ''' + { + "metric": "stb2_6", + "timestamp": { + "value": 1626006833, + "type": "s" + }, + "value": { + "value": 22.123456789, + "type": "double" + }, + "tags": { + "t1": true, + "t2": false, + "t3": 10, + "t4": "123_abc_.!@#$%^&*:;,./?|+-=()[]{}<>" + } + } + ''' + code = self._conn.insert_json_payload(payload) + print("insert_json_payload result {}".format(code)) + + tdSql.query("describe stb2_6") + tdSql.checkData(1, 1, "DOUBLE") + + payload = ''' + { + "metric": "stb2_7", + "timestamp": { + "value": 1626006833, + "type": "s" + }, + "value": { + "value": "123_abc_.!@#$%^&*:;,./?|+-=()[]{}<>", + "type": "binary" + }, + "tags": { + "t1": true, + "t2": false, + "t3": 10, + "t4": "123_abc_.!@#$%^&*:;,./?|+-=()[]{}<>" + } + } + ''' + code = self._conn.insert_json_payload(payload) + print("insert_json_payload result {}".format(code)) + + tdSql.query("describe stb2_7") + tdSql.checkData(1, 1, "BINARY") + + payload = ''' + { + "metric": "stb2_8", + "timestamp": { + "value": 1626006833, + "type": "s" + }, + "value": { + "value": "你好", + "type": "nchar" + }, + "tags": { + "t1": true, + "t2": false, + "t3": 10, + "t4": "123_abc_.!@#$%^&*:;,./?|+-=()[]{}<>" + } + } + ''' + code = self._conn.insert_json_payload(payload) + print("insert_json_payload result {}".format(code)) + + tdSql.query("describe stb2_8") + tdSql.checkData(1, 1, "NCHAR") + + ### tag value ### + + payload = ''' + { + "metric": "stb3_0", + "timestamp": { + "value": 1626006833, + "type": "s" + }, + "value": { + "value": "hello", + "type": "nchar" + }, + "tags": { + "t1": { + "value": true, + "type": "bool" + }, + "t2": { + "value": 127, + "type": "tinyint" + }, + "t3": { + "value": 32767, + "type": "smallint" + }, + "t4": { + "value": 2147483647, + "type": "int" + }, + "t5": { + "value": 9.2233720368547758e+18, + "type": "bigint" + }, + "t6": { + "value": 11.12345, + "type": "float" + }, + "t7": { + "value": 22.123456789, + "type": "double" + }, + "t8": { + "value": "binary_val", + "type": "binary" + }, + "t9": { + "value": "你好", + "type": "nchar" + } + } + } + ''' + code = self._conn.insert_json_payload(payload) + print("insert_json_payload result {}".format(code)) + + tdSql.query("describe stb3_0") + tdSql.checkData(2, 1, "BOOL") + tdSql.checkData(3, 1, "TINYINT") + tdSql.checkData(4, 1, "SMALLINT") + tdSql.checkData(5, 1, "INT") + tdSql.checkData(6, 1, "BIGINT") + tdSql.checkData(7, 1, "FLOAT") + tdSql.checkData(8, 1, "DOUBLE") + tdSql.checkData(9, 1, "BINARY") + tdSql.checkData(10, 1, "NCHAR") + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) From 10b6eafca7dae65b7fbc9d00f38d5987654e5ea4 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Thu, 9 Sep 2021 09:22:13 +0800 Subject: [PATCH 38/39] [TD-6560]: Add unit test cases for parsing OpenTSDB HTTP JSON data import format --- tests/examples/c/makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/examples/c/makefile b/tests/examples/c/makefile index 304623c27a..c85eb4adc5 100644 --- a/tests/examples/c/makefile +++ b/tests/examples/c/makefile @@ -6,8 +6,8 @@ TARGET=exe LFLAGS = '-Wl,-rpath,/usr/local/taos/driver/' -ltaos -lpthread -lm -lrt CFLAGS = -O3 -g -Wall -Wno-deprecated -fPIC -Wno-unused-result -Wconversion \ -Wno-char-subscripts -D_REENTRANT -Wno-format -D_REENTRANT -DLINUX \ - -Wno-unused-function -D_M_X64 -I/usr/local/taos/include -std=gnu99 - + -Wno-unused-function -D_M_X64 -I/usr/local/taos/include -std=gnu99 \ + -I../../../deps/cJson/inc all: $(TARGET) exe: From 3fd7e3511b24f129256c4c3957eba342bd37a327 Mon Sep 17 00:00:00 2001 From: Zhiyu Yang <69311263+zyyang-taosdata@users.noreply.github.com> Date: Mon, 13 Sep 2021 10:15:01 +0800 Subject: [PATCH 39/39] Enh/ts-339 refactor code in springbootdemo and JDBCDemo (#7867) * fix confilct * refactor code in springbootdemo --- tests/examples/JDBC/JDBCDemo/pom.xml | 2 +- .../java/com/taosdata/example/JdbcDemo.java | 18 +++---- .../com/taosdata/example/JdbcRestfulDemo.java | 23 ++++---- .../com/taosdata/example/SubscribeDemo.java | 3 +- tests/examples/JDBC/springbootdemo/pom.xml | 9 ++-- .../SpringbootdemoApplication.java | 2 +- .../controller/WeatherController.java | 24 ++------- .../springbootdemo/dao/WeatherMapper.java | 2 + .../springbootdemo/dao/WeatherMapper.xml | 54 +++++++++++++++---- .../springbootdemo/domain/Weather.java | 9 ++++ .../service/WeatherService.java | 18 +++++++ .../springbootdemo/util/TaosAspect.java | 36 +++++++++++++ .../src/main/resources/application.properties | 10 ++-- 13 files changed, 144 insertions(+), 66 deletions(-) create mode 100644 tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/util/TaosAspect.java diff --git a/tests/examples/JDBC/JDBCDemo/pom.xml b/tests/examples/JDBC/JDBCDemo/pom.xml index fed00c147b..8cf0356721 100644 --- a/tests/examples/JDBC/JDBCDemo/pom.xml +++ b/tests/examples/JDBC/JDBCDemo/pom.xml @@ -17,7 +17,7 @@ com.taosdata.jdbc taos-jdbcdriver - 2.0.31 + 2.0.34 diff --git a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/JdbcDemo.java b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/JdbcDemo.java index d4ea5f919d..5bc2340308 100644 --- a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/JdbcDemo.java +++ b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/JdbcDemo.java @@ -7,6 +7,9 @@ public class JdbcDemo { private static String host; private static final String dbName = "test"; private static final String tbName = "weather"; + private static final String user = "root"; + private static final String password = "taosdata"; + private Connection connection; public static void main(String[] args) { @@ -30,10 +33,9 @@ public class JdbcDemo { } private void init() { - final String url = "jdbc:TAOS://" + host + ":6030/?user=root&password=taosdata"; + final String url = "jdbc:TAOS://" + host + ":6030/?user=" + user + "&password=" + password; // get connection try { - Class.forName("com.taosdata.jdbc.TSDBDriver"); Properties properties = new Properties(); properties.setProperty("charset", "UTF-8"); properties.setProperty("locale", "en_US.UTF-8"); @@ -42,8 +44,7 @@ public class JdbcDemo { connection = DriverManager.getConnection(url, properties); if (connection != null) System.out.println("[ OK ] Connection established."); - } catch (ClassNotFoundException | SQLException e) { - System.out.println("[ ERROR! ] Connection establish failed."); + } catch (SQLException e) { e.printStackTrace(); } } @@ -74,7 +75,7 @@ public class JdbcDemo { } private void select() { - final String sql = "select * from "+ dbName + "." + tbName; + final String sql = "select * from " + dbName + "." + tbName; executeQuery(sql); } @@ -89,8 +90,6 @@ public class JdbcDemo { } } - /************************************************************************/ - private void executeQuery(String sql) { long start = System.currentTimeMillis(); try (Statement statement = connection.createStatement()) { @@ -117,7 +116,6 @@ public class JdbcDemo { } } - private void printSql(String sql, boolean succeed, long cost) { System.out.println("[ " + (succeed ? "OK" : "ERROR!") + " ] time cost: " + cost + " ms, execute statement ====> " + sql); } @@ -132,7 +130,6 @@ public class JdbcDemo { long end = System.currentTimeMillis(); printSql(sql, false, (end - start)); e.printStackTrace(); - } } @@ -141,5 +138,4 @@ public class JdbcDemo { System.exit(0); } - -} \ No newline at end of file +} diff --git a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/JdbcRestfulDemo.java b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/JdbcRestfulDemo.java index 5bf980f6d8..d89476b8ca 100644 --- a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/JdbcRestfulDemo.java +++ b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/JdbcRestfulDemo.java @@ -4,14 +4,15 @@ import java.sql.*; import java.util.Properties; public class JdbcRestfulDemo { - private static final String host = "127.0.0.1"; + private static final String host = "localhost"; + private static final String dbname = "test"; + private static final String user = "root"; + private static final String password = "taosdata"; public static void main(String[] args) { try { - // load JDBC-restful driver - Class.forName("com.taosdata.jdbc.rs.RestfulDriver"); // use port 6041 in url when use JDBC-restful - String url = "jdbc:TAOS-RS://" + host + ":6041/?user=root&password=taosdata"; + String url = "jdbc:TAOS-RS://" + host + ":6041/?user=" + user + "&password=" + password; Properties properties = new Properties(); properties.setProperty("charset", "UTF-8"); @@ -21,12 +22,12 @@ public class JdbcRestfulDemo { Connection conn = DriverManager.getConnection(url, properties); Statement stmt = conn.createStatement(); - stmt.execute("drop database if exists restful_test"); - stmt.execute("create database if not exists restful_test"); - stmt.execute("use restful_test"); - stmt.execute("create table restful_test.weather(ts timestamp, temperature float) tags(location nchar(64))"); - stmt.executeUpdate("insert into t1 using restful_test.weather tags('北京') values(now, 18.2)"); - ResultSet rs = stmt.executeQuery("select * from restful_test.weather"); + stmt.execute("drop database if exists " + dbname); + stmt.execute("create database if not exists " + dbname); + stmt.execute("use " + dbname); + stmt.execute("create table " + dbname + ".weather(ts timestamp, temperature float) tags(location nchar(64))"); + stmt.executeUpdate("insert into t1 using " + dbname + ".weather tags('北京') values(now, 18.2)"); + ResultSet rs = stmt.executeQuery("select * from " + dbname + ".weather"); ResultSetMetaData meta = rs.getMetaData(); while (rs.next()) { for (int i = 1; i <= meta.getColumnCount(); i++) { @@ -38,8 +39,6 @@ public class JdbcRestfulDemo { rs.close(); stmt.close(); conn.close(); - } catch (ClassNotFoundException e) { - e.printStackTrace(); } catch (SQLException e) { e.printStackTrace(); } diff --git a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/SubscribeDemo.java b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/SubscribeDemo.java index def4c64902..4c499b0b3a 100644 --- a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/SubscribeDemo.java +++ b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/SubscribeDemo.java @@ -34,9 +34,8 @@ public class SubscribeDemo { System.out.println(usage); return; } - /*********************************************************************************************/ + try { - Class.forName("com.taosdata.jdbc.TSDBDriver"); Properties properties = new Properties(); properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); diff --git a/tests/examples/JDBC/springbootdemo/pom.xml b/tests/examples/JDBC/springbootdemo/pom.xml index 6c83718896..9126813b67 100644 --- a/tests/examples/JDBC/springbootdemo/pom.xml +++ b/tests/examples/JDBC/springbootdemo/pom.xml @@ -60,12 +60,15 @@ + + org.springframework.boot + spring-boot-starter-aop + + com.taosdata.jdbc taos-jdbcdriver - 2.0.28 - - + 2.0.34 diff --git a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/SpringbootdemoApplication.java b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/SpringbootdemoApplication.java index fa10f3b092..53edaa5796 100644 --- a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/SpringbootdemoApplication.java +++ b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/SpringbootdemoApplication.java @@ -4,7 +4,7 @@ import org.mybatis.spring.annotation.MapperScan; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; -@MapperScan(basePackages = {"com.taosdata.example.springbootdemo.dao"}) +@MapperScan(basePackages = {"com.taosdata.example.springbootdemo"}) @SpringBootApplication public class SpringbootdemoApplication { diff --git a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/controller/WeatherController.java b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/controller/WeatherController.java index cf14f5d84a..ed720fe6c0 100644 --- a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/controller/WeatherController.java +++ b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/controller/WeatherController.java @@ -15,35 +15,21 @@ public class WeatherController { @Autowired private WeatherService weatherService; - /** - * create database and table - * - * @return - */ + @GetMapping("/lastOne") + public Weather lastOne() { + return weatherService.lastOne(); + } + @GetMapping("/init") public int init() { return weatherService.init(); } - /** - * Pagination Query - * - * @param limit - * @param offset - * @return - */ @GetMapping("/{limit}/{offset}") public List queryWeather(@PathVariable Long limit, @PathVariable Long offset) { return weatherService.query(limit, offset); } - /** - * upload single weather info - * - * @param temperature - * @param humidity - * @return - */ @PostMapping("/{temperature}/{humidity}") public int saveWeather(@PathVariable float temperature, @PathVariable float humidity) { return weatherService.save(temperature, humidity); diff --git a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/dao/WeatherMapper.java b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/dao/WeatherMapper.java index ad6733558a..d9202b45b4 100644 --- a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/dao/WeatherMapper.java +++ b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/dao/WeatherMapper.java @@ -8,6 +8,8 @@ import java.util.Map; public interface WeatherMapper { + Map lastOne(); + void dropDB(); void createDB(); diff --git a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/dao/WeatherMapper.xml b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/dao/WeatherMapper.xml index 2d3e054065..91938ca24e 100644 --- a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/dao/WeatherMapper.xml +++ b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/dao/WeatherMapper.xml @@ -9,20 +9,48 @@ + + - drop database if exists test + drop + database if exists test - create database if not exists test + create + database if not exists test - create table if not exists test.weather(ts timestamp, temperature float, humidity float) tags(location nchar(64), groupId int) + create table if not exists test.weather + ( + ts + timestamp, + temperature + float, + humidity + float, + note + binary + ( + 64 + )) tags + ( + location nchar + ( + 64 + ), groupId int) - create table if not exists test.t#{groupId} using test.weather tags(#{location}, #{groupId}) + create table if not exists test.t#{groupId} using test.weather tags + ( + #{location}, + #{groupId} + ) - insert into test.t#{groupId} (ts, temperature, humidity) values (#{ts}, ${temperature}, ${humidity}) + insert into test.t#{groupId} (ts, temperature, humidity, note) + values (#{ts}, ${temperature}, ${humidity}, #{note}) - - - + + + \ No newline at end of file diff --git a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/domain/Weather.java b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/domain/Weather.java index c11b9a6f50..e4238127bd 100644 --- a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/domain/Weather.java +++ b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/domain/Weather.java @@ -11,6 +11,7 @@ public class Weather { private Float temperature; private Float humidity; private String location; + private String note; private int groupId; public Weather() { @@ -61,4 +62,12 @@ public class Weather { public void setGroupId(int groupId) { this.groupId = groupId; } + + public String getNote() { + return note; + } + + public void setNote(String note) { + this.note = note; + } } diff --git a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/service/WeatherService.java b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/service/WeatherService.java index 26d09c7d12..2264b200af 100644 --- a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/service/WeatherService.java +++ b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/service/WeatherService.java @@ -29,6 +29,7 @@ public class WeatherService { Weather weather = new Weather(new Timestamp(ts + (thirtySec * i)), 30 * random.nextFloat(), random.nextInt(100)); weather.setLocation(locations[random.nextInt(locations.length)]); weather.setGroupId(i % locations.length); + weather.setNote("note-" + i); weatherMapper.createTable(weather); count += weatherMapper.insert(weather); } @@ -58,4 +59,21 @@ public class WeatherService { public List avg() { return weatherMapper.avg(); } + + public Weather lastOne() { + Map result = weatherMapper.lastOne(); + + long ts = (long) result.get("ts"); + float temperature = (float) result.get("temperature"); + float humidity = (float) result.get("humidity"); + String note = (String) result.get("note"); + int groupId = (int) result.get("groupid"); + String location = (String) result.get("location"); + + Weather weather = new Weather(new Timestamp(ts), temperature, humidity); + weather.setNote(note); + weather.setGroupId(groupId); + weather.setLocation(location); + return weather; + } } diff --git a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/util/TaosAspect.java b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/util/TaosAspect.java new file mode 100644 index 0000000000..80dad1bd7d --- /dev/null +++ b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/util/TaosAspect.java @@ -0,0 +1,36 @@ +package com.taosdata.example.springbootdemo.util; + +import org.aspectj.lang.ProceedingJoinPoint; +import org.aspectj.lang.annotation.Around; +import org.aspectj.lang.annotation.Aspect; +import org.springframework.stereotype.Component; + +import java.sql.Timestamp; +import java.util.Map; + +@Aspect +@Component +public class TaosAspect { + + @Around("execution(java.util.Map com.taosdata.example.springbootdemo.dao.*.*(..))") + public Object handleType(ProceedingJoinPoint joinPoint) { + Map result = null; + try { + result = (Map) joinPoint.proceed(); + for (String key : result.keySet()) { + Object obj = result.get(key); + if (obj instanceof byte[]) { + obj = new String((byte[]) obj); + result.put(key, obj); + } + if (obj instanceof Timestamp) { + obj = ((Timestamp) obj).getTime(); + result.put(key, obj); + } + } + } catch (Throwable e) { + e.printStackTrace(); + } + return result; + } +} diff --git a/tests/examples/JDBC/springbootdemo/src/main/resources/application.properties b/tests/examples/JDBC/springbootdemo/src/main/resources/application.properties index 4d7e64d105..06daa81bbb 100644 --- a/tests/examples/JDBC/springbootdemo/src/main/resources/application.properties +++ b/tests/examples/JDBC/springbootdemo/src/main/resources/application.properties @@ -1,22 +1,20 @@ # datasource config - JDBC-JNI #spring.datasource.driver-class-name=com.taosdata.jdbc.TSDBDriver -#spring.datasource.url=jdbc:TAOS://127.0.0.1:6030/test?timezone=UTC-8&charset=UTF-8&locale=en_US.UTF-8 +#spring.datasource.url=jdbc:TAOS://localhost:6030/?timezone=UTC-8&charset=UTF-8&locale=en_US.UTF-8 #spring.datasource.username=root #spring.datasource.password=taosdata - # datasource config - JDBC-RESTful spring.datasource.driver-class-name=com.taosdata.jdbc.rs.RestfulDriver -spring.datasource.url=jdbc:TAOS-RS://master:6041/test?timezone=UTC-8&charset=UTF-8&locale=en_US.UTF-8 +spring.datasource.url=jdbc:TAOS-RS://localhsot:6041/test?timezone=UTC-8&charset=UTF-8&locale=en_US.UTF-8 spring.datasource.username=root spring.datasource.password=taosdata - spring.datasource.druid.initial-size=5 spring.datasource.druid.min-idle=5 spring.datasource.druid.max-active=5 spring.datasource.druid.max-wait=30000 spring.datasource.druid.validation-query=select server_status(); - +spring.aop.auto=true +spring.aop.proxy-target-class=true #mybatis mybatis.mapper-locations=classpath:mapper/*.xml - logging.level.com.taosdata.jdbc.springbootdemo.dao=debug