From 9eff7ff43a2e9ca0cf9a91b0c380944e2abb404b Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 25 Apr 2022 15:54:34 +0800 Subject: [PATCH 01/31] refactor: schemaless function --- include/util/taoserror.h | 4 + source/client/inc/tscParseLine.h | 119 + source/client/schemaless/CMakeLists.txt | 17 + .../client/schemaless/src/tscParseOpenTSDB.c | 1113 ++++++ source/client/src/tscParseLineProtocol.c | 3134 +++++++++++++++++ source/libs/CMakeLists.txt | 2 +- .../libs/scalar/test/scalar/scalarTests.cpp | 6 +- source/util/src/terror.c | 5 + 8 files changed, 4396 insertions(+), 4 deletions(-) create mode 100644 source/client/inc/tscParseLine.h create mode 100644 source/client/schemaless/CMakeLists.txt create mode 100644 source/client/schemaless/src/tscParseOpenTSDB.c create mode 100644 source/client/src/tscParseLineProtocol.c diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 215e83c7d4..6147b3495a 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -625,6 +625,10 @@ int32_t* taosGetErrno(); #define TSDB_CODE_FUNC_FUNTION_PARA_TYPE TAOS_DEF_ERROR_CODE(0, 0x2802) #define TSDB_CODE_FUNC_FUNTION_PARA_VALUE TAOS_DEF_ERROR_CODE(0, 0x2803) +#define TSDB_CODE_SML_INVALID_PROTOCOL_TYPE TAOS_DEF_ERROR_CODE(0, 0x3000) +#define TSDB_CODE_SML_INVALID_PRECISION_TYPE TAOS_DEF_ERROR_CODE(0, 0x3001) +#define TSDB_CODE_SML_INVALID_DATA TAOS_DEF_ERROR_CODE(0, 0x3002) + #ifdef __cplusplus } #endif diff --git a/source/client/inc/tscParseLine.h b/source/client/inc/tscParseLine.h new file mode 100644 index 0000000000..b70280f380 --- /dev/null +++ b/source/client/inc/tscParseLine.h @@ -0,0 +1,119 @@ +/* + * Copyright (c) 2021 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_TSCPARSELINE_H +#define TDENGINE_TSCPARSELINE_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include "thash.h" +#include "clientint.h" + +#define SML_TIMESTAMP_SECOND_DIGITS 10 +#define SML_TIMESTAMP_MILLI_SECOND_DIGITS 13 + +typedef TSDB_SML_PROTOCOL_TYPE SMLProtocolType; + +typedef struct { + const char* key; + int32_t keyLen; + uint8_t type; + int16_t length; + const char* value; + int32_t valueLen; +} TAOS_SML_KV; + +typedef struct { + const char* measure; + const char* tags; + const char* cols; + const char* timestamp; + + int32_t measureLen; + int32_t measureTagsLen; + int32_t tagsLen; + int32_t colsLen; +} TAOS_PARSE_ELEMENTS; + +typedef struct { + char* childTableName; + + SArray* tags; + SArray *cols; +} TAOS_SML_DATA_POINT_TAGS; + +typedef struct SSmlSTableMeta { + char *sTableName; // super table name + uint8_t sTableNameLen; + uint8_t precision; // the number of precision + SHashObj* tagHash; + SHashObj* fieldHash; +} SSmlSTableMeta; + +typedef enum { + SML_TIME_STAMP_NOT_CONFIGURED, + SML_TIME_STAMP_HOURS, + SML_TIME_STAMP_MINUTES, + SML_TIME_STAMP_SECONDS, + SML_TIME_STAMP_MILLI_SECONDS, + SML_TIME_STAMP_MICRO_SECONDS, + SML_TIME_STAMP_NANO_SECONDS, + SML_TIME_STAMP_NOW +} SMLTimeStampType; + +typedef struct { + uint64_t id; + + STscObj* taos; + SCatalog* pCatalog; + + SMLProtocolType protocol; + SMLTimeStampType tsType; + + int32_t affectedRows; + + SHashObj* childTables; + SHashObj* superTables; +} SSmlLinesInfo; + +int tscSmlInsert(TAOS* taos, TAOS_SML_DATA_POINT* points, int numPoint, SSmlLinesInfo* info); +bool checkDuplicateKey(char *key, SHashObj *pHash, SSmlLinesInfo* info); +bool isValidInteger(char *str); +bool isValidFloat(char *str); + +int32_t isValidChildTableName(const char *pTbName, int16_t len, SSmlLinesInfo* info); + +bool convertSmlValueType(TAOS_SML_KV *pVal, char *value, + uint16_t len, SSmlLinesInfo* info, bool isTag); +int32_t convertSmlTimeStamp(TAOS_SML_KV *pVal, char *value, + uint16_t len, SSmlLinesInfo* info); + +void destroySmlDataPoint(TAOS_SML_DATA_POINT* point); + +int taos_insert_lines(TAOS* taos, char* lines[], int numLines, SMLProtocolType protocol, + SMLTimeStampType tsType, int* affectedRows); +int taos_insert_telnet_lines(TAOS* taos, char* lines[], int numLines, SMLProtocolType protocol, + SMLTimeStampType tsType, int* affectedRows); +int taos_insert_json_payload(TAOS* taos, char* payload, SMLProtocolType protocol, + SMLTimeStampType tsType, int* affectedRows); + + +#ifdef __cplusplus +} +#endif + +#endif // TDENGINE_TSCPARSELINE_H diff --git a/source/client/schemaless/CMakeLists.txt b/source/client/schemaless/CMakeLists.txt new file mode 100644 index 0000000000..29be674934 --- /dev/null +++ b/source/client/schemaless/CMakeLists.txt @@ -0,0 +1,17 @@ +aux_source_directory(src SCHEMALESS_SRC) +add_library(schemaless STATIC ${SCHEMALESS_SRC}) + +target_include_directories( + schemaless + PUBLIC "${TD_SOURCE_DIR}/include/libs/schemaless" + PRIVATE "inc" +) + +target_link_libraries( + schemaless + PUBLIC os util common catalog qcom +) + +if(${BUILD_TEST}) + ADD_SUBDIRECTORY(test) +endif(${BUILD_TEST}) diff --git a/source/client/schemaless/src/tscParseOpenTSDB.c b/source/client/schemaless/src/tscParseOpenTSDB.c new file mode 100644 index 0000000000..5bb9b09cf1 --- /dev/null +++ b/source/client/schemaless/src/tscParseOpenTSDB.c @@ -0,0 +1,1113 @@ +//#include +//#include +//#include +//#include +// +//#include "cJSON.h" +//#include "hash.h" +//#include "taos.h" +// +//#include "tscUtil.h" +//#include "tsclient.h" +//#include "tscLog.h" +// +//#include "tscParseLine.h" +// +//#define OTD_MAX_FIELDS_NUM 2 +//#define OTD_JSON_SUB_FIELDS_NUM 2 +//#define OTD_JSON_FIELDS_NUM 4 +// +//#define OTD_TIMESTAMP_COLUMN_NAME "ts" +//#define OTD_METRIC_VALUE_COLUMN_NAME "value" +// +///* telnet style API parser */ +//static uint64_t HandleId = 0; +// +//static uint64_t genUID() { +// uint64_t id; +// +// do { +// id = atomic_add_fetch_64(&HandleId, 1); +// } while (id == 0); +// +// return id; +//} +// +//static int32_t parseTelnetMetric(TAOS_SML_DATA_POINT *pSml, const char **index, SSmlLinesInfo* info) { +// const char *cur = *index; +// uint16_t len = 0; +// +// pSml->stableName = tcalloc(TSDB_TABLE_NAME_LEN + TS_BACKQUOTE_CHAR_SIZE, 1); +// if (pSml->stableName == NULL) { +// return TSDB_CODE_TSC_OUT_OF_MEMORY; +// } +// /* +// if (isdigit(*cur)) { +// tscError("OTD:0x%"PRIx64" Metric cannot start with digit", info->id); +// tfree(pSml->stableName); +// return TSDB_CODE_TSC_LINE_SYNTAX_ERROR; +// } +// */ +// +// while (*cur != '\0') { +// if (len > TSDB_TABLE_NAME_LEN - 1) { +// tscError("OTD:0x%"PRIx64" Metric cannot exceeds %d characters", info->id, TSDB_TABLE_NAME_LEN - 1); +// tfree(pSml->stableName); +// return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; +// } +// +// if (*cur == ' ') { +// if (*(cur + 1) != ' ') { +// break; +// } else { +// cur++; +// continue; +// } +// } +// +// pSml->stableName[len] = *cur; +// +// cur++; +// len++; +// } +// if (len == 0 || *cur == '\0') { +// tfree(pSml->stableName); +// return TSDB_CODE_TSC_LINE_SYNTAX_ERROR; +// } +// +// addEscapeCharToString(pSml->stableName, len); +// *index = cur + 1; +// tscDebug("OTD:0x%"PRIx64" Stable name in metric:%s|len:%d", info->id, pSml->stableName, len); +// +// return TSDB_CODE_SUCCESS; +//} +// +//static int32_t parseTelnetTimeStamp(TAOS_SML_KV **pTS, int *num_kvs, const char **index, SSmlLinesInfo* info) { +// //Timestamp must be the first KV to parse +// assert(*num_kvs == 0); +// +// const char *start, *cur; +// int32_t ret = TSDB_CODE_SUCCESS; +// int len = 0; +// char key[] = OTD_TIMESTAMP_COLUMN_NAME; +// char *value = NULL; +// +// start = cur = *index; +// //allocate fields for timestamp and value +// *pTS = tcalloc(OTD_MAX_FIELDS_NUM, sizeof(TAOS_SML_KV)); +// +// while(*cur != '\0') { +// if (*cur == ' ') { +// if (*(cur + 1) != ' ') { +// break; +// } else { +// cur++; +// continue; +// } +// } +// cur++; +// len++; +// } +// +// if (len > 0 && *cur != '\0') { +// value = tcalloc(len + 1, 1); +// memcpy(value, start, len); +// } else { +// tfree(*pTS); +// return TSDB_CODE_TSC_LINE_SYNTAX_ERROR; +// } +// +// ret = convertSmlTimeStamp(*pTS, value, len, info); +// if (ret) { +// tfree(value); +// tfree(*pTS); +// return ret; +// } +// tfree(value); +// +// (*pTS)->key = tcalloc(sizeof(key) + TS_BACKQUOTE_CHAR_SIZE, 1); +// memcpy((*pTS)->key, key, sizeof(key)); +// addEscapeCharToString((*pTS)->key, (int32_t)strlen(key)); +// +// *num_kvs += 1; +// *index = cur + 1; +// +// return ret; +//} +// +//static int32_t parseTelnetMetricValue(TAOS_SML_KV **pKVs, int *num_kvs, const char **index, SSmlLinesInfo* info) { +// //skip timestamp +// TAOS_SML_KV *pVal = *pKVs + 1; +// const char *start, *cur; +// int32_t ret = TSDB_CODE_SUCCESS; +// int len = 0; +// bool searchQuote = false; +// char key[] = OTD_METRIC_VALUE_COLUMN_NAME; +// char *value = NULL; +// +// start = cur = *index; +// +// //if metric value is string +// if (*cur == '"') { +// searchQuote = true; +// cur += 1; +// len += 1; +// } else if (*cur == 'L' && *(cur + 1) == '"') { +// searchQuote = true; +// cur += 2; +// len += 2; +// } +// +// while(*cur != '\0') { +// if (*cur == ' ') { +// if (searchQuote == true) { +// if (*(cur - 1) == '"' && len != 1 && len != 2) { +// searchQuote = false; +// } else { +// cur++; +// len++; +// continue; +// } +// } +// +// if (*(cur + 1) != ' ') { +// break; +// } else { +// cur++; +// continue; +// } +// } +// cur++; +// len++; +// } +// +// if (len > 0 && *cur != '\0') { +// value = tcalloc(len + 1, 1); +// memcpy(value, start, len); +// } else { +// return TSDB_CODE_TSC_LINE_SYNTAX_ERROR; +// } +// +// if (!convertSmlValueType(pVal, value, len, info, false)) { +// tscError("OTD:0x%"PRIx64" Failed to convert metric value string(%s) to any type", +// info->id, value); +// tfree(value); +// return TSDB_CODE_TSC_INVALID_VALUE; +// } +// tfree(value); +// +// pVal->key = tcalloc(sizeof(key) + TS_BACKQUOTE_CHAR_SIZE, 1); +// memcpy(pVal->key, key, sizeof(key)); +// addEscapeCharToString(pVal->key, (int32_t)strlen(pVal->key)); +// *num_kvs += 1; +// +// *index = cur + 1; +// return ret; +//} +// +//static int32_t parseTelnetTagKey(TAOS_SML_KV *pKV, const char **index, SHashObj *pHash, SSmlLinesInfo* info) { +// const char *cur = *index; +// char key[TSDB_COL_NAME_LEN]; +// uint16_t len = 0; +// +// //key field cannot start with digit +// //if (isdigit(*cur)) { +// // tscError("OTD:0x%"PRIx64" Tag key cannot start with digit", info->id); +// // return TSDB_CODE_TSC_LINE_SYNTAX_ERROR; +// //} +// while (*cur != '\0') { +// if (len > TSDB_COL_NAME_LEN - 1) { +// tscError("OTD:0x%"PRIx64" Tag key cannot exceeds %d characters", info->id, TSDB_COL_NAME_LEN - 1); +// return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; +// } +// if (*cur == ' ') { +// return TSDB_CODE_TSC_LINE_SYNTAX_ERROR; +// } +// if (*cur == '=') { +// break; +// } +// +// key[len] = *cur; +// cur++; +// len++; +// } +// if (len == 0 || *cur == '\0') { +// return TSDB_CODE_TSC_LINE_SYNTAX_ERROR; +// } +// key[len] = '\0'; +// +// if (checkDuplicateKey(key, pHash, info)) { +// return TSDB_CODE_TSC_DUP_TAG_NAMES; +// } +// +// pKV->key = tcalloc(len + TS_BACKQUOTE_CHAR_SIZE + 1, 1); +// memcpy(pKV->key, key, len + 1); +// addEscapeCharToString(pKV->key, len); +// //tscDebug("OTD:0x%"PRIx64" Key:%s|len:%d", info->id, pKV->key, len); +// *index = cur + 1; +// return TSDB_CODE_SUCCESS; +//} +// +// +//static int32_t parseTelnetTagValue(TAOS_SML_KV *pKV, const char **index, +// bool *is_last_kv, SSmlLinesInfo* info) { +// const char *start, *cur; +// char *value = NULL; +// uint16_t len = 0; +// start = cur = *index; +// +// while (1) { +// // whitespace or '\0' identifies a value +// if (*cur == ' ' || *cur == '\0') { +// // '\0' indicates end of value +// *is_last_kv = (*cur == '\0') ? true : false; +// if (*cur == ' ' && *(cur + 1) == ' ') { +// cur++; +// continue; +// } else { +// break; +// } +// } +// cur++; +// len++; +// } +// +// if (len == 0) { +// tfree(pKV->key); +// return TSDB_CODE_TSC_LINE_SYNTAX_ERROR; +// } +// +// value = tcalloc(len + 1, 1); +// memcpy(value, start, len); +// value[len] = '\0'; +// if (!convertSmlValueType(pKV, value, len, info, true)) { +// tscError("OTD:0x%"PRIx64" Failed to convert sml value string(%s) to any type", +// info->id, value); +// //free previous alocated key field +// tfree(pKV->key); +// tfree(value); +// return TSDB_CODE_TSC_INVALID_VALUE; +// } +// tfree(value); +// +// *index = (*cur == '\0') ? cur : cur + 1; +// return TSDB_CODE_SUCCESS; +//} +// +//static int32_t parseTelnetTagKvs(TAOS_SML_KV **pKVs, int *num_kvs, +// const char **index, char **childTableName, +// SHashObj *pHash, SSmlLinesInfo* info) { +// const char *cur = *index; +// int32_t ret = TSDB_CODE_SUCCESS; +// TAOS_SML_KV *pkv; +// bool is_last_kv = false; +// +// int32_t capacity = 4; +// *pKVs = tcalloc(capacity, sizeof(TAOS_SML_KV)); +// pkv = *pKVs; +// +// size_t childTableNameLen = strlen(tsSmlChildTableName); +// char childTbName[TSDB_TABLE_NAME_LEN + TS_BACKQUOTE_CHAR_SIZE] = {0}; +// if (childTableNameLen != 0) { +// memcpy(childTbName, tsSmlChildTableName, childTableNameLen); +// addEscapeCharToString(childTbName, (int32_t)(childTableNameLen)); +// } +// while (*cur != '\0') { +// ret = parseTelnetTagKey(pkv, &cur, pHash, info); +// if (ret) { +// tscError("OTD:0x%"PRIx64" Unable to parse key", info->id); +// return ret; +// } +// ret = parseTelnetTagValue(pkv, &cur, &is_last_kv, info); +// if (ret) { +// tscError("OTD:0x%"PRIx64" Unable to parse value", info->id); +// return ret; +// } +// if (childTableNameLen != 0 && strcasecmp(pkv->key, childTbName) == 0) { +// *childTableName = tcalloc(pkv->length + TS_BACKQUOTE_CHAR_SIZE + 1, 1); +// memcpy(*childTableName, pkv->value, pkv->length); +// (*childTableName)[pkv->length] = '\0'; +// addEscapeCharToString(*childTableName, pkv->length); +// tfree(pkv->key); +// tfree(pkv->value); +// } else { +// *num_kvs += 1; +// } +// +// if (is_last_kv) { +// break; +// } +// +// //reallocate addtional memory for more kvs +// if ((*num_kvs + 1) > capacity) { +// TAOS_SML_KV *more_kvs = NULL; +// capacity *= 3; capacity /= 2; +// more_kvs = realloc(*pKVs, capacity * sizeof(TAOS_SML_KV)); +// if (!more_kvs) { +// return TSDB_CODE_TSC_OUT_OF_MEMORY; +// } +// *pKVs = more_kvs; +// } +// +// //move pKV points to next TAOS_SML_KV block +// pkv = *pKVs + *num_kvs; +// } +// +// return ret; +//} +// +//static int32_t tscParseTelnetLine(const char* line, TAOS_SML_DATA_POINT* smlData, SSmlLinesInfo* info) { +// const char* index = line; +// int32_t ret = TSDB_CODE_SUCCESS; +// +// //Parse metric +// ret = parseTelnetMetric(smlData, &index, info); +// if (ret) { +// tscError("OTD:0x%"PRIx64" Unable to parse metric", info->id); +// return ret; +// } +// tscDebug("OTD:0x%"PRIx64" Parse metric finished", info->id); +// +// //Parse timestamp +// ret = parseTelnetTimeStamp(&smlData->fields, &smlData->fieldNum, &index, info); +// if (ret) { +// tscError("OTD:0x%"PRIx64" Unable to parse timestamp", info->id); +// return ret; +// } +// tscDebug("OTD:0x%"PRIx64" Parse timestamp finished", info->id); +// +// //Parse value +// ret = parseTelnetMetricValue(&smlData->fields, &smlData->fieldNum, &index, info); +// if (ret) { +// tscError("OTD:0x%"PRIx64" Unable to parse metric value", info->id); +// return ret; +// } +// tscDebug("OTD:0x%"PRIx64" Parse metric value finished", info->id); +// +// //Parse tagKVs +// SHashObj *keyHashTable = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); +// ret = parseTelnetTagKvs(&smlData->tags, &smlData->tagNum, &index, &smlData->childTableName, keyHashTable, info); +// if (ret) { +// tscError("OTD:0x%"PRIx64" Unable to parse tags", info->id); +// taosHashCleanup(keyHashTable); +// return ret; +// } +// tscDebug("OTD:0x%"PRIx64" Parse tags finished", info->id); +// taosHashCleanup(keyHashTable); +// +// +// return TSDB_CODE_SUCCESS; +//} +// +//static int32_t tscParseTelnetLines(char* lines[], int numLines, SArray* points, SArray* failedLines, SSmlLinesInfo* info) { +// for (int32_t i = 0; i < numLines; ++i) { +// TAOS_SML_DATA_POINT point = {0}; +// int32_t code = tscParseTelnetLine(lines[i], &point, info); +// if (code != TSDB_CODE_SUCCESS) { +// tscError("OTD:0x%"PRIx64" data point line parse failed. line %d : %s", info->id, i, lines[i]); +// destroySmlDataPoint(&point); +// return code; +// } else { +// tscDebug("OTD:0x%"PRIx64" data point line parse success. line %d", info->id, i); +// } +// +// taosArrayPush(points, &point); +// } +// return TSDB_CODE_SUCCESS; +//} +// +//int taos_insert_telnet_lines(TAOS* taos, char* lines[], int numLines, SMLProtocolType protocol, SMLTimeStampType tsType, int* affectedRows) { +// int32_t code = 0; +// +// SSmlLinesInfo* info = tcalloc(1, sizeof(SSmlLinesInfo)); +// info->id = genUID(); +// info->tsType = tsType; +// info->protocol = protocol; +// +// if (numLines <= 0 || numLines > 65536) { +// tscError("OTD:0x%"PRIx64" taos_insert_telnet_lines numLines should be between 1 and 65536. numLines: %d", info->id, numLines); +// tfree(info); +// code = TSDB_CODE_TSC_APP_ERROR; +// return code; +// } +// +// for (int i = 0; i < numLines; ++i) { +// if (lines[i] == NULL) { +// tscError("OTD:0x%"PRIx64" taos_insert_telnet_lines line %d is NULL", info->id, i); +// tfree(info); +// code = TSDB_CODE_TSC_APP_ERROR; +// return code; +// } +// } +// +// SArray* lpPoints = taosArrayInit(numLines, sizeof(TAOS_SML_DATA_POINT)); +// if (lpPoints == NULL) { +// tscError("OTD:0x%"PRIx64" taos_insert_telnet_lines failed to allocate memory", info->id); +// tfree(info); +// return TSDB_CODE_TSC_OUT_OF_MEMORY; +// } +// +// tscDebug("OTD:0x%"PRIx64" taos_insert_telnet_lines begin inserting %d lines, first line: %s", info->id, numLines, lines[0]); +// code = tscParseTelnetLines(lines, numLines, lpPoints, NULL, info); +// size_t numPoints = taosArrayGetSize(lpPoints); +// +// if (code != 0) { +// goto cleanup; +// } +// +// TAOS_SML_DATA_POINT* points = TARRAY_GET_START(lpPoints); +// code = tscSmlInsert(taos, points, (int)numPoints, info); +// if (code != 0) { +// tscError("OTD:0x%"PRIx64" taos_insert_telnet_lines error: %s", info->id, tstrerror((code))); +// } +// if (affectedRows != NULL) { +// *affectedRows = info->affectedRows; +// } +// +//cleanup: +// tscDebug("OTD:0x%"PRIx64" taos_insert_telnet_lines finish inserting %d lines. code: %d", info->id, numLines, code); +// points = TARRAY_GET_START(lpPoints); +// numPoints = taosArrayGetSize(lpPoints); +// for (int i = 0; i < numPoints; ++i) { +// destroySmlDataPoint(points+i); +// } +// +// taosArrayDestroy(&lpPoints); +// +// tfree(info); +// return code; +//} +// +//int taos_telnet_insert(TAOS* taos, TAOS_SML_DATA_POINT* points, int numPoint) { +// SSmlLinesInfo* info = tcalloc(1, sizeof(SSmlLinesInfo)); +// info->id = genUID(); +// int code = tscSmlInsert(taos, points, numPoint, info); +// tfree(info); +// return code; +//} +// +// +///* telnet style API parser */ +//static int32_t parseMetricFromJSON(cJSON *root, TAOS_SML_DATA_POINT* pSml, SSmlLinesInfo* info) { +// cJSON *metric = cJSON_GetObjectItem(root, "metric"); +// if (!cJSON_IsString(metric)) { +// return TSDB_CODE_TSC_INVALID_JSON; +// } +// +// size_t stableLen = strlen(metric->valuestring); +// if (stableLen > TSDB_TABLE_NAME_LEN - 1) { +// tscError("OTD:0x%"PRIx64" Metric cannot exceeds %d characters in JSON", info->id, TSDB_TABLE_NAME_LEN - 1); +// return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; +// } +// +// pSml->stableName = tcalloc(stableLen + TS_BACKQUOTE_CHAR_SIZE + 1, sizeof(char)); +// if (pSml->stableName == NULL){ +// return TSDB_CODE_TSC_OUT_OF_MEMORY; +// } +// +// /* +// if (isdigit(metric->valuestring[0])) { +// tscError("OTD:0x%"PRIx64" Metric cannot start with digit in JSON", info->id); +// tfree(pSml->stableName); +// return TSDB_CODE_TSC_INVALID_JSON; +// } +// */ +// +// tstrncpy(pSml->stableName, metric->valuestring, stableLen + 1); +// addEscapeCharToString(pSml->stableName, (int32_t)stableLen); +// +// return TSDB_CODE_SUCCESS; +// +//} +// +//static int32_t parseTimestampFromJSONObj(cJSON *root, int64_t *tsVal, SSmlLinesInfo* info) { +// int32_t size = cJSON_GetArraySize(root); +// if (size != OTD_JSON_SUB_FIELDS_NUM) { +// return TSDB_CODE_TSC_INVALID_JSON; +// } +// +// cJSON *value = cJSON_GetObjectItem(root, "value"); +// if (!cJSON_IsNumber(value)) { +// return TSDB_CODE_TSC_INVALID_JSON; +// } +// +// cJSON *type = cJSON_GetObjectItem(root, "type"); +// if (!cJSON_IsString(type)) { +// return TSDB_CODE_TSC_INVALID_JSON; +// } +// +// *tsVal = strtoll(value->numberstring, NULL, 10); +// //if timestamp value is 0 use current system time +// if (*tsVal == 0) { +// *tsVal = taosGetTimestampNs(); +// return TSDB_CODE_SUCCESS; +// } +// +// size_t typeLen = strlen(type->valuestring); +// if (typeLen == 1 && type->valuestring[0] == 's') { +// //seconds +// *tsVal = (int64_t)(*tsVal * 1e9); +// } else if (typeLen == 2 && type->valuestring[1] == 's') { +// switch (type->valuestring[0]) { +// case 'm': +// //milliseconds +// *tsVal = convertTimePrecision(*tsVal, TSDB_TIME_PRECISION_MILLI, TSDB_TIME_PRECISION_NANO); +// break; +// case 'u': +// //microseconds +// *tsVal = convertTimePrecision(*tsVal, TSDB_TIME_PRECISION_MICRO, TSDB_TIME_PRECISION_NANO); +// break; +// case 'n': +// //nanoseconds +// *tsVal = *tsVal * 1; +// break; +// default: +// return TSDB_CODE_TSC_INVALID_JSON; +// } +// } else { +// return TSDB_CODE_TSC_INVALID_JSON; +// } +// +// return TSDB_CODE_SUCCESS; +//} +// +//static int32_t parseTimestampFromJSON(cJSON *root, TAOS_SML_KV **pTS, int *num_kvs, SSmlLinesInfo* info) { +// //Timestamp must be the first KV to parse +// assert(*num_kvs == 0); +// int64_t tsVal; +// char key[] = OTD_TIMESTAMP_COLUMN_NAME; +// +// cJSON *timestamp = cJSON_GetObjectItem(root, "timestamp"); +// if (cJSON_IsNumber(timestamp)) { +// //timestamp value 0 indicates current system time +// if (timestamp->valueint == 0) { +// tsVal = taosGetTimestampNs(); +// } else { +// tsVal = strtoll(timestamp->numberstring, NULL, 10); +// size_t tsLen = strlen(timestamp->numberstring); +// if (tsLen == SML_TIMESTAMP_SECOND_DIGITS) { +// tsVal = (int64_t)(tsVal * 1e9); +// } else if (tsLen == SML_TIMESTAMP_MILLI_SECOND_DIGITS) { +// tsVal = convertTimePrecision(tsVal, TSDB_TIME_PRECISION_MILLI, TSDB_TIME_PRECISION_NANO); +// } else { +// return TSDB_CODE_TSC_INVALID_TIME_STAMP; +// } +// } +// } else if (cJSON_IsObject(timestamp)) { +// int32_t ret = parseTimestampFromJSONObj(timestamp, &tsVal, info); +// if (ret != TSDB_CODE_SUCCESS) { +// tscError("OTD:0x%"PRIx64" Failed to parse timestamp from JSON Obj", info->id); +// return ret; +// } +// } else { +// return TSDB_CODE_TSC_INVALID_JSON; +// } +// +// //allocate fields for timestamp and value +// *pTS = tcalloc(OTD_MAX_FIELDS_NUM, sizeof(TAOS_SML_KV)); +// +// +// (*pTS)->key = tcalloc(sizeof(key), 1); +// memcpy((*pTS)->key, key, sizeof(key)); +// +// (*pTS)->type = TSDB_DATA_TYPE_TIMESTAMP; +// (*pTS)->length = (int16_t)tDataTypes[(*pTS)->type].bytes; +// (*pTS)->value = tcalloc((*pTS)->length, 1); +// memcpy((*pTS)->value, &tsVal, (*pTS)->length); +// +// *num_kvs += 1; +// return TSDB_CODE_SUCCESS; +// +//} +// +//static int32_t convertJSONBool(TAOS_SML_KV *pVal, char* typeStr, int64_t valueInt, SSmlLinesInfo* info) { +// if (strcasecmp(typeStr, "bool") != 0) { +// tscError("OTD:0x%"PRIx64" invalid type(%s) for JSON Bool", info->id, typeStr); +// return TSDB_CODE_TSC_INVALID_JSON_TYPE; +// } +// pVal->type = TSDB_DATA_TYPE_BOOL; +// pVal->length = (int16_t)tDataTypes[pVal->type].bytes; +// pVal->value = tcalloc(pVal->length, 1); +// *(bool *)(pVal->value) = valueInt ? true : false; +// +// return TSDB_CODE_SUCCESS; +//} +// +//static int32_t convertJSONNumber(TAOS_SML_KV *pVal, char* typeStr, cJSON *value, SSmlLinesInfo* info) { +// //tinyint +// if (strcasecmp(typeStr, "i8") == 0 || +// strcasecmp(typeStr, "tinyint") == 0) { +// if (!IS_VALID_TINYINT(value->valueint)) { +// tscError("OTD:0x%"PRIx64" JSON value(%"PRId64") cannot fit in type(tinyint)", info->id, value->valueint); +// return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; +// } +// pVal->type = TSDB_DATA_TYPE_TINYINT; +// pVal->length = (int16_t)tDataTypes[pVal->type].bytes; +// pVal->value = tcalloc(pVal->length, 1); +// *(int8_t *)(pVal->value) = (int8_t)(value->valueint); +// return TSDB_CODE_SUCCESS; +// } +// //smallint +// if (strcasecmp(typeStr, "i16") == 0 || +// strcasecmp(typeStr, "smallint") == 0) { +// if (!IS_VALID_SMALLINT(value->valueint)) { +// tscError("OTD:0x%"PRIx64" JSON value(%"PRId64") cannot fit in type(smallint)", info->id, value->valueint); +// return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; +// } +// pVal->type = TSDB_DATA_TYPE_SMALLINT; +// pVal->length = (int16_t)tDataTypes[pVal->type].bytes; +// pVal->value = tcalloc(pVal->length, 1); +// *(int16_t *)(pVal->value) = (int16_t)(value->valueint); +// return TSDB_CODE_SUCCESS; +// } +// //int +// if (strcasecmp(typeStr, "i32") == 0 || +// strcasecmp(typeStr, "int") == 0) { +// if (!IS_VALID_INT(value->valueint)) { +// tscError("OTD:0x%"PRIx64" JSON value(%"PRId64") cannot fit in type(int)", info->id, value->valueint); +// return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; +// } +// pVal->type = TSDB_DATA_TYPE_INT; +// pVal->length = (int16_t)tDataTypes[pVal->type].bytes; +// pVal->value = tcalloc(pVal->length, 1); +// *(int32_t *)(pVal->value) = (int32_t)(value->valueint); +// return TSDB_CODE_SUCCESS; +// } +// //bigint +// if (strcasecmp(typeStr, "i64") == 0 || +// strcasecmp(typeStr, "bigint") == 0) { +// pVal->type = TSDB_DATA_TYPE_BIGINT; +// pVal->length = (int16_t)tDataTypes[pVal->type].bytes; +// pVal->value = tcalloc(pVal->length, 1); +// /* cJSON conversion of legit BIGINT may overflow, +// * use original string to do the conversion. +// */ +// errno = 0; +// int64_t val = (int64_t)strtoll(value->numberstring, NULL, 10); +// if (errno == ERANGE || !IS_VALID_BIGINT(val)) { +// tscError("OTD:0x%"PRIx64" JSON value(%s) cannot fit in type(bigint)", info->id, value->numberstring); +// return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; +// } +// *(int64_t *)(pVal->value) = val; +// return TSDB_CODE_SUCCESS; +// } +// //float +// if (strcasecmp(typeStr, "f32") == 0 || +// strcasecmp(typeStr, "float") == 0) { +// if (!IS_VALID_FLOAT(value->valuedouble)) { +// tscError("OTD:0x%"PRIx64" JSON value(%f) cannot fit in type(float)", info->id, value->valuedouble); +// return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; +// } +// pVal->type = TSDB_DATA_TYPE_FLOAT; +// pVal->length = (int16_t)tDataTypes[pVal->type].bytes; +// pVal->value = tcalloc(pVal->length, 1); +// *(float *)(pVal->value) = (float)(value->valuedouble); +// return TSDB_CODE_SUCCESS; +// } +// //double +// if (strcasecmp(typeStr, "f64") == 0 || +// strcasecmp(typeStr, "double") == 0) { +// if (!IS_VALID_DOUBLE(value->valuedouble)) { +// tscError("OTD:0x%"PRIx64" JSON value(%f) cannot fit in type(double)", info->id, value->valuedouble); +// return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; +// } +// pVal->type = TSDB_DATA_TYPE_DOUBLE; +// pVal->length = (int16_t)tDataTypes[pVal->type].bytes; +// pVal->value = tcalloc(pVal->length, 1); +// *(double *)(pVal->value) = (double)(value->valuedouble); +// return TSDB_CODE_SUCCESS; +// } +// +// //if reach here means type is unsupported +// tscError("OTD:0x%"PRIx64" invalid type(%s) for JSON Number", info->id, typeStr); +// return TSDB_CODE_TSC_INVALID_JSON_TYPE; +//} +// +//static int32_t convertJSONString(TAOS_SML_KV *pVal, char* typeStr, cJSON *value, SSmlLinesInfo* info) { +// if (strcasecmp(typeStr, "binary") == 0) { +// pVal->type = TSDB_DATA_TYPE_BINARY; +// } else if (strcasecmp(typeStr, "nchar") == 0) { +// pVal->type = TSDB_DATA_TYPE_NCHAR; +// } else { +// tscError("OTD:0x%"PRIx64" invalid type(%s) for JSON String", info->id, typeStr); +// return TSDB_CODE_TSC_INVALID_JSON_TYPE; +// } +// pVal->length = (int16_t)strlen(value->valuestring); +// pVal->value = tcalloc(pVal->length + 1, 1); +// memcpy(pVal->value, value->valuestring, pVal->length); +// return TSDB_CODE_SUCCESS; +//} +// +//static int32_t parseValueFromJSONObj(cJSON *root, TAOS_SML_KV *pVal, SSmlLinesInfo* info) { +// int32_t ret = TSDB_CODE_SUCCESS; +// int32_t size = cJSON_GetArraySize(root); +// +// if (size != OTD_JSON_SUB_FIELDS_NUM) { +// return TSDB_CODE_TSC_INVALID_JSON; +// } +// +// cJSON *value = cJSON_GetObjectItem(root, "value"); +// if (value == NULL) { +// return TSDB_CODE_TSC_INVALID_JSON; +// } +// +// cJSON *type = cJSON_GetObjectItem(root, "type"); +// if (!cJSON_IsString(type)) { +// return TSDB_CODE_TSC_INVALID_JSON; +// } +// +// switch (value->type) { +// case cJSON_True: +// case cJSON_False: { +// ret = convertJSONBool(pVal, type->valuestring, value->valueint, info); +// if (ret != TSDB_CODE_SUCCESS) { +// return ret; +// } +// break; +// } +// case cJSON_Number: { +// ret = convertJSONNumber(pVal, type->valuestring, value, info); +// if (ret != TSDB_CODE_SUCCESS) { +// return ret; +// } +// break; +// } +// case cJSON_String: { +// ret = convertJSONString(pVal, type->valuestring, value, info); +// if (ret != TSDB_CODE_SUCCESS) { +// return ret; +// } +// break; +// } +// default: +// return TSDB_CODE_TSC_INVALID_JSON_TYPE; +// } +// +// return TSDB_CODE_SUCCESS; +//} +// +//static int32_t parseValueFromJSON(cJSON *root, TAOS_SML_KV *pVal, SSmlLinesInfo* info) { +// int type = root->type; +// +// switch (type) { +// case cJSON_True: +// case cJSON_False: { +// pVal->type = TSDB_DATA_TYPE_BOOL; +// pVal->length = (int16_t)tDataTypes[pVal->type].bytes; +// pVal->value = tcalloc(pVal->length, 1); +// *(bool *)(pVal->value) = root->valueint ? true : false; +// break; +// } +// case cJSON_Number: { +// //convert default JSON Number type to BIGINT/DOUBLE +// //if (isValidInteger(root->numberstring)) { +// // pVal->type = TSDB_DATA_TYPE_BIGINT; +// // pVal->length = (int16_t)tDataTypes[pVal->type].bytes; +// // pVal->value = tcalloc(pVal->length, 1); +// // /* cJSON conversion of legit BIGINT may overflow, +// // * use original string to do the conversion. +// // */ +// // errno = 0; +// // int64_t val = (int64_t)strtoll(root->numberstring, NULL, 10); +// // if (errno == ERANGE || !IS_VALID_BIGINT(val)) { +// // tscError("OTD:0x%"PRIx64" JSON value(%s) cannot fit in type(bigint)", info->id, root->numberstring); +// // return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; +// // } +// // *(int64_t *)(pVal->value) = val; +// //} else if (isValidFloat(root->numberstring)) { +// // pVal->type = TSDB_DATA_TYPE_DOUBLE; +// // pVal->length = (int16_t)tDataTypes[pVal->type].bytes; +// // pVal->value = tcalloc(pVal->length, 1); +// // *(double *)(pVal->value) = (double)(root->valuedouble); +// //} else { +// // return TSDB_CODE_TSC_INVALID_JSON_TYPE; +// //} +// if (isValidInteger(root->numberstring) || isValidFloat(root->numberstring)) { +// pVal->type = TSDB_DATA_TYPE_DOUBLE; +// pVal->length = (int16_t)tDataTypes[pVal->type].bytes; +// pVal->value = tcalloc(pVal->length, 1); +// *(double *)(pVal->value) = (double)(root->valuedouble); +// } +// +// break; +// } +// case cJSON_String: { +// /* set default JSON type to binary/nchar according to +// * user configured parameter tsDefaultJSONStrType +// */ +// if (strcasecmp(tsDefaultJSONStrType, "binary") == 0) { +// pVal->type = TSDB_DATA_TYPE_BINARY; +// } else if (strcasecmp(tsDefaultJSONStrType, "nchar") == 0) { +// pVal->type = TSDB_DATA_TYPE_NCHAR; +// } else { +// tscError("OTD:0x%"PRIx64" Invalid default JSON string type set from config %s", info->id, tsDefaultJSONStrType); +// return TSDB_CODE_TSC_INVALID_JSON_CONFIG; +// } +// //pVal->length = wcslen((wchar_t *)root->valuestring) * TSDB_NCHAR_SIZE; +// pVal->length = (int16_t)strlen(root->valuestring); +// pVal->value = tcalloc(pVal->length + 1, 1); +// memcpy(pVal->value, root->valuestring, pVal->length); +// break; +// } +// case cJSON_Object: { +// int32_t ret = parseValueFromJSONObj(root, pVal, info); +// if (ret != TSDB_CODE_SUCCESS) { +// tscError("OTD:0x%"PRIx64" Failed to parse timestamp from JSON Obj", info->id); +// return ret; +// } +// break; +// } +// default: +// return TSDB_CODE_TSC_INVALID_JSON; +// } +// +// return TSDB_CODE_SUCCESS; +//} +// +//static int32_t parseMetricValueFromJSON(cJSON *root, TAOS_SML_KV **pKVs, int *num_kvs, SSmlLinesInfo* info) { +// //skip timestamp +// TAOS_SML_KV *pVal = *pKVs + 1; +// char key[] = OTD_METRIC_VALUE_COLUMN_NAME; +// +// cJSON *metricVal = cJSON_GetObjectItem(root, "value"); +// if (metricVal == NULL) { +// return TSDB_CODE_TSC_INVALID_JSON; +// } +// +// int32_t ret = parseValueFromJSON(metricVal, pVal, info); +// if (ret != TSDB_CODE_SUCCESS) { +// return ret; +// } +// +// pVal->key = tcalloc(sizeof(key) + TS_BACKQUOTE_CHAR_SIZE, 1); +// memcpy(pVal->key, key, sizeof(key)); +// addEscapeCharToString(pVal->key, (int32_t)strlen(pVal->key)); +// +// *num_kvs += 1; +// return TSDB_CODE_SUCCESS; +// +//} +// +// +//static int32_t parseTagsFromJSON(cJSON *root, TAOS_SML_KV **pKVs, int *num_kvs, char **childTableName, +// SHashObj *pHash, SSmlLinesInfo* info) { +// int32_t ret = TSDB_CODE_SUCCESS; +// +// cJSON *tags = cJSON_GetObjectItem(root, "tags"); +// if (tags == NULL || tags->type != cJSON_Object) { +// return TSDB_CODE_TSC_INVALID_JSON; +// } +// +// //handle child table name +// size_t childTableNameLen = strlen(tsSmlChildTableName); +// char childTbName[TSDB_TABLE_NAME_LEN] = {0}; +// if (childTableNameLen != 0) { +// memcpy(childTbName, tsSmlChildTableName, childTableNameLen); +// cJSON *id = cJSON_GetObjectItem(tags, childTbName); +// if (id != NULL) { +// if (!cJSON_IsString(id)) { +// tscError("OTD:0x%"PRIx64" ID must be JSON string", info->id); +// return TSDB_CODE_TSC_INVALID_JSON; +// } +// size_t idLen = strlen(id->valuestring); +// *childTableName = tcalloc(idLen + TS_BACKQUOTE_CHAR_SIZE + 1, sizeof(char)); +// memcpy(*childTableName, id->valuestring, idLen); +// addEscapeCharToString(*childTableName, (int32_t)idLen); +// +// //check duplicate IDs +// cJSON_DeleteItemFromObject(tags, childTbName); +// id = cJSON_GetObjectItem(tags, childTbName); +// if (id != NULL) { +// return TSDB_CODE_TSC_DUP_TAG_NAMES; +// } +// } +// } +// +// int32_t tagNum = cJSON_GetArraySize(tags); +// //at least one tag pair required +// if (tagNum <= 0) { +// return TSDB_CODE_TSC_INVALID_JSON; +// } +// +// //allocate memory for tags +// *pKVs = tcalloc(tagNum, sizeof(TAOS_SML_KV)); +// TAOS_SML_KV *pkv = *pKVs; +// +// for (int32_t i = 0; i < tagNum; ++i) { +// cJSON *tag = cJSON_GetArrayItem(tags, i); +// if (tag == NULL) { +// return TSDB_CODE_TSC_INVALID_JSON; +// } +// //check duplicate keys +// if (checkDuplicateKey(tag->string, pHash, info)) { +// return TSDB_CODE_TSC_DUP_TAG_NAMES; +// } +// //key +// size_t keyLen = strlen(tag->string); +// if (keyLen > TSDB_COL_NAME_LEN - 1) { +// tscError("OTD:0x%"PRIx64" Tag key cannot exceeds %d characters in JSON", info->id, TSDB_COL_NAME_LEN - 1); +// return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; +// } +// pkv->key = tcalloc(keyLen + TS_BACKQUOTE_CHAR_SIZE + 1, sizeof(char)); +// strncpy(pkv->key, tag->string, keyLen); +// addEscapeCharToString(pkv->key, (int32_t)keyLen); +// //value +// ret = parseValueFromJSON(tag, pkv, info); +// if (ret != TSDB_CODE_SUCCESS) { +// return ret; +// } +// *num_kvs += 1; +// pkv++; +// +// } +// +// return ret; +// +//} +// +//static int32_t tscParseJSONPayload(cJSON *root, TAOS_SML_DATA_POINT* pSml, SSmlLinesInfo* info) { +// int32_t ret = TSDB_CODE_SUCCESS; +// +// if (!cJSON_IsObject(root)) { +// tscError("OTD:0x%"PRIx64" data point needs to be JSON object", info->id); +// return TSDB_CODE_TSC_INVALID_JSON; +// } +// +// int32_t size = cJSON_GetArraySize(root); +// //outmost json fields has to be exactly 4 +// if (size != OTD_JSON_FIELDS_NUM) { +// tscError("OTD:0x%"PRIx64" Invalid number of JSON fields in data point %d", info->id, size); +// return TSDB_CODE_TSC_INVALID_JSON; +// } +// +// //Parse metric +// ret = parseMetricFromJSON(root, pSml, info); +// if (ret != TSDB_CODE_SUCCESS) { +// tscError("OTD:0x%"PRIx64" Unable to parse metric from JSON payload", info->id); +// return ret; +// } +// tscDebug("OTD:0x%"PRIx64" Parse metric from JSON payload finished", info->id); +// +// //Parse timestamp +// ret = parseTimestampFromJSON(root, &pSml->fields, &pSml->fieldNum, info); +// if (ret) { +// tscError("OTD:0x%"PRIx64" Unable to parse timestamp from JSON payload", info->id); +// return ret; +// } +// tscDebug("OTD:0x%"PRIx64" Parse timestamp from JSON payload finished", info->id); +// +// //Parse metric value +// ret = parseMetricValueFromJSON(root, &pSml->fields, &pSml->fieldNum, info); +// if (ret) { +// tscError("OTD:0x%"PRIx64" Unable to parse metric value from JSON payload", info->id); +// return ret; +// } +// tscDebug("OTD:0x%"PRIx64" Parse metric value from JSON payload finished", info->id); +// +// //Parse tags +// SHashObj *keyHashTable = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); +// ret = parseTagsFromJSON(root, &pSml->tags, &pSml->tagNum, &pSml->childTableName, keyHashTable, info); +// if (ret) { +// tscError("OTD:0x%"PRIx64" Unable to parse tags from JSON payload", info->id); +// taosHashCleanup(keyHashTable); +// return ret; +// } +// tscDebug("OTD:0x%"PRIx64" Parse tags from JSON payload finished", info->id); +// taosHashCleanup(keyHashTable); +// +// return TSDB_CODE_SUCCESS; +//} +// +//static int32_t tscParseMultiJSONPayload(char* payload, SArray* points, SSmlLinesInfo* info) { +// int32_t payloadNum, ret; +// ret = TSDB_CODE_SUCCESS; +// +// if (payload == NULL) { +// tscError("OTD:0x%"PRIx64" empty JSON Payload", info->id); +// return TSDB_CODE_TSC_INVALID_JSON; +// } +// +// cJSON *root = cJSON_Parse(payload); +// //multiple data points must be sent in JSON array +// if (cJSON_IsObject(root)) { +// payloadNum = 1; +// } else if (cJSON_IsArray(root)) { +// payloadNum = cJSON_GetArraySize(root); +// } else { +// tscError("OTD:0x%"PRIx64" Invalid JSON Payload", info->id); +// ret = TSDB_CODE_TSC_INVALID_JSON; +// goto PARSE_JSON_OVER; +// } +// +// for (int32_t i = 0; i < payloadNum; ++i) { +// TAOS_SML_DATA_POINT point = {0}; +// cJSON *dataPoint = (payloadNum == 1 && cJSON_IsObject(root)) ? root : cJSON_GetArrayItem(root, i); +// +// ret = tscParseJSONPayload(dataPoint, &point, info); +// if (ret != TSDB_CODE_SUCCESS) { +// tscError("OTD:0x%"PRIx64" JSON data point parse failed", info->id); +// destroySmlDataPoint(&point); +// goto PARSE_JSON_OVER; +// } else { +// tscDebug("OTD:0x%"PRIx64" JSON data point parse success", info->id); +// } +// taosArrayPush(points, &point); +// } +// +//PARSE_JSON_OVER: +// cJSON_Delete(root); +// return ret; +//} +// +//int taos_insert_json_payload(TAOS* taos, char* payload, SMLProtocolType protocol, SMLTimeStampType tsType, int* affectedRows) { +// int32_t code = 0; +// +// SSmlLinesInfo* info = tcalloc(1, sizeof(SSmlLinesInfo)); +// info->id = genUID(); +// info->tsType = tsType; +// info->protocol = protocol; +// +// if (payload == NULL) { +// tscError("OTD:0x%"PRIx64" taos_insert_json_payload payload is NULL", info->id); +// tfree(info); +// code = TSDB_CODE_TSC_APP_ERROR; +// return code; +// } +// +// SArray* lpPoints = taosArrayInit(1, sizeof(TAOS_SML_DATA_POINT)); +// if (lpPoints == NULL) { +// tscError("OTD:0x%"PRIx64" taos_insert_json_payload failed to allocate memory", info->id); +// tfree(info); +// return TSDB_CODE_TSC_OUT_OF_MEMORY; +// } +// +// tscDebug("OTD:0x%"PRIx64" taos_insert_telnet_lines begin inserting %d points", info->id, 1); +// code = tscParseMultiJSONPayload(payload, lpPoints, info); +// size_t numPoints = taosArrayGetSize(lpPoints); +// +// if (code != 0) { +// goto cleanup; +// } +// +// TAOS_SML_DATA_POINT* points = TARRAY_GET_START(lpPoints); +// code = tscSmlInsert(taos, points, (int)numPoints, info); +// if (code != 0) { +// tscError("OTD:0x%"PRIx64" taos_insert_json_payload error: %s", info->id, tstrerror((code))); +// } +// if (affectedRows != NULL) { +// *affectedRows = info->affectedRows; +// } +// +//cleanup: +// tscDebug("OTD:0x%"PRIx64" taos_insert_json_payload finish inserting 1 Point. code: %d", info->id, code); +// points = TARRAY_GET_START(lpPoints); +// numPoints = taosArrayGetSize(lpPoints); +// for (int i = 0; i < numPoints; ++i) { +// destroySmlDataPoint(points+i); +// } +// +// taosArrayDestroy(&lpPoints); +// +// tfree(info); +// return code; +//} diff --git a/source/client/src/tscParseLineProtocol.c b/source/client/src/tscParseLineProtocol.c new file mode 100644 index 0000000000..e0b4cc0d7a --- /dev/null +++ b/source/client/src/tscParseLineProtocol.c @@ -0,0 +1,3134 @@ +#include +#include +#include +#include + +#include "tscParseLine.h" + +#include "tdef.h" +#include "ttypes.h" +#include "tmsg.h" +#include "tlog.h" +#include "query.h" +#include "taoserror.h" +#include "taos.h" +#include "ttime.h" + + +typedef struct { + char sTableName[TSDB_TABLE_NAME_LEN]; + SHashObj* tagHash; + SHashObj* fieldHash; + SArray* tags; //SArray + SArray* fields; //SArray + uint8_t precision; +} SSmlSTableSchema; + +#define SPACE ' ' +#define COMMA ',' +#define EQUAL '=' +#define QUOTE '"' +#define SLASH '\\' + +//================================================================================================= + +static uint64_t linesSmlHandleId = 0; + +static int32_t insertChildTablePointsBatch(void* pVoid, char* name, char* name1, SArray* pArray, SArray* pArray1, + SArray* pArray2, SArray* pArray3, size_t size, SSmlLinesInfo* info); +static int32_t doInsertChildTablePoints(void* pVoid, char* sql, char* name, SArray* pArray, SArray* pArray1, + SSmlLinesInfo* info); +uint64_t genLinesSmlId() { + uint64_t id; + + do { + id = atomic_add_fetch_64(&linesSmlHandleId, 1); + } while (id == 0); + + return id; +} + +int compareSmlColKv(const void* p1, const void* p2) { + TAOS_SML_KV* kv1 = (TAOS_SML_KV*)p1; + TAOS_SML_KV* kv2 = (TAOS_SML_KV*)p2; + int kvLen1 = (int)strlen(kv1->key); + int kvLen2 = (int)strlen(kv2->key); + int res = strncasecmp(kv1->key, kv2->key, MIN(kvLen1, kvLen2)); + if (res != 0) { + return res; + } else { + return kvLen1-kvLen2; + } +} + +typedef enum { + SCHEMA_ACTION_CREATE_STABLE, + SCHEMA_ACTION_ADD_COLUMN, + SCHEMA_ACTION_ADD_TAG, + SCHEMA_ACTION_CHANGE_COLUMN_SIZE, + SCHEMA_ACTION_CHANGE_TAG_SIZE, +} ESchemaAction; + +typedef struct { + char sTableName[TSDB_TABLE_NAME_LEN]; + SArray* tags; //SArray + SArray* fields; //SArray +} SCreateSTableActionInfo; + +typedef struct { + char sTableName[TSDB_TABLE_NAME_LEN]; + SSchema* field; +} SAlterSTableActionInfo; + +typedef struct { + ESchemaAction action; + union { + SCreateSTableActionInfo createSTable; + SAlterSTableActionInfo alterSTable; + }; +} SSchemaAction; + +static int32_t getFieldBytesFromSmlKv(TAOS_SML_KV* kv, int32_t* bytes, uint64_t id) { + if (!IS_VAR_DATA_TYPE(kv->type)) { + *bytes = tDataTypes[kv->type].bytes; + } else { + if (kv->type == TSDB_DATA_TYPE_NCHAR) { + TdUcs4 *ucs = taosMemoryMalloc(kv->length * TSDB_NCHAR_SIZE + 1); + int32_t bytesNeeded = 0; + bool succ = taosMbsToUcs4(kv->value, kv->length, ucs, kv->length * TSDB_NCHAR_SIZE, &bytesNeeded); + if (!succ) { + taosMemoryFree(ucs); + uError("SML:0x%"PRIx64" convert nchar string to UCS4_LE failed:%s", id, kv->value); + return TSDB_CODE_TSC_INVALID_VALUE; + } + taosMemoryFree(ucs); + *bytes = bytesNeeded + VARSTR_HEADER_SIZE; + } else if (kv->type == TSDB_DATA_TYPE_BINARY) { + *bytes = kv->length + VARSTR_HEADER_SIZE; + } + } + return 0; +} + +static int32_t buildSmlKvSchema(TAOS_SML_KV* smlKv, SHashObj* hash, SArray* array, SSmlLinesInfo* info) { + SSchema* pField = NULL; + size_t* pFieldIdx = taosHashGet(hash, smlKv->key, strlen(smlKv->key)); + size_t fieldIdx = -1; + int32_t code = 0; + if (pFieldIdx) { + fieldIdx = *pFieldIdx; + pField = taosArrayGet(array, fieldIdx); + + if (pField->type != smlKv->type) { + uError("SML:0x%"PRIx64" type mismatch. key %s, type %d. type before %d", info->id, smlKv->key, smlKv->type, pField->type); + return TSDB_CODE_TSC_INVALID_VALUE; + } + + int32_t bytes = 0; + code = getFieldBytesFromSmlKv(smlKv, &bytes, info->id); + if (code != 0) { + return code; + } + pField->bytes = MAX(pField->bytes, bytes); + + } else { + SSchema field = {0}; + size_t tagKeyLen = strlen(smlKv->key); + strncpy(field.name, smlKv->key, tagKeyLen); + field.name[tagKeyLen] = '\0'; + field.type = smlKv->type; + + int32_t bytes = 0; + code = getFieldBytesFromSmlKv(smlKv, &bytes, info->id); + if (code != 0) { + return code; + } + field.bytes = bytes; + + pField = taosArrayPush(array, &field); + fieldIdx = taosArrayGetSize(array) - 1; + taosHashPut(hash, field.name, tagKeyLen, &fieldIdx, sizeof(fieldIdx)); + } + + smlKv->fieldSchemaIdx = (uint32_t)fieldIdx; + + return 0; +} + +static int32_t getSmlMd5ChildTableName(TAOS_SML_DATA_POINT* point, char* tableName, int* tableNameLen, + SSmlLinesInfo* info) { + uDebug("SML:0x%"PRIx64" taos_sml_insert get child table name through md5", info->id); + if (point->tagNum) { + qsort(point->tags, point->tagNum, sizeof(TAOS_SML_KV), compareSmlColKv); + } + + SStringBuilder sb; memset(&sb, 0, sizeof(sb)); + char sTableName[TSDB_TABLE_NAME_LEN] = {0}; + strncpy(sTableName, point->stableName, strlen(point->stableName)); + //strtolower(sTableName, point->stableName); + taosStringBuilderAppendString(&sb, sTableName); + for (int j = 0; j < point->tagNum; ++j) { + taosStringBuilderAppendChar(&sb, ','); + TAOS_SML_KV* tagKv = point->tags + j; + char tagName[TSDB_COL_NAME_LEN] = {0}; + strncpy(tagName, tagKv->key, strlen(tagKv->key)); + //strtolower(tagName, tagKv->key); + taosStringBuilderAppendString(&sb, tagName); + taosStringBuilderAppendChar(&sb, '='); + taosStringBuilderAppend(&sb, tagKv->value, tagKv->length); + } + size_t len = 0; + char* keyJoined = taosStringBuilderGetResult(&sb, &len); + T_MD5_CTX context; + tMD5Init(&context); + tMD5Update(&context, (uint8_t *)keyJoined, (uint32_t)len); + tMD5Final(&context); + uint64_t digest1 = *(uint64_t*)(context.digest); + uint64_t digest2 = *(uint64_t*)(context.digest + 8); + *tableNameLen = snprintf(tableName, *tableNameLen, + "t_%016"PRIx64"%016"PRIx64, digest1, digest2); + taosStringBuilderDestroy(&sb); + uDebug("SML:0x%"PRIx64" child table name: %s", info->id, tableName); + return 0; +} + +static int32_t buildSmlChildTableName(TAOS_SML_DATA_POINT* point, SSmlLinesInfo* info) { + uDebug("SML:0x%"PRIx64" taos_sml_insert build child table name", info->id); + char childTableName[TSDB_TABLE_NAME_LEN]; + int32_t tableNameLen = TSDB_TABLE_NAME_LEN; + getSmlMd5ChildTableName(point, childTableName, &tableNameLen, info); + point->childTableName = calloc(1, tableNameLen+1); + strncpy(point->childTableName, childTableName, tableNameLen); + point->childTableName[tableNameLen] = '\0'; + return 0; +} + +static int32_t buildDataPointSchemas(TAOS_SML_DATA_POINT* points, int numPoint, SArray* stableSchemas, SSmlLinesInfo* info) { + int32_t code = 0; + SHashObj* sname2shema = taosHashInit(32, + taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); + + for (int i = 0; i < numPoint; ++i) { + TAOS_SML_DATA_POINT* point = &points[i]; + size_t stableNameLen = strlen(point->stableName); + size_t* pStableIdx = taosHashGet(sname2shema, point->stableName, stableNameLen); + SSmlSTableSchema* pStableSchema = NULL; + size_t stableIdx = -1; + if (pStableIdx) { + pStableSchema= taosArrayGet(stableSchemas, *pStableIdx); + stableIdx = *pStableIdx; + } else { + SSmlSTableSchema schema; + strncpy(schema.sTableName, point->stableName, stableNameLen); + schema.sTableName[stableNameLen] = '\0'; + schema.fields = taosArrayInit(64, sizeof(SSchema)); + schema.tags = taosArrayInit(8, sizeof(SSchema)); + schema.tagHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); + schema.fieldHash = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); + + pStableSchema = taosArrayPush(stableSchemas, &schema); + stableIdx = taosArrayGetSize(stableSchemas) - 1; + taosHashPut(sname2shema, schema.sTableName, stableNameLen, &stableIdx, sizeof(size_t)); + } + + for (int j = 0; j < point->tagNum; ++j) { + TAOS_SML_KV* tagKv = point->tags + j; + if (!point->childTableName) { + buildSmlChildTableName(point, info); + } + + code = buildSmlKvSchema(tagKv, pStableSchema->tagHash, pStableSchema->tags, info); + if (code != 0) { + uError("SML:0x%"PRIx64" build data point schema failed. point no.: %d, tag key: %s", info->id, i, tagKv->key); + return code; + } + } + + //for Line Protocol tags may be omitted, add a tag with NULL value + if (point->tagNum == 0) { + if (!point->childTableName) { + buildSmlChildTableName(point, info); + } + char tagNullName[TSDB_COL_NAME_LEN] = {0}; + size_t nameLen = strlen(tsSmlTagNullName); + strncpy(tagNullName, tsSmlTagNullName, nameLen); + addEscapeCharToString(tagNullName, (int32_t)nameLen); + size_t* pTagNullIdx = taosHashGet(pStableSchema->tagHash, tagNullName, nameLen); + if (!pTagNullIdx) { + SSchema tagNull = {0}; + tagNull.type = TSDB_DATA_TYPE_NCHAR; + tagNull.bytes = TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE; + strncpy(tagNull.name, tagNullName, nameLen); + taosArrayPush(pStableSchema->tags, &tagNull); + size_t tagNullIdx = taosArrayGetSize(pStableSchema->tags) - 1; + taosHashPut(pStableSchema->tagHash, tagNull.name, nameLen, &tagNullIdx, sizeof(tagNullIdx)); + } + } + + for (int j = 0; j < point->fieldNum; ++j) { + TAOS_SML_KV* fieldKv = point->fields + j; + code = buildSmlKvSchema(fieldKv, pStableSchema->fieldHash, pStableSchema->fields, info); + if (code != 0) { + uError("SML:0x%"PRIx64" build data point schema failed. point no.: %d, tag key: %s", info->id, i, fieldKv->key); + return code; + } + } + + point->schemaIdx = (uint32_t)stableIdx; + } + + size_t numStables = taosArrayGetSize(stableSchemas); + for (int32_t i = 0; i < numStables; ++i) { + SSmlSTableSchema* schema = taosArrayGet(stableSchemas, i); + taosHashCleanup(schema->tagHash); + taosHashCleanup(schema->fieldHash); + } + taosHashCleanup(sname2shema); + + uDebug("SML:0x%"PRIx64" build point schema succeed. num of super table: %zu", info->id, numStables); + for (int32_t i = 0; i < numStables; ++i) { + SSmlSTableSchema* schema = taosArrayGet(stableSchemas, i); + uDebug("\ttable name: %s, tags number: %zu, fields number: %zu", schema->sTableName, + taosArrayGetSize(schema->tags), taosArrayGetSize(schema->fields)); + } + + return 0; +} + +static int32_t generateSchemaAction(SSchema* pointColField, SHashObj* dbAttrHash, SArray* dbAttrArray, bool isTag, char sTableName[], + SSchemaAction* action, bool* actionNeeded, SSmlLinesInfo* info) { + char fieldName[TSDB_COL_NAME_LEN] = {0}; + strcpy(fieldName, pointColField->name); + + size_t* pDbIndex = taosHashGet(dbAttrHash, fieldName, strlen(fieldName)); + if (pDbIndex) { + SSchema* dbAttr = taosArrayGet(dbAttrArray, *pDbIndex); + assert(strcasecmp(dbAttr->name, pointColField->name) == 0); + if (pointColField->type != dbAttr->type) { + uError("SML:0x%"PRIx64" point type and db type mismatch. key: %s. point type: %d, db type: %d", info->id, pointColField->name, + pointColField->type, dbAttr->type); + return TSDB_CODE_TSC_INVALID_VALUE; + } + + if (IS_VAR_DATA_TYPE(pointColField->type) && (pointColField->bytes > dbAttr->bytes)) { + if (isTag) { + action->action = SCHEMA_ACTION_CHANGE_TAG_SIZE; + } else { + action->action = SCHEMA_ACTION_CHANGE_COLUMN_SIZE; + } + memset(&action->alterSTable, 0, sizeof(SAlterSTableActionInfo)); + memcpy(action->alterSTable.sTableName, sTableName, TSDB_TABLE_NAME_LEN); + action->alterSTable.field = pointColField; + *actionNeeded = true; + } + } else { + if (isTag) { + action->action = SCHEMA_ACTION_ADD_TAG; + } else { + action->action = SCHEMA_ACTION_ADD_COLUMN; + } + memset(&action->alterSTable, 0, sizeof(SAlterSTableActionInfo)); + memcpy(action->alterSTable.sTableName, sTableName, TSDB_TABLE_NAME_LEN); + action->alterSTable.field = pointColField; + *actionNeeded = true; + } + if (*actionNeeded) { + uDebug("SML:0x%" PRIx64 " generate schema action. column name: %s, action: %d", info->id, fieldName, + action->action); + } + return 0; +} + +static int32_t buildColumnDescription(SSchema* field, + char* buf, int32_t bufSize, int32_t* outBytes) { + uint8_t type = field->type; + + if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { + int32_t bytes = field->bytes - VARSTR_HEADER_SIZE; + if (type == TSDB_DATA_TYPE_NCHAR) { + bytes = bytes/TSDB_NCHAR_SIZE; + } + int out = snprintf(buf, bufSize,"%s %s(%d)", + field->name,tDataTypes[field->type].name, bytes); + *outBytes = out; + } else { + int out = snprintf(buf, bufSize, "%s %s", + field->name, tDataTypes[type].name); + *outBytes = out; + } + + return 0; +} + + +static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInfo* info) { + int32_t code = 0; + int32_t outBytes = 0; + char *result = (char *)calloc(1, tsMaxSQLStringLen+1); + int32_t capacity = tsMaxSQLStringLen + 1; + + uDebug("SML:0x%"PRIx64" apply schema action. action: %d", info->id, action->action); + switch (action->action) { + case SCHEMA_ACTION_ADD_COLUMN: { + int n = sprintf(result, "alter stable %s add column ", action->alterSTable.sTableName); + buildColumnDescription(action->alterSTable.field, result+n, capacity-n, &outBytes); + TAOS_RES* res = taos_query(taos, result); //TODO async doAsyncQuery + code = taos_errno(res); + char* errStr = taos_errstr(res); + char* begin = strstr(errStr, "duplicated column names"); + bool tscDupColNames = (begin != NULL); + if (code != TSDB_CODE_SUCCESS) { + uError("SML:0x%"PRIx64" apply schema action. error: %s", info->id, errStr); + } + taos_free_result(res); + + if (code == TSDB_CODE_MND_FIELD_ALREAY_EXIST || code == TSDB_CODE_MND_TAG_ALREAY_EXIST || tscDupColNames) { + TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); + code = taos_errno(res2); + if (code != TSDB_CODE_SUCCESS) { + uError("SML:0x%" PRIx64 " apply schema action. reset query cache. error: %s", info->id, taos_errstr(res2)); + } + taos_free_result(res2); + taosMsleep(500); + } + break; + } + case SCHEMA_ACTION_ADD_TAG: { + int n = sprintf(result, "alter stable %s add tag ", action->alterSTable.sTableName); + buildColumnDescription(action->alterSTable.field, + result+n, capacity-n, &outBytes); + TAOS_RES* res = taos_query(taos, result); //TODO async doAsyncQuery + code = taos_errno(res); + char* errStr = taos_errstr(res); + char* begin = strstr(errStr, "duplicated column names"); + bool tscDupColNames = (begin != NULL); + if (code != TSDB_CODE_SUCCESS) { + uError("SML:0x%"PRIx64" apply schema action. error : %s", info->id, taos_errstr(res)); + } + taos_free_result(res); + + if (code == TSDB_CODE_MND_TAG_ALREAY_EXIST || code == TSDB_CODE_MND_FIELD_ALREAY_EXIST || tscDupColNames) { + TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); + code = taos_errno(res2); + if (code != TSDB_CODE_SUCCESS) { + uError("SML:0x%" PRIx64 " apply schema action. reset query cache. error: %s", info->id, taos_errstr(res2)); + } + taos_free_result(res2); + taosMsleep(500); + } + break; + } + case SCHEMA_ACTION_CHANGE_COLUMN_SIZE: { + int n = sprintf(result, "alter stable %s modify column ", action->alterSTable.sTableName); + buildColumnDescription(action->alterSTable.field, result+n, + capacity-n, &outBytes); + TAOS_RES* res = taos_query(taos, result); //TODO async doAsyncQuery + code = taos_errno(res); + if (code != TSDB_CODE_SUCCESS) { + uError("SML:0x%"PRIx64" apply schema action. error : %s", info->id, taos_errstr(res)); + } + taos_free_result(res); + + if (code == TSDB_CODE_MND_INVALID_COLUMN_LENGTH || code == TSDB_CODE_TSC_INVALID_COLUMN_LENGTH) { + TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); + code = taos_errno(res2); + if (code != TSDB_CODE_SUCCESS) { + uError("SML:0x%" PRIx64 " apply schema action. reset query cache. error: %s", info->id, taos_errstr(res2)); + } + taos_free_result(res2); + taosMsleep(500); + } + break; + } + case SCHEMA_ACTION_CHANGE_TAG_SIZE: { + int n = sprintf(result, "alter stable %s modify tag ", action->alterSTable.sTableName); + buildColumnDescription(action->alterSTable.field, result+n, + capacity-n, &outBytes); + TAOS_RES* res = taos_query(taos, result); //TODO async doAsyncQuery + code = taos_errno(res); + if (code != TSDB_CODE_SUCCESS) { + uError("SML:0x%"PRIx64" apply schema action. error : %s", info->id, taos_errstr(res)); + } + taos_free_result(res); + + if (code == TSDB_CODE_MND_INVALID_TAG_LENGTH || code == TSDB_CODE_TSC_INVALID_TAG_LENGTH) { + TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); + code = taos_errno(res2); + if (code != TSDB_CODE_SUCCESS) { + uError("SML:0x%" PRIx64 " apply schema action. reset query cache. error: %s", info->id, taos_errstr(res2)); + } + taos_free_result(res2); + taosMsleep(500); + } + break; + } + case SCHEMA_ACTION_CREATE_STABLE: { + int n = sprintf(result, "create stable %s (", action->createSTable.sTableName); + char* pos = result + n; int freeBytes = capacity - n; + size_t numCols = taosArrayGetSize(action->createSTable.fields); + for (int32_t i = 0; i < numCols; ++i) { + SSchema* field = taosArrayGet(action->createSTable.fields, i); + buildColumnDescription(field, pos, freeBytes, &outBytes); + pos += outBytes; freeBytes -= outBytes; + *pos = ','; ++pos; --freeBytes; + } + --pos; ++freeBytes; + + outBytes = snprintf(pos, freeBytes, ") tags ("); + pos += outBytes; freeBytes -= outBytes; + + size_t numTags = taosArrayGetSize(action->createSTable.tags); + for (int32_t i = 0; i < numTags; ++i) { + SSchema* field = taosArrayGet(action->createSTable.tags, i); + buildColumnDescription(field, pos, freeBytes, &outBytes); + pos += outBytes; freeBytes -= outBytes; + *pos = ','; ++pos; --freeBytes; + } + pos--; ++freeBytes; + outBytes = snprintf(pos, freeBytes, ")"); + TAOS_RES* res = taos_query(taos, result); + code = taos_errno(res); + if (code != TSDB_CODE_SUCCESS) { + uError("SML:0x%"PRIx64" apply schema action. error : %s", info->id, taos_errstr(res)); + } + taos_free_result(res); + + if (code == TSDB_CODE_MND_TABLE_ALREADY_EXIST) { + TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); + code = taos_errno(res2); + if (code != TSDB_CODE_SUCCESS) { + uError("SML:0x%" PRIx64 " apply schema action. reset query cache. error: %s", info->id, taos_errstr(res2)); + } + taos_free_result(res2); + taosMsleep(500); + } + break; + } + + default: + break; + } + + taosMemoryFree(result); + if (code != 0) { + uError("SML:0x%"PRIx64 " apply schema action failure. %s", info->id, tstrerror(code)); + } + return code; +} + +static int32_t destroySmlSTableSchema(SSmlSTableSchema* schema) { + taosHashCleanup(schema->tagHash); + taosHashCleanup(schema->fieldHash); + taosArrayDestroy(&schema->tags); + taosArrayDestroy(&schema->fields); + return 0; +} + +static int32_t fillDbSchema(STableMeta* tableMeta, char* tableName, SSmlSTableSchema* schema, SSmlLinesInfo* info) { + schema->tags = taosArrayInit(8, sizeof(SSchema)); + schema->fields = taosArrayInit(64, sizeof(SSchema)); + schema->tagHash = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); + schema->fieldHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); + + tstrncpy(schema->sTableName, tableName, strlen(tableName)+1); + schema->precision = tableMeta->tableInfo.precision; + for (int i=0; itableInfo.numOfColumns; ++i) { + SSchema field; + tstrncpy(field.name, tableMeta->schema[i].name, strlen(tableMeta->schema[i].name)+1); + addEscapeCharToString(field.name, (int16_t)strlen(field.name)); + field.type = tableMeta->schema[i].type; + field.bytes = tableMeta->schema[i].bytes; + taosArrayPush(schema->fields, &field); + size_t fieldIndex = taosArrayGetSize(schema->fields) - 1; + taosHashPut(schema->fieldHash, field.name, strlen(field.name), &fieldIndex, sizeof(fieldIndex)); + } + + for (int i=0; itableInfo.numOfTags; ++i) { + int j = i + tableMeta->tableInfo.numOfColumns; + SSchema field; + tstrncpy(field.name, tableMeta->schema[j].name, strlen(tableMeta->schema[j].name)+1); + addEscapeCharToString(field.name, (int16_t)strlen(field.name)); + field.type = tableMeta->schema[j].type; + field.bytes = tableMeta->schema[j].bytes; + taosArrayPush(schema->tags, &field); + size_t tagIndex = taosArrayGetSize(schema->tags) - 1; + taosHashPut(schema->tagHash, field.name, strlen(field.name), &tagIndex, sizeof(tagIndex)); + } + uDebug("SML:0x%"PRIx64 " load table schema succeed. table name: %s, columns number: %d, tag number: %d, precision: %d", + info->id, tableName, tableMeta->tableInfo.numOfColumns, tableMeta->tableInfo.numOfTags, schema->precision); + return TSDB_CODE_SUCCESS; +} + +static int32_t getSuperTableMetaFromLocalCache(TAOS* taos, char* tableName, STableMeta** outTableMeta, SSmlLinesInfo* info) { + int32_t code = 0; + STableMeta* tableMeta = NULL; + + SSqlObj* pSql = calloc(1, sizeof(SSqlObj)); + if (pSql == NULL) { + uError("SML:0x%" PRIx64 " failed to allocate memory, reason:%s", info->id, strerror(errno)); + code = TSDB_CODE_TSC_OUT_OF_MEMORY; + return code; + } + pSql->pTscObj = taos; + pSql->signature = pSql; + pSql->fp = NULL; + + registerSqlObj(pSql); + char tableNameBuf[TSDB_TABLE_NAME_LEN + TS_BACKQUOTE_CHAR_SIZE] = {0}; + memcpy(tableNameBuf, tableName, strlen(tableName)); + SStrToken tableToken = {.z = tableNameBuf, .n = (uint32_t)strlen(tableName), .type = TK_ID}; + tGetToken(tableNameBuf, &tableToken.type); + bool dbIncluded = false; + // Check if the table name available or not + if (tscValidateName(&tableToken, true, &dbIncluded) != TSDB_CODE_SUCCESS) { + code = TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; + sprintf(pSql->cmd.payload, "table name is invalid"); + taosReleaseRef(tscObjRef, pSql->self); + return code; + } + + SName sname = {0}; + if ((code = tscSetTableFullName(&sname, &tableToken, pSql, dbIncluded)) != TSDB_CODE_SUCCESS) { + taosReleaseRef(tscObjRef, pSql->self); + return code; + } + + char fullTableName[TSDB_TABLE_FNAME_LEN] = {0}; + memset(fullTableName, 0, tListLen(fullTableName)); + tNameExtractFullName(&sname, fullTableName); + + size_t size = 0; + taosHashGetCloneExt(UTIL_GET_TABLEMETA(pSql), fullTableName, strlen(fullTableName), NULL, (void**)&tableMeta, &size); + + STableMeta* stableMeta = tableMeta; + if (tableMeta != NULL && tableMeta->tableType == TSDB_CHILD_TABLE) { + taosHashGetCloneExt(UTIL_GET_TABLEMETA(pSql), tableMeta->sTableName, strlen(tableMeta->sTableName), NULL, + (void**)stableMeta, &size); + } + taosReleaseRef(tscObjRef, pSql->self); + + if (stableMeta != tableMeta) { + taosMemoryFree(tableMeta); + } + + if (stableMeta != NULL) { + if (outTableMeta != NULL) { + *outTableMeta = stableMeta; + } else { + taosMemoryFree(stableMeta); + } + return TSDB_CODE_SUCCESS; + } else { + return TSDB_CODE_TSC_NO_META_CACHED; + } +} + +static int32_t retrieveTableMeta(TAOS* taos, char* tableName, STableMeta** pTableMeta, SSmlLinesInfo* info) { + int32_t code = 0; + int32_t retries = 0; + STableMeta* tableMeta = NULL; + while (retries++ <= TSDB_MAX_REPLICA && tableMeta == NULL) { + STscObj* pObj = (STscObj*)taos; + if (pObj == NULL || pObj->signature != pObj) { + terrno = TSDB_CODE_TSC_DISCONNECTED; + return TSDB_CODE_TSC_DISCONNECTED; + } + + uDebug("SML:0x%" PRIx64 " retrieve table meta. super table name: %s", info->id, tableName); + code = getSuperTableMetaFromLocalCache(taos, tableName, &tableMeta, info); + if (code == TSDB_CODE_SUCCESS) { + uDebug("SML:0x%" PRIx64 " successfully retrieved table meta. super table name: %s", info->id, tableName); + break; + } else if (code == TSDB_CODE_TSC_NO_META_CACHED) { + char sql[256]; + snprintf(sql, 256, "describe %s", tableName); + TAOS_RES* res = taos_query(taos, sql); + code = taos_errno(res); + if (code != 0) { + uError("SML:0x%" PRIx64 " describe table failure. %s", info->id, taos_errstr(res)); + taos_free_result(res); + return code; + } + taos_free_result(res); + } else { + return code; + } + } + + if (tableMeta != NULL) { + *pTableMeta = tableMeta; + return TSDB_CODE_SUCCESS; + } else { + uError("SML:0x%" PRIx64 " failed to retrieve table meta. super table name: %s", info->id, tableName); + return TSDB_CODE_TSC_NO_META_CACHED; + } +} + +static int32_t loadTableSchemaFromDB(TAOS* taos, char* tableName, SSmlSTableSchema* schema, SSmlLinesInfo* info) { + int32_t code = 0; + STableMeta* tableMeta = NULL; + code = retrieveTableMeta(taos, tableName, &tableMeta, info); + if (code == TSDB_CODE_SUCCESS) { + assert(tableMeta != NULL); + fillDbSchema(tableMeta, tableName, schema, info); + taosMemoryFree(tableMeta); + tableMeta = NULL; + } + + return code; +} + +static int32_t modifyDBSchemas(TAOS* taos, SSmlLinesInfo* info) { + int32_t code = 0; + size_t numStable = taosHashGetSize(info->superTables); + + SSmlSTableMeta** tableMetaSml = taosHashIterate(info->superTables, NULL); + while (tableMetaSml) { + SSmlSTableMeta* cTablePoints = *tableMetaSml; + + if (NULL == pStmt->pCatalog) { + STMT_ERR_RET(catalogGetHandle(pStmt->taos->pAppInfo->clusterId, &pStmt->pCatalog)); + } + + STableMeta *pTableMeta = NULL; + SEpSet ep = getEpSet_s(&pStmt->taos->pAppInfo->mgmtEp); + STMT_ERR_RET(catalogGetTableMeta(pStmt->pCatalog, pStmt->taos->pAppInfo->pTransporter, &ep, &pStmt->bInfo.sname, &pTableMeta)); + + if (pTableMeta->uid == pStmt->bInfo.tbUid) { + pStmt->bInfo.needParse = false; + + return TSDB_CODE_SUCCESS; + } + +// for (int i = 0; i < numStable; ++i) { + SSmlSTableSchema* pointSchema = taosArrayGet(stableSchemas, i); + SSmlSTableSchema dbSchema; + memset(&dbSchema, 0, sizeof(SSmlSTableSchema)); + + code = loadTableSchemaFromDB(taos, pointSchema->sTableName, &dbSchema, info); + if (code == TSDB_CODE_MND_INVALID_TABLE_NAME) { + SSchemaAction schemaAction = {0}; + schemaAction.action = SCHEMA_ACTION_CREATE_STABLE; + memset(&schemaAction.createSTable, 0, sizeof(SCreateSTableActionInfo)); + memcpy(schemaAction.createSTable.sTableName, pointSchema->sTableName, TSDB_TABLE_NAME_LEN); + schemaAction.createSTable.tags = pointSchema->tags; + schemaAction.createSTable.fields = pointSchema->fields; + applySchemaAction(taos, &schemaAction, info); + code = loadTableSchemaFromDB(taos, pointSchema->sTableName, &dbSchema, info); + if (code != 0) { + uError("SML:0x%"PRIx64" reconcile point schema failed. can not create %s", info->id, pointSchema->sTableName); + return code; + } + } + + if (code == TSDB_CODE_SUCCESS) { + pointSchema->precision = dbSchema.precision; + + size_t pointTagSize = taosArrayGetSize(pointSchema->tags); + size_t pointFieldSize = taosArrayGetSize(pointSchema->fields); + + SHashObj* dbTagHash = dbSchema.tagHash; + SHashObj* dbFieldHash = dbSchema.fieldHash; + + for (int j = 0; j < pointTagSize; ++j) { + SSchema* pointTag = taosArrayGet(pointSchema->tags, j); + SSchemaAction schemaAction = {0}; + bool actionNeeded = false; + generateSchemaAction(pointTag, dbTagHash, dbSchema.tags, true, pointSchema->sTableName, + &schemaAction, &actionNeeded, info); + if (actionNeeded) { + code = applySchemaAction(taos, &schemaAction, info); + if (code != 0) { + destroySmlSTableSchema(&dbSchema); + return code; + } + } + } + + SSchema* pointColTs = taosArrayGet(pointSchema->fields, 0); + SSchema* dbColTs = taosArrayGet(dbSchema.fields, 0); + memcpy(pointColTs->name, dbColTs->name, TSDB_COL_NAME_LEN); + + for (int j = 1; j < pointFieldSize; ++j) { + SSchema* pointCol = taosArrayGet(pointSchema->fields, j); + SSchemaAction schemaAction = {0}; + bool actionNeeded = false; + generateSchemaAction(pointCol, dbFieldHash, dbSchema.fields,false, pointSchema->sTableName, + &schemaAction, &actionNeeded, info); + if (actionNeeded) { + code = applySchemaAction(taos, &schemaAction, info); + if (code != 0) { + destroySmlSTableSchema(&dbSchema); + return code; + } + } + } + + pointSchema->precision = dbSchema.precision; + + destroySmlSTableSchema(&dbSchema); + } else { + uError("SML:0x%"PRIx64" load table meta error: %s", info->id, tstrerror(code)); + return code; + } + tableMetaSml = taosHashIterate(info->superTables, tableMetaSml); + } +// } + return 0; +} + +static int32_t arrangePointsByChildTableName(TAOS_SML_DATA_POINT* points, int numPoints, + SHashObj* cname2points, SArray* stableSchemas, SSmlLinesInfo* info) { + for (int32_t i = 0; i < numPoints; ++i) { + TAOS_SML_DATA_POINT * point = points + i; + SSmlSTableSchema* stableSchema = taosArrayGet(stableSchemas, point->schemaIdx); + + for (int j = 0; j < point->tagNum; ++j) { + TAOS_SML_KV* kv = point->tags + j; + if (kv->type == TSDB_DATA_TYPE_TIMESTAMP) { + int64_t ts = *(int64_t*)(kv->value); + ts = convertTimePrecision(ts, TSDB_TIME_PRECISION_NANO, stableSchema->precision); + *(int64_t*)(kv->value) = ts; + } + } + + for (int j = 0; j < point->fieldNum; ++j) { + TAOS_SML_KV* kv = point->fields + j; + if (kv->type == TSDB_DATA_TYPE_TIMESTAMP) { + int64_t ts = *(int64_t*)(kv->value); + ts = convertTimePrecision(ts, TSDB_TIME_PRECISION_NANO, stableSchema->precision); + *(int64_t*)(kv->value) = ts; + } + } + + SArray* cTablePoints = NULL; + SArray** pCTablePoints = taosHashGet(cname2points, point->childTableName, strlen(point->childTableName)); + if (pCTablePoints) { + cTablePoints = *pCTablePoints; + } else { + cTablePoints = taosArrayInit(64, sizeof(point)); + taosHashPut(cname2points, point->childTableName, strlen(point->childTableName), &cTablePoints, POINTER_BYTES); + } + taosArrayPush(cTablePoints, &point); + } + + return 0; +} + +static int32_t applyChildTableDataPointsWithInsertSQL(TAOS* taos, char* cTableName, char* sTableName, SSmlSTableSchema* sTableSchema, + SArray* cTablePoints, size_t rowSize, SSmlLinesInfo* info) { + int32_t code = TSDB_CODE_SUCCESS; + size_t numTags = taosArrayGetSize(sTableSchema->tags); + size_t numCols = taosArrayGetSize(sTableSchema->fields); + size_t rows = taosArrayGetSize(cTablePoints); + SArray* tagsSchema = sTableSchema->tags; + SArray* colsSchema = sTableSchema->fields; + + TAOS_SML_KV* tagKVs[TSDB_MAX_TAGS] = {0}; + for (int i = 0; i < rows; ++i) { + TAOS_SML_DATA_POINT* pDataPoint = taosArrayGetP(cTablePoints, i); + for (int j = 0; j < pDataPoint->tagNum; ++j) { + TAOS_SML_KV* kv = pDataPoint->tags + j; + tagKVs[kv->fieldSchemaIdx] = kv; + } + } + + char* sql = taosMemoryMalloc(tsMaxSQLStringLen + 1); + if (sql == NULL) { + uError("taosMemoryMalloc sql memory error"); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + int32_t freeBytes = tsMaxSQLStringLen + 1; + int32_t totalLen = 0; + totalLen += sprintf(sql, "insert into %s using %s (", cTableName, sTableName); + for (int i = 0; i < numTags; ++i) { + SSchema* tagSchema = taosArrayGet(tagsSchema, i); + totalLen += snprintf(sql + totalLen, freeBytes - totalLen, "%s,", tagSchema->name); + } + --totalLen; + totalLen += snprintf(sql + totalLen, freeBytes - totalLen, ")"); + + totalLen += snprintf(sql + totalLen, freeBytes - totalLen, " tags ("); + + // for (int i = 0; i < numTags; ++i) { + // snprintf(sql+strlen(sql), freeBytes-strlen(sql), "?,"); + // } + for (int i = 0; i < numTags; ++i) { + if (tagKVs[i] == NULL) { + totalLen += snprintf(sql + totalLen, freeBytes - totalLen, "NULL,"); + } else { + TAOS_SML_KV* kv = tagKVs[i]; + size_t beforeLen = totalLen; + int32_t len = 0; + converToStr(sql + beforeLen, kv->type, kv->value, kv->length, &len); + totalLen += len; + totalLen += snprintf(sql + totalLen, freeBytes - totalLen, ","); + } + } + --totalLen; + totalLen += snprintf(sql + totalLen, freeBytes - totalLen, ") ("); + + for (int i = 0; i < numCols; ++i) { + SSchema* colSchema = taosArrayGet(colsSchema, i); + totalLen += snprintf(sql + totalLen, freeBytes - totalLen, "%s,", colSchema->name); + } + --totalLen; + totalLen += snprintf(sql + totalLen, freeBytes - totalLen, ") values "); + + TAOS_SML_KV** colKVs = taosMemoryMalloc(numCols * sizeof(TAOS_SML_KV*)); + for (int r = 0; r < rows; ++r) { + totalLen += snprintf(sql + totalLen, freeBytes - totalLen, "("); + + memset(colKVs, 0, numCols * sizeof(TAOS_SML_KV*)); + + TAOS_SML_DATA_POINT* point = taosArrayGetP(cTablePoints, r); + for (int i = 0; i < point->fieldNum; ++i) { + TAOS_SML_KV* kv = point->fields + i; + colKVs[kv->fieldSchemaIdx] = kv; + } + + for (int i = 0; i < numCols; ++i) { + if (colKVs[i] == NULL) { + totalLen += snprintf(sql + totalLen, freeBytes - totalLen, "NULL,"); + } else { + TAOS_SML_KV* kv = colKVs[i]; + size_t beforeLen = totalLen; + int32_t len = 0; + converToStr(sql + beforeLen, kv->type, kv->value, kv->length, &len); + totalLen += len; + totalLen += snprintf(sql + totalLen, freeBytes - totalLen, ","); + } + } + --totalLen; + totalLen += snprintf(sql + totalLen, freeBytes - totalLen, ")"); + } + taosMemoryFree(colKVs); + sql[totalLen] = '\0'; + + uDebug("SML:0x%" PRIx64 " insert child table table %s of super table %s sql: %s", info->id, cTableName, sTableName, + sql); + + bool tryAgain = false; + int32_t try = 0; + do { + TAOS_RES* res = taos_query(taos, sql); + code = taos_errno(res); + if (code != 0) { + uError("SML:0x%"PRIx64 " taos_query return %d:%s", info->id, code, taos_errstr(res)); + } + + uDebug("SML:0x%"PRIx64 " taos_query inserted %d rows", info->id, taos_affected_rows(res)); + info->affectedRows += taos_affected_rows(res); + taos_free_result(res); + + tryAgain = false; + if ((code == TSDB_CODE_TDB_INVALID_TABLE_ID + || code == TSDB_CODE_VND_INVALID_VGROUP_ID + || code == TSDB_CODE_TDB_TABLE_RECONFIGURE + || code == TSDB_CODE_APP_NOT_READY + || code == TSDB_CODE_RPC_NETWORK_UNAVAIL) && try++ < TSDB_MAX_REPLICA) { + tryAgain = true; + } + + if (code == TSDB_CODE_TDB_INVALID_TABLE_ID || code == TSDB_CODE_VND_INVALID_VGROUP_ID) { + TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); + int32_t code2 = taos_errno(res2); + if (code2 != TSDB_CODE_SUCCESS) { + uError("SML:0x%" PRIx64 " insert child table by sql. reset query cache. error: %s", info->id, taos_errstr(res2)); + } + taos_free_result(res2); + if (tryAgain) { + taosMsleep(100 * (2 << try)); + } + } + + if (code == TSDB_CODE_APP_NOT_READY || code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { + if (tryAgain) { + taosMsleep( 100 * (2 << try)); + } + } + } while (tryAgain); + + taosMemoryFree(sql); + + return code; +} + +static int32_t applyChildTableDataPointsWithStmt(TAOS* taos, char* cTableName, char* sTableName, SSmlSTableSchema* sTableSchema, + SArray* cTablePoints, size_t rowSize, SSmlLinesInfo* info) { + size_t numTags = taosArrayGetSize(sTableSchema->tags); + size_t numCols = taosArrayGetSize(sTableSchema->fields); + size_t rows = taosArrayGetSize(cTablePoints); + + TAOS_SML_KV* tagKVs[TSDB_MAX_TAGS] = {0}; + for (int i= 0; i < rows; ++i) { + TAOS_SML_DATA_POINT * pDataPoint = taosArrayGetP(cTablePoints, i); + for (int j = 0; j < pDataPoint->tagNum; ++j) { + TAOS_SML_KV* kv = pDataPoint->tags + j; + tagKVs[kv->fieldSchemaIdx] = kv; + } + } + + //tag bind + SArray* tagBinds = taosArrayInit(numTags, sizeof(TAOS_BIND)); + taosArraySetSize(tagBinds, numTags); + int isNullColBind = TSDB_TRUE; + for (int j = 0; j < numTags; ++j) { + TAOS_BIND* bind = taosArrayGet(tagBinds, j); + bind->is_null = &isNullColBind; + } + for (int j = 0; j < numTags; ++j) { + if (tagKVs[j] == NULL) continue; + TAOS_SML_KV* kv = tagKVs[j]; + TAOS_BIND* bind = taosArrayGet(tagBinds, kv->fieldSchemaIdx); + bind->buffer_type = kv->type; + bind->length = taosMemoryMalloc(sizeof(uintptr_t*)); + *bind->length = kv->length; + bind->buffer = kv->value; + bind->is_null = NULL; + } + + //rows bind + SArray* rowsBind = taosArrayInit(rows, POINTER_BYTES); + for (int i = 0; i < rows; ++i) { + TAOS_SML_DATA_POINT* point = taosArrayGetP(cTablePoints, i); + + TAOS_BIND* colBinds = calloc(numCols, sizeof(TAOS_BIND)); + if (colBinds == NULL) { + uError("SML:0x%"PRIx64" taos_sml_insert insert points, failed to allocated memory for TAOS_BIND, " + "num of rows: %zu, num of cols: %zu", info->id, rows, numCols); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + for (int j = 0; j < numCols; ++j) { + TAOS_BIND* bind = colBinds + j; + bind->is_null = &isNullColBind; + } + for (int j = 0; j < point->fieldNum; ++j) { + TAOS_SML_KV* kv = point->fields + j; + TAOS_BIND* bind = colBinds + kv->fieldSchemaIdx; + bind->buffer_type = kv->type; + bind->length = taosMemoryMalloc(sizeof(uintptr_t*)); + *bind->length = kv->length; + bind->buffer = kv->value; + bind->is_null = NULL; + } + taosArrayPush(rowsBind, &colBinds); + } + + int32_t code = 0; + code = insertChildTablePointsBatch(taos, cTableName, sTableName, sTableSchema->tags, tagBinds, sTableSchema->fields, rowsBind, rowSize, info); + if (code != 0) { + uError("SML:0x%"PRIx64" insert into child table %s failed. error %s", info->id, cTableName, tstrerror(code)); + } + + //taosMemoryFree rows bind + for (int i = 0; i < rows; ++i) { + TAOS_BIND* colBinds = taosArrayGetP(rowsBind, i); + for (int j = 0; j < numCols; ++j) { + TAOS_BIND* bind = colBinds + j; + taosMemoryFree(bind->length); + } + taosMemoryFree(colBinds); + } + taosArrayDestroy(&rowsBind); + //taosMemoryFree tag bind + for (int i = 0; i < taosArrayGetSize(tagBinds); ++i) { + TAOS_BIND* bind = taosArrayGet(tagBinds, i); + taosMemoryFree(bind->length); + } + taosArrayDestroy(&tagBinds); + return code; +} + +static int32_t insertChildTablePointsBatch(TAOS* taos, char* cTableName, char* sTableName, + SArray* tagsSchema, SArray* tagsBind, + SArray* colsSchema, SArray* rowsBind, + size_t rowSize, SSmlLinesInfo* info) { + size_t numTags = taosArrayGetSize(tagsSchema); + size_t numCols = taosArrayGetSize(colsSchema); + char* sql = taosMemoryMalloc(tsMaxSQLStringLen+1); + if (sql == NULL) { + uError("taosMemoryMalloc sql memory error"); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + int32_t freeBytes = tsMaxSQLStringLen + 1 ; + sprintf(sql, "insert into ? using %s (", sTableName); + for (int i = 0; i < numTags; ++i) { + SSchema* tagSchema = taosArrayGet(tagsSchema, i); + snprintf(sql+strlen(sql), freeBytes-strlen(sql), "%s,", tagSchema->name); + } + snprintf(sql + strlen(sql) - 1, freeBytes-strlen(sql)+1, ")"); + + snprintf(sql + strlen(sql), freeBytes-strlen(sql), " tags ("); + + for (int i = 0; i < numTags; ++i) { + snprintf(sql+strlen(sql), freeBytes-strlen(sql), "?,"); + } + snprintf(sql + strlen(sql) - 1, freeBytes-strlen(sql)+1, ") ("); + + for (int i = 0; i < numCols; ++i) { + SSchema* colSchema = taosArrayGet(colsSchema, i); + snprintf(sql+strlen(sql), freeBytes-strlen(sql), "%s,", colSchema->name); + } + snprintf(sql + strlen(sql)-1, freeBytes-strlen(sql)+1, ") values ("); + + for (int i = 0; i < numCols; ++i) { + snprintf(sql+strlen(sql), freeBytes-strlen(sql), "?,"); + } + snprintf(sql + strlen(sql)-1, freeBytes-strlen(sql)+1, ")"); + sql[strlen(sql)] = '\0'; + + uDebug("SML:0x%"PRIx64" insert child table table %s of super table %s : %s", info->id, cTableName, sTableName, sql); + + size_t maxBatchSize = TSDB_MAX_WAL_SIZE/rowSize * 2 / 3; + size_t rows = taosArrayGetSize(rowsBind); + size_t batchSize = MIN(maxBatchSize, rows); + uDebug("SML:0x%"PRIx64" insert rows into child table %s. num of rows: %zu, batch size: %zu", + info->id, cTableName, rows, batchSize); + SArray* batchBind = taosArrayInit(batchSize, POINTER_BYTES); + int32_t code = TSDB_CODE_SUCCESS; + for (int i = 0; i < rows;) { + int j = i; + for (; j < i + batchSize && j i) { + uDebug("SML:0x%"PRIx64" insert child table batch from line %d to line %d.", info->id, i, j - 1); + code = doInsertChildTablePoints(taos, sql, cTableName, tagsBind, batchBind, info); + if (code != 0) { + taosArrayDestroy(&batchBind); + tfree(sql); + return code; + } + taosArrayClear(batchBind); + } + i = j; + } + taosArrayDestroy(&batchBind); + tfree(sql); + return code; + +} +static int32_t doInsertChildTablePoints(TAOS* taos, char* sql, char* cTableName, SArray* tagsBind, SArray* batchBind, + SSmlLinesInfo* info) { + int32_t code = 0; + + TAOS_STMT* stmt = taos_stmt_init(taos); + if (stmt == NULL) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + code = taos_stmt_prepare(stmt, sql, (unsigned long)strlen(sql)); + + if (code != 0) { + uError("SML:0x%"PRIx64" taos_stmt_prepare return %d:%s", info->id, code, taos_stmt_errstr(stmt)); + taos_stmt_close(stmt); + return code; + } + + bool tryAgain = false; + int32_t try = 0; + do { + code = taos_stmt_set_tbname_tags(stmt, cTableName, TARRAY_GET_START(tagsBind)); + if (code != 0) { + uError("SML:0x%"PRIx64" taos_stmt_set_tbname return %d:%s", info->id, code, taos_stmt_errstr(stmt)); + + int affectedRows = taos_stmt_affected_rows(stmt); + info->affectedRows += affectedRows; + + taos_stmt_close(stmt); + return code; + } + + size_t rows = taosArrayGetSize(batchBind); + for (int32_t i = 0; i < rows; ++i) { + TAOS_BIND* colsBinds = taosArrayGetP(batchBind, i); + code = taos_stmt_bind_param(stmt, colsBinds); + if (code != 0) { + uError("SML:0x%"PRIx64" taos_stmt_bind_param return %d:%s", info->id, code, taos_stmt_errstr(stmt)); + + int affectedRows = taos_stmt_affected_rows(stmt); + info->affectedRows += affectedRows; + + taos_stmt_close(stmt); + return code; + } + code = taos_stmt_add_batch(stmt); + if (code != 0) { + uError("SML:0x%"PRIx64" taos_stmt_add_batch return %d:%s", info->id, code, taos_stmt_errstr(stmt)); + + int affectedRows = taos_stmt_affected_rows(stmt); + info->affectedRows += affectedRows; + + taos_stmt_close(stmt); + return code; + } + } + + code = taos_stmt_execute(stmt); + if (code != 0) { + uError("SML:0x%"PRIx64" taos_stmt_execute return %d:%s, try:%d", info->id, code, taos_stmt_errstr(stmt), try); + } + uDebug("SML:0x%"PRIx64" taos_stmt_execute inserted %d rows", info->id, taos_stmt_affected_rows(stmt)); + + tryAgain = false; + if ((code == TSDB_CODE_TDB_INVALID_TABLE_ID + || code == TSDB_CODE_VND_INVALID_VGROUP_ID + || code == TSDB_CODE_TDB_TABLE_RECONFIGURE + || code == TSDB_CODE_APP_NOT_READY + || code == TSDB_CODE_RPC_NETWORK_UNAVAIL) && try++ < TSDB_MAX_REPLICA) { + tryAgain = true; + } + + if (code == TSDB_CODE_TDB_INVALID_TABLE_ID || code == TSDB_CODE_VND_INVALID_VGROUP_ID) { + TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); + int32_t code2 = taos_errno(res2); + if (code2 != TSDB_CODE_SUCCESS) { + uError("SML:0x%" PRIx64 " insert child table. reset query cache. error: %s", info->id, taos_errstr(res2)); + } + taos_free_result(res2); + if (tryAgain) { + taosMsleep(100 * (2 << try)); + } + } + if (code == TSDB_CODE_APP_NOT_READY || code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { + if (tryAgain) { + taosMsleep( 100 * (2 << try)); + } + } + } while (tryAgain); + + int affectedRows = taos_stmt_affected_rows(stmt); + info->affectedRows += affectedRows; + + taos_stmt_close(stmt); + return code; + + return 0; +} + +static int32_t applyChildTableDataPoints(TAOS* taos, char* cTableName, char* sTableName, SSmlSTableSchema* sTableSchema, + SArray* cTablePoints, size_t rowSize, SSmlLinesInfo* info) { + int32_t code = TSDB_CODE_SUCCESS; + size_t childTableDataPoints = taosArrayGetSize(cTablePoints); + if (childTableDataPoints < 10) { + code = applyChildTableDataPointsWithInsertSQL(taos, cTableName, sTableName, sTableSchema, cTablePoints, rowSize, info); + } else { + code = applyChildTableDataPointsWithStmt(taos, cTableName, sTableName, sTableSchema, cTablePoints, rowSize, info); + } + return code; +} + +static int32_t applyDataPoints(TAOS* taos, TAOS_SML_DATA_POINT* points, int32_t numPoints, SArray* stableSchemas, SSmlLinesInfo* info) { + int32_t code = TSDB_CODE_SUCCESS; + + SHashObj* cname2points = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); + arrangePointsByChildTableName(points, numPoints, cname2points, stableSchemas, info); + + SArray** pCTablePoints = taosHashIterate(cname2points, NULL); + while (pCTablePoints) { + SArray* cTablePoints = *pCTablePoints; + + TAOS_SML_DATA_POINT* point = taosArrayGetP(cTablePoints, 0); + SSmlSTableSchema* sTableSchema = taosArrayGet(stableSchemas, point->schemaIdx); + + size_t rowSize = 0; + size_t numCols = taosArrayGetSize(sTableSchema->fields); + for (int i = 0; i < numCols; ++i) { + SSchema* colSchema = taosArrayGet(sTableSchema->fields, i); + rowSize += colSchema->bytes; + } + + uDebug("SML:0x%"PRIx64" apply child table points. child table: %s of super table %s, row size: %zu", + info->id, point->childTableName, point->stableName, rowSize); + code = applyChildTableDataPoints(taos, point->childTableName, point->stableName, sTableSchema, cTablePoints, rowSize, info); + if (code != 0) { + uError("SML:0x%"PRIx64" Apply child table points failed. child table %s, error %s", info->id, point->childTableName, tstrerror(code)); + goto cleanup; + } + + uDebug("SML:0x%"PRIx64" successfully applied data points of child table %s", info->id, point->childTableName); + + pCTablePoints = taosHashIterate(cname2points, pCTablePoints); + } + +cleanup: + pCTablePoints = taosHashIterate(cname2points, NULL); + while (pCTablePoints) { + SArray* pPoints = *pCTablePoints; + taosArrayDestroy(&pPoints); + pCTablePoints = taosHashIterate(cname2points, pCTablePoints); + } + taosHashCleanup(cname2points); + return code; +} + +static int doSmlInsertOneDataPoint(TAOS* taos, TAOS_SML_DATA_POINT* point, SSmlLinesInfo* info) { + int32_t code = TSDB_CODE_SUCCESS; + + if (!point->childTableName) { + int tableNameLen = TSDB_TABLE_NAME_LEN; + point->childTableName = calloc(1, tableNameLen + 1); + getSmlMd5ChildTableName(point, point->childTableName, &tableNameLen, info); + point->childTableName[tableNameLen] = '\0'; + } + + STableMeta* tableMeta = NULL; + int32_t ret = getSuperTableMetaFromLocalCache(taos, point->stableName, &tableMeta, info); + if (ret != TSDB_CODE_SUCCESS) { + return ret; + } + uint8_t precision = tableMeta->tableInfo.precision; + taosMemoryFree(tableMeta); + + char* sql = taosMemoryMalloc(TSDB_MAX_SQL_LEN + 1); + int freeBytes = TSDB_MAX_SQL_LEN; + int sqlLen = 0; + sqlLen += snprintf(sql + sqlLen, freeBytes - sqlLen, "insert into %s(", point->childTableName); + for (int col = 0; col < point->fieldNum; ++col) { + TAOS_SML_KV* kv = point->fields + col; + sqlLen += snprintf(sql + sqlLen, freeBytes - sqlLen, "%s,", kv->key); + } + --sqlLen; + sqlLen += snprintf(sql + sqlLen, freeBytes - sqlLen, ") values ("); + TAOS_SML_KV* tsField = point->fields + 0; + int64_t ts = *(int64_t*)(tsField->value); + ts = convertTimePrecision(ts, TSDB_TIME_PRECISION_NANO, precision); + sqlLen += snprintf(sql + sqlLen, freeBytes - sqlLen, "%" PRId64 ",", ts); + for (int col = 1; col < point->fieldNum; ++col) { + TAOS_SML_KV* kv = point->fields + col; + int32_t len = 0; + converToStr(sql + sqlLen, kv->type, kv->value, kv->length, &len); + sqlLen += len; + sqlLen += snprintf(sql + sqlLen, freeBytes - sqlLen, ","); + } + --sqlLen; + sqlLen += snprintf(sql + sqlLen, freeBytes - sqlLen, ")"); + sql[sqlLen] = 0; + + uDebug("SML:0x%" PRIx64 " insert child table table %s of super table %s sql: %s", info->id, + point->childTableName, point->stableName, sql); + TAOS_RES* res = taos_query(taos, sql); + taosMemoryFree(sql); + code = taos_errno(res); + info->affectedRows = taos_affected_rows(res); + taos_free_result(res); + + return code; +} + +int tscSmlInsert(TAOS* taos, SSmlLinesInfo* info) { + uDebug("SML:0x%"PRIx64" taos_sml_insert. number of super tables: %d", info->id, taosHashGetSize(info->superTables)); + int32_t code = TSDB_CODE_SUCCESS; + info->affectedRows = 0; + + uDebug("SML:0x%"PRIx64" modify db schemas", info->id); + code = modifyDBSchemas(taos, info); + if (code != 0) { + uError("SML:0x%"PRIx64" error change db schema : %s", info->id, tstrerror(code)); + goto clean_up; + } + + uDebug("SML:0x%"PRIx64" apply data points", info->id); + code = applyDataPoints(taos, points, numPoint, stableSchemas, info); + if (code != 0) { + uError("SML:0x%"PRIx64" error apply data points : %s", info->id, tstrerror(code)); + } + +clean_up: + for (int i = 0; i < taosArrayGetSize(stableSchemas); ++i) { + SSmlSTableSchema* schema = taosArrayGet(stableSchemas, i); + taosArrayDestroy(&schema->fields); + taosArrayDestroy(&schema->tags); + } + taosArrayDestroy(&stableSchemas); + return code; +} + +//========================================================================= + +/* Field Escape charaters + 1: measurement Comma,Space + 2: tag_key, tag_value, field_key Comma,Equal Sign,Space + 3: field_value Double quote,Backslash +*/ +static void escapeSpecialCharacter(uint8_t field, const char **pos) { + const char *cur = *pos; + if (*cur != '\\') { + return; + } + switch (field) { + case 1: + switch (*(cur + 1)) { + case ',': + case ' ': + cur++; + break; + default: + break; + } + break; + case 2: + switch (*(cur + 1)) { + case ',': + case ' ': + case '=': + cur++; + break; + default: + break; + } + break; + case 3: + switch (*(cur + 1)) { + case '"': + case '\\': + cur++; + break; + default: + break; + } + break; + default: + break; + } + *pos = cur; +} + +char* addEscapeCharToString(char *str, int32_t len) { + if (str == NULL) { + return NULL; + } + memmove(str + 1, str, len); + str[0] = str[len + 1] = TS_BACKQUOTE_CHAR; + str[len + 2] = '\0'; + return str; +} + +bool isValidInteger(char *str) { + char *c = str; + if (*c != '+' && *c != '-' && !isdigit(*c)) { + return false; + } + c++; + while (*c != '\0') { + if (!isdigit(*c)) { + return false; + } + c++; + } + return true; +} + +bool isValidFloat(char *str) { + char *c = str; + uint8_t has_dot, has_exp, has_sign; + has_dot = 0; + has_exp = 0; + has_sign = 0; + + if (*c != '+' && *c != '-' && *c != '.' && !isdigit(*c)) { + return false; + } + if (*c == '.' && isdigit(*(c + 1))) { + has_dot = 1; + } + c++; + while (*c != '\0') { + if (!isdigit(*c)) { + switch (*c) { + case '.': { + if (!has_dot && !has_exp && isdigit(*(c + 1))) { + has_dot = 1; + } else { + return false; + } + break; + } + case 'e': + case 'E': { + if (!has_exp && isdigit(*(c - 1)) && + (isdigit(*(c + 1)) || + *(c + 1) == '+' || + *(c + 1) == '-')) { + has_exp = 1; + } else { + return false; + } + break; + } + case '+': + case '-': { + if (!has_sign && has_exp && isdigit(*(c + 1))) { + has_sign = 1; + } else { + return false; + } + break; + } + default: { + return false; + } + } + } + c++; + } //while + return true; +} + +static bool isInteger(char *pVal, uint16_t len, bool *has_sign) { + if (len <= 1) { + return false; + } + if (pVal[len - 1] == 'i') { + *has_sign = true; + return true; + } + if (pVal[len - 1] == 'u') { + *has_sign = false; + return true; + } + + return false; +} + +static bool isTinyInt(char *pVal, uint16_t len) { + if (len <= 2) { + return false; + } + if (!strcasecmp(&pVal[len - 2], "i8")) { + //printf("Type is int8(%s)\n", pVal); + return true; + } + return false; +} + +static bool isTinyUint(char *pVal, uint16_t len) { + if (len <= 2) { + return false; + } + if (pVal[0] == '-') { + return false; + } + if (!strcasecmp(&pVal[len - 2], "u8")) { + //printf("Type is uint8(%s)\n", pVal); + return true; + } + return false; +} + +static bool isSmallInt(char *pVal, uint16_t len) { + if (len <= 3) { + return false; + } + if (!strcasecmp(&pVal[len - 3], "i16")) { + //printf("Type is int16(%s)\n", pVal); + return true; + } + return false; +} + +static bool isSmallUint(char *pVal, uint16_t len) { + if (len <= 3) { + return false; + } + if (pVal[0] == '-') { + return false; + } + if (strcasecmp(&pVal[len - 3], "u16") == 0) { + //printf("Type is uint16(%s)\n", pVal); + return true; + } + return false; +} + +static bool isInt(char *pVal, uint16_t len) { + if (len <= 3) { + return false; + } + if (strcasecmp(&pVal[len - 3], "i32") == 0) { + //printf("Type is int32(%s)\n", pVal); + return true; + } + return false; +} + +static bool isUint(char *pVal, uint16_t len) { + if (len <= 3) { + return false; + } + if (pVal[0] == '-') { + return false; + } + if (strcasecmp(&pVal[len - 3], "u32") == 0) { + //printf("Type is uint32(%s)\n", pVal); + return true; + } + return false; +} + +static bool isBigInt(char *pVal, uint16_t len) { + if (len <= 3) { + return false; + } + if (strcasecmp(&pVal[len - 3], "i64") == 0) { + //printf("Type is int64(%s)\n", pVal); + return true; + } + return false; +} + +static bool isBigUint(char *pVal, uint16_t len) { + if (len <= 3) { + return false; + } + if (pVal[0] == '-') { + return false; + } + if (strcasecmp(&pVal[len - 3], "u64") == 0) { + //printf("Type is uint64(%s)\n", pVal); + return true; + } + return false; +} + +static bool isFloat(char *pVal, uint16_t len) { + if (len <= 3) { + return false; + } + if (strcasecmp(&pVal[len - 3], "f32") == 0) { + //printf("Type is float(%s)\n", pVal); + return true; + } + return false; +} + +static bool isDouble(char *pVal, uint16_t len) { + if (len <= 3) { + return false; + } + if (strcasecmp(&pVal[len - 3], "f64") == 0) { + //printf("Type is double(%s)\n", pVal); + return true; + } + return false; +} + +static bool isBool(char *pVal, uint16_t len, bool *bVal) { + if ((len == 1) && !strcasecmp(&pVal[len - 1], "t")) { + //printf("Type is bool(%c)\n", pVal[len - 1]); + *bVal = true; + return true; + } + + if ((len == 1) && !strcasecmp(&pVal[len - 1], "f")) { + //printf("Type is bool(%c)\n", pVal[len - 1]); + *bVal = false; + return true; + } + + if((len == 4) && !strcasecmp(&pVal[len - 4], "true")) { + //printf("Type is bool(%s)\n", &pVal[len - 4]); + *bVal = true; + return true; + } + if((len == 5) && !strcasecmp(&pVal[len - 5], "false")) { + //printf("Type is bool(%s)\n", &pVal[len - 5]); + *bVal = false; + return true; + } + return false; +} + +static bool isBinary(char *pVal, uint16_t len) { + //binary: "abc" + if (len < 2) { + return false; + } + //binary + if (pVal[0] == '"' && pVal[len - 1] == '"') { + //printf("Type is binary(%s)\n", pVal); + return true; + } + return false; +} + +static bool isNchar(char *pVal, uint16_t len) { + //nchar: L"abc" + if (len < 3) { + return false; + } + if ((pVal[0] == 'l' || pVal[0] == 'L')&& pVal[1] == '"' && pVal[len - 1] == '"') { + //printf("Type is nchar(%s)\n", pVal); + return true; + } + return false; +} + +static int32_t isTimeStamp(char *pVal, uint16_t len, SMLTimeStampType *tsType, SSmlLinesInfo* info) { + if (len == 0) { + return TSDB_CODE_SUCCESS; + } + if ((len == 1) && pVal[0] == '0') { + *tsType = SML_TIME_STAMP_NOW; + return TSDB_CODE_SUCCESS; + } + + for (int i = 0; i < len; ++i) { + if(!isdigit(pVal[i])) { + return TSDB_CODE_TSC_INVALID_TIME_STAMP; + } + } + + /* For InfluxDB line protocol use user passed timestamp precision + * For OpenTSDB protocols only 10 digit(seconds) or 13 digits(milliseconds) + * precision allowed + */ + if (info->protocol == TSDB_SML_LINE_PROTOCOL) { + if (info->tsType != SML_TIME_STAMP_NOT_CONFIGURED) { + *tsType = info->tsType; + } else { + *tsType = SML_TIME_STAMP_NANO_SECONDS; + } + } else if (info->protocol == TSDB_SML_TELNET_PROTOCOL) { + if (len == SML_TIMESTAMP_SECOND_DIGITS) { + *tsType = SML_TIME_STAMP_SECONDS; + } else if (len == SML_TIMESTAMP_MILLI_SECOND_DIGITS) { + *tsType = SML_TIME_STAMP_MILLI_SECONDS; + } else { + return TSDB_CODE_TSC_INVALID_TIME_STAMP; + } + } + return TSDB_CODE_SUCCESS; + + //if (pVal[len - 1] == 's') { + // switch (pVal[len - 2]) { + // case 'm': + // *tsType = SML_TIME_STAMP_MILLI_SECONDS; + // break; + // case 'u': + // *tsType = SML_TIME_STAMP_MICRO_SECONDS; + // break; + // case 'n': + // *tsType = SML_TIME_STAMP_NANO_SECONDS; + // break; + // default: + // if (isdigit(pVal[len - 2])) { + // *tsType = SML_TIME_STAMP_SECONDS; + // break; + // } else { + // return false; + // } + // } + // //printf("Type is timestamp(%s)\n", pVal); + // return true; + //} + //return false; +} + +static bool convertStrToNumber(TAOS_SML_KV *pVal, char *str, SSmlLinesInfo* info) { + errno = 0; + uint8_t type = pVal->type; + int16_t length = pVal->length; + int64_t val_s = 0; + uint64_t val_u = 0; + double val_d = 0.0; + + strntolower_s(str, str, (int32_t)strlen(str)); + if (IS_FLOAT_TYPE(type)) { + val_d = strtod(str, NULL); + } else { + if (IS_SIGNED_NUMERIC_TYPE(type)) { + val_s = strtoll(str, NULL, 10); + } else { + val_u = strtoull(str, NULL, 10); + } + } + + if (errno == ERANGE) { + uError("SML:0x%"PRIx64" Convert number(%s) out of range", info->id, str); + return false; + } + + switch (type) { + case TSDB_DATA_TYPE_TINYINT: + if (!IS_VALID_TINYINT(val_s)) { + return false; + } + pVal->value = calloc(length, 1); + *(int8_t *)(pVal->value) = (int8_t)val_s; + break; + case TSDB_DATA_TYPE_UTINYINT: + if (!IS_VALID_UTINYINT(val_u)) { + return false; + } + pVal->value = calloc(length, 1); + *(uint8_t *)(pVal->value) = (uint8_t)val_u; + break; + case TSDB_DATA_TYPE_SMALLINT: + if (!IS_VALID_SMALLINT(val_s)) { + return false; + } + pVal->value = calloc(length, 1); + *(int16_t *)(pVal->value) = (int16_t)val_s; + break; + case TSDB_DATA_TYPE_USMALLINT: + if (!IS_VALID_USMALLINT(val_u)) { + return false; + } + pVal->value = calloc(length, 1); + *(uint16_t *)(pVal->value) = (uint16_t)val_u; + break; + case TSDB_DATA_TYPE_INT: + if (!IS_VALID_INT(val_s)) { + return false; + } + pVal->value = calloc(length, 1); + *(int32_t *)(pVal->value) = (int32_t)val_s; + break; + case TSDB_DATA_TYPE_UINT: + if (!IS_VALID_UINT(val_u)) { + return false; + } + pVal->value = calloc(length, 1); + *(uint32_t *)(pVal->value) = (uint32_t)val_u; + break; + case TSDB_DATA_TYPE_BIGINT: + if (!IS_VALID_BIGINT(val_s)) { + return false; + } + pVal->value = calloc(length, 1); + *(int64_t *)(pVal->value) = (int64_t)val_s; + break; + case TSDB_DATA_TYPE_UBIGINT: + if (!IS_VALID_UBIGINT(val_u)) { + return false; + } + pVal->value = calloc(length, 1); + *(uint64_t *)(pVal->value) = (uint64_t)val_u; + break; + case TSDB_DATA_TYPE_FLOAT: + if (!IS_VALID_FLOAT(val_d)) { + return false; + } + pVal->value = calloc(length, 1); + *(float *)(pVal->value) = (float)val_d; + break; + case TSDB_DATA_TYPE_DOUBLE: + if (!IS_VALID_DOUBLE(val_d)) { + return false; + } + pVal->value = calloc(length, 1); + *(double *)(pVal->value) = (double)val_d; + break; + default: + return false; + } + return true; +} +//len does not include '\0' from value. +bool convertSmlValueType(TAOS_SML_KV *pVal, char *value, + uint16_t len, SSmlLinesInfo* info, bool isTag) { + if (len <= 0) { + return false; + } + + //convert tags value to Nchar + if (isTag) { + pVal->type = TSDB_DATA_TYPE_NCHAR; + pVal->length = len; + pVal->value = calloc(pVal->length, 1); + memcpy(pVal->value, value, pVal->length); + return true; + } + + //integer number + bool has_sign; + if (isInteger(value, len, &has_sign)) { + pVal->type = has_sign ? TSDB_DATA_TYPE_BIGINT : TSDB_DATA_TYPE_UBIGINT; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + value[len - 1] = '\0'; + if (!isValidInteger(value) || !convertStrToNumber(pVal, value, info)) { + return false; + } + return true; + } + if (isTinyInt(value, len)) { + pVal->type = TSDB_DATA_TYPE_TINYINT; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + value[len - 2] = '\0'; + if (!isValidInteger(value) || !convertStrToNumber(pVal, value, info)) { + return false; + } + return true; + } + if (isTinyUint(value, len)) { + pVal->type = TSDB_DATA_TYPE_UTINYINT; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + value[len - 2] = '\0'; + if (!isValidInteger(value) || !convertStrToNumber(pVal, value, info)) { + return false; + } + return true; + } + if (isSmallInt(value, len)) { + pVal->type = TSDB_DATA_TYPE_SMALLINT; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + value[len - 3] = '\0'; + if (!isValidInteger(value) || !convertStrToNumber(pVal, value, info)) { + return false; + } + return true; + } + if (isSmallUint(value, len)) { + pVal->type = TSDB_DATA_TYPE_USMALLINT; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + value[len - 3] = '\0'; + if (!isValidInteger(value) || !convertStrToNumber(pVal, value, info)) { + return false; + } + return true; + } + if (isInt(value, len)) { + pVal->type = TSDB_DATA_TYPE_INT; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + value[len - 3] = '\0'; + if (!isValidInteger(value) || !convertStrToNumber(pVal, value, info)) { + return false; + } + return true; + } + if (isUint(value, len)) { + pVal->type = TSDB_DATA_TYPE_UINT; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + value[len - 3] = '\0'; + if (!isValidInteger(value) || !convertStrToNumber(pVal, value, info)) { + return false; + } + return true; + } + if (isBigInt(value, len)) { + pVal->type = TSDB_DATA_TYPE_BIGINT; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + value[len - 3] = '\0'; + if (!isValidInteger(value) || !convertStrToNumber(pVal, value, info)) { + return false; + } + return true; + } + if (isBigUint(value, len)) { + pVal->type = TSDB_DATA_TYPE_UBIGINT; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + value[len - 3] = '\0'; + if (!isValidInteger(value) || !convertStrToNumber(pVal, value, info)) { + return false; + } + return true; + } + //floating number + if (isFloat(value, len)) { + pVal->type = TSDB_DATA_TYPE_FLOAT; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + value[len - 3] = '\0'; + if (!isValidFloat(value) || !convertStrToNumber(pVal, value, info)) { + return false; + } + return true; + } + if (isDouble(value, len)) { + pVal->type = TSDB_DATA_TYPE_DOUBLE; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + value[len - 3] = '\0'; + if (!isValidFloat(value) || !convertStrToNumber(pVal, value, info)) { + return false; + } + return true; + } + //binary + if (isBinary(value, len)) { + pVal->type = TSDB_DATA_TYPE_BINARY; + pVal->length = len - 2; + pVal->value = calloc(pVal->length, 1); + //copy after " + memcpy(pVal->value, value + 1, pVal->length); + return true; + } + //nchar + if (isNchar(value, len)) { + pVal->type = TSDB_DATA_TYPE_NCHAR; + pVal->length = len - 3; + pVal->value = calloc(pVal->length, 1); + //copy after L" + memcpy(pVal->value, value + 2, pVal->length); + return true; + } + //bool + bool bVal; + if (isBool(value, len, &bVal)) { + pVal->type = TSDB_DATA_TYPE_BOOL; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + pVal->value = calloc(pVal->length, 1); + memcpy(pVal->value, &bVal, pVal->length); + return true; + } + + //Handle default(no appendix) type as DOUBLE + if (isValidInteger(value) || isValidFloat(value)) { + pVal->type = TSDB_DATA_TYPE_DOUBLE; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + if (!convertStrToNumber(pVal, value, info)) { + return false; + } + return true; + } + return false; +} + +static int32_t getTimeStampValue(char *value, uint16_t len, + SMLTimeStampType type, int64_t *ts, SSmlLinesInfo* info) { + + //No appendix or no timestamp given (len = 0) + if (len != 0 && type != SML_TIME_STAMP_NOW) { + *ts = (int64_t)strtoll(value, NULL, 10); + } else { + type = SML_TIME_STAMP_NOW; + } + switch (type) { + case SML_TIME_STAMP_NOW: { + *ts = taosGetTimestampNs(); + break; + } + case SML_TIME_STAMP_HOURS: { + *ts = (int64_t)(*ts * 3600 * 1e9); + break; + } + case SML_TIME_STAMP_MINUTES: { + *ts = (int64_t)(*ts * 60 * 1e9); + break; + } + case SML_TIME_STAMP_SECONDS: { + *ts = (int64_t)(*ts * 1e9); + break; + } + case SML_TIME_STAMP_MILLI_SECONDS: { + *ts = convertTimePrecision(*ts, TSDB_TIME_PRECISION_MILLI, TSDB_TIME_PRECISION_NANO); + break; + } + case SML_TIME_STAMP_MICRO_SECONDS: { + *ts = convertTimePrecision(*ts, TSDB_TIME_PRECISION_MICRO, TSDB_TIME_PRECISION_NANO); + break; + } + case SML_TIME_STAMP_NANO_SECONDS: { + *ts = *ts * 1; + break; + } + default: { + return TSDB_CODE_TSC_INVALID_TIME_STAMP; + } + } + return TSDB_CODE_SUCCESS; +} + +int32_t convertSmlTimeStamp(TAOS_SML_KV *pVal, char *value, + uint16_t len, SSmlLinesInfo* info) { + int32_t ret; + SMLTimeStampType type = SML_TIME_STAMP_NOW; + int64_t tsVal; + + ret = isTimeStamp(value, len, &type, info); + if (ret != TSDB_CODE_SUCCESS) { + return ret; + } + + ret = getTimeStampValue(value, len, type, &tsVal, info); + if (ret != TSDB_CODE_SUCCESS) { + return ret; + } + uDebug("SML:0x%"PRIx64"Timestamp after conversion:%"PRId64, info->id, tsVal); + + pVal->type = TSDB_DATA_TYPE_TIMESTAMP; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + pVal->value = calloc(pVal->length, 1); + memcpy(pVal->value, &tsVal, pVal->length); + return TSDB_CODE_SUCCESS; +} + +static int32_t parseSmlTimeStamp(TAOS_SML_KV **pTS, const char **index, SSmlLinesInfo* info) { + const char *start, *cur; + int32_t ret = TSDB_CODE_SUCCESS; + int len = 0; + char key[] = "ts"; + char *value = NULL; + + start = cur = *index; + *pTS = calloc(1, sizeof(TAOS_SML_KV)); + + while(*cur != '\0') { + cur++; + len++; + } + + if (len > 0) { + value = calloc(len + 1, 1); + memcpy(value, start, len); + } + + ret = convertSmlTimeStamp(*pTS, value, len, info); + if (ret) { + taosMemoryFree(value); + taosMemoryFree(*pTS); + return ret; + } + taosMemoryFree(value); + + (*pTS)->key = calloc(sizeof(key), 1); + memcpy((*pTS)->key, key, sizeof(key)); + return ret; +} + +bool checkDuplicateKey(char *key, SHashObj *pHash, SSmlLinesInfo* info) { + char *val = NULL; + val = taosHashGet(pHash, key, strlen(key)); + if (val) { + uError("SML:0x%"PRIx64" Duplicate key detected:%s", info->id, key); + return true; + } + + uint8_t dummy_val = 0; + taosHashPut(pHash, key, strlen(key), &dummy_val, sizeof(uint8_t)); + + return false; +} + +static int32_t parseSmlKey(TAOS_SML_KV *pKV, const char **index, SHashObj *pHash, SSmlLinesInfo* info) { + const char *cur = *index; + char key[TSDB_COL_NAME_LEN + 1]; // +1 to avoid key[len] over write + int16_t len = 0; + + while (*cur != '\0') { + if (len > TSDB_COL_NAME_LEN - 1) { + uError("SML:0x%"PRIx64" Key field cannot exceeds %d characters", info->id, TSDB_COL_NAME_LEN - 1); + return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; + } + //unescaped '=' identifies a tag key + if (*cur == '=' && *(cur - 1) != '\\') { + break; + } + //Escape special character + if (*cur == '\\') { + escapeSpecialCharacter(2, &cur); + } + key[len] = *cur; + cur++; + len++; + } + if (len == 0) { + return TSDB_CODE_TSC_LINE_SYNTAX_ERROR; + } + key[len] = '\0'; + + if (checkDuplicateKey(key, pHash, info)) { + return TSDB_CODE_TSC_LINE_SYNTAX_ERROR; + } + + pKV->key = calloc(len + TS_BACKQUOTE_CHAR_SIZE + 1, 1); + memcpy(pKV->key, key, len + 1); + addEscapeCharToString(pKV->key, len); + uDebug("SML:0x%"PRIx64" Key:%s|len:%d", info->id, pKV->key, len); + *index = cur + 1; + return TSDB_CODE_SUCCESS; +} + + +static int32_t parseSmlValue(TAOS_SML_KV *pKV, const char **index, + bool *is_last_kv, SSmlLinesInfo* info, bool isTag) { + const char *start, *cur; + int32_t ret = TSDB_CODE_SUCCESS; + char *value = NULL; + int16_t len = 0; + + bool kv_done = false; + bool back_slash = false; + bool double_quote = false; + size_t line_len = 0; + + enum { + tag_common, + tag_lqoute, + tag_rqoute + } tag_state; + + enum { + val_common, + val_lqoute, + val_rqoute + } val_state; + + start = cur = *index; + tag_state = tag_common; + val_state = val_common; + + while (1) { + if (isTag) { + /* ',', '=' and spaces MUST be escaped */ + switch (tag_state) { + case tag_common: + if (back_slash == true) { + if (*cur != ',' && *cur != '=' && *cur != ' ') { + uError("SML:0x%"PRIx64" tag value: state(%d), incorrect character(%c) escaped", info->id, tag_state, *cur); + ret = TSDB_CODE_TSC_LINE_SYNTAX_ERROR; + goto error; + } + + back_slash = false; + cur++; + len++; + break; + } + + if (*cur == '"') { + if (cur == *index) { + tag_state = tag_lqoute; + } + cur += 1; + len += 1; + break; + } else if (*cur == 'L') { + line_len = strlen(*index); + + /* common character at the end */ + if (cur + 1 >= *index + line_len) { + *is_last_kv = true; + kv_done = true; + break; + } + + if (*(cur + 1) == '"') { + /* string starts here */ + if (cur + 1 == *index + 1) { + tag_state = tag_lqoute; + } + cur += 2; + len += 2; + break; + } + } + + switch (*cur) { + case '\\': + back_slash = true; + cur++; + len++; + break; + case ',': + kv_done = true; + break; + + case ' ': + /* fall through */ + case '\0': + *is_last_kv = true; + kv_done = true; + break; + + default: + cur++; + len++; + } + + break; + case tag_lqoute: + if (back_slash == true) { + if (*cur != ',' && *cur != '=' && *cur != ' ') { + uError("SML:0x%"PRIx64" tag value: state(%d), incorrect character(%c) escaped", info->id, tag_state, *cur); + ret = TSDB_CODE_TSC_LINE_SYNTAX_ERROR; + goto error; + } + + back_slash = false; + cur++; + len++; + break; + } else if (double_quote == true) { + if (*cur != ' ' && *cur != ',' && *cur != '\0') { + uError("SML:0x%"PRIx64" tag value: state(%d), incorrect character(%c) behind closing \"", info->id, tag_state, *cur); + ret = TSDB_CODE_TSC_LINE_SYNTAX_ERROR; + goto error; + } + + if (*cur == ' ' || *cur == '\0') { + *is_last_kv = true; + } + + double_quote = false; + tag_state = tag_rqoute; + break; + } + + switch (*cur) { + case '\\': + back_slash = true; + cur++; + len++; + break; + + case '"': + double_quote = true; + cur++; + len++; + break; + + case ',': + /* fall through */ + case '=': + /* fall through */ + case ' ': + if (*(cur - 1) != '\\') { + uError("SML:0x%"PRIx64" tag value: state(%d), character(%c) not escaped", info->id, tag_state, *cur); + ret = TSDB_CODE_TSC_LINE_SYNTAX_ERROR; + kv_done = true; + } + break; + + case '\0': + uError("SML:0x%"PRIx64" tag value: state(%d), closing \" not found", info->id, tag_state); + ret = TSDB_CODE_TSC_LINE_SYNTAX_ERROR; + kv_done = true; + break; + + default: + cur++; + len++; + } + + break; + + default: + kv_done = true; + } + } else { + switch (val_state) { + case val_common: + if (back_slash == true) { + if (*cur != '\\' && *cur != '"') { + uError("SML:0x%"PRIx64" field value: state(%d), incorrect character(%c) escaped", info->id, val_state, *cur); + ret = TSDB_CODE_TSC_LINE_SYNTAX_ERROR; + goto error; + } + + back_slash = false; + cur++; + len++; + break; + } + + if (*cur == '"') { + if (cur == *index) { + val_state = val_lqoute; + } else { + if (*(cur - 1) != '\\') { + uError("SML:0x%"PRIx64" field value: state(%d), \" not escaped", info->id, val_state); + ret = TSDB_CODE_TSC_LINE_SYNTAX_ERROR; + goto error; + } + } + + cur += 1; + len += 1; + break; + } else if (*cur == 'L') { + line_len = strlen(*index); + + /* common character at the end */ + if (cur + 1 >= *index + line_len) { + *is_last_kv = true; + kv_done = true; + break; + } + + if (*(cur + 1) == '"') { + /* string starts here */ + if (cur + 1 == *index + 1) { + val_state = val_lqoute; + cur += 2; + len += 2; + } else { + /* MUST at the end of string */ + if (cur + 2 >= *index + line_len) { + cur += 2; + len += 2; + *is_last_kv = true; + kv_done = true; + } else { + if (*(cur + 2) != ' ' && *(cur + 2) != ',') { + uError("SML:0x%"PRIx64" field value: state(%d), not closing character(L\")", info->id, val_state); + ret = TSDB_CODE_TSC_LINE_SYNTAX_ERROR; + goto error; + } else { + if (*(cur + 2) == ' ') { + *is_last_kv = true; + } + + cur += 2; + len += 2; + kv_done = true; + } + } + } + break; + } + } + + switch (*cur) { + case '\\': + back_slash = true; + cur++; + len++; + break; + + case ',': + kv_done = true; + break; + + case ' ': + /* fall through */ + case '\0': + *is_last_kv = true; + kv_done = true; + break; + + default: + cur++; + len++; + } + + break; + case val_lqoute: + if (back_slash == true) { + if (*cur != '\\' && *cur != '"') { + uError("SML:0x%"PRIx64" field value: state(%d), incorrect character(%c) escaped", info->id, val_state, *cur); + ret = TSDB_CODE_TSC_LINE_SYNTAX_ERROR; + goto error; + } + + back_slash = false; + cur++; + len++; + break; + } else if (double_quote == true) { + if (*cur != ' ' && *cur != ',' && *cur != '\0') { + uError("SML:0x%"PRIx64" field value: state(%d), incorrect character(%c) behind closing \"", info->id, val_state, *cur); + ret = TSDB_CODE_TSC_LINE_SYNTAX_ERROR; + goto error; + } + + if (*cur == ' ' || *cur == '\0') { + *is_last_kv = true; + } + + double_quote = false; + val_state = val_rqoute; + break; + } + + switch (*cur) { + case '\\': + back_slash = true; + cur++; + len++; + break; + + case '"': + double_quote = true; + cur++; + len++; + break; + + case '\0': + uError("SML:0x%"PRIx64" field value: state(%d), closing \" not found", info->id, val_state); + ret = TSDB_CODE_TSC_LINE_SYNTAX_ERROR; + kv_done = true; + break; + + default: + cur++; + len++; + } + + break; + default: + kv_done = true; + } + } + + if (kv_done == true) { + break; + } + } + + if (len == 0 || ret != TSDB_CODE_SUCCESS) { + taosMemoryFree(pKV->key); + pKV->key = NULL; + return TSDB_CODE_TSC_LINE_SYNTAX_ERROR; + } + + value = calloc(len + 1, 1); + memcpy(value, start, len); + value[len] = '\0'; + if (!convertSmlValueType(pKV, value, len, info, isTag)) { + uError("SML:0x%"PRIx64" Failed to convert sml value string(%s) to any type", + info->id, value); + taosMemoryFree(value); + ret = TSDB_CODE_TSC_INVALID_VALUE; + goto error; + } + taosMemoryFree(value); + + *index = (*cur == '\0') ? cur : cur + 1; + return ret; + +error: + //taosMemoryFree previous alocated key field + taosMemoryFree(pKV->key); + pKV->key = NULL; + return ret; +} + +/* Field Escape charaters + 1: measurement Comma,Space + 2: tag_key, tag_value, field_key Comma,Equal Sign,Space + 3: field_value Double quote,Backslash +*/ + +static int32_t parseSmlMeasurement(TAOS_SML_DATA_POINT *pSml, const char **index, + uint8_t *has_tags, SSmlLinesInfo* info) { + const char *cur = *index; + int16_t len = 0; + + pSml->stableName = calloc(TSDB_TABLE_NAME_LEN + TS_BACKQUOTE_CHAR_SIZE, 1); + if (pSml->stableName == NULL){ + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + while (*cur != '\0') { + if (len > TSDB_TABLE_NAME_LEN - 1) { + uError("SML:0x%"PRIx64" Measurement field cannot exceeds %d characters", info->id, TSDB_TABLE_NAME_LEN - 1); + taosMemoryFree(pSml->stableName); + pSml->stableName = NULL; + return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; + } + //first unescaped comma or space identifies measurement + //if space detected first, meaning no tag in the input + if (*cur == ',' && *(cur - 1) != '\\') { + *has_tags = 1; + break; + } + if (*cur == ' ' && *(cur - 1) != '\\') { + if (*(cur + 1) != ' ') { + break; + } + else { + cur++; + continue; + } + } + //Comma, Space, Backslash needs to be escaped if any + if (*cur == '\\') { + escapeSpecialCharacter(1, &cur); + } + pSml->stableName[len] = *cur; + cur++; + len++; + } + if (len == 0) { + taosMemoryFree(pSml->stableName); + pSml->stableName = NULL; + return TSDB_CODE_TSC_LINE_SYNTAX_ERROR; + } + addEscapeCharToString(pSml->stableName, len); + *index = cur + 1; + uDebug("SML:0x%"PRIx64" Stable name in measurement:%s|len:%d", info->id, pSml->stableName, len); + + return TSDB_CODE_SUCCESS; +} + +//Table name can only contain digits(0-9),alphebet(a-z),underscore(_) +int32_t isValidChildTableName(const char *pTbName, int16_t len, SSmlLinesInfo* info) { + if (len > TSDB_TABLE_NAME_LEN - 1) { + uError("SML:0x%"PRIx64" child table name cannot exceeds %d characters", info->id, TSDB_TABLE_NAME_LEN - 1); + return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; + } + const char *cur = pTbName; + for (int i = 0; i < len; ++i) { + if(!isdigit(cur[i]) && !isalpha(cur[i]) && (cur[i] != '_')) { + return TSDB_CODE_TSC_LINE_SYNTAX_ERROR; + } + } + return TSDB_CODE_SUCCESS; +} + + +static int32_t parseSmlKvPairs(TAOS_SML_KV **pKVs, int *num_kvs, + const char **index, bool isField, + TAOS_SML_DATA_POINT* smlData, SHashObj *pHash, + SSmlLinesInfo* info) { + const char *cur = *index; + int32_t ret = TSDB_CODE_SUCCESS; + TAOS_SML_KV *pkv; + bool is_last_kv = false; + + int32_t capacity = 0; + if (isField) { + capacity = 64; + *pKVs = calloc(capacity, sizeof(TAOS_SML_KV)); + // leave space for timestamp; + pkv = *pKVs; + pkv++; + } else { + capacity = 8; + *pKVs = calloc(capacity, sizeof(TAOS_SML_KV)); + pkv = *pKVs; + } + + size_t childTableNameLen = strlen(tsSmlChildTableName); + char childTableName[TSDB_TABLE_NAME_LEN + TS_BACKQUOTE_CHAR_SIZE] = {0}; + if (childTableNameLen != 0) { + memcpy(childTableName, tsSmlChildTableName, childTableNameLen); + addEscapeCharToString(childTableName, (int32_t)(childTableNameLen)); + } + + while (*cur != '\0') { + ret = parseSmlKey(pkv, &cur, pHash, info); + if (ret) { + uError("SML:0x%"PRIx64" Unable to parse key", info->id); + goto error; + } + ret = parseSmlValue(pkv, &cur, &is_last_kv, info, !isField); + if (ret) { + uError("SML:0x%"PRIx64" Unable to parse value", info->id); + goto error; + } + + if (!isField && childTableNameLen != 0 && strcasecmp(pkv->key, childTableName) == 0) { + smlData->childTableName = taosMemoryMalloc(pkv->length + TS_BACKQUOTE_CHAR_SIZE + 1); + memcpy(smlData->childTableName, pkv->value, pkv->length); + addEscapeCharToString(smlData->childTableName, (int32_t)pkv->length); + taosMemoryFree(pkv->key); + taosMemoryFree(pkv->value); + } else { + *num_kvs += 1; + } + if (is_last_kv) { + goto done; + } + + //reallocate addtional memory for more kvs + TAOS_SML_KV *more_kvs = NULL; + + if (isField) { + if ((*num_kvs + 2) > capacity) { + capacity *= 3; capacity /= 2; + more_kvs = realloc(*pKVs, capacity * sizeof(TAOS_SML_KV)); + } else { + more_kvs = *pKVs; + } + } else { + if ((*num_kvs + 1) > capacity) { + capacity *= 3; capacity /= 2; + more_kvs = realloc(*pKVs, capacity * sizeof(TAOS_SML_KV)); + } else { + more_kvs = *pKVs; + } + } + + if (!more_kvs) { + goto error; + } + *pKVs = more_kvs; + //move pKV points to next TAOS_SML_KV block + if (isField) { + pkv = *pKVs + *num_kvs + 1; + } else { + pkv = *pKVs + *num_kvs; + } + } + goto done; + +error: + return ret; +done: + *index = cur; + return ret; +} + +static void moveTimeStampToFirstKv(TAOS_SML_DATA_POINT** smlData, TAOS_SML_KV *ts) { + TAOS_SML_KV* tsField = (*smlData)->fields; + tsField->length = ts->length; + tsField->type = ts->type; + tsField->value = taosMemoryMalloc(ts->length); + tsField->key = taosMemoryMalloc(strlen(ts->key) + 1); + memcpy(tsField->key, ts->key, strlen(ts->key) + 1); + memcpy(tsField->value, ts->value, ts->length); + (*smlData)->fieldNum = (*smlData)->fieldNum + 1; + + taosMemoryFree(ts->key); + taosMemoryFree(ts->value); + taosMemoryFree(ts); +} + +/* Field Escape charaters + 1: measurement Comma,Space + 2: tag_key, tag_value, field_key Comma,Equal Sign,Space + 3: field_value Double quote,Backslash +*/ + +//void findSpace(const char** sql, const char **tags, int32_t *tagLen){ +// const char *cur = *sql; +// *tagLen = 0; +// *tags = NULL; +// if(!cur) return; +// while (*cur != '\0') { // jump the space at the begining +// if(*cur != SPACE) { +// *tags = cur; +// break; +// } +// cur++; +// } +// +// while (*cur != '\0') { // find the first space +// if (*cur == SPACE && *(cur - 1) != SLASH) { +// *tagLen = cur - *tags; +// break; +// } +// +// cur++; +// } +// *sql = cur; +// return; +//} + + +int32_t parseSml(const char* sql, TAOS_PARSE_ELEMENTS *elements){ + if(!sql) return TSDB_CODE_SML_INVALID_DATA; + while (*sql != '\0') { // jump the space at the begining + if(*sql != SPACE) { + elements->measure = sql; + break; + } + sql++; + } + if (!elements->measure || *sql == COMMA) return TSDB_CODE_SML_INVALID_DATA; + + // parse measure and tag + while (*sql != '\0') { + if (elements->measureLen == 0 && *sql == COMMA && *(sql - 1) != SLASH) { // find the first comma + elements->measureLen = sql - elements->measure; + sql++; + elements->tags = sql; + continue; + } + + if (*sql == SPACE && *(sql - 1) != SLASH) { // find the first space + if (elements->measureLen == 0) { + elements->measureLen = sql - elements->measure; + elements->tags = sql; + } + elements->tagsLen = sql - elements->tags; + elements->measureTagsLen = sql - elements->measure; + break; + } + + sql++; + } + if(elements->measureLen == 0) return TSDB_CODE_SML_INVALID_DATA; + + // parse cols + while (*sql != '\0') { + if(*sql != SPACE) { + elements->cols = sql; + break; + } + sql++; + } + if(!elements->cols) return TSDB_CODE_SML_INVALID_DATA; + + while (*sql != '\0') { + if(*sql == SPACE && *(sql - 1) != SLASH) { + elements->colsLen = sql - elements->cols; + break; + } + sql++; + } + if(elements->colsLen == 0) return TSDB_CODE_SML_INVALID_DATA; + + // parse ts + while (*sql != '\0') { + if(*sql != SPACE) { + elements->timestamp = sql; + break; + } + sql++; + } + if(!elements->timestamp) return TSDB_CODE_SML_INVALID_DATA; + + return TSDB_CODE_SUCCESS; +} + +int32_t parseSmlKV(const char* data, int32_t len, SArray *tags){ + for(int i = 0; i < len; i++){ + const char *key = data + i; + int32_t keyLen = 0; + while(i < len){ + if(data[i] == EQUAL && i > 0 && data[i-1] != SLASH){ + keyLen = data + i - key; + break; + } + i++; + } + if(keyLen == 0){ + return TSDB_CODE_SML_INVALID_DATA; + } + + i++; + const char *value = data + i; + int32_t valueLen = 0; + while(i < len){ + if(data[i] == COMMA && i > 0 && data[i-1] != SLASH){ + valueLen = data + i - value; + break; + } + i++; + } + if(valueLen == 0){ + return TSDB_CODE_SML_INVALID_DATA; + } + TAOS_SML_KV *kv = taosMemoryCalloc(sizeof(TAOS_SML_KV), 1); + kv->key = key; + kv->keyLen = keyLen; + kv->value = value; + kv->valueLen = valueLen; + kv->type = TSDB_DATA_TYPE_NCHAR; + if(tags) taosArrayPush(tags, &kv); + } + return TSDB_CODE_SUCCESS; +} + +int32_t parseSmlTS(const char* data, int32_t len, SArray *tags){ + TAOS_SML_KV *kv = taosMemoryCalloc(sizeof(TAOS_SML_KV), 1); + kv->value = data; + kv->valueLen = len; + kv->type = TSDB_DATA_TYPE_TIMESTAMP; + if(tags) taosArrayPush(tags, &kv); + return TSDB_CODE_SUCCESS; +} + +//int32_t parseSmlCols(const char* data, SArray *cols){ +// while(*data != '\0'){ +// if(*data == EQUAL) return TSDB_CODE_SML_INVALID_DATA; +// const char *key = data; +// int32_t keyLen = 0; +// while(*data != '\0'){ +// if(*data == EQUAL && *(data-1) != SLASH){ +// keyLen = data - key; +// data ++; +// break; +// } +// data++; +// } +// if(keyLen == 0){ +// return TSDB_CODE_SML_INVALID_DATA; +// } +// +// if(*data == COMMA) return TSDB_CODE_SML_INVALID_DATA; +// const char *value = data; +// int32_t valueLen = 0; +// while(*data != '\0'){ +// if(*data == COMMA && *(data-1) != SLASH){ +// valueLen = data - value; +// data ++; +// break; +// } +// data++; +// } +// if(valueLen == 0){ +// return TSDB_CODE_SML_INVALID_DATA; +// } +// +// TAOS_SML_KV *kv = taosMemoryCalloc(sizeof(TAOS_SML_KV), 1); +// kv->key = key; +// kv->keyLen = keyLen; +// kv->value = value; +// kv->valueLen = valueLen; +// kv->type = TSDB_DATA_TYPE_NCHAR; +// if(cols) taosArrayPush(cols, &kv); +// } +// return TSDB_CODE_SUCCESS; +//} + +void updateMeta(SSmlSTableMeta* tableMeta, SArray *tags, SArray *cols){ + if(tags){ + for (int i = 0; i < taosArrayGetSize(tags); ++i) { + TAOS_SML_KV *kv = taosArrayGetP(tags, i); + TAOS_SML_KV **value = taosHashGet(tableMeta->tagHash, kv->key, kv->keyLen); + if(value){ + if(kv->type != (*value)->type){ + // todo + } + }else{ + taosHashPut(tableMeta->tagHash, kv->key, kv->keyLen, &kv, POINTER_BYTES); + } + + } + } + + if(cols){ + for (int i = 1; i < taosArrayGetSize(cols); ++i) { //jump timestamp + TAOS_SML_KV *kv = taosArrayGetP(cols, i); + TAOS_SML_KV **value = taosHashGet(tableMeta->fieldHash, kv->key, kv->keyLen); + if(value){ + if(kv->type != (*value)->type){ + // todo + } + }else{ + taosHashPut(tableMeta->fieldHash, kv->key, kv->keyLen, &kv, POINTER_BYTES); + } + } + } +} + +void insertMeta(SSmlSTableMeta* tableMeta, SArray *tags, SArray *cols){ + if(tags){ + for (int i = 0; i < taosArrayGetSize(tags); ++i) { + TAOS_SML_KV *kv = taosArrayGetP(tags, i); + taosHashPut(tableMeta->tagHash, kv->key, kv->keyLen, &kv, POINTER_BYTES); + } + } + + if(cols){ + for (int i = 0; i < taosArrayGetSize(cols); ++i) { + TAOS_SML_KV *kv = taosArrayGetP(cols, i); + taosHashPut(tableMeta->fieldHash, kv->key, kv->keyLen, &kv, POINTER_BYTES); + } + } +} + +int32_t tscParseLine(const char* sql, SSmlLinesInfo* info) { + TAOS_PARSE_ELEMENTS elements = {0}; + int ret = parseSml(sql, &elements); + if(ret != TSDB_CODE_SUCCESS){ + return ret; + } + + SArray *cols = taosArrayInit(16, POINTER_BYTES); + if (cols == NULL) { + uError("SML:0x%"PRIx64" taos_insert_lines failed to allocate memory", info->id); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + parseSmlTS(elements.timestamp, strlen(elements.timestamp), cols); + ret = parseSmlKV(elements.cols, elements.colsLen, cols); + if(ret != TSDB_CODE_SUCCESS){ + return ret; + } + + TAOS_SML_DATA_POINT_TAGS** oneTable = taosHashGet(info->childTables, elements.measure, elements.measureTagsLen); + if(oneTable){ + SSmlSTableMeta** tableMeta = taosHashGet(info->superTables, elements.measure, elements.measureLen); + ASSERT(tableMeta); + updateMeta(*tableMeta, NULL, cols); // update meta + + taosArrayPush((*oneTable)->cols, &cols); + }else{ + TAOS_SML_DATA_POINT_TAGS *tag = taosMemoryCalloc(sizeof(TAOS_SML_DATA_POINT_TAGS), 1); + tag->cols = taosArrayInit(16, POINTER_BYTES); + if (tag->cols == NULL) { + uError("SML:0x%"PRIx64" taos_insert_lines failed to allocate memory", info->id); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + taosArrayPush(tag->cols, &cols); + + tag->tags = taosArrayInit(16, POINTER_BYTES); + if (tag->tags == NULL) { + uError("SML:0x%"PRIx64" taos_insert_lines failed to allocate memory", info->id); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + ret = parseSmlKV(elements.tags, elements.tagsLen, tag->tags); + if(ret != TSDB_CODE_SUCCESS){ + return ret; + } + + SSmlSTableMeta** tableMeta = taosHashGet(info->superTables, elements.measure, elements.measureLen); + if(tableMeta){ // update meta + updateMeta(*tableMeta, tag->tags, cols); + }else{ + SSmlSTableMeta* meta = taosMemoryCalloc(sizeof(SSmlSTableMeta), 1); + insertMeta(meta, tag->tags, cols); + taosHashPut(info->superTables, elements.measure, elements.measureLen, &meta, POINTER_BYTES); + } + + taosHashPut(info->childTables, elements.measure, elements.measureTagsLen, &tag, POINTER_BYTES); + } + return TSDB_CODE_SUCCESS; +} + + +int32_t tscParseLines(char* lines[], int numLines, SSmlLinesInfo* info) { + for (int32_t i = 0; i < numLines; ++i) { + int32_t code = tscParseLine(lines[i], info); + if (code != TSDB_CODE_SUCCESS) { + uError("SML:0x%"PRIx64" data point line parse failed. line %d : %s", info->id, i, lines[i]); + return code; + } + } + uDebug("SML:0x%"PRIx64" data point line parse success. tables %d", info->id, taosHashGetSize(info->childTables)); + + return TSDB_CODE_SUCCESS; +} + +int taos_insert_lines(TAOS* taos, char* lines[], int numLines, SMLProtocolType protocol, SMLTimeStampType tsType, int *affectedRows) { + int32_t code = 0; + + SSmlLinesInfo* info = taosMemoryMalloc(sizeof(SSmlLinesInfo)); + info->id = genLinesSmlId(); + info->tsType = tsType; + info->taos = (STscObj*)taos; + info->protocol = protocol; + + if (numLines <= 0 || numLines > 65536) { + uError("SML:0x%"PRIx64" taos_insert_lines numLines should be between 1 and 65536. numLines: %d", info->id, numLines); + code = TSDB_CODE_TSC_APP_ERROR; + goto cleanup; + } + + info->childTables = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, false); + info->superTables = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, false); + + uDebug("SML:0x%"PRIx64" taos_insert_lines begin inserting %d lines, first line: %s", info->id, numLines, lines[0]); + code = tscParseLines(lines, numLines, info); + + if (code != 0) { + goto cleanup; + } + + code = tscSmlInsert(taos, info); + if (code != 0) { + uError("SML:0x%"PRIx64" taos_sml_insert error: %s", info->id, tstrerror((code))); + goto cleanup; + } + if (affectedRows != NULL) { + *affectedRows = info->affectedRows; + } + + uDebug("SML:0x%"PRIx64" taos_insert_lines finish inserting %d lines. code: %d", info->id, numLines, code); + +cleanup: + taosMemoryFree(info); + return code; +} + +static int32_t convertPrecisionType(int precision, SMLTimeStampType *tsType) { + switch (precision) { + case TSDB_SML_TIMESTAMP_NOT_CONFIGURED: + *tsType = SML_TIME_STAMP_NOT_CONFIGURED; + break; + case TSDB_SML_TIMESTAMP_HOURS: + *tsType = SML_TIME_STAMP_HOURS; + break; + case TSDB_SML_TIMESTAMP_MILLI_SECONDS: + *tsType = SML_TIME_STAMP_MILLI_SECONDS; + break; + case TSDB_SML_TIMESTAMP_NANO_SECONDS: + *tsType = SML_TIME_STAMP_NANO_SECONDS; + break; + case TSDB_SML_TIMESTAMP_MICRO_SECONDS: + *tsType = SML_TIME_STAMP_MICRO_SECONDS; + break; + case TSDB_SML_TIMESTAMP_SECONDS: + *tsType = SML_TIME_STAMP_SECONDS; + break; + case TSDB_SML_TIMESTAMP_MINUTES: + *tsType = SML_TIME_STAMP_MINUTES; + break; + default: + return TSDB_CODE_SML_INVALID_PRECISION_TYPE; + } + + return TSDB_CODE_SUCCESS; +} + +//make a dummy SSqlObj +static SSqlObj* createSmlQueryObj(TAOS* taos, int32_t affected_rows, int32_t code) { + SSqlObj *pNew = (SSqlObj*)calloc(1, sizeof(SSqlObj)); + if (pNew == NULL) { + return NULL; + } + pNew->signature = pNew; + pNew->pTscObj = taos; + pNew->fp = NULL; + + tsem_init(&pNew->rspSem, 0, 0); + registerSqlObj(pNew); + + pNew->res.numOfRows = affected_rows; + pNew->res.code = code; + + + return pNew; +} + + +/** + * taos_schemaless_insert() parse and insert data points into database according to + * different protocol. + * + * @param $lines input array may contain multiple lines, each line indicates a data point. + * If protocol=2 is used input array should contain single JSON + * string(e.g. char *lines[] = {"$JSON_string"}). If need to insert + * multiple data points in JSON format, should include them in $JSON_string + * as a JSON array. + * @param $numLines indicates how many data points in $lines. + * If protocol = 2 is used this param will be ignored as $lines should + * contain single JSON string. + * @param $protocol indicates which protocol to use for parsing: + * 0 - influxDB line protocol + * 1 - OpenTSDB telnet line protocol + * 2 - OpenTSDB JSON format protocol + * @return return zero for successful insertion. Otherwise return none-zero error code of + * failure reason. + * + */ + +TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int protocol, int precision) { + int code = TSDB_CODE_SUCCESS; + int affected_rows = 0; + SMLTimeStampType tsType = SML_TIME_STAMP_NOW; + + if (protocol == TSDB_SML_LINE_PROTOCOL) { + code = convertPrecisionType(precision, &tsType); + if (code != TSDB_CODE_SUCCESS) { + return NULL; + } + } + + switch (protocol) { + case TSDB_SML_LINE_PROTOCOL: + code = taos_insert_lines(taos, lines, numLines, protocol, tsType, &affected_rows); + break; + case TSDB_SML_TELNET_PROTOCOL: + //code = taos_insert_telnet_lines(taos, lines, numLines, protocol, tsType, &affected_rows); + break; + case TSDB_SML_JSON_PROTOCOL: + //code = taos_insert_json_payload(taos, *lines, protocol, tsType, &affected_rows); + break; + default: + code = TSDB_CODE_SML_INVALID_PROTOCOL_TYPE; + break; + } + + + SSqlObj *pSql = createSmlQueryObj(taos, affected_rows, code); + + return (TAOS_RES*)pSql; +} diff --git a/source/libs/CMakeLists.txt b/source/libs/CMakeLists.txt index b1e8be6528..ade2487239 100644 --- a/source/libs/CMakeLists.txt +++ b/source/libs/CMakeLists.txt @@ -17,4 +17,4 @@ add_subdirectory(tfs) add_subdirectory(monitor) add_subdirectory(nodes) add_subdirectory(scalar) -add_subdirectory(command) +add_subdirectory(command) \ No newline at end of file diff --git a/source/libs/scalar/test/scalar/scalarTests.cpp b/source/libs/scalar/test/scalar/scalarTests.cpp index de76e7e274..206b2ac542 100644 --- a/source/libs/scalar/test/scalar/scalarTests.cpp +++ b/source/libs/scalar/test/scalar/scalarTests.cpp @@ -127,7 +127,7 @@ void scltMakeValueNode(SNode **pNode, int32_t dataType, void *value) { *pNode = (SNode *)vnode; } -void scltMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, int32_t dataBytes, int32_t rowNum, void *value) { +void scltMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, int32_t dataBytes, int32_t ronwNum, void *value) { SNode *node = (SNode*)nodesMakeNode(QUERY_NODE_COLUMN); SColumnNode *rnode = (SColumnNode *)node; rnode->node.resType.type = dataType; @@ -156,9 +156,9 @@ void scltMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in idata.info.colId = 3; int32_t size = idata.info.bytes * rowNum; idata.pData = (char *)taosMemoryCalloc(1, size); + colInfoDataEnsureCapacity(&idata, 0, rowNum); + taosArrayPush(res->pDataBlock, &idata); - - blockDataEnsureCapacity(res, rowNum); SColumnInfoData *pColumn = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); for (int32_t i = 0; i < rowNum; ++i) { diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 9332cb481e..6c77d943f3 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -438,6 +438,11 @@ TAOS_DEFINE_ERROR(TSDB_CODE_QW_MSG_ERROR, "Invalid msg order") //planner TAOS_DEFINE_ERROR(TSDB_CODE_PLAN_INTERNAL_ERROR, "planner internal error") +//schemaless +TAOS_DEFINE_ERROR(TSDB_CODE_SML_INVALID_PROTOCOL_TYPE, "Invalid line protocol type") +TAOS_DEFINE_ERROR(TSDB_CODE_SML_INVALID_PRECISION_TYPE, "Invalid timestamp precision type") +TAOS_DEFINE_ERROR(TSDB_CODE_SML_INVALID_DATA, "Invalid data type") + #ifdef TAOS_ERROR_C }; #endif From 990205d6847b919c6c238b3c6fdaa309446a7c65 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 27 Apr 2022 21:29:25 +0800 Subject: [PATCH 02/31] refactor: schemaless tmp commit --- include/common/tcommon.h | 9 + include/libs/parser/parser.h | 6 +- include/util/tdef.h | 4 + .../inc/{tscParseLine.h => clientSml.h} | 78 +- source/client/schemaless/CMakeLists.txt | 17 - .../client/schemaless/src/tscParseOpenTSDB.c | 1113 ----------- .../{tscParseLineProtocol.c => clientSml.c} | 1767 +++-------------- source/client/src/clientStmt.c | 2 +- source/libs/parser/inc/parInsertData.h | 2 +- source/libs/parser/inc/parUtil.h | 1 - source/libs/parser/src/parInsert.c | 228 ++- source/libs/parser/src/parInsertData.c | 14 +- source/libs/parser/src/parUtil.c | 20 - tools/shell/src/shellCommand.c | 4 +- tools/shell/src/shellEngine.c | 54 +- 15 files changed, 548 insertions(+), 2771 deletions(-) rename source/client/inc/{tscParseLine.h => clientSml.h} (59%) delete mode 100644 source/client/schemaless/CMakeLists.txt delete mode 100644 source/client/schemaless/src/tscParseOpenTSDB.c rename source/client/src/{tscParseLineProtocol.c => clientSml.c} (51%) diff --git a/include/common/tcommon.h b/include/common/tcommon.h index 7c308f9354..1c516e8a96 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -241,6 +241,15 @@ typedef struct SExprInfo { struct tExprNode* pExpr; } SExprInfo; +typedef struct { + const char* key; + int32_t keyLen; + uint8_t type; + int16_t length; + const char* value; + int32_t valueLen; +} SSmlKv; + #define QUERY_ASC_FORWARD_STEP 1 #define QUERY_DESC_FORWARD_STEP -1 diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index 58482735ba..248a6c1237 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -92,10 +92,14 @@ int32_t qBindStmtColsValue(void *pBlock, TAOS_BIND_v2 *bind, char *msgBuf, int32 int32_t qBindStmtSingleColValue(void *pBlock, TAOS_BIND_v2 *bind, char *msgBuf, int32_t msgBufLen, int32_t colIdx, int32_t rowNum); int32_t qBuildStmtColFields(void *pDataBlock, int32_t *fieldNum, TAOS_FIELD** fields); int32_t qBuildStmtTagFields(void *pBlock, void *boundTags, int32_t *fieldNum, TAOS_FIELD** fields); -int32_t qBindStmtTagsValue(void *pBlock, void *boundTags, int64_t suid, SName *pName, TAOS_BIND_v2 *bind, char *msgBuf, int32_t msgBufLen); +int32_t qBindStmtTagsValue(void *pBlock, void *boundTags, int64_t suid, char *tName, TAOS_BIND_v2 *bind, char *msgBuf, int32_t msgBufLen); void destroyBoundColumnInfo(void* pBoundInfo); int32_t qCreateSName(SName* pName, const char* pTableName, int32_t acctId, char* dbName, char *msgBuf, int32_t msgBufLen); +void* tscSmlInitHandle(SQuery *pQuery); +void tscSmlDestroyHandle(void *pHandle); +int32_t smlBind(void *handle, SArray *tags, SArray *cols, STableMeta *pTableMeta, char *msgBuf, int16_t msgBufLen); +int32_t smlBuildOutput(void* handle, SHashObj* pVgHash); #ifdef __cplusplus } diff --git a/include/util/tdef.h b/include/util/tdef.h index 2548df7186..71d5f090a3 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -86,9 +86,13 @@ extern const int32_t TYPE_BYTES[15]; #define TS_PATH_DELIMITER "." #define TS_ESCAPE_CHAR '`' + #define TSDB_TIME_PRECISION_MILLI 0 #define TSDB_TIME_PRECISION_MICRO 1 #define TSDB_TIME_PRECISION_NANO 2 +#define TSDB_TIME_PRECISION_HOURS 3 +#define TSDB_TIME_PRECISION_MINUTES 4 +#define TSDB_TIME_PRECISION_SECONDS 5 #define TSDB_TIME_PRECISION_MILLI_STR "ms" #define TSDB_TIME_PRECISION_MICRO_STR "us" diff --git a/source/client/inc/tscParseLine.h b/source/client/inc/clientSml.h similarity index 59% rename from source/client/inc/tscParseLine.h rename to source/client/inc/clientSml.h index b70280f380..b711c837c0 100644 --- a/source/client/inc/tscParseLine.h +++ b/source/client/inc/clientSml.h @@ -13,30 +13,19 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_TSCPARSELINE_H -#define TDENGINE_TSCPARSELINE_H +#ifndef TDENGINE_CLIENTSML_H +#define TDENGINE_CLIENTSML_H #ifdef __cplusplus extern "C" { #endif #include "thash.h" -#include "clientint.h" - -#define SML_TIMESTAMP_SECOND_DIGITS 10 -#define SML_TIMESTAMP_MILLI_SECOND_DIGITS 13 +#include "clientInt.h" +#include "catalog.h" typedef TSDB_SML_PROTOCOL_TYPE SMLProtocolType; -typedef struct { - const char* key; - int32_t keyLen; - uint8_t type; - int16_t length; - const char* value; - int32_t valueLen; -} TAOS_SML_KV; - typedef struct { const char* measure; const char* tags; @@ -50,65 +39,66 @@ typedef struct { } TAOS_PARSE_ELEMENTS; typedef struct { - char* childTableName; + const char *sTableName; // super table name + uint8_t sTableNameLen; + char childTableName[TSDB_TABLE_NAME_LEN]; + uint64_t uid; SArray* tags; SArray *cols; } TAOS_SML_DATA_POINT_TAGS; typedef struct SSmlSTableMeta { - char *sTableName; // super table name - uint8_t sTableNameLen; +// char *sTableName; // super table name +// uint8_t sTableNameLen; uint8_t precision; // the number of precision SHashObj* tagHash; SHashObj* fieldHash; } SSmlSTableMeta; -typedef enum { - SML_TIME_STAMP_NOT_CONFIGURED, - SML_TIME_STAMP_HOURS, - SML_TIME_STAMP_MINUTES, - SML_TIME_STAMP_SECONDS, - SML_TIME_STAMP_MILLI_SECONDS, - SML_TIME_STAMP_MICRO_SECONDS, - SML_TIME_STAMP_NANO_SECONDS, - SML_TIME_STAMP_NOW -} SMLTimeStampType; - typedef struct { uint64_t id; - STscObj* taos; - SCatalog* pCatalog; - SMLProtocolType protocol; - SMLTimeStampType tsType; - - int32_t affectedRows; + int32_t tsType; SHashObj* childTables; SHashObj* superTables; + + SHashObj* metaHashObj; + SHashObj* pVgHash; + + void* exec; + + STscObj* taos; + SCatalog* pCatalog; + SRequestObj* pRequest; + SQuery* pQuery; + + int32_t affectedRows; + char *msgBuf; + int16_t msgLen; } SSmlLinesInfo; -int tscSmlInsert(TAOS* taos, TAOS_SML_DATA_POINT* points, int numPoint, SSmlLinesInfo* info); +int smlInsert(TAOS* taos, SSmlLinesInfo* info); + bool checkDuplicateKey(char *key, SHashObj *pHash, SSmlLinesInfo* info); bool isValidInteger(char *str); bool isValidFloat(char *str); int32_t isValidChildTableName(const char *pTbName, int16_t len, SSmlLinesInfo* info); -bool convertSmlValueType(TAOS_SML_KV *pVal, char *value, +bool convertSmlValueType(SSmlKv *pVal, char *value, uint16_t len, SSmlLinesInfo* info, bool isTag); -int32_t convertSmlTimeStamp(TAOS_SML_KV *pVal, char *value, +int32_t convertSmlTimeStamp(SSmlKv *pVal, char *value, uint16_t len, SSmlLinesInfo* info); -void destroySmlDataPoint(TAOS_SML_DATA_POINT* point); -int taos_insert_lines(TAOS* taos, char* lines[], int numLines, SMLProtocolType protocol, - SMLTimeStampType tsType, int* affectedRows); -int taos_insert_telnet_lines(TAOS* taos, char* lines[], int numLines, SMLProtocolType protocol, +int sml_insert_lines(TAOS* taos, SRequestObj* request, char* lines[], int numLines, SMLProtocolType protocol, + SMLTimeStampType tsType); +int sml_insert_telnet_lines(TAOS* taos, char* lines[], int numLines, SMLProtocolType protocol, SMLTimeStampType tsType, int* affectedRows); -int taos_insert_json_payload(TAOS* taos, char* payload, SMLProtocolType protocol, +int sml_insert_json_payload(TAOS* taos, char* payload, SMLProtocolType protocol, SMLTimeStampType tsType, int* affectedRows); @@ -116,4 +106,4 @@ int taos_insert_json_payload(TAOS* taos, char* payload, SMLProtocolType protocol } #endif -#endif // TDENGINE_TSCPARSELINE_H +#endif // TDENGINE_CLIENTSML_H diff --git a/source/client/schemaless/CMakeLists.txt b/source/client/schemaless/CMakeLists.txt deleted file mode 100644 index 29be674934..0000000000 --- a/source/client/schemaless/CMakeLists.txt +++ /dev/null @@ -1,17 +0,0 @@ -aux_source_directory(src SCHEMALESS_SRC) -add_library(schemaless STATIC ${SCHEMALESS_SRC}) - -target_include_directories( - schemaless - PUBLIC "${TD_SOURCE_DIR}/include/libs/schemaless" - PRIVATE "inc" -) - -target_link_libraries( - schemaless - PUBLIC os util common catalog qcom -) - -if(${BUILD_TEST}) - ADD_SUBDIRECTORY(test) -endif(${BUILD_TEST}) diff --git a/source/client/schemaless/src/tscParseOpenTSDB.c b/source/client/schemaless/src/tscParseOpenTSDB.c deleted file mode 100644 index 5bb9b09cf1..0000000000 --- a/source/client/schemaless/src/tscParseOpenTSDB.c +++ /dev/null @@ -1,1113 +0,0 @@ -//#include -//#include -//#include -//#include -// -//#include "cJSON.h" -//#include "hash.h" -//#include "taos.h" -// -//#include "tscUtil.h" -//#include "tsclient.h" -//#include "tscLog.h" -// -//#include "tscParseLine.h" -// -//#define OTD_MAX_FIELDS_NUM 2 -//#define OTD_JSON_SUB_FIELDS_NUM 2 -//#define OTD_JSON_FIELDS_NUM 4 -// -//#define OTD_TIMESTAMP_COLUMN_NAME "ts" -//#define OTD_METRIC_VALUE_COLUMN_NAME "value" -// -///* telnet style API parser */ -//static uint64_t HandleId = 0; -// -//static uint64_t genUID() { -// uint64_t id; -// -// do { -// id = atomic_add_fetch_64(&HandleId, 1); -// } while (id == 0); -// -// return id; -//} -// -//static int32_t parseTelnetMetric(TAOS_SML_DATA_POINT *pSml, const char **index, SSmlLinesInfo* info) { -// const char *cur = *index; -// uint16_t len = 0; -// -// pSml->stableName = tcalloc(TSDB_TABLE_NAME_LEN + TS_BACKQUOTE_CHAR_SIZE, 1); -// if (pSml->stableName == NULL) { -// return TSDB_CODE_TSC_OUT_OF_MEMORY; -// } -// /* -// if (isdigit(*cur)) { -// tscError("OTD:0x%"PRIx64" Metric cannot start with digit", info->id); -// tfree(pSml->stableName); -// return TSDB_CODE_TSC_LINE_SYNTAX_ERROR; -// } -// */ -// -// while (*cur != '\0') { -// if (len > TSDB_TABLE_NAME_LEN - 1) { -// tscError("OTD:0x%"PRIx64" Metric cannot exceeds %d characters", info->id, TSDB_TABLE_NAME_LEN - 1); -// tfree(pSml->stableName); -// return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; -// } -// -// if (*cur == ' ') { -// if (*(cur + 1) != ' ') { -// break; -// } else { -// cur++; -// continue; -// } -// } -// -// pSml->stableName[len] = *cur; -// -// cur++; -// len++; -// } -// if (len == 0 || *cur == '\0') { -// tfree(pSml->stableName); -// return TSDB_CODE_TSC_LINE_SYNTAX_ERROR; -// } -// -// addEscapeCharToString(pSml->stableName, len); -// *index = cur + 1; -// tscDebug("OTD:0x%"PRIx64" Stable name in metric:%s|len:%d", info->id, pSml->stableName, len); -// -// return TSDB_CODE_SUCCESS; -//} -// -//static int32_t parseTelnetTimeStamp(TAOS_SML_KV **pTS, int *num_kvs, const char **index, SSmlLinesInfo* info) { -// //Timestamp must be the first KV to parse -// assert(*num_kvs == 0); -// -// const char *start, *cur; -// int32_t ret = TSDB_CODE_SUCCESS; -// int len = 0; -// char key[] = OTD_TIMESTAMP_COLUMN_NAME; -// char *value = NULL; -// -// start = cur = *index; -// //allocate fields for timestamp and value -// *pTS = tcalloc(OTD_MAX_FIELDS_NUM, sizeof(TAOS_SML_KV)); -// -// while(*cur != '\0') { -// if (*cur == ' ') { -// if (*(cur + 1) != ' ') { -// break; -// } else { -// cur++; -// continue; -// } -// } -// cur++; -// len++; -// } -// -// if (len > 0 && *cur != '\0') { -// value = tcalloc(len + 1, 1); -// memcpy(value, start, len); -// } else { -// tfree(*pTS); -// return TSDB_CODE_TSC_LINE_SYNTAX_ERROR; -// } -// -// ret = convertSmlTimeStamp(*pTS, value, len, info); -// if (ret) { -// tfree(value); -// tfree(*pTS); -// return ret; -// } -// tfree(value); -// -// (*pTS)->key = tcalloc(sizeof(key) + TS_BACKQUOTE_CHAR_SIZE, 1); -// memcpy((*pTS)->key, key, sizeof(key)); -// addEscapeCharToString((*pTS)->key, (int32_t)strlen(key)); -// -// *num_kvs += 1; -// *index = cur + 1; -// -// return ret; -//} -// -//static int32_t parseTelnetMetricValue(TAOS_SML_KV **pKVs, int *num_kvs, const char **index, SSmlLinesInfo* info) { -// //skip timestamp -// TAOS_SML_KV *pVal = *pKVs + 1; -// const char *start, *cur; -// int32_t ret = TSDB_CODE_SUCCESS; -// int len = 0; -// bool searchQuote = false; -// char key[] = OTD_METRIC_VALUE_COLUMN_NAME; -// char *value = NULL; -// -// start = cur = *index; -// -// //if metric value is string -// if (*cur == '"') { -// searchQuote = true; -// cur += 1; -// len += 1; -// } else if (*cur == 'L' && *(cur + 1) == '"') { -// searchQuote = true; -// cur += 2; -// len += 2; -// } -// -// while(*cur != '\0') { -// if (*cur == ' ') { -// if (searchQuote == true) { -// if (*(cur - 1) == '"' && len != 1 && len != 2) { -// searchQuote = false; -// } else { -// cur++; -// len++; -// continue; -// } -// } -// -// if (*(cur + 1) != ' ') { -// break; -// } else { -// cur++; -// continue; -// } -// } -// cur++; -// len++; -// } -// -// if (len > 0 && *cur != '\0') { -// value = tcalloc(len + 1, 1); -// memcpy(value, start, len); -// } else { -// return TSDB_CODE_TSC_LINE_SYNTAX_ERROR; -// } -// -// if (!convertSmlValueType(pVal, value, len, info, false)) { -// tscError("OTD:0x%"PRIx64" Failed to convert metric value string(%s) to any type", -// info->id, value); -// tfree(value); -// return TSDB_CODE_TSC_INVALID_VALUE; -// } -// tfree(value); -// -// pVal->key = tcalloc(sizeof(key) + TS_BACKQUOTE_CHAR_SIZE, 1); -// memcpy(pVal->key, key, sizeof(key)); -// addEscapeCharToString(pVal->key, (int32_t)strlen(pVal->key)); -// *num_kvs += 1; -// -// *index = cur + 1; -// return ret; -//} -// -//static int32_t parseTelnetTagKey(TAOS_SML_KV *pKV, const char **index, SHashObj *pHash, SSmlLinesInfo* info) { -// const char *cur = *index; -// char key[TSDB_COL_NAME_LEN]; -// uint16_t len = 0; -// -// //key field cannot start with digit -// //if (isdigit(*cur)) { -// // tscError("OTD:0x%"PRIx64" Tag key cannot start with digit", info->id); -// // return TSDB_CODE_TSC_LINE_SYNTAX_ERROR; -// //} -// while (*cur != '\0') { -// if (len > TSDB_COL_NAME_LEN - 1) { -// tscError("OTD:0x%"PRIx64" Tag key cannot exceeds %d characters", info->id, TSDB_COL_NAME_LEN - 1); -// return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; -// } -// if (*cur == ' ') { -// return TSDB_CODE_TSC_LINE_SYNTAX_ERROR; -// } -// if (*cur == '=') { -// break; -// } -// -// key[len] = *cur; -// cur++; -// len++; -// } -// if (len == 0 || *cur == '\0') { -// return TSDB_CODE_TSC_LINE_SYNTAX_ERROR; -// } -// key[len] = '\0'; -// -// if (checkDuplicateKey(key, pHash, info)) { -// return TSDB_CODE_TSC_DUP_TAG_NAMES; -// } -// -// pKV->key = tcalloc(len + TS_BACKQUOTE_CHAR_SIZE + 1, 1); -// memcpy(pKV->key, key, len + 1); -// addEscapeCharToString(pKV->key, len); -// //tscDebug("OTD:0x%"PRIx64" Key:%s|len:%d", info->id, pKV->key, len); -// *index = cur + 1; -// return TSDB_CODE_SUCCESS; -//} -// -// -//static int32_t parseTelnetTagValue(TAOS_SML_KV *pKV, const char **index, -// bool *is_last_kv, SSmlLinesInfo* info) { -// const char *start, *cur; -// char *value = NULL; -// uint16_t len = 0; -// start = cur = *index; -// -// while (1) { -// // whitespace or '\0' identifies a value -// if (*cur == ' ' || *cur == '\0') { -// // '\0' indicates end of value -// *is_last_kv = (*cur == '\0') ? true : false; -// if (*cur == ' ' && *(cur + 1) == ' ') { -// cur++; -// continue; -// } else { -// break; -// } -// } -// cur++; -// len++; -// } -// -// if (len == 0) { -// tfree(pKV->key); -// return TSDB_CODE_TSC_LINE_SYNTAX_ERROR; -// } -// -// value = tcalloc(len + 1, 1); -// memcpy(value, start, len); -// value[len] = '\0'; -// if (!convertSmlValueType(pKV, value, len, info, true)) { -// tscError("OTD:0x%"PRIx64" Failed to convert sml value string(%s) to any type", -// info->id, value); -// //free previous alocated key field -// tfree(pKV->key); -// tfree(value); -// return TSDB_CODE_TSC_INVALID_VALUE; -// } -// tfree(value); -// -// *index = (*cur == '\0') ? cur : cur + 1; -// return TSDB_CODE_SUCCESS; -//} -// -//static int32_t parseTelnetTagKvs(TAOS_SML_KV **pKVs, int *num_kvs, -// const char **index, char **childTableName, -// SHashObj *pHash, SSmlLinesInfo* info) { -// const char *cur = *index; -// int32_t ret = TSDB_CODE_SUCCESS; -// TAOS_SML_KV *pkv; -// bool is_last_kv = false; -// -// int32_t capacity = 4; -// *pKVs = tcalloc(capacity, sizeof(TAOS_SML_KV)); -// pkv = *pKVs; -// -// size_t childTableNameLen = strlen(tsSmlChildTableName); -// char childTbName[TSDB_TABLE_NAME_LEN + TS_BACKQUOTE_CHAR_SIZE] = {0}; -// if (childTableNameLen != 0) { -// memcpy(childTbName, tsSmlChildTableName, childTableNameLen); -// addEscapeCharToString(childTbName, (int32_t)(childTableNameLen)); -// } -// while (*cur != '\0') { -// ret = parseTelnetTagKey(pkv, &cur, pHash, info); -// if (ret) { -// tscError("OTD:0x%"PRIx64" Unable to parse key", info->id); -// return ret; -// } -// ret = parseTelnetTagValue(pkv, &cur, &is_last_kv, info); -// if (ret) { -// tscError("OTD:0x%"PRIx64" Unable to parse value", info->id); -// return ret; -// } -// if (childTableNameLen != 0 && strcasecmp(pkv->key, childTbName) == 0) { -// *childTableName = tcalloc(pkv->length + TS_BACKQUOTE_CHAR_SIZE + 1, 1); -// memcpy(*childTableName, pkv->value, pkv->length); -// (*childTableName)[pkv->length] = '\0'; -// addEscapeCharToString(*childTableName, pkv->length); -// tfree(pkv->key); -// tfree(pkv->value); -// } else { -// *num_kvs += 1; -// } -// -// if (is_last_kv) { -// break; -// } -// -// //reallocate addtional memory for more kvs -// if ((*num_kvs + 1) > capacity) { -// TAOS_SML_KV *more_kvs = NULL; -// capacity *= 3; capacity /= 2; -// more_kvs = realloc(*pKVs, capacity * sizeof(TAOS_SML_KV)); -// if (!more_kvs) { -// return TSDB_CODE_TSC_OUT_OF_MEMORY; -// } -// *pKVs = more_kvs; -// } -// -// //move pKV points to next TAOS_SML_KV block -// pkv = *pKVs + *num_kvs; -// } -// -// return ret; -//} -// -//static int32_t tscParseTelnetLine(const char* line, TAOS_SML_DATA_POINT* smlData, SSmlLinesInfo* info) { -// const char* index = line; -// int32_t ret = TSDB_CODE_SUCCESS; -// -// //Parse metric -// ret = parseTelnetMetric(smlData, &index, info); -// if (ret) { -// tscError("OTD:0x%"PRIx64" Unable to parse metric", info->id); -// return ret; -// } -// tscDebug("OTD:0x%"PRIx64" Parse metric finished", info->id); -// -// //Parse timestamp -// ret = parseTelnetTimeStamp(&smlData->fields, &smlData->fieldNum, &index, info); -// if (ret) { -// tscError("OTD:0x%"PRIx64" Unable to parse timestamp", info->id); -// return ret; -// } -// tscDebug("OTD:0x%"PRIx64" Parse timestamp finished", info->id); -// -// //Parse value -// ret = parseTelnetMetricValue(&smlData->fields, &smlData->fieldNum, &index, info); -// if (ret) { -// tscError("OTD:0x%"PRIx64" Unable to parse metric value", info->id); -// return ret; -// } -// tscDebug("OTD:0x%"PRIx64" Parse metric value finished", info->id); -// -// //Parse tagKVs -// SHashObj *keyHashTable = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); -// ret = parseTelnetTagKvs(&smlData->tags, &smlData->tagNum, &index, &smlData->childTableName, keyHashTable, info); -// if (ret) { -// tscError("OTD:0x%"PRIx64" Unable to parse tags", info->id); -// taosHashCleanup(keyHashTable); -// return ret; -// } -// tscDebug("OTD:0x%"PRIx64" Parse tags finished", info->id); -// taosHashCleanup(keyHashTable); -// -// -// return TSDB_CODE_SUCCESS; -//} -// -//static int32_t tscParseTelnetLines(char* lines[], int numLines, SArray* points, SArray* failedLines, SSmlLinesInfo* info) { -// for (int32_t i = 0; i < numLines; ++i) { -// TAOS_SML_DATA_POINT point = {0}; -// int32_t code = tscParseTelnetLine(lines[i], &point, info); -// if (code != TSDB_CODE_SUCCESS) { -// tscError("OTD:0x%"PRIx64" data point line parse failed. line %d : %s", info->id, i, lines[i]); -// destroySmlDataPoint(&point); -// return code; -// } else { -// tscDebug("OTD:0x%"PRIx64" data point line parse success. line %d", info->id, i); -// } -// -// taosArrayPush(points, &point); -// } -// return TSDB_CODE_SUCCESS; -//} -// -//int taos_insert_telnet_lines(TAOS* taos, char* lines[], int numLines, SMLProtocolType protocol, SMLTimeStampType tsType, int* affectedRows) { -// int32_t code = 0; -// -// SSmlLinesInfo* info = tcalloc(1, sizeof(SSmlLinesInfo)); -// info->id = genUID(); -// info->tsType = tsType; -// info->protocol = protocol; -// -// if (numLines <= 0 || numLines > 65536) { -// tscError("OTD:0x%"PRIx64" taos_insert_telnet_lines numLines should be between 1 and 65536. numLines: %d", info->id, numLines); -// tfree(info); -// code = TSDB_CODE_TSC_APP_ERROR; -// return code; -// } -// -// for (int i = 0; i < numLines; ++i) { -// if (lines[i] == NULL) { -// tscError("OTD:0x%"PRIx64" taos_insert_telnet_lines line %d is NULL", info->id, i); -// tfree(info); -// code = TSDB_CODE_TSC_APP_ERROR; -// return code; -// } -// } -// -// SArray* lpPoints = taosArrayInit(numLines, sizeof(TAOS_SML_DATA_POINT)); -// if (lpPoints == NULL) { -// tscError("OTD:0x%"PRIx64" taos_insert_telnet_lines failed to allocate memory", info->id); -// tfree(info); -// return TSDB_CODE_TSC_OUT_OF_MEMORY; -// } -// -// tscDebug("OTD:0x%"PRIx64" taos_insert_telnet_lines begin inserting %d lines, first line: %s", info->id, numLines, lines[0]); -// code = tscParseTelnetLines(lines, numLines, lpPoints, NULL, info); -// size_t numPoints = taosArrayGetSize(lpPoints); -// -// if (code != 0) { -// goto cleanup; -// } -// -// TAOS_SML_DATA_POINT* points = TARRAY_GET_START(lpPoints); -// code = tscSmlInsert(taos, points, (int)numPoints, info); -// if (code != 0) { -// tscError("OTD:0x%"PRIx64" taos_insert_telnet_lines error: %s", info->id, tstrerror((code))); -// } -// if (affectedRows != NULL) { -// *affectedRows = info->affectedRows; -// } -// -//cleanup: -// tscDebug("OTD:0x%"PRIx64" taos_insert_telnet_lines finish inserting %d lines. code: %d", info->id, numLines, code); -// points = TARRAY_GET_START(lpPoints); -// numPoints = taosArrayGetSize(lpPoints); -// for (int i = 0; i < numPoints; ++i) { -// destroySmlDataPoint(points+i); -// } -// -// taosArrayDestroy(&lpPoints); -// -// tfree(info); -// return code; -//} -// -//int taos_telnet_insert(TAOS* taos, TAOS_SML_DATA_POINT* points, int numPoint) { -// SSmlLinesInfo* info = tcalloc(1, sizeof(SSmlLinesInfo)); -// info->id = genUID(); -// int code = tscSmlInsert(taos, points, numPoint, info); -// tfree(info); -// return code; -//} -// -// -///* telnet style API parser */ -//static int32_t parseMetricFromJSON(cJSON *root, TAOS_SML_DATA_POINT* pSml, SSmlLinesInfo* info) { -// cJSON *metric = cJSON_GetObjectItem(root, "metric"); -// if (!cJSON_IsString(metric)) { -// return TSDB_CODE_TSC_INVALID_JSON; -// } -// -// size_t stableLen = strlen(metric->valuestring); -// if (stableLen > TSDB_TABLE_NAME_LEN - 1) { -// tscError("OTD:0x%"PRIx64" Metric cannot exceeds %d characters in JSON", info->id, TSDB_TABLE_NAME_LEN - 1); -// return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; -// } -// -// pSml->stableName = tcalloc(stableLen + TS_BACKQUOTE_CHAR_SIZE + 1, sizeof(char)); -// if (pSml->stableName == NULL){ -// return TSDB_CODE_TSC_OUT_OF_MEMORY; -// } -// -// /* -// if (isdigit(metric->valuestring[0])) { -// tscError("OTD:0x%"PRIx64" Metric cannot start with digit in JSON", info->id); -// tfree(pSml->stableName); -// return TSDB_CODE_TSC_INVALID_JSON; -// } -// */ -// -// tstrncpy(pSml->stableName, metric->valuestring, stableLen + 1); -// addEscapeCharToString(pSml->stableName, (int32_t)stableLen); -// -// return TSDB_CODE_SUCCESS; -// -//} -// -//static int32_t parseTimestampFromJSONObj(cJSON *root, int64_t *tsVal, SSmlLinesInfo* info) { -// int32_t size = cJSON_GetArraySize(root); -// if (size != OTD_JSON_SUB_FIELDS_NUM) { -// return TSDB_CODE_TSC_INVALID_JSON; -// } -// -// cJSON *value = cJSON_GetObjectItem(root, "value"); -// if (!cJSON_IsNumber(value)) { -// return TSDB_CODE_TSC_INVALID_JSON; -// } -// -// cJSON *type = cJSON_GetObjectItem(root, "type"); -// if (!cJSON_IsString(type)) { -// return TSDB_CODE_TSC_INVALID_JSON; -// } -// -// *tsVal = strtoll(value->numberstring, NULL, 10); -// //if timestamp value is 0 use current system time -// if (*tsVal == 0) { -// *tsVal = taosGetTimestampNs(); -// return TSDB_CODE_SUCCESS; -// } -// -// size_t typeLen = strlen(type->valuestring); -// if (typeLen == 1 && type->valuestring[0] == 's') { -// //seconds -// *tsVal = (int64_t)(*tsVal * 1e9); -// } else if (typeLen == 2 && type->valuestring[1] == 's') { -// switch (type->valuestring[0]) { -// case 'm': -// //milliseconds -// *tsVal = convertTimePrecision(*tsVal, TSDB_TIME_PRECISION_MILLI, TSDB_TIME_PRECISION_NANO); -// break; -// case 'u': -// //microseconds -// *tsVal = convertTimePrecision(*tsVal, TSDB_TIME_PRECISION_MICRO, TSDB_TIME_PRECISION_NANO); -// break; -// case 'n': -// //nanoseconds -// *tsVal = *tsVal * 1; -// break; -// default: -// return TSDB_CODE_TSC_INVALID_JSON; -// } -// } else { -// return TSDB_CODE_TSC_INVALID_JSON; -// } -// -// return TSDB_CODE_SUCCESS; -//} -// -//static int32_t parseTimestampFromJSON(cJSON *root, TAOS_SML_KV **pTS, int *num_kvs, SSmlLinesInfo* info) { -// //Timestamp must be the first KV to parse -// assert(*num_kvs == 0); -// int64_t tsVal; -// char key[] = OTD_TIMESTAMP_COLUMN_NAME; -// -// cJSON *timestamp = cJSON_GetObjectItem(root, "timestamp"); -// if (cJSON_IsNumber(timestamp)) { -// //timestamp value 0 indicates current system time -// if (timestamp->valueint == 0) { -// tsVal = taosGetTimestampNs(); -// } else { -// tsVal = strtoll(timestamp->numberstring, NULL, 10); -// size_t tsLen = strlen(timestamp->numberstring); -// if (tsLen == SML_TIMESTAMP_SECOND_DIGITS) { -// tsVal = (int64_t)(tsVal * 1e9); -// } else if (tsLen == SML_TIMESTAMP_MILLI_SECOND_DIGITS) { -// tsVal = convertTimePrecision(tsVal, TSDB_TIME_PRECISION_MILLI, TSDB_TIME_PRECISION_NANO); -// } else { -// return TSDB_CODE_TSC_INVALID_TIME_STAMP; -// } -// } -// } else if (cJSON_IsObject(timestamp)) { -// int32_t ret = parseTimestampFromJSONObj(timestamp, &tsVal, info); -// if (ret != TSDB_CODE_SUCCESS) { -// tscError("OTD:0x%"PRIx64" Failed to parse timestamp from JSON Obj", info->id); -// return ret; -// } -// } else { -// return TSDB_CODE_TSC_INVALID_JSON; -// } -// -// //allocate fields for timestamp and value -// *pTS = tcalloc(OTD_MAX_FIELDS_NUM, sizeof(TAOS_SML_KV)); -// -// -// (*pTS)->key = tcalloc(sizeof(key), 1); -// memcpy((*pTS)->key, key, sizeof(key)); -// -// (*pTS)->type = TSDB_DATA_TYPE_TIMESTAMP; -// (*pTS)->length = (int16_t)tDataTypes[(*pTS)->type].bytes; -// (*pTS)->value = tcalloc((*pTS)->length, 1); -// memcpy((*pTS)->value, &tsVal, (*pTS)->length); -// -// *num_kvs += 1; -// return TSDB_CODE_SUCCESS; -// -//} -// -//static int32_t convertJSONBool(TAOS_SML_KV *pVal, char* typeStr, int64_t valueInt, SSmlLinesInfo* info) { -// if (strcasecmp(typeStr, "bool") != 0) { -// tscError("OTD:0x%"PRIx64" invalid type(%s) for JSON Bool", info->id, typeStr); -// return TSDB_CODE_TSC_INVALID_JSON_TYPE; -// } -// pVal->type = TSDB_DATA_TYPE_BOOL; -// pVal->length = (int16_t)tDataTypes[pVal->type].bytes; -// pVal->value = tcalloc(pVal->length, 1); -// *(bool *)(pVal->value) = valueInt ? true : false; -// -// return TSDB_CODE_SUCCESS; -//} -// -//static int32_t convertJSONNumber(TAOS_SML_KV *pVal, char* typeStr, cJSON *value, SSmlLinesInfo* info) { -// //tinyint -// if (strcasecmp(typeStr, "i8") == 0 || -// strcasecmp(typeStr, "tinyint") == 0) { -// if (!IS_VALID_TINYINT(value->valueint)) { -// tscError("OTD:0x%"PRIx64" JSON value(%"PRId64") cannot fit in type(tinyint)", info->id, value->valueint); -// return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; -// } -// pVal->type = TSDB_DATA_TYPE_TINYINT; -// pVal->length = (int16_t)tDataTypes[pVal->type].bytes; -// pVal->value = tcalloc(pVal->length, 1); -// *(int8_t *)(pVal->value) = (int8_t)(value->valueint); -// return TSDB_CODE_SUCCESS; -// } -// //smallint -// if (strcasecmp(typeStr, "i16") == 0 || -// strcasecmp(typeStr, "smallint") == 0) { -// if (!IS_VALID_SMALLINT(value->valueint)) { -// tscError("OTD:0x%"PRIx64" JSON value(%"PRId64") cannot fit in type(smallint)", info->id, value->valueint); -// return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; -// } -// pVal->type = TSDB_DATA_TYPE_SMALLINT; -// pVal->length = (int16_t)tDataTypes[pVal->type].bytes; -// pVal->value = tcalloc(pVal->length, 1); -// *(int16_t *)(pVal->value) = (int16_t)(value->valueint); -// return TSDB_CODE_SUCCESS; -// } -// //int -// if (strcasecmp(typeStr, "i32") == 0 || -// strcasecmp(typeStr, "int") == 0) { -// if (!IS_VALID_INT(value->valueint)) { -// tscError("OTD:0x%"PRIx64" JSON value(%"PRId64") cannot fit in type(int)", info->id, value->valueint); -// return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; -// } -// pVal->type = TSDB_DATA_TYPE_INT; -// pVal->length = (int16_t)tDataTypes[pVal->type].bytes; -// pVal->value = tcalloc(pVal->length, 1); -// *(int32_t *)(pVal->value) = (int32_t)(value->valueint); -// return TSDB_CODE_SUCCESS; -// } -// //bigint -// if (strcasecmp(typeStr, "i64") == 0 || -// strcasecmp(typeStr, "bigint") == 0) { -// pVal->type = TSDB_DATA_TYPE_BIGINT; -// pVal->length = (int16_t)tDataTypes[pVal->type].bytes; -// pVal->value = tcalloc(pVal->length, 1); -// /* cJSON conversion of legit BIGINT may overflow, -// * use original string to do the conversion. -// */ -// errno = 0; -// int64_t val = (int64_t)strtoll(value->numberstring, NULL, 10); -// if (errno == ERANGE || !IS_VALID_BIGINT(val)) { -// tscError("OTD:0x%"PRIx64" JSON value(%s) cannot fit in type(bigint)", info->id, value->numberstring); -// return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; -// } -// *(int64_t *)(pVal->value) = val; -// return TSDB_CODE_SUCCESS; -// } -// //float -// if (strcasecmp(typeStr, "f32") == 0 || -// strcasecmp(typeStr, "float") == 0) { -// if (!IS_VALID_FLOAT(value->valuedouble)) { -// tscError("OTD:0x%"PRIx64" JSON value(%f) cannot fit in type(float)", info->id, value->valuedouble); -// return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; -// } -// pVal->type = TSDB_DATA_TYPE_FLOAT; -// pVal->length = (int16_t)tDataTypes[pVal->type].bytes; -// pVal->value = tcalloc(pVal->length, 1); -// *(float *)(pVal->value) = (float)(value->valuedouble); -// return TSDB_CODE_SUCCESS; -// } -// //double -// if (strcasecmp(typeStr, "f64") == 0 || -// strcasecmp(typeStr, "double") == 0) { -// if (!IS_VALID_DOUBLE(value->valuedouble)) { -// tscError("OTD:0x%"PRIx64" JSON value(%f) cannot fit in type(double)", info->id, value->valuedouble); -// return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; -// } -// pVal->type = TSDB_DATA_TYPE_DOUBLE; -// pVal->length = (int16_t)tDataTypes[pVal->type].bytes; -// pVal->value = tcalloc(pVal->length, 1); -// *(double *)(pVal->value) = (double)(value->valuedouble); -// return TSDB_CODE_SUCCESS; -// } -// -// //if reach here means type is unsupported -// tscError("OTD:0x%"PRIx64" invalid type(%s) for JSON Number", info->id, typeStr); -// return TSDB_CODE_TSC_INVALID_JSON_TYPE; -//} -// -//static int32_t convertJSONString(TAOS_SML_KV *pVal, char* typeStr, cJSON *value, SSmlLinesInfo* info) { -// if (strcasecmp(typeStr, "binary") == 0) { -// pVal->type = TSDB_DATA_TYPE_BINARY; -// } else if (strcasecmp(typeStr, "nchar") == 0) { -// pVal->type = TSDB_DATA_TYPE_NCHAR; -// } else { -// tscError("OTD:0x%"PRIx64" invalid type(%s) for JSON String", info->id, typeStr); -// return TSDB_CODE_TSC_INVALID_JSON_TYPE; -// } -// pVal->length = (int16_t)strlen(value->valuestring); -// pVal->value = tcalloc(pVal->length + 1, 1); -// memcpy(pVal->value, value->valuestring, pVal->length); -// return TSDB_CODE_SUCCESS; -//} -// -//static int32_t parseValueFromJSONObj(cJSON *root, TAOS_SML_KV *pVal, SSmlLinesInfo* info) { -// int32_t ret = TSDB_CODE_SUCCESS; -// int32_t size = cJSON_GetArraySize(root); -// -// if (size != OTD_JSON_SUB_FIELDS_NUM) { -// return TSDB_CODE_TSC_INVALID_JSON; -// } -// -// cJSON *value = cJSON_GetObjectItem(root, "value"); -// if (value == NULL) { -// return TSDB_CODE_TSC_INVALID_JSON; -// } -// -// cJSON *type = cJSON_GetObjectItem(root, "type"); -// if (!cJSON_IsString(type)) { -// return TSDB_CODE_TSC_INVALID_JSON; -// } -// -// switch (value->type) { -// case cJSON_True: -// case cJSON_False: { -// ret = convertJSONBool(pVal, type->valuestring, value->valueint, info); -// if (ret != TSDB_CODE_SUCCESS) { -// return ret; -// } -// break; -// } -// case cJSON_Number: { -// ret = convertJSONNumber(pVal, type->valuestring, value, info); -// if (ret != TSDB_CODE_SUCCESS) { -// return ret; -// } -// break; -// } -// case cJSON_String: { -// ret = convertJSONString(pVal, type->valuestring, value, info); -// if (ret != TSDB_CODE_SUCCESS) { -// return ret; -// } -// break; -// } -// default: -// return TSDB_CODE_TSC_INVALID_JSON_TYPE; -// } -// -// return TSDB_CODE_SUCCESS; -//} -// -//static int32_t parseValueFromJSON(cJSON *root, TAOS_SML_KV *pVal, SSmlLinesInfo* info) { -// int type = root->type; -// -// switch (type) { -// case cJSON_True: -// case cJSON_False: { -// pVal->type = TSDB_DATA_TYPE_BOOL; -// pVal->length = (int16_t)tDataTypes[pVal->type].bytes; -// pVal->value = tcalloc(pVal->length, 1); -// *(bool *)(pVal->value) = root->valueint ? true : false; -// break; -// } -// case cJSON_Number: { -// //convert default JSON Number type to BIGINT/DOUBLE -// //if (isValidInteger(root->numberstring)) { -// // pVal->type = TSDB_DATA_TYPE_BIGINT; -// // pVal->length = (int16_t)tDataTypes[pVal->type].bytes; -// // pVal->value = tcalloc(pVal->length, 1); -// // /* cJSON conversion of legit BIGINT may overflow, -// // * use original string to do the conversion. -// // */ -// // errno = 0; -// // int64_t val = (int64_t)strtoll(root->numberstring, NULL, 10); -// // if (errno == ERANGE || !IS_VALID_BIGINT(val)) { -// // tscError("OTD:0x%"PRIx64" JSON value(%s) cannot fit in type(bigint)", info->id, root->numberstring); -// // return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; -// // } -// // *(int64_t *)(pVal->value) = val; -// //} else if (isValidFloat(root->numberstring)) { -// // pVal->type = TSDB_DATA_TYPE_DOUBLE; -// // pVal->length = (int16_t)tDataTypes[pVal->type].bytes; -// // pVal->value = tcalloc(pVal->length, 1); -// // *(double *)(pVal->value) = (double)(root->valuedouble); -// //} else { -// // return TSDB_CODE_TSC_INVALID_JSON_TYPE; -// //} -// if (isValidInteger(root->numberstring) || isValidFloat(root->numberstring)) { -// pVal->type = TSDB_DATA_TYPE_DOUBLE; -// pVal->length = (int16_t)tDataTypes[pVal->type].bytes; -// pVal->value = tcalloc(pVal->length, 1); -// *(double *)(pVal->value) = (double)(root->valuedouble); -// } -// -// break; -// } -// case cJSON_String: { -// /* set default JSON type to binary/nchar according to -// * user configured parameter tsDefaultJSONStrType -// */ -// if (strcasecmp(tsDefaultJSONStrType, "binary") == 0) { -// pVal->type = TSDB_DATA_TYPE_BINARY; -// } else if (strcasecmp(tsDefaultJSONStrType, "nchar") == 0) { -// pVal->type = TSDB_DATA_TYPE_NCHAR; -// } else { -// tscError("OTD:0x%"PRIx64" Invalid default JSON string type set from config %s", info->id, tsDefaultJSONStrType); -// return TSDB_CODE_TSC_INVALID_JSON_CONFIG; -// } -// //pVal->length = wcslen((wchar_t *)root->valuestring) * TSDB_NCHAR_SIZE; -// pVal->length = (int16_t)strlen(root->valuestring); -// pVal->value = tcalloc(pVal->length + 1, 1); -// memcpy(pVal->value, root->valuestring, pVal->length); -// break; -// } -// case cJSON_Object: { -// int32_t ret = parseValueFromJSONObj(root, pVal, info); -// if (ret != TSDB_CODE_SUCCESS) { -// tscError("OTD:0x%"PRIx64" Failed to parse timestamp from JSON Obj", info->id); -// return ret; -// } -// break; -// } -// default: -// return TSDB_CODE_TSC_INVALID_JSON; -// } -// -// return TSDB_CODE_SUCCESS; -//} -// -//static int32_t parseMetricValueFromJSON(cJSON *root, TAOS_SML_KV **pKVs, int *num_kvs, SSmlLinesInfo* info) { -// //skip timestamp -// TAOS_SML_KV *pVal = *pKVs + 1; -// char key[] = OTD_METRIC_VALUE_COLUMN_NAME; -// -// cJSON *metricVal = cJSON_GetObjectItem(root, "value"); -// if (metricVal == NULL) { -// return TSDB_CODE_TSC_INVALID_JSON; -// } -// -// int32_t ret = parseValueFromJSON(metricVal, pVal, info); -// if (ret != TSDB_CODE_SUCCESS) { -// return ret; -// } -// -// pVal->key = tcalloc(sizeof(key) + TS_BACKQUOTE_CHAR_SIZE, 1); -// memcpy(pVal->key, key, sizeof(key)); -// addEscapeCharToString(pVal->key, (int32_t)strlen(pVal->key)); -// -// *num_kvs += 1; -// return TSDB_CODE_SUCCESS; -// -//} -// -// -//static int32_t parseTagsFromJSON(cJSON *root, TAOS_SML_KV **pKVs, int *num_kvs, char **childTableName, -// SHashObj *pHash, SSmlLinesInfo* info) { -// int32_t ret = TSDB_CODE_SUCCESS; -// -// cJSON *tags = cJSON_GetObjectItem(root, "tags"); -// if (tags == NULL || tags->type != cJSON_Object) { -// return TSDB_CODE_TSC_INVALID_JSON; -// } -// -// //handle child table name -// size_t childTableNameLen = strlen(tsSmlChildTableName); -// char childTbName[TSDB_TABLE_NAME_LEN] = {0}; -// if (childTableNameLen != 0) { -// memcpy(childTbName, tsSmlChildTableName, childTableNameLen); -// cJSON *id = cJSON_GetObjectItem(tags, childTbName); -// if (id != NULL) { -// if (!cJSON_IsString(id)) { -// tscError("OTD:0x%"PRIx64" ID must be JSON string", info->id); -// return TSDB_CODE_TSC_INVALID_JSON; -// } -// size_t idLen = strlen(id->valuestring); -// *childTableName = tcalloc(idLen + TS_BACKQUOTE_CHAR_SIZE + 1, sizeof(char)); -// memcpy(*childTableName, id->valuestring, idLen); -// addEscapeCharToString(*childTableName, (int32_t)idLen); -// -// //check duplicate IDs -// cJSON_DeleteItemFromObject(tags, childTbName); -// id = cJSON_GetObjectItem(tags, childTbName); -// if (id != NULL) { -// return TSDB_CODE_TSC_DUP_TAG_NAMES; -// } -// } -// } -// -// int32_t tagNum = cJSON_GetArraySize(tags); -// //at least one tag pair required -// if (tagNum <= 0) { -// return TSDB_CODE_TSC_INVALID_JSON; -// } -// -// //allocate memory for tags -// *pKVs = tcalloc(tagNum, sizeof(TAOS_SML_KV)); -// TAOS_SML_KV *pkv = *pKVs; -// -// for (int32_t i = 0; i < tagNum; ++i) { -// cJSON *tag = cJSON_GetArrayItem(tags, i); -// if (tag == NULL) { -// return TSDB_CODE_TSC_INVALID_JSON; -// } -// //check duplicate keys -// if (checkDuplicateKey(tag->string, pHash, info)) { -// return TSDB_CODE_TSC_DUP_TAG_NAMES; -// } -// //key -// size_t keyLen = strlen(tag->string); -// if (keyLen > TSDB_COL_NAME_LEN - 1) { -// tscError("OTD:0x%"PRIx64" Tag key cannot exceeds %d characters in JSON", info->id, TSDB_COL_NAME_LEN - 1); -// return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; -// } -// pkv->key = tcalloc(keyLen + TS_BACKQUOTE_CHAR_SIZE + 1, sizeof(char)); -// strncpy(pkv->key, tag->string, keyLen); -// addEscapeCharToString(pkv->key, (int32_t)keyLen); -// //value -// ret = parseValueFromJSON(tag, pkv, info); -// if (ret != TSDB_CODE_SUCCESS) { -// return ret; -// } -// *num_kvs += 1; -// pkv++; -// -// } -// -// return ret; -// -//} -// -//static int32_t tscParseJSONPayload(cJSON *root, TAOS_SML_DATA_POINT* pSml, SSmlLinesInfo* info) { -// int32_t ret = TSDB_CODE_SUCCESS; -// -// if (!cJSON_IsObject(root)) { -// tscError("OTD:0x%"PRIx64" data point needs to be JSON object", info->id); -// return TSDB_CODE_TSC_INVALID_JSON; -// } -// -// int32_t size = cJSON_GetArraySize(root); -// //outmost json fields has to be exactly 4 -// if (size != OTD_JSON_FIELDS_NUM) { -// tscError("OTD:0x%"PRIx64" Invalid number of JSON fields in data point %d", info->id, size); -// return TSDB_CODE_TSC_INVALID_JSON; -// } -// -// //Parse metric -// ret = parseMetricFromJSON(root, pSml, info); -// if (ret != TSDB_CODE_SUCCESS) { -// tscError("OTD:0x%"PRIx64" Unable to parse metric from JSON payload", info->id); -// return ret; -// } -// tscDebug("OTD:0x%"PRIx64" Parse metric from JSON payload finished", info->id); -// -// //Parse timestamp -// ret = parseTimestampFromJSON(root, &pSml->fields, &pSml->fieldNum, info); -// if (ret) { -// tscError("OTD:0x%"PRIx64" Unable to parse timestamp from JSON payload", info->id); -// return ret; -// } -// tscDebug("OTD:0x%"PRIx64" Parse timestamp from JSON payload finished", info->id); -// -// //Parse metric value -// ret = parseMetricValueFromJSON(root, &pSml->fields, &pSml->fieldNum, info); -// if (ret) { -// tscError("OTD:0x%"PRIx64" Unable to parse metric value from JSON payload", info->id); -// return ret; -// } -// tscDebug("OTD:0x%"PRIx64" Parse metric value from JSON payload finished", info->id); -// -// //Parse tags -// SHashObj *keyHashTable = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); -// ret = parseTagsFromJSON(root, &pSml->tags, &pSml->tagNum, &pSml->childTableName, keyHashTable, info); -// if (ret) { -// tscError("OTD:0x%"PRIx64" Unable to parse tags from JSON payload", info->id); -// taosHashCleanup(keyHashTable); -// return ret; -// } -// tscDebug("OTD:0x%"PRIx64" Parse tags from JSON payload finished", info->id); -// taosHashCleanup(keyHashTable); -// -// return TSDB_CODE_SUCCESS; -//} -// -//static int32_t tscParseMultiJSONPayload(char* payload, SArray* points, SSmlLinesInfo* info) { -// int32_t payloadNum, ret; -// ret = TSDB_CODE_SUCCESS; -// -// if (payload == NULL) { -// tscError("OTD:0x%"PRIx64" empty JSON Payload", info->id); -// return TSDB_CODE_TSC_INVALID_JSON; -// } -// -// cJSON *root = cJSON_Parse(payload); -// //multiple data points must be sent in JSON array -// if (cJSON_IsObject(root)) { -// payloadNum = 1; -// } else if (cJSON_IsArray(root)) { -// payloadNum = cJSON_GetArraySize(root); -// } else { -// tscError("OTD:0x%"PRIx64" Invalid JSON Payload", info->id); -// ret = TSDB_CODE_TSC_INVALID_JSON; -// goto PARSE_JSON_OVER; -// } -// -// for (int32_t i = 0; i < payloadNum; ++i) { -// TAOS_SML_DATA_POINT point = {0}; -// cJSON *dataPoint = (payloadNum == 1 && cJSON_IsObject(root)) ? root : cJSON_GetArrayItem(root, i); -// -// ret = tscParseJSONPayload(dataPoint, &point, info); -// if (ret != TSDB_CODE_SUCCESS) { -// tscError("OTD:0x%"PRIx64" JSON data point parse failed", info->id); -// destroySmlDataPoint(&point); -// goto PARSE_JSON_OVER; -// } else { -// tscDebug("OTD:0x%"PRIx64" JSON data point parse success", info->id); -// } -// taosArrayPush(points, &point); -// } -// -//PARSE_JSON_OVER: -// cJSON_Delete(root); -// return ret; -//} -// -//int taos_insert_json_payload(TAOS* taos, char* payload, SMLProtocolType protocol, SMLTimeStampType tsType, int* affectedRows) { -// int32_t code = 0; -// -// SSmlLinesInfo* info = tcalloc(1, sizeof(SSmlLinesInfo)); -// info->id = genUID(); -// info->tsType = tsType; -// info->protocol = protocol; -// -// if (payload == NULL) { -// tscError("OTD:0x%"PRIx64" taos_insert_json_payload payload is NULL", info->id); -// tfree(info); -// code = TSDB_CODE_TSC_APP_ERROR; -// return code; -// } -// -// SArray* lpPoints = taosArrayInit(1, sizeof(TAOS_SML_DATA_POINT)); -// if (lpPoints == NULL) { -// tscError("OTD:0x%"PRIx64" taos_insert_json_payload failed to allocate memory", info->id); -// tfree(info); -// return TSDB_CODE_TSC_OUT_OF_MEMORY; -// } -// -// tscDebug("OTD:0x%"PRIx64" taos_insert_telnet_lines begin inserting %d points", info->id, 1); -// code = tscParseMultiJSONPayload(payload, lpPoints, info); -// size_t numPoints = taosArrayGetSize(lpPoints); -// -// if (code != 0) { -// goto cleanup; -// } -// -// TAOS_SML_DATA_POINT* points = TARRAY_GET_START(lpPoints); -// code = tscSmlInsert(taos, points, (int)numPoints, info); -// if (code != 0) { -// tscError("OTD:0x%"PRIx64" taos_insert_json_payload error: %s", info->id, tstrerror((code))); -// } -// if (affectedRows != NULL) { -// *affectedRows = info->affectedRows; -// } -// -//cleanup: -// tscDebug("OTD:0x%"PRIx64" taos_insert_json_payload finish inserting 1 Point. code: %d", info->id, code); -// points = TARRAY_GET_START(lpPoints); -// numPoints = taosArrayGetSize(lpPoints); -// for (int i = 0; i < numPoints; ++i) { -// destroySmlDataPoint(points+i); -// } -// -// taosArrayDestroy(&lpPoints); -// -// tfree(info); -// return code; -//} diff --git a/source/client/src/tscParseLineProtocol.c b/source/client/src/clientSml.c similarity index 51% rename from source/client/src/tscParseLineProtocol.c rename to source/client/src/clientSml.c index e0b4cc0d7a..dfbefcb0bf 100644 --- a/source/client/src/tscParseLineProtocol.c +++ b/source/client/src/clientSml.c @@ -3,7 +3,7 @@ #include #include -#include "tscParseLine.h" +#include "clientSml.h" #include "tdef.h" #include "ttypes.h" @@ -29,6 +29,7 @@ typedef struct { #define EQUAL '=' #define QUOTE '"' #define SLASH '\\' +#define tsMaxSQLStringLen (1024*1024) //================================================================================================= @@ -71,8 +72,8 @@ typedef enum { typedef struct { char sTableName[TSDB_TABLE_NAME_LEN]; - SArray* tags; //SArray - SArray* fields; //SArray + SHashObj *tags; + SHashObj *fields; } SCreateSTableActionInfo; typedef struct { @@ -339,21 +340,22 @@ static int32_t generateSchemaAction(SSchema* pointColField, SHashObj* dbAttrHash return 0; } -static int32_t buildColumnDescription(SSchema* field, +static int32_t buildColumnDescription(TAOS_SML_KV* field, char* buf, int32_t bufSize, int32_t* outBytes) { uint8_t type = field->type; - + char tname[TSDB_TABLE_NAME_LEN] = {0}; + memcpy(tname, field->key, field->keyLen); if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { - int32_t bytes = field->bytes - VARSTR_HEADER_SIZE; + int32_t bytes = field->length - VARSTR_HEADER_SIZE; if (type == TSDB_DATA_TYPE_NCHAR) { bytes = bytes/TSDB_NCHAR_SIZE; } int out = snprintf(buf, bufSize,"%s %s(%d)", - field->name,tDataTypes[field->type].name, bytes); + tname,tDataTypes[field->type].name, bytes); *outBytes = out; } else { int out = snprintf(buf, bufSize, "%s %s", - field->name, tDataTypes[type].name); + tname, tDataTypes[type].name); *outBytes = out; } @@ -364,7 +366,7 @@ static int32_t buildColumnDescription(SSchema* field, static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInfo* info) { int32_t code = 0; int32_t outBytes = 0; - char *result = (char *)calloc(1, tsMaxSQLStringLen+1); + char *result = (char *)taosMemoryCalloc(1, tsMaxSQLStringLen+1); int32_t capacity = tsMaxSQLStringLen + 1; uDebug("SML:0x%"PRIx64" apply schema action. action: %d", info->id, action->action); @@ -374,7 +376,7 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInf buildColumnDescription(action->alterSTable.field, result+n, capacity-n, &outBytes); TAOS_RES* res = taos_query(taos, result); //TODO async doAsyncQuery code = taos_errno(res); - char* errStr = taos_errstr(res); + const char* errStr = taos_errstr(res); char* begin = strstr(errStr, "duplicated column names"); bool tscDupColNames = (begin != NULL); if (code != TSDB_CODE_SUCCESS) { @@ -382,7 +384,8 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInf } taos_free_result(res); - if (code == TSDB_CODE_MND_FIELD_ALREAY_EXIST || code == TSDB_CODE_MND_TAG_ALREAY_EXIST || tscDupColNames) { +// if (code == TSDB_CODE_MND_FIELD_ALREADY_EXIST || code == TSDB_CODE_MND_TAG_ALREADY_EXIST || tscDupColNames) { + if (code == TSDB_CODE_MND_TAG_ALREADY_EXIST || tscDupColNames) { TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); code = taos_errno(res2); if (code != TSDB_CODE_SUCCESS) { @@ -399,7 +402,7 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInf result+n, capacity-n, &outBytes); TAOS_RES* res = taos_query(taos, result); //TODO async doAsyncQuery code = taos_errno(res); - char* errStr = taos_errstr(res); + const char* errStr = taos_errstr(res); char* begin = strstr(errStr, "duplicated column names"); bool tscDupColNames = (begin != NULL); if (code != TSDB_CODE_SUCCESS) { @@ -407,7 +410,8 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInf } taos_free_result(res); - if (code == TSDB_CODE_MND_TAG_ALREAY_EXIST || code == TSDB_CODE_MND_FIELD_ALREAY_EXIST || tscDupColNames) { +// if (code ==TSDB_CODE_MND_TAG_ALREADY_EXIST || code == TSDB_CODE_MND_FIELD_ALREAY_EXIST || tscDupColNames) { + if (code ==TSDB_CODE_MND_TAG_ALREADY_EXIST || tscDupColNames) { TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); code = taos_errno(res2); if (code != TSDB_CODE_SUCCESS) { @@ -429,7 +433,8 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInf } taos_free_result(res); - if (code == TSDB_CODE_MND_INVALID_COLUMN_LENGTH || code == TSDB_CODE_TSC_INVALID_COLUMN_LENGTH) { +// if (code == TSDB_CODE_MND_INVALID_COLUMN_LENGTH || code == TSDB_CODE_TSC_INVALID_COLUMN_LENGTH) { + if (code == TSDB_CODE_TSC_INVALID_COLUMN_LENGTH) { TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); code = taos_errno(res2); if (code != TSDB_CODE_SUCCESS) { @@ -451,7 +456,8 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInf } taos_free_result(res); - if (code == TSDB_CODE_MND_INVALID_TAG_LENGTH || code == TSDB_CODE_TSC_INVALID_TAG_LENGTH) { +// if (code == TSDB_CODE_MND_INVALID_TAG_LENGTH || code == TSDB_CODE_TSC_INVALID_TAG_LENGTH) { + if (code == TSDB_CODE_TSC_INVALID_TAG_LENGTH) { TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); code = taos_errno(res2); if (code != TSDB_CODE_SUCCESS) { @@ -465,24 +471,25 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInf case SCHEMA_ACTION_CREATE_STABLE: { int n = sprintf(result, "create stable %s (", action->createSTable.sTableName); char* pos = result + n; int freeBytes = capacity - n; - size_t numCols = taosArrayGetSize(action->createSTable.fields); - for (int32_t i = 0; i < numCols; ++i) { - SSchema* field = taosArrayGet(action->createSTable.fields, i); - buildColumnDescription(field, pos, freeBytes, &outBytes); + + TAOS_SML_KV **kv = taosHashIterate(action->createSTable.fields, NULL); + while(kv){ + buildColumnDescription(*kv, pos, freeBytes, &outBytes); pos += outBytes; freeBytes -= outBytes; *pos = ','; ++pos; --freeBytes; + kv = taosHashIterate(action->createSTable.fields, kv); } --pos; ++freeBytes; outBytes = snprintf(pos, freeBytes, ") tags ("); pos += outBytes; freeBytes -= outBytes; - size_t numTags = taosArrayGetSize(action->createSTable.tags); - for (int32_t i = 0; i < numTags; ++i) { - SSchema* field = taosArrayGet(action->createSTable.tags, i); - buildColumnDescription(field, pos, freeBytes, &outBytes); + kv = taosHashIterate(action->createSTable.tags, NULL); + while(kv){ + buildColumnDescription(*kv, pos, freeBytes, &outBytes); pos += outBytes; freeBytes -= outBytes; *pos = ','; ++pos; --freeBytes; + kv = taosHashIterate(action->createSTable.tags, kv); } pos--; ++freeBytes; outBytes = snprintf(pos, freeBytes, ")"); @@ -493,7 +500,7 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInf } taos_free_result(res); - if (code == TSDB_CODE_MND_TABLE_ALREADY_EXIST) { + if (code == TSDB_CODE_MND_STB_ALREADY_EXIST) { TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); code = taos_errno(res2); if (code != TSDB_CODE_SUCCESS) { @@ -680,643 +687,79 @@ static int32_t loadTableSchemaFromDB(TAOS* taos, char* tableName, SSmlSTableSche static int32_t modifyDBSchemas(TAOS* taos, SSmlLinesInfo* info) { int32_t code = 0; - size_t numStable = taosHashGetSize(info->superTables); SSmlSTableMeta** tableMetaSml = taosHashIterate(info->superTables, NULL); while (tableMetaSml) { SSmlSTableMeta* cTablePoints = *tableMetaSml; - if (NULL == pStmt->pCatalog) { - STMT_ERR_RET(catalogGetHandle(pStmt->taos->pAppInfo->clusterId, &pStmt->pCatalog)); - } - STableMeta *pTableMeta = NULL; - SEpSet ep = getEpSet_s(&pStmt->taos->pAppInfo->mgmtEp); - STMT_ERR_RET(catalogGetTableMeta(pStmt->pCatalog, pStmt->taos->pAppInfo->pTransporter, &ep, &pStmt->bInfo.sname, &pTableMeta)); + SEpSet ep = getEpSet_s(&info->taos->pAppInfo->mgmtEp); - if (pTableMeta->uid == pStmt->bInfo.tbUid) { - pStmt->bInfo.needParse = false; + size_t superTableLen = 0; + void *superTable = taosHashGetKey(tableMetaSml, &superTableLen); + SName pName = {TSDB_TABLE_NAME_T, info->taos->acctId, {0}, {0}}; + strcpy(pName.dbname, info->pRequest->pDb); + memcpy(pName.tname, superTable, superTableLen); - return TSDB_CODE_SUCCESS; - } + code = catalogGetSTableMeta(info->pCatalog, info->taos->pAppInfo->pTransporter, &ep, &pName, &pTableMeta); -// for (int i = 0; i < numStable; ++i) { - SSmlSTableSchema* pointSchema = taosArrayGet(stableSchemas, i); - SSmlSTableSchema dbSchema; - memset(&dbSchema, 0, sizeof(SSmlSTableSchema)); - - code = loadTableSchemaFromDB(taos, pointSchema->sTableName, &dbSchema, info); - if (code == TSDB_CODE_MND_INVALID_TABLE_NAME) { + if (code == TSDB_CODE_TDB_INVALID_TABLE_ID) { SSchemaAction schemaAction = {0}; schemaAction.action = SCHEMA_ACTION_CREATE_STABLE; memset(&schemaAction.createSTable, 0, sizeof(SCreateSTableActionInfo)); - memcpy(schemaAction.createSTable.sTableName, pointSchema->sTableName, TSDB_TABLE_NAME_LEN); - schemaAction.createSTable.tags = pointSchema->tags; - schemaAction.createSTable.fields = pointSchema->fields; + memcpy(schemaAction.createSTable.sTableName, superTable, superTableLen); + schemaAction.createSTable.tags = cTablePoints->tagHash; + schemaAction.createSTable.fields = cTablePoints->fieldHash; applySchemaAction(taos, &schemaAction, info); - code = loadTableSchemaFromDB(taos, pointSchema->sTableName, &dbSchema, info); + code = catalogGetSTableMeta(info->pCatalog, info->taos->pAppInfo->pTransporter, &ep, &pName, &pTableMeta); if (code != 0) { - uError("SML:0x%"PRIx64" reconcile point schema failed. can not create %s", info->id, pointSchema->sTableName); + uError("SML:0x%"PRIx64" reconcile point schema failed. can not create %s", info->id, schemaAction.createSTable.sTableName); return code; } - } - - if (code == TSDB_CODE_SUCCESS) { - pointSchema->precision = dbSchema.precision; - - size_t pointTagSize = taosArrayGetSize(pointSchema->tags); - size_t pointFieldSize = taosArrayGetSize(pointSchema->fields); - - SHashObj* dbTagHash = dbSchema.tagHash; - SHashObj* dbFieldHash = dbSchema.fieldHash; - - for (int j = 0; j < pointTagSize; ++j) { - SSchema* pointTag = taosArrayGet(pointSchema->tags, j); - SSchemaAction schemaAction = {0}; - bool actionNeeded = false; - generateSchemaAction(pointTag, dbTagHash, dbSchema.tags, true, pointSchema->sTableName, - &schemaAction, &actionNeeded, info); - if (actionNeeded) { - code = applySchemaAction(taos, &schemaAction, info); - if (code != 0) { - destroySmlSTableSchema(&dbSchema); - return code; - } - } - } - - SSchema* pointColTs = taosArrayGet(pointSchema->fields, 0); - SSchema* dbColTs = taosArrayGet(dbSchema.fields, 0); - memcpy(pointColTs->name, dbColTs->name, TSDB_COL_NAME_LEN); - - for (int j = 1; j < pointFieldSize; ++j) { - SSchema* pointCol = taosArrayGet(pointSchema->fields, j); - SSchemaAction schemaAction = {0}; - bool actionNeeded = false; - generateSchemaAction(pointCol, dbFieldHash, dbSchema.fields,false, pointSchema->sTableName, - &schemaAction, &actionNeeded, info); - if (actionNeeded) { - code = applySchemaAction(taos, &schemaAction, info); - if (code != 0) { - destroySmlSTableSchema(&dbSchema); - return code; - } - } - } - - pointSchema->precision = dbSchema.precision; - - destroySmlSTableSchema(&dbSchema); + }else if (code == TSDB_CODE_SUCCESS) { } else { uError("SML:0x%"PRIx64" load table meta error: %s", info->id, tstrerror(code)); return code; } + taosHashPut(info->metaHashObj, superTable, superTableLen, &pTableMeta, POINTER_BYTES); + tableMetaSml = taosHashIterate(info->superTables, tableMetaSml); } -// } return 0; } -static int32_t arrangePointsByChildTableName(TAOS_SML_DATA_POINT* points, int numPoints, - SHashObj* cname2points, SArray* stableSchemas, SSmlLinesInfo* info) { - for (int32_t i = 0; i < numPoints; ++i) { - TAOS_SML_DATA_POINT * point = points + i; - SSmlSTableSchema* stableSchema = taosArrayGet(stableSchemas, point->schemaIdx); - - for (int j = 0; j < point->tagNum; ++j) { - TAOS_SML_KV* kv = point->tags + j; - if (kv->type == TSDB_DATA_TYPE_TIMESTAMP) { - int64_t ts = *(int64_t*)(kv->value); - ts = convertTimePrecision(ts, TSDB_TIME_PRECISION_NANO, stableSchema->precision); - *(int64_t*)(kv->value) = ts; - } - } - - for (int j = 0; j < point->fieldNum; ++j) { - TAOS_SML_KV* kv = point->fields + j; - if (kv->type == TSDB_DATA_TYPE_TIMESTAMP) { - int64_t ts = *(int64_t*)(kv->value); - ts = convertTimePrecision(ts, TSDB_TIME_PRECISION_NANO, stableSchema->precision); - *(int64_t*)(kv->value) = ts; - } - } - - SArray* cTablePoints = NULL; - SArray** pCTablePoints = taosHashGet(cname2points, point->childTableName, strlen(point->childTableName)); - if (pCTablePoints) { - cTablePoints = *pCTablePoints; - } else { - cTablePoints = taosArrayInit(64, sizeof(point)); - taosHashPut(cname2points, point->childTableName, strlen(point->childTableName), &cTablePoints, POINTER_BYTES); - } - taosArrayPush(cTablePoints, &point); - } - - return 0; -} - -static int32_t applyChildTableDataPointsWithInsertSQL(TAOS* taos, char* cTableName, char* sTableName, SSmlSTableSchema* sTableSchema, - SArray* cTablePoints, size_t rowSize, SSmlLinesInfo* info) { - int32_t code = TSDB_CODE_SUCCESS; - size_t numTags = taosArrayGetSize(sTableSchema->tags); - size_t numCols = taosArrayGetSize(sTableSchema->fields); - size_t rows = taosArrayGetSize(cTablePoints); - SArray* tagsSchema = sTableSchema->tags; - SArray* colsSchema = sTableSchema->fields; - - TAOS_SML_KV* tagKVs[TSDB_MAX_TAGS] = {0}; - for (int i = 0; i < rows; ++i) { - TAOS_SML_DATA_POINT* pDataPoint = taosArrayGetP(cTablePoints, i); - for (int j = 0; j < pDataPoint->tagNum; ++j) { - TAOS_SML_KV* kv = pDataPoint->tags + j; - tagKVs[kv->fieldSchemaIdx] = kv; - } - } - - char* sql = taosMemoryMalloc(tsMaxSQLStringLen + 1); - if (sql == NULL) { - uError("taosMemoryMalloc sql memory error"); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - int32_t freeBytes = tsMaxSQLStringLen + 1; - int32_t totalLen = 0; - totalLen += sprintf(sql, "insert into %s using %s (", cTableName, sTableName); - for (int i = 0; i < numTags; ++i) { - SSchema* tagSchema = taosArrayGet(tagsSchema, i); - totalLen += snprintf(sql + totalLen, freeBytes - totalLen, "%s,", tagSchema->name); - } - --totalLen; - totalLen += snprintf(sql + totalLen, freeBytes - totalLen, ")"); - - totalLen += snprintf(sql + totalLen, freeBytes - totalLen, " tags ("); - - // for (int i = 0; i < numTags; ++i) { - // snprintf(sql+strlen(sql), freeBytes-strlen(sql), "?,"); - // } - for (int i = 0; i < numTags; ++i) { - if (tagKVs[i] == NULL) { - totalLen += snprintf(sql + totalLen, freeBytes - totalLen, "NULL,"); - } else { - TAOS_SML_KV* kv = tagKVs[i]; - size_t beforeLen = totalLen; - int32_t len = 0; - converToStr(sql + beforeLen, kv->type, kv->value, kv->length, &len); - totalLen += len; - totalLen += snprintf(sql + totalLen, freeBytes - totalLen, ","); - } - } - --totalLen; - totalLen += snprintf(sql + totalLen, freeBytes - totalLen, ") ("); - - for (int i = 0; i < numCols; ++i) { - SSchema* colSchema = taosArrayGet(colsSchema, i); - totalLen += snprintf(sql + totalLen, freeBytes - totalLen, "%s,", colSchema->name); - } - --totalLen; - totalLen += snprintf(sql + totalLen, freeBytes - totalLen, ") values "); - - TAOS_SML_KV** colKVs = taosMemoryMalloc(numCols * sizeof(TAOS_SML_KV*)); - for (int r = 0; r < rows; ++r) { - totalLen += snprintf(sql + totalLen, freeBytes - totalLen, "("); - - memset(colKVs, 0, numCols * sizeof(TAOS_SML_KV*)); - - TAOS_SML_DATA_POINT* point = taosArrayGetP(cTablePoints, r); - for (int i = 0; i < point->fieldNum; ++i) { - TAOS_SML_KV* kv = point->fields + i; - colKVs[kv->fieldSchemaIdx] = kv; - } - - for (int i = 0; i < numCols; ++i) { - if (colKVs[i] == NULL) { - totalLen += snprintf(sql + totalLen, freeBytes - totalLen, "NULL,"); - } else { - TAOS_SML_KV* kv = colKVs[i]; - size_t beforeLen = totalLen; - int32_t len = 0; - converToStr(sql + beforeLen, kv->type, kv->value, kv->length, &len); - totalLen += len; - totalLen += snprintf(sql + totalLen, freeBytes - totalLen, ","); - } - } - --totalLen; - totalLen += snprintf(sql + totalLen, freeBytes - totalLen, ")"); - } - taosMemoryFree(colKVs); - sql[totalLen] = '\0'; - - uDebug("SML:0x%" PRIx64 " insert child table table %s of super table %s sql: %s", info->id, cTableName, sTableName, - sql); - - bool tryAgain = false; - int32_t try = 0; - do { - TAOS_RES* res = taos_query(taos, sql); - code = taos_errno(res); - if (code != 0) { - uError("SML:0x%"PRIx64 " taos_query return %d:%s", info->id, code, taos_errstr(res)); - } - - uDebug("SML:0x%"PRIx64 " taos_query inserted %d rows", info->id, taos_affected_rows(res)); - info->affectedRows += taos_affected_rows(res); - taos_free_result(res); - - tryAgain = false; - if ((code == TSDB_CODE_TDB_INVALID_TABLE_ID - || code == TSDB_CODE_VND_INVALID_VGROUP_ID - || code == TSDB_CODE_TDB_TABLE_RECONFIGURE - || code == TSDB_CODE_APP_NOT_READY - || code == TSDB_CODE_RPC_NETWORK_UNAVAIL) && try++ < TSDB_MAX_REPLICA) { - tryAgain = true; - } - - if (code == TSDB_CODE_TDB_INVALID_TABLE_ID || code == TSDB_CODE_VND_INVALID_VGROUP_ID) { - TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); - int32_t code2 = taos_errno(res2); - if (code2 != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " insert child table by sql. reset query cache. error: %s", info->id, taos_errstr(res2)); - } - taos_free_result(res2); - if (tryAgain) { - taosMsleep(100 * (2 << try)); - } - } - - if (code == TSDB_CODE_APP_NOT_READY || code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { - if (tryAgain) { - taosMsleep( 100 * (2 << try)); - } - } - } while (tryAgain); - - taosMemoryFree(sql); - - return code; -} - -static int32_t applyChildTableDataPointsWithStmt(TAOS* taos, char* cTableName, char* sTableName, SSmlSTableSchema* sTableSchema, - SArray* cTablePoints, size_t rowSize, SSmlLinesInfo* info) { - size_t numTags = taosArrayGetSize(sTableSchema->tags); - size_t numCols = taosArrayGetSize(sTableSchema->fields); - size_t rows = taosArrayGetSize(cTablePoints); - - TAOS_SML_KV* tagKVs[TSDB_MAX_TAGS] = {0}; - for (int i= 0; i < rows; ++i) { - TAOS_SML_DATA_POINT * pDataPoint = taosArrayGetP(cTablePoints, i); - for (int j = 0; j < pDataPoint->tagNum; ++j) { - TAOS_SML_KV* kv = pDataPoint->tags + j; - tagKVs[kv->fieldSchemaIdx] = kv; - } - } - - //tag bind - SArray* tagBinds = taosArrayInit(numTags, sizeof(TAOS_BIND)); - taosArraySetSize(tagBinds, numTags); - int isNullColBind = TSDB_TRUE; - for (int j = 0; j < numTags; ++j) { - TAOS_BIND* bind = taosArrayGet(tagBinds, j); - bind->is_null = &isNullColBind; - } - for (int j = 0; j < numTags; ++j) { - if (tagKVs[j] == NULL) continue; - TAOS_SML_KV* kv = tagKVs[j]; - TAOS_BIND* bind = taosArrayGet(tagBinds, kv->fieldSchemaIdx); - bind->buffer_type = kv->type; - bind->length = taosMemoryMalloc(sizeof(uintptr_t*)); - *bind->length = kv->length; - bind->buffer = kv->value; - bind->is_null = NULL; - } - - //rows bind - SArray* rowsBind = taosArrayInit(rows, POINTER_BYTES); - for (int i = 0; i < rows; ++i) { - TAOS_SML_DATA_POINT* point = taosArrayGetP(cTablePoints, i); - - TAOS_BIND* colBinds = calloc(numCols, sizeof(TAOS_BIND)); - if (colBinds == NULL) { - uError("SML:0x%"PRIx64" taos_sml_insert insert points, failed to allocated memory for TAOS_BIND, " - "num of rows: %zu, num of cols: %zu", info->id, rows, numCols); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - for (int j = 0; j < numCols; ++j) { - TAOS_BIND* bind = colBinds + j; - bind->is_null = &isNullColBind; - } - for (int j = 0; j < point->fieldNum; ++j) { - TAOS_SML_KV* kv = point->fields + j; - TAOS_BIND* bind = colBinds + kv->fieldSchemaIdx; - bind->buffer_type = kv->type; - bind->length = taosMemoryMalloc(sizeof(uintptr_t*)); - *bind->length = kv->length; - bind->buffer = kv->value; - bind->is_null = NULL; - } - taosArrayPush(rowsBind, &colBinds); - } - - int32_t code = 0; - code = insertChildTablePointsBatch(taos, cTableName, sTableName, sTableSchema->tags, tagBinds, sTableSchema->fields, rowsBind, rowSize, info); - if (code != 0) { - uError("SML:0x%"PRIx64" insert into child table %s failed. error %s", info->id, cTableName, tstrerror(code)); - } - - //taosMemoryFree rows bind - for (int i = 0; i < rows; ++i) { - TAOS_BIND* colBinds = taosArrayGetP(rowsBind, i); - for (int j = 0; j < numCols; ++j) { - TAOS_BIND* bind = colBinds + j; - taosMemoryFree(bind->length); - } - taosMemoryFree(colBinds); - } - taosArrayDestroy(&rowsBind); - //taosMemoryFree tag bind - for (int i = 0; i < taosArrayGetSize(tagBinds); ++i) { - TAOS_BIND* bind = taosArrayGet(tagBinds, i); - taosMemoryFree(bind->length); - } - taosArrayDestroy(&tagBinds); - return code; -} - -static int32_t insertChildTablePointsBatch(TAOS* taos, char* cTableName, char* sTableName, - SArray* tagsSchema, SArray* tagsBind, - SArray* colsSchema, SArray* rowsBind, - size_t rowSize, SSmlLinesInfo* info) { - size_t numTags = taosArrayGetSize(tagsSchema); - size_t numCols = taosArrayGetSize(colsSchema); - char* sql = taosMemoryMalloc(tsMaxSQLStringLen+1); - if (sql == NULL) { - uError("taosMemoryMalloc sql memory error"); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - int32_t freeBytes = tsMaxSQLStringLen + 1 ; - sprintf(sql, "insert into ? using %s (", sTableName); - for (int i = 0; i < numTags; ++i) { - SSchema* tagSchema = taosArrayGet(tagsSchema, i); - snprintf(sql+strlen(sql), freeBytes-strlen(sql), "%s,", tagSchema->name); - } - snprintf(sql + strlen(sql) - 1, freeBytes-strlen(sql)+1, ")"); - - snprintf(sql + strlen(sql), freeBytes-strlen(sql), " tags ("); - - for (int i = 0; i < numTags; ++i) { - snprintf(sql+strlen(sql), freeBytes-strlen(sql), "?,"); - } - snprintf(sql + strlen(sql) - 1, freeBytes-strlen(sql)+1, ") ("); - - for (int i = 0; i < numCols; ++i) { - SSchema* colSchema = taosArrayGet(colsSchema, i); - snprintf(sql+strlen(sql), freeBytes-strlen(sql), "%s,", colSchema->name); - } - snprintf(sql + strlen(sql)-1, freeBytes-strlen(sql)+1, ") values ("); - - for (int i = 0; i < numCols; ++i) { - snprintf(sql+strlen(sql), freeBytes-strlen(sql), "?,"); - } - snprintf(sql + strlen(sql)-1, freeBytes-strlen(sql)+1, ")"); - sql[strlen(sql)] = '\0'; - - uDebug("SML:0x%"PRIx64" insert child table table %s of super table %s : %s", info->id, cTableName, sTableName, sql); - - size_t maxBatchSize = TSDB_MAX_WAL_SIZE/rowSize * 2 / 3; - size_t rows = taosArrayGetSize(rowsBind); - size_t batchSize = MIN(maxBatchSize, rows); - uDebug("SML:0x%"PRIx64" insert rows into child table %s. num of rows: %zu, batch size: %zu", - info->id, cTableName, rows, batchSize); - SArray* batchBind = taosArrayInit(batchSize, POINTER_BYTES); - int32_t code = TSDB_CODE_SUCCESS; - for (int i = 0; i < rows;) { - int j = i; - for (; j < i + batchSize && j i) { - uDebug("SML:0x%"PRIx64" insert child table batch from line %d to line %d.", info->id, i, j - 1); - code = doInsertChildTablePoints(taos, sql, cTableName, tagsBind, batchBind, info); - if (code != 0) { - taosArrayDestroy(&batchBind); - tfree(sql); - return code; - } - taosArrayClear(batchBind); - } - i = j; - } - taosArrayDestroy(&batchBind); - tfree(sql); - return code; - -} -static int32_t doInsertChildTablePoints(TAOS* taos, char* sql, char* cTableName, SArray* tagsBind, SArray* batchBind, - SSmlLinesInfo* info) { - int32_t code = 0; - - TAOS_STMT* stmt = taos_stmt_init(taos); - if (stmt == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - code = taos_stmt_prepare(stmt, sql, (unsigned long)strlen(sql)); - - if (code != 0) { - uError("SML:0x%"PRIx64" taos_stmt_prepare return %d:%s", info->id, code, taos_stmt_errstr(stmt)); - taos_stmt_close(stmt); - return code; - } - - bool tryAgain = false; - int32_t try = 0; - do { - code = taos_stmt_set_tbname_tags(stmt, cTableName, TARRAY_GET_START(tagsBind)); - if (code != 0) { - uError("SML:0x%"PRIx64" taos_stmt_set_tbname return %d:%s", info->id, code, taos_stmt_errstr(stmt)); - - int affectedRows = taos_stmt_affected_rows(stmt); - info->affectedRows += affectedRows; - - taos_stmt_close(stmt); - return code; - } - - size_t rows = taosArrayGetSize(batchBind); - for (int32_t i = 0; i < rows; ++i) { - TAOS_BIND* colsBinds = taosArrayGetP(batchBind, i); - code = taos_stmt_bind_param(stmt, colsBinds); - if (code != 0) { - uError("SML:0x%"PRIx64" taos_stmt_bind_param return %d:%s", info->id, code, taos_stmt_errstr(stmt)); - - int affectedRows = taos_stmt_affected_rows(stmt); - info->affectedRows += affectedRows; - - taos_stmt_close(stmt); - return code; - } - code = taos_stmt_add_batch(stmt); - if (code != 0) { - uError("SML:0x%"PRIx64" taos_stmt_add_batch return %d:%s", info->id, code, taos_stmt_errstr(stmt)); - - int affectedRows = taos_stmt_affected_rows(stmt); - info->affectedRows += affectedRows; - - taos_stmt_close(stmt); - return code; - } - } - - code = taos_stmt_execute(stmt); - if (code != 0) { - uError("SML:0x%"PRIx64" taos_stmt_execute return %d:%s, try:%d", info->id, code, taos_stmt_errstr(stmt), try); - } - uDebug("SML:0x%"PRIx64" taos_stmt_execute inserted %d rows", info->id, taos_stmt_affected_rows(stmt)); - - tryAgain = false; - if ((code == TSDB_CODE_TDB_INVALID_TABLE_ID - || code == TSDB_CODE_VND_INVALID_VGROUP_ID - || code == TSDB_CODE_TDB_TABLE_RECONFIGURE - || code == TSDB_CODE_APP_NOT_READY - || code == TSDB_CODE_RPC_NETWORK_UNAVAIL) && try++ < TSDB_MAX_REPLICA) { - tryAgain = true; - } - - if (code == TSDB_CODE_TDB_INVALID_TABLE_ID || code == TSDB_CODE_VND_INVALID_VGROUP_ID) { - TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); - int32_t code2 = taos_errno(res2); - if (code2 != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " insert child table. reset query cache. error: %s", info->id, taos_errstr(res2)); - } - taos_free_result(res2); - if (tryAgain) { - taosMsleep(100 * (2 << try)); - } - } - if (code == TSDB_CODE_APP_NOT_READY || code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { - if (tryAgain) { - taosMsleep( 100 * (2 << try)); - } - } - } while (tryAgain); - - int affectedRows = taos_stmt_affected_rows(stmt); - info->affectedRows += affectedRows; - - taos_stmt_close(stmt); - return code; - - return 0; -} - -static int32_t applyChildTableDataPoints(TAOS* taos, char* cTableName, char* sTableName, SSmlSTableSchema* sTableSchema, - SArray* cTablePoints, size_t rowSize, SSmlLinesInfo* info) { - int32_t code = TSDB_CODE_SUCCESS; - size_t childTableDataPoints = taosArrayGetSize(cTablePoints); - if (childTableDataPoints < 10) { - code = applyChildTableDataPointsWithInsertSQL(taos, cTableName, sTableName, sTableSchema, cTablePoints, rowSize, info); - } else { - code = applyChildTableDataPointsWithStmt(taos, cTableName, sTableName, sTableSchema, cTablePoints, rowSize, info); - } - return code; -} - -static int32_t applyDataPoints(TAOS* taos, TAOS_SML_DATA_POINT* points, int32_t numPoints, SArray* stableSchemas, SSmlLinesInfo* info) { +static int32_t applyDataPoints(TAOS* taos, SSmlLinesInfo* info) { int32_t code = TSDB_CODE_SUCCESS; - SHashObj* cname2points = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); - arrangePointsByChildTableName(points, numPoints, cname2points, stableSchemas, info); + TAOS_SML_DATA_POINT_TAGS** oneTable = taosHashIterate(info->childTables, NULL); + while (oneTable) { + TAOS_SML_DATA_POINT_TAGS* tableData = *oneTable; - SArray** pCTablePoints = taosHashIterate(cname2points, NULL); - while (pCTablePoints) { - SArray* cTablePoints = *pCTablePoints; + SName pName = {TSDB_TABLE_NAME_T, info->taos->acctId, {0}, {0}}; + strcpy(pName.dbname, info->pRequest->pDb); + memcpy(pName.tname, tableData->childTableName, strlen(tableData->childTableName)); + SEpSet ep = getEpSet_s(&info->taos->pAppInfo->mgmtEp); + SVgroupInfo vg; + catalogGetTableHashVgroup(info->pCatalog, info->taos->pAppInfo->pTransporter, &ep, &pName, &vg); + taosHashPut(info->pVgHash, (const char*)&vg.vgId, sizeof(vg.vgId), (char*)&vg, sizeof(vg)); - TAOS_SML_DATA_POINT* point = taosArrayGetP(cTablePoints, 0); - SSmlSTableSchema* sTableSchema = taosArrayGet(stableSchemas, point->schemaIdx); + STableMeta** pMeta = taosHashGet(info->metaHashObj, tableData->sTableName, tableData->sTableNameLen); + ASSERT (NULL != pMeta && NULL != *pMeta); + (*pMeta)->vgId = vg.vgId; + (*pMeta)->uid = tableData->uid; - size_t rowSize = 0; - size_t numCols = taosArrayGetSize(sTableSchema->fields); - for (int i = 0; i < numCols; ++i) { - SSchema* colSchema = taosArrayGet(sTableSchema->fields, i); - rowSize += colSchema->bytes; - } + smlBind(info->exec, tableData->tags, tableData->cols, *pMeta, info->msgBuf, info->msgLen); - uDebug("SML:0x%"PRIx64" apply child table points. child table: %s of super table %s, row size: %zu", - info->id, point->childTableName, point->stableName, rowSize); - code = applyChildTableDataPoints(taos, point->childTableName, point->stableName, sTableSchema, cTablePoints, rowSize, info); - if (code != 0) { - uError("SML:0x%"PRIx64" Apply child table points failed. child table %s, error %s", info->id, point->childTableName, tstrerror(code)); - goto cleanup; - } - - uDebug("SML:0x%"PRIx64" successfully applied data points of child table %s", info->id, point->childTableName); - - pCTablePoints = taosHashIterate(cname2points, pCTablePoints); + oneTable = taosHashIterate(info->childTables, oneTable); } -cleanup: - pCTablePoints = taosHashIterate(cname2points, NULL); - while (pCTablePoints) { - SArray* pPoints = *pCTablePoints; - taosArrayDestroy(&pPoints); - pCTablePoints = taosHashIterate(cname2points, pCTablePoints); - } - taosHashCleanup(cname2points); - return code; -} + smlBuildOutput(info->exec, info->pVgHash); + launchQueryImpl(info->pRequest, info->pQuery, TSDB_CODE_SUCCESS, true); + if(info->pRequest->code != TSDB_CODE_SUCCESS){ -static int doSmlInsertOneDataPoint(TAOS* taos, TAOS_SML_DATA_POINT* point, SSmlLinesInfo* info) { - int32_t code = TSDB_CODE_SUCCESS; - - if (!point->childTableName) { - int tableNameLen = TSDB_TABLE_NAME_LEN; - point->childTableName = calloc(1, tableNameLen + 1); - getSmlMd5ChildTableName(point, point->childTableName, &tableNameLen, info); - point->childTableName[tableNameLen] = '\0'; } - STableMeta* tableMeta = NULL; - int32_t ret = getSuperTableMetaFromLocalCache(taos, point->stableName, &tableMeta, info); - if (ret != TSDB_CODE_SUCCESS) { - return ret; - } - uint8_t precision = tableMeta->tableInfo.precision; - taosMemoryFree(tableMeta); - - char* sql = taosMemoryMalloc(TSDB_MAX_SQL_LEN + 1); - int freeBytes = TSDB_MAX_SQL_LEN; - int sqlLen = 0; - sqlLen += snprintf(sql + sqlLen, freeBytes - sqlLen, "insert into %s(", point->childTableName); - for (int col = 0; col < point->fieldNum; ++col) { - TAOS_SML_KV* kv = point->fields + col; - sqlLen += snprintf(sql + sqlLen, freeBytes - sqlLen, "%s,", kv->key); - } - --sqlLen; - sqlLen += snprintf(sql + sqlLen, freeBytes - sqlLen, ") values ("); - TAOS_SML_KV* tsField = point->fields + 0; - int64_t ts = *(int64_t*)(tsField->value); - ts = convertTimePrecision(ts, TSDB_TIME_PRECISION_NANO, precision); - sqlLen += snprintf(sql + sqlLen, freeBytes - sqlLen, "%" PRId64 ",", ts); - for (int col = 1; col < point->fieldNum; ++col) { - TAOS_SML_KV* kv = point->fields + col; - int32_t len = 0; - converToStr(sql + sqlLen, kv->type, kv->value, kv->length, &len); - sqlLen += len; - sqlLen += snprintf(sql + sqlLen, freeBytes - sqlLen, ","); - } - --sqlLen; - sqlLen += snprintf(sql + sqlLen, freeBytes - sqlLen, ")"); - sql[sqlLen] = 0; - - uDebug("SML:0x%" PRIx64 " insert child table table %s of super table %s sql: %s", info->id, - point->childTableName, point->stableName, sql); - TAOS_RES* res = taos_query(taos, sql); - taosMemoryFree(sql); - code = taos_errno(res); - info->affectedRows = taos_affected_rows(res); - taos_free_result(res); - + info->affectedRows = taos_affected_rows(info->pRequest); return code; } @@ -1333,18 +776,12 @@ int tscSmlInsert(TAOS* taos, SSmlLinesInfo* info) { } uDebug("SML:0x%"PRIx64" apply data points", info->id); - code = applyDataPoints(taos, points, numPoint, stableSchemas, info); + code = applyDataPoints(taos, info); if (code != 0) { uError("SML:0x%"PRIx64" error apply data points : %s", info->id, tstrerror(code)); } clean_up: - for (int i = 0; i < taosArrayGetSize(stableSchemas); ++i) { - SSmlSTableSchema* schema = taosArrayGet(stableSchemas, i); - taosArrayDestroy(&schema->fields); - taosArrayDestroy(&schema->tags); - } - taosArrayDestroy(&stableSchemas); return code; } @@ -1398,16 +835,6 @@ static void escapeSpecialCharacter(uint8_t field, const char **pos) { *pos = cur; } -char* addEscapeCharToString(char *str, int32_t len) { - if (str == NULL) { - return NULL; - } - memmove(str + 1, str, len); - str[0] = str[len + 1] = TS_BACKQUOTE_CHAR; - str[len + 2] = '\0'; - return str; -} - bool isValidInteger(char *str) { char *c = str; if (*c != '+' && *c != '-' && !isdigit(*c)) { @@ -1668,67 +1095,6 @@ static bool isNchar(char *pVal, uint16_t len) { return false; } -static int32_t isTimeStamp(char *pVal, uint16_t len, SMLTimeStampType *tsType, SSmlLinesInfo* info) { - if (len == 0) { - return TSDB_CODE_SUCCESS; - } - if ((len == 1) && pVal[0] == '0') { - *tsType = SML_TIME_STAMP_NOW; - return TSDB_CODE_SUCCESS; - } - - for (int i = 0; i < len; ++i) { - if(!isdigit(pVal[i])) { - return TSDB_CODE_TSC_INVALID_TIME_STAMP; - } - } - - /* For InfluxDB line protocol use user passed timestamp precision - * For OpenTSDB protocols only 10 digit(seconds) or 13 digits(milliseconds) - * precision allowed - */ - if (info->protocol == TSDB_SML_LINE_PROTOCOL) { - if (info->tsType != SML_TIME_STAMP_NOT_CONFIGURED) { - *tsType = info->tsType; - } else { - *tsType = SML_TIME_STAMP_NANO_SECONDS; - } - } else if (info->protocol == TSDB_SML_TELNET_PROTOCOL) { - if (len == SML_TIMESTAMP_SECOND_DIGITS) { - *tsType = SML_TIME_STAMP_SECONDS; - } else if (len == SML_TIMESTAMP_MILLI_SECOND_DIGITS) { - *tsType = SML_TIME_STAMP_MILLI_SECONDS; - } else { - return TSDB_CODE_TSC_INVALID_TIME_STAMP; - } - } - return TSDB_CODE_SUCCESS; - - //if (pVal[len - 1] == 's') { - // switch (pVal[len - 2]) { - // case 'm': - // *tsType = SML_TIME_STAMP_MILLI_SECONDS; - // break; - // case 'u': - // *tsType = SML_TIME_STAMP_MICRO_SECONDS; - // break; - // case 'n': - // *tsType = SML_TIME_STAMP_NANO_SECONDS; - // break; - // default: - // if (isdigit(pVal[len - 2])) { - // *tsType = SML_TIME_STAMP_SECONDS; - // break; - // } else { - // return false; - // } - // } - // //printf("Type is timestamp(%s)\n", pVal); - // return true; - //} - //return false; -} - static bool convertStrToNumber(TAOS_SML_KV *pVal, char *str, SSmlLinesInfo* info) { errno = 0; uint8_t type = pVal->type; @@ -1987,107 +1353,7 @@ bool convertSmlValueType(TAOS_SML_KV *pVal, char *value, return false; } -static int32_t getTimeStampValue(char *value, uint16_t len, - SMLTimeStampType type, int64_t *ts, SSmlLinesInfo* info) { - //No appendix or no timestamp given (len = 0) - if (len != 0 && type != SML_TIME_STAMP_NOW) { - *ts = (int64_t)strtoll(value, NULL, 10); - } else { - type = SML_TIME_STAMP_NOW; - } - switch (type) { - case SML_TIME_STAMP_NOW: { - *ts = taosGetTimestampNs(); - break; - } - case SML_TIME_STAMP_HOURS: { - *ts = (int64_t)(*ts * 3600 * 1e9); - break; - } - case SML_TIME_STAMP_MINUTES: { - *ts = (int64_t)(*ts * 60 * 1e9); - break; - } - case SML_TIME_STAMP_SECONDS: { - *ts = (int64_t)(*ts * 1e9); - break; - } - case SML_TIME_STAMP_MILLI_SECONDS: { - *ts = convertTimePrecision(*ts, TSDB_TIME_PRECISION_MILLI, TSDB_TIME_PRECISION_NANO); - break; - } - case SML_TIME_STAMP_MICRO_SECONDS: { - *ts = convertTimePrecision(*ts, TSDB_TIME_PRECISION_MICRO, TSDB_TIME_PRECISION_NANO); - break; - } - case SML_TIME_STAMP_NANO_SECONDS: { - *ts = *ts * 1; - break; - } - default: { - return TSDB_CODE_TSC_INVALID_TIME_STAMP; - } - } - return TSDB_CODE_SUCCESS; -} - -int32_t convertSmlTimeStamp(TAOS_SML_KV *pVal, char *value, - uint16_t len, SSmlLinesInfo* info) { - int32_t ret; - SMLTimeStampType type = SML_TIME_STAMP_NOW; - int64_t tsVal; - - ret = isTimeStamp(value, len, &type, info); - if (ret != TSDB_CODE_SUCCESS) { - return ret; - } - - ret = getTimeStampValue(value, len, type, &tsVal, info); - if (ret != TSDB_CODE_SUCCESS) { - return ret; - } - uDebug("SML:0x%"PRIx64"Timestamp after conversion:%"PRId64, info->id, tsVal); - - pVal->type = TSDB_DATA_TYPE_TIMESTAMP; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - pVal->value = calloc(pVal->length, 1); - memcpy(pVal->value, &tsVal, pVal->length); - return TSDB_CODE_SUCCESS; -} - -static int32_t parseSmlTimeStamp(TAOS_SML_KV **pTS, const char **index, SSmlLinesInfo* info) { - const char *start, *cur; - int32_t ret = TSDB_CODE_SUCCESS; - int len = 0; - char key[] = "ts"; - char *value = NULL; - - start = cur = *index; - *pTS = calloc(1, sizeof(TAOS_SML_KV)); - - while(*cur != '\0') { - cur++; - len++; - } - - if (len > 0) { - value = calloc(len + 1, 1); - memcpy(value, start, len); - } - - ret = convertSmlTimeStamp(*pTS, value, len, info); - if (ret) { - taosMemoryFree(value); - taosMemoryFree(*pTS); - return ret; - } - taosMemoryFree(value); - - (*pTS)->key = calloc(sizeof(key), 1); - memcpy((*pTS)->key, key, sizeof(key)); - return ret; -} bool checkDuplicateKey(char *key, SHashObj *pHash, SSmlLinesInfo* info) { char *val = NULL; @@ -2103,458 +1369,6 @@ bool checkDuplicateKey(char *key, SHashObj *pHash, SSmlLinesInfo* info) { return false; } -static int32_t parseSmlKey(TAOS_SML_KV *pKV, const char **index, SHashObj *pHash, SSmlLinesInfo* info) { - const char *cur = *index; - char key[TSDB_COL_NAME_LEN + 1]; // +1 to avoid key[len] over write - int16_t len = 0; - - while (*cur != '\0') { - if (len > TSDB_COL_NAME_LEN - 1) { - uError("SML:0x%"PRIx64" Key field cannot exceeds %d characters", info->id, TSDB_COL_NAME_LEN - 1); - return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; - } - //unescaped '=' identifies a tag key - if (*cur == '=' && *(cur - 1) != '\\') { - break; - } - //Escape special character - if (*cur == '\\') { - escapeSpecialCharacter(2, &cur); - } - key[len] = *cur; - cur++; - len++; - } - if (len == 0) { - return TSDB_CODE_TSC_LINE_SYNTAX_ERROR; - } - key[len] = '\0'; - - if (checkDuplicateKey(key, pHash, info)) { - return TSDB_CODE_TSC_LINE_SYNTAX_ERROR; - } - - pKV->key = calloc(len + TS_BACKQUOTE_CHAR_SIZE + 1, 1); - memcpy(pKV->key, key, len + 1); - addEscapeCharToString(pKV->key, len); - uDebug("SML:0x%"PRIx64" Key:%s|len:%d", info->id, pKV->key, len); - *index = cur + 1; - return TSDB_CODE_SUCCESS; -} - - -static int32_t parseSmlValue(TAOS_SML_KV *pKV, const char **index, - bool *is_last_kv, SSmlLinesInfo* info, bool isTag) { - const char *start, *cur; - int32_t ret = TSDB_CODE_SUCCESS; - char *value = NULL; - int16_t len = 0; - - bool kv_done = false; - bool back_slash = false; - bool double_quote = false; - size_t line_len = 0; - - enum { - tag_common, - tag_lqoute, - tag_rqoute - } tag_state; - - enum { - val_common, - val_lqoute, - val_rqoute - } val_state; - - start = cur = *index; - tag_state = tag_common; - val_state = val_common; - - while (1) { - if (isTag) { - /* ',', '=' and spaces MUST be escaped */ - switch (tag_state) { - case tag_common: - if (back_slash == true) { - if (*cur != ',' && *cur != '=' && *cur != ' ') { - uError("SML:0x%"PRIx64" tag value: state(%d), incorrect character(%c) escaped", info->id, tag_state, *cur); - ret = TSDB_CODE_TSC_LINE_SYNTAX_ERROR; - goto error; - } - - back_slash = false; - cur++; - len++; - break; - } - - if (*cur == '"') { - if (cur == *index) { - tag_state = tag_lqoute; - } - cur += 1; - len += 1; - break; - } else if (*cur == 'L') { - line_len = strlen(*index); - - /* common character at the end */ - if (cur + 1 >= *index + line_len) { - *is_last_kv = true; - kv_done = true; - break; - } - - if (*(cur + 1) == '"') { - /* string starts here */ - if (cur + 1 == *index + 1) { - tag_state = tag_lqoute; - } - cur += 2; - len += 2; - break; - } - } - - switch (*cur) { - case '\\': - back_slash = true; - cur++; - len++; - break; - case ',': - kv_done = true; - break; - - case ' ': - /* fall through */ - case '\0': - *is_last_kv = true; - kv_done = true; - break; - - default: - cur++; - len++; - } - - break; - case tag_lqoute: - if (back_slash == true) { - if (*cur != ',' && *cur != '=' && *cur != ' ') { - uError("SML:0x%"PRIx64" tag value: state(%d), incorrect character(%c) escaped", info->id, tag_state, *cur); - ret = TSDB_CODE_TSC_LINE_SYNTAX_ERROR; - goto error; - } - - back_slash = false; - cur++; - len++; - break; - } else if (double_quote == true) { - if (*cur != ' ' && *cur != ',' && *cur != '\0') { - uError("SML:0x%"PRIx64" tag value: state(%d), incorrect character(%c) behind closing \"", info->id, tag_state, *cur); - ret = TSDB_CODE_TSC_LINE_SYNTAX_ERROR; - goto error; - } - - if (*cur == ' ' || *cur == '\0') { - *is_last_kv = true; - } - - double_quote = false; - tag_state = tag_rqoute; - break; - } - - switch (*cur) { - case '\\': - back_slash = true; - cur++; - len++; - break; - - case '"': - double_quote = true; - cur++; - len++; - break; - - case ',': - /* fall through */ - case '=': - /* fall through */ - case ' ': - if (*(cur - 1) != '\\') { - uError("SML:0x%"PRIx64" tag value: state(%d), character(%c) not escaped", info->id, tag_state, *cur); - ret = TSDB_CODE_TSC_LINE_SYNTAX_ERROR; - kv_done = true; - } - break; - - case '\0': - uError("SML:0x%"PRIx64" tag value: state(%d), closing \" not found", info->id, tag_state); - ret = TSDB_CODE_TSC_LINE_SYNTAX_ERROR; - kv_done = true; - break; - - default: - cur++; - len++; - } - - break; - - default: - kv_done = true; - } - } else { - switch (val_state) { - case val_common: - if (back_slash == true) { - if (*cur != '\\' && *cur != '"') { - uError("SML:0x%"PRIx64" field value: state(%d), incorrect character(%c) escaped", info->id, val_state, *cur); - ret = TSDB_CODE_TSC_LINE_SYNTAX_ERROR; - goto error; - } - - back_slash = false; - cur++; - len++; - break; - } - - if (*cur == '"') { - if (cur == *index) { - val_state = val_lqoute; - } else { - if (*(cur - 1) != '\\') { - uError("SML:0x%"PRIx64" field value: state(%d), \" not escaped", info->id, val_state); - ret = TSDB_CODE_TSC_LINE_SYNTAX_ERROR; - goto error; - } - } - - cur += 1; - len += 1; - break; - } else if (*cur == 'L') { - line_len = strlen(*index); - - /* common character at the end */ - if (cur + 1 >= *index + line_len) { - *is_last_kv = true; - kv_done = true; - break; - } - - if (*(cur + 1) == '"') { - /* string starts here */ - if (cur + 1 == *index + 1) { - val_state = val_lqoute; - cur += 2; - len += 2; - } else { - /* MUST at the end of string */ - if (cur + 2 >= *index + line_len) { - cur += 2; - len += 2; - *is_last_kv = true; - kv_done = true; - } else { - if (*(cur + 2) != ' ' && *(cur + 2) != ',') { - uError("SML:0x%"PRIx64" field value: state(%d), not closing character(L\")", info->id, val_state); - ret = TSDB_CODE_TSC_LINE_SYNTAX_ERROR; - goto error; - } else { - if (*(cur + 2) == ' ') { - *is_last_kv = true; - } - - cur += 2; - len += 2; - kv_done = true; - } - } - } - break; - } - } - - switch (*cur) { - case '\\': - back_slash = true; - cur++; - len++; - break; - - case ',': - kv_done = true; - break; - - case ' ': - /* fall through */ - case '\0': - *is_last_kv = true; - kv_done = true; - break; - - default: - cur++; - len++; - } - - break; - case val_lqoute: - if (back_slash == true) { - if (*cur != '\\' && *cur != '"') { - uError("SML:0x%"PRIx64" field value: state(%d), incorrect character(%c) escaped", info->id, val_state, *cur); - ret = TSDB_CODE_TSC_LINE_SYNTAX_ERROR; - goto error; - } - - back_slash = false; - cur++; - len++; - break; - } else if (double_quote == true) { - if (*cur != ' ' && *cur != ',' && *cur != '\0') { - uError("SML:0x%"PRIx64" field value: state(%d), incorrect character(%c) behind closing \"", info->id, val_state, *cur); - ret = TSDB_CODE_TSC_LINE_SYNTAX_ERROR; - goto error; - } - - if (*cur == ' ' || *cur == '\0') { - *is_last_kv = true; - } - - double_quote = false; - val_state = val_rqoute; - break; - } - - switch (*cur) { - case '\\': - back_slash = true; - cur++; - len++; - break; - - case '"': - double_quote = true; - cur++; - len++; - break; - - case '\0': - uError("SML:0x%"PRIx64" field value: state(%d), closing \" not found", info->id, val_state); - ret = TSDB_CODE_TSC_LINE_SYNTAX_ERROR; - kv_done = true; - break; - - default: - cur++; - len++; - } - - break; - default: - kv_done = true; - } - } - - if (kv_done == true) { - break; - } - } - - if (len == 0 || ret != TSDB_CODE_SUCCESS) { - taosMemoryFree(pKV->key); - pKV->key = NULL; - return TSDB_CODE_TSC_LINE_SYNTAX_ERROR; - } - - value = calloc(len + 1, 1); - memcpy(value, start, len); - value[len] = '\0'; - if (!convertSmlValueType(pKV, value, len, info, isTag)) { - uError("SML:0x%"PRIx64" Failed to convert sml value string(%s) to any type", - info->id, value); - taosMemoryFree(value); - ret = TSDB_CODE_TSC_INVALID_VALUE; - goto error; - } - taosMemoryFree(value); - - *index = (*cur == '\0') ? cur : cur + 1; - return ret; - -error: - //taosMemoryFree previous alocated key field - taosMemoryFree(pKV->key); - pKV->key = NULL; - return ret; -} - -/* Field Escape charaters - 1: measurement Comma,Space - 2: tag_key, tag_value, field_key Comma,Equal Sign,Space - 3: field_value Double quote,Backslash -*/ - -static int32_t parseSmlMeasurement(TAOS_SML_DATA_POINT *pSml, const char **index, - uint8_t *has_tags, SSmlLinesInfo* info) { - const char *cur = *index; - int16_t len = 0; - - pSml->stableName = calloc(TSDB_TABLE_NAME_LEN + TS_BACKQUOTE_CHAR_SIZE, 1); - if (pSml->stableName == NULL){ - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - while (*cur != '\0') { - if (len > TSDB_TABLE_NAME_LEN - 1) { - uError("SML:0x%"PRIx64" Measurement field cannot exceeds %d characters", info->id, TSDB_TABLE_NAME_LEN - 1); - taosMemoryFree(pSml->stableName); - pSml->stableName = NULL; - return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; - } - //first unescaped comma or space identifies measurement - //if space detected first, meaning no tag in the input - if (*cur == ',' && *(cur - 1) != '\\') { - *has_tags = 1; - break; - } - if (*cur == ' ' && *(cur - 1) != '\\') { - if (*(cur + 1) != ' ') { - break; - } - else { - cur++; - continue; - } - } - //Comma, Space, Backslash needs to be escaped if any - if (*cur == '\\') { - escapeSpecialCharacter(1, &cur); - } - pSml->stableName[len] = *cur; - cur++; - len++; - } - if (len == 0) { - taosMemoryFree(pSml->stableName); - pSml->stableName = NULL; - return TSDB_CODE_TSC_LINE_SYNTAX_ERROR; - } - addEscapeCharToString(pSml->stableName, len); - *index = cur + 1; - uDebug("SML:0x%"PRIx64" Stable name in measurement:%s|len:%d", info->id, pSml->stableName, len); - - return TSDB_CODE_SUCCESS; -} - //Table name can only contain digits(0-9),alphebet(a-z),underscore(_) int32_t isValidChildTableName(const char *pTbName, int16_t len, SSmlLinesInfo* info) { if (len > TSDB_TABLE_NAME_LEN - 1) { @@ -2570,147 +1384,6 @@ int32_t isValidChildTableName(const char *pTbName, int16_t len, SSmlLinesInfo* i return TSDB_CODE_SUCCESS; } - -static int32_t parseSmlKvPairs(TAOS_SML_KV **pKVs, int *num_kvs, - const char **index, bool isField, - TAOS_SML_DATA_POINT* smlData, SHashObj *pHash, - SSmlLinesInfo* info) { - const char *cur = *index; - int32_t ret = TSDB_CODE_SUCCESS; - TAOS_SML_KV *pkv; - bool is_last_kv = false; - - int32_t capacity = 0; - if (isField) { - capacity = 64; - *pKVs = calloc(capacity, sizeof(TAOS_SML_KV)); - // leave space for timestamp; - pkv = *pKVs; - pkv++; - } else { - capacity = 8; - *pKVs = calloc(capacity, sizeof(TAOS_SML_KV)); - pkv = *pKVs; - } - - size_t childTableNameLen = strlen(tsSmlChildTableName); - char childTableName[TSDB_TABLE_NAME_LEN + TS_BACKQUOTE_CHAR_SIZE] = {0}; - if (childTableNameLen != 0) { - memcpy(childTableName, tsSmlChildTableName, childTableNameLen); - addEscapeCharToString(childTableName, (int32_t)(childTableNameLen)); - } - - while (*cur != '\0') { - ret = parseSmlKey(pkv, &cur, pHash, info); - if (ret) { - uError("SML:0x%"PRIx64" Unable to parse key", info->id); - goto error; - } - ret = parseSmlValue(pkv, &cur, &is_last_kv, info, !isField); - if (ret) { - uError("SML:0x%"PRIx64" Unable to parse value", info->id); - goto error; - } - - if (!isField && childTableNameLen != 0 && strcasecmp(pkv->key, childTableName) == 0) { - smlData->childTableName = taosMemoryMalloc(pkv->length + TS_BACKQUOTE_CHAR_SIZE + 1); - memcpy(smlData->childTableName, pkv->value, pkv->length); - addEscapeCharToString(smlData->childTableName, (int32_t)pkv->length); - taosMemoryFree(pkv->key); - taosMemoryFree(pkv->value); - } else { - *num_kvs += 1; - } - if (is_last_kv) { - goto done; - } - - //reallocate addtional memory for more kvs - TAOS_SML_KV *more_kvs = NULL; - - if (isField) { - if ((*num_kvs + 2) > capacity) { - capacity *= 3; capacity /= 2; - more_kvs = realloc(*pKVs, capacity * sizeof(TAOS_SML_KV)); - } else { - more_kvs = *pKVs; - } - } else { - if ((*num_kvs + 1) > capacity) { - capacity *= 3; capacity /= 2; - more_kvs = realloc(*pKVs, capacity * sizeof(TAOS_SML_KV)); - } else { - more_kvs = *pKVs; - } - } - - if (!more_kvs) { - goto error; - } - *pKVs = more_kvs; - //move pKV points to next TAOS_SML_KV block - if (isField) { - pkv = *pKVs + *num_kvs + 1; - } else { - pkv = *pKVs + *num_kvs; - } - } - goto done; - -error: - return ret; -done: - *index = cur; - return ret; -} - -static void moveTimeStampToFirstKv(TAOS_SML_DATA_POINT** smlData, TAOS_SML_KV *ts) { - TAOS_SML_KV* tsField = (*smlData)->fields; - tsField->length = ts->length; - tsField->type = ts->type; - tsField->value = taosMemoryMalloc(ts->length); - tsField->key = taosMemoryMalloc(strlen(ts->key) + 1); - memcpy(tsField->key, ts->key, strlen(ts->key) + 1); - memcpy(tsField->value, ts->value, ts->length); - (*smlData)->fieldNum = (*smlData)->fieldNum + 1; - - taosMemoryFree(ts->key); - taosMemoryFree(ts->value); - taosMemoryFree(ts); -} - -/* Field Escape charaters - 1: measurement Comma,Space - 2: tag_key, tag_value, field_key Comma,Equal Sign,Space - 3: field_value Double quote,Backslash -*/ - -//void findSpace(const char** sql, const char **tags, int32_t *tagLen){ -// const char *cur = *sql; -// *tagLen = 0; -// *tags = NULL; -// if(!cur) return; -// while (*cur != '\0') { // jump the space at the begining -// if(*cur != SPACE) { -// *tags = cur; -// break; -// } -// cur++; -// } -// -// while (*cur != '\0') { // find the first space -// if (*cur == SPACE && *(cur - 1) != SLASH) { -// *tagLen = cur - *tags; -// break; -// } -// -// cur++; -// } -// *sql = cur; -// return; -//} - - int32_t parseSml(const char* sql, TAOS_PARSE_ELEMENTS *elements){ if(!sql) return TSDB_CODE_SML_INVALID_DATA; while (*sql != '\0') { // jump the space at the begining @@ -2757,14 +1430,13 @@ int32_t parseSml(const char* sql, TAOS_PARSE_ELEMENTS *elements){ while (*sql != '\0') { if(*sql == SPACE && *(sql - 1) != SLASH) { - elements->colsLen = sql - elements->cols; break; } sql++; } - if(elements->colsLen == 0) return TSDB_CODE_SML_INVALID_DATA; + elements->colsLen = sql - elements->cols; - // parse ts + // parse ts,ts can be empty while (*sql != '\0') { if(*sql != SPACE) { elements->timestamp = sql; @@ -2772,12 +1444,11 @@ int32_t parseSml(const char* sql, TAOS_PARSE_ELEMENTS *elements){ } sql++; } - if(!elements->timestamp) return TSDB_CODE_SML_INVALID_DATA; return TSDB_CODE_SUCCESS; } -int32_t parseSmlKV(const char* data, int32_t len, SArray *tags){ +int32_t parseSmlKV(const char* data, int32_t len, SArray *cols, bool isTag){ for(int i = 0; i < len; i++){ const char *key = data + i; int32_t keyLen = 0; @@ -2805,24 +1476,112 @@ int32_t parseSmlKV(const char* data, int32_t len, SArray *tags){ if(valueLen == 0){ return TSDB_CODE_SML_INVALID_DATA; } - TAOS_SML_KV *kv = taosMemoryCalloc(sizeof(TAOS_SML_KV), 1); + SSmlKv *kv = taosMemoryCalloc(sizeof(SSmlKv), 1); kv->key = key; kv->keyLen = keyLen; kv->value = value; kv->valueLen = valueLen; - kv->type = TSDB_DATA_TYPE_NCHAR; - if(tags) taosArrayPush(tags, &kv); + if(isTag){ + kv->type = TSDB_DATA_TYPE_NCHAR; + } + if(cols) taosArrayPush(cols, &kv); } return TSDB_CODE_SUCCESS; } -int32_t parseSmlTS(const char* data, int32_t len, SArray *tags){ - TAOS_SML_KV *kv = taosMemoryCalloc(sizeof(TAOS_SML_KV), 1); - kv->value = data; - kv->valueLen = len; +static int64_t getTimeStampValue(const char *value, int32_t type) { + double ts = (double)strtoll(value, NULL, 10); + switch (type) { + case TSDB_TIME_PRECISION_HOURS: + ts *= (3600 * 1e9); + case TSDB_TIME_PRECISION_MINUTES: + ts *= (60 * 1e9); + case TSDB_TIME_PRECISION_SECONDS: + ts *= (1e9); + case TSDB_TIME_PRECISION_MICRO: + ts *= (1e6); + case TSDB_TIME_PRECISION_MILLI: + ts *= (1e3); + default: + break; + } + if(ts > (double)INT64_MAX || ts < 0){ + return -1; + }else{ + return (int64_t)ts; + } +} + +static int64_t getTimeStampNow(int32_t precision) { + switch (precision) { + case TSDB_TIME_PRECISION_HOURS: + return taosGetTimestampMs()/1000/3600; + case TSDB_TIME_PRECISION_MINUTES: + return taosGetTimestampMs()/1000/60; + + case TSDB_TIME_PRECISION_SECONDS: + return taosGetTimestampMs()/1000; + default: + return taosGetTimestamp(precision); + } +} + +static int32_t isValidateTimeStamp(const char *pVal, int32_t len) { + for (int i = 0; i < len; ++i) { + if (!isdigit(pVal[i])) { + return TSDB_CODE_TSC_INVALID_TIME_STAMP; + } + } + return TSDB_CODE_SUCCESS; +} + +static int32_t getTsType(int32_t len) { + if (len == TSDB_TIME_PRECISION_SEC_DIGITS) { + return TSDB_TIME_PRECISION_SECONDS; + } else if (len == TSDB_TIME_PRECISION_MILLI_DIGITS) { + return TSDB_TIME_PRECISION_MILLI_DIGITS; + } else { + return TSDB_CODE_TSC_INVALID_TIME_STAMP; + } +} + +static int32_t parseSmlTS(const char* data, SArray *tags, int8_t tsType, SMLProtocolType protocolType){ + int64_t *ts = taosMemoryCalloc(1, sizeof(int64_t)); + if(data == NULL){ + if(protocolType == TSDB_SML_LINE_PROTOCOL){ + *ts = getTimeStampNow(tsType); + }else{ + goto cleanup; + } + }else{ + int32_t len = strlen(data); + int ret = isValidateTimeStamp(data, len); + if(!ret){ + goto cleanup; + } + if(protocolType != TSDB_SML_LINE_PROTOCOL){ + tsType = getTsType(len); + if (tsType == TSDB_CODE_TSC_INVALID_TIME_STAMP) { + goto cleanup; + } + } + *ts = getTimeStampValue(data, tsType); + if(*ts == -1){ + goto cleanup; + } + } + + SSmlKv *kv = taosMemoryCalloc(sizeof(SSmlKv), 1); + kv->value = (const char*)ts; + kv->valueLen = sizeof(int64_t); kv->type = TSDB_DATA_TYPE_TIMESTAMP; + kv->length = (int16_t)tDataTypes[kv->type].bytes; if(tags) taosArrayPush(tags, &kv); return TSDB_CODE_SUCCESS; + +cleanup: + taosMemoryFree(ts); + return TSDB_CODE_TSC_INVALID_TIME_STAMP; } //int32_t parseSmlCols(const char* data, SArray *cols){ @@ -2871,8 +1630,8 @@ int32_t parseSmlTS(const char* data, int32_t len, SArray *tags){ void updateMeta(SSmlSTableMeta* tableMeta, SArray *tags, SArray *cols){ if(tags){ for (int i = 0; i < taosArrayGetSize(tags); ++i) { - TAOS_SML_KV *kv = taosArrayGetP(tags, i); - TAOS_SML_KV **value = taosHashGet(tableMeta->tagHash, kv->key, kv->keyLen); + SSmlKv *kv = taosArrayGetP(tags, i); + SSmlKv **value = taosHashGet(tableMeta->tagHash, kv->key, kv->keyLen); if(value){ if(kv->type != (*value)->type){ // todo @@ -2886,8 +1645,8 @@ void updateMeta(SSmlSTableMeta* tableMeta, SArray *tags, SArray *cols){ if(cols){ for (int i = 1; i < taosArrayGetSize(cols); ++i) { //jump timestamp - TAOS_SML_KV *kv = taosArrayGetP(cols, i); - TAOS_SML_KV **value = taosHashGet(tableMeta->fieldHash, kv->key, kv->keyLen); + SSmlKv *kv = taosArrayGetP(cols, i); + SSmlKv **value = taosHashGet(tableMeta->fieldHash, kv->key, kv->keyLen); if(value){ if(kv->type != (*value)->type){ // todo @@ -2902,20 +1661,20 @@ void updateMeta(SSmlSTableMeta* tableMeta, SArray *tags, SArray *cols){ void insertMeta(SSmlSTableMeta* tableMeta, SArray *tags, SArray *cols){ if(tags){ for (int i = 0; i < taosArrayGetSize(tags); ++i) { - TAOS_SML_KV *kv = taosArrayGetP(tags, i); + SSmlKv *kv = taosArrayGetP(tags, i); taosHashPut(tableMeta->tagHash, kv->key, kv->keyLen, &kv, POINTER_BYTES); } } if(cols){ for (int i = 0; i < taosArrayGetSize(cols); ++i) { - TAOS_SML_KV *kv = taosArrayGetP(cols, i); + SSmlKv *kv = taosArrayGetP(cols, i); taosHashPut(tableMeta->fieldHash, kv->key, kv->keyLen, &kv, POINTER_BYTES); } } } -int32_t tscParseLine(const char* sql, SSmlLinesInfo* info) { +static int32_t smlParseLine(const char* sql, SSmlLinesInfo* info) { TAOS_PARSE_ELEMENTS elements = {0}; int ret = parseSml(sql, &elements); if(ret != TSDB_CODE_SUCCESS){ @@ -2927,8 +1686,9 @@ int32_t tscParseLine(const char* sql, SSmlLinesInfo* info) { uError("SML:0x%"PRIx64" taos_insert_lines failed to allocate memory", info->id); return TSDB_CODE_TSC_OUT_OF_MEMORY; } - parseSmlTS(elements.timestamp, strlen(elements.timestamp), cols); - ret = parseSmlKV(elements.cols, elements.colsLen, cols); + + parseSmlTS(elements.timestamp, cols, info->tsType); + ret = parseSmlCols(elements.cols, elements.colsLen, cols, false); if(ret != TSDB_CODE_SUCCESS){ return ret; } @@ -2954,7 +1714,7 @@ int32_t tscParseLine(const char* sql, SSmlLinesInfo* info) { uError("SML:0x%"PRIx64" taos_insert_lines failed to allocate memory", info->id); return TSDB_CODE_TSC_OUT_OF_MEMORY; } - ret = parseSmlKV(elements.tags, elements.tagsLen, tag->tags); + ret = parseSmlTags(elements.tags, elements.tagsLen, tag->tags); if(ret != TSDB_CODE_SUCCESS){ return ret; } @@ -2973,112 +1733,115 @@ int32_t tscParseLine(const char* sql, SSmlLinesInfo* info) { return TSDB_CODE_SUCCESS; } - -int32_t tscParseLines(char* lines[], int numLines, SSmlLinesInfo* info) { - for (int32_t i = 0; i < numLines; ++i) { - int32_t code = tscParseLine(lines[i], info); - if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%"PRIx64" data point line parse failed. line %d : %s", info->id, i, lines[i]); - return code; - } - } - uDebug("SML:0x%"PRIx64" data point line parse success. tables %d", info->id, taosHashGetSize(info->childTables)); - - return TSDB_CODE_SUCCESS; +static void smlDestroyInfo(SSmlLinesInfo* info){ + if(!info) return; + qDestroyQuery(info->pQuery); + tscSmlDestroyHandle(info->exec); + taosHashCleanup(info->childTables); + taosHashCleanup(info->superTables); + taosHashCleanup(info->metaHashObj); + taosHashCleanup(info->pVgHash); + taosMemoryFree(info); } - -int taos_insert_lines(TAOS* taos, char* lines[], int numLines, SMLProtocolType protocol, SMLTimeStampType tsType, int *affectedRows) { - int32_t code = 0; - +static SSmlLinesInfo* smlBuildInfo(TAOS* taos, SRequestObj* request, SMLProtocolType protocol, int32_t tsType){ SSmlLinesInfo* info = taosMemoryMalloc(sizeof(SSmlLinesInfo)); + if (NULL == info) { + return NULL; + } info->id = genLinesSmlId(); info->tsType = tsType; - info->taos = (STscObj*)taos; + info->taos = taos; info->protocol = protocol; + info->pQuery = taosMemoryCalloc(1, sizeof(SQuery)); + if (NULL == info->pQuery) { + goto cleanup; + } + info->pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE; + info->pQuery->haveResultSet = false; + info->pQuery->msgType = TDMT_VND_SUBMIT; + info->pQuery->pRoot = (SNode*)nodesMakeNode(QUERY_NODE_VNODE_MODIF_STMT); + ((SVnodeModifOpStmt*)(info->pQuery->pRoot))->payloadType = PAYLOAD_TYPE_KV; + + info->exec = tscSmlInitHandle(info->pQuery); + + int32_t code = catalogGetHandle(info->taos->pAppInfo->clusterId, &info->pCatalog); + if(code != TSDB_CODE_SUCCESS){ + uError("SML:0x%"PRIx64" get catalog error %d", info->id, code); + goto cleanup; + } + info->pRequest = request; + info->msgBuf = info->pRequest->msgBuf; + info->msgLen = ERROR_MSG_BUF_DEFAULT_SIZE; + + + info->childTables = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, false); + info->superTables = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, false); + info->metaHashObj = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), true, false); + info->pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, false); + + return info; + +cleanup: + smlDestroyInfo(info); + return NULL; +} + +int sml_insert_lines(TAOS* taos, SRequestObj* request, char* lines[], int numLines, SMLProtocolType protocol, int32_t tsType) { + int32_t code = TSDB_CODE_SUCCESS; + + SSmlLinesInfo* info = smlBuildInfo(taos, request, protocol, tsType); + if(!info){ + code = TSDB_CODE_OUT_OF_MEMORY; + goto cleanup; + } if (numLines <= 0 || numLines > 65536) { uError("SML:0x%"PRIx64" taos_insert_lines numLines should be between 1 and 65536. numLines: %d", info->id, numLines); code = TSDB_CODE_TSC_APP_ERROR; goto cleanup; } - - info->childTables = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, false); - info->superTables = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, false); - - uDebug("SML:0x%"PRIx64" taos_insert_lines begin inserting %d lines, first line: %s", info->id, numLines, lines[0]); - code = tscParseLines(lines, numLines, info); - - if (code != 0) { - goto cleanup; + for (int32_t i = 0; i < numLines; ++i) { + code = smlParseLine(lines[i], info); + if (code != TSDB_CODE_SUCCESS) { + uError("SML:0x%"PRIx64" data point line parse failed. line %d : %s", info->id, i, lines[i]); + goto cleanup; + } } + uDebug("SML:0x%"PRIx64" data point line parse success. tables %d", info->id, taosHashGetSize(info->childTables)); - code = tscSmlInsert(taos, info); - if (code != 0) { + code = smlInsert(taos, info); + if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%"PRIx64" taos_sml_insert error: %s", info->id, tstrerror((code))); goto cleanup; } - if (affectedRows != NULL) { - *affectedRows = info->affectedRows; - } uDebug("SML:0x%"PRIx64" taos_insert_lines finish inserting %d lines. code: %d", info->id, numLines, code); cleanup: - taosMemoryFree(info); + smlDestroyInfo(info); return code; } -static int32_t convertPrecisionType(int precision, SMLTimeStampType *tsType) { +static int32_t convertPrecisionType(int precision) { switch (precision) { - case TSDB_SML_TIMESTAMP_NOT_CONFIGURED: - *tsType = SML_TIME_STAMP_NOT_CONFIGURED; - break; case TSDB_SML_TIMESTAMP_HOURS: - *tsType = SML_TIME_STAMP_HOURS; - break; + return TSDB_TIME_PRECISION_HOURS; case TSDB_SML_TIMESTAMP_MILLI_SECONDS: - *tsType = SML_TIME_STAMP_MILLI_SECONDS; - break; + return TSDB_TIME_PRECISION_MILLI; case TSDB_SML_TIMESTAMP_NANO_SECONDS: - *tsType = SML_TIME_STAMP_NANO_SECONDS; - break; + case TSDB_SML_TIMESTAMP_NOT_CONFIGURED: + return TSDB_TIME_PRECISION_NANO; case TSDB_SML_TIMESTAMP_MICRO_SECONDS: - *tsType = SML_TIME_STAMP_MICRO_SECONDS; - break; + return TSDB_TIME_PRECISION_MICRO; case TSDB_SML_TIMESTAMP_SECONDS: - *tsType = SML_TIME_STAMP_SECONDS; - break; + return TSDB_TIME_PRECISION_SECONDS; case TSDB_SML_TIMESTAMP_MINUTES: - *tsType = SML_TIME_STAMP_MINUTES; - break; + return TSDB_TIME_PRECISION_MINUTES; default: - return TSDB_CODE_SML_INVALID_PRECISION_TYPE; + return -1; } - - return TSDB_CODE_SUCCESS; } -//make a dummy SSqlObj -static SSqlObj* createSmlQueryObj(TAOS* taos, int32_t affected_rows, int32_t code) { - SSqlObj *pNew = (SSqlObj*)calloc(1, sizeof(SSqlObj)); - if (pNew == NULL) { - return NULL; - } - pNew->signature = pNew; - pNew->pTscObj = taos; - pNew->fp = NULL; - - tsem_init(&pNew->rspSem, 0, 0); - registerSqlObj(pNew); - - pNew->res.numOfRows = affected_rows; - pNew->res.code = code; - - - return pNew; -} - - /** * taos_schemaless_insert() parse and insert data points into database according to * different protocol. @@ -3102,19 +1865,17 @@ static SSqlObj* createSmlQueryObj(TAOS* taos, int32_t affected_rows, int32_t cod TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int protocol, int precision) { int code = TSDB_CODE_SUCCESS; - int affected_rows = 0; - SMLTimeStampType tsType = SML_TIME_STAMP_NOW; - - if (protocol == TSDB_SML_LINE_PROTOCOL) { - code = convertPrecisionType(precision, &tsType); - if (code != TSDB_CODE_SUCCESS) { - return NULL; - } - } + SRequestObj* request = createRequest(taos, NULL, NULL, TSDB_SQL_INSERT); switch (protocol) { case TSDB_SML_LINE_PROTOCOL: - code = taos_insert_lines(taos, lines, numLines, protocol, tsType, &affected_rows); + int32_t tsType = convertPrecisionType(precision); + if(tsType == -1){ + request->code = TSDB_CODE_SML_INVALID_PRECISION_TYPE; + goto end; + } + + code = sml_insert_lines(taos, request, lines, numLines, protocol, tsType); break; case TSDB_SML_TELNET_PROTOCOL: //code = taos_insert_telnet_lines(taos, lines, numLines, protocol, tsType, &affected_rows); @@ -3127,8 +1888,6 @@ TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int pr break; } - - SSqlObj *pSql = createSmlQueryObj(taos, affected_rows, code); - - return (TAOS_RES*)pSql; +end: + return (TAOS_RES*)request; } diff --git a/source/client/src/clientStmt.c b/source/client/src/clientStmt.c index 0972ff3477..e3480b2490 100644 --- a/source/client/src/clientStmt.c +++ b/source/client/src/clientStmt.c @@ -364,7 +364,7 @@ int stmtSetTbTags(TAOS_STMT *stmt, TAOS_BIND_v2 *tags) { STMT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); } - STMT_ERR_RET(qBindStmtTagsValue(*pDataBlock, pStmt->bInfo.boundTags, pStmt->bInfo.tbSuid, &pStmt->bInfo.sname, tags, pStmt->exec.pRequest->msgBuf, pStmt->exec.pRequest->msgBufLen)); + STMT_ERR_RET(qBindStmtTagsValue(*pDataBlock, pStmt->bInfo.boundTags, pStmt->bInfo.tbSuid, pStmt->bInfo.sname.tname, tags, pStmt->exec.pRequest->msgBuf, pStmt->exec.pRequest->msgBufLen)); return TSDB_CODE_SUCCESS; } diff --git a/source/libs/parser/inc/parInsertData.h b/source/libs/parser/inc/parInsertData.h index ed7267655c..046e9ef0f8 100644 --- a/source/libs/parser/inc/parInsertData.h +++ b/source/libs/parser/inc/parInsertData.h @@ -136,7 +136,7 @@ void destroyBlockHashmap(SHashObj* pDataBlockHash); int initRowBuilder(SRowBuilder *pBuilder, int16_t schemaVer, SParsedDataColInfo *pColInfo); int32_t allocateMemIfNeed(STableDataBlocks *pDataBlock, int32_t rowSize, int32_t * numOfRows); int32_t getDataBlockFromList(SHashObj* pHashList, int64_t id, int32_t size, int32_t startOffset, int32_t rowSize, - const STableMeta* pTableMeta, STableDataBlocks** dataBlocks, SArray* pBlockList, SVCreateTbReq* pCreateTbReq); + STableMeta* pTableMeta, STableDataBlocks** dataBlocks, SArray* pBlockList, SVCreateTbReq* pCreateTbReq); int32_t mergeTableDataBlocks(SHashObj* pHashObj, uint8_t payloadType, SArray** pVgDataBlocks); int32_t buildCreateTbMsg(STableDataBlocks* pBlocks, SVCreateTbReq* pCreateTbReq); int32_t allocateMemForSize(STableDataBlocks *pDataBlock, int32_t allSize); diff --git a/source/libs/parser/inc/parUtil.h b/source/libs/parser/inc/parUtil.h index 460a089802..365d317fdb 100644 --- a/source/libs/parser/inc/parUtil.h +++ b/source/libs/parser/inc/parUtil.h @@ -41,7 +41,6 @@ int32_t generateSyntaxErrMsg(SMsgBuf* pBuf, int32_t errCode, ...); int32_t buildInvalidOperationMsg(SMsgBuf* pMsgBuf, const char* msg); int32_t buildSyntaxErrMsg(SMsgBuf* pBuf, const char* additionalInfo, const char* sourceStr); -STableMeta* tableMetaDup(const STableMeta* pTableMeta); SSchema *getTableColumnSchema(const STableMeta *pTableMeta); SSchema *getTableTagSchema(const STableMeta* pTableMeta); int32_t getNumOfColumns(const STableMeta* pTableMeta); diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index f21a738032..0e87d8611e 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -763,11 +763,9 @@ static int32_t KvRowAppend(SMsgBuf* pMsgBuf, const void *value, int32_t len, voi return TSDB_CODE_SUCCESS; } -static int32_t buildCreateTbReq(SVCreateTbReq *pTbReq, const SName* pName, SKVRow row, int64_t suid) { - char dbFName[TSDB_DB_FNAME_LEN] = {0}; - tNameGetFullDbName(pName, dbFName); +static int32_t buildCreateTbReq(SVCreateTbReq *pTbReq, const char* tname, SKVRow row, int64_t suid) { pTbReq->type = TD_CHILD_TABLE; - pTbReq->name = strdup(pName->tname); + pTbReq->name = strdup(tname); pTbReq->ctbCfg.suid = suid; pTbReq->ctbCfg.pTag = row; @@ -775,7 +773,7 @@ static int32_t buildCreateTbReq(SVCreateTbReq *pTbReq, const SName* pName, SKVRo } // pSql -> tag1_value, ...) -static int32_t parseTagsClause(SInsertParseContext* pCxt, SSchema* pSchema, uint8_t precision, const SName* pName) { +static int32_t parseTagsClause(SInsertParseContext* pCxt, SSchema* pSchema, uint8_t precision, const char* tName) { if (tdInitKVRowBuilder(&pCxt->tagsBuilder) < 0) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } @@ -816,7 +814,7 @@ static int32_t parseTagsClause(SInsertParseContext* pCxt, SSchema* pSchema, uint } tdSortKVRowByColIdx(row); - return buildCreateTbReq(&pCxt->createTblReq, pName, row, pCxt->pTableMeta->suid); + return buildCreateTbReq(&pCxt->createTblReq, tName, row, pCxt->pTableMeta->suid); } static int32_t cloneTableMeta(STableMeta* pSrc, STableMeta** pDst) { @@ -876,7 +874,7 @@ static int32_t parseUsingClause(SInsertParseContext* pCxt, SToken* pTbnameToken) if (TK_NK_LP != sToken.type) { return buildSyntaxErrMsg(&pCxt->msg, "( is expected", sToken.z); } - CHECK_CODE(parseTagsClause(pCxt, pCxt->pTableMeta->schema, getTableInfo(pCxt->pTableMeta).precision, &name)); + CHECK_CODE(parseTagsClause(pCxt, pCxt->pTableMeta->schema, getTableInfo(pCxt->pTableMeta).precision, name.tname)); NEXT_TOKEN(pCxt->pSql, sToken); if (TK_NK_RP != sToken.type) { return buildSyntaxErrMsg(&pCxt->msg, ") is expected", sToken.z); @@ -1024,11 +1022,6 @@ static void destroyDataBlock(STableDataBlocks* pDataBlock) { taosMemoryFreeClear(pDataBlock->pData); if (!pDataBlock->cloned) { - // free the refcount for metermeta - if (pDataBlock->pTableMeta != NULL) { - taosMemoryFreeClear(pDataBlock->pTableMeta); - } - destroyBoundColumnInfo(&pDataBlock->boundColumnInfo); } taosMemoryFreeClear(pDataBlock); @@ -1256,7 +1249,7 @@ int32_t qBuildStmtOutput(SQuery* pQuery, SHashObj* pVgHash, SHashObj* pBlockHash return TSDB_CODE_SUCCESS; } -int32_t qBindStmtTagsValue(void *pBlock, void *boundTags, int64_t suid, SName *pName, TAOS_BIND_v2 *bind, char *msgBuf, int32_t msgBufLen){ +int32_t qBindStmtTagsValue(void *pBlock, void *boundTags, int64_t suid, char *tName, TAOS_BIND_v2 *bind, char *msgBuf, int32_t msgBufLen){ STableDataBlocks *pDataBlock = (STableDataBlocks *)pBlock; SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; SParsedDataColInfo* tags = (SParsedDataColInfo*)boundTags; @@ -1297,7 +1290,7 @@ int32_t qBindStmtTagsValue(void *pBlock, void *boundTags, int64_t suid, SName *p tdSortKVRowByColIdx(row); SVCreateTbReq tbReq = {0}; - CHECK_CODE(buildCreateTbReq(&tbReq, pName, row, suid)); + CHECK_CODE(buildCreateTbReq(&tbReq, tName, row, suid)); CHECK_CODE(buildCreateTbMsg(pDataBlock, &tbReq)); destroyCreateSubTbReq(&tbReq); @@ -1522,4 +1515,211 @@ int32_t qBuildStmtColFields(void *pBlock, int32_t *fieldNum, TAOS_FIELD** fields } +typedef struct SmlExecHandle { + SHashObj* pBlockHash; + + SParsedDataColInfo tags; // each table + SKVRowBuilder tagsBuilder; // each table + SVCreateTbReq createTblReq; // each table + + SQuery* pQuery; +} SmlExecHandle; + +static int32_t smlBoundColumns(SArray *cols, SParsedDataColInfo* pColList, SSchema* pSchema) { + col_id_t nCols = pColList->numOfCols; + + pColList->numOfBound = 0; + pColList->boundNullLen = 0; + memset(pColList->boundColumns, 0, sizeof(col_id_t) * nCols); + for (col_id_t i = 0; i < nCols; ++i) { + pColList->cols[i].valStat = VAL_STAT_NONE; + } + + bool isOrdered = true; + col_id_t lastColIdx = -1; // last column found + for (int i = 0; i < taosArrayGetSize(cols); ++i) { + SSmlKv *kv = taosArrayGetP(cols, i); + SToken sToken = {.n=kv->keyLen, .z=kv->key}; + col_id_t t = lastColIdx + 1; + col_id_t index = findCol(&sToken, t, nCols, pSchema); + if (index < 0 && t > 0) { + index = findCol(&sToken, 0, t, pSchema); + isOrdered = false; + } + if (index < 0) { + return TSDB_CODE_SML_INVALID_DATA; + } + if (pColList->cols[index].valStat == VAL_STAT_HAS) { + return TSDB_CODE_SML_INVALID_DATA; + } + lastColIdx = index; + pColList->cols[index].valStat = VAL_STAT_HAS; + pColList->boundColumns[pColList->numOfBound] = index + PRIMARYKEY_TIMESTAMP_COL_ID; + ++pColList->numOfBound; + switch (pSchema[t].type) { + case TSDB_DATA_TYPE_BINARY: + pColList->boundNullLen += (sizeof(VarDataOffsetT) + VARSTR_HEADER_SIZE + CHAR_BYTES); + break; + case TSDB_DATA_TYPE_NCHAR: + pColList->boundNullLen += (sizeof(VarDataOffsetT) + VARSTR_HEADER_SIZE + TSDB_NCHAR_SIZE); + break; + default: + pColList->boundNullLen += TYPE_BYTES[pSchema[t].type]; + break; + } + } + + pColList->orderStatus = isOrdered ? ORDER_STATUS_ORDERED : ORDER_STATUS_DISORDERED; + + if (!isOrdered) { + pColList->colIdxInfo = taosMemoryCalloc(pColList->numOfBound, sizeof(SBoundIdxInfo)); + if (NULL == pColList->colIdxInfo) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + SBoundIdxInfo* pColIdx = pColList->colIdxInfo; + for (col_id_t i = 0; i < pColList->numOfBound; ++i) { + pColIdx[i].schemaColIdx = pColList->boundColumns[i]; + pColIdx[i].boundIdx = i; + } + qsort(pColIdx, pColList->numOfBound, sizeof(SBoundIdxInfo), schemaIdxCompar); + for (col_id_t i = 0; i < pColList->numOfBound; ++i) { + pColIdx[i].finalIdx = i; + } + qsort(pColIdx, pColList->numOfBound, sizeof(SBoundIdxInfo), boundIdxCompar); + } + + if(pColList->numOfCols > pColList->numOfBound){ + memset(&pColList->boundColumns[pColList->numOfBound], 0, + sizeof(col_id_t) * (pColList->numOfCols - pColList->numOfBound)); + } + + return TSDB_CODE_SUCCESS; +} + +static int32_t smlParseTags(SArray *cols, SKVRowBuilder *tagsBuilder, SParsedDataColInfo* tags, SSchema* pSchema, SVCreateTbReq *createTblReq) { + if (tdInitKVRowBuilder(tagsBuilder) < 0) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + SKvParam param = {.builder = tagsBuilder}; + for (int i = 0; i < tags->numOfBound; ++i) { + + SSchema* pTagSchema = &pSchema[tags->boundColumns[i] - 1]; // colId starts with 1 + param.schema = pTagSchema; + SSmlKv *kv = taosArrayGetP(cols, i); + KvRowAppend(NULL, kv->value, kv->valueLen, ¶m) ; + } + + + SKVRow row = tdGetKVRowFromBuilder(tagsBuilder); + if(row == NULL){ + return TSDB_CODE_SML_INVALID_DATA; + } + tdSortKVRowByColIdx(row); + createTblReq->type = TD_CHILD_TABLE; + createTblReq->ctbCfg.pTag = row; + return TSDB_CODE_SUCCESS; +} + +int32_t smlBind(void *handle, SArray *tags, SArray *cols, STableMeta *pTableMeta, char *msgBuf, int16_t msgBufLen) { + SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; + int32_t rowNum = taosArrayGetSize(cols); + if(rowNum <= 0) { + return buildInvalidOperationMsg(&pBuf, "cols size <= 0"); + } + + SmlExecHandle *smlHandle = (SmlExecHandle *)handle; + SSchema* pTagsSchema = getTableTagSchema(pTableMeta); + smlBoundColumns(tags, &smlHandle->tags, pTagsSchema); + smlParseTags(tags, &smlHandle->tagsBuilder, &smlHandle->tags, pTagsSchema, &smlHandle->createTblReq); + + STableDataBlocks* pDataBlock = NULL; + getDataBlockFromList(smlHandle->pBlockHash, pTableMeta->uid, TSDB_DEFAULT_PAYLOAD_SIZE, + sizeof(SSubmitBlk), getTableInfo(pTableMeta).rowSize, pTableMeta, + &pDataBlock, NULL, &smlHandle->createTblReq); + + SSchema* pSchema = getTableColumnSchema(pTableMeta); + + smlBoundColumns(taosArrayGetP(cols, 0), &pDataBlock->boundColumnInfo, pSchema); + + int32_t extendedRowSize = getExtendedRowSize(pDataBlock); + SParsedDataColInfo* spd = &pDataBlock->boundColumnInfo; + SRowBuilder* pBuilder = &pDataBlock->rowBuilder; + SMemParam param = {.rb = pBuilder}; + + initRowBuilder(&pDataBlock->rowBuilder, pDataBlock->pTableMeta->sversion, &pDataBlock->boundColumnInfo); + + allocateMemForSize(pDataBlock, extendedRowSize * rowNum); + + for (int32_t r = 0; r < rowNum; ++r) { + STSRow* row = (STSRow*)(pDataBlock->pData + pDataBlock->size); // skip the SSubmitBlk header + tdSRowResetBuf(pBuilder, row); + SArray *rowData = taosArrayGetP(cols, r); + + // 1. set the parsed value from sql string + for (int c = 0; c < spd->numOfBound; ++c) { + SSchema* pColSchema = &pSchema[spd->boundColumns[c] - 1]; + + param.schema = pColSchema; + getSTSRowAppendInfo(pBuilder->rowType, spd, c, ¶m.toffset, ¶m.colIdx); + + SSmlKv *kv = taosArrayGetP(rowData, c); + + if (kv->valueLen == 0) { + MemRowAppend(&pBuf, NULL, 0, ¶m); + } else { + int32_t colLen = pColSchema->bytes; + if (IS_VAR_DATA_TYPE(pColSchema->type)) { + colLen = kv->valueLen; + } + + MemRowAppend(&pBuf, kv->value, colLen, ¶m); + } + + if (PRIMARYKEY_TIMESTAMP_COL_ID == pColSchema->colId) { + TSKEY tsKey = TD_ROW_KEY(row); + checkTimestamp(pDataBlock, (const char *)&tsKey); + } + } + + // set the null value for the columns that do not assign values + if ((spd->numOfBound < spd->numOfCols) && TD_IS_TP_ROW(row)) { + for (int32_t i = 0; i < spd->numOfCols; ++i) { + if (spd->cols[i].valStat == VAL_STAT_NONE) { // the primary TS key is not VAL_STAT_NONE + tdAppendColValToTpRow(pBuilder, TD_VTYPE_NONE, getNullValue(pSchema[i].type), true, pSchema[i].type, i, + spd->cols[i].toffset); + } + } + } + + pDataBlock->size += extendedRowSize; + } + + SSubmitBlk *pBlocks = (SSubmitBlk *)(pDataBlock->pData); + if (TSDB_CODE_SUCCESS != setBlockInfo(pBlocks, pDataBlock, rowNum)) { + return buildInvalidOperationMsg(&pBuf, "too many rows in sql, total number of rows should be less than 32767"); + } + + return TSDB_CODE_SUCCESS; +} + +void* tscSmlInitHandle(SQuery *pQuery){ + SmlExecHandle *handle = taosMemoryCalloc(sizeof(SmlExecHandle)); + handle->pBlockHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false); + handle->pQuery = pQuery; + + return handle; +} + +void tscSmlDestroyHandle(void *pHandle){ + if(!pHandle) return; + SmlExecHandle *handle = (SmlExecHandle *)pHandle; + taosHashCleanup(handle->pBlockHash); + taosMemoryFree(handle); +} + +int32_t smlBuildOutput(void* handle, SHashObj* pVgHash) { + SmlExecHandle *smlHandle = (SmlExecHandle *)handle; + return qBuildStmtOutput(smlHandle->pQuery, pVgHash, smlHandle->pBlockHash); +} diff --git a/source/libs/parser/src/parInsertData.c b/source/libs/parser/src/parInsertData.c index bf30915fcb..6df8f2eaf3 100644 --- a/source/libs/parser/src/parInsertData.c +++ b/source/libs/parser/src/parInsertData.c @@ -116,7 +116,7 @@ void destroyBoundColumnInfo(void* pBoundInfo) { } static int32_t createDataBlock(size_t defaultSize, int32_t rowSize, int32_t startOffset, - const STableMeta* pTableMeta, STableDataBlocks** dataBlocks) { + STableMeta* pTableMeta, STableDataBlocks** dataBlocks) { STableDataBlocks* dataBuf = (STableDataBlocks*)taosMemoryCalloc(1, sizeof(STableDataBlocks)); if (dataBuf == NULL) { return TSDB_CODE_TSC_OUT_OF_MEMORY; @@ -137,8 +137,7 @@ static int32_t createDataBlock(size_t defaultSize, int32_t rowSize, int32_t star } memset(dataBuf->pData, 0, sizeof(SSubmitBlk)); - //Here we keep the tableMeta to avoid it to be remove by other threads. - dataBuf->pTableMeta = tableMetaDup(pTableMeta); + dataBuf->pTableMeta = pTableMeta; SParsedDataColInfo* pColInfo = &dataBuf->boundColumnInfo; SSchema* pSchema = getTableColumnSchema(dataBuf->pTableMeta); @@ -176,8 +175,7 @@ int32_t buildCreateTbMsg(STableDataBlocks* pBlocks, SVCreateTbReq* pCreateTbReq) return TSDB_CODE_SUCCESS; } -int32_t getDataBlockFromList(SHashObj* pHashList, int64_t id, int32_t size, int32_t startOffset, int32_t rowSize, - const STableMeta* pTableMeta, STableDataBlocks** dataBlocks, SArray* pBlockList, SVCreateTbReq* pCreateTbReq) { +int32_t getDataBlockFromList(SHashObj* pHashList, int64_t id, int32_t size, int32_t startOffset, int32_t rowSize, STableMeta* pTableMeta, STableDataBlocks** dataBlocks, SArray* pBlockList, SVCreateTbReq* pCreateTbReq) { *dataBlocks = NULL; STableDataBlocks** t1 = (STableDataBlocks**)taosHashGet(pHashList, (const char*)&id, sizeof(id)); if (t1 != NULL) { @@ -227,9 +225,9 @@ static void destroyDataBlock(STableDataBlocks* pDataBlock) { taosMemoryFreeClear(pDataBlock->pData); if (!pDataBlock->cloned) { // free the refcount for metermeta - if (pDataBlock->pTableMeta != NULL) { - taosMemoryFreeClear(pDataBlock->pTableMeta); - } +// if (pDataBlock->pTableMeta != NULL) { +// taosMemoryFreeClear(pDataBlock->pTableMeta); +// } destroyBoundColumnInfo(&pDataBlock->boundColumnInfo); } diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index 075caa868c..c98cfcc671 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -166,26 +166,6 @@ int32_t buildSyntaxErrMsg(SMsgBuf* pBuf, const char* additionalInfo, const char* return TSDB_CODE_TSC_SQL_SYNTAX_ERROR; } -static uint32_t getTableMetaSize(const STableMeta* pTableMeta) { - assert(pTableMeta != NULL); - - int32_t totalCols = 0; - if (pTableMeta->tableInfo.numOfColumns >= 0) { - totalCols = pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags; - } - - return sizeof(STableMeta) + totalCols * sizeof(SSchema); -} - -STableMeta* tableMetaDup(const STableMeta* pTableMeta) { - assert(pTableMeta != NULL); - size_t size = getTableMetaSize(pTableMeta); - - STableMeta* p = taosMemoryMalloc(size); - memcpy(p, pTableMeta, size); - return p; -} - SSchema *getTableColumnSchema(const STableMeta *pTableMeta) { assert(pTableMeta != NULL); return (SSchema*) pTableMeta->schema; diff --git a/tools/shell/src/shellCommand.c b/tools/shell/src/shellCommand.c index 546b19f83c..1034c034d4 100644 --- a/tools/shell/src/shellCommand.c +++ b/tools/shell/src/shellCommand.c @@ -51,8 +51,8 @@ void getPrevCharSize(const char *str, int pos, int *size, int *width) { if (str[pos] > 0 || countPrefixOnes((unsigned char)str[pos]) > 1) break; } - int rc = taosMbToWchar(&wc, str + pos, MB_CUR_MAX); - assert(rc == *size); + taosMbToWchar(&wc, str + pos, MB_CUR_MAX); + // assert(rc == *size); // it will be core, if str is encode by utf8 and taos charset is gbk *width = taosWcharWidth(wc); } diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 36d2866fb5..c7a61fb303 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -224,63 +224,27 @@ int32_t shellRunCommand(TAOS *con, char *command) { } } - bool esc = false; - char quote = 0, *cmd = command, *p = command; + char quote = 0, *cmd = command; for (char c = *command++; c != 0; c = *command++) { - if (esc) { - switch (c) { - case 'n': - c = '\n'; - break; - case 'r': - c = '\r'; - break; - case 't': - c = '\t'; - break; - case 'G': - *p++ = '\\'; - break; - case '\'': - case '"': - if (quote) { - *p++ = '\\'; - } - break; - } - *p++ = c; - esc = false; + if (c == '\\' && (*command == '\'' || *command == '"' || *command == '`')) { + command ++; continue; } - if (c == '\\') { - if (quote != 0 && (*command == '_' || *command == '\\')) { - // DO nothing - } else { - esc = true; - continue; - } - } - if (quote == c) { quote = 0; - } else if (quote == 0 && (c == '\'' || c == '"')) { + } else if (quote == 0 && (c == '\'' || c == '"' || c == '`')) { quote = c; - } - - *p++ = c; - if (c == ';' && quote == 0) { - c = *p; - *p = 0; + } else if (c == ';' && quote == 0) { + c = *command; + *command = 0; if (shellRunSingleCommand(con, cmd) < 0) { return -1; } - *p = c; - p = cmd; + *command = c; + cmd = command; } } - - *p = 0; return shellRunSingleCommand(con, cmd); } From a4629e56cecdfdda88d139f0cf3ebe66d33f6277 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 29 Apr 2022 10:58:46 +0800 Subject: [PATCH 03/31] refactor:add schemaless function --- include/common/tcommon.h | 8 +- include/common/ttypes.h | 16 +- source/client/inc/clientSml.h | 51 +- source/client/src/clientSml.c | 1335 ++++++++++------------------ source/libs/parser/src/parInsert.c | 39 +- 5 files changed, 545 insertions(+), 904 deletions(-) diff --git a/include/common/tcommon.h b/include/common/tcommon.h index 1c516e8a96..61f8ebae66 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -246,7 +246,13 @@ typedef struct { int32_t keyLen; uint8_t type; int16_t length; - const char* value; + union{ + const char* value; + int64_t i; + uint64_t u; + double d; + float f; + }; int32_t valueLen; } SSmlKv; diff --git a/include/common/ttypes.h b/include/common/ttypes.h index 405b20c521..377b443843 100644 --- a/include/common/ttypes.h +++ b/include/common/ttypes.h @@ -186,14 +186,14 @@ typedef struct { #define IS_NUMERIC_TYPE(_t) ((IS_SIGNED_NUMERIC_TYPE(_t)) || (IS_UNSIGNED_NUMERIC_TYPE(_t)) || (IS_FLOAT_TYPE(_t))) #define IS_MATHABLE_TYPE(_t) (IS_NUMERIC_TYPE(_t) || (_t) == (TSDB_DATA_TYPE_BOOL) || (_t) == (TSDB_DATA_TYPE_TIMESTAMP)) -#define IS_VALID_TINYINT(_t) ((_t) > INT8_MIN && (_t) <= INT8_MAX) -#define IS_VALID_SMALLINT(_t) ((_t) > INT16_MIN && (_t) <= INT16_MAX) -#define IS_VALID_INT(_t) ((_t) > INT32_MIN && (_t) <= INT32_MAX) -#define IS_VALID_BIGINT(_t) ((_t) > INT64_MIN && (_t) <= INT64_MAX) -#define IS_VALID_UTINYINT(_t) ((_t) >= 0 && (_t) < UINT8_MAX) -#define IS_VALID_USMALLINT(_t) ((_t) >= 0 && (_t) < UINT16_MAX) -#define IS_VALID_UINT(_t) ((_t) >= 0 && (_t) < UINT32_MAX) -#define IS_VALID_UBIGINT(_t) ((_t) >= 0 && (_t) < UINT64_MAX) +#define IS_VALID_TINYINT(_t) ((_t) >= INT8_MIN && (_t) <= INT8_MAX) +#define IS_VALID_SMALLINT(_t) ((_t) >= INT16_MIN && (_t) <= INT16_MAX) +#define IS_VALID_INT(_t) ((_t) >= INT32_MIN && (_t) <= INT32_MAX) +#define IS_VALID_BIGINT(_t) ((_t) >= INT64_MIN && (_t) <= INT64_MAX) +#define IS_VALID_UTINYINT(_t) ((_t) >= 0 && (_t) <= UINT8_MAX) +#define IS_VALID_USMALLINT(_t) ((_t) >= 0 && (_t) <= UINT16_MAX) +#define IS_VALID_UINT(_t) ((_t) >= 0 && (_t) <= UINT32_MAX) +#define IS_VALID_UBIGINT(_t) ((_t) >= 0 && (_t) <= UINT64_MAX) #define IS_VALID_FLOAT(_t) ((_t) >= -FLT_MAX && (_t) <= FLT_MAX) #define IS_VALID_DOUBLE(_t) ((_t) >= -DBL_MAX && (_t) <= DBL_MAX) diff --git a/source/client/inc/clientSml.h b/source/client/inc/clientSml.h index b711c837c0..c970f1e954 100644 --- a/source/client/inc/clientSml.h +++ b/source/client/inc/clientSml.h @@ -36,48 +36,55 @@ typedef struct { int32_t measureTagsLen; int32_t tagsLen; int32_t colsLen; + int32_t timestampLen; } TAOS_PARSE_ELEMENTS; typedef struct { const char *sTableName; // super table name uint8_t sTableNameLen; - char childTableName[TSDB_TABLE_NAME_LEN]; - uint64_t uid; + char childTableName[TSDB_TABLE_NAME_LEN]; + uint64_t uid; - SArray* tags; - SArray *cols; + SArray *tags; + SArray *cols; // elements are SHashObj for find by key quickly + + SArray colsColumn; // elements are cols key string } TAOS_SML_DATA_POINT_TAGS; typedef struct SSmlSTableMeta { // char *sTableName; // super table name // uint8_t sTableNameLen; - uint8_t precision; // the number of precision - SHashObj* tagHash; - SHashObj* fieldHash; + uint8_t precision; // the number of precision + SHashObj *tagHash; + SHashObj *fieldHash; } SSmlSTableMeta; +typedef struct SMsgBuf { + int32_t len; + char *buf; +} SMsgBuf; + typedef struct { - uint64_t id; + uint64_t id; - SMLProtocolType protocol; - int32_t tsType; + SMLProtocolType protocol; + int32_t tsType; - SHashObj* childTables; - SHashObj* superTables; + SHashObj *childTables; + SHashObj *superTables; - SHashObj* metaHashObj; - SHashObj* pVgHash; + SHashObj *metaHashObj; + SHashObj *pVgHash; - void* exec; + void *exec; - STscObj* taos; - SCatalog* pCatalog; - SRequestObj* pRequest; - SQuery* pQuery; + STscObj *taos; + SCatalog *pCatalog; + SRequestObj *pRequest; + SQuery *pQuery; - int32_t affectedRows; - char *msgBuf; - int16_t msgLen; + int32_t affectedRows; + SMsgBuf msgBuf; } SSmlLinesInfo; int smlInsert(TAOS* taos, SSmlLinesInfo* info); diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index dfbefcb0bf..3739c83109 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -13,6 +13,7 @@ #include "taoserror.h" #include "taos.h" #include "ttime.h" +#include "tstrbuild.h" typedef struct { @@ -31,14 +32,15 @@ typedef struct { #define SLASH '\\' #define tsMaxSQLStringLen (1024*1024) +#define TSNAMELEN 2 +#define TAGNAMELEN 3 //================================================================================================= static uint64_t linesSmlHandleId = 0; +static const char* TS = "ts"; +static const char* TAG = "tag"; + -static int32_t insertChildTablePointsBatch(void* pVoid, char* name, char* name1, SArray* pArray, SArray* pArray1, - SArray* pArray2, SArray* pArray3, size_t size, SSmlLinesInfo* info); -static int32_t doInsertChildTablePoints(void* pVoid, char* sql, char* name, SArray* pArray, SArray* pArray1, - SSmlLinesInfo* info); uint64_t genLinesSmlId() { uint64_t id; @@ -49,9 +51,15 @@ uint64_t genLinesSmlId() { return id; } +static int32_t buildInvalidDataMsg(SMsgBuf* pBuf, const char *msg1, const char *msg2) { + if(msg1) snprintf(pBuf->buf, pBuf->len, "%s:", msg1); + if(msg2) strncpy(pBuf->buf, msg2, pBuf->len); + return TSDB_CODE_SML_INVALID_DATA; +} + int compareSmlColKv(const void* p1, const void* p2) { - TAOS_SML_KV* kv1 = (TAOS_SML_KV*)p1; - TAOS_SML_KV* kv2 = (TAOS_SML_KV*)p2; + SSmlKv* kv1 = (SSmlKv *)p1; + SSmlKv* kv2 = (SSmlKv*)p2; int kvLen1 = (int)strlen(kv1->key); int kvLen2 = (int)strlen(kv2->key); int res = strncasecmp(kv1->key, kv2->key, MIN(kvLen1, kvLen2)); @@ -78,7 +86,7 @@ typedef struct { typedef struct { char sTableName[TSDB_TABLE_NAME_LEN]; - SSchema* field; + SSmlKv * field; } SAlterSTableActionInfo; typedef struct { @@ -89,94 +97,17 @@ typedef struct { }; } SSchemaAction; -static int32_t getFieldBytesFromSmlKv(TAOS_SML_KV* kv, int32_t* bytes, uint64_t id) { - if (!IS_VAR_DATA_TYPE(kv->type)) { - *bytes = tDataTypes[kv->type].bytes; - } else { - if (kv->type == TSDB_DATA_TYPE_NCHAR) { - TdUcs4 *ucs = taosMemoryMalloc(kv->length * TSDB_NCHAR_SIZE + 1); - int32_t bytesNeeded = 0; - bool succ = taosMbsToUcs4(kv->value, kv->length, ucs, kv->length * TSDB_NCHAR_SIZE, &bytesNeeded); - if (!succ) { - taosMemoryFree(ucs); - uError("SML:0x%"PRIx64" convert nchar string to UCS4_LE failed:%s", id, kv->value); - return TSDB_CODE_TSC_INVALID_VALUE; - } - taosMemoryFree(ucs); - *bytes = bytesNeeded + VARSTR_HEADER_SIZE; - } else if (kv->type == TSDB_DATA_TYPE_BINARY) { - *bytes = kv->length + VARSTR_HEADER_SIZE; - } - } - return 0; -} +static int32_t buildSmlChildTableName(TAOS_SML_DATA_POINT_TAGS *tags) { + int32_t size = taosArrayGetSize(tags->tags); + ASSERT(size > 0); + qsort(tags->tags, size, POINTER_BYTES, compareSmlColKv); -static int32_t buildSmlKvSchema(TAOS_SML_KV* smlKv, SHashObj* hash, SArray* array, SSmlLinesInfo* info) { - SSchema* pField = NULL; - size_t* pFieldIdx = taosHashGet(hash, smlKv->key, strlen(smlKv->key)); - size_t fieldIdx = -1; - int32_t code = 0; - if (pFieldIdx) { - fieldIdx = *pFieldIdx; - pField = taosArrayGet(array, fieldIdx); - - if (pField->type != smlKv->type) { - uError("SML:0x%"PRIx64" type mismatch. key %s, type %d. type before %d", info->id, smlKv->key, smlKv->type, pField->type); - return TSDB_CODE_TSC_INVALID_VALUE; - } - - int32_t bytes = 0; - code = getFieldBytesFromSmlKv(smlKv, &bytes, info->id); - if (code != 0) { - return code; - } - pField->bytes = MAX(pField->bytes, bytes); - - } else { - SSchema field = {0}; - size_t tagKeyLen = strlen(smlKv->key); - strncpy(field.name, smlKv->key, tagKeyLen); - field.name[tagKeyLen] = '\0'; - field.type = smlKv->type; - - int32_t bytes = 0; - code = getFieldBytesFromSmlKv(smlKv, &bytes, info->id); - if (code != 0) { - return code; - } - field.bytes = bytes; - - pField = taosArrayPush(array, &field); - fieldIdx = taosArrayGetSize(array) - 1; - taosHashPut(hash, field.name, tagKeyLen, &fieldIdx, sizeof(fieldIdx)); - } - - smlKv->fieldSchemaIdx = (uint32_t)fieldIdx; - - return 0; -} - -static int32_t getSmlMd5ChildTableName(TAOS_SML_DATA_POINT* point, char* tableName, int* tableNameLen, - SSmlLinesInfo* info) { - uDebug("SML:0x%"PRIx64" taos_sml_insert get child table name through md5", info->id); - if (point->tagNum) { - qsort(point->tags, point->tagNum, sizeof(TAOS_SML_KV), compareSmlColKv); - } - - SStringBuilder sb; memset(&sb, 0, sizeof(sb)); - char sTableName[TSDB_TABLE_NAME_LEN] = {0}; - strncpy(sTableName, point->stableName, strlen(point->stableName)); - //strtolower(sTableName, point->stableName); - taosStringBuilderAppendString(&sb, sTableName); - for (int j = 0; j < point->tagNum; ++j) { - taosStringBuilderAppendChar(&sb, ','); - TAOS_SML_KV* tagKv = point->tags + j; - char tagName[TSDB_COL_NAME_LEN] = {0}; - strncpy(tagName, tagKv->key, strlen(tagKv->key)); - //strtolower(tagName, tagKv->key); - taosStringBuilderAppendString(&sb, tagName); - taosStringBuilderAppendChar(&sb, '='); - taosStringBuilderAppend(&sb, tagKv->value, tagKv->length); + SStringBuilder sb = {0}; + taosStringBuilderAppendStringLen(&sb, tags->sTableName, tags->sTableNameLen); + for (int j = 0; j < size; ++j) { + SSmlKv *tagKv = taosArrayGetP(tags->tags, j); + taosStringBuilderAppendStringLen(&sb, tagKv->key, tagKv->keyLen); + taosStringBuilderAppendStringLen(&sb, tagKv->value, tagKv->valueLen); } size_t len = 0; char* keyJoined = taosStringBuilderGetResult(&sb, &len); @@ -186,183 +117,74 @@ static int32_t getSmlMd5ChildTableName(TAOS_SML_DATA_POINT* point, char* tableNa tMD5Final(&context); uint64_t digest1 = *(uint64_t*)(context.digest); uint64_t digest2 = *(uint64_t*)(context.digest + 8); - *tableNameLen = snprintf(tableName, *tableNameLen, - "t_%016"PRIx64"%016"PRIx64, digest1, digest2); + snprintf(tags->childTableName, TSDB_TABLE_NAME_LEN, "t_%016"PRIx64"%016"PRIx64, digest1, digest2); taosStringBuilderDestroy(&sb); - uDebug("SML:0x%"PRIx64" child table name: %s", info->id, tableName); - return 0; -} - -static int32_t buildSmlChildTableName(TAOS_SML_DATA_POINT* point, SSmlLinesInfo* info) { - uDebug("SML:0x%"PRIx64" taos_sml_insert build child table name", info->id); - char childTableName[TSDB_TABLE_NAME_LEN]; - int32_t tableNameLen = TSDB_TABLE_NAME_LEN; - getSmlMd5ChildTableName(point, childTableName, &tableNameLen, info); - point->childTableName = calloc(1, tableNameLen+1); - strncpy(point->childTableName, childTableName, tableNameLen); - point->childTableName[tableNameLen] = '\0'; - return 0; -} - -static int32_t buildDataPointSchemas(TAOS_SML_DATA_POINT* points, int numPoint, SArray* stableSchemas, SSmlLinesInfo* info) { - int32_t code = 0; - SHashObj* sname2shema = taosHashInit(32, - taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); - - for (int i = 0; i < numPoint; ++i) { - TAOS_SML_DATA_POINT* point = &points[i]; - size_t stableNameLen = strlen(point->stableName); - size_t* pStableIdx = taosHashGet(sname2shema, point->stableName, stableNameLen); - SSmlSTableSchema* pStableSchema = NULL; - size_t stableIdx = -1; - if (pStableIdx) { - pStableSchema= taosArrayGet(stableSchemas, *pStableIdx); - stableIdx = *pStableIdx; - } else { - SSmlSTableSchema schema; - strncpy(schema.sTableName, point->stableName, stableNameLen); - schema.sTableName[stableNameLen] = '\0'; - schema.fields = taosArrayInit(64, sizeof(SSchema)); - schema.tags = taosArrayInit(8, sizeof(SSchema)); - schema.tagHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); - schema.fieldHash = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); - - pStableSchema = taosArrayPush(stableSchemas, &schema); - stableIdx = taosArrayGetSize(stableSchemas) - 1; - taosHashPut(sname2shema, schema.sTableName, stableNameLen, &stableIdx, sizeof(size_t)); - } - - for (int j = 0; j < point->tagNum; ++j) { - TAOS_SML_KV* tagKv = point->tags + j; - if (!point->childTableName) { - buildSmlChildTableName(point, info); - } - - code = buildSmlKvSchema(tagKv, pStableSchema->tagHash, pStableSchema->tags, info); - if (code != 0) { - uError("SML:0x%"PRIx64" build data point schema failed. point no.: %d, tag key: %s", info->id, i, tagKv->key); - return code; - } - } - - //for Line Protocol tags may be omitted, add a tag with NULL value - if (point->tagNum == 0) { - if (!point->childTableName) { - buildSmlChildTableName(point, info); - } - char tagNullName[TSDB_COL_NAME_LEN] = {0}; - size_t nameLen = strlen(tsSmlTagNullName); - strncpy(tagNullName, tsSmlTagNullName, nameLen); - addEscapeCharToString(tagNullName, (int32_t)nameLen); - size_t* pTagNullIdx = taosHashGet(pStableSchema->tagHash, tagNullName, nameLen); - if (!pTagNullIdx) { - SSchema tagNull = {0}; - tagNull.type = TSDB_DATA_TYPE_NCHAR; - tagNull.bytes = TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE; - strncpy(tagNull.name, tagNullName, nameLen); - taosArrayPush(pStableSchema->tags, &tagNull); - size_t tagNullIdx = taosArrayGetSize(pStableSchema->tags) - 1; - taosHashPut(pStableSchema->tagHash, tagNull.name, nameLen, &tagNullIdx, sizeof(tagNullIdx)); - } - } - - for (int j = 0; j < point->fieldNum; ++j) { - TAOS_SML_KV* fieldKv = point->fields + j; - code = buildSmlKvSchema(fieldKv, pStableSchema->fieldHash, pStableSchema->fields, info); - if (code != 0) { - uError("SML:0x%"PRIx64" build data point schema failed. point no.: %d, tag key: %s", info->id, i, fieldKv->key); - return code; - } - } - - point->schemaIdx = (uint32_t)stableIdx; - } - - size_t numStables = taosArrayGetSize(stableSchemas); - for (int32_t i = 0; i < numStables; ++i) { - SSmlSTableSchema* schema = taosArrayGet(stableSchemas, i); - taosHashCleanup(schema->tagHash); - taosHashCleanup(schema->fieldHash); - } - taosHashCleanup(sname2shema); - - uDebug("SML:0x%"PRIx64" build point schema succeed. num of super table: %zu", info->id, numStables); - for (int32_t i = 0; i < numStables; ++i) { - SSmlSTableSchema* schema = taosArrayGet(stableSchemas, i); - uDebug("\ttable name: %s, tags number: %zu, fields number: %zu", schema->sTableName, - taosArrayGetSize(schema->tags), taosArrayGetSize(schema->fields)); - } - + tags->uid = digest1; + uDebug("SML: child table name: %s", tags->childTableName); return 0; } static int32_t generateSchemaAction(SSchema* pointColField, SHashObj* dbAttrHash, SArray* dbAttrArray, bool isTag, char sTableName[], SSchemaAction* action, bool* actionNeeded, SSmlLinesInfo* info) { - char fieldName[TSDB_COL_NAME_LEN] = {0}; - strcpy(fieldName, pointColField->name); - - size_t* pDbIndex = taosHashGet(dbAttrHash, fieldName, strlen(fieldName)); - if (pDbIndex) { - SSchema* dbAttr = taosArrayGet(dbAttrArray, *pDbIndex); - assert(strcasecmp(dbAttr->name, pointColField->name) == 0); - if (pointColField->type != dbAttr->type) { - uError("SML:0x%"PRIx64" point type and db type mismatch. key: %s. point type: %d, db type: %d", info->id, pointColField->name, - pointColField->type, dbAttr->type); - return TSDB_CODE_TSC_INVALID_VALUE; - } - - if (IS_VAR_DATA_TYPE(pointColField->type) && (pointColField->bytes > dbAttr->bytes)) { - if (isTag) { - action->action = SCHEMA_ACTION_CHANGE_TAG_SIZE; - } else { - action->action = SCHEMA_ACTION_CHANGE_COLUMN_SIZE; - } - memset(&action->alterSTable, 0, sizeof(SAlterSTableActionInfo)); - memcpy(action->alterSTable.sTableName, sTableName, TSDB_TABLE_NAME_LEN); - action->alterSTable.field = pointColField; - *actionNeeded = true; - } - } else { - if (isTag) { - action->action = SCHEMA_ACTION_ADD_TAG; - } else { - action->action = SCHEMA_ACTION_ADD_COLUMN; - } - memset(&action->alterSTable, 0, sizeof(SAlterSTableActionInfo)); - memcpy(action->alterSTable.sTableName, sTableName, TSDB_TABLE_NAME_LEN); - action->alterSTable.field = pointColField; - *actionNeeded = true; - } - if (*actionNeeded) { - uDebug("SML:0x%" PRIx64 " generate schema action. column name: %s, action: %d", info->id, fieldName, - action->action); - } +// char fieldName[TSDB_COL_NAME_LEN] = {0}; +// strcpy(fieldName, pointColField->name); +// +// size_t* pDbIndex = taosHashGet(dbAttrHash, fieldName, strlen(fieldName)); +// if (pDbIndex) { +// SSchema* dbAttr = taosArrayGet(dbAttrArray, *pDbIndex); +// assert(strcasecmp(dbAttr->name, pointColField->name) == 0); +// if (pointColField->type != dbAttr->type) { +// uError("SML:0x%"PRIx64" point type and db type mismatch. key: %s. point type: %d, db type: %d", info->id, pointColField->name, +// pointColField->type, dbAttr->type); +// return TSDB_CODE_TSC_INVALID_VALUE; +// } +// +// if (IS_VAR_DATA_TYPE(pointColField->type) && (pointColField->bytes > dbAttr->bytes)) { +// if (isTag) { +// action->action = SCHEMA_ACTION_CHANGE_TAG_SIZE; +// } else { +// action->action = SCHEMA_ACTION_CHANGE_COLUMN_SIZE; +// } +// memset(&action->alterSTable, 0, sizeof(SAlterSTableActionInfo)); +// memcpy(action->alterSTable.sTableName, sTableName, TSDB_TABLE_NAME_LEN); +// action->alterSTable.field = pointColField; +// *actionNeeded = true; +// } +// } else { +// if (isTag) { +// action->action = SCHEMA_ACTION_ADD_TAG; +// } else { +// action->action = SCHEMA_ACTION_ADD_COLUMN; +// } +// memset(&action->alterSTable, 0, sizeof(SAlterSTableActionInfo)); +// memcpy(action->alterSTable.sTableName, sTableName, TSDB_TABLE_NAME_LEN); +// action->alterSTable.field = pointColField; +// *actionNeeded = true; +// } +// if (*actionNeeded) { +// uDebug("SML:0x%" PRIx64 " generate schema action. column name: %s, action: %d", info->id, fieldName, +// action->action); +// } return 0; } -static int32_t buildColumnDescription(TAOS_SML_KV* field, - char* buf, int32_t bufSize, int32_t* outBytes) { +static int32_t buildColumnDescription(SSmlKv* field, char* buf, int32_t bufSize, int32_t* outBytes) { uint8_t type = field->type; char tname[TSDB_TABLE_NAME_LEN] = {0}; memcpy(tname, field->key, field->keyLen); if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { - int32_t bytes = field->length - VARSTR_HEADER_SIZE; - if (type == TSDB_DATA_TYPE_NCHAR) { - bytes = bytes/TSDB_NCHAR_SIZE; - } + int32_t bytes = field->valueLen; // todo int out = snprintf(buf, bufSize,"%s %s(%d)", tname,tDataTypes[field->type].name, bytes); *outBytes = out; } else { - int out = snprintf(buf, bufSize, "%s %s", - tname, tDataTypes[type].name); + int out = snprintf(buf, bufSize, "%s %s", tname, tDataTypes[type].name); *outBytes = out; } return 0; } - static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInfo* info) { int32_t code = 0; int32_t outBytes = 0; @@ -472,7 +294,7 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInf int n = sprintf(result, "create stable %s (", action->createSTable.sTableName); char* pos = result + n; int freeBytes = capacity - n; - TAOS_SML_KV **kv = taosHashIterate(action->createSTable.fields, NULL); + SSmlKv **kv = taosHashIterate(action->createSTable.fields, NULL); while(kv){ buildColumnDescription(*kv, pos, freeBytes, &outBytes); pos += outBytes; freeBytes -= outBytes; @@ -523,168 +345,6 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInf return code; } -static int32_t destroySmlSTableSchema(SSmlSTableSchema* schema) { - taosHashCleanup(schema->tagHash); - taosHashCleanup(schema->fieldHash); - taosArrayDestroy(&schema->tags); - taosArrayDestroy(&schema->fields); - return 0; -} - -static int32_t fillDbSchema(STableMeta* tableMeta, char* tableName, SSmlSTableSchema* schema, SSmlLinesInfo* info) { - schema->tags = taosArrayInit(8, sizeof(SSchema)); - schema->fields = taosArrayInit(64, sizeof(SSchema)); - schema->tagHash = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); - schema->fieldHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); - - tstrncpy(schema->sTableName, tableName, strlen(tableName)+1); - schema->precision = tableMeta->tableInfo.precision; - for (int i=0; itableInfo.numOfColumns; ++i) { - SSchema field; - tstrncpy(field.name, tableMeta->schema[i].name, strlen(tableMeta->schema[i].name)+1); - addEscapeCharToString(field.name, (int16_t)strlen(field.name)); - field.type = tableMeta->schema[i].type; - field.bytes = tableMeta->schema[i].bytes; - taosArrayPush(schema->fields, &field); - size_t fieldIndex = taosArrayGetSize(schema->fields) - 1; - taosHashPut(schema->fieldHash, field.name, strlen(field.name), &fieldIndex, sizeof(fieldIndex)); - } - - for (int i=0; itableInfo.numOfTags; ++i) { - int j = i + tableMeta->tableInfo.numOfColumns; - SSchema field; - tstrncpy(field.name, tableMeta->schema[j].name, strlen(tableMeta->schema[j].name)+1); - addEscapeCharToString(field.name, (int16_t)strlen(field.name)); - field.type = tableMeta->schema[j].type; - field.bytes = tableMeta->schema[j].bytes; - taosArrayPush(schema->tags, &field); - size_t tagIndex = taosArrayGetSize(schema->tags) - 1; - taosHashPut(schema->tagHash, field.name, strlen(field.name), &tagIndex, sizeof(tagIndex)); - } - uDebug("SML:0x%"PRIx64 " load table schema succeed. table name: %s, columns number: %d, tag number: %d, precision: %d", - info->id, tableName, tableMeta->tableInfo.numOfColumns, tableMeta->tableInfo.numOfTags, schema->precision); - return TSDB_CODE_SUCCESS; -} - -static int32_t getSuperTableMetaFromLocalCache(TAOS* taos, char* tableName, STableMeta** outTableMeta, SSmlLinesInfo* info) { - int32_t code = 0; - STableMeta* tableMeta = NULL; - - SSqlObj* pSql = calloc(1, sizeof(SSqlObj)); - if (pSql == NULL) { - uError("SML:0x%" PRIx64 " failed to allocate memory, reason:%s", info->id, strerror(errno)); - code = TSDB_CODE_TSC_OUT_OF_MEMORY; - return code; - } - pSql->pTscObj = taos; - pSql->signature = pSql; - pSql->fp = NULL; - - registerSqlObj(pSql); - char tableNameBuf[TSDB_TABLE_NAME_LEN + TS_BACKQUOTE_CHAR_SIZE] = {0}; - memcpy(tableNameBuf, tableName, strlen(tableName)); - SStrToken tableToken = {.z = tableNameBuf, .n = (uint32_t)strlen(tableName), .type = TK_ID}; - tGetToken(tableNameBuf, &tableToken.type); - bool dbIncluded = false; - // Check if the table name available or not - if (tscValidateName(&tableToken, true, &dbIncluded) != TSDB_CODE_SUCCESS) { - code = TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; - sprintf(pSql->cmd.payload, "table name is invalid"); - taosReleaseRef(tscObjRef, pSql->self); - return code; - } - - SName sname = {0}; - if ((code = tscSetTableFullName(&sname, &tableToken, pSql, dbIncluded)) != TSDB_CODE_SUCCESS) { - taosReleaseRef(tscObjRef, pSql->self); - return code; - } - - char fullTableName[TSDB_TABLE_FNAME_LEN] = {0}; - memset(fullTableName, 0, tListLen(fullTableName)); - tNameExtractFullName(&sname, fullTableName); - - size_t size = 0; - taosHashGetCloneExt(UTIL_GET_TABLEMETA(pSql), fullTableName, strlen(fullTableName), NULL, (void**)&tableMeta, &size); - - STableMeta* stableMeta = tableMeta; - if (tableMeta != NULL && tableMeta->tableType == TSDB_CHILD_TABLE) { - taosHashGetCloneExt(UTIL_GET_TABLEMETA(pSql), tableMeta->sTableName, strlen(tableMeta->sTableName), NULL, - (void**)stableMeta, &size); - } - taosReleaseRef(tscObjRef, pSql->self); - - if (stableMeta != tableMeta) { - taosMemoryFree(tableMeta); - } - - if (stableMeta != NULL) { - if (outTableMeta != NULL) { - *outTableMeta = stableMeta; - } else { - taosMemoryFree(stableMeta); - } - return TSDB_CODE_SUCCESS; - } else { - return TSDB_CODE_TSC_NO_META_CACHED; - } -} - -static int32_t retrieveTableMeta(TAOS* taos, char* tableName, STableMeta** pTableMeta, SSmlLinesInfo* info) { - int32_t code = 0; - int32_t retries = 0; - STableMeta* tableMeta = NULL; - while (retries++ <= TSDB_MAX_REPLICA && tableMeta == NULL) { - STscObj* pObj = (STscObj*)taos; - if (pObj == NULL || pObj->signature != pObj) { - terrno = TSDB_CODE_TSC_DISCONNECTED; - return TSDB_CODE_TSC_DISCONNECTED; - } - - uDebug("SML:0x%" PRIx64 " retrieve table meta. super table name: %s", info->id, tableName); - code = getSuperTableMetaFromLocalCache(taos, tableName, &tableMeta, info); - if (code == TSDB_CODE_SUCCESS) { - uDebug("SML:0x%" PRIx64 " successfully retrieved table meta. super table name: %s", info->id, tableName); - break; - } else if (code == TSDB_CODE_TSC_NO_META_CACHED) { - char sql[256]; - snprintf(sql, 256, "describe %s", tableName); - TAOS_RES* res = taos_query(taos, sql); - code = taos_errno(res); - if (code != 0) { - uError("SML:0x%" PRIx64 " describe table failure. %s", info->id, taos_errstr(res)); - taos_free_result(res); - return code; - } - taos_free_result(res); - } else { - return code; - } - } - - if (tableMeta != NULL) { - *pTableMeta = tableMeta; - return TSDB_CODE_SUCCESS; - } else { - uError("SML:0x%" PRIx64 " failed to retrieve table meta. super table name: %s", info->id, tableName); - return TSDB_CODE_TSC_NO_META_CACHED; - } -} - -static int32_t loadTableSchemaFromDB(TAOS* taos, char* tableName, SSmlSTableSchema* schema, SSmlLinesInfo* info) { - int32_t code = 0; - STableMeta* tableMeta = NULL; - code = retrieveTableMeta(taos, tableName, &tableMeta, info); - if (code == TSDB_CODE_SUCCESS) { - assert(tableMeta != NULL); - fillDbSchema(tableMeta, tableName, schema, info); - taosMemoryFree(tableMeta); - tableMeta = NULL; - } - - return code; -} - static int32_t modifyDBSchemas(TAOS* taos, SSmlLinesInfo* info) { int32_t code = 0; @@ -706,7 +366,6 @@ static int32_t modifyDBSchemas(TAOS* taos, SSmlLinesInfo* info) { if (code == TSDB_CODE_TDB_INVALID_TABLE_ID) { SSchemaAction schemaAction = {0}; schemaAction.action = SCHEMA_ACTION_CREATE_STABLE; - memset(&schemaAction.createSTable, 0, sizeof(SCreateSTableActionInfo)); memcpy(schemaAction.createSTable.sTableName, superTable, superTableLen); schemaAction.createSTable.tags = cTablePoints->tagHash; schemaAction.createSTable.fields = cTablePoints->fieldHash; @@ -728,7 +387,7 @@ static int32_t modifyDBSchemas(TAOS* taos, SSmlLinesInfo* info) { return 0; } -static int32_t applyDataPoints(TAOS* taos, SSmlLinesInfo* info) { +static int32_t applyDataPoints(SSmlLinesInfo* info) { int32_t code = TSDB_CODE_SUCCESS; TAOS_SML_DATA_POINT_TAGS** oneTable = taosHashIterate(info->childTables, NULL); @@ -746,43 +405,40 @@ static int32_t applyDataPoints(TAOS* taos, SSmlLinesInfo* info) { STableMeta** pMeta = taosHashGet(info->metaHashObj, tableData->sTableName, tableData->sTableNameLen); ASSERT (NULL != pMeta && NULL != *pMeta); (*pMeta)->vgId = vg.vgId; - (*pMeta)->uid = tableData->uid; - - smlBind(info->exec, tableData->tags, tableData->cols, *pMeta, info->msgBuf, info->msgLen); + (*pMeta)->uid = tableData->uid; // one table merge data block together according uid + code = smlBind(info->exec, tableData->tags, tableData->cols, *pMeta, info->msgBuf.buf, info->msgBuf.len); + if(code != TSDB_CODE_SUCCESS){ + return code; + } oneTable = taosHashIterate(info->childTables, oneTable); } smlBuildOutput(info->exec, info->pVgHash); launchQueryImpl(info->pRequest, info->pQuery, TSDB_CODE_SUCCESS, true); - if(info->pRequest->code != TSDB_CODE_SUCCESS){ - - } info->affectedRows = taos_affected_rows(info->pRequest); - return code; + return info->pRequest->code; } -int tscSmlInsert(TAOS* taos, SSmlLinesInfo* info) { +int smlInsert(TAOS* taos, SSmlLinesInfo* info) { uDebug("SML:0x%"PRIx64" taos_sml_insert. number of super tables: %d", info->id, taosHashGetSize(info->superTables)); - int32_t code = TSDB_CODE_SUCCESS; - info->affectedRows = 0; uDebug("SML:0x%"PRIx64" modify db schemas", info->id); - code = modifyDBSchemas(taos, info); + int32_t code = modifyDBSchemas(taos, info); if (code != 0) { uError("SML:0x%"PRIx64" error change db schema : %s", info->id, tstrerror(code)); - goto clean_up; + return code; } uDebug("SML:0x%"PRIx64" apply data points", info->id); - code = applyDataPoints(taos, info); + code = applyDataPoints(info); if (code != 0) { uError("SML:0x%"PRIx64" error apply data points : %s", info->id, tstrerror(code)); + return code; } -clean_up: - return code; + return TSDB_CODE_SUCCESS; } //========================================================================= @@ -835,242 +491,307 @@ static void escapeSpecialCharacter(uint8_t field, const char **pos) { *pos = cur; } -bool isValidInteger(char *str) { - char *c = str; - if (*c != '+' && *c != '-' && !isdigit(*c)) { - return false; - } - c++; - while (*c != '\0') { - if (!isdigit(*c)) { - return false; - } - c++; - } - return true; -} - -bool isValidFloat(char *str) { - char *c = str; - uint8_t has_dot, has_exp, has_sign; - has_dot = 0; - has_exp = 0; - has_sign = 0; - - if (*c != '+' && *c != '-' && *c != '.' && !isdigit(*c)) { - return false; - } - if (*c == '.' && isdigit(*(c + 1))) { - has_dot = 1; - } - c++; - while (*c != '\0') { - if (!isdigit(*c)) { - switch (*c) { - case '.': { - if (!has_dot && !has_exp && isdigit(*(c + 1))) { - has_dot = 1; - } else { - return false; - } - break; - } - case 'e': - case 'E': { - if (!has_exp && isdigit(*(c - 1)) && - (isdigit(*(c + 1)) || - *(c + 1) == '+' || - *(c + 1) == '-')) { - has_exp = 1; - } else { - return false; - } - break; - } - case '+': - case '-': { - if (!has_sign && has_exp && isdigit(*(c + 1))) { - has_sign = 1; - } else { - return false; - } - break; - } - default: { - return false; - } - } - } - c++; - } //while - return true; -} - -static bool isInteger(char *pVal, uint16_t len, bool *has_sign) { - if (len <= 1) { - return false; - } - if (pVal[len - 1] == 'i') { - *has_sign = true; - return true; - } - if (pVal[len - 1] == 'u') { - *has_sign = false; - return true; - } - - return false; -} - -static bool isTinyInt(char *pVal, uint16_t len) { +static bool parseTinyInt(SSmlKv *kvVal, bool *isValid, SMsgBuf *msg) { + const char *pVal = kvVal->value; + int32_t len = kvVal->valueLen; if (len <= 2) { return false; } - if (!strcasecmp(&pVal[len - 2], "i8")) { - //printf("Type is int8(%s)\n", pVal); + const char *signalPos = pVal + len - 2; + if (!strcasecmp(signalPos, "i8")) { + char *endptr = NULL; + int64_t result = strtoll(pVal, &endptr, 10); + if(endptr != signalPos){ // 78ri8 + *isValid = false; + buildInvalidDataMsg(msg, "invalid tiny int", endptr); + }else if(!IS_VALID_TINYINT(result)){ + *isValid = false; + buildInvalidDataMsg(msg, "tiny int out of range[-128,127]", endptr); + }else{ + kvVal->i = result; + *isValid = true; + } return true; } return false; } -static bool isTinyUint(char *pVal, uint16_t len) { +static bool parseTinyUint(SSmlKv *kvVal, bool *isValid, SMsgBuf *msg) { + const char *pVal = kvVal->value; + int32_t len = kvVal->valueLen; if (len <= 2) { return false; } if (pVal[0] == '-') { return false; } - if (!strcasecmp(&pVal[len - 2], "u8")) { - //printf("Type is uint8(%s)\n", pVal); + const char *signalPos = pVal + len - 2; + if (!strcasecmp(signalPos, "u8")) { + char *endptr = NULL; + int64_t result = strtoll(pVal, &endptr, 10); + if(endptr != signalPos){ // 78ri8 + *isValid = false; + buildInvalidDataMsg(msg, "invalid unsigned tiny int", endptr); + }else if(!IS_VALID_UTINYINT(result)){ + *isValid = false; + buildInvalidDataMsg(msg, "unsigned tiny int out of range[0,255]", endptr); + }else{ + kvVal->i = result; + *isValid = true; + } return true; } return false; } -static bool isSmallInt(char *pVal, uint16_t len) { +static bool parseSmallInt(SSmlKv *kvVal, bool *isValid, SMsgBuf *msg) { + const char *pVal = kvVal->value; + int32_t len = kvVal->valueLen; if (len <= 3) { return false; } - if (!strcasecmp(&pVal[len - 3], "i16")) { - //printf("Type is int16(%s)\n", pVal); + const char *signalPos = pVal + len - 3; + if (!strcasecmp(signalPos, "i16")) { + char *endptr = NULL; + int64_t result = strtoll(pVal, &endptr, 10); + if(endptr != signalPos){ // 78ri8 + *isValid = false; + buildInvalidDataMsg(msg, "invalid small int", endptr); + }else if(!IS_VALID_SMALLINT(result)){ + *isValid = false; + buildInvalidDataMsg(msg, "small int our of range[-32768,32767]", endptr); + }else{ + kvVal->i = result; + *isValid = true; + } return true; } return false; } -static bool isSmallUint(char *pVal, uint16_t len) { +static bool parseSmallUint(SSmlKv *kvVal, bool *isValid, SMsgBuf *msg) { + const char *pVal = kvVal->value; + int32_t len = kvVal->valueLen; if (len <= 3) { return false; } if (pVal[0] == '-') { return false; } - if (strcasecmp(&pVal[len - 3], "u16") == 0) { - //printf("Type is uint16(%s)\n", pVal); + const char *signalPos = pVal + len - 3; + if (strcasecmp(signalPos, "u16") == 0) { + char *endptr = NULL; + int64_t result = strtoll(pVal, &endptr, 10); + if(endptr != signalPos){ // 78ri8 + *isValid = false; + buildInvalidDataMsg(msg, "invalid unsigned small int", endptr); + }else if(!IS_VALID_USMALLINT(result)){ + *isValid = false; + buildInvalidDataMsg(msg, "unsigned small int out of rang[0,65535]", endptr); + }else{ + kvVal->i = result; + *isValid = true; + } return true; } return false; } -static bool isInt(char *pVal, uint16_t len) { +static bool parseInt(SSmlKv *kvVal, bool *isValid, SMsgBuf *msg) { + const char *pVal = kvVal->value; + int32_t len = kvVal->valueLen; if (len <= 3) { return false; } - if (strcasecmp(&pVal[len - 3], "i32") == 0) { - //printf("Type is int32(%s)\n", pVal); + const char *signalPos = pVal + len - 3; + if (strcasecmp(signalPos, "i32") == 0) { + char *endptr = NULL; + int64_t result = strtoll(pVal, &endptr, 10); + if(endptr != signalPos){ // 78ri8 + *isValid = false; + buildInvalidDataMsg(msg, "invalid int", endptr); + }else if(!IS_VALID_INT(result)){ + *isValid = false; + buildInvalidDataMsg(msg, "int out of range[-2147483648,2147483647]", endptr); + }else{ + kvVal->i = result; + *isValid = true; + } return true; } return false; } -static bool isUint(char *pVal, uint16_t len) { +static bool parseUint(SSmlKv *kvVal, bool *isValid, SMsgBuf *msg) { + const char *pVal = kvVal->value; + int32_t len = kvVal->valueLen; if (len <= 3) { return false; } if (pVal[0] == '-') { return false; } - if (strcasecmp(&pVal[len - 3], "u32") == 0) { - //printf("Type is uint32(%s)\n", pVal); + const char *signalPos = pVal + len - 3; + if (strcasecmp(signalPos, "u32") == 0) { + char *endptr = NULL; + int64_t result = strtoll(pVal, &endptr, 10); + if(endptr != signalPos){ // 78ri8 + *isValid = false; + buildInvalidDataMsg(msg, "invalid unsigned int", endptr); + }else if(!IS_VALID_UINT(result)){ + *isValid = false; + buildInvalidDataMsg(msg, "unsigned int out of range[0,4294967295]", endptr); + }else{ + kvVal->i = result; + *isValid = true; + } return true; } return false; } -static bool isBigInt(char *pVal, uint16_t len) { - if (len <= 3) { - return false; - } - if (strcasecmp(&pVal[len - 3], "i64") == 0) { - //printf("Type is int64(%s)\n", pVal); +static bool parseBigInt(SSmlKv *kvVal, bool *isValid, SMsgBuf *msg) { + const char *pVal = kvVal->value; + int32_t len = kvVal->valueLen; + if (len > 3 && strcasecmp(pVal + len - 3, "i64") == 0) { + char *endptr = NULL; + int64_t result = strtoll(pVal, &endptr, 10); + if(endptr != pVal + len - 3){ // 78ri8 + *isValid = false; + }else if(!IS_VALID_BIGINT(result)){ + *isValid = false; + }else{ + kvVal->i = result; + *isValid = true; + } + return true; + }else if (len > 1 && pVal[len - 1] == 'i') { + char *endptr = NULL; + int64_t result = strtoll(pVal, &endptr, 10); + if(endptr != pVal + len - 1){ // 78ri8 + *isValid = false; + buildInvalidDataMsg(msg, "invalid big int", endptr); + }else if(!IS_VALID_BIGINT(result)){ + *isValid = false; + buildInvalidDataMsg(msg, "big int out of range[-9223372036854775808,9223372036854775807]", endptr); + }else{ + kvVal->i = result; + *isValid = true; + } return true; } return false; } -static bool isBigUint(char *pVal, uint16_t len) { +static bool parseBigUint(SSmlKv *kvVal, bool *isValid, SMsgBuf *msg) { + const char *pVal = kvVal->value; + int32_t len = kvVal->valueLen; if (len <= 3) { return false; } if (pVal[0] == '-') { return false; } - if (strcasecmp(&pVal[len - 3], "u64") == 0) { - //printf("Type is uint64(%s)\n", pVal); + const char *signalPos = pVal + len - 3; + if (strcasecmp(signalPos, "u64") == 0) { + char *endptr = NULL; + uint64_t result = strtoull(pVal, &endptr, 10); + if(endptr != signalPos){ // 78ri8 + *isValid = false; + buildInvalidDataMsg(msg, "invalid unsigned big int", endptr); + }else if(!IS_VALID_UBIGINT(result)){ + *isValid = false; + buildInvalidDataMsg(msg, "unsigned big int out of range[0,18446744073709551615]", endptr); + }else{ + kvVal->u = result; + *isValid = true; + } return true; } return false; } -static bool isFloat(char *pVal, uint16_t len) { +static bool parseFloat(SSmlKv *kvVal, bool *isValid, SMsgBuf *msg) { + const char *pVal = kvVal->value; + int32_t len = kvVal->valueLen; + char *endptr = NULL; + float result = strtof(pVal, &endptr); + if(endptr == pVal + len && IS_VALID_FLOAT(result)){ // 78 + kvVal->f = result; + *isValid = true; + return true; + } + + if (len > 3 && len f = result; + *isValid = true; + } + return true; + } + return false; +} + +static bool parseDouble(SSmlKv *kvVal, bool *isValid, SMsgBuf *msg) { + const char *pVal = kvVal->value; + int32_t len = kvVal->valueLen; if (len <= 3) { return false; } - if (strcasecmp(&pVal[len - 3], "f32") == 0) { - //printf("Type is float(%s)\n", pVal); + const char *signalPos = pVal + len - 3; + if (len d = result; + *isValid = true; + } return true; } return false; } -static bool isDouble(char *pVal, uint16_t len) { - if (len <= 3) { - return false; - } - if (strcasecmp(&pVal[len - 3], "f64") == 0) { - //printf("Type is double(%s)\n", pVal); - return true; - } - return false; -} - -static bool isBool(char *pVal, uint16_t len, bool *bVal) { - if ((len == 1) && !strcasecmp(&pVal[len - 1], "t")) { +static bool parseBool(SSmlKv *kvVal) { + const char *pVal = kvVal->value; + int32_t len = kvVal->valueLen; + if ((len == 1) && pVal[len - 1] == 't') { //printf("Type is bool(%c)\n", pVal[len - 1]); - *bVal = true; + kvVal->i = true; return true; } - if ((len == 1) && !strcasecmp(&pVal[len - 1], "f")) { + if ((len == 1) && pVal[len - 1] == 'f') { //printf("Type is bool(%c)\n", pVal[len - 1]); - *bVal = false; + kvVal->i = false; return true; } - if((len == 4) && !strcasecmp(&pVal[len - 4], "true")) { + if((len == 4) && !strcasecmp(pVal, "true")) { //printf("Type is bool(%s)\n", &pVal[len - 4]); - *bVal = true; + kvVal->i = true; return true; } - if((len == 5) && !strcasecmp(&pVal[len - 5], "false")) { + if((len == 5) && !strcasecmp(pVal, "false")) { //printf("Type is bool(%s)\n", &pVal[len - 5]); - *bVal = false; + kvVal->i = false; return true; } return false; } -static bool isBinary(char *pVal, uint16_t len) { +static bool isBinary(const char *pVal, uint16_t len) { //binary: "abc" if (len < 2) { return false; @@ -1083,7 +804,7 @@ static bool isBinary(char *pVal, uint16_t len) { return false; } -static bool isNchar(char *pVal, uint16_t len) { +static bool isNchar(const char *pVal, uint16_t len) { //nchar: L"abc" if (len < 3) { return false; @@ -1095,266 +816,97 @@ static bool isNchar(char *pVal, uint16_t len) { return false; } -static bool convertStrToNumber(TAOS_SML_KV *pVal, char *str, SSmlLinesInfo* info) { - errno = 0; - uint8_t type = pVal->type; - int16_t length = pVal->length; - int64_t val_s = 0; - uint64_t val_u = 0; - double val_d = 0.0; - - strntolower_s(str, str, (int32_t)strlen(str)); - if (IS_FLOAT_TYPE(type)) { - val_d = strtod(str, NULL); - } else { - if (IS_SIGNED_NUMERIC_TYPE(type)) { - val_s = strtoll(str, NULL, 10); - } else { - val_u = strtoull(str, NULL, 10); - } - } - - if (errno == ERANGE) { - uError("SML:0x%"PRIx64" Convert number(%s) out of range", info->id, str); - return false; - } - - switch (type) { - case TSDB_DATA_TYPE_TINYINT: - if (!IS_VALID_TINYINT(val_s)) { - return false; - } - pVal->value = calloc(length, 1); - *(int8_t *)(pVal->value) = (int8_t)val_s; - break; - case TSDB_DATA_TYPE_UTINYINT: - if (!IS_VALID_UTINYINT(val_u)) { - return false; - } - pVal->value = calloc(length, 1); - *(uint8_t *)(pVal->value) = (uint8_t)val_u; - break; - case TSDB_DATA_TYPE_SMALLINT: - if (!IS_VALID_SMALLINT(val_s)) { - return false; - } - pVal->value = calloc(length, 1); - *(int16_t *)(pVal->value) = (int16_t)val_s; - break; - case TSDB_DATA_TYPE_USMALLINT: - if (!IS_VALID_USMALLINT(val_u)) { - return false; - } - pVal->value = calloc(length, 1); - *(uint16_t *)(pVal->value) = (uint16_t)val_u; - break; - case TSDB_DATA_TYPE_INT: - if (!IS_VALID_INT(val_s)) { - return false; - } - pVal->value = calloc(length, 1); - *(int32_t *)(pVal->value) = (int32_t)val_s; - break; - case TSDB_DATA_TYPE_UINT: - if (!IS_VALID_UINT(val_u)) { - return false; - } - pVal->value = calloc(length, 1); - *(uint32_t *)(pVal->value) = (uint32_t)val_u; - break; - case TSDB_DATA_TYPE_BIGINT: - if (!IS_VALID_BIGINT(val_s)) { - return false; - } - pVal->value = calloc(length, 1); - *(int64_t *)(pVal->value) = (int64_t)val_s; - break; - case TSDB_DATA_TYPE_UBIGINT: - if (!IS_VALID_UBIGINT(val_u)) { - return false; - } - pVal->value = calloc(length, 1); - *(uint64_t *)(pVal->value) = (uint64_t)val_u; - break; - case TSDB_DATA_TYPE_FLOAT: - if (!IS_VALID_FLOAT(val_d)) { - return false; - } - pVal->value = calloc(length, 1); - *(float *)(pVal->value) = (float)val_d; - break; - case TSDB_DATA_TYPE_DOUBLE: - if (!IS_VALID_DOUBLE(val_d)) { - return false; - } - pVal->value = calloc(length, 1); - *(double *)(pVal->value) = (double)val_d; - break; - default: - return false; - } - return true; -} -//len does not include '\0' from value. -bool convertSmlValueType(TAOS_SML_KV *pVal, char *value, - uint16_t len, SSmlLinesInfo* info, bool isTag) { - if (len <= 0) { - return false; - } - - //convert tags value to Nchar - if (isTag) { - pVal->type = TSDB_DATA_TYPE_NCHAR; - pVal->length = len; - pVal->value = calloc(pVal->length, 1); - memcpy(pVal->value, value, pVal->length); - return true; - } - - //integer number - bool has_sign; - if (isInteger(value, len, &has_sign)) { - pVal->type = has_sign ? TSDB_DATA_TYPE_BIGINT : TSDB_DATA_TYPE_UBIGINT; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - value[len - 1] = '\0'; - if (!isValidInteger(value) || !convertStrToNumber(pVal, value, info)) { - return false; - } - return true; - } - if (isTinyInt(value, len)) { - pVal->type = TSDB_DATA_TYPE_TINYINT; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - value[len - 2] = '\0'; - if (!isValidInteger(value) || !convertStrToNumber(pVal, value, info)) { - return false; - } - return true; - } - if (isTinyUint(value, len)) { - pVal->type = TSDB_DATA_TYPE_UTINYINT; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - value[len - 2] = '\0'; - if (!isValidInteger(value) || !convertStrToNumber(pVal, value, info)) { - return false; - } - return true; - } - if (isSmallInt(value, len)) { - pVal->type = TSDB_DATA_TYPE_SMALLINT; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - value[len - 3] = '\0'; - if (!isValidInteger(value) || !convertStrToNumber(pVal, value, info)) { - return false; - } - return true; - } - if (isSmallUint(value, len)) { - pVal->type = TSDB_DATA_TYPE_USMALLINT; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - value[len - 3] = '\0'; - if (!isValidInteger(value) || !convertStrToNumber(pVal, value, info)) { - return false; - } - return true; - } - if (isInt(value, len)) { - pVal->type = TSDB_DATA_TYPE_INT; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - value[len - 3] = '\0'; - if (!isValidInteger(value) || !convertStrToNumber(pVal, value, info)) { - return false; - } - return true; - } - if (isUint(value, len)) { - pVal->type = TSDB_DATA_TYPE_UINT; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - value[len - 3] = '\0'; - if (!isValidInteger(value) || !convertStrToNumber(pVal, value, info)) { - return false; - } - return true; - } - if (isBigInt(value, len)) { - pVal->type = TSDB_DATA_TYPE_BIGINT; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - value[len - 3] = '\0'; - if (!isValidInteger(value) || !convertStrToNumber(pVal, value, info)) { - return false; - } - return true; - } - if (isBigUint(value, len)) { - pVal->type = TSDB_DATA_TYPE_UBIGINT; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - value[len - 3] = '\0'; - if (!isValidInteger(value) || !convertStrToNumber(pVal, value, info)) { - return false; - } - return true; - } - //floating number - if (isFloat(value, len)) { +static bool convertSmlValue(SSmlKv *pVal, SMsgBuf *msg) { + // put high probability matching type first + bool isValid = false; + if (parseFloat(pVal, &isValid, msg)) { + if(!isValid) return false; pVal->type = TSDB_DATA_TYPE_FLOAT; pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - value[len - 3] = '\0'; - if (!isValidFloat(value) || !convertStrToNumber(pVal, value, info)) { - return false; - } - return true; - } - if (isDouble(value, len)) { - pVal->type = TSDB_DATA_TYPE_DOUBLE; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - value[len - 3] = '\0'; - if (!isValidFloat(value) || !convertStrToNumber(pVal, value, info)) { - return false; - } return true; } //binary - if (isBinary(value, len)) { + if (isBinary(pVal->value, pVal->valueLen)) { pVal->type = TSDB_DATA_TYPE_BINARY; - pVal->length = len - 2; - pVal->value = calloc(pVal->length, 1); - //copy after " - memcpy(pVal->value, value + 1, pVal->length); + pVal->length = pVal->valueLen - 2; + pVal->valueLen -= 2; + pVal->value = pVal->value++; return true; } //nchar - if (isNchar(value, len)) { + if (isNchar(pVal->value, pVal->valueLen)) { pVal->type = TSDB_DATA_TYPE_NCHAR; - pVal->length = len - 3; - pVal->value = calloc(pVal->length, 1); - //copy after L" - memcpy(pVal->value, value + 2, pVal->length); + pVal->length = pVal->valueLen - 3; + pVal->value = pVal->value+2; + return true; + } + if (parseDouble(pVal, &isValid, msg)) { + if(!isValid) return false; + pVal->type = TSDB_DATA_TYPE_DOUBLE; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + return true; } //bool - bool bVal; - if (isBool(value, len, &bVal)) { + if (parseBool(pVal)) { pVal->type = TSDB_DATA_TYPE_BOOL; pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - pVal->value = calloc(pVal->length, 1); - memcpy(pVal->value, &bVal, pVal->length); return true; } - //Handle default(no appendix) type as DOUBLE - if (isValidInteger(value) || isValidFloat(value)) { - pVal->type = TSDB_DATA_TYPE_DOUBLE; + if (parseTinyInt(pVal, &isValid, msg)) { + if(!isValid) return false; + pVal->type = TSDB_DATA_TYPE_TINYINT; pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - if (!convertStrToNumber(pVal, value, info)) { - return false; - } return true; } + if (parseTinyUint(pVal, &isValid, msg)) { + if(!isValid) return false; + pVal->type = TSDB_DATA_TYPE_UTINYINT; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + return true; + } + if (parseSmallInt(pVal, &isValid, msg)) { + if(!isValid) return false; + pVal->type = TSDB_DATA_TYPE_SMALLINT; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + return true; + } + if (parseSmallUint(pVal, &isValid, msg)) { + if(!isValid) return false; + pVal->type = TSDB_DATA_TYPE_USMALLINT; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + return true; + } + if (parseInt(pVal, &isValid, msg)) { + if(!isValid) return false; + pVal->type = TSDB_DATA_TYPE_INT; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + return true; + } + if (parseUint(pVal, &isValid, msg)) { + if(!isValid) return false; + pVal->type = TSDB_DATA_TYPE_UINT; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + return true; + } + if (parseBigInt(pVal, &isValid, msg)) { + if(!isValid) return false; + pVal->type = TSDB_DATA_TYPE_BIGINT; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + return true; + } + if (parseBigUint(pVal, &isValid, msg)) { + if(!isValid) return false; + pVal->type = TSDB_DATA_TYPE_UBIGINT; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + return true; + } + + buildInvalidDataMsg(msg, "invalid data", pVal->value); return false; } - - bool checkDuplicateKey(char *key, SHashObj *pHash, SSmlLinesInfo* info) { char *val = NULL; val = taosHashGet(pHash, key, strlen(key)); @@ -1369,21 +921,6 @@ bool checkDuplicateKey(char *key, SHashObj *pHash, SSmlLinesInfo* info) { return false; } -//Table name can only contain digits(0-9),alphebet(a-z),underscore(_) -int32_t isValidChildTableName(const char *pTbName, int16_t len, SSmlLinesInfo* info) { - if (len > TSDB_TABLE_NAME_LEN - 1) { - uError("SML:0x%"PRIx64" child table name cannot exceeds %d characters", info->id, TSDB_TABLE_NAME_LEN - 1); - return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; - } - const char *cur = pTbName; - for (int i = 0; i < len; ++i) { - if(!isdigit(cur[i]) && !isalpha(cur[i]) && (cur[i] != '_')) { - return TSDB_CODE_TSC_LINE_SYNTAX_ERROR; - } - } - return TSDB_CODE_SUCCESS; -} - int32_t parseSml(const char* sql, TAOS_PARSE_ELEMENTS *elements){ if(!sql) return TSDB_CODE_SML_INVALID_DATA; while (*sql != '\0') { // jump the space at the begining @@ -1428,8 +965,12 @@ int32_t parseSml(const char* sql, TAOS_PARSE_ELEMENTS *elements){ } if(!elements->cols) return TSDB_CODE_SML_INVALID_DATA; + bool isInQuote = false; while (*sql != '\0') { - if(*sql == SPACE && *(sql - 1) != SLASH) { + if(*sql == QUOTE && *(sql - 1) != SLASH){ + isInQuote = !isInQuote; + } + if(!isInQuote && *sql == SPACE && *(sql - 1) != SLASH) { break; } sql++; @@ -1444,12 +985,27 @@ int32_t parseSml(const char* sql, TAOS_PARSE_ELEMENTS *elements){ } sql++; } + if(elements->timestamp){ + elements->timestampLen = sql - elements->timestamp; + } return TSDB_CODE_SUCCESS; } -int32_t parseSmlKV(const char* data, int32_t len, SArray *cols, bool isTag){ +bool parseSmlCols(const char* data, int32_t len, SArray *cols, bool isTag, SMsgBuf *msg){ + if(isTag && len == 0){ + SSmlKv *kv = taosMemoryCalloc(sizeof(SSmlKv), 1); + kv->key = TAG; + kv->keyLen = TAGNAMELEN; + kv->value = TAG; + kv->valueLen = TAGNAMELEN; + kv->type = TSDB_DATA_TYPE_NCHAR; + if(cols) taosArrayPush(cols, &kv); + return true; + } + for(int i = 0; i < len; i++){ + // parse key const char *key = data + i; int32_t keyLen = 0; while(i < len){ @@ -1459,23 +1015,27 @@ int32_t parseSmlKV(const char* data, int32_t len, SArray *cols, bool isTag){ } i++; } - if(keyLen == 0){ + if(keyLen == 0 || keyLen >= TSDB_COL_NAME_LEN){ + buildInvalidDataMsg(msg, "invalid key or key is too long than 64", key); return TSDB_CODE_SML_INVALID_DATA; } + // parse value i++; const char *value = data + i; - int32_t valueLen = 0; while(i < len){ if(data[i] == COMMA && i > 0 && data[i-1] != SLASH){ - valueLen = data + i - value; break; } i++; } + int32_t valueLen = data + i - value; if(valueLen == 0){ + buildInvalidDataMsg(msg, "invalid value", value); return TSDB_CODE_SML_INVALID_DATA; } + + // add kv to SSmlKv SSmlKv *kv = taosMemoryCalloc(sizeof(SSmlKv), 1); kv->key = key; kv->keyLen = keyLen; @@ -1483,9 +1043,15 @@ int32_t parseSmlKV(const char* data, int32_t len, SArray *cols, bool isTag){ kv->valueLen = valueLen; if(isTag){ kv->type = TSDB_DATA_TYPE_NCHAR; + }else{ + if(!convertSmlValue(kv, msg)){ + return TSDB_CODE_SML_INVALID_DATA; + } } + if(cols) taosArrayPush(cols, &kv); } + return TSDB_CODE_SUCCESS; } @@ -1526,13 +1092,13 @@ static int64_t getTimeStampNow(int32_t precision) { } } -static int32_t isValidateTimeStamp(const char *pVal, int32_t len) { +static bool isValidateTimeStamp(const char *pVal, int32_t len) { for (int i = 0; i < len; ++i) { if (!isdigit(pVal[i])) { - return TSDB_CODE_TSC_INVALID_TIME_STAMP; + return false; } } - return TSDB_CODE_SUCCESS; + return true; } static int32_t getTsType(int32_t len) { @@ -1541,47 +1107,53 @@ static int32_t getTsType(int32_t len) { } else if (len == TSDB_TIME_PRECISION_MILLI_DIGITS) { return TSDB_TIME_PRECISION_MILLI_DIGITS; } else { - return TSDB_CODE_TSC_INVALID_TIME_STAMP; + return -1; } } -static int32_t parseSmlTS(const char* data, SArray *tags, int8_t tsType, SMLProtocolType protocolType){ - int64_t *ts = taosMemoryCalloc(1, sizeof(int64_t)); +static int32_t parseSmlTS(const char* data, int32_t len, SArray *tags, SSmlLinesInfo* info){ + int64_t ts = 0; if(data == NULL){ - if(protocolType == TSDB_SML_LINE_PROTOCOL){ - *ts = getTimeStampNow(tsType); - }else{ - goto cleanup; + if(info->protocol != TSDB_SML_LINE_PROTOCOL){ + buildInvalidDataMsg(&info->msgBuf, "timestamp can not be null", NULL); + return TSDB_CODE_TSC_INVALID_TIME_STAMP; } + ts = getTimeStampNow(info->tsType); }else{ - int32_t len = strlen(data); int ret = isValidateTimeStamp(data, len); if(!ret){ - goto cleanup; + buildInvalidDataMsg(&info->msgBuf, "timestamp must be digit", data); + return TSDB_CODE_TSC_INVALID_TIME_STAMP; } - if(protocolType != TSDB_SML_LINE_PROTOCOL){ + int32_t tsType = -1; + if(info->protocol != TSDB_SML_LINE_PROTOCOL){ tsType = getTsType(len); - if (tsType == TSDB_CODE_TSC_INVALID_TIME_STAMP) { - goto cleanup; + if (tsType == -1) { + buildInvalidDataMsg(&info->msgBuf, "timestamp precision can only be seconds(10 digits) or milli seconds(13 digits)", data); + return TSDB_CODE_TSC_INVALID_TIME_STAMP; } + }else{ + tsType = info->tsType; } - *ts = getTimeStampValue(data, tsType); - if(*ts == -1){ - goto cleanup; + ts = getTimeStampValue(data, tsType); + if(ts == -1){ + buildInvalidDataMsg(&info->msgBuf, "invalid timestamp", data); + return TSDB_CODE_TSC_INVALID_TIME_STAMP; } } SSmlKv *kv = taosMemoryCalloc(sizeof(SSmlKv), 1); - kv->value = (const char*)ts; - kv->valueLen = sizeof(int64_t); + if(!kv){ + return TSDB_CODE_OUT_OF_MEMORY; + } + + kv->key = TS; + kv->keyLen = TSNAMELEN; + kv->i = ts; kv->type = TSDB_DATA_TYPE_TIMESTAMP; kv->length = (int16_t)tDataTypes[kv->type].bytes; if(tags) taosArrayPush(tags, &kv); return TSDB_CODE_SUCCESS; - -cleanup: - taosMemoryFree(ts); - return TSDB_CODE_TSC_INVALID_TIME_STAMP; } //int32_t parseSmlCols(const char* data, SArray *cols){ @@ -1627,19 +1199,21 @@ cleanup: // return TSDB_CODE_SUCCESS; //} -void updateMeta(SSmlSTableMeta* tableMeta, SArray *tags, SArray *cols){ +bool updateMeta(SSmlSTableMeta* tableMeta, SArray *tags, SArray *cols, SMsgBuf *msg){ if(tags){ for (int i = 0; i < taosArrayGetSize(tags); ++i) { SSmlKv *kv = taosArrayGetP(tags, i); + ASSERT(kv->type == TSDB_DATA_TYPE_NCHAR); + SSmlKv **value = taosHashGet(tableMeta->tagHash, kv->key, kv->keyLen); if(value){ - if(kv->type != (*value)->type){ - // todo + ASSERT((*value)->type == TSDB_DATA_TYPE_NCHAR); + if(kv->valueLen > (*value)->valueLen){ // tags type is nchar + *value = kv; } }else{ taosHashPut(tableMeta->tagHash, kv->key, kv->keyLen, &kv, POINTER_BYTES); } - } } @@ -1649,7 +1223,14 @@ void updateMeta(SSmlSTableMeta* tableMeta, SArray *tags, SArray *cols){ SSmlKv **value = taosHashGet(tableMeta->fieldHash, kv->key, kv->keyLen); if(value){ if(kv->type != (*value)->type){ - // todo + buildInvalidDataMsg(msg, "the type is not the same like before", kv->key); + return false; + }else{ + if(IS_VAR_DATA_TYPE(kv->type)){ // update string len, if bigger + if(kv->valueLen > (*value)->valueLen){ + *value = kv; + } + } } }else{ taosHashPut(tableMeta->fieldHash, kv->key, kv->keyLen, &kv, POINTER_BYTES); @@ -1687,21 +1268,33 @@ static int32_t smlParseLine(const char* sql, SSmlLinesInfo* info) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } - parseSmlTS(elements.timestamp, cols, info->tsType); - ret = parseSmlCols(elements.cols, elements.colsLen, cols, false); + ret = parseSmlTS(elements.timestamp, elements.timestampLen, cols, info); if(ret != TSDB_CODE_SUCCESS){ return ret; } + ret = parseSmlCols(elements.cols, elements.colsLen, cols, false, &info->msgBuf); + if(ret != TSDB_CODE_SUCCESS){ + return ret; + } + if(taosArrayGetSize(cols) > TSDB_MAX_COLUMNS){ + buildInvalidDataMsg(&info->msgBuf, "too many columns than 4096", NULL); + return TSDB_CODE_SML_INVALID_DATA; + } TAOS_SML_DATA_POINT_TAGS** oneTable = taosHashGet(info->childTables, elements.measure, elements.measureTagsLen); if(oneTable){ SSmlSTableMeta** tableMeta = taosHashGet(info->superTables, elements.measure, elements.measureLen); ASSERT(tableMeta); - updateMeta(*tableMeta, NULL, cols); // update meta - + ret = updateMeta(*tableMeta, NULL, cols, &info->msgBuf); // update meta + if(!ret){ + return TSDB_CODE_SML_INVALID_DATA; + } taosArrayPush((*oneTable)->cols, &cols); }else{ TAOS_SML_DATA_POINT_TAGS *tag = taosMemoryCalloc(sizeof(TAOS_SML_DATA_POINT_TAGS), 1); + if(!tag){ + return TSDB_CODE_OUT_OF_MEMORY; + } tag->cols = taosArrayInit(16, POINTER_BYTES); if (tag->cols == NULL) { uError("SML:0x%"PRIx64" taos_insert_lines failed to allocate memory", info->id); @@ -1709,19 +1302,37 @@ static int32_t smlParseLine(const char* sql, SSmlLinesInfo* info) { } taosArrayPush(tag->cols, &cols); + tag->colsColumn = taosArrayInit(16, POINTER_BYTES); + if (tag->cols == NULL) { + uError("SML:0x%"PRIx64" taos_insert_lines failed to allocate memory", info->id); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + tag->tags = taosArrayInit(16, POINTER_BYTES); if (tag->tags == NULL) { uError("SML:0x%"PRIx64" taos_insert_lines failed to allocate memory", info->id); return TSDB_CODE_TSC_OUT_OF_MEMORY; } - ret = parseSmlTags(elements.tags, elements.tagsLen, tag->tags); + ret = parseSmlCols(elements.tags, elements.tagsLen, tag->tags, true, &info->msgBuf); if(ret != TSDB_CODE_SUCCESS){ return ret; } + if(taosArrayGetSize(tag->tags) > TSDB_MAX_TAGS){ + buildInvalidDataMsg(&info->msgBuf, "too many tags than 128", NULL); + return TSDB_CODE_SML_INVALID_DATA; + } + + tag->sTableName = elements.measure; + tag->sTableNameLen = elements.measureLen; + buildSmlChildTableName(tag); + SSmlSTableMeta** tableMeta = taosHashGet(info->superTables, elements.measure, elements.measureLen); if(tableMeta){ // update meta - updateMeta(*tableMeta, tag->tags, cols); + ret = updateMeta(*tableMeta, tag->tags, cols, &info->msgBuf); + if(!ret){ + return TSDB_CODE_SML_INVALID_DATA; + } }else{ SSmlSTableMeta* meta = taosMemoryCalloc(sizeof(SSmlSTableMeta), 1); insertMeta(meta, tag->tags, cols); @@ -1771,9 +1382,8 @@ static SSmlLinesInfo* smlBuildInfo(TAOS* taos, SRequestObj* request, SMLProtocol goto cleanup; } info->pRequest = request; - info->msgBuf = info->pRequest->msgBuf; - info->msgLen = ERROR_MSG_BUF_DEFAULT_SIZE; - + info->msgBuf.buf = info->pRequest->msgBuf; + info->msgBuf.len = ERROR_MSG_BUF_DEFAULT_SIZE; info->childTables = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, false); info->superTables = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, false); @@ -1868,7 +1478,7 @@ TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int pr SRequestObj* request = createRequest(taos, NULL, NULL, TSDB_SQL_INSERT); switch (protocol) { - case TSDB_SML_LINE_PROTOCOL: + case TSDB_SML_LINE_PROTOCOL:{ int32_t tsType = convertPrecisionType(precision); if(tsType == -1){ request->code = TSDB_CODE_SML_INVALID_PRECISION_TYPE; @@ -1877,6 +1487,7 @@ TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int pr code = sml_insert_lines(taos, request, lines, numLines, protocol, tsType); break; + } case TSDB_SML_TELNET_PROTOCOL: //code = taos_insert_telnet_lines(taos, lines, numLines, protocol, tsType, &affected_rows); break; diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index 0e87d8611e..54e05f9264 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -1539,7 +1539,7 @@ static int32_t smlBoundColumns(SArray *cols, SParsedDataColInfo* pColList, SSche col_id_t lastColIdx = -1; // last column found for (int i = 0; i < taosArrayGetSize(cols); ++i) { SSmlKv *kv = taosArrayGetP(cols, i); - SToken sToken = {.n=kv->keyLen, .z=kv->key}; + SToken sToken = {.n=kv->keyLen, .z=(char*)kv->key}; col_id_t t = lastColIdx + 1; col_id_t index = findCol(&sToken, t, nCols, pSchema); if (index < 0 && t > 0) { @@ -1596,18 +1596,17 @@ static int32_t smlBoundColumns(SArray *cols, SParsedDataColInfo* pColList, SSche return TSDB_CODE_SUCCESS; } -static int32_t smlParseTags(SArray *cols, SKVRowBuilder *tagsBuilder, SParsedDataColInfo* tags, SSchema* pSchema, SVCreateTbReq *createTblReq) { +static int32_t smlParseTags(SArray *cols, SKVRowBuilder *tagsBuilder, SParsedDataColInfo* tags, SSchema* pSchema, SVCreateTbReq *createTblReq, SMsgBuf *msg) { if (tdInitKVRowBuilder(tagsBuilder) < 0) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } SKvParam param = {.builder = tagsBuilder}; for (int i = 0; i < tags->numOfBound; ++i) { - SSchema* pTagSchema = &pSchema[tags->boundColumns[i] - 1]; // colId starts with 1 param.schema = pTagSchema; SSmlKv *kv = taosArrayGetP(cols, i); - KvRowAppend(NULL, kv->value, kv->valueLen, ¶m) ; + KvRowAppend(msg, kv->value, kv->valueLen, ¶m) ; } @@ -1630,18 +1629,33 @@ int32_t smlBind(void *handle, SArray *tags, SArray *cols, STableMeta *pTableMeta SmlExecHandle *smlHandle = (SmlExecHandle *)handle; SSchema* pTagsSchema = getTableTagSchema(pTableMeta); - smlBoundColumns(tags, &smlHandle->tags, pTagsSchema); - smlParseTags(tags, &smlHandle->tagsBuilder, &smlHandle->tags, pTagsSchema, &smlHandle->createTblReq); + setBoundColumnInfo(&smlHandle->tags, pTagsSchema, getNumOfTags(pTableMeta)); + int ret = smlBoundColumns(tags, &smlHandle->tags, pTagsSchema); + if(ret != TSDB_CODE_SUCCESS){ + buildInvalidOperationMsg(&pBuf, "bound tags error"); + return ret; + } + ret = smlParseTags(tags, &smlHandle->tagsBuilder, &smlHandle->tags, pTagsSchema, &smlHandle->createTblReq, &pBuf); + if(ret != TSDB_CODE_SUCCESS){ + return ret; + } STableDataBlocks* pDataBlock = NULL; - getDataBlockFromList(smlHandle->pBlockHash, pTableMeta->uid, TSDB_DEFAULT_PAYLOAD_SIZE, + ret = getDataBlockFromList(smlHandle->pBlockHash, pTableMeta->uid, TSDB_DEFAULT_PAYLOAD_SIZE, sizeof(SSubmitBlk), getTableInfo(pTableMeta).rowSize, pTableMeta, &pDataBlock, NULL, &smlHandle->createTblReq); + if(ret != TSDB_CODE_SUCCESS){ + buildInvalidOperationMsg(&pBuf, "create data block error"); + return ret; + } SSchema* pSchema = getTableColumnSchema(pTableMeta); - smlBoundColumns(taosArrayGetP(cols, 0), &pDataBlock->boundColumnInfo, pSchema); - + ret = smlBoundColumns(taosArrayGetP(cols, 0), &pDataBlock->boundColumnInfo, pSchema); + if(ret != TSDB_CODE_SUCCESS){ + buildInvalidOperationMsg(&pBuf, "bound cols error"); + return ret; + } int32_t extendedRowSize = getExtendedRowSize(pDataBlock); SParsedDataColInfo* spd = &pDataBlock->boundColumnInfo; SRowBuilder* pBuilder = &pDataBlock->rowBuilder; @@ -1649,8 +1663,11 @@ int32_t smlBind(void *handle, SArray *tags, SArray *cols, STableMeta *pTableMeta initRowBuilder(&pDataBlock->rowBuilder, pDataBlock->pTableMeta->sversion, &pDataBlock->boundColumnInfo); - allocateMemForSize(pDataBlock, extendedRowSize * rowNum); - + ret = allocateMemForSize(pDataBlock, extendedRowSize * rowNum); + if(ret != TSDB_CODE_SUCCESS){ + buildInvalidOperationMsg(&pBuf, "allocate memory error"); + return ret; + } for (int32_t r = 0; r < rowNum; ++r) { STSRow* row = (STSRow*)(pDataBlock->pData + pDataBlock->size); // skip the SSubmitBlk header tdSRowResetBuf(pBuilder, row); From ec97d053000bb633b882d4b6861ce122369b02a6 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 29 Apr 2022 17:39:33 +0800 Subject: [PATCH 04/31] refactor:add schemaless function --- include/client/taos.h | 2 +- include/libs/parser/parser.h | 8 +- source/client/inc/clientSml.h | 116 ---- source/client/src/clientSml.c | 1007 +++++++++++++++++----------- source/libs/parser/src/parInsert.c | 58 +- 5 files changed, 657 insertions(+), 534 deletions(-) delete mode 100644 source/client/inc/clientSml.h diff --git a/include/client/taos.h b/include/client/taos.h index 55deee4fad..fa27eb2459 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -199,7 +199,7 @@ DLL_EXPORT void taos_close_stream(TAOS_STREAM *tstr); #endif DLL_EXPORT int taos_load_table_info(TAOS *taos, const char *tableNameList); -DLL_EXPORT TAOS_RES *taos_schemaless_insert(TAOS *taos, char *lines[], int numLines, int protocol, int precision); +DLL_EXPORT TAOS_RES *taos_schemaless_insert(TAOS *taos, char *lines[], int numLines, int protocol, int precision, bool dataFormat); /* --------------------------TMQ INTERFACE------------------------------- */ diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index 248a6c1237..f608e38226 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -93,12 +93,12 @@ int32_t qBindStmtSingleColValue(void *pBlock, TAOS_BIND_v2 *bind, char *msgBuf, int32_t qBuildStmtColFields(void *pDataBlock, int32_t *fieldNum, TAOS_FIELD** fields); int32_t qBuildStmtTagFields(void *pBlock, void *boundTags, int32_t *fieldNum, TAOS_FIELD** fields); int32_t qBindStmtTagsValue(void *pBlock, void *boundTags, int64_t suid, char *tName, TAOS_BIND_v2 *bind, char *msgBuf, int32_t msgBufLen); -void destroyBoundColumnInfo(void* pBoundInfo); +void destroyBoundColumnInfo(void* pBoundInfo); int32_t qCreateSName(SName* pName, const char* pTableName, int32_t acctId, char* dbName, char *msgBuf, int32_t msgBufLen); -void* tscSmlInitHandle(SQuery *pQuery); -void tscSmlDestroyHandle(void *pHandle); -int32_t smlBind(void *handle, SArray *tags, SArray *cols, STableMeta *pTableMeta, char *msgBuf, int16_t msgBufLen); +void* smlInitHandle(SQuery *pQuery); +void smlDestroyHandle(void *pHandle); +int32_t smlBindData(void *handle, SArray *tags, SArray *cols, STableMeta *pTableMeta, char *msgBuf, int16_t msgBufLen); int32_t smlBuildOutput(void* handle, SHashObj* pVgHash); #ifdef __cplusplus diff --git a/source/client/inc/clientSml.h b/source/client/inc/clientSml.h deleted file mode 100644 index c970f1e954..0000000000 --- a/source/client/inc/clientSml.h +++ /dev/null @@ -1,116 +0,0 @@ -/* - * Copyright (c) 2021 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_CLIENTSML_H -#define TDENGINE_CLIENTSML_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include "thash.h" -#include "clientInt.h" -#include "catalog.h" - -typedef TSDB_SML_PROTOCOL_TYPE SMLProtocolType; - -typedef struct { - const char* measure; - const char* tags; - const char* cols; - const char* timestamp; - - int32_t measureLen; - int32_t measureTagsLen; - int32_t tagsLen; - int32_t colsLen; - int32_t timestampLen; -} TAOS_PARSE_ELEMENTS; - -typedef struct { - const char *sTableName; // super table name - uint8_t sTableNameLen; - char childTableName[TSDB_TABLE_NAME_LEN]; - uint64_t uid; - - SArray *tags; - SArray *cols; // elements are SHashObj for find by key quickly - - SArray colsColumn; // elements are cols key string -} TAOS_SML_DATA_POINT_TAGS; - -typedef struct SSmlSTableMeta { -// char *sTableName; // super table name -// uint8_t sTableNameLen; - uint8_t precision; // the number of precision - SHashObj *tagHash; - SHashObj *fieldHash; -} SSmlSTableMeta; - -typedef struct SMsgBuf { - int32_t len; - char *buf; -} SMsgBuf; - -typedef struct { - uint64_t id; - - SMLProtocolType protocol; - int32_t tsType; - - SHashObj *childTables; - SHashObj *superTables; - - SHashObj *metaHashObj; - SHashObj *pVgHash; - - void *exec; - - STscObj *taos; - SCatalog *pCatalog; - SRequestObj *pRequest; - SQuery *pQuery; - - int32_t affectedRows; - SMsgBuf msgBuf; -} SSmlLinesInfo; - -int smlInsert(TAOS* taos, SSmlLinesInfo* info); - -bool checkDuplicateKey(char *key, SHashObj *pHash, SSmlLinesInfo* info); -bool isValidInteger(char *str); -bool isValidFloat(char *str); - -int32_t isValidChildTableName(const char *pTbName, int16_t len, SSmlLinesInfo* info); - -bool convertSmlValueType(SSmlKv *pVal, char *value, - uint16_t len, SSmlLinesInfo* info, bool isTag); -int32_t convertSmlTimeStamp(SSmlKv *pVal, char *value, - uint16_t len, SSmlLinesInfo* info); - - -int sml_insert_lines(TAOS* taos, SRequestObj* request, char* lines[], int numLines, SMLProtocolType protocol, - SMLTimeStampType tsType); -int sml_insert_telnet_lines(TAOS* taos, char* lines[], int numLines, SMLProtocolType protocol, - SMLTimeStampType tsType, int* affectedRows); -int sml_insert_json_payload(TAOS* taos, char* payload, SMLProtocolType protocol, - SMLTimeStampType tsType, int* affectedRows); - - -#ifdef __cplusplus -} -#endif - -#endif // TDENGINE_CLIENTSML_H diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 3739c83109..bde9aeed49 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -1,29 +1,21 @@ +#include #include #include #include #include -#include "clientSml.h" - -#include "tdef.h" -#include "ttypes.h" -#include "tmsg.h" -#include "tlog.h" #include "query.h" -#include "taoserror.h" #include "taos.h" -#include "ttime.h" +#include "taoserror.h" +#include "tdef.h" +#include "tlog.h" +#include "tmsg.h" #include "tstrbuild.h" - - -typedef struct { - char sTableName[TSDB_TABLE_NAME_LEN]; - SHashObj* tagHash; - SHashObj* fieldHash; - SArray* tags; //SArray - SArray* fields; //SArray - uint8_t precision; -} SSmlSTableSchema; +#include "ttime.h" +#include "ttypes.h" +#include "tcommon.h" +#include "catalog.h" +//================================================================================================= #define SPACE ' ' #define COMMA ',' @@ -32,43 +24,8 @@ typedef struct { #define SLASH '\\' #define tsMaxSQLStringLen (1024*1024) -#define TSNAMELEN 2 -#define TAGNAMELEN 3 //================================================================================================= - -static uint64_t linesSmlHandleId = 0; -static const char* TS = "ts"; -static const char* TAG = "tag"; - - -uint64_t genLinesSmlId() { - uint64_t id; - - do { - id = atomic_add_fetch_64(&linesSmlHandleId, 1); - } while (id == 0); - - return id; -} - -static int32_t buildInvalidDataMsg(SMsgBuf* pBuf, const char *msg1, const char *msg2) { - if(msg1) snprintf(pBuf->buf, pBuf->len, "%s:", msg1); - if(msg2) strncpy(pBuf->buf, msg2, pBuf->len); - return TSDB_CODE_SML_INVALID_DATA; -} - -int compareSmlColKv(const void* p1, const void* p2) { - SSmlKv* kv1 = (SSmlKv *)p1; - SSmlKv* kv2 = (SSmlKv*)p2; - int kvLen1 = (int)strlen(kv1->key); - int kvLen2 = (int)strlen(kv2->key); - int res = strncasecmp(kv1->key, kv2->key, MIN(kvLen1, kvLen2)); - if (res != 0) { - return res; - } else { - return kvLen1-kvLen2; - } -} +typedef TSDB_SML_PROTOCOL_TYPE SMLProtocolType; typedef enum { SCHEMA_ACTION_CREATE_STABLE, @@ -97,10 +54,107 @@ typedef struct { }; } SSchemaAction; -static int32_t buildSmlChildTableName(TAOS_SML_DATA_POINT_TAGS *tags) { +typedef struct { + const char* measure; + const char* tags; + const char* cols; + const char* timestamp; + + int32_t measureLen; + int32_t measureTagsLen; + int32_t tagsLen; + int32_t colsLen; + int32_t timestampLen; +} SSmlLineInfo; + +typedef struct { + const char *sTableName; // super table name + uint8_t sTableNameLen; + char childTableName[TSDB_TABLE_NAME_LEN]; + uint64_t uid; + + SArray *tags; + + // colsFormat store cols formated, for quick parse, if info->formatData is true + SArray *colsFormat; // elements are SArray + + // cols & colsColumn store cols un formated + SArray *cols; // elements are SHashObj for find by key quickly + SHashObj *columnsHash; // elements are , just for judge if key exists quickly. +} SSmlTableInfo; + +typedef struct { + SHashObj *tagHash; + SHashObj *fieldHash; + STableMeta *tableMeta; +} SSmlSTableMeta; + +typedef struct { + int32_t len; + char *buf; +} SSmlMsgBuf; + +typedef struct { + uint64_t id; + + SMLProtocolType protocol; + int8_t precision; + bool dataFormat; // true means that the name, number and order of keys in each line are the same + + SHashObj *childTables; + SHashObj *superTables; + SHashObj *pVgHash; + void *exec; + + STscObj *taos; + SCatalog *pCatalog; + SRequestObj *pRequest; + SQuery *pQuery; + + int32_t affectedRows; + SSmlMsgBuf msgBuf; +} SSmlHandle; +//================================================================================================= + +static uint64_t linesSmlHandleId = 0; +static const char* TS = "ts"; +static const char* TAG = "tagNone"; + +//================================================================================================= + +static uint64_t smlGenId() { + uint64_t id; + + do { + id = atomic_add_fetch_64(&linesSmlHandleId, 1); + } while (id == 0); + + return id; +} + +static int32_t smlBuildInvalidDataMsg(SSmlMsgBuf* pBuf, const char *msg1, const char *msg2) { + if(msg1) snprintf(pBuf->buf, pBuf->len, "%s:", msg1); + if(msg2) strncpy(pBuf->buf, msg2, pBuf->len); + return TSDB_CODE_SML_INVALID_DATA; +} + +static int smlCompareKv(const void* p1, const void* p2) { + SSmlKv* kv1 = (SSmlKv *)p1; + SSmlKv* kv2 = (SSmlKv*)p2; + int kvLen1 = (int)strlen(kv1->key); + int kvLen2 = (int)strlen(kv2->key); + int res = strncasecmp(kv1->key, kv2->key, MIN(kvLen1, kvLen2)); + if (res != 0) { + return res; + } else { + return kvLen1-kvLen2; + } +} + +static void smlBuildChildTableName(SSmlTableInfo *tags) { int32_t size = taosArrayGetSize(tags->tags); ASSERT(size > 0); - qsort(tags->tags, size, POINTER_BYTES, compareSmlColKv); + qsort(tags->tags, size, POINTER_BYTES, smlCompareKv); SStringBuilder sb = {0}; taosStringBuilderAppendStringLen(&sb, tags->sTableName, tags->sTableNameLen); @@ -120,12 +174,10 @@ static int32_t buildSmlChildTableName(TAOS_SML_DATA_POINT_TAGS *tags) { snprintf(tags->childTableName, TSDB_TABLE_NAME_LEN, "t_%016"PRIx64"%016"PRIx64, digest1, digest2); taosStringBuilderDestroy(&sb); tags->uid = digest1; - uDebug("SML: child table name: %s", tags->childTableName); - return 0; } -static int32_t generateSchemaAction(SSchema* pointColField, SHashObj* dbAttrHash, SArray* dbAttrArray, bool isTag, char sTableName[], - SSchemaAction* action, bool* actionNeeded, SSmlLinesInfo* info) { +static int32_t smlGenerateSchemaAction(SSchema* pointColField, SHashObj* dbAttrHash, SArray* dbAttrArray, bool isTag, char sTableName[], + SSchemaAction* action, bool* actionNeeded, SSmlHandle* info) { // char fieldName[TSDB_COL_NAME_LEN] = {0}; // strcpy(fieldName, pointColField->name); // @@ -168,7 +220,7 @@ static int32_t generateSchemaAction(SSchema* pointColField, SHashObj* dbAttrHash return 0; } -static int32_t buildColumnDescription(SSmlKv* field, char* buf, int32_t bufSize, int32_t* outBytes) { +static int32_t smlBuildColumnDescription(SSmlKv* field, char* buf, int32_t bufSize, int32_t* outBytes) { uint8_t type = field->type; char tname[TSDB_TABLE_NAME_LEN] = {0}; memcpy(tname, field->key, field->keyLen); @@ -185,7 +237,7 @@ static int32_t buildColumnDescription(SSmlKv* field, char* buf, int32_t bufSize, return 0; } -static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInfo* info) { +static int32_t smlApplySchemaAction(SSmlHandle* info, SSchemaAction* action) { int32_t code = 0; int32_t outBytes = 0; char *result = (char *)taosMemoryCalloc(1, tsMaxSQLStringLen+1); @@ -195,8 +247,8 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInf switch (action->action) { case SCHEMA_ACTION_ADD_COLUMN: { int n = sprintf(result, "alter stable %s add column ", action->alterSTable.sTableName); - buildColumnDescription(action->alterSTable.field, result+n, capacity-n, &outBytes); - TAOS_RES* res = taos_query(taos, result); //TODO async doAsyncQuery + smlBuildColumnDescription(action->alterSTable.field, result+n, capacity-n, &outBytes); + TAOS_RES* res = taos_query(info->taos, result); //TODO async doAsyncQuery code = taos_errno(res); const char* errStr = taos_errstr(res); char* begin = strstr(errStr, "duplicated column names"); @@ -208,7 +260,7 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInf // if (code == TSDB_CODE_MND_FIELD_ALREADY_EXIST || code == TSDB_CODE_MND_TAG_ALREADY_EXIST || tscDupColNames) { if (code == TSDB_CODE_MND_TAG_ALREADY_EXIST || tscDupColNames) { - TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); + TAOS_RES* res2 = taos_query(info->taos, "RESET QUERY CACHE"); code = taos_errno(res2); if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%" PRIx64 " apply schema action. reset query cache. error: %s", info->id, taos_errstr(res2)); @@ -220,9 +272,9 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInf } case SCHEMA_ACTION_ADD_TAG: { int n = sprintf(result, "alter stable %s add tag ", action->alterSTable.sTableName); - buildColumnDescription(action->alterSTable.field, + smlBuildColumnDescription(action->alterSTable.field, result+n, capacity-n, &outBytes); - TAOS_RES* res = taos_query(taos, result); //TODO async doAsyncQuery + TAOS_RES* res = taos_query(info->taos, result); //TODO async doAsyncQuery code = taos_errno(res); const char* errStr = taos_errstr(res); char* begin = strstr(errStr, "duplicated column names"); @@ -234,7 +286,7 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInf // if (code ==TSDB_CODE_MND_TAG_ALREADY_EXIST || code == TSDB_CODE_MND_FIELD_ALREAY_EXIST || tscDupColNames) { if (code ==TSDB_CODE_MND_TAG_ALREADY_EXIST || tscDupColNames) { - TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); + TAOS_RES* res2 = taos_query(info->taos, "RESET QUERY CACHE"); code = taos_errno(res2); if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%" PRIx64 " apply schema action. reset query cache. error: %s", info->id, taos_errstr(res2)); @@ -246,9 +298,9 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInf } case SCHEMA_ACTION_CHANGE_COLUMN_SIZE: { int n = sprintf(result, "alter stable %s modify column ", action->alterSTable.sTableName); - buildColumnDescription(action->alterSTable.field, result+n, + smlBuildColumnDescription(action->alterSTable.field, result+n, capacity-n, &outBytes); - TAOS_RES* res = taos_query(taos, result); //TODO async doAsyncQuery + TAOS_RES* res = taos_query(info->taos, result); //TODO async doAsyncQuery code = taos_errno(res); if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%"PRIx64" apply schema action. error : %s", info->id, taos_errstr(res)); @@ -257,7 +309,7 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInf // if (code == TSDB_CODE_MND_INVALID_COLUMN_LENGTH || code == TSDB_CODE_TSC_INVALID_COLUMN_LENGTH) { if (code == TSDB_CODE_TSC_INVALID_COLUMN_LENGTH) { - TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); + TAOS_RES* res2 = taos_query(info->taos, "RESET QUERY CACHE"); code = taos_errno(res2); if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%" PRIx64 " apply schema action. reset query cache. error: %s", info->id, taos_errstr(res2)); @@ -269,9 +321,9 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInf } case SCHEMA_ACTION_CHANGE_TAG_SIZE: { int n = sprintf(result, "alter stable %s modify tag ", action->alterSTable.sTableName); - buildColumnDescription(action->alterSTable.field, result+n, + smlBuildColumnDescription(action->alterSTable.field, result+n, capacity-n, &outBytes); - TAOS_RES* res = taos_query(taos, result); //TODO async doAsyncQuery + TAOS_RES* res = taos_query(info->taos, result); //TODO async doAsyncQuery code = taos_errno(res); if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%"PRIx64" apply schema action. error : %s", info->id, taos_errstr(res)); @@ -280,7 +332,7 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInf // if (code == TSDB_CODE_MND_INVALID_TAG_LENGTH || code == TSDB_CODE_TSC_INVALID_TAG_LENGTH) { if (code == TSDB_CODE_TSC_INVALID_TAG_LENGTH) { - TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); + TAOS_RES* res2 = taos_query(info->taos, "RESET QUERY CACHE"); code = taos_errno(res2); if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%" PRIx64 " apply schema action. reset query cache. error: %s", info->id, taos_errstr(res2)); @@ -296,7 +348,7 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInf SSmlKv **kv = taosHashIterate(action->createSTable.fields, NULL); while(kv){ - buildColumnDescription(*kv, pos, freeBytes, &outBytes); + smlBuildColumnDescription(*kv, pos, freeBytes, &outBytes); pos += outBytes; freeBytes -= outBytes; *pos = ','; ++pos; --freeBytes; kv = taosHashIterate(action->createSTable.fields, kv); @@ -308,14 +360,14 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInf kv = taosHashIterate(action->createSTable.tags, NULL); while(kv){ - buildColumnDescription(*kv, pos, freeBytes, &outBytes); + smlBuildColumnDescription(*kv, pos, freeBytes, &outBytes); pos += outBytes; freeBytes -= outBytes; *pos = ','; ++pos; --freeBytes; kv = taosHashIterate(action->createSTable.tags, kv); } pos--; ++freeBytes; outBytes = snprintf(pos, freeBytes, ")"); - TAOS_RES* res = taos_query(taos, result); + TAOS_RES* res = taos_query(info->taos, result); code = taos_errno(res); if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%"PRIx64" apply schema action. error : %s", info->id, taos_errstr(res)); @@ -323,7 +375,7 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInf taos_free_result(res); if (code == TSDB_CODE_MND_STB_ALREADY_EXIST) { - TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); + TAOS_RES* res2 = taos_query(info->taos, "RESET QUERY CACHE"); code = taos_errno(res2); if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%" PRIx64 " apply schema action. reset query cache. error: %s", info->id, taos_errstr(res2)); @@ -338,14 +390,14 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInf break; } - taosMemoryFree(result); + taosMemoryFreeClear(result); if (code != 0) { uError("SML:0x%"PRIx64 " apply schema action failure. %s", info->id, tstrerror(code)); } return code; } -static int32_t modifyDBSchemas(TAOS* taos, SSmlLinesInfo* info) { +static int32_t smlModifyDBSchemas(SSmlHandle* info) { int32_t code = 0; SSmlSTableMeta** tableMetaSml = taosHashIterate(info->superTables, NULL); @@ -356,7 +408,7 @@ static int32_t modifyDBSchemas(TAOS* taos, SSmlLinesInfo* info) { SEpSet ep = getEpSet_s(&info->taos->pAppInfo->mgmtEp); size_t superTableLen = 0; - void *superTable = taosHashGetKey(tableMetaSml, &superTableLen); + void *superTable = taosHashGetKey(tableMetaSml, &superTableLen); // todo escape SName pName = {TSDB_TABLE_NAME_T, info->taos->acctId, {0}, {0}}; strcpy(pName.dbname, info->pRequest->pDb); memcpy(pName.tname, superTable, superTableLen); @@ -369,10 +421,15 @@ static int32_t modifyDBSchemas(TAOS* taos, SSmlLinesInfo* info) { memcpy(schemaAction.createSTable.sTableName, superTable, superTableLen); schemaAction.createSTable.tags = cTablePoints->tagHash; schemaAction.createSTable.fields = cTablePoints->fieldHash; - applySchemaAction(taos, &schemaAction, info); + code = smlApplySchemaAction(info, &schemaAction); + if (code != 0) { + uError("SML:0x%"PRIx64" smlApplySchemaAction failed. can not create %s", info->id, schemaAction.createSTable.sTableName); + return code; + } + code = catalogGetSTableMeta(info->pCatalog, info->taos->pAppInfo->pTransporter, &ep, &pName, &pTableMeta); if (code != 0) { - uError("SML:0x%"PRIx64" reconcile point schema failed. can not create %s", info->id, schemaAction.createSTable.sTableName); + uError("SML:0x%"PRIx64" catalogGetSTableMeta failed. super table name %s", info->id, schemaAction.createSTable.sTableName); return code; } }else if (code == TSDB_CODE_SUCCESS) { @@ -380,67 +437,13 @@ static int32_t modifyDBSchemas(TAOS* taos, SSmlLinesInfo* info) { uError("SML:0x%"PRIx64" load table meta error: %s", info->id, tstrerror(code)); return code; } - taosHashPut(info->metaHashObj, superTable, superTableLen, &pTableMeta, POINTER_BYTES); + cTablePoints->tableMeta = pTableMeta; tableMetaSml = taosHashIterate(info->superTables, tableMetaSml); } return 0; } -static int32_t applyDataPoints(SSmlLinesInfo* info) { - int32_t code = TSDB_CODE_SUCCESS; - - TAOS_SML_DATA_POINT_TAGS** oneTable = taosHashIterate(info->childTables, NULL); - while (oneTable) { - TAOS_SML_DATA_POINT_TAGS* tableData = *oneTable; - - SName pName = {TSDB_TABLE_NAME_T, info->taos->acctId, {0}, {0}}; - strcpy(pName.dbname, info->pRequest->pDb); - memcpy(pName.tname, tableData->childTableName, strlen(tableData->childTableName)); - SEpSet ep = getEpSet_s(&info->taos->pAppInfo->mgmtEp); - SVgroupInfo vg; - catalogGetTableHashVgroup(info->pCatalog, info->taos->pAppInfo->pTransporter, &ep, &pName, &vg); - taosHashPut(info->pVgHash, (const char*)&vg.vgId, sizeof(vg.vgId), (char*)&vg, sizeof(vg)); - - STableMeta** pMeta = taosHashGet(info->metaHashObj, tableData->sTableName, tableData->sTableNameLen); - ASSERT (NULL != pMeta && NULL != *pMeta); - (*pMeta)->vgId = vg.vgId; - (*pMeta)->uid = tableData->uid; // one table merge data block together according uid - - code = smlBind(info->exec, tableData->tags, tableData->cols, *pMeta, info->msgBuf.buf, info->msgBuf.len); - if(code != TSDB_CODE_SUCCESS){ - return code; - } - oneTable = taosHashIterate(info->childTables, oneTable); - } - - smlBuildOutput(info->exec, info->pVgHash); - launchQueryImpl(info->pRequest, info->pQuery, TSDB_CODE_SUCCESS, true); - - info->affectedRows = taos_affected_rows(info->pRequest); - return info->pRequest->code; -} - -int smlInsert(TAOS* taos, SSmlLinesInfo* info) { - uDebug("SML:0x%"PRIx64" taos_sml_insert. number of super tables: %d", info->id, taosHashGetSize(info->superTables)); - - uDebug("SML:0x%"PRIx64" modify db schemas", info->id); - int32_t code = modifyDBSchemas(taos, info); - if (code != 0) { - uError("SML:0x%"PRIx64" error change db schema : %s", info->id, tstrerror(code)); - return code; - } - - uDebug("SML:0x%"PRIx64" apply data points", info->id); - code = applyDataPoints(info); - if (code != 0) { - uError("SML:0x%"PRIx64" error apply data points : %s", info->id, tstrerror(code)); - return code; - } - - return TSDB_CODE_SUCCESS; -} - //========================================================================= /* Field Escape charaters @@ -448,50 +451,50 @@ int smlInsert(TAOS* taos, SSmlLinesInfo* info) { 2: tag_key, tag_value, field_key Comma,Equal Sign,Space 3: field_value Double quote,Backslash */ -static void escapeSpecialCharacter(uint8_t field, const char **pos) { - const char *cur = *pos; - if (*cur != '\\') { - return; - } - switch (field) { - case 1: - switch (*(cur + 1)) { - case ',': - case ' ': - cur++; - break; - default: - break; - } - break; - case 2: - switch (*(cur + 1)) { - case ',': - case ' ': - case '=': - cur++; - break; - default: - break; - } - break; - case 3: - switch (*(cur + 1)) { - case '"': - case '\\': - cur++; - break; - default: - break; - } - break; - default: - break; - } - *pos = cur; -} +//static void escapeSpecialCharacter(uint8_t field, const char **pos) { +// const char *cur = *pos; +// if (*cur != '\\') { +// return; +// } +// switch (field) { +// case 1: +// switch (*(cur + 1)) { +// case ',': +// case ' ': +// cur++; +// break; +// default: +// break; +// } +// break; +// case 2: +// switch (*(cur + 1)) { +// case ',': +// case ' ': +// case '=': +// cur++; +// break; +// default: +// break; +// } +// break; +// case 3: +// switch (*(cur + 1)) { +// case '"': +// case '\\': +// cur++; +// break; +// default: +// break; +// } +// break; +// default: +// break; +// } +// *pos = cur; +//} -static bool parseTinyInt(SSmlKv *kvVal, bool *isValid, SMsgBuf *msg) { +static bool smlParseTinyInt(SSmlKv *kvVal, bool *isValid, SSmlMsgBuf *msg) { const char *pVal = kvVal->value; int32_t len = kvVal->valueLen; if (len <= 2) { @@ -503,10 +506,10 @@ static bool parseTinyInt(SSmlKv *kvVal, bool *isValid, SMsgBuf *msg) { int64_t result = strtoll(pVal, &endptr, 10); if(endptr != signalPos){ // 78ri8 *isValid = false; - buildInvalidDataMsg(msg, "invalid tiny int", endptr); + smlBuildInvalidDataMsg(msg, "invalid tiny int", endptr); }else if(!IS_VALID_TINYINT(result)){ *isValid = false; - buildInvalidDataMsg(msg, "tiny int out of range[-128,127]", endptr); + smlBuildInvalidDataMsg(msg, "tiny int out of range[-128,127]", endptr); }else{ kvVal->i = result; *isValid = true; @@ -516,7 +519,7 @@ static bool parseTinyInt(SSmlKv *kvVal, bool *isValid, SMsgBuf *msg) { return false; } -static bool parseTinyUint(SSmlKv *kvVal, bool *isValid, SMsgBuf *msg) { +static bool smlParseTinyUint(SSmlKv *kvVal, bool *isValid, SSmlMsgBuf *msg) { const char *pVal = kvVal->value; int32_t len = kvVal->valueLen; if (len <= 2) { @@ -531,10 +534,10 @@ static bool parseTinyUint(SSmlKv *kvVal, bool *isValid, SMsgBuf *msg) { int64_t result = strtoll(pVal, &endptr, 10); if(endptr != signalPos){ // 78ri8 *isValid = false; - buildInvalidDataMsg(msg, "invalid unsigned tiny int", endptr); + smlBuildInvalidDataMsg(msg, "invalid unsigned tiny int", endptr); }else if(!IS_VALID_UTINYINT(result)){ *isValid = false; - buildInvalidDataMsg(msg, "unsigned tiny int out of range[0,255]", endptr); + smlBuildInvalidDataMsg(msg, "unsigned tiny int out of range[0,255]", endptr); }else{ kvVal->i = result; *isValid = true; @@ -544,7 +547,7 @@ static bool parseTinyUint(SSmlKv *kvVal, bool *isValid, SMsgBuf *msg) { return false; } -static bool parseSmallInt(SSmlKv *kvVal, bool *isValid, SMsgBuf *msg) { +static bool smlParseSmallInt(SSmlKv *kvVal, bool *isValid, SSmlMsgBuf *msg) { const char *pVal = kvVal->value; int32_t len = kvVal->valueLen; if (len <= 3) { @@ -556,10 +559,10 @@ static bool parseSmallInt(SSmlKv *kvVal, bool *isValid, SMsgBuf *msg) { int64_t result = strtoll(pVal, &endptr, 10); if(endptr != signalPos){ // 78ri8 *isValid = false; - buildInvalidDataMsg(msg, "invalid small int", endptr); + smlBuildInvalidDataMsg(msg, "invalid small int", endptr); }else if(!IS_VALID_SMALLINT(result)){ *isValid = false; - buildInvalidDataMsg(msg, "small int our of range[-32768,32767]", endptr); + smlBuildInvalidDataMsg(msg, "small int our of range[-32768,32767]", endptr); }else{ kvVal->i = result; *isValid = true; @@ -569,7 +572,7 @@ static bool parseSmallInt(SSmlKv *kvVal, bool *isValid, SMsgBuf *msg) { return false; } -static bool parseSmallUint(SSmlKv *kvVal, bool *isValid, SMsgBuf *msg) { +static bool smlParseSmallUint(SSmlKv *kvVal, bool *isValid, SSmlMsgBuf *msg) { const char *pVal = kvVal->value; int32_t len = kvVal->valueLen; if (len <= 3) { @@ -584,10 +587,10 @@ static bool parseSmallUint(SSmlKv *kvVal, bool *isValid, SMsgBuf *msg) { int64_t result = strtoll(pVal, &endptr, 10); if(endptr != signalPos){ // 78ri8 *isValid = false; - buildInvalidDataMsg(msg, "invalid unsigned small int", endptr); + smlBuildInvalidDataMsg(msg, "invalid unsigned small int", endptr); }else if(!IS_VALID_USMALLINT(result)){ *isValid = false; - buildInvalidDataMsg(msg, "unsigned small int out of rang[0,65535]", endptr); + smlBuildInvalidDataMsg(msg, "unsigned small int out of rang[0,65535]", endptr); }else{ kvVal->i = result; *isValid = true; @@ -597,7 +600,7 @@ static bool parseSmallUint(SSmlKv *kvVal, bool *isValid, SMsgBuf *msg) { return false; } -static bool parseInt(SSmlKv *kvVal, bool *isValid, SMsgBuf *msg) { +static bool smlParseInt(SSmlKv *kvVal, bool *isValid, SSmlMsgBuf *msg) { const char *pVal = kvVal->value; int32_t len = kvVal->valueLen; if (len <= 3) { @@ -609,10 +612,10 @@ static bool parseInt(SSmlKv *kvVal, bool *isValid, SMsgBuf *msg) { int64_t result = strtoll(pVal, &endptr, 10); if(endptr != signalPos){ // 78ri8 *isValid = false; - buildInvalidDataMsg(msg, "invalid int", endptr); + smlBuildInvalidDataMsg(msg, "invalid int", endptr); }else if(!IS_VALID_INT(result)){ *isValid = false; - buildInvalidDataMsg(msg, "int out of range[-2147483648,2147483647]", endptr); + smlBuildInvalidDataMsg(msg, "int out of range[-2147483648,2147483647]", endptr); }else{ kvVal->i = result; *isValid = true; @@ -622,7 +625,7 @@ static bool parseInt(SSmlKv *kvVal, bool *isValid, SMsgBuf *msg) { return false; } -static bool parseUint(SSmlKv *kvVal, bool *isValid, SMsgBuf *msg) { +static bool smlParseUint(SSmlKv *kvVal, bool *isValid, SSmlMsgBuf *msg) { const char *pVal = kvVal->value; int32_t len = kvVal->valueLen; if (len <= 3) { @@ -637,10 +640,10 @@ static bool parseUint(SSmlKv *kvVal, bool *isValid, SMsgBuf *msg) { int64_t result = strtoll(pVal, &endptr, 10); if(endptr != signalPos){ // 78ri8 *isValid = false; - buildInvalidDataMsg(msg, "invalid unsigned int", endptr); + smlBuildInvalidDataMsg(msg, "invalid unsigned int", endptr); }else if(!IS_VALID_UINT(result)){ *isValid = false; - buildInvalidDataMsg(msg, "unsigned int out of range[0,4294967295]", endptr); + smlBuildInvalidDataMsg(msg, "unsigned int out of range[0,4294967295]", endptr); }else{ kvVal->i = result; *isValid = true; @@ -650,7 +653,7 @@ static bool parseUint(SSmlKv *kvVal, bool *isValid, SMsgBuf *msg) { return false; } -static bool parseBigInt(SSmlKv *kvVal, bool *isValid, SMsgBuf *msg) { +static bool smlParseBigInt(SSmlKv *kvVal, bool *isValid, SSmlMsgBuf *msg) { const char *pVal = kvVal->value; int32_t len = kvVal->valueLen; if (len > 3 && strcasecmp(pVal + len - 3, "i64") == 0) { @@ -670,10 +673,10 @@ static bool parseBigInt(SSmlKv *kvVal, bool *isValid, SMsgBuf *msg) { int64_t result = strtoll(pVal, &endptr, 10); if(endptr != pVal + len - 1){ // 78ri8 *isValid = false; - buildInvalidDataMsg(msg, "invalid big int", endptr); + smlBuildInvalidDataMsg(msg, "invalid big int", endptr); }else if(!IS_VALID_BIGINT(result)){ *isValid = false; - buildInvalidDataMsg(msg, "big int out of range[-9223372036854775808,9223372036854775807]", endptr); + smlBuildInvalidDataMsg(msg, "big int out of range[-9223372036854775808,9223372036854775807]", endptr); }else{ kvVal->i = result; *isValid = true; @@ -683,7 +686,7 @@ static bool parseBigInt(SSmlKv *kvVal, bool *isValid, SMsgBuf *msg) { return false; } -static bool parseBigUint(SSmlKv *kvVal, bool *isValid, SMsgBuf *msg) { +static bool smlParseBigUint(SSmlKv *kvVal, bool *isValid, SSmlMsgBuf *msg) { const char *pVal = kvVal->value; int32_t len = kvVal->valueLen; if (len <= 3) { @@ -698,10 +701,10 @@ static bool parseBigUint(SSmlKv *kvVal, bool *isValid, SMsgBuf *msg) { uint64_t result = strtoull(pVal, &endptr, 10); if(endptr != signalPos){ // 78ri8 *isValid = false; - buildInvalidDataMsg(msg, "invalid unsigned big int", endptr); + smlBuildInvalidDataMsg(msg, "invalid unsigned big int", endptr); }else if(!IS_VALID_UBIGINT(result)){ *isValid = false; - buildInvalidDataMsg(msg, "unsigned big int out of range[0,18446744073709551615]", endptr); + smlBuildInvalidDataMsg(msg, "unsigned big int out of range[0,18446744073709551615]", endptr); }else{ kvVal->u = result; *isValid = true; @@ -711,7 +714,7 @@ static bool parseBigUint(SSmlKv *kvVal, bool *isValid, SMsgBuf *msg) { return false; } -static bool parseFloat(SSmlKv *kvVal, bool *isValid, SMsgBuf *msg) { +static bool smlParseFloat(SSmlKv *kvVal, bool *isValid, SSmlMsgBuf *msg) { const char *pVal = kvVal->value; int32_t len = kvVal->valueLen; char *endptr = NULL; @@ -725,10 +728,10 @@ static bool parseFloat(SSmlKv *kvVal, bool *isValid, SMsgBuf *msg) { if (len > 3 && len f = result; *isValid = true; @@ -738,7 +741,7 @@ static bool parseFloat(SSmlKv *kvVal, bool *isValid, SMsgBuf *msg) { return false; } -static bool parseDouble(SSmlKv *kvVal, bool *isValid, SMsgBuf *msg) { +static bool smlParseDouble(SSmlKv *kvVal, bool *isValid, SSmlMsgBuf *msg) { const char *pVal = kvVal->value; int32_t len = kvVal->valueLen; if (len <= 3) { @@ -750,10 +753,10 @@ static bool parseDouble(SSmlKv *kvVal, bool *isValid, SMsgBuf *msg) { double result = strtod(pVal, &endptr); if(endptr != signalPos){ // 78ri8 *isValid = false; - buildInvalidDataMsg(msg, "invalid double", endptr); + smlBuildInvalidDataMsg(msg, "invalid double", endptr); }else if(!IS_VALID_DOUBLE(result)){ *isValid = false; - buildInvalidDataMsg(msg, "double out of range[-1.7976931348623158e+308,1.7976931348623158e+308]", endptr); + smlBuildInvalidDataMsg(msg, "double out of range[-1.7976931348623158e+308,1.7976931348623158e+308]", endptr); }else{ kvVal->d = result; *isValid = true; @@ -763,70 +766,63 @@ static bool parseDouble(SSmlKv *kvVal, bool *isValid, SMsgBuf *msg) { return false; } -static bool parseBool(SSmlKv *kvVal) { +static bool smlParseBool(SSmlKv *kvVal) { const char *pVal = kvVal->value; int32_t len = kvVal->valueLen; if ((len == 1) && pVal[len - 1] == 't') { - //printf("Type is bool(%c)\n", pVal[len - 1]); kvVal->i = true; return true; } if ((len == 1) && pVal[len - 1] == 'f') { - //printf("Type is bool(%c)\n", pVal[len - 1]); kvVal->i = false; return true; } if((len == 4) && !strcasecmp(pVal, "true")) { - //printf("Type is bool(%s)\n", &pVal[len - 4]); kvVal->i = true; return true; } if((len == 5) && !strcasecmp(pVal, "false")) { - //printf("Type is bool(%s)\n", &pVal[len - 5]); kvVal->i = false; return true; } return false; } -static bool isBinary(const char *pVal, uint16_t len) { +static bool smlIsBinary(const char *pVal, uint16_t len) { //binary: "abc" if (len < 2) { return false; } - //binary if (pVal[0] == '"' && pVal[len - 1] == '"') { - //printf("Type is binary(%s)\n", pVal); return true; } return false; } -static bool isNchar(const char *pVal, uint16_t len) { +static bool smlIsNchar(const char *pVal, uint16_t len) { //nchar: L"abc" if (len < 3) { return false; } if ((pVal[0] == 'l' || pVal[0] == 'L')&& pVal[1] == '"' && pVal[len - 1] == '"') { - //printf("Type is nchar(%s)\n", pVal); return true; } return false; } -static bool convertSmlValue(SSmlKv *pVal, SMsgBuf *msg) { +static bool smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg) { // put high probability matching type first bool isValid = false; - if (parseFloat(pVal, &isValid, msg)) { + if (smlParseFloat(pVal, &isValid, msg)) { if(!isValid) return false; pVal->type = TSDB_DATA_TYPE_FLOAT; pVal->length = (int16_t)tDataTypes[pVal->type].bytes; return true; } //binary - if (isBinary(pVal->value, pVal->valueLen)) { + if (smlIsBinary(pVal->value, pVal->valueLen)) { pVal->type = TSDB_DATA_TYPE_BINARY; pVal->length = pVal->valueLen - 2; pVal->valueLen -= 2; @@ -834,13 +830,13 @@ static bool convertSmlValue(SSmlKv *pVal, SMsgBuf *msg) { return true; } //nchar - if (isNchar(pVal->value, pVal->valueLen)) { + if (smlIsNchar(pVal->value, pVal->valueLen)) { pVal->type = TSDB_DATA_TYPE_NCHAR; pVal->length = pVal->valueLen - 3; pVal->value = pVal->value+2; return true; } - if (parseDouble(pVal, &isValid, msg)) { + if (smlParseDouble(pVal, &isValid, msg)) { if(!isValid) return false; pVal->type = TSDB_DATA_TYPE_DOUBLE; pVal->length = (int16_t)tDataTypes[pVal->type].bytes; @@ -848,66 +844,66 @@ static bool convertSmlValue(SSmlKv *pVal, SMsgBuf *msg) { return true; } //bool - if (parseBool(pVal)) { + if (smlParseBool(pVal)) { pVal->type = TSDB_DATA_TYPE_BOOL; pVal->length = (int16_t)tDataTypes[pVal->type].bytes; return true; } - if (parseTinyInt(pVal, &isValid, msg)) { + if (smlParseTinyInt(pVal, &isValid, msg)) { if(!isValid) return false; pVal->type = TSDB_DATA_TYPE_TINYINT; pVal->length = (int16_t)tDataTypes[pVal->type].bytes; return true; } - if (parseTinyUint(pVal, &isValid, msg)) { + if (smlParseTinyUint(pVal, &isValid, msg)) { if(!isValid) return false; pVal->type = TSDB_DATA_TYPE_UTINYINT; pVal->length = (int16_t)tDataTypes[pVal->type].bytes; return true; } - if (parseSmallInt(pVal, &isValid, msg)) { + if (smlParseSmallInt(pVal, &isValid, msg)) { if(!isValid) return false; pVal->type = TSDB_DATA_TYPE_SMALLINT; pVal->length = (int16_t)tDataTypes[pVal->type].bytes; return true; } - if (parseSmallUint(pVal, &isValid, msg)) { + if (smlParseSmallUint(pVal, &isValid, msg)) { if(!isValid) return false; pVal->type = TSDB_DATA_TYPE_USMALLINT; pVal->length = (int16_t)tDataTypes[pVal->type].bytes; return true; } - if (parseInt(pVal, &isValid, msg)) { + if (smlParseInt(pVal, &isValid, msg)) { if(!isValid) return false; pVal->type = TSDB_DATA_TYPE_INT; pVal->length = (int16_t)tDataTypes[pVal->type].bytes; return true; } - if (parseUint(pVal, &isValid, msg)) { + if (smlParseUint(pVal, &isValid, msg)) { if(!isValid) return false; pVal->type = TSDB_DATA_TYPE_UINT; pVal->length = (int16_t)tDataTypes[pVal->type].bytes; return true; } - if (parseBigInt(pVal, &isValid, msg)) { + if (smlParseBigInt(pVal, &isValid, msg)) { if(!isValid) return false; pVal->type = TSDB_DATA_TYPE_BIGINT; pVal->length = (int16_t)tDataTypes[pVal->type].bytes; return true; } - if (parseBigUint(pVal, &isValid, msg)) { + if (smlParseBigUint(pVal, &isValid, msg)) { if(!isValid) return false; pVal->type = TSDB_DATA_TYPE_UBIGINT; pVal->length = (int16_t)tDataTypes[pVal->type].bytes; return true; } - buildInvalidDataMsg(msg, "invalid data", pVal->value); + smlBuildInvalidDataMsg(msg, "invalid data", pVal->value); return false; } -bool checkDuplicateKey(char *key, SHashObj *pHash, SSmlLinesInfo* info) { +static bool checkDuplicateKey(char *key, SHashObj *pHash, SSmlHandle* info) { char *val = NULL; val = taosHashGet(pHash, key, strlen(key)); if (val) { @@ -921,7 +917,7 @@ bool checkDuplicateKey(char *key, SHashObj *pHash, SSmlLinesInfo* info) { return false; } -int32_t parseSml(const char* sql, TAOS_PARSE_ELEMENTS *elements){ +static int32_t smlParseString(const char* sql, SSmlLineInfo *elements, SSmlMsgBuf *msg){ if(!sql) return TSDB_CODE_SML_INVALID_DATA; while (*sql != '\0') { // jump the space at the begining if(*sql != SPACE) { @@ -930,7 +926,10 @@ int32_t parseSml(const char* sql, TAOS_PARSE_ELEMENTS *elements){ } sql++; } - if (!elements->measure || *sql == COMMA) return TSDB_CODE_SML_INVALID_DATA; + if (!elements->measure || *sql == COMMA) { + smlBuildInvalidDataMsg(msg, "invalid data", sql); + return TSDB_CODE_SML_INVALID_DATA; + } // parse measure and tag while (*sql != '\0') { @@ -953,7 +952,13 @@ int32_t parseSml(const char* sql, TAOS_PARSE_ELEMENTS *elements){ sql++; } - if(elements->measureLen == 0) return TSDB_CODE_SML_INVALID_DATA; + if(elements->tagsLen == 0){ // measure, cols1=a measure cols1=a + elements->measureTagsLen = elements->measureLen; + } + if(elements->measureLen == 0) { + smlBuildInvalidDataMsg(msg, "invalid measure", elements->measure); + return TSDB_CODE_SML_INVALID_DATA; + } // parse cols while (*sql != '\0') { @@ -963,7 +968,10 @@ int32_t parseSml(const char* sql, TAOS_PARSE_ELEMENTS *elements){ } sql++; } - if(!elements->cols) return TSDB_CODE_SML_INVALID_DATA; + if(!elements->cols) { + smlBuildInvalidDataMsg(msg, "invalid columns", elements->cols); + return TSDB_CODE_SML_INVALID_DATA; + } bool isInQuote = false; while (*sql != '\0') { @@ -992,13 +1000,13 @@ int32_t parseSml(const char* sql, TAOS_PARSE_ELEMENTS *elements){ return TSDB_CODE_SUCCESS; } -bool parseSmlCols(const char* data, int32_t len, SArray *cols, bool isTag, SMsgBuf *msg){ +static int32_t smlParseCols(const char* data, int32_t len, SArray *cols, bool isTag, SSmlMsgBuf *msg){ if(isTag && len == 0){ SSmlKv *kv = taosMemoryCalloc(sizeof(SSmlKv), 1); kv->key = TAG; - kv->keyLen = TAGNAMELEN; + kv->keyLen = strlen(TAG); kv->value = TAG; - kv->valueLen = TAGNAMELEN; + kv->valueLen = strlen(TAG); kv->type = TSDB_DATA_TYPE_NCHAR; if(cols) taosArrayPush(cols, &kv); return true; @@ -1016,7 +1024,7 @@ bool parseSmlCols(const char* data, int32_t len, SArray *cols, bool isTag, SMsgB i++; } if(keyLen == 0 || keyLen >= TSDB_COL_NAME_LEN){ - buildInvalidDataMsg(msg, "invalid key or key is too long than 64", key); + smlBuildInvalidDataMsg(msg, "invalid key or key is too long than 64", key); return TSDB_CODE_SML_INVALID_DATA; } @@ -1031,7 +1039,7 @@ bool parseSmlCols(const char* data, int32_t len, SArray *cols, bool isTag, SMsgB } int32_t valueLen = data + i - value; if(valueLen == 0){ - buildInvalidDataMsg(msg, "invalid value", value); + smlBuildInvalidDataMsg(msg, "invalid value", value); return TSDB_CODE_SML_INVALID_DATA; } @@ -1044,7 +1052,7 @@ bool parseSmlCols(const char* data, int32_t len, SArray *cols, bool isTag, SMsgB if(isTag){ kv->type = TSDB_DATA_TYPE_NCHAR; }else{ - if(!convertSmlValue(kv, msg)){ + if(!smlParseValue(kv, msg)){ return TSDB_CODE_SML_INVALID_DATA; } } @@ -1055,53 +1063,56 @@ bool parseSmlCols(const char* data, int32_t len, SArray *cols, bool isTag, SMsgB return TSDB_CODE_SUCCESS; } -static int64_t getTimeStampValue(const char *value, int32_t type) { - double ts = (double)strtoll(value, NULL, 10); +static int64_t smlGetTimeValue(const char *value, int32_t len, int8_t type) { + char *endPtr = NULL; + double ts = (double)strtoll(value, &endPtr, 10); + if(value + len != endPtr){ + return -1; + } switch (type) { case TSDB_TIME_PRECISION_HOURS: ts *= (3600 * 1e9); + break; case TSDB_TIME_PRECISION_MINUTES: ts *= (60 * 1e9); + break; case TSDB_TIME_PRECISION_SECONDS: ts *= (1e9); + break; case TSDB_TIME_PRECISION_MICRO: ts *= (1e6); + break; case TSDB_TIME_PRECISION_MILLI: ts *= (1e3); - default: break; + default: + ASSERT(0); } if(ts > (double)INT64_MAX || ts < 0){ return -1; - }else{ - return (int64_t)ts; } + + return (int64_t)ts; } -static int64_t getTimeStampNow(int32_t precision) { +static int64_t smlGetTimeNow(int8_t precision) { switch (precision) { case TSDB_TIME_PRECISION_HOURS: return taosGetTimestampMs()/1000/3600; case TSDB_TIME_PRECISION_MINUTES: return taosGetTimestampMs()/1000/60; - case TSDB_TIME_PRECISION_SECONDS: return taosGetTimestampMs()/1000; - default: + case TSDB_TIME_PRECISION_MILLI: + case TSDB_TIME_PRECISION_MICRO: + case TSDB_TIME_PRECISION_NANO: return taosGetTimestamp(precision); + default: + ASSERT(0); } } -static bool isValidateTimeStamp(const char *pVal, int32_t len) { - for (int i = 0; i < len; ++i) { - if (!isdigit(pVal[i])) { - return false; - } - } - return true; -} - -static int32_t getTsType(int32_t len) { +static int8_t smlGetTsTypeByLen(int32_t len) { if (len == TSDB_TIME_PRECISION_SEC_DIGITS) { return TSDB_TIME_PRECISION_SECONDS; } else if (len == TSDB_TIME_PRECISION_MILLI_DIGITS) { @@ -1111,52 +1122,87 @@ static int32_t getTsType(int32_t len) { } } -static int32_t parseSmlTS(const char* data, int32_t len, SArray *tags, SSmlLinesInfo* info){ - int64_t ts = 0; - if(data == NULL){ - if(info->protocol != TSDB_SML_LINE_PROTOCOL){ - buildInvalidDataMsg(&info->msgBuf, "timestamp can not be null", NULL); - return TSDB_CODE_TSC_INVALID_TIME_STAMP; - } - ts = getTimeStampNow(info->tsType); - }else{ - int ret = isValidateTimeStamp(data, len); - if(!ret){ - buildInvalidDataMsg(&info->msgBuf, "timestamp must be digit", data); - return TSDB_CODE_TSC_INVALID_TIME_STAMP; - } - int32_t tsType = -1; - if(info->protocol != TSDB_SML_LINE_PROTOCOL){ - tsType = getTsType(len); - if (tsType == -1) { - buildInvalidDataMsg(&info->msgBuf, "timestamp precision can only be seconds(10 digits) or milli seconds(13 digits)", data); - return TSDB_CODE_TSC_INVALID_TIME_STAMP; - } - }else{ - tsType = info->tsType; - } - ts = getTimeStampValue(data, tsType); - if(ts == -1){ - buildInvalidDataMsg(&info->msgBuf, "invalid timestamp", data); - return TSDB_CODE_TSC_INVALID_TIME_STAMP; - } +static int8_t smlGetTsTypeByPrecision(int8_t precision) { + switch (precision) { + case TSDB_SML_TIMESTAMP_HOURS: + return TSDB_TIME_PRECISION_HOURS; + case TSDB_SML_TIMESTAMP_MILLI_SECONDS: + return TSDB_TIME_PRECISION_MILLI; + case TSDB_SML_TIMESTAMP_NANO_SECONDS: + case TSDB_SML_TIMESTAMP_NOT_CONFIGURED: + return TSDB_TIME_PRECISION_NANO; + case TSDB_SML_TIMESTAMP_MICRO_SECONDS: + return TSDB_TIME_PRECISION_MICRO; + case TSDB_SML_TIMESTAMP_SECONDS: + return TSDB_TIME_PRECISION_SECONDS; + case TSDB_SML_TIMESTAMP_MINUTES: + return TSDB_TIME_PRECISION_MINUTES; + default: + return -1; + } +} + +static int64_t smlParseInfluxTime(SSmlHandle* info, const char* data, int32_t len){ + int8_t tsType = smlGetTsTypeByPrecision(info->precision); + if (tsType == -1) { + smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp precision", NULL); + return -1; + } + if(!data){ + return smlGetTimeNow(tsType); } + int64_t ts = smlGetTimeValue(data, len, tsType); + if(ts == -1){ + smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp", data); + return -1; + } + return ts; +} + +static int64_t smlParseOpenTsdbTime(SSmlHandle* info, const char* data, int32_t len){ + if(!data){ + smlBuildInvalidDataMsg(&info->msgBuf, "timestamp can not be null", NULL); + return -1; + } + int8_t tsType = smlGetTsTypeByLen(len); + if (tsType == -1) { + smlBuildInvalidDataMsg(&info->msgBuf, "timestamp precision can only be seconds(10 digits) or milli seconds(13 digits)", data); + return -1; + } + int64_t ts = smlGetTimeValue(data, len, tsType); + if(ts == -1){ + smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp", data); + return -1; + } + return ts; +} + +static int32_t smlParseTS(SSmlHandle* info, const char* data, int32_t len, SArray *cols){ + int64_t ts = 0; + if(info->protocol == TSDB_SML_LINE_PROTOCOL){ + ts = smlParseInfluxTime(info, data, len); + }else{ + ts = smlParseOpenTsdbTime(info, data, len); + } + if(ts == -1) return TSDB_CODE_TSC_INVALID_TIME_STAMP; + + // add ts to SSmlKv *kv = taosMemoryCalloc(sizeof(SSmlKv), 1); if(!kv){ return TSDB_CODE_OUT_OF_MEMORY; } kv->key = TS; - kv->keyLen = TSNAMELEN; + kv->keyLen = strlen(kv->key); kv->i = ts; kv->type = TSDB_DATA_TYPE_TIMESTAMP; kv->length = (int16_t)tDataTypes[kv->type].bytes; - if(tags) taosArrayPush(tags, &kv); + if(cols) taosArrayPush(cols, &kv); return TSDB_CODE_SUCCESS; } -//int32_t parseSmlCols(const char* data, SArray *cols){ +//static int32_t parseSmlCols(const char* data, SArray *cols){ // while(*data != '\0'){ // if(*data == EQUAL) return TSDB_CODE_SML_INVALID_DATA; // const char *key = data; @@ -1199,7 +1245,7 @@ static int32_t parseSmlTS(const char* data, int32_t len, SArray *tags, SSmlLines // return TSDB_CODE_SUCCESS; //} -bool updateMeta(SSmlSTableMeta* tableMeta, SArray *tags, SArray *cols, SMsgBuf *msg){ +static bool smlUpdateMeta(SSmlSTableMeta* tableMeta, SArray *tags, SArray *cols, SSmlMsgBuf *msg){ if(tags){ for (int i = 0; i < taosArrayGetSize(tags); ++i) { SSmlKv *kv = taosArrayGetP(tags, i); @@ -1223,7 +1269,7 @@ bool updateMeta(SSmlSTableMeta* tableMeta, SArray *tags, SArray *cols, SMsgBuf * SSmlKv **value = taosHashGet(tableMeta->fieldHash, kv->key, kv->keyLen); if(value){ if(kv->type != (*value)->type){ - buildInvalidDataMsg(msg, "the type is not the same like before", kv->key); + smlBuildInvalidDataMsg(msg, "the type is not the same like before", kv->key); return false; }else{ if(IS_VAR_DATA_TYPE(kv->type)){ // update string len, if bigger @@ -1237,9 +1283,10 @@ bool updateMeta(SSmlSTableMeta* tableMeta, SArray *tags, SArray *cols, SMsgBuf * } } } + return true; } -void insertMeta(SSmlSTableMeta* tableMeta, SArray *tags, SArray *cols){ +static void smlInsertMeta(SSmlSTableMeta* tableMeta, SArray *tags, SArray *cols){ if(tags){ for (int i = 0; i < taosArrayGetSize(tags); ++i) { SSmlKv *kv = taosArrayGetP(tags, i); @@ -1255,87 +1302,194 @@ void insertMeta(SSmlSTableMeta* tableMeta, SArray *tags, SArray *cols){ } } -static int32_t smlParseLine(const char* sql, SSmlLinesInfo* info) { - TAOS_PARSE_ELEMENTS elements = {0}; - int ret = parseSml(sql, &elements); +static SSmlTableInfo* smlBuildTableInfo(bool format){ + SSmlTableInfo *tag = taosMemoryCalloc(sizeof(SSmlTableInfo), 1); + if(!tag){ + return NULL; + } + + if(format){ + tag->colsFormat = taosArrayInit(16, POINTER_BYTES); + if (tag->colsFormat == NULL) { + uError("SML:smlParseLine failed to allocate memory"); + goto cleanup; + } + }else{ + tag->cols = taosArrayInit(16, POINTER_BYTES); + if (tag->cols == NULL) { + uError("SML:smlParseLine failed to allocate memory"); + goto cleanup; + } + + tag->columnsHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, false); + if (tag->columnsHash == NULL) { + uError("SML:smlParseLine failed to allocate memory"); + goto cleanup; + } + } + + tag->tags = taosArrayInit(16, POINTER_BYTES); + if (tag->tags == NULL) { + uError("SML:smlParseLine failed to allocate memory"); + goto cleanup; + } + return tag; + +cleanup: + taosMemoryFreeClear(tag); + return NULL; +} + +static void smlDestroyBuildTableInfo(SSmlTableInfo *tag, bool format){ + if(format){ + taosArrayDestroy(tag->colsFormat); + }else{ + tag->cols = taosArrayInit(16, POINTER_BYTES); + for(size_t i = 0; i < taosArrayGetSize(tag->cols); i++){ + SHashObj *kvHash = taosArrayGetP(tag->cols, i); + void** p1 = taosHashIterate(kvHash, NULL); + while (p1) { + SSmlKv* kv = *p1; + taosMemoryFreeClear(kv); + p1 = taosHashIterate(kvHash, p1); + } + taosHashCleanup(kvHash); + } + taosHashCleanup(tag->columnsHash); + } + taosArrayDestroy(tag->tags); + taosMemoryFreeClear(tag); +} + +static int32_t smlDealCols(SSmlTableInfo* oneTable, bool dataFormat, SArray *cols){ + if(dataFormat){ + taosArrayPush(oneTable->colsFormat, &cols); + }else{ + SHashObj *kvHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, false); + if(!kvHash){ + uError("SML:smlDealCols failed to allocate memory"); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + for(size_t i = 0; i < taosArrayGetSize(cols); i++){ + SSmlKv *kv = taosArrayGetP(cols, i); + taosHashPut(kvHash, kv->key, kv->keyLen, &kv, POINTER_BYTES); // todo key need escape, like \=, because find by schema name later + + if(taosHashGet(oneTable->columnsHash, kv->key, kv->keyLen) != NULL){ + continue; + } + taosHashPut(oneTable->columnsHash, kv->key, kv->keyLen, &kv, POINTER_BYTES); + } + taosArrayPush(oneTable->cols, &kvHash); + } +} + +static SSmlSTableMeta* smlBuildSTableMeta(){ + SSmlSTableMeta* meta = taosMemoryCalloc(sizeof(SSmlSTableMeta), 1); + if(!meta){ + return NULL; + } + meta->tagHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, false); + if (meta->tagHash == NULL) { + uError("SML:smlBuildSTableMeta failed to allocate memory"); + goto cleanup; + } + + meta->fieldHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, false); + if (meta->fieldHash == NULL) { + uError("SML:smlBuildSTableMeta failed to allocate memory"); + goto cleanup; + } + return meta; + +cleanup: + taosMemoryFreeClear(meta); + return NULL; +} + +static void smlDestroySTableMeta(SSmlSTableMeta *meta){ + taosHashCleanup(meta->tagHash); + taosHashCleanup(meta->fieldHash); + taosMemoryFree(meta->tableMeta); +} + +static int32_t smlParseLine(SSmlHandle* info, const char* sql) { + SSmlLineInfo elements = {0}; + int ret = smlParseString(sql, &elements, &info->msgBuf); if(ret != TSDB_CODE_SUCCESS){ + uError("SML:0x%"PRIx64" smlParseString failed", info->id); return ret; } SArray *cols = taosArrayInit(16, POINTER_BYTES); if (cols == NULL) { - uError("SML:0x%"PRIx64" taos_insert_lines failed to allocate memory", info->id); + uError("SML:0x%"PRIx64" smlParseLine failed to allocate memory", info->id); return TSDB_CODE_TSC_OUT_OF_MEMORY; } - ret = parseSmlTS(elements.timestamp, elements.timestampLen, cols, info); + ret = smlParseTS(info, elements.timestamp, elements.timestampLen, cols); if(ret != TSDB_CODE_SUCCESS){ + uError("SML:0x%"PRIx64" smlParseTS failed", info->id); return ret; } - ret = parseSmlCols(elements.cols, elements.colsLen, cols, false, &info->msgBuf); + ret = smlParseCols(elements.cols, elements.colsLen, cols, false, &info->msgBuf); if(ret != TSDB_CODE_SUCCESS){ + uError("SML:0x%"PRIx64" smlParseCols parse cloums fields failed", info->id); return ret; } if(taosArrayGetSize(cols) > TSDB_MAX_COLUMNS){ - buildInvalidDataMsg(&info->msgBuf, "too many columns than 4096", NULL); + smlBuildInvalidDataMsg(&info->msgBuf, "too many columns than 4096", NULL); return TSDB_CODE_SML_INVALID_DATA; } - TAOS_SML_DATA_POINT_TAGS** oneTable = taosHashGet(info->childTables, elements.measure, elements.measureTagsLen); + SSmlTableInfo **oneTable = taosHashGet(info->childTables, elements.measure, elements.measureTagsLen); if(oneTable){ SSmlSTableMeta** tableMeta = taosHashGet(info->superTables, elements.measure, elements.measureLen); ASSERT(tableMeta); - ret = updateMeta(*tableMeta, NULL, cols, &info->msgBuf); // update meta + ret = smlUpdateMeta(*tableMeta, NULL, cols, &info->msgBuf); // update meta cols if(!ret){ + uError("SML:0x%"PRIx64" smlUpdateMeta cols failed", info->id); return TSDB_CODE_SML_INVALID_DATA; } - taosArrayPush((*oneTable)->cols, &cols); + ret = smlDealCols(*oneTable, info->dataFormat, cols); + if(ret != TSDB_CODE_SUCCESS){ + return ret; + } }else{ - TAOS_SML_DATA_POINT_TAGS *tag = taosMemoryCalloc(sizeof(TAOS_SML_DATA_POINT_TAGS), 1); + SSmlTableInfo *tag = smlBuildTableInfo(info->dataFormat); if(!tag){ - return TSDB_CODE_OUT_OF_MEMORY; - } - tag->cols = taosArrayInit(16, POINTER_BYTES); - if (tag->cols == NULL) { - uError("SML:0x%"PRIx64" taos_insert_lines failed to allocate memory", info->id); return TSDB_CODE_TSC_OUT_OF_MEMORY; } - taosArrayPush(tag->cols, &cols); - - tag->colsColumn = taosArrayInit(16, POINTER_BYTES); - if (tag->cols == NULL) { - uError("SML:0x%"PRIx64" taos_insert_lines failed to allocate memory", info->id); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - tag->tags = taosArrayInit(16, POINTER_BYTES); - if (tag->tags == NULL) { - uError("SML:0x%"PRIx64" taos_insert_lines failed to allocate memory", info->id); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - ret = parseSmlCols(elements.tags, elements.tagsLen, tag->tags, true, &info->msgBuf); + ret = smlDealCols(tag, info->dataFormat, cols); if(ret != TSDB_CODE_SUCCESS){ return ret; } + ret = smlParseCols(elements.tags, elements.tagsLen, tag->tags, true, &info->msgBuf); + if(ret != TSDB_CODE_SUCCESS){ + uError("SML:0x%"PRIx64" smlParseCols parse tag fields failed", info->id); + return ret; + } + if(taosArrayGetSize(tag->tags) > TSDB_MAX_TAGS){ - buildInvalidDataMsg(&info->msgBuf, "too many tags than 128", NULL); + smlBuildInvalidDataMsg(&info->msgBuf, "too many tags than 128", NULL); return TSDB_CODE_SML_INVALID_DATA; } tag->sTableName = elements.measure; tag->sTableNameLen = elements.measureLen; - buildSmlChildTableName(tag); + smlBuildChildTableName(tag); + uDebug("SML:0x%"PRIx64" child table name: %s", info->id, tag->childTableName); SSmlSTableMeta** tableMeta = taosHashGet(info->superTables, elements.measure, elements.measureLen); if(tableMeta){ // update meta - ret = updateMeta(*tableMeta, tag->tags, cols, &info->msgBuf); + ret = smlUpdateMeta(*tableMeta, tag->tags, cols, &info->msgBuf); if(!ret){ + uError("SML:0x%"PRIx64" smlUpdateMeta failed", info->id); return TSDB_CODE_SML_INVALID_DATA; } }else{ - SSmlSTableMeta* meta = taosMemoryCalloc(sizeof(SSmlSTableMeta), 1); - insertMeta(meta, tag->tags, cols); + SSmlSTableMeta *meta = smlBuildSTableMeta(); + smlInsertMeta(meta, tag->tags, cols); taosHashPut(info->superTables, elements.measure, elements.measureLen, &meta, POINTER_BYTES); } @@ -1344,114 +1498,165 @@ static int32_t smlParseLine(const char* sql, SSmlLinesInfo* info) { return TSDB_CODE_SUCCESS; } -static void smlDestroyInfo(SSmlLinesInfo* info){ +static void smlDestroyInfo(SSmlHandle* info){ if(!info) return; qDestroyQuery(info->pQuery); - tscSmlDestroyHandle(info->exec); + smlDestroyHandle(info->exec); + + // destroy info->childTables + void** p1 = taosHashIterate(info->childTables, NULL); + while (p1) { + SSmlTableInfo* oneTable = *p1; + smlDestroyBuildTableInfo(oneTable, info->dataFormat); + p1 = taosHashIterate(info->childTables, p1); + } taosHashCleanup(info->childTables); + + // destroy info->superTables + p1 = taosHashIterate(info->superTables, NULL); + while (p1) { + SSmlSTableMeta* oneTable = *p1; + smlDestroySTableMeta(oneTable); + p1 = taosHashIterate(info->superTables, p1); + } taosHashCleanup(info->superTables); - taosHashCleanup(info->metaHashObj); + + // destroy info->pVgHash taosHashCleanup(info->pVgHash); - taosMemoryFree(info); + + taosMemoryFreeClear(info); } -static SSmlLinesInfo* smlBuildInfo(TAOS* taos, SRequestObj* request, SMLProtocolType protocol, int32_t tsType){ - SSmlLinesInfo* info = taosMemoryMalloc(sizeof(SSmlLinesInfo)); + +static SSmlHandle* smlBuildSmlInfo(TAOS* taos, SRequestObj* request, SMLProtocolType protocol, int8_t precision, bool dataFormat){ + SSmlHandle* info = taosMemoryMalloc(sizeof(SSmlHandle)); if (NULL == info) { return NULL; } - info->id = genLinesSmlId(); - info->tsType = tsType; - info->taos = taos; - info->protocol = protocol; + info->id = smlGenId(); - info->pQuery = taosMemoryCalloc(1, sizeof(SQuery)); + info->pQuery = taosMemoryCalloc(1, sizeof(SQuery)); if (NULL == info->pQuery) { + uError("SML:0x%"PRIx64" create info->pQuery error", info->id); goto cleanup; } - info->pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE; + info->pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE; info->pQuery->haveResultSet = false; - info->pQuery->msgType = TDMT_VND_SUBMIT; - info->pQuery->pRoot = (SNode*)nodesMakeNode(QUERY_NODE_VNODE_MODIF_STMT); + info->pQuery->msgType = TDMT_VND_SUBMIT; + info->pQuery->pRoot = (SNode*)nodesMakeNode(QUERY_NODE_VNODE_MODIF_STMT); + if(NULL == info->pQuery->pRoot){ + uError("SML:0x%"PRIx64" create info->pQuery->pRoot error", info->id); + goto cleanup; + } ((SVnodeModifOpStmt*)(info->pQuery->pRoot))->payloadType = PAYLOAD_TYPE_KV; - info->exec = tscSmlInitHandle(info->pQuery); - - int32_t code = catalogGetHandle(info->taos->pAppInfo->clusterId, &info->pCatalog); + info->taos = taos; + int32_t code = catalogGetHandle(info->taos->pAppInfo->clusterId, &info->pCatalog); if(code != TSDB_CODE_SUCCESS){ uError("SML:0x%"PRIx64" get catalog error %d", info->id, code); goto cleanup; } - info->pRequest = request; - info->msgBuf.buf = info->pRequest->msgBuf; - info->msgBuf.len = ERROR_MSG_BUF_DEFAULT_SIZE; + info->precision = precision; + info->protocol = protocol; + info->dataFormat = dataFormat; + info->pRequest = request; + info->msgBuf.buf = info->pRequest->msgBuf; + info->msgBuf.len = ERROR_MSG_BUF_DEFAULT_SIZE; + + info->exec = smlInitHandle(info->pQuery); info->childTables = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, false); info->superTables = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, false); - info->metaHashObj = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), true, false); info->pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, false); - return info; + if(NULL == info->exec || NULL == info->childTables + || NULL == info->superTables || NULL == info->pVgHash){ + uError("SML:0x%"PRIx64" create info failed", info->id); + goto cleanup; + } + return info; cleanup: smlDestroyInfo(info); return NULL; } - -int sml_insert_lines(TAOS* taos, SRequestObj* request, char* lines[], int numLines, SMLProtocolType protocol, int32_t tsType) { +static int32_t smlInsertData(SSmlHandle* info) { int32_t code = TSDB_CODE_SUCCESS; - SSmlLinesInfo* info = smlBuildInfo(taos, request, protocol, tsType); - if(!info){ - code = TSDB_CODE_OUT_OF_MEMORY; - goto cleanup; + SSmlTableInfo** oneTable = taosHashIterate(info->childTables, NULL); + while (oneTable) { + SSmlTableInfo* tableData = *oneTable; + + SName pName = {TSDB_TABLE_NAME_T, info->taos->acctId, {0}, {0}}; + strcpy(pName.dbname, info->pRequest->pDb); + memcpy(pName.tname, tableData->childTableName, strlen(tableData->childTableName)); + SEpSet ep = getEpSet_s(&info->taos->pAppInfo->mgmtEp); + SVgroupInfo vg; + code = catalogGetTableHashVgroup(info->pCatalog, info->taos->pAppInfo->pTransporter, &ep, &pName, &vg); + if (code != 0) { + uError("SML:0x%"PRIx64" catalogGetTableHashVgroup failed. table name: %s", info->id, tableData->childTableName); + return code; + } + taosHashPut(info->pVgHash, (const char*)&vg.vgId, sizeof(vg.vgId), (char*)&vg, sizeof(vg)); + + SSmlSTableMeta** pMeta = taosHashGet(info->superTables, tableData->sTableName, tableData->sTableNameLen); + ASSERT (NULL != pMeta && NULL != *pMeta); + + (*pMeta)->tableMeta->vgId = vg.vgId; + (*pMeta)->tableMeta->uid = tableData->uid; // one table merge data block together according uid + + code = smlBindData(info->exec, tableData->tags, tableData->colsFormat, tableData->columnsHash, + tableData->cols, info->dataFormat, (*pMeta)->tableMeta, info->msgBuf.buf, info->msgBuf.len); + if(code != TSDB_CODE_SUCCESS){ + return code; + } + oneTable = taosHashIterate(info->childTables, oneTable); } + + smlBuildOutput(info->exec, info->pVgHash); + launchQueryImpl(info->pRequest, info->pQuery, TSDB_CODE_SUCCESS, true); + + info->affectedRows = taos_affected_rows(info->pRequest); + return info->pRequest->code; +} + +static int smlInsertLines(SSmlHandle *info, char* lines[], int numLines) { + int32_t code = TSDB_CODE_SUCCESS; + if (numLines <= 0 || numLines > 65536) { - uError("SML:0x%"PRIx64" taos_insert_lines numLines should be between 1 and 65536. numLines: %d", info->id, numLines); + uError("SML:0x%"PRIx64" smlInsertLines numLines should be between 1 and 65536. numLines: %d", info->id, numLines); code = TSDB_CODE_TSC_APP_ERROR; goto cleanup; } + for (int32_t i = 0; i < numLines; ++i) { - code = smlParseLine(lines[i], info); + code = smlParseLine(info, lines[i]); if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%"PRIx64" data point line parse failed. line %d : %s", info->id, i, lines[i]); + uError("SML:0x%"PRIx64" smlParseLine failed. line %d : %s", info->id, i, lines[i]); goto cleanup; } } - uDebug("SML:0x%"PRIx64" data point line parse success. tables %d", info->id, taosHashGetSize(info->childTables)); + uDebug("SML:0x%"PRIx64" smlInsertLines parse success. tables %d", info->id, taosHashGetSize(info->childTables)); + uDebug("SML:0x%"PRIx64" smlInsertLines parse success. super tables %d", info->id, taosHashGetSize(info->superTables)); - code = smlInsert(taos, info); - if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%"PRIx64" taos_sml_insert error: %s", info->id, tstrerror((code))); + code = smlModifyDBSchemas(info); + if (code != 0) { + uError("SML:0x%"PRIx64" smlModifyDBSchemas error : %s", info->id, tstrerror(code)); goto cleanup; } - uDebug("SML:0x%"PRIx64" taos_insert_lines finish inserting %d lines. code: %d", info->id, numLines, code); + code = smlInsertData(info); + if (code != 0) { + uError("SML:0x%"PRIx64" smlInsertData error : %s", info->id, tstrerror(code)); + goto cleanup; + } + + uDebug("SML:0x%"PRIx64" smlInsertLines finish inserting %d lines.", info->id, numLines); cleanup: smlDestroyInfo(info); return code; } -static int32_t convertPrecisionType(int precision) { - switch (precision) { - case TSDB_SML_TIMESTAMP_HOURS: - return TSDB_TIME_PRECISION_HOURS; - case TSDB_SML_TIMESTAMP_MILLI_SECONDS: - return TSDB_TIME_PRECISION_MILLI; - case TSDB_SML_TIMESTAMP_NANO_SECONDS: - case TSDB_SML_TIMESTAMP_NOT_CONFIGURED: - return TSDB_TIME_PRECISION_NANO; - case TSDB_SML_TIMESTAMP_MICRO_SECONDS: - return TSDB_TIME_PRECISION_MICRO; - case TSDB_SML_TIMESTAMP_SECONDS: - return TSDB_TIME_PRECISION_SECONDS; - case TSDB_SML_TIMESTAMP_MINUTES: - return TSDB_TIME_PRECISION_MINUTES; - default: - return -1; - } -} - /** * taos_schemaless_insert() parse and insert data points into database according to * different protocol. @@ -1473,19 +1678,20 @@ static int32_t convertPrecisionType(int precision) { * */ -TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int protocol, int precision) { - int code = TSDB_CODE_SUCCESS; - +TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int protocol, int precision, bool dataFormat) { SRequestObj* request = createRequest(taos, NULL, NULL, TSDB_SQL_INSERT); + if(!request){ + goto end; + } + + SSmlHandle* info = smlBuildSmlInfo(taos, request, protocol, precision, dataFormat); + if(!info){ + goto end; + } + switch (protocol) { case TSDB_SML_LINE_PROTOCOL:{ - int32_t tsType = convertPrecisionType(precision); - if(tsType == -1){ - request->code = TSDB_CODE_SML_INVALID_PRECISION_TYPE; - goto end; - } - - code = sml_insert_lines(taos, request, lines, numLines, protocol, tsType); + smlInsertLines(info, lines, numLines); break; } case TSDB_SML_TELNET_PROTOCOL: @@ -1495,7 +1701,6 @@ TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int pr //code = taos_insert_json_payload(taos, *lines, protocol, tsType, &affected_rows); break; default: - code = TSDB_CODE_SML_INVALID_PROTOCOL_TYPE; break; } diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index 54e05f9264..d59473b26b 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -1514,6 +1514,7 @@ int32_t qBuildStmtColFields(void *pBlock, int32_t *fieldNum, TAOS_FIELD** fields return TSDB_CODE_SUCCESS; } +// schemaless logic start typedef struct SmlExecHandle { SHashObj* pBlockHash; @@ -1523,7 +1524,7 @@ typedef struct SmlExecHandle { SVCreateTbReq createTblReq; // each table SQuery* pQuery; -} SmlExecHandle; +} SSmlExecHandle; static int32_t smlBoundColumns(SArray *cols, SParsedDataColInfo* pColList, SSchema* pSchema) { col_id_t nCols = pColList->numOfCols; @@ -1620,14 +1621,15 @@ static int32_t smlParseTags(SArray *cols, SKVRowBuilder *tagsBuilder, SParsedDat return TSDB_CODE_SUCCESS; } -int32_t smlBind(void *handle, SArray *tags, SArray *cols, STableMeta *pTableMeta, char *msgBuf, int16_t msgBufLen) { +int32_t smlBindData(void *handle, SArray *tags, SArray *colsFormat, SHashObj *colsHash, SArray *cols, bool format, + STableMeta *pTableMeta, char *msgBuf, int16_t msgBufLen) { SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; int32_t rowNum = taosArrayGetSize(cols); if(rowNum <= 0) { return buildInvalidOperationMsg(&pBuf, "cols size <= 0"); } - SmlExecHandle *smlHandle = (SmlExecHandle *)handle; + SSmlExecHandle *smlHandle = (SSmlExecHandle *)handle; SSchema* pTagsSchema = getTableTagSchema(pTableMeta); setBoundColumnInfo(&smlHandle->tags, pTagsSchema, getNumOfTags(pTableMeta)); int ret = smlBoundColumns(tags, &smlHandle->tags, pTagsSchema); @@ -1651,7 +1653,21 @@ int32_t smlBind(void *handle, SArray *tags, SArray *cols, STableMeta *pTableMeta SSchema* pSchema = getTableColumnSchema(pTableMeta); - ret = smlBoundColumns(taosArrayGetP(cols, 0), &pDataBlock->boundColumnInfo, pSchema); + + if(format){ + ret = smlBoundColumns(taosArrayGetP(colsFormat, 0), &pDataBlock->boundColumnInfo, pSchema); + }else{ + SArray *columns = taosArrayInit(16, POINTER_BYTES); + void **p1 = taosHashIterate(colsHash, NULL); + while (p1) { + SSmlKv* kv = *p1; + taosArrayPush(columns, &kv); + p1 = taosHashIterate(colsHash, p1); + } + ret = smlBoundColumns(columns, &pDataBlock->boundColumnInfo, pSchema); + taosArrayDestroy(columns); + } + if(ret != TSDB_CODE_SUCCESS){ buildInvalidOperationMsg(&pBuf, "bound cols error"); return ret; @@ -1671,7 +1687,12 @@ int32_t smlBind(void *handle, SArray *tags, SArray *cols, STableMeta *pTableMeta for (int32_t r = 0; r < rowNum; ++r) { STSRow* row = (STSRow*)(pDataBlock->pData + pDataBlock->size); // skip the SSubmitBlk header tdSRowResetBuf(pBuilder, row); - SArray *rowData = taosArrayGetP(cols, r); + void *rowData = NULL; + if(format){ + rowData = taosArrayGetP(colsFormat, r); + }else{ + rowData = taosArrayGetP(cols, r); + } // 1. set the parsed value from sql string for (int c = 0; c < spd->numOfBound; ++c) { @@ -1680,7 +1701,18 @@ int32_t smlBind(void *handle, SArray *tags, SArray *cols, STableMeta *pTableMeta param.schema = pColSchema; getSTSRowAppendInfo(pBuilder->rowType, spd, c, ¶m.toffset, ¶m.colIdx); - SSmlKv *kv = taosArrayGetP(rowData, c); + SSmlKv *kv = NULL; + if(format){ + kv = taosArrayGetP(rowData, c); + if (!kv){ + char msg[64] = {0}; + sprintf(msg, "cols num not the same like before:%d", r); + return buildInvalidOperationMsg(&pBuf, msg); + } + }else{ + void **p =taosHashGet(rowData, pColSchema->name, strlen(pColSchema->name)); + kv = *p; + } if (kv->valueLen == 0) { MemRowAppend(&pBuf, NULL, 0, ¶m); @@ -1720,23 +1752,25 @@ int32_t smlBind(void *handle, SArray *tags, SArray *cols, STableMeta *pTableMeta return TSDB_CODE_SUCCESS; } -void* tscSmlInitHandle(SQuery *pQuery){ - SmlExecHandle *handle = taosMemoryCalloc(sizeof(SmlExecHandle)); +void* smlInitHandle(SQuery *pQuery){ + SSmlExecHandle *handle = taosMemoryCalloc(1, sizeof(SSmlExecHandle)); + if(!handle) return NULL; handle->pBlockHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false); handle->pQuery = pQuery; return handle; } -void tscSmlDestroyHandle(void *pHandle){ +void smlDestroyHandle(void *pHandle){ if(!pHandle) return; - SmlExecHandle *handle = (SmlExecHandle *)pHandle; - taosHashCleanup(handle->pBlockHash); + SSmlExecHandle *handle = (SSmlExecHandle *)pHandle; + destroyBlockHashmap(handle->pBlockHash); taosMemoryFree(handle); } int32_t smlBuildOutput(void* handle, SHashObj* pVgHash) { - SmlExecHandle *smlHandle = (SmlExecHandle *)handle; + SSmlExecHandle *smlHandle = (SSmlExecHandle *)handle; return qBuildStmtOutput(smlHandle->pQuery, pVgHash, smlHandle->pBlockHash); } +// schemaless logic end From 4fa52dc6d63ae9f5f72b62f1e7a83a9307c0fc19 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Sun, 1 May 2022 22:08:18 +0800 Subject: [PATCH 05/31] refactor:fix schemaless error & add unit test cases --- cmake/cmake.define | 2 +- include/libs/parser/parser.h | 2 +- source/client/src/clientMain.c | 5 - source/client/src/clientSml.c | 37 +++-- source/client/test/CMakeLists.txt | 21 ++- source/client/test/smlTest.cpp | 134 ++++++++++++++++++ .../libs/scalar/test/scalar/scalarTests.cpp | 2 +- 7 files changed, 177 insertions(+), 26 deletions(-) create mode 100644 source/client/test/smlTest.cpp diff --git a/cmake/cmake.define b/cmake/cmake.define index c985ba1cc7..69fc3cd0eb 100644 --- a/cmake/cmake.define +++ b/cmake/cmake.define @@ -56,7 +56,7 @@ ELSE () MESSAGE(STATUS "Will compile with Address Sanitizer!") ELSE () SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror -Werror=return-type -fPIC -gdwarf-2 -g3") - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Werror=return-type -fPIC -gdwarf-2 -g3") + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-literal-suffix -Werror=return-type -fpermissive -fPIC -gdwarf-2 -g3") ENDIF () MESSAGE("System processor ID: ${CMAKE_SYSTEM_PROCESSOR}") diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index f608e38226..d8a4eac4e5 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -98,7 +98,7 @@ int32_t qCreateSName(SName* pName, const char* pTableName, int32_t acctId, char* void* smlInitHandle(SQuery *pQuery); void smlDestroyHandle(void *pHandle); -int32_t smlBindData(void *handle, SArray *tags, SArray *cols, STableMeta *pTableMeta, char *msgBuf, int16_t msgBufLen); +int32_t smlBindData(void *handle, SArray *tags, SArray *colsFormat, SHashObj *colsHash, SArray *cols, bool format, STableMeta *pTableMeta, char *msgBuf, int16_t msgBufLen); int32_t smlBuildOutput(void* handle, SHashObj* pVgHash); #ifdef __cplusplus diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 27efcee76e..54ac42743c 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -758,8 +758,3 @@ int taos_stmt_close(TAOS_STMT *stmt) { return stmtClose(stmt); } -TAOS_RES *taos_schemaless_insert(TAOS *taos, char *lines[], int numLines, int protocol, int precision) { - // TODO - return NULL; -} - diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index bde9aeed49..8bdf737180 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -1,4 +1,3 @@ -#include #include #include #include @@ -15,6 +14,7 @@ #include "ttypes.h" #include "tcommon.h" #include "catalog.h" +#include "clientInt.h" //================================================================================================= #define SPACE ' ' @@ -971,7 +971,7 @@ static int32_t smlParseString(const char* sql, SSmlLineInfo *elements, SSmlMsgBu if(!elements->cols) { smlBuildInvalidDataMsg(msg, "invalid columns", elements->cols); return TSDB_CODE_SML_INVALID_DATA; - } + } bool isInQuote = false; while (*sql != '\0') { @@ -983,12 +983,18 @@ static int32_t smlParseString(const char* sql, SSmlLineInfo *elements, SSmlMsgBu } sql++; } + if(isInQuote){ + smlBuildInvalidDataMsg(msg, "only one quote", elements->cols); + return TSDB_CODE_SML_INVALID_DATA; + } elements->colsLen = sql - elements->cols; // parse ts,ts can be empty while (*sql != '\0') { - if(*sql != SPACE) { + if(*sql != SPACE && elements->timestamp == NULL) { elements->timestamp = sql; + } + if(*sql == SPACE && elements->timestamp != NULL){ break; } sql++; @@ -1321,7 +1327,7 @@ static SSmlTableInfo* smlBuildTableInfo(bool format){ goto cleanup; } - tag->columnsHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, false); + tag->columnsHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); if (tag->columnsHash == NULL) { uError("SML:smlParseLine failed to allocate memory"); goto cleanup; @@ -1365,7 +1371,7 @@ static int32_t smlDealCols(SSmlTableInfo* oneTable, bool dataFormat, SArray *col if(dataFormat){ taosArrayPush(oneTable->colsFormat, &cols); }else{ - SHashObj *kvHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, false); + SHashObj *kvHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); if(!kvHash){ uError("SML:smlDealCols failed to allocate memory"); return TSDB_CODE_TSC_OUT_OF_MEMORY; @@ -1381,6 +1387,7 @@ static int32_t smlDealCols(SSmlTableInfo* oneTable, bool dataFormat, SArray *col } taosArrayPush(oneTable->cols, &kvHash); } + return TSDB_CODE_SUCCESS; } static SSmlSTableMeta* smlBuildSTableMeta(){ @@ -1388,13 +1395,13 @@ static SSmlSTableMeta* smlBuildSTableMeta(){ if(!meta){ return NULL; } - meta->tagHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, false); + meta->tagHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); if (meta->tagHash == NULL) { uError("SML:smlBuildSTableMeta failed to allocate memory"); goto cleanup; } - meta->fieldHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, false); + meta->fieldHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); if (meta->fieldHash == NULL) { uError("SML:smlBuildSTableMeta failed to allocate memory"); goto cleanup; @@ -1528,6 +1535,7 @@ static void smlDestroyInfo(SSmlHandle* info){ } static SSmlHandle* smlBuildSmlInfo(TAOS* taos, SRequestObj* request, SMLProtocolType protocol, int8_t precision, bool dataFormat){ + int32_t code = TSDB_CODE_SUCCESS; SSmlHandle* info = taosMemoryMalloc(sizeof(SSmlHandle)); if (NULL == info) { return NULL; @@ -1550,7 +1558,7 @@ static SSmlHandle* smlBuildSmlInfo(TAOS* taos, SRequestObj* request, SMLProtocol ((SVnodeModifOpStmt*)(info->pQuery->pRoot))->payloadType = PAYLOAD_TYPE_KV; info->taos = taos; - int32_t code = catalogGetHandle(info->taos->pAppInfo->clusterId, &info->pCatalog); + code = catalogGetHandle(info->taos->pAppInfo->clusterId, &info->pCatalog); if(code != TSDB_CODE_SUCCESS){ uError("SML:0x%"PRIx64" get catalog error %d", info->id, code); goto cleanup; @@ -1564,9 +1572,9 @@ static SSmlHandle* smlBuildSmlInfo(TAOS* taos, SRequestObj* request, SMLProtocol info->msgBuf.len = ERROR_MSG_BUF_DEFAULT_SIZE; info->exec = smlInitHandle(info->pQuery); - info->childTables = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, false); - info->superTables = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, false); - info->pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, false); + info->childTables = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + info->superTables = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + info->pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); if(NULL == info->exec || NULL == info->childTables || NULL == info->superTables || NULL == info->pVgHash){ @@ -1653,7 +1661,6 @@ static int smlInsertLines(SSmlHandle *info, char* lines[], int numLines) { uDebug("SML:0x%"PRIx64" smlInsertLines finish inserting %d lines.", info->id, numLines); cleanup: - smlDestroyInfo(info); return code; } @@ -1681,12 +1688,12 @@ cleanup: TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int protocol, int precision, bool dataFormat) { SRequestObj* request = createRequest(taos, NULL, NULL, TSDB_SQL_INSERT); if(!request){ - goto end; + return NULL; } SSmlHandle* info = smlBuildSmlInfo(taos, request, protocol, precision, dataFormat); if(!info){ - goto end; + return (TAOS_RES*)request; } switch (protocol) { @@ -1703,7 +1710,9 @@ TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int pr default: break; } + smlDestroyInfo(info); end: return (TAOS_RES*)request; } + diff --git a/source/client/test/CMakeLists.txt b/source/client/test/CMakeLists.txt index 9df6b260f6..3fc91d07b6 100644 --- a/source/client/test/CMakeLists.txt +++ b/source/client/test/CMakeLists.txt @@ -17,14 +17,27 @@ TARGET_LINK_LIBRARIES( PUBLIC os util common transport parser catalog scheduler function gtest taos_static qcom ) +ADD_EXECUTABLE(smlTest smlTest.cpp) +TARGET_LINK_LIBRARIES( + smlTest + PUBLIC os util common transport parser catalog scheduler function gtest taos_static qcom +) + TARGET_INCLUDE_DIRECTORIES( clientTest - PUBLIC "${TD_SOURCE_DIR}/include/libs/client/" - PRIVATE "${TD_SOURCE_DIR}/source/libs/client/inc" + PUBLIC "${TD_SOURCE_DIR}/include/client/" + PRIVATE "${TD_SOURCE_DIR}/source/client/inc" ) TARGET_INCLUDE_DIRECTORIES( tmqTest - PUBLIC "${TD_SOURCE_DIR}/include/libs/client/" - PRIVATE "${TD_SOURCE_DIR}/source/libs/client/inc" + PUBLIC "${TD_SOURCE_DIR}/include/client/" + PRIVATE "${TD_SOURCE_DIR}/source/client/inc" ) + +TARGET_INCLUDE_DIRECTORIES( + smlTest + PUBLIC "${TD_SOURCE_DIR}/include/client/" + PRIVATE "${TD_SOURCE_DIR}/source/client/inc" +) + diff --git a/source/client/test/smlTest.cpp b/source/client/test/smlTest.cpp new file mode 100644 index 0000000000..823432d95d --- /dev/null +++ b/source/client/test/smlTest.cpp @@ -0,0 +1,134 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include + +#include +#include +#include + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wwrite-strings" +#pragma GCC diagnostic ignored "-Wunused-function" +#pragma GCC diagnostic ignored "-Wunused-variable" +#pragma GCC diagnostic ignored "-Wsign-compare" + +#include "../src/clientSml.c" +#include "taos.h" + +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + +TEST(testCase, sml_Test) { + char msg[256] = {0}; + SSmlMsgBuf msgBuf; + msgBuf.buf = msg; + msgBuf.len = 256; + SSmlLineInfo elements = {0}; + + //case 1 + char *sql = "st,t1=3,t2=4,t3=t3 c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 1626006833639000000 ,32,c=3"; + int ret = smlParseString(sql, &elements, &msgBuf); + ASSERT_EQ(ret, 0); + ASSERT_EQ(elements.measure, sql); + ASSERT_EQ(elements.measureLen, strlen("st")); + ASSERT_EQ(elements.measureTagsLen, strlen("st,t1=3,t2=4,t3=t3")); + + ASSERT_EQ(elements.tags, sql + elements.measureLen + 1); + ASSERT_EQ(elements.tagsLen, strlen("t1=3,t2=4,t3=t3")); + + ASSERT_EQ(elements.cols, sql + elements.measureTagsLen + 1); + ASSERT_EQ(elements.colsLen, strlen("c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64")); + + ASSERT_EQ(elements.timestamp, sql + elements.measureTagsLen + 1 + elements.colsLen + 1); + ASSERT_EQ(elements.timestampLen, strlen("1626006833639000000")); + + // case 2 false + sql = "st,t1=3,t2=4,t3=t3 c1=3i64,c3=\"passit hello,c1=2,c2=false,c4=4f64 1626006833639000000"; + memset(&elements, 0, sizeof(SSmlLineInfo)); + ret = smlParseString(sql, &elements, &msgBuf); + ASSERT_NE(ret, 0); + + // case 3 false + sql = "st, t1=3,t2=4,t3=t3 c1=3i64,c3=\"passit hello,c1=2,c2=false,c4=4f64 1626006833639000000"; + memset(&elements, 0, sizeof(SSmlLineInfo)); + ret = smlParseString(sql, &elements, &msgBuf); + ASSERT_EQ(ret, 0); + ASSERT_EQ(elements.cols, sql + elements.measureTagsLen + 2); + ASSERT_EQ(elements.colsLen, strlen("t1=3,t2=4,t3=t3")); + + + // case 4 tag is null + sql = "st, c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 1626006833639000000"; + memset(&elements, 0, sizeof(SSmlLineInfo)); + ret = smlParseString(sql, &elements, &msgBuf); + ASSERT_EQ(ret, 0); + ASSERT_EQ(elements.measure, sql); + ASSERT_EQ(elements.measureLen, strlen("st")); + ASSERT_EQ(elements.measureTagsLen, strlen("st")); + + ASSERT_EQ(elements.tags, sql + elements.measureLen + 1); + ASSERT_EQ(elements.tagsLen, 0); + + ASSERT_EQ(elements.cols, sql + elements.measureTagsLen + 2); + ASSERT_EQ(elements.colsLen, strlen("c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64")); + + ASSERT_EQ(elements.timestamp, sql + elements.measureTagsLen + 2 + elements.colsLen + 1); + ASSERT_EQ(elements.timestampLen, strlen("1626006833639000000")); + + // case 5 tag is null + sql = " st c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 1626006833639000000 "; + memset(&elements, 0, sizeof(SSmlLineInfo)); + ret = smlParseString(sql, &elements, &msgBuf); + sql++; + ASSERT_EQ(ret, 0); + ASSERT_EQ(elements.measure, sql); + ASSERT_EQ(elements.measureLen, strlen("st")); + ASSERT_EQ(elements.measureTagsLen, strlen("st")); + + ASSERT_EQ(elements.tags, sql + elements.measureLen); + ASSERT_EQ(elements.tagsLen, 0); + + ASSERT_EQ(elements.cols, sql + elements.measureTagsLen + 3); + ASSERT_EQ(elements.colsLen, strlen("c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64")); + + ASSERT_EQ(elements.timestamp, sql + elements.measureTagsLen + 3 + elements.colsLen + 2); + ASSERT_EQ(elements.timestampLen, strlen("1626006833639000000")); + + // case 6 + sql = " st c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 "; + memset(&elements, 0, sizeof(SSmlLineInfo)); + ret = smlParseString(sql, &elements, &msgBuf); + ASSERT_EQ(ret, 0); + + // case 7 + sql = " st , "; + memset(&elements, 0, sizeof(SSmlLineInfo)); + ret = smlParseString(sql, &elements, &msgBuf); + sql++; + ASSERT_EQ(ret, 0); + ASSERT_EQ(elements.cols, sql + elements.measureTagsLen + 3); + ASSERT_EQ(elements.colsLen, strlen(",")); + + // case 8 false + sql = ", st , "; + memset(&elements, 0, sizeof(SSmlLineInfo)); + ret = smlParseString(sql, &elements, &msgBuf); + ASSERT_NE(ret, 0); +} + + diff --git a/source/libs/scalar/test/scalar/scalarTests.cpp b/source/libs/scalar/test/scalar/scalarTests.cpp index 206b2ac542..c03e3408d4 100644 --- a/source/libs/scalar/test/scalar/scalarTests.cpp +++ b/source/libs/scalar/test/scalar/scalarTests.cpp @@ -127,7 +127,7 @@ void scltMakeValueNode(SNode **pNode, int32_t dataType, void *value) { *pNode = (SNode *)vnode; } -void scltMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, int32_t dataBytes, int32_t ronwNum, void *value) { +void scltMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, int32_t dataBytes, int32_t rowNum, void *value) { SNode *node = (SNode*)nodesMakeNode(QUERY_NODE_COLUMN); SColumnNode *rnode = (SColumnNode *)node; rnode->node.resType.type = dataType; From 119e7952ded8ee60f0ebee9d0913dfb4d37c0b11 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 5 May 2022 12:08:34 +0800 Subject: [PATCH 06/31] refactor:add schemaless test cases --- source/client/src/clientSml.c | 40 +++-- source/client/test/smlTest.cpp | 313 ++++++++++++++++++++++++++++++++- 2 files changed, 329 insertions(+), 24 deletions(-) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 8bdf737180..de034957dc 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -141,9 +141,9 @@ static int32_t smlBuildInvalidDataMsg(SSmlMsgBuf* pBuf, const char *msg1, const static int smlCompareKv(const void* p1, const void* p2) { SSmlKv* kv1 = (SSmlKv *)p1; SSmlKv* kv2 = (SSmlKv*)p2; - int kvLen1 = (int)strlen(kv1->key); - int kvLen2 = (int)strlen(kv2->key); - int res = strncasecmp(kv1->key, kv2->key, MIN(kvLen1, kvLen2)); + int32_t kvLen1 = kv1->keyLen; + int32_t kvLen2 = kv2->keyLen; + int32_t res = strncasecmp(kv1->key, kv2->key, MIN(kvLen1, kvLen2)); if (res != 0) { return res; } else { @@ -154,7 +154,7 @@ static int smlCompareKv(const void* p1, const void* p2) { static void smlBuildChildTableName(SSmlTableInfo *tags) { int32_t size = taosArrayGetSize(tags->tags); ASSERT(size > 0); - qsort(tags->tags, size, POINTER_BYTES, smlCompareKv); + taosArraySort(tags->tags, smlCompareKv); SStringBuilder sb = {0}; taosStringBuilderAppendStringLen(&sb, tags->sTableName, tags->sTableNameLen); @@ -501,7 +501,7 @@ static bool smlParseTinyInt(SSmlKv *kvVal, bool *isValid, SSmlMsgBuf *msg) { return false; } const char *signalPos = pVal + len - 2; - if (!strcasecmp(signalPos, "i8")) { + if (!strncasecmp(signalPos, "i8", 2)) { char *endptr = NULL; int64_t result = strtoll(pVal, &endptr, 10); if(endptr != signalPos){ // 78ri8 @@ -529,7 +529,7 @@ static bool smlParseTinyUint(SSmlKv *kvVal, bool *isValid, SSmlMsgBuf *msg) { return false; } const char *signalPos = pVal + len - 2; - if (!strcasecmp(signalPos, "u8")) { + if (!strncasecmp(signalPos, "u8", 2)) { char *endptr = NULL; int64_t result = strtoll(pVal, &endptr, 10); if(endptr != signalPos){ // 78ri8 @@ -554,7 +554,7 @@ static bool smlParseSmallInt(SSmlKv *kvVal, bool *isValid, SSmlMsgBuf *msg) { return false; } const char *signalPos = pVal + len - 3; - if (!strcasecmp(signalPos, "i16")) { + if (!strncasecmp(signalPos, "i16", 3)) { char *endptr = NULL; int64_t result = strtoll(pVal, &endptr, 10); if(endptr != signalPos){ // 78ri8 @@ -582,7 +582,7 @@ static bool smlParseSmallUint(SSmlKv *kvVal, bool *isValid, SSmlMsgBuf *msg) { return false; } const char *signalPos = pVal + len - 3; - if (strcasecmp(signalPos, "u16") == 0) { + if (strncasecmp(signalPos, "u16", 3) == 0) { char *endptr = NULL; int64_t result = strtoll(pVal, &endptr, 10); if(endptr != signalPos){ // 78ri8 @@ -607,7 +607,7 @@ static bool smlParseInt(SSmlKv *kvVal, bool *isValid, SSmlMsgBuf *msg) { return false; } const char *signalPos = pVal + len - 3; - if (strcasecmp(signalPos, "i32") == 0) { + if (strncasecmp(signalPos, "i32", 3) == 0) { char *endptr = NULL; int64_t result = strtoll(pVal, &endptr, 10); if(endptr != signalPos){ // 78ri8 @@ -635,7 +635,7 @@ static bool smlParseUint(SSmlKv *kvVal, bool *isValid, SSmlMsgBuf *msg) { return false; } const char *signalPos = pVal + len - 3; - if (strcasecmp(signalPos, "u32") == 0) { + if (strncasecmp(signalPos, "u32", 3) == 0) { char *endptr = NULL; int64_t result = strtoll(pVal, &endptr, 10); if(endptr != signalPos){ // 78ri8 @@ -656,13 +656,15 @@ static bool smlParseUint(SSmlKv *kvVal, bool *isValid, SSmlMsgBuf *msg) { static bool smlParseBigInt(SSmlKv *kvVal, bool *isValid, SSmlMsgBuf *msg) { const char *pVal = kvVal->value; int32_t len = kvVal->valueLen; - if (len > 3 && strcasecmp(pVal + len - 3, "i64") == 0) { + if (len > 3 && strncasecmp(pVal + len - 3, "i64", 3) == 0) { char *endptr = NULL; int64_t result = strtoll(pVal, &endptr, 10); if(endptr != pVal + len - 3){ // 78ri8 *isValid = false; + smlBuildInvalidDataMsg(msg, "invalid big int", endptr); }else if(!IS_VALID_BIGINT(result)){ *isValid = false; + smlBuildInvalidDataMsg(msg, "big int out of range[-9223372036854775808,9223372036854775807]", endptr); }else{ kvVal->i = result; *isValid = true; @@ -696,7 +698,7 @@ static bool smlParseBigUint(SSmlKv *kvVal, bool *isValid, SSmlMsgBuf *msg) { return false; } const char *signalPos = pVal + len - 3; - if (strcasecmp(signalPos, "u64") == 0) { + if (strncasecmp(signalPos, "u64", 3) == 0) { char *endptr = NULL; uint64_t result = strtoull(pVal, &endptr, 10); if(endptr != signalPos){ // 78ri8 @@ -725,7 +727,7 @@ static bool smlParseFloat(SSmlKv *kvVal, bool *isValid, SSmlMsgBuf *msg) { return true; } - if (len > 3 && len 3 && len i = true; return true; } - if((len == 5) && !strcasecmp(pVal, "false")) { + if((len == 5) && !strncasecmp(pVal, "false", len)) { kvVal->i = false; return true; } @@ -1085,12 +1087,14 @@ static int64_t smlGetTimeValue(const char *value, int32_t len, int8_t type) { case TSDB_TIME_PRECISION_SECONDS: ts *= (1e9); break; - case TSDB_TIME_PRECISION_MICRO: + case TSDB_TIME_PRECISION_MILLI: ts *= (1e6); break; - case TSDB_TIME_PRECISION_MILLI: + case TSDB_TIME_PRECISION_MICRO: ts *= (1e3); break; + case TSDB_TIME_PRECISION_NANO: + break; default: ASSERT(0); } diff --git a/source/client/test/smlTest.cpp b/source/client/test/smlTest.cpp index 823432d95d..9337263173 100644 --- a/source/client/test/smlTest.cpp +++ b/source/client/test/smlTest.cpp @@ -28,21 +28,21 @@ #include "../src/clientSml.c" #include "taos.h" -int main(int argc, char** argv) { +int main(int argc, char **argv) { testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); } -TEST(testCase, sml_Test) { - char msg[256] = {0}; +TEST(testCase, smlParseString_Test) { + char msg[256] = {0}; SSmlMsgBuf msgBuf; msgBuf.buf = msg; msgBuf.len = 256; SSmlLineInfo elements = {0}; - //case 1 + // case 1 char *sql = "st,t1=3,t2=4,t3=t3 c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 1626006833639000000 ,32,c=3"; - int ret = smlParseString(sql, &elements, &msgBuf); + int ret = smlParseString(sql, &elements, &msgBuf); ASSERT_EQ(ret, 0); ASSERT_EQ(elements.measure, sql); ASSERT_EQ(elements.measureLen, strlen("st")); @@ -71,7 +71,6 @@ TEST(testCase, sml_Test) { ASSERT_EQ(elements.cols, sql + elements.measureTagsLen + 2); ASSERT_EQ(elements.colsLen, strlen("t1=3,t2=4,t3=t3")); - // case 4 tag is null sql = "st, c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 1626006833639000000"; memset(&elements, 0, sizeof(SSmlLineInfo)); @@ -131,4 +130,306 @@ TEST(testCase, sml_Test) { ASSERT_NE(ret, 0); } +TEST(testCase, smlParseCols_Error_Test) { + char msg[256] = {0}; + SSmlMsgBuf msgBuf; + msgBuf.buf = msg; + msgBuf.len = 256; + const char *data[] = { + "c=\"89sd", // binary, nchar + "c=j\"89sd\"", + "c=\"89sd\"k", + "c=u", // bool + "c=truet", + "c=f64", // double + "c=8f64f", + "c=8ef64", + "c=1.7976931348623158e+390f64", + "c=f32", // float + "c=8f32f", + "c=8wef32", + "c=-3.402823466e+39f32", + "c=", // float + "c=8f", + "c=8we", + "c=3.402823466e+39", + "c=i8", // tiny int + "c=-8i8f", + "c=8wei8", + "c=-999i8", + "c=u8", // u tiny int + "c=8fu8", + "c=8weu8", + "c=999u8", + "c=-8u8", + "c=i16", // small int + "c=8fi16u", + "c=8wei16", + "c=-67787i16", + "c=u16", // u small int + "c=8u16f", + "c=8weu16", + "c=-9u16", + "c=67787u16", + "c=i32", // int + "c=8i32f", + "c=8wei32", + "c=2147483649i32", + "c=u32", // u int + "c=8u32f", + "c=8weu32", + "c=-4u32", + "c=42949672958u32", + "c=i64", // big int + "c=8i64i", + "c=8wei64", + "c=-9223372036854775809i64", + "c=i", // big int + "c=8fi", + "c=8wei", + "c=9223372036854775808i", + "c=u64", // u big int + "c=8u64f", + "c=8weu64", + "c=-3.402823466e+39u64", + "c=-339u64", + "c=18446744073709551616u64", + }; + + for(int i = 0; i < sizeof(data)/sizeof(data[0]); i++){ + int32_t len = strlen(data[i]); + int32_t ret = smlParseCols(data[i], len, NULL, false, &msgBuf); + ASSERT_NE(ret, TSDB_CODE_SUCCESS); + } +} + +TEST(testCase, smlParseCols_Test) { + char msg[256] = {0}; + SSmlMsgBuf msgBuf; + msgBuf.buf = msg; + msgBuf.len = 256; + + SArray *cols = taosArrayInit(16, POINTER_BYTES); + ASSERT_NE(cols, NULL); + + const char *data = "cbin=\"passit hello,c=2\",cnch=L\"iisdfsf\",cbool=false,cf64=4f64,cf32_=8.32,cf32=8.23f32,ci8=-34i8,cu8=89u8,ci16=233i16,cu16=898u16,ci32=98289i32,cu32=12323u32,ci64=-89238i64,ci=989i,cu64=8989323u64,cbooltrue=true,cboolt=t,cboolf=f,cnch_=l\"iuwq\""; + int32_t len = strlen(data); + int32_t ret = smlParseCols(data, len, cols, false, &msgBuf); + ASSERT_EQ(ret, TSDB_CODE_SUCCESS); + int32_t size = taosArrayGetSize(cols); + ASSERT_EQ(size, 19); + + // binary + SSmlKv *kv = taosArrayGetP(cols, 0); + ASSERT_EQ(strncasecmp(kv->key, "cbin", 4), 0); + ASSERT_EQ(kv->keyLen, 4; + ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BINARY; + ASSERT_EQ(kv->length, 16); + ASSERT_EQ(strncasecmp(kv->value, "passit", 6), 0); + taosMemoryFree(kv); + + // nchar + kv = taosArrayGetP(cols, 1); + ASSERT_EQ(strncasecmp(kv->key, "cnch", 4), 0); + ASSERT_EQ(kv->keyLen, 4); + ASSERT_EQ(kv->type, TSDB_DATA_TYPE_NCHAR); + ASSERT_EQ(kv->length, 7); + ASSERT_EQ(strncasecmp(kv->value, "iisd", 4), 0); + taosMemoryFree(kv); + + // bool + kv = taosArrayGetP(cols, 2); + ASSERT_EQ(strncasecmp(kv->key, "cbool", 5), 0); + ASSERT_EQ(kv->keyLen, 5); + ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(kv->length, 1; + ASSERT_EQ(kv->i, false); + taosMemoryFree(kv); + + // double + kv = taosArrayGetP(cols, 3); + ASSERT_EQ(strncasecmp(kv->key, "cf64", 4), 0); + ASSERT_EQ(kv->keyLen, 4); + ASSERT_EQ(kv->type, TSDB_DATA_TYPE_DOUBLE); + ASSERT_EQ(kv->length, 8; + ASSERT_EQ(kv->d, 4); + taosMemoryFree(kv); + + // float + kv = taosArrayGetP(cols, 4); + ASSERT_EQ(strncasecmp(kv->key, "cf32_", 5), 0); + ASSERT_EQ(kv->keyLen, 5); + ASSERT_EQ(kv->type, TSDB_DATA_TYPE_FLOAT); + ASSERT_EQ(kv->length, 4); + ASSERT_EQ(kv->f, 8.32); + taosMemoryFree(kv); + + // float + kv = taosArrayGetP(cols, 5); + ASSERT_EQ(strncasecmp(kv->key, "cf32", 4), 0); + ASSERT_EQ(kv->keyLen, 4); + ASSERT_EQ(kv->type, TSDB_DATA_TYPE_FLOAT); + ASSERT_EQ(kv->length, 4); + ASSERT_EQ(kv->f, 8.23); + taosMemoryFree(kv); + + // tiny int + kv = taosArrayGetP(cols, 6); + ASSERT_EQ(strncasecmp(kv->key, "ci8", 3), 0); + ASSERT_EQ(kv->keyLen, 3); + ASSERT_EQ(kv->type, TSDB_DATA_TYPE_TINYINT; + ASSERT_EQ(kv->length, 1); + ASSERT_EQ(i, -34); + taosMemoryFree(kv); + + // unsigned tiny int + kv = taosArrayGetP(cols, 7); + ASSERT_EQ(strncasecmp(kv->key, "cu8", 3), 0); + ASSERT_EQ(kv->keyLen, 3); + ASSERT_EQ(kv->type, TSDB_DATA_TYPE_UTINYINT); + ASSERT_EQ(kv->length, 1); + ASSERT_EQ(kv->u, 89); + taosMemoryFree(kv); + + // small int + kv = taosArrayGetP(cols, 8); + ASSERT_EQ(strncasecmp(kv->key, "ci16", 4), 0); + ASSERT_EQ(kv->keyLen, 4); + ASSERT_EQ(kv->type, TSDB_DATA_TYPE_SMALLINT); + ASSERT_EQ(kv->length, 2); + ASSERT_EQ(kv->u, 233); + taosMemoryFree(kv); + + // unsigned smallint + kv = taosArrayGetP(cols, 9); + ASSERT_EQ(strncasecmp(kv->key, "cu16", 4), 0); + ASSERT_EQ(kv->keyLen, 4); + ASSERT_EQ(kv->type, TSDB_DATA_TYPE_USMALLINT); + ASSERT_EQ(kv->length, 2); + ASSERT_EQ(kv->u, 898); + taosMemoryFree(kv); + + // int + kv = taosArrayGetP(cols, 10); + ASSERT_EQ(strncasecmp(kv->key, "ci32", 4), 0); + ASSERT_EQ(kv->keyLen, 4); + ASSERT_EQ(kv->type, TSDB_DATA_TYPE_INT); + ASSERT_EQ(kv->length, 4); + ASSERT_EQ(kv->u, 98289); + taosMemoryFree(kv); + + // unsigned int + kv = taosArrayGetP(cols, 11); + ASSERT_EQ(strncasecmp(kv->key, "cu32", 4), 0); + ASSERT_EQ(kv->keyLen, 4); + ASSERT_EQ(kv->type, TSDB_DATA_TYPE_UINT); + ASSERT_EQ(kv->length, 4); + ASSERT_EQ(kv->u, 12323); + taosMemoryFree(kv); + + + // bigint + kv = taosArrayGetP(cols, 12); + ASSERT_EQ(strncasecmp(kv->key, "ci64", 4), 0); + ASSERT_EQ(kv->keyLen, 4; + ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BIGINT; + ASSERT_EQ(kv->length, 8); + ASSERT_EQ(kv->i, -89238); + taosMemoryFree(kv); + + // bigint + kv = taosArrayGetP(cols, 13); + ASSERT_EQ(strncasecmp(kv->key, "ci", 2), 0); + ASSERT_EQ(kv->keyLen, 2); + ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BIGINT); + ASSERT_EQ(kv->length, 8); + ASSERT_EQ(kv->i, 989); + taosMemoryFree(kv); + + // unsigned bigint + kv = taosArrayGetP(cols, 14); + ASSERT_EQ(strncasecmp(kv->key, "cu64", 4), 0); + ASSERT_EQ(kv->keyLen, 4); + ASSERT_EQ(kv->type, TSDB_DATA_TYPE_UBIGINT); + ASSERT_EQ(kv->length, 8); + ASSERT_EQ(kv->u, 8989323); + taosMemoryFree(kv); + + // bool + kv = taosArrayGetP(cols, 15); + ASSERT_EQ(strncasecmp(kv->key, "cbooltrue", 9), 0); + ASSERT_EQ(kv->keyLen, 9); + ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(kv->length, 1); + ASSERT_EQ(kv->i, true); + taosMemoryFree(kv); + + + // bool + kv = taosArrayGetP(cols, 16); + ASSERT_EQ(strncasecmp(kv->key, "cboolt", 6), 0); + ASSERT_EQ(kv->keyLen, 6; + ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BOOL; + ASSERT_EQ(kv->length, 8); + ASSERT_EQ(kv->i, -89238); + taosMemoryFree(kv); + + // bool + kv = taosArrayGetP(cols, 17); + ASSERT_EQ(strncasecmp(kv->key, "cboolf", 6), 0); + ASSERT_EQ(kv->keyLen, 6); + ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(kv->length, 8); + ASSERT_EQ(kv->i, 989); + taosMemoryFree(kv); + + // unsigned bigint + kv = taosArrayGetP(cols, 18); + ASSERT_EQ(strncasecmp(kv->key, "cnch_", 5), 0); + ASSERT_EQ(kv->keyLen, 5); + ASSERT_EQ(kv->type, TSDB_DATA_TYPE_NCHAR); + ASSERT_EQ(kv->length, 4); + ASSERT_EQ(strncasecmp(kv->value, "iuwq", 4), 0); + taosMemoryFree(kv); + + taosArrayDestroy(cols); +} + +TEST(testCase, smlParseLine_Test) { + TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + ASSERT_NE(taos, NULL); + + SRequestObj *request = createRequest(taos, NULL, NULL, TSDB_SQL_INSERT); + ASSERT_NE(request, NULL); + + SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS, true); + ASSERT_NE(info, NULL); + + const char *sql[3] = { + "readings,name=truck_0,fleet=South,driver=Trish,model=H-2,device_version=v2.3 load_capacity=1500,fuel_capacity=150,nominal_fuel_consumption=12,latitude=52.31854,longitude=4.72037,elevation=124,velocity=0,heading=221,grade=0,fuel_consumption=25 1451606400000000000", + "readings,name=truck_1,fleet=South,driver=Albert,model=F-150,device_version=v1.5 load_capacity=2000,fuel_capacity=200,nominal_fuel_consumption=15,latitude=72.45258,longitude=68.83761,elevation=255,velocity=0,heading=181,grade=0,fuel_consumption=25 1451606400000000000", + "readings,name=truck_2,fleet=North,driver=Derek,model=F-150,device_version=v1.5 load_capacity=2000,fuel_capacity=200,nominal_fuel_consumption=15,latitude=24.5208,longitude=28.09377,elevation=428,velocity=0,heading=304,grade=0,fuel_consumption=25 1451606400000000000" + }; + for (int i = 0; i < 3; i++) { + smlParseLine(info, sql[i]); + } +} + +// TEST(testCase, smlParseTS_Test) { +// char msg[256] = {0}; +// SSmlMsgBuf msgBuf; +// msgBuf.buf = msg; +// msgBuf.len = 256; +// SSmlLineInfo elements = {0}; +// +// SSmlHandle* info = smlBuildSmlInfo(taos, request, protocol, precision, dataFormat); +// if(!info){ +// return (TAOS_RES*)request; +// } +// ret = smlParseTS(info, elements.timestamp, elements.timestampLen, cols); +// if(ret != TSDB_CODE_SUCCESS){ +// uError("SML:0x%"PRIx64" smlParseTS failed", info->id); +// return ret; +// } +// } From 26155b9511fd6a98f62c686f85c9cdea2bf13401 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 5 May 2022 18:05:06 +0800 Subject: [PATCH 07/31] refactor:add schemaless test cases --- source/client/src/clientSml.c | 75 +++++++++++++++--------- source/client/test/smlTest.cpp | 104 +++++++++++++++++++++++++-------- 2 files changed, 128 insertions(+), 51 deletions(-) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index de034957dc..139336e547 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -133,8 +133,12 @@ static uint64_t smlGenId() { } static int32_t smlBuildInvalidDataMsg(SSmlMsgBuf* pBuf, const char *msg1, const char *msg2) { - if(msg1) snprintf(pBuf->buf, pBuf->len, "%s:", msg1); - if(msg2) strncpy(pBuf->buf, msg2, pBuf->len); + if(msg1) strncat(pBuf->buf, msg1, pBuf->len); + int32_t left = pBuf->len - strlen(pBuf->buf); + if(left > 2 && msg2) { + strncat(pBuf->buf, ":", left - 1); + strncat(pBuf->buf, msg2, left - 2); + } return TSDB_CODE_SML_INVALID_DATA; } @@ -658,11 +662,12 @@ static bool smlParseBigInt(SSmlKv *kvVal, bool *isValid, SSmlMsgBuf *msg) { int32_t len = kvVal->valueLen; if (len > 3 && strncasecmp(pVal + len - 3, "i64", 3) == 0) { char *endptr = NULL; + errno = 0; int64_t result = strtoll(pVal, &endptr, 10); if(endptr != pVal + len - 3){ // 78ri8 *isValid = false; smlBuildInvalidDataMsg(msg, "invalid big int", endptr); - }else if(!IS_VALID_BIGINT(result)){ + }else if(errno == ERANGE || !IS_VALID_BIGINT(result)){ *isValid = false; smlBuildInvalidDataMsg(msg, "big int out of range[-9223372036854775808,9223372036854775807]", endptr); }else{ @@ -672,11 +677,12 @@ static bool smlParseBigInt(SSmlKv *kvVal, bool *isValid, SSmlMsgBuf *msg) { return true; }else if (len > 1 && pVal[len - 1] == 'i') { char *endptr = NULL; + errno = 0; int64_t result = strtoll(pVal, &endptr, 10); if(endptr != pVal + len - 1){ // 78ri8 *isValid = false; smlBuildInvalidDataMsg(msg, "invalid big int", endptr); - }else if(!IS_VALID_BIGINT(result)){ + }else if(errno == ERANGE || !IS_VALID_BIGINT(result)){ *isValid = false; smlBuildInvalidDataMsg(msg, "big int out of range[-9223372036854775808,9223372036854775807]", endptr); }else{ @@ -700,11 +706,12 @@ static bool smlParseBigUint(SSmlKv *kvVal, bool *isValid, SSmlMsgBuf *msg) { const char *signalPos = pVal + len - 3; if (strncasecmp(signalPos, "u64", 3) == 0) { char *endptr = NULL; + errno = 0; uint64_t result = strtoull(pVal, &endptr, 10); if(endptr != signalPos){ // 78ri8 *isValid = false; smlBuildInvalidDataMsg(msg, "invalid unsigned big int", endptr); - }else if(!IS_VALID_UBIGINT(result)){ + }else if(errno == ERANGE || !IS_VALID_UBIGINT(result)){ *isValid = false; smlBuildInvalidDataMsg(msg, "unsigned big int out of range[0,18446744073709551615]", endptr); }else{ @@ -720,18 +727,19 @@ static bool smlParseFloat(SSmlKv *kvVal, bool *isValid, SSmlMsgBuf *msg) { const char *pVal = kvVal->value; int32_t len = kvVal->valueLen; char *endptr = NULL; + errno = 0; float result = strtof(pVal, &endptr); - if(endptr == pVal + len && IS_VALID_FLOAT(result)){ // 78 + if(endptr == pVal + len && errno != ERANGE && IS_VALID_FLOAT(result)){ // 78 kvVal->f = result; *isValid = true; return true; } - if (len > 3 && len 3 && strncasecmp(pVal + len - 3, "f32", 3) == 0) { if(endptr != pVal + len - 3){ // 78ri8 *isValid = false; smlBuildInvalidDataMsg(msg, "invalid float", endptr); - }else if(!IS_VALID_FLOAT(result)){ + }else if(errno == ERANGE || !IS_VALID_FLOAT(result)){ *isValid = false; smlBuildInvalidDataMsg(msg, "float out of range[-3.402823466e+38,3.402823466e+38]", endptr); }else{ @@ -750,13 +758,14 @@ static bool smlParseDouble(SSmlKv *kvVal, bool *isValid, SSmlMsgBuf *msg) { return false; } const char *signalPos = pVal + len - 3; - if (len value, pVal->valueLen)) { + pVal->type = TSDB_DATA_TYPE_BINARY; + pVal->valueLen -= 2; + pVal->length = pVal->valueLen; + pVal->value++; + return true; + } + //nchar + if (smlIsNchar(pVal->value, pVal->valueLen)) { + pVal->type = TSDB_DATA_TYPE_NCHAR; + pVal->valueLen -= 3; + pVal->length = pVal->valueLen; + pVal->value += 2; + return true; + } + //float if (smlParseFloat(pVal, &isValid, msg)) { if(!isValid) return false; pVal->type = TSDB_DATA_TYPE_FLOAT; pVal->length = (int16_t)tDataTypes[pVal->type].bytes; return true; } - //binary - if (smlIsBinary(pVal->value, pVal->valueLen)) { - pVal->type = TSDB_DATA_TYPE_BINARY; - pVal->length = pVal->valueLen - 2; - pVal->valueLen -= 2; - pVal->value = pVal->value++; - return true; - } - //nchar - if (smlIsNchar(pVal->value, pVal->valueLen)) { - pVal->type = TSDB_DATA_TYPE_NCHAR; - pVal->length = pVal->valueLen - 3; - pVal->value = pVal->value+2; - return true; - } + //double if (smlParseDouble(pVal, &isValid, msg)) { if(!isValid) return false; pVal->type = TSDB_DATA_TYPE_DOUBLE; @@ -1017,7 +1030,7 @@ static int32_t smlParseCols(const char* data, int32_t len, SArray *cols, bool is kv->valueLen = strlen(TAG); kv->type = TSDB_DATA_TYPE_NCHAR; if(cols) taosArrayPush(cols, &kv); - return true; + return TSDB_CODE_SUCCESS; } for(int i = 0; i < len; i++){ @@ -1039,12 +1052,20 @@ static int32_t smlParseCols(const char* data, int32_t len, SArray *cols, bool is // parse value i++; const char *value = data + i; + bool isInQuote = false; while(i < len){ - if(data[i] == COMMA && i > 0 && data[i-1] != SLASH){ + if(data[i] == QUOTE && data[i-1] != SLASH){ + isInQuote = !isInQuote; + } + if(!isInQuote && data[i] == COMMA && i > 0 && data[i-1] != SLASH){ break; } i++; } + if(isInQuote){ + smlBuildInvalidDataMsg(msg, "only one quote", value); + return TSDB_CODE_SML_INVALID_DATA; + } int32_t valueLen = data + i - value; if(valueLen == 0){ smlBuildInvalidDataMsg(msg, "invalid value", value); diff --git a/source/client/test/smlTest.cpp b/source/client/test/smlTest.cpp index 9337263173..190abd0145 100644 --- a/source/client/test/smlTest.cpp +++ b/source/client/test/smlTest.cpp @@ -131,11 +131,6 @@ TEST(testCase, smlParseString_Test) { } TEST(testCase, smlParseCols_Error_Test) { - char msg[256] = {0}; - SSmlMsgBuf msgBuf; - msgBuf.buf = msg; - msgBuf.len = 256; - const char *data[] = { "c=\"89sd", // binary, nchar "c=j\"89sd\"", @@ -198,12 +193,70 @@ TEST(testCase, smlParseCols_Error_Test) { }; for(int i = 0; i < sizeof(data)/sizeof(data[0]); i++){ + char msg[256] = {0}; + SSmlMsgBuf msgBuf; + msgBuf.buf = msg; + msgBuf.len = 256; int32_t len = strlen(data[i]); int32_t ret = smlParseCols(data[i], len, NULL, false, &msgBuf); ASSERT_NE(ret, TSDB_CODE_SUCCESS); } } +TEST(testCase, smlParseCols_tag_Test) { + char msg[256] = {0}; + SSmlMsgBuf msgBuf; + msgBuf.buf = msg; + msgBuf.len = 256; + + SArray *cols = taosArrayInit(16, POINTER_BYTES); + ASSERT_NE(cols, NULL); + + const char *data = + "cbin=\"passit hello,c=2\",cnch=L\"iisdfsf\",cbool=false,cf64=4.31f64,cf32_=8.32,cf32=8.23f32,ci8=-34i8,cu8=89u8,ci16=233i16,cu16=898u16,ci32=98289i32,cu32=12323u32,ci64=-89238i64,ci=989i,cu64=8989323u64,cbooltrue=true,cboolt=t,cboolf=f,cnch_=l\"iuwq\""; + int32_t len = strlen(data); + int32_t ret = smlParseCols(data, len, cols, true, &msgBuf); + ASSERT_EQ(ret, TSDB_CODE_SUCCESS); + int32_t size = taosArrayGetSize(cols); + ASSERT_EQ(size, 19); + + // nchar + SSmlKv *kv = taosArrayGetP(cols, 0); + ASSERT_EQ(strncasecmp(kv->key, "cbin", 4), 0); + ASSERT_EQ(kv->keyLen, 4); + ASSERT_EQ(kv->type, TSDB_DATA_TYPE_NCHAR); + ASSERT_EQ(kv->valueLen, 18); + ASSERT_EQ(strncasecmp(kv->value, "\"passit", 7), 0); + taosMemoryFree(kv); + + // nchar + kv = taosArrayGetP(cols, 3); + ASSERT_EQ(strncasecmp(kv->key, "cf64", 4), 0); + ASSERT_EQ(kv->keyLen, 4); + ASSERT_EQ(kv->type, TSDB_DATA_TYPE_NCHAR); + ASSERT_EQ(kv->valueLen, 7); + ASSERT_EQ(strncasecmp(kv->value, "4.31f64", 7), 0); + taosMemoryFree(kv); + + taosArrayClear(cols); + data = "t=3e"; + len = 0; + memset(msgBuf.buf, 0, msgBuf.len); + ret = smlParseCols(data, len, cols, true, &msgBuf); + ASSERT_EQ(ret, TSDB_CODE_SUCCESS); + size = taosArrayGetSize(cols); + ASSERT_EQ(size, 1); + + // nchar + kv = taosArrayGetP(cols, 0); + ASSERT_EQ(strncasecmp(kv->key, TAG, strlen(TAG)), 0); + ASSERT_EQ(kv->keyLen, strlen(TAG)); + ASSERT_EQ(kv->type, TSDB_DATA_TYPE_NCHAR); + ASSERT_EQ(kv->valueLen, strlen(TAG)); + ASSERT_EQ(strncasecmp(kv->value, TAG, strlen(TAG)), 0); + taosMemoryFree(kv); +} + TEST(testCase, smlParseCols_Test) { char msg[256] = {0}; SSmlMsgBuf msgBuf; @@ -213,7 +266,7 @@ TEST(testCase, smlParseCols_Test) { SArray *cols = taosArrayInit(16, POINTER_BYTES); ASSERT_NE(cols, NULL); - const char *data = "cbin=\"passit hello,c=2\",cnch=L\"iisdfsf\",cbool=false,cf64=4f64,cf32_=8.32,cf32=8.23f32,ci8=-34i8,cu8=89u8,ci16=233i16,cu16=898u16,ci32=98289i32,cu32=12323u32,ci64=-89238i64,ci=989i,cu64=8989323u64,cbooltrue=true,cboolt=t,cboolf=f,cnch_=l\"iuwq\""; + const char *data = "cbin=\"passit hello,c=2\",cnch=L\"iisdfsf\",cbool=false,cf64=4.31f64,cf32_=8.32,cf32=8.23f32,ci8=-34i8,cu8=89u8,ci16=233i16,cu16=898u16,ci32=98289i32,cu32=12323u32,ci64=-89238i64,ci=989i,cu64=8989323u64,cbooltrue=true,cboolt=t,cboolf=f,cnch_=l\"iuwq\""; int32_t len = strlen(data); int32_t ret = smlParseCols(data, len, cols, false, &msgBuf); ASSERT_EQ(ret, TSDB_CODE_SUCCESS); @@ -223,8 +276,8 @@ TEST(testCase, smlParseCols_Test) { // binary SSmlKv *kv = taosArrayGetP(cols, 0); ASSERT_EQ(strncasecmp(kv->key, "cbin", 4), 0); - ASSERT_EQ(kv->keyLen, 4; - ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BINARY; + ASSERT_EQ(kv->keyLen, 4); + ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BINARY); ASSERT_EQ(kv->length, 16); ASSERT_EQ(strncasecmp(kv->value, "passit", 6), 0); taosMemoryFree(kv); @@ -243,7 +296,7 @@ TEST(testCase, smlParseCols_Test) { ASSERT_EQ(strncasecmp(kv->key, "cbool", 5), 0); ASSERT_EQ(kv->keyLen, 5); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BOOL); - ASSERT_EQ(kv->length, 1; + ASSERT_EQ(kv->length, 1); ASSERT_EQ(kv->i, false); taosMemoryFree(kv); @@ -252,8 +305,9 @@ TEST(testCase, smlParseCols_Test) { ASSERT_EQ(strncasecmp(kv->key, "cf64", 4), 0); ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_DOUBLE); - ASSERT_EQ(kv->length, 8; - ASSERT_EQ(kv->d, 4); + ASSERT_EQ(kv->length, 8); + //ASSERT_EQ(kv->d, 4.31); + printf("4.31 = kv->f:%f\n", kv->d); taosMemoryFree(kv); // float @@ -262,7 +316,8 @@ TEST(testCase, smlParseCols_Test) { ASSERT_EQ(kv->keyLen, 5); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_FLOAT); ASSERT_EQ(kv->length, 4); - ASSERT_EQ(kv->f, 8.32); + //ASSERT_EQ(kv->f, 8.32); + printf("8.32 = kv->f:%f\n", kv->f); taosMemoryFree(kv); // float @@ -271,16 +326,17 @@ TEST(testCase, smlParseCols_Test) { ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_FLOAT); ASSERT_EQ(kv->length, 4); - ASSERT_EQ(kv->f, 8.23); + //ASSERT_EQ(kv->f, 8.23); + printf("8.23 = kv->f:%f\n", kv->f); taosMemoryFree(kv); // tiny int kv = taosArrayGetP(cols, 6); ASSERT_EQ(strncasecmp(kv->key, "ci8", 3), 0); ASSERT_EQ(kv->keyLen, 3); - ASSERT_EQ(kv->type, TSDB_DATA_TYPE_TINYINT; + ASSERT_EQ(kv->type, TSDB_DATA_TYPE_TINYINT); ASSERT_EQ(kv->length, 1); - ASSERT_EQ(i, -34); + ASSERT_EQ(kv->i, -34); taosMemoryFree(kv); // unsigned tiny int @@ -332,8 +388,8 @@ TEST(testCase, smlParseCols_Test) { // bigint kv = taosArrayGetP(cols, 12); ASSERT_EQ(strncasecmp(kv->key, "ci64", 4), 0); - ASSERT_EQ(kv->keyLen, 4; - ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BIGINT; + ASSERT_EQ(kv->keyLen, 4); + ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BIGINT); ASSERT_EQ(kv->length, 8); ASSERT_EQ(kv->i, -89238); taosMemoryFree(kv); @@ -369,10 +425,10 @@ TEST(testCase, smlParseCols_Test) { // bool kv = taosArrayGetP(cols, 16); ASSERT_EQ(strncasecmp(kv->key, "cboolt", 6), 0); - ASSERT_EQ(kv->keyLen, 6; - ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BOOL; - ASSERT_EQ(kv->length, 8); - ASSERT_EQ(kv->i, -89238); + ASSERT_EQ(kv->keyLen, 6); + ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(kv->length, 1); + ASSERT_EQ(kv->i, true); taosMemoryFree(kv); // bool @@ -380,11 +436,11 @@ TEST(testCase, smlParseCols_Test) { ASSERT_EQ(strncasecmp(kv->key, "cboolf", 6), 0); ASSERT_EQ(kv->keyLen, 6); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BOOL); - ASSERT_EQ(kv->length, 8); - ASSERT_EQ(kv->i, 989); + ASSERT_EQ(kv->length, 1); + ASSERT_EQ(kv->i, false); taosMemoryFree(kv); - // unsigned bigint + // nchar kv = taosArrayGetP(cols, 18); ASSERT_EQ(strncasecmp(kv->key, "cnch_", 5), 0); ASSERT_EQ(kv->keyLen, 5); From 07a02b25b17e671e0c69e23c0dc1d357a998dd53 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 5 May 2022 21:27:02 +0800 Subject: [PATCH 08/31] refactor:add schemaless test cases --- source/client/src/clientSml.c | 25 +++++++++++++++++++------ source/client/test/smlTest.cpp | 13 ++++++++++--- source/libs/parser/src/parInsert.c | 8 ++++---- source/libs/transport/src/rpcTcp.c | 2 +- 4 files changed, 34 insertions(+), 14 deletions(-) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 139336e547..55e95c2aa5 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -117,8 +117,8 @@ typedef struct { //================================================================================================= static uint64_t linesSmlHandleId = 0; -static const char* TS = "ts"; -static const char* TAG = "tagNone"; +static const char* TS = "_ts"; +static const char* TAG = "_tagNone"; //================================================================================================= @@ -350,13 +350,26 @@ static int32_t smlApplySchemaAction(SSmlHandle* info, SSchemaAction* action) { int n = sprintf(result, "create stable %s (", action->createSTable.sTableName); char* pos = result + n; int freeBytes = capacity - n; + size_t size = taosHashGetSize(action->createSTable.fields); + SArray *cols = taosArrayInit(size, POINTER_BYTES); SSmlKv **kv = taosHashIterate(action->createSTable.fields, NULL); while(kv){ - smlBuildColumnDescription(*kv, pos, freeBytes, &outBytes); - pos += outBytes; freeBytes -= outBytes; - *pos = ','; ++pos; --freeBytes; + if(strncmp((*kv)->key, TS, strlen(TS)) == 0 && (*kv)->type == TSDB_DATA_TYPE_TIMESTAMP){ + taosArrayInsert(cols, 0, kv); + }else{ + taosArrayPush(cols, kv); + } kv = taosHashIterate(action->createSTable.fields, kv); } + + for(int i = 0; i < taosArrayGetSize(cols); i++){ + SSmlKv *kvNew = taosArrayGetP(cols, i); + smlBuildColumnDescription(kvNew, pos, freeBytes, &outBytes); + pos += outBytes; freeBytes -= outBytes; + *pos = ','; ++pos; --freeBytes; + } + taosArrayDestroy(cols); + --pos; ++freeBytes; outBytes = snprintf(pos, freeBytes, ") tags ("); @@ -419,7 +432,7 @@ static int32_t smlModifyDBSchemas(SSmlHandle* info) { code = catalogGetSTableMeta(info->pCatalog, info->taos->pAppInfo->pTransporter, &ep, &pName, &pTableMeta); - if (code == TSDB_CODE_TDB_INVALID_TABLE_ID) { + if (code == TSDB_CODE_TDB_INVALID_TABLE_ID || code == TSDB_CODE_MND_INVALID_STB) { SSchemaAction schemaAction = {0}; schemaAction.action = SCHEMA_ACTION_CREATE_STABLE; memcpy(schemaAction.createSTable.sTableName, superTable, superTableLen); diff --git a/source/client/test/smlTest.cpp b/source/client/test/smlTest.cpp index 190abd0145..5f0f188b0b 100644 --- a/source/client/test/smlTest.cpp +++ b/source/client/test/smlTest.cpp @@ -456,6 +456,12 @@ TEST(testCase, smlParseLine_Test) { TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); ASSERT_NE(taos, NULL); + TAOS_RES* pRes = taos_query(taos, "create database if not exists sml_db"); + taos_free_result(pRes); + + pRes = taos_query(taos, "use sml_db"); + taos_free_result(pRes); + SRequestObj *request = createRequest(taos, NULL, NULL, TSDB_SQL_INSERT); ASSERT_NE(request, NULL); @@ -467,9 +473,10 @@ TEST(testCase, smlParseLine_Test) { "readings,name=truck_1,fleet=South,driver=Albert,model=F-150,device_version=v1.5 load_capacity=2000,fuel_capacity=200,nominal_fuel_consumption=15,latitude=72.45258,longitude=68.83761,elevation=255,velocity=0,heading=181,grade=0,fuel_consumption=25 1451606400000000000", "readings,name=truck_2,fleet=North,driver=Derek,model=F-150,device_version=v1.5 load_capacity=2000,fuel_capacity=200,nominal_fuel_consumption=15,latitude=24.5208,longitude=28.09377,elevation=428,velocity=0,heading=304,grade=0,fuel_consumption=25 1451606400000000000" }; - for (int i = 0; i < 3; i++) { - smlParseLine(info, sql[i]); - } + smlInsertLines(info, sql, 3); +// for (int i = 0; i < 3; i++) { +// smlParseLine(info, sql[i]); +// } } // TEST(testCase, smlParseTS_Test) { diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index d59473b26b..1e90d664a9 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -1624,10 +1624,6 @@ static int32_t smlParseTags(SArray *cols, SKVRowBuilder *tagsBuilder, SParsedDat int32_t smlBindData(void *handle, SArray *tags, SArray *colsFormat, SHashObj *colsHash, SArray *cols, bool format, STableMeta *pTableMeta, char *msgBuf, int16_t msgBufLen) { SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; - int32_t rowNum = taosArrayGetSize(cols); - if(rowNum <= 0) { - return buildInvalidOperationMsg(&pBuf, "cols size <= 0"); - } SSmlExecHandle *smlHandle = (SSmlExecHandle *)handle; SSchema* pTagsSchema = getTableTagSchema(pTableMeta); @@ -1679,6 +1675,10 @@ int32_t smlBindData(void *handle, SArray *tags, SArray *colsFormat, SHashObj *co initRowBuilder(&pDataBlock->rowBuilder, pDataBlock->pTableMeta->sversion, &pDataBlock->boundColumnInfo); + int32_t rowNum = format ? taosArrayGetSize(colsFormat) : taosArrayGetSize(cols); + if(rowNum <= 0) { + return buildInvalidOperationMsg(&pBuf, "cols size <= 0"); + } ret = allocateMemForSize(pDataBlock, extendedRowSize * rowNum); if(ret != TSDB_CODE_SUCCESS){ buildInvalidOperationMsg(&pBuf, "allocate memory error"); diff --git a/source/libs/transport/src/rpcTcp.c b/source/libs/transport/src/rpcTcp.c index 52c5ddcf63..edc685f498 100644 --- a/source/libs/transport/src/rpcTcp.c +++ b/source/libs/transport/src/rpcTcp.c @@ -21,7 +21,7 @@ #include "taoserror.h" #include "tutil.h" -#ifndef USE_UV +#ifndef USEf_UV typedef struct SFdObj { void * signature; TdSocketPtr pSocket; // TCP socket FD From 72e8d5cea4b91efa47572b34ebd2ce69dfc46379 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 6 May 2022 10:49:32 +0800 Subject: [PATCH 09/31] refactor:add schemaless test cases --- source/libs/parser/src/parInsert.c | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index 1e90d664a9..2269fd10a5 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -1597,7 +1597,7 @@ static int32_t smlBoundColumns(SArray *cols, SParsedDataColInfo* pColList, SSche return TSDB_CODE_SUCCESS; } -static int32_t smlParseTags(SArray *cols, SKVRowBuilder *tagsBuilder, SParsedDataColInfo* tags, SSchema* pSchema, SVCreateTbReq *createTblReq, SMsgBuf *msg) { +static int32_t smlBoundTags(SArray *cols, SKVRowBuilder *tagsBuilder, SParsedDataColInfo* tags, SSchema* pSchema, SKVRow *row, SMsgBuf *msg) { if (tdInitKVRowBuilder(tagsBuilder) < 0) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } @@ -1611,13 +1611,11 @@ static int32_t smlParseTags(SArray *cols, SKVRowBuilder *tagsBuilder, SParsedDat } - SKVRow row = tdGetKVRowFromBuilder(tagsBuilder); - if(row == NULL){ + *row = tdGetKVRowFromBuilder(tagsBuilder); + if(*row == NULL){ return TSDB_CODE_SML_INVALID_DATA; } - tdSortKVRowByColIdx(row); - createTblReq->type = TD_CHILD_TABLE; - createTblReq->ctbCfg.pTag = row; + tdSortKVRowByColIdx(*row); return TSDB_CODE_SUCCESS; } @@ -1633,11 +1631,14 @@ int32_t smlBindData(void *handle, SArray *tags, SArray *colsFormat, SHashObj *co buildInvalidOperationMsg(&pBuf, "bound tags error"); return ret; } - ret = smlParseTags(tags, &smlHandle->tagsBuilder, &smlHandle->tags, pTagsSchema, &smlHandle->createTblReq, &pBuf); + SKVRow row = NULL; + ret = smlBoundTags(tags, &smlHandle->tagsBuilder, &smlHandle->tags, pTagsSchema, &row, &pBuf); if(ret != TSDB_CODE_SUCCESS){ return ret; } + buildCreateTbReq(&smlHandle->createTblReq, pTableMeta->schema->name, row, pTableMeta->suid); + STableDataBlocks* pDataBlock = NULL; ret = getDataBlockFromList(smlHandle->pBlockHash, pTableMeta->uid, TSDB_DEFAULT_PAYLOAD_SIZE, sizeof(SSubmitBlk), getTableInfo(pTableMeta).rowSize, pTableMeta, @@ -1714,15 +1715,15 @@ int32_t smlBindData(void *handle, SArray *tags, SArray *colsFormat, SHashObj *co kv = *p; } - if (kv->valueLen == 0) { + if (kv->length == 0) { MemRowAppend(&pBuf, NULL, 0, ¶m); } else { int32_t colLen = pColSchema->bytes; if (IS_VAR_DATA_TYPE(pColSchema->type)) { - colLen = kv->valueLen; + colLen = kv->length; } - MemRowAppend(&pBuf, kv->value, colLen, ¶m); + MemRowAppend(&pBuf, &(kv->value), colLen, ¶m); } if (PRIMARYKEY_TIMESTAMP_COL_ID == pColSchema->colId) { From 7e6c12cc396b84fd7aede089ada68ef3901b9fff Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 6 May 2022 11:52:03 +0800 Subject: [PATCH 10/31] refactor:fix error merge from 3.0 --- include/libs/parser/parser.h | 2 +- source/libs/parser/src/parInsert.c | 2 +- source/libs/parser/src/parInsertData.c | 2 +- source/libs/transport/src/rpcTcp.c | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index 4e140d3d4f..0b9fdf94f9 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -97,7 +97,7 @@ int32_t qBindStmtSingleColValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBu int32_t rowNum); int32_t qBuildStmtColFields(void* pDataBlock, int32_t* fieldNum, TAOS_FIELD** fields); int32_t qBuildStmtTagFields(void* pBlock, void* boundTags, int32_t* fieldNum, TAOS_FIELD** fields); -int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, SName* pName, TAOS_MULTI_BIND* bind, +int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, char* tName, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen); void destroyBoundColumnInfo(void* pBoundInfo); int32_t qCreateSName(SName* pName, const char* pTableName, int32_t acctId, char* dbName, char* msgBuf, diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index 33d742d167..17c75dac99 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -1255,7 +1255,7 @@ int32_t qBuildStmtOutput(SQuery* pQuery, SHashObj* pVgHash, SHashObj* pBlockHash return TSDB_CODE_SUCCESS; } -int32_t qBindStmtTagsValue(void *pBlock, void *boundTags, int64_t suid, SName *pName, TAOS_MULTI_BIND *bind, char *msgBuf, int32_t msgBufLen){ +int32_t qBindStmtTagsValue(void *pBlock, void *boundTags, int64_t suid, char *tName, TAOS_MULTI_BIND *bind, char *msgBuf, int32_t msgBufLen){ STableDataBlocks *pDataBlock = (STableDataBlocks *)pBlock; SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; SParsedDataColInfo* tags = (SParsedDataColInfo*)boundTags; diff --git a/source/libs/parser/src/parInsertData.c b/source/libs/parser/src/parInsertData.c index fdb7379152..21d4f13669 100644 --- a/source/libs/parser/src/parInsertData.c +++ b/source/libs/parser/src/parInsertData.c @@ -186,7 +186,7 @@ int32_t buildCreateTbMsg(STableDataBlocks* pBlocks, SVCreateTbReq* pCreateTbReq) } int32_t getDataBlockFromList(SHashObj* pHashList, int64_t id, int32_t size, int32_t startOffset, int32_t rowSize, - const STableMeta* pTableMeta, STableDataBlocks** dataBlocks, SArray* pBlockList, + STableMeta* pTableMeta, STableDataBlocks** dataBlocks, SArray* pBlockList, SVCreateTbReq* pCreateTbReq) { *dataBlocks = NULL; STableDataBlocks** t1 = (STableDataBlocks**)taosHashGet(pHashList, (const char*)&id, sizeof(id)); diff --git a/source/libs/transport/src/rpcTcp.c b/source/libs/transport/src/rpcTcp.c index edc685f498..52c5ddcf63 100644 --- a/source/libs/transport/src/rpcTcp.c +++ b/source/libs/transport/src/rpcTcp.c @@ -21,7 +21,7 @@ #include "taoserror.h" #include "tutil.h" -#ifndef USEf_UV +#ifndef USE_UV typedef struct SFdObj { void * signature; TdSocketPtr pSocket; // TCP socket FD From 666b0b676dacec6a56e40fc972d86926f09d1627 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 6 May 2022 15:30:00 +0800 Subject: [PATCH 11/31] fix: alter db replications --- source/dnode/mgmt/mgmt_vnode/src/vmHandle.c | 4 -- source/dnode/mnode/impl/src/mndDb.c | 50 ++++++++++++--------- 2 files changed, 28 insertions(+), 26 deletions(-) diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index e3816396c9..9329e7b15d 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -293,14 +293,10 @@ void vmInitMsgHandle(SMgmtWrapper *pWrapper) { dmSetMsgHandle(pWrapper, TDMT_VND_TASK_MERGE_EXEC, vmProcessMergeMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_VND_TASK_WRITE_EXEC, vmProcessWriteMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_VND_STREAM_TRIGGER, vmProcessFetchMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_ALTER_VNODE, vmProcessWriteMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_VND_COMPACT_VNODE, vmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_DND_CREATE_VNODE, vmProcessMgmtMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_DND_DROP_VNODE, vmProcessMgmtMsg, DEFAULT_HANDLE); - // dmSetMsgHandle(pWrapper, TDMT_DND_SYNC_VNODE, vmProcessMgmtMsg, DEFAULT_HANDLE); - // dmSetMsgHandle(pWrapper, TDMT_DND_COMPACT_VNODE, vmProcessMgmtMsg, DEFAULT_HANDLE); // sync integration dmSetMsgHandle(pWrapper, TDMT_VND_SYNC_TIMEOUT, (NodeMsgFp)vmProcessSyncMsg, DEFAULT_HANDLE); diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index af7efb5543..96a8969b56 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -627,6 +627,7 @@ static int32_t mndSetDbCfgFromAlterDbReq(SDbObj *pDb, SAlterDbReq *pAlter) { if (pAlter->replications >= 0 && pAlter->replications != pDb->cfg.replications) { pDb->cfg.replications = pAlter->replications; + pDb->vgVersion++; terrno = 0; } @@ -652,26 +653,30 @@ static int32_t mndSetAlterDbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *p } static int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup) { - for (int32_t vn = 0; vn < pVgroup->replica; ++vn) { - STransAction action = {0}; - SVnodeGid *pVgid = pVgroup->vnodeGid + vn; + if (pVgroup->replica == pDb->cfg.replications) { + for (int32_t vn = 0; vn < pVgroup->replica; ++vn) { + STransAction action = {0}; + SVnodeGid *pVgid = pVgroup->vnodeGid + vn; - SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); - if (pDnode == NULL) return -1; - action.epSet = mndGetDnodeEpset(pDnode); - mndReleaseDnode(pMnode, pDnode); + SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); + if (pDnode == NULL) return -1; + action.epSet = mndGetDnodeEpset(pDnode); + mndReleaseDnode(pMnode, pDnode); - int32_t contLen = 0; - void *pReq = mndBuildAlterVnodeReq(pMnode, pDnode, pDb, pVgroup, &contLen); - if (pReq == NULL) return -1; + int32_t contLen = 0; + void *pReq = mndBuildAlterVnodeReq(pMnode, pDnode, pDb, pVgroup, &contLen); + if (pReq == NULL) return -1; - action.pCont = pReq; - action.contLen = contLen; - action.msgType = TDMT_VND_ALTER_VNODE; - if (mndTransAppendRedoAction(pTrans, &action) != 0) { - taosMemoryFree(pReq); - return -1; + action.pCont = pReq; + action.contLen = contLen; + action.msgType = TDMT_VND_ALTER_VNODE; + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + taosMemoryFree(pReq); + return -1; + } } + } else if (pVgroup->replica < pDb->cfg.replications) { + } else { } return 0; @@ -726,6 +731,7 @@ static int32_t mndProcessAlterDbReq(SNodeMsg *pReq) { SDbObj *pDb = NULL; SUserObj *pUser = NULL; SAlterDbReq alterReq = {0}; + SDbObj dbObj = {0}; if (tDeserializeSAlterDbReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &alterReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; @@ -749,15 +755,14 @@ static int32_t mndProcessAlterDbReq(SNodeMsg *pReq) { goto _OVER; } - SDbObj dbObj = {0}; memcpy(&dbObj, pDb, sizeof(SDbObj)); - dbObj.cfg.numOfRetensions = 0; - dbObj.cfg.pRetensions = NULL; + if (dbObj.cfg.pRetensions != NULL) { + dbObj.cfg.pRetensions = taosArrayDup(pDb->cfg.pRetensions); + if (dbObj.cfg.pRetensions == NULL) goto _OVER; + } code = mndSetDbCfgFromAlterDbReq(&dbObj, &alterReq); - if (code != 0) { - goto _OVER; - } + if (code != 0) goto _OVER; dbObj.cfgVersion++; dbObj.updateTime = taosGetTimestampMs(); @@ -771,6 +776,7 @@ _OVER: mndReleaseDb(pMnode, pDb); mndReleaseUser(pMnode, pUser); + taosArrayDestroy(dbObj.cfg.pRetensions); return code; } From b744c0f524da7dda0f394377184f1159210a921e Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 6 May 2022 16:42:02 +0800 Subject: [PATCH 12/31] refactor:fix schemaless error in table name,vgId,uid --- include/libs/parser/parser.h | 2 +- source/client/src/clientSml.c | 5 +++-- source/libs/parser/src/parInsert.c | 7 ++++--- source/libs/parser/src/parInsertData.c | 1 + 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index 0b9fdf94f9..7c9602734b 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -105,7 +105,7 @@ int32_t qCreateSName(SName* pName, const char* pTableName, int32_t acctId, char* void* smlInitHandle(SQuery *pQuery); void smlDestroyHandle(void *pHandle); -int32_t smlBindData(void *handle, SArray *tags, SArray *colsFormat, SHashObj *colsHash, SArray *cols, bool format, STableMeta *pTableMeta, char *msgBuf, int16_t msgBufLen); +int32_t smlBindData(void *handle, SArray *tags, SArray *colsFormat, SHashObj *colsHash, SArray *cols, bool format, STableMeta *pTableMeta, char *tableName, char *msgBuf, int16_t msgBufLen); int32_t smlBuildOutput(void* handle, SHashObj* pVgHash); #ifdef __cplusplus diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 55e95c2aa5..8b5712a619 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -143,7 +143,7 @@ static int32_t smlBuildInvalidDataMsg(SSmlMsgBuf* pBuf, const char *msg1, const } static int smlCompareKv(const void* p1, const void* p2) { - SSmlKv* kv1 = (SSmlKv *)p1; + SSmlKv* kv1 = (SSmlKv*)p1; SSmlKv* kv2 = (SSmlKv*)p2; int32_t kvLen1 = kv1->keyLen; int32_t kvLen2 = kv2->keyLen; @@ -1647,11 +1647,12 @@ static int32_t smlInsertData(SSmlHandle* info) { SSmlSTableMeta** pMeta = taosHashGet(info->superTables, tableData->sTableName, tableData->sTableNameLen); ASSERT (NULL != pMeta && NULL != *pMeta); + // use tablemeta of stable to save vgid and uid of child table (*pMeta)->tableMeta->vgId = vg.vgId; (*pMeta)->tableMeta->uid = tableData->uid; // one table merge data block together according uid code = smlBindData(info->exec, tableData->tags, tableData->colsFormat, tableData->columnsHash, - tableData->cols, info->dataFormat, (*pMeta)->tableMeta, info->msgBuf.buf, info->msgBuf.len); + tableData->cols, info->dataFormat, (*pMeta)->tableMeta, tableData->childTableName, info->msgBuf.buf, info->msgBuf.len); if(code != TSDB_CODE_SUCCESS){ return code; } diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index 4da5fcda69..05d247c037 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -279,6 +279,7 @@ static void buildMsgHeader(STableDataBlocks* src, SVgDataBlocks* blocks) { int32_t numOfBlocks = blocks->numOfTables; while (numOfBlocks--) { int32_t dataLen = blk->dataLen; + int32_t schemaLen = blk->schemaLen; blk->uid = htobe64(blk->uid); blk->suid = htobe64(blk->suid); blk->padding = htonl(blk->padding); @@ -286,7 +287,7 @@ static void buildMsgHeader(STableDataBlocks* src, SVgDataBlocks* blocks) { blk->dataLen = htonl(blk->dataLen); blk->schemaLen = htonl(blk->schemaLen); blk->numOfRows = htons(blk->numOfRows); - blk = (SSubmitBlk*)(blk->data + dataLen); + blk = (SSubmitBlk*)(blk->data + schemaLen + dataLen); } } @@ -1642,7 +1643,7 @@ static int32_t smlBoundTags(SArray *cols, SKVRowBuilder *tagsBuilder, SParsedDat } int32_t smlBindData(void *handle, SArray *tags, SArray *colsFormat, SHashObj *colsHash, SArray *cols, bool format, - STableMeta *pTableMeta, char *msgBuf, int16_t msgBufLen) { + STableMeta *pTableMeta, char *tableName, char *msgBuf, int16_t msgBufLen) { SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; SSmlExecHandle *smlHandle = (SSmlExecHandle *)handle; @@ -1659,7 +1660,7 @@ int32_t smlBindData(void *handle, SArray *tags, SArray *colsFormat, SHashObj *co return ret; } - buildCreateTbReq(&smlHandle->createTblReq, pTableMeta->schema->name, row, pTableMeta->suid); + buildCreateTbReq(&smlHandle->createTblReq, tableName, row, pTableMeta->suid); STableDataBlocks* pDataBlock = NULL; ret = getDataBlockFromList(smlHandle->pBlockHash, pTableMeta->uid, TSDB_DEFAULT_PAYLOAD_SIZE, diff --git a/source/libs/parser/src/parInsertData.c b/source/libs/parser/src/parInsertData.c index d24fca757e..ac197077a4 100644 --- a/source/libs/parser/src/parInsertData.c +++ b/source/libs/parser/src/parInsertData.c @@ -455,6 +455,7 @@ int32_t mergeTableDataBlocks(SHashObj* pHashObj, uint8_t payloadType, SArray** p SSubmitBlk* pBlocks = (SSubmitBlk*)pOneTableBlock->pData; if (pBlocks->numOfRows > 0) { STableDataBlocks* dataBuf = NULL; + pOneTableBlock->pTableMeta->vgId = pOneTableBlock->vgId; // for schemaless, restore origin vgId int32_t ret = getDataBlockFromList(pVnodeDataBlockHashList, pOneTableBlock->vgId, TSDB_PAYLOAD_SIZE, INSERT_HEAD_SIZE, 0, pOneTableBlock->pTableMeta, &dataBuf, pVnodeDataBlockList, NULL); From ae23dd2382a5d8a06dbb4e281bea2901c898ee80 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 6 May 2022 17:28:58 +0800 Subject: [PATCH 13/31] fix: alter db replications --- source/dnode/mnode/impl/inc/mndVgroup.h | 5 +- source/dnode/mnode/impl/src/mndDb.c | 235 +++++++++++++++--------- source/dnode/mnode/impl/src/mndVgroup.c | 8 + source/dnode/mnode/impl/test/db/db.cpp | 13 +- 4 files changed, 173 insertions(+), 88 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndVgroup.h b/source/dnode/mnode/impl/inc/mndVgroup.h index 8aecc22454..87be15a4fd 100644 --- a/source/dnode/mnode/impl/inc/mndVgroup.h +++ b/source/dnode/mnode/impl/inc/mndVgroup.h @@ -27,10 +27,13 @@ void mndCleanupVgroup(SMnode *pMnode); SVgObj *mndAcquireVgroup(SMnode *pMnode, int32_t vgId); void mndReleaseVgroup(SMnode *pMnode, SVgObj *pVgroup); SSdbRaw *mndVgroupActionEncode(SVgObj *pVgroup); -int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups); SEpSet mndGetVgroupEpset(SMnode *pMnode, const SVgObj *pVgroup); int32_t mndGetVnodesNum(SMnode *pMnode, int32_t dnodeId); +int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups); +int32_t mndAddVnodeToVgroup(SMnode *pMnode, SVgObj *pVgroup, SVnodeGid *new1, SVnodeGid *new2, SVnodeGid *exist); +int32_t mndRemoveVnodeFromVgroup(SMnode *pMnode, SVgObj *pVgroup, SVnodeGid *del1, SVnodeGid *del2, SVnodeGid *exist); + void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen); void *mndBuildDropVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen); void *mndBuildAlterVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen); diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 96a8969b56..cf9dd664e3 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -261,6 +261,104 @@ void mndReleaseDb(SMnode *pMnode, SDbObj *pDb) { sdbRelease(pSdb, pDb); } +static int32_t mndAddCreateVnodeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SVnodeGid *pVgid, + bool isRedo) { + STransAction action = {0}; + + SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); + if (pDnode == NULL) return -1; + action.epSet = mndGetDnodeEpset(pDnode); + mndReleaseDnode(pMnode, pDnode); + + int32_t contLen = 0; + void *pReq = mndBuildCreateVnodeReq(pMnode, pDnode, pDb, pVgroup, &contLen); + if (pReq == NULL) return -1; + + action.pCont = pReq; + action.contLen = contLen; + action.msgType = TDMT_DND_CREATE_VNODE; + action.acceptableCode = TSDB_CODE_NODE_ALREADY_DEPLOYED; + + if (isRedo) { + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + taosMemoryFree(pReq); + return -1; + } + } else { + if (mndTransAppendUndoAction(pTrans, &action) != 0) { + taosMemoryFree(pReq); + return -1; + } + } + + return 0; +} + +static int32_t mndAddAlterVnodeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SVnodeGid *pVgid, + bool isRedo) { + STransAction action = {0}; + + SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); + if (pDnode == NULL) return -1; + action.epSet = mndGetDnodeEpset(pDnode); + mndReleaseDnode(pMnode, pDnode); + + int32_t contLen = 0; + void *pReq = mndBuildAlterVnodeReq(pMnode, pDnode, pDb, pVgroup, &contLen); + if (pReq == NULL) return -1; + + action.pCont = pReq; + action.contLen = contLen; + action.msgType = TDMT_VND_ALTER_VNODE; + + if (isRedo) { + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + taosMemoryFree(pReq); + return -1; + } + } else { + if (mndTransAppendUndoAction(pTrans, &action) != 0) { + taosMemoryFree(pReq); + return -1; + } + } + + return 0; +} + +static int32_t mndAddDropVnodeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SVnodeGid *pVgid, + bool isRedo) { + STransAction action = {0}; + + SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); + if (pDnode == NULL) return -1; + action.epSet = mndGetDnodeEpset(pDnode); + mndReleaseDnode(pMnode, pDnode); + + int32_t contLen = 0; + void *pReq = mndBuildDropVnodeReq(pMnode, pDnode, pDb, pVgroup, &contLen); + if (pReq == NULL) return -1; + + action.pCont = pReq; + action.contLen = contLen; + action.msgType = TDMT_DND_DROP_VNODE; + action.acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED; + + if (isRedo) { + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + taosMemoryFree(pReq); + return -1; + } + } else { + if (mndTransAppendUndoAction(pTrans, &action) != 0) { + taosMemoryFree(pReq); + return -1; + } + } + + return 0; +} + static int32_t mndCheckDbName(const char *dbName, SUserObj *pUser) { char *pos = strstr(dbName, TS_PATH_DELIMITER); if (pos == NULL) { @@ -278,6 +376,8 @@ static int32_t mndCheckDbName(const char *dbName, SUserObj *pUser) { } static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg) { + terrno = TSDB_CODE_MND_INVALID_DB_OPTION; + if (pCfg->numOfVgroups < TSDB_MIN_VNODES_PER_DB || pCfg->numOfVgroups > TSDB_MAX_VNODES_PER_DB) return -1; if (pCfg->numOfStables < TSDB_DB_STREAM_MODE_OFF || pCfg->numOfStables > TSDB_DB_STREAM_MODE_ON) return -1; if (pCfg->buffer < TSDB_MIN_BUFFER_PER_VNODE || pCfg->buffer > TSDB_MAX_BUFFER_PER_VNODE) return -1; @@ -300,9 +400,10 @@ static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg) { if (pCfg->replications < TSDB_MIN_DB_REPLICA || pCfg->replications > TSDB_MAX_DB_REPLICA) return -1; if (pCfg->replications > mndGetDnodeSize(pMnode)) return -1; if (pCfg->strict < TSDB_DB_STRICT_OFF || pCfg->strict > TSDB_DB_STRICT_ON) return -1; - if (pCfg->strict > pCfg->replications) return -1; if (pCfg->cacheLastRow < TSDB_MIN_DB_CACHE_LAST_ROW || pCfg->cacheLastRow > TSDB_MAX_DB_CACHE_LAST_ROW) return -1; if (pCfg->hashMethod != 1) return -1; + + terrno = 0; return TSDB_CODE_SUCCESS; } @@ -381,24 +482,8 @@ static int32_t mndSetCreateDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj SVgObj *pVgroup = pVgroups + vg; for (int32_t vn = 0; vn < pVgroup->replica; ++vn) { - STransAction action = {0}; - SVnodeGid *pVgid = pVgroup->vnodeGid + vn; - - SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); - if (pDnode == NULL) return -1; - action.epSet = mndGetDnodeEpset(pDnode); - mndReleaseDnode(pMnode, pDnode); - - int32_t contLen = 0; - void *pReq = mndBuildCreateVnodeReq(pMnode, pDnode, pDb, pVgroup, &contLen); - if (pReq == NULL) return -1; - - action.pCont = pReq; - action.contLen = contLen; - action.msgType = TDMT_DND_CREATE_VNODE; - action.acceptableCode = TSDB_CODE_NODE_ALREADY_DEPLOYED; - if (mndTransAppendRedoAction(pTrans, &action) != 0) { - taosMemoryFree(pReq); + SVnodeGid *pVgid = pVgroup->vnodeGid + vn; + if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, pVgroup, pVgid, true) != 0) { return -1; } } @@ -412,24 +497,8 @@ static int32_t mndSetCreateDbUndoActions(SMnode *pMnode, STrans *pTrans, SDbObj SVgObj *pVgroup = pVgroups + vg; for (int32_t vn = 0; vn < pVgroup->replica; ++vn) { - STransAction action = {0}; - SVnodeGid *pVgid = pVgroup->vnodeGid + vn; - - SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); - if (pDnode == NULL) return -1; - action.epSet = mndGetDnodeEpset(pDnode); - mndReleaseDnode(pMnode, pDnode); - - int32_t contLen = 0; - void *pReq = mndBuildDropVnodeReq(pMnode, pDnode, pDb, pVgroup, &contLen); - if (pReq == NULL) return -1; - - action.pCont = pReq; - action.contLen = contLen; - action.msgType = TDMT_DND_DROP_VNODE; - action.acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED; - if (mndTransAppendUndoAction(pTrans, &action) != 0) { - taosMemoryFree(pReq); + SVnodeGid *pVgid = pVgroup->vnodeGid + vn; + if (mndAddDropVnodeAction(pMnode, pTrans, pDb, pVgroup, pVgid, false) != 0) { return -1; } } @@ -482,7 +551,6 @@ static int32_t mndCreateDb(SMnode *pMnode, SNodeMsg *pReq, SCreateDbReq *pCreate } if (mndCheckDbCfg(pMnode, &dbObj.cfg) != 0) { - terrno = TSDB_CODE_MND_INVALID_DB_OPTION; mError("db:%s, failed to create since %s", pCreate->db, terrstr()); return -1; } @@ -570,37 +638,37 @@ _OVER: static int32_t mndSetDbCfgFromAlterDbReq(SDbObj *pDb, SAlterDbReq *pAlter) { terrno = TSDB_CODE_MND_DB_OPTION_UNCHANGED; - if (pAlter->buffer >= 0 && pAlter->buffer != pDb->cfg.buffer) { + if (pAlter->buffer > 0 && pAlter->buffer != pDb->cfg.buffer) { pDb->cfg.buffer = pAlter->buffer; terrno = 0; } - if (pAlter->pages >= 0 && pAlter->pages != pDb->cfg.pages) { + if (pAlter->pages > 0 && pAlter->pages != pDb->cfg.pages) { pDb->cfg.pages = pAlter->pages; terrno = 0; } - if (pAlter->pageSize >= 0 && pAlter->pageSize != pDb->cfg.pageSize) { + if (pAlter->pageSize > 0 && pAlter->pageSize != pDb->cfg.pageSize) { pDb->cfg.pageSize = pAlter->pageSize; terrno = 0; } - if (pAlter->daysPerFile >= 0 && pAlter->daysPerFile != pDb->cfg.daysPerFile) { + if (pAlter->daysPerFile > 0 && pAlter->daysPerFile != pDb->cfg.daysPerFile) { pDb->cfg.daysPerFile = pAlter->daysPerFile; terrno = 0; } - if (pAlter->daysToKeep0 >= 0 && pAlter->daysToKeep0 != pDb->cfg.daysToKeep0) { + if (pAlter->daysToKeep0 > 0 && pAlter->daysToKeep0 != pDb->cfg.daysToKeep0) { pDb->cfg.daysToKeep0 = pAlter->daysToKeep0; terrno = 0; } - if (pAlter->daysToKeep1 >= 0 && pAlter->daysToKeep1 != pDb->cfg.daysToKeep1) { + if (pAlter->daysToKeep1 > 0 && pAlter->daysToKeep1 != pDb->cfg.daysToKeep1) { pDb->cfg.daysToKeep1 = pAlter->daysToKeep1; terrno = 0; } - if (pAlter->daysToKeep2 >= 0 && pAlter->daysToKeep2 != pDb->cfg.daysToKeep2) { + if (pAlter->daysToKeep2 > 0 && pAlter->daysToKeep2 != pDb->cfg.daysToKeep2) { pDb->cfg.daysToKeep2 = pAlter->daysToKeep2; terrno = 0; } @@ -625,7 +693,7 @@ static int32_t mndSetDbCfgFromAlterDbReq(SDbObj *pDb, SAlterDbReq *pAlter) { terrno = 0; } - if (pAlter->replications >= 0 && pAlter->replications != pDb->cfg.replications) { + if (pAlter->replications > 0 && pAlter->replications != pDb->cfg.replications) { pDb->cfg.replications = pAlter->replications; pDb->vgVersion++; terrno = 0; @@ -653,30 +721,44 @@ static int32_t mndSetAlterDbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *p } static int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup) { - if (pVgroup->replica == pDb->cfg.replications) { + if (pVgroup->replica <= 0 || pVgroup->replica == pDb->cfg.replications) { for (int32_t vn = 0; vn < pVgroup->replica; ++vn) { - STransAction action = {0}; - SVnodeGid *pVgid = pVgroup->vnodeGid + vn; - - SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); - if (pDnode == NULL) return -1; - action.epSet = mndGetDnodeEpset(pDnode); - mndReleaseDnode(pMnode, pDnode); - - int32_t contLen = 0; - void *pReq = mndBuildAlterVnodeReq(pMnode, pDnode, pDb, pVgroup, &contLen); - if (pReq == NULL) return -1; - - action.pCont = pReq; - action.contLen = contLen; - action.msgType = TDMT_VND_ALTER_VNODE; - if (mndTransAppendRedoAction(pTrans, &action) != 0) { - taosMemoryFree(pReq); + SVnodeGid *pVgid = pVgroup->vnodeGid + vn; + if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, pVgroup, pVgid, true) != 0) { return -1; } } - } else if (pVgroup->replica < pDb->cfg.replications) { } else { + SVgObj newVgroup = {0}; + memcpy(&newVgroup, pVgroup, sizeof(SVgObj)); + if (newVgroup.replica < pDb->cfg.replications) { + SVnodeGid new1 = {0}; + SVnodeGid new2 = {0}; + SVnodeGid exist = {0}; + if (mndAddVnodeToVgroup(pMnode, &newVgroup, &new1, &new2, &exist) != 0) { + mError("db:%s, failed to add vnode to vgId:%d since %s", pDb->name, newVgroup.vgId, terrstr()); + return -1; + } + if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVgroup, &new1, true) != 0) return -1; + if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVgroup, &new2, true) != 0) return -1; + if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVgroup, &exist, true) != 0) return -1; + } else { + SVnodeGid del1 = {0}; + SVnodeGid del2 = {0}; + SVnodeGid exist = {0}; + if (mndRemoveVnodeFromVgroup(pMnode, &newVgroup, &del1, &del2, &exist) != 0) { + mError("db:%s, failed to remove vnode from vgId:%d since %s", pDb->name, newVgroup.vgId, terrstr()); + return -1; + } + if (mndAddDropVnodeAction(pMnode, pTrans, pDb, &newVgroup, &del1, true) != 0) return -1; + if (mndAddDropVnodeAction(pMnode, pTrans, pDb, &newVgroup, &del2, true) != 0) return -1; + if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVgroup, &exist, true) != 0) return -1; + } + + SSdbRaw *pVgRaw = mndVgroupActionEncode(&newVgroup); + if (pVgRaw == NULL) return -1; + if (mndTransAppendCommitlog(pTrans, pVgRaw) != 0) return -1; + if (sdbSetRawStatus(pVgRaw, SDB_STATUS_READY) != 0) return -1; } return 0; @@ -764,6 +846,9 @@ static int32_t mndProcessAlterDbReq(SNodeMsg *pReq) { code = mndSetDbCfgFromAlterDbReq(&dbObj, &alterReq); if (code != 0) goto _OVER; + code = mndCheckDbCfg(pMnode, &dbObj.cfg); + if (code != 0) goto _OVER; + dbObj.cfgVersion++; dbObj.updateTime = taosGetTimestampMs(); code = mndAlterDb(pMnode, pReq, pDb, &dbObj); @@ -905,24 +990,8 @@ static int32_t mndSetDropDbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pD static int32_t mndBuildDropVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup) { for (int32_t vn = 0; vn < pVgroup->replica; ++vn) { - STransAction action = {0}; - SVnodeGid *pVgid = pVgroup->vnodeGid + vn; - - SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); - if (pDnode == NULL) return -1; - action.epSet = mndGetDnodeEpset(pDnode); - mndReleaseDnode(pMnode, pDnode); - - int32_t contLen = 0; - void *pReq = mndBuildDropVnodeReq(pMnode, pDnode, pDb, pVgroup, &contLen); - if (pReq == NULL) return -1; - - action.pCont = pReq; - action.contLen = contLen; - action.msgType = TDMT_DND_DROP_VNODE; - action.acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED; - if (mndTransAppendRedoAction(pTrans, &action) != 0) { - taosMemoryFree(pReq); + SVnodeGid *pVgid = pVgroup->vnodeGid + vn; + if (mndAddDropVnodeAction(pMnode, pTrans, pDb, pVgroup, pVgid, true) != 0) { return -1; } } diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index d1e4be1161..ca60a3b9ed 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -491,6 +491,14 @@ _OVER: return code; } +int32_t mndAddVnodeToVgroup(SMnode *pMnode, SVgObj *pVgroup, SVnodeGid *new1, SVnodeGid *new2, SVnodeGid *exist) { + return 0; +} + +int32_t mndRemoveVnodeFromVgroup(SMnode *pMnode, SVgObj *pVgroup, SVnodeGid *del1, SVnodeGid *del2, SVnodeGid *exist) { + return 0; +} + SEpSet mndGetVgroupEpset(SMnode *pMnode, const SVgObj *pVgroup) { SEpSet epset = {0}; diff --git a/source/dnode/mnode/impl/test/db/db.cpp b/source/dnode/mnode/impl/test/db/db.cpp index 3c5bf5c083..545f9f22bb 100644 --- a/source/dnode/mnode/impl/test/db/db.cpp +++ b/source/dnode/mnode/impl/test/db/db.cpp @@ -73,14 +73,19 @@ TEST_F(MndTestDb, 02_Create_Alter_Drop_Db) { { SAlterDbReq alterdbReq = {0}; strcpy(alterdbReq.db, "1.d1"); + alterdbReq.buffer = 12; - alterdbReq.daysToKeep0 = 300; - alterdbReq.daysToKeep1 = 400; - alterdbReq.daysToKeep2 = 500; + alterdbReq.pageSize = -1; + alterdbReq.pages = -1; + alterdbReq.daysPerFile = -1; + alterdbReq.daysToKeep0 = -1; + alterdbReq.daysToKeep1 = -1; + alterdbReq.daysToKeep2 = -1; alterdbReq.fsyncPeriod = 4000; alterdbReq.walLevel = 2; - alterdbReq.strict = 2; + alterdbReq.strict = 1; alterdbReq.cacheLastRow = 1; + alterdbReq.replications = 1; int32_t contLen = tSerializeSAlterDbReq(NULL, 0, &alterdbReq); void* pReq = rpcMallocCont(contLen); From 79b70ac2875b9293914d88aac4e2cff67b3bb8d8 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 6 May 2022 10:03:54 +0000 Subject: [PATCH 14/31] refact more --- source/dnode/vnode/src/tsdb/tsdbMemTable2.c | 98 ++++++++++++++++++--- 1 file changed, 86 insertions(+), 12 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c index ae0d4630d1..fd753afb2f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c @@ -15,10 +15,10 @@ #include "tsdb.h" -typedef struct SMemTable SMemTable; -typedef struct SMemData SMemData; -typedef struct SMemSkipList SMemSkipList; -typedef struct SMemSkipListCfg SMemSkipListCfg; +typedef struct SMemTable SMemTable; +typedef struct SMemData SMemData; +typedef struct SMemSkipList SMemSkipList; +typedef struct SMemSkipListNode SMemSkipListNode; struct SMemTable { STsdb *pTsdb; @@ -32,15 +32,16 @@ struct SMemTable { SMemData **pBuckets; }; -struct SMemSkipListCfg { - int8_t maxLevel; - int32_t nKey; - int32_t nData; +struct SMemSkipListNode { + int8_t level; + SMemSkipListNode *forwards[]; }; struct SMemSkipList { - int8_t level; - uint32_t seed; + uint32_t seed; + int8_t level; + int32_t size; + SMemSkipListNode pHead[]; }; struct SMemData { @@ -76,6 +77,7 @@ int32_t tsdbMemTableCreate2(STsdb *pTsdb, SMemTable **ppMemTb) { pMemTb->pBuckets = taosMemoryCalloc(pMemTb->nBucket, sizeof(*pMemTb->pBuckets)); if (pMemTb->pBuckets == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; + taosMemoryFree(pMemTb); return -1; } @@ -83,11 +85,83 @@ int32_t tsdbMemTableCreate2(STsdb *pTsdb, SMemTable **ppMemTb) { return 0; } -int32_t tsdbMemTableDestroy2(STsdb *pTsdb, SMemTable *pMT) { - // TODO +int32_t tsdbMemTableDestroy2(STsdb *pTsdb, SMemTable *pMemTb) { + if (pMemTb) { + // loop to destroy the contents (todo) + taosMemoryFree(pMemTb->pBuckets); + taosMemoryFree(pMemTb); + } return 0; } +int32_t tsdbInsertData2(SMemTable *pMemTb, int64_t version, const SVSubmitBlk *pSubmitBlk) { + SMemData *pMemData; + SVBufPool *pPool = pMemTb->pTsdb->pVnode->inUse; + int32_t hash; + int32_t tlen; + uint8_t buf[16]; + int32_t rSize; + SMemSkipListNode *pSlNode; + const STSRow *pTSRow; + + // search hash + hash = (pSubmitBlk->suid + pSubmitBlk->uid) % pMemTb->nBucket; + for (pMemData = pMemTb->pBuckets[hash]; pMemData; pMemData = pMemData->pHashNext) { + if (pMemData->suid == pSubmitBlk->suid && pMemData->uid == pSubmitBlk->uid) break; + } + + // create pMemData if need + if (pMemData == NULL) { + pMemData = vnodeBufPoolMalloc(pPool, sizeof(*pMemData)); + if (pMemData == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + pMemData->pHashNext = NULL; + pMemData->suid = pSubmitBlk->suid; + pMemData->uid = pSubmitBlk->uid; + pMemData->minKey = TSKEY_MAX; + pMemData->maxKey = TSKEY_MIN; + pMemData->minVer = -1; + pMemData->maxVer = -1; + pMemData->nRows = 0; + pMemData->sl.level = 0; + pMemData->sl.seed = taosRand(); + pMemData->sl.size = 0; + + // add to MemTable + hash = (pMemData->suid + pMemData->uid) % pMemTb->nBucket; + pMemData->pHashNext = pMemTb->pBuckets[hash]; + pMemTb->pBuckets[hash] = pMemData; + } + + // loop to insert data to skiplist + for (;;) { + rSize = 0; + pTSRow = NULL; + if (pTSRow == NULL) break; + + // check the row (todo) + + // move the cursor to position to write (todo) + + // insert the row + int8_t level = 1; + tlen = 0; // sizeof(int64_t) + tsdbPutLen(rSize) + rSize; + pSlNode = vnodeBufPoolMalloc(pPool, tlen); + if (pSlNode == NULL) { + ASSERT(0); + } + } + + return 0; +} + +static void tsdbEncodeRow(int64_t version, int32_t rSize, const STSRow *pRow) {} + +static void tsdbDecodeRow(int64_t *version, int32_t *rSize, const STSRow **ppRow) {} + // SMemData // SMemSkipList \ No newline at end of file From 5ec83cbf11cf415faaf66ddd9895b64e3da09163 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 6 May 2022 10:21:55 +0000 Subject: [PATCH 15/31] more --- source/dnode/vnode/src/tsdb/tsdbMemTable2.c | 25 ++++++++++++++------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c index fd753afb2f..1c263b7d00 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c @@ -39,6 +39,7 @@ struct SMemSkipListNode { struct SMemSkipList { uint32_t seed; + int8_t maxLevel; int8_t level; int32_t size; SMemSkipListNode pHead[]; @@ -96,11 +97,14 @@ int32_t tsdbMemTableDestroy2(STsdb *pTsdb, SMemTable *pMemTb) { int32_t tsdbInsertData2(SMemTable *pMemTb, int64_t version, const SVSubmitBlk *pSubmitBlk) { SMemData *pMemData; - SVBufPool *pPool = pMemTb->pTsdb->pVnode->inUse; + STsdb *pTsdb = pMemTb->pTsdb; + SVnode *pVnode = pTsdb->pVnode; + SVBufPool *pPool = pVnode->inUse; int32_t hash; int32_t tlen; uint8_t buf[16]; - int32_t rSize; + int32_t rlen; + const uint8_t *p; SMemSkipListNode *pSlNode; const STSRow *pTSRow; @@ -126,8 +130,9 @@ int32_t tsdbInsertData2(SMemTable *pMemTb, int64_t version, const SVSubmitBlk *p pMemData->minVer = -1; pMemData->maxVer = -1; pMemData->nRows = 0; - pMemData->sl.level = 0; pMemData->sl.seed = taosRand(); + pMemData->sl.maxLevel = pVnode->config.tsdbCfg.slLevel; + pMemData->sl.level = 0; pMemData->sl.size = 0; // add to MemTable @@ -137,9 +142,13 @@ int32_t tsdbInsertData2(SMemTable *pMemTb, int64_t version, const SVSubmitBlk *p } // loop to insert data to skiplist + p = pSubmitBlk->pData; for (;;) { - rSize = 0; - pTSRow = NULL; + if (p - (uint8_t *)pSubmitBlk->pData >= pSubmitBlk->nData) break; + + // p = tGetLen(p, &rlen); + pTSRow = (STSRow *)p; + p += rlen; if (pTSRow == NULL) break; // check the row (todo) @@ -148,7 +157,7 @@ int32_t tsdbInsertData2(SMemTable *pMemTb, int64_t version, const SVSubmitBlk *p // insert the row int8_t level = 1; - tlen = 0; // sizeof(int64_t) + tsdbPutLen(rSize) + rSize; + tlen = 0; // sizeof(int64_t) + tsdbPutLen(rlen) + rlen; pSlNode = vnodeBufPoolMalloc(pPool, tlen); if (pSlNode == NULL) { ASSERT(0); @@ -158,9 +167,9 @@ int32_t tsdbInsertData2(SMemTable *pMemTb, int64_t version, const SVSubmitBlk *p return 0; } -static void tsdbEncodeRow(int64_t version, int32_t rSize, const STSRow *pRow) {} +static void tsdbEncodeRow(int64_t version, int32_t rlen, const STSRow *pRow) {} -static void tsdbDecodeRow(int64_t *version, int32_t *rSize, const STSRow **ppRow) {} +static void tsdbDecodeRow(int64_t *version, int32_t *rlen, const STSRow **ppRow) {} // SMemData From 68fc2acc9caad5f1587f83d3767db1c5b529d0c4 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 6 May 2022 11:16:53 +0000 Subject: [PATCH 16/31] refact --- source/dnode/vnode/src/tsdb/tsdbMemTable2.c | 114 ++++++++++++++------ 1 file changed, 81 insertions(+), 33 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c index 1c263b7d00..e7b733369a 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c @@ -15,10 +15,11 @@ #include "tsdb.h" -typedef struct SMemTable SMemTable; -typedef struct SMemData SMemData; -typedef struct SMemSkipList SMemSkipList; -typedef struct SMemSkipListNode SMemSkipListNode; +typedef struct SMemTable SMemTable; +typedef struct SMemData SMemData; +typedef struct SMemSkipList SMemSkipList; +typedef struct SMemSkipListNode SMemSkipListNode; +typedef struct SMemSkipListCurosr SMemSkipListCurosr; struct SMemTable { STsdb *pTsdb; @@ -57,6 +58,20 @@ struct SMemData { SMemSkipList sl; }; +struct SMemSkipListCurosr { + SMemSkipList *pSl; + SMemSkipListNode *pNodeC; +}; + +#define SL_NODE_SIZE(l) (sizeof(SMemSkipListNode) + sizeof(SMemSkipListNode *) * (l)*2) +#define SL_NODE_HALF_SIZE(l) (sizeof(SMemSkipListNode) + sizeof(SMemSkipListNode *) * (l)) +#define SL_NODE_FORWARD(n, l) ((n)->forwards[l]) +#define SL_NODE_BACKWARD(n, l) ((n)->forwards[(n)->level + (l)]) +#define SL_NODE_DATA(n) (&SL_NODE_BACKWARD(n, (n)->level)) + +#define SL_HEAD_NODE(sl) ((sl)->pHead) +#define SL_TAIL_NODE(sl) ((SMemSkipListNode *)&SL_NODE_FORWARD(SL_HEAD_NODE(sl), (sl)->maxLevel)) + // SMemTable int32_t tsdbMemTableCreate2(STsdb *pTsdb, SMemTable **ppMemTb) { SMemTable *pMemTb = NULL; @@ -96,17 +111,18 @@ int32_t tsdbMemTableDestroy2(STsdb *pTsdb, SMemTable *pMemTb) { } int32_t tsdbInsertData2(SMemTable *pMemTb, int64_t version, const SVSubmitBlk *pSubmitBlk) { - SMemData *pMemData; - STsdb *pTsdb = pMemTb->pTsdb; - SVnode *pVnode = pTsdb->pVnode; - SVBufPool *pPool = pVnode->inUse; - int32_t hash; - int32_t tlen; - uint8_t buf[16]; - int32_t rlen; - const uint8_t *p; - SMemSkipListNode *pSlNode; - const STSRow *pTSRow; + SMemData *pMemData; + STsdb *pTsdb = pMemTb->pTsdb; + SVnode *pVnode = pTsdb->pVnode; + SVBufPool *pPool = pVnode->inUse; + int32_t hash; + int32_t tlen; + uint8_t buf[16]; + int32_t rlen; + const uint8_t *p; + SMemSkipListNode *pSlNode; + const STSRow *pTSRow; + SMemSkipListCurosr slc = {0}; // search hash hash = (pSubmitBlk->suid + pSubmitBlk->uid) % pMemTb->nBucket; @@ -116,7 +132,11 @@ int32_t tsdbInsertData2(SMemTable *pMemTb, int64_t version, const SVSubmitBlk *p // create pMemData if need if (pMemData == NULL) { - pMemData = vnodeBufPoolMalloc(pPool, sizeof(*pMemData)); + int8_t maxLevel = pVnode->config.tsdbCfg.slLevel; + int32_t tsize = sizeof(*pMemData) + SL_NODE_HALF_SIZE(maxLevel) * 2; + SMemSkipListNode *pHead, *pTail; + + pMemData = vnodeBufPoolMalloc(pPool, tsize); if (pMemData == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -131,46 +151,74 @@ int32_t tsdbInsertData2(SMemTable *pMemTb, int64_t version, const SVSubmitBlk *p pMemData->maxVer = -1; pMemData->nRows = 0; pMemData->sl.seed = taosRand(); - pMemData->sl.maxLevel = pVnode->config.tsdbCfg.slLevel; + pMemData->sl.maxLevel = maxLevel; pMemData->sl.level = 0; pMemData->sl.size = 0; + pHead = SL_HEAD_NODE(&pMemData->sl); + pTail = SL_TAIL_NODE(&pMemData->sl); + pHead->level = maxLevel; + pTail->level = maxLevel; + for (int iLevel = 0; iLevel < maxLevel; iLevel++) { + SL_NODE_FORWARD(pHead, iLevel) = pTail; + SL_NODE_FORWARD(pTail, iLevel) = pHead; + } // add to MemTable hash = (pMemData->suid + pMemData->uid) % pMemTb->nBucket; pMemData->pHashNext = pMemTb->pBuckets[hash]; pMemTb->pBuckets[hash] = pMemData; + pMemTb->nHash++; } - // loop to insert data to skiplist +// loop to insert data to skiplist +#if 0 + tsdbMemSkipListCursorOpen(&slc, &pMemData->sl); p = pSubmitBlk->pData; for (;;) { if (p - (uint8_t *)pSubmitBlk->pData >= pSubmitBlk->nData) break; - // p = tGetLen(p, &rlen); - pTSRow = (STSRow *)p; - p += rlen; - if (pTSRow == NULL) break; + const uint8_t *pt = p; + p = tGetBinary(p, &pTSRow, &rlen); // check the row (todo) // move the cursor to position to write (todo) + int32_t c; + tsdbMemSkipListCursorMoveTo(&slc, pTSRow, version, &c); + ASSERT(c); - // insert the row - int8_t level = 1; - tlen = 0; // sizeof(int64_t) + tsdbPutLen(rlen) + rlen; - pSlNode = vnodeBufPoolMalloc(pPool, tlen); - if (pSlNode == NULL) { - ASSERT(0); - } + // encode row + int8_t level = tsdbMemSkipListRandLevel(&pMemData->sl); + int32_t tsize = SL_NODE_SIZE(level) + sizeof(version) + (p - pt); + pSlNode = vnodeBufPoolMalloc(pPool, tsize); + pSlNode->level = level; + + uint8_t *pData = SL_NODE_DATA(pSlNode); + *(int64_t *)pData = version; + pData += sizeof(version); + memcpy(pData, pt, p - pt); + + // insert row + tsdbMemSkipListCursorPut(&slc, pSlNode); + + // update status + if (pTSRow->ts < pMemData->minKey) pMemData->minKey = pTSRow->ts; + if (pTSRow->ts > pMemData->maxKey) pMemData->maxKey = pTSRow->ts; } + tsdbMemSkipListCursorClose(&slc); +#endif + + if (pMemData->minVer == -1) pMemData->minVer = version; + if (pMemData->maxVer == -1 || pMemData->maxVer < version) pMemData->maxVer = version; + + if (pMemTb->minKey < pMemData->minKey) pMemTb->minKey = pMemData->minKey; + if (pMemTb->maxKey < pMemData->maxKey) pMemTb->maxKey = pMemData->maxKey; + if (pMemTb->minVer == -1) pMemTb->minVer = version; + if (pMemTb->maxVer == -1 || pMemTb->maxVer < version) pMemTb->maxVer = version; return 0; } -static void tsdbEncodeRow(int64_t version, int32_t rlen, const STSRow *pRow) {} - -static void tsdbDecodeRow(int64_t *version, int32_t *rlen, const STSRow **ppRow) {} - // SMemData // SMemSkipList \ No newline at end of file From 7e30e1b50baaaa874b2e15255fde5666627af912 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 6 May 2022 20:00:30 +0800 Subject: [PATCH 17/31] refactor:merge from 3.0 --- source/client/src/clientSml.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 8b5712a619..85cd0661e3 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -147,7 +147,7 @@ static int smlCompareKv(const void* p1, const void* p2) { SSmlKv* kv2 = (SSmlKv*)p2; int32_t kvLen1 = kv1->keyLen; int32_t kvLen2 = kv2->keyLen; - int32_t res = strncasecmp(kv1->key, kv2->key, MIN(kvLen1, kvLen2)); + int32_t res = strncasecmp(kv1->key, kv2->key, TMIN(kvLen1, kvLen2)); if (res != 0) { return res; } else { From aed53ac9e168a5efcb38571f10bf6dba87eefcfb Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 6 May 2022 13:06:10 +0000 Subject: [PATCH 18/31] feat: create table if not exists --- include/common/tmsg.h | 4 ++-- source/common/src/tmsg.c | 10 +++++----- source/dnode/vnode/src/vnd/vnodeSvr.c | 6 +++++- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 3889784f2f..763ff61e80 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -672,7 +672,6 @@ typedef struct { SArray* pArray; // Array of SUseDbRsp } SUseDbBatchRsp; - int32_t tSerializeSUseDbBatchRsp(void* buf, int32_t bufLen, SUseDbBatchRsp* pRsp); int32_t tDeserializeSUseDbBatchRsp(void* buf, int32_t bufLen, SUseDbBatchRsp* pRsp); void tFreeSUseDbBatchRsp(SUseDbBatchRsp* pRsp); @@ -685,7 +684,6 @@ int32_t tSerializeSUserAuthBatchRsp(void* buf, int32_t bufLen, SUserAuthBatchRsp int32_t tDeserializeSUserAuthBatchRsp(void* buf, int32_t bufLen, SUserAuthBatchRsp* pRsp); void tFreeSUserAuthBatchRsp(SUserAuthBatchRsp* pRsp); - typedef struct { char db[TSDB_DB_FNAME_LEN]; } SCompactDbReq; @@ -1554,7 +1552,9 @@ typedef struct SVDropStbReq { int32_t tEncodeSVDropStbReq(SCoder* pCoder, const SVDropStbReq* pReq); int32_t tDecodeSVDropStbReq(SCoder* pCoder, SVDropStbReq* pReq); +#define TD_CREATE_IF_NOT_EXISTS 0x1 typedef struct SVCreateTbReq { + int32_t flags; tb_uid_t uid; int64_t ctime; const char* name; diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 6278b52a04..85845dd5c4 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -1255,7 +1255,6 @@ int32_t tDeserializeSGetUserAuthRspImpl(SCoder *pDecoder, SGetUserAuthRsp *pRsp) return 0; } - int32_t tDeserializeSGetUserAuthRsp(void *buf, int32_t bufLen, SGetUserAuthRsp *pRsp) { SCoder decoder = {0}; tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); @@ -2091,7 +2090,7 @@ void tFreeSUseDbBatchRsp(SUseDbBatchRsp *pRsp) { taosArrayDestroy(pRsp->pArray); } -int32_t tSerializeSUserAuthBatchRsp(void* buf, int32_t bufLen, SUserAuthBatchRsp* pRsp){ +int32_t tSerializeSUserAuthBatchRsp(void *buf, int32_t bufLen, SUserAuthBatchRsp *pRsp) { SCoder encoder = {0}; tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); @@ -2110,7 +2109,7 @@ int32_t tSerializeSUserAuthBatchRsp(void* buf, int32_t bufLen, SUserAuthBatchRsp return tlen; } -int32_t tDeserializeSUserAuthBatchRsp(void* buf, int32_t bufLen, SUserAuthBatchRsp* pRsp){ +int32_t tDeserializeSUserAuthBatchRsp(void *buf, int32_t bufLen, SUserAuthBatchRsp *pRsp) { SCoder decoder = {0}; tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); @@ -2136,7 +2135,7 @@ int32_t tDeserializeSUserAuthBatchRsp(void* buf, int32_t bufLen, SUserAuthBatchR return 0; } -void tFreeSUserAuthBatchRsp(SUserAuthBatchRsp* pRsp){ +void tFreeSUserAuthBatchRsp(SUserAuthBatchRsp *pRsp) { int32_t numOfBatch = taosArrayGetSize(pRsp->pArray); for (int32_t i = 0; i < numOfBatch; ++i) { SGetUserAuthRsp *pUserAuthRsp = taosArrayGet(pRsp->pArray, i); @@ -2146,7 +2145,6 @@ void tFreeSUserAuthBatchRsp(SUserAuthBatchRsp* pRsp){ taosArrayDestroy(pRsp->pArray); } - int32_t tSerializeSDbCfgReq(void *buf, int32_t bufLen, SDbCfgReq *pReq) { SCoder encoder = {0}; tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); @@ -3746,6 +3744,7 @@ STSchema *tdGetSTSChemaFromSSChema(SSchema **pSchema, int32_t nCols) { int tEncodeSVCreateTbReq(SCoder *pCoder, const SVCreateTbReq *pReq) { if (tStartEncode(pCoder) < 0) return -1; + if (tEncodeI32v(pCoder, pReq->flags) < 0) return -1; if (tEncodeI64(pCoder, pReq->uid) < 0) return -1; if (tEncodeI64(pCoder, pReq->ctime) < 0) return -1; @@ -3771,6 +3770,7 @@ int tDecodeSVCreateTbReq(SCoder *pCoder, SVCreateTbReq *pReq) { if (tStartDecode(pCoder) < 0) return -1; + if (tDecodeI32v(pCoder, &pReq->flags) < 0) return -1; if (tDecodeI64(pCoder, &pReq->uid) < 0) return -1; if (tDecodeI64(pCoder, &pReq->ctime) < 0) return -1; diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 8460400b59..811c5c10e4 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -360,7 +360,11 @@ static int vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pReq, // do create table if (metaCreateTable(pVnode->pMeta, version, pCreateReq) < 0) { - cRsp.code = terrno; + if (pCreateReq->flags & TD_CREATE_IF_NOT_EXISTS && terrno == TSDB_CODE_TDB_TABLE_ALREADY_EXIST) { + cRsp.code = TSDB_CODE_SUCCESS; + } else { + cRsp.code = terrno; + } } else { cRsp.code = TSDB_CODE_SUCCESS; tsdbFetchTbUidList(pVnode->pTsdb, &pStore, pCreateReq->ctb.suid, pCreateReq->uid); From 3ac4beb2eea0aa86d3293a8b6e7d329f2960c233 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 6 May 2022 21:06:27 +0800 Subject: [PATCH 19/31] refactor:add compile option in windows --- cmake/cmake.define | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/cmake.define b/cmake/cmake.define index aeab39cab4..2b4fe5c42f 100644 --- a/cmake/cmake.define +++ b/cmake/cmake.define @@ -46,7 +46,7 @@ ENDIF () IF (TD_WINDOWS) MESSAGE("${Yellow} set compiler flag for Windows! ${ColourReset}") - SET(COMMON_FLAGS "/W3 /D_WIN32") + SET(COMMON_FLAGS "/W3 /D_WIN32 /vmg") # IF (MSVC AND (MSVC_VERSION GREATER_EQUAL 1900)) # SET(COMMON_FLAGS "${COMMON_FLAGS} /Wv:18") From 0886d23e02d1c91ca323bf397034dbf3746feaae Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 6 May 2022 22:17:56 +0800 Subject: [PATCH 20/31] fix: alter db replications --- source/dnode/mnode/impl/inc/mndVgroup.h | 5 +- source/dnode/mnode/impl/src/mndDb.c | 35 ++++--- source/dnode/mnode/impl/src/mndVgroup.c | 94 ++++++++++++++++--- source/dnode/mnode/impl/test/dnode/mdnode.cpp | 92 ++++++++++++++++++ 4 files changed, 199 insertions(+), 27 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndVgroup.h b/source/dnode/mnode/impl/inc/mndVgroup.h index 87be15a4fd..1e95859157 100644 --- a/source/dnode/mnode/impl/inc/mndVgroup.h +++ b/source/dnode/mnode/impl/inc/mndVgroup.h @@ -31,8 +31,9 @@ SEpSet mndGetVgroupEpset(SMnode *pMnode, const SVgObj *pVgroup); int32_t mndGetVnodesNum(SMnode *pMnode, int32_t dnodeId); int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups); -int32_t mndAddVnodeToVgroup(SMnode *pMnode, SVgObj *pVgroup, SVnodeGid *new1, SVnodeGid *new2, SVnodeGid *exist); -int32_t mndRemoveVnodeFromVgroup(SMnode *pMnode, SVgObj *pVgroup, SVnodeGid *del1, SVnodeGid *del2, SVnodeGid *exist); +SArray *mndBuildDnodesArray(SMnode *pMnode); +int32_t mndAddVnodeToVgroup(SMnode *pMnode, SVgObj *pVgroup, SArray *pArray); +int32_t mndRemoveVnodeFromVgroup(SMnode *pMnode, SVgObj *pVgroup, SArray *pArray, SVnodeGid *del1, SVnodeGid *del2); void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen); void *mndBuildDropVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen); diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index cf9dd664e3..c3e4629f2a 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -399,6 +399,7 @@ static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg) { if (pCfg->compression < TSDB_MIN_COMP_LEVEL || pCfg->compression > TSDB_MAX_COMP_LEVEL) return -1; if (pCfg->replications < TSDB_MIN_DB_REPLICA || pCfg->replications > TSDB_MAX_DB_REPLICA) return -1; if (pCfg->replications > mndGetDnodeSize(pMnode)) return -1; + if (pCfg->replications != 1 && pCfg->replications != 3) return -1; if (pCfg->strict < TSDB_DB_STRICT_OFF || pCfg->strict > TSDB_DB_STRICT_ON) return -1; if (pCfg->cacheLastRow < TSDB_MIN_DB_CACHE_LAST_ROW || pCfg->cacheLastRow > TSDB_MAX_DB_CACHE_LAST_ROW) return -1; if (pCfg->hashMethod != 1) return -1; @@ -720,7 +721,7 @@ static int32_t mndSetAlterDbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *p return 0; } -static int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup) { +static int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SArray *pArray) { if (pVgroup->replica <= 0 || pVgroup->replica == pDb->cfg.replications) { for (int32_t vn = 0; vn < pVgroup->replica; ++vn) { SVnodeGid *pVgid = pVgroup->vnodeGid + vn; @@ -732,27 +733,30 @@ static int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj SVgObj newVgroup = {0}; memcpy(&newVgroup, pVgroup, sizeof(SVgObj)); if (newVgroup.replica < pDb->cfg.replications) { - SVnodeGid new1 = {0}; - SVnodeGid new2 = {0}; - SVnodeGid exist = {0}; - if (mndAddVnodeToVgroup(pMnode, &newVgroup, &new1, &new2, &exist) != 0) { + mInfo("db:%s, vgId:%d, will add 2 vnodes, vn:0 dnode:%d", pVgroup->dbName, pVgroup->vgId, + pVgroup->vnodeGid[0].dnodeId); + + if (mndAddVnodeToVgroup(pMnode, &newVgroup, pArray) != 0) { mError("db:%s, failed to add vnode to vgId:%d since %s", pDb->name, newVgroup.vgId, terrstr()); return -1; } - if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVgroup, &new1, true) != 0) return -1; - if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVgroup, &new2, true) != 0) return -1; - if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVgroup, &exist, true) != 0) return -1; + newVgroup.replica = pDb->cfg.replications; + if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVgroup, &newVgroup.vnodeGid[0], true) != 0) return -1; + if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVgroup, &newVgroup.vnodeGid[1], true) != 0) return -1; + if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVgroup, &newVgroup.vnodeGid[2], true) != 0) return -1; } else { + mInfo("db:%s, vgId:%d, will remove 2 vnodes", pVgroup->dbName, pVgroup->vgId); + SVnodeGid del1 = {0}; SVnodeGid del2 = {0}; - SVnodeGid exist = {0}; - if (mndRemoveVnodeFromVgroup(pMnode, &newVgroup, &del1, &del2, &exist) != 0) { + if (mndRemoveVnodeFromVgroup(pMnode, &newVgroup, pArray, &del1, &del2) != 0) { mError("db:%s, failed to remove vnode from vgId:%d since %s", pDb->name, newVgroup.vgId, terrstr()); return -1; } + newVgroup.replica = pDb->cfg.replications; + if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVgroup, &newVgroup.vnodeGid[0], true) != 0) return -1; if (mndAddDropVnodeAction(pMnode, pTrans, pDb, &newVgroup, &del1, true) != 0) return -1; if (mndAddDropVnodeAction(pMnode, pTrans, pDb, &newVgroup, &del2, true) != 0) return -1; - if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVgroup, &exist, true) != 0) return -1; } SSdbRaw *pVgRaw = mndVgroupActionEncode(&newVgroup); @@ -765,8 +769,9 @@ static int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj } static int32_t mndSetAlterDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pOld, SDbObj *pNew) { - SSdb *pSdb = pMnode->pSdb; - void *pIter = NULL; + SSdb *pSdb = pMnode->pSdb; + void *pIter = NULL; + SArray *pArray = mndBuildDnodesArray(pMnode); while (1) { SVgObj *pVgroup = NULL; @@ -774,9 +779,10 @@ static int32_t mndSetAlterDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj * if (pIter == NULL) break; if (pVgroup->dbUid == pNew->uid) { - if (mndBuildAlterVgroupAction(pMnode, pTrans, pNew, pVgroup) != 0) { + if (mndBuildAlterVgroupAction(pMnode, pTrans, pNew, pVgroup, pArray) != 0) { sdbCancelFetch(pSdb, pIter); sdbRelease(pSdb, pVgroup); + taosArrayDestroy(pArray); return -1; } } @@ -784,6 +790,7 @@ static int32_t mndSetAlterDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj * sdbRelease(pSdb, pVgroup); } + taosArrayDestroy(pArray); return 0; } diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index ca60a3b9ed..48a475ec63 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -370,7 +370,7 @@ static bool mndBuildDnodesArrayFp(SMnode *pMnode, void *pObj, void *p1, void *p2 return true; } -static SArray *mndBuildDnodesArray(SMnode *pMnode) { +SArray *mndBuildDnodesArray(SMnode *pMnode) { SSdb *pSdb = pMnode->pSdb; int32_t numOfDnodes = mndGetDnodeSize(pMnode); @@ -421,7 +421,7 @@ static int32_t mndGetAvailableDnode(SMnode *pMnode, SVgObj *pVgroup, SArray *pAr pVgid->role = TAOS_SYNC_STATE_FOLLOWER; } - mDebug("db:%s, vgId:%d, vn:%d dnode:%d is alloced", pVgroup->dbName, pVgroup->vgId, v, pVgid->dnodeId); + mInfo("db:%s, vgId:%d, vn:%d dnode:%d is alloced", pVgroup->dbName, pVgroup->vgId, v, pVgid->dnodeId); pDnode->numOfVnodes++; } @@ -440,12 +440,10 @@ int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups) { } pArray = mndBuildDnodesArray(pMnode); - if (pArray == NULL) { - goto _OVER; - } + if (pArray == NULL) goto _OVER; - mDebug("db:%s, total %d dnodes used to create %d vgroups (%d vnodes)", pDb->name, (int32_t)taosArrayGetSize(pArray), - pDb->cfg.numOfVgroups, pDb->cfg.numOfVgroups * pDb->cfg.replications); + mInfo("db:%s, total %d dnodes used to create %d vgroups (%d vnodes)", pDb->name, (int32_t)taosArrayGetSize(pArray), + pDb->cfg.numOfVgroups, pDb->cfg.numOfVgroups * pDb->cfg.replications); int32_t allocedVgroups = 0; int32_t maxVgId = sdbGetMaxId(pMnode->pSdb, SDB_VGROUP); @@ -483,7 +481,7 @@ int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups) { *ppVgroups = pVgroups; code = 0; - mDebug("db:%s, %d vgroups is alloced, replica:%d", pDb->name, pDb->cfg.numOfVgroups, pDb->cfg.replications); + mInfo("db:%s, %d vgroups is alloced, replica:%d", pDb->name, pDb->cfg.numOfVgroups, pDb->cfg.replications); _OVER: if (code != 0) taosMemoryFree(pVgroups); @@ -491,11 +489,85 @@ _OVER: return code; } -int32_t mndAddVnodeToVgroup(SMnode *pMnode, SVgObj *pVgroup, SVnodeGid *new1, SVnodeGid *new2, SVnodeGid *exist) { - return 0; +int32_t mndAddVnodeToVgroup(SMnode *pMnode, SVgObj *pVgroup, SArray *pArray) { + taosArraySort(pArray, (__compar_fn_t)mndCompareDnodeVnodes); + for (int32_t i = 0; i < taosArrayGetSize(pArray); ++i) { + SDnodeObj *pDnode = taosArrayGet(pArray, i); + mDebug("dnode:%d, equivalent vnodes:%d", pDnode->id, pDnode->numOfVnodes); + } + + int32_t maxPos = 1; + for (int32_t d = 0; d < taosArrayGetSize(pArray); ++d) { + SDnodeObj *pDnode = taosArrayGet(pArray, d); + + bool used = false; + for (int32_t vn = 0; vn < maxPos; ++vn) { + if (pDnode->id == pVgroup->vnodeGid[vn].dnodeId) { + used = true; + break; + } + } + if (used) continue; + + if (pDnode == NULL || pDnode->numOfVnodes > pDnode->numOfSupportVnodes) { + terrno = TSDB_CODE_MND_NO_ENOUGH_DNODES; + return -1; + } + + SVnodeGid *pVgid = &pVgroup->vnodeGid[maxPos]; + pVgid->dnodeId = pDnode->id; + pVgid->role = TAOS_SYNC_STATE_FOLLOWER; + pDnode->numOfVnodes++; + + mInfo("db:%s, vgId:%d, vn:%d dnode:%d is added", pVgroup->dbName, pVgroup->vgId, maxPos, pVgid->dnodeId); + maxPos++; + if (maxPos == 3) return 0; + } + + terrno = TSDB_CODE_MND_NO_ENOUGH_DNODES; + return -1; } -int32_t mndRemoveVnodeFromVgroup(SMnode *pMnode, SVgObj *pVgroup, SVnodeGid *del1, SVnodeGid *del2, SVnodeGid *exist) { +int32_t mndRemoveVnodeFromVgroup(SMnode *pMnode, SVgObj *pVgroup, SArray *pArray, SVnodeGid *del1, SVnodeGid *del2) { + int32_t removedNum = 0; + + taosArraySort(pArray, (__compar_fn_t)mndCompareDnodeVnodes); + for (int32_t i = 0; i < taosArrayGetSize(pArray); ++i) { + SDnodeObj *pDnode = taosArrayGet(pArray, i); + mDebug("dnode:%d, equivalent vnodes:%d", pDnode->id, pDnode->numOfVnodes); + } + + for (int32_t d = taosArrayGetSize(pArray) - 1; d >= 0; --d) { + SDnodeObj *pDnode = taosArrayGet(pArray, d); + + for (int32_t vn = 0; vn < TSDB_MAX_REPLICA; ++vn) { + SVnodeGid *pVgid = &pVgroup->vnodeGid[vn]; + if (pVgid->dnodeId == pDnode->id) { + if (removedNum == 0) *del1 = *pVgid; + if (removedNum == 1) *del2 = *pVgid; + + mInfo("db:%s, vgId:%d, vn:%d dnode:%d is removed", pVgroup->dbName, pVgroup->vgId, vn, pVgid->dnodeId); + memset(pVgid, 0, sizeof(SVnodeGid)); + removedNum++; + pDnode->numOfVnodes--; + + if (removedNum == 2) goto _OVER; + } + } + } + +_OVER: + if (removedNum != 2) return -1; + + for (int32_t vn = 1; vn < TSDB_MAX_REPLICA; ++vn) { + SVnodeGid *pVgid = &pVgroup->vnodeGid[vn]; + if (pVgid->dnodeId != 0) { + memcpy(&pVgroup->vnodeGid[0], pVgid, sizeof(SVnodeGid)); + memset(pVgid, 0, sizeof(SVnodeGid)); + } + } + + mInfo("db:%s, vgId:%d, dnode:%d is keeped", pVgroup->dbName, pVgroup->vgId, pVgroup->vnodeGid[0].dnodeId); return 0; } diff --git a/source/dnode/mnode/impl/test/dnode/mdnode.cpp b/source/dnode/mnode/impl/test/dnode/mdnode.cpp index a4cbc201a9..15ade8166f 100644 --- a/source/dnode/mnode/impl/test/dnode/mdnode.cpp +++ b/source/dnode/mnode/impl/test/dnode/mdnode.cpp @@ -266,4 +266,96 @@ TEST_F(MndTestDnode, 05_Create_Drop_Restart_Dnode) { taosMsleep(1300); test.SendShowReq(TSDB_MGMT_TABLE_DNODE, "dnodes", ""); EXPECT_EQ(test.GetShowRows(), 4); + + // alter replica +#if 0 + { + SCreateDbReq createReq = {0}; + strcpy(createReq.db, "1.d2"); + createReq.numOfVgroups = 2; + createReq.buffer = -1; + createReq.pageSize = -1; + createReq.pages = -1; + createReq.daysPerFile = 1000; + createReq.daysToKeep0 = 3650; + createReq.daysToKeep1 = 3650; + createReq.daysToKeep2 = 3650; + createReq.minRows = 100; + createReq.maxRows = 4096; + createReq.fsyncPeriod = 3000; + createReq.walLevel = 1; + createReq.precision = 0; + createReq.compression = 2; + createReq.replications = 1; + createReq.strict = 1; + createReq.cacheLastRow = 0; + createReq.ignoreExist = 1; + createReq.numOfStables = 0; + createReq.numOfRetensions = 0; + + int32_t contLen = tSerializeSCreateDbReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSCreateDbReq(pReq, contLen, &createReq); + + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); + + test.SendShowReq(TSDB_MGMT_TABLE_DB, "user_databases", ""); + EXPECT_EQ(test.GetShowRows(), 3); + } + + { + SAlterDbReq alterdbReq = {0}; + strcpy(alterdbReq.db, "1.d2"); + + alterdbReq.buffer = 12; + alterdbReq.pageSize = -1; + alterdbReq.pages = -1; + alterdbReq.daysPerFile = -1; + alterdbReq.daysToKeep0 = -1; + alterdbReq.daysToKeep1 = -1; + alterdbReq.daysToKeep2 = -1; + alterdbReq.fsyncPeriod = 4000; + alterdbReq.walLevel = 2; + alterdbReq.strict = 1; + alterdbReq.cacheLastRow = 1; + alterdbReq.replications = 3; + + int32_t contLen = tSerializeSAlterDbReq(NULL, 0, &alterdbReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSAlterDbReq(pReq, contLen, &alterdbReq); + + SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_DB, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); + } + + { + SAlterDbReq alterdbReq = {0}; + strcpy(alterdbReq.db, "1.d2"); + + alterdbReq.buffer = 12; + alterdbReq.pageSize = -1; + alterdbReq.pages = -1; + alterdbReq.daysPerFile = -1; + alterdbReq.daysToKeep0 = -1; + alterdbReq.daysToKeep1 = -1; + alterdbReq.daysToKeep2 = -1; + alterdbReq.fsyncPeriod = 4000; + alterdbReq.walLevel = 2; + alterdbReq.strict = 1; + alterdbReq.cacheLastRow = 1; + alterdbReq.replications = 1; + + int32_t contLen = tSerializeSAlterDbReq(NULL, 0, &alterdbReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSAlterDbReq(pReq, contLen, &alterdbReq); + + SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_DB, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); + } + +#endif } From 384fed49f986d9af907109648b76708090cc745b Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 6 May 2022 22:32:22 +0800 Subject: [PATCH 21/31] enh: add authVersion for user privilege --- source/dnode/mnode/impl/src/mndUser.c | 21 ++++++++++----------- tools/taos-tools | 2 +- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index 8b5fa06faf..81ded4ef8b 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -106,6 +106,7 @@ SSdbRaw *mndUserActionEncode(SUserObj *pUser) { SDB_SET_INT64(pRaw, dataPos, pUser->createdTime, _OVER) SDB_SET_INT64(pRaw, dataPos, pUser->updateTime, _OVER) SDB_SET_INT8(pRaw, dataPos, pUser->superUser, _OVER) + SDB_SET_INT32(pRaw, dataPos, pUser->authVersion, _OVER) SDB_SET_INT32(pRaw, dataPos, numOfReadDbs, _OVER) SDB_SET_INT32(pRaw, dataPos, numOfWriteDbs, _OVER) @@ -161,6 +162,7 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) { SDB_GET_INT64(pRaw, dataPos, &pUser->createdTime, _OVER) SDB_GET_INT64(pRaw, dataPos, &pUser->updateTime, _OVER) SDB_GET_INT8(pRaw, dataPos, &pUser->superUser, _OVER) + SDB_GET_INT32(pRaw, dataPos, &pUser->authVersion, _OVER) int32_t numOfReadDbs = 0; int32_t numOfWriteDbs = 0; @@ -588,7 +590,7 @@ _OVER: return code; } -static int32_t mndSetUserAuthRsp(SMnode *pMnode, SUserObj *pUser, SGetUserAuthRsp *pRsp) { +static int32_t mndSetUserAuthRsp(SMnode *pMnode, SUserObj *pUser, SGetUserAuthRsp *pRsp) { memcpy(pRsp->user, pUser->user, TSDB_USER_LEN); pRsp->superAuth = pUser->superUser; pRsp->version = pUser->authVersion; @@ -601,7 +603,7 @@ static int32_t mndSetUserAuthRsp(SMnode *pMnode, SUserObj *pUser, SGetUser terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - + SSdb *pSdb = pMnode->pSdb; void *pIter = NULL; while (1) { @@ -659,7 +661,7 @@ static int32_t mndProcessGetUserAuthReq(SNodeMsg *pReq) { code = 0; _OVER: - + mndReleaseUser(pMnode, pUser); tFreeSGetUserAuthRsp(&authRsp); @@ -711,7 +713,8 @@ static void mndCancelGetNextUser(SMnode *pMnode, void *pIter) { sdbCancelFetch(pSdb, pIter); } -int32_t mndValidateUserAuthInfo(SMnode *pMnode, SUserAuthVersion *pUsers, int32_t numOfUses, void **ppRsp, int32_t *pRspLen) { +int32_t mndValidateUserAuthInfo(SMnode *pMnode, SUserAuthVersion *pUsers, int32_t numOfUses, void **ppRsp, + int32_t *pRspLen) { SUserAuthBatchRsp batchRsp = {0}; batchRsp.pArray = taosArrayInit(numOfUses, sizeof(SGetUserAuthRsp)); if (batchRsp.pArray == NULL) { @@ -731,7 +734,7 @@ int32_t mndValidateUserAuthInfo(SMnode *pMnode, SUserAuthVersion *pUsers, int32_ mndReleaseUser(pMnode, pUser); continue; } - + SGetUserAuthRsp rsp = {0}; code = mndSetUserAuthRsp(pMnode, pUser, &rsp); if (code) { @@ -740,7 +743,6 @@ int32_t mndValidateUserAuthInfo(SMnode *pMnode, SUserAuthVersion *pUsers, int32_ goto _OVER; } - taosArrayPush(batchRsp.pArray, &rsp); mndReleaseUser(pMnode, pUser); } @@ -748,7 +750,7 @@ int32_t mndValidateUserAuthInfo(SMnode *pMnode, SUserAuthVersion *pUsers, int32_ if (taosArrayGetSize(batchRsp.pArray) <= 0) { *ppRsp = NULL; *pRspLen = 0; - + tFreeSUserAuthBatchRsp(&batchRsp); return 0; } @@ -772,10 +774,7 @@ _OVER: *ppRsp = NULL; *pRspLen = 0; - + tFreeSUserAuthBatchRsp(&batchRsp); return code; } - - - diff --git a/tools/taos-tools b/tools/taos-tools index 59e0ebaf49..2f3dfddd4d 160000 --- a/tools/taos-tools +++ b/tools/taos-tools @@ -1 +1 @@ -Subproject commit 59e0ebaf4905e4cb6d95a01c58b3fa507abc5a20 +Subproject commit 2f3dfddd4d9a869e706ba3cf98fb6d769404cd7c From 99efc7334aa59741cd1784574a8449f0505f5cdf Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 6 May 2022 22:53:49 +0800 Subject: [PATCH 22/31] enh: add authVersion for user privilege --- source/dnode/mnode/impl/src/mndDb.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index c3e4629f2a..70bdda5855 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -1070,6 +1070,17 @@ static int32_t mndDropDb(SMnode *pMnode, SNodeMsg *pReq, SDbObj *pDb) { /*if (mndDropTopicByDB(pMnode, pTrans, pDb) != 0) goto _OVER;*/ if (mndSetDropDbRedoActions(pMnode, pTrans, pDb) != 0) goto _OVER; + SUserObj *pUser = mndAcquireUser(pMnode, pDb->createUser); + if (pUser != NULL) { + pUser->authVersion++; + SSdbRaw *pCommitRaw = mndUserActionEncode(pUser); + if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { + mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr()); + goto _OVER; + } + sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); + } + int32_t rspLen = 0; void *pRsp = NULL; if (mndBuildDropDbRsp(pDb, &rspLen, &pRsp, false) < 0) goto _OVER; From 612bdec1f746eebd57b12db9519c6924f18fb041 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 6 May 2022 22:58:02 +0800 Subject: [PATCH 23/31] fix: invalid header file --- source/dnode/mnode/impl/src/mndSma.c | 2 +- source/dnode/mnode/impl/src/mndStb.c | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index 8619df978b..0ec4332d37 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -21,7 +21,7 @@ #include "mndInfoSchema.h" #include "mndMnode.h" #include "mndShow.h" -#include "mndStb.c" +#include "mndStb.h" #include "mndStream.h" #include "mndTrans.h" #include "mndUser.h" diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 8ae0d5d19c..c32008cb5c 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include "mndStb.h" #include "mndAuth.h" #include "mndDb.h" From b38f18020d73bf43d68387660195f8891fb1d401 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 6 May 2022 23:04:25 +0800 Subject: [PATCH 24/31] enh: check kill trans auth --- source/dnode/mnode/impl/inc/mndAuth.h | 1 + source/dnode/mnode/impl/src/mndAuth.c | 24 +++++++++--------------- source/dnode/mnode/impl/src/mndTrans.c | 3 +-- 3 files changed, 11 insertions(+), 17 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndAuth.h b/source/dnode/mnode/impl/inc/mndAuth.h index ae7663a775..890879912f 100644 --- a/source/dnode/mnode/impl/inc/mndAuth.h +++ b/source/dnode/mnode/impl/inc/mndAuth.h @@ -31,6 +31,7 @@ int32_t mndCheckDropUserAuth(SUserObj *pOperUser); int32_t mndCheckNodeAuth(SUserObj *pOperUser); int32_t mndCheckFuncAuth(SUserObj *pOperUser); +int32_t mndCheckTransAuth(SUserObj *pOperUser); int32_t mndCheckCreateDbAuth(SUserObj *pOperUser); int32_t mndCheckAlterDropCompactDbAuth(SUserObj *pOperUser, SDbObj *pDb); diff --git a/source/dnode/mnode/impl/src/mndAuth.c b/source/dnode/mnode/impl/src/mndAuth.c index 696d850266..8e5ec40c47 100644 --- a/source/dnode/mnode/impl/src/mndAuth.c +++ b/source/dnode/mnode/impl/src/mndAuth.c @@ -74,10 +74,7 @@ static int32_t mndProcessAuthReq(SNodeMsg *pReq) { } int32_t mndCheckCreateUserAuth(SUserObj *pOperUser) { - if (pOperUser->superUser) { - return 0; - } - + if (pOperUser->superUser) return 0; terrno = TSDB_CODE_MND_NO_RIGHTS; return -1; } @@ -118,28 +115,25 @@ int32_t mndCheckAlterUserAuth(SUserObj *pOperUser, SUserObj *pUser, SDbObj *pDb, } int32_t mndCheckDropUserAuth(SUserObj *pOperUser) { - if (pOperUser->superUser) { - return 0; - } - + if (pOperUser->superUser) return 0; terrno = TSDB_CODE_MND_NO_RIGHTS; return -1; } int32_t mndCheckNodeAuth(SUserObj *pOperUser) { - if (pOperUser->superUser) { - return 0; - } - + if (pOperUser->superUser) return 0; terrno = TSDB_CODE_MND_NO_RIGHTS; return -1; } int32_t mndCheckFuncAuth(SUserObj *pOperUser) { - if (pOperUser->superUser) { - return 0; - } + if (pOperUser->superUser) return 0; + terrno = TSDB_CODE_MND_NO_RIGHTS; + return -1; +} +int32_t mndCheckTransAuth(SUserObj *pOperUser) { + if (pOperUser->superUser) return 0; terrno = TSDB_CODE_MND_NO_RIGHTS; return -1; } diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 43e2735984..e57e9a0461 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -1336,8 +1336,7 @@ static int32_t mndProcessKillTransReq(SNodeMsg *pReq) { goto _OVER; } - if (!pUser->superUser) { - terrno = TSDB_CODE_MND_NO_RIGHTS; + if (mndCheckTransAuth(pUser) != 0) { goto _OVER; } From d75b3568a70dce7e39c22fc982f69c852878be73 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Fri, 6 May 2022 23:40:34 +0800 Subject: [PATCH 25/31] enh(sync): add trace log --- source/libs/sync/src/syncRaftLog.c | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index ae153251c3..031722ab3c 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -57,7 +57,13 @@ int32_t logStoreAppendEntry(SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry) { syncMeta.seqNum = pEntry->seqNum; syncMeta.term = pEntry->term; code = walWriteWithSyncInfo(pWal, pEntry->index, pEntry->originalRpcType, syncMeta, pEntry->data, pEntry->dataLen); - assert(code == 0); + if (code != 0) { + int32_t err = terrno; + const char *errStr = tstrerror(err); + sError("walWriteWithSyncInfo error, err:%d, msg:%s", err, errStr); + ASSERT(0); + } + //assert(code == 0); walFsync(pWal, true); return code; @@ -69,7 +75,14 @@ SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) { if (index >= SYNC_INDEX_BEGIN && index <= logStoreLastIndex(pLogStore)) { SWalReadHandle* pWalHandle = walOpenReadHandle(pWal); - assert(walReadWithHandle(pWalHandle, index) == 0); + int32_t code = walReadWithHandle(pWalHandle, index); + if (code != 0) { + int32_t err = terrno; + const char *errStr = tstrerror(err); + sError("walReadWithHandle error, err:%d, msg:%s", err, errStr); + ASSERT(0); + } + //assert(walReadWithHandle(pWalHandle, index) == 0); SSyncRaftEntry* pEntry = syncEntryBuild(pWalHandle->pHead->head.bodyLen); assert(pEntry != NULL); From ce0841e398972eab05044a2f29a727b6e4ee8316 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sat, 7 May 2022 01:58:01 +0000 Subject: [PATCH 26/31] fix: auto-create table --- source/dnode/vnode/src/meta/metaTable.c | 2 +- source/libs/parser/src/parInsert.c | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 397e074061..ffe176f85e 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -196,7 +196,7 @@ int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq) { int8_t type; int64_t ctime; tb_uid_t suid; - int c, ret; + int c = 0, ret; // search & delete the name idx tdbDbcOpen(pMeta->pNameIdx, &pNameIdxc, &pMeta->txn); diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index e82873b923..8396a33122 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -279,6 +279,7 @@ static void buildMsgHeader(STableDataBlocks* src, SVgDataBlocks* blocks) { int32_t numOfBlocks = blocks->numOfTables; while (numOfBlocks--) { int32_t dataLen = blk->dataLen; + int32_t schemaLen = blk->schemaLen; blk->uid = htobe64(blk->uid); blk->suid = htobe64(blk->suid); blk->padding = htonl(blk->padding); @@ -286,7 +287,7 @@ static void buildMsgHeader(STableDataBlocks* src, SVgDataBlocks* blocks) { blk->dataLen = htonl(blk->dataLen); blk->schemaLen = htonl(blk->schemaLen); blk->numOfRows = htons(blk->numOfRows); - blk = (SSubmitBlk*)(blk->data + dataLen); + blk = (SSubmitBlk*)(blk->data + schemaLen + dataLen); } } From 861fde25087bec9594cf7c6f324e07754ed9a34c Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Sat, 7 May 2022 10:21:51 +0800 Subject: [PATCH 27/31] fix(stream): create stb in stream --- example/src/tstream.c | 8 +++--- source/dnode/mnode/impl/inc/mndDef.h | 3 +-- source/dnode/mnode/impl/src/mndSma.c | 1 - source/dnode/mnode/impl/src/mndStream.c | 34 +++++++++++++++---------- source/libs/wal/src/walRead.c | 5 +++- 5 files changed, 30 insertions(+), 21 deletions(-) diff --git a/example/src/tstream.c b/example/src/tstream.c index 5bd833213d..65fd005954 100644 --- a/example/src/tstream.c +++ b/example/src/tstream.c @@ -81,10 +81,10 @@ int32_t create_stream() { /*const char* sql = "select min(k), max(k), sum(k) as sum_of_k from st1";*/ /*const char* sql = "select sum(k) from tu1 interval(10m)";*/ /*pRes = tmq_create_stream(pConn, "stream1", "out1", sql);*/ - pRes = - taos_query(pConn, - "create stream stream1 trigger window_close as select _wstartts, min(k), max(k), sum(k) as sum_of_k " - "from tu1 interval(10m)"); + pRes = taos_query( + pConn, + "create stream stream1 trigger window_close into outstb as select _wstartts, min(k), max(k), sum(k) as sum_of_k " + "from tu1 interval(10m)"); if (taos_errno(pRes) != 0) { printf("failed to create stream stream1, reason:%s\n", taos_errstr(pRes)); return -1; diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 89638524ae..9ae7922f6b 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -72,7 +72,7 @@ typedef enum { TRN_TYPE_DROP_USER = 1003, TRN_TYPE_CREATE_FUNC = 1004, TRN_TYPE_DROP_FUNC = 1005, - + TRN_TYPE_CREATE_SNODE = 1006, TRN_TYPE_DROP_SNODE = 1007, TRN_TYPE_CREATE_QNODE = 1008, @@ -601,7 +601,6 @@ typedef struct { int32_t triggerParam; int64_t waterMark; char* sql; - char* logicalPlan; char* physicalPlan; SArray* tasks; // SArray> SSchemaWrapper outputSchema; diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index 8619df978b..91a2bd4740 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -419,7 +419,6 @@ static int32_t mndCreateSma(SMnode *pMnode, SNodeMsg *pReq, SMCreateSmaReq *pCre streamObj.fixedSinkVgId = smaObj.dstVgId; streamObj.smaId = smaObj.uid; /*streamObj.physicalPlan = "";*/ - streamObj.logicalPlan = "not implemented"; int32_t code = -1; STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_CREATE_SMA, &pReq->rpcMsg); diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 1404b1cd94..aa7f16e10d 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -290,7 +290,7 @@ int32_t mndAddStreamToTrans(SMnode *pMnode, SStreamObj *pStream, const char *ast return 0; } -static SStbObj *mndCreateStbForStream(SMnode *pMnode, STrans *pTrans, const SStreamObj *pStream, const char *user) { +static int32_t mndCreateStbForStream(SMnode *pMnode, STrans *pTrans, const SStreamObj *pStream, const char *user) { SStbObj *pStb = NULL; SDbObj *pDb = NULL; SUserObj *pUser = NULL; @@ -301,7 +301,22 @@ static SStbObj *mndCreateStbForStream(SMnode *pMnode, STrans *pTrans, const SStr createReq.numOfTags = 1; // group id createReq.pColumns = taosArrayInit(createReq.numOfColumns, sizeof(SField)); // build fields + taosArraySetSize(createReq.pColumns, createReq.numOfColumns); + for (int32_t i = 0; i < createReq.numOfColumns; i++) { + SField *pField = taosArrayGet(createReq.pColumns, i); + tstrncpy(pField->name, pStream->outputSchema.pSchema[i].name, TSDB_COL_NAME_LEN); + pField->flags = pStream->outputSchema.pSchema[i].flags; + pField->type = pStream->outputSchema.pSchema[i].type; + pField->bytes = pStream->outputSchema.pSchema[i].bytes; + } + createReq.pTags = taosArrayInit(createReq.numOfTags, sizeof(SField)); + taosArraySetSize(createReq.pTags, 1); // build tags + SField *pField = taosArrayGet(createReq.pTags, 0); + strcpy(pField->name, "group_id"); + pField->type = TSDB_DATA_TYPE_UBIGINT; + pField->flags = 0; + pField->bytes = 8; if (mndCheckCreateStbReq(&createReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; @@ -342,18 +357,14 @@ static SStbObj *mndCreateStbForStream(SMnode *pMnode, STrans *pTrans, const SStr goto _OVER; } - if (mndBuildStbFromReq(pMnode, pStb, &createReq, pDb) != 0) { - goto _OVER; - } - if (mndAddStbToTrans(pMnode, pTrans, pDb, &stbObj) < 0) goto _OVER; - return pStb; + return 0; _OVER: mndReleaseStb(pMnode, pStb); mndReleaseDb(pMnode, pDb); mndReleaseUser(pMnode, pUser); - return NULL; + return -1; } static int32_t mndCreateStream(SMnode *pMnode, SNodeMsg *pReq, SCMCreateStreamReq *pCreate, SDbObj *pDb) { @@ -373,7 +384,6 @@ static int32_t mndCreateStream(SMnode *pMnode, SNodeMsg *pReq, SCMCreateStreamRe streamObj.fixedSinkVgId = 0; streamObj.smaId = 0; /*streamObj.physicalPlan = "";*/ - streamObj.logicalPlan = "not implemented"; streamObj.trigger = pCreate->triggerType; streamObj.waterMark = pCreate->watermark; @@ -384,16 +394,14 @@ static int32_t mndCreateStream(SMnode *pMnode, SNodeMsg *pReq, SCMCreateStreamRe } mDebug("trans:%d, used to create stream:%s", pTrans->id, pCreate->name); -#if 0 - if (mndCreateStbForStream(pMnode, pTrans, &streamObj, pReq->user) < 0) { + if (mndAddStreamToTrans(pMnode, &streamObj, pCreate->ast, pCreate->triggerType, pCreate->watermark, pTrans) != 0) { mError("trans:%d, failed to add stream since %s", pTrans->id, terrstr()); mndTransDrop(pTrans); return -1; } -#endif - if (mndAddStreamToTrans(pMnode, &streamObj, pCreate->ast, pCreate->triggerType, pCreate->watermark, pTrans) != 0) { - mError("trans:%d, failed to add stream since %s", pTrans->id, terrstr()); + if (streamObj.targetSTbName[0] && mndCreateStbForStream(pMnode, pTrans, &streamObj, pReq->user) < 0) { + mError("trans:%d, failed to create stb for stream since %s", pTrans->id, terrstr()); mndTransDrop(pTrans); return -1; } diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index 70a3559cd9..4fe07029f1 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -49,7 +49,10 @@ void walCloseReadHandle(SWalReadHandle *pRead) { taosMemoryFree(pRead); } -int32_t walRegisterRead(SWalReadHandle *pRead, int64_t ver) { return 0; } +int32_t walRegisterRead(SWalReadHandle *pRead, int64_t ver) { + // TODO + return 0; +} static int32_t walReadSeekFilePos(SWalReadHandle *pRead, int64_t fileFirstVer, int64_t ver) { int code = 0; From f942f144e9b3958a72189a2d568c7c7496f77b68 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Sat, 7 May 2022 11:12:46 +0800 Subject: [PATCH 28/31] feat: bitmap operation optimization --- source/dnode/vnode/src/inc/tsdb.h | 5 ++-- source/dnode/vnode/src/tsdb/tsdbCommit.c | 10 ++++---- source/dnode/vnode/src/tsdb/tsdbRead.c | 29 +++++++++++++++------- source/dnode/vnode/src/tsdb/tsdbReadImpl.c | 29 ++++------------------ 4 files changed, 32 insertions(+), 41 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index d84c1b1d07..58003a97d7 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -325,9 +325,8 @@ typedef struct { typedef struct { int16_t colId; uint16_t type : 6; - uint16_t blen : 10; // bitmap length(TODO: full UT for the bitmap compress of various data input) - uint32_t bitmap : 1; // 0: no bitmap if all rows are NORM, 1: has bitmap if has NULL/NORM rows - uint32_t len : 31; // data length + bitmap length + uint16_t blen : 10; // 0 no bitmap if all rows are NORM, > 0 bitmap length + uint32_t len; // data length + bitmap length uint32_t offset; } SBlockColV0; diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index a008c4a0e5..f0b1baf1da 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -943,16 +943,16 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDF &(pAggrBlkCol->numOfNull)); if (pAggrBlkCol->numOfNull == 0) { - TD_SET_COL_ROWS_NORM(pBlockCol); + pBlockCol->blen = 0; } else { - TD_SET_COL_ROWS_MISC(pBlockCol); + pBlockCol->blen = 1; } ++nColsOfBlockSma; } else if (tdIsBitmapBlkNorm(pDataCol->pBitmap, rowsToWrite, pDataCols->bitmapMode)) { // check if all rows normal - TD_SET_COL_ROWS_NORM(pBlockCol); + pBlockCol->blen = 0; } else { - TD_SET_COL_ROWS_MISC(pBlockCol); + pBlockCol->blen = 1; } ++nColsNotAllNull; @@ -985,7 +985,7 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDF #ifdef TD_SUPPORT_BITMAP int32_t tBitmaps = 0; int32_t tBitmapsLen = 0; - if ((ncol != 0) && !TD_COL_ROWS_NORM(pBlockCol)) { + if ((ncol != 0) && (pBlockCol->blen > 0)) { tBitmaps = isSuper ? sBitmaps : nBitmaps; } #endif diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index aa305dd71c..0b9f21dbcf 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -1619,7 +1619,7 @@ static int32_t mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capa SCellVal sVal = {0}; TSKEY rowKey = TSKEY_INITIAL_VAL; int32_t nResult = 0; - bool isMerge = true; + int32_t mergeOption = 0; // 0 discard 1 overwrite 2 merge // the schema version info is embeded in STSRow int32_t numOfColsOfRow1 = 0; @@ -1715,12 +1715,18 @@ static int32_t mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capa if (colId == PRIMARYKEY_TIMESTAMP_COL_ID) { rowKey = *(TSKEY*)sVal.val; if (rowKey != *lastRowKey) { - isMerge = false; + mergeOption = 1; if (*lastRowKey != TSKEY_INITIAL_VAL) { ++(*curRow); } ++nResult; + } else if (update){ + mergeOption = 2; + } else { + mergeOption = 0; + break; } + *lastRowKey = rowKey; } } else { @@ -1730,11 +1736,16 @@ static int32_t mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capa tdSKvRowGetVal(row, PRIMARYKEY_TIMESTAMP_COL_ID, -1, -1, &sVal); rowKey = *(TSKEY*)sVal.val; if (rowKey != *lastRowKey) { - isMerge = false; + mergeOption = 1; if (*lastRowKey != TSKEY_INITIAL_VAL) { ++(*curRow); } ++nResult; + } else if(update) { + mergeOption = 2; + } else { + mergeOption = 0; + break; } *lastRowKey = rowKey; } else { @@ -1754,7 +1765,7 @@ static int32_t mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capa colDataAppend(pColInfo, *curRow, NULL, true); } else if (tdValTypeIsNone(sVal.valType)) { // TODO: Set null if nothing append for this row - if (!isMerge) { + if (mergeOption == 1) { colDataAppend(pColInfo, *curRow, NULL, true); } } else { @@ -1769,14 +1780,14 @@ static int32_t mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capa ++k; } } else { - if (!isMerge) { + if (mergeOption == 1) { colDataAppend(pColInfo, *curRow, NULL, true); } ++i; } } - if (*lastRowKey != rowKey) { + if (mergeOption == 1) { while (i < numOfCols) { // the remain columns are all null data SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i); colDataAppend(pColInfo, *curRow, NULL, true); @@ -2008,7 +2019,7 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf } numOfRows += mergeTwoRowFromMem(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, &curRow, row1, row2, numOfCols, - pCheckInfo->tableId, pSchema1, pSchema2, true, &lastRowKey); + pCheckInfo->tableId, pSchema1, pSchema2, pCfg->update, &lastRowKey); // numOfRows += 1; if (cur->win.skey == TSKEY_INITIAL_VAL) { cur->win.skey = key; @@ -2065,7 +2076,7 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf } numOfRows += mergeTwoRowFromMem(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, &curRow, row1, row2, numOfCols, - pCheckInfo->tableId, pSchema1, pSchema2, true, &lastRowKey); + pCheckInfo->tableId, pSchema1, pSchema2, pCfg->update, &lastRowKey); // ++numOfRows; if (cur->win.skey == TSKEY_INITIAL_VAL) { cur->win.skey = key; @@ -2747,7 +2758,7 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int rv = TD_ROW_SVER(row); } numOfRows += mergeTwoRowFromMem(pTsdbReadHandle, maxRowsToRead, &curRows, row, NULL, numOfCols, pCheckInfo->tableId, pSchema, - NULL, true, &lastRowKey); + NULL, pCfg->update, &lastRowKey); if (numOfRows >= maxRowsToRead) { moveToNextRowInMem(pCheckInfo); diff --git a/source/dnode/vnode/src/tsdb/tsdbReadImpl.c b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c index 49d02adc3a..bebdfb3b63 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadImpl.c +++ b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c @@ -616,7 +616,7 @@ static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDat tcolId = pBlockCol->colId; toffset = tsdbGetBlockColOffset(pBlockCol); tlen = pBlockCol->len; - pDataCol->bitmap = pBlockCol->bitmap; + pDataCol->bitmap = pBlockCol->blen > 0 ? 1 : 0; } else { ASSERT(pDataCol->colId == tcolId); TD_SET_COL_ROWS_NORM(pDataCol); @@ -624,17 +624,8 @@ static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDat // int32_t tBitmaps = 0; int32_t tLenBitmap = 0; - if ((dcol != 0) && !TD_COL_ROWS_NORM(pBlockCol)) { + if ((dcol != 0) && (pBlockCol->blen > 0)) { tLenBitmap = nBitmaps; -#if 0 - if (IS_VAR_DATA_TYPE(pDataCol->type)) { - tBitmaps = nBitmaps; - tLenBitmap = tBitmaps; - } else { - tBitmaps = (int32_t)ceil((double)nBitmaps / TYPE_BYTES[pDataCol->type]); - tLenBitmap = tBitmaps * TYPE_BYTES[pDataCol->type]; - } -#endif } if (tcolId == pDataCol->colId) { @@ -784,8 +775,7 @@ static int tsdbLoadBlockDataColsImpl(SReadH *pReadh, SBlock *pBlock, SDataCols * if (colId == PRIMARYKEY_TIMESTAMP_COL_ID) { // load the key row blockCol.colId = colId; - TD_SET_COL_ROWS_NORM(&blockCol); // default is NORM for the primary key column - blockCol.blen = 0; + blockCol.blen = 0; // default is NORM for the primary key column blockCol.len = pBlock->keyLen; blockCol.type = pDataCol->type; blockCol.offset = TSDB_KEY_COL_OFFSET; @@ -815,7 +805,7 @@ static int tsdbLoadBlockDataColsImpl(SReadH *pReadh, SBlock *pBlock, SDataCols * ASSERT(pBlockCol->colId == pDataCol->colId); } // set the bitmap - pDataCol->bitmap = pBlockCol->bitmap; + pDataCol->bitmap = pBlockCol->blen > 0 ? 1 : 0; if (tsdbLoadColData(pReadh, pDFile, pBlock, pBlockCol, pDataCol) < 0) return -1; } @@ -833,17 +823,8 @@ static int tsdbLoadColData(SReadH *pReadh, SDFile *pDFile, SBlock *pBlock, SBloc // int32_t tBitmaps = 0; int32_t tLenBitmap = 0; - if (!TD_COL_ROWS_NORM(pBlockCol)) { + if (pBlockCol->blen) { tLenBitmap = nBitmaps; -#if 0 - if (IS_VAR_DATA_TYPE(pDataCol->type)) { - tBitmaps = nBitmaps; - tLenBitmap = tBitmaps; - } else { - tBitmaps = (int32_t)ceil((double)nBitmaps / TYPE_BYTES[pDataCol->type]); - tLenBitmap = tBitmaps * TYPE_BYTES[pDataCol->type]; - } -#endif } int tsize = pDataCol->bytes * pBlock->numOfRows + tLenBitmap + 2 * COMP_OVERFLOW_BYTES; From fdd901ae5db522a6d190778d654ea00f5650e9fd Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sat, 7 May 2022 03:28:15 +0000 Subject: [PATCH 29/31] fix auto-create table problem --- source/dnode/vnode/src/vnd/vnodeSvr.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 811c5c10e4..403c02b440 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -533,6 +533,13 @@ static int vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, in } } + msgIter.uid = createTbReq.uid; + if (createTbReq.type == TSDB_CHILD_TABLE) { + msgIter.suid = createTbReq.ctb.suid; + } else { + msgIter.suid = 0; + } + tCoderClear(&coder); } From 3b73dc295f8d34341ff6ff820dca9161600b2a0d Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Sat, 7 May 2022 11:53:08 +0800 Subject: [PATCH 30/31] set table uid --- source/dnode/vnode/src/tq/tqRead.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 5ec3ab0b47..f531d3f5fb 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -173,7 +173,7 @@ int32_t tqRetrieveDataBlock(SArray** ppCols, STqReadHandle* pHandle, uint64_t* p int32_t curRow = 0; tInitSubmitBlkIter(&pHandle->msgIter, pHandle->pBlock, &pHandle->blkIter); - *pUid = pHandle->pBlock->uid; // set the uid of table for submit block + *pUid = pHandle->msgIter.uid; // set the uid of table for submit block while ((row = tGetSubmitBlkNext(&pHandle->blkIter)) != NULL) { tdSTSRowIterReset(&iter, row); From 80cfde72ad7199906267a107fa0066cd681ee611 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sat, 7 May 2022 05:34:30 +0000 Subject: [PATCH 31/31] fix: drop table if exists --- source/dnode/vnode/src/meta/metaTable.c | 2 +- source/libs/tdb/inc/tdb.h | 1 + source/libs/tdb/src/db/tdbBtree.c | 8 ++++++++ source/libs/tdb/src/db/tdbDb.c | 2 ++ source/libs/tdb/src/inc/tdbInt.h | 1 + 5 files changed, 13 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index ffe176f85e..a68e176fa9 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -201,7 +201,7 @@ int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq) { // search & delete the name idx tdbDbcOpen(pMeta->pNameIdx, &pNameIdxc, &pMeta->txn); ret = tdbDbcMoveTo(pNameIdxc, pReq->name, strlen(pReq->name) + 1, &c); - if (ret < 0 || c) { + if (ret < 0 || !tdbDbcIsValid(pNameIdxc) || c) { tdbDbcClose(pNameIdxc); terrno = TSDB_CODE_VND_TABLE_NOT_EXIST; return -1; diff --git a/source/libs/tdb/inc/tdb.h b/source/libs/tdb/inc/tdb.h index 18f8ddec5c..90b07fb6ae 100644 --- a/source/libs/tdb/inc/tdb.h +++ b/source/libs/tdb/inc/tdb.h @@ -49,6 +49,7 @@ int tdbDbPGet(TDB *pDb, const void *pKey, int kLen, void **ppKey, int *pkLen, vo // TDBC int tdbDbcOpen(TDB *pDb, TDBC **ppDbc, TXN *pTxn); int tdbDbcClose(TDBC *pDbc); +int tdbDbcIsValid(TDBC *pDbc); int tdbDbcMoveTo(TDBC *pDbc, const void *pKey, int kLen, int *c); int tdbDbcMoveToFirst(TDBC *pDbc); int tdbDbcMoveToLast(TDBC *pDbc); diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 0d13ec4d2b..43822cd311 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -1678,6 +1678,14 @@ int tdbBtcClose(SBTC *pBtc) { return 0; } + +int tdbBtcIsValid(SBTC *pBtc) { + if (pBtc->idx < 0) { + return 0; + } else { + return 1; + } +} // TDB_BTREE_CURSOR // TDB_BTREE_DEBUG ===================== diff --git a/source/libs/tdb/src/db/tdbDb.c b/source/libs/tdb/src/db/tdbDb.c index 553bb2c646..ceaac6dff1 100644 --- a/source/libs/tdb/src/db/tdbDb.c +++ b/source/libs/tdb/src/db/tdbDb.c @@ -141,3 +141,5 @@ int tdbDbcClose(TDBC *pDbc) { return 0; } + +int tdbDbcIsValid(TDBC *pDbc) { return tdbBtcIsValid(&pDbc->btc); } \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdbInt.h b/source/libs/tdb/src/inc/tdbInt.h index 76dacf7b84..ee431ac638 100644 --- a/source/libs/tdb/src/inc/tdbInt.h +++ b/source/libs/tdb/src/inc/tdbInt.h @@ -136,6 +136,7 @@ int tdbBtreePGet(SBTree *pBt, const void *pKey, int kLen, void **ppKey, int *pkL // SBTC int tdbBtcOpen(SBTC *pBtc, SBTree *pBt, TXN *pTxn); int tdbBtcClose(SBTC *pBtc); +int tdbBtcIsValid(SBTC *pBtc); int tdbBtcMoveTo(SBTC *pBtc, const void *pKey, int kLen, int *pCRst); int tdbBtcMoveToFirst(SBTC *pBtc); int tdbBtcMoveToLast(SBTC *pBtc);