From 9eff7ff43a2e9ca0cf9a91b0c380944e2abb404b Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 25 Apr 2022 15:54:34 +0800 Subject: [PATCH 01/83] refactor: schemaless function --- include/util/taoserror.h | 4 + source/client/inc/tscParseLine.h | 119 + source/client/schemaless/CMakeLists.txt | 17 + .../client/schemaless/src/tscParseOpenTSDB.c | 1113 ++++++ source/client/src/tscParseLineProtocol.c | 3134 +++++++++++++++++ source/libs/CMakeLists.txt | 2 +- .../libs/scalar/test/scalar/scalarTests.cpp | 6 +- source/util/src/terror.c | 5 + 8 files changed, 4396 insertions(+), 4 deletions(-) create mode 100644 source/client/inc/tscParseLine.h create mode 100644 source/client/schemaless/CMakeLists.txt create mode 100644 source/client/schemaless/src/tscParseOpenTSDB.c create mode 100644 source/client/src/tscParseLineProtocol.c diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 215e83c7d4..6147b3495a 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -625,6 +625,10 @@ int32_t* taosGetErrno(); #define TSDB_CODE_FUNC_FUNTION_PARA_TYPE TAOS_DEF_ERROR_CODE(0, 0x2802) #define TSDB_CODE_FUNC_FUNTION_PARA_VALUE TAOS_DEF_ERROR_CODE(0, 0x2803) +#define TSDB_CODE_SML_INVALID_PROTOCOL_TYPE TAOS_DEF_ERROR_CODE(0, 0x3000) +#define TSDB_CODE_SML_INVALID_PRECISION_TYPE TAOS_DEF_ERROR_CODE(0, 0x3001) +#define TSDB_CODE_SML_INVALID_DATA TAOS_DEF_ERROR_CODE(0, 0x3002) + #ifdef __cplusplus } #endif diff --git a/source/client/inc/tscParseLine.h b/source/client/inc/tscParseLine.h new file mode 100644 index 0000000000..b70280f380 --- /dev/null +++ b/source/client/inc/tscParseLine.h @@ -0,0 +1,119 @@ +/* + * Copyright (c) 2021 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_TSCPARSELINE_H +#define TDENGINE_TSCPARSELINE_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include "thash.h" +#include "clientint.h" + +#define SML_TIMESTAMP_SECOND_DIGITS 10 +#define SML_TIMESTAMP_MILLI_SECOND_DIGITS 13 + +typedef TSDB_SML_PROTOCOL_TYPE SMLProtocolType; + +typedef struct { + const char* key; + int32_t keyLen; + uint8_t type; + int16_t length; + const char* value; + int32_t valueLen; +} TAOS_SML_KV; + +typedef struct { + const char* measure; + const char* tags; + const char* cols; + const char* timestamp; + + int32_t measureLen; + int32_t measureTagsLen; + int32_t tagsLen; + int32_t colsLen; +} TAOS_PARSE_ELEMENTS; + +typedef struct { + char* childTableName; + + SArray* tags; + SArray *cols; +} TAOS_SML_DATA_POINT_TAGS; + +typedef struct SSmlSTableMeta { + char *sTableName; // super table name + uint8_t sTableNameLen; + uint8_t precision; // the number of precision + SHashObj* tagHash; + SHashObj* fieldHash; +} SSmlSTableMeta; + +typedef enum { + SML_TIME_STAMP_NOT_CONFIGURED, + SML_TIME_STAMP_HOURS, + SML_TIME_STAMP_MINUTES, + SML_TIME_STAMP_SECONDS, + SML_TIME_STAMP_MILLI_SECONDS, + SML_TIME_STAMP_MICRO_SECONDS, + SML_TIME_STAMP_NANO_SECONDS, + SML_TIME_STAMP_NOW +} SMLTimeStampType; + +typedef struct { + uint64_t id; + + STscObj* taos; + SCatalog* pCatalog; + + SMLProtocolType protocol; + SMLTimeStampType tsType; + + int32_t affectedRows; + + SHashObj* childTables; + SHashObj* superTables; +} SSmlLinesInfo; + +int tscSmlInsert(TAOS* taos, TAOS_SML_DATA_POINT* points, int numPoint, SSmlLinesInfo* info); +bool checkDuplicateKey(char *key, SHashObj *pHash, SSmlLinesInfo* info); +bool isValidInteger(char *str); +bool isValidFloat(char *str); + +int32_t isValidChildTableName(const char *pTbName, int16_t len, SSmlLinesInfo* info); + +bool convertSmlValueType(TAOS_SML_KV *pVal, char *value, + uint16_t len, SSmlLinesInfo* info, bool isTag); +int32_t convertSmlTimeStamp(TAOS_SML_KV *pVal, char *value, + uint16_t len, SSmlLinesInfo* info); + +void destroySmlDataPoint(TAOS_SML_DATA_POINT* point); + +int taos_insert_lines(TAOS* taos, char* lines[], int numLines, SMLProtocolType protocol, + SMLTimeStampType tsType, int* affectedRows); +int taos_insert_telnet_lines(TAOS* taos, char* lines[], int numLines, SMLProtocolType protocol, + SMLTimeStampType tsType, int* affectedRows); +int taos_insert_json_payload(TAOS* taos, char* payload, SMLProtocolType protocol, + SMLTimeStampType tsType, int* affectedRows); + + +#ifdef __cplusplus +} +#endif + +#endif // TDENGINE_TSCPARSELINE_H diff --git a/source/client/schemaless/CMakeLists.txt b/source/client/schemaless/CMakeLists.txt new file mode 100644 index 0000000000..29be674934 --- /dev/null +++ b/source/client/schemaless/CMakeLists.txt @@ -0,0 +1,17 @@ +aux_source_directory(src SCHEMALESS_SRC) +add_library(schemaless STATIC ${SCHEMALESS_SRC}) + +target_include_directories( + schemaless + PUBLIC "${TD_SOURCE_DIR}/include/libs/schemaless" + PRIVATE "inc" +) + +target_link_libraries( + schemaless + PUBLIC os util common catalog qcom +) + +if(${BUILD_TEST}) + ADD_SUBDIRECTORY(test) +endif(${BUILD_TEST}) diff --git a/source/client/schemaless/src/tscParseOpenTSDB.c b/source/client/schemaless/src/tscParseOpenTSDB.c new file mode 100644 index 0000000000..5bb9b09cf1 --- /dev/null +++ b/source/client/schemaless/src/tscParseOpenTSDB.c @@ -0,0 +1,1113 @@ +//#include +//#include +//#include +//#include +// +//#include "cJSON.h" +//#include "hash.h" +//#include "taos.h" +// +//#include "tscUtil.h" +//#include "tsclient.h" +//#include "tscLog.h" +// +//#include "tscParseLine.h" +// +//#define OTD_MAX_FIELDS_NUM 2 +//#define OTD_JSON_SUB_FIELDS_NUM 2 +//#define OTD_JSON_FIELDS_NUM 4 +// +//#define OTD_TIMESTAMP_COLUMN_NAME "ts" +//#define OTD_METRIC_VALUE_COLUMN_NAME "value" +// +///* telnet style API parser */ +//static uint64_t HandleId = 0; +// +//static uint64_t genUID() { +// uint64_t id; +// +// do { +// id = atomic_add_fetch_64(&HandleId, 1); +// } while (id == 0); +// +// return id; +//} +// +//static int32_t parseTelnetMetric(TAOS_SML_DATA_POINT *pSml, const char **index, SSmlLinesInfo* info) { +// const char *cur = *index; +// uint16_t len = 0; +// +// pSml->stableName = tcalloc(TSDB_TABLE_NAME_LEN + TS_BACKQUOTE_CHAR_SIZE, 1); +// if (pSml->stableName == NULL) { +// return TSDB_CODE_TSC_OUT_OF_MEMORY; +// } +// /* +// if (isdigit(*cur)) { +// tscError("OTD:0x%"PRIx64" Metric cannot start with digit", info->id); +// tfree(pSml->stableName); +// return TSDB_CODE_TSC_LINE_SYNTAX_ERROR; +// } +// */ +// +// while (*cur != '\0') { +// if (len > TSDB_TABLE_NAME_LEN - 1) { +// tscError("OTD:0x%"PRIx64" Metric cannot exceeds %d characters", info->id, TSDB_TABLE_NAME_LEN - 1); +// tfree(pSml->stableName); +// return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; +// } +// +// if (*cur == ' ') { +// if (*(cur + 1) != ' ') { +// break; +// } else { +// cur++; +// continue; +// } +// } +// +// pSml->stableName[len] = *cur; +// +// cur++; +// len++; +// } +// if (len == 0 || *cur == '\0') { +// tfree(pSml->stableName); +// return TSDB_CODE_TSC_LINE_SYNTAX_ERROR; +// } +// +// addEscapeCharToString(pSml->stableName, len); +// *index = cur + 1; +// tscDebug("OTD:0x%"PRIx64" Stable name in metric:%s|len:%d", info->id, pSml->stableName, len); +// +// return TSDB_CODE_SUCCESS; +//} +// +//static int32_t parseTelnetTimeStamp(TAOS_SML_KV **pTS, int *num_kvs, const char **index, SSmlLinesInfo* info) { +// //Timestamp must be the first KV to parse +// assert(*num_kvs == 0); +// +// const char *start, *cur; +// int32_t ret = TSDB_CODE_SUCCESS; +// int len = 0; +// char key[] = OTD_TIMESTAMP_COLUMN_NAME; +// char *value = NULL; +// +// start = cur = *index; +// //allocate fields for timestamp and value +// *pTS = tcalloc(OTD_MAX_FIELDS_NUM, sizeof(TAOS_SML_KV)); +// +// while(*cur != '\0') { +// if (*cur == ' ') { +// if (*(cur + 1) != ' ') { +// break; +// } else { +// cur++; +// continue; +// } +// } +// cur++; +// len++; +// } +// +// if (len > 0 && *cur != '\0') { +// value = tcalloc(len + 1, 1); +// memcpy(value, start, len); +// } else { +// tfree(*pTS); +// return TSDB_CODE_TSC_LINE_SYNTAX_ERROR; +// } +// +// ret = convertSmlTimeStamp(*pTS, value, len, info); +// if (ret) { +// tfree(value); +// tfree(*pTS); +// return ret; +// } +// tfree(value); +// +// (*pTS)->key = tcalloc(sizeof(key) + TS_BACKQUOTE_CHAR_SIZE, 1); +// memcpy((*pTS)->key, key, sizeof(key)); +// addEscapeCharToString((*pTS)->key, (int32_t)strlen(key)); +// +// *num_kvs += 1; +// *index = cur + 1; +// +// return ret; +//} +// +//static int32_t parseTelnetMetricValue(TAOS_SML_KV **pKVs, int *num_kvs, const char **index, SSmlLinesInfo* info) { +// //skip timestamp +// TAOS_SML_KV *pVal = *pKVs + 1; +// const char *start, *cur; +// int32_t ret = TSDB_CODE_SUCCESS; +// int len = 0; +// bool searchQuote = false; +// char key[] = OTD_METRIC_VALUE_COLUMN_NAME; +// char *value = NULL; +// +// start = cur = *index; +// +// //if metric value is string +// if (*cur == '"') { +// searchQuote = true; +// cur += 1; +// len += 1; +// } else if (*cur == 'L' && *(cur + 1) == '"') { +// searchQuote = true; +// cur += 2; +// len += 2; +// } +// +// while(*cur != '\0') { +// if (*cur == ' ') { +// if (searchQuote == true) { +// if (*(cur - 1) == '"' && len != 1 && len != 2) { +// searchQuote = false; +// } else { +// cur++; +// len++; +// continue; +// } +// } +// +// if (*(cur + 1) != ' ') { +// break; +// } else { +// cur++; +// continue; +// } +// } +// cur++; +// len++; +// } +// +// if (len > 0 && *cur != '\0') { +// value = tcalloc(len + 1, 1); +// memcpy(value, start, len); +// } else { +// return TSDB_CODE_TSC_LINE_SYNTAX_ERROR; +// } +// +// if (!convertSmlValueType(pVal, value, len, info, false)) { +// tscError("OTD:0x%"PRIx64" Failed to convert metric value string(%s) to any type", +// info->id, value); +// tfree(value); +// return TSDB_CODE_TSC_INVALID_VALUE; +// } +// tfree(value); +// +// pVal->key = tcalloc(sizeof(key) + TS_BACKQUOTE_CHAR_SIZE, 1); +// memcpy(pVal->key, key, sizeof(key)); +// addEscapeCharToString(pVal->key, (int32_t)strlen(pVal->key)); +// *num_kvs += 1; +// +// *index = cur + 1; +// return ret; +//} +// +//static int32_t parseTelnetTagKey(TAOS_SML_KV *pKV, const char **index, SHashObj *pHash, SSmlLinesInfo* info) { +// const char *cur = *index; +// char key[TSDB_COL_NAME_LEN]; +// uint16_t len = 0; +// +// //key field cannot start with digit +// //if (isdigit(*cur)) { +// // tscError("OTD:0x%"PRIx64" Tag key cannot start with digit", info->id); +// // return TSDB_CODE_TSC_LINE_SYNTAX_ERROR; +// //} +// while (*cur != '\0') { +// if (len > TSDB_COL_NAME_LEN - 1) { +// tscError("OTD:0x%"PRIx64" Tag key cannot exceeds %d characters", info->id, TSDB_COL_NAME_LEN - 1); +// return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; +// } +// if (*cur == ' ') { +// return TSDB_CODE_TSC_LINE_SYNTAX_ERROR; +// } +// if (*cur == '=') { +// break; +// } +// +// key[len] = *cur; +// cur++; +// len++; +// } +// if (len == 0 || *cur == '\0') { +// return TSDB_CODE_TSC_LINE_SYNTAX_ERROR; +// } +// key[len] = '\0'; +// +// if (checkDuplicateKey(key, pHash, info)) { +// return TSDB_CODE_TSC_DUP_TAG_NAMES; +// } +// +// pKV->key = tcalloc(len + TS_BACKQUOTE_CHAR_SIZE + 1, 1); +// memcpy(pKV->key, key, len + 1); +// addEscapeCharToString(pKV->key, len); +// //tscDebug("OTD:0x%"PRIx64" Key:%s|len:%d", info->id, pKV->key, len); +// *index = cur + 1; +// return TSDB_CODE_SUCCESS; +//} +// +// +//static int32_t parseTelnetTagValue(TAOS_SML_KV *pKV, const char **index, +// bool *is_last_kv, SSmlLinesInfo* info) { +// const char *start, *cur; +// char *value = NULL; +// uint16_t len = 0; +// start = cur = *index; +// +// while (1) { +// // whitespace or '\0' identifies a value +// if (*cur == ' ' || *cur == '\0') { +// // '\0' indicates end of value +// *is_last_kv = (*cur == '\0') ? true : false; +// if (*cur == ' ' && *(cur + 1) == ' ') { +// cur++; +// continue; +// } else { +// break; +// } +// } +// cur++; +// len++; +// } +// +// if (len == 0) { +// tfree(pKV->key); +// return TSDB_CODE_TSC_LINE_SYNTAX_ERROR; +// } +// +// value = tcalloc(len + 1, 1); +// memcpy(value, start, len); +// value[len] = '\0'; +// if (!convertSmlValueType(pKV, value, len, info, true)) { +// tscError("OTD:0x%"PRIx64" Failed to convert sml value string(%s) to any type", +// info->id, value); +// //free previous alocated key field +// tfree(pKV->key); +// tfree(value); +// return TSDB_CODE_TSC_INVALID_VALUE; +// } +// tfree(value); +// +// *index = (*cur == '\0') ? cur : cur + 1; +// return TSDB_CODE_SUCCESS; +//} +// +//static int32_t parseTelnetTagKvs(TAOS_SML_KV **pKVs, int *num_kvs, +// const char **index, char **childTableName, +// SHashObj *pHash, SSmlLinesInfo* info) { +// const char *cur = *index; +// int32_t ret = TSDB_CODE_SUCCESS; +// TAOS_SML_KV *pkv; +// bool is_last_kv = false; +// +// int32_t capacity = 4; +// *pKVs = tcalloc(capacity, sizeof(TAOS_SML_KV)); +// pkv = *pKVs; +// +// size_t childTableNameLen = strlen(tsSmlChildTableName); +// char childTbName[TSDB_TABLE_NAME_LEN + TS_BACKQUOTE_CHAR_SIZE] = {0}; +// if (childTableNameLen != 0) { +// memcpy(childTbName, tsSmlChildTableName, childTableNameLen); +// addEscapeCharToString(childTbName, (int32_t)(childTableNameLen)); +// } +// while (*cur != '\0') { +// ret = parseTelnetTagKey(pkv, &cur, pHash, info); +// if (ret) { +// tscError("OTD:0x%"PRIx64" Unable to parse key", info->id); +// return ret; +// } +// ret = parseTelnetTagValue(pkv, &cur, &is_last_kv, info); +// if (ret) { +// tscError("OTD:0x%"PRIx64" Unable to parse value", info->id); +// return ret; +// } +// if (childTableNameLen != 0 && strcasecmp(pkv->key, childTbName) == 0) { +// *childTableName = tcalloc(pkv->length + TS_BACKQUOTE_CHAR_SIZE + 1, 1); +// memcpy(*childTableName, pkv->value, pkv->length); +// (*childTableName)[pkv->length] = '\0'; +// addEscapeCharToString(*childTableName, pkv->length); +// tfree(pkv->key); +// tfree(pkv->value); +// } else { +// *num_kvs += 1; +// } +// +// if (is_last_kv) { +// break; +// } +// +// //reallocate addtional memory for more kvs +// if ((*num_kvs + 1) > capacity) { +// TAOS_SML_KV *more_kvs = NULL; +// capacity *= 3; capacity /= 2; +// more_kvs = realloc(*pKVs, capacity * sizeof(TAOS_SML_KV)); +// if (!more_kvs) { +// return TSDB_CODE_TSC_OUT_OF_MEMORY; +// } +// *pKVs = more_kvs; +// } +// +// //move pKV points to next TAOS_SML_KV block +// pkv = *pKVs + *num_kvs; +// } +// +// return ret; +//} +// +//static int32_t tscParseTelnetLine(const char* line, TAOS_SML_DATA_POINT* smlData, SSmlLinesInfo* info) { +// const char* index = line; +// int32_t ret = TSDB_CODE_SUCCESS; +// +// //Parse metric +// ret = parseTelnetMetric(smlData, &index, info); +// if (ret) { +// tscError("OTD:0x%"PRIx64" Unable to parse metric", info->id); +// return ret; +// } +// tscDebug("OTD:0x%"PRIx64" Parse metric finished", info->id); +// +// //Parse timestamp +// ret = parseTelnetTimeStamp(&smlData->fields, &smlData->fieldNum, &index, info); +// if (ret) { +// tscError("OTD:0x%"PRIx64" Unable to parse timestamp", info->id); +// return ret; +// } +// tscDebug("OTD:0x%"PRIx64" Parse timestamp finished", info->id); +// +// //Parse value +// ret = parseTelnetMetricValue(&smlData->fields, &smlData->fieldNum, &index, info); +// if (ret) { +// tscError("OTD:0x%"PRIx64" Unable to parse metric value", info->id); +// return ret; +// } +// tscDebug("OTD:0x%"PRIx64" Parse metric value finished", info->id); +// +// //Parse tagKVs +// SHashObj *keyHashTable = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); +// ret = parseTelnetTagKvs(&smlData->tags, &smlData->tagNum, &index, &smlData->childTableName, keyHashTable, info); +// if (ret) { +// tscError("OTD:0x%"PRIx64" Unable to parse tags", info->id); +// taosHashCleanup(keyHashTable); +// return ret; +// } +// tscDebug("OTD:0x%"PRIx64" Parse tags finished", info->id); +// taosHashCleanup(keyHashTable); +// +// +// return TSDB_CODE_SUCCESS; +//} +// +//static int32_t tscParseTelnetLines(char* lines[], int numLines, SArray* points, SArray* failedLines, SSmlLinesInfo* info) { +// for (int32_t i = 0; i < numLines; ++i) { +// TAOS_SML_DATA_POINT point = {0}; +// int32_t code = tscParseTelnetLine(lines[i], &point, info); +// if (code != TSDB_CODE_SUCCESS) { +// tscError("OTD:0x%"PRIx64" data point line parse failed. line %d : %s", info->id, i, lines[i]); +// destroySmlDataPoint(&point); +// return code; +// } else { +// tscDebug("OTD:0x%"PRIx64" data point line parse success. line %d", info->id, i); +// } +// +// taosArrayPush(points, &point); +// } +// return TSDB_CODE_SUCCESS; +//} +// +//int taos_insert_telnet_lines(TAOS* taos, char* lines[], int numLines, SMLProtocolType protocol, SMLTimeStampType tsType, int* affectedRows) { +// int32_t code = 0; +// +// SSmlLinesInfo* info = tcalloc(1, sizeof(SSmlLinesInfo)); +// info->id = genUID(); +// info->tsType = tsType; +// info->protocol = protocol; +// +// if (numLines <= 0 || numLines > 65536) { +// tscError("OTD:0x%"PRIx64" taos_insert_telnet_lines numLines should be between 1 and 65536. numLines: %d", info->id, numLines); +// tfree(info); +// code = TSDB_CODE_TSC_APP_ERROR; +// return code; +// } +// +// for (int i = 0; i < numLines; ++i) { +// if (lines[i] == NULL) { +// tscError("OTD:0x%"PRIx64" taos_insert_telnet_lines line %d is NULL", info->id, i); +// tfree(info); +// code = TSDB_CODE_TSC_APP_ERROR; +// return code; +// } +// } +// +// SArray* lpPoints = taosArrayInit(numLines, sizeof(TAOS_SML_DATA_POINT)); +// if (lpPoints == NULL) { +// tscError("OTD:0x%"PRIx64" taos_insert_telnet_lines failed to allocate memory", info->id); +// tfree(info); +// return TSDB_CODE_TSC_OUT_OF_MEMORY; +// } +// +// tscDebug("OTD:0x%"PRIx64" taos_insert_telnet_lines begin inserting %d lines, first line: %s", info->id, numLines, lines[0]); +// code = tscParseTelnetLines(lines, numLines, lpPoints, NULL, info); +// size_t numPoints = taosArrayGetSize(lpPoints); +// +// if (code != 0) { +// goto cleanup; +// } +// +// TAOS_SML_DATA_POINT* points = TARRAY_GET_START(lpPoints); +// code = tscSmlInsert(taos, points, (int)numPoints, info); +// if (code != 0) { +// tscError("OTD:0x%"PRIx64" taos_insert_telnet_lines error: %s", info->id, tstrerror((code))); +// } +// if (affectedRows != NULL) { +// *affectedRows = info->affectedRows; +// } +// +//cleanup: +// tscDebug("OTD:0x%"PRIx64" taos_insert_telnet_lines finish inserting %d lines. code: %d", info->id, numLines, code); +// points = TARRAY_GET_START(lpPoints); +// numPoints = taosArrayGetSize(lpPoints); +// for (int i = 0; i < numPoints; ++i) { +// destroySmlDataPoint(points+i); +// } +// +// taosArrayDestroy(&lpPoints); +// +// tfree(info); +// return code; +//} +// +//int taos_telnet_insert(TAOS* taos, TAOS_SML_DATA_POINT* points, int numPoint) { +// SSmlLinesInfo* info = tcalloc(1, sizeof(SSmlLinesInfo)); +// info->id = genUID(); +// int code = tscSmlInsert(taos, points, numPoint, info); +// tfree(info); +// return code; +//} +// +// +///* telnet style API parser */ +//static int32_t parseMetricFromJSON(cJSON *root, TAOS_SML_DATA_POINT* pSml, SSmlLinesInfo* info) { +// cJSON *metric = cJSON_GetObjectItem(root, "metric"); +// if (!cJSON_IsString(metric)) { +// return TSDB_CODE_TSC_INVALID_JSON; +// } +// +// size_t stableLen = strlen(metric->valuestring); +// if (stableLen > TSDB_TABLE_NAME_LEN - 1) { +// tscError("OTD:0x%"PRIx64" Metric cannot exceeds %d characters in JSON", info->id, TSDB_TABLE_NAME_LEN - 1); +// return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; +// } +// +// pSml->stableName = tcalloc(stableLen + TS_BACKQUOTE_CHAR_SIZE + 1, sizeof(char)); +// if (pSml->stableName == NULL){ +// return TSDB_CODE_TSC_OUT_OF_MEMORY; +// } +// +// /* +// if (isdigit(metric->valuestring[0])) { +// tscError("OTD:0x%"PRIx64" Metric cannot start with digit in JSON", info->id); +// tfree(pSml->stableName); +// return TSDB_CODE_TSC_INVALID_JSON; +// } +// */ +// +// tstrncpy(pSml->stableName, metric->valuestring, stableLen + 1); +// addEscapeCharToString(pSml->stableName, (int32_t)stableLen); +// +// return TSDB_CODE_SUCCESS; +// +//} +// +//static int32_t parseTimestampFromJSONObj(cJSON *root, int64_t *tsVal, SSmlLinesInfo* info) { +// int32_t size = cJSON_GetArraySize(root); +// if (size != OTD_JSON_SUB_FIELDS_NUM) { +// return TSDB_CODE_TSC_INVALID_JSON; +// } +// +// cJSON *value = cJSON_GetObjectItem(root, "value"); +// if (!cJSON_IsNumber(value)) { +// return TSDB_CODE_TSC_INVALID_JSON; +// } +// +// cJSON *type = cJSON_GetObjectItem(root, "type"); +// if (!cJSON_IsString(type)) { +// return TSDB_CODE_TSC_INVALID_JSON; +// } +// +// *tsVal = strtoll(value->numberstring, NULL, 10); +// //if timestamp value is 0 use current system time +// if (*tsVal == 0) { +// *tsVal = taosGetTimestampNs(); +// return TSDB_CODE_SUCCESS; +// } +// +// size_t typeLen = strlen(type->valuestring); +// if (typeLen == 1 && type->valuestring[0] == 's') { +// //seconds +// *tsVal = (int64_t)(*tsVal * 1e9); +// } else if (typeLen == 2 && type->valuestring[1] == 's') { +// switch (type->valuestring[0]) { +// case 'm': +// //milliseconds +// *tsVal = convertTimePrecision(*tsVal, TSDB_TIME_PRECISION_MILLI, TSDB_TIME_PRECISION_NANO); +// break; +// case 'u': +// //microseconds +// *tsVal = convertTimePrecision(*tsVal, TSDB_TIME_PRECISION_MICRO, TSDB_TIME_PRECISION_NANO); +// break; +// case 'n': +// //nanoseconds +// *tsVal = *tsVal * 1; +// break; +// default: +// return TSDB_CODE_TSC_INVALID_JSON; +// } +// } else { +// return TSDB_CODE_TSC_INVALID_JSON; +// } +// +// return TSDB_CODE_SUCCESS; +//} +// +//static int32_t parseTimestampFromJSON(cJSON *root, TAOS_SML_KV **pTS, int *num_kvs, SSmlLinesInfo* info) { +// //Timestamp must be the first KV to parse +// assert(*num_kvs == 0); +// int64_t tsVal; +// char key[] = OTD_TIMESTAMP_COLUMN_NAME; +// +// cJSON *timestamp = cJSON_GetObjectItem(root, "timestamp"); +// if (cJSON_IsNumber(timestamp)) { +// //timestamp value 0 indicates current system time +// if (timestamp->valueint == 0) { +// tsVal = taosGetTimestampNs(); +// } else { +// tsVal = strtoll(timestamp->numberstring, NULL, 10); +// size_t tsLen = strlen(timestamp->numberstring); +// if (tsLen == SML_TIMESTAMP_SECOND_DIGITS) { +// tsVal = (int64_t)(tsVal * 1e9); +// } else if (tsLen == SML_TIMESTAMP_MILLI_SECOND_DIGITS) { +// tsVal = convertTimePrecision(tsVal, TSDB_TIME_PRECISION_MILLI, TSDB_TIME_PRECISION_NANO); +// } else { +// return TSDB_CODE_TSC_INVALID_TIME_STAMP; +// } +// } +// } else if (cJSON_IsObject(timestamp)) { +// int32_t ret = parseTimestampFromJSONObj(timestamp, &tsVal, info); +// if (ret != TSDB_CODE_SUCCESS) { +// tscError("OTD:0x%"PRIx64" Failed to parse timestamp from JSON Obj", info->id); +// return ret; +// } +// } else { +// return TSDB_CODE_TSC_INVALID_JSON; +// } +// +// //allocate fields for timestamp and value +// *pTS = tcalloc(OTD_MAX_FIELDS_NUM, sizeof(TAOS_SML_KV)); +// +// +// (*pTS)->key = tcalloc(sizeof(key), 1); +// memcpy((*pTS)->key, key, sizeof(key)); +// +// (*pTS)->type = TSDB_DATA_TYPE_TIMESTAMP; +// (*pTS)->length = (int16_t)tDataTypes[(*pTS)->type].bytes; +// (*pTS)->value = tcalloc((*pTS)->length, 1); +// memcpy((*pTS)->value, &tsVal, (*pTS)->length); +// +// *num_kvs += 1; +// return TSDB_CODE_SUCCESS; +// +//} +// +//static int32_t convertJSONBool(TAOS_SML_KV *pVal, char* typeStr, int64_t valueInt, SSmlLinesInfo* info) { +// if (strcasecmp(typeStr, "bool") != 0) { +// tscError("OTD:0x%"PRIx64" invalid type(%s) for JSON Bool", info->id, typeStr); +// return TSDB_CODE_TSC_INVALID_JSON_TYPE; +// } +// pVal->type = TSDB_DATA_TYPE_BOOL; +// pVal->length = (int16_t)tDataTypes[pVal->type].bytes; +// pVal->value = tcalloc(pVal->length, 1); +// *(bool *)(pVal->value) = valueInt ? true : false; +// +// return TSDB_CODE_SUCCESS; +//} +// +//static int32_t convertJSONNumber(TAOS_SML_KV *pVal, char* typeStr, cJSON *value, SSmlLinesInfo* info) { +// //tinyint +// if (strcasecmp(typeStr, "i8") == 0 || +// strcasecmp(typeStr, "tinyint") == 0) { +// if (!IS_VALID_TINYINT(value->valueint)) { +// tscError("OTD:0x%"PRIx64" JSON value(%"PRId64") cannot fit in type(tinyint)", info->id, value->valueint); +// return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; +// } +// pVal->type = TSDB_DATA_TYPE_TINYINT; +// pVal->length = (int16_t)tDataTypes[pVal->type].bytes; +// pVal->value = tcalloc(pVal->length, 1); +// *(int8_t *)(pVal->value) = (int8_t)(value->valueint); +// return TSDB_CODE_SUCCESS; +// } +// //smallint +// if (strcasecmp(typeStr, "i16") == 0 || +// strcasecmp(typeStr, "smallint") == 0) { +// if (!IS_VALID_SMALLINT(value->valueint)) { +// tscError("OTD:0x%"PRIx64" JSON value(%"PRId64") cannot fit in type(smallint)", info->id, value->valueint); +// return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; +// } +// pVal->type = TSDB_DATA_TYPE_SMALLINT; +// pVal->length = (int16_t)tDataTypes[pVal->type].bytes; +// pVal->value = tcalloc(pVal->length, 1); +// *(int16_t *)(pVal->value) = (int16_t)(value->valueint); +// return TSDB_CODE_SUCCESS; +// } +// //int +// if (strcasecmp(typeStr, "i32") == 0 || +// strcasecmp(typeStr, "int") == 0) { +// if (!IS_VALID_INT(value->valueint)) { +// tscError("OTD:0x%"PRIx64" JSON value(%"PRId64") cannot fit in type(int)", info->id, value->valueint); +// return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; +// } +// pVal->type = TSDB_DATA_TYPE_INT; +// pVal->length = (int16_t)tDataTypes[pVal->type].bytes; +// pVal->value = tcalloc(pVal->length, 1); +// *(int32_t *)(pVal->value) = (int32_t)(value->valueint); +// return TSDB_CODE_SUCCESS; +// } +// //bigint +// if (strcasecmp(typeStr, "i64") == 0 || +// strcasecmp(typeStr, "bigint") == 0) { +// pVal->type = TSDB_DATA_TYPE_BIGINT; +// pVal->length = (int16_t)tDataTypes[pVal->type].bytes; +// pVal->value = tcalloc(pVal->length, 1); +// /* cJSON conversion of legit BIGINT may overflow, +// * use original string to do the conversion. +// */ +// errno = 0; +// int64_t val = (int64_t)strtoll(value->numberstring, NULL, 10); +// if (errno == ERANGE || !IS_VALID_BIGINT(val)) { +// tscError("OTD:0x%"PRIx64" JSON value(%s) cannot fit in type(bigint)", info->id, value->numberstring); +// return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; +// } +// *(int64_t *)(pVal->value) = val; +// return TSDB_CODE_SUCCESS; +// } +// //float +// if (strcasecmp(typeStr, "f32") == 0 || +// strcasecmp(typeStr, "float") == 0) { +// if (!IS_VALID_FLOAT(value->valuedouble)) { +// tscError("OTD:0x%"PRIx64" JSON value(%f) cannot fit in type(float)", info->id, value->valuedouble); +// return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; +// } +// pVal->type = TSDB_DATA_TYPE_FLOAT; +// pVal->length = (int16_t)tDataTypes[pVal->type].bytes; +// pVal->value = tcalloc(pVal->length, 1); +// *(float *)(pVal->value) = (float)(value->valuedouble); +// return TSDB_CODE_SUCCESS; +// } +// //double +// if (strcasecmp(typeStr, "f64") == 0 || +// strcasecmp(typeStr, "double") == 0) { +// if (!IS_VALID_DOUBLE(value->valuedouble)) { +// tscError("OTD:0x%"PRIx64" JSON value(%f) cannot fit in type(double)", info->id, value->valuedouble); +// return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; +// } +// pVal->type = TSDB_DATA_TYPE_DOUBLE; +// pVal->length = (int16_t)tDataTypes[pVal->type].bytes; +// pVal->value = tcalloc(pVal->length, 1); +// *(double *)(pVal->value) = (double)(value->valuedouble); +// return TSDB_CODE_SUCCESS; +// } +// +// //if reach here means type is unsupported +// tscError("OTD:0x%"PRIx64" invalid type(%s) for JSON Number", info->id, typeStr); +// return TSDB_CODE_TSC_INVALID_JSON_TYPE; +//} +// +//static int32_t convertJSONString(TAOS_SML_KV *pVal, char* typeStr, cJSON *value, SSmlLinesInfo* info) { +// if (strcasecmp(typeStr, "binary") == 0) { +// pVal->type = TSDB_DATA_TYPE_BINARY; +// } else if (strcasecmp(typeStr, "nchar") == 0) { +// pVal->type = TSDB_DATA_TYPE_NCHAR; +// } else { +// tscError("OTD:0x%"PRIx64" invalid type(%s) for JSON String", info->id, typeStr); +// return TSDB_CODE_TSC_INVALID_JSON_TYPE; +// } +// pVal->length = (int16_t)strlen(value->valuestring); +// pVal->value = tcalloc(pVal->length + 1, 1); +// memcpy(pVal->value, value->valuestring, pVal->length); +// return TSDB_CODE_SUCCESS; +//} +// +//static int32_t parseValueFromJSONObj(cJSON *root, TAOS_SML_KV *pVal, SSmlLinesInfo* info) { +// int32_t ret = TSDB_CODE_SUCCESS; +// int32_t size = cJSON_GetArraySize(root); +// +// if (size != OTD_JSON_SUB_FIELDS_NUM) { +// return TSDB_CODE_TSC_INVALID_JSON; +// } +// +// cJSON *value = cJSON_GetObjectItem(root, "value"); +// if (value == NULL) { +// return TSDB_CODE_TSC_INVALID_JSON; +// } +// +// cJSON *type = cJSON_GetObjectItem(root, "type"); +// if (!cJSON_IsString(type)) { +// return TSDB_CODE_TSC_INVALID_JSON; +// } +// +// switch (value->type) { +// case cJSON_True: +// case cJSON_False: { +// ret = convertJSONBool(pVal, type->valuestring, value->valueint, info); +// if (ret != TSDB_CODE_SUCCESS) { +// return ret; +// } +// break; +// } +// case cJSON_Number: { +// ret = convertJSONNumber(pVal, type->valuestring, value, info); +// if (ret != TSDB_CODE_SUCCESS) { +// return ret; +// } +// break; +// } +// case cJSON_String: { +// ret = convertJSONString(pVal, type->valuestring, value, info); +// if (ret != TSDB_CODE_SUCCESS) { +// return ret; +// } +// break; +// } +// default: +// return TSDB_CODE_TSC_INVALID_JSON_TYPE; +// } +// +// return TSDB_CODE_SUCCESS; +//} +// +//static int32_t parseValueFromJSON(cJSON *root, TAOS_SML_KV *pVal, SSmlLinesInfo* info) { +// int type = root->type; +// +// switch (type) { +// case cJSON_True: +// case cJSON_False: { +// pVal->type = TSDB_DATA_TYPE_BOOL; +// pVal->length = (int16_t)tDataTypes[pVal->type].bytes; +// pVal->value = tcalloc(pVal->length, 1); +// *(bool *)(pVal->value) = root->valueint ? true : false; +// break; +// } +// case cJSON_Number: { +// //convert default JSON Number type to BIGINT/DOUBLE +// //if (isValidInteger(root->numberstring)) { +// // pVal->type = TSDB_DATA_TYPE_BIGINT; +// // pVal->length = (int16_t)tDataTypes[pVal->type].bytes; +// // pVal->value = tcalloc(pVal->length, 1); +// // /* cJSON conversion of legit BIGINT may overflow, +// // * use original string to do the conversion. +// // */ +// // errno = 0; +// // int64_t val = (int64_t)strtoll(root->numberstring, NULL, 10); +// // if (errno == ERANGE || !IS_VALID_BIGINT(val)) { +// // tscError("OTD:0x%"PRIx64" JSON value(%s) cannot fit in type(bigint)", info->id, root->numberstring); +// // return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; +// // } +// // *(int64_t *)(pVal->value) = val; +// //} else if (isValidFloat(root->numberstring)) { +// // pVal->type = TSDB_DATA_TYPE_DOUBLE; +// // pVal->length = (int16_t)tDataTypes[pVal->type].bytes; +// // pVal->value = tcalloc(pVal->length, 1); +// // *(double *)(pVal->value) = (double)(root->valuedouble); +// //} else { +// // return TSDB_CODE_TSC_INVALID_JSON_TYPE; +// //} +// if (isValidInteger(root->numberstring) || isValidFloat(root->numberstring)) { +// pVal->type = TSDB_DATA_TYPE_DOUBLE; +// pVal->length = (int16_t)tDataTypes[pVal->type].bytes; +// pVal->value = tcalloc(pVal->length, 1); +// *(double *)(pVal->value) = (double)(root->valuedouble); +// } +// +// break; +// } +// case cJSON_String: { +// /* set default JSON type to binary/nchar according to +// * user configured parameter tsDefaultJSONStrType +// */ +// if (strcasecmp(tsDefaultJSONStrType, "binary") == 0) { +// pVal->type = TSDB_DATA_TYPE_BINARY; +// } else if (strcasecmp(tsDefaultJSONStrType, "nchar") == 0) { +// pVal->type = TSDB_DATA_TYPE_NCHAR; +// } else { +// tscError("OTD:0x%"PRIx64" Invalid default JSON string type set from config %s", info->id, tsDefaultJSONStrType); +// return TSDB_CODE_TSC_INVALID_JSON_CONFIG; +// } +// //pVal->length = wcslen((wchar_t *)root->valuestring) * TSDB_NCHAR_SIZE; +// pVal->length = (int16_t)strlen(root->valuestring); +// pVal->value = tcalloc(pVal->length + 1, 1); +// memcpy(pVal->value, root->valuestring, pVal->length); +// break; +// } +// case cJSON_Object: { +// int32_t ret = parseValueFromJSONObj(root, pVal, info); +// if (ret != TSDB_CODE_SUCCESS) { +// tscError("OTD:0x%"PRIx64" Failed to parse timestamp from JSON Obj", info->id); +// return ret; +// } +// break; +// } +// default: +// return TSDB_CODE_TSC_INVALID_JSON; +// } +// +// return TSDB_CODE_SUCCESS; +//} +// +//static int32_t parseMetricValueFromJSON(cJSON *root, TAOS_SML_KV **pKVs, int *num_kvs, SSmlLinesInfo* info) { +// //skip timestamp +// TAOS_SML_KV *pVal = *pKVs + 1; +// char key[] = OTD_METRIC_VALUE_COLUMN_NAME; +// +// cJSON *metricVal = cJSON_GetObjectItem(root, "value"); +// if (metricVal == NULL) { +// return TSDB_CODE_TSC_INVALID_JSON; +// } +// +// int32_t ret = parseValueFromJSON(metricVal, pVal, info); +// if (ret != TSDB_CODE_SUCCESS) { +// return ret; +// } +// +// pVal->key = tcalloc(sizeof(key) + TS_BACKQUOTE_CHAR_SIZE, 1); +// memcpy(pVal->key, key, sizeof(key)); +// addEscapeCharToString(pVal->key, (int32_t)strlen(pVal->key)); +// +// *num_kvs += 1; +// return TSDB_CODE_SUCCESS; +// +//} +// +// +//static int32_t parseTagsFromJSON(cJSON *root, TAOS_SML_KV **pKVs, int *num_kvs, char **childTableName, +// SHashObj *pHash, SSmlLinesInfo* info) { +// int32_t ret = TSDB_CODE_SUCCESS; +// +// cJSON *tags = cJSON_GetObjectItem(root, "tags"); +// if (tags == NULL || tags->type != cJSON_Object) { +// return TSDB_CODE_TSC_INVALID_JSON; +// } +// +// //handle child table name +// size_t childTableNameLen = strlen(tsSmlChildTableName); +// char childTbName[TSDB_TABLE_NAME_LEN] = {0}; +// if (childTableNameLen != 0) { +// memcpy(childTbName, tsSmlChildTableName, childTableNameLen); +// cJSON *id = cJSON_GetObjectItem(tags, childTbName); +// if (id != NULL) { +// if (!cJSON_IsString(id)) { +// tscError("OTD:0x%"PRIx64" ID must be JSON string", info->id); +// return TSDB_CODE_TSC_INVALID_JSON; +// } +// size_t idLen = strlen(id->valuestring); +// *childTableName = tcalloc(idLen + TS_BACKQUOTE_CHAR_SIZE + 1, sizeof(char)); +// memcpy(*childTableName, id->valuestring, idLen); +// addEscapeCharToString(*childTableName, (int32_t)idLen); +// +// //check duplicate IDs +// cJSON_DeleteItemFromObject(tags, childTbName); +// id = cJSON_GetObjectItem(tags, childTbName); +// if (id != NULL) { +// return TSDB_CODE_TSC_DUP_TAG_NAMES; +// } +// } +// } +// +// int32_t tagNum = cJSON_GetArraySize(tags); +// //at least one tag pair required +// if (tagNum <= 0) { +// return TSDB_CODE_TSC_INVALID_JSON; +// } +// +// //allocate memory for tags +// *pKVs = tcalloc(tagNum, sizeof(TAOS_SML_KV)); +// TAOS_SML_KV *pkv = *pKVs; +// +// for (int32_t i = 0; i < tagNum; ++i) { +// cJSON *tag = cJSON_GetArrayItem(tags, i); +// if (tag == NULL) { +// return TSDB_CODE_TSC_INVALID_JSON; +// } +// //check duplicate keys +// if (checkDuplicateKey(tag->string, pHash, info)) { +// return TSDB_CODE_TSC_DUP_TAG_NAMES; +// } +// //key +// size_t keyLen = strlen(tag->string); +// if (keyLen > TSDB_COL_NAME_LEN - 1) { +// tscError("OTD:0x%"PRIx64" Tag key cannot exceeds %d characters in JSON", info->id, TSDB_COL_NAME_LEN - 1); +// return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; +// } +// pkv->key = tcalloc(keyLen + TS_BACKQUOTE_CHAR_SIZE + 1, sizeof(char)); +// strncpy(pkv->key, tag->string, keyLen); +// addEscapeCharToString(pkv->key, (int32_t)keyLen); +// //value +// ret = parseValueFromJSON(tag, pkv, info); +// if (ret != TSDB_CODE_SUCCESS) { +// return ret; +// } +// *num_kvs += 1; +// pkv++; +// +// } +// +// return ret; +// +//} +// +//static int32_t tscParseJSONPayload(cJSON *root, TAOS_SML_DATA_POINT* pSml, SSmlLinesInfo* info) { +// int32_t ret = TSDB_CODE_SUCCESS; +// +// if (!cJSON_IsObject(root)) { +// tscError("OTD:0x%"PRIx64" data point needs to be JSON object", info->id); +// return TSDB_CODE_TSC_INVALID_JSON; +// } +// +// int32_t size = cJSON_GetArraySize(root); +// //outmost json fields has to be exactly 4 +// if (size != OTD_JSON_FIELDS_NUM) { +// tscError("OTD:0x%"PRIx64" Invalid number of JSON fields in data point %d", info->id, size); +// return TSDB_CODE_TSC_INVALID_JSON; +// } +// +// //Parse metric +// ret = parseMetricFromJSON(root, pSml, info); +// if (ret != TSDB_CODE_SUCCESS) { +// tscError("OTD:0x%"PRIx64" Unable to parse metric from JSON payload", info->id); +// return ret; +// } +// tscDebug("OTD:0x%"PRIx64" Parse metric from JSON payload finished", info->id); +// +// //Parse timestamp +// ret = parseTimestampFromJSON(root, &pSml->fields, &pSml->fieldNum, info); +// if (ret) { +// tscError("OTD:0x%"PRIx64" Unable to parse timestamp from JSON payload", info->id); +// return ret; +// } +// tscDebug("OTD:0x%"PRIx64" Parse timestamp from JSON payload finished", info->id); +// +// //Parse metric value +// ret = parseMetricValueFromJSON(root, &pSml->fields, &pSml->fieldNum, info); +// if (ret) { +// tscError("OTD:0x%"PRIx64" Unable to parse metric value from JSON payload", info->id); +// return ret; +// } +// tscDebug("OTD:0x%"PRIx64" Parse metric value from JSON payload finished", info->id); +// +// //Parse tags +// SHashObj *keyHashTable = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); +// ret = parseTagsFromJSON(root, &pSml->tags, &pSml->tagNum, &pSml->childTableName, keyHashTable, info); +// if (ret) { +// tscError("OTD:0x%"PRIx64" Unable to parse tags from JSON payload", info->id); +// taosHashCleanup(keyHashTable); +// return ret; +// } +// tscDebug("OTD:0x%"PRIx64" Parse tags from JSON payload finished", info->id); +// taosHashCleanup(keyHashTable); +// +// return TSDB_CODE_SUCCESS; +//} +// +//static int32_t tscParseMultiJSONPayload(char* payload, SArray* points, SSmlLinesInfo* info) { +// int32_t payloadNum, ret; +// ret = TSDB_CODE_SUCCESS; +// +// if (payload == NULL) { +// tscError("OTD:0x%"PRIx64" empty JSON Payload", info->id); +// return TSDB_CODE_TSC_INVALID_JSON; +// } +// +// cJSON *root = cJSON_Parse(payload); +// //multiple data points must be sent in JSON array +// if (cJSON_IsObject(root)) { +// payloadNum = 1; +// } else if (cJSON_IsArray(root)) { +// payloadNum = cJSON_GetArraySize(root); +// } else { +// tscError("OTD:0x%"PRIx64" Invalid JSON Payload", info->id); +// ret = TSDB_CODE_TSC_INVALID_JSON; +// goto PARSE_JSON_OVER; +// } +// +// for (int32_t i = 0; i < payloadNum; ++i) { +// TAOS_SML_DATA_POINT point = {0}; +// cJSON *dataPoint = (payloadNum == 1 && cJSON_IsObject(root)) ? root : cJSON_GetArrayItem(root, i); +// +// ret = tscParseJSONPayload(dataPoint, &point, info); +// if (ret != TSDB_CODE_SUCCESS) { +// tscError("OTD:0x%"PRIx64" JSON data point parse failed", info->id); +// destroySmlDataPoint(&point); +// goto PARSE_JSON_OVER; +// } else { +// tscDebug("OTD:0x%"PRIx64" JSON data point parse success", info->id); +// } +// taosArrayPush(points, &point); +// } +// +//PARSE_JSON_OVER: +// cJSON_Delete(root); +// return ret; +//} +// +//int taos_insert_json_payload(TAOS* taos, char* payload, SMLProtocolType protocol, SMLTimeStampType tsType, int* affectedRows) { +// int32_t code = 0; +// +// SSmlLinesInfo* info = tcalloc(1, sizeof(SSmlLinesInfo)); +// info->id = genUID(); +// info->tsType = tsType; +// info->protocol = protocol; +// +// if (payload == NULL) { +// tscError("OTD:0x%"PRIx64" taos_insert_json_payload payload is NULL", info->id); +// tfree(info); +// code = TSDB_CODE_TSC_APP_ERROR; +// return code; +// } +// +// SArray* lpPoints = taosArrayInit(1, sizeof(TAOS_SML_DATA_POINT)); +// if (lpPoints == NULL) { +// tscError("OTD:0x%"PRIx64" taos_insert_json_payload failed to allocate memory", info->id); +// tfree(info); +// return TSDB_CODE_TSC_OUT_OF_MEMORY; +// } +// +// tscDebug("OTD:0x%"PRIx64" taos_insert_telnet_lines begin inserting %d points", info->id, 1); +// code = tscParseMultiJSONPayload(payload, lpPoints, info); +// size_t numPoints = taosArrayGetSize(lpPoints); +// +// if (code != 0) { +// goto cleanup; +// } +// +// TAOS_SML_DATA_POINT* points = TARRAY_GET_START(lpPoints); +// code = tscSmlInsert(taos, points, (int)numPoints, info); +// if (code != 0) { +// tscError("OTD:0x%"PRIx64" taos_insert_json_payload error: %s", info->id, tstrerror((code))); +// } +// if (affectedRows != NULL) { +// *affectedRows = info->affectedRows; +// } +// +//cleanup: +// tscDebug("OTD:0x%"PRIx64" taos_insert_json_payload finish inserting 1 Point. code: %d", info->id, code); +// points = TARRAY_GET_START(lpPoints); +// numPoints = taosArrayGetSize(lpPoints); +// for (int i = 0; i < numPoints; ++i) { +// destroySmlDataPoint(points+i); +// } +// +// taosArrayDestroy(&lpPoints); +// +// tfree(info); +// return code; +//} diff --git a/source/client/src/tscParseLineProtocol.c b/source/client/src/tscParseLineProtocol.c new file mode 100644 index 0000000000..e0b4cc0d7a --- /dev/null +++ b/source/client/src/tscParseLineProtocol.c @@ -0,0 +1,3134 @@ +#include +#include +#include +#include + +#include "tscParseLine.h" + +#include "tdef.h" +#include "ttypes.h" +#include "tmsg.h" +#include "tlog.h" +#include "query.h" +#include "taoserror.h" +#include "taos.h" +#include "ttime.h" + + +typedef struct { + char sTableName[TSDB_TABLE_NAME_LEN]; + SHashObj* tagHash; + SHashObj* fieldHash; + SArray* tags; //SArray + SArray* fields; //SArray + uint8_t precision; +} SSmlSTableSchema; + +#define SPACE ' ' +#define COMMA ',' +#define EQUAL '=' +#define QUOTE '"' +#define SLASH '\\' + +//================================================================================================= + +static uint64_t linesSmlHandleId = 0; + +static int32_t insertChildTablePointsBatch(void* pVoid, char* name, char* name1, SArray* pArray, SArray* pArray1, + SArray* pArray2, SArray* pArray3, size_t size, SSmlLinesInfo* info); +static int32_t doInsertChildTablePoints(void* pVoid, char* sql, char* name, SArray* pArray, SArray* pArray1, + SSmlLinesInfo* info); +uint64_t genLinesSmlId() { + uint64_t id; + + do { + id = atomic_add_fetch_64(&linesSmlHandleId, 1); + } while (id == 0); + + return id; +} + +int compareSmlColKv(const void* p1, const void* p2) { + TAOS_SML_KV* kv1 = (TAOS_SML_KV*)p1; + TAOS_SML_KV* kv2 = (TAOS_SML_KV*)p2; + int kvLen1 = (int)strlen(kv1->key); + int kvLen2 = (int)strlen(kv2->key); + int res = strncasecmp(kv1->key, kv2->key, MIN(kvLen1, kvLen2)); + if (res != 0) { + return res; + } else { + return kvLen1-kvLen2; + } +} + +typedef enum { + SCHEMA_ACTION_CREATE_STABLE, + SCHEMA_ACTION_ADD_COLUMN, + SCHEMA_ACTION_ADD_TAG, + SCHEMA_ACTION_CHANGE_COLUMN_SIZE, + SCHEMA_ACTION_CHANGE_TAG_SIZE, +} ESchemaAction; + +typedef struct { + char sTableName[TSDB_TABLE_NAME_LEN]; + SArray* tags; //SArray + SArray* fields; //SArray +} SCreateSTableActionInfo; + +typedef struct { + char sTableName[TSDB_TABLE_NAME_LEN]; + SSchema* field; +} SAlterSTableActionInfo; + +typedef struct { + ESchemaAction action; + union { + SCreateSTableActionInfo createSTable; + SAlterSTableActionInfo alterSTable; + }; +} SSchemaAction; + +static int32_t getFieldBytesFromSmlKv(TAOS_SML_KV* kv, int32_t* bytes, uint64_t id) { + if (!IS_VAR_DATA_TYPE(kv->type)) { + *bytes = tDataTypes[kv->type].bytes; + } else { + if (kv->type == TSDB_DATA_TYPE_NCHAR) { + TdUcs4 *ucs = taosMemoryMalloc(kv->length * TSDB_NCHAR_SIZE + 1); + int32_t bytesNeeded = 0; + bool succ = taosMbsToUcs4(kv->value, kv->length, ucs, kv->length * TSDB_NCHAR_SIZE, &bytesNeeded); + if (!succ) { + taosMemoryFree(ucs); + uError("SML:0x%"PRIx64" convert nchar string to UCS4_LE failed:%s", id, kv->value); + return TSDB_CODE_TSC_INVALID_VALUE; + } + taosMemoryFree(ucs); + *bytes = bytesNeeded + VARSTR_HEADER_SIZE; + } else if (kv->type == TSDB_DATA_TYPE_BINARY) { + *bytes = kv->length + VARSTR_HEADER_SIZE; + } + } + return 0; +} + +static int32_t buildSmlKvSchema(TAOS_SML_KV* smlKv, SHashObj* hash, SArray* array, SSmlLinesInfo* info) { + SSchema* pField = NULL; + size_t* pFieldIdx = taosHashGet(hash, smlKv->key, strlen(smlKv->key)); + size_t fieldIdx = -1; + int32_t code = 0; + if (pFieldIdx) { + fieldIdx = *pFieldIdx; + pField = taosArrayGet(array, fieldIdx); + + if (pField->type != smlKv->type) { + uError("SML:0x%"PRIx64" type mismatch. key %s, type %d. type before %d", info->id, smlKv->key, smlKv->type, pField->type); + return TSDB_CODE_TSC_INVALID_VALUE; + } + + int32_t bytes = 0; + code = getFieldBytesFromSmlKv(smlKv, &bytes, info->id); + if (code != 0) { + return code; + } + pField->bytes = MAX(pField->bytes, bytes); + + } else { + SSchema field = {0}; + size_t tagKeyLen = strlen(smlKv->key); + strncpy(field.name, smlKv->key, tagKeyLen); + field.name[tagKeyLen] = '\0'; + field.type = smlKv->type; + + int32_t bytes = 0; + code = getFieldBytesFromSmlKv(smlKv, &bytes, info->id); + if (code != 0) { + return code; + } + field.bytes = bytes; + + pField = taosArrayPush(array, &field); + fieldIdx = taosArrayGetSize(array) - 1; + taosHashPut(hash, field.name, tagKeyLen, &fieldIdx, sizeof(fieldIdx)); + } + + smlKv->fieldSchemaIdx = (uint32_t)fieldIdx; + + return 0; +} + +static int32_t getSmlMd5ChildTableName(TAOS_SML_DATA_POINT* point, char* tableName, int* tableNameLen, + SSmlLinesInfo* info) { + uDebug("SML:0x%"PRIx64" taos_sml_insert get child table name through md5", info->id); + if (point->tagNum) { + qsort(point->tags, point->tagNum, sizeof(TAOS_SML_KV), compareSmlColKv); + } + + SStringBuilder sb; memset(&sb, 0, sizeof(sb)); + char sTableName[TSDB_TABLE_NAME_LEN] = {0}; + strncpy(sTableName, point->stableName, strlen(point->stableName)); + //strtolower(sTableName, point->stableName); + taosStringBuilderAppendString(&sb, sTableName); + for (int j = 0; j < point->tagNum; ++j) { + taosStringBuilderAppendChar(&sb, ','); + TAOS_SML_KV* tagKv = point->tags + j; + char tagName[TSDB_COL_NAME_LEN] = {0}; + strncpy(tagName, tagKv->key, strlen(tagKv->key)); + //strtolower(tagName, tagKv->key); + taosStringBuilderAppendString(&sb, tagName); + taosStringBuilderAppendChar(&sb, '='); + taosStringBuilderAppend(&sb, tagKv->value, tagKv->length); + } + size_t len = 0; + char* keyJoined = taosStringBuilderGetResult(&sb, &len); + T_MD5_CTX context; + tMD5Init(&context); + tMD5Update(&context, (uint8_t *)keyJoined, (uint32_t)len); + tMD5Final(&context); + uint64_t digest1 = *(uint64_t*)(context.digest); + uint64_t digest2 = *(uint64_t*)(context.digest + 8); + *tableNameLen = snprintf(tableName, *tableNameLen, + "t_%016"PRIx64"%016"PRIx64, digest1, digest2); + taosStringBuilderDestroy(&sb); + uDebug("SML:0x%"PRIx64" child table name: %s", info->id, tableName); + return 0; +} + +static int32_t buildSmlChildTableName(TAOS_SML_DATA_POINT* point, SSmlLinesInfo* info) { + uDebug("SML:0x%"PRIx64" taos_sml_insert build child table name", info->id); + char childTableName[TSDB_TABLE_NAME_LEN]; + int32_t tableNameLen = TSDB_TABLE_NAME_LEN; + getSmlMd5ChildTableName(point, childTableName, &tableNameLen, info); + point->childTableName = calloc(1, tableNameLen+1); + strncpy(point->childTableName, childTableName, tableNameLen); + point->childTableName[tableNameLen] = '\0'; + return 0; +} + +static int32_t buildDataPointSchemas(TAOS_SML_DATA_POINT* points, int numPoint, SArray* stableSchemas, SSmlLinesInfo* info) { + int32_t code = 0; + SHashObj* sname2shema = taosHashInit(32, + taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); + + for (int i = 0; i < numPoint; ++i) { + TAOS_SML_DATA_POINT* point = &points[i]; + size_t stableNameLen = strlen(point->stableName); + size_t* pStableIdx = taosHashGet(sname2shema, point->stableName, stableNameLen); + SSmlSTableSchema* pStableSchema = NULL; + size_t stableIdx = -1; + if (pStableIdx) { + pStableSchema= taosArrayGet(stableSchemas, *pStableIdx); + stableIdx = *pStableIdx; + } else { + SSmlSTableSchema schema; + strncpy(schema.sTableName, point->stableName, stableNameLen); + schema.sTableName[stableNameLen] = '\0'; + schema.fields = taosArrayInit(64, sizeof(SSchema)); + schema.tags = taosArrayInit(8, sizeof(SSchema)); + schema.tagHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); + schema.fieldHash = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); + + pStableSchema = taosArrayPush(stableSchemas, &schema); + stableIdx = taosArrayGetSize(stableSchemas) - 1; + taosHashPut(sname2shema, schema.sTableName, stableNameLen, &stableIdx, sizeof(size_t)); + } + + for (int j = 0; j < point->tagNum; ++j) { + TAOS_SML_KV* tagKv = point->tags + j; + if (!point->childTableName) { + buildSmlChildTableName(point, info); + } + + code = buildSmlKvSchema(tagKv, pStableSchema->tagHash, pStableSchema->tags, info); + if (code != 0) { + uError("SML:0x%"PRIx64" build data point schema failed. point no.: %d, tag key: %s", info->id, i, tagKv->key); + return code; + } + } + + //for Line Protocol tags may be omitted, add a tag with NULL value + if (point->tagNum == 0) { + if (!point->childTableName) { + buildSmlChildTableName(point, info); + } + char tagNullName[TSDB_COL_NAME_LEN] = {0}; + size_t nameLen = strlen(tsSmlTagNullName); + strncpy(tagNullName, tsSmlTagNullName, nameLen); + addEscapeCharToString(tagNullName, (int32_t)nameLen); + size_t* pTagNullIdx = taosHashGet(pStableSchema->tagHash, tagNullName, nameLen); + if (!pTagNullIdx) { + SSchema tagNull = {0}; + tagNull.type = TSDB_DATA_TYPE_NCHAR; + tagNull.bytes = TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE; + strncpy(tagNull.name, tagNullName, nameLen); + taosArrayPush(pStableSchema->tags, &tagNull); + size_t tagNullIdx = taosArrayGetSize(pStableSchema->tags) - 1; + taosHashPut(pStableSchema->tagHash, tagNull.name, nameLen, &tagNullIdx, sizeof(tagNullIdx)); + } + } + + for (int j = 0; j < point->fieldNum; ++j) { + TAOS_SML_KV* fieldKv = point->fields + j; + code = buildSmlKvSchema(fieldKv, pStableSchema->fieldHash, pStableSchema->fields, info); + if (code != 0) { + uError("SML:0x%"PRIx64" build data point schema failed. point no.: %d, tag key: %s", info->id, i, fieldKv->key); + return code; + } + } + + point->schemaIdx = (uint32_t)stableIdx; + } + + size_t numStables = taosArrayGetSize(stableSchemas); + for (int32_t i = 0; i < numStables; ++i) { + SSmlSTableSchema* schema = taosArrayGet(stableSchemas, i); + taosHashCleanup(schema->tagHash); + taosHashCleanup(schema->fieldHash); + } + taosHashCleanup(sname2shema); + + uDebug("SML:0x%"PRIx64" build point schema succeed. num of super table: %zu", info->id, numStables); + for (int32_t i = 0; i < numStables; ++i) { + SSmlSTableSchema* schema = taosArrayGet(stableSchemas, i); + uDebug("\ttable name: %s, tags number: %zu, fields number: %zu", schema->sTableName, + taosArrayGetSize(schema->tags), taosArrayGetSize(schema->fields)); + } + + return 0; +} + +static int32_t generateSchemaAction(SSchema* pointColField, SHashObj* dbAttrHash, SArray* dbAttrArray, bool isTag, char sTableName[], + SSchemaAction* action, bool* actionNeeded, SSmlLinesInfo* info) { + char fieldName[TSDB_COL_NAME_LEN] = {0}; + strcpy(fieldName, pointColField->name); + + size_t* pDbIndex = taosHashGet(dbAttrHash, fieldName, strlen(fieldName)); + if (pDbIndex) { + SSchema* dbAttr = taosArrayGet(dbAttrArray, *pDbIndex); + assert(strcasecmp(dbAttr->name, pointColField->name) == 0); + if (pointColField->type != dbAttr->type) { + uError("SML:0x%"PRIx64" point type and db type mismatch. key: %s. point type: %d, db type: %d", info->id, pointColField->name, + pointColField->type, dbAttr->type); + return TSDB_CODE_TSC_INVALID_VALUE; + } + + if (IS_VAR_DATA_TYPE(pointColField->type) && (pointColField->bytes > dbAttr->bytes)) { + if (isTag) { + action->action = SCHEMA_ACTION_CHANGE_TAG_SIZE; + } else { + action->action = SCHEMA_ACTION_CHANGE_COLUMN_SIZE; + } + memset(&action->alterSTable, 0, sizeof(SAlterSTableActionInfo)); + memcpy(action->alterSTable.sTableName, sTableName, TSDB_TABLE_NAME_LEN); + action->alterSTable.field = pointColField; + *actionNeeded = true; + } + } else { + if (isTag) { + action->action = SCHEMA_ACTION_ADD_TAG; + } else { + action->action = SCHEMA_ACTION_ADD_COLUMN; + } + memset(&action->alterSTable, 0, sizeof(SAlterSTableActionInfo)); + memcpy(action->alterSTable.sTableName, sTableName, TSDB_TABLE_NAME_LEN); + action->alterSTable.field = pointColField; + *actionNeeded = true; + } + if (*actionNeeded) { + uDebug("SML:0x%" PRIx64 " generate schema action. column name: %s, action: %d", info->id, fieldName, + action->action); + } + return 0; +} + +static int32_t buildColumnDescription(SSchema* field, + char* buf, int32_t bufSize, int32_t* outBytes) { + uint8_t type = field->type; + + if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { + int32_t bytes = field->bytes - VARSTR_HEADER_SIZE; + if (type == TSDB_DATA_TYPE_NCHAR) { + bytes = bytes/TSDB_NCHAR_SIZE; + } + int out = snprintf(buf, bufSize,"%s %s(%d)", + field->name,tDataTypes[field->type].name, bytes); + *outBytes = out; + } else { + int out = snprintf(buf, bufSize, "%s %s", + field->name, tDataTypes[type].name); + *outBytes = out; + } + + return 0; +} + + +static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInfo* info) { + int32_t code = 0; + int32_t outBytes = 0; + char *result = (char *)calloc(1, tsMaxSQLStringLen+1); + int32_t capacity = tsMaxSQLStringLen + 1; + + uDebug("SML:0x%"PRIx64" apply schema action. action: %d", info->id, action->action); + switch (action->action) { + case SCHEMA_ACTION_ADD_COLUMN: { + int n = sprintf(result, "alter stable %s add column ", action->alterSTable.sTableName); + buildColumnDescription(action->alterSTable.field, result+n, capacity-n, &outBytes); + TAOS_RES* res = taos_query(taos, result); //TODO async doAsyncQuery + code = taos_errno(res); + char* errStr = taos_errstr(res); + char* begin = strstr(errStr, "duplicated column names"); + bool tscDupColNames = (begin != NULL); + if (code != TSDB_CODE_SUCCESS) { + uError("SML:0x%"PRIx64" apply schema action. error: %s", info->id, errStr); + } + taos_free_result(res); + + if (code == TSDB_CODE_MND_FIELD_ALREAY_EXIST || code == TSDB_CODE_MND_TAG_ALREAY_EXIST || tscDupColNames) { + TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); + code = taos_errno(res2); + if (code != TSDB_CODE_SUCCESS) { + uError("SML:0x%" PRIx64 " apply schema action. reset query cache. error: %s", info->id, taos_errstr(res2)); + } + taos_free_result(res2); + taosMsleep(500); + } + break; + } + case SCHEMA_ACTION_ADD_TAG: { + int n = sprintf(result, "alter stable %s add tag ", action->alterSTable.sTableName); + buildColumnDescription(action->alterSTable.field, + result+n, capacity-n, &outBytes); + TAOS_RES* res = taos_query(taos, result); //TODO async doAsyncQuery + code = taos_errno(res); + char* errStr = taos_errstr(res); + char* begin = strstr(errStr, "duplicated column names"); + bool tscDupColNames = (begin != NULL); + if (code != TSDB_CODE_SUCCESS) { + uError("SML:0x%"PRIx64" apply schema action. error : %s", info->id, taos_errstr(res)); + } + taos_free_result(res); + + if (code == TSDB_CODE_MND_TAG_ALREAY_EXIST || code == TSDB_CODE_MND_FIELD_ALREAY_EXIST || tscDupColNames) { + TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); + code = taos_errno(res2); + if (code != TSDB_CODE_SUCCESS) { + uError("SML:0x%" PRIx64 " apply schema action. reset query cache. error: %s", info->id, taos_errstr(res2)); + } + taos_free_result(res2); + taosMsleep(500); + } + break; + } + case SCHEMA_ACTION_CHANGE_COLUMN_SIZE: { + int n = sprintf(result, "alter stable %s modify column ", action->alterSTable.sTableName); + buildColumnDescription(action->alterSTable.field, result+n, + capacity-n, &outBytes); + TAOS_RES* res = taos_query(taos, result); //TODO async doAsyncQuery + code = taos_errno(res); + if (code != TSDB_CODE_SUCCESS) { + uError("SML:0x%"PRIx64" apply schema action. error : %s", info->id, taos_errstr(res)); + } + taos_free_result(res); + + if (code == TSDB_CODE_MND_INVALID_COLUMN_LENGTH || code == TSDB_CODE_TSC_INVALID_COLUMN_LENGTH) { + TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); + code = taos_errno(res2); + if (code != TSDB_CODE_SUCCESS) { + uError("SML:0x%" PRIx64 " apply schema action. reset query cache. error: %s", info->id, taos_errstr(res2)); + } + taos_free_result(res2); + taosMsleep(500); + } + break; + } + case SCHEMA_ACTION_CHANGE_TAG_SIZE: { + int n = sprintf(result, "alter stable %s modify tag ", action->alterSTable.sTableName); + buildColumnDescription(action->alterSTable.field, result+n, + capacity-n, &outBytes); + TAOS_RES* res = taos_query(taos, result); //TODO async doAsyncQuery + code = taos_errno(res); + if (code != TSDB_CODE_SUCCESS) { + uError("SML:0x%"PRIx64" apply schema action. error : %s", info->id, taos_errstr(res)); + } + taos_free_result(res); + + if (code == TSDB_CODE_MND_INVALID_TAG_LENGTH || code == TSDB_CODE_TSC_INVALID_TAG_LENGTH) { + TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); + code = taos_errno(res2); + if (code != TSDB_CODE_SUCCESS) { + uError("SML:0x%" PRIx64 " apply schema action. reset query cache. error: %s", info->id, taos_errstr(res2)); + } + taos_free_result(res2); + taosMsleep(500); + } + break; + } + case SCHEMA_ACTION_CREATE_STABLE: { + int n = sprintf(result, "create stable %s (", action->createSTable.sTableName); + char* pos = result + n; int freeBytes = capacity - n; + size_t numCols = taosArrayGetSize(action->createSTable.fields); + for (int32_t i = 0; i < numCols; ++i) { + SSchema* field = taosArrayGet(action->createSTable.fields, i); + buildColumnDescription(field, pos, freeBytes, &outBytes); + pos += outBytes; freeBytes -= outBytes; + *pos = ','; ++pos; --freeBytes; + } + --pos; ++freeBytes; + + outBytes = snprintf(pos, freeBytes, ") tags ("); + pos += outBytes; freeBytes -= outBytes; + + size_t numTags = taosArrayGetSize(action->createSTable.tags); + for (int32_t i = 0; i < numTags; ++i) { + SSchema* field = taosArrayGet(action->createSTable.tags, i); + buildColumnDescription(field, pos, freeBytes, &outBytes); + pos += outBytes; freeBytes -= outBytes; + *pos = ','; ++pos; --freeBytes; + } + pos--; ++freeBytes; + outBytes = snprintf(pos, freeBytes, ")"); + TAOS_RES* res = taos_query(taos, result); + code = taos_errno(res); + if (code != TSDB_CODE_SUCCESS) { + uError("SML:0x%"PRIx64" apply schema action. error : %s", info->id, taos_errstr(res)); + } + taos_free_result(res); + + if (code == TSDB_CODE_MND_TABLE_ALREADY_EXIST) { + TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); + code = taos_errno(res2); + if (code != TSDB_CODE_SUCCESS) { + uError("SML:0x%" PRIx64 " apply schema action. reset query cache. error: %s", info->id, taos_errstr(res2)); + } + taos_free_result(res2); + taosMsleep(500); + } + break; + } + + default: + break; + } + + taosMemoryFree(result); + if (code != 0) { + uError("SML:0x%"PRIx64 " apply schema action failure. %s", info->id, tstrerror(code)); + } + return code; +} + +static int32_t destroySmlSTableSchema(SSmlSTableSchema* schema) { + taosHashCleanup(schema->tagHash); + taosHashCleanup(schema->fieldHash); + taosArrayDestroy(&schema->tags); + taosArrayDestroy(&schema->fields); + return 0; +} + +static int32_t fillDbSchema(STableMeta* tableMeta, char* tableName, SSmlSTableSchema* schema, SSmlLinesInfo* info) { + schema->tags = taosArrayInit(8, sizeof(SSchema)); + schema->fields = taosArrayInit(64, sizeof(SSchema)); + schema->tagHash = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); + schema->fieldHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); + + tstrncpy(schema->sTableName, tableName, strlen(tableName)+1); + schema->precision = tableMeta->tableInfo.precision; + for (int i=0; itableInfo.numOfColumns; ++i) { + SSchema field; + tstrncpy(field.name, tableMeta->schema[i].name, strlen(tableMeta->schema[i].name)+1); + addEscapeCharToString(field.name, (int16_t)strlen(field.name)); + field.type = tableMeta->schema[i].type; + field.bytes = tableMeta->schema[i].bytes; + taosArrayPush(schema->fields, &field); + size_t fieldIndex = taosArrayGetSize(schema->fields) - 1; + taosHashPut(schema->fieldHash, field.name, strlen(field.name), &fieldIndex, sizeof(fieldIndex)); + } + + for (int i=0; itableInfo.numOfTags; ++i) { + int j = i + tableMeta->tableInfo.numOfColumns; + SSchema field; + tstrncpy(field.name, tableMeta->schema[j].name, strlen(tableMeta->schema[j].name)+1); + addEscapeCharToString(field.name, (int16_t)strlen(field.name)); + field.type = tableMeta->schema[j].type; + field.bytes = tableMeta->schema[j].bytes; + taosArrayPush(schema->tags, &field); + size_t tagIndex = taosArrayGetSize(schema->tags) - 1; + taosHashPut(schema->tagHash, field.name, strlen(field.name), &tagIndex, sizeof(tagIndex)); + } + uDebug("SML:0x%"PRIx64 " load table schema succeed. table name: %s, columns number: %d, tag number: %d, precision: %d", + info->id, tableName, tableMeta->tableInfo.numOfColumns, tableMeta->tableInfo.numOfTags, schema->precision); + return TSDB_CODE_SUCCESS; +} + +static int32_t getSuperTableMetaFromLocalCache(TAOS* taos, char* tableName, STableMeta** outTableMeta, SSmlLinesInfo* info) { + int32_t code = 0; + STableMeta* tableMeta = NULL; + + SSqlObj* pSql = calloc(1, sizeof(SSqlObj)); + if (pSql == NULL) { + uError("SML:0x%" PRIx64 " failed to allocate memory, reason:%s", info->id, strerror(errno)); + code = TSDB_CODE_TSC_OUT_OF_MEMORY; + return code; + } + pSql->pTscObj = taos; + pSql->signature = pSql; + pSql->fp = NULL; + + registerSqlObj(pSql); + char tableNameBuf[TSDB_TABLE_NAME_LEN + TS_BACKQUOTE_CHAR_SIZE] = {0}; + memcpy(tableNameBuf, tableName, strlen(tableName)); + SStrToken tableToken = {.z = tableNameBuf, .n = (uint32_t)strlen(tableName), .type = TK_ID}; + tGetToken(tableNameBuf, &tableToken.type); + bool dbIncluded = false; + // Check if the table name available or not + if (tscValidateName(&tableToken, true, &dbIncluded) != TSDB_CODE_SUCCESS) { + code = TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; + sprintf(pSql->cmd.payload, "table name is invalid"); + taosReleaseRef(tscObjRef, pSql->self); + return code; + } + + SName sname = {0}; + if ((code = tscSetTableFullName(&sname, &tableToken, pSql, dbIncluded)) != TSDB_CODE_SUCCESS) { + taosReleaseRef(tscObjRef, pSql->self); + return code; + } + + char fullTableName[TSDB_TABLE_FNAME_LEN] = {0}; + memset(fullTableName, 0, tListLen(fullTableName)); + tNameExtractFullName(&sname, fullTableName); + + size_t size = 0; + taosHashGetCloneExt(UTIL_GET_TABLEMETA(pSql), fullTableName, strlen(fullTableName), NULL, (void**)&tableMeta, &size); + + STableMeta* stableMeta = tableMeta; + if (tableMeta != NULL && tableMeta->tableType == TSDB_CHILD_TABLE) { + taosHashGetCloneExt(UTIL_GET_TABLEMETA(pSql), tableMeta->sTableName, strlen(tableMeta->sTableName), NULL, + (void**)stableMeta, &size); + } + taosReleaseRef(tscObjRef, pSql->self); + + if (stableMeta != tableMeta) { + taosMemoryFree(tableMeta); + } + + if (stableMeta != NULL) { + if (outTableMeta != NULL) { + *outTableMeta = stableMeta; + } else { + taosMemoryFree(stableMeta); + } + return TSDB_CODE_SUCCESS; + } else { + return TSDB_CODE_TSC_NO_META_CACHED; + } +} + +static int32_t retrieveTableMeta(TAOS* taos, char* tableName, STableMeta** pTableMeta, SSmlLinesInfo* info) { + int32_t code = 0; + int32_t retries = 0; + STableMeta* tableMeta = NULL; + while (retries++ <= TSDB_MAX_REPLICA && tableMeta == NULL) { + STscObj* pObj = (STscObj*)taos; + if (pObj == NULL || pObj->signature != pObj) { + terrno = TSDB_CODE_TSC_DISCONNECTED; + return TSDB_CODE_TSC_DISCONNECTED; + } + + uDebug("SML:0x%" PRIx64 " retrieve table meta. super table name: %s", info->id, tableName); + code = getSuperTableMetaFromLocalCache(taos, tableName, &tableMeta, info); + if (code == TSDB_CODE_SUCCESS) { + uDebug("SML:0x%" PRIx64 " successfully retrieved table meta. super table name: %s", info->id, tableName); + break; + } else if (code == TSDB_CODE_TSC_NO_META_CACHED) { + char sql[256]; + snprintf(sql, 256, "describe %s", tableName); + TAOS_RES* res = taos_query(taos, sql); + code = taos_errno(res); + if (code != 0) { + uError("SML:0x%" PRIx64 " describe table failure. %s", info->id, taos_errstr(res)); + taos_free_result(res); + return code; + } + taos_free_result(res); + } else { + return code; + } + } + + if (tableMeta != NULL) { + *pTableMeta = tableMeta; + return TSDB_CODE_SUCCESS; + } else { + uError("SML:0x%" PRIx64 " failed to retrieve table meta. super table name: %s", info->id, tableName); + return TSDB_CODE_TSC_NO_META_CACHED; + } +} + +static int32_t loadTableSchemaFromDB(TAOS* taos, char* tableName, SSmlSTableSchema* schema, SSmlLinesInfo* info) { + int32_t code = 0; + STableMeta* tableMeta = NULL; + code = retrieveTableMeta(taos, tableName, &tableMeta, info); + if (code == TSDB_CODE_SUCCESS) { + assert(tableMeta != NULL); + fillDbSchema(tableMeta, tableName, schema, info); + taosMemoryFree(tableMeta); + tableMeta = NULL; + } + + return code; +} + +static int32_t modifyDBSchemas(TAOS* taos, SSmlLinesInfo* info) { + int32_t code = 0; + size_t numStable = taosHashGetSize(info->superTables); + + SSmlSTableMeta** tableMetaSml = taosHashIterate(info->superTables, NULL); + while (tableMetaSml) { + SSmlSTableMeta* cTablePoints = *tableMetaSml; + + if (NULL == pStmt->pCatalog) { + STMT_ERR_RET(catalogGetHandle(pStmt->taos->pAppInfo->clusterId, &pStmt->pCatalog)); + } + + STableMeta *pTableMeta = NULL; + SEpSet ep = getEpSet_s(&pStmt->taos->pAppInfo->mgmtEp); + STMT_ERR_RET(catalogGetTableMeta(pStmt->pCatalog, pStmt->taos->pAppInfo->pTransporter, &ep, &pStmt->bInfo.sname, &pTableMeta)); + + if (pTableMeta->uid == pStmt->bInfo.tbUid) { + pStmt->bInfo.needParse = false; + + return TSDB_CODE_SUCCESS; + } + +// for (int i = 0; i < numStable; ++i) { + SSmlSTableSchema* pointSchema = taosArrayGet(stableSchemas, i); + SSmlSTableSchema dbSchema; + memset(&dbSchema, 0, sizeof(SSmlSTableSchema)); + + code = loadTableSchemaFromDB(taos, pointSchema->sTableName, &dbSchema, info); + if (code == TSDB_CODE_MND_INVALID_TABLE_NAME) { + SSchemaAction schemaAction = {0}; + schemaAction.action = SCHEMA_ACTION_CREATE_STABLE; + memset(&schemaAction.createSTable, 0, sizeof(SCreateSTableActionInfo)); + memcpy(schemaAction.createSTable.sTableName, pointSchema->sTableName, TSDB_TABLE_NAME_LEN); + schemaAction.createSTable.tags = pointSchema->tags; + schemaAction.createSTable.fields = pointSchema->fields; + applySchemaAction(taos, &schemaAction, info); + code = loadTableSchemaFromDB(taos, pointSchema->sTableName, &dbSchema, info); + if (code != 0) { + uError("SML:0x%"PRIx64" reconcile point schema failed. can not create %s", info->id, pointSchema->sTableName); + return code; + } + } + + if (code == TSDB_CODE_SUCCESS) { + pointSchema->precision = dbSchema.precision; + + size_t pointTagSize = taosArrayGetSize(pointSchema->tags); + size_t pointFieldSize = taosArrayGetSize(pointSchema->fields); + + SHashObj* dbTagHash = dbSchema.tagHash; + SHashObj* dbFieldHash = dbSchema.fieldHash; + + for (int j = 0; j < pointTagSize; ++j) { + SSchema* pointTag = taosArrayGet(pointSchema->tags, j); + SSchemaAction schemaAction = {0}; + bool actionNeeded = false; + generateSchemaAction(pointTag, dbTagHash, dbSchema.tags, true, pointSchema->sTableName, + &schemaAction, &actionNeeded, info); + if (actionNeeded) { + code = applySchemaAction(taos, &schemaAction, info); + if (code != 0) { + destroySmlSTableSchema(&dbSchema); + return code; + } + } + } + + SSchema* pointColTs = taosArrayGet(pointSchema->fields, 0); + SSchema* dbColTs = taosArrayGet(dbSchema.fields, 0); + memcpy(pointColTs->name, dbColTs->name, TSDB_COL_NAME_LEN); + + for (int j = 1; j < pointFieldSize; ++j) { + SSchema* pointCol = taosArrayGet(pointSchema->fields, j); + SSchemaAction schemaAction = {0}; + bool actionNeeded = false; + generateSchemaAction(pointCol, dbFieldHash, dbSchema.fields,false, pointSchema->sTableName, + &schemaAction, &actionNeeded, info); + if (actionNeeded) { + code = applySchemaAction(taos, &schemaAction, info); + if (code != 0) { + destroySmlSTableSchema(&dbSchema); + return code; + } + } + } + + pointSchema->precision = dbSchema.precision; + + destroySmlSTableSchema(&dbSchema); + } else { + uError("SML:0x%"PRIx64" load table meta error: %s", info->id, tstrerror(code)); + return code; + } + tableMetaSml = taosHashIterate(info->superTables, tableMetaSml); + } +// } + return 0; +} + +static int32_t arrangePointsByChildTableName(TAOS_SML_DATA_POINT* points, int numPoints, + SHashObj* cname2points, SArray* stableSchemas, SSmlLinesInfo* info) { + for (int32_t i = 0; i < numPoints; ++i) { + TAOS_SML_DATA_POINT * point = points + i; + SSmlSTableSchema* stableSchema = taosArrayGet(stableSchemas, point->schemaIdx); + + for (int j = 0; j < point->tagNum; ++j) { + TAOS_SML_KV* kv = point->tags + j; + if (kv->type == TSDB_DATA_TYPE_TIMESTAMP) { + int64_t ts = *(int64_t*)(kv->value); + ts = convertTimePrecision(ts, TSDB_TIME_PRECISION_NANO, stableSchema->precision); + *(int64_t*)(kv->value) = ts; + } + } + + for (int j = 0; j < point->fieldNum; ++j) { + TAOS_SML_KV* kv = point->fields + j; + if (kv->type == TSDB_DATA_TYPE_TIMESTAMP) { + int64_t ts = *(int64_t*)(kv->value); + ts = convertTimePrecision(ts, TSDB_TIME_PRECISION_NANO, stableSchema->precision); + *(int64_t*)(kv->value) = ts; + } + } + + SArray* cTablePoints = NULL; + SArray** pCTablePoints = taosHashGet(cname2points, point->childTableName, strlen(point->childTableName)); + if (pCTablePoints) { + cTablePoints = *pCTablePoints; + } else { + cTablePoints = taosArrayInit(64, sizeof(point)); + taosHashPut(cname2points, point->childTableName, strlen(point->childTableName), &cTablePoints, POINTER_BYTES); + } + taosArrayPush(cTablePoints, &point); + } + + return 0; +} + +static int32_t applyChildTableDataPointsWithInsertSQL(TAOS* taos, char* cTableName, char* sTableName, SSmlSTableSchema* sTableSchema, + SArray* cTablePoints, size_t rowSize, SSmlLinesInfo* info) { + int32_t code = TSDB_CODE_SUCCESS; + size_t numTags = taosArrayGetSize(sTableSchema->tags); + size_t numCols = taosArrayGetSize(sTableSchema->fields); + size_t rows = taosArrayGetSize(cTablePoints); + SArray* tagsSchema = sTableSchema->tags; + SArray* colsSchema = sTableSchema->fields; + + TAOS_SML_KV* tagKVs[TSDB_MAX_TAGS] = {0}; + for (int i = 0; i < rows; ++i) { + TAOS_SML_DATA_POINT* pDataPoint = taosArrayGetP(cTablePoints, i); + for (int j = 0; j < pDataPoint->tagNum; ++j) { + TAOS_SML_KV* kv = pDataPoint->tags + j; + tagKVs[kv->fieldSchemaIdx] = kv; + } + } + + char* sql = taosMemoryMalloc(tsMaxSQLStringLen + 1); + if (sql == NULL) { + uError("taosMemoryMalloc sql memory error"); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + int32_t freeBytes = tsMaxSQLStringLen + 1; + int32_t totalLen = 0; + totalLen += sprintf(sql, "insert into %s using %s (", cTableName, sTableName); + for (int i = 0; i < numTags; ++i) { + SSchema* tagSchema = taosArrayGet(tagsSchema, i); + totalLen += snprintf(sql + totalLen, freeBytes - totalLen, "%s,", tagSchema->name); + } + --totalLen; + totalLen += snprintf(sql + totalLen, freeBytes - totalLen, ")"); + + totalLen += snprintf(sql + totalLen, freeBytes - totalLen, " tags ("); + + // for (int i = 0; i < numTags; ++i) { + // snprintf(sql+strlen(sql), freeBytes-strlen(sql), "?,"); + // } + for (int i = 0; i < numTags; ++i) { + if (tagKVs[i] == NULL) { + totalLen += snprintf(sql + totalLen, freeBytes - totalLen, "NULL,"); + } else { + TAOS_SML_KV* kv = tagKVs[i]; + size_t beforeLen = totalLen; + int32_t len = 0; + converToStr(sql + beforeLen, kv->type, kv->value, kv->length, &len); + totalLen += len; + totalLen += snprintf(sql + totalLen, freeBytes - totalLen, ","); + } + } + --totalLen; + totalLen += snprintf(sql + totalLen, freeBytes - totalLen, ") ("); + + for (int i = 0; i < numCols; ++i) { + SSchema* colSchema = taosArrayGet(colsSchema, i); + totalLen += snprintf(sql + totalLen, freeBytes - totalLen, "%s,", colSchema->name); + } + --totalLen; + totalLen += snprintf(sql + totalLen, freeBytes - totalLen, ") values "); + + TAOS_SML_KV** colKVs = taosMemoryMalloc(numCols * sizeof(TAOS_SML_KV*)); + for (int r = 0; r < rows; ++r) { + totalLen += snprintf(sql + totalLen, freeBytes - totalLen, "("); + + memset(colKVs, 0, numCols * sizeof(TAOS_SML_KV*)); + + TAOS_SML_DATA_POINT* point = taosArrayGetP(cTablePoints, r); + for (int i = 0; i < point->fieldNum; ++i) { + TAOS_SML_KV* kv = point->fields + i; + colKVs[kv->fieldSchemaIdx] = kv; + } + + for (int i = 0; i < numCols; ++i) { + if (colKVs[i] == NULL) { + totalLen += snprintf(sql + totalLen, freeBytes - totalLen, "NULL,"); + } else { + TAOS_SML_KV* kv = colKVs[i]; + size_t beforeLen = totalLen; + int32_t len = 0; + converToStr(sql + beforeLen, kv->type, kv->value, kv->length, &len); + totalLen += len; + totalLen += snprintf(sql + totalLen, freeBytes - totalLen, ","); + } + } + --totalLen; + totalLen += snprintf(sql + totalLen, freeBytes - totalLen, ")"); + } + taosMemoryFree(colKVs); + sql[totalLen] = '\0'; + + uDebug("SML:0x%" PRIx64 " insert child table table %s of super table %s sql: %s", info->id, cTableName, sTableName, + sql); + + bool tryAgain = false; + int32_t try = 0; + do { + TAOS_RES* res = taos_query(taos, sql); + code = taos_errno(res); + if (code != 0) { + uError("SML:0x%"PRIx64 " taos_query return %d:%s", info->id, code, taos_errstr(res)); + } + + uDebug("SML:0x%"PRIx64 " taos_query inserted %d rows", info->id, taos_affected_rows(res)); + info->affectedRows += taos_affected_rows(res); + taos_free_result(res); + + tryAgain = false; + if ((code == TSDB_CODE_TDB_INVALID_TABLE_ID + || code == TSDB_CODE_VND_INVALID_VGROUP_ID + || code == TSDB_CODE_TDB_TABLE_RECONFIGURE + || code == TSDB_CODE_APP_NOT_READY + || code == TSDB_CODE_RPC_NETWORK_UNAVAIL) && try++ < TSDB_MAX_REPLICA) { + tryAgain = true; + } + + if (code == TSDB_CODE_TDB_INVALID_TABLE_ID || code == TSDB_CODE_VND_INVALID_VGROUP_ID) { + TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); + int32_t code2 = taos_errno(res2); + if (code2 != TSDB_CODE_SUCCESS) { + uError("SML:0x%" PRIx64 " insert child table by sql. reset query cache. error: %s", info->id, taos_errstr(res2)); + } + taos_free_result(res2); + if (tryAgain) { + taosMsleep(100 * (2 << try)); + } + } + + if (code == TSDB_CODE_APP_NOT_READY || code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { + if (tryAgain) { + taosMsleep( 100 * (2 << try)); + } + } + } while (tryAgain); + + taosMemoryFree(sql); + + return code; +} + +static int32_t applyChildTableDataPointsWithStmt(TAOS* taos, char* cTableName, char* sTableName, SSmlSTableSchema* sTableSchema, + SArray* cTablePoints, size_t rowSize, SSmlLinesInfo* info) { + size_t numTags = taosArrayGetSize(sTableSchema->tags); + size_t numCols = taosArrayGetSize(sTableSchema->fields); + size_t rows = taosArrayGetSize(cTablePoints); + + TAOS_SML_KV* tagKVs[TSDB_MAX_TAGS] = {0}; + for (int i= 0; i < rows; ++i) { + TAOS_SML_DATA_POINT * pDataPoint = taosArrayGetP(cTablePoints, i); + for (int j = 0; j < pDataPoint->tagNum; ++j) { + TAOS_SML_KV* kv = pDataPoint->tags + j; + tagKVs[kv->fieldSchemaIdx] = kv; + } + } + + //tag bind + SArray* tagBinds = taosArrayInit(numTags, sizeof(TAOS_BIND)); + taosArraySetSize(tagBinds, numTags); + int isNullColBind = TSDB_TRUE; + for (int j = 0; j < numTags; ++j) { + TAOS_BIND* bind = taosArrayGet(tagBinds, j); + bind->is_null = &isNullColBind; + } + for (int j = 0; j < numTags; ++j) { + if (tagKVs[j] == NULL) continue; + TAOS_SML_KV* kv = tagKVs[j]; + TAOS_BIND* bind = taosArrayGet(tagBinds, kv->fieldSchemaIdx); + bind->buffer_type = kv->type; + bind->length = taosMemoryMalloc(sizeof(uintptr_t*)); + *bind->length = kv->length; + bind->buffer = kv->value; + bind->is_null = NULL; + } + + //rows bind + SArray* rowsBind = taosArrayInit(rows, POINTER_BYTES); + for (int i = 0; i < rows; ++i) { + TAOS_SML_DATA_POINT* point = taosArrayGetP(cTablePoints, i); + + TAOS_BIND* colBinds = calloc(numCols, sizeof(TAOS_BIND)); + if (colBinds == NULL) { + uError("SML:0x%"PRIx64" taos_sml_insert insert points, failed to allocated memory for TAOS_BIND, " + "num of rows: %zu, num of cols: %zu", info->id, rows, numCols); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + for (int j = 0; j < numCols; ++j) { + TAOS_BIND* bind = colBinds + j; + bind->is_null = &isNullColBind; + } + for (int j = 0; j < point->fieldNum; ++j) { + TAOS_SML_KV* kv = point->fields + j; + TAOS_BIND* bind = colBinds + kv->fieldSchemaIdx; + bind->buffer_type = kv->type; + bind->length = taosMemoryMalloc(sizeof(uintptr_t*)); + *bind->length = kv->length; + bind->buffer = kv->value; + bind->is_null = NULL; + } + taosArrayPush(rowsBind, &colBinds); + } + + int32_t code = 0; + code = insertChildTablePointsBatch(taos, cTableName, sTableName, sTableSchema->tags, tagBinds, sTableSchema->fields, rowsBind, rowSize, info); + if (code != 0) { + uError("SML:0x%"PRIx64" insert into child table %s failed. error %s", info->id, cTableName, tstrerror(code)); + } + + //taosMemoryFree rows bind + for (int i = 0; i < rows; ++i) { + TAOS_BIND* colBinds = taosArrayGetP(rowsBind, i); + for (int j = 0; j < numCols; ++j) { + TAOS_BIND* bind = colBinds + j; + taosMemoryFree(bind->length); + } + taosMemoryFree(colBinds); + } + taosArrayDestroy(&rowsBind); + //taosMemoryFree tag bind + for (int i = 0; i < taosArrayGetSize(tagBinds); ++i) { + TAOS_BIND* bind = taosArrayGet(tagBinds, i); + taosMemoryFree(bind->length); + } + taosArrayDestroy(&tagBinds); + return code; +} + +static int32_t insertChildTablePointsBatch(TAOS* taos, char* cTableName, char* sTableName, + SArray* tagsSchema, SArray* tagsBind, + SArray* colsSchema, SArray* rowsBind, + size_t rowSize, SSmlLinesInfo* info) { + size_t numTags = taosArrayGetSize(tagsSchema); + size_t numCols = taosArrayGetSize(colsSchema); + char* sql = taosMemoryMalloc(tsMaxSQLStringLen+1); + if (sql == NULL) { + uError("taosMemoryMalloc sql memory error"); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + int32_t freeBytes = tsMaxSQLStringLen + 1 ; + sprintf(sql, "insert into ? using %s (", sTableName); + for (int i = 0; i < numTags; ++i) { + SSchema* tagSchema = taosArrayGet(tagsSchema, i); + snprintf(sql+strlen(sql), freeBytes-strlen(sql), "%s,", tagSchema->name); + } + snprintf(sql + strlen(sql) - 1, freeBytes-strlen(sql)+1, ")"); + + snprintf(sql + strlen(sql), freeBytes-strlen(sql), " tags ("); + + for (int i = 0; i < numTags; ++i) { + snprintf(sql+strlen(sql), freeBytes-strlen(sql), "?,"); + } + snprintf(sql + strlen(sql) - 1, freeBytes-strlen(sql)+1, ") ("); + + for (int i = 0; i < numCols; ++i) { + SSchema* colSchema = taosArrayGet(colsSchema, i); + snprintf(sql+strlen(sql), freeBytes-strlen(sql), "%s,", colSchema->name); + } + snprintf(sql + strlen(sql)-1, freeBytes-strlen(sql)+1, ") values ("); + + for (int i = 0; i < numCols; ++i) { + snprintf(sql+strlen(sql), freeBytes-strlen(sql), "?,"); + } + snprintf(sql + strlen(sql)-1, freeBytes-strlen(sql)+1, ")"); + sql[strlen(sql)] = '\0'; + + uDebug("SML:0x%"PRIx64" insert child table table %s of super table %s : %s", info->id, cTableName, sTableName, sql); + + size_t maxBatchSize = TSDB_MAX_WAL_SIZE/rowSize * 2 / 3; + size_t rows = taosArrayGetSize(rowsBind); + size_t batchSize = MIN(maxBatchSize, rows); + uDebug("SML:0x%"PRIx64" insert rows into child table %s. num of rows: %zu, batch size: %zu", + info->id, cTableName, rows, batchSize); + SArray* batchBind = taosArrayInit(batchSize, POINTER_BYTES); + int32_t code = TSDB_CODE_SUCCESS; + for (int i = 0; i < rows;) { + int j = i; + for (; j < i + batchSize && j i) { + uDebug("SML:0x%"PRIx64" insert child table batch from line %d to line %d.", info->id, i, j - 1); + code = doInsertChildTablePoints(taos, sql, cTableName, tagsBind, batchBind, info); + if (code != 0) { + taosArrayDestroy(&batchBind); + tfree(sql); + return code; + } + taosArrayClear(batchBind); + } + i = j; + } + taosArrayDestroy(&batchBind); + tfree(sql); + return code; + +} +static int32_t doInsertChildTablePoints(TAOS* taos, char* sql, char* cTableName, SArray* tagsBind, SArray* batchBind, + SSmlLinesInfo* info) { + int32_t code = 0; + + TAOS_STMT* stmt = taos_stmt_init(taos); + if (stmt == NULL) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + code = taos_stmt_prepare(stmt, sql, (unsigned long)strlen(sql)); + + if (code != 0) { + uError("SML:0x%"PRIx64" taos_stmt_prepare return %d:%s", info->id, code, taos_stmt_errstr(stmt)); + taos_stmt_close(stmt); + return code; + } + + bool tryAgain = false; + int32_t try = 0; + do { + code = taos_stmt_set_tbname_tags(stmt, cTableName, TARRAY_GET_START(tagsBind)); + if (code != 0) { + uError("SML:0x%"PRIx64" taos_stmt_set_tbname return %d:%s", info->id, code, taos_stmt_errstr(stmt)); + + int affectedRows = taos_stmt_affected_rows(stmt); + info->affectedRows += affectedRows; + + taos_stmt_close(stmt); + return code; + } + + size_t rows = taosArrayGetSize(batchBind); + for (int32_t i = 0; i < rows; ++i) { + TAOS_BIND* colsBinds = taosArrayGetP(batchBind, i); + code = taos_stmt_bind_param(stmt, colsBinds); + if (code != 0) { + uError("SML:0x%"PRIx64" taos_stmt_bind_param return %d:%s", info->id, code, taos_stmt_errstr(stmt)); + + int affectedRows = taos_stmt_affected_rows(stmt); + info->affectedRows += affectedRows; + + taos_stmt_close(stmt); + return code; + } + code = taos_stmt_add_batch(stmt); + if (code != 0) { + uError("SML:0x%"PRIx64" taos_stmt_add_batch return %d:%s", info->id, code, taos_stmt_errstr(stmt)); + + int affectedRows = taos_stmt_affected_rows(stmt); + info->affectedRows += affectedRows; + + taos_stmt_close(stmt); + return code; + } + } + + code = taos_stmt_execute(stmt); + if (code != 0) { + uError("SML:0x%"PRIx64" taos_stmt_execute return %d:%s, try:%d", info->id, code, taos_stmt_errstr(stmt), try); + } + uDebug("SML:0x%"PRIx64" taos_stmt_execute inserted %d rows", info->id, taos_stmt_affected_rows(stmt)); + + tryAgain = false; + if ((code == TSDB_CODE_TDB_INVALID_TABLE_ID + || code == TSDB_CODE_VND_INVALID_VGROUP_ID + || code == TSDB_CODE_TDB_TABLE_RECONFIGURE + || code == TSDB_CODE_APP_NOT_READY + || code == TSDB_CODE_RPC_NETWORK_UNAVAIL) && try++ < TSDB_MAX_REPLICA) { + tryAgain = true; + } + + if (code == TSDB_CODE_TDB_INVALID_TABLE_ID || code == TSDB_CODE_VND_INVALID_VGROUP_ID) { + TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); + int32_t code2 = taos_errno(res2); + if (code2 != TSDB_CODE_SUCCESS) { + uError("SML:0x%" PRIx64 " insert child table. reset query cache. error: %s", info->id, taos_errstr(res2)); + } + taos_free_result(res2); + if (tryAgain) { + taosMsleep(100 * (2 << try)); + } + } + if (code == TSDB_CODE_APP_NOT_READY || code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { + if (tryAgain) { + taosMsleep( 100 * (2 << try)); + } + } + } while (tryAgain); + + int affectedRows = taos_stmt_affected_rows(stmt); + info->affectedRows += affectedRows; + + taos_stmt_close(stmt); + return code; + + return 0; +} + +static int32_t applyChildTableDataPoints(TAOS* taos, char* cTableName, char* sTableName, SSmlSTableSchema* sTableSchema, + SArray* cTablePoints, size_t rowSize, SSmlLinesInfo* info) { + int32_t code = TSDB_CODE_SUCCESS; + size_t childTableDataPoints = taosArrayGetSize(cTablePoints); + if (childTableDataPoints < 10) { + code = applyChildTableDataPointsWithInsertSQL(taos, cTableName, sTableName, sTableSchema, cTablePoints, rowSize, info); + } else { + code = applyChildTableDataPointsWithStmt(taos, cTableName, sTableName, sTableSchema, cTablePoints, rowSize, info); + } + return code; +} + +static int32_t applyDataPoints(TAOS* taos, TAOS_SML_DATA_POINT* points, int32_t numPoints, SArray* stableSchemas, SSmlLinesInfo* info) { + int32_t code = TSDB_CODE_SUCCESS; + + SHashObj* cname2points = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); + arrangePointsByChildTableName(points, numPoints, cname2points, stableSchemas, info); + + SArray** pCTablePoints = taosHashIterate(cname2points, NULL); + while (pCTablePoints) { + SArray* cTablePoints = *pCTablePoints; + + TAOS_SML_DATA_POINT* point = taosArrayGetP(cTablePoints, 0); + SSmlSTableSchema* sTableSchema = taosArrayGet(stableSchemas, point->schemaIdx); + + size_t rowSize = 0; + size_t numCols = taosArrayGetSize(sTableSchema->fields); + for (int i = 0; i < numCols; ++i) { + SSchema* colSchema = taosArrayGet(sTableSchema->fields, i); + rowSize += colSchema->bytes; + } + + uDebug("SML:0x%"PRIx64" apply child table points. child table: %s of super table %s, row size: %zu", + info->id, point->childTableName, point->stableName, rowSize); + code = applyChildTableDataPoints(taos, point->childTableName, point->stableName, sTableSchema, cTablePoints, rowSize, info); + if (code != 0) { + uError("SML:0x%"PRIx64" Apply child table points failed. child table %s, error %s", info->id, point->childTableName, tstrerror(code)); + goto cleanup; + } + + uDebug("SML:0x%"PRIx64" successfully applied data points of child table %s", info->id, point->childTableName); + + pCTablePoints = taosHashIterate(cname2points, pCTablePoints); + } + +cleanup: + pCTablePoints = taosHashIterate(cname2points, NULL); + while (pCTablePoints) { + SArray* pPoints = *pCTablePoints; + taosArrayDestroy(&pPoints); + pCTablePoints = taosHashIterate(cname2points, pCTablePoints); + } + taosHashCleanup(cname2points); + return code; +} + +static int doSmlInsertOneDataPoint(TAOS* taos, TAOS_SML_DATA_POINT* point, SSmlLinesInfo* info) { + int32_t code = TSDB_CODE_SUCCESS; + + if (!point->childTableName) { + int tableNameLen = TSDB_TABLE_NAME_LEN; + point->childTableName = calloc(1, tableNameLen + 1); + getSmlMd5ChildTableName(point, point->childTableName, &tableNameLen, info); + point->childTableName[tableNameLen] = '\0'; + } + + STableMeta* tableMeta = NULL; + int32_t ret = getSuperTableMetaFromLocalCache(taos, point->stableName, &tableMeta, info); + if (ret != TSDB_CODE_SUCCESS) { + return ret; + } + uint8_t precision = tableMeta->tableInfo.precision; + taosMemoryFree(tableMeta); + + char* sql = taosMemoryMalloc(TSDB_MAX_SQL_LEN + 1); + int freeBytes = TSDB_MAX_SQL_LEN; + int sqlLen = 0; + sqlLen += snprintf(sql + sqlLen, freeBytes - sqlLen, "insert into %s(", point->childTableName); + for (int col = 0; col < point->fieldNum; ++col) { + TAOS_SML_KV* kv = point->fields + col; + sqlLen += snprintf(sql + sqlLen, freeBytes - sqlLen, "%s,", kv->key); + } + --sqlLen; + sqlLen += snprintf(sql + sqlLen, freeBytes - sqlLen, ") values ("); + TAOS_SML_KV* tsField = point->fields + 0; + int64_t ts = *(int64_t*)(tsField->value); + ts = convertTimePrecision(ts, TSDB_TIME_PRECISION_NANO, precision); + sqlLen += snprintf(sql + sqlLen, freeBytes - sqlLen, "%" PRId64 ",", ts); + for (int col = 1; col < point->fieldNum; ++col) { + TAOS_SML_KV* kv = point->fields + col; + int32_t len = 0; + converToStr(sql + sqlLen, kv->type, kv->value, kv->length, &len); + sqlLen += len; + sqlLen += snprintf(sql + sqlLen, freeBytes - sqlLen, ","); + } + --sqlLen; + sqlLen += snprintf(sql + sqlLen, freeBytes - sqlLen, ")"); + sql[sqlLen] = 0; + + uDebug("SML:0x%" PRIx64 " insert child table table %s of super table %s sql: %s", info->id, + point->childTableName, point->stableName, sql); + TAOS_RES* res = taos_query(taos, sql); + taosMemoryFree(sql); + code = taos_errno(res); + info->affectedRows = taos_affected_rows(res); + taos_free_result(res); + + return code; +} + +int tscSmlInsert(TAOS* taos, SSmlLinesInfo* info) { + uDebug("SML:0x%"PRIx64" taos_sml_insert. number of super tables: %d", info->id, taosHashGetSize(info->superTables)); + int32_t code = TSDB_CODE_SUCCESS; + info->affectedRows = 0; + + uDebug("SML:0x%"PRIx64" modify db schemas", info->id); + code = modifyDBSchemas(taos, info); + if (code != 0) { + uError("SML:0x%"PRIx64" error change db schema : %s", info->id, tstrerror(code)); + goto clean_up; + } + + uDebug("SML:0x%"PRIx64" apply data points", info->id); + code = applyDataPoints(taos, points, numPoint, stableSchemas, info); + if (code != 0) { + uError("SML:0x%"PRIx64" error apply data points : %s", info->id, tstrerror(code)); + } + +clean_up: + for (int i = 0; i < taosArrayGetSize(stableSchemas); ++i) { + SSmlSTableSchema* schema = taosArrayGet(stableSchemas, i); + taosArrayDestroy(&schema->fields); + taosArrayDestroy(&schema->tags); + } + taosArrayDestroy(&stableSchemas); + return code; +} + +//========================================================================= + +/* Field Escape charaters + 1: measurement Comma,Space + 2: tag_key, tag_value, field_key Comma,Equal Sign,Space + 3: field_value Double quote,Backslash +*/ +static void escapeSpecialCharacter(uint8_t field, const char **pos) { + const char *cur = *pos; + if (*cur != '\\') { + return; + } + switch (field) { + case 1: + switch (*(cur + 1)) { + case ',': + case ' ': + cur++; + break; + default: + break; + } + break; + case 2: + switch (*(cur + 1)) { + case ',': + case ' ': + case '=': + cur++; + break; + default: + break; + } + break; + case 3: + switch (*(cur + 1)) { + case '"': + case '\\': + cur++; + break; + default: + break; + } + break; + default: + break; + } + *pos = cur; +} + +char* addEscapeCharToString(char *str, int32_t len) { + if (str == NULL) { + return NULL; + } + memmove(str + 1, str, len); + str[0] = str[len + 1] = TS_BACKQUOTE_CHAR; + str[len + 2] = '\0'; + return str; +} + +bool isValidInteger(char *str) { + char *c = str; + if (*c != '+' && *c != '-' && !isdigit(*c)) { + return false; + } + c++; + while (*c != '\0') { + if (!isdigit(*c)) { + return false; + } + c++; + } + return true; +} + +bool isValidFloat(char *str) { + char *c = str; + uint8_t has_dot, has_exp, has_sign; + has_dot = 0; + has_exp = 0; + has_sign = 0; + + if (*c != '+' && *c != '-' && *c != '.' && !isdigit(*c)) { + return false; + } + if (*c == '.' && isdigit(*(c + 1))) { + has_dot = 1; + } + c++; + while (*c != '\0') { + if (!isdigit(*c)) { + switch (*c) { + case '.': { + if (!has_dot && !has_exp && isdigit(*(c + 1))) { + has_dot = 1; + } else { + return false; + } + break; + } + case 'e': + case 'E': { + if (!has_exp && isdigit(*(c - 1)) && + (isdigit(*(c + 1)) || + *(c + 1) == '+' || + *(c + 1) == '-')) { + has_exp = 1; + } else { + return false; + } + break; + } + case '+': + case '-': { + if (!has_sign && has_exp && isdigit(*(c + 1))) { + has_sign = 1; + } else { + return false; + } + break; + } + default: { + return false; + } + } + } + c++; + } //while + return true; +} + +static bool isInteger(char *pVal, uint16_t len, bool *has_sign) { + if (len <= 1) { + return false; + } + if (pVal[len - 1] == 'i') { + *has_sign = true; + return true; + } + if (pVal[len - 1] == 'u') { + *has_sign = false; + return true; + } + + return false; +} + +static bool isTinyInt(char *pVal, uint16_t len) { + if (len <= 2) { + return false; + } + if (!strcasecmp(&pVal[len - 2], "i8")) { + //printf("Type is int8(%s)\n", pVal); + return true; + } + return false; +} + +static bool isTinyUint(char *pVal, uint16_t len) { + if (len <= 2) { + return false; + } + if (pVal[0] == '-') { + return false; + } + if (!strcasecmp(&pVal[len - 2], "u8")) { + //printf("Type is uint8(%s)\n", pVal); + return true; + } + return false; +} + +static bool isSmallInt(char *pVal, uint16_t len) { + if (len <= 3) { + return false; + } + if (!strcasecmp(&pVal[len - 3], "i16")) { + //printf("Type is int16(%s)\n", pVal); + return true; + } + return false; +} + +static bool isSmallUint(char *pVal, uint16_t len) { + if (len <= 3) { + return false; + } + if (pVal[0] == '-') { + return false; + } + if (strcasecmp(&pVal[len - 3], "u16") == 0) { + //printf("Type is uint16(%s)\n", pVal); + return true; + } + return false; +} + +static bool isInt(char *pVal, uint16_t len) { + if (len <= 3) { + return false; + } + if (strcasecmp(&pVal[len - 3], "i32") == 0) { + //printf("Type is int32(%s)\n", pVal); + return true; + } + return false; +} + +static bool isUint(char *pVal, uint16_t len) { + if (len <= 3) { + return false; + } + if (pVal[0] == '-') { + return false; + } + if (strcasecmp(&pVal[len - 3], "u32") == 0) { + //printf("Type is uint32(%s)\n", pVal); + return true; + } + return false; +} + +static bool isBigInt(char *pVal, uint16_t len) { + if (len <= 3) { + return false; + } + if (strcasecmp(&pVal[len - 3], "i64") == 0) { + //printf("Type is int64(%s)\n", pVal); + return true; + } + return false; +} + +static bool isBigUint(char *pVal, uint16_t len) { + if (len <= 3) { + return false; + } + if (pVal[0] == '-') { + return false; + } + if (strcasecmp(&pVal[len - 3], "u64") == 0) { + //printf("Type is uint64(%s)\n", pVal); + return true; + } + return false; +} + +static bool isFloat(char *pVal, uint16_t len) { + if (len <= 3) { + return false; + } + if (strcasecmp(&pVal[len - 3], "f32") == 0) { + //printf("Type is float(%s)\n", pVal); + return true; + } + return false; +} + +static bool isDouble(char *pVal, uint16_t len) { + if (len <= 3) { + return false; + } + if (strcasecmp(&pVal[len - 3], "f64") == 0) { + //printf("Type is double(%s)\n", pVal); + return true; + } + return false; +} + +static bool isBool(char *pVal, uint16_t len, bool *bVal) { + if ((len == 1) && !strcasecmp(&pVal[len - 1], "t")) { + //printf("Type is bool(%c)\n", pVal[len - 1]); + *bVal = true; + return true; + } + + if ((len == 1) && !strcasecmp(&pVal[len - 1], "f")) { + //printf("Type is bool(%c)\n", pVal[len - 1]); + *bVal = false; + return true; + } + + if((len == 4) && !strcasecmp(&pVal[len - 4], "true")) { + //printf("Type is bool(%s)\n", &pVal[len - 4]); + *bVal = true; + return true; + } + if((len == 5) && !strcasecmp(&pVal[len - 5], "false")) { + //printf("Type is bool(%s)\n", &pVal[len - 5]); + *bVal = false; + return true; + } + return false; +} + +static bool isBinary(char *pVal, uint16_t len) { + //binary: "abc" + if (len < 2) { + return false; + } + //binary + if (pVal[0] == '"' && pVal[len - 1] == '"') { + //printf("Type is binary(%s)\n", pVal); + return true; + } + return false; +} + +static bool isNchar(char *pVal, uint16_t len) { + //nchar: L"abc" + if (len < 3) { + return false; + } + if ((pVal[0] == 'l' || pVal[0] == 'L')&& pVal[1] == '"' && pVal[len - 1] == '"') { + //printf("Type is nchar(%s)\n", pVal); + return true; + } + return false; +} + +static int32_t isTimeStamp(char *pVal, uint16_t len, SMLTimeStampType *tsType, SSmlLinesInfo* info) { + if (len == 0) { + return TSDB_CODE_SUCCESS; + } + if ((len == 1) && pVal[0] == '0') { + *tsType = SML_TIME_STAMP_NOW; + return TSDB_CODE_SUCCESS; + } + + for (int i = 0; i < len; ++i) { + if(!isdigit(pVal[i])) { + return TSDB_CODE_TSC_INVALID_TIME_STAMP; + } + } + + /* For InfluxDB line protocol use user passed timestamp precision + * For OpenTSDB protocols only 10 digit(seconds) or 13 digits(milliseconds) + * precision allowed + */ + if (info->protocol == TSDB_SML_LINE_PROTOCOL) { + if (info->tsType != SML_TIME_STAMP_NOT_CONFIGURED) { + *tsType = info->tsType; + } else { + *tsType = SML_TIME_STAMP_NANO_SECONDS; + } + } else if (info->protocol == TSDB_SML_TELNET_PROTOCOL) { + if (len == SML_TIMESTAMP_SECOND_DIGITS) { + *tsType = SML_TIME_STAMP_SECONDS; + } else if (len == SML_TIMESTAMP_MILLI_SECOND_DIGITS) { + *tsType = SML_TIME_STAMP_MILLI_SECONDS; + } else { + return TSDB_CODE_TSC_INVALID_TIME_STAMP; + } + } + return TSDB_CODE_SUCCESS; + + //if (pVal[len - 1] == 's') { + // switch (pVal[len - 2]) { + // case 'm': + // *tsType = SML_TIME_STAMP_MILLI_SECONDS; + // break; + // case 'u': + // *tsType = SML_TIME_STAMP_MICRO_SECONDS; + // break; + // case 'n': + // *tsType = SML_TIME_STAMP_NANO_SECONDS; + // break; + // default: + // if (isdigit(pVal[len - 2])) { + // *tsType = SML_TIME_STAMP_SECONDS; + // break; + // } else { + // return false; + // } + // } + // //printf("Type is timestamp(%s)\n", pVal); + // return true; + //} + //return false; +} + +static bool convertStrToNumber(TAOS_SML_KV *pVal, char *str, SSmlLinesInfo* info) { + errno = 0; + uint8_t type = pVal->type; + int16_t length = pVal->length; + int64_t val_s = 0; + uint64_t val_u = 0; + double val_d = 0.0; + + strntolower_s(str, str, (int32_t)strlen(str)); + if (IS_FLOAT_TYPE(type)) { + val_d = strtod(str, NULL); + } else { + if (IS_SIGNED_NUMERIC_TYPE(type)) { + val_s = strtoll(str, NULL, 10); + } else { + val_u = strtoull(str, NULL, 10); + } + } + + if (errno == ERANGE) { + uError("SML:0x%"PRIx64" Convert number(%s) out of range", info->id, str); + return false; + } + + switch (type) { + case TSDB_DATA_TYPE_TINYINT: + if (!IS_VALID_TINYINT(val_s)) { + return false; + } + pVal->value = calloc(length, 1); + *(int8_t *)(pVal->value) = (int8_t)val_s; + break; + case TSDB_DATA_TYPE_UTINYINT: + if (!IS_VALID_UTINYINT(val_u)) { + return false; + } + pVal->value = calloc(length, 1); + *(uint8_t *)(pVal->value) = (uint8_t)val_u; + break; + case TSDB_DATA_TYPE_SMALLINT: + if (!IS_VALID_SMALLINT(val_s)) { + return false; + } + pVal->value = calloc(length, 1); + *(int16_t *)(pVal->value) = (int16_t)val_s; + break; + case TSDB_DATA_TYPE_USMALLINT: + if (!IS_VALID_USMALLINT(val_u)) { + return false; + } + pVal->value = calloc(length, 1); + *(uint16_t *)(pVal->value) = (uint16_t)val_u; + break; + case TSDB_DATA_TYPE_INT: + if (!IS_VALID_INT(val_s)) { + return false; + } + pVal->value = calloc(length, 1); + *(int32_t *)(pVal->value) = (int32_t)val_s; + break; + case TSDB_DATA_TYPE_UINT: + if (!IS_VALID_UINT(val_u)) { + return false; + } + pVal->value = calloc(length, 1); + *(uint32_t *)(pVal->value) = (uint32_t)val_u; + break; + case TSDB_DATA_TYPE_BIGINT: + if (!IS_VALID_BIGINT(val_s)) { + return false; + } + pVal->value = calloc(length, 1); + *(int64_t *)(pVal->value) = (int64_t)val_s; + break; + case TSDB_DATA_TYPE_UBIGINT: + if (!IS_VALID_UBIGINT(val_u)) { + return false; + } + pVal->value = calloc(length, 1); + *(uint64_t *)(pVal->value) = (uint64_t)val_u; + break; + case TSDB_DATA_TYPE_FLOAT: + if (!IS_VALID_FLOAT(val_d)) { + return false; + } + pVal->value = calloc(length, 1); + *(float *)(pVal->value) = (float)val_d; + break; + case TSDB_DATA_TYPE_DOUBLE: + if (!IS_VALID_DOUBLE(val_d)) { + return false; + } + pVal->value = calloc(length, 1); + *(double *)(pVal->value) = (double)val_d; + break; + default: + return false; + } + return true; +} +//len does not include '\0' from value. +bool convertSmlValueType(TAOS_SML_KV *pVal, char *value, + uint16_t len, SSmlLinesInfo* info, bool isTag) { + if (len <= 0) { + return false; + } + + //convert tags value to Nchar + if (isTag) { + pVal->type = TSDB_DATA_TYPE_NCHAR; + pVal->length = len; + pVal->value = calloc(pVal->length, 1); + memcpy(pVal->value, value, pVal->length); + return true; + } + + //integer number + bool has_sign; + if (isInteger(value, len, &has_sign)) { + pVal->type = has_sign ? TSDB_DATA_TYPE_BIGINT : TSDB_DATA_TYPE_UBIGINT; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + value[len - 1] = '\0'; + if (!isValidInteger(value) || !convertStrToNumber(pVal, value, info)) { + return false; + } + return true; + } + if (isTinyInt(value, len)) { + pVal->type = TSDB_DATA_TYPE_TINYINT; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + value[len - 2] = '\0'; + if (!isValidInteger(value) || !convertStrToNumber(pVal, value, info)) { + return false; + } + return true; + } + if (isTinyUint(value, len)) { + pVal->type = TSDB_DATA_TYPE_UTINYINT; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + value[len - 2] = '\0'; + if (!isValidInteger(value) || !convertStrToNumber(pVal, value, info)) { + return false; + } + return true; + } + if (isSmallInt(value, len)) { + pVal->type = TSDB_DATA_TYPE_SMALLINT; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + value[len - 3] = '\0'; + if (!isValidInteger(value) || !convertStrToNumber(pVal, value, info)) { + return false; + } + return true; + } + if (isSmallUint(value, len)) { + pVal->type = TSDB_DATA_TYPE_USMALLINT; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + value[len - 3] = '\0'; + if (!isValidInteger(value) || !convertStrToNumber(pVal, value, info)) { + return false; + } + return true; + } + if (isInt(value, len)) { + pVal->type = TSDB_DATA_TYPE_INT; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + value[len - 3] = '\0'; + if (!isValidInteger(value) || !convertStrToNumber(pVal, value, info)) { + return false; + } + return true; + } + if (isUint(value, len)) { + pVal->type = TSDB_DATA_TYPE_UINT; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + value[len - 3] = '\0'; + if (!isValidInteger(value) || !convertStrToNumber(pVal, value, info)) { + return false; + } + return true; + } + if (isBigInt(value, len)) { + pVal->type = TSDB_DATA_TYPE_BIGINT; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + value[len - 3] = '\0'; + if (!isValidInteger(value) || !convertStrToNumber(pVal, value, info)) { + return false; + } + return true; + } + if (isBigUint(value, len)) { + pVal->type = TSDB_DATA_TYPE_UBIGINT; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + value[len - 3] = '\0'; + if (!isValidInteger(value) || !convertStrToNumber(pVal, value, info)) { + return false; + } + return true; + } + //floating number + if (isFloat(value, len)) { + pVal->type = TSDB_DATA_TYPE_FLOAT; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + value[len - 3] = '\0'; + if (!isValidFloat(value) || !convertStrToNumber(pVal, value, info)) { + return false; + } + return true; + } + if (isDouble(value, len)) { + pVal->type = TSDB_DATA_TYPE_DOUBLE; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + value[len - 3] = '\0'; + if (!isValidFloat(value) || !convertStrToNumber(pVal, value, info)) { + return false; + } + return true; + } + //binary + if (isBinary(value, len)) { + pVal->type = TSDB_DATA_TYPE_BINARY; + pVal->length = len - 2; + pVal->value = calloc(pVal->length, 1); + //copy after " + memcpy(pVal->value, value + 1, pVal->length); + return true; + } + //nchar + if (isNchar(value, len)) { + pVal->type = TSDB_DATA_TYPE_NCHAR; + pVal->length = len - 3; + pVal->value = calloc(pVal->length, 1); + //copy after L" + memcpy(pVal->value, value + 2, pVal->length); + return true; + } + //bool + bool bVal; + if (isBool(value, len, &bVal)) { + pVal->type = TSDB_DATA_TYPE_BOOL; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + pVal->value = calloc(pVal->length, 1); + memcpy(pVal->value, &bVal, pVal->length); + return true; + } + + //Handle default(no appendix) type as DOUBLE + if (isValidInteger(value) || isValidFloat(value)) { + pVal->type = TSDB_DATA_TYPE_DOUBLE; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + if (!convertStrToNumber(pVal, value, info)) { + return false; + } + return true; + } + return false; +} + +static int32_t getTimeStampValue(char *value, uint16_t len, + SMLTimeStampType type, int64_t *ts, SSmlLinesInfo* info) { + + //No appendix or no timestamp given (len = 0) + if (len != 0 && type != SML_TIME_STAMP_NOW) { + *ts = (int64_t)strtoll(value, NULL, 10); + } else { + type = SML_TIME_STAMP_NOW; + } + switch (type) { + case SML_TIME_STAMP_NOW: { + *ts = taosGetTimestampNs(); + break; + } + case SML_TIME_STAMP_HOURS: { + *ts = (int64_t)(*ts * 3600 * 1e9); + break; + } + case SML_TIME_STAMP_MINUTES: { + *ts = (int64_t)(*ts * 60 * 1e9); + break; + } + case SML_TIME_STAMP_SECONDS: { + *ts = (int64_t)(*ts * 1e9); + break; + } + case SML_TIME_STAMP_MILLI_SECONDS: { + *ts = convertTimePrecision(*ts, TSDB_TIME_PRECISION_MILLI, TSDB_TIME_PRECISION_NANO); + break; + } + case SML_TIME_STAMP_MICRO_SECONDS: { + *ts = convertTimePrecision(*ts, TSDB_TIME_PRECISION_MICRO, TSDB_TIME_PRECISION_NANO); + break; + } + case SML_TIME_STAMP_NANO_SECONDS: { + *ts = *ts * 1; + break; + } + default: { + return TSDB_CODE_TSC_INVALID_TIME_STAMP; + } + } + return TSDB_CODE_SUCCESS; +} + +int32_t convertSmlTimeStamp(TAOS_SML_KV *pVal, char *value, + uint16_t len, SSmlLinesInfo* info) { + int32_t ret; + SMLTimeStampType type = SML_TIME_STAMP_NOW; + int64_t tsVal; + + ret = isTimeStamp(value, len, &type, info); + if (ret != TSDB_CODE_SUCCESS) { + return ret; + } + + ret = getTimeStampValue(value, len, type, &tsVal, info); + if (ret != TSDB_CODE_SUCCESS) { + return ret; + } + uDebug("SML:0x%"PRIx64"Timestamp after conversion:%"PRId64, info->id, tsVal); + + pVal->type = TSDB_DATA_TYPE_TIMESTAMP; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + pVal->value = calloc(pVal->length, 1); + memcpy(pVal->value, &tsVal, pVal->length); + return TSDB_CODE_SUCCESS; +} + +static int32_t parseSmlTimeStamp(TAOS_SML_KV **pTS, const char **index, SSmlLinesInfo* info) { + const char *start, *cur; + int32_t ret = TSDB_CODE_SUCCESS; + int len = 0; + char key[] = "ts"; + char *value = NULL; + + start = cur = *index; + *pTS = calloc(1, sizeof(TAOS_SML_KV)); + + while(*cur != '\0') { + cur++; + len++; + } + + if (len > 0) { + value = calloc(len + 1, 1); + memcpy(value, start, len); + } + + ret = convertSmlTimeStamp(*pTS, value, len, info); + if (ret) { + taosMemoryFree(value); + taosMemoryFree(*pTS); + return ret; + } + taosMemoryFree(value); + + (*pTS)->key = calloc(sizeof(key), 1); + memcpy((*pTS)->key, key, sizeof(key)); + return ret; +} + +bool checkDuplicateKey(char *key, SHashObj *pHash, SSmlLinesInfo* info) { + char *val = NULL; + val = taosHashGet(pHash, key, strlen(key)); + if (val) { + uError("SML:0x%"PRIx64" Duplicate key detected:%s", info->id, key); + return true; + } + + uint8_t dummy_val = 0; + taosHashPut(pHash, key, strlen(key), &dummy_val, sizeof(uint8_t)); + + return false; +} + +static int32_t parseSmlKey(TAOS_SML_KV *pKV, const char **index, SHashObj *pHash, SSmlLinesInfo* info) { + const char *cur = *index; + char key[TSDB_COL_NAME_LEN + 1]; // +1 to avoid key[len] over write + int16_t len = 0; + + while (*cur != '\0') { + if (len > TSDB_COL_NAME_LEN - 1) { + uError("SML:0x%"PRIx64" Key field cannot exceeds %d characters", info->id, TSDB_COL_NAME_LEN - 1); + return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; + } + //unescaped '=' identifies a tag key + if (*cur == '=' && *(cur - 1) != '\\') { + break; + } + //Escape special character + if (*cur == '\\') { + escapeSpecialCharacter(2, &cur); + } + key[len] = *cur; + cur++; + len++; + } + if (len == 0) { + return TSDB_CODE_TSC_LINE_SYNTAX_ERROR; + } + key[len] = '\0'; + + if (checkDuplicateKey(key, pHash, info)) { + return TSDB_CODE_TSC_LINE_SYNTAX_ERROR; + } + + pKV->key = calloc(len + TS_BACKQUOTE_CHAR_SIZE + 1, 1); + memcpy(pKV->key, key, len + 1); + addEscapeCharToString(pKV->key, len); + uDebug("SML:0x%"PRIx64" Key:%s|len:%d", info->id, pKV->key, len); + *index = cur + 1; + return TSDB_CODE_SUCCESS; +} + + +static int32_t parseSmlValue(TAOS_SML_KV *pKV, const char **index, + bool *is_last_kv, SSmlLinesInfo* info, bool isTag) { + const char *start, *cur; + int32_t ret = TSDB_CODE_SUCCESS; + char *value = NULL; + int16_t len = 0; + + bool kv_done = false; + bool back_slash = false; + bool double_quote = false; + size_t line_len = 0; + + enum { + tag_common, + tag_lqoute, + tag_rqoute + } tag_state; + + enum { + val_common, + val_lqoute, + val_rqoute + } val_state; + + start = cur = *index; + tag_state = tag_common; + val_state = val_common; + + while (1) { + if (isTag) { + /* ',', '=' and spaces MUST be escaped */ + switch (tag_state) { + case tag_common: + if (back_slash == true) { + if (*cur != ',' && *cur != '=' && *cur != ' ') { + uError("SML:0x%"PRIx64" tag value: state(%d), incorrect character(%c) escaped", info->id, tag_state, *cur); + ret = TSDB_CODE_TSC_LINE_SYNTAX_ERROR; + goto error; + } + + back_slash = false; + cur++; + len++; + break; + } + + if (*cur == '"') { + if (cur == *index) { + tag_state = tag_lqoute; + } + cur += 1; + len += 1; + break; + } else if (*cur == 'L') { + line_len = strlen(*index); + + /* common character at the end */ + if (cur + 1 >= *index + line_len) { + *is_last_kv = true; + kv_done = true; + break; + } + + if (*(cur + 1) == '"') { + /* string starts here */ + if (cur + 1 == *index + 1) { + tag_state = tag_lqoute; + } + cur += 2; + len += 2; + break; + } + } + + switch (*cur) { + case '\\': + back_slash = true; + cur++; + len++; + break; + case ',': + kv_done = true; + break; + + case ' ': + /* fall through */ + case '\0': + *is_last_kv = true; + kv_done = true; + break; + + default: + cur++; + len++; + } + + break; + case tag_lqoute: + if (back_slash == true) { + if (*cur != ',' && *cur != '=' && *cur != ' ') { + uError("SML:0x%"PRIx64" tag value: state(%d), incorrect character(%c) escaped", info->id, tag_state, *cur); + ret = TSDB_CODE_TSC_LINE_SYNTAX_ERROR; + goto error; + } + + back_slash = false; + cur++; + len++; + break; + } else if (double_quote == true) { + if (*cur != ' ' && *cur != ',' && *cur != '\0') { + uError("SML:0x%"PRIx64" tag value: state(%d), incorrect character(%c) behind closing \"", info->id, tag_state, *cur); + ret = TSDB_CODE_TSC_LINE_SYNTAX_ERROR; + goto error; + } + + if (*cur == ' ' || *cur == '\0') { + *is_last_kv = true; + } + + double_quote = false; + tag_state = tag_rqoute; + break; + } + + switch (*cur) { + case '\\': + back_slash = true; + cur++; + len++; + break; + + case '"': + double_quote = true; + cur++; + len++; + break; + + case ',': + /* fall through */ + case '=': + /* fall through */ + case ' ': + if (*(cur - 1) != '\\') { + uError("SML:0x%"PRIx64" tag value: state(%d), character(%c) not escaped", info->id, tag_state, *cur); + ret = TSDB_CODE_TSC_LINE_SYNTAX_ERROR; + kv_done = true; + } + break; + + case '\0': + uError("SML:0x%"PRIx64" tag value: state(%d), closing \" not found", info->id, tag_state); + ret = TSDB_CODE_TSC_LINE_SYNTAX_ERROR; + kv_done = true; + break; + + default: + cur++; + len++; + } + + break; + + default: + kv_done = true; + } + } else { + switch (val_state) { + case val_common: + if (back_slash == true) { + if (*cur != '\\' && *cur != '"') { + uError("SML:0x%"PRIx64" field value: state(%d), incorrect character(%c) escaped", info->id, val_state, *cur); + ret = TSDB_CODE_TSC_LINE_SYNTAX_ERROR; + goto error; + } + + back_slash = false; + cur++; + len++; + break; + } + + if (*cur == '"') { + if (cur == *index) { + val_state = val_lqoute; + } else { + if (*(cur - 1) != '\\') { + uError("SML:0x%"PRIx64" field value: state(%d), \" not escaped", info->id, val_state); + ret = TSDB_CODE_TSC_LINE_SYNTAX_ERROR; + goto error; + } + } + + cur += 1; + len += 1; + break; + } else if (*cur == 'L') { + line_len = strlen(*index); + + /* common character at the end */ + if (cur + 1 >= *index + line_len) { + *is_last_kv = true; + kv_done = true; + break; + } + + if (*(cur + 1) == '"') { + /* string starts here */ + if (cur + 1 == *index + 1) { + val_state = val_lqoute; + cur += 2; + len += 2; + } else { + /* MUST at the end of string */ + if (cur + 2 >= *index + line_len) { + cur += 2; + len += 2; + *is_last_kv = true; + kv_done = true; + } else { + if (*(cur + 2) != ' ' && *(cur + 2) != ',') { + uError("SML:0x%"PRIx64" field value: state(%d), not closing character(L\")", info->id, val_state); + ret = TSDB_CODE_TSC_LINE_SYNTAX_ERROR; + goto error; + } else { + if (*(cur + 2) == ' ') { + *is_last_kv = true; + } + + cur += 2; + len += 2; + kv_done = true; + } + } + } + break; + } + } + + switch (*cur) { + case '\\': + back_slash = true; + cur++; + len++; + break; + + case ',': + kv_done = true; + break; + + case ' ': + /* fall through */ + case '\0': + *is_last_kv = true; + kv_done = true; + break; + + default: + cur++; + len++; + } + + break; + case val_lqoute: + if (back_slash == true) { + if (*cur != '\\' && *cur != '"') { + uError("SML:0x%"PRIx64" field value: state(%d), incorrect character(%c) escaped", info->id, val_state, *cur); + ret = TSDB_CODE_TSC_LINE_SYNTAX_ERROR; + goto error; + } + + back_slash = false; + cur++; + len++; + break; + } else if (double_quote == true) { + if (*cur != ' ' && *cur != ',' && *cur != '\0') { + uError("SML:0x%"PRIx64" field value: state(%d), incorrect character(%c) behind closing \"", info->id, val_state, *cur); + ret = TSDB_CODE_TSC_LINE_SYNTAX_ERROR; + goto error; + } + + if (*cur == ' ' || *cur == '\0') { + *is_last_kv = true; + } + + double_quote = false; + val_state = val_rqoute; + break; + } + + switch (*cur) { + case '\\': + back_slash = true; + cur++; + len++; + break; + + case '"': + double_quote = true; + cur++; + len++; + break; + + case '\0': + uError("SML:0x%"PRIx64" field value: state(%d), closing \" not found", info->id, val_state); + ret = TSDB_CODE_TSC_LINE_SYNTAX_ERROR; + kv_done = true; + break; + + default: + cur++; + len++; + } + + break; + default: + kv_done = true; + } + } + + if (kv_done == true) { + break; + } + } + + if (len == 0 || ret != TSDB_CODE_SUCCESS) { + taosMemoryFree(pKV->key); + pKV->key = NULL; + return TSDB_CODE_TSC_LINE_SYNTAX_ERROR; + } + + value = calloc(len + 1, 1); + memcpy(value, start, len); + value[len] = '\0'; + if (!convertSmlValueType(pKV, value, len, info, isTag)) { + uError("SML:0x%"PRIx64" Failed to convert sml value string(%s) to any type", + info->id, value); + taosMemoryFree(value); + ret = TSDB_CODE_TSC_INVALID_VALUE; + goto error; + } + taosMemoryFree(value); + + *index = (*cur == '\0') ? cur : cur + 1; + return ret; + +error: + //taosMemoryFree previous alocated key field + taosMemoryFree(pKV->key); + pKV->key = NULL; + return ret; +} + +/* Field Escape charaters + 1: measurement Comma,Space + 2: tag_key, tag_value, field_key Comma,Equal Sign,Space + 3: field_value Double quote,Backslash +*/ + +static int32_t parseSmlMeasurement(TAOS_SML_DATA_POINT *pSml, const char **index, + uint8_t *has_tags, SSmlLinesInfo* info) { + const char *cur = *index; + int16_t len = 0; + + pSml->stableName = calloc(TSDB_TABLE_NAME_LEN + TS_BACKQUOTE_CHAR_SIZE, 1); + if (pSml->stableName == NULL){ + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + while (*cur != '\0') { + if (len > TSDB_TABLE_NAME_LEN - 1) { + uError("SML:0x%"PRIx64" Measurement field cannot exceeds %d characters", info->id, TSDB_TABLE_NAME_LEN - 1); + taosMemoryFree(pSml->stableName); + pSml->stableName = NULL; + return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; + } + //first unescaped comma or space identifies measurement + //if space detected first, meaning no tag in the input + if (*cur == ',' && *(cur - 1) != '\\') { + *has_tags = 1; + break; + } + if (*cur == ' ' && *(cur - 1) != '\\') { + if (*(cur + 1) != ' ') { + break; + } + else { + cur++; + continue; + } + } + //Comma, Space, Backslash needs to be escaped if any + if (*cur == '\\') { + escapeSpecialCharacter(1, &cur); + } + pSml->stableName[len] = *cur; + cur++; + len++; + } + if (len == 0) { + taosMemoryFree(pSml->stableName); + pSml->stableName = NULL; + return TSDB_CODE_TSC_LINE_SYNTAX_ERROR; + } + addEscapeCharToString(pSml->stableName, len); + *index = cur + 1; + uDebug("SML:0x%"PRIx64" Stable name in measurement:%s|len:%d", info->id, pSml->stableName, len); + + return TSDB_CODE_SUCCESS; +} + +//Table name can only contain digits(0-9),alphebet(a-z),underscore(_) +int32_t isValidChildTableName(const char *pTbName, int16_t len, SSmlLinesInfo* info) { + if (len > TSDB_TABLE_NAME_LEN - 1) { + uError("SML:0x%"PRIx64" child table name cannot exceeds %d characters", info->id, TSDB_TABLE_NAME_LEN - 1); + return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; + } + const char *cur = pTbName; + for (int i = 0; i < len; ++i) { + if(!isdigit(cur[i]) && !isalpha(cur[i]) && (cur[i] != '_')) { + return TSDB_CODE_TSC_LINE_SYNTAX_ERROR; + } + } + return TSDB_CODE_SUCCESS; +} + + +static int32_t parseSmlKvPairs(TAOS_SML_KV **pKVs, int *num_kvs, + const char **index, bool isField, + TAOS_SML_DATA_POINT* smlData, SHashObj *pHash, + SSmlLinesInfo* info) { + const char *cur = *index; + int32_t ret = TSDB_CODE_SUCCESS; + TAOS_SML_KV *pkv; + bool is_last_kv = false; + + int32_t capacity = 0; + if (isField) { + capacity = 64; + *pKVs = calloc(capacity, sizeof(TAOS_SML_KV)); + // leave space for timestamp; + pkv = *pKVs; + pkv++; + } else { + capacity = 8; + *pKVs = calloc(capacity, sizeof(TAOS_SML_KV)); + pkv = *pKVs; + } + + size_t childTableNameLen = strlen(tsSmlChildTableName); + char childTableName[TSDB_TABLE_NAME_LEN + TS_BACKQUOTE_CHAR_SIZE] = {0}; + if (childTableNameLen != 0) { + memcpy(childTableName, tsSmlChildTableName, childTableNameLen); + addEscapeCharToString(childTableName, (int32_t)(childTableNameLen)); + } + + while (*cur != '\0') { + ret = parseSmlKey(pkv, &cur, pHash, info); + if (ret) { + uError("SML:0x%"PRIx64" Unable to parse key", info->id); + goto error; + } + ret = parseSmlValue(pkv, &cur, &is_last_kv, info, !isField); + if (ret) { + uError("SML:0x%"PRIx64" Unable to parse value", info->id); + goto error; + } + + if (!isField && childTableNameLen != 0 && strcasecmp(pkv->key, childTableName) == 0) { + smlData->childTableName = taosMemoryMalloc(pkv->length + TS_BACKQUOTE_CHAR_SIZE + 1); + memcpy(smlData->childTableName, pkv->value, pkv->length); + addEscapeCharToString(smlData->childTableName, (int32_t)pkv->length); + taosMemoryFree(pkv->key); + taosMemoryFree(pkv->value); + } else { + *num_kvs += 1; + } + if (is_last_kv) { + goto done; + } + + //reallocate addtional memory for more kvs + TAOS_SML_KV *more_kvs = NULL; + + if (isField) { + if ((*num_kvs + 2) > capacity) { + capacity *= 3; capacity /= 2; + more_kvs = realloc(*pKVs, capacity * sizeof(TAOS_SML_KV)); + } else { + more_kvs = *pKVs; + } + } else { + if ((*num_kvs + 1) > capacity) { + capacity *= 3; capacity /= 2; + more_kvs = realloc(*pKVs, capacity * sizeof(TAOS_SML_KV)); + } else { + more_kvs = *pKVs; + } + } + + if (!more_kvs) { + goto error; + } + *pKVs = more_kvs; + //move pKV points to next TAOS_SML_KV block + if (isField) { + pkv = *pKVs + *num_kvs + 1; + } else { + pkv = *pKVs + *num_kvs; + } + } + goto done; + +error: + return ret; +done: + *index = cur; + return ret; +} + +static void moveTimeStampToFirstKv(TAOS_SML_DATA_POINT** smlData, TAOS_SML_KV *ts) { + TAOS_SML_KV* tsField = (*smlData)->fields; + tsField->length = ts->length; + tsField->type = ts->type; + tsField->value = taosMemoryMalloc(ts->length); + tsField->key = taosMemoryMalloc(strlen(ts->key) + 1); + memcpy(tsField->key, ts->key, strlen(ts->key) + 1); + memcpy(tsField->value, ts->value, ts->length); + (*smlData)->fieldNum = (*smlData)->fieldNum + 1; + + taosMemoryFree(ts->key); + taosMemoryFree(ts->value); + taosMemoryFree(ts); +} + +/* Field Escape charaters + 1: measurement Comma,Space + 2: tag_key, tag_value, field_key Comma,Equal Sign,Space + 3: field_value Double quote,Backslash +*/ + +//void findSpace(const char** sql, const char **tags, int32_t *tagLen){ +// const char *cur = *sql; +// *tagLen = 0; +// *tags = NULL; +// if(!cur) return; +// while (*cur != '\0') { // jump the space at the begining +// if(*cur != SPACE) { +// *tags = cur; +// break; +// } +// cur++; +// } +// +// while (*cur != '\0') { // find the first space +// if (*cur == SPACE && *(cur - 1) != SLASH) { +// *tagLen = cur - *tags; +// break; +// } +// +// cur++; +// } +// *sql = cur; +// return; +//} + + +int32_t parseSml(const char* sql, TAOS_PARSE_ELEMENTS *elements){ + if(!sql) return TSDB_CODE_SML_INVALID_DATA; + while (*sql != '\0') { // jump the space at the begining + if(*sql != SPACE) { + elements->measure = sql; + break; + } + sql++; + } + if (!elements->measure || *sql == COMMA) return TSDB_CODE_SML_INVALID_DATA; + + // parse measure and tag + while (*sql != '\0') { + if (elements->measureLen == 0 && *sql == COMMA && *(sql - 1) != SLASH) { // find the first comma + elements->measureLen = sql - elements->measure; + sql++; + elements->tags = sql; + continue; + } + + if (*sql == SPACE && *(sql - 1) != SLASH) { // find the first space + if (elements->measureLen == 0) { + elements->measureLen = sql - elements->measure; + elements->tags = sql; + } + elements->tagsLen = sql - elements->tags; + elements->measureTagsLen = sql - elements->measure; + break; + } + + sql++; + } + if(elements->measureLen == 0) return TSDB_CODE_SML_INVALID_DATA; + + // parse cols + while (*sql != '\0') { + if(*sql != SPACE) { + elements->cols = sql; + break; + } + sql++; + } + if(!elements->cols) return TSDB_CODE_SML_INVALID_DATA; + + while (*sql != '\0') { + if(*sql == SPACE && *(sql - 1) != SLASH) { + elements->colsLen = sql - elements->cols; + break; + } + sql++; + } + if(elements->colsLen == 0) return TSDB_CODE_SML_INVALID_DATA; + + // parse ts + while (*sql != '\0') { + if(*sql != SPACE) { + elements->timestamp = sql; + break; + } + sql++; + } + if(!elements->timestamp) return TSDB_CODE_SML_INVALID_DATA; + + return TSDB_CODE_SUCCESS; +} + +int32_t parseSmlKV(const char* data, int32_t len, SArray *tags){ + for(int i = 0; i < len; i++){ + const char *key = data + i; + int32_t keyLen = 0; + while(i < len){ + if(data[i] == EQUAL && i > 0 && data[i-1] != SLASH){ + keyLen = data + i - key; + break; + } + i++; + } + if(keyLen == 0){ + return TSDB_CODE_SML_INVALID_DATA; + } + + i++; + const char *value = data + i; + int32_t valueLen = 0; + while(i < len){ + if(data[i] == COMMA && i > 0 && data[i-1] != SLASH){ + valueLen = data + i - value; + break; + } + i++; + } + if(valueLen == 0){ + return TSDB_CODE_SML_INVALID_DATA; + } + TAOS_SML_KV *kv = taosMemoryCalloc(sizeof(TAOS_SML_KV), 1); + kv->key = key; + kv->keyLen = keyLen; + kv->value = value; + kv->valueLen = valueLen; + kv->type = TSDB_DATA_TYPE_NCHAR; + if(tags) taosArrayPush(tags, &kv); + } + return TSDB_CODE_SUCCESS; +} + +int32_t parseSmlTS(const char* data, int32_t len, SArray *tags){ + TAOS_SML_KV *kv = taosMemoryCalloc(sizeof(TAOS_SML_KV), 1); + kv->value = data; + kv->valueLen = len; + kv->type = TSDB_DATA_TYPE_TIMESTAMP; + if(tags) taosArrayPush(tags, &kv); + return TSDB_CODE_SUCCESS; +} + +//int32_t parseSmlCols(const char* data, SArray *cols){ +// while(*data != '\0'){ +// if(*data == EQUAL) return TSDB_CODE_SML_INVALID_DATA; +// const char *key = data; +// int32_t keyLen = 0; +// while(*data != '\0'){ +// if(*data == EQUAL && *(data-1) != SLASH){ +// keyLen = data - key; +// data ++; +// break; +// } +// data++; +// } +// if(keyLen == 0){ +// return TSDB_CODE_SML_INVALID_DATA; +// } +// +// if(*data == COMMA) return TSDB_CODE_SML_INVALID_DATA; +// const char *value = data; +// int32_t valueLen = 0; +// while(*data != '\0'){ +// if(*data == COMMA && *(data-1) != SLASH){ +// valueLen = data - value; +// data ++; +// break; +// } +// data++; +// } +// if(valueLen == 0){ +// return TSDB_CODE_SML_INVALID_DATA; +// } +// +// TAOS_SML_KV *kv = taosMemoryCalloc(sizeof(TAOS_SML_KV), 1); +// kv->key = key; +// kv->keyLen = keyLen; +// kv->value = value; +// kv->valueLen = valueLen; +// kv->type = TSDB_DATA_TYPE_NCHAR; +// if(cols) taosArrayPush(cols, &kv); +// } +// return TSDB_CODE_SUCCESS; +//} + +void updateMeta(SSmlSTableMeta* tableMeta, SArray *tags, SArray *cols){ + if(tags){ + for (int i = 0; i < taosArrayGetSize(tags); ++i) { + TAOS_SML_KV *kv = taosArrayGetP(tags, i); + TAOS_SML_KV **value = taosHashGet(tableMeta->tagHash, kv->key, kv->keyLen); + if(value){ + if(kv->type != (*value)->type){ + // todo + } + }else{ + taosHashPut(tableMeta->tagHash, kv->key, kv->keyLen, &kv, POINTER_BYTES); + } + + } + } + + if(cols){ + for (int i = 1; i < taosArrayGetSize(cols); ++i) { //jump timestamp + TAOS_SML_KV *kv = taosArrayGetP(cols, i); + TAOS_SML_KV **value = taosHashGet(tableMeta->fieldHash, kv->key, kv->keyLen); + if(value){ + if(kv->type != (*value)->type){ + // todo + } + }else{ + taosHashPut(tableMeta->fieldHash, kv->key, kv->keyLen, &kv, POINTER_BYTES); + } + } + } +} + +void insertMeta(SSmlSTableMeta* tableMeta, SArray *tags, SArray *cols){ + if(tags){ + for (int i = 0; i < taosArrayGetSize(tags); ++i) { + TAOS_SML_KV *kv = taosArrayGetP(tags, i); + taosHashPut(tableMeta->tagHash, kv->key, kv->keyLen, &kv, POINTER_BYTES); + } + } + + if(cols){ + for (int i = 0; i < taosArrayGetSize(cols); ++i) { + TAOS_SML_KV *kv = taosArrayGetP(cols, i); + taosHashPut(tableMeta->fieldHash, kv->key, kv->keyLen, &kv, POINTER_BYTES); + } + } +} + +int32_t tscParseLine(const char* sql, SSmlLinesInfo* info) { + TAOS_PARSE_ELEMENTS elements = {0}; + int ret = parseSml(sql, &elements); + if(ret != TSDB_CODE_SUCCESS){ + return ret; + } + + SArray *cols = taosArrayInit(16, POINTER_BYTES); + if (cols == NULL) { + uError("SML:0x%"PRIx64" taos_insert_lines failed to allocate memory", info->id); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + parseSmlTS(elements.timestamp, strlen(elements.timestamp), cols); + ret = parseSmlKV(elements.cols, elements.colsLen, cols); + if(ret != TSDB_CODE_SUCCESS){ + return ret; + } + + TAOS_SML_DATA_POINT_TAGS** oneTable = taosHashGet(info->childTables, elements.measure, elements.measureTagsLen); + if(oneTable){ + SSmlSTableMeta** tableMeta = taosHashGet(info->superTables, elements.measure, elements.measureLen); + ASSERT(tableMeta); + updateMeta(*tableMeta, NULL, cols); // update meta + + taosArrayPush((*oneTable)->cols, &cols); + }else{ + TAOS_SML_DATA_POINT_TAGS *tag = taosMemoryCalloc(sizeof(TAOS_SML_DATA_POINT_TAGS), 1); + tag->cols = taosArrayInit(16, POINTER_BYTES); + if (tag->cols == NULL) { + uError("SML:0x%"PRIx64" taos_insert_lines failed to allocate memory", info->id); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + taosArrayPush(tag->cols, &cols); + + tag->tags = taosArrayInit(16, POINTER_BYTES); + if (tag->tags == NULL) { + uError("SML:0x%"PRIx64" taos_insert_lines failed to allocate memory", info->id); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + ret = parseSmlKV(elements.tags, elements.tagsLen, tag->tags); + if(ret != TSDB_CODE_SUCCESS){ + return ret; + } + + SSmlSTableMeta** tableMeta = taosHashGet(info->superTables, elements.measure, elements.measureLen); + if(tableMeta){ // update meta + updateMeta(*tableMeta, tag->tags, cols); + }else{ + SSmlSTableMeta* meta = taosMemoryCalloc(sizeof(SSmlSTableMeta), 1); + insertMeta(meta, tag->tags, cols); + taosHashPut(info->superTables, elements.measure, elements.measureLen, &meta, POINTER_BYTES); + } + + taosHashPut(info->childTables, elements.measure, elements.measureTagsLen, &tag, POINTER_BYTES); + } + return TSDB_CODE_SUCCESS; +} + + +int32_t tscParseLines(char* lines[], int numLines, SSmlLinesInfo* info) { + for (int32_t i = 0; i < numLines; ++i) { + int32_t code = tscParseLine(lines[i], info); + if (code != TSDB_CODE_SUCCESS) { + uError("SML:0x%"PRIx64" data point line parse failed. line %d : %s", info->id, i, lines[i]); + return code; + } + } + uDebug("SML:0x%"PRIx64" data point line parse success. tables %d", info->id, taosHashGetSize(info->childTables)); + + return TSDB_CODE_SUCCESS; +} + +int taos_insert_lines(TAOS* taos, char* lines[], int numLines, SMLProtocolType protocol, SMLTimeStampType tsType, int *affectedRows) { + int32_t code = 0; + + SSmlLinesInfo* info = taosMemoryMalloc(sizeof(SSmlLinesInfo)); + info->id = genLinesSmlId(); + info->tsType = tsType; + info->taos = (STscObj*)taos; + info->protocol = protocol; + + if (numLines <= 0 || numLines > 65536) { + uError("SML:0x%"PRIx64" taos_insert_lines numLines should be between 1 and 65536. numLines: %d", info->id, numLines); + code = TSDB_CODE_TSC_APP_ERROR; + goto cleanup; + } + + info->childTables = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, false); + info->superTables = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, false); + + uDebug("SML:0x%"PRIx64" taos_insert_lines begin inserting %d lines, first line: %s", info->id, numLines, lines[0]); + code = tscParseLines(lines, numLines, info); + + if (code != 0) { + goto cleanup; + } + + code = tscSmlInsert(taos, info); + if (code != 0) { + uError("SML:0x%"PRIx64" taos_sml_insert error: %s", info->id, tstrerror((code))); + goto cleanup; + } + if (affectedRows != NULL) { + *affectedRows = info->affectedRows; + } + + uDebug("SML:0x%"PRIx64" taos_insert_lines finish inserting %d lines. code: %d", info->id, numLines, code); + +cleanup: + taosMemoryFree(info); + return code; +} + +static int32_t convertPrecisionType(int precision, SMLTimeStampType *tsType) { + switch (precision) { + case TSDB_SML_TIMESTAMP_NOT_CONFIGURED: + *tsType = SML_TIME_STAMP_NOT_CONFIGURED; + break; + case TSDB_SML_TIMESTAMP_HOURS: + *tsType = SML_TIME_STAMP_HOURS; + break; + case TSDB_SML_TIMESTAMP_MILLI_SECONDS: + *tsType = SML_TIME_STAMP_MILLI_SECONDS; + break; + case TSDB_SML_TIMESTAMP_NANO_SECONDS: + *tsType = SML_TIME_STAMP_NANO_SECONDS; + break; + case TSDB_SML_TIMESTAMP_MICRO_SECONDS: + *tsType = SML_TIME_STAMP_MICRO_SECONDS; + break; + case TSDB_SML_TIMESTAMP_SECONDS: + *tsType = SML_TIME_STAMP_SECONDS; + break; + case TSDB_SML_TIMESTAMP_MINUTES: + *tsType = SML_TIME_STAMP_MINUTES; + break; + default: + return TSDB_CODE_SML_INVALID_PRECISION_TYPE; + } + + return TSDB_CODE_SUCCESS; +} + +//make a dummy SSqlObj +static SSqlObj* createSmlQueryObj(TAOS* taos, int32_t affected_rows, int32_t code) { + SSqlObj *pNew = (SSqlObj*)calloc(1, sizeof(SSqlObj)); + if (pNew == NULL) { + return NULL; + } + pNew->signature = pNew; + pNew->pTscObj = taos; + pNew->fp = NULL; + + tsem_init(&pNew->rspSem, 0, 0); + registerSqlObj(pNew); + + pNew->res.numOfRows = affected_rows; + pNew->res.code = code; + + + return pNew; +} + + +/** + * taos_schemaless_insert() parse and insert data points into database according to + * different protocol. + * + * @param $lines input array may contain multiple lines, each line indicates a data point. + * If protocol=2 is used input array should contain single JSON + * string(e.g. char *lines[] = {"$JSON_string"}). If need to insert + * multiple data points in JSON format, should include them in $JSON_string + * as a JSON array. + * @param $numLines indicates how many data points in $lines. + * If protocol = 2 is used this param will be ignored as $lines should + * contain single JSON string. + * @param $protocol indicates which protocol to use for parsing: + * 0 - influxDB line protocol + * 1 - OpenTSDB telnet line protocol + * 2 - OpenTSDB JSON format protocol + * @return return zero for successful insertion. Otherwise return none-zero error code of + * failure reason. + * + */ + +TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int protocol, int precision) { + int code = TSDB_CODE_SUCCESS; + int affected_rows = 0; + SMLTimeStampType tsType = SML_TIME_STAMP_NOW; + + if (protocol == TSDB_SML_LINE_PROTOCOL) { + code = convertPrecisionType(precision, &tsType); + if (code != TSDB_CODE_SUCCESS) { + return NULL; + } + } + + switch (protocol) { + case TSDB_SML_LINE_PROTOCOL: + code = taos_insert_lines(taos, lines, numLines, protocol, tsType, &affected_rows); + break; + case TSDB_SML_TELNET_PROTOCOL: + //code = taos_insert_telnet_lines(taos, lines, numLines, protocol, tsType, &affected_rows); + break; + case TSDB_SML_JSON_PROTOCOL: + //code = taos_insert_json_payload(taos, *lines, protocol, tsType, &affected_rows); + break; + default: + code = TSDB_CODE_SML_INVALID_PROTOCOL_TYPE; + break; + } + + + SSqlObj *pSql = createSmlQueryObj(taos, affected_rows, code); + + return (TAOS_RES*)pSql; +} diff --git a/source/libs/CMakeLists.txt b/source/libs/CMakeLists.txt index b1e8be6528..ade2487239 100644 --- a/source/libs/CMakeLists.txt +++ b/source/libs/CMakeLists.txt @@ -17,4 +17,4 @@ add_subdirectory(tfs) add_subdirectory(monitor) add_subdirectory(nodes) add_subdirectory(scalar) -add_subdirectory(command) +add_subdirectory(command) \ No newline at end of file diff --git a/source/libs/scalar/test/scalar/scalarTests.cpp b/source/libs/scalar/test/scalar/scalarTests.cpp index de76e7e274..206b2ac542 100644 --- a/source/libs/scalar/test/scalar/scalarTests.cpp +++ b/source/libs/scalar/test/scalar/scalarTests.cpp @@ -127,7 +127,7 @@ void scltMakeValueNode(SNode **pNode, int32_t dataType, void *value) { *pNode = (SNode *)vnode; } -void scltMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, int32_t dataBytes, int32_t rowNum, void *value) { +void scltMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, int32_t dataBytes, int32_t ronwNum, void *value) { SNode *node = (SNode*)nodesMakeNode(QUERY_NODE_COLUMN); SColumnNode *rnode = (SColumnNode *)node; rnode->node.resType.type = dataType; @@ -156,9 +156,9 @@ void scltMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in idata.info.colId = 3; int32_t size = idata.info.bytes * rowNum; idata.pData = (char *)taosMemoryCalloc(1, size); + colInfoDataEnsureCapacity(&idata, 0, rowNum); + taosArrayPush(res->pDataBlock, &idata); - - blockDataEnsureCapacity(res, rowNum); SColumnInfoData *pColumn = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); for (int32_t i = 0; i < rowNum; ++i) { diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 9332cb481e..6c77d943f3 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -438,6 +438,11 @@ TAOS_DEFINE_ERROR(TSDB_CODE_QW_MSG_ERROR, "Invalid msg order") //planner TAOS_DEFINE_ERROR(TSDB_CODE_PLAN_INTERNAL_ERROR, "planner internal error") +//schemaless +TAOS_DEFINE_ERROR(TSDB_CODE_SML_INVALID_PROTOCOL_TYPE, "Invalid line protocol type") +TAOS_DEFINE_ERROR(TSDB_CODE_SML_INVALID_PRECISION_TYPE, "Invalid timestamp precision type") +TAOS_DEFINE_ERROR(TSDB_CODE_SML_INVALID_DATA, "Invalid data type") + #ifdef TAOS_ERROR_C }; #endif From 990205d6847b919c6c238b3c6fdaa309446a7c65 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 27 Apr 2022 21:29:25 +0800 Subject: [PATCH 02/83] refactor: schemaless tmp commit --- include/common/tcommon.h | 9 + include/libs/parser/parser.h | 6 +- include/util/tdef.h | 4 + .../inc/{tscParseLine.h => clientSml.h} | 78 +- source/client/schemaless/CMakeLists.txt | 17 - .../client/schemaless/src/tscParseOpenTSDB.c | 1113 ----------- .../{tscParseLineProtocol.c => clientSml.c} | 1767 +++-------------- source/client/src/clientStmt.c | 2 +- source/libs/parser/inc/parInsertData.h | 2 +- source/libs/parser/inc/parUtil.h | 1 - source/libs/parser/src/parInsert.c | 228 ++- source/libs/parser/src/parInsertData.c | 14 +- source/libs/parser/src/parUtil.c | 20 - tools/shell/src/shellCommand.c | 4 +- tools/shell/src/shellEngine.c | 54 +- 15 files changed, 548 insertions(+), 2771 deletions(-) rename source/client/inc/{tscParseLine.h => clientSml.h} (59%) delete mode 100644 source/client/schemaless/CMakeLists.txt delete mode 100644 source/client/schemaless/src/tscParseOpenTSDB.c rename source/client/src/{tscParseLineProtocol.c => clientSml.c} (51%) diff --git a/include/common/tcommon.h b/include/common/tcommon.h index 7c308f9354..1c516e8a96 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -241,6 +241,15 @@ typedef struct SExprInfo { struct tExprNode* pExpr; } SExprInfo; +typedef struct { + const char* key; + int32_t keyLen; + uint8_t type; + int16_t length; + const char* value; + int32_t valueLen; +} SSmlKv; + #define QUERY_ASC_FORWARD_STEP 1 #define QUERY_DESC_FORWARD_STEP -1 diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index 58482735ba..248a6c1237 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -92,10 +92,14 @@ int32_t qBindStmtColsValue(void *pBlock, TAOS_BIND_v2 *bind, char *msgBuf, int32 int32_t qBindStmtSingleColValue(void *pBlock, TAOS_BIND_v2 *bind, char *msgBuf, int32_t msgBufLen, int32_t colIdx, int32_t rowNum); int32_t qBuildStmtColFields(void *pDataBlock, int32_t *fieldNum, TAOS_FIELD** fields); int32_t qBuildStmtTagFields(void *pBlock, void *boundTags, int32_t *fieldNum, TAOS_FIELD** fields); -int32_t qBindStmtTagsValue(void *pBlock, void *boundTags, int64_t suid, SName *pName, TAOS_BIND_v2 *bind, char *msgBuf, int32_t msgBufLen); +int32_t qBindStmtTagsValue(void *pBlock, void *boundTags, int64_t suid, char *tName, TAOS_BIND_v2 *bind, char *msgBuf, int32_t msgBufLen); void destroyBoundColumnInfo(void* pBoundInfo); int32_t qCreateSName(SName* pName, const char* pTableName, int32_t acctId, char* dbName, char *msgBuf, int32_t msgBufLen); +void* tscSmlInitHandle(SQuery *pQuery); +void tscSmlDestroyHandle(void *pHandle); +int32_t smlBind(void *handle, SArray *tags, SArray *cols, STableMeta *pTableMeta, char *msgBuf, int16_t msgBufLen); +int32_t smlBuildOutput(void* handle, SHashObj* pVgHash); #ifdef __cplusplus } diff --git a/include/util/tdef.h b/include/util/tdef.h index 2548df7186..71d5f090a3 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -86,9 +86,13 @@ extern const int32_t TYPE_BYTES[15]; #define TS_PATH_DELIMITER "." #define TS_ESCAPE_CHAR '`' + #define TSDB_TIME_PRECISION_MILLI 0 #define TSDB_TIME_PRECISION_MICRO 1 #define TSDB_TIME_PRECISION_NANO 2 +#define TSDB_TIME_PRECISION_HOURS 3 +#define TSDB_TIME_PRECISION_MINUTES 4 +#define TSDB_TIME_PRECISION_SECONDS 5 #define TSDB_TIME_PRECISION_MILLI_STR "ms" #define TSDB_TIME_PRECISION_MICRO_STR "us" diff --git a/source/client/inc/tscParseLine.h b/source/client/inc/clientSml.h similarity index 59% rename from source/client/inc/tscParseLine.h rename to source/client/inc/clientSml.h index b70280f380..b711c837c0 100644 --- a/source/client/inc/tscParseLine.h +++ b/source/client/inc/clientSml.h @@ -13,30 +13,19 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_TSCPARSELINE_H -#define TDENGINE_TSCPARSELINE_H +#ifndef TDENGINE_CLIENTSML_H +#define TDENGINE_CLIENTSML_H #ifdef __cplusplus extern "C" { #endif #include "thash.h" -#include "clientint.h" - -#define SML_TIMESTAMP_SECOND_DIGITS 10 -#define SML_TIMESTAMP_MILLI_SECOND_DIGITS 13 +#include "clientInt.h" +#include "catalog.h" typedef TSDB_SML_PROTOCOL_TYPE SMLProtocolType; -typedef struct { - const char* key; - int32_t keyLen; - uint8_t type; - int16_t length; - const char* value; - int32_t valueLen; -} TAOS_SML_KV; - typedef struct { const char* measure; const char* tags; @@ -50,65 +39,66 @@ typedef struct { } TAOS_PARSE_ELEMENTS; typedef struct { - char* childTableName; + const char *sTableName; // super table name + uint8_t sTableNameLen; + char childTableName[TSDB_TABLE_NAME_LEN]; + uint64_t uid; SArray* tags; SArray *cols; } TAOS_SML_DATA_POINT_TAGS; typedef struct SSmlSTableMeta { - char *sTableName; // super table name - uint8_t sTableNameLen; +// char *sTableName; // super table name +// uint8_t sTableNameLen; uint8_t precision; // the number of precision SHashObj* tagHash; SHashObj* fieldHash; } SSmlSTableMeta; -typedef enum { - SML_TIME_STAMP_NOT_CONFIGURED, - SML_TIME_STAMP_HOURS, - SML_TIME_STAMP_MINUTES, - SML_TIME_STAMP_SECONDS, - SML_TIME_STAMP_MILLI_SECONDS, - SML_TIME_STAMP_MICRO_SECONDS, - SML_TIME_STAMP_NANO_SECONDS, - SML_TIME_STAMP_NOW -} SMLTimeStampType; - typedef struct { uint64_t id; - STscObj* taos; - SCatalog* pCatalog; - SMLProtocolType protocol; - SMLTimeStampType tsType; - - int32_t affectedRows; + int32_t tsType; SHashObj* childTables; SHashObj* superTables; + + SHashObj* metaHashObj; + SHashObj* pVgHash; + + void* exec; + + STscObj* taos; + SCatalog* pCatalog; + SRequestObj* pRequest; + SQuery* pQuery; + + int32_t affectedRows; + char *msgBuf; + int16_t msgLen; } SSmlLinesInfo; -int tscSmlInsert(TAOS* taos, TAOS_SML_DATA_POINT* points, int numPoint, SSmlLinesInfo* info); +int smlInsert(TAOS* taos, SSmlLinesInfo* info); + bool checkDuplicateKey(char *key, SHashObj *pHash, SSmlLinesInfo* info); bool isValidInteger(char *str); bool isValidFloat(char *str); int32_t isValidChildTableName(const char *pTbName, int16_t len, SSmlLinesInfo* info); -bool convertSmlValueType(TAOS_SML_KV *pVal, char *value, +bool convertSmlValueType(SSmlKv *pVal, char *value, uint16_t len, SSmlLinesInfo* info, bool isTag); -int32_t convertSmlTimeStamp(TAOS_SML_KV *pVal, char *value, +int32_t convertSmlTimeStamp(SSmlKv *pVal, char *value, uint16_t len, SSmlLinesInfo* info); -void destroySmlDataPoint(TAOS_SML_DATA_POINT* point); -int taos_insert_lines(TAOS* taos, char* lines[], int numLines, SMLProtocolType protocol, - SMLTimeStampType tsType, int* affectedRows); -int taos_insert_telnet_lines(TAOS* taos, char* lines[], int numLines, SMLProtocolType protocol, +int sml_insert_lines(TAOS* taos, SRequestObj* request, char* lines[], int numLines, SMLProtocolType protocol, + SMLTimeStampType tsType); +int sml_insert_telnet_lines(TAOS* taos, char* lines[], int numLines, SMLProtocolType protocol, SMLTimeStampType tsType, int* affectedRows); -int taos_insert_json_payload(TAOS* taos, char* payload, SMLProtocolType protocol, +int sml_insert_json_payload(TAOS* taos, char* payload, SMLProtocolType protocol, SMLTimeStampType tsType, int* affectedRows); @@ -116,4 +106,4 @@ int taos_insert_json_payload(TAOS* taos, char* payload, SMLProtocolType protocol } #endif -#endif // TDENGINE_TSCPARSELINE_H +#endif // TDENGINE_CLIENTSML_H diff --git a/source/client/schemaless/CMakeLists.txt b/source/client/schemaless/CMakeLists.txt deleted file mode 100644 index 29be674934..0000000000 --- a/source/client/schemaless/CMakeLists.txt +++ /dev/null @@ -1,17 +0,0 @@ -aux_source_directory(src SCHEMALESS_SRC) -add_library(schemaless STATIC ${SCHEMALESS_SRC}) - -target_include_directories( - schemaless - PUBLIC "${TD_SOURCE_DIR}/include/libs/schemaless" - PRIVATE "inc" -) - -target_link_libraries( - schemaless - PUBLIC os util common catalog qcom -) - -if(${BUILD_TEST}) - ADD_SUBDIRECTORY(test) -endif(${BUILD_TEST}) diff --git a/source/client/schemaless/src/tscParseOpenTSDB.c b/source/client/schemaless/src/tscParseOpenTSDB.c deleted file mode 100644 index 5bb9b09cf1..0000000000 --- a/source/client/schemaless/src/tscParseOpenTSDB.c +++ /dev/null @@ -1,1113 +0,0 @@ -//#include -//#include -//#include -//#include -// -//#include "cJSON.h" -//#include "hash.h" -//#include "taos.h" -// -//#include "tscUtil.h" -//#include "tsclient.h" -//#include "tscLog.h" -// -//#include "tscParseLine.h" -// -//#define OTD_MAX_FIELDS_NUM 2 -//#define OTD_JSON_SUB_FIELDS_NUM 2 -//#define OTD_JSON_FIELDS_NUM 4 -// -//#define OTD_TIMESTAMP_COLUMN_NAME "ts" -//#define OTD_METRIC_VALUE_COLUMN_NAME "value" -// -///* telnet style API parser */ -//static uint64_t HandleId = 0; -// -//static uint64_t genUID() { -// uint64_t id; -// -// do { -// id = atomic_add_fetch_64(&HandleId, 1); -// } while (id == 0); -// -// return id; -//} -// -//static int32_t parseTelnetMetric(TAOS_SML_DATA_POINT *pSml, const char **index, SSmlLinesInfo* info) { -// const char *cur = *index; -// uint16_t len = 0; -// -// pSml->stableName = tcalloc(TSDB_TABLE_NAME_LEN + TS_BACKQUOTE_CHAR_SIZE, 1); -// if (pSml->stableName == NULL) { -// return TSDB_CODE_TSC_OUT_OF_MEMORY; -// } -// /* -// if (isdigit(*cur)) { -// tscError("OTD:0x%"PRIx64" Metric cannot start with digit", info->id); -// tfree(pSml->stableName); -// return TSDB_CODE_TSC_LINE_SYNTAX_ERROR; -// } -// */ -// -// while (*cur != '\0') { -// if (len > TSDB_TABLE_NAME_LEN - 1) { -// tscError("OTD:0x%"PRIx64" Metric cannot exceeds %d characters", info->id, TSDB_TABLE_NAME_LEN - 1); -// tfree(pSml->stableName); -// return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; -// } -// -// if (*cur == ' ') { -// if (*(cur + 1) != ' ') { -// break; -// } else { -// cur++; -// continue; -// } -// } -// -// pSml->stableName[len] = *cur; -// -// cur++; -// len++; -// } -// if (len == 0 || *cur == '\0') { -// tfree(pSml->stableName); -// return TSDB_CODE_TSC_LINE_SYNTAX_ERROR; -// } -// -// addEscapeCharToString(pSml->stableName, len); -// *index = cur + 1; -// tscDebug("OTD:0x%"PRIx64" Stable name in metric:%s|len:%d", info->id, pSml->stableName, len); -// -// return TSDB_CODE_SUCCESS; -//} -// -//static int32_t parseTelnetTimeStamp(TAOS_SML_KV **pTS, int *num_kvs, const char **index, SSmlLinesInfo* info) { -// //Timestamp must be the first KV to parse -// assert(*num_kvs == 0); -// -// const char *start, *cur; -// int32_t ret = TSDB_CODE_SUCCESS; -// int len = 0; -// char key[] = OTD_TIMESTAMP_COLUMN_NAME; -// char *value = NULL; -// -// start = cur = *index; -// //allocate fields for timestamp and value -// *pTS = tcalloc(OTD_MAX_FIELDS_NUM, sizeof(TAOS_SML_KV)); -// -// while(*cur != '\0') { -// if (*cur == ' ') { -// if (*(cur + 1) != ' ') { -// break; -// } else { -// cur++; -// continue; -// } -// } -// cur++; -// len++; -// } -// -// if (len > 0 && *cur != '\0') { -// value = tcalloc(len + 1, 1); -// memcpy(value, start, len); -// } else { -// tfree(*pTS); -// return TSDB_CODE_TSC_LINE_SYNTAX_ERROR; -// } -// -// ret = convertSmlTimeStamp(*pTS, value, len, info); -// if (ret) { -// tfree(value); -// tfree(*pTS); -// return ret; -// } -// tfree(value); -// -// (*pTS)->key = tcalloc(sizeof(key) + TS_BACKQUOTE_CHAR_SIZE, 1); -// memcpy((*pTS)->key, key, sizeof(key)); -// addEscapeCharToString((*pTS)->key, (int32_t)strlen(key)); -// -// *num_kvs += 1; -// *index = cur + 1; -// -// return ret; -//} -// -//static int32_t parseTelnetMetricValue(TAOS_SML_KV **pKVs, int *num_kvs, const char **index, SSmlLinesInfo* info) { -// //skip timestamp -// TAOS_SML_KV *pVal = *pKVs + 1; -// const char *start, *cur; -// int32_t ret = TSDB_CODE_SUCCESS; -// int len = 0; -// bool searchQuote = false; -// char key[] = OTD_METRIC_VALUE_COLUMN_NAME; -// char *value = NULL; -// -// start = cur = *index; -// -// //if metric value is string -// if (*cur == '"') { -// searchQuote = true; -// cur += 1; -// len += 1; -// } else if (*cur == 'L' && *(cur + 1) == '"') { -// searchQuote = true; -// cur += 2; -// len += 2; -// } -// -// while(*cur != '\0') { -// if (*cur == ' ') { -// if (searchQuote == true) { -// if (*(cur - 1) == '"' && len != 1 && len != 2) { -// searchQuote = false; -// } else { -// cur++; -// len++; -// continue; -// } -// } -// -// if (*(cur + 1) != ' ') { -// break; -// } else { -// cur++; -// continue; -// } -// } -// cur++; -// len++; -// } -// -// if (len > 0 && *cur != '\0') { -// value = tcalloc(len + 1, 1); -// memcpy(value, start, len); -// } else { -// return TSDB_CODE_TSC_LINE_SYNTAX_ERROR; -// } -// -// if (!convertSmlValueType(pVal, value, len, info, false)) { -// tscError("OTD:0x%"PRIx64" Failed to convert metric value string(%s) to any type", -// info->id, value); -// tfree(value); -// return TSDB_CODE_TSC_INVALID_VALUE; -// } -// tfree(value); -// -// pVal->key = tcalloc(sizeof(key) + TS_BACKQUOTE_CHAR_SIZE, 1); -// memcpy(pVal->key, key, sizeof(key)); -// addEscapeCharToString(pVal->key, (int32_t)strlen(pVal->key)); -// *num_kvs += 1; -// -// *index = cur + 1; -// return ret; -//} -// -//static int32_t parseTelnetTagKey(TAOS_SML_KV *pKV, const char **index, SHashObj *pHash, SSmlLinesInfo* info) { -// const char *cur = *index; -// char key[TSDB_COL_NAME_LEN]; -// uint16_t len = 0; -// -// //key field cannot start with digit -// //if (isdigit(*cur)) { -// // tscError("OTD:0x%"PRIx64" Tag key cannot start with digit", info->id); -// // return TSDB_CODE_TSC_LINE_SYNTAX_ERROR; -// //} -// while (*cur != '\0') { -// if (len > TSDB_COL_NAME_LEN - 1) { -// tscError("OTD:0x%"PRIx64" Tag key cannot exceeds %d characters", info->id, TSDB_COL_NAME_LEN - 1); -// return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; -// } -// if (*cur == ' ') { -// return TSDB_CODE_TSC_LINE_SYNTAX_ERROR; -// } -// if (*cur == '=') { -// break; -// } -// -// key[len] = *cur; -// cur++; -// len++; -// } -// if (len == 0 || *cur == '\0') { -// return TSDB_CODE_TSC_LINE_SYNTAX_ERROR; -// } -// key[len] = '\0'; -// -// if (checkDuplicateKey(key, pHash, info)) { -// return TSDB_CODE_TSC_DUP_TAG_NAMES; -// } -// -// pKV->key = tcalloc(len + TS_BACKQUOTE_CHAR_SIZE + 1, 1); -// memcpy(pKV->key, key, len + 1); -// addEscapeCharToString(pKV->key, len); -// //tscDebug("OTD:0x%"PRIx64" Key:%s|len:%d", info->id, pKV->key, len); -// *index = cur + 1; -// return TSDB_CODE_SUCCESS; -//} -// -// -//static int32_t parseTelnetTagValue(TAOS_SML_KV *pKV, const char **index, -// bool *is_last_kv, SSmlLinesInfo* info) { -// const char *start, *cur; -// char *value = NULL; -// uint16_t len = 0; -// start = cur = *index; -// -// while (1) { -// // whitespace or '\0' identifies a value -// if (*cur == ' ' || *cur == '\0') { -// // '\0' indicates end of value -// *is_last_kv = (*cur == '\0') ? true : false; -// if (*cur == ' ' && *(cur + 1) == ' ') { -// cur++; -// continue; -// } else { -// break; -// } -// } -// cur++; -// len++; -// } -// -// if (len == 0) { -// tfree(pKV->key); -// return TSDB_CODE_TSC_LINE_SYNTAX_ERROR; -// } -// -// value = tcalloc(len + 1, 1); -// memcpy(value, start, len); -// value[len] = '\0'; -// if (!convertSmlValueType(pKV, value, len, info, true)) { -// tscError("OTD:0x%"PRIx64" Failed to convert sml value string(%s) to any type", -// info->id, value); -// //free previous alocated key field -// tfree(pKV->key); -// tfree(value); -// return TSDB_CODE_TSC_INVALID_VALUE; -// } -// tfree(value); -// -// *index = (*cur == '\0') ? cur : cur + 1; -// return TSDB_CODE_SUCCESS; -//} -// -//static int32_t parseTelnetTagKvs(TAOS_SML_KV **pKVs, int *num_kvs, -// const char **index, char **childTableName, -// SHashObj *pHash, SSmlLinesInfo* info) { -// const char *cur = *index; -// int32_t ret = TSDB_CODE_SUCCESS; -// TAOS_SML_KV *pkv; -// bool is_last_kv = false; -// -// int32_t capacity = 4; -// *pKVs = tcalloc(capacity, sizeof(TAOS_SML_KV)); -// pkv = *pKVs; -// -// size_t childTableNameLen = strlen(tsSmlChildTableName); -// char childTbName[TSDB_TABLE_NAME_LEN + TS_BACKQUOTE_CHAR_SIZE] = {0}; -// if (childTableNameLen != 0) { -// memcpy(childTbName, tsSmlChildTableName, childTableNameLen); -// addEscapeCharToString(childTbName, (int32_t)(childTableNameLen)); -// } -// while (*cur != '\0') { -// ret = parseTelnetTagKey(pkv, &cur, pHash, info); -// if (ret) { -// tscError("OTD:0x%"PRIx64" Unable to parse key", info->id); -// return ret; -// } -// ret = parseTelnetTagValue(pkv, &cur, &is_last_kv, info); -// if (ret) { -// tscError("OTD:0x%"PRIx64" Unable to parse value", info->id); -// return ret; -// } -// if (childTableNameLen != 0 && strcasecmp(pkv->key, childTbName) == 0) { -// *childTableName = tcalloc(pkv->length + TS_BACKQUOTE_CHAR_SIZE + 1, 1); -// memcpy(*childTableName, pkv->value, pkv->length); -// (*childTableName)[pkv->length] = '\0'; -// addEscapeCharToString(*childTableName, pkv->length); -// tfree(pkv->key); -// tfree(pkv->value); -// } else { -// *num_kvs += 1; -// } -// -// if (is_last_kv) { -// break; -// } -// -// //reallocate addtional memory for more kvs -// if ((*num_kvs + 1) > capacity) { -// TAOS_SML_KV *more_kvs = NULL; -// capacity *= 3; capacity /= 2; -// more_kvs = realloc(*pKVs, capacity * sizeof(TAOS_SML_KV)); -// if (!more_kvs) { -// return TSDB_CODE_TSC_OUT_OF_MEMORY; -// } -// *pKVs = more_kvs; -// } -// -// //move pKV points to next TAOS_SML_KV block -// pkv = *pKVs + *num_kvs; -// } -// -// return ret; -//} -// -//static int32_t tscParseTelnetLine(const char* line, TAOS_SML_DATA_POINT* smlData, SSmlLinesInfo* info) { -// const char* index = line; -// int32_t ret = TSDB_CODE_SUCCESS; -// -// //Parse metric -// ret = parseTelnetMetric(smlData, &index, info); -// if (ret) { -// tscError("OTD:0x%"PRIx64" Unable to parse metric", info->id); -// return ret; -// } -// tscDebug("OTD:0x%"PRIx64" Parse metric finished", info->id); -// -// //Parse timestamp -// ret = parseTelnetTimeStamp(&smlData->fields, &smlData->fieldNum, &index, info); -// if (ret) { -// tscError("OTD:0x%"PRIx64" Unable to parse timestamp", info->id); -// return ret; -// } -// tscDebug("OTD:0x%"PRIx64" Parse timestamp finished", info->id); -// -// //Parse value -// ret = parseTelnetMetricValue(&smlData->fields, &smlData->fieldNum, &index, info); -// if (ret) { -// tscError("OTD:0x%"PRIx64" Unable to parse metric value", info->id); -// return ret; -// } -// tscDebug("OTD:0x%"PRIx64" Parse metric value finished", info->id); -// -// //Parse tagKVs -// SHashObj *keyHashTable = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); -// ret = parseTelnetTagKvs(&smlData->tags, &smlData->tagNum, &index, &smlData->childTableName, keyHashTable, info); -// if (ret) { -// tscError("OTD:0x%"PRIx64" Unable to parse tags", info->id); -// taosHashCleanup(keyHashTable); -// return ret; -// } -// tscDebug("OTD:0x%"PRIx64" Parse tags finished", info->id); -// taosHashCleanup(keyHashTable); -// -// -// return TSDB_CODE_SUCCESS; -//} -// -//static int32_t tscParseTelnetLines(char* lines[], int numLines, SArray* points, SArray* failedLines, SSmlLinesInfo* info) { -// for (int32_t i = 0; i < numLines; ++i) { -// TAOS_SML_DATA_POINT point = {0}; -// int32_t code = tscParseTelnetLine(lines[i], &point, info); -// if (code != TSDB_CODE_SUCCESS) { -// tscError("OTD:0x%"PRIx64" data point line parse failed. line %d : %s", info->id, i, lines[i]); -// destroySmlDataPoint(&point); -// return code; -// } else { -// tscDebug("OTD:0x%"PRIx64" data point line parse success. line %d", info->id, i); -// } -// -// taosArrayPush(points, &point); -// } -// return TSDB_CODE_SUCCESS; -//} -// -//int taos_insert_telnet_lines(TAOS* taos, char* lines[], int numLines, SMLProtocolType protocol, SMLTimeStampType tsType, int* affectedRows) { -// int32_t code = 0; -// -// SSmlLinesInfo* info = tcalloc(1, sizeof(SSmlLinesInfo)); -// info->id = genUID(); -// info->tsType = tsType; -// info->protocol = protocol; -// -// if (numLines <= 0 || numLines > 65536) { -// tscError("OTD:0x%"PRIx64" taos_insert_telnet_lines numLines should be between 1 and 65536. numLines: %d", info->id, numLines); -// tfree(info); -// code = TSDB_CODE_TSC_APP_ERROR; -// return code; -// } -// -// for (int i = 0; i < numLines; ++i) { -// if (lines[i] == NULL) { -// tscError("OTD:0x%"PRIx64" taos_insert_telnet_lines line %d is NULL", info->id, i); -// tfree(info); -// code = TSDB_CODE_TSC_APP_ERROR; -// return code; -// } -// } -// -// SArray* lpPoints = taosArrayInit(numLines, sizeof(TAOS_SML_DATA_POINT)); -// if (lpPoints == NULL) { -// tscError("OTD:0x%"PRIx64" taos_insert_telnet_lines failed to allocate memory", info->id); -// tfree(info); -// return TSDB_CODE_TSC_OUT_OF_MEMORY; -// } -// -// tscDebug("OTD:0x%"PRIx64" taos_insert_telnet_lines begin inserting %d lines, first line: %s", info->id, numLines, lines[0]); -// code = tscParseTelnetLines(lines, numLines, lpPoints, NULL, info); -// size_t numPoints = taosArrayGetSize(lpPoints); -// -// if (code != 0) { -// goto cleanup; -// } -// -// TAOS_SML_DATA_POINT* points = TARRAY_GET_START(lpPoints); -// code = tscSmlInsert(taos, points, (int)numPoints, info); -// if (code != 0) { -// tscError("OTD:0x%"PRIx64" taos_insert_telnet_lines error: %s", info->id, tstrerror((code))); -// } -// if (affectedRows != NULL) { -// *affectedRows = info->affectedRows; -// } -// -//cleanup: -// tscDebug("OTD:0x%"PRIx64" taos_insert_telnet_lines finish inserting %d lines. code: %d", info->id, numLines, code); -// points = TARRAY_GET_START(lpPoints); -// numPoints = taosArrayGetSize(lpPoints); -// for (int i = 0; i < numPoints; ++i) { -// destroySmlDataPoint(points+i); -// } -// -// taosArrayDestroy(&lpPoints); -// -// tfree(info); -// return code; -//} -// -//int taos_telnet_insert(TAOS* taos, TAOS_SML_DATA_POINT* points, int numPoint) { -// SSmlLinesInfo* info = tcalloc(1, sizeof(SSmlLinesInfo)); -// info->id = genUID(); -// int code = tscSmlInsert(taos, points, numPoint, info); -// tfree(info); -// return code; -//} -// -// -///* telnet style API parser */ -//static int32_t parseMetricFromJSON(cJSON *root, TAOS_SML_DATA_POINT* pSml, SSmlLinesInfo* info) { -// cJSON *metric = cJSON_GetObjectItem(root, "metric"); -// if (!cJSON_IsString(metric)) { -// return TSDB_CODE_TSC_INVALID_JSON; -// } -// -// size_t stableLen = strlen(metric->valuestring); -// if (stableLen > TSDB_TABLE_NAME_LEN - 1) { -// tscError("OTD:0x%"PRIx64" Metric cannot exceeds %d characters in JSON", info->id, TSDB_TABLE_NAME_LEN - 1); -// return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; -// } -// -// pSml->stableName = tcalloc(stableLen + TS_BACKQUOTE_CHAR_SIZE + 1, sizeof(char)); -// if (pSml->stableName == NULL){ -// return TSDB_CODE_TSC_OUT_OF_MEMORY; -// } -// -// /* -// if (isdigit(metric->valuestring[0])) { -// tscError("OTD:0x%"PRIx64" Metric cannot start with digit in JSON", info->id); -// tfree(pSml->stableName); -// return TSDB_CODE_TSC_INVALID_JSON; -// } -// */ -// -// tstrncpy(pSml->stableName, metric->valuestring, stableLen + 1); -// addEscapeCharToString(pSml->stableName, (int32_t)stableLen); -// -// return TSDB_CODE_SUCCESS; -// -//} -// -//static int32_t parseTimestampFromJSONObj(cJSON *root, int64_t *tsVal, SSmlLinesInfo* info) { -// int32_t size = cJSON_GetArraySize(root); -// if (size != OTD_JSON_SUB_FIELDS_NUM) { -// return TSDB_CODE_TSC_INVALID_JSON; -// } -// -// cJSON *value = cJSON_GetObjectItem(root, "value"); -// if (!cJSON_IsNumber(value)) { -// return TSDB_CODE_TSC_INVALID_JSON; -// } -// -// cJSON *type = cJSON_GetObjectItem(root, "type"); -// if (!cJSON_IsString(type)) { -// return TSDB_CODE_TSC_INVALID_JSON; -// } -// -// *tsVal = strtoll(value->numberstring, NULL, 10); -// //if timestamp value is 0 use current system time -// if (*tsVal == 0) { -// *tsVal = taosGetTimestampNs(); -// return TSDB_CODE_SUCCESS; -// } -// -// size_t typeLen = strlen(type->valuestring); -// if (typeLen == 1 && type->valuestring[0] == 's') { -// //seconds -// *tsVal = (int64_t)(*tsVal * 1e9); -// } else if (typeLen == 2 && type->valuestring[1] == 's') { -// switch (type->valuestring[0]) { -// case 'm': -// //milliseconds -// *tsVal = convertTimePrecision(*tsVal, TSDB_TIME_PRECISION_MILLI, TSDB_TIME_PRECISION_NANO); -// break; -// case 'u': -// //microseconds -// *tsVal = convertTimePrecision(*tsVal, TSDB_TIME_PRECISION_MICRO, TSDB_TIME_PRECISION_NANO); -// break; -// case 'n': -// //nanoseconds -// *tsVal = *tsVal * 1; -// break; -// default: -// return TSDB_CODE_TSC_INVALID_JSON; -// } -// } else { -// return TSDB_CODE_TSC_INVALID_JSON; -// } -// -// return TSDB_CODE_SUCCESS; -//} -// -//static int32_t parseTimestampFromJSON(cJSON *root, TAOS_SML_KV **pTS, int *num_kvs, SSmlLinesInfo* info) { -// //Timestamp must be the first KV to parse -// assert(*num_kvs == 0); -// int64_t tsVal; -// char key[] = OTD_TIMESTAMP_COLUMN_NAME; -// -// cJSON *timestamp = cJSON_GetObjectItem(root, "timestamp"); -// if (cJSON_IsNumber(timestamp)) { -// //timestamp value 0 indicates current system time -// if (timestamp->valueint == 0) { -// tsVal = taosGetTimestampNs(); -// } else { -// tsVal = strtoll(timestamp->numberstring, NULL, 10); -// size_t tsLen = strlen(timestamp->numberstring); -// if (tsLen == SML_TIMESTAMP_SECOND_DIGITS) { -// tsVal = (int64_t)(tsVal * 1e9); -// } else if (tsLen == SML_TIMESTAMP_MILLI_SECOND_DIGITS) { -// tsVal = convertTimePrecision(tsVal, TSDB_TIME_PRECISION_MILLI, TSDB_TIME_PRECISION_NANO); -// } else { -// return TSDB_CODE_TSC_INVALID_TIME_STAMP; -// } -// } -// } else if (cJSON_IsObject(timestamp)) { -// int32_t ret = parseTimestampFromJSONObj(timestamp, &tsVal, info); -// if (ret != TSDB_CODE_SUCCESS) { -// tscError("OTD:0x%"PRIx64" Failed to parse timestamp from JSON Obj", info->id); -// return ret; -// } -// } else { -// return TSDB_CODE_TSC_INVALID_JSON; -// } -// -// //allocate fields for timestamp and value -// *pTS = tcalloc(OTD_MAX_FIELDS_NUM, sizeof(TAOS_SML_KV)); -// -// -// (*pTS)->key = tcalloc(sizeof(key), 1); -// memcpy((*pTS)->key, key, sizeof(key)); -// -// (*pTS)->type = TSDB_DATA_TYPE_TIMESTAMP; -// (*pTS)->length = (int16_t)tDataTypes[(*pTS)->type].bytes; -// (*pTS)->value = tcalloc((*pTS)->length, 1); -// memcpy((*pTS)->value, &tsVal, (*pTS)->length); -// -// *num_kvs += 1; -// return TSDB_CODE_SUCCESS; -// -//} -// -//static int32_t convertJSONBool(TAOS_SML_KV *pVal, char* typeStr, int64_t valueInt, SSmlLinesInfo* info) { -// if (strcasecmp(typeStr, "bool") != 0) { -// tscError("OTD:0x%"PRIx64" invalid type(%s) for JSON Bool", info->id, typeStr); -// return TSDB_CODE_TSC_INVALID_JSON_TYPE; -// } -// pVal->type = TSDB_DATA_TYPE_BOOL; -// pVal->length = (int16_t)tDataTypes[pVal->type].bytes; -// pVal->value = tcalloc(pVal->length, 1); -// *(bool *)(pVal->value) = valueInt ? true : false; -// -// return TSDB_CODE_SUCCESS; -//} -// -//static int32_t convertJSONNumber(TAOS_SML_KV *pVal, char* typeStr, cJSON *value, SSmlLinesInfo* info) { -// //tinyint -// if (strcasecmp(typeStr, "i8") == 0 || -// strcasecmp(typeStr, "tinyint") == 0) { -// if (!IS_VALID_TINYINT(value->valueint)) { -// tscError("OTD:0x%"PRIx64" JSON value(%"PRId64") cannot fit in type(tinyint)", info->id, value->valueint); -// return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; -// } -// pVal->type = TSDB_DATA_TYPE_TINYINT; -// pVal->length = (int16_t)tDataTypes[pVal->type].bytes; -// pVal->value = tcalloc(pVal->length, 1); -// *(int8_t *)(pVal->value) = (int8_t)(value->valueint); -// return TSDB_CODE_SUCCESS; -// } -// //smallint -// if (strcasecmp(typeStr, "i16") == 0 || -// strcasecmp(typeStr, "smallint") == 0) { -// if (!IS_VALID_SMALLINT(value->valueint)) { -// tscError("OTD:0x%"PRIx64" JSON value(%"PRId64") cannot fit in type(smallint)", info->id, value->valueint); -// return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; -// } -// pVal->type = TSDB_DATA_TYPE_SMALLINT; -// pVal->length = (int16_t)tDataTypes[pVal->type].bytes; -// pVal->value = tcalloc(pVal->length, 1); -// *(int16_t *)(pVal->value) = (int16_t)(value->valueint); -// return TSDB_CODE_SUCCESS; -// } -// //int -// if (strcasecmp(typeStr, "i32") == 0 || -// strcasecmp(typeStr, "int") == 0) { -// if (!IS_VALID_INT(value->valueint)) { -// tscError("OTD:0x%"PRIx64" JSON value(%"PRId64") cannot fit in type(int)", info->id, value->valueint); -// return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; -// } -// pVal->type = TSDB_DATA_TYPE_INT; -// pVal->length = (int16_t)tDataTypes[pVal->type].bytes; -// pVal->value = tcalloc(pVal->length, 1); -// *(int32_t *)(pVal->value) = (int32_t)(value->valueint); -// return TSDB_CODE_SUCCESS; -// } -// //bigint -// if (strcasecmp(typeStr, "i64") == 0 || -// strcasecmp(typeStr, "bigint") == 0) { -// pVal->type = TSDB_DATA_TYPE_BIGINT; -// pVal->length = (int16_t)tDataTypes[pVal->type].bytes; -// pVal->value = tcalloc(pVal->length, 1); -// /* cJSON conversion of legit BIGINT may overflow, -// * use original string to do the conversion. -// */ -// errno = 0; -// int64_t val = (int64_t)strtoll(value->numberstring, NULL, 10); -// if (errno == ERANGE || !IS_VALID_BIGINT(val)) { -// tscError("OTD:0x%"PRIx64" JSON value(%s) cannot fit in type(bigint)", info->id, value->numberstring); -// return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; -// } -// *(int64_t *)(pVal->value) = val; -// return TSDB_CODE_SUCCESS; -// } -// //float -// if (strcasecmp(typeStr, "f32") == 0 || -// strcasecmp(typeStr, "float") == 0) { -// if (!IS_VALID_FLOAT(value->valuedouble)) { -// tscError("OTD:0x%"PRIx64" JSON value(%f) cannot fit in type(float)", info->id, value->valuedouble); -// return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; -// } -// pVal->type = TSDB_DATA_TYPE_FLOAT; -// pVal->length = (int16_t)tDataTypes[pVal->type].bytes; -// pVal->value = tcalloc(pVal->length, 1); -// *(float *)(pVal->value) = (float)(value->valuedouble); -// return TSDB_CODE_SUCCESS; -// } -// //double -// if (strcasecmp(typeStr, "f64") == 0 || -// strcasecmp(typeStr, "double") == 0) { -// if (!IS_VALID_DOUBLE(value->valuedouble)) { -// tscError("OTD:0x%"PRIx64" JSON value(%f) cannot fit in type(double)", info->id, value->valuedouble); -// return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; -// } -// pVal->type = TSDB_DATA_TYPE_DOUBLE; -// pVal->length = (int16_t)tDataTypes[pVal->type].bytes; -// pVal->value = tcalloc(pVal->length, 1); -// *(double *)(pVal->value) = (double)(value->valuedouble); -// return TSDB_CODE_SUCCESS; -// } -// -// //if reach here means type is unsupported -// tscError("OTD:0x%"PRIx64" invalid type(%s) for JSON Number", info->id, typeStr); -// return TSDB_CODE_TSC_INVALID_JSON_TYPE; -//} -// -//static int32_t convertJSONString(TAOS_SML_KV *pVal, char* typeStr, cJSON *value, SSmlLinesInfo* info) { -// if (strcasecmp(typeStr, "binary") == 0) { -// pVal->type = TSDB_DATA_TYPE_BINARY; -// } else if (strcasecmp(typeStr, "nchar") == 0) { -// pVal->type = TSDB_DATA_TYPE_NCHAR; -// } else { -// tscError("OTD:0x%"PRIx64" invalid type(%s) for JSON String", info->id, typeStr); -// return TSDB_CODE_TSC_INVALID_JSON_TYPE; -// } -// pVal->length = (int16_t)strlen(value->valuestring); -// pVal->value = tcalloc(pVal->length + 1, 1); -// memcpy(pVal->value, value->valuestring, pVal->length); -// return TSDB_CODE_SUCCESS; -//} -// -//static int32_t parseValueFromJSONObj(cJSON *root, TAOS_SML_KV *pVal, SSmlLinesInfo* info) { -// int32_t ret = TSDB_CODE_SUCCESS; -// int32_t size = cJSON_GetArraySize(root); -// -// if (size != OTD_JSON_SUB_FIELDS_NUM) { -// return TSDB_CODE_TSC_INVALID_JSON; -// } -// -// cJSON *value = cJSON_GetObjectItem(root, "value"); -// if (value == NULL) { -// return TSDB_CODE_TSC_INVALID_JSON; -// } -// -// cJSON *type = cJSON_GetObjectItem(root, "type"); -// if (!cJSON_IsString(type)) { -// return TSDB_CODE_TSC_INVALID_JSON; -// } -// -// switch (value->type) { -// case cJSON_True: -// case cJSON_False: { -// ret = convertJSONBool(pVal, type->valuestring, value->valueint, info); -// if (ret != TSDB_CODE_SUCCESS) { -// return ret; -// } -// break; -// } -// case cJSON_Number: { -// ret = convertJSONNumber(pVal, type->valuestring, value, info); -// if (ret != TSDB_CODE_SUCCESS) { -// return ret; -// } -// break; -// } -// case cJSON_String: { -// ret = convertJSONString(pVal, type->valuestring, value, info); -// if (ret != TSDB_CODE_SUCCESS) { -// return ret; -// } -// break; -// } -// default: -// return TSDB_CODE_TSC_INVALID_JSON_TYPE; -// } -// -// return TSDB_CODE_SUCCESS; -//} -// -//static int32_t parseValueFromJSON(cJSON *root, TAOS_SML_KV *pVal, SSmlLinesInfo* info) { -// int type = root->type; -// -// switch (type) { -// case cJSON_True: -// case cJSON_False: { -// pVal->type = TSDB_DATA_TYPE_BOOL; -// pVal->length = (int16_t)tDataTypes[pVal->type].bytes; -// pVal->value = tcalloc(pVal->length, 1); -// *(bool *)(pVal->value) = root->valueint ? true : false; -// break; -// } -// case cJSON_Number: { -// //convert default JSON Number type to BIGINT/DOUBLE -// //if (isValidInteger(root->numberstring)) { -// // pVal->type = TSDB_DATA_TYPE_BIGINT; -// // pVal->length = (int16_t)tDataTypes[pVal->type].bytes; -// // pVal->value = tcalloc(pVal->length, 1); -// // /* cJSON conversion of legit BIGINT may overflow, -// // * use original string to do the conversion. -// // */ -// // errno = 0; -// // int64_t val = (int64_t)strtoll(root->numberstring, NULL, 10); -// // if (errno == ERANGE || !IS_VALID_BIGINT(val)) { -// // tscError("OTD:0x%"PRIx64" JSON value(%s) cannot fit in type(bigint)", info->id, root->numberstring); -// // return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; -// // } -// // *(int64_t *)(pVal->value) = val; -// //} else if (isValidFloat(root->numberstring)) { -// // pVal->type = TSDB_DATA_TYPE_DOUBLE; -// // pVal->length = (int16_t)tDataTypes[pVal->type].bytes; -// // pVal->value = tcalloc(pVal->length, 1); -// // *(double *)(pVal->value) = (double)(root->valuedouble); -// //} else { -// // return TSDB_CODE_TSC_INVALID_JSON_TYPE; -// //} -// if (isValidInteger(root->numberstring) || isValidFloat(root->numberstring)) { -// pVal->type = TSDB_DATA_TYPE_DOUBLE; -// pVal->length = (int16_t)tDataTypes[pVal->type].bytes; -// pVal->value = tcalloc(pVal->length, 1); -// *(double *)(pVal->value) = (double)(root->valuedouble); -// } -// -// break; -// } -// case cJSON_String: { -// /* set default JSON type to binary/nchar according to -// * user configured parameter tsDefaultJSONStrType -// */ -// if (strcasecmp(tsDefaultJSONStrType, "binary") == 0) { -// pVal->type = TSDB_DATA_TYPE_BINARY; -// } else if (strcasecmp(tsDefaultJSONStrType, "nchar") == 0) { -// pVal->type = TSDB_DATA_TYPE_NCHAR; -// } else { -// tscError("OTD:0x%"PRIx64" Invalid default JSON string type set from config %s", info->id, tsDefaultJSONStrType); -// return TSDB_CODE_TSC_INVALID_JSON_CONFIG; -// } -// //pVal->length = wcslen((wchar_t *)root->valuestring) * TSDB_NCHAR_SIZE; -// pVal->length = (int16_t)strlen(root->valuestring); -// pVal->value = tcalloc(pVal->length + 1, 1); -// memcpy(pVal->value, root->valuestring, pVal->length); -// break; -// } -// case cJSON_Object: { -// int32_t ret = parseValueFromJSONObj(root, pVal, info); -// if (ret != TSDB_CODE_SUCCESS) { -// tscError("OTD:0x%"PRIx64" Failed to parse timestamp from JSON Obj", info->id); -// return ret; -// } -// break; -// } -// default: -// return TSDB_CODE_TSC_INVALID_JSON; -// } -// -// return TSDB_CODE_SUCCESS; -//} -// -//static int32_t parseMetricValueFromJSON(cJSON *root, TAOS_SML_KV **pKVs, int *num_kvs, SSmlLinesInfo* info) { -// //skip timestamp -// TAOS_SML_KV *pVal = *pKVs + 1; -// char key[] = OTD_METRIC_VALUE_COLUMN_NAME; -// -// cJSON *metricVal = cJSON_GetObjectItem(root, "value"); -// if (metricVal == NULL) { -// return TSDB_CODE_TSC_INVALID_JSON; -// } -// -// int32_t ret = parseValueFromJSON(metricVal, pVal, info); -// if (ret != TSDB_CODE_SUCCESS) { -// return ret; -// } -// -// pVal->key = tcalloc(sizeof(key) + TS_BACKQUOTE_CHAR_SIZE, 1); -// memcpy(pVal->key, key, sizeof(key)); -// addEscapeCharToString(pVal->key, (int32_t)strlen(pVal->key)); -// -// *num_kvs += 1; -// return TSDB_CODE_SUCCESS; -// -//} -// -// -//static int32_t parseTagsFromJSON(cJSON *root, TAOS_SML_KV **pKVs, int *num_kvs, char **childTableName, -// SHashObj *pHash, SSmlLinesInfo* info) { -// int32_t ret = TSDB_CODE_SUCCESS; -// -// cJSON *tags = cJSON_GetObjectItem(root, "tags"); -// if (tags == NULL || tags->type != cJSON_Object) { -// return TSDB_CODE_TSC_INVALID_JSON; -// } -// -// //handle child table name -// size_t childTableNameLen = strlen(tsSmlChildTableName); -// char childTbName[TSDB_TABLE_NAME_LEN] = {0}; -// if (childTableNameLen != 0) { -// memcpy(childTbName, tsSmlChildTableName, childTableNameLen); -// cJSON *id = cJSON_GetObjectItem(tags, childTbName); -// if (id != NULL) { -// if (!cJSON_IsString(id)) { -// tscError("OTD:0x%"PRIx64" ID must be JSON string", info->id); -// return TSDB_CODE_TSC_INVALID_JSON; -// } -// size_t idLen = strlen(id->valuestring); -// *childTableName = tcalloc(idLen + TS_BACKQUOTE_CHAR_SIZE + 1, sizeof(char)); -// memcpy(*childTableName, id->valuestring, idLen); -// addEscapeCharToString(*childTableName, (int32_t)idLen); -// -// //check duplicate IDs -// cJSON_DeleteItemFromObject(tags, childTbName); -// id = cJSON_GetObjectItem(tags, childTbName); -// if (id != NULL) { -// return TSDB_CODE_TSC_DUP_TAG_NAMES; -// } -// } -// } -// -// int32_t tagNum = cJSON_GetArraySize(tags); -// //at least one tag pair required -// if (tagNum <= 0) { -// return TSDB_CODE_TSC_INVALID_JSON; -// } -// -// //allocate memory for tags -// *pKVs = tcalloc(tagNum, sizeof(TAOS_SML_KV)); -// TAOS_SML_KV *pkv = *pKVs; -// -// for (int32_t i = 0; i < tagNum; ++i) { -// cJSON *tag = cJSON_GetArrayItem(tags, i); -// if (tag == NULL) { -// return TSDB_CODE_TSC_INVALID_JSON; -// } -// //check duplicate keys -// if (checkDuplicateKey(tag->string, pHash, info)) { -// return TSDB_CODE_TSC_DUP_TAG_NAMES; -// } -// //key -// size_t keyLen = strlen(tag->string); -// if (keyLen > TSDB_COL_NAME_LEN - 1) { -// tscError("OTD:0x%"PRIx64" Tag key cannot exceeds %d characters in JSON", info->id, TSDB_COL_NAME_LEN - 1); -// return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; -// } -// pkv->key = tcalloc(keyLen + TS_BACKQUOTE_CHAR_SIZE + 1, sizeof(char)); -// strncpy(pkv->key, tag->string, keyLen); -// addEscapeCharToString(pkv->key, (int32_t)keyLen); -// //value -// ret = parseValueFromJSON(tag, pkv, info); -// if (ret != TSDB_CODE_SUCCESS) { -// return ret; -// } -// *num_kvs += 1; -// pkv++; -// -// } -// -// return ret; -// -//} -// -//static int32_t tscParseJSONPayload(cJSON *root, TAOS_SML_DATA_POINT* pSml, SSmlLinesInfo* info) { -// int32_t ret = TSDB_CODE_SUCCESS; -// -// if (!cJSON_IsObject(root)) { -// tscError("OTD:0x%"PRIx64" data point needs to be JSON object", info->id); -// return TSDB_CODE_TSC_INVALID_JSON; -// } -// -// int32_t size = cJSON_GetArraySize(root); -// //outmost json fields has to be exactly 4 -// if (size != OTD_JSON_FIELDS_NUM) { -// tscError("OTD:0x%"PRIx64" Invalid number of JSON fields in data point %d", info->id, size); -// return TSDB_CODE_TSC_INVALID_JSON; -// } -// -// //Parse metric -// ret = parseMetricFromJSON(root, pSml, info); -// if (ret != TSDB_CODE_SUCCESS) { -// tscError("OTD:0x%"PRIx64" Unable to parse metric from JSON payload", info->id); -// return ret; -// } -// tscDebug("OTD:0x%"PRIx64" Parse metric from JSON payload finished", info->id); -// -// //Parse timestamp -// ret = parseTimestampFromJSON(root, &pSml->fields, &pSml->fieldNum, info); -// if (ret) { -// tscError("OTD:0x%"PRIx64" Unable to parse timestamp from JSON payload", info->id); -// return ret; -// } -// tscDebug("OTD:0x%"PRIx64" Parse timestamp from JSON payload finished", info->id); -// -// //Parse metric value -// ret = parseMetricValueFromJSON(root, &pSml->fields, &pSml->fieldNum, info); -// if (ret) { -// tscError("OTD:0x%"PRIx64" Unable to parse metric value from JSON payload", info->id); -// return ret; -// } -// tscDebug("OTD:0x%"PRIx64" Parse metric value from JSON payload finished", info->id); -// -// //Parse tags -// SHashObj *keyHashTable = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); -// ret = parseTagsFromJSON(root, &pSml->tags, &pSml->tagNum, &pSml->childTableName, keyHashTable, info); -// if (ret) { -// tscError("OTD:0x%"PRIx64" Unable to parse tags from JSON payload", info->id); -// taosHashCleanup(keyHashTable); -// return ret; -// } -// tscDebug("OTD:0x%"PRIx64" Parse tags from JSON payload finished", info->id); -// taosHashCleanup(keyHashTable); -// -// return TSDB_CODE_SUCCESS; -//} -// -//static int32_t tscParseMultiJSONPayload(char* payload, SArray* points, SSmlLinesInfo* info) { -// int32_t payloadNum, ret; -// ret = TSDB_CODE_SUCCESS; -// -// if (payload == NULL) { -// tscError("OTD:0x%"PRIx64" empty JSON Payload", info->id); -// return TSDB_CODE_TSC_INVALID_JSON; -// } -// -// cJSON *root = cJSON_Parse(payload); -// //multiple data points must be sent in JSON array -// if (cJSON_IsObject(root)) { -// payloadNum = 1; -// } else if (cJSON_IsArray(root)) { -// payloadNum = cJSON_GetArraySize(root); -// } else { -// tscError("OTD:0x%"PRIx64" Invalid JSON Payload", info->id); -// ret = TSDB_CODE_TSC_INVALID_JSON; -// goto PARSE_JSON_OVER; -// } -// -// for (int32_t i = 0; i < payloadNum; ++i) { -// TAOS_SML_DATA_POINT point = {0}; -// cJSON *dataPoint = (payloadNum == 1 && cJSON_IsObject(root)) ? root : cJSON_GetArrayItem(root, i); -// -// ret = tscParseJSONPayload(dataPoint, &point, info); -// if (ret != TSDB_CODE_SUCCESS) { -// tscError("OTD:0x%"PRIx64" JSON data point parse failed", info->id); -// destroySmlDataPoint(&point); -// goto PARSE_JSON_OVER; -// } else { -// tscDebug("OTD:0x%"PRIx64" JSON data point parse success", info->id); -// } -// taosArrayPush(points, &point); -// } -// -//PARSE_JSON_OVER: -// cJSON_Delete(root); -// return ret; -//} -// -//int taos_insert_json_payload(TAOS* taos, char* payload, SMLProtocolType protocol, SMLTimeStampType tsType, int* affectedRows) { -// int32_t code = 0; -// -// SSmlLinesInfo* info = tcalloc(1, sizeof(SSmlLinesInfo)); -// info->id = genUID(); -// info->tsType = tsType; -// info->protocol = protocol; -// -// if (payload == NULL) { -// tscError("OTD:0x%"PRIx64" taos_insert_json_payload payload is NULL", info->id); -// tfree(info); -// code = TSDB_CODE_TSC_APP_ERROR; -// return code; -// } -// -// SArray* lpPoints = taosArrayInit(1, sizeof(TAOS_SML_DATA_POINT)); -// if (lpPoints == NULL) { -// tscError("OTD:0x%"PRIx64" taos_insert_json_payload failed to allocate memory", info->id); -// tfree(info); -// return TSDB_CODE_TSC_OUT_OF_MEMORY; -// } -// -// tscDebug("OTD:0x%"PRIx64" taos_insert_telnet_lines begin inserting %d points", info->id, 1); -// code = tscParseMultiJSONPayload(payload, lpPoints, info); -// size_t numPoints = taosArrayGetSize(lpPoints); -// -// if (code != 0) { -// goto cleanup; -// } -// -// TAOS_SML_DATA_POINT* points = TARRAY_GET_START(lpPoints); -// code = tscSmlInsert(taos, points, (int)numPoints, info); -// if (code != 0) { -// tscError("OTD:0x%"PRIx64" taos_insert_json_payload error: %s", info->id, tstrerror((code))); -// } -// if (affectedRows != NULL) { -// *affectedRows = info->affectedRows; -// } -// -//cleanup: -// tscDebug("OTD:0x%"PRIx64" taos_insert_json_payload finish inserting 1 Point. code: %d", info->id, code); -// points = TARRAY_GET_START(lpPoints); -// numPoints = taosArrayGetSize(lpPoints); -// for (int i = 0; i < numPoints; ++i) { -// destroySmlDataPoint(points+i); -// } -// -// taosArrayDestroy(&lpPoints); -// -// tfree(info); -// return code; -//} diff --git a/source/client/src/tscParseLineProtocol.c b/source/client/src/clientSml.c similarity index 51% rename from source/client/src/tscParseLineProtocol.c rename to source/client/src/clientSml.c index e0b4cc0d7a..dfbefcb0bf 100644 --- a/source/client/src/tscParseLineProtocol.c +++ b/source/client/src/clientSml.c @@ -3,7 +3,7 @@ #include #include -#include "tscParseLine.h" +#include "clientSml.h" #include "tdef.h" #include "ttypes.h" @@ -29,6 +29,7 @@ typedef struct { #define EQUAL '=' #define QUOTE '"' #define SLASH '\\' +#define tsMaxSQLStringLen (1024*1024) //================================================================================================= @@ -71,8 +72,8 @@ typedef enum { typedef struct { char sTableName[TSDB_TABLE_NAME_LEN]; - SArray* tags; //SArray - SArray* fields; //SArray + SHashObj *tags; + SHashObj *fields; } SCreateSTableActionInfo; typedef struct { @@ -339,21 +340,22 @@ static int32_t generateSchemaAction(SSchema* pointColField, SHashObj* dbAttrHash return 0; } -static int32_t buildColumnDescription(SSchema* field, +static int32_t buildColumnDescription(TAOS_SML_KV* field, char* buf, int32_t bufSize, int32_t* outBytes) { uint8_t type = field->type; - + char tname[TSDB_TABLE_NAME_LEN] = {0}; + memcpy(tname, field->key, field->keyLen); if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { - int32_t bytes = field->bytes - VARSTR_HEADER_SIZE; + int32_t bytes = field->length - VARSTR_HEADER_SIZE; if (type == TSDB_DATA_TYPE_NCHAR) { bytes = bytes/TSDB_NCHAR_SIZE; } int out = snprintf(buf, bufSize,"%s %s(%d)", - field->name,tDataTypes[field->type].name, bytes); + tname,tDataTypes[field->type].name, bytes); *outBytes = out; } else { int out = snprintf(buf, bufSize, "%s %s", - field->name, tDataTypes[type].name); + tname, tDataTypes[type].name); *outBytes = out; } @@ -364,7 +366,7 @@ static int32_t buildColumnDescription(SSchema* field, static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInfo* info) { int32_t code = 0; int32_t outBytes = 0; - char *result = (char *)calloc(1, tsMaxSQLStringLen+1); + char *result = (char *)taosMemoryCalloc(1, tsMaxSQLStringLen+1); int32_t capacity = tsMaxSQLStringLen + 1; uDebug("SML:0x%"PRIx64" apply schema action. action: %d", info->id, action->action); @@ -374,7 +376,7 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInf buildColumnDescription(action->alterSTable.field, result+n, capacity-n, &outBytes); TAOS_RES* res = taos_query(taos, result); //TODO async doAsyncQuery code = taos_errno(res); - char* errStr = taos_errstr(res); + const char* errStr = taos_errstr(res); char* begin = strstr(errStr, "duplicated column names"); bool tscDupColNames = (begin != NULL); if (code != TSDB_CODE_SUCCESS) { @@ -382,7 +384,8 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInf } taos_free_result(res); - if (code == TSDB_CODE_MND_FIELD_ALREAY_EXIST || code == TSDB_CODE_MND_TAG_ALREAY_EXIST || tscDupColNames) { +// if (code == TSDB_CODE_MND_FIELD_ALREADY_EXIST || code == TSDB_CODE_MND_TAG_ALREADY_EXIST || tscDupColNames) { + if (code == TSDB_CODE_MND_TAG_ALREADY_EXIST || tscDupColNames) { TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); code = taos_errno(res2); if (code != TSDB_CODE_SUCCESS) { @@ -399,7 +402,7 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInf result+n, capacity-n, &outBytes); TAOS_RES* res = taos_query(taos, result); //TODO async doAsyncQuery code = taos_errno(res); - char* errStr = taos_errstr(res); + const char* errStr = taos_errstr(res); char* begin = strstr(errStr, "duplicated column names"); bool tscDupColNames = (begin != NULL); if (code != TSDB_CODE_SUCCESS) { @@ -407,7 +410,8 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInf } taos_free_result(res); - if (code == TSDB_CODE_MND_TAG_ALREAY_EXIST || code == TSDB_CODE_MND_FIELD_ALREAY_EXIST || tscDupColNames) { +// if (code ==TSDB_CODE_MND_TAG_ALREADY_EXIST || code == TSDB_CODE_MND_FIELD_ALREAY_EXIST || tscDupColNames) { + if (code ==TSDB_CODE_MND_TAG_ALREADY_EXIST || tscDupColNames) { TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); code = taos_errno(res2); if (code != TSDB_CODE_SUCCESS) { @@ -429,7 +433,8 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInf } taos_free_result(res); - if (code == TSDB_CODE_MND_INVALID_COLUMN_LENGTH || code == TSDB_CODE_TSC_INVALID_COLUMN_LENGTH) { +// if (code == TSDB_CODE_MND_INVALID_COLUMN_LENGTH || code == TSDB_CODE_TSC_INVALID_COLUMN_LENGTH) { + if (code == TSDB_CODE_TSC_INVALID_COLUMN_LENGTH) { TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); code = taos_errno(res2); if (code != TSDB_CODE_SUCCESS) { @@ -451,7 +456,8 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInf } taos_free_result(res); - if (code == TSDB_CODE_MND_INVALID_TAG_LENGTH || code == TSDB_CODE_TSC_INVALID_TAG_LENGTH) { +// if (code == TSDB_CODE_MND_INVALID_TAG_LENGTH || code == TSDB_CODE_TSC_INVALID_TAG_LENGTH) { + if (code == TSDB_CODE_TSC_INVALID_TAG_LENGTH) { TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); code = taos_errno(res2); if (code != TSDB_CODE_SUCCESS) { @@ -465,24 +471,25 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInf case SCHEMA_ACTION_CREATE_STABLE: { int n = sprintf(result, "create stable %s (", action->createSTable.sTableName); char* pos = result + n; int freeBytes = capacity - n; - size_t numCols = taosArrayGetSize(action->createSTable.fields); - for (int32_t i = 0; i < numCols; ++i) { - SSchema* field = taosArrayGet(action->createSTable.fields, i); - buildColumnDescription(field, pos, freeBytes, &outBytes); + + TAOS_SML_KV **kv = taosHashIterate(action->createSTable.fields, NULL); + while(kv){ + buildColumnDescription(*kv, pos, freeBytes, &outBytes); pos += outBytes; freeBytes -= outBytes; *pos = ','; ++pos; --freeBytes; + kv = taosHashIterate(action->createSTable.fields, kv); } --pos; ++freeBytes; outBytes = snprintf(pos, freeBytes, ") tags ("); pos += outBytes; freeBytes -= outBytes; - size_t numTags = taosArrayGetSize(action->createSTable.tags); - for (int32_t i = 0; i < numTags; ++i) { - SSchema* field = taosArrayGet(action->createSTable.tags, i); - buildColumnDescription(field, pos, freeBytes, &outBytes); + kv = taosHashIterate(action->createSTable.tags, NULL); + while(kv){ + buildColumnDescription(*kv, pos, freeBytes, &outBytes); pos += outBytes; freeBytes -= outBytes; *pos = ','; ++pos; --freeBytes; + kv = taosHashIterate(action->createSTable.tags, kv); } pos--; ++freeBytes; outBytes = snprintf(pos, freeBytes, ")"); @@ -493,7 +500,7 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInf } taos_free_result(res); - if (code == TSDB_CODE_MND_TABLE_ALREADY_EXIST) { + if (code == TSDB_CODE_MND_STB_ALREADY_EXIST) { TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); code = taos_errno(res2); if (code != TSDB_CODE_SUCCESS) { @@ -680,643 +687,79 @@ static int32_t loadTableSchemaFromDB(TAOS* taos, char* tableName, SSmlSTableSche static int32_t modifyDBSchemas(TAOS* taos, SSmlLinesInfo* info) { int32_t code = 0; - size_t numStable = taosHashGetSize(info->superTables); SSmlSTableMeta** tableMetaSml = taosHashIterate(info->superTables, NULL); while (tableMetaSml) { SSmlSTableMeta* cTablePoints = *tableMetaSml; - if (NULL == pStmt->pCatalog) { - STMT_ERR_RET(catalogGetHandle(pStmt->taos->pAppInfo->clusterId, &pStmt->pCatalog)); - } - STableMeta *pTableMeta = NULL; - SEpSet ep = getEpSet_s(&pStmt->taos->pAppInfo->mgmtEp); - STMT_ERR_RET(catalogGetTableMeta(pStmt->pCatalog, pStmt->taos->pAppInfo->pTransporter, &ep, &pStmt->bInfo.sname, &pTableMeta)); + SEpSet ep = getEpSet_s(&info->taos->pAppInfo->mgmtEp); - if (pTableMeta->uid == pStmt->bInfo.tbUid) { - pStmt->bInfo.needParse = false; + size_t superTableLen = 0; + void *superTable = taosHashGetKey(tableMetaSml, &superTableLen); + SName pName = {TSDB_TABLE_NAME_T, info->taos->acctId, {0}, {0}}; + strcpy(pName.dbname, info->pRequest->pDb); + memcpy(pName.tname, superTable, superTableLen); - return TSDB_CODE_SUCCESS; - } + code = catalogGetSTableMeta(info->pCatalog, info->taos->pAppInfo->pTransporter, &ep, &pName, &pTableMeta); -// for (int i = 0; i < numStable; ++i) { - SSmlSTableSchema* pointSchema = taosArrayGet(stableSchemas, i); - SSmlSTableSchema dbSchema; - memset(&dbSchema, 0, sizeof(SSmlSTableSchema)); - - code = loadTableSchemaFromDB(taos, pointSchema->sTableName, &dbSchema, info); - if (code == TSDB_CODE_MND_INVALID_TABLE_NAME) { + if (code == TSDB_CODE_TDB_INVALID_TABLE_ID) { SSchemaAction schemaAction = {0}; schemaAction.action = SCHEMA_ACTION_CREATE_STABLE; memset(&schemaAction.createSTable, 0, sizeof(SCreateSTableActionInfo)); - memcpy(schemaAction.createSTable.sTableName, pointSchema->sTableName, TSDB_TABLE_NAME_LEN); - schemaAction.createSTable.tags = pointSchema->tags; - schemaAction.createSTable.fields = pointSchema->fields; + memcpy(schemaAction.createSTable.sTableName, superTable, superTableLen); + schemaAction.createSTable.tags = cTablePoints->tagHash; + schemaAction.createSTable.fields = cTablePoints->fieldHash; applySchemaAction(taos, &schemaAction, info); - code = loadTableSchemaFromDB(taos, pointSchema->sTableName, &dbSchema, info); + code = catalogGetSTableMeta(info->pCatalog, info->taos->pAppInfo->pTransporter, &ep, &pName, &pTableMeta); if (code != 0) { - uError("SML:0x%"PRIx64" reconcile point schema failed. can not create %s", info->id, pointSchema->sTableName); + uError("SML:0x%"PRIx64" reconcile point schema failed. can not create %s", info->id, schemaAction.createSTable.sTableName); return code; } - } - - if (code == TSDB_CODE_SUCCESS) { - pointSchema->precision = dbSchema.precision; - - size_t pointTagSize = taosArrayGetSize(pointSchema->tags); - size_t pointFieldSize = taosArrayGetSize(pointSchema->fields); - - SHashObj* dbTagHash = dbSchema.tagHash; - SHashObj* dbFieldHash = dbSchema.fieldHash; - - for (int j = 0; j < pointTagSize; ++j) { - SSchema* pointTag = taosArrayGet(pointSchema->tags, j); - SSchemaAction schemaAction = {0}; - bool actionNeeded = false; - generateSchemaAction(pointTag, dbTagHash, dbSchema.tags, true, pointSchema->sTableName, - &schemaAction, &actionNeeded, info); - if (actionNeeded) { - code = applySchemaAction(taos, &schemaAction, info); - if (code != 0) { - destroySmlSTableSchema(&dbSchema); - return code; - } - } - } - - SSchema* pointColTs = taosArrayGet(pointSchema->fields, 0); - SSchema* dbColTs = taosArrayGet(dbSchema.fields, 0); - memcpy(pointColTs->name, dbColTs->name, TSDB_COL_NAME_LEN); - - for (int j = 1; j < pointFieldSize; ++j) { - SSchema* pointCol = taosArrayGet(pointSchema->fields, j); - SSchemaAction schemaAction = {0}; - bool actionNeeded = false; - generateSchemaAction(pointCol, dbFieldHash, dbSchema.fields,false, pointSchema->sTableName, - &schemaAction, &actionNeeded, info); - if (actionNeeded) { - code = applySchemaAction(taos, &schemaAction, info); - if (code != 0) { - destroySmlSTableSchema(&dbSchema); - return code; - } - } - } - - pointSchema->precision = dbSchema.precision; - - destroySmlSTableSchema(&dbSchema); + }else if (code == TSDB_CODE_SUCCESS) { } else { uError("SML:0x%"PRIx64" load table meta error: %s", info->id, tstrerror(code)); return code; } + taosHashPut(info->metaHashObj, superTable, superTableLen, &pTableMeta, POINTER_BYTES); + tableMetaSml = taosHashIterate(info->superTables, tableMetaSml); } -// } return 0; } -static int32_t arrangePointsByChildTableName(TAOS_SML_DATA_POINT* points, int numPoints, - SHashObj* cname2points, SArray* stableSchemas, SSmlLinesInfo* info) { - for (int32_t i = 0; i < numPoints; ++i) { - TAOS_SML_DATA_POINT * point = points + i; - SSmlSTableSchema* stableSchema = taosArrayGet(stableSchemas, point->schemaIdx); - - for (int j = 0; j < point->tagNum; ++j) { - TAOS_SML_KV* kv = point->tags + j; - if (kv->type == TSDB_DATA_TYPE_TIMESTAMP) { - int64_t ts = *(int64_t*)(kv->value); - ts = convertTimePrecision(ts, TSDB_TIME_PRECISION_NANO, stableSchema->precision); - *(int64_t*)(kv->value) = ts; - } - } - - for (int j = 0; j < point->fieldNum; ++j) { - TAOS_SML_KV* kv = point->fields + j; - if (kv->type == TSDB_DATA_TYPE_TIMESTAMP) { - int64_t ts = *(int64_t*)(kv->value); - ts = convertTimePrecision(ts, TSDB_TIME_PRECISION_NANO, stableSchema->precision); - *(int64_t*)(kv->value) = ts; - } - } - - SArray* cTablePoints = NULL; - SArray** pCTablePoints = taosHashGet(cname2points, point->childTableName, strlen(point->childTableName)); - if (pCTablePoints) { - cTablePoints = *pCTablePoints; - } else { - cTablePoints = taosArrayInit(64, sizeof(point)); - taosHashPut(cname2points, point->childTableName, strlen(point->childTableName), &cTablePoints, POINTER_BYTES); - } - taosArrayPush(cTablePoints, &point); - } - - return 0; -} - -static int32_t applyChildTableDataPointsWithInsertSQL(TAOS* taos, char* cTableName, char* sTableName, SSmlSTableSchema* sTableSchema, - SArray* cTablePoints, size_t rowSize, SSmlLinesInfo* info) { - int32_t code = TSDB_CODE_SUCCESS; - size_t numTags = taosArrayGetSize(sTableSchema->tags); - size_t numCols = taosArrayGetSize(sTableSchema->fields); - size_t rows = taosArrayGetSize(cTablePoints); - SArray* tagsSchema = sTableSchema->tags; - SArray* colsSchema = sTableSchema->fields; - - TAOS_SML_KV* tagKVs[TSDB_MAX_TAGS] = {0}; - for (int i = 0; i < rows; ++i) { - TAOS_SML_DATA_POINT* pDataPoint = taosArrayGetP(cTablePoints, i); - for (int j = 0; j < pDataPoint->tagNum; ++j) { - TAOS_SML_KV* kv = pDataPoint->tags + j; - tagKVs[kv->fieldSchemaIdx] = kv; - } - } - - char* sql = taosMemoryMalloc(tsMaxSQLStringLen + 1); - if (sql == NULL) { - uError("taosMemoryMalloc sql memory error"); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - int32_t freeBytes = tsMaxSQLStringLen + 1; - int32_t totalLen = 0; - totalLen += sprintf(sql, "insert into %s using %s (", cTableName, sTableName); - for (int i = 0; i < numTags; ++i) { - SSchema* tagSchema = taosArrayGet(tagsSchema, i); - totalLen += snprintf(sql + totalLen, freeBytes - totalLen, "%s,", tagSchema->name); - } - --totalLen; - totalLen += snprintf(sql + totalLen, freeBytes - totalLen, ")"); - - totalLen += snprintf(sql + totalLen, freeBytes - totalLen, " tags ("); - - // for (int i = 0; i < numTags; ++i) { - // snprintf(sql+strlen(sql), freeBytes-strlen(sql), "?,"); - // } - for (int i = 0; i < numTags; ++i) { - if (tagKVs[i] == NULL) { - totalLen += snprintf(sql + totalLen, freeBytes - totalLen, "NULL,"); - } else { - TAOS_SML_KV* kv = tagKVs[i]; - size_t beforeLen = totalLen; - int32_t len = 0; - converToStr(sql + beforeLen, kv->type, kv->value, kv->length, &len); - totalLen += len; - totalLen += snprintf(sql + totalLen, freeBytes - totalLen, ","); - } - } - --totalLen; - totalLen += snprintf(sql + totalLen, freeBytes - totalLen, ") ("); - - for (int i = 0; i < numCols; ++i) { - SSchema* colSchema = taosArrayGet(colsSchema, i); - totalLen += snprintf(sql + totalLen, freeBytes - totalLen, "%s,", colSchema->name); - } - --totalLen; - totalLen += snprintf(sql + totalLen, freeBytes - totalLen, ") values "); - - TAOS_SML_KV** colKVs = taosMemoryMalloc(numCols * sizeof(TAOS_SML_KV*)); - for (int r = 0; r < rows; ++r) { - totalLen += snprintf(sql + totalLen, freeBytes - totalLen, "("); - - memset(colKVs, 0, numCols * sizeof(TAOS_SML_KV*)); - - TAOS_SML_DATA_POINT* point = taosArrayGetP(cTablePoints, r); - for (int i = 0; i < point->fieldNum; ++i) { - TAOS_SML_KV* kv = point->fields + i; - colKVs[kv->fieldSchemaIdx] = kv; - } - - for (int i = 0; i < numCols; ++i) { - if (colKVs[i] == NULL) { - totalLen += snprintf(sql + totalLen, freeBytes - totalLen, "NULL,"); - } else { - TAOS_SML_KV* kv = colKVs[i]; - size_t beforeLen = totalLen; - int32_t len = 0; - converToStr(sql + beforeLen, kv->type, kv->value, kv->length, &len); - totalLen += len; - totalLen += snprintf(sql + totalLen, freeBytes - totalLen, ","); - } - } - --totalLen; - totalLen += snprintf(sql + totalLen, freeBytes - totalLen, ")"); - } - taosMemoryFree(colKVs); - sql[totalLen] = '\0'; - - uDebug("SML:0x%" PRIx64 " insert child table table %s of super table %s sql: %s", info->id, cTableName, sTableName, - sql); - - bool tryAgain = false; - int32_t try = 0; - do { - TAOS_RES* res = taos_query(taos, sql); - code = taos_errno(res); - if (code != 0) { - uError("SML:0x%"PRIx64 " taos_query return %d:%s", info->id, code, taos_errstr(res)); - } - - uDebug("SML:0x%"PRIx64 " taos_query inserted %d rows", info->id, taos_affected_rows(res)); - info->affectedRows += taos_affected_rows(res); - taos_free_result(res); - - tryAgain = false; - if ((code == TSDB_CODE_TDB_INVALID_TABLE_ID - || code == TSDB_CODE_VND_INVALID_VGROUP_ID - || code == TSDB_CODE_TDB_TABLE_RECONFIGURE - || code == TSDB_CODE_APP_NOT_READY - || code == TSDB_CODE_RPC_NETWORK_UNAVAIL) && try++ < TSDB_MAX_REPLICA) { - tryAgain = true; - } - - if (code == TSDB_CODE_TDB_INVALID_TABLE_ID || code == TSDB_CODE_VND_INVALID_VGROUP_ID) { - TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); - int32_t code2 = taos_errno(res2); - if (code2 != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " insert child table by sql. reset query cache. error: %s", info->id, taos_errstr(res2)); - } - taos_free_result(res2); - if (tryAgain) { - taosMsleep(100 * (2 << try)); - } - } - - if (code == TSDB_CODE_APP_NOT_READY || code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { - if (tryAgain) { - taosMsleep( 100 * (2 << try)); - } - } - } while (tryAgain); - - taosMemoryFree(sql); - - return code; -} - -static int32_t applyChildTableDataPointsWithStmt(TAOS* taos, char* cTableName, char* sTableName, SSmlSTableSchema* sTableSchema, - SArray* cTablePoints, size_t rowSize, SSmlLinesInfo* info) { - size_t numTags = taosArrayGetSize(sTableSchema->tags); - size_t numCols = taosArrayGetSize(sTableSchema->fields); - size_t rows = taosArrayGetSize(cTablePoints); - - TAOS_SML_KV* tagKVs[TSDB_MAX_TAGS] = {0}; - for (int i= 0; i < rows; ++i) { - TAOS_SML_DATA_POINT * pDataPoint = taosArrayGetP(cTablePoints, i); - for (int j = 0; j < pDataPoint->tagNum; ++j) { - TAOS_SML_KV* kv = pDataPoint->tags + j; - tagKVs[kv->fieldSchemaIdx] = kv; - } - } - - //tag bind - SArray* tagBinds = taosArrayInit(numTags, sizeof(TAOS_BIND)); - taosArraySetSize(tagBinds, numTags); - int isNullColBind = TSDB_TRUE; - for (int j = 0; j < numTags; ++j) { - TAOS_BIND* bind = taosArrayGet(tagBinds, j); - bind->is_null = &isNullColBind; - } - for (int j = 0; j < numTags; ++j) { - if (tagKVs[j] == NULL) continue; - TAOS_SML_KV* kv = tagKVs[j]; - TAOS_BIND* bind = taosArrayGet(tagBinds, kv->fieldSchemaIdx); - bind->buffer_type = kv->type; - bind->length = taosMemoryMalloc(sizeof(uintptr_t*)); - *bind->length = kv->length; - bind->buffer = kv->value; - bind->is_null = NULL; - } - - //rows bind - SArray* rowsBind = taosArrayInit(rows, POINTER_BYTES); - for (int i = 0; i < rows; ++i) { - TAOS_SML_DATA_POINT* point = taosArrayGetP(cTablePoints, i); - - TAOS_BIND* colBinds = calloc(numCols, sizeof(TAOS_BIND)); - if (colBinds == NULL) { - uError("SML:0x%"PRIx64" taos_sml_insert insert points, failed to allocated memory for TAOS_BIND, " - "num of rows: %zu, num of cols: %zu", info->id, rows, numCols); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - for (int j = 0; j < numCols; ++j) { - TAOS_BIND* bind = colBinds + j; - bind->is_null = &isNullColBind; - } - for (int j = 0; j < point->fieldNum; ++j) { - TAOS_SML_KV* kv = point->fields + j; - TAOS_BIND* bind = colBinds + kv->fieldSchemaIdx; - bind->buffer_type = kv->type; - bind->length = taosMemoryMalloc(sizeof(uintptr_t*)); - *bind->length = kv->length; - bind->buffer = kv->value; - bind->is_null = NULL; - } - taosArrayPush(rowsBind, &colBinds); - } - - int32_t code = 0; - code = insertChildTablePointsBatch(taos, cTableName, sTableName, sTableSchema->tags, tagBinds, sTableSchema->fields, rowsBind, rowSize, info); - if (code != 0) { - uError("SML:0x%"PRIx64" insert into child table %s failed. error %s", info->id, cTableName, tstrerror(code)); - } - - //taosMemoryFree rows bind - for (int i = 0; i < rows; ++i) { - TAOS_BIND* colBinds = taosArrayGetP(rowsBind, i); - for (int j = 0; j < numCols; ++j) { - TAOS_BIND* bind = colBinds + j; - taosMemoryFree(bind->length); - } - taosMemoryFree(colBinds); - } - taosArrayDestroy(&rowsBind); - //taosMemoryFree tag bind - for (int i = 0; i < taosArrayGetSize(tagBinds); ++i) { - TAOS_BIND* bind = taosArrayGet(tagBinds, i); - taosMemoryFree(bind->length); - } - taosArrayDestroy(&tagBinds); - return code; -} - -static int32_t insertChildTablePointsBatch(TAOS* taos, char* cTableName, char* sTableName, - SArray* tagsSchema, SArray* tagsBind, - SArray* colsSchema, SArray* rowsBind, - size_t rowSize, SSmlLinesInfo* info) { - size_t numTags = taosArrayGetSize(tagsSchema); - size_t numCols = taosArrayGetSize(colsSchema); - char* sql = taosMemoryMalloc(tsMaxSQLStringLen+1); - if (sql == NULL) { - uError("taosMemoryMalloc sql memory error"); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - int32_t freeBytes = tsMaxSQLStringLen + 1 ; - sprintf(sql, "insert into ? using %s (", sTableName); - for (int i = 0; i < numTags; ++i) { - SSchema* tagSchema = taosArrayGet(tagsSchema, i); - snprintf(sql+strlen(sql), freeBytes-strlen(sql), "%s,", tagSchema->name); - } - snprintf(sql + strlen(sql) - 1, freeBytes-strlen(sql)+1, ")"); - - snprintf(sql + strlen(sql), freeBytes-strlen(sql), " tags ("); - - for (int i = 0; i < numTags; ++i) { - snprintf(sql+strlen(sql), freeBytes-strlen(sql), "?,"); - } - snprintf(sql + strlen(sql) - 1, freeBytes-strlen(sql)+1, ") ("); - - for (int i = 0; i < numCols; ++i) { - SSchema* colSchema = taosArrayGet(colsSchema, i); - snprintf(sql+strlen(sql), freeBytes-strlen(sql), "%s,", colSchema->name); - } - snprintf(sql + strlen(sql)-1, freeBytes-strlen(sql)+1, ") values ("); - - for (int i = 0; i < numCols; ++i) { - snprintf(sql+strlen(sql), freeBytes-strlen(sql), "?,"); - } - snprintf(sql + strlen(sql)-1, freeBytes-strlen(sql)+1, ")"); - sql[strlen(sql)] = '\0'; - - uDebug("SML:0x%"PRIx64" insert child table table %s of super table %s : %s", info->id, cTableName, sTableName, sql); - - size_t maxBatchSize = TSDB_MAX_WAL_SIZE/rowSize * 2 / 3; - size_t rows = taosArrayGetSize(rowsBind); - size_t batchSize = MIN(maxBatchSize, rows); - uDebug("SML:0x%"PRIx64" insert rows into child table %s. num of rows: %zu, batch size: %zu", - info->id, cTableName, rows, batchSize); - SArray* batchBind = taosArrayInit(batchSize, POINTER_BYTES); - int32_t code = TSDB_CODE_SUCCESS; - for (int i = 0; i < rows;) { - int j = i; - for (; j < i + batchSize && j i) { - uDebug("SML:0x%"PRIx64" insert child table batch from line %d to line %d.", info->id, i, j - 1); - code = doInsertChildTablePoints(taos, sql, cTableName, tagsBind, batchBind, info); - if (code != 0) { - taosArrayDestroy(&batchBind); - tfree(sql); - return code; - } - taosArrayClear(batchBind); - } - i = j; - } - taosArrayDestroy(&batchBind); - tfree(sql); - return code; - -} -static int32_t doInsertChildTablePoints(TAOS* taos, char* sql, char* cTableName, SArray* tagsBind, SArray* batchBind, - SSmlLinesInfo* info) { - int32_t code = 0; - - TAOS_STMT* stmt = taos_stmt_init(taos); - if (stmt == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - code = taos_stmt_prepare(stmt, sql, (unsigned long)strlen(sql)); - - if (code != 0) { - uError("SML:0x%"PRIx64" taos_stmt_prepare return %d:%s", info->id, code, taos_stmt_errstr(stmt)); - taos_stmt_close(stmt); - return code; - } - - bool tryAgain = false; - int32_t try = 0; - do { - code = taos_stmt_set_tbname_tags(stmt, cTableName, TARRAY_GET_START(tagsBind)); - if (code != 0) { - uError("SML:0x%"PRIx64" taos_stmt_set_tbname return %d:%s", info->id, code, taos_stmt_errstr(stmt)); - - int affectedRows = taos_stmt_affected_rows(stmt); - info->affectedRows += affectedRows; - - taos_stmt_close(stmt); - return code; - } - - size_t rows = taosArrayGetSize(batchBind); - for (int32_t i = 0; i < rows; ++i) { - TAOS_BIND* colsBinds = taosArrayGetP(batchBind, i); - code = taos_stmt_bind_param(stmt, colsBinds); - if (code != 0) { - uError("SML:0x%"PRIx64" taos_stmt_bind_param return %d:%s", info->id, code, taos_stmt_errstr(stmt)); - - int affectedRows = taos_stmt_affected_rows(stmt); - info->affectedRows += affectedRows; - - taos_stmt_close(stmt); - return code; - } - code = taos_stmt_add_batch(stmt); - if (code != 0) { - uError("SML:0x%"PRIx64" taos_stmt_add_batch return %d:%s", info->id, code, taos_stmt_errstr(stmt)); - - int affectedRows = taos_stmt_affected_rows(stmt); - info->affectedRows += affectedRows; - - taos_stmt_close(stmt); - return code; - } - } - - code = taos_stmt_execute(stmt); - if (code != 0) { - uError("SML:0x%"PRIx64" taos_stmt_execute return %d:%s, try:%d", info->id, code, taos_stmt_errstr(stmt), try); - } - uDebug("SML:0x%"PRIx64" taos_stmt_execute inserted %d rows", info->id, taos_stmt_affected_rows(stmt)); - - tryAgain = false; - if ((code == TSDB_CODE_TDB_INVALID_TABLE_ID - || code == TSDB_CODE_VND_INVALID_VGROUP_ID - || code == TSDB_CODE_TDB_TABLE_RECONFIGURE - || code == TSDB_CODE_APP_NOT_READY - || code == TSDB_CODE_RPC_NETWORK_UNAVAIL) && try++ < TSDB_MAX_REPLICA) { - tryAgain = true; - } - - if (code == TSDB_CODE_TDB_INVALID_TABLE_ID || code == TSDB_CODE_VND_INVALID_VGROUP_ID) { - TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); - int32_t code2 = taos_errno(res2); - if (code2 != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " insert child table. reset query cache. error: %s", info->id, taos_errstr(res2)); - } - taos_free_result(res2); - if (tryAgain) { - taosMsleep(100 * (2 << try)); - } - } - if (code == TSDB_CODE_APP_NOT_READY || code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { - if (tryAgain) { - taosMsleep( 100 * (2 << try)); - } - } - } while (tryAgain); - - int affectedRows = taos_stmt_affected_rows(stmt); - info->affectedRows += affectedRows; - - taos_stmt_close(stmt); - return code; - - return 0; -} - -static int32_t applyChildTableDataPoints(TAOS* taos, char* cTableName, char* sTableName, SSmlSTableSchema* sTableSchema, - SArray* cTablePoints, size_t rowSize, SSmlLinesInfo* info) { - int32_t code = TSDB_CODE_SUCCESS; - size_t childTableDataPoints = taosArrayGetSize(cTablePoints); - if (childTableDataPoints < 10) { - code = applyChildTableDataPointsWithInsertSQL(taos, cTableName, sTableName, sTableSchema, cTablePoints, rowSize, info); - } else { - code = applyChildTableDataPointsWithStmt(taos, cTableName, sTableName, sTableSchema, cTablePoints, rowSize, info); - } - return code; -} - -static int32_t applyDataPoints(TAOS* taos, TAOS_SML_DATA_POINT* points, int32_t numPoints, SArray* stableSchemas, SSmlLinesInfo* info) { +static int32_t applyDataPoints(TAOS* taos, SSmlLinesInfo* info) { int32_t code = TSDB_CODE_SUCCESS; - SHashObj* cname2points = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); - arrangePointsByChildTableName(points, numPoints, cname2points, stableSchemas, info); + TAOS_SML_DATA_POINT_TAGS** oneTable = taosHashIterate(info->childTables, NULL); + while (oneTable) { + TAOS_SML_DATA_POINT_TAGS* tableData = *oneTable; - SArray** pCTablePoints = taosHashIterate(cname2points, NULL); - while (pCTablePoints) { - SArray* cTablePoints = *pCTablePoints; + SName pName = {TSDB_TABLE_NAME_T, info->taos->acctId, {0}, {0}}; + strcpy(pName.dbname, info->pRequest->pDb); + memcpy(pName.tname, tableData->childTableName, strlen(tableData->childTableName)); + SEpSet ep = getEpSet_s(&info->taos->pAppInfo->mgmtEp); + SVgroupInfo vg; + catalogGetTableHashVgroup(info->pCatalog, info->taos->pAppInfo->pTransporter, &ep, &pName, &vg); + taosHashPut(info->pVgHash, (const char*)&vg.vgId, sizeof(vg.vgId), (char*)&vg, sizeof(vg)); - TAOS_SML_DATA_POINT* point = taosArrayGetP(cTablePoints, 0); - SSmlSTableSchema* sTableSchema = taosArrayGet(stableSchemas, point->schemaIdx); + STableMeta** pMeta = taosHashGet(info->metaHashObj, tableData->sTableName, tableData->sTableNameLen); + ASSERT (NULL != pMeta && NULL != *pMeta); + (*pMeta)->vgId = vg.vgId; + (*pMeta)->uid = tableData->uid; - size_t rowSize = 0; - size_t numCols = taosArrayGetSize(sTableSchema->fields); - for (int i = 0; i < numCols; ++i) { - SSchema* colSchema = taosArrayGet(sTableSchema->fields, i); - rowSize += colSchema->bytes; - } + smlBind(info->exec, tableData->tags, tableData->cols, *pMeta, info->msgBuf, info->msgLen); - uDebug("SML:0x%"PRIx64" apply child table points. child table: %s of super table %s, row size: %zu", - info->id, point->childTableName, point->stableName, rowSize); - code = applyChildTableDataPoints(taos, point->childTableName, point->stableName, sTableSchema, cTablePoints, rowSize, info); - if (code != 0) { - uError("SML:0x%"PRIx64" Apply child table points failed. child table %s, error %s", info->id, point->childTableName, tstrerror(code)); - goto cleanup; - } - - uDebug("SML:0x%"PRIx64" successfully applied data points of child table %s", info->id, point->childTableName); - - pCTablePoints = taosHashIterate(cname2points, pCTablePoints); + oneTable = taosHashIterate(info->childTables, oneTable); } -cleanup: - pCTablePoints = taosHashIterate(cname2points, NULL); - while (pCTablePoints) { - SArray* pPoints = *pCTablePoints; - taosArrayDestroy(&pPoints); - pCTablePoints = taosHashIterate(cname2points, pCTablePoints); - } - taosHashCleanup(cname2points); - return code; -} + smlBuildOutput(info->exec, info->pVgHash); + launchQueryImpl(info->pRequest, info->pQuery, TSDB_CODE_SUCCESS, true); + if(info->pRequest->code != TSDB_CODE_SUCCESS){ -static int doSmlInsertOneDataPoint(TAOS* taos, TAOS_SML_DATA_POINT* point, SSmlLinesInfo* info) { - int32_t code = TSDB_CODE_SUCCESS; - - if (!point->childTableName) { - int tableNameLen = TSDB_TABLE_NAME_LEN; - point->childTableName = calloc(1, tableNameLen + 1); - getSmlMd5ChildTableName(point, point->childTableName, &tableNameLen, info); - point->childTableName[tableNameLen] = '\0'; } - STableMeta* tableMeta = NULL; - int32_t ret = getSuperTableMetaFromLocalCache(taos, point->stableName, &tableMeta, info); - if (ret != TSDB_CODE_SUCCESS) { - return ret; - } - uint8_t precision = tableMeta->tableInfo.precision; - taosMemoryFree(tableMeta); - - char* sql = taosMemoryMalloc(TSDB_MAX_SQL_LEN + 1); - int freeBytes = TSDB_MAX_SQL_LEN; - int sqlLen = 0; - sqlLen += snprintf(sql + sqlLen, freeBytes - sqlLen, "insert into %s(", point->childTableName); - for (int col = 0; col < point->fieldNum; ++col) { - TAOS_SML_KV* kv = point->fields + col; - sqlLen += snprintf(sql + sqlLen, freeBytes - sqlLen, "%s,", kv->key); - } - --sqlLen; - sqlLen += snprintf(sql + sqlLen, freeBytes - sqlLen, ") values ("); - TAOS_SML_KV* tsField = point->fields + 0; - int64_t ts = *(int64_t*)(tsField->value); - ts = convertTimePrecision(ts, TSDB_TIME_PRECISION_NANO, precision); - sqlLen += snprintf(sql + sqlLen, freeBytes - sqlLen, "%" PRId64 ",", ts); - for (int col = 1; col < point->fieldNum; ++col) { - TAOS_SML_KV* kv = point->fields + col; - int32_t len = 0; - converToStr(sql + sqlLen, kv->type, kv->value, kv->length, &len); - sqlLen += len; - sqlLen += snprintf(sql + sqlLen, freeBytes - sqlLen, ","); - } - --sqlLen; - sqlLen += snprintf(sql + sqlLen, freeBytes - sqlLen, ")"); - sql[sqlLen] = 0; - - uDebug("SML:0x%" PRIx64 " insert child table table %s of super table %s sql: %s", info->id, - point->childTableName, point->stableName, sql); - TAOS_RES* res = taos_query(taos, sql); - taosMemoryFree(sql); - code = taos_errno(res); - info->affectedRows = taos_affected_rows(res); - taos_free_result(res); - + info->affectedRows = taos_affected_rows(info->pRequest); return code; } @@ -1333,18 +776,12 @@ int tscSmlInsert(TAOS* taos, SSmlLinesInfo* info) { } uDebug("SML:0x%"PRIx64" apply data points", info->id); - code = applyDataPoints(taos, points, numPoint, stableSchemas, info); + code = applyDataPoints(taos, info); if (code != 0) { uError("SML:0x%"PRIx64" error apply data points : %s", info->id, tstrerror(code)); } clean_up: - for (int i = 0; i < taosArrayGetSize(stableSchemas); ++i) { - SSmlSTableSchema* schema = taosArrayGet(stableSchemas, i); - taosArrayDestroy(&schema->fields); - taosArrayDestroy(&schema->tags); - } - taosArrayDestroy(&stableSchemas); return code; } @@ -1398,16 +835,6 @@ static void escapeSpecialCharacter(uint8_t field, const char **pos) { *pos = cur; } -char* addEscapeCharToString(char *str, int32_t len) { - if (str == NULL) { - return NULL; - } - memmove(str + 1, str, len); - str[0] = str[len + 1] = TS_BACKQUOTE_CHAR; - str[len + 2] = '\0'; - return str; -} - bool isValidInteger(char *str) { char *c = str; if (*c != '+' && *c != '-' && !isdigit(*c)) { @@ -1668,67 +1095,6 @@ static bool isNchar(char *pVal, uint16_t len) { return false; } -static int32_t isTimeStamp(char *pVal, uint16_t len, SMLTimeStampType *tsType, SSmlLinesInfo* info) { - if (len == 0) { - return TSDB_CODE_SUCCESS; - } - if ((len == 1) && pVal[0] == '0') { - *tsType = SML_TIME_STAMP_NOW; - return TSDB_CODE_SUCCESS; - } - - for (int i = 0; i < len; ++i) { - if(!isdigit(pVal[i])) { - return TSDB_CODE_TSC_INVALID_TIME_STAMP; - } - } - - /* For InfluxDB line protocol use user passed timestamp precision - * For OpenTSDB protocols only 10 digit(seconds) or 13 digits(milliseconds) - * precision allowed - */ - if (info->protocol == TSDB_SML_LINE_PROTOCOL) { - if (info->tsType != SML_TIME_STAMP_NOT_CONFIGURED) { - *tsType = info->tsType; - } else { - *tsType = SML_TIME_STAMP_NANO_SECONDS; - } - } else if (info->protocol == TSDB_SML_TELNET_PROTOCOL) { - if (len == SML_TIMESTAMP_SECOND_DIGITS) { - *tsType = SML_TIME_STAMP_SECONDS; - } else if (len == SML_TIMESTAMP_MILLI_SECOND_DIGITS) { - *tsType = SML_TIME_STAMP_MILLI_SECONDS; - } else { - return TSDB_CODE_TSC_INVALID_TIME_STAMP; - } - } - return TSDB_CODE_SUCCESS; - - //if (pVal[len - 1] == 's') { - // switch (pVal[len - 2]) { - // case 'm': - // *tsType = SML_TIME_STAMP_MILLI_SECONDS; - // break; - // case 'u': - // *tsType = SML_TIME_STAMP_MICRO_SECONDS; - // break; - // case 'n': - // *tsType = SML_TIME_STAMP_NANO_SECONDS; - // break; - // default: - // if (isdigit(pVal[len - 2])) { - // *tsType = SML_TIME_STAMP_SECONDS; - // break; - // } else { - // return false; - // } - // } - // //printf("Type is timestamp(%s)\n", pVal); - // return true; - //} - //return false; -} - static bool convertStrToNumber(TAOS_SML_KV *pVal, char *str, SSmlLinesInfo* info) { errno = 0; uint8_t type = pVal->type; @@ -1987,107 +1353,7 @@ bool convertSmlValueType(TAOS_SML_KV *pVal, char *value, return false; } -static int32_t getTimeStampValue(char *value, uint16_t len, - SMLTimeStampType type, int64_t *ts, SSmlLinesInfo* info) { - //No appendix or no timestamp given (len = 0) - if (len != 0 && type != SML_TIME_STAMP_NOW) { - *ts = (int64_t)strtoll(value, NULL, 10); - } else { - type = SML_TIME_STAMP_NOW; - } - switch (type) { - case SML_TIME_STAMP_NOW: { - *ts = taosGetTimestampNs(); - break; - } - case SML_TIME_STAMP_HOURS: { - *ts = (int64_t)(*ts * 3600 * 1e9); - break; - } - case SML_TIME_STAMP_MINUTES: { - *ts = (int64_t)(*ts * 60 * 1e9); - break; - } - case SML_TIME_STAMP_SECONDS: { - *ts = (int64_t)(*ts * 1e9); - break; - } - case SML_TIME_STAMP_MILLI_SECONDS: { - *ts = convertTimePrecision(*ts, TSDB_TIME_PRECISION_MILLI, TSDB_TIME_PRECISION_NANO); - break; - } - case SML_TIME_STAMP_MICRO_SECONDS: { - *ts = convertTimePrecision(*ts, TSDB_TIME_PRECISION_MICRO, TSDB_TIME_PRECISION_NANO); - break; - } - case SML_TIME_STAMP_NANO_SECONDS: { - *ts = *ts * 1; - break; - } - default: { - return TSDB_CODE_TSC_INVALID_TIME_STAMP; - } - } - return TSDB_CODE_SUCCESS; -} - -int32_t convertSmlTimeStamp(TAOS_SML_KV *pVal, char *value, - uint16_t len, SSmlLinesInfo* info) { - int32_t ret; - SMLTimeStampType type = SML_TIME_STAMP_NOW; - int64_t tsVal; - - ret = isTimeStamp(value, len, &type, info); - if (ret != TSDB_CODE_SUCCESS) { - return ret; - } - - ret = getTimeStampValue(value, len, type, &tsVal, info); - if (ret != TSDB_CODE_SUCCESS) { - return ret; - } - uDebug("SML:0x%"PRIx64"Timestamp after conversion:%"PRId64, info->id, tsVal); - - pVal->type = TSDB_DATA_TYPE_TIMESTAMP; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - pVal->value = calloc(pVal->length, 1); - memcpy(pVal->value, &tsVal, pVal->length); - return TSDB_CODE_SUCCESS; -} - -static int32_t parseSmlTimeStamp(TAOS_SML_KV **pTS, const char **index, SSmlLinesInfo* info) { - const char *start, *cur; - int32_t ret = TSDB_CODE_SUCCESS; - int len = 0; - char key[] = "ts"; - char *value = NULL; - - start = cur = *index; - *pTS = calloc(1, sizeof(TAOS_SML_KV)); - - while(*cur != '\0') { - cur++; - len++; - } - - if (len > 0) { - value = calloc(len + 1, 1); - memcpy(value, start, len); - } - - ret = convertSmlTimeStamp(*pTS, value, len, info); - if (ret) { - taosMemoryFree(value); - taosMemoryFree(*pTS); - return ret; - } - taosMemoryFree(value); - - (*pTS)->key = calloc(sizeof(key), 1); - memcpy((*pTS)->key, key, sizeof(key)); - return ret; -} bool checkDuplicateKey(char *key, SHashObj *pHash, SSmlLinesInfo* info) { char *val = NULL; @@ -2103,458 +1369,6 @@ bool checkDuplicateKey(char *key, SHashObj *pHash, SSmlLinesInfo* info) { return false; } -static int32_t parseSmlKey(TAOS_SML_KV *pKV, const char **index, SHashObj *pHash, SSmlLinesInfo* info) { - const char *cur = *index; - char key[TSDB_COL_NAME_LEN + 1]; // +1 to avoid key[len] over write - int16_t len = 0; - - while (*cur != '\0') { - if (len > TSDB_COL_NAME_LEN - 1) { - uError("SML:0x%"PRIx64" Key field cannot exceeds %d characters", info->id, TSDB_COL_NAME_LEN - 1); - return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; - } - //unescaped '=' identifies a tag key - if (*cur == '=' && *(cur - 1) != '\\') { - break; - } - //Escape special character - if (*cur == '\\') { - escapeSpecialCharacter(2, &cur); - } - key[len] = *cur; - cur++; - len++; - } - if (len == 0) { - return TSDB_CODE_TSC_LINE_SYNTAX_ERROR; - } - key[len] = '\0'; - - if (checkDuplicateKey(key, pHash, info)) { - return TSDB_CODE_TSC_LINE_SYNTAX_ERROR; - } - - pKV->key = calloc(len + TS_BACKQUOTE_CHAR_SIZE + 1, 1); - memcpy(pKV->key, key, len + 1); - addEscapeCharToString(pKV->key, len); - uDebug("SML:0x%"PRIx64" Key:%s|len:%d", info->id, pKV->key, len); - *index = cur + 1; - return TSDB_CODE_SUCCESS; -} - - -static int32_t parseSmlValue(TAOS_SML_KV *pKV, const char **index, - bool *is_last_kv, SSmlLinesInfo* info, bool isTag) { - const char *start, *cur; - int32_t ret = TSDB_CODE_SUCCESS; - char *value = NULL; - int16_t len = 0; - - bool kv_done = false; - bool back_slash = false; - bool double_quote = false; - size_t line_len = 0; - - enum { - tag_common, - tag_lqoute, - tag_rqoute - } tag_state; - - enum { - val_common, - val_lqoute, - val_rqoute - } val_state; - - start = cur = *index; - tag_state = tag_common; - val_state = val_common; - - while (1) { - if (isTag) { - /* ',', '=' and spaces MUST be escaped */ - switch (tag_state) { - case tag_common: - if (back_slash == true) { - if (*cur != ',' && *cur != '=' && *cur != ' ') { - uError("SML:0x%"PRIx64" tag value: state(%d), incorrect character(%c) escaped", info->id, tag_state, *cur); - ret = TSDB_CODE_TSC_LINE_SYNTAX_ERROR; - goto error; - } - - back_slash = false; - cur++; - len++; - break; - } - - if (*cur == '"') { - if (cur == *index) { - tag_state = tag_lqoute; - } - cur += 1; - len += 1; - break; - } else if (*cur == 'L') { - line_len = strlen(*index); - - /* common character at the end */ - if (cur + 1 >= *index + line_len) { - *is_last_kv = true; - kv_done = true; - break; - } - - if (*(cur + 1) == '"') { - /* string starts here */ - if (cur + 1 == *index + 1) { - tag_state = tag_lqoute; - } - cur += 2; - len += 2; - break; - } - } - - switch (*cur) { - case '\\': - back_slash = true; - cur++; - len++; - break; - case ',': - kv_done = true; - break; - - case ' ': - /* fall through */ - case '\0': - *is_last_kv = true; - kv_done = true; - break; - - default: - cur++; - len++; - } - - break; - case tag_lqoute: - if (back_slash == true) { - if (*cur != ',' && *cur != '=' && *cur != ' ') { - uError("SML:0x%"PRIx64" tag value: state(%d), incorrect character(%c) escaped", info->id, tag_state, *cur); - ret = TSDB_CODE_TSC_LINE_SYNTAX_ERROR; - goto error; - } - - back_slash = false; - cur++; - len++; - break; - } else if (double_quote == true) { - if (*cur != ' ' && *cur != ',' && *cur != '\0') { - uError("SML:0x%"PRIx64" tag value: state(%d), incorrect character(%c) behind closing \"", info->id, tag_state, *cur); - ret = TSDB_CODE_TSC_LINE_SYNTAX_ERROR; - goto error; - } - - if (*cur == ' ' || *cur == '\0') { - *is_last_kv = true; - } - - double_quote = false; - tag_state = tag_rqoute; - break; - } - - switch (*cur) { - case '\\': - back_slash = true; - cur++; - len++; - break; - - case '"': - double_quote = true; - cur++; - len++; - break; - - case ',': - /* fall through */ - case '=': - /* fall through */ - case ' ': - if (*(cur - 1) != '\\') { - uError("SML:0x%"PRIx64" tag value: state(%d), character(%c) not escaped", info->id, tag_state, *cur); - ret = TSDB_CODE_TSC_LINE_SYNTAX_ERROR; - kv_done = true; - } - break; - - case '\0': - uError("SML:0x%"PRIx64" tag value: state(%d), closing \" not found", info->id, tag_state); - ret = TSDB_CODE_TSC_LINE_SYNTAX_ERROR; - kv_done = true; - break; - - default: - cur++; - len++; - } - - break; - - default: - kv_done = true; - } - } else { - switch (val_state) { - case val_common: - if (back_slash == true) { - if (*cur != '\\' && *cur != '"') { - uError("SML:0x%"PRIx64" field value: state(%d), incorrect character(%c) escaped", info->id, val_state, *cur); - ret = TSDB_CODE_TSC_LINE_SYNTAX_ERROR; - goto error; - } - - back_slash = false; - cur++; - len++; - break; - } - - if (*cur == '"') { - if (cur == *index) { - val_state = val_lqoute; - } else { - if (*(cur - 1) != '\\') { - uError("SML:0x%"PRIx64" field value: state(%d), \" not escaped", info->id, val_state); - ret = TSDB_CODE_TSC_LINE_SYNTAX_ERROR; - goto error; - } - } - - cur += 1; - len += 1; - break; - } else if (*cur == 'L') { - line_len = strlen(*index); - - /* common character at the end */ - if (cur + 1 >= *index + line_len) { - *is_last_kv = true; - kv_done = true; - break; - } - - if (*(cur + 1) == '"') { - /* string starts here */ - if (cur + 1 == *index + 1) { - val_state = val_lqoute; - cur += 2; - len += 2; - } else { - /* MUST at the end of string */ - if (cur + 2 >= *index + line_len) { - cur += 2; - len += 2; - *is_last_kv = true; - kv_done = true; - } else { - if (*(cur + 2) != ' ' && *(cur + 2) != ',') { - uError("SML:0x%"PRIx64" field value: state(%d), not closing character(L\")", info->id, val_state); - ret = TSDB_CODE_TSC_LINE_SYNTAX_ERROR; - goto error; - } else { - if (*(cur + 2) == ' ') { - *is_last_kv = true; - } - - cur += 2; - len += 2; - kv_done = true; - } - } - } - break; - } - } - - switch (*cur) { - case '\\': - back_slash = true; - cur++; - len++; - break; - - case ',': - kv_done = true; - break; - - case ' ': - /* fall through */ - case '\0': - *is_last_kv = true; - kv_done = true; - break; - - default: - cur++; - len++; - } - - break; - case val_lqoute: - if (back_slash == true) { - if (*cur != '\\' && *cur != '"') { - uError("SML:0x%"PRIx64" field value: state(%d), incorrect character(%c) escaped", info->id, val_state, *cur); - ret = TSDB_CODE_TSC_LINE_SYNTAX_ERROR; - goto error; - } - - back_slash = false; - cur++; - len++; - break; - } else if (double_quote == true) { - if (*cur != ' ' && *cur != ',' && *cur != '\0') { - uError("SML:0x%"PRIx64" field value: state(%d), incorrect character(%c) behind closing \"", info->id, val_state, *cur); - ret = TSDB_CODE_TSC_LINE_SYNTAX_ERROR; - goto error; - } - - if (*cur == ' ' || *cur == '\0') { - *is_last_kv = true; - } - - double_quote = false; - val_state = val_rqoute; - break; - } - - switch (*cur) { - case '\\': - back_slash = true; - cur++; - len++; - break; - - case '"': - double_quote = true; - cur++; - len++; - break; - - case '\0': - uError("SML:0x%"PRIx64" field value: state(%d), closing \" not found", info->id, val_state); - ret = TSDB_CODE_TSC_LINE_SYNTAX_ERROR; - kv_done = true; - break; - - default: - cur++; - len++; - } - - break; - default: - kv_done = true; - } - } - - if (kv_done == true) { - break; - } - } - - if (len == 0 || ret != TSDB_CODE_SUCCESS) { - taosMemoryFree(pKV->key); - pKV->key = NULL; - return TSDB_CODE_TSC_LINE_SYNTAX_ERROR; - } - - value = calloc(len + 1, 1); - memcpy(value, start, len); - value[len] = '\0'; - if (!convertSmlValueType(pKV, value, len, info, isTag)) { - uError("SML:0x%"PRIx64" Failed to convert sml value string(%s) to any type", - info->id, value); - taosMemoryFree(value); - ret = TSDB_CODE_TSC_INVALID_VALUE; - goto error; - } - taosMemoryFree(value); - - *index = (*cur == '\0') ? cur : cur + 1; - return ret; - -error: - //taosMemoryFree previous alocated key field - taosMemoryFree(pKV->key); - pKV->key = NULL; - return ret; -} - -/* Field Escape charaters - 1: measurement Comma,Space - 2: tag_key, tag_value, field_key Comma,Equal Sign,Space - 3: field_value Double quote,Backslash -*/ - -static int32_t parseSmlMeasurement(TAOS_SML_DATA_POINT *pSml, const char **index, - uint8_t *has_tags, SSmlLinesInfo* info) { - const char *cur = *index; - int16_t len = 0; - - pSml->stableName = calloc(TSDB_TABLE_NAME_LEN + TS_BACKQUOTE_CHAR_SIZE, 1); - if (pSml->stableName == NULL){ - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - while (*cur != '\0') { - if (len > TSDB_TABLE_NAME_LEN - 1) { - uError("SML:0x%"PRIx64" Measurement field cannot exceeds %d characters", info->id, TSDB_TABLE_NAME_LEN - 1); - taosMemoryFree(pSml->stableName); - pSml->stableName = NULL; - return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; - } - //first unescaped comma or space identifies measurement - //if space detected first, meaning no tag in the input - if (*cur == ',' && *(cur - 1) != '\\') { - *has_tags = 1; - break; - } - if (*cur == ' ' && *(cur - 1) != '\\') { - if (*(cur + 1) != ' ') { - break; - } - else { - cur++; - continue; - } - } - //Comma, Space, Backslash needs to be escaped if any - if (*cur == '\\') { - escapeSpecialCharacter(1, &cur); - } - pSml->stableName[len] = *cur; - cur++; - len++; - } - if (len == 0) { - taosMemoryFree(pSml->stableName); - pSml->stableName = NULL; - return TSDB_CODE_TSC_LINE_SYNTAX_ERROR; - } - addEscapeCharToString(pSml->stableName, len); - *index = cur + 1; - uDebug("SML:0x%"PRIx64" Stable name in measurement:%s|len:%d", info->id, pSml->stableName, len); - - return TSDB_CODE_SUCCESS; -} - //Table name can only contain digits(0-9),alphebet(a-z),underscore(_) int32_t isValidChildTableName(const char *pTbName, int16_t len, SSmlLinesInfo* info) { if (len > TSDB_TABLE_NAME_LEN - 1) { @@ -2570,147 +1384,6 @@ int32_t isValidChildTableName(const char *pTbName, int16_t len, SSmlLinesInfo* i return TSDB_CODE_SUCCESS; } - -static int32_t parseSmlKvPairs(TAOS_SML_KV **pKVs, int *num_kvs, - const char **index, bool isField, - TAOS_SML_DATA_POINT* smlData, SHashObj *pHash, - SSmlLinesInfo* info) { - const char *cur = *index; - int32_t ret = TSDB_CODE_SUCCESS; - TAOS_SML_KV *pkv; - bool is_last_kv = false; - - int32_t capacity = 0; - if (isField) { - capacity = 64; - *pKVs = calloc(capacity, sizeof(TAOS_SML_KV)); - // leave space for timestamp; - pkv = *pKVs; - pkv++; - } else { - capacity = 8; - *pKVs = calloc(capacity, sizeof(TAOS_SML_KV)); - pkv = *pKVs; - } - - size_t childTableNameLen = strlen(tsSmlChildTableName); - char childTableName[TSDB_TABLE_NAME_LEN + TS_BACKQUOTE_CHAR_SIZE] = {0}; - if (childTableNameLen != 0) { - memcpy(childTableName, tsSmlChildTableName, childTableNameLen); - addEscapeCharToString(childTableName, (int32_t)(childTableNameLen)); - } - - while (*cur != '\0') { - ret = parseSmlKey(pkv, &cur, pHash, info); - if (ret) { - uError("SML:0x%"PRIx64" Unable to parse key", info->id); - goto error; - } - ret = parseSmlValue(pkv, &cur, &is_last_kv, info, !isField); - if (ret) { - uError("SML:0x%"PRIx64" Unable to parse value", info->id); - goto error; - } - - if (!isField && childTableNameLen != 0 && strcasecmp(pkv->key, childTableName) == 0) { - smlData->childTableName = taosMemoryMalloc(pkv->length + TS_BACKQUOTE_CHAR_SIZE + 1); - memcpy(smlData->childTableName, pkv->value, pkv->length); - addEscapeCharToString(smlData->childTableName, (int32_t)pkv->length); - taosMemoryFree(pkv->key); - taosMemoryFree(pkv->value); - } else { - *num_kvs += 1; - } - if (is_last_kv) { - goto done; - } - - //reallocate addtional memory for more kvs - TAOS_SML_KV *more_kvs = NULL; - - if (isField) { - if ((*num_kvs + 2) > capacity) { - capacity *= 3; capacity /= 2; - more_kvs = realloc(*pKVs, capacity * sizeof(TAOS_SML_KV)); - } else { - more_kvs = *pKVs; - } - } else { - if ((*num_kvs + 1) > capacity) { - capacity *= 3; capacity /= 2; - more_kvs = realloc(*pKVs, capacity * sizeof(TAOS_SML_KV)); - } else { - more_kvs = *pKVs; - } - } - - if (!more_kvs) { - goto error; - } - *pKVs = more_kvs; - //move pKV points to next TAOS_SML_KV block - if (isField) { - pkv = *pKVs + *num_kvs + 1; - } else { - pkv = *pKVs + *num_kvs; - } - } - goto done; - -error: - return ret; -done: - *index = cur; - return ret; -} - -static void moveTimeStampToFirstKv(TAOS_SML_DATA_POINT** smlData, TAOS_SML_KV *ts) { - TAOS_SML_KV* tsField = (*smlData)->fields; - tsField->length = ts->length; - tsField->type = ts->type; - tsField->value = taosMemoryMalloc(ts->length); - tsField->key = taosMemoryMalloc(strlen(ts->key) + 1); - memcpy(tsField->key, ts->key, strlen(ts->key) + 1); - memcpy(tsField->value, ts->value, ts->length); - (*smlData)->fieldNum = (*smlData)->fieldNum + 1; - - taosMemoryFree(ts->key); - taosMemoryFree(ts->value); - taosMemoryFree(ts); -} - -/* Field Escape charaters - 1: measurement Comma,Space - 2: tag_key, tag_value, field_key Comma,Equal Sign,Space - 3: field_value Double quote,Backslash -*/ - -//void findSpace(const char** sql, const char **tags, int32_t *tagLen){ -// const char *cur = *sql; -// *tagLen = 0; -// *tags = NULL; -// if(!cur) return; -// while (*cur != '\0') { // jump the space at the begining -// if(*cur != SPACE) { -// *tags = cur; -// break; -// } -// cur++; -// } -// -// while (*cur != '\0') { // find the first space -// if (*cur == SPACE && *(cur - 1) != SLASH) { -// *tagLen = cur - *tags; -// break; -// } -// -// cur++; -// } -// *sql = cur; -// return; -//} - - int32_t parseSml(const char* sql, TAOS_PARSE_ELEMENTS *elements){ if(!sql) return TSDB_CODE_SML_INVALID_DATA; while (*sql != '\0') { // jump the space at the begining @@ -2757,14 +1430,13 @@ int32_t parseSml(const char* sql, TAOS_PARSE_ELEMENTS *elements){ while (*sql != '\0') { if(*sql == SPACE && *(sql - 1) != SLASH) { - elements->colsLen = sql - elements->cols; break; } sql++; } - if(elements->colsLen == 0) return TSDB_CODE_SML_INVALID_DATA; + elements->colsLen = sql - elements->cols; - // parse ts + // parse ts,ts can be empty while (*sql != '\0') { if(*sql != SPACE) { elements->timestamp = sql; @@ -2772,12 +1444,11 @@ int32_t parseSml(const char* sql, TAOS_PARSE_ELEMENTS *elements){ } sql++; } - if(!elements->timestamp) return TSDB_CODE_SML_INVALID_DATA; return TSDB_CODE_SUCCESS; } -int32_t parseSmlKV(const char* data, int32_t len, SArray *tags){ +int32_t parseSmlKV(const char* data, int32_t len, SArray *cols, bool isTag){ for(int i = 0; i < len; i++){ const char *key = data + i; int32_t keyLen = 0; @@ -2805,24 +1476,112 @@ int32_t parseSmlKV(const char* data, int32_t len, SArray *tags){ if(valueLen == 0){ return TSDB_CODE_SML_INVALID_DATA; } - TAOS_SML_KV *kv = taosMemoryCalloc(sizeof(TAOS_SML_KV), 1); + SSmlKv *kv = taosMemoryCalloc(sizeof(SSmlKv), 1); kv->key = key; kv->keyLen = keyLen; kv->value = value; kv->valueLen = valueLen; - kv->type = TSDB_DATA_TYPE_NCHAR; - if(tags) taosArrayPush(tags, &kv); + if(isTag){ + kv->type = TSDB_DATA_TYPE_NCHAR; + } + if(cols) taosArrayPush(cols, &kv); } return TSDB_CODE_SUCCESS; } -int32_t parseSmlTS(const char* data, int32_t len, SArray *tags){ - TAOS_SML_KV *kv = taosMemoryCalloc(sizeof(TAOS_SML_KV), 1); - kv->value = data; - kv->valueLen = len; +static int64_t getTimeStampValue(const char *value, int32_t type) { + double ts = (double)strtoll(value, NULL, 10); + switch (type) { + case TSDB_TIME_PRECISION_HOURS: + ts *= (3600 * 1e9); + case TSDB_TIME_PRECISION_MINUTES: + ts *= (60 * 1e9); + case TSDB_TIME_PRECISION_SECONDS: + ts *= (1e9); + case TSDB_TIME_PRECISION_MICRO: + ts *= (1e6); + case TSDB_TIME_PRECISION_MILLI: + ts *= (1e3); + default: + break; + } + if(ts > (double)INT64_MAX || ts < 0){ + return -1; + }else{ + return (int64_t)ts; + } +} + +static int64_t getTimeStampNow(int32_t precision) { + switch (precision) { + case TSDB_TIME_PRECISION_HOURS: + return taosGetTimestampMs()/1000/3600; + case TSDB_TIME_PRECISION_MINUTES: + return taosGetTimestampMs()/1000/60; + + case TSDB_TIME_PRECISION_SECONDS: + return taosGetTimestampMs()/1000; + default: + return taosGetTimestamp(precision); + } +} + +static int32_t isValidateTimeStamp(const char *pVal, int32_t len) { + for (int i = 0; i < len; ++i) { + if (!isdigit(pVal[i])) { + return TSDB_CODE_TSC_INVALID_TIME_STAMP; + } + } + return TSDB_CODE_SUCCESS; +} + +static int32_t getTsType(int32_t len) { + if (len == TSDB_TIME_PRECISION_SEC_DIGITS) { + return TSDB_TIME_PRECISION_SECONDS; + } else if (len == TSDB_TIME_PRECISION_MILLI_DIGITS) { + return TSDB_TIME_PRECISION_MILLI_DIGITS; + } else { + return TSDB_CODE_TSC_INVALID_TIME_STAMP; + } +} + +static int32_t parseSmlTS(const char* data, SArray *tags, int8_t tsType, SMLProtocolType protocolType){ + int64_t *ts = taosMemoryCalloc(1, sizeof(int64_t)); + if(data == NULL){ + if(protocolType == TSDB_SML_LINE_PROTOCOL){ + *ts = getTimeStampNow(tsType); + }else{ + goto cleanup; + } + }else{ + int32_t len = strlen(data); + int ret = isValidateTimeStamp(data, len); + if(!ret){ + goto cleanup; + } + if(protocolType != TSDB_SML_LINE_PROTOCOL){ + tsType = getTsType(len); + if (tsType == TSDB_CODE_TSC_INVALID_TIME_STAMP) { + goto cleanup; + } + } + *ts = getTimeStampValue(data, tsType); + if(*ts == -1){ + goto cleanup; + } + } + + SSmlKv *kv = taosMemoryCalloc(sizeof(SSmlKv), 1); + kv->value = (const char*)ts; + kv->valueLen = sizeof(int64_t); kv->type = TSDB_DATA_TYPE_TIMESTAMP; + kv->length = (int16_t)tDataTypes[kv->type].bytes; if(tags) taosArrayPush(tags, &kv); return TSDB_CODE_SUCCESS; + +cleanup: + taosMemoryFree(ts); + return TSDB_CODE_TSC_INVALID_TIME_STAMP; } //int32_t parseSmlCols(const char* data, SArray *cols){ @@ -2871,8 +1630,8 @@ int32_t parseSmlTS(const char* data, int32_t len, SArray *tags){ void updateMeta(SSmlSTableMeta* tableMeta, SArray *tags, SArray *cols){ if(tags){ for (int i = 0; i < taosArrayGetSize(tags); ++i) { - TAOS_SML_KV *kv = taosArrayGetP(tags, i); - TAOS_SML_KV **value = taosHashGet(tableMeta->tagHash, kv->key, kv->keyLen); + SSmlKv *kv = taosArrayGetP(tags, i); + SSmlKv **value = taosHashGet(tableMeta->tagHash, kv->key, kv->keyLen); if(value){ if(kv->type != (*value)->type){ // todo @@ -2886,8 +1645,8 @@ void updateMeta(SSmlSTableMeta* tableMeta, SArray *tags, SArray *cols){ if(cols){ for (int i = 1; i < taosArrayGetSize(cols); ++i) { //jump timestamp - TAOS_SML_KV *kv = taosArrayGetP(cols, i); - TAOS_SML_KV **value = taosHashGet(tableMeta->fieldHash, kv->key, kv->keyLen); + SSmlKv *kv = taosArrayGetP(cols, i); + SSmlKv **value = taosHashGet(tableMeta->fieldHash, kv->key, kv->keyLen); if(value){ if(kv->type != (*value)->type){ // todo @@ -2902,20 +1661,20 @@ void updateMeta(SSmlSTableMeta* tableMeta, SArray *tags, SArray *cols){ void insertMeta(SSmlSTableMeta* tableMeta, SArray *tags, SArray *cols){ if(tags){ for (int i = 0; i < taosArrayGetSize(tags); ++i) { - TAOS_SML_KV *kv = taosArrayGetP(tags, i); + SSmlKv *kv = taosArrayGetP(tags, i); taosHashPut(tableMeta->tagHash, kv->key, kv->keyLen, &kv, POINTER_BYTES); } } if(cols){ for (int i = 0; i < taosArrayGetSize(cols); ++i) { - TAOS_SML_KV *kv = taosArrayGetP(cols, i); + SSmlKv *kv = taosArrayGetP(cols, i); taosHashPut(tableMeta->fieldHash, kv->key, kv->keyLen, &kv, POINTER_BYTES); } } } -int32_t tscParseLine(const char* sql, SSmlLinesInfo* info) { +static int32_t smlParseLine(const char* sql, SSmlLinesInfo* info) { TAOS_PARSE_ELEMENTS elements = {0}; int ret = parseSml(sql, &elements); if(ret != TSDB_CODE_SUCCESS){ @@ -2927,8 +1686,9 @@ int32_t tscParseLine(const char* sql, SSmlLinesInfo* info) { uError("SML:0x%"PRIx64" taos_insert_lines failed to allocate memory", info->id); return TSDB_CODE_TSC_OUT_OF_MEMORY; } - parseSmlTS(elements.timestamp, strlen(elements.timestamp), cols); - ret = parseSmlKV(elements.cols, elements.colsLen, cols); + + parseSmlTS(elements.timestamp, cols, info->tsType); + ret = parseSmlCols(elements.cols, elements.colsLen, cols, false); if(ret != TSDB_CODE_SUCCESS){ return ret; } @@ -2954,7 +1714,7 @@ int32_t tscParseLine(const char* sql, SSmlLinesInfo* info) { uError("SML:0x%"PRIx64" taos_insert_lines failed to allocate memory", info->id); return TSDB_CODE_TSC_OUT_OF_MEMORY; } - ret = parseSmlKV(elements.tags, elements.tagsLen, tag->tags); + ret = parseSmlTags(elements.tags, elements.tagsLen, tag->tags); if(ret != TSDB_CODE_SUCCESS){ return ret; } @@ -2973,112 +1733,115 @@ int32_t tscParseLine(const char* sql, SSmlLinesInfo* info) { return TSDB_CODE_SUCCESS; } - -int32_t tscParseLines(char* lines[], int numLines, SSmlLinesInfo* info) { - for (int32_t i = 0; i < numLines; ++i) { - int32_t code = tscParseLine(lines[i], info); - if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%"PRIx64" data point line parse failed. line %d : %s", info->id, i, lines[i]); - return code; - } - } - uDebug("SML:0x%"PRIx64" data point line parse success. tables %d", info->id, taosHashGetSize(info->childTables)); - - return TSDB_CODE_SUCCESS; +static void smlDestroyInfo(SSmlLinesInfo* info){ + if(!info) return; + qDestroyQuery(info->pQuery); + tscSmlDestroyHandle(info->exec); + taosHashCleanup(info->childTables); + taosHashCleanup(info->superTables); + taosHashCleanup(info->metaHashObj); + taosHashCleanup(info->pVgHash); + taosMemoryFree(info); } - -int taos_insert_lines(TAOS* taos, char* lines[], int numLines, SMLProtocolType protocol, SMLTimeStampType tsType, int *affectedRows) { - int32_t code = 0; - +static SSmlLinesInfo* smlBuildInfo(TAOS* taos, SRequestObj* request, SMLProtocolType protocol, int32_t tsType){ SSmlLinesInfo* info = taosMemoryMalloc(sizeof(SSmlLinesInfo)); + if (NULL == info) { + return NULL; + } info->id = genLinesSmlId(); info->tsType = tsType; - info->taos = (STscObj*)taos; + info->taos = taos; info->protocol = protocol; + info->pQuery = taosMemoryCalloc(1, sizeof(SQuery)); + if (NULL == info->pQuery) { + goto cleanup; + } + info->pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE; + info->pQuery->haveResultSet = false; + info->pQuery->msgType = TDMT_VND_SUBMIT; + info->pQuery->pRoot = (SNode*)nodesMakeNode(QUERY_NODE_VNODE_MODIF_STMT); + ((SVnodeModifOpStmt*)(info->pQuery->pRoot))->payloadType = PAYLOAD_TYPE_KV; + + info->exec = tscSmlInitHandle(info->pQuery); + + int32_t code = catalogGetHandle(info->taos->pAppInfo->clusterId, &info->pCatalog); + if(code != TSDB_CODE_SUCCESS){ + uError("SML:0x%"PRIx64" get catalog error %d", info->id, code); + goto cleanup; + } + info->pRequest = request; + info->msgBuf = info->pRequest->msgBuf; + info->msgLen = ERROR_MSG_BUF_DEFAULT_SIZE; + + + info->childTables = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, false); + info->superTables = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, false); + info->metaHashObj = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), true, false); + info->pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, false); + + return info; + +cleanup: + smlDestroyInfo(info); + return NULL; +} + +int sml_insert_lines(TAOS* taos, SRequestObj* request, char* lines[], int numLines, SMLProtocolType protocol, int32_t tsType) { + int32_t code = TSDB_CODE_SUCCESS; + + SSmlLinesInfo* info = smlBuildInfo(taos, request, protocol, tsType); + if(!info){ + code = TSDB_CODE_OUT_OF_MEMORY; + goto cleanup; + } if (numLines <= 0 || numLines > 65536) { uError("SML:0x%"PRIx64" taos_insert_lines numLines should be between 1 and 65536. numLines: %d", info->id, numLines); code = TSDB_CODE_TSC_APP_ERROR; goto cleanup; } - - info->childTables = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, false); - info->superTables = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, false); - - uDebug("SML:0x%"PRIx64" taos_insert_lines begin inserting %d lines, first line: %s", info->id, numLines, lines[0]); - code = tscParseLines(lines, numLines, info); - - if (code != 0) { - goto cleanup; + for (int32_t i = 0; i < numLines; ++i) { + code = smlParseLine(lines[i], info); + if (code != TSDB_CODE_SUCCESS) { + uError("SML:0x%"PRIx64" data point line parse failed. line %d : %s", info->id, i, lines[i]); + goto cleanup; + } } + uDebug("SML:0x%"PRIx64" data point line parse success. tables %d", info->id, taosHashGetSize(info->childTables)); - code = tscSmlInsert(taos, info); - if (code != 0) { + code = smlInsert(taos, info); + if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%"PRIx64" taos_sml_insert error: %s", info->id, tstrerror((code))); goto cleanup; } - if (affectedRows != NULL) { - *affectedRows = info->affectedRows; - } uDebug("SML:0x%"PRIx64" taos_insert_lines finish inserting %d lines. code: %d", info->id, numLines, code); cleanup: - taosMemoryFree(info); + smlDestroyInfo(info); return code; } -static int32_t convertPrecisionType(int precision, SMLTimeStampType *tsType) { +static int32_t convertPrecisionType(int precision) { switch (precision) { - case TSDB_SML_TIMESTAMP_NOT_CONFIGURED: - *tsType = SML_TIME_STAMP_NOT_CONFIGURED; - break; case TSDB_SML_TIMESTAMP_HOURS: - *tsType = SML_TIME_STAMP_HOURS; - break; + return TSDB_TIME_PRECISION_HOURS; case TSDB_SML_TIMESTAMP_MILLI_SECONDS: - *tsType = SML_TIME_STAMP_MILLI_SECONDS; - break; + return TSDB_TIME_PRECISION_MILLI; case TSDB_SML_TIMESTAMP_NANO_SECONDS: - *tsType = SML_TIME_STAMP_NANO_SECONDS; - break; + case TSDB_SML_TIMESTAMP_NOT_CONFIGURED: + return TSDB_TIME_PRECISION_NANO; case TSDB_SML_TIMESTAMP_MICRO_SECONDS: - *tsType = SML_TIME_STAMP_MICRO_SECONDS; - break; + return TSDB_TIME_PRECISION_MICRO; case TSDB_SML_TIMESTAMP_SECONDS: - *tsType = SML_TIME_STAMP_SECONDS; - break; + return TSDB_TIME_PRECISION_SECONDS; case TSDB_SML_TIMESTAMP_MINUTES: - *tsType = SML_TIME_STAMP_MINUTES; - break; + return TSDB_TIME_PRECISION_MINUTES; default: - return TSDB_CODE_SML_INVALID_PRECISION_TYPE; + return -1; } - - return TSDB_CODE_SUCCESS; } -//make a dummy SSqlObj -static SSqlObj* createSmlQueryObj(TAOS* taos, int32_t affected_rows, int32_t code) { - SSqlObj *pNew = (SSqlObj*)calloc(1, sizeof(SSqlObj)); - if (pNew == NULL) { - return NULL; - } - pNew->signature = pNew; - pNew->pTscObj = taos; - pNew->fp = NULL; - - tsem_init(&pNew->rspSem, 0, 0); - registerSqlObj(pNew); - - pNew->res.numOfRows = affected_rows; - pNew->res.code = code; - - - return pNew; -} - - /** * taos_schemaless_insert() parse and insert data points into database according to * different protocol. @@ -3102,19 +1865,17 @@ static SSqlObj* createSmlQueryObj(TAOS* taos, int32_t affected_rows, int32_t cod TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int protocol, int precision) { int code = TSDB_CODE_SUCCESS; - int affected_rows = 0; - SMLTimeStampType tsType = SML_TIME_STAMP_NOW; - - if (protocol == TSDB_SML_LINE_PROTOCOL) { - code = convertPrecisionType(precision, &tsType); - if (code != TSDB_CODE_SUCCESS) { - return NULL; - } - } + SRequestObj* request = createRequest(taos, NULL, NULL, TSDB_SQL_INSERT); switch (protocol) { case TSDB_SML_LINE_PROTOCOL: - code = taos_insert_lines(taos, lines, numLines, protocol, tsType, &affected_rows); + int32_t tsType = convertPrecisionType(precision); + if(tsType == -1){ + request->code = TSDB_CODE_SML_INVALID_PRECISION_TYPE; + goto end; + } + + code = sml_insert_lines(taos, request, lines, numLines, protocol, tsType); break; case TSDB_SML_TELNET_PROTOCOL: //code = taos_insert_telnet_lines(taos, lines, numLines, protocol, tsType, &affected_rows); @@ -3127,8 +1888,6 @@ TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int pr break; } - - SSqlObj *pSql = createSmlQueryObj(taos, affected_rows, code); - - return (TAOS_RES*)pSql; +end: + return (TAOS_RES*)request; } diff --git a/source/client/src/clientStmt.c b/source/client/src/clientStmt.c index 0972ff3477..e3480b2490 100644 --- a/source/client/src/clientStmt.c +++ b/source/client/src/clientStmt.c @@ -364,7 +364,7 @@ int stmtSetTbTags(TAOS_STMT *stmt, TAOS_BIND_v2 *tags) { STMT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); } - STMT_ERR_RET(qBindStmtTagsValue(*pDataBlock, pStmt->bInfo.boundTags, pStmt->bInfo.tbSuid, &pStmt->bInfo.sname, tags, pStmt->exec.pRequest->msgBuf, pStmt->exec.pRequest->msgBufLen)); + STMT_ERR_RET(qBindStmtTagsValue(*pDataBlock, pStmt->bInfo.boundTags, pStmt->bInfo.tbSuid, pStmt->bInfo.sname.tname, tags, pStmt->exec.pRequest->msgBuf, pStmt->exec.pRequest->msgBufLen)); return TSDB_CODE_SUCCESS; } diff --git a/source/libs/parser/inc/parInsertData.h b/source/libs/parser/inc/parInsertData.h index ed7267655c..046e9ef0f8 100644 --- a/source/libs/parser/inc/parInsertData.h +++ b/source/libs/parser/inc/parInsertData.h @@ -136,7 +136,7 @@ void destroyBlockHashmap(SHashObj* pDataBlockHash); int initRowBuilder(SRowBuilder *pBuilder, int16_t schemaVer, SParsedDataColInfo *pColInfo); int32_t allocateMemIfNeed(STableDataBlocks *pDataBlock, int32_t rowSize, int32_t * numOfRows); int32_t getDataBlockFromList(SHashObj* pHashList, int64_t id, int32_t size, int32_t startOffset, int32_t rowSize, - const STableMeta* pTableMeta, STableDataBlocks** dataBlocks, SArray* pBlockList, SVCreateTbReq* pCreateTbReq); + STableMeta* pTableMeta, STableDataBlocks** dataBlocks, SArray* pBlockList, SVCreateTbReq* pCreateTbReq); int32_t mergeTableDataBlocks(SHashObj* pHashObj, uint8_t payloadType, SArray** pVgDataBlocks); int32_t buildCreateTbMsg(STableDataBlocks* pBlocks, SVCreateTbReq* pCreateTbReq); int32_t allocateMemForSize(STableDataBlocks *pDataBlock, int32_t allSize); diff --git a/source/libs/parser/inc/parUtil.h b/source/libs/parser/inc/parUtil.h index 460a089802..365d317fdb 100644 --- a/source/libs/parser/inc/parUtil.h +++ b/source/libs/parser/inc/parUtil.h @@ -41,7 +41,6 @@ int32_t generateSyntaxErrMsg(SMsgBuf* pBuf, int32_t errCode, ...); int32_t buildInvalidOperationMsg(SMsgBuf* pMsgBuf, const char* msg); int32_t buildSyntaxErrMsg(SMsgBuf* pBuf, const char* additionalInfo, const char* sourceStr); -STableMeta* tableMetaDup(const STableMeta* pTableMeta); SSchema *getTableColumnSchema(const STableMeta *pTableMeta); SSchema *getTableTagSchema(const STableMeta* pTableMeta); int32_t getNumOfColumns(const STableMeta* pTableMeta); diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index f21a738032..0e87d8611e 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -763,11 +763,9 @@ static int32_t KvRowAppend(SMsgBuf* pMsgBuf, const void *value, int32_t len, voi return TSDB_CODE_SUCCESS; } -static int32_t buildCreateTbReq(SVCreateTbReq *pTbReq, const SName* pName, SKVRow row, int64_t suid) { - char dbFName[TSDB_DB_FNAME_LEN] = {0}; - tNameGetFullDbName(pName, dbFName); +static int32_t buildCreateTbReq(SVCreateTbReq *pTbReq, const char* tname, SKVRow row, int64_t suid) { pTbReq->type = TD_CHILD_TABLE; - pTbReq->name = strdup(pName->tname); + pTbReq->name = strdup(tname); pTbReq->ctbCfg.suid = suid; pTbReq->ctbCfg.pTag = row; @@ -775,7 +773,7 @@ static int32_t buildCreateTbReq(SVCreateTbReq *pTbReq, const SName* pName, SKVRo } // pSql -> tag1_value, ...) -static int32_t parseTagsClause(SInsertParseContext* pCxt, SSchema* pSchema, uint8_t precision, const SName* pName) { +static int32_t parseTagsClause(SInsertParseContext* pCxt, SSchema* pSchema, uint8_t precision, const char* tName) { if (tdInitKVRowBuilder(&pCxt->tagsBuilder) < 0) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } @@ -816,7 +814,7 @@ static int32_t parseTagsClause(SInsertParseContext* pCxt, SSchema* pSchema, uint } tdSortKVRowByColIdx(row); - return buildCreateTbReq(&pCxt->createTblReq, pName, row, pCxt->pTableMeta->suid); + return buildCreateTbReq(&pCxt->createTblReq, tName, row, pCxt->pTableMeta->suid); } static int32_t cloneTableMeta(STableMeta* pSrc, STableMeta** pDst) { @@ -876,7 +874,7 @@ static int32_t parseUsingClause(SInsertParseContext* pCxt, SToken* pTbnameToken) if (TK_NK_LP != sToken.type) { return buildSyntaxErrMsg(&pCxt->msg, "( is expected", sToken.z); } - CHECK_CODE(parseTagsClause(pCxt, pCxt->pTableMeta->schema, getTableInfo(pCxt->pTableMeta).precision, &name)); + CHECK_CODE(parseTagsClause(pCxt, pCxt->pTableMeta->schema, getTableInfo(pCxt->pTableMeta).precision, name.tname)); NEXT_TOKEN(pCxt->pSql, sToken); if (TK_NK_RP != sToken.type) { return buildSyntaxErrMsg(&pCxt->msg, ") is expected", sToken.z); @@ -1024,11 +1022,6 @@ static void destroyDataBlock(STableDataBlocks* pDataBlock) { taosMemoryFreeClear(pDataBlock->pData); if (!pDataBlock->cloned) { - // free the refcount for metermeta - if (pDataBlock->pTableMeta != NULL) { - taosMemoryFreeClear(pDataBlock->pTableMeta); - } - destroyBoundColumnInfo(&pDataBlock->boundColumnInfo); } taosMemoryFreeClear(pDataBlock); @@ -1256,7 +1249,7 @@ int32_t qBuildStmtOutput(SQuery* pQuery, SHashObj* pVgHash, SHashObj* pBlockHash return TSDB_CODE_SUCCESS; } -int32_t qBindStmtTagsValue(void *pBlock, void *boundTags, int64_t suid, SName *pName, TAOS_BIND_v2 *bind, char *msgBuf, int32_t msgBufLen){ +int32_t qBindStmtTagsValue(void *pBlock, void *boundTags, int64_t suid, char *tName, TAOS_BIND_v2 *bind, char *msgBuf, int32_t msgBufLen){ STableDataBlocks *pDataBlock = (STableDataBlocks *)pBlock; SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; SParsedDataColInfo* tags = (SParsedDataColInfo*)boundTags; @@ -1297,7 +1290,7 @@ int32_t qBindStmtTagsValue(void *pBlock, void *boundTags, int64_t suid, SName *p tdSortKVRowByColIdx(row); SVCreateTbReq tbReq = {0}; - CHECK_CODE(buildCreateTbReq(&tbReq, pName, row, suid)); + CHECK_CODE(buildCreateTbReq(&tbReq, tName, row, suid)); CHECK_CODE(buildCreateTbMsg(pDataBlock, &tbReq)); destroyCreateSubTbReq(&tbReq); @@ -1522,4 +1515,211 @@ int32_t qBuildStmtColFields(void *pBlock, int32_t *fieldNum, TAOS_FIELD** fields } +typedef struct SmlExecHandle { + SHashObj* pBlockHash; + + SParsedDataColInfo tags; // each table + SKVRowBuilder tagsBuilder; // each table + SVCreateTbReq createTblReq; // each table + + SQuery* pQuery; +} SmlExecHandle; + +static int32_t smlBoundColumns(SArray *cols, SParsedDataColInfo* pColList, SSchema* pSchema) { + col_id_t nCols = pColList->numOfCols; + + pColList->numOfBound = 0; + pColList->boundNullLen = 0; + memset(pColList->boundColumns, 0, sizeof(col_id_t) * nCols); + for (col_id_t i = 0; i < nCols; ++i) { + pColList->cols[i].valStat = VAL_STAT_NONE; + } + + bool isOrdered = true; + col_id_t lastColIdx = -1; // last column found + for (int i = 0; i < taosArrayGetSize(cols); ++i) { + SSmlKv *kv = taosArrayGetP(cols, i); + SToken sToken = {.n=kv->keyLen, .z=kv->key}; + col_id_t t = lastColIdx + 1; + col_id_t index = findCol(&sToken, t, nCols, pSchema); + if (index < 0 && t > 0) { + index = findCol(&sToken, 0, t, pSchema); + isOrdered = false; + } + if (index < 0) { + return TSDB_CODE_SML_INVALID_DATA; + } + if (pColList->cols[index].valStat == VAL_STAT_HAS) { + return TSDB_CODE_SML_INVALID_DATA; + } + lastColIdx = index; + pColList->cols[index].valStat = VAL_STAT_HAS; + pColList->boundColumns[pColList->numOfBound] = index + PRIMARYKEY_TIMESTAMP_COL_ID; + ++pColList->numOfBound; + switch (pSchema[t].type) { + case TSDB_DATA_TYPE_BINARY: + pColList->boundNullLen += (sizeof(VarDataOffsetT) + VARSTR_HEADER_SIZE + CHAR_BYTES); + break; + case TSDB_DATA_TYPE_NCHAR: + pColList->boundNullLen += (sizeof(VarDataOffsetT) + VARSTR_HEADER_SIZE + TSDB_NCHAR_SIZE); + break; + default: + pColList->boundNullLen += TYPE_BYTES[pSchema[t].type]; + break; + } + } + + pColList->orderStatus = isOrdered ? ORDER_STATUS_ORDERED : ORDER_STATUS_DISORDERED; + + if (!isOrdered) { + pColList->colIdxInfo = taosMemoryCalloc(pColList->numOfBound, sizeof(SBoundIdxInfo)); + if (NULL == pColList->colIdxInfo) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + SBoundIdxInfo* pColIdx = pColList->colIdxInfo; + for (col_id_t i = 0; i < pColList->numOfBound; ++i) { + pColIdx[i].schemaColIdx = pColList->boundColumns[i]; + pColIdx[i].boundIdx = i; + } + qsort(pColIdx, pColList->numOfBound, sizeof(SBoundIdxInfo), schemaIdxCompar); + for (col_id_t i = 0; i < pColList->numOfBound; ++i) { + pColIdx[i].finalIdx = i; + } + qsort(pColIdx, pColList->numOfBound, sizeof(SBoundIdxInfo), boundIdxCompar); + } + + if(pColList->numOfCols > pColList->numOfBound){ + memset(&pColList->boundColumns[pColList->numOfBound], 0, + sizeof(col_id_t) * (pColList->numOfCols - pColList->numOfBound)); + } + + return TSDB_CODE_SUCCESS; +} + +static int32_t smlParseTags(SArray *cols, SKVRowBuilder *tagsBuilder, SParsedDataColInfo* tags, SSchema* pSchema, SVCreateTbReq *createTblReq) { + if (tdInitKVRowBuilder(tagsBuilder) < 0) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + SKvParam param = {.builder = tagsBuilder}; + for (int i = 0; i < tags->numOfBound; ++i) { + + SSchema* pTagSchema = &pSchema[tags->boundColumns[i] - 1]; // colId starts with 1 + param.schema = pTagSchema; + SSmlKv *kv = taosArrayGetP(cols, i); + KvRowAppend(NULL, kv->value, kv->valueLen, ¶m) ; + } + + + SKVRow row = tdGetKVRowFromBuilder(tagsBuilder); + if(row == NULL){ + return TSDB_CODE_SML_INVALID_DATA; + } + tdSortKVRowByColIdx(row); + createTblReq->type = TD_CHILD_TABLE; + createTblReq->ctbCfg.pTag = row; + return TSDB_CODE_SUCCESS; +} + +int32_t smlBind(void *handle, SArray *tags, SArray *cols, STableMeta *pTableMeta, char *msgBuf, int16_t msgBufLen) { + SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; + int32_t rowNum = taosArrayGetSize(cols); + if(rowNum <= 0) { + return buildInvalidOperationMsg(&pBuf, "cols size <= 0"); + } + + SmlExecHandle *smlHandle = (SmlExecHandle *)handle; + SSchema* pTagsSchema = getTableTagSchema(pTableMeta); + smlBoundColumns(tags, &smlHandle->tags, pTagsSchema); + smlParseTags(tags, &smlHandle->tagsBuilder, &smlHandle->tags, pTagsSchema, &smlHandle->createTblReq); + + STableDataBlocks* pDataBlock = NULL; + getDataBlockFromList(smlHandle->pBlockHash, pTableMeta->uid, TSDB_DEFAULT_PAYLOAD_SIZE, + sizeof(SSubmitBlk), getTableInfo(pTableMeta).rowSize, pTableMeta, + &pDataBlock, NULL, &smlHandle->createTblReq); + + SSchema* pSchema = getTableColumnSchema(pTableMeta); + + smlBoundColumns(taosArrayGetP(cols, 0), &pDataBlock->boundColumnInfo, pSchema); + + int32_t extendedRowSize = getExtendedRowSize(pDataBlock); + SParsedDataColInfo* spd = &pDataBlock->boundColumnInfo; + SRowBuilder* pBuilder = &pDataBlock->rowBuilder; + SMemParam param = {.rb = pBuilder}; + + initRowBuilder(&pDataBlock->rowBuilder, pDataBlock->pTableMeta->sversion, &pDataBlock->boundColumnInfo); + + allocateMemForSize(pDataBlock, extendedRowSize * rowNum); + + for (int32_t r = 0; r < rowNum; ++r) { + STSRow* row = (STSRow*)(pDataBlock->pData + pDataBlock->size); // skip the SSubmitBlk header + tdSRowResetBuf(pBuilder, row); + SArray *rowData = taosArrayGetP(cols, r); + + // 1. set the parsed value from sql string + for (int c = 0; c < spd->numOfBound; ++c) { + SSchema* pColSchema = &pSchema[spd->boundColumns[c] - 1]; + + param.schema = pColSchema; + getSTSRowAppendInfo(pBuilder->rowType, spd, c, ¶m.toffset, ¶m.colIdx); + + SSmlKv *kv = taosArrayGetP(rowData, c); + + if (kv->valueLen == 0) { + MemRowAppend(&pBuf, NULL, 0, ¶m); + } else { + int32_t colLen = pColSchema->bytes; + if (IS_VAR_DATA_TYPE(pColSchema->type)) { + colLen = kv->valueLen; + } + + MemRowAppend(&pBuf, kv->value, colLen, ¶m); + } + + if (PRIMARYKEY_TIMESTAMP_COL_ID == pColSchema->colId) { + TSKEY tsKey = TD_ROW_KEY(row); + checkTimestamp(pDataBlock, (const char *)&tsKey); + } + } + + // set the null value for the columns that do not assign values + if ((spd->numOfBound < spd->numOfCols) && TD_IS_TP_ROW(row)) { + for (int32_t i = 0; i < spd->numOfCols; ++i) { + if (spd->cols[i].valStat == VAL_STAT_NONE) { // the primary TS key is not VAL_STAT_NONE + tdAppendColValToTpRow(pBuilder, TD_VTYPE_NONE, getNullValue(pSchema[i].type), true, pSchema[i].type, i, + spd->cols[i].toffset); + } + } + } + + pDataBlock->size += extendedRowSize; + } + + SSubmitBlk *pBlocks = (SSubmitBlk *)(pDataBlock->pData); + if (TSDB_CODE_SUCCESS != setBlockInfo(pBlocks, pDataBlock, rowNum)) { + return buildInvalidOperationMsg(&pBuf, "too many rows in sql, total number of rows should be less than 32767"); + } + + return TSDB_CODE_SUCCESS; +} + +void* tscSmlInitHandle(SQuery *pQuery){ + SmlExecHandle *handle = taosMemoryCalloc(sizeof(SmlExecHandle)); + handle->pBlockHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false); + handle->pQuery = pQuery; + + return handle; +} + +void tscSmlDestroyHandle(void *pHandle){ + if(!pHandle) return; + SmlExecHandle *handle = (SmlExecHandle *)pHandle; + taosHashCleanup(handle->pBlockHash); + taosMemoryFree(handle); +} + +int32_t smlBuildOutput(void* handle, SHashObj* pVgHash) { + SmlExecHandle *smlHandle = (SmlExecHandle *)handle; + return qBuildStmtOutput(smlHandle->pQuery, pVgHash, smlHandle->pBlockHash); +} diff --git a/source/libs/parser/src/parInsertData.c b/source/libs/parser/src/parInsertData.c index bf30915fcb..6df8f2eaf3 100644 --- a/source/libs/parser/src/parInsertData.c +++ b/source/libs/parser/src/parInsertData.c @@ -116,7 +116,7 @@ void destroyBoundColumnInfo(void* pBoundInfo) { } static int32_t createDataBlock(size_t defaultSize, int32_t rowSize, int32_t startOffset, - const STableMeta* pTableMeta, STableDataBlocks** dataBlocks) { + STableMeta* pTableMeta, STableDataBlocks** dataBlocks) { STableDataBlocks* dataBuf = (STableDataBlocks*)taosMemoryCalloc(1, sizeof(STableDataBlocks)); if (dataBuf == NULL) { return TSDB_CODE_TSC_OUT_OF_MEMORY; @@ -137,8 +137,7 @@ static int32_t createDataBlock(size_t defaultSize, int32_t rowSize, int32_t star } memset(dataBuf->pData, 0, sizeof(SSubmitBlk)); - //Here we keep the tableMeta to avoid it to be remove by other threads. - dataBuf->pTableMeta = tableMetaDup(pTableMeta); + dataBuf->pTableMeta = pTableMeta; SParsedDataColInfo* pColInfo = &dataBuf->boundColumnInfo; SSchema* pSchema = getTableColumnSchema(dataBuf->pTableMeta); @@ -176,8 +175,7 @@ int32_t buildCreateTbMsg(STableDataBlocks* pBlocks, SVCreateTbReq* pCreateTbReq) return TSDB_CODE_SUCCESS; } -int32_t getDataBlockFromList(SHashObj* pHashList, int64_t id, int32_t size, int32_t startOffset, int32_t rowSize, - const STableMeta* pTableMeta, STableDataBlocks** dataBlocks, SArray* pBlockList, SVCreateTbReq* pCreateTbReq) { +int32_t getDataBlockFromList(SHashObj* pHashList, int64_t id, int32_t size, int32_t startOffset, int32_t rowSize, STableMeta* pTableMeta, STableDataBlocks** dataBlocks, SArray* pBlockList, SVCreateTbReq* pCreateTbReq) { *dataBlocks = NULL; STableDataBlocks** t1 = (STableDataBlocks**)taosHashGet(pHashList, (const char*)&id, sizeof(id)); if (t1 != NULL) { @@ -227,9 +225,9 @@ static void destroyDataBlock(STableDataBlocks* pDataBlock) { taosMemoryFreeClear(pDataBlock->pData); if (!pDataBlock->cloned) { // free the refcount for metermeta - if (pDataBlock->pTableMeta != NULL) { - taosMemoryFreeClear(pDataBlock->pTableMeta); - } +// if (pDataBlock->pTableMeta != NULL) { +// taosMemoryFreeClear(pDataBlock->pTableMeta); +// } destroyBoundColumnInfo(&pDataBlock->boundColumnInfo); } diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index 075caa868c..c98cfcc671 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -166,26 +166,6 @@ int32_t buildSyntaxErrMsg(SMsgBuf* pBuf, const char* additionalInfo, const char* return TSDB_CODE_TSC_SQL_SYNTAX_ERROR; } -static uint32_t getTableMetaSize(const STableMeta* pTableMeta) { - assert(pTableMeta != NULL); - - int32_t totalCols = 0; - if (pTableMeta->tableInfo.numOfColumns >= 0) { - totalCols = pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags; - } - - return sizeof(STableMeta) + totalCols * sizeof(SSchema); -} - -STableMeta* tableMetaDup(const STableMeta* pTableMeta) { - assert(pTableMeta != NULL); - size_t size = getTableMetaSize(pTableMeta); - - STableMeta* p = taosMemoryMalloc(size); - memcpy(p, pTableMeta, size); - return p; -} - SSchema *getTableColumnSchema(const STableMeta *pTableMeta) { assert(pTableMeta != NULL); return (SSchema*) pTableMeta->schema; diff --git a/tools/shell/src/shellCommand.c b/tools/shell/src/shellCommand.c index 546b19f83c..1034c034d4 100644 --- a/tools/shell/src/shellCommand.c +++ b/tools/shell/src/shellCommand.c @@ -51,8 +51,8 @@ void getPrevCharSize(const char *str, int pos, int *size, int *width) { if (str[pos] > 0 || countPrefixOnes((unsigned char)str[pos]) > 1) break; } - int rc = taosMbToWchar(&wc, str + pos, MB_CUR_MAX); - assert(rc == *size); + taosMbToWchar(&wc, str + pos, MB_CUR_MAX); + // assert(rc == *size); // it will be core, if str is encode by utf8 and taos charset is gbk *width = taosWcharWidth(wc); } diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 36d2866fb5..c7a61fb303 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -224,63 +224,27 @@ int32_t shellRunCommand(TAOS *con, char *command) { } } - bool esc = false; - char quote = 0, *cmd = command, *p = command; + char quote = 0, *cmd = command; for (char c = *command++; c != 0; c = *command++) { - if (esc) { - switch (c) { - case 'n': - c = '\n'; - break; - case 'r': - c = '\r'; - break; - case 't': - c = '\t'; - break; - case 'G': - *p++ = '\\'; - break; - case '\'': - case '"': - if (quote) { - *p++ = '\\'; - } - break; - } - *p++ = c; - esc = false; + if (c == '\\' && (*command == '\'' || *command == '"' || *command == '`')) { + command ++; continue; } - if (c == '\\') { - if (quote != 0 && (*command == '_' || *command == '\\')) { - // DO nothing - } else { - esc = true; - continue; - } - } - if (quote == c) { quote = 0; - } else if (quote == 0 && (c == '\'' || c == '"')) { + } else if (quote == 0 && (c == '\'' || c == '"' || c == '`')) { quote = c; - } - - *p++ = c; - if (c == ';' && quote == 0) { - c = *p; - *p = 0; + } else if (c == ';' && quote == 0) { + c = *command; + *command = 0; if (shellRunSingleCommand(con, cmd) < 0) { return -1; } - *p = c; - p = cmd; + *command = c; + cmd = command; } } - - *p = 0; return shellRunSingleCommand(con, cmd); } From ca21807d14b46264152bb11fda7dcda29f57c2a8 Mon Sep 17 00:00:00 2001 From: jiacy-jcy <714897623@qq.com> Date: Thu, 28 Apr 2022 10:52:26 +0800 Subject: [PATCH 03/83] update --- tests/system-test/2-query/Now.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/tests/system-test/2-query/Now.py b/tests/system-test/2-query/Now.py index 99fca2e88d..9d073eb4f7 100644 --- a/tests/system-test/2-query/Now.py +++ b/tests/system-test/2-query/Now.py @@ -144,6 +144,12 @@ class TDTestCase: tdSql.error("select now()+1.5 from ntb") tdSql.error("select now()+1.5 from db.ntb") + tdSql.error("select now()-1.5 from ntb") + tdSql.error("select now()-1.5 from db.ntb") + tdSql.error("select now()*1.5 from ntb") + tdSql.error("select now()*1.5 from db.ntb") + tdSql.error("select now()/1.5 from ntb") + tdSql.error("select now()/1.5 from db.ntb") tdSql.error("select now()+'abc' from ntb") tdSql.error("select now()+'abc' from db.ntb") tdSql.error("select now()+abc from ntb") @@ -262,7 +268,10 @@ class TDTestCase: tdSql.checkRows(3) tdSql.query("select now() +1 from db.stb") tdSql.checkRows(3) - + tdSql.error("select now() +1.5 from stb") + tdSql.error("select now() -1.5 from stb") + tdSql.error("select now() *1.5 from stb") + tdSql.error("select now() /1.5 from stb") tdSql.error("select now() +'abc' from stb") tdSql.error("select now() +'abc' from db.stb") tdSql.error("select now() + ! from stb") @@ -272,6 +281,7 @@ class TDTestCase: tdSql.error("select now() -today() from stb") tdSql.error("select now() - today() from db.stb") + tdSql.query("select now()+null from stb") tdSql.checkData(0,0,None) tdSql.query("select now()+null from db.stb") From e1ac1226a24a5f5db5641f3bc3a5f948b16eb9dd Mon Sep 17 00:00:00 2001 From: jiacy-jcy <714897623@qq.com> Date: Thu, 28 Apr 2022 18:05:14 +0800 Subject: [PATCH 04/83] add case for to_iso8601 --- .../2-query/{To_iso8061.py => To_iso8601.py} | 17 ++++ tests/system-test/2-query/Today.py | 80 ++++++++++++++++++- 2 files changed, 96 insertions(+), 1 deletion(-) rename tests/system-test/2-query/{To_iso8061.py => To_iso8601.py} (88%) diff --git a/tests/system-test/2-query/To_iso8061.py b/tests/system-test/2-query/To_iso8601.py similarity index 88% rename from tests/system-test/2-query/To_iso8061.py rename to tests/system-test/2-query/To_iso8601.py index 86322ed076..cd22ffb90c 100644 --- a/tests/system-test/2-query/To_iso8061.py +++ b/tests/system-test/2-query/To_iso8601.py @@ -92,6 +92,21 @@ class TDTestCase: tdSql.query("select to_iso8601(today()) /null from db.ntb") tdSql.checkRows(3) tdSql.checkData(0,0,None) + # tdSql.query("select to_iso8601(-1) from ntb") + tdSql.query("select to_iso8601(9223372036854775807) from ntb") + tdSql.checkRows(3) + + # tdSql.query("select to_iso8601(10000000000) from ntb") + # tdSql.checkData(0,0,None) + # tdSql.query("select to_iso8601(-1) from ntb") + # tdSql.checkRows(3) + # tdSql.query("select to_iso8601(-10000000000) from ntb") + # tdSql.checkData(0,0,None) + tdSql.error("select to_iso8601(1.5) from ntb") + tdSql.error("select to_iso8601(1.5) from db.ntb") + tdSql.error("select to_iso8601('a') from ntb") + tdSql.error("select to_iso8601(c2) from ntb") + @@ -137,6 +152,8 @@ class TDTestCase: tdSql.checkRows(3) tdSql.checkData(0,0,None) + # tdSql.query("select to_iso8601(-1) from ntb") + def stop(self): diff --git a/tests/system-test/2-query/Today.py b/tests/system-test/2-query/Today.py index 09f018dc11..9eb06de9fb 100644 --- a/tests/system-test/2-query/Today.py +++ b/tests/system-test/2-query/Today.py @@ -134,6 +134,35 @@ class TDTestCase: tdSql.checkRows(3) tdSql.checkData(0, 0, str(today_date)) + tdSql.error("select today()+1.5 from ntb") + tdSql.error("select today()-1.5 from ntb") + tdSql.error("select today()*1.5 from ntb") + tdSql.error("select today()/1.5 from ntb") + tdSql.error("select today()+1.5 from db.ntb") + tdSql.error("select today()-1.5 from db.ntb") + tdSql.error("select today()*1.5 from db.ntb") + tdSql.error("select today()/1.5 from db.ntb") + tdSql.query("select today()+null from ntb") + tdSql.checkData(0,0,None) + tdSql.query("select today()+null from db.ntb") + tdSql.checkData(0,0,None) + tdSql.query("select today()-null from ntb") + tdSql.checkData(0,0,None) + tdSql.query("select today()-null from db.ntb") + tdSql.checkData(0,0,None) + tdSql.query("select today()*null from ntb") + tdSql.checkData(0,0,None) + tdSql.query("select today()*null from db.ntb") + tdSql.checkData(0,0,None) + tdSql.query("select today()/null from ntb") + tdSql.checkData(0,0,None) + tdSql.query("select today()/null from db.ntb") + tdSql.checkData(0,0,None) + tdSql.query("select today()/0 from db.ntb") + tdSql.checkRows(3) + tdSql.checkData(0,0,None) + tdSql.checkData(1,0,None) + tdSql.checkData(2,0,None) # stable tdSql.query("select today() from stb") tdSql.checkRows(3) @@ -208,6 +237,26 @@ class TDTestCase: tdSql.query("select ts from stb where ts<=today()") tdSql.checkRows(2) + tdSql.error("select today()+1.5 from stb") + tdSql.error("select today()-1.5 from stb") + tdSql.error("select today()*1.5 from stb") + tdSql.error("select today()/1.5 from stb") + tdSql.query("select today()+null from stb") + tdSql.checkData(0,0,None) + tdSql.query("select today()+null from db.stb") + tdSql.checkData(0,0,None) + tdSql.query("select today()-null from stb") + tdSql.checkData(0,0,None) + tdSql.query("select today()-null from db.stb") + tdSql.checkData(0,0,None) + tdSql.query("select today()*null from stb") + tdSql.checkData(0,0,None) + tdSql.query("select today()*null from db.stb") + tdSql.checkData(0,0,None) + tdSql.query("select today()/null from stb") + tdSql.checkData(0,0,None) + tdSql.query("select today()/null from db.stb") + tdSql.checkData(0,0,None) # # tdSql.query("select * from ntb where ts Date: Fri, 29 Apr 2022 09:32:57 +0800 Subject: [PATCH 05/83] add to_unixtimestamp case --- tests/system-test/2-query/To_unixtimestamp.py | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 tests/system-test/2-query/To_unixtimestamp.py diff --git a/tests/system-test/2-query/To_unixtimestamp.py b/tests/system-test/2-query/To_unixtimestamp.py new file mode 100644 index 0000000000..7088142f9a --- /dev/null +++ b/tests/system-test/2-query/To_unixtimestamp.py @@ -0,0 +1,52 @@ +from time import sleep + +from util.log import * +from util.sql import * +from util.cases import * + + + + +class TDTestCase: + + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + + def run(self): # sourcery skip: extract-duplicate-method + tdSql.prepare() + tdLog.printNoPrefix("==========step1:create tables==========") + tdSql.execute( + '''create table if not exists ntb + (ts timestamp, c1 int, c2 float,c3 double,c4 timestamp) + ''' + ) + tdSql.execute( + '''create table if not exists stb + (ts timestamp, c1 int, c2 float,c3 double,c4 timestamp) tags(t0 int) + ''' + ) + tdSql.execute( + '''create table if not exists stb_1 using stb tags(100) + ''' + ) + tdLog.printNoPrefix("==========step2:insert data into ntb==========") + + + + + + + + + + + + + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file From a4629e56cecdfdda88d139f0cf3ebe66d33f6277 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 29 Apr 2022 10:58:46 +0800 Subject: [PATCH 06/83] refactor:add schemaless function --- include/common/tcommon.h | 8 +- include/common/ttypes.h | 16 +- source/client/inc/clientSml.h | 51 +- source/client/src/clientSml.c | 1335 ++++++++++------------------ source/libs/parser/src/parInsert.c | 39 +- 5 files changed, 545 insertions(+), 904 deletions(-) diff --git a/include/common/tcommon.h b/include/common/tcommon.h index 1c516e8a96..61f8ebae66 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -246,7 +246,13 @@ typedef struct { int32_t keyLen; uint8_t type; int16_t length; - const char* value; + union{ + const char* value; + int64_t i; + uint64_t u; + double d; + float f; + }; int32_t valueLen; } SSmlKv; diff --git a/include/common/ttypes.h b/include/common/ttypes.h index 405b20c521..377b443843 100644 --- a/include/common/ttypes.h +++ b/include/common/ttypes.h @@ -186,14 +186,14 @@ typedef struct { #define IS_NUMERIC_TYPE(_t) ((IS_SIGNED_NUMERIC_TYPE(_t)) || (IS_UNSIGNED_NUMERIC_TYPE(_t)) || (IS_FLOAT_TYPE(_t))) #define IS_MATHABLE_TYPE(_t) (IS_NUMERIC_TYPE(_t) || (_t) == (TSDB_DATA_TYPE_BOOL) || (_t) == (TSDB_DATA_TYPE_TIMESTAMP)) -#define IS_VALID_TINYINT(_t) ((_t) > INT8_MIN && (_t) <= INT8_MAX) -#define IS_VALID_SMALLINT(_t) ((_t) > INT16_MIN && (_t) <= INT16_MAX) -#define IS_VALID_INT(_t) ((_t) > INT32_MIN && (_t) <= INT32_MAX) -#define IS_VALID_BIGINT(_t) ((_t) > INT64_MIN && (_t) <= INT64_MAX) -#define IS_VALID_UTINYINT(_t) ((_t) >= 0 && (_t) < UINT8_MAX) -#define IS_VALID_USMALLINT(_t) ((_t) >= 0 && (_t) < UINT16_MAX) -#define IS_VALID_UINT(_t) ((_t) >= 0 && (_t) < UINT32_MAX) -#define IS_VALID_UBIGINT(_t) ((_t) >= 0 && (_t) < UINT64_MAX) +#define IS_VALID_TINYINT(_t) ((_t) >= INT8_MIN && (_t) <= INT8_MAX) +#define IS_VALID_SMALLINT(_t) ((_t) >= INT16_MIN && (_t) <= INT16_MAX) +#define IS_VALID_INT(_t) ((_t) >= INT32_MIN && (_t) <= INT32_MAX) +#define IS_VALID_BIGINT(_t) ((_t) >= INT64_MIN && (_t) <= INT64_MAX) +#define IS_VALID_UTINYINT(_t) ((_t) >= 0 && (_t) <= UINT8_MAX) +#define IS_VALID_USMALLINT(_t) ((_t) >= 0 && (_t) <= UINT16_MAX) +#define IS_VALID_UINT(_t) ((_t) >= 0 && (_t) <= UINT32_MAX) +#define IS_VALID_UBIGINT(_t) ((_t) >= 0 && (_t) <= UINT64_MAX) #define IS_VALID_FLOAT(_t) ((_t) >= -FLT_MAX && (_t) <= FLT_MAX) #define IS_VALID_DOUBLE(_t) ((_t) >= -DBL_MAX && (_t) <= DBL_MAX) diff --git a/source/client/inc/clientSml.h b/source/client/inc/clientSml.h index b711c837c0..c970f1e954 100644 --- a/source/client/inc/clientSml.h +++ b/source/client/inc/clientSml.h @@ -36,48 +36,55 @@ typedef struct { int32_t measureTagsLen; int32_t tagsLen; int32_t colsLen; + int32_t timestampLen; } TAOS_PARSE_ELEMENTS; typedef struct { const char *sTableName; // super table name uint8_t sTableNameLen; - char childTableName[TSDB_TABLE_NAME_LEN]; - uint64_t uid; + char childTableName[TSDB_TABLE_NAME_LEN]; + uint64_t uid; - SArray* tags; - SArray *cols; + SArray *tags; + SArray *cols; // elements are SHashObj for find by key quickly + + SArray colsColumn; // elements are cols key string } TAOS_SML_DATA_POINT_TAGS; typedef struct SSmlSTableMeta { // char *sTableName; // super table name // uint8_t sTableNameLen; - uint8_t precision; // the number of precision - SHashObj* tagHash; - SHashObj* fieldHash; + uint8_t precision; // the number of precision + SHashObj *tagHash; + SHashObj *fieldHash; } SSmlSTableMeta; +typedef struct SMsgBuf { + int32_t len; + char *buf; +} SMsgBuf; + typedef struct { - uint64_t id; + uint64_t id; - SMLProtocolType protocol; - int32_t tsType; + SMLProtocolType protocol; + int32_t tsType; - SHashObj* childTables; - SHashObj* superTables; + SHashObj *childTables; + SHashObj *superTables; - SHashObj* metaHashObj; - SHashObj* pVgHash; + SHashObj *metaHashObj; + SHashObj *pVgHash; - void* exec; + void *exec; - STscObj* taos; - SCatalog* pCatalog; - SRequestObj* pRequest; - SQuery* pQuery; + STscObj *taos; + SCatalog *pCatalog; + SRequestObj *pRequest; + SQuery *pQuery; - int32_t affectedRows; - char *msgBuf; - int16_t msgLen; + int32_t affectedRows; + SMsgBuf msgBuf; } SSmlLinesInfo; int smlInsert(TAOS* taos, SSmlLinesInfo* info); diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index dfbefcb0bf..3739c83109 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -13,6 +13,7 @@ #include "taoserror.h" #include "taos.h" #include "ttime.h" +#include "tstrbuild.h" typedef struct { @@ -31,14 +32,15 @@ typedef struct { #define SLASH '\\' #define tsMaxSQLStringLen (1024*1024) +#define TSNAMELEN 2 +#define TAGNAMELEN 3 //================================================================================================= static uint64_t linesSmlHandleId = 0; +static const char* TS = "ts"; +static const char* TAG = "tag"; + -static int32_t insertChildTablePointsBatch(void* pVoid, char* name, char* name1, SArray* pArray, SArray* pArray1, - SArray* pArray2, SArray* pArray3, size_t size, SSmlLinesInfo* info); -static int32_t doInsertChildTablePoints(void* pVoid, char* sql, char* name, SArray* pArray, SArray* pArray1, - SSmlLinesInfo* info); uint64_t genLinesSmlId() { uint64_t id; @@ -49,9 +51,15 @@ uint64_t genLinesSmlId() { return id; } +static int32_t buildInvalidDataMsg(SMsgBuf* pBuf, const char *msg1, const char *msg2) { + if(msg1) snprintf(pBuf->buf, pBuf->len, "%s:", msg1); + if(msg2) strncpy(pBuf->buf, msg2, pBuf->len); + return TSDB_CODE_SML_INVALID_DATA; +} + int compareSmlColKv(const void* p1, const void* p2) { - TAOS_SML_KV* kv1 = (TAOS_SML_KV*)p1; - TAOS_SML_KV* kv2 = (TAOS_SML_KV*)p2; + SSmlKv* kv1 = (SSmlKv *)p1; + SSmlKv* kv2 = (SSmlKv*)p2; int kvLen1 = (int)strlen(kv1->key); int kvLen2 = (int)strlen(kv2->key); int res = strncasecmp(kv1->key, kv2->key, MIN(kvLen1, kvLen2)); @@ -78,7 +86,7 @@ typedef struct { typedef struct { char sTableName[TSDB_TABLE_NAME_LEN]; - SSchema* field; + SSmlKv * field; } SAlterSTableActionInfo; typedef struct { @@ -89,94 +97,17 @@ typedef struct { }; } SSchemaAction; -static int32_t getFieldBytesFromSmlKv(TAOS_SML_KV* kv, int32_t* bytes, uint64_t id) { - if (!IS_VAR_DATA_TYPE(kv->type)) { - *bytes = tDataTypes[kv->type].bytes; - } else { - if (kv->type == TSDB_DATA_TYPE_NCHAR) { - TdUcs4 *ucs = taosMemoryMalloc(kv->length * TSDB_NCHAR_SIZE + 1); - int32_t bytesNeeded = 0; - bool succ = taosMbsToUcs4(kv->value, kv->length, ucs, kv->length * TSDB_NCHAR_SIZE, &bytesNeeded); - if (!succ) { - taosMemoryFree(ucs); - uError("SML:0x%"PRIx64" convert nchar string to UCS4_LE failed:%s", id, kv->value); - return TSDB_CODE_TSC_INVALID_VALUE; - } - taosMemoryFree(ucs); - *bytes = bytesNeeded + VARSTR_HEADER_SIZE; - } else if (kv->type == TSDB_DATA_TYPE_BINARY) { - *bytes = kv->length + VARSTR_HEADER_SIZE; - } - } - return 0; -} +static int32_t buildSmlChildTableName(TAOS_SML_DATA_POINT_TAGS *tags) { + int32_t size = taosArrayGetSize(tags->tags); + ASSERT(size > 0); + qsort(tags->tags, size, POINTER_BYTES, compareSmlColKv); -static int32_t buildSmlKvSchema(TAOS_SML_KV* smlKv, SHashObj* hash, SArray* array, SSmlLinesInfo* info) { - SSchema* pField = NULL; - size_t* pFieldIdx = taosHashGet(hash, smlKv->key, strlen(smlKv->key)); - size_t fieldIdx = -1; - int32_t code = 0; - if (pFieldIdx) { - fieldIdx = *pFieldIdx; - pField = taosArrayGet(array, fieldIdx); - - if (pField->type != smlKv->type) { - uError("SML:0x%"PRIx64" type mismatch. key %s, type %d. type before %d", info->id, smlKv->key, smlKv->type, pField->type); - return TSDB_CODE_TSC_INVALID_VALUE; - } - - int32_t bytes = 0; - code = getFieldBytesFromSmlKv(smlKv, &bytes, info->id); - if (code != 0) { - return code; - } - pField->bytes = MAX(pField->bytes, bytes); - - } else { - SSchema field = {0}; - size_t tagKeyLen = strlen(smlKv->key); - strncpy(field.name, smlKv->key, tagKeyLen); - field.name[tagKeyLen] = '\0'; - field.type = smlKv->type; - - int32_t bytes = 0; - code = getFieldBytesFromSmlKv(smlKv, &bytes, info->id); - if (code != 0) { - return code; - } - field.bytes = bytes; - - pField = taosArrayPush(array, &field); - fieldIdx = taosArrayGetSize(array) - 1; - taosHashPut(hash, field.name, tagKeyLen, &fieldIdx, sizeof(fieldIdx)); - } - - smlKv->fieldSchemaIdx = (uint32_t)fieldIdx; - - return 0; -} - -static int32_t getSmlMd5ChildTableName(TAOS_SML_DATA_POINT* point, char* tableName, int* tableNameLen, - SSmlLinesInfo* info) { - uDebug("SML:0x%"PRIx64" taos_sml_insert get child table name through md5", info->id); - if (point->tagNum) { - qsort(point->tags, point->tagNum, sizeof(TAOS_SML_KV), compareSmlColKv); - } - - SStringBuilder sb; memset(&sb, 0, sizeof(sb)); - char sTableName[TSDB_TABLE_NAME_LEN] = {0}; - strncpy(sTableName, point->stableName, strlen(point->stableName)); - //strtolower(sTableName, point->stableName); - taosStringBuilderAppendString(&sb, sTableName); - for (int j = 0; j < point->tagNum; ++j) { - taosStringBuilderAppendChar(&sb, ','); - TAOS_SML_KV* tagKv = point->tags + j; - char tagName[TSDB_COL_NAME_LEN] = {0}; - strncpy(tagName, tagKv->key, strlen(tagKv->key)); - //strtolower(tagName, tagKv->key); - taosStringBuilderAppendString(&sb, tagName); - taosStringBuilderAppendChar(&sb, '='); - taosStringBuilderAppend(&sb, tagKv->value, tagKv->length); + SStringBuilder sb = {0}; + taosStringBuilderAppendStringLen(&sb, tags->sTableName, tags->sTableNameLen); + for (int j = 0; j < size; ++j) { + SSmlKv *tagKv = taosArrayGetP(tags->tags, j); + taosStringBuilderAppendStringLen(&sb, tagKv->key, tagKv->keyLen); + taosStringBuilderAppendStringLen(&sb, tagKv->value, tagKv->valueLen); } size_t len = 0; char* keyJoined = taosStringBuilderGetResult(&sb, &len); @@ -186,183 +117,74 @@ static int32_t getSmlMd5ChildTableName(TAOS_SML_DATA_POINT* point, char* tableNa tMD5Final(&context); uint64_t digest1 = *(uint64_t*)(context.digest); uint64_t digest2 = *(uint64_t*)(context.digest + 8); - *tableNameLen = snprintf(tableName, *tableNameLen, - "t_%016"PRIx64"%016"PRIx64, digest1, digest2); + snprintf(tags->childTableName, TSDB_TABLE_NAME_LEN, "t_%016"PRIx64"%016"PRIx64, digest1, digest2); taosStringBuilderDestroy(&sb); - uDebug("SML:0x%"PRIx64" child table name: %s", info->id, tableName); - return 0; -} - -static int32_t buildSmlChildTableName(TAOS_SML_DATA_POINT* point, SSmlLinesInfo* info) { - uDebug("SML:0x%"PRIx64" taos_sml_insert build child table name", info->id); - char childTableName[TSDB_TABLE_NAME_LEN]; - int32_t tableNameLen = TSDB_TABLE_NAME_LEN; - getSmlMd5ChildTableName(point, childTableName, &tableNameLen, info); - point->childTableName = calloc(1, tableNameLen+1); - strncpy(point->childTableName, childTableName, tableNameLen); - point->childTableName[tableNameLen] = '\0'; - return 0; -} - -static int32_t buildDataPointSchemas(TAOS_SML_DATA_POINT* points, int numPoint, SArray* stableSchemas, SSmlLinesInfo* info) { - int32_t code = 0; - SHashObj* sname2shema = taosHashInit(32, - taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); - - for (int i = 0; i < numPoint; ++i) { - TAOS_SML_DATA_POINT* point = &points[i]; - size_t stableNameLen = strlen(point->stableName); - size_t* pStableIdx = taosHashGet(sname2shema, point->stableName, stableNameLen); - SSmlSTableSchema* pStableSchema = NULL; - size_t stableIdx = -1; - if (pStableIdx) { - pStableSchema= taosArrayGet(stableSchemas, *pStableIdx); - stableIdx = *pStableIdx; - } else { - SSmlSTableSchema schema; - strncpy(schema.sTableName, point->stableName, stableNameLen); - schema.sTableName[stableNameLen] = '\0'; - schema.fields = taosArrayInit(64, sizeof(SSchema)); - schema.tags = taosArrayInit(8, sizeof(SSchema)); - schema.tagHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); - schema.fieldHash = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); - - pStableSchema = taosArrayPush(stableSchemas, &schema); - stableIdx = taosArrayGetSize(stableSchemas) - 1; - taosHashPut(sname2shema, schema.sTableName, stableNameLen, &stableIdx, sizeof(size_t)); - } - - for (int j = 0; j < point->tagNum; ++j) { - TAOS_SML_KV* tagKv = point->tags + j; - if (!point->childTableName) { - buildSmlChildTableName(point, info); - } - - code = buildSmlKvSchema(tagKv, pStableSchema->tagHash, pStableSchema->tags, info); - if (code != 0) { - uError("SML:0x%"PRIx64" build data point schema failed. point no.: %d, tag key: %s", info->id, i, tagKv->key); - return code; - } - } - - //for Line Protocol tags may be omitted, add a tag with NULL value - if (point->tagNum == 0) { - if (!point->childTableName) { - buildSmlChildTableName(point, info); - } - char tagNullName[TSDB_COL_NAME_LEN] = {0}; - size_t nameLen = strlen(tsSmlTagNullName); - strncpy(tagNullName, tsSmlTagNullName, nameLen); - addEscapeCharToString(tagNullName, (int32_t)nameLen); - size_t* pTagNullIdx = taosHashGet(pStableSchema->tagHash, tagNullName, nameLen); - if (!pTagNullIdx) { - SSchema tagNull = {0}; - tagNull.type = TSDB_DATA_TYPE_NCHAR; - tagNull.bytes = TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE; - strncpy(tagNull.name, tagNullName, nameLen); - taosArrayPush(pStableSchema->tags, &tagNull); - size_t tagNullIdx = taosArrayGetSize(pStableSchema->tags) - 1; - taosHashPut(pStableSchema->tagHash, tagNull.name, nameLen, &tagNullIdx, sizeof(tagNullIdx)); - } - } - - for (int j = 0; j < point->fieldNum; ++j) { - TAOS_SML_KV* fieldKv = point->fields + j; - code = buildSmlKvSchema(fieldKv, pStableSchema->fieldHash, pStableSchema->fields, info); - if (code != 0) { - uError("SML:0x%"PRIx64" build data point schema failed. point no.: %d, tag key: %s", info->id, i, fieldKv->key); - return code; - } - } - - point->schemaIdx = (uint32_t)stableIdx; - } - - size_t numStables = taosArrayGetSize(stableSchemas); - for (int32_t i = 0; i < numStables; ++i) { - SSmlSTableSchema* schema = taosArrayGet(stableSchemas, i); - taosHashCleanup(schema->tagHash); - taosHashCleanup(schema->fieldHash); - } - taosHashCleanup(sname2shema); - - uDebug("SML:0x%"PRIx64" build point schema succeed. num of super table: %zu", info->id, numStables); - for (int32_t i = 0; i < numStables; ++i) { - SSmlSTableSchema* schema = taosArrayGet(stableSchemas, i); - uDebug("\ttable name: %s, tags number: %zu, fields number: %zu", schema->sTableName, - taosArrayGetSize(schema->tags), taosArrayGetSize(schema->fields)); - } - + tags->uid = digest1; + uDebug("SML: child table name: %s", tags->childTableName); return 0; } static int32_t generateSchemaAction(SSchema* pointColField, SHashObj* dbAttrHash, SArray* dbAttrArray, bool isTag, char sTableName[], SSchemaAction* action, bool* actionNeeded, SSmlLinesInfo* info) { - char fieldName[TSDB_COL_NAME_LEN] = {0}; - strcpy(fieldName, pointColField->name); - - size_t* pDbIndex = taosHashGet(dbAttrHash, fieldName, strlen(fieldName)); - if (pDbIndex) { - SSchema* dbAttr = taosArrayGet(dbAttrArray, *pDbIndex); - assert(strcasecmp(dbAttr->name, pointColField->name) == 0); - if (pointColField->type != dbAttr->type) { - uError("SML:0x%"PRIx64" point type and db type mismatch. key: %s. point type: %d, db type: %d", info->id, pointColField->name, - pointColField->type, dbAttr->type); - return TSDB_CODE_TSC_INVALID_VALUE; - } - - if (IS_VAR_DATA_TYPE(pointColField->type) && (pointColField->bytes > dbAttr->bytes)) { - if (isTag) { - action->action = SCHEMA_ACTION_CHANGE_TAG_SIZE; - } else { - action->action = SCHEMA_ACTION_CHANGE_COLUMN_SIZE; - } - memset(&action->alterSTable, 0, sizeof(SAlterSTableActionInfo)); - memcpy(action->alterSTable.sTableName, sTableName, TSDB_TABLE_NAME_LEN); - action->alterSTable.field = pointColField; - *actionNeeded = true; - } - } else { - if (isTag) { - action->action = SCHEMA_ACTION_ADD_TAG; - } else { - action->action = SCHEMA_ACTION_ADD_COLUMN; - } - memset(&action->alterSTable, 0, sizeof(SAlterSTableActionInfo)); - memcpy(action->alterSTable.sTableName, sTableName, TSDB_TABLE_NAME_LEN); - action->alterSTable.field = pointColField; - *actionNeeded = true; - } - if (*actionNeeded) { - uDebug("SML:0x%" PRIx64 " generate schema action. column name: %s, action: %d", info->id, fieldName, - action->action); - } +// char fieldName[TSDB_COL_NAME_LEN] = {0}; +// strcpy(fieldName, pointColField->name); +// +// size_t* pDbIndex = taosHashGet(dbAttrHash, fieldName, strlen(fieldName)); +// if (pDbIndex) { +// SSchema* dbAttr = taosArrayGet(dbAttrArray, *pDbIndex); +// assert(strcasecmp(dbAttr->name, pointColField->name) == 0); +// if (pointColField->type != dbAttr->type) { +// uError("SML:0x%"PRIx64" point type and db type mismatch. key: %s. point type: %d, db type: %d", info->id, pointColField->name, +// pointColField->type, dbAttr->type); +// return TSDB_CODE_TSC_INVALID_VALUE; +// } +// +// if (IS_VAR_DATA_TYPE(pointColField->type) && (pointColField->bytes > dbAttr->bytes)) { +// if (isTag) { +// action->action = SCHEMA_ACTION_CHANGE_TAG_SIZE; +// } else { +// action->action = SCHEMA_ACTION_CHANGE_COLUMN_SIZE; +// } +// memset(&action->alterSTable, 0, sizeof(SAlterSTableActionInfo)); +// memcpy(action->alterSTable.sTableName, sTableName, TSDB_TABLE_NAME_LEN); +// action->alterSTable.field = pointColField; +// *actionNeeded = true; +// } +// } else { +// if (isTag) { +// action->action = SCHEMA_ACTION_ADD_TAG; +// } else { +// action->action = SCHEMA_ACTION_ADD_COLUMN; +// } +// memset(&action->alterSTable, 0, sizeof(SAlterSTableActionInfo)); +// memcpy(action->alterSTable.sTableName, sTableName, TSDB_TABLE_NAME_LEN); +// action->alterSTable.field = pointColField; +// *actionNeeded = true; +// } +// if (*actionNeeded) { +// uDebug("SML:0x%" PRIx64 " generate schema action. column name: %s, action: %d", info->id, fieldName, +// action->action); +// } return 0; } -static int32_t buildColumnDescription(TAOS_SML_KV* field, - char* buf, int32_t bufSize, int32_t* outBytes) { +static int32_t buildColumnDescription(SSmlKv* field, char* buf, int32_t bufSize, int32_t* outBytes) { uint8_t type = field->type; char tname[TSDB_TABLE_NAME_LEN] = {0}; memcpy(tname, field->key, field->keyLen); if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { - int32_t bytes = field->length - VARSTR_HEADER_SIZE; - if (type == TSDB_DATA_TYPE_NCHAR) { - bytes = bytes/TSDB_NCHAR_SIZE; - } + int32_t bytes = field->valueLen; // todo int out = snprintf(buf, bufSize,"%s %s(%d)", tname,tDataTypes[field->type].name, bytes); *outBytes = out; } else { - int out = snprintf(buf, bufSize, "%s %s", - tname, tDataTypes[type].name); + int out = snprintf(buf, bufSize, "%s %s", tname, tDataTypes[type].name); *outBytes = out; } return 0; } - static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInfo* info) { int32_t code = 0; int32_t outBytes = 0; @@ -472,7 +294,7 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInf int n = sprintf(result, "create stable %s (", action->createSTable.sTableName); char* pos = result + n; int freeBytes = capacity - n; - TAOS_SML_KV **kv = taosHashIterate(action->createSTable.fields, NULL); + SSmlKv **kv = taosHashIterate(action->createSTable.fields, NULL); while(kv){ buildColumnDescription(*kv, pos, freeBytes, &outBytes); pos += outBytes; freeBytes -= outBytes; @@ -523,168 +345,6 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInf return code; } -static int32_t destroySmlSTableSchema(SSmlSTableSchema* schema) { - taosHashCleanup(schema->tagHash); - taosHashCleanup(schema->fieldHash); - taosArrayDestroy(&schema->tags); - taosArrayDestroy(&schema->fields); - return 0; -} - -static int32_t fillDbSchema(STableMeta* tableMeta, char* tableName, SSmlSTableSchema* schema, SSmlLinesInfo* info) { - schema->tags = taosArrayInit(8, sizeof(SSchema)); - schema->fields = taosArrayInit(64, sizeof(SSchema)); - schema->tagHash = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); - schema->fieldHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); - - tstrncpy(schema->sTableName, tableName, strlen(tableName)+1); - schema->precision = tableMeta->tableInfo.precision; - for (int i=0; itableInfo.numOfColumns; ++i) { - SSchema field; - tstrncpy(field.name, tableMeta->schema[i].name, strlen(tableMeta->schema[i].name)+1); - addEscapeCharToString(field.name, (int16_t)strlen(field.name)); - field.type = tableMeta->schema[i].type; - field.bytes = tableMeta->schema[i].bytes; - taosArrayPush(schema->fields, &field); - size_t fieldIndex = taosArrayGetSize(schema->fields) - 1; - taosHashPut(schema->fieldHash, field.name, strlen(field.name), &fieldIndex, sizeof(fieldIndex)); - } - - for (int i=0; itableInfo.numOfTags; ++i) { - int j = i + tableMeta->tableInfo.numOfColumns; - SSchema field; - tstrncpy(field.name, tableMeta->schema[j].name, strlen(tableMeta->schema[j].name)+1); - addEscapeCharToString(field.name, (int16_t)strlen(field.name)); - field.type = tableMeta->schema[j].type; - field.bytes = tableMeta->schema[j].bytes; - taosArrayPush(schema->tags, &field); - size_t tagIndex = taosArrayGetSize(schema->tags) - 1; - taosHashPut(schema->tagHash, field.name, strlen(field.name), &tagIndex, sizeof(tagIndex)); - } - uDebug("SML:0x%"PRIx64 " load table schema succeed. table name: %s, columns number: %d, tag number: %d, precision: %d", - info->id, tableName, tableMeta->tableInfo.numOfColumns, tableMeta->tableInfo.numOfTags, schema->precision); - return TSDB_CODE_SUCCESS; -} - -static int32_t getSuperTableMetaFromLocalCache(TAOS* taos, char* tableName, STableMeta** outTableMeta, SSmlLinesInfo* info) { - int32_t code = 0; - STableMeta* tableMeta = NULL; - - SSqlObj* pSql = calloc(1, sizeof(SSqlObj)); - if (pSql == NULL) { - uError("SML:0x%" PRIx64 " failed to allocate memory, reason:%s", info->id, strerror(errno)); - code = TSDB_CODE_TSC_OUT_OF_MEMORY; - return code; - } - pSql->pTscObj = taos; - pSql->signature = pSql; - pSql->fp = NULL; - - registerSqlObj(pSql); - char tableNameBuf[TSDB_TABLE_NAME_LEN + TS_BACKQUOTE_CHAR_SIZE] = {0}; - memcpy(tableNameBuf, tableName, strlen(tableName)); - SStrToken tableToken = {.z = tableNameBuf, .n = (uint32_t)strlen(tableName), .type = TK_ID}; - tGetToken(tableNameBuf, &tableToken.type); - bool dbIncluded = false; - // Check if the table name available or not - if (tscValidateName(&tableToken, true, &dbIncluded) != TSDB_CODE_SUCCESS) { - code = TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; - sprintf(pSql->cmd.payload, "table name is invalid"); - taosReleaseRef(tscObjRef, pSql->self); - return code; - } - - SName sname = {0}; - if ((code = tscSetTableFullName(&sname, &tableToken, pSql, dbIncluded)) != TSDB_CODE_SUCCESS) { - taosReleaseRef(tscObjRef, pSql->self); - return code; - } - - char fullTableName[TSDB_TABLE_FNAME_LEN] = {0}; - memset(fullTableName, 0, tListLen(fullTableName)); - tNameExtractFullName(&sname, fullTableName); - - size_t size = 0; - taosHashGetCloneExt(UTIL_GET_TABLEMETA(pSql), fullTableName, strlen(fullTableName), NULL, (void**)&tableMeta, &size); - - STableMeta* stableMeta = tableMeta; - if (tableMeta != NULL && tableMeta->tableType == TSDB_CHILD_TABLE) { - taosHashGetCloneExt(UTIL_GET_TABLEMETA(pSql), tableMeta->sTableName, strlen(tableMeta->sTableName), NULL, - (void**)stableMeta, &size); - } - taosReleaseRef(tscObjRef, pSql->self); - - if (stableMeta != tableMeta) { - taosMemoryFree(tableMeta); - } - - if (stableMeta != NULL) { - if (outTableMeta != NULL) { - *outTableMeta = stableMeta; - } else { - taosMemoryFree(stableMeta); - } - return TSDB_CODE_SUCCESS; - } else { - return TSDB_CODE_TSC_NO_META_CACHED; - } -} - -static int32_t retrieveTableMeta(TAOS* taos, char* tableName, STableMeta** pTableMeta, SSmlLinesInfo* info) { - int32_t code = 0; - int32_t retries = 0; - STableMeta* tableMeta = NULL; - while (retries++ <= TSDB_MAX_REPLICA && tableMeta == NULL) { - STscObj* pObj = (STscObj*)taos; - if (pObj == NULL || pObj->signature != pObj) { - terrno = TSDB_CODE_TSC_DISCONNECTED; - return TSDB_CODE_TSC_DISCONNECTED; - } - - uDebug("SML:0x%" PRIx64 " retrieve table meta. super table name: %s", info->id, tableName); - code = getSuperTableMetaFromLocalCache(taos, tableName, &tableMeta, info); - if (code == TSDB_CODE_SUCCESS) { - uDebug("SML:0x%" PRIx64 " successfully retrieved table meta. super table name: %s", info->id, tableName); - break; - } else if (code == TSDB_CODE_TSC_NO_META_CACHED) { - char sql[256]; - snprintf(sql, 256, "describe %s", tableName); - TAOS_RES* res = taos_query(taos, sql); - code = taos_errno(res); - if (code != 0) { - uError("SML:0x%" PRIx64 " describe table failure. %s", info->id, taos_errstr(res)); - taos_free_result(res); - return code; - } - taos_free_result(res); - } else { - return code; - } - } - - if (tableMeta != NULL) { - *pTableMeta = tableMeta; - return TSDB_CODE_SUCCESS; - } else { - uError("SML:0x%" PRIx64 " failed to retrieve table meta. super table name: %s", info->id, tableName); - return TSDB_CODE_TSC_NO_META_CACHED; - } -} - -static int32_t loadTableSchemaFromDB(TAOS* taos, char* tableName, SSmlSTableSchema* schema, SSmlLinesInfo* info) { - int32_t code = 0; - STableMeta* tableMeta = NULL; - code = retrieveTableMeta(taos, tableName, &tableMeta, info); - if (code == TSDB_CODE_SUCCESS) { - assert(tableMeta != NULL); - fillDbSchema(tableMeta, tableName, schema, info); - taosMemoryFree(tableMeta); - tableMeta = NULL; - } - - return code; -} - static int32_t modifyDBSchemas(TAOS* taos, SSmlLinesInfo* info) { int32_t code = 0; @@ -706,7 +366,6 @@ static int32_t modifyDBSchemas(TAOS* taos, SSmlLinesInfo* info) { if (code == TSDB_CODE_TDB_INVALID_TABLE_ID) { SSchemaAction schemaAction = {0}; schemaAction.action = SCHEMA_ACTION_CREATE_STABLE; - memset(&schemaAction.createSTable, 0, sizeof(SCreateSTableActionInfo)); memcpy(schemaAction.createSTable.sTableName, superTable, superTableLen); schemaAction.createSTable.tags = cTablePoints->tagHash; schemaAction.createSTable.fields = cTablePoints->fieldHash; @@ -728,7 +387,7 @@ static int32_t modifyDBSchemas(TAOS* taos, SSmlLinesInfo* info) { return 0; } -static int32_t applyDataPoints(TAOS* taos, SSmlLinesInfo* info) { +static int32_t applyDataPoints(SSmlLinesInfo* info) { int32_t code = TSDB_CODE_SUCCESS; TAOS_SML_DATA_POINT_TAGS** oneTable = taosHashIterate(info->childTables, NULL); @@ -746,43 +405,40 @@ static int32_t applyDataPoints(TAOS* taos, SSmlLinesInfo* info) { STableMeta** pMeta = taosHashGet(info->metaHashObj, tableData->sTableName, tableData->sTableNameLen); ASSERT (NULL != pMeta && NULL != *pMeta); (*pMeta)->vgId = vg.vgId; - (*pMeta)->uid = tableData->uid; - - smlBind(info->exec, tableData->tags, tableData->cols, *pMeta, info->msgBuf, info->msgLen); + (*pMeta)->uid = tableData->uid; // one table merge data block together according uid + code = smlBind(info->exec, tableData->tags, tableData->cols, *pMeta, info->msgBuf.buf, info->msgBuf.len); + if(code != TSDB_CODE_SUCCESS){ + return code; + } oneTable = taosHashIterate(info->childTables, oneTable); } smlBuildOutput(info->exec, info->pVgHash); launchQueryImpl(info->pRequest, info->pQuery, TSDB_CODE_SUCCESS, true); - if(info->pRequest->code != TSDB_CODE_SUCCESS){ - - } info->affectedRows = taos_affected_rows(info->pRequest); - return code; + return info->pRequest->code; } -int tscSmlInsert(TAOS* taos, SSmlLinesInfo* info) { +int smlInsert(TAOS* taos, SSmlLinesInfo* info) { uDebug("SML:0x%"PRIx64" taos_sml_insert. number of super tables: %d", info->id, taosHashGetSize(info->superTables)); - int32_t code = TSDB_CODE_SUCCESS; - info->affectedRows = 0; uDebug("SML:0x%"PRIx64" modify db schemas", info->id); - code = modifyDBSchemas(taos, info); + int32_t code = modifyDBSchemas(taos, info); if (code != 0) { uError("SML:0x%"PRIx64" error change db schema : %s", info->id, tstrerror(code)); - goto clean_up; + return code; } uDebug("SML:0x%"PRIx64" apply data points", info->id); - code = applyDataPoints(taos, info); + code = applyDataPoints(info); if (code != 0) { uError("SML:0x%"PRIx64" error apply data points : %s", info->id, tstrerror(code)); + return code; } -clean_up: - return code; + return TSDB_CODE_SUCCESS; } //========================================================================= @@ -835,242 +491,307 @@ static void escapeSpecialCharacter(uint8_t field, const char **pos) { *pos = cur; } -bool isValidInteger(char *str) { - char *c = str; - if (*c != '+' && *c != '-' && !isdigit(*c)) { - return false; - } - c++; - while (*c != '\0') { - if (!isdigit(*c)) { - return false; - } - c++; - } - return true; -} - -bool isValidFloat(char *str) { - char *c = str; - uint8_t has_dot, has_exp, has_sign; - has_dot = 0; - has_exp = 0; - has_sign = 0; - - if (*c != '+' && *c != '-' && *c != '.' && !isdigit(*c)) { - return false; - } - if (*c == '.' && isdigit(*(c + 1))) { - has_dot = 1; - } - c++; - while (*c != '\0') { - if (!isdigit(*c)) { - switch (*c) { - case '.': { - if (!has_dot && !has_exp && isdigit(*(c + 1))) { - has_dot = 1; - } else { - return false; - } - break; - } - case 'e': - case 'E': { - if (!has_exp && isdigit(*(c - 1)) && - (isdigit(*(c + 1)) || - *(c + 1) == '+' || - *(c + 1) == '-')) { - has_exp = 1; - } else { - return false; - } - break; - } - case '+': - case '-': { - if (!has_sign && has_exp && isdigit(*(c + 1))) { - has_sign = 1; - } else { - return false; - } - break; - } - default: { - return false; - } - } - } - c++; - } //while - return true; -} - -static bool isInteger(char *pVal, uint16_t len, bool *has_sign) { - if (len <= 1) { - return false; - } - if (pVal[len - 1] == 'i') { - *has_sign = true; - return true; - } - if (pVal[len - 1] == 'u') { - *has_sign = false; - return true; - } - - return false; -} - -static bool isTinyInt(char *pVal, uint16_t len) { +static bool parseTinyInt(SSmlKv *kvVal, bool *isValid, SMsgBuf *msg) { + const char *pVal = kvVal->value; + int32_t len = kvVal->valueLen; if (len <= 2) { return false; } - if (!strcasecmp(&pVal[len - 2], "i8")) { - //printf("Type is int8(%s)\n", pVal); + const char *signalPos = pVal + len - 2; + if (!strcasecmp(signalPos, "i8")) { + char *endptr = NULL; + int64_t result = strtoll(pVal, &endptr, 10); + if(endptr != signalPos){ // 78ri8 + *isValid = false; + buildInvalidDataMsg(msg, "invalid tiny int", endptr); + }else if(!IS_VALID_TINYINT(result)){ + *isValid = false; + buildInvalidDataMsg(msg, "tiny int out of range[-128,127]", endptr); + }else{ + kvVal->i = result; + *isValid = true; + } return true; } return false; } -static bool isTinyUint(char *pVal, uint16_t len) { +static bool parseTinyUint(SSmlKv *kvVal, bool *isValid, SMsgBuf *msg) { + const char *pVal = kvVal->value; + int32_t len = kvVal->valueLen; if (len <= 2) { return false; } if (pVal[0] == '-') { return false; } - if (!strcasecmp(&pVal[len - 2], "u8")) { - //printf("Type is uint8(%s)\n", pVal); + const char *signalPos = pVal + len - 2; + if (!strcasecmp(signalPos, "u8")) { + char *endptr = NULL; + int64_t result = strtoll(pVal, &endptr, 10); + if(endptr != signalPos){ // 78ri8 + *isValid = false; + buildInvalidDataMsg(msg, "invalid unsigned tiny int", endptr); + }else if(!IS_VALID_UTINYINT(result)){ + *isValid = false; + buildInvalidDataMsg(msg, "unsigned tiny int out of range[0,255]", endptr); + }else{ + kvVal->i = result; + *isValid = true; + } return true; } return false; } -static bool isSmallInt(char *pVal, uint16_t len) { +static bool parseSmallInt(SSmlKv *kvVal, bool *isValid, SMsgBuf *msg) { + const char *pVal = kvVal->value; + int32_t len = kvVal->valueLen; if (len <= 3) { return false; } - if (!strcasecmp(&pVal[len - 3], "i16")) { - //printf("Type is int16(%s)\n", pVal); + const char *signalPos = pVal + len - 3; + if (!strcasecmp(signalPos, "i16")) { + char *endptr = NULL; + int64_t result = strtoll(pVal, &endptr, 10); + if(endptr != signalPos){ // 78ri8 + *isValid = false; + buildInvalidDataMsg(msg, "invalid small int", endptr); + }else if(!IS_VALID_SMALLINT(result)){ + *isValid = false; + buildInvalidDataMsg(msg, "small int our of range[-32768,32767]", endptr); + }else{ + kvVal->i = result; + *isValid = true; + } return true; } return false; } -static bool isSmallUint(char *pVal, uint16_t len) { +static bool parseSmallUint(SSmlKv *kvVal, bool *isValid, SMsgBuf *msg) { + const char *pVal = kvVal->value; + int32_t len = kvVal->valueLen; if (len <= 3) { return false; } if (pVal[0] == '-') { return false; } - if (strcasecmp(&pVal[len - 3], "u16") == 0) { - //printf("Type is uint16(%s)\n", pVal); + const char *signalPos = pVal + len - 3; + if (strcasecmp(signalPos, "u16") == 0) { + char *endptr = NULL; + int64_t result = strtoll(pVal, &endptr, 10); + if(endptr != signalPos){ // 78ri8 + *isValid = false; + buildInvalidDataMsg(msg, "invalid unsigned small int", endptr); + }else if(!IS_VALID_USMALLINT(result)){ + *isValid = false; + buildInvalidDataMsg(msg, "unsigned small int out of rang[0,65535]", endptr); + }else{ + kvVal->i = result; + *isValid = true; + } return true; } return false; } -static bool isInt(char *pVal, uint16_t len) { +static bool parseInt(SSmlKv *kvVal, bool *isValid, SMsgBuf *msg) { + const char *pVal = kvVal->value; + int32_t len = kvVal->valueLen; if (len <= 3) { return false; } - if (strcasecmp(&pVal[len - 3], "i32") == 0) { - //printf("Type is int32(%s)\n", pVal); + const char *signalPos = pVal + len - 3; + if (strcasecmp(signalPos, "i32") == 0) { + char *endptr = NULL; + int64_t result = strtoll(pVal, &endptr, 10); + if(endptr != signalPos){ // 78ri8 + *isValid = false; + buildInvalidDataMsg(msg, "invalid int", endptr); + }else if(!IS_VALID_INT(result)){ + *isValid = false; + buildInvalidDataMsg(msg, "int out of range[-2147483648,2147483647]", endptr); + }else{ + kvVal->i = result; + *isValid = true; + } return true; } return false; } -static bool isUint(char *pVal, uint16_t len) { +static bool parseUint(SSmlKv *kvVal, bool *isValid, SMsgBuf *msg) { + const char *pVal = kvVal->value; + int32_t len = kvVal->valueLen; if (len <= 3) { return false; } if (pVal[0] == '-') { return false; } - if (strcasecmp(&pVal[len - 3], "u32") == 0) { - //printf("Type is uint32(%s)\n", pVal); + const char *signalPos = pVal + len - 3; + if (strcasecmp(signalPos, "u32") == 0) { + char *endptr = NULL; + int64_t result = strtoll(pVal, &endptr, 10); + if(endptr != signalPos){ // 78ri8 + *isValid = false; + buildInvalidDataMsg(msg, "invalid unsigned int", endptr); + }else if(!IS_VALID_UINT(result)){ + *isValid = false; + buildInvalidDataMsg(msg, "unsigned int out of range[0,4294967295]", endptr); + }else{ + kvVal->i = result; + *isValid = true; + } return true; } return false; } -static bool isBigInt(char *pVal, uint16_t len) { - if (len <= 3) { - return false; - } - if (strcasecmp(&pVal[len - 3], "i64") == 0) { - //printf("Type is int64(%s)\n", pVal); +static bool parseBigInt(SSmlKv *kvVal, bool *isValid, SMsgBuf *msg) { + const char *pVal = kvVal->value; + int32_t len = kvVal->valueLen; + if (len > 3 && strcasecmp(pVal + len - 3, "i64") == 0) { + char *endptr = NULL; + int64_t result = strtoll(pVal, &endptr, 10); + if(endptr != pVal + len - 3){ // 78ri8 + *isValid = false; + }else if(!IS_VALID_BIGINT(result)){ + *isValid = false; + }else{ + kvVal->i = result; + *isValid = true; + } + return true; + }else if (len > 1 && pVal[len - 1] == 'i') { + char *endptr = NULL; + int64_t result = strtoll(pVal, &endptr, 10); + if(endptr != pVal + len - 1){ // 78ri8 + *isValid = false; + buildInvalidDataMsg(msg, "invalid big int", endptr); + }else if(!IS_VALID_BIGINT(result)){ + *isValid = false; + buildInvalidDataMsg(msg, "big int out of range[-9223372036854775808,9223372036854775807]", endptr); + }else{ + kvVal->i = result; + *isValid = true; + } return true; } return false; } -static bool isBigUint(char *pVal, uint16_t len) { +static bool parseBigUint(SSmlKv *kvVal, bool *isValid, SMsgBuf *msg) { + const char *pVal = kvVal->value; + int32_t len = kvVal->valueLen; if (len <= 3) { return false; } if (pVal[0] == '-') { return false; } - if (strcasecmp(&pVal[len - 3], "u64") == 0) { - //printf("Type is uint64(%s)\n", pVal); + const char *signalPos = pVal + len - 3; + if (strcasecmp(signalPos, "u64") == 0) { + char *endptr = NULL; + uint64_t result = strtoull(pVal, &endptr, 10); + if(endptr != signalPos){ // 78ri8 + *isValid = false; + buildInvalidDataMsg(msg, "invalid unsigned big int", endptr); + }else if(!IS_VALID_UBIGINT(result)){ + *isValid = false; + buildInvalidDataMsg(msg, "unsigned big int out of range[0,18446744073709551615]", endptr); + }else{ + kvVal->u = result; + *isValid = true; + } return true; } return false; } -static bool isFloat(char *pVal, uint16_t len) { +static bool parseFloat(SSmlKv *kvVal, bool *isValid, SMsgBuf *msg) { + const char *pVal = kvVal->value; + int32_t len = kvVal->valueLen; + char *endptr = NULL; + float result = strtof(pVal, &endptr); + if(endptr == pVal + len && IS_VALID_FLOAT(result)){ // 78 + kvVal->f = result; + *isValid = true; + return true; + } + + if (len > 3 && len f = result; + *isValid = true; + } + return true; + } + return false; +} + +static bool parseDouble(SSmlKv *kvVal, bool *isValid, SMsgBuf *msg) { + const char *pVal = kvVal->value; + int32_t len = kvVal->valueLen; if (len <= 3) { return false; } - if (strcasecmp(&pVal[len - 3], "f32") == 0) { - //printf("Type is float(%s)\n", pVal); + const char *signalPos = pVal + len - 3; + if (len d = result; + *isValid = true; + } return true; } return false; } -static bool isDouble(char *pVal, uint16_t len) { - if (len <= 3) { - return false; - } - if (strcasecmp(&pVal[len - 3], "f64") == 0) { - //printf("Type is double(%s)\n", pVal); - return true; - } - return false; -} - -static bool isBool(char *pVal, uint16_t len, bool *bVal) { - if ((len == 1) && !strcasecmp(&pVal[len - 1], "t")) { +static bool parseBool(SSmlKv *kvVal) { + const char *pVal = kvVal->value; + int32_t len = kvVal->valueLen; + if ((len == 1) && pVal[len - 1] == 't') { //printf("Type is bool(%c)\n", pVal[len - 1]); - *bVal = true; + kvVal->i = true; return true; } - if ((len == 1) && !strcasecmp(&pVal[len - 1], "f")) { + if ((len == 1) && pVal[len - 1] == 'f') { //printf("Type is bool(%c)\n", pVal[len - 1]); - *bVal = false; + kvVal->i = false; return true; } - if((len == 4) && !strcasecmp(&pVal[len - 4], "true")) { + if((len == 4) && !strcasecmp(pVal, "true")) { //printf("Type is bool(%s)\n", &pVal[len - 4]); - *bVal = true; + kvVal->i = true; return true; } - if((len == 5) && !strcasecmp(&pVal[len - 5], "false")) { + if((len == 5) && !strcasecmp(pVal, "false")) { //printf("Type is bool(%s)\n", &pVal[len - 5]); - *bVal = false; + kvVal->i = false; return true; } return false; } -static bool isBinary(char *pVal, uint16_t len) { +static bool isBinary(const char *pVal, uint16_t len) { //binary: "abc" if (len < 2) { return false; @@ -1083,7 +804,7 @@ static bool isBinary(char *pVal, uint16_t len) { return false; } -static bool isNchar(char *pVal, uint16_t len) { +static bool isNchar(const char *pVal, uint16_t len) { //nchar: L"abc" if (len < 3) { return false; @@ -1095,266 +816,97 @@ static bool isNchar(char *pVal, uint16_t len) { return false; } -static bool convertStrToNumber(TAOS_SML_KV *pVal, char *str, SSmlLinesInfo* info) { - errno = 0; - uint8_t type = pVal->type; - int16_t length = pVal->length; - int64_t val_s = 0; - uint64_t val_u = 0; - double val_d = 0.0; - - strntolower_s(str, str, (int32_t)strlen(str)); - if (IS_FLOAT_TYPE(type)) { - val_d = strtod(str, NULL); - } else { - if (IS_SIGNED_NUMERIC_TYPE(type)) { - val_s = strtoll(str, NULL, 10); - } else { - val_u = strtoull(str, NULL, 10); - } - } - - if (errno == ERANGE) { - uError("SML:0x%"PRIx64" Convert number(%s) out of range", info->id, str); - return false; - } - - switch (type) { - case TSDB_DATA_TYPE_TINYINT: - if (!IS_VALID_TINYINT(val_s)) { - return false; - } - pVal->value = calloc(length, 1); - *(int8_t *)(pVal->value) = (int8_t)val_s; - break; - case TSDB_DATA_TYPE_UTINYINT: - if (!IS_VALID_UTINYINT(val_u)) { - return false; - } - pVal->value = calloc(length, 1); - *(uint8_t *)(pVal->value) = (uint8_t)val_u; - break; - case TSDB_DATA_TYPE_SMALLINT: - if (!IS_VALID_SMALLINT(val_s)) { - return false; - } - pVal->value = calloc(length, 1); - *(int16_t *)(pVal->value) = (int16_t)val_s; - break; - case TSDB_DATA_TYPE_USMALLINT: - if (!IS_VALID_USMALLINT(val_u)) { - return false; - } - pVal->value = calloc(length, 1); - *(uint16_t *)(pVal->value) = (uint16_t)val_u; - break; - case TSDB_DATA_TYPE_INT: - if (!IS_VALID_INT(val_s)) { - return false; - } - pVal->value = calloc(length, 1); - *(int32_t *)(pVal->value) = (int32_t)val_s; - break; - case TSDB_DATA_TYPE_UINT: - if (!IS_VALID_UINT(val_u)) { - return false; - } - pVal->value = calloc(length, 1); - *(uint32_t *)(pVal->value) = (uint32_t)val_u; - break; - case TSDB_DATA_TYPE_BIGINT: - if (!IS_VALID_BIGINT(val_s)) { - return false; - } - pVal->value = calloc(length, 1); - *(int64_t *)(pVal->value) = (int64_t)val_s; - break; - case TSDB_DATA_TYPE_UBIGINT: - if (!IS_VALID_UBIGINT(val_u)) { - return false; - } - pVal->value = calloc(length, 1); - *(uint64_t *)(pVal->value) = (uint64_t)val_u; - break; - case TSDB_DATA_TYPE_FLOAT: - if (!IS_VALID_FLOAT(val_d)) { - return false; - } - pVal->value = calloc(length, 1); - *(float *)(pVal->value) = (float)val_d; - break; - case TSDB_DATA_TYPE_DOUBLE: - if (!IS_VALID_DOUBLE(val_d)) { - return false; - } - pVal->value = calloc(length, 1); - *(double *)(pVal->value) = (double)val_d; - break; - default: - return false; - } - return true; -} -//len does not include '\0' from value. -bool convertSmlValueType(TAOS_SML_KV *pVal, char *value, - uint16_t len, SSmlLinesInfo* info, bool isTag) { - if (len <= 0) { - return false; - } - - //convert tags value to Nchar - if (isTag) { - pVal->type = TSDB_DATA_TYPE_NCHAR; - pVal->length = len; - pVal->value = calloc(pVal->length, 1); - memcpy(pVal->value, value, pVal->length); - return true; - } - - //integer number - bool has_sign; - if (isInteger(value, len, &has_sign)) { - pVal->type = has_sign ? TSDB_DATA_TYPE_BIGINT : TSDB_DATA_TYPE_UBIGINT; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - value[len - 1] = '\0'; - if (!isValidInteger(value) || !convertStrToNumber(pVal, value, info)) { - return false; - } - return true; - } - if (isTinyInt(value, len)) { - pVal->type = TSDB_DATA_TYPE_TINYINT; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - value[len - 2] = '\0'; - if (!isValidInteger(value) || !convertStrToNumber(pVal, value, info)) { - return false; - } - return true; - } - if (isTinyUint(value, len)) { - pVal->type = TSDB_DATA_TYPE_UTINYINT; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - value[len - 2] = '\0'; - if (!isValidInteger(value) || !convertStrToNumber(pVal, value, info)) { - return false; - } - return true; - } - if (isSmallInt(value, len)) { - pVal->type = TSDB_DATA_TYPE_SMALLINT; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - value[len - 3] = '\0'; - if (!isValidInteger(value) || !convertStrToNumber(pVal, value, info)) { - return false; - } - return true; - } - if (isSmallUint(value, len)) { - pVal->type = TSDB_DATA_TYPE_USMALLINT; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - value[len - 3] = '\0'; - if (!isValidInteger(value) || !convertStrToNumber(pVal, value, info)) { - return false; - } - return true; - } - if (isInt(value, len)) { - pVal->type = TSDB_DATA_TYPE_INT; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - value[len - 3] = '\0'; - if (!isValidInteger(value) || !convertStrToNumber(pVal, value, info)) { - return false; - } - return true; - } - if (isUint(value, len)) { - pVal->type = TSDB_DATA_TYPE_UINT; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - value[len - 3] = '\0'; - if (!isValidInteger(value) || !convertStrToNumber(pVal, value, info)) { - return false; - } - return true; - } - if (isBigInt(value, len)) { - pVal->type = TSDB_DATA_TYPE_BIGINT; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - value[len - 3] = '\0'; - if (!isValidInteger(value) || !convertStrToNumber(pVal, value, info)) { - return false; - } - return true; - } - if (isBigUint(value, len)) { - pVal->type = TSDB_DATA_TYPE_UBIGINT; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - value[len - 3] = '\0'; - if (!isValidInteger(value) || !convertStrToNumber(pVal, value, info)) { - return false; - } - return true; - } - //floating number - if (isFloat(value, len)) { +static bool convertSmlValue(SSmlKv *pVal, SMsgBuf *msg) { + // put high probability matching type first + bool isValid = false; + if (parseFloat(pVal, &isValid, msg)) { + if(!isValid) return false; pVal->type = TSDB_DATA_TYPE_FLOAT; pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - value[len - 3] = '\0'; - if (!isValidFloat(value) || !convertStrToNumber(pVal, value, info)) { - return false; - } - return true; - } - if (isDouble(value, len)) { - pVal->type = TSDB_DATA_TYPE_DOUBLE; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - value[len - 3] = '\0'; - if (!isValidFloat(value) || !convertStrToNumber(pVal, value, info)) { - return false; - } return true; } //binary - if (isBinary(value, len)) { + if (isBinary(pVal->value, pVal->valueLen)) { pVal->type = TSDB_DATA_TYPE_BINARY; - pVal->length = len - 2; - pVal->value = calloc(pVal->length, 1); - //copy after " - memcpy(pVal->value, value + 1, pVal->length); + pVal->length = pVal->valueLen - 2; + pVal->valueLen -= 2; + pVal->value = pVal->value++; return true; } //nchar - if (isNchar(value, len)) { + if (isNchar(pVal->value, pVal->valueLen)) { pVal->type = TSDB_DATA_TYPE_NCHAR; - pVal->length = len - 3; - pVal->value = calloc(pVal->length, 1); - //copy after L" - memcpy(pVal->value, value + 2, pVal->length); + pVal->length = pVal->valueLen - 3; + pVal->value = pVal->value+2; + return true; + } + if (parseDouble(pVal, &isValid, msg)) { + if(!isValid) return false; + pVal->type = TSDB_DATA_TYPE_DOUBLE; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + return true; } //bool - bool bVal; - if (isBool(value, len, &bVal)) { + if (parseBool(pVal)) { pVal->type = TSDB_DATA_TYPE_BOOL; pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - pVal->value = calloc(pVal->length, 1); - memcpy(pVal->value, &bVal, pVal->length); return true; } - //Handle default(no appendix) type as DOUBLE - if (isValidInteger(value) || isValidFloat(value)) { - pVal->type = TSDB_DATA_TYPE_DOUBLE; + if (parseTinyInt(pVal, &isValid, msg)) { + if(!isValid) return false; + pVal->type = TSDB_DATA_TYPE_TINYINT; pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - if (!convertStrToNumber(pVal, value, info)) { - return false; - } return true; } + if (parseTinyUint(pVal, &isValid, msg)) { + if(!isValid) return false; + pVal->type = TSDB_DATA_TYPE_UTINYINT; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + return true; + } + if (parseSmallInt(pVal, &isValid, msg)) { + if(!isValid) return false; + pVal->type = TSDB_DATA_TYPE_SMALLINT; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + return true; + } + if (parseSmallUint(pVal, &isValid, msg)) { + if(!isValid) return false; + pVal->type = TSDB_DATA_TYPE_USMALLINT; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + return true; + } + if (parseInt(pVal, &isValid, msg)) { + if(!isValid) return false; + pVal->type = TSDB_DATA_TYPE_INT; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + return true; + } + if (parseUint(pVal, &isValid, msg)) { + if(!isValid) return false; + pVal->type = TSDB_DATA_TYPE_UINT; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + return true; + } + if (parseBigInt(pVal, &isValid, msg)) { + if(!isValid) return false; + pVal->type = TSDB_DATA_TYPE_BIGINT; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + return true; + } + if (parseBigUint(pVal, &isValid, msg)) { + if(!isValid) return false; + pVal->type = TSDB_DATA_TYPE_UBIGINT; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + return true; + } + + buildInvalidDataMsg(msg, "invalid data", pVal->value); return false; } - - bool checkDuplicateKey(char *key, SHashObj *pHash, SSmlLinesInfo* info) { char *val = NULL; val = taosHashGet(pHash, key, strlen(key)); @@ -1369,21 +921,6 @@ bool checkDuplicateKey(char *key, SHashObj *pHash, SSmlLinesInfo* info) { return false; } -//Table name can only contain digits(0-9),alphebet(a-z),underscore(_) -int32_t isValidChildTableName(const char *pTbName, int16_t len, SSmlLinesInfo* info) { - if (len > TSDB_TABLE_NAME_LEN - 1) { - uError("SML:0x%"PRIx64" child table name cannot exceeds %d characters", info->id, TSDB_TABLE_NAME_LEN - 1); - return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; - } - const char *cur = pTbName; - for (int i = 0; i < len; ++i) { - if(!isdigit(cur[i]) && !isalpha(cur[i]) && (cur[i] != '_')) { - return TSDB_CODE_TSC_LINE_SYNTAX_ERROR; - } - } - return TSDB_CODE_SUCCESS; -} - int32_t parseSml(const char* sql, TAOS_PARSE_ELEMENTS *elements){ if(!sql) return TSDB_CODE_SML_INVALID_DATA; while (*sql != '\0') { // jump the space at the begining @@ -1428,8 +965,12 @@ int32_t parseSml(const char* sql, TAOS_PARSE_ELEMENTS *elements){ } if(!elements->cols) return TSDB_CODE_SML_INVALID_DATA; + bool isInQuote = false; while (*sql != '\0') { - if(*sql == SPACE && *(sql - 1) != SLASH) { + if(*sql == QUOTE && *(sql - 1) != SLASH){ + isInQuote = !isInQuote; + } + if(!isInQuote && *sql == SPACE && *(sql - 1) != SLASH) { break; } sql++; @@ -1444,12 +985,27 @@ int32_t parseSml(const char* sql, TAOS_PARSE_ELEMENTS *elements){ } sql++; } + if(elements->timestamp){ + elements->timestampLen = sql - elements->timestamp; + } return TSDB_CODE_SUCCESS; } -int32_t parseSmlKV(const char* data, int32_t len, SArray *cols, bool isTag){ +bool parseSmlCols(const char* data, int32_t len, SArray *cols, bool isTag, SMsgBuf *msg){ + if(isTag && len == 0){ + SSmlKv *kv = taosMemoryCalloc(sizeof(SSmlKv), 1); + kv->key = TAG; + kv->keyLen = TAGNAMELEN; + kv->value = TAG; + kv->valueLen = TAGNAMELEN; + kv->type = TSDB_DATA_TYPE_NCHAR; + if(cols) taosArrayPush(cols, &kv); + return true; + } + for(int i = 0; i < len; i++){ + // parse key const char *key = data + i; int32_t keyLen = 0; while(i < len){ @@ -1459,23 +1015,27 @@ int32_t parseSmlKV(const char* data, int32_t len, SArray *cols, bool isTag){ } i++; } - if(keyLen == 0){ + if(keyLen == 0 || keyLen >= TSDB_COL_NAME_LEN){ + buildInvalidDataMsg(msg, "invalid key or key is too long than 64", key); return TSDB_CODE_SML_INVALID_DATA; } + // parse value i++; const char *value = data + i; - int32_t valueLen = 0; while(i < len){ if(data[i] == COMMA && i > 0 && data[i-1] != SLASH){ - valueLen = data + i - value; break; } i++; } + int32_t valueLen = data + i - value; if(valueLen == 0){ + buildInvalidDataMsg(msg, "invalid value", value); return TSDB_CODE_SML_INVALID_DATA; } + + // add kv to SSmlKv SSmlKv *kv = taosMemoryCalloc(sizeof(SSmlKv), 1); kv->key = key; kv->keyLen = keyLen; @@ -1483,9 +1043,15 @@ int32_t parseSmlKV(const char* data, int32_t len, SArray *cols, bool isTag){ kv->valueLen = valueLen; if(isTag){ kv->type = TSDB_DATA_TYPE_NCHAR; + }else{ + if(!convertSmlValue(kv, msg)){ + return TSDB_CODE_SML_INVALID_DATA; + } } + if(cols) taosArrayPush(cols, &kv); } + return TSDB_CODE_SUCCESS; } @@ -1526,13 +1092,13 @@ static int64_t getTimeStampNow(int32_t precision) { } } -static int32_t isValidateTimeStamp(const char *pVal, int32_t len) { +static bool isValidateTimeStamp(const char *pVal, int32_t len) { for (int i = 0; i < len; ++i) { if (!isdigit(pVal[i])) { - return TSDB_CODE_TSC_INVALID_TIME_STAMP; + return false; } } - return TSDB_CODE_SUCCESS; + return true; } static int32_t getTsType(int32_t len) { @@ -1541,47 +1107,53 @@ static int32_t getTsType(int32_t len) { } else if (len == TSDB_TIME_PRECISION_MILLI_DIGITS) { return TSDB_TIME_PRECISION_MILLI_DIGITS; } else { - return TSDB_CODE_TSC_INVALID_TIME_STAMP; + return -1; } } -static int32_t parseSmlTS(const char* data, SArray *tags, int8_t tsType, SMLProtocolType protocolType){ - int64_t *ts = taosMemoryCalloc(1, sizeof(int64_t)); +static int32_t parseSmlTS(const char* data, int32_t len, SArray *tags, SSmlLinesInfo* info){ + int64_t ts = 0; if(data == NULL){ - if(protocolType == TSDB_SML_LINE_PROTOCOL){ - *ts = getTimeStampNow(tsType); - }else{ - goto cleanup; + if(info->protocol != TSDB_SML_LINE_PROTOCOL){ + buildInvalidDataMsg(&info->msgBuf, "timestamp can not be null", NULL); + return TSDB_CODE_TSC_INVALID_TIME_STAMP; } + ts = getTimeStampNow(info->tsType); }else{ - int32_t len = strlen(data); int ret = isValidateTimeStamp(data, len); if(!ret){ - goto cleanup; + buildInvalidDataMsg(&info->msgBuf, "timestamp must be digit", data); + return TSDB_CODE_TSC_INVALID_TIME_STAMP; } - if(protocolType != TSDB_SML_LINE_PROTOCOL){ + int32_t tsType = -1; + if(info->protocol != TSDB_SML_LINE_PROTOCOL){ tsType = getTsType(len); - if (tsType == TSDB_CODE_TSC_INVALID_TIME_STAMP) { - goto cleanup; + if (tsType == -1) { + buildInvalidDataMsg(&info->msgBuf, "timestamp precision can only be seconds(10 digits) or milli seconds(13 digits)", data); + return TSDB_CODE_TSC_INVALID_TIME_STAMP; } + }else{ + tsType = info->tsType; } - *ts = getTimeStampValue(data, tsType); - if(*ts == -1){ - goto cleanup; + ts = getTimeStampValue(data, tsType); + if(ts == -1){ + buildInvalidDataMsg(&info->msgBuf, "invalid timestamp", data); + return TSDB_CODE_TSC_INVALID_TIME_STAMP; } } SSmlKv *kv = taosMemoryCalloc(sizeof(SSmlKv), 1); - kv->value = (const char*)ts; - kv->valueLen = sizeof(int64_t); + if(!kv){ + return TSDB_CODE_OUT_OF_MEMORY; + } + + kv->key = TS; + kv->keyLen = TSNAMELEN; + kv->i = ts; kv->type = TSDB_DATA_TYPE_TIMESTAMP; kv->length = (int16_t)tDataTypes[kv->type].bytes; if(tags) taosArrayPush(tags, &kv); return TSDB_CODE_SUCCESS; - -cleanup: - taosMemoryFree(ts); - return TSDB_CODE_TSC_INVALID_TIME_STAMP; } //int32_t parseSmlCols(const char* data, SArray *cols){ @@ -1627,19 +1199,21 @@ cleanup: // return TSDB_CODE_SUCCESS; //} -void updateMeta(SSmlSTableMeta* tableMeta, SArray *tags, SArray *cols){ +bool updateMeta(SSmlSTableMeta* tableMeta, SArray *tags, SArray *cols, SMsgBuf *msg){ if(tags){ for (int i = 0; i < taosArrayGetSize(tags); ++i) { SSmlKv *kv = taosArrayGetP(tags, i); + ASSERT(kv->type == TSDB_DATA_TYPE_NCHAR); + SSmlKv **value = taosHashGet(tableMeta->tagHash, kv->key, kv->keyLen); if(value){ - if(kv->type != (*value)->type){ - // todo + ASSERT((*value)->type == TSDB_DATA_TYPE_NCHAR); + if(kv->valueLen > (*value)->valueLen){ // tags type is nchar + *value = kv; } }else{ taosHashPut(tableMeta->tagHash, kv->key, kv->keyLen, &kv, POINTER_BYTES); } - } } @@ -1649,7 +1223,14 @@ void updateMeta(SSmlSTableMeta* tableMeta, SArray *tags, SArray *cols){ SSmlKv **value = taosHashGet(tableMeta->fieldHash, kv->key, kv->keyLen); if(value){ if(kv->type != (*value)->type){ - // todo + buildInvalidDataMsg(msg, "the type is not the same like before", kv->key); + return false; + }else{ + if(IS_VAR_DATA_TYPE(kv->type)){ // update string len, if bigger + if(kv->valueLen > (*value)->valueLen){ + *value = kv; + } + } } }else{ taosHashPut(tableMeta->fieldHash, kv->key, kv->keyLen, &kv, POINTER_BYTES); @@ -1687,21 +1268,33 @@ static int32_t smlParseLine(const char* sql, SSmlLinesInfo* info) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } - parseSmlTS(elements.timestamp, cols, info->tsType); - ret = parseSmlCols(elements.cols, elements.colsLen, cols, false); + ret = parseSmlTS(elements.timestamp, elements.timestampLen, cols, info); if(ret != TSDB_CODE_SUCCESS){ return ret; } + ret = parseSmlCols(elements.cols, elements.colsLen, cols, false, &info->msgBuf); + if(ret != TSDB_CODE_SUCCESS){ + return ret; + } + if(taosArrayGetSize(cols) > TSDB_MAX_COLUMNS){ + buildInvalidDataMsg(&info->msgBuf, "too many columns than 4096", NULL); + return TSDB_CODE_SML_INVALID_DATA; + } TAOS_SML_DATA_POINT_TAGS** oneTable = taosHashGet(info->childTables, elements.measure, elements.measureTagsLen); if(oneTable){ SSmlSTableMeta** tableMeta = taosHashGet(info->superTables, elements.measure, elements.measureLen); ASSERT(tableMeta); - updateMeta(*tableMeta, NULL, cols); // update meta - + ret = updateMeta(*tableMeta, NULL, cols, &info->msgBuf); // update meta + if(!ret){ + return TSDB_CODE_SML_INVALID_DATA; + } taosArrayPush((*oneTable)->cols, &cols); }else{ TAOS_SML_DATA_POINT_TAGS *tag = taosMemoryCalloc(sizeof(TAOS_SML_DATA_POINT_TAGS), 1); + if(!tag){ + return TSDB_CODE_OUT_OF_MEMORY; + } tag->cols = taosArrayInit(16, POINTER_BYTES); if (tag->cols == NULL) { uError("SML:0x%"PRIx64" taos_insert_lines failed to allocate memory", info->id); @@ -1709,19 +1302,37 @@ static int32_t smlParseLine(const char* sql, SSmlLinesInfo* info) { } taosArrayPush(tag->cols, &cols); + tag->colsColumn = taosArrayInit(16, POINTER_BYTES); + if (tag->cols == NULL) { + uError("SML:0x%"PRIx64" taos_insert_lines failed to allocate memory", info->id); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + tag->tags = taosArrayInit(16, POINTER_BYTES); if (tag->tags == NULL) { uError("SML:0x%"PRIx64" taos_insert_lines failed to allocate memory", info->id); return TSDB_CODE_TSC_OUT_OF_MEMORY; } - ret = parseSmlTags(elements.tags, elements.tagsLen, tag->tags); + ret = parseSmlCols(elements.tags, elements.tagsLen, tag->tags, true, &info->msgBuf); if(ret != TSDB_CODE_SUCCESS){ return ret; } + if(taosArrayGetSize(tag->tags) > TSDB_MAX_TAGS){ + buildInvalidDataMsg(&info->msgBuf, "too many tags than 128", NULL); + return TSDB_CODE_SML_INVALID_DATA; + } + + tag->sTableName = elements.measure; + tag->sTableNameLen = elements.measureLen; + buildSmlChildTableName(tag); + SSmlSTableMeta** tableMeta = taosHashGet(info->superTables, elements.measure, elements.measureLen); if(tableMeta){ // update meta - updateMeta(*tableMeta, tag->tags, cols); + ret = updateMeta(*tableMeta, tag->tags, cols, &info->msgBuf); + if(!ret){ + return TSDB_CODE_SML_INVALID_DATA; + } }else{ SSmlSTableMeta* meta = taosMemoryCalloc(sizeof(SSmlSTableMeta), 1); insertMeta(meta, tag->tags, cols); @@ -1771,9 +1382,8 @@ static SSmlLinesInfo* smlBuildInfo(TAOS* taos, SRequestObj* request, SMLProtocol goto cleanup; } info->pRequest = request; - info->msgBuf = info->pRequest->msgBuf; - info->msgLen = ERROR_MSG_BUF_DEFAULT_SIZE; - + info->msgBuf.buf = info->pRequest->msgBuf; + info->msgBuf.len = ERROR_MSG_BUF_DEFAULT_SIZE; info->childTables = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, false); info->superTables = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, false); @@ -1868,7 +1478,7 @@ TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int pr SRequestObj* request = createRequest(taos, NULL, NULL, TSDB_SQL_INSERT); switch (protocol) { - case TSDB_SML_LINE_PROTOCOL: + case TSDB_SML_LINE_PROTOCOL:{ int32_t tsType = convertPrecisionType(precision); if(tsType == -1){ request->code = TSDB_CODE_SML_INVALID_PRECISION_TYPE; @@ -1877,6 +1487,7 @@ TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int pr code = sml_insert_lines(taos, request, lines, numLines, protocol, tsType); break; + } case TSDB_SML_TELNET_PROTOCOL: //code = taos_insert_telnet_lines(taos, lines, numLines, protocol, tsType, &affected_rows); break; diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index 0e87d8611e..54e05f9264 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -1539,7 +1539,7 @@ static int32_t smlBoundColumns(SArray *cols, SParsedDataColInfo* pColList, SSche col_id_t lastColIdx = -1; // last column found for (int i = 0; i < taosArrayGetSize(cols); ++i) { SSmlKv *kv = taosArrayGetP(cols, i); - SToken sToken = {.n=kv->keyLen, .z=kv->key}; + SToken sToken = {.n=kv->keyLen, .z=(char*)kv->key}; col_id_t t = lastColIdx + 1; col_id_t index = findCol(&sToken, t, nCols, pSchema); if (index < 0 && t > 0) { @@ -1596,18 +1596,17 @@ static int32_t smlBoundColumns(SArray *cols, SParsedDataColInfo* pColList, SSche return TSDB_CODE_SUCCESS; } -static int32_t smlParseTags(SArray *cols, SKVRowBuilder *tagsBuilder, SParsedDataColInfo* tags, SSchema* pSchema, SVCreateTbReq *createTblReq) { +static int32_t smlParseTags(SArray *cols, SKVRowBuilder *tagsBuilder, SParsedDataColInfo* tags, SSchema* pSchema, SVCreateTbReq *createTblReq, SMsgBuf *msg) { if (tdInitKVRowBuilder(tagsBuilder) < 0) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } SKvParam param = {.builder = tagsBuilder}; for (int i = 0; i < tags->numOfBound; ++i) { - SSchema* pTagSchema = &pSchema[tags->boundColumns[i] - 1]; // colId starts with 1 param.schema = pTagSchema; SSmlKv *kv = taosArrayGetP(cols, i); - KvRowAppend(NULL, kv->value, kv->valueLen, ¶m) ; + KvRowAppend(msg, kv->value, kv->valueLen, ¶m) ; } @@ -1630,18 +1629,33 @@ int32_t smlBind(void *handle, SArray *tags, SArray *cols, STableMeta *pTableMeta SmlExecHandle *smlHandle = (SmlExecHandle *)handle; SSchema* pTagsSchema = getTableTagSchema(pTableMeta); - smlBoundColumns(tags, &smlHandle->tags, pTagsSchema); - smlParseTags(tags, &smlHandle->tagsBuilder, &smlHandle->tags, pTagsSchema, &smlHandle->createTblReq); + setBoundColumnInfo(&smlHandle->tags, pTagsSchema, getNumOfTags(pTableMeta)); + int ret = smlBoundColumns(tags, &smlHandle->tags, pTagsSchema); + if(ret != TSDB_CODE_SUCCESS){ + buildInvalidOperationMsg(&pBuf, "bound tags error"); + return ret; + } + ret = smlParseTags(tags, &smlHandle->tagsBuilder, &smlHandle->tags, pTagsSchema, &smlHandle->createTblReq, &pBuf); + if(ret != TSDB_CODE_SUCCESS){ + return ret; + } STableDataBlocks* pDataBlock = NULL; - getDataBlockFromList(smlHandle->pBlockHash, pTableMeta->uid, TSDB_DEFAULT_PAYLOAD_SIZE, + ret = getDataBlockFromList(smlHandle->pBlockHash, pTableMeta->uid, TSDB_DEFAULT_PAYLOAD_SIZE, sizeof(SSubmitBlk), getTableInfo(pTableMeta).rowSize, pTableMeta, &pDataBlock, NULL, &smlHandle->createTblReq); + if(ret != TSDB_CODE_SUCCESS){ + buildInvalidOperationMsg(&pBuf, "create data block error"); + return ret; + } SSchema* pSchema = getTableColumnSchema(pTableMeta); - smlBoundColumns(taosArrayGetP(cols, 0), &pDataBlock->boundColumnInfo, pSchema); - + ret = smlBoundColumns(taosArrayGetP(cols, 0), &pDataBlock->boundColumnInfo, pSchema); + if(ret != TSDB_CODE_SUCCESS){ + buildInvalidOperationMsg(&pBuf, "bound cols error"); + return ret; + } int32_t extendedRowSize = getExtendedRowSize(pDataBlock); SParsedDataColInfo* spd = &pDataBlock->boundColumnInfo; SRowBuilder* pBuilder = &pDataBlock->rowBuilder; @@ -1649,8 +1663,11 @@ int32_t smlBind(void *handle, SArray *tags, SArray *cols, STableMeta *pTableMeta initRowBuilder(&pDataBlock->rowBuilder, pDataBlock->pTableMeta->sversion, &pDataBlock->boundColumnInfo); - allocateMemForSize(pDataBlock, extendedRowSize * rowNum); - + ret = allocateMemForSize(pDataBlock, extendedRowSize * rowNum); + if(ret != TSDB_CODE_SUCCESS){ + buildInvalidOperationMsg(&pBuf, "allocate memory error"); + return ret; + } for (int32_t r = 0; r < rowNum; ++r) { STSRow* row = (STSRow*)(pDataBlock->pData + pDataBlock->size); // skip the SSubmitBlk header tdSRowResetBuf(pBuilder, row); From ec97d053000bb633b882d4b6861ce122369b02a6 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 29 Apr 2022 17:39:33 +0800 Subject: [PATCH 07/83] refactor:add schemaless function --- include/client/taos.h | 2 +- include/libs/parser/parser.h | 8 +- source/client/inc/clientSml.h | 116 ---- source/client/src/clientSml.c | 1007 +++++++++++++++++----------- source/libs/parser/src/parInsert.c | 58 +- 5 files changed, 657 insertions(+), 534 deletions(-) delete mode 100644 source/client/inc/clientSml.h diff --git a/include/client/taos.h b/include/client/taos.h index 55deee4fad..fa27eb2459 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -199,7 +199,7 @@ DLL_EXPORT void taos_close_stream(TAOS_STREAM *tstr); #endif DLL_EXPORT int taos_load_table_info(TAOS *taos, const char *tableNameList); -DLL_EXPORT TAOS_RES *taos_schemaless_insert(TAOS *taos, char *lines[], int numLines, int protocol, int precision); +DLL_EXPORT TAOS_RES *taos_schemaless_insert(TAOS *taos, char *lines[], int numLines, int protocol, int precision, bool dataFormat); /* --------------------------TMQ INTERFACE------------------------------- */ diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index 248a6c1237..f608e38226 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -93,12 +93,12 @@ int32_t qBindStmtSingleColValue(void *pBlock, TAOS_BIND_v2 *bind, char *msgBuf, int32_t qBuildStmtColFields(void *pDataBlock, int32_t *fieldNum, TAOS_FIELD** fields); int32_t qBuildStmtTagFields(void *pBlock, void *boundTags, int32_t *fieldNum, TAOS_FIELD** fields); int32_t qBindStmtTagsValue(void *pBlock, void *boundTags, int64_t suid, char *tName, TAOS_BIND_v2 *bind, char *msgBuf, int32_t msgBufLen); -void destroyBoundColumnInfo(void* pBoundInfo); +void destroyBoundColumnInfo(void* pBoundInfo); int32_t qCreateSName(SName* pName, const char* pTableName, int32_t acctId, char* dbName, char *msgBuf, int32_t msgBufLen); -void* tscSmlInitHandle(SQuery *pQuery); -void tscSmlDestroyHandle(void *pHandle); -int32_t smlBind(void *handle, SArray *tags, SArray *cols, STableMeta *pTableMeta, char *msgBuf, int16_t msgBufLen); +void* smlInitHandle(SQuery *pQuery); +void smlDestroyHandle(void *pHandle); +int32_t smlBindData(void *handle, SArray *tags, SArray *cols, STableMeta *pTableMeta, char *msgBuf, int16_t msgBufLen); int32_t smlBuildOutput(void* handle, SHashObj* pVgHash); #ifdef __cplusplus diff --git a/source/client/inc/clientSml.h b/source/client/inc/clientSml.h deleted file mode 100644 index c970f1e954..0000000000 --- a/source/client/inc/clientSml.h +++ /dev/null @@ -1,116 +0,0 @@ -/* - * Copyright (c) 2021 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_CLIENTSML_H -#define TDENGINE_CLIENTSML_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include "thash.h" -#include "clientInt.h" -#include "catalog.h" - -typedef TSDB_SML_PROTOCOL_TYPE SMLProtocolType; - -typedef struct { - const char* measure; - const char* tags; - const char* cols; - const char* timestamp; - - int32_t measureLen; - int32_t measureTagsLen; - int32_t tagsLen; - int32_t colsLen; - int32_t timestampLen; -} TAOS_PARSE_ELEMENTS; - -typedef struct { - const char *sTableName; // super table name - uint8_t sTableNameLen; - char childTableName[TSDB_TABLE_NAME_LEN]; - uint64_t uid; - - SArray *tags; - SArray *cols; // elements are SHashObj for find by key quickly - - SArray colsColumn; // elements are cols key string -} TAOS_SML_DATA_POINT_TAGS; - -typedef struct SSmlSTableMeta { -// char *sTableName; // super table name -// uint8_t sTableNameLen; - uint8_t precision; // the number of precision - SHashObj *tagHash; - SHashObj *fieldHash; -} SSmlSTableMeta; - -typedef struct SMsgBuf { - int32_t len; - char *buf; -} SMsgBuf; - -typedef struct { - uint64_t id; - - SMLProtocolType protocol; - int32_t tsType; - - SHashObj *childTables; - SHashObj *superTables; - - SHashObj *metaHashObj; - SHashObj *pVgHash; - - void *exec; - - STscObj *taos; - SCatalog *pCatalog; - SRequestObj *pRequest; - SQuery *pQuery; - - int32_t affectedRows; - SMsgBuf msgBuf; -} SSmlLinesInfo; - -int smlInsert(TAOS* taos, SSmlLinesInfo* info); - -bool checkDuplicateKey(char *key, SHashObj *pHash, SSmlLinesInfo* info); -bool isValidInteger(char *str); -bool isValidFloat(char *str); - -int32_t isValidChildTableName(const char *pTbName, int16_t len, SSmlLinesInfo* info); - -bool convertSmlValueType(SSmlKv *pVal, char *value, - uint16_t len, SSmlLinesInfo* info, bool isTag); -int32_t convertSmlTimeStamp(SSmlKv *pVal, char *value, - uint16_t len, SSmlLinesInfo* info); - - -int sml_insert_lines(TAOS* taos, SRequestObj* request, char* lines[], int numLines, SMLProtocolType protocol, - SMLTimeStampType tsType); -int sml_insert_telnet_lines(TAOS* taos, char* lines[], int numLines, SMLProtocolType protocol, - SMLTimeStampType tsType, int* affectedRows); -int sml_insert_json_payload(TAOS* taos, char* payload, SMLProtocolType protocol, - SMLTimeStampType tsType, int* affectedRows); - - -#ifdef __cplusplus -} -#endif - -#endif // TDENGINE_CLIENTSML_H diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 3739c83109..bde9aeed49 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -1,29 +1,21 @@ +#include #include #include #include #include -#include "clientSml.h" - -#include "tdef.h" -#include "ttypes.h" -#include "tmsg.h" -#include "tlog.h" #include "query.h" -#include "taoserror.h" #include "taos.h" -#include "ttime.h" +#include "taoserror.h" +#include "tdef.h" +#include "tlog.h" +#include "tmsg.h" #include "tstrbuild.h" - - -typedef struct { - char sTableName[TSDB_TABLE_NAME_LEN]; - SHashObj* tagHash; - SHashObj* fieldHash; - SArray* tags; //SArray - SArray* fields; //SArray - uint8_t precision; -} SSmlSTableSchema; +#include "ttime.h" +#include "ttypes.h" +#include "tcommon.h" +#include "catalog.h" +//================================================================================================= #define SPACE ' ' #define COMMA ',' @@ -32,43 +24,8 @@ typedef struct { #define SLASH '\\' #define tsMaxSQLStringLen (1024*1024) -#define TSNAMELEN 2 -#define TAGNAMELEN 3 //================================================================================================= - -static uint64_t linesSmlHandleId = 0; -static const char* TS = "ts"; -static const char* TAG = "tag"; - - -uint64_t genLinesSmlId() { - uint64_t id; - - do { - id = atomic_add_fetch_64(&linesSmlHandleId, 1); - } while (id == 0); - - return id; -} - -static int32_t buildInvalidDataMsg(SMsgBuf* pBuf, const char *msg1, const char *msg2) { - if(msg1) snprintf(pBuf->buf, pBuf->len, "%s:", msg1); - if(msg2) strncpy(pBuf->buf, msg2, pBuf->len); - return TSDB_CODE_SML_INVALID_DATA; -} - -int compareSmlColKv(const void* p1, const void* p2) { - SSmlKv* kv1 = (SSmlKv *)p1; - SSmlKv* kv2 = (SSmlKv*)p2; - int kvLen1 = (int)strlen(kv1->key); - int kvLen2 = (int)strlen(kv2->key); - int res = strncasecmp(kv1->key, kv2->key, MIN(kvLen1, kvLen2)); - if (res != 0) { - return res; - } else { - return kvLen1-kvLen2; - } -} +typedef TSDB_SML_PROTOCOL_TYPE SMLProtocolType; typedef enum { SCHEMA_ACTION_CREATE_STABLE, @@ -97,10 +54,107 @@ typedef struct { }; } SSchemaAction; -static int32_t buildSmlChildTableName(TAOS_SML_DATA_POINT_TAGS *tags) { +typedef struct { + const char* measure; + const char* tags; + const char* cols; + const char* timestamp; + + int32_t measureLen; + int32_t measureTagsLen; + int32_t tagsLen; + int32_t colsLen; + int32_t timestampLen; +} SSmlLineInfo; + +typedef struct { + const char *sTableName; // super table name + uint8_t sTableNameLen; + char childTableName[TSDB_TABLE_NAME_LEN]; + uint64_t uid; + + SArray *tags; + + // colsFormat store cols formated, for quick parse, if info->formatData is true + SArray *colsFormat; // elements are SArray + + // cols & colsColumn store cols un formated + SArray *cols; // elements are SHashObj for find by key quickly + SHashObj *columnsHash; // elements are , just for judge if key exists quickly. +} SSmlTableInfo; + +typedef struct { + SHashObj *tagHash; + SHashObj *fieldHash; + STableMeta *tableMeta; +} SSmlSTableMeta; + +typedef struct { + int32_t len; + char *buf; +} SSmlMsgBuf; + +typedef struct { + uint64_t id; + + SMLProtocolType protocol; + int8_t precision; + bool dataFormat; // true means that the name, number and order of keys in each line are the same + + SHashObj *childTables; + SHashObj *superTables; + SHashObj *pVgHash; + void *exec; + + STscObj *taos; + SCatalog *pCatalog; + SRequestObj *pRequest; + SQuery *pQuery; + + int32_t affectedRows; + SSmlMsgBuf msgBuf; +} SSmlHandle; +//================================================================================================= + +static uint64_t linesSmlHandleId = 0; +static const char* TS = "ts"; +static const char* TAG = "tagNone"; + +//================================================================================================= + +static uint64_t smlGenId() { + uint64_t id; + + do { + id = atomic_add_fetch_64(&linesSmlHandleId, 1); + } while (id == 0); + + return id; +} + +static int32_t smlBuildInvalidDataMsg(SSmlMsgBuf* pBuf, const char *msg1, const char *msg2) { + if(msg1) snprintf(pBuf->buf, pBuf->len, "%s:", msg1); + if(msg2) strncpy(pBuf->buf, msg2, pBuf->len); + return TSDB_CODE_SML_INVALID_DATA; +} + +static int smlCompareKv(const void* p1, const void* p2) { + SSmlKv* kv1 = (SSmlKv *)p1; + SSmlKv* kv2 = (SSmlKv*)p2; + int kvLen1 = (int)strlen(kv1->key); + int kvLen2 = (int)strlen(kv2->key); + int res = strncasecmp(kv1->key, kv2->key, MIN(kvLen1, kvLen2)); + if (res != 0) { + return res; + } else { + return kvLen1-kvLen2; + } +} + +static void smlBuildChildTableName(SSmlTableInfo *tags) { int32_t size = taosArrayGetSize(tags->tags); ASSERT(size > 0); - qsort(tags->tags, size, POINTER_BYTES, compareSmlColKv); + qsort(tags->tags, size, POINTER_BYTES, smlCompareKv); SStringBuilder sb = {0}; taosStringBuilderAppendStringLen(&sb, tags->sTableName, tags->sTableNameLen); @@ -120,12 +174,10 @@ static int32_t buildSmlChildTableName(TAOS_SML_DATA_POINT_TAGS *tags) { snprintf(tags->childTableName, TSDB_TABLE_NAME_LEN, "t_%016"PRIx64"%016"PRIx64, digest1, digest2); taosStringBuilderDestroy(&sb); tags->uid = digest1; - uDebug("SML: child table name: %s", tags->childTableName); - return 0; } -static int32_t generateSchemaAction(SSchema* pointColField, SHashObj* dbAttrHash, SArray* dbAttrArray, bool isTag, char sTableName[], - SSchemaAction* action, bool* actionNeeded, SSmlLinesInfo* info) { +static int32_t smlGenerateSchemaAction(SSchema* pointColField, SHashObj* dbAttrHash, SArray* dbAttrArray, bool isTag, char sTableName[], + SSchemaAction* action, bool* actionNeeded, SSmlHandle* info) { // char fieldName[TSDB_COL_NAME_LEN] = {0}; // strcpy(fieldName, pointColField->name); // @@ -168,7 +220,7 @@ static int32_t generateSchemaAction(SSchema* pointColField, SHashObj* dbAttrHash return 0; } -static int32_t buildColumnDescription(SSmlKv* field, char* buf, int32_t bufSize, int32_t* outBytes) { +static int32_t smlBuildColumnDescription(SSmlKv* field, char* buf, int32_t bufSize, int32_t* outBytes) { uint8_t type = field->type; char tname[TSDB_TABLE_NAME_LEN] = {0}; memcpy(tname, field->key, field->keyLen); @@ -185,7 +237,7 @@ static int32_t buildColumnDescription(SSmlKv* field, char* buf, int32_t bufSize, return 0; } -static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInfo* info) { +static int32_t smlApplySchemaAction(SSmlHandle* info, SSchemaAction* action) { int32_t code = 0; int32_t outBytes = 0; char *result = (char *)taosMemoryCalloc(1, tsMaxSQLStringLen+1); @@ -195,8 +247,8 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInf switch (action->action) { case SCHEMA_ACTION_ADD_COLUMN: { int n = sprintf(result, "alter stable %s add column ", action->alterSTable.sTableName); - buildColumnDescription(action->alterSTable.field, result+n, capacity-n, &outBytes); - TAOS_RES* res = taos_query(taos, result); //TODO async doAsyncQuery + smlBuildColumnDescription(action->alterSTable.field, result+n, capacity-n, &outBytes); + TAOS_RES* res = taos_query(info->taos, result); //TODO async doAsyncQuery code = taos_errno(res); const char* errStr = taos_errstr(res); char* begin = strstr(errStr, "duplicated column names"); @@ -208,7 +260,7 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInf // if (code == TSDB_CODE_MND_FIELD_ALREADY_EXIST || code == TSDB_CODE_MND_TAG_ALREADY_EXIST || tscDupColNames) { if (code == TSDB_CODE_MND_TAG_ALREADY_EXIST || tscDupColNames) { - TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); + TAOS_RES* res2 = taos_query(info->taos, "RESET QUERY CACHE"); code = taos_errno(res2); if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%" PRIx64 " apply schema action. reset query cache. error: %s", info->id, taos_errstr(res2)); @@ -220,9 +272,9 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInf } case SCHEMA_ACTION_ADD_TAG: { int n = sprintf(result, "alter stable %s add tag ", action->alterSTable.sTableName); - buildColumnDescription(action->alterSTable.field, + smlBuildColumnDescription(action->alterSTable.field, result+n, capacity-n, &outBytes); - TAOS_RES* res = taos_query(taos, result); //TODO async doAsyncQuery + TAOS_RES* res = taos_query(info->taos, result); //TODO async doAsyncQuery code = taos_errno(res); const char* errStr = taos_errstr(res); char* begin = strstr(errStr, "duplicated column names"); @@ -234,7 +286,7 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInf // if (code ==TSDB_CODE_MND_TAG_ALREADY_EXIST || code == TSDB_CODE_MND_FIELD_ALREAY_EXIST || tscDupColNames) { if (code ==TSDB_CODE_MND_TAG_ALREADY_EXIST || tscDupColNames) { - TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); + TAOS_RES* res2 = taos_query(info->taos, "RESET QUERY CACHE"); code = taos_errno(res2); if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%" PRIx64 " apply schema action. reset query cache. error: %s", info->id, taos_errstr(res2)); @@ -246,9 +298,9 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInf } case SCHEMA_ACTION_CHANGE_COLUMN_SIZE: { int n = sprintf(result, "alter stable %s modify column ", action->alterSTable.sTableName); - buildColumnDescription(action->alterSTable.field, result+n, + smlBuildColumnDescription(action->alterSTable.field, result+n, capacity-n, &outBytes); - TAOS_RES* res = taos_query(taos, result); //TODO async doAsyncQuery + TAOS_RES* res = taos_query(info->taos, result); //TODO async doAsyncQuery code = taos_errno(res); if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%"PRIx64" apply schema action. error : %s", info->id, taos_errstr(res)); @@ -257,7 +309,7 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInf // if (code == TSDB_CODE_MND_INVALID_COLUMN_LENGTH || code == TSDB_CODE_TSC_INVALID_COLUMN_LENGTH) { if (code == TSDB_CODE_TSC_INVALID_COLUMN_LENGTH) { - TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); + TAOS_RES* res2 = taos_query(info->taos, "RESET QUERY CACHE"); code = taos_errno(res2); if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%" PRIx64 " apply schema action. reset query cache. error: %s", info->id, taos_errstr(res2)); @@ -269,9 +321,9 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInf } case SCHEMA_ACTION_CHANGE_TAG_SIZE: { int n = sprintf(result, "alter stable %s modify tag ", action->alterSTable.sTableName); - buildColumnDescription(action->alterSTable.field, result+n, + smlBuildColumnDescription(action->alterSTable.field, result+n, capacity-n, &outBytes); - TAOS_RES* res = taos_query(taos, result); //TODO async doAsyncQuery + TAOS_RES* res = taos_query(info->taos, result); //TODO async doAsyncQuery code = taos_errno(res); if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%"PRIx64" apply schema action. error : %s", info->id, taos_errstr(res)); @@ -280,7 +332,7 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInf // if (code == TSDB_CODE_MND_INVALID_TAG_LENGTH || code == TSDB_CODE_TSC_INVALID_TAG_LENGTH) { if (code == TSDB_CODE_TSC_INVALID_TAG_LENGTH) { - TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); + TAOS_RES* res2 = taos_query(info->taos, "RESET QUERY CACHE"); code = taos_errno(res2); if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%" PRIx64 " apply schema action. reset query cache. error: %s", info->id, taos_errstr(res2)); @@ -296,7 +348,7 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInf SSmlKv **kv = taosHashIterate(action->createSTable.fields, NULL); while(kv){ - buildColumnDescription(*kv, pos, freeBytes, &outBytes); + smlBuildColumnDescription(*kv, pos, freeBytes, &outBytes); pos += outBytes; freeBytes -= outBytes; *pos = ','; ++pos; --freeBytes; kv = taosHashIterate(action->createSTable.fields, kv); @@ -308,14 +360,14 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInf kv = taosHashIterate(action->createSTable.tags, NULL); while(kv){ - buildColumnDescription(*kv, pos, freeBytes, &outBytes); + smlBuildColumnDescription(*kv, pos, freeBytes, &outBytes); pos += outBytes; freeBytes -= outBytes; *pos = ','; ++pos; --freeBytes; kv = taosHashIterate(action->createSTable.tags, kv); } pos--; ++freeBytes; outBytes = snprintf(pos, freeBytes, ")"); - TAOS_RES* res = taos_query(taos, result); + TAOS_RES* res = taos_query(info->taos, result); code = taos_errno(res); if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%"PRIx64" apply schema action. error : %s", info->id, taos_errstr(res)); @@ -323,7 +375,7 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInf taos_free_result(res); if (code == TSDB_CODE_MND_STB_ALREADY_EXIST) { - TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); + TAOS_RES* res2 = taos_query(info->taos, "RESET QUERY CACHE"); code = taos_errno(res2); if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%" PRIx64 " apply schema action. reset query cache. error: %s", info->id, taos_errstr(res2)); @@ -338,14 +390,14 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInf break; } - taosMemoryFree(result); + taosMemoryFreeClear(result); if (code != 0) { uError("SML:0x%"PRIx64 " apply schema action failure. %s", info->id, tstrerror(code)); } return code; } -static int32_t modifyDBSchemas(TAOS* taos, SSmlLinesInfo* info) { +static int32_t smlModifyDBSchemas(SSmlHandle* info) { int32_t code = 0; SSmlSTableMeta** tableMetaSml = taosHashIterate(info->superTables, NULL); @@ -356,7 +408,7 @@ static int32_t modifyDBSchemas(TAOS* taos, SSmlLinesInfo* info) { SEpSet ep = getEpSet_s(&info->taos->pAppInfo->mgmtEp); size_t superTableLen = 0; - void *superTable = taosHashGetKey(tableMetaSml, &superTableLen); + void *superTable = taosHashGetKey(tableMetaSml, &superTableLen); // todo escape SName pName = {TSDB_TABLE_NAME_T, info->taos->acctId, {0}, {0}}; strcpy(pName.dbname, info->pRequest->pDb); memcpy(pName.tname, superTable, superTableLen); @@ -369,10 +421,15 @@ static int32_t modifyDBSchemas(TAOS* taos, SSmlLinesInfo* info) { memcpy(schemaAction.createSTable.sTableName, superTable, superTableLen); schemaAction.createSTable.tags = cTablePoints->tagHash; schemaAction.createSTable.fields = cTablePoints->fieldHash; - applySchemaAction(taos, &schemaAction, info); + code = smlApplySchemaAction(info, &schemaAction); + if (code != 0) { + uError("SML:0x%"PRIx64" smlApplySchemaAction failed. can not create %s", info->id, schemaAction.createSTable.sTableName); + return code; + } + code = catalogGetSTableMeta(info->pCatalog, info->taos->pAppInfo->pTransporter, &ep, &pName, &pTableMeta); if (code != 0) { - uError("SML:0x%"PRIx64" reconcile point schema failed. can not create %s", info->id, schemaAction.createSTable.sTableName); + uError("SML:0x%"PRIx64" catalogGetSTableMeta failed. super table name %s", info->id, schemaAction.createSTable.sTableName); return code; } }else if (code == TSDB_CODE_SUCCESS) { @@ -380,67 +437,13 @@ static int32_t modifyDBSchemas(TAOS* taos, SSmlLinesInfo* info) { uError("SML:0x%"PRIx64" load table meta error: %s", info->id, tstrerror(code)); return code; } - taosHashPut(info->metaHashObj, superTable, superTableLen, &pTableMeta, POINTER_BYTES); + cTablePoints->tableMeta = pTableMeta; tableMetaSml = taosHashIterate(info->superTables, tableMetaSml); } return 0; } -static int32_t applyDataPoints(SSmlLinesInfo* info) { - int32_t code = TSDB_CODE_SUCCESS; - - TAOS_SML_DATA_POINT_TAGS** oneTable = taosHashIterate(info->childTables, NULL); - while (oneTable) { - TAOS_SML_DATA_POINT_TAGS* tableData = *oneTable; - - SName pName = {TSDB_TABLE_NAME_T, info->taos->acctId, {0}, {0}}; - strcpy(pName.dbname, info->pRequest->pDb); - memcpy(pName.tname, tableData->childTableName, strlen(tableData->childTableName)); - SEpSet ep = getEpSet_s(&info->taos->pAppInfo->mgmtEp); - SVgroupInfo vg; - catalogGetTableHashVgroup(info->pCatalog, info->taos->pAppInfo->pTransporter, &ep, &pName, &vg); - taosHashPut(info->pVgHash, (const char*)&vg.vgId, sizeof(vg.vgId), (char*)&vg, sizeof(vg)); - - STableMeta** pMeta = taosHashGet(info->metaHashObj, tableData->sTableName, tableData->sTableNameLen); - ASSERT (NULL != pMeta && NULL != *pMeta); - (*pMeta)->vgId = vg.vgId; - (*pMeta)->uid = tableData->uid; // one table merge data block together according uid - - code = smlBind(info->exec, tableData->tags, tableData->cols, *pMeta, info->msgBuf.buf, info->msgBuf.len); - if(code != TSDB_CODE_SUCCESS){ - return code; - } - oneTable = taosHashIterate(info->childTables, oneTable); - } - - smlBuildOutput(info->exec, info->pVgHash); - launchQueryImpl(info->pRequest, info->pQuery, TSDB_CODE_SUCCESS, true); - - info->affectedRows = taos_affected_rows(info->pRequest); - return info->pRequest->code; -} - -int smlInsert(TAOS* taos, SSmlLinesInfo* info) { - uDebug("SML:0x%"PRIx64" taos_sml_insert. number of super tables: %d", info->id, taosHashGetSize(info->superTables)); - - uDebug("SML:0x%"PRIx64" modify db schemas", info->id); - int32_t code = modifyDBSchemas(taos, info); - if (code != 0) { - uError("SML:0x%"PRIx64" error change db schema : %s", info->id, tstrerror(code)); - return code; - } - - uDebug("SML:0x%"PRIx64" apply data points", info->id); - code = applyDataPoints(info); - if (code != 0) { - uError("SML:0x%"PRIx64" error apply data points : %s", info->id, tstrerror(code)); - return code; - } - - return TSDB_CODE_SUCCESS; -} - //========================================================================= /* Field Escape charaters @@ -448,50 +451,50 @@ int smlInsert(TAOS* taos, SSmlLinesInfo* info) { 2: tag_key, tag_value, field_key Comma,Equal Sign,Space 3: field_value Double quote,Backslash */ -static void escapeSpecialCharacter(uint8_t field, const char **pos) { - const char *cur = *pos; - if (*cur != '\\') { - return; - } - switch (field) { - case 1: - switch (*(cur + 1)) { - case ',': - case ' ': - cur++; - break; - default: - break; - } - break; - case 2: - switch (*(cur + 1)) { - case ',': - case ' ': - case '=': - cur++; - break; - default: - break; - } - break; - case 3: - switch (*(cur + 1)) { - case '"': - case '\\': - cur++; - break; - default: - break; - } - break; - default: - break; - } - *pos = cur; -} +//static void escapeSpecialCharacter(uint8_t field, const char **pos) { +// const char *cur = *pos; +// if (*cur != '\\') { +// return; +// } +// switch (field) { +// case 1: +// switch (*(cur + 1)) { +// case ',': +// case ' ': +// cur++; +// break; +// default: +// break; +// } +// break; +// case 2: +// switch (*(cur + 1)) { +// case ',': +// case ' ': +// case '=': +// cur++; +// break; +// default: +// break; +// } +// break; +// case 3: +// switch (*(cur + 1)) { +// case '"': +// case '\\': +// cur++; +// break; +// default: +// break; +// } +// break; +// default: +// break; +// } +// *pos = cur; +//} -static bool parseTinyInt(SSmlKv *kvVal, bool *isValid, SMsgBuf *msg) { +static bool smlParseTinyInt(SSmlKv *kvVal, bool *isValid, SSmlMsgBuf *msg) { const char *pVal = kvVal->value; int32_t len = kvVal->valueLen; if (len <= 2) { @@ -503,10 +506,10 @@ static bool parseTinyInt(SSmlKv *kvVal, bool *isValid, SMsgBuf *msg) { int64_t result = strtoll(pVal, &endptr, 10); if(endptr != signalPos){ // 78ri8 *isValid = false; - buildInvalidDataMsg(msg, "invalid tiny int", endptr); + smlBuildInvalidDataMsg(msg, "invalid tiny int", endptr); }else if(!IS_VALID_TINYINT(result)){ *isValid = false; - buildInvalidDataMsg(msg, "tiny int out of range[-128,127]", endptr); + smlBuildInvalidDataMsg(msg, "tiny int out of range[-128,127]", endptr); }else{ kvVal->i = result; *isValid = true; @@ -516,7 +519,7 @@ static bool parseTinyInt(SSmlKv *kvVal, bool *isValid, SMsgBuf *msg) { return false; } -static bool parseTinyUint(SSmlKv *kvVal, bool *isValid, SMsgBuf *msg) { +static bool smlParseTinyUint(SSmlKv *kvVal, bool *isValid, SSmlMsgBuf *msg) { const char *pVal = kvVal->value; int32_t len = kvVal->valueLen; if (len <= 2) { @@ -531,10 +534,10 @@ static bool parseTinyUint(SSmlKv *kvVal, bool *isValid, SMsgBuf *msg) { int64_t result = strtoll(pVal, &endptr, 10); if(endptr != signalPos){ // 78ri8 *isValid = false; - buildInvalidDataMsg(msg, "invalid unsigned tiny int", endptr); + smlBuildInvalidDataMsg(msg, "invalid unsigned tiny int", endptr); }else if(!IS_VALID_UTINYINT(result)){ *isValid = false; - buildInvalidDataMsg(msg, "unsigned tiny int out of range[0,255]", endptr); + smlBuildInvalidDataMsg(msg, "unsigned tiny int out of range[0,255]", endptr); }else{ kvVal->i = result; *isValid = true; @@ -544,7 +547,7 @@ static bool parseTinyUint(SSmlKv *kvVal, bool *isValid, SMsgBuf *msg) { return false; } -static bool parseSmallInt(SSmlKv *kvVal, bool *isValid, SMsgBuf *msg) { +static bool smlParseSmallInt(SSmlKv *kvVal, bool *isValid, SSmlMsgBuf *msg) { const char *pVal = kvVal->value; int32_t len = kvVal->valueLen; if (len <= 3) { @@ -556,10 +559,10 @@ static bool parseSmallInt(SSmlKv *kvVal, bool *isValid, SMsgBuf *msg) { int64_t result = strtoll(pVal, &endptr, 10); if(endptr != signalPos){ // 78ri8 *isValid = false; - buildInvalidDataMsg(msg, "invalid small int", endptr); + smlBuildInvalidDataMsg(msg, "invalid small int", endptr); }else if(!IS_VALID_SMALLINT(result)){ *isValid = false; - buildInvalidDataMsg(msg, "small int our of range[-32768,32767]", endptr); + smlBuildInvalidDataMsg(msg, "small int our of range[-32768,32767]", endptr); }else{ kvVal->i = result; *isValid = true; @@ -569,7 +572,7 @@ static bool parseSmallInt(SSmlKv *kvVal, bool *isValid, SMsgBuf *msg) { return false; } -static bool parseSmallUint(SSmlKv *kvVal, bool *isValid, SMsgBuf *msg) { +static bool smlParseSmallUint(SSmlKv *kvVal, bool *isValid, SSmlMsgBuf *msg) { const char *pVal = kvVal->value; int32_t len = kvVal->valueLen; if (len <= 3) { @@ -584,10 +587,10 @@ static bool parseSmallUint(SSmlKv *kvVal, bool *isValid, SMsgBuf *msg) { int64_t result = strtoll(pVal, &endptr, 10); if(endptr != signalPos){ // 78ri8 *isValid = false; - buildInvalidDataMsg(msg, "invalid unsigned small int", endptr); + smlBuildInvalidDataMsg(msg, "invalid unsigned small int", endptr); }else if(!IS_VALID_USMALLINT(result)){ *isValid = false; - buildInvalidDataMsg(msg, "unsigned small int out of rang[0,65535]", endptr); + smlBuildInvalidDataMsg(msg, "unsigned small int out of rang[0,65535]", endptr); }else{ kvVal->i = result; *isValid = true; @@ -597,7 +600,7 @@ static bool parseSmallUint(SSmlKv *kvVal, bool *isValid, SMsgBuf *msg) { return false; } -static bool parseInt(SSmlKv *kvVal, bool *isValid, SMsgBuf *msg) { +static bool smlParseInt(SSmlKv *kvVal, bool *isValid, SSmlMsgBuf *msg) { const char *pVal = kvVal->value; int32_t len = kvVal->valueLen; if (len <= 3) { @@ -609,10 +612,10 @@ static bool parseInt(SSmlKv *kvVal, bool *isValid, SMsgBuf *msg) { int64_t result = strtoll(pVal, &endptr, 10); if(endptr != signalPos){ // 78ri8 *isValid = false; - buildInvalidDataMsg(msg, "invalid int", endptr); + smlBuildInvalidDataMsg(msg, "invalid int", endptr); }else if(!IS_VALID_INT(result)){ *isValid = false; - buildInvalidDataMsg(msg, "int out of range[-2147483648,2147483647]", endptr); + smlBuildInvalidDataMsg(msg, "int out of range[-2147483648,2147483647]", endptr); }else{ kvVal->i = result; *isValid = true; @@ -622,7 +625,7 @@ static bool parseInt(SSmlKv *kvVal, bool *isValid, SMsgBuf *msg) { return false; } -static bool parseUint(SSmlKv *kvVal, bool *isValid, SMsgBuf *msg) { +static bool smlParseUint(SSmlKv *kvVal, bool *isValid, SSmlMsgBuf *msg) { const char *pVal = kvVal->value; int32_t len = kvVal->valueLen; if (len <= 3) { @@ -637,10 +640,10 @@ static bool parseUint(SSmlKv *kvVal, bool *isValid, SMsgBuf *msg) { int64_t result = strtoll(pVal, &endptr, 10); if(endptr != signalPos){ // 78ri8 *isValid = false; - buildInvalidDataMsg(msg, "invalid unsigned int", endptr); + smlBuildInvalidDataMsg(msg, "invalid unsigned int", endptr); }else if(!IS_VALID_UINT(result)){ *isValid = false; - buildInvalidDataMsg(msg, "unsigned int out of range[0,4294967295]", endptr); + smlBuildInvalidDataMsg(msg, "unsigned int out of range[0,4294967295]", endptr); }else{ kvVal->i = result; *isValid = true; @@ -650,7 +653,7 @@ static bool parseUint(SSmlKv *kvVal, bool *isValid, SMsgBuf *msg) { return false; } -static bool parseBigInt(SSmlKv *kvVal, bool *isValid, SMsgBuf *msg) { +static bool smlParseBigInt(SSmlKv *kvVal, bool *isValid, SSmlMsgBuf *msg) { const char *pVal = kvVal->value; int32_t len = kvVal->valueLen; if (len > 3 && strcasecmp(pVal + len - 3, "i64") == 0) { @@ -670,10 +673,10 @@ static bool parseBigInt(SSmlKv *kvVal, bool *isValid, SMsgBuf *msg) { int64_t result = strtoll(pVal, &endptr, 10); if(endptr != pVal + len - 1){ // 78ri8 *isValid = false; - buildInvalidDataMsg(msg, "invalid big int", endptr); + smlBuildInvalidDataMsg(msg, "invalid big int", endptr); }else if(!IS_VALID_BIGINT(result)){ *isValid = false; - buildInvalidDataMsg(msg, "big int out of range[-9223372036854775808,9223372036854775807]", endptr); + smlBuildInvalidDataMsg(msg, "big int out of range[-9223372036854775808,9223372036854775807]", endptr); }else{ kvVal->i = result; *isValid = true; @@ -683,7 +686,7 @@ static bool parseBigInt(SSmlKv *kvVal, bool *isValid, SMsgBuf *msg) { return false; } -static bool parseBigUint(SSmlKv *kvVal, bool *isValid, SMsgBuf *msg) { +static bool smlParseBigUint(SSmlKv *kvVal, bool *isValid, SSmlMsgBuf *msg) { const char *pVal = kvVal->value; int32_t len = kvVal->valueLen; if (len <= 3) { @@ -698,10 +701,10 @@ static bool parseBigUint(SSmlKv *kvVal, bool *isValid, SMsgBuf *msg) { uint64_t result = strtoull(pVal, &endptr, 10); if(endptr != signalPos){ // 78ri8 *isValid = false; - buildInvalidDataMsg(msg, "invalid unsigned big int", endptr); + smlBuildInvalidDataMsg(msg, "invalid unsigned big int", endptr); }else if(!IS_VALID_UBIGINT(result)){ *isValid = false; - buildInvalidDataMsg(msg, "unsigned big int out of range[0,18446744073709551615]", endptr); + smlBuildInvalidDataMsg(msg, "unsigned big int out of range[0,18446744073709551615]", endptr); }else{ kvVal->u = result; *isValid = true; @@ -711,7 +714,7 @@ static bool parseBigUint(SSmlKv *kvVal, bool *isValid, SMsgBuf *msg) { return false; } -static bool parseFloat(SSmlKv *kvVal, bool *isValid, SMsgBuf *msg) { +static bool smlParseFloat(SSmlKv *kvVal, bool *isValid, SSmlMsgBuf *msg) { const char *pVal = kvVal->value; int32_t len = kvVal->valueLen; char *endptr = NULL; @@ -725,10 +728,10 @@ static bool parseFloat(SSmlKv *kvVal, bool *isValid, SMsgBuf *msg) { if (len > 3 && len f = result; *isValid = true; @@ -738,7 +741,7 @@ static bool parseFloat(SSmlKv *kvVal, bool *isValid, SMsgBuf *msg) { return false; } -static bool parseDouble(SSmlKv *kvVal, bool *isValid, SMsgBuf *msg) { +static bool smlParseDouble(SSmlKv *kvVal, bool *isValid, SSmlMsgBuf *msg) { const char *pVal = kvVal->value; int32_t len = kvVal->valueLen; if (len <= 3) { @@ -750,10 +753,10 @@ static bool parseDouble(SSmlKv *kvVal, bool *isValid, SMsgBuf *msg) { double result = strtod(pVal, &endptr); if(endptr != signalPos){ // 78ri8 *isValid = false; - buildInvalidDataMsg(msg, "invalid double", endptr); + smlBuildInvalidDataMsg(msg, "invalid double", endptr); }else if(!IS_VALID_DOUBLE(result)){ *isValid = false; - buildInvalidDataMsg(msg, "double out of range[-1.7976931348623158e+308,1.7976931348623158e+308]", endptr); + smlBuildInvalidDataMsg(msg, "double out of range[-1.7976931348623158e+308,1.7976931348623158e+308]", endptr); }else{ kvVal->d = result; *isValid = true; @@ -763,70 +766,63 @@ static bool parseDouble(SSmlKv *kvVal, bool *isValid, SMsgBuf *msg) { return false; } -static bool parseBool(SSmlKv *kvVal) { +static bool smlParseBool(SSmlKv *kvVal) { const char *pVal = kvVal->value; int32_t len = kvVal->valueLen; if ((len == 1) && pVal[len - 1] == 't') { - //printf("Type is bool(%c)\n", pVal[len - 1]); kvVal->i = true; return true; } if ((len == 1) && pVal[len - 1] == 'f') { - //printf("Type is bool(%c)\n", pVal[len - 1]); kvVal->i = false; return true; } if((len == 4) && !strcasecmp(pVal, "true")) { - //printf("Type is bool(%s)\n", &pVal[len - 4]); kvVal->i = true; return true; } if((len == 5) && !strcasecmp(pVal, "false")) { - //printf("Type is bool(%s)\n", &pVal[len - 5]); kvVal->i = false; return true; } return false; } -static bool isBinary(const char *pVal, uint16_t len) { +static bool smlIsBinary(const char *pVal, uint16_t len) { //binary: "abc" if (len < 2) { return false; } - //binary if (pVal[0] == '"' && pVal[len - 1] == '"') { - //printf("Type is binary(%s)\n", pVal); return true; } return false; } -static bool isNchar(const char *pVal, uint16_t len) { +static bool smlIsNchar(const char *pVal, uint16_t len) { //nchar: L"abc" if (len < 3) { return false; } if ((pVal[0] == 'l' || pVal[0] == 'L')&& pVal[1] == '"' && pVal[len - 1] == '"') { - //printf("Type is nchar(%s)\n", pVal); return true; } return false; } -static bool convertSmlValue(SSmlKv *pVal, SMsgBuf *msg) { +static bool smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg) { // put high probability matching type first bool isValid = false; - if (parseFloat(pVal, &isValid, msg)) { + if (smlParseFloat(pVal, &isValid, msg)) { if(!isValid) return false; pVal->type = TSDB_DATA_TYPE_FLOAT; pVal->length = (int16_t)tDataTypes[pVal->type].bytes; return true; } //binary - if (isBinary(pVal->value, pVal->valueLen)) { + if (smlIsBinary(pVal->value, pVal->valueLen)) { pVal->type = TSDB_DATA_TYPE_BINARY; pVal->length = pVal->valueLen - 2; pVal->valueLen -= 2; @@ -834,13 +830,13 @@ static bool convertSmlValue(SSmlKv *pVal, SMsgBuf *msg) { return true; } //nchar - if (isNchar(pVal->value, pVal->valueLen)) { + if (smlIsNchar(pVal->value, pVal->valueLen)) { pVal->type = TSDB_DATA_TYPE_NCHAR; pVal->length = pVal->valueLen - 3; pVal->value = pVal->value+2; return true; } - if (parseDouble(pVal, &isValid, msg)) { + if (smlParseDouble(pVal, &isValid, msg)) { if(!isValid) return false; pVal->type = TSDB_DATA_TYPE_DOUBLE; pVal->length = (int16_t)tDataTypes[pVal->type].bytes; @@ -848,66 +844,66 @@ static bool convertSmlValue(SSmlKv *pVal, SMsgBuf *msg) { return true; } //bool - if (parseBool(pVal)) { + if (smlParseBool(pVal)) { pVal->type = TSDB_DATA_TYPE_BOOL; pVal->length = (int16_t)tDataTypes[pVal->type].bytes; return true; } - if (parseTinyInt(pVal, &isValid, msg)) { + if (smlParseTinyInt(pVal, &isValid, msg)) { if(!isValid) return false; pVal->type = TSDB_DATA_TYPE_TINYINT; pVal->length = (int16_t)tDataTypes[pVal->type].bytes; return true; } - if (parseTinyUint(pVal, &isValid, msg)) { + if (smlParseTinyUint(pVal, &isValid, msg)) { if(!isValid) return false; pVal->type = TSDB_DATA_TYPE_UTINYINT; pVal->length = (int16_t)tDataTypes[pVal->type].bytes; return true; } - if (parseSmallInt(pVal, &isValid, msg)) { + if (smlParseSmallInt(pVal, &isValid, msg)) { if(!isValid) return false; pVal->type = TSDB_DATA_TYPE_SMALLINT; pVal->length = (int16_t)tDataTypes[pVal->type].bytes; return true; } - if (parseSmallUint(pVal, &isValid, msg)) { + if (smlParseSmallUint(pVal, &isValid, msg)) { if(!isValid) return false; pVal->type = TSDB_DATA_TYPE_USMALLINT; pVal->length = (int16_t)tDataTypes[pVal->type].bytes; return true; } - if (parseInt(pVal, &isValid, msg)) { + if (smlParseInt(pVal, &isValid, msg)) { if(!isValid) return false; pVal->type = TSDB_DATA_TYPE_INT; pVal->length = (int16_t)tDataTypes[pVal->type].bytes; return true; } - if (parseUint(pVal, &isValid, msg)) { + if (smlParseUint(pVal, &isValid, msg)) { if(!isValid) return false; pVal->type = TSDB_DATA_TYPE_UINT; pVal->length = (int16_t)tDataTypes[pVal->type].bytes; return true; } - if (parseBigInt(pVal, &isValid, msg)) { + if (smlParseBigInt(pVal, &isValid, msg)) { if(!isValid) return false; pVal->type = TSDB_DATA_TYPE_BIGINT; pVal->length = (int16_t)tDataTypes[pVal->type].bytes; return true; } - if (parseBigUint(pVal, &isValid, msg)) { + if (smlParseBigUint(pVal, &isValid, msg)) { if(!isValid) return false; pVal->type = TSDB_DATA_TYPE_UBIGINT; pVal->length = (int16_t)tDataTypes[pVal->type].bytes; return true; } - buildInvalidDataMsg(msg, "invalid data", pVal->value); + smlBuildInvalidDataMsg(msg, "invalid data", pVal->value); return false; } -bool checkDuplicateKey(char *key, SHashObj *pHash, SSmlLinesInfo* info) { +static bool checkDuplicateKey(char *key, SHashObj *pHash, SSmlHandle* info) { char *val = NULL; val = taosHashGet(pHash, key, strlen(key)); if (val) { @@ -921,7 +917,7 @@ bool checkDuplicateKey(char *key, SHashObj *pHash, SSmlLinesInfo* info) { return false; } -int32_t parseSml(const char* sql, TAOS_PARSE_ELEMENTS *elements){ +static int32_t smlParseString(const char* sql, SSmlLineInfo *elements, SSmlMsgBuf *msg){ if(!sql) return TSDB_CODE_SML_INVALID_DATA; while (*sql != '\0') { // jump the space at the begining if(*sql != SPACE) { @@ -930,7 +926,10 @@ int32_t parseSml(const char* sql, TAOS_PARSE_ELEMENTS *elements){ } sql++; } - if (!elements->measure || *sql == COMMA) return TSDB_CODE_SML_INVALID_DATA; + if (!elements->measure || *sql == COMMA) { + smlBuildInvalidDataMsg(msg, "invalid data", sql); + return TSDB_CODE_SML_INVALID_DATA; + } // parse measure and tag while (*sql != '\0') { @@ -953,7 +952,13 @@ int32_t parseSml(const char* sql, TAOS_PARSE_ELEMENTS *elements){ sql++; } - if(elements->measureLen == 0) return TSDB_CODE_SML_INVALID_DATA; + if(elements->tagsLen == 0){ // measure, cols1=a measure cols1=a + elements->measureTagsLen = elements->measureLen; + } + if(elements->measureLen == 0) { + smlBuildInvalidDataMsg(msg, "invalid measure", elements->measure); + return TSDB_CODE_SML_INVALID_DATA; + } // parse cols while (*sql != '\0') { @@ -963,7 +968,10 @@ int32_t parseSml(const char* sql, TAOS_PARSE_ELEMENTS *elements){ } sql++; } - if(!elements->cols) return TSDB_CODE_SML_INVALID_DATA; + if(!elements->cols) { + smlBuildInvalidDataMsg(msg, "invalid columns", elements->cols); + return TSDB_CODE_SML_INVALID_DATA; + } bool isInQuote = false; while (*sql != '\0') { @@ -992,13 +1000,13 @@ int32_t parseSml(const char* sql, TAOS_PARSE_ELEMENTS *elements){ return TSDB_CODE_SUCCESS; } -bool parseSmlCols(const char* data, int32_t len, SArray *cols, bool isTag, SMsgBuf *msg){ +static int32_t smlParseCols(const char* data, int32_t len, SArray *cols, bool isTag, SSmlMsgBuf *msg){ if(isTag && len == 0){ SSmlKv *kv = taosMemoryCalloc(sizeof(SSmlKv), 1); kv->key = TAG; - kv->keyLen = TAGNAMELEN; + kv->keyLen = strlen(TAG); kv->value = TAG; - kv->valueLen = TAGNAMELEN; + kv->valueLen = strlen(TAG); kv->type = TSDB_DATA_TYPE_NCHAR; if(cols) taosArrayPush(cols, &kv); return true; @@ -1016,7 +1024,7 @@ bool parseSmlCols(const char* data, int32_t len, SArray *cols, bool isTag, SMsgB i++; } if(keyLen == 0 || keyLen >= TSDB_COL_NAME_LEN){ - buildInvalidDataMsg(msg, "invalid key or key is too long than 64", key); + smlBuildInvalidDataMsg(msg, "invalid key or key is too long than 64", key); return TSDB_CODE_SML_INVALID_DATA; } @@ -1031,7 +1039,7 @@ bool parseSmlCols(const char* data, int32_t len, SArray *cols, bool isTag, SMsgB } int32_t valueLen = data + i - value; if(valueLen == 0){ - buildInvalidDataMsg(msg, "invalid value", value); + smlBuildInvalidDataMsg(msg, "invalid value", value); return TSDB_CODE_SML_INVALID_DATA; } @@ -1044,7 +1052,7 @@ bool parseSmlCols(const char* data, int32_t len, SArray *cols, bool isTag, SMsgB if(isTag){ kv->type = TSDB_DATA_TYPE_NCHAR; }else{ - if(!convertSmlValue(kv, msg)){ + if(!smlParseValue(kv, msg)){ return TSDB_CODE_SML_INVALID_DATA; } } @@ -1055,53 +1063,56 @@ bool parseSmlCols(const char* data, int32_t len, SArray *cols, bool isTag, SMsgB return TSDB_CODE_SUCCESS; } -static int64_t getTimeStampValue(const char *value, int32_t type) { - double ts = (double)strtoll(value, NULL, 10); +static int64_t smlGetTimeValue(const char *value, int32_t len, int8_t type) { + char *endPtr = NULL; + double ts = (double)strtoll(value, &endPtr, 10); + if(value + len != endPtr){ + return -1; + } switch (type) { case TSDB_TIME_PRECISION_HOURS: ts *= (3600 * 1e9); + break; case TSDB_TIME_PRECISION_MINUTES: ts *= (60 * 1e9); + break; case TSDB_TIME_PRECISION_SECONDS: ts *= (1e9); + break; case TSDB_TIME_PRECISION_MICRO: ts *= (1e6); + break; case TSDB_TIME_PRECISION_MILLI: ts *= (1e3); - default: break; + default: + ASSERT(0); } if(ts > (double)INT64_MAX || ts < 0){ return -1; - }else{ - return (int64_t)ts; } + + return (int64_t)ts; } -static int64_t getTimeStampNow(int32_t precision) { +static int64_t smlGetTimeNow(int8_t precision) { switch (precision) { case TSDB_TIME_PRECISION_HOURS: return taosGetTimestampMs()/1000/3600; case TSDB_TIME_PRECISION_MINUTES: return taosGetTimestampMs()/1000/60; - case TSDB_TIME_PRECISION_SECONDS: return taosGetTimestampMs()/1000; - default: + case TSDB_TIME_PRECISION_MILLI: + case TSDB_TIME_PRECISION_MICRO: + case TSDB_TIME_PRECISION_NANO: return taosGetTimestamp(precision); + default: + ASSERT(0); } } -static bool isValidateTimeStamp(const char *pVal, int32_t len) { - for (int i = 0; i < len; ++i) { - if (!isdigit(pVal[i])) { - return false; - } - } - return true; -} - -static int32_t getTsType(int32_t len) { +static int8_t smlGetTsTypeByLen(int32_t len) { if (len == TSDB_TIME_PRECISION_SEC_DIGITS) { return TSDB_TIME_PRECISION_SECONDS; } else if (len == TSDB_TIME_PRECISION_MILLI_DIGITS) { @@ -1111,52 +1122,87 @@ static int32_t getTsType(int32_t len) { } } -static int32_t parseSmlTS(const char* data, int32_t len, SArray *tags, SSmlLinesInfo* info){ - int64_t ts = 0; - if(data == NULL){ - if(info->protocol != TSDB_SML_LINE_PROTOCOL){ - buildInvalidDataMsg(&info->msgBuf, "timestamp can not be null", NULL); - return TSDB_CODE_TSC_INVALID_TIME_STAMP; - } - ts = getTimeStampNow(info->tsType); - }else{ - int ret = isValidateTimeStamp(data, len); - if(!ret){ - buildInvalidDataMsg(&info->msgBuf, "timestamp must be digit", data); - return TSDB_CODE_TSC_INVALID_TIME_STAMP; - } - int32_t tsType = -1; - if(info->protocol != TSDB_SML_LINE_PROTOCOL){ - tsType = getTsType(len); - if (tsType == -1) { - buildInvalidDataMsg(&info->msgBuf, "timestamp precision can only be seconds(10 digits) or milli seconds(13 digits)", data); - return TSDB_CODE_TSC_INVALID_TIME_STAMP; - } - }else{ - tsType = info->tsType; - } - ts = getTimeStampValue(data, tsType); - if(ts == -1){ - buildInvalidDataMsg(&info->msgBuf, "invalid timestamp", data); - return TSDB_CODE_TSC_INVALID_TIME_STAMP; - } +static int8_t smlGetTsTypeByPrecision(int8_t precision) { + switch (precision) { + case TSDB_SML_TIMESTAMP_HOURS: + return TSDB_TIME_PRECISION_HOURS; + case TSDB_SML_TIMESTAMP_MILLI_SECONDS: + return TSDB_TIME_PRECISION_MILLI; + case TSDB_SML_TIMESTAMP_NANO_SECONDS: + case TSDB_SML_TIMESTAMP_NOT_CONFIGURED: + return TSDB_TIME_PRECISION_NANO; + case TSDB_SML_TIMESTAMP_MICRO_SECONDS: + return TSDB_TIME_PRECISION_MICRO; + case TSDB_SML_TIMESTAMP_SECONDS: + return TSDB_TIME_PRECISION_SECONDS; + case TSDB_SML_TIMESTAMP_MINUTES: + return TSDB_TIME_PRECISION_MINUTES; + default: + return -1; + } +} + +static int64_t smlParseInfluxTime(SSmlHandle* info, const char* data, int32_t len){ + int8_t tsType = smlGetTsTypeByPrecision(info->precision); + if (tsType == -1) { + smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp precision", NULL); + return -1; + } + if(!data){ + return smlGetTimeNow(tsType); } + int64_t ts = smlGetTimeValue(data, len, tsType); + if(ts == -1){ + smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp", data); + return -1; + } + return ts; +} + +static int64_t smlParseOpenTsdbTime(SSmlHandle* info, const char* data, int32_t len){ + if(!data){ + smlBuildInvalidDataMsg(&info->msgBuf, "timestamp can not be null", NULL); + return -1; + } + int8_t tsType = smlGetTsTypeByLen(len); + if (tsType == -1) { + smlBuildInvalidDataMsg(&info->msgBuf, "timestamp precision can only be seconds(10 digits) or milli seconds(13 digits)", data); + return -1; + } + int64_t ts = smlGetTimeValue(data, len, tsType); + if(ts == -1){ + smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp", data); + return -1; + } + return ts; +} + +static int32_t smlParseTS(SSmlHandle* info, const char* data, int32_t len, SArray *cols){ + int64_t ts = 0; + if(info->protocol == TSDB_SML_LINE_PROTOCOL){ + ts = smlParseInfluxTime(info, data, len); + }else{ + ts = smlParseOpenTsdbTime(info, data, len); + } + if(ts == -1) return TSDB_CODE_TSC_INVALID_TIME_STAMP; + + // add ts to SSmlKv *kv = taosMemoryCalloc(sizeof(SSmlKv), 1); if(!kv){ return TSDB_CODE_OUT_OF_MEMORY; } kv->key = TS; - kv->keyLen = TSNAMELEN; + kv->keyLen = strlen(kv->key); kv->i = ts; kv->type = TSDB_DATA_TYPE_TIMESTAMP; kv->length = (int16_t)tDataTypes[kv->type].bytes; - if(tags) taosArrayPush(tags, &kv); + if(cols) taosArrayPush(cols, &kv); return TSDB_CODE_SUCCESS; } -//int32_t parseSmlCols(const char* data, SArray *cols){ +//static int32_t parseSmlCols(const char* data, SArray *cols){ // while(*data != '\0'){ // if(*data == EQUAL) return TSDB_CODE_SML_INVALID_DATA; // const char *key = data; @@ -1199,7 +1245,7 @@ static int32_t parseSmlTS(const char* data, int32_t len, SArray *tags, SSmlLines // return TSDB_CODE_SUCCESS; //} -bool updateMeta(SSmlSTableMeta* tableMeta, SArray *tags, SArray *cols, SMsgBuf *msg){ +static bool smlUpdateMeta(SSmlSTableMeta* tableMeta, SArray *tags, SArray *cols, SSmlMsgBuf *msg){ if(tags){ for (int i = 0; i < taosArrayGetSize(tags); ++i) { SSmlKv *kv = taosArrayGetP(tags, i); @@ -1223,7 +1269,7 @@ bool updateMeta(SSmlSTableMeta* tableMeta, SArray *tags, SArray *cols, SMsgBuf * SSmlKv **value = taosHashGet(tableMeta->fieldHash, kv->key, kv->keyLen); if(value){ if(kv->type != (*value)->type){ - buildInvalidDataMsg(msg, "the type is not the same like before", kv->key); + smlBuildInvalidDataMsg(msg, "the type is not the same like before", kv->key); return false; }else{ if(IS_VAR_DATA_TYPE(kv->type)){ // update string len, if bigger @@ -1237,9 +1283,10 @@ bool updateMeta(SSmlSTableMeta* tableMeta, SArray *tags, SArray *cols, SMsgBuf * } } } + return true; } -void insertMeta(SSmlSTableMeta* tableMeta, SArray *tags, SArray *cols){ +static void smlInsertMeta(SSmlSTableMeta* tableMeta, SArray *tags, SArray *cols){ if(tags){ for (int i = 0; i < taosArrayGetSize(tags); ++i) { SSmlKv *kv = taosArrayGetP(tags, i); @@ -1255,87 +1302,194 @@ void insertMeta(SSmlSTableMeta* tableMeta, SArray *tags, SArray *cols){ } } -static int32_t smlParseLine(const char* sql, SSmlLinesInfo* info) { - TAOS_PARSE_ELEMENTS elements = {0}; - int ret = parseSml(sql, &elements); +static SSmlTableInfo* smlBuildTableInfo(bool format){ + SSmlTableInfo *tag = taosMemoryCalloc(sizeof(SSmlTableInfo), 1); + if(!tag){ + return NULL; + } + + if(format){ + tag->colsFormat = taosArrayInit(16, POINTER_BYTES); + if (tag->colsFormat == NULL) { + uError("SML:smlParseLine failed to allocate memory"); + goto cleanup; + } + }else{ + tag->cols = taosArrayInit(16, POINTER_BYTES); + if (tag->cols == NULL) { + uError("SML:smlParseLine failed to allocate memory"); + goto cleanup; + } + + tag->columnsHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, false); + if (tag->columnsHash == NULL) { + uError("SML:smlParseLine failed to allocate memory"); + goto cleanup; + } + } + + tag->tags = taosArrayInit(16, POINTER_BYTES); + if (tag->tags == NULL) { + uError("SML:smlParseLine failed to allocate memory"); + goto cleanup; + } + return tag; + +cleanup: + taosMemoryFreeClear(tag); + return NULL; +} + +static void smlDestroyBuildTableInfo(SSmlTableInfo *tag, bool format){ + if(format){ + taosArrayDestroy(tag->colsFormat); + }else{ + tag->cols = taosArrayInit(16, POINTER_BYTES); + for(size_t i = 0; i < taosArrayGetSize(tag->cols); i++){ + SHashObj *kvHash = taosArrayGetP(tag->cols, i); + void** p1 = taosHashIterate(kvHash, NULL); + while (p1) { + SSmlKv* kv = *p1; + taosMemoryFreeClear(kv); + p1 = taosHashIterate(kvHash, p1); + } + taosHashCleanup(kvHash); + } + taosHashCleanup(tag->columnsHash); + } + taosArrayDestroy(tag->tags); + taosMemoryFreeClear(tag); +} + +static int32_t smlDealCols(SSmlTableInfo* oneTable, bool dataFormat, SArray *cols){ + if(dataFormat){ + taosArrayPush(oneTable->colsFormat, &cols); + }else{ + SHashObj *kvHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, false); + if(!kvHash){ + uError("SML:smlDealCols failed to allocate memory"); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + for(size_t i = 0; i < taosArrayGetSize(cols); i++){ + SSmlKv *kv = taosArrayGetP(cols, i); + taosHashPut(kvHash, kv->key, kv->keyLen, &kv, POINTER_BYTES); // todo key need escape, like \=, because find by schema name later + + if(taosHashGet(oneTable->columnsHash, kv->key, kv->keyLen) != NULL){ + continue; + } + taosHashPut(oneTable->columnsHash, kv->key, kv->keyLen, &kv, POINTER_BYTES); + } + taosArrayPush(oneTable->cols, &kvHash); + } +} + +static SSmlSTableMeta* smlBuildSTableMeta(){ + SSmlSTableMeta* meta = taosMemoryCalloc(sizeof(SSmlSTableMeta), 1); + if(!meta){ + return NULL; + } + meta->tagHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, false); + if (meta->tagHash == NULL) { + uError("SML:smlBuildSTableMeta failed to allocate memory"); + goto cleanup; + } + + meta->fieldHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, false); + if (meta->fieldHash == NULL) { + uError("SML:smlBuildSTableMeta failed to allocate memory"); + goto cleanup; + } + return meta; + +cleanup: + taosMemoryFreeClear(meta); + return NULL; +} + +static void smlDestroySTableMeta(SSmlSTableMeta *meta){ + taosHashCleanup(meta->tagHash); + taosHashCleanup(meta->fieldHash); + taosMemoryFree(meta->tableMeta); +} + +static int32_t smlParseLine(SSmlHandle* info, const char* sql) { + SSmlLineInfo elements = {0}; + int ret = smlParseString(sql, &elements, &info->msgBuf); if(ret != TSDB_CODE_SUCCESS){ + uError("SML:0x%"PRIx64" smlParseString failed", info->id); return ret; } SArray *cols = taosArrayInit(16, POINTER_BYTES); if (cols == NULL) { - uError("SML:0x%"PRIx64" taos_insert_lines failed to allocate memory", info->id); + uError("SML:0x%"PRIx64" smlParseLine failed to allocate memory", info->id); return TSDB_CODE_TSC_OUT_OF_MEMORY; } - ret = parseSmlTS(elements.timestamp, elements.timestampLen, cols, info); + ret = smlParseTS(info, elements.timestamp, elements.timestampLen, cols); if(ret != TSDB_CODE_SUCCESS){ + uError("SML:0x%"PRIx64" smlParseTS failed", info->id); return ret; } - ret = parseSmlCols(elements.cols, elements.colsLen, cols, false, &info->msgBuf); + ret = smlParseCols(elements.cols, elements.colsLen, cols, false, &info->msgBuf); if(ret != TSDB_CODE_SUCCESS){ + uError("SML:0x%"PRIx64" smlParseCols parse cloums fields failed", info->id); return ret; } if(taosArrayGetSize(cols) > TSDB_MAX_COLUMNS){ - buildInvalidDataMsg(&info->msgBuf, "too many columns than 4096", NULL); + smlBuildInvalidDataMsg(&info->msgBuf, "too many columns than 4096", NULL); return TSDB_CODE_SML_INVALID_DATA; } - TAOS_SML_DATA_POINT_TAGS** oneTable = taosHashGet(info->childTables, elements.measure, elements.measureTagsLen); + SSmlTableInfo **oneTable = taosHashGet(info->childTables, elements.measure, elements.measureTagsLen); if(oneTable){ SSmlSTableMeta** tableMeta = taosHashGet(info->superTables, elements.measure, elements.measureLen); ASSERT(tableMeta); - ret = updateMeta(*tableMeta, NULL, cols, &info->msgBuf); // update meta + ret = smlUpdateMeta(*tableMeta, NULL, cols, &info->msgBuf); // update meta cols if(!ret){ + uError("SML:0x%"PRIx64" smlUpdateMeta cols failed", info->id); return TSDB_CODE_SML_INVALID_DATA; } - taosArrayPush((*oneTable)->cols, &cols); + ret = smlDealCols(*oneTable, info->dataFormat, cols); + if(ret != TSDB_CODE_SUCCESS){ + return ret; + } }else{ - TAOS_SML_DATA_POINT_TAGS *tag = taosMemoryCalloc(sizeof(TAOS_SML_DATA_POINT_TAGS), 1); + SSmlTableInfo *tag = smlBuildTableInfo(info->dataFormat); if(!tag){ - return TSDB_CODE_OUT_OF_MEMORY; - } - tag->cols = taosArrayInit(16, POINTER_BYTES); - if (tag->cols == NULL) { - uError("SML:0x%"PRIx64" taos_insert_lines failed to allocate memory", info->id); return TSDB_CODE_TSC_OUT_OF_MEMORY; } - taosArrayPush(tag->cols, &cols); - - tag->colsColumn = taosArrayInit(16, POINTER_BYTES); - if (tag->cols == NULL) { - uError("SML:0x%"PRIx64" taos_insert_lines failed to allocate memory", info->id); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - tag->tags = taosArrayInit(16, POINTER_BYTES); - if (tag->tags == NULL) { - uError("SML:0x%"PRIx64" taos_insert_lines failed to allocate memory", info->id); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - ret = parseSmlCols(elements.tags, elements.tagsLen, tag->tags, true, &info->msgBuf); + ret = smlDealCols(tag, info->dataFormat, cols); if(ret != TSDB_CODE_SUCCESS){ return ret; } + ret = smlParseCols(elements.tags, elements.tagsLen, tag->tags, true, &info->msgBuf); + if(ret != TSDB_CODE_SUCCESS){ + uError("SML:0x%"PRIx64" smlParseCols parse tag fields failed", info->id); + return ret; + } + if(taosArrayGetSize(tag->tags) > TSDB_MAX_TAGS){ - buildInvalidDataMsg(&info->msgBuf, "too many tags than 128", NULL); + smlBuildInvalidDataMsg(&info->msgBuf, "too many tags than 128", NULL); return TSDB_CODE_SML_INVALID_DATA; } tag->sTableName = elements.measure; tag->sTableNameLen = elements.measureLen; - buildSmlChildTableName(tag); + smlBuildChildTableName(tag); + uDebug("SML:0x%"PRIx64" child table name: %s", info->id, tag->childTableName); SSmlSTableMeta** tableMeta = taosHashGet(info->superTables, elements.measure, elements.measureLen); if(tableMeta){ // update meta - ret = updateMeta(*tableMeta, tag->tags, cols, &info->msgBuf); + ret = smlUpdateMeta(*tableMeta, tag->tags, cols, &info->msgBuf); if(!ret){ + uError("SML:0x%"PRIx64" smlUpdateMeta failed", info->id); return TSDB_CODE_SML_INVALID_DATA; } }else{ - SSmlSTableMeta* meta = taosMemoryCalloc(sizeof(SSmlSTableMeta), 1); - insertMeta(meta, tag->tags, cols); + SSmlSTableMeta *meta = smlBuildSTableMeta(); + smlInsertMeta(meta, tag->tags, cols); taosHashPut(info->superTables, elements.measure, elements.measureLen, &meta, POINTER_BYTES); } @@ -1344,114 +1498,165 @@ static int32_t smlParseLine(const char* sql, SSmlLinesInfo* info) { return TSDB_CODE_SUCCESS; } -static void smlDestroyInfo(SSmlLinesInfo* info){ +static void smlDestroyInfo(SSmlHandle* info){ if(!info) return; qDestroyQuery(info->pQuery); - tscSmlDestroyHandle(info->exec); + smlDestroyHandle(info->exec); + + // destroy info->childTables + void** p1 = taosHashIterate(info->childTables, NULL); + while (p1) { + SSmlTableInfo* oneTable = *p1; + smlDestroyBuildTableInfo(oneTable, info->dataFormat); + p1 = taosHashIterate(info->childTables, p1); + } taosHashCleanup(info->childTables); + + // destroy info->superTables + p1 = taosHashIterate(info->superTables, NULL); + while (p1) { + SSmlSTableMeta* oneTable = *p1; + smlDestroySTableMeta(oneTable); + p1 = taosHashIterate(info->superTables, p1); + } taosHashCleanup(info->superTables); - taosHashCleanup(info->metaHashObj); + + // destroy info->pVgHash taosHashCleanup(info->pVgHash); - taosMemoryFree(info); + + taosMemoryFreeClear(info); } -static SSmlLinesInfo* smlBuildInfo(TAOS* taos, SRequestObj* request, SMLProtocolType protocol, int32_t tsType){ - SSmlLinesInfo* info = taosMemoryMalloc(sizeof(SSmlLinesInfo)); + +static SSmlHandle* smlBuildSmlInfo(TAOS* taos, SRequestObj* request, SMLProtocolType protocol, int8_t precision, bool dataFormat){ + SSmlHandle* info = taosMemoryMalloc(sizeof(SSmlHandle)); if (NULL == info) { return NULL; } - info->id = genLinesSmlId(); - info->tsType = tsType; - info->taos = taos; - info->protocol = protocol; + info->id = smlGenId(); - info->pQuery = taosMemoryCalloc(1, sizeof(SQuery)); + info->pQuery = taosMemoryCalloc(1, sizeof(SQuery)); if (NULL == info->pQuery) { + uError("SML:0x%"PRIx64" create info->pQuery error", info->id); goto cleanup; } - info->pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE; + info->pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE; info->pQuery->haveResultSet = false; - info->pQuery->msgType = TDMT_VND_SUBMIT; - info->pQuery->pRoot = (SNode*)nodesMakeNode(QUERY_NODE_VNODE_MODIF_STMT); + info->pQuery->msgType = TDMT_VND_SUBMIT; + info->pQuery->pRoot = (SNode*)nodesMakeNode(QUERY_NODE_VNODE_MODIF_STMT); + if(NULL == info->pQuery->pRoot){ + uError("SML:0x%"PRIx64" create info->pQuery->pRoot error", info->id); + goto cleanup; + } ((SVnodeModifOpStmt*)(info->pQuery->pRoot))->payloadType = PAYLOAD_TYPE_KV; - info->exec = tscSmlInitHandle(info->pQuery); - - int32_t code = catalogGetHandle(info->taos->pAppInfo->clusterId, &info->pCatalog); + info->taos = taos; + int32_t code = catalogGetHandle(info->taos->pAppInfo->clusterId, &info->pCatalog); if(code != TSDB_CODE_SUCCESS){ uError("SML:0x%"PRIx64" get catalog error %d", info->id, code); goto cleanup; } - info->pRequest = request; - info->msgBuf.buf = info->pRequest->msgBuf; - info->msgBuf.len = ERROR_MSG_BUF_DEFAULT_SIZE; + info->precision = precision; + info->protocol = protocol; + info->dataFormat = dataFormat; + info->pRequest = request; + info->msgBuf.buf = info->pRequest->msgBuf; + info->msgBuf.len = ERROR_MSG_BUF_DEFAULT_SIZE; + + info->exec = smlInitHandle(info->pQuery); info->childTables = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, false); info->superTables = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, false); - info->metaHashObj = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), true, false); info->pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, false); - return info; + if(NULL == info->exec || NULL == info->childTables + || NULL == info->superTables || NULL == info->pVgHash){ + uError("SML:0x%"PRIx64" create info failed", info->id); + goto cleanup; + } + return info; cleanup: smlDestroyInfo(info); return NULL; } - -int sml_insert_lines(TAOS* taos, SRequestObj* request, char* lines[], int numLines, SMLProtocolType protocol, int32_t tsType) { +static int32_t smlInsertData(SSmlHandle* info) { int32_t code = TSDB_CODE_SUCCESS; - SSmlLinesInfo* info = smlBuildInfo(taos, request, protocol, tsType); - if(!info){ - code = TSDB_CODE_OUT_OF_MEMORY; - goto cleanup; + SSmlTableInfo** oneTable = taosHashIterate(info->childTables, NULL); + while (oneTable) { + SSmlTableInfo* tableData = *oneTable; + + SName pName = {TSDB_TABLE_NAME_T, info->taos->acctId, {0}, {0}}; + strcpy(pName.dbname, info->pRequest->pDb); + memcpy(pName.tname, tableData->childTableName, strlen(tableData->childTableName)); + SEpSet ep = getEpSet_s(&info->taos->pAppInfo->mgmtEp); + SVgroupInfo vg; + code = catalogGetTableHashVgroup(info->pCatalog, info->taos->pAppInfo->pTransporter, &ep, &pName, &vg); + if (code != 0) { + uError("SML:0x%"PRIx64" catalogGetTableHashVgroup failed. table name: %s", info->id, tableData->childTableName); + return code; + } + taosHashPut(info->pVgHash, (const char*)&vg.vgId, sizeof(vg.vgId), (char*)&vg, sizeof(vg)); + + SSmlSTableMeta** pMeta = taosHashGet(info->superTables, tableData->sTableName, tableData->sTableNameLen); + ASSERT (NULL != pMeta && NULL != *pMeta); + + (*pMeta)->tableMeta->vgId = vg.vgId; + (*pMeta)->tableMeta->uid = tableData->uid; // one table merge data block together according uid + + code = smlBindData(info->exec, tableData->tags, tableData->colsFormat, tableData->columnsHash, + tableData->cols, info->dataFormat, (*pMeta)->tableMeta, info->msgBuf.buf, info->msgBuf.len); + if(code != TSDB_CODE_SUCCESS){ + return code; + } + oneTable = taosHashIterate(info->childTables, oneTable); } + + smlBuildOutput(info->exec, info->pVgHash); + launchQueryImpl(info->pRequest, info->pQuery, TSDB_CODE_SUCCESS, true); + + info->affectedRows = taos_affected_rows(info->pRequest); + return info->pRequest->code; +} + +static int smlInsertLines(SSmlHandle *info, char* lines[], int numLines) { + int32_t code = TSDB_CODE_SUCCESS; + if (numLines <= 0 || numLines > 65536) { - uError("SML:0x%"PRIx64" taos_insert_lines numLines should be between 1 and 65536. numLines: %d", info->id, numLines); + uError("SML:0x%"PRIx64" smlInsertLines numLines should be between 1 and 65536. numLines: %d", info->id, numLines); code = TSDB_CODE_TSC_APP_ERROR; goto cleanup; } + for (int32_t i = 0; i < numLines; ++i) { - code = smlParseLine(lines[i], info); + code = smlParseLine(info, lines[i]); if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%"PRIx64" data point line parse failed. line %d : %s", info->id, i, lines[i]); + uError("SML:0x%"PRIx64" smlParseLine failed. line %d : %s", info->id, i, lines[i]); goto cleanup; } } - uDebug("SML:0x%"PRIx64" data point line parse success. tables %d", info->id, taosHashGetSize(info->childTables)); + uDebug("SML:0x%"PRIx64" smlInsertLines parse success. tables %d", info->id, taosHashGetSize(info->childTables)); + uDebug("SML:0x%"PRIx64" smlInsertLines parse success. super tables %d", info->id, taosHashGetSize(info->superTables)); - code = smlInsert(taos, info); - if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%"PRIx64" taos_sml_insert error: %s", info->id, tstrerror((code))); + code = smlModifyDBSchemas(info); + if (code != 0) { + uError("SML:0x%"PRIx64" smlModifyDBSchemas error : %s", info->id, tstrerror(code)); goto cleanup; } - uDebug("SML:0x%"PRIx64" taos_insert_lines finish inserting %d lines. code: %d", info->id, numLines, code); + code = smlInsertData(info); + if (code != 0) { + uError("SML:0x%"PRIx64" smlInsertData error : %s", info->id, tstrerror(code)); + goto cleanup; + } + + uDebug("SML:0x%"PRIx64" smlInsertLines finish inserting %d lines.", info->id, numLines); cleanup: smlDestroyInfo(info); return code; } -static int32_t convertPrecisionType(int precision) { - switch (precision) { - case TSDB_SML_TIMESTAMP_HOURS: - return TSDB_TIME_PRECISION_HOURS; - case TSDB_SML_TIMESTAMP_MILLI_SECONDS: - return TSDB_TIME_PRECISION_MILLI; - case TSDB_SML_TIMESTAMP_NANO_SECONDS: - case TSDB_SML_TIMESTAMP_NOT_CONFIGURED: - return TSDB_TIME_PRECISION_NANO; - case TSDB_SML_TIMESTAMP_MICRO_SECONDS: - return TSDB_TIME_PRECISION_MICRO; - case TSDB_SML_TIMESTAMP_SECONDS: - return TSDB_TIME_PRECISION_SECONDS; - case TSDB_SML_TIMESTAMP_MINUTES: - return TSDB_TIME_PRECISION_MINUTES; - default: - return -1; - } -} - /** * taos_schemaless_insert() parse and insert data points into database according to * different protocol. @@ -1473,19 +1678,20 @@ static int32_t convertPrecisionType(int precision) { * */ -TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int protocol, int precision) { - int code = TSDB_CODE_SUCCESS; - +TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int protocol, int precision, bool dataFormat) { SRequestObj* request = createRequest(taos, NULL, NULL, TSDB_SQL_INSERT); + if(!request){ + goto end; + } + + SSmlHandle* info = smlBuildSmlInfo(taos, request, protocol, precision, dataFormat); + if(!info){ + goto end; + } + switch (protocol) { case TSDB_SML_LINE_PROTOCOL:{ - int32_t tsType = convertPrecisionType(precision); - if(tsType == -1){ - request->code = TSDB_CODE_SML_INVALID_PRECISION_TYPE; - goto end; - } - - code = sml_insert_lines(taos, request, lines, numLines, protocol, tsType); + smlInsertLines(info, lines, numLines); break; } case TSDB_SML_TELNET_PROTOCOL: @@ -1495,7 +1701,6 @@ TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int pr //code = taos_insert_json_payload(taos, *lines, protocol, tsType, &affected_rows); break; default: - code = TSDB_CODE_SML_INVALID_PROTOCOL_TYPE; break; } diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index 54e05f9264..d59473b26b 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -1514,6 +1514,7 @@ int32_t qBuildStmtColFields(void *pBlock, int32_t *fieldNum, TAOS_FIELD** fields return TSDB_CODE_SUCCESS; } +// schemaless logic start typedef struct SmlExecHandle { SHashObj* pBlockHash; @@ -1523,7 +1524,7 @@ typedef struct SmlExecHandle { SVCreateTbReq createTblReq; // each table SQuery* pQuery; -} SmlExecHandle; +} SSmlExecHandle; static int32_t smlBoundColumns(SArray *cols, SParsedDataColInfo* pColList, SSchema* pSchema) { col_id_t nCols = pColList->numOfCols; @@ -1620,14 +1621,15 @@ static int32_t smlParseTags(SArray *cols, SKVRowBuilder *tagsBuilder, SParsedDat return TSDB_CODE_SUCCESS; } -int32_t smlBind(void *handle, SArray *tags, SArray *cols, STableMeta *pTableMeta, char *msgBuf, int16_t msgBufLen) { +int32_t smlBindData(void *handle, SArray *tags, SArray *colsFormat, SHashObj *colsHash, SArray *cols, bool format, + STableMeta *pTableMeta, char *msgBuf, int16_t msgBufLen) { SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; int32_t rowNum = taosArrayGetSize(cols); if(rowNum <= 0) { return buildInvalidOperationMsg(&pBuf, "cols size <= 0"); } - SmlExecHandle *smlHandle = (SmlExecHandle *)handle; + SSmlExecHandle *smlHandle = (SSmlExecHandle *)handle; SSchema* pTagsSchema = getTableTagSchema(pTableMeta); setBoundColumnInfo(&smlHandle->tags, pTagsSchema, getNumOfTags(pTableMeta)); int ret = smlBoundColumns(tags, &smlHandle->tags, pTagsSchema); @@ -1651,7 +1653,21 @@ int32_t smlBind(void *handle, SArray *tags, SArray *cols, STableMeta *pTableMeta SSchema* pSchema = getTableColumnSchema(pTableMeta); - ret = smlBoundColumns(taosArrayGetP(cols, 0), &pDataBlock->boundColumnInfo, pSchema); + + if(format){ + ret = smlBoundColumns(taosArrayGetP(colsFormat, 0), &pDataBlock->boundColumnInfo, pSchema); + }else{ + SArray *columns = taosArrayInit(16, POINTER_BYTES); + void **p1 = taosHashIterate(colsHash, NULL); + while (p1) { + SSmlKv* kv = *p1; + taosArrayPush(columns, &kv); + p1 = taosHashIterate(colsHash, p1); + } + ret = smlBoundColumns(columns, &pDataBlock->boundColumnInfo, pSchema); + taosArrayDestroy(columns); + } + if(ret != TSDB_CODE_SUCCESS){ buildInvalidOperationMsg(&pBuf, "bound cols error"); return ret; @@ -1671,7 +1687,12 @@ int32_t smlBind(void *handle, SArray *tags, SArray *cols, STableMeta *pTableMeta for (int32_t r = 0; r < rowNum; ++r) { STSRow* row = (STSRow*)(pDataBlock->pData + pDataBlock->size); // skip the SSubmitBlk header tdSRowResetBuf(pBuilder, row); - SArray *rowData = taosArrayGetP(cols, r); + void *rowData = NULL; + if(format){ + rowData = taosArrayGetP(colsFormat, r); + }else{ + rowData = taosArrayGetP(cols, r); + } // 1. set the parsed value from sql string for (int c = 0; c < spd->numOfBound; ++c) { @@ -1680,7 +1701,18 @@ int32_t smlBind(void *handle, SArray *tags, SArray *cols, STableMeta *pTableMeta param.schema = pColSchema; getSTSRowAppendInfo(pBuilder->rowType, spd, c, ¶m.toffset, ¶m.colIdx); - SSmlKv *kv = taosArrayGetP(rowData, c); + SSmlKv *kv = NULL; + if(format){ + kv = taosArrayGetP(rowData, c); + if (!kv){ + char msg[64] = {0}; + sprintf(msg, "cols num not the same like before:%d", r); + return buildInvalidOperationMsg(&pBuf, msg); + } + }else{ + void **p =taosHashGet(rowData, pColSchema->name, strlen(pColSchema->name)); + kv = *p; + } if (kv->valueLen == 0) { MemRowAppend(&pBuf, NULL, 0, ¶m); @@ -1720,23 +1752,25 @@ int32_t smlBind(void *handle, SArray *tags, SArray *cols, STableMeta *pTableMeta return TSDB_CODE_SUCCESS; } -void* tscSmlInitHandle(SQuery *pQuery){ - SmlExecHandle *handle = taosMemoryCalloc(sizeof(SmlExecHandle)); +void* smlInitHandle(SQuery *pQuery){ + SSmlExecHandle *handle = taosMemoryCalloc(1, sizeof(SSmlExecHandle)); + if(!handle) return NULL; handle->pBlockHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false); handle->pQuery = pQuery; return handle; } -void tscSmlDestroyHandle(void *pHandle){ +void smlDestroyHandle(void *pHandle){ if(!pHandle) return; - SmlExecHandle *handle = (SmlExecHandle *)pHandle; - taosHashCleanup(handle->pBlockHash); + SSmlExecHandle *handle = (SSmlExecHandle *)pHandle; + destroyBlockHashmap(handle->pBlockHash); taosMemoryFree(handle); } int32_t smlBuildOutput(void* handle, SHashObj* pVgHash) { - SmlExecHandle *smlHandle = (SmlExecHandle *)handle; + SSmlExecHandle *smlHandle = (SSmlExecHandle *)handle; return qBuildStmtOutput(smlHandle->pQuery, pVgHash, smlHandle->pBlockHash); } +// schemaless logic end From 4fa52dc6d63ae9f5f72b62f1e7a83a9307c0fc19 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Sun, 1 May 2022 22:08:18 +0800 Subject: [PATCH 08/83] refactor:fix schemaless error & add unit test cases --- cmake/cmake.define | 2 +- include/libs/parser/parser.h | 2 +- source/client/src/clientMain.c | 5 - source/client/src/clientSml.c | 37 +++-- source/client/test/CMakeLists.txt | 21 ++- source/client/test/smlTest.cpp | 134 ++++++++++++++++++ .../libs/scalar/test/scalar/scalarTests.cpp | 2 +- 7 files changed, 177 insertions(+), 26 deletions(-) create mode 100644 source/client/test/smlTest.cpp diff --git a/cmake/cmake.define b/cmake/cmake.define index c985ba1cc7..69fc3cd0eb 100644 --- a/cmake/cmake.define +++ b/cmake/cmake.define @@ -56,7 +56,7 @@ ELSE () MESSAGE(STATUS "Will compile with Address Sanitizer!") ELSE () SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror -Werror=return-type -fPIC -gdwarf-2 -g3") - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Werror=return-type -fPIC -gdwarf-2 -g3") + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-literal-suffix -Werror=return-type -fpermissive -fPIC -gdwarf-2 -g3") ENDIF () MESSAGE("System processor ID: ${CMAKE_SYSTEM_PROCESSOR}") diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index f608e38226..d8a4eac4e5 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -98,7 +98,7 @@ int32_t qCreateSName(SName* pName, const char* pTableName, int32_t acctId, char* void* smlInitHandle(SQuery *pQuery); void smlDestroyHandle(void *pHandle); -int32_t smlBindData(void *handle, SArray *tags, SArray *cols, STableMeta *pTableMeta, char *msgBuf, int16_t msgBufLen); +int32_t smlBindData(void *handle, SArray *tags, SArray *colsFormat, SHashObj *colsHash, SArray *cols, bool format, STableMeta *pTableMeta, char *msgBuf, int16_t msgBufLen); int32_t smlBuildOutput(void* handle, SHashObj* pVgHash); #ifdef __cplusplus diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 27efcee76e..54ac42743c 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -758,8 +758,3 @@ int taos_stmt_close(TAOS_STMT *stmt) { return stmtClose(stmt); } -TAOS_RES *taos_schemaless_insert(TAOS *taos, char *lines[], int numLines, int protocol, int precision) { - // TODO - return NULL; -} - diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index bde9aeed49..8bdf737180 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -1,4 +1,3 @@ -#include #include #include #include @@ -15,6 +14,7 @@ #include "ttypes.h" #include "tcommon.h" #include "catalog.h" +#include "clientInt.h" //================================================================================================= #define SPACE ' ' @@ -971,7 +971,7 @@ static int32_t smlParseString(const char* sql, SSmlLineInfo *elements, SSmlMsgBu if(!elements->cols) { smlBuildInvalidDataMsg(msg, "invalid columns", elements->cols); return TSDB_CODE_SML_INVALID_DATA; - } + } bool isInQuote = false; while (*sql != '\0') { @@ -983,12 +983,18 @@ static int32_t smlParseString(const char* sql, SSmlLineInfo *elements, SSmlMsgBu } sql++; } + if(isInQuote){ + smlBuildInvalidDataMsg(msg, "only one quote", elements->cols); + return TSDB_CODE_SML_INVALID_DATA; + } elements->colsLen = sql - elements->cols; // parse ts,ts can be empty while (*sql != '\0') { - if(*sql != SPACE) { + if(*sql != SPACE && elements->timestamp == NULL) { elements->timestamp = sql; + } + if(*sql == SPACE && elements->timestamp != NULL){ break; } sql++; @@ -1321,7 +1327,7 @@ static SSmlTableInfo* smlBuildTableInfo(bool format){ goto cleanup; } - tag->columnsHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, false); + tag->columnsHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); if (tag->columnsHash == NULL) { uError("SML:smlParseLine failed to allocate memory"); goto cleanup; @@ -1365,7 +1371,7 @@ static int32_t smlDealCols(SSmlTableInfo* oneTable, bool dataFormat, SArray *col if(dataFormat){ taosArrayPush(oneTable->colsFormat, &cols); }else{ - SHashObj *kvHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, false); + SHashObj *kvHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); if(!kvHash){ uError("SML:smlDealCols failed to allocate memory"); return TSDB_CODE_TSC_OUT_OF_MEMORY; @@ -1381,6 +1387,7 @@ static int32_t smlDealCols(SSmlTableInfo* oneTable, bool dataFormat, SArray *col } taosArrayPush(oneTable->cols, &kvHash); } + return TSDB_CODE_SUCCESS; } static SSmlSTableMeta* smlBuildSTableMeta(){ @@ -1388,13 +1395,13 @@ static SSmlSTableMeta* smlBuildSTableMeta(){ if(!meta){ return NULL; } - meta->tagHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, false); + meta->tagHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); if (meta->tagHash == NULL) { uError("SML:smlBuildSTableMeta failed to allocate memory"); goto cleanup; } - meta->fieldHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, false); + meta->fieldHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); if (meta->fieldHash == NULL) { uError("SML:smlBuildSTableMeta failed to allocate memory"); goto cleanup; @@ -1528,6 +1535,7 @@ static void smlDestroyInfo(SSmlHandle* info){ } static SSmlHandle* smlBuildSmlInfo(TAOS* taos, SRequestObj* request, SMLProtocolType protocol, int8_t precision, bool dataFormat){ + int32_t code = TSDB_CODE_SUCCESS; SSmlHandle* info = taosMemoryMalloc(sizeof(SSmlHandle)); if (NULL == info) { return NULL; @@ -1550,7 +1558,7 @@ static SSmlHandle* smlBuildSmlInfo(TAOS* taos, SRequestObj* request, SMLProtocol ((SVnodeModifOpStmt*)(info->pQuery->pRoot))->payloadType = PAYLOAD_TYPE_KV; info->taos = taos; - int32_t code = catalogGetHandle(info->taos->pAppInfo->clusterId, &info->pCatalog); + code = catalogGetHandle(info->taos->pAppInfo->clusterId, &info->pCatalog); if(code != TSDB_CODE_SUCCESS){ uError("SML:0x%"PRIx64" get catalog error %d", info->id, code); goto cleanup; @@ -1564,9 +1572,9 @@ static SSmlHandle* smlBuildSmlInfo(TAOS* taos, SRequestObj* request, SMLProtocol info->msgBuf.len = ERROR_MSG_BUF_DEFAULT_SIZE; info->exec = smlInitHandle(info->pQuery); - info->childTables = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, false); - info->superTables = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, false); - info->pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, false); + info->childTables = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + info->superTables = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + info->pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); if(NULL == info->exec || NULL == info->childTables || NULL == info->superTables || NULL == info->pVgHash){ @@ -1653,7 +1661,6 @@ static int smlInsertLines(SSmlHandle *info, char* lines[], int numLines) { uDebug("SML:0x%"PRIx64" smlInsertLines finish inserting %d lines.", info->id, numLines); cleanup: - smlDestroyInfo(info); return code; } @@ -1681,12 +1688,12 @@ cleanup: TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int protocol, int precision, bool dataFormat) { SRequestObj* request = createRequest(taos, NULL, NULL, TSDB_SQL_INSERT); if(!request){ - goto end; + return NULL; } SSmlHandle* info = smlBuildSmlInfo(taos, request, protocol, precision, dataFormat); if(!info){ - goto end; + return (TAOS_RES*)request; } switch (protocol) { @@ -1703,7 +1710,9 @@ TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int pr default: break; } + smlDestroyInfo(info); end: return (TAOS_RES*)request; } + diff --git a/source/client/test/CMakeLists.txt b/source/client/test/CMakeLists.txt index 9df6b260f6..3fc91d07b6 100644 --- a/source/client/test/CMakeLists.txt +++ b/source/client/test/CMakeLists.txt @@ -17,14 +17,27 @@ TARGET_LINK_LIBRARIES( PUBLIC os util common transport parser catalog scheduler function gtest taos_static qcom ) +ADD_EXECUTABLE(smlTest smlTest.cpp) +TARGET_LINK_LIBRARIES( + smlTest + PUBLIC os util common transport parser catalog scheduler function gtest taos_static qcom +) + TARGET_INCLUDE_DIRECTORIES( clientTest - PUBLIC "${TD_SOURCE_DIR}/include/libs/client/" - PRIVATE "${TD_SOURCE_DIR}/source/libs/client/inc" + PUBLIC "${TD_SOURCE_DIR}/include/client/" + PRIVATE "${TD_SOURCE_DIR}/source/client/inc" ) TARGET_INCLUDE_DIRECTORIES( tmqTest - PUBLIC "${TD_SOURCE_DIR}/include/libs/client/" - PRIVATE "${TD_SOURCE_DIR}/source/libs/client/inc" + PUBLIC "${TD_SOURCE_DIR}/include/client/" + PRIVATE "${TD_SOURCE_DIR}/source/client/inc" ) + +TARGET_INCLUDE_DIRECTORIES( + smlTest + PUBLIC "${TD_SOURCE_DIR}/include/client/" + PRIVATE "${TD_SOURCE_DIR}/source/client/inc" +) + diff --git a/source/client/test/smlTest.cpp b/source/client/test/smlTest.cpp new file mode 100644 index 0000000000..823432d95d --- /dev/null +++ b/source/client/test/smlTest.cpp @@ -0,0 +1,134 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include + +#include +#include +#include + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wwrite-strings" +#pragma GCC diagnostic ignored "-Wunused-function" +#pragma GCC diagnostic ignored "-Wunused-variable" +#pragma GCC diagnostic ignored "-Wsign-compare" + +#include "../src/clientSml.c" +#include "taos.h" + +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + +TEST(testCase, sml_Test) { + char msg[256] = {0}; + SSmlMsgBuf msgBuf; + msgBuf.buf = msg; + msgBuf.len = 256; + SSmlLineInfo elements = {0}; + + //case 1 + char *sql = "st,t1=3,t2=4,t3=t3 c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 1626006833639000000 ,32,c=3"; + int ret = smlParseString(sql, &elements, &msgBuf); + ASSERT_EQ(ret, 0); + ASSERT_EQ(elements.measure, sql); + ASSERT_EQ(elements.measureLen, strlen("st")); + ASSERT_EQ(elements.measureTagsLen, strlen("st,t1=3,t2=4,t3=t3")); + + ASSERT_EQ(elements.tags, sql + elements.measureLen + 1); + ASSERT_EQ(elements.tagsLen, strlen("t1=3,t2=4,t3=t3")); + + ASSERT_EQ(elements.cols, sql + elements.measureTagsLen + 1); + ASSERT_EQ(elements.colsLen, strlen("c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64")); + + ASSERT_EQ(elements.timestamp, sql + elements.measureTagsLen + 1 + elements.colsLen + 1); + ASSERT_EQ(elements.timestampLen, strlen("1626006833639000000")); + + // case 2 false + sql = "st,t1=3,t2=4,t3=t3 c1=3i64,c3=\"passit hello,c1=2,c2=false,c4=4f64 1626006833639000000"; + memset(&elements, 0, sizeof(SSmlLineInfo)); + ret = smlParseString(sql, &elements, &msgBuf); + ASSERT_NE(ret, 0); + + // case 3 false + sql = "st, t1=3,t2=4,t3=t3 c1=3i64,c3=\"passit hello,c1=2,c2=false,c4=4f64 1626006833639000000"; + memset(&elements, 0, sizeof(SSmlLineInfo)); + ret = smlParseString(sql, &elements, &msgBuf); + ASSERT_EQ(ret, 0); + ASSERT_EQ(elements.cols, sql + elements.measureTagsLen + 2); + ASSERT_EQ(elements.colsLen, strlen("t1=3,t2=4,t3=t3")); + + + // case 4 tag is null + sql = "st, c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 1626006833639000000"; + memset(&elements, 0, sizeof(SSmlLineInfo)); + ret = smlParseString(sql, &elements, &msgBuf); + ASSERT_EQ(ret, 0); + ASSERT_EQ(elements.measure, sql); + ASSERT_EQ(elements.measureLen, strlen("st")); + ASSERT_EQ(elements.measureTagsLen, strlen("st")); + + ASSERT_EQ(elements.tags, sql + elements.measureLen + 1); + ASSERT_EQ(elements.tagsLen, 0); + + ASSERT_EQ(elements.cols, sql + elements.measureTagsLen + 2); + ASSERT_EQ(elements.colsLen, strlen("c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64")); + + ASSERT_EQ(elements.timestamp, sql + elements.measureTagsLen + 2 + elements.colsLen + 1); + ASSERT_EQ(elements.timestampLen, strlen("1626006833639000000")); + + // case 5 tag is null + sql = " st c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 1626006833639000000 "; + memset(&elements, 0, sizeof(SSmlLineInfo)); + ret = smlParseString(sql, &elements, &msgBuf); + sql++; + ASSERT_EQ(ret, 0); + ASSERT_EQ(elements.measure, sql); + ASSERT_EQ(elements.measureLen, strlen("st")); + ASSERT_EQ(elements.measureTagsLen, strlen("st")); + + ASSERT_EQ(elements.tags, sql + elements.measureLen); + ASSERT_EQ(elements.tagsLen, 0); + + ASSERT_EQ(elements.cols, sql + elements.measureTagsLen + 3); + ASSERT_EQ(elements.colsLen, strlen("c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64")); + + ASSERT_EQ(elements.timestamp, sql + elements.measureTagsLen + 3 + elements.colsLen + 2); + ASSERT_EQ(elements.timestampLen, strlen("1626006833639000000")); + + // case 6 + sql = " st c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 "; + memset(&elements, 0, sizeof(SSmlLineInfo)); + ret = smlParseString(sql, &elements, &msgBuf); + ASSERT_EQ(ret, 0); + + // case 7 + sql = " st , "; + memset(&elements, 0, sizeof(SSmlLineInfo)); + ret = smlParseString(sql, &elements, &msgBuf); + sql++; + ASSERT_EQ(ret, 0); + ASSERT_EQ(elements.cols, sql + elements.measureTagsLen + 3); + ASSERT_EQ(elements.colsLen, strlen(",")); + + // case 8 false + sql = ", st , "; + memset(&elements, 0, sizeof(SSmlLineInfo)); + ret = smlParseString(sql, &elements, &msgBuf); + ASSERT_NE(ret, 0); +} + + diff --git a/source/libs/scalar/test/scalar/scalarTests.cpp b/source/libs/scalar/test/scalar/scalarTests.cpp index 206b2ac542..c03e3408d4 100644 --- a/source/libs/scalar/test/scalar/scalarTests.cpp +++ b/source/libs/scalar/test/scalar/scalarTests.cpp @@ -127,7 +127,7 @@ void scltMakeValueNode(SNode **pNode, int32_t dataType, void *value) { *pNode = (SNode *)vnode; } -void scltMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, int32_t dataBytes, int32_t ronwNum, void *value) { +void scltMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, int32_t dataBytes, int32_t rowNum, void *value) { SNode *node = (SNode*)nodesMakeNode(QUERY_NODE_COLUMN); SColumnNode *rnode = (SColumnNode *)node; rnode->node.resType.type = dataType; From c5b62ad05368ec232778295022d0f06af9ffa0ad Mon Sep 17 00:00:00 2001 From: jiacy-jcy <714897623@qq.com> Date: Thu, 5 May 2022 11:12:21 +0800 Subject: [PATCH 09/83] update test case --- tests/system-test/2-query/Timediff.py | 196 ++++++++++++++++++ tests/system-test/2-query/To_unixtimestamp.py | 43 +++- tests/system-test/2-query/count.py | 182 ++++++++++++++++ 3 files changed, 410 insertions(+), 11 deletions(-) create mode 100644 tests/system-test/2-query/Timediff.py create mode 100644 tests/system-test/2-query/count.py diff --git a/tests/system-test/2-query/Timediff.py b/tests/system-test/2-query/Timediff.py new file mode 100644 index 0000000000..2824fea5a2 --- /dev/null +++ b/tests/system-test/2-query/Timediff.py @@ -0,0 +1,196 @@ +from util.log import * +from util.sql import * +from util.cases import * + +class TDTestCase: + + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + + def run(self): # sourcery skip: extract-duplicate-method + tdSql.prepare() + tdLog.printNoPrefix("==========step1:create tables==========") + tdSql.execute( + '''create table if not exists ntb + (ts timestamp, c1 int, c2 float,c3 double,c4 timestamp) + ''' + ) + tdSql.execute( + '''create table if not exists stb + (ts timestamp, c1 int, c2 float,c3 double,c4 timestamp) tags(t0 int) + ''' + ) + tdSql.execute( + '''create table if not exists stb_1 using stb tags(100) + ''' + ) + tdLog.printNoPrefix("==========step2:insert data into ntb==========") + + # RFC3339:2020-01-01T00:00:00+8:00 + # ISO8601:2020-01-01T00:00:00.000+0800 + tdSql.execute( + 'insert into ntb values(now,1,1.55,100.555555,today())("2020-1-1 00:00:00",10,11.11,99.999999,now())(today(),3,3.333,333.333333,now())') + tdSql.execute( + 'insert into stb_1 values(now,1,1.55,100.555555,today())("2020-1-1 00:00:00",10,11.11,99.999999,now())(today(),3,3.333,333.333333,now())') + + tdSql.query("select timediff('2020-1-1 00:00:00','2020-1-2 00:00:00') from ntb") + tdSql.checkRows(3) + tdSql.query("select timediff(1,0,1d) from ntb") + tdSql.checkRows(3) + tdSql.checkData(0,0,0) + tdSql.query("select timediff(1,0,1d) from db.ntb") + tdSql.checkRows(3) + tdSql.checkData(0,0,0) + tdSql.query("select timediff(1,0,1s) from ntb") + tdSql.checkRows(3) + tdSql.checkData(0,0,1) + tdSql.query("select timediff(1,0,1s) from db.ntb") + tdSql.checkRows(3) + tdSql.checkData(0,0,1) + tdSql.query("select timediff(1,0,1w) from ntb") + tdSql.checkRows(3) + tdSql.checkData(0,0,0) + tdSql.query("select timediff(1,0,1w) from db.ntb") + tdSql.checkRows(3) + tdSql.checkData(0,0,0) + tdSql.query("select timediff(1,0,1h) from ntb") + tdSql.checkRows(3) + tdSql.checkData(0,0,0) + tdSql.query("select timediff(1,0,1h) from db.ntb") + tdSql.checkRows(3) + tdSql.checkData(0,0,0) + tdSql.query("select timediff(1,0,1m) from ntb") + tdSql.checkRows(3) + tdSql.checkData(0,0,0) + tdSql.query("select timediff(1,0,1m) from db.ntb") + tdSql.checkRows(3) + tdSql.checkData(0,0,0) + tdSql.query("select timediff(1,0,1a) from ntb") + tdSql.checkRows(3) + tdSql.checkData(0,0,1000) + tdSql.query("select timediff(1,0,1a) from db.ntb") + tdSql.checkRows(3) + tdSql.checkData(0,0,1000) + tdSql.query("select timediff(1,0,1u) from ntb") + tdSql.checkRows(3) + tdSql.checkData(0,0,1000000) + tdSql.query("select timediff(1,0,1u) from db.ntb") + tdSql.checkRows(3) + tdSql.checkData(0,0,1000000) + + tdSql.query("select timediff('2020-1-1 00:00:00','2020-1-2 00:00:00') from stb") + tdSql.checkRows(3) + tdSql.query("select timediff('2020-1-1 00:00:00','2020-1-2 00:00:00') from db.stb") + tdSql.checkRows(3) + tdSql.query("select timediff('2020-1-1 00:00:00','2020-1-2 00:00:00',1d) from stb") + tdSql.checkRows(3) + tdSql.checkData(0,0,1) + tdSql.query("select timediff('2020-1-1 00:00:00','2020-1-2 00:00:00',1d) from db.stb") + tdSql.checkRows(3) + tdSql.checkData(0,0,1) + tdSql.query("select timediff('2020-1-1 00:00:00','2020-1-2 00:00:00',1h) from stb") + tdSql.checkRows(3) + tdSql.checkData(0,0,24) + tdSql.query("select timediff('2020-1-1 00:00:00','2020-1-2 00:00:00',1h) from db.stb") + tdSql.checkRows(3) + tdSql.checkData(0,0,24) + tdSql.query("select timediff('2020-1-1 00:00:00','2020-1-2 00:00:00',1w) from stb") + tdSql.checkRows(3) + tdSql.checkData(0,0,0) + tdSql.query("select timediff('2020-1-1 00:00:00','2020-1-2 00:00:00',1m) from stb") + tdSql.checkRows(3) + tdSql.checkData(0,0,1440) + tdSql.query("select timediff('2020-1-1 00:00:00','2020-1-2 00:00:00',1m) from db.stb") + tdSql.checkRows(3) + tdSql.checkData(0,0,1440) + tdSql.query("select timediff('2020-1-1 00:00:00','2020-1-2 00:00:00',1s) from stb") + tdSql.checkRows(3) + tdSql.checkData(0,0,86400) + tdSql.query("select timediff('2020-1-1 00:00:00','2020-1-2 00:00:00',1s) from db.stb") + tdSql.checkRows(3) + tdSql.checkData(0,0,86400) + tdSql.query("select timediff('2020-1-1 00:00:00','2020-1-2 00:00:00',1a) from stb") + tdSql.checkRows(3) + tdSql.checkData(0,0,86400000) + tdSql.query("select timediff('2020-1-1 00:00:00','2020-1-2 00:00:00',1a) from db.stb") + tdSql.checkRows(3) + tdSql.checkData(0,0,86400000) + tdSql.query("select timediff('2020-1-1 00:00:00','2020-1-2 00:00:00',1u) from stb") + tdSql.checkRows(3) + tdSql.checkData(0,0,86400000000) + tdSql.query("select timediff('2020-1-1 00:00:00','2020-1-2 00:00:00',1u) from db.stb") + tdSql.checkRows(3) + tdSql.checkData(0,0,86400000000) + + + tdSql.query("select timediff('2020-1-1 00:00:00','2020-1-1 12:00:00') from stb_1") + tdSql.checkRows(3) + tdSql.query("select timediff('2020-1-1 00:00:00','2020-1-1 12:00:00') from db.stb_1") + tdSql.checkRows(3) + tdSql.query("select timediff('2020-1-1 00:00:00','2020-1-1 12:00:00',1w) from stb_1 ") + tdSql.checkRows(3) + tdSql.checkData(0,0,0) + tdSql.query("select timediff('2020-1-1 00:00:00','2020-1-1 12:00:00',1w) from db.stb_1 ") + tdSql.checkRows(3) + tdSql.checkData(0,0,0) + tdSql.query("select timediff('2020-1-1 00:00:00','2020-1-1 12:00:00',1d) from stb_1 ") + tdSql.checkRows(3) + tdSql.checkData(0,0,0) + tdSql.query("select timediff('2020-1-1 00:00:00','2020-1-1 12:00:00',1d) from db.stb_1 ") + tdSql.checkRows(3) + tdSql.checkData(0,0,0) + tdSql.query("select timediff('2020-1-1 00:00:00','2020-1-1 12:00:00',1h) from stb_1 ") + tdSql.checkRows(3) + tdSql.checkData(0,0,12) + tdSql.query("select timediff('2020-1-1 00:00:00','2020-1-1 12:00:00',1h) from db.stb_1 ") + tdSql.checkRows(3) + tdSql.checkData(0,0,12) + tdSql.query("select timediff('2020-1-1 00:00:00','2020-1-1 12:00:00',1m) from stb_1" ) + tdSql.checkRows(3) + tdSql.checkData(0,0,720) + tdSql.query("select timediff('2020-1-1 00:00:00','2020-1-1 12:00:00',1m) from db.stb_1" ) + tdSql.checkRows(3) + tdSql.checkData(0,0,720) + tdSql.query("select timediff('2020-1-1 00:00:00','2020-1-1 12:00:00',1s) from stb_1") + tdSql.checkRows(3) + tdSql.checkData(0,0,43200) + tdSql.query("select timediff('2020-1-1 00:00:00','2020-1-1 12:00:00',1s) from db.stb_1") + tdSql.checkRows(3) + tdSql.checkData(0,0,43200) + tdSql.query("select timediff('2020-1-1 00:00:00','2020-1-1 12:00:00',1a) from stb_1") + tdSql.checkRows(3) + tdSql.checkData(0,0,43200000) + tdSql.query("select timediff('2020-1-1 00:00:00','2020-1-1 12:00:00',1a) from db.stb_1") + tdSql.checkRows(3) + tdSql.checkData(0,0,43200000) + tdSql.query("select timediff('2020-1-1 00:00:00','2020-1-1 12:00:00',1u) from stb_1") + tdSql.checkRows(3) + tdSql.checkData(0,0,43200000000) + tdSql.query("select timediff('2020-1-1 00:00:00','2020-1-1 12:00:00',1u) from db.stb_1") + tdSql.checkRows(3) + tdSql.checkData(0,0,43200000000) + + tdSql.query("select timediff('a','b') from stb") + tdSql.checkRows(3) + tdSql.checkData(0,0,None) + tdSql.checkData(1,0,None) + tdSql.checkData(2,0,None) + tdSql.error("select timediff(1.5,1.5) from stb") + tdSql.error("select timediff(1) from stb") + tdSql.error("select timediff(10,1,1.5) from stb") + # tdSql.error("select timediff(10,1,2s) from stb") + # tdSql.error("select timedifff(10,1,c1) from stb") + + + + + + + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/system-test/2-query/To_unixtimestamp.py b/tests/system-test/2-query/To_unixtimestamp.py index 7088142f9a..43315ff0d8 100644 --- a/tests/system-test/2-query/To_unixtimestamp.py +++ b/tests/system-test/2-query/To_unixtimestamp.py @@ -31,19 +31,40 @@ class TDTestCase: ''' ) tdLog.printNoPrefix("==========step2:insert data into ntb==========") + + # RFC3339:2020-01-01T00:00:00+8:00 + # ISO8601:2020-01-01T00:00:00.000+0800 + tdSql.execute( + 'insert into ntb values(now,1,1.55,100.555555,today())("2020-1-1 00:00:00",10,11.11,99.999999,now())(today(),3,3.333,333.333333,now())') + tdSql.execute( + 'insert into stb_1 values(now,1,1.55,100.555555,today())("2020-1-1 00:00:00",10,11.11,99.999999,now())(today(),3,3.333,333.333333,now())') + tdSql.query("select to_unixtimestamp('1970-01-01T08:00:00+0800') from ntb") + tdSql.checkData(0,0,0) + tdSql.checkData(1,0,0) + tdSql.checkData(2,0,0) + tdSql.checkRows(3) + tdSql.query("select to_unixtimestamp('1970-01-01T08:00:00+08:00') from ntb") + tdSql.checkData(0,0,0) + tdSql.checkRows(3) + tdSql.query("select to_unixtimestamp('1900-01-01T08:00:00+08:00') from ntb") + tdSql.checkRows(3) + tdSql.query("select to_unixtimestamp('2020-01-32T08:00:00') from ntb") + tdSql.checkRows(3) + tdSql.checkData(0,0,None) + tdSql.query("select to_unixtimestamp('2020-13-32T08:00:00') from ntb") + tdSql.checkRows(3) + tdSql.checkData(0,0,None) + tdSql.query("select to_unixtimestamp('acd') from ntb") + tdSql.checkRows(3) + tdSql.checkData(0,0,None) + tdSql.error("select to_unixtimestamp(1) from ntb") + tdSql.error("select to_unixtimestamp(1.5) from ntb") + tdSql.error("select to_unixtimestamp(ts) from ntb") + + tdSql.query("select ts from ntb where to_unixtimestamp('1970-01-01T08:00:00+08:00')=0") + tdSql.checkRows(3) - - - - - - - - - - - def stop(self): tdSql.close() tdLog.success(f"{__file__} successfully executed") diff --git a/tests/system-test/2-query/count.py b/tests/system-test/2-query/count.py new file mode 100644 index 0000000000..8888a182b8 --- /dev/null +++ b/tests/system-test/2-query/count.py @@ -0,0 +1,182 @@ +from util.log import * +from util.sql import * +from util.cases import * + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.rowNum = 10 + self.ts = 1537146000000 + + def run(self): + tdSql.prepare() + + tdSql.execute('''create table stb(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, + col7 bool, col8 binary(20), col9 nchar(20), col11 tinyint unsigned, col12 smallint unsigned, col13 int unsigned, col14 bigint unsigned) tags(loc nchar(20))''') + tdSql.execute("create table stb_1 using stb tags('beijing')") + + tdSql.execute('''create table ntb(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, + col7 bool, col8 binary(20), col9 nchar(20), col11 tinyint unsigned, col12 smallint unsigned, col13 int unsigned, col14 bigint unsigned)''') + + for i in range(self.rowNum): + tdSql.execute("insert into stb_1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1)) + + for i in range(self.rowNum): + tdSql.execute("insert into ntb values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1)) + + tdSql.query("select count(*) from stb") + tdSql.checkData(0,0,10) + tdSql.query("select count(*) from db.stb") + tdSql.checkData(0,0,10) + tdSql.query("select count(ts) from stb") + tdSql.checkData(0,0,10) + tdSql.query("select count(ts) from db.stb") + tdSql.checkData(0,0,10) + tdSql.query("select count(col1) from stb") + tdSql.checkData(0,0,10) + tdSql.query("select count(col1) from db.stb") + tdSql.checkData(0,0,10) + tdSql.query("select count(col2) from stb") + tdSql.checkData(0,0,10) + tdSql.query("select count(col2) from db.stb") + tdSql.checkData(0,0,10) + tdSql.query("select count(col3) from stb") + tdSql.checkData(0,0,10) + tdSql.query("select count(col3) from db.stb") + tdSql.checkData(0,0,10) + tdSql.query("select count(col4) from stb") + tdSql.checkData(0,0,10) + tdSql.query("select count(col4) from db.stb") + tdSql.checkData(0,0,10) + tdSql.query("select count(col5) from stb") + tdSql.checkData(0,0,10) + tdSql.query("select count(col5) from db.stb") + tdSql.checkData(0,0,10) + tdSql.query("select count(col6) from stb") + tdSql.checkData(0,0,10) + tdSql.query("select count(col6) from db.stb") + tdSql.checkData(0,0,10) + tdSql.query("select count(col7) from stb") + tdSql.checkData(0,0,10) + tdSql.query("select count(col7) from db.stb") + tdSql.checkData(0,0,10) + tdSql.query("select count(col8) from stb") + tdSql.checkData(0,0,10) + tdSql.query("select count(col8) from db.stb") + tdSql.checkData(0,0,10) + tdSql.query("select count(col9) from stb") + tdSql.checkData(0,0,10) + tdSql.query("select count(col9) from db.stb") + tdSql.checkData(0,0,10) + tdSql.query("select count(col11) from stb") + tdSql.checkData(0,0,10) + tdSql.query("select count(col11) from db.stb") + tdSql.checkData(0,0,10) + tdSql.query("select count(col12) from stb") + tdSql.checkData(0,0,10) + tdSql.query("select count(col12) from db.stb") + tdSql.checkData(0,0,10) + tdSql.query("select count(col13) from stb") + tdSql.checkData(0,0,10) + tdSql.query("select count(col13) from db.stb") + tdSql.checkData(0,0,10) + tdSql.query("select count(col14) from stb") + tdSql.checkData(0,0,10) + tdSql.query("select count(col14) from db.stb") + tdSql.checkData(0,0,10) + tdSql.query("select count(ts) from db.stb_1") + tdSql.checkData(0,0,10) + tdSql.query("select count(ts) from db.stb_1") + tdSql.checkData(0,0,10) + tdSql.query("select count(col1) from stb_1") + tdSql.checkData(0,0,10) + tdSql.query("select count(col1) from db.stb_1") + tdSql.checkData(0,0,10) + tdSql.query("select count(col2) from stb_1") + tdSql.checkData(0,0,10) + tdSql.query("select count(col2) from db.stb_1") + tdSql.checkData(0,0,10) + tdSql.query("select count(col3) from stb_1") + tdSql.checkData(0,0,10) + tdSql.query("select count(col3) from db.stb_1") + tdSql.checkData(0,0,10) + tdSql.query("select count(col4) from stb_1") + tdSql.checkData(0,0,10) + tdSql.query("select count(col4) from db.stb_1") + tdSql.checkData(0,0,10) + tdSql.query("select count(col5) from stb_1") + tdSql.checkData(0,0,10) + tdSql.query("select count(col5) from db.stb_1") + tdSql.checkData(0,0,10) + tdSql.query("select count(col6) from stb_1") + tdSql.checkData(0,0,10) + tdSql.query("select count(col6) from db.stb_1") + tdSql.checkData(0,0,10) + tdSql.query("select count(col7) from stb_1") + tdSql.checkData(0,0,10) + tdSql.query("select count(col7) from db.stb_1") + tdSql.checkData(0,0,10) + tdSql.query("select count(col8) from stb_1") + tdSql.checkData(0,0,10) + tdSql.query("select count(col8) from db.stb_1") + tdSql.checkData(0,0,10) + tdSql.query("select count(col9) from stb_1") + tdSql.checkData(0,0,10) + tdSql.query("select count(col9) from db.stb_1") + tdSql.checkData(0,0,10) + tdSql.query("select count(col11) from stb_1") + tdSql.checkData(0,0,10) + tdSql.query("select count(col11) from db.stb_1") + tdSql.checkData(0,0,10) + tdSql.query("select count(col12) from stb_1") + tdSql.checkData(0,0,10) + tdSql.query("select count(col12) from db.stb_1") + tdSql.checkData(0,0,10) + tdSql.query("select count(col13) from stb_1") + tdSql.checkData(0,0,10) + tdSql.query("select count(col13) from db.stb_1") + tdSql.checkData(0,0,10) + tdSql.query("select count(col14) from stb_1") + tdSql.checkData(0,0,10) + tdSql.query("select count(col14) from db.stb_1") + tdSql.checkData(0,0,10) + + tdSql.execute("insert into stb_1 values(now,null,null,null,null,null,null,null,null,null,null,null,null,null)") + tdSql.query("select count(col1) from stb_1") + tdSql.checkData(0,0,10) + tdSql.query("select count(col1),count(ts) from stb_1") + tdSql.checkData(0,0,10) + tdSql.checkData(0,1,11) + + tdSql.query("select count(col1) from db.stb_1") + tdSql.checkData(0,0,10) + tdSql.query("select count(col1),count(ts) from db.stb_1") + tdSql.checkData(0,0,10) + tdSql.checkData(0,1,11) + + tdSql.query("select count(col1) from stb") + tdSql.checkData(0,0,10) + tdSql.query("select count(col1),count(ts) from stb") + tdSql.checkData(0,0,10) + tdSql.checkData(0,1,11) + + tdSql.query("select count(col1) from db.stb") + tdSql.checkData(0,0,10) + tdSql.query("select count(col1),count(ts) from db.stb") + tdSql.checkData(0,0,10) + tdSql.checkData(0,1,11) + + + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file From 119e7952ded8ee60f0ebee9d0913dfb4d37c0b11 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 5 May 2022 12:08:34 +0800 Subject: [PATCH 10/83] refactor:add schemaless test cases --- source/client/src/clientSml.c | 40 +++-- source/client/test/smlTest.cpp | 313 ++++++++++++++++++++++++++++++++- 2 files changed, 329 insertions(+), 24 deletions(-) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 8bdf737180..de034957dc 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -141,9 +141,9 @@ static int32_t smlBuildInvalidDataMsg(SSmlMsgBuf* pBuf, const char *msg1, const static int smlCompareKv(const void* p1, const void* p2) { SSmlKv* kv1 = (SSmlKv *)p1; SSmlKv* kv2 = (SSmlKv*)p2; - int kvLen1 = (int)strlen(kv1->key); - int kvLen2 = (int)strlen(kv2->key); - int res = strncasecmp(kv1->key, kv2->key, MIN(kvLen1, kvLen2)); + int32_t kvLen1 = kv1->keyLen; + int32_t kvLen2 = kv2->keyLen; + int32_t res = strncasecmp(kv1->key, kv2->key, MIN(kvLen1, kvLen2)); if (res != 0) { return res; } else { @@ -154,7 +154,7 @@ static int smlCompareKv(const void* p1, const void* p2) { static void smlBuildChildTableName(SSmlTableInfo *tags) { int32_t size = taosArrayGetSize(tags->tags); ASSERT(size > 0); - qsort(tags->tags, size, POINTER_BYTES, smlCompareKv); + taosArraySort(tags->tags, smlCompareKv); SStringBuilder sb = {0}; taosStringBuilderAppendStringLen(&sb, tags->sTableName, tags->sTableNameLen); @@ -501,7 +501,7 @@ static bool smlParseTinyInt(SSmlKv *kvVal, bool *isValid, SSmlMsgBuf *msg) { return false; } const char *signalPos = pVal + len - 2; - if (!strcasecmp(signalPos, "i8")) { + if (!strncasecmp(signalPos, "i8", 2)) { char *endptr = NULL; int64_t result = strtoll(pVal, &endptr, 10); if(endptr != signalPos){ // 78ri8 @@ -529,7 +529,7 @@ static bool smlParseTinyUint(SSmlKv *kvVal, bool *isValid, SSmlMsgBuf *msg) { return false; } const char *signalPos = pVal + len - 2; - if (!strcasecmp(signalPos, "u8")) { + if (!strncasecmp(signalPos, "u8", 2)) { char *endptr = NULL; int64_t result = strtoll(pVal, &endptr, 10); if(endptr != signalPos){ // 78ri8 @@ -554,7 +554,7 @@ static bool smlParseSmallInt(SSmlKv *kvVal, bool *isValid, SSmlMsgBuf *msg) { return false; } const char *signalPos = pVal + len - 3; - if (!strcasecmp(signalPos, "i16")) { + if (!strncasecmp(signalPos, "i16", 3)) { char *endptr = NULL; int64_t result = strtoll(pVal, &endptr, 10); if(endptr != signalPos){ // 78ri8 @@ -582,7 +582,7 @@ static bool smlParseSmallUint(SSmlKv *kvVal, bool *isValid, SSmlMsgBuf *msg) { return false; } const char *signalPos = pVal + len - 3; - if (strcasecmp(signalPos, "u16") == 0) { + if (strncasecmp(signalPos, "u16", 3) == 0) { char *endptr = NULL; int64_t result = strtoll(pVal, &endptr, 10); if(endptr != signalPos){ // 78ri8 @@ -607,7 +607,7 @@ static bool smlParseInt(SSmlKv *kvVal, bool *isValid, SSmlMsgBuf *msg) { return false; } const char *signalPos = pVal + len - 3; - if (strcasecmp(signalPos, "i32") == 0) { + if (strncasecmp(signalPos, "i32", 3) == 0) { char *endptr = NULL; int64_t result = strtoll(pVal, &endptr, 10); if(endptr != signalPos){ // 78ri8 @@ -635,7 +635,7 @@ static bool smlParseUint(SSmlKv *kvVal, bool *isValid, SSmlMsgBuf *msg) { return false; } const char *signalPos = pVal + len - 3; - if (strcasecmp(signalPos, "u32") == 0) { + if (strncasecmp(signalPos, "u32", 3) == 0) { char *endptr = NULL; int64_t result = strtoll(pVal, &endptr, 10); if(endptr != signalPos){ // 78ri8 @@ -656,13 +656,15 @@ static bool smlParseUint(SSmlKv *kvVal, bool *isValid, SSmlMsgBuf *msg) { static bool smlParseBigInt(SSmlKv *kvVal, bool *isValid, SSmlMsgBuf *msg) { const char *pVal = kvVal->value; int32_t len = kvVal->valueLen; - if (len > 3 && strcasecmp(pVal + len - 3, "i64") == 0) { + if (len > 3 && strncasecmp(pVal + len - 3, "i64", 3) == 0) { char *endptr = NULL; int64_t result = strtoll(pVal, &endptr, 10); if(endptr != pVal + len - 3){ // 78ri8 *isValid = false; + smlBuildInvalidDataMsg(msg, "invalid big int", endptr); }else if(!IS_VALID_BIGINT(result)){ *isValid = false; + smlBuildInvalidDataMsg(msg, "big int out of range[-9223372036854775808,9223372036854775807]", endptr); }else{ kvVal->i = result; *isValid = true; @@ -696,7 +698,7 @@ static bool smlParseBigUint(SSmlKv *kvVal, bool *isValid, SSmlMsgBuf *msg) { return false; } const char *signalPos = pVal + len - 3; - if (strcasecmp(signalPos, "u64") == 0) { + if (strncasecmp(signalPos, "u64", 3) == 0) { char *endptr = NULL; uint64_t result = strtoull(pVal, &endptr, 10); if(endptr != signalPos){ // 78ri8 @@ -725,7 +727,7 @@ static bool smlParseFloat(SSmlKv *kvVal, bool *isValid, SSmlMsgBuf *msg) { return true; } - if (len > 3 && len 3 && len i = true; return true; } - if((len == 5) && !strcasecmp(pVal, "false")) { + if((len == 5) && !strncasecmp(pVal, "false", len)) { kvVal->i = false; return true; } @@ -1085,12 +1087,14 @@ static int64_t smlGetTimeValue(const char *value, int32_t len, int8_t type) { case TSDB_TIME_PRECISION_SECONDS: ts *= (1e9); break; - case TSDB_TIME_PRECISION_MICRO: + case TSDB_TIME_PRECISION_MILLI: ts *= (1e6); break; - case TSDB_TIME_PRECISION_MILLI: + case TSDB_TIME_PRECISION_MICRO: ts *= (1e3); break; + case TSDB_TIME_PRECISION_NANO: + break; default: ASSERT(0); } diff --git a/source/client/test/smlTest.cpp b/source/client/test/smlTest.cpp index 823432d95d..9337263173 100644 --- a/source/client/test/smlTest.cpp +++ b/source/client/test/smlTest.cpp @@ -28,21 +28,21 @@ #include "../src/clientSml.c" #include "taos.h" -int main(int argc, char** argv) { +int main(int argc, char **argv) { testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); } -TEST(testCase, sml_Test) { - char msg[256] = {0}; +TEST(testCase, smlParseString_Test) { + char msg[256] = {0}; SSmlMsgBuf msgBuf; msgBuf.buf = msg; msgBuf.len = 256; SSmlLineInfo elements = {0}; - //case 1 + // case 1 char *sql = "st,t1=3,t2=4,t3=t3 c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 1626006833639000000 ,32,c=3"; - int ret = smlParseString(sql, &elements, &msgBuf); + int ret = smlParseString(sql, &elements, &msgBuf); ASSERT_EQ(ret, 0); ASSERT_EQ(elements.measure, sql); ASSERT_EQ(elements.measureLen, strlen("st")); @@ -71,7 +71,6 @@ TEST(testCase, sml_Test) { ASSERT_EQ(elements.cols, sql + elements.measureTagsLen + 2); ASSERT_EQ(elements.colsLen, strlen("t1=3,t2=4,t3=t3")); - // case 4 tag is null sql = "st, c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 1626006833639000000"; memset(&elements, 0, sizeof(SSmlLineInfo)); @@ -131,4 +130,306 @@ TEST(testCase, sml_Test) { ASSERT_NE(ret, 0); } +TEST(testCase, smlParseCols_Error_Test) { + char msg[256] = {0}; + SSmlMsgBuf msgBuf; + msgBuf.buf = msg; + msgBuf.len = 256; + const char *data[] = { + "c=\"89sd", // binary, nchar + "c=j\"89sd\"", + "c=\"89sd\"k", + "c=u", // bool + "c=truet", + "c=f64", // double + "c=8f64f", + "c=8ef64", + "c=1.7976931348623158e+390f64", + "c=f32", // float + "c=8f32f", + "c=8wef32", + "c=-3.402823466e+39f32", + "c=", // float + "c=8f", + "c=8we", + "c=3.402823466e+39", + "c=i8", // tiny int + "c=-8i8f", + "c=8wei8", + "c=-999i8", + "c=u8", // u tiny int + "c=8fu8", + "c=8weu8", + "c=999u8", + "c=-8u8", + "c=i16", // small int + "c=8fi16u", + "c=8wei16", + "c=-67787i16", + "c=u16", // u small int + "c=8u16f", + "c=8weu16", + "c=-9u16", + "c=67787u16", + "c=i32", // int + "c=8i32f", + "c=8wei32", + "c=2147483649i32", + "c=u32", // u int + "c=8u32f", + "c=8weu32", + "c=-4u32", + "c=42949672958u32", + "c=i64", // big int + "c=8i64i", + "c=8wei64", + "c=-9223372036854775809i64", + "c=i", // big int + "c=8fi", + "c=8wei", + "c=9223372036854775808i", + "c=u64", // u big int + "c=8u64f", + "c=8weu64", + "c=-3.402823466e+39u64", + "c=-339u64", + "c=18446744073709551616u64", + }; + + for(int i = 0; i < sizeof(data)/sizeof(data[0]); i++){ + int32_t len = strlen(data[i]); + int32_t ret = smlParseCols(data[i], len, NULL, false, &msgBuf); + ASSERT_NE(ret, TSDB_CODE_SUCCESS); + } +} + +TEST(testCase, smlParseCols_Test) { + char msg[256] = {0}; + SSmlMsgBuf msgBuf; + msgBuf.buf = msg; + msgBuf.len = 256; + + SArray *cols = taosArrayInit(16, POINTER_BYTES); + ASSERT_NE(cols, NULL); + + const char *data = "cbin=\"passit hello,c=2\",cnch=L\"iisdfsf\",cbool=false,cf64=4f64,cf32_=8.32,cf32=8.23f32,ci8=-34i8,cu8=89u8,ci16=233i16,cu16=898u16,ci32=98289i32,cu32=12323u32,ci64=-89238i64,ci=989i,cu64=8989323u64,cbooltrue=true,cboolt=t,cboolf=f,cnch_=l\"iuwq\""; + int32_t len = strlen(data); + int32_t ret = smlParseCols(data, len, cols, false, &msgBuf); + ASSERT_EQ(ret, TSDB_CODE_SUCCESS); + int32_t size = taosArrayGetSize(cols); + ASSERT_EQ(size, 19); + + // binary + SSmlKv *kv = taosArrayGetP(cols, 0); + ASSERT_EQ(strncasecmp(kv->key, "cbin", 4), 0); + ASSERT_EQ(kv->keyLen, 4; + ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BINARY; + ASSERT_EQ(kv->length, 16); + ASSERT_EQ(strncasecmp(kv->value, "passit", 6), 0); + taosMemoryFree(kv); + + // nchar + kv = taosArrayGetP(cols, 1); + ASSERT_EQ(strncasecmp(kv->key, "cnch", 4), 0); + ASSERT_EQ(kv->keyLen, 4); + ASSERT_EQ(kv->type, TSDB_DATA_TYPE_NCHAR); + ASSERT_EQ(kv->length, 7); + ASSERT_EQ(strncasecmp(kv->value, "iisd", 4), 0); + taosMemoryFree(kv); + + // bool + kv = taosArrayGetP(cols, 2); + ASSERT_EQ(strncasecmp(kv->key, "cbool", 5), 0); + ASSERT_EQ(kv->keyLen, 5); + ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(kv->length, 1; + ASSERT_EQ(kv->i, false); + taosMemoryFree(kv); + + // double + kv = taosArrayGetP(cols, 3); + ASSERT_EQ(strncasecmp(kv->key, "cf64", 4), 0); + ASSERT_EQ(kv->keyLen, 4); + ASSERT_EQ(kv->type, TSDB_DATA_TYPE_DOUBLE); + ASSERT_EQ(kv->length, 8; + ASSERT_EQ(kv->d, 4); + taosMemoryFree(kv); + + // float + kv = taosArrayGetP(cols, 4); + ASSERT_EQ(strncasecmp(kv->key, "cf32_", 5), 0); + ASSERT_EQ(kv->keyLen, 5); + ASSERT_EQ(kv->type, TSDB_DATA_TYPE_FLOAT); + ASSERT_EQ(kv->length, 4); + ASSERT_EQ(kv->f, 8.32); + taosMemoryFree(kv); + + // float + kv = taosArrayGetP(cols, 5); + ASSERT_EQ(strncasecmp(kv->key, "cf32", 4), 0); + ASSERT_EQ(kv->keyLen, 4); + ASSERT_EQ(kv->type, TSDB_DATA_TYPE_FLOAT); + ASSERT_EQ(kv->length, 4); + ASSERT_EQ(kv->f, 8.23); + taosMemoryFree(kv); + + // tiny int + kv = taosArrayGetP(cols, 6); + ASSERT_EQ(strncasecmp(kv->key, "ci8", 3), 0); + ASSERT_EQ(kv->keyLen, 3); + ASSERT_EQ(kv->type, TSDB_DATA_TYPE_TINYINT; + ASSERT_EQ(kv->length, 1); + ASSERT_EQ(i, -34); + taosMemoryFree(kv); + + // unsigned tiny int + kv = taosArrayGetP(cols, 7); + ASSERT_EQ(strncasecmp(kv->key, "cu8", 3), 0); + ASSERT_EQ(kv->keyLen, 3); + ASSERT_EQ(kv->type, TSDB_DATA_TYPE_UTINYINT); + ASSERT_EQ(kv->length, 1); + ASSERT_EQ(kv->u, 89); + taosMemoryFree(kv); + + // small int + kv = taosArrayGetP(cols, 8); + ASSERT_EQ(strncasecmp(kv->key, "ci16", 4), 0); + ASSERT_EQ(kv->keyLen, 4); + ASSERT_EQ(kv->type, TSDB_DATA_TYPE_SMALLINT); + ASSERT_EQ(kv->length, 2); + ASSERT_EQ(kv->u, 233); + taosMemoryFree(kv); + + // unsigned smallint + kv = taosArrayGetP(cols, 9); + ASSERT_EQ(strncasecmp(kv->key, "cu16", 4), 0); + ASSERT_EQ(kv->keyLen, 4); + ASSERT_EQ(kv->type, TSDB_DATA_TYPE_USMALLINT); + ASSERT_EQ(kv->length, 2); + ASSERT_EQ(kv->u, 898); + taosMemoryFree(kv); + + // int + kv = taosArrayGetP(cols, 10); + ASSERT_EQ(strncasecmp(kv->key, "ci32", 4), 0); + ASSERT_EQ(kv->keyLen, 4); + ASSERT_EQ(kv->type, TSDB_DATA_TYPE_INT); + ASSERT_EQ(kv->length, 4); + ASSERT_EQ(kv->u, 98289); + taosMemoryFree(kv); + + // unsigned int + kv = taosArrayGetP(cols, 11); + ASSERT_EQ(strncasecmp(kv->key, "cu32", 4), 0); + ASSERT_EQ(kv->keyLen, 4); + ASSERT_EQ(kv->type, TSDB_DATA_TYPE_UINT); + ASSERT_EQ(kv->length, 4); + ASSERT_EQ(kv->u, 12323); + taosMemoryFree(kv); + + + // bigint + kv = taosArrayGetP(cols, 12); + ASSERT_EQ(strncasecmp(kv->key, "ci64", 4), 0); + ASSERT_EQ(kv->keyLen, 4; + ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BIGINT; + ASSERT_EQ(kv->length, 8); + ASSERT_EQ(kv->i, -89238); + taosMemoryFree(kv); + + // bigint + kv = taosArrayGetP(cols, 13); + ASSERT_EQ(strncasecmp(kv->key, "ci", 2), 0); + ASSERT_EQ(kv->keyLen, 2); + ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BIGINT); + ASSERT_EQ(kv->length, 8); + ASSERT_EQ(kv->i, 989); + taosMemoryFree(kv); + + // unsigned bigint + kv = taosArrayGetP(cols, 14); + ASSERT_EQ(strncasecmp(kv->key, "cu64", 4), 0); + ASSERT_EQ(kv->keyLen, 4); + ASSERT_EQ(kv->type, TSDB_DATA_TYPE_UBIGINT); + ASSERT_EQ(kv->length, 8); + ASSERT_EQ(kv->u, 8989323); + taosMemoryFree(kv); + + // bool + kv = taosArrayGetP(cols, 15); + ASSERT_EQ(strncasecmp(kv->key, "cbooltrue", 9), 0); + ASSERT_EQ(kv->keyLen, 9); + ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(kv->length, 1); + ASSERT_EQ(kv->i, true); + taosMemoryFree(kv); + + + // bool + kv = taosArrayGetP(cols, 16); + ASSERT_EQ(strncasecmp(kv->key, "cboolt", 6), 0); + ASSERT_EQ(kv->keyLen, 6; + ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BOOL; + ASSERT_EQ(kv->length, 8); + ASSERT_EQ(kv->i, -89238); + taosMemoryFree(kv); + + // bool + kv = taosArrayGetP(cols, 17); + ASSERT_EQ(strncasecmp(kv->key, "cboolf", 6), 0); + ASSERT_EQ(kv->keyLen, 6); + ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(kv->length, 8); + ASSERT_EQ(kv->i, 989); + taosMemoryFree(kv); + + // unsigned bigint + kv = taosArrayGetP(cols, 18); + ASSERT_EQ(strncasecmp(kv->key, "cnch_", 5), 0); + ASSERT_EQ(kv->keyLen, 5); + ASSERT_EQ(kv->type, TSDB_DATA_TYPE_NCHAR); + ASSERT_EQ(kv->length, 4); + ASSERT_EQ(strncasecmp(kv->value, "iuwq", 4), 0); + taosMemoryFree(kv); + + taosArrayDestroy(cols); +} + +TEST(testCase, smlParseLine_Test) { + TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + ASSERT_NE(taos, NULL); + + SRequestObj *request = createRequest(taos, NULL, NULL, TSDB_SQL_INSERT); + ASSERT_NE(request, NULL); + + SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS, true); + ASSERT_NE(info, NULL); + + const char *sql[3] = { + "readings,name=truck_0,fleet=South,driver=Trish,model=H-2,device_version=v2.3 load_capacity=1500,fuel_capacity=150,nominal_fuel_consumption=12,latitude=52.31854,longitude=4.72037,elevation=124,velocity=0,heading=221,grade=0,fuel_consumption=25 1451606400000000000", + "readings,name=truck_1,fleet=South,driver=Albert,model=F-150,device_version=v1.5 load_capacity=2000,fuel_capacity=200,nominal_fuel_consumption=15,latitude=72.45258,longitude=68.83761,elevation=255,velocity=0,heading=181,grade=0,fuel_consumption=25 1451606400000000000", + "readings,name=truck_2,fleet=North,driver=Derek,model=F-150,device_version=v1.5 load_capacity=2000,fuel_capacity=200,nominal_fuel_consumption=15,latitude=24.5208,longitude=28.09377,elevation=428,velocity=0,heading=304,grade=0,fuel_consumption=25 1451606400000000000" + }; + for (int i = 0; i < 3; i++) { + smlParseLine(info, sql[i]); + } +} + +// TEST(testCase, smlParseTS_Test) { +// char msg[256] = {0}; +// SSmlMsgBuf msgBuf; +// msgBuf.buf = msg; +// msgBuf.len = 256; +// SSmlLineInfo elements = {0}; +// +// SSmlHandle* info = smlBuildSmlInfo(taos, request, protocol, precision, dataFormat); +// if(!info){ +// return (TAOS_RES*)request; +// } +// ret = smlParseTS(info, elements.timestamp, elements.timestampLen, cols); +// if(ret != TSDB_CODE_SUCCESS){ +// uError("SML:0x%"PRIx64" smlParseTS failed", info->id); +// return ret; +// } +// } From 7a8f8e11a5d0727ceec022231f49f49d84d35d1e Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Thu, 5 May 2022 16:00:35 +0800 Subject: [PATCH 11/83] feat: support update in mem/file --- include/common/tdataformat.h | 6 +- include/common/trow.h | 49 ++-- include/util/tdef.h | 2 + source/common/src/trow.c | 273 ++++++++++++++------- source/common/src/ttypes.c | 2 +- source/dnode/vnode/src/inc/tsdb.h | 3 +- source/dnode/vnode/src/tsdb/tsdbCommit.c | 10 +- source/dnode/vnode/src/tsdb/tsdbMemTable.c | 7 +- source/dnode/vnode/src/tsdb/tsdbRead.c | 257 ++++++++++++++----- source/dnode/vnode/src/tsdb/tsdbReadImpl.c | 38 ++- source/dnode/vnode/src/tsdb/tsdbSma.c | 9 +- 11 files changed, 460 insertions(+), 196 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 36286d5944..0472c7ea52 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -27,7 +27,7 @@ extern "C" { // Imported since 3.0 and use bitmap to demonstrate None/Null/Norm, while use Null/Norm below 3.0 without of bitmap. #define TD_SUPPORT_BITMAP -#define TD_SUPPORT_READ2 +#undef TD_SUPPORT_READ2 #define TD_SUPPORT_BACK2 // suppport back compatibility of 2.0 #define TASSERT(x) ASSERT(x) @@ -387,8 +387,6 @@ typedef struct SDataCol { TSKEY ts; // only used in last NULL column } SDataCol; - - #define isAllRowsNull(pCol) ((pCol)->len == 0) #define isAllRowsNone(pCol) ((pCol)->len == 0) static FORCE_INLINE void dataColReset(SDataCol *pDataCol) { pDataCol->len = 0; } @@ -482,7 +480,7 @@ void tdResetDataCols(SDataCols *pCols); int32_t tdInitDataCols(SDataCols *pCols, STSchema *pSchema); SDataCols *tdDupDataCols(SDataCols *pCols, bool keepData); SDataCols *tdFreeDataCols(SDataCols *pCols); -int32_t tdMergeDataCols(SDataCols *target, SDataCols *source, int32_t rowsToMerge, int32_t *pOffset, bool forceSetNull, TDRowVerT maxVer); +int32_t tdMergeDataCols(SDataCols *target, SDataCols *source, int32_t rowsToMerge, int32_t *pOffset, bool update, TDRowVerT maxVer); // ----------------- K-V data row structure /* |<-------------------------------------- len -------------------------------------------->| diff --git a/include/common/trow.h b/include/common/trow.h index 02f84b372e..fe1cf0d967 100644 --- a/include/common/trow.h +++ b/include/common/trow.h @@ -42,7 +42,7 @@ extern "C" { * @brief value type * - for data from client input and STSRow in memory, 3 types of value none/null/norm available */ -#define TD_VTYPE_NORM 0x00U // normal val: not none, not null(no need assign value) +#define TD_VTYPE_NORM 0x00U // normal val: not none, not null #define TD_VTYPE_NULL 0x01U // null val #define TD_VTYPE_NONE 0x02U // none or unknown/undefined #define TD_VTYPE_MAX 0x03U // @@ -122,6 +122,8 @@ typedef struct { typedef struct { /// timestamp TSKEY ts; + /// row version + uint64_t ver; union { /// union field for encode and decode uint32_t info; @@ -140,8 +142,6 @@ typedef struct { }; /// row total length uint32_t len; - /// row version - uint64_t ver; /// the inline data, maybe a tuple or a k-v tuple char data[]; } STSRow; @@ -241,13 +241,13 @@ static FORCE_INLINE int32_t tdGetBitmapValType(const void *pBitmap, int16_t colI static FORCE_INLINE bool tdIsBitmapValTypeNorm(const void *pBitmap, int16_t idx, int8_t bitmapMode); bool tdIsBitmapBlkNorm(const void *pBitmap, int32_t numOfBits, int8_t bitmapMode); int32_t tdAppendValToDataCol(SDataCol *pCol, TDRowValT valType, const void *val, int32_t numOfRows, int32_t maxPoints, - int8_t bitmapMode); + int8_t bitmapMode, bool isMerge); static FORCE_INLINE int32_t tdAppendColValToTpRow(SRowBuilder *pBuilder, TDRowValT valType, const void *val, bool isCopyVarData, int8_t colType, int16_t colIdx, int32_t offset); static FORCE_INLINE int32_t tdAppendColValToKvRow(SRowBuilder *pBuilder, TDRowValT valType, const void *val, bool isCopyVarData, int8_t colType, int16_t colIdx, int32_t offset, col_id_t colId); -int32_t tdAppendSTSRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols *pCols); +int32_t tdAppendSTSRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols *pCols, bool isMerge); /** * @brief @@ -622,7 +622,6 @@ static FORCE_INLINE int32_t tdSRowSetTpInfo(SRowBuilder *pBuilder, int32_t nCols return TSDB_CODE_SUCCESS; } - /** * @brief To judge row type: STpRow/SKvRow * @@ -719,6 +718,7 @@ static int32_t tdSRowResetBuf(SRowBuilder *pBuilder, void *pBuf) { terrno = TSDB_CODE_INVALID_PARA; return terrno; } + return TSDB_CODE_SUCCESS; } @@ -758,7 +758,6 @@ static int32_t tdSRowGetBuf(SRowBuilder *pBuilder, void *pBuf) { return TSDB_CODE_SUCCESS; } - /** * @brief 由调用方管理存储空间的分配及释放，一次输入多个参数 * @@ -1250,16 +1249,16 @@ static FORCE_INLINE int32_t tdGetColDataOfRow(SCellVal *pVal, SDataCol *pCol, in } /** - * @brief - * - * @param pRow - * @param colId - * @param colType - * @param flen - * @param offset + * @brief + * + * @param pRow + * @param colId + * @param colType + * @param flen + * @param offset * @param colIdx start from 0 - * @param pVal - * @return FORCE_INLINE + * @param pVal + * @return FORCE_INLINE */ static FORCE_INLINE bool tdSTpRowGetVal(STSRow *pRow, col_id_t colId, col_type_t colType, int32_t flen, uint32_t offset, col_id_t colIdx, SCellVal *pVal) { @@ -1273,14 +1272,14 @@ static FORCE_INLINE bool tdSTpRowGetVal(STSRow *pRow, col_id_t colId, col_type_t } /** - * @brief - * - * @param pRow - * @param colId - * @param offset + * @brief + * + * @param pRow + * @param colId + * @param offset * @param colIdx start from 0 - * @param pVal - * @return FORCE_INLINE + * @param pVal + * @return FORCE_INLINE */ static FORCE_INLINE bool tdSKvRowGetVal(STSRow *pRow, col_id_t colId, uint32_t offset, col_id_t colIdx, SCellVal *pVal) { @@ -1397,14 +1396,14 @@ static void tdSCellValPrint(SCellVal *pVal, int8_t colType) { } } -static void tdSRowPrint(STSRow *row, STSchema *pSchema, const char* tag) { +static void tdSRowPrint(STSRow *row, STSchema *pSchema, const char *tag) { STSRowIter iter = {0}; tdSTSRowIterInit(&iter, pSchema); tdSTSRowIterReset(&iter, row); printf("%s >>>", tag); for (int i = 0; i < pSchema->numOfCols; ++i) { STColumn *stCol = pSchema->columns + i; - SCellVal sVal = { 255, NULL}; + SCellVal sVal = {255, NULL}; if (!tdSTSRowIterNext(&iter, stCol->colId, stCol->type, &sVal)) { break; } diff --git a/include/util/tdef.h b/include/util/tdef.h index 316640797b..0ab277e0c3 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -29,6 +29,8 @@ extern "C" { #define TSKEY_MAX (INT64_MAX - 1) #define TSKEY_INITIAL_VAL TSKEY_MIN +#define TD_VER_MAX UINT64_MAX // TODO: use the real max version from query handle + // Bytes for each type. extern const int32_t TYPE_BYTES[15]; diff --git a/source/common/src/trow.c b/source/common/src/trow.c index 7157c1e0f0..9f4f053028 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -33,6 +33,24 @@ const uint8_t tdVTypeByte[2][3] = {{ // declaration static uint8_t tdGetBitmapByte(uint8_t byte); +// static void dataColSetNEleNull(SDataCol *pCol, int nEle); + +/** + * @brief src2 data has more priority than src1 + * + * @param target + * @param src1 + * @param iter1 + * @param limit1 + * @param src2 + * @param iter2 + * @param limit2 + * @param tRows + * @param update + */ +static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limit1, SDataCols *src2, int *iter2, + int limit2, int tRows, bool update); + // implementation /** * @brief Compress bitmap bytes comprised of 2-bits to counterpart of 1-bit. @@ -229,23 +247,23 @@ static uint8_t tdGetMergedBitmapByte(uint8_t byte) { void tdMergeBitmap(uint8_t *srcBitmap, int32_t nBits, uint8_t *dstBitmap) { int32_t i = 0, j = 0; int32_t nBytes = TD_BITMAP_BYTES(nBits); - int32_t nStrictBytes = nBits / 4; - int32_t nPartialBits = nBits - nStrictBytes * 4; + int32_t nRoundBytes = nBits / 4; + int32_t nRemainderBits = nBits - nRoundBytes * 4; - switch (nPartialBits) { + switch (nRemainderBits) { case 0: // NOTHING TODO break; case 1: { - void *lastByte = POINTER_SHIFT(srcBitmap, nStrictBytes); + void *lastByte = POINTER_SHIFT(srcBitmap, nRoundBytes); *(uint8_t *)lastByte &= 0xC0; } break; case 2: { - void *lastByte = POINTER_SHIFT(srcBitmap, nStrictBytes); + void *lastByte = POINTER_SHIFT(srcBitmap, nRoundBytes); *(uint8_t *)lastByte &= 0xF0; } break; case 3: { - void *lastByte = POINTER_SHIFT(srcBitmap, nStrictBytes); + void *lastByte = POINTER_SHIFT(srcBitmap, nRoundBytes); *(uint8_t *)lastByte &= 0xFC; } break; default: @@ -266,10 +284,6 @@ void tdMergeBitmap(uint8_t *srcBitmap, int32_t nBits, uint8_t *dstBitmap) { } } -// static void dataColSetNEleNull(SDataCol *pCol, int nEle); -static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limit1, SDataCols *src2, int *iter2, - int limit2, int tRows, bool forceSetNull); - static FORCE_INLINE void dataColSetNullAt(SDataCol *pCol, int index, bool setBitmap, int8_t bitmapMode) { if (IS_VAR_DATA_TYPE(pCol->type)) { pCol->dataOff[index] = pCol->len; @@ -410,11 +424,12 @@ STSRow *tdRowDup(STSRow *row) { * @param val * @param numOfRows * @param maxPoints - * @param bitmapMode default is 0(2 bits), otherwise 1(1 bit) + * @param bitmapMode default is 0(2 bits), otherwise 1(1 bit) + * @param isMerge merge to current row * @return int */ int tdAppendValToDataCol(SDataCol *pCol, TDRowValT valType, const void *val, int numOfRows, int maxPoints, - int8_t bitmapMode) { + int8_t bitmapMode, bool isMerge) { TASSERT(pCol != NULL); // Assume that the columns not specified during insert/upsert mean None. @@ -436,18 +451,35 @@ int tdAppendValToDataCol(SDataCol *pCol, TDRowValT valType, const void *val, int // 2. later on, considering further optimization, don't save Null/None for VarType. val = getNullValue(pCol->type); } - if (IS_VAR_DATA_TYPE(pCol->type)) { - // set offset - pCol->dataOff[numOfRows] = pCol->len; - // Copy data - memcpy(POINTER_SHIFT(pCol->pData, pCol->len), val, varDataTLen(val)); - // Update the length - pCol->len += varDataTLen(val); + if (!isMerge) { + if (IS_VAR_DATA_TYPE(pCol->type)) { + // set offset + pCol->dataOff[numOfRows] = pCol->len; + // Copy data + memcpy(POINTER_SHIFT(pCol->pData, pCol->len), val, varDataTLen(val)); + // Update the length + pCol->len += varDataTLen(val); + } else { + ASSERT(pCol->len == TYPE_BYTES[pCol->type] * numOfRows); + memcpy(POINTER_SHIFT(pCol->pData, pCol->len), val, pCol->bytes); + pCol->len += pCol->bytes; + } } else { - ASSERT(pCol->len == TYPE_BYTES[pCol->type] * numOfRows); - memcpy(POINTER_SHIFT(pCol->pData, pCol->len), val, pCol->bytes); - pCol->len += pCol->bytes; + if (IS_VAR_DATA_TYPE(pCol->type)) { + // keep the last offset + // discard the last var data + int32_t lastVarLen = varDataTLen(POINTER_SHIFT(pCol->pData, pCol->dataOff[numOfRows])); + pCol->len -= lastVarLen; + // Copy data + memcpy(POINTER_SHIFT(pCol->pData, pCol->len), val, varDataTLen(val)); + // Update the length + pCol->len += varDataTLen(val); + } else { + ASSERT(pCol->len - TYPE_BYTES[pCol->type] == TYPE_BYTES[pCol->type] * numOfRows); + memcpy(POINTER_SHIFT(pCol->pData, pCol->len - TYPE_BYTES[pCol->type]), val, pCol->bytes); + } } + #ifdef TD_SUPPORT_BITMAP tdSetBitmapValType(pCol->pBitmap, numOfRows, valType, bitmapMode); #endif @@ -455,8 +487,13 @@ int tdAppendValToDataCol(SDataCol *pCol, TDRowValT valType, const void *val, int } // internal -static int32_t tdAppendTpRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols *pCols) { +static int32_t tdAppendTpRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols *pCols, bool isMerge) { +#if 0 ASSERT(pCols->numOfRows == 0 || dataColsKeyLast(pCols) < TD_ROW_KEY(pRow)); +#endif + + // Multi-Version rows with the same key and different versions supported + ASSERT(pCols->numOfRows == 0 || dataColsKeyLast(pCols) <= TD_ROW_KEY(pRow)); int rcol = 1; int dcol = 1; @@ -464,12 +501,14 @@ static int32_t tdAppendTpRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols SDataCol *pDataCol = &(pCols->cols[0]); ASSERT(pDataCol->colId == PRIMARYKEY_TIMESTAMP_COL_ID); - tdAppendValToDataCol(pDataCol, TD_VTYPE_NORM, &pRow->ts, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode); + tdAppendValToDataCol(pDataCol, TD_VTYPE_NORM, &pRow->ts, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode, + isMerge); while (dcol < pCols->numOfCols) { pDataCol = &(pCols->cols[dcol]); if (rcol >= schemaNCols(pSchema)) { - tdAppendValToDataCol(pDataCol, TD_VTYPE_NULL, NULL, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode); + tdAppendValToDataCol(pDataCol, TD_VTYPE_NULL, NULL, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode, + isMerge); ++dcol; continue; } @@ -480,13 +519,15 @@ static int32_t tdAppendTpRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols if (tdGetTpRowValOfCol(&sVal, pRow, pBitmap, pRowCol->type, pRowCol->offset - sizeof(TSKEY), rcol - 1) < 0) { return terrno; } - tdAppendValToDataCol(pDataCol, sVal.valType, sVal.val, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode); + tdAppendValToDataCol(pDataCol, sVal.valType, sVal.val, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode, + isMerge); ++dcol; ++rcol; } else if (pRowCol->colId < pDataCol->colId) { ++rcol; } else { - tdAppendValToDataCol(pDataCol, TD_VTYPE_NULL, NULL, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode); + tdAppendValToDataCol(pDataCol, TD_VTYPE_NULL, NULL, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode, + isMerge); ++dcol; } } @@ -495,7 +536,7 @@ static int32_t tdAppendTpRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols return TSDB_CODE_SUCCESS; } // internal -static int32_t tdAppendKvRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols *pCols) { +static int32_t tdAppendKvRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols *pCols, bool isMerge) { ASSERT(pCols->numOfRows == 0 || dataColsKeyLast(pCols) < TD_ROW_KEY(pRow)); int rcol = 0; @@ -506,12 +547,14 @@ static int32_t tdAppendKvRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols SDataCol *pDataCol = &(pCols->cols[0]); ASSERT(pDataCol->colId == PRIMARYKEY_TIMESTAMP_COL_ID); - tdAppendValToDataCol(pDataCol, TD_VTYPE_NORM, &pRow->ts, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode); + tdAppendValToDataCol(pDataCol, TD_VTYPE_NORM, &pRow->ts, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode, + isMerge); while (dcol < pCols->numOfCols) { pDataCol = &(pCols->cols[dcol]); if (rcol >= tRowCols || rcol >= tSchemaCols) { - tdAppendValToDataCol(pDataCol, TD_VTYPE_NULL, NULL, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode); + tdAppendValToDataCol(pDataCol, TD_VTYPE_NULL, NULL, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode, + isMerge); ++dcol; continue; } @@ -527,13 +570,15 @@ static int32_t tdAppendKvRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols if (tdGetKvRowValOfCol(&sVal, pRow, pBitmap, pIdx->offset, colIdx) < 0) { return terrno; } - tdAppendValToDataCol(pDataCol, sVal.valType, sVal.val, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode); + tdAppendValToDataCol(pDataCol, sVal.valType, sVal.val, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode, + isMerge); ++dcol; ++rcol; } else if (pIdx->colId < pDataCol->colId) { ++rcol; } else { - tdAppendValToDataCol(pDataCol, TD_VTYPE_NULL, NULL, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode); + tdAppendValToDataCol(pDataCol, TD_VTYPE_NULL, NULL, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode, + isMerge); ++dcol; } } @@ -548,20 +593,30 @@ static int32_t tdAppendKvRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols * @param pRow * @param pSchema * @param pCols - * @param forceSetNull */ -int32_t tdAppendSTSRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols *pCols) { +int32_t tdAppendSTSRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols *pCols, bool isMerge) { if (TD_IS_TP_ROW(pRow)) { - return tdAppendTpRowToDataCol(pRow, pSchema, pCols); + return tdAppendTpRowToDataCol(pRow, pSchema, pCols, isMerge); } else if (TD_IS_KV_ROW(pRow)) { - return tdAppendKvRowToDataCol(pRow, pSchema, pCols); + return tdAppendKvRowToDataCol(pRow, pSchema, pCols, isMerge); } else { ASSERT(0); } return TSDB_CODE_SUCCESS; } -int tdMergeDataCols(SDataCols *target, SDataCols *source, int rowsToMerge, int *pOffset, bool forceSetNull, +/** + * @brief source data has more priority than target + * + * @param target + * @param source + * @param rowsToMerge + * @param pOffset + * @param update + * @param maxVer + * @return int + */ +int tdMergeDataCols(SDataCols *target, SDataCols *source, int rowsToMerge, int *pOffset, bool update, TDRowVerT maxVer) { ASSERT(rowsToMerge > 0 && rowsToMerge <= source->numOfRows); ASSERT(target->numOfCols == source->numOfCols); @@ -576,17 +631,35 @@ int tdMergeDataCols(SDataCols *target, SDataCols *source, int rowsToMerge, int * if ((target->numOfRows == 0) || (dataColsKeyLast(target) < dataColsKeyAtRow(source, *pOffset))) { // No overlap ASSERT(target->numOfRows + rowsToMerge <= target->maxPoints); // TODO: filter the maxVer - for (int i = 0; i < rowsToMerge; i++) { + TSKEY lastKey = TSKEY_INITIAL_VAL; + for (int i = 0; i < rowsToMerge; ++i) { + bool merge = false; for (int j = 0; j < source->numOfCols; j++) { if (source->cols[j].len > 0 || target->cols[j].len > 0) { SCellVal sVal = {0}; if (tdGetColDataOfRow(&sVal, source->cols + j, i + (*pOffset), source->bitmapMode) < 0) { TASSERT(0); } + + if (j == 0) { + if (lastKey == *(TSKEY *)sVal.val) { + if (!update) { + break; + } + merge = true; + } else if (lastKey != TSKEY_INITIAL_VAL) { + ++target->numOfRows; + } + + lastKey = *(TSKEY *)sVal.val; + } + tdAppendValToDataCol(target->cols + j, sVal.valType, sVal.val, target->numOfRows, target->maxPoints, - target->bitmapMode); + target->bitmapMode, merge); } } + } + if (rowsToMerge > 0) { ++target->numOfRows; } (*pOffset) += rowsToMerge; @@ -596,7 +669,7 @@ int tdMergeDataCols(SDataCols *target, SDataCols *source, int rowsToMerge, int * int iter1 = 0; tdMergeTwoDataCols(target, pTarget, &iter1, pTarget->numOfRows, source, pOffset, source->numOfRows, - pTarget->numOfRows + rowsToMerge, forceSetNull); + pTarget->numOfRows + rowsToMerge, update); } tdFreeDataCols(pTarget); @@ -606,68 +679,95 @@ _err: tdFreeDataCols(pTarget); return -1; } - -// src2 data has more priority than src1 +static void tdAppendValToDataCols(SDataCols *target, SDataCols *src, int iter, bool isMerge) { + for (int i = 0; i < src->numOfCols; ++i) { + ASSERT(target->cols[i].type == src->cols[i].type); + if (src->cols[i].len > 0 || target->cols[i].len > 0) { + SCellVal sVal = {0}; + if (tdGetColDataOfRow(&sVal, src->cols + i, iter, src->bitmapMode) < 0) { + TASSERT(0); + } + if (isMerge) { + if (!tdValTypeIsNone(sVal.valType)) { + tdAppendValToDataCol(&(target->cols[i]), sVal.valType, sVal.val, target->numOfRows, target->maxPoints, + target->bitmapMode, isMerge); + } else { + // Keep the origi value for None + } + } else { + tdAppendValToDataCol(&(target->cols[i]), sVal.valType, sVal.val, target->numOfRows, target->maxPoints, + target->bitmapMode, isMerge); + } + } + } +} +/** + * @brief src2 data has more priority than src1 + * + * @param target + * @param src1 + * @param iter1 + * @param limit1 + * @param src2 + * @param iter2 + * @param limit2 + * @param tRows + * @param update + */ static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limit1, SDataCols *src2, int *iter2, - int limit2, int tRows, bool forceSetNull) { + int limit2, int tRows, bool update) { tdResetDataCols(target); + target->bitmapMode = src1->bitmapMode; ASSERT(limit1 <= src1->numOfRows && limit2 <= src2->numOfRows); + int32_t nRows = 0; - while (target->numOfRows < tRows) { + // TODO: filter the maxVer + // TODO: handle the delete function + TSKEY lastKey = TSKEY_INITIAL_VAL; + while (nRows < tRows) { if (*iter1 >= limit1 && *iter2 >= limit2) break; TSKEY key1 = (*iter1 >= limit1) ? INT64_MAX : dataColsKeyAt(src1, *iter1); - TKEY tkey1 = (*iter1 >= limit1) ? TKEY_NULL : dataColsTKeyAt(src1, *iter1); + // TKEY tkey1 = (*iter1 >= limit1) ? TKEY_NULL : dataColsTKeyAt(src1, *iter1); TSKEY key2 = (*iter2 >= limit2) ? INT64_MAX : dataColsKeyAt(src2, *iter2); // TKEY tkey2 = (*iter2 >= limit2) ? TKEY_NULL : dataColsTKeyAt(src2, *iter2); - ASSERT(tkey1 == TKEY_NULL || (!TKEY_IS_DELETED(tkey1))); - // TODO: filter the maxVer - if (key1 < key2) { - for (int i = 0; i < src1->numOfCols; ++i) { - ASSERT(target->cols[i].type == src1->cols[i].type); - if (src1->cols[i].len > 0 || target->cols[i].len > 0) { - SCellVal sVal = {0}; - if (tdGetColDataOfRow(&sVal, src1->cols + i, *iter1, src1->bitmapMode) < 0) { - TASSERT(0); - } - tdAppendValToDataCol(&(target->cols[i]), sVal.valType, sVal.val, target->numOfRows, target->maxPoints, - target->bitmapMode); - } - } + // ASSERT(tkey1 == TKEY_NULL || (!TKEY_IS_DELETED(tkey1))); - ++target->numOfRows; + if (key1 <= key2) { + // select key1 if not delete + if (update && (lastKey == key1)) { + tdAppendValToDataCols(target, src1, *iter1, true); + } else if (lastKey != key1) { + if (lastKey != TSKEY_INITIAL_VAL) { + ++target->numOfRows; + } + tdAppendValToDataCols(target, src1, *iter1, false); + } + ++nRows; ++(*iter1); - } else if (key1 >= key2) { - // TODO: filter the maxVer - if ((key1 > key2) || ((key1 == key2) && !TKEY_IS_DELETED(key2))) { - for (int i = 0; i < src2->numOfCols; ++i) { - SCellVal sVal = {0}; - ASSERT(target->cols[i].type == src2->cols[i].type); - if (tdGetColDataOfRow(&sVal, src2->cols + i, *iter2, src2->bitmapMode) < 0) { - TASSERT(0); - } - if (src2->cols[i].len > 0 && !tdValTypeIsNull(sVal.valType)) { - tdAppendValToDataCol(&(target->cols[i]), sVal.valType, sVal.val, target->numOfRows, target->maxPoints, - target->bitmapMode); - } else if (!forceSetNull && key1 == key2 && src1->cols[i].len > 0) { - if (tdGetColDataOfRow(&sVal, src1->cols + i, *iter1, src1->bitmapMode) < 0) { - TASSERT(0); - } - tdAppendValToDataCol(&(target->cols[i]), sVal.valType, sVal.val, target->numOfRows, target->maxPoints, - target->bitmapMode); - } else if (target->cols[i].len > 0) { - dataColSetNullAt(&target->cols[i], target->numOfRows, true, target->bitmapMode); - } + lastKey = key1; + } else { + // use key2 if not deleted + // TODO: handle the delete function + if (update && (lastKey == key2)) { + tdAppendValToDataCols(target, src2, *iter2, true); + } else if (lastKey != key2) { + if (lastKey != TSKEY_INITIAL_VAL) { + ++target->numOfRows; } - ++target->numOfRows; + tdAppendValToDataCols(target, src2, *iter2, false); } + ++nRows; ++(*iter2); - if (key1 == key2) ++(*iter1); + lastKey = key2; } - ASSERT(target->numOfRows <= target->maxPoints); + ASSERT(target->numOfRows <= target->maxPoints - 1); + } + if (nRows > 0) { + ++target->numOfRows; } } @@ -777,7 +877,7 @@ SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) { pRet->sversion = pDataCols->sversion; if (keepData) pRet->numOfRows = pDataCols->numOfRows; - for (int i = 0; i < pDataCols->numOfCols; i++) { + for (int i = 0; i < pDataCols->numOfCols; ++i) { pRet->cols[i].type = pDataCols->cols[i].type; pRet->cols[i].bitmap = pDataCols->cols[i].bitmap; pRet->cols[i].colId = pDataCols->cols[i].colId; @@ -797,8 +897,7 @@ SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) { memcpy(pRet->cols[i].dataOff, pDataCols->cols[i].dataOff, dataOffSize); } if (!TD_COL_ROWS_NORM(pRet->cols + i)) { - int32_t nBitmapBytes = (int32_t)TD_BITMAP_BYTES(pDataCols->numOfRows); - memcpy(pRet->cols[i].pBitmap, pDataCols->cols[i].pBitmap, nBitmapBytes); + memcpy(pRet->cols[i].pBitmap, pDataCols->cols[i].pBitmap, TD_BITMAP_BYTES(pDataCols->numOfRows)); } } } diff --git a/source/common/src/ttypes.c b/source/common/src/ttypes.c index e3d67cd488..3fd2ef8e73 100644 --- a/source/common/src/ttypes.c +++ b/source/common/src/ttypes.c @@ -557,7 +557,7 @@ static const void *nullValues[] = { }; const void *getNullValue(int32_t type) { - assert(type >= TSDB_DATA_TYPE_BOOL && type <= TSDB_DATA_TYPE_UBIGINT); + assert(type >= TSDB_DATA_TYPE_BOOL && type <= TSDB_DATA_TYPE_UBIGINT); // TODO: extend the types return nullValues[type - 1]; } diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 078a493773..ffa6f81c0d 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -274,7 +274,8 @@ typedef enum { typedef struct { uint8_t last : 1; - uint8_t blkVer : 7; + uint8_t hasDupKey : 1; // 0: no dup TS key, 1: has dup TS key(since supporting Multi-Version) + uint8_t blkVer : 6; uint8_t numOfSubBlocks; col_id_t numOfCols; // not including timestamp column uint32_t len; // data block length diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index fc23534247..754023c49d 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -1360,7 +1360,7 @@ static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIt TASSERT(0); } tdAppendValToDataCol(pTarget->cols + i, sVal.valType, sVal.val, pTarget->numOfRows, pTarget->maxPoints, - pTarget->bitmapMode); + pTarget->bitmapMode, false); } ++pTarget->numOfRows; @@ -1371,7 +1371,7 @@ static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIt ASSERT(pSchema != NULL); } - tdAppendSTSRowToDataCol(row, pSchema, pTarget); + tdAppendSTSRowToDataCol(row, pSchema, pTarget, false); tSkipListIterNext(pCommitIter->pIter); } else { @@ -1409,7 +1409,7 @@ static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIt } // TODO: tdAppendValToDataCol may fail tdAppendValToDataCol(pTarget->cols + i, sVal.valType, sVal.val, pTarget->numOfRows, pTarget->maxPoints, - pTarget->bitmapMode); + pTarget->bitmapMode, false); } if (TD_SUPPORT_UPDATE(update)) { @@ -1427,9 +1427,9 @@ static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIt STSRow *nextRow = tsdbNextIterRow(pCommitIter->pIter); if (key2 < TD_ROW_KEY(nextRow)) { - tdAppendSTSRowToDataCol(row, pSchema, pTarget); + tdAppendSTSRowToDataCol(row, pSchema, pTarget, false); } else { - tdAppendSTSRowToDataCol(row, pSchema, pTarget); + tdAppendSTSRowToDataCol(row, pSchema, pTarget, false); } // TODO: merge with Multi-Version } else { diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index ff4d99f510..ebe5ce9a8f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -256,9 +256,12 @@ static STbData *tsdbNewTbData(tb_uid_t uid) { pTbData->keyMin = TSKEY_MAX; pTbData->keyMax = TSKEY_MIN; pTbData->nrows = 0; - +#if 0 pTbData->pData = tSkipListCreate(5, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), tkeyComparFn, SL_DISCARD_DUP_KEY, tsdbGetTsTupleKey); +#endif + pTbData->pData = tSkipListCreate(5, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), tkeyComparFn, SL_ALLOW_DUP_KEY, + tsdbGetTsTupleKey); if (pTbData->pData == NULL) { taosMemoryFree(pTbData); return NULL; @@ -303,7 +306,7 @@ static int tsdbAppendTableRowToCols(STable *pTable, SDataCols *pCols, STSchema * } } - tdAppendSTSRowToDataCol(row, *ppSchema, pCols); + tdAppendSTSRowToDataCol(row, *ppSchema, pCols, false); } return 0; diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index b6dcc52b5e..72fada4e97 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -163,7 +163,7 @@ static int32_t tsdbCheckInfoCompar(const void* key1, const void* key2); // static void* destroyTableCheckInfo(SArray* pTableCheckInfo); static bool tsdbGetExternalRow(tsdbReaderT pHandle); -static STsdb* getTsdbByRetentions(SVnode* pVnode, TSKEY winSKey, SRetention* retentions); +static STsdb* getTsdbByRetentions(SVnode* pVnode, STsdbReadHandle* pReadHandle, TSKEY winSKey, SRetention* retentions); static void tsdbInitDataBlockLoadInfo(SDataBlockLoadInfo* pBlockLoadInfo) { pBlockLoadInfo->slot = -1; @@ -351,36 +351,43 @@ static void setQueryTimewindow(STsdbReadHandle* pTsdbReadHandle, SQueryTableData pTsdbReadHandle->window.ekey, pTsdbReadHandle->idStr); } } -#if 0 -int nQUERY = 0; +#if 1 +int nQUERY = 0; #endif -static STsdb* getTsdbByRetentions(SVnode* pVnode, TSKEY winSKey, SRetention* retentions) { +static STsdb* getTsdbByRetentions(SVnode* pVnode, STsdbReadHandle* pReadHandle, TSKEY winSKey, SRetention* retentions) { if (vnodeIsRollup(pVnode)) { int level = 0; -#if 1 +#if 0 int64_t now = taosGetTimestamp(pVnode->config.tsdbCfg.precision); for (int i = 0; i < TSDB_RETENTION_MAX; ++i) { SRetention* pRetention = retentions + i; if (pRetention->keep <= 0 || (now - pRetention->keep) >= winSKey) { break; } + ++level; } #endif -#if 0 - ++nQUERY; - if(nQUERY%3 == 0) { - level = 2; - } else if(nQUERY%2 == 0) { - level = 1; - } else { - level = 0; +#if 1 + switch ((nQUERY++) % 3) { + case 0: + level = 0; + break; + case 1: + level = 1; + break; + default: + level = 2; + break; } #endif if (level == TSDB_RETENTION_L0) { + tsdbDebug("%p rsma level %d is selected to query\n", pReadHandle, level); return VND_RSMA0(pVnode); } else if (level == TSDB_RETENTION_L1) { + tsdbDebug("%p rsma level %d is selected to query\n", pReadHandle, level); return VND_RSMA1(pVnode); } else { + tsdbDebug("%p rsma level %d is selected to query\n", pReadHandle, level); return VND_RSMA2(pVnode); } } @@ -393,7 +400,7 @@ static STsdbReadHandle* tsdbQueryTablesImpl(SVnode* pVnode, SQueryTableDataCond* goto _end; } - STsdb* pTsdb = getTsdbByRetentions(pVnode, pCond->twindow.skey, pVnode->config.tsdbCfg.retentions); + STsdb* pTsdb = getTsdbByRetentions(pVnode, pReadHandle, pCond->twindow.skey, pVnode->config.tsdbCfg.retentions); pReadHandle->order = pCond->order; pReadHandle->pTsdb = pTsdb; @@ -803,12 +810,15 @@ static void destroyTableMemIterator(STableCheckInfo* pCheckInfo) { tSkipListDestroyIter(pCheckInfo->iiter); } -static TSKEY extractFirstTraverseKey(STableCheckInfo* pCheckInfo, int32_t order, int32_t update) { +static TSKEY extractFirstTraverseKey(STableCheckInfo* pCheckInfo, int32_t order, int32_t update, TDRowVerT maxVer) { STSRow *rmem = NULL, *rimem = NULL; if (pCheckInfo->iter) { SSkipListNode* node = tSkipListIterGet(pCheckInfo->iter); if (node != NULL) { rmem = (STSRow*)SL_GET_NODE_DATA(node); + if (TD_ROW_KEY(rmem) > maxVer) { + rmem = NULL; + } } } @@ -816,6 +826,9 @@ static TSKEY extractFirstTraverseKey(STableCheckInfo* pCheckInfo, int32_t order, SSkipListNode* node = tSkipListIterGet(pCheckInfo->iiter); if (node != NULL) { rimem = (STSRow*)SL_GET_NODE_DATA(node); + if (TD_ROW_KEY(rimem) > maxVer) { + rimem = NULL; + } } } @@ -837,6 +850,7 @@ static TSKEY extractFirstTraverseKey(STableCheckInfo* pCheckInfo, int32_t order, TSKEY r2 = TD_ROW_KEY(rimem); if (r1 == r2) { +#if 0 if (update == TD_ROW_DISCARD_UPDATE) { pCheckInfo->chosen = CHECKINFO_CHOSEN_IMEM; tSkipListIterNext(pCheckInfo->iter); @@ -846,6 +860,13 @@ static TSKEY extractFirstTraverseKey(STableCheckInfo* pCheckInfo, int32_t order, } else { pCheckInfo->chosen = CHECKINFO_CHOSEN_BOTH; } +#endif + if (TD_SUPPORT_UPDATE(update)) { + pCheckInfo->chosen = CHECKINFO_CHOSEN_BOTH; + } else { + pCheckInfo->chosen = CHECKINFO_CHOSEN_IMEM; + tSkipListIterNext(pCheckInfo->iter); + } return r1; } else if (r1 < r2 && ASCENDING_TRAVERSE(order)) { pCheckInfo->chosen = CHECKINFO_CHOSEN_MEM; @@ -856,12 +877,16 @@ static TSKEY extractFirstTraverseKey(STableCheckInfo* pCheckInfo, int32_t order, } } -static STSRow* getSRowInTableMem(STableCheckInfo* pCheckInfo, int32_t order, int32_t update, STSRow** extraRow) { + +static STSRow* getSRowInTableMem(STableCheckInfo* pCheckInfo, int32_t order, int32_t update, STSRow** extraRow, TDRowVerT maxVer) { STSRow *rmem = NULL, *rimem = NULL; if (pCheckInfo->iter) { SSkipListNode* node = tSkipListIterGet(pCheckInfo->iter); if (node != NULL) { rmem = (STSRow*)SL_GET_NODE_DATA(node); + if (TD_ROW_VER(rmem) > maxVer) { + rmem = NULL; + } } } @@ -869,6 +894,9 @@ static STSRow* getSRowInTableMem(STableCheckInfo* pCheckInfo, int32_t order, int SSkipListNode* node = tSkipListIterGet(pCheckInfo->iiter); if (node != NULL) { rimem = (STSRow*)SL_GET_NODE_DATA(node); + if (TD_ROW_VER(rimem) > maxVer) { + rimem = NULL; + } } } @@ -890,6 +918,7 @@ static STSRow* getSRowInTableMem(STableCheckInfo* pCheckInfo, int32_t order, int TSKEY r2 = TD_ROW_KEY(rimem); if (r1 == r2) { +#if 0 if (update == TD_ROW_DISCARD_UPDATE) { tSkipListIterNext(pCheckInfo->iter); pCheckInfo->chosen = CHECKINFO_CHOSEN_IMEM; @@ -903,6 +932,16 @@ static STSRow* getSRowInTableMem(STableCheckInfo* pCheckInfo, int32_t order, int *extraRow = rimem; return rmem; } +#endif + if (TD_SUPPORT_UPDATE(update)) { + pCheckInfo->chosen = CHECKINFO_CHOSEN_BOTH; + *extraRow = rimem; + return rmem; + } else { + tSkipListIterNext(pCheckInfo->iter); + pCheckInfo->chosen = CHECKINFO_CHOSEN_IMEM; + return rimem; + } } else { if (ASCENDING_TRAVERSE(order)) { if (r1 < r2) { @@ -973,7 +1012,7 @@ static bool hasMoreDataInCache(STsdbReadHandle* pHandle) { initTableMemIterator(pHandle, pCheckInfo); } - STSRow* row = getSRowInTableMem(pCheckInfo, pHandle->order, pCfg->update, NULL); + STSRow* row = getSRowInTableMem(pCheckInfo, pHandle->order, pCfg->update, NULL, TD_VER_MAX); if (row == NULL) { return false; } @@ -1250,7 +1289,7 @@ static int32_t handleDataMergeIfNeeded(STsdbReadHandle* pTsdbReadHandle, SBlock* /*bool hasData = */ initTableMemIterator(pTsdbReadHandle, pCheckInfo); assert(cur->pos >= 0 && cur->pos <= binfo.rows); - key = extractFirstTraverseKey(pCheckInfo, pTsdbReadHandle->order, pCfg->update); + key = extractFirstTraverseKey(pCheckInfo, pTsdbReadHandle->order, pCfg->update, TD_VER_MAX); if (key != TSKEY_INITIAL_VAL) { tsdbDebug("%p key in mem:%" PRId64 ", %s", pTsdbReadHandle, key, pTsdbReadHandle->idStr); @@ -1497,10 +1536,10 @@ static int32_t doCopyRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, int32_t TASSERT(0); } - if (sVal.valType == TD_VTYPE_NULL) { - colDataAppendNULL(pColInfo, rowIndex); - } else { + if (sVal.valType == TD_VTYPE_NORM) { colDataAppend(pColInfo, rowIndex, sVal.val, false); + } else { + colDataAppendNULL(pColInfo, rowIndex); } } } else { // handle the var-string @@ -1513,10 +1552,10 @@ static int32_t doCopyRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, int32_t TASSERT(0); } - if (sVal.valType == TD_VTYPE_NULL) { - colDataAppendNULL(pColInfo, rowIndex); - } else { + if (sVal.valType == TD_VTYPE_NORM) { colDataAppend(pColInfo, rowIndex, sVal.val, false); + } else { + colDataAppendNULL(pColInfo, rowIndex); } } } @@ -1541,11 +1580,26 @@ static int32_t doCopyRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, int32_t return numOfRows + num; } -// TODO fix bug for reverse copy data problem -// Note: row1 always has high priority -static void mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capacity, int32_t numOfRows, STSRow* row1, - STSRow* row2, int32_t numOfCols, uint64_t uid, STSchema* pSchema1, STSchema* pSchema2, - bool forceSetNull) { +/** + * @brief // TODO fix bug for reverse copy data problem + * Note: row1 always has high priority + * + * @param pTsdbReadHandle + * @param capacity + * @param curRow + * @param row1 + * @param row2 + * @param numOfCols + * @param uid + * @param pSchema1 + * @param pSchema2 + * @param update + * @param lastRowKey + * @return int32_t The quantity of rows appended + */ +static int32_t mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capacity, int32_t* curRow, STSRow* row1, + STSRow* row2, int32_t numOfCols, uint64_t uid, STSchema* pSchema1, STSchema* pSchema2, + bool update, TSKEY* lastRowKey) { #if 1 STSchema* pSchema; STSRow* row; @@ -1557,12 +1611,16 @@ static void mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capacit bool isChosenRowDataRow; int32_t chosen_itr; SCellVal sVal = {0}; + TSKEY rowKey = TSKEY_INITIAL_VAL; + int32_t nResult = 0; // the schema version info is embeded in STSRow int32_t numOfColsOfRow1 = 0; if (pSchema1 == NULL) { - pSchema1 = metaGetTbTSchema(REPO_META(pTsdbReadHandle->pTsdb), uid, TD_ROW_SVER(row1)); + // pSchema1 = metaGetTbTSchema(REPO_META(pTsdbReadHandle->pTsdb), uid, TD_ROW_SVER(row1)); + // TODO: use the real schemaVersion + pSchema1 = metaGetTbTSchema(REPO_META(pTsdbReadHandle->pTsdb), uid, 0); } #ifdef TD_DEBUG_PRINT_ROW @@ -1579,7 +1637,9 @@ static void mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capacit if (row2) { isRow2DataRow = TD_IS_TP_ROW(row2); if (pSchema2 == NULL) { - pSchema2 = metaGetTbTSchema(REPO_META(pTsdbReadHandle->pTsdb), uid, TD_ROW_SVER(row2)); + // pSchema2 = metaGetTbTSchema(REPO_META(pTsdbReadHandle->pTsdb), uid, TD_ROW_SVER(row2)); + // TODO: use the real schemaVersion + pSchema2 = metaGetTbTSchema(REPO_META(pTsdbReadHandle->pTsdb), uid, 0); } if (isRow2DataRow) { numOfColsOfRow2 = schemaNCols(pSchema2); @@ -1610,19 +1670,19 @@ static void mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capacit colIdOfRow2 = tdKvRowColIdAt(row2, k); } - if (colIdOfRow1 == colIdOfRow2) { + if (colIdOfRow1 < colIdOfRow2) { // the most probability if (colIdOfRow1 < pColInfo->info.colId) { - j++; - k++; + ++j; continue; } row = row1; pSchema = pSchema1; isChosenRowDataRow = isRow1DataRow; chosen_itr = j; - } else if (colIdOfRow1 < colIdOfRow2) { + } else if (colIdOfRow1 == colIdOfRow2) { if (colIdOfRow1 < pColInfo->info.colId) { - j++; + ++j; + ++k; continue; } row = row1; @@ -1631,7 +1691,7 @@ static void mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capacit chosen_itr = j; } else { if (colIdOfRow2 < pColInfo->info.colId) { - k++; + ++k; continue; } row = row2; @@ -1639,16 +1699,35 @@ static void mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capacit chosen_itr = k; isChosenRowDataRow = isRow2DataRow; } + if (isChosenRowDataRow) { colId = pSchema->columns[chosen_itr].colId; offset = pSchema->columns[chosen_itr].offset; // TODO: use STSRowIter tdSTpRowGetVal(row, colId, pSchema->columns[chosen_itr].type, pSchema->flen, offset, chosen_itr - 1, &sVal); + if (colId == PRIMARYKEY_TIMESTAMP_COL_ID) { + rowKey = *(TSKEY*)sVal.val; + if (rowKey != *lastRowKey) { + if (*lastRowKey != TSKEY_INITIAL_VAL) { + ++(*curRow); + } + ++nResult; + } + *lastRowKey = rowKey; + } } else { // TODO: use STSRowIter if (chosen_itr == 0) { colId = PRIMARYKEY_TIMESTAMP_COL_ID; tdSKvRowGetVal(row, PRIMARYKEY_TIMESTAMP_COL_ID, -1, -1, &sVal); + rowKey = *(TSKEY*)sVal.val; + if (rowKey != *lastRowKey) { + if (*lastRowKey != TSKEY_INITIAL_VAL) { + ++(*curRow); + } + ++nResult; + } + *lastRowKey = rowKey; } else { SKvRowIdx* pColIdx = tdKvRowColIdxAt(row, chosen_itr - 1); colId = pColIdx->colId; @@ -1657,35 +1736,46 @@ static void mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capacit } } + ASSERT(rowKey != TSKEY_INITIAL_VAL); + if (colId == pColInfo->info.colId) { if (tdValTypeIsNorm(sVal.valType)) { - colDataAppend(pColInfo, numOfRows, sVal.val, false); - } else if (forceSetNull) { - colDataAppend(pColInfo, numOfRows, NULL, true); + colDataAppend(pColInfo, *curRow, sVal.val, false); + } else if (tdValTypeIsNull(sVal.valType)) { + colDataAppend(pColInfo, *curRow, NULL, true); + } else if (tdValTypeIsNone(sVal.valType)) { + // TODO: Set null if nothing append for this row + if (*lastRowKey != rowKey) { + colDataAppend(pColInfo, *curRow, NULL, true); + } + } else { + ASSERT(0); } - i++; + ++i; if (row == row1) { - j++; + ++j; } else { - k++; + ++k; } } else { - if (forceSetNull) { - colDataAppend(pColInfo, numOfRows, NULL, true); + if (*lastRowKey != rowKey) { + colDataAppend(pColInfo, *curRow, NULL, true); } - i++; + ++i; } } - if (forceSetNull) { + if (*lastRowKey != rowKey) { while (i < numOfCols) { // the remain columns are all null data SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i); - colDataAppend(pColInfo, numOfRows, NULL, true); - i++; + colDataAppend(pColInfo, *curRow, NULL, true); + ++i; } } + + return nResult; #endif } @@ -1859,6 +1949,7 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf // compared with the data from in-memory buffer, to generate the correct timestamp array list int32_t numOfRows = 0; + int32_t curRow = 0; int16_t rv1 = -1; int16_t rv2 = -1; @@ -1874,9 +1965,11 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf return; } else if (pCheckInfo->iter != NULL || pCheckInfo->iiter != NULL) { SSkipListNode* node = NULL; + TSKEY lastRowKey = TSKEY_INITIAL_VAL; + do { STSRow* row2 = NULL; - STSRow* row1 = getSRowInTableMem(pCheckInfo, pTsdbReadHandle->order, pCfg->update, &row2); + STSRow* row1 = getSRowInTableMem(pCheckInfo, pTsdbReadHandle->order, pCfg->update, &row2, TD_VER_MAX); if (row1 == NULL) { break; } @@ -1905,9 +1998,9 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf rv2 = TD_ROW_SVER(row2); } - mergeTwoRowFromMem(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, numOfRows, row1, row2, numOfCols, - pCheckInfo->tableId, pSchema1, pSchema2, true); - numOfRows += 1; + numOfRows += mergeTwoRowFromMem(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, &curRow, row1, row2, numOfCols, + pCheckInfo->tableId, pSchema1, pSchema2, true, &lastRowKey); + // numOfRows += 1; if (cur->win.skey == TSKEY_INITIAL_VAL) { cur->win.skey = key; } @@ -1918,6 +2011,7 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf moveToNextRowInMem(pCheckInfo); } else if (key == tsArray[pos]) { // data in buffer has the same timestamp of data in file block, ignore it +#if 0 if (pCfg->update) { if (pCfg->update == TD_ROW_PARTIAL_UPDATE) { doCopyRowsFromFileBlock(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, numOfRows, pos, pos); @@ -1933,7 +2027,7 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf bool forceSetNull = pCfg->update != TD_ROW_PARTIAL_UPDATE; mergeTwoRowFromMem(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, numOfRows, row1, row2, numOfCols, - pCheckInfo->tableId, pSchema1, pSchema2, forceSetNull); + pCheckInfo->tableId, pSchema1, pSchema2, forceSetNull, &lastRowKey); numOfRows += 1; if (cur->win.skey == TSKEY_INITIAL_VAL) { cur->win.skey = key; @@ -1943,6 +2037,35 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf cur->lastKey = key + step; cur->mixBlock = true; + moveToNextRowInMem(pCheckInfo); + pos += step; + } else { + moveToNextRowInMem(pCheckInfo); + } +#endif + if (TD_SUPPORT_UPDATE(pCfg->update)) { + doCopyRowsFromFileBlock(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, curRow, pos, pos); + + if (rv1 != TD_ROW_SVER(row1)) { + // pSchema1 = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row1)); + rv1 = TD_ROW_SVER(row1); + } + if (row2 && rv2 != TD_ROW_SVER(row2)) { + // pSchema2 = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row2)); + rv2 = TD_ROW_SVER(row2); + } + + numOfRows += mergeTwoRowFromMem(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, &curRow, row1, row2, numOfCols, + pCheckInfo->tableId, pSchema1, pSchema2, true, &lastRowKey); + // ++numOfRows; + if (cur->win.skey == TSKEY_INITIAL_VAL) { + cur->win.skey = key; + } + + cur->win.ekey = key; + cur->lastKey = key + step; + cur->mixBlock = true; + moveToNextRowInMem(pCheckInfo); pos += step; } else { @@ -1958,11 +2081,18 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf assert(end != -1); if (tsArray[end] == key) { // the value of key in cache equals to the end timestamp value, ignore it +#if 0 if (pCfg->update == TD_ROW_DISCARD_UPDATE) { moveToNextRowInMem(pCheckInfo); } else { end -= step; } +#endif + if (!TD_SUPPORT_UPDATE(pCfg->update)) { + moveToNextRowInMem(pCheckInfo); + } else { + end -= step; + } } int32_t qstart = 0, qend = 0; @@ -2572,6 +2702,7 @@ static UNUSED_FUNC void changeQueryHandleForInterpQuery(tsdbReaderT pHandle) { static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int maxRowsToRead, STimeWindow* win, STsdbReadHandle* pTsdbReadHandle) { int numOfRows = 0; + int curRows = 0; int32_t numOfCols = (int32_t)taosArrayGetSize(pTsdbReadHandle->pColumns); STsdbCfg* pCfg = REPO_CFG(pTsdbReadHandle->pTsdb); win->skey = TSKEY_INITIAL_VAL; @@ -2579,9 +2710,11 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int int64_t st = taosGetTimestampUs(); int16_t rv = -1; STSchema* pSchema = NULL; + TSKEY lastRowKey = TSKEY_INITIAL_VAL; + do { - STSRow* row = getSRowInTableMem(pCheckInfo, pTsdbReadHandle->order, pCfg->update, NULL); + STSRow* row = getSRowInTableMem(pCheckInfo, pTsdbReadHandle->order, pCfg->update, NULL, TD_VER_MAX); if (row == NULL) { break; } @@ -2604,16 +2737,18 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int pSchema = metaGetTbTSchema(REPO_META(pTsdbReadHandle->pTsdb), pCheckInfo->tableId, 0); rv = TD_ROW_SVER(row); } - mergeTwoRowFromMem(pTsdbReadHandle, maxRowsToRead, numOfRows, row, NULL, numOfCols, pCheckInfo->tableId, pSchema, - NULL, true); + numOfRows += mergeTwoRowFromMem(pTsdbReadHandle, maxRowsToRead, &curRows, row, NULL, numOfCols, pCheckInfo->tableId, pSchema, + NULL, true, &lastRowKey); - if (++numOfRows >= maxRowsToRead) { + if (numOfRows >= maxRowsToRead) { moveToNextRowInMem(pCheckInfo); break; } } while (moveToNextRowInMem(pCheckInfo)); + taosMemoryFreeClear(pSchema); // free the STSChema + assert(numOfRows <= maxRowsToRead); // if the buffer is not full in case of descending order query, move the data in the front of the buffer @@ -2731,6 +2866,8 @@ static bool loadCachedLastRow(STsdbReadHandle* pTsdbReadHandle) { STSRow* pRow = NULL; TSKEY key = TSKEY_INITIAL_VAL; int32_t step = ASCENDING_TRAVERSE(pTsdbReadHandle->order) ? 1 : -1; + TSKEY lastRowKey = TSKEY_INITIAL_VAL; + int32_t curRow = 0; if (++pTsdbReadHandle->activeIndex < numOfTables) { STableCheckInfo* pCheckInfo = taosArrayGet(pTsdbReadHandle->pTableCheckInfo, pTsdbReadHandle->activeIndex); @@ -2738,8 +2875,8 @@ static bool loadCachedLastRow(STsdbReadHandle* pTsdbReadHandle) { // if (ret != TSDB_CODE_SUCCESS) { // return false; // } - mergeTwoRowFromMem(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, 0, pRow, NULL, numOfCols, pCheckInfo->tableId, - NULL, NULL, true); + mergeTwoRowFromMem(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, &curRow, pRow, NULL, numOfCols, pCheckInfo->tableId, + NULL, NULL, true, &lastRowKey); taosMemoryFreeClear(pRow); // update the last key value diff --git a/source/dnode/vnode/src/tsdb/tsdbReadImpl.c b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c index d7c9a70c18..8f2013e8e0 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadImpl.c +++ b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c @@ -266,11 +266,22 @@ int tsdbLoadBlockData(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo) { if (tsdbLoadBlockDataImpl(pReadh, iBlock, pReadh->pDCols[1]) < 0) return -1; // TODO: use the real maxVersion to replace the UINT64_MAX to support Multi-Version if (tdMergeDataCols(pReadh->pDCols[0], pReadh->pDCols[1], pReadh->pDCols[1]->numOfRows, NULL, - update != TD_ROW_PARTIAL_UPDATE, UINT64_MAX) < 0) + TD_SUPPORT_UPDATE(update), TD_VER_MAX) < 0) return -1; } + // if ((pBlock->numOfSubBlocks == 1) && (iBlock->hasDupKey)) { // TODO: use this line + if (pBlock->numOfSubBlocks == 1) { + tdResetDataCols(pReadh->pDCols[1]); + pReadh->pDCols[1]->bitmapMode = pReadh->pDCols[0]->bitmapMode; + if (tdMergeDataCols(pReadh->pDCols[1], pReadh->pDCols[0], pReadh->pDCols[0]->numOfRows, NULL, + TD_SUPPORT_UPDATE(update), TD_VER_MAX) < 0) { + return -1; + } + memcpy(pReadh->pDCols[0], pReadh->pDCols[1], sizeof(SDataCols)); + ASSERT(pReadh->pDCols[0]->bitmapMode != 0); + } - ASSERT(pReadh->pDCols[0]->numOfRows == pBlock->numOfRows); + ASSERT(pReadh->pDCols[0]->numOfRows <= pBlock->numOfRows); ASSERT(dataColsKeyFirst(pReadh->pDCols[0]) == pBlock->keyFirst); ASSERT(dataColsKeyLast(pReadh->pDCols[0]) == pBlock->keyLast); @@ -297,9 +308,20 @@ int tsdbLoadBlockDataCols(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo, if (tsdbLoadBlockDataColsImpl(pReadh, iBlock, pReadh->pDCols[1], colIds, numOfColsIds) < 0) return -1; // TODO: use the real maxVersion to replace the UINT64_MAX to support Multi-Version if (tdMergeDataCols(pReadh->pDCols[0], pReadh->pDCols[1], pReadh->pDCols[1]->numOfRows, NULL, - update != TD_ROW_PARTIAL_UPDATE, UINT64_MAX) < 0) + TD_SUPPORT_UPDATE(update), TD_VER_MAX) < 0) return -1; } + // if ((pBlock->numOfSubBlocks == 1) && (iBlock->hasDupKey)) { // TODO: use this line + if (pBlock->numOfSubBlocks == 1) { + tdResetDataCols(pReadh->pDCols[1]); + pReadh->pDCols[1]->bitmapMode = pReadh->pDCols[0]->bitmapMode; + if (tdMergeDataCols(pReadh->pDCols[1], pReadh->pDCols[0], pReadh->pDCols[0]->numOfRows, NULL, + TD_SUPPORT_UPDATE(update), TD_VER_MAX) < 0) { + return -1; + } + memcpy(pReadh->pDCols[0], pReadh->pDCols[1], sizeof(SDataCols)); + ASSERT(pReadh->pDCols[0]->bitmapMode != 0); + } if (mergeBitmap && !tdDataColsIsBitmapI(pReadh->pDCols[0])) { for (int i = 0; i < numOfColsIds; ++i) { @@ -312,7 +334,7 @@ int tsdbLoadBlockDataCols(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo, } } - ASSERT(pReadh->pDCols[0]->numOfRows == pBlock->numOfRows); + ASSERT(pReadh->pDCols[0]->numOfRows <= pBlock->numOfRows); ASSERT(dataColsKeyFirst(pReadh->pDCols[0]) == pBlock->keyFirst); ASSERT(dataColsKeyLast(pReadh->pDCols[0]) == pBlock->keyLast); @@ -623,15 +645,15 @@ static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDat } if (dcol != 0) { - ccol++; + ++ccol; } - dcol++; + ++dcol; } else if (tcolId < pDataCol->colId) { - ccol++; + ++ccol; } else { // Set current column as NULL and forward dataColReset(pDataCol); - dcol++; + ++dcol; } } diff --git a/source/dnode/vnode/src/tsdb/tsdbSma.c b/source/dnode/vnode/src/tsdb/tsdbSma.c index 362e31ff61..83adb545ba 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSma.c +++ b/source/dnode/vnode/src/tsdb/tsdbSma.c @@ -1635,17 +1635,20 @@ int32_t tsdbCreateTSma(STsdb *pTsdb, char *pMsg) { SSmaCfg vCreateSmaReq = {0}; if (!tDeserializeSVCreateTSmaReq(pMsg, &vCreateSmaReq)) { terrno = TSDB_CODE_OUT_OF_MEMORY; - tsdbWarn("vgId:%d TDMT_VND_CREATE_SMA received but deserialize failed since %s", REPO_ID(pTsdb), terrstr(terrno)); + tsdbWarn("vgId:%d tsma create msg received but deserialize failed since %s", REPO_ID(pTsdb), terrstr(terrno)); return -1; } - tsdbDebug("vgId:%d TDMT_VND_CREATE_SMA msg received for %s:%" PRIi64, REPO_ID(pTsdb), vCreateSmaReq.tSma.indexName, - vCreateSmaReq.tSma.indexUid); + + tsdbDebug("vgId:%d tsma create msg %s:%" PRIi64 " for table %" PRIi64 " received", REPO_ID(pTsdb), + vCreateSmaReq.tSma.indexName, vCreateSmaReq.tSma.indexUid, vCreateSmaReq.tSma.tableUid); // record current timezone of server side vCreateSmaReq.tSma.timezoneInt = tsTimezone; if (metaCreateTSma(REPO_META(pTsdb), &vCreateSmaReq) < 0) { // TODO: handle error + tsdbWarn("vgId:%d tsma %s:%" PRIi64 " create failed for table %" PRIi64 " since %s", REPO_ID(pTsdb), + vCreateSmaReq.tSma.indexName, vCreateSmaReq.tSma.indexUid, vCreateSmaReq.tSma.tableUid, terrstr(terrno)); tdDestroyTSma(&vCreateSmaReq.tSma); return -1; } From 26155b9511fd6a98f62c686f85c9cdea2bf13401 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 5 May 2022 18:05:06 +0800 Subject: [PATCH 12/83] refactor:add schemaless test cases --- source/client/src/clientSml.c | 75 +++++++++++++++--------- source/client/test/smlTest.cpp | 104 +++++++++++++++++++++++++-------- 2 files changed, 128 insertions(+), 51 deletions(-) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index de034957dc..139336e547 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -133,8 +133,12 @@ static uint64_t smlGenId() { } static int32_t smlBuildInvalidDataMsg(SSmlMsgBuf* pBuf, const char *msg1, const char *msg2) { - if(msg1) snprintf(pBuf->buf, pBuf->len, "%s:", msg1); - if(msg2) strncpy(pBuf->buf, msg2, pBuf->len); + if(msg1) strncat(pBuf->buf, msg1, pBuf->len); + int32_t left = pBuf->len - strlen(pBuf->buf); + if(left > 2 && msg2) { + strncat(pBuf->buf, ":", left - 1); + strncat(pBuf->buf, msg2, left - 2); + } return TSDB_CODE_SML_INVALID_DATA; } @@ -658,11 +662,12 @@ static bool smlParseBigInt(SSmlKv *kvVal, bool *isValid, SSmlMsgBuf *msg) { int32_t len = kvVal->valueLen; if (len > 3 && strncasecmp(pVal + len - 3, "i64", 3) == 0) { char *endptr = NULL; + errno = 0; int64_t result = strtoll(pVal, &endptr, 10); if(endptr != pVal + len - 3){ // 78ri8 *isValid = false; smlBuildInvalidDataMsg(msg, "invalid big int", endptr); - }else if(!IS_VALID_BIGINT(result)){ + }else if(errno == ERANGE || !IS_VALID_BIGINT(result)){ *isValid = false; smlBuildInvalidDataMsg(msg, "big int out of range[-9223372036854775808,9223372036854775807]", endptr); }else{ @@ -672,11 +677,12 @@ static bool smlParseBigInt(SSmlKv *kvVal, bool *isValid, SSmlMsgBuf *msg) { return true; }else if (len > 1 && pVal[len - 1] == 'i') { char *endptr = NULL; + errno = 0; int64_t result = strtoll(pVal, &endptr, 10); if(endptr != pVal + len - 1){ // 78ri8 *isValid = false; smlBuildInvalidDataMsg(msg, "invalid big int", endptr); - }else if(!IS_VALID_BIGINT(result)){ + }else if(errno == ERANGE || !IS_VALID_BIGINT(result)){ *isValid = false; smlBuildInvalidDataMsg(msg, "big int out of range[-9223372036854775808,9223372036854775807]", endptr); }else{ @@ -700,11 +706,12 @@ static bool smlParseBigUint(SSmlKv *kvVal, bool *isValid, SSmlMsgBuf *msg) { const char *signalPos = pVal + len - 3; if (strncasecmp(signalPos, "u64", 3) == 0) { char *endptr = NULL; + errno = 0; uint64_t result = strtoull(pVal, &endptr, 10); if(endptr != signalPos){ // 78ri8 *isValid = false; smlBuildInvalidDataMsg(msg, "invalid unsigned big int", endptr); - }else if(!IS_VALID_UBIGINT(result)){ + }else if(errno == ERANGE || !IS_VALID_UBIGINT(result)){ *isValid = false; smlBuildInvalidDataMsg(msg, "unsigned big int out of range[0,18446744073709551615]", endptr); }else{ @@ -720,18 +727,19 @@ static bool smlParseFloat(SSmlKv *kvVal, bool *isValid, SSmlMsgBuf *msg) { const char *pVal = kvVal->value; int32_t len = kvVal->valueLen; char *endptr = NULL; + errno = 0; float result = strtof(pVal, &endptr); - if(endptr == pVal + len && IS_VALID_FLOAT(result)){ // 78 + if(endptr == pVal + len && errno != ERANGE && IS_VALID_FLOAT(result)){ // 78 kvVal->f = result; *isValid = true; return true; } - if (len > 3 && len 3 && strncasecmp(pVal + len - 3, "f32", 3) == 0) { if(endptr != pVal + len - 3){ // 78ri8 *isValid = false; smlBuildInvalidDataMsg(msg, "invalid float", endptr); - }else if(!IS_VALID_FLOAT(result)){ + }else if(errno == ERANGE || !IS_VALID_FLOAT(result)){ *isValid = false; smlBuildInvalidDataMsg(msg, "float out of range[-3.402823466e+38,3.402823466e+38]", endptr); }else{ @@ -750,13 +758,14 @@ static bool smlParseDouble(SSmlKv *kvVal, bool *isValid, SSmlMsgBuf *msg) { return false; } const char *signalPos = pVal + len - 3; - if (len value, pVal->valueLen)) { + pVal->type = TSDB_DATA_TYPE_BINARY; + pVal->valueLen -= 2; + pVal->length = pVal->valueLen; + pVal->value++; + return true; + } + //nchar + if (smlIsNchar(pVal->value, pVal->valueLen)) { + pVal->type = TSDB_DATA_TYPE_NCHAR; + pVal->valueLen -= 3; + pVal->length = pVal->valueLen; + pVal->value += 2; + return true; + } + //float if (smlParseFloat(pVal, &isValid, msg)) { if(!isValid) return false; pVal->type = TSDB_DATA_TYPE_FLOAT; pVal->length = (int16_t)tDataTypes[pVal->type].bytes; return true; } - //binary - if (smlIsBinary(pVal->value, pVal->valueLen)) { - pVal->type = TSDB_DATA_TYPE_BINARY; - pVal->length = pVal->valueLen - 2; - pVal->valueLen -= 2; - pVal->value = pVal->value++; - return true; - } - //nchar - if (smlIsNchar(pVal->value, pVal->valueLen)) { - pVal->type = TSDB_DATA_TYPE_NCHAR; - pVal->length = pVal->valueLen - 3; - pVal->value = pVal->value+2; - return true; - } + //double if (smlParseDouble(pVal, &isValid, msg)) { if(!isValid) return false; pVal->type = TSDB_DATA_TYPE_DOUBLE; @@ -1017,7 +1030,7 @@ static int32_t smlParseCols(const char* data, int32_t len, SArray *cols, bool is kv->valueLen = strlen(TAG); kv->type = TSDB_DATA_TYPE_NCHAR; if(cols) taosArrayPush(cols, &kv); - return true; + return TSDB_CODE_SUCCESS; } for(int i = 0; i < len; i++){ @@ -1039,12 +1052,20 @@ static int32_t smlParseCols(const char* data, int32_t len, SArray *cols, bool is // parse value i++; const char *value = data + i; + bool isInQuote = false; while(i < len){ - if(data[i] == COMMA && i > 0 && data[i-1] != SLASH){ + if(data[i] == QUOTE && data[i-1] != SLASH){ + isInQuote = !isInQuote; + } + if(!isInQuote && data[i] == COMMA && i > 0 && data[i-1] != SLASH){ break; } i++; } + if(isInQuote){ + smlBuildInvalidDataMsg(msg, "only one quote", value); + return TSDB_CODE_SML_INVALID_DATA; + } int32_t valueLen = data + i - value; if(valueLen == 0){ smlBuildInvalidDataMsg(msg, "invalid value", value); diff --git a/source/client/test/smlTest.cpp b/source/client/test/smlTest.cpp index 9337263173..190abd0145 100644 --- a/source/client/test/smlTest.cpp +++ b/source/client/test/smlTest.cpp @@ -131,11 +131,6 @@ TEST(testCase, smlParseString_Test) { } TEST(testCase, smlParseCols_Error_Test) { - char msg[256] = {0}; - SSmlMsgBuf msgBuf; - msgBuf.buf = msg; - msgBuf.len = 256; - const char *data[] = { "c=\"89sd", // binary, nchar "c=j\"89sd\"", @@ -198,12 +193,70 @@ TEST(testCase, smlParseCols_Error_Test) { }; for(int i = 0; i < sizeof(data)/sizeof(data[0]); i++){ + char msg[256] = {0}; + SSmlMsgBuf msgBuf; + msgBuf.buf = msg; + msgBuf.len = 256; int32_t len = strlen(data[i]); int32_t ret = smlParseCols(data[i], len, NULL, false, &msgBuf); ASSERT_NE(ret, TSDB_CODE_SUCCESS); } } +TEST(testCase, smlParseCols_tag_Test) { + char msg[256] = {0}; + SSmlMsgBuf msgBuf; + msgBuf.buf = msg; + msgBuf.len = 256; + + SArray *cols = taosArrayInit(16, POINTER_BYTES); + ASSERT_NE(cols, NULL); + + const char *data = + "cbin=\"passit hello,c=2\",cnch=L\"iisdfsf\",cbool=false,cf64=4.31f64,cf32_=8.32,cf32=8.23f32,ci8=-34i8,cu8=89u8,ci16=233i16,cu16=898u16,ci32=98289i32,cu32=12323u32,ci64=-89238i64,ci=989i,cu64=8989323u64,cbooltrue=true,cboolt=t,cboolf=f,cnch_=l\"iuwq\""; + int32_t len = strlen(data); + int32_t ret = smlParseCols(data, len, cols, true, &msgBuf); + ASSERT_EQ(ret, TSDB_CODE_SUCCESS); + int32_t size = taosArrayGetSize(cols); + ASSERT_EQ(size, 19); + + // nchar + SSmlKv *kv = taosArrayGetP(cols, 0); + ASSERT_EQ(strncasecmp(kv->key, "cbin", 4), 0); + ASSERT_EQ(kv->keyLen, 4); + ASSERT_EQ(kv->type, TSDB_DATA_TYPE_NCHAR); + ASSERT_EQ(kv->valueLen, 18); + ASSERT_EQ(strncasecmp(kv->value, "\"passit", 7), 0); + taosMemoryFree(kv); + + // nchar + kv = taosArrayGetP(cols, 3); + ASSERT_EQ(strncasecmp(kv->key, "cf64", 4), 0); + ASSERT_EQ(kv->keyLen, 4); + ASSERT_EQ(kv->type, TSDB_DATA_TYPE_NCHAR); + ASSERT_EQ(kv->valueLen, 7); + ASSERT_EQ(strncasecmp(kv->value, "4.31f64", 7), 0); + taosMemoryFree(kv); + + taosArrayClear(cols); + data = "t=3e"; + len = 0; + memset(msgBuf.buf, 0, msgBuf.len); + ret = smlParseCols(data, len, cols, true, &msgBuf); + ASSERT_EQ(ret, TSDB_CODE_SUCCESS); + size = taosArrayGetSize(cols); + ASSERT_EQ(size, 1); + + // nchar + kv = taosArrayGetP(cols, 0); + ASSERT_EQ(strncasecmp(kv->key, TAG, strlen(TAG)), 0); + ASSERT_EQ(kv->keyLen, strlen(TAG)); + ASSERT_EQ(kv->type, TSDB_DATA_TYPE_NCHAR); + ASSERT_EQ(kv->valueLen, strlen(TAG)); + ASSERT_EQ(strncasecmp(kv->value, TAG, strlen(TAG)), 0); + taosMemoryFree(kv); +} + TEST(testCase, smlParseCols_Test) { char msg[256] = {0}; SSmlMsgBuf msgBuf; @@ -213,7 +266,7 @@ TEST(testCase, smlParseCols_Test) { SArray *cols = taosArrayInit(16, POINTER_BYTES); ASSERT_NE(cols, NULL); - const char *data = "cbin=\"passit hello,c=2\",cnch=L\"iisdfsf\",cbool=false,cf64=4f64,cf32_=8.32,cf32=8.23f32,ci8=-34i8,cu8=89u8,ci16=233i16,cu16=898u16,ci32=98289i32,cu32=12323u32,ci64=-89238i64,ci=989i,cu64=8989323u64,cbooltrue=true,cboolt=t,cboolf=f,cnch_=l\"iuwq\""; + const char *data = "cbin=\"passit hello,c=2\",cnch=L\"iisdfsf\",cbool=false,cf64=4.31f64,cf32_=8.32,cf32=8.23f32,ci8=-34i8,cu8=89u8,ci16=233i16,cu16=898u16,ci32=98289i32,cu32=12323u32,ci64=-89238i64,ci=989i,cu64=8989323u64,cbooltrue=true,cboolt=t,cboolf=f,cnch_=l\"iuwq\""; int32_t len = strlen(data); int32_t ret = smlParseCols(data, len, cols, false, &msgBuf); ASSERT_EQ(ret, TSDB_CODE_SUCCESS); @@ -223,8 +276,8 @@ TEST(testCase, smlParseCols_Test) { // binary SSmlKv *kv = taosArrayGetP(cols, 0); ASSERT_EQ(strncasecmp(kv->key, "cbin", 4), 0); - ASSERT_EQ(kv->keyLen, 4; - ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BINARY; + ASSERT_EQ(kv->keyLen, 4); + ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BINARY); ASSERT_EQ(kv->length, 16); ASSERT_EQ(strncasecmp(kv->value, "passit", 6), 0); taosMemoryFree(kv); @@ -243,7 +296,7 @@ TEST(testCase, smlParseCols_Test) { ASSERT_EQ(strncasecmp(kv->key, "cbool", 5), 0); ASSERT_EQ(kv->keyLen, 5); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BOOL); - ASSERT_EQ(kv->length, 1; + ASSERT_EQ(kv->length, 1); ASSERT_EQ(kv->i, false); taosMemoryFree(kv); @@ -252,8 +305,9 @@ TEST(testCase, smlParseCols_Test) { ASSERT_EQ(strncasecmp(kv->key, "cf64", 4), 0); ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_DOUBLE); - ASSERT_EQ(kv->length, 8; - ASSERT_EQ(kv->d, 4); + ASSERT_EQ(kv->length, 8); + //ASSERT_EQ(kv->d, 4.31); + printf("4.31 = kv->f:%f\n", kv->d); taosMemoryFree(kv); // float @@ -262,7 +316,8 @@ TEST(testCase, smlParseCols_Test) { ASSERT_EQ(kv->keyLen, 5); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_FLOAT); ASSERT_EQ(kv->length, 4); - ASSERT_EQ(kv->f, 8.32); + //ASSERT_EQ(kv->f, 8.32); + printf("8.32 = kv->f:%f\n", kv->f); taosMemoryFree(kv); // float @@ -271,16 +326,17 @@ TEST(testCase, smlParseCols_Test) { ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_FLOAT); ASSERT_EQ(kv->length, 4); - ASSERT_EQ(kv->f, 8.23); + //ASSERT_EQ(kv->f, 8.23); + printf("8.23 = kv->f:%f\n", kv->f); taosMemoryFree(kv); // tiny int kv = taosArrayGetP(cols, 6); ASSERT_EQ(strncasecmp(kv->key, "ci8", 3), 0); ASSERT_EQ(kv->keyLen, 3); - ASSERT_EQ(kv->type, TSDB_DATA_TYPE_TINYINT; + ASSERT_EQ(kv->type, TSDB_DATA_TYPE_TINYINT); ASSERT_EQ(kv->length, 1); - ASSERT_EQ(i, -34); + ASSERT_EQ(kv->i, -34); taosMemoryFree(kv); // unsigned tiny int @@ -332,8 +388,8 @@ TEST(testCase, smlParseCols_Test) { // bigint kv = taosArrayGetP(cols, 12); ASSERT_EQ(strncasecmp(kv->key, "ci64", 4), 0); - ASSERT_EQ(kv->keyLen, 4; - ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BIGINT; + ASSERT_EQ(kv->keyLen, 4); + ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BIGINT); ASSERT_EQ(kv->length, 8); ASSERT_EQ(kv->i, -89238); taosMemoryFree(kv); @@ -369,10 +425,10 @@ TEST(testCase, smlParseCols_Test) { // bool kv = taosArrayGetP(cols, 16); ASSERT_EQ(strncasecmp(kv->key, "cboolt", 6), 0); - ASSERT_EQ(kv->keyLen, 6; - ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BOOL; - ASSERT_EQ(kv->length, 8); - ASSERT_EQ(kv->i, -89238); + ASSERT_EQ(kv->keyLen, 6); + ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(kv->length, 1); + ASSERT_EQ(kv->i, true); taosMemoryFree(kv); // bool @@ -380,11 +436,11 @@ TEST(testCase, smlParseCols_Test) { ASSERT_EQ(strncasecmp(kv->key, "cboolf", 6), 0); ASSERT_EQ(kv->keyLen, 6); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BOOL); - ASSERT_EQ(kv->length, 8); - ASSERT_EQ(kv->i, 989); + ASSERT_EQ(kv->length, 1); + ASSERT_EQ(kv->i, false); taosMemoryFree(kv); - // unsigned bigint + // nchar kv = taosArrayGetP(cols, 18); ASSERT_EQ(strncasecmp(kv->key, "cnch_", 5), 0); ASSERT_EQ(kv->keyLen, 5); From 319d2785abd24ec3ca82af084d2990258bf79027 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Thu, 5 May 2022 18:33:31 +0800 Subject: [PATCH 13/83] feat: update support merge in mem --- source/dnode/vnode/src/tsdb/tsdbRead.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 72fada4e97..7f733e0e4b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -1613,6 +1613,7 @@ static int32_t mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capa SCellVal sVal = {0}; TSKEY rowKey = TSKEY_INITIAL_VAL; int32_t nResult = 0; + bool isMerge = true; // the schema version info is embeded in STSRow int32_t numOfColsOfRow1 = 0; @@ -1708,6 +1709,7 @@ static int32_t mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capa if (colId == PRIMARYKEY_TIMESTAMP_COL_ID) { rowKey = *(TSKEY*)sVal.val; if (rowKey != *lastRowKey) { + isMerge = false; if (*lastRowKey != TSKEY_INITIAL_VAL) { ++(*curRow); } @@ -1722,6 +1724,7 @@ static int32_t mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capa tdSKvRowGetVal(row, PRIMARYKEY_TIMESTAMP_COL_ID, -1, -1, &sVal); rowKey = *(TSKEY*)sVal.val; if (rowKey != *lastRowKey) { + isMerge = false; if (*lastRowKey != TSKEY_INITIAL_VAL) { ++(*curRow); } @@ -1745,7 +1748,7 @@ static int32_t mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capa colDataAppend(pColInfo, *curRow, NULL, true); } else if (tdValTypeIsNone(sVal.valType)) { // TODO: Set null if nothing append for this row - if (*lastRowKey != rowKey) { + if (!isMerge) { colDataAppend(pColInfo, *curRow, NULL, true); } } else { @@ -1760,7 +1763,7 @@ static int32_t mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capa ++k; } } else { - if (*lastRowKey != rowKey) { + if (!isMerge) { colDataAppend(pColInfo, *curRow, NULL, true); } ++i; From 1854f065c9dbaa3a9b1c7060756a43f154f9e782 Mon Sep 17 00:00:00 2001 From: jiacy-jcy <714897623@qq.com> Date: Thu, 5 May 2022 20:22:53 +0800 Subject: [PATCH 14/83] add case for max and min --- tests/system-test/2-query/max.py | 205 +++++++++++++++++++++ tests/system-test/2-query/min.py | 215 ++++++++++++++++++++++ tests/system-test/2-query/timetruncate.py | 104 +++++++++++ 3 files changed, 524 insertions(+) create mode 100644 tests/system-test/2-query/max.py create mode 100644 tests/system-test/2-query/min.py create mode 100644 tests/system-test/2-query/timetruncate.py diff --git a/tests/system-test/2-query/max.py b/tests/system-test/2-query/max.py new file mode 100644 index 0000000000..5342c7d449 --- /dev/null +++ b/tests/system-test/2-query/max.py @@ -0,0 +1,205 @@ +from util.log import * +from util.cases import * +from util.sql import * +import numpy as np + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.rowNum = 10 + self.ts = 1537146000000 + + def prepare_data(self): + + pass + def run(self): + tdSql.prepare() + + intData = [] + floatData = [] + + tdSql.execute('''create table stb(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, + col7 bool, col8 binary(20), col9 nchar(20), col11 tinyint unsigned, col12 smallint unsigned, col13 int unsigned, col14 bigint unsigned) tags(loc nchar(20))''') + tdSql.execute("create table stb_1 using stb tags('beijing')") + tdSql.execute('''create table ntb(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, + col7 bool, col8 binary(20), col9 nchar(20), col11 tinyint unsigned, col12 smallint unsigned, col13 int unsigned, col14 bigint unsigned)''') + for i in range(self.rowNum): + tdSql.execute("insert into ntb values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1)) + intData.append(i + 1) + floatData.append(i + 0.1) + for i in range(self.rowNum): + tdSql.execute("insert into stb_1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1)) + intData.append(i + 1) + floatData.append(i + 0.1) + + # max verifacation + tdSql.error("select max(ts) from stb_1") + tdSql.error("select max(ts) from db.stb_1") + tdSql.error("select max(col7) from stb_1") + tdSql.error("select max(col7) from db.stb_1") + tdSql.error("select max(col8) from stb_1") + tdSql.error("select max(col8) from db.stb_1") + tdSql.error("select max(col9) from stb_1") + tdSql.error("select max(col9) from db.stb_1") + + tdSql.query("select max(col1) from stb_1") + tdSql.checkData(0, 0, np.max(intData)) + tdSql.query("select max(col1) from db.stb_1") + tdSql.checkData(0, 0, np.max(intData)) + tdSql.query("select max(col2) from stb_1") + tdSql.checkData(0, 0, np.max(intData)) + tdSql.query("select max(col2) from db.stb_1") + tdSql.checkData(0, 0, np.max(intData)) + tdSql.query("select max(col3) from stb_1") + tdSql.checkData(0, 0, np.max(intData)) + tdSql.query("select max(col3) from db.stb_1") + tdSql.checkData(0, 0, np.max(intData)) + tdSql.query("select max(col4) from stb_1") + tdSql.checkData(0, 0, np.max(intData)) + tdSql.query("select max(col4) from db.stb_1") + tdSql.checkData(0, 0, np.max(intData)) + tdSql.query("select max(col11) from stb_1") + tdSql.checkData(0, 0, np.max(intData)) + tdSql.query("select max(col11) from db.stb_1") + tdSql.checkData(0, 0, np.max(intData)) + tdSql.query("select max(col12) from stb_1") + tdSql.checkData(0, 0, np.max(intData)) + tdSql.query("select max(col12) from db.stb_1") + tdSql.checkData(0, 0, np.max(intData)) + tdSql.query("select max(col13) from stb_1") + tdSql.checkData(0, 0, np.max(intData)) + tdSql.query("select max(col13) from db.stb_1") + tdSql.checkData(0, 0, np.max(intData)) + tdSql.query("select max(col14) from stb_1") + tdSql.checkData(0, 0, np.max(intData)) + tdSql.query("select max(col14) from db.stb_1") + tdSql.checkData(0, 0, np.max(intData)) + tdSql.query("select max(col5) from stb_1") + tdSql.checkData(0, 0, np.max(floatData)) + tdSql.query("select max(col5) from db.stb_1") + tdSql.checkData(0, 0, np.max(floatData)) + tdSql.query("select max(col6) from stb_1") + tdSql.checkData(0, 0, np.max(floatData)) + tdSql.query("select max(col6) from db.stb_1") + tdSql.checkData(0, 0, np.max(floatData)) + tdSql.query("select max(col1) from stb_1 where col2<=5") + tdSql.checkData(0,0,5) + + + + tdSql.error("select max(ts) from stb") + tdSql.error("select max(ts) from db.stb") + tdSql.error("select max(col7) from stb") + tdSql.error("select max(col7) from db.stb") + tdSql.error("select max(col8) from stb") + tdSql.error("select max(col8) from db.stb") + tdSql.error("select max(col9) from stb") + tdSql.error("select max(col9) from db.stb") + + tdSql.query("select max(col1) from stb") + tdSql.checkData(0, 0, np.max(intData)) + tdSql.query("select max(col1) from db.stb") + tdSql.checkData(0, 0, np.max(intData)) + tdSql.query("select max(col2) from stb") + tdSql.checkData(0, 0, np.max(intData)) + tdSql.query("select max(col2) from db.stb") + tdSql.checkData(0, 0, np.max(intData)) + tdSql.query("select max(col3) from stb") + tdSql.checkData(0, 0, np.max(intData)) + tdSql.query("select max(col3) from db.stb") + tdSql.checkData(0, 0, np.max(intData)) + tdSql.query("select max(col4) from stb") + tdSql.checkData(0, 0, np.max(intData)) + tdSql.query("select max(col4) from db.stb") + tdSql.checkData(0, 0, np.max(intData)) + tdSql.query("select max(col11) from stb") + tdSql.checkData(0, 0, np.max(intData)) + tdSql.query("select max(col11) from db.stb") + tdSql.checkData(0, 0, np.max(intData)) + tdSql.query("select max(col12) from stb") + tdSql.checkData(0, 0, np.max(intData)) + tdSql.query("select max(col12) from db.stb") + tdSql.checkData(0, 0, np.max(intData)) + tdSql.query("select max(col13) from stb") + tdSql.checkData(0, 0, np.max(intData)) + tdSql.query("select max(col13) from db.stb") + tdSql.checkData(0, 0, np.max(intData)) + tdSql.query("select max(col14) from stb") + tdSql.checkData(0, 0, np.max(intData)) + tdSql.query("select max(col14) from db.stb") + tdSql.checkData(0, 0, np.max(intData)) + tdSql.query("select max(col5) from stb") + tdSql.checkData(0, 0, np.max(floatData)) + tdSql.query("select max(col5) from db.stb") + tdSql.checkData(0, 0, np.max(floatData)) + tdSql.query("select max(col6) from stb") + tdSql.checkData(0, 0, np.max(floatData)) + tdSql.query("select max(col6) from db.stb") + tdSql.checkData(0, 0, np.max(floatData)) + tdSql.query("select max(col1) from stb where col2<=5") + tdSql.checkData(0,0,5) + + + + tdSql.error("select max(ts) from ntb") + tdSql.error("select max(ts) from db.ntb") + tdSql.error("select max(col7) from ntb") + tdSql.error("select max(col7) from db.ntb") + tdSql.error("select max(col8) from ntb") + tdSql.error("select max(col8) from db.ntb") + tdSql.error("select max(col9) from ntb") + tdSql.error("select max(col9) from db.ntb") + + tdSql.query("select max(col1) from ntb") + tdSql.checkData(0, 0, np.max(intData)) + tdSql.query("select max(col1) from db.ntb") + tdSql.checkData(0, 0, np.max(intData)) + tdSql.query("select max(col2) from ntb") + tdSql.checkData(0, 0, np.max(intData)) + tdSql.query("select max(col2) from db.ntb") + tdSql.checkData(0, 0, np.max(intData)) + tdSql.query("select max(col3) from ntb") + tdSql.checkData(0, 0, np.max(intData)) + tdSql.query("select max(col3) from db.ntb") + tdSql.checkData(0, 0, np.max(intData)) + tdSql.query("select max(col4) from ntb") + tdSql.checkData(0, 0, np.max(intData)) + tdSql.query("select max(col4) from db.ntb") + tdSql.checkData(0, 0, np.max(intData)) + tdSql.query("select max(col11) from ntb") + tdSql.checkData(0, 0, np.max(intData)) + tdSql.query("select max(col11) from db.ntb") + tdSql.checkData(0, 0, np.max(intData)) + tdSql.query("select max(col12) from ntb") + tdSql.checkData(0, 0, np.max(intData)) + tdSql.query("select max(col12) from db.ntb") + tdSql.checkData(0, 0, np.max(intData)) + tdSql.query("select max(col13) from ntb") + tdSql.checkData(0, 0, np.max(intData)) + tdSql.query("select max(col13) from db.ntb") + tdSql.checkData(0, 0, np.max(intData)) + tdSql.query("select max(col14) from ntb") + tdSql.checkData(0, 0, np.max(intData)) + tdSql.query("select max(col14) from db.ntb") + tdSql.checkData(0, 0, np.max(intData)) + tdSql.query("select max(col5) from ntb") + tdSql.checkData(0, 0, np.max(floatData)) + tdSql.query("select max(col5) from db.ntb") + tdSql.checkData(0, 0, np.max(floatData)) + tdSql.query("select max(col6) from ntb") + tdSql.checkData(0, 0, np.max(floatData)) + tdSql.query("select max(col6) from db.ntb") + tdSql.checkData(0, 0, np.max(floatData)) + tdSql.query("select max(col1) from stb_1 where col2<=5") + tdSql.checkData(0,0,5) + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/system-test/2-query/min.py b/tests/system-test/2-query/min.py new file mode 100644 index 0000000000..60a8c105f1 --- /dev/null +++ b/tests/system-test/2-query/min.py @@ -0,0 +1,215 @@ +from wsgiref.headers import tspecials +from util.log import * +from util.cases import * +from util.sql import * +import numpy as np + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.rowNum = 10 + self.ts = 1537146000000 + + def run(self): + tdSql.prepare() + + intData = [] + floatData = [] + + tdSql.execute('''create table stb(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, + col7 bool, col8 binary(20), col9 nchar(20), col11 tinyint unsigned, col12 smallint unsigned, col13 int unsigned, col14 bigint unsigned) tags(loc nchar(20))''') + tdSql.execute("create table stb_1 using stb tags('beijing')") + tdSql.execute('''create table ntb(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, + col7 bool, col8 binary(20), col9 nchar(20), col11 tinyint unsigned, col12 smallint unsigned, col13 int unsigned, col14 bigint unsigned)''') + for i in range(self.rowNum): + tdSql.execute("insert into ntb values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1)) + intData.append(i + 1) + floatData.append(i + 0.1) + for i in range(self.rowNum): + tdSql.execute("insert into stb_1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1)) + intData.append(i + 1) + floatData.append(i + 0.1) + + # max verifacation + tdSql.error("select min(ts) from stb_1") + tdSql.error("select min(ts) from db.stb_1") + tdSql.error("select min(col7) from stb_1") + tdSql.error("select min(col7) from db.stb_1") + tdSql.error("select min(col8) from stb_1") + tdSql.error("select min(col8) from db.stb_1") + tdSql.error("select min(col9) from stb_1") + tdSql.error("select min(col9) from db.stb_1") + # tdSql.error("select min(a) from stb_1") + # tdSql.error("select min(1) from stb_1") + tdSql.error("select min(now()) from stb_1") + tdSql.error("select min(count(c1),count(c2)) from stb_1") + + tdSql.query("select min(col1) from stb_1") + tdSql.checkData(0, 0, np.min(intData)) + tdSql.query("select min(col1) from db.stb_1") + tdSql.checkData(0, 0, np.min(intData)) + tdSql.query("select min(col2) from stb_1") + tdSql.checkData(0, 0, np.min(intData)) + tdSql.query("select min(col2) from db.stb_1") + tdSql.checkData(0, 0, np.min(intData)) + tdSql.query("select min(col3) from stb_1") + tdSql.checkData(0, 0, np.min(intData)) + tdSql.query("select min(col3) from db.stb_1") + tdSql.checkData(0, 0, np.min(intData)) + tdSql.query("select min(col4) from stb_1") + tdSql.checkData(0, 0, np.min(intData)) + tdSql.query("select min(col4) from db.stb_1") + tdSql.checkData(0, 0, np.min(intData)) + tdSql.query("select min(col11) from stb_1") + tdSql.checkData(0, 0, np.min(intData)) + tdSql.query("select min(col11) from db.stb_1") + tdSql.checkData(0, 0, np.min(intData)) + tdSql.query("select min(col12) from stb_1") + tdSql.checkData(0, 0, np.min(intData)) + tdSql.query("select min(col12) from db.stb_1") + tdSql.checkData(0, 0, np.min(intData)) + tdSql.query("select min(col13) from stb_1") + tdSql.checkData(0, 0, np.min(intData)) + tdSql.query("select min(col13) from db.stb_1") + tdSql.checkData(0, 0, np.min(intData)) + tdSql.query("select min(col14) from stb_1") + tdSql.checkData(0, 0, np.min(intData)) + tdSql.query("select min(col14) from db.stb_1") + tdSql.checkData(0, 0, np.min(intData)) + tdSql.query("select min(col5) from stb_1") + tdSql.checkData(0, 0, np.min(floatData)) + tdSql.query("select min(col5) from db.stb_1") + tdSql.checkData(0, 0, np.min(floatData)) + tdSql.query("select min(col6) from stb_1") + tdSql.checkData(0, 0, np.min(floatData)) + tdSql.query("select min(col6) from db.stb_1") + tdSql.checkData(0, 0, np.min(floatData)) + tdSql.query("select min(col1) from stb_1 where col2>=5") + tdSql.checkData(0,0,5) + + + tdSql.error("select min(ts) from stb_1") + tdSql.error("select min(ts) from db.stb_1") + tdSql.error("select min(col7) from stb_1") + tdSql.error("select min(col7) from db.stb_1") + tdSql.error("select min(col8) from stb_1") + tdSql.error("select min(col8) from db.stb_1") + tdSql.error("select min(col9) from stb_1") + tdSql.error("select min(col9) from db.stb_1") + # tdSql.error("select min(a) from stb_1") + # tdSql.error("select min(1) from stb_1") + tdSql.error("select min(now()) from stb_1") + tdSql.error("select min(count(c1),count(c2)) from stb_1") + + tdSql.query("select min(col1) from stb") + tdSql.checkData(0, 0, np.min(intData)) + tdSql.query("select min(col1) from db.stb") + tdSql.checkData(0, 0, np.min(intData)) + tdSql.query("select min(col2) from stb") + tdSql.checkData(0, 0, np.min(intData)) + tdSql.query("select min(col2) from db.stb") + tdSql.checkData(0, 0, np.min(intData)) + tdSql.query("select min(col3) from stb") + tdSql.checkData(0, 0, np.min(intData)) + tdSql.query("select min(col3) from db.stb") + tdSql.checkData(0, 0, np.min(intData)) + tdSql.query("select min(col4) from stb") + tdSql.checkData(0, 0, np.min(intData)) + tdSql.query("select min(col4) from db.stb") + tdSql.checkData(0, 0, np.min(intData)) + tdSql.query("select min(col11) from stb") + tdSql.checkData(0, 0, np.min(intData)) + tdSql.query("select min(col11) from db.stb") + tdSql.checkData(0, 0, np.min(intData)) + tdSql.query("select min(col12) from stb") + tdSql.checkData(0, 0, np.min(intData)) + tdSql.query("select min(col12) from db.stb") + tdSql.checkData(0, 0, np.min(intData)) + tdSql.query("select min(col13) from stb") + tdSql.checkData(0, 0, np.min(intData)) + tdSql.query("select min(col13) from db.stb") + tdSql.checkData(0, 0, np.min(intData)) + tdSql.query("select min(col14) from stb") + tdSql.checkData(0, 0, np.min(intData)) + tdSql.query("select min(col14) from db.stb") + tdSql.checkData(0, 0, np.min(intData)) + tdSql.query("select min(col5) from stb") + tdSql.checkData(0, 0, np.min(floatData)) + tdSql.query("select min(col5) from db.stb") + tdSql.checkData(0, 0, np.min(floatData)) + tdSql.query("select min(col6) from stb") + tdSql.checkData(0, 0, np.min(floatData)) + tdSql.query("select min(col6) from db.stb") + tdSql.checkData(0, 0, np.min(floatData)) + tdSql.query("select min(col1) from stb where col2>=5") + tdSql.checkData(0,0,5) + + + tdSql.error("select min(ts) from ntb") + tdSql.error("select min(ts) from db.ntb") + tdSql.error("select min(col7) from ntb") + tdSql.error("select min(col7) from db.ntb") + tdSql.error("select min(col8) from ntb") + tdSql.error("select min(col8) from db.ntb") + tdSql.error("select min(col9) from ntb") + tdSql.error("select min(col9) from db.ntb") + # tdSql.error("select min(a) from stb_1") + # tdSql.error("select min(1) from stb_1") + tdSql.error("select min(now()) from ntb") + tdSql.error("select min(count(c1),count(c2)) from ntb") + + tdSql.query("select min(col1) from ntb") + tdSql.checkData(0, 0, np.min(intData)) + tdSql.query("select min(col1) from db.ntb") + tdSql.checkData(0, 0, np.min(intData)) + tdSql.query("select min(col2) from ntb") + tdSql.checkData(0, 0, np.min(intData)) + tdSql.query("select min(col2) from db.ntb") + tdSql.checkData(0, 0, np.min(intData)) + tdSql.query("select min(col3) from ntb") + tdSql.checkData(0, 0, np.min(intData)) + tdSql.query("select min(col3) from db.ntb") + tdSql.checkData(0, 0, np.min(intData)) + tdSql.query("select min(col4) from ntb") + tdSql.checkData(0, 0, np.min(intData)) + tdSql.query("select min(col4) from db.ntb") + tdSql.checkData(0, 0, np.min(intData)) + tdSql.query("select min(col11) from ntb") + tdSql.checkData(0, 0, np.min(intData)) + tdSql.query("select min(col11) from db.ntb") + tdSql.checkData(0, 0, np.min(intData)) + tdSql.query("select min(col12) from ntb") + tdSql.checkData(0, 0, np.min(intData)) + tdSql.query("select min(col12) from db.ntb") + tdSql.checkData(0, 0, np.min(intData)) + tdSql.query("select min(col13) from ntb") + tdSql.checkData(0, 0, np.min(intData)) + tdSql.query("select min(col13) from db.ntb") + tdSql.checkData(0, 0, np.min(intData)) + tdSql.query("select min(col14) from ntb") + tdSql.checkData(0, 0, np.min(intData)) + tdSql.query("select min(col14) from db.ntb") + tdSql.checkData(0, 0, np.min(intData)) + tdSql.query("select min(col5) from ntb") + tdSql.checkData(0, 0, np.min(floatData)) + tdSql.query("select min(col5) from db.ntb") + tdSql.checkData(0, 0, np.min(floatData)) + tdSql.query("select min(col6) from ntb") + tdSql.checkData(0, 0, np.min(floatData)) + tdSql.query("select min(col6) from db.ntb") + tdSql.checkData(0, 0, np.min(floatData)) + tdSql.query("select min(col1) from ntb where col2>=5") + tdSql.checkData(0,0,5) + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/system-test/2-query/timetruncate.py b/tests/system-test/2-query/timetruncate.py new file mode 100644 index 0000000000..4c4ebb09df --- /dev/null +++ b/tests/system-test/2-query/timetruncate.py @@ -0,0 +1,104 @@ +import taos +from util.log import * +from util.cases import * +from util.sql import * +import numpy as np + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.rowNum = 10 + self.ts = 1537146000000 # 2018-9-17 09:00:00.000 + + def run(self): + tdSql.prepare() + + intData = [] + floatData = [] + + tdSql.execute('''create table stb(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, + col7 bool, col8 binary(20), col9 nchar(20), col11 tinyint unsigned, col12 smallint unsigned, col13 int unsigned, col14 bigint unsigned) tags(loc nchar(20))''') + tdSql.execute("create table stb_1 using stb tags('beijing')") + tdSql.execute('''create table ntb(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, + col7 bool, col8 binary(20), col9 nchar(20), col11 tinyint unsigned, col12 smallint unsigned, col13 int unsigned, col14 bigint unsigned)''') + for i in range(self.rowNum): + tdSql.execute("insert into ntb values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1)) + intData.append(i + 1) + floatData.append(i + 0.1) + for i in range(self.rowNum): + tdSql.execute("insert into stb_1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1)) + intData.append(i + 1) + floatData.append(i + 0.1) + + tdSql.query("select timetruncate(1,1d) from ntb") + tdSql.checkRows(10) + tdSql.query("select timetruncate(1,1u) from ntb") + tdSql.checkRows(10) + tdSql.query("select timetruncate(1,1a) from ntb") + tdSql.checkRows(10) + tdSql.query("select timetruncate(1,1m) from ntb") + tdSql.checkRows(10) + tdSql.query("select timetruncate(1,1h) from ntb") + tdSql.checkRows(10) + tdSql.query("select timetruncate(ts,1d) from ntb") + tdSql.checkRows(10) + tdSql.checkData(0,0,"2018-09-17 08:00:00.000") + tdSql.query("select timetruncate(ts,1h) from ntb") + tdSql.checkRows(10) + tdSql.checkData(0,0,"2018-09-17 09:00:00.000") + tdSql.query("select timetruncate(ts,1m) from ntb") + tdSql.checkRows(10) + tdSql.checkData(0,0,"2018-09-17 09:00:00.000") + tdSql.query("select timetruncate(ts,1s) from ntb") + tdSql.checkRows(10) + tdSql.checkData(0,0,"2018-09-17 09:00:00.000") + tdSql.query("select timetruncate(ts,1a) from ntb") + tdSql.checkRows(10) + tdSql.checkData(0,0,"2018-09-17 09:00:00.000") + tdSql.checkData(1,0,"2018-09-17 09:00:00.001") + tdSql.checkData(2,0,"2018-09-17 09:00:00.002") + tdSql.checkData(3,0,"2018-09-17 09:00:00.003") + tdSql.checkData(4,0,"2018-09-17 09:00:00.004") + tdSql.checkData(5,0,"2018-09-17 09:00:00.005") + tdSql.checkData(6,0,"2018-09-17 09:00:00.006") + tdSql.checkData(7,0,"2018-09-17 09:00:00.007") + tdSql.checkData(8,0,"2018-09-17 09:00:00.008") + tdSql.checkData(9,0,"2018-09-17 09:00:00.009") + # tdSql.query("select timetruncate(ts,1u) from ntb") + # tdSql.checkRows(10) + # tdSql.checkData(0,0,"2018-09-17 09:00:00.000000") + # tdSql.checkData(1,0,"2018-09-17 09:00:00.001000") + # tdSql.checkData(2,0,"2018-09-17 09:00:00.002000") + # tdSql.checkData(3,0,"2018-09-17 09:00:00.003000") + # tdSql.checkData(4,0,"2018-09-17 09:00:00.004000") + # tdSql.checkData(5,0,"2018-09-17 09:00:00.005000") + # tdSql.checkData(6,0,"2018-09-17 09:00:00.006000") + # tdSql.checkData(7,0,"2018-09-17 09:00:00.007000") + # tdSql.checkData(8,0,"2018-09-17 09:00:00.008000") + # tdSql.checkData(9,0,"2018-09-17 09:00:00.009000") + # tdSql.query("select timetruncate(ts,1b) from ntb") + # tdSql.checkRows(10) + # tdSql.checkData(0,0,"2018-09-17 09:00:00.000000000") + # tdSql.checkData(1,0,"2018-09-17 09:00:00.001000000") + # tdSql.checkData(2,0,"2018-09-17 09:00:00.002000000") + # tdSql.checkData(3,0,"2018-09-17 09:00:00.003000000") + # tdSql.checkData(4,0,"2018-09-17 09:00:00.004000000") + # tdSql.checkData(5,0,"2018-09-17 09:00:00.005000000") + # tdSql.checkData(6,0,"2018-09-17 09:00:00.006000000") + # tdSql.checkData(7,0,"2018-09-17 09:00:00.007000000") + # tdSql.checkData(8,0,"2018-09-17 09:00:00.008000000") + # tdSql.checkData(9,0,"2018-09-17 09:00:00.009000000") + + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file From 4a649ec925d18d20a1c9e29918c6591357d48dca Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Thu, 5 May 2022 21:07:55 +0800 Subject: [PATCH 15/83] fix(tsdb): swap pointer of data cols --- source/common/src/trow.c | 5 +++-- source/dnode/vnode/src/tsdb/tsdbReadImpl.c | 11 +++++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/source/common/src/trow.c b/source/common/src/trow.c index 9f4f053028..6391744851 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -24,7 +24,8 @@ const uint8_t tdVTypeByte[2][3] = {{ }, { // 1 bit - TD_VTYPE_NORM_BYTE_I, TD_VTYPE_NULL_BYTE_I, + TD_VTYPE_NORM_BYTE_I, // normal + TD_VTYPE_NULL_BYTE_I, TD_VTYPE_NULL_BYTE_I, // padding } @@ -343,7 +344,7 @@ bool tdIsBitmapBlkNorm(const void *pBitmap, int32_t numOfBits, int8_t bitmapMode if (*((uint8_t *)pBitmap) != vTypeByte) { return false; } - pBitmap = POINTER_SHIFT(pBitmap, 1); + pBitmap = POINTER_SHIFT(pBitmap, i); } int32_t nLeft = numOfBits - nBytes * (bitmapMode == 0 ? TD_VTYPE_BITS : TD_VTYPE_BITS_I); diff --git a/source/dnode/vnode/src/tsdb/tsdbReadImpl.c b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c index 8f2013e8e0..8f68883fde 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadImpl.c +++ b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c @@ -246,6 +246,12 @@ int tsdbLoadBlockInfo(SReadH *pReadh, void *pTarget) { return 0; } +static FORCE_INLINE void tsdbSwapDataCols(SDataCols *pDest, SDataCols *pSrc) { + SDataCol *pCols = pDest->cols; + memcpy(pDest, pSrc, sizeof(SDataCols)); + pSrc->cols = pCols; +} + int tsdbLoadBlockData(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo) { ASSERT(pBlock->numOfSubBlocks > 0); STsdbCfg *pCfg = REPO_CFG(pReadh->pRepo); @@ -277,7 +283,7 @@ int tsdbLoadBlockData(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo) { TD_SUPPORT_UPDATE(update), TD_VER_MAX) < 0) { return -1; } - memcpy(pReadh->pDCols[0], pReadh->pDCols[1], sizeof(SDataCols)); + tsdbSwapDataCols(pReadh->pDCols[0], pReadh->pDCols[1]); ASSERT(pReadh->pDCols[0]->bitmapMode != 0); } @@ -319,10 +325,11 @@ int tsdbLoadBlockDataCols(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo, TD_SUPPORT_UPDATE(update), TD_VER_MAX) < 0) { return -1; } - memcpy(pReadh->pDCols[0], pReadh->pDCols[1], sizeof(SDataCols)); + tsdbSwapDataCols(pReadh->pDCols[0], pReadh->pDCols[1]); ASSERT(pReadh->pDCols[0]->bitmapMode != 0); } + if (mergeBitmap && !tdDataColsIsBitmapI(pReadh->pDCols[0])) { for (int i = 0; i < numOfColsIds; ++i) { SDataCol *pDataCol = pReadh->pDCols[0]->cols + i; From 07a02b25b17e671e0c69e23c0dc1d357a998dd53 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 5 May 2022 21:27:02 +0800 Subject: [PATCH 16/83] refactor:add schemaless test cases --- source/client/src/clientSml.c | 25 +++++++++++++++++++------ source/client/test/smlTest.cpp | 13 ++++++++++--- source/libs/parser/src/parInsert.c | 8 ++++---- source/libs/transport/src/rpcTcp.c | 2 +- 4 files changed, 34 insertions(+), 14 deletions(-) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 139336e547..55e95c2aa5 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -117,8 +117,8 @@ typedef struct { //================================================================================================= static uint64_t linesSmlHandleId = 0; -static const char* TS = "ts"; -static const char* TAG = "tagNone"; +static const char* TS = "_ts"; +static const char* TAG = "_tagNone"; //================================================================================================= @@ -350,13 +350,26 @@ static int32_t smlApplySchemaAction(SSmlHandle* info, SSchemaAction* action) { int n = sprintf(result, "create stable %s (", action->createSTable.sTableName); char* pos = result + n; int freeBytes = capacity - n; + size_t size = taosHashGetSize(action->createSTable.fields); + SArray *cols = taosArrayInit(size, POINTER_BYTES); SSmlKv **kv = taosHashIterate(action->createSTable.fields, NULL); while(kv){ - smlBuildColumnDescription(*kv, pos, freeBytes, &outBytes); - pos += outBytes; freeBytes -= outBytes; - *pos = ','; ++pos; --freeBytes; + if(strncmp((*kv)->key, TS, strlen(TS)) == 0 && (*kv)->type == TSDB_DATA_TYPE_TIMESTAMP){ + taosArrayInsert(cols, 0, kv); + }else{ + taosArrayPush(cols, kv); + } kv = taosHashIterate(action->createSTable.fields, kv); } + + for(int i = 0; i < taosArrayGetSize(cols); i++){ + SSmlKv *kvNew = taosArrayGetP(cols, i); + smlBuildColumnDescription(kvNew, pos, freeBytes, &outBytes); + pos += outBytes; freeBytes -= outBytes; + *pos = ','; ++pos; --freeBytes; + } + taosArrayDestroy(cols); + --pos; ++freeBytes; outBytes = snprintf(pos, freeBytes, ") tags ("); @@ -419,7 +432,7 @@ static int32_t smlModifyDBSchemas(SSmlHandle* info) { code = catalogGetSTableMeta(info->pCatalog, info->taos->pAppInfo->pTransporter, &ep, &pName, &pTableMeta); - if (code == TSDB_CODE_TDB_INVALID_TABLE_ID) { + if (code == TSDB_CODE_TDB_INVALID_TABLE_ID || code == TSDB_CODE_MND_INVALID_STB) { SSchemaAction schemaAction = {0}; schemaAction.action = SCHEMA_ACTION_CREATE_STABLE; memcpy(schemaAction.createSTable.sTableName, superTable, superTableLen); diff --git a/source/client/test/smlTest.cpp b/source/client/test/smlTest.cpp index 190abd0145..5f0f188b0b 100644 --- a/source/client/test/smlTest.cpp +++ b/source/client/test/smlTest.cpp @@ -456,6 +456,12 @@ TEST(testCase, smlParseLine_Test) { TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); ASSERT_NE(taos, NULL); + TAOS_RES* pRes = taos_query(taos, "create database if not exists sml_db"); + taos_free_result(pRes); + + pRes = taos_query(taos, "use sml_db"); + taos_free_result(pRes); + SRequestObj *request = createRequest(taos, NULL, NULL, TSDB_SQL_INSERT); ASSERT_NE(request, NULL); @@ -467,9 +473,10 @@ TEST(testCase, smlParseLine_Test) { "readings,name=truck_1,fleet=South,driver=Albert,model=F-150,device_version=v1.5 load_capacity=2000,fuel_capacity=200,nominal_fuel_consumption=15,latitude=72.45258,longitude=68.83761,elevation=255,velocity=0,heading=181,grade=0,fuel_consumption=25 1451606400000000000", "readings,name=truck_2,fleet=North,driver=Derek,model=F-150,device_version=v1.5 load_capacity=2000,fuel_capacity=200,nominal_fuel_consumption=15,latitude=24.5208,longitude=28.09377,elevation=428,velocity=0,heading=304,grade=0,fuel_consumption=25 1451606400000000000" }; - for (int i = 0; i < 3; i++) { - smlParseLine(info, sql[i]); - } + smlInsertLines(info, sql, 3); +// for (int i = 0; i < 3; i++) { +// smlParseLine(info, sql[i]); +// } } // TEST(testCase, smlParseTS_Test) { diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index d59473b26b..1e90d664a9 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -1624,10 +1624,6 @@ static int32_t smlParseTags(SArray *cols, SKVRowBuilder *tagsBuilder, SParsedDat int32_t smlBindData(void *handle, SArray *tags, SArray *colsFormat, SHashObj *colsHash, SArray *cols, bool format, STableMeta *pTableMeta, char *msgBuf, int16_t msgBufLen) { SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; - int32_t rowNum = taosArrayGetSize(cols); - if(rowNum <= 0) { - return buildInvalidOperationMsg(&pBuf, "cols size <= 0"); - } SSmlExecHandle *smlHandle = (SSmlExecHandle *)handle; SSchema* pTagsSchema = getTableTagSchema(pTableMeta); @@ -1679,6 +1675,10 @@ int32_t smlBindData(void *handle, SArray *tags, SArray *colsFormat, SHashObj *co initRowBuilder(&pDataBlock->rowBuilder, pDataBlock->pTableMeta->sversion, &pDataBlock->boundColumnInfo); + int32_t rowNum = format ? taosArrayGetSize(colsFormat) : taosArrayGetSize(cols); + if(rowNum <= 0) { + return buildInvalidOperationMsg(&pBuf, "cols size <= 0"); + } ret = allocateMemForSize(pDataBlock, extendedRowSize * rowNum); if(ret != TSDB_CODE_SUCCESS){ buildInvalidOperationMsg(&pBuf, "allocate memory error"); diff --git a/source/libs/transport/src/rpcTcp.c b/source/libs/transport/src/rpcTcp.c index 52c5ddcf63..edc685f498 100644 --- a/source/libs/transport/src/rpcTcp.c +++ b/source/libs/transport/src/rpcTcp.c @@ -21,7 +21,7 @@ #include "taoserror.h" #include "tutil.h" -#ifndef USE_UV +#ifndef USEf_UV typedef struct SFdObj { void * signature; TdSocketPtr pSocket; // TCP socket FD From 934636d23f7b2277a482e86fb658fe4ba43abe3c Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Thu, 5 May 2022 23:25:55 +0800 Subject: [PATCH 17/83] fix(tsdb): update function --- source/common/src/trow.c | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/source/common/src/trow.c b/source/common/src/trow.c index 6391744851..590e6097c4 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -446,23 +446,24 @@ int tdAppendValToDataCol(SDataCol *pCol, TDRowValT valType, const void *val, int dataColSetNEleNone(pCol, numOfRows, bitmapMode); } } - if (!tdValTypeIsNorm(valType)) { + const void *value = val; + if (!tdValTypeIsNorm(valType) || !val) { // TODO: // 1. back compatibility and easy to debug with codes of 2.0 to save NULL values. // 2. later on, considering further optimization, don't save Null/None for VarType. - val = getNullValue(pCol->type); + value = getNullValue(pCol->type); } if (!isMerge) { if (IS_VAR_DATA_TYPE(pCol->type)) { // set offset pCol->dataOff[numOfRows] = pCol->len; // Copy data - memcpy(POINTER_SHIFT(pCol->pData, pCol->len), val, varDataTLen(val)); + memcpy(POINTER_SHIFT(pCol->pData, pCol->len), value, varDataTLen(value)); // Update the length - pCol->len += varDataTLen(val); + pCol->len += varDataTLen(value); } else { ASSERT(pCol->len == TYPE_BYTES[pCol->type] * numOfRows); - memcpy(POINTER_SHIFT(pCol->pData, pCol->len), val, pCol->bytes); + memcpy(POINTER_SHIFT(pCol->pData, pCol->len), value, pCol->bytes); pCol->len += pCol->bytes; } } else { @@ -472,12 +473,12 @@ int tdAppendValToDataCol(SDataCol *pCol, TDRowValT valType, const void *val, int int32_t lastVarLen = varDataTLen(POINTER_SHIFT(pCol->pData, pCol->dataOff[numOfRows])); pCol->len -= lastVarLen; // Copy data - memcpy(POINTER_SHIFT(pCol->pData, pCol->len), val, varDataTLen(val)); + memcpy(POINTER_SHIFT(pCol->pData, pCol->len), value, varDataTLen(value)); // Update the length - pCol->len += varDataTLen(val); + pCol->len += varDataTLen(value); } else { ASSERT(pCol->len - TYPE_BYTES[pCol->type] == TYPE_BYTES[pCol->type] * numOfRows); - memcpy(POINTER_SHIFT(pCol->pData, pCol->len - TYPE_BYTES[pCol->type]), val, pCol->bytes); + memcpy(POINTER_SHIFT(pCol->pData, pCol->len - TYPE_BYTES[pCol->type]), value, pCol->bytes); } } @@ -654,6 +655,9 @@ int tdMergeDataCols(SDataCols *target, SDataCols *source, int rowsToMerge, int * lastKey = *(TSKEY *)sVal.val; } + if (i == 0) { + (target->cols + j)->bitmap = (source->cols + j)->bitmap; + } tdAppendValToDataCol(target->cols + j, sVal.valType, sVal.val, target->numOfRows, target->maxPoints, target->bitmapMode, merge); From 3bcea212e4d8a38d3f0e1ad0ac32e3695caaf15f Mon Sep 17 00:00:00 2001 From: jiacy-jcy <714897623@qq.com> Date: Fri, 6 May 2022 10:33:37 +0800 Subject: [PATCH 18/83] add case for last --- tests/system-test/2-query/last.py | 132 ++++++++++++++++++++++++++++++ 1 file changed, 132 insertions(+) create mode 100644 tests/system-test/2-query/last.py diff --git a/tests/system-test/2-query/last.py b/tests/system-test/2-query/last.py new file mode 100644 index 0000000000..82e648f60a --- /dev/null +++ b/tests/system-test/2-query/last.py @@ -0,0 +1,132 @@ +from util.log import * +from util.cases import * +from util.sql import * +import numpy as np + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.rowNum = 10 + self.ts = 1537146000000 + + def run(self): + tdSql.prepare() + + tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, + col7 bool, col8 binary(20), col9 nchar(20), col11 tinyint unsigned, col12 smallint unsigned, col13 int unsigned, col14 bigint unsigned) tags(loc nchar(20))''') + tdSql.execute("create table test1 using test tags('beijing')") + tdSql.execute("insert into test1(ts) values(%d)" % (self.ts - 1)) + + # last verifacation + tdSql.query("select last(*) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 1, None) + + tdSql.query("select last(col1) from test1") + tdSql.checkRows(0) + + tdSql.query("select last(col2) from test1") + tdSql.checkRows(0) + + tdSql.query("select last(col3) from test1") + tdSql.checkRows(0) + + tdSql.query("select last(col4) from test1") + tdSql.checkRows(0) + + tdSql.query("select last(col11) from test1") + tdSql.checkRows(0) + + tdSql.query("select last(col12) from test1") + tdSql.checkRows(0) + + tdSql.query("select last(col13) from test1") + tdSql.checkRows(0) + + tdSql.query("select last(col14) from test1") + tdSql.checkRows(0) + + tdSql.query("select last(col5) from test1") + tdSql.checkRows(0) + + tdSql.query("select last(col6) from test1") + tdSql.checkRows(0) + + tdSql.query("select last(col7) from test1") + tdSql.checkRows(0) + + tdSql.query("select last(col8) from test1") + tdSql.checkRows(0) + + tdSql.query("select last(col9) from test1") + tdSql.checkRows(0) + + for i in range(self.rowNum): + tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1)) + + tdSql.query("select last(*) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 1, 10) + + tdSql.query("select last(col1) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + + tdSql.query("select last(col2) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + + tdSql.query("select last(col3) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + + tdSql.query("select last(col4) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + + tdSql.query("select last(col11) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + + tdSql.query("select last(col12) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + + tdSql.query("select last(col13) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + + tdSql.query("select last(col14) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + + tdSql.query("select last(col5) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 9.1) + + tdSql.query("select last(col6) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 9.1) + + tdSql.query("select last(col7) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, True) + + tdSql.query("select last(col8) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 'taosdata10') + + tdSql.query("select last(col9) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '涛思数据10') + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file From 72e8d5cea4b91efa47572b34ebd2ce69dfc46379 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 6 May 2022 10:49:32 +0800 Subject: [PATCH 19/83] refactor:add schemaless test cases --- source/libs/parser/src/parInsert.c | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index 1e90d664a9..2269fd10a5 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -1597,7 +1597,7 @@ static int32_t smlBoundColumns(SArray *cols, SParsedDataColInfo* pColList, SSche return TSDB_CODE_SUCCESS; } -static int32_t smlParseTags(SArray *cols, SKVRowBuilder *tagsBuilder, SParsedDataColInfo* tags, SSchema* pSchema, SVCreateTbReq *createTblReq, SMsgBuf *msg) { +static int32_t smlBoundTags(SArray *cols, SKVRowBuilder *tagsBuilder, SParsedDataColInfo* tags, SSchema* pSchema, SKVRow *row, SMsgBuf *msg) { if (tdInitKVRowBuilder(tagsBuilder) < 0) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } @@ -1611,13 +1611,11 @@ static int32_t smlParseTags(SArray *cols, SKVRowBuilder *tagsBuilder, SParsedDat } - SKVRow row = tdGetKVRowFromBuilder(tagsBuilder); - if(row == NULL){ + *row = tdGetKVRowFromBuilder(tagsBuilder); + if(*row == NULL){ return TSDB_CODE_SML_INVALID_DATA; } - tdSortKVRowByColIdx(row); - createTblReq->type = TD_CHILD_TABLE; - createTblReq->ctbCfg.pTag = row; + tdSortKVRowByColIdx(*row); return TSDB_CODE_SUCCESS; } @@ -1633,11 +1631,14 @@ int32_t smlBindData(void *handle, SArray *tags, SArray *colsFormat, SHashObj *co buildInvalidOperationMsg(&pBuf, "bound tags error"); return ret; } - ret = smlParseTags(tags, &smlHandle->tagsBuilder, &smlHandle->tags, pTagsSchema, &smlHandle->createTblReq, &pBuf); + SKVRow row = NULL; + ret = smlBoundTags(tags, &smlHandle->tagsBuilder, &smlHandle->tags, pTagsSchema, &row, &pBuf); if(ret != TSDB_CODE_SUCCESS){ return ret; } + buildCreateTbReq(&smlHandle->createTblReq, pTableMeta->schema->name, row, pTableMeta->suid); + STableDataBlocks* pDataBlock = NULL; ret = getDataBlockFromList(smlHandle->pBlockHash, pTableMeta->uid, TSDB_DEFAULT_PAYLOAD_SIZE, sizeof(SSubmitBlk), getTableInfo(pTableMeta).rowSize, pTableMeta, @@ -1714,15 +1715,15 @@ int32_t smlBindData(void *handle, SArray *tags, SArray *colsFormat, SHashObj *co kv = *p; } - if (kv->valueLen == 0) { + if (kv->length == 0) { MemRowAppend(&pBuf, NULL, 0, ¶m); } else { int32_t colLen = pColSchema->bytes; if (IS_VAR_DATA_TYPE(pColSchema->type)) { - colLen = kv->valueLen; + colLen = kv->length; } - MemRowAppend(&pBuf, kv->value, colLen, ¶m); + MemRowAppend(&pBuf, &(kv->value), colLen, ¶m); } if (PRIMARYKEY_TIMESTAMP_COL_ID == pColSchema->colId) { From 7e6c12cc396b84fd7aede089ada68ef3901b9fff Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 6 May 2022 11:52:03 +0800 Subject: [PATCH 20/83] refactor:fix error merge from 3.0 --- include/libs/parser/parser.h | 2 +- source/libs/parser/src/parInsert.c | 2 +- source/libs/parser/src/parInsertData.c | 2 +- source/libs/transport/src/rpcTcp.c | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index 4e140d3d4f..0b9fdf94f9 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -97,7 +97,7 @@ int32_t qBindStmtSingleColValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBu int32_t rowNum); int32_t qBuildStmtColFields(void* pDataBlock, int32_t* fieldNum, TAOS_FIELD** fields); int32_t qBuildStmtTagFields(void* pBlock, void* boundTags, int32_t* fieldNum, TAOS_FIELD** fields); -int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, SName* pName, TAOS_MULTI_BIND* bind, +int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, char* tName, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen); void destroyBoundColumnInfo(void* pBoundInfo); int32_t qCreateSName(SName* pName, const char* pTableName, int32_t acctId, char* dbName, char* msgBuf, diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index 33d742d167..17c75dac99 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -1255,7 +1255,7 @@ int32_t qBuildStmtOutput(SQuery* pQuery, SHashObj* pVgHash, SHashObj* pBlockHash return TSDB_CODE_SUCCESS; } -int32_t qBindStmtTagsValue(void *pBlock, void *boundTags, int64_t suid, SName *pName, TAOS_MULTI_BIND *bind, char *msgBuf, int32_t msgBufLen){ +int32_t qBindStmtTagsValue(void *pBlock, void *boundTags, int64_t suid, char *tName, TAOS_MULTI_BIND *bind, char *msgBuf, int32_t msgBufLen){ STableDataBlocks *pDataBlock = (STableDataBlocks *)pBlock; SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; SParsedDataColInfo* tags = (SParsedDataColInfo*)boundTags; diff --git a/source/libs/parser/src/parInsertData.c b/source/libs/parser/src/parInsertData.c index fdb7379152..21d4f13669 100644 --- a/source/libs/parser/src/parInsertData.c +++ b/source/libs/parser/src/parInsertData.c @@ -186,7 +186,7 @@ int32_t buildCreateTbMsg(STableDataBlocks* pBlocks, SVCreateTbReq* pCreateTbReq) } int32_t getDataBlockFromList(SHashObj* pHashList, int64_t id, int32_t size, int32_t startOffset, int32_t rowSize, - const STableMeta* pTableMeta, STableDataBlocks** dataBlocks, SArray* pBlockList, + STableMeta* pTableMeta, STableDataBlocks** dataBlocks, SArray* pBlockList, SVCreateTbReq* pCreateTbReq) { *dataBlocks = NULL; STableDataBlocks** t1 = (STableDataBlocks**)taosHashGet(pHashList, (const char*)&id, sizeof(id)); diff --git a/source/libs/transport/src/rpcTcp.c b/source/libs/transport/src/rpcTcp.c index edc685f498..52c5ddcf63 100644 --- a/source/libs/transport/src/rpcTcp.c +++ b/source/libs/transport/src/rpcTcp.c @@ -21,7 +21,7 @@ #include "taoserror.h" #include "tutil.h" -#ifndef USEf_UV +#ifndef USE_UV typedef struct SFdObj { void * signature; TdSocketPtr pSocket; // TCP socket FD From 971c13719e759fcf3dd2aa5e0a8bf0383293fcf9 Mon Sep 17 00:00:00 2001 From: jiacy-jcy <714897623@qq.com> Date: Fri, 6 May 2022 15:19:33 +0800 Subject: [PATCH 21/83] update last.py --- tests/system-test/2-query/last.py | 322 ++++++++++++++++++++++++------ 1 file changed, 260 insertions(+), 62 deletions(-) diff --git a/tests/system-test/2-query/last.py b/tests/system-test/2-query/last.py index 82e648f60a..fe5beefea5 100644 --- a/tests/system-test/2-query/last.py +++ b/tests/system-test/2-query/last.py @@ -15,115 +15,313 @@ class TDTestCase: def run(self): tdSql.prepare() - tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, + tdSql.execute('''create table stb(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, col7 bool, col8 binary(20), col9 nchar(20), col11 tinyint unsigned, col12 smallint unsigned, col13 int unsigned, col14 bigint unsigned) tags(loc nchar(20))''') - tdSql.execute("create table test1 using test tags('beijing')") - tdSql.execute("insert into test1(ts) values(%d)" % (self.ts - 1)) + tdSql.execute("create table stb_1 using stb tags('beijing')") + tdSql.execute("insert into stb_1(ts) values(%d)" % (self.ts - 1)) # last verifacation - tdSql.query("select last(*) from test1") + tdSql.query("select last(*) from stb_1") tdSql.checkRows(1) tdSql.checkData(0, 1, None) - - tdSql.query("select last(col1) from test1") - tdSql.checkRows(0) - - tdSql.query("select last(col2) from test1") + tdSql.query("select last(*) from db.stb_1") + tdSql.checkRows(1) + tdSql.checkData(0, 1, None) + tdSql.query("select last(col1) from stb_1") + tdSql.checkRows(0) + tdSql.query("select last(col1) from db.stb_1") + tdSql.checkRows(0) + tdSql.query("select last(col2) from stb_1") tdSql.checkRows(0) - - tdSql.query("select last(col3) from test1") + tdSql.query("select last(col2) from db.stb_1") tdSql.checkRows(0) - - tdSql.query("select last(col4) from test1") + tdSql.query("select last(col3) from stb_1") tdSql.checkRows(0) - - tdSql.query("select last(col11) from test1") - tdSql.checkRows(0) - - tdSql.query("select last(col12) from test1") + tdSql.query("select last(col3) from db.stb_1") tdSql.checkRows(0) - - tdSql.query("select last(col13) from test1") + tdSql.query("select last(col4) from stb_1") tdSql.checkRows(0) - - tdSql.query("select last(col14) from test1") + tdSql.query("select last(col4) from db.stb_1") tdSql.checkRows(0) - - tdSql.query("select last(col5) from test1") + tdSql.query("select last(col11) from stb_1") + tdSql.checkRows(0) + tdSql.query("select last(col11) from db.stb_1") + tdSql.checkRows(0) + tdSql.query("select last(col12) from stb_1") tdSql.checkRows(0) - - tdSql.query("select last(col6) from test1") + tdSql.query("select last(col12) from db.stb_1") tdSql.checkRows(0) - - tdSql.query("select last(col7) from test1") + tdSql.query("select last(col13) from stb_1") tdSql.checkRows(0) - - tdSql.query("select last(col8) from test1") + tdSql.query("select last(col13) from db.stb_1") tdSql.checkRows(0) - - tdSql.query("select last(col9) from test1") + tdSql.query("select last(col14) from stb_1") + tdSql.checkRows(0) + tdSql.query("select last(col14) from db.stb_1") + tdSql.checkRows(0) + tdSql.query("select last(col5) from stb_1") + tdSql.checkRows(0) + tdSql.query("select last(col5) from db.stb_1") + tdSql.checkRows(0) + tdSql.query("select last(col6) from stb_1") + tdSql.checkRows(0) + tdSql.query("select last(col6) from db.stb_1") + tdSql.checkRows(0) + tdSql.query("select last(col7) from stb_1") + tdSql.checkRows(0) + tdSql.query("select last(col7) from db.stb_1") + tdSql.checkRows(0) + tdSql.query("select last(col8) from stb_1") + tdSql.checkRows(0) + tdSql.query("select last(col8) from db.stb_1") + tdSql.checkRows(0) + tdSql.query("select last(col9) from stb_1") + tdSql.checkRows(0) + tdSql.query("select last(col9) from db.stb_1") tdSql.checkRows(0) for i in range(self.rowNum): - tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" + tdSql.execute("insert into stb_1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1)) - tdSql.query("select last(*) from test1") + tdSql.query("select last(*) from stb_1") tdSql.checkRows(1) tdSql.checkData(0, 1, 10) - - tdSql.query("select last(col1) from test1") + tdSql.query("select last(*) from db.stb_1") + tdSql.checkRows(1) + tdSql.checkData(0, 1, 10) + tdSql.query("select last(col1) from stb_1") tdSql.checkRows(1) tdSql.checkData(0, 0, 10) - - tdSql.query("select last(col2) from test1") + tdSql.query("select last(col1) from db.stb_1") tdSql.checkRows(1) tdSql.checkData(0, 0, 10) - - tdSql.query("select last(col3) from test1") + tdSql.query("select last(col2) from stb_1") tdSql.checkRows(1) tdSql.checkData(0, 0, 10) - - tdSql.query("select last(col4) from test1") + tdSql.query("select last(col2) from db.stb_1") tdSql.checkRows(1) tdSql.checkData(0, 0, 10) - - tdSql.query("select last(col11) from test1") + tdSql.query("select last(col3) from stb_1") tdSql.checkRows(1) tdSql.checkData(0, 0, 10) - - tdSql.query("select last(col12) from test1") + tdSql.query("select last(col3) from db.stb_1") tdSql.checkRows(1) tdSql.checkData(0, 0, 10) - - tdSql.query("select last(col13) from test1") + tdSql.query("select last(col4) from stb_1") tdSql.checkRows(1) tdSql.checkData(0, 0, 10) - - tdSql.query("select last(col14) from test1") + tdSql.query("select last(col4) from db.stb_1") tdSql.checkRows(1) tdSql.checkData(0, 0, 10) - - tdSql.query("select last(col5) from test1") + tdSql.query("select last(col11) from stb_1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + tdSql.query("select last(col11) from db.stb_1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + tdSql.query("select last(col12) from stb_1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + tdSql.query("select last(col12) from db.stb_1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + tdSql.query("select last(col13) from stb_1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + tdSql.query("select last(col13) from db.stb_1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + tdSql.query("select last(col14) from stb_1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + tdSql.query("select last(col14) from db.stb_1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + tdSql.query("select last(col5) from stb_1") tdSql.checkRows(1) tdSql.checkData(0, 0, 9.1) - - tdSql.query("select last(col6) from test1") + tdSql.query("select last(col5) from db.stb_1") tdSql.checkRows(1) tdSql.checkData(0, 0, 9.1) - - tdSql.query("select last(col7) from test1") + tdSql.query("select last(col6) from stb_1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 9.1) + tdSql.query("select last(col6) from db.stb_1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 9.1) + tdSql.query("select last(col7) from stb_1") tdSql.checkRows(1) tdSql.checkData(0, 0, True) - - tdSql.query("select last(col8) from test1") + tdSql.query("select last(col7) from db.stb_1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, True) + tdSql.query("select last(col8) from stb_1") tdSql.checkRows(1) tdSql.checkData(0, 0, 'taosdata10') - - tdSql.query("select last(col9) from test1") + tdSql.query("select last(col8) from db.stb_1") tdSql.checkRows(1) - tdSql.checkData(0, 0, '涛思数据10') - + tdSql.checkData(0, 0, 'taosdata10') + # tdSql.query("select last(col9) from stb_1") + # tdSql.checkRows(1) + # tdSql.checkData(0, 0, '涛思数据10') + # tdSql.query("select last(col9) from db.stb_1") + # tdSql.checkRows(1) + # tdSql.checkData(0, 0, '涛思数据10') + + + + tdSql.execute('''create table ntb(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, + col7 bool, col8 binary(20), col9 nchar(20), col11 tinyint unsigned, col12 smallint unsigned, col13 int unsigned, col14 bigint unsigned)''') + tdSql.execute("insert into ntb(ts) values(%d)" % (self.ts - 1)) + tdSql.query("select last(*) from ntb") + tdSql.checkRows(1) + tdSql.checkData(0, 1, None) + tdSql.query("select last(*) from db.ntb") + tdSql.checkRows(1) + tdSql.checkData(0, 1, None) + tdSql.query("select last(col1) from ntb") + tdSql.checkRows(0) + tdSql.query("select last(col1) from db.ntb") + tdSql.checkRows(0) + tdSql.query("select last(col2) from ntb") + tdSql.checkRows(0) + tdSql.query("select last(col2) from db.ntb") + tdSql.checkRows(0) + tdSql.query("select last(col3) from ntb") + tdSql.checkRows(0) + tdSql.query("select last(col3) from db.ntb") + tdSql.checkRows(0) + tdSql.query("select last(col4) from ntb") + tdSql.checkRows(0) + tdSql.query("select last(col4) from db.ntb") + tdSql.checkRows(0) + tdSql.query("select last(col11) from ntb") + tdSql.checkRows(0) + tdSql.query("select last(col11) from db.ntb") + tdSql.checkRows(0) + tdSql.query("select last(col12) from ntb") + tdSql.checkRows(0) + tdSql.query("select last(col12) from db.ntb") + tdSql.checkRows(0) + tdSql.query("select last(col13) from ntb") + tdSql.checkRows(0) + tdSql.query("select last(col13) from db.ntb") + tdSql.checkRows(0) + tdSql.query("select last(col14) from ntb") + tdSql.checkRows(0) + tdSql.query("select last(col14) from db.ntb") + tdSql.checkRows(0) + tdSql.query("select last(col5) from ntb") + tdSql.checkRows(0) + tdSql.query("select last(col5) from db.ntb") + tdSql.checkRows(0) + tdSql.query("select last(col6) from ntb") + tdSql.checkRows(0) + tdSql.query("select last(col6) from db.ntb") + tdSql.checkRows(0) + tdSql.query("select last(col7) from ntb") + tdSql.checkRows(0) + tdSql.query("select last(col7) from db.ntb") + tdSql.checkRows(0) + tdSql.query("select last(col8) from ntb") + tdSql.checkRows(0) + tdSql.query("select last(col8) from db.ntb") + tdSql.checkRows(0) + tdSql.query("select last(col9) from ntb") + tdSql.checkRows(0) + tdSql.query("select last(col9) from db.ntb") + tdSql.checkRows(0) + + for i in range(self.rowNum): + tdSql.execute("insert into ntb values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1)) + + tdSql.query("select last(*) from ntb") + tdSql.checkRows(1) + tdSql.checkData(0, 1, 10) + tdSql.query("select last(*) from db.ntb") + tdSql.checkRows(1) + tdSql.checkData(0, 1, 10) + tdSql.query("select last(col1) from ntb") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + tdSql.query("select last(col1) from db.ntb") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + tdSql.query("select last(col2) from ntb") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + tdSql.query("select last(col2) from db.ntb") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + tdSql.query("select last(col3) from ntb") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + tdSql.query("select last(col3) from db.ntb") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + tdSql.query("select last(col4) from ntb") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + tdSql.query("select last(col4) from db.ntb") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + tdSql.query("select last(col11) from ntb") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + tdSql.query("select last(col11) from db.ntb") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + tdSql.query("select last(col12) from ntb") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + tdSql.query("select last(col12) from db.ntb") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + tdSql.query("select last(col13) from ntb") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + tdSql.query("select last(col13) from db.ntb") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + tdSql.query("select last(col14) from ntb") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + tdSql.query("select last(col14) from db.ntb") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + tdSql.query("select last(col5) from ntb") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 9.1) + tdSql.query("select last(col5) from db.ntb") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 9.1) + tdSql.query("select last(col6) from ntb") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 9.1) + tdSql.query("select last(col6) from db.ntb") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 9.1) + tdSql.query("select last(col7) from ntb") + tdSql.checkRows(1) + tdSql.checkData(0, 0, True) + tdSql.query("select last(col7) from db.ntb") + tdSql.checkRows(1) + tdSql.checkData(0, 0, True) + tdSql.query("select last(col8) from ntb") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 'taosdata10') + tdSql.query("select last(col8) from db.ntb") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 'taosdata10') + # tdSql.query("select last(col9) from ntb") + # tdSql.checkRows(1) + # tdSql.checkData(0, 0, '涛思数据10') + # tdSql.query("select last(col9) from db.ntb") + # tdSql.checkRows(1) + # tdSql.checkData(0, 0, '涛思数据10') + def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) From 666b0b676dacec6a56e40fc972d86926f09d1627 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 6 May 2022 15:30:00 +0800 Subject: [PATCH 22/83] fix: alter db replications --- source/dnode/mgmt/mgmt_vnode/src/vmHandle.c | 4 -- source/dnode/mnode/impl/src/mndDb.c | 50 ++++++++++++--------- 2 files changed, 28 insertions(+), 26 deletions(-) diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index e3816396c9..9329e7b15d 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -293,14 +293,10 @@ void vmInitMsgHandle(SMgmtWrapper *pWrapper) { dmSetMsgHandle(pWrapper, TDMT_VND_TASK_MERGE_EXEC, vmProcessMergeMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_VND_TASK_WRITE_EXEC, vmProcessWriteMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_VND_STREAM_TRIGGER, vmProcessFetchMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_VND_ALTER_VNODE, vmProcessWriteMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_VND_COMPACT_VNODE, vmProcessWriteMsg, DEFAULT_HANDLE); - dmSetMsgHandle(pWrapper, TDMT_DND_CREATE_VNODE, vmProcessMgmtMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_DND_DROP_VNODE, vmProcessMgmtMsg, DEFAULT_HANDLE); - // dmSetMsgHandle(pWrapper, TDMT_DND_SYNC_VNODE, vmProcessMgmtMsg, DEFAULT_HANDLE); - // dmSetMsgHandle(pWrapper, TDMT_DND_COMPACT_VNODE, vmProcessMgmtMsg, DEFAULT_HANDLE); // sync integration dmSetMsgHandle(pWrapper, TDMT_VND_SYNC_TIMEOUT, (NodeMsgFp)vmProcessSyncMsg, DEFAULT_HANDLE); diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index af7efb5543..96a8969b56 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -627,6 +627,7 @@ static int32_t mndSetDbCfgFromAlterDbReq(SDbObj *pDb, SAlterDbReq *pAlter) { if (pAlter->replications >= 0 && pAlter->replications != pDb->cfg.replications) { pDb->cfg.replications = pAlter->replications; + pDb->vgVersion++; terrno = 0; } @@ -652,26 +653,30 @@ static int32_t mndSetAlterDbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *p } static int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup) { - for (int32_t vn = 0; vn < pVgroup->replica; ++vn) { - STransAction action = {0}; - SVnodeGid *pVgid = pVgroup->vnodeGid + vn; + if (pVgroup->replica == pDb->cfg.replications) { + for (int32_t vn = 0; vn < pVgroup->replica; ++vn) { + STransAction action = {0}; + SVnodeGid *pVgid = pVgroup->vnodeGid + vn; - SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); - if (pDnode == NULL) return -1; - action.epSet = mndGetDnodeEpset(pDnode); - mndReleaseDnode(pMnode, pDnode); + SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); + if (pDnode == NULL) return -1; + action.epSet = mndGetDnodeEpset(pDnode); + mndReleaseDnode(pMnode, pDnode); - int32_t contLen = 0; - void *pReq = mndBuildAlterVnodeReq(pMnode, pDnode, pDb, pVgroup, &contLen); - if (pReq == NULL) return -1; + int32_t contLen = 0; + void *pReq = mndBuildAlterVnodeReq(pMnode, pDnode, pDb, pVgroup, &contLen); + if (pReq == NULL) return -1; - action.pCont = pReq; - action.contLen = contLen; - action.msgType = TDMT_VND_ALTER_VNODE; - if (mndTransAppendRedoAction(pTrans, &action) != 0) { - taosMemoryFree(pReq); - return -1; + action.pCont = pReq; + action.contLen = contLen; + action.msgType = TDMT_VND_ALTER_VNODE; + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + taosMemoryFree(pReq); + return -1; + } } + } else if (pVgroup->replica < pDb->cfg.replications) { + } else { } return 0; @@ -726,6 +731,7 @@ static int32_t mndProcessAlterDbReq(SNodeMsg *pReq) { SDbObj *pDb = NULL; SUserObj *pUser = NULL; SAlterDbReq alterReq = {0}; + SDbObj dbObj = {0}; if (tDeserializeSAlterDbReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &alterReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; @@ -749,15 +755,14 @@ static int32_t mndProcessAlterDbReq(SNodeMsg *pReq) { goto _OVER; } - SDbObj dbObj = {0}; memcpy(&dbObj, pDb, sizeof(SDbObj)); - dbObj.cfg.numOfRetensions = 0; - dbObj.cfg.pRetensions = NULL; + if (dbObj.cfg.pRetensions != NULL) { + dbObj.cfg.pRetensions = taosArrayDup(pDb->cfg.pRetensions); + if (dbObj.cfg.pRetensions == NULL) goto _OVER; + } code = mndSetDbCfgFromAlterDbReq(&dbObj, &alterReq); - if (code != 0) { - goto _OVER; - } + if (code != 0) goto _OVER; dbObj.cfgVersion++; dbObj.updateTime = taosGetTimestampMs(); @@ -771,6 +776,7 @@ _OVER: mndReleaseDb(pMnode, pDb); mndReleaseUser(pMnode, pUser); + taosArrayDestroy(dbObj.cfg.pRetensions); return code; } From d12532b4d24527fef9a57f1a20518265e15a1a59 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 6 May 2022 16:35:24 +0800 Subject: [PATCH 23/83] feature/qnodew --- source/dnode/mgmt/mgmt_qnode/src/qmHandle.c | 1 + source/dnode/qnode/src/qnode.c | 2 + tests/script/api/batchprepare.c | 131 +++++++++++++++++++- 3 files changed, 130 insertions(+), 4 deletions(-) diff --git a/source/dnode/mgmt/mgmt_qnode/src/qmHandle.c b/source/dnode/mgmt/mgmt_qnode/src/qmHandle.c index 805b4c7264..11b91f0568 100644 --- a/source/dnode/mgmt/mgmt_qnode/src/qmHandle.c +++ b/source/dnode/mgmt/mgmt_qnode/src/qmHandle.c @@ -101,6 +101,7 @@ void qmInitMsgHandle(SMgmtWrapper *pWrapper) { dmSetMsgHandle(pWrapper, TDMT_VND_QUERY_CONTINUE, qmProcessQueryMsg, QNODE_HANDLE); dmSetMsgHandle(pWrapper, TDMT_VND_FETCH, qmProcessFetchMsg, QNODE_HANDLE); dmSetMsgHandle(pWrapper, TDMT_VND_FETCH_RSP, qmProcessFetchMsg, QNODE_HANDLE); + dmSetMsgHandle(pWrapper, TDMT_VND_QUERY_HEARTBEAT, qmProcessFetchMsg, QNODE_HANDLE); dmSetMsgHandle(pWrapper, TDMT_VND_RES_READY, qmProcessFetchMsg, QNODE_HANDLE); dmSetMsgHandle(pWrapper, TDMT_VND_TASKS_STATUS, qmProcessFetchMsg, QNODE_HANDLE); diff --git a/source/dnode/qnode/src/qnode.c b/source/dnode/qnode/src/qnode.c index 7a226a4c6b..b21141001a 100644 --- a/source/dnode/qnode/src/qnode.c +++ b/source/dnode/qnode/src/qnode.c @@ -84,6 +84,8 @@ int32_t qndProcessFetchMsg(SQnode *pQnode, SRpcMsg *pMsg) { // return vnodeGetTableMeta(pQnode, pMsg); case TDMT_VND_CONSUME: // return tqProcessConsumeReq(pQnode->pTq, pMsg); + case TDMT_VND_QUERY_HEARTBEAT: + return qWorkerProcessHbMsg(pQnode, pQnode->pQuery, pMsg); default: qError("unknown msg type:%d in fetch queue", pMsg->msgType); return TSDB_CODE_VND_APP_ERROR; diff --git a/tests/script/api/batchprepare.c b/tests/script/api/batchprepare.c index 828216d1d3..99e546fcd0 100644 --- a/tests/script/api/batchprepare.c +++ b/tests/script/api/batchprepare.c @@ -398,7 +398,7 @@ void bpAppendOperatorParam(BindData *data, int32_t *len, int32_t dataType, int32 } } -void generateQuerySQL(BindData *data, int32_t tblIdx) { +void generateQueryCondSQL(BindData *data, int32_t tblIdx) { int32_t len = sprintf(data->sql, "select * from %s%d where ", bpTbPrefix, tblIdx); if (!gCurCase->fullCol) { for (int c = 0; c < gCurCase->bindColNum; ++c) { @@ -462,6 +462,72 @@ void generateQuerySQL(BindData *data, int32_t tblIdx) { } } + +void generateQueryMiscSQL(BindData *data, int32_t tblIdx) { + int32_t len = sprintf(data->sql, "select * from %s%d where ", bpTbPrefix, tblIdx); + if (!gCurCase->fullCol) { + for (int c = 0; c < gCurCase->bindColNum; ++c) { + if (c) { + len += sprintf(data->sql + len, " and "); + } + switch (data->pBind[c].buffer_type) { + case TSDB_DATA_TYPE_BOOL: + len += sprintf(data->sql + len, "booldata"); + break; + case TSDB_DATA_TYPE_TINYINT: + len += sprintf(data->sql + len, "tinydata"); + break; + case TSDB_DATA_TYPE_SMALLINT: + len += sprintf(data->sql + len, "smalldata"); + break; + case TSDB_DATA_TYPE_INT: + len += sprintf(data->sql + len, "intdata"); + break; + case TSDB_DATA_TYPE_BIGINT: + len += sprintf(data->sql + len, "bigdata"); + break; + case TSDB_DATA_TYPE_FLOAT: + len += sprintf(data->sql + len, "floatdata"); + break; + case TSDB_DATA_TYPE_DOUBLE: + len += sprintf(data->sql + len, "doubledata"); + break; + case TSDB_DATA_TYPE_VARCHAR: + len += sprintf(data->sql + len, "binarydata"); + break; + case TSDB_DATA_TYPE_TIMESTAMP: + len += sprintf(data->sql + len, "ts"); + break; + case TSDB_DATA_TYPE_NCHAR: + len += sprintf(data->sql + len, "nchardata"); + break; + case TSDB_DATA_TYPE_UTINYINT: + len += sprintf(data->sql + len, "utinydata"); + break; + case TSDB_DATA_TYPE_USMALLINT: + len += sprintf(data->sql + len, "usmalldata"); + break; + case TSDB_DATA_TYPE_UINT: + len += sprintf(data->sql + len, "uintdata"); + break; + case TSDB_DATA_TYPE_UBIGINT: + len += sprintf(data->sql + len, "ubigdata"); + break; + default: + printf("!!!invalid col type:%d", data->pBind[c].buffer_type); + exit(1); + } + + bpAppendOperatorParam(data, &len, data->pBind[c].buffer_type, c); + } + } + + if (gCaseCtrl.printStmtSql) { + printf("\tSTMT SQL: %s\n", data->sql); + } +} + + void generateErrorSQL(BindData *data, int32_t tblIdx) { int32_t len = 0; data->sql = taosMemoryCalloc(1, 1024); @@ -677,7 +743,7 @@ int32_t prepareInsertData(BindData *data) { return 0; } -int32_t prepareQueryData(BindData *data, int32_t tblIdx) { +int32_t prepareQueryCondData(BindData *data, int32_t tblIdx) { static int64_t tsData = 1591060628000; uint64_t bindNum = gCurCase->rowNum / gCurCase->bindRowNum; @@ -735,6 +801,63 @@ int32_t prepareQueryData(BindData *data, int32_t tblIdx) { } +int32_t prepareQueryMiscData(BindData *data, int32_t tblIdx) { + static int64_t tsData = 1591060628000; + uint64_t bindNum = gCurCase->rowNum / gCurCase->bindRowNum; + + data->colNum = 0; + data->colTypes = taosMemoryCalloc(30, sizeof(int32_t)); + data->sql = taosMemoryCalloc(1, 1024); + data->pBind = taosMemoryCalloc(bindNum*gCurCase->bindColNum, sizeof(TAOS_MULTI_BIND)); + data->tsData = taosMemoryMalloc(bindNum * sizeof(int64_t)); + data->boolData = taosMemoryMalloc(bindNum * sizeof(bool)); + data->tinyData = taosMemoryMalloc(bindNum * sizeof(int8_t)); + data->utinyData = taosMemoryMalloc(bindNum * sizeof(uint8_t)); + data->smallData = taosMemoryMalloc(bindNum * sizeof(int16_t)); + data->usmallData = taosMemoryMalloc(bindNum * sizeof(uint16_t)); + data->intData = taosMemoryMalloc(bindNum * sizeof(int32_t)); + data->uintData = taosMemoryMalloc(bindNum * sizeof(uint32_t)); + data->bigData = taosMemoryMalloc(bindNum * sizeof(int64_t)); + data->ubigData = taosMemoryMalloc(bindNum * sizeof(uint64_t)); + data->floatData = taosMemoryMalloc(bindNum * sizeof(float)); + data->doubleData = taosMemoryMalloc(bindNum * sizeof(double)); + data->binaryData = taosMemoryMalloc(bindNum * gVarCharSize); + data->binaryLen = taosMemoryMalloc(bindNum * sizeof(int32_t)); + if (gCurCase->bindNullNum) { + data->isNull = taosMemoryCalloc(bindNum, sizeof(char)); + } + + for (int32_t i = 0; i < bindNum; ++i) { + data->tsData[i] = tsData + tblIdx*gCurCase->rowNum + rand()%gCurCase->rowNum; + data->boolData[i] = (bool)(tblIdx*gCurCase->rowNum + rand() % gCurCase->rowNum); + data->tinyData[i] = (int8_t)(tblIdx*gCurCase->rowNum + rand() % gCurCase->rowNum); + data->utinyData[i] = (uint8_t)(tblIdx*gCurCase->rowNum + rand() % gCurCase->rowNum); + data->smallData[i] = (int16_t)(tblIdx*gCurCase->rowNum + rand() % gCurCase->rowNum); + data->usmallData[i] = (uint16_t)(tblIdx*gCurCase->rowNum + rand() % gCurCase->rowNum); + data->intData[i] = (int32_t)(tblIdx*gCurCase->rowNum + rand() % gCurCase->rowNum); + data->uintData[i] = (uint32_t)(tblIdx*gCurCase->rowNum + rand() % gCurCase->rowNum); + data->bigData[i] = (int64_t)(tblIdx*gCurCase->rowNum + rand() % gCurCase->rowNum); + data->ubigData[i] = (uint64_t)(tblIdx*gCurCase->rowNum + rand() % gCurCase->rowNum); + data->floatData[i] = (float)(tblIdx*gCurCase->rowNum + rand() % gCurCase->rowNum); + data->doubleData[i] = (double)(tblIdx*gCurCase->rowNum + rand() % gCurCase->rowNum); + memset(data->binaryData + gVarCharSize * i, 'a'+i%26, gVarCharLen); + if (gCurCase->bindNullNum) { + data->isNull[i] = i % 2; + } + data->binaryLen[i] = gVarCharLen; + } + + for (int b = 0; b < bindNum; b++) { + for (int c = 0; c < gCurCase->bindColNum; ++c) { + prepareColData(data, b*gCurCase->bindColNum+c, b*gCurCase->bindRowNum, c); + } + } + + generateQueryMiscSQL(data, tblIdx); + + return 0; +} + void destroyData(BindData *data) { @@ -1385,7 +1508,7 @@ int querySUBTTest1(TAOS_STMT *stmt, TAOS *taos) { for (int32_t t = 0; t< gCurCase->tblNum; ++t) { memset(&data, 0, sizeof(data)); - prepareQueryData(&data, t); + prepareQueryCondData(&data, t); int code = taos_stmt_prepare(stmt, data.sql, 0); if (code != 0){ @@ -1431,7 +1554,7 @@ int querySUBTTest2(TAOS_STMT *stmt, TAOS *taos) { for (int32_t t = 0; t< gCurCase->tblNum; ++t) { memset(&data, 0, sizeof(data)); - prepareQueryData(&data, t); + prepareQueryMiscData(&data, t); int code = taos_stmt_prepare(stmt, data.sql, 0); if (code != 0){ From b744c0f524da7dda0f394377184f1159210a921e Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 6 May 2022 16:42:02 +0800 Subject: [PATCH 24/83] refactor:fix schemaless error in table name,vgId,uid --- include/libs/parser/parser.h | 2 +- source/client/src/clientSml.c | 5 +++-- source/libs/parser/src/parInsert.c | 7 ++++--- source/libs/parser/src/parInsertData.c | 1 + 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index 0b9fdf94f9..7c9602734b 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -105,7 +105,7 @@ int32_t qCreateSName(SName* pName, const char* pTableName, int32_t acctId, char* void* smlInitHandle(SQuery *pQuery); void smlDestroyHandle(void *pHandle); -int32_t smlBindData(void *handle, SArray *tags, SArray *colsFormat, SHashObj *colsHash, SArray *cols, bool format, STableMeta *pTableMeta, char *msgBuf, int16_t msgBufLen); +int32_t smlBindData(void *handle, SArray *tags, SArray *colsFormat, SHashObj *colsHash, SArray *cols, bool format, STableMeta *pTableMeta, char *tableName, char *msgBuf, int16_t msgBufLen); int32_t smlBuildOutput(void* handle, SHashObj* pVgHash); #ifdef __cplusplus diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 55e95c2aa5..8b5712a619 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -143,7 +143,7 @@ static int32_t smlBuildInvalidDataMsg(SSmlMsgBuf* pBuf, const char *msg1, const } static int smlCompareKv(const void* p1, const void* p2) { - SSmlKv* kv1 = (SSmlKv *)p1; + SSmlKv* kv1 = (SSmlKv*)p1; SSmlKv* kv2 = (SSmlKv*)p2; int32_t kvLen1 = kv1->keyLen; int32_t kvLen2 = kv2->keyLen; @@ -1647,11 +1647,12 @@ static int32_t smlInsertData(SSmlHandle* info) { SSmlSTableMeta** pMeta = taosHashGet(info->superTables, tableData->sTableName, tableData->sTableNameLen); ASSERT (NULL != pMeta && NULL != *pMeta); + // use tablemeta of stable to save vgid and uid of child table (*pMeta)->tableMeta->vgId = vg.vgId; (*pMeta)->tableMeta->uid = tableData->uid; // one table merge data block together according uid code = smlBindData(info->exec, tableData->tags, tableData->colsFormat, tableData->columnsHash, - tableData->cols, info->dataFormat, (*pMeta)->tableMeta, info->msgBuf.buf, info->msgBuf.len); + tableData->cols, info->dataFormat, (*pMeta)->tableMeta, tableData->childTableName, info->msgBuf.buf, info->msgBuf.len); if(code != TSDB_CODE_SUCCESS){ return code; } diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index 4da5fcda69..05d247c037 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -279,6 +279,7 @@ static void buildMsgHeader(STableDataBlocks* src, SVgDataBlocks* blocks) { int32_t numOfBlocks = blocks->numOfTables; while (numOfBlocks--) { int32_t dataLen = blk->dataLen; + int32_t schemaLen = blk->schemaLen; blk->uid = htobe64(blk->uid); blk->suid = htobe64(blk->suid); blk->padding = htonl(blk->padding); @@ -286,7 +287,7 @@ static void buildMsgHeader(STableDataBlocks* src, SVgDataBlocks* blocks) { blk->dataLen = htonl(blk->dataLen); blk->schemaLen = htonl(blk->schemaLen); blk->numOfRows = htons(blk->numOfRows); - blk = (SSubmitBlk*)(blk->data + dataLen); + blk = (SSubmitBlk*)(blk->data + schemaLen + dataLen); } } @@ -1642,7 +1643,7 @@ static int32_t smlBoundTags(SArray *cols, SKVRowBuilder *tagsBuilder, SParsedDat } int32_t smlBindData(void *handle, SArray *tags, SArray *colsFormat, SHashObj *colsHash, SArray *cols, bool format, - STableMeta *pTableMeta, char *msgBuf, int16_t msgBufLen) { + STableMeta *pTableMeta, char *tableName, char *msgBuf, int16_t msgBufLen) { SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; SSmlExecHandle *smlHandle = (SSmlExecHandle *)handle; @@ -1659,7 +1660,7 @@ int32_t smlBindData(void *handle, SArray *tags, SArray *colsFormat, SHashObj *co return ret; } - buildCreateTbReq(&smlHandle->createTblReq, pTableMeta->schema->name, row, pTableMeta->suid); + buildCreateTbReq(&smlHandle->createTblReq, tableName, row, pTableMeta->suid); STableDataBlocks* pDataBlock = NULL; ret = getDataBlockFromList(smlHandle->pBlockHash, pTableMeta->uid, TSDB_DEFAULT_PAYLOAD_SIZE, diff --git a/source/libs/parser/src/parInsertData.c b/source/libs/parser/src/parInsertData.c index d24fca757e..ac197077a4 100644 --- a/source/libs/parser/src/parInsertData.c +++ b/source/libs/parser/src/parInsertData.c @@ -455,6 +455,7 @@ int32_t mergeTableDataBlocks(SHashObj* pHashObj, uint8_t payloadType, SArray** p SSubmitBlk* pBlocks = (SSubmitBlk*)pOneTableBlock->pData; if (pBlocks->numOfRows > 0) { STableDataBlocks* dataBuf = NULL; + pOneTableBlock->pTableMeta->vgId = pOneTableBlock->vgId; // for schemaless, restore origin vgId int32_t ret = getDataBlockFromList(pVnodeDataBlockHashList, pOneTableBlock->vgId, TSDB_PAYLOAD_SIZE, INSERT_HEAD_SIZE, 0, pOneTableBlock->pTableMeta, &dataBuf, pVnodeDataBlockList, NULL); From b3ec7b895a2c486085b2f0900c9bfa52c1f07e9b Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Fri, 6 May 2022 16:50:25 +0800 Subject: [PATCH 25/83] add case for math function sqrt --- tests/system-test/2-query/sqrt.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/system-test/2-query/sqrt.py b/tests/system-test/2-query/sqrt.py index b41a41010e..2d72a5170b 100644 --- a/tests/system-test/2-query/sqrt.py +++ b/tests/system-test/2-query/sqrt.py @@ -498,6 +498,7 @@ class TDTestCase: tdSql.checkData(3,3,math.sqrt(126)) tdSql.checkData(3,4,math.sqrt(339999995214436424907732413799364296704.00000)) + # check + - * / in functions tdSql.query("select sqrt(abs(c1+1)) ,sqrt(abs(c2)) , sqrt(abs(c3*1)) , sqrt(abs(c4/2)), sqrt(abs(c5))/2, sqrt(abs(c6)) from sub1_bound ") tdSql.checkData(0,0,math.sqrt(2147483648.000000000)) From ae23dd2382a5d8a06dbb4e281bea2901c898ee80 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 6 May 2022 17:28:58 +0800 Subject: [PATCH 26/83] fix: alter db replications --- source/dnode/mnode/impl/inc/mndVgroup.h | 5 +- source/dnode/mnode/impl/src/mndDb.c | 235 +++++++++++++++--------- source/dnode/mnode/impl/src/mndVgroup.c | 8 + source/dnode/mnode/impl/test/db/db.cpp | 13 +- 4 files changed, 173 insertions(+), 88 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndVgroup.h b/source/dnode/mnode/impl/inc/mndVgroup.h index 8aecc22454..87be15a4fd 100644 --- a/source/dnode/mnode/impl/inc/mndVgroup.h +++ b/source/dnode/mnode/impl/inc/mndVgroup.h @@ -27,10 +27,13 @@ void mndCleanupVgroup(SMnode *pMnode); SVgObj *mndAcquireVgroup(SMnode *pMnode, int32_t vgId); void mndReleaseVgroup(SMnode *pMnode, SVgObj *pVgroup); SSdbRaw *mndVgroupActionEncode(SVgObj *pVgroup); -int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups); SEpSet mndGetVgroupEpset(SMnode *pMnode, const SVgObj *pVgroup); int32_t mndGetVnodesNum(SMnode *pMnode, int32_t dnodeId); +int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups); +int32_t mndAddVnodeToVgroup(SMnode *pMnode, SVgObj *pVgroup, SVnodeGid *new1, SVnodeGid *new2, SVnodeGid *exist); +int32_t mndRemoveVnodeFromVgroup(SMnode *pMnode, SVgObj *pVgroup, SVnodeGid *del1, SVnodeGid *del2, SVnodeGid *exist); + void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen); void *mndBuildDropVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen); void *mndBuildAlterVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen); diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 96a8969b56..cf9dd664e3 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -261,6 +261,104 @@ void mndReleaseDb(SMnode *pMnode, SDbObj *pDb) { sdbRelease(pSdb, pDb); } +static int32_t mndAddCreateVnodeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SVnodeGid *pVgid, + bool isRedo) { + STransAction action = {0}; + + SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); + if (pDnode == NULL) return -1; + action.epSet = mndGetDnodeEpset(pDnode); + mndReleaseDnode(pMnode, pDnode); + + int32_t contLen = 0; + void *pReq = mndBuildCreateVnodeReq(pMnode, pDnode, pDb, pVgroup, &contLen); + if (pReq == NULL) return -1; + + action.pCont = pReq; + action.contLen = contLen; + action.msgType = TDMT_DND_CREATE_VNODE; + action.acceptableCode = TSDB_CODE_NODE_ALREADY_DEPLOYED; + + if (isRedo) { + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + taosMemoryFree(pReq); + return -1; + } + } else { + if (mndTransAppendUndoAction(pTrans, &action) != 0) { + taosMemoryFree(pReq); + return -1; + } + } + + return 0; +} + +static int32_t mndAddAlterVnodeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SVnodeGid *pVgid, + bool isRedo) { + STransAction action = {0}; + + SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); + if (pDnode == NULL) return -1; + action.epSet = mndGetDnodeEpset(pDnode); + mndReleaseDnode(pMnode, pDnode); + + int32_t contLen = 0; + void *pReq = mndBuildAlterVnodeReq(pMnode, pDnode, pDb, pVgroup, &contLen); + if (pReq == NULL) return -1; + + action.pCont = pReq; + action.contLen = contLen; + action.msgType = TDMT_VND_ALTER_VNODE; + + if (isRedo) { + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + taosMemoryFree(pReq); + return -1; + } + } else { + if (mndTransAppendUndoAction(pTrans, &action) != 0) { + taosMemoryFree(pReq); + return -1; + } + } + + return 0; +} + +static int32_t mndAddDropVnodeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SVnodeGid *pVgid, + bool isRedo) { + STransAction action = {0}; + + SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); + if (pDnode == NULL) return -1; + action.epSet = mndGetDnodeEpset(pDnode); + mndReleaseDnode(pMnode, pDnode); + + int32_t contLen = 0; + void *pReq = mndBuildDropVnodeReq(pMnode, pDnode, pDb, pVgroup, &contLen); + if (pReq == NULL) return -1; + + action.pCont = pReq; + action.contLen = contLen; + action.msgType = TDMT_DND_DROP_VNODE; + action.acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED; + + if (isRedo) { + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + taosMemoryFree(pReq); + return -1; + } + } else { + if (mndTransAppendUndoAction(pTrans, &action) != 0) { + taosMemoryFree(pReq); + return -1; + } + } + + return 0; +} + static int32_t mndCheckDbName(const char *dbName, SUserObj *pUser) { char *pos = strstr(dbName, TS_PATH_DELIMITER); if (pos == NULL) { @@ -278,6 +376,8 @@ static int32_t mndCheckDbName(const char *dbName, SUserObj *pUser) { } static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg) { + terrno = TSDB_CODE_MND_INVALID_DB_OPTION; + if (pCfg->numOfVgroups < TSDB_MIN_VNODES_PER_DB || pCfg->numOfVgroups > TSDB_MAX_VNODES_PER_DB) return -1; if (pCfg->numOfStables < TSDB_DB_STREAM_MODE_OFF || pCfg->numOfStables > TSDB_DB_STREAM_MODE_ON) return -1; if (pCfg->buffer < TSDB_MIN_BUFFER_PER_VNODE || pCfg->buffer > TSDB_MAX_BUFFER_PER_VNODE) return -1; @@ -300,9 +400,10 @@ static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg) { if (pCfg->replications < TSDB_MIN_DB_REPLICA || pCfg->replications > TSDB_MAX_DB_REPLICA) return -1; if (pCfg->replications > mndGetDnodeSize(pMnode)) return -1; if (pCfg->strict < TSDB_DB_STRICT_OFF || pCfg->strict > TSDB_DB_STRICT_ON) return -1; - if (pCfg->strict > pCfg->replications) return -1; if (pCfg->cacheLastRow < TSDB_MIN_DB_CACHE_LAST_ROW || pCfg->cacheLastRow > TSDB_MAX_DB_CACHE_LAST_ROW) return -1; if (pCfg->hashMethod != 1) return -1; + + terrno = 0; return TSDB_CODE_SUCCESS; } @@ -381,24 +482,8 @@ static int32_t mndSetCreateDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj SVgObj *pVgroup = pVgroups + vg; for (int32_t vn = 0; vn < pVgroup->replica; ++vn) { - STransAction action = {0}; - SVnodeGid *pVgid = pVgroup->vnodeGid + vn; - - SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); - if (pDnode == NULL) return -1; - action.epSet = mndGetDnodeEpset(pDnode); - mndReleaseDnode(pMnode, pDnode); - - int32_t contLen = 0; - void *pReq = mndBuildCreateVnodeReq(pMnode, pDnode, pDb, pVgroup, &contLen); - if (pReq == NULL) return -1; - - action.pCont = pReq; - action.contLen = contLen; - action.msgType = TDMT_DND_CREATE_VNODE; - action.acceptableCode = TSDB_CODE_NODE_ALREADY_DEPLOYED; - if (mndTransAppendRedoAction(pTrans, &action) != 0) { - taosMemoryFree(pReq); + SVnodeGid *pVgid = pVgroup->vnodeGid + vn; + if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, pVgroup, pVgid, true) != 0) { return -1; } } @@ -412,24 +497,8 @@ static int32_t mndSetCreateDbUndoActions(SMnode *pMnode, STrans *pTrans, SDbObj SVgObj *pVgroup = pVgroups + vg; for (int32_t vn = 0; vn < pVgroup->replica; ++vn) { - STransAction action = {0}; - SVnodeGid *pVgid = pVgroup->vnodeGid + vn; - - SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); - if (pDnode == NULL) return -1; - action.epSet = mndGetDnodeEpset(pDnode); - mndReleaseDnode(pMnode, pDnode); - - int32_t contLen = 0; - void *pReq = mndBuildDropVnodeReq(pMnode, pDnode, pDb, pVgroup, &contLen); - if (pReq == NULL) return -1; - - action.pCont = pReq; - action.contLen = contLen; - action.msgType = TDMT_DND_DROP_VNODE; - action.acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED; - if (mndTransAppendUndoAction(pTrans, &action) != 0) { - taosMemoryFree(pReq); + SVnodeGid *pVgid = pVgroup->vnodeGid + vn; + if (mndAddDropVnodeAction(pMnode, pTrans, pDb, pVgroup, pVgid, false) != 0) { return -1; } } @@ -482,7 +551,6 @@ static int32_t mndCreateDb(SMnode *pMnode, SNodeMsg *pReq, SCreateDbReq *pCreate } if (mndCheckDbCfg(pMnode, &dbObj.cfg) != 0) { - terrno = TSDB_CODE_MND_INVALID_DB_OPTION; mError("db:%s, failed to create since %s", pCreate->db, terrstr()); return -1; } @@ -570,37 +638,37 @@ _OVER: static int32_t mndSetDbCfgFromAlterDbReq(SDbObj *pDb, SAlterDbReq *pAlter) { terrno = TSDB_CODE_MND_DB_OPTION_UNCHANGED; - if (pAlter->buffer >= 0 && pAlter->buffer != pDb->cfg.buffer) { + if (pAlter->buffer > 0 && pAlter->buffer != pDb->cfg.buffer) { pDb->cfg.buffer = pAlter->buffer; terrno = 0; } - if (pAlter->pages >= 0 && pAlter->pages != pDb->cfg.pages) { + if (pAlter->pages > 0 && pAlter->pages != pDb->cfg.pages) { pDb->cfg.pages = pAlter->pages; terrno = 0; } - if (pAlter->pageSize >= 0 && pAlter->pageSize != pDb->cfg.pageSize) { + if (pAlter->pageSize > 0 && pAlter->pageSize != pDb->cfg.pageSize) { pDb->cfg.pageSize = pAlter->pageSize; terrno = 0; } - if (pAlter->daysPerFile >= 0 && pAlter->daysPerFile != pDb->cfg.daysPerFile) { + if (pAlter->daysPerFile > 0 && pAlter->daysPerFile != pDb->cfg.daysPerFile) { pDb->cfg.daysPerFile = pAlter->daysPerFile; terrno = 0; } - if (pAlter->daysToKeep0 >= 0 && pAlter->daysToKeep0 != pDb->cfg.daysToKeep0) { + if (pAlter->daysToKeep0 > 0 && pAlter->daysToKeep0 != pDb->cfg.daysToKeep0) { pDb->cfg.daysToKeep0 = pAlter->daysToKeep0; terrno = 0; } - if (pAlter->daysToKeep1 >= 0 && pAlter->daysToKeep1 != pDb->cfg.daysToKeep1) { + if (pAlter->daysToKeep1 > 0 && pAlter->daysToKeep1 != pDb->cfg.daysToKeep1) { pDb->cfg.daysToKeep1 = pAlter->daysToKeep1; terrno = 0; } - if (pAlter->daysToKeep2 >= 0 && pAlter->daysToKeep2 != pDb->cfg.daysToKeep2) { + if (pAlter->daysToKeep2 > 0 && pAlter->daysToKeep2 != pDb->cfg.daysToKeep2) { pDb->cfg.daysToKeep2 = pAlter->daysToKeep2; terrno = 0; } @@ -625,7 +693,7 @@ static int32_t mndSetDbCfgFromAlterDbReq(SDbObj *pDb, SAlterDbReq *pAlter) { terrno = 0; } - if (pAlter->replications >= 0 && pAlter->replications != pDb->cfg.replications) { + if (pAlter->replications > 0 && pAlter->replications != pDb->cfg.replications) { pDb->cfg.replications = pAlter->replications; pDb->vgVersion++; terrno = 0; @@ -653,30 +721,44 @@ static int32_t mndSetAlterDbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *p } static int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup) { - if (pVgroup->replica == pDb->cfg.replications) { + if (pVgroup->replica <= 0 || pVgroup->replica == pDb->cfg.replications) { for (int32_t vn = 0; vn < pVgroup->replica; ++vn) { - STransAction action = {0}; - SVnodeGid *pVgid = pVgroup->vnodeGid + vn; - - SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); - if (pDnode == NULL) return -1; - action.epSet = mndGetDnodeEpset(pDnode); - mndReleaseDnode(pMnode, pDnode); - - int32_t contLen = 0; - void *pReq = mndBuildAlterVnodeReq(pMnode, pDnode, pDb, pVgroup, &contLen); - if (pReq == NULL) return -1; - - action.pCont = pReq; - action.contLen = contLen; - action.msgType = TDMT_VND_ALTER_VNODE; - if (mndTransAppendRedoAction(pTrans, &action) != 0) { - taosMemoryFree(pReq); + SVnodeGid *pVgid = pVgroup->vnodeGid + vn; + if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, pVgroup, pVgid, true) != 0) { return -1; } } - } else if (pVgroup->replica < pDb->cfg.replications) { } else { + SVgObj newVgroup = {0}; + memcpy(&newVgroup, pVgroup, sizeof(SVgObj)); + if (newVgroup.replica < pDb->cfg.replications) { + SVnodeGid new1 = {0}; + SVnodeGid new2 = {0}; + SVnodeGid exist = {0}; + if (mndAddVnodeToVgroup(pMnode, &newVgroup, &new1, &new2, &exist) != 0) { + mError("db:%s, failed to add vnode to vgId:%d since %s", pDb->name, newVgroup.vgId, terrstr()); + return -1; + } + if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVgroup, &new1, true) != 0) return -1; + if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVgroup, &new2, true) != 0) return -1; + if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVgroup, &exist, true) != 0) return -1; + } else { + SVnodeGid del1 = {0}; + SVnodeGid del2 = {0}; + SVnodeGid exist = {0}; + if (mndRemoveVnodeFromVgroup(pMnode, &newVgroup, &del1, &del2, &exist) != 0) { + mError("db:%s, failed to remove vnode from vgId:%d since %s", pDb->name, newVgroup.vgId, terrstr()); + return -1; + } + if (mndAddDropVnodeAction(pMnode, pTrans, pDb, &newVgroup, &del1, true) != 0) return -1; + if (mndAddDropVnodeAction(pMnode, pTrans, pDb, &newVgroup, &del2, true) != 0) return -1; + if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVgroup, &exist, true) != 0) return -1; + } + + SSdbRaw *pVgRaw = mndVgroupActionEncode(&newVgroup); + if (pVgRaw == NULL) return -1; + if (mndTransAppendCommitlog(pTrans, pVgRaw) != 0) return -1; + if (sdbSetRawStatus(pVgRaw, SDB_STATUS_READY) != 0) return -1; } return 0; @@ -764,6 +846,9 @@ static int32_t mndProcessAlterDbReq(SNodeMsg *pReq) { code = mndSetDbCfgFromAlterDbReq(&dbObj, &alterReq); if (code != 0) goto _OVER; + code = mndCheckDbCfg(pMnode, &dbObj.cfg); + if (code != 0) goto _OVER; + dbObj.cfgVersion++; dbObj.updateTime = taosGetTimestampMs(); code = mndAlterDb(pMnode, pReq, pDb, &dbObj); @@ -905,24 +990,8 @@ static int32_t mndSetDropDbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pD static int32_t mndBuildDropVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup) { for (int32_t vn = 0; vn < pVgroup->replica; ++vn) { - STransAction action = {0}; - SVnodeGid *pVgid = pVgroup->vnodeGid + vn; - - SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); - if (pDnode == NULL) return -1; - action.epSet = mndGetDnodeEpset(pDnode); - mndReleaseDnode(pMnode, pDnode); - - int32_t contLen = 0; - void *pReq = mndBuildDropVnodeReq(pMnode, pDnode, pDb, pVgroup, &contLen); - if (pReq == NULL) return -1; - - action.pCont = pReq; - action.contLen = contLen; - action.msgType = TDMT_DND_DROP_VNODE; - action.acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED; - if (mndTransAppendRedoAction(pTrans, &action) != 0) { - taosMemoryFree(pReq); + SVnodeGid *pVgid = pVgroup->vnodeGid + vn; + if (mndAddDropVnodeAction(pMnode, pTrans, pDb, pVgroup, pVgid, true) != 0) { return -1; } } diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index d1e4be1161..ca60a3b9ed 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -491,6 +491,14 @@ _OVER: return code; } +int32_t mndAddVnodeToVgroup(SMnode *pMnode, SVgObj *pVgroup, SVnodeGid *new1, SVnodeGid *new2, SVnodeGid *exist) { + return 0; +} + +int32_t mndRemoveVnodeFromVgroup(SMnode *pMnode, SVgObj *pVgroup, SVnodeGid *del1, SVnodeGid *del2, SVnodeGid *exist) { + return 0; +} + SEpSet mndGetVgroupEpset(SMnode *pMnode, const SVgObj *pVgroup) { SEpSet epset = {0}; diff --git a/source/dnode/mnode/impl/test/db/db.cpp b/source/dnode/mnode/impl/test/db/db.cpp index 3c5bf5c083..545f9f22bb 100644 --- a/source/dnode/mnode/impl/test/db/db.cpp +++ b/source/dnode/mnode/impl/test/db/db.cpp @@ -73,14 +73,19 @@ TEST_F(MndTestDb, 02_Create_Alter_Drop_Db) { { SAlterDbReq alterdbReq = {0}; strcpy(alterdbReq.db, "1.d1"); + alterdbReq.buffer = 12; - alterdbReq.daysToKeep0 = 300; - alterdbReq.daysToKeep1 = 400; - alterdbReq.daysToKeep2 = 500; + alterdbReq.pageSize = -1; + alterdbReq.pages = -1; + alterdbReq.daysPerFile = -1; + alterdbReq.daysToKeep0 = -1; + alterdbReq.daysToKeep1 = -1; + alterdbReq.daysToKeep2 = -1; alterdbReq.fsyncPeriod = 4000; alterdbReq.walLevel = 2; - alterdbReq.strict = 2; + alterdbReq.strict = 1; alterdbReq.cacheLastRow = 1; + alterdbReq.replications = 1; int32_t contLen = tSerializeSAlterDbReq(NULL, 0, &alterdbReq); void* pReq = rpcMallocCont(contLen); From 79b70ac2875b9293914d88aac4e2cff67b3bb8d8 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 6 May 2022 10:03:54 +0000 Subject: [PATCH 27/83] refact more --- source/dnode/vnode/src/tsdb/tsdbMemTable2.c | 98 ++++++++++++++++++--- 1 file changed, 86 insertions(+), 12 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c index ae0d4630d1..fd753afb2f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c @@ -15,10 +15,10 @@ #include "tsdb.h" -typedef struct SMemTable SMemTable; -typedef struct SMemData SMemData; -typedef struct SMemSkipList SMemSkipList; -typedef struct SMemSkipListCfg SMemSkipListCfg; +typedef struct SMemTable SMemTable; +typedef struct SMemData SMemData; +typedef struct SMemSkipList SMemSkipList; +typedef struct SMemSkipListNode SMemSkipListNode; struct SMemTable { STsdb *pTsdb; @@ -32,15 +32,16 @@ struct SMemTable { SMemData **pBuckets; }; -struct SMemSkipListCfg { - int8_t maxLevel; - int32_t nKey; - int32_t nData; +struct SMemSkipListNode { + int8_t level; + SMemSkipListNode *forwards[]; }; struct SMemSkipList { - int8_t level; - uint32_t seed; + uint32_t seed; + int8_t level; + int32_t size; + SMemSkipListNode pHead[]; }; struct SMemData { @@ -76,6 +77,7 @@ int32_t tsdbMemTableCreate2(STsdb *pTsdb, SMemTable **ppMemTb) { pMemTb->pBuckets = taosMemoryCalloc(pMemTb->nBucket, sizeof(*pMemTb->pBuckets)); if (pMemTb->pBuckets == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; + taosMemoryFree(pMemTb); return -1; } @@ -83,11 +85,83 @@ int32_t tsdbMemTableCreate2(STsdb *pTsdb, SMemTable **ppMemTb) { return 0; } -int32_t tsdbMemTableDestroy2(STsdb *pTsdb, SMemTable *pMT) { - // TODO +int32_t tsdbMemTableDestroy2(STsdb *pTsdb, SMemTable *pMemTb) { + if (pMemTb) { + // loop to destroy the contents (todo) + taosMemoryFree(pMemTb->pBuckets); + taosMemoryFree(pMemTb); + } return 0; } +int32_t tsdbInsertData2(SMemTable *pMemTb, int64_t version, const SVSubmitBlk *pSubmitBlk) { + SMemData *pMemData; + SVBufPool *pPool = pMemTb->pTsdb->pVnode->inUse; + int32_t hash; + int32_t tlen; + uint8_t buf[16]; + int32_t rSize; + SMemSkipListNode *pSlNode; + const STSRow *pTSRow; + + // search hash + hash = (pSubmitBlk->suid + pSubmitBlk->uid) % pMemTb->nBucket; + for (pMemData = pMemTb->pBuckets[hash]; pMemData; pMemData = pMemData->pHashNext) { + if (pMemData->suid == pSubmitBlk->suid && pMemData->uid == pSubmitBlk->uid) break; + } + + // create pMemData if need + if (pMemData == NULL) { + pMemData = vnodeBufPoolMalloc(pPool, sizeof(*pMemData)); + if (pMemData == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + pMemData->pHashNext = NULL; + pMemData->suid = pSubmitBlk->suid; + pMemData->uid = pSubmitBlk->uid; + pMemData->minKey = TSKEY_MAX; + pMemData->maxKey = TSKEY_MIN; + pMemData->minVer = -1; + pMemData->maxVer = -1; + pMemData->nRows = 0; + pMemData->sl.level = 0; + pMemData->sl.seed = taosRand(); + pMemData->sl.size = 0; + + // add to MemTable + hash = (pMemData->suid + pMemData->uid) % pMemTb->nBucket; + pMemData->pHashNext = pMemTb->pBuckets[hash]; + pMemTb->pBuckets[hash] = pMemData; + } + + // loop to insert data to skiplist + for (;;) { + rSize = 0; + pTSRow = NULL; + if (pTSRow == NULL) break; + + // check the row (todo) + + // move the cursor to position to write (todo) + + // insert the row + int8_t level = 1; + tlen = 0; // sizeof(int64_t) + tsdbPutLen(rSize) + rSize; + pSlNode = vnodeBufPoolMalloc(pPool, tlen); + if (pSlNode == NULL) { + ASSERT(0); + } + } + + return 0; +} + +static void tsdbEncodeRow(int64_t version, int32_t rSize, const STSRow *pRow) {} + +static void tsdbDecodeRow(int64_t *version, int32_t *rSize, const STSRow **ppRow) {} + // SMemData // SMemSkipList \ No newline at end of file From c9e6bd4afd45c4ee188b1d42828ba23980ccbf91 Mon Sep 17 00:00:00 2001 From: jiacy-jcy <714897623@qq.com> Date: Fri, 6 May 2022 18:07:51 +0800 Subject: [PATCH 28/83] update testcase --- tests/system-test/2-query/timetruncate.py | 118 +++++++++++++++++++++- tests/system-test/fulltest.sh | 5 +- 2 files changed, 121 insertions(+), 2 deletions(-) diff --git a/tests/system-test/2-query/timetruncate.py b/tests/system-test/2-query/timetruncate.py index 4c4ebb09df..a48851b251 100644 --- a/tests/system-test/2-query/timetruncate.py +++ b/tests/system-test/2-query/timetruncate.py @@ -95,7 +95,123 @@ class TDTestCase: # tdSql.checkData(9,0,"2018-09-17 09:00:00.009000000") - + tdSql.query("select timetruncate(1,1d) from stb") + tdSql.checkRows(10) + tdSql.query("select timetruncate(1,1u) from stb") + tdSql.checkRows(10) + tdSql.query("select timetruncate(1,1a) from stb") + tdSql.checkRows(10) + tdSql.query("select timetruncate(1,1m) from stb") + tdSql.checkRows(10) + tdSql.query("select timetruncate(1,1h) from stb") + tdSql.checkRows(10) + tdSql.query("select timetruncate(ts,1d) from stb") + tdSql.checkRows(10) + tdSql.checkData(0,0,"2018-09-17 08:00:00.000") + tdSql.query("select timetruncate(ts,1h) from stb") + tdSql.checkRows(10) + tdSql.checkData(0,0,"2018-09-17 09:00:00.000") + tdSql.query("select timetruncate(ts,1m) from stb") + tdSql.checkRows(10) + tdSql.checkData(0,0,"2018-09-17 09:00:00.000") + tdSql.query("select timetruncate(ts,1s) from stb") + tdSql.checkRows(10) + tdSql.checkData(0,0,"2018-09-17 09:00:00.000") + tdSql.query("select timetruncate(ts,1a) from stb") + tdSql.checkRows(10) + tdSql.checkData(0,0,"2018-09-17 09:00:00.000") + tdSql.checkData(1,0,"2018-09-17 09:00:00.001") + tdSql.checkData(2,0,"2018-09-17 09:00:00.002") + tdSql.checkData(3,0,"2018-09-17 09:00:00.003") + tdSql.checkData(4,0,"2018-09-17 09:00:00.004") + tdSql.checkData(5,0,"2018-09-17 09:00:00.005") + tdSql.checkData(6,0,"2018-09-17 09:00:00.006") + tdSql.checkData(7,0,"2018-09-17 09:00:00.007") + tdSql.checkData(8,0,"2018-09-17 09:00:00.008") + tdSql.checkData(9,0,"2018-09-17 09:00:00.009") + # tdSql.query("select timetruncate(ts,1u) from stb") + # tdSql.checkRows(10) + # tdSql.checkData(0,0,"2018-09-17 09:00:00.000000") + # tdSql.checkData(1,0,"2018-09-17 09:00:00.001000") + # tdSql.checkData(2,0,"2018-09-17 09:00:00.002000") + # tdSql.checkData(3,0,"2018-09-17 09:00:00.003000") + # tdSql.checkData(4,0,"2018-09-17 09:00:00.004000") + # tdSql.checkData(5,0,"2018-09-17 09:00:00.005000") + # tdSql.checkData(6,0,"2018-09-17 09:00:00.006000") + # tdSql.checkData(7,0,"2018-09-17 09:00:00.007000") + # tdSql.checkData(8,0,"2018-09-17 09:00:00.008000") + # tdSql.checkData(9,0,"2018-09-17 09:00:00.009000") + # tdSql.query("select timetruncate(ts,1b) from stb") + # tdSql.checkRows(10) + # tdSql.checkData(0,0,"2018-09-17 09:00:00.000000000") + # tdSql.checkData(1,0,"2018-09-17 09:00:00.001000000") + # tdSql.checkData(2,0,"2018-09-17 09:00:00.002000000") + # tdSql.checkData(3,0,"2018-09-17 09:00:00.003000000") + # tdSql.checkData(4,0,"2018-09-17 09:00:00.004000000") + # tdSql.checkData(5,0,"2018-09-17 09:00:00.005000000") + # tdSql.checkData(6,0,"2018-09-17 09:00:00.006000000") + # tdSql.checkData(7,0,"2018-09-17 09:00:00.007000000") + # tdSql.checkData(8,0,"2018-09-17 09:00:00.008000000") + # tdSql.checkData(9,0,"2018-09-17 09:00:00.009000000") + + tdSql.query("select timetruncate(1,1d) from stb_1") + tdSql.checkRows(10) + tdSql.query("select timetruncate(1,1u) from stb_1") + tdSql.checkRows(10) + tdSql.query("select timetruncate(1,1a) from stb_1") + tdSql.checkRows(10) + tdSql.query("select timetruncate(1,1m) from stb_1") + tdSql.checkRows(10) + tdSql.query("select timetruncate(1,1h) from stb_1") + tdSql.checkRows(10) + tdSql.query("select timetruncate(ts,1d) from stb_1") + tdSql.checkRows(10) + tdSql.checkData(0,0,"2018-09-17 08:00:00.000") + tdSql.query("select timetruncate(ts,1h) from stb_1") + tdSql.checkRows(10) + tdSql.checkData(0,0,"2018-09-17 09:00:00.000") + tdSql.query("select timetruncate(ts,1m) from stb_1") + tdSql.checkRows(10) + tdSql.checkData(0,0,"2018-09-17 09:00:00.000") + tdSql.query("select timetruncate(ts,1s) from stb_1") + tdSql.checkRows(10) + tdSql.checkData(0,0,"2018-09-17 09:00:00.000") + tdSql.query("select timetruncate(ts,1a) from stb_1") + tdSql.checkRows(10) + tdSql.checkData(0,0,"2018-09-17 09:00:00.000") + tdSql.checkData(1,0,"2018-09-17 09:00:00.001") + tdSql.checkData(2,0,"2018-09-17 09:00:00.002") + tdSql.checkData(3,0,"2018-09-17 09:00:00.003") + tdSql.checkData(4,0,"2018-09-17 09:00:00.004") + tdSql.checkData(5,0,"2018-09-17 09:00:00.005") + tdSql.checkData(6,0,"2018-09-17 09:00:00.006") + tdSql.checkData(7,0,"2018-09-17 09:00:00.007") + tdSql.checkData(8,0,"2018-09-17 09:00:00.008") + tdSql.checkData(9,0,"2018-09-17 09:00:00.009") + # tdSql.query("select timetruncate(ts,1u) from stb_1") + # tdSql.checkRows(10) + # tdSql.checkData(0,0,"2018-09-17 09:00:00.000000") + # tdSql.checkData(1,0,"2018-09-17 09:00:00.001000") + # tdSql.checkData(2,0,"2018-09-17 09:00:00.002000") + # tdSql.checkData(3,0,"2018-09-17 09:00:00.003000") + # tdSql.checkData(4,0,"2018-09-17 09:00:00.004000") + # tdSql.checkData(5,0,"2018-09-17 09:00:00.005000") + # tdSql.checkData(6,0,"2018-09-17 09:00:00.006000") + # tdSql.checkData(7,0,"2018-09-17 09:00:00.007000") + # tdSql.checkData(8,0,"2018-09-17 09:00:00.008000") + # tdSql.checkData(9,0,"2018-09-17 09:00:00.009000") + # tdSql.query("select timetruncate(ts,1b) from stb_1") + # tdSql.checkRows(10) + # tdSql.checkData(0,0,"2018-09-17 09:00:00.000000000") + # tdSql.checkData(1,0,"2018-09-17 09:00:00.001000000") + # tdSql.checkData(2,0,"2018-09-17 09:00:00.002000000") + # tdSql.checkData(3,0,"2018-09-17 09:00:00.003000000") + # tdSql.checkData(4,0,"2018-09-17 09:00:00.004000000") + # tdSql.checkData(5,0,"2018-09-17 09:00:00.005000000") + # tdSql.checkData(6,0,"2018-09-17 09:00:00.006000000") + # tdSql.checkData(7,0,"2018-09-17 09:00:00.007000000") + # tdSql.checkData(8,0,"2018-09-17 09:00:00.008000000") + # tdSql.checkData(9,0,"2018-09-17 09:00:00.009000000") def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 0b7a71bbe4..026dba6490 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -14,7 +14,10 @@ python3 ./test.py -f 2-query/varchar.py #python3 ./test.py -f 2-query/timezone.py python3 ./test.py -f 2-query/Now.py python3 ./test.py -f 2-query/Today.py - +python3 ./test.py -f 2-query/max.py +python3 ./test.py -f 2-query/min.py +python3 ./test.py -f 2-query/count.py +# python3 ./test.py -f 2-query/Timediff.py #python3 ./test.py -f 2-query/cast.py From 5ec83cbf11cf415faaf66ddd9895b64e3da09163 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 6 May 2022 10:21:55 +0000 Subject: [PATCH 29/83] more --- source/dnode/vnode/src/tsdb/tsdbMemTable2.c | 25 ++++++++++++++------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c index fd753afb2f..1c263b7d00 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c @@ -39,6 +39,7 @@ struct SMemSkipListNode { struct SMemSkipList { uint32_t seed; + int8_t maxLevel; int8_t level; int32_t size; SMemSkipListNode pHead[]; @@ -96,11 +97,14 @@ int32_t tsdbMemTableDestroy2(STsdb *pTsdb, SMemTable *pMemTb) { int32_t tsdbInsertData2(SMemTable *pMemTb, int64_t version, const SVSubmitBlk *pSubmitBlk) { SMemData *pMemData; - SVBufPool *pPool = pMemTb->pTsdb->pVnode->inUse; + STsdb *pTsdb = pMemTb->pTsdb; + SVnode *pVnode = pTsdb->pVnode; + SVBufPool *pPool = pVnode->inUse; int32_t hash; int32_t tlen; uint8_t buf[16]; - int32_t rSize; + int32_t rlen; + const uint8_t *p; SMemSkipListNode *pSlNode; const STSRow *pTSRow; @@ -126,8 +130,9 @@ int32_t tsdbInsertData2(SMemTable *pMemTb, int64_t version, const SVSubmitBlk *p pMemData->minVer = -1; pMemData->maxVer = -1; pMemData->nRows = 0; - pMemData->sl.level = 0; pMemData->sl.seed = taosRand(); + pMemData->sl.maxLevel = pVnode->config.tsdbCfg.slLevel; + pMemData->sl.level = 0; pMemData->sl.size = 0; // add to MemTable @@ -137,9 +142,13 @@ int32_t tsdbInsertData2(SMemTable *pMemTb, int64_t version, const SVSubmitBlk *p } // loop to insert data to skiplist + p = pSubmitBlk->pData; for (;;) { - rSize = 0; - pTSRow = NULL; + if (p - (uint8_t *)pSubmitBlk->pData >= pSubmitBlk->nData) break; + + // p = tGetLen(p, &rlen); + pTSRow = (STSRow *)p; + p += rlen; if (pTSRow == NULL) break; // check the row (todo) @@ -148,7 +157,7 @@ int32_t tsdbInsertData2(SMemTable *pMemTb, int64_t version, const SVSubmitBlk *p // insert the row int8_t level = 1; - tlen = 0; // sizeof(int64_t) + tsdbPutLen(rSize) + rSize; + tlen = 0; // sizeof(int64_t) + tsdbPutLen(rlen) + rlen; pSlNode = vnodeBufPoolMalloc(pPool, tlen); if (pSlNode == NULL) { ASSERT(0); @@ -158,9 +167,9 @@ int32_t tsdbInsertData2(SMemTable *pMemTb, int64_t version, const SVSubmitBlk *p return 0; } -static void tsdbEncodeRow(int64_t version, int32_t rSize, const STSRow *pRow) {} +static void tsdbEncodeRow(int64_t version, int32_t rlen, const STSRow *pRow) {} -static void tsdbDecodeRow(int64_t *version, int32_t *rSize, const STSRow **ppRow) {} +static void tsdbDecodeRow(int64_t *version, int32_t *rlen, const STSRow **ppRow) {} // SMemData From e08dd100a44d4b30f35964754683840b38279cd6 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 6 May 2022 18:39:28 +0800 Subject: [PATCH 30/83] enh(index): support more data type --- include/libs/index/index.h | 7 +- source/libs/executor/src/indexoperator.c | 4 +- source/libs/index/inc/indexCache.h | 1 + source/libs/index/inc/indexInt.h | 4 +- source/libs/index/src/index.c | 4 +- source/libs/index/src/indexCache.c | 190 +++++++++++++++++------ source/libs/index/src/indexTfile.c | 14 +- source/libs/index/test/indexTests.cc | 44 +++--- source/libs/index/test/jsonUT.cc | 16 +- source/libs/transport/src/transSrv.c | 5 + 10 files changed, 194 insertions(+), 95 deletions(-) diff --git a/include/libs/index/index.h b/include/libs/index/index.h index c94d75338a..fa4cb1d2bd 100644 --- a/include/libs/index/index.h +++ b/include/libs/index/index.h @@ -56,7 +56,8 @@ typedef enum { QUERY_LESS_EQUAL, QUERY_GREATER_THAN, QUERY_GREATER_EQUAL, - QUERY_RANGE + QUERY_RANGE, + QUERY_MAX } EIndexQueryType; /* @@ -178,8 +179,8 @@ void indexOptsDestroy(SIndexOpts* opts); * @param: */ -SIndexTerm* indexTermCreate(int64_t suid, SIndexOperOnColumn operType, int8_t qType, uint8_t colType, - const char* colName, int32_t nColName, const char* colVal, int32_t nColVal); +SIndexTerm* indexTermCreate(int64_t suid, SIndexOperOnColumn operType, uint8_t colType, const char* colName, + int32_t nColName, const char* colVal, int32_t nColVal); void indexTermDestroy(SIndexTerm* p); /* diff --git a/source/libs/executor/src/indexoperator.c b/source/libs/executor/src/indexoperator.c index 3b62bdd664..86a28605b2 100644 --- a/source/libs/executor/src/indexoperator.c +++ b/source/libs/executor/src/indexoperator.c @@ -252,8 +252,8 @@ static int32_t sifExecFunction(SFunctionNode *node, SIFCtx *ctx, SIFParam *outpu return TSDB_CODE_QRY_INVALID_INPUT; } static int32_t sifDoIndex(SIFParam *left, SIFParam *right, int8_t operType, SIFParam *output) { - SIndexTerm *tm = indexTermCreate(left->suid, DEFAULT, operType, left->colValType, left->colName, - strlen(left->colName), right->condValue, strlen(right->condValue)); + SIndexTerm *tm = indexTermCreate(left->suid, DEFAULT, left->colValType, left->colName, strlen(left->colName), + right->condValue, strlen(right->condValue)); if (tm == NULL) { return TSDB_CODE_QRY_OUT_OF_MEMORY; } diff --git a/source/libs/index/inc/indexCache.h b/source/libs/index/inc/indexCache.h index 3ea986ad48..0daca26ec9 100644 --- a/source/libs/index/inc/indexCache.h +++ b/source/libs/index/inc/indexCache.h @@ -31,6 +31,7 @@ extern "C" { typedef struct MemTable { T_REF_DECLARE() SSkipList* mem; + void* pCache; } MemTable; typedef struct IndexCache { T_REF_DECLARE() diff --git a/source/libs/index/inc/indexInt.h b/source/libs/index/inc/indexInt.h index 5c7b8b9afe..7b7050d80e 100644 --- a/source/libs/index/inc/indexInt.h +++ b/source/libs/index/inc/indexInt.h @@ -166,7 +166,9 @@ int32_t indexSerialCacheKey(ICacheKey* key, char* buf); } while (0) #define INDEX_TYPE_CONTAIN_EXTERN_TYPE(ty, exTy) (((ty >> 4) & (exTy)) != 0) -#define INDEX_TYPE_GET_TYPE(ty) (ty & 0x0F) + +#define INDEX_TYPE_GET_TYPE(ty) (ty & 0x0F) + #define INDEX_TYPE_ADD_EXTERN_TYPE(ty, exTy) \ do { \ uint8_t oldTy = ty; \ diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index 83b5025ad0..e0c24ac3bd 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -244,8 +244,8 @@ int indexMultiTermQueryAdd(SIndexMultiTermQuery* pQuery, SIndexTerm* term, EInde return 0; } -SIndexTerm* indexTermCreate(int64_t suid, SIndexOperOnColumn oper, int8_t queryType, uint8_t colType, - const char* colName, int32_t nColName, const char* colVal, int32_t nColVal) { +SIndexTerm* indexTermCreate(int64_t suid, SIndexOperOnColumn oper, uint8_t colType, const char* colName, + int32_t nColName, const char* colVal, int32_t nColVal) { SIndexTerm* tm = (SIndexTerm*)taosMemoryCalloc(1, (sizeof(SIndexTerm))); if (tm == NULL) { return NULL; diff --git a/source/libs/index/src/indexCache.c b/source/libs/index/src/indexCache.c index 13768ce682..ad82dd0748 100644 --- a/source/libs/index/src/indexCache.c +++ b/source/libs/index/src/indexCache.c @@ -34,23 +34,58 @@ static char* indexCacheTermGet(const void* pData); static MemTable* indexInternalCacheCreate(int8_t type); -static int32_t cacheSearchTerm(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s); -static int32_t cacheSearchPrefix(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s); -static int32_t cacheSearchSuffix(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s); -static int32_t cacheSearchRegex(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s); -static int32_t cacheSearchLessThan(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s); -static int32_t cacheSearchLessEqual(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s); -static int32_t cacheSearchGreaterThan(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s); -static int32_t cacheSearchGreaterEqual(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s); -static int32_t cacheSearchRange(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s); +static int32_t cacheSearchTerm(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s); +static int32_t cacheSearchPrefix(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s); +static int32_t cacheSearchSuffix(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s); +static int32_t cacheSearchRegex(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s); +static int32_t cacheSearchLessThan(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s); +static int32_t cacheSearchLessEqual(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s); +static int32_t cacheSearchGreaterThan(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s); +static int32_t cacheSearchGreaterEqual(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s); +static int32_t cacheSearchRange(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s); /*comm func of compare, used in (LE/LT/GE/GT compare)*/ -static int32_t cacheSearchCompareFunc(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s, +static int32_t cacheSearchCompareFunc(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s, RangeType type); +static int32_t cacheSearchTerm_JSON(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s); +static int32_t cacheSearchPrefix_JSON(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s); +static int32_t cacheSearchSuffix_JSON(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s); +static int32_t cacheSearchRegex_JSON(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s); +static int32_t cacheSearchLessThan_JSON(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s); +static int32_t cacheSearchLessEqual_JSON(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s); +static int32_t cacheSearchGreaterThan_JSON(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s); +static int32_t cacheSearchGreaterEqual_JSON(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s); +static int32_t cacheSearchRange_JSON(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s); + +static int32_t cacheSearchCompareFunc_JSON(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s, + RangeType type); typedef enum { MATCH, CONTINUE, BREAK } TExeCond; typedef TExeCond (*_cache_range_compare)(void* a, void* b, int8_t type); -static TExeCond tCompareLessThan(void* a, void* b, int8_t type) { return MATCH; } +static TExeCond tDoCommpare(__compar_fn_t func, int8_t comType, void* a, void* b) { + int32_t ret = func(a, b); + switch (comType) { + case QUERY_LESS_THAN: { + if (ret < 0) return MATCH; + } break; + case QUERY_LESS_EQUAL: { + if (ret <= 0) return MATCH; + break; + } + case QUERY_GREATER_THAN: { + if (ret > 0) return MATCH; + break; + } + case QUERY_GREATER_EQUAL: { + if (ret >= 0) return MATCH; + } + } + return CONTINUE; +} +static TExeCond tCompareLessThan(void* a, void* b, int8_t type) { + __compar_fn_t func = getComparFunc(type, 0); + return tDoCommpare(func, QUERY_LESS_THAN, a, b); +} static TExeCond tCompareLessEqual(void* a, void* b, int8_t type) { return MATCH; } static TExeCond tCompareGreaterThan(void* a, void* b, int8_t type) { return MATCH; } static TExeCond tCompareGreaterEqual(void* a, void* b, int8_t type) { return MATCH; } @@ -58,20 +93,26 @@ static TExeCond tCompareGreaterEqual(void* a, void* b, int8_t type) { return MAT static TExeCond (*rangeCompare[])(void* a, void* b, int8_t type) = {tCompareLessThan, tCompareLessEqual, tCompareGreaterThan, tCompareGreaterEqual}; -static int32_t (*cacheSearch[])(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) = { - cacheSearchTerm, cacheSearchPrefix, cacheSearchSuffix, cacheSearchRegex, cacheSearchLessThan, - cacheSearchLessEqual, cacheSearchGreaterThan, cacheSearchGreaterEqual, cacheSearchRange}; +static int32_t (*cacheSearch[][QUERY_MAX])(void* cache, SIndexTerm* ct, SIdxTempResult* tr, STermValueType* s) = { + {cacheSearchTerm, cacheSearchPrefix, cacheSearchSuffix, cacheSearchRegex, cacheSearchLessThan, cacheSearchLessEqual, + cacheSearchGreaterThan, cacheSearchGreaterEqual, cacheSearchRange}, + {cacheSearchTerm_JSON, cacheSearchPrefix_JSON, cacheSearchSuffix_JSON, cacheSearchRegex_JSON, + cacheSearchLessThan_JSON, cacheSearchLessEqual_JSON, cacheSearchGreaterThan_JSON, cacheSearchGreaterEqual_JSON, + cacheSearchRange_JSON}}; static void doMergeWork(SSchedMsg* msg); static bool indexCacheIteratorNext(Iterate* itera); -static int32_t cacheSearchTerm(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) { +static int32_t cacheSearchTerm(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) { if (cache == NULL) { return 0; } + MemTable* mem = cache; + IndexCache* pCache = mem->pCache; + CacheTerm ct = {.colVal = term->colVal, .version = atomic_load_32(&pCache->version)}; + CacheTerm* pCt = &ct; - MemTable* mem = cache; - char* key = indexCacheTermGet(ct); + char* key = indexCacheTermGet(&ct); SSkipListIterator* iter = tSkipListCreateIterFromVal(mem->mem, key, TSDB_DATA_TYPE_BINARY, TSDB_ORDER_ASC); while (tSkipListIterNext(iter)) { @@ -80,7 +121,7 @@ static int32_t cacheSearchTerm(void* cache, CacheTerm* ct, SIdxTempResult* tr, S break; } CacheTerm* c = (CacheTerm*)SL_GET_NODE_DATA(node); - if (0 == strcmp(c->colVal, ct->colVal)) { + if (0 == strcmp(c->colVal, pCt->colVal)) { if (c->operaType == ADD_VALUE) { INDEX_MERGE_ADD_DEL(tr->deled, tr->added, c->uid) // taosArrayPush(result, &c->uid); @@ -95,27 +136,33 @@ static int32_t cacheSearchTerm(void* cache, CacheTerm* ct, SIdxTempResult* tr, S tSkipListDestroyIter(iter); return 0; } -static int32_t cacheSearchPrefix(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) { +static int32_t cacheSearchPrefix(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) { // impl later return 0; } -static int32_t cacheSearchSuffix(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) { +static int32_t cacheSearchSuffix(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) { // impl later return 0; } -static int32_t cacheSearchRegex(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) { +static int32_t cacheSearchRegex(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) { // impl later return 0; } -static int32_t cacheSearchCompareFunc(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s, +static int32_t cacheSearchCompareFunc(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s, RangeType type) { if (cache == NULL) { return 0; } _cache_range_compare cmpFn = rangeCompare[type]; - MemTable* mem = cache; - char* key = indexCacheTermGet(ct); + MemTable* mem = cache; + IndexCache* pCache = mem->pCache; + + CacheTerm* pCt = taosMemoryCalloc(1, sizeof(CacheTerm)); + pCt->colVal = term->colVal; + pCt->version = atomic_load_32(&pCache->version); + + char* key = indexCacheTermGet(pCt); SSkipListIterator* iter = tSkipListCreateIter(mem->mem); while (tSkipListIterNext(iter)) { @@ -124,7 +171,7 @@ static int32_t cacheSearchCompareFunc(void* cache, CacheTerm* ct, SIdxTempResult break; } CacheTerm* c = (CacheTerm*)SL_GET_NODE_DATA(node); - TExeCond cond = cmpFn(c->colVal, ct->colVal, ct->colType); + TExeCond cond = cmpFn(c->colVal, pCt->colVal, pCt->colType); if (cond == MATCH) { if (c->operaType == ADD_VALUE) { INDEX_MERGE_ADD_DEL(tr->deled, tr->added, c->uid) @@ -134,26 +181,61 @@ static int32_t cacheSearchCompareFunc(void* cache, CacheTerm* ct, SIdxTempResult INDEX_MERGE_ADD_DEL(tr->added, tr->deled, c->uid) } } else if (cond == CONTINUE) { + continue; } else if (cond == BREAK) { break; } } + taosMemoryFree(pCt); tSkipListDestroyIter(iter); return TSDB_CODE_SUCCESS; } -static int32_t cacheSearchLessThan(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) { - return cacheSearchCompareFunc(cache, ct, tr, s, LT); +static int32_t cacheSearchLessThan(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) { + return cacheSearchCompareFunc(cache, term, tr, s, LT); } -static int32_t cacheSearchLessEqual(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) { - return cacheSearchCompareFunc(cache, ct, tr, s, LE); +static int32_t cacheSearchLessEqual(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) { + return cacheSearchCompareFunc(cache, term, tr, s, LE); } -static int32_t cacheSearchGreaterThan(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) { - return cacheSearchCompareFunc(cache, ct, tr, s, GT); +static int32_t cacheSearchGreaterThan(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) { + return cacheSearchCompareFunc(cache, term, tr, s, GT); } -static int32_t cacheSearchGreaterEqual(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) { - return cacheSearchCompareFunc(cache, ct, tr, s, GE); +static int32_t cacheSearchGreaterEqual(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) { + return cacheSearchCompareFunc(cache, term, tr, s, GE); } -static int32_t cacheSearchRange(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) { + +static int32_t cacheSearchTerm_JSON(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) { + return TSDB_CODE_SUCCESS; +} +static int32_t cacheSearchPrefix_JSON(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) { + return TSDB_CODE_SUCCESS; +} +static int32_t cacheSearchSuffix_JSON(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) { + return TSDB_CODE_SUCCESS; +} +static int32_t cacheSearchRegex_JSON(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) { + return TSDB_CODE_SUCCESS; +} +static int32_t cacheSearchLessThan_JSON(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) { + return cacheSearchCompareFunc_JSON(cache, term, tr, s, LT); +} +static int32_t cacheSearchLessEqual_JSON(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) { + return cacheSearchCompareFunc_JSON(cache, term, tr, s, LE); +} +static int32_t cacheSearchGreaterThan_JSON(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) { + return cacheSearchCompareFunc_JSON(cache, term, tr, s, GT); +} +static int32_t cacheSearchGreaterEqual_JSON(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) { + return cacheSearchCompareFunc_JSON(cache, term, tr, s, GE); +} +static int32_t cacheSearchRange_JSON(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) { + return TSDB_CODE_SUCCESS; +} + +static int32_t cacheSearchCompareFunc_JSON(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s, + RangeType type) { + return TSDB_CODE_SUCCESS; +} +static int32_t cacheSearchRange(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) { // impl later return 0; } @@ -167,6 +249,7 @@ IndexCache* indexCacheCreate(SIndex* idx, uint64_t suid, const char* colName, in }; cache->mem = indexInternalCacheCreate(type); + cache->mem->pCache = cache; cache->colName = INDEX_TYPE_CONTAIN_EXTERN_TYPE(type, TSDB_DATA_TYPE_JSON) ? tstrdup(JSON_COLUMN) : tstrdup(colName); cache->type = type; cache->index = idx; @@ -379,11 +462,19 @@ int indexCacheDel(void* cache, const char* fieldValue, int32_t fvlen, uint64_t u return 0; } -static int indexQueryMem(MemTable* mem, CacheTerm* ct, EIndexQueryType qtype, SIdxTempResult* tr, STermValueType* s) { +static int32_t indexQueryMem(MemTable* mem, SIndexTermQuery* query, SIdxTempResult* tr, STermValueType* s) { if (mem == NULL) { return 0; } - return cacheSearch[qtype](mem, ct, tr, s); + + SIndexTerm* term = query->term; + EIndexQueryType qtype = query->qType; + + if (INDEX_TYPE_CONTAIN_EXTERN_TYPE(term->colType, TSDB_DATA_TYPE_JSON)) { + return cacheSearch[1][qtype](mem, term, tr, s); + } else { + return cacheSearch[0][qtype](mem, term, tr, s); + } } int indexCacheSearch(void* cache, SIndexTermQuery* query, SIdxTempResult* result, STermValueType* s) { int64_t st = taosGetTimestampUs(); @@ -400,25 +491,24 @@ int indexCacheSearch(void* cache, SIndexTermQuery* query, SIdxTempResult* result indexMemRef(imm); taosThreadMutexUnlock(&pCache->mtx); - SIndexTerm* term = query->term; - EIndexQueryType qtype = query->qType; + // SIndexTerm* term = query->term; + // EIndexQueryType qtype = query->qType; - bool hasJson = INDEX_TYPE_CONTAIN_EXTERN_TYPE(term->colType, TSDB_DATA_TYPE_JSON); - char* p = term->colVal; - if (hasJson) { - p = indexPackJsonData(term); - } - CacheTerm ct = {.colVal = p, .version = atomic_load_32(&pCache->version)}; + // bool isJson = INDEX_TYPE_CONTAIN_EXTERN_TYPE(term->colType, TSDB_DATA_TYPE_JSON); + // char* p = term->colVal; + // if (isJson) { + // p = indexPackJsonData(term); + //} + // CacheTerm ct = {.colVal = p, .version = atomic_load_32(&pCache->version)}; - int ret = indexQueryMem(mem, &ct, qtype, result, s); + int ret = indexQueryMem(mem, query, result, s); if (ret == 0 && *s != kTypeDeletion) { // continue search in imm - ret = indexQueryMem(imm, &ct, qtype, result, s); - } - - if (hasJson) { - taosMemoryFreeClear(p); + ret = indexQueryMem(imm, query, result, s); } + // if (isJson) { + // taosMemoryFreeClear(p); + //} indexMemUnRef(mem); indexMemUnRef(imm); diff --git a/source/libs/index/src/indexTfile.c b/source/libs/index/src/indexTfile.c index 5aed2bd6b0..9edd868272 100644 --- a/source/libs/index/src/indexTfile.c +++ b/source/libs/index/src/indexTfile.c @@ -606,16 +606,16 @@ static bool tfileIteratorNext(Iterate* iiter) { static IterateValue* tifileIterateGetValue(Iterate* iter) { return &iter->val; } static TFileFstIter* tfileFstIteratorCreate(TFileReader* reader) { - TFileFstIter* tIter = taosMemoryCalloc(1, sizeof(TFileFstIter)); - if (tIter == NULL) { + TFileFstIter* iter = taosMemoryCalloc(1, sizeof(TFileFstIter)); + if (iter == NULL) { return NULL; } - tIter->ctx = automCtxCreate(NULL, AUTOMATION_ALWAYS); - tIter->fb = fstSearch(reader->fst, tIter->ctx); - tIter->st = streamBuilderIntoStream(tIter->fb); - tIter->rdr = reader; - return tIter; + iter->ctx = automCtxCreate(NULL, AUTOMATION_ALWAYS); + iter->fb = fstSearch(reader->fst, iter->ctx); + iter->st = streamBuilderIntoStream(iter->fb); + iter->rdr = reader; + return iter; } Iterate* tfileIteratorCreate(TFileReader* reader) { diff --git a/source/libs/index/test/indexTests.cc b/source/libs/index/test/indexTests.cc index d8ea6a8233..896451c686 100644 --- a/source/libs/index/test/indexTests.cc +++ b/source/libs/index/test/indexTests.cc @@ -483,7 +483,7 @@ TEST_F(IndexTFileEnv, test_tfile_write) { std::string colName("voltage"); std::string colVal("ab"); - SIndexTerm* term = indexTermCreate(1, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SIndexTermQuery query = {term, QUERY_TERM}; @@ -557,7 +557,7 @@ TEST_F(IndexCacheEnv, cache_test) { std::string colName("voltage"); { std::string colVal("v1"); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); coj->Put(term, colId, version++, suid++); indexTermDestroy(term); @@ -565,28 +565,28 @@ TEST_F(IndexCacheEnv, cache_test) { } { std::string colVal("v3"); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); coj->Put(term, colId, version++, suid++); indexTermDestroy(term); } { std::string colVal("v2"); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); coj->Put(term, colId, version++, suid++); indexTermDestroy(term); } { std::string colVal("v3"); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); coj->Put(term, colId, version++, suid++); indexTermDestroy(term); } { std::string colVal("v3"); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); coj->Put(term, colId, version++, suid++); indexTermDestroy(term); @@ -595,14 +595,14 @@ TEST_F(IndexCacheEnv, cache_test) { std::cout << "--------first----------" << std::endl; { std::string colVal("v3"); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); coj->Put(term, othColId, version++, suid++); indexTermDestroy(term); } { std::string colVal("v4"); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); coj->Put(term, othColId, version++, suid++); indexTermDestroy(term); @@ -613,7 +613,7 @@ TEST_F(IndexCacheEnv, cache_test) { std::string colVal("v4"); for (size_t i = 0; i < 10; i++) { colVal[colVal.size() - 1] = 'a' + i; - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); coj->Put(term, colId, version++, suid++); indexTermDestroy(term); @@ -623,7 +623,7 @@ TEST_F(IndexCacheEnv, cache_test) { // begin query { std::string colVal("v3"); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SIndexTermQuery query = {term, QUERY_TERM}; SArray* ret = (SArray*)taosArrayInit(4, sizeof(suid)); @@ -638,7 +638,7 @@ TEST_F(IndexCacheEnv, cache_test) { } { std::string colVal("v2"); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SIndexTermQuery query = {term, QUERY_TERM}; SArray* ret = (SArray*)taosArrayInit(4, sizeof(suid)); @@ -670,7 +670,7 @@ class IndexObj { return ret; } void Del(const std::string& colName, const std::string& colVal, uint64_t uid) { - SIndexTerm* term = indexTermCreate(0, DEL_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, DEL_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); @@ -679,7 +679,7 @@ class IndexObj { } int WriteMillonData(const std::string& colName, const std::string& colVal = "Hello world", size_t numOfTable = 100 * 10000) { - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); @@ -701,7 +701,7 @@ class IndexObj { // opt tColVal[taosRand() % colValSize] = 'a' + k % 26; } - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), tColVal.c_str(), tColVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); @@ -737,7 +737,7 @@ class IndexObj { int SearchOne(const std::string& colName, const std::string& colVal) { SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); indexMultiTermQueryAdd(mq, term, QUERY_TERM); @@ -759,7 +759,7 @@ class IndexObj { } int SearchOneTarget(const std::string& colName, const std::string& colVal, uint64_t val) { SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); indexMultiTermQueryAdd(mq, term, QUERY_TERM); @@ -784,7 +784,7 @@ class IndexObj { void PutOne(const std::string& colName, const std::string& colVal) { SIndexMultiTerm* terms = indexMultiTermCreate(); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); indexMultiTermAdd(terms, term); Put(terms, 10); @@ -792,7 +792,7 @@ class IndexObj { } void PutOneTarge(const std::string& colName, const std::string& colVal, uint64_t val) { SIndexMultiTerm* terms = indexMultiTermCreate(); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); indexMultiTermAdd(terms, term); Put(terms, val); @@ -832,7 +832,7 @@ TEST_F(IndexEnv2, testIndexOpen) { { std::string colName("tag1"), colVal("Hello"); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); @@ -847,7 +847,7 @@ TEST_F(IndexEnv2, testIndexOpen) { size_t size = 200; std::string colName("tag1"), colVal("hello"); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); @@ -862,7 +862,7 @@ TEST_F(IndexEnv2, testIndexOpen) { size_t size = 200; std::string colName("tag1"), colVal("Hello"); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); @@ -877,7 +877,7 @@ TEST_F(IndexEnv2, testIndexOpen) { { std::string colName("tag1"), colVal("Hello"); SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); indexMultiTermQueryAdd(mq, term, QUERY_TERM); diff --git a/source/libs/index/test/jsonUT.cc b/source/libs/index/test/jsonUT.cc index e1e5004701..f789d23136 100644 --- a/source/libs/index/test/jsonUT.cc +++ b/source/libs/index/test/jsonUT.cc @@ -40,7 +40,7 @@ TEST_F(JsonEnv, testWrite) { { std::string colName("test"); std::string colVal("ab"); - SIndexTerm* term = indexTermCreate(1, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); @@ -53,7 +53,7 @@ TEST_F(JsonEnv, testWrite) { { std::string colName("voltage"); std::string colVal("ab1"); - SIndexTerm* term = indexTermCreate(1, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); @@ -66,7 +66,7 @@ TEST_F(JsonEnv, testWrite) { { std::string colName("voltage"); std::string colVal("123"); - SIndexTerm* term = indexTermCreate(1, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); @@ -81,7 +81,7 @@ TEST_F(JsonEnv, testWrite) { std::string colVal("ab"); SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); - SIndexTerm* q = indexTermCreate(1, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SArray* result = taosArrayInit(1, sizeof(uint64_t)); @@ -95,7 +95,7 @@ TEST_F(JsonEnv, testWriteMillonData) { { std::string colName("test"); std::string colVal("ab"); - SIndexTerm* term = indexTermCreate(1, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); @@ -110,7 +110,7 @@ TEST_F(JsonEnv, testWriteMillonData) { std::string colVal("abxxxxxxxxxxxx"); for (int i = 0; i < 1000; i++) { colVal[i % colVal.size()] = '0' + i % 128; - SIndexTerm* term = indexTermCreate(1, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); @@ -124,7 +124,7 @@ TEST_F(JsonEnv, testWriteMillonData) { { std::string colName("voltagefdadfa"); std::string colVal("abxxxxxxxxxxxx"); - SIndexTerm* term = indexTermCreate(1, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); @@ -139,7 +139,7 @@ TEST_F(JsonEnv, testWriteMillonData) { std::string colVal("ab"); SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); - SIndexTerm* q = indexTermCreate(1, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); SArray* result = taosArrayInit(1, sizeof(uint64_t)); diff --git a/source/libs/transport/src/transSrv.c b/source/libs/transport/src/transSrv.c index b78edf6cd0..9c905771d5 100644 --- a/source/libs/transport/src/transSrv.c +++ b/source/libs/transport/src/transSrv.c @@ -782,6 +782,11 @@ static void uvDestroyConn(uv_handle_t* handle) { tDebug("server conn %p destroy", conn); // uv_timer_stop(&conn->pTimer); transQueueDestroy(&conn->srvMsgs); + + if (conn->regArg.init == 1) { + transFreeMsg(conn->regArg.msg.pCont); + conn->regArg.init = 0; + } QUEUE_REMOVE(&conn->queue); taosMemoryFree(conn->pTcp); taosMemoryFree(conn); From 68fc2acc9caad5f1587f83d3767db1c5b529d0c4 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 6 May 2022 11:16:53 +0000 Subject: [PATCH 31/83] refact --- source/dnode/vnode/src/tsdb/tsdbMemTable2.c | 114 ++++++++++++++------ 1 file changed, 81 insertions(+), 33 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c index 1c263b7d00..e7b733369a 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c @@ -15,10 +15,11 @@ #include "tsdb.h" -typedef struct SMemTable SMemTable; -typedef struct SMemData SMemData; -typedef struct SMemSkipList SMemSkipList; -typedef struct SMemSkipListNode SMemSkipListNode; +typedef struct SMemTable SMemTable; +typedef struct SMemData SMemData; +typedef struct SMemSkipList SMemSkipList; +typedef struct SMemSkipListNode SMemSkipListNode; +typedef struct SMemSkipListCurosr SMemSkipListCurosr; struct SMemTable { STsdb *pTsdb; @@ -57,6 +58,20 @@ struct SMemData { SMemSkipList sl; }; +struct SMemSkipListCurosr { + SMemSkipList *pSl; + SMemSkipListNode *pNodeC; +}; + +#define SL_NODE_SIZE(l) (sizeof(SMemSkipListNode) + sizeof(SMemSkipListNode *) * (l)*2) +#define SL_NODE_HALF_SIZE(l) (sizeof(SMemSkipListNode) + sizeof(SMemSkipListNode *) * (l)) +#define SL_NODE_FORWARD(n, l) ((n)->forwards[l]) +#define SL_NODE_BACKWARD(n, l) ((n)->forwards[(n)->level + (l)]) +#define SL_NODE_DATA(n) (&SL_NODE_BACKWARD(n, (n)->level)) + +#define SL_HEAD_NODE(sl) ((sl)->pHead) +#define SL_TAIL_NODE(sl) ((SMemSkipListNode *)&SL_NODE_FORWARD(SL_HEAD_NODE(sl), (sl)->maxLevel)) + // SMemTable int32_t tsdbMemTableCreate2(STsdb *pTsdb, SMemTable **ppMemTb) { SMemTable *pMemTb = NULL; @@ -96,17 +111,18 @@ int32_t tsdbMemTableDestroy2(STsdb *pTsdb, SMemTable *pMemTb) { } int32_t tsdbInsertData2(SMemTable *pMemTb, int64_t version, const SVSubmitBlk *pSubmitBlk) { - SMemData *pMemData; - STsdb *pTsdb = pMemTb->pTsdb; - SVnode *pVnode = pTsdb->pVnode; - SVBufPool *pPool = pVnode->inUse; - int32_t hash; - int32_t tlen; - uint8_t buf[16]; - int32_t rlen; - const uint8_t *p; - SMemSkipListNode *pSlNode; - const STSRow *pTSRow; + SMemData *pMemData; + STsdb *pTsdb = pMemTb->pTsdb; + SVnode *pVnode = pTsdb->pVnode; + SVBufPool *pPool = pVnode->inUse; + int32_t hash; + int32_t tlen; + uint8_t buf[16]; + int32_t rlen; + const uint8_t *p; + SMemSkipListNode *pSlNode; + const STSRow *pTSRow; + SMemSkipListCurosr slc = {0}; // search hash hash = (pSubmitBlk->suid + pSubmitBlk->uid) % pMemTb->nBucket; @@ -116,7 +132,11 @@ int32_t tsdbInsertData2(SMemTable *pMemTb, int64_t version, const SVSubmitBlk *p // create pMemData if need if (pMemData == NULL) { - pMemData = vnodeBufPoolMalloc(pPool, sizeof(*pMemData)); + int8_t maxLevel = pVnode->config.tsdbCfg.slLevel; + int32_t tsize = sizeof(*pMemData) + SL_NODE_HALF_SIZE(maxLevel) * 2; + SMemSkipListNode *pHead, *pTail; + + pMemData = vnodeBufPoolMalloc(pPool, tsize); if (pMemData == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -131,46 +151,74 @@ int32_t tsdbInsertData2(SMemTable *pMemTb, int64_t version, const SVSubmitBlk *p pMemData->maxVer = -1; pMemData->nRows = 0; pMemData->sl.seed = taosRand(); - pMemData->sl.maxLevel = pVnode->config.tsdbCfg.slLevel; + pMemData->sl.maxLevel = maxLevel; pMemData->sl.level = 0; pMemData->sl.size = 0; + pHead = SL_HEAD_NODE(&pMemData->sl); + pTail = SL_TAIL_NODE(&pMemData->sl); + pHead->level = maxLevel; + pTail->level = maxLevel; + for (int iLevel = 0; iLevel < maxLevel; iLevel++) { + SL_NODE_FORWARD(pHead, iLevel) = pTail; + SL_NODE_FORWARD(pTail, iLevel) = pHead; + } // add to MemTable hash = (pMemData->suid + pMemData->uid) % pMemTb->nBucket; pMemData->pHashNext = pMemTb->pBuckets[hash]; pMemTb->pBuckets[hash] = pMemData; + pMemTb->nHash++; } - // loop to insert data to skiplist +// loop to insert data to skiplist +#if 0 + tsdbMemSkipListCursorOpen(&slc, &pMemData->sl); p = pSubmitBlk->pData; for (;;) { if (p - (uint8_t *)pSubmitBlk->pData >= pSubmitBlk->nData) break; - // p = tGetLen(p, &rlen); - pTSRow = (STSRow *)p; - p += rlen; - if (pTSRow == NULL) break; + const uint8_t *pt = p; + p = tGetBinary(p, &pTSRow, &rlen); // check the row (todo) // move the cursor to position to write (todo) + int32_t c; + tsdbMemSkipListCursorMoveTo(&slc, pTSRow, version, &c); + ASSERT(c); - // insert the row - int8_t level = 1; - tlen = 0; // sizeof(int64_t) + tsdbPutLen(rlen) + rlen; - pSlNode = vnodeBufPoolMalloc(pPool, tlen); - if (pSlNode == NULL) { - ASSERT(0); - } + // encode row + int8_t level = tsdbMemSkipListRandLevel(&pMemData->sl); + int32_t tsize = SL_NODE_SIZE(level) + sizeof(version) + (p - pt); + pSlNode = vnodeBufPoolMalloc(pPool, tsize); + pSlNode->level = level; + + uint8_t *pData = SL_NODE_DATA(pSlNode); + *(int64_t *)pData = version; + pData += sizeof(version); + memcpy(pData, pt, p - pt); + + // insert row + tsdbMemSkipListCursorPut(&slc, pSlNode); + + // update status + if (pTSRow->ts < pMemData->minKey) pMemData->minKey = pTSRow->ts; + if (pTSRow->ts > pMemData->maxKey) pMemData->maxKey = pTSRow->ts; } + tsdbMemSkipListCursorClose(&slc); +#endif + + if (pMemData->minVer == -1) pMemData->minVer = version; + if (pMemData->maxVer == -1 || pMemData->maxVer < version) pMemData->maxVer = version; + + if (pMemTb->minKey < pMemData->minKey) pMemTb->minKey = pMemData->minKey; + if (pMemTb->maxKey < pMemData->maxKey) pMemTb->maxKey = pMemData->maxKey; + if (pMemTb->minVer == -1) pMemTb->minVer = version; + if (pMemTb->maxVer == -1 || pMemTb->maxVer < version) pMemTb->maxVer = version; return 0; } -static void tsdbEncodeRow(int64_t version, int32_t rlen, const STSRow *pRow) {} - -static void tsdbDecodeRow(int64_t *version, int32_t *rlen, const STSRow **ppRow) {} - // SMemData // SMemSkipList \ No newline at end of file From bcdf9c19b2dc0eab830877a620f1d8a3cc2d51db Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Fri, 6 May 2022 19:31:20 +0800 Subject: [PATCH 32/83] feat: update refactor --- Jenkinsfile2 | 279 +++++--- cmake/cmake.define | 2 + include/common/tdataformat.h | 7 +- include/common/tmsg.h | 37 +- include/common/trow.h | 4 +- include/libs/catalog/catalog.h | 17 + include/libs/function/tudf.h | 15 +- include/libs/scalar/scalar.h | 2 + include/libs/transport/trpc.h | 1 + include/os/osSocket.h | 1 + source/client/src/clientHb.c | 77 +++ source/client/test/clientTests.cpp | 75 +- source/common/src/tdatablock.c | 10 +- source/common/src/tmsg.c | 204 +++++- source/common/src/trow.c | 1 + source/dnode/mgmt/implement/src/dmTransport.c | 12 +- source/dnode/mnode/impl/inc/mndDef.h | 2 + source/dnode/mnode/impl/inc/mndSma.h | 1 - source/dnode/mnode/impl/inc/mndUser.h | 1 + source/dnode/mnode/impl/inc/mndVgroup.h | 1 + source/dnode/mnode/impl/src/mndDb.c | 219 ++---- source/dnode/mnode/impl/src/mndFunc.c | 2 +- source/dnode/mnode/impl/src/mndProfile.c | 10 + source/dnode/mnode/impl/src/mndSma.c | 49 +- source/dnode/mnode/impl/src/mndTrans.c | 4 +- source/dnode/mnode/impl/src/mndUser.c | 140 +++- source/dnode/mnode/impl/src/mndVgroup.c | 166 +++-- source/dnode/mnode/impl/test/func/func.cpp | 6 + source/dnode/vnode/CMakeLists.txt | 1 + source/dnode/vnode/src/inc/tsdb.h | 1 - source/dnode/vnode/src/inc/vnodeInt.h | 2 +- source/dnode/vnode/src/tsdb/tsdbCommit.c | 4 - source/dnode/vnode/src/tsdb/tsdbMemTable.c | 55 +- source/dnode/vnode/src/tsdb/tsdbMemTable2.c | 93 +++ source/dnode/vnode/src/tsdb/tsdbRead.c | 4 + source/dnode/vnode/src/tsdb/tsdbReadImpl.c | 1 + source/dnode/vnode/src/tsdb/tsdbSma.c | 15 + source/dnode/vnode/src/vnd/vnodeSvr.c | 76 +- source/libs/catalog/inc/catalogInt.h | 20 + source/libs/catalog/src/catalog.c | 271 ++++++++ source/libs/executor/inc/executorimpl.h | 21 +- source/libs/executor/inc/tsort.h | 15 +- source/libs/executor/src/executorimpl.c | 63 +- source/libs/executor/src/scanoperator.c | 82 ++- source/libs/executor/src/sortoperator.c | 59 +- source/libs/executor/src/tsort.c | 40 +- source/libs/function/src/builtins.c | 2 +- source/libs/function/src/tudf.c | 4 +- source/libs/function/src/udfd.c | 25 +- source/libs/parser/src/parInsert.c | 52 +- source/libs/parser/src/parInsertData.c | 5 - source/libs/qcom/src/querymsg.c | 36 + source/libs/scalar/src/sclfunc.c | 6 + source/libs/scalar/src/sclvector.c | 6 +- source/libs/transport/src/trans.c | 14 +- source/libs/transport/src/transSrv.c | 5 +- source/os/src/osDir.c | 2 +- source/os/src/osSocket.c | 42 ++ tests/requirements.txt | 5 + tests/script/tsim/query/scalarFunction.sim | 5 + tests/script/tsim/query/udf.sim | 21 + .../1-insert/insertWithMoreVgroup.py | 291 ++++++++ tests/system-test/2-query/join.py | 238 +++++-- tests/system-test/2-query/log.py | 14 +- tests/system-test/2-query/pow.py | 652 ++++++++++++++++++ tests/system-test/2-query/smaTest.py | 131 ++++ tests/system-test/2-query/sqrt.py | 551 +++++++++++++++ tests/system-test/fulltest.sh | 4 +- tools/shell/src/shellEngine.c | 3 + tools/shell/src/shellNettest.c | 5 +- 70 files changed, 3588 insertions(+), 699 deletions(-) create mode 100644 source/dnode/vnode/src/tsdb/tsdbMemTable2.c create mode 100644 tests/requirements.txt create mode 100644 tests/system-test/1-insert/insertWithMoreVgroup.py create mode 100644 tests/system-test/2-query/pow.py create mode 100644 tests/system-test/2-query/smaTest.py create mode 100644 tests/system-test/2-query/sqrt.py diff --git a/Jenkinsfile2 b/Jenkinsfile2 index c9c9c3a7ca..49e13b7831 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -8,135 +8,202 @@ def skipbuild=0 def win_stop=0 def abortPreviousBuilds() { - def currentJobName = env.JOB_NAME - def currentBuildNumber = env.BUILD_NUMBER.toInteger() - def jobs = Jenkins.instance.getItemByFullName(currentJobName) - def builds = jobs.getBuilds() + def currentJobName = env.JOB_NAME + def currentBuildNumber = env.BUILD_NUMBER.toInteger() + def jobs = Jenkins.instance.getItemByFullName(currentJobName) + def builds = jobs.getBuilds() - for (build in builds) { - if (!build.isBuilding()) { - continue; + for (build in builds) { + if (!build.isBuilding()) { + continue; + } + + if (currentBuildNumber == build.getNumber().toInteger()) { + continue; + } + + build.doKill() //doTerm(),doKill(),doTerm() } - - if (currentBuildNumber == build.getNumber().toInteger()) { - continue; - } - - build.doKill() //doTerm(),doKill(),doTerm() - } } // abort previous build abortPreviousBuilds() def abort_previous(){ - def buildNumber = env.BUILD_NUMBER as int - if (buildNumber > 1) milestone(buildNumber - 1) - milestone(buildNumber) + def buildNumber = env.BUILD_NUMBER as int + if (buildNumber > 1) milestone(buildNumber - 1) + milestone(buildNumber) } def pre_test(){ - sh'hostname' + sh 'hostname' sh ''' - date - sudo rmtaos || echo "taosd has not installed" + date + sudo rmtaos || echo "taosd has not installed" ''' sh ''' - killall -9 taosd ||echo "no taosd running" - killall -9 gdb || echo "no gdb running" - killall -9 python3.8 || echo "no python program running" - cd ${WKC} + killall -9 taosd ||echo "no taosd running" + killall -9 gdb || echo "no gdb running" + killall -9 python3.8 || echo "no python program running" + cd ${WKC} ''' script { - if (env.CHANGE_TARGET == 'master') { - sh ''' - cd ${WKC} - git checkout master - ''' + if (env.CHANGE_TARGET == 'master') { + sh ''' + cd ${WKC} + git checkout master + ''' + } else if(env.CHANGE_TARGET == '2.0') { + sh ''' + cd ${WKC} + git checkout 2.0 + ''' + } else if(env.CHANGE_TARGET == '3.0') { + sh ''' + cd ${WKC} + git checkout 3.0 + [ -d contrib/bdb ] && cd contrib/bdb && git clean -fxd && cd ../.. + ''' + } else { + sh ''' + cd ${WKC} + git checkout develop + ''' } - else if(env.CHANGE_TARGET == '2.0'){ - sh ''' - cd ${WKC} - git checkout 2.0 - ''' - } - else if(env.CHANGE_TARGET == '3.0'){ - sh ''' - cd ${WKC} - git checkout 3.0 - [ -d contrib/bdb ] && cd contrib/bdb && git clean -fxd && cd ../.. - ''' - } - else{ - sh ''' - cd ${WKC} - git checkout develop - ''' - } } - sh''' - cd ${WKC} - git pull >/dev/null - git fetch origin +refs/pull/${CHANGE_ID}/merge - git checkout -qf FETCH_HEAD - git submodule update --init --recursive + sh ''' + cd ${WKC} + git pull >/dev/null + git fetch origin +refs/pull/${CHANGE_ID}/merge + git checkout -qf FETCH_HEAD + git submodule update --init --recursive ''' - sh''' - cd ${WKC} - export TZ=Asia/Harbin - date - rm -rf debug - mkdir debug - cd debug - cmake .. > /dev/null - make -j4> /dev/null + sh ''' + cd ${WKC} + export TZ=Asia/Harbin + date + rm -rf debug + mkdir debug + cd debug + cmake .. > /dev/null + make -j4> /dev/null ''' - sh''' - cd ${WKPY} - git reset --hard - git pull - pip3 install . + sh ''' + cd ${WKPY} + git reset --hard + git pull + pip3 install . + ''' + return 1 +} +def pre_test_win(){ + bat ''' + hostname + date /t + time /t + taskkill /f /t /im python.exe + taskkill /f /t /im bash.exe + cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDengine + rd /s /Q C:\\workspace\\%EXECUTOR_NUMBER%\\TDengine\\debug + exit 0 + ''' + bat ''' + cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDengine + git reset --hard + git fetch || git fetch + git checkout -f + ''' + script { + if (env.CHANGE_TARGET == 'master') { + bat ''' + cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDengine + git checkout master + ''' + } else if(env.CHANGE_TARGET == '2.0') { + bat ''' + cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDengine + git checkout 2.0 + ''' + } else if(env.CHANGE_TARGET == '3.0') { + bat ''' + cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDengine + git checkout 3.0 + ''' + } else { + bat ''' + cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDengine + git checkout develop + ''' + } + } + bat ''' + cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDengine + git branch + git pull || git pull + git fetch origin +refs/pull/%CHANGE_ID%/merge + git checkout -qf FETCH_HEAD + ''' +} +def pre_test_build_win() { + bat ''' + echo "building ..." + time /t + cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDengine + mkdir debug + cd debug + call "C:\\Program Files (x86)\\Microsoft Visual Studio\\2017\\Community\\VC\\Auxiliary\\Build\\vcvarsall.bat" x64 + set CL=/MP8 + cmake .. -G "NMake Makefiles JOM" + jom -j 4 || exit 8 + time /t ''' return 1 } pipeline { - agent none - options { skipDefaultCheckout() } - environment{ - WK = '/var/lib/jenkins/workspace/TDinternal' - WKC= '/var/lib/jenkins/workspace/TDengine' - WKPY= '/var/lib/jenkins/workspace/taos-connector-python' - } - stages { - stage('pre_build'){ - agent{label " slave3_0 || slave15 || slave16 || slave17 "} - options { skipDefaultCheckout() } - when { - changeRequest() - } - steps { - script{ - abort_previous() - abortPreviousBuilds() + agent none + options { skipDefaultCheckout() } + environment{ + WK = '/var/lib/jenkins/workspace/TDinternal' + WKC= '/var/lib/jenkins/workspace/TDengine' + WKPY= '/var/lib/jenkins/workspace/taos-connector-python' + } + stages { + stage('run test') { + parallel { + stage('windows test') { + agent {label " windows11 "} + steps { + pre_test_win() + pre_test_build_win() + } + } + stage('linux test') { + agent{label " slave3_0 || slave15 || slave16 || slave17 "} + options { skipDefaultCheckout() } + when { + changeRequest() + } + steps { + timeout(time: 45, unit: 'MINUTES'){ + pre_test() + sh ''' + cd ${WKC}/debug + ctest -VV + ''' + sh ''' + export LD_LIBRARY_PATH=${WKC}/debug/build/lib + cd ${WKC}/tests/system-test + ./fulltest.sh + ''' + sh ''' + cd ${WKC}/tests + ./test-all.sh b1fq + ''' + } + } + } } - timeout(time: 45, unit: 'MINUTES'){ - pre_test() - sh''' - cd ${WKC}/debug - ctest -VV - ''' - sh''' - export LD_LIBRARY_PATH=${WKC}/debug/build/lib - cd ${WKC}/tests/system-test - ./fulltest.sh - ''' - sh''' - cd ${WKC}/tests - ./test-all.sh b1fq - ''' - } - } - } - } - post { + } + } + post { success { emailext ( subject: "PR-result: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]' SUCCESS", diff --git a/cmake/cmake.define b/cmake/cmake.define index 1655154506..d1d9266bca 100644 --- a/cmake/cmake.define +++ b/cmake/cmake.define @@ -2,6 +2,8 @@ cmake_minimum_required(VERSION 3.16) set(CMAKE_VERBOSE_MAKEFILE OFF) +SET(BUILD_SHARED_LIBS "OFF") + #set output directory SET(LIBRARY_OUTPUT_PATH ${PROJECT_BINARY_DIR}/build/lib) SET(EXECUTABLE_OUTPUT_PATH ${PROJECT_BINARY_DIR}/build/bin) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 35b57ba117..74d11a5ffb 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -372,10 +372,9 @@ static FORCE_INLINE void tdCopyColOfRowBySchema(SDataRow dst, STSchema *pDstSche // ----------------- Data column structure // SDataCol arrangement: data => bitmap => dataOffset typedef struct SDataCol { - int8_t type; // column type - uint8_t bitmap : 1; // 0: no bitmap if all rows are NORM, 1: has bitmap if has NULL/NORM rows - uint8_t bitmapMode : 1; // default is 0(2 bits), otherwise 1(1 bit) - uint8_t reserve : 6; + int8_t type; // column type + uint8_t bitmap : 1; // 0: no bitmap if all rows are NORM, 1: has bitmap if has NULL/NORM rows + uint8_t reserve : 7; int16_t colId; // column ID int32_t bytes; // column data bytes defined int32_t offset; // data offset in a SDataRow (including the header size) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index f6160cbbf5..3889784f2f 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -73,7 +73,8 @@ typedef uint16_t tmsg_t; enum { CONN_TYPE__QUERY = 1, CONN_TYPE__TMQ, CONN_TYPE__MAX }; enum { - HEARTBEAT_KEY_DBINFO = 1, + HEARTBEAT_KEY_USER_AUTHINFO = 1, + HEARTBEAT_KEY_DBINFO, HEARTBEAT_KEY_STBINFO, HEARTBEAT_KEY_MQ_TMP, }; @@ -426,7 +427,9 @@ int32_t tDeserializeSGetUserAuthReq(void* buf, int32_t bufLen, SGetUserAuthReq* typedef struct { char user[TSDB_USER_LEN]; + int32_t version; int8_t superAuth; + SHashObj* createdDbs; SHashObj* readDbs; SHashObj* writeDbs; } SGetUserAuthRsp; @@ -669,10 +672,20 @@ typedef struct { SArray* pArray; // Array of SUseDbRsp } SUseDbBatchRsp; + int32_t tSerializeSUseDbBatchRsp(void* buf, int32_t bufLen, SUseDbBatchRsp* pRsp); int32_t tDeserializeSUseDbBatchRsp(void* buf, int32_t bufLen, SUseDbBatchRsp* pRsp); void tFreeSUseDbBatchRsp(SUseDbBatchRsp* pRsp); +typedef struct { + SArray* pArray; // Array of SGetUserAuthRsp +} SUserAuthBatchRsp; + +int32_t tSerializeSUserAuthBatchRsp(void* buf, int32_t bufLen, SUserAuthBatchRsp* pRsp); +int32_t tDeserializeSUserAuthBatchRsp(void* buf, int32_t bufLen, SUserAuthBatchRsp* pRsp); +void tFreeSUserAuthBatchRsp(SUserAuthBatchRsp* pRsp); + + typedef struct { char db[TSDB_DB_FNAME_LEN]; } SCompactDbReq; @@ -2578,6 +2591,28 @@ static FORCE_INLINE void tDeleteSMqAskEpRsp(SMqAskEpRsp* pRsp) { taosArrayDestroyEx(pRsp->topics, (void (*)(void*))tDeleteSMqSubTopicEp); } +#define TD_AUTO_CREATE_TABLE 0x1 +typedef struct { + int64_t suid; + int64_t uid; + int32_t sver; + uint64_t nData; + const void* pData; + SVCreateTbReq cTbReq; +} SVSubmitBlk; + +typedef struct { + int32_t flags; + int32_t nBlocks; + union { + SArray* pArray; + SVSubmitBlk* pBlocks; + }; +} SVSubmitReq; + +int32_t tEncodeSVSubmitReq(SCoder* pCoder, const SVSubmitReq* pReq); +int32_t tDecodeSVSubmitReq(SCoder* pCoder, SVSubmitReq* pReq); + #pragma pack(pop) #ifdef __cplusplus diff --git a/include/common/trow.h b/include/common/trow.h index fe1cf0d967..0d34c6e49f 100644 --- a/include/common/trow.h +++ b/include/common/trow.h @@ -122,8 +122,6 @@ typedef struct { typedef struct { /// timestamp TSKEY ts; - /// row version - uint64_t ver; union { /// union field for encode and decode uint32_t info; @@ -176,7 +174,7 @@ typedef struct { #define TD_ROW_DATA(r) ((r)->data) #define TD_ROW_LEN(r) ((r)->len) #define TD_ROW_KEY(r) ((r)->ts) -#define TD_ROW_VER(r) ((r)->ver) +// #define TD_ROW_VER(r) ((r)->ver) #define TD_ROW_KEY_ADDR(r) (r) // N.B. If without STSchema, getExtendedRowSize() is used to get the rowMaxBytes and diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index 30d1bd0a51..04a24c4f32 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -40,6 +40,11 @@ enum { CTG_DBG_STB_RENT_NUM, }; +typedef enum { + AUTH_TYPE_READ = 1, + AUTH_TYPE_WRITE, + AUTH_TYPE_OTHER, +} AUTH_TYPE; typedef struct SCatalogReq { SArray *pTableName; // element is SNAME @@ -57,6 +62,7 @@ typedef struct SMetaData { typedef struct SCatalogCfg { uint32_t maxTblCacheNum; uint32_t maxDBCacheNum; + uint32_t maxUserCacheNum; uint32_t dbRentSec; uint32_t stbRentSec; } SCatalogCfg; @@ -77,6 +83,11 @@ typedef struct SDbVgVersion { int32_t numOfTable; // unit is TSDB_TABLE_NUM_UNIT } SDbVgVersion; +typedef struct SUserAuthVersion { + char user[TSDB_USER_LEN]; + int32_t version; +} SUserAuthVersion; + typedef SDbCfgRsp SDbCfgInfo; typedef SUserIndexRsp SIndexInfo; @@ -219,12 +230,18 @@ int32_t catalogGetExpiredSTables(SCatalog* pCatalog, SSTableMetaVersion **stable int32_t catalogGetExpiredDBs(SCatalog* pCatalog, SDbVgVersion **dbs, uint32_t *num); +int32_t catalogGetExpiredUsers(SCatalog* pCtg, SUserAuthVersion **users, uint32_t *num); + int32_t catalogGetDBCfg(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* dbFName, SDbCfgInfo* pDbCfg); int32_t catalogGetIndexInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* indexName, SIndexInfo* pInfo); int32_t catalogGetUdfInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* funcName, SFuncInfo** pInfo); +int32_t catalogChkAuth(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* user, const char* dbFName, AUTH_TYPE type, bool *pass); + +int32_t catalogUpdateUserAuthInfo(SCatalog* pCtg, SGetUserAuthRsp* pAuth); + /** * Destroy catalog and relase all resources diff --git a/include/libs/function/tudf.h b/include/libs/function/tudf.h index 22314e4bd6..d59a7c23f7 100644 --- a/include/libs/function/tudf.h +++ b/include/libs/function/tudf.h @@ -29,7 +29,11 @@ extern "C" { #endif #define UDF_LISTEN_PIPE_NAME_LEN 32 -#define UDF_LISTEN_PIPE_NAME_PREFIX "udfd.sock." +#ifdef _WIN32 +#define UDF_LISTEN_PIPE_NAME_PREFIX "\\\\?\\pipe\\udfd.sock" +#else +#define UDF_LISTEN_PIPE_NAME_PREFIX ".udfd.sock." +#endif #define UDF_DNODE_ID_ENV_NAME "DNODE_ID" //====================================================================================== @@ -129,8 +133,8 @@ int32_t udfAggFinalize(struct SqlFunctionCtx *pCtx, SSDataBlock* pBlock); // begin API to UDF writer. // dynamic lib init and destroy -typedef int32_t (*TUdfSetupFunc)(); -typedef int32_t (*TUdfTeardownFunc)(); +typedef int32_t (*TUdfInitFunc)(); +typedef int32_t (*TUdfDestroyFunc)(); //TODO: add API to check function arguments type, number etc. @@ -144,7 +148,7 @@ static FORCE_INLINE int32_t udfColEnsureCapacity(SUdfColumn* pColumn, int32_t ne return TSDB_CODE_SUCCESS; } - int allocCapacity = MAX(data->rowsAlloc, 8); + int allocCapacity = TMAX(data->rowsAlloc, 8); while (allocCapacity < newCapacity) { allocCapacity *= UDF_MEMORY_EXP_GROWTH; } @@ -238,11 +242,10 @@ static FORCE_INLINE int32_t udfColSetRow(SUdfColumn* pColumn, uint32_t currentRo data->varLenCol.payloadLen += dataLen; } } - data->numOfRows = MAX(currentRow + 1, data->numOfRows); + data->numOfRows = TMAX(currentRow + 1, data->numOfRows); return 0; } -typedef int32_t (*TUdfFreeUdfColumnFunc)(SUdfColumn* column); typedef int32_t (*TUdfScalarProcFunc)(SUdfDataBlock* block, SUdfColumn *resultCol); typedef int32_t (*TUdfAggStartFunc)(SUdfInterBuf *buf); diff --git a/include/libs/scalar/scalar.h b/include/libs/scalar/scalar.h index 0c7db45c4b..555274599a 100644 --- a/include/libs/scalar/scalar.h +++ b/include/libs/scalar/scalar.h @@ -91,6 +91,8 @@ int32_t winDurFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu int32_t qStartTsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t qEndTsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t qTbnameFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); + #ifdef __cplusplus } #endif diff --git a/include/libs/transport/trpc.h b/include/libs/transport/trpc.h index 0e7d486eab..a7d1522d12 100644 --- a/include/libs/transport/trpc.h +++ b/include/libs/transport/trpc.h @@ -68,6 +68,7 @@ typedef int (*RpcAfp)(void *parent, char *tableId, char *spi, char *encrypt, cha typedef bool (*RpcRfp)(int32_t code); typedef struct SRpcInit { + char localFqdn[TSDB_FQDN_LEN]; uint16_t localPort; // local port char * label; // for debug purpose int numOfThreads; // number of threads to handle connections diff --git a/include/os/osSocket.h b/include/os/osSocket.h index 62c3771669..213a6930ee 100644 --- a/include/os/osSocket.h +++ b/include/os/osSocket.h @@ -161,6 +161,7 @@ int taosCreateSocketWithTimeOutOpt(uint32_t conn_timeout_sec); TdSocketPtr taosOpenUdpSocket(uint32_t localIp, uint16_t localPort); TdSocketPtr taosOpenTcpClientSocket(uint32_t ip, uint16_t port, uint32_t localIp); +bool taosValidIpAndPort(uint32_t ip, uint16_t port); TdSocketServerPtr taosOpenTcpServerSocket(uint32_t ip, uint16_t port); int32_t taosKeepTcpAlive(TdSocketPtr pSocket); TdSocketPtr taosAcceptTcpConnectSocket(TdSocketServerPtr pServerSocket, struct sockaddr *destAddr, int *addrLen); diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index b11a49fa1a..fc39e80c1e 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -28,6 +28,27 @@ static int32_t hbMqHbReqHandle(SClientHbKey *connKey, void *param, SClientHbReq static int32_t hbMqHbRspHandle(SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) { return 0; } +static int32_t hbProcessUserAuthInfoRsp(void *value, int32_t valueLen, struct SCatalog *pCatalog) { + int32_t code = 0; + + SUserAuthBatchRsp batchRsp = {0}; + if (tDeserializeSUserAuthBatchRsp(value, valueLen, &batchRsp) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } + + int32_t numOfBatchs = taosArrayGetSize(batchRsp.pArray); + for (int32_t i = 0; i < numOfBatchs; ++i) { + SGetUserAuthRsp *rsp = taosArrayGet(batchRsp.pArray, i); + tscDebug("hb user auth rsp, user:%s, version:%d", rsp->user, rsp->version); + + catalogUpdateUserAuthInfo(pCatalog, rsp); + } + + tFreeSUserAuthBatchRsp(&batchRsp); + return TSDB_CODE_SUCCESS; +} + static int32_t hbProcessDBInfoRsp(void *value, int32_t valueLen, struct SCatalog *pCatalog) { int32_t code = 0; @@ -148,6 +169,24 @@ static int32_t hbQueryHbRspHandle(SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) { for (int32_t i = 0; i < kvNum; ++i) { SKv *kv = taosArrayGet(pRsp->info, i); switch (kv->key) { + case HEARTBEAT_KEY_USER_AUTHINFO: { + if (kv->valueLen <= 0 || NULL == kv->value) { + tscError("invalid hb user auth info, len:%d, value:%p", kv->valueLen, kv->value); + break; + } + + int64_t *clusterId = (int64_t *)info->param; + struct SCatalog *pCatalog = NULL; + + int32_t code = catalogGetHandle(*clusterId, &pCatalog); + if (code != TSDB_CODE_SUCCESS) { + tscWarn("catalogGetHandle failed, clusterId:%" PRIx64 ", error:%s", *clusterId, tstrerror(code)); + break; + } + + hbProcessUserAuthInfoRsp(kv->value, kv->valueLen, pCatalog); + break; + } case HEARTBEAT_KEY_DBINFO: { if (kv->valueLen <= 0 || NULL == kv->value) { tscError("invalid hb db info, len:%d, value:%p", kv->valueLen, kv->value); @@ -327,6 +366,39 @@ int32_t hbGetQueryBasicInfo(SClientHbKey *connKey, SClientHbReq *req) { return TSDB_CODE_SUCCESS; } +int32_t hbGetExpiredUserInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SClientHbReq *req) { + SUserAuthVersion *users = NULL; + uint32_t userNum = 0; + int32_t code = 0; + + code = catalogGetExpiredUsers(pCatalog, &users, &userNum); + if (TSDB_CODE_SUCCESS != code) { + return code; + } + + if (userNum <= 0) { + return TSDB_CODE_SUCCESS; + } + + for (int32_t i = 0; i < userNum; ++i) { + SUserAuthVersion *user = &users[i]; + user->version = htonl(user->version); + } + + SKv kv = { + .key = HEARTBEAT_KEY_USER_AUTHINFO, + .valueLen = sizeof(SUserAuthVersion) * userNum, + .value = users, + }; + + tscDebug("hb got %d expired users, valueLen:%d", userNum, kv.valueLen); + + taosHashPut(req->info, &kv.key, sizeof(kv.key), &kv, sizeof(kv)); + + return TSDB_CODE_SUCCESS; +} + + int32_t hbGetExpiredDBInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SClientHbReq *req) { SDbVgVersion *dbs = NULL; uint32_t dbNum = 0; @@ -407,6 +479,11 @@ int32_t hbQueryHbReqHandle(SClientHbKey *connKey, void *param, SClientHbReq *req hbGetQueryBasicInfo(connKey, req); + code = hbGetExpiredUserInfo(connKey, pCatalog, req); + if (TSDB_CODE_SUCCESS != code) { + return code; + } + code = hbGetExpiredDBInfo(connKey, pCatalog, req); if (TSDB_CODE_SUCCESS != code) { return code; diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index 2874220121..fc5781cb4d 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -587,15 +587,34 @@ TEST(testCase, projection_query_tables) { } taos_free_result(pRes); + pRes = taos_query(pConn, "create stable st2 (ts timestamp, k int) tags(a int)"); + if (taos_errno(pRes) != 0) { + printf("failed to create table tu, reason:%s\n", taos_errstr(pRes)); + } + taos_free_result(pRes); + pRes = taos_query(pConn, "create table tu using st1 tags(1)"); if (taos_errno(pRes) != 0) { printf("failed to create table tu, reason:%s\n", taos_errstr(pRes)); } taos_free_result(pRes); - for(int32_t i = 0; i < 10000; ++i) { - char sql[512] = {0}; - sprintf(sql, "insert into tu values(now+%da, %d)", i, i); + pRes = taos_query(pConn, "create table tu2 using st2 tags(1)"); + if (taos_errno(pRes) != 0) { + printf("failed to create table tu, reason:%s\n", taos_errstr(pRes)); + } + taos_free_result(pRes); + + for(int32_t i = 0; i < 10000000; i += 20) { + char sql[1024] = {0}; + sprintf(sql, + "insert into tu values(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)" + "(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)" + "(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)" + "(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)", + i, i, i + 1, i + 1, i + 2, i + 2, i + 3, i + 3, i + 4, i + 4, i + 5, i + 5, i + 6, i + 6, i + 7, i + 7, + i + 8, i + 8, i + 9, i + 9, i + 10, i + 10, i + 11, i + 11, i + 12, i + 12, i + 13, i + 13, i + 14, i + 14, + i + 15, i + 15, i + 16, i + 16, i + 17, i + 17, i + 18, i + 18, i + 19, i + 19); TAOS_RES* p = taos_query(pConn, sql); if (taos_errno(p) != 0) { printf("failed to insert data, reason:%s\n", taos_errstr(p)); @@ -604,24 +623,44 @@ TEST(testCase, projection_query_tables) { taos_free_result(p); } - pRes = taos_query(pConn, "select * from tu"); - if (taos_errno(pRes) != 0) { - printf("failed to select from table, reason:%s\n", taos_errstr(pRes)); - taos_free_result(pRes); - ASSERT_TRUE(false); + printf("start to insert next table\n"); + + for(int32_t i = 0; i < 10000000; i += 20) { + char sql[1024] = {0}; + sprintf(sql, + "insert into tu2 values(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)" + "(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)" + "(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)" + "(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)", + i, i, i + 1, i + 1, i + 2, i + 2, i + 3, i + 3, i + 4, i + 4, i + 5, i + 5, i + 6, i + 6, i + 7, i + 7, + i + 8, i + 8, i + 9, i + 9, i + 10, i + 10, i + 11, i + 11, i + 12, i + 12, i + 13, i + 13, i + 14, i + 14, + i + 15, i + 15, i + 16, i + 16, i + 17, i + 17, i + 18, i + 18, i + 19, i + 19); + TAOS_RES* p = taos_query(pConn, sql); + if (taos_errno(p) != 0) { + printf("failed to insert data, reason:%s\n", taos_errstr(p)); + } + + taos_free_result(p); } - TAOS_ROW pRow = NULL; - TAOS_FIELD* pFields = taos_fetch_fields(pRes); - int32_t numOfFields = taos_num_fields(pRes); +// pRes = taos_query(pConn, "select * from tu"); +// if (taos_errno(pRes) != 0) { +// printf("failed to select from table, reason:%s\n", taos_errstr(pRes)); +// taos_free_result(pRes); +// ASSERT_TRUE(false); +// } - char str[512] = {0}; - while ((pRow = taos_fetch_row(pRes)) != NULL) { - int32_t code = taos_print_row(str, pRow, pFields, numOfFields); - printf("%s\n", str); - } +// TAOS_ROW pRow = NULL; +// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +// int32_t numOfFields = taos_num_fields(pRes); +// +// char str[512] = {0}; +// while ((pRow = taos_fetch_row(pRes)) != NULL) { +// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); +// printf("%s\n", str); +// } - taos_free_result(pRes); +// taos_free_result(pRes); taos_close(pConn); } @@ -659,7 +698,7 @@ TEST(testCase, agg_query_tables) { TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); ASSERT_NE(pConn, nullptr); - TAOS_RES* pRes = taos_query(pConn, "use db"); + TAOS_RES* pRes = taos_query(pConn, "use abc1"); if (taos_errno(pRes) != 0) { printf("failed to use db, reason:%s\n", taos_errstr(pRes)); taos_free_result(pRes); diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 4f4b8b196f..a4b36dcb67 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -363,9 +363,9 @@ int32_t blockDataMerge(SSDataBlock* pDest, const SSDataBlock* pSrc, SArray* pInd for (int32_t i = 0; i < pDest->info.numOfCols; ++i) { int32_t mapIndex = i; - if (pIndexMap) { - mapIndex = *(int32_t*)taosArrayGet(pIndexMap, i); - } +// if (pIndexMap) { +// mapIndex = *(int32_t*)taosArrayGet(pIndexMap, i); +// } SColumnInfoData* pCol2 = taosArrayGet(pDest->pDataBlock, i); SColumnInfoData* pCol1 = taosArrayGet(pSrc->pDataBlock, mapIndex); @@ -493,12 +493,12 @@ SSDataBlock* blockDataExtractBlock(SSDataBlock* pBlock, int32_t startIndex, int3 for (int32_t j = startIndex; j < (startIndex + rowCount); ++j) { bool isNull = false; if (pBlock->pBlockAgg == NULL) { - isNull = colDataIsNull(pColData, pBlock->info.rows, j, NULL); + isNull = colDataIsNull_s(pColData, pBlock->info.rows); } else { isNull = colDataIsNull(pColData, pBlock->info.rows, j, pBlock->pBlockAgg[i]); } - char* p = colDataGetData(pColData, j); + char* p = colDataGetData(pColData, j); colDataAppend(pDstCol, j - startIndex, p, isNull); } } diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index a91f5f8223..6278b52a04 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -1164,31 +1164,47 @@ int32_t tDeserializeSGetUserAuthReq(void *buf, int32_t bufLen, SGetUserAuthReq * return 0; } -int32_t tSerializeSGetUserAuthRsp(void *buf, int32_t bufLen, SGetUserAuthRsp *pRsp) { - SCoder encoder = {0}; - tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); - - if (tStartEncode(&encoder) < 0) return -1; - if (tEncodeCStr(&encoder, pRsp->user) < 0) return -1; - if (tEncodeI8(&encoder, pRsp->superAuth) < 0) return -1; +int32_t tSerializeSGetUserAuthRspImpl(SCoder *pEncoder, SGetUserAuthRsp *pRsp) { + if (tEncodeCStr(pEncoder, pRsp->user) < 0) return -1; + if (tEncodeI8(pEncoder, pRsp->superAuth) < 0) return -1; + if (tEncodeI32(pEncoder, pRsp->version) < 0) return -1; + int32_t numOfCreatedDbs = taosHashGetSize(pRsp->createdDbs); int32_t numOfReadDbs = taosHashGetSize(pRsp->readDbs); int32_t numOfWriteDbs = taosHashGetSize(pRsp->writeDbs); - if (tEncodeI32(&encoder, numOfReadDbs) < 0) return -1; - if (tEncodeI32(&encoder, numOfWriteDbs) < 0) return -1; + if (tEncodeI32(pEncoder, numOfCreatedDbs) < 0) return -1; + if (tEncodeI32(pEncoder, numOfReadDbs) < 0) return -1; + if (tEncodeI32(pEncoder, numOfWriteDbs) < 0) return -1; - char *db = taosHashIterate(pRsp->readDbs, NULL); + char *db = taosHashIterate(pRsp->createdDbs, NULL); while (db != NULL) { - if (tEncodeCStr(&encoder, db) < 0) return -1; + if (tEncodeCStr(pEncoder, db) < 0) return -1; + db = taosHashIterate(pRsp->createdDbs, db); + } + + db = taosHashIterate(pRsp->readDbs, NULL); + while (db != NULL) { + if (tEncodeCStr(pEncoder, db) < 0) return -1; db = taosHashIterate(pRsp->readDbs, db); } db = taosHashIterate(pRsp->writeDbs, NULL); while (db != NULL) { - if (tEncodeCStr(&encoder, db) < 0) return -1; + if (tEncodeCStr(pEncoder, db) < 0) return -1; db = taosHashIterate(pRsp->writeDbs, db); } + return 0; +} + +int32_t tSerializeSGetUserAuthRsp(void *buf, int32_t bufLen, SGetUserAuthRsp *pRsp) { + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); + + if (tStartEncode(&encoder) < 0) return -1; + + if (tSerializeSGetUserAuthRspImpl(&encoder, pRsp) < 0) return -1; + tEndEncode(&encoder); int32_t tlen = encoder.pos; @@ -1196,39 +1212,58 @@ int32_t tSerializeSGetUserAuthRsp(void *buf, int32_t bufLen, SGetUserAuthRsp *pR return tlen; } -int32_t tDeserializeSGetUserAuthRsp(void *buf, int32_t bufLen, SGetUserAuthRsp *pRsp) { - pRsp->readDbs = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); - pRsp->writeDbs = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); +int32_t tDeserializeSGetUserAuthRspImpl(SCoder *pDecoder, SGetUserAuthRsp *pRsp) { + pRsp->createdDbs = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + pRsp->readDbs = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + pRsp->writeDbs = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); if (pRsp->readDbs == NULL || pRsp->writeDbs == NULL) { return -1; } - SCoder decoder = {0}; - tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); - - if (tStartDecode(&decoder) < 0) return -1; - if (tDecodeCStrTo(&decoder, pRsp->user) < 0) return -1; - if (tDecodeI8(&decoder, &pRsp->superAuth) < 0) return -1; + if (tDecodeCStrTo(pDecoder, pRsp->user) < 0) return -1; + if (tDecodeI8(pDecoder, &pRsp->superAuth) < 0) return -1; + if (tDecodeI32(pDecoder, &pRsp->version) < 0) return -1; + int32_t numOfCreatedDbs = 0; int32_t numOfReadDbs = 0; int32_t numOfWriteDbs = 0; - if (tDecodeI32(&decoder, &numOfReadDbs) < 0) return -1; - if (tDecodeI32(&decoder, &numOfWriteDbs) < 0) return -1; + if (tDecodeI32(pDecoder, &numOfCreatedDbs) < 0) return -1; + if (tDecodeI32(pDecoder, &numOfReadDbs) < 0) return -1; + if (tDecodeI32(pDecoder, &numOfWriteDbs) < 0) return -1; + + for (int32_t i = 0; i < numOfCreatedDbs; ++i) { + char db[TSDB_DB_FNAME_LEN] = {0}; + if (tDecodeCStrTo(pDecoder, db) < 0) return -1; + int32_t len = strlen(db) + 1; + taosHashPut(pRsp->createdDbs, db, len, db, len); + } for (int32_t i = 0; i < numOfReadDbs; ++i) { char db[TSDB_DB_FNAME_LEN] = {0}; - if (tDecodeCStrTo(&decoder, db) < 0) return -1; + if (tDecodeCStrTo(pDecoder, db) < 0) return -1; int32_t len = strlen(db) + 1; taosHashPut(pRsp->readDbs, db, len, db, len); } for (int32_t i = 0; i < numOfWriteDbs; ++i) { char db[TSDB_DB_FNAME_LEN] = {0}; - if (tDecodeCStrTo(&decoder, db) < 0) return -1; + if (tDecodeCStrTo(pDecoder, db) < 0) return -1; int32_t len = strlen(db) + 1; taosHashPut(pRsp->writeDbs, db, len, db, len); } + return 0; +} + + +int32_t tDeserializeSGetUserAuthRsp(void *buf, int32_t bufLen, SGetUserAuthRsp *pRsp) { + SCoder decoder = {0}; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); + + if (tStartDecode(&decoder) < 0) return -1; + + if (tDeserializeSGetUserAuthRspImpl(&decoder, pRsp) < 0) return -1; + tEndDecode(&decoder); tCoderClear(&decoder); @@ -1236,6 +1271,7 @@ int32_t tDeserializeSGetUserAuthRsp(void *buf, int32_t bufLen, SGetUserAuthRsp * } void tFreeSGetUserAuthRsp(SGetUserAuthRsp *pRsp) { + taosHashCleanup(pRsp->createdDbs); taosHashCleanup(pRsp->readDbs); taosHashCleanup(pRsp->writeDbs); } @@ -2055,6 +2091,62 @@ void tFreeSUseDbBatchRsp(SUseDbBatchRsp *pRsp) { taosArrayDestroy(pRsp->pArray); } +int32_t tSerializeSUserAuthBatchRsp(void* buf, int32_t bufLen, SUserAuthBatchRsp* pRsp){ + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); + + if (tStartEncode(&encoder) < 0) return -1; + + int32_t numOfBatch = taosArrayGetSize(pRsp->pArray); + if (tEncodeI32(&encoder, numOfBatch) < 0) return -1; + for (int32_t i = 0; i < numOfBatch; ++i) { + SGetUserAuthRsp *pUserAuthRsp = taosArrayGet(pRsp->pArray, i); + if (tSerializeSGetUserAuthRspImpl(&encoder, pUserAuthRsp) < 0) return -1; + } + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tCoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSUserAuthBatchRsp(void* buf, int32_t bufLen, SUserAuthBatchRsp* pRsp){ + SCoder decoder = {0}; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); + + if (tStartDecode(&decoder) < 0) return -1; + + int32_t numOfBatch = taosArrayGetSize(pRsp->pArray); + if (tDecodeI32(&decoder, &numOfBatch) < 0) return -1; + + pRsp->pArray = taosArrayInit(numOfBatch, sizeof(SGetUserAuthRsp)); + if (pRsp->pArray == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + for (int32_t i = 0; i < numOfBatch; ++i) { + SGetUserAuthRsp rsp = {0}; + if (tDeserializeSGetUserAuthRspImpl(&decoder, &rsp) < 0) return -1; + taosArrayPush(pRsp->pArray, &rsp); + } + tEndDecode(&decoder); + + tCoderClear(&decoder); + return 0; +} + +void tFreeSUserAuthBatchRsp(SUserAuthBatchRsp* pRsp){ + int32_t numOfBatch = taosArrayGetSize(pRsp->pArray); + for (int32_t i = 0; i < numOfBatch; ++i) { + SGetUserAuthRsp *pUserAuthRsp = taosArrayGet(pRsp->pArray, i); + tFreeSGetUserAuthRsp(pUserAuthRsp); + } + + taosArrayDestroy(pRsp->pArray); +} + + int32_t tSerializeSDbCfgReq(void *buf, int32_t bufLen, SDbCfgReq *pReq) { SCoder encoder = {0}; tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); @@ -3860,3 +3952,65 @@ int32_t tDecodeSVDropStbReq(SCoder *pCoder, SVDropStbReq *pReq) { tEndDecode(pCoder); return 0; } + +static int32_t tEncodeSVSubmitBlk(SCoder *pCoder, const SVSubmitBlk *pBlock, int32_t flags) { + if (tStartEncode(pCoder) < 0) return -1; + + if (tEncodeI64(pCoder, pBlock->suid) < 0) return -1; + if (tEncodeI64(pCoder, pBlock->uid) < 0) return -1; + if (tEncodeI32v(pCoder, pBlock->sver) < 0) return -1; + if (tEncodeBinary(pCoder, pBlock->pData, pBlock->nData) < 0) return -1; + + if (flags & TD_AUTO_CREATE_TABLE) { + if (tEncodeSVCreateTbReq(pCoder, &pBlock->cTbReq) < 0) return -1; + } + + tEndEncode(pCoder); + return 0; +} + +static int32_t tDecodeSVSubmitBlk(SCoder *pCoder, SVSubmitBlk *pBlock, int32_t flags) { + if (tStartDecode(pCoder) < 0) return -1; + + if (tDecodeI64(pCoder, &pBlock->suid) < 0) return -1; + if (tDecodeI64(pCoder, &pBlock->uid) < 0) return -1; + if (tDecodeI32v(pCoder, &pBlock->sver) < 0) return -1; + if (tDecodeBinary(pCoder, &pBlock->pData, &pBlock->nData) < 0) return -1; + + if (flags & TD_AUTO_CREATE_TABLE) { + if (tDecodeSVCreateTbReq(pCoder, &pBlock->cTbReq) < 0) return -1; + } + + tEndDecode(pCoder); + return 0; +} + +int32_t tEncodeSVSubmitReq(SCoder *pCoder, const SVSubmitReq *pReq) { + int32_t nBlocks = taosArrayGetSize(pReq->pArray); + + if (tStartEncode(pCoder) < 0) return -1; + + if (tEncodeI32v(pCoder, pReq->flags) < 0) return -1; + if (tEncodeI32v(pCoder, nBlocks) < 0) return -1; + for (int32_t iBlock = 0; iBlock < nBlocks; iBlock++) { + if (tEncodeSVSubmitBlk(pCoder, (SVSubmitBlk *)taosArrayGet(pReq->pArray, iBlock), pReq->flags) < 0) return -1; + } + + tEndEncode(pCoder); + return 0; +} + +int32_t tDecodeSVSubmitReq(SCoder *pCoder, SVSubmitReq *pReq) { + if (tStartDecode(pCoder) < 0) return -1; + + if (tDecodeI32v(pCoder, &pReq->flags) < 0) return -1; + if (tDecodeI32v(pCoder, &pReq->nBlocks) < 0) return -1; + pReq->pBlocks = tCoderMalloc(pCoder, sizeof(SVSubmitBlk) * pReq->nBlocks); + if (pReq->pBlocks == NULL) return -1; + for (int32_t iBlock = 0; iBlock < pReq->nBlocks; iBlock++) { + if (tDecodeSVSubmitBlk(pCoder, pReq->pBlocks + iBlock, pReq->flags) < 0) return -1; + } + + tEndDecode(pCoder); + return 0; +} \ No newline at end of file diff --git a/source/common/src/trow.c b/source/common/src/trow.c index 590e6097c4..4c063b205f 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -684,6 +684,7 @@ _err: tdFreeDataCols(pTarget); return -1; } + static void tdAppendValToDataCols(SDataCols *target, SDataCols *src, int iter, bool isMerge) { for (int i = 0; i < src->numOfCols; ++i) { ASSERT(target->cols[i].type == src->cols[i].type); diff --git a/source/dnode/mgmt/implement/src/dmTransport.c b/source/dnode/mgmt/implement/src/dmTransport.c index 446894556e..114d7b6dfc 100644 --- a/source/dnode/mgmt/implement/src/dmTransport.c +++ b/source/dnode/mgmt/implement/src/dmTransport.c @@ -16,8 +16,8 @@ #define _DEFAULT_SOURCE #include "dmImp.h" -#define INTERNAL_USER "_dnd" -#define INTERNAL_CKEY "_key" +#define INTERNAL_USER "_dnd" +#define INTERNAL_CKEY "_key" #define INTERNAL_SECRET "_pwd" static void dmGetMnodeEpSet(SDnode *pDnode, SEpSet *pEpSet) { @@ -130,10 +130,10 @@ _OVER: } static void dmProcessMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { - SDnodeTrans *pTrans = &pDnode->trans; + SDnodeTrans * pTrans = &pDnode->trans; tmsg_t msgType = pMsg->msgType; bool isReq = msgType & 1u; - SMsgHandle *pHandle = &pTrans->msgHandles[TMSG_INDEX(msgType)]; + SMsgHandle * pHandle = &pTrans->msgHandles[TMSG_INDEX(msgType)]; SMgmtWrapper *pWrapper = pHandle->pNdWrapper; if (msgType == TDMT_DND_SERVER_STATUS) { @@ -517,7 +517,7 @@ static inline int32_t dmRetrieveUserAuthInfo(SDnode *pDnode, char *user, char *s SAuthReq authReq = {0}; tstrncpy(authReq.user, user, TSDB_USER_LEN); int32_t contLen = tSerializeSAuthReq(NULL, 0, &authReq); - void *pReq = rpcMallocCont(contLen); + void * pReq = rpcMallocCont(contLen); tSerializeSAuthReq(pReq, contLen, &authReq); SRpcMsg rpcMsg = {.pCont = pReq, .contLen = contLen, .msgType = TDMT_MND_AUTH, .ahandle = (void *)9528}; @@ -547,6 +547,8 @@ static int32_t dmInitServer(SDnode *pDnode) { SDnodeTrans *pTrans = &pDnode->trans; SRpcInit rpcInit = {0}; + + strncpy(rpcInit.localFqdn, pDnode->data.localFqdn, strlen(pDnode->data.localFqdn)); rpcInit.localPort = pDnode->data.serverPort; rpcInit.label = "DND"; rpcInit.numOfThreads = tsNumOfRpcThreads; diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 4a0df80358..89638524ae 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -255,8 +255,10 @@ typedef struct { int64_t updateTime; int8_t superUser; int32_t acctId; + int32_t authVersion; SHashObj* readDbs; SHashObj* writeDbs; + SRWLatch lock; } SUserObj; typedef struct { diff --git a/source/dnode/mnode/impl/inc/mndSma.h b/source/dnode/mnode/impl/inc/mndSma.h index 91c6e24e28..4a80f619d3 100644 --- a/source/dnode/mnode/impl/inc/mndSma.h +++ b/source/dnode/mnode/impl/inc/mndSma.h @@ -26,7 +26,6 @@ int32_t mndInitSma(SMnode *pMnode); void mndCleanupSma(SMnode *pMnode); SSmaObj *mndAcquireSma(SMnode *pMnode, char *smaName); void mndReleaseSma(SMnode *pMnode, SSmaObj *pSma); -int32_t mndProcessGetSmaReq(SMnode *pMnode, SUserIndexReq *indexReq, SUserIndexRsp *rsp, bool *exist); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/inc/mndUser.h b/source/dnode/mnode/impl/inc/mndUser.h index b3eb7f2f95..2140d0fa67 100644 --- a/source/dnode/mnode/impl/inc/mndUser.h +++ b/source/dnode/mnode/impl/inc/mndUser.h @@ -29,6 +29,7 @@ void mndReleaseUser(SMnode *pMnode, SUserObj *pUser); // for trans test SSdbRaw *mndUserActionEncode(SUserObj *pUser); +int32_t mndValidateUserAuthInfo(SMnode *pMnode, SUserAuthVersion *pUsers, int32_t numOfUses, void **ppRsp, int32_t *pRspLen); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/inc/mndVgroup.h b/source/dnode/mnode/impl/inc/mndVgroup.h index f42829eddf..8aecc22454 100644 --- a/source/dnode/mnode/impl/inc/mndVgroup.h +++ b/source/dnode/mnode/impl/inc/mndVgroup.h @@ -33,6 +33,7 @@ int32_t mndGetVnodesNum(SMnode *pMnode, int32_t dnodeId); void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen); void *mndBuildDropVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen); +void *mndBuildAlterVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index dfad99bdfb..af7efb5543 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -44,16 +44,17 @@ static int32_t mndProcessCompactDbReq(SNodeMsg *pReq); static int32_t mndRetrieveDbs(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rowsCapacity); static void mndCancelGetNextDb(SMnode *pMnode, void *pIter); static int32_t mndProcessGetDbCfgReq(SNodeMsg *pReq); -static int32_t mndProcessGetIndexReq(SNodeMsg *pReq); int32_t mndInitDb(SMnode *pMnode) { - SSdbTable table = {.sdbType = SDB_DB, - .keyType = SDB_KEY_BINARY, - .encodeFp = (SdbEncodeFp)mndDbActionEncode, - .decodeFp = (SdbDecodeFp)mndDbActionDecode, - .insertFp = (SdbInsertFp)mndDbActionInsert, - .updateFp = (SdbUpdateFp)mndDbActionUpdate, - .deleteFp = (SdbDeleteFp)mndDbActionDelete}; + SSdbTable table = { + .sdbType = SDB_DB, + .keyType = SDB_KEY_BINARY, + .encodeFp = (SdbEncodeFp)mndDbActionEncode, + .decodeFp = (SdbDecodeFp)mndDbActionDecode, + .insertFp = (SdbInsertFp)mndDbActionInsert, + .updateFp = (SdbUpdateFp)mndDbActionUpdate, + .deleteFp = (SdbDeleteFp)mndDbActionDelete, + }; mndSetMsgHandle(pMnode, TDMT_MND_CREATE_DB, mndProcessCreateDbReq); mndSetMsgHandle(pMnode, TDMT_MND_ALTER_DB, mndProcessAlterDbReq); @@ -61,7 +62,6 @@ int32_t mndInitDb(SMnode *pMnode) { mndSetMsgHandle(pMnode, TDMT_MND_USE_DB, mndProcessUseDbReq); mndSetMsgHandle(pMnode, TDMT_MND_COMPACT_DB, mndProcessCompactDbReq); mndSetMsgHandle(pMnode, TDMT_MND_GET_DB_CFG, mndProcessGetDbCfgReq); - mndSetMsgHandle(pMnode, TDMT_MND_GET_INDEX, mndProcessGetIndexReq); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_DB, mndRetrieveDbs); mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_DB, mndCancelGetNextDb); @@ -194,6 +194,7 @@ static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw) { } SDB_GET_RESERVE(pRaw, dataPos, DB_RESERVE_SIZE, _OVER) + taosInitRWLatch(&pDb->lock); terrno = 0; @@ -222,17 +223,29 @@ static int32_t mndDbActionDelete(SSdb *pSdb, SDbObj *pDb) { static int32_t mndDbActionUpdate(SSdb *pSdb, SDbObj *pOld, SDbObj *pNew) { mTrace("db:%s, perform update action, old row:%p new row:%p", pOld->name, pOld, pNew); taosWLockLatch(&pOld->lock); - SArray *pOldRetensions = pOld->cfg.pRetensions; pOld->updateTime = pNew->updateTime; pOld->cfgVersion = pNew->cfgVersion; pOld->vgVersion = pNew->vgVersion; - memcpy(&pOld->cfg, &pNew->cfg, sizeof(SDbCfg)); - pNew->cfg.pRetensions = pOldRetensions; + pOld->cfg.buffer = pNew->cfg.buffer; + pOld->cfg.pages = pNew->cfg.pages; + pOld->cfg.pageSize = pNew->cfg.pageSize; + pOld->cfg.daysPerFile = pNew->cfg.daysPerFile; + pOld->cfg.daysToKeep0 = pNew->cfg.daysToKeep0; + pOld->cfg.daysToKeep1 = pNew->cfg.daysToKeep1; + pOld->cfg.daysToKeep2 = pNew->cfg.daysToKeep2; + pOld->cfg.fsyncPeriod = pNew->cfg.fsyncPeriod; + pOld->cfg.walLevel = pNew->cfg.walLevel; + pOld->cfg.strict = pNew->cfg.strict; + pOld->cfg.cacheLastRow = pNew->cfg.cacheLastRow; + pOld->cfg.replications = pNew->cfg.replications; taosWUnLockLatch(&pOld->lock); return 0; } -static int32_t mndGetGlobalVgroupVersion(SMnode *pMnode) { return sdbGetTableVer(pMnode->pSdb, SDB_VGROUP); } +static inline int32_t mndGetGlobalVgroupVersion(SMnode *pMnode) { + SSdb *pSdb = pMnode->pSdb; + return sdbGetTableVer(pSdb, SDB_VGROUP); +} SDbObj *mndAcquireDb(SMnode *pMnode, const char *db) { SSdb *pSdb = pMnode->pSdb; @@ -638,69 +651,7 @@ static int32_t mndSetAlterDbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *p return 0; } -void *mndBuildAlterVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen) { - SAlterVnodeReq alterReq = {0}; - alterReq.vgVersion = pVgroup->version; - alterReq.buffer = pDb->cfg.buffer; - alterReq.pages = pDb->cfg.pages; - alterReq.pageSize = pDb->cfg.pageSize; - alterReq.daysPerFile = pDb->cfg.daysPerFile; - alterReq.daysToKeep0 = pDb->cfg.daysToKeep0; - alterReq.daysToKeep1 = pDb->cfg.daysToKeep1; - alterReq.daysToKeep2 = pDb->cfg.daysToKeep2; - alterReq.fsyncPeriod = pDb->cfg.fsyncPeriod; - alterReq.walLevel = pDb->cfg.walLevel; - alterReq.strict = pDb->cfg.strict; - alterReq.cacheLastRow = pDb->cfg.cacheLastRow; - alterReq.replica = pVgroup->replica; - alterReq.selfIndex = -1; - - for (int32_t v = 0; v < pVgroup->replica; ++v) { - SReplica *pReplica = &alterReq.replicas[v]; - SVnodeGid *pVgid = &pVgroup->vnodeGid[v]; - SDnodeObj *pVgidDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); - if (pVgidDnode == NULL) { - return NULL; - } - - pReplica->id = pVgidDnode->id; - pReplica->port = pVgidDnode->port; - memcpy(pReplica->fqdn, pVgidDnode->fqdn, TSDB_FQDN_LEN); - mndReleaseDnode(pMnode, pVgidDnode); - - if (pDnode->id == pVgid->dnodeId) { - alterReq.selfIndex = v; - } - } - - if (alterReq.selfIndex == -1) { - terrno = TSDB_CODE_MND_APP_ERROR; - return NULL; - } - - int32_t contLen = tSerializeSAlterVnodeReq(NULL, 0, &alterReq); - if (contLen < 0) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; - } - contLen += +sizeof(SMsgHead); - - void *pReq = taosMemoryMalloc(contLen); - if (pReq == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; - } - - SMsgHead *pHead = pReq; - pHead->contLen = htonl(contLen); - pHead->vgId = htonl(pVgroup->vgId); - - tSerializeSAlterVnodeReq((char *)pReq + sizeof(SMsgHead), contLen, &alterReq); - *pContLen = contLen; - return pReq; -} - -static int32_t mndBuilAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup) { +static int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup) { for (int32_t vn = 0; vn < pVgroup->replica; ++vn) { STransAction action = {0}; SVnodeGid *pVgid = pVgroup->vnodeGid + vn; @@ -736,7 +687,7 @@ static int32_t mndSetAlterDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj * if (pIter == NULL) break; if (pVgroup->dbUid == pNew->uid) { - if (mndBuilAlterVgroupAction(pMnode, pTrans, pNew, pVgroup) != 0) { + if (mndBuildAlterVgroupAction(pMnode, pTrans, pNew, pVgroup) != 0) { sdbCancelFetch(pSdb, pIter); sdbRelease(pSdb, pVgroup); return -1; @@ -752,19 +703,19 @@ static int32_t mndSetAlterDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj * static int32_t mndAlterDb(SMnode *pMnode, SNodeMsg *pReq, SDbObj *pOld, SDbObj *pNew) { int32_t code = -1; STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_ALTER_DB, &pReq->rpcMsg); - if (pTrans == NULL) goto UPDATE_DB_OVER; + if (pTrans == NULL) goto _OVER; mDebug("trans:%d, used to alter db:%s", pTrans->id, pOld->name); mndTransSetDbInfo(pTrans, pOld); - if (mndSetAlterDbRedoLogs(pMnode, pTrans, pOld, pNew) != 0) goto UPDATE_DB_OVER; - if (mndSetAlterDbCommitLogs(pMnode, pTrans, pOld, pNew) != 0) goto UPDATE_DB_OVER; - if (mndSetAlterDbRedoActions(pMnode, pTrans, pOld, pNew) != 0) goto UPDATE_DB_OVER; - if (mndTransPrepare(pMnode, pTrans) != 0) goto UPDATE_DB_OVER; + if (mndSetAlterDbRedoLogs(pMnode, pTrans, pOld, pNew) != 0) goto _OVER; + if (mndSetAlterDbCommitLogs(pMnode, pTrans, pOld, pNew) != 0) goto _OVER; + if (mndSetAlterDbRedoActions(pMnode, pTrans, pOld, pNew) != 0) goto _OVER; + if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; code = 0; -UPDATE_DB_OVER: +_OVER: mndTransDrop(pTrans); return code; } @@ -778,7 +729,7 @@ static int32_t mndProcessAlterDbReq(SNodeMsg *pReq) { if (tDeserializeSAlterDbReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &alterReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; - goto ALTER_DB_OVER; + goto _OVER; } mDebug("db:%s, start to alter", alterReq.db); @@ -786,24 +737,26 @@ static int32_t mndProcessAlterDbReq(SNodeMsg *pReq) { pDb = mndAcquireDb(pMnode, alterReq.db); if (pDb == NULL) { terrno = TSDB_CODE_MND_DB_NOT_EXIST; - goto ALTER_DB_OVER; + goto _OVER; } pUser = mndAcquireUser(pMnode, pReq->user); if (pUser == NULL) { - goto ALTER_DB_OVER; + goto _OVER; } if (mndCheckAlterDropCompactDbAuth(pUser, pDb) != 0) { - goto ALTER_DB_OVER; + goto _OVER; } SDbObj dbObj = {0}; memcpy(&dbObj, pDb, sizeof(SDbObj)); + dbObj.cfg.numOfRetensions = 0; + dbObj.cfg.pRetensions = NULL; code = mndSetDbCfgFromAlterDbReq(&dbObj, &alterReq); if (code != 0) { - goto ALTER_DB_OVER; + goto _OVER; } dbObj.cfgVersion++; @@ -811,7 +764,7 @@ static int32_t mndProcessAlterDbReq(SNodeMsg *pReq) { code = mndAlterDb(pMnode, pReq, pDb, &dbObj); if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; -ALTER_DB_OVER: +_OVER: if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { mError("db:%s, failed to alter since %s", alterReq.db, terrstr()); } @@ -831,13 +784,13 @@ static int32_t mndProcessGetDbCfgReq(SNodeMsg *pReq) { if (tDeserializeSDbCfgReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &cfgReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; - goto GET_DB_CFG_OVER; + goto _OVER; } pDb = mndAcquireDb(pMnode, cfgReq.db); if (pDb == NULL) { terrno = TSDB_CODE_MND_DB_NOT_EXIST; - goto GET_DB_CFG_OVER; + goto _OVER; } cfgRsp.numOfVgroups = pDb->cfg.numOfVgroups; @@ -866,7 +819,7 @@ static int32_t mndProcessGetDbCfgReq(SNodeMsg *pReq) { if (pRsp == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; code = -1; - goto GET_DB_CFG_OVER; + goto _OVER; } tSerializeSDbCfgRsp(pRsp, contLen, &cfgRsp); @@ -876,9 +829,9 @@ static int32_t mndProcessGetDbCfgReq(SNodeMsg *pReq) { code = 0; -GET_DB_CFG_OVER: +_OVER: - if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + if (code != 0) { mError("db:%s, failed to get cfg since %s", cfgReq.db, terrstr()); } @@ -1097,7 +1050,8 @@ _OVER: return code; } -void mndGetDBTableNum(SDbObj *pDb, SMnode *pMnode, int32_t *num) { +static int32_t mndGetDBTableNum(SDbObj *pDb, SMnode *pMnode) { + int32_t numOfTables = 0; int32_t vindex = 0; SSdb *pSdb = pMnode->pSdb; @@ -1108,8 +1062,7 @@ void mndGetDBTableNum(SDbObj *pDb, SMnode *pMnode, int32_t *num) { if (pIter == NULL) break; if (pVgroup->dbUid == pDb->uid) { - *num += pVgroup->numOfTables / TSDB_TABLE_NUM_UNIT; - + numOfTables += pVgroup->numOfTables / TSDB_TABLE_NUM_UNIT; vindex++; } @@ -1117,6 +1070,7 @@ void mndGetDBTableNum(SDbObj *pDb, SMnode *pMnode, int32_t *num) { } sdbCancelFetch(pSdb, pIter); + return numOfTables; } static void mndBuildDBVgroupInfo(SDbObj *pDb, SMnode *pMnode, SArray *pVgList) { @@ -1170,8 +1124,7 @@ int32_t mndExtractDbInfo(SMnode *pMnode, SDbObj *pDb, SUseDbRsp *pRsp, const SUs return -1; } - int32_t numOfTable = 0; - mndGetDBTableNum(pDb, pMnode, &numOfTable); + int32_t numOfTable = mndGetDBTableNum(pDb, pMnode); if (pReq == NULL || pReq->vgVersion < pDb->vgVersion || pReq->dbId != pDb->uid || numOfTable != pReq->numOfTable) { mndBuildDBVgroupInfo(pDb, pMnode, pRsp->pVgroupInfos); @@ -1195,7 +1148,7 @@ static int32_t mndProcessUseDbReq(SNodeMsg *pReq) { if (tDeserializeSUseDbReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &usedbReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; - goto USE_DB_OVER; + goto _OVER; } char *p = strchr(usedbReq.db, '.'); @@ -1206,12 +1159,11 @@ static int32_t mndProcessUseDbReq(SNodeMsg *pReq) { usedbRsp.pVgroupInfos = taosArrayInit(10, sizeof(SVgroupInfo)); if (usedbRsp.pVgroupInfos == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; - goto USE_DB_OVER; + goto _OVER; } mndBuildDBVgroupInfo(NULL, pMnode, usedbRsp.pVgroupInfos); usedbRsp.vgVersion = vgVersion++; - } else { usedbRsp.vgVersion = usedbReq.vgVersion; } @@ -1232,15 +1184,15 @@ static int32_t mndProcessUseDbReq(SNodeMsg *pReq) { } else { pUser = mndAcquireUser(pMnode, pReq->user); if (pUser == NULL) { - goto USE_DB_OVER; + goto _OVER; } if (mndCheckUseDbAuth(pUser, pDb) != 0) { - goto USE_DB_OVER; + goto _OVER; } if (mndExtractDbInfo(pMnode, pDb, &usedbRsp, &usedbReq) < 0) { - goto USE_DB_OVER; + goto _OVER; } code = 0; @@ -1252,7 +1204,7 @@ static int32_t mndProcessUseDbReq(SNodeMsg *pReq) { if (pRsp == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; code = -1; - goto USE_DB_OVER; + goto _OVER; } tSerializeSUseDbRsp(pRsp, contLen, &usedbRsp); @@ -1260,7 +1212,7 @@ static int32_t mndProcessUseDbReq(SNodeMsg *pReq) { pReq->pRsp = pRsp; pReq->rspLen = contLen; -USE_DB_OVER: +_OVER: if (code != 0) { mError("db:%s, failed to process use db req since %s", usedbReq.db, terrstr()); } @@ -1298,8 +1250,7 @@ int32_t mndValidateDbInfo(SMnode *pMnode, SDbVgVersion *pDbs, int32_t numOfDbs, continue; } - int32_t numOfTable = 0; - mndGetDBTableNum(pDb, pMnode, &numOfTable); + int32_t numOfTable = mndGetDBTableNum(pDb, pMnode); if (pDbVgVersion->vgVersion >= pDb->vgVersion && numOfTable == pDbVgVersion->numOfTable) { mDebug("db:%s, version & numOfTable not changed", pDbVgVersion->dbFName); @@ -1514,9 +1465,6 @@ static void dumpDbInfoData(SSDataBlock *pBlock, SDbObj *pDb, SShowObj *pShow, in pColInfo = taosArrayGet(pBlock->pDataBlock, cols); colDataAppend(pColInfo, rows, (const char *)b, false); } - - // pWrite = getDataPosition(data, pShow, cols, rows, rowCapacity); - // *(int8_t *)pWrite = pDb->cfg.update; } static void setInformationSchemaDbCfg(SDbObj *pDbObj) { @@ -1544,7 +1492,6 @@ static void setPerfSchemaDbCfg(SDbObj *pDbObj) { static bool mndGetTablesOfDbFp(SMnode *pMnode, void *pObj, void *p1, void *p2, void *p3) { SVgObj *pVgroup = pObj; int32_t *numOfTables = p1; - *numOfTables += pVgroup->numOfTables; return true; } @@ -1594,49 +1541,3 @@ static void mndCancelGetNextDb(SMnode *pMnode, void *pIter) { SSdb *pSdb = pMnode->pSdb; sdbCancelFetch(pSdb, pIter); } - -static int32_t mndProcessGetIndexReq(SNodeMsg *pReq) { - SUserIndexReq indexReq = {0}; - SMnode *pMnode = pReq->pNode; - int32_t code = -1; - SUserIndexRsp rsp = {0}; - bool exist = false; - - if (tDeserializeSUserIndexReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &indexReq) != 0) { - terrno = TSDB_CODE_INVALID_MSG; - goto _OVER; - } - - code = mndProcessGetSmaReq(pMnode, &indexReq, &rsp, &exist); - if (code) { - goto _OVER; - } - - if (!exist) { - // TODO GET INDEX FROM FULLTEXT - code = -1; - terrno = TSDB_CODE_MND_DB_INDEX_NOT_EXIST; - } else { - int32_t contLen = tSerializeSUserIndexRsp(NULL, 0, &rsp); - void *pRsp = rpcMallocCont(contLen); - if (pRsp == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - code = -1; - goto _OVER; - } - - tSerializeSUserIndexRsp(pRsp, contLen, &rsp); - - pReq->pRsp = pRsp; - pReq->rspLen = contLen; - - code = 0; - } - -_OVER: - if (code != 0) { - mError("failed to get index %s since %s", indexReq.indexFName, terrstr()); - } - - return code; -} diff --git a/source/dnode/mnode/impl/src/mndFunc.c b/source/dnode/mnode/impl/src/mndFunc.c index b9331e6e03..3ac2951b6f 100644 --- a/source/dnode/mnode/impl/src/mndFunc.c +++ b/source/dnode/mnode/impl/src/mndFunc.c @@ -314,7 +314,7 @@ static int32_t mndProcessCreateFuncReq(SNodeMsg *pReq) { goto _OVER; } - if (createReq.bufSize <= 0 || createReq.bufSize > TSDB_FUNC_BUF_SIZE) { + if (createReq.bufSize < 0 || createReq.bufSize > TSDB_FUNC_BUF_SIZE) { terrno = TSDB_CODE_MND_INVALID_FUNC_BUFSIZE; goto _OVER; } diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index 59d07fd4aa..2de337537f 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -403,6 +403,16 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb SKv *kv = pIter; switch (kv->key) { + case HEARTBEAT_KEY_USER_AUTHINFO: { + void * rspMsg = NULL; + int32_t rspLen = 0; + mndValidateUserAuthInfo(pMnode, kv->value, kv->valueLen / sizeof(SUserAuthVersion), &rspMsg, &rspLen); + if (rspMsg && rspLen > 0) { + SKv kv1 = {.key = HEARTBEAT_KEY_USER_AUTHINFO, .valueLen = rspLen, .value = rspMsg}; + taosArrayPush(hbRsp.info, &kv1); + } + break; + } case HEARTBEAT_KEY_DBINFO: { void * rspMsg = NULL; int32_t rspLen = 0; diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index 0e3e8b68cc..8619df978b 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -40,6 +40,7 @@ static int32_t mndProcessMCreateSmaReq(SNodeMsg *pReq); static int32_t mndProcessMDropSmaReq(SNodeMsg *pReq); static int32_t mndProcessVCreateSmaRsp(SNodeMsg *pRsp); static int32_t mndProcessVDropSmaRsp(SNodeMsg *pRsp); +static int32_t mndProcessGetSmaReq(SNodeMsg *pReq); static int32_t mndRetrieveSma(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static void mndCancelGetNextSma(SMnode *pMnode, void *pIter); @@ -56,6 +57,7 @@ int32_t mndInitSma(SMnode *pMnode) { mndSetMsgHandle(pMnode, TDMT_MND_DROP_SMA, mndProcessMDropSmaReq); mndSetMsgHandle(pMnode, TDMT_VND_CREATE_SMA_RSP, mndProcessVCreateSmaRsp); mndSetMsgHandle(pMnode, TDMT_VND_DROP_SMA_RSP, mndProcessVDropSmaRsp); + mndSetMsgHandle(pMnode, TDMT_MND_GET_INDEX, mndProcessGetSmaReq); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_INDEX, mndRetrieveSma); mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_INDEX, mndCancelGetNextSma); @@ -686,7 +688,7 @@ _OVER: return code; } -int32_t mndProcessGetSmaReq(SMnode *pMnode, SUserIndexReq *indexReq, SUserIndexRsp *rsp, bool *exist) { +static int32_t mndGetSma(SMnode *pMnode, SUserIndexReq *indexReq, SUserIndexRsp *rsp, bool *exist) { int32_t code = -1; SSmaObj *pSma = NULL; @@ -715,6 +717,51 @@ int32_t mndProcessGetSmaReq(SMnode *pMnode, SUserIndexReq *indexReq, SUserIndexR } mndReleaseSma(pMnode, pSma); + return code; +} + +static int32_t mndProcessGetSmaReq(SNodeMsg *pReq) { + SUserIndexReq indexReq = {0}; + SMnode *pMnode = pReq->pNode; + int32_t code = -1; + SUserIndexRsp rsp = {0}; + bool exist = false; + + if (tDeserializeSUserIndexReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &indexReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + goto _OVER; + } + + code = mndGetSma(pMnode, &indexReq, &rsp, &exist); + if (code) { + goto _OVER; + } + + if (!exist) { + // TODO GET INDEX FROM FULLTEXT + code = -1; + terrno = TSDB_CODE_MND_DB_INDEX_NOT_EXIST; + } else { + int32_t contLen = tSerializeSUserIndexRsp(NULL, 0, &rsp); + void *pRsp = rpcMallocCont(contLen); + if (pRsp == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + code = -1; + goto _OVER; + } + + tSerializeSUserIndexRsp(pRsp, contLen, &rsp); + + pReq->pRsp = pRsp; + pReq->rspLen = contLen; + + code = 0; + } + +_OVER: + if (code != 0) { + mError("failed to get index %s since %s", indexReq.indexFName, terrstr()); + } return code; } diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index f3438bd0dc..43e2735984 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -1050,7 +1050,7 @@ static int32_t mndTransExecuteActions(SMnode *pMnode, STrans *pTrans, SArray *pA static int32_t mndTransExecuteRedoActions(SMnode *pMnode, STrans *pTrans) { int32_t code = mndTransExecuteActions(pMnode, pTrans, pTrans->redoActions); - if (code != 0) { + if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { mError("failed to execute redoActions since %s", terrstr()); } return code; @@ -1058,7 +1058,7 @@ static int32_t mndTransExecuteRedoActions(SMnode *pMnode, STrans *pTrans) { static int32_t mndTransExecuteUndoActions(SMnode *pMnode, STrans *pTrans) { int32_t code = mndTransExecuteActions(pMnode, pTrans, pTrans->undoActions); - if (code != 0) { + if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { mError("failed to execute undoActions since %s", terrstr()); } return code; diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index 5e15bdeb43..8b5fa06faf 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -186,6 +186,7 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) { } SDB_GET_RESERVE(pRaw, dataPos, USER_RESERVE_SIZE, _OVER) + taosInitRWLatch(&pUser->lock); terrno = 0; @@ -228,11 +229,12 @@ static int32_t mndUserActionDelete(SSdb *pSdb, SUserObj *pUser) { static int32_t mndUserActionUpdate(SSdb *pSdb, SUserObj *pOld, SUserObj *pNew) { mTrace("user:%s, perform update action, old row:%p new row:%p", pOld->user, pOld, pNew); - memcpy(pOld->pass, pNew->pass, TSDB_PASSWORD_LEN); + taosWLockLatch(&pOld->lock); pOld->updateTime = pNew->updateTime; - + memcpy(pOld->pass, pNew->pass, TSDB_PASSWORD_LEN); TSWAP(pOld->readDbs, pNew->readDbs); TSWAP(pOld->writeDbs, pNew->writeDbs); + taosWUnLockLatch(&pOld->lock); return 0; } @@ -426,8 +428,12 @@ static int32_t mndProcessAlterUserReq(SNodeMsg *pReq) { } memcpy(&newUser, pUser, sizeof(SUserObj)); + + taosRLockLatch(&pUser->lock); newUser.readDbs = mndDupDbHash(pUser->readDbs); newUser.writeDbs = mndDupDbHash(pUser->writeDbs); + taosRUnLockLatch(&pUser->lock); + if (newUser.readDbs == NULL || newUser.writeDbs == NULL) { goto _OVER; } @@ -451,13 +457,16 @@ static int32_t mndProcessAlterUserReq(SNodeMsg *pReq) { terrno = TSDB_CODE_OUT_OF_MEMORY; goto _OVER; } + newUser.authVersion++; } else if (alterReq.alterType == TSDB_ALTER_USER_REMOVE_READ_DB) { if (taosHashRemove(newUser.readDbs, alterReq.dbname, len) != 0) { terrno = TSDB_CODE_MND_DB_NOT_EXIST; goto _OVER; } + newUser.authVersion++; } else if (alterReq.alterType == TSDB_ALTER_USER_CLEAR_READ_DB) { taosHashClear(newUser.readDbs); + newUser.authVersion++; } else if (alterReq.alterType == TSDB_ALTER_USER_ADD_WRITE_DB) { if (pDb == NULL) { terrno = TSDB_CODE_MND_DB_NOT_EXIST; @@ -467,13 +476,16 @@ static int32_t mndProcessAlterUserReq(SNodeMsg *pReq) { terrno = TSDB_CODE_OUT_OF_MEMORY; goto _OVER; } + newUser.authVersion++; } else if (alterReq.alterType == TSDB_ALTER_USER_REMOVE_WRITE_DB) { if (taosHashRemove(newUser.writeDbs, alterReq.dbname, len) != 0) { terrno = TSDB_CODE_MND_DB_NOT_EXIST; goto _OVER; } + newUser.authVersion++; } else if (alterReq.alterType == TSDB_ALTER_USER_CLEAR_WRITE_DB) { taosHashClear(newUser.writeDbs); + newUser.authVersion++; } else { terrno = TSDB_CODE_MND_INVALID_ALTER_OPER; goto _OVER; @@ -576,6 +588,38 @@ _OVER: return code; } +static int32_t mndSetUserAuthRsp(SMnode *pMnode, SUserObj *pUser, SGetUserAuthRsp *pRsp) { + memcpy(pRsp->user, pUser->user, TSDB_USER_LEN); + pRsp->superAuth = pUser->superUser; + pRsp->version = pUser->authVersion; + taosRLockLatch(&pUser->lock); + pRsp->readDbs = mndDupDbHash(pUser->readDbs); + pRsp->writeDbs = mndDupDbHash(pUser->writeDbs); + taosRUnLockLatch(&pUser->lock); + pRsp->createdDbs = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + if (NULL == pRsp->createdDbs) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + SSdb *pSdb = pMnode->pSdb; + void *pIter = NULL; + while (1) { + SDbObj *pDb = NULL; + pIter = sdbFetch(pSdb, SDB_DB, pIter, (void **)&pDb); + if (pIter == NULL) break; + + if (strcmp(pDb->createUser, pUser->user) == 0) { + int32_t len = strlen(pDb->name) + 1; + taosHashPut(pRsp->createdDbs, pDb->name, len, pDb->name, len); + } + + sdbRelease(pSdb, pDb); + } + + return 0; +} + static int32_t mndProcessGetUserAuthReq(SNodeMsg *pReq) { SMnode *pMnode = pReq->pNode; int32_t code = -1; @@ -596,25 +640,9 @@ static int32_t mndProcessGetUserAuthReq(SNodeMsg *pReq) { goto _OVER; } - memcpy(authRsp.user, pUser->user, TSDB_USER_LEN); - authRsp.superAuth = pUser->superUser; - authRsp.readDbs = mndDupDbHash(pUser->readDbs); - authRsp.writeDbs = mndDupDbHash(pUser->writeDbs); - - SSdb *pSdb = pMnode->pSdb; - void *pIter = NULL; - while (1) { - SDbObj *pDb = NULL; - pIter = sdbFetch(pSdb, SDB_DB, pIter, (void **)&pDb); - if (pIter == NULL) break; - - if (strcmp(pDb->createUser, pUser->user) == 0) { - int32_t len = strlen(pDb->name) + 1; - taosHashPut(authRsp.readDbs, pDb->name, len, pDb->name, len); - taosHashPut(authRsp.writeDbs, pDb->name, len, pDb->name, len); - } - - sdbRelease(pSdb, pDb); + code = mndSetUserAuthRsp(pMnode, pUser, &authRsp); + if (code) { + goto _OVER; } int32_t contLen = tSerializeSGetUserAuthRsp(NULL, 0, &authRsp); @@ -631,6 +659,7 @@ static int32_t mndProcessGetUserAuthReq(SNodeMsg *pReq) { code = 0; _OVER: + mndReleaseUser(pMnode, pUser); tFreeSGetUserAuthRsp(&authRsp); @@ -681,3 +710,72 @@ static void mndCancelGetNextUser(SMnode *pMnode, void *pIter) { SSdb *pSdb = pMnode->pSdb; sdbCancelFetch(pSdb, pIter); } + +int32_t mndValidateUserAuthInfo(SMnode *pMnode, SUserAuthVersion *pUsers, int32_t numOfUses, void **ppRsp, int32_t *pRspLen) { + SUserAuthBatchRsp batchRsp = {0}; + batchRsp.pArray = taosArrayInit(numOfUses, sizeof(SGetUserAuthRsp)); + if (batchRsp.pArray == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + int32_t code = 0; + for (int32_t i = 0; i < numOfUses; ++i) { + SUserObj *pUser = mndAcquireUser(pMnode, pUsers[i].user); + if (pUser == NULL) { + mError("user:%s, failed to auth user since %s", pUsers[i].user, terrstr()); + continue; + } + + if (pUser->authVersion <= pUsers[i].version) { + mndReleaseUser(pMnode, pUser); + continue; + } + + SGetUserAuthRsp rsp = {0}; + code = mndSetUserAuthRsp(pMnode, pUser, &rsp); + if (code) { + mndReleaseUser(pMnode, pUser); + tFreeSGetUserAuthRsp(&rsp); + goto _OVER; + } + + + taosArrayPush(batchRsp.pArray, &rsp); + mndReleaseUser(pMnode, pUser); + } + + if (taosArrayGetSize(batchRsp.pArray) <= 0) { + *ppRsp = NULL; + *pRspLen = 0; + + tFreeSUserAuthBatchRsp(&batchRsp); + return 0; + } + + int32_t rspLen = tSerializeSUserAuthBatchRsp(NULL, 0, &batchRsp); + void *pRsp = taosMemoryMalloc(rspLen); + if (pRsp == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + tFreeSUserAuthBatchRsp(&batchRsp); + return -1; + } + tSerializeSUserAuthBatchRsp(pRsp, rspLen, &batchRsp); + + *ppRsp = pRsp; + *pRspLen = rspLen; + + tFreeSUserAuthBatchRsp(&batchRsp); + return 0; + +_OVER: + + *ppRsp = NULL; + *pRspLen = 0; + + tFreeSUserAuthBatchRsp(&batchRsp); + return code; +} + + + diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 31e4a8ea7d..d1e4be1161 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -21,8 +21,8 @@ #include "mndShow.h" #include "mndTrans.h" -#define TSDB_VGROUP_VER_NUMBER 1 -#define TSDB_VGROUP_RESERVE_SIZE 64 +#define VGROUP_VER_NUMBER 1 +#define VGROUP_RESERVE_SIZE 64 static SSdbRow *mndVgroupActionDecode(SSdbRaw *pRaw); static int32_t mndVgroupActionInsert(SSdb *pSdb, SVgObj *pVgroup); @@ -34,19 +34,21 @@ static int32_t mndProcessAlterVnodeRsp(SNodeMsg *pRsp); static int32_t mndProcessDropVnodeRsp(SNodeMsg *pRsp); static int32_t mndProcessCompactVnodeRsp(SNodeMsg *pRsp); -static int32_t mndRetrieveVgroups(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* pBlock, int32_t rows); +static int32_t mndRetrieveVgroups(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static void mndCancelGetNextVgroup(SMnode *pMnode, void *pIter); -static int32_t mndRetrieveVnodes(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* pBlock, int32_t rows); +static int32_t mndRetrieveVnodes(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static void mndCancelGetNextVnode(SMnode *pMnode, void *pIter); int32_t mndInitVgroup(SMnode *pMnode) { - SSdbTable table = {.sdbType = SDB_VGROUP, - .keyType = SDB_KEY_INT32, - .encodeFp = (SdbEncodeFp)mndVgroupActionEncode, - .decodeFp = (SdbDecodeFp)mndVgroupActionDecode, - .insertFp = (SdbInsertFp)mndVgroupActionInsert, - .updateFp = (SdbUpdateFp)mndVgroupActionUpdate, - .deleteFp = (SdbDeleteFp)mndVgroupActionDelete}; + SSdbTable table = { + .sdbType = SDB_VGROUP, + .keyType = SDB_KEY_INT32, + .encodeFp = (SdbEncodeFp)mndVgroupActionEncode, + .decodeFp = (SdbDecodeFp)mndVgroupActionDecode, + .insertFp = (SdbInsertFp)mndVgroupActionInsert, + .updateFp = (SdbUpdateFp)mndVgroupActionUpdate, + .deleteFp = (SdbDeleteFp)mndVgroupActionDelete, + }; mndSetMsgHandle(pMnode, TDMT_DND_CREATE_VNODE_RSP, mndProcessCreateVnodeRsp); mndSetMsgHandle(pMnode, TDMT_VND_ALTER_VNODE_RSP, mndProcessAlterVnodeRsp); @@ -66,29 +68,29 @@ void mndCleanupVgroup(SMnode *pMnode) {} SSdbRaw *mndVgroupActionEncode(SVgObj *pVgroup) { terrno = TSDB_CODE_OUT_OF_MEMORY; - SSdbRaw *pRaw = sdbAllocRaw(SDB_VGROUP, TSDB_VGROUP_VER_NUMBER, sizeof(SVgObj) + TSDB_VGROUP_RESERVE_SIZE); - if (pRaw == NULL) goto VG_ENCODE_OVER; + SSdbRaw *pRaw = sdbAllocRaw(SDB_VGROUP, VGROUP_VER_NUMBER, sizeof(SVgObj) + VGROUP_RESERVE_SIZE); + if (pRaw == NULL) goto _OVER; int32_t dataPos = 0; - SDB_SET_INT32(pRaw, dataPos, pVgroup->vgId, VG_ENCODE_OVER) - SDB_SET_INT64(pRaw, dataPos, pVgroup->createdTime, VG_ENCODE_OVER) - SDB_SET_INT64(pRaw, dataPos, pVgroup->updateTime, VG_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, pVgroup->version, VG_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, pVgroup->hashBegin, VG_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, pVgroup->hashEnd, VG_ENCODE_OVER) - SDB_SET_BINARY(pRaw, dataPos, pVgroup->dbName, TSDB_DB_FNAME_LEN, VG_ENCODE_OVER) - SDB_SET_INT64(pRaw, dataPos, pVgroup->dbUid, VG_ENCODE_OVER) - SDB_SET_INT8(pRaw, dataPos, pVgroup->replica, VG_ENCODE_OVER) + SDB_SET_INT32(pRaw, dataPos, pVgroup->vgId, _OVER) + SDB_SET_INT64(pRaw, dataPos, pVgroup->createdTime, _OVER) + SDB_SET_INT64(pRaw, dataPos, pVgroup->updateTime, _OVER) + SDB_SET_INT32(pRaw, dataPos, pVgroup->version, _OVER) + SDB_SET_INT32(pRaw, dataPos, pVgroup->hashBegin, _OVER) + SDB_SET_INT32(pRaw, dataPos, pVgroup->hashEnd, _OVER) + SDB_SET_BINARY(pRaw, dataPos, pVgroup->dbName, TSDB_DB_FNAME_LEN, _OVER) + SDB_SET_INT64(pRaw, dataPos, pVgroup->dbUid, _OVER) + SDB_SET_INT8(pRaw, dataPos, pVgroup->replica, _OVER) for (int8_t i = 0; i < pVgroup->replica; ++i) { SVnodeGid *pVgid = &pVgroup->vnodeGid[i]; - SDB_SET_INT32(pRaw, dataPos, pVgid->dnodeId, VG_ENCODE_OVER) + SDB_SET_INT32(pRaw, dataPos, pVgid->dnodeId, _OVER) } - SDB_SET_RESERVE(pRaw, dataPos, TSDB_VGROUP_RESERVE_SIZE, VG_ENCODE_OVER) - SDB_SET_DATALEN(pRaw, dataPos, VG_ENCODE_OVER) + SDB_SET_RESERVE(pRaw, dataPos, VGROUP_RESERVE_SIZE, _OVER) + SDB_SET_DATALEN(pRaw, dataPos, _OVER) terrno = 0; -VG_ENCODE_OVER: +_OVER: if (terrno != 0) { mError("vgId:%d, failed to encode to raw:%p since %s", pVgroup->vgId, pRaw, terrstr()); sdbFreeRaw(pRaw); @@ -103,41 +105,41 @@ SSdbRow *mndVgroupActionDecode(SSdbRaw *pRaw) { terrno = TSDB_CODE_OUT_OF_MEMORY; int8_t sver = 0; - if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto VG_DECODE_OVER; + if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER; - if (sver != TSDB_VGROUP_VER_NUMBER) { + if (sver != VGROUP_VER_NUMBER) { terrno = TSDB_CODE_SDB_INVALID_DATA_VER; - goto VG_DECODE_OVER; + goto _OVER; } SSdbRow *pRow = sdbAllocRow(sizeof(SVgObj)); - if (pRow == NULL) goto VG_DECODE_OVER; + if (pRow == NULL) goto _OVER; SVgObj *pVgroup = sdbGetRowObj(pRow); - if (pVgroup == NULL) goto VG_DECODE_OVER; + if (pVgroup == NULL) goto _OVER; int32_t dataPos = 0; - SDB_GET_INT32(pRaw, dataPos, &pVgroup->vgId, VG_DECODE_OVER) - SDB_GET_INT64(pRaw, dataPos, &pVgroup->createdTime, VG_DECODE_OVER) - SDB_GET_INT64(pRaw, dataPos, &pVgroup->updateTime, VG_DECODE_OVER) - SDB_GET_INT32(pRaw, dataPos, &pVgroup->version, VG_DECODE_OVER) - SDB_GET_INT32(pRaw, dataPos, &pVgroup->hashBegin, VG_DECODE_OVER) - SDB_GET_INT32(pRaw, dataPos, &pVgroup->hashEnd, VG_DECODE_OVER) - SDB_GET_BINARY(pRaw, dataPos, pVgroup->dbName, TSDB_DB_FNAME_LEN, VG_DECODE_OVER) - SDB_GET_INT64(pRaw, dataPos, &pVgroup->dbUid, VG_DECODE_OVER) - SDB_GET_INT8(pRaw, dataPos, &pVgroup->replica, VG_DECODE_OVER) + SDB_GET_INT32(pRaw, dataPos, &pVgroup->vgId, _OVER) + SDB_GET_INT64(pRaw, dataPos, &pVgroup->createdTime, _OVER) + SDB_GET_INT64(pRaw, dataPos, &pVgroup->updateTime, _OVER) + SDB_GET_INT32(pRaw, dataPos, &pVgroup->version, _OVER) + SDB_GET_INT32(pRaw, dataPos, &pVgroup->hashBegin, _OVER) + SDB_GET_INT32(pRaw, dataPos, &pVgroup->hashEnd, _OVER) + SDB_GET_BINARY(pRaw, dataPos, pVgroup->dbName, TSDB_DB_FNAME_LEN, _OVER) + SDB_GET_INT64(pRaw, dataPos, &pVgroup->dbUid, _OVER) + SDB_GET_INT8(pRaw, dataPos, &pVgroup->replica, _OVER) for (int8_t i = 0; i < pVgroup->replica; ++i) { SVnodeGid *pVgid = &pVgroup->vnodeGid[i]; - SDB_GET_INT32(pRaw, dataPos, &pVgid->dnodeId, VG_DECODE_OVER) + SDB_GET_INT32(pRaw, dataPos, &pVgid->dnodeId, _OVER) if (pVgroup->replica == 1) { pVgid->role = TAOS_SYNC_STATE_LEADER; } } - SDB_GET_RESERVE(pRaw, dataPos, TSDB_VGROUP_RESERVE_SIZE, VG_DECODE_OVER) + SDB_GET_RESERVE(pRaw, dataPos, VGROUP_RESERVE_SIZE, _OVER) terrno = 0; -VG_DECODE_OVER: +_OVER: if (terrno != 0) { mError("vgId:%d, failed to decode from raw:%p since %s", pVgroup->vgId, pRaw, terrstr()); taosMemoryFreeClear(pRow); @@ -254,6 +256,68 @@ void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVg return pReq; } +void *mndBuildAlterVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen) { + SAlterVnodeReq alterReq = {0}; + alterReq.vgVersion = pVgroup->version; + alterReq.buffer = pDb->cfg.buffer; + alterReq.pages = pDb->cfg.pages; + alterReq.pageSize = pDb->cfg.pageSize; + alterReq.daysPerFile = pDb->cfg.daysPerFile; + alterReq.daysToKeep0 = pDb->cfg.daysToKeep0; + alterReq.daysToKeep1 = pDb->cfg.daysToKeep1; + alterReq.daysToKeep2 = pDb->cfg.daysToKeep2; + alterReq.fsyncPeriod = pDb->cfg.fsyncPeriod; + alterReq.walLevel = pDb->cfg.walLevel; + alterReq.strict = pDb->cfg.strict; + alterReq.cacheLastRow = pDb->cfg.cacheLastRow; + alterReq.replica = pVgroup->replica; + alterReq.selfIndex = -1; + + for (int32_t v = 0; v < pVgroup->replica; ++v) { + SReplica *pReplica = &alterReq.replicas[v]; + SVnodeGid *pVgid = &pVgroup->vnodeGid[v]; + SDnodeObj *pVgidDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); + if (pVgidDnode == NULL) { + return NULL; + } + + pReplica->id = pVgidDnode->id; + pReplica->port = pVgidDnode->port; + memcpy(pReplica->fqdn, pVgidDnode->fqdn, TSDB_FQDN_LEN); + mndReleaseDnode(pMnode, pVgidDnode); + + if (pDnode->id == pVgid->dnodeId) { + alterReq.selfIndex = v; + } + } + + if (alterReq.selfIndex == -1) { + terrno = TSDB_CODE_MND_APP_ERROR; + return NULL; + } + + int32_t contLen = tSerializeSAlterVnodeReq(NULL, 0, &alterReq); + if (contLen < 0) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + contLen += +sizeof(SMsgHead); + + void *pReq = taosMemoryMalloc(contLen); + if (pReq == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + SMsgHead *pHead = pReq; + pHead->contLen = htonl(contLen); + pHead->vgId = htonl(pVgroup->vgId); + + tSerializeSAlterVnodeReq((char *)pReq + sizeof(SMsgHead), contLen, &alterReq); + *pContLen = contLen; + return pReq; +} + void *mndBuildDropVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen) { SDropVnodeReq dropReq = {0}; dropReq.dnodeId = pDnode->id; @@ -372,12 +436,12 @@ int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups) { pVgroups = taosMemoryCalloc(pDb->cfg.numOfVgroups, sizeof(SVgObj)); if (pVgroups == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; - goto ALLOC_VGROUP_OVER; + goto _OVER; } pArray = mndBuildDnodesArray(pMnode); if (pArray == NULL) { - goto ALLOC_VGROUP_OVER; + goto _OVER; } mDebug("db:%s, total %d dnodes used to create %d vgroups (%d vnodes)", pDb->name, (int32_t)taosArrayGetSize(pArray), @@ -410,7 +474,7 @@ int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups) { if (mndGetAvailableDnode(pMnode, pVgroup, pArray) != 0) { terrno = TSDB_CODE_MND_NO_ENOUGH_DNODES; - goto ALLOC_VGROUP_OVER; + goto _OVER; } allocedVgroups++; @@ -421,7 +485,7 @@ int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups) { mDebug("db:%s, %d vgroups is alloced, replica:%d", pDb->name, pDb->cfg.numOfVgroups, pDb->cfg.replications); -ALLOC_VGROUP_OVER: +_OVER: if (code != 0) taosMemoryFree(pVgroups); taosArrayDestroy(pArray); return code; @@ -492,7 +556,7 @@ static int32_t mndGetVgroupMaxReplica(SMnode *pMnode, char *dbName, int8_t *pRep return 0; } -static int32_t mndRetrieveVgroups(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* pBlock, int32_t rows) { +static int32_t mndRetrieveVgroups(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; int32_t numOfRows = 0; @@ -533,14 +597,13 @@ static int32_t mndRetrieveVgroups(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* // default 3 replica for (int32_t i = 0; i < 3; ++i) { - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); if (i < pVgroup->replica) { colDataAppend(pColInfo, numOfRows, (const char *)&pVgroup->vnodeGid[i].dnodeId, false); char buf1[20] = {0}; const char *role = syncStr(pVgroup->vnodeGid[i].role); - STR_WITH_MAXSIZE_TO_VARSTR(buf1, role, pShow->pMeta->pSchemas[cols].bytes); + STR_WITH_MAXSIZE_TO_VARSTR(buf1, role, pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)buf1, false); @@ -597,13 +660,12 @@ int32_t mndGetVnodesNum(SMnode *pMnode, int32_t dnodeId) { return numOfVnodes; } -static int32_t mndRetrieveVnodes(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* pBlock, int32_t rows) { +static int32_t mndRetrieveVnodes(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; int32_t numOfRows = 0; SVgObj *pVgroup = NULL; int32_t cols = 0; -// int32_t dnodeId = pShow->replica; while (numOfRows < rows) { pShow->pIter = sdbFetch(pSdb, SDB_VGROUP, pShow->pIter, (void **)&pVgroup); diff --git a/source/dnode/mnode/impl/test/func/func.cpp b/source/dnode/mnode/impl/test/func/func.cpp index 0473fa375e..c8f832160b 100644 --- a/source/dnode/mnode/impl/test/func/func.cpp +++ b/source/dnode/mnode/impl/test/func/func.cpp @@ -24,6 +24,7 @@ class MndTestFunc : public ::testing::Test { void SetCode(SCreateFuncReq* pReq, const char* pCode, int32_t size); void SetComment(SCreateFuncReq* pReq, const char* pComment); + void SetBufSize(SCreateFuncReq* pReq, int32_t size); }; Testbase MndTestFunc::test; @@ -40,6 +41,10 @@ void MndTestFunc::SetComment(SCreateFuncReq* pReq, const char* pComment) { strcpy(pReq->pComment, pComment); } +void MndTestFunc::SetBufSize(SCreateFuncReq* pReq, int32_t size) { + pReq->bufSize = size; +} + TEST_F(MndTestFunc, 01_Show_Func) { test.SendShowReq(TSDB_MGMT_TABLE_FUNC, "user_functions", ""); EXPECT_EQ(test.GetShowRows(), 0); @@ -96,6 +101,7 @@ TEST_F(MndTestFunc, 02_Create_Func) { strcpy(createReq.name, "f1"); SetCode(&createReq, "code1", 6); SetComment(&createReq, "comment1"); + SetBufSize(&createReq, -1); int32_t contLen = tSerializeSCreateFuncReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index 85d0f76268..58e00ee34a 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -30,6 +30,7 @@ target_sources( "src/tsdb/tsdbFS.c" "src/tsdb/tsdbOpen.c" "src/tsdb/tsdbMemTable.c" + "src/tsdb/tsdbMemTable2.c" "src/tsdb/tsdbRead.c" "src/tsdb/tsdbReadImpl.c" "src/tsdb/tsdbSma.c" diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index ffa6f81c0d..d84c1b1d07 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -40,7 +40,6 @@ typedef struct STable STable; int tsdbMemTableCreate(STsdb *pTsdb, STsdbMemTable **ppMemTable); void tsdbMemTableDestroy(STsdb *pTsdb, STsdbMemTable *pMemTable); -int tsdbInsertTableData(STsdb *pTsdb, SSubmitMsgIter *pMsgIter, SSubmitBlk *pBlock, int32_t *pAffectedRows); int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey, int maxRowsToRead, SDataCols *pCols, TKEY *filterKeys, int nFilterKeys, bool keepDup, SMergeInfo *pMergeInfo); diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 5538a63b4b..3724a98a5e 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -100,6 +100,7 @@ int32_t tsdbUpdateSmaWindow(STsdb* pTsdb, SSubmitReq* pMsg, int64_t version int32_t tsdbCreateTSma(STsdb* pTsdb, char* pMsg); int32_t tsdbInsertTSmaData(STsdb* pTsdb, int64_t indexUid, const char* msg); int tsdbInsertData(STsdb* pTsdb, int64_t version, SSubmitReq* pMsg, SSubmitRsp* pRsp); +int tsdbInsertTableData(STsdb* pTsdb, SSubmitMsgIter* pMsgIter, SSubmitBlk* pBlock, int32_t* pAffectedRows); tsdbReaderT* tsdbQueryTables(SVnode* pVnode, SQueryTableDataCond* pCond, STableGroupInfo* groupList, uint64_t qId, uint64_t taskId); tsdbReaderT tsdbQueryCacheLastT(STsdb* tsdb, SQueryTableDataCond* pCond, STableGroupInfo* groupList, uint64_t qId, @@ -125,7 +126,6 @@ int32_t tsdbUpdateTbUidList(STsdb* pTsdb, STbUidStore* pUidStore); void tsdbUidStoreDestory(STbUidStore* pStore); void* tsdbUidStoreFree(STbUidStore* pStore); int32_t tsdbTriggerRSma(STsdb* pTsdb, void* pMsg, int32_t inputType); -int32_t tsdbProcessSubmitReq(STsdb* pTsdb, int64_t version, void* pReq); typedef struct { int8_t streamType; // sma or other diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 754023c49d..b255622c60 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -69,10 +69,6 @@ static void tsdbDestroyCommitIters(SCommitH *pCommith); static int tsdbCommitToFile(SCommitH *pCommith, SDFileSet *pSet, int fid); static void tsdbResetCommitFile(SCommitH *pCommith); static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid); -// static int tsdbCommitMeta(STsdbRepo *pRepo); -// static int tsdbUpdateMetaRecord(STsdbFS *pfs, SMFile *pMFile, uint64_t uid, void *cont, int contLen, bool compact); -// static int tsdbDropMetaRecord(STsdbFS *pfs, SMFile *pMFile, uint64_t uid); -// static int tsdbCompactMetaFile(STsdbRepo *pRepo, STsdbFS *pfs, SMFile *pMFile); static int tsdbCommitToTable(SCommitH *pCommith, int tid); static int tsdbSetCommitTable(SCommitH *pCommith, STable *pTable); static int tsdbComparKeyBlock(const void *arg1, const void *arg2); diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index ebe5ce9a8f..6520f371f6 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -85,6 +85,7 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey STSchema *pSchema = NULL; TSKEY rowKey = 0; TSKEY fKey = 0; + TSKEY lastKey = TSKEY_INITIAL_VAL; bool isRowDel = false; int filterIter = 0; STSRow *row = NULL; @@ -111,7 +112,8 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey } else { fKey = tdGetKey(filterKeys[filterIter]); } - + // 1. fkey - no dup since merged up to maxVersion of each query handle by tsdbLoadBlockDataCols + // 2. rowKey - would dup since Multi-Version supported while (true) { if (fKey == INT64_MAX && rowKey == INT64_MAX) break; @@ -125,12 +127,14 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey } else { fKey = tdGetKey(filterKeys[filterIter]); } +#if 0 } else if (fKey > rowKey) { if (isRowDel) { pMergeInfo->rowsDeleteFailed++; } else { if (pMergeInfo->rowsInserted - pMergeInfo->rowsDeleteSucceed >= maxRowsToRead) break; if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break; + pMergeInfo->rowsInserted++; pMergeInfo->nOperations++; pMergeInfo->keyFirst = TMIN(pMergeInfo->keyFirst, rowKey); @@ -178,6 +182,45 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey isRowDel = TD_ROW_IS_DELETED(row); } + filterIter++; + if (filterIter >= nFilterKeys) { + fKey = INT64_MAX; + } else { + fKey = tdGetKey(filterKeys[filterIter]); + } + } +#endif + } else { // fkey >= rowKey + if (isRowDel) { + ASSERT(!keepDup); + if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break; + pMergeInfo->rowsDeleteSucceed++; + pMergeInfo->nOperations++; + tsdbAppendTableRowToCols(pTable, pCols, &pSchema, row); + } else { + if (keepDup) { + if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break; + pMergeInfo->rowsUpdated++; + pMergeInfo->nOperations++; + pMergeInfo->keyFirst = TMIN(pMergeInfo->keyFirst, rowKey); + pMergeInfo->keyLast = TMAX(pMergeInfo->keyLast, rowKey); + tsdbAppendTableRowToCols(pTable, pCols, &pSchema, row); + } else { + pMergeInfo->keyFirst = TMIN(pMergeInfo->keyFirst, fKey); + pMergeInfo->keyLast = TMAX(pMergeInfo->keyLast, fKey); + } + } + + tSkipListIterNext(pIter); + row = tsdbNextIterRow(pIter); + if (row == NULL || TD_ROW_KEY(row) > maxKey) { + rowKey = INT64_MAX; + isRowDel = false; + } else { + rowKey = TD_ROW_KEY(row); + isRowDel = TD_ROW_IS_DELETED(row); + } + filterIter++; if (filterIter >= nFilterKeys) { fKey = INT64_MAX; @@ -191,9 +234,6 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey } int tsdbInsertTableData(STsdb *pTsdb, SSubmitMsgIter *pMsgIter, SSubmitBlk *pBlock, int32_t *pAffectedRows) { - // STsdbMeta *pMeta = pRepo->tsdbMeta; - // int32_t points = 0; - // STable *pTable = NULL; SSubmitBlkIter blkIter = {0}; STsdbMemTable *pMemTable = pTsdb->mem; void *tptr; @@ -221,8 +261,9 @@ int tsdbInsertTableData(STsdb *pTsdb, SSubmitMsgIter *pMsgIter, SSubmitBlk *pBlo } // copy data to buffer pool - pBlkCopy = (SSubmitBlk *)vnodeBufPoolMalloc(pTsdb->mem->pPool, pMsgIter->dataLen + sizeof(*pBlock)); - memcpy(pBlkCopy, pBlock, pMsgIter->dataLen + sizeof(*pBlock)); + int32_t tlen = pMsgIter->dataLen + pMsgIter->schemaLen + sizeof(*pBlock); + pBlkCopy = (SSubmitBlk *)vnodeBufPoolMalloc(pTsdb->mem->pPool, tlen); + memcpy(pBlkCopy, pBlock, tlen); tInitSubmitBlkIter(pMsgIter, pBlkCopy, &blkIter); if (blkIter.row == NULL) return 0; @@ -241,7 +282,7 @@ int tsdbInsertTableData(STsdb *pTsdb, SSubmitMsgIter *pMsgIter, SSubmitBlk *pBlo if (pMemTable->keyMin > keyMin) pMemTable->keyMin = keyMin; if (pMemTable->keyMax < keyMax) pMemTable->keyMax = keyMax; - (*pAffectedRows) += pMsgIter->numOfRows; + (*pAffectedRows) = pMsgIter->numOfRows; return 0; } diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c new file mode 100644 index 0000000000..ae0d4630d1 --- /dev/null +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c @@ -0,0 +1,93 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tsdb.h" + +typedef struct SMemTable SMemTable; +typedef struct SMemData SMemData; +typedef struct SMemSkipList SMemSkipList; +typedef struct SMemSkipListCfg SMemSkipListCfg; + +struct SMemTable { + STsdb *pTsdb; + TSKEY minKey; + TSKEY maxKey; + int64_t minVer; + int64_t maxVer; + int64_t nRows; + int32_t nHash; + int32_t nBucket; + SMemData **pBuckets; +}; + +struct SMemSkipListCfg { + int8_t maxLevel; + int32_t nKey; + int32_t nData; +}; + +struct SMemSkipList { + int8_t level; + uint32_t seed; +}; + +struct SMemData { + SMemData *pHashNext; + tb_uid_t suid; + tb_uid_t uid; + TSKEY minKey; + TSKEY maxKey; + int64_t minVer; + int64_t maxVer; + int64_t nRows; + SMemSkipList sl; +}; + +// SMemTable +int32_t tsdbMemTableCreate2(STsdb *pTsdb, SMemTable **ppMemTb) { + SMemTable *pMemTb = NULL; + + pMemTb = taosMemoryCalloc(1, sizeof(*pMemTb)); + if (pMemTb == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + pMemTb->pTsdb = pTsdb; + pMemTb->minKey = TSKEY_MAX; + pMemTb->maxKey = TSKEY_MIN; + pMemTb->minVer = -1; + pMemTb->maxVer = -1; + pMemTb->nRows = 0; + pMemTb->nHash = 0; + pMemTb->nBucket = 1024; + pMemTb->pBuckets = taosMemoryCalloc(pMemTb->nBucket, sizeof(*pMemTb->pBuckets)); + if (pMemTb->pBuckets == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + *ppMemTb = pMemTb; + return 0; +} + +int32_t tsdbMemTableDestroy2(STsdb *pTsdb, SMemTable *pMT) { + // TODO + return 0; +} + +// SMemData + +// SMemSkipList \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 4f27e4616a..54b3eea988 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -884,9 +884,11 @@ static STSRow* getSRowInTableMem(STableCheckInfo* pCheckInfo, int32_t order, int SSkipListNode* node = tSkipListIterGet(pCheckInfo->iter); if (node != NULL) { rmem = (STSRow*)SL_GET_NODE_DATA(node); +#if 0 // TODO: skiplist refactor if (TD_ROW_VER(rmem) > maxVer) { rmem = NULL; } +#endif } } @@ -894,9 +896,11 @@ static STSRow* getSRowInTableMem(STableCheckInfo* pCheckInfo, int32_t order, int SSkipListNode* node = tSkipListIterGet(pCheckInfo->iiter); if (node != NULL) { rimem = (STSRow*)SL_GET_NODE_DATA(node); +#if 0 // TODO: skiplist refactor if (TD_ROW_VER(rimem) > maxVer) { rimem = NULL; } +#endif } } diff --git a/source/dnode/vnode/src/tsdb/tsdbReadImpl.c b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c index 8f68883fde..49d02adc3a 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadImpl.c +++ b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c @@ -294,6 +294,7 @@ int tsdbLoadBlockData(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo) { return 0; } +// TODO: filter by Multi-Version int tsdbLoadBlockDataCols(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo, const int16_t *colIds, int numOfColsIds, bool mergeBitmap) { ASSERT(pBlock->numOfSubBlocks > 0); diff --git a/source/dnode/vnode/src/tsdb/tsdbSma.c b/source/dnode/vnode/src/tsdb/tsdbSma.c index 83adb545ba..105bc330d3 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSma.c +++ b/source/dnode/vnode/src/tsdb/tsdbSma.c @@ -1965,6 +1965,21 @@ int32_t tsdbUpdateTbUidList(STsdb *pTsdb, STbUidStore *pStore) { return TSDB_CODE_SUCCESS; } +static int32_t tsdbProcessSubmitReq(STsdb *pTsdb, int64_t version, void *pReq) { + if (!pReq) { + terrno = TSDB_CODE_INVALID_PTR; + return TSDB_CODE_FAILED; + } + + SSubmitReq *pSubmitReq = (SSubmitReq *)pReq; + + if (tsdbInsertData(pTsdb, version, pSubmitReq, NULL) < 0) { + return TSDB_CODE_FAILED; + } + + return TSDB_CODE_SUCCESS; +} + static int32_t tsdbFetchSubmitReqSuids(SSubmitReq *pMsg, STbUidStore *pStore) { ASSERT(pMsg != NULL); SSubmitMsgIter msgIter = {0}; diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 878dd5fca4..8460400b59 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -339,7 +339,7 @@ static int vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pReq, goto _exit; } - rsp.pArray = taosArrayInit(sizeof(cRsp), req.nReqs); + rsp.pArray = taosArrayInit(req.nReqs, sizeof(cRsp)); if (rsp.pArray == NULL) { rcode = -1; terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -450,7 +450,7 @@ static int vnodeProcessDropTbReq(SVnode *pVnode, int64_t version, void *pReq, in SCoder coder = {0}; int ret; - pRsp->msgType = TDMT_VND_CREATE_STB_RSP; + pRsp->msgType = TDMT_VND_DROP_TABLE_RSP; pRsp->pCont = NULL; pRsp->contLen = 0; pRsp->code = TSDB_CODE_SUCCESS; @@ -473,9 +473,13 @@ static int vnodeProcessDropTbReq(SVnode *pVnode, int64_t version, void *pReq, in /* code */ ret = metaDropTable(pVnode->pMeta, version, pDropTbReq); if (ret < 0) { - dropTbRsp.code = TSDB_CODE_SUCCESS; + if (pDropTbReq->igNotExists && terrno == TSDB_CODE_VND_TABLE_NOT_EXIST) { + dropTbRsp.code = TSDB_CODE_SUCCESS; + } else { + dropTbRsp.code = terrno; + } } else { - dropTbRsp.code = terrno; + dropTbRsp.code = TSDB_CODE_SUCCESS; } taosArrayPush(rsp.pArray, &dropTbRsp); @@ -488,20 +492,55 @@ _exit: } static int vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { - SSubmitReq *pSubmitReq = (SSubmitReq *)pReq; - SSubmitRsp rsp = {0}; + SSubmitReq *pSubmitReq = (SSubmitReq *)pReq; + SSubmitMsgIter msgIter = {0}; + SSubmitBlk *pBlock; + SSubmitRsp rsp = {0}; + SVCreateTbReq createTbReq = {0}; + SCoder coder = {0}; + int32_t nRows; pRsp->code = 0; // handle the request - if (tsdbInsertData(pVnode->pTsdb, version, pSubmitReq, &rsp) < 0) { - pRsp->code = terrno; - return -1; + if (tInitSubmitMsgIter(pSubmitReq, &msgIter) < 0) { + pRsp->code = TSDB_CODE_INVALID_MSG; + goto _exit; } - // pRsp->msgType = TDMT_VND_SUBMIT_RSP; - // vnodeProcessSubmitReq(pVnode, ptr, pRsp); + for (;;) { + tGetSubmitMsgNext(&msgIter, &pBlock); + if (pBlock == NULL) break; + // create table for auto create table mode + if (msgIter.schemaLen > 0) { + tCoderInit(&coder, TD_LITTLE_ENDIAN, pBlock->data, msgIter.schemaLen, TD_DECODER); + if (tDecodeSVCreateTbReq(&coder, &createTbReq) < 0) { + pRsp->code = TSDB_CODE_INVALID_MSG; + tCoderClear(&coder); + goto _exit; + } + + if (metaCreateTable(pVnode->pMeta, version, &createTbReq) < 0) { + if (terrno != TSDB_CODE_TDB_TABLE_ALREADY_EXIST) { + pRsp->code = terrno; + tCoderClear(&coder); + goto _exit; + } + } + + tCoderClear(&coder); + } + + if (tsdbInsertTableData(pVnode->pTsdb, &msgIter, pBlock, &nRows) < 0) { + pRsp->code = terrno; + goto _exit; + } + + rsp.numOfRows += nRows; + } + +_exit: // encode the response (TODO) pRsp->pCont = rpcMallocCont(sizeof(SSubmitRsp)); memcpy(pRsp->pCont, &rsp, sizeof(rsp)); @@ -511,18 +550,3 @@ static int vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, in return 0; } - -int32_t tsdbProcessSubmitReq(STsdb *pTsdb, int64_t version, void *pReq) { - if (!pReq) { - terrno = TSDB_CODE_INVALID_PTR; - return TSDB_CODE_FAILED; - } - - SSubmitReq *pSubmitReq = (SSubmitReq *)pReq; - - if (tsdbInsertData(pTsdb, version, pSubmitReq, NULL) < 0) { - return TSDB_CODE_FAILED; - } - - return TSDB_CODE_SUCCESS; -} diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 918892b786..3e8528e3d9 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -54,6 +54,7 @@ enum { CTG_ACT_REMOVE_DB, CTG_ACT_REMOVE_STB, CTG_ACT_REMOVE_TBL, + CTG_ACT_UPDATE_USER, CTG_ACT_MAX }; @@ -95,8 +96,18 @@ typedef struct SCtgRentMgmt { SCtgRentSlot *slots; } SCtgRentMgmt; +typedef struct SCtgUserAuth { + int32_t version; + SRWLatch lock; + bool superUser; + SHashObj *createdDbs; + SHashObj *readDbs; + SHashObj *writeDbs; +} SCtgUserAuth; + typedef struct SCatalog { uint64_t clusterId; + SHashObj *userCache; //key:user, value:SCtgUserAuth SHashObj *dbCache; //key:dbname, value:SCtgDBCache SCtgRentMgmt dbRent; SCtgRentMgmt stbRent; @@ -124,6 +135,8 @@ typedef struct SCtgCacheStat { uint64_t vgMissNum; uint64_t tblHitNum; uint64_t tblMissNum; + uint64_t userHitNum; + uint64_t userMissNum; } SCtgCacheStat; typedef struct SCatalogStat { @@ -169,6 +182,11 @@ typedef struct SCtgRemoveTblMsg { uint64_t dbId; } SCtgRemoveTblMsg; +typedef struct SCtgUpdateUserMsg { + SCatalog* pCtg; + SGetUserAuthRsp userAuth; +} SCtgUpdateUserMsg; + typedef struct SCtgMetaAction { int32_t act; @@ -234,6 +252,8 @@ typedef struct SCtgAction { #define CTG_FLAG_SYS_DB 0x8 #define CTG_FLAG_FORCE_UPDATE 0x10 +#define CTG_FLAG_SET(_flag, _v) ((_flag) |= (_v)) + #define CTG_FLAG_IS_STB(_flag) ((_flag) & CTG_FLAG_STB) #define CTG_FLAG_IS_NOT_STB(_flag) ((_flag) & CTG_FLAG_NOT_STB) #define CTG_FLAG_IS_UNKNOWN_STB(_flag) ((_flag) & CTG_FLAG_UNKNOWN_STB) diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 6f1f34a57b..f485f85809 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -24,6 +24,7 @@ int32_t ctgActUpdateTbl(SCtgMetaAction *action); int32_t ctgActRemoveDB(SCtgMetaAction *action); int32_t ctgActRemoveStb(SCtgMetaAction *action); int32_t ctgActRemoveTbl(SCtgMetaAction *action); +int32_t ctgActUpdateUser(SCtgMetaAction *action); extern SCtgDebug gCTGDebug; SCatalogMgmt gCtgMgmt = {0}; @@ -51,6 +52,11 @@ SCtgAction gCtgAction[CTG_ACT_MAX] = {{ CTG_ACT_REMOVE_TBL, "remove tbMeta", ctgActRemoveTbl + }, + { + CTG_ACT_UPDATE_USER, + "update user", + ctgActUpdateUser } }; @@ -357,6 +363,31 @@ _return: CTG_RET(code); } +int32_t ctgPushUpdateUserMsgInQueue(SCatalog* pCtg, SGetUserAuthRsp *pAuth, bool syncReq) { + int32_t code = 0; + SCtgMetaAction action= {.act = CTG_ACT_UPDATE_USER, .syncReq = syncReq}; + SCtgUpdateUserMsg *msg = taosMemoryMalloc(sizeof(SCtgUpdateUserMsg)); + if (NULL == msg) { + ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateUserMsg)); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + } + + msg->pCtg = pCtg; + msg->userAuth = *pAuth; + + action.data = msg; + + CTG_ERR_JRET(ctgPushAction(pCtg, &action)); + + return TSDB_CODE_SUCCESS; + +_return: + + tFreeSGetUserAuthRsp(pAuth); + taosMemoryFreeClear(msg); + + CTG_RET(code); +} int32_t ctgAcquireVgInfo(SCatalog *pCtg, SCtgDBCache *dbCache, bool *inCache) { CTG_LOCK(CTG_READ, &dbCache->vgLock); @@ -687,6 +718,43 @@ int32_t ctgGetUdfInfoFromMnode(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEp return TSDB_CODE_SUCCESS; } +int32_t ctgGetUserDbAuthFromMnode(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char *user, SGetUserAuthRsp *authRsp) { + char *msg = NULL; + int32_t msgLen = 0; + + ctgDebug("try to get user auth from mnode, user:%s", user); + + int32_t code = queryBuildMsg[TMSG_INDEX(TDMT_MND_GET_USER_AUTH)]((void *)user, &msg, 0, &msgLen); + if (code) { + ctgError("Build get user auth msg failed, code:%x, db:%s", code, user); + CTG_ERR_RET(code); + } + + SRpcMsg rpcMsg = { + .msgType = TDMT_MND_GET_USER_AUTH, + .pCont = msg, + .contLen = msgLen, + }; + + SRpcMsg rpcRsp = {0}; + + rpcSendRecv(pRpc, (SEpSet*)pMgmtEps, &rpcMsg, &rpcRsp); + if (TSDB_CODE_SUCCESS != rpcRsp.code) { + ctgError("error rsp for get user auth, error:%s, user:%s", tstrerror(rpcRsp.code), user); + CTG_ERR_RET(rpcRsp.code); + } + + code = queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_GET_USER_AUTH)](authRsp, rpcRsp.pCont, rpcRsp.contLen); + if (code) { + ctgError("Process get user auth rsp failed, code:%x, user:%s", code, user); + CTG_ERR_RET(code); + } + + ctgDebug("Got user auth from mnode, user:%s", user); + + return TSDB_CODE_SUCCESS; +} + int32_t ctgIsTableMetaExistInCache(SCatalog* pCtg, char *dbFName, char* tbName, int32_t *exist) { if (NULL == pCtg->dbCache) { @@ -859,6 +927,55 @@ int32_t ctgGetTableTypeFromCache(SCatalog* pCtg, const char* dbFName, const char return TSDB_CODE_SUCCESS; } +int32_t ctgChkAuthFromCache(SCatalog* pCtg, const char* user, const char* dbFName, AUTH_TYPE type, bool *inCache, bool *pass) { + if (NULL == pCtg->userCache) { + ctgDebug("empty user auth cache, user:%s", user); + goto _return; + } + + SCtgUserAuth *pUser = (SCtgUserAuth *)taosHashGet(pCtg->userCache, user, strlen(user)); + if (NULL == pUser) { + ctgDebug("user not in cache, user:%s", user); + goto _return; + } + + *inCache = true; + + ctgDebug("Got user from cache, user:%s", user); + CTG_CACHE_STAT_ADD(userHitNum, 1); + + if (pUser->superUser) { + *pass = true; + return TSDB_CODE_SUCCESS; + } + + CTG_LOCK(CTG_READ, &pUser->lock); + if (pUser->createdDbs && taosHashGet(pUser->createdDbs, dbFName, strlen(dbFName))) { + *pass = true; + CTG_UNLOCK(CTG_READ, &pUser->lock); + return TSDB_CODE_SUCCESS; + } + + if (pUser->readDbs && taosHashGet(pUser->readDbs, dbFName, strlen(dbFName)) && type == AUTH_TYPE_READ) { + *pass = true; + } + + if (pUser->writeDbs && taosHashGet(pUser->writeDbs, dbFName, strlen(dbFName)) && type == AUTH_TYPE_WRITE) { + *pass = true; + } + + CTG_UNLOCK(CTG_READ, &pUser->lock); + + return TSDB_CODE_SUCCESS; + +_return: + + *inCache = false; + CTG_CACHE_STAT_ADD(userMissNum, 1); + + return TSDB_CODE_SUCCESS; +} + int32_t ctgGetTableMetaFromMnodeImpl(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, char *dbFName, char* tbName, STableMetaOutput* output) { SBuildTableMetaInput bInput = {.vgId = 0, .dbFName = dbFName, .tbName = tbName}; char *msg = NULL; @@ -1952,6 +2069,45 @@ _return: CTG_RET(code); } +int32_t ctgChkAuth(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* user, const char* dbFName, AUTH_TYPE type, bool *pass) { + bool inCache = false; + int32_t code = 0; + + *pass = false; + + CTG_ERR_RET(ctgChkAuthFromCache(pCtg, user, dbFName, type, &inCache, pass)); + + if (inCache) { + return TSDB_CODE_SUCCESS; + } + + SGetUserAuthRsp authRsp = {0}; + CTG_ERR_RET(ctgGetUserDbAuthFromMnode(pCtg, pRpc, pMgmtEps, user, &authRsp)); + + if (authRsp.superAuth) { + *pass = true; + goto _return; + } + + if (authRsp.createdDbs && taosHashGet(authRsp.createdDbs, dbFName, strlen(dbFName))) { + *pass = true; + goto _return; + } + + if (authRsp.readDbs && taosHashGet(authRsp.readDbs, dbFName, strlen(dbFName)) && type == AUTH_TYPE_READ) { + *pass = true; + } + + if (authRsp.writeDbs && taosHashGet(authRsp.writeDbs, dbFName, strlen(dbFName)) && type == AUTH_TYPE_WRITE) { + *pass = true; + } + +_return: + + ctgPushUpdateUserMsgInQueue(pCtg, &authRsp, false); + + return TSDB_CODE_SUCCESS; +} int32_t ctgActUpdateVg(SCtgMetaAction *action) { @@ -2121,6 +2277,67 @@ _return: CTG_RET(code); } +int32_t ctgActUpdateUser(SCtgMetaAction *action) { + int32_t code = 0; + SCtgUpdateUserMsg *msg = action->data; + SCatalog* pCtg = msg->pCtg; + + if (NULL == pCtg->userCache) { + pCtg->userCache = taosHashInit(gCtgMgmt.cfg.maxUserCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); + if (NULL == pCtg->userCache) { + ctgError("taosHashInit %d user cache failed", gCtgMgmt.cfg.maxUserCacheNum); + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } + } + + SCtgUserAuth *pUser = (SCtgUserAuth *)taosHashGet(pCtg->userCache, msg->userAuth.user, strlen(msg->userAuth.user)); + if (NULL == pUser) { + SCtgUserAuth userAuth = {0}; + + userAuth.version = msg->userAuth.version; + userAuth.superUser = msg->userAuth.superAuth; + userAuth.createdDbs = msg->userAuth.createdDbs; + userAuth.readDbs = msg->userAuth.readDbs; + userAuth.writeDbs = msg->userAuth.writeDbs; + + if (taosHashPut(pCtg->userCache, msg->userAuth.user, strlen(msg->userAuth.user), &userAuth, sizeof(userAuth))) { + ctgError("taosHashPut user %s to cache failed", msg->userAuth.user); + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } + + return TSDB_CODE_SUCCESS; + } + + pUser->version = msg->userAuth.version; + + CTG_LOCK(CTG_WRITE, &pUser->lock); + + taosHashCleanup(pUser->createdDbs); + pUser->createdDbs = msg->userAuth.createdDbs; + msg->userAuth.createdDbs = NULL; + + taosHashCleanup(pUser->readDbs); + pUser->readDbs = msg->userAuth.readDbs; + msg->userAuth.readDbs = NULL; + + taosHashCleanup(pUser->writeDbs); + pUser->writeDbs = msg->userAuth.writeDbs; + msg->userAuth.writeDbs = NULL; + + CTG_UNLOCK(CTG_WRITE, &pUser->lock); + +_return: + + + taosHashCleanup(msg->userAuth.createdDbs); + taosHashCleanup(msg->userAuth.readDbs); + taosHashCleanup(msg->userAuth.writeDbs); + + taosMemoryFreeClear(msg); + + CTG_RET(code); +} + void* ctgUpdateThreadFunc(void* param) { setThreadName("catalog"); @@ -2836,6 +3053,35 @@ int32_t catalogGetExpiredDBs(SCatalog* pCtg, SDbVgVersion **dbs, uint32_t *num) CTG_API_LEAVE(ctgMetaRentGet(&pCtg->dbRent, (void **)dbs, num, sizeof(SDbVgVersion))); } +int32_t catalogGetExpiredUsers(SCatalog* pCtg, SUserAuthVersion **users, uint32_t *num) { + CTG_API_ENTER(); + + if (NULL == pCtg || NULL == users || NULL == num) { + CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); + } + + *num = taosHashGetSize(pCtg->userCache); + if (*num > 0) { + *users = taosMemoryCalloc(*num, sizeof(SUserAuthVersion)); + if (NULL == *users) { + ctgError("calloc %d userAuthVersion failed", *num); + CTG_API_LEAVE(TSDB_CODE_OUT_OF_MEMORY); + } + } + + uint32_t i = 0; + SCtgUserAuth *pAuth = taosHashIterate(pCtg->userCache, NULL); + while (pAuth != NULL) { + void *key = taosHashGetKey(pAuth, NULL); + strncpy((*users)[i].user, key, sizeof((*users)[i].user)); + (*users)[i].version = pAuth->version; + pAuth = taosHashIterate(pCtg->userCache, pAuth); + } + + CTG_API_LEAVE(TSDB_CODE_SUCCESS); +} + + int32_t catalogGetDBCfg(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* dbFName, SDbCfgInfo* pDbCfg) { CTG_API_ENTER(); @@ -2880,6 +3126,31 @@ _return: CTG_API_LEAVE(code); } +int32_t catalogChkAuth(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* user, const char* dbFName, AUTH_TYPE type, bool *pass) { + CTG_API_ENTER(); + + if (NULL == pCtg || NULL == pRpc || NULL == pMgmtEps || NULL == user || NULL == dbFName || NULL == pass) { + CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); + } + + int32_t code = 0; + CTG_ERR_JRET(ctgChkAuth(pCtg, pRpc, pMgmtEps, user, dbFName, type, pass)); + +_return: + + CTG_API_LEAVE(code); +} + +int32_t catalogUpdateUserAuthInfo(SCatalog* pCtg, SGetUserAuthRsp* pAuth) { + CTG_API_ENTER(); + + if (NULL == pCtg || NULL == pAuth) { + CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); + } + + CTG_API_LEAVE(ctgPushUpdateUserMsgInQueue(pCtg, pAuth, false)); +} + void catalogDestroy(void) { qInfo("start to destroy catalog"); diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index bc6139c304..3b5d0c209f 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -87,9 +87,7 @@ typedef struct SResultInfo { // TODO refactor typedef struct STableQueryInfo { TSKEY lastKey; // last check ts, todo remove it later SResultRowPosition pos; // current active time window -// int32_t groupIndex; // group id in table list // SVariant tag; -// SResultRowInfo resInfo; // result info } STableQueryInfo; typedef enum { @@ -363,11 +361,12 @@ typedef struct STableScanInfo { } STableScanInfo; typedef struct STagScanInfo { - SColumnInfo *pCols; - SSDataBlock *pRes; - int32_t totalTables; - int32_t curPos; - void *pReader; + SColumnInfo *pCols; + SSDataBlock *pRes; + SArray *pColMatchInfo; + int32_t curPos; + SReadHandle readHandle; + STableGroupInfo *pTableGroups; } STagScanInfo; typedef struct SStreamBlockScanInfo { @@ -579,9 +578,8 @@ typedef struct SSortOperatorInfo { uint32_t sortBufSize; // max buffer size for in-memory sort SArray* pSortInfo; SSortHandle* pSortHandle; - SArray* inputSlotMap; // for index map from table scan output + SArray* pColMatchInfo; // for index map from table scan output int32_t bufPageSize; -// int32_t numOfRowsInRes; // TODO extact struct int64_t startTs; // sort start time @@ -646,7 +644,7 @@ void cleanupAggSup(SAggSupporter* pAggSup); void destroyBasicOperatorInfo(void* param, int32_t numOfOutput); void appendOneRowToDataBlock(SSDataBlock* pBlock, STupleHandle* pTupleHandle); -SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, int32_t capacity); +SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, int32_t capacity, SArray* pColMatchInfo); SSDataBlock* loadNextDataBlock(void* param); void setResultRowInitCtx(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowCellInfoOffset); @@ -704,7 +702,7 @@ SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SExprInfo* SSDataBlock* pResultBlock, SExecTaskInfo* pTaskInfo); SOperatorInfo* createJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t numOfDownstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SNode* pOnCondition, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createTagScanOperatorInfo(void* pReaderHandle, SExprInfo* pExpr, int32_t numOfOutput, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, SExprInfo* pExpr, int32_t numOfOutput, SSDataBlock* pResBlock, SArray* pColMatchInfo, STableGroupInfo* pTableGroupInfo, SExecTaskInfo* pTaskInfo); #if 0 SOperatorInfo* createTableSeqScanOperatorInfo(void* pTsdbReadHandle, STaskRuntimeEnv* pRuntimeEnv); @@ -717,7 +715,6 @@ int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBloc void setInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order, bool createDummyCol); -void finalizeQueryResult(SqlFunctionCtx* pCtx, int32_t numOfOutput); void copyTsColoum(SSDataBlock* pRes, SqlFunctionCtx* pCtx, int32_t numOfOutput); STableQueryInfo* createTableQueryInfo(void* buf, STimeWindow win); diff --git a/source/libs/executor/inc/tsort.h b/source/libs/executor/inc/tsort.h index 2072707b30..d74628a72f 100644 --- a/source/libs/executor/inc/tsort.h +++ b/source/libs/executor/inc/tsort.h @@ -117,18 +117,25 @@ STupleHandle* tsortNextTuple(SSortHandle* pHandle); /** * * @param pHandle - * @param colIndex + * @param colId * @return */ -bool tsortIsNullVal(STupleHandle* pVHandle, int32_t colIndex); +bool tsortIsNullVal(STupleHandle* pVHandle, int32_t colId); /** * * @param pHandle - * @param colIndex + * @param colId * @return */ -void* tsortGetValue(STupleHandle* pVHandle, int32_t colIndex); +void* tsortGetValue(STupleHandle* pVHandle, int32_t colId); + +/** + * + * @param pSortHandle + * @return + */ +SSDataBlock* tsortGetSortedDataBlock(const SSortHandle* pSortHandle); #ifdef __cplusplus } diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index a237eb0e7d..943d4b2783 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -3520,7 +3520,7 @@ static SSDataBlock* doSortedMerge(SOperatorInfo* pOperator) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SSortedMergeOperatorInfo* pInfo = pOperator->info; if (pOperator->status == OP_RES_TO_RETURN) { - return getSortedBlockData(pInfo->pSortHandle, pInfo->binfo.pRes, pOperator->resultInfo.capacity); + return getSortedBlockData(pInfo->pSortHandle, pInfo->binfo.pRes, pOperator->resultInfo.capacity, NULL); } int32_t numOfBufPage = pInfo->sortBufSize / pInfo->bufPageSize; @@ -4701,7 +4701,7 @@ static SArray* extractTableIdList(const STableGroupInfo* pTableGroupInfo); static SArray* extractColumnInfo(SNodeList* pNodeList); static SArray* extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNodeList, int32_t* numOfOutputCols); -static SArray* createSortInfo(SNodeList* pNodeList, SNodeList* pNodeListTarget); +static SArray* createSortInfo(SNodeList* pNodeList); static SArray* createIndexMap(SNodeList* pNodeList); static SArray* extractPartitionColInfo(SNodeList* pNodeList); static int32_t initQueryTableDataCond(SQueryTableDataCond* pCond, const STableScanPhysiNode* pTableScanNode); @@ -4739,7 +4739,6 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo SSDataBlock* pResBlock = createResDataBlock(pScanPhyNode->node.pOutputDataBlockDesc); SQueryTableDataCond cond = {0}; - int32_t code = initQueryTableDataCond(&cond, pTableScanNode); if (code != TSDB_CODE_SUCCESS) { return NULL; @@ -4783,6 +4782,25 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo pHandle, pResBlock, &pScanNode->tableName, pScanNode->node.pConditions, pSysScanPhyNode->mgmtEpSet, colList, pTaskInfo, pSysScanPhyNode->showRewrite, pSysScanPhyNode->accountId); return pOperator; + } else if (QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN == type) { + STagScanPhysiNode* pScanPhyNode = (STagScanPhysiNode*) pPhyNode; + SSDataBlock* pResBlock = createResDataBlock(pScanPhyNode->node.pOutputDataBlockDesc); + + int32_t code = + doCreateTableGroup(pHandle->meta, pScanPhyNode->tableType, pScanPhyNode->uid, pTableGroupInfo, queryId, taskId); + if (code != TSDB_CODE_SUCCESS) { + return NULL; + } + + int32_t num = 0; + SExprInfo* pExprInfo = createExprInfo(pScanPhyNode->pScanPseudoCols, NULL, &num); + + int32_t numOfOutputCols = 0; + SArray* colList = + extractColMatchInfo(pScanPhyNode->pScanPseudoCols, pScanPhyNode->node.pOutputDataBlockDesc, &numOfOutputCols); + + SOperatorInfo* pOperator = createTagScanOperatorInfo(pHandle, pExprInfo, num, pResBlock, colList, pTableGroupInfo, pTaskInfo); + return pOperator; } else { ASSERT(0); } @@ -4852,16 +4870,16 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo SSortPhysiNode* pSortPhyNode = (SSortPhysiNode*)pPhyNode; SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc); - SArray* info = createSortInfo(pSortPhyNode->pSortKeys, pSortPhyNode->pTargets); - SArray* slotMap = createIndexMap(pSortPhyNode->pTargets); + SArray* info = createSortInfo(pSortPhyNode->pSortKeys); int32_t numOfCols = 0; - SExprInfo* pExprInfo = NULL; - if (pSortPhyNode->pExprs != NULL) { - pExprInfo = createExprInfo(pSortPhyNode->pExprs, NULL, &numOfCols); - } + SExprInfo* pExprInfo = createExprInfo(pSortPhyNode->pExprs, NULL, &numOfCols); - pOptr = createSortOperatorInfo(ops[0], pResBlock, info, pExprInfo, numOfCols, slotMap, pTaskInfo); + int32_t numOfOutputCols = 0; + SArray* pColList = + extractColMatchInfo(pSortPhyNode->pTargets, pSortPhyNode->node.pOutputDataBlockDesc, &numOfOutputCols); + + pOptr = createSortOperatorInfo(ops[0], pResBlock, info, pExprInfo, numOfCols, pColList, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW == type) { SSessionWinodwPhysiNode* pSessionNode = (SSessionWinodwPhysiNode*)pPhyNode; @@ -5019,7 +5037,7 @@ SArray* extractPartitionColInfo(SNodeList* pNodeList) { return pList; } -SArray* createSortInfo(SNodeList* pNodeList, SNodeList* pNodeListTarget) { +SArray* createSortInfo(SNodeList* pNodeList) { size_t numOfCols = LIST_LENGTH(pNodeList); SArray* pList = taosArrayInit(numOfCols, sizeof(SBlockOrderInfo)); if (pList == NULL) { @@ -5034,22 +5052,7 @@ SArray* createSortInfo(SNodeList* pNodeList, SNodeList* pNodeListTarget) { bi.nullFirst = (pSortKey->nullOrder == NULL_ORDER_FIRST); SColumnNode* pColNode = (SColumnNode*)pSortKey->pExpr; - - bool found = false; - for (int32_t j = 0; j < LIST_LENGTH(pNodeListTarget); ++j) { - STargetNode* pTarget = (STargetNode*)nodesListGetNode(pNodeListTarget, j); - - SColumnNode* pColNodeT = (SColumnNode*)pTarget->pExpr; - if (pColNode->slotId == pColNodeT->slotId) { // to find slotId in PhysiSort OutputDataBlockDesc - bi.slotId = pTarget->slotId; - found = true; - break; - } - } - - if (!found) { - qError("sort slot id does not found"); - } + bi.slotId = pColNode->slotId; taosArrayPush(pList, &bi); } @@ -5088,7 +5091,7 @@ SArray* extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNod SColMatchInfo c = {0}; c.output = true; - c.colId = pColNode->colId; + c.colId = pColNode->colId; c.targetSlotId = pNode->slotId; taosArrayPush(pList, &c); } @@ -5166,9 +5169,7 @@ tsdbReaderT doCreateDataReader(STableScanPhysiNode* pTableScanNode, SReadHandle* if (code != TSDB_CODE_SUCCESS) { goto _error; } -#if 0 - return tsdbQueryTables(pHandle->reader, &cond, pTableGroupInfo, queryId, taskId); -#endif + return tsdbQueryTables(pHandle->vnode, &cond, pTableGroupInfo, queryId, taskId); _error: diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 8c9fdfe4e6..b728daa3bb 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -13,15 +13,16 @@ * along with this program. If not, see . */ -#include "ttime.h" +#include #include "filter.h" #include "function.h" #include "functionMgt.h" #include "os.h" #include "querynodes.h" +#include "systable.h" #include "tglobal.h" #include "tname.h" -#include "systable.h" +#include "ttime.h" #include "tdatablock.h" #include "tmsg.h" @@ -1159,16 +1160,17 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSDataBlock* pRe } static SSDataBlock* doTagScan(SOperatorInfo* pOperator) { -#if 0 if (pOperator->status == OP_EXEC_DONE) { return NULL; } + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + +#if 0 int32_t maxNumOfTables = (int32_t)pResultInfo->capacity; STagScanInfo *pInfo = pOperator->info; SSDataBlock *pRes = pInfo->pRes; - *newgroup = false; int32_t count = 0; SArray* pa = GET_TABLEGROUP(pRuntimeEnv, 0); @@ -1237,55 +1239,54 @@ static SSDataBlock* doTagScan(SOperatorInfo* pOperator) { pOperator->status = OP_EXEC_DONE; //qDebug("QInfo:0x%"PRIx64" create count(tbname) query, res:%d rows:1", GET_TASKID(pRuntimeEnv), count); } else { // return only the tags|table name etc. - SExprInfo* pExprInfo = &pOperator->pExpr[0]; // todo use the column list instead of exprinfo +#endif - count = 0; - while(pInfo->curPos < pInfo->totalTables && count < maxNumOfTables) { - int32_t i = pInfo->curPos++; + STagScanInfo* pInfo = pOperator->info; + SExprInfo* pExprInfo = &pOperator->pExpr[0]; + SSDataBlock* pRes = pInfo->pRes; - STableQueryInfo* item = taosArrayGetP(pa, i); + SArray* pa = taosArrayGetP(pInfo->pTableGroups->pGroupList, 0); - char *data = NULL, *dst = NULL; - int16_t type = 0, bytes = 0; - for(int32_t j = 0; j < pOperator->numOfExprs; ++j) { - // not assign value in case of user defined constant output column - if (TSDB_COL_IS_UD_COL(pExprInfo[j].base.pColumns->flag)) { - continue; - } + char str[512] = {0}; + int32_t count = 0; + SMetaReader mr = {0}; - SColumnInfoData* pColInfo = taosArrayGet(pRes->pDataBlock, j); - type = pExprInfo[j].base.resSchema.type; - bytes = pExprInfo[j].base.resSchema.bytes; + while (pInfo->curPos < pInfo->pTableGroups->numOfTables && count < pOperator->resultInfo.capacity) { + STableKeyInfo* item = taosArrayGet(pa, pInfo->curPos); - if (pExprInfo[j].base.pColumns->info.colId == TSDB_TBNAME_COLUMN_INDEX) { - data = tsdbGetTableName(item->pTable); - } else { - data = tsdbGetTableTagVal(item->pTable, pExprInfo[j].base.pColumns->info.colId, type, bytes); - } + for (int32_t j = 0; j < pOperator->numOfExprs; ++j) { + SColumnInfoData* pDst = taosArrayGet(pRes->pDataBlock, pExprInfo[j].base.resSchema.slotId); - dst = pColInfo->pData + count * pExprInfo[j].base.resSchema.bytes; - doSetTagValueToResultBuf(dst, data, type, bytes); + // refactor later + if (fmIsScanPseudoColumnFunc(pExprInfo[j].pExpr->_function.functionId)) { + metaReaderInit(&mr, pInfo->readHandle.meta, 0); + metaGetTableEntryByUid(&mr, item->uid); + + STR_TO_VARSTR(str, mr.me.name); + metaReaderClear(&mr); + + colDataAppend(pDst, count, str, false); + + // data = tsdbGetTableTagVal(item->pTable, pExprInfo[j].base.pColumns->info.colId, type, bytes); + // dst = pColInfo->pData + count * pExprInfo[j].base.resSchema.bytes; + // doSetTagValueToResultBuf(dst, data, type, bytes); } count += 1; } - if (pInfo->curPos >= pInfo->totalTables) { + if (++pInfo->curPos >= pInfo->pTableGroups->numOfTables) { pOperator->status = OP_EXEC_DONE; } - - //qDebug("QInfo:0x%"PRIx64" create tag values results completed, rows:%d", GET_TASKID(pRuntimeEnv), count); } + // qDebug("QInfo:0x%"PRIx64" create tag values results completed, rows:%d", GET_TASKID(pRuntimeEnv), count); if (pOperator->status == OP_EXEC_DONE) { - setTaskStatus(pOperator->pRuntimeEnv, TASK_COMPLETED); + setTaskStatus(pTaskInfo, TASK_COMPLETED); } pRes->info.rows = count; - return (pRes->info.rows == 0)? NULL:pInfo->pRes; - -#endif - return TSDB_CODE_SUCCESS; + return (pRes->info.rows == 0) ? NULL : pInfo->pRes; } static void destroyTagScanOperatorInfo(void* param, int32_t numOfOutput) { @@ -1293,14 +1294,18 @@ static void destroyTagScanOperatorInfo(void* param, int32_t numOfOutput) { pInfo->pRes = blockDataDestroy(pInfo->pRes); } -SOperatorInfo* createTagScanOperatorInfo(void* readHandle, SExprInfo* pExpr, int32_t numOfOutput, SExecTaskInfo* pTaskInfo) { +SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, SExprInfo* pExpr, int32_t numOfOutput, + SSDataBlock* pResBlock, SArray* pColMatchInfo, STableGroupInfo* pTableGroupInfo, SExecTaskInfo* pTaskInfo) { STagScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STagScanInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { goto _error; } - pInfo->pReader = readHandle; + pInfo->pTableGroups = pTableGroupInfo; + pInfo->pColMatchInfo = pColMatchInfo; + pInfo->pRes = pResBlock; + pInfo->readHandle = *pReadHandle; pInfo->curPos = 0; pOperator->name = "TagScanOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN; @@ -1308,9 +1313,12 @@ SOperatorInfo* createTagScanOperatorInfo(void* readHandle, SExprInfo* pExpr, int pOperator->status = OP_NOT_OPENED; pOperator->info = pInfo; pOperator->pExpr = pExpr; - pOperator->numOfExprs = numOfOutput; + pOperator->numOfExprs = numOfOutput; pOperator->pTaskInfo = pTaskInfo; + initResultSizeInfo(pOperator, 4096); + blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity); + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doTagScan, NULL, NULL, destroyTagScanOperatorInfo, NULL, NULL, NULL); diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c index 0f973b0cf0..619651f11f 100644 --- a/source/libs/executor/src/sortoperator.c +++ b/source/libs/executor/src/sortoperator.c @@ -5,7 +5,7 @@ static SSDataBlock* doSort(SOperatorInfo* pOperator); static void destroyOrderOperatorInfo(void* param, int32_t numOfOutput); SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSDataBlock* pResBlock, SArray* pSortInfo, SExprInfo* pExprInfo, int32_t numOfCols, - SArray* pIndexMap, SExecTaskInfo* pTaskInfo) { + SArray* pColMatchColInfo, SExecTaskInfo* pTaskInfo) { SSortOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SSortOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); int32_t rowSize = pResBlock->info.rowSize; @@ -20,17 +20,19 @@ SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSDataBlock* pR pInfo->binfo.pRes = pResBlock; initResultSizeInfo(pOperator, 1024); - pInfo->bufPageSize = rowSize < 1024 ? 1024 * 2 : rowSize * 2; // there are headers, so pageSize = rowSize + header - pInfo->sortBufSize = pInfo->bufPageSize * 16; // TODO dynamic set the available sort buffer pInfo->pSortInfo = pSortInfo; - pInfo->inputSlotMap = pIndexMap; + pInfo->pColMatchInfo= pColMatchColInfo; pOperator->name = "SortOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_SORT; pOperator->blocking = true; pOperator->status = OP_NOT_OPENED; pOperator->info = pInfo; + // lazy evaluation for the following parameter since the input datablock is not known till now. +// pInfo->bufPageSize = rowSize < 1024 ? 1024 * 2 : rowSize * 2; // there are headers, so pageSize = rowSize + header +// pInfo->sortBufSize = pInfo->bufPageSize * 16; // TODO dynamic set the available sort buffer + pOperator->pTaskInfo = pTaskInfo; pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doSort, NULL, NULL, destroyOrderOperatorInfo, NULL, NULL, NULL); @@ -45,14 +47,12 @@ SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSDataBlock* pR return NULL; } -// TODO merge aggregate super table void appendOneRowToDataBlock(SSDataBlock* pBlock, STupleHandle* pTupleHandle) { for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i); - bool isNull = tsortIsNullVal(pTupleHandle, i); if (isNull) { - colDataAppend(pColInfo, pBlock->info.rows, NULL, true); + colDataAppendNULL(pColInfo, pBlock->info.rows); } else { char* pData = tsortGetValue(pTupleHandle, i); colDataAppend(pColInfo, pBlock->info.rows, pData, false); @@ -62,11 +62,12 @@ void appendOneRowToDataBlock(SSDataBlock* pBlock, STupleHandle* pTupleHandle) { pBlock->info.rows += 1; } -SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, int32_t capacity) { +SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, int32_t capacity, SArray* pColMatchInfo) { blockDataCleanup(pDataBlock); - blockDataEnsureCapacity(pDataBlock, capacity); + ASSERT(taosArrayGetSize(pColMatchInfo) == pDataBlock->info.numOfCols); - blockDataEnsureCapacity(pDataBlock, capacity); + SSDataBlock* p = tsortGetSortedDataBlock(pHandle); + blockDataEnsureCapacity(p, capacity); while (1) { STupleHandle* pTupleHandle = tsortNextTuple(pHandle); @@ -74,12 +75,32 @@ SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, i break; } - appendOneRowToDataBlock(pDataBlock, pTupleHandle); - if (pDataBlock->info.rows >= capacity) { + appendOneRowToDataBlock(p, pTupleHandle); + if (p->info.rows >= capacity) { return pDataBlock; } } + if (p->info.rows > 0) { + int32_t numOfCols = taosArrayGetSize(pColMatchInfo); + for(int32_t i = 0; i < numOfCols; ++i) { + SColMatchInfo* pmInfo = taosArrayGet(pColMatchInfo, i); + + for(int32_t j = 0; j < p->info.numOfCols; ++j) { + SColumnInfoData* pSrc = taosArrayGet(p->pDataBlock, j); + if (pSrc->info.colId == pmInfo->colId) { + SColumnInfoData* pDst = taosArrayGet(pDataBlock->pDataBlock, pmInfo->targetSlotId); + colDataAssign(pDst, pSrc, p->info.rows); + break; + } + } + } + + pDataBlock->info.rows = p->info.rows; + pDataBlock->info.capacity = p->info.rows; + } + + blockDataDestroy(p); return (pDataBlock->info.rows > 0) ? pDataBlock : NULL; } @@ -106,16 +127,16 @@ SSDataBlock* doSort(SOperatorInfo* pOperator) { SSortOperatorInfo* pInfo = pOperator->info; if (pOperator->status == OP_RES_TO_RETURN) { - return getSortedBlockData(pInfo->pSortHandle, pInfo->binfo.pRes, pOperator->resultInfo.capacity); + return getSortedBlockData(pInfo->pSortHandle, pInfo->binfo.pRes, pOperator->resultInfo.capacity, pInfo->pColMatchInfo); } - int32_t numOfBufPage = pInfo->sortBufSize / pInfo->bufPageSize; - pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, pInfo->inputSlotMap, SORT_SINGLESOURCE_SORT, - pInfo->bufPageSize, numOfBufPage, pInfo->binfo.pRes, pTaskInfo->id.str); +// pInfo->binfo.pRes is not equalled to the input datablock. +// int32_t numOfBufPage = pInfo->sortBufSize / pInfo->bufPageSize; + pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, pInfo->pColMatchInfo, SORT_SINGLESOURCE_SORT, + -1, -1, NULL, pTaskInfo->id.str); tsortSetFetchRawDataFp(pInfo->pSortHandle, loadNextDataBlock, applyScalarFunction, pOperator); - SSortSource* ps = taosMemoryCalloc(1, sizeof(SSortSource)); ps->param = pOperator->pDownstream[0]; tsortAddSource(pInfo->pSortHandle, ps); @@ -127,7 +148,7 @@ SSDataBlock* doSort(SOperatorInfo* pOperator) { } pOperator->status = OP_RES_TO_RETURN; - return getSortedBlockData(pInfo->pSortHandle, pInfo->binfo.pRes, pOperator->resultInfo.capacity); + return getSortedBlockData(pInfo->pSortHandle, pInfo->binfo.pRes, pOperator->resultInfo.capacity, pInfo->pColMatchInfo); } void destroyOrderOperatorInfo(void* param, int32_t numOfOutput) { @@ -135,5 +156,5 @@ void destroyOrderOperatorInfo(void* param, int32_t numOfOutput) { pInfo->binfo.pRes = blockDataDestroy(pInfo->binfo.pRes); taosArrayDestroy(pInfo->pSortInfo); - taosArrayDestroy(pInfo->inputSlotMap); + taosArrayDestroy(pInfo->pColMatchInfo); } diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c index 50aa4cfc01..040ee8c7f5 100644 --- a/source/libs/executor/src/tsort.c +++ b/source/libs/executor/src/tsort.c @@ -64,25 +64,8 @@ struct SSortHandle { static int32_t msortComparFn(const void *pLeft, const void *pRight, void *param); -static SSDataBlock* createDataBlock_rv(SSchema* pSchema, int32_t numOfCols) { - SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); - pBlock->pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); - pBlock->info.numOfCols = numOfCols; - - for(int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData colInfo = {0}; - - colInfo.info.type = pSchema[i].type; - colInfo.info.bytes = pSchema[i].bytes; - colInfo.info.colId = pSchema[i].colId; - taosArrayPush(pBlock->pDataBlock, &colInfo); - - if (IS_VAR_DATA_TYPE(colInfo.info.type)) { - pBlock->info.hasVarCol = true; - } - } - - return pBlock; +SSDataBlock* tsortGetSortedDataBlock(const SSortHandle* pSortHandle) { + return createOneDataBlock(pSortHandle->pDataBlock, false); } /** @@ -98,7 +81,10 @@ SSortHandle* tsortCreateSortHandle(SArray* pSortInfo, SArray* pIndexMap, int32_t pSortHandle->numOfPages = numOfPages; pSortHandle->pSortInfo = pSortInfo; pSortHandle->pIndexMap = pIndexMap; - pSortHandle->pDataBlock = createOneDataBlock(pBlock, false); + + if (pBlock != NULL) { + pSortHandle->pDataBlock = createOneDataBlock(pBlock, false); + } pSortHandle->pOrderedSource = taosArrayInit(4, POINTER_BYTES); pSortHandle->cmpParam.orderInfo = pSortInfo; @@ -530,6 +516,17 @@ static int32_t createInitialSortedMultiSources(SSortHandle* pHandle) { if (pHandle->pDataBlock == NULL) { pHandle->pDataBlock = createOneDataBlock(pBlock, false); + + // calculate the buffer pages according to the total available buffers. + int32_t rowSize = blockDataGetRowSize(pBlock); + if (rowSize * 4 > 4096) { + pHandle->pageSize = rowSize * 4; + } else { + pHandle->pageSize = 4096; + } + // todo!! + pHandle->numOfPages = 1024; + sortBufSize = pHandle->numOfPages * pHandle->pageSize; } // perform the scalar function calculation before apply the sort @@ -538,7 +535,6 @@ static int32_t createInitialSortedMultiSources(SSortHandle* pHandle) { } // todo relocate the columns - int32_t code = blockDataMerge(pHandle->pDataBlock, pBlock, pHandle->pIndexMap); if (code != 0) { return code; @@ -689,7 +685,7 @@ STupleHandle* tsortNextTuple(SSortHandle* pHandle) { bool tsortIsNullVal(STupleHandle* pVHandle, int32_t colIndex) { SColumnInfoData* pColInfoSrc = taosArrayGet(pVHandle->pBlock->pDataBlock, colIndex); - return colDataIsNull(pColInfoSrc, 0, pVHandle->rowIndex, NULL); + return colDataIsNull_s(pColInfoSrc, pVHandle->rowIndex); } void* tsortGetValue(STupleHandle* pVHandle, int32_t colIndex) { diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 38922833f9..eac11558cb 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -917,7 +917,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .translateFunc = translateTbnameColumn, .getEnvFunc = NULL, .initFunc = NULL, - .sprocessFunc = NULL, + .sprocessFunc = qTbnameFunction, .finalizeFunc = NULL }, { diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index 5b1573c88f..0a99ef61ce 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -594,7 +594,9 @@ int32_t convertScalarParamToDataBlock(SScalarParam *input, int32_t numOfCols, SS //TODO: free the array output->pDataBlock output->pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); - taosArrayPush(output->pDataBlock, input->columnData); + for (int32_t i = 0; i < numOfCols; ++i) { + taosArrayPush(output->pDataBlock, (input + i)->columnData); + } return 0; } diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c index ba9fca2969..f5e4a9c6e6 100644 --- a/source/libs/function/src/udfd.c +++ b/source/libs/function/src/udfd.c @@ -81,6 +81,9 @@ typedef struct SUdf { TUdfAggStartFunc aggStartFunc; TUdfAggProcessFunc aggProcFunc; TUdfAggFinishFunc aggFinishFunc; + + TUdfInitFunc initFunc; + TUdfDestroyFunc destroyFunc; } SUdf; // TODO: low priority: change name onxxx to xxxCb, and udfc or udfd as prefix @@ -101,7 +104,19 @@ int32_t udfdLoadUdf(char *udfName, SUdf *udf) { fnError("can not load library %s. error: %s", udf->path, uv_strerror(err)); return UDFC_CODE_LOAD_UDF_FAILURE; } - //TODO: init and destroy function + + char initFuncName[TSDB_FUNC_NAME_LEN+5] = {0}; + char *initSuffix = "_init"; + strcpy(initFuncName, udfName); + strncat(initFuncName, initSuffix, strlen(initSuffix)); + uv_dlsym(&udf->lib, initFuncName, (void**)(&udf->initFunc)); + + char destroyFuncName[TSDB_FUNC_NAME_LEN+5] = {0}; + char *destroySuffix = "_destroy"; + strcpy(destroyFuncName, udfName); + strncat(destroyFuncName, destroySuffix, strlen(destroySuffix)); + uv_dlsym(&udf->lib, destroyFuncName, (void**)(&udf->destroyFunc)); + if (udf->funcType == TSDB_FUNC_TYPE_SCALAR) { char processFuncName[TSDB_FUNC_NAME_LEN] = {0}; strcpy(processFuncName, udfName); @@ -159,6 +174,9 @@ void udfdProcessRequest(uv_work_t *req) { if (udf->state == UDF_STATE_INIT) { udf->state = UDF_STATE_LOADING; udfdLoadUdf(setup->udfName, udf); + if (udf->initFunc) { + udf->initFunc(); + } udf->state = UDF_STATE_READY; uv_cond_broadcast(&udf->condReady); uv_mutex_unlock(&udf->lock); @@ -170,7 +188,6 @@ void udfdProcessRequest(uv_work_t *req) { } SUdfcFuncHandle *handle = taosMemoryMalloc(sizeof(SUdfcFuncHandle)); handle->udf = udf; - // TODO: allocate private structure and call init function and set it to handle SUdfResponse rsp; rsp.seqNum = request.seqNum; rsp.type = request.type; @@ -275,10 +292,12 @@ void udfdProcessRequest(uv_work_t *req) { if (unloadUdf) { uv_cond_destroy(&udf->condReady); uv_mutex_destroy(&udf->lock); + if (udf->destroyFunc) { + (udf->destroyFunc)(); + } uv_dlclose(&udf->lib); taosMemoryFree(udf); } - // TODO: call destroy and free udf private taosMemoryFree(handle); SUdfResponse response; diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index 152aa6a01e..e82873b923 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -22,7 +22,6 @@ #include "ttime.h" #include "ttypes.h" -// clang-format off #define NEXT_TOKEN(pSql, sToken) \ do { \ int32_t index = 0; \ @@ -248,12 +247,11 @@ static int32_t getTableMetaImpl(SInsertParseContext* pCxt, SToken* pTname, bool } else { CHECK_CODE(catalogGetTableMeta(pBasicCtx->pCatalog, pBasicCtx->pTransporter, &pBasicCtx->mgmtEpSet, &name, &pCxt->pTableMeta)); + SVgroupInfo vg; + CHECK_CODE( + catalogGetTableHashVgroup(pBasicCtx->pCatalog, pBasicCtx->pTransporter, &pBasicCtx->mgmtEpSet, &name, &vg)); + CHECK_CODE(taosHashPut(pCxt->pVgroupsHashObj, (const char*)&vg.vgId, sizeof(vg.vgId), (char*)&vg, sizeof(vg))); } - SVgroupInfo vg; - CHECK_CODE( - catalogGetTableHashVgroup(pBasicCtx->pCatalog, pBasicCtx->pTransporter, &pBasicCtx->mgmtEpSet, &name, &vg)); - CHECK_CODE(taosHashPut(pCxt->pVgroupsHashObj, (const char*)&vg.vgId, sizeof(vg.vgId), (char*)&vg, sizeof(vg))); - return TSDB_CODE_SUCCESS; } @@ -828,12 +826,21 @@ static int32_t cloneTableMeta(STableMeta* pSrc, STableMeta** pDst) { return TSDB_CODE_SUCCESS; } -static int32_t storeTableMeta(SHashObj* pHash, const char* pName, int32_t len, STableMeta* pMeta) { +static int32_t storeTableMeta(SInsertParseContext* pCxt, SHashObj* pHash, SName* pTableName, const char* pName, + int32_t len, STableMeta* pMeta) { + SVgroupInfo vg; + SParseContext* pBasicCtx = pCxt->pComCxt; + CHECK_CODE( + catalogGetTableHashVgroup(pBasicCtx->pCatalog, pBasicCtx->pTransporter, &pBasicCtx->mgmtEpSet, pTableName, &vg)); + CHECK_CODE(taosHashPut(pCxt->pVgroupsHashObj, (const char*)&vg.vgId, sizeof(vg.vgId), (char*)&vg, sizeof(vg))); + + pMeta->uid = tGenIdPI64(); + pMeta->vgId = vg.vgId; + STableMeta* pBackup = NULL; if (TSDB_CODE_SUCCESS != cloneTableMeta(pMeta, &pBackup)) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } - pBackup->uid = tGenIdPI64(); return taosHashPut(pHash, pName, len, &pBackup, POINTER_BYTES); } @@ -856,7 +863,7 @@ static int32_t parseUsingClause(SInsertParseContext* pCxt, SToken* pTbnameToken) if (TSDB_SUPER_TABLE != pCxt->pTableMeta->tableType) { return buildInvalidOperationMsg(&pCxt->msg, "create table only from super table is allowed"); } - CHECK_CODE(storeTableMeta(pCxt->pSubTableHashObj, tbFName, len, pCxt->pTableMeta)); + CHECK_CODE(storeTableMeta(pCxt, pCxt->pSubTableHashObj, &name, tbFName, len, pCxt->pTableMeta)); SSchema* pTagsSchema = getTableTagSchema(pCxt->pTableMeta); setBoundColumnInfo(&pCxt->tags, pTagsSchema, getNumOfTags(pCxt->pTableMeta)); @@ -1262,9 +1269,10 @@ int32_t qBuildStmtOutput(SQuery* pQuery, SHashObj* pVgHash, SHashObj* pBlockHash return TSDB_CODE_SUCCESS; } -int32_t qBindStmtTagsValue(void *pBlock, void *boundTags, int64_t suid, SName *pName, TAOS_MULTI_BIND *bind, char *msgBuf, int32_t msgBufLen){ - STableDataBlocks *pDataBlock = (STableDataBlocks *)pBlock; - SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; +int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, SName* pName, TAOS_MULTI_BIND* bind, + char* msgBuf, int32_t msgBufLen) { + STableDataBlocks* pDataBlock = (STableDataBlocks*)pBlock; + SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; SParsedDataColInfo* tags = (SParsedDataColInfo*)boundTags; if (NULL == tags) { return TSDB_CODE_QRY_APP_ERROR; @@ -1312,11 +1320,10 @@ int32_t qBindStmtTagsValue(void *pBlock, void *boundTags, int64_t suid, SName *p return TSDB_CODE_SUCCESS; } - -int32_t qBindStmtColsValue(void *pBlock, TAOS_MULTI_BIND *bind, char *msgBuf, int32_t msgBufLen) { - STableDataBlocks *pDataBlock = (STableDataBlocks *)pBlock; - SSchema* pSchema = getTableColumnSchema(pDataBlock->pTableMeta); - int32_t extendedRowSize = getExtendedRowSize(pDataBlock); +int32_t qBindStmtColsValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen) { + STableDataBlocks* pDataBlock = (STableDataBlocks*)pBlock; + SSchema* pSchema = getTableColumnSchema(pDataBlock->pTableMeta); + int32_t extendedRowSize = getExtendedRowSize(pDataBlock); SParsedDataColInfo* spd = &pDataBlock->boundColumnInfo; SRowBuilder* pBuilder = &pDataBlock->rowBuilder; SMemParam param = {.rb = pBuilder}; @@ -1391,10 +1398,11 @@ int32_t qBindStmtColsValue(void *pBlock, TAOS_MULTI_BIND *bind, char *msgBuf, in return TSDB_CODE_SUCCESS; } -int32_t qBindStmtSingleColValue(void *pBlock, TAOS_MULTI_BIND *bind, char *msgBuf, int32_t msgBufLen, int32_t colIdx, int32_t rowNum) { - STableDataBlocks *pDataBlock = (STableDataBlocks *)pBlock; - SSchema* pSchema = getTableColumnSchema(pDataBlock->pTableMeta); - int32_t extendedRowSize = getExtendedRowSize(pDataBlock); +int32_t qBindStmtSingleColValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen, int32_t colIdx, + int32_t rowNum) { + STableDataBlocks* pDataBlock = (STableDataBlocks*)pBlock; + SSchema* pSchema = getTableColumnSchema(pDataBlock->pTableMeta); + int32_t extendedRowSize = getExtendedRowSize(pDataBlock); SParsedDataColInfo* spd = &pDataBlock->boundColumnInfo; SRowBuilder* pBuilder = &pDataBlock->rowBuilder; SMemParam param = {.rb = pBuilder}; @@ -1459,7 +1467,7 @@ int32_t qBindStmtSingleColValue(void *pBlock, TAOS_MULTI_BIND *bind, char *msgBu } #ifdef TD_DEBUG_PRINT_ROW - if(rowEnd) { + if (rowEnd) { STSchema* pSTSchema = tdGetSTSChemaFromSSChema(&pSchema, spd->numOfCols); tdSRowPrint(row, pSTSchema, __func__); taosMemoryFree(pSTSchema); diff --git a/source/libs/parser/src/parInsertData.c b/source/libs/parser/src/parInsertData.c index ae05d2293e..4a96301d95 100644 --- a/source/libs/parser/src/parInsertData.c +++ b/source/libs/parser/src/parInsertData.c @@ -498,14 +498,9 @@ int32_t mergeTableDataBlocks(SHashObj* pHashObj, uint8_t payloadType, SArray** p ASSERT(blkKeyInfo.pKeyTuple != NULL && pBlocks->numOfRows > 0); } - int32_t len = pBlocks->numOfRows * - (isRawPayload ? (pOneTableBlock->rowSize + expandSize) : getExtendedRowSize(pOneTableBlock)) + - sizeof(STColumn) * getNumOfColumns(pOneTableBlock->pTableMeta); - // erase the empty space reserved for binary data int32_t finalLen = trimDataBlock(dataBuf->pData + dataBuf->size, pOneTableBlock, blkKeyInfo.pKeyTuple, isRawPayload); - assert(finalLen <= len); dataBuf->size += (finalLen + sizeof(SSubmitBlk)); assert(dataBuf->size <= dataBuf->nAllocSize); diff --git a/source/libs/qcom/src/querymsg.c b/source/libs/qcom/src/querymsg.c index 286e7d3d44..822c214fe5 100644 --- a/source/libs/qcom/src/querymsg.c +++ b/source/libs/qcom/src/querymsg.c @@ -181,6 +181,25 @@ int32_t queryBuildRetrieveFuncMsg(void *input, char **msg, int32_t msgSize, int3 return TSDB_CODE_SUCCESS; } +int32_t queryBuildGetUserAuthMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen) { + if (NULL == msg || NULL == msgLen) { + return TSDB_CODE_TSC_INVALID_INPUT; + } + + SGetUserAuthReq req = {0}; + strncpy(req.user, input, sizeof(req.user)); + + int32_t bufLen = tSerializeSGetUserAuthReq(NULL, 0, &req); + void *pBuf = rpcMallocCont(bufLen); + tSerializeSGetUserAuthReq(pBuf, bufLen, &req); + + *msg = pBuf; + *msgLen = bufLen; + + return TSDB_CODE_SUCCESS; +} + + int32_t queryProcessUseDBRsp(void *output, char *msg, int32_t msgSize) { SUseDbOutput *pOut = output; SUseDbRsp usedbRsp = {0}; @@ -419,6 +438,20 @@ int32_t queryProcessRetrieveFuncRsp(void *output, char *msg, int32_t msgSize) { return TSDB_CODE_SUCCESS; } +int32_t queryProcessGetUserAuthRsp(void *output, char *msg, int32_t msgSize) { + if (NULL == output || NULL == msg || msgSize <= 0) { + return TSDB_CODE_TSC_INVALID_INPUT; + } + + if (tDeserializeSGetUserAuthRsp(msg, msgSize, (SGetUserAuthRsp *)output) != 0) { + qError("tDeserializeSGetUserAuthRsp failed, msgSize:%d", msgSize); + return TSDB_CODE_INVALID_MSG; + } + + return TSDB_CODE_SUCCESS; +} + + void initQueryModuleMsgHandle() { queryBuildMsg[TMSG_INDEX(TDMT_VND_TABLE_META)] = queryBuildTableMetaReqMsg; queryBuildMsg[TMSG_INDEX(TDMT_MND_TABLE_META)] = queryBuildTableMetaReqMsg; @@ -427,6 +460,8 @@ void initQueryModuleMsgHandle() { queryBuildMsg[TMSG_INDEX(TDMT_MND_GET_DB_CFG)] = queryBuildGetDBCfgMsg; queryBuildMsg[TMSG_INDEX(TDMT_MND_GET_INDEX)] = queryBuildGetIndexMsg; queryBuildMsg[TMSG_INDEX(TDMT_MND_RETRIEVE_FUNC)] = queryBuildRetrieveFuncMsg; + queryBuildMsg[TMSG_INDEX(TDMT_MND_GET_USER_AUTH)] = queryBuildGetUserAuthMsg; + queryProcessMsgRsp[TMSG_INDEX(TDMT_VND_TABLE_META)] = queryProcessTableMetaRsp; queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_TABLE_META)] = queryProcessTableMetaRsp; @@ -435,6 +470,7 @@ void initQueryModuleMsgHandle() { queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_GET_DB_CFG)] = queryProcessGetDbCfgRsp; queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_GET_INDEX)] = queryProcessGetIndexRsp; queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_RETRIEVE_FUNC)] = queryProcessRetrieveFuncRsp; + queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_GET_USER_AUTH)] = queryProcessGetUserAuthRsp; } #pragma GCC diagnostic pop diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 51ac48adaf..77caf67a2b 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -1513,3 +1513,9 @@ int32_t winEndTsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p colDataAppendInt64(pOutput->columnData, pOutput->numOfRows, (int64_t*) colDataGetData(pInput->columnData, 4)); return TSDB_CODE_SUCCESS; } + +int32_t qTbnameFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + ASSERT(inputNum == 1); + colDataAppend(pOutput->columnData, pOutput->numOfRows, colDataGetData(pInput->columnData, 0), false); + return TSDB_CODE_SUCCESS; +} diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index 84aa5559d0..1e37533f2c 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -1023,8 +1023,7 @@ static void vectorMathMultiplyHelper(SColumnInfoData* pLeftCol, SColumnInfoData* colDataAppendNULL(pOutputCol, i); continue; // TODO set null or ignore } - *output = getVectorDoubleValueFnLeft(LEFT_COL, i) - * getVectorDoubleValueFnRight(RIGHT_COL, 0); + *output = getVectorDoubleValueFnLeft(LEFT_COL, i) * getVectorDoubleValueFnRight(RIGHT_COL, 0); } } } @@ -1050,8 +1049,7 @@ void vectorMathMultiply(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam colDataAppendNULL(pOutputCol, i); continue; // TODO set null or ignore } - *output = getVectorDoubleValueFnLeft(LEFT_COL, i) - * getVectorDoubleValueFnRight(RIGHT_COL, i); + *output = getVectorDoubleValueFnLeft(LEFT_COL, i) * getVectorDoubleValueFnRight(RIGHT_COL, i); } } else if (pLeft->numOfRows == 1) { vectorMathMultiplyHelper(pRightCol, pLeftCol, pOutputCol, pRight->numOfRows, step, i); diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index f776fb3764..846cf6f967 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -46,9 +46,21 @@ void* rpcOpen(const SRpcInit* pInit) { pRpc->numOfThreads = pInit->numOfThreads > TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS : pInit->numOfThreads; } + uint32_t ip = 0; + if (pInit->connType == TAOS_CONN_SERVER) { + ip = taosGetIpv4FromFqdn(pInit->localFqdn); + if (ip == 0xFFFFFFFF) { + tError("invalid fqdn: %s", pInit->localFqdn); + terrno = TSDB_CODE_RPC_FQDN_ERROR; + taosMemoryFree(pRpc); + return NULL; + } + } + pRpc->connType = pInit->connType; pRpc->idleTime = pInit->idleTime; - pRpc->tcphandle = (*taosInitHandle[pRpc->connType])(0, pInit->localPort, pRpc->label, pRpc->numOfThreads, NULL, pRpc); + pRpc->tcphandle = + (*taosInitHandle[pRpc->connType])(ip, pInit->localPort, pRpc->label, pRpc->numOfThreads, NULL, pRpc); if (pRpc->tcphandle == NULL) { taosMemoryFree(pRpc); return NULL; diff --git a/source/libs/transport/src/transSrv.c b/source/libs/transport/src/transSrv.c index 7378ca3241..ad3f520210 100644 --- a/source/libs/transport/src/transSrv.c +++ b/source/libs/transport/src/transSrv.c @@ -817,7 +817,6 @@ void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, srv->pipe[i] = (uv_pipe_t*)taosMemoryCalloc(2, sizeof(uv_pipe_t)); - uv_os_sock_t fds[2]; if (uv_socketpair(SOCK_STREAM, 0, fds, UV_NONBLOCK_PIPE, UV_NONBLOCK_PIPE) != 0) { goto End; @@ -841,6 +840,10 @@ void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, goto End; } } + if (false == taosValidIpAndPort(srv->ip, srv->port)) { + tError("failed to bind, reason: %s", terrstr()); + goto End; + } if (false == addHandleToAcceptloop(srv)) { goto End; } diff --git a/source/os/src/osDir.c b/source/os/src/osDir.c index 6d90773d1b..19e4defafc 100644 --- a/source/os/src/osDir.c +++ b/source/os/src/osDir.c @@ -44,7 +44,7 @@ int wordexp(char *words, wordexp_t *pwordexp, int flags) { return -1; } - printf("parse relative path:%s to abs path:%s\n", words, pwordexp->wordPos); + // printf("parse relative path:%s to abs path:%s\n", words, pwordexp->wordPos); return 0; } diff --git a/source/os/src/osSocket.c b/source/os/src/osSocket.c index 6aa8520082..8cac660039 100644 --- a/source/os/src/osSocket.c +++ b/source/os/src/osSocket.c @@ -638,6 +638,48 @@ int32_t taosKeepTcpAlive(TdSocketPtr pSocket) { return 0; } +bool taosValidIpAndPort(uint32_t ip, uint16_t port) { + struct sockaddr_in serverAdd; + SocketFd fd; + int32_t reuse; + + // printf("open tcp server socket:0x%x:%hu", ip, port); + + bzero((char *)&serverAdd, sizeof(serverAdd)); + serverAdd.sin_family = AF_INET; + serverAdd.sin_addr.s_addr = ip; + serverAdd.sin_port = (uint16_t)htons(port); + + if ((fd = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) <= 2) { + // printf("failed to open TCP socket: %d (%s)", errno, strerror(errno)); + taosCloseSocketNoCheck1(fd); + return false; + } + + TdSocketPtr pSocket = (TdSocketPtr)taosMemoryMalloc(sizeof(TdSocket)); + if (pSocket == NULL) { + taosCloseSocketNoCheck1(fd); + return false; + } + pSocket->refId = 0; + pSocket->fd = fd; + + /* set REUSEADDR option, so the portnumber can be re-used */ + reuse = 1; + if (taosSetSockOpt(pSocket, SOL_SOCKET, SO_REUSEADDR, (void *)&reuse, sizeof(reuse)) < 0) { + // printf("setsockopt SO_REUSEADDR failed: %d (%s)", errno, strerror(errno)); + taosCloseSocket(&pSocket); + return NULL; + } + /* bind socket to server address */ + if (bind(pSocket->fd, (struct sockaddr *)&serverAdd, sizeof(serverAdd)) < 0) { + // printf("bind tcp server socket failed, 0x%x:%hu(%s)", ip, port, strerror(errno)); + taosCloseSocket(&pSocket); + return false; + } + taosCloseSocket(&pSocket); + return true; +} TdSocketServerPtr taosOpenTcpServerSocket(uint32_t ip, uint16_t port) { struct sockaddr_in serverAdd; SocketFd fd; diff --git a/tests/requirements.txt b/tests/requirements.txt new file mode 100644 index 0000000000..ce459414c4 --- /dev/null +++ b/tests/requirements.txt @@ -0,0 +1,5 @@ +taospy +numpy +fabric2 +psutil +pandas diff --git a/tests/script/tsim/query/scalarFunction.sim b/tests/script/tsim/query/scalarFunction.sim index 80d2dafa84..97d49ae901 100644 --- a/tests/script/tsim/query/scalarFunction.sim +++ b/tests/script/tsim/query/scalarFunction.sim @@ -105,17 +105,22 @@ print ====> $data90 $data91 $data92 $data93 $data94 $data95 if $rows != $rowNum then return -1 endi + print ====> select c1, abs(c1), c2, abs(c2), c3, abs(c3) from stb sql select c1, abs(c1), c2, abs(c2), c3, abs(c3) from stb +print ====> rows = $rows and totalRows = $totalRows for stb if $rows != $totalRows then return -1 endi + print ====> select c1, abs(c1), c2, abs(c2), c3, abs(c3) from ntb sql select c1, abs(c1), c2, abs(c2), c3, abs(c3) from ntb +print ====> rows = $rows and rowNum = $rowNum for ntb if $rows != $rowNum then return -1 endi + print ====> log sql select c1, log(c1, 10), c2, log(c2, 10), c3, log(c3, 10) from ct1 print ====> select c1, log(c1, 10), c2, log(c2, 10), c3, log(c3, 10) from ct1 diff --git a/tests/script/tsim/query/udf.sim b/tests/script/tsim/query/udf.sim index 8acd07cfe4..c76569b40f 100644 --- a/tests/script/tsim/query/udf.sim +++ b/tests/script/tsim/query/udf.sim @@ -43,6 +43,27 @@ if $data00 != 2.236067977 then return -1 endi +sql create table t2 (ts timestamp, f1 int, f2 int); +sql insert into t2 values(now, 0, 0)(now+1s, 1, 1); +sql select udf1(f1, f2) from t2; +if $rows != 2 then + return -1 +endi +if $data00 != 88 then + return -1 +endi +if $data10 != 88 then + return -1 +endi + +sql select udf2(f1, f2) from t2; +if $rows != 1 then + return -1 +endi +if $data00 != 1.414213562 then + return -1 +endi + #sql drop function udf1; #sql drop function udf2; system sh/exec.sh -n dnode1 -s stop -x SIGKILL diff --git a/tests/system-test/1-insert/insertWithMoreVgroup.py b/tests/system-test/1-insert/insertWithMoreVgroup.py new file mode 100644 index 0000000000..a7d17bc41e --- /dev/null +++ b/tests/system-test/1-insert/insertWithMoreVgroup.py @@ -0,0 +1,291 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import threading +import multiprocessing as mp +from numpy.lib.function_base import insert +import taos +from util.log import * +from util.cases import * +from util.sql import * +import numpy as np +import datetime as dt +import time +# constant define +WAITS = 5 # wait seconds + +class TDTestCase: + # + # --------------- main frame ------------------- + # + + def caseDescription(self): + ''' + limit and offset keyword function test cases; + case1: limit offset base function test + case2: offset return valid + ''' + return + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root)-len("/build/bin")] + break + return buildPath + + # init + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + # tdSql.prepare() + # self.create_tables(); + self.ts = 1500000000000 + + + # run case + def run(self): + + # test base case + self.test_case1() + tdLog.debug(" LIMIT test_case1 ............ [OK]") + + # test advance case + # self.test_case2() + # tdLog.debug(" LIMIT test_case2 ............ [OK]") + + # stop + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + # --------------- case ------------------- + + # create tables + def create_tables(self,dbname,stbname,count): + tdSql.execute("use %s" %dbname) + tdSql.execute("create stable %s(ts timestamp, c1 int, c2 binary(10)) tags(t1 int)"%stbname) + pre_create = "create table" + sql = pre_create + tdLog.debug("doing create one stable %s and %d child table in %s ..." %(stbname, count ,dbname)) + # print(time.time()) + exeStartTime=time.time() + for i in range(count): + sql += " %s_%d using %s tags(%d)"%(stbname,i,stbname,i+1) + if i >0 and i%3000 == 0: + tdSql.execute(sql) + sql = pre_create + # print(time.time()) + # end sql + if sql != pre_create: + tdSql.execute(sql) + exeEndTime=time.time() + spendTime=exeEndTime-exeStartTime + speedCreate=count/spendTime + tdLog.debug("spent %.2fs to create 1 stable and %d table, create speed is %.2f table/s... [OK]"% (spendTime,count,speedCreate)) + return + + def newcur(self,host,cfg): + user = "root" + password = "taosdata" + port =6030 + con=taos.connect(host=host, user=user, password=password, config=cfg ,port=port) + cur=con.cursor() + print(cur) + return cur + + def new_create_tables(self,dbname,vgroups,stbname,tcountStart,tcountStop): + host = "chenhaoran02" + buildPath = self.getBuildPath() + config = buildPath+ "../sim/dnode1/cfg/" + + tsql=self.newcur(host,config) + tsql.execute("create database %s vgroups %d"%(dbname,vgroups)) + tsql.execute("use %s" %dbname) + tsql.execute("create stable %s(ts timestamp, c1 int, c2 binary(10)) tags(t1 int)"%stbname) + + pre_create = "create table" + sql = pre_create + tcountStop=int(tcountStop) + tcountStart=int(tcountStart) + count=tcountStop-tcountStart + + tdLog.debug("doing create one stable %s and %d child table in %s ..." %(stbname, count ,dbname)) + # print(time.time()) + exeStartTime=time.time() + # print(type(tcountStop),type(tcountStart)) + for i in range(tcountStart,tcountStop): + sql += " %s_%d using %s tags(%d)"%(stbname,i,stbname,i+1) + if i >0 and i%20000 == 0: + # print(sql) + tsql.execute(sql) + sql = pre_create + # print(time.time()) + # end sql + if sql != pre_create: + # print(sql) + tsql.execute(sql) + exeEndTime=time.time() + spendTime=exeEndTime-exeStartTime + speedCreate=count/spendTime + # tdLog.debug("spent %.2fs to create 1 stable and %d table, create speed is %.2f table/s... [OK]"% (spendTime,count,speedCreate)) + return + + + + # insert data + def insert_data(self, dbname, stbname, ts_start, tcountStart,tcountStop,rowCount): + tdSql.execute("use %s" %dbname) + pre_insert = "insert into " + sql = pre_insert + tcount=tcountStop-tcountStart + allRows=tcount*rowCount + tdLog.debug("doing insert data into stable:%s rows:%d ..."%(stbname, allRows)) + exeStartTime=time.time() + for i in range(tcountStart,tcountStop): + sql += " %s_%d values "%(stbname,i) + for j in range(rowCount): + sql += "(%d, %d, 'taos_%d') "%(ts_start + j*1000, j, j) + if j >0 and j%5000 == 0: + # print(sql) + tdSql.execute(sql) + sql = "insert into %s_%d values " %(stbname,i) + # end sql + if sql != pre_insert: + # print(sql) + tdSql.execute(sql) + exeEndTime=time.time() + spendTime=exeEndTime-exeStartTime + speedInsert=allRows/spendTime + # tdLog.debug("spent %.2fs to INSERT %d rows , insert rate is %.2f rows/s... [OK]"% (spendTime,allRows,speedInsert)) + + tdLog.debug("INSERT TABLE DATA ............ [OK]") + return + + + # test case1 base + def test_case1(self): + tdLog.debug("-----create database and tables test------- ") + tdSql.execute("drop database if exists db1") + tdSql.execute("drop database if exists db4") + tdSql.execute("drop database if exists db6") + tdSql.execute("drop database if exists db8") + tdSql.execute("drop database if exists db12") + tdSql.execute("drop database if exists db16") + + #create database and tables; + + # tdSql.execute("create database db11 vgroups 1") + # # self.create_tables("db1", "stb1", 30*10000) + # tdSql.execute("use db1") + # tdSql.execute("create stable stb1(ts timestamp, c1 int, c2 binary(10)) tags(t1 int)") + + # tdSql.execute("create database db12 vgroups 1") + # # self.create_tables("db1", "stb1", 30*10000) + # tdSql.execute("use db1") + + # t1 = threading.Thread(target=self.new_create_tables("db1", "stb1", 15*10000), args=(1,)) + # t2 = threading.Thread(target=self.new_create_tables("db1", "stb1", 15*10000), args=(2,)) + # t1 = mp.Process(target=self.new_create_tables, args=("db1", "stb1", 0,count/2,)) + # t2 = mp.Process(target=self.new_create_tables, args=("db1", "stb1", count/2,count,)) + + count=50000 + vgroups=1 + threads = [] + threadNumbers=2 + for i in range(threadNumbers): + threads.append(mp.Process(target=self.new_create_tables, args=("db1%d"%i, vgroups, "stb1", 0,count,))) + start_time = time.time() + for tr in threads: + tr.start() + for tr in threads: + tr.join() + end_time = time.time() + spendTime=end_time-start_time + speedCreate=count/spendTime + tdLog.debug("spent %.2fs to create 1 stable and %d table, create speed is %.2f table/s... [OK]"% (spendTime,count,speedCreate)) + # self.new_create_tables("db1", "stb1", 15*10000) + # self.new_create_tables("db1", "stb1", 15*10000) + + # tdSql.execute("create database db4 vgroups 4") + # self.create_tables("db4", "stb4", 30*10000) + + # tdSql.execute("create database db6 vgroups 6") + # self.create_tables("db6", "stb6", 30*10000) + + # tdSql.execute("create database db8 vgroups 8") + # self.create_tables("db8", "stb8", 30*10000) + + # tdSql.execute("create database db12 vgroups 12") + # self.create_tables("db12", "stb12", 30*10000) + + # tdSql.execute("create database db16 vgroups 16") + # self.create_tables("db16", "stb16", 30*10000) + return + + # test case2 base:insert data + def test_case2(self): + + tdLog.debug("-----insert data test------- ") + # drop database + tdSql.execute("drop database if exists db1") + tdSql.execute("drop database if exists db4") + tdSql.execute("drop database if exists db6") + tdSql.execute("drop database if exists db8") + tdSql.execute("drop database if exists db12") + tdSql.execute("drop database if exists db16") + + #create database and tables; + + tdSql.execute("create database db1 vgroups 1") + self.create_tables("db1", "stb1", 1*100) + self.insert_data("db1", "stb1", self.ts, 1*50,1*10000) + + + tdSql.execute("create database db4 vgroups 4") + self.create_tables("db4", "stb4", 1*100) + self.insert_data("db4", "stb4", self.ts, 1*100,1*10000) + + tdSql.execute("create database db6 vgroups 6") + self.create_tables("db6", "stb6", 1*100) + self.insert_data("db6", "stb6", self.ts, 1*100,1*10000) + + tdSql.execute("create database db8 vgroups 8") + self.create_tables("db8", "stb8", 1*100) + self.insert_data("db8", "stb8", self.ts, 1*100,1*10000) + + tdSql.execute("create database db12 vgroups 12") + self.create_tables("db12", "stb12", 1*100) + self.insert_data("db12", "stb12", self.ts, 1*100,1*10000) + + tdSql.execute("create database db16 vgroups 16") + self.create_tables("db16", "stb16", 1*100) + self.insert_data("db16", "stb16", self.ts, 1*100,1*10000) + + return + +# +# add case with filename +# +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/system-test/2-query/join.py b/tests/system-test/2-query/join.py index b4878e42c9..a39bc21946 100644 --- a/tests/system-test/2-query/join.py +++ b/tests/system-test/2-query/join.py @@ -5,6 +5,7 @@ from util.sql import * from util.cases import * from util.dnodes import * +PRIMARY_COL = "ts" INT_COL = "c1" BINT_COL = "c2" @@ -18,9 +19,10 @@ BINARY_COL = "c8" NCHAR_COL = "c9" TS_COL = "c10" -UN_CHAR_COL = [INT_COL, BINT_COL, SINT_COL, TINT_COL, FLOAT_COL, DOUBLE_COL, BOOL_COL, ] +NUM_COL = [ INT_COL, BINT_COL, SINT_COL, TINT_COL, FLOAT_COL, DOUBLE_COL, ] CHAR_COL = [ BINARY_COL, NCHAR_COL, ] -TS_TYPE_COL = [TS_COL] +BOOLEAN_COL = [ BOOL_COL, ] +TS_TYPE_COL = [ TS_COL, ] class TDTestCase: @@ -28,88 +30,171 @@ class TDTestCase: tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) - def __length_condition(self): - length_condition = [] + def __query_condition(self,tbname): + query_condition = [] for char_col in CHAR_COL: - length_condition.extend( + query_condition.extend( ( - char_col, - f"upper( {char_col} )", + f"{tbname}.{char_col}", + f"upper( {tbname}.{char_col} )", ) ) - length_condition.extend( f"cast( {un_char_col} as binary(16) ) " for un_char_col in UN_CHAR_COL) - length_condition.extend( f"cast( {char_col} + {char_col_2} as binary(32) ) " for char_col_2 in CHAR_COL ) - length_condition.extend( f"cast( {char_col} + {un_char_col} as binary(32) ) " for un_char_col in UN_CHAR_COL ) + query_condition.extend( f"cast( {tbname}.{un_char_col} as binary(16) ) " for un_char_col in NUM_COL) + query_condition.extend( f"cast( {tbname}.{char_col} + {tbname}.{char_col_2} as binary(32) ) " for char_col_2 in CHAR_COL ) + query_condition.extend( f"cast( {tbname}.{char_col} + {tbname}.{un_char_col} as binary(32) ) " for un_char_col in NUM_COL ) + for num_col in NUM_COL: + query_condition.extend( + ( + f"{tbname}.{num_col}", + f"sin( {tbname}.{num_col} )" + ) + ) + query_condition.extend( f"{tbname}.{num_col} + {tbname}.{num_col_1} " for num_col_1 in NUM_COL ) - length_condition.append('''"test1234!@#$%^&*():'> 0 " - return "" + def __join_condition(self, tb_list, filter=PRIMARY_COL): + # sourcery skip: flip-comparison + if 1 == len(tb_list): + join_filter = f"{tb_list[0]}.{filter} = {tb_list[0]}.{filter} " + elif 2 == len(tb_list): + join_filter = f"{tb_list[0]}.{filter} = {tb_list[1]}.{filter} " + else: + join_filter = f"{tb_list[0]}.{filter} = {tb_list[1]}.{filter} " + for i in range(1, len(tb_list)-1 ): + join_filter += f"and {tb_list[i]}.{filter} = {tb_list[i+1]}.{filter}" - def __group_condition(self, col, having = ""): + return join_filter + + def __where_condition(self, col, tbname): + if col in NUM_COL: + return f" abs( {tbname}.{col} ) >= 0" + elif col in CHAR_COL: + return f" lower( {tbname}.{col} ) like 'bina%' or lower( {tbname}.{col} ) like '_cha%' " + elif col in BOOLEAN_COL: + return f" {tbname}.{col} in (false, true) " + elif col in TS_TYPE_COL or col in PRIMARY_COL: + return f" cast( {tbname}.{col} as binary(16) ) is not null " + else: + return "" + + def __group_condition(self, tbname, col, having = ""): return f" group by {col} having {having}" if having else f" group by {col} " - def __length_current_check(self, tbname): - length_condition = self.__length_condition() - for condition in length_condition: - where_condition = self.__where_condition(condition) - group_having = self.__group_condition(condition, having=f"{condition} is not null " ) - group_no_having= self.__group_condition(condition ) + def __join_check(self, tblist, checkrows, join_flag=True): + query_conditions = self.__query_condition(tblist[0]) + join_condition = self.__join_condition(tb_list=tblist) if join_flag else " " + for condition in query_conditions: + where_condition = self.__where_condition(col=condition, tbname=tblist[0]) + group_having = self.__group_condition(tbname=tblist[0], col=condition, having=f"{condition} is not null " ) + group_no_having= self.__group_condition(tbname=tblist[0], col=condition ) groups = ["", group_having, group_no_having] - for group_condition in groups: - tdSql.query(f"select {condition} from {tbname} {where_condition} {group_condition} ") - datas = [tdSql.getData(i,0) for i in range(tdSql.queryRows)] - length_data = [ len(str(data)) if data else None for data in datas ] - tdSql.query(f"select length( {condition} ) from {tbname} {where_condition} {group_condition}") - for i in range(len(length_data)): - tdSql.checkData(i, 0, length_data[i] ) if length_data[i] else tdSql.checkData(i, 0, None) + if where_condition: + sql = f" select {condition} from {tblist[0]},{tblist[1]} where {join_condition} and {where_condition} {group_condition} " + else: + sql = f" select {condition} from {tblist[0]},{tblist[1]} where {join_condition} {group_condition} " - def __length_err_check(self,tbname): - sqls = [] + if not join_flag : + tdSql.error(sql=sql) + if len(tblist) == 2: + if "ct1" in tblist or "t1" in tblist: + self.__join_current(sql, checkrows) + elif where_condition or "not null" in group_condition: + self.__join_current(sql, checkrows + 2 ) + elif group_condition: + self.__join_current(sql, checkrows + 3 ) + else: + self.__join_current(sql, checkrows + 5 ) + if len(tblist) > 2 or len(tblist) < 1: + tdSql.error(sql=sql) - for un_char_col in UN_CHAR_COL: - sqls.extend( - ( - f"select length( {un_char_col} ) from {tbname} ", - f"select length(ceil( {un_char_col} )) from {tbname} ", - f"select {un_char_col} from {tbname} group by length( {un_char_col} ) ", - ) - ) + # def __join_err_check(self,tbname): + # sqls = [] - sqls.extend( f"select length( {un_char_col} + {un_char_col_2} ) from {tbname} " for un_char_col_2 in UN_CHAR_COL ) - sqls.extend( f"select length( {un_char_col} + {ts_col} ) from {tbname} " for ts_col in TS_TYPE_COL ) + # for un_char_col in NUM_COL: + # sqls.extend( + # ( + # f"select length( {un_char_col} ) from {tbname} ", + # f"select length(ceil( {un_char_col} )) from {tbname} ", + # f"select {un_char_col} from {tbname} group by length( {un_char_col} ) ", + # ) + # ) - sqls.extend( f"select {char_col} from {tbname} group by length( {char_col} ) " for char_col in CHAR_COL) - sqls.extend( f"select length( {ts_col} ) from {tbname} " for ts_col in TS_TYPE_COL ) - sqls.extend( f"select length( {char_col} + {ts_col} ) from {tbname} " for char_col in UN_CHAR_COL for ts_col in TS_TYPE_COL) - sqls.extend( f"select length( {char_col} + {char_col_2} ) from {tbname} " for char_col in CHAR_COL for char_col_2 in CHAR_COL ) - sqls.extend( f"select upper({char_col}, 11) from {tbname} " for char_col in CHAR_COL ) - sqls.extend( f"select upper({char_col}) from {tbname} interval(2d) sliding(1d)" for char_col in CHAR_COL ) - sqls.extend( - ( - f"select length() from {tbname} ", - f"select length(*) from {tbname} ", - f"select length(ccccccc) from {tbname} ", - f"select length(111) from {tbname} ", - f"select length(c8, 11) from {tbname} ", - ) - ) + # sqls.extend( f"select length( {un_char_col} + {un_char_col_2} ) from {tbname} " for un_char_col_2 in NUM_COL ) + # sqls.extend( f"select length( {un_char_col} + {ts_col} ) from {tbname} " for ts_col in TS_TYPE_COL ) + + # sqls.extend( f"select {char_col} from {tbname} group by length( {char_col} ) " for char_col in CHAR_COL) + # sqls.extend( f"select length( {ts_col} ) from {tbname} " for ts_col in TS_TYPE_COL ) + # sqls.extend( f"select length( {char_col} + {ts_col} ) from {tbname} " for char_col in NUM_COL for ts_col in TS_TYPE_COL) + # sqls.extend( f"select length( {char_col} + {char_col_2} ) from {tbname} " for char_col in CHAR_COL for char_col_2 in CHAR_COL ) + # sqls.extend( f"select upper({char_col}, 11) from {tbname} " for char_col in CHAR_COL ) + # sqls.extend( f"select upper({char_col}) from {tbname} interval(2d) sliding(1d)" for char_col in CHAR_COL ) + # sqls.extend( + # ( + # f"select length() from {tbname} ", + # f"select length(*) from {tbname} ", + # f"select length(ccccccc) from {tbname} ", + # f"select length(111) from {tbname} ", + # f"select length(c8, 11) from {tbname} ", + # ) + # ) + + # return sqls + + def __join_current(self, sql, checkrows): + tdSql.query(sql=sql) + tdSql.checkRows(checkrows) - return sqls def __test_current(self): + # sourcery skip: extract-duplicate-method, inline-immediately-returned-variable tdLog.printNoPrefix("==========current sql condition check , must return query ok==========") - tbname = ["ct1", "ct2", "ct4", "t1"] - for tb in tbname: - self.__length_current_check(tb) - tdLog.printNoPrefix(f"==========current sql condition check in {tb} over==========") + tblist_1 = ["ct1", "ct2"] + self.__join_check(tblist_1, 1) + tdLog.printNoPrefix(f"==========current sql condition check in {tblist_1} over==========") + tblist_2 = ["ct2", "ct4"] + self.__join_check(tblist_2, self.rows) + tdLog.printNoPrefix(f"==========current sql condition check in {tblist_2} over==========") + tblist_3 = ["t1", "ct4"] + self.__join_check(tblist_3, 1) + tdLog.printNoPrefix(f"==========current sql condition check in {tblist_3} over==========") + tblist_4 = ["t1", "ct1"] + self.__join_check(tblist_4, 1) + tdLog.printNoPrefix(f"==========current sql condition check in {tblist_4} over==========") def __test_error(self): + # sourcery skip: extract-duplicate-method, move-assign-in-block tdLog.printNoPrefix("==========err sql condition check , must return error==========") + err_list_1 = ["ct1","ct2", "ct4"] + err_list_2 = ["ct1","ct2", "t1"] + err_list_3 = ["ct1","ct4", "t1"] + err_list_4 = ["ct2","ct4", "t1"] + err_list_5 = ["ct1", "ct2","ct4", "t1"] + self.__join_check(err_list_1, -1) + tdLog.printNoPrefix(f"==========err sql condition check in {err_list_1} over==========") + self.__join_check(err_list_2, -1) + tdLog.printNoPrefix(f"==========err sql condition check in {err_list_2} over==========") + self.__join_check(err_list_3, -1) + tdLog.printNoPrefix(f"==========err sql condition check in {err_list_3} over==========") + self.__join_check(err_list_4, -1) + tdLog.printNoPrefix(f"==========err sql condition check in {err_list_4} over==========") + self.__join_check(err_list_5, -1) + tdLog.printNoPrefix(f"==========err sql condition check in {err_list_5} over==========") + self.__join_check(["ct2", "ct4"], -1, join_flag=False) + tdLog.printNoPrefix("==========err sql condition check in has no join condition over==========") + + tdSql.error( f"select c1, c2 from ct2, ct4 where ct2.{PRIMARY_COL}=ct4.{PRIMARY_COL}" ) + tdSql.error( f"select ct2.c1, ct2.c2 from ct2, ct4 where ct2.{INT_COL}=ct4.{INT_COL}" ) + tdSql.error( f"select ct2.c1, ct2.c2 from ct2, ct4 where ct2.{TS_COL}=ct4.{TS_COL}" ) + tdSql.error( f"select ct2.c1, ct2.c2 from ct2, ct4 where ct2.{PRIMARY_COL}=ct4.{TS_COL}" ) + tdSql.error( f"select ct2.c1, ct1.c2 from ct2, ct4 where ct2.{PRIMARY_COL}=ct4.{PRIMARY_COL}" ) + tdSql.error( f"select ct2.c1, ct4.c2 from ct2, ct4 where ct2.{PRIMARY_COL}=ct4.{PRIMARY_COL} and c1 is not null " ) + tdSql.error( f"select ct2.c1, ct4.c2 from ct2, ct4 where ct2.{PRIMARY_COL}=ct4.{PRIMARY_COL} and ct1.c1 is not null " ) + + tbname = ["ct1", "ct2", "ct4", "t1"] for tb in tbname: @@ -150,33 +235,33 @@ class TDTestCase: now_time = int(datetime.datetime.timestamp(datetime.datetime.now()) * 1000) for i in range(rows): tdSql.execute( - f"insert into ct1 values ( { now_time - i * 1000 }, {i}, {11111 * i}, {111 * i % 32767 }, {11 * i % 127}, {1.11*i}, {1100.0011*i}, {i%2}, 'binary{i}', 'nchar{i}', { now_time + 1 * i } )" + f"insert into ct1 values ( { now_time - i * 1000 }, {i}, {11111 * i}, {111 * i % 32767 }, {11 * i % 127}, {1.11*i}, {1100.0011*i}, {i%2}, 'binary{i}', 'nchar_测试_{i}', { now_time + 1 * i } )" ) tdSql.execute( - f"insert into ct4 values ( { now_time - i * 7776000000 }, {i}, {11111 * i}, {111 * i % 32767 }, {11 * i % 127}, {1.11*i}, {1100.0011*i}, {i%2}, 'binary{i}', 'nchar{i}', { now_time + 1 * i } )" + f"insert into ct4 values ( { now_time - i * 7776000000 }, {i}, {11111 * i}, {111 * i % 32767 }, {11 * i % 127}, {1.11*i}, {1100.0011*i}, {i%2}, 'binary{i}', 'nchar_测试_{i}', { now_time + 1 * i } )" ) tdSql.execute( - f"insert into ct2 values ( { now_time - i * 7776000000 }, {-i}, {-11111 * i}, {-111 * i % 32767 }, {-11 * i % 127}, {-1.11*i}, {-1100.0011*i}, {i%2}, 'binary{i}', 'nchar{i}', { now_time + 1 * i } )" + f"insert into ct2 values ( { now_time - i * 7776000000 }, {-i}, {-11111 * i}, {-111 * i % 32767 }, {-11 * i % 127}, {-1.11*i}, {-1100.0011*i}, {i%2}, 'binary{i}', 'nchar_测试_{i}', { now_time + 1 * i } )" ) tdSql.execute( f'''insert into ct1 values - ( { now_time - rows * 5 }, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', { now_time + 8 } ) - ( { now_time + 10000 }, { rows }, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', { now_time + 9 } ) + ( { now_time - rows * 5 }, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar_测试_0', { now_time + 8 } ) + ( { now_time + 10000 }, { rows }, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar_测试_9', { now_time + 9 } ) ''' ) tdSql.execute( f'''insert into ct4 values ( { now_time - rows * 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) - ( { now_time - rows * 3888000000+ 10800000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( { now_time - rows * 3888000000 + 10800000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ( { now_time + 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ( { now_time + 5184000000}, {pow(2,31)-pow(2,15)}, {pow(2,63)-pow(2,30)}, 32767, 127, - { 3.3 * pow(10,38) }, { 1.3 * pow(10,308) }, { rows % 2 }, "binary_limit-1", "nchar_limit-1", { now_time - 86400000} + { 3.3 * pow(10,38) }, { 1.3 * pow(10,308) }, { rows % 2 }, "binary_limit-1", "nchar_测试_limit-1", { now_time - 86400000} ) ( { now_time + 2592000000 }, {pow(2,31)-pow(2,16)}, {pow(2,63)-pow(2,31)}, 32766, 126, - { 3.2 * pow(10,38) }, { 1.2 * pow(10,308) }, { (rows-1) % 2 }, "binary_limit-2", "nchar_limit-2", { now_time - 172800000} + { 3.2 * pow(10,38) }, { 1.2 * pow(10,308) }, { (rows-1) % 2 }, "binary_limit-2", "nchar_测试_limit-2", { now_time - 172800000} ) ''' ) @@ -184,15 +269,15 @@ class TDTestCase: tdSql.execute( f'''insert into ct2 values ( { now_time - rows * 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) - ( { now_time - rows * 3888000000+ 10800000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( { now_time - rows * 3888000000 + 10800000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ( { now_time + 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ( { now_time + 5184000000 }, { -1 * pow(2,31) + pow(2,15) }, { -1 * pow(2,63) + pow(2,30) }, -32766, -126, - { -1 * 3.2 * pow(10,38) }, { -1.2 * pow(10,308) }, { rows % 2 }, "binary_limit-1", "nchar_limit-1", { now_time - 86400000 } + { -1 * 3.2 * pow(10,38) }, { -1.2 * pow(10,308) }, { rows % 2 }, "binary_limit-1", "nchar_测试_limit-1", { now_time - 86400000 } ) ( { now_time + 2592000000 }, { -1 * pow(2,31) + pow(2,16) }, { -1 * pow(2,63) + pow(2,31) }, -32767, -127, - { - 3.3 * pow(10,38) }, { -1.3 * pow(10,308) }, { (rows-1) % 2 }, "binary_limit-2", "nchar_limit-2", { now_time - 172800000 } + { - 3.3 * pow(10,38) }, { -1.3 * pow(10,308) }, { (rows-1) % 2 }, "binary_limit-2", "nchar_测试_limit-2", { now_time - 172800000 } ) ''' ) @@ -200,7 +285,7 @@ class TDTestCase: for i in range(rows): insert_data = f'''insert into t1 values ( { now_time - i * 3600000 }, {i}, {i * 11111}, { i % 32767 }, { i % 127}, { i * 1.11111 }, { i * 1000.1111 }, { i % 2}, - "binary_{i}", "nchar_{i}", { now_time - 1000 * i } ) + "binary_{i}", "nchar_测试_{i}", { now_time - 1000 * i } ) ''' tdSql.execute(insert_data) tdSql.execute( @@ -210,12 +295,12 @@ class TDTestCase: ( { now_time - rows * 3600000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ( { now_time + 7200000 }, { pow(2,31) - pow(2,15) }, { pow(2,63) - pow(2,30) }, 32767, 127, { 3.3 * pow(10,38) }, { 1.3 * pow(10,308) }, { rows % 2 }, - "binary_limit-1", "nchar_limit-1", { now_time - 86400000 } + "binary_limit-1", "nchar_测试_limit-1", { now_time - 86400000 } ) ( { now_time + 3600000 } , { pow(2,31) - pow(2,16) }, { pow(2,63) - pow(2,31) }, 32766, 126, { 3.2 * pow(10,38) }, { 1.2 * pow(10,308) }, { (rows-1) % 2 }, - "binary_limit-2", "nchar_limit-2", { now_time - 172800000 } + "binary_limit-2", "nchar_测试_limit-2", { now_time - 172800000 } ) ''' ) @@ -228,7 +313,8 @@ class TDTestCase: self.__create_tb() tdLog.printNoPrefix("==========step2:insert data") - self.__insert_data(10) + self.rows = 10 + self.__insert_data(self.rows) tdLog.printNoPrefix("==========step3:all check") self.all_test() diff --git a/tests/system-test/2-query/log.py b/tests/system-test/2-query/log.py index e6762b2d61..6e4c292183 100644 --- a/tests/system-test/2-query/log.py +++ b/tests/system-test/2-query/log.py @@ -191,13 +191,13 @@ class TDTestCase: def support_types(self): type_error_sql_lists = [ "select log(ts ,2 ) from t1" , - "select log(c7,2 ) from t1", - "select log(c8,2 ) from t1", - "select log(c9,2 ) from t1", - "select log(ts,2 ) from ct1" , - "select log(c7,2 ) from ct1", - "select log(c8,2 ) from ct1", - "select log(c9,2 ) from ct1", + "select log(c7,c2 ) from t1", + "select log(c8,c1 ) from t1", + "select log(c9,c2 ) from t1", + "select log(ts,c7 ) from ct1" , + "select log(c7,c9 ) from ct1", + "select log(c8,c2 ) from ct1", + "select log(c9,c1 ) from ct1", "select log(ts,2 ) from ct3" , "select log(c7,2 ) from ct3", "select log(c8,2 ) from ct3", diff --git a/tests/system-test/2-query/pow.py b/tests/system-test/2-query/pow.py new file mode 100644 index 0000000000..8b0137b411 --- /dev/null +++ b/tests/system-test/2-query/pow.py @@ -0,0 +1,652 @@ +import taos +import sys +import datetime +import inspect +import math +from util.log import * +from util.sql import * +from util.cases import * + + +class TDTestCase: + updatecfgDict = {'debugFlag': 143 ,"cDebugFlag":143,"uDebugFlag":143 ,"rpcDebugFlag":143 , "tmrDebugFlag":143 , + "jniDebugFlag":143 ,"simDebugFlag":143,"dDebugFlag":143, "dDebugFlag":143,"vDebugFlag":143,"mDebugFlag":143,"qDebugFlag":143, + "wDebugFlag":143,"sDebugFlag":143,"tsdbDebugFlag":143,"tqDebugFlag":143 ,"fsDebugFlag":143 ,"fnDebugFlag":143} + def init(self, conn, powSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + + def prepare_datas(self): + tdSql.execute( + '''create table stb1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + tags (t1 int) + ''' + ) + + tdSql.execute( + ''' + create table t1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + ''' + ) + for i in range(4): + tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )') + + for i in range(9): + tdSql.execute( + f"insert into ct1 values ( now()-{i*10}s, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + ) + tdSql.execute( + f"insert into ct4 values ( now()-{i*90}d, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + ) + tdSql.execute("insert into ct1 values (now()-45s, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a )") + tdSql.execute("insert into ct1 values (now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute("insert into ct1 values (now()+15s, 9, -99999, -999, -99, -9.99, NULL, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute("insert into ct1 values (now()+20s, 9, -99999, -999, NULL, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + + tdSql.execute("insert into ct4 values (now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute("insert into ct4 values (now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute("insert into ct4 values (now()+90d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + + tdSql.execute( + f'''insert into t1 values + ( '2020-04-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2020-10-21 01:01:01.000', 1, 11111, 111, 11, 1.11, 11.11, 1, "binary1", "nchar1", now()+1a ) + ( '2020-12-31 01:01:01.000', 2, 22222, 222, 22, 2.22, 22.22, 0, "binary2", "nchar2", now()+2a ) + ( '2021-01-01 01:01:06.000', 3, 33333, 333, 33, 3.33, 33.33, 0, "binary3", "nchar3", now()+3a ) + ( '2021-05-07 01:01:10.000', 4, 44444, 444, 44, 4.44, 44.44, 1, "binary4", "nchar4", now()+4a ) + ( '2021-07-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2021-09-30 01:01:16.000', 5, 55555, 555, 55, 5.55, 55.55, 0, "binary5", "nchar5", now()+5a ) + ( '2022-02-01 01:01:20.000', 6, 66666, 666, 66, 6.66, 66.66, 1, "binary6", "nchar6", now()+6a ) + ( '2022-10-28 01:01:26.000', 7, 00000, 000, 00, 0.00, 00.00, 1, "binary7", "nchar7", "1970-01-01 08:00:00.000" ) + ( '2022-12-01 01:01:30.000', 8, -88888, -888, -88, -8.88, -88.88, 0, "binary8", "nchar8", "1969-01-01 01:00:00.000" ) + ( '2022-12-31 01:01:36.000', 9, -99999999999999999, -999, -99, -9.99, -999999999999999999999.99, 1, "binary9", "nchar9", "1900-01-01 00:00:00.000" ) + ( '2023-02-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ''' + ) + + def check_result_auto_pow2(self ,origin_query , pow_query): + + pow_result = tdSql.getResult(pow_query) + origin_result = tdSql.getResult(origin_query) + + auto_result =[] + + for row in origin_result: + row_check = [] + for elem in row: + if elem == None: + elem = None + else: + elem = math.pow(elem,2) + row_check.append(elem) + auto_result.append(row_check) + + check_status = True + + for row_index , row in enumerate(pow_result): + for col_index , elem in enumerate(row): + if auto_result[row_index][col_index] == None and not (auto_result[row_index][col_index] == None and elem == None): + check_status = False + elif auto_result[row_index][col_index] != None and (auto_result[row_index][col_index] - elem > 0.00000001): + check_status = False + else: + pass + if not check_status: + tdLog.notice("pow function value has not as expected , sql is \"%s\" "%pow_query ) + sys.exit(1) + else: + tdLog.info("pow value check pass , it work as expected ,sql is \"%s\" "%pow_query ) + + def check_result_auto_pow1(self ,origin_query , pow_query): + pow_result = tdSql.getResult(pow_query) + origin_result = tdSql.getResult(origin_query) + + auto_result =[] + + for row in origin_result: + row_check = [] + for elem in row: + if elem == None: + elem = None + else : + elem = pow(elem ,1) + row_check.append(elem) + auto_result.append(row_check) + + check_status = True + for row_index , row in enumerate(pow_result): + for col_index , elem in enumerate(row): + if auto_result[row_index][col_index] == None and not (auto_result[row_index][col_index] == None and elem == None): + check_status = False + elif auto_result[row_index][col_index] != None and (auto_result[row_index][col_index] - elem > 0.00000001): + check_status = False + else: + pass + if not check_status: + tdLog.notice("pow function value has not as expected , sql is \"%s\" "%pow_query ) + sys.exit(1) + else: + tdLog.info("pow value check pass , it work as expected ,sql is \"%s\" "%pow_query ) + + def check_result_auto_pow__10(self ,origin_query , pow_query): + pow_result = tdSql.getResult(pow_query) + origin_result = tdSql.getResult(origin_query) + + auto_result =[] + + for row in origin_result: + row_check = [] + for elem in row: + if elem == None: + elem = None + elif elem == 0: + elem = None + else: + elem = pow(elem ,-10) + row_check.append(elem) + auto_result.append(row_check) + + check_status = True + for row_index , row in enumerate(pow_result): + for col_index , elem in enumerate(row): + if auto_result[row_index][col_index] == None and not (auto_result[row_index][col_index] == None and elem == None): + check_status = False + elif auto_result[row_index][col_index] != None and (auto_result[row_index][col_index] - elem > 0.00000001): + check_status = False + else: + pass + if not check_status: + tdLog.notice("pow function value has not as expected , sql is \"%s\" "%pow_query ) + sys.exit(1) + else: + tdLog.info("pow value check pass , it work as expected ,sql is \"%s\" "%pow_query ) + + def test_errors(self): + error_sql_lists = [ + "select pow from t1", + # "select pow(-+--+c1 ,2) from t1", + # "select +-pow(c1,2) from t1", + # "select ++-pow(c1,2) from t1", + # "select ++--pow(c1,2) from t1", + # "select - -pow(c1,2)*0 from t1", + # "select pow(tbname+1,2) from t1 ", + "select pow(123--123,2)==1 from t1", + "select pow(c1,2) as 'd1' from t1", + "select pow(c1 ,c2 ,2) from t1", + "select pow(c1 ,NULL ,2) from t1", + "select pow(, 2) from t1;", + "select pow(pow(c1, 2) ab from t1)", + "select pow(c1 ,2 ) as int from t1", + "select pow from stb1", + # "select pow(-+--+c1) from stb1", + # "select +-pow(c1) from stb1", + # "select ++-pow(c1) from stb1", + # "select ++--pow(c1) from stb1", + # "select - -pow(c1)*0 from stb1", + # "select pow(tbname+1) from stb1 ", + "select pow(123--123 ,2)==1 from stb1", + "select pow(c1 ,2) as 'd1' from stb1", + "select pow(c1 ,c2 ,2 ) from stb1", + "select pow(c1 ,NULL,2) from stb1", + "select pow(,) from stb1;", + "select pow(pow(c1 , 2) ab from stb1)", + "select pow(c1 , 2) as int from stb1" + ] + for error_sql in error_sql_lists: + tdSql.error(error_sql) + + def support_types(self): + type_error_sql_lists = [ + "select pow(ts ,2 ) from t1" , + "select pow(c7,c1 ) from t1", + "select pow(c8,c2) from t1", + "select pow(c9,c3 ) from t1", + "select pow(ts,c4 ) from ct1" , + "select pow(c7,c5 ) from ct1", + "select pow(c8,c6 ) from ct1", + "select pow(c9,c8 ) from ct1", + "select pow(ts,2 ) from ct3" , + "select pow(c7,2 ) from ct3", + "select pow(c8,2 ) from ct3", + "select pow(c9,2 ) from ct3", + "select pow(ts,2 ) from ct4" , + "select pow(c7,2 ) from ct4", + "select pow(c8,2 ) from ct4", + "select pow(c9,2 ) from ct4", + "select pow(ts,2 ) from stb1" , + "select pow(c7,2 ) from stb1", + "select pow(c8,2 ) from stb1", + "select pow(c9,2 ) from stb1" , + + "select pow(ts,2 ) from stbbb1" , + "select pow(c7,2 ) from stbbb1", + + "select pow(ts,2 ) from tbname", + "select pow(c9,2 ) from tbname" + + ] + + for type_sql in type_error_sql_lists: + tdSql.error(type_sql) + + + type_sql_lists = [ + "select pow(c1,2 ) from t1", + "select pow(c2,2 ) from t1", + "select pow(c3,2 ) from t1", + "select pow(c4,2 ) from t1", + "select pow(c5,2 ) from t1", + "select pow(c6,2 ) from t1", + + "select pow(c1,2 ) from ct1", + "select pow(c2,2 ) from ct1", + "select pow(c3,2 ) from ct1", + "select pow(c4,2 ) from ct1", + "select pow(c5,2 ) from ct1", + "select pow(c6,2 ) from ct1", + + "select pow(c1,2 ) from ct3", + "select pow(c2,2 ) from ct3", + "select pow(c3,2 ) from ct3", + "select pow(c4,2 ) from ct3", + "select pow(c5,2 ) from ct3", + "select pow(c6,2 ) from ct3", + + "select pow(c1,2 ) from stb1", + "select pow(c2,2 ) from stb1", + "select pow(c3,2 ) from stb1", + "select pow(c4,2 ) from stb1", + "select pow(c5,2 ) from stb1", + "select pow(c6,2 ) from stb1", + + "select pow(c6,2) as alisb from stb1", + "select pow(c6,2) alisb from stb1", + ] + + for type_sql in type_sql_lists: + tdSql.query(type_sql) + + def basic_pow_function(self): + + # basic query + tdSql.query("select c1 from ct3") + tdSql.checkRows(0) + tdSql.query("select c1 from t1") + tdSql.checkRows(12) + tdSql.query("select c1 from stb1") + tdSql.checkRows(25) + + # used for empty table , ct3 is empty + tdSql.query("select pow(c1 ,2) from ct3") + tdSql.checkRows(0) + tdSql.query("select pow(c2 ,2) from ct3") + tdSql.checkRows(0) + tdSql.query("select pow(c3 ,2) from ct3") + tdSql.checkRows(0) + tdSql.query("select pow(c4 ,2) from ct3") + tdSql.checkRows(0) + tdSql.query("select pow(c5 ,2) from ct3") + tdSql.checkRows(0) + tdSql.query("select pow(c6 ,2) from ct3") + tdSql.checkRows(0) + + + # # used for regular table + tdSql.query("select pow(c1 ,2) from t1") + tdSql.checkData(0, 0, None) + tdSql.checkData(1 , 0, 1.000000000) + tdSql.checkData(3 , 0, 9.000000000) + tdSql.checkData(5 , 0, None) + + tdSql.query("select c1, c2, c3 , c4, c5 from t1") + tdSql.checkData(1, 4, 1.11000) + tdSql.checkData(3, 3, 33) + tdSql.checkData(5, 4, None) + + tdSql.query("select ts,c1, c2, c3 , c4, c5 from t1") + tdSql.checkData(1, 5, 1.11000) + tdSql.checkData(3, 4, 33) + tdSql.checkData(5, 5, None) + + self.check_result_auto_pow2( "select c1, c2, c3 , c4, c5 from t1", "select pow(c1 ,2), pow(c2 ,2) ,pow(c3, 2), pow(c4 ,2), pow(c5 ,2) from t1") + self.check_result_auto_pow1( "select c1, c2, c3 , c4, c5 from t1", "select pow(c1 ,1), pow(c2 ,1) ,pow(c3, 1), pow(c4 ,1), pow(c5 ,1) from t1") + self.check_result_auto_pow__10( "select c1, c2, c3 , c4, c5 from t1", "select pow(c1 ,-10), pow(c2 ,-10) ,pow(c3, -10), pow(c4 ,-10), pow(c5 ,-10) from t1") + + # used for sub table + tdSql.query("select c1 ,pow(c1 ,2) from ct1") + tdSql.checkData(0, 1, 64.000000000) + tdSql.checkData(1 , 1, 49.000000000) + tdSql.checkData(3 , 1, 25.000000000) + tdSql.checkData(4 , 1, 0) + + # # test bug fix for pow(c1,c2) + + tdSql.query("select c1, c5 ,pow(c1,c5) from ct4") + tdSql.checkData(0 , 2, None) + tdSql.checkData(1 , 2, 104577724.506799981) + tdSql.checkData(2 , 2, 3684781.623933245) + tdSql.checkData(3 , 2, 152225.429759376) + tdSql.checkData(4 , 2, 7573.273783071) + + + self.check_result_auto_pow2( "select c1, c2, c3 , c4, c5 from ct1", "select pow(c1,2), pow(c2,2) ,pow(c3,2), pow(c4,2), pow(c5,2) from ct1") + self.check_result_auto_pow__10( "select c1, c2, c3 , c4, c5 from ct1", "select pow(c1,-10), pow(c2,-10) ,pow(c3,-10), pow(c4,-10), pow(c5,-10) from ct1") + + # nest query for pow functions + tdSql.query("select c1 , pow(c1,2) ,pow(pow(c1,2),2) , pow(pow(pow(c1,2),2),2) from ct1;") + tdSql.checkData(0 , 0 , 8) + tdSql.checkData(0 , 1 , 64.000000000) + tdSql.checkData(0 , 2 , 4096.000000000) + tdSql.checkData(0 , 3 , 16777216.000000000) + + tdSql.checkData(1 , 0 , 7) + tdSql.checkData(1 , 1 , 49.000000000) + tdSql.checkData(1 , 2 , 2401.000000000) + tdSql.checkData(1 , 3 , 5764801.000000000) + + tdSql.checkData(4 , 0 , 0) + tdSql.checkData(4 , 1 , 0.000000000) + tdSql.checkData(4 , 2 , 0.000000000) + tdSql.checkData(4 , 3 , 0.000000000) + + # # used for stable table + + tdSql.query("select pow(c1, 2) from stb1") + tdSql.checkRows(25) + + + # used for not exists table + tdSql.error("select pow(c1, 2) from stbbb1") + tdSql.error("select pow(c1, 2) from tbname") + tdSql.error("select pow(c1, 2) from ct5") + + # mix with common col + tdSql.query("select c1, pow(c1 ,2) from ct1") + tdSql.checkData(0 , 0 ,8) + tdSql.checkData(0 , 1 ,64.000000000) + tdSql.checkData(4 , 0 ,0) + tdSql.checkData(4 , 1 ,0.000000000) + tdSql.query("select c1, pow(c1,2) from ct4") + tdSql.checkData(0 , 0 , None) + tdSql.checkData(0 , 1 ,None) + tdSql.checkData(4 , 0 ,5) + tdSql.checkData(4 , 1 ,25.000000000) + tdSql.checkData(5 , 0 ,None) + tdSql.checkData(5 , 1 ,None) + + # mix with common functions + tdSql.query("select c1, pow(c1 ,2),pow(c1,2), log(pow(c1,2) ,2) from ct4 ") + tdSql.checkData(0 , 0 ,None) + tdSql.checkData(0 , 1 ,None) + tdSql.checkData(0 , 2 ,None) + tdSql.checkData(0 , 3 ,None) + + tdSql.checkData(3 , 0 , 6) + tdSql.checkData(3 , 1 ,36.000000000) + tdSql.checkData(3 , 2 ,36.000000000) + tdSql.checkData(3 , 3 ,5.169925001) + + tdSql.query("select c1, pow(c1,1),c5, floor(c5 ) from stb1 ") + + # # mix with agg functions , not support + tdSql.error("select c1, pow(c1 ,2),c5, count(c5) from stb1 ") + tdSql.error("select c1, pow(c1 ,2),c5, count(c5) from ct1 ") + tdSql.error("select pow(c1 ,2), count(c5) from stb1 ") + tdSql.error("select pow(c1 ,2), count(c5) from ct1 ") + tdSql.error("select c1, count(c5) from ct1 ") + tdSql.error("select c1, count(c5) from stb1 ") + + # agg functions mix with agg functions + + tdSql.query("select max(c5), count(c5) from stb1") + tdSql.query("select max(c5), count(c5) from ct1") + + + # bug fix for count + tdSql.query("select count(c1) from ct4 ") + tdSql.checkData(0,0,9) + tdSql.query("select count(*) from ct4 ") + tdSql.checkData(0,0,12) + tdSql.query("select count(c1) from stb1 ") + tdSql.checkData(0,0,22) + tdSql.query("select count(*) from stb1 ") + tdSql.checkData(0,0,25) + + # # bug fix for compute + tdSql.query("select c1, pow(c1 ,2) -0 ,pow(c1-4 ,2)-0 from ct4 ") + tdSql.checkData(0, 0, None) + tdSql.checkData(0, 1, None) + tdSql.checkData(0, 2, None) + tdSql.checkData(1, 0, 8) + tdSql.checkData(1, 1, 64.000000000) + tdSql.checkData(1, 2, 16.000000000) + + tdSql.query(" select c1, pow(c1 ,2) -0 ,pow(c1-0.1 ,2)-0.1 from ct4") + tdSql.checkData(0, 0, None) + tdSql.checkData(0, 1, None) + tdSql.checkData(0, 2, None) + tdSql.checkData(1, 0, 8) + tdSql.checkData(1, 1, 64.000000000) + tdSql.checkData(1, 2, 62.310000000) + + tdSql.query("select c1, pow(c1, -10), c2, pow(c2, -10), c3, pow(c3, -10) from ct1") + + def test_big_number(self): + + tdSql.query("select c1, pow(c1, 100000000) from ct1") # bigint to double data overflow + tdSql.checkData(0, 1, None) + tdSql.checkData(1, 1, None) + tdSql.checkData(4, 1, 0.000000000) + + + tdSql.query("select c1, pow(c1, 10000000000000) from ct1") # bigint to double data overflow + tdSql.checkData(0, 1, None) + tdSql.checkData(1, 1, None) + tdSql.checkData(4, 1, 0.000000000) + + tdSql.query("select c1, pow(c1, 10000000000000000000000000) from ct1") # bigint to double data overflow + tdSql.query("select c1, pow(c1, 10000000000000000000000000.0) from ct1") # 10000000000000000000000000.0 is a double value + tdSql.checkData(0, 1, None) + tdSql.checkData(1, 1, None) + tdSql.checkData(4, 1, 0.000000000) + + tdSql.query("select c1, pow(c1, 10000000000000000000000000000000000) from ct1") # bigint to double data overflow + tdSql.query("select c1, pow(c1, 10000000000000000000000000000000000.0) from ct1") # 10000000000000000000000000.0 is a double value + tdSql.checkData(0, 1, None) + tdSql.checkData(1, 1, None) + tdSql.checkData(4, 1, 0.000000000) + + tdSql.query("select c1, pow(c1, 10000000000000000000000000000000000000000) from ct1") # bigint to double data overflow + tdSql.query("select c1, pow(c1, 10000000000000000000000000000000000000000.0) from ct1") # 10000000000000000000000000.0 is a double value + tdSql.checkData(0, 1, None) + tdSql.checkData(1, 1, None) + tdSql.checkData(4, 1, 0.000000000) + + tdSql.query("select c1, pow(c1, 10000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000) from ct1") # bigint to double data overflow + + def pow_base_test(self): + + # base is an regular number ,int or double + tdSql.query("select c1, pow(c1, 2) from ct1") + tdSql.checkData(0, 1,64.000000000) + tdSql.query("select c1, pow(c1, 2.0) from ct1") + tdSql.checkData(0, 1, 64.000000000) + + tdSql.query("select c1, pow(1, 2.0) from ct1") + tdSql.checkData(0, 1, 1.000000000) + tdSql.checkRows(13) + + + # # bug for compute in functions + # tdSql.query("select c1, abs(1/0) from ct1") + # tdSql.checkData(0, 0, 8) + # tdSql.checkData(0, 1, 1) + + tdSql.query("select c1, pow(1, 2.0) from ct1") + tdSql.checkData(0, 1, 1.000000000) + tdSql.checkRows(13) + + # two cols start pow(x,y) + tdSql.query("select c1,c2, pow(c1,c2) from ct1") + tdSql.checkData(0, 2, None) + tdSql.checkData(1, 2, None) + tdSql.checkData(4, 2, 1.000000000) + + tdSql.query("select c1,c2, pow(c2,c1) from ct1") + tdSql.checkData(0, 2, 3897131646727578700481513520437089271808.000000000) + tdSql.checkData(1, 2, 17217033054561120738612297152331776.000000000) + tdSql.checkData(4, 2, 1.000000000) + + tdSql.query("select c1, pow(2.0 , c1) from ct1") + tdSql.checkData(0, 1, 256.000000000) + tdSql.checkData(1, 1, 128.000000000) + tdSql.checkData(4, 1, 1.000000000) + + tdSql.query("select c1, pow(2.0 , c1) from ct1") + tdSql.checkData(0, 1, 256.000000000) + tdSql.checkData(1, 1, 128.000000000) + tdSql.checkData(4, 1, 1.000000000) + + def abs_func_filter(self): + tdSql.execute("use db") + tdSql.query("select c1, abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(pow(c1,2)-0.5) from ct4 where c1>5 ") + tdSql.checkRows(3) + tdSql.checkData(0,0,8) + tdSql.checkData(0,1,8.000000000) + tdSql.checkData(0,2,8.000000000) + tdSql.checkData(0,3,7.900000000) + tdSql.checkData(0,4,64.000000000) + + tdSql.query("select c1, abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(pow(c1,2)-0.5) from ct4 where c1=5 ") + tdSql.checkRows(1) + tdSql.checkData(0,0,5) + tdSql.checkData(0,1,5.000000000) + tdSql.checkData(0,2,5.000000000) + tdSql.checkData(0,3,4.900000000) + tdSql.checkData(0,4,25.000000000) + + tdSql.query("select c1, abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(pow(c1,2)-0.5) from ct4 where c1=5 ") + tdSql.checkRows(1) + tdSql.checkData(0,0,5) + tdSql.checkData(0,1,5.000000000) + tdSql.checkData(0,2,5.000000000) + tdSql.checkData(0,3,4.900000000) + tdSql.checkData(0,4,25.000000000) + + tdSql.query("select c1,c2 , abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(pow(c1,2)-0.5) from ct4 where c10 and i%30000 == 0: + tdSql.execute(sql) + sql = pre_insert + # end sql + if sql != pre_insert: + tdSql.execute(sql) + + tdLog.debug("INSERT TABLE DATA ............ [OK]") + return + + def insert_data1(self, tbname, ts_start, count): + pre_insert = "insert into %s values"%tbname + sql = pre_insert + tdLog.debug("doing insert table %s rows=%d ..."%(tbname, count)) + for i in range(count): + sql += " (%d,%d,%d)"%(ts_start + i*1000, i , i+1) + if i >0 and i%30000 == 0: + tdSql.execute(sql) + sql = pre_insert + # end sql + if sql != pre_insert: + tdSql.execute(sql) + + tdLog.debug("INSERT TABLE DATA ............ [OK]") + return + + # test case1 base + # def test_case1(self): + # # + # # limit base function + # # + # # base no where + # sql = "select * from t1 limit 10" + # tdSql.waitedQuery(sql, 10, WAITS) + + +# +# add case with filename +# +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/system-test/2-query/sqrt.py b/tests/system-test/2-query/sqrt.py new file mode 100644 index 0000000000..28e869e044 --- /dev/null +++ b/tests/system-test/2-query/sqrt.py @@ -0,0 +1,551 @@ +import taos +import sys +import datetime +import inspect +import math +from util.log import * +from util.sql import * +from util.cases import * + + +class TDTestCase: + updatecfgDict = {'debugFlag': 143 ,"cDebugFlag":143,"uDebugFlag":143 ,"rpcDebugFlag":143 , "tmrDebugFlag":143 , + "jniDebugFlag":143 ,"simDebugFlag":143,"dDebugFlag":143, "dDebugFlag":143,"vDebugFlag":143,"mDebugFlag":143,"qDebugFlag":143, + "wDebugFlag":143,"sDebugFlag":143,"tsdbDebugFlag":143,"tqDebugFlag":143 ,"fsDebugFlag":143 ,"fnDebugFlag":143} + def init(self, conn, powSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + + def prepare_datas(self): + tdSql.execute( + '''create table stb1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + tags (t1 int) + ''' + ) + + tdSql.execute( + ''' + create table t1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + ''' + ) + for i in range(4): + tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )') + + for i in range(9): + tdSql.execute( + f"insert into ct1 values ( now()-{i*10}s, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + ) + tdSql.execute( + f"insert into ct4 values ( now()-{i*90}d, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + ) + tdSql.execute("insert into ct1 values (now()-45s, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a )") + tdSql.execute("insert into ct1 values (now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute("insert into ct1 values (now()+15s, 9, -99999, -999, -99, -9.99, NULL, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute("insert into ct1 values (now()+20s, 9, -99999, -999, NULL, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + + tdSql.execute("insert into ct4 values (now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute("insert into ct4 values (now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute("insert into ct4 values (now()+90d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + + tdSql.execute( + f'''insert into t1 values + ( '2020-04-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2020-10-21 01:01:01.000', 1, 11111, 111, 11, 1.11, 11.11, 1, "binary1", "nchar1", now()+1a ) + ( '2020-12-31 01:01:01.000', 2, 22222, 222, 22, 2.22, 22.22, 0, "binary2", "nchar2", now()+2a ) + ( '2021-01-01 01:01:06.000', 3, 33333, 333, 33, 3.33, 33.33, 0, "binary3", "nchar3", now()+3a ) + ( '2021-05-07 01:01:10.000', 4, 44444, 444, 44, 4.44, 44.44, 1, "binary4", "nchar4", now()+4a ) + ( '2021-07-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2021-09-30 01:01:16.000', 5, 55555, 555, 55, 5.55, 55.55, 0, "binary5", "nchar5", now()+5a ) + ( '2022-02-01 01:01:20.000', 6, 66666, 666, 66, 6.66, 66.66, 1, "binary6", "nchar6", now()+6a ) + ( '2022-10-28 01:01:26.000', 7, 00000, 000, 00, 0.00, 00.00, 1, "binary7", "nchar7", "1970-01-01 08:00:00.000" ) + ( '2022-12-01 01:01:30.000', 8, -88888, -888, -88, -8.88, -88.88, 0, "binary8", "nchar8", "1969-01-01 01:00:00.000" ) + ( '2022-12-31 01:01:36.000', 9, -99999999999999999, -999, -99, -9.99, -999999999999999999999.99, 1, "binary9", "nchar9", "1900-01-01 00:00:00.000" ) + ( '2023-02-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ''' + ) + + def check_result_auto_sqrt(self ,origin_query , pow_query): + + pow_result = tdSql.getResult(pow_query) + origin_result = tdSql.getResult(origin_query) + + auto_result =[] + + for row in origin_result: + row_check = [] + for elem in row: + if elem == None: + elem = None + elif elem < 0: + elem = None + else: + elem = math.sqrt(elem) + row_check.append(elem) + auto_result.append(row_check) + + check_status = True + + for row_index , row in enumerate(pow_result): + for col_index , elem in enumerate(row): + if auto_result[row_index][col_index] == None and not (auto_result[row_index][col_index] == None and elem == None): + check_status = False + elif auto_result[row_index][col_index] != None and (auto_result[row_index][col_index] - elem > 0.00000001): + check_status = False + else: + pass + if not check_status: + tdLog.notice("sqrt function value has not as expected , sql is \"%s\" "%pow_query ) + sys.exit(1) + else: + tdLog.info("sqrt value check pass , it work as expected ,sql is \"%s\" "%pow_query ) + + def test_errors(self): + error_sql_lists = [ + "select sqrt from t1", + # "select sqrt(-+--+c1 ) from t1", + # "select +-sqrt(c1) from t1", + # "select ++-sqrt(c1) from t1", + # "select ++--sqrt(c1) from t1", + # "select - -sqrt(c1)*0 from t1", + # "select sqrt(tbname+1) from t1 ", + "select sqrt(123--123)==1 from t1", + "select sqrt(c1) as 'd1' from t1", + "select sqrt(c1 ,c2) from t1", + "select sqrt(c1 ,NULL ) from t1", + "select sqrt(,) from t1;", + "select sqrt(sqrt(c1) ab from t1)", + "select sqrt(c1 ) as int from t1", + "select sqrt from stb1", + # "select sqrt(-+--+c1) from stb1", + # "select +-sqrt(c1) from stb1", + # "select ++-sqrt(c1) from stb1", + # "select ++--sqrt(c1) from stb1", + # "select - -sqrt(c1)*0 from stb1", + # "select sqrt(tbname+1) from stb1 ", + "select sqrt(123--123)==1 from stb1", + "select sqrt(c1) as 'd1' from stb1", + "select sqrt(c1 ,c2 ) from stb1", + "select sqrt(c1 ,NULL) from stb1", + "select sqrt(,) from stb1;", + "select sqrt(sqrt(c1) ab from stb1)", + "select sqrt(c1) as int from stb1" + ] + for error_sql in error_sql_lists: + tdSql.error(error_sql) + + def support_types(self): + type_error_sql_lists = [ + "select sqrt(ts) from t1" , + "select sqrt(c7) from t1", + "select sqrt(c8) from t1", + "select sqrt(c9) from t1", + "select sqrt(ts) from ct1" , + "select sqrt(c7) from ct1", + "select sqrt(c8) from ct1", + "select sqrt(c9) from ct1", + "select sqrt(ts) from ct3" , + "select sqrt(c7) from ct3", + "select sqrt(c8) from ct3", + "select sqrt(c9) from ct3", + "select sqrt(ts) from ct4" , + "select sqrt(c7) from ct4", + "select sqrt(c8) from ct4", + "select sqrt(c9) from ct4", + "select sqrt(ts) from stb1" , + "select sqrt(c7) from stb1", + "select sqrt(c8) from stb1", + "select sqrt(c9) from stb1" , + + "select sqrt(ts) from stbbb1" , + "select sqrt(c7) from stbbb1", + + "select sqrt(ts) from tbname", + "select sqrt(c9) from tbname" + + ] + + for type_sql in type_error_sql_lists: + tdSql.error(type_sql) + + + type_sql_lists = [ + "select sqrt(c1) from t1", + "select sqrt(c2) from t1", + "select sqrt(c3) from t1", + "select sqrt(c4) from t1", + "select sqrt(c5) from t1", + "select sqrt(c6) from t1", + + "select sqrt(c1) from ct1", + "select sqrt(c2) from ct1", + "select sqrt(c3) from ct1", + "select sqrt(c4) from ct1", + "select sqrt(c5) from ct1", + "select sqrt(c6) from ct1", + + "select sqrt(c1) from ct3", + "select sqrt(c2) from ct3", + "select sqrt(c3) from ct3", + "select sqrt(c4) from ct3", + "select sqrt(c5) from ct3", + "select sqrt(c6) from ct3", + + "select sqrt(c1) from stb1", + "select sqrt(c2) from stb1", + "select sqrt(c3) from stb1", + "select sqrt(c4) from stb1", + "select sqrt(c5) from stb1", + "select sqrt(c6) from stb1", + + "select sqrt(c6) as alisb from stb1", + "select sqrt(c6) alisb from stb1", + ] + + for type_sql in type_sql_lists: + tdSql.query(type_sql) + + def basic_sqrt_function(self): + + # basic query + tdSql.query("select c1 from ct3") + tdSql.checkRows(0) + tdSql.query("select c1 from t1") + tdSql.checkRows(12) + tdSql.query("select c1 from stb1") + tdSql.checkRows(25) + + # used for empty table , ct3 is empty + tdSql.query("select sqrt(c1) from ct3") + tdSql.checkRows(0) + tdSql.query("select sqrt(c2) from ct3") + tdSql.checkRows(0) + tdSql.query("select sqrt(c3) from ct3") + tdSql.checkRows(0) + tdSql.query("select sqrt(c4) from ct3") + tdSql.checkRows(0) + tdSql.query("select sqrt(c5) from ct3") + tdSql.checkRows(0) + tdSql.query("select sqrt(c6) from ct3") + tdSql.checkRows(0) + + + # # used for regular table + tdSql.query("select sqrt(c1) from t1") + tdSql.checkData(0, 0, None) + tdSql.checkData(1 , 0, 1.000000000) + tdSql.checkData(3 , 0, 1.732050808) + tdSql.checkData(5 , 0, None) + + tdSql.query("select c1, c2, c3 , c4, c5 from t1") + tdSql.checkData(1, 4, 1.11000) + tdSql.checkData(3, 3, 33) + tdSql.checkData(5, 4, None) + + tdSql.query("select ts,c1, c2, c3 , c4, c5 from t1") + tdSql.checkData(1, 5, 1.11000) + tdSql.checkData(3, 4, 33) + tdSql.checkData(5, 5, None) + + self.check_result_auto_sqrt( "select abs(c1), abs(c2), abs(c3) , abs(c4), abs(c5) from t1", "select sqrt(abs(c1)), sqrt(abs(c2)) ,sqrt(abs(c3)), sqrt(abs(c4)), sqrt(abs(c5)) from t1") + + # used for sub table + tdSql.query("select c2 ,sqrt(c2) from ct1") + tdSql.checkData(0, 1, 298.140906284) + tdSql.checkData(1 , 1, 278.885281074) + tdSql.checkData(3 , 1, 235.701081881) + tdSql.checkData(4 , 1, 0.000000000) + + tdSql.query("select c1, c5 ,sqrt(c5) from ct4") + tdSql.checkData(0 , 2, None) + tdSql.checkData(1 , 2, 2.979932904) + tdSql.checkData(2 , 2, 2.787471970) + tdSql.checkData(3 , 2, 2.580697551) + tdSql.checkData(5 , 2, None) + + self.check_result_auto_sqrt( "select c1, c2, c3 , c4, c5 from ct1", "select sqrt(c1), sqrt(c2) ,sqrt(c3), sqrt(c4), sqrt(c5) from ct1") + + # nest query for sqrt functions + tdSql.query("select c4 , sqrt(c4) ,sqrt(sqrt(c4)) , sqrt(sqrt(sqrt(c4))) from ct1;") + tdSql.checkData(0 , 0 , 88) + tdSql.checkData(0 , 1 , 9.380831520) + tdSql.checkData(0 , 2 , 3.062814314) + tdSql.checkData(0 , 3 , 1.750089802) + + tdSql.checkData(1 , 0 , 77) + tdSql.checkData(1 , 1 , 8.774964387) + tdSql.checkData(1 , 2 , 2.962256638) + tdSql.checkData(1 , 3 , 1.721120750) + + tdSql.checkData(11 , 0 , -99) + tdSql.checkData(11 , 1 , None) + tdSql.checkData(11 , 2 , None) + tdSql.checkData(11 , 3 , None) + + # used for stable table + + tdSql.query("select sqrt(c1) from stb1") + tdSql.checkRows(25) + + + # used for not exists table + tdSql.error("select sqrt(c1) from stbbb1") + tdSql.error("select sqrt(c1) from tbname") + tdSql.error("select sqrt(c1) from ct5") + + # mix with common col + tdSql.query("select c1, sqrt(c1) from ct1") + tdSql.checkData(0 , 0 ,8) + tdSql.checkData(0 , 1 ,2.828427125) + tdSql.checkData(4 , 0 ,0) + tdSql.checkData(4 , 1 ,0.000000000) + tdSql.query("select c2, sqrt(c2) from ct4") + tdSql.checkData(0 , 0 , None) + tdSql.checkData(0 , 1 ,None) + tdSql.checkData(4 , 0 ,55555) + tdSql.checkData(4 , 1 ,235.701081881) + tdSql.checkData(5 , 0 ,None) + tdSql.checkData(5 , 1 ,None) + + # mix with common functions + tdSql.query("select c1, sqrt(c1),sqrt(c1), sqrt(sqrt(c1)) from ct4 ") + tdSql.checkData(0 , 0 ,None) + tdSql.checkData(0 , 1 ,None) + tdSql.checkData(0 , 2 ,None) + tdSql.checkData(0 , 3 ,None) + + tdSql.checkData(3 , 0 , 6) + tdSql.checkData(3 , 1 ,2.449489743) + tdSql.checkData(3 , 2 ,2.449489743) + tdSql.checkData(3 , 3 ,1.565084580) + + tdSql.query("select c1, sqrt(c1),c5, floor(c5) from stb1 ") + + # # mix with agg functions , not support + tdSql.error("select c1, sqrt(c1),c5, count(c5) from stb1 ") + tdSql.error("select c1, sqrt(c1),c5, count(c5) from ct1 ") + tdSql.error("select sqrt(c1), count(c5) from stb1 ") + tdSql.error("select sqrt(c1), count(c5) from ct1 ") + tdSql.error("select c1, count(c5) from ct1 ") + tdSql.error("select c1, count(c5) from stb1 ") + + # agg functions mix with agg functions + + tdSql.query("select max(c5), count(c5) from stb1") + tdSql.query("select max(c5), count(c5) from ct1") + + + # bug fix for count + tdSql.query("select count(c1) from ct4 ") + tdSql.checkData(0,0,9) + tdSql.query("select count(*) from ct4 ") + tdSql.checkData(0,0,12) + tdSql.query("select count(c1) from stb1 ") + tdSql.checkData(0,0,22) + tdSql.query("select count(*) from stb1 ") + tdSql.checkData(0,0,25) + + # # bug fix for compute + tdSql.query("select c1, sqrt(c1) -0 ,sqrt(c1-4)-0 from ct4 ") + tdSql.checkData(0, 0, None) + tdSql.checkData(0, 1, None) + tdSql.checkData(0, 2, None) + tdSql.checkData(1, 0, 8) + tdSql.checkData(1, 1, 2.828427125) + tdSql.checkData(1, 2, 2.000000000) + + tdSql.query(" select c1, sqrt(c1) -0 ,sqrt(c1-0.1)-0.1 from ct4") + tdSql.checkData(0, 0, None) + tdSql.checkData(0, 1, None) + tdSql.checkData(0, 2, None) + tdSql.checkData(1, 0, 8) + tdSql.checkData(1, 1, 2.828427125) + tdSql.checkData(1, 2, 2.710693865) + + tdSql.query("select c1, sqrt(c1), c2, sqrt(c2), c3, sqrt(c3) from ct1") + + def test_big_number(self): + + tdSql.query("select c1, sqrt(100000000) from ct1") # bigint to double data overflow + tdSql.checkData(4, 1, 10000.000000000) + + + tdSql.query("select c1, sqrt(10000000000000) from ct1") # bigint to double data overflow + tdSql.checkData(4, 1, 3162277.660168380) + + tdSql.query("select c1, sqrt(c1) + sqrt(10000000000000000000000000) from ct1") # bigint to double data overflow + tdSql.query("select c1, sqrt(c1) + sqrt(10000000000000000000000000.0) from ct1") # 10000000000000000000000000.0 is a double value + tdSql.checkData(1, 1, 3162277660171.025390625) + + tdSql.query("select c1, sqrt(10000000000000000000000000000000000) from ct1") # bigint to double data overflow + tdSql.query("select c1, sqrt(10000000000000000000000000000000000.0) from ct1") # 10000000000000000000000000.0 is a double value + tdSql.checkData(4, 1, 100000000000000000.000000000) + + tdSql.query("select c1, sqrt(10000000000000000000000000000000000000000) from ct1") # bigint to double data overflow + tdSql.query("select c1, sqrt(10000000000000000000000000000000000000000.0) from ct1") # 10000000000000000000000000.0 is a double value + + tdSql.checkData(4, 1, 100000000000000000000.000000000) + + tdSql.query("select c1, sqrt(10000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000) from ct1") # bigint to double data overflow + + def pow_base_test(self): + + # base is an regular number ,int or double + tdSql.query("select c1, sqrt(c1) from ct1") + tdSql.checkData(0, 1,2.828427125) + tdSql.checkRows(13) + + # # bug for compute in functions + # tdSql.query("select c1, abs(1/0) from ct1") + # tdSql.checkData(0, 0, 8) + # tdSql.checkData(0, 1, 1) + + tdSql.query("select c1, sqrt(1) from ct1") + tdSql.checkData(0, 1, 1.000000000) + tdSql.checkRows(13) + + # two cols start sqrt(x,y) + tdSql.query("select c1,c2, sqrt(c2) from ct1") + tdSql.checkData(0, 2, 298.140906284) + tdSql.checkData(1, 2, 278.885281074) + tdSql.checkData(4, 2, 0.000000000) + + def abs_func_filter(self): + tdSql.execute("use db") + tdSql.query("select c1, abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(sqrt(c1)-0.5) from ct4 where c1>5 ") + tdSql.checkRows(3) + tdSql.checkData(0,0,8) + tdSql.checkData(0,1,8.000000000) + tdSql.checkData(0,2,8.000000000) + tdSql.checkData(0,3,7.900000000) + tdSql.checkData(0,4,3.000000000) + + tdSql.query("select c1, abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(sqrt(c1)-0.5) from ct4 where c1=5 ") + tdSql.checkRows(1) + tdSql.checkData(0,0,5) + tdSql.checkData(0,1,5.000000000) + tdSql.checkData(0,2,5.000000000) + tdSql.checkData(0,3,4.900000000) + tdSql.checkData(0,4,2.000000000) + + tdSql.query("select c1, abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(sqrt(c1)-0.5) from ct4 where c1=5 ") + tdSql.checkRows(1) + tdSql.checkData(0,0,5) + tdSql.checkData(0,1,5.000000000) + tdSql.checkData(0,2,5.000000000) + tdSql.checkData(0,3,4.900000000) + tdSql.checkData(0,4,2.000000000) + + tdSql.query("select c1,c2 , abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(sqrt(c1)-0.5) from ct4 where c1=sqrt(c1) limit 1 ") + tdSql.checkRows(1) + tdSql.checkData(0,0,1) + tdSql.checkData(0,1,11111) + tdSql.checkData(0,2,1.000000000) + tdSql.checkData(0,3,1.000000000) + tdSql.checkData(0,4,0.900000000) + tdSql.checkData(0,5,1.000000000) + + def pow_Arithmetic(self): + pass + + def check_boundary_values(self): + + tdSql.execute("drop database if exists bound_test") + tdSql.execute("create database if not exists bound_test") + time.sleep(3) + tdSql.execute("use bound_test") + tdSql.execute( + "create table stb_bound (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(32),c9 nchar(32), c10 timestamp) tags (t1 int);" + ) + tdSql.execute(f'create table sub1_bound using stb_bound tags ( 1 )') + tdSql.execute( + f"insert into sub1_bound values ( now()-1s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + tdSql.execute( + f"insert into sub1_bound values ( now()-1s, -2147483647, -9223372036854775807, -32767, -127, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + tdSql.execute( + f"insert into sub1_bound values ( now(), 2147483646, 9223372036854775806, 32766, 126, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + tdSql.execute( + f"insert into sub1_bound values ( now(), -2147483646, -9223372036854775806, -32766, -126, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + tdSql.error( + f"insert into sub1_bound values ( now()+1s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + self.check_result_auto_sqrt( "select abs(c1), abs(c2), abs(c3) , abs(c4), abs(c5) from sub1_bound ", "select sqrt(abs(c1)), sqrt(abs(c2)) ,sqrt(abs(c3)), sqrt(abs(c4)), sqrt(abs(c5)) from sub1_bound") + + self.check_result_auto_sqrt( "select c1, c2, c3 , c3, c2 ,c1 from sub1_bound ", "select sqrt(c1), sqrt(c2) ,sqrt(c3), sqrt(c3), sqrt(c2) ,sqrt(c1) from sub1_bound") + + self.check_result_auto_sqrt("select abs(abs(abs(abs(abs(abs(abs(abs(abs(c1))))))))) nest_col_func from sub1_bound" , "select sqrt(abs(c1)) from sub1_bound" ) + + # check basic elem for table per row + tdSql.query("select sqrt(abs(c1)) ,sqrt(abs(c2)) , sqrt(abs(c3)) , sqrt(abs(c4)), sqrt(abs(c5)), sqrt(abs(c6)) from sub1_bound ") + tdSql.checkData(0,0,math.sqrt(2147483647)) + tdSql.checkData(0,1,math.sqrt(9223372036854775807)) + tdSql.checkData(0,2,math.sqrt(32767)) + tdSql.checkData(0,3,math.sqrt(127)) + tdSql.checkData(0,4,math.sqrt(339999995214436424907732413799364296704.00000)) + tdSql.checkData(1,0,math.sqrt(2147483647)) + tdSql.checkData(1,1,math.sqrt(9223372036854775807)) + tdSql.checkData(1,2,math.sqrt(32767)) + tdSql.checkData(1,3,math.sqrt(127)) + tdSql.checkData(1,4,math.sqrt(339999995214436424907732413799364296704.00000)) + tdSql.checkData(3,0,math.sqrt(2147483646)) + tdSql.checkData(3,1,math.sqrt(9223372036854775806)) + tdSql.checkData(3,2,math.sqrt(32766)) + tdSql.checkData(3,3,math.sqrt(126)) + tdSql.checkData(3,4,math.sqrt(339999995214436424907732413799364296704.00000)) + + # check + - * / in functions + tdSql.query("select sqrt(abs(c1+1)) ,sqrt(abs(c2)) , sqrt(abs(c3*1)) , sqrt(abs(c4/2)), sqrt(abs(c5))/2, sqrt(abs(c6)) from sub1_bound ") + tdSql.checkData(0,0,math.sqrt(2147483648.000000000)) + tdSql.checkData(0,1,math.sqrt(9223372036854775807)) + tdSql.checkData(0,2,math.sqrt(32767.000000000)) + tdSql.checkData(0,3,math.sqrt(63.500000000)) + + + def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring + tdSql.prepare() + + tdLog.printNoPrefix("==========step1:create table ==============") + + self.prepare_datas() + + tdLog.printNoPrefix("==========step2:test errors ==============") + + self.test_errors() + + tdLog.printNoPrefix("==========step3:support types ============") + + self.support_types() + + tdLog.printNoPrefix("==========step4: sqrt basic query ============") + + self.basic_sqrt_function() + + tdLog.printNoPrefix("==========step5: big number sqrt query ============") + + self.test_big_number() + + tdLog.printNoPrefix("==========step6: base number for sqrt query ============") + + self.pow_base_test() + + tdLog.printNoPrefix("==========step7: sqrt boundary query ============") + + self.check_boundary_values() + + tdLog.printNoPrefix("==========step8: sqrt filter query ============") + + self.abs_func_filter() + + + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 0b7a71bbe4..51f1649cc8 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -22,4 +22,6 @@ python3 ./test.py -f 2-query/abs.py python3 ./test.py -f 2-query/ceil.py python3 ./test.py -f 2-query/floor.py python3 ./test.py -f 2-query/round.py -python3 ./test.py -f 2-query/log.py \ No newline at end of file +python3 ./test.py -f 2-query/log.py +python3 ./test.py -f 2-query/pow.py +python3 ./test.py -f 2-query/sqrt.py diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index befcfc0731..08fe2cbc32 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -141,6 +141,9 @@ int32_t shellRunCommand(char *command) { *p++ = '\\'; } break; + default: + *p++ = '\\'; + break; } *p++ = c; esc = false; diff --git a/tools/shell/src/shellNettest.c b/tools/shell/src/shellNettest.c index c8ec31c48b..345b85d896 100644 --- a/tools/shell/src/shellNettest.c +++ b/tools/shell/src/shellNettest.c @@ -21,7 +21,7 @@ static void shellWorkAsClient() { SRpcInit rpcInit = {0}; SEpSet epSet = {.inUse = 0, .numOfEps = 1}; SRpcMsg rpcRsp = {0}; - void *clientRpc = NULL; + void * clientRpc = NULL; char pass[TSDB_PASSWORD_LEN + 1] = {0}; taosEncryptPass_c((uint8_t *)("_pwd"), strlen("_pwd"), pass); @@ -116,6 +116,7 @@ static void shellWorkAsServer() { } SRpcInit rpcInit = {0}; + memcpy(rpcInit.localFqdn, tsLocalFqdn, strlen(tsLocalFqdn)); rpcInit.localPort = pArgs->port; rpcInit.label = "CHK"; rpcInit.numOfThreads = tsNumOfRpcThreads; @@ -126,7 +127,7 @@ static void shellWorkAsServer() { void *serverRpc = rpcOpen(&rpcInit); if (serverRpc == NULL) { - printf("failed to init net test server since %s", terrstr()); + printf("failed to init net test server since %s\n", terrstr()); } else { printf("network test server is initialized, port:%u\n", pArgs->port); taosSetSignal(SIGTERM, shellNettestHandler); From e45e953e44e512d4240ec38b4ca76e6d051f37f2 Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Fri, 6 May 2022 19:52:09 +0800 Subject: [PATCH 33/83] add test case for sin functions --- tests/system-test/2-query/sin.py | 519 +++++++++++++++++++++++++++++++ tests/system-test/fulltest.sh | 1 + 2 files changed, 520 insertions(+) create mode 100644 tests/system-test/2-query/sin.py diff --git a/tests/system-test/2-query/sin.py b/tests/system-test/2-query/sin.py new file mode 100644 index 0000000000..c2c5b8794e --- /dev/null +++ b/tests/system-test/2-query/sin.py @@ -0,0 +1,519 @@ +import taos +import sys +import datetime +import inspect +import math +from util.log import * +from util.sql import * +from util.cases import * + + +class TDTestCase: + updatecfgDict = {'debugFlag': 143 ,"cDebugFlag":143,"uDebugFlag":143 ,"rpcDebugFlag":143 , "tmrDebugFlag":143 , + "jniDebugFlag":143 ,"simDebugFlag":143,"dDebugFlag":143, "dDebugFlag":143,"vDebugFlag":143,"mDebugFlag":143,"qDebugFlag":143, + "wDebugFlag":143,"sDebugFlag":143,"tsdbDebugFlag":143,"tqDebugFlag":143 ,"fsDebugFlag":143 ,"fnDebugFlag":143} + def init(self, conn, powSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + + def prepare_datas(self): + tdSql.execute( + '''create table stb1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + tags (t1 int) + ''' + ) + + tdSql.execute( + ''' + create table t1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + ''' + ) + for i in range(4): + tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )') + + for i in range(9): + tdSql.execute( + f"insert into ct1 values ( now()-{i*10}s, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + ) + tdSql.execute( + f"insert into ct4 values ( now()-{i*90}d, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + ) + tdSql.execute("insert into ct1 values (now()-45s, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a )") + tdSql.execute("insert into ct1 values (now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute("insert into ct1 values (now()+15s, 9, -99999, -999, -99, -9.99, NULL, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute("insert into ct1 values (now()+20s, 9, -99999, -999, NULL, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + + tdSql.execute("insert into ct4 values (now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute("insert into ct4 values (now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute("insert into ct4 values (now()+90d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + + tdSql.execute( + f'''insert into t1 values + ( '2020-04-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2020-10-21 01:01:01.000', 1, 11111, 111, 11, 1.11, 11.11, 1, "binary1", "nchar1", now()+1a ) + ( '2020-12-31 01:01:01.000', 2, 22222, 222, 22, 2.22, 22.22, 0, "binary2", "nchar2", now()+2a ) + ( '2021-01-01 01:01:06.000', 3, 33333, 333, 33, 3.33, 33.33, 0, "binary3", "nchar3", now()+3a ) + ( '2021-05-07 01:01:10.000', 4, 44444, 444, 44, 4.44, 44.44, 1, "binary4", "nchar4", now()+4a ) + ( '2021-07-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2021-09-30 01:01:16.000', 5, 55555, 555, 55, 5.55, 55.55, 0, "binary5", "nchar5", now()+5a ) + ( '2022-02-01 01:01:20.000', 6, 66666, 666, 66, 6.66, 66.66, 1, "binary6", "nchar6", now()+6a ) + ( '2022-10-28 01:01:26.000', 7, 00000, 000, 00, 0.00, 00.00, 1, "binary7", "nchar7", "1970-01-01 08:00:00.000" ) + ( '2022-12-01 01:01:30.000', 8, -88888, -888, -88, -8.88, -88.88, 0, "binary8", "nchar8", "1969-01-01 01:00:00.000" ) + ( '2022-12-31 01:01:36.000', 9, -99999999999999999, -999, -99, -9.99, -999999999999999999999.99, 1, "binary9", "nchar9", "1900-01-01 00:00:00.000" ) + ( '2023-02-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ''' + ) + + def check_result_auto_sin(self ,origin_query , pow_query): + + pow_result = tdSql.getResult(pow_query) + origin_result = tdSql.getResult(origin_query) + + auto_result =[] + + for row in origin_result: + row_check = [] + for elem in row: + if elem == None: + elem = None + else: + elem = math.sin(elem) + row_check.append(elem) + auto_result.append(row_check) + + check_status = True + + for row_index , row in enumerate(pow_result): + for col_index , elem in enumerate(row): + if auto_result[row_index][col_index] == None and not (auto_result[row_index][col_index] == None and elem == None): + check_status = False + elif auto_result[row_index][col_index] != None and (auto_result[row_index][col_index] - elem > 0.00000001): + check_status = False + else: + pass + if not check_status: + tdLog.notice("sin function value has not as expected , sql is \"%s\" "%pow_query ) + sys.exit(1) + else: + tdLog.info("sin value check pass , it work as expected ,sql is \"%s\" "%pow_query ) + + def test_errors(self): + error_sql_lists = [ + "select sin from t1", + # "select sin(-+--+c1 ) from t1", + # "select +-sin(c1) from t1", + # "select ++-sin(c1) from t1", + # "select ++--sin(c1) from t1", + # "select - -sin(c1)*0 from t1", + # "select sin(tbname+1) from t1 ", + "select sin(123--123)==1 from t1", + "select sin(c1) as 'd1' from t1", + "select sin(c1 ,c2) from t1", + "select sin(c1 ,NULL ) from t1", + "select sin(,) from t1;", + "select sin(sin(c1) ab from t1)", + "select sin(c1 ) as int from t1", + "select sin from stb1", + # "select sin(-+--+c1) from stb1", + # "select +-sin(c1) from stb1", + # "select ++-sin(c1) from stb1", + # "select ++--sin(c1) from stb1", + # "select - -sin(c1)*0 from stb1", + # "select sin(tbname+1) from stb1 ", + "select sin(123--123)==1 from stb1", + "select sin(c1) as 'd1' from stb1", + "select sin(c1 ,c2 ) from stb1", + "select sin(c1 ,NULL) from stb1", + "select sin(,) from stb1;", + "select sin(sin(c1) ab from stb1)", + "select sin(c1) as int from stb1" + ] + for error_sql in error_sql_lists: + tdSql.error(error_sql) + + def support_types(self): + type_error_sql_lists = [ + "select sin(ts) from t1" , + "select sin(c7) from t1", + "select sin(c8) from t1", + "select sin(c9) from t1", + "select sin(ts) from ct1" , + "select sin(c7) from ct1", + "select sin(c8) from ct1", + "select sin(c9) from ct1", + "select sin(ts) from ct3" , + "select sin(c7) from ct3", + "select sin(c8) from ct3", + "select sin(c9) from ct3", + "select sin(ts) from ct4" , + "select sin(c7) from ct4", + "select sin(c8) from ct4", + "select sin(c9) from ct4", + "select sin(ts) from stb1" , + "select sin(c7) from stb1", + "select sin(c8) from stb1", + "select sin(c9) from stb1" , + + "select sin(ts) from stbbb1" , + "select sin(c7) from stbbb1", + + "select sin(ts) from tbname", + "select sin(c9) from tbname" + + ] + + for type_sql in type_error_sql_lists: + tdSql.error(type_sql) + + + type_sql_lists = [ + "select sin(c1) from t1", + "select sin(c2) from t1", + "select sin(c3) from t1", + "select sin(c4) from t1", + "select sin(c5) from t1", + "select sin(c6) from t1", + + "select sin(c1) from ct1", + "select sin(c2) from ct1", + "select sin(c3) from ct1", + "select sin(c4) from ct1", + "select sin(c5) from ct1", + "select sin(c6) from ct1", + + "select sin(c1) from ct3", + "select sin(c2) from ct3", + "select sin(c3) from ct3", + "select sin(c4) from ct3", + "select sin(c5) from ct3", + "select sin(c6) from ct3", + + "select sin(c1) from stb1", + "select sin(c2) from stb1", + "select sin(c3) from stb1", + "select sin(c4) from stb1", + "select sin(c5) from stb1", + "select sin(c6) from stb1", + + "select sin(c6) as alisb from stb1", + "select sin(c6) alisb from stb1", + ] + + for type_sql in type_sql_lists: + tdSql.query(type_sql) + + def basic_sqrt_function(self): + + # basic query + tdSql.query("select c1 from ct3") + tdSql.checkRows(0) + tdSql.query("select c1 from t1") + tdSql.checkRows(12) + tdSql.query("select c1 from stb1") + tdSql.checkRows(25) + + # used for empty table , ct3 is empty + tdSql.query("select sin(c1) from ct3") + tdSql.checkRows(0) + tdSql.query("select sin(c2) from ct3") + tdSql.checkRows(0) + tdSql.query("select sin(c3) from ct3") + tdSql.checkRows(0) + tdSql.query("select sin(c4) from ct3") + tdSql.checkRows(0) + tdSql.query("select sin(c5) from ct3") + tdSql.checkRows(0) + tdSql.query("select sin(c6) from ct3") + tdSql.checkRows(0) + + + # # used for regular table + tdSql.query("select sin(c1) from t1") + tdSql.checkData(0, 0, None) + tdSql.checkData(1 , 0, 0.841470985) + tdSql.checkData(3 , 0, 0.141120008) + tdSql.checkData(5 , 0, None) + + tdSql.query("select c1, c2, c3 , c4, c5 from t1") + tdSql.checkData(1, 4, 1.11000) + tdSql.checkData(3, 3, 33) + tdSql.checkData(5, 4, None) + + tdSql.query("select ts,c1, c2, c3 , c4, c5 from t1") + tdSql.checkData(1, 5, 1.11000) + tdSql.checkData(3, 4, 33) + tdSql.checkData(5, 5, None) + + self.check_result_auto_sin( "select abs(c1), abs(c2), abs(c3) , abs(c4), abs(c5) from t1", "select sin(abs(c1)), sin(abs(c2)) ,sin(abs(c3)), sin(abs(c4)), sin(abs(c5)) from t1") + + # used for sub table + tdSql.query("select c2 ,sin(c2) from ct1") + tdSql.checkData(0, 1, -0.220708349) + tdSql.checkData(1 , 1, -0.556921845) + tdSql.checkData(3 , 1, -0.798311364) + tdSql.checkData(4 , 1, 0.000000000) + + tdSql.query("select c1, c5 ,sin(c5) from ct4") + tdSql.checkData(0 , 2, None) + tdSql.checkData(1 , 2, 0.518228108) + tdSql.checkData(2 , 2, 0.996475613) + tdSql.checkData(3 , 2, 0.367960369) + tdSql.checkData(5 , 2, None) + + self.check_result_auto_sin( "select c1, c2, c3 , c4, c5 from ct1", "select sin(c1), sin(c2) ,sin(c3), sin(c4), sin(c5) from ct1") + + # nest query for sin functions + tdSql.query("select c4 , sin(c4) ,sin(sin(c4)) , sin(sin(sin(c4))) from ct1;") + tdSql.checkData(0 , 0 , 88) + tdSql.checkData(0 , 1 , 0.035398303) + tdSql.checkData(0 , 2 , 0.035390911) + tdSql.checkData(0 , 3 , 0.035383523) + + tdSql.checkData(1 , 0 , 77) + tdSql.checkData(1 , 1 , 0.999520159) + tdSql.checkData(1 , 2 , 0.841211629) + tdSql.checkData(1 , 3 , 0.745451290) + + tdSql.checkData(11 , 0 , -99) + tdSql.checkData(11 , 1 , 0.999206834) + tdSql.checkData(11 , 2 , 0.841042171) + tdSql.checkData(11 , 3 , 0.745338326) + + # used for stable table + + tdSql.query("select sin(c1) from stb1") + tdSql.checkRows(25) + + + # used for not exists table + tdSql.error("select sin(c1) from stbbb1") + tdSql.error("select sin(c1) from tbname") + tdSql.error("select sin(c1) from ct5") + + # mix with common col + tdSql.query("select c1, sin(c1) from ct1") + tdSql.query("select c2, sin(c2) from ct4") + + + # mix with common functions + tdSql.query("select c1, sin(c1),sin(c1), sin(sin(c1)) from ct4 ") + tdSql.checkData(0 , 0 ,None) + tdSql.checkData(0 , 1 ,None) + tdSql.checkData(0 , 2 ,None) + tdSql.checkData(0 , 3 ,None) + + tdSql.checkData(3 , 0 , 6) + tdSql.checkData(3 , 1 ,-0.279415498) + tdSql.checkData(3 , 2 ,-0.279415498) + tdSql.checkData(3 , 3 ,-0.275793863) + + tdSql.query("select c1, sin(c1),c5, floor(c5) from stb1 ") + + # # mix with agg functions , not support + tdSql.error("select c1, sin(c1),c5, count(c5) from stb1 ") + tdSql.error("select c1, sin(c1),c5, count(c5) from ct1 ") + tdSql.error("select sin(c1), count(c5) from stb1 ") + tdSql.error("select sin(c1), count(c5) from ct1 ") + tdSql.error("select c1, count(c5) from ct1 ") + tdSql.error("select c1, count(c5) from stb1 ") + + # agg functions mix with agg functions + + tdSql.query("select max(c5), count(c5) from stb1") + tdSql.query("select max(c5), count(c5) from ct1") + + + # # bug fix for compute + tdSql.query("select c1, sin(c1) -0 ,sin(c1-4)-0 from ct4 ") + tdSql.checkData(0, 0, None) + tdSql.checkData(0, 1, None) + tdSql.checkData(0, 2, None) + tdSql.checkData(1, 0, 8) + tdSql.checkData(1, 1, 0.989358247) + tdSql.checkData(1, 2, -0.756802495) + + tdSql.query(" select c1, sin(c1) -0 ,sin(c1-0.1)-0.1 from ct4") + tdSql.checkData(0, 0, None) + tdSql.checkData(0, 1, None) + tdSql.checkData(0, 2, None) + tdSql.checkData(1, 0, 8) + tdSql.checkData(1, 1, 0.989358247) + tdSql.checkData(1, 2, 0.898941342) + + tdSql.query("select c1, sin(c1), c2, sin(c2), c3, sin(c3) from ct1") + + def test_big_number(self): + + tdSql.query("select c1, sin(100000000) from ct1") # bigint to double data overflow + tdSql.checkData(4, 1, math.sin(100000000)) + + + tdSql.query("select c1, sin(10000000000000) from ct1") # bigint to double data overflow + tdSql.checkData(4, 1, math.sin(10000000000000)) + + tdSql.query("select c1, sin(10000000000000000000000000) from ct1") # bigint to double data overflow + tdSql.query("select c1, sin(10000000000000000000000000.0) from ct1") # 10000000000000000000000000.0 is a double value + tdSql.checkData(1, 1, math.sin(10000000000000000000000000.0)) + + tdSql.query("select c1, sin(10000000000000000000000000000000000) from ct1") # bigint to double data overflow + tdSql.query("select c1, sin(10000000000000000000000000000000000.0) from ct1") # 10000000000000000000000000.0 is a double value + tdSql.checkData(4, 1, math.sin(10000000000000000000000000000000000.0)) + + tdSql.query("select c1, sin(10000000000000000000000000000000000000000) from ct1") # bigint to double data overflow + tdSql.query("select c1, sin(10000000000000000000000000000000000000000.0) from ct1") # 10000000000000000000000000.0 is a double value + + tdSql.checkData(4, 1, math.sin(10000000000000000000000000000000000000000.0)) + + tdSql.query("select c1, sin(10000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000) from ct1") # bigint to double data overflow + + def abs_func_filter(self): + tdSql.execute("use db") + tdSql.query("select c1, abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(sin(c1)-0.5) from ct4 where c1>5 ") + tdSql.checkRows(3) + tdSql.checkData(0,0,8) + tdSql.checkData(0,1,8.000000000) + tdSql.checkData(0,2,8.000000000) + tdSql.checkData(0,3,7.900000000) + tdSql.checkData(0,4,1.000000000) + + tdSql.query("select c1, abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(sin(c1)-0.5) from ct4 where c1=5 ") + tdSql.checkRows(1) + tdSql.checkData(0,0,5) + tdSql.checkData(0,1,5.000000000) + tdSql.checkData(0,2,5.000000000) + tdSql.checkData(0,3,4.900000000) + tdSql.checkData(0,4,-1.000000000) + + tdSql.query("select c1,c2 , abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(sin(c1)-0.5) from ct4 where c1=sin(c1) limit 1 ") + tdSql.checkRows(1) + tdSql.checkData(0,0,0) + tdSql.checkData(0,1,0) + tdSql.checkData(0,2,0.000000000) + tdSql.checkData(0,3,0.000000000) + tdSql.checkData(0,4,-0.100000000) + tdSql.checkData(0,5,0.000000000) + + def pow_Arithmetic(self): + pass + + def check_boundary_values(self): + + PI=3.1415926 + + tdSql.execute("drop database if exists bound_test") + tdSql.execute("create database if not exists bound_test") + time.sleep(3) + tdSql.execute("use bound_test") + tdSql.execute( + "create table stb_bound (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(32),c9 nchar(32), c10 timestamp) tags (t1 int);" + ) + tdSql.execute(f'create table sub1_bound using stb_bound tags ( 1 )') + tdSql.execute( + f"insert into sub1_bound values ( now()-1s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + tdSql.execute( + f"insert into sub1_bound values ( now()-1s, -2147483647, -9223372036854775807, -32767, -127, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + tdSql.execute( + f"insert into sub1_bound values ( now(), 2147483646, 9223372036854775806, 32766, 126, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + tdSql.execute( + f"insert into sub1_bound values ( now(), -2147483646, -9223372036854775806, -32766, -126, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + tdSql.error( + f"insert into sub1_bound values ( now()+1s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + self.check_result_auto_sin( "select abs(c1), abs(c2), abs(c3) , abs(c4), abs(c5) from sub1_bound ", "select sin(abs(c1)), sin(abs(c2)) ,sin(abs(c3)), sin(abs(c4)), sin(abs(c5)) from sub1_bound") + + self.check_result_auto_sin( "select c1, c2, c3 , c3, c2 ,c1 from sub1_bound ", "select sin(c1), sin(c2) ,sin(c3), sin(c3), sin(c2) ,sin(c1) from sub1_bound") + + self.check_result_auto_sin("select abs(abs(abs(abs(abs(abs(abs(abs(abs(c1))))))))) nest_col_func from sub1_bound" , "select sin(abs(c1)) from sub1_bound" ) + + # check basic elem for table per row + tdSql.query("select sin(abs(c1)) ,sin(abs(c2)) , sin(abs(c3)) , sin(abs(c4)), sin(abs(c5)), sin(abs(c6)) from sub1_bound ") + tdSql.checkData(0,0,math.sin(2147483647)) + tdSql.checkData(0,1,math.sin(9223372036854775807)) + tdSql.checkData(0,2,math.sin(32767)) + tdSql.checkData(0,3,math.sin(127)) + tdSql.checkData(0,4,math.sin(339999995214436424907732413799364296704.00000)) + tdSql.checkData(1,0,math.sin(2147483647)) + tdSql.checkData(1,1,math.sin(9223372036854775807)) + tdSql.checkData(1,2,math.sin(32767)) + tdSql.checkData(1,3,math.sin(127)) + tdSql.checkData(1,4,math.sin(339999995214436424907732413799364296704.00000)) + tdSql.checkData(3,0,math.sin(2147483646)) + tdSql.checkData(3,1,math.sin(9223372036854775806)) + tdSql.checkData(3,2,math.sin(32766)) + tdSql.checkData(3,3,math.sin(126)) + tdSql.checkData(3,4,math.sin(339999995214436424907732413799364296704.00000)) + + # check + - * / in functions + tdSql.query("select sin(abs(c1+1)) ,sin(abs(c2)) , sin(abs(c3*1)) , sin(abs(c4/2)), sin(abs(c5))/2, sin(abs(c6)) from sub1_bound ") + tdSql.checkData(0,0,math.sin(2147483648.000000000)) + tdSql.checkData(0,1,math.sin(9223372036854775807)) + tdSql.checkData(0,2,math.sin(32767.000000000)) + tdSql.checkData(0,3,math.sin(63.500000000)) + + tdSql.execute("create stable st (ts timestamp, num1 float, num2 double) tags (t1 int);") + tdSql.execute(f'create table tb1 using st tags (1)') + tdSql.execute(f'create table tb2 using st tags (2)') + tdSql.execute(f'create table tb3 using st tags (3)') + tdSql.execute('insert into tb1 values (now()-40s, {}, {})'.format(PI/2 ,PI/2 )) + tdSql.execute('insert into tb1 values (now()-30s, {}, {})'.format(PI ,PI )) + tdSql.execute('insert into tb1 values (now()-20s, {}, {})'.format(PI*1.5 ,PI*1.5)) + tdSql.execute('insert into tb1 values (now()-10s, {}, {})'.format(PI*2 ,PI*2)) + tdSql.execute('insert into tb1 values (now(), {}, {})'.format(PI*2.5 ,PI*2.5)) + + tdSql.execute('insert into tb2 values (now()-40s, {}, {})'.format(PI/2 ,PI/2 )) + tdSql.execute('insert into tb2 values (now()-30s, {}, {})'.format(PI ,PI )) + tdSql.execute('insert into tb2 values (now()-20s, {}, {})'.format(PI*1.5 ,PI*1.5)) + tdSql.execute('insert into tb2 values (now()-10s, {}, {})'.format(PI*2 ,PI*2)) + tdSql.execute('insert into tb2 values (now(), {}, {})'.format(PI*2.5 ,PI*2.5)) + + for i in range(100): + tdSql.execute('insert into tb3 values (now()+{}s, {}, {})'.format(i,PI*(2.5+i) ,PI*(2.5+i))) + + self.check_result_auto_sin("select num1,num2 from tb3;" , "select sin(num1),sin(num2) from tb3") + + def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring + tdSql.prepare() + + tdLog.printNoPrefix("==========step1:create table ==============") + + self.prepare_datas() + + tdLog.printNoPrefix("==========step2:test errors ==============") + + self.test_errors() + + tdLog.printNoPrefix("==========step3:support types ============") + + self.support_types() + + tdLog.printNoPrefix("==========step4: sin basic query ============") + + self.basic_sqrt_function() + + tdLog.printNoPrefix("==========step5: big number sin query ============") + + self.test_big_number() + + + tdLog.printNoPrefix("==========step6: sin boundary query ============") + + self.check_boundary_values() + + tdLog.printNoPrefix("==========step7: sin filter query ============") + + self.abs_func_filter() + + + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 1dfb160987..dec31b137c 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -24,3 +24,4 @@ python3 ./test.py -f 2-query/floor.py python3 ./test.py -f 2-query/round.py python3 ./test.py -f 2-query/log.py python3 ./test.py -f 2-query/pow.py +python3 ./test.py -f 2-query/sin.py \ No newline at end of file From 31f1af34e530aa99d205ea76e9e6dd3b9511cdc1 Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Fri, 6 May 2022 19:56:51 +0800 Subject: [PATCH 34/83] update sin case --- tests/system-test/2-query/sin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/2-query/sin.py b/tests/system-test/2-query/sin.py index c2c5b8794e..1bc01dbb8a 100644 --- a/tests/system-test/2-query/sin.py +++ b/tests/system-test/2-query/sin.py @@ -204,7 +204,7 @@ class TDTestCase: for type_sql in type_sql_lists: tdSql.query(type_sql) - def basic_sqrt_function(self): + def basic_sin_function(self): # basic query tdSql.query("select c1 from ct3") From c1cc6417cb9e8233dc7d1ac4bb09d989b604d34f Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Fri, 6 May 2022 19:57:29 +0800 Subject: [PATCH 35/83] update sin case --- tests/system-test/2-query/sin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/2-query/sin.py b/tests/system-test/2-query/sin.py index 1bc01dbb8a..138027d26c 100644 --- a/tests/system-test/2-query/sin.py +++ b/tests/system-test/2-query/sin.py @@ -494,7 +494,7 @@ class TDTestCase: tdLog.printNoPrefix("==========step4: sin basic query ============") - self.basic_sqrt_function() + self.basic_sin_function() tdLog.printNoPrefix("==========step5: big number sin query ============") From 7e30e1b50baaaa874b2e15255fde5666627af912 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 6 May 2022 20:00:30 +0800 Subject: [PATCH 36/83] refactor:merge from 3.0 --- source/client/src/clientSml.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 8b5712a619..85cd0661e3 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -147,7 +147,7 @@ static int smlCompareKv(const void* p1, const void* p2) { SSmlKv* kv2 = (SSmlKv*)p2; int32_t kvLen1 = kv1->keyLen; int32_t kvLen2 = kv2->keyLen; - int32_t res = strncasecmp(kv1->key, kv2->key, MIN(kvLen1, kvLen2)); + int32_t res = strncasecmp(kv1->key, kv2->key, TMIN(kvLen1, kvLen2)); if (res != 0) { return res; } else { From 20cf55c1fc99d4bc1a772f20bc8f16c623f6a7c3 Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Fri, 6 May 2022 20:31:52 +0800 Subject: [PATCH 37/83] add case for cosin function --- tests/system-test/2-query/cos.py | 518 +++++++++++++++++++++++++++++++ tests/system-test/2-query/sin.py | 2 +- tests/system-test/fulltest.sh | 3 +- 3 files changed, 521 insertions(+), 2 deletions(-) create mode 100644 tests/system-test/2-query/cos.py diff --git a/tests/system-test/2-query/cos.py b/tests/system-test/2-query/cos.py new file mode 100644 index 0000000000..0cb9f66937 --- /dev/null +++ b/tests/system-test/2-query/cos.py @@ -0,0 +1,518 @@ +import taos +import sys +import datetime +import inspect +import math +from util.log import * +from util.sql import * +from util.cases import * + + +class TDTestCase: + updatecfgDict = {'debugFlag': 143 ,"cDebugFlag":143,"uDebugFlag":143 ,"rpcDebugFlag":143 , "tmrDebugFlag":143 , + "jniDebugFlag":143 ,"simDebugFlag":143,"dDebugFlag":143, "dDebugFlag":143,"vDebugFlag":143,"mDebugFlag":143,"qDebugFlag":143, + "wDebugFlag":143,"sDebugFlag":143,"tsdbDebugFlag":143,"tqDebugFlag":143 ,"fsDebugFlag":143 ,"fnDebugFlag":143} + def init(self, conn, powSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + + def prepare_datas(self): + tdSql.execute( + '''create table stb1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + tags (t1 int) + ''' + ) + + tdSql.execute( + ''' + create table t1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + ''' + ) + for i in range(4): + tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )') + + for i in range(9): + tdSql.execute( + f"insert into ct1 values ( now()-{i*10}s, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + ) + tdSql.execute( + f"insert into ct4 values ( now()-{i*90}d, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + ) + tdSql.execute("insert into ct1 values (now()-45s, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a )") + tdSql.execute("insert into ct1 values (now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute("insert into ct1 values (now()+15s, 9, -99999, -999, -99, -9.99, NULL, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute("insert into ct1 values (now()+20s, 9, -99999, -999, NULL, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + + tdSql.execute("insert into ct4 values (now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute("insert into ct4 values (now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute("insert into ct4 values (now()+90d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + + tdSql.execute( + f'''insert into t1 values + ( '2020-04-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2020-10-21 01:01:01.000', 1, 11111, 111, 11, 1.11, 11.11, 1, "binary1", "nchar1", now()+1a ) + ( '2020-12-31 01:01:01.000', 2, 22222, 222, 22, 2.22, 22.22, 0, "binary2", "nchar2", now()+2a ) + ( '2021-01-01 01:01:06.000', 3, 33333, 333, 33, 3.33, 33.33, 0, "binary3", "nchar3", now()+3a ) + ( '2021-05-07 01:01:10.000', 4, 44444, 444, 44, 4.44, 44.44, 1, "binary4", "nchar4", now()+4a ) + ( '2021-07-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2021-09-30 01:01:16.000', 5, 55555, 555, 55, 5.55, 55.55, 0, "binary5", "nchar5", now()+5a ) + ( '2022-02-01 01:01:20.000', 6, 66666, 666, 66, 6.66, 66.66, 1, "binary6", "nchar6", now()+6a ) + ( '2022-10-28 01:01:26.000', 7, 00000, 000, 00, 0.00, 00.00, 1, "binary7", "nchar7", "1970-01-01 08:00:00.000" ) + ( '2022-12-01 01:01:30.000', 8, -88888, -888, -88, -8.88, -88.88, 0, "binary8", "nchar8", "1969-01-01 01:00:00.000" ) + ( '2022-12-31 01:01:36.000', 9, -99999999999999999, -999, -99, -9.99, -999999999999999999999.99, 1, "binary9", "nchar9", "1900-01-01 00:00:00.000" ) + ( '2023-02-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ''' + ) + + def check_result_auto_cos(self ,origin_query , pow_query): + + pow_result = tdSql.getResult(pow_query) + origin_result = tdSql.getResult(origin_query) + + auto_result =[] + + for row in origin_result: + row_check = [] + for elem in row: + if elem == None: + elem = None + else: + elem = math.cos(elem) + row_check.append(elem) + auto_result.append(row_check) + + check_status = True + + for row_index , row in enumerate(pow_result): + for col_index , elem in enumerate(row): + if auto_result[row_index][col_index] == None and not (auto_result[row_index][col_index] == None and elem == None): + check_status = False + elif auto_result[row_index][col_index] != None and (auto_result[row_index][col_index] - elem > 0.00000001): + check_status = False + else: + pass + if not check_status: + tdLog.notice("cos function value has not as expected , sql is \"%s\" "%pow_query ) + sys.exit(1) + else: + tdLog.info("cos value check pass , it work as expected ,sql is \"%s\" "%pow_query ) + + def test_errors(self): + error_sql_lists = [ + "select cos from t1", + # "select cos(-+--+c1 ) from t1", + # "select +-cos(c1) from t1", + # "select ++-cos(c1) from t1", + # "select ++--cos(c1) from t1", + # "select - -cos(c1)*0 from t1", + # "select cos(tbname+1) from t1 ", + "select cos(123--123)==1 from t1", + "select cos(c1) as 'd1' from t1", + "select cos(c1 ,c2) from t1", + "select cos(c1 ,NULL ) from t1", + "select cos(,) from t1;", + "select cos(cos(c1) ab from t1)", + "select cos(c1 ) as int from t1", + "select cos from stb1", + # "select cos(-+--+c1) from stb1", + # "select +-cos(c1) from stb1", + # "select ++-cos(c1) from stb1", + # "select ++--cos(c1) from stb1", + # "select - -cos(c1)*0 from stb1", + # "select cos(tbname+1) from stb1 ", + "select cos(123--123)==1 from stb1", + "select cos(c1) as 'd1' from stb1", + "select cos(c1 ,c2 ) from stb1", + "select cos(c1 ,NULL) from stb1", + "select cos(,) from stb1;", + "select cos(cos(c1) ab from stb1)", + "select cos(c1) as int from stb1" + ] + for error_sql in error_sql_lists: + tdSql.error(error_sql) + + def support_types(self): + type_error_sql_lists = [ + "select cos(ts) from t1" , + "select cos(c7) from t1", + "select cos(c8) from t1", + "select cos(c9) from t1", + "select cos(ts) from ct1" , + "select cos(c7) from ct1", + "select cos(c8) from ct1", + "select cos(c9) from ct1", + "select cos(ts) from ct3" , + "select cos(c7) from ct3", + "select cos(c8) from ct3", + "select cos(c9) from ct3", + "select cos(ts) from ct4" , + "select cos(c7) from ct4", + "select cos(c8) from ct4", + "select cos(c9) from ct4", + "select cos(ts) from stb1" , + "select cos(c7) from stb1", + "select cos(c8) from stb1", + "select cos(c9) from stb1" , + + "select cos(ts) from stbbb1" , + "select cos(c7) from stbbb1", + + "select cos(ts) from tbname", + "select cos(c9) from tbname" + + ] + + for type_sql in type_error_sql_lists: + tdSql.error(type_sql) + + + type_sql_lists = [ + "select cos(c1) from t1", + "select cos(c2) from t1", + "select cos(c3) from t1", + "select cos(c4) from t1", + "select cos(c5) from t1", + "select cos(c6) from t1", + + "select cos(c1) from ct1", + "select cos(c2) from ct1", + "select cos(c3) from ct1", + "select cos(c4) from ct1", + "select cos(c5) from ct1", + "select cos(c6) from ct1", + + "select cos(c1) from ct3", + "select cos(c2) from ct3", + "select cos(c3) from ct3", + "select cos(c4) from ct3", + "select cos(c5) from ct3", + "select cos(c6) from ct3", + + "select cos(c1) from stb1", + "select cos(c2) from stb1", + "select cos(c3) from stb1", + "select cos(c4) from stb1", + "select cos(c5) from stb1", + "select cos(c6) from stb1", + + "select cos(c6) as alisb from stb1", + "select cos(c6) alisb from stb1", + ] + + for type_sql in type_sql_lists: + tdSql.query(type_sql) + + def basic_cosin_function(self): + + # basic query + tdSql.query("select c1 from ct3") + tdSql.checkRows(0) + tdSql.query("select c1 from t1") + tdSql.checkRows(12) + tdSql.query("select c1 from stb1") + tdSql.checkRows(25) + + # used for empty table , ct3 is empty + tdSql.query("select cos(c1) from ct3") + tdSql.checkRows(0) + tdSql.query("select cos(c2) from ct3") + tdSql.checkRows(0) + tdSql.query("select cos(c3) from ct3") + tdSql.checkRows(0) + tdSql.query("select cos(c4) from ct3") + tdSql.checkRows(0) + tdSql.query("select cos(c5) from ct3") + tdSql.checkRows(0) + tdSql.query("select cos(c6) from ct3") + tdSql.checkRows(0) + + + # # used for regular table + tdSql.query("select cos(c1) from t1") + tdSql.checkData(0, 0, None) + tdSql.checkData(1 , 0, 0.540302306) + tdSql.checkData(3 , 0, -0.989992497) + tdSql.checkData(5 , 0, None) + + tdSql.query("select c1, c2, c3 , c4, c5 from t1") + tdSql.checkData(1, 4, 1.11000) + tdSql.checkData(3, 3, 33) + tdSql.checkData(5, 4, None) + + tdSql.query("select ts,c1, c2, c3 , c4, c5 from t1") + tdSql.checkData(1, 5, 1.11000) + tdSql.checkData(3, 4, 33) + tdSql.checkData(5, 5, None) + + self.check_result_auto_cos( "select abs(c1), abs(c2), abs(c3) , abs(c4), abs(c5) from t1", "select cos(abs(c1)), cos(abs(c2)) ,cos(abs(c3)), cos(abs(c4)), cos(abs(c5)) from t1") + + # used for sub table + tdSql.query("select c2 ,cos(c2) from ct1") + tdSql.checkData(0, 1, 0.975339851) + tdSql.checkData(1 , 1, -0.830564903) + tdSql.checkData(3 , 1, 0.602244939) + tdSql.checkData(4 , 1, 1.000000000) + + tdSql.query("select c1, c5 ,cos(c5) from ct4") + tdSql.checkData(0 , 2, None) + tdSql.checkData(1 , 2, -0.855242438) + tdSql.checkData(2 , 2, 0.083882969) + tdSql.checkData(3 , 2, 0.929841474) + tdSql.checkData(5 , 2, None) + + self.check_result_auto_cos( "select c1, c2, c3 , c4, c5 from ct1", "select cos(c1), cos(c2) ,cos(c3), cos(c4), cos(c5) from ct1") + + # nest query for cos functions + tdSql.query("select c4 , cos(c4) ,cos(cos(c4)) , cos(cos(cos(c4))) from ct1;") + tdSql.checkData(0 , 0 , 88) + tdSql.checkData(0 , 1 , 0.999373284) + tdSql.checkData(0 , 2 , 0.540829563) + tdSql.checkData(0 , 3 , 0.857281878) + + tdSql.checkData(1 , 0 , 77) + tdSql.checkData(1 , 1 , -0.030975032) + tdSql.checkData(1 , 2 , 0.999520312) + tdSql.checkData(1 , 3 , 0.540705887) + + tdSql.checkData(11 , 0 , -99) + tdSql.checkData(11 , 1 , 0.039820880) + tdSql.checkData(11 , 2 , 0.999207254) + tdSql.checkData(11 , 3 , 0.540969209) + + # used for stable table + + tdSql.query("select cos(c1) from stb1") + tdSql.checkRows(25) + + + # used for not exists table + tdSql.error("select cos(c1) from stbbb1") + tdSql.error("select cos(c1) from tbname") + tdSql.error("select cos(c1) from ct5") + + # mix with common col + tdSql.query("select c1, cos(c1) from ct1") + tdSql.query("select c2, cos(c2) from ct4") + + + # mix with common functions + tdSql.query("select c1, cos(c1),cos(c1), cos(cos(c1)) from ct4 ") + tdSql.checkData(0 , 0 ,None) + tdSql.checkData(0 , 1 ,None) + tdSql.checkData(0 , 2 ,None) + tdSql.checkData(0 , 3 ,None) + + tdSql.checkData(3 , 0 , 6) + tdSql.checkData(3 , 1 ,0.960170287) + tdSql.checkData(3 , 2 ,0.960170287) + tdSql.checkData(3 , 3 ,0.573380480) + + tdSql.query("select c1, cos(c1),c5, floor(c5) from stb1 ") + + # # mix with agg functions , not support + tdSql.error("select c1, cos(c1),c5, count(c5) from stb1 ") + tdSql.error("select c1, cos(c1),c5, count(c5) from ct1 ") + tdSql.error("select cos(c1), count(c5) from stb1 ") + tdSql.error("select cos(c1), count(c5) from ct1 ") + tdSql.error("select c1, count(c5) from ct1 ") + tdSql.error("select c1, count(c5) from stb1 ") + + # agg functions mix with agg functions + + tdSql.query("select max(c5), count(c5) from stb1") + tdSql.query("select max(c5), count(c5) from ct1") + + + # # bug fix for compute + tdSql.query("select c1, cos(c1) -0 ,cos(c1-4)-0 from ct4 ") + tdSql.checkData(0, 0, None) + tdSql.checkData(0, 1, None) + tdSql.checkData(0, 2, None) + tdSql.checkData(1, 0, 8) + tdSql.checkData(1, 1, -0.145500034) + tdSql.checkData(1, 2, -0.653643621) + + tdSql.query(" select c1, cos(c1) -0 ,cos(c1-0.1)-0.1 from ct4") + tdSql.checkData(0, 0, None) + tdSql.checkData(0, 1, None) + tdSql.checkData(0, 2, None) + tdSql.checkData(1, 0, 8) + tdSql.checkData(1, 1, -0.145500034) + tdSql.checkData(1, 2, -0.146002126) + + tdSql.query("select c1, cos(c1), c2, cos(c2), c3, cos(c3) from ct1") + + def test_big_number(self): + + tdSql.query("select c1, cos(100000000) from ct1") # bigint to double data overflow + tdSql.checkData(4, 1, math.cos(100000000)) + + + tdSql.query("select c1, cos(10000000000000) from ct1") # bigint to double data overflow + tdSql.checkData(4, 1, math.cos(10000000000000)) + + tdSql.query("select c1, cos(10000000000000000000000000) from ct1") # bigint to double data overflow + tdSql.query("select c1, cos(10000000000000000000000000.0) from ct1") # 10000000000000000000000000.0 is a double value + tdSql.checkData(1, 1, math.cos(10000000000000000000000000.0)) + + tdSql.query("select c1, cos(10000000000000000000000000000000000) from ct1") # bigint to double data overflow + tdSql.query("select c1, cos(10000000000000000000000000000000000.0) from ct1") # 10000000000000000000000000.0 is a double value + tdSql.checkData(4, 1, math.cos(10000000000000000000000000000000000.0)) + + tdSql.query("select c1, cos(10000000000000000000000000000000000000000) from ct1") # bigint to double data overflow + tdSql.query("select c1, cos(10000000000000000000000000000000000000000.0) from ct1") # 10000000000000000000000000.0 is a double value + + tdSql.checkData(4, 1, math.cos(10000000000000000000000000000000000000000.0)) + + tdSql.query("select c1, cos(10000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000) from ct1") # bigint to double data overflow + + def abs_func_filter(self): + tdSql.execute("use db") + tdSql.query("select c1, abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(cos(c1)-0.5) from ct4 where c1>5 ") + tdSql.checkRows(3) + tdSql.checkData(0,0,8) + tdSql.checkData(0,1,8.000000000) + tdSql.checkData(0,2,8.000000000) + tdSql.checkData(0,3,7.900000000) + tdSql.checkData(0,4,0.000000000) + + tdSql.query("select c1, abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(cos(c1)-0.5) from ct4 where c1=5 ") + tdSql.checkRows(1) + tdSql.checkData(0,0,5) + tdSql.checkData(0,1,5.000000000) + tdSql.checkData(0,2,5.000000000) + tdSql.checkData(0,3,4.900000000) + tdSql.checkData(0,4,0.000000000) + + tdSql.query("select c1,c2 , abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(cos(c1)-0.5) from ct4 where c1>cos(c1) limit 1 ") + tdSql.checkRows(1) + tdSql.checkData(0,0,8) + tdSql.checkData(0,1,88888) + tdSql.checkData(0,2,8.000000000) + tdSql.checkData(0,3,8.000000000) + tdSql.checkData(0,4,7.900000000) + tdSql.checkData(0,5,0.000000000) + + def pow_Arithmetic(self): + pass + + def check_boundary_values(self): + + PI=3.1415926 + + tdSql.execute("drop database if exists bound_test") + tdSql.execute("create database if not exists bound_test") + time.sleep(3) + tdSql.execute("use bound_test") + tdSql.execute( + "create table stb_bound (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(32),c9 nchar(32), c10 timestamp) tags (t1 int);" + ) + tdSql.execute(f'create table sub1_bound using stb_bound tags ( 1 )') + tdSql.execute( + f"insert into sub1_bound values ( now()-1s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + tdSql.execute( + f"insert into sub1_bound values ( now()-1s, -2147483647, -9223372036854775807, -32767, -127, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + tdSql.execute( + f"insert into sub1_bound values ( now(), 2147483646, 9223372036854775806, 32766, 126, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + tdSql.execute( + f"insert into sub1_bound values ( now(), -2147483646, -9223372036854775806, -32766, -126, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + tdSql.error( + f"insert into sub1_bound values ( now()+1s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + self.check_result_auto_cos( "select abs(c1), abs(c2), abs(c3) , abs(c4), abs(c5) from sub1_bound ", "select cos(abs(c1)), cos(abs(c2)) ,cos(abs(c3)), cos(abs(c4)), cos(abs(c5)) from sub1_bound") + + self.check_result_auto_cos( "select c1, c2, c3 , c3, c2 ,c1 from sub1_bound ", "select cos(c1), cos(c2) ,cos(c3), cos(c3), cos(c2) ,cos(c1) from sub1_bound") + + self.check_result_auto_cos("select abs(abs(abs(abs(abs(abs(abs(abs(abs(c1))))))))) nest_col_func from sub1_bound" , "select cos(abs(c1)) from sub1_bound" ) + + # check basic elem for table per row + tdSql.query("select cos(abs(c1)) ,cos(abs(c2)) , cos(abs(c3)) , cos(abs(c4)), cos(abs(c5)), cos(abs(c6)) from sub1_bound ") + tdSql.checkData(0,0,math.cos(2147483647)) + tdSql.checkData(0,1,math.cos(9223372036854775807)) + tdSql.checkData(0,2,math.cos(32767)) + tdSql.checkData(0,3,math.cos(127)) + tdSql.checkData(0,4,math.cos(339999995214436424907732413799364296704.00000)) + tdSql.checkData(1,0,math.cos(2147483647)) + tdSql.checkData(1,1,math.cos(9223372036854775807)) + tdSql.checkData(1,2,math.cos(32767)) + tdSql.checkData(1,3,math.cos(127)) + tdSql.checkData(1,4,math.cos(339999995214436424907732413799364296704.00000)) + tdSql.checkData(3,0,math.cos(2147483646)) + tdSql.checkData(3,1,math.cos(9223372036854775806)) + tdSql.checkData(3,2,math.cos(32766)) + tdSql.checkData(3,3,math.cos(126)) + tdSql.checkData(3,4,math.cos(339999995214436424907732413799364296704.00000)) + + # check + - * / in functions + tdSql.query("select cos(abs(c1+1)) ,cos(abs(c2)) , cos(abs(c3*1)) , cos(abs(c4/2)), cos(abs(c5))/2, cos(abs(c6)) from sub1_bound ") + tdSql.checkData(0,0,math.cos(2147483648.000000000)) + tdSql.checkData(0,1,math.cos(9223372036854775807)) + tdSql.checkData(0,2,math.cos(32767.000000000)) + tdSql.checkData(0,3,math.cos(63.500000000)) + + tdSql.execute("create stable st (ts timestamp, num1 float, num2 double) tags (t1 int);") + tdSql.execute(f'create table tb1 using st tags (1)') + tdSql.execute(f'create table tb2 using st tags (2)') + tdSql.execute(f'create table tb3 using st tags (3)') + tdSql.execute('insert into tb1 values (now()-40s, {}, {})'.format(PI/2 ,PI/2 )) + tdSql.execute('insert into tb1 values (now()-30s, {}, {})'.format(PI ,PI )) + tdSql.execute('insert into tb1 values (now()-20s, {}, {})'.format(PI*1.5 ,PI*1.5)) + tdSql.execute('insert into tb1 values (now()-10s, {}, {})'.format(PI*2 ,PI*2)) + tdSql.execute('insert into tb1 values (now(), {}, {})'.format(PI*2.5 ,PI*2.5)) + + tdSql.execute('insert into tb2 values (now()-40s, {}, {})'.format(PI/2 ,PI/2 )) + tdSql.execute('insert into tb2 values (now()-30s, {}, {})'.format(PI ,PI )) + tdSql.execute('insert into tb2 values (now()-20s, {}, {})'.format(PI*1.5 ,PI*1.5)) + tdSql.execute('insert into tb2 values (now()-10s, {}, {})'.format(PI*2 ,PI*2)) + tdSql.execute('insert into tb2 values (now(), {}, {})'.format(PI*2.5 ,PI*2.5)) + + for i in range(100): + tdSql.execute('insert into tb3 values (now()+{}s, {}, {})'.format(i,PI*(5+i)/2 ,PI*(5+i)/2)) + + self.check_result_auto_cos("select num1,num2 from tb3;" , "select cos(num1),cos(num2) from tb3") + + def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring + tdSql.prepare() + + tdLog.printNoPrefix("==========step1:create table ==============") + + self.prepare_datas() + + tdLog.printNoPrefix("==========step2:test errors ==============") + + self.test_errors() + + tdLog.printNoPrefix("==========step3:support types ============") + + self.support_types() + + tdLog.printNoPrefix("==========step4: cos basic query ============") + + self.basic_cosin_function() + + tdLog.printNoPrefix("==========step5: big number cos query ============") + + self.test_big_number() + + + tdLog.printNoPrefix("==========step6: cos boundary query ============") + + self.check_boundary_values() + + tdLog.printNoPrefix("==========step7: cos filter query ============") + + self.abs_func_filter() + + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/2-query/sin.py b/tests/system-test/2-query/sin.py index 138027d26c..62d42a991e 100644 --- a/tests/system-test/2-query/sin.py +++ b/tests/system-test/2-query/sin.py @@ -473,7 +473,7 @@ class TDTestCase: tdSql.execute('insert into tb2 values (now(), {}, {})'.format(PI*2.5 ,PI*2.5)) for i in range(100): - tdSql.execute('insert into tb3 values (now()+{}s, {}, {})'.format(i,PI*(2.5+i) ,PI*(2.5+i))) + tdSql.execute('insert into tb3 values (now()+{}s, {}, {})'.format(i,PI*(5+i)/2 ,PI*(5+i)/2)) self.check_result_auto_sin("select num1,num2 from tb3;" , "select sin(num1),sin(num2) from tb3") diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index dec31b137c..bdef427fdb 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -24,4 +24,5 @@ python3 ./test.py -f 2-query/floor.py python3 ./test.py -f 2-query/round.py python3 ./test.py -f 2-query/log.py python3 ./test.py -f 2-query/pow.py -python3 ./test.py -f 2-query/sin.py \ No newline at end of file +python3 ./test.py -f 2-query/sin.py +python3 ./test.py -f 2-query/cos.py \ No newline at end of file From 679726ebb7cee70f0eb88e9efbb737fb96fbec51 Mon Sep 17 00:00:00 2001 From: jiacy-jcy <714897623@qq.com> Date: Fri, 6 May 2022 20:47:27 +0800 Subject: [PATCH 38/83] update test case --- tests/system-test/2-query/count.py | 86 +++++++++++++++++------------- tests/system-test/2-query/last.py | 3 +- 2 files changed, 52 insertions(+), 37 deletions(-) diff --git a/tests/system-test/2-query/count.py b/tests/system-test/2-query/count.py index 8888a182b8..a70a2e72f2 100644 --- a/tests/system-test/2-query/count.py +++ b/tests/system-test/2-query/count.py @@ -16,6 +16,7 @@ class TDTestCase: tdSql.execute('''create table stb(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, col7 bool, col8 binary(20), col9 nchar(20), col11 tinyint unsigned, col12 smallint unsigned, col13 int unsigned, col14 bigint unsigned) tags(loc nchar(20))''') tdSql.execute("create table stb_1 using stb tags('beijing')") + tdSql.execute("create table stb_2 using stb tags('shanghai')") tdSql.execute('''create table ntb(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, col7 bool, col8 binary(20), col9 nchar(20), col11 tinyint unsigned, col12 smallint unsigned, col13 int unsigned, col14 bigint unsigned)''') @@ -23,71 +24,76 @@ class TDTestCase: for i in range(self.rowNum): tdSql.execute("insert into stb_1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1)) + tdSql.execute("insert into stb_2 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1)) for i in range(self.rowNum): tdSql.execute("insert into ntb values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1)) tdSql.query("select count(*) from stb") - tdSql.checkData(0,0,10) + tdSql.checkData(0,0,20) tdSql.query("select count(*) from db.stb") - tdSql.checkData(0,0,10) + tdSql.checkData(0,0,20) tdSql.query("select count(ts) from stb") - tdSql.checkData(0,0,10) + tdSql.checkData(0,0,20) tdSql.query("select count(ts) from db.stb") - tdSql.checkData(0,0,10) + tdSql.checkData(0,0,20) tdSql.query("select count(col1) from stb") - tdSql.checkData(0,0,10) + tdSql.checkData(0,0,20) tdSql.query("select count(col1) from db.stb") - tdSql.checkData(0,0,10) + tdSql.checkData(0,0,20) tdSql.query("select count(col2) from stb") - tdSql.checkData(0,0,10) + tdSql.checkData(0,0,20) tdSql.query("select count(col2) from db.stb") - tdSql.checkData(0,0,10) + tdSql.checkData(0,0,20) tdSql.query("select count(col3) from stb") - tdSql.checkData(0,0,10) + tdSql.checkData(0,0,20) tdSql.query("select count(col3) from db.stb") - tdSql.checkData(0,0,10) + tdSql.checkData(0,0,20) tdSql.query("select count(col4) from stb") - tdSql.checkData(0,0,10) + tdSql.checkData(0,0,20) tdSql.query("select count(col4) from db.stb") - tdSql.checkData(0,0,10) + tdSql.checkData(0,0,20) tdSql.query("select count(col5) from stb") - tdSql.checkData(0,0,10) + tdSql.checkData(0,0,20) tdSql.query("select count(col5) from db.stb") - tdSql.checkData(0,0,10) + tdSql.checkData(0,0,20) tdSql.query("select count(col6) from stb") - tdSql.checkData(0,0,10) + tdSql.checkData(0,0,20) tdSql.query("select count(col6) from db.stb") - tdSql.checkData(0,0,10) + tdSql.checkData(0,0,20) tdSql.query("select count(col7) from stb") - tdSql.checkData(0,0,10) + tdSql.checkData(0,0,20) tdSql.query("select count(col7) from db.stb") - tdSql.checkData(0,0,10) + tdSql.checkData(0,0,20) tdSql.query("select count(col8) from stb") - tdSql.checkData(0,0,10) + tdSql.checkData(0,0,20) tdSql.query("select count(col8) from db.stb") - tdSql.checkData(0,0,10) + tdSql.checkData(0,0,20) tdSql.query("select count(col9) from stb") - tdSql.checkData(0,0,10) + tdSql.checkData(0,0,20) tdSql.query("select count(col9) from db.stb") - tdSql.checkData(0,0,10) + tdSql.checkData(0,0,20) tdSql.query("select count(col11) from stb") - tdSql.checkData(0,0,10) + tdSql.checkData(0,0,20) tdSql.query("select count(col11) from db.stb") - tdSql.checkData(0,0,10) + tdSql.checkData(0,0,20) tdSql.query("select count(col12) from stb") - tdSql.checkData(0,0,10) + tdSql.checkData(0,0,20) tdSql.query("select count(col12) from db.stb") - tdSql.checkData(0,0,10) + tdSql.checkData(0,0,20) tdSql.query("select count(col13) from stb") - tdSql.checkData(0,0,10) + tdSql.checkData(0,0,20) tdSql.query("select count(col13) from db.stb") - tdSql.checkData(0,0,10) + tdSql.checkData(0,0,20) tdSql.query("select count(col14) from stb") - tdSql.checkData(0,0,10) + tdSql.checkData(0,0,20) tdSql.query("select count(col14) from db.stb") - tdSql.checkData(0,0,10) + tdSql.checkData(0,0,20) + + + tdSql.query("select count(ts) from db.stb_1") tdSql.checkData(0,0,10) tdSql.query("select count(ts) from db.stb_1") @@ -144,6 +150,8 @@ class TDTestCase: tdSql.checkData(0,0,10) tdSql.query("select count(col14) from db.stb_1") tdSql.checkData(0,0,10) + tdSql.query("select count(col1) from stb_1 group by col7") + tdSql.checkRows(2) tdSql.execute("insert into stb_1 values(now,null,null,null,null,null,null,null,null,null,null,null,null,null)") tdSql.query("select count(col1) from stb_1") @@ -159,16 +167,22 @@ class TDTestCase: tdSql.checkData(0,1,11) tdSql.query("select count(col1) from stb") - tdSql.checkData(0,0,10) + tdSql.checkData(0,0,20) tdSql.query("select count(col1),count(ts) from stb") - tdSql.checkData(0,0,10) - tdSql.checkData(0,1,11) + tdSql.checkData(0,0,20) + tdSql.checkData(0,1,21) tdSql.query("select count(col1) from db.stb") - tdSql.checkData(0,0,10) + tdSql.checkData(0,0,20) tdSql.query("select count(col1),count(ts) from db.stb") - tdSql.checkData(0,0,10) - tdSql.checkData(0,1,11) + tdSql.checkData(0,0,20) + tdSql.checkData(0,1,21) + tdSql.query("select count(col1) from stb_1 group by col7") + tdSql.checkRows(3) + tdSql.query("select count(col1) from stb_2 group by col7") + tdSql.checkRows(2) + tdSql.query("select count(col1) from stb group by col7") + tdSql.checkRows(3) diff --git a/tests/system-test/2-query/last.py b/tests/system-test/2-query/last.py index fe5beefea5..a5407c7b12 100644 --- a/tests/system-test/2-query/last.py +++ b/tests/system-test/2-query/last.py @@ -79,6 +79,8 @@ class TDTestCase: tdSql.checkRows(0) tdSql.query("select last(col9) from db.stb_1") tdSql.checkRows(0) + tdSql.query("select count(col1) from stb_1 group by col7") + tdSql.checkRows(1) for i in range(self.rowNum): tdSql.execute("insert into stb_1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" @@ -170,7 +172,6 @@ class TDTestCase: # tdSql.checkData(0, 0, '涛思数据10') - tdSql.execute('''create table ntb(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, col7 bool, col8 binary(20), col9 nchar(20), col11 tinyint unsigned, col12 smallint unsigned, col13 int unsigned, col14 bigint unsigned)''') tdSql.execute("insert into ntb(ts) values(%d)" % (self.ts - 1)) From a65d5a47c7c3139d496aaa68e738a968dffc9c33 Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Fri, 6 May 2022 20:47:45 +0800 Subject: [PATCH 39/83] add case for tan functions --- tests/system-test/2-query/tan.py | 518 +++++++++++++++++++++++++++++++ tests/system-test/fulltest.sh | 3 +- 2 files changed, 520 insertions(+), 1 deletion(-) create mode 100644 tests/system-test/2-query/tan.py diff --git a/tests/system-test/2-query/tan.py b/tests/system-test/2-query/tan.py new file mode 100644 index 0000000000..c8914d18c0 --- /dev/null +++ b/tests/system-test/2-query/tan.py @@ -0,0 +1,518 @@ +import taos +import sys +import datetime +import inspect +import math +from util.log import * +from util.sql import * +from util.cases import * + + +class TDTestCase: + updatecfgDict = {'debugFlag': 143 ,"cDebugFlag":143,"uDebugFlag":143 ,"rpcDebugFlag":143 , "tmrDebugFlag":143 , + "jniDebugFlag":143 ,"simDebugFlag":143,"dDebugFlag":143, "dDebugFlag":143,"vDebugFlag":143,"mDebugFlag":143,"qDebugFlag":143, + "wDebugFlag":143,"sDebugFlag":143,"tsdbDebugFlag":143,"tqDebugFlag":143 ,"fsDebugFlag":143 ,"fnDebugFlag":143} + def init(self, conn, powSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + + def prepare_datas(self): + tdSql.execute( + '''create table stb1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + tags (t1 int) + ''' + ) + + tdSql.execute( + ''' + create table t1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + ''' + ) + for i in range(4): + tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )') + + for i in range(9): + tdSql.execute( + f"insert into ct1 values ( now()-{i*10}s, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + ) + tdSql.execute( + f"insert into ct4 values ( now()-{i*90}d, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + ) + tdSql.execute("insert into ct1 values (now()-45s, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a )") + tdSql.execute("insert into ct1 values (now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute("insert into ct1 values (now()+15s, 9, -99999, -999, -99, -9.99, NULL, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute("insert into ct1 values (now()+20s, 9, -99999, -999, NULL, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + + tdSql.execute("insert into ct4 values (now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute("insert into ct4 values (now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute("insert into ct4 values (now()+90d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + + tdSql.execute( + f'''insert into t1 values + ( '2020-04-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2020-10-21 01:01:01.000', 1, 11111, 111, 11, 1.11, 11.11, 1, "binary1", "nchar1", now()+1a ) + ( '2020-12-31 01:01:01.000', 2, 22222, 222, 22, 2.22, 22.22, 0, "binary2", "nchar2", now()+2a ) + ( '2021-01-01 01:01:06.000', 3, 33333, 333, 33, 3.33, 33.33, 0, "binary3", "nchar3", now()+3a ) + ( '2021-05-07 01:01:10.000', 4, 44444, 444, 44, 4.44, 44.44, 1, "binary4", "nchar4", now()+4a ) + ( '2021-07-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2021-09-30 01:01:16.000', 5, 55555, 555, 55, 5.55, 55.55, 0, "binary5", "nchar5", now()+5a ) + ( '2022-02-01 01:01:20.000', 6, 66666, 666, 66, 6.66, 66.66, 1, "binary6", "nchar6", now()+6a ) + ( '2022-10-28 01:01:26.000', 7, 00000, 000, 00, 0.00, 00.00, 1, "binary7", "nchar7", "1970-01-01 08:00:00.000" ) + ( '2022-12-01 01:01:30.000', 8, -88888, -888, -88, -8.88, -88.88, 0, "binary8", "nchar8", "1969-01-01 01:00:00.000" ) + ( '2022-12-31 01:01:36.000', 9, -99999999999999999, -999, -99, -9.99, -999999999999999999999.99, 1, "binary9", "nchar9", "1900-01-01 00:00:00.000" ) + ( '2023-02-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ''' + ) + + def check_result_auto_tan(self ,origin_query , pow_query): + + pow_result = tdSql.getResult(pow_query) + origin_result = tdSql.getResult(origin_query) + + auto_result =[] + + for row in origin_result: + row_check = [] + for elem in row: + if elem == None: + elem = None + else: + elem = math.tan(elem) + row_check.append(elem) + auto_result.append(row_check) + + check_status = True + + for row_index , row in enumerate(pow_result): + for col_index , elem in enumerate(row): + if auto_result[row_index][col_index] == None and not (auto_result[row_index][col_index] == None and elem == None): + check_status = False + elif auto_result[row_index][col_index] != None and (auto_result[row_index][col_index] - elem > 0.00000001): + check_status = False + else: + pass + if not check_status: + tdLog.notice("tan function value has not as expected , sql is \"%s\" "%pow_query ) + sys.exit(1) + else: + tdLog.info("tan value check pass , it work as expected ,sql is \"%s\" "%pow_query ) + + def test_errors(self): + error_sql_lists = [ + "select tan from t1", + # "select tan(-+--+c1 ) from t1", + # "select +-tan(c1) from t1", + # "select ++-tan(c1) from t1", + # "select ++--tan(c1) from t1", + # "select - -tan(c1)*0 from t1", + # "select tan(tbname+1) from t1 ", + "select tan(123--123)==1 from t1", + "select tan(c1) as 'd1' from t1", + "select tan(c1 ,c2) from t1", + "select tan(c1 ,NULL ) from t1", + "select tan(,) from t1;", + "select tan(tan(c1) ab from t1)", + "select tan(c1 ) as int from t1", + "select tan from stb1", + # "select tan(-+--+c1) from stb1", + # "select +-tan(c1) from stb1", + # "select ++-tan(c1) from stb1", + # "select ++--tan(c1) from stb1", + # "select - -tan(c1)*0 from stb1", + # "select tan(tbname+1) from stb1 ", + "select tan(123--123)==1 from stb1", + "select tan(c1) as 'd1' from stb1", + "select tan(c1 ,c2 ) from stb1", + "select tan(c1 ,NULL) from stb1", + "select tan(,) from stb1;", + "select tan(tan(c1) ab from stb1)", + "select tan(c1) as int from stb1" + ] + for error_sql in error_sql_lists: + tdSql.error(error_sql) + + def support_types(self): + type_error_sql_lists = [ + "select tan(ts) from t1" , + "select tan(c7) from t1", + "select tan(c8) from t1", + "select tan(c9) from t1", + "select tan(ts) from ct1" , + "select tan(c7) from ct1", + "select tan(c8) from ct1", + "select tan(c9) from ct1", + "select tan(ts) from ct3" , + "select tan(c7) from ct3", + "select tan(c8) from ct3", + "select tan(c9) from ct3", + "select tan(ts) from ct4" , + "select tan(c7) from ct4", + "select tan(c8) from ct4", + "select tan(c9) from ct4", + "select tan(ts) from stb1" , + "select tan(c7) from stb1", + "select tan(c8) from stb1", + "select tan(c9) from stb1" , + + "select tan(ts) from stbbb1" , + "select tan(c7) from stbbb1", + + "select tan(ts) from tbname", + "select tan(c9) from tbname" + + ] + + for type_sql in type_error_sql_lists: + tdSql.error(type_sql) + + + type_sql_lists = [ + "select tan(c1) from t1", + "select tan(c2) from t1", + "select tan(c3) from t1", + "select tan(c4) from t1", + "select tan(c5) from t1", + "select tan(c6) from t1", + + "select tan(c1) from ct1", + "select tan(c2) from ct1", + "select tan(c3) from ct1", + "select tan(c4) from ct1", + "select tan(c5) from ct1", + "select tan(c6) from ct1", + + "select tan(c1) from ct3", + "select tan(c2) from ct3", + "select tan(c3) from ct3", + "select tan(c4) from ct3", + "select tan(c5) from ct3", + "select tan(c6) from ct3", + + "select tan(c1) from stb1", + "select tan(c2) from stb1", + "select tan(c3) from stb1", + "select tan(c4) from stb1", + "select tan(c5) from stb1", + "select tan(c6) from stb1", + + "select tan(c6) as alisb from stb1", + "select tan(c6) alisb from stb1", + ] + + for type_sql in type_sql_lists: + tdSql.query(type_sql) + + def basic_tan_function(self): + + # basic query + tdSql.query("select c1 from ct3") + tdSql.checkRows(0) + tdSql.query("select c1 from t1") + tdSql.checkRows(12) + tdSql.query("select c1 from stb1") + tdSql.checkRows(25) + + # used for empty table , ct3 is empty + tdSql.query("select tan(c1) from ct3") + tdSql.checkRows(0) + tdSql.query("select tan(c2) from ct3") + tdSql.checkRows(0) + tdSql.query("select tan(c3) from ct3") + tdSql.checkRows(0) + tdSql.query("select tan(c4) from ct3") + tdSql.checkRows(0) + tdSql.query("select tan(c5) from ct3") + tdSql.checkRows(0) + tdSql.query("select tan(c6) from ct3") + tdSql.checkRows(0) + + + # # used for regular table + tdSql.query("select tan(c1) from t1") + tdSql.checkData(0, 0, None) + tdSql.checkData(1 , 0, 1.557407725) + tdSql.checkData(3 , 0, -0.142546543) + tdSql.checkData(5 , 0, None) + + tdSql.query("select c1, c2, c3 , c4, c5 from t1") + tdSql.checkData(1, 4, 1.11000) + tdSql.checkData(3, 3, 33) + tdSql.checkData(5, 4, None) + + tdSql.query("select ts,c1, c2, c3 , c4, c5 from t1") + tdSql.checkData(1, 5, 1.11000) + tdSql.checkData(3, 4, 33) + tdSql.checkData(5, 5, None) + + self.check_result_auto_tan( "select abs(c1), abs(c2), abs(c3) , abs(c4), abs(c5) from t1", "select tan(abs(c1)), tan(abs(c2)) ,tan(abs(c3)), tan(abs(c4)), tan(abs(c5)) from t1") + + # used for sub table + tdSql.query("select c2 ,tan(c2) from ct1") + tdSql.checkData(0, 1, -0.226288661) + tdSql.checkData(1 , 1, 0.670533806) + tdSql.checkData(3 , 1, -1.325559275) + tdSql.checkData(4 , 1, 0.000000000) + + tdSql.query("select c1, c5 ,tan(c5) from ct4") + tdSql.checkData(0 , 2, None) + tdSql.checkData(1 , 2, -0.605942929) + tdSql.checkData(2 , 2, 11.879355609) + tdSql.checkData(3 , 2, 0.395723765) + tdSql.checkData(5 , 2, None) + + self.check_result_auto_tan( "select c1, c2, c3 , c4, c5 from ct1", "select tan(c1), tan(c2) ,tan(c3), tan(c4), tan(c5) from ct1") + + # nest query for tan functions + tdSql.query("select c4 , tan(c4) ,tan(tan(c4)) , tan(tan(tan(c4))) from ct1;") + tdSql.checkData(0 , 0 , 88) + tdSql.checkData(0 , 1 , 0.035420501) + tdSql.checkData(0 , 2 , 0.035435322) + tdSql.checkData(0 , 3 , 0.035450161) + + tdSql.checkData(1 , 0 , 77) + tdSql.checkData(1 , 1 , -32.268575776) + tdSql.checkData(1 , 2 , -1.144433254) + tdSql.checkData(1 , 3 , -2.201545113) + + tdSql.checkData(11 , 0 , -99) + tdSql.checkData(11 , 1 , 25.092534980) + tdSql.checkData(11 , 2 , -0.040227928) + tdSql.checkData(11 , 3 , -0.040249642) + + # used for stable table + + tdSql.query("select tan(c1) from stb1") + tdSql.checkRows(25) + + + # used for not exists table + tdSql.error("select tan(c1) from stbbb1") + tdSql.error("select tan(c1) from tbname") + tdSql.error("select tan(c1) from ct5") + + # mix with common col + tdSql.query("select c1, tan(c1) from ct1") + tdSql.query("select c2, tan(c2) from ct4") + + + # mix with common functions + tdSql.query("select c1, tan(c1),tan(c1), tan(tan(c1)) from ct4 ") + tdSql.checkData(0 , 0 ,None) + tdSql.checkData(0 , 1 ,None) + tdSql.checkData(0 , 2 ,None) + tdSql.checkData(0 , 3 ,None) + + tdSql.checkData(3 , 0 , 6) + tdSql.checkData(3 , 1 ,-0.291006191) + tdSql.checkData(3 , 2 ,-0.291006191) + tdSql.checkData(3 , 3 ,-0.299508909) + + tdSql.query("select c1, tan(c1),c5, floor(c5) from stb1 ") + + # # mix with agg functions , not support + tdSql.error("select c1, tan(c1),c5, count(c5) from stb1 ") + tdSql.error("select c1, tan(c1),c5, count(c5) from ct1 ") + tdSql.error("select tan(c1), count(c5) from stb1 ") + tdSql.error("select tan(c1), count(c5) from ct1 ") + tdSql.error("select c1, count(c5) from ct1 ") + tdSql.error("select c1, count(c5) from stb1 ") + + # agg functions mix with agg functions + + tdSql.query("select max(c5), count(c5) from stb1") + tdSql.query("select max(c5), count(c5) from ct1") + + + # # bug fix for compute + tdSql.query("select c1, tan(c1) -0 ,tan(c1-4)-0 from ct4 ") + tdSql.checkData(0, 0, None) + tdSql.checkData(0, 1, None) + tdSql.checkData(0, 2, None) + tdSql.checkData(1, 0, 8) + tdSql.checkData(1, 1, -6.799711455) + tdSql.checkData(1, 2, 1.157821282) + + tdSql.query(" select c1, tan(c1) -0 ,tan(c1-0.1)-0.1 from ct4") + tdSql.checkData(0, 0, None) + tdSql.checkData(0, 1, None) + tdSql.checkData(0, 2, None) + tdSql.checkData(1, 0, 8) + tdSql.checkData(1, 1, -6.799711455) + tdSql.checkData(1, 2, -21.815112681) + + tdSql.query("select c1, tan(c1), c2, tan(c2), c3, tan(c3) from ct1") + + def test_big_number(self): + + tdSql.query("select c1, tan(100000000) from ct1") # bigint to double data overflow + tdSql.checkData(4, 1, math.tan(100000000)) + + + tdSql.query("select c1, tan(10000000000000) from ct1") # bigint to double data overflow + tdSql.checkData(4, 1, math.tan(10000000000000)) + + tdSql.query("select c1, tan(10000000000000000000000000) from ct1") # bigint to double data overflow + tdSql.query("select c1, tan(10000000000000000000000000.0) from ct1") # 10000000000000000000000000.0 is a double value + tdSql.checkData(1, 1, math.tan(10000000000000000000000000.0)) + + tdSql.query("select c1, tan(10000000000000000000000000000000000) from ct1") # bigint to double data overflow + tdSql.query("select c1, tan(10000000000000000000000000000000000.0) from ct1") # 10000000000000000000000000.0 is a double value + tdSql.checkData(4, 1, math.tan(10000000000000000000000000000000000.0)) + + tdSql.query("select c1, tan(10000000000000000000000000000000000000000) from ct1") # bigint to double data overflow + tdSql.query("select c1, tan(10000000000000000000000000000000000000000.0) from ct1") # 10000000000000000000000000.0 is a double value + + tdSql.checkData(4, 1, math.tan(10000000000000000000000000000000000000000.0)) + + tdSql.query("select c1, tan(10000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000) from ct1") # bigint to double data overflow + + def abs_func_filter(self): + tdSql.execute("use db") + tdSql.query("select c1, abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(tan(c1)-0.5) from ct4 where c1>5 ") + tdSql.checkRows(3) + tdSql.checkData(0,0,8) + tdSql.checkData(0,1,8.000000000) + tdSql.checkData(0,2,8.000000000) + tdSql.checkData(0,3,7.900000000) + tdSql.checkData(0,4,-7.000000000) + + tdSql.query("select c1, abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(tan(c1)-0.5) from ct4 where c1=5 ") + tdSql.checkRows(1) + tdSql.checkData(0,0,5) + tdSql.checkData(0,1,5.000000000) + tdSql.checkData(0,2,5.000000000) + tdSql.checkData(0,3,4.900000000) + tdSql.checkData(0,4,-3.000000000) + + tdSql.query("select c1,c2 , abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(tan(c1)-0.5) from ct4 where c1>tan(c1) limit 1 ") + tdSql.checkRows(1) + tdSql.checkData(0,0,8) + tdSql.checkData(0,1,88888) + tdSql.checkData(0,2,8.000000000) + tdSql.checkData(0,3,8.000000000) + tdSql.checkData(0,4,7.900000000) + tdSql.checkData(0,5,-7.000000000) + + def pow_Arithmetic(self): + pass + + def check_boundary_values(self): + + PI=3.1415926 + + tdSql.execute("drop database if exists bound_test") + tdSql.execute("create database if not exists bound_test") + time.sleep(3) + tdSql.execute("use bound_test") + tdSql.execute( + "create table stb_bound (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(32),c9 nchar(32), c10 timestamp) tags (t1 int);" + ) + tdSql.execute(f'create table sub1_bound using stb_bound tags ( 1 )') + tdSql.execute( + f"insert into sub1_bound values ( now()-1s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + tdSql.execute( + f"insert into sub1_bound values ( now()-1s, -2147483647, -9223372036854775807, -32767, -127, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + tdSql.execute( + f"insert into sub1_bound values ( now(), 2147483646, 9223372036854775806, 32766, 126, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + tdSql.execute( + f"insert into sub1_bound values ( now(), -2147483646, -9223372036854775806, -32766, -126, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + tdSql.error( + f"insert into sub1_bound values ( now()+1s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + self.check_result_auto_tan( "select abs(c1), abs(c2), abs(c3) , abs(c4), abs(c5) from sub1_bound ", "select tan(abs(c1)), tan(abs(c2)) ,tan(abs(c3)), tan(abs(c4)), tan(abs(c5)) from sub1_bound") + + self.check_result_auto_tan( "select c1, c2, c3 , c3, c2 ,c1 from sub1_bound ", "select tan(c1), tan(c2) ,tan(c3), tan(c3), tan(c2) ,tan(c1) from sub1_bound") + + self.check_result_auto_tan("select abs(abs(abs(abs(abs(abs(abs(abs(abs(c1))))))))) nest_col_func from sub1_bound" , "select tan(abs(c1)) from sub1_bound" ) + + # check basic elem for table per row + tdSql.query("select tan(abs(c1)) ,tan(abs(c2)) , tan(abs(c3)) , tan(abs(c4)), tan(abs(c5)), tan(abs(c6)) from sub1_bound ") + tdSql.checkData(0,0,math.tan(2147483647)) + tdSql.checkData(0,1,math.tan(9223372036854775807)) + tdSql.checkData(0,2,math.tan(32767)) + tdSql.checkData(0,3,math.tan(127)) + tdSql.checkData(0,4,math.tan(339999995214436424907732413799364296704.00000)) + tdSql.checkData(1,0,math.tan(2147483647)) + tdSql.checkData(1,1,math.tan(9223372036854775807)) + tdSql.checkData(1,2,math.tan(32767)) + tdSql.checkData(1,3,math.tan(127)) + tdSql.checkData(1,4,math.tan(339999995214436424907732413799364296704.00000)) + tdSql.checkData(3,0,math.tan(2147483646)) + tdSql.checkData(3,1,math.tan(9223372036854775806)) + tdSql.checkData(3,2,math.tan(32766)) + tdSql.checkData(3,3,math.tan(126)) + tdSql.checkData(3,4,math.tan(339999995214436424907732413799364296704.00000)) + + # check + - * / in functions + tdSql.query("select tan(abs(c1+1)) ,tan(abs(c2)) , tan(abs(c3*1)) , tan(abs(c4/2)), tan(abs(c5))/2, tan(abs(c6)) from sub1_bound ") + tdSql.checkData(0,0,math.tan(2147483648.000000000)) + tdSql.checkData(0,1,math.tan(9223372036854775807)) + tdSql.checkData(0,2,math.tan(32767.000000000)) + tdSql.checkData(0,3,math.tan(63.500000000)) + + tdSql.execute("create stable st (ts timestamp, num1 float, num2 double) tags (t1 int);") + tdSql.execute(f'create table tb1 using st tags (1)') + tdSql.execute(f'create table tb2 using st tags (2)') + tdSql.execute(f'create table tb3 using st tags (3)') + tdSql.execute('insert into tb1 values (now()-40s, {}, {})'.format(PI/2 ,PI/2 )) + tdSql.execute('insert into tb1 values (now()-30s, {}, {})'.format(PI ,PI )) + tdSql.execute('insert into tb1 values (now()-20s, {}, {})'.format(PI*1.5 ,PI*1.5)) + tdSql.execute('insert into tb1 values (now()-10s, {}, {})'.format(PI*2 ,PI*2)) + tdSql.execute('insert into tb1 values (now(), {}, {})'.format(PI*2.5 ,PI*2.5)) + + tdSql.execute('insert into tb2 values (now()-40s, {}, {})'.format(PI/2 ,PI/2 )) + tdSql.execute('insert into tb2 values (now()-30s, {}, {})'.format(PI ,PI )) + tdSql.execute('insert into tb2 values (now()-20s, {}, {})'.format(PI*1.5 ,PI*1.5)) + tdSql.execute('insert into tb2 values (now()-10s, {}, {})'.format(PI*2 ,PI*2)) + tdSql.execute('insert into tb2 values (now(), {}, {})'.format(PI*2.5 ,PI*2.5)) + + for i in range(100): + tdSql.execute('insert into tb3 values (now()+{}s, {}, {})'.format(i,PI*(5+i)/2 ,PI*(5+i)/2)) + + self.check_result_auto_tan("select num1,num2 from tb3;" , "select tan(num1),tan(num2) from tb3") + + def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring + tdSql.prepare() + + tdLog.printNoPrefix("==========step1:create table ==============") + + self.prepare_datas() + + tdLog.printNoPrefix("==========step2:test errors ==============") + + self.test_errors() + + tdLog.printNoPrefix("==========step3:support types ============") + + self.support_types() + + tdLog.printNoPrefix("==========step4: tan basic query ============") + + self.basic_tan_function() + + tdLog.printNoPrefix("==========step5: big number tan query ============") + + self.test_big_number() + + + tdLog.printNoPrefix("==========step6: tan boundary query ============") + + self.check_boundary_values() + + tdLog.printNoPrefix("==========step7: tan filter query ============") + + self.abs_func_filter() + + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index bdef427fdb..fd9248d892 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -25,4 +25,5 @@ python3 ./test.py -f 2-query/round.py python3 ./test.py -f 2-query/log.py python3 ./test.py -f 2-query/pow.py python3 ./test.py -f 2-query/sin.py -python3 ./test.py -f 2-query/cos.py \ No newline at end of file +python3 ./test.py -f 2-query/cos.py +python3 ./test.py -f 2-query/tan.py \ No newline at end of file From aed53ac9e168a5efcb38571f10bf6dba87eefcfb Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 6 May 2022 13:06:10 +0000 Subject: [PATCH 40/83] feat: create table if not exists --- include/common/tmsg.h | 4 ++-- source/common/src/tmsg.c | 10 +++++----- source/dnode/vnode/src/vnd/vnodeSvr.c | 6 +++++- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 3889784f2f..763ff61e80 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -672,7 +672,6 @@ typedef struct { SArray* pArray; // Array of SUseDbRsp } SUseDbBatchRsp; - int32_t tSerializeSUseDbBatchRsp(void* buf, int32_t bufLen, SUseDbBatchRsp* pRsp); int32_t tDeserializeSUseDbBatchRsp(void* buf, int32_t bufLen, SUseDbBatchRsp* pRsp); void tFreeSUseDbBatchRsp(SUseDbBatchRsp* pRsp); @@ -685,7 +684,6 @@ int32_t tSerializeSUserAuthBatchRsp(void* buf, int32_t bufLen, SUserAuthBatchRsp int32_t tDeserializeSUserAuthBatchRsp(void* buf, int32_t bufLen, SUserAuthBatchRsp* pRsp); void tFreeSUserAuthBatchRsp(SUserAuthBatchRsp* pRsp); - typedef struct { char db[TSDB_DB_FNAME_LEN]; } SCompactDbReq; @@ -1554,7 +1552,9 @@ typedef struct SVDropStbReq { int32_t tEncodeSVDropStbReq(SCoder* pCoder, const SVDropStbReq* pReq); int32_t tDecodeSVDropStbReq(SCoder* pCoder, SVDropStbReq* pReq); +#define TD_CREATE_IF_NOT_EXISTS 0x1 typedef struct SVCreateTbReq { + int32_t flags; tb_uid_t uid; int64_t ctime; const char* name; diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 6278b52a04..85845dd5c4 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -1255,7 +1255,6 @@ int32_t tDeserializeSGetUserAuthRspImpl(SCoder *pDecoder, SGetUserAuthRsp *pRsp) return 0; } - int32_t tDeserializeSGetUserAuthRsp(void *buf, int32_t bufLen, SGetUserAuthRsp *pRsp) { SCoder decoder = {0}; tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); @@ -2091,7 +2090,7 @@ void tFreeSUseDbBatchRsp(SUseDbBatchRsp *pRsp) { taosArrayDestroy(pRsp->pArray); } -int32_t tSerializeSUserAuthBatchRsp(void* buf, int32_t bufLen, SUserAuthBatchRsp* pRsp){ +int32_t tSerializeSUserAuthBatchRsp(void *buf, int32_t bufLen, SUserAuthBatchRsp *pRsp) { SCoder encoder = {0}; tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); @@ -2110,7 +2109,7 @@ int32_t tSerializeSUserAuthBatchRsp(void* buf, int32_t bufLen, SUserAuthBatchRsp return tlen; } -int32_t tDeserializeSUserAuthBatchRsp(void* buf, int32_t bufLen, SUserAuthBatchRsp* pRsp){ +int32_t tDeserializeSUserAuthBatchRsp(void *buf, int32_t bufLen, SUserAuthBatchRsp *pRsp) { SCoder decoder = {0}; tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); @@ -2136,7 +2135,7 @@ int32_t tDeserializeSUserAuthBatchRsp(void* buf, int32_t bufLen, SUserAuthBatchR return 0; } -void tFreeSUserAuthBatchRsp(SUserAuthBatchRsp* pRsp){ +void tFreeSUserAuthBatchRsp(SUserAuthBatchRsp *pRsp) { int32_t numOfBatch = taosArrayGetSize(pRsp->pArray); for (int32_t i = 0; i < numOfBatch; ++i) { SGetUserAuthRsp *pUserAuthRsp = taosArrayGet(pRsp->pArray, i); @@ -2146,7 +2145,6 @@ void tFreeSUserAuthBatchRsp(SUserAuthBatchRsp* pRsp){ taosArrayDestroy(pRsp->pArray); } - int32_t tSerializeSDbCfgReq(void *buf, int32_t bufLen, SDbCfgReq *pReq) { SCoder encoder = {0}; tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); @@ -3746,6 +3744,7 @@ STSchema *tdGetSTSChemaFromSSChema(SSchema **pSchema, int32_t nCols) { int tEncodeSVCreateTbReq(SCoder *pCoder, const SVCreateTbReq *pReq) { if (tStartEncode(pCoder) < 0) return -1; + if (tEncodeI32v(pCoder, pReq->flags) < 0) return -1; if (tEncodeI64(pCoder, pReq->uid) < 0) return -1; if (tEncodeI64(pCoder, pReq->ctime) < 0) return -1; @@ -3771,6 +3770,7 @@ int tDecodeSVCreateTbReq(SCoder *pCoder, SVCreateTbReq *pReq) { if (tStartDecode(pCoder) < 0) return -1; + if (tDecodeI32v(pCoder, &pReq->flags) < 0) return -1; if (tDecodeI64(pCoder, &pReq->uid) < 0) return -1; if (tDecodeI64(pCoder, &pReq->ctime) < 0) return -1; diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 8460400b59..811c5c10e4 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -360,7 +360,11 @@ static int vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pReq, // do create table if (metaCreateTable(pVnode->pMeta, version, pCreateReq) < 0) { - cRsp.code = terrno; + if (pCreateReq->flags & TD_CREATE_IF_NOT_EXISTS && terrno == TSDB_CODE_TDB_TABLE_ALREADY_EXIST) { + cRsp.code = TSDB_CODE_SUCCESS; + } else { + cRsp.code = terrno; + } } else { cRsp.code = TSDB_CODE_SUCCESS; tsdbFetchTbUidList(pVnode->pTsdb, &pStore, pCreateReq->ctb.suid, pCreateReq->uid); From 3ac4beb2eea0aa86d3293a8b6e7d329f2960c233 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 6 May 2022 21:06:27 +0800 Subject: [PATCH 41/83] refactor:add compile option in windows --- cmake/cmake.define | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/cmake.define b/cmake/cmake.define index aeab39cab4..2b4fe5c42f 100644 --- a/cmake/cmake.define +++ b/cmake/cmake.define @@ -46,7 +46,7 @@ ENDIF () IF (TD_WINDOWS) MESSAGE("${Yellow} set compiler flag for Windows! ${ColourReset}") - SET(COMMON_FLAGS "/W3 /D_WIN32") + SET(COMMON_FLAGS "/W3 /D_WIN32 /vmg") # IF (MSVC AND (MSVC_VERSION GREATER_EQUAL 1900)) # SET(COMMON_FLAGS "${COMMON_FLAGS} /Wv:18") From 944a23c271072afac9c97a912bd390de47ee56ab Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Fri, 6 May 2022 21:25:58 +0800 Subject: [PATCH 42/83] add case for function arcsin --- tests/system-test/2-query/arcsin.py | 521 ++++++++++++++++++++++++++++ tests/system-test/fulltest.sh | 3 +- 2 files changed, 523 insertions(+), 1 deletion(-) create mode 100644 tests/system-test/2-query/arcsin.py diff --git a/tests/system-test/2-query/arcsin.py b/tests/system-test/2-query/arcsin.py new file mode 100644 index 0000000000..ee134a56a8 --- /dev/null +++ b/tests/system-test/2-query/arcsin.py @@ -0,0 +1,521 @@ +import taos +import sys +import datetime +import inspect +import math +from util.log import * +from util.sql import * +from util.cases import * + + +class TDTestCase: + updatecfgDict = {'debugFlag': 143 ,"cDebugFlag":143,"uDebugFlag":143 ,"rpcDebugFlag":143 , "tmrDebugFlag":143 , + "jniDebugFlag":143 ,"simDebugFlag":143,"dDebugFlag":143, "dDebugFlag":143,"vDebugFlag":143,"mDebugFlag":143,"qDebugFlag":143, + "wDebugFlag":143,"sDebugFlag":143,"tsdbDebugFlag":143,"tqDebugFlag":143 ,"fsDebugFlag":143 ,"fnDebugFlag":143} + def init(self, conn, powSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + self.PI =3.1415926 + + def prepare_datas(self): + tdSql.execute( + '''create table stb1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + tags (t1 int) + ''' + ) + + tdSql.execute( + ''' + create table t1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + ''' + ) + for i in range(4): + tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )') + + for i in range(9): + tdSql.execute( + f"insert into ct1 values ( now()-{i*10}s, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + ) + tdSql.execute( + f"insert into ct4 values ( now()-{i*90}d, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + ) + tdSql.execute("insert into ct1 values (now()-45s, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a )") + tdSql.execute("insert into ct1 values (now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute("insert into ct1 values (now()+15s, 9, -99999, -999, -99, -9.99, NULL, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute("insert into ct1 values (now()+20s, 9, -99999, -999, NULL, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + + tdSql.execute("insert into ct4 values (now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute("insert into ct4 values (now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute("insert into ct4 values (now()+90d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + + tdSql.execute( + f'''insert into t1 values + ( '2020-04-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2020-10-21 01:01:01.000', 1, 11111, 111, 11, 1.11, 11.11, 1, "binary1", "nchar1", now()+1a ) + ( '2020-12-31 01:01:01.000', 2, 22222, 222, 22, 2.22, 22.22, 0, "binary2", "nchar2", now()+2a ) + ( '2021-01-01 01:01:06.000', 3, 33333, 333, 33, 3.33, 33.33, 0, "binary3", "nchar3", now()+3a ) + ( '2021-05-07 01:01:10.000', 4, 44444, 444, 44, 4.44, 44.44, 1, "binary4", "nchar4", now()+4a ) + ( '2021-07-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2021-09-30 01:01:16.000', 5, 55555, 555, 55, 5.55, 55.55, 0, "binary5", "nchar5", now()+5a ) + ( '2022-02-01 01:01:20.000', 6, 66666, 666, 66, 6.66, 66.66, 1, "binary6", "nchar6", now()+6a ) + ( '2022-10-28 01:01:26.000', 7, 00000, 000, 00, 0.00, 00.00, 1, "binary7", "nchar7", "1970-01-01 08:00:00.000" ) + ( '2022-12-01 01:01:30.000', 8, -88888, -888, -88, -8.88, -88.88, 0, "binary8", "nchar8", "1969-01-01 01:00:00.000" ) + ( '2022-12-31 01:01:36.000', 9, -99999999999999999, -999, -99, -9.99, -999999999999999999999.99, 1, "binary9", "nchar9", "1900-01-01 00:00:00.000" ) + ( '2023-02-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ''' + ) + + def check_result_auto_asin(self ,origin_query , pow_query): + + pow_result = tdSql.getResult(pow_query) + origin_result = tdSql.getResult(origin_query) + + auto_result =[] + + for row in origin_result: + row_check = [] + for elem in row: + if elem == None: + elem = None + elif elem >=-1 and elem <=1: + elem = math.asin(elem) + else: + elem = None + row_check.append(elem) + auto_result.append(row_check) + + check_status = True + + for row_index , row in enumerate(pow_result): + for col_index , elem in enumerate(row): + if auto_result[row_index][col_index] == None and not (auto_result[row_index][col_index] == None and elem == None): + check_status = False + elif auto_result[row_index][col_index] != None and (auto_result[row_index][col_index] - elem > 0.00000001): + check_status = False + else: + pass + if not check_status: + tdLog.notice("asin function value has not as expected , sql is \"%s\" "%pow_query ) + sys.exit(1) + else: + tdLog.info("asin value check pass , it work as expected ,sql is \"%s\" "%pow_query ) + + def test_errors(self): + error_sql_lists = [ + "select asin from t1", + # "select asin(-+--+c1 ) from t1", + # "select +-asin(c1) from t1", + # "select ++-asin(c1) from t1", + # "select ++--asin(c1) from t1", + # "select - -asin(c1)*0 from t1", + # "select asin(tbname+1) from t1 ", + "select asin(123--123)==1 from t1", + "select asin(c1) as 'd1' from t1", + "select asin(c1 ,c2) from t1", + "select asin(c1 ,NULL ) from t1", + "select asin(,) from t1;", + "select asin(asin(c1) ab from t1)", + "select asin(c1 ) as int from t1", + "select asin from stb1", + # "select asin(-+--+c1) from stb1", + # "select +-asin(c1) from stb1", + # "select ++-asin(c1) from stb1", + # "select ++--asin(c1) from stb1", + # "select - -asin(c1)*0 from stb1", + # "select asin(tbname+1) from stb1 ", + "select asin(123--123)==1 from stb1", + "select asin(c1) as 'd1' from stb1", + "select asin(c1 ,c2 ) from stb1", + "select asin(c1 ,NULL) from stb1", + "select asin(,) from stb1;", + "select asin(asin(c1) ab from stb1)", + "select asin(c1) as int from stb1" + ] + for error_sql in error_sql_lists: + tdSql.error(error_sql) + + def support_types(self): + type_error_sql_lists = [ + "select asin(ts) from t1" , + "select asin(c7) from t1", + "select asin(c8) from t1", + "select asin(c9) from t1", + "select asin(ts) from ct1" , + "select asin(c7) from ct1", + "select asin(c8) from ct1", + "select asin(c9) from ct1", + "select asin(ts) from ct3" , + "select asin(c7) from ct3", + "select asin(c8) from ct3", + "select asin(c9) from ct3", + "select asin(ts) from ct4" , + "select asin(c7) from ct4", + "select asin(c8) from ct4", + "select asin(c9) from ct4", + "select asin(ts) from stb1" , + "select asin(c7) from stb1", + "select asin(c8) from stb1", + "select asin(c9) from stb1" , + + "select asin(ts) from stbbb1" , + "select asin(c7) from stbbb1", + + "select asin(ts) from tbname", + "select asin(c9) from tbname" + + ] + + for type_sql in type_error_sql_lists: + tdSql.error(type_sql) + + + type_sql_lists = [ + "select asin(c1) from t1", + "select asin(c2) from t1", + "select asin(c3) from t1", + "select asin(c4) from t1", + "select asin(c5) from t1", + "select asin(c6) from t1", + + "select asin(c1) from ct1", + "select asin(c2) from ct1", + "select asin(c3) from ct1", + "select asin(c4) from ct1", + "select asin(c5) from ct1", + "select asin(c6) from ct1", + + "select asin(c1) from ct3", + "select asin(c2) from ct3", + "select asin(c3) from ct3", + "select asin(c4) from ct3", + "select asin(c5) from ct3", + "select asin(c6) from ct3", + + "select asin(c1) from stb1", + "select asin(c2) from stb1", + "select asin(c3) from stb1", + "select asin(c4) from stb1", + "select asin(c5) from stb1", + "select asin(c6) from stb1", + + "select asin(c6) as alisb from stb1", + "select asin(c6) alisb from stb1", + ] + + for type_sql in type_sql_lists: + tdSql.query(type_sql) + + def basic_asin_function(self): + + # basic query + tdSql.query("select c1 from ct3") + tdSql.checkRows(0) + tdSql.query("select c1 from t1") + tdSql.checkRows(12) + tdSql.query("select c1 from stb1") + tdSql.checkRows(25) + + # used for empty table , ct3 is empty + tdSql.query("select asin(c1) from ct3") + tdSql.checkRows(0) + tdSql.query("select asin(c2) from ct3") + tdSql.checkRows(0) + tdSql.query("select asin(c3) from ct3") + tdSql.checkRows(0) + tdSql.query("select asin(c4) from ct3") + tdSql.checkRows(0) + tdSql.query("select asin(c5) from ct3") + tdSql.checkRows(0) + tdSql.query("select asin(c6) from ct3") + tdSql.checkRows(0) + + + # # used for regular table + tdSql.query("select asin(c1) from t1") + tdSql.checkData(0, 0, None) + tdSql.checkData(1 , 0, 1.570796327) + tdSql.checkData(3 , 0, None) + tdSql.checkData(5 , 0, None) + + tdSql.query("select c1, c2, c3 , c4, c5 from t1") + tdSql.checkData(1, 4, 1.11000) + tdSql.checkData(3, 3, 33) + tdSql.checkData(5, 4, None) + + tdSql.query("select ts,c1, c2, c3 , c4, c5 from t1") + tdSql.checkData(1, 5, 1.11000) + tdSql.checkData(3, 4, 33) + tdSql.checkData(5, 5, None) + + self.check_result_auto_asin( "select abs(c1), abs(c2), abs(c3) , abs(c4), abs(c5) from t1", "select asin(abs(c1)), asin(abs(c2)) ,asin(abs(c3)), asin(abs(c4)), asin(abs(c5)) from t1") + + # used for sub table + tdSql.query("select c2 ,asin(c2) from ct1") + tdSql.checkData(0, 1, None) + tdSql.checkData(1 , 1, None) + tdSql.checkData(3 , 1, None) + tdSql.checkData(4 , 1, 0.000000000) + + tdSql.query("select c1, c5 ,asin(c5) from ct4") + tdSql.checkData(0 , 2, None) + tdSql.checkData(1 , 2, None) + tdSql.checkData(2 , 2, None) + tdSql.checkData(3 , 2, None) + tdSql.checkData(5 , 2, None) + + self.check_result_auto_asin( "select c1, c2, c3 , c4, c5 from ct1", "select asin(c1), asin(c2) ,asin(c3), asin(c4), asin(c5) from ct1") + + # nest query for asin functions + tdSql.query("select c4 , asin(c4) ,asin(asin(c4)) , asin(asin(asin(c4))) from ct1;") + tdSql.checkData(0 , 0 , 88) + tdSql.checkData(0 , 1 , None) + tdSql.checkData(0 , 2 , None) + tdSql.checkData(0 , 3 , None) + + tdSql.checkData(1 , 0 , 77) + tdSql.checkData(1 , 1 , None) + tdSql.checkData(1 , 2 , None) + tdSql.checkData(1 , 3 , None) + + tdSql.checkData(11 , 0 , -99) + tdSql.checkData(11 , 1 , None) + tdSql.checkData(11 , 2 , None) + tdSql.checkData(11 , 3 , None) + + # used for stable table + + tdSql.query("select asin(c1) from stb1") + tdSql.checkRows(25) + + + # used for not exists table + tdSql.error("select asin(c1) from stbbb1") + tdSql.error("select asin(c1) from tbname") + tdSql.error("select asin(c1) from ct5") + + # mix with common col + tdSql.query("select c1, asin(c1) from ct1") + tdSql.query("select c2, asin(c2) from ct4") + + + # mix with common functions + tdSql.query("select c1, asin(c1),asin(c1), asin(asin(c1)) from ct4 ") + tdSql.checkData(0 , 0 ,None) + tdSql.checkData(0 , 1 ,None) + tdSql.checkData(0 , 2 ,None) + tdSql.checkData(0 , 3 ,None) + + tdSql.checkData(3 , 0 , 6) + tdSql.checkData(3 , 1 ,None) + tdSql.checkData(3 , 2 ,None) + tdSql.checkData(3 , 3 ,None) + + tdSql.query("select c1, asin(c1),c5, floor(c5) from stb1 ") + + # # mix with agg functions , not support + tdSql.error("select c1, asin(c1),c5, count(c5) from stb1 ") + tdSql.error("select c1, asin(c1),c5, count(c5) from ct1 ") + tdSql.error("select asin(c1), count(c5) from stb1 ") + tdSql.error("select asin(c1), count(c5) from ct1 ") + tdSql.error("select c1, count(c5) from ct1 ") + tdSql.error("select c1, count(c5) from stb1 ") + + # agg functions mix with agg functions + + tdSql.query("select max(c5), count(c5) from stb1") + tdSql.query("select max(c5), count(c5) from ct1") + + + # # bug fix for compute + tdSql.query("select c1, asin(c1) -0 ,asin(c1-4)-0 from ct4 ") + tdSql.checkData(0, 0, None) + tdSql.checkData(0, 1, None) + tdSql.checkData(0, 2, None) + tdSql.checkData(1, 0, 8) + tdSql.checkData(1, 1, None) + tdSql.checkData(1, 2, None) + + tdSql.query(" select c1, asin(c1) -0 ,asin(c1-0.1)-0.1 from ct4") + tdSql.checkData(0, 0, None) + tdSql.checkData(0, 1, None) + tdSql.checkData(0, 2, None) + tdSql.checkData(1, 0, 8) + tdSql.checkData(1, 1, None) + tdSql.checkData(1, 2, None) + + tdSql.query("select c1, asin(c1), c2, asin(c2), c3, asin(c3) from ct1") + + def test_big_number(self): + + tdSql.query("select c1, asin(100000000) from ct1") # bigint to double data overflow + tdSql.checkData(4, 1, None) + + + tdSql.query("select c1, asin(10000000000000) from ct1") # bigint to double data overflow + tdSql.checkData(4, 1, None) + + tdSql.query("select c1, asin(10000000000000000000000000) from ct1") # bigint to double data overflow + tdSql.query("select c1, asin(10000000000000000000000000.0) from ct1") # 10000000000000000000000000.0 is a double value + tdSql.checkData(1, 1, None) + + tdSql.query("select c1, asin(10000000000000000000000000000000000) from ct1") # bigint to double data overflow + tdSql.query("select c1, asin(10000000000000000000000000000000000.0) from ct1") # 10000000000000000000000000.0 is a double value + tdSql.checkData(4, 1, None) + + tdSql.query("select c1, asin(10000000000000000000000000000000000000000) from ct1") # bigint to double data overflow + tdSql.query("select c1, asin(10000000000000000000000000000000000000000.0) from ct1") # 10000000000000000000000000.0 is a double value + + tdSql.checkData(4, 1, None) + + tdSql.query("select c1, asin(10000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000) from ct1") # bigint to double data overflow + + def abs_func_filter(self): + tdSql.execute("use db") + tdSql.query("select c1, abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(asin(c1)-0.5) from ct4 where c1>5 ") + tdSql.checkRows(3) + tdSql.checkData(0,0,8) + tdSql.checkData(0,1,8.000000000) + tdSql.checkData(0,2,8.000000000) + tdSql.checkData(0,3,7.900000000) + tdSql.checkData(0,4,None) + + tdSql.query("select c1, abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(asin(c1)-0.5) from ct4 where c1=5 ") + tdSql.checkRows(1) + tdSql.checkData(0,0,5) + tdSql.checkData(0,1,5.000000000) + tdSql.checkData(0,2,5.000000000) + tdSql.checkData(0,3,4.900000000) + tdSql.checkData(0,4,None) + + tdSql.query("select c1,c2 , abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(asin(c1)-0.5) from ct4 where c1 Date: Fri, 6 May 2022 21:34:26 +0800 Subject: [PATCH 43/83] add case for function arccos --- tests/system-test/2-query/arccos.py | 521 ++++++++++++++++++++++++++++ tests/system-test/fulltest.sh | 3 +- 2 files changed, 523 insertions(+), 1 deletion(-) create mode 100644 tests/system-test/2-query/arccos.py diff --git a/tests/system-test/2-query/arccos.py b/tests/system-test/2-query/arccos.py new file mode 100644 index 0000000000..e15f7675f2 --- /dev/null +++ b/tests/system-test/2-query/arccos.py @@ -0,0 +1,521 @@ +import taos +import sys +import datetime +import inspect +import math +from util.log import * +from util.sql import * +from util.cases import * + + +class TDTestCase: + updatecfgDict = {'debugFlag': 143 ,"cDebugFlag":143,"uDebugFlag":143 ,"rpcDebugFlag":143 , "tmrDebugFlag":143 , + "jniDebugFlag":143 ,"simDebugFlag":143,"dDebugFlag":143, "dDebugFlag":143,"vDebugFlag":143,"mDebugFlag":143,"qDebugFlag":143, + "wDebugFlag":143,"sDebugFlag":143,"tsdbDebugFlag":143,"tqDebugFlag":143 ,"fsDebugFlag":143 ,"fnDebugFlag":143} + def init(self, conn, powSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + self.PI =3.1415926 + + def prepare_datas(self): + tdSql.execute( + '''create table stb1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + tags (t1 int) + ''' + ) + + tdSql.execute( + ''' + create table t1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + ''' + ) + for i in range(4): + tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )') + + for i in range(9): + tdSql.execute( + f"insert into ct1 values ( now()-{i*10}s, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + ) + tdSql.execute( + f"insert into ct4 values ( now()-{i*90}d, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + ) + tdSql.execute("insert into ct1 values (now()-45s, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a )") + tdSql.execute("insert into ct1 values (now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute("insert into ct1 values (now()+15s, 9, -99999, -999, -99, -9.99, NULL, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute("insert into ct1 values (now()+20s, 9, -99999, -999, NULL, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + + tdSql.execute("insert into ct4 values (now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute("insert into ct4 values (now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute("insert into ct4 values (now()+90d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + + tdSql.execute( + f'''insert into t1 values + ( '2020-04-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2020-10-21 01:01:01.000', 1, 11111, 111, 11, 1.11, 11.11, 1, "binary1", "nchar1", now()+1a ) + ( '2020-12-31 01:01:01.000', 2, 22222, 222, 22, 2.22, 22.22, 0, "binary2", "nchar2", now()+2a ) + ( '2021-01-01 01:01:06.000', 3, 33333, 333, 33, 3.33, 33.33, 0, "binary3", "nchar3", now()+3a ) + ( '2021-05-07 01:01:10.000', 4, 44444, 444, 44, 4.44, 44.44, 1, "binary4", "nchar4", now()+4a ) + ( '2021-07-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2021-09-30 01:01:16.000', 5, 55555, 555, 55, 5.55, 55.55, 0, "binary5", "nchar5", now()+5a ) + ( '2022-02-01 01:01:20.000', 6, 66666, 666, 66, 6.66, 66.66, 1, "binary6", "nchar6", now()+6a ) + ( '2022-10-28 01:01:26.000', 7, 00000, 000, 00, 0.00, 00.00, 1, "binary7", "nchar7", "1970-01-01 08:00:00.000" ) + ( '2022-12-01 01:01:30.000', 8, -88888, -888, -88, -8.88, -88.88, 0, "binary8", "nchar8", "1969-01-01 01:00:00.000" ) + ( '2022-12-31 01:01:36.000', 9, -99999999999999999, -999, -99, -9.99, -999999999999999999999.99, 1, "binary9", "nchar9", "1900-01-01 00:00:00.000" ) + ( '2023-02-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ''' + ) + + def check_result_auto_acos(self ,origin_query , pow_query): + + pow_result = tdSql.getResult(pow_query) + origin_result = tdSql.getResult(origin_query) + + auto_result =[] + + for row in origin_result: + row_check = [] + for elem in row: + if elem == None: + elem = None + elif elem >=-1 and elem <=1: + elem = math.acos(elem) + else: + elem = None + row_check.append(elem) + auto_result.append(row_check) + + check_status = True + + for row_index , row in enumerate(pow_result): + for col_index , elem in enumerate(row): + if auto_result[row_index][col_index] == None and not (auto_result[row_index][col_index] == None and elem == None): + check_status = False + elif auto_result[row_index][col_index] != None and (auto_result[row_index][col_index] - elem > 0.00000001): + check_status = False + else: + pass + if not check_status: + tdLog.notice("acos function value has not as expected , sql is \"%s\" "%pow_query ) + sys.exit(1) + else: + tdLog.info("acos value check pass , it work as expected ,sql is \"%s\" "%pow_query ) + + def test_errors(self): + error_sql_lists = [ + "select acos from t1", + # "select acos(-+--+c1 ) from t1", + # "select +-acos(c1) from t1", + # "select ++-acos(c1) from t1", + # "select ++--acos(c1) from t1", + # "select - -acos(c1)*0 from t1", + # "select acos(tbname+1) from t1 ", + "select acos(123--123)==1 from t1", + "select acos(c1) as 'd1' from t1", + "select acos(c1 ,c2) from t1", + "select acos(c1 ,NULL ) from t1", + "select acos(,) from t1;", + "select acos(acos(c1) ab from t1)", + "select acos(c1 ) as int from t1", + "select acos from stb1", + # "select acos(-+--+c1) from stb1", + # "select +-acos(c1) from stb1", + # "select ++-acos(c1) from stb1", + # "select ++--acos(c1) from stb1", + # "select - -acos(c1)*0 from stb1", + # "select acos(tbname+1) from stb1 ", + "select acos(123--123)==1 from stb1", + "select acos(c1) as 'd1' from stb1", + "select acos(c1 ,c2 ) from stb1", + "select acos(c1 ,NULL) from stb1", + "select acos(,) from stb1;", + "select acos(acos(c1) ab from stb1)", + "select acos(c1) as int from stb1" + ] + for error_sql in error_sql_lists: + tdSql.error(error_sql) + + def support_types(self): + type_error_sql_lists = [ + "select acos(ts) from t1" , + "select acos(c7) from t1", + "select acos(c8) from t1", + "select acos(c9) from t1", + "select acos(ts) from ct1" , + "select acos(c7) from ct1", + "select acos(c8) from ct1", + "select acos(c9) from ct1", + "select acos(ts) from ct3" , + "select acos(c7) from ct3", + "select acos(c8) from ct3", + "select acos(c9) from ct3", + "select acos(ts) from ct4" , + "select acos(c7) from ct4", + "select acos(c8) from ct4", + "select acos(c9) from ct4", + "select acos(ts) from stb1" , + "select acos(c7) from stb1", + "select acos(c8) from stb1", + "select acos(c9) from stb1" , + + "select acos(ts) from stbbb1" , + "select acos(c7) from stbbb1", + + "select acos(ts) from tbname", + "select acos(c9) from tbname" + + ] + + for type_sql in type_error_sql_lists: + tdSql.error(type_sql) + + + type_sql_lists = [ + "select acos(c1) from t1", + "select acos(c2) from t1", + "select acos(c3) from t1", + "select acos(c4) from t1", + "select acos(c5) from t1", + "select acos(c6) from t1", + + "select acos(c1) from ct1", + "select acos(c2) from ct1", + "select acos(c3) from ct1", + "select acos(c4) from ct1", + "select acos(c5) from ct1", + "select acos(c6) from ct1", + + "select acos(c1) from ct3", + "select acos(c2) from ct3", + "select acos(c3) from ct3", + "select acos(c4) from ct3", + "select acos(c5) from ct3", + "select acos(c6) from ct3", + + "select acos(c1) from stb1", + "select acos(c2) from stb1", + "select acos(c3) from stb1", + "select acos(c4) from stb1", + "select acos(c5) from stb1", + "select acos(c6) from stb1", + + "select acos(c6) as alisb from stb1", + "select acos(c6) alisb from stb1", + ] + + for type_sql in type_sql_lists: + tdSql.query(type_sql) + + def basic_acos_function(self): + + # basic query + tdSql.query("select c1 from ct3") + tdSql.checkRows(0) + tdSql.query("select c1 from t1") + tdSql.checkRows(12) + tdSql.query("select c1 from stb1") + tdSql.checkRows(25) + + # used for empty table , ct3 is empty + tdSql.query("select acos(c1) from ct3") + tdSql.checkRows(0) + tdSql.query("select acos(c2) from ct3") + tdSql.checkRows(0) + tdSql.query("select acos(c3) from ct3") + tdSql.checkRows(0) + tdSql.query("select acos(c4) from ct3") + tdSql.checkRows(0) + tdSql.query("select acos(c5) from ct3") + tdSql.checkRows(0) + tdSql.query("select acos(c6) from ct3") + tdSql.checkRows(0) + + + # # used for regular table + tdSql.query("select acos(c1) from t1") + tdSql.checkData(0, 0, None) + tdSql.checkData(1 , 0, 0.000000000) + tdSql.checkData(3 , 0, None) + tdSql.checkData(5 , 0, None) + + tdSql.query("select c1, c2, c3 , c4, c5 from t1") + tdSql.checkData(1, 4, 1.11000) + tdSql.checkData(3, 3, 33) + tdSql.checkData(5, 4, None) + + tdSql.query("select ts,c1, c2, c3 , c4, c5 from t1") + tdSql.checkData(1, 5, 1.11000) + tdSql.checkData(3, 4, 33) + tdSql.checkData(5, 5, None) + + self.check_result_auto_acos( "select abs(c1), abs(c2), abs(c3) , abs(c4), abs(c5) from t1", "select acos(abs(c1)), acos(abs(c2)) ,acos(abs(c3)), acos(abs(c4)), acos(abs(c5)) from t1") + + # used for sub table + tdSql.query("select c2 ,acos(c2) from ct1") + tdSql.checkData(0, 1, None) + tdSql.checkData(1 , 1, None) + tdSql.checkData(3 , 1, None) + tdSql.checkData(4 , 1, 1.570796327) + + tdSql.query("select c1, c5 ,acos(c5) from ct4") + tdSql.checkData(0 , 2, None) + tdSql.checkData(1 , 2, None) + tdSql.checkData(2 , 2, None) + tdSql.checkData(3 , 2, None) + tdSql.checkData(5 , 2, None) + + self.check_result_auto_acos( "select c1, c2, c3 , c4, c5 from ct1", "select acos(c1), acos(c2) ,acos(c3), acos(c4), acos(c5) from ct1") + + # nest query for acos functions + tdSql.query("select c4 , acos(c4) ,acos(acos(c4)) , acos(acos(acos(c4))) from ct1;") + tdSql.checkData(0 , 0 , 88) + tdSql.checkData(0 , 1 , None) + tdSql.checkData(0 , 2 , None) + tdSql.checkData(0 , 3 , None) + + tdSql.checkData(1 , 0 , 77) + tdSql.checkData(1 , 1 , None) + tdSql.checkData(1 , 2 , None) + tdSql.checkData(1 , 3 , None) + + tdSql.checkData(11 , 0 , -99) + tdSql.checkData(11 , 1 , None) + tdSql.checkData(11 , 2 , None) + tdSql.checkData(11 , 3 , None) + + # used for stable table + + tdSql.query("select acos(c1) from stb1") + tdSql.checkRows(25) + + + # used for not exists table + tdSql.error("select acos(c1) from stbbb1") + tdSql.error("select acos(c1) from tbname") + tdSql.error("select acos(c1) from ct5") + + # mix with common col + tdSql.query("select c1, acos(c1) from ct1") + tdSql.query("select c2, acos(c2) from ct4") + + + # mix with common functions + tdSql.query("select c1, acos(c1),acos(c1), acos(acos(c1)) from ct4 ") + tdSql.checkData(0 , 0 ,None) + tdSql.checkData(0 , 1 ,None) + tdSql.checkData(0 , 2 ,None) + tdSql.checkData(0 , 3 ,None) + + tdSql.checkData(3 , 0 , 6) + tdSql.checkData(3 , 1 ,None) + tdSql.checkData(3 , 2 ,None) + tdSql.checkData(3 , 3 ,None) + + tdSql.query("select c1, acos(c1),c5, floor(c5) from stb1 ") + + # # mix with agg functions , not support + tdSql.error("select c1, acos(c1),c5, count(c5) from stb1 ") + tdSql.error("select c1, acos(c1),c5, count(c5) from ct1 ") + tdSql.error("select acos(c1), count(c5) from stb1 ") + tdSql.error("select acos(c1), count(c5) from ct1 ") + tdSql.error("select c1, count(c5) from ct1 ") + tdSql.error("select c1, count(c5) from stb1 ") + + # agg functions mix with agg functions + + tdSql.query("select max(c5), count(c5) from stb1") + tdSql.query("select max(c5), count(c5) from ct1") + + + # # bug fix for compute + tdSql.query("select c1, acos(c1) -0 ,acos(c1-4)-0 from ct4 ") + tdSql.checkData(0, 0, None) + tdSql.checkData(0, 1, None) + tdSql.checkData(0, 2, None) + tdSql.checkData(1, 0, 8) + tdSql.checkData(1, 1, None) + tdSql.checkData(1, 2, None) + + tdSql.query(" select c1, acos(c1) -0 ,acos(c1-0.1)-0.1 from ct4") + tdSql.checkData(0, 0, None) + tdSql.checkData(0, 1, None) + tdSql.checkData(0, 2, None) + tdSql.checkData(1, 0, 8) + tdSql.checkData(1, 1, None) + tdSql.checkData(1, 2, None) + + tdSql.query("select c1, acos(c1), c2, acos(c2), c3, acos(c3) from ct1") + + def test_big_number(self): + + tdSql.query("select c1, acos(100000000) from ct1") # bigint to double data overflow + tdSql.checkData(4, 1, None) + + + tdSql.query("select c1, acos(10000000000000) from ct1") # bigint to double data overflow + tdSql.checkData(4, 1, None) + + tdSql.query("select c1, acos(10000000000000000000000000) from ct1") # bigint to double data overflow + tdSql.query("select c1, acos(10000000000000000000000000.0) from ct1") # 10000000000000000000000000.0 is a double value + tdSql.checkData(1, 1, None) + + tdSql.query("select c1, acos(10000000000000000000000000000000000) from ct1") # bigint to double data overflow + tdSql.query("select c1, acos(10000000000000000000000000000000000.0) from ct1") # 10000000000000000000000000.0 is a double value + tdSql.checkData(4, 1, None) + + tdSql.query("select c1, acos(10000000000000000000000000000000000000000) from ct1") # bigint to double data overflow + tdSql.query("select c1, acos(10000000000000000000000000000000000000000.0) from ct1") # 10000000000000000000000000.0 is a double value + + tdSql.checkData(4, 1, None) + + tdSql.query("select c1, acos(10000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000) from ct1") # bigint to double data overflow + + def abs_func_filter(self): + tdSql.execute("use db") + tdSql.query("select c1, abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(acos(c1)-0.5) from ct4 where c1>5 ") + tdSql.checkRows(3) + tdSql.checkData(0,0,8) + tdSql.checkData(0,1,8.000000000) + tdSql.checkData(0,2,8.000000000) + tdSql.checkData(0,3,7.900000000) + tdSql.checkData(0,4,None) + + tdSql.query("select c1, abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(acos(c1)-0.5) from ct4 where c1=5 ") + tdSql.checkRows(1) + tdSql.checkData(0,0,5) + tdSql.checkData(0,1,5.000000000) + tdSql.checkData(0,2,5.000000000) + tdSql.checkData(0,3,4.900000000) + tdSql.checkData(0,4,None) + + tdSql.query("select c1,c2 , abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(acos(c1)-0.5) from ct4 where c1 Date: Fri, 6 May 2022 21:36:39 +0800 Subject: [PATCH 44/83] enh(index): support more data type --- source/libs/index/inc/indexCache.h | 1 + source/libs/index/inc/indexComm.h | 2 +- source/libs/index/src/indexCache.c | 119 +++++++++++++++++++++++++++-- source/libs/index/src/indexComm.c | 27 +++++++ source/libs/index/src/indexTfile.c | 14 ++-- 5 files changed, 149 insertions(+), 14 deletions(-) diff --git a/source/libs/index/inc/indexCache.h b/source/libs/index/inc/indexCache.h index 0daca26ec9..d474d87409 100644 --- a/source/libs/index/inc/indexCache.h +++ b/source/libs/index/inc/indexCache.h @@ -48,6 +48,7 @@ typedef struct IndexCache { } IndexCache; #define CACHE_VERSION(cache) atomic_load_32(&cache->version) + typedef struct CacheTerm { // key char* colVal; diff --git a/source/libs/index/inc/indexComm.h b/source/libs/index/inc/indexComm.h index 0d8418ba65..3b07429089 100644 --- a/source/libs/index/inc/indexComm.h +++ b/source/libs/index/inc/indexComm.h @@ -24,7 +24,7 @@ extern char JSON_COLUMN[]; extern char JSON_VALUE_DELIM; char* indexPackJsonData(SIndexTerm* itm); - +char* indexPackJsonDataPrefix(SIndexTerm* itm, int32_t* skip); #ifdef __cplusplus } #endif diff --git a/source/libs/index/src/indexCache.c b/source/libs/index/src/indexCache.c index ad82dd0748..b3ae7b7dbe 100644 --- a/source/libs/index/src/indexCache.c +++ b/source/libs/index/src/indexCache.c @@ -86,9 +86,18 @@ static TExeCond tCompareLessThan(void* a, void* b, int8_t type) { __compar_fn_t func = getComparFunc(type, 0); return tDoCommpare(func, QUERY_LESS_THAN, a, b); } -static TExeCond tCompareLessEqual(void* a, void* b, int8_t type) { return MATCH; } -static TExeCond tCompareGreaterThan(void* a, void* b, int8_t type) { return MATCH; } -static TExeCond tCompareGreaterEqual(void* a, void* b, int8_t type) { return MATCH; } +static TExeCond tCompareLessEqual(void* a, void* b, int8_t type) { + __compar_fn_t func = getComparFunc(type, 0); + return tDoCommpare(func, QUERY_LESS_EQUAL, a, b); +} +static TExeCond tCompareGreaterThan(void* a, void* b, int8_t type) { + __compar_fn_t func = getComparFunc(type, 0); + return tDoCommpare(func, QUERY_GREATER_THAN, a, b); +} +static TExeCond tCompareGreaterEqual(void* a, void* b, int8_t type) { + __compar_fn_t func = getComparFunc(type, 0); + return tDoCommpare(func, QUERY_GREATER_EQUAL, a, b); +} static TExeCond (*rangeCompare[])(void* a, void* b, int8_t type) = {tCompareLessThan, tCompareLessEqual, tCompareGreaterThan, tCompareGreaterEqual}; @@ -109,10 +118,12 @@ static int32_t cacheSearchTerm(void* cache, SIndexTerm* term, SIdxTempResult* tr } MemTable* mem = cache; IndexCache* pCache = mem->pCache; - CacheTerm ct = {.colVal = term->colVal, .version = atomic_load_32(&pCache->version)}; - CacheTerm* pCt = &ct; - char* key = indexCacheTermGet(&ct); + CacheTerm* pCt = taosMemoryCalloc(1, sizeof(CacheTerm)); + pCt->colVal = term->colVal; + pCt->version = atomic_load_32(&pCache->version); + + char* key = indexCacheTermGet(pCt); SSkipListIterator* iter = tSkipListCreateIterFromVal(mem->mem, key, TSDB_DATA_TYPE_BINARY, TSDB_ORDER_ASC); while (tSkipListIterNext(iter)) { @@ -133,6 +144,8 @@ static int32_t cacheSearchTerm(void* cache, SIndexTerm* term, SIdxTempResult* tr break; } } + + taosMemoryFree(pCt); tSkipListDestroyIter(iter); return 0; } @@ -153,6 +166,7 @@ static int32_t cacheSearchCompareFunc(void* cache, SIndexTerm* term, SIdxTempRes if (cache == NULL) { return 0; } + _cache_range_compare cmpFn = rangeCompare[type]; MemTable* mem = cache; @@ -204,6 +218,48 @@ static int32_t cacheSearchGreaterEqual(void* cache, SIndexTerm* term, SIdxTempRe } static int32_t cacheSearchTerm_JSON(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) { + if (cache == NULL) { + return 0; + } + MemTable* mem = cache; + IndexCache* pCache = mem->pCache; + + CacheTerm* pCt = taosMemoryCalloc(1, sizeof(CacheTerm)); + pCt->colVal = term->colVal; + pCt->version = atomic_load_32(&pCache->version); + + char* exBuf = NULL; + if (INDEX_TYPE_CONTAIN_EXTERN_TYPE(term->colType, TSDB_DATA_TYPE_JSON)) { + exBuf = indexPackJsonData(term); + pCt->colVal = exBuf; + } + char* key = indexCacheTermGet(pCt); + + SSkipListIterator* iter = tSkipListCreateIterFromVal(mem->mem, key, TSDB_DATA_TYPE_BINARY, TSDB_ORDER_ASC); + while (tSkipListIterNext(iter)) { + SSkipListNode* node = tSkipListIterGet(iter); + if (node == NULL) { + break; + } + CacheTerm* c = (CacheTerm*)SL_GET_NODE_DATA(node); + if (0 == strcmp(c->colVal, pCt->colVal)) { + if (c->operaType == ADD_VALUE) { + INDEX_MERGE_ADD_DEL(tr->deled, tr->added, c->uid) + // taosArrayPush(result, &c->uid); + *s = kTypeValue; + } else if (c->operaType == DEL_VALUE) { + INDEX_MERGE_ADD_DEL(tr->added, tr->deled, c->uid) + } + } else { + break; + } + } + + taosMemoryFree(pCt); + taosMemoryFree(exBuf); + tSkipListDestroyIter(iter); + return 0; + return TSDB_CODE_SUCCESS; } static int32_t cacheSearchPrefix_JSON(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) { @@ -233,6 +289,56 @@ static int32_t cacheSearchRange_JSON(void* cache, SIndexTerm* term, SIdxTempResu static int32_t cacheSearchCompareFunc_JSON(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s, RangeType type) { + if (cache == NULL) { + return 0; + } + _cache_range_compare cmpFn = rangeCompare[type]; + + MemTable* mem = cache; + IndexCache* pCache = mem->pCache; + + CacheTerm* pCt = taosMemoryCalloc(1, sizeof(CacheTerm)); + pCt->colVal = term->colVal; + pCt->version = atomic_load_32(&pCache->version); + + int8_t dType = INDEX_TYPE_GET_TYPE(term->colType); + int skip = 0; + char* exBuf = NULL; + + if (INDEX_TYPE_CONTAIN_EXTERN_TYPE(term->colType, TSDB_DATA_TYPE_JSON)) { + exBuf = indexPackJsonDataPrefix(term, &skip); + pCt->colVal = exBuf; + } + char* key = indexCacheTermGet(pCt); + + SSkipListIterator* iter = tSkipListCreateIterFromVal(mem->mem, key, TSDB_DATA_TYPE_BINARY, TSDB_ORDER_ASC); + while (tSkipListIterNext(iter)) { + SSkipListNode* node = tSkipListIterGet(iter); + if (node == NULL) { + break; + } + CacheTerm* c = (CacheTerm*)SL_GET_NODE_DATA(node); + + TExeCond cond = cmpFn(c->colVal + skip, term->colVal, dType); + if (cond == MATCH) { + if (c->operaType == ADD_VALUE) { + INDEX_MERGE_ADD_DEL(tr->deled, tr->added, c->uid) + // taosArrayPush(result, &c->uid); + *s = kTypeValue; + } else if (c->operaType == DEL_VALUE) { + INDEX_MERGE_ADD_DEL(tr->added, tr->deled, c->uid) + } + } else if (cond == CONTINUE) { + continue; + } else if (cond == BREAK) { + break; + } + } + + taosMemoryFree(pCt); + taosMemoryFree(exBuf); + tSkipListDestroyIter(iter); + return TSDB_CODE_SUCCESS; } static int32_t cacheSearchRange(void* cache, SIndexTerm* term, SIdxTempResult* tr, STermValueType* s) { @@ -408,6 +514,7 @@ static void indexCacheMakeRoomForWrite(IndexCache* cache) { indexCacheRef(cache); cache->imm = cache->mem; cache->mem = indexInternalCacheCreate(cache->type); + cache->mem->pCache = cache; cache->occupiedMem = 0; // sched to merge // unref cache in bgwork diff --git a/source/libs/index/src/indexComm.c b/source/libs/index/src/indexComm.c index 4dea5fa011..cdd7b35675 100644 --- a/source/libs/index/src/indexComm.c +++ b/source/libs/index/src/indexComm.c @@ -46,3 +46,30 @@ char* indexPackJsonData(SIndexTerm* itm) { return buf; } +char* indexPackJsonDataPrefix(SIndexTerm* itm, int32_t* skip) { + /* + * |<-----colname---->|<-----dataType---->|<--------colVal---------->| + * |<-----string----->|<-----uint8_t----->|<----depend on dataType-->| + */ + uint8_t ty = INDEX_TYPE_GET_TYPE(itm->colType); + + int32_t sz = itm->nColName + itm->nColVal + sizeof(uint8_t) + sizeof(JSON_VALUE_DELIM) * 2 + 1; + char* buf = (char*)taosMemoryCalloc(1, sz); + char* p = buf; + + memcpy(p, itm->colName, itm->nColName); + p += itm->nColName; + + memcpy(p, &JSON_VALUE_DELIM, sizeof(JSON_VALUE_DELIM)); + p += sizeof(JSON_VALUE_DELIM); + + memcpy(p, &ty, sizeof(ty)); + p += sizeof(ty); + + memcpy(p, &JSON_VALUE_DELIM, sizeof(JSON_VALUE_DELIM)); + p += sizeof(JSON_VALUE_DELIM); + + *skip = p - buf; + + return buf; +} diff --git a/source/libs/index/src/indexTfile.c b/source/libs/index/src/indexTfile.c index 9edd868272..b5551e825f 100644 --- a/source/libs/index/src/indexTfile.c +++ b/source/libs/index/src/indexTfile.c @@ -308,20 +308,20 @@ static int32_t tfSearchRegex(void* reader, SIndexTerm* tem, SIdxTempResult* tr) } static int32_t tfSearchCompareFunc(void* reader, SIndexTerm* tem, SIdxTempResult* tr, RangeType type) { - bool hasJson = INDEX_TYPE_CONTAIN_EXTERN_TYPE(tem->colType, TSDB_DATA_TYPE_JSON); - int ret = 0; - char* p = tem->colVal; - uint64_t sz = tem->nColVal; + bool hasJson = INDEX_TYPE_CONTAIN_EXTERN_TYPE(tem->colType, TSDB_DATA_TYPE_JSON); + int ret = 0; + char* p = tem->colVal; + int skip = 0; + if (hasJson) { - p = indexPackJsonData(tem); - sz = strlen(p); + p = indexPackJsonDataPrefix(tem, &skip); } SArray* offsets = taosArrayInit(16, sizeof(uint64_t)); AutomationCtx* ctx = automCtxCreate((void*)p, AUTOMATION_ALWAYS); FstStreamBuilder* sb = fstSearch(((TFileReader*)reader)->fst, ctx); - FstSlice h = fstSliceCreate((uint8_t*)p, sz); + FstSlice h = fstSliceCreate((uint8_t*)p, skip); fstStreamBuilderSetRange(sb, &h, type); fstSliceDestroy(&h); From bc5e63b0a78bfd62dce88e3b5e81c0cd889f8a6b Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Fri, 6 May 2022 21:44:25 +0800 Subject: [PATCH 45/83] feat(query): add histogram param parsing --- source/libs/function/src/builtinsimpl.c | 169 +++++++++++++++++++++++- 1 file changed, 163 insertions(+), 6 deletions(-) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 3c9eca85dd..64cca47da0 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -14,13 +14,14 @@ */ #include "builtinsimpl.h" +#include "cJSON.h" #include "function.h" #include "querynodes.h" #include "taggfunction.h" #include "tdatablock.h" #include "tpercentile.h" -#define HISTOGRAM_MAX_BINS_NUM 100 +#define HISTOGRAM_MAX_BINS_NUM 100 typedef struct SSumRes { union { @@ -106,6 +107,13 @@ typedef struct SHistoFuncInfo { SHistoFuncBin bins[]; } SHistoFuncInfo; +typedef enum { + UNKNOWN_BIN = 0, + USER_INPUT_BIN, + LINEAR_BIN, + LOG_BIN +} EHistoBinType; + #define SET_VAL(_info, numOfElem, res) \ do { \ @@ -1801,16 +1809,165 @@ bool getHistogramFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) return true; } -bool histogramFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { +static int8_t getHistogramBinType(char *binTypeStr) { + int8_t binType; + if (strcasecmp(binTypeStr, "user_input") == 0) { + binType = USER_INPUT_BIN; + } else if (strcasecmp(binTypeStr, "linear_bin") == 0) { + binType = LINEAR_BIN; + } else if (strcasecmp(binTypeStr, "log_bin") == 0) { + binType = LOG_BIN; + } else { + binType = UNKNOWN_BIN; + } + + return binType; +} + +static bool getHistogramBinDesc(SHistoFuncInfo *pInfo, char *binDescStr, int8_t binType, bool normalized) { + cJSON* binDesc = cJSON_Parse(binDescStr); + int32_t counter; + int32_t numOfBins; + double* intervals; + if (cJSON_IsObject(binDesc)) { /* linaer/log bins */ + int32_t numOfParams = cJSON_GetArraySize(binDesc); + int32_t startIndex; + if (numOfParams != 4) { + return false; + } + + cJSON* start = cJSON_GetObjectItem(binDesc, "start"); + cJSON* factor = cJSON_GetObjectItem(binDesc, "factor"); + cJSON* width = cJSON_GetObjectItem(binDesc, "width"); + cJSON* count = cJSON_GetObjectItem(binDesc, "count"); + cJSON* infinity = cJSON_GetObjectItem(binDesc, "infinity"); + + if (!cJSON_IsNumber(start) || !cJSON_IsNumber(count) || !cJSON_IsBool(infinity)) { + return false; + } + + if (count->valueint <= 0 || count->valueint > 1000) { // limit count to 1000 + return false; + } + + if (isinf(start->valuedouble) || (width != NULL && isinf(width->valuedouble)) || + (factor != NULL && isinf(factor->valuedouble)) || (count != NULL && isinf(count->valuedouble))) { + return false; + } + + counter = (int32_t)count->valueint; + if (infinity->valueint == false) { + startIndex = 0; + numOfBins = counter + 1; + } else { + startIndex = 1; + numOfBins = counter + 3; + } + + intervals = taosMemoryCalloc(numOfBins, sizeof(double)); + if (cJSON_IsNumber(width) && factor == NULL && binType == LINEAR_BIN) { + // linear bin process + if (width->valuedouble == 0) { + taosMemoryFree(intervals); + return false; + } + for (int i = 0; i < counter + 1; ++i) { + intervals[startIndex] = start->valuedouble + i * width->valuedouble; + if (isinf(intervals[startIndex])) { + taosMemoryFree(intervals); + return false; + } + startIndex++; + } + } else if (cJSON_IsNumber(factor) && width == NULL && binType == LOG_BIN) { + // log bin process + if (start->valuedouble == 0) { + taosMemoryFree(intervals); + return false; + } + if (factor->valuedouble < 0 || factor->valuedouble == 0 || factor->valuedouble == 1) { + taosMemoryFree(intervals); + return false; + } + for (int i = 0; i < counter + 1; ++i) { + intervals[startIndex] = start->valuedouble * pow(factor->valuedouble, i * 1.0); + if (isinf(intervals[startIndex])) { + taosMemoryFree(intervals); + return false; + } + startIndex++; + } + } else { + taosMemoryFree(intervals); + return false; + } + + if (infinity->valueint == true) { + intervals[0] = -INFINITY; + intervals[numOfBins - 1] = INFINITY; + // in case of desc bin orders, -inf/inf should be swapped + ASSERT(numOfBins >= 4); + if (intervals[1] > intervals[numOfBins - 2]) { + TSWAP(intervals[0], intervals[numOfBins - 1]); + } + } + } else if (cJSON_IsArray(binDesc)) { /* user input bins */ + if (binType != USER_INPUT_BIN) { + return false; + } + counter = numOfBins = cJSON_GetArraySize(binDesc); + intervals = taosMemoryCalloc(numOfBins, sizeof(double)); + cJSON* bin = binDesc->child; + if (bin == NULL) { + taosMemoryFree(intervals); + return false; + } + int i = 0; + while (bin) { + intervals[i] = bin->valuedouble; + if (!cJSON_IsNumber(bin)) { + taosMemoryFree(intervals); + return false; + } + if (i != 0 && intervals[i] <= intervals[i - 1]) { + taosMemoryFree(intervals); + return false; + } + bin = bin->next; + i++; + } + } else { + return false; + } + + pInfo->numOfBins = numOfBins; + pInfo->normalized = normalized; + for (int32_t i = 0; i < numOfBins; ++i) { + pInfo->bins[i].lower = intervals[i] < intervals[i + 1] ? intervals[i] : intervals[i + 1]; + pInfo->bins[i].upper = intervals[i + 1] > intervals[i] ? intervals[i + 1] : intervals[i]; + pInfo->bins[i].count = 0; + } + + taosMemoryFree(intervals); + return true; +} + +bool histogramFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo *pResultInfo) { if (!functionSetup(pCtx, pResultInfo)) { return false; } - SHistoFuncInfo* pInfo = GET_ROWCELL_INTERBUF(pResultInfo); - char* binType = pCtx->param[1].param.pz; - char* binDesc = pCtx->param[2].param.pz; - int64_t nornalized = pCtx->param[3].param.i; + SHistoFuncInfo *pInfo = GET_ROWCELL_INTERBUF(pResultInfo); + int8_t binType = getHistogramBinType(varDataVal(pCtx->param[1].param.pz)); + if (binType == UNKNOWN_BIN) { + return false; + } + char* binDesc = varDataVal(pCtx->param[2].param.pz); + int64_t normalized = pCtx->param[3].param.i; + if (!getHistogramBinDesc(pInfo, binDesc, binType, (bool)normalized)) { + return false; + } return true; } From 1f9b6f14dd70c968bf8391afd5d0a74fe24ec527 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 6 May 2022 22:11:33 +0800 Subject: [PATCH 46/83] fix mem leak --- source/libs/executor/src/dataSinkMgt.c | 1 + source/libs/executor/src/executorimpl.c | 4 +++- source/libs/qworker/inc/qworkerInt.h | 6 ++++-- source/libs/qworker/src/qworker.c | 21 ++++++++++++++++++++- source/libs/transport/src/transSrv.c | 4 ++++ 5 files changed, 32 insertions(+), 4 deletions(-) diff --git a/source/libs/executor/src/dataSinkMgt.c b/source/libs/executor/src/dataSinkMgt.c index 997397314f..64206fc10a 100644 --- a/source/libs/executor/src/dataSinkMgt.c +++ b/source/libs/executor/src/dataSinkMgt.c @@ -60,4 +60,5 @@ void dsScheduleProcess(void* ahandle, void* pItem) { void dsDestroyDataSinker(DataSinkHandle handle) { SDataSinkHandle* pHandleImpl = (SDataSinkHandle*)handle; pHandleImpl->fDestroy(pHandleImpl); + taosMemoryFree(pHandleImpl); } diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index a237eb0e7d..10d5e717bd 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -2826,6 +2826,7 @@ void relocateColumnData(SSDataBlock* pBlock, const SArray* pColMatchInfo, SArray } if (p->info.colId == pmInfo->colId) { + colDataDestroy(taosArrayGet(pBlock->pDataBlock, pmInfo->targetSlotId)); taosArraySet(pBlock->pDataBlock, pmInfo->targetSlotId, p); i++; j++; @@ -4687,7 +4688,7 @@ static SExecTaskInfo* createExecTaskInfo(uint64_t queryId, uint64_t taskId, EOPT char* p = taosMemoryCalloc(1, 128); snprintf(p, 128, "TID:0x%" PRIx64 " QID:0x%" PRIx64, taskId, queryId); - pTaskInfo->id.str = strdup(p); + pTaskInfo->id.str = p; return pTaskInfo; } @@ -5300,6 +5301,7 @@ void doDestroyTask(SExecTaskInfo* pTaskInfo) { // taosArrayDestroy(pTaskInfo->summary.queryProfEvents); // taosHashCleanup(pTaskInfo->summary.operatorProfResults); + destroyOperatorInfo(pTaskInfo->pRoot); taosMemoryFreeClear(pTaskInfo->sql); taosMemoryFreeClear(pTaskInfo->id.str); taosMemoryFreeClear(pTaskInfo); diff --git a/source/libs/qworker/inc/qworkerInt.h b/source/libs/qworker/inc/qworkerInt.h index ca471262ff..3ecf987811 100644 --- a/source/libs/qworker/inc/qworkerInt.h +++ b/source/libs/qworker/inc/qworkerInt.h @@ -25,6 +25,7 @@ extern "C" { #include "tlockfree.h" #include "ttimer.h" #include "tref.h" +#include "plannodes.h" #define QW_DEFAULT_SCHEDULER_NUMBER 10000 #define QW_DEFAULT_TASK_NUMBER 10000 @@ -131,8 +132,9 @@ typedef struct SQWTaskCtx { int8_t events[QW_EVENT_MAX]; - void *taskHandle; - void *sinkHandle; + void *taskHandle; + void *sinkHandle; + SSubplan *plan; } SQWTaskCtx; typedef struct SQWSchStatus { diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index bd7ee6321a..717958c033 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -424,6 +424,11 @@ void qwFreeTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx) { dsDestroyDataSinker(ctx->sinkHandle); ctx->sinkHandle = NULL; } + + if (ctx->plan) { + nodesDestroyNode(ctx->plan); + ctx->plan = NULL; + } } int32_t qwDropTaskCtx(QW_FPARAMS_DEF) { @@ -440,6 +445,7 @@ int32_t qwDropTaskCtx(QW_FPARAMS_DEF) { atomic_store_ptr(&ctx->taskHandle, NULL); atomic_store_ptr(&ctx->sinkHandle, NULL); + atomic_store_ptr(&ctx->plan, NULL); QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_DROP); @@ -922,7 +928,7 @@ _return: int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, int8_t taskType, int8_t explain) { int32_t code = 0; bool queryRsped = false; - struct SSubplan *plan = NULL; + SSubplan* plan = NULL; SQWPhaseInput input = {0}; qTaskInfo_t pTaskInfo = NULL; DataSinkHandle sinkHandle = NULL; @@ -950,6 +956,8 @@ int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, int8_t taskType, int8_t ex QW_ERR_JRET(code); } + ctx->plan = plan; + code = qCreateExecTask(qwMsg->node, mgmt->nodeId, tId, plan, &pTaskInfo, &sinkHandle, OPTR_EXEC_MODEL_BATCH); if (code) { QW_TASK_ELOG("qCreateExecTask failed, code:%x - %s", code, tstrerror(code)); @@ -1428,6 +1436,10 @@ void qwCloseRef(void) { taosWUnLockLatch(&gQwMgmt.lock); } +void qwDestroySchStatus(SQWSchStatus *pStatus) { + taosHashCleanup(pStatus->tasksHash); +} + void qwDestroyImpl(void *pMgmt) { SQWorker *mgmt = (SQWorker *)pMgmt; @@ -1439,6 +1451,13 @@ void qwDestroyImpl(void *pMgmt) { // TODO FREE ALL taosHashCleanup(mgmt->ctxHash); + + void *pIter = taosHashIterate(mgmt->schHash, NULL); + while (pIter) { + SQWSchStatus *sch = (SQWSchStatus *)pIter; + qwDestroySchStatus(sch); + pIter = taosHashIterate(mgmt->schHash, pIter); + } taosHashCleanup(mgmt->schHash); taosMemoryFree(mgmt); diff --git a/source/libs/transport/src/transSrv.c b/source/libs/transport/src/transSrv.c index ad3f520210..c0d8ca44ee 100644 --- a/source/libs/transport/src/transSrv.c +++ b/source/libs/transport/src/transSrv.c @@ -786,6 +786,10 @@ static void uvDestroyConn(uv_handle_t* handle) { transQueueDestroy(&conn->srvMsgs); QUEUE_REMOVE(&conn->queue); taosMemoryFree(conn->pTcp); + if (conn->regArg.init == 1) { + transFreeMsg(conn->regArg.msg.pCont); + conn->regArg.init = 0; + } taosMemoryFree(conn); if (thrd->quit && QUEUE_IS_EMPTY(&thrd->conn)) { From 0886d23e02d1c91ca323bf397034dbf3746feaae Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 6 May 2022 22:17:56 +0800 Subject: [PATCH 47/83] fix: alter db replications --- source/dnode/mnode/impl/inc/mndVgroup.h | 5 +- source/dnode/mnode/impl/src/mndDb.c | 35 ++++--- source/dnode/mnode/impl/src/mndVgroup.c | 94 ++++++++++++++++--- source/dnode/mnode/impl/test/dnode/mdnode.cpp | 92 ++++++++++++++++++ 4 files changed, 199 insertions(+), 27 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndVgroup.h b/source/dnode/mnode/impl/inc/mndVgroup.h index 87be15a4fd..1e95859157 100644 --- a/source/dnode/mnode/impl/inc/mndVgroup.h +++ b/source/dnode/mnode/impl/inc/mndVgroup.h @@ -31,8 +31,9 @@ SEpSet mndGetVgroupEpset(SMnode *pMnode, const SVgObj *pVgroup); int32_t mndGetVnodesNum(SMnode *pMnode, int32_t dnodeId); int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups); -int32_t mndAddVnodeToVgroup(SMnode *pMnode, SVgObj *pVgroup, SVnodeGid *new1, SVnodeGid *new2, SVnodeGid *exist); -int32_t mndRemoveVnodeFromVgroup(SMnode *pMnode, SVgObj *pVgroup, SVnodeGid *del1, SVnodeGid *del2, SVnodeGid *exist); +SArray *mndBuildDnodesArray(SMnode *pMnode); +int32_t mndAddVnodeToVgroup(SMnode *pMnode, SVgObj *pVgroup, SArray *pArray); +int32_t mndRemoveVnodeFromVgroup(SMnode *pMnode, SVgObj *pVgroup, SArray *pArray, SVnodeGid *del1, SVnodeGid *del2); void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen); void *mndBuildDropVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen); diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index cf9dd664e3..c3e4629f2a 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -399,6 +399,7 @@ static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg) { if (pCfg->compression < TSDB_MIN_COMP_LEVEL || pCfg->compression > TSDB_MAX_COMP_LEVEL) return -1; if (pCfg->replications < TSDB_MIN_DB_REPLICA || pCfg->replications > TSDB_MAX_DB_REPLICA) return -1; if (pCfg->replications > mndGetDnodeSize(pMnode)) return -1; + if (pCfg->replications != 1 && pCfg->replications != 3) return -1; if (pCfg->strict < TSDB_DB_STRICT_OFF || pCfg->strict > TSDB_DB_STRICT_ON) return -1; if (pCfg->cacheLastRow < TSDB_MIN_DB_CACHE_LAST_ROW || pCfg->cacheLastRow > TSDB_MAX_DB_CACHE_LAST_ROW) return -1; if (pCfg->hashMethod != 1) return -1; @@ -720,7 +721,7 @@ static int32_t mndSetAlterDbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *p return 0; } -static int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup) { +static int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SArray *pArray) { if (pVgroup->replica <= 0 || pVgroup->replica == pDb->cfg.replications) { for (int32_t vn = 0; vn < pVgroup->replica; ++vn) { SVnodeGid *pVgid = pVgroup->vnodeGid + vn; @@ -732,27 +733,30 @@ static int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj SVgObj newVgroup = {0}; memcpy(&newVgroup, pVgroup, sizeof(SVgObj)); if (newVgroup.replica < pDb->cfg.replications) { - SVnodeGid new1 = {0}; - SVnodeGid new2 = {0}; - SVnodeGid exist = {0}; - if (mndAddVnodeToVgroup(pMnode, &newVgroup, &new1, &new2, &exist) != 0) { + mInfo("db:%s, vgId:%d, will add 2 vnodes, vn:0 dnode:%d", pVgroup->dbName, pVgroup->vgId, + pVgroup->vnodeGid[0].dnodeId); + + if (mndAddVnodeToVgroup(pMnode, &newVgroup, pArray) != 0) { mError("db:%s, failed to add vnode to vgId:%d since %s", pDb->name, newVgroup.vgId, terrstr()); return -1; } - if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVgroup, &new1, true) != 0) return -1; - if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVgroup, &new2, true) != 0) return -1; - if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVgroup, &exist, true) != 0) return -1; + newVgroup.replica = pDb->cfg.replications; + if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVgroup, &newVgroup.vnodeGid[0], true) != 0) return -1; + if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVgroup, &newVgroup.vnodeGid[1], true) != 0) return -1; + if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVgroup, &newVgroup.vnodeGid[2], true) != 0) return -1; } else { + mInfo("db:%s, vgId:%d, will remove 2 vnodes", pVgroup->dbName, pVgroup->vgId); + SVnodeGid del1 = {0}; SVnodeGid del2 = {0}; - SVnodeGid exist = {0}; - if (mndRemoveVnodeFromVgroup(pMnode, &newVgroup, &del1, &del2, &exist) != 0) { + if (mndRemoveVnodeFromVgroup(pMnode, &newVgroup, pArray, &del1, &del2) != 0) { mError("db:%s, failed to remove vnode from vgId:%d since %s", pDb->name, newVgroup.vgId, terrstr()); return -1; } + newVgroup.replica = pDb->cfg.replications; + if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVgroup, &newVgroup.vnodeGid[0], true) != 0) return -1; if (mndAddDropVnodeAction(pMnode, pTrans, pDb, &newVgroup, &del1, true) != 0) return -1; if (mndAddDropVnodeAction(pMnode, pTrans, pDb, &newVgroup, &del2, true) != 0) return -1; - if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVgroup, &exist, true) != 0) return -1; } SSdbRaw *pVgRaw = mndVgroupActionEncode(&newVgroup); @@ -765,8 +769,9 @@ static int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj } static int32_t mndSetAlterDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pOld, SDbObj *pNew) { - SSdb *pSdb = pMnode->pSdb; - void *pIter = NULL; + SSdb *pSdb = pMnode->pSdb; + void *pIter = NULL; + SArray *pArray = mndBuildDnodesArray(pMnode); while (1) { SVgObj *pVgroup = NULL; @@ -774,9 +779,10 @@ static int32_t mndSetAlterDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj * if (pIter == NULL) break; if (pVgroup->dbUid == pNew->uid) { - if (mndBuildAlterVgroupAction(pMnode, pTrans, pNew, pVgroup) != 0) { + if (mndBuildAlterVgroupAction(pMnode, pTrans, pNew, pVgroup, pArray) != 0) { sdbCancelFetch(pSdb, pIter); sdbRelease(pSdb, pVgroup); + taosArrayDestroy(pArray); return -1; } } @@ -784,6 +790,7 @@ static int32_t mndSetAlterDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj * sdbRelease(pSdb, pVgroup); } + taosArrayDestroy(pArray); return 0; } diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index ca60a3b9ed..48a475ec63 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -370,7 +370,7 @@ static bool mndBuildDnodesArrayFp(SMnode *pMnode, void *pObj, void *p1, void *p2 return true; } -static SArray *mndBuildDnodesArray(SMnode *pMnode) { +SArray *mndBuildDnodesArray(SMnode *pMnode) { SSdb *pSdb = pMnode->pSdb; int32_t numOfDnodes = mndGetDnodeSize(pMnode); @@ -421,7 +421,7 @@ static int32_t mndGetAvailableDnode(SMnode *pMnode, SVgObj *pVgroup, SArray *pAr pVgid->role = TAOS_SYNC_STATE_FOLLOWER; } - mDebug("db:%s, vgId:%d, vn:%d dnode:%d is alloced", pVgroup->dbName, pVgroup->vgId, v, pVgid->dnodeId); + mInfo("db:%s, vgId:%d, vn:%d dnode:%d is alloced", pVgroup->dbName, pVgroup->vgId, v, pVgid->dnodeId); pDnode->numOfVnodes++; } @@ -440,12 +440,10 @@ int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups) { } pArray = mndBuildDnodesArray(pMnode); - if (pArray == NULL) { - goto _OVER; - } + if (pArray == NULL) goto _OVER; - mDebug("db:%s, total %d dnodes used to create %d vgroups (%d vnodes)", pDb->name, (int32_t)taosArrayGetSize(pArray), - pDb->cfg.numOfVgroups, pDb->cfg.numOfVgroups * pDb->cfg.replications); + mInfo("db:%s, total %d dnodes used to create %d vgroups (%d vnodes)", pDb->name, (int32_t)taosArrayGetSize(pArray), + pDb->cfg.numOfVgroups, pDb->cfg.numOfVgroups * pDb->cfg.replications); int32_t allocedVgroups = 0; int32_t maxVgId = sdbGetMaxId(pMnode->pSdb, SDB_VGROUP); @@ -483,7 +481,7 @@ int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups) { *ppVgroups = pVgroups; code = 0; - mDebug("db:%s, %d vgroups is alloced, replica:%d", pDb->name, pDb->cfg.numOfVgroups, pDb->cfg.replications); + mInfo("db:%s, %d vgroups is alloced, replica:%d", pDb->name, pDb->cfg.numOfVgroups, pDb->cfg.replications); _OVER: if (code != 0) taosMemoryFree(pVgroups); @@ -491,11 +489,85 @@ _OVER: return code; } -int32_t mndAddVnodeToVgroup(SMnode *pMnode, SVgObj *pVgroup, SVnodeGid *new1, SVnodeGid *new2, SVnodeGid *exist) { - return 0; +int32_t mndAddVnodeToVgroup(SMnode *pMnode, SVgObj *pVgroup, SArray *pArray) { + taosArraySort(pArray, (__compar_fn_t)mndCompareDnodeVnodes); + for (int32_t i = 0; i < taosArrayGetSize(pArray); ++i) { + SDnodeObj *pDnode = taosArrayGet(pArray, i); + mDebug("dnode:%d, equivalent vnodes:%d", pDnode->id, pDnode->numOfVnodes); + } + + int32_t maxPos = 1; + for (int32_t d = 0; d < taosArrayGetSize(pArray); ++d) { + SDnodeObj *pDnode = taosArrayGet(pArray, d); + + bool used = false; + for (int32_t vn = 0; vn < maxPos; ++vn) { + if (pDnode->id == pVgroup->vnodeGid[vn].dnodeId) { + used = true; + break; + } + } + if (used) continue; + + if (pDnode == NULL || pDnode->numOfVnodes > pDnode->numOfSupportVnodes) { + terrno = TSDB_CODE_MND_NO_ENOUGH_DNODES; + return -1; + } + + SVnodeGid *pVgid = &pVgroup->vnodeGid[maxPos]; + pVgid->dnodeId = pDnode->id; + pVgid->role = TAOS_SYNC_STATE_FOLLOWER; + pDnode->numOfVnodes++; + + mInfo("db:%s, vgId:%d, vn:%d dnode:%d is added", pVgroup->dbName, pVgroup->vgId, maxPos, pVgid->dnodeId); + maxPos++; + if (maxPos == 3) return 0; + } + + terrno = TSDB_CODE_MND_NO_ENOUGH_DNODES; + return -1; } -int32_t mndRemoveVnodeFromVgroup(SMnode *pMnode, SVgObj *pVgroup, SVnodeGid *del1, SVnodeGid *del2, SVnodeGid *exist) { +int32_t mndRemoveVnodeFromVgroup(SMnode *pMnode, SVgObj *pVgroup, SArray *pArray, SVnodeGid *del1, SVnodeGid *del2) { + int32_t removedNum = 0; + + taosArraySort(pArray, (__compar_fn_t)mndCompareDnodeVnodes); + for (int32_t i = 0; i < taosArrayGetSize(pArray); ++i) { + SDnodeObj *pDnode = taosArrayGet(pArray, i); + mDebug("dnode:%d, equivalent vnodes:%d", pDnode->id, pDnode->numOfVnodes); + } + + for (int32_t d = taosArrayGetSize(pArray) - 1; d >= 0; --d) { + SDnodeObj *pDnode = taosArrayGet(pArray, d); + + for (int32_t vn = 0; vn < TSDB_MAX_REPLICA; ++vn) { + SVnodeGid *pVgid = &pVgroup->vnodeGid[vn]; + if (pVgid->dnodeId == pDnode->id) { + if (removedNum == 0) *del1 = *pVgid; + if (removedNum == 1) *del2 = *pVgid; + + mInfo("db:%s, vgId:%d, vn:%d dnode:%d is removed", pVgroup->dbName, pVgroup->vgId, vn, pVgid->dnodeId); + memset(pVgid, 0, sizeof(SVnodeGid)); + removedNum++; + pDnode->numOfVnodes--; + + if (removedNum == 2) goto _OVER; + } + } + } + +_OVER: + if (removedNum != 2) return -1; + + for (int32_t vn = 1; vn < TSDB_MAX_REPLICA; ++vn) { + SVnodeGid *pVgid = &pVgroup->vnodeGid[vn]; + if (pVgid->dnodeId != 0) { + memcpy(&pVgroup->vnodeGid[0], pVgid, sizeof(SVnodeGid)); + memset(pVgid, 0, sizeof(SVnodeGid)); + } + } + + mInfo("db:%s, vgId:%d, dnode:%d is keeped", pVgroup->dbName, pVgroup->vgId, pVgroup->vnodeGid[0].dnodeId); return 0; } diff --git a/source/dnode/mnode/impl/test/dnode/mdnode.cpp b/source/dnode/mnode/impl/test/dnode/mdnode.cpp index a4cbc201a9..15ade8166f 100644 --- a/source/dnode/mnode/impl/test/dnode/mdnode.cpp +++ b/source/dnode/mnode/impl/test/dnode/mdnode.cpp @@ -266,4 +266,96 @@ TEST_F(MndTestDnode, 05_Create_Drop_Restart_Dnode) { taosMsleep(1300); test.SendShowReq(TSDB_MGMT_TABLE_DNODE, "dnodes", ""); EXPECT_EQ(test.GetShowRows(), 4); + + // alter replica +#if 0 + { + SCreateDbReq createReq = {0}; + strcpy(createReq.db, "1.d2"); + createReq.numOfVgroups = 2; + createReq.buffer = -1; + createReq.pageSize = -1; + createReq.pages = -1; + createReq.daysPerFile = 1000; + createReq.daysToKeep0 = 3650; + createReq.daysToKeep1 = 3650; + createReq.daysToKeep2 = 3650; + createReq.minRows = 100; + createReq.maxRows = 4096; + createReq.fsyncPeriod = 3000; + createReq.walLevel = 1; + createReq.precision = 0; + createReq.compression = 2; + createReq.replications = 1; + createReq.strict = 1; + createReq.cacheLastRow = 0; + createReq.ignoreExist = 1; + createReq.numOfStables = 0; + createReq.numOfRetensions = 0; + + int32_t contLen = tSerializeSCreateDbReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSCreateDbReq(pReq, contLen, &createReq); + + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); + + test.SendShowReq(TSDB_MGMT_TABLE_DB, "user_databases", ""); + EXPECT_EQ(test.GetShowRows(), 3); + } + + { + SAlterDbReq alterdbReq = {0}; + strcpy(alterdbReq.db, "1.d2"); + + alterdbReq.buffer = 12; + alterdbReq.pageSize = -1; + alterdbReq.pages = -1; + alterdbReq.daysPerFile = -1; + alterdbReq.daysToKeep0 = -1; + alterdbReq.daysToKeep1 = -1; + alterdbReq.daysToKeep2 = -1; + alterdbReq.fsyncPeriod = 4000; + alterdbReq.walLevel = 2; + alterdbReq.strict = 1; + alterdbReq.cacheLastRow = 1; + alterdbReq.replications = 3; + + int32_t contLen = tSerializeSAlterDbReq(NULL, 0, &alterdbReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSAlterDbReq(pReq, contLen, &alterdbReq); + + SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_DB, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); + } + + { + SAlterDbReq alterdbReq = {0}; + strcpy(alterdbReq.db, "1.d2"); + + alterdbReq.buffer = 12; + alterdbReq.pageSize = -1; + alterdbReq.pages = -1; + alterdbReq.daysPerFile = -1; + alterdbReq.daysToKeep0 = -1; + alterdbReq.daysToKeep1 = -1; + alterdbReq.daysToKeep2 = -1; + alterdbReq.fsyncPeriod = 4000; + alterdbReq.walLevel = 2; + alterdbReq.strict = 1; + alterdbReq.cacheLastRow = 1; + alterdbReq.replications = 1; + + int32_t contLen = tSerializeSAlterDbReq(NULL, 0, &alterdbReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSAlterDbReq(pReq, contLen, &alterdbReq); + + SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_DB, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); + } + +#endif } From 384fed49f986d9af907109648b76708090cc745b Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 6 May 2022 22:32:22 +0800 Subject: [PATCH 48/83] enh: add authVersion for user privilege --- source/dnode/mnode/impl/src/mndUser.c | 21 ++++++++++----------- tools/taos-tools | 2 +- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index 8b5fa06faf..81ded4ef8b 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -106,6 +106,7 @@ SSdbRaw *mndUserActionEncode(SUserObj *pUser) { SDB_SET_INT64(pRaw, dataPos, pUser->createdTime, _OVER) SDB_SET_INT64(pRaw, dataPos, pUser->updateTime, _OVER) SDB_SET_INT8(pRaw, dataPos, pUser->superUser, _OVER) + SDB_SET_INT32(pRaw, dataPos, pUser->authVersion, _OVER) SDB_SET_INT32(pRaw, dataPos, numOfReadDbs, _OVER) SDB_SET_INT32(pRaw, dataPos, numOfWriteDbs, _OVER) @@ -161,6 +162,7 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) { SDB_GET_INT64(pRaw, dataPos, &pUser->createdTime, _OVER) SDB_GET_INT64(pRaw, dataPos, &pUser->updateTime, _OVER) SDB_GET_INT8(pRaw, dataPos, &pUser->superUser, _OVER) + SDB_GET_INT32(pRaw, dataPos, &pUser->authVersion, _OVER) int32_t numOfReadDbs = 0; int32_t numOfWriteDbs = 0; @@ -588,7 +590,7 @@ _OVER: return code; } -static int32_t mndSetUserAuthRsp(SMnode *pMnode, SUserObj *pUser, SGetUserAuthRsp *pRsp) { +static int32_t mndSetUserAuthRsp(SMnode *pMnode, SUserObj *pUser, SGetUserAuthRsp *pRsp) { memcpy(pRsp->user, pUser->user, TSDB_USER_LEN); pRsp->superAuth = pUser->superUser; pRsp->version = pUser->authVersion; @@ -601,7 +603,7 @@ static int32_t mndSetUserAuthRsp(SMnode *pMnode, SUserObj *pUser, SGetUser terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - + SSdb *pSdb = pMnode->pSdb; void *pIter = NULL; while (1) { @@ -659,7 +661,7 @@ static int32_t mndProcessGetUserAuthReq(SNodeMsg *pReq) { code = 0; _OVER: - + mndReleaseUser(pMnode, pUser); tFreeSGetUserAuthRsp(&authRsp); @@ -711,7 +713,8 @@ static void mndCancelGetNextUser(SMnode *pMnode, void *pIter) { sdbCancelFetch(pSdb, pIter); } -int32_t mndValidateUserAuthInfo(SMnode *pMnode, SUserAuthVersion *pUsers, int32_t numOfUses, void **ppRsp, int32_t *pRspLen) { +int32_t mndValidateUserAuthInfo(SMnode *pMnode, SUserAuthVersion *pUsers, int32_t numOfUses, void **ppRsp, + int32_t *pRspLen) { SUserAuthBatchRsp batchRsp = {0}; batchRsp.pArray = taosArrayInit(numOfUses, sizeof(SGetUserAuthRsp)); if (batchRsp.pArray == NULL) { @@ -731,7 +734,7 @@ int32_t mndValidateUserAuthInfo(SMnode *pMnode, SUserAuthVersion *pUsers, int32_ mndReleaseUser(pMnode, pUser); continue; } - + SGetUserAuthRsp rsp = {0}; code = mndSetUserAuthRsp(pMnode, pUser, &rsp); if (code) { @@ -740,7 +743,6 @@ int32_t mndValidateUserAuthInfo(SMnode *pMnode, SUserAuthVersion *pUsers, int32_ goto _OVER; } - taosArrayPush(batchRsp.pArray, &rsp); mndReleaseUser(pMnode, pUser); } @@ -748,7 +750,7 @@ int32_t mndValidateUserAuthInfo(SMnode *pMnode, SUserAuthVersion *pUsers, int32_ if (taosArrayGetSize(batchRsp.pArray) <= 0) { *ppRsp = NULL; *pRspLen = 0; - + tFreeSUserAuthBatchRsp(&batchRsp); return 0; } @@ -772,10 +774,7 @@ _OVER: *ppRsp = NULL; *pRspLen = 0; - + tFreeSUserAuthBatchRsp(&batchRsp); return code; } - - - diff --git a/tools/taos-tools b/tools/taos-tools index 59e0ebaf49..2f3dfddd4d 160000 --- a/tools/taos-tools +++ b/tools/taos-tools @@ -1 +1 @@ -Subproject commit 59e0ebaf4905e4cb6d95a01c58b3fa507abc5a20 +Subproject commit 2f3dfddd4d9a869e706ba3cf98fb6d769404cd7c From 99efc7334aa59741cd1784574a8449f0505f5cdf Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 6 May 2022 22:53:49 +0800 Subject: [PATCH 49/83] enh: add authVersion for user privilege --- source/dnode/mnode/impl/src/mndDb.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index c3e4629f2a..70bdda5855 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -1070,6 +1070,17 @@ static int32_t mndDropDb(SMnode *pMnode, SNodeMsg *pReq, SDbObj *pDb) { /*if (mndDropTopicByDB(pMnode, pTrans, pDb) != 0) goto _OVER;*/ if (mndSetDropDbRedoActions(pMnode, pTrans, pDb) != 0) goto _OVER; + SUserObj *pUser = mndAcquireUser(pMnode, pDb->createUser); + if (pUser != NULL) { + pUser->authVersion++; + SSdbRaw *pCommitRaw = mndUserActionEncode(pUser); + if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { + mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr()); + goto _OVER; + } + sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); + } + int32_t rspLen = 0; void *pRsp = NULL; if (mndBuildDropDbRsp(pDb, &rspLen, &pRsp, false) < 0) goto _OVER; From 612bdec1f746eebd57b12db9519c6924f18fb041 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 6 May 2022 22:58:02 +0800 Subject: [PATCH 50/83] fix: invalid header file --- source/dnode/mnode/impl/src/mndSma.c | 2 +- source/dnode/mnode/impl/src/mndStb.c | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index 8619df978b..0ec4332d37 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -21,7 +21,7 @@ #include "mndInfoSchema.h" #include "mndMnode.h" #include "mndShow.h" -#include "mndStb.c" +#include "mndStb.h" #include "mndStream.h" #include "mndTrans.h" #include "mndUser.h" diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 8ae0d5d19c..c32008cb5c 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include "mndStb.h" #include "mndAuth.h" #include "mndDb.h" From b38f18020d73bf43d68387660195f8891fb1d401 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 6 May 2022 23:04:25 +0800 Subject: [PATCH 51/83] enh: check kill trans auth --- source/dnode/mnode/impl/inc/mndAuth.h | 1 + source/dnode/mnode/impl/src/mndAuth.c | 24 +++++++++--------------- source/dnode/mnode/impl/src/mndTrans.c | 3 +-- 3 files changed, 11 insertions(+), 17 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndAuth.h b/source/dnode/mnode/impl/inc/mndAuth.h index ae7663a775..890879912f 100644 --- a/source/dnode/mnode/impl/inc/mndAuth.h +++ b/source/dnode/mnode/impl/inc/mndAuth.h @@ -31,6 +31,7 @@ int32_t mndCheckDropUserAuth(SUserObj *pOperUser); int32_t mndCheckNodeAuth(SUserObj *pOperUser); int32_t mndCheckFuncAuth(SUserObj *pOperUser); +int32_t mndCheckTransAuth(SUserObj *pOperUser); int32_t mndCheckCreateDbAuth(SUserObj *pOperUser); int32_t mndCheckAlterDropCompactDbAuth(SUserObj *pOperUser, SDbObj *pDb); diff --git a/source/dnode/mnode/impl/src/mndAuth.c b/source/dnode/mnode/impl/src/mndAuth.c index 696d850266..8e5ec40c47 100644 --- a/source/dnode/mnode/impl/src/mndAuth.c +++ b/source/dnode/mnode/impl/src/mndAuth.c @@ -74,10 +74,7 @@ static int32_t mndProcessAuthReq(SNodeMsg *pReq) { } int32_t mndCheckCreateUserAuth(SUserObj *pOperUser) { - if (pOperUser->superUser) { - return 0; - } - + if (pOperUser->superUser) return 0; terrno = TSDB_CODE_MND_NO_RIGHTS; return -1; } @@ -118,28 +115,25 @@ int32_t mndCheckAlterUserAuth(SUserObj *pOperUser, SUserObj *pUser, SDbObj *pDb, } int32_t mndCheckDropUserAuth(SUserObj *pOperUser) { - if (pOperUser->superUser) { - return 0; - } - + if (pOperUser->superUser) return 0; terrno = TSDB_CODE_MND_NO_RIGHTS; return -1; } int32_t mndCheckNodeAuth(SUserObj *pOperUser) { - if (pOperUser->superUser) { - return 0; - } - + if (pOperUser->superUser) return 0; terrno = TSDB_CODE_MND_NO_RIGHTS; return -1; } int32_t mndCheckFuncAuth(SUserObj *pOperUser) { - if (pOperUser->superUser) { - return 0; - } + if (pOperUser->superUser) return 0; + terrno = TSDB_CODE_MND_NO_RIGHTS; + return -1; +} +int32_t mndCheckTransAuth(SUserObj *pOperUser) { + if (pOperUser->superUser) return 0; terrno = TSDB_CODE_MND_NO_RIGHTS; return -1; } diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 43e2735984..e57e9a0461 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -1336,8 +1336,7 @@ static int32_t mndProcessKillTransReq(SNodeMsg *pReq) { goto _OVER; } - if (!pUser->superUser) { - terrno = TSDB_CODE_MND_NO_RIGHTS; + if (mndCheckTransAuth(pUser) != 0) { goto _OVER; } From d75b3568a70dce7e39c22fc982f69c852878be73 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Fri, 6 May 2022 23:40:34 +0800 Subject: [PATCH 52/83] enh(sync): add trace log --- source/libs/sync/src/syncRaftLog.c | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index ae153251c3..031722ab3c 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -57,7 +57,13 @@ int32_t logStoreAppendEntry(SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry) { syncMeta.seqNum = pEntry->seqNum; syncMeta.term = pEntry->term; code = walWriteWithSyncInfo(pWal, pEntry->index, pEntry->originalRpcType, syncMeta, pEntry->data, pEntry->dataLen); - assert(code == 0); + if (code != 0) { + int32_t err = terrno; + const char *errStr = tstrerror(err); + sError("walWriteWithSyncInfo error, err:%d, msg:%s", err, errStr); + ASSERT(0); + } + //assert(code == 0); walFsync(pWal, true); return code; @@ -69,7 +75,14 @@ SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) { if (index >= SYNC_INDEX_BEGIN && index <= logStoreLastIndex(pLogStore)) { SWalReadHandle* pWalHandle = walOpenReadHandle(pWal); - assert(walReadWithHandle(pWalHandle, index) == 0); + int32_t code = walReadWithHandle(pWalHandle, index); + if (code != 0) { + int32_t err = terrno; + const char *errStr = tstrerror(err); + sError("walReadWithHandle error, err:%d, msg:%s", err, errStr); + ASSERT(0); + } + //assert(walReadWithHandle(pWalHandle, index) == 0); SSyncRaftEntry* pEntry = syncEntryBuild(pWalHandle->pHead->head.bodyLen); assert(pEntry != NULL); From 27a74a56dfe52a29eaab8cdea615a0778d99a619 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Sat, 7 May 2022 00:13:25 +0800 Subject: [PATCH 53/83] feat: merge rows in mem/file during commit for update --- source/common/src/trow.c | 12 +++- source/dnode/vnode/src/tsdb/tsdbCommit.c | 52 ++++++++------ source/dnode/vnode/src/tsdb/tsdbMemTable.c | 83 ++++++++++++++++++---- tests/script/tsim/query/scalarFunction.sim | 1 + 4 files changed, 110 insertions(+), 38 deletions(-) diff --git a/source/common/src/trow.c b/source/common/src/trow.c index 4c063b205f..261b6e1d67 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -466,7 +466,7 @@ int tdAppendValToDataCol(SDataCol *pCol, TDRowValT valType, const void *val, int memcpy(POINTER_SHIFT(pCol->pData, pCol->len), value, pCol->bytes); pCol->len += pCol->bytes; } - } else { + } else if (!tdValTypeIsNone(valType)) { if (IS_VAR_DATA_TYPE(pCol->type)) { // keep the last offset // discard the last var data @@ -483,7 +483,9 @@ int tdAppendValToDataCol(SDataCol *pCol, TDRowValT valType, const void *val, int } #ifdef TD_SUPPORT_BITMAP - tdSetBitmapValType(pCol->pBitmap, numOfRows, valType, bitmapMode); + if (!isMerge || !tdValTypeIsNone(valType)) { + tdSetBitmapValType(pCol->pBitmap, numOfRows, valType, bitmapMode); + } #endif return 0; } @@ -533,7 +535,9 @@ static int32_t tdAppendTpRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols ++dcol; } } +#if 0 ++pCols->numOfRows; +#endif return TSDB_CODE_SUCCESS; } @@ -584,7 +588,9 @@ static int32_t tdAppendKvRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols ++dcol; } } +#if 0 ++pCols->numOfRows; +#endif return TSDB_CODE_SUCCESS; } @@ -698,7 +704,7 @@ static void tdAppendValToDataCols(SDataCols *target, SDataCols *src, int iter, b tdAppendValToDataCol(&(target->cols[i]), sVal.valType, sVal.val, target->numOfRows, target->maxPoints, target->bitmapMode, isMerge); } else { - // Keep the origi value for None + // Keep the origin value for None } } else { tdAppendValToDataCol(&(target->cols[i]), sVal.valType, sVal.val, target->numOfRows, target->maxPoints, diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index b255622c60..a008c4a0e5 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -1330,13 +1330,15 @@ static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIt TSKEY maxKey, int maxRows, int8_t update) { TSKEY key1 = INT64_MAX; TSKEY key2 = INT64_MAX; + TSKEY lastKey = TSKEY_INITIAL_VAL; STSchema *pSchema = NULL; ASSERT(maxRows > 0 && dataColsKeyLast(pDataCols) <= maxKey); tdResetDataCols(pTarget); pTarget->bitmapMode = pDataCols->bitmapMode; - + // TODO: filter Multi-Version + // TODO: support delete function while (true) { key1 = (*iter >= pDataCols->numOfRows) ? INT64_MAX : dataColsKeyAt(pDataCols, *iter); STSRow *row = tsdbNextIterRow(pCommitIter->pIter); @@ -1349,6 +1351,9 @@ static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIt if (key1 == INT64_MAX && key2 == INT64_MAX) break; if (key1 < key2) { + if (lastKey != TSKEY_INITIAL_VAL) { + ++pTarget->numOfRows; + } for (int i = 0; i < pDataCols->numOfCols; ++i) { // TODO: dataColAppendVal may fail SCellVal sVal = {0}; @@ -1359,7 +1364,7 @@ static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIt pTarget->bitmapMode, false); } - ++pTarget->numOfRows; + lastKey = key1; ++(*iter); } else if (key1 > key2) { if (pSchema == NULL || schemaVersion(pSchema) != TD_ROW_SVER(row)) { @@ -1367,7 +1372,17 @@ static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIt ASSERT(pSchema != NULL); } - tdAppendSTSRowToDataCol(row, pSchema, pTarget, false); + if (key2 == lastKey) { + if (TD_SUPPORT_UPDATE(update)) { + tdAppendSTSRowToDataCol(row, pSchema, pTarget, true); + } + } else { + if (lastKey != TSKEY_INITIAL_VAL) { + ++pTarget->numOfRows; + } + tdAppendSTSRowToDataCol(row, pSchema, pTarget, false); + lastKey = key2; + } tSkipListIterNext(pCommitIter->pIter); } else { @@ -1397,6 +1412,12 @@ static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIt ++(*iter); tSkipListIterNext(pCommitIter->pIter); #endif + + if(lastKey != key1) { + lastKey = key1; + ++pTarget->numOfRows; + } + // copy disk data for (int i = 0; i < pDataCols->numOfCols; ++i) { SCellVal sVal = {0}; @@ -1416,26 +1437,17 @@ static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIt } // TODO: merge with Multi-Version - STSRow *curRow = row; - - ++(*iter); - tSkipListIterNext(pCommitIter->pIter); - STSRow *nextRow = tsdbNextIterRow(pCommitIter->pIter); - - if (key2 < TD_ROW_KEY(nextRow)) { - tdAppendSTSRowToDataCol(row, pSchema, pTarget, false); - } else { - tdAppendSTSRowToDataCol(row, pSchema, pTarget, false); - } - // TODO: merge with Multi-Version - } else { - ++pTarget->numOfRows; - ++(*iter); - tSkipListIterNext(pCommitIter->pIter); + tdAppendSTSRowToDataCol(row, pSchema, pTarget, true); } + ++(*iter); + tSkipListIterNext(pCommitIter->pIter); } - if (pTarget->numOfRows >= maxRows) break; + if (pTarget->numOfRows >= (maxRows - 1)) break; + } + + if (lastKey != TSKEY_INITIAL_VAL) { + ++pTarget->numOfRows; } } diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index 6520f371f6..48d32c0dd3 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -20,7 +20,7 @@ static void tsdbFreeTbData(STbData *pTbData); static char *tsdbGetTsTupleKey(const void *data); static int tsdbTbDataComp(const void *arg1, const void *arg2); static char *tsdbTbDataGetUid(const void *arg); -static int tsdbAppendTableRowToCols(STable *pTable, SDataCols *pCols, STSchema **ppSchema, STSRow *row); +static int tsdbAppendTableRowToCols(STable *pTable, SDataCols *pCols, STSchema **ppSchema, STSRow *row, bool merge); int tsdbMemTableCreate(STsdb *pTsdb, STsdbMemTable **ppMemTable) { STsdbMemTable *pMemTable; @@ -82,15 +82,19 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey TKEY *filterKeys, int nFilterKeys, bool keepDup, SMergeInfo *pMergeInfo) { ASSERT(maxRowsToRead > 0 && nFilterKeys >= 0); if (pIter == NULL) return 0; - STSchema *pSchema = NULL; - TSKEY rowKey = 0; - TSKEY fKey = 0; + STSchema *pSchema = NULL; + TSKEY rowKey = 0; + TSKEY fKey = 0; + // only fetch lastKey from mem data as file data not used in this function actually TSKEY lastKey = TSKEY_INITIAL_VAL; bool isRowDel = false; int filterIter = 0; STSRow *row = NULL; SMergeInfo mInfo; + // TODO: support Multi-Version(the rows with the same TS keys in memory can't be merged if its version refered by + // query handle) + if (pMergeInfo == NULL) pMergeInfo = &mInfo; memset(pMergeInfo, 0, sizeof(*pMergeInfo)); @@ -190,21 +194,66 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey } } #endif - } else { // fkey >= rowKey +#if 1 + } else if (fKey > rowKey) { if (isRowDel) { + // TODO: support delete function + pMergeInfo->rowsDeleteFailed++; + } else { + if (pMergeInfo->rowsInserted - pMergeInfo->rowsDeleteSucceed >= maxRowsToRead) break; + if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break; + + if (lastKey != rowKey) { + pMergeInfo->rowsInserted++; + pMergeInfo->nOperations++; + pMergeInfo->keyFirst = TMIN(pMergeInfo->keyFirst, rowKey); + pMergeInfo->keyLast = TMAX(pMergeInfo->keyLast, rowKey); + if (pCols) { + if (lastKey != TSKEY_INITIAL_VAL) { + ++pCols->numOfRows; + } + tsdbAppendTableRowToCols(pTable, pCols, &pSchema, row, false); + } + lastKey = rowKey; + } else { + if (keepDup) { + tsdbAppendTableRowToCols(pTable, pCols, &pSchema, row, true); + } else { + // discard + } + } + } + + tSkipListIterNext(pIter); + row = tsdbNextIterRow(pIter); + if (row == NULL || TD_ROW_KEY(row) > maxKey) { + rowKey = INT64_MAX; + isRowDel = false; + } else { + rowKey = TD_ROW_KEY(row); + isRowDel = TD_ROW_IS_DELETED(row); + } + } else { // fkey == rowKey + if (isRowDel) { // TODO: support delete function(How to stands for delete in file? rowVersion = -1?) ASSERT(!keepDup); if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break; pMergeInfo->rowsDeleteSucceed++; pMergeInfo->nOperations++; - tsdbAppendTableRowToCols(pTable, pCols, &pSchema, row); + tsdbAppendTableRowToCols(pTable, pCols, &pSchema, row, false); } else { if (keepDup) { if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break; - pMergeInfo->rowsUpdated++; - pMergeInfo->nOperations++; - pMergeInfo->keyFirst = TMIN(pMergeInfo->keyFirst, rowKey); - pMergeInfo->keyLast = TMAX(pMergeInfo->keyLast, rowKey); - tsdbAppendTableRowToCols(pTable, pCols, &pSchema, row); + if (lastKey != rowKey) { + pMergeInfo->rowsUpdated++; + pMergeInfo->nOperations++; + pMergeInfo->keyFirst = TMIN(pMergeInfo->keyFirst, rowKey); + pMergeInfo->keyLast = TMAX(pMergeInfo->keyLast, rowKey); + lastKey = rowKey; + ++pCols->numOfRows; + tsdbAppendTableRowToCols(pTable, pCols, &pSchema, row, false); + } else { + tsdbAppendTableRowToCols(pTable, pCols, &pSchema, row, true); + } } else { pMergeInfo->keyFirst = TMIN(pMergeInfo->keyFirst, fKey); pMergeInfo->keyLast = TMAX(pMergeInfo->keyLast, fKey); @@ -228,6 +277,10 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey fKey = tdGetKey(filterKeys[filterIter]); } } +#endif + } + if (lastKey != TSKEY_INITIAL_VAL) { + ++pCols->numOfRows; } return 0; @@ -301,8 +354,8 @@ static STbData *tsdbNewTbData(tb_uid_t uid) { pTbData->pData = tSkipListCreate(5, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), tkeyComparFn, SL_DISCARD_DUP_KEY, tsdbGetTsTupleKey); #endif - pTbData->pData = tSkipListCreate(5, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), tkeyComparFn, SL_ALLOW_DUP_KEY, - tsdbGetTsTupleKey); + pTbData->pData = + tSkipListCreate(5, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), tkeyComparFn, SL_ALLOW_DUP_KEY, tsdbGetTsTupleKey); if (pTbData->pData == NULL) { taosMemoryFree(pTbData); return NULL; @@ -337,7 +390,7 @@ static char *tsdbTbDataGetUid(const void *arg) { STbData *pTbData = (STbData *)arg; return (char *)(&(pTbData->uid)); } -static int tsdbAppendTableRowToCols(STable *pTable, SDataCols *pCols, STSchema **ppSchema, STSRow *row) { +static int tsdbAppendTableRowToCols(STable *pTable, SDataCols *pCols, STSchema **ppSchema, STSRow *row, bool merge) { if (pCols) { if (*ppSchema == NULL || schemaVersion(*ppSchema) != TD_ROW_SVER(row)) { *ppSchema = tsdbGetTableSchemaImpl(pTable, false, false, TD_ROW_SVER(row)); @@ -347,7 +400,7 @@ static int tsdbAppendTableRowToCols(STable *pTable, SDataCols *pCols, STSchema * } } - tdAppendSTSRowToDataCol(row, *ppSchema, pCols, false); + tdAppendSTSRowToDataCol(row, *ppSchema, pCols, merge); } return 0; diff --git a/tests/script/tsim/query/scalarFunction.sim b/tests/script/tsim/query/scalarFunction.sim index 97d49ae901..2946a89ff6 100644 --- a/tests/script/tsim/query/scalarFunction.sim +++ b/tests/script/tsim/query/scalarFunction.sim @@ -102,6 +102,7 @@ print ====> $data60 $data61 $data62 $data63 $data64 $data65 print ====> $data70 $data71 $data72 $data73 $data74 $data75 print ====> $data80 $data81 $data82 $data83 $data84 $data85 print ====> $data90 $data91 $data92 $data93 $data94 $data95 +print ====> rows = $rows and rowNum = $rowNum for ct1 if $rows != $rowNum then return -1 endi From 6b18948884f00184aaa22d743423cdcd51d362e8 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Sat, 7 May 2022 00:14:52 +0800 Subject: [PATCH 54/83] enh(wal): set error code --- source/libs/sync/src/syncRaftLog.c | 1 + source/libs/wal/src/walWrite.c | 15 +++++++++++---- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index ae153251c3..5fa77c9964 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -57,6 +57,7 @@ int32_t logStoreAppendEntry(SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry) { syncMeta.seqNum = pEntry->seqNum; syncMeta.term = pEntry->term; code = walWriteWithSyncInfo(pWal, pEntry->index, pEntry->originalRpcType, syncMeta, pEntry->data, pEntry->dataLen); + perror("wal write error: "); assert(code == 0); walFsync(pWal, true); diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index dc31086e9f..da1c36dcc4 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -198,12 +198,14 @@ int walRoll(SWal *pWal) { if (pWal->pWriteIdxTFile != NULL) { code = taosCloseFile(&pWal->pWriteIdxTFile); if (code != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); return -1; } } if (pWal->pWriteLogTFile != NULL) { code = taosCloseFile(&pWal->pWriteLogTFile); if (code != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); return -1; } } @@ -263,14 +265,19 @@ int64_t walWriteWithSyncInfo(SWal *pWal, int64_t index, tmsg_t msgType, SSyncLog if (index == pWal->vers.lastVer + 1) { if (taosArrayGetSize(pWal->fileInfoSet) == 0) { pWal->vers.firstVer = index; - code = walRoll(pWal); - ASSERT(code == 0); + if (walRoll(pWal) < 0) { + return -1; + } } else { int64_t passed = walGetSeq() - pWal->lastRollSeq; if (pWal->cfg.rollPeriod != -1 && pWal->cfg.rollPeriod != 0 && passed > pWal->cfg.rollPeriod) { - walRoll(pWal); + if (walRoll(pWal) < 0) { + return -1; + } } else if (pWal->cfg.segSize != -1 && pWal->cfg.segSize != 0 && walGetLastFileSize(pWal) > pWal->cfg.segSize) { - walRoll(pWal); + if (walRoll(pWal) < 0) { + return -1; + } } } } else { From 1fa906cde0e12f2af29490d817423fe5a2cef923 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Sat, 7 May 2022 00:46:09 +0800 Subject: [PATCH 55/83] fix: fill null val --- include/common/tdataformat.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 74d11a5ffb..e94be797b9 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -27,7 +27,7 @@ extern "C" { // Imported since 3.0 and use bitmap to demonstrate None/Null/Norm, while use Null/Norm below 3.0 without of bitmap. #define TD_SUPPORT_BITMAP -#undef TD_SUPPORT_READ2 +#define TD_SUPPORT_READ2 #define TD_SUPPORT_BACK2 // suppport back compatibility of 2.0 #define TASSERT(x) ASSERT(x) From b1a429f7df17f1687d4d87134a83b13915fe4838 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Sat, 7 May 2022 00:56:07 +0800 Subject: [PATCH 56/83] feat: code optimization for update --- source/common/src/trow.c | 4 ++-- source/dnode/vnode/src/tsdb/tsdbRead.c | 14 ++++++++------ 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/source/common/src/trow.c b/source/common/src/trow.c index 261b6e1d67..50e51a40dd 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -670,7 +670,7 @@ int tdMergeDataCols(SDataCols *target, SDataCols *source, int rowsToMerge, int * } } } - if (rowsToMerge > 0) { + if (lastKey != TSKEY_INITIAL_VAL) { ++target->numOfRows; } (*pOffset) += rowsToMerge; @@ -778,7 +778,7 @@ static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, i ASSERT(target->numOfRows <= target->maxPoints - 1); } - if (nRows > 0) { + if (lastKey != TSKEY_INITIAL_VAL) { ++target->numOfRows; } } diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 54b3eea988..aa305dd71c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -816,9 +816,10 @@ static TSKEY extractFirstTraverseKey(STableCheckInfo* pCheckInfo, int32_t order, SSkipListNode* node = tSkipListIterGet(pCheckInfo->iter); if (node != NULL) { rmem = (STSRow*)SL_GET_NODE_DATA(node); - if (TD_ROW_KEY(rmem) > maxVer) { - rmem = NULL; - } + // TODO: filter max version + // if (TD_ROW_VER(rmem) > maxVer) { + // rmem = NULL; + // } } } @@ -826,9 +827,10 @@ static TSKEY extractFirstTraverseKey(STableCheckInfo* pCheckInfo, int32_t order, SSkipListNode* node = tSkipListIterGet(pCheckInfo->iiter); if (node != NULL) { rimem = (STSRow*)SL_GET_NODE_DATA(node); - if (TD_ROW_KEY(rimem) > maxVer) { - rimem = NULL; - } + // TODO: filter max version + // if (TD_ROW_VER(rimem) > maxVer) { + // rimem = NULL; + // } } } From 3ae79a77c75b0a3a4443f7d1017342438743e005 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Sat, 7 May 2022 01:47:45 +0800 Subject: [PATCH 57/83] enh(stream): fix create stb in stream --- source/common/src/tdatablock.c | 29 +++--- source/dnode/mnode/impl/inc/mndStb.h | 5 ++ source/dnode/mnode/impl/src/mndStb.c | 112 +++++++++++++----------- source/dnode/mnode/impl/src/mndStream.c | 76 +++++++++++++++- source/libs/sync/src/syncRaftLog.c | 2 +- 5 files changed, 159 insertions(+), 65 deletions(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index a4b36dcb67..2c17f2c2fc 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1478,11 +1478,11 @@ void blockDebugShowData(const SArray* dataBlocks) { * @param uid set as parameter temporarily // TODO: remove this parameter, and the executor should set uid in * SDataBlock->info.uid * @param suid // TODO: check with Liao whether suid response is reasonable - * + * * TODO: colId should be set */ -int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks, STSchema *pTSchema, int32_t vgId, tb_uid_t uid, - tb_uid_t suid) { +int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks, STSchema* pTSchema, int32_t vgId, + tb_uid_t uid, tb_uid_t suid) { int32_t sz = taosArrayGetSize(pDataBlocks); int32_t bufSize = sizeof(SSubmitReq); for (int32_t i = 0; i < sz; ++i) { @@ -1494,16 +1494,16 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks ASSERT(bufSize < 3 * 1024 * 1024); *pReq = taosMemoryCalloc(1, bufSize); - if(!(*pReq)) { + if (!(*pReq)) { terrno = TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_FAILED; } void* pDataBuf = *pReq; - int32_t msgLen = sizeof(SSubmitReq); + int32_t msgLen = sizeof(SSubmitReq); int32_t numOfBlks = 0; SRowBuilder rb = {0}; - tdSRowInit(&rb, 0); // TODO: use the latest version + tdSRowInit(&rb, 0); // TODO: use the latest version for (int32_t i = 0; i < sz; ++i) { SSDataBlock* pDataBlock = taosArrayGet(pDataBlocks, i); @@ -1511,8 +1511,8 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks int32_t rows = pDataBlock->info.rows; int32_t rowSize = pDataBlock->info.rowSize; int64_t groupId = pDataBlock->info.groupId; - - if(rb.nCols != colNum) { + + if (rb.nCols != colNum) { tdSRowSetTpInfo(&rb, colNum, pTSchema->flen); } @@ -1525,10 +1525,10 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks msgLen += sizeof(SSubmitBlk); int32_t dataLen = 0; - for (int32_t j = 0; j < rows; ++j) { // iterate by row - tdSRowResetBuf(&rb, POINTER_SHIFT(pDataBuf, msgLen)); // set row buf + for (int32_t j = 0; j < rows; ++j) { // iterate by row + tdSRowResetBuf(&rb, POINTER_SHIFT(pDataBuf, msgLen)); // set row buf printf("|"); - bool isStartKey = false; + bool isStartKey = false; for (int32_t k = 0; k < colNum; ++k) { // iterate by column SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, k); void* var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes); @@ -1536,7 +1536,8 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks case TSDB_DATA_TYPE_TIMESTAMP: if (!isStartKey) { isStartKey = true; - tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID, TSDB_DATA_TYPE_TIMESTAMP, TD_VTYPE_NORM, var, true, 0, 0); + tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID, TSDB_DATA_TYPE_TIMESTAMP, TD_VTYPE_NORM, var, true, + 0, 0); } else { tdAppendColValToRow(&rb, 2, TSDB_DATA_TYPE_TIMESTAMP, TD_VTYPE_NORM, var, true, 8, k); break; @@ -1629,14 +1630,14 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema) { blkHead->uid = htobe64(pDataBlock->info.uid); int32_t rows = pDataBlock->info.rows; - int32_t maxLen = TD_ROW_MAX_BYTES_FROM_SCHEMA(pTSchema); + /*int32_t maxLen = TD_ROW_MAX_BYTES_FROM_SCHEMA(pTSchema);*/ /*blkHead->dataLen = htonl(rows * maxLen);*/ blkHead->dataLen = 0; void* blockData = POINTER_SHIFT(submitBlk, sizeof(SSubmitBlk)); STSRow* rowData = blockData; - for (int32_t j = 0; j < pDataBlock->info.rows; j++) { + for (int32_t j = 0; j < rows; j++) { SRowBuilder rb = {0}; tdSRowInit(&rb, pTSchema->version); tdSRowSetTpInfo(&rb, pTSchema->numOfCols, pTSchema->flen); diff --git a/source/dnode/mnode/impl/inc/mndStb.h b/source/dnode/mnode/impl/inc/mndStb.h index a415d39434..28d3215b98 100644 --- a/source/dnode/mnode/impl/inc/mndStb.h +++ b/source/dnode/mnode/impl/inc/mndStb.h @@ -31,6 +31,11 @@ int32_t mndValidateStbInfo(SMnode *pMnode, SSTableMetaVersion *pStbs, int32_t n int32_t *pRspLen); int32_t mndGetNumOfStbs(SMnode *pMnode, char *dbName, int32_t *pNumOfStbs); +int32_t mndCheckCreateStbReq(SMCreateStbReq *pCreate); +SDbObj *mndAcquireDbByStb(SMnode *pMnode, const char *stbName); +int32_t mndBuildStbFromReq(SMnode *pMnode, SStbObj *pDst, SMCreateStbReq *pCreate, SDbObj *pDb); +int32_t mndAddStbToTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb); + #ifdef __cplusplus } #endif diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 8ae0d5d19c..cd76c7c8bb 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -342,7 +342,7 @@ void mndReleaseStb(SMnode *pMnode, SStbObj *pStb) { sdbRelease(pSdb, pStb); } -static SDbObj *mndAcquireDbByStb(SMnode *pMnode, const char *stbName) { +SDbObj *mndAcquireDbByStb(SMnode *pMnode, const char *stbName) { SName name = {0}; tNameFromString(&name, stbName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); @@ -463,7 +463,7 @@ static void *mndBuildVDropStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pStb, return pHead; } -static int32_t mndCheckCreateStbReq(SMCreateStbReq *pCreate) { +int32_t mndCheckCreateStbReq(SMCreateStbReq *pCreate) { if (pCreate->igExists < 0 || pCreate->igExists > 1) { terrno = TSDB_CODE_MND_INVALID_STB_OPTION; return -1; @@ -634,91 +634,96 @@ static SSchema *mndFindStbColumns(const SStbObj *pStb, const char *colName) { return NULL; } -static int32_t mndCreateStb(SMnode *pMnode, SNodeMsg *pReq, SMCreateStbReq *pCreate, SDbObj *pDb) { - SStbObj stbObj = {0}; - memcpy(stbObj.name, pCreate->name, TSDB_TABLE_FNAME_LEN); - memcpy(stbObj.db, pDb->name, TSDB_DB_FNAME_LEN); - stbObj.createdTime = taosGetTimestampMs(); - stbObj.updateTime = stbObj.createdTime; - stbObj.uid = mndGenerateUid(pCreate->name, TSDB_TABLE_FNAME_LEN); - stbObj.dbUid = pDb->uid; - stbObj.version = 1; - stbObj.nextColId = 1; - stbObj.xFilesFactor = pCreate->xFilesFactor; - stbObj.delay = pCreate->delay; - stbObj.ttl = pCreate->ttl; - stbObj.numOfColumns = pCreate->numOfColumns; - stbObj.numOfTags = pCreate->numOfTags; - stbObj.commentLen = pCreate->commentLen; - if (stbObj.commentLen > 0) { - stbObj.comment = taosMemoryCalloc(stbObj.commentLen, 1); - if (stbObj.comment == NULL) { +int32_t mndBuildStbFromReq(SMnode *pMnode, SStbObj *pDst, SMCreateStbReq *pCreate, SDbObj *pDb) { + memcpy(pDst->name, pCreate->name, TSDB_TABLE_FNAME_LEN); + memcpy(pDst->db, pDb->name, TSDB_DB_FNAME_LEN); + pDst->createdTime = taosGetTimestampMs(); + pDst->updateTime = pDst->createdTime; + pDst->uid = mndGenerateUid(pCreate->name, TSDB_TABLE_FNAME_LEN); + pDst->dbUid = pDb->uid; + pDst->version = 1; + pDst->nextColId = 1; + pDst->xFilesFactor = pCreate->xFilesFactor; + pDst->delay = pCreate->delay; + pDst->ttl = pCreate->ttl; + pDst->numOfColumns = pCreate->numOfColumns; + pDst->numOfTags = pCreate->numOfTags; + pDst->commentLen = pCreate->commentLen; + if (pDst->commentLen > 0) { + pDst->comment = taosMemoryCalloc(pDst->commentLen, 1); + if (pDst->comment == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - memcpy(stbObj.comment, pCreate->comment, stbObj.commentLen); + memcpy(pDst->comment, pCreate->comment, pDst->commentLen); } - stbObj.ast1Len = pCreate->ast1Len; - if (stbObj.ast1Len > 0) { - stbObj.pAst1 = taosMemoryCalloc(stbObj.ast1Len, 1); - if (stbObj.pAst1 == NULL) { + pDst->ast1Len = pCreate->ast1Len; + if (pDst->ast1Len > 0) { + pDst->pAst1 = taosMemoryCalloc(pDst->ast1Len, 1); + if (pDst->pAst1 == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - memcpy(stbObj.pAst1, pCreate->pAst1, stbObj.ast1Len); + memcpy(pDst->pAst1, pCreate->pAst1, pDst->ast1Len); } - stbObj.ast2Len = pCreate->ast2Len; - if (stbObj.ast2Len > 0) { - stbObj.pAst2 = taosMemoryCalloc(stbObj.ast2Len, 1); - if (stbObj.pAst2 == NULL) { + pDst->ast2Len = pCreate->ast2Len; + if (pDst->ast2Len > 0) { + pDst->pAst2 = taosMemoryCalloc(pDst->ast2Len, 1); + if (pDst->pAst2 == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - memcpy(stbObj.pAst2, pCreate->pAst2, stbObj.ast2Len); + memcpy(pDst->pAst2, pCreate->pAst2, pDst->ast2Len); } - stbObj.pColumns = taosMemoryCalloc(1, stbObj.numOfColumns * sizeof(SSchema)); - stbObj.pTags = taosMemoryCalloc(1, stbObj.numOfTags * sizeof(SSchema)); - if (stbObj.pColumns == NULL || stbObj.pTags == NULL) { + pDst->pColumns = taosMemoryCalloc(1, pDst->numOfColumns * sizeof(SSchema)); + pDst->pTags = taosMemoryCalloc(1, pDst->numOfTags * sizeof(SSchema)); + if (pDst->pColumns == NULL || pDst->pTags == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - for (int32_t i = 0; i < stbObj.numOfColumns; ++i) { + for (int32_t i = 0; i < pDst->numOfColumns; ++i) { SField *pField = taosArrayGet(pCreate->pColumns, i); - SSchema *pSchema = &stbObj.pColumns[i]; + SSchema *pSchema = &pDst->pColumns[i]; pSchema->type = pField->type; pSchema->bytes = pField->bytes; pSchema->flags = pField->flags; memcpy(pSchema->name, pField->name, TSDB_COL_NAME_LEN); - pSchema->colId = stbObj.nextColId; - stbObj.nextColId++; + pSchema->colId = pDst->nextColId; + pDst->nextColId++; } - for (int32_t i = 0; i < stbObj.numOfTags; ++i) { + for (int32_t i = 0; i < pDst->numOfTags; ++i) { SField *pField = taosArrayGet(pCreate->pTags, i); - SSchema *pSchema = &stbObj.pTags[i]; + SSchema *pSchema = &pDst->pTags[i]; pSchema->type = pField->type; pSchema->bytes = pField->bytes; memcpy(pSchema->name, pField->name, TSDB_COL_NAME_LEN); - pSchema->colId = stbObj.nextColId; - stbObj.nextColId++; + pSchema->colId = pDst->nextColId; + pDst->nextColId++; } + return 0; +} + +static int32_t mndCreateStb(SMnode *pMnode, SNodeMsg *pReq, SMCreateStbReq *pCreate, SDbObj *pDb) { + SStbObj stbObj = {0}; int32_t code = -1; + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_CREATE_STB, &pReq->rpcMsg); if (pTrans == NULL) goto _OVER; mDebug("trans:%d, used to create stb:%s", pTrans->id, pCreate->name); - mndTransSetDbInfo(pTrans, pDb); - if (mndSetCreateStbRedoLogs(pMnode, pTrans, pDb, &stbObj) != 0) goto _OVER; - if (mndSetCreateStbUndoLogs(pMnode, pTrans, pDb, &stbObj) != 0) goto _OVER; - if (mndSetCreateStbCommitLogs(pMnode, pTrans, pDb, &stbObj) != 0) goto _OVER; - if (mndSetCreateStbRedoActions(pMnode, pTrans, pDb, &stbObj) != 0) goto _OVER; - if (mndSetCreateStbUndoActions(pMnode, pTrans, pDb, &stbObj) != 0) goto _OVER; + if (mndBuildStbFromReq(pMnode, &stbObj, pCreate, pDb) != 0) { + goto _OVER; + } + + if (mndAddStbToTrans(pMnode, pTrans, pDb, &stbObj) < 0) goto _OVER; + if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; code = 0; @@ -727,6 +732,15 @@ _OVER: mndTransDrop(pTrans); return code; } +int32_t mndAddStbToTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) { + mndTransSetDbInfo(pTrans, pDb); + if (mndSetCreateStbRedoLogs(pMnode, pTrans, pDb, pStb) != 0) return -1; + if (mndSetCreateStbUndoLogs(pMnode, pTrans, pDb, pStb) != 0) return -1; + if (mndSetCreateStbCommitLogs(pMnode, pTrans, pDb, pStb) != 0) return -1; + if (mndSetCreateStbRedoActions(pMnode, pTrans, pDb, pStb) != 0) return -1; + if (mndSetCreateStbUndoActions(pMnode, pTrans, pDb, pStb) != 0) return -1; + return 0; +} static int32_t mndProcessMCreateStbReq(SNodeMsg *pReq) { SMnode *pMnode = pReq->pNode; diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 5c6e2ce771..1404b1cd94 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -290,6 +290,72 @@ int32_t mndAddStreamToTrans(SMnode *pMnode, SStreamObj *pStream, const char *ast return 0; } +static SStbObj *mndCreateStbForStream(SMnode *pMnode, STrans *pTrans, const SStreamObj *pStream, const char *user) { + SStbObj *pStb = NULL; + SDbObj *pDb = NULL; + SUserObj *pUser = NULL; + + SMCreateStbReq createReq = {0}; + tstrncpy(createReq.name, pStream->targetSTbName, TSDB_TABLE_FNAME_LEN); + createReq.numOfColumns = pStream->outputSchema.nCols; + createReq.numOfTags = 1; // group id + createReq.pColumns = taosArrayInit(createReq.numOfColumns, sizeof(SField)); + // build fields + // build tags + + if (mndCheckCreateStbReq(&createReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + goto _OVER; + } + + pStb = mndAcquireStb(pMnode, createReq.name); + if (pStb != NULL) { + terrno = TSDB_CODE_MND_STB_ALREADY_EXIST; + goto _OVER; + } + + pDb = mndAcquireDbByStb(pMnode, createReq.name); + if (pDb == NULL) { + terrno = TSDB_CODE_MND_DB_NOT_SELECTED; + goto _OVER; + } + + pUser = mndAcquireUser(pMnode, user); + if (pUser == NULL) { + goto _OVER; + } + + if (mndCheckWriteAuth(pUser, pDb) != 0) { + goto _OVER; + } + + int32_t numOfStbs = -1; + mndGetNumOfStbs(pMnode, pDb->name, &numOfStbs); + if (pDb->cfg.numOfStables == 1 && numOfStbs != 0) { + terrno = TSDB_CODE_MND_SINGLE_STB_MODE_DB; + goto _OVER; + } + + SStbObj stbObj = {0}; + + if (mndBuildStbFromReq(pMnode, &stbObj, &createReq, pDb) != 0) { + goto _OVER; + } + + if (mndBuildStbFromReq(pMnode, pStb, &createReq, pDb) != 0) { + goto _OVER; + } + + if (mndAddStbToTrans(pMnode, pTrans, pDb, &stbObj) < 0) goto _OVER; + + return pStb; +_OVER: + mndReleaseStb(pMnode, pStb); + mndReleaseDb(pMnode, pDb); + mndReleaseUser(pMnode, pUser); + return NULL; +} + static int32_t mndCreateStream(SMnode *pMnode, SNodeMsg *pReq, SCMCreateStreamReq *pCreate, SDbObj *pDb) { mDebug("stream:%s to create", pCreate->name); SStreamObj streamObj = {0}; @@ -311,13 +377,21 @@ static int32_t mndCreateStream(SMnode *pMnode, SNodeMsg *pReq, SCMCreateStreamRe streamObj.trigger = pCreate->triggerType; streamObj.waterMark = pCreate->watermark; - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_CREATE_STREAM, &pReq->rpcMsg); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_CREATE_STREAM, &pReq->rpcMsg); if (pTrans == NULL) { mError("stream:%s, failed to create since %s", pCreate->name, terrstr()); return -1; } mDebug("trans:%d, used to create stream:%s", pTrans->id, pCreate->name); +#if 0 + if (mndCreateStbForStream(pMnode, pTrans, &streamObj, pReq->user) < 0) { + mError("trans:%d, failed to add stream since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } +#endif + if (mndAddStreamToTrans(pMnode, &streamObj, pCreate->ast, pCreate->triggerType, pCreate->watermark, pTrans) != 0) { mError("trans:%d, failed to add stream since %s", pTrans->id, terrstr()); mndTransDrop(pTrans); diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index 5fa77c9964..6bf17b8a82 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -57,7 +57,7 @@ int32_t logStoreAppendEntry(SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry) { syncMeta.seqNum = pEntry->seqNum; syncMeta.term = pEntry->term; code = walWriteWithSyncInfo(pWal, pEntry->index, pEntry->originalRpcType, syncMeta, pEntry->data, pEntry->dataLen); - perror("wal write error: "); + if (code < 0) perror("wal write error: "); assert(code == 0); walFsync(pWal, true); From 6b05911e45f5c34bace8ab9721ff58e9277ad49d Mon Sep 17 00:00:00 2001 From: tangfangzhi Date: Sat, 7 May 2022 06:49:06 +0800 Subject: [PATCH 58/83] remove windows test --- Jenkinsfile2 | 7 ------- 1 file changed, 7 deletions(-) diff --git a/Jenkinsfile2 b/Jenkinsfile2 index 49e13b7831..d827c6640a 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -168,13 +168,6 @@ pipeline { stages { stage('run test') { parallel { - stage('windows test') { - agent {label " windows11 "} - steps { - pre_test_win() - pre_test_build_win() - } - } stage('linux test') { agent{label " slave3_0 || slave15 || slave16 || slave17 "} options { skipDefaultCheckout() } From 20e0d336c7d1e28f40d663752e73ce4778d7c491 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Sat, 7 May 2022 08:47:52 +0800 Subject: [PATCH 59/83] fix memory leak issue --- source/libs/executor/src/executorimpl.c | 1 - 1 file changed, 1 deletion(-) diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 10d5e717bd..dc0070cbee 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -2826,7 +2826,6 @@ void relocateColumnData(SSDataBlock* pBlock, const SArray* pColMatchInfo, SArray } if (p->info.colId == pmInfo->colId) { - colDataDestroy(taosArrayGet(pBlock->pDataBlock, pmInfo->targetSlotId)); taosArraySet(pBlock->pDataBlock, pmInfo->targetSlotId, p); i++; j++; From cb4dd0f948c738935499f4a6eb7edf393b9fa7b8 Mon Sep 17 00:00:00 2001 From: jiacy-jcy <714897623@qq.com> Date: Sat, 7 May 2022 09:03:46 +0800 Subject: [PATCH 60/83] update last.py --- tests/system-test/2-query/last.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/system-test/2-query/last.py b/tests/system-test/2-query/last.py index a5407c7b12..531afd0117 100644 --- a/tests/system-test/2-query/last.py +++ b/tests/system-test/2-query/last.py @@ -164,12 +164,12 @@ class TDTestCase: tdSql.query("select last(col8) from db.stb_1") tdSql.checkRows(1) tdSql.checkData(0, 0, 'taosdata10') - # tdSql.query("select last(col9) from stb_1") - # tdSql.checkRows(1) - # tdSql.checkData(0, 0, '涛思数据10') - # tdSql.query("select last(col9) from db.stb_1") - # tdSql.checkRows(1) - # tdSql.checkData(0, 0, '涛思数据10') + tdSql.query("select last(col9) from stb_1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '涛思数据10') + tdSql.query("select last(col9) from db.stb_1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '涛思数据10') tdSql.execute('''create table ntb(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, From 285c396b0bcc82923dfaf6afb25117c9ba6463f3 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 7 May 2022 09:23:06 +0800 Subject: [PATCH 61/83] refactor: adjust cluster mgmt code --- source/dnode/mnode/impl/src/mndCluster.c | 68 ++++++++++++------------ 1 file changed, 35 insertions(+), 33 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndCluster.c b/source/dnode/mnode/impl/src/mndCluster.c index 257db0dcd4..96845fcd42 100644 --- a/source/dnode/mnode/impl/src/mndCluster.c +++ b/source/dnode/mnode/impl/src/mndCluster.c @@ -17,8 +17,8 @@ #include "mndCluster.h" #include "mndShow.h" -#define TSDB_CLUSTER_VER_NUMBE 1 -#define TSDB_CLUSTER_RESERVE_SIZE 64 +#define CLUSTER_VER_NUMBE 1 +#define CLUSTER_RESERVE_SIZE 64 static SSdbRaw *mndClusterActionEncode(SClusterObj *pCluster); static SSdbRow *mndClusterActionDecode(SSdbRaw *pRaw); @@ -30,14 +30,16 @@ static int32_t mndRetrieveClusters(SNodeMsg *pMsg, SShowObj *pShow, SSDataBlock static void mndCancelGetNextCluster(SMnode *pMnode, void *pIter); int32_t mndInitCluster(SMnode *pMnode) { - SSdbTable table = {.sdbType = SDB_CLUSTER, - .keyType = SDB_KEY_INT64, - .deployFp = (SdbDeployFp)mndCreateDefaultCluster, - .encodeFp = (SdbEncodeFp)mndClusterActionEncode, - .decodeFp = (SdbDecodeFp)mndClusterActionDecode, - .insertFp = (SdbInsertFp)mndClusterActionInsert, - .updateFp = (SdbUpdateFp)mndClusterActionUpdate, - .deleteFp = (SdbDeleteFp)mndClusterActionDelete}; + SSdbTable table = { + .sdbType = SDB_CLUSTER, + .keyType = SDB_KEY_INT64, + .deployFp = (SdbDeployFp)mndCreateDefaultCluster, + .encodeFp = (SdbEncodeFp)mndClusterActionEncode, + .decodeFp = (SdbDecodeFp)mndClusterActionDecode, + .insertFp = (SdbInsertFp)mndClusterActionInsert, + .updateFp = (SdbUpdateFp)mndClusterActionUpdate, + .deleteFp = (SdbDeleteFp)mndClusterActionDelete, + }; mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_CLUSTER, mndRetrieveClusters); mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_CLUSTER, mndCancelGetNextCluster); @@ -79,19 +81,19 @@ int64_t mndGetClusterId(SMnode *pMnode) { static SSdbRaw *mndClusterActionEncode(SClusterObj *pCluster) { terrno = TSDB_CODE_OUT_OF_MEMORY; - SSdbRaw *pRaw = sdbAllocRaw(SDB_CLUSTER, TSDB_CLUSTER_VER_NUMBE, sizeof(SClusterObj) + TSDB_CLUSTER_RESERVE_SIZE); - if (pRaw == NULL) goto CLUSTER_ENCODE_OVER; + SSdbRaw *pRaw = sdbAllocRaw(SDB_CLUSTER, CLUSTER_VER_NUMBE, sizeof(SClusterObj) + CLUSTER_RESERVE_SIZE); + if (pRaw == NULL) goto _OVER; int32_t dataPos = 0; - SDB_SET_INT64(pRaw, dataPos, pCluster->id, CLUSTER_ENCODE_OVER) - SDB_SET_INT64(pRaw, dataPos, pCluster->createdTime, CLUSTER_ENCODE_OVER) - SDB_SET_INT64(pRaw, dataPos, pCluster->updateTime, CLUSTER_ENCODE_OVER) - SDB_SET_BINARY(pRaw, dataPos, pCluster->name, TSDB_CLUSTER_ID_LEN, CLUSTER_ENCODE_OVER) - SDB_SET_RESERVE(pRaw, dataPos, TSDB_CLUSTER_RESERVE_SIZE, CLUSTER_ENCODE_OVER) + SDB_SET_INT64(pRaw, dataPos, pCluster->id, _OVER) + SDB_SET_INT64(pRaw, dataPos, pCluster->createdTime, _OVER) + SDB_SET_INT64(pRaw, dataPos, pCluster->updateTime, _OVER) + SDB_SET_BINARY(pRaw, dataPos, pCluster->name, TSDB_CLUSTER_ID_LEN, _OVER) + SDB_SET_RESERVE(pRaw, dataPos, CLUSTER_RESERVE_SIZE, _OVER) terrno = 0; -CLUSTER_ENCODE_OVER: +_OVER: if (terrno != 0) { mError("cluster:%" PRId64 ", failed to encode to raw:%p since %s", pCluster->id, pRaw, terrstr()); sdbFreeRaw(pRaw); @@ -106,29 +108,29 @@ static SSdbRow *mndClusterActionDecode(SSdbRaw *pRaw) { terrno = TSDB_CODE_OUT_OF_MEMORY; int8_t sver = 0; - if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto CLUSTER_DECODE_OVER; + if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER; - if (sver != TSDB_CLUSTER_VER_NUMBE) { + if (sver != CLUSTER_VER_NUMBE) { terrno = TSDB_CODE_SDB_INVALID_DATA_VER; - goto CLUSTER_DECODE_OVER; + goto _OVER; } SSdbRow *pRow = sdbAllocRow(sizeof(SClusterObj)); - if (pRow == NULL) goto CLUSTER_DECODE_OVER; + if (pRow == NULL) goto _OVER; SClusterObj *pCluster = sdbGetRowObj(pRow); - if (pCluster == NULL) goto CLUSTER_DECODE_OVER; + if (pCluster == NULL) goto _OVER; int32_t dataPos = 0; - SDB_GET_INT64(pRaw, dataPos, &pCluster->id, CLUSTER_DECODE_OVER) - SDB_GET_INT64(pRaw, dataPos, &pCluster->createdTime, CLUSTER_DECODE_OVER) - SDB_GET_INT64(pRaw, dataPos, &pCluster->updateTime, CLUSTER_DECODE_OVER) - SDB_GET_BINARY(pRaw, dataPos, pCluster->name, TSDB_CLUSTER_ID_LEN, CLUSTER_DECODE_OVER) - SDB_GET_RESERVE(pRaw, dataPos, TSDB_CLUSTER_RESERVE_SIZE, CLUSTER_DECODE_OVER) + SDB_GET_INT64(pRaw, dataPos, &pCluster->id, _OVER) + SDB_GET_INT64(pRaw, dataPos, &pCluster->createdTime, _OVER) + SDB_GET_INT64(pRaw, dataPos, &pCluster->updateTime, _OVER) + SDB_GET_BINARY(pRaw, dataPos, pCluster->name, TSDB_CLUSTER_ID_LEN, _OVER) + SDB_GET_RESERVE(pRaw, dataPos, CLUSTER_RESERVE_SIZE, _OVER) terrno = 0; -CLUSTER_DECODE_OVER: +_OVER: if (terrno != 0) { mError("cluster:%" PRId64 ", failed to decode from raw:%p since %s", pCluster->id, pRaw, terrstr()); taosMemoryFreeClear(pRow); @@ -161,7 +163,7 @@ static int32_t mndCreateDefaultCluster(SMnode *pMnode) { int32_t code = taosGetSystemUUID(clusterObj.name, TSDB_CLUSTER_ID_LEN); if (code != 0) { - strcpy(clusterObj.name, "tdengine2.0"); + strcpy(clusterObj.name, "tdengine3.0"); mError("failed to get name from system, set to default val %s", clusterObj.name); } @@ -190,8 +192,8 @@ static int32_t mndRetrieveClusters(SNodeMsg *pMsg, SShowObj *pShow, SSDataBlock if (pShow->pIter == NULL) break; cols = 0; - SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char*) &pCluster->id, false); + SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, (const char *)&pCluster->id, false); char buf[tListLen(pCluster->name) + VARSTR_HEADER_SIZE] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(buf, pCluster->name, pShow->pMeta->pSchemas[cols].bytes); @@ -200,7 +202,7 @@ static int32_t mndRetrieveClusters(SNodeMsg *pMsg, SShowObj *pShow, SSDataBlock colDataAppend(pColInfo, numOfRows, buf, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char*) &pCluster->createdTime, false); + colDataAppend(pColInfo, numOfRows, (const char *)&pCluster->createdTime, false); sdbRelease(pSdb, pCluster); numOfRows++; From f018ccc28e91ac619b8fdf0267f1a56114c1781d Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 7 May 2022 09:36:36 +0800 Subject: [PATCH 62/83] refactor: adjust mnode def --- source/dnode/mnode/impl/inc/mndDef.h | 84 ++++++++++++---------------- source/dnode/mnode/impl/src/mndDef.c | 15 +++++ 2 files changed, 51 insertions(+), 48 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 89638524ae..c113cd0bbe 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -72,7 +72,7 @@ typedef enum { TRN_TYPE_DROP_USER = 1003, TRN_TYPE_CREATE_FUNC = 1004, TRN_TYPE_DROP_FUNC = 1005, - + TRN_TYPE_CREATE_SNODE = 1006, TRN_TYPE_DROP_SNODE = 1007, TRN_TYPE_CREATE_QNODE = 1008, @@ -115,7 +115,10 @@ typedef enum { TRN_TYPE_STB_SCOPE_END, } ETrnType; -typedef enum { TRN_POLICY_ROLLBACK = 0, TRN_POLICY_RETRY = 1 } ETrnPolicy; +typedef enum { + TRN_POLICY_ROLLBACK = 0, + TRN_POLICY_RETRY = 1, +} ETrnPolicy; typedef enum { DND_REASON_ONLINE = 0, @@ -131,6 +134,15 @@ typedef enum { DND_REASON_OTHERS } EDndReason; +typedef enum { + CONSUMER_UPDATE__TOUCH = 1, + CONSUMER_UPDATE__ADD, + CONSUMER_UPDATE__REMOVE, + CONSUMER_UPDATE__LOST, + CONSUMER_UPDATE__RECOVER, + CONSUMER_UPDATE__MODIFY, +} ECsmUpdateType; + typedef struct { int32_t id; ETrnStage stage; @@ -386,7 +398,6 @@ typedef struct { int32_t codeSize; char* pComment; char* pCode; - char pData[]; } SFuncObj; typedef struct { @@ -425,18 +436,8 @@ typedef struct { int64_t offset; } SMqOffsetObj; -static FORCE_INLINE int32_t tEncodeSMqOffsetObj(void** buf, const SMqOffsetObj* pOffset) { - int32_t tlen = 0; - tlen += taosEncodeString(buf, pOffset->key); - tlen += taosEncodeFixedI64(buf, pOffset->offset); - return tlen; -} - -static FORCE_INLINE void* tDecodeSMqOffsetObj(void* buf, SMqOffsetObj* pOffset) { - buf = taosDecodeStringTo(buf, pOffset->key); - buf = taosDecodeFixedI64(buf, &pOffset->offset); - return buf; -} +int32_t tEncodeSMqOffsetObj(void** buf, const SMqOffsetObj* pOffset); +void* tDecodeSMqOffsetObj(void* buf, SMqOffsetObj* pOffset); typedef struct { char name[TSDB_TOPIC_FNAME_LEN]; @@ -459,26 +460,15 @@ typedef struct { SSchemaWrapper schema; } SMqTopicObj; -enum { - CONSUMER_UPDATE__TOUCH = 1, - CONSUMER_UPDATE__ADD, - CONSUMER_UPDATE__REMOVE, - CONSUMER_UPDATE__LOST, - CONSUMER_UPDATE__RECOVER, - CONSUMER_UPDATE__MODIFY, -}; - typedef struct { - int64_t consumerId; - char cgroup[TSDB_CGROUP_LEN]; - char appId[TSDB_CGROUP_LEN]; - int8_t updateType; // used only for update - int32_t epoch; - int32_t status; - // hbStatus is not applicable to serialization - int32_t hbStatus; - // lock is used for topics update - SRWLatch lock; + int64_t consumerId; + char cgroup[TSDB_CGROUP_LEN]; + char appId[TSDB_CGROUP_LEN]; + int8_t updateType; // used only for update + int32_t epoch; + int32_t status; + int32_t hbStatus; // hbStatus is not applicable to serialization + SRWLatch lock; // lock is used for topics update SArray* currentTopics; // SArray SArray* rebNewTopics; // SArray SArray* rebRemovedTopics; // SArray @@ -492,7 +482,6 @@ typedef struct { int64_t upTime; int64_t subscribeTime; int64_t rebalanceTime; - } SMqConsumerObj; SMqConsumerObj* tNewSMqConsumerObj(int64_t consumerId, char cgroup[TSDB_CGROUP_LEN]); @@ -581,19 +570,18 @@ typedef struct { } SMqRebOutputObj; typedef struct { - char name[TSDB_TOPIC_FNAME_LEN]; - char sourceDb[TSDB_DB_FNAME_LEN]; - char targetDb[TSDB_DB_FNAME_LEN]; - char targetSTbName[TSDB_TABLE_FNAME_LEN]; - int64_t createTime; - int64_t updateTime; - int64_t uid; - int64_t dbUid; - int32_t version; - int32_t vgNum; - SRWLatch lock; - int8_t status; - // int32_t sqlLen; + char name[TSDB_TOPIC_FNAME_LEN]; + char sourceDb[TSDB_DB_FNAME_LEN]; + char targetDb[TSDB_DB_FNAME_LEN]; + char targetSTbName[TSDB_TABLE_FNAME_LEN]; + int64_t createTime; + int64_t updateTime; + int64_t uid; + int64_t dbUid; + int32_t version; + int32_t vgNum; + SRWLatch lock; + int8_t status; int8_t createdBy; // STREAM_CREATED_BY__USER or SMA int32_t fixedSinkVgId; // 0 for shuffle int64_t smaId; // 0 for unused diff --git a/source/dnode/mnode/impl/src/mndDef.c b/source/dnode/mnode/impl/src/mndDef.c index 5bce6ac218..b8c120905c 100644 --- a/source/dnode/mnode/impl/src/mndDef.c +++ b/source/dnode/mnode/impl/src/mndDef.c @@ -13,12 +13,14 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include "mndDef.h" #include "mndConsumer.h" SMqConsumerObj *tNewSMqConsumerObj(int64_t consumerId, char cgroup[TSDB_CGROUP_LEN]) { SMqConsumerObj *pConsumer = taosMemoryCalloc(1, sizeof(SMqConsumerObj)); if (pConsumer == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } @@ -515,3 +517,16 @@ int32_t tDecodeSStreamObj(SCoder *pDecoder, SStreamObj *pObj) { #endif return 0; } + +int32_t tEncodeSMqOffsetObj(void **buf, const SMqOffsetObj *pOffset) { + int32_t tlen = 0; + tlen += taosEncodeString(buf, pOffset->key); + tlen += taosEncodeFixedI64(buf, pOffset->offset); + return tlen; +} + +void *tDecodeSMqOffsetObj(void *buf, SMqOffsetObj *pOffset) { + buf = taosDecodeStringTo(buf, pOffset->key); + buf = taosDecodeFixedI64(buf, &pOffset->offset); + return buf; +} \ No newline at end of file From ce0841e398972eab05044a2f29a727b6e4ee8316 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sat, 7 May 2022 01:58:01 +0000 Subject: [PATCH 63/83] fix: auto-create table --- source/dnode/vnode/src/meta/metaTable.c | 2 +- source/libs/parser/src/parInsert.c | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 397e074061..ffe176f85e 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -196,7 +196,7 @@ int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq) { int8_t type; int64_t ctime; tb_uid_t suid; - int c, ret; + int c = 0, ret; // search & delete the name idx tdbDbcOpen(pMeta->pNameIdx, &pNameIdxc, &pMeta->txn); diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index e82873b923..8396a33122 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -279,6 +279,7 @@ static void buildMsgHeader(STableDataBlocks* src, SVgDataBlocks* blocks) { int32_t numOfBlocks = blocks->numOfTables; while (numOfBlocks--) { int32_t dataLen = blk->dataLen; + int32_t schemaLen = blk->schemaLen; blk->uid = htobe64(blk->uid); blk->suid = htobe64(blk->suid); blk->padding = htonl(blk->padding); @@ -286,7 +287,7 @@ static void buildMsgHeader(STableDataBlocks* src, SVgDataBlocks* blocks) { blk->dataLen = htonl(blk->dataLen); blk->schemaLen = htonl(blk->schemaLen); blk->numOfRows = htons(blk->numOfRows); - blk = (SSubmitBlk*)(blk->data + dataLen); + blk = (SSubmitBlk*)(blk->data + schemaLen + dataLen); } } From 59fe535581a86f35f647d4f6f180e1688196def3 Mon Sep 17 00:00:00 2001 From: jiacy-jcy <714897623@qq.com> Date: Sat, 7 May 2022 10:19:26 +0800 Subject: [PATCH 64/83] update --- tests/system-test/fulltest.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 884d18f7cd..4149c2cd9d 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -11,12 +11,17 @@ python3 ./test.py -f 0-others/taosShellNetChk.py python3 ./test.py -f 2-query/distinct.py python3 ./test.py -f 2-query/varchar.py -#python3 ./test.py -f 2-query/timezone.py +python3 ./test.py -f 2-query/timezone.py python3 ./test.py -f 2-query/Now.py python3 ./test.py -f 2-query/Today.py python3 ./test.py -f 2-query/max.py python3 ./test.py -f 2-query/min.py python3 ./test.py -f 2-query/count.py +python3 ./test.py -f 2-query/last.py +#python3 ./test.py -f 2-query/To_iso8601.py +python3 ./test.py -f 2-query/To_unixtimestamp.py +python3 ./test.py -f 2-query/timetruncate.py + # python3 ./test.py -f 2-query/Timediff.py #python3 ./test.py -f 2-query/cast.py From 861fde25087bec9594cf7c6f324e07754ed9a34c Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Sat, 7 May 2022 10:21:51 +0800 Subject: [PATCH 65/83] fix(stream): create stb in stream --- example/src/tstream.c | 8 +++--- source/dnode/mnode/impl/inc/mndDef.h | 3 +-- source/dnode/mnode/impl/src/mndSma.c | 1 - source/dnode/mnode/impl/src/mndStream.c | 34 +++++++++++++++---------- source/libs/wal/src/walRead.c | 5 +++- 5 files changed, 30 insertions(+), 21 deletions(-) diff --git a/example/src/tstream.c b/example/src/tstream.c index 5bd833213d..65fd005954 100644 --- a/example/src/tstream.c +++ b/example/src/tstream.c @@ -81,10 +81,10 @@ int32_t create_stream() { /*const char* sql = "select min(k), max(k), sum(k) as sum_of_k from st1";*/ /*const char* sql = "select sum(k) from tu1 interval(10m)";*/ /*pRes = tmq_create_stream(pConn, "stream1", "out1", sql);*/ - pRes = - taos_query(pConn, - "create stream stream1 trigger window_close as select _wstartts, min(k), max(k), sum(k) as sum_of_k " - "from tu1 interval(10m)"); + pRes = taos_query( + pConn, + "create stream stream1 trigger window_close into outstb as select _wstartts, min(k), max(k), sum(k) as sum_of_k " + "from tu1 interval(10m)"); if (taos_errno(pRes) != 0) { printf("failed to create stream stream1, reason:%s\n", taos_errstr(pRes)); return -1; diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 89638524ae..9ae7922f6b 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -72,7 +72,7 @@ typedef enum { TRN_TYPE_DROP_USER = 1003, TRN_TYPE_CREATE_FUNC = 1004, TRN_TYPE_DROP_FUNC = 1005, - + TRN_TYPE_CREATE_SNODE = 1006, TRN_TYPE_DROP_SNODE = 1007, TRN_TYPE_CREATE_QNODE = 1008, @@ -601,7 +601,6 @@ typedef struct { int32_t triggerParam; int64_t waterMark; char* sql; - char* logicalPlan; char* physicalPlan; SArray* tasks; // SArray> SSchemaWrapper outputSchema; diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index 8619df978b..91a2bd4740 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -419,7 +419,6 @@ static int32_t mndCreateSma(SMnode *pMnode, SNodeMsg *pReq, SMCreateSmaReq *pCre streamObj.fixedSinkVgId = smaObj.dstVgId; streamObj.smaId = smaObj.uid; /*streamObj.physicalPlan = "";*/ - streamObj.logicalPlan = "not implemented"; int32_t code = -1; STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_CREATE_SMA, &pReq->rpcMsg); diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 1404b1cd94..aa7f16e10d 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -290,7 +290,7 @@ int32_t mndAddStreamToTrans(SMnode *pMnode, SStreamObj *pStream, const char *ast return 0; } -static SStbObj *mndCreateStbForStream(SMnode *pMnode, STrans *pTrans, const SStreamObj *pStream, const char *user) { +static int32_t mndCreateStbForStream(SMnode *pMnode, STrans *pTrans, const SStreamObj *pStream, const char *user) { SStbObj *pStb = NULL; SDbObj *pDb = NULL; SUserObj *pUser = NULL; @@ -301,7 +301,22 @@ static SStbObj *mndCreateStbForStream(SMnode *pMnode, STrans *pTrans, const SStr createReq.numOfTags = 1; // group id createReq.pColumns = taosArrayInit(createReq.numOfColumns, sizeof(SField)); // build fields + taosArraySetSize(createReq.pColumns, createReq.numOfColumns); + for (int32_t i = 0; i < createReq.numOfColumns; i++) { + SField *pField = taosArrayGet(createReq.pColumns, i); + tstrncpy(pField->name, pStream->outputSchema.pSchema[i].name, TSDB_COL_NAME_LEN); + pField->flags = pStream->outputSchema.pSchema[i].flags; + pField->type = pStream->outputSchema.pSchema[i].type; + pField->bytes = pStream->outputSchema.pSchema[i].bytes; + } + createReq.pTags = taosArrayInit(createReq.numOfTags, sizeof(SField)); + taosArraySetSize(createReq.pTags, 1); // build tags + SField *pField = taosArrayGet(createReq.pTags, 0); + strcpy(pField->name, "group_id"); + pField->type = TSDB_DATA_TYPE_UBIGINT; + pField->flags = 0; + pField->bytes = 8; if (mndCheckCreateStbReq(&createReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; @@ -342,18 +357,14 @@ static SStbObj *mndCreateStbForStream(SMnode *pMnode, STrans *pTrans, const SStr goto _OVER; } - if (mndBuildStbFromReq(pMnode, pStb, &createReq, pDb) != 0) { - goto _OVER; - } - if (mndAddStbToTrans(pMnode, pTrans, pDb, &stbObj) < 0) goto _OVER; - return pStb; + return 0; _OVER: mndReleaseStb(pMnode, pStb); mndReleaseDb(pMnode, pDb); mndReleaseUser(pMnode, pUser); - return NULL; + return -1; } static int32_t mndCreateStream(SMnode *pMnode, SNodeMsg *pReq, SCMCreateStreamReq *pCreate, SDbObj *pDb) { @@ -373,7 +384,6 @@ static int32_t mndCreateStream(SMnode *pMnode, SNodeMsg *pReq, SCMCreateStreamRe streamObj.fixedSinkVgId = 0; streamObj.smaId = 0; /*streamObj.physicalPlan = "";*/ - streamObj.logicalPlan = "not implemented"; streamObj.trigger = pCreate->triggerType; streamObj.waterMark = pCreate->watermark; @@ -384,16 +394,14 @@ static int32_t mndCreateStream(SMnode *pMnode, SNodeMsg *pReq, SCMCreateStreamRe } mDebug("trans:%d, used to create stream:%s", pTrans->id, pCreate->name); -#if 0 - if (mndCreateStbForStream(pMnode, pTrans, &streamObj, pReq->user) < 0) { + if (mndAddStreamToTrans(pMnode, &streamObj, pCreate->ast, pCreate->triggerType, pCreate->watermark, pTrans) != 0) { mError("trans:%d, failed to add stream since %s", pTrans->id, terrstr()); mndTransDrop(pTrans); return -1; } -#endif - if (mndAddStreamToTrans(pMnode, &streamObj, pCreate->ast, pCreate->triggerType, pCreate->watermark, pTrans) != 0) { - mError("trans:%d, failed to add stream since %s", pTrans->id, terrstr()); + if (streamObj.targetSTbName[0] && mndCreateStbForStream(pMnode, pTrans, &streamObj, pReq->user) < 0) { + mError("trans:%d, failed to create stb for stream since %s", pTrans->id, terrstr()); mndTransDrop(pTrans); return -1; } diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index 70a3559cd9..4fe07029f1 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -49,7 +49,10 @@ void walCloseReadHandle(SWalReadHandle *pRead) { taosMemoryFree(pRead); } -int32_t walRegisterRead(SWalReadHandle *pRead, int64_t ver) { return 0; } +int32_t walRegisterRead(SWalReadHandle *pRead, int64_t ver) { + // TODO + return 0; +} static int32_t walReadSeekFilePos(SWalReadHandle *pRead, int64_t fileFirstVer, int64_t ver) { int code = 0; From bb3681cd713577f475746fc49d28c828501d9e33 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Fri, 6 May 2022 21:44:25 +0800 Subject: [PATCH 66/83] feat(query): add histogram param parsing --- source/libs/function/src/builtinsimpl.c | 52 +++++++++++++++++++++++-- 1 file changed, 49 insertions(+), 3 deletions(-) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 64cca47da0..69d3e079c6 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -1973,13 +1973,59 @@ bool histogramFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo *pResultIn } int32_t histogramFunction(SqlFunctionCtx *pCtx) { + int32_t numOfElems = 0; + SHistoFuncInfo* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); + + SInputColumnInfoData* pInput = &pCtx->input; + SColumnInfoData* pCol = pInput->pData[0]; + + int32_t type = pInput->pData[0]->info.type; + + int32_t start = pInput->startRowIndex; + int32_t numOfRows = pInput->numOfRows; + + for (int32_t i = start; i < numOfRows + start; ++i) { + if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) { + continue; + } + + char* data = colDataGetData(pCol, i); + double v; + GET_TYPED_DATA(v, double, type, data); + + for (int32_t k = 0; k < pInfo->numOfBins; ++k) { + if (v > pInfo->bins[k].lower && v <= pInfo->bins[k].upper) { + pInfo->bins[k].count++; + numOfElems++; + break; + } + } + } + return TSDB_CODE_SUCCESS; } int32_t histogramFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { SHistoFuncInfo* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); - //if (pInfo->hasResult == true) { - // SET_DOUBLE_VAL(&pInfo->result, pInfo->max - pInfo->min); - //} + int32_t slotId = pCtx->pExpr->base.resSchema.slotId; + SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); + + int32_t currentRow = pBlock->info.rows; + + for (int32_t i = 0; i < pInfo->numOfBins; ++i) { + int32_t len; + char buf[400] = {0}; + if (!pInfo->normalized) { + len = sprintf(buf + VARSTR_HEADER_SIZE, "{\"lower_bin\":%g, \"upper_bin\":%g, \"count\":%"PRId64"}", + pInfo->bins[i].lower, pInfo->bins[i].upper, pInfo->bins[i].count); + } else { + len = sprintf(buf + VARSTR_HEADER_SIZE, "{\"lower_bin\":%g, \"upper_bin\":%g, \"count\":%lf}", + pInfo->bins[i].lower, pInfo->bins[i].upper, pInfo->bins[i].percentage); + } + varDataSetLen(buf, len); + colDataAppend(pCol, currentRow, buf, false); + currentRow++; + } + return functionFinalize(pCtx, pBlock); } From f942f144e9b3958a72189a2d568c7c7496f77b68 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Sat, 7 May 2022 11:12:46 +0800 Subject: [PATCH 67/83] feat: bitmap operation optimization --- source/dnode/vnode/src/inc/tsdb.h | 5 ++-- source/dnode/vnode/src/tsdb/tsdbCommit.c | 10 ++++---- source/dnode/vnode/src/tsdb/tsdbRead.c | 29 +++++++++++++++------- source/dnode/vnode/src/tsdb/tsdbReadImpl.c | 29 ++++------------------ 4 files changed, 32 insertions(+), 41 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index d84c1b1d07..58003a97d7 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -325,9 +325,8 @@ typedef struct { typedef struct { int16_t colId; uint16_t type : 6; - uint16_t blen : 10; // bitmap length(TODO: full UT for the bitmap compress of various data input) - uint32_t bitmap : 1; // 0: no bitmap if all rows are NORM, 1: has bitmap if has NULL/NORM rows - uint32_t len : 31; // data length + bitmap length + uint16_t blen : 10; // 0 no bitmap if all rows are NORM, > 0 bitmap length + uint32_t len; // data length + bitmap length uint32_t offset; } SBlockColV0; diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index a008c4a0e5..f0b1baf1da 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -943,16 +943,16 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDF &(pAggrBlkCol->numOfNull)); if (pAggrBlkCol->numOfNull == 0) { - TD_SET_COL_ROWS_NORM(pBlockCol); + pBlockCol->blen = 0; } else { - TD_SET_COL_ROWS_MISC(pBlockCol); + pBlockCol->blen = 1; } ++nColsOfBlockSma; } else if (tdIsBitmapBlkNorm(pDataCol->pBitmap, rowsToWrite, pDataCols->bitmapMode)) { // check if all rows normal - TD_SET_COL_ROWS_NORM(pBlockCol); + pBlockCol->blen = 0; } else { - TD_SET_COL_ROWS_MISC(pBlockCol); + pBlockCol->blen = 1; } ++nColsNotAllNull; @@ -985,7 +985,7 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDF #ifdef TD_SUPPORT_BITMAP int32_t tBitmaps = 0; int32_t tBitmapsLen = 0; - if ((ncol != 0) && !TD_COL_ROWS_NORM(pBlockCol)) { + if ((ncol != 0) && (pBlockCol->blen > 0)) { tBitmaps = isSuper ? sBitmaps : nBitmaps; } #endif diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index aa305dd71c..0b9f21dbcf 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -1619,7 +1619,7 @@ static int32_t mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capa SCellVal sVal = {0}; TSKEY rowKey = TSKEY_INITIAL_VAL; int32_t nResult = 0; - bool isMerge = true; + int32_t mergeOption = 0; // 0 discard 1 overwrite 2 merge // the schema version info is embeded in STSRow int32_t numOfColsOfRow1 = 0; @@ -1715,12 +1715,18 @@ static int32_t mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capa if (colId == PRIMARYKEY_TIMESTAMP_COL_ID) { rowKey = *(TSKEY*)sVal.val; if (rowKey != *lastRowKey) { - isMerge = false; + mergeOption = 1; if (*lastRowKey != TSKEY_INITIAL_VAL) { ++(*curRow); } ++nResult; + } else if (update){ + mergeOption = 2; + } else { + mergeOption = 0; + break; } + *lastRowKey = rowKey; } } else { @@ -1730,11 +1736,16 @@ static int32_t mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capa tdSKvRowGetVal(row, PRIMARYKEY_TIMESTAMP_COL_ID, -1, -1, &sVal); rowKey = *(TSKEY*)sVal.val; if (rowKey != *lastRowKey) { - isMerge = false; + mergeOption = 1; if (*lastRowKey != TSKEY_INITIAL_VAL) { ++(*curRow); } ++nResult; + } else if(update) { + mergeOption = 2; + } else { + mergeOption = 0; + break; } *lastRowKey = rowKey; } else { @@ -1754,7 +1765,7 @@ static int32_t mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capa colDataAppend(pColInfo, *curRow, NULL, true); } else if (tdValTypeIsNone(sVal.valType)) { // TODO: Set null if nothing append for this row - if (!isMerge) { + if (mergeOption == 1) { colDataAppend(pColInfo, *curRow, NULL, true); } } else { @@ -1769,14 +1780,14 @@ static int32_t mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capa ++k; } } else { - if (!isMerge) { + if (mergeOption == 1) { colDataAppend(pColInfo, *curRow, NULL, true); } ++i; } } - if (*lastRowKey != rowKey) { + if (mergeOption == 1) { while (i < numOfCols) { // the remain columns are all null data SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i); colDataAppend(pColInfo, *curRow, NULL, true); @@ -2008,7 +2019,7 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf } numOfRows += mergeTwoRowFromMem(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, &curRow, row1, row2, numOfCols, - pCheckInfo->tableId, pSchema1, pSchema2, true, &lastRowKey); + pCheckInfo->tableId, pSchema1, pSchema2, pCfg->update, &lastRowKey); // numOfRows += 1; if (cur->win.skey == TSKEY_INITIAL_VAL) { cur->win.skey = key; @@ -2065,7 +2076,7 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf } numOfRows += mergeTwoRowFromMem(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, &curRow, row1, row2, numOfCols, - pCheckInfo->tableId, pSchema1, pSchema2, true, &lastRowKey); + pCheckInfo->tableId, pSchema1, pSchema2, pCfg->update, &lastRowKey); // ++numOfRows; if (cur->win.skey == TSKEY_INITIAL_VAL) { cur->win.skey = key; @@ -2747,7 +2758,7 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int rv = TD_ROW_SVER(row); } numOfRows += mergeTwoRowFromMem(pTsdbReadHandle, maxRowsToRead, &curRows, row, NULL, numOfCols, pCheckInfo->tableId, pSchema, - NULL, true, &lastRowKey); + NULL, pCfg->update, &lastRowKey); if (numOfRows >= maxRowsToRead) { moveToNextRowInMem(pCheckInfo); diff --git a/source/dnode/vnode/src/tsdb/tsdbReadImpl.c b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c index 49d02adc3a..bebdfb3b63 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadImpl.c +++ b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c @@ -616,7 +616,7 @@ static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDat tcolId = pBlockCol->colId; toffset = tsdbGetBlockColOffset(pBlockCol); tlen = pBlockCol->len; - pDataCol->bitmap = pBlockCol->bitmap; + pDataCol->bitmap = pBlockCol->blen > 0 ? 1 : 0; } else { ASSERT(pDataCol->colId == tcolId); TD_SET_COL_ROWS_NORM(pDataCol); @@ -624,17 +624,8 @@ static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDat // int32_t tBitmaps = 0; int32_t tLenBitmap = 0; - if ((dcol != 0) && !TD_COL_ROWS_NORM(pBlockCol)) { + if ((dcol != 0) && (pBlockCol->blen > 0)) { tLenBitmap = nBitmaps; -#if 0 - if (IS_VAR_DATA_TYPE(pDataCol->type)) { - tBitmaps = nBitmaps; - tLenBitmap = tBitmaps; - } else { - tBitmaps = (int32_t)ceil((double)nBitmaps / TYPE_BYTES[pDataCol->type]); - tLenBitmap = tBitmaps * TYPE_BYTES[pDataCol->type]; - } -#endif } if (tcolId == pDataCol->colId) { @@ -784,8 +775,7 @@ static int tsdbLoadBlockDataColsImpl(SReadH *pReadh, SBlock *pBlock, SDataCols * if (colId == PRIMARYKEY_TIMESTAMP_COL_ID) { // load the key row blockCol.colId = colId; - TD_SET_COL_ROWS_NORM(&blockCol); // default is NORM for the primary key column - blockCol.blen = 0; + blockCol.blen = 0; // default is NORM for the primary key column blockCol.len = pBlock->keyLen; blockCol.type = pDataCol->type; blockCol.offset = TSDB_KEY_COL_OFFSET; @@ -815,7 +805,7 @@ static int tsdbLoadBlockDataColsImpl(SReadH *pReadh, SBlock *pBlock, SDataCols * ASSERT(pBlockCol->colId == pDataCol->colId); } // set the bitmap - pDataCol->bitmap = pBlockCol->bitmap; + pDataCol->bitmap = pBlockCol->blen > 0 ? 1 : 0; if (tsdbLoadColData(pReadh, pDFile, pBlock, pBlockCol, pDataCol) < 0) return -1; } @@ -833,17 +823,8 @@ static int tsdbLoadColData(SReadH *pReadh, SDFile *pDFile, SBlock *pBlock, SBloc // int32_t tBitmaps = 0; int32_t tLenBitmap = 0; - if (!TD_COL_ROWS_NORM(pBlockCol)) { + if (pBlockCol->blen) { tLenBitmap = nBitmaps; -#if 0 - if (IS_VAR_DATA_TYPE(pDataCol->type)) { - tBitmaps = nBitmaps; - tLenBitmap = tBitmaps; - } else { - tBitmaps = (int32_t)ceil((double)nBitmaps / TYPE_BYTES[pDataCol->type]); - tLenBitmap = tBitmaps * TYPE_BYTES[pDataCol->type]; - } -#endif } int tsize = pDataCol->bytes * pBlock->numOfRows + tLenBitmap + 2 * COMP_OVERFLOW_BYTES; From fdd901ae5db522a6d190778d654ea00f5650e9fd Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sat, 7 May 2022 03:28:15 +0000 Subject: [PATCH 68/83] fix auto-create table problem --- source/dnode/vnode/src/vnd/vnodeSvr.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 811c5c10e4..403c02b440 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -533,6 +533,13 @@ static int vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, in } } + msgIter.uid = createTbReq.uid; + if (createTbReq.type == TSDB_CHILD_TABLE) { + msgIter.suid = createTbReq.ctb.suid; + } else { + msgIter.suid = 0; + } + tCoderClear(&coder); } From 660b610f0411cb6e817aca4680c16578e791ebe6 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Sat, 7 May 2022 11:35:03 +0800 Subject: [PATCH 69/83] fix(query): fix histogram function output --- source/libs/function/src/builtins.c | 2 +- source/libs/function/src/builtinsimpl.c | 20 +++++++++++--------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index e0d9fb60f5..cb58a7a1d0 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -241,7 +241,7 @@ static int32_t translateHistogram(SFunctionNode* pFunc, char* pErrBuf, int32_t l return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); } - pFunc->node.resType = (SDataType) { .bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes, .type = TSDB_DATA_TYPE_DOUBLE }; + pFunc->node.resType = (SDataType) { .bytes = 512, .type = TSDB_DATA_TYPE_BINARY }; return TSDB_CODE_SUCCESS; } diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 69d3e079c6..909659701c 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -1826,7 +1826,6 @@ static int8_t getHistogramBinType(char *binTypeStr) { static bool getHistogramBinDesc(SHistoFuncInfo *pInfo, char *binDescStr, int8_t binType, bool normalized) { cJSON* binDesc = cJSON_Parse(binDescStr); - int32_t counter; int32_t numOfBins; double* intervals; if (cJSON_IsObject(binDesc)) { /* linaer/log bins */ @@ -1855,7 +1854,7 @@ static bool getHistogramBinDesc(SHistoFuncInfo *pInfo, char *binDescStr, int8_t return false; } - counter = (int32_t)count->valueint; + int32_t counter = (int32_t)count->valueint; if (infinity->valueint == false) { startIndex = 0; numOfBins = counter + 1; @@ -1915,7 +1914,7 @@ static bool getHistogramBinDesc(SHistoFuncInfo *pInfo, char *binDescStr, int8_t if (binType != USER_INPUT_BIN) { return false; } - counter = numOfBins = cJSON_GetArraySize(binDesc); + numOfBins = cJSON_GetArraySize(binDesc); intervals = taosMemoryCalloc(numOfBins, sizeof(double)); cJSON* bin = binDesc->child; if (bin == NULL) { @@ -1940,9 +1939,9 @@ static bool getHistogramBinDesc(SHistoFuncInfo *pInfo, char *binDescStr, int8_t return false; } - pInfo->numOfBins = numOfBins; + pInfo->numOfBins = numOfBins - 1; pInfo->normalized = normalized; - for (int32_t i = 0; i < numOfBins; ++i) { + for (int32_t i = 0; i < pInfo->numOfBins; ++i) { pInfo->bins[i].lower = intervals[i] < intervals[i + 1] ? intervals[i] : intervals[i + 1]; pInfo->bins[i].upper = intervals[i + 1] > intervals[i] ? intervals[i + 1] : intervals[i]; pInfo->bins[i].count = 0; @@ -1989,6 +1988,8 @@ int32_t histogramFunction(SqlFunctionCtx *pCtx) { continue; } + numOfElems++; + char* data = colDataGetData(pCol, i); double v; GET_TYPED_DATA(v, double, type, data); @@ -1996,25 +1997,26 @@ int32_t histogramFunction(SqlFunctionCtx *pCtx) { for (int32_t k = 0; k < pInfo->numOfBins; ++k) { if (v > pInfo->bins[k].lower && v <= pInfo->bins[k].upper) { pInfo->bins[k].count++; - numOfElems++; break; } } } + SET_VAL(GET_RES_INFO(pCtx), numOfElems, pInfo->numOfBins); return TSDB_CODE_SUCCESS; } int32_t histogramFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { + SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); SHistoFuncInfo* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); int32_t slotId = pCtx->pExpr->base.resSchema.slotId; SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); int32_t currentRow = pBlock->info.rows; - for (int32_t i = 0; i < pInfo->numOfBins; ++i) { + for (int32_t i = 0; i < pResInfo->numOfRes; ++i) { int32_t len; - char buf[400] = {0}; + char buf[512] = {0}; if (!pInfo->normalized) { len = sprintf(buf + VARSTR_HEADER_SIZE, "{\"lower_bin\":%g, \"upper_bin\":%g, \"count\":%"PRId64"}", pInfo->bins[i].lower, pInfo->bins[i].upper, pInfo->bins[i].count); @@ -2027,5 +2029,5 @@ int32_t histogramFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { currentRow++; } - return functionFinalize(pCtx, pBlock); + return pResInfo->numOfRes; } From 3b73dc295f8d34341ff6ff820dca9161600b2a0d Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Sat, 7 May 2022 11:53:08 +0800 Subject: [PATCH 70/83] set table uid --- source/dnode/vnode/src/tq/tqRead.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 5ec3ab0b47..f531d3f5fb 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -173,7 +173,7 @@ int32_t tqRetrieveDataBlock(SArray** ppCols, STqReadHandle* pHandle, uint64_t* p int32_t curRow = 0; tInitSubmitBlkIter(&pHandle->msgIter, pHandle->pBlock, &pHandle->blkIter); - *pUid = pHandle->pBlock->uid; // set the uid of table for submit block + *pUid = pHandle->msgIter.uid; // set the uid of table for submit block while ((row = tGetSubmitBlkNext(&pHandle->blkIter)) != NULL) { tdSTSRowIterReset(&iter, row); From 3ab513f0f022f32093940f022948e67857e3ffdb Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Sat, 7 May 2022 11:55:29 +0800 Subject: [PATCH 71/83] add test case for atan functions --- tests/system-test/2-query/arctan.py | 519 ++++++++++++++++++++++++++++ tests/system-test/fulltest.sh | 1 + 2 files changed, 520 insertions(+) create mode 100644 tests/system-test/2-query/arctan.py diff --git a/tests/system-test/2-query/arctan.py b/tests/system-test/2-query/arctan.py new file mode 100644 index 0000000000..1890c4605c --- /dev/null +++ b/tests/system-test/2-query/arctan.py @@ -0,0 +1,519 @@ +import taos +import sys +import datetime +import inspect +import math +from util.log import * +from util.sql import * +from util.cases import * + + +class TDTestCase: + updatecfgDict = {'debugFlag': 143 ,"cDebugFlag":143,"uDebugFlag":143 ,"rpcDebugFlag":143 , "tmrDebugFlag":143 , + "jniDebugFlag":143 ,"simDebugFlag":143,"dDebugFlag":143, "dDebugFlag":143,"vDebugFlag":143,"mDebugFlag":143,"qDebugFlag":143, + "wDebugFlag":143,"sDebugFlag":143,"tsdbDebugFlag":143,"tqDebugFlag":143 ,"fsDebugFlag":143 ,"fnDebugFlag":143} + def init(self, conn, powSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + + def prepare_datas(self): + tdSql.execute( + '''create table stb1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + tags (t1 int) + ''' + ) + + tdSql.execute( + ''' + create table t1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + ''' + ) + for i in range(4): + tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )') + + for i in range(9): + tdSql.execute( + f"insert into ct1 values ( now()-{i*10}s, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + ) + tdSql.execute( + f"insert into ct4 values ( now()-{i*90}d, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + ) + tdSql.execute("insert into ct1 values (now()-45s, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a )") + tdSql.execute("insert into ct1 values (now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute("insert into ct1 values (now()+15s, 9, -99999, -999, -99, -9.99, NULL, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute("insert into ct1 values (now()+20s, 9, -99999, -999, NULL, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + + tdSql.execute("insert into ct4 values (now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute("insert into ct4 values (now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute("insert into ct4 values (now()+90d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + + tdSql.execute( + f'''insert into t1 values + ( '2020-04-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2020-10-21 01:01:01.000', 1, 11111, 111, 11, 1.11, 11.11, 1, "binary1", "nchar1", now()+1a ) + ( '2020-12-31 01:01:01.000', 2, 22222, 222, 22, 2.22, 22.22, 0, "binary2", "nchar2", now()+2a ) + ( '2021-01-01 01:01:06.000', 3, 33333, 333, 33, 3.33, 33.33, 0, "binary3", "nchar3", now()+3a ) + ( '2021-05-07 01:01:10.000', 4, 44444, 444, 44, 4.44, 44.44, 1, "binary4", "nchar4", now()+4a ) + ( '2021-07-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2021-09-30 01:01:16.000', 5, 55555, 555, 55, 5.55, 55.55, 0, "binary5", "nchar5", now()+5a ) + ( '2022-02-01 01:01:20.000', 6, 66666, 666, 66, 6.66, 66.66, 1, "binary6", "nchar6", now()+6a ) + ( '2022-10-28 01:01:26.000', 7, 00000, 000, 00, 0.00, 00.00, 1, "binary7", "nchar7", "1970-01-01 08:00:00.000" ) + ( '2022-12-01 01:01:30.000', 8, -88888, -888, -88, -8.88, -88.88, 0, "binary8", "nchar8", "1969-01-01 01:00:00.000" ) + ( '2022-12-31 01:01:36.000', 9, -99999999999999999, -999, -99, -9.99, -999999999999999999999.99, 1, "binary9", "nchar9", "1900-01-01 00:00:00.000" ) + ( '2023-02-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ''' + ) + + def check_result_auto_atan(self ,origin_query , pow_query): + + pow_result = tdSql.getResult(pow_query) + origin_result = tdSql.getResult(origin_query) + + auto_result =[] + + for row in origin_result: + row_check = [] + for elem in row: + if elem == None: + elem = None + else: + elem = math.atan(elem) + row_check.append(elem) + auto_result.append(row_check) + + check_status = True + + for row_index , row in enumerate(pow_result): + for col_index , elem in enumerate(row): + if auto_result[row_index][col_index] == None and not (auto_result[row_index][col_index] == None and elem == None): + check_status = False + elif auto_result[row_index][col_index] != None and (auto_result[row_index][col_index] - elem > 0.00000001): + check_status = False + else: + pass + if not check_status: + tdLog.notice("atan function value has not as expected , sql is \"%s\" "%pow_query ) + sys.exit(1) + else: + tdLog.info("atan value check pass , it work as expected ,sql is \"%s\" "%pow_query ) + + def test_errors(self): + error_sql_lists = [ + "select atan from t1", + # "select atan(-+--+c1 ) from t1", + # "select +-atan(c1) from t1", + # "select ++-atan(c1) from t1", + # "select ++--atan(c1) from t1", + # "select - -atan(c1)*0 from t1", + # "select atan(tbname+1) from t1 ", + "select atan(123--123)==1 from t1", + "select atan(c1) as 'd1' from t1", + "select atan(c1 ,c2) from t1", + "select atan(c1 ,NULL ) from t1", + "select atan(,) from t1;", + "select atan(atan(c1) ab from t1)", + "select atan(c1 ) as int from t1", + "select atan from stb1", + # "select atan(-+--+c1) from stb1", + # "select +-atan(c1) from stb1", + # "select ++-atan(c1) from stb1", + # "select ++--atan(c1) from stb1", + # "select - -atan(c1)*0 from stb1", + # "select atan(tbname+1) from stb1 ", + "select atan(123--123)==1 from stb1", + "select atan(c1) as 'd1' from stb1", + "select atan(c1 ,c2 ) from stb1", + "select atan(c1 ,NULL) from stb1", + "select atan(,) from stb1;", + "select atan(atan(c1) ab from stb1)", + "select atan(c1) as int from stb1" + ] + for error_sql in error_sql_lists: + tdSql.error(error_sql) + + def support_types(self): + type_error_sql_lists = [ + "select atan(ts) from t1" , + "select atan(c7) from t1", + "select atan(c8) from t1", + "select atan(c9) from t1", + "select atan(ts) from ct1" , + "select atan(c7) from ct1", + "select atan(c8) from ct1", + "select atan(c9) from ct1", + "select atan(ts) from ct3" , + "select atan(c7) from ct3", + "select atan(c8) from ct3", + "select atan(c9) from ct3", + "select atan(ts) from ct4" , + "select atan(c7) from ct4", + "select atan(c8) from ct4", + "select atan(c9) from ct4", + "select atan(ts) from stb1" , + "select atan(c7) from stb1", + "select atan(c8) from stb1", + "select atan(c9) from stb1" , + + "select atan(ts) from stbbb1" , + "select atan(c7) from stbbb1", + + "select atan(ts) from tbname", + "select atan(c9) from tbname" + + ] + + for type_sql in type_error_sql_lists: + tdSql.error(type_sql) + + + type_sql_lists = [ + "select atan(c1) from t1", + "select atan(c2) from t1", + "select atan(c3) from t1", + "select atan(c4) from t1", + "select atan(c5) from t1", + "select atan(c6) from t1", + + "select atan(c1) from ct1", + "select atan(c2) from ct1", + "select atan(c3) from ct1", + "select atan(c4) from ct1", + "select atan(c5) from ct1", + "select atan(c6) from ct1", + + "select atan(c1) from ct3", + "select atan(c2) from ct3", + "select atan(c3) from ct3", + "select atan(c4) from ct3", + "select atan(c5) from ct3", + "select atan(c6) from ct3", + + "select atan(c1) from stb1", + "select atan(c2) from stb1", + "select atan(c3) from stb1", + "select atan(c4) from stb1", + "select atan(c5) from stb1", + "select atan(c6) from stb1", + + "select atan(c6) as alisb from stb1", + "select atan(c6) alisb from stb1", + ] + + for type_sql in type_sql_lists: + tdSql.query(type_sql) + + def basic_atan_function(self): + + # basic query + tdSql.query("select c1 from ct3") + tdSql.checkRows(0) + tdSql.query("select c1 from t1") + tdSql.checkRows(12) + tdSql.query("select c1 from stb1") + tdSql.checkRows(25) + + # used for empty table , ct3 is empty + tdSql.query("select atan(c1) from ct3") + tdSql.checkRows(0) + tdSql.query("select atan(c2) from ct3") + tdSql.checkRows(0) + tdSql.query("select atan(c3) from ct3") + tdSql.checkRows(0) + tdSql.query("select atan(c4) from ct3") + tdSql.checkRows(0) + tdSql.query("select atan(c5) from ct3") + tdSql.checkRows(0) + tdSql.query("select atan(c6) from ct3") + tdSql.checkRows(0) + + + # # used for regular table + tdSql.query("select atan(c1) from t1") + tdSql.checkData(0, 0, None) + tdSql.checkData(1 , 0, 0.785398163) + tdSql.checkData(3 , 0, 1.249045772) + tdSql.checkData(5 , 0, None) + + tdSql.query("select c1, c2, c3 , c4, c5 from t1") + tdSql.checkData(1, 4, 1.11000) + tdSql.checkData(3, 3, 33) + tdSql.checkData(5, 4, None) + + tdSql.query("select ts,c1, c2, c3 , c4, c5 from t1") + tdSql.checkData(1, 5, 1.11000) + tdSql.checkData(3, 4, 33) + tdSql.checkData(5, 5, None) + + self.check_result_auto_atan( "select abs(c1), abs(c2), abs(c3) , abs(c4), abs(c5) from t1", "select atan(abs(c1)), atan(abs(c2)) ,atan(abs(c3)), atan(abs(c4)), atan(abs(c5)) from t1") + + # used for sub table + tdSql.query("select c2 ,atan(c2) from ct1") + tdSql.checkData(0, 1, 1.570785077) + tdSql.checkData(1 , 1, 1.570783470) + tdSql.checkData(3 , 1, 1.570778327) + tdSql.checkData(4 , 1, 0.000000000) + + tdSql.query("select c1, c5 ,atan(c5) from ct4") + tdSql.checkData(0 , 2, None) + tdSql.checkData(1 , 2, 1.458656162) + tdSql.checkData(2 , 2, 1.442799803) + tdSql.checkData(3 , 2, 1.421759533) + tdSql.checkData(5 , 2, None) + + self.check_result_auto_atan( "select c1, c2, c3 , c4, c5 from ct1", "select atan(c1), atan(c2) ,atan(c3), atan(c4), atan(c5) from ct1") + + # nest query for atan functions + tdSql.query("select c4 , atan(c4) ,atan(atan(c4)) , atan(atan(atan(c4))) from ct1;") + tdSql.checkData(0 , 0 , 88) + tdSql.checkData(0 , 1 , 1.559433180) + tdSql.checkData(0 , 2 , 1.000590740) + tdSql.checkData(0 , 3 , 0.785693446) + + tdSql.checkData(1 , 0 , 77) + tdSql.checkData(1 , 1 , 1.557810044) + tdSql.checkData(1 , 2 , 1.000117426) + tdSql.checkData(1 , 3 , 0.785456873) + + tdSql.checkData(11 , 0 , -99) + tdSql.checkData(11 , 1 , -1.560695660) + tdSql.checkData(11 , 2 , -1.000958403) + tdSql.checkData(11 , 3 , -0.785877135) + + # used for stable table + + tdSql.query("select atan(c1) from stb1") + tdSql.checkRows(25) + + + # used for not exists table + tdSql.error("select atan(c1) from stbbb1") + tdSql.error("select atan(c1) from tbname") + tdSql.error("select atan(c1) from ct5") + + # mix with common col + tdSql.query("select c1, atan(c1) from ct1") + tdSql.query("select c2, atan(c2) from ct4") + + + # mix with common functions + tdSql.query("select c1, atan(c1),atan(c1), atan(atan(c1)) from ct4 ") + tdSql.checkData(0 , 0 ,None) + tdSql.checkData(0 , 1 ,None) + tdSql.checkData(0 , 2 ,None) + tdSql.checkData(0 , 3 ,None) + + tdSql.checkData(3 , 0 , 6) + tdSql.checkData(3 , 1 ,1.405647649) + tdSql.checkData(3 , 2 ,1.405647649) + tdSql.checkData(3 , 3 ,0.952449745) + + tdSql.query("select c1, atan(c1),c5, floor(c5) from stb1 ") + + # # mix with agg functions , not support + tdSql.error("select c1, atan(c1),c5, count(c5) from stb1 ") + tdSql.error("select c1, atan(c1),c5, count(c5) from ct1 ") + tdSql.error("select atan(c1), count(c5) from stb1 ") + tdSql.error("select atan(c1), count(c5) from ct1 ") + tdSql.error("select c1, count(c5) from ct1 ") + tdSql.error("select c1, count(c5) from stb1 ") + + # agg functions mix with agg functions + + tdSql.query("select max(c5), count(c5) from stb1") + tdSql.query("select max(c5), count(c5) from ct1") + + + # # bug fix for compute + tdSql.query("select c1, atan(c1) -0 ,atan(c1-4)-0 from ct4 ") + tdSql.checkData(0, 0, None) + tdSql.checkData(0, 1, None) + tdSql.checkData(0, 2, None) + tdSql.checkData(1, 0, 8) + tdSql.checkData(1, 1, 1.446441332) + tdSql.checkData(1, 2, 1.325817664) + + tdSql.query(" select c1, atan(c1) -0 ,atan(c1-0.1)-0.1 from ct4") + tdSql.checkData(0, 0, None) + tdSql.checkData(0, 1, None) + tdSql.checkData(0, 2, None) + tdSql.checkData(1, 0, 8) + tdSql.checkData(1, 1, 1.446441332) + tdSql.checkData(1, 2, 1.344883701) + + tdSql.query("select c1, atan(c1), c2, atan(c2), c3, atan(c3) from ct1") + + def test_big_number(self): + + tdSql.query("select c1, atan(100000000) from ct1") # bigint to double data overflow + tdSql.checkData(4, 1, math.atan(100000000)) + + + tdSql.query("select c1, atan(10000000000000) from ct1") # bigint to double data overflow + tdSql.checkData(4, 1, math.atan(10000000000000)) + + tdSql.query("select c1, atan(10000000000000000000000000) from ct1") # bigint to double data overflow + tdSql.query("select c1, atan(10000000000000000000000000.0) from ct1") # 10000000000000000000000000.0 is a double value + tdSql.checkData(1, 1, math.atan(10000000000000000000000000.0)) + + tdSql.query("select c1, atan(10000000000000000000000000000000000) from ct1") # bigint to double data overflow + tdSql.query("select c1, atan(10000000000000000000000000000000000.0) from ct1") # 10000000000000000000000000.0 is a double value + tdSql.checkData(4, 1, math.atan(10000000000000000000000000000000000.0)) + + tdSql.query("select c1, atan(10000000000000000000000000000000000000000) from ct1") # bigint to double data overflow + tdSql.query("select c1, atan(10000000000000000000000000000000000000000.0) from ct1") # 10000000000000000000000000.0 is a double value + + tdSql.checkData(4, 1, math.atan(10000000000000000000000000000000000000000.0)) + + tdSql.query("select c1, atan(10000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000) from ct1") # bigint to double data overflow + + def abs_func_filter(self): + tdSql.execute("use db") + tdSql.query("select c1, abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(atan(c1)-0.5) from ct4 where c1>5 ") + tdSql.checkRows(3) + tdSql.checkData(0,0,8) + tdSql.checkData(0,1,8.000000000) + tdSql.checkData(0,2,8.000000000) + tdSql.checkData(0,3,7.900000000) + tdSql.checkData(0,4,1.000000000) + + tdSql.query("select c1, abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(atan(c1)-0.5) from ct4 where c1=5 ") + tdSql.checkRows(1) + tdSql.checkData(0,0,5) + tdSql.checkData(0,1,5.000000000) + tdSql.checkData(0,2,5.000000000) + tdSql.checkData(0,3,4.900000000) + tdSql.checkData(0,4,1.000000000) + + tdSql.query("select c1,c2 , abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(atan(c1)-0.5) from ct4 where c1=atan(c1) limit 1 ") + tdSql.checkRows(1) + tdSql.checkData(0,0,0) + tdSql.checkData(0,1,0) + tdSql.checkData(0,2,0.000000000) + tdSql.checkData(0,3,0.000000000) + tdSql.checkData(0,4,-0.100000000) + tdSql.checkData(0,5,0.000000000) + + def pow_Arithmetic(self): + pass + + def check_boundary_values(self): + + PI=3.1415926 + + tdSql.execute("drop database if exists bound_test") + tdSql.execute("create database if not exists bound_test") + time.sleep(3) + tdSql.execute("use bound_test") + tdSql.execute( + "create table stb_bound (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(32),c9 nchar(32), c10 timestamp) tags (t1 int);" + ) + tdSql.execute(f'create table sub1_bound using stb_bound tags ( 1 )') + tdSql.execute( + f"insert into sub1_bound values ( now()-1s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + tdSql.execute( + f"insert into sub1_bound values ( now()-1s, -2147483647, -9223372036854775807, -32767, -127, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + tdSql.execute( + f"insert into sub1_bound values ( now(), 2147483646, 9223372036854775806, 32766, 126, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + tdSql.execute( + f"insert into sub1_bound values ( now(), -2147483646, -9223372036854775806, -32766, -126, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + tdSql.error( + f"insert into sub1_bound values ( now()+1s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + self.check_result_auto_atan( "select abs(c1), abs(c2), abs(c3) , abs(c4), abs(c5) from sub1_bound ", "select atan(abs(c1)), atan(abs(c2)) ,atan(abs(c3)), atan(abs(c4)), atan(abs(c5)) from sub1_bound") + + self.check_result_auto_atan( "select c1, c2, c3 , c3, c2 ,c1 from sub1_bound ", "select atan(c1), atan(c2) ,atan(c3), atan(c3), atan(c2) ,atan(c1) from sub1_bound") + + self.check_result_auto_atan("select abs(abs(abs(abs(abs(abs(abs(abs(abs(c1))))))))) nest_col_func from sub1_bound" , "select atan(abs(c1)) from sub1_bound" ) + + # check basic elem for table per row + tdSql.query("select atan(abs(c1)) ,atan(abs(c2)) , atan(abs(c3)) , atan(abs(c4)), atan(abs(c5)), atan(abs(c6)) from sub1_bound ") + tdSql.checkData(0,0,math.atan(2147483647)) + tdSql.checkData(0,1,math.atan(9223372036854775807)) + tdSql.checkData(0,2,math.atan(32767)) + tdSql.checkData(0,3,math.atan(127)) + tdSql.checkData(0,4,math.atan(339999995214436424907732413799364296704.00000)) + tdSql.checkData(1,0,math.atan(2147483647)) + tdSql.checkData(1,1,math.atan(9223372036854775807)) + tdSql.checkData(1,2,math.atan(32767)) + tdSql.checkData(1,3,math.atan(127)) + tdSql.checkData(1,4,math.atan(339999995214436424907732413799364296704.00000)) + tdSql.checkData(3,0,math.atan(2147483646)) + tdSql.checkData(3,1,math.atan(9223372036854775806)) + tdSql.checkData(3,2,math.atan(32766)) + tdSql.checkData(3,3,math.atan(126)) + tdSql.checkData(3,4,math.atan(339999995214436424907732413799364296704.00000)) + + # check + - * / in functions + tdSql.query("select atan(abs(c1+1)) ,atan(abs(c2)) , atan(abs(c3*1)) , atan(abs(c4/2)), atan(abs(c5))/2, atan(abs(c6)) from sub1_bound ") + tdSql.checkData(0,0,math.atan(2147483648.000000000)) + tdSql.checkData(0,1,math.atan(9223372036854775807)) + tdSql.checkData(0,2,math.atan(32767.000000000)) + tdSql.checkData(0,3,math.atan(63.500000000)) + + tdSql.execute("create stable st (ts timestamp, num1 float, num2 double) tags (t1 int);") + tdSql.execute(f'create table tb1 using st tags (1)') + tdSql.execute(f'create table tb2 using st tags (2)') + tdSql.execute(f'create table tb3 using st tags (3)') + tdSql.execute('insert into tb1 values (now()-40s, {}, {})'.format(PI/2 ,PI/2 )) + tdSql.execute('insert into tb1 values (now()-30s, {}, {})'.format(PI ,PI )) + tdSql.execute('insert into tb1 values (now()-20s, {}, {})'.format(PI*1.5 ,PI*1.5)) + tdSql.execute('insert into tb1 values (now()-10s, {}, {})'.format(PI*2 ,PI*2)) + tdSql.execute('insert into tb1 values (now(), {}, {})'.format(PI*2.5 ,PI*2.5)) + + tdSql.execute('insert into tb2 values (now()-40s, {}, {})'.format(PI/2 ,PI/2 )) + tdSql.execute('insert into tb2 values (now()-30s, {}, {})'.format(PI ,PI )) + tdSql.execute('insert into tb2 values (now()-20s, {}, {})'.format(PI*1.5 ,PI*1.5)) + tdSql.execute('insert into tb2 values (now()-10s, {}, {})'.format(PI*2 ,PI*2)) + tdSql.execute('insert into tb2 values (now(), {}, {})'.format(PI*2.5 ,PI*2.5)) + + for i in range(100): + tdSql.execute('insert into tb3 values (now()+{}s, {}, {})'.format(i,PI*(5+i)/2 ,PI*(5+i)/2)) + + self.check_result_auto_atan("select num1,num2 from tb3;" , "select atan(num1),atan(num2) from tb3") + + def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring + tdSql.prepare() + + tdLog.printNoPrefix("==========step1:create table ==============") + + self.prepare_datas() + + tdLog.printNoPrefix("==========step2:test errors ==============") + + self.test_errors() + + tdLog.printNoPrefix("==========step3:support types ============") + + self.support_types() + + tdLog.printNoPrefix("==========step4: atan basic query ============") + + self.basic_atan_function() + + tdLog.printNoPrefix("==========step5: big number atan query ============") + + self.test_big_number() + + + tdLog.printNoPrefix("==========step6: atan boundary query ============") + + self.check_boundary_values() + + tdLog.printNoPrefix("==========step7: atan filter query ============") + + self.abs_func_filter() + + + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 7553be344f..2df7d61e10 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -29,3 +29,4 @@ python3 ./test.py -f 2-query/cos.py python3 ./test.py -f 2-query/tan.py python3 ./test.py -f 2-query/arcsin.py python3 ./test.py -f 2-query/arccos.py +python3 ./test.py -f 2-query/arctan.py \ No newline at end of file From 935684e19026ca296b587b3c8718e16f57a63132 Mon Sep 17 00:00:00 2001 From: wenzhouwww Date: Sat, 7 May 2022 12:04:46 +0800 Subject: [PATCH 72/83] Update sqrt.py --- tests/system-test/2-query/sqrt.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/system-test/2-query/sqrt.py b/tests/system-test/2-query/sqrt.py index 2d72a5170b..28e869e044 100644 --- a/tests/system-test/2-query/sqrt.py +++ b/tests/system-test/2-query/sqrt.py @@ -497,8 +497,7 @@ class TDTestCase: tdSql.checkData(3,2,math.sqrt(32766)) tdSql.checkData(3,3,math.sqrt(126)) tdSql.checkData(3,4,math.sqrt(339999995214436424907732413799364296704.00000)) - - + # check + - * / in functions tdSql.query("select sqrt(abs(c1+1)) ,sqrt(abs(c2)) , sqrt(abs(c3*1)) , sqrt(abs(c4/2)), sqrt(abs(c5))/2, sqrt(abs(c6)) from sub1_bound ") tdSql.checkData(0,0,math.sqrt(2147483648.000000000)) From cdb7c37d16a0ac95a7a92d12816d8568081d755d Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 7 May 2022 12:07:45 +0800 Subject: [PATCH 73/83] refactor: adjust stb codes --- include/util/taoserror.h | 49 ++++++++++++----------- source/dnode/mnode/impl/inc/mndUser.h | 3 +- source/dnode/mnode/impl/src/mndStb.c | 44 ++++++++++++-------- source/dnode/mnode/impl/src/mndStream.c | 5 ++- source/util/src/terror.c | 3 ++ tests/script/general/stable/testSuite.sim | 5 --- 6 files changed, 63 insertions(+), 46 deletions(-) delete mode 100644 tests/script/general/stable/testSuite.sim diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 441e87eff7..53781c0f0a 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -327,29 +327,32 @@ int32_t* taosGetErrno(); #define TSDB_CODE_TDB_INVALID_TABLE_TYPE TAOS_DEF_ERROR_CODE(0, 0x0601) #define TSDB_CODE_TDB_IVD_TB_SCHEMA_VERSION TAOS_DEF_ERROR_CODE(0, 0x0602) #define TSDB_CODE_TDB_TABLE_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0603) -#define TSDB_CODE_TDB_INVALID_CONFIG TAOS_DEF_ERROR_CODE(0, 0x0604) -#define TSDB_CODE_TDB_INIT_FAILED TAOS_DEF_ERROR_CODE(0, 0x0605) -#define TSDB_CODE_TDB_NO_DISKSPACE TAOS_DEF_ERROR_CODE(0, 0x0606) -#define TSDB_CODE_TDB_NO_DISK_PERMISSIONS TAOS_DEF_ERROR_CODE(0, 0x0607) -#define TSDB_CODE_TDB_FILE_CORRUPTED TAOS_DEF_ERROR_CODE(0, 0x0608) -#define TSDB_CODE_TDB_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0609) -#define TSDB_CODE_TDB_TAG_VER_OUT_OF_DATE TAOS_DEF_ERROR_CODE(0, 0x060A) -#define TSDB_CODE_TDB_TIMESTAMP_OUT_OF_RANGE TAOS_DEF_ERROR_CODE(0, 0x060B) -#define TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP TAOS_DEF_ERROR_CODE(0, 0x060C) -#define TSDB_CODE_TDB_INVALID_ACTION TAOS_DEF_ERROR_CODE(0, 0x060D) -#define TSDB_CODE_TDB_INVALID_CREATE_TB_MSG TAOS_DEF_ERROR_CODE(0, 0x060E) -#define TSDB_CODE_TDB_NO_TABLE_DATA_IN_MEM TAOS_DEF_ERROR_CODE(0, 0x060F) -#define TSDB_CODE_TDB_FILE_ALREADY_EXISTS TAOS_DEF_ERROR_CODE(0, 0x0610) -#define TSDB_CODE_TDB_TABLE_RECONFIGURE TAOS_DEF_ERROR_CODE(0, 0x0611) -#define TSDB_CODE_TDB_IVD_CREATE_TABLE_INFO TAOS_DEF_ERROR_CODE(0, 0x0612) -#define TSDB_CODE_TDB_NO_AVAIL_DISK TAOS_DEF_ERROR_CODE(0, 0x0613) -#define TSDB_CODE_TDB_MESSED_MSG TAOS_DEF_ERROR_CODE(0, 0x0614) -#define TSDB_CODE_TDB_IVLD_TAG_VAL TAOS_DEF_ERROR_CODE(0, 0x0615) -#define TSDB_CODE_TDB_NO_CACHE_LAST_ROW TAOS_DEF_ERROR_CODE(0, 0x0616) -#define TSDB_CODE_TDB_TABLE_RECREATED TAOS_DEF_ERROR_CODE(0, 0x0617) -#define TSDB_CODE_TDB_TDB_ENV_OPEN_ERROR TAOS_DEF_ERROR_CODE(0, 0x0618) -#define TSDB_CODE_TDB_NO_SMA_INDEX_IN_META TAOS_DEF_ERROR_CODE(0, 0x0619) -#define TSDB_CODE_TDB_INVALID_SMA_STAT TAOS_DEF_ERROR_CODE(0, 0x0620) +#define TSDB_CODE_TDB_TABLE_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0604) +#define TSDB_CODE_TDB_STB_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0605) +#define TSDB_CODE_TDB_STB_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0606) +#define TSDB_CODE_TDB_INVALID_CONFIG TAOS_DEF_ERROR_CODE(0, 0x0607) +#define TSDB_CODE_TDB_INIT_FAILED TAOS_DEF_ERROR_CODE(0, 0x0608) +#define TSDB_CODE_TDB_NO_DISKSPACE TAOS_DEF_ERROR_CODE(0, 0x0609) +#define TSDB_CODE_TDB_NO_DISK_PERMISSIONS TAOS_DEF_ERROR_CODE(0, 0x060A) +#define TSDB_CODE_TDB_FILE_CORRUPTED TAOS_DEF_ERROR_CODE(0, 0x060B) +#define TSDB_CODE_TDB_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x060C) +#define TSDB_CODE_TDB_TAG_VER_OUT_OF_DATE TAOS_DEF_ERROR_CODE(0, 0x060D) +#define TSDB_CODE_TDB_TIMESTAMP_OUT_OF_RANGE TAOS_DEF_ERROR_CODE(0, 0x060E) +#define TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP TAOS_DEF_ERROR_CODE(0, 0x060F) +#define TSDB_CODE_TDB_INVALID_ACTION TAOS_DEF_ERROR_CODE(0, 0x0600) +#define TSDB_CODE_TDB_INVALID_CREATE_TB_MSG TAOS_DEF_ERROR_CODE(0, 0x0601) +#define TSDB_CODE_TDB_NO_TABLE_DATA_IN_MEM TAOS_DEF_ERROR_CODE(0, 0x0602) +#define TSDB_CODE_TDB_FILE_ALREADY_EXISTS TAOS_DEF_ERROR_CODE(0, 0x0613) +#define TSDB_CODE_TDB_TABLE_RECONFIGURE TAOS_DEF_ERROR_CODE(0, 0x0614) +#define TSDB_CODE_TDB_IVD_CREATE_TABLE_INFO TAOS_DEF_ERROR_CODE(0, 0x0615) +#define TSDB_CODE_TDB_NO_AVAIL_DISK TAOS_DEF_ERROR_CODE(0, 0x0616) +#define TSDB_CODE_TDB_MESSED_MSG TAOS_DEF_ERROR_CODE(0, 0x0617) +#define TSDB_CODE_TDB_IVLD_TAG_VAL TAOS_DEF_ERROR_CODE(0, 0x0618) +#define TSDB_CODE_TDB_NO_CACHE_LAST_ROW TAOS_DEF_ERROR_CODE(0, 0x0619) +#define TSDB_CODE_TDB_TABLE_RECREATED TAOS_DEF_ERROR_CODE(0, 0x061A) +#define TSDB_CODE_TDB_TDB_ENV_OPEN_ERROR TAOS_DEF_ERROR_CODE(0, 0x061B) +#define TSDB_CODE_TDB_NO_SMA_INDEX_IN_META TAOS_DEF_ERROR_CODE(0, 0x061C) +#define TSDB_CODE_TDB_INVALID_SMA_STAT TAOS_DEF_ERROR_CODE(0, 0x062D) // query #define TSDB_CODE_QRY_INVALID_QHANDLE TAOS_DEF_ERROR_CODE(0, 0x0700) diff --git a/source/dnode/mnode/impl/inc/mndUser.h b/source/dnode/mnode/impl/inc/mndUser.h index 2140d0fa67..016ec3e6e9 100644 --- a/source/dnode/mnode/impl/inc/mndUser.h +++ b/source/dnode/mnode/impl/inc/mndUser.h @@ -29,7 +29,8 @@ void mndReleaseUser(SMnode *pMnode, SUserObj *pUser); // for trans test SSdbRaw *mndUserActionEncode(SUserObj *pUser); -int32_t mndValidateUserAuthInfo(SMnode *pMnode, SUserAuthVersion *pUsers, int32_t numOfUses, void **ppRsp, int32_t *pRspLen); +int32_t mndValidateUserAuthInfo(SMnode *pMnode, SUserAuthVersion *pUsers, int32_t numOfUses, void **ppRsp, + int32_t *pRspLen); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 0fe78f4dcc..726631adea 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -28,8 +28,8 @@ #include "mndVgroup.h" #include "tname.h" -#define TSDB_STB_VER_NUMBER 1 -#define TSDB_STB_RESERVE_SIZE 64 +#define STB_VER_NUMBER 1 +#define STB_RESERVE_SIZE 64 static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw); static int32_t mndStbActionInsert(SSdb *pSdb, SStbObj *pStb); @@ -46,13 +46,15 @@ static int32_t mndRetrieveStb(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBl static void mndCancelGetNextStb(SMnode *pMnode, void *pIter); int32_t mndInitStb(SMnode *pMnode) { - SSdbTable table = {.sdbType = SDB_STB, - .keyType = SDB_KEY_BINARY, - .encodeFp = (SdbEncodeFp)mndStbActionEncode, - .decodeFp = (SdbDecodeFp)mndStbActionDecode, - .insertFp = (SdbInsertFp)mndStbActionInsert, - .updateFp = (SdbUpdateFp)mndStbActionUpdate, - .deleteFp = (SdbDeleteFp)mndStbActionDelete}; + SSdbTable table = { + .sdbType = SDB_STB, + .keyType = SDB_KEY_BINARY, + .encodeFp = (SdbEncodeFp)mndStbActionEncode, + .decodeFp = (SdbDecodeFp)mndStbActionDecode, + .insertFp = (SdbInsertFp)mndStbActionInsert, + .updateFp = (SdbUpdateFp)mndStbActionUpdate, + .deleteFp = (SdbDeleteFp)mndStbActionDelete, + }; mndSetMsgHandle(pMnode, TDMT_MND_CREATE_STB, mndProcessMCreateStbReq); mndSetMsgHandle(pMnode, TDMT_MND_ALTER_STB, mndProcessMAlterStbReq); @@ -74,8 +76,8 @@ SSdbRaw *mndStbActionEncode(SStbObj *pStb) { terrno = TSDB_CODE_OUT_OF_MEMORY; int32_t size = sizeof(SStbObj) + (pStb->numOfColumns + pStb->numOfTags) * sizeof(SSchema) + +pStb->commentLen + - pStb->ast1Len + pStb->ast2Len + TSDB_STB_RESERVE_SIZE; - SSdbRaw *pRaw = sdbAllocRaw(SDB_STB, TSDB_STB_VER_NUMBER, size); + pStb->ast1Len + pStb->ast2Len + STB_RESERVE_SIZE; + SSdbRaw *pRaw = sdbAllocRaw(SDB_STB, STB_VER_NUMBER, size); if (pRaw == NULL) goto _OVER; int32_t dataPos = 0; @@ -99,6 +101,7 @@ SSdbRaw *mndStbActionEncode(SStbObj *pStb) { for (int32_t i = 0; i < pStb->numOfColumns; ++i) { SSchema *pSchema = &pStb->pColumns[i]; SDB_SET_INT8(pRaw, dataPos, pSchema->type, _OVER) + SDB_SET_INT8(pRaw, dataPos, pSchema->flags, _OVER) SDB_SET_INT16(pRaw, dataPos, pSchema->colId, _OVER) SDB_SET_INT32(pRaw, dataPos, pSchema->bytes, _OVER) SDB_SET_BINARY(pRaw, dataPos, pSchema->name, TSDB_COL_NAME_LEN, _OVER) @@ -107,6 +110,7 @@ SSdbRaw *mndStbActionEncode(SStbObj *pStb) { for (int32_t i = 0; i < pStb->numOfTags; ++i) { SSchema *pSchema = &pStb->pTags[i]; SDB_SET_INT8(pRaw, dataPos, pSchema->type, _OVER) + SDB_SET_INT8(pRaw, dataPos, pSchema->flags, _OVER) SDB_SET_INT16(pRaw, dataPos, pSchema->colId, _OVER) SDB_SET_INT32(pRaw, dataPos, pSchema->bytes, _OVER) SDB_SET_BINARY(pRaw, dataPos, pSchema->name, TSDB_COL_NAME_LEN, _OVER) @@ -121,7 +125,7 @@ SSdbRaw *mndStbActionEncode(SStbObj *pStb) { if (pStb->ast2Len > 0) { SDB_SET_BINARY(pRaw, dataPos, pStb->pAst2, pStb->ast2Len, _OVER) } - SDB_SET_RESERVE(pRaw, dataPos, TSDB_STB_RESERVE_SIZE, _OVER) + SDB_SET_RESERVE(pRaw, dataPos, STB_RESERVE_SIZE, _OVER) SDB_SET_DATALEN(pRaw, dataPos, _OVER) terrno = 0; @@ -143,7 +147,7 @@ static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw) { int8_t sver = 0; if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER; - if (sver != TSDB_STB_VER_NUMBER) { + if (sver != STB_VER_NUMBER) { terrno = TSDB_CODE_SDB_INVALID_DATA_VER; goto _OVER; } @@ -183,6 +187,7 @@ static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw) { for (int32_t i = 0; i < pStb->numOfColumns; ++i) { SSchema *pSchema = &pStb->pColumns[i]; SDB_GET_INT8(pRaw, dataPos, &pSchema->type, _OVER) + SDB_GET_INT8(pRaw, dataPos, &pSchema->flags, _OVER) SDB_GET_INT16(pRaw, dataPos, &pSchema->colId, _OVER) SDB_GET_INT32(pRaw, dataPos, &pSchema->bytes, _OVER) SDB_GET_BINARY(pRaw, dataPos, pSchema->name, TSDB_COL_NAME_LEN, _OVER) @@ -191,6 +196,7 @@ static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw) { for (int32_t i = 0; i < pStb->numOfTags; ++i) { SSchema *pSchema = &pStb->pTags[i]; SDB_GET_INT8(pRaw, dataPos, &pSchema->type, _OVER) + SDB_GET_INT8(pRaw, dataPos, &pSchema->flags, _OVER) SDB_GET_INT16(pRaw, dataPos, &pSchema->colId, _OVER) SDB_GET_INT32(pRaw, dataPos, &pSchema->bytes, _OVER) SDB_GET_BINARY(pRaw, dataPos, pSchema->name, TSDB_COL_NAME_LEN, _OVER) @@ -211,7 +217,7 @@ static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw) { if (pStb->pAst2 == NULL) goto _OVER; SDB_GET_BINARY(pRaw, dataPos, pStb->pAst2, pStb->ast2Len, _OVER) } - SDB_GET_RESERVE(pRaw, dataPos, TSDB_STB_RESERVE_SIZE, _OVER) + SDB_GET_RESERVE(pRaw, dataPos, STB_RESERVE_SIZE, _OVER) terrno = 0; @@ -488,7 +494,7 @@ int32_t mndCheckCreateStbReq(SMCreateStbReq *pCreate) { for (int32_t i = 0; i < pCreate->numOfColumns; ++i) { SField *pField1 = taosArrayGet(pCreate->pColumns, i); - if (pField->type < 0) { + if (pField1->type < 0) { terrno = TSDB_CODE_MND_INVALID_STB_OPTION; return -1; } @@ -574,6 +580,7 @@ static int32_t mndSetCreateStbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj action.pCont = pReq; action.contLen = contLen; action.msgType = TDMT_VND_CREATE_STB; + action.acceptableCode = TSDB_CODE_TDB_STB_ALREADY_EXIST; if (mndTransAppendRedoAction(pTrans, &action) != 0) { taosMemoryFree(pReq); sdbCancelFetch(pSdb, pIter); @@ -613,6 +620,7 @@ static int32_t mndSetCreateStbUndoActions(SMnode *pMnode, STrans *pTrans, SDbObj action.pCont = pReq; action.contLen = contLen; action.msgType = TDMT_VND_DROP_STB; + action.acceptableCode = TSDB_CODE_TDB_STB_NOT_EXIST; if (mndTransAppendUndoAction(pTrans, &action) != 0) { taosMemoryFree(pReq); sdbCancelFetch(pSdb, pIter); @@ -733,6 +741,7 @@ _OVER: mndTransDrop(pTrans); return code; } + int32_t mndAddStbToTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) { mndTransSetDbInfo(pTrans, pDb); if (mndSetCreateStbRedoLogs(pMnode, pTrans, pDb, pStb) != 0) return -1; @@ -792,7 +801,10 @@ static int32_t mndProcessMCreateStbReq(SNodeMsg *pReq) { } int32_t numOfStbs = -1; - mndGetNumOfStbs(pMnode, pDb->name, &numOfStbs); + if (mndGetNumOfStbs(pMnode, pDb->name, &numOfStbs) != 0) { + goto _OVER; + } + if (pDb->cfg.numOfStables == 1 && numOfStbs != 0) { terrno = TSDB_CODE_MND_SINGLE_STB_MODE_DB; goto _OVER; diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 1404b1cd94..2260b178c8 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -330,7 +330,10 @@ static SStbObj *mndCreateStbForStream(SMnode *pMnode, STrans *pTrans, const SStr } int32_t numOfStbs = -1; - mndGetNumOfStbs(pMnode, pDb->name, &numOfStbs); + if (mndGetNumOfStbs(pMnode, pDb->name, &numOfStbs) != 0) { + goto _OVER; + } + if (pDb->cfg.numOfStables == 1 && numOfStbs != 0) { terrno = TSDB_CODE_MND_SINGLE_STB_MODE_DB; goto _OVER; diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 1470496c68..8ebcaede8c 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -326,6 +326,9 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_TABLE_ID, "Invalid table ID") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_TABLE_TYPE, "Invalid table type") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_IVD_TB_SCHEMA_VERSION, "Invalid table schema version") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TABLE_ALREADY_EXIST, "Table already exists") +TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TABLE_NOT_EXIST, "Table not exists") +TAOS_DEFINE_ERROR(TSDB_CODE_TDB_STB_ALREADY_EXIST, "Stable already exists") +TAOS_DEFINE_ERROR(TSDB_CODE_TDB_STB_NOT_EXIST, "Stable not exists") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_CONFIG, "Invalid configuration") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INIT_FAILED, "Tsdb init failed") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_NO_DISKSPACE, "No diskspace for tsdb") diff --git a/tests/script/general/stable/testSuite.sim b/tests/script/general/stable/testSuite.sim deleted file mode 100644 index e786ac9ca4..0000000000 --- a/tests/script/general/stable/testSuite.sim +++ /dev/null @@ -1,5 +0,0 @@ -run general/stable/disk.sim -run general/stable/dnode3.sim -run general/stable/metrics.sim -run general/stable/values.sim -run general/stable/vnode3.sim From 073745c8937775107cf1260f31a11f77d4f13f2b Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 7 May 2022 13:06:23 +0800 Subject: [PATCH 74/83] refactor: alter stb --- include/common/tmsg.h | 8 ++++---- source/common/src/tmsg.c | 6 +++--- source/dnode/mnode/impl/src/mndStb.c | 16 ++++++++-------- source/dnode/mnode/impl/test/stb/stb.cpp | 14 +++++++------- source/libs/parser/src/parTranslater.c | 6 +++--- 5 files changed, 25 insertions(+), 25 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 3889784f2f..f22e8fbb97 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -326,11 +326,11 @@ typedef struct { int8_t alterType; int32_t numOfFields; SArray* pFields; -} SMAltertbReq; +} SMAlterStbReq; -int32_t tSerializeSMAlterStbReq(void* buf, int32_t bufLen, SMAltertbReq* pReq); -int32_t tDeserializeSMAlterStbReq(void* buf, int32_t bufLen, SMAltertbReq* pReq); -void tFreeSMAltertbReq(SMAltertbReq* pReq); +int32_t tSerializeSMAlterStbReq(void* buf, int32_t bufLen, SMAlterStbReq* pReq); +int32_t tDeserializeSMAlterStbReq(void* buf, int32_t bufLen, SMAlterStbReq* pReq); +void tFreeSMAltertbReq(SMAlterStbReq* pReq); typedef struct SEpSet { int8_t inUse; diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 6278b52a04..24e6fc9e63 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -594,7 +594,7 @@ int32_t tDeserializeSMDropStbReq(void *buf, int32_t bufLen, SMDropStbReq *pReq) return 0; } -int32_t tSerializeSMAlterStbReq(void *buf, int32_t bufLen, SMAltertbReq *pReq) { +int32_t tSerializeSMAlterStbReq(void *buf, int32_t bufLen, SMAlterStbReq *pReq) { SCoder encoder = {0}; tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); @@ -615,7 +615,7 @@ int32_t tSerializeSMAlterStbReq(void *buf, int32_t bufLen, SMAltertbReq *pReq) { return tlen; } -int32_t tDeserializeSMAlterStbReq(void *buf, int32_t bufLen, SMAltertbReq *pReq) { +int32_t tDeserializeSMAlterStbReq(void *buf, int32_t bufLen, SMAlterStbReq *pReq) { SCoder decoder = {0}; tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); @@ -645,7 +645,7 @@ int32_t tDeserializeSMAlterStbReq(void *buf, int32_t bufLen, SMAltertbReq *pReq) return 0; } -void tFreeSMAltertbReq(SMAltertbReq *pReq) { +void tFreeSMAltertbReq(SMAlterStbReq *pReq) { taosArrayDestroy(pReq->pFields); pReq->pFields = NULL; } diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 726631adea..b6912d0777 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -831,7 +831,7 @@ static int32_t mndProcessVCreateStbRsp(SNodeMsg *pRsp) { return 0; } -static int32_t mndCheckAlterStbReq(SMAltertbReq *pAlter) { +static int32_t mndCheckAlterStbReq(SMAlterStbReq *pAlter) { if (pAlter->numOfFields < 1 || pAlter->numOfFields != (int32_t)taosArrayGetSize(pAlter->pFields)) { terrno = TSDB_CODE_MND_INVALID_STB_OPTION; return -1; @@ -1182,7 +1182,7 @@ static int32_t mndSetAlterStbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj return 0; } -static int32_t mndAlterStb(SMnode *pMnode, SNodeMsg *pReq, const SMAltertbReq *pAlter, SDbObj *pDb, SStbObj *pOld) { +static int32_t mndAlterStb(SMnode *pMnode, SNodeMsg *pReq, const SMAlterStbReq *pAlter, SDbObj *pDb, SStbObj *pOld) { SStbObj stbObj = {0}; taosRLockLatch(&pOld->lock); memcpy(&stbObj, pOld, sizeof(SStbObj)); @@ -1246,12 +1246,12 @@ _OVER: } static int32_t mndProcessMAlterStbReq(SNodeMsg *pReq) { - SMnode *pMnode = pReq->pNode; - int32_t code = -1; - SDbObj *pDb = NULL; - SStbObj *pStb = NULL; - SUserObj *pUser = NULL; - SMAltertbReq alterReq = {0}; + SMnode *pMnode = pReq->pNode; + int32_t code = -1; + SDbObj *pDb = NULL; + SStbObj *pStb = NULL; + SUserObj *pUser = NULL; + SMAlterStbReq alterReq = {0}; if (tDeserializeSMAlterStbReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &alterReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; diff --git a/source/dnode/mnode/impl/test/stb/stb.cpp b/source/dnode/mnode/impl/test/stb/stb.cpp index 220a73def6..16974ad541 100644 --- a/source/dnode/mnode/impl/test/stb/stb.cpp +++ b/source/dnode/mnode/impl/test/stb/stb.cpp @@ -137,7 +137,7 @@ void* MndTestStb::BuildCreateStbReq(const char* stbname, int32_t* pContLen) { } void* MndTestStb::BuildAlterStbAddTagReq(const char* stbname, const char* tagname, int32_t* pContLen) { - SMAltertbReq req = {0}; + SMAlterStbReq req = {0}; strcpy(req.name, stbname); req.numOfFields = 1; req.pFields = taosArrayInit(1, sizeof(SField)); @@ -158,7 +158,7 @@ void* MndTestStb::BuildAlterStbAddTagReq(const char* stbname, const char* tagnam } void* MndTestStb::BuildAlterStbDropTagReq(const char* stbname, const char* tagname, int32_t* pContLen) { - SMAltertbReq req = {0}; + SMAlterStbReq req = {0}; strcpy(req.name, stbname); req.numOfFields = 1; req.pFields = taosArrayInit(1, sizeof(SField)); @@ -180,7 +180,7 @@ void* MndTestStb::BuildAlterStbDropTagReq(const char* stbname, const char* tagna void* MndTestStb::BuildAlterStbUpdateTagNameReq(const char* stbname, const char* tagname, const char* newtagname, int32_t* pContLen) { - SMAltertbReq req = {0}; + SMAlterStbReq req = {0}; strcpy(req.name, stbname); req.numOfFields = 2; req.pFields = taosArrayInit(2, sizeof(SField)); @@ -208,7 +208,7 @@ void* MndTestStb::BuildAlterStbUpdateTagNameReq(const char* stbname, const char* void* MndTestStb::BuildAlterStbUpdateTagBytesReq(const char* stbname, const char* tagname, int32_t bytes, int32_t* pContLen) { - SMAltertbReq req = {0}; + SMAlterStbReq req = {0}; strcpy(req.name, stbname); req.numOfFields = 1; req.pFields = taosArrayInit(1, sizeof(SField)); @@ -229,7 +229,7 @@ void* MndTestStb::BuildAlterStbUpdateTagBytesReq(const char* stbname, const char } void* MndTestStb::BuildAlterStbAddColumnReq(const char* stbname, const char* colname, int32_t* pContLen) { - SMAltertbReq req = {0}; + SMAlterStbReq req = {0}; strcpy(req.name, stbname); req.numOfFields = 1; req.pFields = taosArrayInit(1, sizeof(SField)); @@ -250,7 +250,7 @@ void* MndTestStb::BuildAlterStbAddColumnReq(const char* stbname, const char* col } void* MndTestStb::BuildAlterStbDropColumnReq(const char* stbname, const char* colname, int32_t* pContLen) { - SMAltertbReq req = {0}; + SMAlterStbReq req = {0}; strcpy(req.name, stbname); req.numOfFields = 1; req.pFields = taosArrayInit(1, sizeof(SField)); @@ -272,7 +272,7 @@ void* MndTestStb::BuildAlterStbDropColumnReq(const char* stbname, const char* co void* MndTestStb::BuildAlterStbUpdateColumnBytesReq(const char* stbname, const char* colname, int32_t bytes, int32_t* pContLen) { - SMAltertbReq req = {0}; + SMAlterStbReq req = {0}; strcpy(req.name, stbname); req.numOfFields = 1; req.pFields = taosArrayInit(1, sizeof(SField)); diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 6874b5b7d4..bd614d0165 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -2596,7 +2596,7 @@ static int32_t translateDropSuperTable(STranslateContext* pCxt, SDropSuperTableS pStmt->ignoreNotExists); } -static int32_t setAlterTableField(SAlterTableStmt* pStmt, SMAltertbReq* pAlterReq) { +static int32_t setAlterTableField(SAlterTableStmt* pStmt, SMAlterStbReq* pAlterReq) { pAlterReq->pFields = taosArrayInit(2, sizeof(TAOS_FIELD)); if (NULL == pAlterReq->pFields) { return TSDB_CODE_OUT_OF_MEMORY; @@ -2632,8 +2632,8 @@ static int32_t setAlterTableField(SAlterTableStmt* pStmt, SMAltertbReq* pAlterRe } static int32_t translateAlterTable(STranslateContext* pCxt, SAlterTableStmt* pStmt) { - SMAltertbReq alterReq = {0}; - SName tableName; + SMAlterStbReq alterReq = {0}; + SName tableName; tNameExtractFullName(toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &tableName), alterReq.name); alterReq.alterType = pStmt->alterType; alterReq.numOfFields = 1; From 7375cf4921ec0438be0af588e7a7774e4cbcb369 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Sat, 7 May 2022 13:20:48 +0800 Subject: [PATCH 75/83] fix mem issue --- source/libs/nodes/src/nodesUtilFuncs.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 0bceb84f4a..319b680161 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -1337,7 +1337,9 @@ void valueNodeToVariant(const SValueNode* pNode, SVariant* pVal) { case TSDB_DATA_TYPE_NCHAR: case TSDB_DATA_TYPE_VARCHAR: case TSDB_DATA_TYPE_VARBINARY: - pVal->pz = pNode->datum.p; + pVal->pz = taosMemoryMalloc(pVal->nLen + VARSTR_HEADER_SIZE + 1); + memcpy(pVal->pz, pNode->datum.p, pVal->nLen + VARSTR_HEADER_SIZE); + pVal->pz[pVal->nLen + VARSTR_HEADER_SIZE] = 0; break; case TSDB_DATA_TYPE_JSON: case TSDB_DATA_TYPE_DECIMAL: From 80cfde72ad7199906267a107fa0066cd681ee611 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sat, 7 May 2022 05:34:30 +0000 Subject: [PATCH 76/83] fix: drop table if exists --- source/dnode/vnode/src/meta/metaTable.c | 2 +- source/libs/tdb/inc/tdb.h | 1 + source/libs/tdb/src/db/tdbBtree.c | 8 ++++++++ source/libs/tdb/src/db/tdbDb.c | 2 ++ source/libs/tdb/src/inc/tdbInt.h | 1 + 5 files changed, 13 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index ffe176f85e..a68e176fa9 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -201,7 +201,7 @@ int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq) { // search & delete the name idx tdbDbcOpen(pMeta->pNameIdx, &pNameIdxc, &pMeta->txn); ret = tdbDbcMoveTo(pNameIdxc, pReq->name, strlen(pReq->name) + 1, &c); - if (ret < 0 || c) { + if (ret < 0 || !tdbDbcIsValid(pNameIdxc) || c) { tdbDbcClose(pNameIdxc); terrno = TSDB_CODE_VND_TABLE_NOT_EXIST; return -1; diff --git a/source/libs/tdb/inc/tdb.h b/source/libs/tdb/inc/tdb.h index 18f8ddec5c..90b07fb6ae 100644 --- a/source/libs/tdb/inc/tdb.h +++ b/source/libs/tdb/inc/tdb.h @@ -49,6 +49,7 @@ int tdbDbPGet(TDB *pDb, const void *pKey, int kLen, void **ppKey, int *pkLen, vo // TDBC int tdbDbcOpen(TDB *pDb, TDBC **ppDbc, TXN *pTxn); int tdbDbcClose(TDBC *pDbc); +int tdbDbcIsValid(TDBC *pDbc); int tdbDbcMoveTo(TDBC *pDbc, const void *pKey, int kLen, int *c); int tdbDbcMoveToFirst(TDBC *pDbc); int tdbDbcMoveToLast(TDBC *pDbc); diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 0d13ec4d2b..43822cd311 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -1678,6 +1678,14 @@ int tdbBtcClose(SBTC *pBtc) { return 0; } + +int tdbBtcIsValid(SBTC *pBtc) { + if (pBtc->idx < 0) { + return 0; + } else { + return 1; + } +} // TDB_BTREE_CURSOR // TDB_BTREE_DEBUG ===================== diff --git a/source/libs/tdb/src/db/tdbDb.c b/source/libs/tdb/src/db/tdbDb.c index 553bb2c646..ceaac6dff1 100644 --- a/source/libs/tdb/src/db/tdbDb.c +++ b/source/libs/tdb/src/db/tdbDb.c @@ -141,3 +141,5 @@ int tdbDbcClose(TDBC *pDbc) { return 0; } + +int tdbDbcIsValid(TDBC *pDbc) { return tdbBtcIsValid(&pDbc->btc); } \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdbInt.h b/source/libs/tdb/src/inc/tdbInt.h index 76dacf7b84..ee431ac638 100644 --- a/source/libs/tdb/src/inc/tdbInt.h +++ b/source/libs/tdb/src/inc/tdbInt.h @@ -136,6 +136,7 @@ int tdbBtreePGet(SBTree *pBt, const void *pKey, int kLen, void **ppKey, int *pkL // SBTC int tdbBtcOpen(SBTC *pBtc, SBTree *pBt, TXN *pTxn); int tdbBtcClose(SBTC *pBtc); +int tdbBtcIsValid(SBTC *pBtc); int tdbBtcMoveTo(SBTC *pBtc, const void *pKey, int kLen, int *pCRst); int tdbBtcMoveToFirst(SBTC *pBtc); int tdbBtcMoveToLast(SBTC *pBtc); From 174bc6a5680e17fefe0800aade5ed607998ad416 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Sat, 7 May 2022 13:58:10 +0800 Subject: [PATCH 77/83] refactor:modify schemaless interface --- cmake/cmake.define | 2 +- include/client/taos.h | 2 +- source/client/src/clientSml.c | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cmake/cmake.define b/cmake/cmake.define index 2b4fe5c42f..aeab39cab4 100644 --- a/cmake/cmake.define +++ b/cmake/cmake.define @@ -46,7 +46,7 @@ ENDIF () IF (TD_WINDOWS) MESSAGE("${Yellow} set compiler flag for Windows! ${ColourReset}") - SET(COMMON_FLAGS "/W3 /D_WIN32 /vmg") + SET(COMMON_FLAGS "/W3 /D_WIN32") # IF (MSVC AND (MSVC_VERSION GREATER_EQUAL 1900)) # SET(COMMON_FLAGS "${COMMON_FLAGS} /Wv:18") diff --git a/include/client/taos.h b/include/client/taos.h index 9d8b5107bb..6e20900668 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -195,7 +195,7 @@ DLL_EXPORT void taos_unsubscribe(TAOS_SUB *tsub, int keepProgress); #endif DLL_EXPORT int taos_load_table_info(TAOS *taos, const char *tableNameList); -DLL_EXPORT TAOS_RES *taos_schemaless_insert(TAOS *taos, char *lines[], int numLines, int protocol, int precision, bool dataFormat); +DLL_EXPORT TAOS_RES *taos_schemaless_insert(TAOS *taos, char *lines[], int numLines, int protocol, int precision); /* --------------------------TMQ INTERFACE------------------------------- */ diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 85cd0661e3..5f9138bb43 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -1724,13 +1724,13 @@ cleanup: * */ -TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int protocol, int precision, bool dataFormat) { +TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int protocol, int precision) { SRequestObj* request = createRequest(taos, NULL, NULL, TSDB_SQL_INSERT); if(!request){ return NULL; } - SSmlHandle* info = smlBuildSmlInfo(taos, request, protocol, precision, dataFormat); + SSmlHandle* info = smlBuildSmlInfo(taos, request, protocol, precision, false); if(!info){ return (TAOS_RES*)request; } From 67e8dccfec0f0c302d6d190ac440e590a36d51c7 Mon Sep 17 00:00:00 2001 From: jiacy-jcy <714897623@qq.com> Date: Sat, 7 May 2022 14:15:32 +0800 Subject: [PATCH 78/83] test case --- tests/system-test/2-query/last.py | 36 +++++++++++++++---------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/tests/system-test/2-query/last.py b/tests/system-test/2-query/last.py index 531afd0117..b5c0498e40 100644 --- a/tests/system-test/2-query/last.py +++ b/tests/system-test/2-query/last.py @@ -21,12 +21,12 @@ class TDTestCase: tdSql.execute("insert into stb_1(ts) values(%d)" % (self.ts - 1)) # last verifacation - tdSql.query("select last(*) from stb_1") - tdSql.checkRows(1) - tdSql.checkData(0, 1, None) - tdSql.query("select last(*) from db.stb_1") - tdSql.checkRows(1) - tdSql.checkData(0, 1, None) + # tdSql.query("select last(*) from stb_1") + # tdSql.checkRows(1) + # tdSql.checkData(0, 1, None) + # tdSql.query("select last(*) from db.stb_1") + # tdSql.checkRows(1) + # tdSql.checkData(0, 1, None) tdSql.query("select last(col1) from stb_1") tdSql.checkRows(0) tdSql.query("select last(col1) from db.stb_1") @@ -86,12 +86,12 @@ class TDTestCase: tdSql.execute("insert into stb_1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1)) - tdSql.query("select last(*) from stb_1") - tdSql.checkRows(1) - tdSql.checkData(0, 1, 10) - tdSql.query("select last(*) from db.stb_1") - tdSql.checkRows(1) - tdSql.checkData(0, 1, 10) + # tdSql.query("select last(*) from stb_1") + # tdSql.checkRows(1) + # tdSql.checkData(0, 1, 10) + # tdSql.query("select last(*) from db.stb_1") + # tdSql.checkRows(1) + # tdSql.checkData(0, 1, 10) tdSql.query("select last(col1) from stb_1") tdSql.checkRows(1) tdSql.checkData(0, 0, 10) @@ -175,12 +175,12 @@ class TDTestCase: tdSql.execute('''create table ntb(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, col7 bool, col8 binary(20), col9 nchar(20), col11 tinyint unsigned, col12 smallint unsigned, col13 int unsigned, col14 bigint unsigned)''') tdSql.execute("insert into ntb(ts) values(%d)" % (self.ts - 1)) - tdSql.query("select last(*) from ntb") - tdSql.checkRows(1) - tdSql.checkData(0, 1, None) - tdSql.query("select last(*) from db.ntb") - tdSql.checkRows(1) - tdSql.checkData(0, 1, None) + # tdSql.query("select last(*) from ntb") + # tdSql.checkRows(1) + # tdSql.checkData(0, 1, None) + # tdSql.query("select last(*) from db.ntb") + # tdSql.checkRows(1) + # tdSql.checkData(0, 1, None) tdSql.query("select last(col1) from ntb") tdSql.checkRows(0) tdSql.query("select last(col1) from db.ntb") From dd7cec41324e5cfd0037302c30290f90d053804d Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Sat, 7 May 2022 14:23:07 +0800 Subject: [PATCH 79/83] feat(query): add histogram normalized processing --- source/libs/function/src/builtinsimpl.c | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 909659701c..7229cf98b0 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -1972,7 +1972,6 @@ bool histogramFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo *pResultIn } int32_t histogramFunction(SqlFunctionCtx *pCtx) { - int32_t numOfElems = 0; SHistoFuncInfo* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); SInputColumnInfoData* pInput = &pCtx->input; @@ -1983,6 +1982,8 @@ int32_t histogramFunction(SqlFunctionCtx *pCtx) { int32_t start = pInput->startRowIndex; int32_t numOfRows = pInput->numOfRows; + int32_t numOfElems = 0; + int32_t totalElems = 0; for (int32_t i = start; i < numOfRows + start; ++i) { if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) { continue; @@ -1997,9 +1998,21 @@ int32_t histogramFunction(SqlFunctionCtx *pCtx) { for (int32_t k = 0; k < pInfo->numOfBins; ++k) { if (v > pInfo->bins[k].lower && v <= pInfo->bins[k].upper) { pInfo->bins[k].count++; + totalElems++; break; } } + + } + + if (pInfo->normalized) { + for (int32_t k = 0; k < pInfo->numOfBins; ++k) { + if(totalElems != 0) { + pInfo->bins[k].percentage = pInfo->bins[k].count / (double)totalElems; + } else { + pInfo->bins[k].percentage = 0; + } + } } SET_VAL(GET_RES_INFO(pCtx), numOfElems, pInfo->numOfBins); From e2413a062c477784aaf40062f9db7330c303177d Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Sat, 7 May 2022 14:29:50 +0800 Subject: [PATCH 80/83] fix: histogram normalized can only be 0/1 --- source/libs/function/src/builtinsimpl.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 7229cf98b0..0808ec58b0 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -1964,6 +1964,9 @@ bool histogramFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo *pResultIn } char* binDesc = varDataVal(pCtx->param[2].param.pz); int64_t normalized = pCtx->param[3].param.i; + if (normalized != 0 && normalized != 1) { + return false; + } if (!getHistogramBinDesc(pInfo, binDesc, binType, (bool)normalized)) { return false; } From 5bf762d50d20c6a99f4cd6e6eebd4092cd1e9829 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Sat, 7 May 2022 15:15:31 +0800 Subject: [PATCH 81/83] [test: add test cases for telemetry] --- tests/system-test/0-others/telemetry.py | 197 ++++++++++++++++++++++++ 1 file changed, 197 insertions(+) create mode 100644 tests/system-test/0-others/telemetry.py diff --git a/tests/system-test/0-others/telemetry.py b/tests/system-test/0-others/telemetry.py new file mode 100644 index 0000000000..3ab39f9e7b --- /dev/null +++ b/tests/system-test/0-others/telemetry.py @@ -0,0 +1,197 @@ +import taos +import sys +import time +import socket +import pexpect +import os +import http.server +import gzip +import threading +import json + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * + +telemetryPort = '80' + +#{ +# "instanceId": "5cf4cd7a-acd4-43ba-8b0d-e84395b76a65", +# "reportVersion": 1, +# "os": "Ubuntu 20.04.3 LTS", +# "cpuModel": "Intel(R) Xeon(R) CPU E5-2620 v3 @ 2.40GHz", +# "numOfCpu": 6, +# "memory": "65860292 kB", +# "version": "3.0.0.0", +# "buildInfo": "Built at 2022-05-07 14:09:02", +# "gitInfo": "2139ccceb0946cde86b6b553b11e338f1ba437e5", +# "email": "user@taosdata.com", +# "numOfDnode": 1, +# "numOfMnode": 1, +# "numOfVgroup": 32, +# "numOfDatabase": 2, +# "numOfSuperTable": 0, +# "numOfChildTable": 100, +# "numOfColumn": 200, +# "numOfPoint": 300, +# "totalStorage": 400, +# "compStorage": 500 +#} + +def telemetryInfoCheck(infoDict=''): + if "instanceId" not in infoDict or len(infoDict["instanceId"]) == 0: + tdLog.exit("instanceId is null!") + + if "reportVersion" not in infoDict or infoDict["reportVersion"] != 1: + tdLog.exit("reportVersion is null!") + + if "os" not in infoDict: + tdLog.exit("os is null!") + + if "cpuModel" not in infoDict: + tdLog.exit("cpuModel is null!") + + if "numOfCpu" not in infoDict or infoDict["numOfCpu"] == 0: + tdLog.exit("numOfCpu is null!") + + if "memory" not in infoDict: + tdLog.exit("memory is null!") + + if "version" not in infoDict: + tdLog.exit("version is null!") + + if "buildInfo" not in infoDict: + tdLog.exit("buildInfo is null!") + + if "gitInfo" not in infoDict: + tdLog.exit("gitInfo is null!") + + if "email" not in infoDict: + tdLog.exit("email is not exists!") + + if "numOfDnode" not in infoDict or infoDict["numOfDnode"] < 1: + tdLog.exit("numOfDnode is null!") + + if "numOfMnode" not in infoDict or infoDict["numOfMnode"] < 1: + tdLog.exit("numOfMnode is null!") + + if "numOfVgroup" not in infoDict or infoDict["numOfVgroup"] <= 0: + tdLog.exit("numOfVgroup is null!") + + if "numOfDatabase" not in infoDict or infoDict["numOfDatabase"] <= 0: + tdLog.exit("numOfDatabase is null!") + + if "numOfSuperTable" not in infoDict or infoDict["numOfSuperTable"] < 0: + tdLog.exit("numOfSuperTable is null!") + + if "numOfChildTable" not in infoDict or infoDict["numOfChildTable"] < 0: + tdLog.exit("numOfChildTable is null!") + + if "numOfColumn" not in infoDict or infoDict["numOfColumn"] < 0: + tdLog.exit("numOfColumn is null!") + + if "numOfPoint" not in infoDict or infoDict["numOfPoint"] < 0: + tdLog.exit("numOfPoint is null!") + + if "totalStorage" not in infoDict or infoDict["totalStorage"] < 0: + tdLog.exit("totalStorage is null!") + + if "compStorage" not in infoDict or infoDict["compStorage"] < 0: + tdLog.exit("compStorage is null!") + + +class RequestHandlerImpl(http.server.BaseHTTPRequestHandler): + def do_GET(self): + """ + process GET request + """ + + def do_POST(self): + """ + process POST request + """ + contentEncoding = self.headers["Content-Encoding"] + + if contentEncoding == 'gzip': + req_body = self.rfile.read(int(self.headers["Content-Length"])) + plainText = gzip.decompress(req_body).decode() + else: + plainText = self.rfile.read(int(self.headers["Content-Length"])).decode() + + print("monitor info:\n%s"%plainText) + + # 1. send response code and header + self.send_response(200) + self.send_header("Content-Type", "text/html; charset=utf-8") + self.end_headers() + + # 2. send response content + #self.wfile.write(("Hello World: " + req_body + "\n").encode("utf-8")) + + # 3. check request body info + infoDict = json.loads(plainText) + #print("================") + #print(infoDict) + telemetryInfoCheck(infoDict) + + # 4. shutdown the server and exit case + assassin = threading.Thread(target=httpServer.shutdown) + assassin.daemon = True + assassin.start() + print ("==== shutdown http server ====") + +class TDTestCase: + hostname = socket.gethostname() + serverPort = '7080' + rpcDebugFlagVal = '143' + clientCfgDict = {'serverPort': '', 'firstEp': '', 'secondEp':'', 'rpcDebugFlag':'135', 'fqdn':''} + clientCfgDict["serverPort"] = serverPort + clientCfgDict["firstEp"] = hostname + ':' + serverPort + clientCfgDict["secondEp"] = hostname + ':' + serverPort + clientCfgDict["rpcDebugFlag"] = rpcDebugFlagVal + clientCfgDict["fqdn"] = hostname + + updatecfgDict = {'clientCfg': {}, 'serverPort': '', 'firstEp': '', 'secondEp':'', 'rpcDebugFlag':'135', 'fqdn':''} + updatecfgDict["clientCfg"] = clientCfgDict + updatecfgDict["serverPort"] = serverPort + updatecfgDict["firstEp"] = hostname + ':' + serverPort + updatecfgDict["secondEp"] = hostname + ':' + serverPort + updatecfgDict["fqdn"] = hostname + + updatecfgDict["telemetryReporting"] = '1' + updatecfgDict["telemetryServer"] = hostname + updatecfgDict["telemetryPort"] = telemetryPort + updatecfgDict["telemetryInterval"] = "3" + + print ("===================: ", updatecfgDict) + + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + + def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring + tdSql.prepare() + # time.sleep(2) + vgroups = "30" + sql = "create database db3 vgroups " + vgroups + tdSql.query(sql) + + # loop to wait request + httpServer.serve_forever() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +# create http server: bing ip/port , and request processor +serverAddress = ("", int(telemetryPort)) +httpServer = http.server.HTTPServer(serverAddress, RequestHandlerImpl) + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) + + + + + From 3a7b65b038a54d295c5df65a8499485d29b660de Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Sat, 7 May 2022 15:18:42 +0800 Subject: [PATCH 82/83] [test: add test cases for telemetry] --- tests/system-test/fulltest.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 4149c2cd9d..dfb6fdbefb 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -5,6 +5,7 @@ set -x python3 ./test.py -f 0-others/taosShell.py python3 ./test.py -f 0-others/taosShellError.py python3 ./test.py -f 0-others/taosShellNetChk.py +python3 ./test.py -f 0-others/telemetry.py #python3 ./test.py -f 2-query/between.py From 5763b24843059d8bef618f1df32c3f43509d0416 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Sat, 7 May 2022 15:51:02 +0800 Subject: [PATCH 83/83] fix(query): fix timeunit not processed in time functions when calculate constant values. --- source/libs/parser/src/parTranslater.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index bd614d0165..ea5d373a73 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -481,6 +481,7 @@ static EDealRes translateValue(STranslateContext* pCxt, SValueNode* pVal) { TSDB_CODE_SUCCESS) { return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, pVal->literal); } + *(int64_t*)&pVal->typeData = pVal->datum.i; } else { switch (pVal->node.resType.type) { case TSDB_DATA_TYPE_NULL: