diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 4d0f5365f3..45f0def157 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -560,9 +560,14 @@ static int32_t smlGenerateSchemaAction(SSchema *colField, SHashObj *colHash, SSm static int32_t smlFindNearestPowerOf2(int32_t length, uint8_t type) { int32_t result = 1; - while (result <= length) { - result *= 2; + if (length < 1024) { + while (result <= length) { + result <<= 1; + } + } else { + result = length; } + if (type == TSDB_DATA_TYPE_BINARY && result > TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) { result = TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE; } else if (type == TSDB_DATA_TYPE_NCHAR && result > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { @@ -624,10 +629,6 @@ static int32_t getBytes(uint8_t type, int32_t length) { static int32_t smlBuildFieldsList(SSmlHandle *info, SSchema *schemaField, SHashObj *schemaHash, SArray *cols, SArray *results, int32_t numOfCols, bool isTag) { - bool check = numOfCols == 0 ? true : false; - int32_t maxLen = isTag ? TSDB_MAX_TAGS_LEN : TSDB_MAX_BYTES_PER_ROW; - - int32_t len = 0; for (int j = 0; j < taosArrayGetSize(cols); ++j) { SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, j); ESchemaAction action = SCHEMA_ACTION_NULL; @@ -639,15 +640,6 @@ static int32_t smlBuildFieldsList(SSmlHandle *info, SSchema *schemaField, SHashO SField field = {0}; field.type = kv->type; field.bytes = getBytes(kv->type, kv->length); - if (check) { - len += field.bytes; - if (len > maxLen) { - code = TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; - uError("smlBuildFieldsList add %s failed since %s", isTag ? "tag" : "col", tstrerror(code)); - return code; - } - } - memcpy(field.name, kv->key, kv->keyLen); taosArrayPush(results, &field); } else if (action == SCHEMA_ACTION_CHANGE_COLUMN_SIZE || action == SCHEMA_ACTION_CHANGE_TAG_SIZE) { @@ -660,16 +652,19 @@ static int32_t smlBuildFieldsList(SSmlHandle *info, SSchema *schemaField, SHashO if (isTag) newIndex -= numOfCols; SField *field = (SField *)taosArrayGet(results, newIndex); field->bytes = getBytes(kv->type, kv->length); - if (check) { - len += (kv->length - schemaField[*index].bytes + VARSTR_HEADER_SIZE); - if (len > maxLen) { - code = TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; - uError("smlBuildFieldsList change %s failed since %s", isTag ? "tag" : "col", tstrerror(code)); - return code; - } - } } } + + int32_t maxLen = isTag ? TSDB_MAX_TAGS_LEN : TSDB_MAX_BYTES_PER_ROW; + int32_t len = 0; + for (int j = 0; j < taosArrayGetSize(results); ++j) { + SField *field = taosArrayGet(results, j); + len += field->bytes; + } + if (len > maxLen) { + return isTag ? TSDB_CODE_PAR_INVALID_TAGS_LENGTH : TSDB_CODE_PAR_INVALID_ROW_LENGTH; + } + return TSDB_CODE_SUCCESS; } @@ -867,6 +862,21 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { goto end; } + if (taosArrayGetSize(pTags) + pTableMeta->tableInfo.numOfColumns > TSDB_MAX_COLUMNS) { + uError("SML:0x%" PRIx64 " too many columns than 4096", info->id); + code = TSDB_CODE_PAR_TOO_MANY_COLUMNS; + taosArrayDestroy(pColumns); + taosArrayDestroy(pTags); + goto end; + } + if (taosArrayGetSize(pTags) > TSDB_MAX_TAGS) { + uError("SML:0x%" PRIx64 " too many tags than 128", info->id); + code = TSDB_CODE_PAR_INVALID_TAGS_NUM; + taosArrayDestroy(pColumns); + taosArrayDestroy(pTags); + goto end; + } + code = smlSendMetaMsg(info, &pName, pColumns, pTags, pTableMeta, action); if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%" PRIx64 " smlSendMetaMsg failed. can not create %s", info->id, pName.tname); @@ -923,6 +933,14 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { goto end; } + if (taosArrayGetSize(pColumns) + pTableMeta->tableInfo.numOfTags > TSDB_MAX_COLUMNS) { + uError("SML:0x%" PRIx64 " too many columns than 4096", info->id); + code = TSDB_CODE_PAR_TOO_MANY_COLUMNS; + taosArrayDestroy(pColumns); + taosArrayDestroy(pTags); + goto end; + } + code = smlSendMetaMsg(info, &pName, pColumns, pTags, pTableMeta, action); if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%" PRIx64 " smlSendMetaMsg failed. can not create %s", info->id, pName.tname); @@ -1527,8 +1545,11 @@ static int smlProcess(SSmlHandle *info, char *lines[], char *rawLine, char *rawL do { code = smlModifyDBSchemas(info); - if (code == 0) break; - taosMsleep(500); + if (code == 0 || code == TSDB_CODE_SML_INVALID_DATA || code == TSDB_CODE_PAR_TOO_MANY_COLUMNS || + code == TSDB_CODE_PAR_INVALID_TAGS_NUM || code == TSDB_CODE_PAR_INVALID_ROW_LENGTH || + code == TSDB_CODE_PAR_INVALID_TAGS_LENGTH) + break; + taosMsleep(100); uInfo("SML:0x%" PRIx64 " smlModifyDBSchemas retry code:%s, times:%d", info->id, tstrerror(code), retryNum); } while (retryNum++ < taosHashGetSize(info->superTables) * MAX_RETRY_TIMES); diff --git a/source/client/src/clientSmlJson.c b/source/client/src/clientSmlJson.c index c3a6e15697..b0ae316031 100644 --- a/source/client/src/clientSmlJson.c +++ b/source/client/src/clientSmlJson.c @@ -575,7 +575,7 @@ static int32_t smlConvertJSONString(SSmlKv *pVal, char *typeStr, cJSON *value) { uError("OTD:invalid type(%s) for JSON String", typeStr); return TSDB_CODE_TSC_INVALID_JSON_TYPE; } - pVal->length = (uint16_t)strlen(value->valuestring); + pVal->length = strlen(value->valuestring); if (pVal->type == TSDB_DATA_TYPE_BINARY && pVal->length > TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) { return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; diff --git a/tests/system-test/1-insert/influxdb_line_taosc_insert.py b/tests/system-test/1-insert/influxdb_line_taosc_insert.py index 667ce3239c..46aec2909a 100644 --- a/tests/system-test/1-insert/influxdb_line_taosc_insert.py +++ b/tests/system-test/1-insert/influxdb_line_taosc_insert.py @@ -439,7 +439,7 @@ class TDTestCase: for input_sql in [self.genLongSql(127, 1)[0], self.genLongSql(1, 4093)[0]]: tdCom.cleanTb(dbname="test") self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) - for input_sql in [self.genLongSql(129, 1)[0], self.genLongSql(1, 4095)[0]]: + for input_sql in [self.genLongSql(128, 1)[0], self.genLongSql(1, 4094)[0]]: tdCom.cleanTb(dbname="test") try: self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) @@ -578,10 +578,16 @@ class TDTestCase: # binary stb_name = tdCom.getLongName(7, "letters") - input_sql = f'{stb_name},t0=t,t1="{tdCom.getLongName(16374, "letters")}" c0=f 1626006833639000000' + input_sql = f'{stb_name},t0=t,t1="{tdCom.getLongName(4091, "letters")}" c0=f 1626006833639000000' self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) - input_sql = f'{stb_name},t0=t,t1="{tdCom.getLongName(16375, "letters")}" c0=f 1626006833639000000' + input_sql = f'{stb_name},t0="a",t1="{tdCom.getLongName(4088, "letters")}" c0=f 1626006833639000000' + try: + self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) + except SchemalessError as err: + tdSql.checkNotEqual(err.errno, 0) + + input_sql = f'{stb_name},t0=t,t1="{tdCom.getLongName(4092, "letters")}" c0=f 1626006833639000000' try: self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) except SchemalessError as err: @@ -590,10 +596,10 @@ class TDTestCase: # nchar # * legal nchar could not be larger than 16374/4 stb_name = tdCom.getLongName(7, "letters") - input_sql = f'{stb_name},t0=t,t1=L"{tdCom.getLongName(4093, "letters")}" c0=f 1626006833639000000' + input_sql = f'{stb_name},t0=t,t1=L"{tdCom.getLongName(4090, "letters")}" c0=f 1626006833639000000' self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) - input_sql = f'{stb_name},t0=t,t1=L"{tdCom.getLongName(4094, "letters")}" c0=f 1626006833639000000' + input_sql = f'{stb_name},t0=t,t1=L"{tdCom.getLongName(4091, "letters")}" c0=f 1626006833639000000' try: self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) except SchemalessError as err: @@ -674,9 +680,15 @@ class TDTestCase: # binary stb_name = tdCom.getLongName(7, "letters") - input_sql = f'{stb_name},t0=t c0=f,c1="{tdCom.getLongName(65517, "letters")}" 1626006833639000000' + input_sql = f'{stb_name},t0=t c0=1i32,c1="{tdCom.getLongName(65517, "letters")}" 1626006833639000000' self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) + input_sql = f'{stb_name},t0=t c0=1i32,c1="{tdCom.getLongName(65517, "letters")},c2=f" 1626006833639000000' + try: + self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) + except SchemalessError as err: + tdSql.checkNotEqual(err.errno, 0) + input_sql = f'{stb_name},t0=t c0=f,c1="{tdCom.getLongName(65518, "letters")}" 1626006833639000000' try: self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) @@ -686,10 +698,10 @@ class TDTestCase: # nchar # * legal nchar could not be larger than 16374/4 stb_name = tdCom.getLongName(7, "letters") - input_sql = f'{stb_name},t0=t c0=f,c1=L"{tdCom.getLongName(16379, "letters")}" 1626006833639000000' + input_sql = f'{stb_name},t0=t c0=1i32,c1=L"{tdCom.getLongName(16379, "letters")}",c2=f 1626006833639000000' self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) - input_sql = f'{stb_name},t0=t c0=f,c1=L"{tdCom.getLongName(16380, "letters")}" 1626006833639000000' + input_sql = f'{stb_name},t0=t c0=1i32,c1=L"{tdCom.getLongName(16380, "letters")}",c2=1i16 1626006833639000000' try: self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) except SchemalessError as err: @@ -896,52 +908,57 @@ class TDTestCase: self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) # * every binary and nchar must be length+2, so here is two tag, max length could not larger than 16384-2*2 - # input_sql = f'{stb_name}, t1="{tdCom.getLongName(4095, "letters")}"" c0=f 1626006833639000000' - # self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) - - # tdSql.query(f"select * from {stb_name}") - # tdSql.checkRows(2) - # input_sql = f'{stb_name},t0=t,t1="{tdCom.getLongName(4084, "letters")}",t2="{tdCom.getLongName(6, "letters")}" c0=f 1626006833639000000' - # try: - # self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) - # raise Exception("should not reach here") - # except SchemalessError as err: - # tdSql.checkNotEqual(err.errno, 0) - # tdSql.query(f"select * from {stb_name}") - # tdSql.checkRows(2) - - # # * check col，col+ts max in describe ---> 16143 - input_sql = f'{stb_name},t0=t c0=i32,c1="{tdCom.getLongName(65517, "letters")}" 1626006833639000000' + input_sql = f'{stb_name}, t0=f,t1="{tdCom.getLongName(4093, "letters")}" 1626006833639000000' self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) - input_sql = f'{stb_name},t0=t c0=i32,c1="{tdCom.getLongName(65517, "letters")},c2=f" 1626006833639000000' - self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) + tdSql.query(f"select * from {stb_name}") + tdSql.checkRows(2) + input_sql = f'{stb_name},t0=t,t1="{tdCom.getLongName(4084, "letters")}",t2="{tdCom.getLongName(6, "letters")}" c0=f 1626006833639000000' try: self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) + raise Exception("should not reach here") except SchemalessError as err: tdSql.checkNotEqual(err.errno, 0) + tdSql.query(f"select * from {stb_name}") + tdSql.checkRows(2) - input_sql = f'{stb_name},t0=t c0=i16,c1="{tdCom.getLongName(49133, "letters")}",c2="{tdCom.getLongName(16384, "letters")}" 1626006833639000000' - self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) - input_sql = f'{stb_name},t0=t c0=i16,c1="{tdCom.getLongName(49133, "letters")}",c2="{tdCom.getLongName(16384, "letters")},c3=t" 1626006833639000000' + stb_name = tdCom.getLongName(8, "letters") + # # * check col，col+ts max in describe ---> 16143 + input_sql = f'{stb_name},t0=t c0=1i32,c1="{tdCom.getLongName(65517, "letters")}" 1626006833639000000' self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) + + input_sql = f'{stb_name},t0=t c0=1i32,c1="{tdCom.getLongName(65517, "letters")}",c2=f 1626006833639000000' try: self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) except SchemalessError as err: tdSql.checkNotEqual(err.errno, 0) tdSql.query(f"select * from {stb_name}") - tdSql.checkRows(3) + tdSql.checkRows(1) + + + stb_name = tdCom.getLongName(9, "letters") + input_sql = f'{stb_name},t0=t c0=1i16,c1="{tdCom.getLongName(49133, "letters")}",c2="{tdCom.getLongName(16384, "letters")}" 1626006833639000000' + self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) + + input_sql = f'{stb_name},t0=t c0=1i16,c1="{tdCom.getLongName(49133, "letters")}",c2="{tdCom.getLongName(16384, "letters")},c3=t" 1626006833639000000' + try: + self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) + except SchemalessError as err: + tdSql.checkNotEqual(err.errno, 0) + + tdSql.query(f"select * from {stb_name}") + tdSql.checkRows(1) + input_sql = f'{stb_name},t0=t c0=f,c1="{tdCom.getLongName(16374, "letters")}",c2="{tdCom.getLongName(16374, "letters")}",c3="{tdCom.getLongName(16374, "letters")}",c4="{tdCom.getLongName(13, "letters")}" 1626006833639000000' try: self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) except SchemalessError as err: tdSql.checkNotEqual(err.errno, 0) - tdSql.query(f"select * from {stb_name}") - tdSql.checkRows(3) - # * tag nchar max is 16374/4, col+ts nchar max 49151 + + # * tag nchar max is 16384/4, col+ts nchar max 65531 def tagColNcharMaxLengthCheckCase(self): """ check nchar length limit @@ -952,7 +969,7 @@ class TDTestCase: input_sql = f'{stb_name},id="{tb_name}",t0=t c0=f 1626006833639000000' code = self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) - # * legal nchar could not be larger than 16374/4 + # * legal tag nchar could not be larger than 16384/4 # input_sql = f'{stb_name},t0=t,t1=L"{tdCom.getLongName(4093, "letters")}",t2=L"{tdCom.getLongName(1, "letters")}" c0=f 1626006833639000000' # self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) # tdSql.query(f"select * from {stb_name}") @@ -968,14 +985,15 @@ class TDTestCase: input_sql = f'{stb_name},t0=t c0=f,c1=L"{tdCom.getLongName(4093, "letters")}",c2=L"{tdCom.getLongName(4093, "letters")}",c3=L"{tdCom.getLongName(4093, "letters")}",c4=L"{tdCom.getLongName(4, "letters")}" 1626006833639000000' self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) tdSql.query(f"select * from {stb_name}") - tdSql.checkRows(3) + tdSql.checkRows(2) + input_sql = f'{stb_name},t0=t c0=f,c1=L"{tdCom.getLongName(4093, "letters")}",c2=L"{tdCom.getLongName(4093, "letters")}",c3=L"{tdCom.getLongName(4093, "letters")}",c4=L"{tdCom.getLongName(5, "letters")}" 1626006833639000000' try: self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) except SchemalessError as err: tdSql.checkNotEqual(err.errno, 0) tdSql.query(f"select * from {stb_name}") - tdSql.checkRows(3) + tdSql.checkRows(2) def batchInsertCheckCase(self): """ @@ -1291,13 +1309,13 @@ class TDTestCase: self.idSeqCheckCase() self.idUpperCheckCase() self.noIdCheckCase() - # self.maxColTagCheckCase() + self.maxColTagCheckCase() self.idIllegalNameCheckCase() self.idStartWithNumCheckCase() self.nowTsCheckCase() self.dateFormatTsCheckCase() self.illegalTsCheckCase() - # self.tagValueLengthCheckCase() + self.tagValueLengthCheckCase() self.colValueLengthCheckCase() self.tagColIllegalValueCheckCase() self.duplicateIdTagColInsertCheckCase() @@ -1307,8 +1325,8 @@ class TDTestCase: self.tagColAddDupIDCheckCase() self.tagColAddCheckCase() self.tagMd5Check() - # self.tagColBinaryMaxLengthCheckCase() - # self.tagColNcharMaxLengthCheckCase() + self.tagColBinaryMaxLengthCheckCase() + self.tagColNcharMaxLengthCheckCase() self.batchInsertCheckCase() self.multiInsertCheckCase(10) self.batchErrorInsertCheckCase()