xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

refactor: fix error in schemaless

This commit is contained in:
wangmm0220 2022-05-13 11:19:13 +08:00
parent d3d4ac9b3f
commit 140c319e6b
4 changed files with 85 additions and 96 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

165
source/client/src/clientSml.c
View File

@ -103,7 +103,7 @@ typedef struct {
SHashObj *tagHash; // elements are <key, index in tags>
SArray *cols;
SHashObj *fieldHash;
SHashObj *colHash;
STableMeta *tableMeta;
} SSmlSTableMeta;
@ -238,7 +238,7 @@ static int32_t smlBuildColumnDescription(SSmlKv* field, char* buf, int32_t bufSi
char tname[TSDB_TABLE_NAME_LEN] = {0};
memcpy(tname, field->key, field->keyLen);
if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) {
int32_t bytes = field->valueLen; // todo
int32_t bytes = field->length; // todo
int out = snprintf(buf, bufSize,"`%s` %s(%d)",
tname,tDataTypes[field->type].name, bytes);
*outBytes = out;
@ -507,7 +507,7 @@ static int32_t smlModifyDBSchemas(SSmlHandle* info) {
static bool smlParseNumber(SSmlKv *kvVal, SSmlMsgBuf *msg){
const char *pVal = kvVal->value;
int32_t len = kvVal->valueLen;
int32_t len = kvVal->length;
char *endptr = NULL;
double result = strtod(pVal, &endptr);
if(pVal == endptr){
@ -591,7 +591,7 @@ static bool smlParseNumber(SSmlKv *kvVal, SSmlMsgBuf *msg){
static bool smlParseBool(SSmlKv *kvVal) {
const char *pVal = kvVal->value;
int32_t len = kvVal->valueLen;
int32_t len = kvVal->length;
if ((len == 1) && pVal[0] == 't') {
kvVal->i = true;
return true;
@ -778,18 +778,16 @@ static int32_t smlParseTS(SSmlHandle* info, const char* data, int32_t len, SArra
static bool smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg) {
//binary
if (smlIsBinary(pVal->value, pVal->valueLen)) {
if (smlIsBinary(pVal->value, pVal->length)) {
pVal->type = TSDB_DATA_TYPE_BINARY;
pVal->valueLen -= BINARY_ADD_LEN;
pVal->length = pVal->valueLen;
pVal->length -= BINARY_ADD_LEN;
pVal->value += (BINARY_ADD_LEN - 1);
return true;
}
//nchar
if (smlIsNchar(pVal->value, pVal->valueLen)) {
if (smlIsNchar(pVal->value, pVal->length)) {
pVal->type = TSDB_DATA_TYPE_NCHAR;
pVal->valueLen -= NCHAR_ADD_LEN;
pVal->length = pVal->valueLen;
pVal->length -= NCHAR_ADD_LEN;
pVal->value += (NCHAR_ADD_LEN - 1);
return true;
}
@ -953,7 +951,7 @@ static int32_t smlParseTelnetTags(const char* data, int32_t len, SArray *cols, S
kv->key = key;
kv->keyLen = keyLen;
kv->value = value;
kv->valueLen = valueLen;
kv->length = valueLen;
kv->type = TSDB_DATA_TYPE_NCHAR;
if(cols) taosArrayPush(cols, &kv);
@ -1002,7 +1000,7 @@ static int32_t smlParseTelnetString(SSmlHandle *info, const char* sql, SSmlTable
kv->key = VALUE;
kv->keyLen = VALUE_LEN;
kv->value = value;
kv->valueLen = valueLen;
kv->length = valueLen;
if(!smlParseValue(kv, &info->msgBuf) || kv->type == TSDB_DATA_TYPE_BINARY
|| kv->type == TSDB_DATA_TYPE_NCHAR || kv->type == TSDB_DATA_TYPE_BOOL){
return TSDB_CODE_SML_INVALID_DATA;
@ -1028,7 +1026,7 @@ static int32_t smlParseCols(const char* data, int32_t len, SArray *cols, bool is
kv->key = TAG;
kv->keyLen = TAG_LEN;
kv->value = TAG;
kv->valueLen = TAG_LEN;
kv->length = TAG_LEN;
kv->type = TSDB_DATA_TYPE_NCHAR;
if(cols) taosArrayPush(cols, &kv);
return TSDB_CODE_SUCCESS;
@ -1086,7 +1084,7 @@ static int32_t smlParseCols(const char* data, int32_t len, SArray *cols, bool is
kv->key = key;
kv->keyLen = keyLen;
kv->value = value;
kv->valueLen = valueLen;
kv->length = valueLen;
if(isTag){
kv->type = TSDB_DATA_TYPE_NCHAR;
}else{
@ -1142,73 +1140,40 @@ static int32_t smlParseCols(const char* data, int32_t len, SArray *cols, bool is
// return TSDB_CODE_SUCCESS;
//}
static bool smlUpdateMeta(SSmlSTableMeta* tableMeta, SArray *tags, SArray *cols, SSmlMsgBuf *msg){
if(tags){
for (int i = 0; i < taosArrayGetSize(tags); ++i) {
SSmlKv *kv = (SSmlKv *)taosArrayGetP(tags, i);
ASSERT(kv->type == TSDB_DATA_TYPE_NCHAR);
static bool smlUpdateMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols, SSmlMsgBuf *msg){
for (int i = 1; i < taosArrayGetSize(cols); ++i) { //jump timestamp
SSmlKv *kv = (SSmlKv *)taosArrayGetP(cols, i);
uint8_t *index = (uint8_t *)taosHashGet(tableMeta->tagHash, kv->key, kv->keyLen);
if(index){
SSmlKv **value = (SSmlKv **)taosArrayGet(tableMeta->tags, *index);
ASSERT((*value)->type == TSDB_DATA_TYPE_NCHAR);
if(kv->valueLen > (*value)->valueLen){ // tags type is nchar
*value = kv;
}
int16_t *index = (int16_t *)taosHashGet(metaHash, kv->key, kv->keyLen);
if(index){
SSmlKv **value = (SSmlKv **)taosArrayGet(metaArray, *index);
if(kv->type != (*value)->type){
smlBuildInvalidDataMsg(msg, "the type is not the same like before", kv->key);
return false;
}else{
size_t tmp = taosArrayGetSize(tableMeta->tags);
ASSERT(tmp <= UINT8_MAX);
uint8_t size = tmp;
taosArrayPush(tableMeta->tags, &kv);
taosHashPut(tableMeta->tagHash, kv->key, kv->keyLen, &size, CHAR_BYTES);
}
}
}
if(cols){
for (int i = 1; i < taosArrayGetSize(cols); ++i) { //jump timestamp
SSmlKv *kv = (SSmlKv *)taosArrayGetP(cols, i);
int16_t *index = (int16_t *)taosHashGet(tableMeta->fieldHash, kv->key, kv->keyLen);
if(index){
SSmlKv **value = (SSmlKv **)taosArrayGet(tableMeta->cols, *index);
if(kv->type != (*value)->type){
smlBuildInvalidDataMsg(msg, "the type is not the same like before", kv->key);
return false;
}else{
if(IS_VAR_DATA_TYPE(kv->type)){ // update string len, if bigger
if(kv->valueLen > (*value)->valueLen){
*value = kv;
}
if(IS_VAR_DATA_TYPE(kv->type)){ // update string len, if bigger
if(kv->length > (*value)->length){
*value = kv;
}
}
}else{
size_t tmp = taosArrayGetSize(tableMeta->cols);
ASSERT(tmp <= INT16_MAX);
int16_t size = tmp;
taosArrayPush(tableMeta->cols, &kv);
taosHashPut(tableMeta->fieldHash, kv->key, kv->keyLen, &size, SHORT_BYTES);
}
}else{
size_t tmp = taosArrayGetSize(metaArray);
ASSERT(tmp <= INT16_MAX);
int16_t size = tmp;
taosArrayPush(metaArray, &kv);
taosHashPut(metaHash, kv->key, kv->keyLen, &size, SHORT_BYTES);
}
}
return true;
}
static void smlInsertMeta(SSmlSTableMeta* tableMeta, SArray *tags, SArray *cols){
if(tags){
for (uint8_t i = 0; i < taosArrayGetSize(tags); ++i) {
SSmlKv *kv = (SSmlKv *)taosArrayGetP(tags, i);
taosArrayPush(tableMeta->tags, &kv);
taosHashPut(tableMeta->tagHash, kv->key, kv->keyLen, &i, CHAR_BYTES);
}
}
if(cols){
for (int16_t i = 0; i < taosArrayGetSize(cols); ++i) {
SSmlKv *kv = (SSmlKv *)taosArrayGetP(cols, i);
taosArrayPush(tableMeta->cols, &kv);
taosHashPut(tableMeta->fieldHash, kv->key, kv->keyLen, &i, SHORT_BYTES);
}
static void smlInsertMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols){
for (int16_t i = 0; i < taosArrayGetSize(cols); ++i) {
SSmlKv *kv = (SSmlKv *)taosArrayGetP(cols, i);
taosArrayPush(metaArray, &kv);
taosHashPut(metaHash, kv->key, kv->keyLen, &i, SHORT_BYTES);
}
}
@ -1236,12 +1201,16 @@ cleanup:
return NULL;
}
static void smlDestroyTableInfo(SSmlTableInfo *tag, bool format){
if(format){
static void smlDestroyTableInfo(SSmlHandle* info, SSmlTableInfo *tag){
if(info->dataFormat){
for(size_t i = 0; i < taosArrayGetSize(tag->cols); i++){
SArray *kvArray = (SArray *)taosArrayGetP(tag->cols, i);
for (int j = 0; j < taosArrayGetSize(kvArray); ++j) {
void *p = taosArrayGetP(kvArray, j);
SSmlKv *p = (SSmlKv *)taosArrayGetP(kvArray, j);
if(info->protocol == TSDB_SML_JSON_PROTOCOL &&
(p->type == TSDB_DATA_TYPE_NCHAR || p->type == TSDB_DATA_TYPE_BINARY)){
taosMemoryFree((void*)p->value);
}
taosMemoryFree(p);
}
taosArrayDestroy(kvArray);
@ -1257,6 +1226,19 @@ static void smlDestroyTableInfo(SSmlTableInfo *tag, bool format){
taosHashCleanup(kvHash);
}
}
for(size_t i = 0; i < taosArrayGetSize(tag->tags); i++){
SSmlKv *p = (SSmlKv *)taosArrayGetP(tag->tags, i);
if(info->protocol == TSDB_SML_JSON_PROTOCOL){
taosMemoryFree((void*)p->key);
if(p->type == TSDB_DATA_TYPE_NCHAR || p->type == TSDB_DATA_TYPE_BINARY){
taosMemoryFree((void*)p->value);
}
}
taosMemoryFree(p);
}
if(info->protocol == TSDB_SML_JSON_PROTOCOL && tag->sTableName){
taosMemoryFree((void*)tag->sTableName);
}
taosArrayDestroy(tag->cols);
taosArrayDestroy(tag->tags);
taosMemoryFree(tag);
@ -1293,8 +1275,8 @@ static SSmlSTableMeta* smlBuildSTableMeta(){
goto cleanup;
}
meta->fieldHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
if (meta->fieldHash == NULL) {
meta->colHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
if (meta->colHash == NULL) {
uError("SML:smlBuildSTableMeta failed to allocate memory");
goto cleanup;
}
@ -1319,7 +1301,7 @@ cleanup:
static void smlDestroySTableMeta(SSmlSTableMeta *meta){
taosHashCleanup(meta->tagHash);
taosHashCleanup(meta->fieldHash);
taosHashCleanup(meta->colHash);
taosArrayDestroy(meta->tags);
taosArrayDestroy(meta->cols);
taosMemoryFree(meta->tableMeta);
@ -1341,7 +1323,7 @@ static void smlDestroyInfo(SSmlHandle* info){
// destroy info->childTables
void** p1 = (void**)taosHashIterate(info->childTables, NULL);
while (p1) {
smlDestroyTableInfo((SSmlTableInfo*)(*p1), info->dataFormat);
smlDestroyTableInfo(info, (SSmlTableInfo*)(*p1));
p1 = (void**)taosHashIterate(info->childTables, p1);
}
taosHashCleanup(info->childTables);
@ -1357,7 +1339,9 @@ static void smlDestroyInfo(SSmlHandle* info){
// destroy info->pVgHash
taosHashCleanup(info->pVgHash);
taosHashCleanup(info->dumplicateKey);
if(!info->dataFormat){
taosArrayDestroy(info->colsContainer);
}
taosMemoryFreeClear(info);
}
@ -1682,8 +1666,7 @@ static int32_t smlConvertJSONString(SSmlKv *pVal, char* typeStr, cJSON *value) {
return TSDB_CODE_TSC_INVALID_JSON_TYPE;
}
pVal->length = (int16_t)strlen(value->valuestring);
pVal->valueLen = pVal->length;
return smlJsonCreateSring(&pVal->value, value->valuestring, pVal->valueLen);
return smlJsonCreateSring(&pVal->value, value->valuestring, pVal->length);
}
static int32_t smlParseValueFromJSONObj(cJSON *root, SSmlKv *kv) {
@ -1994,14 +1977,18 @@ static int32_t smlParseInfluxLine(SSmlHandle* info, const char* sql) {
SSmlSTableMeta** tableMeta = (SSmlSTableMeta**)taosHashGet(info->superTables, elements.measure, elements.measureLen);
if(tableMeta){ // update meta
ret = smlUpdateMeta(*tableMeta, hasTable ? NULL : (*oneTable)->tags, cols, &info->msgBuf);
ret = smlUpdateMeta((*tableMeta)->colHash, (*tableMeta)->cols, cols, &info->msgBuf);
if(!hasTable && ret){
ret = smlUpdateMeta((*tableMeta)->tagHash, (*tableMeta)->tags, (*oneTable)->tags, &info->msgBuf);
}
if(!ret){
uError("SML:0x%"PRIx64" smlUpdateMeta failed", info->id);
return TSDB_CODE_SML_INVALID_DATA;
}
}else{
SSmlSTableMeta *meta = smlBuildSTableMeta();
smlInsertMeta(meta, (*oneTable)->tags, cols);
smlInsertMeta(meta->tagHash, meta->tags, (*oneTable)->tags);
smlInsertMeta(meta->colHash, meta->cols, cols);
taosHashPut(info->superTables, elements.measure, elements.measureLen, &meta, POINTER_BYTES);
}
@ -2034,7 +2021,7 @@ static int32_t smlParseTelnetLine(SSmlHandle* info, void *data) {
}
if(ret != TSDB_CODE_SUCCESS){
uError("SML:0x%"PRIx64" smlParseTelnetLine failed", info->id);
smlDestroyTableInfo(tinfo, true);
smlDestroyTableInfo(info, tinfo);
taosArrayDestroy(cols);
return ret;
}
@ -2057,20 +2044,24 @@ static int32_t smlParseTelnetLine(SSmlHandle* info, void *data) {
oneTable = &tinfo;
hasTable = false;
}else{
smlDestroyTableInfo(tinfo, true);
smlDestroyTableInfo(info, tinfo);
}
taosArrayPush((*oneTable)->cols, &cols);
SSmlSTableMeta** tableMeta = (SSmlSTableMeta** )taosHashGet(info->superTables, (*oneTable)->sTableName, (*oneTable)->sTableNameLen);
if(tableMeta){ // update meta
ret = smlUpdateMeta(*tableMeta, hasTable ? NULL : (*oneTable)->tags, cols, &info->msgBuf);
ret = smlUpdateMeta((*tableMeta)->colHash, (*tableMeta)->cols, cols, &info->msgBuf);
if(!hasTable && ret){
ret = smlUpdateMeta((*tableMeta)->tagHash, (*tableMeta)->tags, (*oneTable)->tags, &info->msgBuf);
}
if(!ret){
uError("SML:0x%"PRIx64" smlUpdateMeta failed", info->id);
return TSDB_CODE_SML_INVALID_DATA;
}
}else{
SSmlSTableMeta *meta = smlBuildSTableMeta();
smlInsertMeta(meta, (*oneTable)->tags, cols);
smlInsertMeta(meta->tagHash, meta->tags, (*oneTable)->tags);
smlInsertMeta(meta->colHash, meta->cols, cols);
taosHashPut(info->superTables, (*oneTable)->sTableName, (*oneTable)->sTableNameLen, &meta, POINTER_BYTES);
}

6
source/client/test/smlTest.cpp
View File

@ -228,7 +228,7 @@ TEST(testCase, smlParseCols_tag_Test) {
ASSERT_EQ(strncasecmp(kv->key, "cbin", 4), 0);
ASSERT_EQ(kv->keyLen, 4);
ASSERT_EQ(kv->type, TSDB_DATA_TYPE_NCHAR);
ASSERT_EQ(kv->valueLen, 17);
ASSERT_EQ(kv->length, 17);
ASSERT_EQ(strncasecmp(kv->value, "\"passit", 7), 0);
taosMemoryFree(kv);
@ -237,7 +237,7 @@ TEST(testCase, smlParseCols_tag_Test) {
ASSERT_EQ(strncasecmp(kv->key, "cf64", 4), 0);
ASSERT_EQ(kv->keyLen, 4);
ASSERT_EQ(kv->type, TSDB_DATA_TYPE_NCHAR);
ASSERT_EQ(kv->valueLen, 7);
ASSERT_EQ(kv->length, 7);
ASSERT_EQ(strncasecmp(kv->value, "4.31f64", 7), 0);
taosMemoryFree(kv);
@ -259,7 +259,7 @@ TEST(testCase, smlParseCols_tag_Test) {
ASSERT_EQ(strncasecmp(kv->key, TAG, strlen(TAG)), 0);
ASSERT_EQ(kv->keyLen, strlen(TAG));
ASSERT_EQ(kv->type, TSDB_DATA_TYPE_NCHAR);
ASSERT_EQ(kv->valueLen, strlen(TAG));
ASSERT_EQ(kv->length, strlen(TAG));
ASSERT_EQ(strncasecmp(kv->value, TAG, strlen(TAG)), 0);
taosMemoryFree(kv);

2
source/common/src/tname.c
View File

@ -317,7 +317,7 @@ void buildChildTableName(RandTableName* rName) {
for (int j = 0; j < size; ++j) {
SSmlKv* tagKv = taosArrayGetP(rName->tags, j);
taosStringBuilderAppendStringLen(&sb, tagKv->key, tagKv->keyLen);
taosStringBuilderAppendStringLen(&sb, tagKv->value, tagKv->valueLen);
taosStringBuilderAppendStringLen(&sb, tagKv->value, tagKv->length);
}
size_t len = 0;
char* keyJoined = taosStringBuilderGetResult(&sb, &len);

8
source/libs/parser/src/parInsert.c
View File

@ -1668,7 +1668,7 @@ static int32_t smlBuildTagRow(SArray* cols, SKVRowBuilder* tagsBuilder, SParsedD
SSchema* pTagSchema = &pSchema[tags->boundColumns[i] - 1]; // colId starts with 1
param.schema = pTagSchema;
SSmlKv* kv = taosArrayGetP(cols, i);
KvRowAppend(msg, kv->value, kv->valueLen, &param);
KvRowAppend(msg, kv->value, kv->length, &param);
}
*row = tdGetKVRowFromBuilder(tagsBuilder);
@ -1766,10 +1766,8 @@ int32_t smlBindData(void *handle, SArray *tags, SArray *colsSchema, SArray *cols
if (!kv || kv->length == 0) {
MemRowAppend(&pBuf, NULL, 0, &param);
} else {
int32_t colLen = pColSchema->bytes;
if (IS_VAR_DATA_TYPE(pColSchema->type)) {
colLen = kv->length;
} else if (pColSchema->type == TSDB_DATA_TYPE_TIMESTAMP) {
int32_t colLen = kv->length;
if (pColSchema->type == TSDB_DATA_TYPE_TIMESTAMP) {
kv->i = convertTimePrecision(kv->i, TSDB_TIME_PRECISION_NANO, pTableMeta->tableInfo.precision);
}