From f171e8033a4141f7d0c93568fd41989411fb0517 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Thu, 12 Aug 2021 12:24:41 +0800 Subject: [PATCH 01/30] [TD-xxxx]:begin coding taos_sml_insert --- src/client/src/tscParseLineProtocol.c | 199 ++++++++++++++------------ 1 file changed, 111 insertions(+), 88 deletions(-) diff --git a/src/client/src/tscParseLineProtocol.c b/src/client/src/tscParseLineProtocol.c index 2a16f1aad6..43509d3533 100644 --- a/src/client/src/tscParseLineProtocol.c +++ b/src/client/src/tscParseLineProtocol.c @@ -32,9 +32,6 @@ typedef struct { uint8_t type; int16_t length; char* value; - - //=================================== - uint32_t fieldSchemaIdx; } TAOS_SML_KV; typedef struct { @@ -47,9 +44,6 @@ typedef struct { // first kv must be timestamp TAOS_SML_KV* fields; int32_t fieldNum; - - //================================ - uint32_t schemaIdx; } TAOS_SML_DATA_POINT; typedef enum { @@ -62,10 +56,23 @@ typedef enum { typedef struct { uint64_t id; - + SHashObj* smlDataToSchema; } SSmlLinesInfo; + //================================================================================================= +static uint64_t linesSmlHandleId = 0; + +uint64_t genLinesSmlId() { + uint64_t id; + + do { + id = atomic_add_fetch_64(&linesSmlHandleId, 1); + } while (id == 0); + + return id; +} + int compareSmlColKv(const void* p1, const void* p2) { TAOS_SML_KV* kv1 = (TAOS_SML_KV*)p1; TAOS_SML_KV* kv2 = (TAOS_SML_KV*)p2; @@ -168,11 +175,46 @@ static int32_t buildSmlKvSchema(TAOS_SML_KV* smlKv, SHashObj* hash, SArray* arra taosHashPut(hash, field.name, tagKeyLen, &fieldIdx, sizeof(fieldIdx)); } - smlKv->fieldSchemaIdx = (uint32_t)fieldIdx; + uintptr_t valPointer = (uintptr_t)smlKv; + taosHashPut(info->smlDataToSchema, &valPointer, sizeof(uintptr_t), &fieldIdx, sizeof(fieldIdx)); return 0; } +static int32_t getSmlMd5ChildTableName(TAOS_SML_DATA_POINT* point, char* tableName, int* tableNameLen, + SSmlLinesInfo* info) { + tscDebug("SML:0x%"PRIx64" taos_sml_insert get child table name through md5", info->id); + qsort(point->tags, point->tagNum, sizeof(TAOS_SML_KV), compareSmlColKv); + + SStringBuilder sb; memset(&sb, 0, sizeof(sb)); + char sTableName[TSDB_TABLE_NAME_LEN] = {0}; + strtolower(sTableName, point->stableName); + taosStringBuilderAppendString(&sb, sTableName); + for (int j = 0; j < point->tagNum; ++j) { + taosStringBuilderAppendChar(&sb, ','); + TAOS_SML_KV* tagKv = point->tags + j; + char tagName[TSDB_COL_NAME_LEN] = {0}; + strtolower(tagName, tagKv->key); + taosStringBuilderAppendString(&sb, tagName); + taosStringBuilderAppendChar(&sb, '='); + taosStringBuilderAppend(&sb, tagKv->value, tagKv->length); + } + size_t len = 0; + char* keyJoined = taosStringBuilderGetResult(&sb, &len); + MD5_CTX context; + MD5Init(&context); + MD5Update(&context, (uint8_t *)keyJoined, (uint32_t)len); + MD5Final(&context); + *tableNameLen = snprintf(tableName, *tableNameLen, + "t_%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x", context.digest[0], + context.digest[1], context.digest[2], context.digest[3], context.digest[4], context.digest[5], context.digest[6], + context.digest[7], context.digest[8], context.digest[9], context.digest[10], context.digest[11], + context.digest[12], context.digest[13], context.digest[14], context.digest[15]); + taosStringBuilderDestroy(&sb); + tscDebug("SML:0x%"PRIx64" child table name: %s", info->id, tableName); + return 0; +} + static int32_t buildDataPointSchemas(TAOS_SML_DATA_POINT* points, int numPoint, SArray* stableSchemas, SSmlLinesInfo* info) { int32_t code = 0; SHashObj* sname2shema = taosHashInit(32, @@ -203,6 +245,15 @@ static int32_t buildDataPointSchemas(TAOS_SML_DATA_POINT* points, int numPoint, for (int j = 0; j < point->tagNum; ++j) { TAOS_SML_KV* tagKv = point->tags + j; + if (!point->childTableName) { + char childTableName[TSDB_TABLE_NAME_LEN]; + int32_t tableNameLen = TSDB_TABLE_NAME_LEN; + getSmlMd5ChildTableName(point, childTableName, &tableNameLen, info); + point->childTableName = calloc(1, tableNameLen+1); + strncpy(point->childTableName, childTableName, tableNameLen); + point->childTableName[tableNameLen] = '\0'; + } + code = buildSmlKvSchema(tagKv, pStableSchema->tagHash, pStableSchema->tags, info); if (code != 0) { tscError("SML:0x%"PRIx64" build data point schema failed. point no.: %d, tag key: %s", info->id, i, tagKv->key); @@ -219,7 +270,8 @@ static int32_t buildDataPointSchemas(TAOS_SML_DATA_POINT* points, int numPoint, } } - point->schemaIdx = (uint32_t)stableIdx; + uintptr_t valPointer = (uintptr_t)point; + taosHashPut(info->smlDataToSchema, &valPointer, sizeof(uintptr_t), &stableIdx, sizeof(stableIdx)); } size_t numStables = taosArrayGetSize(stableSchemas); @@ -567,41 +619,6 @@ static int32_t modifyDBSchemas(TAOS* taos, SArray* stableSchemas, SSmlLinesInfo* return 0; } -static int32_t getSmlMd5ChildTableName(TAOS_SML_DATA_POINT* point, char* tableName, int* tableNameLen, - SSmlLinesInfo* info) { - tscDebug("SML:0x%"PRIx64" taos_sml_insert get child table name through md5", info->id); - qsort(point->tags, point->tagNum, sizeof(TAOS_SML_KV), compareSmlColKv); - - SStringBuilder sb; memset(&sb, 0, sizeof(sb)); - char sTableName[TSDB_TABLE_NAME_LEN] = {0}; - strtolower(sTableName, point->stableName); - taosStringBuilderAppendString(&sb, sTableName); - for (int j = 0; j < point->tagNum; ++j) { - taosStringBuilderAppendChar(&sb, ','); - TAOS_SML_KV* tagKv = point->tags + j; - char tagName[TSDB_COL_NAME_LEN] = {0}; - strtolower(tagName, tagKv->key); - taosStringBuilderAppendString(&sb, tagName); - taosStringBuilderAppendChar(&sb, '='); - taosStringBuilderAppend(&sb, tagKv->value, tagKv->length); - } - size_t len = 0; - char* keyJoined = taosStringBuilderGetResult(&sb, &len); - MD5_CTX context; - MD5Init(&context); - MD5Update(&context, (uint8_t *)keyJoined, (uint32_t)len); - MD5Final(&context); - *tableNameLen = snprintf(tableName, *tableNameLen, - "t_%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x", context.digest[0], - context.digest[1], context.digest[2], context.digest[3], context.digest[4], context.digest[5], context.digest[6], - context.digest[7], context.digest[8], context.digest[9], context.digest[10], context.digest[11], - context.digest[12], context.digest[13], context.digest[14], context.digest[15]); - taosStringBuilderDestroy(&sb); - tscDebug("SML:0x%"PRIx64" child table name: %s", info->id, tableName); - return 0; -} - - static int32_t changeChildTableTagValue(TAOS* taos, const char* cTableName, const char* tagName, TAOS_BIND* bind, SSmlLinesInfo* info) { char sql[512]; sprintf(sql, "alter table %s set tag %s=?", cTableName, tagName); @@ -611,25 +628,25 @@ static int32_t changeChildTableTagValue(TAOS* taos, const char* cTableName, cons code = taos_stmt_prepare(stmt, sql, (unsigned long)strlen(sql)); if (code != 0) { - tscError("SML:0x%"PRIx64" %s", info->id, taos_stmt_errstr(stmt)); + tscError("SML:0x%"PRIx64" taos_stmt_prepare return %d:%s", info->id, code, tstrerror(code)); return code; } code = taos_stmt_bind_param(stmt, bind); if (code != 0) { - tscError("SML:0x%"PRIx64" %s", info->id, taos_stmt_errstr(stmt)); + tscError("SML:0x%"PRIx64" taos_stmt_bind_param return %d:%s", info->id, code, tstrerror(code)); return code; } code = taos_stmt_execute(stmt); if (code != 0) { - tscError("SML:0x%"PRIx64" %s", info->id, taos_stmt_errstr(stmt)); + tscError("SML:0x%"PRIx64" taos_stmt_execute return %d:%s", info->id, code, tstrerror(code)); return code; } code = taos_stmt_close(stmt); if (code != 0) { - tscError("SML:0x%"PRIx64" %s", info->id, taos_stmt_errstr(stmt)); + tscError("SML:0x%"PRIx64" taos_stmt_close return %d:%s", info->id, code, tstrerror(code)); return code; } return code; @@ -674,27 +691,27 @@ static int32_t creatChildTableIfNotExists(TAOS* taos, const char* cTableName, co if (code != 0) { tfree(stmt); - tscError("SML:0x%"PRIx64" %s", info->id, taos_stmt_errstr(stmt)); + tscError("SML:0x%"PRIx64" taos_stmt_prepare returns %d:%s", info->id, code, tstrerror(code)); return code; } code = taos_stmt_bind_param(stmt, TARRAY_GET_START(tagsBind)); if (code != 0) { tfree(stmt); - tscError("SML:0x%"PRIx64" %s", info->id, taos_stmt_errstr(stmt)); + tscError("SML:0x%"PRIx64" taos_stmt_bind_param returns %d:%s", info->id, code, tstrerror(code)); return code; } code = taos_stmt_execute(stmt); if (code != 0) { tfree(stmt); - tscError("SML:0x%"PRIx64" %s", info->id, taos_stmt_errstr(stmt)); + tscError("SML:0x%"PRIx64" taos_stmt_execute returns %d:%s", info->id, code, tstrerror(code)); return code; } code = taos_stmt_close(stmt); if (code != 0) { - tscError("SML:0x%"PRIx64" %s", info->id, taos_stmt_errstr(stmt)); + tscError("SML:0x%"PRIx64" taos_stmt_close return %d:%s", info->id, code, tstrerror(code)); return code; } return code; @@ -738,7 +755,7 @@ static int32_t insertChildTableBatch(TAOS* taos, char* cTableName, SArray* cols if (code != 0) { tfree(stmt); - tscError("SML:0x%"PRIx64" %s", info->id, taos_stmt_errstr(stmt)); + tscError("SML:0x%"PRIx64" taos_stmt_prepare return %d:%s", info->id, code, tstrerror(code)); return code; } @@ -746,7 +763,7 @@ static int32_t insertChildTableBatch(TAOS* taos, char* cTableName, SArray* cols code = taos_stmt_set_tbname(stmt, cTableName); if (code != 0) { tfree(stmt); - tscError("SML:0x%"PRIx64" %s", info->id, taos_stmt_errstr(stmt)); + tscError("SML:0x%"PRIx64" taos_stmt_set_tbname return %d:%s", info->id, code, tstrerror(code)); return code; } @@ -756,25 +773,25 @@ static int32_t insertChildTableBatch(TAOS* taos, char* cTableName, SArray* cols code = taos_stmt_bind_param(stmt, colsBinds); if (code != 0) { tfree(stmt); - tscError("SML:0x%"PRIx64" %s", info->id, taos_stmt_errstr(stmt)); + tscError("SML:0x%"PRIx64" taos_stmt_bind_param return %d:%s", info->id, code, tstrerror(code)); return code; } code = taos_stmt_add_batch(stmt); if (code != 0) { tfree(stmt); - tscError("SML:0x%"PRIx64" %s", info->id, taos_stmt_errstr(stmt)); + tscError("SML:0x%"PRIx64" taos_stmt_add_batch return %d:%s", info->id, code, tstrerror(code)); return code; } } code = taos_stmt_execute(stmt); if (code != 0) { - tscError("SML:0x%"PRIx64" %s", info->id, taos_stmt_errstr(stmt)); + tscError("SML:0x%"PRIx64" taos_stmt_execute return %d:%s", info->id, code, tstrerror(code)); } } while (code == TSDB_CODE_TDB_TABLE_RECONFIGURE && try++ < TSDB_MAX_REPLICA); if (code != 0) { - tscError("SML:0x%"PRIx64" %s", info->id, taos_stmt_errstr(stmt)); + tscError("SML:0x%"PRIx64" %d:%s", info->id, code, tstrerror(code)); taos_stmt_close(stmt); } else { taos_stmt_close(stmt); @@ -787,16 +804,10 @@ static int32_t arrangePointsByChildTableName(TAOS_SML_DATA_POINT* points, int nu SHashObj* cname2points, SArray* stableSchemas, SSmlLinesInfo* info) { for (int32_t i = 0; i < numPoints; ++i) { TAOS_SML_DATA_POINT * point = points + i; - if (!point->childTableName) { - char childTableName[TSDB_TABLE_NAME_LEN]; - int32_t tableNameLen = TSDB_TABLE_NAME_LEN; - getSmlMd5ChildTableName(point, childTableName, &tableNameLen, info); - point->childTableName = calloc(1, tableNameLen+1); - strncpy(point->childTableName, childTableName, tableNameLen); - point->childTableName[tableNameLen] = '\0'; - } - - SSmlSTableSchema* stableSchema = taosArrayGet(stableSchemas, point->schemaIdx); + uintptr_t valPointer = (uintptr_t)point; + size_t* pSchemaIndex = taosHashGet(info->smlDataToSchema, &valPointer, sizeof(uintptr_t)); + assert(pSchemaIndex != NULL); + SSmlSTableSchema* stableSchema = taosArrayGet(stableSchemas, *pSchemaIndex); for (int j = 0; j < point->tagNum; ++j) { TAOS_SML_KV* kv = point->tags + j; @@ -840,7 +851,10 @@ static int32_t applyChildTableTags(TAOS* taos, char* cTableName, char* sTableNam TAOS_SML_DATA_POINT * pDataPoint = taosArrayGetP(cTablePoints, i); for (int j = 0; j < pDataPoint->tagNum; ++j) { TAOS_SML_KV* kv = pDataPoint->tags + j; - tagKVs[kv->fieldSchemaIdx] = kv; + uintptr_t valPointer = (uintptr_t)kv; + size_t* pFieldSchemaIdx = taosHashGet(info->smlDataToSchema, &valPointer, sizeof(uintptr_t)); + assert(pFieldSchemaIdx != NULL); + tagKVs[*pFieldSchemaIdx] = kv; } } @@ -863,7 +877,10 @@ static int32_t applyChildTableTags(TAOS* taos, char* cTableName, char* sTableNam for (int j = 0; j < numTags; ++j) { if (tagKVs[j] == NULL) continue; TAOS_SML_KV* kv = tagKVs[j]; - TAOS_BIND* bind = taosArrayGet(tagBinds, kv->fieldSchemaIdx); + uintptr_t valPointer = (uintptr_t)kv; + size_t* pFieldSchemaIdx = taosHashGet(info->smlDataToSchema, &valPointer, sizeof(uintptr_t)); + assert(pFieldSchemaIdx != NULL); + TAOS_BIND* bind = taosArrayGet(tagBinds, *pFieldSchemaIdx); bind->buffer_type = kv->type; bind->length = malloc(sizeof(uintptr_t*)); *bind->length = kv->length; @@ -912,7 +929,10 @@ static int32_t applyChildTableTags(TAOS* taos, char* cTableName, char* sTableNam assert(tagKV->value); if (val == NULL || length != tagKV->length || memcmp(tagKV->value, val, length) != 0) { - TAOS_BIND* bind = taosArrayGet(tagBinds, tagKV->fieldSchemaIdx); + uintptr_t valPointer = (uintptr_t)tagKV; + size_t* pFieldSchemaIdx = taosHashGet(info->smlDataToSchema, &valPointer, sizeof(uintptr_t)); + assert(pFieldSchemaIdx != NULL); + TAOS_BIND* bind = taosArrayGet(tagBinds, *pFieldSchemaIdx); code = changeChildTableTagValue(taos, cTableName, tagKV->key, bind, info); if (code != 0) { tscError("SML:0x%"PRIx64" change child table tag failed. table name %s, tag %s", info->id, cTableName, tagKV->key); @@ -963,7 +983,10 @@ static int32_t applyChildTableFields(TAOS* taos, SSmlSTableSchema* sTableSchema, } for (int j = 0; j < point->fieldNum; ++j) { TAOS_SML_KV* kv = point->fields + j; - TAOS_BIND* bind = colBinds + kv->fieldSchemaIdx; + uintptr_t valPointer = (uintptr_t)kv; + size_t* pFieldSchemaIdx = taosHashGet(info->smlDataToSchema, &valPointer, sizeof(uintptr_t)); + assert(pFieldSchemaIdx != NULL); + TAOS_BIND* bind = colBinds + *pFieldSchemaIdx; bind->buffer_type = kv->type; bind->length = malloc(sizeof(uintptr_t*)); *bind->length = kv->length; @@ -1000,9 +1023,11 @@ static int32_t applyDataPoints(TAOS* taos, TAOS_SML_DATA_POINT* points, int32_t while (pCTablePoints) { SArray* cTablePoints = *pCTablePoints; - TAOS_SML_DATA_POINT* point = taosArrayGetP(cTablePoints, 0); - SSmlSTableSchema* sTableSchema = taosArrayGet(stableSchemas, point->schemaIdx); + uintptr_t valPointer = (uintptr_t)point; + size_t* pSchemaIndex = taosHashGet(info->smlDataToSchema, &valPointer, sizeof(uintptr_t)); + assert(pSchemaIndex != NULL); + SSmlSTableSchema* sTableSchema = taosArrayGet(stableSchemas, *pSchemaIndex); tscDebug("SML:0x%"PRIx64" apply child table tags. child table: %s", info->id, point->childTableName); code = applyChildTableTags(taos, point->childTableName, point->stableName, sTableSchema, cTablePoints, info); @@ -1034,10 +1059,11 @@ cleanup: return code; } -int taos_sml_insert(TAOS* taos, TAOS_SML_DATA_POINT* points, int numPoint, SSmlLinesInfo* info) { +int tscSmlInsert(TAOS* taos, TAOS_SML_DATA_POINT* points, int numPoint, SSmlLinesInfo* info) { tscDebug("SML:0x%"PRIx64" taos_sml_insert. number of points: %d", info->id, numPoint); int32_t code = TSDB_CODE_SUCCESS; + info->smlDataToSchema = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), true, false); tscDebug("SML:0x%"PRIx64" build data point schemas", info->id); SArray* stableSchemas = taosArrayInit(32, sizeof(SSmlSTableSchema)); // SArray @@ -1067,6 +1093,15 @@ clean_up: taosArrayDestroy(schema->tags); } taosArrayDestroy(stableSchemas); + taosHashCleanup(info->smlDataToSchema); + return code; +} + +int taos_sml_insert(TAOS* taos, TAOS_SML_DATA_POINT* points, int numPoint) { + SSmlLinesInfo* info = calloc(1, sizeof(SSmlLinesInfo)); + info->id = genLinesSmlId(); + int code = tscSmlInsert(taos, points, numPoint, info); + free(info); return code; } @@ -2076,18 +2111,6 @@ int32_t tscParseLine(const char* sql, TAOS_SML_DATA_POINT* smlData, SSmlLinesInf //========================================================================= -static uint64_t linesSmlHandleId = 0; - -uint64_t genLinesSmlId() { - uint64_t id; - - do { - id = atomic_add_fetch_64(&linesSmlHandleId, 1); - } while (id == 0); - - return id; -} - void destroySmlDataPoint(TAOS_SML_DATA_POINT* point) { for (int i=0; itagNum; ++i) { free((point->tags+i)->key); @@ -2157,7 +2180,7 @@ int taos_insert_lines(TAOS* taos, char* lines[], int numLines) { } TAOS_SML_DATA_POINT* points = TARRAY_GET_START(lpPoints); - code = taos_sml_insert(taos, points, (int)numPoints, info); + code = tscSmlInsert(taos, points, (int)numPoints, info); if (code != 0) { tscError("SML:0x%"PRIx64" taos_sml_insert error: %s", info->id, tstrerror((code))); } From 2afaf1a0e4ad48f3a1a73a94acae4f19b147aa68 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Fri, 20 Aug 2021 13:11:47 +0800 Subject: [PATCH 02/30] schemaless:multi-thread ignore some errors --- src/client/src/tscParseLineProtocol.c | 25 +++++++++++++++++++++++++ src/inc/taoserror.h | 3 +++ src/mnode/src/mnodeTable.c | 13 +++++++++++++ 3 files changed, 41 insertions(+) diff --git a/src/client/src/tscParseLineProtocol.c b/src/client/src/tscParseLineProtocol.c index 43509d3533..25438ef940 100644 --- a/src/client/src/tscParseLineProtocol.c +++ b/src/client/src/tscParseLineProtocol.c @@ -371,6 +371,11 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInf buildColumnDescription(action->alterSTable.field, result+n, capacity-n, &outBytes); TAOS_RES* res = taos_query(taos, result); //TODO async doAsyncQuery code = taos_errno(res); + if (code == TSDB_CODE_MND_FIELD_ALREAY_EXIST) { + TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); + code = taos_errno(res2); + taos_free_result(res2); + } taos_free_result(res); break; } @@ -380,6 +385,11 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInf result+n, capacity-n, &outBytes); TAOS_RES* res = taos_query(taos, result); //TODO async doAsyncQuery code = taos_errno(res); + if (code == TSDB_CODE_MND_TAG_ALREAY_EXIST) { + TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); + code = taos_errno(res2); + taos_free_result(res2); + } taos_free_result(res); break; } @@ -389,6 +399,11 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInf capacity-n, &outBytes); TAOS_RES* res = taos_query(taos, result); //TODO async doAsyncQuery code = taos_errno(res); + if (code == TSDB_CODE_MND_INVALID_COLUMN_LENGTH) { + TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); + code = taos_errno(res2); + taos_free_result(res2); + } taos_free_result(res); break; } @@ -398,6 +413,11 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInf capacity-n, &outBytes); TAOS_RES* res = taos_query(taos, result); //TODO async doAsyncQuery code = taos_errno(res); + if (code == TSDB_CODE_MND_INVALID_TAG_LENGTH) { + TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); + code = taos_errno(res2); + taos_free_result(res2); + } taos_free_result(res); break; } @@ -427,6 +447,11 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInf outBytes = snprintf(pos, freeBytes, ")"); TAOS_RES* res = taos_query(taos, result); code = taos_errno(res); + if (code == TSDB_CODE_MND_TABLE_ALREADY_EXIST) { + TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); + code = taos_errno(res2); + taos_free_result(res2); + } taos_free_result(res); break; } diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index 368658377c..b8ccada105 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -211,6 +211,9 @@ int32_t* taosGetErrno(); #define TSDB_CODE_DND_TOO_MANY_VNODES TAOS_DEF_ERROR_CODE(0, 0x0405) //"Too many vnode directories") #define TSDB_CODE_DND_EXITING TAOS_DEF_ERROR_CODE(0, 0x0406) //"Dnode is exiting" +#define TSDB_CODE_MND_INVALID_TAG_LENGTH TAOS_DEF_ERROR_CODE(0, 0x410) //"invalid tag length") +#define TSDB_CODE_MND_INVALID_COLUMN_LENGTH TAOS_DEF_ERROR_CODE(0, 0x411) //"invalid column length") + // vnode #define TSDB_CODE_VND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0500) //"Action in progress") #define TSDB_CODE_VND_MSG_NOT_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0501) //"Message not processed") diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index 0bc114ffdf..39a21a6747 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -1519,6 +1519,13 @@ static int32_t mnodeChangeSuperTableColumn(SMnodeMsg *pMsg) { SSchema *schema = (SSchema *) (pStable->schema + col); ASSERT(schema->type == TSDB_DATA_TYPE_BINARY || schema->type == TSDB_DATA_TYPE_NCHAR); schema->bytes = pAlter->schema[0].bytes; + + if (pAlter->schema[0].bytes <= schema->bytes) { + mError("msg:%p, app:%p stable:%s, modify column len. column:%s, len from %d to %d", pMsg, pMsg->rpcMsg.ahandle, + pStable->info.tableId, name, schema->bytes, pAlter->schema[0].bytes); + return TSDB_CODE_MND_INVALID_COLUMN_LENGTH; + } + pStable->sversion++; mInfo("msg:%p, app:%p stable %s, start to modify column %s len to %d", pMsg, pMsg->rpcMsg.ahandle, pStable->info.tableId, name, schema->bytes); @@ -1548,6 +1555,12 @@ static int32_t mnodeChangeSuperTableTag(SMnodeMsg *pMsg) { // update SSchema *schema = (SSchema *) (pStable->schema + col + pStable->numOfColumns); ASSERT(schema->type == TSDB_DATA_TYPE_BINARY || schema->type == TSDB_DATA_TYPE_NCHAR); + if (pAlter->schema[0].bytes <= schema->bytes) { + mError("msg:%p, app:%p stable:%s, modify tag len. tag:%s, len from %d to %d", pMsg, pMsg->rpcMsg.ahandle, + pStable->info.tableId, name, schema->bytes, pAlter->schema[0].bytes); + return TSDB_CODE_MND_INVALID_TAG_LENGTH; + } + schema->bytes = pAlter->schema[0].bytes; pStable->tversion++; mInfo("msg:%p, app:%p stable %s, start to modify tag len %s to %d", pMsg, pMsg->rpcMsg.ahandle, pStable->info.tableId, From a687df67376d80e01fd5ef0cc678842bca6c6c89 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Fri, 20 Aug 2021 15:51:47 +0800 Subject: [PATCH 03/30] save before invalid operation --- src/client/inc/tsclient.h | 1 + src/client/src/tscParseLineProtocol.c | 21 ++++++- src/client/src/tscSQLParser.c | 41 ++++++++++---- src/inc/taoserror.h | 1 + tests/pytest/insert/schemalessInsert.py | 74 ++++++++++++------------- 5 files changed, 87 insertions(+), 51 deletions(-) diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index 4ed7894931..4847a34dd1 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -486,6 +486,7 @@ bool tscHasReachLimitation(SQueryInfo *pQueryInfo, SSqlRes *pRes); void tscSetBoundColumnInfo(SParsedDataColInfo *pColInfo, SSchema *pSchema, int32_t numOfCols); char *tscGetErrorMsgPayload(SSqlCmd *pCmd); +int32_t tscErrorMsgWithCode(int32_t code, char* dstBuffer, const char* errMsg, const char* sql); int32_t tscInvalidOperationMsg(char *msg, const char *additionalInfo, const char *sql); int32_t tscSQLSyntaxErrMsg(char* msg, const char* additionalInfo, const char* sql); diff --git a/src/client/src/tscParseLineProtocol.c b/src/client/src/tscParseLineProtocol.c index 25438ef940..ee63290a3e 100644 --- a/src/client/src/tscParseLineProtocol.c +++ b/src/client/src/tscParseLineProtocol.c @@ -371,7 +371,10 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInf buildColumnDescription(action->alterSTable.field, result+n, capacity-n, &outBytes); TAOS_RES* res = taos_query(taos, result); //TODO async doAsyncQuery code = taos_errno(res); - if (code == TSDB_CODE_MND_FIELD_ALREAY_EXIST) { + if (code != TSDB_CODE_SUCCESS) { + tscError("SML:0x%"PRIx64" apply schema action. error : %s", info->id, taos_errstr(res)); + } + if (code == TSDB_CODE_MND_FIELD_ALREAY_EXIST || code == TSDB_CODE_TSC_DUP_COL_NAMES) { TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); code = taos_errno(res2); taos_free_result(res2); @@ -385,7 +388,10 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInf result+n, capacity-n, &outBytes); TAOS_RES* res = taos_query(taos, result); //TODO async doAsyncQuery code = taos_errno(res); - if (code == TSDB_CODE_MND_TAG_ALREAY_EXIST) { + if (code != TSDB_CODE_SUCCESS) { + tscError("SML:0x%"PRIx64" apply schema action. error : %s", info->id, taos_errstr(res)); + } + if (code == TSDB_CODE_MND_TAG_ALREAY_EXIST || code == TSDB_CODE_TSC_DUP_COL_NAMES) { TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); code = taos_errno(res2); taos_free_result(res2); @@ -399,6 +405,9 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInf capacity-n, &outBytes); TAOS_RES* res = taos_query(taos, result); //TODO async doAsyncQuery code = taos_errno(res); + if (code != TSDB_CODE_SUCCESS) { + tscError("SML:0x%"PRIx64" apply schema action. error : %s", info->id, taos_errstr(res)); + } if (code == TSDB_CODE_MND_INVALID_COLUMN_LENGTH) { TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); code = taos_errno(res2); @@ -413,6 +422,9 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInf capacity-n, &outBytes); TAOS_RES* res = taos_query(taos, result); //TODO async doAsyncQuery code = taos_errno(res); + if (code != TSDB_CODE_SUCCESS) { + tscError("SML:0x%"PRIx64" apply schema action. error : %s", info->id, taos_errstr(res)); + } if (code == TSDB_CODE_MND_INVALID_TAG_LENGTH) { TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); code = taos_errno(res2); @@ -447,6 +459,9 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInf outBytes = snprintf(pos, freeBytes, ")"); TAOS_RES* res = taos_query(taos, result); code = taos_errno(res); + if (code != TSDB_CODE_SUCCESS) { + tscError("SML:0x%"PRIx64" apply schema action. error : %s", info->id, taos_errstr(res)); + } if (code == TSDB_CODE_MND_TABLE_ALREADY_EXIST) { TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); code = taos_errno(res2); @@ -462,7 +477,7 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInf free(result); if (code != 0) { - tscError("SML:0x%"PRIx64 "apply schema action failure. %s", info->id, tstrerror(code)); + tscError("SML:0x%"PRIx64 " apply schema action failure. %s", info->id, tstrerror(code)); } return code; } diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 612a3d4798..2699ad2eb6 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -307,6 +307,31 @@ static int32_t invalidOperationMsg(char* dstBuffer, const char* errMsg) { return tscInvalidOperationMsg(dstBuffer, errMsg, NULL); } +int32_t tscErrorMsgWithCode(int32_t code, char* dstBuffer, const char* errMsg, const char* sql) { + const char* msgFormat1 = "%s:%s"; + const char* msgFormat2 = "%s:\'%s\' (%s)"; + const char* msgFormat3 = "%s:\'%s\'"; + + const int32_t BACKWARD_CHAR_STEP = 0; + + if (sql == NULL) { + assert(errMsg != NULL); + sprintf(dstBuffer, msgFormat1, tstrerror(code), errMsg); + return code; + } + + char buf[64] = {0}; // only extract part of sql string + strncpy(buf, (sql - BACKWARD_CHAR_STEP), tListLen(buf) - 1); + + if (errMsg != NULL) { + sprintf(dstBuffer, msgFormat2, tstrerror(code), buf, errMsg); + } else { + sprintf(dstBuffer, msgFormat3, tstrerror(code), buf); // no additional information for invalid sql error + } + + return code; +} + static int convertTimestampStrToInt64(tVariant *pVar, int32_t precision) { int64_t time = 0; strdequote(pVar->pz); @@ -1393,7 +1418,6 @@ static bool validateTableColumnInfo(SArray* pFieldList, SSqlCmd* pCmd) { const char* msg = "illegal number of columns"; const char* msg1 = "first column must be timestamp"; const char* msg2 = "row length exceeds max length"; - const char* msg3 = "duplicated column names"; const char* msg4 = "invalid data type"; const char* msg5 = "invalid binary/nchar column length"; const char* msg6 = "invalid column name"; @@ -1442,7 +1466,7 @@ static bool validateTableColumnInfo(SArray* pFieldList, SSqlCmd* pCmd) { // field name must be unique if (has(pFieldList, i + 1, pField->name) == true) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); + tscErrorMsgWithCode(TSDB_CODE_TSC_DUP_COL_NAMES, tscGetErrorMsgPayload(pCmd), pField->name, NULL); return false; } @@ -1464,8 +1488,6 @@ static bool validateTagParams(SArray* pTagsList, SArray* pFieldList, SSqlCmd* pC const char* msg1 = "invalid number of tag columns"; const char* msg2 = "tag length too long"; - const char* msg3 = "duplicated column names"; - //const char* msg4 = "timestamp not allowed in tags"; const char* msg5 = "invalid data type in tags"; const char* msg6 = "invalid tag name"; const char* msg7 = "invalid binary/nchar tag length"; @@ -1496,7 +1518,7 @@ static bool validateTagParams(SArray* pTagsList, SArray* pFieldList, SSqlCmd* pC } if (has(pTagsList, i + 1, p->name) == true) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); + tscErrorMsgWithCode(TSDB_CODE_TSC_DUP_COL_NAMES, tscGetErrorMsgPayload(pCmd), p->name, NULL); return false; } } @@ -1523,7 +1545,7 @@ static bool validateTagParams(SArray* pTagsList, SArray* pFieldList, SSqlCmd* pC TAOS_FIELD* p = taosArrayGet(pTagsList, i); if (has(pFieldList, 0, p->name) == true) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); + tscErrorMsgWithCode(TSDB_CODE_TSC_DUP_COL_NAMES, tscGetErrorMsgPayload(pCmd), p->name, NULL); return false; } } @@ -1535,8 +1557,6 @@ static bool validateTagParams(SArray* pTagsList, SArray* pFieldList, SSqlCmd* pC * tags name /column name is truncated in sql.y */ bool validateOneTags(SSqlCmd* pCmd, TAOS_FIELD* pTagField) { - //const char* msg1 = "timestamp not allowed in tags"; - const char* msg2 = "duplicated column names"; const char* msg3 = "tag length too long"; const char* msg4 = "invalid tag name"; const char* msg5 = "invalid binary/nchar tag length"; @@ -1605,7 +1625,7 @@ bool validateOneTags(SSqlCmd* pCmd, TAOS_FIELD* pTagField) { for (int32_t i = 0; i < numOfTags + numOfCols; ++i) { if (strncasecmp(pTagField->name, pSchema[i].name, sizeof(pTagField->name) - 1) == 0) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); + tscErrorMsgWithCode(TSDB_CODE_TSC_DUP_COL_NAMES, tscGetErrorMsgPayload(pCmd), pTagField->name, NULL); return false; } } @@ -1615,7 +1635,6 @@ bool validateOneTags(SSqlCmd* pCmd, TAOS_FIELD* pTagField) { bool validateOneColumn(SSqlCmd* pCmd, TAOS_FIELD* pColField) { const char* msg1 = "too many columns"; - const char* msg2 = "duplicated column names"; const char* msg3 = "column length too long"; const char* msg4 = "invalid data type"; const char* msg5 = "invalid column name"; @@ -1665,7 +1684,7 @@ bool validateOneColumn(SSqlCmd* pCmd, TAOS_FIELD* pColField) { // field name must be unique for (int32_t i = 0; i < numOfTags + numOfCols; ++i) { if (strncasecmp(pColField->name, pSchema[i].name, sizeof(pColField->name) - 1) == 0) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); + tscErrorMsgWithCode(TSDB_CODE_TSC_DUP_COL_NAMES, tscGetErrorMsgPayload(pCmd), pColField->name, NULL); return false; } } diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index b8ccada105..f54150d76b 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -103,6 +103,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_TSC_FILE_EMPTY TAOS_DEF_ERROR_CODE(0, 0x021A) //"File is empty") #define TSDB_CODE_TSC_LINE_SYNTAX_ERROR TAOS_DEF_ERROR_CODE(0, 0x021B) //"Syntax error in Line") #define TSDB_CODE_TSC_NO_META_CACHED TAOS_DEF_ERROR_CODE(0, 0x021C) //"No table meta cached") +#define TSDB_CODE_TSC_DUP_COL_NAMES TAOS_DEF_ERROR_CODE(0, 0x021D) //"duplicated column names") // mnode #define TSDB_CODE_MND_MSG_NOT_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0300) //"Message not processed") diff --git a/tests/pytest/insert/schemalessInsert.py b/tests/pytest/insert/schemalessInsert.py index 49c3223588..00e805d791 100644 --- a/tests/pytest/insert/schemalessInsert.py +++ b/tests/pytest/insert/schemalessInsert.py @@ -1261,52 +1261,52 @@ class TDTestCase: # self._conn.insert_lines([input_sql4]) def runAll(self): - self.initCheckCase() - self.boolTypeCheckCase() - self.symbolsCheckCase() - self.tsCheckCase() - self.idSeqCheckCase() - self.idUpperCheckCase() - self.noIdCheckCase() - self.maxColTagCheckCase() - self.idIllegalNameCheckCase() - self.idStartWithNumCheckCase() - self.nowTsCheckCase() - self.dateFormatTsCheckCase() - self.illegalTsCheckCase() - self.tagValueLengthCheckCase() - self.colValueLengthCheckCase() - self.tagColIllegalValueCheckCase() - self.duplicateIdTagColInsertCheckCase() - self.noIdStbExistCheckCase() - self.duplicateInsertExistCheckCase() - self.tagColBinaryNcharLengthCheckCase() - self.tagColAddDupIDCheckCase() - self.tagColAddCheckCase() - self.tagMd5Check() - self.tagColBinaryMaxLengthCheckCase() - # self.tagColNcharMaxLengthCheckCase() - self.batchInsertCheckCase() - self.multiInsertCheckCase(1000) - self.batchErrorInsertCheckCase() - # MultiThreads - self.stbInsertMultiThreadCheckCase() - self.sStbStbDdataInsertMultiThreadCheckCase() - self.sStbStbDdataAtcInsertMultiThreadCheckCase() - self.sStbStbDdataMtcInsertMultiThreadCheckCase() - self.sStbDtbDdataInsertMultiThreadCheckCase() + # self.initCheckCase() + # self.boolTypeCheckCase() + # self.symbolsCheckCase() + # self.tsCheckCase() + # self.idSeqCheckCase() + # self.idUpperCheckCase() + # self.noIdCheckCase() + # self.maxColTagCheckCase() + # self.idIllegalNameCheckCase() + # self.idStartWithNumCheckCase() + # self.nowTsCheckCase() + # self.dateFormatTsCheckCase() + # self.illegalTsCheckCase() + # self.tagValueLengthCheckCase() + # self.colValueLengthCheckCase() + # self.tagColIllegalValueCheckCase() + # self.duplicateIdTagColInsertCheckCase() + # self.noIdStbExistCheckCase() + # self.duplicateInsertExistCheckCase() + # self.tagColBinaryNcharLengthCheckCase() + # self.tagColAddDupIDCheckCase() + # self.tagColAddCheckCase() + # self.tagMd5Check() + # self.tagColBinaryMaxLengthCheckCase() + # # self.tagColNcharMaxLengthCheckCase() + # self.batchInsertCheckCase() + # self.multiInsertCheckCase(1000) + # self.batchErrorInsertCheckCase() + # # MultiThreads + # self.stbInsertMultiThreadCheckCase() + # self.sStbStbDdataInsertMultiThreadCheckCase() + # self.sStbStbDdataAtcInsertMultiThreadCheckCase() + # self.sStbStbDdataMtcInsertMultiThreadCheckCase() + # self.sStbDtbDdataInsertMultiThreadCheckCase() # # ! concurrency conflict - # self.sStbDtbDdataAcMtInsertMultiThreadCheckCase() + self.sStbDtbDdataAcMtInsertMultiThreadCheckCase() # self.sStbDtbDdataAtMcInsertMultiThreadCheckCase() - self.sStbStbDdataDtsInsertMultiThreadCheckCase() + # self.sStbStbDdataDtsInsertMultiThreadCheckCase() # # ! concurrency conflict # self.sStbStbDdataDtsAcMtInsertMultiThreadCheckCase() # self.sStbStbDdataDtsAtMcInsertMultiThreadCheckCase() - self.sStbDtbDdataDtsInsertMultiThreadCheckCase() + # self.sStbDtbDdataDtsInsertMultiThreadCheckCase() # ! concurrency conflict # self.sStbDtbDdataDtsAcMtInsertMultiThreadCheckCase() From 559dcff66ec77422cf33744f3c2d4aaf63fae62e Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Fri, 20 Aug 2021 17:45:58 +0800 Subject: [PATCH 04/30] schemaless:before retrying loading meta --- src/client/src/tscSQLParser.c | 95 +++++++++++------------------------ src/client/src/tscUtil.c | 25 +++++++++ 2 files changed, 55 insertions(+), 65 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 2699ad2eb6..64f88b373d 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -116,7 +116,7 @@ static int32_t validateColumnName(char* name); static int32_t setKillInfo(SSqlObj* pSql, struct SSqlInfo* pInfo, int32_t killType); static int32_t setCompactVnodeInfo(SSqlObj* pSql, struct SSqlInfo* pInfo); -static bool validateOneTags(SSqlCmd* pCmd, TAOS_FIELD* pTagField); +static int32_t validateOneTag(SSqlCmd* pCmd, TAOS_FIELD* pTagField); static bool hasTimestampForPointInterpQuery(SQueryInfo* pQueryInfo); static bool hasNormalColumnFilter(SQueryInfo* pQueryInfo); @@ -307,31 +307,6 @@ static int32_t invalidOperationMsg(char* dstBuffer, const char* errMsg) { return tscInvalidOperationMsg(dstBuffer, errMsg, NULL); } -int32_t tscErrorMsgWithCode(int32_t code, char* dstBuffer, const char* errMsg, const char* sql) { - const char* msgFormat1 = "%s:%s"; - const char* msgFormat2 = "%s:\'%s\' (%s)"; - const char* msgFormat3 = "%s:\'%s\'"; - - const int32_t BACKWARD_CHAR_STEP = 0; - - if (sql == NULL) { - assert(errMsg != NULL); - sprintf(dstBuffer, msgFormat1, tstrerror(code), errMsg); - return code; - } - - char buf[64] = {0}; // only extract part of sql string - strncpy(buf, (sql - BACKWARD_CHAR_STEP), tListLen(buf) - 1); - - if (errMsg != NULL) { - sprintf(dstBuffer, msgFormat2, tstrerror(code), buf, errMsg); - } else { - sprintf(dstBuffer, msgFormat3, tstrerror(code), buf); // no additional information for invalid sql error - } - - return code; -} - static int convertTimestampStrToInt64(tVariant *pVar, int32_t precision) { int64_t time = 0; strdequote(pVar->pz); @@ -1418,6 +1393,7 @@ static bool validateTableColumnInfo(SArray* pFieldList, SSqlCmd* pCmd) { const char* msg = "illegal number of columns"; const char* msg1 = "first column must be timestamp"; const char* msg2 = "row length exceeds max length"; + const char* msg3 = "duplicated column names"; const char* msg4 = "invalid data type"; const char* msg5 = "invalid binary/nchar column length"; const char* msg6 = "invalid column name"; @@ -1466,7 +1442,7 @@ static bool validateTableColumnInfo(SArray* pFieldList, SSqlCmd* pCmd) { // field name must be unique if (has(pFieldList, i + 1, pField->name) == true) { - tscErrorMsgWithCode(TSDB_CODE_TSC_DUP_COL_NAMES, tscGetErrorMsgPayload(pCmd), pField->name, NULL); + invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); return false; } @@ -1488,6 +1464,8 @@ static bool validateTagParams(SArray* pTagsList, SArray* pFieldList, SSqlCmd* pC const char* msg1 = "invalid number of tag columns"; const char* msg2 = "tag length too long"; + const char* msg3 = "duplicated column names"; + //const char* msg4 = "timestamp not allowed in tags"; const char* msg5 = "invalid data type in tags"; const char* msg6 = "invalid tag name"; const char* msg7 = "invalid binary/nchar tag length"; @@ -1518,7 +1496,7 @@ static bool validateTagParams(SArray* pTagsList, SArray* pFieldList, SSqlCmd* pC } if (has(pTagsList, i + 1, p->name) == true) { - tscErrorMsgWithCode(TSDB_CODE_TSC_DUP_COL_NAMES, tscGetErrorMsgPayload(pCmd), p->name, NULL); + invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); return false; } } @@ -1545,7 +1523,7 @@ static bool validateTagParams(SArray* pTagsList, SArray* pFieldList, SSqlCmd* pC TAOS_FIELD* p = taosArrayGet(pTagsList, i); if (has(pFieldList, 0, p->name) == true) { - tscErrorMsgWithCode(TSDB_CODE_TSC_DUP_COL_NAMES, tscGetErrorMsgPayload(pCmd), p->name, NULL); + invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); return false; } } @@ -1556,7 +1534,7 @@ static bool validateTagParams(SArray* pTagsList, SArray* pFieldList, SSqlCmd* pC /* * tags name /column name is truncated in sql.y */ -bool validateOneTags(SSqlCmd* pCmd, TAOS_FIELD* pTagField) { +int32_t validateOneTag(SSqlCmd* pCmd, TAOS_FIELD* pTagField) { const char* msg3 = "tag length too long"; const char* msg4 = "invalid tag name"; const char* msg5 = "invalid binary/nchar tag length"; @@ -1571,8 +1549,7 @@ bool validateOneTags(SSqlCmd* pCmd, TAOS_FIELD* pTagField) { // no more max columns if (numOfTags + numOfCols >= TSDB_MAX_COLUMNS) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg7); - return false; + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg7); } // no more than 6 tags @@ -1580,8 +1557,7 @@ bool validateOneTags(SSqlCmd* pCmd, TAOS_FIELD* pTagField) { char msg[128] = {0}; sprintf(msg, "tags no more than %d", TSDB_MAX_TAGS); - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg); - return false; + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg); } // no timestamp allowable @@ -1591,8 +1567,7 @@ bool validateOneTags(SSqlCmd* pCmd, TAOS_FIELD* pTagField) { //} if ((pTagField->type < TSDB_DATA_TYPE_BOOL) || (pTagField->type > TSDB_DATA_TYPE_UBIGINT)) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6); - return false; + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6); } SSchema* pTagSchema = tscGetTableTagSchema(pTableMetaInfo->pTableMeta); @@ -1604,20 +1579,17 @@ bool validateOneTags(SSqlCmd* pCmd, TAOS_FIELD* pTagField) { // length less than TSDB_MAX_TASG_LEN if (nLen + pTagField->bytes > TSDB_MAX_TAGS_LEN) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - return false; + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); } // tags name can not be a keyword if (validateColumnName(pTagField->name) != TSDB_CODE_SUCCESS) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); - return false; + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); } // binary(val), val can not be equalled to or less than 0 if ((pTagField->type == TSDB_DATA_TYPE_BINARY || pTagField->type == TSDB_DATA_TYPE_NCHAR) && pTagField->bytes <= 0) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5); - return false; + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5); } // field name must be unique @@ -1625,15 +1597,14 @@ bool validateOneTags(SSqlCmd* pCmd, TAOS_FIELD* pTagField) { for (int32_t i = 0; i < numOfTags + numOfCols; ++i) { if (strncasecmp(pTagField->name, pSchema[i].name, sizeof(pTagField->name) - 1) == 0) { - tscErrorMsgWithCode(TSDB_CODE_TSC_DUP_COL_NAMES, tscGetErrorMsgPayload(pCmd), pTagField->name, NULL); - return false; + return tscErrorMsgWithCode(TSDB_CODE_TSC_DUP_COL_NAMES, tscGetErrorMsgPayload(pCmd), pTagField->name, NULL); } } - return true; + return TSDB_CODE_SUCCESS; } -bool validateOneColumn(SSqlCmd* pCmd, TAOS_FIELD* pColField) { +int32_t validateOneColumn(SSqlCmd* pCmd, TAOS_FIELD* pColField) { const char* msg1 = "too many columns"; const char* msg3 = "column length too long"; const char* msg4 = "invalid data type"; @@ -1649,18 +1620,15 @@ bool validateOneColumn(SSqlCmd* pCmd, TAOS_FIELD* pColField) { // no more max columns if (numOfCols >= TSDB_MAX_COLUMNS || numOfTags + numOfCols >= TSDB_MAX_COLUMNS) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - return false; + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); } if (pColField->type < TSDB_DATA_TYPE_BOOL || pColField->type > TSDB_DATA_TYPE_UBIGINT) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); - return false; + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); } if (validateColumnName(pColField->name) != TSDB_CODE_SUCCESS) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5); - return false; + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5); } SSchema* pSchema = tscGetTableSchema(pTableMeta); @@ -1671,25 +1639,22 @@ bool validateOneColumn(SSqlCmd* pCmd, TAOS_FIELD* pColField) { } if (pColField->bytes <= 0) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6); - return false; + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6); } // length less than TSDB_MAX_BYTES_PER_ROW if (nLen + pColField->bytes > TSDB_MAX_BYTES_PER_ROW) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - return false; + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); } // field name must be unique for (int32_t i = 0; i < numOfTags + numOfCols; ++i) { if (strncasecmp(pColField->name, pSchema[i].name, sizeof(pColField->name) - 1) == 0) { - tscErrorMsgWithCode(TSDB_CODE_TSC_DUP_COL_NAMES, tscGetErrorMsgPayload(pCmd), pColField->name, NULL); - return false; + return tscErrorMsgWithCode(TSDB_CODE_TSC_DUP_COL_NAMES, tscGetErrorMsgPayload(pCmd), pColField->name, NULL); } } - return true; + return TSDB_CODE_SUCCESS; } /* is contained in pFieldList or not */ @@ -6108,8 +6073,9 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { } TAOS_FIELD* p = taosArrayGet(pFieldList, 0); - if (!validateOneTags(pCmd, p)) { - return TSDB_CODE_TSC_INVALID_OPERATION; + int32_t ret = validateOneTag(pCmd, p); + if (ret != TSDB_CODE_SUCCESS) { + return ret; } tscFieldInfoAppend(&pQueryInfo->fieldsInfo, p); @@ -6286,8 +6252,9 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { } TAOS_FIELD* p = taosArrayGet(pFieldList, 0); - if (!validateOneColumn(pCmd, p)) { - return TSDB_CODE_TSC_INVALID_OPERATION; + int32_t ret = validateOneColumn(pCmd, p); + if (ret != TSDB_CODE_SUCCESS) { + return ret; } tscFieldInfoAppend(&pQueryInfo->fieldsInfo, p); @@ -8702,8 +8669,6 @@ static STableMeta* extractTempTableMetaFromSubquery(SQueryInfo* pUpstream) { n += 1; } - info->numOfColumns = n; - return meta; } diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 591a6bba34..583f50aecd 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -4096,6 +4096,31 @@ int32_t tscInvalidOperationMsg(char* msg, const char* additionalInfo, const char return TSDB_CODE_TSC_INVALID_OPERATION; } +int32_t tscErrorMsgWithCode(int32_t code, char* dstBuffer, const char* errMsg, const char* sql) { + const char* msgFormat1 = "%s:%s"; + const char* msgFormat2 = "%s:\'%s\' (%s)"; + const char* msgFormat3 = "%s:\'%s\'"; + + const int32_t BACKWARD_CHAR_STEP = 0; + + if (sql == NULL) { + assert(errMsg != NULL); + sprintf(dstBuffer, msgFormat1, tstrerror(code), errMsg); + return code; + } + + char buf[64] = {0}; // only extract part of sql string + strncpy(buf, (sql - BACKWARD_CHAR_STEP), tListLen(buf) - 1); + + if (errMsg != NULL) { + sprintf(dstBuffer, msgFormat2, tstrerror(code), buf, errMsg); + } else { + sprintf(dstBuffer, msgFormat3, tstrerror(code), buf); // no additional information for invalid sql error + } + + return code; +} + bool tscHasReachLimitation(SQueryInfo* pQueryInfo, SSqlRes* pRes) { assert(pQueryInfo != NULL && pQueryInfo->clauseLimit != 0); return (pQueryInfo->clauseLimit > 0 && pRes->numOfClauseTotal >= pQueryInfo->clauseLimit); From f26174745865e2300e4b133a3f4df2d6f32cf03a Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Fri, 20 Aug 2021 21:57:30 +0800 Subject: [PATCH 05/30] schemaless: multi-threaded cases runw --- src/client/src/tscParseLineProtocol.c | 183 +++++++++++++++--------- src/mnode/src/mnodeTable.c | 4 +- tests/pytest/insert/schemalessInsert.py | 26 ++-- 3 files changed, 129 insertions(+), 84 deletions(-) diff --git a/src/client/src/tscParseLineProtocol.c b/src/client/src/tscParseLineProtocol.c index ee63290a3e..03e361c950 100644 --- a/src/client/src/tscParseLineProtocol.c +++ b/src/client/src/tscParseLineProtocol.c @@ -372,14 +372,18 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInf TAOS_RES* res = taos_query(taos, result); //TODO async doAsyncQuery code = taos_errno(res); if (code != TSDB_CODE_SUCCESS) { - tscError("SML:0x%"PRIx64" apply schema action. error : %s", info->id, taos_errstr(res)); + tscError("SML:0x%"PRIx64" apply schema action. error: %s", info->id, taos_errstr(res)); } + taos_free_result(res); + if (code == TSDB_CODE_MND_FIELD_ALREAY_EXIST || code == TSDB_CODE_TSC_DUP_COL_NAMES) { TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); code = taos_errno(res2); + if (code != TSDB_CODE_SUCCESS) { + tscError("SML:0x%" PRIx64 " apply schema action. reset query cache. error: %s", info->id, taos_errstr(res2)); + } taos_free_result(res2); } - taos_free_result(res); break; } case SCHEMA_ACTION_ADD_TAG: { @@ -391,12 +395,16 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInf if (code != TSDB_CODE_SUCCESS) { tscError("SML:0x%"PRIx64" apply schema action. error : %s", info->id, taos_errstr(res)); } + taos_free_result(res); + if (code == TSDB_CODE_MND_TAG_ALREAY_EXIST || code == TSDB_CODE_TSC_DUP_COL_NAMES) { TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); code = taos_errno(res2); + if (code != TSDB_CODE_SUCCESS) { + tscError("SML:0x%" PRIx64 " apply schema action. reset query cache. error: %s", info->id, taos_errstr(res2)); + } taos_free_result(res2); } - taos_free_result(res); break; } case SCHEMA_ACTION_CHANGE_COLUMN_SIZE: { @@ -408,12 +416,16 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInf if (code != TSDB_CODE_SUCCESS) { tscError("SML:0x%"PRIx64" apply schema action. error : %s", info->id, taos_errstr(res)); } + taos_free_result(res); + if (code == TSDB_CODE_MND_INVALID_COLUMN_LENGTH) { TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); code = taos_errno(res2); + if (code != TSDB_CODE_SUCCESS) { + tscError("SML:0x%" PRIx64 " apply schema action. reset query cache. error: %s", info->id, taos_errstr(res2)); + } taos_free_result(res2); } - taos_free_result(res); break; } case SCHEMA_ACTION_CHANGE_TAG_SIZE: { @@ -425,12 +437,16 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInf if (code != TSDB_CODE_SUCCESS) { tscError("SML:0x%"PRIx64" apply schema action. error : %s", info->id, taos_errstr(res)); } + taos_free_result(res); + if (code == TSDB_CODE_MND_INVALID_TAG_LENGTH) { TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); code = taos_errno(res2); + if (code != TSDB_CODE_SUCCESS) { + tscError("SML:0x%" PRIx64 " apply schema action. reset query cache. error: %s", info->id, taos_errstr(res2)); + } taos_free_result(res2); } - taos_free_result(res); break; } case SCHEMA_ACTION_CREATE_STABLE: { @@ -462,12 +478,16 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInf if (code != TSDB_CODE_SUCCESS) { tscError("SML:0x%"PRIx64" apply schema action. error : %s", info->id, taos_errstr(res)); } + taos_free_result(res); + if (code == TSDB_CODE_MND_TABLE_ALREADY_EXIST) { TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); code = taos_errno(res2); + if (code != TSDB_CODE_SUCCESS) { + tscError("SML:0x%" PRIx64 " apply schema action. reset query cache. error: %s", info->id, taos_errstr(res2)); + } taos_free_result(res2); } - taos_free_result(res); break; } @@ -490,70 +510,12 @@ static int32_t destroySmlSTableSchema(SSmlSTableSchema* schema) { return 0; } -int32_t loadTableMeta(TAOS* taos, char* tableName, SSmlSTableSchema* schema, SSmlLinesInfo* info) { - int32_t code = 0; - - STscObj *pObj = (STscObj *)taos; - if (pObj == NULL || pObj->signature != pObj) { - terrno = TSDB_CODE_TSC_DISCONNECTED; - return TSDB_CODE_TSC_DISCONNECTED; - } - - tscDebug("SML:0x%"PRIx64" load table schema. super table name: %s", info->id, tableName); - - char tableNameLowerCase[TSDB_TABLE_NAME_LEN]; - strtolower(tableNameLowerCase, tableName); - - char sql[256]; - snprintf(sql, 256, "describe %s", tableNameLowerCase); - TAOS_RES* res = taos_query(taos, sql); - code = taos_errno(res); - if (code != 0) { - tscError("SML:0x%"PRIx64" describe table failure. %s", info->id, taos_errstr(res)); - taos_free_result(res); - return code; - } - taos_free_result(res); - - SSqlObj* pSql = calloc(1, sizeof(SSqlObj)); - if (pSql == NULL){ - tscError("failed to allocate memory, reason:%s", strerror(errno)); - code = TSDB_CODE_TSC_OUT_OF_MEMORY; - return code; - } - pSql->pTscObj = taos; - pSql->signature = pSql; - pSql->fp = NULL; - - SStrToken tableToken = {.z=tableNameLowerCase, .n=(uint32_t)strlen(tableNameLowerCase), .type=TK_ID}; - tGetToken(tableNameLowerCase, &tableToken.type); - // Check if the table name available or not - if (tscValidateName(&tableToken) != TSDB_CODE_SUCCESS) { - code = TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; - sprintf(pSql->cmd.payload, "table name is invalid"); - tscFreeSqlObj(pSql); - return code; - } - - SName sname = {0}; - if ((code = tscSetTableFullName(&sname, &tableToken, pSql)) != TSDB_CODE_SUCCESS) { - tscFreeSqlObj(pSql); - return code; - } - char fullTableName[TSDB_TABLE_FNAME_LEN] = {0}; - memset(fullTableName, 0, tListLen(fullTableName)); - tNameExtractFullName(&sname, fullTableName); - tscFreeSqlObj(pSql); - +static int32_t fillDbSchema(STableMeta* tableMeta, char* tableName, SSmlSTableSchema* schema, SSmlLinesInfo* info) { schema->tags = taosArrayInit(8, sizeof(SSchema)); schema->fields = taosArrayInit(64, sizeof(SSchema)); schema->tagHash = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); schema->fieldHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); - size_t size = 0; - STableMeta* tableMeta = NULL; - taosHashGetCloneExt(tscTableMetaMap, fullTableName, strlen(fullTableName), NULL, (void **)&tableMeta, &size); - tstrncpy(schema->sTableName, tableName, strlen(tableName)+1); schema->precision = tableMeta->tableInfo.precision; for (int i=0; itableInfo.numOfColumns; ++i) { @@ -576,9 +538,92 @@ int32_t loadTableMeta(TAOS* taos, char* tableName, SSmlSTableSchema* schema, SSm size_t tagIndex = taosArrayGetSize(schema->tags) - 1; taosHashPut(schema->tagHash, field.name, strlen(field.name), &tagIndex, sizeof(tagIndex)); } - tscDebug("SML:0x%"PRIx64 " load table meta succeed. table name: %s, columns number: %d, tag number: %d, precision: %d", + tscDebug("SML:0x%"PRIx64 " load table schema succeed. table name: %s, columns number: %d, tag number: %d, precision: %d", info->id, tableName, tableMeta->tableInfo.numOfColumns, tableMeta->tableInfo.numOfTags, schema->precision); - free(tableMeta); tableMeta = NULL; + return TSDB_CODE_SUCCESS; +} + +static int32_t retrieveTableMeta(TAOS* taos, char* tableName, STableMeta** pTableMeta, SSmlLinesInfo* info) { + int32_t code = 0; + int32_t retries = 0; + STableMeta* tableMeta = NULL; + while (retries++ < TSDB_MAX_REPLICA && tableMeta == NULL) { + STscObj* pObj = (STscObj*)taos; + if (pObj == NULL || pObj->signature != pObj) { + terrno = TSDB_CODE_TSC_DISCONNECTED; + return TSDB_CODE_TSC_DISCONNECTED; + } + + tscDebug("SML:0x%" PRIx64 " retrieve table meta. super table name: %s", info->id, tableName); + + char tableNameLowerCase[TSDB_TABLE_NAME_LEN]; + strtolower(tableNameLowerCase, tableName); + + char sql[256]; + snprintf(sql, 256, "describe %s", tableNameLowerCase); + TAOS_RES* res = taos_query(taos, sql); + code = taos_errno(res); + if (code != 0) { + tscError("SML:0x%" PRIx64 " describe table failure. %s", info->id, taos_errstr(res)); + taos_free_result(res); + return code; + } + taos_free_result(res); + + SSqlObj* pSql = calloc(1, sizeof(SSqlObj)); + if (pSql == NULL) { + tscError("SML:0x%" PRIx64 " failed to allocate memory, reason:%s", info->id, strerror(errno)); + code = TSDB_CODE_TSC_OUT_OF_MEMORY; + return code; + } + pSql->pTscObj = taos; + pSql->signature = pSql; + pSql->fp = NULL; + + SStrToken tableToken = {.z = tableNameLowerCase, .n = (uint32_t)strlen(tableNameLowerCase), .type = TK_ID}; + tGetToken(tableNameLowerCase, &tableToken.type); + // Check if the table name available or not + if (tscValidateName(&tableToken) != TSDB_CODE_SUCCESS) { + code = TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; + sprintf(pSql->cmd.payload, "table name is invalid"); + tscFreeSqlObj(pSql); + return code; + } + + SName sname = {0}; + if ((code = tscSetTableFullName(&sname, &tableToken, pSql)) != TSDB_CODE_SUCCESS) { + tscFreeSqlObj(pSql); + return code; + } + char fullTableName[TSDB_TABLE_FNAME_LEN] = {0}; + memset(fullTableName, 0, tListLen(fullTableName)); + tNameExtractFullName(&sname, fullTableName); + tscFreeSqlObj(pSql); + + size_t size = 0; + taosHashGetCloneExt(tscTableMetaMap, fullTableName, strlen(fullTableName), NULL, (void**)&tableMeta, &size); + } + + if (tableMeta != NULL) { + *pTableMeta = tableMeta; + return TSDB_CODE_SUCCESS; + } else { + tscError("SML:0x%" PRIx64 " failed to retrieve table meta. super table name: %s", info->id, tableName); + return TSDB_CODE_TSC_NO_META_CACHED; + } +} + +static int32_t loadTableSchemaFromDB(TAOS* taos, char* tableName, SSmlSTableSchema* schema, SSmlLinesInfo* info) { + int32_t code = 0; + STableMeta* tableMeta = NULL; + code = retrieveTableMeta(taos, tableName, &tableMeta, info); + if (code == TSDB_CODE_SUCCESS) { + assert(tableMeta != NULL); + fillDbSchema(tableMeta, tableName, schema, info); + free(tableMeta); + tableMeta = NULL; + } + return code; } @@ -590,7 +635,7 @@ static int32_t modifyDBSchemas(TAOS* taos, SArray* stableSchemas, SSmlLinesInfo* SSmlSTableSchema dbSchema; memset(&dbSchema, 0, sizeof(SSmlSTableSchema)); - code = loadTableMeta(taos, pointSchema->sTableName, &dbSchema, info); + code = loadTableSchemaFromDB(taos, pointSchema->sTableName, &dbSchema, info); if (code == TSDB_CODE_MND_INVALID_TABLE_NAME) { SSchemaAction schemaAction = {0}; schemaAction.action = SCHEMA_ACTION_CREATE_STABLE; @@ -599,7 +644,7 @@ static int32_t modifyDBSchemas(TAOS* taos, SArray* stableSchemas, SSmlLinesInfo* schemaAction.createSTable.tags = pointSchema->tags; schemaAction.createSTable.fields = pointSchema->fields; applySchemaAction(taos, &schemaAction, info); - code = loadTableMeta(taos, pointSchema->sTableName, &dbSchema, info); + code = loadTableSchemaFromDB(taos, pointSchema->sTableName, &dbSchema, info); if (code != 0) { tscError("SML:0x%"PRIx64" reconcile point schema failed. can not create %s", info->id, pointSchema->sTableName); return code; diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index 39a21a6747..1bc5607da5 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -1246,13 +1246,13 @@ static int32_t mnodeAddSuperTableTag(SMnodeMsg *pMsg, SSchema schema[], int32_t if (mnodeFindSuperTableColumnIndex(pStable, schema[i].name) > 0) { mError("msg:%p, app:%p stable:%s, add tag, column:%s already exist", pMsg, pMsg->rpcMsg.ahandle, pStable->info.tableId, schema[i].name); - return TSDB_CODE_MND_TAG_ALREAY_EXIST; + return TSDB_CODE_MND_FIELD_ALREAY_EXIST; } if (mnodeFindSuperTableTagIndex(pStable, schema[i].name) > 0) { mError("msg:%p, app:%p stable:%s, add tag, tag:%s already exist", pMsg, pMsg->rpcMsg.ahandle, pStable->info.tableId, schema[i].name); - return TSDB_CODE_MND_FIELD_ALREAY_EXIST; + return TSDB_CODE_MND_TAG_ALREAY_EXIST; } } diff --git a/tests/pytest/insert/schemalessInsert.py b/tests/pytest/insert/schemalessInsert.py index 00e805d791..9871af4b6d 100644 --- a/tests/pytest/insert/schemalessInsert.py +++ b/tests/pytest/insert/schemalessInsert.py @@ -1145,7 +1145,7 @@ class TDTestCase: s_stb_d_tb_a_col_m_tag_list = self.genSqlList(stb_name=stb_name)[5] self.multiThreadRun(self.genMultiThreadSeq(s_stb_d_tb_a_col_m_tag_list)) tdSql.query(f"show tables;") - tdSql.checkRows(6) + tdSql.checkRows(3) def sStbDtbDdataAtMcInsertMultiThreadCheckCase(self): """ @@ -1242,7 +1242,7 @@ class TDTestCase: s_stb_d_tb_d_ts_a_col_m_tag_list = self.genSqlList(stb_name=stb_name)[11] self.multiThreadRun(self.genMultiThreadSeq(s_stb_d_tb_d_ts_a_col_m_tag_list)) tdSql.query(f"show tables;") - tdSql.checkRows(6) + #tdSql.checkRows(6) def test(self): input_sql1 = "rfasta,id=\"rfasta_1\",t0=true,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64,t7=\"ddzhiksj\",t8=L\"ncharTagValue\" c0=True,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=\"bnhwlgvj\",c8=L\"ncharTagValue\",c9=7u64 1626006933640000000ns" @@ -1290,26 +1290,26 @@ class TDTestCase: # self.multiInsertCheckCase(1000) # self.batchErrorInsertCheckCase() # # MultiThreads - # self.stbInsertMultiThreadCheckCase() - # self.sStbStbDdataInsertMultiThreadCheckCase() - # self.sStbStbDdataAtcInsertMultiThreadCheckCase() - # self.sStbStbDdataMtcInsertMultiThreadCheckCase() - # self.sStbDtbDdataInsertMultiThreadCheckCase() + self.stbInsertMultiThreadCheckCase() + self.sStbStbDdataInsertMultiThreadCheckCase() + self.sStbStbDdataAtcInsertMultiThreadCheckCase() + self.sStbStbDdataMtcInsertMultiThreadCheckCase() + self.sStbDtbDdataInsertMultiThreadCheckCase() # # ! concurrency conflict self.sStbDtbDdataAcMtInsertMultiThreadCheckCase() - # self.sStbDtbDdataAtMcInsertMultiThreadCheckCase() + self.sStbDtbDdataAtMcInsertMultiThreadCheckCase() - # self.sStbStbDdataDtsInsertMultiThreadCheckCase() + self.sStbStbDdataDtsInsertMultiThreadCheckCase() # # ! concurrency conflict - # self.sStbStbDdataDtsAcMtInsertMultiThreadCheckCase() - # self.sStbStbDdataDtsAtMcInsertMultiThreadCheckCase() + self.sStbStbDdataDtsAcMtInsertMultiThreadCheckCase() + self.sStbStbDdataDtsAtMcInsertMultiThreadCheckCase() - # self.sStbDtbDdataDtsInsertMultiThreadCheckCase() + self.sStbDtbDdataDtsInsertMultiThreadCheckCase() # ! concurrency conflict - # self.sStbDtbDdataDtsAcMtInsertMultiThreadCheckCase() + self.sStbDtbDdataDtsAcMtInsertMultiThreadCheckCase() From 806451934b73165472da0d93948b116587887822 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Mon, 23 Aug 2021 08:39:29 +0800 Subject: [PATCH 06/30] [TD-6091]:ignore tag size/column size error during multi-thread schemaless insertion --- src/client/src/tscParseLineProtocol.c | 4 ++-- src/client/src/tscSQLParser.c | 5 ++--- src/inc/taoserror.h | 2 ++ 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/client/src/tscParseLineProtocol.c b/src/client/src/tscParseLineProtocol.c index 03e361c950..575cd80db6 100644 --- a/src/client/src/tscParseLineProtocol.c +++ b/src/client/src/tscParseLineProtocol.c @@ -418,7 +418,7 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInf } taos_free_result(res); - if (code == TSDB_CODE_MND_INVALID_COLUMN_LENGTH) { + if (code == TSDB_CODE_MND_INVALID_COLUMN_LENGTH || code == TSDB_CODE_TSC_INVALID_COLUMN_LENGTH) { TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); code = taos_errno(res2); if (code != TSDB_CODE_SUCCESS) { @@ -439,7 +439,7 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInf } taos_free_result(res); - if (code == TSDB_CODE_MND_INVALID_TAG_LENGTH) { + if (code == TSDB_CODE_MND_INVALID_TAG_LENGTH || code == TSDB_CODE_TSC_INVALID_TAG_LENGTH) { TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); code = taos_errno(res2); if (code != TSDB_CODE_SUCCESS) { diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 64f88b373d..bb79b93613 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -6021,7 +6021,6 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { const char* msg19 = "invalid new tag name"; const char* msg20 = "table is not super table"; const char* msg21 = "only binary/nchar column length could be modified"; - const char* msg22 = "new column length should be bigger than old one"; const char* msg23 = "only column length coulbe be modified"; const char* msg24 = "invalid binary/nchar column length"; @@ -6317,7 +6316,7 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { } if (pItem->bytes <= pColSchema->bytes) { - return invalidOperationMsg(pMsg, msg22); + return tscErrorMsgWithCode(TSDB_CODE_TSC_INVALID_COLUMN_LENGTH, tscGetErrorMsgPayload(pCmd), pItem->name, NULL); } SSchema* pSchema = (SSchema*) pTableMetaInfo->pTableMeta->schema; @@ -6368,7 +6367,7 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { } if (pItem->bytes <= pColSchema->bytes) { - return invalidOperationMsg(pMsg, msg22); + return tscErrorMsgWithCode(TSDB_CODE_TSC_INVALID_TAG_LENGTH, tscGetErrorMsgPayload(pCmd), pItem->name, NULL); } SSchema* pSchema = tscGetTableTagSchema(pTableMetaInfo->pTableMeta); diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index f54150d76b..08f6d0bf4d 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -104,6 +104,8 @@ int32_t* taosGetErrno(); #define TSDB_CODE_TSC_LINE_SYNTAX_ERROR TAOS_DEF_ERROR_CODE(0, 0x021B) //"Syntax error in Line") #define TSDB_CODE_TSC_NO_META_CACHED TAOS_DEF_ERROR_CODE(0, 0x021C) //"No table meta cached") #define TSDB_CODE_TSC_DUP_COL_NAMES TAOS_DEF_ERROR_CODE(0, 0x021D) //"duplicated column names") +#define TSDB_CODE_TSC_INVALID_TAG_LENGTH TAOS_DEF_ERROR_CODE(0, 0x021E) //"Invalid tag length") +#define TSDB_CODE_TSC_INVALID_COLUMN_LENGTH TAOS_DEF_ERROR_CODE(0, 0x021F) //"Invalid tag length") // mnode #define TSDB_CODE_MND_MSG_NOT_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0300) //"Message not processed") From 4714688f30198d84da2532d767ff726cb350e15a Mon Sep 17 00:00:00 2001 From: Shenglian Zhou Date: Mon, 23 Aug 2021 10:08:29 +0800 Subject: [PATCH 07/30] [TD-6199]:restore schemless test file before development --- tests/pytest/insert/schemalessInsert.py | 72 ++++++++++++------------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/tests/pytest/insert/schemalessInsert.py b/tests/pytest/insert/schemalessInsert.py index 9871af4b6d..49c3223588 100644 --- a/tests/pytest/insert/schemalessInsert.py +++ b/tests/pytest/insert/schemalessInsert.py @@ -1145,7 +1145,7 @@ class TDTestCase: s_stb_d_tb_a_col_m_tag_list = self.genSqlList(stb_name=stb_name)[5] self.multiThreadRun(self.genMultiThreadSeq(s_stb_d_tb_a_col_m_tag_list)) tdSql.query(f"show tables;") - tdSql.checkRows(3) + tdSql.checkRows(6) def sStbDtbDdataAtMcInsertMultiThreadCheckCase(self): """ @@ -1242,7 +1242,7 @@ class TDTestCase: s_stb_d_tb_d_ts_a_col_m_tag_list = self.genSqlList(stb_name=stb_name)[11] self.multiThreadRun(self.genMultiThreadSeq(s_stb_d_tb_d_ts_a_col_m_tag_list)) tdSql.query(f"show tables;") - #tdSql.checkRows(6) + tdSql.checkRows(6) def test(self): input_sql1 = "rfasta,id=\"rfasta_1\",t0=true,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64,t7=\"ddzhiksj\",t8=L\"ncharTagValue\" c0=True,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=\"bnhwlgvj\",c8=L\"ncharTagValue\",c9=7u64 1626006933640000000ns" @@ -1261,35 +1261,35 @@ class TDTestCase: # self._conn.insert_lines([input_sql4]) def runAll(self): - # self.initCheckCase() - # self.boolTypeCheckCase() - # self.symbolsCheckCase() - # self.tsCheckCase() - # self.idSeqCheckCase() - # self.idUpperCheckCase() - # self.noIdCheckCase() - # self.maxColTagCheckCase() - # self.idIllegalNameCheckCase() - # self.idStartWithNumCheckCase() - # self.nowTsCheckCase() - # self.dateFormatTsCheckCase() - # self.illegalTsCheckCase() - # self.tagValueLengthCheckCase() - # self.colValueLengthCheckCase() - # self.tagColIllegalValueCheckCase() - # self.duplicateIdTagColInsertCheckCase() - # self.noIdStbExistCheckCase() - # self.duplicateInsertExistCheckCase() - # self.tagColBinaryNcharLengthCheckCase() - # self.tagColAddDupIDCheckCase() - # self.tagColAddCheckCase() - # self.tagMd5Check() - # self.tagColBinaryMaxLengthCheckCase() - # # self.tagColNcharMaxLengthCheckCase() - # self.batchInsertCheckCase() - # self.multiInsertCheckCase(1000) - # self.batchErrorInsertCheckCase() - # # MultiThreads + self.initCheckCase() + self.boolTypeCheckCase() + self.symbolsCheckCase() + self.tsCheckCase() + self.idSeqCheckCase() + self.idUpperCheckCase() + self.noIdCheckCase() + self.maxColTagCheckCase() + self.idIllegalNameCheckCase() + self.idStartWithNumCheckCase() + self.nowTsCheckCase() + self.dateFormatTsCheckCase() + self.illegalTsCheckCase() + self.tagValueLengthCheckCase() + self.colValueLengthCheckCase() + self.tagColIllegalValueCheckCase() + self.duplicateIdTagColInsertCheckCase() + self.noIdStbExistCheckCase() + self.duplicateInsertExistCheckCase() + self.tagColBinaryNcharLengthCheckCase() + self.tagColAddDupIDCheckCase() + self.tagColAddCheckCase() + self.tagMd5Check() + self.tagColBinaryMaxLengthCheckCase() + # self.tagColNcharMaxLengthCheckCase() + self.batchInsertCheckCase() + self.multiInsertCheckCase(1000) + self.batchErrorInsertCheckCase() + # MultiThreads self.stbInsertMultiThreadCheckCase() self.sStbStbDdataInsertMultiThreadCheckCase() self.sStbStbDdataAtcInsertMultiThreadCheckCase() @@ -1297,19 +1297,19 @@ class TDTestCase: self.sStbDtbDdataInsertMultiThreadCheckCase() # # ! concurrency conflict - self.sStbDtbDdataAcMtInsertMultiThreadCheckCase() - self.sStbDtbDdataAtMcInsertMultiThreadCheckCase() + # self.sStbDtbDdataAcMtInsertMultiThreadCheckCase() + # self.sStbDtbDdataAtMcInsertMultiThreadCheckCase() self.sStbStbDdataDtsInsertMultiThreadCheckCase() # # ! concurrency conflict - self.sStbStbDdataDtsAcMtInsertMultiThreadCheckCase() - self.sStbStbDdataDtsAtMcInsertMultiThreadCheckCase() + # self.sStbStbDdataDtsAcMtInsertMultiThreadCheckCase() + # self.sStbStbDdataDtsAtMcInsertMultiThreadCheckCase() self.sStbDtbDdataDtsInsertMultiThreadCheckCase() # ! concurrency conflict - self.sStbDtbDdataDtsAcMtInsertMultiThreadCheckCase() + # self.sStbDtbDdataDtsAcMtInsertMultiThreadCheckCase() From 0c09bbb9ba0a0c74296e687a084f19e5643841ef Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Mon, 23 Aug 2021 15:45:34 +0800 Subject: [PATCH 08/30] only performance test and leave corretness test to python test cases --- tests/examples/c/schemaless.c | 118 +--------------------------------- 1 file changed, 2 insertions(+), 116 deletions(-) diff --git a/tests/examples/c/schemaless.c b/tests/examples/c/schemaless.c index 3ea199c914..1a551cc5f7 100644 --- a/tests/examples/c/schemaless.c +++ b/tests/examples/c/schemaless.c @@ -61,7 +61,7 @@ int main(int argc, char* argv[]) { time_t ct = time(0); int64_t ts = ct * 1000; - char* lineFormat = "sta%d,t0=true,t1=127i8,t2=32767i16,t3=%di32,t4=9223372036854775807i64,t9=11.12345f32,t10=22.123456789f64,t11=\"binaryTagValue\",t12=L\"ncharTagValue\" c0=true,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=255u8,c6=32770u16,c7=2147483699u32,c8=9223372036854775899u64,c9=11.12345f32,c10=22.123456789f64,c11=\"binaryValue\",c12=L\"ncharValue\" %lldms"; + char* lineFormat = "sta%d,t0=true,t1=127i8,t2=32767i16,t3=%di32,t4=9223372036854775807i64,t9=11.12345f32,t10=22.123456789f64,t11=\"binaryTagValue\",t12=L\"ncharTagValue\" c0=true,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=254u8,c6=32770u16,c7=2147483699u32,c8=9223372036854775899u64,c9=11.12345f32,c10=22.123456789f64,c11=\"binaryValue\",c12=L\"ncharValue\" %lldms"; char** lines = calloc(numSuperTables * numChildTables * numRowsPerChildTable, sizeof(char*)); int l = 0; @@ -75,7 +75,7 @@ int main(int argc, char* argv[]) { } } } - shuffle(lines, numSuperTables * numChildTables * numRowsPerChildTable); + //shuffle(lines, numSuperTables * numChildTables * numRowsPerChildTable); printf("%s\n", "begin taos_insert_lines"); int64_t begin = getTimeInUs(); @@ -83,119 +83,5 @@ int main(int argc, char* argv[]) { int64_t end = getTimeInUs(); printf("code: %d, %s. time used: %"PRId64"\n", code, tstrerror(code), end-begin); - char* lines_000_0[] = { - "sta1,id=sta1_1,t0=true,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=255u8,t6=32770u16,t7=2147483699u32,t8=9223372036854775899u64,t9=11.12345f32,t10=22.123456789f64,t11=\"binaryTagValue\",t12=L\"ncharTagValue\" c0=true,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=255u8,c6=32770u16,c7=2147483699u32,c8=9223372036854775899u64,c9=11.12345f32,c10=22.123456789f64,c11=\"binaryValue\",c12=L\"ncharValue\" 1626006833639000us" - }; - - code = taos_insert_lines(taos, lines_000_0 , sizeof(lines_000_0)/sizeof(char*)); - if (0 == code) { - printf("taos_insert_lines() lines_000_0 should return error\n"); - return -1; - } - - char* lines_000_1[] = { - "sta2,id=\"sta2_1\",t0=true,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=255u8,t6=32770u16,t7=2147483699u32,t8=9223372036854775899u64,t9=11.12345f32,t10=22.123456789f64,t11=\"binaryTagValue\",t12=L\"ncharTagValue\" c0=true,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=255u8,c6=32770u16,c7=2147483699u32,c8=9223372036854775899u64,c9=11.12345f32,c10=22.123456789f64,c11=\"binaryValue\",c12=L\"ncharValue\" 1626006833639001" - }; - - code = taos_insert_lines(taos, lines_000_1 , sizeof(lines_000_1)/sizeof(char*)); - if (0 == code) { - printf("taos_insert_lines() lines_000_1 should return error\n"); - return -1; - } - - char* lines_000_2[] = { - "sta3,id=\"sta3_1\",t0=true,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t9=11.12345f32,t10=22.123456789f64,t11=\"binaryTagValue\",t12=L\"ncharTagValue\" c0=true,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=255u8,c6=32770u16,c7=2147483699u32,c8=9223372036854775899u64,c9=11.12345f32,c10=22.123456789f64,c11=\"binaryValue\",c12=L\"ncharValue\" 0" - }; - - code = taos_insert_lines(taos, lines_000_2 , sizeof(lines_000_2)/sizeof(char*)); - if (0 != code) { - printf("taos_insert_lines() lines_000_2 return code:%d (%s)\n", code, (char*)tstrerror(code)); - return -1; - } - - char* lines_001_0[] = { - "sta4,t0=true,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t9=11.12345f32,t10=22.123456789f64,t11=\"binaryTagValue\",t12=L\"ncharTagValue\" c0=true,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c9=11.12345f32,c10=22.123456789f64,c11=\"binaryValue\",c12=L\"ncharValue\" 1626006833639000us", - - }; - - code = taos_insert_lines(taos, lines_001_0 , sizeof(lines_001_0)/sizeof(char*)); - if (0 != code) { - printf("taos_insert_lines() lines_001_0 return code:%d (%s)\n", code, (char*)tstrerror(code)); - return -1; - } - - char* lines_001_1[] = { - "sta5,id=\"sta5_1\",t0=true,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t9=11.12345f32,t10=22.123456789f64,t11=\"binaryTagValue\",t12=L\"ncharTagValue\" c0=true,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c9=11.12345f32,c10=22.123456789f64,c11=\"binaryValue\",c12=L\"ncharValue\" 1626006833639001" - }; - - code = taos_insert_lines(taos, lines_001_1 , sizeof(lines_001_1)/sizeof(char*)); - if (0 != code) { - printf("taos_insert_lines() lines_001_1 return code:%d (%s)\n", code, (char*)tstrerror(code)); - return -1; - } - - char* lines_001_2[] = { - "sta6,id=\"sta6_1\",t0=true,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t9=11.12345f32,t10=22.123456789f64,t11=\"binaryTagValue\",t12=L\"ncharTagValue\" c0=true,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c9=11.12345f32,c10=22.123456789f64,c11=\"binaryValue\",c12=L\"ncharValue\" 0" - }; - - code = taos_insert_lines(taos, lines_001_2 , sizeof(lines_001_2)/sizeof(char*)); - if (0 != code) { - printf("taos_insert_lines() lines_001_2 return code:%d (%s)\n", code, (char*)tstrerror(code)); - return -1; - } - - char* lines_002[] = { - "stb,id=\"stb_1\",t20=t,t21=T,t22=true,t23=True,t24=TRUE,t25=f,t26=F,t27=false,t28=False,t29=FALSE,t10=33.12345,t11=\"binaryTagValue\",t12=L\"ncharTagValue\" c20=t,c21=T,c22=true,c23=True,c24=TRUE,c25=f,c26=F,c27=false,c28=False,c29=FALSE,c10=33.12345,c11=\"binaryValue\",c12=L\"ncharValue\" 1626006833639000000ns", - "stc,id=\"stc_1\",t20=t,t21=T,t22=true,t23=True,t24=TRUE,t25=f,t26=F,t27=false,t28=False,t29=FALSE,t10=33.12345,t11=\"binaryTagValue\",t12=L\"ncharTagValue\" c20=t,c21=T,c22=true,c23=True,c24=TRUE,c25=f,c26=F,c27=false,c28=False,c29=FALSE,c10=33.12345,c11=\"binaryValue\",c12=L\"ncharValue\" 1626006833639019us", - "stc,id=\"stc_1\",t20=t,t21=T,t22=true,t23=True,t24=TRUE,t25=f,t26=F,t27=false,t28=False,t29=FALSE,t10=33.12345,t11=\"binaryTagValue\",t12=L\"ncharTagValue\" c20=t,c21=T,c22=true,c23=True,c24=TRUE,c25=f,c26=F,c27=false,c28=False,c29=FALSE,c10=33.12345,c11=\"binaryValue\",c12=L\"ncharValue\" 1626006833640ms", - "stc,id=\"stc_1\",t20=t,t21=T,t22=true,t23=True,t24=TRUE,t25=f,t26=F,t27=false,t28=False,t29=FALSE,t10=33.12345,t11=\"binaryTagValue\",t12=L\"ncharTagValue\" c20=t,c21=T,c22=true,c23=True,c24=TRUE,c25=f,c26=F,c27=false,c28=False,c29=FALSE,c10=33.12345,c11=\"binaryValue\",c12=L\"ncharValue\" 1626006834s" - }; - - code = taos_insert_lines(taos, lines_002 , sizeof(lines_002)/sizeof(char*)); - if (0 != code) { - printf("taos_insert_lines() lines_002 return code:%d (%s)\n", code, (char*)tstrerror(code)); - return -1; - } - - //Duplicate key check; - char* lines_003_1[] = { - "std,id=\"std_3_1\",t1=4i64,Id=\"std\",t2=true c1=true 1626006834s" - }; - - code = taos_insert_lines(taos, lines_003_1 , sizeof(lines_003_1)/sizeof(char*)); - if (0 == code) { - printf("taos_insert_lines() lines_003_1 return code:%d (%s)\n", code, (char*)tstrerror(code)); - return -1; - } - - char* lines_003_2[] = { - "std,id=\"std_3_2\",tag1=4i64,Tag2=true,tAg3=2,TaG2=\"dup!\" c1=true 1626006834s" - }; - - code = taos_insert_lines(taos, lines_003_2 , sizeof(lines_003_2)/sizeof(char*)); - if (0 == code) { - printf("taos_insert_lines() lines_003_2 return code:%d (%s)\n", code, (char*)tstrerror(code)); - return -1; - } - - char* lines_003_3[] = { - "std,id=\"std_3_3\",tag1=4i64 field1=true,Field2=2,FIElD1=\"dup!\",fIeLd4=true 1626006834s" - }; - - code = taos_insert_lines(taos, lines_003_3 , sizeof(lines_003_3)/sizeof(char*)); - if (0 == code) { - printf("taos_insert_lines() lines_003_3 return code:%d (%s)\n", code, (char*)tstrerror(code)); - return -1; - } - - char* lines_003_4[] = { - "std,id=\"std_3_4\",tag1=4i64,dupkey=4i16,tag2=T field1=true,dUpkEy=1e3f32,field2=\"1234\" 1626006834s" - }; - - code = taos_insert_lines(taos, lines_003_4 , sizeof(lines_003_4)/sizeof(char*)); - if (0 == code) { - printf("taos_insert_lines() lines_003_4 return code:%d (%s)\n", code, (char*)tstrerror(code)); - return -1; - } return 0; } From e9b9b9439d55ea6529ba273769f95d0a3aa10fc1 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Tue, 24 Aug 2021 11:42:15 +0800 Subject: [PATCH 09/30] [TD-5201]:register sql object whenever --- src/client/src/tscParseLineProtocol.c | 29 +++++++++++++++------------ src/client/src/tscPrepare.c | 4 ++-- 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/src/client/src/tscParseLineProtocol.c b/src/client/src/tscParseLineProtocol.c index 575cd80db6..bde68c306a 100644 --- a/src/client/src/tscParseLineProtocol.c +++ b/src/client/src/tscParseLineProtocol.c @@ -580,25 +580,26 @@ static int32_t retrieveTableMeta(TAOS* taos, char* tableName, STableMeta** pTabl pSql->signature = pSql; pSql->fp = NULL; + registerSqlObj(pSql); SStrToken tableToken = {.z = tableNameLowerCase, .n = (uint32_t)strlen(tableNameLowerCase), .type = TK_ID}; tGetToken(tableNameLowerCase, &tableToken.type); // Check if the table name available or not if (tscValidateName(&tableToken) != TSDB_CODE_SUCCESS) { code = TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; sprintf(pSql->cmd.payload, "table name is invalid"); - tscFreeSqlObj(pSql); + tscFreeRegisteredSqlObj(pSql); return code; } SName sname = {0}; if ((code = tscSetTableFullName(&sname, &tableToken, pSql)) != TSDB_CODE_SUCCESS) { - tscFreeSqlObj(pSql); + tscFreeRegisteredSqlObj(pSql); return code; } char fullTableName[TSDB_TABLE_FNAME_LEN] = {0}; memset(fullTableName, 0, tListLen(fullTableName)); tNameExtractFullName(&sname, fullTableName); - tscFreeSqlObj(pSql); + tscFreeRegisteredSqlObj(pSql); size_t size = 0; taosHashGetCloneExt(tscTableMetaMap, fullTableName, strlen(fullTableName), NULL, (void**)&tableMeta, &size); @@ -714,18 +715,21 @@ static int32_t changeChildTableTagValue(TAOS* taos, const char* cTableName, cons if (code != 0) { tscError("SML:0x%"PRIx64" taos_stmt_prepare return %d:%s", info->id, code, tstrerror(code)); + taos_stmt_close(stmt); return code; } code = taos_stmt_bind_param(stmt, bind); if (code != 0) { tscError("SML:0x%"PRIx64" taos_stmt_bind_param return %d:%s", info->id, code, tstrerror(code)); + taos_stmt_close(stmt); return code; } code = taos_stmt_execute(stmt); if (code != 0) { tscError("SML:0x%"PRIx64" taos_stmt_execute return %d:%s", info->id, code, tstrerror(code)); + taos_stmt_close(stmt); return code; } @@ -775,22 +779,22 @@ static int32_t creatChildTableIfNotExists(TAOS* taos, const char* cTableName, co free(sql); if (code != 0) { - tfree(stmt); tscError("SML:0x%"PRIx64" taos_stmt_prepare returns %d:%s", info->id, code, tstrerror(code)); + taos_stmt_close(stmt); return code; } code = taos_stmt_bind_param(stmt, TARRAY_GET_START(tagsBind)); if (code != 0) { - tfree(stmt); tscError("SML:0x%"PRIx64" taos_stmt_bind_param returns %d:%s", info->id, code, tstrerror(code)); + taos_stmt_close(stmt); return code; } code = taos_stmt_execute(stmt); if (code != 0) { - tfree(stmt); tscError("SML:0x%"PRIx64" taos_stmt_execute returns %d:%s", info->id, code, tstrerror(code)); + taos_stmt_close(stmt); return code; } @@ -835,20 +839,21 @@ static int32_t insertChildTableBatch(TAOS* taos, char* cTableName, SArray* cols tfree(sql); return TSDB_CODE_TSC_OUT_OF_MEMORY; } + code = taos_stmt_prepare(stmt, sql, (unsigned long)strlen(sql)); tfree(sql); if (code != 0) { - tfree(stmt); tscError("SML:0x%"PRIx64" taos_stmt_prepare return %d:%s", info->id, code, tstrerror(code)); + taos_stmt_close(stmt); return code; } do { code = taos_stmt_set_tbname(stmt, cTableName); if (code != 0) { - tfree(stmt); tscError("SML:0x%"PRIx64" taos_stmt_set_tbname return %d:%s", info->id, code, tstrerror(code)); + taos_stmt_close(stmt); return code; } @@ -857,14 +862,14 @@ static int32_t insertChildTableBatch(TAOS* taos, char* cTableName, SArray* cols TAOS_BIND* colsBinds = taosArrayGetP(rowsBind, i); code = taos_stmt_bind_param(stmt, colsBinds); if (code != 0) { - tfree(stmt); tscError("SML:0x%"PRIx64" taos_stmt_bind_param return %d:%s", info->id, code, tstrerror(code)); + taos_stmt_close(stmt); return code; } code = taos_stmt_add_batch(stmt); if (code != 0) { - tfree(stmt); tscError("SML:0x%"PRIx64" taos_stmt_add_batch return %d:%s", info->id, code, tstrerror(code)); + taos_stmt_close(stmt); return code; } } @@ -877,11 +882,9 @@ static int32_t insertChildTableBatch(TAOS* taos, char* cTableName, SArray* cols if (code != 0) { tscError("SML:0x%"PRIx64" %d:%s", info->id, code, tstrerror(code)); - taos_stmt_close(stmt); - } else { - taos_stmt_close(stmt); } + taos_stmt_close(stmt); return code; } diff --git a/src/client/src/tscPrepare.c b/src/client/src/tscPrepare.c index 7f7966559b..48393a3dda 100644 --- a/src/client/src/tscPrepare.c +++ b/src/client/src/tscPrepare.c @@ -1537,6 +1537,8 @@ int taos_stmt_prepare(TAOS_STMT* stmt, const char* sql, unsigned long length) { pRes->qId = 0; pRes->numOfRows = 1; + registerSqlObj(pSql); + strtolower(pSql->sqlstr, sql); tscDebugL("0x%"PRIx64" SQL: %s", pSql->self, pSql->sqlstr); @@ -1546,8 +1548,6 @@ int taos_stmt_prepare(TAOS_STMT* stmt, const char* sql, unsigned long length) { pSql->cmd.insertParam.numOfParams = 0; pSql->cmd.batchSize = 0; - registerSqlObj(pSql); - int32_t ret = stmtParseInsertTbTags(pSql, pStmt); if (ret != TSDB_CODE_SUCCESS) { STMT_RET(ret); From 060a7451c48cc0c8e9bd2b7911026af4cbaadd90 Mon Sep 17 00:00:00 2001 From: jiajingbin Date: Tue, 17 Aug 2021 16:36:08 +0800 Subject: [PATCH 10/30] add testcases 5 --- tests/pytest/query/queryDiffColsOr.py | 436 ++++++++++++++++++++++++++ 1 file changed, 436 insertions(+) create mode 100644 tests/pytest/query/queryDiffColsOr.py diff --git a/tests/pytest/query/queryDiffColsOr.py b/tests/pytest/query/queryDiffColsOr.py new file mode 100644 index 0000000000..68c8f14869 --- /dev/null +++ b/tests/pytest/query/queryDiffColsOr.py @@ -0,0 +1,436 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- +from copy import deepcopy +from util.log import tdLog +from util.cases import tdCases +from util.sql import tdSql +from util.common import tdCom + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def insertData(self, tb_name): + insert_sql_list = [f'insert into {tb_name} values ("2021-01-01 12:00:00", 1, 1, 1, 3, 1.1, 1.1, "binary", "nchar", true, 1)', + f'insert into {tb_name} values ("2021-01-05 12:00:00", 2, 2, 1, 3, 1.1, 1.1, "binary", "nchar", true, 2)', + f'insert into {tb_name} values ("2021-01-07 12:00:00", 1, 3, 1, 2, 1.1, 1.1, "binary", "nchar", true, 3)', + f'insert into {tb_name} values ("2021-01-09 12:00:00", 1, 2, 4, 3, 1.1, 1.1, "binary", "nchar", true, 4)', + f'insert into {tb_name} values ("2021-01-11 12:00:00", 1, 2, 5, 5, 1.1, 1.1, "binary", "nchar", true, 5)', + f'insert into {tb_name} values ("2021-01-13 12:00:00", 1, 2, 1, 3, 6.6, 1.1, "binary", "nchar", true, 6)', + f'insert into {tb_name} values ("2021-01-15 12:00:00", 1, 2, 1, 3, 1.1, 7.7, "binary", "nchar", true, 7)', + f'insert into {tb_name} values ("2021-01-17 12:00:00", 1, 2, 1, 3, 1.1, 1.1, "binary8", "nchar", true, 8)', + f'insert into {tb_name} values ("2021-01-19 12:00:00", 1, 2, 1, 3, 1.1, 1.1, "binary", "nchar9", true, 9)', + f'insert into {tb_name} values ("2021-01-21 12:00:00", 1, 2, 1, 3, 1.1, 1.1, "binary", "nchar", false, 10)', + f'insert into {tb_name} values ("2021-01-23 12:00:00", 1, 3, 1, 3, 1.1, 1.1, Null, Null, false, 11)' + ] + for sql in insert_sql_list: + tdSql.execute(sql) + + def initTb(self): + tdCom.cleanTb() + tb_name = tdCom.getLongName(8, "letters") + tdSql.execute( + f"CREATE TABLE {tb_name} (ts timestamp, c1 tinyint, c2 smallint, c3 int, c4 bigint, c5 float, c6 double, c7 binary(100), c8 nchar(200), c9 bool, c10 int)") + self.insertData(tb_name) + return tb_name + + def queryLastC10(self, query_sql, multi=False): + if multi: + res = tdSql.query(query_sql.replace('c10', 'last(*)'), True) + else: + res = tdSql.query(query_sql.replace('*', 'last(*)'), True) + return int(res[0][-1]) + + def queryFullColType(self, tb_name): + ## ts + query_sql = f'select * from {tb_name} where ts > "2021-01-11 12:00:00" or ts < "2021-01-13 12:00:00"' + tdSql.query(query_sql) + tdSql.checkRows(11) + tdSql.checkEqual(self.queryLastC10(query_sql), 11) + + ## != or + query_sql = f'select * from {tb_name} where c1 != 1 or c2 = 3' + tdSql.query(query_sql) + tdSql.checkRows(3) + tdSql.checkEqual(self.queryLastC10(query_sql), 11) + + ## <> or + query_sql = f'select * from {tb_name} where c1 <> 1 or c3 = 3' + tdSql.query(query_sql) + tdSql.checkRows(1) + tdSql.checkEqual(self.queryLastC10(query_sql), 2) + + ## >= or + query_sql = f'select * from {tb_name} where c1 >= 2 or c3 = 4' + tdSql.query(query_sql) + tdSql.checkRows(2) + tdSql.checkEqual(self.queryLastC10(query_sql), 4) + + ## <= or + query_sql = f'select * from {tb_name} where c1 <= 1 or c3 = 4' + tdSql.query(query_sql) + tdSql.checkRows(10) + tdSql.checkEqual(self.queryLastC10(query_sql), 11) + + ## <> or is Null + query_sql = f'select * from {tb_name} where c1 <> 1 or c7 is Null' + tdSql.query(query_sql) + tdSql.checkRows(2) + tdSql.checkEqual(self.queryLastC10(query_sql), 11) + + ## > or is not Null + query_sql = f'select * from {tb_name} where c2 > 2 or c8 is not Null' + tdSql.query(query_sql) + tdSql.checkRows(11) + tdSql.checkEqual(self.queryLastC10(query_sql), 11) + + ## > or < or >= or <= or != or <> or = Null + query_sql = f'select * from {tb_name} where c1 > 1 or c2 < 2 or c3 >= 4 or c4 <= 2 or c5 != 1.1 or c6 <> 1.1 or c7 is Null' + tdSql.query(query_sql) + tdSql.checkRows(8) + tdSql.checkEqual(self.queryLastC10(query_sql), 11) + + ## tiny small int big or + query_sql = f'select * from {tb_name} where c1 = 2 or c2 = 3 or c3 = 4 or c4 = 5' + tdSql.query(query_sql) + tdSql.checkRows(5) + tdSql.checkEqual(self.queryLastC10(query_sql), 11) + + ## float double binary nchar bool or + query_sql = f'select * from {tb_name} where c5=6.6 or c6=7.7 or c7="binary8" or c8="nchar9" or c9=false' + tdSql.query(query_sql) + tdSql.checkRows(6) + tdSql.checkEqual(self.queryLastC10(query_sql), 11) + + ## all types or + query_sql = f'select * from {tb_name} where c1=2 or c2=3 or c3=4 or c4=5 or c5=6.6 or c6=7.7 or c7="binary8" or c8="nchar9" or c9=false' + tdSql.query(query_sql) + tdSql.checkRows(10) + tdSql.checkEqual(self.queryLastC10(query_sql), 11) + + def checkTbColTypeOperator(self): + ''' + Ordinary table full column type and operator + ''' + tb_name = self.initTb() + self.queryFullColType(tb_name) + + def checkTbMultiExpression(self): + ''' + Ordinary table multiExpression + ''' + tb_name = self.initTb() + ## condition_A and condition_B or condition_C (> < >=) + query_sql = f'select * from {tb_name} where c1 > 2 and c2 < 4 or c3 >= 4' + tdSql.query(query_sql) + tdSql.checkRows(2) + tdSql.checkEqual(self.queryLastC10(query_sql), 5) + + ## (condition_A and condition_B) or condition_C (<= != <>) + query_sql = f'select * from {tb_name} where (c1 <= 1 and c2 != 2) or c4 <> 3' + tdSql.query(query_sql) + tdSql.checkRows(4) + tdSql.checkEqual(self.queryLastC10(query_sql), 11) + + ## condition_A and (condition_B or condition_C) (Null not Null) + query_sql = f'select * from {tb_name} where c1 is not Null and (c6 = 7.7 or c8 is Null)' + tdSql.query(query_sql) + tdSql.checkRows(2) + tdSql.checkEqual(self.queryLastC10(query_sql), 11) + + ## condition_A or condition_B and condition_C (> < >=) + query_sql = f'select * from {tb_name} where c1 > 2 or c2 < 4 and c3 >= 4' + tdSql.query(query_sql) + tdSql.checkRows(2) + tdSql.checkEqual(self.queryLastC10(query_sql), 5) + + ## (condition_A or condition_B) and condition_C (<= != <>) + query_sql = f'select * from {tb_name} where (c1 <= 1 or c2 != 2) and c4 <> 3' + tdSql.query(query_sql) + tdSql.checkRows(2) + tdSql.checkEqual(self.queryLastC10(query_sql), 5) + + ## condition_A or (condition_B and condition_C) (Null not Null) + query_sql = f'select * from {tb_name} where c6 >= 7.7 or (c1 is not Null and c3 =5)' + tdSql.query(query_sql) + tdSql.checkRows(2) + tdSql.checkEqual(self.queryLastC10(query_sql), 7) + + ## condition_A or (condition_B and condition_C) or condition_D (> != < Null) + query_sql = f'select * from {tb_name} where c1 != 1 or (c2 >2 and c3 < 1) or c7 is Null' + tdSql.query(query_sql) + tdSql.checkRows(2) + tdSql.checkEqual(self.queryLastC10(query_sql), 11) + + ## condition_A and (condition_B or condition_C) and condition_D (>= = <= not Null) + query_sql = f'select * from {tb_name} where c4 >= 4 and (c1 = 2 or c5 <= 1.1) and c7 is not Null' + tdSql.query(query_sql) + tdSql.checkRows(1) + tdSql.checkEqual(self.queryLastC10(query_sql), 5) + + ## (condition_A and condition_B) or (condition_C or condition_D) (Null >= > =) + query_sql = f'select * from {tb_name} where (c8 is Null and c1 >= 1) or (c3 > 3 or c4 =2)' + tdSql.query(query_sql) + tdSql.checkRows(4) + tdSql.checkEqual(self.queryLastC10(query_sql), 11) + + ## (condition_A or condition_B) or condition_C or (condition_D and condition_E) (>= <= = not Null <>) + query_sql = f'select * from {tb_name} where (c1 >= 2 or c2 <= 1) or c3 = 4 or (c7 is not Null and c6 <> 1.1)' + tdSql.query(query_sql) + tdSql.checkRows(4) + tdSql.checkEqual(self.queryLastC10(query_sql), 7) + + ## condition_A or (condition_B and condition_C) or (condition_D and condition_E) and condition_F + query_sql = f'select * from {tb_name} where c1 != 1 or (c2 <= 1 and c3 <4) or (c3 >= 4 or c7 is not Null) and c9 <> true' + tdSql.query(query_sql) + tdSql.checkRows(3) + tdSql.checkEqual(self.queryLastC10(query_sql), 10) + + ## (condition_A or (condition_B and condition_C) or (condition_D and condition_E)) and condition_F + query_sql = f'select * from {tb_name} where (c1 != 1 or (c2 <= 2 and c3 >= 4) or (c3 >= 4 or c7 is not Null)) and c9 != false' + tdSql.query(query_sql) + tdSql.checkRows(9) + tdSql.checkEqual(self.queryLastC10(query_sql), 9) + + ## (condition_A or condition_B) or (condition_C or condition_D) and (condition_E or condition_F or condition_G) + query_sql = f'select * from {tb_name} where c1 != 1 or (c2 <= 3 and c3 > 4) and c3 <= 5 and (c7 is not Null and c9 != false)' + tdSql.query(query_sql) + tdSql.checkRows(2) + tdSql.checkEqual(self.queryLastC10(query_sql), 5) + + def checkTbMultiIn(self): + ''' + Ordinary table multiIn + ''' + tb_name = self.initTb() + ## in and in + query_sql = f'select * from {tb_name} where c7 in ("binary") and c8 in ("nchar")' + tdSql.query(query_sql) + tdSql.checkRows(8) + tdSql.checkEqual(self.queryLastC10(query_sql), 10) + + ## in or in + query_sql = f'select * from {tb_name} where c1 in (2, 4) or c2 in (1, 4)' + tdSql.query(query_sql) + tdSql.checkRows(2) + tdSql.checkEqual(self.queryLastC10(query_sql), 2) + + ## in and in or condition_A + query_sql = f'select * from {tb_name} where c7 in ("binary") and c8 in ("nchar") or c10 != 10' + tdSql.query(query_sql) + tdSql.checkRows(11) + tdSql.checkEqual(self.queryLastC10(query_sql), 11) + + ## in or in and condition_A + query_sql = f'select * from {tb_name} where c7 in ("binary") or c8 in ("nchar") and c10 != 10' + tdSql.query(query_sql) + tdSql.checkRows(10) + tdSql.checkEqual(self.queryLastC10(query_sql), 10) + + ## in or in or condition_A + query_sql = f'select * from {tb_name} where c1 in (2, 4) or c2 in (3, 4) or c9 != true' + tdSql.query(query_sql) + tdSql.checkRows(4) + tdSql.checkEqual(self.queryLastC10(query_sql), 11) + + ## in or in or in or in + query_sql = f'select * from {tb_name} where c1 in (2, 4) or c2 in (3, 4) or c9 in (false) or c10 in (5, 6 ,22)' + tdSql.query(query_sql) + tdSql.checkRows(6) + tdSql.checkEqual(self.queryLastC10(query_sql), 11) + + ## in or in and in or in + query_sql = f'select * from {tb_name} where c1 in (2, 4) or c2 in (3, 4) and c9 in (false) or c10 in (5, 6 ,22)' + tdSql.query(query_sql) + tdSql.checkRows(4) + tdSql.checkEqual(self.queryLastC10(query_sql), 11) + + ## condition_A or in or condition_B and in + query_sql = f'select * from {tb_name} where c1 = 2 or c2 in (2, 4) and c9 = false or c10 in (6 ,22)' + tdSql.query(query_sql) + tdSql.checkRows(3) + tdSql.checkEqual(self.queryLastC10(query_sql), 10) + + ## in and condition_A or in and in and condition_B + query_sql = f'select * from {tb_name} where c1 in (2, 3) and c2 <> 3 or c10 <= 4 and c10 in (4 ,22) and c9 != false' + tdSql.query(query_sql) + tdSql.checkRows(2) + tdSql.checkEqual(self.queryLastC10(query_sql), 4) + + ## (in and condition_A or in) and in and condition_B + query_sql = f'select * from {tb_name} where (c1 in (2, 3) and c2 <> 3 or c10 <= 4) and c10 in (4 ,22) and c9 != false' + tdSql.query(query_sql) + tdSql.checkRows(1) + tdSql.checkEqual(self.queryLastC10(query_sql), 4) + + def checkTbMultiLike(self): + ''' + Ordinary table multiLike + ''' + tb_name = self.initTb() + ## like and like + query_sql = f'select * from {tb_name} where c7 like "bi%" and c8 like ("ncha_")' + tdSql.query(query_sql) + tdSql.checkRows(9) + tdSql.checkEqual(self.queryLastC10(query_sql), 10) + + ## like or like + query_sql = f'select * from {tb_name} where c7 like "binar12345" or c8 like "nchar_"' + tdSql.query(query_sql) + tdSql.checkRows(1) + tdSql.checkEqual(self.queryLastC10(query_sql), 9) + + ## like and like or condition_A + query_sql = f'select * from {tb_name} where c7 like "binary_" and c8 like "ncha_" or c1 != 1' + tdSql.query(query_sql) + tdSql.checkRows(2) + tdSql.checkEqual(self.queryLastC10(query_sql), 8) + + ## like or like and condition_A + query_sql = f'select * from {tb_name} where c7 like ("binar_") or c8 like ("nchar_") and c10 != 8' + tdSql.query(query_sql) + tdSql.checkRows(9) + tdSql.checkEqual(self.queryLastC10(query_sql), 10) + + ## like or like or condition_A + query_sql = f'select * from {tb_name} where c7 like ("binary_") or c8 like ("nchar_") or c10 = 6' + tdSql.query(query_sql) + tdSql.checkRows(3) + tdSql.checkEqual(self.queryLastC10(query_sql), 9) + + ## like or like or like or like + query_sql = f'select * from {tb_name} where c7 like ("binary_") or c8 like ("nchar_") or c10 = 6 or c7 is Null' + tdSql.query(query_sql) + tdSql.checkRows(4) + tdSql.checkEqual(self.queryLastC10(query_sql), 11) + + ## like or like and like or like + query_sql = f'select * from {tb_name} where c7 like ("binary_") or c8 like ("ncha_") and c10 = 6 or c10 = 9' + tdSql.query(query_sql) + tdSql.checkRows(3) + tdSql.checkEqual(self.queryLastC10(query_sql), 9) + + ## condition_A or like or condition_B and like + query_sql = f'select * from {tb_name} where c1 = 2 or c7 like "binary_" or c10 = 3 and c8 like "ncha%"' + tdSql.query(query_sql) + tdSql.checkRows(3) + tdSql.checkEqual(self.queryLastC10(query_sql), 8) + + ## like and condition_A or like and like and condition_B + query_sql = f'select * from {tb_name} where c7 like "bin%" and c2 = 3 or c10 <= 4 and c7 like "binar_" and c8 like "ncha_"' + tdSql.query(query_sql) + tdSql.checkRows(4) + tdSql.checkEqual(self.queryLastC10(query_sql), 4) + + ## (like and condition_A or like) and like and condition_B + query_sql = f'select * from {tb_name} where (c7 like "bin%" and c2 = 3 or c8 like "nchar_") and c7 like "binar_" and c9 != false' + tdSql.query(query_sql) + tdSql.checkRows(2) + tdSql.checkEqual(self.queryLastC10(query_sql), 9) + + def checkTbPreCal(self): + ''' + Ordinary table precal + ''' + tb_name = self.initTb() + ## avg sum condition_A or condition_B + query_sql = f'select avg(c3), sum(c3) from {tb_name} where c10 = 5 or c8 is Null' + res = tdSql.query(query_sql, True)[0] + tdSql.checkEqual(int(res[0]), 3) + tdSql.checkEqual(int(res[1]), 6) + + ## avg sum condition_A or condition_B or condition_C + query_sql = f'select avg(c3), sum(c3) from {tb_name} where c10 = 4 or c8 is Null or c9 = false ' + res = tdSql.query(query_sql, True)[0] + tdSql.checkEqual(int(res[0]), 2) + tdSql.checkEqual(int(res[1]), 6) + + ## count avg sum condition_A or condition_B or condition_C interval + query_sql = f'select count(*), avg(c3), sum(c3) from {tb_name} where c10 = 4 or c8 is Null or c9 = false interval(16d)' + res = tdSql.query(query_sql, True) + tdSql.checkRows(2) + tdSql.checkEqual(int(res[0][1]), 1) + tdSql.checkEqual(int(res[0][2]), 4) + tdSql.checkEqual(int(res[0][3]), 4) + tdSql.checkEqual(int(res[1][1]), 2) + tdSql.checkEqual(int(res[1][2]), 1) + tdSql.checkEqual(int(res[1][3]), 2) + + ## count avg sum condition_A or condition_B or in and like or condition_C interval + query_sql = f'select count(*), sum(c3) from {tb_name} where c10 = 4 or c8 is Null or c2 in (1, 2) and c7 like "binary_" or c1 <> 1 interval(16d)' + res = tdSql.query(query_sql, True) + tdSql.checkRows(2) + tdSql.checkEqual(int(res[0][1]), 2) + tdSql.checkEqual(int(res[0][2]), 5) + tdSql.checkEqual(int(res[1][1]), 2) + tdSql.checkEqual(int(res[1][2]), 2) + + def queryMultiTb(self): + ''' + test "or" in multi ordinary table + ''' + tdCom.cleanTb() + tb_name = self.initTb() + ## select from (condition_A or condition_B) + query_sql = f'select c10 from (select * from {tb_name} where c1 >1 or c2 >=3)' + tdSql.query(query_sql) + tdSql.checkRows(3) + tdSql.checkEqual(self.queryLastC10(query_sql, True), 11) + + + + # tb_name1 = tdCom.getLongName(8, "letters") + # tb_name2 = tdCom.getLongName(8, "letters") + # tb_name3 = tdCom.getLongName(8, "letters") + # tdSql.execute( + # f"CREATE TABLE {tb_name1} (ts timestamp, c1 tinyint, c2 smallint, c3 int)") + # tdSql.execute( + # f"CREATE TABLE {tb_name2} (ts timestamp, c1 tinyint, c2 smallint, c3 int)") + # tdSql.execute( + # f"CREATE TABLE {tb_name3} (ts timestamp, c1 tinyint, c2 smallint, c3 int)") + # insert_sql_list = [f'insert into {tb_name1} values ("2021-01-01 12:00:00", 1, 5, 1)', + # f'insert into {tb_name1} values ("2021-01-03 12:00:00", 2, 4, 1)', + # f'insert into {tb_name1} values ("2021-01-05 12:00:00", 3, 2, 1)', + # f'insert into {tb_name2} values ("2021-01-01 12:00:00", 4, 2, 1)', + # f'insert into {tb_name2} values ("2021-01-02 12:00:00", 5, 1, 1)', + # f'insert into {tb_name2} values ("2021-01-04 12:00:00", 1, 2, 1)', + # f'insert into {tb_name3} values ("2021-01-02 12:00:00", 4, 2, 1)', + # f'insert into {tb_name3} values ("2021-01-06 12:00:00", 5, 1, 1)', + # f'insert into {tb_name3} values ("2021-01-07 12:00:00", 1, 2, 1)', + # ] + # for sql in insert_sql_list: + # tdSql.execute(sql) + # tdSql.query( + # f'select * from {tb_name1} t1, {tb_name2}, {tb_name3} t3 t2 where (t1.ts=t2.ts or t2.ts=t3.ts)') + # tdSql.checkRows(4) + + + def run(self): + tdSql.prepare() + self.checkTbColTypeOperator() + self.checkTbMultiExpression() + self.checkTbMultiIn() + self.checkTbMultiLike() + self.checkTbPreCal() + self.queryMultiTb() + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) From 617c59f4a8db05ed803b28f7329a0ad493afcab6 Mon Sep 17 00:00:00 2001 From: jiajingbin Date: Wed, 18 Aug 2021 21:27:21 +0800 Subject: [PATCH 11/30] save --- tests/pytest/query/queryDiffColsOr.py | 30 +++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/tests/pytest/query/queryDiffColsOr.py b/tests/pytest/query/queryDiffColsOr.py index 68c8f14869..d07867ee27 100644 --- a/tests/pytest/query/queryDiffColsOr.py +++ b/tests/pytest/query/queryDiffColsOr.py @@ -389,7 +389,37 @@ class TDTestCase: tdSql.checkRows(3) tdSql.checkEqual(self.queryLastC10(query_sql, True), 11) + ## select from (condition_A or condition_B) where condition_A or condition_B + query_sql = f'select c10 from (select * from {tb_name} where c1 >1 or c2 >=3) where c1 =2 or c4 = 2' + tdSql.query(query_sql) + tdSql.checkRows(2) + tdSql.checkEqual(self.queryLastC10(query_sql, True), 3) + ## select from (condition_A or condition_B and like and in) where condition_A or condition_B or like and in + query_sql = f'select c10 from (select * from {tb_name} where c1 >1 or c2 = 2 and c7 like "binar_" and c4 in (3, 5)) where c1 != 2 or c3 = 1 or c8 like "ncha_" and c9 in (true)' + tdSql.query(query_sql) + tdSql.checkRows(7) + tdSql.checkEqual(self.queryLastC10(query_sql, True), 10) + + ## select count avg sum from (condition_A or condition_B and like and in) where condition_A or condition_B or like and in interval + query_sql = f'select count(*), avg(c6), sum(c3) from (select * from {tb_name} where c1 >1 or c2 = 2 and c7 like "binar_" and c4 in (3, 5)) where c1 != 2 or c3 = 1 or c8 like "ncha_" and c9 in (true) interval(8d)' + res = tdSql.query(query_sql, True) + tdSql.checkRows(3) + tdSql.checkEqual(int(res[0][1]), 3) + tdSql.checkEqual(int(res[0][2]), 1) + tdSql.checkEqual(int(res[0][3]), 10) + tdSql.checkEqual(int(res[1][1]), 3) + tdSql.checkEqual(int(res[1][2]), 3) + tdSql.checkEqual(int(res[1][3]), 3) + tdSql.checkEqual(int(res[2][1]), 1) + tdSql.checkEqual(int(res[2][2]), 1) + tdSql.checkEqual(int(res[2][3]), 1) + + ## cname + query_sql = f'select c10 from (select * from {tb_name} where c1 >1 or c2 = 2 and c7 like "binar_" and c4 in (3, 5)) a where a.c1 != 2 or a.c3 = 1 or a.c8 like "ncha_" and a.c9 in (true)' + tdSql.query(query_sql) + tdSql.checkRows(7) + tdSql.checkEqual(self.queryLastC10(query_sql, True), 10) # tb_name1 = tdCom.getLongName(8, "letters") # tb_name2 = tdCom.getLongName(8, "letters") From ad0d0ab93d60741509fc27480e148ec16f2d289a Mon Sep 17 00:00:00 2001 From: jiajingbin Date: Thu, 19 Aug 2021 15:36:10 +0800 Subject: [PATCH 12/30] save --- tests/pytest/query/queryDiffColsOr.py | 167 +++++++++++++++++++------- 1 file changed, 123 insertions(+), 44 deletions(-) diff --git a/tests/pytest/query/queryDiffColsOr.py b/tests/pytest/query/queryDiffColsOr.py index d07867ee27..6f32a15727 100644 --- a/tests/pytest/query/queryDiffColsOr.py +++ b/tests/pytest/query/queryDiffColsOr.py @@ -46,6 +46,16 @@ class TDTestCase: self.insertData(tb_name) return tb_name + def initStb(self): + tdCom.cleanTb() + tb_name = tdCom.getLongName(8, "letters") + tdSql.execute( + f"CREATE TABLE {tb_name} (ts timestamp, c1 tinyint, c2 smallint, c3 int, c4 bigint, c5 float, c6 double, c7 binary(100), c8 nchar(200), c9 bool, c10 int) tags (t1 tinyint, t2 smallint, t3 int, t4 bigint, t5 float, t6 double, t7 binary(100), t8 nchar(200), t9 bool, t10 int)") + tdSql.execute( + f'CREATE TABLE {tb_name}_sub using {tb_name} tags (1, 1, 1, 3, 1.1, 1.1, "binary", "nchar", true, 1)') + self.insertData(f'{tb_name}_sub') + return tb_name + def queryLastC10(self, query_sql, multi=False): if multi: res = tdSql.query(query_sql.replace('c10', 'last(*)'), True) @@ -120,18 +130,7 @@ class TDTestCase: tdSql.checkRows(10) tdSql.checkEqual(self.queryLastC10(query_sql), 11) - def checkTbColTypeOperator(self): - ''' - Ordinary table full column type and operator - ''' - tb_name = self.initTb() - self.queryFullColType(tb_name) - - def checkTbMultiExpression(self): - ''' - Ordinary table multiExpression - ''' - tb_name = self.initTb() + def queryMultiExpression(self, tb_name): ## condition_A and condition_B or condition_C (> < >=) query_sql = f'select * from {tb_name} where c1 > 2 and c2 < 4 or c3 >= 4' tdSql.query(query_sql) @@ -209,12 +208,8 @@ class TDTestCase: tdSql.query(query_sql) tdSql.checkRows(2) tdSql.checkEqual(self.queryLastC10(query_sql), 5) - - def checkTbMultiIn(self): - ''' - Ordinary table multiIn - ''' - tb_name = self.initTb() + + def queryMultiIn(self, tb_name): ## in and in query_sql = f'select * from {tb_name} where c7 in ("binary") and c8 in ("nchar")' tdSql.query(query_sql) @@ -275,11 +270,7 @@ class TDTestCase: tdSql.checkRows(1) tdSql.checkEqual(self.queryLastC10(query_sql), 4) - def checkTbMultiLike(self): - ''' - Ordinary table multiLike - ''' - tb_name = self.initTb() + def queryMultiLike(self, tb_name): ## like and like query_sql = f'select * from {tb_name} where c7 like "bi%" and c8 like ("ncha_")' tdSql.query(query_sql) @@ -340,11 +331,7 @@ class TDTestCase: tdSql.checkRows(2) tdSql.checkEqual(self.queryLastC10(query_sql), 9) - def checkTbPreCal(self): - ''' - Ordinary table precal - ''' - tb_name = self.initTb() + def queryPreCal(self, tb_name): ## avg sum condition_A or condition_B query_sql = f'select avg(c3), sum(c3) from {tb_name} where c10 = 5 or c8 is Null' res = tdSql.query(query_sql, True)[0] @@ -377,29 +364,24 @@ class TDTestCase: tdSql.checkEqual(int(res[1][1]), 2) tdSql.checkEqual(int(res[1][2]), 2) - def queryMultiTb(self): - ''' - test "or" in multi ordinary table - ''' - tdCom.cleanTb() - tb_name = self.initTb() + def queryMultiTb(self, tb_name): ## select from (condition_A or condition_B) query_sql = f'select c10 from (select * from {tb_name} where c1 >1 or c2 >=3)' - tdSql.query(query_sql) + res = tdSql.query(query_sql, True) tdSql.checkRows(3) - tdSql.checkEqual(self.queryLastC10(query_sql, True), 11) + tdSql.checkEqual(int(res[2][0]), 11) ## select from (condition_A or condition_B) where condition_A or condition_B query_sql = f'select c10 from (select * from {tb_name} where c1 >1 or c2 >=3) where c1 =2 or c4 = 2' - tdSql.query(query_sql) + res = tdSql.query(query_sql, True) tdSql.checkRows(2) - tdSql.checkEqual(self.queryLastC10(query_sql, True), 3) + tdSql.checkEqual(int(res[1][0]), 3) ## select from (condition_A or condition_B and like and in) where condition_A or condition_B or like and in query_sql = f'select c10 from (select * from {tb_name} where c1 >1 or c2 = 2 and c7 like "binar_" and c4 in (3, 5)) where c1 != 2 or c3 = 1 or c8 like "ncha_" and c9 in (true)' - tdSql.query(query_sql) + res = tdSql.query(query_sql, True) tdSql.checkRows(7) - tdSql.checkEqual(self.queryLastC10(query_sql, True), 10) + tdSql.checkEqual(int(res[6][0]), 10) ## select count avg sum from (condition_A or condition_B and like and in) where condition_A or condition_B or like and in interval query_sql = f'select count(*), avg(c6), sum(c3) from (select * from {tb_name} where c1 >1 or c2 = 2 and c7 like "binar_" and c4 in (3, 5)) where c1 != 2 or c3 = 1 or c8 like "ncha_" and c9 in (true) interval(8d)' @@ -415,12 +397,103 @@ class TDTestCase: tdSql.checkEqual(int(res[2][2]), 1) tdSql.checkEqual(int(res[2][3]), 1) - ## cname + # cname query_sql = f'select c10 from (select * from {tb_name} where c1 >1 or c2 = 2 and c7 like "binar_" and c4 in (3, 5)) a where a.c1 != 2 or a.c3 = 1 or a.c8 like "ncha_" and a.c9 in (true)' - tdSql.query(query_sql) + res = tdSql.query(query_sql, True) tdSql.checkRows(7) - tdSql.checkEqual(self.queryLastC10(query_sql, True), 10) + tdSql.checkEqual(int(res[6][0]), 10) + # ## multi cname + query_sql = f'select b.c10 from (select * from {tb_name} where c9 = true or c2 = 2) a, (select * from {tb_name} where c7 like "binar_" or c4 in (3, 5)) b where a.ts = b.ts' + res = tdSql.query(query_sql, True) + tdSql.checkRows(10) + tdSql.checkEqual(int(res[9][0]), 10) + + def checkTbColTypeOperator(self): + ''' + Ordinary table full column type and operator + ''' + tb_name = self.initTb() + self.queryFullColType(tb_name) + + def checkStbColTypeOperator(self): + ''' + Super table full column type and operator + ''' + tb_name = self.initStb() + self.queryFullColType(tb_name) + + def checkTbMultiExpression(self): + ''' + Ordinary table multiExpression + ''' + tb_name = self.initTb() + self.queryMultiExpression(tb_name) + + def checkStbMultiExpression(self): + ''' + Super table multiExpression + ''' + tb_name = self.initStb() + self.queryMultiExpression(tb_name) + + def checkTbMultiIn(self): + ''' + Ordinary table multiIn + ''' + tb_name = self.initTb() + self.queryMultiIn(tb_name) + + def checkStbMultiIn(self): + ''' + Super table multiIn + ''' + tb_name = self.initStb() + self.queryMultiIn(tb_name) + + def checkTbMultiLike(self): + ''' + Ordinary table multiLike + ''' + tb_name = self.initTb() + self.queryMultiLike(tb_name) + + def checkStbMultiLike(self): + ''' + Super table multiLike + ''' + tb_name = self.initStb() + self.queryMultiLike(tb_name) + + def checkTbPreCal(self): + ''' + Ordinary table precal + ''' + tb_name = self.initTb() + self.queryPreCal(tb_name) + + def checkStbPreCal(self): + ''' + Super table precal + ''' + tb_name = self.initStb() + self.queryPreCal(tb_name) + + def checkMultiTb(self): + ''' + test "or" in multi ordinary table + ''' + tb_name = self.initTb() + self.queryMultiTb(tb_name) + + def checkMultiStb(self): + ''' + test "or" in multi super table + ''' + tb_name = self.initStb() + self.queryMultiTb(tb_name) + + # tb_name1 = tdCom.getLongName(8, "letters") # tb_name2 = tdCom.getLongName(8, "letters") # tb_name3 = tdCom.getLongName(8, "letters") @@ -450,11 +523,17 @@ class TDTestCase: def run(self): tdSql.prepare() self.checkTbColTypeOperator() + self.checkStbColTypeOperator() self.checkTbMultiExpression() + self.checkStbMultiExpression() self.checkTbMultiIn() + self.checkStbMultiIn() self.checkTbMultiLike() + self.checkStbMultiLike() self.checkTbPreCal() - self.queryMultiTb() + self.checkStbPreCal() + self.checkMultiTb() + self.checkMultiStb() def stop(self): From 3ef88ec1969bec595a757784e19504072f154e94 Mon Sep 17 00:00:00 2001 From: jiajingbin Date: Fri, 20 Aug 2021 09:58:06 +0800 Subject: [PATCH 13/30] save --- tests/pytest/query/queryDiffColsOr.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/pytest/query/queryDiffColsOr.py b/tests/pytest/query/queryDiffColsOr.py index 6f32a15727..feeab84a7e 100644 --- a/tests/pytest/query/queryDiffColsOr.py +++ b/tests/pytest/query/queryDiffColsOr.py @@ -397,13 +397,13 @@ class TDTestCase: tdSql.checkEqual(int(res[2][2]), 1) tdSql.checkEqual(int(res[2][3]), 1) - # cname + ## cname query_sql = f'select c10 from (select * from {tb_name} where c1 >1 or c2 = 2 and c7 like "binar_" and c4 in (3, 5)) a where a.c1 != 2 or a.c3 = 1 or a.c8 like "ncha_" and a.c9 in (true)' res = tdSql.query(query_sql, True) tdSql.checkRows(7) tdSql.checkEqual(int(res[6][0]), 10) - # ## multi cname + ## multi cname query_sql = f'select b.c10 from (select * from {tb_name} where c9 = true or c2 = 2) a, (select * from {tb_name} where c7 like "binar_" or c4 in (3, 5)) b where a.ts = b.ts' res = tdSql.query(query_sql, True) tdSql.checkRows(10) From 1517dd9de64952e3d2cc2be7b846765a1a8c7b29 Mon Sep 17 00:00:00 2001 From: jiajingbin Date: Fri, 20 Aug 2021 19:20:49 +0800 Subject: [PATCH 14/30] save --- .../tools/schemalessInsertPerformance.py | 228 ++++++++++++++++++ 1 file changed, 228 insertions(+) create mode 100644 tests/pytest/tools/schemalessInsertPerformance.py diff --git a/tests/pytest/tools/schemalessInsertPerformance.py b/tests/pytest/tools/schemalessInsertPerformance.py new file mode 100644 index 0000000000..3d44f023b4 --- /dev/null +++ b/tests/pytest/tools/schemalessInsertPerformance.py @@ -0,0 +1,228 @@ +################################################################### +# Copyright (c) 2021 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import traceback +import random +import string +from taos.error import LinesError +import datetime +import time +from copy import deepcopy +import numpy as np +from util.log import * +from util.cases import * +from util.sql import * +from util.common import tdCom +import threading + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + self._conn = conn + + def genRandomTs(self): + year = random.randint(2000, 2021) + month = random.randint(10, 12) + day = random.randint(10, 29) + hour = random.randint(10, 24) + minute = random.randint(10, 59) + second = random.randint(10, 59) + m_second = random.randint(101, 199) + date_time = f'{year}-{month}-{day} {hour}:{minute}:{second}' + print(date_time) + timeArray = time.strptime(date_time, "%Y-%m-%d %H:%M:%S") + ts = int(time.mktime(timeArray)) + print("------", ts) + # timestamp = time.mktime(datetime.datetime.strptime(date_time, "%Y-%m-%d %H:%M:%S.%f").timetuple()) + return f'{ts}s' + + def genMultiColStr(self, int_count=4, double_count=0, binary_count=0): + """ + genType must be tag/col + """ + col_str = "" + if double_count == 0 and binary_count == 0: + for i in range(0, int_count): + if i < (int_count-1): + col_str += f'c{i}={random.randint(0, 255)}i32,' + else: + col_str += f'c{i}={random.randint(0, 255)}i32 ' + elif double_count > 0 and binary_count == 0: + for i in range(0, int_count): + col_str += f'c{i}={random.randint(0, 255)}i32,' + for i in range(0, double_count): + if i < (double_count-1): + col_str += f'c{i+int_count}={random.randint(1, 255)}.{i}f64,' + else: + col_str += f'c{i+int_count}={random.randint(1, 255)}.{i}f64 ' + elif double_count == 0 and binary_count > 0: + for i in range(0, int_count): + col_str += f'c{i}={random.randint(0, 255)}i32,' + for i in range(0, binary_count): + if i < (binary_count-1): + col_str += f'c{i+int_count}=\"{tdCom.getLongName(5, "letters")}\",' + else: + col_str += f'c{i+int_count}=\"{tdCom.getLongName(5, "letters")}\" ' + elif double_count > 0 and binary_count > 0: + for i in range(0, int_count): + col_str += f'c{i}={random.randint(0, 255)}i32,' + for i in range(0, double_count): + col_str += f'c{i+int_count}={random.randint(1, 255)}.{i}f64,' + for i in range(0, binary_count): + if i < (binary_count-1): + col_str += f'c{i+int_count+double_count}=\"{tdCom.getLongName(5, "letters")}\",' + else: + col_str += f'c{i+int_count+double_count}=\"{tdCom.getLongName(5, "letters")}\" ' + return col_str + + def genLongSql(self, int_count=4, double_count=0, binary_count=0, init=False): + if init: + tag_str = f'id="init",t0={random.randint(0, 65535)}i32,t1=\"{tdCom.getLongName(10, "letters")}\"' + else: + tag_str = f'id=sub_{tdCom.getLongName(5, "letters")}_{tdCom.getLongName(5, "letters")},t0={random.randint(0, 65535)}i32,t1=\"{tdCom.getLongName(10, "letters")}\"' + col_str = self.genMultiColStr(int_count, double_count, binary_count) + long_sql = 'stb' + ',' + tag_str + ' ' + col_str + '0' + return long_sql + + def getPerfSql(self, count=4, init=False): + if count == 4: + input_sql = self.genLongSql(init=init) + elif count == 1000: + input_sql = self.genLongSql(400, 400, 200, init=init) + elif count == 4000: + input_sql = self.genLongSql(1900, 1900, 200, init=init) + return input_sql + + def tableGenerator(self, count=4, table_count=1000): + for i in range(table_count): + yield self.getPerfSql(count) + + def genTableList(self, count=4, table_count=10000): + table_list = list() + for i in range(1, table_count+1): + table_list.append(self.getPerfSql(count)) + return table_list + + def splitTableList(self, count=4, thread_count=10, table_count=10000): + per_list_len = int(table_count/thread_count) + table_list = self.genTableList(count=count) + # ts = int(time.time()) + list_of_group = zip(*(iter(table_list),) *per_list_len) + end_list = [list(i) for i in list_of_group] # i is a tuple + count = len(table_list) % per_list_len + end_list.append(table_list[-count:]) if count !=0 else end_list + return table_list, end_list + + def replaceLastStr(self, str, new): + list_ori = list(str) + list_ori[-1] = new + return ''.join(list_ori) + + def genDataList(self, table_list, row_count=10): + data_list = list() + ts = int(time.time()) + for table_str in table_list: + for i in range(1, row_count+1): + ts -= 1 + table_str_new = self.replaceLastStr(table_str, str(ts)) + data_list.append(table_str_new) + print(data_list) + return data_list + + def perfTableInsert(self): + table_generator = self.tableGenerator() + for input_sql in table_generator: + self._conn.insert_lines([input_sql]) + for i in range(10): + self._conn.insert_lines([input_sql]) + + def perfDataInsert(self, input_sql): + for i in range(10000): + self._conn.insert_lines([input_sql]) + + def genTableThread(self, thread_count=10): + threads = list() + for i in range(thread_count): + t = threading.Thread(target=self.perfTableInsert) + threads.append(t) + return threads + + def genMultiThread(self, input_sql, thread_count=10): + threads = list() + for i in range(thread_count): + t = threading.Thread(target=self.perfDataInsert,args=(input_sql,)) + threads.append(t) + return threads + + def multiThreadRun(self, threads): + for t in threads: + t.start() + for t in threads: + t.join() + + def createStb(self, count=4): + input_sql = self.getPerfSql(count=count, init=True) + print("stb-----", input_sql) + self._conn.insert_lines([input_sql]) + + # def createTb(self, count=4): + # input_sql = self.getPerfSql(count=count) + # for i in range(10000): + # self._conn.insert_lines([input_sql]) + + # def createTb1(self, count=4): + # start_time = time.time() + # self.multiThreadRun(self.genMultiThread(input_sql)) + # end_time = time.time() + # return end_time - start_time + + def calInsertTableTime(self): + start_time = time.time() + self.createStb() + self.multiThreadRun(self.genTableThread()) + end_time = time.time() + return end_time - start_time + + def calRunTime(self, input_sql): + start_time = time.time() + self.multiThreadRun(self.genMultiThread(input_sql)) + end_time = time.time() + return end_time - start_time + + def schemalessInsertPerfTest(self, count=4): + input_sql = self.getPerfSql(count) + self.calRunTime(input_sql) + + def test(self): + sql1 = 'stb,id="init",t0=14865i32,t1="tvnqbjuqck" c0=37i32,c1=217i32,c2=3i32,c3=88i32 1626006833640ms' + sql2 = 'stb,id="init",t0=14865i32,t1="tvnqbjuqck" c0=38i32,c1=217i32,c2=3i32,c3=88i32 1626006833641ms' + self._conn.insert_lines([sql1]) + self._conn.insert_lines([sql2]) + def run(self): + print("running {}".format(__file__)) + tdSql.prepare() + # print(self.genRandomTs()) + # self.calInsertTableTime() + # self.test() + table_list = self.splitTableList()[0] + data_list = self.genDataList(table_list) + print(len(data_list)) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) From a3b56039ba3afd98287eee4d91625d2e09e6d70e Mon Sep 17 00:00:00 2001 From: jiajingbin Date: Mon, 23 Aug 2021 17:51:09 +0800 Subject: [PATCH 15/30] save for tag TD-6293 --- .../tools/schemalessInsertPerformance.py | 124 ++++++++++++++---- 1 file changed, 97 insertions(+), 27 deletions(-) diff --git a/tests/pytest/tools/schemalessInsertPerformance.py b/tests/pytest/tools/schemalessInsertPerformance.py index 3d44f023b4..0dcbc5ffe2 100644 --- a/tests/pytest/tools/schemalessInsertPerformance.py +++ b/tests/pytest/tools/schemalessInsertPerformance.py @@ -91,7 +91,7 @@ class TDTestCase: if init: tag_str = f'id="init",t0={random.randint(0, 65535)}i32,t1=\"{tdCom.getLongName(10, "letters")}\"' else: - tag_str = f'id=sub_{tdCom.getLongName(5, "letters")}_{tdCom.getLongName(5, "letters")},t0={random.randint(0, 65535)}i32,t1=\"{tdCom.getLongName(10, "letters")}\"' + tag_str = f'id="sub_{tdCom.getLongName(5, "letters")}_{tdCom.getLongName(5, "letters")}",t0={random.randint(0, 65535)}i32,t1=\"{tdCom.getLongName(10, "letters")}\"' col_str = self.genMultiColStr(int_count, double_count, binary_count) long_sql = 'stb' + ',' + tag_str + ' ' + col_str + '0' return long_sql @@ -109,13 +109,19 @@ class TDTestCase: for i in range(table_count): yield self.getPerfSql(count) + + + + + + def genTableList(self, count=4, table_count=10000): table_list = list() for i in range(1, table_count+1): table_list.append(self.getPerfSql(count)) return table_list - def splitTableList(self, count=4, thread_count=10, table_count=10000): + def splitTableList(self, count=4, thread_count=10, table_count=1000): per_list_len = int(table_count/thread_count) table_list = self.genTableList(count=count) # ts = int(time.time()) @@ -125,6 +131,22 @@ class TDTestCase: end_list.append(table_list[-count:]) if count !=0 else end_list return table_list, end_list + def rowsGenerator(self, end_list): + ts = int(time.time()) + input_sql_list = list() + for elm_list in end_list: + for elm in elm_list: + for i in range(1, 10000): + ts -= 1 + elm_new = self.replaceLastStr(elm, str(ts)) + 's' + input_sql_list.append(elm_new) + yield input_sql_list + + # def insertRows(self, count=4, thread_count=10): + # table_list = self.splitTableList(count=count, thread_count=thread_count)[0] + # for + + def replaceLastStr(self, str, new): list_ori = list(str) list_ori[-1] = new @@ -136,21 +158,39 @@ class TDTestCase: for table_str in table_list: for i in range(1, row_count+1): ts -= 1 - table_str_new = self.replaceLastStr(table_str, str(ts)) + table_str_new = self.replaceLastStr(table_str, f'{str(ts)}s') data_list.append(table_str_new) print(data_list) return data_list + + def insertRows(self, count=4, table_count=1000): + table_generator = self.tableGenerator(count=count, table_count=table_count) + for table_name in table_generator: + pass + def perfTableInsert(self): table_generator = self.tableGenerator() for input_sql in table_generator: self._conn.insert_lines([input_sql]) - for i in range(10): - self._conn.insert_lines([input_sql]) + # for i in range(10): + # self._conn.insert_lines([input_sql]) - def perfDataInsert(self, input_sql): - for i in range(10000): + def perfDataInsert(self, count=4): + table_generator = self.tableGenerator(count=count) + ts = int(time.time()) + for input_sql in table_generator: + print("input_sql-----------", input_sql) self._conn.insert_lines([input_sql]) + for i in range(100000): + ts -= 1 + input_sql_new = self.replaceLastStr(input_sql, str(ts)) + 's' + print("input_sql_new---------", input_sql_new) + self._conn.insert_lines([input_sql_new]) + + def batchInsertTable(self, batch_list): + for insert_list in batch_list: + self._conn.insert_lines(insert_list) def genTableThread(self, thread_count=10): threads = list() @@ -159,10 +199,10 @@ class TDTestCase: threads.append(t) return threads - def genMultiThread(self, input_sql, thread_count=10): + def genMultiThread(self, count, thread_count=10): threads = list() for i in range(thread_count): - t = threading.Thread(target=self.perfDataInsert,args=(input_sql,)) + t = threading.Thread(target=self.perfDataInsert,args=(count,)) threads.append(t) return threads @@ -174,9 +214,24 @@ class TDTestCase: def createStb(self, count=4): input_sql = self.getPerfSql(count=count, init=True) - print("stb-----", input_sql) self._conn.insert_lines([input_sql]) + def threadInsertTable(self, end_list, thread_count=10): + threads = list() + for i in range(thread_count): + t = threading.Thread(target=self.batchInsertTable, args=(end_list,)) + threads.append(t) + return threads + + + def finalRun(self): + self.createStb() + table_list, end_list = self.splitTableList() + batchInsertTableThread = self.threadInsertTable(end_list=end_list) + print(end_list) + self.multiThreadRun(batchInsertTableThread) + # print(end_list) + # def createTb(self, count=4): # input_sql = self.getPerfSql(count=count) # for i in range(10000): @@ -188,37 +243,52 @@ class TDTestCase: # end_time = time.time() # return end_time - start_time - def calInsertTableTime(self): + # def calInsertTableTime(self): + # start_time = time.time() + # self.createStb() + # self.multiThreadRun(self.genMultiThread()) + # end_time = time.time() + # return end_time - start_time + + def calRunTime(self, count=4): start_time = time.time() self.createStb() - self.multiThreadRun(self.genTableThread()) + self.multiThreadRun(self.genMultiThread(count=count)) end_time = time.time() return end_time - start_time - def calRunTime(self, input_sql): + def calRunTime1(self, count=4): start_time = time.time() - self.multiThreadRun(self.genMultiThread(input_sql)) - end_time = time.time() - return end_time - start_time + self.createStb() + self.multiThreadRun(self.perfTableInsert()) + # self.perfTableInsert() - def schemalessInsertPerfTest(self, count=4): - input_sql = self.getPerfSql(count) - self.calRunTime(input_sql) + # def schemalessInsertPerfTest(self, count=4): + # input_sql = self.getPerfSql(count) + # self.calRunTime(input_sql) + + # def test(self): + # sql1 = 'stb,id="init",t0=14865i32,t1="tvnqbjuqck" c0=37i32,c1=217i32,c2=3i32,c3=88i32 1626006833640ms' + # sql2 = 'stb,id="init",t0=14865i32,t1="tvnqbjuqck" c0=38i32,c1=217i32,c2=3i32,c3=88i32 1626006833641ms' + # self._conn.insert_lines([sql1]) + # self._conn.insert_lines([sql2]) - def test(self): - sql1 = 'stb,id="init",t0=14865i32,t1="tvnqbjuqck" c0=37i32,c1=217i32,c2=3i32,c3=88i32 1626006833640ms' - sql2 = 'stb,id="init",t0=14865i32,t1="tvnqbjuqck" c0=38i32,c1=217i32,c2=3i32,c3=88i32 1626006833641ms' - self._conn.insert_lines([sql1]) - self._conn.insert_lines([sql2]) def run(self): print("running {}".format(__file__)) tdSql.prepare() + self.finalRun() + # print(self.calRunTime1(count=4)) + # print(self.calRunTime(count=4)) # print(self.genRandomTs()) # self.calInsertTableTime() # self.test() - table_list = self.splitTableList()[0] - data_list = self.genDataList(table_list) - print(len(data_list)) + # table_list = self.splitTableList()[0] + # data_list = self.genDataList(table_list) + # print(len(data_list)) + # end_list = [['stb,id="sub_vzvfx_dbuxp",t0=9961i32,t1="zjjfayhfep" c0=83i32,c1=169i32,c2=177i32,c3=4i32 0','stb,id="sub_vzvfx_dbuxp",t0=9961i32,t1="zjjfayhfep" c0=83i32,c1=169i32,c2=177i32,c3=4i32 0'], ['stb,id="sub_vzvfx_dbuxp",t0=9961i32,t1="zjjfayhfep" c0=83i32,c1=169i32,c2=177i32,c3=4i32 0','stb,id="sub_vzvfx_dbuxp",t0=9961i32,t1="zjjfayhfep" c0=83i32,c1=169i32,c2=177i32,c3=4i32 0']] + # rowsGenerator = self.rowsGenerator(end_list) + # for i in rowsGenerator: + # print(i) def stop(self): tdSql.close() From 9126417086cb6d7f28d6952cc7f613869b2b347c Mon Sep 17 00:00:00 2001 From: jiajingbin Date: Tue, 24 Aug 2021 09:47:06 +0800 Subject: [PATCH 16/30] add common.py --- tests/pytest/util/common.py | 53 +++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 tests/pytest/util/common.py diff --git a/tests/pytest/util/common.py b/tests/pytest/util/common.py new file mode 100644 index 0000000000..35abc4802f --- /dev/null +++ b/tests/pytest/util/common.py @@ -0,0 +1,53 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import random +import string +from util.sql import tdSql + +class TDCom: + def init(self, conn, logSql): + tdSql.init(conn.cursor(), logSql) + + def cleanTb(self): + query_sql = "show stables" + res_row_list = tdSql.query(query_sql, True) + stb_list = map(lambda x: x[0], res_row_list) + for stb in stb_list: + tdSql.execute(f'drop table if exists {stb}') + + query_sql = "show tables" + res_row_list = tdSql.query(query_sql, True) + tb_list = map(lambda x: x[0], res_row_list) + for tb in tb_list: + tdSql.execute(f'drop table if exists {tb}') + + def getLongName(self, len, mode = "mixed"): + """ + generate long name + mode could be numbers/letters/letters_mixed/mixed + """ + if mode == "numbers": + chars = ''.join(random.choice(string.digits) for i in range(len)) + elif mode == "letters": + chars = ''.join(random.choice(string.ascii_letters.lower()) for i in range(len)) + elif mode == "letters_mixed": + chars = ''.join(random.choice(string.ascii_letters.upper() + string.ascii_letters.lower()) for i in range(len)) + else: + chars = ''.join(random.choice(string.ascii_letters.lower() + string.digits) for i in range(len)) + return chars + + def close(self): + self.cursor.close() + +tdCom = TDCom() From 1380e6c4ba10aacac99add769ea4575cac6f2af9 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Tue, 24 Aug 2021 17:21:35 +0800 Subject: [PATCH 17/30] add log information to printlines --- tests/pytest/tools/schemalessInsertPerformance.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/pytest/tools/schemalessInsertPerformance.py b/tests/pytest/tools/schemalessInsertPerformance.py index 0dcbc5ffe2..68802310a2 100644 --- a/tests/pytest/tools/schemalessInsertPerformance.py +++ b/tests/pytest/tools/schemalessInsertPerformance.py @@ -190,7 +190,10 @@ class TDTestCase: def batchInsertTable(self, batch_list): for insert_list in batch_list: + print(threading.current_thread().name, "length=", len(insert_list)) + print(threading.current_thread().name, 'firstline', insert_list[0], 'lastline:', insert_list[-1]) self._conn.insert_lines(insert_list) + print(threading.current_thread().name, 'end') def genTableThread(self, thread_count=10): threads = list() @@ -228,7 +231,6 @@ class TDTestCase: self.createStb() table_list, end_list = self.splitTableList() batchInsertTableThread = self.threadInsertTable(end_list=end_list) - print(end_list) self.multiThreadRun(batchInsertTableThread) # print(end_list) From b6896fc5f9e480f3d64bd970827d8cefe23e372e Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Tue, 24 Aug 2021 17:43:33 +0800 Subject: [PATCH 18/30] before insert retry, refer to needRetryInsert --- src/client/src/tscParseLineProtocol.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/src/tscParseLineProtocol.c b/src/client/src/tscParseLineProtocol.c index bde68c306a..9fb6cb4717 100644 --- a/src/client/src/tscParseLineProtocol.c +++ b/src/client/src/tscParseLineProtocol.c @@ -1127,7 +1127,7 @@ static int32_t applyDataPoints(TAOS* taos, TAOS_SML_DATA_POINT* points, int32_t tscDebug("SML:0x%"PRIx64" apply child table points. child table: %s", info->id, point->childTableName); code = applyChildTableFields(taos, sTableSchema, point->childTableName, cTablePoints, info); if (code != 0) { - tscError("Apply child table fields failed. child table %s, error %s", point->childTableName, tstrerror(code)); + tscError("SML:0x%"PRIx64" Apply child table fields failed. child table %s, error %s", info->id, point->childTableName, tstrerror(code)); goto cleanup; } From 88109bbdbc0ca2364e951050e290d058b1f7330f Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Tue, 24 Aug 2021 21:51:50 +0800 Subject: [PATCH 19/30] add retry for invalid table id and invalid group id --- src/client/src/tscParseLineProtocol.c | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/src/client/src/tscParseLineProtocol.c b/src/client/src/tscParseLineProtocol.c index 9fb6cb4717..1bce10f95e 100644 --- a/src/client/src/tscParseLineProtocol.c +++ b/src/client/src/tscParseLineProtocol.c @@ -1084,10 +1084,28 @@ static int32_t applyChildTableFields(TAOS* taos, SSmlSTableSchema* sTableSchema, taosArrayPush(rowsBind, &colBinds); } - code = insertChildTableBatch(taos, cTableName, sTableSchema->fields, rowsBind, info); - if (code != 0) { - tscError("SML:0x%"PRIx64" insert into child table %s failed. error %s", info->id, cTableName, tstrerror(code)); - } + int32_t retry = 0; + bool retryAgain = false; + do { + code = insertChildTableBatch(taos, cTableName, sTableSchema->fields, rowsBind, info); + if (code != 0) { + tscError("SML:0x%" PRIx64 " insert into child table %s failed. error %s, retry %d", info->id, cTableName, tstrerror(code), retry); + if (code == TSDB_CODE_TDB_INVALID_TABLE_ID || code == TSDB_CODE_VND_INVALID_VGROUP_ID) { + TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); + int32_t code2 = taos_errno(res2); + if (code2 != TSDB_CODE_SUCCESS) { + tscError("SML:0x%" PRIx64 " apply schema action. reset query cache. error: %s", info->id, taos_errstr(res2)); + } + taos_free_result(res2); + retryAgain = (++retry < TSDB_MAX_REPLICA) ? true : false; + if (retryAgain) { + taosMsleep(100 * (2 << (retry))); + } else { + tscError("SML:0x%" PRIx64 " insert into child table %s reached max retry", info->id, cTableName); + } + } + } + } while (retryAgain); for (int i = 0; i < rows; ++i) { TAOS_BIND* colBinds = taosArrayGetP(rowsBind, i); From 532fb5f7c1a02008990269e2b42ae67ee1925a5d Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Tue, 24 Aug 2021 22:04:28 +0800 Subject: [PATCH 20/30] show firstline and lastline of schemaless performace python scripts --- tests/pytest/tools/schemalessInsertPerformance.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/pytest/tools/schemalessInsertPerformance.py b/tests/pytest/tools/schemalessInsertPerformance.py index 68802310a2..5e009a2129 100644 --- a/tests/pytest/tools/schemalessInsertPerformance.py +++ b/tests/pytest/tools/schemalessInsertPerformance.py @@ -191,7 +191,8 @@ class TDTestCase: def batchInsertTable(self, batch_list): for insert_list in batch_list: print(threading.current_thread().name, "length=", len(insert_list)) - print(threading.current_thread().name, 'firstline', insert_list[0], 'lastline:', insert_list[-1]) + print(threading.current_thread().name, 'firstline', insert_list[0]) + print(threading.current_thread().name, 'lastline:', insert_list[-1]) self._conn.insert_lines(insert_list) print(threading.current_thread().name, 'end') From 1408c0c0ad3ff3ffbc63ea2c9c8da194d486ff2f Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Wed, 25 Aug 2021 08:35:04 +0800 Subject: [PATCH 21/30] schemaless: fix schema modification error --- src/client/src/tscParseLineProtocol.c | 49 ++++++++++++++------------- 1 file changed, 26 insertions(+), 23 deletions(-) diff --git a/src/client/src/tscParseLineProtocol.c b/src/client/src/tscParseLineProtocol.c index 1bce10f95e..e3ed89a591 100644 --- a/src/client/src/tscParseLineProtocol.c +++ b/src/client/src/tscParseLineProtocol.c @@ -849,6 +849,8 @@ static int32_t insertChildTableBatch(TAOS* taos, char* cTableName, SArray* cols return code; } + bool tryAgain = false; + do { code = taos_stmt_set_tbname(stmt, cTableName); if (code != 0) { @@ -878,7 +880,26 @@ static int32_t insertChildTableBatch(TAOS* taos, char* cTableName, SArray* cols if (code != 0) { tscError("SML:0x%"PRIx64" taos_stmt_execute return %d:%s", info->id, code, tstrerror(code)); } - } while (code == TSDB_CODE_TDB_TABLE_RECONFIGURE && try++ < TSDB_MAX_REPLICA); + + tryAgain = false; + if ((code == TSDB_CODE_TDB_INVALID_TABLE_ID + || code == TSDB_CODE_VND_INVALID_VGROUP_ID + || code == TSDB_CODE_TDB_TABLE_RECONFIGURE) && try++ < TSDB_MAX_REPLICA) { + tryAgain = true; + } + + if (code == TSDB_CODE_TDB_INVALID_TABLE_ID || code == TSDB_CODE_VND_INVALID_VGROUP_ID) { + TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); + int32_t code2 = taos_errno(res2); + if (code2 != TSDB_CODE_SUCCESS) { + tscError("SML:0x%" PRIx64 " apply schema action. reset query cache. error: %s", info->id, taos_errstr(res2)); + } + taos_free_result(res2); + if (tryAgain) { + taosMsleep(100 * (2 << (try))); + } + } + } while (tryAgain); if (code != 0) { tscError("SML:0x%"PRIx64" %d:%s", info->id, code, tstrerror(code)); @@ -1084,28 +1105,10 @@ static int32_t applyChildTableFields(TAOS* taos, SSmlSTableSchema* sTableSchema, taosArrayPush(rowsBind, &colBinds); } - int32_t retry = 0; - bool retryAgain = false; - do { - code = insertChildTableBatch(taos, cTableName, sTableSchema->fields, rowsBind, info); - if (code != 0) { - tscError("SML:0x%" PRIx64 " insert into child table %s failed. error %s, retry %d", info->id, cTableName, tstrerror(code), retry); - if (code == TSDB_CODE_TDB_INVALID_TABLE_ID || code == TSDB_CODE_VND_INVALID_VGROUP_ID) { - TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); - int32_t code2 = taos_errno(res2); - if (code2 != TSDB_CODE_SUCCESS) { - tscError("SML:0x%" PRIx64 " apply schema action. reset query cache. error: %s", info->id, taos_errstr(res2)); - } - taos_free_result(res2); - retryAgain = (++retry < TSDB_MAX_REPLICA) ? true : false; - if (retryAgain) { - taosMsleep(100 * (2 << (retry))); - } else { - tscError("SML:0x%" PRIx64 " insert into child table %s reached max retry", info->id, cTableName); - } - } - } - } while (retryAgain); + code = insertChildTableBatch(taos, cTableName, sTableSchema->fields, rowsBind, info); + if (code != 0) { + tscError("SML:0x%"PRIx64" insert into child table %s failed. error %s", info->id, cTableName, tstrerror(code)); + } for (int i = 0; i < rows; ++i) { TAOS_BIND* colBinds = taosArrayGetP(rowsBind, i); From 0ffb24e73fb961c63507c081b0480c2de04f6d09 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Wed, 25 Aug 2021 08:51:53 +0800 Subject: [PATCH 22/30] schemaless: fix spelling error and add try times --- src/client/src/tscParseLineProtocol.c | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/src/client/src/tscParseLineProtocol.c b/src/client/src/tscParseLineProtocol.c index e3ed89a591..acb28533a9 100644 --- a/src/client/src/tscParseLineProtocol.c +++ b/src/client/src/tscParseLineProtocol.c @@ -832,7 +832,6 @@ static int32_t insertChildTableBatch(TAOS* taos, char* cTableName, SArray* cols tscDebug("SML:0x%"PRIx64" insert rows into child table %s. num of rows: %zu", info->id, cTableName, taosArrayGetSize(rowsBind)); int32_t code = 0; - int32_t try = 0; TAOS_STMT* stmt = taos_stmt_init(taos); if (stmt == NULL) { @@ -850,7 +849,7 @@ static int32_t insertChildTableBatch(TAOS* taos, char* cTableName, SArray* cols } bool tryAgain = false; - + int32_t try = 0; do { code = taos_stmt_set_tbname(stmt, cTableName); if (code != 0) { @@ -878,7 +877,7 @@ static int32_t insertChildTableBatch(TAOS* taos, char* cTableName, SArray* cols code = taos_stmt_execute(stmt); if (code != 0) { - tscError("SML:0x%"PRIx64" taos_stmt_execute return %d:%s", info->id, code, tstrerror(code)); + tscError("SML:0x%"PRIx64" taos_stmt_execute return %d:%s, try:%d", info->id, code, tstrerror(code), try); } tryAgain = false; @@ -892,7 +891,7 @@ static int32_t insertChildTableBatch(TAOS* taos, char* cTableName, SArray* cols TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); int32_t code2 = taos_errno(res2); if (code2 != TSDB_CODE_SUCCESS) { - tscError("SML:0x%" PRIx64 " apply schema action. reset query cache. error: %s", info->id, taos_errstr(res2)); + tscError("SML:0x%" PRIx64 " insert child table. reset query cache. error: %s", info->id, taos_errstr(res2)); } taos_free_result(res2); if (tryAgain) { @@ -901,9 +900,6 @@ static int32_t insertChildTableBatch(TAOS* taos, char* cTableName, SArray* cols } } while (tryAgain); - if (code != 0) { - tscError("SML:0x%"PRIx64" %d:%s", info->id, code, tstrerror(code)); - } taos_stmt_close(stmt); return code; From e39f1ef2b904688a3997e8587a749a0067957506 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Wed, 25 Aug 2021 09:08:06 +0800 Subject: [PATCH 23/30] schemaless:add database not ready and network unavailable processing --- src/client/src/tscParseLineProtocol.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/client/src/tscParseLineProtocol.c b/src/client/src/tscParseLineProtocol.c index acb28533a9..c0ed1feb6b 100644 --- a/src/client/src/tscParseLineProtocol.c +++ b/src/client/src/tscParseLineProtocol.c @@ -883,11 +883,14 @@ static int32_t insertChildTableBatch(TAOS* taos, char* cTableName, SArray* cols tryAgain = false; if ((code == TSDB_CODE_TDB_INVALID_TABLE_ID || code == TSDB_CODE_VND_INVALID_VGROUP_ID - || code == TSDB_CODE_TDB_TABLE_RECONFIGURE) && try++ < TSDB_MAX_REPLICA) { + || code == TSDB_CODE_TDB_TABLE_RECONFIGURE + || code == TSDB_CODE_APP_NOT_READY + || code == TSDB_CODE_RPC_NETWORK_UNAVAIL) && try++ < TSDB_MAX_REPLICA) { tryAgain = true; } - if (code == TSDB_CODE_TDB_INVALID_TABLE_ID || code == TSDB_CODE_VND_INVALID_VGROUP_ID) { + if (code == TSDB_CODE_APP_NOT_READY || code == TSDB_CODE_RPC_NETWORK_UNAVAIL || + code == TSDB_CODE_TDB_INVALID_TABLE_ID || code == TSDB_CODE_VND_INVALID_VGROUP_ID) { TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); int32_t code2 = taos_errno(res2); if (code2 != TSDB_CODE_SUCCESS) { From 954dc3798101522fef211b1a3595cf2ca3fd032e Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Wed, 25 Aug 2021 09:13:41 +0800 Subject: [PATCH 24/30] schemaless:decrease sleep time during errors --- src/client/src/tscParseLineProtocol.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/src/tscParseLineProtocol.c b/src/client/src/tscParseLineProtocol.c index c0ed1feb6b..801d32a1ee 100644 --- a/src/client/src/tscParseLineProtocol.c +++ b/src/client/src/tscParseLineProtocol.c @@ -898,7 +898,7 @@ static int32_t insertChildTableBatch(TAOS* taos, char* cTableName, SArray* cols } taos_free_result(res2); if (tryAgain) { - taosMsleep(100 * (2 << (try))); + taosMsleep(50 * (2 << (try))); } } } while (tryAgain); From b5fb8921b6527520320fdde7c2e2a900d59519d4 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Wed, 25 Aug 2021 16:03:53 +0800 Subject: [PATCH 25/30] schemaless: fix error code definition --- src/client/src/tscParseLineProtocol.c | 21 +++++++++++++-------- src/client/src/tscSQLParser.c | 4 ++-- src/inc/taoserror.h | 8 ++++---- src/util/src/terror.c | 6 ++++++ 4 files changed, 25 insertions(+), 14 deletions(-) diff --git a/src/client/src/tscParseLineProtocol.c b/src/client/src/tscParseLineProtocol.c index 801d32a1ee..c66800620a 100644 --- a/src/client/src/tscParseLineProtocol.c +++ b/src/client/src/tscParseLineProtocol.c @@ -889,16 +889,21 @@ static int32_t insertChildTableBatch(TAOS* taos, char* cTableName, SArray* cols tryAgain = true; } - if (code == TSDB_CODE_APP_NOT_READY || code == TSDB_CODE_RPC_NETWORK_UNAVAIL || + if (code == TSDB_CODE_APP_NOT_READY || code == TSDB_CODE_TDB_INVALID_TABLE_ID || code == TSDB_CODE_VND_INVALID_VGROUP_ID) { - TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); - int32_t code2 = taos_errno(res2); - if (code2 != TSDB_CODE_SUCCESS) { - tscError("SML:0x%" PRIx64 " insert child table. reset query cache. error: %s", info->id, taos_errstr(res2)); - } - taos_free_result(res2); +// TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); +// int32_t code2 = taos_errno(res2); +// if (code2 != TSDB_CODE_SUCCESS) { +// tscError("SML:0x%" PRIx64 " insert child table. reset query cache. error: %s", info->id, taos_errstr(res2)); +// } +// taos_free_result(res2); if (tryAgain) { - taosMsleep(50 * (2 << (try))); + taosMsleep(50 * (2 << try)); + } + } + if (code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { + if (tryAgain) { + taosMsleep( 50 * (2 << try)); } } } while (tryAgain); diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 2a2e4540e1..f80d3e7fd1 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -6319,7 +6319,7 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { } if (pItem->bytes <= pColSchema->bytes) { - return tscErrorMsgWithCode(TSDB_CODE_TSC_INVALID_COLUMN_LENGTH, tscGetErrorMsgPayload(pCmd), pItem->name, NULL); + return tscErrorMsgWithCode(TSDB_CODE_TSC_INVALID_COLUMN_LENGTH, pMsg, pItem->name, NULL); } SSchema* pSchema = (SSchema*) pTableMetaInfo->pTableMeta->schema; @@ -6370,7 +6370,7 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { } if (pItem->bytes <= pColSchema->bytes) { - return tscErrorMsgWithCode(TSDB_CODE_TSC_INVALID_TAG_LENGTH, tscGetErrorMsgPayload(pCmd), pItem->name, NULL); + return tscErrorMsgWithCode(TSDB_CODE_TSC_INVALID_TAG_LENGTH, pMsg, pItem->name, NULL); } SSchema* pSchema = tscGetTableTagSchema(pTableMetaInfo->pTableMeta); diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index 08f6d0bf4d..86ef53e959 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -105,7 +105,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_TSC_NO_META_CACHED TAOS_DEF_ERROR_CODE(0, 0x021C) //"No table meta cached") #define TSDB_CODE_TSC_DUP_COL_NAMES TAOS_DEF_ERROR_CODE(0, 0x021D) //"duplicated column names") #define TSDB_CODE_TSC_INVALID_TAG_LENGTH TAOS_DEF_ERROR_CODE(0, 0x021E) //"Invalid tag length") -#define TSDB_CODE_TSC_INVALID_COLUMN_LENGTH TAOS_DEF_ERROR_CODE(0, 0x021F) //"Invalid tag length") +#define TSDB_CODE_TSC_INVALID_COLUMN_LENGTH TAOS_DEF_ERROR_CODE(0, 0x021F) //"Invalid tag length") // mnode #define TSDB_CODE_MND_MSG_NOT_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0300) //"Message not processed") @@ -188,6 +188,9 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_INVALID_FUNC TAOS_DEF_ERROR_CODE(0, 0x0374) //"Invalid func") #define TSDB_CODE_MND_INVALID_FUNC_BUFSIZE TAOS_DEF_ERROR_CODE(0, 0x0375) //"Invalid func bufSize") +#define TSDB_CODE_MND_INVALID_TAG_LENGTH TAOS_DEF_ERROR_CODE(0, 0x0376) //"invalid tag length") +#define TSDB_CODE_MND_INVALID_COLUMN_LENGTH TAOS_DEF_ERROR_CODE(0, 0x0377) //"invalid column length") + #define TSDB_CODE_MND_DB_NOT_SELECTED TAOS_DEF_ERROR_CODE(0, 0x0380) //"Database not specified or available") #define TSDB_CODE_MND_DB_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0381) //"Database already exists") #define TSDB_CODE_MND_INVALID_DB_OPTION TAOS_DEF_ERROR_CODE(0, 0x0382) //"Invalid database options") @@ -214,9 +217,6 @@ int32_t* taosGetErrno(); #define TSDB_CODE_DND_TOO_MANY_VNODES TAOS_DEF_ERROR_CODE(0, 0x0405) //"Too many vnode directories") #define TSDB_CODE_DND_EXITING TAOS_DEF_ERROR_CODE(0, 0x0406) //"Dnode is exiting" -#define TSDB_CODE_MND_INVALID_TAG_LENGTH TAOS_DEF_ERROR_CODE(0, 0x410) //"invalid tag length") -#define TSDB_CODE_MND_INVALID_COLUMN_LENGTH TAOS_DEF_ERROR_CODE(0, 0x411) //"invalid column length") - // vnode #define TSDB_CODE_VND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0500) //"Action in progress") #define TSDB_CODE_VND_MSG_NOT_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0501) //"Message not processed") diff --git a/src/util/src/terror.c b/src/util/src/terror.c index 49e46cdde8..addaf54e78 100644 --- a/src/util/src/terror.c +++ b/src/util/src/terror.c @@ -112,6 +112,9 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TSC_EXCEED_SQL_LIMIT, "SQL statement too lon TAOS_DEFINE_ERROR(TSDB_CODE_TSC_FILE_EMPTY, "File is empty") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_LINE_SYNTAX_ERROR, "Syntax error in Line") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_NO_META_CACHED, "No table meta cached") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_DUP_COL_NAMES, "duplicated column names") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_TAG_LENGTH, "Invalid tag length") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_COLUMN_LENGTH, "Invalid tag length") // mnode TAOS_DEFINE_ERROR(TSDB_CODE_MND_MSG_NOT_PROCESSED, "Message not processed") @@ -194,6 +197,9 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_FUNC_ALREADY_EXIST, "Func already exists") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC, "Invalid func") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_BUFSIZE, "Invalid func bufSize") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TAG_LENGTH, "invalid tag length") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_COLUMN_LENGTH, "invalid column length") + TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_NOT_SELECTED, "Database not specified or available") TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_ALREADY_EXIST, "Database already exists") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_OPTION, "Invalid database options") From ae58e4cb60ef2d812810d2c5fd701ba1a24a53b1 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Wed, 25 Aug 2021 16:42:59 +0800 Subject: [PATCH 26/30] schemaless: fix error description --- src/inc/taoserror.h | 2 +- src/util/src/terror.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index 86ef53e959..c401ab762e 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -105,7 +105,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_TSC_NO_META_CACHED TAOS_DEF_ERROR_CODE(0, 0x021C) //"No table meta cached") #define TSDB_CODE_TSC_DUP_COL_NAMES TAOS_DEF_ERROR_CODE(0, 0x021D) //"duplicated column names") #define TSDB_CODE_TSC_INVALID_TAG_LENGTH TAOS_DEF_ERROR_CODE(0, 0x021E) //"Invalid tag length") -#define TSDB_CODE_TSC_INVALID_COLUMN_LENGTH TAOS_DEF_ERROR_CODE(0, 0x021F) //"Invalid tag length") +#define TSDB_CODE_TSC_INVALID_COLUMN_LENGTH TAOS_DEF_ERROR_CODE(0, 0x021F) //"Invalid column length") // mnode #define TSDB_CODE_MND_MSG_NOT_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0300) //"Message not processed") diff --git a/src/util/src/terror.c b/src/util/src/terror.c index addaf54e78..311abe70df 100644 --- a/src/util/src/terror.c +++ b/src/util/src/terror.c @@ -114,7 +114,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TSC_LINE_SYNTAX_ERROR, "Syntax error in Line" TAOS_DEFINE_ERROR(TSDB_CODE_TSC_NO_META_CACHED, "No table meta cached") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_DUP_COL_NAMES, "duplicated column names") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_TAG_LENGTH, "Invalid tag length") -TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_COLUMN_LENGTH, "Invalid tag length") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_COLUMN_LENGTH, "Invalid column length") // mnode TAOS_DEFINE_ERROR(TSDB_CODE_MND_MSG_NOT_PROCESSED, "Message not processed") From a243d2b91d4630464568fb0088d5af9cb10e1fa6 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Wed, 25 Aug 2021 17:08:09 +0800 Subject: [PATCH 27/30] schemaless: fix errors about modify_column --- src/client/src/tscParseLineProtocol.c | 12 ++++++------ src/mnode/src/mnodeTable.c | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/client/src/tscParseLineProtocol.c b/src/client/src/tscParseLineProtocol.c index c66800620a..52e4bd3d40 100644 --- a/src/client/src/tscParseLineProtocol.c +++ b/src/client/src/tscParseLineProtocol.c @@ -891,12 +891,12 @@ static int32_t insertChildTableBatch(TAOS* taos, char* cTableName, SArray* cols if (code == TSDB_CODE_APP_NOT_READY || code == TSDB_CODE_TDB_INVALID_TABLE_ID || code == TSDB_CODE_VND_INVALID_VGROUP_ID) { -// TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); -// int32_t code2 = taos_errno(res2); -// if (code2 != TSDB_CODE_SUCCESS) { -// tscError("SML:0x%" PRIx64 " insert child table. reset query cache. error: %s", info->id, taos_errstr(res2)); -// } -// taos_free_result(res2); + TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); + int32_t code2 = taos_errno(res2); + if (code2 != TSDB_CODE_SUCCESS) { + tscError("SML:0x%" PRIx64 " insert child table. reset query cache. error: %s", info->id, taos_errstr(res2)); + } + taos_free_result(res2); if (tryAgain) { taosMsleep(50 * (2 << try)); } diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index 1bc5607da5..7a0108026e 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -1518,7 +1518,6 @@ static int32_t mnodeChangeSuperTableColumn(SMnodeMsg *pMsg) { // update SSchema *schema = (SSchema *) (pStable->schema + col); ASSERT(schema->type == TSDB_DATA_TYPE_BINARY || schema->type == TSDB_DATA_TYPE_NCHAR); - schema->bytes = pAlter->schema[0].bytes; if (pAlter->schema[0].bytes <= schema->bytes) { mError("msg:%p, app:%p stable:%s, modify column len. column:%s, len from %d to %d", pMsg, pMsg->rpcMsg.ahandle, @@ -1526,6 +1525,7 @@ static int32_t mnodeChangeSuperTableColumn(SMnodeMsg *pMsg) { return TSDB_CODE_MND_INVALID_COLUMN_LENGTH; } + schema->bytes = pAlter->schema[0].bytes; pStable->sversion++; mInfo("msg:%p, app:%p stable %s, start to modify column %s len to %d", pMsg, pMsg->rpcMsg.ahandle, pStable->info.tableId, name, schema->bytes); From 934c2c045350a5a8637c47a3a5d6dcb50543f886 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Thu, 26 Aug 2021 08:28:06 +0800 Subject: [PATCH 28/30] schemaless:REMOVE TAG VALUE PROCESSING FOR NAMED CHILD TABLES --- src/client/src/tscParseLineProtocol.c | 107 +------------------------- 1 file changed, 1 insertion(+), 106 deletions(-) diff --git a/src/client/src/tscParseLineProtocol.c b/src/client/src/tscParseLineProtocol.c index 52e4bd3d40..b01e4daa05 100644 --- a/src/client/src/tscParseLineProtocol.c +++ b/src/client/src/tscParseLineProtocol.c @@ -705,42 +705,6 @@ static int32_t modifyDBSchemas(TAOS* taos, SArray* stableSchemas, SSmlLinesInfo* return 0; } -static int32_t changeChildTableTagValue(TAOS* taos, const char* cTableName, const char* tagName, TAOS_BIND* bind, SSmlLinesInfo* info) { - char sql[512]; - sprintf(sql, "alter table %s set tag %s=?", cTableName, tagName); - - int32_t code; - TAOS_STMT* stmt = taos_stmt_init(taos); - code = taos_stmt_prepare(stmt, sql, (unsigned long)strlen(sql)); - - if (code != 0) { - tscError("SML:0x%"PRIx64" taos_stmt_prepare return %d:%s", info->id, code, tstrerror(code)); - taos_stmt_close(stmt); - return code; - } - - code = taos_stmt_bind_param(stmt, bind); - if (code != 0) { - tscError("SML:0x%"PRIx64" taos_stmt_bind_param return %d:%s", info->id, code, tstrerror(code)); - taos_stmt_close(stmt); - return code; - } - - code = taos_stmt_execute(stmt); - if (code != 0) { - tscError("SML:0x%"PRIx64" taos_stmt_execute return %d:%s", info->id, code, tstrerror(code)); - taos_stmt_close(stmt); - return code; - } - - code = taos_stmt_close(stmt); - if (code != 0) { - tscError("SML:0x%"PRIx64" taos_stmt_close return %d:%s", info->id, code, tstrerror(code)); - return code; - } - return code; -} - static int32_t creatChildTableIfNotExists(TAOS* taos, const char* cTableName, const char* sTableName, SArray* tagsSchema, SArray* tagsBind, SSmlLinesInfo* info) { size_t numTags = taosArrayGetSize(tagsSchema); @@ -970,15 +934,6 @@ static int32_t applyChildTableTags(TAOS* taos, char* cTableName, char* sTableNam tagKVs[*pFieldSchemaIdx] = kv; } } - - int32_t notNullTagsIndices[TSDB_MAX_TAGS] = {0}; - int32_t numNotNullTags = 0; - for (int32_t i = 0; i < numTags; ++i) { - if (tagKVs[i] != NULL) { - notNullTagsIndices[numNotNullTags] = i; - ++numNotNullTags; - } - } SArray* tagBinds = taosArrayInit(numTags, sizeof(TAOS_BIND)); taosArraySetSize(tagBinds, numTags); @@ -1001,68 +956,8 @@ static int32_t applyChildTableTags(TAOS* taos, char* cTableName, char* sTableNam bind->is_null = NULL; } - // select tag1,tag2,... from stable where tbname in (ctable) - char* sql = malloc(tsMaxSQLStringLen+1); - int freeBytes = tsMaxSQLStringLen + 1; - snprintf(sql, freeBytes, "select tbname, "); - for (int i = 0; i < numNotNullTags ; ++i) { - snprintf(sql + strlen(sql), freeBytes-strlen(sql), "%s,", tagKVs[notNullTagsIndices[i]]->key); - } - snprintf(sql + strlen(sql) - 1, freeBytes - strlen(sql) + 1, - " from %s where tbname in (\'%s\')", sTableName, cTableName); - sql[strlen(sql)] = '\0'; + int32_t code = creatChildTableIfNotExists(taos, cTableName, sTableName, sTableSchema->tags, tagBinds, info); - TAOS_RES* result = taos_query(taos, sql); - free(sql); - - int32_t code = taos_errno(result); - if (code != 0) { - tscError("SML:0x%"PRIx64" get child table %s tags failed. error string %s", info->id, cTableName, taos_errstr(result)); - goto cleanup; - } - - // check tag value and set tag values if different - TAOS_ROW row = taos_fetch_row(result); - if (row != NULL) { - int numFields = taos_field_count(result); - TAOS_FIELD* fields = taos_fetch_fields(result); - int* lengths = taos_fetch_lengths(result); - for (int i = 1; i < numFields; ++i) { - uint8_t dbType = fields[i].type; - int32_t length = lengths[i]; - char* val = row[i]; - - TAOS_SML_KV* tagKV = tagKVs[notNullTagsIndices[i-1]]; - if (tagKV->type != dbType) { - tscError("SML:0x%"PRIx64" child table %s tag %s type mismatch. point type : %d, db type : %d", - info->id, cTableName, tagKV->key, tagKV->type, dbType); - return TSDB_CODE_TSC_INVALID_VALUE; - } - - assert(tagKV->value); - - if (val == NULL || length != tagKV->length || memcmp(tagKV->value, val, length) != 0) { - uintptr_t valPointer = (uintptr_t)tagKV; - size_t* pFieldSchemaIdx = taosHashGet(info->smlDataToSchema, &valPointer, sizeof(uintptr_t)); - assert(pFieldSchemaIdx != NULL); - TAOS_BIND* bind = taosArrayGet(tagBinds, *pFieldSchemaIdx); - code = changeChildTableTagValue(taos, cTableName, tagKV->key, bind, info); - if (code != 0) { - tscError("SML:0x%"PRIx64" change child table tag failed. table name %s, tag %s", info->id, cTableName, tagKV->key); - goto cleanup; - } - } - } - tscDebug("SML:0x%"PRIx64" successfully applied point tags. child table: %s", info->id, cTableName); - } else { - code = creatChildTableIfNotExists(taos, cTableName, sTableName, sTableSchema->tags, tagBinds, info); - if (code != 0) { - goto cleanup; - } - } - -cleanup: - taos_free_result(result); for (int i = 0; i < taosArrayGetSize(tagBinds); ++i) { TAOS_BIND* bind = taosArrayGet(tagBinds, i); free(bind->length); From 1e244ce6408bbf519c2d339d8cb8d0e4169f04a2 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Thu, 26 Aug 2021 08:33:19 +0800 Subject: [PATCH 29/30] schemaless: change database not ready processing to sleep without clear global meta cache --- src/client/src/tscParseLineProtocol.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/client/src/tscParseLineProtocol.c b/src/client/src/tscParseLineProtocol.c index b01e4daa05..6c30c78643 100644 --- a/src/client/src/tscParseLineProtocol.c +++ b/src/client/src/tscParseLineProtocol.c @@ -853,8 +853,7 @@ static int32_t insertChildTableBatch(TAOS* taos, char* cTableName, SArray* cols tryAgain = true; } - if (code == TSDB_CODE_APP_NOT_READY || - code == TSDB_CODE_TDB_INVALID_TABLE_ID || code == TSDB_CODE_VND_INVALID_VGROUP_ID) { + if (code == TSDB_CODE_TDB_INVALID_TABLE_ID || code == TSDB_CODE_VND_INVALID_VGROUP_ID) { TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); int32_t code2 = taos_errno(res2); if (code2 != TSDB_CODE_SUCCESS) { @@ -865,7 +864,7 @@ static int32_t insertChildTableBatch(TAOS* taos, char* cTableName, SArray* cols taosMsleep(50 * (2 << try)); } } - if (code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { + if (code == TSDB_CODE_APP_NOT_READY || code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { if (tryAgain) { taosMsleep( 50 * (2 << try)); } From a2a845faebc9bceaba06b913bacc83148d65e1b7 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Thu, 26 Aug 2021 08:43:37 +0800 Subject: [PATCH 30/30] [TD-6238]:Not get meta again when tsParseSql succceeded for stmt insert in tscTableMetaCallback --- src/client/src/tscAsync.c | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/client/src/tscAsync.c b/src/client/src/tscAsync.c index eaad783331..6b12cd0da0 100644 --- a/src/client/src/tscAsync.c +++ b/src/client/src/tscAsync.c @@ -363,15 +363,6 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) { } if (TSDB_QUERY_HAS_TYPE(pCmd->insertParam.insertType, TSDB_QUERY_TYPE_STMT_INSERT)) { // stmt insert - STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - code = tscGetTableMeta(pSql, pTableMetaInfo); - if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { - taosReleaseRef(tscObjRef, pSql->self); - return; - } else { - assert(code == TSDB_CODE_SUCCESS); - } - (*pSql->fp)(pSql->param, pSql, code); } else if (TSDB_QUERY_HAS_TYPE(pCmd->insertParam.insertType, TSDB_QUERY_TYPE_FILE_INSERT)) { // file insert tscImportDataFromFile(pSql);