From f171e8033a4141f7d0c93568fd41989411fb0517 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Thu, 12 Aug 2021 12:24:41 +0800 Subject: [PATCH 01/40] [TD-xxxx]:begin coding taos_sml_insert --- src/client/src/tscParseLineProtocol.c | 199 ++++++++++++++------------ 1 file changed, 111 insertions(+), 88 deletions(-) diff --git a/src/client/src/tscParseLineProtocol.c b/src/client/src/tscParseLineProtocol.c index 2a16f1aad6..43509d3533 100644 --- a/src/client/src/tscParseLineProtocol.c +++ b/src/client/src/tscParseLineProtocol.c @@ -32,9 +32,6 @@ typedef struct { uint8_t type; int16_t length; char* value; - - //=================================== - uint32_t fieldSchemaIdx; } TAOS_SML_KV; typedef struct { @@ -47,9 +44,6 @@ typedef struct { // first kv must be timestamp TAOS_SML_KV* fields; int32_t fieldNum; - - //================================ - uint32_t schemaIdx; } TAOS_SML_DATA_POINT; typedef enum { @@ -62,10 +56,23 @@ typedef enum { typedef struct { uint64_t id; - + SHashObj* smlDataToSchema; } SSmlLinesInfo; + //================================================================================================= +static uint64_t linesSmlHandleId = 0; + +uint64_t genLinesSmlId() { + uint64_t id; + + do { + id = atomic_add_fetch_64(&linesSmlHandleId, 1); + } while (id == 0); + + return id; +} + int compareSmlColKv(const void* p1, const void* p2) { TAOS_SML_KV* kv1 = (TAOS_SML_KV*)p1; TAOS_SML_KV* kv2 = (TAOS_SML_KV*)p2; @@ -168,11 +175,46 @@ static int32_t buildSmlKvSchema(TAOS_SML_KV* smlKv, SHashObj* hash, SArray* arra taosHashPut(hash, field.name, tagKeyLen, &fieldIdx, sizeof(fieldIdx)); } - smlKv->fieldSchemaIdx = (uint32_t)fieldIdx; + uintptr_t valPointer = (uintptr_t)smlKv; + taosHashPut(info->smlDataToSchema, &valPointer, sizeof(uintptr_t), &fieldIdx, sizeof(fieldIdx)); return 0; } +static int32_t getSmlMd5ChildTableName(TAOS_SML_DATA_POINT* point, char* tableName, int* tableNameLen, + SSmlLinesInfo* info) { + tscDebug("SML:0x%"PRIx64" taos_sml_insert get child table name through md5", info->id); + qsort(point->tags, point->tagNum, sizeof(TAOS_SML_KV), compareSmlColKv); + + SStringBuilder sb; memset(&sb, 0, sizeof(sb)); + char sTableName[TSDB_TABLE_NAME_LEN] = {0}; + strtolower(sTableName, point->stableName); + taosStringBuilderAppendString(&sb, sTableName); + for (int j = 0; j < point->tagNum; ++j) { + taosStringBuilderAppendChar(&sb, ','); + TAOS_SML_KV* tagKv = point->tags + j; + char tagName[TSDB_COL_NAME_LEN] = {0}; + strtolower(tagName, tagKv->key); + taosStringBuilderAppendString(&sb, tagName); + taosStringBuilderAppendChar(&sb, '='); + taosStringBuilderAppend(&sb, tagKv->value, tagKv->length); + } + size_t len = 0; + char* keyJoined = taosStringBuilderGetResult(&sb, &len); + MD5_CTX context; + MD5Init(&context); + MD5Update(&context, (uint8_t *)keyJoined, (uint32_t)len); + MD5Final(&context); + *tableNameLen = snprintf(tableName, *tableNameLen, + "t_%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x", context.digest[0], + context.digest[1], context.digest[2], context.digest[3], context.digest[4], context.digest[5], context.digest[6], + context.digest[7], context.digest[8], context.digest[9], context.digest[10], context.digest[11], + context.digest[12], context.digest[13], context.digest[14], context.digest[15]); + taosStringBuilderDestroy(&sb); + tscDebug("SML:0x%"PRIx64" child table name: %s", info->id, tableName); + return 0; +} + static int32_t buildDataPointSchemas(TAOS_SML_DATA_POINT* points, int numPoint, SArray* stableSchemas, SSmlLinesInfo* info) { int32_t code = 0; SHashObj* sname2shema = taosHashInit(32, @@ -203,6 +245,15 @@ static int32_t buildDataPointSchemas(TAOS_SML_DATA_POINT* points, int numPoint, for (int j = 0; j < point->tagNum; ++j) { TAOS_SML_KV* tagKv = point->tags + j; + if (!point->childTableName) { + char childTableName[TSDB_TABLE_NAME_LEN]; + int32_t tableNameLen = TSDB_TABLE_NAME_LEN; + getSmlMd5ChildTableName(point, childTableName, &tableNameLen, info); + point->childTableName = calloc(1, tableNameLen+1); + strncpy(point->childTableName, childTableName, tableNameLen); + point->childTableName[tableNameLen] = '\0'; + } + code = buildSmlKvSchema(tagKv, pStableSchema->tagHash, pStableSchema->tags, info); if (code != 0) { tscError("SML:0x%"PRIx64" build data point schema failed. point no.: %d, tag key: %s", info->id, i, tagKv->key); @@ -219,7 +270,8 @@ static int32_t buildDataPointSchemas(TAOS_SML_DATA_POINT* points, int numPoint, } } - point->schemaIdx = (uint32_t)stableIdx; + uintptr_t valPointer = (uintptr_t)point; + taosHashPut(info->smlDataToSchema, &valPointer, sizeof(uintptr_t), &stableIdx, sizeof(stableIdx)); } size_t numStables = taosArrayGetSize(stableSchemas); @@ -567,41 +619,6 @@ static int32_t modifyDBSchemas(TAOS* taos, SArray* stableSchemas, SSmlLinesInfo* return 0; } -static int32_t getSmlMd5ChildTableName(TAOS_SML_DATA_POINT* point, char* tableName, int* tableNameLen, - SSmlLinesInfo* info) { - tscDebug("SML:0x%"PRIx64" taos_sml_insert get child table name through md5", info->id); - qsort(point->tags, point->tagNum, sizeof(TAOS_SML_KV), compareSmlColKv); - - SStringBuilder sb; memset(&sb, 0, sizeof(sb)); - char sTableName[TSDB_TABLE_NAME_LEN] = {0}; - strtolower(sTableName, point->stableName); - taosStringBuilderAppendString(&sb, sTableName); - for (int j = 0; j < point->tagNum; ++j) { - taosStringBuilderAppendChar(&sb, ','); - TAOS_SML_KV* tagKv = point->tags + j; - char tagName[TSDB_COL_NAME_LEN] = {0}; - strtolower(tagName, tagKv->key); - taosStringBuilderAppendString(&sb, tagName); - taosStringBuilderAppendChar(&sb, '='); - taosStringBuilderAppend(&sb, tagKv->value, tagKv->length); - } - size_t len = 0; - char* keyJoined = taosStringBuilderGetResult(&sb, &len); - MD5_CTX context; - MD5Init(&context); - MD5Update(&context, (uint8_t *)keyJoined, (uint32_t)len); - MD5Final(&context); - *tableNameLen = snprintf(tableName, *tableNameLen, - "t_%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x", context.digest[0], - context.digest[1], context.digest[2], context.digest[3], context.digest[4], context.digest[5], context.digest[6], - context.digest[7], context.digest[8], context.digest[9], context.digest[10], context.digest[11], - context.digest[12], context.digest[13], context.digest[14], context.digest[15]); - taosStringBuilderDestroy(&sb); - tscDebug("SML:0x%"PRIx64" child table name: %s", info->id, tableName); - return 0; -} - - static int32_t changeChildTableTagValue(TAOS* taos, const char* cTableName, const char* tagName, TAOS_BIND* bind, SSmlLinesInfo* info) { char sql[512]; sprintf(sql, "alter table %s set tag %s=?", cTableName, tagName); @@ -611,25 +628,25 @@ static int32_t changeChildTableTagValue(TAOS* taos, const char* cTableName, cons code = taos_stmt_prepare(stmt, sql, (unsigned long)strlen(sql)); if (code != 0) { - tscError("SML:0x%"PRIx64" %s", info->id, taos_stmt_errstr(stmt)); + tscError("SML:0x%"PRIx64" taos_stmt_prepare return %d:%s", info->id, code, tstrerror(code)); return code; } code = taos_stmt_bind_param(stmt, bind); if (code != 0) { - tscError("SML:0x%"PRIx64" %s", info->id, taos_stmt_errstr(stmt)); + tscError("SML:0x%"PRIx64" taos_stmt_bind_param return %d:%s", info->id, code, tstrerror(code)); return code; } code = taos_stmt_execute(stmt); if (code != 0) { - tscError("SML:0x%"PRIx64" %s", info->id, taos_stmt_errstr(stmt)); + tscError("SML:0x%"PRIx64" taos_stmt_execute return %d:%s", info->id, code, tstrerror(code)); return code; } code = taos_stmt_close(stmt); if (code != 0) { - tscError("SML:0x%"PRIx64" %s", info->id, taos_stmt_errstr(stmt)); + tscError("SML:0x%"PRIx64" taos_stmt_close return %d:%s", info->id, code, tstrerror(code)); return code; } return code; @@ -674,27 +691,27 @@ static int32_t creatChildTableIfNotExists(TAOS* taos, const char* cTableName, co if (code != 0) { tfree(stmt); - tscError("SML:0x%"PRIx64" %s", info->id, taos_stmt_errstr(stmt)); + tscError("SML:0x%"PRIx64" taos_stmt_prepare returns %d:%s", info->id, code, tstrerror(code)); return code; } code = taos_stmt_bind_param(stmt, TARRAY_GET_START(tagsBind)); if (code != 0) { tfree(stmt); - tscError("SML:0x%"PRIx64" %s", info->id, taos_stmt_errstr(stmt)); + tscError("SML:0x%"PRIx64" taos_stmt_bind_param returns %d:%s", info->id, code, tstrerror(code)); return code; } code = taos_stmt_execute(stmt); if (code != 0) { tfree(stmt); - tscError("SML:0x%"PRIx64" %s", info->id, taos_stmt_errstr(stmt)); + tscError("SML:0x%"PRIx64" taos_stmt_execute returns %d:%s", info->id, code, tstrerror(code)); return code; } code = taos_stmt_close(stmt); if (code != 0) { - tscError("SML:0x%"PRIx64" %s", info->id, taos_stmt_errstr(stmt)); + tscError("SML:0x%"PRIx64" taos_stmt_close return %d:%s", info->id, code, tstrerror(code)); return code; } return code; @@ -738,7 +755,7 @@ static int32_t insertChildTableBatch(TAOS* taos, char* cTableName, SArray* cols if (code != 0) { tfree(stmt); - tscError("SML:0x%"PRIx64" %s", info->id, taos_stmt_errstr(stmt)); + tscError("SML:0x%"PRIx64" taos_stmt_prepare return %d:%s", info->id, code, tstrerror(code)); return code; } @@ -746,7 +763,7 @@ static int32_t insertChildTableBatch(TAOS* taos, char* cTableName, SArray* cols code = taos_stmt_set_tbname(stmt, cTableName); if (code != 0) { tfree(stmt); - tscError("SML:0x%"PRIx64" %s", info->id, taos_stmt_errstr(stmt)); + tscError("SML:0x%"PRIx64" taos_stmt_set_tbname return %d:%s", info->id, code, tstrerror(code)); return code; } @@ -756,25 +773,25 @@ static int32_t insertChildTableBatch(TAOS* taos, char* cTableName, SArray* cols code = taos_stmt_bind_param(stmt, colsBinds); if (code != 0) { tfree(stmt); - tscError("SML:0x%"PRIx64" %s", info->id, taos_stmt_errstr(stmt)); + tscError("SML:0x%"PRIx64" taos_stmt_bind_param return %d:%s", info->id, code, tstrerror(code)); return code; } code = taos_stmt_add_batch(stmt); if (code != 0) { tfree(stmt); - tscError("SML:0x%"PRIx64" %s", info->id, taos_stmt_errstr(stmt)); + tscError("SML:0x%"PRIx64" taos_stmt_add_batch return %d:%s", info->id, code, tstrerror(code)); return code; } } code = taos_stmt_execute(stmt); if (code != 0) { - tscError("SML:0x%"PRIx64" %s", info->id, taos_stmt_errstr(stmt)); + tscError("SML:0x%"PRIx64" taos_stmt_execute return %d:%s", info->id, code, tstrerror(code)); } } while (code == TSDB_CODE_TDB_TABLE_RECONFIGURE && try++ < TSDB_MAX_REPLICA); if (code != 0) { - tscError("SML:0x%"PRIx64" %s", info->id, taos_stmt_errstr(stmt)); + tscError("SML:0x%"PRIx64" %d:%s", info->id, code, tstrerror(code)); taos_stmt_close(stmt); } else { taos_stmt_close(stmt); @@ -787,16 +804,10 @@ static int32_t arrangePointsByChildTableName(TAOS_SML_DATA_POINT* points, int nu SHashObj* cname2points, SArray* stableSchemas, SSmlLinesInfo* info) { for (int32_t i = 0; i < numPoints; ++i) { TAOS_SML_DATA_POINT * point = points + i; - if (!point->childTableName) { - char childTableName[TSDB_TABLE_NAME_LEN]; - int32_t tableNameLen = TSDB_TABLE_NAME_LEN; - getSmlMd5ChildTableName(point, childTableName, &tableNameLen, info); - point->childTableName = calloc(1, tableNameLen+1); - strncpy(point->childTableName, childTableName, tableNameLen); - point->childTableName[tableNameLen] = '\0'; - } - - SSmlSTableSchema* stableSchema = taosArrayGet(stableSchemas, point->schemaIdx); + uintptr_t valPointer = (uintptr_t)point; + size_t* pSchemaIndex = taosHashGet(info->smlDataToSchema, &valPointer, sizeof(uintptr_t)); + assert(pSchemaIndex != NULL); + SSmlSTableSchema* stableSchema = taosArrayGet(stableSchemas, *pSchemaIndex); for (int j = 0; j < point->tagNum; ++j) { TAOS_SML_KV* kv = point->tags + j; @@ -840,7 +851,10 @@ static int32_t applyChildTableTags(TAOS* taos, char* cTableName, char* sTableNam TAOS_SML_DATA_POINT * pDataPoint = taosArrayGetP(cTablePoints, i); for (int j = 0; j < pDataPoint->tagNum; ++j) { TAOS_SML_KV* kv = pDataPoint->tags + j; - tagKVs[kv->fieldSchemaIdx] = kv; + uintptr_t valPointer = (uintptr_t)kv; + size_t* pFieldSchemaIdx = taosHashGet(info->smlDataToSchema, &valPointer, sizeof(uintptr_t)); + assert(pFieldSchemaIdx != NULL); + tagKVs[*pFieldSchemaIdx] = kv; } } @@ -863,7 +877,10 @@ static int32_t applyChildTableTags(TAOS* taos, char* cTableName, char* sTableNam for (int j = 0; j < numTags; ++j) { if (tagKVs[j] == NULL) continue; TAOS_SML_KV* kv = tagKVs[j]; - TAOS_BIND* bind = taosArrayGet(tagBinds, kv->fieldSchemaIdx); + uintptr_t valPointer = (uintptr_t)kv; + size_t* pFieldSchemaIdx = taosHashGet(info->smlDataToSchema, &valPointer, sizeof(uintptr_t)); + assert(pFieldSchemaIdx != NULL); + TAOS_BIND* bind = taosArrayGet(tagBinds, *pFieldSchemaIdx); bind->buffer_type = kv->type; bind->length = malloc(sizeof(uintptr_t*)); *bind->length = kv->length; @@ -912,7 +929,10 @@ static int32_t applyChildTableTags(TAOS* taos, char* cTableName, char* sTableNam assert(tagKV->value); if (val == NULL || length != tagKV->length || memcmp(tagKV->value, val, length) != 0) { - TAOS_BIND* bind = taosArrayGet(tagBinds, tagKV->fieldSchemaIdx); + uintptr_t valPointer = (uintptr_t)tagKV; + size_t* pFieldSchemaIdx = taosHashGet(info->smlDataToSchema, &valPointer, sizeof(uintptr_t)); + assert(pFieldSchemaIdx != NULL); + TAOS_BIND* bind = taosArrayGet(tagBinds, *pFieldSchemaIdx); code = changeChildTableTagValue(taos, cTableName, tagKV->key, bind, info); if (code != 0) { tscError("SML:0x%"PRIx64" change child table tag failed. table name %s, tag %s", info->id, cTableName, tagKV->key); @@ -963,7 +983,10 @@ static int32_t applyChildTableFields(TAOS* taos, SSmlSTableSchema* sTableSchema, } for (int j = 0; j < point->fieldNum; ++j) { TAOS_SML_KV* kv = point->fields + j; - TAOS_BIND* bind = colBinds + kv->fieldSchemaIdx; + uintptr_t valPointer = (uintptr_t)kv; + size_t* pFieldSchemaIdx = taosHashGet(info->smlDataToSchema, &valPointer, sizeof(uintptr_t)); + assert(pFieldSchemaIdx != NULL); + TAOS_BIND* bind = colBinds + *pFieldSchemaIdx; bind->buffer_type = kv->type; bind->length = malloc(sizeof(uintptr_t*)); *bind->length = kv->length; @@ -1000,9 +1023,11 @@ static int32_t applyDataPoints(TAOS* taos, TAOS_SML_DATA_POINT* points, int32_t while (pCTablePoints) { SArray* cTablePoints = *pCTablePoints; - TAOS_SML_DATA_POINT* point = taosArrayGetP(cTablePoints, 0); - SSmlSTableSchema* sTableSchema = taosArrayGet(stableSchemas, point->schemaIdx); + uintptr_t valPointer = (uintptr_t)point; + size_t* pSchemaIndex = taosHashGet(info->smlDataToSchema, &valPointer, sizeof(uintptr_t)); + assert(pSchemaIndex != NULL); + SSmlSTableSchema* sTableSchema = taosArrayGet(stableSchemas, *pSchemaIndex); tscDebug("SML:0x%"PRIx64" apply child table tags. child table: %s", info->id, point->childTableName); code = applyChildTableTags(taos, point->childTableName, point->stableName, sTableSchema, cTablePoints, info); @@ -1034,10 +1059,11 @@ cleanup: return code; } -int taos_sml_insert(TAOS* taos, TAOS_SML_DATA_POINT* points, int numPoint, SSmlLinesInfo* info) { +int tscSmlInsert(TAOS* taos, TAOS_SML_DATA_POINT* points, int numPoint, SSmlLinesInfo* info) { tscDebug("SML:0x%"PRIx64" taos_sml_insert. number of points: %d", info->id, numPoint); int32_t code = TSDB_CODE_SUCCESS; + info->smlDataToSchema = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), true, false); tscDebug("SML:0x%"PRIx64" build data point schemas", info->id); SArray* stableSchemas = taosArrayInit(32, sizeof(SSmlSTableSchema)); // SArray @@ -1067,6 +1093,15 @@ clean_up: taosArrayDestroy(schema->tags); } taosArrayDestroy(stableSchemas); + taosHashCleanup(info->smlDataToSchema); + return code; +} + +int taos_sml_insert(TAOS* taos, TAOS_SML_DATA_POINT* points, int numPoint) { + SSmlLinesInfo* info = calloc(1, sizeof(SSmlLinesInfo)); + info->id = genLinesSmlId(); + int code = tscSmlInsert(taos, points, numPoint, info); + free(info); return code; } @@ -2076,18 +2111,6 @@ int32_t tscParseLine(const char* sql, TAOS_SML_DATA_POINT* smlData, SSmlLinesInf //========================================================================= -static uint64_t linesSmlHandleId = 0; - -uint64_t genLinesSmlId() { - uint64_t id; - - do { - id = atomic_add_fetch_64(&linesSmlHandleId, 1); - } while (id == 0); - - return id; -} - void destroySmlDataPoint(TAOS_SML_DATA_POINT* point) { for (int i=0; itagNum; ++i) { free((point->tags+i)->key); @@ -2157,7 +2180,7 @@ int taos_insert_lines(TAOS* taos, char* lines[], int numLines) { } TAOS_SML_DATA_POINT* points = TARRAY_GET_START(lpPoints); - code = taos_sml_insert(taos, points, (int)numPoints, info); + code = tscSmlInsert(taos, points, (int)numPoints, info); if (code != 0) { tscError("SML:0x%"PRIx64" taos_sml_insert error: %s", info->id, tstrerror((code))); } From 2afaf1a0e4ad48f3a1a73a94acae4f19b147aa68 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Fri, 20 Aug 2021 13:11:47 +0800 Subject: [PATCH 02/40] schemaless:multi-thread ignore some errors --- src/client/src/tscParseLineProtocol.c | 25 +++++++++++++++++++++++++ src/inc/taoserror.h | 3 +++ src/mnode/src/mnodeTable.c | 13 +++++++++++++ 3 files changed, 41 insertions(+) diff --git a/src/client/src/tscParseLineProtocol.c b/src/client/src/tscParseLineProtocol.c index 43509d3533..25438ef940 100644 --- a/src/client/src/tscParseLineProtocol.c +++ b/src/client/src/tscParseLineProtocol.c @@ -371,6 +371,11 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInf buildColumnDescription(action->alterSTable.field, result+n, capacity-n, &outBytes); TAOS_RES* res = taos_query(taos, result); //TODO async doAsyncQuery code = taos_errno(res); + if (code == TSDB_CODE_MND_FIELD_ALREAY_EXIST) { + TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); + code = taos_errno(res2); + taos_free_result(res2); + } taos_free_result(res); break; } @@ -380,6 +385,11 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInf result+n, capacity-n, &outBytes); TAOS_RES* res = taos_query(taos, result); //TODO async doAsyncQuery code = taos_errno(res); + if (code == TSDB_CODE_MND_TAG_ALREAY_EXIST) { + TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); + code = taos_errno(res2); + taos_free_result(res2); + } taos_free_result(res); break; } @@ -389,6 +399,11 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInf capacity-n, &outBytes); TAOS_RES* res = taos_query(taos, result); //TODO async doAsyncQuery code = taos_errno(res); + if (code == TSDB_CODE_MND_INVALID_COLUMN_LENGTH) { + TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); + code = taos_errno(res2); + taos_free_result(res2); + } taos_free_result(res); break; } @@ -398,6 +413,11 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInf capacity-n, &outBytes); TAOS_RES* res = taos_query(taos, result); //TODO async doAsyncQuery code = taos_errno(res); + if (code == TSDB_CODE_MND_INVALID_TAG_LENGTH) { + TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); + code = taos_errno(res2); + taos_free_result(res2); + } taos_free_result(res); break; } @@ -427,6 +447,11 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInf outBytes = snprintf(pos, freeBytes, ")"); TAOS_RES* res = taos_query(taos, result); code = taos_errno(res); + if (code == TSDB_CODE_MND_TABLE_ALREADY_EXIST) { + TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); + code = taos_errno(res2); + taos_free_result(res2); + } taos_free_result(res); break; } diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index 368658377c..b8ccada105 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -211,6 +211,9 @@ int32_t* taosGetErrno(); #define TSDB_CODE_DND_TOO_MANY_VNODES TAOS_DEF_ERROR_CODE(0, 0x0405) //"Too many vnode directories") #define TSDB_CODE_DND_EXITING TAOS_DEF_ERROR_CODE(0, 0x0406) //"Dnode is exiting" +#define TSDB_CODE_MND_INVALID_TAG_LENGTH TAOS_DEF_ERROR_CODE(0, 0x410) //"invalid tag length") +#define TSDB_CODE_MND_INVALID_COLUMN_LENGTH TAOS_DEF_ERROR_CODE(0, 0x411) //"invalid column length") + // vnode #define TSDB_CODE_VND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0500) //"Action in progress") #define TSDB_CODE_VND_MSG_NOT_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0501) //"Message not processed") diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index 0bc114ffdf..39a21a6747 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -1519,6 +1519,13 @@ static int32_t mnodeChangeSuperTableColumn(SMnodeMsg *pMsg) { SSchema *schema = (SSchema *) (pStable->schema + col); ASSERT(schema->type == TSDB_DATA_TYPE_BINARY || schema->type == TSDB_DATA_TYPE_NCHAR); schema->bytes = pAlter->schema[0].bytes; + + if (pAlter->schema[0].bytes <= schema->bytes) { + mError("msg:%p, app:%p stable:%s, modify column len. column:%s, len from %d to %d", pMsg, pMsg->rpcMsg.ahandle, + pStable->info.tableId, name, schema->bytes, pAlter->schema[0].bytes); + return TSDB_CODE_MND_INVALID_COLUMN_LENGTH; + } + pStable->sversion++; mInfo("msg:%p, app:%p stable %s, start to modify column %s len to %d", pMsg, pMsg->rpcMsg.ahandle, pStable->info.tableId, name, schema->bytes); @@ -1548,6 +1555,12 @@ static int32_t mnodeChangeSuperTableTag(SMnodeMsg *pMsg) { // update SSchema *schema = (SSchema *) (pStable->schema + col + pStable->numOfColumns); ASSERT(schema->type == TSDB_DATA_TYPE_BINARY || schema->type == TSDB_DATA_TYPE_NCHAR); + if (pAlter->schema[0].bytes <= schema->bytes) { + mError("msg:%p, app:%p stable:%s, modify tag len. tag:%s, len from %d to %d", pMsg, pMsg->rpcMsg.ahandle, + pStable->info.tableId, name, schema->bytes, pAlter->schema[0].bytes); + return TSDB_CODE_MND_INVALID_TAG_LENGTH; + } + schema->bytes = pAlter->schema[0].bytes; pStable->tversion++; mInfo("msg:%p, app:%p stable %s, start to modify tag len %s to %d", pMsg, pMsg->rpcMsg.ahandle, pStable->info.tableId, From a687df67376d80e01fd5ef0cc678842bca6c6c89 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Fri, 20 Aug 2021 15:51:47 +0800 Subject: [PATCH 03/40] save before invalid operation --- src/client/inc/tsclient.h | 1 + src/client/src/tscParseLineProtocol.c | 21 ++++++- src/client/src/tscSQLParser.c | 41 ++++++++++---- src/inc/taoserror.h | 1 + tests/pytest/insert/schemalessInsert.py | 74 ++++++++++++------------- 5 files changed, 87 insertions(+), 51 deletions(-) diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index 4ed7894931..4847a34dd1 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -486,6 +486,7 @@ bool tscHasReachLimitation(SQueryInfo *pQueryInfo, SSqlRes *pRes); void tscSetBoundColumnInfo(SParsedDataColInfo *pColInfo, SSchema *pSchema, int32_t numOfCols); char *tscGetErrorMsgPayload(SSqlCmd *pCmd); +int32_t tscErrorMsgWithCode(int32_t code, char* dstBuffer, const char* errMsg, const char* sql); int32_t tscInvalidOperationMsg(char *msg, const char *additionalInfo, const char *sql); int32_t tscSQLSyntaxErrMsg(char* msg, const char* additionalInfo, const char* sql); diff --git a/src/client/src/tscParseLineProtocol.c b/src/client/src/tscParseLineProtocol.c index 25438ef940..ee63290a3e 100644 --- a/src/client/src/tscParseLineProtocol.c +++ b/src/client/src/tscParseLineProtocol.c @@ -371,7 +371,10 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInf buildColumnDescription(action->alterSTable.field, result+n, capacity-n, &outBytes); TAOS_RES* res = taos_query(taos, result); //TODO async doAsyncQuery code = taos_errno(res); - if (code == TSDB_CODE_MND_FIELD_ALREAY_EXIST) { + if (code != TSDB_CODE_SUCCESS) { + tscError("SML:0x%"PRIx64" apply schema action. error : %s", info->id, taos_errstr(res)); + } + if (code == TSDB_CODE_MND_FIELD_ALREAY_EXIST || code == TSDB_CODE_TSC_DUP_COL_NAMES) { TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); code = taos_errno(res2); taos_free_result(res2); @@ -385,7 +388,10 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInf result+n, capacity-n, &outBytes); TAOS_RES* res = taos_query(taos, result); //TODO async doAsyncQuery code = taos_errno(res); - if (code == TSDB_CODE_MND_TAG_ALREAY_EXIST) { + if (code != TSDB_CODE_SUCCESS) { + tscError("SML:0x%"PRIx64" apply schema action. error : %s", info->id, taos_errstr(res)); + } + if (code == TSDB_CODE_MND_TAG_ALREAY_EXIST || code == TSDB_CODE_TSC_DUP_COL_NAMES) { TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); code = taos_errno(res2); taos_free_result(res2); @@ -399,6 +405,9 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInf capacity-n, &outBytes); TAOS_RES* res = taos_query(taos, result); //TODO async doAsyncQuery code = taos_errno(res); + if (code != TSDB_CODE_SUCCESS) { + tscError("SML:0x%"PRIx64" apply schema action. error : %s", info->id, taos_errstr(res)); + } if (code == TSDB_CODE_MND_INVALID_COLUMN_LENGTH) { TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); code = taos_errno(res2); @@ -413,6 +422,9 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInf capacity-n, &outBytes); TAOS_RES* res = taos_query(taos, result); //TODO async doAsyncQuery code = taos_errno(res); + if (code != TSDB_CODE_SUCCESS) { + tscError("SML:0x%"PRIx64" apply schema action. error : %s", info->id, taos_errstr(res)); + } if (code == TSDB_CODE_MND_INVALID_TAG_LENGTH) { TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); code = taos_errno(res2); @@ -447,6 +459,9 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInf outBytes = snprintf(pos, freeBytes, ")"); TAOS_RES* res = taos_query(taos, result); code = taos_errno(res); + if (code != TSDB_CODE_SUCCESS) { + tscError("SML:0x%"PRIx64" apply schema action. error : %s", info->id, taos_errstr(res)); + } if (code == TSDB_CODE_MND_TABLE_ALREADY_EXIST) { TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); code = taos_errno(res2); @@ -462,7 +477,7 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInf free(result); if (code != 0) { - tscError("SML:0x%"PRIx64 "apply schema action failure. %s", info->id, tstrerror(code)); + tscError("SML:0x%"PRIx64 " apply schema action failure. %s", info->id, tstrerror(code)); } return code; } diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 612a3d4798..2699ad2eb6 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -307,6 +307,31 @@ static int32_t invalidOperationMsg(char* dstBuffer, const char* errMsg) { return tscInvalidOperationMsg(dstBuffer, errMsg, NULL); } +int32_t tscErrorMsgWithCode(int32_t code, char* dstBuffer, const char* errMsg, const char* sql) { + const char* msgFormat1 = "%s:%s"; + const char* msgFormat2 = "%s:\'%s\' (%s)"; + const char* msgFormat3 = "%s:\'%s\'"; + + const int32_t BACKWARD_CHAR_STEP = 0; + + if (sql == NULL) { + assert(errMsg != NULL); + sprintf(dstBuffer, msgFormat1, tstrerror(code), errMsg); + return code; + } + + char buf[64] = {0}; // only extract part of sql string + strncpy(buf, (sql - BACKWARD_CHAR_STEP), tListLen(buf) - 1); + + if (errMsg != NULL) { + sprintf(dstBuffer, msgFormat2, tstrerror(code), buf, errMsg); + } else { + sprintf(dstBuffer, msgFormat3, tstrerror(code), buf); // no additional information for invalid sql error + } + + return code; +} + static int convertTimestampStrToInt64(tVariant *pVar, int32_t precision) { int64_t time = 0; strdequote(pVar->pz); @@ -1393,7 +1418,6 @@ static bool validateTableColumnInfo(SArray* pFieldList, SSqlCmd* pCmd) { const char* msg = "illegal number of columns"; const char* msg1 = "first column must be timestamp"; const char* msg2 = "row length exceeds max length"; - const char* msg3 = "duplicated column names"; const char* msg4 = "invalid data type"; const char* msg5 = "invalid binary/nchar column length"; const char* msg6 = "invalid column name"; @@ -1442,7 +1466,7 @@ static bool validateTableColumnInfo(SArray* pFieldList, SSqlCmd* pCmd) { // field name must be unique if (has(pFieldList, i + 1, pField->name) == true) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); + tscErrorMsgWithCode(TSDB_CODE_TSC_DUP_COL_NAMES, tscGetErrorMsgPayload(pCmd), pField->name, NULL); return false; } @@ -1464,8 +1488,6 @@ static bool validateTagParams(SArray* pTagsList, SArray* pFieldList, SSqlCmd* pC const char* msg1 = "invalid number of tag columns"; const char* msg2 = "tag length too long"; - const char* msg3 = "duplicated column names"; - //const char* msg4 = "timestamp not allowed in tags"; const char* msg5 = "invalid data type in tags"; const char* msg6 = "invalid tag name"; const char* msg7 = "invalid binary/nchar tag length"; @@ -1496,7 +1518,7 @@ static bool validateTagParams(SArray* pTagsList, SArray* pFieldList, SSqlCmd* pC } if (has(pTagsList, i + 1, p->name) == true) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); + tscErrorMsgWithCode(TSDB_CODE_TSC_DUP_COL_NAMES, tscGetErrorMsgPayload(pCmd), p->name, NULL); return false; } } @@ -1523,7 +1545,7 @@ static bool validateTagParams(SArray* pTagsList, SArray* pFieldList, SSqlCmd* pC TAOS_FIELD* p = taosArrayGet(pTagsList, i); if (has(pFieldList, 0, p->name) == true) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); + tscErrorMsgWithCode(TSDB_CODE_TSC_DUP_COL_NAMES, tscGetErrorMsgPayload(pCmd), p->name, NULL); return false; } } @@ -1535,8 +1557,6 @@ static bool validateTagParams(SArray* pTagsList, SArray* pFieldList, SSqlCmd* pC * tags name /column name is truncated in sql.y */ bool validateOneTags(SSqlCmd* pCmd, TAOS_FIELD* pTagField) { - //const char* msg1 = "timestamp not allowed in tags"; - const char* msg2 = "duplicated column names"; const char* msg3 = "tag length too long"; const char* msg4 = "invalid tag name"; const char* msg5 = "invalid binary/nchar tag length"; @@ -1605,7 +1625,7 @@ bool validateOneTags(SSqlCmd* pCmd, TAOS_FIELD* pTagField) { for (int32_t i = 0; i < numOfTags + numOfCols; ++i) { if (strncasecmp(pTagField->name, pSchema[i].name, sizeof(pTagField->name) - 1) == 0) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); + tscErrorMsgWithCode(TSDB_CODE_TSC_DUP_COL_NAMES, tscGetErrorMsgPayload(pCmd), pTagField->name, NULL); return false; } } @@ -1615,7 +1635,6 @@ bool validateOneTags(SSqlCmd* pCmd, TAOS_FIELD* pTagField) { bool validateOneColumn(SSqlCmd* pCmd, TAOS_FIELD* pColField) { const char* msg1 = "too many columns"; - const char* msg2 = "duplicated column names"; const char* msg3 = "column length too long"; const char* msg4 = "invalid data type"; const char* msg5 = "invalid column name"; @@ -1665,7 +1684,7 @@ bool validateOneColumn(SSqlCmd* pCmd, TAOS_FIELD* pColField) { // field name must be unique for (int32_t i = 0; i < numOfTags + numOfCols; ++i) { if (strncasecmp(pColField->name, pSchema[i].name, sizeof(pColField->name) - 1) == 0) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); + tscErrorMsgWithCode(TSDB_CODE_TSC_DUP_COL_NAMES, tscGetErrorMsgPayload(pCmd), pColField->name, NULL); return false; } } diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index b8ccada105..f54150d76b 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -103,6 +103,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_TSC_FILE_EMPTY TAOS_DEF_ERROR_CODE(0, 0x021A) //"File is empty") #define TSDB_CODE_TSC_LINE_SYNTAX_ERROR TAOS_DEF_ERROR_CODE(0, 0x021B) //"Syntax error in Line") #define TSDB_CODE_TSC_NO_META_CACHED TAOS_DEF_ERROR_CODE(0, 0x021C) //"No table meta cached") +#define TSDB_CODE_TSC_DUP_COL_NAMES TAOS_DEF_ERROR_CODE(0, 0x021D) //"duplicated column names") // mnode #define TSDB_CODE_MND_MSG_NOT_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0300) //"Message not processed") diff --git a/tests/pytest/insert/schemalessInsert.py b/tests/pytest/insert/schemalessInsert.py index 49c3223588..00e805d791 100644 --- a/tests/pytest/insert/schemalessInsert.py +++ b/tests/pytest/insert/schemalessInsert.py @@ -1261,52 +1261,52 @@ class TDTestCase: # self._conn.insert_lines([input_sql4]) def runAll(self): - self.initCheckCase() - self.boolTypeCheckCase() - self.symbolsCheckCase() - self.tsCheckCase() - self.idSeqCheckCase() - self.idUpperCheckCase() - self.noIdCheckCase() - self.maxColTagCheckCase() - self.idIllegalNameCheckCase() - self.idStartWithNumCheckCase() - self.nowTsCheckCase() - self.dateFormatTsCheckCase() - self.illegalTsCheckCase() - self.tagValueLengthCheckCase() - self.colValueLengthCheckCase() - self.tagColIllegalValueCheckCase() - self.duplicateIdTagColInsertCheckCase() - self.noIdStbExistCheckCase() - self.duplicateInsertExistCheckCase() - self.tagColBinaryNcharLengthCheckCase() - self.tagColAddDupIDCheckCase() - self.tagColAddCheckCase() - self.tagMd5Check() - self.tagColBinaryMaxLengthCheckCase() - # self.tagColNcharMaxLengthCheckCase() - self.batchInsertCheckCase() - self.multiInsertCheckCase(1000) - self.batchErrorInsertCheckCase() - # MultiThreads - self.stbInsertMultiThreadCheckCase() - self.sStbStbDdataInsertMultiThreadCheckCase() - self.sStbStbDdataAtcInsertMultiThreadCheckCase() - self.sStbStbDdataMtcInsertMultiThreadCheckCase() - self.sStbDtbDdataInsertMultiThreadCheckCase() + # self.initCheckCase() + # self.boolTypeCheckCase() + # self.symbolsCheckCase() + # self.tsCheckCase() + # self.idSeqCheckCase() + # self.idUpperCheckCase() + # self.noIdCheckCase() + # self.maxColTagCheckCase() + # self.idIllegalNameCheckCase() + # self.idStartWithNumCheckCase() + # self.nowTsCheckCase() + # self.dateFormatTsCheckCase() + # self.illegalTsCheckCase() + # self.tagValueLengthCheckCase() + # self.colValueLengthCheckCase() + # self.tagColIllegalValueCheckCase() + # self.duplicateIdTagColInsertCheckCase() + # self.noIdStbExistCheckCase() + # self.duplicateInsertExistCheckCase() + # self.tagColBinaryNcharLengthCheckCase() + # self.tagColAddDupIDCheckCase() + # self.tagColAddCheckCase() + # self.tagMd5Check() + # self.tagColBinaryMaxLengthCheckCase() + # # self.tagColNcharMaxLengthCheckCase() + # self.batchInsertCheckCase() + # self.multiInsertCheckCase(1000) + # self.batchErrorInsertCheckCase() + # # MultiThreads + # self.stbInsertMultiThreadCheckCase() + # self.sStbStbDdataInsertMultiThreadCheckCase() + # self.sStbStbDdataAtcInsertMultiThreadCheckCase() + # self.sStbStbDdataMtcInsertMultiThreadCheckCase() + # self.sStbDtbDdataInsertMultiThreadCheckCase() # # ! concurrency conflict - # self.sStbDtbDdataAcMtInsertMultiThreadCheckCase() + self.sStbDtbDdataAcMtInsertMultiThreadCheckCase() # self.sStbDtbDdataAtMcInsertMultiThreadCheckCase() - self.sStbStbDdataDtsInsertMultiThreadCheckCase() + # self.sStbStbDdataDtsInsertMultiThreadCheckCase() # # ! concurrency conflict # self.sStbStbDdataDtsAcMtInsertMultiThreadCheckCase() # self.sStbStbDdataDtsAtMcInsertMultiThreadCheckCase() - self.sStbDtbDdataDtsInsertMultiThreadCheckCase() + # self.sStbDtbDdataDtsInsertMultiThreadCheckCase() # ! concurrency conflict # self.sStbDtbDdataDtsAcMtInsertMultiThreadCheckCase() From 559dcff66ec77422cf33744f3c2d4aaf63fae62e Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Fri, 20 Aug 2021 17:45:58 +0800 Subject: [PATCH 04/40] schemaless:before retrying loading meta --- src/client/src/tscSQLParser.c | 95 +++++++++++------------------------ src/client/src/tscUtil.c | 25 +++++++++ 2 files changed, 55 insertions(+), 65 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 2699ad2eb6..64f88b373d 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -116,7 +116,7 @@ static int32_t validateColumnName(char* name); static int32_t setKillInfo(SSqlObj* pSql, struct SSqlInfo* pInfo, int32_t killType); static int32_t setCompactVnodeInfo(SSqlObj* pSql, struct SSqlInfo* pInfo); -static bool validateOneTags(SSqlCmd* pCmd, TAOS_FIELD* pTagField); +static int32_t validateOneTag(SSqlCmd* pCmd, TAOS_FIELD* pTagField); static bool hasTimestampForPointInterpQuery(SQueryInfo* pQueryInfo); static bool hasNormalColumnFilter(SQueryInfo* pQueryInfo); @@ -307,31 +307,6 @@ static int32_t invalidOperationMsg(char* dstBuffer, const char* errMsg) { return tscInvalidOperationMsg(dstBuffer, errMsg, NULL); } -int32_t tscErrorMsgWithCode(int32_t code, char* dstBuffer, const char* errMsg, const char* sql) { - const char* msgFormat1 = "%s:%s"; - const char* msgFormat2 = "%s:\'%s\' (%s)"; - const char* msgFormat3 = "%s:\'%s\'"; - - const int32_t BACKWARD_CHAR_STEP = 0; - - if (sql == NULL) { - assert(errMsg != NULL); - sprintf(dstBuffer, msgFormat1, tstrerror(code), errMsg); - return code; - } - - char buf[64] = {0}; // only extract part of sql string - strncpy(buf, (sql - BACKWARD_CHAR_STEP), tListLen(buf) - 1); - - if (errMsg != NULL) { - sprintf(dstBuffer, msgFormat2, tstrerror(code), buf, errMsg); - } else { - sprintf(dstBuffer, msgFormat3, tstrerror(code), buf); // no additional information for invalid sql error - } - - return code; -} - static int convertTimestampStrToInt64(tVariant *pVar, int32_t precision) { int64_t time = 0; strdequote(pVar->pz); @@ -1418,6 +1393,7 @@ static bool validateTableColumnInfo(SArray* pFieldList, SSqlCmd* pCmd) { const char* msg = "illegal number of columns"; const char* msg1 = "first column must be timestamp"; const char* msg2 = "row length exceeds max length"; + const char* msg3 = "duplicated column names"; const char* msg4 = "invalid data type"; const char* msg5 = "invalid binary/nchar column length"; const char* msg6 = "invalid column name"; @@ -1466,7 +1442,7 @@ static bool validateTableColumnInfo(SArray* pFieldList, SSqlCmd* pCmd) { // field name must be unique if (has(pFieldList, i + 1, pField->name) == true) { - tscErrorMsgWithCode(TSDB_CODE_TSC_DUP_COL_NAMES, tscGetErrorMsgPayload(pCmd), pField->name, NULL); + invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); return false; } @@ -1488,6 +1464,8 @@ static bool validateTagParams(SArray* pTagsList, SArray* pFieldList, SSqlCmd* pC const char* msg1 = "invalid number of tag columns"; const char* msg2 = "tag length too long"; + const char* msg3 = "duplicated column names"; + //const char* msg4 = "timestamp not allowed in tags"; const char* msg5 = "invalid data type in tags"; const char* msg6 = "invalid tag name"; const char* msg7 = "invalid binary/nchar tag length"; @@ -1518,7 +1496,7 @@ static bool validateTagParams(SArray* pTagsList, SArray* pFieldList, SSqlCmd* pC } if (has(pTagsList, i + 1, p->name) == true) { - tscErrorMsgWithCode(TSDB_CODE_TSC_DUP_COL_NAMES, tscGetErrorMsgPayload(pCmd), p->name, NULL); + invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); return false; } } @@ -1545,7 +1523,7 @@ static bool validateTagParams(SArray* pTagsList, SArray* pFieldList, SSqlCmd* pC TAOS_FIELD* p = taosArrayGet(pTagsList, i); if (has(pFieldList, 0, p->name) == true) { - tscErrorMsgWithCode(TSDB_CODE_TSC_DUP_COL_NAMES, tscGetErrorMsgPayload(pCmd), p->name, NULL); + invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); return false; } } @@ -1556,7 +1534,7 @@ static bool validateTagParams(SArray* pTagsList, SArray* pFieldList, SSqlCmd* pC /* * tags name /column name is truncated in sql.y */ -bool validateOneTags(SSqlCmd* pCmd, TAOS_FIELD* pTagField) { +int32_t validateOneTag(SSqlCmd* pCmd, TAOS_FIELD* pTagField) { const char* msg3 = "tag length too long"; const char* msg4 = "invalid tag name"; const char* msg5 = "invalid binary/nchar tag length"; @@ -1571,8 +1549,7 @@ bool validateOneTags(SSqlCmd* pCmd, TAOS_FIELD* pTagField) { // no more max columns if (numOfTags + numOfCols >= TSDB_MAX_COLUMNS) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg7); - return false; + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg7); } // no more than 6 tags @@ -1580,8 +1557,7 @@ bool validateOneTags(SSqlCmd* pCmd, TAOS_FIELD* pTagField) { char msg[128] = {0}; sprintf(msg, "tags no more than %d", TSDB_MAX_TAGS); - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg); - return false; + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg); } // no timestamp allowable @@ -1591,8 +1567,7 @@ bool validateOneTags(SSqlCmd* pCmd, TAOS_FIELD* pTagField) { //} if ((pTagField->type < TSDB_DATA_TYPE_BOOL) || (pTagField->type > TSDB_DATA_TYPE_UBIGINT)) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6); - return false; + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6); } SSchema* pTagSchema = tscGetTableTagSchema(pTableMetaInfo->pTableMeta); @@ -1604,20 +1579,17 @@ bool validateOneTags(SSqlCmd* pCmd, TAOS_FIELD* pTagField) { // length less than TSDB_MAX_TASG_LEN if (nLen + pTagField->bytes > TSDB_MAX_TAGS_LEN) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - return false; + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); } // tags name can not be a keyword if (validateColumnName(pTagField->name) != TSDB_CODE_SUCCESS) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); - return false; + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); } // binary(val), val can not be equalled to or less than 0 if ((pTagField->type == TSDB_DATA_TYPE_BINARY || pTagField->type == TSDB_DATA_TYPE_NCHAR) && pTagField->bytes <= 0) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5); - return false; + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5); } // field name must be unique @@ -1625,15 +1597,14 @@ bool validateOneTags(SSqlCmd* pCmd, TAOS_FIELD* pTagField) { for (int32_t i = 0; i < numOfTags + numOfCols; ++i) { if (strncasecmp(pTagField->name, pSchema[i].name, sizeof(pTagField->name) - 1) == 0) { - tscErrorMsgWithCode(TSDB_CODE_TSC_DUP_COL_NAMES, tscGetErrorMsgPayload(pCmd), pTagField->name, NULL); - return false; + return tscErrorMsgWithCode(TSDB_CODE_TSC_DUP_COL_NAMES, tscGetErrorMsgPayload(pCmd), pTagField->name, NULL); } } - return true; + return TSDB_CODE_SUCCESS; } -bool validateOneColumn(SSqlCmd* pCmd, TAOS_FIELD* pColField) { +int32_t validateOneColumn(SSqlCmd* pCmd, TAOS_FIELD* pColField) { const char* msg1 = "too many columns"; const char* msg3 = "column length too long"; const char* msg4 = "invalid data type"; @@ -1649,18 +1620,15 @@ bool validateOneColumn(SSqlCmd* pCmd, TAOS_FIELD* pColField) { // no more max columns if (numOfCols >= TSDB_MAX_COLUMNS || numOfTags + numOfCols >= TSDB_MAX_COLUMNS) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - return false; + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); } if (pColField->type < TSDB_DATA_TYPE_BOOL || pColField->type > TSDB_DATA_TYPE_UBIGINT) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); - return false; + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); } if (validateColumnName(pColField->name) != TSDB_CODE_SUCCESS) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5); - return false; + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5); } SSchema* pSchema = tscGetTableSchema(pTableMeta); @@ -1671,25 +1639,22 @@ bool validateOneColumn(SSqlCmd* pCmd, TAOS_FIELD* pColField) { } if (pColField->bytes <= 0) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6); - return false; + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6); } // length less than TSDB_MAX_BYTES_PER_ROW if (nLen + pColField->bytes > TSDB_MAX_BYTES_PER_ROW) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - return false; + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); } // field name must be unique for (int32_t i = 0; i < numOfTags + numOfCols; ++i) { if (strncasecmp(pColField->name, pSchema[i].name, sizeof(pColField->name) - 1) == 0) { - tscErrorMsgWithCode(TSDB_CODE_TSC_DUP_COL_NAMES, tscGetErrorMsgPayload(pCmd), pColField->name, NULL); - return false; + return tscErrorMsgWithCode(TSDB_CODE_TSC_DUP_COL_NAMES, tscGetErrorMsgPayload(pCmd), pColField->name, NULL); } } - return true; + return TSDB_CODE_SUCCESS; } /* is contained in pFieldList or not */ @@ -6108,8 +6073,9 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { } TAOS_FIELD* p = taosArrayGet(pFieldList, 0); - if (!validateOneTags(pCmd, p)) { - return TSDB_CODE_TSC_INVALID_OPERATION; + int32_t ret = validateOneTag(pCmd, p); + if (ret != TSDB_CODE_SUCCESS) { + return ret; } tscFieldInfoAppend(&pQueryInfo->fieldsInfo, p); @@ -6286,8 +6252,9 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { } TAOS_FIELD* p = taosArrayGet(pFieldList, 0); - if (!validateOneColumn(pCmd, p)) { - return TSDB_CODE_TSC_INVALID_OPERATION; + int32_t ret = validateOneColumn(pCmd, p); + if (ret != TSDB_CODE_SUCCESS) { + return ret; } tscFieldInfoAppend(&pQueryInfo->fieldsInfo, p); @@ -8702,8 +8669,6 @@ static STableMeta* extractTempTableMetaFromSubquery(SQueryInfo* pUpstream) { n += 1; } - info->numOfColumns = n; - return meta; } diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 591a6bba34..583f50aecd 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -4096,6 +4096,31 @@ int32_t tscInvalidOperationMsg(char* msg, const char* additionalInfo, const char return TSDB_CODE_TSC_INVALID_OPERATION; } +int32_t tscErrorMsgWithCode(int32_t code, char* dstBuffer, const char* errMsg, const char* sql) { + const char* msgFormat1 = "%s:%s"; + const char* msgFormat2 = "%s:\'%s\' (%s)"; + const char* msgFormat3 = "%s:\'%s\'"; + + const int32_t BACKWARD_CHAR_STEP = 0; + + if (sql == NULL) { + assert(errMsg != NULL); + sprintf(dstBuffer, msgFormat1, tstrerror(code), errMsg); + return code; + } + + char buf[64] = {0}; // only extract part of sql string + strncpy(buf, (sql - BACKWARD_CHAR_STEP), tListLen(buf) - 1); + + if (errMsg != NULL) { + sprintf(dstBuffer, msgFormat2, tstrerror(code), buf, errMsg); + } else { + sprintf(dstBuffer, msgFormat3, tstrerror(code), buf); // no additional information for invalid sql error + } + + return code; +} + bool tscHasReachLimitation(SQueryInfo* pQueryInfo, SSqlRes* pRes) { assert(pQueryInfo != NULL && pQueryInfo->clauseLimit != 0); return (pQueryInfo->clauseLimit > 0 && pRes->numOfClauseTotal >= pQueryInfo->clauseLimit); From f26174745865e2300e4b133a3f4df2d6f32cf03a Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Fri, 20 Aug 2021 21:57:30 +0800 Subject: [PATCH 05/40] schemaless: multi-threaded cases runw --- src/client/src/tscParseLineProtocol.c | 183 +++++++++++++++--------- src/mnode/src/mnodeTable.c | 4 +- tests/pytest/insert/schemalessInsert.py | 26 ++-- 3 files changed, 129 insertions(+), 84 deletions(-) diff --git a/src/client/src/tscParseLineProtocol.c b/src/client/src/tscParseLineProtocol.c index ee63290a3e..03e361c950 100644 --- a/src/client/src/tscParseLineProtocol.c +++ b/src/client/src/tscParseLineProtocol.c @@ -372,14 +372,18 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInf TAOS_RES* res = taos_query(taos, result); //TODO async doAsyncQuery code = taos_errno(res); if (code != TSDB_CODE_SUCCESS) { - tscError("SML:0x%"PRIx64" apply schema action. error : %s", info->id, taos_errstr(res)); + tscError("SML:0x%"PRIx64" apply schema action. error: %s", info->id, taos_errstr(res)); } + taos_free_result(res); + if (code == TSDB_CODE_MND_FIELD_ALREAY_EXIST || code == TSDB_CODE_TSC_DUP_COL_NAMES) { TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); code = taos_errno(res2); + if (code != TSDB_CODE_SUCCESS) { + tscError("SML:0x%" PRIx64 " apply schema action. reset query cache. error: %s", info->id, taos_errstr(res2)); + } taos_free_result(res2); } - taos_free_result(res); break; } case SCHEMA_ACTION_ADD_TAG: { @@ -391,12 +395,16 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInf if (code != TSDB_CODE_SUCCESS) { tscError("SML:0x%"PRIx64" apply schema action. error : %s", info->id, taos_errstr(res)); } + taos_free_result(res); + if (code == TSDB_CODE_MND_TAG_ALREAY_EXIST || code == TSDB_CODE_TSC_DUP_COL_NAMES) { TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); code = taos_errno(res2); + if (code != TSDB_CODE_SUCCESS) { + tscError("SML:0x%" PRIx64 " apply schema action. reset query cache. error: %s", info->id, taos_errstr(res2)); + } taos_free_result(res2); } - taos_free_result(res); break; } case SCHEMA_ACTION_CHANGE_COLUMN_SIZE: { @@ -408,12 +416,16 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInf if (code != TSDB_CODE_SUCCESS) { tscError("SML:0x%"PRIx64" apply schema action. error : %s", info->id, taos_errstr(res)); } + taos_free_result(res); + if (code == TSDB_CODE_MND_INVALID_COLUMN_LENGTH) { TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); code = taos_errno(res2); + if (code != TSDB_CODE_SUCCESS) { + tscError("SML:0x%" PRIx64 " apply schema action. reset query cache. error: %s", info->id, taos_errstr(res2)); + } taos_free_result(res2); } - taos_free_result(res); break; } case SCHEMA_ACTION_CHANGE_TAG_SIZE: { @@ -425,12 +437,16 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInf if (code != TSDB_CODE_SUCCESS) { tscError("SML:0x%"PRIx64" apply schema action. error : %s", info->id, taos_errstr(res)); } + taos_free_result(res); + if (code == TSDB_CODE_MND_INVALID_TAG_LENGTH) { TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); code = taos_errno(res2); + if (code != TSDB_CODE_SUCCESS) { + tscError("SML:0x%" PRIx64 " apply schema action. reset query cache. error: %s", info->id, taos_errstr(res2)); + } taos_free_result(res2); } - taos_free_result(res); break; } case SCHEMA_ACTION_CREATE_STABLE: { @@ -462,12 +478,16 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInf if (code != TSDB_CODE_SUCCESS) { tscError("SML:0x%"PRIx64" apply schema action. error : %s", info->id, taos_errstr(res)); } + taos_free_result(res); + if (code == TSDB_CODE_MND_TABLE_ALREADY_EXIST) { TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); code = taos_errno(res2); + if (code != TSDB_CODE_SUCCESS) { + tscError("SML:0x%" PRIx64 " apply schema action. reset query cache. error: %s", info->id, taos_errstr(res2)); + } taos_free_result(res2); } - taos_free_result(res); break; } @@ -490,70 +510,12 @@ static int32_t destroySmlSTableSchema(SSmlSTableSchema* schema) { return 0; } -int32_t loadTableMeta(TAOS* taos, char* tableName, SSmlSTableSchema* schema, SSmlLinesInfo* info) { - int32_t code = 0; - - STscObj *pObj = (STscObj *)taos; - if (pObj == NULL || pObj->signature != pObj) { - terrno = TSDB_CODE_TSC_DISCONNECTED; - return TSDB_CODE_TSC_DISCONNECTED; - } - - tscDebug("SML:0x%"PRIx64" load table schema. super table name: %s", info->id, tableName); - - char tableNameLowerCase[TSDB_TABLE_NAME_LEN]; - strtolower(tableNameLowerCase, tableName); - - char sql[256]; - snprintf(sql, 256, "describe %s", tableNameLowerCase); - TAOS_RES* res = taos_query(taos, sql); - code = taos_errno(res); - if (code != 0) { - tscError("SML:0x%"PRIx64" describe table failure. %s", info->id, taos_errstr(res)); - taos_free_result(res); - return code; - } - taos_free_result(res); - - SSqlObj* pSql = calloc(1, sizeof(SSqlObj)); - if (pSql == NULL){ - tscError("failed to allocate memory, reason:%s", strerror(errno)); - code = TSDB_CODE_TSC_OUT_OF_MEMORY; - return code; - } - pSql->pTscObj = taos; - pSql->signature = pSql; - pSql->fp = NULL; - - SStrToken tableToken = {.z=tableNameLowerCase, .n=(uint32_t)strlen(tableNameLowerCase), .type=TK_ID}; - tGetToken(tableNameLowerCase, &tableToken.type); - // Check if the table name available or not - if (tscValidateName(&tableToken) != TSDB_CODE_SUCCESS) { - code = TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; - sprintf(pSql->cmd.payload, "table name is invalid"); - tscFreeSqlObj(pSql); - return code; - } - - SName sname = {0}; - if ((code = tscSetTableFullName(&sname, &tableToken, pSql)) != TSDB_CODE_SUCCESS) { - tscFreeSqlObj(pSql); - return code; - } - char fullTableName[TSDB_TABLE_FNAME_LEN] = {0}; - memset(fullTableName, 0, tListLen(fullTableName)); - tNameExtractFullName(&sname, fullTableName); - tscFreeSqlObj(pSql); - +static int32_t fillDbSchema(STableMeta* tableMeta, char* tableName, SSmlSTableSchema* schema, SSmlLinesInfo* info) { schema->tags = taosArrayInit(8, sizeof(SSchema)); schema->fields = taosArrayInit(64, sizeof(SSchema)); schema->tagHash = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); schema->fieldHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); - size_t size = 0; - STableMeta* tableMeta = NULL; - taosHashGetCloneExt(tscTableMetaMap, fullTableName, strlen(fullTableName), NULL, (void **)&tableMeta, &size); - tstrncpy(schema->sTableName, tableName, strlen(tableName)+1); schema->precision = tableMeta->tableInfo.precision; for (int i=0; itableInfo.numOfColumns; ++i) { @@ -576,9 +538,92 @@ int32_t loadTableMeta(TAOS* taos, char* tableName, SSmlSTableSchema* schema, SSm size_t tagIndex = taosArrayGetSize(schema->tags) - 1; taosHashPut(schema->tagHash, field.name, strlen(field.name), &tagIndex, sizeof(tagIndex)); } - tscDebug("SML:0x%"PRIx64 " load table meta succeed. table name: %s, columns number: %d, tag number: %d, precision: %d", + tscDebug("SML:0x%"PRIx64 " load table schema succeed. table name: %s, columns number: %d, tag number: %d, precision: %d", info->id, tableName, tableMeta->tableInfo.numOfColumns, tableMeta->tableInfo.numOfTags, schema->precision); - free(tableMeta); tableMeta = NULL; + return TSDB_CODE_SUCCESS; +} + +static int32_t retrieveTableMeta(TAOS* taos, char* tableName, STableMeta** pTableMeta, SSmlLinesInfo* info) { + int32_t code = 0; + int32_t retries = 0; + STableMeta* tableMeta = NULL; + while (retries++ < TSDB_MAX_REPLICA && tableMeta == NULL) { + STscObj* pObj = (STscObj*)taos; + if (pObj == NULL || pObj->signature != pObj) { + terrno = TSDB_CODE_TSC_DISCONNECTED; + return TSDB_CODE_TSC_DISCONNECTED; + } + + tscDebug("SML:0x%" PRIx64 " retrieve table meta. super table name: %s", info->id, tableName); + + char tableNameLowerCase[TSDB_TABLE_NAME_LEN]; + strtolower(tableNameLowerCase, tableName); + + char sql[256]; + snprintf(sql, 256, "describe %s", tableNameLowerCase); + TAOS_RES* res = taos_query(taos, sql); + code = taos_errno(res); + if (code != 0) { + tscError("SML:0x%" PRIx64 " describe table failure. %s", info->id, taos_errstr(res)); + taos_free_result(res); + return code; + } + taos_free_result(res); + + SSqlObj* pSql = calloc(1, sizeof(SSqlObj)); + if (pSql == NULL) { + tscError("SML:0x%" PRIx64 " failed to allocate memory, reason:%s", info->id, strerror(errno)); + code = TSDB_CODE_TSC_OUT_OF_MEMORY; + return code; + } + pSql->pTscObj = taos; + pSql->signature = pSql; + pSql->fp = NULL; + + SStrToken tableToken = {.z = tableNameLowerCase, .n = (uint32_t)strlen(tableNameLowerCase), .type = TK_ID}; + tGetToken(tableNameLowerCase, &tableToken.type); + // Check if the table name available or not + if (tscValidateName(&tableToken) != TSDB_CODE_SUCCESS) { + code = TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; + sprintf(pSql->cmd.payload, "table name is invalid"); + tscFreeSqlObj(pSql); + return code; + } + + SName sname = {0}; + if ((code = tscSetTableFullName(&sname, &tableToken, pSql)) != TSDB_CODE_SUCCESS) { + tscFreeSqlObj(pSql); + return code; + } + char fullTableName[TSDB_TABLE_FNAME_LEN] = {0}; + memset(fullTableName, 0, tListLen(fullTableName)); + tNameExtractFullName(&sname, fullTableName); + tscFreeSqlObj(pSql); + + size_t size = 0; + taosHashGetCloneExt(tscTableMetaMap, fullTableName, strlen(fullTableName), NULL, (void**)&tableMeta, &size); + } + + if (tableMeta != NULL) { + *pTableMeta = tableMeta; + return TSDB_CODE_SUCCESS; + } else { + tscError("SML:0x%" PRIx64 " failed to retrieve table meta. super table name: %s", info->id, tableName); + return TSDB_CODE_TSC_NO_META_CACHED; + } +} + +static int32_t loadTableSchemaFromDB(TAOS* taos, char* tableName, SSmlSTableSchema* schema, SSmlLinesInfo* info) { + int32_t code = 0; + STableMeta* tableMeta = NULL; + code = retrieveTableMeta(taos, tableName, &tableMeta, info); + if (code == TSDB_CODE_SUCCESS) { + assert(tableMeta != NULL); + fillDbSchema(tableMeta, tableName, schema, info); + free(tableMeta); + tableMeta = NULL; + } + return code; } @@ -590,7 +635,7 @@ static int32_t modifyDBSchemas(TAOS* taos, SArray* stableSchemas, SSmlLinesInfo* SSmlSTableSchema dbSchema; memset(&dbSchema, 0, sizeof(SSmlSTableSchema)); - code = loadTableMeta(taos, pointSchema->sTableName, &dbSchema, info); + code = loadTableSchemaFromDB(taos, pointSchema->sTableName, &dbSchema, info); if (code == TSDB_CODE_MND_INVALID_TABLE_NAME) { SSchemaAction schemaAction = {0}; schemaAction.action = SCHEMA_ACTION_CREATE_STABLE; @@ -599,7 +644,7 @@ static int32_t modifyDBSchemas(TAOS* taos, SArray* stableSchemas, SSmlLinesInfo* schemaAction.createSTable.tags = pointSchema->tags; schemaAction.createSTable.fields = pointSchema->fields; applySchemaAction(taos, &schemaAction, info); - code = loadTableMeta(taos, pointSchema->sTableName, &dbSchema, info); + code = loadTableSchemaFromDB(taos, pointSchema->sTableName, &dbSchema, info); if (code != 0) { tscError("SML:0x%"PRIx64" reconcile point schema failed. can not create %s", info->id, pointSchema->sTableName); return code; diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index 39a21a6747..1bc5607da5 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -1246,13 +1246,13 @@ static int32_t mnodeAddSuperTableTag(SMnodeMsg *pMsg, SSchema schema[], int32_t if (mnodeFindSuperTableColumnIndex(pStable, schema[i].name) > 0) { mError("msg:%p, app:%p stable:%s, add tag, column:%s already exist", pMsg, pMsg->rpcMsg.ahandle, pStable->info.tableId, schema[i].name); - return TSDB_CODE_MND_TAG_ALREAY_EXIST; + return TSDB_CODE_MND_FIELD_ALREAY_EXIST; } if (mnodeFindSuperTableTagIndex(pStable, schema[i].name) > 0) { mError("msg:%p, app:%p stable:%s, add tag, tag:%s already exist", pMsg, pMsg->rpcMsg.ahandle, pStable->info.tableId, schema[i].name); - return TSDB_CODE_MND_FIELD_ALREAY_EXIST; + return TSDB_CODE_MND_TAG_ALREAY_EXIST; } } diff --git a/tests/pytest/insert/schemalessInsert.py b/tests/pytest/insert/schemalessInsert.py index 00e805d791..9871af4b6d 100644 --- a/tests/pytest/insert/schemalessInsert.py +++ b/tests/pytest/insert/schemalessInsert.py @@ -1145,7 +1145,7 @@ class TDTestCase: s_stb_d_tb_a_col_m_tag_list = self.genSqlList(stb_name=stb_name)[5] self.multiThreadRun(self.genMultiThreadSeq(s_stb_d_tb_a_col_m_tag_list)) tdSql.query(f"show tables;") - tdSql.checkRows(6) + tdSql.checkRows(3) def sStbDtbDdataAtMcInsertMultiThreadCheckCase(self): """ @@ -1242,7 +1242,7 @@ class TDTestCase: s_stb_d_tb_d_ts_a_col_m_tag_list = self.genSqlList(stb_name=stb_name)[11] self.multiThreadRun(self.genMultiThreadSeq(s_stb_d_tb_d_ts_a_col_m_tag_list)) tdSql.query(f"show tables;") - tdSql.checkRows(6) + #tdSql.checkRows(6) def test(self): input_sql1 = "rfasta,id=\"rfasta_1\",t0=true,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64,t7=\"ddzhiksj\",t8=L\"ncharTagValue\" c0=True,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=\"bnhwlgvj\",c8=L\"ncharTagValue\",c9=7u64 1626006933640000000ns" @@ -1290,26 +1290,26 @@ class TDTestCase: # self.multiInsertCheckCase(1000) # self.batchErrorInsertCheckCase() # # MultiThreads - # self.stbInsertMultiThreadCheckCase() - # self.sStbStbDdataInsertMultiThreadCheckCase() - # self.sStbStbDdataAtcInsertMultiThreadCheckCase() - # self.sStbStbDdataMtcInsertMultiThreadCheckCase() - # self.sStbDtbDdataInsertMultiThreadCheckCase() + self.stbInsertMultiThreadCheckCase() + self.sStbStbDdataInsertMultiThreadCheckCase() + self.sStbStbDdataAtcInsertMultiThreadCheckCase() + self.sStbStbDdataMtcInsertMultiThreadCheckCase() + self.sStbDtbDdataInsertMultiThreadCheckCase() # # ! concurrency conflict self.sStbDtbDdataAcMtInsertMultiThreadCheckCase() - # self.sStbDtbDdataAtMcInsertMultiThreadCheckCase() + self.sStbDtbDdataAtMcInsertMultiThreadCheckCase() - # self.sStbStbDdataDtsInsertMultiThreadCheckCase() + self.sStbStbDdataDtsInsertMultiThreadCheckCase() # # ! concurrency conflict - # self.sStbStbDdataDtsAcMtInsertMultiThreadCheckCase() - # self.sStbStbDdataDtsAtMcInsertMultiThreadCheckCase() + self.sStbStbDdataDtsAcMtInsertMultiThreadCheckCase() + self.sStbStbDdataDtsAtMcInsertMultiThreadCheckCase() - # self.sStbDtbDdataDtsInsertMultiThreadCheckCase() + self.sStbDtbDdataDtsInsertMultiThreadCheckCase() # ! concurrency conflict - # self.sStbDtbDdataDtsAcMtInsertMultiThreadCheckCase() + self.sStbDtbDdataDtsAcMtInsertMultiThreadCheckCase() From 806451934b73165472da0d93948b116587887822 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Mon, 23 Aug 2021 08:39:29 +0800 Subject: [PATCH 06/40] [TD-6091]:ignore tag size/column size error during multi-thread schemaless insertion --- src/client/src/tscParseLineProtocol.c | 4 ++-- src/client/src/tscSQLParser.c | 5 ++--- src/inc/taoserror.h | 2 ++ 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/client/src/tscParseLineProtocol.c b/src/client/src/tscParseLineProtocol.c index 03e361c950..575cd80db6 100644 --- a/src/client/src/tscParseLineProtocol.c +++ b/src/client/src/tscParseLineProtocol.c @@ -418,7 +418,7 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInf } taos_free_result(res); - if (code == TSDB_CODE_MND_INVALID_COLUMN_LENGTH) { + if (code == TSDB_CODE_MND_INVALID_COLUMN_LENGTH || code == TSDB_CODE_TSC_INVALID_COLUMN_LENGTH) { TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); code = taos_errno(res2); if (code != TSDB_CODE_SUCCESS) { @@ -439,7 +439,7 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInf } taos_free_result(res); - if (code == TSDB_CODE_MND_INVALID_TAG_LENGTH) { + if (code == TSDB_CODE_MND_INVALID_TAG_LENGTH || code == TSDB_CODE_TSC_INVALID_TAG_LENGTH) { TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); code = taos_errno(res2); if (code != TSDB_CODE_SUCCESS) { diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 64f88b373d..bb79b93613 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -6021,7 +6021,6 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { const char* msg19 = "invalid new tag name"; const char* msg20 = "table is not super table"; const char* msg21 = "only binary/nchar column length could be modified"; - const char* msg22 = "new column length should be bigger than old one"; const char* msg23 = "only column length coulbe be modified"; const char* msg24 = "invalid binary/nchar column length"; @@ -6317,7 +6316,7 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { } if (pItem->bytes <= pColSchema->bytes) { - return invalidOperationMsg(pMsg, msg22); + return tscErrorMsgWithCode(TSDB_CODE_TSC_INVALID_COLUMN_LENGTH, tscGetErrorMsgPayload(pCmd), pItem->name, NULL); } SSchema* pSchema = (SSchema*) pTableMetaInfo->pTableMeta->schema; @@ -6368,7 +6367,7 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { } if (pItem->bytes <= pColSchema->bytes) { - return invalidOperationMsg(pMsg, msg22); + return tscErrorMsgWithCode(TSDB_CODE_TSC_INVALID_TAG_LENGTH, tscGetErrorMsgPayload(pCmd), pItem->name, NULL); } SSchema* pSchema = tscGetTableTagSchema(pTableMetaInfo->pTableMeta); diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index f54150d76b..08f6d0bf4d 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -104,6 +104,8 @@ int32_t* taosGetErrno(); #define TSDB_CODE_TSC_LINE_SYNTAX_ERROR TAOS_DEF_ERROR_CODE(0, 0x021B) //"Syntax error in Line") #define TSDB_CODE_TSC_NO_META_CACHED TAOS_DEF_ERROR_CODE(0, 0x021C) //"No table meta cached") #define TSDB_CODE_TSC_DUP_COL_NAMES TAOS_DEF_ERROR_CODE(0, 0x021D) //"duplicated column names") +#define TSDB_CODE_TSC_INVALID_TAG_LENGTH TAOS_DEF_ERROR_CODE(0, 0x021E) //"Invalid tag length") +#define TSDB_CODE_TSC_INVALID_COLUMN_LENGTH TAOS_DEF_ERROR_CODE(0, 0x021F) //"Invalid tag length") // mnode #define TSDB_CODE_MND_MSG_NOT_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0300) //"Message not processed") From 4714688f30198d84da2532d767ff726cb350e15a Mon Sep 17 00:00:00 2001 From: Shenglian Zhou Date: Mon, 23 Aug 2021 10:08:29 +0800 Subject: [PATCH 07/40] [TD-6199]:restore schemless test file before development --- tests/pytest/insert/schemalessInsert.py | 72 ++++++++++++------------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/tests/pytest/insert/schemalessInsert.py b/tests/pytest/insert/schemalessInsert.py index 9871af4b6d..49c3223588 100644 --- a/tests/pytest/insert/schemalessInsert.py +++ b/tests/pytest/insert/schemalessInsert.py @@ -1145,7 +1145,7 @@ class TDTestCase: s_stb_d_tb_a_col_m_tag_list = self.genSqlList(stb_name=stb_name)[5] self.multiThreadRun(self.genMultiThreadSeq(s_stb_d_tb_a_col_m_tag_list)) tdSql.query(f"show tables;") - tdSql.checkRows(3) + tdSql.checkRows(6) def sStbDtbDdataAtMcInsertMultiThreadCheckCase(self): """ @@ -1242,7 +1242,7 @@ class TDTestCase: s_stb_d_tb_d_ts_a_col_m_tag_list = self.genSqlList(stb_name=stb_name)[11] self.multiThreadRun(self.genMultiThreadSeq(s_stb_d_tb_d_ts_a_col_m_tag_list)) tdSql.query(f"show tables;") - #tdSql.checkRows(6) + tdSql.checkRows(6) def test(self): input_sql1 = "rfasta,id=\"rfasta_1\",t0=true,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64,t7=\"ddzhiksj\",t8=L\"ncharTagValue\" c0=True,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=\"bnhwlgvj\",c8=L\"ncharTagValue\",c9=7u64 1626006933640000000ns" @@ -1261,35 +1261,35 @@ class TDTestCase: # self._conn.insert_lines([input_sql4]) def runAll(self): - # self.initCheckCase() - # self.boolTypeCheckCase() - # self.symbolsCheckCase() - # self.tsCheckCase() - # self.idSeqCheckCase() - # self.idUpperCheckCase() - # self.noIdCheckCase() - # self.maxColTagCheckCase() - # self.idIllegalNameCheckCase() - # self.idStartWithNumCheckCase() - # self.nowTsCheckCase() - # self.dateFormatTsCheckCase() - # self.illegalTsCheckCase() - # self.tagValueLengthCheckCase() - # self.colValueLengthCheckCase() - # self.tagColIllegalValueCheckCase() - # self.duplicateIdTagColInsertCheckCase() - # self.noIdStbExistCheckCase() - # self.duplicateInsertExistCheckCase() - # self.tagColBinaryNcharLengthCheckCase() - # self.tagColAddDupIDCheckCase() - # self.tagColAddCheckCase() - # self.tagMd5Check() - # self.tagColBinaryMaxLengthCheckCase() - # # self.tagColNcharMaxLengthCheckCase() - # self.batchInsertCheckCase() - # self.multiInsertCheckCase(1000) - # self.batchErrorInsertCheckCase() - # # MultiThreads + self.initCheckCase() + self.boolTypeCheckCase() + self.symbolsCheckCase() + self.tsCheckCase() + self.idSeqCheckCase() + self.idUpperCheckCase() + self.noIdCheckCase() + self.maxColTagCheckCase() + self.idIllegalNameCheckCase() + self.idStartWithNumCheckCase() + self.nowTsCheckCase() + self.dateFormatTsCheckCase() + self.illegalTsCheckCase() + self.tagValueLengthCheckCase() + self.colValueLengthCheckCase() + self.tagColIllegalValueCheckCase() + self.duplicateIdTagColInsertCheckCase() + self.noIdStbExistCheckCase() + self.duplicateInsertExistCheckCase() + self.tagColBinaryNcharLengthCheckCase() + self.tagColAddDupIDCheckCase() + self.tagColAddCheckCase() + self.tagMd5Check() + self.tagColBinaryMaxLengthCheckCase() + # self.tagColNcharMaxLengthCheckCase() + self.batchInsertCheckCase() + self.multiInsertCheckCase(1000) + self.batchErrorInsertCheckCase() + # MultiThreads self.stbInsertMultiThreadCheckCase() self.sStbStbDdataInsertMultiThreadCheckCase() self.sStbStbDdataAtcInsertMultiThreadCheckCase() @@ -1297,19 +1297,19 @@ class TDTestCase: self.sStbDtbDdataInsertMultiThreadCheckCase() # # ! concurrency conflict - self.sStbDtbDdataAcMtInsertMultiThreadCheckCase() - self.sStbDtbDdataAtMcInsertMultiThreadCheckCase() + # self.sStbDtbDdataAcMtInsertMultiThreadCheckCase() + # self.sStbDtbDdataAtMcInsertMultiThreadCheckCase() self.sStbStbDdataDtsInsertMultiThreadCheckCase() # # ! concurrency conflict - self.sStbStbDdataDtsAcMtInsertMultiThreadCheckCase() - self.sStbStbDdataDtsAtMcInsertMultiThreadCheckCase() + # self.sStbStbDdataDtsAcMtInsertMultiThreadCheckCase() + # self.sStbStbDdataDtsAtMcInsertMultiThreadCheckCase() self.sStbDtbDdataDtsInsertMultiThreadCheckCase() # ! concurrency conflict - self.sStbDtbDdataDtsAcMtInsertMultiThreadCheckCase() + # self.sStbDtbDdataDtsAcMtInsertMultiThreadCheckCase() From 0c09bbb9ba0a0c74296e687a084f19e5643841ef Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Mon, 23 Aug 2021 15:45:34 +0800 Subject: [PATCH 08/40] only performance test and leave corretness test to python test cases --- tests/examples/c/schemaless.c | 118 +--------------------------------- 1 file changed, 2 insertions(+), 116 deletions(-) diff --git a/tests/examples/c/schemaless.c b/tests/examples/c/schemaless.c index 3ea199c914..1a551cc5f7 100644 --- a/tests/examples/c/schemaless.c +++ b/tests/examples/c/schemaless.c @@ -61,7 +61,7 @@ int main(int argc, char* argv[]) { time_t ct = time(0); int64_t ts = ct * 1000; - char* lineFormat = "sta%d,t0=true,t1=127i8,t2=32767i16,t3=%di32,t4=9223372036854775807i64,t9=11.12345f32,t10=22.123456789f64,t11=\"binaryTagValue\",t12=L\"ncharTagValue\" c0=true,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=255u8,c6=32770u16,c7=2147483699u32,c8=9223372036854775899u64,c9=11.12345f32,c10=22.123456789f64,c11=\"binaryValue\",c12=L\"ncharValue\" %lldms"; + char* lineFormat = "sta%d,t0=true,t1=127i8,t2=32767i16,t3=%di32,t4=9223372036854775807i64,t9=11.12345f32,t10=22.123456789f64,t11=\"binaryTagValue\",t12=L\"ncharTagValue\" c0=true,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=254u8,c6=32770u16,c7=2147483699u32,c8=9223372036854775899u64,c9=11.12345f32,c10=22.123456789f64,c11=\"binaryValue\",c12=L\"ncharValue\" %lldms"; char** lines = calloc(numSuperTables * numChildTables * numRowsPerChildTable, sizeof(char*)); int l = 0; @@ -75,7 +75,7 @@ int main(int argc, char* argv[]) { } } } - shuffle(lines, numSuperTables * numChildTables * numRowsPerChildTable); + //shuffle(lines, numSuperTables * numChildTables * numRowsPerChildTable); printf("%s\n", "begin taos_insert_lines"); int64_t begin = getTimeInUs(); @@ -83,119 +83,5 @@ int main(int argc, char* argv[]) { int64_t end = getTimeInUs(); printf("code: %d, %s. time used: %"PRId64"\n", code, tstrerror(code), end-begin); - char* lines_000_0[] = { - "sta1,id=sta1_1,t0=true,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=255u8,t6=32770u16,t7=2147483699u32,t8=9223372036854775899u64,t9=11.12345f32,t10=22.123456789f64,t11=\"binaryTagValue\",t12=L\"ncharTagValue\" c0=true,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=255u8,c6=32770u16,c7=2147483699u32,c8=9223372036854775899u64,c9=11.12345f32,c10=22.123456789f64,c11=\"binaryValue\",c12=L\"ncharValue\" 1626006833639000us" - }; - - code = taos_insert_lines(taos, lines_000_0 , sizeof(lines_000_0)/sizeof(char*)); - if (0 == code) { - printf("taos_insert_lines() lines_000_0 should return error\n"); - return -1; - } - - char* lines_000_1[] = { - "sta2,id=\"sta2_1\",t0=true,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=255u8,t6=32770u16,t7=2147483699u32,t8=9223372036854775899u64,t9=11.12345f32,t10=22.123456789f64,t11=\"binaryTagValue\",t12=L\"ncharTagValue\" c0=true,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=255u8,c6=32770u16,c7=2147483699u32,c8=9223372036854775899u64,c9=11.12345f32,c10=22.123456789f64,c11=\"binaryValue\",c12=L\"ncharValue\" 1626006833639001" - }; - - code = taos_insert_lines(taos, lines_000_1 , sizeof(lines_000_1)/sizeof(char*)); - if (0 == code) { - printf("taos_insert_lines() lines_000_1 should return error\n"); - return -1; - } - - char* lines_000_2[] = { - "sta3,id=\"sta3_1\",t0=true,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t9=11.12345f32,t10=22.123456789f64,t11=\"binaryTagValue\",t12=L\"ncharTagValue\" c0=true,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=255u8,c6=32770u16,c7=2147483699u32,c8=9223372036854775899u64,c9=11.12345f32,c10=22.123456789f64,c11=\"binaryValue\",c12=L\"ncharValue\" 0" - }; - - code = taos_insert_lines(taos, lines_000_2 , sizeof(lines_000_2)/sizeof(char*)); - if (0 != code) { - printf("taos_insert_lines() lines_000_2 return code:%d (%s)\n", code, (char*)tstrerror(code)); - return -1; - } - - char* lines_001_0[] = { - "sta4,t0=true,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t9=11.12345f32,t10=22.123456789f64,t11=\"binaryTagValue\",t12=L\"ncharTagValue\" c0=true,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c9=11.12345f32,c10=22.123456789f64,c11=\"binaryValue\",c12=L\"ncharValue\" 1626006833639000us", - - }; - - code = taos_insert_lines(taos, lines_001_0 , sizeof(lines_001_0)/sizeof(char*)); - if (0 != code) { - printf("taos_insert_lines() lines_001_0 return code:%d (%s)\n", code, (char*)tstrerror(code)); - return -1; - } - - char* lines_001_1[] = { - "sta5,id=\"sta5_1\",t0=true,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t9=11.12345f32,t10=22.123456789f64,t11=\"binaryTagValue\",t12=L\"ncharTagValue\" c0=true,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c9=11.12345f32,c10=22.123456789f64,c11=\"binaryValue\",c12=L\"ncharValue\" 1626006833639001" - }; - - code = taos_insert_lines(taos, lines_001_1 , sizeof(lines_001_1)/sizeof(char*)); - if (0 != code) { - printf("taos_insert_lines() lines_001_1 return code:%d (%s)\n", code, (char*)tstrerror(code)); - return -1; - } - - char* lines_001_2[] = { - "sta6,id=\"sta6_1\",t0=true,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t9=11.12345f32,t10=22.123456789f64,t11=\"binaryTagValue\",t12=L\"ncharTagValue\" c0=true,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c9=11.12345f32,c10=22.123456789f64,c11=\"binaryValue\",c12=L\"ncharValue\" 0" - }; - - code = taos_insert_lines(taos, lines_001_2 , sizeof(lines_001_2)/sizeof(char*)); - if (0 != code) { - printf("taos_insert_lines() lines_001_2 return code:%d (%s)\n", code, (char*)tstrerror(code)); - return -1; - } - - char* lines_002[] = { - "stb,id=\"stb_1\",t20=t,t21=T,t22=true,t23=True,t24=TRUE,t25=f,t26=F,t27=false,t28=False,t29=FALSE,t10=33.12345,t11=\"binaryTagValue\",t12=L\"ncharTagValue\" c20=t,c21=T,c22=true,c23=True,c24=TRUE,c25=f,c26=F,c27=false,c28=False,c29=FALSE,c10=33.12345,c11=\"binaryValue\",c12=L\"ncharValue\" 1626006833639000000ns", - "stc,id=\"stc_1\",t20=t,t21=T,t22=true,t23=True,t24=TRUE,t25=f,t26=F,t27=false,t28=False,t29=FALSE,t10=33.12345,t11=\"binaryTagValue\",t12=L\"ncharTagValue\" c20=t,c21=T,c22=true,c23=True,c24=TRUE,c25=f,c26=F,c27=false,c28=False,c29=FALSE,c10=33.12345,c11=\"binaryValue\",c12=L\"ncharValue\" 1626006833639019us", - "stc,id=\"stc_1\",t20=t,t21=T,t22=true,t23=True,t24=TRUE,t25=f,t26=F,t27=false,t28=False,t29=FALSE,t10=33.12345,t11=\"binaryTagValue\",t12=L\"ncharTagValue\" c20=t,c21=T,c22=true,c23=True,c24=TRUE,c25=f,c26=F,c27=false,c28=False,c29=FALSE,c10=33.12345,c11=\"binaryValue\",c12=L\"ncharValue\" 1626006833640ms", - "stc,id=\"stc_1\",t20=t,t21=T,t22=true,t23=True,t24=TRUE,t25=f,t26=F,t27=false,t28=False,t29=FALSE,t10=33.12345,t11=\"binaryTagValue\",t12=L\"ncharTagValue\" c20=t,c21=T,c22=true,c23=True,c24=TRUE,c25=f,c26=F,c27=false,c28=False,c29=FALSE,c10=33.12345,c11=\"binaryValue\",c12=L\"ncharValue\" 1626006834s" - }; - - code = taos_insert_lines(taos, lines_002 , sizeof(lines_002)/sizeof(char*)); - if (0 != code) { - printf("taos_insert_lines() lines_002 return code:%d (%s)\n", code, (char*)tstrerror(code)); - return -1; - } - - //Duplicate key check; - char* lines_003_1[] = { - "std,id=\"std_3_1\",t1=4i64,Id=\"std\",t2=true c1=true 1626006834s" - }; - - code = taos_insert_lines(taos, lines_003_1 , sizeof(lines_003_1)/sizeof(char*)); - if (0 == code) { - printf("taos_insert_lines() lines_003_1 return code:%d (%s)\n", code, (char*)tstrerror(code)); - return -1; - } - - char* lines_003_2[] = { - "std,id=\"std_3_2\",tag1=4i64,Tag2=true,tAg3=2,TaG2=\"dup!\" c1=true 1626006834s" - }; - - code = taos_insert_lines(taos, lines_003_2 , sizeof(lines_003_2)/sizeof(char*)); - if (0 == code) { - printf("taos_insert_lines() lines_003_2 return code:%d (%s)\n", code, (char*)tstrerror(code)); - return -1; - } - - char* lines_003_3[] = { - "std,id=\"std_3_3\",tag1=4i64 field1=true,Field2=2,FIElD1=\"dup!\",fIeLd4=true 1626006834s" - }; - - code = taos_insert_lines(taos, lines_003_3 , sizeof(lines_003_3)/sizeof(char*)); - if (0 == code) { - printf("taos_insert_lines() lines_003_3 return code:%d (%s)\n", code, (char*)tstrerror(code)); - return -1; - } - - char* lines_003_4[] = { - "std,id=\"std_3_4\",tag1=4i64,dupkey=4i16,tag2=T field1=true,dUpkEy=1e3f32,field2=\"1234\" 1626006834s" - }; - - code = taos_insert_lines(taos, lines_003_4 , sizeof(lines_003_4)/sizeof(char*)); - if (0 == code) { - printf("taos_insert_lines() lines_003_4 return code:%d (%s)\n", code, (char*)tstrerror(code)); - return -1; - } return 0; } From e9b9b9439d55ea6529ba273769f95d0a3aa10fc1 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Tue, 24 Aug 2021 11:42:15 +0800 Subject: [PATCH 09/40] [TD-5201]:register sql object whenever --- src/client/src/tscParseLineProtocol.c | 29 +++++++++++++++------------ src/client/src/tscPrepare.c | 4 ++-- 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/src/client/src/tscParseLineProtocol.c b/src/client/src/tscParseLineProtocol.c index 575cd80db6..bde68c306a 100644 --- a/src/client/src/tscParseLineProtocol.c +++ b/src/client/src/tscParseLineProtocol.c @@ -580,25 +580,26 @@ static int32_t retrieveTableMeta(TAOS* taos, char* tableName, STableMeta** pTabl pSql->signature = pSql; pSql->fp = NULL; + registerSqlObj(pSql); SStrToken tableToken = {.z = tableNameLowerCase, .n = (uint32_t)strlen(tableNameLowerCase), .type = TK_ID}; tGetToken(tableNameLowerCase, &tableToken.type); // Check if the table name available or not if (tscValidateName(&tableToken) != TSDB_CODE_SUCCESS) { code = TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; sprintf(pSql->cmd.payload, "table name is invalid"); - tscFreeSqlObj(pSql); + tscFreeRegisteredSqlObj(pSql); return code; } SName sname = {0}; if ((code = tscSetTableFullName(&sname, &tableToken, pSql)) != TSDB_CODE_SUCCESS) { - tscFreeSqlObj(pSql); + tscFreeRegisteredSqlObj(pSql); return code; } char fullTableName[TSDB_TABLE_FNAME_LEN] = {0}; memset(fullTableName, 0, tListLen(fullTableName)); tNameExtractFullName(&sname, fullTableName); - tscFreeSqlObj(pSql); + tscFreeRegisteredSqlObj(pSql); size_t size = 0; taosHashGetCloneExt(tscTableMetaMap, fullTableName, strlen(fullTableName), NULL, (void**)&tableMeta, &size); @@ -714,18 +715,21 @@ static int32_t changeChildTableTagValue(TAOS* taos, const char* cTableName, cons if (code != 0) { tscError("SML:0x%"PRIx64" taos_stmt_prepare return %d:%s", info->id, code, tstrerror(code)); + taos_stmt_close(stmt); return code; } code = taos_stmt_bind_param(stmt, bind); if (code != 0) { tscError("SML:0x%"PRIx64" taos_stmt_bind_param return %d:%s", info->id, code, tstrerror(code)); + taos_stmt_close(stmt); return code; } code = taos_stmt_execute(stmt); if (code != 0) { tscError("SML:0x%"PRIx64" taos_stmt_execute return %d:%s", info->id, code, tstrerror(code)); + taos_stmt_close(stmt); return code; } @@ -775,22 +779,22 @@ static int32_t creatChildTableIfNotExists(TAOS* taos, const char* cTableName, co free(sql); if (code != 0) { - tfree(stmt); tscError("SML:0x%"PRIx64" taos_stmt_prepare returns %d:%s", info->id, code, tstrerror(code)); + taos_stmt_close(stmt); return code; } code = taos_stmt_bind_param(stmt, TARRAY_GET_START(tagsBind)); if (code != 0) { - tfree(stmt); tscError("SML:0x%"PRIx64" taos_stmt_bind_param returns %d:%s", info->id, code, tstrerror(code)); + taos_stmt_close(stmt); return code; } code = taos_stmt_execute(stmt); if (code != 0) { - tfree(stmt); tscError("SML:0x%"PRIx64" taos_stmt_execute returns %d:%s", info->id, code, tstrerror(code)); + taos_stmt_close(stmt); return code; } @@ -835,20 +839,21 @@ static int32_t insertChildTableBatch(TAOS* taos, char* cTableName, SArray* cols tfree(sql); return TSDB_CODE_TSC_OUT_OF_MEMORY; } + code = taos_stmt_prepare(stmt, sql, (unsigned long)strlen(sql)); tfree(sql); if (code != 0) { - tfree(stmt); tscError("SML:0x%"PRIx64" taos_stmt_prepare return %d:%s", info->id, code, tstrerror(code)); + taos_stmt_close(stmt); return code; } do { code = taos_stmt_set_tbname(stmt, cTableName); if (code != 0) { - tfree(stmt); tscError("SML:0x%"PRIx64" taos_stmt_set_tbname return %d:%s", info->id, code, tstrerror(code)); + taos_stmt_close(stmt); return code; } @@ -857,14 +862,14 @@ static int32_t insertChildTableBatch(TAOS* taos, char* cTableName, SArray* cols TAOS_BIND* colsBinds = taosArrayGetP(rowsBind, i); code = taos_stmt_bind_param(stmt, colsBinds); if (code != 0) { - tfree(stmt); tscError("SML:0x%"PRIx64" taos_stmt_bind_param return %d:%s", info->id, code, tstrerror(code)); + taos_stmt_close(stmt); return code; } code = taos_stmt_add_batch(stmt); if (code != 0) { - tfree(stmt); tscError("SML:0x%"PRIx64" taos_stmt_add_batch return %d:%s", info->id, code, tstrerror(code)); + taos_stmt_close(stmt); return code; } } @@ -877,11 +882,9 @@ static int32_t insertChildTableBatch(TAOS* taos, char* cTableName, SArray* cols if (code != 0) { tscError("SML:0x%"PRIx64" %d:%s", info->id, code, tstrerror(code)); - taos_stmt_close(stmt); - } else { - taos_stmt_close(stmt); } + taos_stmt_close(stmt); return code; } diff --git a/src/client/src/tscPrepare.c b/src/client/src/tscPrepare.c index 7f7966559b..48393a3dda 100644 --- a/src/client/src/tscPrepare.c +++ b/src/client/src/tscPrepare.c @@ -1537,6 +1537,8 @@ int taos_stmt_prepare(TAOS_STMT* stmt, const char* sql, unsigned long length) { pRes->qId = 0; pRes->numOfRows = 1; + registerSqlObj(pSql); + strtolower(pSql->sqlstr, sql); tscDebugL("0x%"PRIx64" SQL: %s", pSql->self, pSql->sqlstr); @@ -1546,8 +1548,6 @@ int taos_stmt_prepare(TAOS_STMT* stmt, const char* sql, unsigned long length) { pSql->cmd.insertParam.numOfParams = 0; pSql->cmd.batchSize = 0; - registerSqlObj(pSql); - int32_t ret = stmtParseInsertTbTags(pSql, pStmt); if (ret != TSDB_CODE_SUCCESS) { STMT_RET(ret); From 060a7451c48cc0c8e9bd2b7911026af4cbaadd90 Mon Sep 17 00:00:00 2001 From: jiajingbin Date: Tue, 17 Aug 2021 16:36:08 +0800 Subject: [PATCH 10/40] add testcases 5 --- tests/pytest/query/queryDiffColsOr.py | 436 ++++++++++++++++++++++++++ 1 file changed, 436 insertions(+) create mode 100644 tests/pytest/query/queryDiffColsOr.py diff --git a/tests/pytest/query/queryDiffColsOr.py b/tests/pytest/query/queryDiffColsOr.py new file mode 100644 index 0000000000..68c8f14869 --- /dev/null +++ b/tests/pytest/query/queryDiffColsOr.py @@ -0,0 +1,436 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- +from copy import deepcopy +from util.log import tdLog +from util.cases import tdCases +from util.sql import tdSql +from util.common import tdCom + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def insertData(self, tb_name): + insert_sql_list = [f'insert into {tb_name} values ("2021-01-01 12:00:00", 1, 1, 1, 3, 1.1, 1.1, "binary", "nchar", true, 1)', + f'insert into {tb_name} values ("2021-01-05 12:00:00", 2, 2, 1, 3, 1.1, 1.1, "binary", "nchar", true, 2)', + f'insert into {tb_name} values ("2021-01-07 12:00:00", 1, 3, 1, 2, 1.1, 1.1, "binary", "nchar", true, 3)', + f'insert into {tb_name} values ("2021-01-09 12:00:00", 1, 2, 4, 3, 1.1, 1.1, "binary", "nchar", true, 4)', + f'insert into {tb_name} values ("2021-01-11 12:00:00", 1, 2, 5, 5, 1.1, 1.1, "binary", "nchar", true, 5)', + f'insert into {tb_name} values ("2021-01-13 12:00:00", 1, 2, 1, 3, 6.6, 1.1, "binary", "nchar", true, 6)', + f'insert into {tb_name} values ("2021-01-15 12:00:00", 1, 2, 1, 3, 1.1, 7.7, "binary", "nchar", true, 7)', + f'insert into {tb_name} values ("2021-01-17 12:00:00", 1, 2, 1, 3, 1.1, 1.1, "binary8", "nchar", true, 8)', + f'insert into {tb_name} values ("2021-01-19 12:00:00", 1, 2, 1, 3, 1.1, 1.1, "binary", "nchar9", true, 9)', + f'insert into {tb_name} values ("2021-01-21 12:00:00", 1, 2, 1, 3, 1.1, 1.1, "binary", "nchar", false, 10)', + f'insert into {tb_name} values ("2021-01-23 12:00:00", 1, 3, 1, 3, 1.1, 1.1, Null, Null, false, 11)' + ] + for sql in insert_sql_list: + tdSql.execute(sql) + + def initTb(self): + tdCom.cleanTb() + tb_name = tdCom.getLongName(8, "letters") + tdSql.execute( + f"CREATE TABLE {tb_name} (ts timestamp, c1 tinyint, c2 smallint, c3 int, c4 bigint, c5 float, c6 double, c7 binary(100), c8 nchar(200), c9 bool, c10 int)") + self.insertData(tb_name) + return tb_name + + def queryLastC10(self, query_sql, multi=False): + if multi: + res = tdSql.query(query_sql.replace('c10', 'last(*)'), True) + else: + res = tdSql.query(query_sql.replace('*', 'last(*)'), True) + return int(res[0][-1]) + + def queryFullColType(self, tb_name): + ## ts + query_sql = f'select * from {tb_name} where ts > "2021-01-11 12:00:00" or ts < "2021-01-13 12:00:00"' + tdSql.query(query_sql) + tdSql.checkRows(11) + tdSql.checkEqual(self.queryLastC10(query_sql), 11) + + ## != or + query_sql = f'select * from {tb_name} where c1 != 1 or c2 = 3' + tdSql.query(query_sql) + tdSql.checkRows(3) + tdSql.checkEqual(self.queryLastC10(query_sql), 11) + + ## <> or + query_sql = f'select * from {tb_name} where c1 <> 1 or c3 = 3' + tdSql.query(query_sql) + tdSql.checkRows(1) + tdSql.checkEqual(self.queryLastC10(query_sql), 2) + + ## >= or + query_sql = f'select * from {tb_name} where c1 >= 2 or c3 = 4' + tdSql.query(query_sql) + tdSql.checkRows(2) + tdSql.checkEqual(self.queryLastC10(query_sql), 4) + + ## <= or + query_sql = f'select * from {tb_name} where c1 <= 1 or c3 = 4' + tdSql.query(query_sql) + tdSql.checkRows(10) + tdSql.checkEqual(self.queryLastC10(query_sql), 11) + + ## <> or is Null + query_sql = f'select * from {tb_name} where c1 <> 1 or c7 is Null' + tdSql.query(query_sql) + tdSql.checkRows(2) + tdSql.checkEqual(self.queryLastC10(query_sql), 11) + + ## > or is not Null + query_sql = f'select * from {tb_name} where c2 > 2 or c8 is not Null' + tdSql.query(query_sql) + tdSql.checkRows(11) + tdSql.checkEqual(self.queryLastC10(query_sql), 11) + + ## > or < or >= or <= or != or <> or = Null + query_sql = f'select * from {tb_name} where c1 > 1 or c2 < 2 or c3 >= 4 or c4 <= 2 or c5 != 1.1 or c6 <> 1.1 or c7 is Null' + tdSql.query(query_sql) + tdSql.checkRows(8) + tdSql.checkEqual(self.queryLastC10(query_sql), 11) + + ## tiny small int big or + query_sql = f'select * from {tb_name} where c1 = 2 or c2 = 3 or c3 = 4 or c4 = 5' + tdSql.query(query_sql) + tdSql.checkRows(5) + tdSql.checkEqual(self.queryLastC10(query_sql), 11) + + ## float double binary nchar bool or + query_sql = f'select * from {tb_name} where c5=6.6 or c6=7.7 or c7="binary8" or c8="nchar9" or c9=false' + tdSql.query(query_sql) + tdSql.checkRows(6) + tdSql.checkEqual(self.queryLastC10(query_sql), 11) + + ## all types or + query_sql = f'select * from {tb_name} where c1=2 or c2=3 or c3=4 or c4=5 or c5=6.6 or c6=7.7 or c7="binary8" or c8="nchar9" or c9=false' + tdSql.query(query_sql) + tdSql.checkRows(10) + tdSql.checkEqual(self.queryLastC10(query_sql), 11) + + def checkTbColTypeOperator(self): + ''' + Ordinary table full column type and operator + ''' + tb_name = self.initTb() + self.queryFullColType(tb_name) + + def checkTbMultiExpression(self): + ''' + Ordinary table multiExpression + ''' + tb_name = self.initTb() + ## condition_A and condition_B or condition_C (> < >=) + query_sql = f'select * from {tb_name} where c1 > 2 and c2 < 4 or c3 >= 4' + tdSql.query(query_sql) + tdSql.checkRows(2) + tdSql.checkEqual(self.queryLastC10(query_sql), 5) + + ## (condition_A and condition_B) or condition_C (<= != <>) + query_sql = f'select * from {tb_name} where (c1 <= 1 and c2 != 2) or c4 <> 3' + tdSql.query(query_sql) + tdSql.checkRows(4) + tdSql.checkEqual(self.queryLastC10(query_sql), 11) + + ## condition_A and (condition_B or condition_C) (Null not Null) + query_sql = f'select * from {tb_name} where c1 is not Null and (c6 = 7.7 or c8 is Null)' + tdSql.query(query_sql) + tdSql.checkRows(2) + tdSql.checkEqual(self.queryLastC10(query_sql), 11) + + ## condition_A or condition_B and condition_C (> < >=) + query_sql = f'select * from {tb_name} where c1 > 2 or c2 < 4 and c3 >= 4' + tdSql.query(query_sql) + tdSql.checkRows(2) + tdSql.checkEqual(self.queryLastC10(query_sql), 5) + + ## (condition_A or condition_B) and condition_C (<= != <>) + query_sql = f'select * from {tb_name} where (c1 <= 1 or c2 != 2) and c4 <> 3' + tdSql.query(query_sql) + tdSql.checkRows(2) + tdSql.checkEqual(self.queryLastC10(query_sql), 5) + + ## condition_A or (condition_B and condition_C) (Null not Null) + query_sql = f'select * from {tb_name} where c6 >= 7.7 or (c1 is not Null and c3 =5)' + tdSql.query(query_sql) + tdSql.checkRows(2) + tdSql.checkEqual(self.queryLastC10(query_sql), 7) + + ## condition_A or (condition_B and condition_C) or condition_D (> != < Null) + query_sql = f'select * from {tb_name} where c1 != 1 or (c2 >2 and c3 < 1) or c7 is Null' + tdSql.query(query_sql) + tdSql.checkRows(2) + tdSql.checkEqual(self.queryLastC10(query_sql), 11) + + ## condition_A and (condition_B or condition_C) and condition_D (>= = <= not Null) + query_sql = f'select * from {tb_name} where c4 >= 4 and (c1 = 2 or c5 <= 1.1) and c7 is not Null' + tdSql.query(query_sql) + tdSql.checkRows(1) + tdSql.checkEqual(self.queryLastC10(query_sql), 5) + + ## (condition_A and condition_B) or (condition_C or condition_D) (Null >= > =) + query_sql = f'select * from {tb_name} where (c8 is Null and c1 >= 1) or (c3 > 3 or c4 =2)' + tdSql.query(query_sql) + tdSql.checkRows(4) + tdSql.checkEqual(self.queryLastC10(query_sql), 11) + + ## (condition_A or condition_B) or condition_C or (condition_D and condition_E) (>= <= = not Null <>) + query_sql = f'select * from {tb_name} where (c1 >= 2 or c2 <= 1) or c3 = 4 or (c7 is not Null and c6 <> 1.1)' + tdSql.query(query_sql) + tdSql.checkRows(4) + tdSql.checkEqual(self.queryLastC10(query_sql), 7) + + ## condition_A or (condition_B and condition_C) or (condition_D and condition_E) and condition_F + query_sql = f'select * from {tb_name} where c1 != 1 or (c2 <= 1 and c3 <4) or (c3 >= 4 or c7 is not Null) and c9 <> true' + tdSql.query(query_sql) + tdSql.checkRows(3) + tdSql.checkEqual(self.queryLastC10(query_sql), 10) + + ## (condition_A or (condition_B and condition_C) or (condition_D and condition_E)) and condition_F + query_sql = f'select * from {tb_name} where (c1 != 1 or (c2 <= 2 and c3 >= 4) or (c3 >= 4 or c7 is not Null)) and c9 != false' + tdSql.query(query_sql) + tdSql.checkRows(9) + tdSql.checkEqual(self.queryLastC10(query_sql), 9) + + ## (condition_A or condition_B) or (condition_C or condition_D) and (condition_E or condition_F or condition_G) + query_sql = f'select * from {tb_name} where c1 != 1 or (c2 <= 3 and c3 > 4) and c3 <= 5 and (c7 is not Null and c9 != false)' + tdSql.query(query_sql) + tdSql.checkRows(2) + tdSql.checkEqual(self.queryLastC10(query_sql), 5) + + def checkTbMultiIn(self): + ''' + Ordinary table multiIn + ''' + tb_name = self.initTb() + ## in and in + query_sql = f'select * from {tb_name} where c7 in ("binary") and c8 in ("nchar")' + tdSql.query(query_sql) + tdSql.checkRows(8) + tdSql.checkEqual(self.queryLastC10(query_sql), 10) + + ## in or in + query_sql = f'select * from {tb_name} where c1 in (2, 4) or c2 in (1, 4)' + tdSql.query(query_sql) + tdSql.checkRows(2) + tdSql.checkEqual(self.queryLastC10(query_sql), 2) + + ## in and in or condition_A + query_sql = f'select * from {tb_name} where c7 in ("binary") and c8 in ("nchar") or c10 != 10' + tdSql.query(query_sql) + tdSql.checkRows(11) + tdSql.checkEqual(self.queryLastC10(query_sql), 11) + + ## in or in and condition_A + query_sql = f'select * from {tb_name} where c7 in ("binary") or c8 in ("nchar") and c10 != 10' + tdSql.query(query_sql) + tdSql.checkRows(10) + tdSql.checkEqual(self.queryLastC10(query_sql), 10) + + ## in or in or condition_A + query_sql = f'select * from {tb_name} where c1 in (2, 4) or c2 in (3, 4) or c9 != true' + tdSql.query(query_sql) + tdSql.checkRows(4) + tdSql.checkEqual(self.queryLastC10(query_sql), 11) + + ## in or in or in or in + query_sql = f'select * from {tb_name} where c1 in (2, 4) or c2 in (3, 4) or c9 in (false) or c10 in (5, 6 ,22)' + tdSql.query(query_sql) + tdSql.checkRows(6) + tdSql.checkEqual(self.queryLastC10(query_sql), 11) + + ## in or in and in or in + query_sql = f'select * from {tb_name} where c1 in (2, 4) or c2 in (3, 4) and c9 in (false) or c10 in (5, 6 ,22)' + tdSql.query(query_sql) + tdSql.checkRows(4) + tdSql.checkEqual(self.queryLastC10(query_sql), 11) + + ## condition_A or in or condition_B and in + query_sql = f'select * from {tb_name} where c1 = 2 or c2 in (2, 4) and c9 = false or c10 in (6 ,22)' + tdSql.query(query_sql) + tdSql.checkRows(3) + tdSql.checkEqual(self.queryLastC10(query_sql), 10) + + ## in and condition_A or in and in and condition_B + query_sql = f'select * from {tb_name} where c1 in (2, 3) and c2 <> 3 or c10 <= 4 and c10 in (4 ,22) and c9 != false' + tdSql.query(query_sql) + tdSql.checkRows(2) + tdSql.checkEqual(self.queryLastC10(query_sql), 4) + + ## (in and condition_A or in) and in and condition_B + query_sql = f'select * from {tb_name} where (c1 in (2, 3) and c2 <> 3 or c10 <= 4) and c10 in (4 ,22) and c9 != false' + tdSql.query(query_sql) + tdSql.checkRows(1) + tdSql.checkEqual(self.queryLastC10(query_sql), 4) + + def checkTbMultiLike(self): + ''' + Ordinary table multiLike + ''' + tb_name = self.initTb() + ## like and like + query_sql = f'select * from {tb_name} where c7 like "bi%" and c8 like ("ncha_")' + tdSql.query(query_sql) + tdSql.checkRows(9) + tdSql.checkEqual(self.queryLastC10(query_sql), 10) + + ## like or like + query_sql = f'select * from {tb_name} where c7 like "binar12345" or c8 like "nchar_"' + tdSql.query(query_sql) + tdSql.checkRows(1) + tdSql.checkEqual(self.queryLastC10(query_sql), 9) + + ## like and like or condition_A + query_sql = f'select * from {tb_name} where c7 like "binary_" and c8 like "ncha_" or c1 != 1' + tdSql.query(query_sql) + tdSql.checkRows(2) + tdSql.checkEqual(self.queryLastC10(query_sql), 8) + + ## like or like and condition_A + query_sql = f'select * from {tb_name} where c7 like ("binar_") or c8 like ("nchar_") and c10 != 8' + tdSql.query(query_sql) + tdSql.checkRows(9) + tdSql.checkEqual(self.queryLastC10(query_sql), 10) + + ## like or like or condition_A + query_sql = f'select * from {tb_name} where c7 like ("binary_") or c8 like ("nchar_") or c10 = 6' + tdSql.query(query_sql) + tdSql.checkRows(3) + tdSql.checkEqual(self.queryLastC10(query_sql), 9) + + ## like or like or like or like + query_sql = f'select * from {tb_name} where c7 like ("binary_") or c8 like ("nchar_") or c10 = 6 or c7 is Null' + tdSql.query(query_sql) + tdSql.checkRows(4) + tdSql.checkEqual(self.queryLastC10(query_sql), 11) + + ## like or like and like or like + query_sql = f'select * from {tb_name} where c7 like ("binary_") or c8 like ("ncha_") and c10 = 6 or c10 = 9' + tdSql.query(query_sql) + tdSql.checkRows(3) + tdSql.checkEqual(self.queryLastC10(query_sql), 9) + + ## condition_A or like or condition_B and like + query_sql = f'select * from {tb_name} where c1 = 2 or c7 like "binary_" or c10 = 3 and c8 like "ncha%"' + tdSql.query(query_sql) + tdSql.checkRows(3) + tdSql.checkEqual(self.queryLastC10(query_sql), 8) + + ## like and condition_A or like and like and condition_B + query_sql = f'select * from {tb_name} where c7 like "bin%" and c2 = 3 or c10 <= 4 and c7 like "binar_" and c8 like "ncha_"' + tdSql.query(query_sql) + tdSql.checkRows(4) + tdSql.checkEqual(self.queryLastC10(query_sql), 4) + + ## (like and condition_A or like) and like and condition_B + query_sql = f'select * from {tb_name} where (c7 like "bin%" and c2 = 3 or c8 like "nchar_") and c7 like "binar_" and c9 != false' + tdSql.query(query_sql) + tdSql.checkRows(2) + tdSql.checkEqual(self.queryLastC10(query_sql), 9) + + def checkTbPreCal(self): + ''' + Ordinary table precal + ''' + tb_name = self.initTb() + ## avg sum condition_A or condition_B + query_sql = f'select avg(c3), sum(c3) from {tb_name} where c10 = 5 or c8 is Null' + res = tdSql.query(query_sql, True)[0] + tdSql.checkEqual(int(res[0]), 3) + tdSql.checkEqual(int(res[1]), 6) + + ## avg sum condition_A or condition_B or condition_C + query_sql = f'select avg(c3), sum(c3) from {tb_name} where c10 = 4 or c8 is Null or c9 = false ' + res = tdSql.query(query_sql, True)[0] + tdSql.checkEqual(int(res[0]), 2) + tdSql.checkEqual(int(res[1]), 6) + + ## count avg sum condition_A or condition_B or condition_C interval + query_sql = f'select count(*), avg(c3), sum(c3) from {tb_name} where c10 = 4 or c8 is Null or c9 = false interval(16d)' + res = tdSql.query(query_sql, True) + tdSql.checkRows(2) + tdSql.checkEqual(int(res[0][1]), 1) + tdSql.checkEqual(int(res[0][2]), 4) + tdSql.checkEqual(int(res[0][3]), 4) + tdSql.checkEqual(int(res[1][1]), 2) + tdSql.checkEqual(int(res[1][2]), 1) + tdSql.checkEqual(int(res[1][3]), 2) + + ## count avg sum condition_A or condition_B or in and like or condition_C interval + query_sql = f'select count(*), sum(c3) from {tb_name} where c10 = 4 or c8 is Null or c2 in (1, 2) and c7 like "binary_" or c1 <> 1 interval(16d)' + res = tdSql.query(query_sql, True) + tdSql.checkRows(2) + tdSql.checkEqual(int(res[0][1]), 2) + tdSql.checkEqual(int(res[0][2]), 5) + tdSql.checkEqual(int(res[1][1]), 2) + tdSql.checkEqual(int(res[1][2]), 2) + + def queryMultiTb(self): + ''' + test "or" in multi ordinary table + ''' + tdCom.cleanTb() + tb_name = self.initTb() + ## select from (condition_A or condition_B) + query_sql = f'select c10 from (select * from {tb_name} where c1 >1 or c2 >=3)' + tdSql.query(query_sql) + tdSql.checkRows(3) + tdSql.checkEqual(self.queryLastC10(query_sql, True), 11) + + + + # tb_name1 = tdCom.getLongName(8, "letters") + # tb_name2 = tdCom.getLongName(8, "letters") + # tb_name3 = tdCom.getLongName(8, "letters") + # tdSql.execute( + # f"CREATE TABLE {tb_name1} (ts timestamp, c1 tinyint, c2 smallint, c3 int)") + # tdSql.execute( + # f"CREATE TABLE {tb_name2} (ts timestamp, c1 tinyint, c2 smallint, c3 int)") + # tdSql.execute( + # f"CREATE TABLE {tb_name3} (ts timestamp, c1 tinyint, c2 smallint, c3 int)") + # insert_sql_list = [f'insert into {tb_name1} values ("2021-01-01 12:00:00", 1, 5, 1)', + # f'insert into {tb_name1} values ("2021-01-03 12:00:00", 2, 4, 1)', + # f'insert into {tb_name1} values ("2021-01-05 12:00:00", 3, 2, 1)', + # f'insert into {tb_name2} values ("2021-01-01 12:00:00", 4, 2, 1)', + # f'insert into {tb_name2} values ("2021-01-02 12:00:00", 5, 1, 1)', + # f'insert into {tb_name2} values ("2021-01-04 12:00:00", 1, 2, 1)', + # f'insert into {tb_name3} values ("2021-01-02 12:00:00", 4, 2, 1)', + # f'insert into {tb_name3} values ("2021-01-06 12:00:00", 5, 1, 1)', + # f'insert into {tb_name3} values ("2021-01-07 12:00:00", 1, 2, 1)', + # ] + # for sql in insert_sql_list: + # tdSql.execute(sql) + # tdSql.query( + # f'select * from {tb_name1} t1, {tb_name2}, {tb_name3} t3 t2 where (t1.ts=t2.ts or t2.ts=t3.ts)') + # tdSql.checkRows(4) + + + def run(self): + tdSql.prepare() + self.checkTbColTypeOperator() + self.checkTbMultiExpression() + self.checkTbMultiIn() + self.checkTbMultiLike() + self.checkTbPreCal() + self.queryMultiTb() + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) From 617c59f4a8db05ed803b28f7329a0ad493afcab6 Mon Sep 17 00:00:00 2001 From: jiajingbin Date: Wed, 18 Aug 2021 21:27:21 +0800 Subject: [PATCH 11/40] save --- tests/pytest/query/queryDiffColsOr.py | 30 +++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/tests/pytest/query/queryDiffColsOr.py b/tests/pytest/query/queryDiffColsOr.py index 68c8f14869..d07867ee27 100644 --- a/tests/pytest/query/queryDiffColsOr.py +++ b/tests/pytest/query/queryDiffColsOr.py @@ -389,7 +389,37 @@ class TDTestCase: tdSql.checkRows(3) tdSql.checkEqual(self.queryLastC10(query_sql, True), 11) + ## select from (condition_A or condition_B) where condition_A or condition_B + query_sql = f'select c10 from (select * from {tb_name} where c1 >1 or c2 >=3) where c1 =2 or c4 = 2' + tdSql.query(query_sql) + tdSql.checkRows(2) + tdSql.checkEqual(self.queryLastC10(query_sql, True), 3) + ## select from (condition_A or condition_B and like and in) where condition_A or condition_B or like and in + query_sql = f'select c10 from (select * from {tb_name} where c1 >1 or c2 = 2 and c7 like "binar_" and c4 in (3, 5)) where c1 != 2 or c3 = 1 or c8 like "ncha_" and c9 in (true)' + tdSql.query(query_sql) + tdSql.checkRows(7) + tdSql.checkEqual(self.queryLastC10(query_sql, True), 10) + + ## select count avg sum from (condition_A or condition_B and like and in) where condition_A or condition_B or like and in interval + query_sql = f'select count(*), avg(c6), sum(c3) from (select * from {tb_name} where c1 >1 or c2 = 2 and c7 like "binar_" and c4 in (3, 5)) where c1 != 2 or c3 = 1 or c8 like "ncha_" and c9 in (true) interval(8d)' + res = tdSql.query(query_sql, True) + tdSql.checkRows(3) + tdSql.checkEqual(int(res[0][1]), 3) + tdSql.checkEqual(int(res[0][2]), 1) + tdSql.checkEqual(int(res[0][3]), 10) + tdSql.checkEqual(int(res[1][1]), 3) + tdSql.checkEqual(int(res[1][2]), 3) + tdSql.checkEqual(int(res[1][3]), 3) + tdSql.checkEqual(int(res[2][1]), 1) + tdSql.checkEqual(int(res[2][2]), 1) + tdSql.checkEqual(int(res[2][3]), 1) + + ## cname + query_sql = f'select c10 from (select * from {tb_name} where c1 >1 or c2 = 2 and c7 like "binar_" and c4 in (3, 5)) a where a.c1 != 2 or a.c3 = 1 or a.c8 like "ncha_" and a.c9 in (true)' + tdSql.query(query_sql) + tdSql.checkRows(7) + tdSql.checkEqual(self.queryLastC10(query_sql, True), 10) # tb_name1 = tdCom.getLongName(8, "letters") # tb_name2 = tdCom.getLongName(8, "letters") From ad0d0ab93d60741509fc27480e148ec16f2d289a Mon Sep 17 00:00:00 2001 From: jiajingbin Date: Thu, 19 Aug 2021 15:36:10 +0800 Subject: [PATCH 12/40] save --- tests/pytest/query/queryDiffColsOr.py | 167 +++++++++++++++++++------- 1 file changed, 123 insertions(+), 44 deletions(-) diff --git a/tests/pytest/query/queryDiffColsOr.py b/tests/pytest/query/queryDiffColsOr.py index d07867ee27..6f32a15727 100644 --- a/tests/pytest/query/queryDiffColsOr.py +++ b/tests/pytest/query/queryDiffColsOr.py @@ -46,6 +46,16 @@ class TDTestCase: self.insertData(tb_name) return tb_name + def initStb(self): + tdCom.cleanTb() + tb_name = tdCom.getLongName(8, "letters") + tdSql.execute( + f"CREATE TABLE {tb_name} (ts timestamp, c1 tinyint, c2 smallint, c3 int, c4 bigint, c5 float, c6 double, c7 binary(100), c8 nchar(200), c9 bool, c10 int) tags (t1 tinyint, t2 smallint, t3 int, t4 bigint, t5 float, t6 double, t7 binary(100), t8 nchar(200), t9 bool, t10 int)") + tdSql.execute( + f'CREATE TABLE {tb_name}_sub using {tb_name} tags (1, 1, 1, 3, 1.1, 1.1, "binary", "nchar", true, 1)') + self.insertData(f'{tb_name}_sub') + return tb_name + def queryLastC10(self, query_sql, multi=False): if multi: res = tdSql.query(query_sql.replace('c10', 'last(*)'), True) @@ -120,18 +130,7 @@ class TDTestCase: tdSql.checkRows(10) tdSql.checkEqual(self.queryLastC10(query_sql), 11) - def checkTbColTypeOperator(self): - ''' - Ordinary table full column type and operator - ''' - tb_name = self.initTb() - self.queryFullColType(tb_name) - - def checkTbMultiExpression(self): - ''' - Ordinary table multiExpression - ''' - tb_name = self.initTb() + def queryMultiExpression(self, tb_name): ## condition_A and condition_B or condition_C (> < >=) query_sql = f'select * from {tb_name} where c1 > 2 and c2 < 4 or c3 >= 4' tdSql.query(query_sql) @@ -209,12 +208,8 @@ class TDTestCase: tdSql.query(query_sql) tdSql.checkRows(2) tdSql.checkEqual(self.queryLastC10(query_sql), 5) - - def checkTbMultiIn(self): - ''' - Ordinary table multiIn - ''' - tb_name = self.initTb() + + def queryMultiIn(self, tb_name): ## in and in query_sql = f'select * from {tb_name} where c7 in ("binary") and c8 in ("nchar")' tdSql.query(query_sql) @@ -275,11 +270,7 @@ class TDTestCase: tdSql.checkRows(1) tdSql.checkEqual(self.queryLastC10(query_sql), 4) - def checkTbMultiLike(self): - ''' - Ordinary table multiLike - ''' - tb_name = self.initTb() + def queryMultiLike(self, tb_name): ## like and like query_sql = f'select * from {tb_name} where c7 like "bi%" and c8 like ("ncha_")' tdSql.query(query_sql) @@ -340,11 +331,7 @@ class TDTestCase: tdSql.checkRows(2) tdSql.checkEqual(self.queryLastC10(query_sql), 9) - def checkTbPreCal(self): - ''' - Ordinary table precal - ''' - tb_name = self.initTb() + def queryPreCal(self, tb_name): ## avg sum condition_A or condition_B query_sql = f'select avg(c3), sum(c3) from {tb_name} where c10 = 5 or c8 is Null' res = tdSql.query(query_sql, True)[0] @@ -377,29 +364,24 @@ class TDTestCase: tdSql.checkEqual(int(res[1][1]), 2) tdSql.checkEqual(int(res[1][2]), 2) - def queryMultiTb(self): - ''' - test "or" in multi ordinary table - ''' - tdCom.cleanTb() - tb_name = self.initTb() + def queryMultiTb(self, tb_name): ## select from (condition_A or condition_B) query_sql = f'select c10 from (select * from {tb_name} where c1 >1 or c2 >=3)' - tdSql.query(query_sql) + res = tdSql.query(query_sql, True) tdSql.checkRows(3) - tdSql.checkEqual(self.queryLastC10(query_sql, True), 11) + tdSql.checkEqual(int(res[2][0]), 11) ## select from (condition_A or condition_B) where condition_A or condition_B query_sql = f'select c10 from (select * from {tb_name} where c1 >1 or c2 >=3) where c1 =2 or c4 = 2' - tdSql.query(query_sql) + res = tdSql.query(query_sql, True) tdSql.checkRows(2) - tdSql.checkEqual(self.queryLastC10(query_sql, True), 3) + tdSql.checkEqual(int(res[1][0]), 3) ## select from (condition_A or condition_B and like and in) where condition_A or condition_B or like and in query_sql = f'select c10 from (select * from {tb_name} where c1 >1 or c2 = 2 and c7 like "binar_" and c4 in (3, 5)) where c1 != 2 or c3 = 1 or c8 like "ncha_" and c9 in (true)' - tdSql.query(query_sql) + res = tdSql.query(query_sql, True) tdSql.checkRows(7) - tdSql.checkEqual(self.queryLastC10(query_sql, True), 10) + tdSql.checkEqual(int(res[6][0]), 10) ## select count avg sum from (condition_A or condition_B and like and in) where condition_A or condition_B or like and in interval query_sql = f'select count(*), avg(c6), sum(c3) from (select * from {tb_name} where c1 >1 or c2 = 2 and c7 like "binar_" and c4 in (3, 5)) where c1 != 2 or c3 = 1 or c8 like "ncha_" and c9 in (true) interval(8d)' @@ -415,12 +397,103 @@ class TDTestCase: tdSql.checkEqual(int(res[2][2]), 1) tdSql.checkEqual(int(res[2][3]), 1) - ## cname + # cname query_sql = f'select c10 from (select * from {tb_name} where c1 >1 or c2 = 2 and c7 like "binar_" and c4 in (3, 5)) a where a.c1 != 2 or a.c3 = 1 or a.c8 like "ncha_" and a.c9 in (true)' - tdSql.query(query_sql) + res = tdSql.query(query_sql, True) tdSql.checkRows(7) - tdSql.checkEqual(self.queryLastC10(query_sql, True), 10) + tdSql.checkEqual(int(res[6][0]), 10) + # ## multi cname + query_sql = f'select b.c10 from (select * from {tb_name} where c9 = true or c2 = 2) a, (select * from {tb_name} where c7 like "binar_" or c4 in (3, 5)) b where a.ts = b.ts' + res = tdSql.query(query_sql, True) + tdSql.checkRows(10) + tdSql.checkEqual(int(res[9][0]), 10) + + def checkTbColTypeOperator(self): + ''' + Ordinary table full column type and operator + ''' + tb_name = self.initTb() + self.queryFullColType(tb_name) + + def checkStbColTypeOperator(self): + ''' + Super table full column type and operator + ''' + tb_name = self.initStb() + self.queryFullColType(tb_name) + + def checkTbMultiExpression(self): + ''' + Ordinary table multiExpression + ''' + tb_name = self.initTb() + self.queryMultiExpression(tb_name) + + def checkStbMultiExpression(self): + ''' + Super table multiExpression + ''' + tb_name = self.initStb() + self.queryMultiExpression(tb_name) + + def checkTbMultiIn(self): + ''' + Ordinary table multiIn + ''' + tb_name = self.initTb() + self.queryMultiIn(tb_name) + + def checkStbMultiIn(self): + ''' + Super table multiIn + ''' + tb_name = self.initStb() + self.queryMultiIn(tb_name) + + def checkTbMultiLike(self): + ''' + Ordinary table multiLike + ''' + tb_name = self.initTb() + self.queryMultiLike(tb_name) + + def checkStbMultiLike(self): + ''' + Super table multiLike + ''' + tb_name = self.initStb() + self.queryMultiLike(tb_name) + + def checkTbPreCal(self): + ''' + Ordinary table precal + ''' + tb_name = self.initTb() + self.queryPreCal(tb_name) + + def checkStbPreCal(self): + ''' + Super table precal + ''' + tb_name = self.initStb() + self.queryPreCal(tb_name) + + def checkMultiTb(self): + ''' + test "or" in multi ordinary table + ''' + tb_name = self.initTb() + self.queryMultiTb(tb_name) + + def checkMultiStb(self): + ''' + test "or" in multi super table + ''' + tb_name = self.initStb() + self.queryMultiTb(tb_name) + + # tb_name1 = tdCom.getLongName(8, "letters") # tb_name2 = tdCom.getLongName(8, "letters") # tb_name3 = tdCom.getLongName(8, "letters") @@ -450,11 +523,17 @@ class TDTestCase: def run(self): tdSql.prepare() self.checkTbColTypeOperator() + self.checkStbColTypeOperator() self.checkTbMultiExpression() + self.checkStbMultiExpression() self.checkTbMultiIn() + self.checkStbMultiIn() self.checkTbMultiLike() + self.checkStbMultiLike() self.checkTbPreCal() - self.queryMultiTb() + self.checkStbPreCal() + self.checkMultiTb() + self.checkMultiStb() def stop(self): From 3ef88ec1969bec595a757784e19504072f154e94 Mon Sep 17 00:00:00 2001 From: jiajingbin Date: Fri, 20 Aug 2021 09:58:06 +0800 Subject: [PATCH 13/40] save --- tests/pytest/query/queryDiffColsOr.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/pytest/query/queryDiffColsOr.py b/tests/pytest/query/queryDiffColsOr.py index 6f32a15727..feeab84a7e 100644 --- a/tests/pytest/query/queryDiffColsOr.py +++ b/tests/pytest/query/queryDiffColsOr.py @@ -397,13 +397,13 @@ class TDTestCase: tdSql.checkEqual(int(res[2][2]), 1) tdSql.checkEqual(int(res[2][3]), 1) - # cname + ## cname query_sql = f'select c10 from (select * from {tb_name} where c1 >1 or c2 = 2 and c7 like "binar_" and c4 in (3, 5)) a where a.c1 != 2 or a.c3 = 1 or a.c8 like "ncha_" and a.c9 in (true)' res = tdSql.query(query_sql, True) tdSql.checkRows(7) tdSql.checkEqual(int(res[6][0]), 10) - # ## multi cname + ## multi cname query_sql = f'select b.c10 from (select * from {tb_name} where c9 = true or c2 = 2) a, (select * from {tb_name} where c7 like "binar_" or c4 in (3, 5)) b where a.ts = b.ts' res = tdSql.query(query_sql, True) tdSql.checkRows(10) From 1517dd9de64952e3d2cc2be7b846765a1a8c7b29 Mon Sep 17 00:00:00 2001 From: jiajingbin Date: Fri, 20 Aug 2021 19:20:49 +0800 Subject: [PATCH 14/40] save --- .../tools/schemalessInsertPerformance.py | 228 ++++++++++++++++++ 1 file changed, 228 insertions(+) create mode 100644 tests/pytest/tools/schemalessInsertPerformance.py diff --git a/tests/pytest/tools/schemalessInsertPerformance.py b/tests/pytest/tools/schemalessInsertPerformance.py new file mode 100644 index 0000000000..3d44f023b4 --- /dev/null +++ b/tests/pytest/tools/schemalessInsertPerformance.py @@ -0,0 +1,228 @@ +################################################################### +# Copyright (c) 2021 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import traceback +import random +import string +from taos.error import LinesError +import datetime +import time +from copy import deepcopy +import numpy as np +from util.log import * +from util.cases import * +from util.sql import * +from util.common import tdCom +import threading + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + self._conn = conn + + def genRandomTs(self): + year = random.randint(2000, 2021) + month = random.randint(10, 12) + day = random.randint(10, 29) + hour = random.randint(10, 24) + minute = random.randint(10, 59) + second = random.randint(10, 59) + m_second = random.randint(101, 199) + date_time = f'{year}-{month}-{day} {hour}:{minute}:{second}' + print(date_time) + timeArray = time.strptime(date_time, "%Y-%m-%d %H:%M:%S") + ts = int(time.mktime(timeArray)) + print("------", ts) + # timestamp = time.mktime(datetime.datetime.strptime(date_time, "%Y-%m-%d %H:%M:%S.%f").timetuple()) + return f'{ts}s' + + def genMultiColStr(self, int_count=4, double_count=0, binary_count=0): + """ + genType must be tag/col + """ + col_str = "" + if double_count == 0 and binary_count == 0: + for i in range(0, int_count): + if i < (int_count-1): + col_str += f'c{i}={random.randint(0, 255)}i32,' + else: + col_str += f'c{i}={random.randint(0, 255)}i32 ' + elif double_count > 0 and binary_count == 0: + for i in range(0, int_count): + col_str += f'c{i}={random.randint(0, 255)}i32,' + for i in range(0, double_count): + if i < (double_count-1): + col_str += f'c{i+int_count}={random.randint(1, 255)}.{i}f64,' + else: + col_str += f'c{i+int_count}={random.randint(1, 255)}.{i}f64 ' + elif double_count == 0 and binary_count > 0: + for i in range(0, int_count): + col_str += f'c{i}={random.randint(0, 255)}i32,' + for i in range(0, binary_count): + if i < (binary_count-1): + col_str += f'c{i+int_count}=\"{tdCom.getLongName(5, "letters")}\",' + else: + col_str += f'c{i+int_count}=\"{tdCom.getLongName(5, "letters")}\" ' + elif double_count > 0 and binary_count > 0: + for i in range(0, int_count): + col_str += f'c{i}={random.randint(0, 255)}i32,' + for i in range(0, double_count): + col_str += f'c{i+int_count}={random.randint(1, 255)}.{i}f64,' + for i in range(0, binary_count): + if i < (binary_count-1): + col_str += f'c{i+int_count+double_count}=\"{tdCom.getLongName(5, "letters")}\",' + else: + col_str += f'c{i+int_count+double_count}=\"{tdCom.getLongName(5, "letters")}\" ' + return col_str + + def genLongSql(self, int_count=4, double_count=0, binary_count=0, init=False): + if init: + tag_str = f'id="init",t0={random.randint(0, 65535)}i32,t1=\"{tdCom.getLongName(10, "letters")}\"' + else: + tag_str = f'id=sub_{tdCom.getLongName(5, "letters")}_{tdCom.getLongName(5, "letters")},t0={random.randint(0, 65535)}i32,t1=\"{tdCom.getLongName(10, "letters")}\"' + col_str = self.genMultiColStr(int_count, double_count, binary_count) + long_sql = 'stb' + ',' + tag_str + ' ' + col_str + '0' + return long_sql + + def getPerfSql(self, count=4, init=False): + if count == 4: + input_sql = self.genLongSql(init=init) + elif count == 1000: + input_sql = self.genLongSql(400, 400, 200, init=init) + elif count == 4000: + input_sql = self.genLongSql(1900, 1900, 200, init=init) + return input_sql + + def tableGenerator(self, count=4, table_count=1000): + for i in range(table_count): + yield self.getPerfSql(count) + + def genTableList(self, count=4, table_count=10000): + table_list = list() + for i in range(1, table_count+1): + table_list.append(self.getPerfSql(count)) + return table_list + + def splitTableList(self, count=4, thread_count=10, table_count=10000): + per_list_len = int(table_count/thread_count) + table_list = self.genTableList(count=count) + # ts = int(time.time()) + list_of_group = zip(*(iter(table_list),) *per_list_len) + end_list = [list(i) for i in list_of_group] # i is a tuple + count = len(table_list) % per_list_len + end_list.append(table_list[-count:]) if count !=0 else end_list + return table_list, end_list + + def replaceLastStr(self, str, new): + list_ori = list(str) + list_ori[-1] = new + return ''.join(list_ori) + + def genDataList(self, table_list, row_count=10): + data_list = list() + ts = int(time.time()) + for table_str in table_list: + for i in range(1, row_count+1): + ts -= 1 + table_str_new = self.replaceLastStr(table_str, str(ts)) + data_list.append(table_str_new) + print(data_list) + return data_list + + def perfTableInsert(self): + table_generator = self.tableGenerator() + for input_sql in table_generator: + self._conn.insert_lines([input_sql]) + for i in range(10): + self._conn.insert_lines([input_sql]) + + def perfDataInsert(self, input_sql): + for i in range(10000): + self._conn.insert_lines([input_sql]) + + def genTableThread(self, thread_count=10): + threads = list() + for i in range(thread_count): + t = threading.Thread(target=self.perfTableInsert) + threads.append(t) + return threads + + def genMultiThread(self, input_sql, thread_count=10): + threads = list() + for i in range(thread_count): + t = threading.Thread(target=self.perfDataInsert,args=(input_sql,)) + threads.append(t) + return threads + + def multiThreadRun(self, threads): + for t in threads: + t.start() + for t in threads: + t.join() + + def createStb(self, count=4): + input_sql = self.getPerfSql(count=count, init=True) + print("stb-----", input_sql) + self._conn.insert_lines([input_sql]) + + # def createTb(self, count=4): + # input_sql = self.getPerfSql(count=count) + # for i in range(10000): + # self._conn.insert_lines([input_sql]) + + # def createTb1(self, count=4): + # start_time = time.time() + # self.multiThreadRun(self.genMultiThread(input_sql)) + # end_time = time.time() + # return end_time - start_time + + def calInsertTableTime(self): + start_time = time.time() + self.createStb() + self.multiThreadRun(self.genTableThread()) + end_time = time.time() + return end_time - start_time + + def calRunTime(self, input_sql): + start_time = time.time() + self.multiThreadRun(self.genMultiThread(input_sql)) + end_time = time.time() + return end_time - start_time + + def schemalessInsertPerfTest(self, count=4): + input_sql = self.getPerfSql(count) + self.calRunTime(input_sql) + + def test(self): + sql1 = 'stb,id="init",t0=14865i32,t1="tvnqbjuqck" c0=37i32,c1=217i32,c2=3i32,c3=88i32 1626006833640ms' + sql2 = 'stb,id="init",t0=14865i32,t1="tvnqbjuqck" c0=38i32,c1=217i32,c2=3i32,c3=88i32 1626006833641ms' + self._conn.insert_lines([sql1]) + self._conn.insert_lines([sql2]) + def run(self): + print("running {}".format(__file__)) + tdSql.prepare() + # print(self.genRandomTs()) + # self.calInsertTableTime() + # self.test() + table_list = self.splitTableList()[0] + data_list = self.genDataList(table_list) + print(len(data_list)) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) From a3b56039ba3afd98287eee4d91625d2e09e6d70e Mon Sep 17 00:00:00 2001 From: jiajingbin Date: Mon, 23 Aug 2021 17:51:09 +0800 Subject: [PATCH 15/40] save for tag TD-6293 --- .../tools/schemalessInsertPerformance.py | 124 ++++++++++++++---- 1 file changed, 97 insertions(+), 27 deletions(-) diff --git a/tests/pytest/tools/schemalessInsertPerformance.py b/tests/pytest/tools/schemalessInsertPerformance.py index 3d44f023b4..0dcbc5ffe2 100644 --- a/tests/pytest/tools/schemalessInsertPerformance.py +++ b/tests/pytest/tools/schemalessInsertPerformance.py @@ -91,7 +91,7 @@ class TDTestCase: if init: tag_str = f'id="init",t0={random.randint(0, 65535)}i32,t1=\"{tdCom.getLongName(10, "letters")}\"' else: - tag_str = f'id=sub_{tdCom.getLongName(5, "letters")}_{tdCom.getLongName(5, "letters")},t0={random.randint(0, 65535)}i32,t1=\"{tdCom.getLongName(10, "letters")}\"' + tag_str = f'id="sub_{tdCom.getLongName(5, "letters")}_{tdCom.getLongName(5, "letters")}",t0={random.randint(0, 65535)}i32,t1=\"{tdCom.getLongName(10, "letters")}\"' col_str = self.genMultiColStr(int_count, double_count, binary_count) long_sql = 'stb' + ',' + tag_str + ' ' + col_str + '0' return long_sql @@ -109,13 +109,19 @@ class TDTestCase: for i in range(table_count): yield self.getPerfSql(count) + + + + + + def genTableList(self, count=4, table_count=10000): table_list = list() for i in range(1, table_count+1): table_list.append(self.getPerfSql(count)) return table_list - def splitTableList(self, count=4, thread_count=10, table_count=10000): + def splitTableList(self, count=4, thread_count=10, table_count=1000): per_list_len = int(table_count/thread_count) table_list = self.genTableList(count=count) # ts = int(time.time()) @@ -125,6 +131,22 @@ class TDTestCase: end_list.append(table_list[-count:]) if count !=0 else end_list return table_list, end_list + def rowsGenerator(self, end_list): + ts = int(time.time()) + input_sql_list = list() + for elm_list in end_list: + for elm in elm_list: + for i in range(1, 10000): + ts -= 1 + elm_new = self.replaceLastStr(elm, str(ts)) + 's' + input_sql_list.append(elm_new) + yield input_sql_list + + # def insertRows(self, count=4, thread_count=10): + # table_list = self.splitTableList(count=count, thread_count=thread_count)[0] + # for + + def replaceLastStr(self, str, new): list_ori = list(str) list_ori[-1] = new @@ -136,21 +158,39 @@ class TDTestCase: for table_str in table_list: for i in range(1, row_count+1): ts -= 1 - table_str_new = self.replaceLastStr(table_str, str(ts)) + table_str_new = self.replaceLastStr(table_str, f'{str(ts)}s') data_list.append(table_str_new) print(data_list) return data_list + + def insertRows(self, count=4, table_count=1000): + table_generator = self.tableGenerator(count=count, table_count=table_count) + for table_name in table_generator: + pass + def perfTableInsert(self): table_generator = self.tableGenerator() for input_sql in table_generator: self._conn.insert_lines([input_sql]) - for i in range(10): - self._conn.insert_lines([input_sql]) + # for i in range(10): + # self._conn.insert_lines([input_sql]) - def perfDataInsert(self, input_sql): - for i in range(10000): + def perfDataInsert(self, count=4): + table_generator = self.tableGenerator(count=count) + ts = int(time.time()) + for input_sql in table_generator: + print("input_sql-----------", input_sql) self._conn.insert_lines([input_sql]) + for i in range(100000): + ts -= 1 + input_sql_new = self.replaceLastStr(input_sql, str(ts)) + 's' + print("input_sql_new---------", input_sql_new) + self._conn.insert_lines([input_sql_new]) + + def batchInsertTable(self, batch_list): + for insert_list in batch_list: + self._conn.insert_lines(insert_list) def genTableThread(self, thread_count=10): threads = list() @@ -159,10 +199,10 @@ class TDTestCase: threads.append(t) return threads - def genMultiThread(self, input_sql, thread_count=10): + def genMultiThread(self, count, thread_count=10): threads = list() for i in range(thread_count): - t = threading.Thread(target=self.perfDataInsert,args=(input_sql,)) + t = threading.Thread(target=self.perfDataInsert,args=(count,)) threads.append(t) return threads @@ -174,9 +214,24 @@ class TDTestCase: def createStb(self, count=4): input_sql = self.getPerfSql(count=count, init=True) - print("stb-----", input_sql) self._conn.insert_lines([input_sql]) + def threadInsertTable(self, end_list, thread_count=10): + threads = list() + for i in range(thread_count): + t = threading.Thread(target=self.batchInsertTable, args=(end_list,)) + threads.append(t) + return threads + + + def finalRun(self): + self.createStb() + table_list, end_list = self.splitTableList() + batchInsertTableThread = self.threadInsertTable(end_list=end_list) + print(end_list) + self.multiThreadRun(batchInsertTableThread) + # print(end_list) + # def createTb(self, count=4): # input_sql = self.getPerfSql(count=count) # for i in range(10000): @@ -188,37 +243,52 @@ class TDTestCase: # end_time = time.time() # return end_time - start_time - def calInsertTableTime(self): + # def calInsertTableTime(self): + # start_time = time.time() + # self.createStb() + # self.multiThreadRun(self.genMultiThread()) + # end_time = time.time() + # return end_time - start_time + + def calRunTime(self, count=4): start_time = time.time() self.createStb() - self.multiThreadRun(self.genTableThread()) + self.multiThreadRun(self.genMultiThread(count=count)) end_time = time.time() return end_time - start_time - def calRunTime(self, input_sql): + def calRunTime1(self, count=4): start_time = time.time() - self.multiThreadRun(self.genMultiThread(input_sql)) - end_time = time.time() - return end_time - start_time + self.createStb() + self.multiThreadRun(self.perfTableInsert()) + # self.perfTableInsert() - def schemalessInsertPerfTest(self, count=4): - input_sql = self.getPerfSql(count) - self.calRunTime(input_sql) + # def schemalessInsertPerfTest(self, count=4): + # input_sql = self.getPerfSql(count) + # self.calRunTime(input_sql) + + # def test(self): + # sql1 = 'stb,id="init",t0=14865i32,t1="tvnqbjuqck" c0=37i32,c1=217i32,c2=3i32,c3=88i32 1626006833640ms' + # sql2 = 'stb,id="init",t0=14865i32,t1="tvnqbjuqck" c0=38i32,c1=217i32,c2=3i32,c3=88i32 1626006833641ms' + # self._conn.insert_lines([sql1]) + # self._conn.insert_lines([sql2]) - def test(self): - sql1 = 'stb,id="init",t0=14865i32,t1="tvnqbjuqck" c0=37i32,c1=217i32,c2=3i32,c3=88i32 1626006833640ms' - sql2 = 'stb,id="init",t0=14865i32,t1="tvnqbjuqck" c0=38i32,c1=217i32,c2=3i32,c3=88i32 1626006833641ms' - self._conn.insert_lines([sql1]) - self._conn.insert_lines([sql2]) def run(self): print("running {}".format(__file__)) tdSql.prepare() + self.finalRun() + # print(self.calRunTime1(count=4)) + # print(self.calRunTime(count=4)) # print(self.genRandomTs()) # self.calInsertTableTime() # self.test() - table_list = self.splitTableList()[0] - data_list = self.genDataList(table_list) - print(len(data_list)) + # table_list = self.splitTableList()[0] + # data_list = self.genDataList(table_list) + # print(len(data_list)) + # end_list = [['stb,id="sub_vzvfx_dbuxp",t0=9961i32,t1="zjjfayhfep" c0=83i32,c1=169i32,c2=177i32,c3=4i32 0','stb,id="sub_vzvfx_dbuxp",t0=9961i32,t1="zjjfayhfep" c0=83i32,c1=169i32,c2=177i32,c3=4i32 0'], ['stb,id="sub_vzvfx_dbuxp",t0=9961i32,t1="zjjfayhfep" c0=83i32,c1=169i32,c2=177i32,c3=4i32 0','stb,id="sub_vzvfx_dbuxp",t0=9961i32,t1="zjjfayhfep" c0=83i32,c1=169i32,c2=177i32,c3=4i32 0']] + # rowsGenerator = self.rowsGenerator(end_list) + # for i in rowsGenerator: + # print(i) def stop(self): tdSql.close() From 9126417086cb6d7f28d6952cc7f613869b2b347c Mon Sep 17 00:00:00 2001 From: jiajingbin Date: Tue, 24 Aug 2021 09:47:06 +0800 Subject: [PATCH 16/40] add common.py --- tests/pytest/util/common.py | 53 +++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 tests/pytest/util/common.py diff --git a/tests/pytest/util/common.py b/tests/pytest/util/common.py new file mode 100644 index 0000000000..35abc4802f --- /dev/null +++ b/tests/pytest/util/common.py @@ -0,0 +1,53 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import random +import string +from util.sql import tdSql + +class TDCom: + def init(self, conn, logSql): + tdSql.init(conn.cursor(), logSql) + + def cleanTb(self): + query_sql = "show stables" + res_row_list = tdSql.query(query_sql, True) + stb_list = map(lambda x: x[0], res_row_list) + for stb in stb_list: + tdSql.execute(f'drop table if exists {stb}') + + query_sql = "show tables" + res_row_list = tdSql.query(query_sql, True) + tb_list = map(lambda x: x[0], res_row_list) + for tb in tb_list: + tdSql.execute(f'drop table if exists {tb}') + + def getLongName(self, len, mode = "mixed"): + """ + generate long name + mode could be numbers/letters/letters_mixed/mixed + """ + if mode == "numbers": + chars = ''.join(random.choice(string.digits) for i in range(len)) + elif mode == "letters": + chars = ''.join(random.choice(string.ascii_letters.lower()) for i in range(len)) + elif mode == "letters_mixed": + chars = ''.join(random.choice(string.ascii_letters.upper() + string.ascii_letters.lower()) for i in range(len)) + else: + chars = ''.join(random.choice(string.ascii_letters.lower() + string.digits) for i in range(len)) + return chars + + def close(self): + self.cursor.close() + +tdCom = TDCom() From 1380e6c4ba10aacac99add769ea4575cac6f2af9 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Tue, 24 Aug 2021 17:21:35 +0800 Subject: [PATCH 17/40] add log information to printlines --- tests/pytest/tools/schemalessInsertPerformance.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/pytest/tools/schemalessInsertPerformance.py b/tests/pytest/tools/schemalessInsertPerformance.py index 0dcbc5ffe2..68802310a2 100644 --- a/tests/pytest/tools/schemalessInsertPerformance.py +++ b/tests/pytest/tools/schemalessInsertPerformance.py @@ -190,7 +190,10 @@ class TDTestCase: def batchInsertTable(self, batch_list): for insert_list in batch_list: + print(threading.current_thread().name, "length=", len(insert_list)) + print(threading.current_thread().name, 'firstline', insert_list[0], 'lastline:', insert_list[-1]) self._conn.insert_lines(insert_list) + print(threading.current_thread().name, 'end') def genTableThread(self, thread_count=10): threads = list() @@ -228,7 +231,6 @@ class TDTestCase: self.createStb() table_list, end_list = self.splitTableList() batchInsertTableThread = self.threadInsertTable(end_list=end_list) - print(end_list) self.multiThreadRun(batchInsertTableThread) # print(end_list) From b6896fc5f9e480f3d64bd970827d8cefe23e372e Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Tue, 24 Aug 2021 17:43:33 +0800 Subject: [PATCH 18/40] before insert retry, refer to needRetryInsert --- src/client/src/tscParseLineProtocol.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/src/tscParseLineProtocol.c b/src/client/src/tscParseLineProtocol.c index bde68c306a..9fb6cb4717 100644 --- a/src/client/src/tscParseLineProtocol.c +++ b/src/client/src/tscParseLineProtocol.c @@ -1127,7 +1127,7 @@ static int32_t applyDataPoints(TAOS* taos, TAOS_SML_DATA_POINT* points, int32_t tscDebug("SML:0x%"PRIx64" apply child table points. child table: %s", info->id, point->childTableName); code = applyChildTableFields(taos, sTableSchema, point->childTableName, cTablePoints, info); if (code != 0) { - tscError("Apply child table fields failed. child table %s, error %s", point->childTableName, tstrerror(code)); + tscError("SML:0x%"PRIx64" Apply child table fields failed. child table %s, error %s", info->id, point->childTableName, tstrerror(code)); goto cleanup; } From 88109bbdbc0ca2364e951050e290d058b1f7330f Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Tue, 24 Aug 2021 21:51:50 +0800 Subject: [PATCH 19/40] add retry for invalid table id and invalid group id --- src/client/src/tscParseLineProtocol.c | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/src/client/src/tscParseLineProtocol.c b/src/client/src/tscParseLineProtocol.c index 9fb6cb4717..1bce10f95e 100644 --- a/src/client/src/tscParseLineProtocol.c +++ b/src/client/src/tscParseLineProtocol.c @@ -1084,10 +1084,28 @@ static int32_t applyChildTableFields(TAOS* taos, SSmlSTableSchema* sTableSchema, taosArrayPush(rowsBind, &colBinds); } - code = insertChildTableBatch(taos, cTableName, sTableSchema->fields, rowsBind, info); - if (code != 0) { - tscError("SML:0x%"PRIx64" insert into child table %s failed. error %s", info->id, cTableName, tstrerror(code)); - } + int32_t retry = 0; + bool retryAgain = false; + do { + code = insertChildTableBatch(taos, cTableName, sTableSchema->fields, rowsBind, info); + if (code != 0) { + tscError("SML:0x%" PRIx64 " insert into child table %s failed. error %s, retry %d", info->id, cTableName, tstrerror(code), retry); + if (code == TSDB_CODE_TDB_INVALID_TABLE_ID || code == TSDB_CODE_VND_INVALID_VGROUP_ID) { + TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); + int32_t code2 = taos_errno(res2); + if (code2 != TSDB_CODE_SUCCESS) { + tscError("SML:0x%" PRIx64 " apply schema action. reset query cache. error: %s", info->id, taos_errstr(res2)); + } + taos_free_result(res2); + retryAgain = (++retry < TSDB_MAX_REPLICA) ? true : false; + if (retryAgain) { + taosMsleep(100 * (2 << (retry))); + } else { + tscError("SML:0x%" PRIx64 " insert into child table %s reached max retry", info->id, cTableName); + } + } + } + } while (retryAgain); for (int i = 0; i < rows; ++i) { TAOS_BIND* colBinds = taosArrayGetP(rowsBind, i); From 532fb5f7c1a02008990269e2b42ae67ee1925a5d Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Tue, 24 Aug 2021 22:04:28 +0800 Subject: [PATCH 20/40] show firstline and lastline of schemaless performace python scripts --- tests/pytest/tools/schemalessInsertPerformance.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/pytest/tools/schemalessInsertPerformance.py b/tests/pytest/tools/schemalessInsertPerformance.py index 68802310a2..5e009a2129 100644 --- a/tests/pytest/tools/schemalessInsertPerformance.py +++ b/tests/pytest/tools/schemalessInsertPerformance.py @@ -191,7 +191,8 @@ class TDTestCase: def batchInsertTable(self, batch_list): for insert_list in batch_list: print(threading.current_thread().name, "length=", len(insert_list)) - print(threading.current_thread().name, 'firstline', insert_list[0], 'lastline:', insert_list[-1]) + print(threading.current_thread().name, 'firstline', insert_list[0]) + print(threading.current_thread().name, 'lastline:', insert_list[-1]) self._conn.insert_lines(insert_list) print(threading.current_thread().name, 'end') From 1408c0c0ad3ff3ffbc63ea2c9c8da194d486ff2f Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Wed, 25 Aug 2021 08:35:04 +0800 Subject: [PATCH 21/40] schemaless: fix schema modification error --- src/client/src/tscParseLineProtocol.c | 49 ++++++++++++++------------- 1 file changed, 26 insertions(+), 23 deletions(-) diff --git a/src/client/src/tscParseLineProtocol.c b/src/client/src/tscParseLineProtocol.c index 1bce10f95e..e3ed89a591 100644 --- a/src/client/src/tscParseLineProtocol.c +++ b/src/client/src/tscParseLineProtocol.c @@ -849,6 +849,8 @@ static int32_t insertChildTableBatch(TAOS* taos, char* cTableName, SArray* cols return code; } + bool tryAgain = false; + do { code = taos_stmt_set_tbname(stmt, cTableName); if (code != 0) { @@ -878,7 +880,26 @@ static int32_t insertChildTableBatch(TAOS* taos, char* cTableName, SArray* cols if (code != 0) { tscError("SML:0x%"PRIx64" taos_stmt_execute return %d:%s", info->id, code, tstrerror(code)); } - } while (code == TSDB_CODE_TDB_TABLE_RECONFIGURE && try++ < TSDB_MAX_REPLICA); + + tryAgain = false; + if ((code == TSDB_CODE_TDB_INVALID_TABLE_ID + || code == TSDB_CODE_VND_INVALID_VGROUP_ID + || code == TSDB_CODE_TDB_TABLE_RECONFIGURE) && try++ < TSDB_MAX_REPLICA) { + tryAgain = true; + } + + if (code == TSDB_CODE_TDB_INVALID_TABLE_ID || code == TSDB_CODE_VND_INVALID_VGROUP_ID) { + TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); + int32_t code2 = taos_errno(res2); + if (code2 != TSDB_CODE_SUCCESS) { + tscError("SML:0x%" PRIx64 " apply schema action. reset query cache. error: %s", info->id, taos_errstr(res2)); + } + taos_free_result(res2); + if (tryAgain) { + taosMsleep(100 * (2 << (try))); + } + } + } while (tryAgain); if (code != 0) { tscError("SML:0x%"PRIx64" %d:%s", info->id, code, tstrerror(code)); @@ -1084,28 +1105,10 @@ static int32_t applyChildTableFields(TAOS* taos, SSmlSTableSchema* sTableSchema, taosArrayPush(rowsBind, &colBinds); } - int32_t retry = 0; - bool retryAgain = false; - do { - code = insertChildTableBatch(taos, cTableName, sTableSchema->fields, rowsBind, info); - if (code != 0) { - tscError("SML:0x%" PRIx64 " insert into child table %s failed. error %s, retry %d", info->id, cTableName, tstrerror(code), retry); - if (code == TSDB_CODE_TDB_INVALID_TABLE_ID || code == TSDB_CODE_VND_INVALID_VGROUP_ID) { - TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); - int32_t code2 = taos_errno(res2); - if (code2 != TSDB_CODE_SUCCESS) { - tscError("SML:0x%" PRIx64 " apply schema action. reset query cache. error: %s", info->id, taos_errstr(res2)); - } - taos_free_result(res2); - retryAgain = (++retry < TSDB_MAX_REPLICA) ? true : false; - if (retryAgain) { - taosMsleep(100 * (2 << (retry))); - } else { - tscError("SML:0x%" PRIx64 " insert into child table %s reached max retry", info->id, cTableName); - } - } - } - } while (retryAgain); + code = insertChildTableBatch(taos, cTableName, sTableSchema->fields, rowsBind, info); + if (code != 0) { + tscError("SML:0x%"PRIx64" insert into child table %s failed. error %s", info->id, cTableName, tstrerror(code)); + } for (int i = 0; i < rows; ++i) { TAOS_BIND* colBinds = taosArrayGetP(rowsBind, i); From 0ffb24e73fb961c63507c081b0480c2de04f6d09 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Wed, 25 Aug 2021 08:51:53 +0800 Subject: [PATCH 22/40] schemaless: fix spelling error and add try times --- src/client/src/tscParseLineProtocol.c | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/src/client/src/tscParseLineProtocol.c b/src/client/src/tscParseLineProtocol.c index e3ed89a591..acb28533a9 100644 --- a/src/client/src/tscParseLineProtocol.c +++ b/src/client/src/tscParseLineProtocol.c @@ -832,7 +832,6 @@ static int32_t insertChildTableBatch(TAOS* taos, char* cTableName, SArray* cols tscDebug("SML:0x%"PRIx64" insert rows into child table %s. num of rows: %zu", info->id, cTableName, taosArrayGetSize(rowsBind)); int32_t code = 0; - int32_t try = 0; TAOS_STMT* stmt = taos_stmt_init(taos); if (stmt == NULL) { @@ -850,7 +849,7 @@ static int32_t insertChildTableBatch(TAOS* taos, char* cTableName, SArray* cols } bool tryAgain = false; - + int32_t try = 0; do { code = taos_stmt_set_tbname(stmt, cTableName); if (code != 0) { @@ -878,7 +877,7 @@ static int32_t insertChildTableBatch(TAOS* taos, char* cTableName, SArray* cols code = taos_stmt_execute(stmt); if (code != 0) { - tscError("SML:0x%"PRIx64" taos_stmt_execute return %d:%s", info->id, code, tstrerror(code)); + tscError("SML:0x%"PRIx64" taos_stmt_execute return %d:%s, try:%d", info->id, code, tstrerror(code), try); } tryAgain = false; @@ -892,7 +891,7 @@ static int32_t insertChildTableBatch(TAOS* taos, char* cTableName, SArray* cols TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); int32_t code2 = taos_errno(res2); if (code2 != TSDB_CODE_SUCCESS) { - tscError("SML:0x%" PRIx64 " apply schema action. reset query cache. error: %s", info->id, taos_errstr(res2)); + tscError("SML:0x%" PRIx64 " insert child table. reset query cache. error: %s", info->id, taos_errstr(res2)); } taos_free_result(res2); if (tryAgain) { @@ -901,9 +900,6 @@ static int32_t insertChildTableBatch(TAOS* taos, char* cTableName, SArray* cols } } while (tryAgain); - if (code != 0) { - tscError("SML:0x%"PRIx64" %d:%s", info->id, code, tstrerror(code)); - } taos_stmt_close(stmt); return code; From e39f1ef2b904688a3997e8587a749a0067957506 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Wed, 25 Aug 2021 09:08:06 +0800 Subject: [PATCH 23/40] schemaless:add database not ready and network unavailable processing --- src/client/src/tscParseLineProtocol.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/client/src/tscParseLineProtocol.c b/src/client/src/tscParseLineProtocol.c index acb28533a9..c0ed1feb6b 100644 --- a/src/client/src/tscParseLineProtocol.c +++ b/src/client/src/tscParseLineProtocol.c @@ -883,11 +883,14 @@ static int32_t insertChildTableBatch(TAOS* taos, char* cTableName, SArray* cols tryAgain = false; if ((code == TSDB_CODE_TDB_INVALID_TABLE_ID || code == TSDB_CODE_VND_INVALID_VGROUP_ID - || code == TSDB_CODE_TDB_TABLE_RECONFIGURE) && try++ < TSDB_MAX_REPLICA) { + || code == TSDB_CODE_TDB_TABLE_RECONFIGURE + || code == TSDB_CODE_APP_NOT_READY + || code == TSDB_CODE_RPC_NETWORK_UNAVAIL) && try++ < TSDB_MAX_REPLICA) { tryAgain = true; } - if (code == TSDB_CODE_TDB_INVALID_TABLE_ID || code == TSDB_CODE_VND_INVALID_VGROUP_ID) { + if (code == TSDB_CODE_APP_NOT_READY || code == TSDB_CODE_RPC_NETWORK_UNAVAIL || + code == TSDB_CODE_TDB_INVALID_TABLE_ID || code == TSDB_CODE_VND_INVALID_VGROUP_ID) { TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); int32_t code2 = taos_errno(res2); if (code2 != TSDB_CODE_SUCCESS) { From 954dc3798101522fef211b1a3595cf2ca3fd032e Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Wed, 25 Aug 2021 09:13:41 +0800 Subject: [PATCH 24/40] schemaless:decrease sleep time during errors --- src/client/src/tscParseLineProtocol.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/src/tscParseLineProtocol.c b/src/client/src/tscParseLineProtocol.c index c0ed1feb6b..801d32a1ee 100644 --- a/src/client/src/tscParseLineProtocol.c +++ b/src/client/src/tscParseLineProtocol.c @@ -898,7 +898,7 @@ static int32_t insertChildTableBatch(TAOS* taos, char* cTableName, SArray* cols } taos_free_result(res2); if (tryAgain) { - taosMsleep(100 * (2 << (try))); + taosMsleep(50 * (2 << (try))); } } } while (tryAgain); From f6c1d031d35f80154a913c0187415f847d613664 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 25 Aug 2021 13:41:04 +0800 Subject: [PATCH 25/40] [TD-6330] fix free error --- src/client/src/tscSQLParser.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 5f888834ec..2a68b3e1ea 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -427,7 +427,6 @@ int32_t readFromFile(char *name, uint32_t *len, void **buf) { return TSDB_CODE_TSC_APP_ERROR; } close(fd); - tfree(*buf); return TSDB_CODE_SUCCESS; } From 6b652fa2c348bcc642868712ff1c360e7f01ea85 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Wed, 25 Aug 2021 15:53:03 +0800 Subject: [PATCH 26/40] [TD-6343]: taos crash at tscSubquery.c:2447 --- src/client/src/tscSubquery.c | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index 45bcf4095a..df9360f331 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -2442,7 +2442,11 @@ static void doSendQueryReqs(SSchedMsg* pSchedMsg) { SSqlObj* pSql = pSchedMsg->ahandle; SPair* p = pSchedMsg->msg; - for(int32_t i = p->first; i < p->second; ++i) { + for (int32_t i = p->first; i < p->second; ++i) { + if (i == pSql->subState.numOfSub) { + tfree(p); + return; + } SSqlObj* pSub = pSql->pSubs[i]; SRetrieveSupport* pSupport = pSub->param; @@ -2582,7 +2586,12 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) { int32_t numOfTasks = (pState->numOfSub + MAX_REQUEST_PER_TASK - 1)/MAX_REQUEST_PER_TASK; assert(numOfTasks >= 1); - int32_t num = (pState->numOfSub/numOfTasks) + 1; + int32_t num; + if (pState->numOfSub / numOfTasks == MAX_REQUEST_PER_TASK) { + num = MAX_REQUEST_PER_TASK; + } else { + num = pState->numOfSub / numOfTasks + 1; + } tscDebug("0x%"PRIx64 " query will be sent by %d threads", pSql->self, numOfTasks); for(int32_t j = 0; j < numOfTasks; ++j) { From b5fb8921b6527520320fdde7c2e2a900d59519d4 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Wed, 25 Aug 2021 16:03:53 +0800 Subject: [PATCH 27/40] schemaless: fix error code definition --- src/client/src/tscParseLineProtocol.c | 21 +++++++++++++-------- src/client/src/tscSQLParser.c | 4 ++-- src/inc/taoserror.h | 8 ++++---- src/util/src/terror.c | 6 ++++++ 4 files changed, 25 insertions(+), 14 deletions(-) diff --git a/src/client/src/tscParseLineProtocol.c b/src/client/src/tscParseLineProtocol.c index 801d32a1ee..c66800620a 100644 --- a/src/client/src/tscParseLineProtocol.c +++ b/src/client/src/tscParseLineProtocol.c @@ -889,16 +889,21 @@ static int32_t insertChildTableBatch(TAOS* taos, char* cTableName, SArray* cols tryAgain = true; } - if (code == TSDB_CODE_APP_NOT_READY || code == TSDB_CODE_RPC_NETWORK_UNAVAIL || + if (code == TSDB_CODE_APP_NOT_READY || code == TSDB_CODE_TDB_INVALID_TABLE_ID || code == TSDB_CODE_VND_INVALID_VGROUP_ID) { - TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); - int32_t code2 = taos_errno(res2); - if (code2 != TSDB_CODE_SUCCESS) { - tscError("SML:0x%" PRIx64 " insert child table. reset query cache. error: %s", info->id, taos_errstr(res2)); - } - taos_free_result(res2); +// TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); +// int32_t code2 = taos_errno(res2); +// if (code2 != TSDB_CODE_SUCCESS) { +// tscError("SML:0x%" PRIx64 " insert child table. reset query cache. error: %s", info->id, taos_errstr(res2)); +// } +// taos_free_result(res2); if (tryAgain) { - taosMsleep(50 * (2 << (try))); + taosMsleep(50 * (2 << try)); + } + } + if (code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { + if (tryAgain) { + taosMsleep( 50 * (2 << try)); } } } while (tryAgain); diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 2a2e4540e1..f80d3e7fd1 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -6319,7 +6319,7 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { } if (pItem->bytes <= pColSchema->bytes) { - return tscErrorMsgWithCode(TSDB_CODE_TSC_INVALID_COLUMN_LENGTH, tscGetErrorMsgPayload(pCmd), pItem->name, NULL); + return tscErrorMsgWithCode(TSDB_CODE_TSC_INVALID_COLUMN_LENGTH, pMsg, pItem->name, NULL); } SSchema* pSchema = (SSchema*) pTableMetaInfo->pTableMeta->schema; @@ -6370,7 +6370,7 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { } if (pItem->bytes <= pColSchema->bytes) { - return tscErrorMsgWithCode(TSDB_CODE_TSC_INVALID_TAG_LENGTH, tscGetErrorMsgPayload(pCmd), pItem->name, NULL); + return tscErrorMsgWithCode(TSDB_CODE_TSC_INVALID_TAG_LENGTH, pMsg, pItem->name, NULL); } SSchema* pSchema = tscGetTableTagSchema(pTableMetaInfo->pTableMeta); diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index 08f6d0bf4d..86ef53e959 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -105,7 +105,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_TSC_NO_META_CACHED TAOS_DEF_ERROR_CODE(0, 0x021C) //"No table meta cached") #define TSDB_CODE_TSC_DUP_COL_NAMES TAOS_DEF_ERROR_CODE(0, 0x021D) //"duplicated column names") #define TSDB_CODE_TSC_INVALID_TAG_LENGTH TAOS_DEF_ERROR_CODE(0, 0x021E) //"Invalid tag length") -#define TSDB_CODE_TSC_INVALID_COLUMN_LENGTH TAOS_DEF_ERROR_CODE(0, 0x021F) //"Invalid tag length") +#define TSDB_CODE_TSC_INVALID_COLUMN_LENGTH TAOS_DEF_ERROR_CODE(0, 0x021F) //"Invalid tag length") // mnode #define TSDB_CODE_MND_MSG_NOT_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0300) //"Message not processed") @@ -188,6 +188,9 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_INVALID_FUNC TAOS_DEF_ERROR_CODE(0, 0x0374) //"Invalid func") #define TSDB_CODE_MND_INVALID_FUNC_BUFSIZE TAOS_DEF_ERROR_CODE(0, 0x0375) //"Invalid func bufSize") +#define TSDB_CODE_MND_INVALID_TAG_LENGTH TAOS_DEF_ERROR_CODE(0, 0x0376) //"invalid tag length") +#define TSDB_CODE_MND_INVALID_COLUMN_LENGTH TAOS_DEF_ERROR_CODE(0, 0x0377) //"invalid column length") + #define TSDB_CODE_MND_DB_NOT_SELECTED TAOS_DEF_ERROR_CODE(0, 0x0380) //"Database not specified or available") #define TSDB_CODE_MND_DB_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0381) //"Database already exists") #define TSDB_CODE_MND_INVALID_DB_OPTION TAOS_DEF_ERROR_CODE(0, 0x0382) //"Invalid database options") @@ -214,9 +217,6 @@ int32_t* taosGetErrno(); #define TSDB_CODE_DND_TOO_MANY_VNODES TAOS_DEF_ERROR_CODE(0, 0x0405) //"Too many vnode directories") #define TSDB_CODE_DND_EXITING TAOS_DEF_ERROR_CODE(0, 0x0406) //"Dnode is exiting" -#define TSDB_CODE_MND_INVALID_TAG_LENGTH TAOS_DEF_ERROR_CODE(0, 0x410) //"invalid tag length") -#define TSDB_CODE_MND_INVALID_COLUMN_LENGTH TAOS_DEF_ERROR_CODE(0, 0x411) //"invalid column length") - // vnode #define TSDB_CODE_VND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0500) //"Action in progress") #define TSDB_CODE_VND_MSG_NOT_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0501) //"Message not processed") diff --git a/src/util/src/terror.c b/src/util/src/terror.c index 49e46cdde8..addaf54e78 100644 --- a/src/util/src/terror.c +++ b/src/util/src/terror.c @@ -112,6 +112,9 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TSC_EXCEED_SQL_LIMIT, "SQL statement too lon TAOS_DEFINE_ERROR(TSDB_CODE_TSC_FILE_EMPTY, "File is empty") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_LINE_SYNTAX_ERROR, "Syntax error in Line") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_NO_META_CACHED, "No table meta cached") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_DUP_COL_NAMES, "duplicated column names") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_TAG_LENGTH, "Invalid tag length") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_COLUMN_LENGTH, "Invalid tag length") // mnode TAOS_DEFINE_ERROR(TSDB_CODE_MND_MSG_NOT_PROCESSED, "Message not processed") @@ -194,6 +197,9 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_FUNC_ALREADY_EXIST, "Func already exists") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC, "Invalid func") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_BUFSIZE, "Invalid func bufSize") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TAG_LENGTH, "invalid tag length") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_COLUMN_LENGTH, "invalid column length") + TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_NOT_SELECTED, "Database not specified or available") TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_ALREADY_EXIST, "Database already exists") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_OPTION, "Invalid database options") From ae58e4cb60ef2d812810d2c5fd701ba1a24a53b1 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Wed, 25 Aug 2021 16:42:59 +0800 Subject: [PATCH 28/40] schemaless: fix error description --- src/inc/taoserror.h | 2 +- src/util/src/terror.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index 86ef53e959..c401ab762e 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -105,7 +105,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_TSC_NO_META_CACHED TAOS_DEF_ERROR_CODE(0, 0x021C) //"No table meta cached") #define TSDB_CODE_TSC_DUP_COL_NAMES TAOS_DEF_ERROR_CODE(0, 0x021D) //"duplicated column names") #define TSDB_CODE_TSC_INVALID_TAG_LENGTH TAOS_DEF_ERROR_CODE(0, 0x021E) //"Invalid tag length") -#define TSDB_CODE_TSC_INVALID_COLUMN_LENGTH TAOS_DEF_ERROR_CODE(0, 0x021F) //"Invalid tag length") +#define TSDB_CODE_TSC_INVALID_COLUMN_LENGTH TAOS_DEF_ERROR_CODE(0, 0x021F) //"Invalid column length") // mnode #define TSDB_CODE_MND_MSG_NOT_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0300) //"Message not processed") diff --git a/src/util/src/terror.c b/src/util/src/terror.c index addaf54e78..311abe70df 100644 --- a/src/util/src/terror.c +++ b/src/util/src/terror.c @@ -114,7 +114,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TSC_LINE_SYNTAX_ERROR, "Syntax error in Line" TAOS_DEFINE_ERROR(TSDB_CODE_TSC_NO_META_CACHED, "No table meta cached") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_DUP_COL_NAMES, "duplicated column names") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_TAG_LENGTH, "Invalid tag length") -TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_COLUMN_LENGTH, "Invalid tag length") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_COLUMN_LENGTH, "Invalid column length") // mnode TAOS_DEFINE_ERROR(TSDB_CODE_MND_MSG_NOT_PROCESSED, "Message not processed") From a243d2b91d4630464568fb0088d5af9cb10e1fa6 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Wed, 25 Aug 2021 17:08:09 +0800 Subject: [PATCH 29/40] schemaless: fix errors about modify_column --- src/client/src/tscParseLineProtocol.c | 12 ++++++------ src/mnode/src/mnodeTable.c | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/client/src/tscParseLineProtocol.c b/src/client/src/tscParseLineProtocol.c index c66800620a..52e4bd3d40 100644 --- a/src/client/src/tscParseLineProtocol.c +++ b/src/client/src/tscParseLineProtocol.c @@ -891,12 +891,12 @@ static int32_t insertChildTableBatch(TAOS* taos, char* cTableName, SArray* cols if (code == TSDB_CODE_APP_NOT_READY || code == TSDB_CODE_TDB_INVALID_TABLE_ID || code == TSDB_CODE_VND_INVALID_VGROUP_ID) { -// TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); -// int32_t code2 = taos_errno(res2); -// if (code2 != TSDB_CODE_SUCCESS) { -// tscError("SML:0x%" PRIx64 " insert child table. reset query cache. error: %s", info->id, taos_errstr(res2)); -// } -// taos_free_result(res2); + TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); + int32_t code2 = taos_errno(res2); + if (code2 != TSDB_CODE_SUCCESS) { + tscError("SML:0x%" PRIx64 " insert child table. reset query cache. error: %s", info->id, taos_errstr(res2)); + } + taos_free_result(res2); if (tryAgain) { taosMsleep(50 * (2 << try)); } diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index 1bc5607da5..7a0108026e 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -1518,7 +1518,6 @@ static int32_t mnodeChangeSuperTableColumn(SMnodeMsg *pMsg) { // update SSchema *schema = (SSchema *) (pStable->schema + col); ASSERT(schema->type == TSDB_DATA_TYPE_BINARY || schema->type == TSDB_DATA_TYPE_NCHAR); - schema->bytes = pAlter->schema[0].bytes; if (pAlter->schema[0].bytes <= schema->bytes) { mError("msg:%p, app:%p stable:%s, modify column len. column:%s, len from %d to %d", pMsg, pMsg->rpcMsg.ahandle, @@ -1526,6 +1525,7 @@ static int32_t mnodeChangeSuperTableColumn(SMnodeMsg *pMsg) { return TSDB_CODE_MND_INVALID_COLUMN_LENGTH; } + schema->bytes = pAlter->schema[0].bytes; pStable->sversion++; mInfo("msg:%p, app:%p stable %s, start to modify column %s len to %d", pMsg, pMsg->rpcMsg.ahandle, pStable->info.tableId, name, schema->bytes); From 934c2c045350a5a8637c47a3a5d6dcb50543f886 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Thu, 26 Aug 2021 08:28:06 +0800 Subject: [PATCH 30/40] schemaless:REMOVE TAG VALUE PROCESSING FOR NAMED CHILD TABLES --- src/client/src/tscParseLineProtocol.c | 107 +------------------------- 1 file changed, 1 insertion(+), 106 deletions(-) diff --git a/src/client/src/tscParseLineProtocol.c b/src/client/src/tscParseLineProtocol.c index 52e4bd3d40..b01e4daa05 100644 --- a/src/client/src/tscParseLineProtocol.c +++ b/src/client/src/tscParseLineProtocol.c @@ -705,42 +705,6 @@ static int32_t modifyDBSchemas(TAOS* taos, SArray* stableSchemas, SSmlLinesInfo* return 0; } -static int32_t changeChildTableTagValue(TAOS* taos, const char* cTableName, const char* tagName, TAOS_BIND* bind, SSmlLinesInfo* info) { - char sql[512]; - sprintf(sql, "alter table %s set tag %s=?", cTableName, tagName); - - int32_t code; - TAOS_STMT* stmt = taos_stmt_init(taos); - code = taos_stmt_prepare(stmt, sql, (unsigned long)strlen(sql)); - - if (code != 0) { - tscError("SML:0x%"PRIx64" taos_stmt_prepare return %d:%s", info->id, code, tstrerror(code)); - taos_stmt_close(stmt); - return code; - } - - code = taos_stmt_bind_param(stmt, bind); - if (code != 0) { - tscError("SML:0x%"PRIx64" taos_stmt_bind_param return %d:%s", info->id, code, tstrerror(code)); - taos_stmt_close(stmt); - return code; - } - - code = taos_stmt_execute(stmt); - if (code != 0) { - tscError("SML:0x%"PRIx64" taos_stmt_execute return %d:%s", info->id, code, tstrerror(code)); - taos_stmt_close(stmt); - return code; - } - - code = taos_stmt_close(stmt); - if (code != 0) { - tscError("SML:0x%"PRIx64" taos_stmt_close return %d:%s", info->id, code, tstrerror(code)); - return code; - } - return code; -} - static int32_t creatChildTableIfNotExists(TAOS* taos, const char* cTableName, const char* sTableName, SArray* tagsSchema, SArray* tagsBind, SSmlLinesInfo* info) { size_t numTags = taosArrayGetSize(tagsSchema); @@ -970,15 +934,6 @@ static int32_t applyChildTableTags(TAOS* taos, char* cTableName, char* sTableNam tagKVs[*pFieldSchemaIdx] = kv; } } - - int32_t notNullTagsIndices[TSDB_MAX_TAGS] = {0}; - int32_t numNotNullTags = 0; - for (int32_t i = 0; i < numTags; ++i) { - if (tagKVs[i] != NULL) { - notNullTagsIndices[numNotNullTags] = i; - ++numNotNullTags; - } - } SArray* tagBinds = taosArrayInit(numTags, sizeof(TAOS_BIND)); taosArraySetSize(tagBinds, numTags); @@ -1001,68 +956,8 @@ static int32_t applyChildTableTags(TAOS* taos, char* cTableName, char* sTableNam bind->is_null = NULL; } - // select tag1,tag2,... from stable where tbname in (ctable) - char* sql = malloc(tsMaxSQLStringLen+1); - int freeBytes = tsMaxSQLStringLen + 1; - snprintf(sql, freeBytes, "select tbname, "); - for (int i = 0; i < numNotNullTags ; ++i) { - snprintf(sql + strlen(sql), freeBytes-strlen(sql), "%s,", tagKVs[notNullTagsIndices[i]]->key); - } - snprintf(sql + strlen(sql) - 1, freeBytes - strlen(sql) + 1, - " from %s where tbname in (\'%s\')", sTableName, cTableName); - sql[strlen(sql)] = '\0'; + int32_t code = creatChildTableIfNotExists(taos, cTableName, sTableName, sTableSchema->tags, tagBinds, info); - TAOS_RES* result = taos_query(taos, sql); - free(sql); - - int32_t code = taos_errno(result); - if (code != 0) { - tscError("SML:0x%"PRIx64" get child table %s tags failed. error string %s", info->id, cTableName, taos_errstr(result)); - goto cleanup; - } - - // check tag value and set tag values if different - TAOS_ROW row = taos_fetch_row(result); - if (row != NULL) { - int numFields = taos_field_count(result); - TAOS_FIELD* fields = taos_fetch_fields(result); - int* lengths = taos_fetch_lengths(result); - for (int i = 1; i < numFields; ++i) { - uint8_t dbType = fields[i].type; - int32_t length = lengths[i]; - char* val = row[i]; - - TAOS_SML_KV* tagKV = tagKVs[notNullTagsIndices[i-1]]; - if (tagKV->type != dbType) { - tscError("SML:0x%"PRIx64" child table %s tag %s type mismatch. point type : %d, db type : %d", - info->id, cTableName, tagKV->key, tagKV->type, dbType); - return TSDB_CODE_TSC_INVALID_VALUE; - } - - assert(tagKV->value); - - if (val == NULL || length != tagKV->length || memcmp(tagKV->value, val, length) != 0) { - uintptr_t valPointer = (uintptr_t)tagKV; - size_t* pFieldSchemaIdx = taosHashGet(info->smlDataToSchema, &valPointer, sizeof(uintptr_t)); - assert(pFieldSchemaIdx != NULL); - TAOS_BIND* bind = taosArrayGet(tagBinds, *pFieldSchemaIdx); - code = changeChildTableTagValue(taos, cTableName, tagKV->key, bind, info); - if (code != 0) { - tscError("SML:0x%"PRIx64" change child table tag failed. table name %s, tag %s", info->id, cTableName, tagKV->key); - goto cleanup; - } - } - } - tscDebug("SML:0x%"PRIx64" successfully applied point tags. child table: %s", info->id, cTableName); - } else { - code = creatChildTableIfNotExists(taos, cTableName, sTableName, sTableSchema->tags, tagBinds, info); - if (code != 0) { - goto cleanup; - } - } - -cleanup: - taos_free_result(result); for (int i = 0; i < taosArrayGetSize(tagBinds); ++i) { TAOS_BIND* bind = taosArrayGet(tagBinds, i); free(bind->length); From 1e244ce6408bbf519c2d339d8cb8d0e4169f04a2 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Thu, 26 Aug 2021 08:33:19 +0800 Subject: [PATCH 31/40] schemaless: change database not ready processing to sleep without clear global meta cache --- src/client/src/tscParseLineProtocol.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/client/src/tscParseLineProtocol.c b/src/client/src/tscParseLineProtocol.c index b01e4daa05..6c30c78643 100644 --- a/src/client/src/tscParseLineProtocol.c +++ b/src/client/src/tscParseLineProtocol.c @@ -853,8 +853,7 @@ static int32_t insertChildTableBatch(TAOS* taos, char* cTableName, SArray* cols tryAgain = true; } - if (code == TSDB_CODE_APP_NOT_READY || - code == TSDB_CODE_TDB_INVALID_TABLE_ID || code == TSDB_CODE_VND_INVALID_VGROUP_ID) { + if (code == TSDB_CODE_TDB_INVALID_TABLE_ID || code == TSDB_CODE_VND_INVALID_VGROUP_ID) { TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); int32_t code2 = taos_errno(res2); if (code2 != TSDB_CODE_SUCCESS) { @@ -865,7 +864,7 @@ static int32_t insertChildTableBatch(TAOS* taos, char* cTableName, SArray* cols taosMsleep(50 * (2 << try)); } } - if (code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { + if (code == TSDB_CODE_APP_NOT_READY || code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { if (tryAgain) { taosMsleep( 50 * (2 << try)); } From a2a845faebc9bceaba06b913bacc83148d65e1b7 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Thu, 26 Aug 2021 08:43:37 +0800 Subject: [PATCH 32/40] [TD-6238]:Not get meta again when tsParseSql succceeded for stmt insert in tscTableMetaCallback --- src/client/src/tscAsync.c | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/client/src/tscAsync.c b/src/client/src/tscAsync.c index eaad783331..6b12cd0da0 100644 --- a/src/client/src/tscAsync.c +++ b/src/client/src/tscAsync.c @@ -363,15 +363,6 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) { } if (TSDB_QUERY_HAS_TYPE(pCmd->insertParam.insertType, TSDB_QUERY_TYPE_STMT_INSERT)) { // stmt insert - STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - code = tscGetTableMeta(pSql, pTableMetaInfo); - if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { - taosReleaseRef(tscObjRef, pSql->self); - return; - } else { - assert(code == TSDB_CODE_SUCCESS); - } - (*pSql->fp)(pSql->param, pSql, code); } else if (TSDB_QUERY_HAS_TYPE(pCmd->insertParam.insertType, TSDB_QUERY_TYPE_FILE_INSERT)) { // file insert tscImportDataFromFile(pSql); From 51193750af2d9f4a2a623be108113f6bad5c435b Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Wed, 25 Aug 2021 15:53:03 +0800 Subject: [PATCH 33/40] [TD-6343]: taos crash at tscSubquery.c:2447 --- src/client/src/tscSubquery.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index df9360f331..c2a9d516c2 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -2443,7 +2443,7 @@ static void doSendQueryReqs(SSchedMsg* pSchedMsg) { SPair* p = pSchedMsg->msg; for (int32_t i = p->first; i < p->second; ++i) { - if (i == pSql->subState.numOfSub) { + if (i >= pSql->subState.numOfSub) { tfree(p); return; } From 5317e22facc54bb47aefdb2f99b90630946488d4 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 26 Aug 2021 15:42:39 +0800 Subject: [PATCH 34/40] [TD-6336] reproduce and fix core --- src/client/src/tscParseInsert.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index b3d6ade319..1bf27e6cad 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -1777,6 +1777,7 @@ static void parseFileSendDataBlock(void *param, TAOS_RES *tres, int32_t numOfRow } _error: + pParentSql->res.code = code; tfree(tokenBuf); tfree(line); taos_free_result(pSql); From b76f9e0793b8a958024a156cc793affb5f34f9bc Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 26 Aug 2021 16:14:10 +0800 Subject: [PATCH 35/40] [TD-6387] To avoid the uncessary check on results To avoid the uncessary check on the number of returned result from virtual nodes. --- src/client/src/tscSubquery.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index 0c31a01be6..a6fb3b5dd6 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -2987,7 +2987,7 @@ static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfR tscDebug("0x%"PRIx64" sub:0x%"PRIx64" retrieve numOfRows:%d totalNumOfRows:%" PRIu64 " from ep:%s, orderOfSub:%d", pParentSql->self, pSql->self, pRes->numOfRows, pState->numOfRetrievedRows, pSql->epSet.fqdn[pSql->epSet.inUse], idx); - if (num > tsMaxNumOfOrderedResults && /*tscIsProjectionQueryOnSTable(pQueryInfo, 0) &&*/ !(tscGetQueryInfo(&pParentSql->cmd)->distinct)) { + if (num > tsMaxNumOfOrderedResults && tscIsProjectionQueryOnSTable(pQueryInfo, 0) && !(tscGetQueryInfo(&pParentSql->cmd)->distinct)) { tscError("0x%"PRIx64" sub:0x%"PRIx64" num of OrderedRes is too many, max allowed:%" PRId32 " , current:%" PRId64, pParentSql->self, pSql->self, tsMaxNumOfOrderedResults, num); tscAbortFurtherRetryRetrieval(trsupport, tres, TSDB_CODE_TSC_SORTED_RES_TOO_MANY); From d921a078e858728320a550f66e1083bc06c64bc5 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Fri, 27 Aug 2021 09:38:37 +0800 Subject: [PATCH 36/40] [ci skip] update cfg key --- tests/pytest/util/dnodes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/pytest/util/dnodes.py b/tests/pytest/util/dnodes.py index 2abb8f5ee7..0208f884b6 100644 --- a/tests/pytest/util/dnodes.py +++ b/tests/pytest/util/dnodes.py @@ -123,7 +123,7 @@ class TDDnode: "charset":"UTF-8", "asyncLog":"0", "anyIp":"0", - "tsEnableTelemetryReporting":"0", + "telemetryReporting":"0", "dDebugFlag":"135", "tsdbDebugFlag":"135", "mDebugFlag":"135", From dc89fa48308b609f259e7ef8fd741e6abac84c73 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Fri, 27 Aug 2021 09:55:35 +0800 Subject: [PATCH 37/40] [TD-6274]: taos shell --file bug. (#7571) From f159933896a7af7e9db61f50c0e008bfa9b04db8 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Fri, 27 Aug 2021 09:55:44 +0800 Subject: [PATCH 38/40] [TD-6294]: taosdemo support long arg (#7589) * [TD-6294]: taosdemo support long arg fix conflict with develop branch. * fix few words. * declare default child tables number. --- src/kit/taosdemo/taosdemo.c | 908 +++++++++++++++++++++++++----------- src/kit/taosdump/taosdump.c | 26 +- 2 files changed, 634 insertions(+), 300 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 7bd6842df9..df3a741c10 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -75,6 +75,8 @@ extern char configDir[]; #define OPT_ABORT 1 /* –abort */ #define MAX_FILE_NAME_LEN 256 // max file name length on linux is 255. +#define DEFAULT_START_TIME 1500000000000 + #define MAX_PREPARED_RAND 1000000 #define INT_BUFF_LEN 11 #define BIGINT_BUFF_LEN 21 @@ -102,6 +104,8 @@ extern char configDir[]; #define NOTE_BUFF_LEN (SMALL_BUFF_LEN*16) #define DEFAULT_TIMESTAMP_STEP 1 +#define DEFAULT_INTERLACE_ROWS 0 +#define DEFAULT_DATATYPE_NUM 3 #define DEFAULT_CHILDTABLES 10000 @@ -202,9 +206,9 @@ enum _describe_table_index { static char *g_dupstr = NULL; typedef struct SArguments_S { - char * metaFile; + char *metaFile; uint32_t test_mode; - char * host; + char *host; uint16_t port; uint16_t iface; char * user; @@ -223,7 +227,7 @@ typedef struct SArguments_S { char * output_file; bool async_mode; char * datatype[MAX_NUM_COLUMNS + 1]; - uint32_t len_of_binary; + uint32_t binwidth; uint32_t num_of_CPR; uint32_t num_of_threads; uint64_t insert_interval; @@ -364,7 +368,7 @@ typedef struct SDbs_S { bool asyncMode; uint32_t threadCount; - uint32_t threadCountByCreateTbl; + uint32_t threadCountForCreateTbl; uint32_t dbCount; SDataBase db[MAX_DB_COUNT]; @@ -583,24 +587,22 @@ char *g_randdouble_buff = NULL; char *g_aggreFunc[] = {"*", "count(*)", "avg(col0)", "sum(col0)", "max(col0)", "min(col0)", "first(col0)", "last(col0)"}; -#define DEFAULT_DATATYPE_NUM 3 - SArguments g_args = { - NULL, // metaFile - 0, // test_mode - "127.0.0.1", // host - 6030, // port - INTERFACE_BUT, // iface - "root", // user + NULL, // metaFile + 0, // test_mode + "localhost", // host + 6030, // port + INTERFACE_BUT, // iface + "root", // user #ifdef _TD_POWER_ "powerdb", // password #elif (_TD_TQ_ == true) - "tqueue", // password + "tqueue", // password #else - "taosdata", // password + "taosdata", // password #endif - "test", // database - 1, // replica + "test", // database + 1, // replica "d", // tb_prefix NULL, // sqlFile true, // use_metric @@ -617,13 +619,13 @@ SArguments g_args = { "INT", // datatype "FLOAT", // datatype. DEFAULT_DATATYPE_NUM is 3 }, - 64, // len_of_binary + 64, // binwidth 4, // num_of_CPR 10, // num_of_connections/thread 0, // insert_interval DEFAULT_TIMESTAMP_STEP, // timestamp_step 1, // query_times - 0, // interlace_rows; + DEFAULT_INTERLACE_ROWS, // interlace_rows; 30000, // num_of_RPR (1024*1024), // max_sql_len DEFAULT_CHILDTABLES, // num_of_tables @@ -637,8 +639,6 @@ SArguments g_args = { true, // demo_mode; }; - - static SDbs g_Dbs; static int64_t g_totalChildTables = DEFAULT_CHILDTABLES; static int64_t g_actualChildTables = 0; @@ -720,86 +720,93 @@ static void printVersion() { } static void printHelp() { - char indent[10] = " "; - printf("%s%s%s%s\n", indent, "-f", indent, + char indent[10] = " "; + printf("%s\n\n", "Usage: taosdemo [OPTION...]"); + printf("%s%s%s%s\n", indent, "-f, --file=FILE", "\t\t", "The meta file to the execution procedure. Default is './meta.json'."); - printf("%s%s%s%s\n", indent, "-u", indent, - "The TDengine user name to use when connecting to the server. Default is 'root'."); + printf("%s%s%s%s\n", indent, "-u, --user=USER", "\t\t", + "The user name to use when connecting to the server."); #ifdef _TD_POWER_ - printf("%s%s%s%s\n", indent, "-p", indent, - "The password to use when connecting to the server. Default is 'powerdb'."); - printf("%s%s%s%s\n", indent, "-c", indent, + printf("%s%s%s%s\n", indent, "-p, --password", "\t\t", + "The password to use when connecting to the server. Default is 'powerdb'"); + printf("%s%s%s%s\n", indent, "-c, --config-dir=CONFIG_DIR", "\t", "Configuration directory. Default is '/etc/power/'."); #elif (_TD_TQ_ == true) - printf("%s%s%s%s\n", indent, "-p", indent, - "The password to use when connecting to the server. Default is 'tqueue'."); - printf("%s%s%s%s\n", indent, "-c", indent, + printf("%s%s%s%s\n", indent, "-p, --password", "\t\t", + "The password to use when connecting to the server. Default is 'tqueue'"); + printf("%s%s%s%s\n", indent, "-c, --config-dir=CONFIG_DIR", "\t", "Configuration directory. Default is '/etc/tq/'."); #else - printf("%s%s%s%s\n", indent, "-p", indent, - "The password to use when connecting to the server. Default is 'taosdata'."); - printf("%s%s%s%s\n", indent, "-c", indent, - "Configuration directory. Default is '/etc/taos/'."); + printf("%s%s%s%s\n", indent, "-p, --password", "\t\t", + "The password to use when connecting to the server."); + printf("%s%s%s%s\n", indent, "-c, --config-dir=CONFIG_DIR", "\t", + "Configuration directory."); #endif - printf("%s%s%s%s\n", indent, "-h", indent, - "The host to connect to TDengine. Default is localhost."); - printf("%s%s%s%s\n", indent, "-P", indent, - "The TCP/IP port number to use for the connection. Default is 0."); - printf("%s%s%s%s\n", indent, "-I", indent, + printf("%s%s%s%s\n", indent, "-h, --host=HOST", "\t\t", + "TDengine server FQDN to connect. The default host is localhost."); + printf("%s%s%s%s\n", indent, "-P, --port=PORT", "\t\t", + "The TCP/IP port number to use for the connection."); + printf("%s%s%s%s\n", indent, "-I, --interface=INTERFACE", "\t", "The interface (taosc, rest, and stmt) taosdemo uses. Default is 'taosc'."); - printf("%s%s%s%s\n", indent, "-d", indent, + printf("%s%s%s%s\n", indent, "-d, --database=DATABASE", "\t", "Destination database. Default is 'test'."); - printf("%s%s%s%s\n", indent, "-a", indent, + printf("%s%s%s%s\n", indent, "-a, --replica=REPLICA", "\t\t", "Set the replica parameters of the database, Default 1, min: 1, max: 3."); - printf("%s%s%s%s\n", indent, "-m", indent, + printf("%s%s%s%s\n", indent, "-m, --table-prefix=TABLEPREFIX", "\t", "Table prefix name. Default is 'd'."); - printf("%s%s%s%s\n", indent, "-s", indent, "The select sql file."); - printf("%s%s%s%s\n", indent, "-N", indent, "Use normal table flag."); - printf("%s%s%s%s\n", indent, "-o", indent, + printf("%s%s%s%s\n", indent, "-s, --sql-file=FILE", "\t\t", "The select sql file."); + printf("%s%s%s%s\n", indent, "-N, --normal-table", "\t\t", "Use normal table flag."); + printf("%s%s%s%s\n", indent, "-o, --output=FILE", "\t\t", "Direct output to the named file. Default is './output.txt'."); - printf("%s%s%s%s\n", indent, "-q", indent, + printf("%s%s%s%s\n", indent, "-s, --sql-file=FILE", "\t\t", + "The select sql file."); + printf("%s%s%s%s\n", indent, "-q, --query-mode=MODE", "\t\t", "Query mode -- 0: SYNC, 1: ASYNC. Default is SYNC."); - printf("%s%s%s%s\n", indent, "-b", indent, + printf("%s%s%s%s\n", indent, "-b, --data-type=DATATYPE", "\t", "The data_type of columns, default: FLOAT, INT, FLOAT."); - printf("%s%s%s%s%d\n", indent, "-w", indent, - "The length of data_type 'BINARY' or 'NCHAR'. Default is ", - g_args.len_of_binary); - printf("%s%s%s%s%d%s%d\n", indent, "-l", indent, + printf("%s%s%s%s%d\n", indent, "-w, --binwidth=WIDTH", "\t\t", + "The width of data_type 'BINARY' or 'NCHAR'. Default is ", + g_args.binwidth); + printf("%s%s%s%s%d%s%d\n", indent, "-l, --columns=COLUMNS", "\t\t", "The number of columns per record. Demo mode by default is ", DEFAULT_DATATYPE_NUM, " (float, int, float). Max values is ", MAX_NUM_COLUMNS); printf("%s%s%s%s\n", indent, indent, indent, - "All of the new column(s) type is INT. If use -b to specify column type, -l will be ignored."); - printf("%s%s%s%s\n", indent, "-T", indent, + "\t\t\t\tAll of the new column(s) type is INT. If use -b to specify column type, -l will be ignored."); + printf("%s%s%s%s\n", indent, "-T, --threads=NUMBER", "\t\t", "The number of threads. Default is 10."); - printf("%s%s%s%s\n", indent, "-i", indent, + printf("%s%s%s%s\n", indent, "-i, --insert-interval=NUMBER", "\t", "The sleep time (ms) between insertion. Default is 0."); - printf("%s%s%s%s%d.\n", indent, "-S", indent, + printf("%s%s%s%s%d.\n", indent, "-S, --time-step=TIME_STEP", "\t", "The timestamp step between insertion. Default is ", DEFAULT_TIMESTAMP_STEP); - printf("%s%s%s%s\n", indent, "-r", indent, + printf("%s%s%s%s%d.\n", indent, "-B, --interlace-rows=NUMBER", "\t", + "The interlace rows of insertion. Default is ", + DEFAULT_INTERLACE_ROWS); + printf("%s%s%s%s\n", indent, "-r, --rec-per-req=NUMBER", "\t", "The number of records per request. Default is 30000."); - printf("%s%s%s%s\n", indent, "-t", indent, + printf("%s%s%s%s\n", indent, "-t, --tables=NUMBER", "\t\t", "The number of tables. Default is 10000."); - printf("%s%s%s%s\n", indent, "-n", indent, + printf("%s%s%s%s\n", indent, "-n, --records=NUMBER", "\t\t", "The number of records per table. Default is 10000."); - printf("%s%s%s%s\n", indent, "-M", indent, + printf("%s%s%s%s\n", indent, "-M, --random", "\t\t\t", "The value of records generated are totally random."); - printf("%s%s%s%s\n", indent, indent, indent, - " The default is to simulate power equipment senario."); - printf("%s%s%s%s\n", indent, "-x", indent, "Not insert only flag."); - printf("%s%s%s%s\n", indent, "-y", indent, "Default input yes for prompt."); - printf("%s%s%s%s\n", indent, "-O", indent, - "Insert mode--0: In order, 1 ~ 50: disorder ratio. Default is in order."); - printf("%s%s%s%s\n", indent, "-R", indent, + printf("%s\n", "\t\t\t\tThe default is to simulate power equipment senario."); + printf("%s%s%s%s\n", indent, "-x, --no-insert", "\t\t", + "No-insert flag."); + printf("%s%s%s%s\n", indent, "-y, --answer-yes", "\t\t", "Default input yes for prompt."); + printf("%s%s%s%s\n", indent, "-O, --disorder=NUMBER", "\t\t", + "Insert order mode--0: In order, 1 ~ 50: disorder ratio. Default is in order."); + printf("%s%s%s%s\n", indent, "-R, --disorder-range=NUMBER", "\t", "Out of order data's range, ms, default is 1000."); - printf("%s%s%s%s\n", indent, "-g", indent, + printf("%s%s%s%s\n", indent, "-g, --debug", "\t\t\t", "Print debug info."); - printf("%s%s%s\n", indent, "-V, --version\t", - "Print version info."); - printf("%s%s%s%s\n", indent, "--help\t", indent, - "Print command line arguments list info."); + printf("%s%s%s%s\n", indent, "-?, --help\t", "\t\t", + "Give this help list"); + printf("%s%s%s%s\n", indent, " --usage\t", "\t\t", + "Give a short usage message"); + printf("%s%s\n", indent, "-V, --version\t\t\tPrint version info."); /* printf("%s%s%s%s\n", indent, "-D", indent, "Delete database if exists. 0: no, 1: yes, default is 1"); */ @@ -820,66 +827,149 @@ static bool isStringNumber(char *input) return true; } +static void errorPrintReqArg(char *program, char *wrong_arg) +{ + fprintf(stderr, + "%s: option requires an argument -- '%s'\n", + program, wrong_arg); + fprintf(stderr, + "Try `taosdemo --help' or `taosdemo --usage' for more information.\n"); +} + +static void errorPrintReqArg2(char *program, char *wrong_arg) +{ + fprintf(stderr, + "%s: option requires a number argument '-%s'\n", + program, wrong_arg); + fprintf(stderr, + "Try `taosdemo --help' or `taosdemo --usage' for more information.\n"); +} + +static void errorPrintReqArg3(char *program, char *wrong_arg) +{ + fprintf(stderr, + "%s: option '%s' requires an argument\n", + program, wrong_arg); + fprintf(stderr, + "Try `taosdemo --help' or `taosdemo --usage' for more information.\n"); +} + static void parse_args(int argc, char *argv[], SArguments *arguments) { for (int i = 1; i < argc; i++) { - if (strcmp(argv[i], "-f") == 0) { - arguments->demo_mode = false; + if ((strcmp(argv[i], "-f") == 0) + || (0 == strncmp(argv[i], "--file", strlen("--file")))) { + if (2 == strlen(argv[i])) { + arguments->demo_mode = false; - if (NULL == argv[i+1]) { - printHelp(); - errorPrint("%s", "\n\t-f need a valid json file following!\n"); + if (NULL == argv[i+1]) { + errorPrintReqArg3(argv[0], "f"); + exit(EXIT_FAILURE); + } + arguments->metaFile = argv[++i]; + } else if (strlen("--file") == strlen(argv[i])) { + errorPrintReqArg3(argv[0], "--file"); exit(EXIT_FAILURE); + } else if (0 == strncmp(argv[i], "--file=", strlen("--file="))) { + arguments->metaFile = (char *)(argv[i] + strlen("--file=")); } - arguments->metaFile = argv[++i]; - } else if (strcmp(argv[i], "-c") == 0) { - if (argc == i+1) { - printHelp(); - errorPrint("%s", "\n\t-c need a valid path following!\n"); + } else if ((strcmp(argv[i], "-c") == 0) + || (0 == strncmp(argv[i], "--config-dir", strlen("--config-dir")))) { + if (2 == strlen(argv[i])) { + if (argc == i+1) { + errorPrintReqArg(argv[0], "c"); + exit(EXIT_FAILURE); + } + tstrncpy(configDir, argv[++i], TSDB_FILENAME_LEN); + } else if (strlen("--config-dir") == strlen(argv[i])) { + errorPrintReqArg3(argv[0], "--config-dir"); exit(EXIT_FAILURE); + } else if (0 == strncmp(argv[i], "--config-dir=", strlen("--config-dir="))) { + tstrncpy(configDir, (char *)(argv[i] + strlen("--config-dir=")), TSDB_FILENAME_LEN); } - tstrncpy(configDir, argv[++i], TSDB_FILENAME_LEN); - } else if (strcmp(argv[i], "-h") == 0) { - if (argc == i+1) { - printHelp(); - errorPrint("%s", "\n\t-h need a valid string following!\n"); + } else if ((strcmp(argv[i], "-h") == 0) + || (0 == strncmp(argv[i], "--host", strlen("--host")))) { + if (2 == strlen(argv[i])) { + if (argc == i+1) { + errorPrintReqArg(argv[0], "h"); + exit(EXIT_FAILURE); + } + arguments->host = argv[++i]; + } else if (strlen("--host") == strlen(argv[i])) { + errorPrintReqArg3(argv[0], "--host"); exit(EXIT_FAILURE); + } else if (0 == strncmp(argv[i], "--host=", strlen("--host="))) { + arguments->host = (char *)(argv[i] + strlen("--host=")); } - arguments->host = argv[++i]; - } else if (strcmp(argv[i], "-P") == 0) { - if ((argc == i+1) || - (!isStringNumber(argv[i+1]))) { - printHelp(); - errorPrint("%s", "\n\t-P need a number following!\n"); + } else if ((strcmp(argv[i], "-P") == 0) + || (0 == strncmp(argv[i], "--port", strlen("--port")))) { + if (2 == strlen(argv[i])) { + if (argc == i+1) { + errorPrintReqArg(argv[0], "P"); + exit(EXIT_FAILURE); + } else if (!isStringNumber(argv[i+1])) { + errorPrintReqArg2(argv[0], "P"); + exit(EXIT_FAILURE); + } + arguments->port = atoi(argv[++i]); + } else if (strlen("--port") == strlen(argv[i])) { + errorPrintReqArg3(argv[0], "--port"); exit(EXIT_FAILURE); + } else if (0 == strncmp(argv[i], "--port=", strlen("--port="))) { + if (isStringNumber((char *)(argv[i] + strlen("--port=")))) { + arguments->port = atoi((char *)(argv[i]+strlen("--port="))); + } } - arguments->port = atoi(argv[++i]); - } else if (strcmp(argv[i], "-I") == 0) { - if (argc == i+1) { - printHelp(); - errorPrint("%s", "\n\t-I need a valid string following!\n"); + } else if ((strcmp(argv[i], "-I") == 0) + || (0 == strncmp(argv[i], "--interface", strlen("--interface")))) { + if (2 == strlen(argv[i])) { + if (argc == i+1) { + errorPrintReqArg(argv[0], "I"); + exit(EXIT_FAILURE); + } + if (0 == strcasecmp(argv[i+1], "taosc")) { + arguments->iface = TAOSC_IFACE; + } else if (0 == strcasecmp(argv[i+1], "rest")) { + arguments->iface = REST_IFACE; + } else if (0 == strcasecmp(argv[i+1], "stmt")) { + arguments->iface = STMT_IFACE; + } else { + errorPrintReqArg(argv[0], "I"); + exit(EXIT_FAILURE); + } + i++; + } else if (strlen("--interface") == strlen(argv[i])) { + errorPrintReqArg3(argv[0], "--interface"); exit(EXIT_FAILURE); + } else if (0 == strncmp(argv[i], "--interface=", strlen("--interface="))) { + if (0 == strcasecmp((char *)(argv[i] + strlen("--interface=")), "taosc")) { + arguments->iface = TAOSC_IFACE; + } else if (0 == strcasecmp((char *)(argv[i] + strlen("--interface=")), "rest")) { + arguments->iface = REST_IFACE; + } else if (0 == strcasecmp((char *)(argv[i] + strlen("--interface=")), "stmt")) { + arguments->iface = STMT_IFACE; + } else { + errorPrintReqArg3(argv[0], "--interface"); + exit(EXIT_FAILURE); + } } - ++i; - if (0 == strcasecmp(argv[i], "taosc")) { - arguments->iface = TAOSC_IFACE; - } else if (0 == strcasecmp(argv[i], "rest")) { - arguments->iface = REST_IFACE; - } else if (0 == strcasecmp(argv[i], "stmt")) { - arguments->iface = STMT_IFACE; - } else { - errorPrint("%s", "\n\t-I need a valid string following!\n"); + } else if ((strcmp(argv[i], "-u") == 0) + || (0 == strncmp(argv[i], "--user", strlen("--user")))) { + if (2 == strlen(argv[i])) { + if (argc == i+1) { + errorPrintReqArg(argv[0], "u"); + exit(EXIT_FAILURE); + } + arguments->user = argv[++i]; + } else if (strlen("--user") == strlen(argv[i])) { + errorPrintReqArg3(argv[0], "--user"); exit(EXIT_FAILURE); + } else if (0 == strncmp(argv[i], "--user=", strlen("--user="))) { + arguments->user = (char *)(argv[i++] + strlen("--user=")); } - } else if (strcmp(argv[i], "-u") == 0) { - if (argc == i+1) { - printHelp(); - errorPrint("%s", "\n\t-u need a valid string following!\n"); - exit(EXIT_FAILURE); - } - arguments->user = argv[++i]; - } else if (strncmp(argv[i], "-p", 2) == 0) { - if (strlen(argv[i]) == 2) { + } else if ((strncmp(argv[i], "-p", 2) == 0) + || (0 == strcmp(argv[i], "--password"))) { + if ((strlen(argv[i]) == 2) || (0 == strcmp(argv[i], "--password"))) { printf("Enter password: "); taosSetConsoleEcho(false); if (scanf("%s", arguments->password) > 1) { @@ -889,37 +979,100 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { } else { tstrncpy(arguments->password, (char *)(argv[i] + 2), SHELL_MAX_PASSWORD_LEN); } - } else if (strcmp(argv[i], "-o") == 0) { - if (argc == i+1) { - printHelp(); - errorPrint("%s", "\n\t-o need a valid string following!\n"); + } else if ((strcmp(argv[i], "-o") == 0) + || (0 == strncmp(argv[i], "--output", strlen("--output")))) { + if (2 == strlen(argv[i])) { + if (argc == i+1) { + errorPrintReqArg3(argv[0], "o"); + exit(EXIT_FAILURE); + } + arguments->output_file = argv[++i]; + } else if (strlen("--output") == strlen(argv[i])) { + errorPrintReqArg3(argv[0], "--output"); exit(EXIT_FAILURE); + } else if (0 == strncmp(argv[i], "--output=", strlen("--output="))) { + arguments->output_file = (char *)(argv[i++] + strlen("--output=")); } - arguments->output_file = argv[++i]; - } else if (strcmp(argv[i], "-s") == 0) { - if (argc == i+1) { - printHelp(); - errorPrint("%s", "\n\t-s need a valid string following!\n"); + } else if ((strcmp(argv[i], "-s") == 0) + || (0 == strncmp(argv[i], "--sql-file", strlen("--sql-file")))) { + if (2 == strlen(argv[i])) { + if (argc == i+1) { + errorPrintReqArg(argv[0], "s"); + exit(EXIT_FAILURE); + } + arguments->sqlFile = argv[++i]; + } else if (strlen("--sql-file") == strlen(argv[i])) { + errorPrintReqArg3(argv[0], "--sql-file"); exit(EXIT_FAILURE); + } else if (0 == strncmp(argv[i], "--sql-file=", strlen("--sql-file="))) { + arguments->host = (char *)(argv[i++] + strlen("--sql-file=")); } - arguments->sqlFile = argv[++i]; - } else if (strcmp(argv[i], "-q") == 0) { - if ((argc == i+1) - || (!isStringNumber(argv[i+1]))) { - printHelp(); - errorPrint("%s", "\n\t-q need a number following!\nQuery mode -- 0: SYNC, not-0: ASYNC. Default is SYNC.\n"); + } else if ((strcmp(argv[i], "-q") == 0) + || (0 == strncmp(argv[i], "--query-mode", strlen("--query-mode")))) { + if (2 == strlen(argv[i])) { + if (argc == i+1) { + errorPrintReqArg(argv[0], "q"); + exit(EXIT_FAILURE); + } else if (!isStringNumber(argv[i+1])) { + errorPrintReqArg2(argv[0], "q"); + exit(EXIT_FAILURE); + } + arguments->async_mode = atoi(argv[++i]); + } else if (strlen("--query-mode") == strlen(argv[i])) { + errorPrintReqArg3(argv[0], "--query-mode"); exit(EXIT_FAILURE); + } else if (0 == strncmp(argv[i], "--query-mode=", strlen("--query-mode="))) { + if (isStringNumber((char *)(argv[i] + strlen("--query-mode=")))) { + arguments->async_mode = atoi((char *)(argv[i]+strlen("--query-mode="))); + } else { + errorPrintReqArg2(argv[0], "--query-mode"); + exit(EXIT_FAILURE); + } } - arguments->async_mode = atoi(argv[++i]); - } else if (strcmp(argv[i], "-T") == 0) { - if ((argc == i+1) - || (!isStringNumber(argv[i+1]))) { - printHelp(); - errorPrint("%s", "\n\t-T need a number following!\n"); + } else if ((strcmp(argv[i], "-T") == 0) + || (0 == strncmp(argv[i], "--threads", strlen("--threads")))) { + if (2 == strlen(argv[i])) { + if (argc == i+1) { + errorPrintReqArg(argv[0], "T"); + exit(EXIT_FAILURE); + } else if (!isStringNumber(argv[i+1])) { + errorPrintReqArg2(argv[0], "T"); + exit(EXIT_FAILURE); + } + arguments->num_of_threads = atoi(argv[++i]); + } else if (strlen("--threads") == strlen(argv[i])) { + errorPrintReqArg3(argv[0], "--threads"); exit(EXIT_FAILURE); + } else if (0 == strncmp(argv[i], "--threads=", strlen("--threads="))) { + if (isStringNumber((char *)(argv[i] + strlen("--threads=")))) { + arguments->num_of_threads = atoi((char *)(argv[i]+strlen("--threads="))); + } else { + errorPrintReqArg2(argv[0], "--threads"); + exit(EXIT_FAILURE); + } + } + } else if ((strcmp(argv[i], "-i") == 0) + || (0 == strncmp(argv[i], "--insert-interval", strlen("--insert-interval")))) { + if (2 == strlen(argv[i])) { + if (argc == i+1) { + errorPrintReqArg(argv[0], "i"); + exit(EXIT_FAILURE); + } else if (!isStringNumber(argv[i+1])) { + errorPrintReqArg2(argv[0], "i"); + exit(EXIT_FAILURE); + } + arguments->insert_interval = atoi(argv[++i]); + } else if (strlen("--insert-interval")== strlen(argv[i])) { + errorPrintReqArg3(argv[0], "--insert-interval"); + exit(EXIT_FAILURE); + } else if (0 == strncmp(argv[i], "--insert-interval=", strlen("--insert-interval="))) { + if (isStringNumber((char *)(argv[i] + 18))) { + arguments->insert_interval = atoi((char *)(argv[i]+strlen("--insert-interval="))); + } else { + errorPrintReqArg3(argv[0], "--insert-innterval"); + exit(EXIT_FAILURE); + } } - arguments->num_of_threads = atoi(argv[++i]); - } else if (strcmp(argv[i], "-i") == 0) { if ((argc == i+1) || (!isStringNumber(argv[i+1]))) { printHelp(); @@ -927,14 +1080,28 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { exit(EXIT_FAILURE); } arguments->insert_interval = atoi(argv[++i]); - } else if (strcmp(argv[i], "-S") == 0) { - if ((argc == i+1) || - (!isStringNumber(argv[i+1]))) { - printHelp(); - errorPrint("\n\t%s%s", argv[i], " need a number following!\n"); + } else if ((strcmp(argv[i], "-S") == 0) + || (0 == strncmp(argv[i], "--time-step", strlen("--time-step")))) { + if (2 == strlen(argv[i])) { + if (argc == i+1) { + errorPrintReqArg(argv[0], "S"); + exit(EXIT_FAILURE); + } else if (!isStringNumber(argv[i+1])) { + errorPrintReqArg2(argv[0], "S"); + exit(EXIT_FAILURE); + } + arguments->async_mode = atoi(argv[++i]); + } else if (strlen("--time-step") == strlen(argv[i])) { + errorPrintReqArg3(argv[0], "--time-step"); exit(EXIT_FAILURE); + } else if (0 == strncmp(argv[i], "--time-step=", strlen("--time-step="))) { + if (isStringNumber((char *)(argv[i] + strlen("--time-step=")))) { + arguments->async_mode = atoi((char *)(argv[i]+strlen("--time-step="))); + } else { + errorPrintReqArg2(argv[0], "--time-step"); + exit(EXIT_FAILURE); + } } - arguments->timestamp_step = atoi(argv[++i]); } else if (strcmp(argv[i], "-qt") == 0) { if ((argc == i+1) || (!isStringNumber(argv[i+1]))) { @@ -943,56 +1110,134 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { exit(EXIT_FAILURE); } arguments->query_times = atoi(argv[++i]); - } else if (strcmp(argv[i], "-B") == 0) { - if ((argc == i+1) - || (!isStringNumber(argv[i+1]))) { - printHelp(); - errorPrint("%s", "\n\t-B need a number following!\n"); + } else if ((strcmp(argv[i], "-B") == 0) + || (0 == strncmp(argv[i], "--interlace-rows", strlen("--interlace-rows")))) { + if (2 == strlen(argv[i])) { + if (argc == i+1) { + errorPrintReqArg(argv[0], "B"); + exit(EXIT_FAILURE); + } else if (!isStringNumber(argv[i+1])) { + errorPrintReqArg2(argv[0], "B"); + exit(EXIT_FAILURE); + } + arguments->interlace_rows = atoi(argv[++i]); + } else if (strlen("--interlace-rows")== strlen(argv[i])) { + errorPrintReqArg3(argv[0], "--interlace-rows"); exit(EXIT_FAILURE); - } - arguments->interlace_rows = atoi(argv[++i]); - } else if (strcmp(argv[i], "-r") == 0) { - if ((argc == i+1) - || (!isStringNumber(argv[i+1]))) { - printHelp(); - errorPrint("%s", "\n\t-r need a number following!\n"); - exit(EXIT_FAILURE); - } - arguments->num_of_RPR = atoi(argv[++i]); - } else if (strcmp(argv[i], "-t") == 0) { - if ((argc == i+1) || - (!isStringNumber(argv[i+1]))) { - printHelp(); - errorPrint("%s", "\n\t-t need a number following!\n"); - exit(EXIT_FAILURE); - } - arguments->num_of_tables = atoi(argv[++i]); - g_totalChildTables = arguments->num_of_tables; - } else if (strcmp(argv[i], "-n") == 0) { - if ((argc == i+1) || - (!isStringNumber(argv[i+1]))) { - printHelp(); - errorPrint("%s", "\n\t-n need a number following!\n"); - exit(EXIT_FAILURE); - } - arguments->num_of_DPT = atoi(argv[++i]); - } else if (strcmp(argv[i], "-d") == 0) { - if (argc == i+1) { - printHelp(); - errorPrint("%s", "\n\t-d need a valid string following!\n"); - exit(EXIT_FAILURE); - } - arguments->database = argv[++i]; - } else if (strcmp(argv[i], "-l") == 0) { - arguments->demo_mode = false; - if (argc == i+1) { - if (!isStringNumber(argv[i+1])) { - printHelp(); - errorPrint("%s", "\n\t-l need a number following!\n"); + } else if (0 == strncmp(argv[i], "--interlace-rows=", strlen("--interlace-rows="))) { + if (isStringNumber((char *)(argv[i] + strlen("--interlace-rows=")))) { + arguments->interlace_rows = atoi((char *)(argv[i]+strlen("--interlace-rows="))); + } else { + errorPrintReqArg2(argv[0], "--interlace-rows"); + exit(EXIT_FAILURE); + } + } + } else if ((strcmp(argv[i], "-r") == 0) + || (0 == strncmp(argv[i], "--rec-per-req", 13))) { + if (2 == strlen(argv[i])) { + if (argc == i+1) { + errorPrintReqArg(argv[0], "r"); + exit(EXIT_FAILURE); + } else if (!isStringNumber(argv[i+1])) { + errorPrintReqArg2(argv[0], "r"); + exit(EXIT_FAILURE); + } + arguments->num_of_RPR = atoi(argv[++i]); + } else if (strlen("--rec-per-req")== strlen(argv[i])) { + errorPrintReqArg3(argv[0], "--rec-per-req"); + exit(EXIT_FAILURE); + } else if (0 == strncmp(argv[i], "--rec-per-req=", strlen("--rec-per-req="))) { + if (isStringNumber((char *)(argv[i] + strlen("--rec-per-req=")))) { + arguments->num_of_RPR = atoi((char *)(argv[i]+strlen("--rec-per-req="))); + } else { + errorPrintReqArg2(argv[0], "--rec-per-req"); + exit(EXIT_FAILURE); + } + } + } else if ((strcmp(argv[i], "-t") == 0) + || (0 == strncmp(argv[i], "--tables", strlen("--tables")))) { + if (2 == strlen(argv[i])) { + if (argc == i+1) { + errorPrintReqArg(argv[0], "t"); + exit(EXIT_FAILURE); + } else if (!isStringNumber(argv[i+1])) { + errorPrintReqArg2(argv[0], "t"); + exit(EXIT_FAILURE); + } + arguments->num_of_tables = atoi(argv[++i]); + } else if (strlen("--tables") == strlen(argv[i])) { + errorPrintReqArg3(argv[0], "--tables"); + exit(EXIT_FAILURE); + } else if (0 == strncmp(argv[i], "--tables=", strlen("--tables="))) { + if (isStringNumber((char *)(argv[i] + strlen("--tables=")))) { + arguments->num_of_tables = atoi((char *)(argv[i]+strlen("--tables="))); + } else { + errorPrintReqArg2(argv[0], "--tables"); + exit(EXIT_FAILURE); + } + } + } else if ((strcmp(argv[i], "-n") == 0) + || (0 == strncmp(argv[i], "--records", strlen("--records")))) { + if (2 == strlen(argv[i])) { + if (argc == i+1) { + errorPrintReqArg(argv[0], "n"); + exit(EXIT_FAILURE); + } else if (!isStringNumber(argv[i+1])) { + errorPrintReqArg2(argv[0], "n"); + exit(EXIT_FAILURE); + } + arguments->num_of_DPT = atoi(argv[++i]); + } else if (strlen("--records") == strlen(argv[i])) { + errorPrintReqArg3(argv[0], "--records"); + exit(EXIT_FAILURE); + } else if (0 == strncmp(argv[i], "--records=", strlen("--records="))) { + if (isStringNumber((char *)(argv[i] + strlen("--records=")))) { + arguments->num_of_DPT = atoi((char *)(argv[i]+strlen("--records="))); + } else { + errorPrintReqArg2(argv[0], "--records"); + exit(EXIT_FAILURE); + } + } + + g_totalChildTables = arguments->num_of_DPT; + } else if ((strcmp(argv[i], "-d") == 0) + + || (0 == strncmp(argv[i], "--database", strlen("--database")))) { + if (2 == strlen(argv[i])) { + if (argc == i+1) { + errorPrintReqArg3(argv[0], "d"); + exit(EXIT_FAILURE); + } + arguments->database = argv[++i]; + } else if (strlen("--database") == strlen(argv[i])) { + errorPrintReqArg3(argv[0], "--database"); + exit(EXIT_FAILURE); + } else if (0 == strncmp(argv[i], "--database=", strlen("--database="))) { + arguments->output_file = (char *)(argv[i] + strlen("--database=")); + } + } else if ((strcmp(argv[i], "-l") == 0) + || (0 == strncmp(argv[i], "--columns", strlen("--columns")))) { + arguments->demo_mode = false; + if (2 == strlen(argv[i])) { + if (argc == i+1) { + errorPrintReqArg(argv[0], "l"); + exit(EXIT_FAILURE); + } else if (!isStringNumber(argv[i+1])) { + errorPrintReqArg2(argv[0], "l"); + exit(EXIT_FAILURE); + } + arguments->num_of_CPR = atoi(argv[++i]); + } else if (strlen("--columns")== strlen(argv[i])) { + errorPrintReqArg3(argv[0], "--columns"); + exit(EXIT_FAILURE); + } else if (0 == strncmp(argv[i], "--columns=", strlen("--columns="))) { + if (isStringNumber((char *)(argv[i] + strlen("--columns=")))) { + arguments->num_of_CPR = atoi((char *)(argv[i]+strlen("--columns"))); + } else { + errorPrintReqArg2(argv[0], "--columns"); exit(EXIT_FAILURE); } } - arguments->num_of_CPR = atoi(argv[++i]); if (arguments->num_of_CPR > MAX_NUM_COLUMNS) { printf("WARNING: max acceptible columns count is %d\n", MAX_NUM_COLUMNS); @@ -1006,36 +1251,46 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { for (int col = arguments->num_of_CPR; col < MAX_NUM_COLUMNS; col++) { arguments->datatype[col] = NULL; } - } else if (strcmp(argv[i], "-b") == 0) { + } else if ((strcmp(argv[i], "-b") == 0) + || (0 == strncmp(argv[i], "--data-type", strlen("--data-type")))) { arguments->demo_mode = false; - if (argc == i+1) { - printHelp(); - errorPrint("%s", "\n\t-b need valid string following!\n"); + + char *dataType; + if (2 == strlen(argv[i])) { + if (argc == i+1) { + errorPrintReqArg(argv[0], "b"); + exit(EXIT_FAILURE); + } + dataType = argv[++i]; + } else if (strlen("--data-type") == strlen(argv[i])) { + errorPrintReqArg3(argv[0], "--data-type"); exit(EXIT_FAILURE); + } else if (0 == strncmp(argv[i], "--data-type=", strlen("--data-type="))) { + dataType = (char *)(argv[i] + strlen("--data-type=")); } - ++i; - if (strstr(argv[i], ",") == NULL) { + + if (strstr(dataType, ",") == NULL) { // only one col - if (strcasecmp(argv[i], "INT") - && strcasecmp(argv[i], "FLOAT") - && strcasecmp(argv[i], "TINYINT") - && strcasecmp(argv[i], "BOOL") - && strcasecmp(argv[i], "SMALLINT") - && strcasecmp(argv[i], "BIGINT") - && strcasecmp(argv[i], "DOUBLE") - && strcasecmp(argv[i], "BINARY") - && strcasecmp(argv[i], "TIMESTAMP") - && strcasecmp(argv[i], "NCHAR")) { + if (strcasecmp(dataType, "INT") + && strcasecmp(dataType, "FLOAT") + && strcasecmp(dataType, "TINYINT") + && strcasecmp(dataType, "BOOL") + && strcasecmp(dataType, "SMALLINT") + && strcasecmp(dataType, "BIGINT") + && strcasecmp(dataType, "DOUBLE") + && strcasecmp(dataType, "BINARY") + && strcasecmp(dataType, "TIMESTAMP") + && strcasecmp(dataType, "NCHAR")) { printHelp(); errorPrint("%s", "-b: Invalid data_type!\n"); exit(EXIT_FAILURE); } - arguments->datatype[0] = argv[i]; + arguments->datatype[0] = dataType; arguments->datatype[1] = NULL; } else { // more than one col int index = 0; - g_dupstr = strdup(argv[i]); + g_dupstr = strdup(dataType); char *running = g_dupstr; char *token = strsep(&running, ","); while(token != NULL) { @@ -1060,75 +1315,155 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { } arguments->datatype[index] = NULL; } - } else if (strcmp(argv[i], "-w") == 0) { - if ((argc == i+1) || - (!isStringNumber(argv[i+1]))) { - printHelp(); - errorPrint("%s", "\n\t-w need a number following!\n"); + } else if ((strcmp(argv[i], "-w") == 0) + || (0 == strncmp(argv[i], "--binwidth", strlen("--binwidth")))) { + if (2 == strlen(argv[i])) { + if (argc == i+1) { + errorPrintReqArg(argv[0], "w"); + exit(EXIT_FAILURE); + } else if (!isStringNumber(argv[i+1])) { + errorPrintReqArg2(argv[0], "w"); + exit(EXIT_FAILURE); + } + arguments->binwidth = atoi(argv[++i]); + } else if (strlen("--binwidth") == strlen(argv[i])) { + errorPrintReqArg3(argv[0], "--binwidth"); exit(EXIT_FAILURE); + } else if (0 == strncmp(argv[i], "--binwidth=", strlen("--binwidth="))) { + if (isStringNumber((char *)(argv[i] + strlen("--binwidth=")))) { + arguments->binwidth = atoi((char *)(argv[i]+strlen("--binwidth="))); + } else { + errorPrintReqArg2(argv[0], "--binwidth"); + exit(EXIT_FAILURE); + } } - arguments->len_of_binary = atoi(argv[++i]); - } else if (strcmp(argv[i], "-m") == 0) { - if ((argc == i+1) || - (isStringNumber(argv[i+1]))) { - printHelp(); - errorPrint("%s", "\n\t-m need a letter-initial string following!\n"); + } else if ((strcmp(argv[i], "-m") == 0) + || (0 == strncmp(argv[i], "--table-prefix", strlen("--table-prefix")))) { + if (2 == strlen(argv[i])) { + if (argc == i+1) { + errorPrintReqArg3(argv[0], "m"); + exit(EXIT_FAILURE); + } + arguments->tb_prefix = argv[++i]; + } else if (strlen("--table-prefix") == strlen(argv[i]) + || (strlen("--table-prefix=") == strlen(argv[i]))) { + errorPrintReqArg3(argv[0], "--table-prefix"); exit(EXIT_FAILURE); + } else if (0 == strncmp(argv[i], "--table-prefix=", strlen("--table-prefix="))) { + arguments->tb_prefix = (char *)(argv[i] + strlen("--table-prefix=")); } - arguments->tb_prefix = argv[++i]; - } else if (strcmp(argv[i], "-N") == 0) { + } else if ((strcmp(argv[i], "-N") == 0) + || (0 == strcmp(argv[i], "--normal-table"))) { arguments->use_metric = false; - } else if (strcmp(argv[i], "-M") == 0) { + } else if ((strcmp(argv[i], "-M") == 0) + || (0 == strcmp(argv[i], "--random"))) { arguments->demo_mode = false; - } else if (strcmp(argv[i], "-x") == 0) { + } else if ((strcmp(argv[i], "-x") == 0) + || (0 == strcmp(argv[i], "--no-insert"))) { arguments->insert_only = false; - } else if (strcmp(argv[i], "-y") == 0) { + } else if ((strcmp(argv[i], "-y") == 0) + || (0 == strcmp(argv[i], "--answer-yes"))) { arguments->answer_yes = true; - } else if (strcmp(argv[i], "-g") == 0) { + } else if ((strcmp(argv[i], "-g") == 0) + || (0 == strcmp(argv[i], "--debug"))) { arguments->debug_print = true; } else if (strcmp(argv[i], "-gg") == 0) { arguments->verbose_print = true; } else if (strcmp(argv[i], "-PP") == 0) { arguments->performance_print = true; - } else if (strcmp(argv[i], "-O") == 0) { - if ((argc == i+1) || - (!isStringNumber(argv[i+1]))) { - printHelp(); - errorPrint("%s", "\n\t-O need a number following!\n"); + } else if ((strcmp(argv[i], "-O") == 0) + || (0 == strncmp(argv[i], "--disorder", strlen("--disorder")))) { + if (2 == strlen(argv[i])) { + if (argc == i+1) { + errorPrintReqArg(argv[0], "O"); + exit(EXIT_FAILURE); + } else if (!isStringNumber(argv[i+1])) { + errorPrintReqArg2(argv[0], "O"); + exit(EXIT_FAILURE); + } + arguments->disorderRatio = atoi(argv[++i]); + } else if (strlen("--disorder") == strlen(argv[i])) { + errorPrintReqArg3(argv[0], "--disorder"); exit(EXIT_FAILURE); + } else if (0 == strncmp(argv[i], "--disorder=", strlen("--disorder="))) { + if (isStringNumber((char *)(argv[i] + strlen("--disorder=")))) { + arguments->disorderRatio = atoi((char *)(argv[i]+strlen("--disorder="))); + } else { + errorPrintReqArg2(argv[0], "--disorder"); + exit(EXIT_FAILURE); + } } - arguments->disorderRatio = atoi(argv[++i]); - if (arguments->disorderRatio > 50) { + errorPrint("Invalid disorder ratio %d, will be set to %d\n", + arguments->disorderRatio, 50); arguments->disorderRatio = 50; } if (arguments->disorderRatio < 0) { + errorPrint("Invalid disorder ratio %d, will be set to %d\n", + arguments->disorderRatio, 0); arguments->disorderRatio = 0; } - - } else if (strcmp(argv[i], "-R") == 0) { - if ((argc == i+1) || - (!isStringNumber(argv[i+1]))) { - printHelp(); - errorPrint("%s", "\n\t-R need a number following!\n"); + } else if ((strcmp(argv[i], "-R") == 0) + || (0 == strncmp(argv[i], "--disorder-range", + strlen("--disorder-range")))) { + if (2 == strlen(argv[i])) { + if (argc == i+1) { + errorPrintReqArg(argv[0], "R"); + exit(EXIT_FAILURE); + } else if (!isStringNumber(argv[i+1])) { + errorPrintReqArg2(argv[0], "R"); + exit(EXIT_FAILURE); + } + arguments->disorderRange = atoi(argv[++i]); + } else if (strlen("--disorder-range") == strlen(argv[i])) { + errorPrintReqArg3(argv[0], "--disorder-range"); exit(EXIT_FAILURE); + } else if (0 == strncmp(argv[i], "--disorder-range=", + strlen("--disorder-range="))) { + if (isStringNumber((char *)(argv[i] + strlen("--disorder-range=")))) { + arguments->disorderRange = + atoi((char *)(argv[i]+strlen("--disorder-rnage="))); + } else { + errorPrintReqArg2(argv[0], "--disorder-range"); + exit(EXIT_FAILURE); + } } - arguments->disorderRange = atoi(argv[++i]); - if (arguments->disorderRange < 0) + if (arguments->disorderRange < 0) { + errorPrint("Invalid disorder range %d, will be set to %d\n", + arguments->disorderRange, 1000); arguments->disorderRange = 1000; - - } else if (strcmp(argv[i], "-a") == 0) { - if ((argc == i+1) || - (!isStringNumber(argv[i+1]))) { - printHelp(); - errorPrint("%s", "\n\t-a need a number following!\n"); - exit(EXIT_FAILURE); } - arguments->replica = atoi(argv[++i]); + } else if ((strcmp(argv[i], "-a") == 0) + || (0 == strncmp(argv[i], "--replica", + strlen("--replica")))) { + if (2 == strlen(argv[i])) { + if (argc == i+1) { + errorPrintReqArg(argv[0], "a"); + exit(EXIT_FAILURE); + } else if (!isStringNumber(argv[i+1])) { + errorPrintReqArg2(argv[0], "a"); + exit(EXIT_FAILURE); + } + arguments->replica = atoi(argv[++i]); + } else if (strlen("--replica") == strlen(argv[i])) { + errorPrintReqArg3(argv[0], "--replica"); + exit(EXIT_FAILURE); + } else if (0 == strncmp(argv[i], "--replica=", + strlen("--replica="))) { + if (isStringNumber((char *)(argv[i] + strlen("--replica=")))) { + arguments->replica = + atoi((char *)(argv[i]+strlen("--replica="))); + } else { + errorPrintReqArg2(argv[0], "--replica"); + exit(EXIT_FAILURE); + } + } if (arguments->replica > 3 || arguments->replica < 1) { + errorPrint("Invalid replica value %d, will be set to %d\n", + arguments->replica, 1); arguments->replica = 1; } } else if (strcmp(argv[i], "-D") == 0) { @@ -1137,16 +1472,32 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { errorPrint("%s", "\n\t-D need a valud (0~3) number following!\n"); exit(EXIT_FAILURE); } - } else if ((strcmp(argv[i], "--version") == 0) || - (strcmp(argv[i], "-V") == 0)) { + } else if ((strcmp(argv[i], "--version") == 0) + || (strcmp(argv[i], "-V") == 0)) { printVersion(); exit(0); - } else if (strcmp(argv[i], "--help") == 0) { + } else if ((strcmp(argv[i], "--help") == 0) + || (strcmp(argv[i], "-?") == 0)) { printHelp(); exit(0); + } else if (strcmp(argv[i], "--usage") == 0) { + printf(" Usage: taosdemo [-f JSONFILE] [-u USER] [-p PASSWORD] [-c CONFIG_DIR]\n\ + [-h HOST] [-P PORT] [-I INTERFACE] [-d DATABASE] [-a REPLICA]\n\ + [-m TABLEPREFIX] [-s SQLFILE] [-N] [-o OUTPUTFILE] [-q QUERYMODE]\n\ + [-b DATATYPES] [-w WIDTH_OF_BINARY] [-l COLUNNS] [-T THREADNUMBER]\n\ + [-i SLEEPTIME] [-S TIME_STEP] [-B INTERLACE_ROWS] [-t TABLES]\n\ + [-n RECORDS] [-M] [-x] [-y] [-O ORDERMODE] [-R RANGE] [-a REPLIcA][-g]\n\ + [--help] [--usage] [--version]\n"); + exit(0); } else { - printHelp(); - errorPrint("%s", "ERROR: wrong options\n"); + // to simulate argp_option output + if (strlen(argv[i]) > 2) { + fprintf(stderr, "%s unrecognized options '%s'\n", argv[0], argv[i]); + } else { + fprintf(stderr, "%s invalid options -- '%s'\n", argv[0], + (char *)((char *)argv[i])+1); + } + fprintf(stderr, "Try `taosdemo --help' or `taosdemo --usage' for more information.\n"); exit(EXIT_FAILURE); } } @@ -1163,7 +1514,7 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { } g_args.num_of_CPR = columnCount; - if (((arguments->debug_print) && (arguments->metaFile == NULL)) + if (((arguments->debug_print) && (NULL != arguments->metaFile)) || arguments->verbose_print) { printf("###################################################################\n"); printf("# meta file: %s\n", arguments->metaFile); @@ -1189,7 +1540,7 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { arguments->num_of_RPR); printf("# Max SQL length: %"PRIu64"\n", arguments->max_sql_len); - printf("# Length of Binary: %d\n", arguments->len_of_binary); + printf("# Length of Binary: %d\n", arguments->binwidth); printf("# Number of Threads: %d\n", arguments->num_of_threads); printf("# Number of Tables: %"PRId64"\n", arguments->num_of_tables); @@ -1645,7 +1996,7 @@ static int printfInsertMeta() { printf("resultFile: \033[33m%s\033[0m\n", g_Dbs.resultFile); printf("thread num of insert data: \033[33m%d\033[0m\n", g_Dbs.threadCount); printf("thread num of create table: \033[33m%d\033[0m\n", - g_Dbs.threadCountByCreateTbl); + g_Dbs.threadCountForCreateTbl); printf("top insert interval: \033[33m%"PRIu64"\033[0m\n", g_args.insert_interval); printf("number of records per req: \033[33m%u\033[0m\n", @@ -1853,7 +2204,7 @@ static void printfInsertMetaToFile(FILE* fp) { fprintf(fp, "configDir: %s\n", configDir); fprintf(fp, "resultFile: %s\n", g_Dbs.resultFile); fprintf(fp, "thread num of insert data: %d\n", g_Dbs.threadCount); - fprintf(fp, "thread num of create table: %d\n", g_Dbs.threadCountByCreateTbl); + fprintf(fp, "thread num of create table: %d\n", g_Dbs.threadCountForCreateTbl); fprintf(fp, "number of records per req: %u\n", g_args.num_of_RPR); fprintf(fp, "max sql length: %"PRIu64"\n", g_args.max_sql_len); fprintf(fp, "database count: %d\n", g_Dbs.dbCount); @@ -3453,7 +3804,7 @@ static void createChildTables() { startMultiThreadCreateChildTable( g_Dbs.db[i].superTbls[j].colsOfCreateChildTable, - g_Dbs.threadCountByCreateTbl, + g_Dbs.threadCountForCreateTbl, startFrom, g_Dbs.db[i].superTbls[j].childTblCount, g_Dbs.db[i].dbName, &(g_Dbs.db[i].superTbls[j])); @@ -3467,7 +3818,7 @@ static void createChildTables() { || (strncasecmp(g_args.datatype[j], "NCHAR", strlen("NCHAR")) == 0)) { snprintf(tblColsBuf + len, TSDB_MAX_BYTES_PER_ROW - len, - ",C%d %s(%d)", j, g_args.datatype[j], g_args.len_of_binary); + ",C%d %s(%d)", j, g_args.datatype[j], g_args.binwidth); } else { snprintf(tblColsBuf + len, TSDB_MAX_BYTES_PER_ROW - len, ",C%d %s", j, g_args.datatype[j]); @@ -3482,7 +3833,7 @@ static void createChildTables() { g_Dbs.db[i].dbName, g_args.num_of_tables, tblColsBuf); startMultiThreadCreateChildTable( tblColsBuf, - g_Dbs.threadCountByCreateTbl, + g_Dbs.threadCountForCreateTbl, 0, g_args.num_of_tables, g_Dbs.db[i].dbName, @@ -3839,9 +4190,9 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { cJSON* threads2 = cJSON_GetObjectItem(root, "thread_count_create_tbl"); if (threads2 && threads2->type == cJSON_Number) { - g_Dbs.threadCountByCreateTbl = threads2->valueint; + g_Dbs.threadCountForCreateTbl = threads2->valueint; } else if (!threads2) { - g_Dbs.threadCountByCreateTbl = 1; + g_Dbs.threadCountForCreateTbl = 1; } else { errorPrint("%s", "failed to read json, threads2 not found\n"); goto PARSE_OVER; @@ -4203,7 +4554,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { if (batchCreateTbl && batchCreateTbl->type == cJSON_Number) { g_Dbs.db[i].superTbls[j].batchCreateTableNum = batchCreateTbl->valueint; } else if (!batchCreateTbl) { - g_Dbs.db[i].superTbls[j].batchCreateTableNum = 1000; + g_Dbs.db[i].superTbls[j].batchCreateTableNum = 10; } else { errorPrint("%s", "failed to read json, batch_create_tbl_num not found\n"); goto PARSE_OVER; @@ -5431,7 +5782,7 @@ static int32_t generateDataTailWithoutStb( int64_t retLen = 0; char **data_type = g_args.datatype; - int lenOfBinary = g_args.len_of_binary; + int lenOfBinary = g_args.binwidth; if (g_args.disorderRatio) { retLen = generateData(data, data_type, @@ -6213,7 +6564,7 @@ static int32_t prepareStmtWithoutStb( if ( -1 == prepareStmtBindArrayByType( bind, data_type[i], - g_args.len_of_binary, + g_args.binwidth, pThreadInfo->time_precision, NULL)) { return -1; @@ -6247,7 +6598,7 @@ static int32_t prepareStbStmtBindTag( char *tagsVal, int32_t timePrec) { - char *bindBuffer = calloc(1, DOUBLE_BUFF_LEN); // g_args.len_of_binary); + char *bindBuffer = calloc(1, DOUBLE_BUFF_LEN); // g_args.binwidth); if (bindBuffer == NULL) { errorPrint2("%s() LN%d, Failed to allocate %d bind buffer\n", __func__, __LINE__, DOUBLE_BUFF_LEN); @@ -6279,7 +6630,7 @@ static int32_t prepareStbStmtBindRand( int64_t startTime, int32_t recSeq, int32_t timePrec) { - char *bindBuffer = calloc(1, DOUBLE_BUFF_LEN); // g_args.len_of_binary); + char *bindBuffer = calloc(1, DOUBLE_BUFF_LEN); // g_args.binwidth); if (bindBuffer == NULL) { errorPrint2("%s() LN%d, Failed to allocate %d bind buffer\n", __func__, __LINE__, DOUBLE_BUFF_LEN); @@ -7390,7 +7741,7 @@ static void startMultiThreadInsertData(int threads, char* db_name, } } } else { - start_time = 1500000000000; + start_time = DEFAULT_START_TIME; } debugPrint("%s() LN%d, start_time= %"PRId64"\n", __func__, __LINE__, start_time); @@ -7945,11 +8296,11 @@ static int insertTestProcess() { if (g_totalChildTables > 0) { fprintf(stderr, "creating %"PRId64" table(s) with %d thread(s)\n\n", - g_totalChildTables, g_Dbs.threadCountByCreateTbl); + g_totalChildTables, g_Dbs.threadCountForCreateTbl); if (g_fpOfInsertResult) { fprintf(g_fpOfInsertResult, "creating %"PRId64" table(s) with %d thread(s)\n\n", - g_totalChildTables, g_Dbs.threadCountByCreateTbl); + g_totalChildTables, g_Dbs.threadCountForCreateTbl); } // create child tables @@ -7960,12 +8311,12 @@ static int insertTestProcess() { fprintf(stderr, "\nSpent %.4f seconds to create %"PRId64" table(s) with %d thread(s), actual %"PRId64" table(s) created\n\n", (end - start)/1000.0, g_totalChildTables, - g_Dbs.threadCountByCreateTbl, g_actualChildTables); + g_Dbs.threadCountForCreateTbl, g_actualChildTables); if (g_fpOfInsertResult) { fprintf(g_fpOfInsertResult, "\nSpent %.4f seconds to create %"PRId64" table(s) with %d thread(s), actual %"PRId64" table(s) created\n\n", (end - start)/1000.0, g_totalChildTables, - g_Dbs.threadCountByCreateTbl, g_actualChildTables); + g_Dbs.threadCountForCreateTbl, g_actualChildTables); } } @@ -8880,7 +9231,7 @@ static void setParaFromArg() { } g_Dbs.threadCount = g_args.num_of_threads; - g_Dbs.threadCountByCreateTbl = g_args.num_of_threads; + g_Dbs.threadCountForCreateTbl = g_args.num_of_threads; g_Dbs.dbCount = 1; g_Dbs.db[0].drop = true; @@ -8912,7 +9263,7 @@ static void setParaFromArg() { tstrncpy(g_Dbs.db[0].superTbls[0].sTblName, "meters", TSDB_TABLE_NAME_LEN); g_Dbs.db[0].superTbls[0].childTblCount = g_args.num_of_tables; g_Dbs.threadCount = g_args.num_of_threads; - g_Dbs.threadCountByCreateTbl = g_args.num_of_threads; + g_Dbs.threadCountForCreateTbl = g_args.num_of_threads; g_Dbs.asyncMode = g_args.async_mode; g_Dbs.db[0].superTbls[0].autoCreateTable = PRE_CREATE_SUBTBL; @@ -8943,7 +9294,7 @@ static void setParaFromArg() { tstrncpy(g_Dbs.db[0].superTbls[0].columns[i].dataType, data_type[i], min(DATATYPE_BUFF_LEN, strlen(data_type[i]) + 1)); - g_Dbs.db[0].superTbls[0].columns[i].dataLen = g_args.len_of_binary; + g_Dbs.db[0].superTbls[0].columns[i].dataLen = g_args.binwidth; g_Dbs.db[0].superTbls[0].columnCount++; } @@ -8965,10 +9316,10 @@ static void setParaFromArg() { tstrncpy(g_Dbs.db[0].superTbls[0].tags[1].dataType, "BINARY", min(DATATYPE_BUFF_LEN, strlen("BINARY") + 1)); - g_Dbs.db[0].superTbls[0].tags[1].dataLen = g_args.len_of_binary; + g_Dbs.db[0].superTbls[0].tags[1].dataLen = g_args.binwidth; g_Dbs.db[0].superTbls[0].tagCount = 2; } else { - g_Dbs.threadCountByCreateTbl = g_args.num_of_threads; + g_Dbs.threadCountForCreateTbl = g_args.num_of_threads; g_Dbs.db[0].superTbls[0].tagCount = 0; } } @@ -9090,7 +9441,7 @@ static void queryResult() { pthread_t read_id; threadInfo *pThreadInfo = calloc(1, sizeof(threadInfo)); assert(pThreadInfo); - pThreadInfo->start_time = 1500000000000; // 2017-07-14 10:40:00.000 + pThreadInfo->start_time = DEFAULT_START_TIME; // 2017-07-14 10:40:00.000 pThreadInfo->start_table_from = 0; //pThreadInfo->do_aggreFunc = g_Dbs.do_aggreFunc; @@ -9144,7 +9495,6 @@ static void testCmdLine() { } g_args.test_mode = INSERT_TEST; - g_totalChildTables = DEFAULT_CHILDTABLES; insertTestProcess(); if (false == g_Dbs.insert_only) diff --git a/src/kit/taosdump/taosdump.c b/src/kit/taosdump/taosdump.c index 30b5d91b10..2dfc069510 100644 --- a/src/kit/taosdump/taosdump.c +++ b/src/kit/taosdump/taosdump.c @@ -225,7 +225,6 @@ static struct argp_option options[] = { {"password", 'p', 0, 0, "User password to connect to server. Default is taosdata.", 0}, #endif {"port", 'P', "PORT", 0, "Port to connect", 0}, - {"cversion", 'v', "CVERION", 0, "client version", 0}, {"mysqlFlag", 'q', "MYSQLFLAG", 0, "mysqlFlag, Default is 0", 0}, // input/output file {"outpath", 'o', "OUTPATH", 0, "Output file path.", 1}, @@ -244,7 +243,7 @@ static struct argp_option options[] = { // dump format options {"schemaonly", 's', 0, 0, "Only dump schema.", 2}, {"without-property", 'N', 0, 0, "Dump schema without properties.", 2}, - {"avro", 'V', 0, 0, "Dump apache avro format data file. By default, dump sql command sequence.", 2}, + {"avro", 'v', 0, 0, "Dump apache avro format data file. By default, dump sql command sequence.", 2}, {"start-time", 'S', "START_TIME", 0, "Start time to dump. Either epoch or ISO8601/RFC3339 format is acceptable. ISO8601 format example: 2017-10-01T00:00:00.000+0800 or 2017-10-0100:00:00:000+0800 or '2017-10-01 00:00:00.000+0800'", 4}, {"end-time", 'E', "END_TIME", 0, "End time to dump. Either epoch or ISO8601/RFC3339 format is acceptable. ISO8601 format example: 2017-10-01T00:00:00.000+0800 or 2017-10-0100:00:00.000+0800 or '2017-10-01 00:00:00.000+0800'", 5}, #if TSDB_SUPPORT_NANOSECOND == 1 @@ -267,7 +266,6 @@ typedef struct arguments { char *user; char password[SHELL_MAX_PASSWORD_LEN]; uint16_t port; - char cversion[12]; uint16_t mysqlFlag; // output file char outpath[MAX_FILE_NAME_LEN]; @@ -338,7 +336,6 @@ struct arguments g_args = { "taosdata", #endif 0, - "", 0, // outpath and inpath "", @@ -395,15 +392,6 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) { case 'q': g_args.mysqlFlag = atoi(arg); break; - case 'v': - if (wordexp(arg, &full_path, 0) != 0) { - errorPrint("Invalid client vesion %s\n", arg); - return -1; - } - tstrncpy(g_args.cversion, full_path.we_wordv[0], 11); - wordfree(&full_path); - break; - // output file path case 'o': if (wordexp(arg, &full_path, 0) != 0) { errorPrint("Invalid path %s\n", arg); @@ -453,7 +441,7 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) { case 'N': g_args.with_property = false; break; - case 'V': + case 'v': g_args.avro = true; break; case 'S': @@ -660,6 +648,9 @@ static void parse_timestamp( } int main(int argc, char *argv[]) { + static char verType[32] = {0}; + sprintf(verType, "version: %s\n", version); + argp_program_version = verType; int ret = 0; /* Parse our arguments; every option seen by parse_opt will be @@ -686,7 +677,6 @@ int main(int argc, char *argv[]) { printf("user: %s\n", g_args.user); printf("password: %s\n", g_args.password); printf("port: %u\n", g_args.port); - printf("cversion: %s\n", g_args.cversion); printf("mysqlFlag: %d\n", g_args.mysqlFlag); printf("outpath: %s\n", g_args.outpath); printf("inpath: %s\n", g_args.inpath); @@ -715,11 +705,6 @@ int main(int argc, char *argv[]) { } } printf("==============================\n"); - - if (g_args.cversion[0] != 0){ - tstrncpy(version, g_args.cversion, 11); - } - if (taosCheckParam(&g_args) < 0) { exit(EXIT_FAILURE); } @@ -737,7 +722,6 @@ int main(int argc, char *argv[]) { fprintf(g_fpOfResult, "user: %s\n", g_args.user); fprintf(g_fpOfResult, "password: %s\n", g_args.password); fprintf(g_fpOfResult, "port: %u\n", g_args.port); - fprintf(g_fpOfResult, "cversion: %s\n", g_args.cversion); fprintf(g_fpOfResult, "mysqlFlag: %d\n", g_args.mysqlFlag); fprintf(g_fpOfResult, "outpath: %s\n", g_args.outpath); fprintf(g_fpOfResult, "inpath: %s\n", g_args.inpath); From 8c28632b8b991e28ef92cec6c73de1c9b572cbc9 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Fri, 27 Aug 2021 09:56:01 +0800 Subject: [PATCH 39/40] [TD-6292]: taosdump --config-dir determine input. (#7611) --- src/kit/taosdump/taosdump.c | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/src/kit/taosdump/taosdump.c b/src/kit/taosdump/taosdump.c index 2dfc069510..cca04f926b 100644 --- a/src/kit/taosdump/taosdump.c +++ b/src/kit/taosdump/taosdump.c @@ -231,9 +231,9 @@ static struct argp_option options[] = { {"inpath", 'i', "INPATH", 0, "Input file path.", 1}, {"resultFile", 'r', "RESULTFILE", 0, "DumpOut/In Result file path and name.", 1}, #ifdef _TD_POWER_ - {"config", 'c', "CONFIG_DIR", 0, "Configure directory. Default is /etc/power/taos.cfg.", 1}, + {"config-dir", 'c', "CONFIG_DIR", 0, "Configure directory. Default is /etc/power/taos.cfg.", 1}, #else - {"config", 'c', "CONFIG_DIR", 0, "Configure directory. Default is /etc/taos/taos.cfg.", 1}, + {"config-dir", 'c', "CONFIG_DIR", 0, "Configure directory. Default is /etc/taos/taos.cfg.", 1}, #endif {"encode", 'e', "ENCODE", 0, "Input file encoding.", 1}, // dump unit options @@ -367,6 +367,15 @@ struct arguments g_args = { false // performance_print }; +static void errorPrintReqArg3(char *program, char *wrong_arg) +{ + fprintf(stderr, + "%s: option '%s' requires an argument\n", + program, wrong_arg); + fprintf(stderr, + "Try `taosdump --help' or `taosdump --usage' for more information.\n"); +} + /* Parse a single option. */ static error_t parse_opt(int key, char *arg, struct argp_state *state) { /* Get the input argument from argp_parse, which we @@ -418,9 +427,13 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) { g_args.resultFile = arg; break; case 'c': + if (0 == strlen(arg)) { + errorPrintReqArg3("taosdump", "-c or --config-dir"); + exit(EXIT_FAILURE); + } if (wordexp(arg, &full_path, 0) != 0) { errorPrint("Invalid path %s\n", arg); - return -1; + exit(EXIT_FAILURE); } tstrncpy(configDir, full_path.we_wordv[0], MAX_FILE_NAME_LEN); wordfree(&full_path); From a8d688a74520ade9f53681e5ac9a4d587a90e163 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Fri, 27 Aug 2021 11:44:33 +0800 Subject: [PATCH 40/40] [TD-6280]: taosdump -P determine input. (#7613) --- src/kit/taosdump/taosdump.c | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/kit/taosdump/taosdump.c b/src/kit/taosdump/taosdump.c index cca04f926b..ae2193a82e 100644 --- a/src/kit/taosdump/taosdump.c +++ b/src/kit/taosdump/taosdump.c @@ -367,6 +367,15 @@ struct arguments g_args = { false // performance_print }; +static void errorPrintReqArg2(char *program, char *wrong_arg) +{ + fprintf(stderr, + "%s: option requires a number argument '-%s'\n", + program, wrong_arg); + fprintf(stderr, + "Try `taosdump --help' or `taosdump --usage' for more information.\n"); +} + static void errorPrintReqArg3(char *program, char *wrong_arg) { fprintf(stderr, @@ -396,6 +405,10 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) { case 'p': break; case 'P': + if (!isStringNumber(arg)) { + errorPrintReqArg2("taosdump", "P"); + exit(EXIT_FAILURE); + } g_args.port = atoi(arg); break; case 'q':