From 438a76c8b2dc4fb6af315e6d8dab260f2d62da48 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 9 May 2022 19:51:52 +0800 Subject: [PATCH 1/5] refactor:add time cost info for schemaless --- source/client/src/clientSml.c | 54 +++++++++++++++++++++++++++++++---- 1 file changed, 49 insertions(+), 5 deletions(-) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 2a0e85092b..a7ce1a30a8 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -97,6 +97,21 @@ typedef struct { char *buf; } SSmlMsgBuf; +typedef struct { + int32_t code; + int32_t lineNum; + + int32_t numOfSTables; + int32_t numOfCTables; + int32_t numOfCreateSTables; + + int64_t parseTime; + int64_t schemaTime; + int64_t insertBindTime; + int64_t insertRpcTime; + int64_t endTime; +} SSmlCostInfo; + typedef struct { uint64_t id; @@ -114,6 +129,7 @@ typedef struct { SRequestObj *pRequest; SQuery *pQuery; + SSmlCostInfo cost; int32_t affectedRows; SSmlMsgBuf msgBuf; SHashObj *dumplicateKey; // for dumplicate key @@ -444,6 +460,7 @@ static int32_t smlModifyDBSchemas(SSmlHandle* info) { uError("SML:0x%"PRIx64" catalogGetSTableMeta failed. super table name %s", info->id, schemaAction.createSTable.sTableName); return code; } + info->cost.numOfCreateSTables++; }else if (code == TSDB_CODE_SUCCESS) { } else { uError("SML:0x%"PRIx64" load table meta error: %s", info->id, tstrerror(code)); @@ -1547,7 +1564,7 @@ static int32_t smlParseLine(SSmlHandle* info, const char* sql) { tinfo->sTableName = elements.measure; tinfo->sTableNameLen = elements.measureLen; smlBuildChildTableName(tinfo); - uDebug("SML:0x%"PRIx64" child table name: %s", info->id, tinfo->childTableName); + //uDebug("SML:0x%"PRIx64" child table name: %s", info->id, tinfo->childTableName); SSmlSTableMeta** tableMeta = taosHashGet(info->superTables, elements.measure, elements.measureLen); if(tableMeta){ // update meta @@ -1699,12 +1716,33 @@ static int32_t smlInsertData(SSmlHandle* info) { } smlBuildOutput(info->exec, info->pVgHash); + info->cost.insertRpcTime = taosGetTimestampUs(); + launchQueryImpl(info->pRequest, info->pQuery, TSDB_CODE_SUCCESS, true); info->affectedRows = taos_affected_rows(info->pRequest); return info->pRequest->code; } +int32_t numOfSTables; +int32_t numOfCTables; +int32_t numOfCreateSTables; + +int64_t parseTime; +int64_t schemaTime; +int64_t insertBindTime; +int64_t insertRpcTime; +int64_t endTime; + +static void printStatisticInfo(SSmlHandle *info){ + uError("SML:0x%"PRIx64" smlInsertLines result, code:%d,lineNum:%d,stable num:%d,ctable num:%d,create stable num:%d \ + parse cost:%lld,schema cost:%lld,bind cost:%lld,rpc cost:%lld,total cost:%lld", info->id, info->cost.code, + info->cost.lineNum, info->cost.numOfSTables, info->cost.numOfCTables, info->cost.numOfCreateSTables, + info->cost.schemaTime-info->cost.parseTime, info->cost.insertBindTime-info->cost.schemaTime, + info->cost.insertRpcTime-info->cost.insertBindTime, info->cost.endTime-info->cost.insertRpcTime, + info->cost.endTime-info->cost.parseTime); +} + static int smlInsertLines(SSmlHandle *info, char* lines[], int numLines) { int32_t code = TSDB_CODE_SUCCESS; @@ -1714,6 +1752,7 @@ static int smlInsertLines(SSmlHandle *info, char* lines[], int numLines) { goto cleanup; } + info->cost.parseTime = taosGetTimestampUs(); for (int32_t i = 0; i < numLines; ++i) { code = smlParseLine(info, lines[i]); if (code != TSDB_CODE_SUCCESS) { @@ -1721,24 +1760,29 @@ static int smlInsertLines(SSmlHandle *info, char* lines[], int numLines) { goto cleanup; } } - uDebug("SML:0x%"PRIx64" smlInsertLines parse success. tables %d", info->id, taosHashGetSize(info->childTables)); - uDebug("SML:0x%"PRIx64" smlInsertLines parse success. super tables %d", info->id, taosHashGetSize(info->superTables)); + info->cost.lineNum = numLines; + info->cost.numOfSTables = taosHashGetSize(info->superTables); + info->cost.numOfCTables = taosHashGetSize(info->childTables); + + info->cost.schemaTime = taosGetTimestampUs(); code = smlModifyDBSchemas(info); if (code != 0) { uError("SML:0x%"PRIx64" smlModifyDBSchemas error : %s", info->id, tstrerror(code)); goto cleanup; } + info->cost.insertBindTime = taosGetTimestampUs(); code = smlInsertData(info); if (code != 0) { uError("SML:0x%"PRIx64" smlInsertData error : %s", info->id, tstrerror(code)); goto cleanup; } - - uDebug("SML:0x%"PRIx64" smlInsertLines finish inserting %d lines.", info->id, numLines); + info->cost.endTime = taosGetTimestampUs(); cleanup: + info->cost.code = code; + printStatisticInfo(info); return code; } From d9f76c7b72527c09706744109c7b9ae6614a439e Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 9 May 2022 20:24:17 +0800 Subject: [PATCH 2/5] refactor:dd a function of generate table name --- include/common/tname.h | 13 ++++++++++ source/client/src/clientSml.c | 48 +++++------------------------------ source/common/src/tname.c | 41 ++++++++++++++++++++++++++++++ 3 files changed, 61 insertions(+), 41 deletions(-) diff --git a/include/common/tname.h b/include/common/tname.h index ffa4f8f253..ae2dc32335 100644 --- a/include/common/tname.h +++ b/include/common/tname.h @@ -17,6 +17,7 @@ #define _TD_COMMON_NAME_H_ #include "tdef.h" +#include "tarray.h" #ifdef __cplusplus extern "C" { @@ -62,6 +63,18 @@ int32_t tNameSetAcctId(SName* dst, int32_t acctId); bool tNameDBNameEqual(SName* left, SName* right); +typedef struct { + // input + SArray *tags; // element is SSmlKV + const char *sTableName; // super table name + uint8_t sTableNameLen; // the length of super table name + + // output + char *childTableName; // must have size of TSDB_TABLE_NAME_LEN; + uint64_t uid; // child table uid, may be useful +} RandTableName; + +void buildChildTableName(RandTableName *rName); #ifdef __cplusplus } diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index a7ce1a30a8..927fec35dc 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -15,6 +15,7 @@ #include "tcommon.h" #include "catalog.h" #include "clientInt.h" +#include "tname.h" //================================================================================================= #define SPACE ' ' @@ -163,19 +164,6 @@ static int32_t smlBuildInvalidDataMsg(SSmlMsgBuf* pBuf, const char *msg1, const return TSDB_CODE_SML_INVALID_DATA; } -static int smlCompareKv(const void* p1, const void* p2) { - SSmlKv* kv1 = *(SSmlKv**)p1; - SSmlKv* kv2 = *(SSmlKv**)p2; - int32_t kvLen1 = kv1->keyLen; - int32_t kvLen2 = kv2->keyLen; - int32_t res = strncasecmp(kv1->key, kv2->key, TMIN(kvLen1, kvLen2)); - if (res != 0) { - return res; - } else { - return kvLen1-kvLen2; - } -} - static void smlBuildChildTableName(SSmlTableInfo *tags) { int32_t size = taosArrayGetSize(tags->tags); ASSERT(size > 0); @@ -943,20 +931,6 @@ static bool smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg) { return false; } -static bool checkDuplicateKey(char *key, SHashObj *pHash, SSmlHandle* info) { - char *val = NULL; - val = taosHashGet(pHash, key, strlen(key)); - if (val) { - uError("SML:0x%"PRIx64" Duplicate key detected:%s", info->id, key); - return true; - } - - uint8_t dummy_val = 0; - taosHashPut(pHash, key, strlen(key), &dummy_val, sizeof(uint8_t)); - - return false; -} - static int32_t smlParseString(const char* sql, SSmlLineInfo *elements, SSmlMsgBuf *msg){ if(!sql) return TSDB_CODE_SML_INVALID_DATA; while (*sql != '\0') { // jump the space at the begining @@ -1563,8 +1537,10 @@ static int32_t smlParseLine(SSmlHandle* info, const char* sql) { tinfo->sTableName = elements.measure; tinfo->sTableNameLen = elements.measureLen; - smlBuildChildTableName(tinfo); - //uDebug("SML:0x%"PRIx64" child table name: %s", info->id, tinfo->childTableName); + RandTableName rName = {.tags=tinfo->tags, .sTableName=tinfo->sTableName, .sTableNameLen=tinfo->sTableNameLen, + .childTableName=tinfo->childTableName}; + buildChildTableName(&rName); + tinfo->uid = rName.uid; SSmlSTableMeta** tableMeta = taosHashGet(info->superTables, elements.measure, elements.measureLen); if(tableMeta){ // update meta @@ -1724,17 +1700,7 @@ static int32_t smlInsertData(SSmlHandle* info) { return info->pRequest->code; } -int32_t numOfSTables; -int32_t numOfCTables; -int32_t numOfCreateSTables; - -int64_t parseTime; -int64_t schemaTime; -int64_t insertBindTime; -int64_t insertRpcTime; -int64_t endTime; - -static void printStatisticInfo(SSmlHandle *info){ +static void smlPrintStatisticInfo(SSmlHandle *info){ uError("SML:0x%"PRIx64" smlInsertLines result, code:%d,lineNum:%d,stable num:%d,ctable num:%d,create stable num:%d \ parse cost:%lld,schema cost:%lld,bind cost:%lld,rpc cost:%lld,total cost:%lld", info->id, info->cost.code, info->cost.lineNum, info->cost.numOfSTables, info->cost.numOfCTables, info->cost.numOfCreateSTables, @@ -1782,7 +1748,7 @@ static int smlInsertLines(SSmlHandle *info, char* lines[], int numLines) { cleanup: info->cost.code = code; - printStatisticInfo(info); + smlPrintStatisticInfo(info); return code; } diff --git a/source/common/src/tname.c b/source/common/src/tname.c index f4755f5b5e..62ba4bfb79 100644 --- a/source/common/src/tname.c +++ b/source/common/src/tname.c @@ -15,6 +15,8 @@ #define _DEFAULT_SOURCE #include "tname.h" +#include "tcommon.h" +#include "tstrbuild.h" #define VALID_NAME_TYPE(x) ((x) == TSDB_DB_NAME_T || (x) == TSDB_TABLE_NAME_T) @@ -294,4 +296,43 @@ int32_t tNameFromString(SName* dst, const char* str, uint32_t type) { return 0; } +static int compareKv(const void* p1, const void* p2) { + SSmlKv* kv1 = *(SSmlKv**)p1; + SSmlKv* kv2 = *(SSmlKv**)p2; + int32_t kvLen1 = kv1->keyLen; + int32_t kvLen2 = kv2->keyLen; + int32_t res = strncasecmp(kv1->key, kv2->key, TMIN(kvLen1, kvLen2)); + if (res != 0) { + return res; + } else { + return kvLen1-kvLen2; + } +} +/* + * use stable name and tags to grearate child table name + */ +void buildChildTableName(RandTableName *rName) { + int32_t size = taosArrayGetSize(rName->tags); + ASSERT(size > 0); + taosArraySort(rName->tags, compareKv); + + SStringBuilder sb = {0}; + taosStringBuilderAppendStringLen(&sb, rName->sTableName, rName->sTableNameLen); + for (int j = 0; j < size; ++j) { + SSmlKv *tagKv = taosArrayGetP(rName->tags, j); + taosStringBuilderAppendStringLen(&sb, tagKv->key, tagKv->keyLen); + taosStringBuilderAppendStringLen(&sb, tagKv->value, tagKv->valueLen); + } + size_t len = 0; + char* keyJoined = taosStringBuilderGetResult(&sb, &len); + T_MD5_CTX context; + tMD5Init(&context); + tMD5Update(&context, (uint8_t *)keyJoined, (uint32_t)len); + tMD5Final(&context); + uint64_t digest1 = *(uint64_t*)(context.digest); + uint64_t digest2 = *(uint64_t*)(context.digest + 8); + snprintf(rName->childTableName, TSDB_TABLE_NAME_LEN, "t_%016"PRIx64"%016"PRIx64, digest1, digest2); + taosStringBuilderDestroy(&sb); + rName->uid = digest1; +} From af60b5743daecd0942e803dd926ee598e8dc004f Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 9 May 2022 20:34:44 +0800 Subject: [PATCH 3/5] refactor:fix error in schemaless --- source/client/src/clientSml.c | 29 +---------------------------- 1 file changed, 1 insertion(+), 28 deletions(-) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 927fec35dc..b251827bd1 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -164,32 +164,6 @@ static int32_t smlBuildInvalidDataMsg(SSmlMsgBuf* pBuf, const char *msg1, const return TSDB_CODE_SML_INVALID_DATA; } -static void smlBuildChildTableName(SSmlTableInfo *tags) { - int32_t size = taosArrayGetSize(tags->tags); - ASSERT(size > 0); - taosArraySort(tags->tags, smlCompareKv); - - SStringBuilder sb = {0}; - taosStringBuilderAppendStringLen(&sb, tags->sTableName, tags->sTableNameLen); - for (int j = 0; j < size; ++j) { - SSmlKv *tagKv = taosArrayGetP(tags->tags, j); - taosStringBuilderAppendStringLen(&sb, tagKv->key, tagKv->keyLen); - taosStringBuilderAppendStringLen(&sb, tagKv->value, tagKv->valueLen); - } - size_t len = 0; - char* keyJoined = taosStringBuilderGetResult(&sb, &len); - T_MD5_CTX context; - tMD5Init(&context); - tMD5Update(&context, (uint8_t *)keyJoined, (uint32_t)len); - tMD5Final(&context); - uint64_t digest1 = *(uint64_t*)(context.digest); - //uint64_t digest2 = *(uint64_t*)(context.digest + 8); - //snprintf(tags->childTableName, TSDB_TABLE_NAME_LEN, "t_%016"PRIx64"%016"PRIx64, digest1, digest2); - snprintf(tags->childTableName, TSDB_TABLE_NAME_LEN, "t_%016"PRIx64, digest1); - taosStringBuilderDestroy(&sb); - tags->uid = digest1; -} - static int32_t smlGenerateSchemaAction(SSchema* pointColField, SHashObj* dbAttrHash, SArray* dbAttrArray, bool isTag, char sTableName[], SSchemaAction* action, bool* actionNeeded, SSmlHandle* info) { // char fieldName[TSDB_COL_NAME_LEN] = {0}; @@ -1702,7 +1676,7 @@ static int32_t smlInsertData(SSmlHandle* info) { static void smlPrintStatisticInfo(SSmlHandle *info){ uError("SML:0x%"PRIx64" smlInsertLines result, code:%d,lineNum:%d,stable num:%d,ctable num:%d,create stable num:%d \ - parse cost:%lld,schema cost:%lld,bind cost:%lld,rpc cost:%lld,total cost:%lld", info->id, info->cost.code, + parse cost:%"PRIx64",schema cost:%"PRIx64",bind cost:%"PRIx64",rpc cost:%"PRIx64",total cost:%"PRIx64"", info->id, info->cost.code, info->cost.lineNum, info->cost.numOfSTables, info->cost.numOfCTables, info->cost.numOfCreateSTables, info->cost.schemaTime-info->cost.parseTime, info->cost.insertBindTime-info->cost.schemaTime, info->cost.insertRpcTime-info->cost.insertBindTime, info->cost.endTime-info->cost.insertRpcTime, @@ -1800,7 +1774,6 @@ TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int pr } smlDestroyInfo(info); -end: return (TAOS_RES*)request; } From f3ed6ba9057c371fb255c255dd54a3bcdf325c25 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 9 May 2022 20:41:00 +0800 Subject: [PATCH 4/5] refactor:fix error in schemaless --- source/client/src/clientSml.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index b251827bd1..6f6d32ee39 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -1676,7 +1676,7 @@ static int32_t smlInsertData(SSmlHandle* info) { static void smlPrintStatisticInfo(SSmlHandle *info){ uError("SML:0x%"PRIx64" smlInsertLines result, code:%d,lineNum:%d,stable num:%d,ctable num:%d,create stable num:%d \ - parse cost:%"PRIx64",schema cost:%"PRIx64",bind cost:%"PRIx64",rpc cost:%"PRIx64",total cost:%"PRIx64"", info->id, info->cost.code, + parse cost:%"PRId64",schema cost:%"PRId64",bind cost:%"PRId64",rpc cost:%"PRId64",total cost:%"PRId64"", info->id, info->cost.code, info->cost.lineNum, info->cost.numOfSTables, info->cost.numOfCTables, info->cost.numOfCreateSTables, info->cost.schemaTime-info->cost.parseTime, info->cost.insertBindTime-info->cost.schemaTime, info->cost.insertRpcTime-info->cost.insertBindTime, info->cost.endTime-info->cost.insertRpcTime, From 771a1e319429b46edd5864dc25cbfe6918cadcd3 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 9 May 2022 20:46:00 +0800 Subject: [PATCH 5/5] refactor:fix error in schemaless --- source/client/src/clientSml.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 6f6d32ee39..15d1500860 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -1571,7 +1571,7 @@ static void smlDestroyInfo(SSmlHandle* info){ static SSmlHandle* smlBuildSmlInfo(TAOS* taos, SRequestObj* request, SMLProtocolType protocol, int8_t precision, bool dataFormat){ int32_t code = TSDB_CODE_SUCCESS; - SSmlHandle* info = taosMemoryMalloc(sizeof(SSmlHandle)); + SSmlHandle* info = taosMemoryCalloc(1, sizeof(SSmlHandle)); if (NULL == info) { return NULL; }