From 240e28ae66f10c6a2ed6cd24f26554f179b704a1 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 27 Apr 2021 20:00:03 +0800 Subject: [PATCH 01/10] [td-225]refactor. --- src/client/inc/tscUtil.h | 2 +- src/client/src/tscParseInsert.c | 10 +++++----- src/client/src/tscServer.c | 13 +++++++++---- src/client/src/tscUtil.c | 12 ++++++------ 4 files changed, 21 insertions(+), 16 deletions(-) diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index 0eda49b1f4..e53a8a7abd 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -297,7 +297,7 @@ STableMeta* createSuperTableMeta(STableMetaMsg* pChild); uint32_t tscGetTableMetaSize(STableMeta* pTableMeta); CChildTableMeta* tscCreateChildMeta(STableMeta* pTableMeta); uint32_t tscGetTableMetaMaxSize(); -int32_t tscCreateTableMetaFromCChildMeta(STableMeta* pChild, const char* name); +int32_t tscCreateTableMetaFromCChildMeta(STableMeta* pChild, const char* name, void* buf); STableMeta* tscTableMetaDup(STableMeta* pTableMeta); diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index 920937928f..6c53074426 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -1317,15 +1317,15 @@ int tsParseSql(SSqlObj *pSql, bool initial) { } // make a backup as tsParseInsertSql may modify the string - char* sqlstr = strdup(pSql->sqlstr); +// char* sqlstr = strdup(pSql->sqlstr); ret = tsParseInsertSql(pSql); - if ((sqlstr == NULL) || (pSql->parseRetry >= 1) || + if (/*(sqlstr == NULL) || */(pSql->parseRetry >= 1) || (ret != TSDB_CODE_TSC_SQL_SYNTAX_ERROR && ret != TSDB_CODE_TSC_INVALID_SQL)) { - free(sqlstr); +// free(sqlstr); } else { tscResetSqlCmd(pCmd, true); - free(pSql->sqlstr); - pSql->sqlstr = sqlstr; +// free(pSql->sqlstr); +// pSql->sqlstr = sqlstr; pSql->parseRetry++; if ((ret = tsInsertInitialCheck(pSql)) == TSDB_CODE_SUCCESS) { ret = tsParseInsertSql(pSql); diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 8889e25177..021aba171b 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -2529,10 +2529,12 @@ static int32_t getTableMetaFromMnode(SSqlObj *pSql, STableMetaInfo *pTableMetaIn int32_t tscGetTableMeta(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo) { assert(tIsValidName(&pTableMetaInfo->name)); - tfree(pTableMetaInfo->pTableMeta); - uint32_t size = tscGetTableMetaMaxSize(); - pTableMetaInfo->pTableMeta = calloc(1, size); + if (pTableMetaInfo->pTableMeta == NULL) { + pTableMetaInfo->pTableMeta = calloc(1, size); + } else { + memset(pTableMetaInfo->pTableMeta, 0, size); + } pTableMetaInfo->pTableMeta->tableType = -1; pTableMetaInfo->pTableMeta->tableInfo.numOfColumns = -1; @@ -2544,10 +2546,13 @@ int32_t tscGetTableMeta(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo) { taosHashGetClone(tscTableMetaInfo, name, len, NULL, pTableMetaInfo->pTableMeta, -1); // TODO resize the tableMeta + char buf[80*1024] = {0}; + assert(size < 80*1024); + STableMeta* pMeta = pTableMetaInfo->pTableMeta; if (pMeta->id.uid > 0) { if (pMeta->tableType == TSDB_CHILD_TABLE) { - int32_t code = tscCreateTableMetaFromCChildMeta(pTableMetaInfo->pTableMeta, name); + int32_t code = tscCreateTableMetaFromCChildMeta(pTableMetaInfo->pTableMeta, name, buf); if (code != TSDB_CODE_SUCCESS) { return getTableMetaFromMnode(pSql, pTableMetaInfo); } diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 4e8bfdf064..c9c6c130d6 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -2915,11 +2915,11 @@ CChildTableMeta* tscCreateChildMeta(STableMeta* pTableMeta) { return cMeta; } -int32_t tscCreateTableMetaFromCChildMeta(STableMeta* pChild, const char* name) { - assert(pChild != NULL); +int32_t tscCreateTableMetaFromCChildMeta(STableMeta* pChild, const char* name, void* buf) { + assert(pChild != NULL && buf != NULL); - uint32_t size = tscGetTableMetaMaxSize(); - STableMeta* p = calloc(1, size); +// uint32_t size = tscGetTableMetaMaxSize(); + STableMeta* p = buf;//calloc(1, size); taosHashGetClone(tscTableMetaInfo, pChild->sTableName, strnlen(pChild->sTableName, TSDB_TABLE_FNAME_LEN), NULL, p, -1); if (p->id.uid > 0) { // tableMeta exists, build child table meta and return @@ -2931,12 +2931,12 @@ int32_t tscCreateTableMetaFromCChildMeta(STableMeta* pChild, const char* name) { memcpy(pChild->schema, p->schema, sizeof(SSchema) *total); - tfree(p); +// tfree(p); return TSDB_CODE_SUCCESS; } else { // super table has been removed, current tableMeta is also expired. remove it here taosHashRemove(tscTableMetaInfo, name, strnlen(name, TSDB_TABLE_FNAME_LEN)); - tfree(p); +// tfree(p); return -1; } } From 1b0e7c70fc635bc358bd36923654a4367b167fc0 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 27 Apr 2021 20:06:33 +0800 Subject: [PATCH 02/10] [td-225] refactor. --- src/client/src/tscServer.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 021aba171b..680a39c7cf 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -2533,7 +2533,8 @@ int32_t tscGetTableMeta(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo) { if (pTableMetaInfo->pTableMeta == NULL) { pTableMetaInfo->pTableMeta = calloc(1, size); } else { - memset(pTableMetaInfo->pTableMeta, 0, size); + uint32_t s = tscGetTableMetaSize(pTableMetaInfo->pTableMeta); + memset(pTableMetaInfo->pTableMeta, 0, s); } pTableMetaInfo->pTableMeta->tableType = -1; From 52b739abe0d1ae394b4d471766caf754f68bb754 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 30 Apr 2021 14:09:34 +0800 Subject: [PATCH 03/10] [TD-3747]enhance performance on win --- src/client/src/tscParseInsert.c | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index 1b8cfdb329..a53dcc05fa 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -705,19 +705,11 @@ static int32_t doParseInsertStatement(SSqlCmd* pCmd, char **str, STableDataBlock } code = TSDB_CODE_TSC_INVALID_SQL; - char *tmpTokenBuf = calloc(1, 16*1024); // used for deleting Escape character: \\, \', \" - if (NULL == tmpTokenBuf) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } + char tmpTokenBuf[16*1024] = {0}; // used for deleting Escape character: \\, \', \" int32_t numOfRows = 0; code = tsParseValues(str, dataBuf, maxNumOfRows, pCmd, &numOfRows, tmpTokenBuf); - free(tmpTokenBuf); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - for (uint32_t i = 0; i < dataBuf->numOfParams; ++i) { SParamInfo *param = dataBuf->params + i; if (param->idx == -1) { From f1098e1cc397b20d648db1b0560712794cc34c25 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 6 May 2021 13:18:45 +0800 Subject: [PATCH 04/10] [TD-3747] --- src/client/inc/tsclient.h | 1 + src/client/src/tscServer.c | 5 +++++ src/client/src/tscUtil.c | 5 ++++- 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index c91943e232..6d94d270a7 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -83,6 +83,7 @@ typedef struct STableMeta { typedef struct STableMetaInfo { STableMeta *pTableMeta; // table meta, cached in client side and acquired by name + uint32_t tableMetaSize; SVgroupsInfo *vgroupList; SArray *pVgroupTables; // SArray diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 5a7bd92994..706055668d 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -2533,6 +2533,11 @@ int32_t tscGetTableMeta(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo) { uint32_t size = tscGetTableMetaMaxSize(); if (pTableMetaInfo->pTableMeta == NULL) { pTableMetaInfo->pTableMeta = calloc(1, size); + } else if (pTableMetaInfo->tableMetaSize < size) { + char *tmp = realloc(pTableMetaInfo->pTableMeta, size); + if (tmp == NULL) { return TSDB_CODE_TSC_OUT_OF_MEMORY;} + pTableMetaInfo->pTableMeta = (STableMeta *)tmp; + pTableMetaInfo->tableMetaSize = size; } else { uint32_t s = tscGetTableMetaSize(pTableMetaInfo->pTableMeta); memset(pTableMetaInfo->pTableMeta, 0, s); diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index a1416284c7..df48486e66 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -693,7 +693,8 @@ int32_t tscCopyDataBlockToPayload(SSqlObj* pSql, STableDataBlocks* pDataBlock) { tfree(pTableMetaInfo->pTableMeta); } - pTableMetaInfo->pTableMeta = tscTableMetaDup(pDataBlock->pTableMeta); + pTableMetaInfo->pTableMeta = tscTableMetaDup(pDataBlock->pTableMeta); + pTableMetaInfo->tableMetaSize = tscGetTableMetaSize(pDataBlock->pTableMeta); } /* @@ -2078,6 +2079,7 @@ STableMetaInfo* tscAddTableMetaInfo(SQueryInfo* pQueryInfo, SName* name, STableM } pTableMetaInfo->pTableMeta = pTableMeta; + pTableMetaInfo->tableMetaSize = tscGetTableMetaSize(pTableMeta); if (vgroupList != NULL) { pTableMetaInfo->vgroupList = tscVgroupInfoClone(vgroupList); @@ -2352,6 +2354,7 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, __async_cb_func_t pFinalInfo = tscAddTableMetaInfo(pNewQueryInfo, &pTableMetaInfo->name, pTableMeta, pTableMetaInfo->vgroupList, pTableMetaInfo->tagColList, pTableMetaInfo->pVgroupTables); + } else { // transfer the ownership of pTableMeta to the newly create sql object. STableMetaInfo* pPrevInfo = tscGetTableMetaInfoFromCmd(&pPrevSql->cmd, pPrevSql->cmd.clauseIndex, 0); if (pPrevInfo->pTableMeta && pPrevInfo->pTableMeta->tableType < 0) { From 014ac9144320f9d253d00589523fc9bc04c9c3ef Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 6 May 2021 13:30:16 +0800 Subject: [PATCH 05/10] [TD-3747] --- src/client/src/tscUtil.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index df48486e66..37875790dd 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -2079,7 +2079,11 @@ STableMetaInfo* tscAddTableMetaInfo(SQueryInfo* pQueryInfo, SName* name, STableM } pTableMetaInfo->pTableMeta = pTableMeta; - pTableMetaInfo->tableMetaSize = tscGetTableMetaSize(pTableMeta); + if (pTableMetaInfo->pTableMeta == NULL) { + pTableMetaInfo->tableMetaSize = 0; + } else { + pTableMetaInfo->tableMetaSize = tscGetTableMetaSize(pTableMeta); + } if (vgroupList != NULL) { pTableMetaInfo->vgroupList = tscVgroupInfoClone(vgroupList); From 8dc64ed8e7f56aeb499d4be37765c8cb76f4d46b Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 6 May 2021 16:55:51 +0800 Subject: [PATCH 06/10] delete some code --- src/client/src/tscParseInsert.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index a53dcc05fa..17f9f27abc 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -1349,11 +1349,8 @@ int tsParseSql(SSqlObj *pSql, bool initial) { ret = tsParseInsertSql(pSql); if (/*(sqlstr == NULL) || */(pSql->parseRetry >= 1) || (ret != TSDB_CODE_TSC_SQL_SYNTAX_ERROR && ret != TSDB_CODE_TSC_INVALID_SQL)) { -// free(sqlstr); } else { tscResetSqlCmd(pCmd, true); -// free(pSql->sqlstr); -// pSql->sqlstr = sqlstr; pSql->parseRetry++; if ((ret = tsInsertInitialCheck(pSql)) == TSDB_CODE_SUCCESS) { ret = tsParseInsertSql(pSql); From 9cd45522ccbff69626613114bd91872565386b49 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 6 May 2021 20:34:11 +0800 Subject: [PATCH 07/10] delete some code --- src/client/src/tscServer.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 706055668d..9e7d778ddd 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -2535,7 +2535,9 @@ int32_t tscGetTableMeta(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo) { pTableMetaInfo->pTableMeta = calloc(1, size); } else if (pTableMetaInfo->tableMetaSize < size) { char *tmp = realloc(pTableMetaInfo->pTableMeta, size); - if (tmp == NULL) { return TSDB_CODE_TSC_OUT_OF_MEMORY;} + if (tmp == NULL) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } pTableMetaInfo->pTableMeta = (STableMeta *)tmp; pTableMetaInfo->tableMetaSize = size; } else { From cf3b1b12fd6a39e5ea567264af62c84c9fd51a86 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 6 May 2021 21:03:54 +0800 Subject: [PATCH 08/10] reset code --- src/client/src/tscParseInsert.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index 17f9f27abc..f17afa0f78 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -1345,12 +1345,15 @@ int tsParseSql(SSqlObj *pSql, bool initial) { } // make a backup as tsParseInsertSql may modify the string -// char* sqlstr = strdup(pSql->sqlstr); + char* sqlstr = strdup(pSql->sqlstr); ret = tsParseInsertSql(pSql); - if (/*(sqlstr == NULL) || */(pSql->parseRetry >= 1) || + if ((sqlstr == NULL) || (pSql->parseRetry >= 1) || (ret != TSDB_CODE_TSC_SQL_SYNTAX_ERROR && ret != TSDB_CODE_TSC_INVALID_SQL)) { + free(sqlstr); } else { tscResetSqlCmd(pCmd, true); + free(pSql->sqlstr); + pSql->sqlstr = sqlstr; pSql->parseRetry++; if ((ret = tsInsertInitialCheck(pSql)) == TSDB_CODE_SUCCESS) { ret = tsParseInsertSql(pSql); From 12cad8c1b1e4d53a1e63fe740cf7634abc6d21ff Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 6 May 2021 22:12:30 +0800 Subject: [PATCH 09/10] [TD-3747]fix crash --- src/client/src/tscServer.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 9e7d778ddd..b691a7ad4a 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -2532,7 +2532,8 @@ int32_t tscGetTableMeta(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo) { uint32_t size = tscGetTableMetaMaxSize(); if (pTableMetaInfo->pTableMeta == NULL) { - pTableMetaInfo->pTableMeta = calloc(1, size); + pTableMetaInfo->pTableMeta = calloc(1, size); + pTableMetaInfo->tableMetaSize = size; } else if (pTableMetaInfo->tableMetaSize < size) { char *tmp = realloc(pTableMetaInfo->pTableMeta, size); if (tmp == NULL) { @@ -2541,8 +2542,8 @@ int32_t tscGetTableMeta(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo) { pTableMetaInfo->pTableMeta = (STableMeta *)tmp; pTableMetaInfo->tableMetaSize = size; } else { - uint32_t s = tscGetTableMetaSize(pTableMetaInfo->pTableMeta); - memset(pTableMetaInfo->pTableMeta, 0, s); + //uint32_t s = tscGetTableMetaSize(pTableMetaInfo->pTableMeta); + memset(pTableMetaInfo->pTableMeta, 0, size); } pTableMetaInfo->pTableMeta->tableType = -1; From 2242be054929222a7cbe7032a5d9417e5e9f299a Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 7 May 2021 00:17:30 +0800 Subject: [PATCH 10/10] [TD-3747] fix stream case failed --- src/client/src/tscServer.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index b691a7ad4a..d9405f300a 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -2544,6 +2544,7 @@ int32_t tscGetTableMeta(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo) { } else { //uint32_t s = tscGetTableMetaSize(pTableMetaInfo->pTableMeta); memset(pTableMetaInfo->pTableMeta, 0, size); + pTableMetaInfo->tableMetaSize = size; } pTableMetaInfo->pTableMeta->tableType = -1;