From 8532227a36930a1286269a4d8486a26d5ee213cb Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Thu, 19 Oct 2023 11:30:44 +0800 Subject: [PATCH 01/45] fix: first schema clause top - target is supertable --- include/libs/nodes/querynodes.h | 1 + source/libs/parser/src/parInsertSql.c | 69 +++++++++++++++++---------- 2 files changed, 46 insertions(+), 24 deletions(-) diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index 98f0a795f7..06e90683ff 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -439,6 +439,7 @@ typedef struct SVnodeModifyOpStmt { FFreeVgourpBlockArray freeArrayFunc; bool usingTableProcessing; bool fileProcessing; + bool stbSyntax; } SVnodeModifyOpStmt; typedef struct SExplainOptions { diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index 0e007e127e..62aef704de 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -989,12 +989,12 @@ static int32_t checkAuthForStable(SParseContext* pCxt, SName* pTbName, bool* pMi return checkAuth(pCxt, pTbName, pMissCache, pTagCond); } -static int32_t getTableMeta(SInsertParseContext* pCxt, SName* pTbName, bool isStb, STableMeta** pTableMeta, - bool* pMissCache) { +static int32_t getTableMeta(SInsertParseContext* pCxt, SName* pTbName, STableMeta** pTableMeta, + bool* pMissCache, bool bUsingTable) { SParseContext* pComCxt = pCxt->pComCxt; int32_t code = TSDB_CODE_SUCCESS; if (pComCxt->async) { - if (isStb) { + if (bUsingTable) { code = catalogGetCachedSTableMeta(pComCxt->pCatalog, pTbName, pTableMeta); } else { code = catalogGetCachedTableMeta(pComCxt->pCatalog, pTbName, pTableMeta); @@ -1004,7 +1004,7 @@ static int32_t getTableMeta(SInsertParseContext* pCxt, SName* pTbName, bool isSt .requestId = pComCxt->requestId, .requestObjRefId = pComCxt->requestRid, .mgmtEps = pComCxt->mgmtEpSet}; - if (isStb) { + if (bUsingTable) { code = catalogGetSTableMeta(pComCxt->pCatalog, &conn, pTbName, pTableMeta); } else { code = catalogGetTableMeta(pComCxt->pCatalog, &conn, pTbName, pTableMeta); @@ -1013,10 +1013,8 @@ static int32_t getTableMeta(SInsertParseContext* pCxt, SName* pTbName, bool isSt if (TSDB_CODE_SUCCESS == code) { if (NULL == *pTableMeta) { *pMissCache = true; - } else if (isStb && TSDB_SUPER_TABLE != (*pTableMeta)->tableType) { + } else if (bUsingTable && TSDB_SUPER_TABLE != (*pTableMeta)->tableType) { code = buildInvalidOperationMsg(&pCxt->msg, "create table only from super table is allowed"); - } else if (!isStb && TSDB_SUPER_TABLE == (*pTableMeta)->tableType) { - code = buildInvalidOperationMsg(&pCxt->msg, "insert data into super table is not supported"); } } return code; @@ -1047,7 +1045,7 @@ static int32_t getTableVgroup(SParseContext* pCxt, SVnodeModifyOpStmt* pStmt, bo return code; } -static int32_t getTableMetaAndVgroupImpl(SParseContext* pCxt, SVnodeModifyOpStmt* pStmt, bool* pMissCache) { +static int32_t getTargetTableMetaAndVgroupImpl(SParseContext* pCxt, SVnodeModifyOpStmt* pStmt, bool* pMissCache) { SVgroupInfo vg; int32_t code = catalogGetCachedTableVgMeta(pCxt->pCatalog, &pStmt->targetTableName, &vg, &pStmt->pTableMeta); if (TSDB_CODE_SUCCESS == code) { @@ -1059,15 +1057,34 @@ static int32_t getTableMetaAndVgroupImpl(SParseContext* pCxt, SVnodeModifyOpStmt return code; } -static int32_t getTableMetaAndVgroup(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, bool* pMissCache) { +static int32_t getTargetTableMetaAndVgroup(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, bool* pMissCache) { SParseContext* pComCxt = pCxt->pComCxt; int32_t code = TSDB_CODE_SUCCESS; if (pComCxt->async) { - code = getTableMetaAndVgroupImpl(pComCxt, pStmt, pMissCache); + { + SVgroupInfo vg; + code = catalogGetCachedTableVgMeta(pComCxt->pCatalog, &pStmt->targetTableName, &vg, &pStmt->pTableMeta); + if (TSDB_CODE_SUCCESS == code) { + if (NULL != pStmt->pTableMeta) { + if (pStmt->pTableMeta->tableType == TSDB_SUPER_TABLE) { + pStmt->stbSyntax = true; + } else { + code = taosHashPut(pStmt->pVgroupsHashObj, (const char*)&vg.vgId, sizeof(vg.vgId), (char*)&vg, sizeof(vg)); + } + } + *pMissCache = (NULL == pStmt->pTableMeta); + } + } } else { - code = getTableMeta(pCxt, &pStmt->targetTableName, false, &pStmt->pTableMeta, pMissCache); + bool bUsingTable = false; + code = getTableMeta(pCxt, &pStmt->targetTableName, &pStmt->pTableMeta, pMissCache, bUsingTable); if (TSDB_CODE_SUCCESS == code && !pCxt->missCache) { - code = getTableVgroup(pCxt->pComCxt, pStmt, false, &pCxt->missCache); + if (TSDB_SUPER_TABLE == pStmt->pTableMeta->tableType) { + pStmt->stbSyntax = true; + } + if (!pStmt->stbSyntax) { + code = getTableVgroup(pCxt->pComCxt, pStmt, false, &pCxt->missCache); + } } } return code; @@ -1093,7 +1110,7 @@ static int32_t getTargetTableSchema(SInsertParseContext* pCxt, SVnodeModifyOpStm int32_t code = checkAuthForTable(pCxt->pComCxt, &pStmt->targetTableName, &pCxt->missCache, &pCxt->needTableTagVal); if (TSDB_CODE_SUCCESS == code && !pCxt->missCache) { - code = getTableMetaAndVgroup(pCxt, pStmt, &pCxt->missCache); + code = getTargetTableMetaAndVgroup(pCxt, pStmt, &pCxt->missCache); } if (TSDB_CODE_SUCCESS == code && !pCxt->pComCxt->async) { code = collectUseDatabase(&pStmt->targetTableName, pStmt->pDbFNameHashObj); @@ -1116,7 +1133,8 @@ static int32_t getUsingTableSchema(SInsertParseContext* pCxt, SVnodeModifyOpStmt int32_t code = checkAuthForStable(pCxt->pComCxt, &pStmt->usingTableName, &pCxt->missCache, &pStmt->pTagCond); if (TSDB_CODE_SUCCESS == code && !pCxt->missCache) { - code = getTableMeta(pCxt, &pStmt->usingTableName, true, &pStmt->pTableMeta, &pCxt->missCache); + bool bUsingTable = true; + code = getTableMeta(pCxt, &pStmt->usingTableName, &pStmt->pTableMeta, &pCxt->missCache, bUsingTable); } if (TSDB_CODE_SUCCESS == code && !pCxt->missCache) { code = getTableVgroup(pCxt->pComCxt, pStmt, true, &pCxt->missCache); @@ -2024,13 +2042,10 @@ static int32_t checkSubtablePrivilegeForTable(const SArray* pTables, SVnodeModif return code; } -static int32_t getTableSchemaFromMetaData(SInsertParseContext* pCxt, const SMetaData* pMetaData, +static int32_t processTableSchemaFromMetaData(SInsertParseContext* pCxt, const SMetaData* pMetaData, SVnodeModifyOpStmt* pStmt, bool isStb) { - int32_t code = checkAuthFromMetaData(pMetaData->pUser, &pStmt->pTagCond); - if (TSDB_CODE_SUCCESS == code) { - code = getTableMetaFromMetaData(pMetaData->pTableMeta, &pStmt->pTableMeta); - } - if (TSDB_CODE_SUCCESS == code && !isStb && TSDB_SUPER_TABLE == pStmt->pTableMeta->tableType) { + int32_t code = TSDB_CODE_SUCCESS; + if (!isStb && TSDB_SUPER_TABLE == pStmt->pTableMeta->tableType) { code = buildInvalidOperationMsg(&pCxt->msg, "insert data into super table is not supported"); } if (TSDB_CODE_SUCCESS == code && isStb) { @@ -2068,11 +2083,17 @@ static void clearCatalogReq(SCatalogReq* pCatalogReq) { static int32_t setVnodeModifOpStmt(SInsertParseContext* pCxt, SCatalogReq* pCatalogReq, const SMetaData* pMetaData, SVnodeModifyOpStmt* pStmt) { clearCatalogReq(pCatalogReq); - - if (pStmt->usingTableProcessing) { - return getTableSchemaFromMetaData(pCxt, pMetaData, pStmt, true); + int32_t code = checkAuthFromMetaData(pMetaData->pUser, &pStmt->pTagCond); + if (code == TSDB_CODE_SUCCESS) { + code = getTableMetaFromMetaData(pMetaData->pTableMeta, &pStmt->pTableMeta); + if (code == TSDB_CODE_SUCCESS && pStmt->pTableMeta->tableType == TSDB_SUPER_TABLE && !pStmt->usingTableProcessing) { + pStmt->stbSyntax = true; + } } - return getTableSchemaFromMetaData(pCxt, pMetaData, pStmt, false); + if (pStmt->usingTableProcessing || pStmt->stbSyntax) { + return processTableSchemaFromMetaData(pCxt, pMetaData, pStmt, true); + } + return processTableSchemaFromMetaData(pCxt, pMetaData, pStmt, false); } static int32_t resetVnodeModifOpStmt(SInsertParseContext* pCxt, SQuery* pQuery) { From 334feb199a8322003f010d5f457da1e7df98f210 Mon Sep 17 00:00:00 2001 From: slzhou Date: Thu, 19 Oct 2023 13:22:28 +0800 Subject: [PATCH 02/45] call processTableSchemaFromMetaData when success --- source/libs/parser/src/parInsertSql.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index 62aef704de..81cfb02ada 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -2089,11 +2089,12 @@ static int32_t setVnodeModifOpStmt(SInsertParseContext* pCxt, SCatalogReq* pCata if (code == TSDB_CODE_SUCCESS && pStmt->pTableMeta->tableType == TSDB_SUPER_TABLE && !pStmt->usingTableProcessing) { pStmt->stbSyntax = true; } + if (code == TSDB_CODE_SUCCESS) { + if (pStmt->usingTableProcessing || pStmt->stbSyntax) { + return processTableSchemaFromMetaData(pCxt, pMetaData, pStmt, true); + } + return processTableSchemaFromMetaData(pCxt, pMetaData, pStmt, false); } - if (pStmt->usingTableProcessing || pStmt->stbSyntax) { - return processTableSchemaFromMetaData(pCxt, pMetaData, pStmt, true); - } - return processTableSchemaFromMetaData(pCxt, pMetaData, pStmt, false); } static int32_t resetVnodeModifOpStmt(SInsertParseContext* pCxt, SQuery* pQuery) { From cb2311fde68b7d01a3943865b1f34027514077ef Mon Sep 17 00:00:00 2001 From: slzhou Date: Thu, 19 Oct 2023 13:28:48 +0800 Subject: [PATCH 03/45] fix minior error --- source/libs/parser/src/parInsertSql.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index 81cfb02ada..c395434ea2 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -2086,15 +2086,17 @@ static int32_t setVnodeModifOpStmt(SInsertParseContext* pCxt, SCatalogReq* pCata int32_t code = checkAuthFromMetaData(pMetaData->pUser, &pStmt->pTagCond); if (code == TSDB_CODE_SUCCESS) { code = getTableMetaFromMetaData(pMetaData->pTableMeta, &pStmt->pTableMeta); - if (code == TSDB_CODE_SUCCESS && pStmt->pTableMeta->tableType == TSDB_SUPER_TABLE && !pStmt->usingTableProcessing) { + } + if (code == TSDB_CODE_SUCCESS) { + if (pStmt->pTableMeta->tableType == TSDB_SUPER_TABLE && !pStmt->usingTableProcessing) { pStmt->stbSyntax = true; } - if (code == TSDB_CODE_SUCCESS) { if (pStmt->usingTableProcessing || pStmt->stbSyntax) { return processTableSchemaFromMetaData(pCxt, pMetaData, pStmt, true); } return processTableSchemaFromMetaData(pCxt, pMetaData, pStmt, false); } + return code; } static int32_t resetVnodeModifOpStmt(SInsertParseContext* pCxt, SQuery* pQuery) { From 332268fd4b05490e4d3a163a6027eb658cecc806 Mon Sep 17 00:00:00 2001 From: slzhou Date: Thu, 19 Oct 2023 15:55:24 +0800 Subject: [PATCH 04/45] set super table name for stb syntax and rename setRefreshMate to setRefreshMeta --- include/libs/nodes/querynodes.h | 3 +++ source/libs/parser/src/parInsertSql.c | 15 ++++++++++++--- source/libs/parser/src/parTranslater.c | 10 +++++----- 3 files changed, 20 insertions(+), 8 deletions(-) diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index 06e90683ff..b2a30d525b 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -439,7 +439,10 @@ typedef struct SVnodeModifyOpStmt { FFreeVgourpBlockArray freeArrayFunc; bool usingTableProcessing; bool fileProcessing; + bool stbSyntax; + SName superTableName; + SName childTableName; } SVnodeModifyOpStmt; typedef struct SExplainOptions { diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index c395434ea2..f03620b25f 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -1068,6 +1068,7 @@ static int32_t getTargetTableMetaAndVgroup(SInsertParseContext* pCxt, SVnodeModi if (NULL != pStmt->pTableMeta) { if (pStmt->pTableMeta->tableType == TSDB_SUPER_TABLE) { pStmt->stbSyntax = true; + tNameAssign(&pStmt->superTableName, &pStmt->targetTableName); } else { code = taosHashPut(pStmt->pVgroupsHashObj, (const char*)&vg.vgId, sizeof(vg.vgId), (char*)&vg, sizeof(vg)); } @@ -1081,6 +1082,7 @@ static int32_t getTargetTableMetaAndVgroup(SInsertParseContext* pCxt, SVnodeModi if (TSDB_CODE_SUCCESS == code && !pCxt->missCache) { if (TSDB_SUPER_TABLE == pStmt->pTableMeta->tableType) { pStmt->stbSyntax = true; + tNameAssign(&pStmt->superTableName, &pStmt->targetTableName); } if (!pStmt->stbSyntax) { code = getTableVgroup(pCxt->pComCxt, pStmt, false, &pCxt->missCache); @@ -1795,6 +1797,7 @@ static void resetEnvPreTable(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStm pStmt->usingTableProcessing = false; pStmt->fileProcessing = false; pStmt->usingTableName.type = 0; + pStmt->stbSyntax = false; } // input pStmt->pSql: [(field1_name, ...)] [ USING ... ] VALUES ... | FILE ... @@ -1802,8 +1805,14 @@ static int32_t parseInsertTableClause(SInsertParseContext* pCxt, SVnodeModifyOpS resetEnvPreTable(pCxt, pStmt); int32_t code = parseSchemaClauseTop(pCxt, pStmt, pTbName); if (TSDB_CODE_SUCCESS == code && !pCxt->missCache) { - code = parseInsertTableClauseBottom(pCxt, pStmt); + if (!pStmt->stbSyntax) { + code = parseInsertTableClauseBottom(pCxt, pStmt); + } else { + //code = parseInsertStbClauseBottom(pCxt, pStmt); + code = TSDB_CODE_SUCCESS; + } } + return code; } @@ -2142,7 +2151,7 @@ static int32_t initInsertQuery(SInsertParseContext* pCxt, SCatalogReq* pCatalogR return TSDB_CODE_SUCCESS; } -static int32_t setRefreshMate(SQuery* pQuery) { +static int32_t setRefreshMeta(SQuery* pQuery) { SVnodeModifyOpStmt* pStmt = (SVnodeModifyOpStmt*)pQuery->pRoot; if (taosHashGetSize(pStmt->pTableNameHashObj) > 0) { @@ -2315,7 +2324,7 @@ int32_t parseInsertSql(SParseContext* pCxt, SQuery** pQuery, SCatalogReq* pCatal } if ((TSDB_CODE_SUCCESS == code || NEED_CLIENT_HANDLE_ERROR(code)) && QUERY_EXEC_STAGE_SCHEDULE == (*pQuery)->execStage) { - code = setRefreshMate(*pQuery); + code = setRefreshMeta(*pQuery); } insDestroyBoundColInfo(&context.tags); return code; diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 1c31993a92..3fa9432099 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -261,7 +261,7 @@ static int32_t createSimpleSelectStmtFromProjList(const char* pDb, const char* SSelectStmt** pStmt); static int32_t createLastTsSelectStmt(char* pDb, char* pTable, STableMeta* pMeta, SNode** pQuery); static int32_t setQuery(STranslateContext* pCxt, SQuery* pQuery); -static int32_t setRefreshMate(STranslateContext* pCxt, SQuery* pQuery); +static int32_t setRefreshMeta(STranslateContext* pCxt, SQuery* pQuery); static bool afterGroupBy(ESqlClause clause) { return clause > SQL_CLAUSE_GROUP_BY; } @@ -6147,7 +6147,7 @@ int32_t translatePostCreateSmaIndex(SParseContext* pParseCxt, SQuery* pQuery, vo if (TSDB_CODE_SUCCESS == code) { code = setQuery(&pCxt, pQuery); } - setRefreshMate(&pCxt, pQuery); + setRefreshMeta(&pCxt, pQuery); destroyTranslateContext(&pCxt); tFreeSMCreateSmaReq(pStmt->pReq); taosMemoryFreeClear(pStmt->pReq); @@ -7366,7 +7366,7 @@ int32_t translatePostCreateStream(SParseContext* pParseCxt, SQuery* pQuery, void if (TSDB_CODE_SUCCESS == code) { code = setQuery(&cxt, pQuery); } - setRefreshMate(&cxt, pQuery); + setRefreshMeta(&cxt, pQuery); destroyTranslateContext(&cxt); tFreeSCMCreateStreamReq(pStmt->pReq); @@ -9590,7 +9590,7 @@ static int32_t toMsgType(ENodeType type) { return TDMT_VND_CREATE_TABLE; } -static int32_t setRefreshMate(STranslateContext* pCxt, SQuery* pQuery) { +static int32_t setRefreshMeta(STranslateContext* pCxt, SQuery* pQuery) { if (NULL != pCxt->pDbs) { taosArrayDestroy(pQuery->pDbList); pQuery->pDbList = taosArrayInit(taosHashGetSize(pCxt->pDbs), TSDB_DB_FNAME_LEN); @@ -9724,7 +9724,7 @@ int32_t translate(SParseContext* pParseCxt, SQuery* pQuery, SParseMetaCache* pMe if (TSDB_CODE_SUCCESS == code) { code = setQuery(&cxt, pQuery); } - setRefreshMate(&cxt, pQuery); + setRefreshMeta(&cxt, pQuery); destroyTranslateContext(&cxt); return code; } From ce6be3d3e7d795fcd9ad712d0bfd84c6d7f50b80 Mon Sep 17 00:00:00 2001 From: slzhou Date: Fri, 20 Oct 2023 14:51:44 +0800 Subject: [PATCH 05/45] fix: refactor checkAuth for target table --- source/libs/parser/src/parInsertSql.c | 43 ++++++++------------------- 1 file changed, 13 insertions(+), 30 deletions(-) diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index f03620b25f..120fe13636 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -974,21 +974,6 @@ static int32_t checkAuth(SParseContext* pCxt, SName* pTbName, bool* pMissCache, return code; } -static int32_t checkAuthForTable(SParseContext* pCxt, SName* pTbName, bool* pMissCache, bool* pNeedTableTagVal) { - SNode* pTagCond = NULL; - int32_t code = checkAuth(pCxt, pTbName, pMissCache, &pTagCond); - if (TSDB_CODE_SUCCESS == code) { - *pNeedTableTagVal = ((*pMissCache) || (NULL != pTagCond)); - *pMissCache = (NULL != pTagCond); - } - nodesDestroyNode(pTagCond); - return code; -} - -static int32_t checkAuthForStable(SParseContext* pCxt, SName* pTbName, bool* pMissCache, SNode** pTagCond) { - return checkAuth(pCxt, pTbName, pMissCache, pTagCond); -} - static int32_t getTableMeta(SInsertParseContext* pCxt, SName* pTbName, STableMeta** pTableMeta, bool* pMissCache, bool bUsingTable) { SParseContext* pComCxt = pCxt->pComCxt; @@ -1045,18 +1030,6 @@ static int32_t getTableVgroup(SParseContext* pCxt, SVnodeModifyOpStmt* pStmt, bo return code; } -static int32_t getTargetTableMetaAndVgroupImpl(SParseContext* pCxt, SVnodeModifyOpStmt* pStmt, bool* pMissCache) { - SVgroupInfo vg; - int32_t code = catalogGetCachedTableVgMeta(pCxt->pCatalog, &pStmt->targetTableName, &vg, &pStmt->pTableMeta); - if (TSDB_CODE_SUCCESS == code) { - if (NULL != pStmt->pTableMeta) { - code = taosHashPut(pStmt->pVgroupsHashObj, (const char*)&vg.vgId, sizeof(vg.vgId), (char*)&vg, sizeof(vg)); - } - *pMissCache = (NULL == pStmt->pTableMeta); - } - return code; -} - static int32_t getTargetTableMetaAndVgroup(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, bool* pMissCache) { SParseContext* pComCxt = pCxt->pComCxt; int32_t code = TSDB_CODE_SUCCESS; @@ -1109,11 +1082,21 @@ static int32_t getTargetTableSchema(SInsertParseContext* pCxt, SVnodeModifyOpStm pCxt->missCache = true; return TSDB_CODE_SUCCESS; } - - int32_t code = checkAuthForTable(pCxt->pComCxt, &pStmt->targetTableName, &pCxt->missCache, &pCxt->needTableTagVal); + SNode* pTagCond = NULL; + int32_t code = checkAuth(pCxt->pComCxt, &pStmt->targetTableName, &pCxt->missCache, &pTagCond); if (TSDB_CODE_SUCCESS == code && !pCxt->missCache) { code = getTargetTableMetaAndVgroup(pCxt, pStmt, &pCxt->missCache); } + if (TSDB_CODE_SUCCESS == code && !pCxt->missCache) { + if (TSDB_SUPER_TABLE != pStmt->pTableMeta->tableType) { + pCxt->needTableTagVal = (NULL != pTagCond); + pCxt->missCache = (NULL != pTagCond); + } else { + pStmt->pTagCond = nodesCloneNode(pTagCond); + } + } + nodesDestroyNode(pTagCond); + if (TSDB_CODE_SUCCESS == code && !pCxt->pComCxt->async) { code = collectUseDatabase(&pStmt->targetTableName, pStmt->pDbFNameHashObj); if (TSDB_CODE_SUCCESS == code) { @@ -1133,7 +1116,7 @@ static int32_t getUsingTableSchema(SInsertParseContext* pCxt, SVnodeModifyOpStmt return TSDB_CODE_SUCCESS; } - int32_t code = checkAuthForStable(pCxt->pComCxt, &pStmt->usingTableName, &pCxt->missCache, &pStmt->pTagCond); + int32_t code = checkAuth(pCxt->pComCxt, &pStmt->usingTableName, &pCxt->missCache, &pStmt->pTagCond); if (TSDB_CODE_SUCCESS == code && !pCxt->missCache) { bool bUsingTable = true; code = getTableMeta(pCxt, &pStmt->usingTableName, &pStmt->pTableMeta, &pCxt->missCache, bUsingTable); From 3fe5c447e78faaccfa76dd6346033a631dc0b465 Mon Sep 17 00:00:00 2001 From: slzhou Date: Fri, 20 Oct 2023 17:16:45 +0800 Subject: [PATCH 06/45] enhance: begin parse cols --- include/libs/nodes/querynodes.h | 2 +- source/libs/parser/src/parInsertSql.c | 22 ++++++++++++++++++++-- source/libs/parser/src/parInsertUtil.c | 4 +++- 3 files changed, 24 insertions(+), 4 deletions(-) diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index b2a30d525b..bfd23d1965 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -439,7 +439,7 @@ typedef struct SVnodeModifyOpStmt { FFreeVgourpBlockArray freeArrayFunc; bool usingTableProcessing; bool fileProcessing; - + bool stbSyntax; SName superTableName; SName childTableName; diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index 120fe13636..1351a48a52 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -1783,6 +1783,25 @@ static void resetEnvPreTable(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStm pStmt->stbSyntax = false; } +static int32_t parseStbBoundColumnsClause(SInsertParseContext* pCxt, const char* pBoundCols, + STableMeta* pTableMeta, SBoundColInfo* pBoundColsInfo) { + return TSDB_CODE_SUCCESS; +} + +static int32_t parseInsertStbClauseBottom(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) { + int32_t code = TSDB_CODE_SUCCESS; + STableComInfo tblInfo = getTableInfo(pStmt->pTableMeta); + SBoundColInfo stbBoundColInfo; + insInitBoundColsInfo(tblInfo.numOfColumns + tblInfo.numOfTags + 1, &stbBoundColInfo); + if (!pStmt->pBoundCols) { + return buildSyntaxErrMsg(&pCxt->msg, "(...tbname...) bounded cols is expected", pStmt->pSql); + } + SToken token; + int32_t index = 0; + parseStbBoundColumnsClause(pCxt, pStmt->pBoundCols, pStmt->pTableMeta, &stbBoundColInfo); + return code; +} + // input pStmt->pSql: [(field1_name, ...)] [ USING ... ] VALUES ... | FILE ... static int32_t parseInsertTableClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, SToken* pTbName) { resetEnvPreTable(pCxt, pStmt); @@ -1791,8 +1810,7 @@ static int32_t parseInsertTableClause(SInsertParseContext* pCxt, SVnodeModifyOpS if (!pStmt->stbSyntax) { code = parseInsertTableClauseBottom(pCxt, pStmt); } else { - //code = parseInsertStbClauseBottom(pCxt, pStmt); - code = TSDB_CODE_SUCCESS; + code = parseInsertStbClauseBottom(pCxt, pStmt); } } diff --git a/source/libs/parser/src/parInsertUtil.c b/source/libs/parser/src/parInsertUtil.c index 3efb5dafcb..1c0b28f883 100644 --- a/source/libs/parser/src/parInsertUtil.c +++ b/source/libs/parser/src/parInsertUtil.c @@ -177,7 +177,9 @@ int32_t insInitBoundColsInfo(int32_t numOfBound, SBoundColInfo* pInfo) { if (NULL == pInfo->pColIndex) { return TSDB_CODE_OUT_OF_MEMORY; } - initBoundCols(numOfBound, pInfo->pColIndex); + for (int32_t i = 0; i < numOfBound; ++i) { + pInfo->pColIndex[i] = i; + } return TSDB_CODE_SUCCESS; } From 438a775c78ffee221c50592d20996279a0b0ba32 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Sat, 21 Oct 2023 14:02:03 +0800 Subject: [PATCH 07/45] fix: parse insert stb columns when meta is ready or ready after get meta --- source/libs/parser/src/parInsertSql.c | 66 ++++++++++++++------------- 1 file changed, 34 insertions(+), 32 deletions(-) diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index 1351a48a52..24e1ab226c 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -1755,33 +1755,6 @@ static int32_t parseDataClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pS return buildSyntaxErrMsg(&pCxt->msg, "keyword VALUES or FILE is expected", token.z); } -// input pStmt->pSql: -// 1. [(tag1_name, ...)] ... -// 2. VALUES ... | FILE ... -static int32_t parseInsertTableClauseBottom(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) { - STableDataCxt* pTableCxt = NULL; - int32_t code = parseSchemaClauseBottom(pCxt, pStmt, &pTableCxt); - if (TSDB_CODE_SUCCESS == code) { - code = parseDataClause(pCxt, pStmt, pTableCxt); - } - return code; -} - -static void resetEnvPreTable(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) { - insDestroyBoundColInfo(&pCxt->tags); - taosMemoryFreeClear(pStmt->pTableMeta); - nodesDestroyNode(pStmt->pTagCond); - taosArrayDestroy(pStmt->pTableTag); - tdDestroySVCreateTbReq(pStmt->pCreateTblReq); - taosMemoryFreeClear(pStmt->pCreateTblReq); - pCxt->missCache = false; - pCxt->usingDuplicateTable = false; - pStmt->pBoundCols = NULL; - pStmt->usingTableProcessing = false; - pStmt->fileProcessing = false; - pStmt->usingTableName.type = 0; - pStmt->stbSyntax = false; -} static int32_t parseStbBoundColumnsClause(SInsertParseContext* pCxt, const char* pBoundCols, STableMeta* pTableMeta, SBoundColInfo* pBoundColsInfo) { @@ -1802,16 +1775,45 @@ static int32_t parseInsertStbClauseBottom(SInsertParseContext* pCxt, SVnodeModif return code; } +// input pStmt->pSql: +// 1. [(tag1_name, ...)] ... +// 2. VALUES ... | FILE ... +static int32_t parseInsertTableClauseBottom(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) { + if (!pStmt->stbSyntax) { + STableDataCxt* pTableCxt = NULL; + int32_t code = parseSchemaClauseBottom(pCxt, pStmt, &pTableCxt); + if (TSDB_CODE_SUCCESS == code) { + code = parseDataClause(pCxt, pStmt, pTableCxt); + } + return code; + } else { + int32_t code = parseInsertStbClauseBottom(pCxt, pStmt); + return code; + } +} + +static void resetEnvPreTable(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) { + insDestroyBoundColInfo(&pCxt->tags); + taosMemoryFreeClear(pStmt->pTableMeta); + nodesDestroyNode(pStmt->pTagCond); + taosArrayDestroy(pStmt->pTableTag); + tdDestroySVCreateTbReq(pStmt->pCreateTblReq); + taosMemoryFreeClear(pStmt->pCreateTblReq); + pCxt->missCache = false; + pCxt->usingDuplicateTable = false; + pStmt->pBoundCols = NULL; + pStmt->usingTableProcessing = false; + pStmt->fileProcessing = false; + pStmt->usingTableName.type = 0; + pStmt->stbSyntax = false; +} + // input pStmt->pSql: [(field1_name, ...)] [ USING ... ] VALUES ... | FILE ... static int32_t parseInsertTableClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, SToken* pTbName) { resetEnvPreTable(pCxt, pStmt); int32_t code = parseSchemaClauseTop(pCxt, pStmt, pTbName); if (TSDB_CODE_SUCCESS == code && !pCxt->missCache) { - if (!pStmt->stbSyntax) { - code = parseInsertTableClauseBottom(pCxt, pStmt); - } else { - code = parseInsertStbClauseBottom(pCxt, pStmt); - } + code = parseInsertTableClauseBottom(pCxt, pStmt); } return code; From 6488672dd642b6e9b3a27fc3f171cff168a761f5 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Sun, 22 Oct 2023 11:21:36 +0800 Subject: [PATCH 08/45] enhance: parse target column --- source/libs/parser/src/parInsertSql.c | 74 +++++++++++++++++++++++++-- 1 file changed, 70 insertions(+), 4 deletions(-) diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index 24e1ab226c..a4f38039de 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -1755,10 +1755,73 @@ static int32_t parseDataClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pS return buildSyntaxErrMsg(&pCxt->msg, "keyword VALUES or FILE is expected", token.z); } +static int32_t parseBoundStbColumnsClause(SInsertParseContext* pCxt, const char** pSql, + STableMeta* pTableMeta, SBoundColInfo* pBoundInfo) { + // tbname schema index == (pBoundInfo->numOfCols - 1) == (tiInfo.numOfColumns + tiInfo.numOfTags) + if (pBoundInfo->numOfCols != pTableMeta->tableInfo.numOfTags + pTableMeta->tableInfo.numOfColumns + 1) { + return TSDB_CODE_PAR_INTERNAL_ERROR; + } + int32_t tbnameSchemaIndex = pTableMeta->tableInfo.numOfTags + pTableMeta->tableInfo.numOfColumns; -static int32_t parseStbBoundColumnsClause(SInsertParseContext* pCxt, const char* pBoundCols, - STableMeta* pTableMeta, SBoundColInfo* pBoundColsInfo) { - return TSDB_CODE_SUCCESS; + bool* pUseCols = taosMemoryCalloc(pBoundInfo->numOfCols, sizeof(bool)); + if (NULL == pUseCols) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + pBoundInfo->numOfBound = 0; + + int16_t lastColIdx = -1; // last column found + int32_t code = TSDB_CODE_SUCCESS; + while (TSDB_CODE_SUCCESS == code) { + SToken token; + NEXT_TOKEN(*pSql, token); + + if (TK_NK_RP == token.type) { + break; + } + + char tmpTokenBuf[TSDB_COL_NAME_LEN + 2] = {0}; // used for deleting Escape character backstick(`) + strncpy(tmpTokenBuf, token.z, token.n); + token.z = tmpTokenBuf; + token.n = strdequote(token.z); + + if (token.n == strlen("tbname") && (strcasecmp(token.z, "tbname") == 0)) { + pBoundInfo->pColIndex[pBoundInfo->numOfBound] = tbnameSchemaIndex; + pUseCols[tbnameSchemaIndex] = true; + ++pBoundInfo->numOfBound; + continue; + } + + int16_t t = lastColIdx + 1; + int16_t index = insFindCol(&token, t, pBoundInfo->numOfCols, pTableMeta->schema); + if (index < 0 && t > 0) { + index = insFindCol(&token, 0, t, pTableMeta->schema); + } + if (index < 0) { + code = generateSyntaxErrMsg(&pCxt->msg, TSDB_CODE_PAR_INVALID_COLUMN, token.z); + } else if (pUseCols[index]) { + code = buildSyntaxErrMsg(&pCxt->msg, "duplicated column name", token.z); + } else { + lastColIdx = index; + pUseCols[index] = true; + pBoundInfo->pColIndex[pBoundInfo->numOfBound] = index; + ++pBoundInfo->numOfBound; + } + } + + if (TSDB_CODE_SUCCESS == code && !pUseCols[0]) { + code = buildInvalidOperationMsg(&pCxt->msg, "primary timestamp column can not be null"); + } + if (TSDB_CODE_SUCCESS == code && !pUseCols[tbnameSchemaIndex]) { + code = buildInvalidOperationMsg(&pCxt->msg, "tbname column can not be null"); + } + taosMemoryFree(pUseCols); + + return code; +} + +static int32_t parseDataStbClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, SBoundColInfo* pBoundColInfo) { + return TSDB_CODE_SUCCESS; } static int32_t parseInsertStbClauseBottom(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) { @@ -1771,7 +1834,10 @@ static int32_t parseInsertStbClauseBottom(SInsertParseContext* pCxt, SVnodeModif } SToken token; int32_t index = 0; - parseStbBoundColumnsClause(pCxt, pStmt->pBoundCols, pStmt->pTableMeta, &stbBoundColInfo); + code = parseBoundStbColumnsClause(pCxt, &pStmt->pBoundCols, pStmt->pTableMeta, &stbBoundColInfo); + if (TSDB_CODE_SUCCESS == code) { + code = parseDataStbClause(pCxt, pStmt, &stbBoundColInfo); + } return code; } From 5156c0d00c7db71990a8e4a99d541e1e04d12960 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Sun, 22 Oct 2023 15:14:50 +0800 Subject: [PATCH 09/45] enhance: get tbname from one row for stb syntax --- source/libs/parser/src/parInsertSql.c | 143 ++++++++++++-------------- 1 file changed, 66 insertions(+), 77 deletions(-) diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index a4f38039de..cf7726e81b 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -182,9 +182,32 @@ static int32_t parseDuplicateUsingClause(SInsertParseContext* pCxt, SVnodeModify return TSDB_CODE_SUCCESS; } +typedef enum { + BOUND_TAGS, + BOUND_COLUMNS, + BOUND_ALL_AND_TBNAME +} EBoundColumnsType; + +static int32_t getTbnameSchemaIndex(STableMeta* pTableMeta) { + return pTableMeta->tableInfo.numOfTags + pTableMeta->tableInfo.numOfColumns; +} + // pStmt->pSql -> field1_name, ...) -static int32_t parseBoundColumns(SInsertParseContext* pCxt, const char** pSql, bool isTags, SSchema* pSchema, +static int32_t parseBoundColumns(SInsertParseContext* pCxt, const char** pSql, EBoundColumnsType boundColsType, STableMeta* pTableMeta, SBoundColInfo* pBoundInfo) { + SSchema* pSchema = NULL; + if (boundColsType == BOUND_TAGS) { + pSchema = getTableTagSchema(pTableMeta); + } else if (boundColsType == BOUND_COLUMNS) { + pSchema = getTableColumnSchema(pTableMeta); + } else { + pSchema = pTableMeta->schema; + if (pBoundInfo->numOfCols != getTbnameSchemaIndex(pTableMeta) + 1) { + return TSDB_CODE_PAR_INTERNAL_ERROR; + } + } + int32_t tbnameSchemaIndex = getTbnameSchemaIndex(pTableMeta); + bool* pUseCols = taosMemoryCalloc(pBoundInfo->numOfCols, sizeof(bool)); if (NULL == pUseCols) { return TSDB_CODE_OUT_OF_MEMORY; @@ -207,6 +230,14 @@ static int32_t parseBoundColumns(SInsertParseContext* pCxt, const char** pSql, b token.z = tmpTokenBuf; token.n = strdequote(token.z); + if (boundColsType == BOUND_ALL_AND_TBNAME) { + if (token.n == strlen("tbname") && (strcasecmp(token.z, "tbname") == 0)) { + pBoundInfo->pColIndex[pBoundInfo->numOfBound] = tbnameSchemaIndex; + pUseCols[tbnameSchemaIndex] = true; + ++pBoundInfo->numOfBound; + continue; + } + } int16_t t = lastColIdx + 1; int16_t index = insFindCol(&token, t, pBoundInfo->numOfCols, pSchema); if (index < 0 && t > 0) { @@ -224,10 +255,12 @@ static int32_t parseBoundColumns(SInsertParseContext* pCxt, const char** pSql, b } } - if (TSDB_CODE_SUCCESS == code && !isTags && !pUseCols[0]) { + if (TSDB_CODE_SUCCESS == code && (BOUND_TAGS != boundColsType) && !pUseCols[0]) { code = buildInvalidOperationMsg(&pCxt->msg, "primary timestamp column can not be null"); } - + if (TSDB_CODE_SUCCESS == code && (BOUND_ALL_AND_TBNAME == boundColsType) &&!pUseCols[tbnameSchemaIndex]) { + code = buildInvalidOperationMsg(&pCxt->msg, "tbname column can not be null"); + } taosMemoryFree(pUseCols); return code; @@ -586,7 +619,7 @@ static int32_t parseBoundTagsClause(SInsertParseContext* pCxt, SVnodeModifyOpStm } pStmt->pSql += index; - return parseBoundColumns(pCxt, &pStmt->pSql, true, getTableTagSchema(pStmt->pTableMeta), &pCxt->tags); + return parseBoundColumns(pCxt, &pStmt->pSql, BOUND_TAGS, pStmt->pTableMeta, &pCxt->tags); } static int32_t parseTagValue(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, SSchema* pTagSchema, SToken* pToken, @@ -1220,12 +1253,12 @@ static int32_t parseBoundColumnsClause(SInsertParseContext* pCxt, SVnodeModifyOp return buildSyntaxErrMsg(&pCxt->msg, "keyword VALUES or FILE is expected", token.z); } // pStmt->pSql -> field1_name, ...) - return parseBoundColumns(pCxt, &pStmt->pSql, false, getTableColumnSchema(pStmt->pTableMeta), + return parseBoundColumns(pCxt, &pStmt->pSql, BOUND_COLUMNS, pStmt->pTableMeta, &pTableCxt->boundColsInfo); } if (NULL != pStmt->pBoundCols) { - return parseBoundColumns(pCxt, &pStmt->pBoundCols, false, getTableColumnSchema(pStmt->pTableMeta), + return parseBoundColumns(pCxt, &pStmt->pBoundCols, BOUND_COLUMNS, pStmt->pTableMeta, &pTableCxt->boundColsInfo); } @@ -1521,8 +1554,12 @@ static void clearColValArray(SArray* pCols) { } } +typedef struct SInsertStbParseContext { + +} SInsertStbParseContext; + static int parseOneRow(SInsertParseContext* pCxt, const char** pSql, STableDataCxt* pTableCxt, bool* pGotRow, - SToken* pToken) { + SToken* pToken, bool *pGotTbName, char* tbName) { SBoundColInfo* pCols = &pTableCxt->boundColsInfo; bool isParseBindParam = false; SSchema* pSchemas = getTableColumnSchema(pTableCxt->pMeta); @@ -1559,7 +1596,21 @@ static int parseOneRow(SInsertParseContext* pCxt, const char** pSql, STableDataC } if (TSDB_CODE_SUCCESS == code) { - code = parseValueToken(pCxt, pSql, pToken, pSchema, getTableInfo(pTableCxt->pMeta).precision, pVal); + if (pCols->pColIndex[i] == getTbnameSchemaIndex(pTableCxt->pMeta)) { + SColVal tbnameVal; + code = parseValueToken(pCxt, pSql, pToken, (SSchema*)tGetTbnameColumnSchema(), getTableInfo(pTableCxt->pMeta).precision, &tbnameVal); + if (code == TSDB_CODE_SUCCESS) { + if (pGotTbName != NULL) { + *pGotTbName = true; + } + if (tbName != NULL) { + memcpy(tbName, tbnameVal.value.pData, tbnameVal.value.nData); + tbName[tbnameVal.value.nData] = '\0'; + } + } + } else { + code = parseValueToken(pCxt, pSql, pToken, pSchema, getTableInfo(pTableCxt->pMeta).precision, pVal); + } } } @@ -1603,8 +1654,10 @@ static int32_t parseValues(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, pStmt->pSql += index; bool gotRow = false; + bool gotTbname = false; + char tbName[TSDB_TABLE_NAME_LEN] = {0}; if (TSDB_CODE_SUCCESS == code) { - code = parseOneRow(pCxt, &pStmt->pSql, pTableCxt, &gotRow, pToken); + code = parseOneRow(pCxt, &pStmt->pSql, pTableCxt, &gotRow, pToken, &gotTbname, tbName); } if (TSDB_CODE_SUCCESS == code) { @@ -1664,8 +1717,9 @@ static int32_t parseCsvFile(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt SToken token; strtolower(pLine, pLine); const char* pRow = pLine; - - code = parseOneRow(pCxt, (const char**)&pRow, pTableCxt, &gotRow, &token); + bool gotTbname = false; + char tbName[TSDB_TABLE_NAME_LEN] = {0}; + code = parseOneRow(pCxt, (const char**)&pRow, pTableCxt, &gotRow, &token, &gotTbname, tbName); if (code && firstLine) { firstLine = false; code = 0; @@ -1755,71 +1809,6 @@ static int32_t parseDataClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pS return buildSyntaxErrMsg(&pCxt->msg, "keyword VALUES or FILE is expected", token.z); } -static int32_t parseBoundStbColumnsClause(SInsertParseContext* pCxt, const char** pSql, - STableMeta* pTableMeta, SBoundColInfo* pBoundInfo) { - // tbname schema index == (pBoundInfo->numOfCols - 1) == (tiInfo.numOfColumns + tiInfo.numOfTags) - if (pBoundInfo->numOfCols != pTableMeta->tableInfo.numOfTags + pTableMeta->tableInfo.numOfColumns + 1) { - return TSDB_CODE_PAR_INTERNAL_ERROR; - } - int32_t tbnameSchemaIndex = pTableMeta->tableInfo.numOfTags + pTableMeta->tableInfo.numOfColumns; - - bool* pUseCols = taosMemoryCalloc(pBoundInfo->numOfCols, sizeof(bool)); - if (NULL == pUseCols) { - return TSDB_CODE_OUT_OF_MEMORY; - } - - pBoundInfo->numOfBound = 0; - - int16_t lastColIdx = -1; // last column found - int32_t code = TSDB_CODE_SUCCESS; - while (TSDB_CODE_SUCCESS == code) { - SToken token; - NEXT_TOKEN(*pSql, token); - - if (TK_NK_RP == token.type) { - break; - } - - char tmpTokenBuf[TSDB_COL_NAME_LEN + 2] = {0}; // used for deleting Escape character backstick(`) - strncpy(tmpTokenBuf, token.z, token.n); - token.z = tmpTokenBuf; - token.n = strdequote(token.z); - - if (token.n == strlen("tbname") && (strcasecmp(token.z, "tbname") == 0)) { - pBoundInfo->pColIndex[pBoundInfo->numOfBound] = tbnameSchemaIndex; - pUseCols[tbnameSchemaIndex] = true; - ++pBoundInfo->numOfBound; - continue; - } - - int16_t t = lastColIdx + 1; - int16_t index = insFindCol(&token, t, pBoundInfo->numOfCols, pTableMeta->schema); - if (index < 0 && t > 0) { - index = insFindCol(&token, 0, t, pTableMeta->schema); - } - if (index < 0) { - code = generateSyntaxErrMsg(&pCxt->msg, TSDB_CODE_PAR_INVALID_COLUMN, token.z); - } else if (pUseCols[index]) { - code = buildSyntaxErrMsg(&pCxt->msg, "duplicated column name", token.z); - } else { - lastColIdx = index; - pUseCols[index] = true; - pBoundInfo->pColIndex[pBoundInfo->numOfBound] = index; - ++pBoundInfo->numOfBound; - } - } - - if (TSDB_CODE_SUCCESS == code && !pUseCols[0]) { - code = buildInvalidOperationMsg(&pCxt->msg, "primary timestamp column can not be null"); - } - if (TSDB_CODE_SUCCESS == code && !pUseCols[tbnameSchemaIndex]) { - code = buildInvalidOperationMsg(&pCxt->msg, "tbname column can not be null"); - } - taosMemoryFree(pUseCols); - - return code; -} - static int32_t parseDataStbClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, SBoundColInfo* pBoundColInfo) { return TSDB_CODE_SUCCESS; } @@ -1834,7 +1823,7 @@ static int32_t parseInsertStbClauseBottom(SInsertParseContext* pCxt, SVnodeModif } SToken token; int32_t index = 0; - code = parseBoundStbColumnsClause(pCxt, &pStmt->pBoundCols, pStmt->pTableMeta, &stbBoundColInfo); + code = parseBoundColumns(pCxt, &pStmt->pBoundCols, BOUND_ALL_AND_TBNAME, pStmt->pTableMeta, &stbBoundColInfo); if (TSDB_CODE_SUCCESS == code) { code = parseDataStbClause(pCxt, pStmt, &stbBoundColInfo); } From e713b7c22fdd2cf9ae610a95513cc5652ae4df57 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Mon, 23 Oct 2023 16:27:00 +0800 Subject: [PATCH 10/45] enhance: insert stb skeleton development --- include/libs/nodes/querynodes.h | 4 +- source/libs/parser/src/parInsertSql.c | 219 ++++++++++++++++++-------- 2 files changed, 158 insertions(+), 65 deletions(-) diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index bfd23d1965..6d8f16986d 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -413,6 +413,7 @@ typedef struct SVgDataBlocks { typedef void (*FFreeTableBlockHash)(SHashObj*); typedef void (*FFreeVgourpBlockArray)(SArray*); +struct SStbRowsDataContext; typedef struct SVnodeModifyOpStmt { ENodeType nodeType; ENodeType sqlNodeType; @@ -441,8 +442,7 @@ typedef struct SVnodeModifyOpStmt { bool fileProcessing; bool stbSyntax; - SName superTableName; - SName childTableName; + struct SStbRowsDataContext* pStbRowsCxt; } SVnodeModifyOpStmt; typedef struct SExplainOptions { diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index cf7726e81b..aa8704a62b 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -622,7 +622,7 @@ static int32_t parseBoundTagsClause(SInsertParseContext* pCxt, SVnodeModifyOpStm return parseBoundColumns(pCxt, &pStmt->pSql, BOUND_TAGS, pStmt->pTableMeta, &pCxt->tags); } -static int32_t parseTagValue(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, SSchema* pTagSchema, SToken* pToken, +static int32_t parseTagValue(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, const char** ppSql, SSchema* pTagSchema, SToken* pToken, SArray* pTagName, SArray* pTagVals, STag** pTag) { if (!isNullValue(pTagSchema->type, pToken)) { taosArrayPush(pTagName, pTagSchema->name); @@ -642,7 +642,7 @@ static int32_t parseTagValue(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStm STagVal val = {0}; int32_t code = - parseTagToken(&pStmt->pSql, pToken, pTagSchema, pStmt->pTableMeta->tableInfo.precision, &val, &pCxt->msg); + parseTagToken(ppSql, pToken, pTagSchema, pStmt->pTableMeta->tableInfo.precision, &val, &pCxt->msg); if (TSDB_CODE_SUCCESS == code) { taosArrayPush(pTagVals, &val); } @@ -845,7 +845,7 @@ static int32_t parseTagsClauseImpl(SInsertParseContext* pCxt, SVnodeModifyOpStmt isJson = pTagSchema->type == TSDB_DATA_TYPE_JSON; code = checkAndTrimValue(&token, pCxt->tmpTokenBuf, &pCxt->msg); if (TSDB_CODE_SUCCESS == code) { - code = parseTagValue(pCxt, pStmt, pTagSchema, &token, pTagName, pTagVals, &pTag); + code = parseTagValue(pCxt, pStmt, &pStmt->pSql, pTagSchema, &token, pTagName, pTagVals, &pTag); } } @@ -900,7 +900,7 @@ static int32_t parseTagsClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pS return code; } -static int32_t storeTableMeta(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) { +static int32_t storeChildTableMeta(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) { pStmt->pTableMeta->suid = pStmt->pTableMeta->uid; pStmt->pTableMeta->uid = pStmt->totalTbNum; pStmt->pTableMeta->tableType = TSDB_CHILD_TABLE; @@ -1038,7 +1038,7 @@ static int32_t getTableMeta(SInsertParseContext* pCxt, SName* pTbName, STableMet return code; } -static int32_t getTableVgroup(SParseContext* pCxt, SVnodeModifyOpStmt* pStmt, bool isStb, bool* pMissCache) { +static int32_t getTargetTableVgroup(SParseContext* pCxt, SVnodeModifyOpStmt* pStmt, bool isStb, bool* pMissCache) { int32_t code = TSDB_CODE_SUCCESS; SVgroupInfo vg; bool exists = true; @@ -1074,7 +1074,6 @@ static int32_t getTargetTableMetaAndVgroup(SInsertParseContext* pCxt, SVnodeModi if (NULL != pStmt->pTableMeta) { if (pStmt->pTableMeta->tableType == TSDB_SUPER_TABLE) { pStmt->stbSyntax = true; - tNameAssign(&pStmt->superTableName, &pStmt->targetTableName); } else { code = taosHashPut(pStmt->pVgroupsHashObj, (const char*)&vg.vgId, sizeof(vg.vgId), (char*)&vg, sizeof(vg)); } @@ -1088,10 +1087,9 @@ static int32_t getTargetTableMetaAndVgroup(SInsertParseContext* pCxt, SVnodeModi if (TSDB_CODE_SUCCESS == code && !pCxt->missCache) { if (TSDB_SUPER_TABLE == pStmt->pTableMeta->tableType) { pStmt->stbSyntax = true; - tNameAssign(&pStmt->superTableName, &pStmt->targetTableName); } if (!pStmt->stbSyntax) { - code = getTableVgroup(pCxt->pComCxt, pStmt, false, &pCxt->missCache); + code = getTargetTableVgroup(pCxt->pComCxt, pStmt, false, &pCxt->missCache); } } } @@ -1155,7 +1153,7 @@ static int32_t getUsingTableSchema(SInsertParseContext* pCxt, SVnodeModifyOpStmt code = getTableMeta(pCxt, &pStmt->usingTableName, &pStmt->pTableMeta, &pCxt->missCache, bUsingTable); } if (TSDB_CODE_SUCCESS == code && !pCxt->missCache) { - code = getTableVgroup(pCxt->pComCxt, pStmt, true, &pCxt->missCache); + code = getTargetTableVgroup(pCxt->pComCxt, pStmt, true, &pCxt->missCache); } if (TSDB_CODE_SUCCESS == code && !pCxt->pComCxt->async) { code = collectUseDatabase(&pStmt->usingTableName, pStmt->pDbFNameHashObj); @@ -1174,7 +1172,7 @@ static int32_t parseUsingTableNameImpl(SInsertParseContext* pCxt, SVnodeModifyOp code = getUsingTableSchema(pCxt, pStmt); } if (TSDB_CODE_SUCCESS == code && !pCxt->missCache) { - code = storeTableMeta(pCxt, pStmt); + code = storeChildTableMeta(pCxt, pStmt); } return code; } @@ -1554,12 +1552,101 @@ static void clearColValArray(SArray* pCols) { } } -typedef struct SInsertStbParseContext { +typedef struct SStbRowsDataContext { + SVnodeModifyOpStmt* pStmt; + STableMeta* pStbMeta; + SName stbName; + SBoundColInfo boundColsInfo; + int32_t numOfBoundCols; + int32_t numOfBoundTags; -} SInsertStbParseContext; + SArray* aChildTableNames; + SArray* aTableDataCxts; + SArray* aCreateTbReqs; -static int parseOneRow(SInsertParseContext* pCxt, const char** pSql, STableDataCxt* pTableCxt, bool* pGotRow, - SToken* pToken, bool *pGotTbName, char* tbName) { + SArray* aTagVals; + SArray* aColVals; + SArray* aTagNames; +} SStbRowsDataContext; + +typedef union SRowsDataContext{ + STableDataCxt* pTableDataCxt; + SStbRowsDataContext* pStbRowsCxt; +} SRowsDataContext; + +static int32_t parseOneStbRow(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, const char** ppSql, SStbRowsDataContext* pStbRowsCxt, bool* pGotRow, + SToken* pToken) { + SBoundColInfo* pCols = &pStbRowsCxt->boundColsInfo; + SSchema* pSchemas = getTableColumnSchema(pStbRowsCxt->pStbMeta); + + bool isJsonTag = false; + SArray* pTagName = taosArrayInit(8, TSDB_COL_NAME_LEN); + SArray* pTagVals = taosArrayInit(pCxt->tags.numOfBound, sizeof(STagVal)); + STag* pTag = NULL; + + int32_t code = TSDB_CODE_SUCCESS; + for (int i = 0; i < pCols->numOfBound && TSDB_CODE_SUCCESS == code; ++i) { + const char* pOrigSql = *ppSql; + bool ignoreComma = false; + NEXT_TOKEN_WITH_PREV_EXT(*ppSql, *pToken, &ignoreComma); + if (ignoreComma) { + code = buildSyntaxErrMsg(&pCxt->msg, "invalid data or symbol", pOrigSql); + break; + } + + if (TK_NK_RP == pToken->type) { + code = generateSyntaxErrMsg(&pCxt->msg, TSDB_CODE_PAR_INVALID_COLUMNS_NUM); + break; + } + + if (pCols->pColIndex[i] < getNumOfColumns(pStbRowsCxt->pStbMeta)) { + SSchema* pSchema = &pSchemas[pCols->pColIndex[i]]; + SColVal* pVal = taosArrayGet(pStbRowsCxt->aColVals, pCols->pColIndex[i]); + code = parseValueToken(pCxt, ppSql, pToken, pSchema, getTableInfo(pStbRowsCxt->pStbMeta).precision, pVal); + } else if (pCols->pColIndex[i] < getTbnameSchemaIndex(pStbRowsCxt->pStbMeta)) { + SSchema* pTagSchema = &pSchemas[pCols->pColIndex[i]]; + isJsonTag = pTagSchema->type == TSDB_DATA_TYPE_JSON; + code = checkAndTrimValue(pToken, pCxt->tmpTokenBuf, &pCxt->msg); + if (TSDB_CODE_SUCCESS == code) { + code = parseTagValue(pCxt, pStmt, ppSql, pTagSchema, pToken, pTagName, pTagVals, &pTag); + } + } + else if (pCols->pColIndex[i] == getTbnameSchemaIndex(pStbRowsCxt->pStbMeta)) { + SColVal tbnameVal; + code = parseValueToken(pCxt, ppSql, pToken, (SSchema*)tGetTbnameColumnSchema(), + getTableInfo(pStbRowsCxt->pStbMeta).precision, &tbnameVal); + if (code == TSDB_CODE_SUCCESS) { + char tbName[TSDB_TABLE_NAME_LEN]; + memcpy(tbName, tbnameVal.value.pData, tbnameVal.value.nData); + tbName[tbnameVal.value.nData] = '\0'; + } + } + + if (TSDB_CODE_SUCCESS == code && i < pCols->numOfBound - 1) { + NEXT_VALID_TOKEN(*ppSql, *pToken); + if (TK_NK_COMMA != pToken->type) { + code = buildSyntaxErrMsg(&pCxt->msg, ", expected", pToken->z); + } + } + } + + // child meta , vgroupid, check privilege + // refer to parInsertSml.c + // create or reuse table data context from tbName + // init submit data + // build create table request, refer to parseTagsClauseImpl + // build row and check table data order, refer to parseOneRow + + if (TSDB_CODE_SUCCESS == code) { + *pGotRow = true; + } + + return code; + + return TSDB_CODE_SUCCESS; +} + +static int parseOneRow(SInsertParseContext* pCxt, const char** pSql, STableDataCxt* pTableCxt, bool* pGotRow, SToken* pToken) { SBoundColInfo* pCols = &pTableCxt->boundColsInfo; bool isParseBindParam = false; SSchema* pSchemas = getTableColumnSchema(pTableCxt->pMeta); @@ -1596,21 +1683,7 @@ static int parseOneRow(SInsertParseContext* pCxt, const char** pSql, STableDataC } if (TSDB_CODE_SUCCESS == code) { - if (pCols->pColIndex[i] == getTbnameSchemaIndex(pTableCxt->pMeta)) { - SColVal tbnameVal; - code = parseValueToken(pCxt, pSql, pToken, (SSchema*)tGetTbnameColumnSchema(), getTableInfo(pTableCxt->pMeta).precision, &tbnameVal); - if (code == TSDB_CODE_SUCCESS) { - if (pGotTbName != NULL) { - *pGotTbName = true; - } - if (tbName != NULL) { - memcpy(tbName, tbnameVal.value.pData, tbnameVal.value.nData); - tbName[tbnameVal.value.nData] = '\0'; - } - } - } else { - code = parseValueToken(pCxt, pSql, pToken, pSchema, getTableInfo(pTableCxt->pMeta).precision, pVal); - } + code = parseValueToken(pCxt, pSql, pToken, pSchema, getTableInfo(pTableCxt->pMeta).precision, pVal); } } @@ -1640,7 +1713,7 @@ static int parseOneRow(SInsertParseContext* pCxt, const char** pSql, STableDataC } // pSql -> (field1_value, ...) [(field1_value2, ...) ...] -static int32_t parseValues(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, STableDataCxt* pTableCxt, +static int32_t parseValues(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, SRowsDataContext rowsDataCxt, int32_t* pNumOfRows, SToken* pToken) { int32_t code = TSDB_CODE_SUCCESS; @@ -1654,10 +1727,12 @@ static int32_t parseValues(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, pStmt->pSql += index; bool gotRow = false; - bool gotTbname = false; - char tbName[TSDB_TABLE_NAME_LEN] = {0}; if (TSDB_CODE_SUCCESS == code) { - code = parseOneRow(pCxt, &pStmt->pSql, pTableCxt, &gotRow, pToken, &gotTbname, tbName); + if (!pStmt->stbSyntax) { + code = parseOneRow(pCxt, &pStmt->pSql, rowsDataCxt.pTableDataCxt, &gotRow, pToken); + } else { + code = parseOneStbRow(pCxt, pStmt, &pStmt->pSql, rowsDataCxt.pStbRowsCxt, &gotRow, pToken); + } } if (TSDB_CODE_SUCCESS == code) { @@ -1682,10 +1757,10 @@ static int32_t parseValues(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, } // VALUES (field1_value, ...) [(field1_value2, ...) ...] -static int32_t parseValuesClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, STableDataCxt* pTableCxt, +static int32_t parseValuesClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, SRowsDataContext rowsDataContext, SToken* pToken) { int32_t numOfRows = 0; - int32_t code = parseValues(pCxt, pStmt, pTableCxt, &numOfRows, pToken); + int32_t code = parseValues(pCxt, pStmt, rowsDataContext, &numOfRows, pToken); if (TSDB_CODE_SUCCESS == code) { pStmt->totalRowsNum += numOfRows; pStmt->totalTbNum += 1; @@ -1694,7 +1769,7 @@ static int32_t parseValuesClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* return code; } -static int32_t parseCsvFile(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, STableDataCxt* pTableCxt, +static int32_t parseCsvFile(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, SRowsDataContext rowsDataCxt, int32_t* pNumOfRows) { int32_t code = TSDB_CODE_SUCCESS; (*pNumOfRows) = 0; @@ -1717,9 +1792,11 @@ static int32_t parseCsvFile(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt SToken token; strtolower(pLine, pLine); const char* pRow = pLine; - bool gotTbname = false; - char tbName[TSDB_TABLE_NAME_LEN] = {0}; - code = parseOneRow(pCxt, (const char**)&pRow, pTableCxt, &gotRow, &token, &gotTbname, tbName); + if (!pStmt->stbSyntax) { + code = parseOneRow(pCxt, (const char**)&pRow, rowsDataCxt.pTableDataCxt, &gotRow, &token); + } else { + code = parseOneStbRow(pCxt, pStmt, (const char**)&pRow, rowsDataCxt.pStbRowsCxt, &gotRow, &token); + } if (code && firstLine) { firstLine = false; code = 0; @@ -1749,9 +1826,9 @@ static int32_t parseCsvFile(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt return code; } -static int32_t parseDataFromFileImpl(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, STableDataCxt* pTableCxt) { +static int32_t parseDataFromFileImpl(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, SRowsDataContext rowsDataCxt) { int32_t numOfRows = 0; - int32_t code = parseCsvFile(pCxt, pStmt, pTableCxt, &numOfRows); + int32_t code = parseCsvFile(pCxt, pStmt, rowsDataCxt, &numOfRows); if (TSDB_CODE_SUCCESS == code) { pStmt->totalRowsNum += numOfRows; pStmt->totalTbNum += 1; @@ -1766,7 +1843,7 @@ static int32_t parseDataFromFileImpl(SInsertParseContext* pCxt, SVnodeModifyOpSt } static int32_t parseDataFromFile(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, SToken* pFilePath, - STableDataCxt* pTableCxt) { + SRowsDataContext rowsDataCxt) { char filePathStr[TSDB_FILENAME_LEN] = {0}; if (TK_NK_STRING == pFilePath->type) { trimString(pFilePath->z, pFilePath->n, filePathStr, sizeof(filePathStr)); @@ -1778,10 +1855,10 @@ static int32_t parseDataFromFile(SInsertParseContext* pCxt, SVnodeModifyOpStmt* return TAOS_SYSTEM_ERROR(errno); } - return parseDataFromFileImpl(pCxt, pStmt, pTableCxt); + return parseDataFromFileImpl(pCxt, pStmt, rowsDataCxt); } -static int32_t parseFileClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, STableDataCxt* pTableCxt, +static int32_t parseFileClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, SRowsDataContext rowsDataCxt, SToken* pToken) { if (tsUseAdapter) { return buildInvalidOperationMsg(&pCxt->msg, "proxy mode does not support csv loading"); @@ -1791,28 +1868,24 @@ static int32_t parseFileClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pS if (0 == pToken->n || (TK_NK_STRING != pToken->type && TK_NK_ID != pToken->type)) { return buildSyntaxErrMsg(&pCxt->msg, "file path is required following keyword FILE", pToken->z); } - return parseDataFromFile(pCxt, pStmt, pToken, pTableCxt); + return parseDataFromFile(pCxt, pStmt, pToken, rowsDataCxt); } // VALUES (field1_value, ...) [(field1_value2, ...) ...] | FILE csv_file_path -static int32_t parseDataClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, STableDataCxt* pTableCxt) { +static int32_t parseDataClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, SRowsDataContext rowsDataCxt) { SToken token; NEXT_TOKEN(pStmt->pSql, token); switch (token.type) { case TK_VALUES: - return parseValuesClause(pCxt, pStmt, pTableCxt, &token); + return parseValuesClause(pCxt, pStmt, rowsDataCxt, &token); case TK_FILE: - return parseFileClause(pCxt, pStmt, pTableCxt, &token); + return parseFileClause(pCxt, pStmt, rowsDataCxt, &token); default: break; } return buildSyntaxErrMsg(&pCxt->msg, "keyword VALUES or FILE is expected", token.z); } -static int32_t parseDataStbClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, SBoundColInfo* pBoundColInfo) { - return TSDB_CODE_SUCCESS; -} - static int32_t parseInsertStbClauseBottom(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) { int32_t code = TSDB_CODE_SUCCESS; STableComInfo tblInfo = getTableInfo(pStmt->pTableMeta); @@ -1824,9 +1897,16 @@ static int32_t parseInsertStbClauseBottom(SInsertParseContext* pCxt, SVnodeModif SToken token; int32_t index = 0; code = parseBoundColumns(pCxt, &pStmt->pBoundCols, BOUND_ALL_AND_TBNAME, pStmt->pTableMeta, &stbBoundColInfo); + pStmt->pStbRowsCxt = taosMemoryCalloc(1, sizeof(SStbRowsDataContext)); + + // fill SStbRowsDataCxt; + + SRowsDataContext rowsDataCxt; + rowsDataCxt.pStbRowsCxt = pStmt->pStbRowsCxt; if (TSDB_CODE_SUCCESS == code) { - code = parseDataStbClause(pCxt, pStmt, &stbBoundColInfo); + code = parseDataClause(pCxt, pStmt, rowsDataCxt); } + return code; } @@ -1837,8 +1917,10 @@ static int32_t parseInsertTableClauseBottom(SInsertParseContext* pCxt, SVnodeMod if (!pStmt->stbSyntax) { STableDataCxt* pTableCxt = NULL; int32_t code = parseSchemaClauseBottom(pCxt, pStmt, &pTableCxt); + SRowsDataContext rowsDataCxt; + rowsDataCxt.pTableDataCxt = pTableCxt; if (TSDB_CODE_SUCCESS == code) { - code = parseDataClause(pCxt, pStmt, pTableCxt); + code = parseDataClause(pCxt, pStmt, rowsDataCxt); } return code; } else { @@ -1848,6 +1930,7 @@ static int32_t parseInsertTableClauseBottom(SInsertParseContext* pCxt, SVnodeMod } static void resetEnvPreTable(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) { + //TODO: reset env for stb syntax insDestroyBoundColInfo(&pCxt->tags); taosMemoryFreeClear(pStmt->pTableMeta); nodesDestroyNode(pStmt->pTagCond); @@ -2060,7 +2143,7 @@ static int32_t getTableMetaFromMetaData(const SArray* pTables, STableMeta** pMet return pRes->code; } -static int32_t getTableVgroupFromMetaData(const SArray* pTables, SVnodeModifyOpStmt* pStmt, bool isStb) { +static int32_t addTableVgroupFromMetaData(const SArray* pTables, SVnodeModifyOpStmt* pStmt, bool isStb) { if (1 != taosArrayGetSize(pTables)) { return TSDB_CODE_FAILED; } @@ -2116,10 +2199,10 @@ static int32_t processTableSchemaFromMetaData(SInsertParseContext* pCxt, const S code = buildInvalidOperationMsg(&pCxt->msg, "insert data into super table is not supported"); } if (TSDB_CODE_SUCCESS == code && isStb) { - code = storeTableMeta(pCxt, pStmt); + code = storeChildTableMeta(pCxt, pStmt); } if (TSDB_CODE_SUCCESS == code) { - code = getTableVgroupFromMetaData(pMetaData->pTableHash, pStmt, isStb); + code = addTableVgroupFromMetaData(pMetaData->pTableHash, pStmt, isStb); } if (TSDB_CODE_SUCCESS == code && !isStb && NULL != pStmt->pTagCond) { code = checkSubtablePrivilegeForTable(pMetaData->pTableTag, pStmt); @@ -2158,10 +2241,12 @@ static int32_t setVnodeModifOpStmt(SInsertParseContext* pCxt, SCatalogReq* pCata if (pStmt->pTableMeta->tableType == TSDB_SUPER_TABLE && !pStmt->usingTableProcessing) { pStmt->stbSyntax = true; } - if (pStmt->usingTableProcessing || pStmt->stbSyntax) { - return processTableSchemaFromMetaData(pCxt, pMetaData, pStmt, true); + if (!pStmt->stbSyntax) { + if (pStmt->usingTableProcessing) { + return processTableSchemaFromMetaData(pCxt, pMetaData, pStmt, true); + } + return processTableSchemaFromMetaData(pCxt, pMetaData, pStmt, false); } - return processTableSchemaFromMetaData(pCxt, pMetaData, pStmt, false); } return code; } @@ -2250,10 +2335,18 @@ static int32_t parseInsertSqlFromStart(SInsertParseContext* pCxt, SVnodeModifyOp } static int32_t parseInsertSqlFromCsv(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) { - STableDataCxt* pTableCxt = NULL; - int32_t code = getTableDataCxt(pCxt, pStmt, &pTableCxt); + int32_t code = TSDB_CODE_SUCCESS; + SRowsDataContext rowsDataCxt; + + if (!pStmt->stbSyntax) { + STableDataCxt* pTableCxt = NULL; + code = getTableDataCxt(pCxt, pStmt, &pTableCxt); + rowsDataCxt.pTableDataCxt = pTableCxt; + } else { + rowsDataCxt.pStbRowsCxt = pStmt->pStbRowsCxt; + } if (TSDB_CODE_SUCCESS == code) { - code = parseDataFromFileImpl(pCxt, pStmt, pTableCxt); + code = parseDataFromFileImpl(pCxt, pStmt, rowsDataCxt); } if (TSDB_CODE_SUCCESS == code) { From 8cdcc422839c765c810e0cd2334c2d9fb1171981 Mon Sep 17 00:00:00 2001 From: slzhou Date: Mon, 23 Oct 2023 17:03:18 +0800 Subject: [PATCH 11/45] enhance: add comments to SVnodeModifyOpStmt --- include/libs/nodes/querynodes.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index 1331e865b2..7dd60cfe5b 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -429,11 +429,11 @@ typedef struct SVnodeModifyOpStmt { struct STableMeta* pTableMeta; SNode* pTagCond; SArray* pTableTag; - SHashObj* pVgroupsHashObj; + SHashObj* pVgroupsHashObj; // SHashObj SHashObj* pTableBlockHashObj; // SHashObj - SHashObj* pSubTableHashObj; - SHashObj* pTableNameHashObj; - SHashObj* pDbFNameHashObj; + SHashObj* pSubTableHashObj; // SHashObj + SHashObj* pTableNameHashObj; // set of table names for refreshing meta + SHashObj* pDbFNameHashObj; // set of db names for refreshing meta SArray* pVgDataBlocks; // SArray SVCreateTbReq* pCreateTblReq; TdFilePtr fp; From 65cfc13703dc3896782ef89b1b1f314d0f20ff5d Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Mon, 23 Oct 2023 19:40:27 +0800 Subject: [PATCH 12/45] fix: memory sanitizer error --- source/libs/parser/src/parInsertSql.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index aa8704a62b..46b8993d12 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -1892,6 +1892,7 @@ static int32_t parseInsertStbClauseBottom(SInsertParseContext* pCxt, SVnodeModif SBoundColInfo stbBoundColInfo; insInitBoundColsInfo(tblInfo.numOfColumns + tblInfo.numOfTags + 1, &stbBoundColInfo); if (!pStmt->pBoundCols) { + insDestroyBoundColInfo(&stbBoundColInfo); return buildSyntaxErrMsg(&pCxt->msg, "(...tbname...) bounded cols is expected", pStmt->pSql); } SToken token; @@ -1906,6 +1907,7 @@ static int32_t parseInsertStbClauseBottom(SInsertParseContext* pCxt, SVnodeModif if (TSDB_CODE_SUCCESS == code) { code = parseDataClause(pCxt, pStmt, rowsDataCxt); } + insDestroyBoundColInfo(&stbBoundColInfo); return code; } From ae1bac251a3af627e19c0b0f2894833689df1a50 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Tue, 24 Oct 2023 16:08:08 +0800 Subject: [PATCH 13/45] enhance: finished skeleton --- include/libs/nodes/querynodes.h | 4 +- source/libs/parser/inc/parInsertUtil.h | 1 + source/libs/parser/src/parInsertSql.c | 183 +++++++++++++++++++------ source/libs/parser/src/parInsertUtil.c | 4 + 4 files changed, 150 insertions(+), 42 deletions(-) diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index 7dd60cfe5b..21e50923a2 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -432,8 +432,8 @@ typedef struct SVnodeModifyOpStmt { SHashObj* pVgroupsHashObj; // SHashObj SHashObj* pTableBlockHashObj; // SHashObj SHashObj* pSubTableHashObj; // SHashObj - SHashObj* pTableNameHashObj; // set of table names for refreshing meta - SHashObj* pDbFNameHashObj; // set of db names for refreshing meta + SHashObj* pTableNameHashObj; // set of table names for refreshing meta, sync mode + SHashObj* pDbFNameHashObj; // set of db names for refreshing meta, sync mode SArray* pVgDataBlocks; // SArray SVCreateTbReq* pCreateTblReq; TdFilePtr fp; diff --git a/source/libs/parser/inc/parInsertUtil.h b/source/libs/parser/inc/parInsertUtil.h index 303d349b34..0a153225a3 100644 --- a/source/libs/parser/inc/parInsertUtil.h +++ b/source/libs/parser/inc/parInsertUtil.h @@ -45,6 +45,7 @@ int16_t insFindCol(struct SToken *pColname, int16_t start, int16_t end, SSchema void insBuildCreateTbReq(SVCreateTbReq *pTbReq, const char *tname, STag *pTag, int64_t suid, const char *sname, SArray *tagName, uint8_t tagNum, int32_t ttl); int32_t insInitBoundColsInfo(int32_t numOfBound, SBoundColInfo *pInfo); +void insInitColValues(STableMeta* pTableMeta, SArray* aColValues); void insCheckTableDataOrder(STableDataCxt *pTableCxt, TSKEY tsKey); int32_t insGetTableDataCxt(SHashObj *pHash, void *id, int32_t idLen, STableMeta *pTableMeta, SVCreateTbReq **pCreateTbReq, STableDataCxt **pTableCxt, bool colMode); diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index 46b8993d12..706621d154 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -1553,20 +1553,20 @@ static void clearColValArray(SArray* pCols) { } typedef struct SStbRowsDataContext { - SVnodeModifyOpStmt* pStmt; - STableMeta* pStbMeta; SName stbName; + + STableMeta* pStbMeta; + SNode* pTagCond; SBoundColInfo boundColsInfo; - int32_t numOfBoundCols; - int32_t numOfBoundTags; - - SArray* aChildTableNames; - SArray* aTableDataCxts; - SArray* aCreateTbReqs; + // the following fields are for each stb row SArray* aTagVals; SArray* aColVals; SArray* aTagNames; + SName ctbName; + STag* pTag; + STableMeta* pCtbMeta; + SVCreateTbReq* pCreateCtbReq; } SStbRowsDataContext; typedef union SRowsDataContext{ @@ -1574,23 +1574,25 @@ typedef union SRowsDataContext{ SStbRowsDataContext* pStbRowsCxt; } SRowsDataContext; -static int32_t parseOneStbRow(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, const char** ppSql, SStbRowsDataContext* pStbRowsCxt, bool* pGotRow, - SToken* pToken) { +static int32_t getStbRowValues(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, const char** ppSql, + SStbRowsDataContext* pStbRowsCxt, bool* pGotRow, SToken* pToken) { SBoundColInfo* pCols = &pStbRowsCxt->boundColsInfo; SSchema* pSchemas = getTableColumnSchema(pStbRowsCxt->pStbMeta); bool isJsonTag = false; - SArray* pTagName = taosArrayInit(8, TSDB_COL_NAME_LEN); - SArray* pTagVals = taosArrayInit(pCxt->tags.numOfBound, sizeof(STagVal)); - STag* pTag = NULL; + SArray* pTagName = pStbRowsCxt->aTagNames; + SArray* pTagVals = pStbRowsCxt->aTagVals; + + bool bFoundTbName = false; + const char* pOrigSql = *ppSql; int32_t code = TSDB_CODE_SUCCESS; - for (int i = 0; i < pCols->numOfBound && TSDB_CODE_SUCCESS == code; ++i) { - const char* pOrigSql = *ppSql; + for (int i = 0; i < pCols->numOfBound && code == TSDB_CODE_SUCCESS; ++i) { + const char* pTmpSql = *ppSql; bool ignoreComma = false; NEXT_TOKEN_WITH_PREV_EXT(*ppSql, *pToken, &ignoreComma); if (ignoreComma) { - code = buildSyntaxErrMsg(&pCxt->msg, "invalid data or symbol", pOrigSql); + code = buildSyntaxErrMsg(&pCxt->msg, "invalid data or symbol", pTmpSql); break; } @@ -1607,29 +1609,114 @@ static int32_t parseOneStbRow(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pSt SSchema* pTagSchema = &pSchemas[pCols->pColIndex[i]]; isJsonTag = pTagSchema->type == TSDB_DATA_TYPE_JSON; code = checkAndTrimValue(pToken, pCxt->tmpTokenBuf, &pCxt->msg); - if (TSDB_CODE_SUCCESS == code) { - code = parseTagValue(pCxt, pStmt, ppSql, pTagSchema, pToken, pTagName, pTagVals, &pTag); + if (code == TSDB_CODE_SUCCESS) { + code = parseTagValue(pCxt, pStmt, ppSql, pTagSchema, pToken, pTagName, pTagVals, &pStbRowsCxt->pTag); } } else if (pCols->pColIndex[i] == getTbnameSchemaIndex(pStbRowsCxt->pStbMeta)) { SColVal tbnameVal; code = parseValueToken(pCxt, ppSql, pToken, (SSchema*)tGetTbnameColumnSchema(), getTableInfo(pStbRowsCxt->pStbMeta).precision, &tbnameVal); - if (code == TSDB_CODE_SUCCESS) { - char tbName[TSDB_TABLE_NAME_LEN]; - memcpy(tbName, tbnameVal.value.pData, tbnameVal.value.nData); - tbName[tbnameVal.value.nData] = '\0'; + if (code == TSDB_CODE_SUCCESS && COL_VAL_IS_VALUE(&tbnameVal)) { + tNameAssign(&pStbRowsCxt->ctbName, &pStbRowsCxt->stbName); + tNameAddTbName(&pStbRowsCxt->ctbName, tbnameVal.value.pData, tbnameVal.value.nData); + bFoundTbName = true; } } - if (TSDB_CODE_SUCCESS == code && i < pCols->numOfBound - 1) { + if (code == TSDB_CODE_SUCCESS && i < pCols->numOfBound - 1) { NEXT_VALID_TOKEN(*ppSql, *pToken); if (TK_NK_COMMA != pToken->type) { code = buildSyntaxErrMsg(&pCxt->msg, ", expected", pToken->z); } } } + if (!bFoundTbName) { + code = buildSyntaxErrMsg(&pCxt->msg, "tbname value expected", pOrigSql); + } + if (code == TSDB_CODE_SUCCESS && !isJsonTag) { + code = tTagNew(pTagVals, 1, false, &pStbRowsCxt->pTag); + } + return code; +} +static int32_t parseOneStbRow(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, const char** ppSql, + SStbRowsDataContext* pStbRowsCxt, bool* pGotRow, SToken* pToken) { + int32_t code = getStbRowValues(pCxt, pStmt, ppSql, pStbRowsCxt, pGotRow, pToken); + if (code != TSDB_CODE_SUCCESS || !*pGotRow) { + return code; + } + + if (pStbRowsCxt->pTagCond) { + code = checkSubtablePrivilege(pStbRowsCxt->aTagVals, pStbRowsCxt->aTagNames, &pStbRowsCxt->pTagCond); + } + + if (code == TSDB_CODE_SUCCESS) { + + collectUseTable(&pStbRowsCxt->ctbName, pStmt->pTableNameHashObj); + + char ctbFName[TSDB_TABLE_FNAME_LEN]; + tNameExtractFullName(&pStbRowsCxt->ctbName, ctbFName); + STableMeta** pMeta = taosHashGet(pStmt->pSubTableHashObj, ctbFName, strlen(ctbFName)); + if (NULL != pMeta) { + cloneTableMeta(*pMeta, &pStbRowsCxt->pCtbMeta); + } else { + SVgroupInfo vg; + SRequestConnInfo conn = {.pTrans = pCxt->pComCxt->pTransporter, + .requestId = pCxt->pComCxt->requestId, + .requestObjRefId = pCxt->pComCxt->requestRid, + .mgmtEps = pCxt->pComCxt->mgmtEpSet}; + code = catalogGetTableHashVgroup(pCxt->pComCxt->pCatalog, &conn, &pStmt->targetTableName, &vg); + taosHashPut(pStmt->pVgroupsHashObj, (const char*)(&vg.vgId), sizeof(vg.vgId), &vg, sizeof(vg)); + cloneTableMeta(pStbRowsCxt->pStbMeta, &pStbRowsCxt->pCtbMeta); + pStbRowsCxt->pCtbMeta->suid = pStbRowsCxt->pStbMeta->uid; + pStbRowsCxt->pCtbMeta->uid = taosHashGetSize(pStmt->pSubTableHashObj) + 1; + pStbRowsCxt->pCtbMeta->vgId = vg.vgId; + pStbRowsCxt->pCtbMeta->tableType = TSDB_CHILD_TABLE; + STableMeta* pBackup = NULL; + cloneTableMeta(pStmt->pTableMeta, &pBackup); + taosHashPut(pStmt->pSubTableHashObj, ctbFName, strlen(ctbFName), &pBackup, POINTER_BYTES); + } + } + + pStbRowsCxt->pCreateCtbReq = taosMemoryCalloc(1, sizeof(SVCreateTbReq)); + if (pStbRowsCxt->pCreateCtbReq == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + if (code == TSDB_CODE_SUCCESS) { + insBuildCreateTbReq(pStbRowsCxt->pCreateCtbReq, pStbRowsCxt->ctbName.tname, pStbRowsCxt->pTag, pStbRowsCxt->pStbMeta->uid, + pStbRowsCxt->stbName.tname, pStbRowsCxt->aTagNames, getNumOfTags(pStbRowsCxt->pStbMeta), + TSDB_DEFAULT_TABLE_TTL); + pStbRowsCxt->pTag = NULL; + } + + STableDataCxt* pTableDataCxt = NULL; + code = insGetTableDataCxt(pStmt->pTableBlockHashObj, &pStbRowsCxt->pCtbMeta->uid, sizeof(pStbRowsCxt->pCtbMeta->uid), + pStbRowsCxt->pCtbMeta, &pStbRowsCxt->pCreateCtbReq, &pTableDataCxt, false); + initTableColSubmitData(pTableDataCxt); + if (code == TSDB_CODE_SUCCESS) { + SRow** pRow = taosArrayReserve(pTableDataCxt->pData->aRowP, 1); + code = tRowBuild(pStbRowsCxt->aColVals, pTableDataCxt->pSchema, pRow); + if (TSDB_CODE_SUCCESS == code) { + insCheckTableDataOrder(pTableDataCxt, TD_ROW_KEY(*pRow)); + } + } + if (code == TSDB_CODE_SUCCESS) { + *pGotRow = true; + } + taosArrayClear(pStbRowsCxt->aTagNames); + for (int i = 0; i < taosArrayGetSize(pStbRowsCxt->aTagVals); ++i) { + STagVal* p = (STagVal*)taosArrayGet(pStbRowsCxt->aTagVals, i); + if (IS_VAR_DATA_TYPE(p->type)) { + taosMemoryFreeClear(p->pData); + } + } + taosArrayClear(pStbRowsCxt->aTagVals); + taosArrayClear(pStbRowsCxt->aColVals); + tTagFree(pStbRowsCxt->pTag); + taosMemoryFree(pStbRowsCxt->pCtbMeta); + tdDestroySVCreateTbReq(pStmt->pCreateTblReq); + taosMemoryFreeClear(pStmt->pCreateTblReq); // child meta , vgroupid, check privilege // refer to parInsertSml.c // create or reuse table data context from tbName @@ -1637,11 +1724,7 @@ static int32_t parseOneStbRow(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pSt // build create table request, refer to parseTagsClauseImpl // build row and check table data order, refer to parseOneRow - if (TSDB_CODE_SUCCESS == code) { - *pGotRow = true; - } - - return code; + // clear the values, etc return TSDB_CODE_SUCCESS; } @@ -1886,28 +1969,48 @@ static int32_t parseDataClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pS return buildSyntaxErrMsg(&pCxt->msg, "keyword VALUES or FILE is expected", token.z); } +static void destroyStbRowsDataContext(SStbRowsDataContext* pStbRowsContext) { + taosArrayDestroy(pStbRowsContext->aColVals); + taosArrayDestroy(pStbRowsContext->aTagVals); + taosArrayDestroy(pStbRowsContext->aTagNames); + insDestroyBoundColInfo(&pStbRowsContext->boundColsInfo); +} + static int32_t parseInsertStbClauseBottom(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) { int32_t code = TSDB_CODE_SUCCESS; - STableComInfo tblInfo = getTableInfo(pStmt->pTableMeta); - SBoundColInfo stbBoundColInfo; - insInitBoundColsInfo(tblInfo.numOfColumns + tblInfo.numOfTags + 1, &stbBoundColInfo); if (!pStmt->pBoundCols) { - insDestroyBoundColInfo(&stbBoundColInfo); - return buildSyntaxErrMsg(&pCxt->msg, "(...tbname...) bounded cols is expected", pStmt->pSql); + return buildSyntaxErrMsg(&pCxt->msg, "(...tbname, ts...) bounded cols is expected for supertable insertion", pStmt->pSql); } - SToken token; - int32_t index = 0; - code = parseBoundColumns(pCxt, &pStmt->pBoundCols, BOUND_ALL_AND_TBNAME, pStmt->pTableMeta, &stbBoundColInfo); - pStmt->pStbRowsCxt = taosMemoryCalloc(1, sizeof(SStbRowsDataContext)); + SStbRowsDataContext* pStbRowsCxt; + pStbRowsCxt = taosMemoryCalloc(1, sizeof(SStbRowsDataContext)); + if (!pStbRowsCxt) { + return TSDB_CODE_OUT_OF_MEMORY; + } + tNameAssign(&pStbRowsCxt->stbName, &pStmt->targetTableName); + collectUseTable(&pStbRowsCxt->stbName, pStmt->pTableNameHashObj); + collectUseDatabase(&pStbRowsCxt->stbName, pStmt->pDbFNameHashObj); + pStbRowsCxt->pTagCond = pStmt->pTagCond; + pStbRowsCxt->pStbMeta = pStmt->pTableMeta; + pStbRowsCxt->aTagNames = taosArrayInit(8, TSDB_COL_NAME_LEN); + pStbRowsCxt->aTagVals = taosArrayInit(8, sizeof(STagVal)); - // fill SStbRowsDataCxt; + // col values and bound cols info of STableDataContext is not used TODO: remove the construction when createing table data context + pStbRowsCxt->aColVals = taosArrayInit(getNumOfColumns(pStbRowsCxt->pStbMeta), sizeof(SColVal)); + insInitColValues(pStbRowsCxt->pStbMeta, pStbRowsCxt->aColVals); + + STableComInfo tblInfo = getTableInfo(pStmt->pTableMeta); + insInitBoundColsInfo(tblInfo.numOfColumns + tblInfo.numOfTags + 1, &pStbRowsCxt->boundColsInfo); + + code = parseBoundColumns(pCxt, &pStmt->pBoundCols, BOUND_ALL_AND_TBNAME, pStmt->pTableMeta, &pStbRowsCxt->boundColsInfo); + + pStmt->pStbRowsCxt = pStbRowsCxt; SRowsDataContext rowsDataCxt; - rowsDataCxt.pStbRowsCxt = pStmt->pStbRowsCxt; + rowsDataCxt.pStbRowsCxt = pStbRowsCxt; if (TSDB_CODE_SUCCESS == code) { code = parseDataClause(pCxt, pStmt, rowsDataCxt); } - insDestroyBoundColInfo(&stbBoundColInfo); + // destroyStbRowsDataContext; TODO: move it to resetEnvPreTable return code; } diff --git a/source/libs/parser/src/parInsertUtil.c b/source/libs/parser/src/parInsertUtil.c index 1c0b28f883..9791319503 100644 --- a/source/libs/parser/src/parInsertUtil.c +++ b/source/libs/parser/src/parInsertUtil.c @@ -170,6 +170,10 @@ static void initColValues(STableMeta* pTableMeta, SArray* pValues) { } } +void insInitColValues(STableMeta* pTableMeta, SArray* aColValues) { + initColValues(pTableMeta, aColValues); +} + int32_t insInitBoundColsInfo(int32_t numOfBound, SBoundColInfo* pInfo) { pInfo->numOfCols = numOfBound; pInfo->numOfBound = numOfBound; From 27302f995bc44cc22da3bbdae886d260485310ec Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Tue, 24 Oct 2023 16:23:03 +0800 Subject: [PATCH 14/45] fix: add memory free --- source/libs/parser/src/parInsertSql.c | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index 706621d154..368295638e 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -1715,8 +1715,8 @@ static int32_t parseOneStbRow(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pSt taosArrayClear(pStbRowsCxt->aColVals); tTagFree(pStbRowsCxt->pTag); taosMemoryFree(pStbRowsCxt->pCtbMeta); - tdDestroySVCreateTbReq(pStmt->pCreateTblReq); - taosMemoryFreeClear(pStmt->pCreateTblReq); + tdDestroySVCreateTbReq(pStbRowsCxt->pCreateCtbReq); + taosMemoryFreeClear(pStbRowsCxt->pCreateCtbReq); // child meta , vgroupid, check privilege // refer to parInsertSml.c // create or reuse table data context from tbName @@ -1969,11 +1969,15 @@ static int32_t parseDataClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pS return buildSyntaxErrMsg(&pCxt->msg, "keyword VALUES or FILE is expected", token.z); } -static void destroyStbRowsDataContext(SStbRowsDataContext* pStbRowsContext) { - taosArrayDestroy(pStbRowsContext->aColVals); - taosArrayDestroy(pStbRowsContext->aTagVals); - taosArrayDestroy(pStbRowsContext->aTagNames); - insDestroyBoundColInfo(&pStbRowsContext->boundColsInfo); +static void destroyStbRowsDataContext(SStbRowsDataContext* pStbRowsCxt) { + taosArrayDestroy(pStbRowsCxt->aColVals); + taosArrayDestroy(pStbRowsCxt->aTagVals); + taosArrayDestroy(pStbRowsCxt->aTagNames); + insDestroyBoundColInfo(&pStbRowsCxt->boundColsInfo); + tTagFree(pStbRowsCxt->pTag); + taosMemoryFree(pStbRowsCxt->pCtbMeta); + tdDestroySVCreateTbReq(pStbRowsCxt->pCreateCtbReq); + taosMemoryFreeClear(pStbRowsCxt->pCreateCtbReq); } static int32_t parseInsertStbClauseBottom(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) { @@ -1981,8 +1985,7 @@ static int32_t parseInsertStbClauseBottom(SInsertParseContext* pCxt, SVnodeModif if (!pStmt->pBoundCols) { return buildSyntaxErrMsg(&pCxt->msg, "(...tbname, ts...) bounded cols is expected for supertable insertion", pStmt->pSql); } - SStbRowsDataContext* pStbRowsCxt; - pStbRowsCxt = taosMemoryCalloc(1, sizeof(SStbRowsDataContext)); + SStbRowsDataContext* pStbRowsCxt = taosMemoryCalloc(1, sizeof(SStbRowsDataContext)); if (!pStbRowsCxt) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -2010,7 +2013,8 @@ static int32_t parseInsertStbClauseBottom(SInsertParseContext* pCxt, SVnodeModif if (TSDB_CODE_SUCCESS == code) { code = parseDataClause(pCxt, pStmt, rowsDataCxt); } - // destroyStbRowsDataContext; TODO: move it to resetEnvPreTable + destroyStbRowsDataContext(pStbRowsCxt); + taosMemoryFree(pStbRowsCxt); return code; } From 0c33a3e60ea605e3a2fc55950b47e3a12c5738d5 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Wed, 25 Oct 2023 08:27:02 +0800 Subject: [PATCH 15/45] fix: child table meta optimization --- source/libs/parser/src/parInsertSql.c | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index 368295638e..8b6a92eea9 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -1652,14 +1652,14 @@ static int32_t parseOneStbRow(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pSt } if (code == TSDB_CODE_SUCCESS) { - collectUseTable(&pStbRowsCxt->ctbName, pStmt->pTableNameHashObj); char ctbFName[TSDB_TABLE_FNAME_LEN]; tNameExtractFullName(&pStbRowsCxt->ctbName, ctbFName); - STableMeta** pMeta = taosHashGet(pStmt->pSubTableHashObj, ctbFName, strlen(ctbFName)); - if (NULL != pMeta) { - cloneTableMeta(*pMeta, &pStbRowsCxt->pCtbMeta); + STableMeta** pCtbMeta = taosHashGet(pStmt->pSubTableHashObj, ctbFName, strlen(ctbFName)); + if (NULL != pCtbMeta) { + pStbRowsCxt->pCtbMeta->uid = (*pCtbMeta)->uid; + pStbRowsCxt->pCtbMeta->vgId = (*pCtbMeta)->vgId; } else { SVgroupInfo vg; SRequestConnInfo conn = {.pTrans = pCxt->pComCxt->pTransporter, @@ -1668,11 +1668,8 @@ static int32_t parseOneStbRow(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pSt .mgmtEps = pCxt->pComCxt->mgmtEpSet}; code = catalogGetTableHashVgroup(pCxt->pComCxt->pCatalog, &conn, &pStmt->targetTableName, &vg); taosHashPut(pStmt->pVgroupsHashObj, (const char*)(&vg.vgId), sizeof(vg.vgId), &vg, sizeof(vg)); - cloneTableMeta(pStbRowsCxt->pStbMeta, &pStbRowsCxt->pCtbMeta); - pStbRowsCxt->pCtbMeta->suid = pStbRowsCxt->pStbMeta->uid; pStbRowsCxt->pCtbMeta->uid = taosHashGetSize(pStmt->pSubTableHashObj) + 1; pStbRowsCxt->pCtbMeta->vgId = vg.vgId; - pStbRowsCxt->pCtbMeta->tableType = TSDB_CHILD_TABLE; STableMeta* pBackup = NULL; cloneTableMeta(pStmt->pTableMeta, &pBackup); taosHashPut(pStmt->pSubTableHashObj, ctbFName, strlen(ctbFName), &pBackup, POINTER_BYTES); @@ -1704,6 +1701,7 @@ static int32_t parseOneStbRow(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pSt if (code == TSDB_CODE_SUCCESS) { *pGotRow = true; } + taosArrayClear(pStbRowsCxt->aTagNames); for (int i = 0; i < taosArrayGetSize(pStbRowsCxt->aTagVals); ++i) { STagVal* p = (STagVal*)taosArrayGet(pStbRowsCxt->aTagVals, i); @@ -1714,7 +1712,9 @@ static int32_t parseOneStbRow(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pSt taosArrayClear(pStbRowsCxt->aTagVals); taosArrayClear(pStbRowsCxt->aColVals); tTagFree(pStbRowsCxt->pTag); - taosMemoryFree(pStbRowsCxt->pCtbMeta); + pStbRowsCxt->pTag = NULL; + pStbRowsCxt->pCtbMeta->uid = 0; + pStbRowsCxt->pCtbMeta->vgId = 0; tdDestroySVCreateTbReq(pStbRowsCxt->pCreateCtbReq); taosMemoryFreeClear(pStbRowsCxt->pCreateCtbReq); // child meta , vgroupid, check privilege @@ -1994,6 +1994,9 @@ static int32_t parseInsertStbClauseBottom(SInsertParseContext* pCxt, SVnodeModif collectUseDatabase(&pStbRowsCxt->stbName, pStmt->pDbFNameHashObj); pStbRowsCxt->pTagCond = pStmt->pTagCond; pStbRowsCxt->pStbMeta = pStmt->pTableMeta; + cloneTableMeta(pStbRowsCxt->pStbMeta, &pStbRowsCxt->pCtbMeta); + pStbRowsCxt->pCtbMeta->tableType = TSDB_CHILD_TABLE; + pStbRowsCxt->pCtbMeta->suid = pStbRowsCxt->pStbMeta->uid; pStbRowsCxt->aTagNames = taosArrayInit(8, TSDB_COL_NAME_LEN); pStbRowsCxt->aTagVals = taosArrayInit(8, sizeof(STagVal)); From a28b3a6e3f9c1e674ce1bca2a29b95d803095100 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Wed, 25 Oct 2023 08:50:08 +0800 Subject: [PATCH 16/45] fix: refactor --- source/libs/parser/src/parInsertSql.c | 78 ++++++++++++++------------- 1 file changed, 41 insertions(+), 37 deletions(-) diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index 8b6a92eea9..e980564031 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -1640,17 +1640,23 @@ static int32_t getStbRowValues(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pS return code; } -static int32_t parseOneStbRow(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, const char** ppSql, - SStbRowsDataContext* pStbRowsCxt, bool* pGotRow, SToken* pToken) { - int32_t code = getStbRowValues(pCxt, pStmt, ppSql, pStbRowsCxt, pGotRow, pToken); - if (code != TSDB_CODE_SUCCESS || !*pGotRow) { - return code; - } - +static int32_t processCtbAutoCreationAndCtbMeta(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, SStbRowsDataContext* pStbRowsCxt) { + int32_t code = TSDB_CODE_SUCCESS; if (pStbRowsCxt->pTagCond) { code = checkSubtablePrivilege(pStbRowsCxt->aTagVals, pStbRowsCxt->aTagNames, &pStbRowsCxt->pTagCond); } + pStbRowsCxt->pCreateCtbReq = taosMemoryCalloc(1, sizeof(SVCreateTbReq)); + if (pStbRowsCxt->pCreateCtbReq == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + if (code == TSDB_CODE_SUCCESS) { + insBuildCreateTbReq(pStbRowsCxt->pCreateCtbReq, pStbRowsCxt->ctbName.tname, pStbRowsCxt->pTag, pStbRowsCxt->pStbMeta->uid, + pStbRowsCxt->stbName.tname, pStbRowsCxt->aTagNames, getNumOfTags(pStbRowsCxt->pStbMeta), + TSDB_DEFAULT_TABLE_TTL); + pStbRowsCxt->pTag = NULL; + } + if (code == TSDB_CODE_SUCCESS) { collectUseTable(&pStbRowsCxt->ctbName, pStmt->pTableNameHashObj); @@ -1675,16 +1681,36 @@ static int32_t parseOneStbRow(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pSt taosHashPut(pStmt->pSubTableHashObj, ctbFName, strlen(ctbFName), &pBackup, POINTER_BYTES); } } + return code; +} - pStbRowsCxt->pCreateCtbReq = taosMemoryCalloc(1, sizeof(SVCreateTbReq)); - if (pStbRowsCxt->pCreateCtbReq == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; +static void clearStbRowsDataContext(SStbRowsDataContext* pStbRowsCxt) { + taosArrayClear(pStbRowsCxt->aTagNames); + for (int i = 0; i < taosArrayGetSize(pStbRowsCxt->aTagVals); ++i) { + STagVal* p = (STagVal*)taosArrayGet(pStbRowsCxt->aTagVals, i); + if (IS_VAR_DATA_TYPE(p->type)) { + taosMemoryFreeClear(p->pData); + } } + taosArrayClear(pStbRowsCxt->aTagVals); + taosArrayClear(pStbRowsCxt->aColVals); + tTagFree(pStbRowsCxt->pTag); + pStbRowsCxt->pTag = NULL; + pStbRowsCxt->pCtbMeta->uid = 0; + pStbRowsCxt->pCtbMeta->vgId = 0; + tdDestroySVCreateTbReq(pStbRowsCxt->pCreateCtbReq); + taosMemoryFreeClear(pStbRowsCxt->pCreateCtbReq); +} + +static int32_t parseOneStbRow(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, const char** ppSql, + SStbRowsDataContext* pStbRowsCxt, bool* pGotRow, SToken* pToken) { + int32_t code = getStbRowValues(pCxt, pStmt, ppSql, pStbRowsCxt, pGotRow, pToken); + if (code != TSDB_CODE_SUCCESS || !*pGotRow) { + return code; + } + if (code == TSDB_CODE_SUCCESS) { - insBuildCreateTbReq(pStbRowsCxt->pCreateCtbReq, pStbRowsCxt->ctbName.tname, pStbRowsCxt->pTag, pStbRowsCxt->pStbMeta->uid, - pStbRowsCxt->stbName.tname, pStbRowsCxt->aTagNames, getNumOfTags(pStbRowsCxt->pStbMeta), - TSDB_DEFAULT_TABLE_TTL); - pStbRowsCxt->pTag = NULL; + code = processCtbAutoCreationAndCtbMeta(pCxt, pStmt, pStbRowsCxt); } STableDataCxt* pTableDataCxt = NULL; @@ -1702,29 +1728,7 @@ static int32_t parseOneStbRow(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pSt *pGotRow = true; } - taosArrayClear(pStbRowsCxt->aTagNames); - for (int i = 0; i < taosArrayGetSize(pStbRowsCxt->aTagVals); ++i) { - STagVal* p = (STagVal*)taosArrayGet(pStbRowsCxt->aTagVals, i); - if (IS_VAR_DATA_TYPE(p->type)) { - taosMemoryFreeClear(p->pData); - } - } - taosArrayClear(pStbRowsCxt->aTagVals); - taosArrayClear(pStbRowsCxt->aColVals); - tTagFree(pStbRowsCxt->pTag); - pStbRowsCxt->pTag = NULL; - pStbRowsCxt->pCtbMeta->uid = 0; - pStbRowsCxt->pCtbMeta->vgId = 0; - tdDestroySVCreateTbReq(pStbRowsCxt->pCreateCtbReq); - taosMemoryFreeClear(pStbRowsCxt->pCreateCtbReq); - // child meta , vgroupid, check privilege - // refer to parInsertSml.c - // create or reuse table data context from tbName - // init submit data - // build create table request, refer to parseTagsClauseImpl - // build row and check table data order, refer to parseOneRow - - // clear the values, etc + clearStbRowsDataContext(pStbRowsCxt); return TSDB_CODE_SUCCESS; } From c9d1553633b7111e12ab10a39b8e6cc6a8a4a202 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Wed, 25 Oct 2023 09:06:45 +0800 Subject: [PATCH 17/45] enhance: refactor stb rows data context --- source/libs/parser/src/parInsertSql.c | 44 ++++++++++++++++++++------- 1 file changed, 33 insertions(+), 11 deletions(-) diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index e980564031..05908d2ece 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -1685,6 +1685,8 @@ static int32_t processCtbAutoCreationAndCtbMeta(SInsertParseContext* pCxt, SVnod } static void clearStbRowsDataContext(SStbRowsDataContext* pStbRowsCxt) { + if (pStbRowsCxt == NULL) return; + taosArrayClear(pStbRowsCxt->aTagNames); for (int i = 0; i < taosArrayGetSize(pStbRowsCxt->aTagVals); ++i) { STagVal* p = (STagVal*)taosArrayGet(pStbRowsCxt->aTagVals, i); @@ -1693,7 +1695,10 @@ static void clearStbRowsDataContext(SStbRowsDataContext* pStbRowsCxt) { } } taosArrayClear(pStbRowsCxt->aTagVals); + + clearColValArray(pStbRowsCxt->aColVals); taosArrayClear(pStbRowsCxt->aColVals); + tTagFree(pStbRowsCxt->pTag); pStbRowsCxt->pTag = NULL; pStbRowsCxt->pCtbMeta->uid = 0; @@ -1974,21 +1979,23 @@ static int32_t parseDataClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pS } static void destroyStbRowsDataContext(SStbRowsDataContext* pStbRowsCxt) { + if (pStbRowsCxt == NULL) return; + clearStbRowsDataContext(pStbRowsCxt); taosArrayDestroy(pStbRowsCxt->aColVals); + pStbRowsCxt->aColVals = NULL; taosArrayDestroy(pStbRowsCxt->aTagVals); + pStbRowsCxt->aTagVals = NULL; taosArrayDestroy(pStbRowsCxt->aTagNames); + pStbRowsCxt->aTagNames = NULL; insDestroyBoundColInfo(&pStbRowsCxt->boundColsInfo); tTagFree(pStbRowsCxt->pTag); - taosMemoryFree(pStbRowsCxt->pCtbMeta); + pStbRowsCxt->pTag = NULL; + taosMemoryFreeClear(pStbRowsCxt->pCtbMeta); tdDestroySVCreateTbReq(pStbRowsCxt->pCreateCtbReq); taosMemoryFreeClear(pStbRowsCxt->pCreateCtbReq); } -static int32_t parseInsertStbClauseBottom(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) { - int32_t code = TSDB_CODE_SUCCESS; - if (!pStmt->pBoundCols) { - return buildSyntaxErrMsg(&pCxt->msg, "(...tbname, ts...) bounded cols is expected for supertable insertion", pStmt->pSql); - } +static int32_t constructStbRowsDataContext(SVnodeModifyOpStmt* pStmt, SStbRowsDataContext** ppStbRowsCxt) { SStbRowsDataContext* pStbRowsCxt = taosMemoryCalloc(1, sizeof(SStbRowsDataContext)); if (!pStbRowsCxt) { return TSDB_CODE_OUT_OF_MEMORY; @@ -2004,20 +2011,35 @@ static int32_t parseInsertStbClauseBottom(SInsertParseContext* pCxt, SVnodeModif pStbRowsCxt->aTagNames = taosArrayInit(8, TSDB_COL_NAME_LEN); pStbRowsCxt->aTagVals = taosArrayInit(8, sizeof(STagVal)); - // col values and bound cols info of STableDataContext is not used TODO: remove the construction when createing table data context + // col values and bound cols info of STableDataContext is not used pStbRowsCxt->aColVals = taosArrayInit(getNumOfColumns(pStbRowsCxt->pStbMeta), sizeof(SColVal)); insInitColValues(pStbRowsCxt->pStbMeta, pStbRowsCxt->aColVals); STableComInfo tblInfo = getTableInfo(pStmt->pTableMeta); insInitBoundColsInfo(tblInfo.numOfColumns + tblInfo.numOfTags + 1, &pStbRowsCxt->boundColsInfo); - code = parseBoundColumns(pCxt, &pStmt->pBoundCols, BOUND_ALL_AND_TBNAME, pStmt->pTableMeta, &pStbRowsCxt->boundColsInfo); + *ppStbRowsCxt = pStbRowsCxt; + return TSDB_CODE_SUCCESS; +} - pStmt->pStbRowsCxt = pStbRowsCxt; +static int32_t parseInsertStbClauseBottom(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) { + int32_t code = TSDB_CODE_SUCCESS; + if (!pStmt->pBoundCols) { + return buildSyntaxErrMsg(&pCxt->msg, "(...tbname, ts...) bounded cols is expected for supertable insertion", pStmt->pSql); + } + + SStbRowsDataContext* pStbRowsCxt = NULL; + code = constructStbRowsDataContext(pStmt, &pStbRowsCxt); + + if (code == TSDB_CODE_SUCCESS) { + code = parseBoundColumns(pCxt, &pStmt->pBoundCols, BOUND_ALL_AND_TBNAME, pStmt->pTableMeta, + &pStbRowsCxt->boundColsInfo); + pStmt->pStbRowsCxt = pStbRowsCxt; + } - SRowsDataContext rowsDataCxt; - rowsDataCxt.pStbRowsCxt = pStbRowsCxt; if (TSDB_CODE_SUCCESS == code) { + SRowsDataContext rowsDataCxt; + rowsDataCxt.pStbRowsCxt = pStbRowsCxt; code = parseDataClause(pCxt, pStmt, rowsDataCxt); } destroyStbRowsDataContext(pStbRowsCxt); From 3de5b841a5536b10247c68cc106165d98849c4e3 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Wed, 25 Oct 2023 09:35:37 +0800 Subject: [PATCH 18/45] fix: large file by batches --- source/libs/parser/src/parInsertSql.c | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index 05908d2ece..ec5aeb950d 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -2037,13 +2037,16 @@ static int32_t parseInsertStbClauseBottom(SInsertParseContext* pCxt, SVnodeModif pStmt->pStbRowsCxt = pStbRowsCxt; } - if (TSDB_CODE_SUCCESS == code) { + if (code == TSDB_CODE_SUCCESS) { SRowsDataContext rowsDataCxt; rowsDataCxt.pStbRowsCxt = pStbRowsCxt; code = parseDataClause(pCxt, pStmt, rowsDataCxt); } - destroyStbRowsDataContext(pStbRowsCxt); - taosMemoryFree(pStbRowsCxt); + + if (code != TSDB_CODE_SUCCESS) { + destroyStbRowsDataContext(pStbRowsCxt); + taosMemoryFreeClear(pStbRowsCxt); + } return code; } @@ -2068,7 +2071,6 @@ static int32_t parseInsertTableClauseBottom(SInsertParseContext* pCxt, SVnodeMod } static void resetEnvPreTable(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) { - //TODO: reset env for stb syntax insDestroyBoundColInfo(&pCxt->tags); taosMemoryFreeClear(pStmt->pTableMeta); nodesDestroyNode(pStmt->pTagCond); @@ -2081,6 +2083,9 @@ static void resetEnvPreTable(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStm pStmt->usingTableProcessing = false; pStmt->fileProcessing = false; pStmt->usingTableName.type = 0; + + destroyStbRowsDataContext(pStmt->pStbRowsCxt); + taosMemoryFreeClear(pStmt->pStbRowsCxt); pStmt->stbSyntax = false; } From f0a44b31fc063adc39c4907130d1c95fae624a89 Mon Sep 17 00:00:00 2001 From: slzhou Date: Wed, 25 Oct 2023 09:56:21 +0800 Subject: [PATCH 19/45] fix: insert into super table syntax is not supported for stmt --- source/libs/parser/src/parInsertSql.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index ec5aeb950d..65ff043e23 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -2055,6 +2055,9 @@ static int32_t parseInsertStbClauseBottom(SInsertParseContext* pCxt, SVnodeModif // 1. [(tag1_name, ...)] ... // 2. VALUES ... | FILE ... static int32_t parseInsertTableClauseBottom(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) { + if (pStmt->stbSyntax && (pStmt->insertType, TSDB_QUERY_TYPE_STMT_INSERT)) { + return buildSyntaxErrMsg(&pCxt->msg, "insert into super table syntax is not supported for stmt", NULL); + } if (!pStmt->stbSyntax) { STableDataCxt* pTableCxt = NULL; int32_t code = parseSchemaClauseBottom(pCxt, pStmt, &pTableCxt); From c17300557d3332574708b1e4281fc9bc49027851 Mon Sep 17 00:00:00 2001 From: slzhou Date: Wed, 25 Oct 2023 10:56:28 +0800 Subject: [PATCH 20/45] fix: memory sanitizer error of simple insert stb --- include/libs/nodes/querynodes.h | 3 ++- source/libs/nodes/src/nodesUtilFuncs.c | 4 ++++ source/libs/parser/src/parInsertSql.c | 19 ++++++++++++++----- 3 files changed, 20 insertions(+), 6 deletions(-) diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index 21e50923a2..379f599786 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -413,8 +413,8 @@ typedef struct SVgDataBlocks { typedef void (*FFreeTableBlockHash)(SHashObj*); typedef void (*FFreeVgourpBlockArray)(SArray*); - struct SStbRowsDataContext; +typedef void (*FFreeStbRowsDataContext)(struct SStbRowsDataContext*); typedef struct SVnodeModifyOpStmt { ENodeType nodeType; ENodeType sqlNodeType; @@ -444,6 +444,7 @@ typedef struct SVnodeModifyOpStmt { bool stbSyntax; struct SStbRowsDataContext* pStbRowsCxt; + FFreeStbRowsDataContext freeStbRowsCxtFunc; } SVnodeModifyOpStmt; typedef struct SExplainOptions { diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index c5a1bfa599..926c907ae4 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -879,6 +879,10 @@ void nodesDestroyNode(SNode* pNode) { } tdDestroySVCreateTbReq(pStmt->pCreateTblReq); taosMemoryFreeClear(pStmt->pCreateTblReq); + if (pStmt->freeStbRowsCxtFunc) { + pStmt->freeStbRowsCxtFunc(pStmt->pStbRowsCxt); + } + taosMemoryFreeClear(pStmt->pStbRowsCxt); taosCloseFile(&pStmt->fp); break; } diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index 65ff043e23..f53192ad5b 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -1614,13 +1614,17 @@ static int32_t getStbRowValues(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pS } } else if (pCols->pColIndex[i] == getTbnameSchemaIndex(pStbRowsCxt->pStbMeta)) { - SColVal tbnameVal; + SColVal tbnameVal = COL_VAL_NONE(-1, TSDB_DATA_TYPE_BINARY); code = parseValueToken(pCxt, ppSql, pToken, (SSchema*)tGetTbnameColumnSchema(), getTableInfo(pStbRowsCxt->pStbMeta).precision, &tbnameVal); if (code == TSDB_CODE_SUCCESS && COL_VAL_IS_VALUE(&tbnameVal)) { - tNameAssign(&pStbRowsCxt->ctbName, &pStbRowsCxt->stbName); - tNameAddTbName(&pStbRowsCxt->ctbName, tbnameVal.value.pData, tbnameVal.value.nData); + tNameSetDbName(&pStbRowsCxt->ctbName, pStbRowsCxt->stbName.acctId, pStbRowsCxt->stbName.dbname, strlen(pStbRowsCxt->stbName.dbname)); + char ctbName[TSDB_TABLE_NAME_LEN]; + memcpy(ctbName, tbnameVal.value.pData, tbnameVal.value.nData); + ctbName[tbnameVal.value.nData] = '\0'; + tNameAddTbName(&pStbRowsCxt->ctbName, ctbName, tbnameVal.value.nData); bFoundTbName = true; + taosMemoryFreeClear(tbnameVal.value.pData); } } @@ -1637,6 +1641,9 @@ static int32_t getStbRowValues(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pS if (code == TSDB_CODE_SUCCESS && !isJsonTag) { code = tTagNew(pTagVals, 1, false, &pStbRowsCxt->pTag); } + if (code == TSDB_CODE_SUCCESS) { + *pGotRow = true; + } return code; } @@ -2055,7 +2062,7 @@ static int32_t parseInsertStbClauseBottom(SInsertParseContext* pCxt, SVnodeModif // 1. [(tag1_name, ...)] ... // 2. VALUES ... | FILE ... static int32_t parseInsertTableClauseBottom(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) { - if (pStmt->stbSyntax && (pStmt->insertType, TSDB_QUERY_TYPE_STMT_INSERT)) { + if (pStmt->stbSyntax && TSDB_QUERY_HAS_TYPE(pStmt->insertType, TSDB_QUERY_TYPE_STMT_INSERT)) { return buildSyntaxErrMsg(&pCxt->msg, "insert into super table syntax is not supported for stmt", NULL); } if (!pStmt->stbSyntax) { @@ -2215,9 +2222,11 @@ static int32_t createVnodeModifOpStmt(SInsertParseContext* pCxt, bool reentry, S TSDB_QUERY_SET_TYPE(pStmt->insertType, TSDB_QUERY_TYPE_STMT_INSERT); } pStmt->pSql = pCxt->pComCxt->pSql; + pStmt->freeHashFunc = insDestroyTableDataCxtHashMap; pStmt->freeArrayFunc = insDestroyVgroupDataCxtList; - + pStmt->freeStbRowsCxtFunc = destroyStbRowsDataContext; + if (!reentry) { pStmt->pVgroupsHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); pStmt->pTableBlockHashObj = From 29db21a3ad88e47c08289663c20647110dbc1318 Mon Sep 17 00:00:00 2001 From: slzhou Date: Wed, 25 Oct 2023 11:22:15 +0800 Subject: [PATCH 21/45] fix: reinit ctb meta and col vals before each stb row --- source/libs/parser/src/parInsertSql.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index f53192ad5b..6e1c599df4 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -1691,6 +1691,13 @@ static int32_t processCtbAutoCreationAndCtbMeta(SInsertParseContext* pCxt, SVnod return code; } +static void resetStbRowsDataContextPreStbRow(SStbRowsDataContext* pStbRowsCxt) { + pStbRowsCxt->pCtbMeta->tableType = TSDB_CHILD_TABLE; + pStbRowsCxt->pCtbMeta->suid = pStbRowsCxt->pStbMeta->uid; + + insInitColValues(pStbRowsCxt->pStbMeta, pStbRowsCxt->aColVals); +} + static void clearStbRowsDataContext(SStbRowsDataContext* pStbRowsCxt) { if (pStbRowsCxt == NULL) return; @@ -1716,6 +1723,7 @@ static void clearStbRowsDataContext(SStbRowsDataContext* pStbRowsCxt) { static int32_t parseOneStbRow(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, const char** ppSql, SStbRowsDataContext* pStbRowsCxt, bool* pGotRow, SToken* pToken) { + resetStbRowsDataContextPreStbRow(pStbRowsCxt); int32_t code = getStbRowValues(pCxt, pStmt, ppSql, pStbRowsCxt, pGotRow, pToken); if (code != TSDB_CODE_SUCCESS || !*pGotRow) { return code; @@ -2010,11 +2018,14 @@ static int32_t constructStbRowsDataContext(SVnodeModifyOpStmt* pStmt, SStbRowsDa tNameAssign(&pStbRowsCxt->stbName, &pStmt->targetTableName); collectUseTable(&pStbRowsCxt->stbName, pStmt->pTableNameHashObj); collectUseDatabase(&pStbRowsCxt->stbName, pStmt->pDbFNameHashObj); + pStbRowsCxt->pTagCond = pStmt->pTagCond; pStbRowsCxt->pStbMeta = pStmt->pTableMeta; + cloneTableMeta(pStbRowsCxt->pStbMeta, &pStbRowsCxt->pCtbMeta); pStbRowsCxt->pCtbMeta->tableType = TSDB_CHILD_TABLE; pStbRowsCxt->pCtbMeta->suid = pStbRowsCxt->pStbMeta->uid; + pStbRowsCxt->aTagNames = taosArrayInit(8, TSDB_COL_NAME_LEN); pStbRowsCxt->aTagVals = taosArrayInit(8, sizeof(STagVal)); From 1135d31caff20a9e08d08a3e16f48b3167693676 Mon Sep 17 00:00:00 2001 From: slzhou Date: Wed, 25 Oct 2023 11:39:59 +0800 Subject: [PATCH 22/45] enhance: add test case --- tests/script/tsim/insert/insert_stb.sim | 36 +++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 tests/script/tsim/insert/insert_stb.sim diff --git a/tests/script/tsim/insert/insert_stb.sim b/tests/script/tsim/insert/insert_stb.sim new file mode 100644 index 0000000000..4defbcdc26 --- /dev/null +++ b/tests/script/tsim/insert/insert_stb.sim @@ -0,0 +1,36 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sql connect + +sql create database d1 +sql create database d2 + +sql use d1; +sql create table st(ts timestamp, f int) tags(t int); +sql insert into ct1 using st tags(1) values('2021-04-19 00:00:00', 1); +sql insert into ct2 using st tags(2) values('2021-04-19 00:00:01', 2); +sql insert into ct1 values('2021-04-19 00:00:02', 2); + +sql use d2; +sql create table st(ts timestamp, f int) tags(t int); +sql insert into ct1 using st tags(1) values('2021-04-19 00:00:00', 1); +sql insert into ct2 using st tags(2) values('2021-04-19 00:00:01', 2); + +sql create database db1 vgroups 1; +sql create table db1.stb (ts timestamp, c1 int, c2 int) tags(t1 int, t2 int); + +sql use d1; +sql insert into st (tbname, ts, f, t) values('ct3', '2021-04-19 08:00:00', 3, 3); +sql insert into d1.st (tbname, ts, f) values('ct6', '2021-04-19 08:00:00', 6); +sql insert into d1.st (tbname, ts, f) values('ct6', '2021-04-19 08:00:00', 7)('ct8', '2021-04-19 08:00:00', 8); +sql insert into d1.st (tbname, ts, f, t) values('ct6', '2021-04-19 08:00:00', 9, 9)('ct8', '2021-04-19 08:00:00', 10, 10); +sql insert into d1.st (tbname, ts, f, t) values('ct6', '2021-04-19 08:00:00', 9, 9)('ct8', '2021-04-19 08:00:00', 10, 10) d2.st (tbname, ts, f, t) values('ct6', '2021-04-19 08:00:00', 9, 9)('ct8', '2021-04-19 08:00:00', 10, 10); + +sql select * from d1.st +print $rows + +sql select * from d2.st; +print $rows + +system sh/exec.sh -n dnode1 -s stop -x SIGINT From 6134346d4e3405d9ea932fda1dadd5044e00bd3b Mon Sep 17 00:00:00 2001 From: slzhou Date: Wed, 25 Oct 2023 13:20:37 +0800 Subject: [PATCH 23/45] enhance: test case insert_stb.sim --- tests/parallel_test/cases.task | 1 + tests/script/tsim/insert/insert_stb.sim | 18 +++++++++++------- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index fcd39092bd..d5ebdaf9d3 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -930,6 +930,7 @@ ,,y,script,./test.sh -f tsim/insert/delete0.sim ,,y,script,./test.sh -f tsim/insert/update1_sort_merge.sim ,,y,script,./test.sh -f tsim/insert/update2.sim +,,y,script,./test.sh -f tsim/insert/insert_stb.sim ,,y,script,./test.sh -f tsim/parser/alter__for_community_version.sim ,,y,script,./test.sh -f tsim/parser/alter_column.sim ,,y,script,./test.sh -f tsim/parser/alter_stable.sim diff --git a/tests/script/tsim/insert/insert_stb.sim b/tests/script/tsim/insert/insert_stb.sim index 4defbcdc26..cee209f4e1 100644 --- a/tests/script/tsim/insert/insert_stb.sim +++ b/tests/script/tsim/insert/insert_stb.sim @@ -21,16 +21,20 @@ sql create database db1 vgroups 1; sql create table db1.stb (ts timestamp, c1 int, c2 int) tags(t1 int, t2 int); sql use d1; -sql insert into st (tbname, ts, f, t) values('ct3', '2021-04-19 08:00:00', 3, 3); -sql insert into d1.st (tbname, ts, f) values('ct6', '2021-04-19 08:00:00', 6); -sql insert into d1.st (tbname, ts, f) values('ct6', '2021-04-19 08:00:00', 7)('ct8', '2021-04-19 08:00:00', 8); -sql insert into d1.st (tbname, ts, f, t) values('ct6', '2021-04-19 08:00:00', 9, 9)('ct8', '2021-04-19 08:00:00', 10, 10); -sql insert into d1.st (tbname, ts, f, t) values('ct6', '2021-04-19 08:00:00', 9, 9)('ct8', '2021-04-19 08:00:00', 10, 10) d2.st (tbname, ts, f, t) values('ct6', '2021-04-19 08:00:00', 9, 9)('ct8', '2021-04-19 08:00:00', 10, 10); +sql insert into st (tbname, ts, f, t) values('ct3', '2021-04-19 08:00:03', 3, 3); +sql insert into d1.st (tbname, ts, f) values('ct6', '2021-04-19 08:00:04', 6); +sql insert into d1.st (tbname, ts, f) values('ct6', '2021-04-19 08:00:05', 7)('ct8', '2021-04-19 08:00:06', 8); +sql insert into d1.st (tbname, ts, f, t) values('ct6', '2021-04-19 08:00:07', 9, 9)('ct8', '2021-04-19 08:00:08', 10, 10); +sql insert into d1.st (tbname, ts, f, t) values('ct6', '2021-04-19 08:00:09', 9, 9)('ct8', '2021-04-19 08:00:10', 10, 10) d2.st (tbname, ts, f, t) values('ct6', '2021-04-19 08:00:11', 9, 9)('ct8', '2021-04-19 08:00:12', 10, 10); sql select * from d1.st print $rows - +if $rows != 11 then + return -1 +endi sql select * from d2.st; print $rows - +if $rows != 4 then + return -1 +endi system sh/exec.sh -n dnode1 -s stop -x SIGINT From 5ca39f961cac35e902c97f86ad2f1b754277cb5f Mon Sep 17 00:00:00 2001 From: slzhou Date: Wed, 25 Oct 2023 16:30:13 +0800 Subject: [PATCH 24/45] fix: add error test case and fix address sanitizer --- source/libs/parser/src/parInsertSql.c | 10 +++------- tests/script/tsim/insert/insert_stb.sim | 6 ++++++ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index 6e1c599df4..13447acab3 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -1615,17 +1615,18 @@ static int32_t getStbRowValues(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pS } else if (pCols->pColIndex[i] == getTbnameSchemaIndex(pStbRowsCxt->pStbMeta)) { SColVal tbnameVal = COL_VAL_NONE(-1, TSDB_DATA_TYPE_BINARY); + tbnameVal.value.val = 0; code = parseValueToken(pCxt, ppSql, pToken, (SSchema*)tGetTbnameColumnSchema(), getTableInfo(pStbRowsCxt->pStbMeta).precision, &tbnameVal); - if (code == TSDB_CODE_SUCCESS && COL_VAL_IS_VALUE(&tbnameVal)) { + if (code == TSDB_CODE_SUCCESS && COL_VAL_IS_VALUE(&tbnameVal) && tbnameVal.value.nData>0) { tNameSetDbName(&pStbRowsCxt->ctbName, pStbRowsCxt->stbName.acctId, pStbRowsCxt->stbName.dbname, strlen(pStbRowsCxt->stbName.dbname)); char ctbName[TSDB_TABLE_NAME_LEN]; memcpy(ctbName, tbnameVal.value.pData, tbnameVal.value.nData); ctbName[tbnameVal.value.nData] = '\0'; tNameAddTbName(&pStbRowsCxt->ctbName, ctbName, tbnameVal.value.nData); bFoundTbName = true; - taosMemoryFreeClear(tbnameVal.value.pData); } + taosMemoryFreeClear(tbnameVal.value.pData); } if (code == TSDB_CODE_SUCCESS && i < pCols->numOfBound - 1) { @@ -2061,11 +2062,6 @@ static int32_t parseInsertStbClauseBottom(SInsertParseContext* pCxt, SVnodeModif code = parseDataClause(pCxt, pStmt, rowsDataCxt); } - if (code != TSDB_CODE_SUCCESS) { - destroyStbRowsDataContext(pStbRowsCxt); - taosMemoryFreeClear(pStbRowsCxt); - } - return code; } diff --git a/tests/script/tsim/insert/insert_stb.sim b/tests/script/tsim/insert/insert_stb.sim index cee209f4e1..afe1976c79 100644 --- a/tests/script/tsim/insert/insert_stb.sim +++ b/tests/script/tsim/insert/insert_stb.sim @@ -37,4 +37,10 @@ print $rows if $rows != 4 then return -1 endi + +sql_error insert into d2.st values(now, 1, 1) +sql_error insert into d2.st(ts, f) values(now, 1); +sql_error insert into d2.st(ts, f, tbname) values(now, 1); +sql_error insert into d2.st(ts, f, tbname) values(now, 1, ''); +sql_error insert into d2.st(ts, tbname) values(now, 1, 34) system sh/exec.sh -n dnode1 -s stop -x SIGINT From 503761a53c3cca569daea2d7182a87c5306c005d Mon Sep 17 00:00:00 2001 From: slzhou Date: Thu, 26 Oct 2023 07:39:09 +0800 Subject: [PATCH 25/45] enhance: add test case --- tests/script/tsim/insert/insert_stb.sim | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/script/tsim/insert/insert_stb.sim b/tests/script/tsim/insert/insert_stb.sim index afe1976c79..7e6db51823 100644 --- a/tests/script/tsim/insert/insert_stb.sim +++ b/tests/script/tsim/insert/insert_stb.sim @@ -38,6 +38,18 @@ if $rows != 4 then return -1 endi +sql insert into d2.st(ts, f, tbname) values('2021-04-19 08:00:13', 1, 'ct1') d1.ct1 values('2021-04-19 08:00:14', 1); + +sql select * from d1.st +print $rows +if $rows != 12 then + return -1 +endi +sql select * from d2.st; +print $rows +if $rows != 5 then + return -1 +endi sql_error insert into d2.st values(now, 1, 1) sql_error insert into d2.st(ts, f) values(now, 1); sql_error insert into d2.st(ts, f, tbname) values(now, 1); From 779c37b4256fb546673312e5e7465458b484e585 Mon Sep 17 00:00:00 2001 From: slzhou Date: Thu, 26 Oct 2023 14:23:21 +0800 Subject: [PATCH 26/45] enhance: change to invalid operation for stmt error --- source/libs/parser/src/parInsertSql.c | 2 +- tests/parallel_test/cases.task | 1 + tests/system-test/1-insert/insert_stb.py | 206 +++++++++++++++++++++++ 3 files changed, 208 insertions(+), 1 deletion(-) create mode 100644 tests/system-test/1-insert/insert_stb.py diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index 13447acab3..f8cb019ced 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -2070,7 +2070,7 @@ static int32_t parseInsertStbClauseBottom(SInsertParseContext* pCxt, SVnodeModif // 2. VALUES ... | FILE ... static int32_t parseInsertTableClauseBottom(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) { if (pStmt->stbSyntax && TSDB_QUERY_HAS_TYPE(pStmt->insertType, TSDB_QUERY_TYPE_STMT_INSERT)) { - return buildSyntaxErrMsg(&pCxt->msg, "insert into super table syntax is not supported for stmt", NULL); + return buildInvalidOperationMsg(&pCxt->msg, "insert into super table syntax is not supported for stmt"); } if (!pStmt->stbSyntax) { STableDataCxt* pTableCxt = NULL; diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 476322ab04..bd20b61706 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -83,6 +83,7 @@ ,,n,system-test,python3 ./test.py -f 7-tmq/tmqDropConsumer.py +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/insert_stb.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/delete_stable.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/out_of_order.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/out_of_order.py diff --git a/tests/system-test/1-insert/insert_stb.py b/tests/system-test/1-insert/insert_stb.py new file mode 100644 index 0000000000..695a5abef7 --- /dev/null +++ b/tests/system-test/1-insert/insert_stb.py @@ -0,0 +1,206 @@ +import sys +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import tdDnodes +from math import inf +import taos + +class TDTestCase: + def caseDescription(self): + ''' + case1: [TS-3932] insert into stb + ''' + return + + def init(self, conn, logSql, replicaVer=1): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), True) + self.conn = conn + + + def restartTaosd(self, index=1, dbname="db"): + tdDnodes.stop(index) + tdDnodes.startWithoutSleep(index) + tdSql.execute(f"use insert_stb") + + + def run_normal(self): + print("running {}".format(__file__)) + tdSql.execute("drop database if exists insert_stb") + tdSql.execute("create database if not exists insert_stb") + tdSql.execute('use insert_stb') + tdSql.execute('create database d1') + + tdSql.execute('create database d2') + + tdSql.execute('use d1;') + + tdSql.execute('create table st(ts timestamp, f int) tags(t int);') + + tdSql.execute("insert into ct1 using st tags(1) values('2021-04-19 00:00:00', 1);") + + tdSql.execute("insert into ct2 using st tags(2) values('2021-04-19 00:00:01', 2);") + + tdSql.execute("insert into ct1 values('2021-04-19 00:00:02', 2);") + + tdSql.execute('use d2;') + + tdSql.execute('create table st(ts timestamp, f int) tags(t int);') + + tdSql.execute("insert into ct1 using st tags(1) values('2021-04-19 00:00:00', 1);") + + tdSql.execute("insert into ct2 using st tags(2) values('2021-04-19 00:00:01', 2);") + + tdSql.execute('create database db1 vgroups 1;') + + tdSql.execute('create table db1.stb (ts timestamp, c1 int, c2 int) tags(t1 int, t2 int);') + + tdSql.execute('use d1;') + + tdSql.execute("insert into st (tbname, ts, f, t) values('ct3', '2021-04-19 08:00:03', 3, 3);") + + tdSql.execute("insert into d1.st (tbname, ts, f) values('ct6', '2021-04-19 08:00:04', 6);") + + tdSql.execute("insert into d1.st (tbname, ts, f) values('ct6', '2021-04-19 08:00:05', 7)('ct8', '2021-04-19 08:00:06', 8);") + + tdSql.execute("insert into d1.st (tbname, ts, f, t) values('ct6', '2021-04-19 08:00:07', 9, 9)('ct8', '2021-04-19 08:00:08', 10, 10);") + + tdSql.execute("insert into d1.st (tbname, ts, f, t) values('ct6', '2021-04-19 08:00:09', 9, 9)('ct8', '2021-04-19 08:00:10', 10, 10) d2.st (tbname, ts, f, t) values('ct6', '2021-04-19 08:00:11', 9, 9)('ct8', '2021-04-19 08:00:12', 10, 10);") + + tdSql.query('select * from d1.st order by ts;') + tdSql.checkRows(11) + tdSql.checkData(0, 0, datetime.datetime(2021, 4, 19, 0, 0)) + tdSql.checkData(0, 1, 1) + tdSql.checkData(0, 2, 1) + tdSql.checkData(1, 0, datetime.datetime(2021, 4, 19, 0, 0, 1)) + tdSql.checkData(1, 1, 2) + tdSql.checkData(1, 2, 2) + tdSql.checkData(2, 0, datetime.datetime(2021, 4, 19, 0, 0, 2)) + tdSql.checkData(2, 1, 2) + tdSql.checkData(2, 2, 1) + tdSql.checkData(3, 0, datetime.datetime(2021, 4, 19, 8, 0, 3)) + tdSql.checkData(3, 1, 3) + tdSql.checkData(3, 2, 3) + tdSql.checkData(4, 0, datetime.datetime(2021, 4, 19, 8, 0, 4)) + tdSql.checkData(4, 1, 6) + tdSql.checkData(4, 2, None) + tdSql.checkData(5, 0, datetime.datetime(2021, 4, 19, 8, 0, 5)) + tdSql.checkData(5, 1, 7) + tdSql.checkData(5, 2, None) + tdSql.checkData(6, 0, datetime.datetime(2021, 4, 19, 8, 0, 6)) + tdSql.checkData(6, 1, 8) + tdSql.checkData(6, 2, None) + tdSql.checkData(7, 0, datetime.datetime(2021, 4, 19, 8, 0, 7)) + tdSql.checkData(7, 1, 9) + tdSql.checkData(7, 2, None) + tdSql.checkData(8, 0, datetime.datetime(2021, 4, 19, 8, 0, 8)) + tdSql.checkData(8, 1, 10) + tdSql.checkData(8, 2, None) + tdSql.checkData(9, 0, datetime.datetime(2021, 4, 19, 8, 0, 9)) + tdSql.checkData(9, 1, 9) + tdSql.checkData(9, 2, None) + tdSql.checkData(10, 0, datetime.datetime(2021, 4, 19, 8, 0, 10)) + tdSql.checkData(10, 1, 10) + tdSql.checkData(10, 2, None) + + tdSql.query('select * from d2.st order by ts;') + tdSql.checkRows(4) + tdSql.checkData(0, 0, datetime.datetime(2021, 4, 19, 0, 0)) + tdSql.checkData(0, 1, 1) + tdSql.checkData(0, 2, 1) + tdSql.checkData(1, 0, datetime.datetime(2021, 4, 19, 0, 0, 1)) + tdSql.checkData(1, 1, 2) + tdSql.checkData(1, 2, 2) + tdSql.checkData(2, 0, datetime.datetime(2021, 4, 19, 8, 0, 11)) + tdSql.checkData(2, 1, 9) + tdSql.checkData(2, 2, 9) + tdSql.checkData(3, 0, datetime.datetime(2021, 4, 19, 8, 0, 12)) + tdSql.checkData(3, 1, 10) + tdSql.checkData(3, 2, 10) + + tdSql.execute("insert into d2.st(ts, f, tbname) values('2021-04-19 08:00:13', 1, 'ct1') d1.ct1 values('2021-04-19 08:00:14', 1);") + + tdSql.query('select * from d1.st order by ts;') + tdSql.checkRows(12) + tdSql.checkData(0, 0, datetime.datetime(2021, 4, 19, 0, 0)) + tdSql.checkData(0, 1, 1) + tdSql.checkData(0, 2, 1) + tdSql.checkData(1, 0, datetime.datetime(2021, 4, 19, 0, 0, 1)) + tdSql.checkData(1, 1, 2) + tdSql.checkData(1, 2, 2) + tdSql.checkData(2, 0, datetime.datetime(2021, 4, 19, 0, 0, 2)) + tdSql.checkData(2, 1, 2) + tdSql.checkData(2, 2, 1) + tdSql.checkData(3, 0, datetime.datetime(2021, 4, 19, 8, 0, 3)) + tdSql.checkData(3, 1, 3) + tdSql.checkData(3, 2, 3) + tdSql.checkData(4, 0, datetime.datetime(2021, 4, 19, 8, 0, 4)) + tdSql.checkData(4, 1, 6) + tdSql.checkData(4, 2, None) + tdSql.checkData(5, 0, datetime.datetime(2021, 4, 19, 8, 0, 5)) + tdSql.checkData(5, 1, 7) + tdSql.checkData(5, 2, None) + tdSql.checkData(6, 0, datetime.datetime(2021, 4, 19, 8, 0, 6)) + tdSql.checkData(6, 1, 8) + tdSql.checkData(6, 2, None) + tdSql.checkData(7, 0, datetime.datetime(2021, 4, 19, 8, 0, 7)) + tdSql.checkData(7, 1, 9) + tdSql.checkData(7, 2, None) + tdSql.checkData(8, 0, datetime.datetime(2021, 4, 19, 8, 0, 8)) + tdSql.checkData(8, 1, 10) + tdSql.checkData(8, 2, None) + tdSql.checkData(9, 0, datetime.datetime(2021, 4, 19, 8, 0, 9)) + tdSql.checkData(9, 1, 9) + tdSql.checkData(9, 2, None) + tdSql.checkData(10, 0, datetime.datetime(2021, 4, 19, 8, 0, 10)) + tdSql.checkData(10, 1, 10) + tdSql.checkData(10, 2, None) + tdSql.checkData(11, 0, datetime.datetime(2021, 4, 19, 8, 0, 14)) + tdSql.checkData(11, 1, 1) + tdSql.checkData(11, 2, 1) + + tdSql.query('select * from d2.st order by ts;') + tdSql.checkRows(5) + tdSql.checkData(0, 0, datetime.datetime(2021, 4, 19, 0, 0)) + tdSql.checkData(0, 1, 1) + tdSql.checkData(0, 2, 1) + tdSql.checkData(1, 0, datetime.datetime(2021, 4, 19, 0, 0, 1)) + tdSql.checkData(1, 1, 2) + tdSql.checkData(1, 2, 2) + tdSql.checkData(2, 0, datetime.datetime(2021, 4, 19, 8, 0, 11)) + tdSql.checkData(2, 1, 9) + tdSql.checkData(2, 2, 9) + tdSql.checkData(3, 0, datetime.datetime(2021, 4, 19, 8, 0, 12)) + tdSql.checkData(3, 1, 10) + tdSql.checkData(3, 2, 10) + tdSql.checkData(4, 0, datetime.datetime(2021, 4, 19, 8, 0, 13)) + tdSql.checkData(4, 1, 1) + tdSql.checkData(4, 2, None) + + def run_stmt_error(self): + conn = self.conn + conn.select_db('insert_stb') + conn.execute('create table stb9(ts timestamp, f int) tags (t int)') + try: + stmt = conn.statement("insert into stb9(tbname, f, t) values('ctb91', 1, ?)") + params = taos.new_bind_params(1) + params[0].int(1) + stmt.bind_param(params) + stmt.execute() + result = stmt.use_result() + except Exception as err: + print(str(err)) + + + def run(self): + self.run_normal() + self.run_stmt_error() + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) From 65386ed2374b4d4f7fd07094e534cc518e42f1cc Mon Sep 17 00:00:00 2001 From: slzhou Date: Fri, 27 Oct 2023 08:26:04 +0800 Subject: [PATCH 27/45] enhance: add test case --- tests/script/tsim/insert/insert_stb.sim | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/script/tsim/insert/insert_stb.sim b/tests/script/tsim/insert/insert_stb.sim index 7e6db51823..890d8937a5 100644 --- a/tests/script/tsim/insert/insert_stb.sim +++ b/tests/script/tsim/insert/insert_stb.sim @@ -11,6 +11,7 @@ sql create table st(ts timestamp, f int) tags(t int); sql insert into ct1 using st tags(1) values('2021-04-19 00:00:00', 1); sql insert into ct2 using st tags(2) values('2021-04-19 00:00:01', 2); sql insert into ct1 values('2021-04-19 00:00:02', 2); +sql create table st2(ts timestamp, f int) tags(t int); sql use d2; sql create table st(ts timestamp, f int) tags(t int); @@ -55,4 +56,5 @@ sql_error insert into d2.st(ts, f) values(now, 1); sql_error insert into d2.st(ts, f, tbname) values(now, 1); sql_error insert into d2.st(ts, f, tbname) values(now, 1, ''); sql_error insert into d2.st(ts, tbname) values(now, 1, 34) +sql_error insert into st using st2 tags(2) values(now,1); system sh/exec.sh -n dnode1 -s stop -x SIGINT From 42618a953a4894bcc020966f817089d5a6b5af3f Mon Sep 17 00:00:00 2001 From: slzhou Date: Fri, 27 Oct 2023 09:07:41 +0800 Subject: [PATCH 28/45] enhance: add test case --- tests/system-test/1-insert/insert_stb.py | 169 +++++++++++++++++++++++ 1 file changed, 169 insertions(+) diff --git a/tests/system-test/1-insert/insert_stb.py b/tests/system-test/1-insert/insert_stb.py index 695a5abef7..6e004a96e9 100644 --- a/tests/system-test/1-insert/insert_stb.py +++ b/tests/system-test/1-insert/insert_stb.py @@ -178,6 +178,174 @@ class TDTestCase: tdSql.checkData(4, 1, 1) tdSql.checkData(4, 2, None) + def run_insert_stb(self): + print("running {}".format('insert_stb')) + self.conn.select_db('insert_stb') + tdSql.execute('create table stb1 (ts timestamp, c1 bool, c2 tinyint, c3 smallint, c4 int, c5 bigint, c6 float, c7 double, c8 binary(10), c9 nchar(10), c10 tinyint unsigned, c11 smallint unsigned, c12 int unsigned, c13 bigint unsigned) TAGS(t1 int, t2 binary(10), t3 double);') + + tdSql.execute('insert into stb1(ts,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,t1,t2,t3,tbname) values (\'2021-11-11 09:00:00\',true,1,1,1,1,1,1,"123","1234",1,1,1,1, 1, \'1\', 1.0, \'tb1\');') + + tdSql.execute("insert into stb1(ts,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,t1,t2,t3,tbname) values ('2021-11-11 09:00:01',true,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL, 2, '2', 2.0, 'tb1');") + + tdSql.execute('insert into stb1(ts,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,t1,t2,t3,tbname) values (\'2021-11-11 09:00:02\',true,2,NULL,2,NULL,2,NULL,"234",NULL,2,NULL,2,NULL, 2, \'2\', 2.0, \'tb2\');') + + tdSql.execute('insert into stb1(ts,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,t1,t2,t3,tbname) values (\'2021-11-11 09:00:03\',false,NULL,3,NULL,3,NULL,3,NULL,"3456",NULL,3,NULL,3, 3, \'3\', 3.0, \'tb3\');') + + tdSql.execute('insert into stb1(ts,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,t1,t2,t3,tbname) values (\'2021-11-11 09:00:04\',true,4,4,4,4,4,4,"456","4567",4,4,4,4, 4, \'4.0\', 4.0, \'tb4\');') + + tdSql.execute('insert into stb1(ts,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,t1,t2,t3,tbname) values (\'2021-11-11 09:00:05\',true,127,32767,2147483647,9223372036854775807,3.402823466e+38,1.79769e+308,"567","5678",254,65534,4294967294,9223372036854775807, 5, \'5\', 5, \'max\' );') + + tdSql.execute('insert into stb1(ts,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,t1,t2,t3,tbname) values (\'2021-11-11 09:00:06\',true,-127,-32767,-2147483647,-9223372036854775807,-3.402823466e+38,-1.79769e+308,"678","6789",0,0,0,0, 6, \'6\', 6, \'min\');') + + tdSql.execute('insert into stb1(ts,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,tbname,t1,t2,t3) values (\'2021-11-11 09:00:07\',true,-127,-32767,-2147483647,-9223372036854775807,-3.402823466e+38,-1.79769e+308,"678","6789",0,0,0,0, \'min\', 6, \'6\', 6);') + + tdSql.query('select tbname,* from stb1 order by ts;') + tdSql.checkRows(8) + tdSql.checkData(0, 0, 'tb1') + tdSql.checkData(0, 1, datetime.datetime(2021, 11, 11, 9, 0)) + tdSql.checkData(0, 2, True) + tdSql.checkData(0, 3, 1) + tdSql.checkData(0, 4, 1) + tdSql.checkData(0, 5, 1) + tdSql.checkData(0, 6, 1) + tdSql.checkData(0, 7, 1.0) + tdSql.checkData(0, 8, 1.0) + tdSql.checkData(0, 9, '123') + tdSql.checkData(0, 10, '1234') + tdSql.checkData(0, 11, 1) + tdSql.checkData(0, 12, 1) + tdSql.checkData(0, 13, 1) + tdSql.checkData(0, 14, 1) + tdSql.checkData(0, 15, 1) + tdSql.checkData(0, 16, '1') + tdSql.checkData(0, 17, 1.0) + tdSql.checkData(1, 0, 'tb1') + tdSql.checkData(1, 1, datetime.datetime(2021, 11, 11, 9, 0, 1)) + tdSql.checkData(1, 2, True) + tdSql.checkData(1, 3, None) + tdSql.checkData(1, 4, None) + tdSql.checkData(1, 5, None) + tdSql.checkData(1, 6, None) + tdSql.checkData(1, 7, None) + tdSql.checkData(1, 8, None) + tdSql.checkData(1, 9, None) + tdSql.checkData(1, 10, None) + tdSql.checkData(1, 11, None) + tdSql.checkData(1, 12, None) + tdSql.checkData(1, 13, None) + tdSql.checkData(1, 14, None) + tdSql.checkData(1, 15, 1) + tdSql.checkData(1, 16, '1') + tdSql.checkData(1, 17, 1.0) + tdSql.checkData(2, 0, 'tb2') + tdSql.checkData(2, 1, datetime.datetime(2021, 11, 11, 9, 0, 2)) + tdSql.checkData(2, 2, True) + tdSql.checkData(2, 3, 2) + tdSql.checkData(2, 4, None) + tdSql.checkData(2, 5, 2) + tdSql.checkData(2, 6, None) + tdSql.checkData(2, 7, 2.0) + tdSql.checkData(2, 8, None) + tdSql.checkData(2, 9, '234') + tdSql.checkData(2, 10, None) + tdSql.checkData(2, 11, 2) + tdSql.checkData(2, 12, None) + tdSql.checkData(2, 13, 2) + tdSql.checkData(2, 14, None) + tdSql.checkData(2, 15, 2) + tdSql.checkData(2, 16, '2') + tdSql.checkData(2, 17, 2.0) + tdSql.checkData(3, 0, 'tb3') + tdSql.checkData(3, 1, datetime.datetime(2021, 11, 11, 9, 0, 3)) + tdSql.checkData(3, 2, False) + tdSql.checkData(3, 3, None) + tdSql.checkData(3, 4, 3) + tdSql.checkData(3, 5, None) + tdSql.checkData(3, 6, 3) + tdSql.checkData(3, 7, None) + tdSql.checkData(3, 8, 3.0) + tdSql.checkData(3, 9, None) + tdSql.checkData(3, 10, '3456') + tdSql.checkData(3, 11, None) + tdSql.checkData(3, 12, 3) + tdSql.checkData(3, 13, None) + tdSql.checkData(3, 14, 3) + tdSql.checkData(3, 15, 3) + tdSql.checkData(3, 16, '3') + tdSql.checkData(3, 17, 3.0) + tdSql.checkData(4, 0, 'tb4') + tdSql.checkData(4, 1, datetime.datetime(2021, 11, 11, 9, 0, 4)) + tdSql.checkData(4, 2, True) + tdSql.checkData(4, 3, 4) + tdSql.checkData(4, 4, 4) + tdSql.checkData(4, 5, 4) + tdSql.checkData(4, 6, 4) + tdSql.checkData(4, 7, 4.0) + tdSql.checkData(4, 8, 4.0) + tdSql.checkData(4, 9, '456') + tdSql.checkData(4, 10, '4567') + tdSql.checkData(4, 11, 4) + tdSql.checkData(4, 12, 4) + tdSql.checkData(4, 13, 4) + tdSql.checkData(4, 14, 4) + tdSql.checkData(4, 15, 4) + tdSql.checkData(4, 16, '4.0') + tdSql.checkData(4, 17, 4.0) + tdSql.checkData(5, 0, 'max') + tdSql.checkData(5, 1, datetime.datetime(2021, 11, 11, 9, 0, 5)) + tdSql.checkData(5, 2, True) + tdSql.checkData(5, 3, 127) + tdSql.checkData(5, 4, 32767) + tdSql.checkData(5, 5, 2147483647) + tdSql.checkData(5, 6, 9223372036854775807) + tdSql.checkData(5, 7, 3.4028234663852886e+38) + tdSql.checkData(5, 8, 1.79769e+308) + tdSql.checkData(5, 9, '567') + tdSql.checkData(5, 10, '5678') + tdSql.checkData(5, 11, 254) + tdSql.checkData(5, 12, 65534) + tdSql.checkData(5, 13, 4294967294) + tdSql.checkData(5, 14, 9223372036854775807) + tdSql.checkData(5, 15, 5) + tdSql.checkData(5, 16, '5') + tdSql.checkData(5, 17, 5.0) + tdSql.checkData(6, 0, 'min') + tdSql.checkData(6, 1, datetime.datetime(2021, 11, 11, 9, 0, 6)) + tdSql.checkData(6, 2, True) + tdSql.checkData(6, 3, -127) + tdSql.checkData(6, 4, -32767) + tdSql.checkData(6, 5, -2147483647) + tdSql.checkData(6, 6, -9223372036854775807) + tdSql.checkData(6, 7, -3.4028234663852886e+38) + tdSql.checkData(6, 8, -1.79769e+308) + tdSql.checkData(6, 9, '678') + tdSql.checkData(6, 10, '6789') + tdSql.checkData(6, 11, 0) + tdSql.checkData(6, 12, 0) + tdSql.checkData(6, 13, 0) + tdSql.checkData(6, 14, 0) + tdSql.checkData(6, 15, 6) + tdSql.checkData(6, 16, '6') + tdSql.checkData(6, 17, 6.0) + tdSql.checkData(7, 0, 'min') + tdSql.checkData(7, 1, datetime.datetime(2021, 11, 11, 9, 0, 7)) + tdSql.checkData(7, 2, True) + tdSql.checkData(7, 3, -127) + tdSql.checkData(7, 4, -32767) + tdSql.checkData(7, 5, -2147483647) + tdSql.checkData(7, 6, -9223372036854775807) + tdSql.checkData(7, 7, -3.4028234663852886e+38) + tdSql.checkData(7, 8, -1.79769e+308) + tdSql.checkData(7, 9, '678') + tdSql.checkData(7, 10, '6789') + tdSql.checkData(7, 11, 0) + tdSql.checkData(7, 12, 0) + tdSql.checkData(7, 13, 0) + tdSql.checkData(7, 14, 0) + tdSql.checkData(7, 15, 6) + tdSql.checkData(7, 16, '6') + tdSql.checkData(7, 17, 6.0) + def run_stmt_error(self): conn = self.conn conn.select_db('insert_stb') @@ -195,6 +363,7 @@ class TDTestCase: def run(self): self.run_normal() + self.run_insert_stb() self.run_stmt_error() From e9a7f6e322982b6d08ddb9cc8c8846846e7a62e4 Mon Sep 17 00:00:00 2001 From: slzhou Date: Mon, 30 Oct 2023 15:54:10 +0800 Subject: [PATCH 29/45] fix: invalid schema error --- source/libs/parser/src/parInsertSql.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index f8cb019ced..85ce0f8225 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -1685,7 +1685,7 @@ static int32_t processCtbAutoCreationAndCtbMeta(SInsertParseContext* pCxt, SVnod pStbRowsCxt->pCtbMeta->uid = taosHashGetSize(pStmt->pSubTableHashObj) + 1; pStbRowsCxt->pCtbMeta->vgId = vg.vgId; STableMeta* pBackup = NULL; - cloneTableMeta(pStmt->pTableMeta, &pBackup); + cloneTableMeta(pStbRowsCxt->pCtbMeta, &pBackup); taosHashPut(pStmt->pSubTableHashObj, ctbFName, strlen(ctbFName), &pBackup, POINTER_BYTES); } } From 45ae3727a7a6dfb29a8a3d169fb5d53b46f8ad92 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Tue, 31 Oct 2023 10:48:10 +0800 Subject: [PATCH 30/45] fix: performace improvement --- source/libs/parser/src/parInsertSql.c | 108 ++++++++++++++++++-------- 1 file changed, 75 insertions(+), 33 deletions(-) diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index 85ce0f8225..231e28878a 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -230,13 +230,12 @@ static int32_t parseBoundColumns(SInsertParseContext* pCxt, const char** pSql, E token.z = tmpTokenBuf; token.n = strdequote(token.z); - if (boundColsType == BOUND_ALL_AND_TBNAME) { - if (token.n == strlen("tbname") && (strcasecmp(token.z, "tbname") == 0)) { - pBoundInfo->pColIndex[pBoundInfo->numOfBound] = tbnameSchemaIndex; - pUseCols[tbnameSchemaIndex] = true; - ++pBoundInfo->numOfBound; - continue; - } + if (boundColsType == BOUND_ALL_AND_TBNAME && + token.n == strlen("tbname") && (strcasecmp(token.z, "tbname") == 0)) { + pBoundInfo->pColIndex[pBoundInfo->numOfBound] = tbnameSchemaIndex; + pUseCols[tbnameSchemaIndex] = true; + ++pBoundInfo->numOfBound; + continue; } int16_t t = lastColIdx + 1; int16_t index = insFindCol(&token, t, pBoundInfo->numOfCols, pSchema); @@ -1567,6 +1566,7 @@ typedef struct SStbRowsDataContext { STag* pTag; STableMeta* pCtbMeta; SVCreateTbReq* pCreateCtbReq; + bool hasTimestampTag; } SStbRowsDataContext; typedef union SRowsDataContext{ @@ -1575,18 +1575,25 @@ typedef union SRowsDataContext{ } SRowsDataContext; static int32_t getStbRowValues(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, const char** ppSql, - SStbRowsDataContext* pStbRowsCxt, bool* pGotRow, SToken* pToken) { + SStbRowsDataContext* pStbRowsCxt, bool* pGotRow, + SToken* pToken, bool *pCtbFirst) { SBoundColInfo* pCols = &pStbRowsCxt->boundColsInfo; SSchema* pSchemas = getTableColumnSchema(pStbRowsCxt->pStbMeta); bool isJsonTag = false; - SArray* pTagName = pStbRowsCxt->aTagNames; + SArray* pTagNames = pStbRowsCxt->aTagNames; SArray* pTagVals = pStbRowsCxt->aTagVals; bool bFoundTbName = false; const char* pOrigSql = *ppSql; int32_t code = TSDB_CODE_SUCCESS; + + bool canParseTagsAfter = !pStbRowsCxt->pTagCond && !pStbRowsCxt->hasTimestampTag; + SToken tagTokens[TSDB_MAX_TAGS] = {0}; + SSchema* tagSchemas[TSDB_MAX_TAGS] = {0}; + int numOfTagTokens = 0; + for (int i = 0; i < pCols->numOfBound && code == TSDB_CODE_SUCCESS; ++i) { const char* pTmpSql = *ppSql; bool ignoreComma = false; @@ -1608,9 +1615,15 @@ static int32_t getStbRowValues(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pS } else if (pCols->pColIndex[i] < getTbnameSchemaIndex(pStbRowsCxt->pStbMeta)) { SSchema* pTagSchema = &pSchemas[pCols->pColIndex[i]]; isJsonTag = pTagSchema->type == TSDB_DATA_TYPE_JSON; - code = checkAndTrimValue(pToken, pCxt->tmpTokenBuf, &pCxt->msg); - if (code == TSDB_CODE_SUCCESS) { - code = parseTagValue(pCxt, pStmt, ppSql, pTagSchema, pToken, pTagName, pTagVals, &pStbRowsCxt->pTag); + if (canParseTagsAfter) { + tagTokens[numOfTagTokens] = *pToken; + tagSchemas[numOfTagTokens] = pTagSchema; + ++numOfTagTokens; + } else { + code = checkAndTrimValue(pToken, pCxt->tmpTokenBuf, &pCxt->msg); + if (code == TSDB_CODE_SUCCESS) { + code = parseTagValue(pCxt, pStmt, ppSql, pTagSchema, pToken, pTagNames, pTagVals, &pStbRowsCxt->pTag); + } } } else if (pCols->pColIndex[i] == getTbnameSchemaIndex(pStbRowsCxt->pStbMeta)) { @@ -1639,9 +1652,36 @@ static int32_t getStbRowValues(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pS if (!bFoundTbName) { code = buildSyntaxErrMsg(&pCxt->msg, "tbname value expected", pOrigSql); } - if (code == TSDB_CODE_SUCCESS && !isJsonTag) { - code = tTagNew(pTagVals, 1, false, &pStbRowsCxt->pTag); + + bool ctbFirst = true; + if (code == TSDB_CODE_SUCCESS) { + char ctbFName[TSDB_TABLE_FNAME_LEN]; + tNameExtractFullName(&pStbRowsCxt->ctbName, ctbFName); + STableMeta** pCtbMeta = taosHashGet(pStmt->pSubTableHashObj, ctbFName, strlen(ctbFName)); + ctbFirst = *pCtbFirst = (pCtbMeta == NULL); + if (!ctbFirst) { + pStbRowsCxt->pCtbMeta = *pCtbMeta; + } } + if (code == TSDB_CODE_SUCCESS && ctbFirst) { + for (int32_t i = 0; code == TSDB_CODE_SUCCESS && i < numOfTagTokens; ++i) { + SToken* pTagToken = tagTokens + i; + SSchema* pTagSchema = tagSchemas[i]; + code = checkAndTrimValue(pTagToken, pCxt->tmpTokenBuf, &pCxt->msg); + if (code == TSDB_CODE_SUCCESS) { + code = parseTagValue(pCxt, pStmt, NULL, pTagSchema, pTagToken, pStbRowsCxt->aTagNames, pStbRowsCxt->aTagVals, + &pStbRowsCxt->pTag); + } + } + if (code == TSDB_CODE_SUCCESS && !isJsonTag) { + code = tTagNew(pStbRowsCxt->aTagVals, 1, false, &pStbRowsCxt->pTag); + } + } + + if (code == TSDB_CODE_SUCCESS && pStbRowsCxt->pTagCond) { + code = checkSubtablePrivilege(pStbRowsCxt->aTagVals, pStbRowsCxt->aTagNames, &pStbRowsCxt->pTagCond); + } + if (code == TSDB_CODE_SUCCESS) { *pGotRow = true; } @@ -1650,9 +1690,6 @@ static int32_t getStbRowValues(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pS static int32_t processCtbAutoCreationAndCtbMeta(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, SStbRowsDataContext* pStbRowsCxt) { int32_t code = TSDB_CODE_SUCCESS; - if (pStbRowsCxt->pTagCond) { - code = checkSubtablePrivilege(pStbRowsCxt->aTagVals, pStbRowsCxt->aTagNames, &pStbRowsCxt->pTagCond); - } pStbRowsCxt->pCreateCtbReq = taosMemoryCalloc(1, sizeof(SVCreateTbReq)); if (pStbRowsCxt->pCreateCtbReq == NULL) { @@ -1666,28 +1703,24 @@ static int32_t processCtbAutoCreationAndCtbMeta(SInsertParseContext* pCxt, SVnod } if (code == TSDB_CODE_SUCCESS) { - collectUseTable(&pStbRowsCxt->ctbName, pStmt->pTableNameHashObj); - char ctbFName[TSDB_TABLE_FNAME_LEN]; tNameExtractFullName(&pStbRowsCxt->ctbName, ctbFName); - STableMeta** pCtbMeta = taosHashGet(pStmt->pSubTableHashObj, ctbFName, strlen(ctbFName)); - if (NULL != pCtbMeta) { - pStbRowsCxt->pCtbMeta->uid = (*pCtbMeta)->uid; - pStbRowsCxt->pCtbMeta->vgId = (*pCtbMeta)->vgId; - } else { - SVgroupInfo vg; - SRequestConnInfo conn = {.pTrans = pCxt->pComCxt->pTransporter, - .requestId = pCxt->pComCxt->requestId, - .requestObjRefId = pCxt->pComCxt->requestRid, - .mgmtEps = pCxt->pComCxt->mgmtEpSet}; - code = catalogGetTableHashVgroup(pCxt->pComCxt->pCatalog, &conn, &pStmt->targetTableName, &vg); + SVgroupInfo vg; + SRequestConnInfo conn = {.pTrans = pCxt->pComCxt->pTransporter, + .requestId = pCxt->pComCxt->requestId, + .requestObjRefId = pCxt->pComCxt->requestRid, + .mgmtEps = pCxt->pComCxt->mgmtEpSet}; + code = catalogGetTableHashVgroup(pCxt->pComCxt->pCatalog, &conn, &pStmt->targetTableName, &vg); + if (code == TSDB_CODE_SUCCESS) { taosHashPut(pStmt->pVgroupsHashObj, (const char*)(&vg.vgId), sizeof(vg.vgId), &vg, sizeof(vg)); pStbRowsCxt->pCtbMeta->uid = taosHashGetSize(pStmt->pSubTableHashObj) + 1; pStbRowsCxt->pCtbMeta->vgId = vg.vgId; + STableMeta* pBackup = NULL; cloneTableMeta(pStbRowsCxt->pCtbMeta, &pBackup); taosHashPut(pStmt->pSubTableHashObj, ctbFName, strlen(ctbFName), &pBackup, POINTER_BYTES); } + collectUseTable(&pStbRowsCxt->ctbName, pStmt->pTableNameHashObj); } return code; } @@ -1724,13 +1757,14 @@ static void clearStbRowsDataContext(SStbRowsDataContext* pStbRowsCxt) { static int32_t parseOneStbRow(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, const char** ppSql, SStbRowsDataContext* pStbRowsCxt, bool* pGotRow, SToken* pToken) { - resetStbRowsDataContextPreStbRow(pStbRowsCxt); - int32_t code = getStbRowValues(pCxt, pStmt, ppSql, pStbRowsCxt, pGotRow, pToken); + resetStbRowsDataContextPreStbRow(pStbRowsCxt); + bool bFirstTable = false; + int32_t code = getStbRowValues(pCxt, pStmt, ppSql, pStbRowsCxt, pGotRow, pToken, &bFirstTable); if (code != TSDB_CODE_SUCCESS || !*pGotRow) { return code; } - if (code == TSDB_CODE_SUCCESS) { + if (code == TSDB_CODE_SUCCESS && bFirstTable) { code = processCtbAutoCreationAndCtbMeta(pCxt, pStmt, pStbRowsCxt); } @@ -2053,6 +2087,14 @@ static int32_t parseInsertStbClauseBottom(SInsertParseContext* pCxt, SVnodeModif if (code == TSDB_CODE_SUCCESS) { code = parseBoundColumns(pCxt, &pStmt->pBoundCols, BOUND_ALL_AND_TBNAME, pStmt->pTableMeta, &pStbRowsCxt->boundColsInfo); + pStbRowsCxt->hasTimestampTag = false; + for (int32_t i = 0; i < pStbRowsCxt->boundColsInfo.numOfBound; ++i) { + int16_t schemaIndex = pStbRowsCxt->boundColsInfo.pColIndex[i]; + if (schemaIndex != getTbnameSchemaIndex(pStmt->pTableMeta) && + schemaIndex >= getNumOfColumns(pStmt->pTableMeta) && pStmt->pTableMeta->schema[schemaIndex].type == TSDB_DATA_TYPE_TIMESTAMP) { + pStbRowsCxt->hasTimestampTag = true; + } + } pStmt->pStbRowsCxt = pStbRowsCxt; } From 1686c70a6048ecf132be8cc0ccc33a70aadbe399 Mon Sep 17 00:00:00 2001 From: slzhou Date: Tue, 31 Oct 2023 13:14:21 +0800 Subject: [PATCH 31/45] fix: not use ctb meta from hash of stmt node --- examples/c/insert_stb.c | 162 ++++++++++++++++++++++++++ examples/c/makefile | 2 + source/libs/parser/src/parInsertSql.c | 3 +- 3 files changed, 166 insertions(+), 1 deletion(-) create mode 100644 examples/c/insert_stb.c diff --git a/examples/c/insert_stb.c b/examples/c/insert_stb.c new file mode 100644 index 0000000000..8316bb929a --- /dev/null +++ b/examples/c/insert_stb.c @@ -0,0 +1,162 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +// TAOS standard API example. The same syntax as MySQL, but only a subset +// to compile: gcc -o demo demo.c -ltaos + +#include +#include +#include +#include +#include + +#include "taos.h" // TAOS header file + +static void executeSql(TAOS *taos, char *command) { + int i; + TAOS_RES *pSql = NULL; + int32_t code = -1; + + for (i = 0; i < 5; i++) { + if (NULL != pSql) { + taos_free_result(pSql); + pSql = NULL; + } + + pSql = taos_query(taos, command); + code = taos_errno(pSql); + if (0 == code) { + break; + } + } + + if (code != 0) { + fprintf(stderr, "Failed to run %s, reason: %s\n", command, taos_errstr(pSql)); + taos_free_result(pSql); + taos_close(taos); + exit(EXIT_FAILURE); + } + + taos_free_result(pSql); +} + +void TestInsert(TAOS *taos, char *qstr) { + executeSql(taos, "drop database if exists demo2"); + executeSql(taos, "create database demo2"); + executeSql(taos, "use demo2"); + + executeSql(taos, "create table st (ts timestamp, ti tinyint, si smallint, i int, bi bigint, f float, d double, b binary(10)) tags(t1 int, t2 float, t3 binary(10))"); + printf("success to create table\n"); + + struct timeval start_time; + gettimeofday(&start_time, NULL); + + for (int tblIdx = 0; tblIdx < 10; ++tblIdx) { + int len = 0; + len += sprintf(qstr+len, "insert into ct%d using st tags(%d, %f, '%s')", tblIdx, tblIdx, (float)tblIdx, "childtable"); + int batchStart = len; + for (int batchIdx = 0; batchIdx < 10000; ++batchIdx) { + len = batchStart; + len += sprintf(qstr+len, " values"); + if (batchIdx % 1000 == 0) + printf("%s %d\n", qstr, batchIdx); + + for (int rowIdx = 0; rowIdx < 100; ++ rowIdx) { + int i = rowIdx + batchIdx * 100 + tblIdx*10000*100; + len += sprintf(qstr+len, " (%" PRId64 ", %d, %d, %d, %d, %f, %lf, '%s')", (uint64_t)(1546300800000 + i), (int8_t)i, (int16_t)i, i, i, i*1.0, i*2.0, "hello"); + } + TAOS_RES *result1 = taos_query(taos, qstr); + if (result1 == NULL || taos_errno(result1) != 0) { + printf("failed to insert row, reason:%s. qstr: %s\n", taos_errstr(result1), qstr); + taos_free_result(result1); + exit(1); + } + taos_free_result(result1); + } + } + struct timeval end_time; + gettimeofday(&end_time, NULL); + double elapsed_time = (end_time.tv_sec - start_time.tv_sec) + + (end_time.tv_usec - start_time.tv_usec) / 1000000.0; + printf("elapsed time: %.3f\n", elapsed_time); + executeSql(taos, "drop database if exists demo2"); +} + +void TestInsertStb(TAOS *taos, char *qstr) { + executeSql(taos, "drop database if exists demo"); + executeSql(taos, "create database demo"); + executeSql(taos, "use demo"); + + executeSql(taos, "create table st (ts timestamp, ti tinyint, si smallint, i int, bi bigint, f float, d double, b binary(10)) tags(t1 int, t2 float, t3 binary(10))"); + printf("success to create table\n"); + + struct timeval start_time; + gettimeofday(&start_time, NULL); + + for (int tblIdx = 0; tblIdx < 10; ++tblIdx) { + int len = 0; + len += sprintf(qstr+len, "insert into st(tbname, t1, t2, t3, ts, ti, si, i, bi, f, d, b)"); + int batchStart = len; + for (int batchIdx = 0; batchIdx < 10000; ++batchIdx) { + len = batchStart; + len += sprintf(qstr+len, " values"); + if (batchIdx % 1000 == 0) + printf("%s %d\n", qstr, batchIdx); + + for (int rowIdx = 0; rowIdx < 100; ++rowIdx) { + int i = rowIdx + batchIdx * 100 + tblIdx*10000*100; + len += sprintf(qstr+len, " ('ct%d', %d, %f, '%s', %" PRId64 ", %d, %d, %d, %d, %f, %lf, '%s')", tblIdx, tblIdx, (float)tblIdx, "childtable", + (uint64_t)(1546300800000 + i), (int8_t)i, (int16_t)i, i, i, i*1.0, i*2.0, "hello"); + } + TAOS_RES *result1 = taos_query(taos, qstr); + if (result1 == NULL || taos_errno(result1) != 0) { + printf("failed to insert row, reason:%s. qstr: %s\n", taos_errstr(result1), qstr); + taos_free_result(result1); + exit(1); + } + taos_free_result(result1); + } + } + struct timeval end_time; + gettimeofday(&end_time, NULL); + double elapsed_time = (end_time.tv_sec - start_time.tv_sec) + + (end_time.tv_usec - start_time.tv_usec) / 1000000.0; + + printf("elapsed time: %.3f\n", elapsed_time); + executeSql(taos, "drop database if exists demo"); +} + + +int main(int argc, char *argv[]) { + + // connect to server + if (argc < 2) { + printf("please input server-ip \n"); + return 0; + } + + TAOS *taos = taos_connect(argv[1], "root", "taosdata", NULL, 0); + if (taos == NULL) { + printf("failed to connect to server, reason:%s\n", taos_errstr(NULL)); + exit(1); + } + char* qstr = malloc(1024*1024); + //TestInsert(taos, qstr); + TestInsertStb(taos, qstr); + free(qstr); + taos_close(taos); + taos_cleanup(); +} + diff --git a/examples/c/makefile b/examples/c/makefile index 244d13fad7..5fc590f424 100644 --- a/examples/c/makefile +++ b/examples/c/makefile @@ -17,6 +17,7 @@ exe: gcc $(CFLAGS) ./stream_demo.c -o $(ROOT)stream_demo $(LFLAGS) gcc $(CFLAGS) ./tmq.c -o $(ROOT)tmq $(LFLAGS) gcc $(CFLAGS) ./schemaless.c -o $(ROOT)schemaless $(LFLAGS) + gcc $(CFLAGS) ./insert_stb.c -o $(ROOT)insert_stb $(LFLAGS) clean: rm $(ROOT)asyncdemo @@ -25,3 +26,4 @@ clean: rm $(ROOT)stream_demo rm $(ROOT)tmq rm $(ROOT)schemaless + rm $(ROOT)insert_stb diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index 231e28878a..663a636ebc 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -1660,7 +1660,8 @@ static int32_t getStbRowValues(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pS STableMeta** pCtbMeta = taosHashGet(pStmt->pSubTableHashObj, ctbFName, strlen(ctbFName)); ctbFirst = *pCtbFirst = (pCtbMeta == NULL); if (!ctbFirst) { - pStbRowsCxt->pCtbMeta = *pCtbMeta; + pStbRowsCxt->pCtbMeta->uid = (*pCtbMeta)->uid; + pStbRowsCxt->pCtbMeta->vgId = (*pCtbMeta)->vgId; } } if (code == TSDB_CODE_SUCCESS && ctbFirst) { From c8767f7614c6192aaeb42580e946478c775515e7 Mon Sep 17 00:00:00 2001 From: slzhou Date: Tue, 31 Oct 2023 14:44:06 +0800 Subject: [PATCH 32/45] enhance: tbname token optimization --- examples/c/insert_stb.c | 8 +++--- source/libs/parser/src/parInsertSql.c | 41 ++++++++++++++++++--------- 2 files changed, 32 insertions(+), 17 deletions(-) diff --git a/examples/c/insert_stb.c b/examples/c/insert_stb.c index 8316bb929a..773fe89ff3 100644 --- a/examples/c/insert_stb.c +++ b/examples/c/insert_stb.c @@ -70,7 +70,7 @@ void TestInsert(TAOS *taos, char *qstr) { for (int batchIdx = 0; batchIdx < 10000; ++batchIdx) { len = batchStart; len += sprintf(qstr+len, " values"); - if (batchIdx % 1000 == 0) + if (batchIdx % 5000 == 1) printf("%s %d\n", qstr, batchIdx); for (int rowIdx = 0; rowIdx < 100; ++ rowIdx) { @@ -112,8 +112,8 @@ void TestInsertStb(TAOS *taos, char *qstr) { for (int batchIdx = 0; batchIdx < 10000; ++batchIdx) { len = batchStart; len += sprintf(qstr+len, " values"); - if (batchIdx % 1000 == 0) - printf("%s %d\n", qstr, batchIdx); + if (batchIdx % 5000 == 1) + printf("%s %d table %d\n", qstr, batchIdx, tblIdx); for (int rowIdx = 0; rowIdx < 100; ++rowIdx) { int i = rowIdx + batchIdx * 100 + tblIdx*10000*100; @@ -153,7 +153,7 @@ int main(int argc, char *argv[]) { exit(1); } char* qstr = malloc(1024*1024); - //TestInsert(taos, qstr); + TestInsert(taos, qstr); TestInsertStb(taos, qstr); free(qstr); taos_close(taos); diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index 663a636ebc..567d050836 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -1574,6 +1574,32 @@ typedef union SRowsDataContext{ SStbRowsDataContext* pStbRowsCxt; } SRowsDataContext; +static int32_t parseTbnameToken(SInsertParseContext* pCxt, SStbRowsDataContext* pStbRowsCxt, SToken* pToken, + char* ctbName, bool* pFoundCtbName) { + *pFoundCtbName = false; + int32_t code = checkAndTrimValue(pToken, pCxt->tmpTokenBuf, &pCxt->msg); + if (code == TSDB_CODE_SUCCESS){ + if (isNullValue(TSDB_DATA_TYPE_BINARY, pToken)) { + return buildInvalidOperationMsg(&pCxt->msg, "tbname can not be null value"); + } + + if (pToken->n > 0) { + if (pToken->n <= TSDB_TABLE_NAME_LEN - 1) { + memcpy(ctbName, pToken->z, pToken->n); + ctbName[pToken->n] = '\0'; + *pFoundCtbName = true; + tNameSetDbName(&pStbRowsCxt->ctbName, pStbRowsCxt->stbName.acctId, pStbRowsCxt->stbName.dbname, strlen(pStbRowsCxt->stbName.dbname)); + tNameAddTbName(&pStbRowsCxt->ctbName, ctbName, pToken->n); + } else { + return buildInvalidOperationMsg(&pCxt->msg, "tbname is too long"); + } + } else { + return buildInvalidOperationMsg(&pCxt->msg, "tbname can not be empty"); + } + } + return code; +} + static int32_t getStbRowValues(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, const char** ppSql, SStbRowsDataContext* pStbRowsCxt, bool* pGotRow, SToken* pToken, bool *pCtbFirst) { @@ -1627,19 +1653,8 @@ static int32_t getStbRowValues(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pS } } else if (pCols->pColIndex[i] == getTbnameSchemaIndex(pStbRowsCxt->pStbMeta)) { - SColVal tbnameVal = COL_VAL_NONE(-1, TSDB_DATA_TYPE_BINARY); - tbnameVal.value.val = 0; - code = parseValueToken(pCxt, ppSql, pToken, (SSchema*)tGetTbnameColumnSchema(), - getTableInfo(pStbRowsCxt->pStbMeta).precision, &tbnameVal); - if (code == TSDB_CODE_SUCCESS && COL_VAL_IS_VALUE(&tbnameVal) && tbnameVal.value.nData>0) { - tNameSetDbName(&pStbRowsCxt->ctbName, pStbRowsCxt->stbName.acctId, pStbRowsCxt->stbName.dbname, strlen(pStbRowsCxt->stbName.dbname)); - char ctbName[TSDB_TABLE_NAME_LEN]; - memcpy(ctbName, tbnameVal.value.pData, tbnameVal.value.nData); - ctbName[tbnameVal.value.nData] = '\0'; - tNameAddTbName(&pStbRowsCxt->ctbName, ctbName, tbnameVal.value.nData); - bFoundTbName = true; - } - taosMemoryFreeClear(tbnameVal.value.pData); + char ctbName[TSDB_TABLE_NAME_LEN]; + code = parseTbnameToken(pCxt, pStbRowsCxt, pToken, ctbName, &bFoundTbName); } if (code == TSDB_CODE_SUCCESS && i < pCols->numOfBound - 1) { From 9a4e4dd4c175301bb00109cbed5fd33c5ecf1a8f Mon Sep 17 00:00:00 2001 From: slzhou Date: Tue, 31 Oct 2023 15:08:39 +0800 Subject: [PATCH 33/45] fix: insert stb test to add one binary column --- examples/c/insert_stb.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/c/insert_stb.c b/examples/c/insert_stb.c index 773fe89ff3..dbe34e3bfb 100644 --- a/examples/c/insert_stb.c +++ b/examples/c/insert_stb.c @@ -57,7 +57,7 @@ void TestInsert(TAOS *taos, char *qstr) { executeSql(taos, "create database demo2"); executeSql(taos, "use demo2"); - executeSql(taos, "create table st (ts timestamp, ti tinyint, si smallint, i int, bi bigint, f float, d double, b binary(10)) tags(t1 int, t2 float, t3 binary(10))"); + executeSql(taos, "create table st (ts timestamp, ti tinyint, si smallint, i int, bi bigint, f float, d double, b binary(10), tbname2 binary(192)) tags(t1 int, t2 float, t3 binary(10))"); printf("success to create table\n"); struct timeval start_time; @@ -75,7 +75,7 @@ void TestInsert(TAOS *taos, char *qstr) { for (int rowIdx = 0; rowIdx < 100; ++ rowIdx) { int i = rowIdx + batchIdx * 100 + tblIdx*10000*100; - len += sprintf(qstr+len, " (%" PRId64 ", %d, %d, %d, %d, %f, %lf, '%s')", (uint64_t)(1546300800000 + i), (int8_t)i, (int16_t)i, i, i, i*1.0, i*2.0, "hello"); + len += sprintf(qstr+len, " (%" PRId64 ", %d, %d, %d, %d, %f, %lf, '%s', 'ct%d')", (uint64_t)(1546300800000 + i), (int8_t)i, (int16_t)i, i, i, i*1.0, i*2.0, "hello", tblIdx); } TAOS_RES *result1 = taos_query(taos, qstr); if (result1 == NULL || taos_errno(result1) != 0) { From 041ef1ef83bbe63400207bf6498336e8480a72ea Mon Sep 17 00:00:00 2001 From: slzhou Date: Tue, 31 Oct 2023 17:05:37 +0800 Subject: [PATCH 34/45] enhance: colvals of table data cxt is not used in insert stb syntax --- source/libs/parser/inc/parInsertUtil.h | 2 +- source/libs/parser/src/parInsertSml.c | 4 ++-- source/libs/parser/src/parInsertSql.c | 6 +++--- source/libs/parser/src/parInsertUtil.c | 14 ++++++++------ {examples/c => tests/script/api}/insert_stb.c | 8 ++++---- 5 files changed, 18 insertions(+), 16 deletions(-) rename {examples/c => tests/script/api}/insert_stb.c (97%) diff --git a/source/libs/parser/inc/parInsertUtil.h b/source/libs/parser/inc/parInsertUtil.h index 0a153225a3..ce8c2d8a3d 100644 --- a/source/libs/parser/inc/parInsertUtil.h +++ b/source/libs/parser/inc/parInsertUtil.h @@ -48,7 +48,7 @@ int32_t insInitBoundColsInfo(int32_t numOfBound, SBoundColInfo *pInfo); void insInitColValues(STableMeta* pTableMeta, SArray* aColValues); void insCheckTableDataOrder(STableDataCxt *pTableCxt, TSKEY tsKey); int32_t insGetTableDataCxt(SHashObj *pHash, void *id, int32_t idLen, STableMeta *pTableMeta, - SVCreateTbReq **pCreateTbReq, STableDataCxt **pTableCxt, bool colMode); + SVCreateTbReq **pCreateTbReq, STableDataCxt **pTableCxt, bool colMode, bool ignoreColVals); int32_t initTableColSubmitData(STableDataCxt *pTableCxt); int32_t insMergeTableDataCxt(SHashObj *pTableHash, SArray **pVgDataBlocks); int32_t insBuildVgDataBlocks(SHashObj *pVgroupsHashObj, SArray *pVgDataBlocks, SArray **pDataBlocks); diff --git a/source/libs/parser/src/parInsertSml.c b/source/libs/parser/src/parInsertSml.c index 517bc630c3..f2194402da 100644 --- a/source/libs/parser/src/parInsertSml.c +++ b/source/libs/parser/src/parInsertSml.c @@ -169,7 +169,7 @@ STableDataCxt* smlInitTableDataCtx(SQuery* query, STableMeta* pTableMeta) { STableDataCxt* pTableCxt = NULL; SVCreateTbReq* pCreateTbReq = NULL; int ret = insGetTableDataCxt(((SVnodeModifyOpStmt*)(query->pRoot))->pTableBlockHashObj, &pTableMeta->uid, - sizeof(pTableMeta->uid), pTableMeta, &pCreateTbReq, &pTableCxt, false); + sizeof(pTableMeta->uid), pTableMeta, &pCreateTbReq, &pTableCxt, false, false); if (ret != TSDB_CODE_SUCCESS) { return NULL; } @@ -305,7 +305,7 @@ int32_t smlBindData(SQuery* query, bool dataFormat, SArray* tags, SArray* colsSc STableDataCxt* pTableCxt = NULL; ret = insGetTableDataCxt(((SVnodeModifyOpStmt*)(query->pRoot))->pTableBlockHashObj, &pTableMeta->uid, - sizeof(pTableMeta->uid), pTableMeta, &pCreateTblReq, &pTableCxt, false); + sizeof(pTableMeta->uid), pTableMeta, &pCreateTblReq, &pTableCxt, false, false); if (ret != TSDB_CODE_SUCCESS) { buildInvalidOperationMsg(&pBuf, "insGetTableDataCxt error"); goto end; diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index 567d050836..270a1b6093 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -1228,7 +1228,7 @@ static int32_t preParseBoundColumnsClause(SInsertParseContext* pCxt, SVnodeModif static int32_t getTableDataCxt(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, STableDataCxt** pTableCxt) { if (pCxt->pComCxt->async) { return insGetTableDataCxt(pStmt->pTableBlockHashObj, &pStmt->pTableMeta->uid, sizeof(pStmt->pTableMeta->uid), - pStmt->pTableMeta, &pStmt->pCreateTblReq, pTableCxt, false); + pStmt->pTableMeta, &pStmt->pCreateTblReq, pTableCxt, false, false); } char tbFName[TSDB_TABLE_FNAME_LEN]; @@ -1237,7 +1237,7 @@ static int32_t getTableDataCxt(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pS pStmt->pTableMeta->uid = 0; } return insGetTableDataCxt(pStmt->pTableBlockHashObj, tbFName, strlen(tbFName), pStmt->pTableMeta, - &pStmt->pCreateTblReq, pTableCxt, NULL != pCxt->pComCxt->pStmtCb); + &pStmt->pCreateTblReq, pTableCxt, NULL != pCxt->pComCxt->pStmtCb, false); } static int32_t parseBoundColumnsClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, STableDataCxt* pTableCxt) { @@ -1786,7 +1786,7 @@ static int32_t parseOneStbRow(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pSt STableDataCxt* pTableDataCxt = NULL; code = insGetTableDataCxt(pStmt->pTableBlockHashObj, &pStbRowsCxt->pCtbMeta->uid, sizeof(pStbRowsCxt->pCtbMeta->uid), - pStbRowsCxt->pCtbMeta, &pStbRowsCxt->pCreateCtbReq, &pTableDataCxt, false); + pStbRowsCxt->pCtbMeta, &pStbRowsCxt->pCreateCtbReq, &pTableDataCxt, false, true); initTableColSubmitData(pTableDataCxt); if (code == TSDB_CODE_SUCCESS) { SRow** pRow = taosArrayReserve(pTableDataCxt->pData->aRowP, 1); diff --git a/source/libs/parser/src/parInsertUtil.c b/source/libs/parser/src/parInsertUtil.c index 9791319503..21b093c76c 100644 --- a/source/libs/parser/src/parInsertUtil.c +++ b/source/libs/parser/src/parInsertUtil.c @@ -208,7 +208,7 @@ void insCheckTableDataOrder(STableDataCxt* pTableCxt, TSKEY tsKey) { void insDestroyBoundColInfo(SBoundColInfo* pInfo) { taosMemoryFreeClear(pInfo->pColIndex); } static int32_t createTableDataCxt(STableMeta* pTableMeta, SVCreateTbReq** pCreateTbReq, STableDataCxt** pOutput, - bool colMode) { + bool colMode, bool ignoreColVals) { STableDataCxt* pTableCxt = taosMemoryCalloc(1, sizeof(STableDataCxt)); if (NULL == pTableCxt) { return TSDB_CODE_OUT_OF_MEMORY; @@ -234,7 +234,7 @@ static int32_t createTableDataCxt(STableMeta* pTableMeta, SVCreateTbReq** pCreat if (TSDB_CODE_SUCCESS == code) { code = insInitBoundColsInfo(pTableMeta->tableInfo.numOfColumns, &pTableCxt->boundColsInfo); } - if (TSDB_CODE_SUCCESS == code) { + if (TSDB_CODE_SUCCESS == code && !ignoreColVals) { pTableCxt->pValues = taosArrayInit(pTableMeta->tableInfo.numOfColumns, sizeof(SColVal)); if (NULL == pTableCxt->pValues) { code = TSDB_CODE_OUT_OF_MEMORY; @@ -322,14 +322,16 @@ static void resetColValues(SArray* pValues) { } int32_t insGetTableDataCxt(SHashObj* pHash, void* id, int32_t idLen, STableMeta* pTableMeta, - SVCreateTbReq** pCreateTbReq, STableDataCxt** pTableCxt, bool colMode) { + SVCreateTbReq** pCreateTbReq, STableDataCxt** pTableCxt, bool colMode, bool ignoreColVals) { STableDataCxt** tmp = (STableDataCxt**)taosHashGet(pHash, id, idLen); if (NULL != tmp) { *pTableCxt = *tmp; - resetColValues((*pTableCxt)->pValues); + if (!ignoreColVals) { + resetColValues((*pTableCxt)->pValues); + } return TSDB_CODE_SUCCESS; } - int32_t code = createTableDataCxt(pTableMeta, pCreateTbReq, pTableCxt, colMode); + int32_t code = createTableDataCxt(pTableMeta, pCreateTbReq, pTableCxt, colMode, ignoreColVals); if (TSDB_CODE_SUCCESS == code) { void* pData = *pTableCxt; // deal scan coverity code = taosHashPut(pHash, id, idLen, &pData, POINTER_BYTES); @@ -622,7 +624,7 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate void* tmp = taosHashGet(((SVnodeModifyOpStmt*)(query->pRoot))->pTableBlockHashObj, &pTableMeta->uid, sizeof(pTableMeta->uid)); STableDataCxt* pTableCxt = NULL; int ret = insGetTableDataCxt(((SVnodeModifyOpStmt*)(query->pRoot))->pTableBlockHashObj, &pTableMeta->uid, - sizeof(pTableMeta->uid), pTableMeta, &pCreateTb, &pTableCxt, true); + sizeof(pTableMeta->uid), pTableMeta, &pCreateTb, &pTableCxt, true, false); if (ret != TSDB_CODE_SUCCESS) { uError("insGetTableDataCxt error"); goto end; diff --git a/examples/c/insert_stb.c b/tests/script/api/insert_stb.c similarity index 97% rename from examples/c/insert_stb.c rename to tests/script/api/insert_stb.c index dbe34e3bfb..4fdcea144a 100644 --- a/examples/c/insert_stb.c +++ b/tests/script/api/insert_stb.c @@ -112,8 +112,6 @@ void TestInsertStb(TAOS *taos, char *qstr) { for (int batchIdx = 0; batchIdx < 10000; ++batchIdx) { len = batchStart; len += sprintf(qstr+len, " values"); - if (batchIdx % 5000 == 1) - printf("%s %d table %d\n", qstr, batchIdx, tblIdx); for (int rowIdx = 0; rowIdx < 100; ++rowIdx) { int i = rowIdx + batchIdx * 100 + tblIdx*10000*100; @@ -153,8 +151,10 @@ int main(int argc, char *argv[]) { exit(1); } char* qstr = malloc(1024*1024); - TestInsert(taos, qstr); - TestInsertStb(taos, qstr); + for (int i =0; i < 5; ++i) + TestInsert(taos, qstr); + for (int i =0; i < 5; ++i) + TestInsertStb(taos, qstr); free(qstr); taos_close(taos); taos_cleanup(); From a5d4cf1432dfbca084f57e2854d05a2b72a8c0a1 Mon Sep 17 00:00:00 2001 From: slzhou Date: Tue, 31 Oct 2023 17:06:44 +0800 Subject: [PATCH 35/45] fix: modify makefile to remove insert_stb --- examples/c/makefile | 2 -- 1 file changed, 2 deletions(-) diff --git a/examples/c/makefile b/examples/c/makefile index 5fc590f424..244d13fad7 100644 --- a/examples/c/makefile +++ b/examples/c/makefile @@ -17,7 +17,6 @@ exe: gcc $(CFLAGS) ./stream_demo.c -o $(ROOT)stream_demo $(LFLAGS) gcc $(CFLAGS) ./tmq.c -o $(ROOT)tmq $(LFLAGS) gcc $(CFLAGS) ./schemaless.c -o $(ROOT)schemaless $(LFLAGS) - gcc $(CFLAGS) ./insert_stb.c -o $(ROOT)insert_stb $(LFLAGS) clean: rm $(ROOT)asyncdemo @@ -26,4 +25,3 @@ clean: rm $(ROOT)stream_demo rm $(ROOT)tmq rm $(ROOT)schemaless - rm $(ROOT)insert_stb From 670a95fabc924ab37a8a3a271135a9f0cbe58020 Mon Sep 17 00:00:00 2001 From: slzhou Date: Tue, 31 Oct 2023 21:10:25 +0800 Subject: [PATCH 36/45] fix: performance test --- tests/script/api/insert_stb.c | 22 ++++++++++------------ tests/script/api/makefile | 2 ++ 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/tests/script/api/insert_stb.c b/tests/script/api/insert_stb.c index 4fdcea144a..cadd3dc98c 100644 --- a/tests/script/api/insert_stb.c +++ b/tests/script/api/insert_stb.c @@ -52,12 +52,12 @@ static void executeSql(TAOS *taos, char *command) { taos_free_result(pSql); } -void TestInsert(TAOS *taos, char *qstr) { +void testInsert(TAOS *taos, char *qstr) { executeSql(taos, "drop database if exists demo2"); executeSql(taos, "create database demo2"); executeSql(taos, "use demo2"); - executeSql(taos, "create table st (ts timestamp, ti tinyint, si smallint, i int, bi bigint, f float, d double, b binary(10), tbname2 binary(192)) tags(t1 int, t2 float, t3 binary(10))"); + executeSql(taos, "create table st (ts timestamp, ti tinyint, si smallint, i int, bi bigint, f float, d double, b binary(10)) tags(t1 int, t2 float, t3 binary(10))"); printf("success to create table\n"); struct timeval start_time; @@ -70,12 +70,10 @@ void TestInsert(TAOS *taos, char *qstr) { for (int batchIdx = 0; batchIdx < 10000; ++batchIdx) { len = batchStart; len += sprintf(qstr+len, " values"); - if (batchIdx % 5000 == 1) - printf("%s %d\n", qstr, batchIdx); for (int rowIdx = 0; rowIdx < 100; ++ rowIdx) { int i = rowIdx + batchIdx * 100 + tblIdx*10000*100; - len += sprintf(qstr+len, " (%" PRId64 ", %d, %d, %d, %d, %f, %lf, '%s', 'ct%d')", (uint64_t)(1546300800000 + i), (int8_t)i, (int16_t)i, i, i, i*1.0, i*2.0, "hello", tblIdx); + len += sprintf(qstr+len, " (%" PRId64 ", %d, %d, %d, %d, %f, %lf, '%s')", (uint64_t)(1546300800000 + i), (int8_t)i, (int16_t)i, i, i, i*1.0, i*2.0, "hello"); } TAOS_RES *result1 = taos_query(taos, qstr); if (result1 == NULL || taos_errno(result1) != 0) { @@ -88,13 +86,13 @@ void TestInsert(TAOS *taos, char *qstr) { } struct timeval end_time; gettimeofday(&end_time, NULL); - double elapsed_time = (end_time.tv_sec - start_time.tv_sec) + - (end_time.tv_usec - start_time.tv_usec) / 1000000.0; + double elapsed_time = (double)(end_time.tv_sec - start_time.tv_sec) + + (double)(end_time.tv_usec - start_time.tv_usec) / 1000000.0; printf("elapsed time: %.3f\n", elapsed_time); executeSql(taos, "drop database if exists demo2"); } -void TestInsertStb(TAOS *taos, char *qstr) { +void testInsertStb(TAOS *taos, char *qstr) { executeSql(taos, "drop database if exists demo"); executeSql(taos, "create database demo"); executeSql(taos, "use demo"); @@ -129,8 +127,8 @@ void TestInsertStb(TAOS *taos, char *qstr) { } struct timeval end_time; gettimeofday(&end_time, NULL); - double elapsed_time = (end_time.tv_sec - start_time.tv_sec) + - (end_time.tv_usec - start_time.tv_usec) / 1000000.0; + double elapsed_time = (double)(end_time.tv_sec - start_time.tv_sec) + + (double)(end_time.tv_usec - start_time.tv_usec) / 1000000.0; printf("elapsed time: %.3f\n", elapsed_time); executeSql(taos, "drop database if exists demo"); @@ -152,9 +150,9 @@ int main(int argc, char *argv[]) { } char* qstr = malloc(1024*1024); for (int i =0; i < 5; ++i) - TestInsert(taos, qstr); + testInsert(taos, qstr); for (int i =0; i < 5; ++i) - TestInsertStb(taos, qstr); + testInsertStb(taos, qstr); free(qstr); taos_close(taos); taos_cleanup(); diff --git a/tests/script/api/makefile b/tests/script/api/makefile index ac5c5c9138..605b5ef717 100644 --- a/tests/script/api/makefile +++ b/tests/script/api/makefile @@ -17,6 +17,7 @@ exe: gcc $(CFLAGS) ./insertSameTs.c -o $(ROOT)insertSameTs $(LFLAGS) gcc $(CFLAGS) ./passwdTest.c -o $(ROOT)passwdTest $(LFLAGS) gcc $(CFLAGS) ./whiteListTest.c -o $(ROOT)whiteListTest $(LFLAGS) + gcc $(CFLAGS) ./insert_stb.c -o $(ROOT)insert_stb $(LFLAGS) clean: rm $(ROOT)batchprepare @@ -25,3 +26,4 @@ clean: rm $(ROOT)insertSameTs rm $(ROOT)passwdTest rm $(ROOT)whiteListTest + rm $(ROOT)insert_stb From 7636f09bfb4a65d1f2e5c1982a03f851addfcf75 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Wed, 1 Nov 2023 08:32:31 +0800 Subject: [PATCH 37/45] enhance: refactor getStbRowValues --- source/libs/parser/src/parInsertSql.c | 116 ++++++++++++++++---------- 1 file changed, 71 insertions(+), 45 deletions(-) diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index 270a1b6093..ebb96925d3 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -1567,6 +1567,7 @@ typedef struct SStbRowsDataContext { STableMeta* pCtbMeta; SVCreateTbReq* pCreateCtbReq; bool hasTimestampTag; + bool isJsonTag; } SStbRowsDataContext; typedef union SRowsDataContext{ @@ -1600,27 +1601,40 @@ static int32_t parseTbnameToken(SInsertParseContext* pCxt, SStbRowsDataContext* return code; } -static int32_t getStbRowValues(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, const char** ppSql, - SStbRowsDataContext* pStbRowsCxt, bool* pGotRow, - SToken* pToken, bool *pCtbFirst) { - SBoundColInfo* pCols = &pStbRowsCxt->boundColsInfo; - SSchema* pSchemas = getTableColumnSchema(pStbRowsCxt->pStbMeta); - - bool isJsonTag = false; - SArray* pTagNames = pStbRowsCxt->aTagNames; - SArray* pTagVals = pStbRowsCxt->aTagVals; - - bool bFoundTbName = false; - const char* pOrigSql = *ppSql; - +static int32_t processCtbTagsAfterCtbName(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, + SStbRowsDataContext* pStbRowsCxt, bool ctbFirst, + const SToken* tagTokens, SSchema* const* tagSchemas, + int numOfTagTokens) { int32_t code = TSDB_CODE_SUCCESS; - bool canParseTagsAfter = !pStbRowsCxt->pTagCond && !pStbRowsCxt->hasTimestampTag; - SToken tagTokens[TSDB_MAX_TAGS] = {0}; - SSchema* tagSchemas[TSDB_MAX_TAGS] = {0}; - int numOfTagTokens = 0; + if (code == TSDB_CODE_SUCCESS && ctbFirst) { + for (int32_t i = 0; code == TSDB_CODE_SUCCESS && i < numOfTagTokens; ++i) { + SToken* pTagToken = (SToken*)(tagTokens + i); + SSchema* pTagSchema = tagSchemas[i]; + code = checkAndTrimValue(pTagToken, pCxt->tmpTokenBuf, &pCxt->msg); + if (code == TSDB_CODE_SUCCESS) { + code = parseTagValue(pCxt, pStmt, NULL, pTagSchema, pTagToken, pStbRowsCxt->aTagNames, pStbRowsCxt->aTagVals, + &pStbRowsCxt->pTag); + } + } + if (code == TSDB_CODE_SUCCESS && !pStbRowsCxt->isJsonTag) { + code = tTagNew(pStbRowsCxt->aTagVals, 1, false, &pStbRowsCxt->pTag); + } + } - for (int i = 0; i < pCols->numOfBound && code == TSDB_CODE_SUCCESS; ++i) { + if (code == TSDB_CODE_SUCCESS && pStbRowsCxt->pTagCond) { + code = checkSubtablePrivilege(pStbRowsCxt->aTagVals, pStbRowsCxt->aTagNames, &pStbRowsCxt->pTagCond); + } + return code; +} + +static int32_t doGetStbRowValues(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, const char* const* ppSql, + SStbRowsDataContext* pStbRowsCxt, SToken* pToken, const SBoundColInfo* pCols, + const SSchema* pSchemas, SArray* pTagNames, SArray* pTagVals, SToken* tagTokens, + SSchema** tagSchemas, int* pNumOfTagTokens, bool* bFoundTbName) { + int32_t code = TSDB_CODE_SUCCESS; + bool canParseTagsAfter = !pStbRowsCxt->pTagCond && !pStbRowsCxt->hasTimestampTag; + for (int i = 0; i < pCols->numOfBound && (code) == TSDB_CODE_SUCCESS; ++i) { const char* pTmpSql = *ppSql; bool ignoreComma = false; NEXT_TOKEN_WITH_PREV_EXT(*ppSql, *pToken, &ignoreComma); @@ -1640,11 +1654,10 @@ static int32_t getStbRowValues(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pS code = parseValueToken(pCxt, ppSql, pToken, pSchema, getTableInfo(pStbRowsCxt->pStbMeta).precision, pVal); } else if (pCols->pColIndex[i] < getTbnameSchemaIndex(pStbRowsCxt->pStbMeta)) { SSchema* pTagSchema = &pSchemas[pCols->pColIndex[i]]; - isJsonTag = pTagSchema->type == TSDB_DATA_TYPE_JSON; if (canParseTagsAfter) { - tagTokens[numOfTagTokens] = *pToken; - tagSchemas[numOfTagTokens] = pTagSchema; - ++numOfTagTokens; + tagTokens[(*pNumOfTagTokens)] = *pToken; + tagSchemas[(*pNumOfTagTokens)] = pTagSchema; + ++(*pNumOfTagTokens); } else { code = checkAndTrimValue(pToken, pCxt->tmpTokenBuf, &pCxt->msg); if (code == TSDB_CODE_SUCCESS) { @@ -1654,7 +1667,7 @@ static int32_t getStbRowValues(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pS } else if (pCols->pColIndex[i] == getTbnameSchemaIndex(pStbRowsCxt->pStbMeta)) { char ctbName[TSDB_TABLE_NAME_LEN]; - code = parseTbnameToken(pCxt, pStbRowsCxt, pToken, ctbName, &bFoundTbName); + code = parseTbnameToken(pCxt, pStbRowsCxt, pToken, ctbName, bFoundTbName); } if (code == TSDB_CODE_SUCCESS && i < pCols->numOfBound - 1) { @@ -1664,38 +1677,47 @@ static int32_t getStbRowValues(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pS } } } - if (!bFoundTbName) { + return code; +} + +static int32_t getStbRowValues(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, const char** ppSql, + SStbRowsDataContext* pStbRowsCxt, bool* pGotRow, + SToken* pToken, bool *pCtbFirst) { + SBoundColInfo* pCols = &pStbRowsCxt->boundColsInfo; + SSchema* pSchemas = getTableColumnSchema(pStbRowsCxt->pStbMeta); + + SArray* pTagNames = pStbRowsCxt->aTagNames; + SArray* pTagVals = pStbRowsCxt->aTagVals; + + bool bFoundTbName = false; + const char* pOrigSql = *ppSql; + + int32_t code = TSDB_CODE_SUCCESS; + SToken tagTokens[TSDB_MAX_TAGS] = {0}; + SSchema* tagSchemas[TSDB_MAX_TAGS] = {0}; + int numOfTagTokens = 0; + + code = doGetStbRowValues(pCxt, pStmt, ppSql, pStbRowsCxt, pToken, pCols, pSchemas, pTagNames, pTagVals, tagTokens, + tagSchemas, &numOfTagTokens, &bFoundTbName); + if (code == TSDB_CODE_SUCCESS && !bFoundTbName) { code = buildSyntaxErrMsg(&pCxt->msg, "tbname value expected", pOrigSql); } - + bool ctbFirst = true; if (code == TSDB_CODE_SUCCESS) { char ctbFName[TSDB_TABLE_FNAME_LEN]; tNameExtractFullName(&pStbRowsCxt->ctbName, ctbFName); STableMeta** pCtbMeta = taosHashGet(pStmt->pSubTableHashObj, ctbFName, strlen(ctbFName)); - ctbFirst = *pCtbFirst = (pCtbMeta == NULL); + ctbFirst = (pCtbMeta == NULL); if (!ctbFirst) { pStbRowsCxt->pCtbMeta->uid = (*pCtbMeta)->uid; pStbRowsCxt->pCtbMeta->vgId = (*pCtbMeta)->vgId; } - } - if (code == TSDB_CODE_SUCCESS && ctbFirst) { - for (int32_t i = 0; code == TSDB_CODE_SUCCESS && i < numOfTagTokens; ++i) { - SToken* pTagToken = tagTokens + i; - SSchema* pTagSchema = tagSchemas[i]; - code = checkAndTrimValue(pTagToken, pCxt->tmpTokenBuf, &pCxt->msg); - if (code == TSDB_CODE_SUCCESS) { - code = parseTagValue(pCxt, pStmt, NULL, pTagSchema, pTagToken, pStbRowsCxt->aTagNames, pStbRowsCxt->aTagVals, - &pStbRowsCxt->pTag); - } - } - if (code == TSDB_CODE_SUCCESS && !isJsonTag) { - code = tTagNew(pStbRowsCxt->aTagVals, 1, false, &pStbRowsCxt->pTag); - } + *pCtbFirst = ctbFirst; } - if (code == TSDB_CODE_SUCCESS && pStbRowsCxt->pTagCond) { - code = checkSubtablePrivilege(pStbRowsCxt->aTagVals, pStbRowsCxt->aTagNames, &pStbRowsCxt->pTagCond); + if (code == TSDB_CODE_SUCCESS) { + code = processCtbTagsAfterCtbName(pCxt, pStmt, pStbRowsCxt, ctbFirst, tagTokens, tagSchemas, numOfTagTokens); } if (code == TSDB_CODE_SUCCESS) { @@ -2106,9 +2128,13 @@ static int32_t parseInsertStbClauseBottom(SInsertParseContext* pCxt, SVnodeModif pStbRowsCxt->hasTimestampTag = false; for (int32_t i = 0; i < pStbRowsCxt->boundColsInfo.numOfBound; ++i) { int16_t schemaIndex = pStbRowsCxt->boundColsInfo.pColIndex[i]; - if (schemaIndex != getTbnameSchemaIndex(pStmt->pTableMeta) && - schemaIndex >= getNumOfColumns(pStmt->pTableMeta) && pStmt->pTableMeta->schema[schemaIndex].type == TSDB_DATA_TYPE_TIMESTAMP) { - pStbRowsCxt->hasTimestampTag = true; + if (schemaIndex != getTbnameSchemaIndex(pStmt->pTableMeta) && schemaIndex >= getNumOfColumns(pStmt->pTableMeta) ) { + if (pStmt->pTableMeta->schema[schemaIndex].type == TSDB_DATA_TYPE_TIMESTAMP) { + pStbRowsCxt->hasTimestampTag = true; + } + if (pStmt->pTableMeta->schema[schemaIndex].type == TSDB_DATA_TYPE_JSON) { + pStbRowsCxt->isJsonTag = true; + } } } pStmt->pStbRowsCxt = pStbRowsCxt; From 42a7553fe04aa9cd2c3b25eb0d07acd09984048b Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Wed, 1 Nov 2023 08:35:38 +0800 Subject: [PATCH 38/45] enhance: refactor getStbRowValues --- source/libs/parser/src/parInsertSql.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index ebb96925d3..c77dd3fc60 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -1629,10 +1629,12 @@ static int32_t processCtbTagsAfterCtbName(SInsertParseContext* pCxt, SVnodeModif } static int32_t doGetStbRowValues(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, const char* const* ppSql, - SStbRowsDataContext* pStbRowsCxt, SToken* pToken, const SBoundColInfo* pCols, - const SSchema* pSchemas, SArray* pTagNames, SArray* pTagVals, SToken* tagTokens, - SSchema** tagSchemas, int* pNumOfTagTokens, bool* bFoundTbName) { + SStbRowsDataContext* pStbRowsCxt, SToken* pToken, + const SBoundColInfo* pCols, const SSchema* pSchemas, + SToken* tagTokens, SSchema** tagSchemas, int* pNumOfTagTokens, bool* bFoundTbName) { int32_t code = TSDB_CODE_SUCCESS; + SArray* pTagNames = pStbRowsCxt->aTagNames; + SArray* pTagVals = pStbRowsCxt->aTagVals; bool canParseTagsAfter = !pStbRowsCxt->pTagCond && !pStbRowsCxt->hasTimestampTag; for (int i = 0; i < pCols->numOfBound && (code) == TSDB_CODE_SUCCESS; ++i) { const char* pTmpSql = *ppSql; @@ -1686,9 +1688,6 @@ static int32_t getStbRowValues(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pS SBoundColInfo* pCols = &pStbRowsCxt->boundColsInfo; SSchema* pSchemas = getTableColumnSchema(pStbRowsCxt->pStbMeta); - SArray* pTagNames = pStbRowsCxt->aTagNames; - SArray* pTagVals = pStbRowsCxt->aTagVals; - bool bFoundTbName = false; const char* pOrigSql = *ppSql; @@ -1697,8 +1696,9 @@ static int32_t getStbRowValues(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pS SSchema* tagSchemas[TSDB_MAX_TAGS] = {0}; int numOfTagTokens = 0; - code = doGetStbRowValues(pCxt, pStmt, ppSql, pStbRowsCxt, pToken, pCols, pSchemas, pTagNames, pTagVals, tagTokens, + code = doGetStbRowValues(pCxt, pStmt, ppSql, pStbRowsCxt, pToken, pCols, pSchemas, tagTokens, tagSchemas, &numOfTagTokens, &bFoundTbName); + if (code == TSDB_CODE_SUCCESS && !bFoundTbName) { code = buildSyntaxErrMsg(&pCxt->msg, "tbname value expected", pOrigSql); } From efb0c61cac03dfcb744d4950ce516d0f88699dbc Mon Sep 17 00:00:00 2001 From: slzhou Date: Wed, 1 Nov 2023 08:43:29 +0800 Subject: [PATCH 39/45] fix: fix compilation error --- source/libs/parser/src/parInsertSql.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index c77dd3fc60..8c404e146c 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -1628,7 +1628,7 @@ static int32_t processCtbTagsAfterCtbName(SInsertParseContext* pCxt, SVnodeModif return code; } -static int32_t doGetStbRowValues(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, const char* const* ppSql, +static int32_t doGetStbRowValues(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, const char** ppSql, SStbRowsDataContext* pStbRowsCxt, SToken* pToken, const SBoundColInfo* pCols, const SSchema* pSchemas, SToken* tagTokens, SSchema** tagSchemas, int* pNumOfTagTokens, bool* bFoundTbName) { @@ -1651,19 +1651,19 @@ static int32_t doGetStbRowValues(SInsertParseContext* pCxt, SVnodeModifyOpStmt* } if (pCols->pColIndex[i] < getNumOfColumns(pStbRowsCxt->pStbMeta)) { - SSchema* pSchema = &pSchemas[pCols->pColIndex[i]]; + const SSchema* pSchema = &pSchemas[pCols->pColIndex[i]]; SColVal* pVal = taosArrayGet(pStbRowsCxt->aColVals, pCols->pColIndex[i]); - code = parseValueToken(pCxt, ppSql, pToken, pSchema, getTableInfo(pStbRowsCxt->pStbMeta).precision, pVal); + code = parseValueToken(pCxt, ppSql, pToken, (SSchema*)pSchema, getTableInfo(pStbRowsCxt->pStbMeta).precision, pVal); } else if (pCols->pColIndex[i] < getTbnameSchemaIndex(pStbRowsCxt->pStbMeta)) { - SSchema* pTagSchema = &pSchemas[pCols->pColIndex[i]]; + const SSchema* pTagSchema = &pSchemas[pCols->pColIndex[i]]; if (canParseTagsAfter) { tagTokens[(*pNumOfTagTokens)] = *pToken; - tagSchemas[(*pNumOfTagTokens)] = pTagSchema; + tagSchemas[(*pNumOfTagTokens)] = (SSchema*)pTagSchema; ++(*pNumOfTagTokens); } else { code = checkAndTrimValue(pToken, pCxt->tmpTokenBuf, &pCxt->msg); if (code == TSDB_CODE_SUCCESS) { - code = parseTagValue(pCxt, pStmt, ppSql, pTagSchema, pToken, pTagNames, pTagVals, &pStbRowsCxt->pTag); + code = parseTagValue(pCxt, pStmt, ppSql, (SSchema*)pTagSchema, pToken, pTagNames, pTagVals, &pStbRowsCxt->pTag); } } } From 4806480b2a0d96875f477ca4e2f8ea5dcd022d78 Mon Sep 17 00:00:00 2001 From: slzhou Date: Wed, 1 Nov 2023 09:46:33 +0800 Subject: [PATCH 40/45] enhance: add test case --- tests/system-test/1-insert/insert_stb.py | 252 +++++++++++++++++++++++ 1 file changed, 252 insertions(+) diff --git a/tests/system-test/1-insert/insert_stb.py b/tests/system-test/1-insert/insert_stb.py index 6e004a96e9..605bee7d8b 100644 --- a/tests/system-test/1-insert/insert_stb.py +++ b/tests/system-test/1-insert/insert_stb.py @@ -360,11 +360,263 @@ class TDTestCase: except Exception as err: print(str(err)) + def run_consecutive_seq(self): + print("running {}".format("consecutive_seq")) + tdSql.execute("drop database if exists insert_stb3") + tdSql.execute("create database if not exists insert_stb3") + tdSql.execute('use insert_stb3') + tdSql.execute('create table st (ts timestamp, ti tinyint, si smallint, i int, bi bigint, f float, d double, b binary(10)) tags(t1 int, t2 float, t3 binary(10))') + + tdSql.execute("insert into st(tbname, t1, t2, t3, ts, ti, si, i, bi, f, d, b) values ('ct0', 0, 0.000000, 'childtable', 1546300800000, 0, 0, 0, 0, 0.000000, 0.000000, 'hello') ('ct0', 0, 0.000000, 'childtable', 1546300800001, 1, 1, 1, 1, 1.000000, 2.000000, 'hello')") + + tdSql.execute("insert into st(tbname, t1, t2, t3, ts, ti, si, i, bi, f, d, b) values ('ct1', 1, 1.000000, 'childtable', 1546301800000, 64, 16960, 1000000, 1000000, 1000000.000000, 2000000.000000, 'hello') ('ct1', 1, 1.000000, 'childtable', 1546301800001, 65, 16961, 1000001, 1000001, 1000001.000000, 2000002.000000, 'hello')") + + tdSql.execute("insert into st(tbname, t1, t2, t3, ts, ti, si, i, bi, f, d, b) values ('ct2', 2, 2.000000, 'childtable', 1546302800000, -128, -31616, 2000000, 2000000, 2000000.000000, 4000000.000000, 'hello') ('ct2', 2, 2.000000, 'childtable', 1546302800001, -127, -31615, 2000001, 2000001, 2000001.000000, 4000002.000000, 'hello')") + + tdSql.execute("insert into st(tbname, t1, t2, t3, ts, ti, si, i, bi, f, d, b) values ('ct3', 3, 3.000000, 'childtable', 1546303800000, -64, -14656, 3000000, 3000000, 3000000.000000, 6000000.000000, 'hello') ('ct3', 3, 3.000000, 'childtable', 1546303800001, -63, -14655, 3000001, 3000001, 3000001.000000, 6000002.000000, 'hello')") + + tdSql.execute("insert into st(tbname, t1, t2, t3, ts, ti, si, i, bi, f, d, b) values ('ct4', 4, 4.000000, 'childtable', 1546304800000, 0, 2304, 4000000, 4000000, 4000000.000000, 8000000.000000, 'hello') ('ct4', 4, 4.000000, 'childtable', 1546304800001, 1, 2305, 4000001, 4000001, 4000001.000000, 8000002.000000, 'hello')") + + tdSql.execute("insert into st(tbname, t1, t2, t3, ts, ti, si, i, bi, f, d, b) values ('ct5', 5, 5.000000, 'childtable', 1546305800000, 64, 19264, 5000000, 5000000, 5000000.000000, 10000000.000000, 'hello') ('ct5', 5, 5.000000, 'childtable', 1546305800001, 65, 19265, 5000001, 5000001, 5000001.000000, 10000002.000000, 'hello')") + + tdSql.execute("insert into st(tbname, t1, t2, t3, ts, ti, si, i, bi, f, d, b) values ('ct6', 6, 6.000000, 'childtable', 1546306800000, -128, -29312, 6000000, 6000000, 6000000.000000, 12000000.000000, 'hello') ('ct6', 6, 6.000000, 'childtable', 1546306800001, -127, -29311, 6000001, 6000001, 6000001.000000, 12000002.000000, 'hello')") + + tdSql.execute("insert into st(tbname, t1, t2, t3, ts, ti, si, i, bi, f, d, b) values ('ct7', 7, 7.000000, 'childtable', 1546307800000, -64, -12352, 7000000, 7000000, 7000000.000000, 14000000.000000, 'hello') ('ct7', 7, 7.000000, 'childtable', 1546307800001, -63, -12351, 7000001, 7000001, 7000001.000000, 14000002.000000, 'hello')") + + tdSql.execute("insert into st(tbname, t1, t2, t3, ts, ti, si, i, bi, f, d, b) values ('ct8', 8, 8.000000, 'childtable', 1546308800000, 0, 4608, 8000000, 8000000, 8000000.000000, 16000000.000000, 'hello') ('ct8', 8, 8.000000, 'childtable', 1546308800001, 1, 4609, 8000001, 8000001, 8000001.000000, 16000002.000000, 'hello')") + + tdSql.execute("insert into st(tbname, t1, t2, t3, ts, ti, si, i, bi, f, d, b) values ('ct9', 9, 9.000000, 'childtable', 1546309800000, 64, 21568, 9000000, 9000000, 9000000.000000, 18000000.000000, 'hello') ('ct9', 9, 9.000000, 'childtable', 1546309800001, 65, 21569, 9000001, 9000001, 9000001.000000, 18000002.000000, 'hello')") + + tdSql.query('select * from st order by ts') + tdSql.checkRows(20) + tdSql.checkData(0, 0, datetime.datetime(2019, 1, 1, 8, 0)) + tdSql.checkData(0, 1, 0) + tdSql.checkData(0, 2, 0) + tdSql.checkData(0, 3, 0) + tdSql.checkData(0, 4, 0) + tdSql.checkData(0, 5, 0.0) + tdSql.checkData(0, 6, 0.0) + tdSql.checkData(0, 7, 'hello') + tdSql.checkData(0, 8, 0) + tdSql.checkData(0, 9, 0.0) + tdSql.checkData(0, 10, 'childtable') + tdSql.checkData(1, 0, datetime.datetime(2019, 1, 1, 8, 0, 0, 1000)) + tdSql.checkData(1, 1, 1) + tdSql.checkData(1, 2, 1) + tdSql.checkData(1, 3, 1) + tdSql.checkData(1, 4, 1) + tdSql.checkData(1, 5, 1.0) + tdSql.checkData(1, 6, 2.0) + tdSql.checkData(1, 7, 'hello') + tdSql.checkData(1, 8, 0) + tdSql.checkData(1, 9, 0.0) + tdSql.checkData(1, 10, 'childtable') + tdSql.checkData(2, 0, datetime.datetime(2019, 1, 1, 8, 16, 40)) + tdSql.checkData(2, 1, 64) + tdSql.checkData(2, 2, 16960) + tdSql.checkData(2, 3, 1000000) + tdSql.checkData(2, 4, 1000000) + tdSql.checkData(2, 5, 1000000.0) + tdSql.checkData(2, 6, 2000000.0) + tdSql.checkData(2, 7, 'hello') + tdSql.checkData(2, 8, 1) + tdSql.checkData(2, 9, 1.0) + tdSql.checkData(2, 10, 'childtable') + tdSql.checkData(3, 0, datetime.datetime(2019, 1, 1, 8, 16, 40, 1000)) + tdSql.checkData(3, 1, 65) + tdSql.checkData(3, 2, 16961) + tdSql.checkData(3, 3, 1000001) + tdSql.checkData(3, 4, 1000001) + tdSql.checkData(3, 5, 1000001.0) + tdSql.checkData(3, 6, 2000002.0) + tdSql.checkData(3, 7, 'hello') + tdSql.checkData(3, 8, 1) + tdSql.checkData(3, 9, 1.0) + tdSql.checkData(3, 10, 'childtable') + tdSql.checkData(4, 0, datetime.datetime(2019, 1, 1, 8, 33, 20)) + tdSql.checkData(4, 1, -128) + tdSql.checkData(4, 2, -31616) + tdSql.checkData(4, 3, 2000000) + tdSql.checkData(4, 4, 2000000) + tdSql.checkData(4, 5, 2000000.0) + tdSql.checkData(4, 6, 4000000.0) + tdSql.checkData(4, 7, 'hello') + tdSql.checkData(4, 8, 2) + tdSql.checkData(4, 9, 2.0) + tdSql.checkData(4, 10, 'childtable') + tdSql.checkData(5, 0, datetime.datetime(2019, 1, 1, 8, 33, 20, 1000)) + tdSql.checkData(5, 1, -127) + tdSql.checkData(5, 2, -31615) + tdSql.checkData(5, 3, 2000001) + tdSql.checkData(5, 4, 2000001) + tdSql.checkData(5, 5, 2000001.0) + tdSql.checkData(5, 6, 4000002.0) + tdSql.checkData(5, 7, 'hello') + tdSql.checkData(5, 8, 2) + tdSql.checkData(5, 9, 2.0) + tdSql.checkData(5, 10, 'childtable') + tdSql.checkData(6, 0, datetime.datetime(2019, 1, 1, 8, 50)) + tdSql.checkData(6, 1, -64) + tdSql.checkData(6, 2, -14656) + tdSql.checkData(6, 3, 3000000) + tdSql.checkData(6, 4, 3000000) + tdSql.checkData(6, 5, 3000000.0) + tdSql.checkData(6, 6, 6000000.0) + tdSql.checkData(6, 7, 'hello') + tdSql.checkData(6, 8, 3) + tdSql.checkData(6, 9, 3.0) + tdSql.checkData(6, 10, 'childtable') + tdSql.checkData(7, 0, datetime.datetime(2019, 1, 1, 8, 50, 0, 1000)) + tdSql.checkData(7, 1, -63) + tdSql.checkData(7, 2, -14655) + tdSql.checkData(7, 3, 3000001) + tdSql.checkData(7, 4, 3000001) + tdSql.checkData(7, 5, 3000001.0) + tdSql.checkData(7, 6, 6000002.0) + tdSql.checkData(7, 7, 'hello') + tdSql.checkData(7, 8, 3) + tdSql.checkData(7, 9, 3.0) + tdSql.checkData(7, 10, 'childtable') + tdSql.checkData(8, 0, datetime.datetime(2019, 1, 1, 9, 6, 40)) + tdSql.checkData(8, 1, 0) + tdSql.checkData(8, 2, 2304) + tdSql.checkData(8, 3, 4000000) + tdSql.checkData(8, 4, 4000000) + tdSql.checkData(8, 5, 4000000.0) + tdSql.checkData(8, 6, 8000000.0) + tdSql.checkData(8, 7, 'hello') + tdSql.checkData(8, 8, 4) + tdSql.checkData(8, 9, 4.0) + tdSql.checkData(8, 10, 'childtable') + tdSql.checkData(9, 0, datetime.datetime(2019, 1, 1, 9, 6, 40, 1000)) + tdSql.checkData(9, 1, 1) + tdSql.checkData(9, 2, 2305) + tdSql.checkData(9, 3, 4000001) + tdSql.checkData(9, 4, 4000001) + tdSql.checkData(9, 5, 4000001.0) + tdSql.checkData(9, 6, 8000002.0) + tdSql.checkData(9, 7, 'hello') + tdSql.checkData(9, 8, 4) + tdSql.checkData(9, 9, 4.0) + tdSql.checkData(9, 10, 'childtable') + tdSql.checkData(10, 0, datetime.datetime(2019, 1, 1, 9, 23, 20)) + tdSql.checkData(10, 1, 64) + tdSql.checkData(10, 2, 19264) + tdSql.checkData(10, 3, 5000000) + tdSql.checkData(10, 4, 5000000) + tdSql.checkData(10, 5, 5000000.0) + tdSql.checkData(10, 6, 10000000.0) + tdSql.checkData(10, 7, 'hello') + tdSql.checkData(10, 8, 5) + tdSql.checkData(10, 9, 5.0) + tdSql.checkData(10, 10, 'childtable') + tdSql.checkData(11, 0, datetime.datetime(2019, 1, 1, 9, 23, 20, 1000)) + tdSql.checkData(11, 1, 65) + tdSql.checkData(11, 2, 19265) + tdSql.checkData(11, 3, 5000001) + tdSql.checkData(11, 4, 5000001) + tdSql.checkData(11, 5, 5000001.0) + tdSql.checkData(11, 6, 10000002.0) + tdSql.checkData(11, 7, 'hello') + tdSql.checkData(11, 8, 5) + tdSql.checkData(11, 9, 5.0) + tdSql.checkData(11, 10, 'childtable') + tdSql.checkData(12, 0, datetime.datetime(2019, 1, 1, 9, 40)) + tdSql.checkData(12, 1, -128) + tdSql.checkData(12, 2, -29312) + tdSql.checkData(12, 3, 6000000) + tdSql.checkData(12, 4, 6000000) + tdSql.checkData(12, 5, 6000000.0) + tdSql.checkData(12, 6, 12000000.0) + tdSql.checkData(12, 7, 'hello') + tdSql.checkData(12, 8, 6) + tdSql.checkData(12, 9, 6.0) + tdSql.checkData(12, 10, 'childtable') + tdSql.checkData(13, 0, datetime.datetime(2019, 1, 1, 9, 40, 0, 1000)) + tdSql.checkData(13, 1, -127) + tdSql.checkData(13, 2, -29311) + tdSql.checkData(13, 3, 6000001) + tdSql.checkData(13, 4, 6000001) + tdSql.checkData(13, 5, 6000001.0) + tdSql.checkData(13, 6, 12000002.0) + tdSql.checkData(13, 7, 'hello') + tdSql.checkData(13, 8, 6) + tdSql.checkData(13, 9, 6.0) + tdSql.checkData(13, 10, 'childtable') + tdSql.checkData(14, 0, datetime.datetime(2019, 1, 1, 9, 56, 40)) + tdSql.checkData(14, 1, -64) + tdSql.checkData(14, 2, -12352) + tdSql.checkData(14, 3, 7000000) + tdSql.checkData(14, 4, 7000000) + tdSql.checkData(14, 5, 7000000.0) + tdSql.checkData(14, 6, 14000000.0) + tdSql.checkData(14, 7, 'hello') + tdSql.checkData(14, 8, 7) + tdSql.checkData(14, 9, 7.0) + tdSql.checkData(14, 10, 'childtable') + tdSql.checkData(15, 0, datetime.datetime(2019, 1, 1, 9, 56, 40, 1000)) + tdSql.checkData(15, 1, -63) + tdSql.checkData(15, 2, -12351) + tdSql.checkData(15, 3, 7000001) + tdSql.checkData(15, 4, 7000001) + tdSql.checkData(15, 5, 7000001.0) + tdSql.checkData(15, 6, 14000002.0) + tdSql.checkData(15, 7, 'hello') + tdSql.checkData(15, 8, 7) + tdSql.checkData(15, 9, 7.0) + tdSql.checkData(15, 10, 'childtable') + tdSql.checkData(16, 0, datetime.datetime(2019, 1, 1, 10, 13, 20)) + tdSql.checkData(16, 1, 0) + tdSql.checkData(16, 2, 4608) + tdSql.checkData(16, 3, 8000000) + tdSql.checkData(16, 4, 8000000) + tdSql.checkData(16, 5, 8000000.0) + tdSql.checkData(16, 6, 16000000.0) + tdSql.checkData(16, 7, 'hello') + tdSql.checkData(16, 8, 8) + tdSql.checkData(16, 9, 8.0) + tdSql.checkData(16, 10, 'childtable') + tdSql.checkData(17, 0, datetime.datetime(2019, 1, 1, 10, 13, 20, 1000)) + tdSql.checkData(17, 1, 1) + tdSql.checkData(17, 2, 4609) + tdSql.checkData(17, 3, 8000001) + tdSql.checkData(17, 4, 8000001) + tdSql.checkData(17, 5, 8000001.0) + tdSql.checkData(17, 6, 16000002.0) + tdSql.checkData(17, 7, 'hello') + tdSql.checkData(17, 8, 8) + tdSql.checkData(17, 9, 8.0) + tdSql.checkData(17, 10, 'childtable') + tdSql.checkData(18, 0, datetime.datetime(2019, 1, 1, 10, 30)) + tdSql.checkData(18, 1, 64) + tdSql.checkData(18, 2, 21568) + tdSql.checkData(18, 3, 9000000) + tdSql.checkData(18, 4, 9000000) + tdSql.checkData(18, 5, 9000000.0) + tdSql.checkData(18, 6, 18000000.0) + tdSql.checkData(18, 7, 'hello') + tdSql.checkData(18, 8, 9) + tdSql.checkData(18, 9, 9.0) + tdSql.checkData(18, 10, 'childtable') + tdSql.checkData(19, 0, datetime.datetime(2019, 1, 1, 10, 30, 0, 1000)) + tdSql.checkData(19, 1, 65) + tdSql.checkData(19, 2, 21569) + tdSql.checkData(19, 3, 9000001) + tdSql.checkData(19, 4, 9000001) + tdSql.checkData(19, 5, 9000001.0) + tdSql.checkData(19, 6, 18000002.0) + tdSql.checkData(19, 7, 'hello') + tdSql.checkData(19, 8, 9) + tdSql.checkData(19, 9, 9.0) + tdSql.checkData(19, 10, 'childtable') + + tdSql.execute('drop database insert_stb3') def run(self): self.run_normal() self.run_insert_stb() self.run_stmt_error() + self.run_consecutive_seq() def stop(self): From bd64b9102db7bb07d429745c02e88a6aab0a86b7 Mon Sep 17 00:00:00 2001 From: slzhou Date: Wed, 1 Nov 2023 13:25:54 +0800 Subject: [PATCH 41/45] fix: fix address sanitizer error --- source/libs/parser/src/parInsertSql.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index 8c404e146c..7f6ce59fdc 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -238,7 +238,8 @@ static int32_t parseBoundColumns(SInsertParseContext* pCxt, const char** pSql, E continue; } int16_t t = lastColIdx + 1; - int16_t index = insFindCol(&token, t, pBoundInfo->numOfCols, pSchema); + int16_t end = (boundColsType == BOUND_ALL_AND_TBNAME) ? (pBoundInfo->numOfCols - 1) : pBoundInfo->numOfCols; + int16_t index = insFindCol(&token, t, end, pSchema); if (index < 0 && t > 0) { index = insFindCol(&token, 0, t, pSchema); } From 5f52f8940f575aeb4eae2008d93214fab9759178 Mon Sep 17 00:00:00 2001 From: slzhou Date: Wed, 1 Nov 2023 16:24:54 +0800 Subject: [PATCH 42/45] restart ci --- tests/script/api/insert_stb.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/script/api/insert_stb.c b/tests/script/api/insert_stb.c index cadd3dc98c..7daf4503b8 100644 --- a/tests/script/api/insert_stb.c +++ b/tests/script/api/insert_stb.c @@ -149,10 +149,13 @@ int main(int argc, char *argv[]) { exit(1); } char* qstr = malloc(1024*1024); - for (int i =0; i < 5; ++i) + for (int i =0; i < 5; ++i) { testInsert(taos, qstr); - for (int i =0; i < 5; ++i) + } + printf("test insert into stb tbname\n\n"); + for (int i =0; i < 5; ++i) { testInsertStb(taos, qstr); + } free(qstr); taos_close(taos); taos_cleanup(); From 9b6e44756c717577ce7120d03c8423e1931cffa2 Mon Sep 17 00:00:00 2001 From: slzhou Date: Wed, 1 Nov 2023 16:32:42 +0800 Subject: [PATCH 43/45] restart ci --- tests/script/api/insert_stb.c | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/script/api/insert_stb.c b/tests/script/api/insert_stb.c index 7daf4503b8..66430b0337 100644 --- a/tests/script/api/insert_stb.c +++ b/tests/script/api/insert_stb.c @@ -149,6 +149,7 @@ int main(int argc, char *argv[]) { exit(1); } char* qstr = malloc(1024*1024); + printf("test insert into tb using stb\n\n"); for (int i =0; i < 5; ++i) { testInsert(taos, qstr); } From 89a9a5f19303953e773e0f31f175585b11213c11 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Wed, 1 Nov 2023 18:17:32 +0800 Subject: [PATCH 44/45] test:add testcase of the same ts data --- .../system-test/2-query/megeFileSttQuery.json | 91 +++++++++++++++++++ .../2-query/megeFileSttQueryUpdate.json | 91 +++++++++++++++++++ .../system-test/2-query/mergeFileSttQuery.py | 71 +++++++++++++++ 3 files changed, 253 insertions(+) create mode 100644 tests/system-test/2-query/megeFileSttQuery.json create mode 100644 tests/system-test/2-query/megeFileSttQueryUpdate.json create mode 100644 tests/system-test/2-query/mergeFileSttQuery.py diff --git a/tests/system-test/2-query/megeFileSttQuery.json b/tests/system-test/2-query/megeFileSttQuery.json new file mode 100644 index 0000000000..5e70674f58 --- /dev/null +++ b/tests/system-test/2-query/megeFileSttQuery.json @@ -0,0 +1,91 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "connection_pool_size": 8, + "thread_count": 100, + "create_table_thread_count": 7, + "result_file": "./insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "interlace_rows": 100, + "num_of_records_per_req": 100, + "prepared_rand": 10000, + "chinese": "no", + "databases": [ + { + "dbinfo": { + "name": "db", + "drop": "yes", + "replica": 1, + "precision": "ms", + "vgroups": 1, + "duration": "1d", + "keep": 3650, + "minRows": 100, + "maxRows": 200, + "STT_TRIGGER": 1, + "comp": 2 + }, + "super_tables": [ + { + "name": "meters", + "child_table_exists": "no", + "childtable_count": 1, + "childtable_prefix": "d", + "escape_character": "yes", + "auto_create_table": "no", + "batch_create_tbl_num": 5, + "data_source": "rand", + "insert_mode": "taosc", + "non_stop_mode": "no", + "line_protocol": "line", + "insert_rows": 10000, + "childtable_limit": 0, + "childtable_offset": 100, + "interlace_rows": 0, + "insert_interval": 0, + "partial_col_num": 0, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1000, + "start_timestamp": "2021-10-02 00:01:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "use_sample_ts": "no", + "tags_file": "", + "columns": [ + { + "type": "FLOAT", + "name": "current", + "count": 1, + "max": 12, + "min": 8 + }, + { "type": "INT", "name": "voltage", "max": 225, "min": 215 }, + { "type": "FLOAT", "name": "phase", "max": 1, "min": 0 } + ], + "tags": [ + { + "type": "INT", + "name": "groupid", + "max": -10000, + "min": 10000 + }, + { + "name": "location", + "type": "BINARY", + "len": 16, + "values": ["San Francisco", "Los Angles", "San Diego", + "San Jose", "Palo Alto", "Campbell", "Mountain View", + "Sunnyvale", "Santa Clara", "Cupertino"] + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/tests/system-test/2-query/megeFileSttQueryUpdate.json b/tests/system-test/2-query/megeFileSttQueryUpdate.json new file mode 100644 index 0000000000..5d5887649f --- /dev/null +++ b/tests/system-test/2-query/megeFileSttQueryUpdate.json @@ -0,0 +1,91 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "connection_pool_size": 8, + "thread_count": 100, + "create_table_thread_count": 7, + "result_file": "./insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "interlace_rows": 100, + "num_of_records_per_req": 100, + "prepared_rand": 10000, + "chinese": "no", + "databases": [ + { + "dbinfo": { + "name": "db", + "drop": "no", + "replica": 1, + "precision": "ms", + "vgroups": 1, + "duration": "1d", + "keep": 3650, + "minRows": 100, + "maxRows": 200, + "STT_TRIGGER": 1, + "comp": 2 + }, + "super_tables": [ + { + "name": "meters", + "child_table_exists": "yes", + "childtable_count": 1, + "childtable_prefix": "d", + "escape_character": "yes", + "auto_create_table": "no", + "batch_create_tbl_num": 5, + "data_source": "rand", + "insert_mode": "taosc", + "non_stop_mode": "no", + "line_protocol": "line", + "insert_rows": 4, + "childtable_limit": 0, + "childtable_offset": 100, + "interlace_rows": 0, + "insert_interval": 0, + "partial_col_num": 0, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 3600000, + "start_timestamp": "2021-10-02 00:00:00.001", + "sample_format": "csv", + "sample_file": "./sample.csv", + "use_sample_ts": "no", + "tags_file": "", + "columns": [ + { + "type": "FLOAT", + "name": "current", + "count": 1, + "max": 12, + "min": 8 + }, + { "type": "INT", "name": "voltage", "max": 225, "min": 215 }, + { "type": "FLOAT", "name": "phase", "max": 1, "min": 0 } + ], + "tags": [ + { + "type": "INT", + "name": "groupid", + "max": -10000, + "min": 10000 + }, + { + "name": "location", + "type": "BINARY", + "len": 16, + "values": ["San Francisco", "Los Angles", "San Diego", + "San Jose", "Palo Alto", "Campbell", "Mountain View", + "Sunnyvale", "Santa Clara", "Cupertino"] + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/tests/system-test/2-query/mergeFileSttQuery.py b/tests/system-test/2-query/mergeFileSttQuery.py new file mode 100644 index 0000000000..7d2695a760 --- /dev/null +++ b/tests/system-test/2-query/mergeFileSttQuery.py @@ -0,0 +1,71 @@ +import sys +import os +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * +import time +from datetime import datetime + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.rowNum = 10 + self.ts = 1537146000000 + + def getPath(self, tool="taosBenchmark"): + if (platform.system().lower() == 'windows'): + tool = tool + ".exe" + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + paths = [] + for root, dirs, files in os.walk(projPath): + if ((tool) in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + paths.append(os.path.join(root, tool)) + break + if (len(paths) == 0): + tdLog.exit("taosBenchmark not found!") + return + else: + tdLog.info("taosBenchmark found in %s" % paths[0]) + return paths[0] + + def run(self): + binPath = self.getPath() + tdLog.debug("insert full data block and flush db") + os.system(f"{binPath} -f ./2-query/megeFileSttQuery.json") + tdSql.execute("flush database db;") + tdLog.debug("insert disorder data and flush db") + os.system(f"{binPath} -f ./2-query/megeFileSttQueryUpdate.json") + tdSql.execute("flush database db;") + tdLog.debug("check data") + tdSql.query("select ts from db.d0 limit 5;") + tdSql.checkData(0, 0, '2021-10-02 00:00:00.001') + tdSql.checkData(1, 0, '2021-10-02 00:01:00.000') + tdLog.debug("update disorder data and flush db") + + os.system(f"{binPath} -f ./2-query/megeFileSttQueryUpdate.json") + tdSql.query("select ts from db.d0 limit 5;") + tdSql.checkData(0, 0, '2021-10-02 00:00:00.001') + tdSql.checkData(1, 0, '2021-10-02 00:01:00.000') + tdSql.execute("flush database db;") + tdSql.query("select ts from db.d0 limit 5;") + tdSql.checkData(0, 0, '2021-10-02 00:00:00.001') + tdSql.checkData(1, 0, '2021-10-02 00:01:00.000') + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) From a6ee7c4685dc0e0b9469f056c04f233a43aafe77 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 1 Nov 2023 20:34:47 +0800 Subject: [PATCH 45/45] fix: possible coredump --- source/dnode/vnode/src/tsdb/tsdbMerge.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbMerge.c b/source/dnode/vnode/src/tsdb/tsdbMerge.c index 0c20a342d3..0db8cf85ed 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMerge.c +++ b/source/dnode/vnode/src/tsdb/tsdbMerge.c @@ -483,11 +483,12 @@ _exit: } static int32_t tsdbDoMerge(SMerger *merger) { - int32_t code = 0; - int32_t lino = 0; - SSttLvl *lvl = TARRAY2_FIRST(merger->fset->lvlArr); + int32_t code = 0; + int32_t lino = 0; if (TARRAY2_SIZE(merger->fset->lvlArr) == 0) return 0; + + SSttLvl *lvl = TARRAY2_FIRST(merger->fset->lvlArr); if (lvl->level != 0 || TARRAY2_SIZE(lvl->fobjArr) < merger->sttTrigger) return 0; code = tsdbMergerOpen(merger);