From 438a775c78ffee221c50592d20996279a0b0ba32 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Sat, 21 Oct 2023 14:02:03 +0800 Subject: [PATCH 01/17] fix: parse insert stb columns when meta is ready or ready after get meta --- source/libs/parser/src/parInsertSql.c | 66 ++++++++++++++------------- 1 file changed, 34 insertions(+), 32 deletions(-) diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index 1351a48a52..24e1ab226c 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -1755,33 +1755,6 @@ static int32_t parseDataClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pS return buildSyntaxErrMsg(&pCxt->msg, "keyword VALUES or FILE is expected", token.z); } -// input pStmt->pSql: -// 1. [(tag1_name, ...)] ... -// 2. VALUES ... | FILE ... -static int32_t parseInsertTableClauseBottom(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) { - STableDataCxt* pTableCxt = NULL; - int32_t code = parseSchemaClauseBottom(pCxt, pStmt, &pTableCxt); - if (TSDB_CODE_SUCCESS == code) { - code = parseDataClause(pCxt, pStmt, pTableCxt); - } - return code; -} - -static void resetEnvPreTable(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) { - insDestroyBoundColInfo(&pCxt->tags); - taosMemoryFreeClear(pStmt->pTableMeta); - nodesDestroyNode(pStmt->pTagCond); - taosArrayDestroy(pStmt->pTableTag); - tdDestroySVCreateTbReq(pStmt->pCreateTblReq); - taosMemoryFreeClear(pStmt->pCreateTblReq); - pCxt->missCache = false; - pCxt->usingDuplicateTable = false; - pStmt->pBoundCols = NULL; - pStmt->usingTableProcessing = false; - pStmt->fileProcessing = false; - pStmt->usingTableName.type = 0; - pStmt->stbSyntax = false; -} static int32_t parseStbBoundColumnsClause(SInsertParseContext* pCxt, const char* pBoundCols, STableMeta* pTableMeta, SBoundColInfo* pBoundColsInfo) { @@ -1802,16 +1775,45 @@ static int32_t parseInsertStbClauseBottom(SInsertParseContext* pCxt, SVnodeModif return code; } +// input pStmt->pSql: +// 1. [(tag1_name, ...)] ... +// 2. VALUES ... | FILE ... +static int32_t parseInsertTableClauseBottom(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) { + if (!pStmt->stbSyntax) { + STableDataCxt* pTableCxt = NULL; + int32_t code = parseSchemaClauseBottom(pCxt, pStmt, &pTableCxt); + if (TSDB_CODE_SUCCESS == code) { + code = parseDataClause(pCxt, pStmt, pTableCxt); + } + return code; + } else { + int32_t code = parseInsertStbClauseBottom(pCxt, pStmt); + return code; + } +} + +static void resetEnvPreTable(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) { + insDestroyBoundColInfo(&pCxt->tags); + taosMemoryFreeClear(pStmt->pTableMeta); + nodesDestroyNode(pStmt->pTagCond); + taosArrayDestroy(pStmt->pTableTag); + tdDestroySVCreateTbReq(pStmt->pCreateTblReq); + taosMemoryFreeClear(pStmt->pCreateTblReq); + pCxt->missCache = false; + pCxt->usingDuplicateTable = false; + pStmt->pBoundCols = NULL; + pStmt->usingTableProcessing = false; + pStmt->fileProcessing = false; + pStmt->usingTableName.type = 0; + pStmt->stbSyntax = false; +} + // input pStmt->pSql: [(field1_name, ...)] [ USING ... ] VALUES ... | FILE ... static int32_t parseInsertTableClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, SToken* pTbName) { resetEnvPreTable(pCxt, pStmt); int32_t code = parseSchemaClauseTop(pCxt, pStmt, pTbName); if (TSDB_CODE_SUCCESS == code && !pCxt->missCache) { - if (!pStmt->stbSyntax) { - code = parseInsertTableClauseBottom(pCxt, pStmt); - } else { - code = parseInsertStbClauseBottom(pCxt, pStmt); - } + code = parseInsertTableClauseBottom(pCxt, pStmt); } return code; From 6488672dd642b6e9b3a27fc3f171cff168a761f5 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Sun, 22 Oct 2023 11:21:36 +0800 Subject: [PATCH 02/17] enhance: parse target column --- source/libs/parser/src/parInsertSql.c | 74 +++++++++++++++++++++++++-- 1 file changed, 70 insertions(+), 4 deletions(-) diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index 24e1ab226c..a4f38039de 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -1755,10 +1755,73 @@ static int32_t parseDataClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pS return buildSyntaxErrMsg(&pCxt->msg, "keyword VALUES or FILE is expected", token.z); } +static int32_t parseBoundStbColumnsClause(SInsertParseContext* pCxt, const char** pSql, + STableMeta* pTableMeta, SBoundColInfo* pBoundInfo) { + // tbname schema index == (pBoundInfo->numOfCols - 1) == (tiInfo.numOfColumns + tiInfo.numOfTags) + if (pBoundInfo->numOfCols != pTableMeta->tableInfo.numOfTags + pTableMeta->tableInfo.numOfColumns + 1) { + return TSDB_CODE_PAR_INTERNAL_ERROR; + } + int32_t tbnameSchemaIndex = pTableMeta->tableInfo.numOfTags + pTableMeta->tableInfo.numOfColumns; -static int32_t parseStbBoundColumnsClause(SInsertParseContext* pCxt, const char* pBoundCols, - STableMeta* pTableMeta, SBoundColInfo* pBoundColsInfo) { - return TSDB_CODE_SUCCESS; + bool* pUseCols = taosMemoryCalloc(pBoundInfo->numOfCols, sizeof(bool)); + if (NULL == pUseCols) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + pBoundInfo->numOfBound = 0; + + int16_t lastColIdx = -1; // last column found + int32_t code = TSDB_CODE_SUCCESS; + while (TSDB_CODE_SUCCESS == code) { + SToken token; + NEXT_TOKEN(*pSql, token); + + if (TK_NK_RP == token.type) { + break; + } + + char tmpTokenBuf[TSDB_COL_NAME_LEN + 2] = {0}; // used for deleting Escape character backstick(`) + strncpy(tmpTokenBuf, token.z, token.n); + token.z = tmpTokenBuf; + token.n = strdequote(token.z); + + if (token.n == strlen("tbname") && (strcasecmp(token.z, "tbname") == 0)) { + pBoundInfo->pColIndex[pBoundInfo->numOfBound] = tbnameSchemaIndex; + pUseCols[tbnameSchemaIndex] = true; + ++pBoundInfo->numOfBound; + continue; + } + + int16_t t = lastColIdx + 1; + int16_t index = insFindCol(&token, t, pBoundInfo->numOfCols, pTableMeta->schema); + if (index < 0 && t > 0) { + index = insFindCol(&token, 0, t, pTableMeta->schema); + } + if (index < 0) { + code = generateSyntaxErrMsg(&pCxt->msg, TSDB_CODE_PAR_INVALID_COLUMN, token.z); + } else if (pUseCols[index]) { + code = buildSyntaxErrMsg(&pCxt->msg, "duplicated column name", token.z); + } else { + lastColIdx = index; + pUseCols[index] = true; + pBoundInfo->pColIndex[pBoundInfo->numOfBound] = index; + ++pBoundInfo->numOfBound; + } + } + + if (TSDB_CODE_SUCCESS == code && !pUseCols[0]) { + code = buildInvalidOperationMsg(&pCxt->msg, "primary timestamp column can not be null"); + } + if (TSDB_CODE_SUCCESS == code && !pUseCols[tbnameSchemaIndex]) { + code = buildInvalidOperationMsg(&pCxt->msg, "tbname column can not be null"); + } + taosMemoryFree(pUseCols); + + return code; +} + +static int32_t parseDataStbClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, SBoundColInfo* pBoundColInfo) { + return TSDB_CODE_SUCCESS; } static int32_t parseInsertStbClauseBottom(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) { @@ -1771,7 +1834,10 @@ static int32_t parseInsertStbClauseBottom(SInsertParseContext* pCxt, SVnodeModif } SToken token; int32_t index = 0; - parseStbBoundColumnsClause(pCxt, pStmt->pBoundCols, pStmt->pTableMeta, &stbBoundColInfo); + code = parseBoundStbColumnsClause(pCxt, &pStmt->pBoundCols, pStmt->pTableMeta, &stbBoundColInfo); + if (TSDB_CODE_SUCCESS == code) { + code = parseDataStbClause(pCxt, pStmt, &stbBoundColInfo); + } return code; } From 5156c0d00c7db71990a8e4a99d541e1e04d12960 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Sun, 22 Oct 2023 15:14:50 +0800 Subject: [PATCH 03/17] enhance: get tbname from one row for stb syntax --- source/libs/parser/src/parInsertSql.c | 143 ++++++++++++-------------- 1 file changed, 66 insertions(+), 77 deletions(-) diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index a4f38039de..cf7726e81b 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -182,9 +182,32 @@ static int32_t parseDuplicateUsingClause(SInsertParseContext* pCxt, SVnodeModify return TSDB_CODE_SUCCESS; } +typedef enum { + BOUND_TAGS, + BOUND_COLUMNS, + BOUND_ALL_AND_TBNAME +} EBoundColumnsType; + +static int32_t getTbnameSchemaIndex(STableMeta* pTableMeta) { + return pTableMeta->tableInfo.numOfTags + pTableMeta->tableInfo.numOfColumns; +} + // pStmt->pSql -> field1_name, ...) -static int32_t parseBoundColumns(SInsertParseContext* pCxt, const char** pSql, bool isTags, SSchema* pSchema, +static int32_t parseBoundColumns(SInsertParseContext* pCxt, const char** pSql, EBoundColumnsType boundColsType, STableMeta* pTableMeta, SBoundColInfo* pBoundInfo) { + SSchema* pSchema = NULL; + if (boundColsType == BOUND_TAGS) { + pSchema = getTableTagSchema(pTableMeta); + } else if (boundColsType == BOUND_COLUMNS) { + pSchema = getTableColumnSchema(pTableMeta); + } else { + pSchema = pTableMeta->schema; + if (pBoundInfo->numOfCols != getTbnameSchemaIndex(pTableMeta) + 1) { + return TSDB_CODE_PAR_INTERNAL_ERROR; + } + } + int32_t tbnameSchemaIndex = getTbnameSchemaIndex(pTableMeta); + bool* pUseCols = taosMemoryCalloc(pBoundInfo->numOfCols, sizeof(bool)); if (NULL == pUseCols) { return TSDB_CODE_OUT_OF_MEMORY; @@ -207,6 +230,14 @@ static int32_t parseBoundColumns(SInsertParseContext* pCxt, const char** pSql, b token.z = tmpTokenBuf; token.n = strdequote(token.z); + if (boundColsType == BOUND_ALL_AND_TBNAME) { + if (token.n == strlen("tbname") && (strcasecmp(token.z, "tbname") == 0)) { + pBoundInfo->pColIndex[pBoundInfo->numOfBound] = tbnameSchemaIndex; + pUseCols[tbnameSchemaIndex] = true; + ++pBoundInfo->numOfBound; + continue; + } + } int16_t t = lastColIdx + 1; int16_t index = insFindCol(&token, t, pBoundInfo->numOfCols, pSchema); if (index < 0 && t > 0) { @@ -224,10 +255,12 @@ static int32_t parseBoundColumns(SInsertParseContext* pCxt, const char** pSql, b } } - if (TSDB_CODE_SUCCESS == code && !isTags && !pUseCols[0]) { + if (TSDB_CODE_SUCCESS == code && (BOUND_TAGS != boundColsType) && !pUseCols[0]) { code = buildInvalidOperationMsg(&pCxt->msg, "primary timestamp column can not be null"); } - + if (TSDB_CODE_SUCCESS == code && (BOUND_ALL_AND_TBNAME == boundColsType) &&!pUseCols[tbnameSchemaIndex]) { + code = buildInvalidOperationMsg(&pCxt->msg, "tbname column can not be null"); + } taosMemoryFree(pUseCols); return code; @@ -586,7 +619,7 @@ static int32_t parseBoundTagsClause(SInsertParseContext* pCxt, SVnodeModifyOpStm } pStmt->pSql += index; - return parseBoundColumns(pCxt, &pStmt->pSql, true, getTableTagSchema(pStmt->pTableMeta), &pCxt->tags); + return parseBoundColumns(pCxt, &pStmt->pSql, BOUND_TAGS, pStmt->pTableMeta, &pCxt->tags); } static int32_t parseTagValue(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, SSchema* pTagSchema, SToken* pToken, @@ -1220,12 +1253,12 @@ static int32_t parseBoundColumnsClause(SInsertParseContext* pCxt, SVnodeModifyOp return buildSyntaxErrMsg(&pCxt->msg, "keyword VALUES or FILE is expected", token.z); } // pStmt->pSql -> field1_name, ...) - return parseBoundColumns(pCxt, &pStmt->pSql, false, getTableColumnSchema(pStmt->pTableMeta), + return parseBoundColumns(pCxt, &pStmt->pSql, BOUND_COLUMNS, pStmt->pTableMeta, &pTableCxt->boundColsInfo); } if (NULL != pStmt->pBoundCols) { - return parseBoundColumns(pCxt, &pStmt->pBoundCols, false, getTableColumnSchema(pStmt->pTableMeta), + return parseBoundColumns(pCxt, &pStmt->pBoundCols, BOUND_COLUMNS, pStmt->pTableMeta, &pTableCxt->boundColsInfo); } @@ -1521,8 +1554,12 @@ static void clearColValArray(SArray* pCols) { } } +typedef struct SInsertStbParseContext { + +} SInsertStbParseContext; + static int parseOneRow(SInsertParseContext* pCxt, const char** pSql, STableDataCxt* pTableCxt, bool* pGotRow, - SToken* pToken) { + SToken* pToken, bool *pGotTbName, char* tbName) { SBoundColInfo* pCols = &pTableCxt->boundColsInfo; bool isParseBindParam = false; SSchema* pSchemas = getTableColumnSchema(pTableCxt->pMeta); @@ -1559,7 +1596,21 @@ static int parseOneRow(SInsertParseContext* pCxt, const char** pSql, STableDataC } if (TSDB_CODE_SUCCESS == code) { - code = parseValueToken(pCxt, pSql, pToken, pSchema, getTableInfo(pTableCxt->pMeta).precision, pVal); + if (pCols->pColIndex[i] == getTbnameSchemaIndex(pTableCxt->pMeta)) { + SColVal tbnameVal; + code = parseValueToken(pCxt, pSql, pToken, (SSchema*)tGetTbnameColumnSchema(), getTableInfo(pTableCxt->pMeta).precision, &tbnameVal); + if (code == TSDB_CODE_SUCCESS) { + if (pGotTbName != NULL) { + *pGotTbName = true; + } + if (tbName != NULL) { + memcpy(tbName, tbnameVal.value.pData, tbnameVal.value.nData); + tbName[tbnameVal.value.nData] = '\0'; + } + } + } else { + code = parseValueToken(pCxt, pSql, pToken, pSchema, getTableInfo(pTableCxt->pMeta).precision, pVal); + } } } @@ -1603,8 +1654,10 @@ static int32_t parseValues(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, pStmt->pSql += index; bool gotRow = false; + bool gotTbname = false; + char tbName[TSDB_TABLE_NAME_LEN] = {0}; if (TSDB_CODE_SUCCESS == code) { - code = parseOneRow(pCxt, &pStmt->pSql, pTableCxt, &gotRow, pToken); + code = parseOneRow(pCxt, &pStmt->pSql, pTableCxt, &gotRow, pToken, &gotTbname, tbName); } if (TSDB_CODE_SUCCESS == code) { @@ -1664,8 +1717,9 @@ static int32_t parseCsvFile(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt SToken token; strtolower(pLine, pLine); const char* pRow = pLine; - - code = parseOneRow(pCxt, (const char**)&pRow, pTableCxt, &gotRow, &token); + bool gotTbname = false; + char tbName[TSDB_TABLE_NAME_LEN] = {0}; + code = parseOneRow(pCxt, (const char**)&pRow, pTableCxt, &gotRow, &token, &gotTbname, tbName); if (code && firstLine) { firstLine = false; code = 0; @@ -1755,71 +1809,6 @@ static int32_t parseDataClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pS return buildSyntaxErrMsg(&pCxt->msg, "keyword VALUES or FILE is expected", token.z); } -static int32_t parseBoundStbColumnsClause(SInsertParseContext* pCxt, const char** pSql, - STableMeta* pTableMeta, SBoundColInfo* pBoundInfo) { - // tbname schema index == (pBoundInfo->numOfCols - 1) == (tiInfo.numOfColumns + tiInfo.numOfTags) - if (pBoundInfo->numOfCols != pTableMeta->tableInfo.numOfTags + pTableMeta->tableInfo.numOfColumns + 1) { - return TSDB_CODE_PAR_INTERNAL_ERROR; - } - int32_t tbnameSchemaIndex = pTableMeta->tableInfo.numOfTags + pTableMeta->tableInfo.numOfColumns; - - bool* pUseCols = taosMemoryCalloc(pBoundInfo->numOfCols, sizeof(bool)); - if (NULL == pUseCols) { - return TSDB_CODE_OUT_OF_MEMORY; - } - - pBoundInfo->numOfBound = 0; - - int16_t lastColIdx = -1; // last column found - int32_t code = TSDB_CODE_SUCCESS; - while (TSDB_CODE_SUCCESS == code) { - SToken token; - NEXT_TOKEN(*pSql, token); - - if (TK_NK_RP == token.type) { - break; - } - - char tmpTokenBuf[TSDB_COL_NAME_LEN + 2] = {0}; // used for deleting Escape character backstick(`) - strncpy(tmpTokenBuf, token.z, token.n); - token.z = tmpTokenBuf; - token.n = strdequote(token.z); - - if (token.n == strlen("tbname") && (strcasecmp(token.z, "tbname") == 0)) { - pBoundInfo->pColIndex[pBoundInfo->numOfBound] = tbnameSchemaIndex; - pUseCols[tbnameSchemaIndex] = true; - ++pBoundInfo->numOfBound; - continue; - } - - int16_t t = lastColIdx + 1; - int16_t index = insFindCol(&token, t, pBoundInfo->numOfCols, pTableMeta->schema); - if (index < 0 && t > 0) { - index = insFindCol(&token, 0, t, pTableMeta->schema); - } - if (index < 0) { - code = generateSyntaxErrMsg(&pCxt->msg, TSDB_CODE_PAR_INVALID_COLUMN, token.z); - } else if (pUseCols[index]) { - code = buildSyntaxErrMsg(&pCxt->msg, "duplicated column name", token.z); - } else { - lastColIdx = index; - pUseCols[index] = true; - pBoundInfo->pColIndex[pBoundInfo->numOfBound] = index; - ++pBoundInfo->numOfBound; - } - } - - if (TSDB_CODE_SUCCESS == code && !pUseCols[0]) { - code = buildInvalidOperationMsg(&pCxt->msg, "primary timestamp column can not be null"); - } - if (TSDB_CODE_SUCCESS == code && !pUseCols[tbnameSchemaIndex]) { - code = buildInvalidOperationMsg(&pCxt->msg, "tbname column can not be null"); - } - taosMemoryFree(pUseCols); - - return code; -} - static int32_t parseDataStbClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, SBoundColInfo* pBoundColInfo) { return TSDB_CODE_SUCCESS; } @@ -1834,7 +1823,7 @@ static int32_t parseInsertStbClauseBottom(SInsertParseContext* pCxt, SVnodeModif } SToken token; int32_t index = 0; - code = parseBoundStbColumnsClause(pCxt, &pStmt->pBoundCols, pStmt->pTableMeta, &stbBoundColInfo); + code = parseBoundColumns(pCxt, &pStmt->pBoundCols, BOUND_ALL_AND_TBNAME, pStmt->pTableMeta, &stbBoundColInfo); if (TSDB_CODE_SUCCESS == code) { code = parseDataStbClause(pCxt, pStmt, &stbBoundColInfo); } From e713b7c22fdd2cf9ae610a95513cc5652ae4df57 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Mon, 23 Oct 2023 16:27:00 +0800 Subject: [PATCH 04/17] enhance: insert stb skeleton development --- include/libs/nodes/querynodes.h | 4 +- source/libs/parser/src/parInsertSql.c | 219 ++++++++++++++++++-------- 2 files changed, 158 insertions(+), 65 deletions(-) diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index bfd23d1965..6d8f16986d 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -413,6 +413,7 @@ typedef struct SVgDataBlocks { typedef void (*FFreeTableBlockHash)(SHashObj*); typedef void (*FFreeVgourpBlockArray)(SArray*); +struct SStbRowsDataContext; typedef struct SVnodeModifyOpStmt { ENodeType nodeType; ENodeType sqlNodeType; @@ -441,8 +442,7 @@ typedef struct SVnodeModifyOpStmt { bool fileProcessing; bool stbSyntax; - SName superTableName; - SName childTableName; + struct SStbRowsDataContext* pStbRowsCxt; } SVnodeModifyOpStmt; typedef struct SExplainOptions { diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index cf7726e81b..aa8704a62b 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -622,7 +622,7 @@ static int32_t parseBoundTagsClause(SInsertParseContext* pCxt, SVnodeModifyOpStm return parseBoundColumns(pCxt, &pStmt->pSql, BOUND_TAGS, pStmt->pTableMeta, &pCxt->tags); } -static int32_t parseTagValue(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, SSchema* pTagSchema, SToken* pToken, +static int32_t parseTagValue(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, const char** ppSql, SSchema* pTagSchema, SToken* pToken, SArray* pTagName, SArray* pTagVals, STag** pTag) { if (!isNullValue(pTagSchema->type, pToken)) { taosArrayPush(pTagName, pTagSchema->name); @@ -642,7 +642,7 @@ static int32_t parseTagValue(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStm STagVal val = {0}; int32_t code = - parseTagToken(&pStmt->pSql, pToken, pTagSchema, pStmt->pTableMeta->tableInfo.precision, &val, &pCxt->msg); + parseTagToken(ppSql, pToken, pTagSchema, pStmt->pTableMeta->tableInfo.precision, &val, &pCxt->msg); if (TSDB_CODE_SUCCESS == code) { taosArrayPush(pTagVals, &val); } @@ -845,7 +845,7 @@ static int32_t parseTagsClauseImpl(SInsertParseContext* pCxt, SVnodeModifyOpStmt isJson = pTagSchema->type == TSDB_DATA_TYPE_JSON; code = checkAndTrimValue(&token, pCxt->tmpTokenBuf, &pCxt->msg); if (TSDB_CODE_SUCCESS == code) { - code = parseTagValue(pCxt, pStmt, pTagSchema, &token, pTagName, pTagVals, &pTag); + code = parseTagValue(pCxt, pStmt, &pStmt->pSql, pTagSchema, &token, pTagName, pTagVals, &pTag); } } @@ -900,7 +900,7 @@ static int32_t parseTagsClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pS return code; } -static int32_t storeTableMeta(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) { +static int32_t storeChildTableMeta(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) { pStmt->pTableMeta->suid = pStmt->pTableMeta->uid; pStmt->pTableMeta->uid = pStmt->totalTbNum; pStmt->pTableMeta->tableType = TSDB_CHILD_TABLE; @@ -1038,7 +1038,7 @@ static int32_t getTableMeta(SInsertParseContext* pCxt, SName* pTbName, STableMet return code; } -static int32_t getTableVgroup(SParseContext* pCxt, SVnodeModifyOpStmt* pStmt, bool isStb, bool* pMissCache) { +static int32_t getTargetTableVgroup(SParseContext* pCxt, SVnodeModifyOpStmt* pStmt, bool isStb, bool* pMissCache) { int32_t code = TSDB_CODE_SUCCESS; SVgroupInfo vg; bool exists = true; @@ -1074,7 +1074,6 @@ static int32_t getTargetTableMetaAndVgroup(SInsertParseContext* pCxt, SVnodeModi if (NULL != pStmt->pTableMeta) { if (pStmt->pTableMeta->tableType == TSDB_SUPER_TABLE) { pStmt->stbSyntax = true; - tNameAssign(&pStmt->superTableName, &pStmt->targetTableName); } else { code = taosHashPut(pStmt->pVgroupsHashObj, (const char*)&vg.vgId, sizeof(vg.vgId), (char*)&vg, sizeof(vg)); } @@ -1088,10 +1087,9 @@ static int32_t getTargetTableMetaAndVgroup(SInsertParseContext* pCxt, SVnodeModi if (TSDB_CODE_SUCCESS == code && !pCxt->missCache) { if (TSDB_SUPER_TABLE == pStmt->pTableMeta->tableType) { pStmt->stbSyntax = true; - tNameAssign(&pStmt->superTableName, &pStmt->targetTableName); } if (!pStmt->stbSyntax) { - code = getTableVgroup(pCxt->pComCxt, pStmt, false, &pCxt->missCache); + code = getTargetTableVgroup(pCxt->pComCxt, pStmt, false, &pCxt->missCache); } } } @@ -1155,7 +1153,7 @@ static int32_t getUsingTableSchema(SInsertParseContext* pCxt, SVnodeModifyOpStmt code = getTableMeta(pCxt, &pStmt->usingTableName, &pStmt->pTableMeta, &pCxt->missCache, bUsingTable); } if (TSDB_CODE_SUCCESS == code && !pCxt->missCache) { - code = getTableVgroup(pCxt->pComCxt, pStmt, true, &pCxt->missCache); + code = getTargetTableVgroup(pCxt->pComCxt, pStmt, true, &pCxt->missCache); } if (TSDB_CODE_SUCCESS == code && !pCxt->pComCxt->async) { code = collectUseDatabase(&pStmt->usingTableName, pStmt->pDbFNameHashObj); @@ -1174,7 +1172,7 @@ static int32_t parseUsingTableNameImpl(SInsertParseContext* pCxt, SVnodeModifyOp code = getUsingTableSchema(pCxt, pStmt); } if (TSDB_CODE_SUCCESS == code && !pCxt->missCache) { - code = storeTableMeta(pCxt, pStmt); + code = storeChildTableMeta(pCxt, pStmt); } return code; } @@ -1554,12 +1552,101 @@ static void clearColValArray(SArray* pCols) { } } -typedef struct SInsertStbParseContext { +typedef struct SStbRowsDataContext { + SVnodeModifyOpStmt* pStmt; + STableMeta* pStbMeta; + SName stbName; + SBoundColInfo boundColsInfo; + int32_t numOfBoundCols; + int32_t numOfBoundTags; -} SInsertStbParseContext; + SArray* aChildTableNames; + SArray* aTableDataCxts; + SArray* aCreateTbReqs; -static int parseOneRow(SInsertParseContext* pCxt, const char** pSql, STableDataCxt* pTableCxt, bool* pGotRow, - SToken* pToken, bool *pGotTbName, char* tbName) { + SArray* aTagVals; + SArray* aColVals; + SArray* aTagNames; +} SStbRowsDataContext; + +typedef union SRowsDataContext{ + STableDataCxt* pTableDataCxt; + SStbRowsDataContext* pStbRowsCxt; +} SRowsDataContext; + +static int32_t parseOneStbRow(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, const char** ppSql, SStbRowsDataContext* pStbRowsCxt, bool* pGotRow, + SToken* pToken) { + SBoundColInfo* pCols = &pStbRowsCxt->boundColsInfo; + SSchema* pSchemas = getTableColumnSchema(pStbRowsCxt->pStbMeta); + + bool isJsonTag = false; + SArray* pTagName = taosArrayInit(8, TSDB_COL_NAME_LEN); + SArray* pTagVals = taosArrayInit(pCxt->tags.numOfBound, sizeof(STagVal)); + STag* pTag = NULL; + + int32_t code = TSDB_CODE_SUCCESS; + for (int i = 0; i < pCols->numOfBound && TSDB_CODE_SUCCESS == code; ++i) { + const char* pOrigSql = *ppSql; + bool ignoreComma = false; + NEXT_TOKEN_WITH_PREV_EXT(*ppSql, *pToken, &ignoreComma); + if (ignoreComma) { + code = buildSyntaxErrMsg(&pCxt->msg, "invalid data or symbol", pOrigSql); + break; + } + + if (TK_NK_RP == pToken->type) { + code = generateSyntaxErrMsg(&pCxt->msg, TSDB_CODE_PAR_INVALID_COLUMNS_NUM); + break; + } + + if (pCols->pColIndex[i] < getNumOfColumns(pStbRowsCxt->pStbMeta)) { + SSchema* pSchema = &pSchemas[pCols->pColIndex[i]]; + SColVal* pVal = taosArrayGet(pStbRowsCxt->aColVals, pCols->pColIndex[i]); + code = parseValueToken(pCxt, ppSql, pToken, pSchema, getTableInfo(pStbRowsCxt->pStbMeta).precision, pVal); + } else if (pCols->pColIndex[i] < getTbnameSchemaIndex(pStbRowsCxt->pStbMeta)) { + SSchema* pTagSchema = &pSchemas[pCols->pColIndex[i]]; + isJsonTag = pTagSchema->type == TSDB_DATA_TYPE_JSON; + code = checkAndTrimValue(pToken, pCxt->tmpTokenBuf, &pCxt->msg); + if (TSDB_CODE_SUCCESS == code) { + code = parseTagValue(pCxt, pStmt, ppSql, pTagSchema, pToken, pTagName, pTagVals, &pTag); + } + } + else if (pCols->pColIndex[i] == getTbnameSchemaIndex(pStbRowsCxt->pStbMeta)) { + SColVal tbnameVal; + code = parseValueToken(pCxt, ppSql, pToken, (SSchema*)tGetTbnameColumnSchema(), + getTableInfo(pStbRowsCxt->pStbMeta).precision, &tbnameVal); + if (code == TSDB_CODE_SUCCESS) { + char tbName[TSDB_TABLE_NAME_LEN]; + memcpy(tbName, tbnameVal.value.pData, tbnameVal.value.nData); + tbName[tbnameVal.value.nData] = '\0'; + } + } + + if (TSDB_CODE_SUCCESS == code && i < pCols->numOfBound - 1) { + NEXT_VALID_TOKEN(*ppSql, *pToken); + if (TK_NK_COMMA != pToken->type) { + code = buildSyntaxErrMsg(&pCxt->msg, ", expected", pToken->z); + } + } + } + + // child meta , vgroupid, check privilege + // refer to parInsertSml.c + // create or reuse table data context from tbName + // init submit data + // build create table request, refer to parseTagsClauseImpl + // build row and check table data order, refer to parseOneRow + + if (TSDB_CODE_SUCCESS == code) { + *pGotRow = true; + } + + return code; + + return TSDB_CODE_SUCCESS; +} + +static int parseOneRow(SInsertParseContext* pCxt, const char** pSql, STableDataCxt* pTableCxt, bool* pGotRow, SToken* pToken) { SBoundColInfo* pCols = &pTableCxt->boundColsInfo; bool isParseBindParam = false; SSchema* pSchemas = getTableColumnSchema(pTableCxt->pMeta); @@ -1596,21 +1683,7 @@ static int parseOneRow(SInsertParseContext* pCxt, const char** pSql, STableDataC } if (TSDB_CODE_SUCCESS == code) { - if (pCols->pColIndex[i] == getTbnameSchemaIndex(pTableCxt->pMeta)) { - SColVal tbnameVal; - code = parseValueToken(pCxt, pSql, pToken, (SSchema*)tGetTbnameColumnSchema(), getTableInfo(pTableCxt->pMeta).precision, &tbnameVal); - if (code == TSDB_CODE_SUCCESS) { - if (pGotTbName != NULL) { - *pGotTbName = true; - } - if (tbName != NULL) { - memcpy(tbName, tbnameVal.value.pData, tbnameVal.value.nData); - tbName[tbnameVal.value.nData] = '\0'; - } - } - } else { - code = parseValueToken(pCxt, pSql, pToken, pSchema, getTableInfo(pTableCxt->pMeta).precision, pVal); - } + code = parseValueToken(pCxt, pSql, pToken, pSchema, getTableInfo(pTableCxt->pMeta).precision, pVal); } } @@ -1640,7 +1713,7 @@ static int parseOneRow(SInsertParseContext* pCxt, const char** pSql, STableDataC } // pSql -> (field1_value, ...) [(field1_value2, ...) ...] -static int32_t parseValues(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, STableDataCxt* pTableCxt, +static int32_t parseValues(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, SRowsDataContext rowsDataCxt, int32_t* pNumOfRows, SToken* pToken) { int32_t code = TSDB_CODE_SUCCESS; @@ -1654,10 +1727,12 @@ static int32_t parseValues(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, pStmt->pSql += index; bool gotRow = false; - bool gotTbname = false; - char tbName[TSDB_TABLE_NAME_LEN] = {0}; if (TSDB_CODE_SUCCESS == code) { - code = parseOneRow(pCxt, &pStmt->pSql, pTableCxt, &gotRow, pToken, &gotTbname, tbName); + if (!pStmt->stbSyntax) { + code = parseOneRow(pCxt, &pStmt->pSql, rowsDataCxt.pTableDataCxt, &gotRow, pToken); + } else { + code = parseOneStbRow(pCxt, pStmt, &pStmt->pSql, rowsDataCxt.pStbRowsCxt, &gotRow, pToken); + } } if (TSDB_CODE_SUCCESS == code) { @@ -1682,10 +1757,10 @@ static int32_t parseValues(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, } // VALUES (field1_value, ...) [(field1_value2, ...) ...] -static int32_t parseValuesClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, STableDataCxt* pTableCxt, +static int32_t parseValuesClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, SRowsDataContext rowsDataContext, SToken* pToken) { int32_t numOfRows = 0; - int32_t code = parseValues(pCxt, pStmt, pTableCxt, &numOfRows, pToken); + int32_t code = parseValues(pCxt, pStmt, rowsDataContext, &numOfRows, pToken); if (TSDB_CODE_SUCCESS == code) { pStmt->totalRowsNum += numOfRows; pStmt->totalTbNum += 1; @@ -1694,7 +1769,7 @@ static int32_t parseValuesClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* return code; } -static int32_t parseCsvFile(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, STableDataCxt* pTableCxt, +static int32_t parseCsvFile(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, SRowsDataContext rowsDataCxt, int32_t* pNumOfRows) { int32_t code = TSDB_CODE_SUCCESS; (*pNumOfRows) = 0; @@ -1717,9 +1792,11 @@ static int32_t parseCsvFile(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt SToken token; strtolower(pLine, pLine); const char* pRow = pLine; - bool gotTbname = false; - char tbName[TSDB_TABLE_NAME_LEN] = {0}; - code = parseOneRow(pCxt, (const char**)&pRow, pTableCxt, &gotRow, &token, &gotTbname, tbName); + if (!pStmt->stbSyntax) { + code = parseOneRow(pCxt, (const char**)&pRow, rowsDataCxt.pTableDataCxt, &gotRow, &token); + } else { + code = parseOneStbRow(pCxt, pStmt, (const char**)&pRow, rowsDataCxt.pStbRowsCxt, &gotRow, &token); + } if (code && firstLine) { firstLine = false; code = 0; @@ -1749,9 +1826,9 @@ static int32_t parseCsvFile(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt return code; } -static int32_t parseDataFromFileImpl(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, STableDataCxt* pTableCxt) { +static int32_t parseDataFromFileImpl(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, SRowsDataContext rowsDataCxt) { int32_t numOfRows = 0; - int32_t code = parseCsvFile(pCxt, pStmt, pTableCxt, &numOfRows); + int32_t code = parseCsvFile(pCxt, pStmt, rowsDataCxt, &numOfRows); if (TSDB_CODE_SUCCESS == code) { pStmt->totalRowsNum += numOfRows; pStmt->totalTbNum += 1; @@ -1766,7 +1843,7 @@ static int32_t parseDataFromFileImpl(SInsertParseContext* pCxt, SVnodeModifyOpSt } static int32_t parseDataFromFile(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, SToken* pFilePath, - STableDataCxt* pTableCxt) { + SRowsDataContext rowsDataCxt) { char filePathStr[TSDB_FILENAME_LEN] = {0}; if (TK_NK_STRING == pFilePath->type) { trimString(pFilePath->z, pFilePath->n, filePathStr, sizeof(filePathStr)); @@ -1778,10 +1855,10 @@ static int32_t parseDataFromFile(SInsertParseContext* pCxt, SVnodeModifyOpStmt* return TAOS_SYSTEM_ERROR(errno); } - return parseDataFromFileImpl(pCxt, pStmt, pTableCxt); + return parseDataFromFileImpl(pCxt, pStmt, rowsDataCxt); } -static int32_t parseFileClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, STableDataCxt* pTableCxt, +static int32_t parseFileClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, SRowsDataContext rowsDataCxt, SToken* pToken) { if (tsUseAdapter) { return buildInvalidOperationMsg(&pCxt->msg, "proxy mode does not support csv loading"); @@ -1791,28 +1868,24 @@ static int32_t parseFileClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pS if (0 == pToken->n || (TK_NK_STRING != pToken->type && TK_NK_ID != pToken->type)) { return buildSyntaxErrMsg(&pCxt->msg, "file path is required following keyword FILE", pToken->z); } - return parseDataFromFile(pCxt, pStmt, pToken, pTableCxt); + return parseDataFromFile(pCxt, pStmt, pToken, rowsDataCxt); } // VALUES (field1_value, ...) [(field1_value2, ...) ...] | FILE csv_file_path -static int32_t parseDataClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, STableDataCxt* pTableCxt) { +static int32_t parseDataClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, SRowsDataContext rowsDataCxt) { SToken token; NEXT_TOKEN(pStmt->pSql, token); switch (token.type) { case TK_VALUES: - return parseValuesClause(pCxt, pStmt, pTableCxt, &token); + return parseValuesClause(pCxt, pStmt, rowsDataCxt, &token); case TK_FILE: - return parseFileClause(pCxt, pStmt, pTableCxt, &token); + return parseFileClause(pCxt, pStmt, rowsDataCxt, &token); default: break; } return buildSyntaxErrMsg(&pCxt->msg, "keyword VALUES or FILE is expected", token.z); } -static int32_t parseDataStbClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, SBoundColInfo* pBoundColInfo) { - return TSDB_CODE_SUCCESS; -} - static int32_t parseInsertStbClauseBottom(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) { int32_t code = TSDB_CODE_SUCCESS; STableComInfo tblInfo = getTableInfo(pStmt->pTableMeta); @@ -1824,9 +1897,16 @@ static int32_t parseInsertStbClauseBottom(SInsertParseContext* pCxt, SVnodeModif SToken token; int32_t index = 0; code = parseBoundColumns(pCxt, &pStmt->pBoundCols, BOUND_ALL_AND_TBNAME, pStmt->pTableMeta, &stbBoundColInfo); + pStmt->pStbRowsCxt = taosMemoryCalloc(1, sizeof(SStbRowsDataContext)); + + // fill SStbRowsDataCxt; + + SRowsDataContext rowsDataCxt; + rowsDataCxt.pStbRowsCxt = pStmt->pStbRowsCxt; if (TSDB_CODE_SUCCESS == code) { - code = parseDataStbClause(pCxt, pStmt, &stbBoundColInfo); + code = parseDataClause(pCxt, pStmt, rowsDataCxt); } + return code; } @@ -1837,8 +1917,10 @@ static int32_t parseInsertTableClauseBottom(SInsertParseContext* pCxt, SVnodeMod if (!pStmt->stbSyntax) { STableDataCxt* pTableCxt = NULL; int32_t code = parseSchemaClauseBottom(pCxt, pStmt, &pTableCxt); + SRowsDataContext rowsDataCxt; + rowsDataCxt.pTableDataCxt = pTableCxt; if (TSDB_CODE_SUCCESS == code) { - code = parseDataClause(pCxt, pStmt, pTableCxt); + code = parseDataClause(pCxt, pStmt, rowsDataCxt); } return code; } else { @@ -1848,6 +1930,7 @@ static int32_t parseInsertTableClauseBottom(SInsertParseContext* pCxt, SVnodeMod } static void resetEnvPreTable(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) { + //TODO: reset env for stb syntax insDestroyBoundColInfo(&pCxt->tags); taosMemoryFreeClear(pStmt->pTableMeta); nodesDestroyNode(pStmt->pTagCond); @@ -2060,7 +2143,7 @@ static int32_t getTableMetaFromMetaData(const SArray* pTables, STableMeta** pMet return pRes->code; } -static int32_t getTableVgroupFromMetaData(const SArray* pTables, SVnodeModifyOpStmt* pStmt, bool isStb) { +static int32_t addTableVgroupFromMetaData(const SArray* pTables, SVnodeModifyOpStmt* pStmt, bool isStb) { if (1 != taosArrayGetSize(pTables)) { return TSDB_CODE_FAILED; } @@ -2116,10 +2199,10 @@ static int32_t processTableSchemaFromMetaData(SInsertParseContext* pCxt, const S code = buildInvalidOperationMsg(&pCxt->msg, "insert data into super table is not supported"); } if (TSDB_CODE_SUCCESS == code && isStb) { - code = storeTableMeta(pCxt, pStmt); + code = storeChildTableMeta(pCxt, pStmt); } if (TSDB_CODE_SUCCESS == code) { - code = getTableVgroupFromMetaData(pMetaData->pTableHash, pStmt, isStb); + code = addTableVgroupFromMetaData(pMetaData->pTableHash, pStmt, isStb); } if (TSDB_CODE_SUCCESS == code && !isStb && NULL != pStmt->pTagCond) { code = checkSubtablePrivilegeForTable(pMetaData->pTableTag, pStmt); @@ -2158,10 +2241,12 @@ static int32_t setVnodeModifOpStmt(SInsertParseContext* pCxt, SCatalogReq* pCata if (pStmt->pTableMeta->tableType == TSDB_SUPER_TABLE && !pStmt->usingTableProcessing) { pStmt->stbSyntax = true; } - if (pStmt->usingTableProcessing || pStmt->stbSyntax) { - return processTableSchemaFromMetaData(pCxt, pMetaData, pStmt, true); + if (!pStmt->stbSyntax) { + if (pStmt->usingTableProcessing) { + return processTableSchemaFromMetaData(pCxt, pMetaData, pStmt, true); + } + return processTableSchemaFromMetaData(pCxt, pMetaData, pStmt, false); } - return processTableSchemaFromMetaData(pCxt, pMetaData, pStmt, false); } return code; } @@ -2250,10 +2335,18 @@ static int32_t parseInsertSqlFromStart(SInsertParseContext* pCxt, SVnodeModifyOp } static int32_t parseInsertSqlFromCsv(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) { - STableDataCxt* pTableCxt = NULL; - int32_t code = getTableDataCxt(pCxt, pStmt, &pTableCxt); + int32_t code = TSDB_CODE_SUCCESS; + SRowsDataContext rowsDataCxt; + + if (!pStmt->stbSyntax) { + STableDataCxt* pTableCxt = NULL; + code = getTableDataCxt(pCxt, pStmt, &pTableCxt); + rowsDataCxt.pTableDataCxt = pTableCxt; + } else { + rowsDataCxt.pStbRowsCxt = pStmt->pStbRowsCxt; + } if (TSDB_CODE_SUCCESS == code) { - code = parseDataFromFileImpl(pCxt, pStmt, pTableCxt); + code = parseDataFromFileImpl(pCxt, pStmt, rowsDataCxt); } if (TSDB_CODE_SUCCESS == code) { From 8cdcc422839c765c810e0cd2334c2d9fb1171981 Mon Sep 17 00:00:00 2001 From: slzhou Date: Mon, 23 Oct 2023 17:03:18 +0800 Subject: [PATCH 05/17] enhance: add comments to SVnodeModifyOpStmt --- include/libs/nodes/querynodes.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index 1331e865b2..7dd60cfe5b 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -429,11 +429,11 @@ typedef struct SVnodeModifyOpStmt { struct STableMeta* pTableMeta; SNode* pTagCond; SArray* pTableTag; - SHashObj* pVgroupsHashObj; + SHashObj* pVgroupsHashObj; // SHashObj SHashObj* pTableBlockHashObj; // SHashObj - SHashObj* pSubTableHashObj; - SHashObj* pTableNameHashObj; - SHashObj* pDbFNameHashObj; + SHashObj* pSubTableHashObj; // SHashObj + SHashObj* pTableNameHashObj; // set of table names for refreshing meta + SHashObj* pDbFNameHashObj; // set of db names for refreshing meta SArray* pVgDataBlocks; // SArray SVCreateTbReq* pCreateTblReq; TdFilePtr fp; From 65cfc13703dc3896782ef89b1b1f314d0f20ff5d Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Mon, 23 Oct 2023 19:40:27 +0800 Subject: [PATCH 06/17] fix: memory sanitizer error --- source/libs/parser/src/parInsertSql.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index aa8704a62b..46b8993d12 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -1892,6 +1892,7 @@ static int32_t parseInsertStbClauseBottom(SInsertParseContext* pCxt, SVnodeModif SBoundColInfo stbBoundColInfo; insInitBoundColsInfo(tblInfo.numOfColumns + tblInfo.numOfTags + 1, &stbBoundColInfo); if (!pStmt->pBoundCols) { + insDestroyBoundColInfo(&stbBoundColInfo); return buildSyntaxErrMsg(&pCxt->msg, "(...tbname...) bounded cols is expected", pStmt->pSql); } SToken token; @@ -1906,6 +1907,7 @@ static int32_t parseInsertStbClauseBottom(SInsertParseContext* pCxt, SVnodeModif if (TSDB_CODE_SUCCESS == code) { code = parseDataClause(pCxt, pStmt, rowsDataCxt); } + insDestroyBoundColInfo(&stbBoundColInfo); return code; } From ae1bac251a3af627e19c0b0f2894833689df1a50 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Tue, 24 Oct 2023 16:08:08 +0800 Subject: [PATCH 07/17] enhance: finished skeleton --- include/libs/nodes/querynodes.h | 4 +- source/libs/parser/inc/parInsertUtil.h | 1 + source/libs/parser/src/parInsertSql.c | 183 +++++++++++++++++++------ source/libs/parser/src/parInsertUtil.c | 4 + 4 files changed, 150 insertions(+), 42 deletions(-) diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index 7dd60cfe5b..21e50923a2 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -432,8 +432,8 @@ typedef struct SVnodeModifyOpStmt { SHashObj* pVgroupsHashObj; // SHashObj SHashObj* pTableBlockHashObj; // SHashObj SHashObj* pSubTableHashObj; // SHashObj - SHashObj* pTableNameHashObj; // set of table names for refreshing meta - SHashObj* pDbFNameHashObj; // set of db names for refreshing meta + SHashObj* pTableNameHashObj; // set of table names for refreshing meta, sync mode + SHashObj* pDbFNameHashObj; // set of db names for refreshing meta, sync mode SArray* pVgDataBlocks; // SArray SVCreateTbReq* pCreateTblReq; TdFilePtr fp; diff --git a/source/libs/parser/inc/parInsertUtil.h b/source/libs/parser/inc/parInsertUtil.h index 303d349b34..0a153225a3 100644 --- a/source/libs/parser/inc/parInsertUtil.h +++ b/source/libs/parser/inc/parInsertUtil.h @@ -45,6 +45,7 @@ int16_t insFindCol(struct SToken *pColname, int16_t start, int16_t end, SSchema void insBuildCreateTbReq(SVCreateTbReq *pTbReq, const char *tname, STag *pTag, int64_t suid, const char *sname, SArray *tagName, uint8_t tagNum, int32_t ttl); int32_t insInitBoundColsInfo(int32_t numOfBound, SBoundColInfo *pInfo); +void insInitColValues(STableMeta* pTableMeta, SArray* aColValues); void insCheckTableDataOrder(STableDataCxt *pTableCxt, TSKEY tsKey); int32_t insGetTableDataCxt(SHashObj *pHash, void *id, int32_t idLen, STableMeta *pTableMeta, SVCreateTbReq **pCreateTbReq, STableDataCxt **pTableCxt, bool colMode); diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index 46b8993d12..706621d154 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -1553,20 +1553,20 @@ static void clearColValArray(SArray* pCols) { } typedef struct SStbRowsDataContext { - SVnodeModifyOpStmt* pStmt; - STableMeta* pStbMeta; SName stbName; + + STableMeta* pStbMeta; + SNode* pTagCond; SBoundColInfo boundColsInfo; - int32_t numOfBoundCols; - int32_t numOfBoundTags; - - SArray* aChildTableNames; - SArray* aTableDataCxts; - SArray* aCreateTbReqs; + // the following fields are for each stb row SArray* aTagVals; SArray* aColVals; SArray* aTagNames; + SName ctbName; + STag* pTag; + STableMeta* pCtbMeta; + SVCreateTbReq* pCreateCtbReq; } SStbRowsDataContext; typedef union SRowsDataContext{ @@ -1574,23 +1574,25 @@ typedef union SRowsDataContext{ SStbRowsDataContext* pStbRowsCxt; } SRowsDataContext; -static int32_t parseOneStbRow(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, const char** ppSql, SStbRowsDataContext* pStbRowsCxt, bool* pGotRow, - SToken* pToken) { +static int32_t getStbRowValues(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, const char** ppSql, + SStbRowsDataContext* pStbRowsCxt, bool* pGotRow, SToken* pToken) { SBoundColInfo* pCols = &pStbRowsCxt->boundColsInfo; SSchema* pSchemas = getTableColumnSchema(pStbRowsCxt->pStbMeta); bool isJsonTag = false; - SArray* pTagName = taosArrayInit(8, TSDB_COL_NAME_LEN); - SArray* pTagVals = taosArrayInit(pCxt->tags.numOfBound, sizeof(STagVal)); - STag* pTag = NULL; + SArray* pTagName = pStbRowsCxt->aTagNames; + SArray* pTagVals = pStbRowsCxt->aTagVals; + + bool bFoundTbName = false; + const char* pOrigSql = *ppSql; int32_t code = TSDB_CODE_SUCCESS; - for (int i = 0; i < pCols->numOfBound && TSDB_CODE_SUCCESS == code; ++i) { - const char* pOrigSql = *ppSql; + for (int i = 0; i < pCols->numOfBound && code == TSDB_CODE_SUCCESS; ++i) { + const char* pTmpSql = *ppSql; bool ignoreComma = false; NEXT_TOKEN_WITH_PREV_EXT(*ppSql, *pToken, &ignoreComma); if (ignoreComma) { - code = buildSyntaxErrMsg(&pCxt->msg, "invalid data or symbol", pOrigSql); + code = buildSyntaxErrMsg(&pCxt->msg, "invalid data or symbol", pTmpSql); break; } @@ -1607,29 +1609,114 @@ static int32_t parseOneStbRow(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pSt SSchema* pTagSchema = &pSchemas[pCols->pColIndex[i]]; isJsonTag = pTagSchema->type == TSDB_DATA_TYPE_JSON; code = checkAndTrimValue(pToken, pCxt->tmpTokenBuf, &pCxt->msg); - if (TSDB_CODE_SUCCESS == code) { - code = parseTagValue(pCxt, pStmt, ppSql, pTagSchema, pToken, pTagName, pTagVals, &pTag); + if (code == TSDB_CODE_SUCCESS) { + code = parseTagValue(pCxt, pStmt, ppSql, pTagSchema, pToken, pTagName, pTagVals, &pStbRowsCxt->pTag); } } else if (pCols->pColIndex[i] == getTbnameSchemaIndex(pStbRowsCxt->pStbMeta)) { SColVal tbnameVal; code = parseValueToken(pCxt, ppSql, pToken, (SSchema*)tGetTbnameColumnSchema(), getTableInfo(pStbRowsCxt->pStbMeta).precision, &tbnameVal); - if (code == TSDB_CODE_SUCCESS) { - char tbName[TSDB_TABLE_NAME_LEN]; - memcpy(tbName, tbnameVal.value.pData, tbnameVal.value.nData); - tbName[tbnameVal.value.nData] = '\0'; + if (code == TSDB_CODE_SUCCESS && COL_VAL_IS_VALUE(&tbnameVal)) { + tNameAssign(&pStbRowsCxt->ctbName, &pStbRowsCxt->stbName); + tNameAddTbName(&pStbRowsCxt->ctbName, tbnameVal.value.pData, tbnameVal.value.nData); + bFoundTbName = true; } } - if (TSDB_CODE_SUCCESS == code && i < pCols->numOfBound - 1) { + if (code == TSDB_CODE_SUCCESS && i < pCols->numOfBound - 1) { NEXT_VALID_TOKEN(*ppSql, *pToken); if (TK_NK_COMMA != pToken->type) { code = buildSyntaxErrMsg(&pCxt->msg, ", expected", pToken->z); } } } + if (!bFoundTbName) { + code = buildSyntaxErrMsg(&pCxt->msg, "tbname value expected", pOrigSql); + } + if (code == TSDB_CODE_SUCCESS && !isJsonTag) { + code = tTagNew(pTagVals, 1, false, &pStbRowsCxt->pTag); + } + return code; +} +static int32_t parseOneStbRow(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, const char** ppSql, + SStbRowsDataContext* pStbRowsCxt, bool* pGotRow, SToken* pToken) { + int32_t code = getStbRowValues(pCxt, pStmt, ppSql, pStbRowsCxt, pGotRow, pToken); + if (code != TSDB_CODE_SUCCESS || !*pGotRow) { + return code; + } + + if (pStbRowsCxt->pTagCond) { + code = checkSubtablePrivilege(pStbRowsCxt->aTagVals, pStbRowsCxt->aTagNames, &pStbRowsCxt->pTagCond); + } + + if (code == TSDB_CODE_SUCCESS) { + + collectUseTable(&pStbRowsCxt->ctbName, pStmt->pTableNameHashObj); + + char ctbFName[TSDB_TABLE_FNAME_LEN]; + tNameExtractFullName(&pStbRowsCxt->ctbName, ctbFName); + STableMeta** pMeta = taosHashGet(pStmt->pSubTableHashObj, ctbFName, strlen(ctbFName)); + if (NULL != pMeta) { + cloneTableMeta(*pMeta, &pStbRowsCxt->pCtbMeta); + } else { + SVgroupInfo vg; + SRequestConnInfo conn = {.pTrans = pCxt->pComCxt->pTransporter, + .requestId = pCxt->pComCxt->requestId, + .requestObjRefId = pCxt->pComCxt->requestRid, + .mgmtEps = pCxt->pComCxt->mgmtEpSet}; + code = catalogGetTableHashVgroup(pCxt->pComCxt->pCatalog, &conn, &pStmt->targetTableName, &vg); + taosHashPut(pStmt->pVgroupsHashObj, (const char*)(&vg.vgId), sizeof(vg.vgId), &vg, sizeof(vg)); + cloneTableMeta(pStbRowsCxt->pStbMeta, &pStbRowsCxt->pCtbMeta); + pStbRowsCxt->pCtbMeta->suid = pStbRowsCxt->pStbMeta->uid; + pStbRowsCxt->pCtbMeta->uid = taosHashGetSize(pStmt->pSubTableHashObj) + 1; + pStbRowsCxt->pCtbMeta->vgId = vg.vgId; + pStbRowsCxt->pCtbMeta->tableType = TSDB_CHILD_TABLE; + STableMeta* pBackup = NULL; + cloneTableMeta(pStmt->pTableMeta, &pBackup); + taosHashPut(pStmt->pSubTableHashObj, ctbFName, strlen(ctbFName), &pBackup, POINTER_BYTES); + } + } + + pStbRowsCxt->pCreateCtbReq = taosMemoryCalloc(1, sizeof(SVCreateTbReq)); + if (pStbRowsCxt->pCreateCtbReq == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + if (code == TSDB_CODE_SUCCESS) { + insBuildCreateTbReq(pStbRowsCxt->pCreateCtbReq, pStbRowsCxt->ctbName.tname, pStbRowsCxt->pTag, pStbRowsCxt->pStbMeta->uid, + pStbRowsCxt->stbName.tname, pStbRowsCxt->aTagNames, getNumOfTags(pStbRowsCxt->pStbMeta), + TSDB_DEFAULT_TABLE_TTL); + pStbRowsCxt->pTag = NULL; + } + + STableDataCxt* pTableDataCxt = NULL; + code = insGetTableDataCxt(pStmt->pTableBlockHashObj, &pStbRowsCxt->pCtbMeta->uid, sizeof(pStbRowsCxt->pCtbMeta->uid), + pStbRowsCxt->pCtbMeta, &pStbRowsCxt->pCreateCtbReq, &pTableDataCxt, false); + initTableColSubmitData(pTableDataCxt); + if (code == TSDB_CODE_SUCCESS) { + SRow** pRow = taosArrayReserve(pTableDataCxt->pData->aRowP, 1); + code = tRowBuild(pStbRowsCxt->aColVals, pTableDataCxt->pSchema, pRow); + if (TSDB_CODE_SUCCESS == code) { + insCheckTableDataOrder(pTableDataCxt, TD_ROW_KEY(*pRow)); + } + } + if (code == TSDB_CODE_SUCCESS) { + *pGotRow = true; + } + taosArrayClear(pStbRowsCxt->aTagNames); + for (int i = 0; i < taosArrayGetSize(pStbRowsCxt->aTagVals); ++i) { + STagVal* p = (STagVal*)taosArrayGet(pStbRowsCxt->aTagVals, i); + if (IS_VAR_DATA_TYPE(p->type)) { + taosMemoryFreeClear(p->pData); + } + } + taosArrayClear(pStbRowsCxt->aTagVals); + taosArrayClear(pStbRowsCxt->aColVals); + tTagFree(pStbRowsCxt->pTag); + taosMemoryFree(pStbRowsCxt->pCtbMeta); + tdDestroySVCreateTbReq(pStmt->pCreateTblReq); + taosMemoryFreeClear(pStmt->pCreateTblReq); // child meta , vgroupid, check privilege // refer to parInsertSml.c // create or reuse table data context from tbName @@ -1637,11 +1724,7 @@ static int32_t parseOneStbRow(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pSt // build create table request, refer to parseTagsClauseImpl // build row and check table data order, refer to parseOneRow - if (TSDB_CODE_SUCCESS == code) { - *pGotRow = true; - } - - return code; + // clear the values, etc return TSDB_CODE_SUCCESS; } @@ -1886,28 +1969,48 @@ static int32_t parseDataClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pS return buildSyntaxErrMsg(&pCxt->msg, "keyword VALUES or FILE is expected", token.z); } +static void destroyStbRowsDataContext(SStbRowsDataContext* pStbRowsContext) { + taosArrayDestroy(pStbRowsContext->aColVals); + taosArrayDestroy(pStbRowsContext->aTagVals); + taosArrayDestroy(pStbRowsContext->aTagNames); + insDestroyBoundColInfo(&pStbRowsContext->boundColsInfo); +} + static int32_t parseInsertStbClauseBottom(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) { int32_t code = TSDB_CODE_SUCCESS; - STableComInfo tblInfo = getTableInfo(pStmt->pTableMeta); - SBoundColInfo stbBoundColInfo; - insInitBoundColsInfo(tblInfo.numOfColumns + tblInfo.numOfTags + 1, &stbBoundColInfo); if (!pStmt->pBoundCols) { - insDestroyBoundColInfo(&stbBoundColInfo); - return buildSyntaxErrMsg(&pCxt->msg, "(...tbname...) bounded cols is expected", pStmt->pSql); + return buildSyntaxErrMsg(&pCxt->msg, "(...tbname, ts...) bounded cols is expected for supertable insertion", pStmt->pSql); } - SToken token; - int32_t index = 0; - code = parseBoundColumns(pCxt, &pStmt->pBoundCols, BOUND_ALL_AND_TBNAME, pStmt->pTableMeta, &stbBoundColInfo); - pStmt->pStbRowsCxt = taosMemoryCalloc(1, sizeof(SStbRowsDataContext)); + SStbRowsDataContext* pStbRowsCxt; + pStbRowsCxt = taosMemoryCalloc(1, sizeof(SStbRowsDataContext)); + if (!pStbRowsCxt) { + return TSDB_CODE_OUT_OF_MEMORY; + } + tNameAssign(&pStbRowsCxt->stbName, &pStmt->targetTableName); + collectUseTable(&pStbRowsCxt->stbName, pStmt->pTableNameHashObj); + collectUseDatabase(&pStbRowsCxt->stbName, pStmt->pDbFNameHashObj); + pStbRowsCxt->pTagCond = pStmt->pTagCond; + pStbRowsCxt->pStbMeta = pStmt->pTableMeta; + pStbRowsCxt->aTagNames = taosArrayInit(8, TSDB_COL_NAME_LEN); + pStbRowsCxt->aTagVals = taosArrayInit(8, sizeof(STagVal)); - // fill SStbRowsDataCxt; + // col values and bound cols info of STableDataContext is not used TODO: remove the construction when createing table data context + pStbRowsCxt->aColVals = taosArrayInit(getNumOfColumns(pStbRowsCxt->pStbMeta), sizeof(SColVal)); + insInitColValues(pStbRowsCxt->pStbMeta, pStbRowsCxt->aColVals); + + STableComInfo tblInfo = getTableInfo(pStmt->pTableMeta); + insInitBoundColsInfo(tblInfo.numOfColumns + tblInfo.numOfTags + 1, &pStbRowsCxt->boundColsInfo); + + code = parseBoundColumns(pCxt, &pStmt->pBoundCols, BOUND_ALL_AND_TBNAME, pStmt->pTableMeta, &pStbRowsCxt->boundColsInfo); + + pStmt->pStbRowsCxt = pStbRowsCxt; SRowsDataContext rowsDataCxt; - rowsDataCxt.pStbRowsCxt = pStmt->pStbRowsCxt; + rowsDataCxt.pStbRowsCxt = pStbRowsCxt; if (TSDB_CODE_SUCCESS == code) { code = parseDataClause(pCxt, pStmt, rowsDataCxt); } - insDestroyBoundColInfo(&stbBoundColInfo); + // destroyStbRowsDataContext; TODO: move it to resetEnvPreTable return code; } diff --git a/source/libs/parser/src/parInsertUtil.c b/source/libs/parser/src/parInsertUtil.c index 1c0b28f883..9791319503 100644 --- a/source/libs/parser/src/parInsertUtil.c +++ b/source/libs/parser/src/parInsertUtil.c @@ -170,6 +170,10 @@ static void initColValues(STableMeta* pTableMeta, SArray* pValues) { } } +void insInitColValues(STableMeta* pTableMeta, SArray* aColValues) { + initColValues(pTableMeta, aColValues); +} + int32_t insInitBoundColsInfo(int32_t numOfBound, SBoundColInfo* pInfo) { pInfo->numOfCols = numOfBound; pInfo->numOfBound = numOfBound; From 27302f995bc44cc22da3bbdae886d260485310ec Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Tue, 24 Oct 2023 16:23:03 +0800 Subject: [PATCH 08/17] fix: add memory free --- source/libs/parser/src/parInsertSql.c | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index 706621d154..368295638e 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -1715,8 +1715,8 @@ static int32_t parseOneStbRow(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pSt taosArrayClear(pStbRowsCxt->aColVals); tTagFree(pStbRowsCxt->pTag); taosMemoryFree(pStbRowsCxt->pCtbMeta); - tdDestroySVCreateTbReq(pStmt->pCreateTblReq); - taosMemoryFreeClear(pStmt->pCreateTblReq); + tdDestroySVCreateTbReq(pStbRowsCxt->pCreateCtbReq); + taosMemoryFreeClear(pStbRowsCxt->pCreateCtbReq); // child meta , vgroupid, check privilege // refer to parInsertSml.c // create or reuse table data context from tbName @@ -1969,11 +1969,15 @@ static int32_t parseDataClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pS return buildSyntaxErrMsg(&pCxt->msg, "keyword VALUES or FILE is expected", token.z); } -static void destroyStbRowsDataContext(SStbRowsDataContext* pStbRowsContext) { - taosArrayDestroy(pStbRowsContext->aColVals); - taosArrayDestroy(pStbRowsContext->aTagVals); - taosArrayDestroy(pStbRowsContext->aTagNames); - insDestroyBoundColInfo(&pStbRowsContext->boundColsInfo); +static void destroyStbRowsDataContext(SStbRowsDataContext* pStbRowsCxt) { + taosArrayDestroy(pStbRowsCxt->aColVals); + taosArrayDestroy(pStbRowsCxt->aTagVals); + taosArrayDestroy(pStbRowsCxt->aTagNames); + insDestroyBoundColInfo(&pStbRowsCxt->boundColsInfo); + tTagFree(pStbRowsCxt->pTag); + taosMemoryFree(pStbRowsCxt->pCtbMeta); + tdDestroySVCreateTbReq(pStbRowsCxt->pCreateCtbReq); + taosMemoryFreeClear(pStbRowsCxt->pCreateCtbReq); } static int32_t parseInsertStbClauseBottom(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) { @@ -1981,8 +1985,7 @@ static int32_t parseInsertStbClauseBottom(SInsertParseContext* pCxt, SVnodeModif if (!pStmt->pBoundCols) { return buildSyntaxErrMsg(&pCxt->msg, "(...tbname, ts...) bounded cols is expected for supertable insertion", pStmt->pSql); } - SStbRowsDataContext* pStbRowsCxt; - pStbRowsCxt = taosMemoryCalloc(1, sizeof(SStbRowsDataContext)); + SStbRowsDataContext* pStbRowsCxt = taosMemoryCalloc(1, sizeof(SStbRowsDataContext)); if (!pStbRowsCxt) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -2010,7 +2013,8 @@ static int32_t parseInsertStbClauseBottom(SInsertParseContext* pCxt, SVnodeModif if (TSDB_CODE_SUCCESS == code) { code = parseDataClause(pCxt, pStmt, rowsDataCxt); } - // destroyStbRowsDataContext; TODO: move it to resetEnvPreTable + destroyStbRowsDataContext(pStbRowsCxt); + taosMemoryFree(pStbRowsCxt); return code; } From 0c33a3e60ea605e3a2fc55950b47e3a12c5738d5 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Wed, 25 Oct 2023 08:27:02 +0800 Subject: [PATCH 09/17] fix: child table meta optimization --- source/libs/parser/src/parInsertSql.c | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index 368295638e..8b6a92eea9 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -1652,14 +1652,14 @@ static int32_t parseOneStbRow(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pSt } if (code == TSDB_CODE_SUCCESS) { - collectUseTable(&pStbRowsCxt->ctbName, pStmt->pTableNameHashObj); char ctbFName[TSDB_TABLE_FNAME_LEN]; tNameExtractFullName(&pStbRowsCxt->ctbName, ctbFName); - STableMeta** pMeta = taosHashGet(pStmt->pSubTableHashObj, ctbFName, strlen(ctbFName)); - if (NULL != pMeta) { - cloneTableMeta(*pMeta, &pStbRowsCxt->pCtbMeta); + STableMeta** pCtbMeta = taosHashGet(pStmt->pSubTableHashObj, ctbFName, strlen(ctbFName)); + if (NULL != pCtbMeta) { + pStbRowsCxt->pCtbMeta->uid = (*pCtbMeta)->uid; + pStbRowsCxt->pCtbMeta->vgId = (*pCtbMeta)->vgId; } else { SVgroupInfo vg; SRequestConnInfo conn = {.pTrans = pCxt->pComCxt->pTransporter, @@ -1668,11 +1668,8 @@ static int32_t parseOneStbRow(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pSt .mgmtEps = pCxt->pComCxt->mgmtEpSet}; code = catalogGetTableHashVgroup(pCxt->pComCxt->pCatalog, &conn, &pStmt->targetTableName, &vg); taosHashPut(pStmt->pVgroupsHashObj, (const char*)(&vg.vgId), sizeof(vg.vgId), &vg, sizeof(vg)); - cloneTableMeta(pStbRowsCxt->pStbMeta, &pStbRowsCxt->pCtbMeta); - pStbRowsCxt->pCtbMeta->suid = pStbRowsCxt->pStbMeta->uid; pStbRowsCxt->pCtbMeta->uid = taosHashGetSize(pStmt->pSubTableHashObj) + 1; pStbRowsCxt->pCtbMeta->vgId = vg.vgId; - pStbRowsCxt->pCtbMeta->tableType = TSDB_CHILD_TABLE; STableMeta* pBackup = NULL; cloneTableMeta(pStmt->pTableMeta, &pBackup); taosHashPut(pStmt->pSubTableHashObj, ctbFName, strlen(ctbFName), &pBackup, POINTER_BYTES); @@ -1704,6 +1701,7 @@ static int32_t parseOneStbRow(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pSt if (code == TSDB_CODE_SUCCESS) { *pGotRow = true; } + taosArrayClear(pStbRowsCxt->aTagNames); for (int i = 0; i < taosArrayGetSize(pStbRowsCxt->aTagVals); ++i) { STagVal* p = (STagVal*)taosArrayGet(pStbRowsCxt->aTagVals, i); @@ -1714,7 +1712,9 @@ static int32_t parseOneStbRow(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pSt taosArrayClear(pStbRowsCxt->aTagVals); taosArrayClear(pStbRowsCxt->aColVals); tTagFree(pStbRowsCxt->pTag); - taosMemoryFree(pStbRowsCxt->pCtbMeta); + pStbRowsCxt->pTag = NULL; + pStbRowsCxt->pCtbMeta->uid = 0; + pStbRowsCxt->pCtbMeta->vgId = 0; tdDestroySVCreateTbReq(pStbRowsCxt->pCreateCtbReq); taosMemoryFreeClear(pStbRowsCxt->pCreateCtbReq); // child meta , vgroupid, check privilege @@ -1994,6 +1994,9 @@ static int32_t parseInsertStbClauseBottom(SInsertParseContext* pCxt, SVnodeModif collectUseDatabase(&pStbRowsCxt->stbName, pStmt->pDbFNameHashObj); pStbRowsCxt->pTagCond = pStmt->pTagCond; pStbRowsCxt->pStbMeta = pStmt->pTableMeta; + cloneTableMeta(pStbRowsCxt->pStbMeta, &pStbRowsCxt->pCtbMeta); + pStbRowsCxt->pCtbMeta->tableType = TSDB_CHILD_TABLE; + pStbRowsCxt->pCtbMeta->suid = pStbRowsCxt->pStbMeta->uid; pStbRowsCxt->aTagNames = taosArrayInit(8, TSDB_COL_NAME_LEN); pStbRowsCxt->aTagVals = taosArrayInit(8, sizeof(STagVal)); From a28b3a6e3f9c1e674ce1bca2a29b95d803095100 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Wed, 25 Oct 2023 08:50:08 +0800 Subject: [PATCH 10/17] fix: refactor --- source/libs/parser/src/parInsertSql.c | 78 ++++++++++++++------------- 1 file changed, 41 insertions(+), 37 deletions(-) diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index 8b6a92eea9..e980564031 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -1640,17 +1640,23 @@ static int32_t getStbRowValues(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pS return code; } -static int32_t parseOneStbRow(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, const char** ppSql, - SStbRowsDataContext* pStbRowsCxt, bool* pGotRow, SToken* pToken) { - int32_t code = getStbRowValues(pCxt, pStmt, ppSql, pStbRowsCxt, pGotRow, pToken); - if (code != TSDB_CODE_SUCCESS || !*pGotRow) { - return code; - } - +static int32_t processCtbAutoCreationAndCtbMeta(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, SStbRowsDataContext* pStbRowsCxt) { + int32_t code = TSDB_CODE_SUCCESS; if (pStbRowsCxt->pTagCond) { code = checkSubtablePrivilege(pStbRowsCxt->aTagVals, pStbRowsCxt->aTagNames, &pStbRowsCxt->pTagCond); } + pStbRowsCxt->pCreateCtbReq = taosMemoryCalloc(1, sizeof(SVCreateTbReq)); + if (pStbRowsCxt->pCreateCtbReq == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + if (code == TSDB_CODE_SUCCESS) { + insBuildCreateTbReq(pStbRowsCxt->pCreateCtbReq, pStbRowsCxt->ctbName.tname, pStbRowsCxt->pTag, pStbRowsCxt->pStbMeta->uid, + pStbRowsCxt->stbName.tname, pStbRowsCxt->aTagNames, getNumOfTags(pStbRowsCxt->pStbMeta), + TSDB_DEFAULT_TABLE_TTL); + pStbRowsCxt->pTag = NULL; + } + if (code == TSDB_CODE_SUCCESS) { collectUseTable(&pStbRowsCxt->ctbName, pStmt->pTableNameHashObj); @@ -1675,16 +1681,36 @@ static int32_t parseOneStbRow(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pSt taosHashPut(pStmt->pSubTableHashObj, ctbFName, strlen(ctbFName), &pBackup, POINTER_BYTES); } } + return code; +} - pStbRowsCxt->pCreateCtbReq = taosMemoryCalloc(1, sizeof(SVCreateTbReq)); - if (pStbRowsCxt->pCreateCtbReq == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; +static void clearStbRowsDataContext(SStbRowsDataContext* pStbRowsCxt) { + taosArrayClear(pStbRowsCxt->aTagNames); + for (int i = 0; i < taosArrayGetSize(pStbRowsCxt->aTagVals); ++i) { + STagVal* p = (STagVal*)taosArrayGet(pStbRowsCxt->aTagVals, i); + if (IS_VAR_DATA_TYPE(p->type)) { + taosMemoryFreeClear(p->pData); + } } + taosArrayClear(pStbRowsCxt->aTagVals); + taosArrayClear(pStbRowsCxt->aColVals); + tTagFree(pStbRowsCxt->pTag); + pStbRowsCxt->pTag = NULL; + pStbRowsCxt->pCtbMeta->uid = 0; + pStbRowsCxt->pCtbMeta->vgId = 0; + tdDestroySVCreateTbReq(pStbRowsCxt->pCreateCtbReq); + taosMemoryFreeClear(pStbRowsCxt->pCreateCtbReq); +} + +static int32_t parseOneStbRow(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, const char** ppSql, + SStbRowsDataContext* pStbRowsCxt, bool* pGotRow, SToken* pToken) { + int32_t code = getStbRowValues(pCxt, pStmt, ppSql, pStbRowsCxt, pGotRow, pToken); + if (code != TSDB_CODE_SUCCESS || !*pGotRow) { + return code; + } + if (code == TSDB_CODE_SUCCESS) { - insBuildCreateTbReq(pStbRowsCxt->pCreateCtbReq, pStbRowsCxt->ctbName.tname, pStbRowsCxt->pTag, pStbRowsCxt->pStbMeta->uid, - pStbRowsCxt->stbName.tname, pStbRowsCxt->aTagNames, getNumOfTags(pStbRowsCxt->pStbMeta), - TSDB_DEFAULT_TABLE_TTL); - pStbRowsCxt->pTag = NULL; + code = processCtbAutoCreationAndCtbMeta(pCxt, pStmt, pStbRowsCxt); } STableDataCxt* pTableDataCxt = NULL; @@ -1702,29 +1728,7 @@ static int32_t parseOneStbRow(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pSt *pGotRow = true; } - taosArrayClear(pStbRowsCxt->aTagNames); - for (int i = 0; i < taosArrayGetSize(pStbRowsCxt->aTagVals); ++i) { - STagVal* p = (STagVal*)taosArrayGet(pStbRowsCxt->aTagVals, i); - if (IS_VAR_DATA_TYPE(p->type)) { - taosMemoryFreeClear(p->pData); - } - } - taosArrayClear(pStbRowsCxt->aTagVals); - taosArrayClear(pStbRowsCxt->aColVals); - tTagFree(pStbRowsCxt->pTag); - pStbRowsCxt->pTag = NULL; - pStbRowsCxt->pCtbMeta->uid = 0; - pStbRowsCxt->pCtbMeta->vgId = 0; - tdDestroySVCreateTbReq(pStbRowsCxt->pCreateCtbReq); - taosMemoryFreeClear(pStbRowsCxt->pCreateCtbReq); - // child meta , vgroupid, check privilege - // refer to parInsertSml.c - // create or reuse table data context from tbName - // init submit data - // build create table request, refer to parseTagsClauseImpl - // build row and check table data order, refer to parseOneRow - - // clear the values, etc + clearStbRowsDataContext(pStbRowsCxt); return TSDB_CODE_SUCCESS; } From c9d1553633b7111e12ab10a39b8e6cc6a8a4a202 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Wed, 25 Oct 2023 09:06:45 +0800 Subject: [PATCH 11/17] enhance: refactor stb rows data context --- source/libs/parser/src/parInsertSql.c | 44 ++++++++++++++++++++------- 1 file changed, 33 insertions(+), 11 deletions(-) diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index e980564031..05908d2ece 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -1685,6 +1685,8 @@ static int32_t processCtbAutoCreationAndCtbMeta(SInsertParseContext* pCxt, SVnod } static void clearStbRowsDataContext(SStbRowsDataContext* pStbRowsCxt) { + if (pStbRowsCxt == NULL) return; + taosArrayClear(pStbRowsCxt->aTagNames); for (int i = 0; i < taosArrayGetSize(pStbRowsCxt->aTagVals); ++i) { STagVal* p = (STagVal*)taosArrayGet(pStbRowsCxt->aTagVals, i); @@ -1693,7 +1695,10 @@ static void clearStbRowsDataContext(SStbRowsDataContext* pStbRowsCxt) { } } taosArrayClear(pStbRowsCxt->aTagVals); + + clearColValArray(pStbRowsCxt->aColVals); taosArrayClear(pStbRowsCxt->aColVals); + tTagFree(pStbRowsCxt->pTag); pStbRowsCxt->pTag = NULL; pStbRowsCxt->pCtbMeta->uid = 0; @@ -1974,21 +1979,23 @@ static int32_t parseDataClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pS } static void destroyStbRowsDataContext(SStbRowsDataContext* pStbRowsCxt) { + if (pStbRowsCxt == NULL) return; + clearStbRowsDataContext(pStbRowsCxt); taosArrayDestroy(pStbRowsCxt->aColVals); + pStbRowsCxt->aColVals = NULL; taosArrayDestroy(pStbRowsCxt->aTagVals); + pStbRowsCxt->aTagVals = NULL; taosArrayDestroy(pStbRowsCxt->aTagNames); + pStbRowsCxt->aTagNames = NULL; insDestroyBoundColInfo(&pStbRowsCxt->boundColsInfo); tTagFree(pStbRowsCxt->pTag); - taosMemoryFree(pStbRowsCxt->pCtbMeta); + pStbRowsCxt->pTag = NULL; + taosMemoryFreeClear(pStbRowsCxt->pCtbMeta); tdDestroySVCreateTbReq(pStbRowsCxt->pCreateCtbReq); taosMemoryFreeClear(pStbRowsCxt->pCreateCtbReq); } -static int32_t parseInsertStbClauseBottom(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) { - int32_t code = TSDB_CODE_SUCCESS; - if (!pStmt->pBoundCols) { - return buildSyntaxErrMsg(&pCxt->msg, "(...tbname, ts...) bounded cols is expected for supertable insertion", pStmt->pSql); - } +static int32_t constructStbRowsDataContext(SVnodeModifyOpStmt* pStmt, SStbRowsDataContext** ppStbRowsCxt) { SStbRowsDataContext* pStbRowsCxt = taosMemoryCalloc(1, sizeof(SStbRowsDataContext)); if (!pStbRowsCxt) { return TSDB_CODE_OUT_OF_MEMORY; @@ -2004,20 +2011,35 @@ static int32_t parseInsertStbClauseBottom(SInsertParseContext* pCxt, SVnodeModif pStbRowsCxt->aTagNames = taosArrayInit(8, TSDB_COL_NAME_LEN); pStbRowsCxt->aTagVals = taosArrayInit(8, sizeof(STagVal)); - // col values and bound cols info of STableDataContext is not used TODO: remove the construction when createing table data context + // col values and bound cols info of STableDataContext is not used pStbRowsCxt->aColVals = taosArrayInit(getNumOfColumns(pStbRowsCxt->pStbMeta), sizeof(SColVal)); insInitColValues(pStbRowsCxt->pStbMeta, pStbRowsCxt->aColVals); STableComInfo tblInfo = getTableInfo(pStmt->pTableMeta); insInitBoundColsInfo(tblInfo.numOfColumns + tblInfo.numOfTags + 1, &pStbRowsCxt->boundColsInfo); - code = parseBoundColumns(pCxt, &pStmt->pBoundCols, BOUND_ALL_AND_TBNAME, pStmt->pTableMeta, &pStbRowsCxt->boundColsInfo); + *ppStbRowsCxt = pStbRowsCxt; + return TSDB_CODE_SUCCESS; +} - pStmt->pStbRowsCxt = pStbRowsCxt; +static int32_t parseInsertStbClauseBottom(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) { + int32_t code = TSDB_CODE_SUCCESS; + if (!pStmt->pBoundCols) { + return buildSyntaxErrMsg(&pCxt->msg, "(...tbname, ts...) bounded cols is expected for supertable insertion", pStmt->pSql); + } + + SStbRowsDataContext* pStbRowsCxt = NULL; + code = constructStbRowsDataContext(pStmt, &pStbRowsCxt); + + if (code == TSDB_CODE_SUCCESS) { + code = parseBoundColumns(pCxt, &pStmt->pBoundCols, BOUND_ALL_AND_TBNAME, pStmt->pTableMeta, + &pStbRowsCxt->boundColsInfo); + pStmt->pStbRowsCxt = pStbRowsCxt; + } - SRowsDataContext rowsDataCxt; - rowsDataCxt.pStbRowsCxt = pStbRowsCxt; if (TSDB_CODE_SUCCESS == code) { + SRowsDataContext rowsDataCxt; + rowsDataCxt.pStbRowsCxt = pStbRowsCxt; code = parseDataClause(pCxt, pStmt, rowsDataCxt); } destroyStbRowsDataContext(pStbRowsCxt); From 3de5b841a5536b10247c68cc106165d98849c4e3 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Wed, 25 Oct 2023 09:35:37 +0800 Subject: [PATCH 12/17] fix: large file by batches --- source/libs/parser/src/parInsertSql.c | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index 05908d2ece..ec5aeb950d 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -2037,13 +2037,16 @@ static int32_t parseInsertStbClauseBottom(SInsertParseContext* pCxt, SVnodeModif pStmt->pStbRowsCxt = pStbRowsCxt; } - if (TSDB_CODE_SUCCESS == code) { + if (code == TSDB_CODE_SUCCESS) { SRowsDataContext rowsDataCxt; rowsDataCxt.pStbRowsCxt = pStbRowsCxt; code = parseDataClause(pCxt, pStmt, rowsDataCxt); } - destroyStbRowsDataContext(pStbRowsCxt); - taosMemoryFree(pStbRowsCxt); + + if (code != TSDB_CODE_SUCCESS) { + destroyStbRowsDataContext(pStbRowsCxt); + taosMemoryFreeClear(pStbRowsCxt); + } return code; } @@ -2068,7 +2071,6 @@ static int32_t parseInsertTableClauseBottom(SInsertParseContext* pCxt, SVnodeMod } static void resetEnvPreTable(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) { - //TODO: reset env for stb syntax insDestroyBoundColInfo(&pCxt->tags); taosMemoryFreeClear(pStmt->pTableMeta); nodesDestroyNode(pStmt->pTagCond); @@ -2081,6 +2083,9 @@ static void resetEnvPreTable(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStm pStmt->usingTableProcessing = false; pStmt->fileProcessing = false; pStmt->usingTableName.type = 0; + + destroyStbRowsDataContext(pStmt->pStbRowsCxt); + taosMemoryFreeClear(pStmt->pStbRowsCxt); pStmt->stbSyntax = false; } From f0a44b31fc063adc39c4907130d1c95fae624a89 Mon Sep 17 00:00:00 2001 From: slzhou Date: Wed, 25 Oct 2023 09:56:21 +0800 Subject: [PATCH 13/17] fix: insert into super table syntax is not supported for stmt --- source/libs/parser/src/parInsertSql.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index ec5aeb950d..65ff043e23 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -2055,6 +2055,9 @@ static int32_t parseInsertStbClauseBottom(SInsertParseContext* pCxt, SVnodeModif // 1. [(tag1_name, ...)] ... // 2. VALUES ... | FILE ... static int32_t parseInsertTableClauseBottom(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) { + if (pStmt->stbSyntax && (pStmt->insertType, TSDB_QUERY_TYPE_STMT_INSERT)) { + return buildSyntaxErrMsg(&pCxt->msg, "insert into super table syntax is not supported for stmt", NULL); + } if (!pStmt->stbSyntax) { STableDataCxt* pTableCxt = NULL; int32_t code = parseSchemaClauseBottom(pCxt, pStmt, &pTableCxt); From c17300557d3332574708b1e4281fc9bc49027851 Mon Sep 17 00:00:00 2001 From: slzhou Date: Wed, 25 Oct 2023 10:56:28 +0800 Subject: [PATCH 14/17] fix: memory sanitizer error of simple insert stb --- include/libs/nodes/querynodes.h | 3 ++- source/libs/nodes/src/nodesUtilFuncs.c | 4 ++++ source/libs/parser/src/parInsertSql.c | 19 ++++++++++++++----- 3 files changed, 20 insertions(+), 6 deletions(-) diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index 21e50923a2..379f599786 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -413,8 +413,8 @@ typedef struct SVgDataBlocks { typedef void (*FFreeTableBlockHash)(SHashObj*); typedef void (*FFreeVgourpBlockArray)(SArray*); - struct SStbRowsDataContext; +typedef void (*FFreeStbRowsDataContext)(struct SStbRowsDataContext*); typedef struct SVnodeModifyOpStmt { ENodeType nodeType; ENodeType sqlNodeType; @@ -444,6 +444,7 @@ typedef struct SVnodeModifyOpStmt { bool stbSyntax; struct SStbRowsDataContext* pStbRowsCxt; + FFreeStbRowsDataContext freeStbRowsCxtFunc; } SVnodeModifyOpStmt; typedef struct SExplainOptions { diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index c5a1bfa599..926c907ae4 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -879,6 +879,10 @@ void nodesDestroyNode(SNode* pNode) { } tdDestroySVCreateTbReq(pStmt->pCreateTblReq); taosMemoryFreeClear(pStmt->pCreateTblReq); + if (pStmt->freeStbRowsCxtFunc) { + pStmt->freeStbRowsCxtFunc(pStmt->pStbRowsCxt); + } + taosMemoryFreeClear(pStmt->pStbRowsCxt); taosCloseFile(&pStmt->fp); break; } diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index 65ff043e23..f53192ad5b 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -1614,13 +1614,17 @@ static int32_t getStbRowValues(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pS } } else if (pCols->pColIndex[i] == getTbnameSchemaIndex(pStbRowsCxt->pStbMeta)) { - SColVal tbnameVal; + SColVal tbnameVal = COL_VAL_NONE(-1, TSDB_DATA_TYPE_BINARY); code = parseValueToken(pCxt, ppSql, pToken, (SSchema*)tGetTbnameColumnSchema(), getTableInfo(pStbRowsCxt->pStbMeta).precision, &tbnameVal); if (code == TSDB_CODE_SUCCESS && COL_VAL_IS_VALUE(&tbnameVal)) { - tNameAssign(&pStbRowsCxt->ctbName, &pStbRowsCxt->stbName); - tNameAddTbName(&pStbRowsCxt->ctbName, tbnameVal.value.pData, tbnameVal.value.nData); + tNameSetDbName(&pStbRowsCxt->ctbName, pStbRowsCxt->stbName.acctId, pStbRowsCxt->stbName.dbname, strlen(pStbRowsCxt->stbName.dbname)); + char ctbName[TSDB_TABLE_NAME_LEN]; + memcpy(ctbName, tbnameVal.value.pData, tbnameVal.value.nData); + ctbName[tbnameVal.value.nData] = '\0'; + tNameAddTbName(&pStbRowsCxt->ctbName, ctbName, tbnameVal.value.nData); bFoundTbName = true; + taosMemoryFreeClear(tbnameVal.value.pData); } } @@ -1637,6 +1641,9 @@ static int32_t getStbRowValues(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pS if (code == TSDB_CODE_SUCCESS && !isJsonTag) { code = tTagNew(pTagVals, 1, false, &pStbRowsCxt->pTag); } + if (code == TSDB_CODE_SUCCESS) { + *pGotRow = true; + } return code; } @@ -2055,7 +2062,7 @@ static int32_t parseInsertStbClauseBottom(SInsertParseContext* pCxt, SVnodeModif // 1. [(tag1_name, ...)] ... // 2. VALUES ... | FILE ... static int32_t parseInsertTableClauseBottom(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) { - if (pStmt->stbSyntax && (pStmt->insertType, TSDB_QUERY_TYPE_STMT_INSERT)) { + if (pStmt->stbSyntax && TSDB_QUERY_HAS_TYPE(pStmt->insertType, TSDB_QUERY_TYPE_STMT_INSERT)) { return buildSyntaxErrMsg(&pCxt->msg, "insert into super table syntax is not supported for stmt", NULL); } if (!pStmt->stbSyntax) { @@ -2215,9 +2222,11 @@ static int32_t createVnodeModifOpStmt(SInsertParseContext* pCxt, bool reentry, S TSDB_QUERY_SET_TYPE(pStmt->insertType, TSDB_QUERY_TYPE_STMT_INSERT); } pStmt->pSql = pCxt->pComCxt->pSql; + pStmt->freeHashFunc = insDestroyTableDataCxtHashMap; pStmt->freeArrayFunc = insDestroyVgroupDataCxtList; - + pStmt->freeStbRowsCxtFunc = destroyStbRowsDataContext; + if (!reentry) { pStmt->pVgroupsHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); pStmt->pTableBlockHashObj = From 29db21a3ad88e47c08289663c20647110dbc1318 Mon Sep 17 00:00:00 2001 From: slzhou Date: Wed, 25 Oct 2023 11:22:15 +0800 Subject: [PATCH 15/17] fix: reinit ctb meta and col vals before each stb row --- source/libs/parser/src/parInsertSql.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index f53192ad5b..6e1c599df4 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -1691,6 +1691,13 @@ static int32_t processCtbAutoCreationAndCtbMeta(SInsertParseContext* pCxt, SVnod return code; } +static void resetStbRowsDataContextPreStbRow(SStbRowsDataContext* pStbRowsCxt) { + pStbRowsCxt->pCtbMeta->tableType = TSDB_CHILD_TABLE; + pStbRowsCxt->pCtbMeta->suid = pStbRowsCxt->pStbMeta->uid; + + insInitColValues(pStbRowsCxt->pStbMeta, pStbRowsCxt->aColVals); +} + static void clearStbRowsDataContext(SStbRowsDataContext* pStbRowsCxt) { if (pStbRowsCxt == NULL) return; @@ -1716,6 +1723,7 @@ static void clearStbRowsDataContext(SStbRowsDataContext* pStbRowsCxt) { static int32_t parseOneStbRow(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, const char** ppSql, SStbRowsDataContext* pStbRowsCxt, bool* pGotRow, SToken* pToken) { + resetStbRowsDataContextPreStbRow(pStbRowsCxt); int32_t code = getStbRowValues(pCxt, pStmt, ppSql, pStbRowsCxt, pGotRow, pToken); if (code != TSDB_CODE_SUCCESS || !*pGotRow) { return code; @@ -2010,11 +2018,14 @@ static int32_t constructStbRowsDataContext(SVnodeModifyOpStmt* pStmt, SStbRowsDa tNameAssign(&pStbRowsCxt->stbName, &pStmt->targetTableName); collectUseTable(&pStbRowsCxt->stbName, pStmt->pTableNameHashObj); collectUseDatabase(&pStbRowsCxt->stbName, pStmt->pDbFNameHashObj); + pStbRowsCxt->pTagCond = pStmt->pTagCond; pStbRowsCxt->pStbMeta = pStmt->pTableMeta; + cloneTableMeta(pStbRowsCxt->pStbMeta, &pStbRowsCxt->pCtbMeta); pStbRowsCxt->pCtbMeta->tableType = TSDB_CHILD_TABLE; pStbRowsCxt->pCtbMeta->suid = pStbRowsCxt->pStbMeta->uid; + pStbRowsCxt->aTagNames = taosArrayInit(8, TSDB_COL_NAME_LEN); pStbRowsCxt->aTagVals = taosArrayInit(8, sizeof(STagVal)); From 1135d31caff20a9e08d08a3e16f48b3167693676 Mon Sep 17 00:00:00 2001 From: slzhou Date: Wed, 25 Oct 2023 11:39:59 +0800 Subject: [PATCH 16/17] enhance: add test case --- tests/script/tsim/insert/insert_stb.sim | 36 +++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 tests/script/tsim/insert/insert_stb.sim diff --git a/tests/script/tsim/insert/insert_stb.sim b/tests/script/tsim/insert/insert_stb.sim new file mode 100644 index 0000000000..4defbcdc26 --- /dev/null +++ b/tests/script/tsim/insert/insert_stb.sim @@ -0,0 +1,36 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sql connect + +sql create database d1 +sql create database d2 + +sql use d1; +sql create table st(ts timestamp, f int) tags(t int); +sql insert into ct1 using st tags(1) values('2021-04-19 00:00:00', 1); +sql insert into ct2 using st tags(2) values('2021-04-19 00:00:01', 2); +sql insert into ct1 values('2021-04-19 00:00:02', 2); + +sql use d2; +sql create table st(ts timestamp, f int) tags(t int); +sql insert into ct1 using st tags(1) values('2021-04-19 00:00:00', 1); +sql insert into ct2 using st tags(2) values('2021-04-19 00:00:01', 2); + +sql create database db1 vgroups 1; +sql create table db1.stb (ts timestamp, c1 int, c2 int) tags(t1 int, t2 int); + +sql use d1; +sql insert into st (tbname, ts, f, t) values('ct3', '2021-04-19 08:00:00', 3, 3); +sql insert into d1.st (tbname, ts, f) values('ct6', '2021-04-19 08:00:00', 6); +sql insert into d1.st (tbname, ts, f) values('ct6', '2021-04-19 08:00:00', 7)('ct8', '2021-04-19 08:00:00', 8); +sql insert into d1.st (tbname, ts, f, t) values('ct6', '2021-04-19 08:00:00', 9, 9)('ct8', '2021-04-19 08:00:00', 10, 10); +sql insert into d1.st (tbname, ts, f, t) values('ct6', '2021-04-19 08:00:00', 9, 9)('ct8', '2021-04-19 08:00:00', 10, 10) d2.st (tbname, ts, f, t) values('ct6', '2021-04-19 08:00:00', 9, 9)('ct8', '2021-04-19 08:00:00', 10, 10); + +sql select * from d1.st +print $rows + +sql select * from d2.st; +print $rows + +system sh/exec.sh -n dnode1 -s stop -x SIGINT From 6134346d4e3405d9ea932fda1dadd5044e00bd3b Mon Sep 17 00:00:00 2001 From: slzhou Date: Wed, 25 Oct 2023 13:20:37 +0800 Subject: [PATCH 17/17] enhance: test case insert_stb.sim --- tests/parallel_test/cases.task | 1 + tests/script/tsim/insert/insert_stb.sim | 18 +++++++++++------- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index fcd39092bd..d5ebdaf9d3 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -930,6 +930,7 @@ ,,y,script,./test.sh -f tsim/insert/delete0.sim ,,y,script,./test.sh -f tsim/insert/update1_sort_merge.sim ,,y,script,./test.sh -f tsim/insert/update2.sim +,,y,script,./test.sh -f tsim/insert/insert_stb.sim ,,y,script,./test.sh -f tsim/parser/alter__for_community_version.sim ,,y,script,./test.sh -f tsim/parser/alter_column.sim ,,y,script,./test.sh -f tsim/parser/alter_stable.sim diff --git a/tests/script/tsim/insert/insert_stb.sim b/tests/script/tsim/insert/insert_stb.sim index 4defbcdc26..cee209f4e1 100644 --- a/tests/script/tsim/insert/insert_stb.sim +++ b/tests/script/tsim/insert/insert_stb.sim @@ -21,16 +21,20 @@ sql create database db1 vgroups 1; sql create table db1.stb (ts timestamp, c1 int, c2 int) tags(t1 int, t2 int); sql use d1; -sql insert into st (tbname, ts, f, t) values('ct3', '2021-04-19 08:00:00', 3, 3); -sql insert into d1.st (tbname, ts, f) values('ct6', '2021-04-19 08:00:00', 6); -sql insert into d1.st (tbname, ts, f) values('ct6', '2021-04-19 08:00:00', 7)('ct8', '2021-04-19 08:00:00', 8); -sql insert into d1.st (tbname, ts, f, t) values('ct6', '2021-04-19 08:00:00', 9, 9)('ct8', '2021-04-19 08:00:00', 10, 10); -sql insert into d1.st (tbname, ts, f, t) values('ct6', '2021-04-19 08:00:00', 9, 9)('ct8', '2021-04-19 08:00:00', 10, 10) d2.st (tbname, ts, f, t) values('ct6', '2021-04-19 08:00:00', 9, 9)('ct8', '2021-04-19 08:00:00', 10, 10); +sql insert into st (tbname, ts, f, t) values('ct3', '2021-04-19 08:00:03', 3, 3); +sql insert into d1.st (tbname, ts, f) values('ct6', '2021-04-19 08:00:04', 6); +sql insert into d1.st (tbname, ts, f) values('ct6', '2021-04-19 08:00:05', 7)('ct8', '2021-04-19 08:00:06', 8); +sql insert into d1.st (tbname, ts, f, t) values('ct6', '2021-04-19 08:00:07', 9, 9)('ct8', '2021-04-19 08:00:08', 10, 10); +sql insert into d1.st (tbname, ts, f, t) values('ct6', '2021-04-19 08:00:09', 9, 9)('ct8', '2021-04-19 08:00:10', 10, 10) d2.st (tbname, ts, f, t) values('ct6', '2021-04-19 08:00:11', 9, 9)('ct8', '2021-04-19 08:00:12', 10, 10); sql select * from d1.st print $rows - +if $rows != 11 then + return -1 +endi sql select * from d2.st; print $rows - +if $rows != 4 then + return -1 +endi system sh/exec.sh -n dnode1 -s stop -x SIGINT