From 8513dc10672a245578742680ceb1dd596da3fae8 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Fri, 4 Nov 2022 15:21:38 +0800 Subject: [PATCH 1/9] enh: insert optimize --- include/libs/catalog/catalog.h | 7 +- include/libs/nodes/querynodes.h | 73 +- include/libs/parser/parser.h | 4 +- source/client/inc/clientInt.h | 3 +- source/client/src/clientImpl.c | 16 +- source/client/src/clientMain.c | 111 +- source/libs/nodes/src/nodesUtilFuncs.c | 19 +- source/libs/parser/inc/parInsertUtil.h | 27 +- source/libs/parser/inc/parInt.h | 3 +- source/libs/parser/inc/parUtil.h | 38 +- source/libs/parser/src/parAstParser.c | 1 + source/libs/parser/src/parInsertSml.c | 6 +- source/libs/parser/src/parInsertSql.c | 2467 +++++++++-------- source/libs/parser/src/parInsertStmt.c | 28 +- source/libs/parser/src/parInsertUtil.c | 86 +- source/libs/parser/src/parUtil.c | 177 +- source/libs/parser/src/parser.c | 36 +- source/libs/parser/test/mockCatalog.cpp | 15 + .../libs/parser/test/mockCatalogService.cpp | 26 +- source/libs/parser/test/mockCatalogService.h | 4 +- source/libs/parser/test/parTestUtil.cpp | 199 +- source/libs/scheduler/src/schRemote.c | 2 +- 22 files changed, 1708 insertions(+), 1640 deletions(-) diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index 9f1513d100..7ce5a58a00 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -186,6 +186,8 @@ int32_t catalogRemoveStbMeta(SCatalog* pCtg, const char* dbFName, uint64_t dbId, */ int32_t catalogGetTableMeta(SCatalog* pCatalog, SRequestConnInfo* pConn, const SName* pTableName, STableMeta** pTableMeta); +int32_t catalogGetCachedTableMeta(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, + STableMeta** pTableMeta); /** * Get a super table's meta data. @@ -198,6 +200,8 @@ int32_t catalogGetTableMeta(SCatalog* pCatalog, SRequestConnInfo* pConn, const S */ int32_t catalogGetSTableMeta(SCatalog* pCatalog, SRequestConnInfo* pConn, const SName* pTableName, STableMeta** pTableMeta); +int32_t catalogGetCachedSTableMeta(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, + STableMeta** pTableMeta); int32_t catalogUpdateTableMeta(SCatalog* pCatalog, STableMetaRsp* rspMsg); @@ -261,7 +265,8 @@ int32_t catalogGetTableDistVgInfo(SCatalog* pCatalog, SRequestConnInfo* pConn, c * @return error code */ int32_t catalogGetTableHashVgroup(SCatalog* pCatalog, SRequestConnInfo* pConn, const SName* pName, SVgroupInfo* vgInfo); - +int32_t catalogGetCachedTableHashVgroup(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, + SVgroupInfo* pVgroup, bool* exists); /** * Get all meta data required in pReq. * @param pCatalog (input, got with catalogGetHandle) diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index 837834f795..a85f8bf63d 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -354,12 +354,33 @@ typedef struct SVgDataBlocks { void* pData; // SMsgDesc + SSubmitReq + SSubmitBlk + ... } SVgDataBlocks; +typedef void (*FFreeDataBlockHash)(SHashObj*); +typedef void (*FFreeDataBlockArray)(SArray*); + typedef struct SVnodeModifOpStmt { - ENodeType nodeType; - ENodeType sqlNodeType; - SArray* pDataBlocks; // data block for each vgroup, SArray. - uint32_t insertType; // insert data from [file|sql statement| bound statement] - const char* sql; // current sql statement position + ENodeType nodeType; + ENodeType sqlNodeType; + SArray* pDataBlocks; // data block for each vgroup, SArray. + uint32_t insertType; // insert data from [file|sql statement| bound statement] + const char* pSql; // current sql statement position + int32_t totalRowsNum; + int32_t totalTbNum; + SName targetTableName; + SName usingTableName; + const char* pBoundCols; + struct STableMeta* pTableMeta; + SHashObj* pVgroupsHashObj; + SHashObj* pTableBlockHashObj; + SHashObj* pSubTableHashObj; + SHashObj* pTableNameHashObj; + SHashObj* pDbFNameHashObj; + SArray* pVgDataBlocks; + SVCreateTbReq createTblReq; + TdFilePtr fp; + FFreeDataBlockHash freeHashFunc; + FFreeDataBlockArray freeArrayFunc; + bool usingTableProcessing; + bool fileProcessing; } SVnodeModifOpStmt; typedef struct SExplainOptions { @@ -389,24 +410,32 @@ typedef enum EQueryExecMode { QUERY_EXEC_MODE_EMPTY_RESULT } EQueryExecMode; +typedef enum EQueryExecStage { + QUERY_EXEC_STAGE_PARSE = 1, + QUERY_EXEC_STAGE_ANALYSE, + QUERY_EXEC_STAGE_SCHEDULE, + QUERY_EXEC_STAGE_END +} EQueryExecStage; + typedef struct SQuery { - ENodeType type; - EQueryExecMode execMode; - bool haveResultSet; - SNode* pRoot; - int32_t numOfResCols; - SSchema* pResSchema; - int8_t precision; - SCmdMsgInfo* pCmdMsg; - int32_t msgType; - SArray* pTargetTableList; - SArray* pTableList; - SArray* pDbList; - bool showRewrite; - int32_t placeholderNum; - SArray* pPlaceholderValues; - SNode* pPrepareRoot; - bool stableQuery; + ENodeType type; + EQueryExecStage execStage; + EQueryExecMode execMode; + bool haveResultSet; + SNode* pRoot; + int32_t numOfResCols; + SSchema* pResSchema; + int8_t precision; + SCmdMsgInfo* pCmdMsg; + int32_t msgType; + SArray* pTargetTableList; + SArray* pTableList; + SArray* pDbList; + bool showRewrite; + int32_t placeholderNum; + SArray* pPlaceholderValues; + SNode* pPrepareRoot; + bool stableQuery; } SQuery; void nodesWalkSelectStmt(SSelectStmt* pSelect, ESqlClause clause, FNodeWalker walker, void* pContext); diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index bcd2316baf..1a7e6dc748 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -64,8 +64,6 @@ typedef struct SParseContext { SArray* pTableMetaPos; // sql table pos => catalog data pos SArray* pTableVgroupPos; // sql table pos => catalog data pos int64_t allocatorId; - bool needMultiParse; - SParseCsvCxt csvCxt; } SParseContext; int32_t qParseSql(SParseContext* pCxt, SQuery** pQuery); @@ -75,6 +73,8 @@ bool qIsInsertValuesSql(const char* pStr, size_t length); int32_t qParseSqlSyntax(SParseContext* pCxt, SQuery** pQuery, struct SCatalogReq* pCatalogReq); int32_t qAnalyseSqlSemantic(SParseContext* pCxt, const struct SCatalogReq* pCatalogReq, const struct SMetaData* pMetaData, SQuery* pQuery); +int32_t qContinueParseSql(SParseContext* pCxt, struct SCatalogReq* pCatalogReq, const struct SMetaData* pMetaData, + SQuery* pQuery); void qDestroyParseContext(SParseContext* pCxt); diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index cf8b0babdd..3d99fc435a 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -379,7 +379,6 @@ void hbMgrInitMqHbRspHandle(); typedef struct SSqlCallbackWrapper { SParseContext* pParseCtx; SCatalogReq* pCatalogReq; - SMetaData* pResultMeta; SRequestObj* pRequest; } SSqlCallbackWrapper; @@ -393,7 +392,7 @@ int32_t removeMeta(STscObj* pTscObj, SArray* tbList); int32_t handleAlterTbExecRes(void* res, struct SCatalog* pCatalog); int32_t handleCreateTbExecRes(void* res, SCatalog* pCatalog); bool qnodeRequired(SRequestObj* pRequest); -int32_t continueInsertFromCsv(SSqlCallbackWrapper* pWrapper, SRequestObj* pRequest); +void continueInsertFromCsv(SSqlCallbackWrapper* pWrapper, SRequestObj* pRequest); void destorySqlCallbackWrapper(SSqlCallbackWrapper* pWrapper); #ifdef __cplusplus diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 0c12d78c18..703ca7d951 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -867,6 +867,10 @@ int32_t handleQueryExecRsp(SRequestObj* pRequest) { return code; } +static bool incompletaFileParsing(SNode* pStmt) { + return QUERY_NODE_VNODE_MODIF_STMT != nodeType(pStmt) ? false : ((SVnodeModifOpStmt*)pStmt)->fileProcessing; +} + // todo refacto the error code mgmt void schedulerExecCb(SExecResult* pResult, void* param, int32_t code) { SSqlCallbackWrapper* pWrapper = param; @@ -921,11 +925,9 @@ void schedulerExecCb(SExecResult* pResult, void* param, int32_t code) { pRequest->code = code1; } - if (pRequest->code == TSDB_CODE_SUCCESS && NULL != pWrapper->pParseCtx && pWrapper->pParseCtx->needMultiParse) { - code = continueInsertFromCsv(pWrapper, pRequest); - if (TSDB_CODE_SUCCESS == code) { - return; - } + if (pRequest->code == TSDB_CODE_SUCCESS && incompletaFileParsing(pRequest->pQuery->pRoot)) { + continueInsertFromCsv(pWrapper, pRequest); + return; } destorySqlCallbackWrapper(pWrapper); @@ -1049,7 +1051,9 @@ static int32_t asyncExecSchQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaDat } if (TSDB_CODE_SUCCESS == code && !pRequest->validateOnly) { SArray* pNodeList = NULL; - buildAsyncExecNodeList(pRequest, &pNodeList, pMnodeList, pResultMeta); + if (QUERY_NODE_VNODE_MODIF_STMT != nodeType(pRequest->pQuery->pRoot)) { + buildAsyncExecNodeList(pRequest, &pNodeList, pMnodeList, pResultMeta); + } SRequestConnInfo conn = {.pTrans = getAppInfo(pRequest)->pTransporter, .requestId = pRequest->requestId, diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index b03576ff01..2b2ee03638 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -682,11 +682,10 @@ void destorySqlCallbackWrapper(SSqlCallbackWrapper *pWrapper) { } destoryCatalogReq(pWrapper->pCatalogReq); qDestroyParseContext(pWrapper->pParseCtx); - catalogFreeMetaData(pWrapper->pResultMeta); taosMemoryFree(pWrapper); } -void retrieveMetaCallback(SMetaData *pResultMeta, void *param, int32_t code) { +static void doAsyncQueryFromAnalyse(SMetaData *pResultMeta, void *param, int32_t code) { SSqlCallbackWrapper *pWrapper = (SSqlCallbackWrapper *)param; SRequestObj *pRequest = pWrapper->pRequest; SQuery *pQuery = pRequest->pQuery; @@ -704,13 +703,6 @@ void retrieveMetaCallback(SMetaData *pResultMeta, void *param, int32_t code) { pRequest->metric.semanticEnd = taosGetTimestampUs(); - if (code == TSDB_CODE_SUCCESS && pWrapper->pParseCtx->needMultiParse) { - pWrapper->pResultMeta = catalogCloneMetaData(pResultMeta); - if (NULL == pWrapper->pResultMeta) { - code = TSDB_CODE_OUT_OF_MEMORY; - } - } - if (code == TSDB_CODE_SUCCESS) { if (pQuery->haveResultSet) { setResSchemaInfo(&pRequest->body.resInfo, pQuery->pResSchema, pQuery->numOfResCols); @@ -747,14 +739,83 @@ void retrieveMetaCallback(SMetaData *pResultMeta, void *param, int32_t code) { } } -int32_t continueInsertFromCsv(SSqlCallbackWrapper *pWrapper, SRequestObj *pRequest) { - qDestroyQuery(pRequest->pQuery); - pRequest->pQuery = (SQuery *)nodesMakeNode(QUERY_NODE_QUERY); - if (NULL == pRequest->pQuery) { - return TSDB_CODE_OUT_OF_MEMORY; +static int32_t getAllMetaAsync(SSqlCallbackWrapper *pWrapper, catalogCallback fp) { + SRequestConnInfo conn = {.pTrans = pWrapper->pParseCtx->pTransporter, + .requestId = pWrapper->pParseCtx->requestId, + .requestObjRefId = pWrapper->pParseCtx->requestRid, + .mgmtEps = pWrapper->pParseCtx->mgmtEpSet}; + + pWrapper->pRequest->metric.ctgStart = taosGetTimestampUs(); + + return catalogAsyncGetAllMeta(pWrapper->pParseCtx->pCatalog, &conn, pWrapper->pCatalogReq, fp, pWrapper, + &pWrapper->pRequest->body.queryJob); +} + +static void doAsyncQueryFromParse(SMetaData *pResultMeta, void *param, int32_t code); + +static int32_t phaseAsyncQuery(SSqlCallbackWrapper *pWrapper) { + int32_t code = TSDB_CODE_SUCCESS; + switch (pWrapper->pRequest->pQuery->execStage) { + case QUERY_EXEC_STAGE_PARSE: { + // continue parse after get metadata + code = getAllMetaAsync(pWrapper, doAsyncQueryFromParse); + break; + } + case QUERY_EXEC_STAGE_ANALYSE: { + // analysis after get metadata + code = getAllMetaAsync(pWrapper, doAsyncQueryFromAnalyse); + break; + } + case QUERY_EXEC_STAGE_SCHEDULE: { + launchAsyncQuery(pWrapper->pRequest, pWrapper->pRequest->pQuery, NULL, pWrapper); + break; + } + default: + break; + } + return code; +} + +static void doAsyncQueryFromParse(SMetaData *pResultMeta, void *param, int32_t code) { + SSqlCallbackWrapper *pWrapper = (SSqlCallbackWrapper *)param; + SRequestObj *pRequest = pWrapper->pRequest; + SQuery *pQuery = pRequest->pQuery; + + pRequest->metric.ctgEnd = taosGetTimestampUs(); + qDebug("0x%" PRIx64 " start to continue parse, reqId:0x%" PRIx64, pRequest->self, pRequest->requestId); + + if (code == TSDB_CODE_SUCCESS) { + code = qContinueParseSql(pWrapper->pParseCtx, pWrapper->pCatalogReq, pResultMeta, pQuery); + } + + if (TSDB_CODE_SUCCESS == code) { + code = phaseAsyncQuery(pWrapper); + } + + if (TSDB_CODE_SUCCESS != code) { + tscError("0x%" PRIx64 " error happens, code:%d - %s, reqId:0x%" PRIx64, pWrapper->pRequest->self, code, + tstrerror(code), pWrapper->pRequest->requestId); + destorySqlCallbackWrapper(pWrapper); + terrno = code; + pWrapper->pRequest->code = code; + pWrapper->pRequest->body.queryFp(pWrapper->pRequest->body.param, pWrapper->pRequest, code); + } +} + +void continueInsertFromCsv(SSqlCallbackWrapper *pWrapper, SRequestObj *pRequest) { + int32_t code = qParseSqlSyntax(pWrapper->pParseCtx, &pRequest->pQuery, pWrapper->pCatalogReq); + if (TSDB_CODE_SUCCESS == code) { + code = phaseAsyncQuery(pWrapper); + } + + if (TSDB_CODE_SUCCESS != code) { + tscError("0x%" PRIx64 " error happens, code:%d - %s, reqId:0x%" PRIx64, pWrapper->pRequest->self, code, + tstrerror(code), pWrapper->pRequest->requestId); + destorySqlCallbackWrapper(pWrapper); + terrno = code; + pWrapper->pRequest->code = code; + pWrapper->pRequest->body.queryFp(pWrapper->pRequest->body.param, pWrapper->pRequest, code); } - retrieveMetaCallback(pWrapper->pResultMeta, pWrapper, TSDB_CODE_SUCCESS); - return TSDB_CODE_SUCCESS; } void taos_query_a(TAOS *taos, const char *sql, __taos_async_fn_t fp, void *param) { @@ -836,26 +897,16 @@ void doAsyncQuery(SRequestObj *pRequest, bool updateMetaForce) { if (TSDB_CODE_SUCCESS == code && !updateMetaForce) { SAppClusterSummary *pActivity = &pTscObj->pAppInfo->summary; - if (NULL == pRequest->pQuery->pRoot) { + if (QUERY_NODE_INSERT_STMT == nodeType(pRequest->pQuery->pRoot)) { atomic_add_fetch_64((int64_t *)&pActivity->numOfInsertsReq, 1); - } else if (QUERY_NODE_SELECT_STMT == pRequest->pQuery->pRoot->type) { + } else if (QUERY_NODE_SELECT_STMT == nodeType(pRequest->pQuery->pRoot)) { atomic_add_fetch_64((int64_t *)&pActivity->numOfQueryReq, 1); } } if (TSDB_CODE_SUCCESS == code) { - SRequestConnInfo conn = {.pTrans = pWrapper->pParseCtx->pTransporter, - .requestId = pWrapper->pParseCtx->requestId, - .requestObjRefId = pWrapper->pParseCtx->requestRid, - .mgmtEps = pWrapper->pParseCtx->mgmtEpSet}; - - pRequest->metric.ctgStart = taosGetTimestampUs(); - - code = catalogAsyncGetAllMeta(pWrapper->pParseCtx->pCatalog, &conn, pWrapper->pCatalogReq, retrieveMetaCallback, - pWrapper, &pRequest->body.queryJob); - } - - if (TSDB_CODE_SUCCESS != code) { + phaseAsyncQuery(pWrapper); + } else { tscError("0x%" PRIx64 " error happens, code:%d - %s, reqId:0x%" PRIx64, pRequest->self, code, tstrerror(code), pRequest->requestId); destorySqlCallbackWrapper(pWrapper); diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 0c13dd822b..778bfcdf82 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -791,9 +791,24 @@ void nodesDestroyNode(SNode* pNode) { nodesDestroyNode((SNode*)pStmt->pSlimit); break; } - case QUERY_NODE_VNODE_MODIF_STMT: - destroyVgDataBlockArray(((SVnodeModifOpStmt*)pNode)->pDataBlocks); + case QUERY_NODE_VNODE_MODIF_STMT: { + SVnodeModifOpStmt* pStmt = (SVnodeModifOpStmt*)pNode; + destroyVgDataBlockArray(pStmt->pDataBlocks); + taosMemoryFreeClear(pStmt->pTableMeta); + taosHashCleanup(pStmt->pVgroupsHashObj); + taosHashCleanup(pStmt->pSubTableHashObj); + taosHashCleanup(pStmt->pTableNameHashObj); + taosHashCleanup(pStmt->pDbFNameHashObj); + if (pStmt->freeHashFunc) { + pStmt->freeHashFunc(pStmt->pTableBlockHashObj); + } + if (pStmt->freeArrayFunc) { + pStmt->freeArrayFunc(pStmt->pVgDataBlocks); + } + tdDestroySVCreateTbReq(&pStmt->createTblReq); + taosCloseFile(&pStmt->fp); break; + } case QUERY_NODE_CREATE_DATABASE_STMT: nodesDestroyNode((SNode*)((SCreateDatabaseStmt*)pNode)->pOptions); break; diff --git a/source/libs/parser/inc/parInsertUtil.h b/source/libs/parser/inc/parInsertUtil.h index 1e941632e7..f747fdc2c9 100644 --- a/source/libs/parser/inc/parInsertUtil.h +++ b/source/libs/parser/inc/parInsertUtil.h @@ -79,29 +79,6 @@ typedef struct SInsertParseBaseContext { SMsgBuf msg; } SInsertParseBaseContext; -typedef struct SInsertParseContext { - SParseContext *pComCxt; // input - char *pSql; // input - SMsgBuf msg; // input - STableMeta *pTableMeta; // each table - SParsedDataColInfo tags; // each table - SVCreateTbReq createTblReq; // each table - SHashObj *pVgroupsHashObj; // global - SHashObj *pTableBlockHashObj; // global - SHashObj *pSubTableHashObj; // global - SArray *pVgDataBlocks; // global - SHashObj *pTableNameHashObj; // global - SHashObj *pDbFNameHashObj; // global - int32_t totalNum; - SVnodeModifOpStmt *pOutput; - SStmtCallback *pStmtCb; - SParseMetaCache *pMetaCache; - char sTableName[TSDB_TABLE_NAME_LEN]; - char tmpTokenBuf[TSDB_MAX_BYTES_PER_ROW]; - int64_t memElapsed; - int64_t parRowElapsed; -} SInsertParseContext; - typedef struct SInsertParseSyntaxCxt { SParseContext *pComCxt; char *pSql; @@ -142,7 +119,7 @@ typedef struct STableDataBlocks { int32_t insGetExtendedRowSize(STableDataBlocks *pBlock); void insGetSTSRowAppendInfo(uint8_t rowType, SParsedDataColInfo *spd, col_id_t idx, int32_t *toffset, col_id_t *colIdx); -int32_t insSetBlockInfo(SSubmitBlk *pBlocks, STableDataBlocks *dataBuf, int32_t numOfRows); +int32_t insSetBlockInfo(SSubmitBlk *pBlocks, STableDataBlocks *dataBuf, int32_t numOfRows, SMsgBuf *pMsg); int32_t insSchemaIdxCompar(const void *lhs, const void *rhs); int32_t insBoundIdxCompar(const void *lhs, const void *rhs); void insSetBoundColumnInfo(SParsedDataColInfo *pColList, SSchema *pSchema, col_id_t numOfCols); @@ -161,7 +138,7 @@ void insBuildCreateTbReq(SVCreateTbReq *pTbReq, const char *tname, STag *pTag SArray *tagName, uint8_t tagNum); int32_t insMemRowAppend(SMsgBuf *pMsgBuf, const void *value, int32_t len, void *param); int32_t insCheckTimestamp(STableDataBlocks *pDataBlocks, const char *start); -int32_t insBuildOutput(SInsertParseContext *pCxt); +int32_t insBuildOutput(SVnodeModifOpStmt *pStmt); void insDestroyDataBlock(STableDataBlocks *pDataBlock); #endif // TDENGINE_PAR_INSERT_UTIL_H diff --git a/source/libs/parser/inc/parInt.h b/source/libs/parser/inc/parInt.h index b799fa9855..66aec272d7 100644 --- a/source/libs/parser/inc/parInt.h +++ b/source/libs/parser/inc/parInt.h @@ -27,8 +27,7 @@ extern "C" { #define QUERY_SMA_OPTIMIZE_DISABLE 0 #define QUERY_SMA_OPTIMIZE_ENABLE 1 -int32_t parseInsertSyntax(SParseContext* pContext, SQuery** pQuery, SParseMetaCache* pMetaCache); -int32_t parseInsertSql(SParseContext* pContext, SQuery** pQuery, SParseMetaCache* pMetaCache); +int32_t parseInsertSql(SParseContext* pCxt, SQuery** pQuery, SCatalogReq* pCatalogReq, const SMetaData* pMetaData); int32_t parse(SParseContext* pParseCxt, SQuery** pQuery); int32_t collectMetaKey(SParseContext* pParseCxt, SQuery* pQuery, SParseMetaCache* pMetaCache); int32_t authenticate(SParseContext* pParseCxt, SQuery* pQuery, SParseMetaCache* pMetaCache); diff --git a/source/libs/parser/inc/parUtil.h b/source/libs/parser/inc/parUtil.h index 75b631ec9d..10a86866d5 100644 --- a/source/libs/parser/inc/parUtil.h +++ b/source/libs/parser/inc/parUtil.h @@ -60,22 +60,17 @@ typedef struct SInsertTablesMetaReq { } SInsertTablesMetaReq; typedef struct SParseMetaCache { - SHashObj* pTableMeta; // key is tbFName, element is STableMeta* - SHashObj* pDbVgroup; // key is dbFName, element is SArray* - SHashObj* pTableVgroup; // key is tbFName, element is SVgroupInfo* - SHashObj* pDbCfg; // key is tbFName, element is SDbCfgInfo* - SHashObj* pDbInfo; // key is tbFName, element is SDbInfo* - SHashObj* pUserAuth; // key is SUserAuthInfo serialized string, element is bool indicating whether or not to pass - SHashObj* pUdf; // key is funcName, element is SFuncInfo* - SHashObj* pTableIndex; // key is tbFName, element is SArray* - SHashObj* pTableCfg; // key is tbFName, element is STableCfg* - SArray* pDnodes; // element is SEpSet - bool dnodeRequired; - SHashObj* pInsertTables; // key is dbName, element is SInsertTablesMetaReq*, for insert - const char* pUser; - const SArray* pTableMetaData; // pRes = STableMeta* - const SArray* pTableVgroupData; // pRes = SVgroupInfo* - int32_t sqlTableNum; + SHashObj* pTableMeta; // key is tbFName, element is STableMeta* + SHashObj* pDbVgroup; // key is dbFName, element is SArray* + SHashObj* pTableVgroup; // key is tbFName, element is SVgroupInfo* + SHashObj* pDbCfg; // key is tbFName, element is SDbCfgInfo* + SHashObj* pDbInfo; // key is tbFName, element is SDbInfo* + SHashObj* pUserAuth; // key is SUserAuthInfo serialized string, element is bool indicating whether or not to pass + SHashObj* pUdf; // key is funcName, element is SFuncInfo* + SHashObj* pTableIndex; // key is tbFName, element is SArray* + SHashObj* pTableCfg; // key is tbFName, element is STableCfg* + SArray* pDnodes; // element is SEpSet + bool dnodeRequired; } SParseMetaCache; int32_t generateSyntaxErrMsg(SMsgBuf* pBuf, int32_t errCode, ...); @@ -93,9 +88,8 @@ STableMeta* tableMetaDup(const STableMeta* pTableMeta); int32_t trimString(const char* src, int32_t len, char* dst, int32_t dlen); int32_t getInsTagsTableTargetName(int32_t acctId, SNode* pWhere, SName* pName); -int32_t buildCatalogReq(SParseContext* pCxt, const SParseMetaCache* pMetaCache, SCatalogReq* pCatalogReq); -int32_t putMetaDataToCache(const SCatalogReq* pCatalogReq, const SMetaData* pMetaData, SParseMetaCache* pMetaCache, - bool insertValuesStmt); +int32_t buildCatalogReq(const SParseMetaCache* pMetaCache, SCatalogReq* pCatalogReq); +int32_t putMetaDataToCache(const SCatalogReq* pCatalogReq, const SMetaData* pMetaData, SParseMetaCache* pMetaCache); int32_t reserveTableMetaInCache(int32_t acctId, const char* pDb, const char* pTable, SParseMetaCache* pMetaCache); int32_t reserveTableMetaInCacheExt(const SName* pName, SParseMetaCache* pMetaCache); int32_t reserveDbVgInfoInCache(int32_t acctId, const char* pDb, SParseMetaCache* pMetaCache); @@ -122,12 +116,6 @@ int32_t getUdfInfoFromCache(SParseMetaCache* pMetaCache, const char* pFunc, SFun int32_t getTableIndexFromCache(SParseMetaCache* pMetaCache, const SName* pName, SArray** pIndexes); int32_t getTableCfgFromCache(SParseMetaCache* pMetaCache, const SName* pName, STableCfg** pOutput); int32_t getDnodeListFromCache(SParseMetaCache* pMetaCache, SArray** pDnodes); -int32_t reserveTableMetaInCacheForInsert(const SName* pName, ECatalogReqType reqType, int32_t tableNo, - SParseMetaCache* pMetaCache); -int32_t getTableMetaFromCacheForInsert(SArray* pTableMetaPos, SParseMetaCache* pMetaCache, int32_t tableNo, - STableMeta** pMeta); -int32_t getTableVgroupFromCacheForInsert(SArray* pTableVgroupPos, SParseMetaCache* pMetaCache, int32_t tableNo, - SVgroupInfo* pVgroup); void destoryParseMetaCache(SParseMetaCache* pMetaCache, bool request); #ifdef __cplusplus diff --git a/source/libs/parser/src/parAstParser.c b/source/libs/parser/src/parAstParser.c index 28dc6179f9..bb3d1a2cb3 100644 --- a/source/libs/parser/src/parAstParser.c +++ b/source/libs/parser/src/parAstParser.c @@ -84,6 +84,7 @@ abort_parse: (*pQuery)->pRoot = cxt.pRootNode; (*pQuery)->placeholderNum = cxt.placeholderNo; TSWAP((*pQuery)->pPlaceholderValues, cxt.pPlaceholderValues); + (*pQuery)->execStage = QUERY_EXEC_STAGE_ANALYSE; } taosArrayDestroy(cxt.pPlaceholderValues); return cxt.errCode; diff --git a/source/libs/parser/src/parInsertSml.c b/source/libs/parser/src/parInsertSml.c index 8be5b86b37..d18b11ad57 100644 --- a/source/libs/parser/src/parInsertSml.c +++ b/source/libs/parser/src/parInsertSml.c @@ -333,11 +333,7 @@ int32_t smlBindData(void* handle, SArray* tags, SArray* colsSchema, SArray* cols } SSubmitBlk* pBlocks = (SSubmitBlk*)(pDataBlock->pData); - if (TSDB_CODE_SUCCESS != insSetBlockInfo(pBlocks, pDataBlock, rowNum)) { - return buildInvalidOperationMsg(&pBuf, "too many rows in sql, total number of rows should be less than INT32_MAX"); - } - - return TSDB_CODE_SUCCESS; + return insSetBlockInfo(pBlocks, pDataBlock, rowNum, &pBuf); } void* smlInitHandle(SQuery* pQuery) { diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index 1ccb34cd1f..d9bfee24d1 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -18,215 +18,38 @@ #include "tglobal.h" #include "ttime.h" -#define NEXT_TOKEN_WITH_PREV(pSql, sToken) \ - do { \ - int32_t index = 0; \ - sToken = tStrGetToken(pSql, &index, true); \ - pSql += index; \ - } while (0) - -#define NEXT_TOKEN_KEEP_SQL(pSql, sToken, index) \ - do { \ - sToken = tStrGetToken(pSql, &index, false); \ - } while (0) - -#define NEXT_VALID_TOKEN(pSql, sToken) \ +#define NEXT_TOKEN_WITH_PREV(pSql, token) \ do { \ - sToken.n = tGetToken(pSql, &sToken.type); \ - sToken.z = pSql; \ - pSql += sToken.n; \ - } while (TK_NK_SPACE == sToken.type) + int32_t index = 0; \ + token = tStrGetToken(pSql, &index, true); \ + pSql += index; \ + } while (0) + +#define NEXT_TOKEN_KEEP_SQL(pSql, token, index) \ + do { \ + token = tStrGetToken(pSql, &index, false); \ + } while (0) + +#define NEXT_VALID_TOKEN(pSql, token) \ + do { \ + (token).n = tGetToken(pSql, &(token).type); \ + (token).z = (char*)pSql; \ + pSql += (token).n; \ + } while (TK_NK_SPACE == (token).type) + +typedef struct SInsertParseContext { + SParseContext* pComCxt; + SMsgBuf msg; + char tmpTokenBuf[TSDB_MAX_BYTES_PER_ROW]; + SParsedDataColInfo tags; // for stmt + bool missCache; +} SInsertParseContext; typedef int32_t (*_row_append_fn_t)(SMsgBuf* pMsgBuf, const void* value, int32_t len, void* param); static uint8_t TRUE_VALUE = (uint8_t)TSDB_TRUE; static uint8_t FALSE_VALUE = (uint8_t)TSDB_FALSE; -static int32_t skipInsertInto(char** pSql, SMsgBuf* pMsg) { - SToken sToken; - NEXT_TOKEN(*pSql, sToken); - if (TK_INSERT != sToken.type && TK_IMPORT != sToken.type) { - return buildSyntaxErrMsg(pMsg, "keyword INSERT is expected", sToken.z); - } - NEXT_TOKEN(*pSql, sToken); - if (TK_INTO != sToken.type) { - return buildSyntaxErrMsg(pMsg, "keyword INTO is expected", sToken.z); - } - return TSDB_CODE_SUCCESS; -} - -static int32_t checkAuth(SInsertParseContext* pCxt, char* pDbFname, bool* pPass) { - SParseContext* pBasicCtx = pCxt->pComCxt; - if (pBasicCtx->async) { - return getUserAuthFromCache(pCxt->pMetaCache, pBasicCtx->pUser, pDbFname, AUTH_TYPE_WRITE, pPass); - } - SRequestConnInfo conn = {.pTrans = pBasicCtx->pTransporter, - .requestId = pBasicCtx->requestId, - .requestObjRefId = pBasicCtx->requestRid, - .mgmtEps = pBasicCtx->mgmtEpSet}; - - return catalogChkAuth(pBasicCtx->pCatalog, &conn, pBasicCtx->pUser, pDbFname, AUTH_TYPE_WRITE, pPass); -} - -static int32_t getTableSchema(SInsertParseContext* pCxt, int32_t tbNo, SName* pTbName, bool isStb, - STableMeta** pTableMeta) { - SParseContext* pBasicCtx = pCxt->pComCxt; - if (pBasicCtx->async) { - return getTableMetaFromCacheForInsert(pBasicCtx->pTableMetaPos, pCxt->pMetaCache, tbNo, pTableMeta); - } - SRequestConnInfo conn = {.pTrans = pBasicCtx->pTransporter, - .requestId = pBasicCtx->requestId, - .requestObjRefId = pBasicCtx->requestRid, - .mgmtEps = pBasicCtx->mgmtEpSet}; - - if (isStb) { - return catalogGetSTableMeta(pBasicCtx->pCatalog, &conn, pTbName, pTableMeta); - } - return catalogGetTableMeta(pBasicCtx->pCatalog, &conn, pTbName, pTableMeta); -} - -static int32_t getTableVgroup(SInsertParseContext* pCxt, int32_t tbNo, SName* pTbName, SVgroupInfo* pVg) { - SParseContext* pBasicCtx = pCxt->pComCxt; - if (pBasicCtx->async) { - return getTableVgroupFromCacheForInsert(pBasicCtx->pTableVgroupPos, pCxt->pMetaCache, tbNo, pVg); - } - SRequestConnInfo conn = {.pTrans = pBasicCtx->pTransporter, - .requestId = pBasicCtx->requestId, - .requestObjRefId = pBasicCtx->requestRid, - .mgmtEps = pBasicCtx->mgmtEpSet}; - return catalogGetTableHashVgroup(pBasicCtx->pCatalog, &conn, pTbName, pVg); -} - -static int32_t getTableMetaImpl(SInsertParseContext* pCxt, int32_t tbNo, SName* name, bool isStb) { - CHECK_CODE(getTableSchema(pCxt, tbNo, name, isStb, &pCxt->pTableMeta)); - if (!isStb) { - SVgroupInfo vg; - CHECK_CODE(getTableVgroup(pCxt, tbNo, name, &vg)); - CHECK_CODE(taosHashPut(pCxt->pVgroupsHashObj, (const char*)&vg.vgId, sizeof(vg.vgId), (char*)&vg, sizeof(vg))); - } - return TSDB_CODE_SUCCESS; -} - -static int32_t getTableMeta(SInsertParseContext* pCxt, int32_t tbNo, SName* name) { - return getTableMetaImpl(pCxt, tbNo, name, false); -} - -static int32_t getSTableMeta(SInsertParseContext* pCxt, int32_t tbNo, SName* name) { - return getTableMetaImpl(pCxt, tbNo, name, true); -} - -static int32_t getDBCfg(SInsertParseContext* pCxt, const char* pDbFName, SDbCfgInfo* pInfo) { - SParseContext* pBasicCtx = pCxt->pComCxt; - if (pBasicCtx->async) { - CHECK_CODE(getDbCfgFromCache(pCxt->pMetaCache, pDbFName, pInfo)); - } else { - SRequestConnInfo conn = {.pTrans = pBasicCtx->pTransporter, - .requestId = pBasicCtx->requestId, - .requestObjRefId = pBasicCtx->requestRid, - .mgmtEps = pBasicCtx->mgmtEpSet}; - CHECK_CODE(catalogGetDBCfg(pBasicCtx->pCatalog, &conn, pDbFName, pInfo)); - } - return TSDB_CODE_SUCCESS; -} - -static int parseTime(char** end, SToken* pToken, int16_t timePrec, int64_t* time, SMsgBuf* pMsgBuf) { - int32_t index = 0; - SToken sToken; - int64_t interval; - int64_t ts = 0; - char* pTokenEnd = *end; - - if (pToken->type == TK_NOW) { - ts = taosGetTimestamp(timePrec); - } else if (pToken->type == TK_TODAY) { - ts = taosGetTimestampToday(timePrec); - } else if (pToken->type == TK_NK_INTEGER) { - if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &ts)) { - return buildSyntaxErrMsg(pMsgBuf, "invalid timestamp format", pToken->z); - } - } else { // parse the RFC-3339/ISO-8601 timestamp format string - if (taosParseTime(pToken->z, time, pToken->n, timePrec, tsDaylight) != TSDB_CODE_SUCCESS) { - return buildSyntaxErrMsg(pMsgBuf, "invalid timestamp format", pToken->z); - } - - return TSDB_CODE_SUCCESS; - } - - for (int k = pToken->n; pToken->z[k] != '\0'; k++) { - if (pToken->z[k] == ' ' || pToken->z[k] == '\t') continue; - if (pToken->z[k] == '(' && pToken->z[k + 1] == ')') { // for insert NOW()/TODAY() - *end = pTokenEnd = &pToken->z[k + 2]; - k++; - continue; - } - if (pToken->z[k] == ',') { - *end = pTokenEnd; - *time = ts; - return 0; - } - - break; - } - - /* - * time expression: - * e.g., now+12a, now-5h - */ - SToken valueToken; - index = 0; - sToken = tStrGetToken(pTokenEnd, &index, false); - pTokenEnd += index; - - if (sToken.type == TK_NK_MINUS || sToken.type == TK_NK_PLUS) { - index = 0; - valueToken = tStrGetToken(pTokenEnd, &index, false); - pTokenEnd += index; - - if (valueToken.n < 2) { - return buildSyntaxErrMsg(pMsgBuf, "value expected in timestamp", sToken.z); - } - - char unit = 0; - if (parseAbsoluteDuration(valueToken.z, valueToken.n, &interval, &unit, timePrec) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - if (sToken.type == TK_NK_PLUS) { - ts += interval; - } else { - ts = ts - interval; - } - - *end = pTokenEnd; - } - - *time = ts; - return TSDB_CODE_SUCCESS; -} - -static FORCE_INLINE int32_t checkAndTrimValue(SToken* pToken, char* tmpTokenBuf, SMsgBuf* pMsgBuf) { - if ((pToken->type != TK_NOW && pToken->type != TK_TODAY && pToken->type != TK_NK_INTEGER && - pToken->type != TK_NK_STRING && pToken->type != TK_NK_FLOAT && pToken->type != TK_NK_BOOL && - pToken->type != TK_NULL && pToken->type != TK_NK_HEX && pToken->type != TK_NK_OCT && - pToken->type != TK_NK_BIN) || - (pToken->n == 0) || (pToken->type == TK_NK_RP)) { - return buildSyntaxErrMsg(pMsgBuf, "invalid data or symbol", pToken->z); - } - - // Remove quotation marks - if (TK_NK_STRING == pToken->type) { - if (pToken->n >= TSDB_MAX_BYTES_PER_ROW) { - return buildSyntaxErrMsg(pMsgBuf, "too long string", pToken->z); - } - - int32_t len = trimString(pToken->z, pToken->n, tmpTokenBuf, TSDB_MAX_BYTES_PER_ROW); - pToken->z = tmpTokenBuf; - pToken->n = len; - } - - return TSDB_CODE_SUCCESS; -} - static bool isNullStr(SToken* pToken) { return ((pToken->type == TK_NK_STRING) && (strlen(TSDB_DATA_NULL_STR_L) == pToken->n) && (strncasecmp(TSDB_DATA_NULL_STR_L, pToken->z, pToken->n) == 0)); @@ -248,182 +71,108 @@ static FORCE_INLINE int32_t toDouble(SToken* pToken, double* value, char** endPt return pToken->type; } -static int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int16_t timePrec, char* tmpTokenBuf, - _row_append_fn_t func, void* param, SMsgBuf* pMsgBuf) { - int64_t iv; - uint64_t uv; - char* endptr = NULL; - - int32_t code = checkAndTrimValue(pToken, tmpTokenBuf, pMsgBuf); - if (code != TSDB_CODE_SUCCESS) { - return code; +static int32_t skipInsertInto(const char** pSql, SMsgBuf* pMsg) { + SToken token; + NEXT_TOKEN(*pSql, token); + if (TK_INSERT != token.type && TK_IMPORT != token.type) { + return buildSyntaxErrMsg(pMsg, "keyword INSERT is expected", token.z); } - - if (isNullValue(pSchema->type, pToken)) { - if (TSDB_DATA_TYPE_TIMESTAMP == pSchema->type && PRIMARYKEY_TIMESTAMP_COL_ID == pSchema->colId) { - return buildSyntaxErrMsg(pMsgBuf, "primary timestamp should not be null", pToken->z); - } - - return func(pMsgBuf, NULL, 0, param); + NEXT_TOKEN(*pSql, token); + if (TK_INTO != token.type) { + return buildSyntaxErrMsg(pMsg, "keyword INTO is expected", token.z); } - - if (IS_NUMERIC_TYPE(pSchema->type) && pToken->n == 0) { - return buildSyntaxErrMsg(pMsgBuf, "invalid numeric data", pToken->z); - } - - switch (pSchema->type) { - case TSDB_DATA_TYPE_BOOL: { - if ((pToken->type == TK_NK_BOOL || pToken->type == TK_NK_STRING) && (pToken->n != 0)) { - if (strncmp(pToken->z, "true", pToken->n) == 0) { - return func(pMsgBuf, &TRUE_VALUE, pSchema->bytes, param); - } else if (strncmp(pToken->z, "false", pToken->n) == 0) { - return func(pMsgBuf, &FALSE_VALUE, pSchema->bytes, param); - } else { - return buildSyntaxErrMsg(pMsgBuf, "invalid bool data", pToken->z); - } - } else if (pToken->type == TK_NK_INTEGER) { - return func(pMsgBuf, ((taosStr2Int64(pToken->z, NULL, 10) == 0) ? &FALSE_VALUE : &TRUE_VALUE), pSchema->bytes, - param); - } else if (pToken->type == TK_NK_FLOAT) { - return func(pMsgBuf, ((taosStr2Double(pToken->z, NULL) == 0) ? &FALSE_VALUE : &TRUE_VALUE), pSchema->bytes, - param); - } else { - return buildSyntaxErrMsg(pMsgBuf, "invalid bool data", pToken->z); - } - } - - case TSDB_DATA_TYPE_TINYINT: { - if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv)) { - return buildSyntaxErrMsg(pMsgBuf, "invalid tinyint data", pToken->z); - } else if (!IS_VALID_TINYINT(iv)) { - return buildSyntaxErrMsg(pMsgBuf, "tinyint data overflow", pToken->z); - } - - uint8_t tmpVal = (uint8_t)iv; - return func(pMsgBuf, &tmpVal, pSchema->bytes, param); - } - - case TSDB_DATA_TYPE_UTINYINT: { - if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &uv)) { - return buildSyntaxErrMsg(pMsgBuf, "invalid unsigned tinyint data", pToken->z); - } else if (uv > UINT8_MAX) { - return buildSyntaxErrMsg(pMsgBuf, "unsigned tinyint data overflow", pToken->z); - } - uint8_t tmpVal = (uint8_t)uv; - return func(pMsgBuf, &tmpVal, pSchema->bytes, param); - } - - case TSDB_DATA_TYPE_SMALLINT: { - if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv)) { - return buildSyntaxErrMsg(pMsgBuf, "invalid smallint data", pToken->z); - } else if (!IS_VALID_SMALLINT(iv)) { - return buildSyntaxErrMsg(pMsgBuf, "smallint data overflow", pToken->z); - } - int16_t tmpVal = (int16_t)iv; - return func(pMsgBuf, &tmpVal, pSchema->bytes, param); - } - - case TSDB_DATA_TYPE_USMALLINT: { - if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &uv)) { - return buildSyntaxErrMsg(pMsgBuf, "invalid unsigned smallint data", pToken->z); - } else if (uv > UINT16_MAX) { - return buildSyntaxErrMsg(pMsgBuf, "unsigned smallint data overflow", pToken->z); - } - uint16_t tmpVal = (uint16_t)uv; - return func(pMsgBuf, &tmpVal, pSchema->bytes, param); - } - - case TSDB_DATA_TYPE_INT: { - if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv)) { - return buildSyntaxErrMsg(pMsgBuf, "invalid int data", pToken->z); - } else if (!IS_VALID_INT(iv)) { - return buildSyntaxErrMsg(pMsgBuf, "int data overflow", pToken->z); - } - int32_t tmpVal = (int32_t)iv; - return func(pMsgBuf, &tmpVal, pSchema->bytes, param); - } - - case TSDB_DATA_TYPE_UINT: { - if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &uv)) { - return buildSyntaxErrMsg(pMsgBuf, "invalid unsigned int data", pToken->z); - } else if (uv > UINT32_MAX) { - return buildSyntaxErrMsg(pMsgBuf, "unsigned int data overflow", pToken->z); - } - uint32_t tmpVal = (uint32_t)uv; - return func(pMsgBuf, &tmpVal, pSchema->bytes, param); - } - - case TSDB_DATA_TYPE_BIGINT: { - if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv)) { - return buildSyntaxErrMsg(pMsgBuf, "invalid bigint data", pToken->z); - } - return func(pMsgBuf, &iv, pSchema->bytes, param); - } - - case TSDB_DATA_TYPE_UBIGINT: { - if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &uv)) { - return buildSyntaxErrMsg(pMsgBuf, "invalid unsigned bigint data", pToken->z); - } - return func(pMsgBuf, &uv, pSchema->bytes, param); - } - - case TSDB_DATA_TYPE_FLOAT: { - double dv; - if (TK_NK_ILLEGAL == toDouble(pToken, &dv, &endptr)) { - return buildSyntaxErrMsg(pMsgBuf, "illegal float data", pToken->z); - } - if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || dv > FLT_MAX || dv < -FLT_MAX || isinf(dv) || - isnan(dv)) { - return buildSyntaxErrMsg(pMsgBuf, "illegal float data", pToken->z); - } - float tmpVal = (float)dv; - return func(pMsgBuf, &tmpVal, pSchema->bytes, param); - } - - case TSDB_DATA_TYPE_DOUBLE: { - double dv; - if (TK_NK_ILLEGAL == toDouble(pToken, &dv, &endptr)) { - return buildSyntaxErrMsg(pMsgBuf, "illegal double data", pToken->z); - } - if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || isinf(dv) || isnan(dv)) { - return buildSyntaxErrMsg(pMsgBuf, "illegal double data", pToken->z); - } - return func(pMsgBuf, &dv, pSchema->bytes, param); - } - - case TSDB_DATA_TYPE_BINARY: { - // Too long values will raise the invalid sql error message - if (pToken->n + VARSTR_HEADER_SIZE > pSchema->bytes) { - return generateSyntaxErrMsg(pMsgBuf, TSDB_CODE_PAR_VALUE_TOO_LONG, pSchema->name); - } - - return func(pMsgBuf, pToken->z, pToken->n, param); - } - - case TSDB_DATA_TYPE_NCHAR: { - return func(pMsgBuf, pToken->z, pToken->n, param); - } - case TSDB_DATA_TYPE_JSON: { - if (pToken->n > (TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { - return buildSyntaxErrMsg(pMsgBuf, "json string too long than 4095", pToken->z); - } - return func(pMsgBuf, pToken->z, pToken->n, param); - } - case TSDB_DATA_TYPE_TIMESTAMP: { - int64_t tmpVal; - if (parseTime(end, pToken, timePrec, &tmpVal, pMsgBuf) != TSDB_CODE_SUCCESS) { - return buildSyntaxErrMsg(pMsgBuf, "invalid timestamp", pToken->z); - } - - return func(pMsgBuf, &tmpVal, pSchema->bytes, param); - } - } - - return TSDB_CODE_FAILED; + return TSDB_CODE_SUCCESS; } -// pSql -> tag1_name, ...) -static int32_t parseBoundColumns(SInsertParseContext* pCxt, SParsedDataColInfo* pColList, SSchema* pSchema) { +static int32_t skipParentheses(SInsertParseContext* pCxt, const char** pSql) { + SToken token; + int32_t expectRightParenthesis = 1; + while (1) { + NEXT_TOKEN(*pSql, token); + if (TK_NK_LP == token.type) { + ++expectRightParenthesis; + } else if (TK_NK_RP == token.type && 0 == --expectRightParenthesis) { + break; + } + if (0 == token.n) { + return buildSyntaxErrMsg(&pCxt->msg, ") expected", NULL); + } + } + return TSDB_CODE_SUCCESS; +} + +static int32_t skipTableOptions(SInsertParseContext* pCxt, const char** pSql) { + do { + int32_t index = 0; + SToken token; + NEXT_TOKEN_KEEP_SQL(*pSql, token, index); + if (TK_TTL == token.type || TK_COMMENT == token.type) { + *pSql += index; + NEXT_TOKEN_WITH_PREV(*pSql, token); + } else { + break; + } + } while (1); + return TSDB_CODE_SUCCESS; +} + +// pSql -> stb_name [(tag1_name, ...)] TAGS (tag1_value, ...) +static int32_t ignoreUsingClause(SInsertParseContext* pCxt, const char** pSql) { + int32_t code = TSDB_CODE_SUCCESS; + SToken token; + NEXT_TOKEN(*pSql, token); + + NEXT_TOKEN(*pSql, token); + if (TK_NK_LP == token.type) { + code = skipParentheses(pCxt, pSql); + if (TSDB_CODE_SUCCESS == code) { + NEXT_TOKEN(*pSql, token); + } + } + + // pSql -> TAGS (tag1_value, ...) + if (TSDB_CODE_SUCCESS == code) { + if (TK_TAGS != token.type) { + code = buildSyntaxErrMsg(&pCxt->msg, "TAGS is expected", token.z); + } else { + NEXT_TOKEN(*pSql, token); + } + } + if (TSDB_CODE_SUCCESS == code) { + if (TK_NK_LP != token.type) { + code = buildSyntaxErrMsg(&pCxt->msg, "( is expected", token.z); + } else { + code = skipParentheses(pCxt, pSql); + } + } + + if (TSDB_CODE_SUCCESS == code) { + code = skipTableOptions(pCxt, pSql); + } + + return code; +} + +static int32_t parseDuplicateUsingClause(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, bool* pDuplicate) { + *pDuplicate = false; + + char tbFName[TSDB_TABLE_FNAME_LEN]; + tNameExtractFullName(&pStmt->targetTableName, tbFName); + STableMeta** pMeta = taosHashGet(pStmt->pSubTableHashObj, tbFName, strlen(tbFName)); + if (NULL != pMeta) { + *pDuplicate = true; + int32_t code = ignoreUsingClause(pCxt, &pStmt->pSql); + if (TSDB_CODE_SUCCESS == code) { + return cloneTableMeta(*pMeta, &pStmt->pTableMeta); + } + } + + return TSDB_CODE_SUCCESS; +} + +// pStmt->pSql -> field1_name, ...) +static int32_t parseBoundColumns(SInsertParseContext* pCxt, const char** pSql, SParsedDataColInfo* pColList, + SSchema* pSchema) { col_id_t nCols = pColList->numOfCols; pColList->numOfBound = 0; @@ -433,32 +182,32 @@ static int32_t parseBoundColumns(SInsertParseContext* pCxt, SParsedDataColInfo* pColList->cols[i].valStat = VAL_STAT_NONE; } - SToken sToken; + SToken token; bool isOrdered = true; col_id_t lastColIdx = -1; // last column found while (1) { - NEXT_TOKEN(pCxt->pSql, sToken); + NEXT_TOKEN(*pSql, token); - if (TK_NK_RP == sToken.type) { + if (TK_NK_RP == token.type) { break; } char tmpTokenBuf[TSDB_COL_NAME_LEN + 2] = {0}; // used for deleting Escape character backstick(`) - strncpy(tmpTokenBuf, sToken.z, sToken.n); - sToken.z = tmpTokenBuf; - sToken.n = strdequote(sToken.z); + strncpy(tmpTokenBuf, token.z, token.n); + token.z = tmpTokenBuf; + token.n = strdequote(token.z); col_id_t t = lastColIdx + 1; - col_id_t index = insFindCol(&sToken, t, nCols, pSchema); + col_id_t index = insFindCol(&token, t, nCols, pSchema); if (index < 0 && t > 0) { - index = insFindCol(&sToken, 0, t, pSchema); + index = insFindCol(&token, 0, t, pSchema); isOrdered = false; } if (index < 0) { - return generateSyntaxErrMsg(&pCxt->msg, TSDB_CODE_PAR_INVALID_COLUMN, sToken.z); + return generateSyntaxErrMsg(&pCxt->msg, TSDB_CODE_PAR_INVALID_COLUMN, token.z); } if (pColList->cols[index].valStat == VAL_STAT_HAS) { - return buildSyntaxErrMsg(&pCxt->msg, "duplicated column name", sToken.z); + return buildSyntaxErrMsg(&pCxt->msg, "duplicated column name", token.z); } lastColIdx = index; pColList->cols[index].valStat = VAL_STAT_HAS; @@ -504,7 +253,80 @@ static int32_t parseBoundColumns(SInsertParseContext* pCxt, SParsedDataColInfo* return TSDB_CODE_SUCCESS; } -static int32_t parseTagToken(char** end, SToken* pToken, SSchema* pSchema, int16_t timePrec, STagVal* val, +static int parseTime(const char** end, SToken* pToken, int16_t timePrec, int64_t* time, SMsgBuf* pMsgBuf) { + int32_t index = 0; + int64_t interval; + int64_t ts = 0; + const char* pTokenEnd = *end; + + if (pToken->type == TK_NOW) { + ts = taosGetTimestamp(timePrec); + } else if (pToken->type == TK_TODAY) { + ts = taosGetTimestampToday(timePrec); + } else if (pToken->type == TK_NK_INTEGER) { + if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &ts)) { + return buildSyntaxErrMsg(pMsgBuf, "invalid timestamp format", pToken->z); + } + } else { // parse the RFC-3339/ISO-8601 timestamp format string + if (taosParseTime(pToken->z, time, pToken->n, timePrec, tsDaylight) != TSDB_CODE_SUCCESS) { + return buildSyntaxErrMsg(pMsgBuf, "invalid timestamp format", pToken->z); + } + + return TSDB_CODE_SUCCESS; + } + + for (int k = pToken->n; pToken->z[k] != '\0'; k++) { + if (pToken->z[k] == ' ' || pToken->z[k] == '\t') continue; + if (pToken->z[k] == '(' && pToken->z[k + 1] == ')') { // for insert NOW()/TODAY() + *end = pTokenEnd = &pToken->z[k + 2]; + k++; + continue; + } + if (pToken->z[k] == ',') { + *end = pTokenEnd; + *time = ts; + return 0; + } + + break; + } + + /* + * time expression: + * e.g., now+12a, now-5h + */ + index = 0; + SToken token = tStrGetToken(pTokenEnd, &index, false); + pTokenEnd += index; + + if (token.type == TK_NK_MINUS || token.type == TK_NK_PLUS) { + index = 0; + SToken valueToken = tStrGetToken(pTokenEnd, &index, false); + pTokenEnd += index; + + if (valueToken.n < 2) { + return buildSyntaxErrMsg(pMsgBuf, "value expected in timestamp", token.z); + } + + char unit = 0; + if (parseAbsoluteDuration(valueToken.z, valueToken.n, &interval, &unit, timePrec) != TSDB_CODE_SUCCESS) { + return TSDB_CODE_TSC_INVALID_OPERATION; + } + + if (token.type == TK_NK_PLUS) { + ts += interval; + } else { + ts = ts - interval; + } + + *end = pTokenEnd; + } + + *time = ts; + return TSDB_CODE_SUCCESS; +} + +static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema, int16_t timePrec, STagVal* val, SMsgBuf* pMsgBuf) { int64_t iv; uint64_t uv; @@ -688,22 +510,100 @@ static int32_t parseTagToken(char** end, SToken* pToken, SSchema* pSchema, int16 return TSDB_CODE_SUCCESS; } -// pSql -> tag1_value, ...) -static int32_t parseTagsClause(SInsertParseContext* pCxt, SSchema* pSchema, uint8_t precision, const char* tName) { - int32_t code = TSDB_CODE_SUCCESS; - SArray* pTagVals = taosArrayInit(pCxt->tags.numOfBound, sizeof(STagVal)); - SArray* tagName = taosArrayInit(8, TSDB_COL_NAME_LEN); - SToken sToken; - bool isParseBindParam = false; - bool isJson = false; - STag* pTag = NULL; - for (int i = 0; i < pCxt->tags.numOfBound; ++i) { - NEXT_TOKEN_WITH_PREV(pCxt->pSql, sToken); +// input pStmt->pSql: [(tag1_name, ...)] TAGS (tag1_value, ...) ... +// output pStmt->pSql: TAGS (tag1_value, ...) ... +static int32_t parseBoundTagsClause(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { + SSchema* pTagsSchema = getTableTagSchema(pStmt->pTableMeta); + insSetBoundColumnInfo(&pCxt->tags, pTagsSchema, getNumOfTags(pStmt->pTableMeta)); - if (sToken.type == TK_NK_QUESTION) { + SToken token; + int32_t index = 0; + NEXT_TOKEN_KEEP_SQL(pStmt->pSql, token, index); + if (TK_NK_LP != token.type) { + return TSDB_CODE_SUCCESS; + } + + pStmt->pSql += index; + return parseBoundColumns(pCxt, &pStmt->pSql, &pCxt->tags, pTagsSchema); +} + +static int32_t parseTagValue(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, SSchema* pTagSchema, SToken* pToken, + SArray* pTagName, SArray* pTagVals, STag** pTag) { + if (!isNullValue(pTagSchema->type, pToken)) { + taosArrayPush(pTagName, pTagSchema->name); + } + + if (pTagSchema->type == TSDB_DATA_TYPE_JSON) { + if (pToken->n > (TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { + return buildSyntaxErrMsg(&pCxt->msg, "json string too long than 4095", pToken->z); + } + + if (isNullValue(pTagSchema->type, pToken)) { + return tTagNew(pTagVals, 1, true, pTag); + } else { + return parseJsontoTagData(pToken->z, pTagVals, pTag, &pCxt->msg); + } + } + + STagVal val = {0}; + int32_t code = + parseTagToken(&pStmt->pSql, pToken, pTagSchema, pStmt->pTableMeta->tableInfo.precision, &val, &pCxt->msg); + if (TSDB_CODE_SUCCESS == code) { + taosArrayPush(pTagVals, &val); + } + + return code; +} + +static void buildCreateTbReq(SVnodeModifOpStmt* pStmt, STag* pTag, SArray* pTagName) { + char tbFName[TSDB_TABLE_FNAME_LEN]; + tNameExtractFullName(&pStmt->targetTableName, tbFName); + char stbFName[TSDB_TABLE_FNAME_LEN]; + tNameExtractFullName(&pStmt->usingTableName, stbFName); + insBuildCreateTbReq(&pStmt->createTblReq, tbFName, pTag, pStmt->pTableMeta->suid, stbFName, pTagName, + pStmt->pTableMeta->tableInfo.numOfTags); +} + +static int32_t checkAndTrimValue(SToken* pToken, char* tmpTokenBuf, SMsgBuf* pMsgBuf) { + if ((pToken->type != TK_NOW && pToken->type != TK_TODAY && pToken->type != TK_NK_INTEGER && + pToken->type != TK_NK_STRING && pToken->type != TK_NK_FLOAT && pToken->type != TK_NK_BOOL && + pToken->type != TK_NULL && pToken->type != TK_NK_HEX && pToken->type != TK_NK_OCT && + pToken->type != TK_NK_BIN) || + (pToken->n == 0) || (pToken->type == TK_NK_RP)) { + return buildSyntaxErrMsg(pMsgBuf, "invalid data or symbol", pToken->z); + } + + // Remove quotation marks + if (TK_NK_STRING == pToken->type) { + if (pToken->n >= TSDB_MAX_BYTES_PER_ROW) { + return buildSyntaxErrMsg(pMsgBuf, "too long string", pToken->z); + } + + int32_t len = trimString(pToken->z, pToken->n, tmpTokenBuf, TSDB_MAX_BYTES_PER_ROW); + pToken->z = tmpTokenBuf; + pToken->n = len; + } + + return TSDB_CODE_SUCCESS; +} + +// pSql -> tag1_value, ...) +static int32_t parseTagsClauseImpl(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { + int32_t code = TSDB_CODE_SUCCESS; + SSchema* pSchema = getTableTagSchema(pStmt->pTableMeta); + SArray* pTagVals = taosArrayInit(pCxt->tags.numOfBound, sizeof(STagVal)); + SArray* pTagName = taosArrayInit(8, TSDB_COL_NAME_LEN); + SToken token; + bool isParseBindParam = false; + bool isJson = false; + STag* pTag = NULL; + for (int i = 0; TSDB_CODE_SUCCESS == code && i < pCxt->tags.numOfBound; ++i) { + NEXT_TOKEN_WITH_PREV(pStmt->pSql, token); + + if (token.type == TK_NK_QUESTION) { isParseBindParam = true; - if (NULL == pCxt->pStmtCb) { - code = buildSyntaxErrMsg(&pCxt->msg, "? only used in stmt", sToken.z); + if (NULL == pCxt->pComCxt->pStmtCb) { + code = buildSyntaxErrMsg(&pCxt->msg, "? only used in stmt", token.z); break; } @@ -716,33 +616,10 @@ static int32_t parseTagsClause(SInsertParseContext* pCxt, SSchema* pSchema, uint } SSchema* pTagSchema = &pSchema[pCxt->tags.boundColumns[i]]; - char tmpTokenBuf[TSDB_MAX_BYTES_PER_ROW] = {0}; // todo this can be optimize with parse column - code = checkAndTrimValue(&sToken, tmpTokenBuf, &pCxt->msg); + isJson = pTagSchema->type == TSDB_DATA_TYPE_JSON; + code = checkAndTrimValue(&token, pCxt->tmpTokenBuf, &pCxt->msg); if (TSDB_CODE_SUCCESS == code) { - if (!isNullValue(pTagSchema->type, &sToken)) { - taosArrayPush(tagName, pTagSchema->name); - } - if (pTagSchema->type == TSDB_DATA_TYPE_JSON) { - isJson = true; - if (sToken.n > (TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { - code = buildSyntaxErrMsg(&pCxt->msg, "json string too long than 4095", sToken.z); - break; - } - if (isNullValue(pTagSchema->type, &sToken)) { - code = tTagNew(pTagVals, 1, true, &pTag); - } else { - code = parseJsontoTagData(sToken.z, pTagVals, &pTag, &pCxt->msg); - } - } else { - STagVal val = {0}; - code = parseTagToken(&pCxt->pSql, &sToken, pTagSchema, precision, &val, &pCxt->msg); - if (TSDB_CODE_SUCCESS == code) { - taosArrayPush(pTagVals, &val); - } - } - } - if (TSDB_CODE_SUCCESS != code) { - break; + code = parseTagValue(pCxt, pStmt, pTagSchema, &token, pTagName, pTagVals, &pTag); } } @@ -751,8 +628,7 @@ static int32_t parseTagsClause(SInsertParseContext* pCxt, SSchema* pSchema, uint } if (TSDB_CODE_SUCCESS == code && !isParseBindParam) { - insBuildCreateTbReq(&pCxt->createTblReq, tName, pTag, pCxt->pTableMeta->suid, pCxt->sTableName, tagName, - pCxt->pTableMeta->tableInfo.numOfTags); + buildCreateTbReq(pStmt, pTag, pTagName); pTag = NULL; } @@ -763,96 +639,81 @@ static int32_t parseTagsClause(SInsertParseContext* pCxt, SSchema* pSchema, uint } } taosArrayDestroy(pTagVals); - taosArrayDestroy(tagName); + taosArrayDestroy(pTagName); tTagFree(pTag); return code; } -static int32_t storeTableMeta(SInsertParseContext* pCxt, SHashObj* pHash, int32_t tbNo, SName* pTableName, - const char* pName, int32_t len, STableMeta* pMeta) { - SVgroupInfo vg; - CHECK_CODE(getTableVgroup(pCxt, tbNo, pTableName, &vg)); - CHECK_CODE(taosHashPut(pCxt->pVgroupsHashObj, (const char*)&vg.vgId, sizeof(vg.vgId), (char*)&vg, sizeof(vg))); +// input pStmt->pSql: TAGS (tag1_value, ...) [table_options] ... +// output pStmt->pSql: [table_options] ... +static int32_t parseTagsClause(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { + SToken token; + NEXT_TOKEN(pStmt->pSql, token); + if (TK_TAGS != token.type) { + return buildSyntaxErrMsg(&pCxt->msg, "TAGS is expected", token.z); + } - pMeta->uid = tbNo; - pMeta->vgId = vg.vgId; - pMeta->tableType = TSDB_CHILD_TABLE; + NEXT_TOKEN(pStmt->pSql, token); + if (TK_NK_LP != token.type) { + return buildSyntaxErrMsg(&pCxt->msg, "( is expected", token.z); + } + + int32_t code = parseTagsClauseImpl(pCxt, pStmt); + if (TSDB_CODE_SUCCESS == code) { + NEXT_VALID_TOKEN(pStmt->pSql, token); + if (TK_NK_COMMA == token.type) { + code = generateSyntaxErrMsg(&pCxt->msg, TSDB_CODE_PAR_TAGS_NOT_MATCHED); + } else if (TK_NK_RP != token.type) { + code = buildSyntaxErrMsg(&pCxt->msg, ") is expected", token.z); + } + } + return code; +} + +static int32_t storeTableMeta(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { + pStmt->pTableMeta->uid = pStmt->totalTbNum; + pStmt->pTableMeta->tableType = TSDB_CHILD_TABLE; STableMeta* pBackup = NULL; - if (TSDB_CODE_SUCCESS != cloneTableMeta(pMeta, &pBackup)) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; + if (TSDB_CODE_SUCCESS != cloneTableMeta(pStmt->pTableMeta, &pBackup)) { + return TSDB_CODE_OUT_OF_MEMORY; } - return taosHashPut(pHash, pName, len, &pBackup, POINTER_BYTES); + + char tbFName[TSDB_TABLE_FNAME_LEN]; + tNameExtractFullName(&pStmt->targetTableName, tbFName); + return taosHashPut(pStmt->pSubTableHashObj, tbFName, strlen(tbFName), &pBackup, POINTER_BYTES); } -static int32_t skipParentheses(SInsertParseSyntaxCxt* pCxt) { - SToken sToken; - int32_t expectRightParenthesis = 1; - while (1) { - NEXT_TOKEN(pCxt->pSql, sToken); - if (TK_NK_LP == sToken.type) { - ++expectRightParenthesis; - } else if (TK_NK_RP == sToken.type && 0 == --expectRightParenthesis) { - break; - } - if (0 == sToken.n) { - return buildSyntaxErrMsg(&pCxt->msg, ") expected", NULL); - } - } - return TSDB_CODE_SUCCESS; -} - -static int32_t skipBoundColumns(SInsertParseSyntaxCxt* pCxt) { return skipParentheses(pCxt); } - -static int32_t ignoreBoundColumns(SInsertParseContext* pCxt) { - SInsertParseSyntaxCxt cxt = {.pComCxt = pCxt->pComCxt, .pSql = pCxt->pSql, .msg = pCxt->msg, .pMetaCache = NULL}; - int32_t code = skipBoundColumns(&cxt); - pCxt->pSql = cxt.pSql; - return code; -} - -static int32_t skipUsingClause(SInsertParseSyntaxCxt* pCxt); - -// pSql -> stb_name [(tag1_name, ...)] TAGS (tag1_value, ...) -static int32_t ignoreAutoCreateTableClause(SInsertParseContext* pCxt) { - SToken sToken; - NEXT_TOKEN(pCxt->pSql, sToken); - SInsertParseSyntaxCxt cxt = {.pComCxt = pCxt->pComCxt, .pSql = pCxt->pSql, .msg = pCxt->msg, .pMetaCache = NULL}; - int32_t code = skipUsingClause(&cxt); - pCxt->pSql = cxt.pSql; - return code; -} - -static int32_t parseTableOptions(SInsertParseContext* pCxt) { +static int32_t parseTableOptions(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { do { int32_t index = 0; - SToken sToken; - NEXT_TOKEN_KEEP_SQL(pCxt->pSql, sToken, index); - if (TK_TTL == sToken.type) { - pCxt->pSql += index; - NEXT_TOKEN_WITH_PREV(pCxt->pSql, sToken); - if (TK_NK_INTEGER != sToken.type) { - return buildSyntaxErrMsg(&pCxt->msg, "Invalid option ttl", sToken.z); + SToken token; + NEXT_TOKEN_KEEP_SQL(pStmt->pSql, token, index); + if (TK_TTL == token.type) { + pStmt->pSql += index; + NEXT_TOKEN_WITH_PREV(pStmt->pSql, token); + if (TK_NK_INTEGER != token.type) { + return buildSyntaxErrMsg(&pCxt->msg, "Invalid option ttl", token.z); } - pCxt->createTblReq.ttl = taosStr2Int32(sToken.z, NULL, 10); - if (pCxt->createTblReq.ttl < 0) { - return buildSyntaxErrMsg(&pCxt->msg, "Invalid option ttl", sToken.z); + pStmt->createTblReq.ttl = taosStr2Int32(token.z, NULL, 10); + if (pStmt->createTblReq.ttl < 0) { + return buildSyntaxErrMsg(&pCxt->msg, "Invalid option ttl", token.z); } - } else if (TK_COMMENT == sToken.type) { - pCxt->pSql += index; - NEXT_TOKEN(pCxt->pSql, sToken); - if (TK_NK_STRING != sToken.type) { - return buildSyntaxErrMsg(&pCxt->msg, "Invalid option comment", sToken.z); + } else if (TK_COMMENT == token.type) { + pStmt->pSql += index; + NEXT_TOKEN(pStmt->pSql, token); + if (TK_NK_STRING != token.type) { + return buildSyntaxErrMsg(&pCxt->msg, "Invalid option comment", token.z); } - if (sToken.n >= TSDB_TB_COMMENT_LEN) { - return buildSyntaxErrMsg(&pCxt->msg, "comment too long", sToken.z); + if (token.n >= TSDB_TB_COMMENT_LEN) { + return buildSyntaxErrMsg(&pCxt->msg, "comment too long", token.z); } - int32_t len = trimString(sToken.z, sToken.n, pCxt->tmpTokenBuf, TSDB_TB_COMMENT_LEN); - pCxt->createTblReq.comment = strndup(pCxt->tmpTokenBuf, len); - if (NULL == pCxt->createTblReq.comment) { + int32_t len = trimString(token.z, token.n, pCxt->tmpTokenBuf, TSDB_TB_COMMENT_LEN); + pStmt->createTblReq.comment = strndup(pCxt->tmpTokenBuf, len); + if (NULL == pStmt->createTblReq.comment) { return TSDB_CODE_OUT_OF_MEMORY; } - pCxt->createTblReq.commentLen = len; + pStmt->createTblReq.commentLen = len; } else { break; } @@ -860,119 +721,501 @@ static int32_t parseTableOptions(SInsertParseContext* pCxt) { return TSDB_CODE_SUCCESS; } -// pSql -> stb_name [(tag1_name, ...)] TAGS (tag1_value, ...) -static int32_t parseUsingClause(SInsertParseContext* pCxt, int32_t tbNo, SName* name, char* tbFName) { - int32_t len = strlen(tbFName); - STableMeta** pMeta = taosHashGet(pCxt->pSubTableHashObj, tbFName, len); - if (NULL != pMeta) { - CHECK_CODE(ignoreAutoCreateTableClause(pCxt)); - return cloneTableMeta(*pMeta, &pCxt->pTableMeta); +// input pStmt->pSql: +// 1. [(tag1_name, ...)] ... +// 2. VALUES ... | FILE ... +// output pStmt->pSql: +// 1. [(field1_name, ...)] +// 2. VALUES ... | FILE ... +static int32_t parseUsingClauseBottom(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { + if ('\0' == pStmt->usingTableName.tname[0]) { + return TSDB_CODE_SUCCESS; } - SToken sToken; - // pSql -> stb_name [(tag1_name, ...)] TAGS (tag1_value, ...) - NEXT_TOKEN(pCxt->pSql, sToken); - - SName sname; - CHECK_CODE(insCreateSName(&sname, &sToken, pCxt->pComCxt->acctId, pCxt->pComCxt->db, &pCxt->msg)); - char dbFName[TSDB_DB_FNAME_LEN]; - tNameGetFullDbName(&sname, dbFName); - strcpy(pCxt->sTableName, sname.tname); - - CHECK_CODE(getSTableMeta(pCxt, tbNo, &sname)); - if (TSDB_SUPER_TABLE != pCxt->pTableMeta->tableType) { - return buildInvalidOperationMsg(&pCxt->msg, "create table only from super table is allowed"); + int32_t code = parseBoundTagsClause(pCxt, pStmt); + if (TSDB_CODE_SUCCESS == code) { + code = parseTagsClause(pCxt, pStmt); } - CHECK_CODE(storeTableMeta(pCxt, pCxt->pSubTableHashObj, tbNo, name, tbFName, len, pCxt->pTableMeta)); - - SSchema* pTagsSchema = getTableTagSchema(pCxt->pTableMeta); - insSetBoundColumnInfo(&pCxt->tags, pTagsSchema, getNumOfTags(pCxt->pTableMeta)); - - // pSql -> [(tag1_name, ...)] TAGS (tag1_value, ...) - NEXT_TOKEN(pCxt->pSql, sToken); - if (TK_NK_LP == sToken.type) { - CHECK_CODE(parseBoundColumns(pCxt, &pCxt->tags, pTagsSchema)); - NEXT_TOKEN(pCxt->pSql, sToken); + if (TSDB_CODE_SUCCESS == code) { + code = parseTableOptions(pCxt, pStmt); } - if (TK_TAGS != sToken.type) { - return buildSyntaxErrMsg(&pCxt->msg, "TAGS is expected", sToken.z); - } - // pSql -> (tag1_value, ...) - NEXT_TOKEN(pCxt->pSql, sToken); - if (TK_NK_LP != sToken.type) { - return buildSyntaxErrMsg(&pCxt->msg, "( is expected", sToken.z); - } - CHECK_CODE(parseTagsClause(pCxt, pTagsSchema, getTableInfo(pCxt->pTableMeta).precision, name->tname)); - NEXT_VALID_TOKEN(pCxt->pSql, sToken); - if (TK_NK_COMMA == sToken.type) { - return generateSyntaxErrMsg(&pCxt->msg, TSDB_CODE_PAR_TAGS_NOT_MATCHED); - } else if (TK_NK_RP != sToken.type) { - return buildSyntaxErrMsg(&pCxt->msg, ") is expected", sToken.z); - } - - return parseTableOptions(pCxt); + return code; } -static int parseOneRow(SInsertParseContext* pCxt, STableDataBlocks* pDataBlocks, int16_t timePrec, bool* gotRow, - char* tmpTokenBuf) { - SParsedDataColInfo* spd = &pDataBlocks->boundColumnInfo; - SRowBuilder* pBuilder = &pDataBlocks->rowBuilder; - STSRow* row = (STSRow*)(pDataBlocks->pData + pDataBlocks->size); // skip the SSubmitBlk header +static int32_t checkAuth(SParseContext* pCxt, SName* pTbName) { + char dbFName[TSDB_DB_FNAME_LEN]; + tNameGetFullDbName(pTbName, dbFName); + SRequestConnInfo conn = {.pTrans = pCxt->pTransporter, + .requestId = pCxt->requestId, + .requestObjRefId = pCxt->requestRid, + .mgmtEps = pCxt->mgmtEpSet}; + int32_t code = TSDB_CODE_SUCCESS; + bool pass = true; + if (pCxt->async) { + // todo replace with cached api + code = catalogChkAuth(pCxt->pCatalog, &conn, pCxt->pUser, dbFName, AUTH_TYPE_WRITE, &pass); + } else { + code = catalogChkAuth(pCxt->pCatalog, &conn, pCxt->pUser, dbFName, AUTH_TYPE_WRITE, &pass); + } + if (TSDB_CODE_SUCCESS == code && !pass) { + code = TSDB_CODE_PAR_PERMISSION_DENIED; + } + return code; +} - tdSRowResetBuf(pBuilder, row); +static int32_t getTableMeta(SInsertParseContext* pCxt, SName* pTbName, bool isStb, STableMeta** pTableMeta, + bool* pMissCache) { + SParseContext* pComCxt = pCxt->pComCxt; + SRequestConnInfo conn = {.pTrans = pComCxt->pTransporter, + .requestId = pComCxt->requestId, + .requestObjRefId = pComCxt->requestRid, + .mgmtEps = pComCxt->mgmtEpSet}; + int32_t code = TSDB_CODE_SUCCESS; + if (pComCxt->async) { + if (isStb) { + code = catalogGetCachedSTableMeta(pComCxt->pCatalog, &conn, pTbName, pTableMeta); + } else { + code = catalogGetCachedTableMeta(pComCxt->pCatalog, &conn, pTbName, pTableMeta); + } + } else { + if (isStb) { + code = catalogGetSTableMeta(pComCxt->pCatalog, &conn, pTbName, pTableMeta); + } else { + code = catalogGetTableMeta(pComCxt->pCatalog, &conn, pTbName, pTableMeta); + } + } + if (TSDB_CODE_SUCCESS == code) { + if (NULL == *pTableMeta) { + *pMissCache = true; + } else if (isStb && TSDB_SUPER_TABLE != (*pTableMeta)->tableType) { + code = buildInvalidOperationMsg(&pCxt->msg, "create table only from super table is allowed"); + } + } + return code; +} - bool isParseBindParam = false; - SSchema* schema = getTableColumnSchema(pDataBlocks->pTableMeta); - SMemParam param = {.rb = pBuilder}; - SToken sToken = {0}; - // 1. set the parsed value from sql string - for (int i = 0; i < spd->numOfBound; ++i) { - NEXT_TOKEN_WITH_PREV(pCxt->pSql, sToken); - SSchema* pSchema = &schema[spd->boundColumns[i]]; +static int32_t getTableVgroup(SParseContext* pCxt, SVnodeModifOpStmt* pStmt, bool isStb, bool* pMissCache) { + SRequestConnInfo conn = {.pTrans = pCxt->pTransporter, + .requestId = pCxt->requestId, + .requestObjRefId = pCxt->requestRid, + .mgmtEps = pCxt->mgmtEpSet}; + int32_t code = TSDB_CODE_SUCCESS; + SVgroupInfo vg; + bool exists = true; + if (pCxt->async) { + code = catalogGetCachedTableHashVgroup(pCxt->pCatalog, &conn, &pStmt->targetTableName, &vg, &exists); + } else { + code = catalogGetTableHashVgroup(pCxt->pCatalog, &conn, &pStmt->targetTableName, &vg); + } + if (TSDB_CODE_SUCCESS == code) { + if (exists) { + if (isStb) { + pStmt->pTableMeta->vgId = vg.vgId; + } + code = taosHashPut(pStmt->pVgroupsHashObj, (const char*)&vg.vgId, sizeof(vg.vgId), (char*)&vg, sizeof(vg)); + } + *pMissCache = !exists; + } + return code; +} - if (sToken.type == TK_NK_QUESTION) { - isParseBindParam = true; - if (NULL == pCxt->pStmtCb) { - return buildSyntaxErrMsg(&pCxt->msg, "? only used in stmt", sToken.z); +static int32_t getTargetTableSchema(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { + int32_t code = checkAuth(pCxt->pComCxt, &pStmt->targetTableName); + if (TSDB_CODE_SUCCESS == code) { + code = getTableMeta(pCxt, &pStmt->targetTableName, false, &pStmt->pTableMeta, &pCxt->missCache); + } + if (TSDB_CODE_SUCCESS == code && !pCxt->missCache) { + code = getTableVgroup(pCxt->pComCxt, pStmt, false, &pCxt->missCache); + } + return code; +} + +static int32_t preParseUsingTableName(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, SToken* pTbName) { + return insCreateSName(&pStmt->usingTableName, pTbName, pCxt->pComCxt->acctId, pCxt->pComCxt->db, &pCxt->msg); +} + +static int32_t getUsingTableSchema(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { + int32_t code = checkAuth(pCxt->pComCxt, &pStmt->targetTableName); + if (TSDB_CODE_SUCCESS == code) { + code = getTableMeta(pCxt, &pStmt->usingTableName, true, &pStmt->pTableMeta, &pCxt->missCache); + } + if (TSDB_CODE_SUCCESS == code && !pCxt->missCache) { + code = getTableVgroup(pCxt->pComCxt, pStmt, true, &pCxt->missCache); + } + return code; +} + +static int32_t parseUsingTableNameImpl(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { + SToken token; + NEXT_TOKEN(pStmt->pSql, token); + int32_t code = preParseUsingTableName(pCxt, pStmt, &token); + if (TSDB_CODE_SUCCESS == code) { + code = getUsingTableSchema(pCxt, pStmt); + } + if (TSDB_CODE_SUCCESS == code) { + code = storeTableMeta(pCxt, pStmt); + } + return code; +} + +// input pStmt->pSql: +// 1(care). [USING stb_name [(tag1_name, ...)] TAGS (tag1_value, ...) [table_options]] ... +// 2. VALUES ... | FILE ... +// output pStmt->pSql: +// 1. [(tag1_name, ...)] TAGS (tag1_value, ...) [table_options]] ... +// 2. VALUES ... | FILE ... +static int32_t parseUsingTableName(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { + SToken token; + int32_t index = 0; + NEXT_TOKEN_KEEP_SQL(pStmt->pSql, token, index); + if (TK_USING != token.type) { + return getTargetTableSchema(pCxt, pStmt); + } + + // pStmt->pSql -> stb_name [(tag1_name, ...) + pStmt->pSql += index; + bool duplicate = false; + int32_t code = parseDuplicateUsingClause(pCxt, pStmt, &duplicate); + if (TSDB_CODE_SUCCESS == code && !duplicate) { + return parseUsingTableNameImpl(pCxt, pStmt); + } + return code; +} + +static int32_t preParseTargetTableName(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, SToken* pTbName) { + return insCreateSName(&pStmt->targetTableName, pTbName, pCxt->pComCxt->acctId, pCxt->pComCxt->db, &pCxt->msg); +} + +// input pStmt->pSql: +// 1(care). [(field1_name, ...)] ... +// 2. [ USING ... ] ... +// 3. VALUES ... | FILE ... +// output pStmt->pSql: +// 1. [ USING ... ] ... +// 2. VALUES ... | FILE ... +static int32_t preParseBoundColumnsClause(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { + SToken token; + int32_t index = 0; + NEXT_TOKEN_KEEP_SQL(pStmt->pSql, token, index); + if (TK_NK_LP != token.type) { + return TSDB_CODE_SUCCESS; + } + + // pStmt->pSql -> field1_name, ...) + pStmt->pSql += index; + pStmt->pBoundCols = pStmt->pSql; + return skipParentheses(pCxt, &pStmt->pSql); +} + +static int32_t getTableDataBlocks(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, STableDataBlocks** pDataBuf) { + if (pCxt->pComCxt->async) { + return insGetDataBlockFromList(pStmt->pTableBlockHashObj, &pStmt->pTableMeta->uid, sizeof(pStmt->pTableMeta->uid), + TSDB_DEFAULT_PAYLOAD_SIZE, sizeof(SSubmitBlk), + getTableInfo(pStmt->pTableMeta).rowSize, pStmt->pTableMeta, pDataBuf, NULL, + &pStmt->createTblReq); + } + char tbFName[TSDB_TABLE_FNAME_LEN]; + tNameExtractFullName(&pStmt->targetTableName, tbFName); + return insGetDataBlockFromList(pStmt->pTableBlockHashObj, tbFName, strlen(tbFName), TSDB_DEFAULT_PAYLOAD_SIZE, + sizeof(SSubmitBlk), getTableInfo(pStmt->pTableMeta).rowSize, pStmt->pTableMeta, + pDataBuf, NULL, &pStmt->createTblReq); +} + +static int32_t parseBoundColumnsClause(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, + STableDataBlocks* pDataBuf) { + SToken token; + int32_t index = 0; + NEXT_TOKEN_KEEP_SQL(pStmt->pSql, token, index); + if (TK_NK_LP == token.type) { + pStmt->pSql += index; + if (NULL != pStmt->pBoundCols) { + return buildSyntaxErrMsg(&pCxt->msg, "keyword VALUES or FILE is expected", token.z); + } + // pStmt->pSql -> field1_name, ...) + return parseBoundColumns(pCxt, &pStmt->pSql, &pDataBuf->boundColumnInfo, getTableColumnSchema(pStmt->pTableMeta)); + } + + if (NULL != pStmt->pBoundCols) { + return parseBoundColumns(pCxt, &pStmt->pBoundCols, &pDataBuf->boundColumnInfo, + getTableColumnSchema(pStmt->pTableMeta)); + } + + return TSDB_CODE_SUCCESS; +} + +// input pStmt->pSql: +// 1. [(tag1_name, ...)] ... +// 2. VALUES ... | FILE ... +// output pStmt->pSql: VALUES ... | FILE ... +static int32_t parseSchemaClauseBottom(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, + STableDataBlocks** pDataBuf) { + int32_t code = parseUsingClauseBottom(pCxt, pStmt); + if (TSDB_CODE_SUCCESS == code) { + code = getTableDataBlocks(pCxt, pStmt, pDataBuf); + } + if (TSDB_CODE_SUCCESS == code) { + code = parseBoundColumnsClause(pCxt, pStmt, *pDataBuf); + } + return code; +} + +// input pStmt->pSql: [(field1_name, ...)] [ USING ... ] VALUES ... | FILE ... +// output pStmt->pSql: +// 1. [(tag1_name, ...)] ... +// 2. VALUES ... | FILE ... +static int32_t parseSchemaClauseTop(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, SToken* pTbName) { + int32_t code = preParseTargetTableName(pCxt, pStmt, pTbName); + if (TSDB_CODE_SUCCESS == code) { + // option: [(field1_name, ...)] + code = preParseBoundColumnsClause(pCxt, pStmt); + } + if (TSDB_CODE_SUCCESS == code) { + // option: [USING stb_name] + code = parseUsingTableName(pCxt, pStmt); + } + return code; +} + +static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql, SToken* pToken, SSchema* pSchema, + int16_t timePrec, _row_append_fn_t func, void* param) { + int64_t iv; + uint64_t uv; + char* endptr = NULL; + + switch (pSchema->type) { + case TSDB_DATA_TYPE_BOOL: { + if ((pToken->type == TK_NK_BOOL || pToken->type == TK_NK_STRING) && (pToken->n != 0)) { + if (strncmp(pToken->z, "true", pToken->n) == 0) { + return func(&pCxt->msg, &TRUE_VALUE, pSchema->bytes, param); + } else if (strncmp(pToken->z, "false", pToken->n) == 0) { + return func(&pCxt->msg, &FALSE_VALUE, pSchema->bytes, param); + } else { + return buildSyntaxErrMsg(&pCxt->msg, "invalid bool data", pToken->z); + } + } else if (pToken->type == TK_NK_INTEGER) { + return func(&pCxt->msg, ((taosStr2Int64(pToken->z, NULL, 10) == 0) ? &FALSE_VALUE : &TRUE_VALUE), + pSchema->bytes, param); + } else if (pToken->type == TK_NK_FLOAT) { + return func(&pCxt->msg, ((taosStr2Double(pToken->z, NULL) == 0) ? &FALSE_VALUE : &TRUE_VALUE), pSchema->bytes, + param); + } else { + return buildSyntaxErrMsg(&pCxt->msg, "invalid bool data", pToken->z); + } + } + + case TSDB_DATA_TYPE_TINYINT: { + if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv)) { + return buildSyntaxErrMsg(&pCxt->msg, "invalid tinyint data", pToken->z); + } else if (!IS_VALID_TINYINT(iv)) { + return buildSyntaxErrMsg(&pCxt->msg, "tinyint data overflow", pToken->z); } + uint8_t tmpVal = (uint8_t)iv; + return func(&pCxt->msg, &tmpVal, pSchema->bytes, param); + } + + case TSDB_DATA_TYPE_UTINYINT: { + if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &uv)) { + return buildSyntaxErrMsg(&pCxt->msg, "invalid unsigned tinyint data", pToken->z); + } else if (uv > UINT8_MAX) { + return buildSyntaxErrMsg(&pCxt->msg, "unsigned tinyint data overflow", pToken->z); + } + uint8_t tmpVal = (uint8_t)uv; + return func(&pCxt->msg, &tmpVal, pSchema->bytes, param); + } + + case TSDB_DATA_TYPE_SMALLINT: { + if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv)) { + return buildSyntaxErrMsg(&pCxt->msg, "invalid smallint data", pToken->z); + } else if (!IS_VALID_SMALLINT(iv)) { + return buildSyntaxErrMsg(&pCxt->msg, "smallint data overflow", pToken->z); + } + int16_t tmpVal = (int16_t)iv; + return func(&pCxt->msg, &tmpVal, pSchema->bytes, param); + } + + case TSDB_DATA_TYPE_USMALLINT: { + if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &uv)) { + return buildSyntaxErrMsg(&pCxt->msg, "invalid unsigned smallint data", pToken->z); + } else if (uv > UINT16_MAX) { + return buildSyntaxErrMsg(&pCxt->msg, "unsigned smallint data overflow", pToken->z); + } + uint16_t tmpVal = (uint16_t)uv; + return func(&pCxt->msg, &tmpVal, pSchema->bytes, param); + } + + case TSDB_DATA_TYPE_INT: { + if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv)) { + return buildSyntaxErrMsg(&pCxt->msg, "invalid int data", pToken->z); + } else if (!IS_VALID_INT(iv)) { + return buildSyntaxErrMsg(&pCxt->msg, "int data overflow", pToken->z); + } + int32_t tmpVal = (int32_t)iv; + return func(&pCxt->msg, &tmpVal, pSchema->bytes, param); + } + + case TSDB_DATA_TYPE_UINT: { + if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &uv)) { + return buildSyntaxErrMsg(&pCxt->msg, "invalid unsigned int data", pToken->z); + } else if (uv > UINT32_MAX) { + return buildSyntaxErrMsg(&pCxt->msg, "unsigned int data overflow", pToken->z); + } + uint32_t tmpVal = (uint32_t)uv; + return func(&pCxt->msg, &tmpVal, pSchema->bytes, param); + } + + case TSDB_DATA_TYPE_BIGINT: { + if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv)) { + return buildSyntaxErrMsg(&pCxt->msg, "invalid bigint data", pToken->z); + } + return func(&pCxt->msg, &iv, pSchema->bytes, param); + } + + case TSDB_DATA_TYPE_UBIGINT: { + if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &uv)) { + return buildSyntaxErrMsg(&pCxt->msg, "invalid unsigned bigint data", pToken->z); + } + return func(&pCxt->msg, &uv, pSchema->bytes, param); + } + + case TSDB_DATA_TYPE_FLOAT: { + double dv; + if (TK_NK_ILLEGAL == toDouble(pToken, &dv, &endptr)) { + return buildSyntaxErrMsg(&pCxt->msg, "illegal float data", pToken->z); + } + if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || dv > FLT_MAX || dv < -FLT_MAX || isinf(dv) || + isnan(dv)) { + return buildSyntaxErrMsg(&pCxt->msg, "illegal float data", pToken->z); + } + float tmpVal = (float)dv; + return func(&pCxt->msg, &tmpVal, pSchema->bytes, param); + } + + case TSDB_DATA_TYPE_DOUBLE: { + double dv; + if (TK_NK_ILLEGAL == toDouble(pToken, &dv, &endptr)) { + return buildSyntaxErrMsg(&pCxt->msg, "illegal double data", pToken->z); + } + if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || isinf(dv) || isnan(dv)) { + return buildSyntaxErrMsg(&pCxt->msg, "illegal double data", pToken->z); + } + return func(&pCxt->msg, &dv, pSchema->bytes, param); + } + + case TSDB_DATA_TYPE_BINARY: { + // Too long values will raise the invalid sql error message + if (pToken->n + VARSTR_HEADER_SIZE > pSchema->bytes) { + return generateSyntaxErrMsg(&pCxt->msg, TSDB_CODE_PAR_VALUE_TOO_LONG, pSchema->name); + } + + return func(&pCxt->msg, pToken->z, pToken->n, param); + } + + case TSDB_DATA_TYPE_NCHAR: { + return func(&pCxt->msg, pToken->z, pToken->n, param); + } + case TSDB_DATA_TYPE_JSON: { + if (pToken->n > (TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { + return buildSyntaxErrMsg(&pCxt->msg, "json string too long than 4095", pToken->z); + } + return func(&pCxt->msg, pToken->z, pToken->n, param); + } + case TSDB_DATA_TYPE_TIMESTAMP: { + int64_t tmpVal; + if (parseTime(pSql, pToken, timePrec, &tmpVal, &pCxt->msg) != TSDB_CODE_SUCCESS) { + return buildSyntaxErrMsg(&pCxt->msg, "invalid timestamp", pToken->z); + } + + return func(&pCxt->msg, &tmpVal, pSchema->bytes, param); + } + } + + return TSDB_CODE_FAILED; +} + +static int32_t parseValueToken(SInsertParseContext* pCxt, const char** pSql, SToken* pToken, SSchema* pSchema, + int16_t timePrec, _row_append_fn_t func, void* param) { + int32_t code = checkAndTrimValue(pToken, pCxt->tmpTokenBuf, &pCxt->msg); + if (TSDB_CODE_SUCCESS == code && isNullValue(pSchema->type, pToken)) { + if (TSDB_DATA_TYPE_TIMESTAMP == pSchema->type && PRIMARYKEY_TIMESTAMP_COL_ID == pSchema->colId) { + return buildSyntaxErrMsg(&pCxt->msg, "primary timestamp should not be null", pToken->z); + } + + return func(&pCxt->msg, NULL, 0, param); + } + + if (TSDB_CODE_SUCCESS == code && IS_NUMERIC_TYPE(pSchema->type) && pToken->n == 0) { + return buildSyntaxErrMsg(&pCxt->msg, "invalid numeric data", pToken->z); + } + + if (TSDB_CODE_SUCCESS == code) { + code = parseValueTokenImpl(pCxt, pSql, pToken, pSchema, timePrec, func, param); + } + + return code; +} + +static int parseOneRow(SInsertParseContext* pCxt, const char** pSql, STableDataBlocks* pDataBuf, bool* pGotRow, + SToken* pToken) { + SRowBuilder* pBuilder = &pDataBuf->rowBuilder; + STSRow* row = (STSRow*)(pDataBuf->pData + pDataBuf->size); // skip the SSubmitBlk header + SParsedDataColInfo* pCols = &pDataBuf->boundColumnInfo; + bool isParseBindParam = false; + SSchema* pSchemas = getTableColumnSchema(pDataBuf->pTableMeta); + SMemParam param = {.rb = pBuilder}; + + int32_t code = tdSRowResetBuf(pBuilder, row); + // 1. set the parsed value from sql string + for (int i = 0; i < pCols->numOfBound && TSDB_CODE_SUCCESS == code; ++i) { + NEXT_TOKEN_WITH_PREV(*pSql, *pToken); + SSchema* pSchema = &pSchemas[pCols->boundColumns[i]]; + + if (pToken->type == TK_NK_QUESTION) { + isParseBindParam = true; + if (NULL == pCxt->pComCxt->pStmtCb) { + code = buildSyntaxErrMsg(&pCxt->msg, "? only used in stmt", pToken->z); + } continue; } - if (TK_NK_RP == sToken.type) { - return generateSyntaxErrMsg(&pCxt->msg, TSDB_CODE_PAR_INVALID_COLUMNS_NUM); + if (TSDB_CODE_SUCCESS == code && TK_NK_RP == pToken->type) { + code = generateSyntaxErrMsg(&pCxt->msg, TSDB_CODE_PAR_INVALID_COLUMNS_NUM); } - if (isParseBindParam) { - return buildInvalidOperationMsg(&pCxt->msg, "no mix usage for ? and values"); + if (TSDB_CODE_SUCCESS == code && isParseBindParam) { + code = buildInvalidOperationMsg(&pCxt->msg, "no mix usage for ? and values"); } - param.schema = pSchema; - insGetSTSRowAppendInfo(pBuilder->rowType, spd, i, ¶m.toffset, ¶m.colIdx); - CHECK_CODE( - parseValueToken(&pCxt->pSql, &sToken, pSchema, timePrec, tmpTokenBuf, insMemRowAppend, ¶m, &pCxt->msg)); + if (TSDB_CODE_SUCCESS == code) { + param.schema = pSchema; + insGetSTSRowAppendInfo(pBuilder->rowType, pCols, i, ¶m.toffset, ¶m.colIdx); + code = parseValueToken(pCxt, pSql, pToken, pSchema, getTableInfo(pDataBuf->pTableMeta).precision, insMemRowAppend, + ¶m); + } - if (i < spd->numOfBound - 1) { - NEXT_VALID_TOKEN(pCxt->pSql, sToken); - if (TK_NK_COMMA != sToken.type) { - return buildSyntaxErrMsg(&pCxt->msg, ", expected", sToken.z); + if (TSDB_CODE_SUCCESS == code && i < pCols->numOfBound - 1) { + NEXT_VALID_TOKEN(*pSql, *pToken); + if (TK_NK_COMMA != pToken->type) { + code = buildSyntaxErrMsg(&pCxt->msg, ", expected", pToken->z); } } } - TSKEY tsKey = TD_ROW_KEY(row); - insCheckTimestamp(pDataBlocks, (const char*)&tsKey); + if (TSDB_CODE_SUCCESS == code) { + TSKEY tsKey = TD_ROW_KEY(row); + code = insCheckTimestamp(pDataBuf, (const char*)&tsKey); + } - if (!isParseBindParam) { + if (TSDB_CODE_SUCCESS == code && !isParseBindParam) { // set the null value for the columns that do not assign values - if ((spd->numOfBound < spd->numOfCols) && TD_IS_TP_ROW(row)) { + if ((pCols->numOfBound < pCols->numOfCols) && TD_IS_TP_ROW(row)) { pBuilder->hasNone = true; } tdSRowEnd(pBuilder); - *gotRow = true; + *pGotRow = true; #ifdef TD_DEBUG_PRINT_ROW STSchema* pSTSchema = tdGetSTSChemaFromSSChema(schema, spd->numOfCols, 1); @@ -981,8 +1224,7 @@ static int parseOneRow(SInsertParseContext* pCxt, STableDataBlocks* pDataBlocks, #endif } - // *len = pBuilder->extendedRowSize; - return TSDB_CODE_SUCCESS; + return code; } static int32_t allocateMemIfNeed(STableDataBlocks* pDataBlock, int32_t rowSize, int32_t* numOfRows) { @@ -1014,84 +1256,81 @@ static int32_t allocateMemIfNeed(STableDataBlocks* pDataBlock, int32_t rowSize, } // pSql -> (field1_value, ...) [(field1_value2, ...) ...] -static int32_t parseValues(SInsertParseContext* pCxt, STableDataBlocks* pDataBlock, int maxRows, int32_t* numOfRows) { - STableComInfo tinfo = getTableInfo(pDataBlock->pTableMeta); - int32_t extendedRowSize = insGetExtendedRowSize(pDataBlock); - CHECK_CODE( - insInitRowBuilder(&pDataBlock->rowBuilder, pDataBlock->pTableMeta->sversion, &pDataBlock->boundColumnInfo)); +static int32_t parseValues(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, STableDataBlocks* pDataBuf, + int32_t maxRows, int32_t* pNumOfRows, SToken* pToken) { + int32_t code = insInitRowBuilder(&pDataBuf->rowBuilder, pDataBuf->pTableMeta->sversion, &pDataBuf->boundColumnInfo); - (*numOfRows) = 0; - // char tmpTokenBuf[TSDB_MAX_BYTES_PER_ROW] = {0}; // used for deleting Escape character: \\, \', \" - SToken sToken; - while (1) { + int32_t extendedRowSize = insGetExtendedRowSize(pDataBuf); + (*pNumOfRows) = 0; + while (TSDB_CODE_SUCCESS == code) { int32_t index = 0; - NEXT_TOKEN_KEEP_SQL(pCxt->pSql, sToken, index); - if (TK_NK_LP != sToken.type) { + NEXT_TOKEN_KEEP_SQL(pStmt->pSql, *pToken, index); + if (TK_NK_LP != pToken->type) { break; } - pCxt->pSql += index; + pStmt->pSql += index; - if ((*numOfRows) >= maxRows || pDataBlock->size + extendedRowSize >= pDataBlock->nAllocSize) { - int32_t tSize; - CHECK_CODE(allocateMemIfNeed(pDataBlock, extendedRowSize, &tSize)); - ASSERT(tSize >= maxRows); - maxRows = tSize; + if ((*pNumOfRows) >= maxRows || pDataBuf->size + extendedRowSize >= pDataBuf->nAllocSize) { + code = allocateMemIfNeed(pDataBuf, extendedRowSize, &maxRows); } bool gotRow = false; - CHECK_CODE(parseOneRow(pCxt, pDataBlock, tinfo.precision, &gotRow, pCxt->tmpTokenBuf)); - if (gotRow) { - pDataBlock->size += extendedRowSize; // len; + if (TSDB_CODE_SUCCESS == code) { + code = parseOneRow(pCxt, &pStmt->pSql, pDataBuf, &gotRow, pToken); } - NEXT_VALID_TOKEN(pCxt->pSql, sToken); - if (TK_NK_COMMA == sToken.type) { - return generateSyntaxErrMsg(&pCxt->msg, TSDB_CODE_PAR_INVALID_COLUMNS_NUM); - } else if (TK_NK_RP != sToken.type) { - return buildSyntaxErrMsg(&pCxt->msg, ") expected", sToken.z); + if (TSDB_CODE_SUCCESS == code) { + NEXT_VALID_TOKEN(pStmt->pSql, *pToken); + if (TK_NK_COMMA == pToken->type) { + code = generateSyntaxErrMsg(&pCxt->msg, TSDB_CODE_PAR_INVALID_COLUMNS_NUM); + } else if (TK_NK_RP != pToken->type) { + code = buildSyntaxErrMsg(&pCxt->msg, ") expected", pToken->z); + } } - if (gotRow) { - (*numOfRows)++; + if (TSDB_CODE_SUCCESS == code && gotRow) { + pDataBuf->size += extendedRowSize; + (*pNumOfRows)++; } } - if (0 == (*numOfRows) && (!TSDB_QUERY_HAS_TYPE(pCxt->pOutput->insertType, TSDB_QUERY_TYPE_STMT_INSERT))) { - return buildSyntaxErrMsg(&pCxt->msg, "no any data points", NULL); + if (TSDB_CODE_SUCCESS == code && 0 == (*pNumOfRows) && + (!TSDB_QUERY_HAS_TYPE(pStmt->insertType, TSDB_QUERY_TYPE_STMT_INSERT))) { + code = buildSyntaxErrMsg(&pCxt->msg, "no any data points", NULL); } - return TSDB_CODE_SUCCESS; + return code; } -static int32_t parseValuesClause(SInsertParseContext* pCxt, STableDataBlocks* dataBuf) { - int32_t maxNumOfRows; - CHECK_CODE(allocateMemIfNeed(dataBuf, insGetExtendedRowSize(dataBuf), &maxNumOfRows)); - +// VALUES (field1_value, ...) [(field1_value2, ...) ...] +static int32_t parseValuesClause(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, STableDataBlocks* pDataBuf, + SToken* pToken) { + int32_t maxNumOfRows = 0; int32_t numOfRows = 0; - CHECK_CODE(parseValues(pCxt, dataBuf, maxNumOfRows, &numOfRows)); - - SSubmitBlk* pBlocks = (SSubmitBlk*)(dataBuf->pData); - if (TSDB_CODE_SUCCESS != insSetBlockInfo(pBlocks, dataBuf, numOfRows)) { - return buildInvalidOperationMsg(&pCxt->msg, - "too many rows in sql, total number of rows should be less than INT32_MAX"); + int32_t code = allocateMemIfNeed(pDataBuf, insGetExtendedRowSize(pDataBuf), &maxNumOfRows); + if (TSDB_CODE_SUCCESS == code) { + code = parseValues(pCxt, pStmt, pDataBuf, maxNumOfRows, &numOfRows, pToken); } - - dataBuf->numOfTables = 1; - pCxt->totalNum += numOfRows; - return TSDB_CODE_SUCCESS; + if (TSDB_CODE_SUCCESS == code) { + code = insSetBlockInfo((SSubmitBlk*)(pDataBuf->pData), pDataBuf, numOfRows, &pCxt->msg); + } + if (TSDB_CODE_SUCCESS == code) { + pDataBuf->numOfTables = 1; + pStmt->totalRowsNum += numOfRows; + pStmt->totalTbNum += 1; + TSDB_QUERY_SET_TYPE(pStmt->insertType, TSDB_QUERY_TYPE_INSERT); + } + return code; } -static int32_t parseCsvFile(SInsertParseContext* pCxt, TdFilePtr fp, STableDataBlocks* pDataBlock, int maxRows, - int32_t* numOfRows) { - STableComInfo tinfo = getTableInfo(pDataBlock->pTableMeta); - int32_t extendedRowSize = insGetExtendedRowSize(pDataBlock); - CHECK_CODE( - insInitRowBuilder(&pDataBlock->rowBuilder, pDataBlock->pTableMeta->sversion, &pDataBlock->boundColumnInfo)); +static int32_t parseCsvFile(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, STableDataBlocks* pDataBuf, + int maxRows, int32_t* pNumOfRows) { + int32_t code = insInitRowBuilder(&pDataBuf->rowBuilder, pDataBuf->pTableMeta->sversion, &pDataBuf->boundColumnInfo); - (*numOfRows) = 0; - char tmpTokenBuf[TSDB_MAX_BYTES_PER_ROW] = {0}; // used for deleting Escape character: \\, \', \" + int32_t extendedRowSize = insGetExtendedRowSize(pDataBuf); + (*pNumOfRows) = 0; char* pLine = NULL; int64_t readLen = 0; - while ((readLen = taosGetLineFile(fp, &pLine)) != -1) { + while (TSDB_CODE_SUCCESS == code && (readLen = taosGetLineFile(pStmt->fp, &pLine)) != -1) { if (('\r' == pLine[readLen - 1]) || ('\n' == pLine[readLen - 1])) { pLine[--readLen] = '\0'; } @@ -1100,588 +1339,516 @@ static int32_t parseCsvFile(SInsertParseContext* pCxt, TdFilePtr fp, STableDataB continue; } - if ((*numOfRows) >= maxRows || pDataBlock->size + extendedRowSize >= pDataBlock->nAllocSize) { - int32_t tSize; - CHECK_CODE(allocateMemIfNeed(pDataBlock, extendedRowSize, &tSize)); - ASSERT(tSize >= maxRows); - maxRows = tSize; + if ((*pNumOfRows) >= maxRows || pDataBuf->size + extendedRowSize >= pDataBuf->nAllocSize) { + code = allocateMemIfNeed(pDataBuf, extendedRowSize, &maxRows); } - strtolower(pLine, pLine); - char* pRawSql = pCxt->pSql; - pCxt->pSql = pLine; - bool gotRow = false; - int32_t code = parseOneRow(pCxt, pDataBlock, tinfo.precision, &gotRow, tmpTokenBuf); - if (TSDB_CODE_SUCCESS != code) { - pCxt->pSql = pRawSql; - return code; + bool gotRow = false; + if (TSDB_CODE_SUCCESS == code) { + SToken token; + strtolower(pLine, pLine); + code = parseOneRow(pCxt, (const char**)&pLine, pDataBuf, &gotRow, &token); } - if (gotRow) { - pDataBlock->size += extendedRowSize; // len; - (*numOfRows)++; - } - pCxt->pSql = pRawSql; - if (pDataBlock->nAllocSize > tsMaxMemUsedByInsert * 1024 * 1024) { + if (TSDB_CODE_SUCCESS == code && gotRow) { + pDataBuf->size += extendedRowSize; + (*pNumOfRows)++; + } + + if (TSDB_CODE_SUCCESS == code && pDataBuf->nAllocSize > tsMaxMemUsedByInsert * 1024 * 1024) { break; } } - if (0 == (*numOfRows) && (!TSDB_QUERY_HAS_TYPE(pCxt->pOutput->insertType, TSDB_QUERY_TYPE_STMT_INSERT))) { - return buildSyntaxErrMsg(&pCxt->msg, "no any data points", NULL); + if (TSDB_CODE_SUCCESS == code && 0 == (*pNumOfRows) && + (!TSDB_QUERY_HAS_TYPE(pStmt->insertType, TSDB_QUERY_TYPE_STMT_INSERT))) { + code = buildSyntaxErrMsg(&pCxt->msg, "no any data points", NULL); } - return TSDB_CODE_SUCCESS; + return code; } -static int32_t parseDataFromFileAgain(SInsertParseContext* pCxt, int16_t tableNo, const SName* pTableName, - STableDataBlocks* dataBuf) { - int32_t maxNumOfRows; - CHECK_CODE(allocateMemIfNeed(dataBuf, insGetExtendedRowSize(dataBuf), &maxNumOfRows)); - +static int32_t parseDataFromFileImpl(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, STableDataBlocks* pDataBuf) { + int32_t maxNumOfRows = 0; int32_t numOfRows = 0; - CHECK_CODE(parseCsvFile(pCxt, pCxt->pComCxt->csvCxt.fp, dataBuf, maxNumOfRows, &numOfRows)); - - SSubmitBlk* pBlocks = (SSubmitBlk*)(dataBuf->pData); - if (TSDB_CODE_SUCCESS != insSetBlockInfo(pBlocks, dataBuf, numOfRows)) { - return buildInvalidOperationMsg(&pCxt->msg, - "too many rows in sql, total number of rows should be less than INT32_MAX"); + int32_t code = allocateMemIfNeed(pDataBuf, insGetExtendedRowSize(pDataBuf), &maxNumOfRows); + if (TSDB_CODE_SUCCESS == code) { + code = parseCsvFile(pCxt, pStmt, pDataBuf, maxNumOfRows, &numOfRows); } - - if (!taosEOFFile(pCxt->pComCxt->csvCxt.fp)) { - pCxt->pComCxt->needMultiParse = true; - pCxt->pComCxt->csvCxt.tableNo = tableNo; - memcpy(&pCxt->pComCxt->csvCxt.tableName, pTableName, sizeof(SName)); - pCxt->pComCxt->csvCxt.pLastSqlPos = pCxt->pSql; + if (TSDB_CODE_SUCCESS == code) { + code = insSetBlockInfo((SSubmitBlk*)(pDataBuf->pData), pDataBuf, numOfRows, &pCxt->msg); + } + if (TSDB_CODE_SUCCESS == code) { + pDataBuf->numOfTables = 1; + pStmt->totalRowsNum += numOfRows; + pStmt->totalTbNum += 1; + TSDB_QUERY_SET_TYPE(pStmt->insertType, TSDB_QUERY_TYPE_FILE_INSERT); + if (taosEOFFile(pStmt->fp)) { + taosCloseFile(&pStmt->fp); + } else { + parserDebug("0x%" PRIx64 " insert from csv. File is too large, do it in batches.", pCxt->pComCxt->requestId); + } } - - dataBuf->numOfTables = 1; - pCxt->totalNum += numOfRows; return TSDB_CODE_SUCCESS; } -static int32_t parseDataFromFile(SInsertParseContext* pCxt, int16_t tableNo, const SName* pTableName, SToken filePath, - STableDataBlocks* dataBuf) { +static int32_t parseDataFromFile(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, SToken* pFilePath, + STableDataBlocks* pDataBuf) { char filePathStr[TSDB_FILENAME_LEN] = {0}; - if (TK_NK_STRING == filePath.type) { - trimString(filePath.z, filePath.n, filePathStr, sizeof(filePathStr)); + if (TK_NK_STRING == pFilePath->type) { + trimString(pFilePath->z, pFilePath->n, filePathStr, sizeof(filePathStr)); } else { - strncpy(filePathStr, filePath.z, filePath.n); + strncpy(filePathStr, pFilePath->z, pFilePath->n); } - pCxt->pComCxt->csvCxt.fp = taosOpenFile(filePathStr, TD_FILE_READ | TD_FILE_STREAM); - if (NULL == pCxt->pComCxt->csvCxt.fp) { + pStmt->fp = taosOpenFile(filePathStr, TD_FILE_READ | TD_FILE_STREAM); + if (NULL == pStmt->fp) { return TAOS_SYSTEM_ERROR(errno); } - return parseDataFromFileAgain(pCxt, tableNo, pTableName, dataBuf); + return parseDataFromFileImpl(pCxt, pStmt, pDataBuf); } -static void destroyInsertParseContextForTable(SInsertParseContext* pCxt) { - if (!pCxt->pComCxt->needMultiParse) { - taosCloseFile(&pCxt->pComCxt->csvCxt.fp); +static int32_t parseFileClause(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, STableDataBlocks* pDataBuf, + SToken* pToken) { + NEXT_TOKEN(pStmt->pSql, *pToken); + if (0 == pToken->n || (TK_NK_STRING != pToken->type && TK_NK_ID != pToken->type)) { + return buildSyntaxErrMsg(&pCxt->msg, "file path is required following keyword FILE", pToken->z); } - taosMemoryFreeClear(pCxt->pTableMeta); + return parseDataFromFile(pCxt, pStmt, pToken, pDataBuf); +} + +// VALUES (field1_value, ...) [(field1_value2, ...) ...] | FILE csv_file_path +static int32_t parseDataClause(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, STableDataBlocks* pDataBuf) { + SToken token; + NEXT_TOKEN(pStmt->pSql, token); + switch (token.type) { + case TK_VALUES: + return parseValuesClause(pCxt, pStmt, pDataBuf, &token); + case TK_FILE: + return parseFileClause(pCxt, pStmt, pDataBuf, &token); + default: + break; + } + return buildSyntaxErrMsg(&pCxt->msg, "keyword VALUES or FILE is expected", token.z); +} + +// input pStmt->pSql: +// 1. [(tag1_name, ...)] ... +// 2. VALUES ... | FILE ... +static int32_t parseInsertTableClauseBottom(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { + STableDataBlocks* pDataBuf = NULL; + int32_t code = parseSchemaClauseBottom(pCxt, pStmt, &pDataBuf); + if (TSDB_CODE_SUCCESS == code) { + code = parseDataClause(pCxt, pStmt, pDataBuf); + } + return code; +} + +// input pStmt->pSql: [(field1_name, ...)] [ USING ... ] VALUES ... | FILE ... +static int32_t parseInsertTableClause(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, SToken* pTbName) { + int32_t code = parseSchemaClauseTop(pCxt, pStmt, pTbName); + if (TSDB_CODE_SUCCESS == code && !pCxt->missCache) { + code = parseInsertTableClauseBottom(pCxt, pStmt); + } + return code; +} + +static int32_t checkTableClauseFirstToken(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, SToken* pTbName, + bool* pHasData) { + // no data in the sql string anymore. + if (0 == pTbName->n) { + if (0 != pTbName->type && '\0' != pStmt->pSql[0]) { + return buildSyntaxErrMsg(&pCxt->msg, "invalid charactor in SQL", pTbName->z); + } + + if (0 == pStmt->totalRowsNum && (!TSDB_QUERY_HAS_TYPE(pStmt->insertType, TSDB_QUERY_TYPE_STMT_INSERT))) { + return buildInvalidOperationMsg(&pCxt->msg, "no data in sql"); + } + + *pHasData = false; + return TSDB_CODE_SUCCESS; + } + + if (TSDB_QUERY_HAS_TYPE(pStmt->insertType, TSDB_QUERY_TYPE_STMT_INSERT) && pStmt->totalTbNum > 0) { + return buildInvalidOperationMsg(&pCxt->msg, "single table allowed in one stmt"); + } + + if (TK_NK_QUESTION == pTbName->type) { + if (NULL == pCxt->pComCxt->pStmtCb) { + return buildSyntaxErrMsg(&pCxt->msg, "? only used in stmt", pTbName->z); + } + + char* tbName = NULL; + int32_t code = (*pCxt->pComCxt->pStmtCb->getTbNameFn)(pCxt->pComCxt->pStmtCb->pStmt, &tbName); + if (TSDB_CODE_SUCCESS == code) { + pTbName->z = tbName; + pTbName->n = strlen(tbName); + } else { + return code; + } + } + + *pHasData = true; + return TSDB_CODE_SUCCESS; +} + +static int32_t setStmtInfo(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { + SParsedDataColInfo* tags = taosMemoryMalloc(sizeof(pCxt->tags)); + if (NULL == tags) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + memcpy(tags, &pCxt->tags, sizeof(pCxt->tags)); + + SStmtCallback* pStmtCb = pCxt->pComCxt->pStmtCb; + char tbFName[TSDB_TABLE_FNAME_LEN]; + tNameExtractFullName(&pStmt->targetTableName, tbFName); + char stbFName[TSDB_TABLE_FNAME_LEN]; + tNameExtractFullName(&pStmt->usingTableName, stbFName); + int32_t code = + (*pStmtCb->setInfoFn)(pStmtCb->pStmt, pStmt->pTableMeta, tags, tbFName, '\0' != pStmt->usingTableName.tname[0], + pStmt->pVgroupsHashObj, pStmt->pTableBlockHashObj, stbFName); + + memset(&pCxt->tags, 0, sizeof(pCxt->tags)); + pStmt->pVgroupsHashObj = NULL; + pStmt->pTableBlockHashObj = NULL; + return code; +} + +static int32_t parseInsertBodyBottom(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { + if (TSDB_QUERY_HAS_TYPE(pStmt->insertType, TSDB_QUERY_TYPE_STMT_INSERT)) { + return setStmtInfo(pCxt, pStmt); + } + + // merge according to vgId + int32_t code = TSDB_CODE_SUCCESS; + if (taosHashGetSize(pStmt->pTableBlockHashObj) > 0) { + code = insMergeTableDataBlocks(pStmt->pTableBlockHashObj, &pStmt->pVgDataBlocks); + } + if (TSDB_CODE_SUCCESS == code) { + code = insBuildOutput(pStmt); + } + return code; +} + +static void destroyEnvPreTable(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { destroyBoundColumnInfo(&pCxt->tags); - tdDestroySVCreateTbReq(&pCxt->createTblReq); + taosMemoryFreeClear(pStmt->pTableMeta); + tdDestroySVCreateTbReq(&pStmt->createTblReq); +} + +// tb_name +// [USING stb_name [(tag1_name, ...)] TAGS (tag1_value, ...)] +// [(field1_name, ...)] +// VALUES (field1_value, ...) [(field1_value2, ...) ...] | FILE csv_file_path +// [...]; +static int32_t parseInsertBody(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { + SToken token; + int32_t code = TSDB_CODE_SUCCESS; + bool hasData = true; + // for each table + while (TSDB_CODE_SUCCESS == code && hasData && !pCxt->missCache) { + destroyEnvPreTable(pCxt, pStmt); + // pStmt->pSql -> tb_name ... + NEXT_TOKEN(pStmt->pSql, token); + code = checkTableClauseFirstToken(pCxt, pStmt, &token, &hasData); + if (TSDB_CODE_SUCCESS == code && hasData) { + code = parseInsertTableClause(pCxt, pStmt, &token); + } + } + + parserDebug("0x%" PRIx64 " insert input rows: %d", pCxt->pComCxt->requestId, pStmt->totalRowsNum); + + if (TSDB_CODE_SUCCESS == code && !pCxt->missCache) { + code = parseInsertBodyBottom(pCxt, pStmt); + } + return code; } static void destroySubTableHashElem(void* p) { taosMemoryFree(*(STableMeta**)p); } -static void destroyInsertParseContext(SInsertParseContext* pCxt) { - destroyInsertParseContextForTable(pCxt); - taosHashCleanup(pCxt->pVgroupsHashObj); - taosHashCleanup(pCxt->pSubTableHashObj); - taosHashCleanup(pCxt->pTableNameHashObj); - taosHashCleanup(pCxt->pDbFNameHashObj); +static int32_t createVnodeModifOpStmt(SParseContext* pCxt, SNode** pOutput) { + SVnodeModifOpStmt* pStmt = (SVnodeModifOpStmt*)nodesMakeNode(QUERY_NODE_VNODE_MODIF_STMT); + if (NULL == pStmt) { + return TSDB_CODE_OUT_OF_MEMORY; + } - insDestroyBlockHashmap(pCxt->pTableBlockHashObj); - insDestroyBlockArrayList(pCxt->pVgDataBlocks); + if (pCxt->pStmtCb) { + TSDB_QUERY_SET_TYPE(pStmt->insertType, TSDB_QUERY_TYPE_STMT_INSERT); + } + pStmt->pSql = pCxt->pSql; + pStmt->freeHashFunc = insDestroyBlockHashmap; + pStmt->freeArrayFunc = insDestroyBlockArrayList; + + pStmt->pVgroupsHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); + pStmt->pTableBlockHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); + pStmt->pSubTableHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), true, HASH_NO_LOCK); + pStmt->pTableNameHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), true, HASH_NO_LOCK); + pStmt->pDbFNameHashObj = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), true, HASH_NO_LOCK); + if (NULL == pStmt->pVgroupsHashObj || NULL == pStmt->pTableBlockHashObj || NULL == pStmt->pSubTableHashObj || + NULL == pStmt->pTableNameHashObj || NULL == pStmt->pDbFNameHashObj) { + nodesDestroyNode((SNode*)pStmt); + return TSDB_CODE_OUT_OF_MEMORY; + } + + taosHashSetFreeFp(pStmt->pSubTableHashObj, destroySubTableHashElem); + + *pOutput = (SNode*)pStmt; + return TSDB_CODE_SUCCESS; } -static int32_t parseTableName(SInsertParseContext* pCxt, SToken* pTbnameToken, SName* pName, char* pDbFName, - char* pTbFName) { - int32_t code = insCreateSName(pName, pTbnameToken, pCxt->pComCxt->acctId, pCxt->pComCxt->db, &pCxt->msg); - if (TSDB_CODE_SUCCESS == code) { - tNameExtractFullName(pName, pTbFName); - code = taosHashPut(pCxt->pTableNameHashObj, pTbFName, strlen(pTbFName), pName, sizeof(SName)); +static int32_t createInsertQuery(SParseContext* pCxt, SQuery** pOutput) { + SQuery* pQuery = (SQuery*)nodesMakeNode(QUERY_NODE_QUERY); + if (NULL == pQuery) { + return TSDB_CODE_OUT_OF_MEMORY; } + + pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE; + pQuery->haveResultSet = false; + pQuery->msgType = TDMT_VND_SUBMIT; + + int32_t code = createVnodeModifOpStmt(pCxt, &pQuery->pRoot); if (TSDB_CODE_SUCCESS == code) { - tNameGetFullDbName(pName, pDbFName); - code = taosHashPut(pCxt->pDbFNameHashObj, pDbFName, strlen(pDbFName), pDbFName, TSDB_DB_FNAME_LEN); + *pOutput = pQuery; + } else { + nodesDestroyNode((SNode*)pQuery); } return code; } -// tb_name -// [USING stb_name [(tag1_name, ...)] TAGS (tag1_value, ...)] -// [(field1_name, ...)] -// VALUES (field1_value, ...) [(field1_value2, ...) ...] | FILE csv_file_path -// [...]; -static int32_t parseInsertBody(SInsertParseContext* pCxt) { - int32_t tbNum = 0; - SName name; - char tbFName[TSDB_TABLE_FNAME_LEN]; - char dbFName[TSDB_DB_FNAME_LEN]; - bool autoCreateTbl = false; - - // for each table - while (1) { - SToken sToken; - char* tbName = NULL; - - // pSql -> tb_name ... - NEXT_TOKEN(pCxt->pSql, sToken); - - // no data in the sql string anymore. - if (sToken.n == 0) { - if (sToken.type && pCxt->pSql[0]) { - return buildSyntaxErrMsg(&pCxt->msg, "invalid charactor in SQL", sToken.z); - } - - if (0 == pCxt->totalNum && (!TSDB_QUERY_HAS_TYPE(pCxt->pOutput->insertType, TSDB_QUERY_TYPE_STMT_INSERT)) && - !pCxt->pComCxt->needMultiParse) { - return buildInvalidOperationMsg(&pCxt->msg, "no data in sql"); - } - break; - } - - if (TSDB_QUERY_HAS_TYPE(pCxt->pOutput->insertType, TSDB_QUERY_TYPE_STMT_INSERT) && tbNum > 0) { - return buildInvalidOperationMsg(&pCxt->msg, "single table allowed in one stmt"); - } - - destroyInsertParseContextForTable(pCxt); - - if (TK_NK_QUESTION == sToken.type) { - if (pCxt->pStmtCb) { - CHECK_CODE((*pCxt->pStmtCb->getTbNameFn)(pCxt->pStmtCb->pStmt, &tbName)); - - sToken.z = tbName; - sToken.n = strlen(tbName); - } else { - return buildSyntaxErrMsg(&pCxt->msg, "? only used in stmt", sToken.z); - } - } - - SToken tbnameToken = sToken; - NEXT_TOKEN(pCxt->pSql, sToken); - - if (!pCxt->pComCxt->async || TK_USING == sToken.type) { - CHECK_CODE(parseTableName(pCxt, &tbnameToken, &name, dbFName, tbFName)); - } - - bool existedUsing = false; - // USING clause - if (TK_USING == sToken.type) { - existedUsing = true; - CHECK_CODE(parseUsingClause(pCxt, tbNum, &name, tbFName)); - NEXT_TOKEN(pCxt->pSql, sToken); - autoCreateTbl = true; - } - - char* pBoundColsStart = NULL; - if (TK_NK_LP == sToken.type) { - // pSql -> field1_name, ...) - pBoundColsStart = pCxt->pSql; - CHECK_CODE(ignoreBoundColumns(pCxt)); - NEXT_TOKEN(pCxt->pSql, sToken); - } - - if (TK_USING == sToken.type) { - if (pCxt->pComCxt->async) { - CHECK_CODE(parseTableName(pCxt, &tbnameToken, &name, dbFName, tbFName)); - } - CHECK_CODE(parseUsingClause(pCxt, tbNum, &name, tbFName)); - NEXT_TOKEN(pCxt->pSql, sToken); - autoCreateTbl = true; - } else if (!existedUsing) { - CHECK_CODE(getTableMeta(pCxt, tbNum, &name)); - if (TSDB_SUPER_TABLE == pCxt->pTableMeta->tableType) { - return buildInvalidOperationMsg(&pCxt->msg, "insert data into super table is not supported"); - } - } - - STableDataBlocks* dataBuf = NULL; - if (pCxt->pComCxt->async) { - CHECK_CODE(insGetDataBlockFromList(pCxt->pTableBlockHashObj, &pCxt->pTableMeta->uid, - sizeof(pCxt->pTableMeta->uid), TSDB_DEFAULT_PAYLOAD_SIZE, sizeof(SSubmitBlk), - getTableInfo(pCxt->pTableMeta).rowSize, pCxt->pTableMeta, &dataBuf, NULL, - &pCxt->createTblReq)); - } else { - CHECK_CODE(insGetDataBlockFromList(pCxt->pTableBlockHashObj, tbFName, strlen(tbFName), TSDB_DEFAULT_PAYLOAD_SIZE, - sizeof(SSubmitBlk), getTableInfo(pCxt->pTableMeta).rowSize, pCxt->pTableMeta, - &dataBuf, NULL, &pCxt->createTblReq)); - } - - if (NULL != pBoundColsStart) { - char* pCurrPos = pCxt->pSql; - pCxt->pSql = pBoundColsStart; - CHECK_CODE(parseBoundColumns(pCxt, &dataBuf->boundColumnInfo, getTableColumnSchema(pCxt->pTableMeta))); - pCxt->pSql = pCurrPos; - } - - if (TK_VALUES == sToken.type) { - // pSql -> (field1_value, ...) [(field1_value2, ...) ...] - CHECK_CODE(parseValuesClause(pCxt, dataBuf)); - TSDB_QUERY_SET_TYPE(pCxt->pOutput->insertType, TSDB_QUERY_TYPE_INSERT); - - tbNum++; - continue; - } - - // FILE csv_file_path - if (TK_FILE == sToken.type) { - // pSql -> csv_file_path - NEXT_TOKEN(pCxt->pSql, sToken); - if (0 == sToken.n || (TK_NK_STRING != sToken.type && TK_NK_ID != sToken.type)) { - return buildSyntaxErrMsg(&pCxt->msg, "file path is required following keyword FILE", sToken.z); - } - CHECK_CODE(parseDataFromFile(pCxt, tbNum, &name, sToken, dataBuf)); - pCxt->pOutput->insertType = TSDB_QUERY_TYPE_FILE_INSERT; - - tbNum++; - if (!pCxt->pComCxt->needMultiParse) { - continue; - } else { - parserDebug("0x%" PRIx64 " insert from csv. File is too large, do it in batches.", pCxt->pComCxt->requestId); - break; - } - } - - return buildSyntaxErrMsg(&pCxt->msg, "keyword VALUES or FILE is expected", sToken.z); +static int32_t checkAuthFromMetaData(const SArray* pUsers) { + if (1 != taosArrayGetSize(pUsers)) { + return TSDB_CODE_FAILED; } - parserDebug("0x%" PRIx64 " insert input rows: %d", pCxt->pComCxt->requestId, pCxt->totalNum); - - if (TSDB_QUERY_HAS_TYPE(pCxt->pOutput->insertType, TSDB_QUERY_TYPE_STMT_INSERT)) { - SParsedDataColInfo* tags = taosMemoryMalloc(sizeof(pCxt->tags)); - if (NULL == tags) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - memcpy(tags, &pCxt->tags, sizeof(pCxt->tags)); - (*pCxt->pStmtCb->setInfoFn)(pCxt->pStmtCb->pStmt, pCxt->pTableMeta, tags, tbFName, autoCreateTbl, - pCxt->pVgroupsHashObj, pCxt->pTableBlockHashObj, pCxt->sTableName); - - memset(&pCxt->tags, 0, sizeof(pCxt->tags)); - pCxt->pVgroupsHashObj = NULL; - pCxt->pTableBlockHashObj = NULL; - - return TSDB_CODE_SUCCESS; + SMetaRes* pRes = taosArrayGet(pUsers, 0); + if (TSDB_CODE_SUCCESS == pRes->code) { + return (*(bool*)pRes->pRes) ? TSDB_CODE_SUCCESS : TSDB_CODE_PAR_PERMISSION_DENIED; } - - // merge according to vgId - if (taosHashGetSize(pCxt->pTableBlockHashObj) > 0) { - CHECK_CODE(insMergeTableDataBlocks(pCxt->pTableBlockHashObj, &pCxt->pVgDataBlocks)); - } - return insBuildOutput(pCxt); + return pRes->code; } -static int32_t parseInsertBodyAgain(SInsertParseContext* pCxt) { - STableDataBlocks* dataBuf = NULL; - CHECK_CODE(getTableMeta(pCxt, pCxt->pComCxt->csvCxt.tableNo, &pCxt->pComCxt->csvCxt.tableName)); - CHECK_CODE(insGetDataBlockFromList(pCxt->pTableBlockHashObj, &pCxt->pTableMeta->uid, sizeof(pCxt->pTableMeta->uid), - TSDB_DEFAULT_PAYLOAD_SIZE, sizeof(SSubmitBlk), - getTableInfo(pCxt->pTableMeta).rowSize, pCxt->pTableMeta, &dataBuf, NULL, - &pCxt->createTblReq)); - CHECK_CODE(parseDataFromFileAgain(pCxt, pCxt->pComCxt->csvCxt.tableNo, &pCxt->pComCxt->csvCxt.tableName, dataBuf)); - if (taosEOFFile(pCxt->pComCxt->csvCxt.fp)) { - CHECK_CODE(parseInsertBody(pCxt)); - pCxt->pComCxt->needMultiParse = false; - return TSDB_CODE_SUCCESS; +static int32_t getTableMetaFromMetaData(const SArray* pTables, STableMeta** pMeta) { + if (1 != taosArrayGetSize(pTables)) { + return TSDB_CODE_FAILED; } - parserDebug("0x%" PRIx64 " insert again input rows: %d", pCxt->pComCxt->requestId, pCxt->totalNum); - // merge according to vgId - if (taosHashGetSize(pCxt->pTableBlockHashObj) > 0) { - CHECK_CODE(insMergeTableDataBlocks(pCxt->pTableBlockHashObj, &pCxt->pVgDataBlocks)); + SMetaRes* pRes = taosArrayGet(pTables, 0); + if (TSDB_CODE_SUCCESS == pRes->code) { + *pMeta = tableMetaDup((const STableMeta*)pRes->pRes); + if (NULL == *pMeta) { + return TSDB_CODE_OUT_OF_MEMORY; + } } - return insBuildOutput(pCxt); + return pRes->code; +} + +static int32_t getTableVgroupFromMetaData(const SArray* pTables, SVnodeModifOpStmt* pStmt, bool isStb) { + if (1 != taosArrayGetSize(pTables)) { + return TSDB_CODE_FAILED; + } + + SMetaRes* pRes = taosArrayGet(pTables, 0); + if (TSDB_CODE_SUCCESS != pRes->code) { + return pRes->code; + } + + SVgroupInfo* pVg = pRes->pRes; + if (isStb) { + pStmt->pTableMeta->vgId = pVg->vgId; + } + return taosHashPut(pStmt->pVgroupsHashObj, (const char*)&pVg->vgId, sizeof(pVg->vgId), (char*)pVg, + sizeof(SVgroupInfo)); +} + +static int32_t getTableSchemaFromMetaData(const SMetaData* pMetaData, SVnodeModifOpStmt* pStmt, bool isStb) { + int32_t code = checkAuthFromMetaData(pMetaData->pUser); + if (TSDB_CODE_SUCCESS == code) { + code = getTableMetaFromMetaData(pMetaData->pTableMeta, &pStmt->pTableMeta); + } + if (TSDB_CODE_SUCCESS == code) { + code = getTableVgroupFromMetaData(pMetaData->pTableHash, pStmt, isStb); + } + return code; +} + +static int32_t setVnodeModifOpStmt(SParseContext* pCxt, const SMetaData* pMetaData, SVnodeModifOpStmt* pStmt) { + if (pCxt->pStmtCb) { + (*pCxt->pStmtCb->getExecInfoFn)(pCxt->pStmtCb->pStmt, &pStmt->pVgroupsHashObj, &pStmt->pTableBlockHashObj); + if (NULL == pStmt->pVgroupsHashObj) { + pStmt->pVgroupsHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); + } + if (NULL == pStmt->pTableBlockHashObj) { + pStmt->pTableBlockHashObj = + taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + } + + TSDB_QUERY_SET_TYPE(pStmt->insertType, TSDB_QUERY_TYPE_STMT_INSERT); + } + + if (pStmt->usingTableProcessing) { + return getTableSchemaFromMetaData(pMetaData, pStmt, true); + } + return getTableSchemaFromMetaData(pMetaData, pStmt, false); +} + +static int32_t initInsertQuery(SParseContext* pCxt, const SMetaData* pMetaData, SQuery** pQuery) { + if (NULL == *pQuery) { + return createInsertQuery(pCxt, pQuery); + } + return setVnodeModifOpStmt(pCxt, pMetaData, (SVnodeModifOpStmt*)(*pQuery)->pRoot); +} + +static int32_t setRefreshMate(SQuery* pQuery) { + SVnodeModifOpStmt* pStmt = (SVnodeModifOpStmt*)pQuery->pRoot; + SName* pTable = taosHashIterate(pStmt->pTableNameHashObj, NULL); + while (NULL != pTable) { + taosArrayPush(pQuery->pTableList, pTable); + pTable = taosHashIterate(pStmt->pTableNameHashObj, pTable); + } + + char* pDb = taosHashIterate(pStmt->pDbFNameHashObj, NULL); + while (NULL != pDb) { + taosArrayPush(pQuery->pDbList, pDb); + pDb = taosHashIterate(pStmt->pDbFNameHashObj, pDb); + } + + return TSDB_CODE_SUCCESS; } // INSERT INTO // tb_name -// [USING stb_name [(tag1_name, ...)] TAGS (tag1_value, ...)] +// [USING stb_name [(tag1_name, ...)] TAGS (tag1_value, ...) [table_options]] // [(field1_name, ...)] // VALUES (field1_value, ...) [(field1_value2, ...) ...] | FILE csv_file_path // [...]; -int32_t parseInsertSql(SParseContext* pContext, SQuery** pQuery, SParseMetaCache* pMetaCache) { - SInsertParseContext context = { - .pComCxt = pContext, - .pSql = pContext->needMultiParse ? (char*)pContext->csvCxt.pLastSqlPos : (char*)pContext->pSql, - .msg = {.buf = pContext->pMsg, .len = pContext->msgLen}, - .pTableMeta = NULL, - .createTblReq = {0}, - .pSubTableHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), true, HASH_NO_LOCK), - .pTableNameHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), true, HASH_NO_LOCK), - .pDbFNameHashObj = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), true, HASH_NO_LOCK), - .totalNum = 0, - .pOutput = (SVnodeModifOpStmt*)nodesMakeNode(QUERY_NODE_VNODE_MODIF_STMT), - .pStmtCb = pContext->pStmtCb, - .pMetaCache = pMetaCache, - .memElapsed = 0, - .parRowElapsed = 0}; - - if (pContext->pStmtCb && *pQuery) { - (*pContext->pStmtCb->getExecInfoFn)(pContext->pStmtCb->pStmt, &context.pVgroupsHashObj, - &context.pTableBlockHashObj); - if (NULL == context.pVgroupsHashObj) { - context.pVgroupsHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); - } - if (NULL == context.pTableBlockHashObj) { - context.pTableBlockHashObj = - taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); - } - } else { - context.pVgroupsHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); - context.pTableBlockHashObj = - taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); +static int32_t parseInsertSqlFromStart(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { + int32_t code = skipInsertInto(&pStmt->pSql, &pCxt->msg); + if (TSDB_CODE_SUCCESS == code) { + code = parseInsertBody(pCxt, pStmt); } - - if (NULL == context.pVgroupsHashObj || NULL == context.pTableBlockHashObj || NULL == context.pSubTableHashObj || - NULL == context.pTableNameHashObj || NULL == context.pDbFNameHashObj || NULL == context.pOutput) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - taosHashSetFreeFp(context.pSubTableHashObj, destroySubTableHashElem); - - if (pContext->pStmtCb) { - TSDB_QUERY_SET_TYPE(context.pOutput->insertType, TSDB_QUERY_TYPE_STMT_INSERT); - } - - if (NULL == *pQuery) { - *pQuery = (SQuery*)nodesMakeNode(QUERY_NODE_QUERY); - if (NULL == *pQuery) { - return TSDB_CODE_OUT_OF_MEMORY; - } - } else { - nodesDestroyNode((*pQuery)->pRoot); - } - - (*pQuery)->execMode = QUERY_EXEC_MODE_SCHEDULE; - (*pQuery)->haveResultSet = false; - (*pQuery)->msgType = TDMT_VND_SUBMIT; - (*pQuery)->pRoot = (SNode*)context.pOutput; - - if (NULL == (*pQuery)->pTableList) { - (*pQuery)->pTableList = taosArrayInit(taosHashGetSize(context.pTableNameHashObj), sizeof(SName)); - if (NULL == (*pQuery)->pTableList) { - return TSDB_CODE_OUT_OF_MEMORY; - } - } - - if (NULL == (*pQuery)->pDbList) { - (*pQuery)->pDbList = taosArrayInit(taosHashGetSize(context.pDbFNameHashObj), TSDB_DB_FNAME_LEN); - if (NULL == (*pQuery)->pDbList) { - return TSDB_CODE_OUT_OF_MEMORY; - } - } - - int32_t code = TSDB_CODE_SUCCESS; - if (!context.pComCxt->needMultiParse) { - code = skipInsertInto(&context.pSql, &context.msg); - if (TSDB_CODE_SUCCESS == code) { - code = parseInsertBody(&context); - } - } else { - code = parseInsertBodyAgain(&context); - } - - if (TSDB_CODE_SUCCESS == code || NEED_CLIENT_HANDLE_ERROR(code)) { - SName* pTable = taosHashIterate(context.pTableNameHashObj, NULL); - while (NULL != pTable) { - taosArrayPush((*pQuery)->pTableList, pTable); - pTable = taosHashIterate(context.pTableNameHashObj, pTable); - } - - char* pDb = taosHashIterate(context.pDbFNameHashObj, NULL); - while (NULL != pDb) { - taosArrayPush((*pQuery)->pDbList, pDb); - pDb = taosHashIterate(context.pDbFNameHashObj, pDb); - } - } - if (pContext->pStmtCb) { - context.pVgroupsHashObj = NULL; - context.pTableBlockHashObj = NULL; - } - destroyInsertParseContext(&context); return code; } -// pSql -> (field1_value, ...) [(field1_value2, ...) ...] -static int32_t skipValuesClause(SInsertParseSyntaxCxt* pCxt) { - int32_t numOfRows = 0; - SToken sToken; - while (1) { - int32_t index = 0; - NEXT_TOKEN_KEEP_SQL(pCxt->pSql, sToken, index); - if (TK_NK_LP != sToken.type) { - break; - } - pCxt->pSql += index; - - CHECK_CODE(skipParentheses(pCxt)); - ++numOfRows; +static int32_t parseInsertSqlFromCsv(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { + STableDataBlocks* pDataBuf = NULL; + int32_t code = getTableDataBlocks(pCxt, pStmt, &pDataBuf); + if (TSDB_CODE_SUCCESS == code) { + code = parseDataFromFileImpl(pCxt, pStmt, pDataBuf); } - if (0 == numOfRows) { - return buildSyntaxErrMsg(&pCxt->msg, "no any data points", NULL); - } - return TSDB_CODE_SUCCESS; -} -static int32_t skipTagsClause(SInsertParseSyntaxCxt* pCxt) { return skipParentheses(pCxt); } + parserDebug("0x%" PRIx64 " insert again input rows: %d", pCxt->pComCxt->requestId, pStmt->totalRowsNum); -static int32_t skipTableOptions(SInsertParseSyntaxCxt* pCxt) { - do { - int32_t index = 0; - SToken sToken; - NEXT_TOKEN_KEEP_SQL(pCxt->pSql, sToken, index); - if (TK_TTL == sToken.type || TK_COMMENT == sToken.type) { - pCxt->pSql += index; - NEXT_TOKEN_WITH_PREV(pCxt->pSql, sToken); + if (TSDB_CODE_SUCCESS == code) { + if (pStmt->fileProcessing) { + code = parseInsertBodyBottom(pCxt, pStmt); } else { - break; + code = parseInsertBody(pCxt, pStmt); } - } while (1); - return TSDB_CODE_SUCCESS; -} - -// pSql -> [(tag1_name, ...)] TAGS (tag1_value, ...) -static int32_t skipUsingClause(SInsertParseSyntaxCxt* pCxt) { - SToken sToken; - NEXT_TOKEN(pCxt->pSql, sToken); - if (TK_NK_LP == sToken.type) { - CHECK_CODE(skipBoundColumns(pCxt)); - NEXT_TOKEN(pCxt->pSql, sToken); } - if (TK_TAGS != sToken.type) { - return buildSyntaxErrMsg(&pCxt->msg, "TAGS is expected", sToken.z); - } - // pSql -> (tag1_value, ...) - NEXT_TOKEN(pCxt->pSql, sToken); - if (TK_NK_LP != sToken.type) { - return buildSyntaxErrMsg(&pCxt->msg, "( is expected", sToken.z); - } - CHECK_CODE(skipTagsClause(pCxt)); - CHECK_CODE(skipTableOptions(pCxt)); - - return TSDB_CODE_SUCCESS; + return code; } -static int32_t collectTableMetaKey(SInsertParseSyntaxCxt* pCxt, bool isStable, int32_t tableNo, SToken* pTbToken) { - SName name = {0}; - CHECK_CODE(insCreateSName(&name, pTbToken, pCxt->pComCxt->acctId, pCxt->pComCxt->db, &pCxt->msg)); - CHECK_CODE(reserveTableMetaInCacheForInsert(&name, isStable ? CATALOG_REQ_TYPE_META : CATALOG_REQ_TYPE_BOTH, tableNo, - pCxt->pMetaCache)); - return TSDB_CODE_SUCCESS; -} - -static int32_t checkTableName(const char* pTableName, SMsgBuf* pMsgBuf) { - if (NULL != strchr(pTableName, '.')) { - return generateSyntaxErrMsgExt(pMsgBuf, TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME, "The table name cannot contain '.'"); - } - return TSDB_CODE_SUCCESS; -} - -static int32_t collectAutoCreateTableMetaKey(SInsertParseSyntaxCxt* pCxt, int32_t tableNo, SToken* pTbToken) { - SName name = {0}; - CHECK_CODE(insCreateSName(&name, pTbToken, pCxt->pComCxt->acctId, pCxt->pComCxt->db, &pCxt->msg)); - CHECK_CODE(checkTableName(name.tname, &pCxt->msg)); - CHECK_CODE(reserveTableMetaInCacheForInsert(&name, CATALOG_REQ_TYPE_VGROUP, tableNo, pCxt->pMetaCache)); - return TSDB_CODE_SUCCESS; -} - -static int32_t parseInsertBodySyntax(SInsertParseSyntaxCxt* pCxt) { - bool hasData = false; - int32_t tableNo = 0; - // for each table - while (1) { - SToken sToken; - - // pSql -> tb_name ... - NEXT_TOKEN(pCxt->pSql, sToken); - - // no data in the sql string anymore. - if (sToken.n == 0) { - if (sToken.type && pCxt->pSql[0]) { - return buildSyntaxErrMsg(&pCxt->msg, "invalid charactor in SQL", sToken.z); - } - - if (!hasData) { - return buildInvalidOperationMsg(&pCxt->msg, "no data in sql"); - } - break; - } - - hasData = false; - - SToken tbnameToken = sToken; - NEXT_TOKEN(pCxt->pSql, sToken); - - bool existedUsing = false; - // USING clause - if (TK_USING == sToken.type) { - existedUsing = true; - CHECK_CODE(collectAutoCreateTableMetaKey(pCxt, tableNo, &tbnameToken)); - NEXT_TOKEN(pCxt->pSql, sToken); - CHECK_CODE(collectTableMetaKey(pCxt, true, tableNo, &sToken)); - CHECK_CODE(skipUsingClause(pCxt)); - NEXT_TOKEN(pCxt->pSql, sToken); - } - - if (TK_NK_LP == sToken.type) { - // pSql -> field1_name, ...) - CHECK_CODE(skipBoundColumns(pCxt)); - NEXT_TOKEN(pCxt->pSql, sToken); - } - - if (TK_USING == sToken.type && !existedUsing) { - existedUsing = true; - CHECK_CODE(collectAutoCreateTableMetaKey(pCxt, tableNo, &tbnameToken)); - NEXT_TOKEN(pCxt->pSql, sToken); - CHECK_CODE(collectTableMetaKey(pCxt, true, tableNo, &sToken)); - CHECK_CODE(skipUsingClause(pCxt)); - NEXT_TOKEN(pCxt->pSql, sToken); - } else if (!existedUsing) { - CHECK_CODE(collectTableMetaKey(pCxt, false, tableNo, &tbnameToken)); - } - - ++tableNo; - - if (TK_VALUES == sToken.type) { - // pSql -> (field1_value, ...) [(field1_value2, ...) ...] - CHECK_CODE(skipValuesClause(pCxt)); - hasData = true; - continue; - } - - // FILE csv_file_path - if (TK_FILE == sToken.type) { - // pSql -> csv_file_path - NEXT_TOKEN(pCxt->pSql, sToken); - if (0 == sToken.n || (TK_NK_STRING != sToken.type && TK_NK_ID != sToken.type)) { - return buildSyntaxErrMsg(&pCxt->msg, "file path is required following keyword FILE", sToken.z); - } - hasData = true; - continue; - } - - return buildSyntaxErrMsg(&pCxt->msg, "keyword VALUES or FILE is expected", sToken.z); - } - - return TSDB_CODE_SUCCESS; -} - -int32_t parseInsertSyntax(SParseContext* pContext, SQuery** pQuery, SParseMetaCache* pMetaCache) { - SInsertParseSyntaxCxt context = {.pComCxt = pContext, - .pSql = (char*)pContext->pSql, - .msg = {.buf = pContext->pMsg, .len = pContext->msgLen}, - .pMetaCache = pMetaCache}; - int32_t code = skipInsertInto(&context.pSql, &context.msg); +static int32_t parseInsertSqlFromTable(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { + int32_t code = parseInsertTableClauseBottom(pCxt, pStmt); if (TSDB_CODE_SUCCESS == code) { - code = parseInsertBodySyntax(&context); - } - if (TSDB_CODE_SUCCESS == code) { - *pQuery = (SQuery*)nodesMakeNode(QUERY_NODE_QUERY); - if (NULL == *pQuery) { - return TSDB_CODE_OUT_OF_MEMORY; - } + code = parseInsertBody(pCxt, pStmt); } return code; } + +static int32_t parseInsertSqlImpl(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { + if (pStmt->pSql == pCxt->pComCxt->pSql || NULL != pCxt->pComCxt->pStmtCb) { + return parseInsertSqlFromStart(pCxt, pStmt); + } + + if (pStmt->fileProcessing) { + return parseInsertSqlFromCsv(pCxt, pStmt); + } + + return parseInsertSqlFromTable(pCxt, pStmt); +} + +static int32_t buildInsertTableReq(SName* pName, SArray** pTables) { + *pTables = taosArrayInit(1, sizeof(SName)); + if (NULL == *pTables) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + taosArrayPush(*pTables, pName); + return TSDB_CODE_SUCCESS; +} + +static int32_t buildInsertDbReq(SName* pName, SArray** pDbs) { + if (NULL == *pDbs) { + *pDbs = taosArrayInit(1, sizeof(STablesReq)); + if (NULL == *pDbs) { + return TSDB_CODE_OUT_OF_MEMORY; + } + } + + STablesReq req = {0}; + tNameGetFullDbName(pName, req.dbFName); + buildInsertTableReq(pName, &req.pTables); + taosArrayPush(*pDbs, &req); + + return TSDB_CODE_SUCCESS; +} + +static int32_t buildInsertUserAuthReq(const char* pUser, SName* pName, SArray** pUserAuth) { + *pUserAuth = taosArrayInit(1, sizeof(SUserAuthInfo)); + if (NULL == *pUserAuth) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + SUserAuthInfo userAuth = {.type = AUTH_TYPE_WRITE}; + snprintf(userAuth.user, sizeof(userAuth.user), "%s", pUser); + tNameGetFullDbName(pName, userAuth.dbFName); + taosArrayPush(*pUserAuth, &userAuth); + + return TSDB_CODE_SUCCESS; +} + +static int32_t buildInsertCatalogReq(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, SCatalogReq* pCatalogReq) { + int32_t code = buildInsertUserAuthReq(pCxt->pComCxt->pUser, &pStmt->targetTableName, &pCatalogReq->pUser); + if (TSDB_CODE_SUCCESS == code) { + if (0 == pStmt->usingTableName.type) { + code = buildInsertDbReq(&pStmt->targetTableName, &pCatalogReq->pTableMeta); + } else { + code = buildInsertDbReq(&pStmt->usingTableName, &pCatalogReq->pTableMeta); + } + } + if (TSDB_CODE_SUCCESS == code) { + code = buildInsertDbReq(&pStmt->targetTableName, &pCatalogReq->pTableHash); + } + return code; +} + +static int32_t setNextStageInfo(SInsertParseContext* pCxt, SQuery* pQuery, SCatalogReq* pCatalogReq) { + if (pCxt->missCache) { + pQuery->execStage = QUERY_EXEC_STAGE_PARSE; + return buildInsertCatalogReq(pCxt, (SVnodeModifOpStmt*)pQuery->pRoot, pCatalogReq); + } + + pQuery->execStage = QUERY_EXEC_STAGE_SCHEDULE; + return TSDB_CODE_SUCCESS; +} + +int32_t parseInsertSql(SParseContext* pCxt, SQuery** pQuery, SCatalogReq* pCatalogReq, const SMetaData* pMetaData) { + SInsertParseContext context = { + .pComCxt = pCxt, + .msg = {.buf = pCxt->pMsg, .len = pCxt->msgLen}, + }; + + int32_t code = initInsertQuery(pCxt, pMetaData, pQuery); + if (TSDB_CODE_SUCCESS == code) { + code = parseInsertSqlImpl(&context, (SVnodeModifOpStmt*)(*pQuery)->pRoot); + } + if (TSDB_CODE_SUCCESS == code) { + code = setNextStageInfo(&context, *pQuery, pCatalogReq); + } + if ((TSDB_CODE_SUCCESS == code || NEED_CLIENT_HANDLE_ERROR(code)) && + QUERY_EXEC_STAGE_SCHEDULE == (*pQuery)->execStage) { + code = setRefreshMate(*pQuery); + } + destroyBoundColumnInfo(&context.tags); + return code; +} diff --git a/source/libs/parser/src/parInsertStmt.c b/source/libs/parser/src/parInsertStmt.c index f85ceccf6e..73557c067e 100644 --- a/source/libs/parser/src/parInsertStmt.c +++ b/source/libs/parser/src/parInsertStmt.c @@ -30,22 +30,17 @@ typedef struct SKvParam { } SKvParam; int32_t qBuildStmtOutput(SQuery* pQuery, SHashObj* pVgHash, SHashObj* pBlockHash) { - SVnodeModifOpStmt* modifyNode = (SVnodeModifOpStmt*)pQuery->pRoot; - int32_t code = 0; - SInsertParseContext insertCtx = { - .pVgroupsHashObj = pVgHash, - .pTableBlockHashObj = pBlockHash, - .pOutput = (SVnodeModifOpStmt*)pQuery->pRoot, - }; + SVnodeModifOpStmt* modifyNode = (SVnodeModifOpStmt*)pQuery->pRoot; + int32_t code = 0; // merge according to vgId - if (taosHashGetSize(insertCtx.pTableBlockHashObj) > 0) { - CHECK_CODE(insMergeTableDataBlocks(insertCtx.pTableBlockHashObj, &insertCtx.pVgDataBlocks)); + if (taosHashGetSize(pBlockHash) > 0) { + CHECK_CODE(insMergeTableDataBlocks(pBlockHash, &modifyNode->pVgDataBlocks)); } - CHECK_CODE(insBuildOutput(&insertCtx)); + CHECK_CODE(insBuildOutput(modifyNode)); - insDestroyBlockArrayList(insertCtx.pVgDataBlocks); + insDestroyBlockArrayList(modifyNode->pVgDataBlocks); return TSDB_CODE_SUCCESS; } @@ -222,11 +217,7 @@ int32_t qBindStmtColsValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBuf, in } SSubmitBlk* pBlocks = (SSubmitBlk*)(pDataBlock->pData); - if (TSDB_CODE_SUCCESS != insSetBlockInfo(pBlocks, pDataBlock, bind->num)) { - return buildInvalidOperationMsg(&pBuf, "too many rows in sql, total number of rows should be less than INT32_MAX"); - } - - return TSDB_CODE_SUCCESS; + return insSetBlockInfo(pBlocks, pDataBlock, bind->num, &pBuf); } int32_t qBindStmtSingleColValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen, int32_t colIdx, @@ -308,10 +299,7 @@ int32_t qBindStmtSingleColValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBu pDataBlock->size += extendedRowSize * bind->num; SSubmitBlk* pBlocks = (SSubmitBlk*)(pDataBlock->pData); - if (TSDB_CODE_SUCCESS != insSetBlockInfo(pBlocks, pDataBlock, bind->num)) { - return buildInvalidOperationMsg(&pBuf, - "too many rows in sql, total number of rows should be less than INT32_MAX"); - } + CHECK_CODE(insSetBlockInfo(pBlocks, pDataBlock, bind->num, &pBuf)); } return TSDB_CODE_SUCCESS; diff --git a/source/libs/parser/src/parInsertUtil.c b/source/libs/parser/src/parInsertUtil.c index 570a6f9859..a0a24d3d31 100644 --- a/source/libs/parser/src/parInsertUtil.c +++ b/source/libs/parser/src/parInsertUtil.c @@ -110,18 +110,17 @@ void insGetSTSRowAppendInfo(uint8_t rowType, SParsedDataColInfo* spd, col_id_t i } } -int32_t insSetBlockInfo(SSubmitBlk* pBlocks, STableDataBlocks* dataBuf, int32_t numOfRows) { +int32_t insSetBlockInfo(SSubmitBlk* pBlocks, STableDataBlocks* dataBuf, int32_t numOfRows, SMsgBuf* pMsg) { pBlocks->suid = (TSDB_NORMAL_TABLE == dataBuf->pTableMeta->tableType ? 0 : dataBuf->pTableMeta->suid); pBlocks->uid = dataBuf->pTableMeta->uid; pBlocks->sversion = dataBuf->pTableMeta->sversion; pBlocks->schemaLen = dataBuf->createTbReqLen; if (pBlocks->numOfRows + numOfRows >= INT32_MAX) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } else { - pBlocks->numOfRows += numOfRows; - return TSDB_CODE_SUCCESS; + return buildInvalidOperationMsg(pMsg, "too many rows in sql, total number of rows should be less than INT32_MAX"); } + pBlocks->numOfRows += numOfRows; + return TSDB_CODE_SUCCESS; } void insSetBoundColumnInfo(SParsedDataColInfo* pColList, SSchema* pSchema, col_id_t numOfCols) { @@ -271,12 +270,8 @@ void insDestroyDataBlock(STableDataBlocks* pDataBlock) { } taosMemoryFreeClear(pDataBlock->pData); - // if (!pDataBlock->cloned) { - // free the refcount for metermeta taosMemoryFreeClear(pDataBlock->pTableMeta); - destroyBoundColumnInfo(&pDataBlock->boundColumnInfo); - // } taosMemoryFreeClear(pDataBlock); } @@ -312,20 +307,6 @@ int32_t insGetDataBlockFromList(SHashObj* pHashList, void* id, int32_t idLen, in return TSDB_CODE_SUCCESS; } -#if 0 -static int32_t getRowExpandSize(STableMeta* pTableMeta) { - int32_t result = TD_ROW_HEAD_LEN - sizeof(TSKEY); - int32_t columns = getNumOfColumns(pTableMeta); - SSchema* pSchema = getTableColumnSchema(pTableMeta); - for (int32_t i = 0; i < columns; ++i) { - if (IS_VAR_DATA_TYPE((pSchema + i)->type)) { - result += TYPE_BYTES[TSDB_DATA_TYPE_BINARY]; - } - } - result += (int32_t)TD_BITMAP_BYTES(columns - 1); - return result; -} -#endif void insDestroyBlockArrayList(SArray* pDataBlockList) { if (pDataBlockList == NULL) { @@ -357,51 +338,6 @@ void insDestroyBlockHashmap(SHashObj* pDataBlockHash) { taosHashCleanup(pDataBlockHash); } -#if 0 -// data block is disordered, sort it in ascending order -void sortRemoveDataBlockDupRowsRaw(STableDataBlocks* dataBuf) { - SSubmitBlk* pBlocks = (SSubmitBlk*)dataBuf->pData; - - // size is less than the total size, since duplicated rows may be removed yet. - assert(pBlocks->numOfRows * dataBuf->rowSize + sizeof(SSubmitBlk) == dataBuf->size); - - if (!dataBuf->ordered) { - char* pBlockData = pBlocks->data; - - // todo. qsort is unstable, if timestamp is same, should get the last one - taosSort(pBlockData, pBlocks->numOfRows, dataBuf->rowSize, rowDataCompar); - - int32_t i = 0; - int32_t j = 1; - - // delete rows with timestamp conflicts - while (j < pBlocks->numOfRows) { - TSKEY ti = *(TSKEY*)(pBlockData + dataBuf->rowSize * i); - TSKEY tj = *(TSKEY*)(pBlockData + dataBuf->rowSize * j); - - if (ti == tj) { - ++j; - continue; - } - - int32_t nextPos = (++i); - if (nextPos != j) { - memmove(pBlockData + dataBuf->rowSize * nextPos, pBlockData + dataBuf->rowSize * j, dataBuf->rowSize); - } - - ++j; - } - - dataBuf->ordered = true; - - pBlocks->numOfRows = i + 1; - dataBuf->size = sizeof(SSubmitBlk) + dataBuf->rowSize * pBlocks->numOfRows; - } - - dataBuf->prevTS = INT64_MIN; -} -#endif - // data block is disordered, sort it in ascending order static int sortRemoveDataBlockDupRows(STableDataBlocks* dataBuf, SBlockKeyInfo* pBlkKeyInfo) { SSubmitBlk* pBlocks = (SSubmitBlk*)dataBuf->pData; @@ -994,24 +930,24 @@ static void buildMsgHeader(STableDataBlocks* src, SVgDataBlocks* blocks) { } } -int32_t insBuildOutput(SInsertParseContext* pCxt) { - size_t numOfVg = taosArrayGetSize(pCxt->pVgDataBlocks); - pCxt->pOutput->pDataBlocks = taosArrayInit(numOfVg, POINTER_BYTES); - if (NULL == pCxt->pOutput->pDataBlocks) { +int32_t insBuildOutput(SVnodeModifOpStmt* pStmt) { + size_t numOfVg = taosArrayGetSize(pStmt->pVgDataBlocks); + pStmt->pDataBlocks = taosArrayInit(numOfVg, POINTER_BYTES); + if (NULL == pStmt->pDataBlocks) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } for (size_t i = 0; i < numOfVg; ++i) { - STableDataBlocks* src = taosArrayGetP(pCxt->pVgDataBlocks, i); + STableDataBlocks* src = taosArrayGetP(pStmt->pVgDataBlocks, i); SVgDataBlocks* dst = taosMemoryCalloc(1, sizeof(SVgDataBlocks)); if (NULL == dst) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } - taosHashGetDup(pCxt->pVgroupsHashObj, (const char*)&src->vgId, sizeof(src->vgId), &dst->vg); + taosHashGetDup(pStmt->pVgroupsHashObj, (const char*)&src->vgId, sizeof(src->vgId), &dst->vg); dst->numOfTables = src->numOfTables; dst->size = src->size; TSWAP(dst->pData, src->pData); buildMsgHeader(src, dst); - taosArrayPush(pCxt->pOutput->pDataBlocks, &dst); + taosArrayPush(pStmt->pDataBlocks, &dst); } return TSDB_CODE_SUCCESS; } diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index d98d513d5d..466c6edd24 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -612,62 +612,7 @@ static int32_t buildUdfReq(SHashObj* pUdfHash, SArray** pUdf) { return TSDB_CODE_SUCCESS; } -static int32_t buildCatalogReqForInsert(SParseContext* pCxt, const SParseMetaCache* pMetaCache, - SCatalogReq* pCatalogReq) { - int32_t ndbs = taosHashGetSize(pMetaCache->pInsertTables); - pCatalogReq->pTableMeta = taosArrayInit(ndbs, sizeof(STablesReq)); - if (NULL == pCatalogReq->pTableMeta) { - return TSDB_CODE_OUT_OF_MEMORY; - } - pCatalogReq->pTableHash = taosArrayInit(ndbs, sizeof(STablesReq)); - if (NULL == pCatalogReq->pTableHash) { - return TSDB_CODE_OUT_OF_MEMORY; - } - pCatalogReq->pUser = taosArrayInit(ndbs, sizeof(SUserAuthInfo)); - if (NULL == pCatalogReq->pUser) { - return TSDB_CODE_OUT_OF_MEMORY; - } - - pCxt->pTableMetaPos = taosArrayInit(pMetaCache->sqlTableNum, sizeof(int32_t)); - pCxt->pTableVgroupPos = taosArrayInit(pMetaCache->sqlTableNum, sizeof(int32_t)); - - int32_t metaReqNo = 0; - int32_t vgroupReqNo = 0; - SInsertTablesMetaReq* p = taosHashIterate(pMetaCache->pInsertTables, NULL); - while (NULL != p) { - STablesReq req = {0}; - strcpy(req.dbFName, p->dbFName); - TSWAP(req.pTables, p->pTableMetaReq); - taosArrayPush(pCatalogReq->pTableMeta, &req); - - req.pTables = NULL; - TSWAP(req.pTables, p->pTableVgroupReq); - taosArrayPush(pCatalogReq->pTableHash, &req); - - int32_t ntables = taosArrayGetSize(p->pTableMetaPos); - for (int32_t i = 0; i < ntables; ++i) { - taosArrayInsert(pCxt->pTableMetaPos, *(int32_t*)taosArrayGet(p->pTableMetaPos, i), &metaReqNo); - ++metaReqNo; - } - - ntables = taosArrayGetSize(p->pTableVgroupPos); - for (int32_t i = 0; i < ntables; ++i) { - taosArrayInsert(pCxt->pTableVgroupPos, *(int32_t*)taosArrayGet(p->pTableVgroupPos, i), &vgroupReqNo); - ++vgroupReqNo; - } - - SUserAuthInfo auth = {0}; - snprintf(auth.user, sizeof(auth.user), "%s", pCxt->pUser); - snprintf(auth.dbFName, sizeof(auth.dbFName), "%s", p->dbFName); - auth.type = AUTH_TYPE_WRITE; - taosArrayPush(pCatalogReq->pUser, &auth); - - p = taosHashIterate(pMetaCache->pInsertTables, p); - } - return TSDB_CODE_SUCCESS; -} - -int32_t buildCatalogReqForQuery(const SParseMetaCache* pMetaCache, SCatalogReq* pCatalogReq) { +int32_t buildCatalogReq(const SParseMetaCache* pMetaCache, SCatalogReq* pCatalogReq) { int32_t code = buildTableReqFromDb(pMetaCache->pTableMeta, &pCatalogReq->pTableMeta); if (TSDB_CODE_SUCCESS == code) { code = buildDbReq(pMetaCache->pDbVgroup, &pCatalogReq->pDbVgroup); @@ -697,13 +642,6 @@ int32_t buildCatalogReqForQuery(const SParseMetaCache* pMetaCache, SCatalogReq* return code; } -int32_t buildCatalogReq(SParseContext* pCxt, const SParseMetaCache* pMetaCache, SCatalogReq* pCatalogReq) { - if (NULL != pMetaCache->pInsertTables) { - return buildCatalogReqForInsert(pCxt, pMetaCache, pCatalogReq); - } - return buildCatalogReqForQuery(pMetaCache, pCatalogReq); -} - static int32_t putMetaDataToHash(const char* pKey, int32_t len, const SArray* pData, int32_t index, SHashObj** pHash) { if (NULL == *pHash) { *pHash = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); @@ -791,8 +729,7 @@ static int32_t putUdfToCache(const SArray* pUdfReq, const SArray* pUdfData, SHas return TSDB_CODE_SUCCESS; } -int32_t putMetaDataToCacheForQuery(const SCatalogReq* pCatalogReq, const SMetaData* pMetaData, - SParseMetaCache* pMetaCache) { +int32_t putMetaDataToCache(const SCatalogReq* pCatalogReq, const SMetaData* pMetaData, SParseMetaCache* pMetaCache) { int32_t code = putDbTableDataToCache(pCatalogReq->pTableMeta, pMetaData->pTableMeta, &pMetaCache->pTableMeta); if (TSDB_CODE_SUCCESS == code) { code = putDbDataToCache(pCatalogReq->pDbVgroup, pMetaData->pDbVgroup, &pMetaCache->pDbVgroup); @@ -822,30 +759,6 @@ int32_t putMetaDataToCacheForQuery(const SCatalogReq* pCatalogReq, const SMetaDa return code; } -int32_t putMetaDataToCacheForInsert(const SMetaData* pMetaData, SParseMetaCache* pMetaCache) { - int32_t ndbs = taosArrayGetSize(pMetaData->pUser); - for (int32_t i = 0; i < ndbs; ++i) { - SMetaRes* pRes = taosArrayGet(pMetaData->pUser, i); - if (TSDB_CODE_SUCCESS != pRes->code) { - return pRes->code; - } - if (!(*(bool*)pRes->pRes)) { - return TSDB_CODE_PAR_PERMISSION_DENIED; - } - } - pMetaCache->pTableMetaData = pMetaData->pTableMeta; - pMetaCache->pTableVgroupData = pMetaData->pTableHash; - return TSDB_CODE_SUCCESS; -} - -int32_t putMetaDataToCache(const SCatalogReq* pCatalogReq, const SMetaData* pMetaData, SParseMetaCache* pMetaCache, - bool insertValuesStmt) { - if (insertValuesStmt) { - return putMetaDataToCacheForInsert(pMetaData, pMetaCache); - } - return putMetaDataToCacheForQuery(pCatalogReq, pMetaData, pMetaCache); -} - static int32_t reserveTableReqInCacheImpl(const char* pTbFName, int32_t len, SHashObj** pTables) { if (NULL == *pTables) { *pTables = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); @@ -1146,82 +1059,6 @@ int32_t getDnodeListFromCache(SParseMetaCache* pMetaCache, SArray** pDnodes) { return TSDB_CODE_SUCCESS; } -static int32_t reserveTableReqInCacheForInsert(const SName* pName, ECatalogReqType reqType, int32_t tableNo, - SInsertTablesMetaReq* pReq) { - switch (reqType) { - case CATALOG_REQ_TYPE_META: - taosArrayPush(pReq->pTableMetaReq, pName); - taosArrayPush(pReq->pTableMetaPos, &tableNo); - break; - case CATALOG_REQ_TYPE_VGROUP: - taosArrayPush(pReq->pTableVgroupReq, pName); - taosArrayPush(pReq->pTableVgroupPos, &tableNo); - break; - case CATALOG_REQ_TYPE_BOTH: - taosArrayPush(pReq->pTableMetaReq, pName); - taosArrayPush(pReq->pTableMetaPos, &tableNo); - taosArrayPush(pReq->pTableVgroupReq, pName); - taosArrayPush(pReq->pTableVgroupPos, &tableNo); - break; - default: - break; - } - return TSDB_CODE_SUCCESS; -} - -static int32_t reserveTableReqInDbCacheForInsert(const SName* pName, ECatalogReqType reqType, int32_t tableNo, - SHashObj* pDbs) { - SInsertTablesMetaReq req = {.pTableMetaReq = taosArrayInit(4, sizeof(SName)), - .pTableMetaPos = taosArrayInit(4, sizeof(int32_t)), - .pTableVgroupReq = taosArrayInit(4, sizeof(SName)), - .pTableVgroupPos = taosArrayInit(4, sizeof(int32_t))}; - tNameGetFullDbName(pName, req.dbFName); - int32_t code = reserveTableReqInCacheForInsert(pName, reqType, tableNo, &req); - if (TSDB_CODE_SUCCESS == code) { - code = taosHashPut(pDbs, pName->dbname, strlen(pName->dbname), &req, sizeof(SInsertTablesMetaReq)); - } - return code; -} - -int32_t reserveTableMetaInCacheForInsert(const SName* pName, ECatalogReqType reqType, int32_t tableNo, - SParseMetaCache* pMetaCache) { - if (NULL == pMetaCache->pInsertTables) { - pMetaCache->pInsertTables = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); - if (NULL == pMetaCache->pInsertTables) { - return TSDB_CODE_OUT_OF_MEMORY; - } - } - pMetaCache->sqlTableNum = tableNo; - SInsertTablesMetaReq* pReq = taosHashGet(pMetaCache->pInsertTables, pName->dbname, strlen(pName->dbname)); - if (NULL == pReq) { - return reserveTableReqInDbCacheForInsert(pName, reqType, tableNo, pMetaCache->pInsertTables); - } - return reserveTableReqInCacheForInsert(pName, reqType, tableNo, pReq); -} - -int32_t getTableMetaFromCacheForInsert(SArray* pTableMetaPos, SParseMetaCache* pMetaCache, int32_t tableNo, - STableMeta** pMeta) { - int32_t reqIndex = *(int32_t*)taosArrayGet(pTableMetaPos, tableNo); - SMetaRes* pRes = taosArrayGet(pMetaCache->pTableMetaData, reqIndex); - if (TSDB_CODE_SUCCESS == pRes->code) { - *pMeta = tableMetaDup((const STableMeta*)pRes->pRes); - if (NULL == *pMeta) { - return TSDB_CODE_OUT_OF_MEMORY; - } - } - return pRes->code; -} - -int32_t getTableVgroupFromCacheForInsert(SArray* pTableVgroupPos, SParseMetaCache* pMetaCache, int32_t tableNo, - SVgroupInfo* pVgroup) { - int32_t reqIndex = *(int32_t*)taosArrayGet(pTableVgroupPos, tableNo); - SMetaRes* pRes = taosArrayGet(pMetaCache->pTableVgroupData, reqIndex); - if (TSDB_CODE_SUCCESS == pRes->code) { - memcpy(pVgroup, pRes->pRes, sizeof(SVgroupInfo)); - } - return pRes->code; -} - void destoryParseTablesMetaReqHash(SHashObj* pHash) { SParseTablesMetaReq* p = taosHashIterate(pHash, NULL); while (NULL != p) { @@ -1239,16 +1076,6 @@ void destoryParseMetaCache(SParseMetaCache* pMetaCache, bool request) { taosHashCleanup(pMetaCache->pTableMeta); taosHashCleanup(pMetaCache->pTableVgroup); } - SInsertTablesMetaReq* p = taosHashIterate(pMetaCache->pInsertTables, NULL); - while (NULL != p) { - taosArrayDestroy(p->pTableMetaPos); - taosArrayDestroy(p->pTableMetaReq); - taosArrayDestroy(p->pTableVgroupPos); - taosArrayDestroy(p->pTableVgroupReq); - - p = taosHashIterate(pMetaCache->pInsertTables, p); - } - taosHashCleanup(pMetaCache->pInsertTables); taosHashCleanup(pMetaCache->pDbVgroup); taosHashCleanup(pMetaCache->pDbCfg); taosHashCleanup(pMetaCache->pDbInfo); diff --git a/source/libs/parser/src/parser.c b/source/libs/parser/src/parser.c index 748478778a..cf338d63ff 100644 --- a/source/libs/parser/src/parser.c +++ b/source/libs/parser/src/parser.c @@ -167,7 +167,7 @@ static void rewriteExprAlias(SNode* pRoot) { int32_t qParseSql(SParseContext* pCxt, SQuery** pQuery) { int32_t code = TSDB_CODE_SUCCESS; if (qIsInsertValuesSql(pCxt->pSql, pCxt->sqlLen)) { - code = parseInsertSql(pCxt, pQuery, NULL); + code = parseInsertSql(pCxt, pQuery, NULL, NULL); } else { code = parseSqlIntoAst(pCxt, pQuery); } @@ -175,21 +175,26 @@ int32_t qParseSql(SParseContext* pCxt, SQuery** pQuery) { return code; } -int32_t qParseSqlSyntax(SParseContext* pCxt, SQuery** pQuery, struct SCatalogReq* pCatalogReq) { +static int32_t parseQuerySyntax(SParseContext* pCxt, SQuery** pQuery, struct SCatalogReq* pCatalogReq) { SParseMetaCache metaCache = {0}; - int32_t code = nodesAcquireAllocator(pCxt->allocatorId); + int32_t code = parseSqlSyntax(pCxt, pQuery, &metaCache); + if (TSDB_CODE_SUCCESS == code) { + code = buildCatalogReq(&metaCache, pCatalogReq); + } + destoryParseMetaCache(&metaCache, true); + return code; +} + +int32_t qParseSqlSyntax(SParseContext* pCxt, SQuery** pQuery, struct SCatalogReq* pCatalogReq) { + int32_t code = nodesAcquireAllocator(pCxt->allocatorId); if (TSDB_CODE_SUCCESS == code) { if (qIsInsertValuesSql(pCxt->pSql, pCxt->sqlLen)) { - code = parseInsertSyntax(pCxt, pQuery, &metaCache); + code = parseInsertSql(pCxt, pQuery, pCatalogReq, NULL); } else { - code = parseSqlSyntax(pCxt, pQuery, &metaCache); + code = parseQuerySyntax(pCxt, pQuery, pCatalogReq); } } - if (TSDB_CODE_SUCCESS == code) { - code = buildCatalogReq(pCxt, &metaCache, pCatalogReq); - } nodesReleaseAllocator(pCxt->allocatorId); - destoryParseMetaCache(&metaCache, true); terrno = code; return code; } @@ -199,14 +204,10 @@ int32_t qAnalyseSqlSemantic(SParseContext* pCxt, const struct SCatalogReq* pCata SParseMetaCache metaCache = {0}; int32_t code = nodesAcquireAllocator(pCxt->allocatorId); if (TSDB_CODE_SUCCESS == code) { - code = putMetaDataToCache(pCatalogReq, pMetaData, &metaCache, NULL == pQuery->pRoot); + code = putMetaDataToCache(pCatalogReq, pMetaData, &metaCache); } if (TSDB_CODE_SUCCESS == code) { - if (NULL == pQuery->pRoot) { - code = parseInsertSql(pCxt, &pQuery, &metaCache); - } else { - code = analyseSemantic(pCxt, pQuery, &metaCache); - } + code = analyseSemantic(pCxt, pQuery, &metaCache); } nodesReleaseAllocator(pCxt->allocatorId); destoryParseMetaCache(&metaCache, false); @@ -214,6 +215,11 @@ int32_t qAnalyseSqlSemantic(SParseContext* pCxt, const struct SCatalogReq* pCata return code; } +int32_t qContinueParseSql(SParseContext* pCxt, struct SCatalogReq* pCatalogReq, const struct SMetaData* pMetaData, + SQuery* pQuery) { + return parseInsertSql(pCxt, &pQuery, pCatalogReq, pMetaData); +} + void qDestroyParseContext(SParseContext* pCxt) { if (NULL == pCxt) { return; diff --git a/source/libs/parser/test/mockCatalog.cpp b/source/libs/parser/test/mockCatalog.cpp index 7f18a7b282..96ba8ab273 100644 --- a/source/libs/parser/test/mockCatalog.cpp +++ b/source/libs/parser/test/mockCatalog.cpp @@ -228,11 +228,23 @@ int32_t __catalogGetTableMeta(struct SCatalog* pCatalog, SRequestConnInfo* pConn return g_mockCatalogService->catalogGetTableMeta(pTableName, pTableMeta); } +int32_t __catalogGetCachedTableMeta(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, + STableMeta** pTableMeta) { + return g_mockCatalogService->catalogGetTableMeta(pTableName, pTableMeta, true); +} + int32_t __catalogGetTableHashVgroup(struct SCatalog* pCatalog, SRequestConnInfo* pConn, const SName* pTableName, SVgroupInfo* vgInfo) { return g_mockCatalogService->catalogGetTableHashVgroup(pTableName, vgInfo); } +int32_t __catalogGetCachedTableHashVgroup(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, + SVgroupInfo* pVgroup, bool* exists) { + int32_t code = g_mockCatalogService->catalogGetTableHashVgroup(pTableName, pVgroup, true); + *exists = 0 != pVgroup->vgId; + return code; +} + int32_t __catalogGetTableDistVgInfo(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, SArray** pVgList) { return g_mockCatalogService->catalogGetTableDistVgInfo(pTableName, pVgList); @@ -289,8 +301,11 @@ void initMetaDataEnv() { static Stub stub; stub.set(catalogGetHandle, __catalogGetHandle); stub.set(catalogGetTableMeta, __catalogGetTableMeta); + stub.set(catalogGetCachedTableMeta, __catalogGetCachedTableMeta); stub.set(catalogGetSTableMeta, __catalogGetTableMeta); + stub.set(catalogGetCachedSTableMeta, __catalogGetCachedTableMeta); stub.set(catalogGetTableHashVgroup, __catalogGetTableHashVgroup); + stub.set(catalogGetCachedTableHashVgroup, __catalogGetCachedTableHashVgroup); stub.set(catalogGetTableDistVgInfo, __catalogGetTableDistVgInfo); stub.set(catalogGetDBVgVersion, __catalogGetDBVgVersion); stub.set(catalogGetDBVgList, __catalogGetDBVgList); diff --git a/source/libs/parser/test/mockCatalogService.cpp b/source/libs/parser/test/mockCatalogService.cpp index f2cebcb08d..95f7af435d 100644 --- a/source/libs/parser/test/mockCatalogService.cpp +++ b/source/libs/parser/test/mockCatalogService.cpp @@ -91,7 +91,7 @@ class MockCatalogServiceImpl { public: static const int32_t numOfDataTypes = sizeof(tDataTypes) / sizeof(tDataTypes[0]); - MockCatalogServiceImpl() : id_(1) {} + MockCatalogServiceImpl() : id_(1), havaCache_(true) {} ~MockCatalogServiceImpl() { for (auto& cfg : dbCfg_) { @@ -106,7 +106,11 @@ class MockCatalogServiceImpl { int32_t catalogGetHandle() const { return 0; } - int32_t catalogGetTableMeta(const SName* pTableName, STableMeta** pTableMeta) const { + int32_t catalogGetTableMeta(const SName* pTableName, STableMeta** pTableMeta, bool onlyCache = false) const { + if (onlyCache && !havaCache_) { + return TSDB_CODE_SUCCESS; + } + std::unique_ptr table; char db[TSDB_DB_NAME_LEN] = {0}; @@ -121,7 +125,12 @@ class MockCatalogServiceImpl { return TSDB_CODE_SUCCESS; } - int32_t catalogGetTableHashVgroup(const SName* pTableName, SVgroupInfo* vgInfo) const { + int32_t catalogGetTableHashVgroup(const SName* pTableName, SVgroupInfo* vgInfo, bool onlyCache = false) const { + if (onlyCache && !havaCache_) { + vgInfo->vgId = 0; + return TSDB_CODE_SUCCESS; + } + vgInfo->vgId = 1; return TSDB_CODE_SUCCESS; } @@ -618,6 +627,7 @@ class MockCatalogServiceImpl { IndexMetaCache index_; DnodeCache dnode_; DbCfgCache dbCfg_; + bool havaCache_; }; MockCatalogService::MockCatalogService() : impl_(new MockCatalogServiceImpl()) {} @@ -651,12 +661,14 @@ void MockCatalogService::createDatabase(const std::string& db, bool rollup, int8 impl_->createDatabase(db, rollup, cacheLast); } -int32_t MockCatalogService::catalogGetTableMeta(const SName* pTableName, STableMeta** pTableMeta) const { - return impl_->catalogGetTableMeta(pTableName, pTableMeta); +int32_t MockCatalogService::catalogGetTableMeta(const SName* pTableName, STableMeta** pTableMeta, + bool onlyCache) const { + return impl_->catalogGetTableMeta(pTableName, pTableMeta, onlyCache); } -int32_t MockCatalogService::catalogGetTableHashVgroup(const SName* pTableName, SVgroupInfo* vgInfo) const { - return impl_->catalogGetTableHashVgroup(pTableName, vgInfo); +int32_t MockCatalogService::catalogGetTableHashVgroup(const SName* pTableName, SVgroupInfo* vgInfo, + bool onlyCache) const { + return impl_->catalogGetTableHashVgroup(pTableName, vgInfo, onlyCache); } int32_t MockCatalogService::catalogGetTableDistVgInfo(const SName* pTableName, SArray** pVgList) const { diff --git a/source/libs/parser/test/mockCatalogService.h b/source/libs/parser/test/mockCatalogService.h index d9d2185728..acd7fab8e1 100644 --- a/source/libs/parser/test/mockCatalogService.h +++ b/source/libs/parser/test/mockCatalogService.h @@ -67,8 +67,8 @@ class MockCatalogService { void createDnode(int32_t dnodeId, const std::string& host, int16_t port); void createDatabase(const std::string& db, bool rollup = false, int8_t cacheLast = 0); - int32_t catalogGetTableMeta(const SName* pTableName, STableMeta** pTableMeta) const; - int32_t catalogGetTableHashVgroup(const SName* pTableName, SVgroupInfo* vgInfo) const; + int32_t catalogGetTableMeta(const SName* pTableName, STableMeta** pTableMeta, bool onlyCache = false) const; + int32_t catalogGetTableHashVgroup(const SName* pTableName, SVgroupInfo* vgInfo, bool onlyCache = false) const; int32_t catalogGetTableDistVgInfo(const SName* pTableName, SArray** pVgList) const; int32_t catalogGetDBVgList(const char* pDbFName, SArray** pVgList) const; int32_t catalogGetDBCfg(const char* pDbFName, SDbCfgInfo* pDbCfg) const; diff --git a/source/libs/parser/test/parTestUtil.cpp b/source/libs/parser/test/parTestUtil.cpp index bf27fd2e13..1063f4fadc 100644 --- a/source/libs/parser/test/parTestUtil.cpp +++ b/source/libs/parser/test/parTestUtil.cpp @@ -233,16 +233,15 @@ class ParserTestBaseImpl { } void doBuildCatalogReq(SParseContext* pCxt, const SParseMetaCache* pMetaCache, SCatalogReq* pCatalogReq) { - DO_WITH_THROW(buildCatalogReq, pCxt, pMetaCache, pCatalogReq); + DO_WITH_THROW(buildCatalogReq, pMetaCache, pCatalogReq); } void doGetAllMeta(const SCatalogReq* pCatalogReq, SMetaData* pMetaData) { DO_WITH_THROW(g_mockCatalogService->catalogGetAllMeta, pCatalogReq, pMetaData); } - void doPutMetaDataToCache(const SCatalogReq* pCatalogReq, const SMetaData* pMetaData, SParseMetaCache* pMetaCache, - bool isInsertValues) { - DO_WITH_THROW(putMetaDataToCache, pCatalogReq, pMetaData, pMetaCache, isInsertValues); + void doPutMetaDataToCache(const SCatalogReq* pCatalogReq, const SMetaData* pMetaData, SParseMetaCache* pMetaCache) { + DO_WITH_THROW(putMetaDataToCache, pCatalogReq, pMetaData, pMetaCache); } void doAuthenticate(SParseContext* pCxt, SQuery* pQuery, SParseMetaCache* pMetaCache) { @@ -280,15 +279,14 @@ class ParserTestBaseImpl { res_.calcConstAst_ = toString(pQuery->pRoot); } - void doParseInsertSql(SParseContext* pCxt, SQuery** pQuery, SParseMetaCache* pMetaCache) { - DO_WITH_THROW(parseInsertSql, pCxt, pQuery, pMetaCache); + void doParseInsertSql(SParseContext* pCxt, SQuery** pQuery, SCatalogReq* pCatalogReq, const SMetaData* pMetaData) { + DO_WITH_THROW(parseInsertSql, pCxt, pQuery, pCatalogReq, pMetaData); ASSERT_NE(*pQuery, nullptr); res_.parsedAst_ = toString((*pQuery)->pRoot); } - void doParseInsertSyntax(SParseContext* pCxt, SQuery** pQuery, SParseMetaCache* pMetaCache) { - DO_WITH_THROW(parseInsertSyntax, pCxt, pQuery, pMetaCache); - ASSERT_NE(*pQuery, nullptr); + void doContinueParseSql(SParseContext* pCxt, SCatalogReq* pCatalogReq, const SMetaData* pMetaData, SQuery* pQuery) { + DO_WITH_THROW(qContinueParseSql, pCxt, pCatalogReq, pMetaData, pQuery); } string toString(const SNode* pRoot) { @@ -310,7 +308,7 @@ class ParserTestBaseImpl { if (qIsInsertValuesSql(cxt.pSql, cxt.sqlLen)) { unique_ptr query((SQuery**)taosMemoryCalloc(1, sizeof(SQuery*)), destroyQuery); - doParseInsertSql(&cxt, query.get(), nullptr); + doParseInsertSql(&cxt, query.get(), nullptr, nullptr); } else { unique_ptr query((SQuery**)taosMemoryCalloc(1, sizeof(SQuery*)), destroyQuery); doParse(&cxt, query.get()); @@ -356,61 +354,102 @@ class ParserTestBaseImpl { } } + void runQueryAsyncInternalFuncs(SParseContext* pParCxt) { + unique_ptr query((SQuery**)taosMemoryCalloc(1, sizeof(SQuery*)), destroyQuery); + bool request = true; + unique_ptr > metaCache( + new SParseMetaCache(), bind(destoryParseMetaCacheWarpper, _1, cref(request))); + doParse(pParCxt, query.get()); + doCollectMetaKey(pParCxt, *(query.get()), metaCache.get()); + + SQuery* pQuery = *(query.get()); + + unique_ptr catalogReq(new SCatalogReq(), + MockCatalogService::destoryCatalogReq); + doBuildCatalogReq(pParCxt, metaCache.get(), catalogReq.get()); + + string err; + thread t1([&]() { + try { + unique_ptr metaData(new SMetaData(), MockCatalogService::destoryMetaData); + doGetAllMeta(catalogReq.get(), metaData.get()); + + metaCache.reset(new SParseMetaCache()); + request = false; + doPutMetaDataToCache(catalogReq.get(), metaData.get(), metaCache.get()); + + doAuthenticate(pParCxt, pQuery, metaCache.get()); + + doTranslate(pParCxt, pQuery, metaCache.get()); + + doCalculateConstant(pParCxt, pQuery); + } catch (const TerminateFlag& e) { + // success and terminate + } catch (const runtime_error& e) { + err = e.what(); + } catch (...) { + err = "unknown error"; + } + }); + + t1.join(); + if (!err.empty()) { + throw runtime_error(err); + } + } + + void runInsertAsyncInternalFuncsImpl(SParseContext* pParCxt, SQuery** pQuery, SCatalogReq* pCatalogReq, + SMetaData* pMetaData) { + doParseInsertSql(pParCxt, pQuery, pCatalogReq, pMetaData); + + if (QUERY_EXEC_STAGE_SCHEDULE == (*pQuery)->execStage) { + return; + } + + string err; + thread t1([&]() { + try { + doGetAllMeta(pCatalogReq, pMetaData); + + doParseInsertSql(pParCxt, pQuery, pCatalogReq, pMetaData); + + if (QUERY_EXEC_STAGE_SCHEDULE != (*pQuery)->execStage) { + runInsertAsyncInternalFuncsImpl(pParCxt, pQuery, pCatalogReq, pMetaData); + } + } catch (const TerminateFlag& e) { + // success and terminate + } catch (const runtime_error& e) { + err = e.what(); + } catch (...) { + err = "unknown error"; + } + }); + + t1.join(); + if (!err.empty()) { + throw runtime_error(err); + } + } + + void runInsertAsyncInternalFuncs(SParseContext* pParCxt) { + unique_ptr query((SQuery**)taosMemoryCalloc(1, sizeof(SQuery*)), destroyQuery); + unique_ptr catalogReq(new SCatalogReq(), + MockCatalogService::destoryCatalogReq); + unique_ptr metaData(new SMetaData(), MockCatalogService::destoryMetaData); + runInsertAsyncInternalFuncsImpl(pParCxt, query.get(), catalogReq.get(), metaData.get()); + } + void runAsyncInternalFuncs(const string& sql, int32_t expect, ParserStage checkStage) { reset(expect, checkStage, TEST_INTERFACE_ASYNC_INTERNAL); try { unique_ptr > cxt(new SParseContext(), destoryParseContext); setParseContext(sql, cxt.get(), true); - unique_ptr query((SQuery**)taosMemoryCalloc(1, sizeof(SQuery*)), destroyQuery); - bool request = true; - unique_ptr > metaCache( - new SParseMetaCache(), bind(destoryParseMetaCacheWarpper, _1, cref(request))); bool isInsertValues = qIsInsertValuesSql(cxt->pSql, cxt->sqlLen); if (isInsertValues) { - doParseInsertSyntax(cxt.get(), query.get(), metaCache.get()); + runInsertAsyncInternalFuncs(cxt.get()); } else { - doParse(cxt.get(), query.get()); - doCollectMetaKey(cxt.get(), *(query.get()), metaCache.get()); - } - - SQuery* pQuery = *(query.get()); - - unique_ptr catalogReq(new SCatalogReq(), - MockCatalogService::destoryCatalogReq); - doBuildCatalogReq(cxt.get(), metaCache.get(), catalogReq.get()); - - string err; - thread t1([&]() { - try { - unique_ptr metaData(new SMetaData(), MockCatalogService::destoryMetaData); - doGetAllMeta(catalogReq.get(), metaData.get()); - - metaCache.reset(new SParseMetaCache()); - request = false; - doPutMetaDataToCache(catalogReq.get(), metaData.get(), metaCache.get(), isInsertValues); - - if (isInsertValues) { - doParseInsertSql(cxt.get(), query.get(), metaCache.get()); - } else { - doAuthenticate(cxt.get(), pQuery, metaCache.get()); - - doTranslate(cxt.get(), pQuery, metaCache.get()); - - doCalculateConstant(cxt.get(), pQuery); - } - } catch (const TerminateFlag& e) { - // success and terminate - } catch (const runtime_error& e) { - err = e.what(); - } catch (...) { - err = "unknown error"; - } - }); - - t1.join(); - if (!err.empty()) { - throw runtime_error(err); + runQueryAsyncInternalFuncs(cxt.get()); } if (g_dump) { @@ -437,25 +476,39 @@ class ParserTestBaseImpl { doParseSqlSyntax(cxt.get(), query.get(), catalogReq.get()); SQuery* pQuery = *(query.get()); - string err; - thread t1([&]() { - try { - unique_ptr metaData(new SMetaData(), MockCatalogService::destoryMetaData); - doGetAllMeta(catalogReq.get(), metaData.get()); + switch (pQuery->execStage) { + case QUERY_EXEC_STAGE_PARSE: + case QUERY_EXEC_STAGE_ANALYSE: { + string err; + thread t1([&]() { + try { + unique_ptr metaData(new SMetaData(), + MockCatalogService::destoryMetaData); + doGetAllMeta(catalogReq.get(), metaData.get()); + if (QUERY_EXEC_STAGE_PARSE == pQuery->execStage) { + doContinueParseSql(cxt.get(), catalogReq.get(), metaData.get(), pQuery); + } else { + doAnalyseSqlSemantic(cxt.get(), catalogReq.get(), metaData.get(), pQuery); + } + } catch (const TerminateFlag& e) { + // success and terminate + } catch (const runtime_error& e) { + err = e.what(); + } catch (...) { + err = "unknown error"; + } + }); - doAnalyseSqlSemantic(cxt.get(), catalogReq.get(), metaData.get(), pQuery); - } catch (const TerminateFlag& e) { - // success and terminate - } catch (const runtime_error& e) { - err = e.what(); - } catch (...) { - err = "unknown error"; + t1.join(); + if (!err.empty()) { + throw runtime_error(err); + } + break; } - }); - - t1.join(); - if (!err.empty()) { - throw runtime_error(err); + case QUERY_EXEC_STAGE_SCHEDULE: + break; + default: + break; } if (g_dump) { diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c index c26ae4e646..e07255b66b 100644 --- a/source/libs/scheduler/src/schRemote.c +++ b/source/libs/scheduler/src/schRemote.c @@ -1134,7 +1134,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); } -#if 1 +#if 0 SSchTrans trans = {.pTrans = pJob->conn.pTrans, .pHandle = SCH_GET_TASK_HANDLE(pTask)}; code = schAsyncSendMsg(pJob, pTask, &trans, addr, msgType, msg, (uint32_t)msgSize, persistHandle, (rpcCtx.args ? &rpcCtx : NULL)); msg = NULL; From 9ae008235c0da4c7a6b99bc24ede73940b772396 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Fri, 4 Nov 2022 15:45:47 +0800 Subject: [PATCH 2/9] enh: insert optimize --- source/libs/scheduler/src/schRemote.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c index e07255b66b..ef5989e66d 100644 --- a/source/libs/scheduler/src/schRemote.c +++ b/source/libs/scheduler/src/schRemote.c @@ -1154,6 +1154,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, SCH_ERR_RET(schAppendTaskExecNode(pJob, pTask, addr, pTask->execId)); } } else { + taosMemoryFree(msg); SCH_ERR_RET(schProcessOnTaskSuccess(pJob, pTask)); } #endif From 91da143ae53f43f5f279df2fd71e12958e934b93 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Fri, 4 Nov 2022 18:01:39 +0800 Subject: [PATCH 3/9] enh: insert optimize --- source/libs/parser/src/parInsertSql.c | 9 +++------ source/libs/scheduler/src/schRemote.c | 2 +- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index d9bfee24d1..af8567a92e 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -556,12 +556,8 @@ static int32_t parseTagValue(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt } static void buildCreateTbReq(SVnodeModifOpStmt* pStmt, STag* pTag, SArray* pTagName) { - char tbFName[TSDB_TABLE_FNAME_LEN]; - tNameExtractFullName(&pStmt->targetTableName, tbFName); - char stbFName[TSDB_TABLE_FNAME_LEN]; - tNameExtractFullName(&pStmt->usingTableName, stbFName); - insBuildCreateTbReq(&pStmt->createTblReq, tbFName, pTag, pStmt->pTableMeta->suid, stbFName, pTagName, - pStmt->pTableMeta->tableInfo.numOfTags); + insBuildCreateTbReq(&pStmt->createTblReq, pStmt->targetTableName.tname, pTag, pStmt->pTableMeta->suid, + pStmt->usingTableName.tname, pTagName, pStmt->pTableMeta->tableInfo.numOfTags); } static int32_t checkAndTrimValue(SToken* pToken, char* tmpTokenBuf, SMsgBuf* pMsgBuf) { @@ -671,6 +667,7 @@ static int32_t parseTagsClause(SInsertParseContext* pCxt, SVnodeModifOpStmt* pSt } static int32_t storeTableMeta(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { + pStmt->pTableMeta->suid = pStmt->pTableMeta->uid; pStmt->pTableMeta->uid = pStmt->totalTbNum; pStmt->pTableMeta->tableType = TSDB_CHILD_TABLE; diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c index ef5989e66d..15e3a8c027 100644 --- a/source/libs/scheduler/src/schRemote.c +++ b/source/libs/scheduler/src/schRemote.c @@ -1134,7 +1134,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); } -#if 0 +#if 1 SSchTrans trans = {.pTrans = pJob->conn.pTrans, .pHandle = SCH_GET_TASK_HANDLE(pTask)}; code = schAsyncSendMsg(pJob, pTask, &trans, addr, msgType, msg, (uint32_t)msgSize, persistHandle, (rpcCtx.args ? &rpcCtx : NULL)); msg = NULL; From 7484ff430cd411ace3604969c9701d2745b63172 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Sat, 5 Nov 2022 12:09:11 +0800 Subject: [PATCH 4/9] enh: insert optimize --- include/libs/catalog/catalog.h | 17 +++---- source/libs/parser/src/parInsertSql.c | 66 ++++++++++++++------------- 2 files changed, 41 insertions(+), 42 deletions(-) diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index 96b49b5dc4..3902224f0b 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -186,8 +186,6 @@ int32_t catalogRemoveStbMeta(SCatalog* pCtg, const char* dbFName, uint64_t dbId, */ int32_t catalogGetTableMeta(SCatalog* pCatalog, SRequestConnInfo* pConn, const SName* pTableName, STableMeta** pTableMeta); -int32_t catalogGetCachedTableMeta(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, - STableMeta** pTableMeta); /** * Get a super table's meta data. @@ -200,8 +198,6 @@ int32_t catalogGetCachedTableMeta(SCatalog* pCtg, SRequestConnInfo* pConn, const */ int32_t catalogGetSTableMeta(SCatalog* pCatalog, SRequestConnInfo* pConn, const SName* pTableName, STableMeta** pTableMeta); -int32_t catalogGetCachedSTableMeta(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, - STableMeta** pTableMeta); int32_t catalogUpdateTableMeta(SCatalog* pCatalog, STableMetaRsp* rspMsg); @@ -211,7 +207,7 @@ int32_t catalogGetCachedTableMeta(SCatalog* pCtg, const SName* pTableName, STabl int32_t catalogGetCachedSTableMeta(SCatalog* pCtg, const SName* pTableName, STableMeta** pTableMeta); -int32_t catalogGetCachedTableHashVgroup(SCatalog* pCtg, const SName* pTableName, SVgroupInfo* pVgroup, bool* exists); +int32_t catalogGetCachedTableHashVgroup(SCatalog* pCtg, const SName* pTableName, SVgroupInfo* pVgroup, bool* exists); /** * Force refresh DB's local cached vgroup info. @@ -271,8 +267,7 @@ int32_t catalogGetTableDistVgInfo(SCatalog* pCatalog, SRequestConnInfo* pConn, c * @return error code */ int32_t catalogGetTableHashVgroup(SCatalog* pCatalog, SRequestConnInfo* pConn, const SName* pName, SVgroupInfo* vgInfo); -int32_t catalogGetCachedTableHashVgroup(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, - SVgroupInfo* pVgroup, bool* exists); + /** * Get all meta data required in pReq. * @param pCatalog (input, got with catalogGetHandle) @@ -312,8 +307,8 @@ int32_t catalogGetUdfInfo(SCatalog* pCtg, SRequestConnInfo* pConn, const char* f int32_t catalogChkAuth(SCatalog* pCtg, SRequestConnInfo* pConn, const char* user, const char* dbFName, AUTH_TYPE type, bool* pass); -int32_t catalogChkAuthFromCache(SCatalog* pCtg, const char* user, const char* dbFName, AUTH_TYPE type, - bool* pass, bool* exists); +int32_t catalogChkAuthFromCache(SCatalog* pCtg, const char* user, const char* dbFName, AUTH_TYPE type, bool* pass, + bool* exists); int32_t catalogUpdateUserAuthInfo(SCatalog* pCtg, SGetUserAuthRsp* pAuth); @@ -329,9 +324,9 @@ SMetaData* catalogCloneMetaData(SMetaData* pData); void catalogFreeMetaData(SMetaData* pData); -int32_t ctgdEnableDebug(char *option, bool enable); +int32_t ctgdEnableDebug(char* option, bool enable); -int32_t ctgdHandleDbgCommand(char *command); +int32_t ctgdHandleDbgCommand(char* command); /** * Destroy catalog and relase all resources diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index af8567a92e..cb7d8f8e88 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -740,42 +740,46 @@ static int32_t parseUsingClauseBottom(SInsertParseContext* pCxt, SVnodeModifOpSt return code; } -static int32_t checkAuth(SParseContext* pCxt, SName* pTbName) { +static int32_t checkAuth(SParseContext* pCxt, SName* pTbName, bool* pMissCache) { char dbFName[TSDB_DB_FNAME_LEN]; tNameGetFullDbName(pTbName, dbFName); - SRequestConnInfo conn = {.pTrans = pCxt->pTransporter, - .requestId = pCxt->requestId, - .requestObjRefId = pCxt->requestRid, - .mgmtEps = pCxt->mgmtEpSet}; - int32_t code = TSDB_CODE_SUCCESS; - bool pass = true; + int32_t code = TSDB_CODE_SUCCESS; + bool pass = true; + bool exists = true; if (pCxt->async) { - // todo replace with cached api - code = catalogChkAuth(pCxt->pCatalog, &conn, pCxt->pUser, dbFName, AUTH_TYPE_WRITE, &pass); + code = catalogChkAuthFromCache(pCxt->pCatalog, pCxt->pUser, dbFName, AUTH_TYPE_WRITE, &pass, &exists); } else { + SRequestConnInfo conn = {.pTrans = pCxt->pTransporter, + .requestId = pCxt->requestId, + .requestObjRefId = pCxt->requestRid, + .mgmtEps = pCxt->mgmtEpSet}; code = catalogChkAuth(pCxt->pCatalog, &conn, pCxt->pUser, dbFName, AUTH_TYPE_WRITE, &pass); } - if (TSDB_CODE_SUCCESS == code && !pass) { - code = TSDB_CODE_PAR_PERMISSION_DENIED; + if (TSDB_CODE_SUCCESS == code) { + if (!exists) { + *pMissCache = true; + } else if (!pass) { + code = TSDB_CODE_PAR_PERMISSION_DENIED; + } } return code; } static int32_t getTableMeta(SInsertParseContext* pCxt, SName* pTbName, bool isStb, STableMeta** pTableMeta, bool* pMissCache) { - SParseContext* pComCxt = pCxt->pComCxt; - SRequestConnInfo conn = {.pTrans = pComCxt->pTransporter, - .requestId = pComCxt->requestId, - .requestObjRefId = pComCxt->requestRid, - .mgmtEps = pComCxt->mgmtEpSet}; - int32_t code = TSDB_CODE_SUCCESS; + SParseContext* pComCxt = pCxt->pComCxt; + int32_t code = TSDB_CODE_SUCCESS; if (pComCxt->async) { if (isStb) { - code = catalogGetCachedSTableMeta(pComCxt->pCatalog, &conn, pTbName, pTableMeta); + code = catalogGetCachedSTableMeta(pComCxt->pCatalog, pTbName, pTableMeta); } else { - code = catalogGetCachedTableMeta(pComCxt->pCatalog, &conn, pTbName, pTableMeta); + code = catalogGetCachedTableMeta(pComCxt->pCatalog, pTbName, pTableMeta); } } else { + SRequestConnInfo conn = {.pTrans = pComCxt->pTransporter, + .requestId = pComCxt->requestId, + .requestObjRefId = pComCxt->requestRid, + .mgmtEps = pComCxt->mgmtEpSet}; if (isStb) { code = catalogGetSTableMeta(pComCxt->pCatalog, &conn, pTbName, pTableMeta); } else { @@ -793,16 +797,16 @@ static int32_t getTableMeta(SInsertParseContext* pCxt, SName* pTbName, bool isSt } static int32_t getTableVgroup(SParseContext* pCxt, SVnodeModifOpStmt* pStmt, bool isStb, bool* pMissCache) { - SRequestConnInfo conn = {.pTrans = pCxt->pTransporter, - .requestId = pCxt->requestId, - .requestObjRefId = pCxt->requestRid, - .mgmtEps = pCxt->mgmtEpSet}; - int32_t code = TSDB_CODE_SUCCESS; - SVgroupInfo vg; - bool exists = true; + int32_t code = TSDB_CODE_SUCCESS; + SVgroupInfo vg; + bool exists = true; if (pCxt->async) { - code = catalogGetCachedTableHashVgroup(pCxt->pCatalog, &conn, &pStmt->targetTableName, &vg, &exists); + code = catalogGetCachedTableHashVgroup(pCxt->pCatalog, &pStmt->targetTableName, &vg, &exists); } else { + SRequestConnInfo conn = {.pTrans = pCxt->pTransporter, + .requestId = pCxt->requestId, + .requestObjRefId = pCxt->requestRid, + .mgmtEps = pCxt->mgmtEpSet}; code = catalogGetTableHashVgroup(pCxt->pCatalog, &conn, &pStmt->targetTableName, &vg); } if (TSDB_CODE_SUCCESS == code) { @@ -818,8 +822,8 @@ static int32_t getTableVgroup(SParseContext* pCxt, SVnodeModifOpStmt* pStmt, boo } static int32_t getTargetTableSchema(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { - int32_t code = checkAuth(pCxt->pComCxt, &pStmt->targetTableName); - if (TSDB_CODE_SUCCESS == code) { + int32_t code = checkAuth(pCxt->pComCxt, &pStmt->targetTableName, &pCxt->missCache); + if (TSDB_CODE_SUCCESS == code && !pCxt->missCache) { code = getTableMeta(pCxt, &pStmt->targetTableName, false, &pStmt->pTableMeta, &pCxt->missCache); } if (TSDB_CODE_SUCCESS == code && !pCxt->missCache) { @@ -833,8 +837,8 @@ static int32_t preParseUsingTableName(SInsertParseContext* pCxt, SVnodeModifOpSt } static int32_t getUsingTableSchema(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { - int32_t code = checkAuth(pCxt->pComCxt, &pStmt->targetTableName); - if (TSDB_CODE_SUCCESS == code) { + int32_t code = checkAuth(pCxt->pComCxt, &pStmt->targetTableName, &pCxt->missCache); + if (TSDB_CODE_SUCCESS == code && !pCxt->missCache) { code = getTableMeta(pCxt, &pStmt->usingTableName, true, &pStmt->pTableMeta, &pCxt->missCache); } if (TSDB_CODE_SUCCESS == code && !pCxt->missCache) { From ac2ce21414d6f2ac7c0e6e6ddb200560e5d5f716 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Sun, 6 Nov 2022 14:26:19 +0800 Subject: [PATCH 5/9] enh: insert optimize --- source/libs/parser/src/parInsertSql.c | 71 ++++++++++++++++++------- source/libs/parser/test/mockCatalog.cpp | 14 +++-- 2 files changed, 63 insertions(+), 22 deletions(-) diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index cb7d8f8e88..ce75e4ff8a 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -43,6 +43,7 @@ typedef struct SInsertParseContext { char tmpTokenBuf[TSDB_MAX_BYTES_PER_ROW]; SParsedDataColInfo tags; // for stmt bool missCache; + bool usingDuplicateTable; } SInsertParseContext; typedef int32_t (*_row_append_fn_t)(SMsgBuf* pMsgBuf, const void* value, int32_t len, void* param); @@ -725,7 +726,7 @@ static int32_t parseTableOptions(SInsertParseContext* pCxt, SVnodeModifOpStmt* p // 1. [(field1_name, ...)] // 2. VALUES ... | FILE ... static int32_t parseUsingClauseBottom(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { - if ('\0' == pStmt->usingTableName.tname[0]) { + if (!pStmt->usingTableProcessing || pCxt->usingDuplicateTable) { return TSDB_CODE_SUCCESS; } @@ -854,7 +855,7 @@ static int32_t parseUsingTableNameImpl(SInsertParseContext* pCxt, SVnodeModifOpS if (TSDB_CODE_SUCCESS == code) { code = getUsingTableSchema(pCxt, pStmt); } - if (TSDB_CODE_SUCCESS == code) { + if (TSDB_CODE_SUCCESS == code && !pCxt->missCache) { code = storeTableMeta(pCxt, pStmt); } return code; @@ -874,11 +875,11 @@ static int32_t parseUsingTableName(SInsertParseContext* pCxt, SVnodeModifOpStmt* return getTargetTableSchema(pCxt, pStmt); } + pStmt->usingTableProcessing = true; // pStmt->pSql -> stb_name [(tag1_name, ...) pStmt->pSql += index; - bool duplicate = false; - int32_t code = parseDuplicateUsingClause(pCxt, pStmt, &duplicate); - if (TSDB_CODE_SUCCESS == code && !duplicate) { + int32_t code = parseDuplicateUsingClause(pCxt, pStmt, &pCxt->usingDuplicateTable); + if (TSDB_CODE_SUCCESS == code && !pCxt->usingDuplicateTable) { return parseUsingTableNameImpl(pCxt, pStmt); } return code; @@ -1331,6 +1332,7 @@ static int32_t parseCsvFile(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, (*pNumOfRows) = 0; char* pLine = NULL; int64_t readLen = 0; + pStmt->fileProcessing = false; while (TSDB_CODE_SUCCESS == code && (readLen = taosGetLineFile(pStmt->fp, &pLine)) != -1) { if (('\r' == pLine[readLen - 1]) || ('\n' == pLine[readLen - 1])) { pLine[--readLen] = '\0'; @@ -1348,7 +1350,8 @@ static int32_t parseCsvFile(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, if (TSDB_CODE_SUCCESS == code) { SToken token; strtolower(pLine, pLine); - code = parseOneRow(pCxt, (const char**)&pLine, pDataBuf, &gotRow, &token); + const char* pRow = pLine; + code = parseOneRow(pCxt, (const char**)&pRow, pDataBuf, &gotRow, &token); } if (TSDB_CODE_SUCCESS == code && gotRow) { @@ -1357,12 +1360,13 @@ static int32_t parseCsvFile(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, } if (TSDB_CODE_SUCCESS == code && pDataBuf->nAllocSize > tsMaxMemUsedByInsert * 1024 * 1024) { + pStmt->fileProcessing = true; break; } } if (TSDB_CODE_SUCCESS == code && 0 == (*pNumOfRows) && - (!TSDB_QUERY_HAS_TYPE(pStmt->insertType, TSDB_QUERY_TYPE_STMT_INSERT))) { + (!TSDB_QUERY_HAS_TYPE(pStmt->insertType, TSDB_QUERY_TYPE_STMT_INSERT)) && !pStmt->fileProcessing) { code = buildSyntaxErrMsg(&pCxt->msg, "no any data points", NULL); } return code; @@ -1383,13 +1387,13 @@ static int32_t parseDataFromFileImpl(SInsertParseContext* pCxt, SVnodeModifOpStm pStmt->totalRowsNum += numOfRows; pStmt->totalTbNum += 1; TSDB_QUERY_SET_TYPE(pStmt->insertType, TSDB_QUERY_TYPE_FILE_INSERT); - if (taosEOFFile(pStmt->fp)) { + if (!pStmt->fileProcessing) { taosCloseFile(&pStmt->fp); } else { parserDebug("0x%" PRIx64 " insert from csv. File is too large, do it in batches.", pCxt->pComCxt->requestId); } } - return TSDB_CODE_SUCCESS; + return code; } static int32_t parseDataFromFile(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, SToken* pFilePath, @@ -1534,6 +1538,10 @@ static void destroyEnvPreTable(SInsertParseContext* pCxt, SVnodeModifOpStmt* pSt destroyBoundColumnInfo(&pCxt->tags); taosMemoryFreeClear(pStmt->pTableMeta); tdDestroySVCreateTbReq(&pStmt->createTblReq); + pCxt->missCache = false; + pCxt->usingDuplicateTable = false; + pStmt->usingTableProcessing = false; + pStmt->fileProcessing = false; } // tb_name @@ -1546,7 +1554,7 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt, SVnodeModifOpStmt* pSt int32_t code = TSDB_CODE_SUCCESS; bool hasData = true; // for each table - while (TSDB_CODE_SUCCESS == code && hasData && !pCxt->missCache) { + while (TSDB_CODE_SUCCESS == code && hasData && !pCxt->missCache && !pStmt->fileProcessing) { destroyEnvPreTable(pCxt, pStmt); // pStmt->pSql -> tb_name ... NEXT_TOKEN(pStmt->pSql, token); @@ -1556,8 +1564,6 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt, SVnodeModifOpStmt* pSt } } - parserDebug("0x%" PRIx64 " insert input rows: %d", pCxt->pComCxt->requestId, pStmt->totalRowsNum); - if (TSDB_CODE_SUCCESS == code && !pCxt->missCache) { code = parseInsertBodyBottom(pCxt, pStmt); } @@ -1670,7 +1676,24 @@ static int32_t getTableSchemaFromMetaData(const SMetaData* pMetaData, SVnodeModi return code; } -static int32_t setVnodeModifOpStmt(SParseContext* pCxt, const SMetaData* pMetaData, SVnodeModifOpStmt* pStmt) { +static void destoryTablesReq(void* p) { + STablesReq* pRes = (STablesReq*)p; + taosArrayDestroy(pRes->pTables); +} + +static void clearCatalogReq(SCatalogReq* pCatalogReq) { + taosArrayDestroyEx(pCatalogReq->pTableMeta, destoryTablesReq); + pCatalogReq->pTableMeta = NULL; + taosArrayDestroyEx(pCatalogReq->pTableHash, destoryTablesReq); + pCatalogReq->pTableHash = NULL; + taosArrayDestroy(pCatalogReq->pUser); + pCatalogReq->pUser = NULL; +} + +static int32_t setVnodeModifOpStmt(SParseContext* pCxt, SCatalogReq* pCatalogReq, const SMetaData* pMetaData, + SVnodeModifOpStmt* pStmt) { + clearCatalogReq(pCatalogReq); + if (pCxt->pStmtCb) { (*pCxt->pStmtCb->getExecInfoFn)(pCxt->pStmtCb->pStmt, &pStmt->pVgroupsHashObj, &pStmt->pTableBlockHashObj); if (NULL == pStmt->pVgroupsHashObj) { @@ -1690,11 +1713,17 @@ static int32_t setVnodeModifOpStmt(SParseContext* pCxt, const SMetaData* pMetaDa return getTableSchemaFromMetaData(pMetaData, pStmt, false); } -static int32_t initInsertQuery(SParseContext* pCxt, const SMetaData* pMetaData, SQuery** pQuery) { +static int32_t initInsertQuery(SParseContext* pCxt, SCatalogReq* pCatalogReq, const SMetaData* pMetaData, + SQuery** pQuery) { if (NULL == *pQuery) { return createInsertQuery(pCxt, pQuery); } - return setVnodeModifOpStmt(pCxt, pMetaData, (SVnodeModifOpStmt*)(*pQuery)->pRoot); + + if (!((SVnodeModifOpStmt*)(*pQuery)->pRoot)->fileProcessing) { + return setVnodeModifOpStmt(pCxt, pCatalogReq, pMetaData, (SVnodeModifOpStmt*)(*pQuery)->pRoot); + } + + return TSDB_CODE_SUCCESS; } static int32_t setRefreshMate(SQuery* pQuery) { @@ -1735,8 +1764,6 @@ static int32_t parseInsertSqlFromCsv(SInsertParseContext* pCxt, SVnodeModifOpStm code = parseDataFromFileImpl(pCxt, pStmt, pDataBuf); } - parserDebug("0x%" PRIx64 " insert again input rows: %d", pCxt->pComCxt->requestId, pStmt->totalRowsNum); - if (TSDB_CODE_SUCCESS == code) { if (pStmt->fileProcessing) { code = parseInsertBodyBottom(pCxt, pStmt); @@ -1825,10 +1852,16 @@ static int32_t buildInsertCatalogReq(SInsertParseContext* pCxt, SVnodeModifOpStm static int32_t setNextStageInfo(SInsertParseContext* pCxt, SQuery* pQuery, SCatalogReq* pCatalogReq) { if (pCxt->missCache) { + parserDebug("0x%" PRIx64 " %d rows have been inserted before cache miss", pCxt->pComCxt->requestId, + ((SVnodeModifOpStmt*)pQuery->pRoot)->totalRowsNum); + pQuery->execStage = QUERY_EXEC_STAGE_PARSE; return buildInsertCatalogReq(pCxt, (SVnodeModifOpStmt*)pQuery->pRoot, pCatalogReq); } + parserDebug("0x%" PRIx64 " %d rows have been inserted", pCxt->pComCxt->requestId, + ((SVnodeModifOpStmt*)pQuery->pRoot)->totalRowsNum); + pQuery->execStage = QUERY_EXEC_STAGE_SCHEDULE; return TSDB_CODE_SUCCESS; } @@ -1837,9 +1870,11 @@ int32_t parseInsertSql(SParseContext* pCxt, SQuery** pQuery, SCatalogReq* pCatal SInsertParseContext context = { .pComCxt = pCxt, .msg = {.buf = pCxt->pMsg, .len = pCxt->msgLen}, + .missCache = false, + .usingDuplicateTable = false, }; - int32_t code = initInsertQuery(pCxt, pMetaData, pQuery); + int32_t code = initInsertQuery(pCxt, pCatalogReq, pMetaData, pQuery); if (TSDB_CODE_SUCCESS == code) { code = parseInsertSqlImpl(&context, (SVnodeModifOpStmt*)(*pQuery)->pRoot); } diff --git a/source/libs/parser/test/mockCatalog.cpp b/source/libs/parser/test/mockCatalog.cpp index 96ba8ab273..4f5ddd9a51 100644 --- a/source/libs/parser/test/mockCatalog.cpp +++ b/source/libs/parser/test/mockCatalog.cpp @@ -228,8 +228,7 @@ int32_t __catalogGetTableMeta(struct SCatalog* pCatalog, SRequestConnInfo* pConn return g_mockCatalogService->catalogGetTableMeta(pTableName, pTableMeta); } -int32_t __catalogGetCachedTableMeta(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, - STableMeta** pTableMeta) { +int32_t __catalogGetCachedTableMeta(SCatalog* pCtg, const SName* pTableName, STableMeta** pTableMeta) { return g_mockCatalogService->catalogGetTableMeta(pTableName, pTableMeta, true); } @@ -238,8 +237,7 @@ int32_t __catalogGetTableHashVgroup(struct SCatalog* pCatalog, SRequestConnInfo* return g_mockCatalogService->catalogGetTableHashVgroup(pTableName, vgInfo); } -int32_t __catalogGetCachedTableHashVgroup(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, - SVgroupInfo* pVgroup, bool* exists) { +int32_t __catalogGetCachedTableHashVgroup(SCatalog* pCtg, const SName* pTableName, SVgroupInfo* pVgroup, bool* exists) { int32_t code = g_mockCatalogService->catalogGetTableHashVgroup(pTableName, pVgroup, true); *exists = 0 != pVgroup->vgId; return code; @@ -269,6 +267,13 @@ int32_t __catalogChkAuth(SCatalog* pCtg, SRequestConnInfo* pConn, const char* us return 0; } +int32_t __catalogChkAuthFromCache(SCatalog* pCtg, const char* user, const char* dbFName, AUTH_TYPE type, bool* pass, + bool* exists) { + *pass = true; + *exists = true; + return 0; +} + int32_t __catalogGetUdfInfo(SCatalog* pCtg, SRequestConnInfo* pConn, const char* funcName, SFuncInfo* pInfo) { return g_mockCatalogService->catalogGetUdfInfo(funcName, pInfo); } @@ -311,6 +316,7 @@ void initMetaDataEnv() { stub.set(catalogGetDBVgList, __catalogGetDBVgList); stub.set(catalogGetDBCfg, __catalogGetDBCfg); stub.set(catalogChkAuth, __catalogChkAuth); + stub.set(catalogChkAuthFromCache, __catalogChkAuthFromCache); stub.set(catalogGetUdfInfo, __catalogGetUdfInfo); stub.set(catalogRefreshGetTableMeta, __catalogRefreshGetTableMeta); stub.set(catalogRemoveTableMeta, __catalogRemoveTableMeta); From b7bc828172420c95d13f7b72587d01b3aaf5010a Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Sun, 6 Nov 2022 18:42:53 +0800 Subject: [PATCH 6/9] enh: insert optimize --- source/client/src/clientImpl.c | 5 +-- source/libs/parser/inc/parInsertUtil.h | 2 +- source/libs/parser/src/parInsertSql.c | 31 ++++++++--------- source/libs/parser/src/parInsertStmt.c | 17 +++++---- source/libs/parser/src/parInsertUtil.c | 18 ++++++---- tests/system-test/output.txt | 48 ++++++++++++++++++++++++++ 6 files changed, 85 insertions(+), 36 deletions(-) create mode 100644 tests/system-test/output.txt diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 367fde355c..a32ebc8694 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -931,7 +931,8 @@ void schedulerExecCb(SExecResult* pResult, void* param, int32_t code) { pRequest->code = code1; } - if (pRequest->code == TSDB_CODE_SUCCESS && incompletaFileParsing(pRequest->pQuery->pRoot)) { + if (pRequest->code == TSDB_CODE_SUCCESS && NULL != pRequest->pQuery && + incompletaFileParsing(pRequest->pQuery->pRoot)) { continueInsertFromCsv(pWrapper, pRequest); return; } @@ -1057,7 +1058,7 @@ static int32_t asyncExecSchQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaDat } if (TSDB_CODE_SUCCESS == code && !pRequest->validateOnly) { SArray* pNodeList = NULL; - if (QUERY_NODE_VNODE_MODIF_STMT != nodeType(pRequest->pQuery->pRoot)) { + if (QUERY_NODE_VNODE_MODIF_STMT != nodeType(pQuery->pRoot)) { buildAsyncExecNodeList(pRequest, &pNodeList, pMnodeList, pResultMeta); } diff --git a/source/libs/parser/inc/parInsertUtil.h b/source/libs/parser/inc/parInsertUtil.h index f747fdc2c9..09d55d369f 100644 --- a/source/libs/parser/inc/parInsertUtil.h +++ b/source/libs/parser/inc/parInsertUtil.h @@ -138,7 +138,7 @@ void insBuildCreateTbReq(SVCreateTbReq *pTbReq, const char *tname, STag *pTag SArray *tagName, uint8_t tagNum); int32_t insMemRowAppend(SMsgBuf *pMsgBuf, const void *value, int32_t len, void *param); int32_t insCheckTimestamp(STableDataBlocks *pDataBlocks, const char *start); -int32_t insBuildOutput(SVnodeModifOpStmt *pStmt); +int32_t insBuildOutput(SHashObj *pVgroupsHashObj, SArray *pVgDataBlocks, SArray **pDataBlocks); void insDestroyDataBlock(STableDataBlocks *pDataBlock); #endif // TDENGINE_PAR_INSERT_UTIL_H diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index ce75e4ff8a..fe12332840 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -1448,8 +1448,19 @@ static int32_t parseInsertTableClauseBottom(SInsertParseContext* pCxt, SVnodeMod return code; } +static void resetEnvPreTable(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { + destroyBoundColumnInfo(&pCxt->tags); + taosMemoryFreeClear(pStmt->pTableMeta); + tdDestroySVCreateTbReq(&pStmt->createTblReq); + pCxt->missCache = false; + pCxt->usingDuplicateTable = false; + pStmt->usingTableProcessing = false; + pStmt->fileProcessing = false; +} + // input pStmt->pSql: [(field1_name, ...)] [ USING ... ] VALUES ... | FILE ... static int32_t parseInsertTableClause(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, SToken* pTbName) { + resetEnvPreTable(pCxt, pStmt); int32_t code = parseSchemaClauseTop(pCxt, pStmt, pTbName); if (TSDB_CODE_SUCCESS == code && !pCxt->missCache) { code = parseInsertTableClauseBottom(pCxt, pStmt); @@ -1506,11 +1517,8 @@ static int32_t setStmtInfo(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) SStmtCallback* pStmtCb = pCxt->pComCxt->pStmtCb; char tbFName[TSDB_TABLE_FNAME_LEN]; tNameExtractFullName(&pStmt->targetTableName, tbFName); - char stbFName[TSDB_TABLE_FNAME_LEN]; - tNameExtractFullName(&pStmt->usingTableName, stbFName); - int32_t code = - (*pStmtCb->setInfoFn)(pStmtCb->pStmt, pStmt->pTableMeta, tags, tbFName, '\0' != pStmt->usingTableName.tname[0], - pStmt->pVgroupsHashObj, pStmt->pTableBlockHashObj, stbFName); + int32_t code = (*pStmtCb->setInfoFn)(pStmtCb->pStmt, pStmt->pTableMeta, tags, tbFName, pStmt->usingTableProcessing, + pStmt->pVgroupsHashObj, pStmt->pTableBlockHashObj, pStmt->usingTableName.tname); memset(&pCxt->tags, 0, sizeof(pCxt->tags)); pStmt->pVgroupsHashObj = NULL; @@ -1529,21 +1537,11 @@ static int32_t parseInsertBodyBottom(SInsertParseContext* pCxt, SVnodeModifOpStm code = insMergeTableDataBlocks(pStmt->pTableBlockHashObj, &pStmt->pVgDataBlocks); } if (TSDB_CODE_SUCCESS == code) { - code = insBuildOutput(pStmt); + code = insBuildOutput(pStmt->pVgroupsHashObj, pStmt->pVgDataBlocks, &pStmt->pDataBlocks); } return code; } -static void destroyEnvPreTable(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { - destroyBoundColumnInfo(&pCxt->tags); - taosMemoryFreeClear(pStmt->pTableMeta); - tdDestroySVCreateTbReq(&pStmt->createTblReq); - pCxt->missCache = false; - pCxt->usingDuplicateTable = false; - pStmt->usingTableProcessing = false; - pStmt->fileProcessing = false; -} - // tb_name // [USING stb_name [(tag1_name, ...)] TAGS (tag1_value, ...)] // [(field1_name, ...)] @@ -1555,7 +1553,6 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt, SVnodeModifOpStmt* pSt bool hasData = true; // for each table while (TSDB_CODE_SUCCESS == code && hasData && !pCxt->missCache && !pStmt->fileProcessing) { - destroyEnvPreTable(pCxt, pStmt); // pStmt->pSql -> tb_name ... NEXT_TOKEN(pStmt->pSql, token); code = checkTableClauseFirstToken(pCxt, pStmt, &token, &hasData); diff --git a/source/libs/parser/src/parInsertStmt.c b/source/libs/parser/src/parInsertStmt.c index 73557c067e..9a5f349d8f 100644 --- a/source/libs/parser/src/parInsertStmt.c +++ b/source/libs/parser/src/parInsertStmt.c @@ -30,18 +30,17 @@ typedef struct SKvParam { } SKvParam; int32_t qBuildStmtOutput(SQuery* pQuery, SHashObj* pVgHash, SHashObj* pBlockHash) { - SVnodeModifOpStmt* modifyNode = (SVnodeModifOpStmt*)pQuery->pRoot; - int32_t code = 0; - + int32_t code = TSDB_CODE_SUCCESS; + SArray* pVgDataBlocks = NULL; // merge according to vgId if (taosHashGetSize(pBlockHash) > 0) { - CHECK_CODE(insMergeTableDataBlocks(pBlockHash, &modifyNode->pVgDataBlocks)); + code = insMergeTableDataBlocks(pBlockHash, &pVgDataBlocks); } - - CHECK_CODE(insBuildOutput(modifyNode)); - - insDestroyBlockArrayList(modifyNode->pVgDataBlocks); - return TSDB_CODE_SUCCESS; + if (TSDB_CODE_SUCCESS == code) { + code = insBuildOutput(pVgHash, pVgDataBlocks, &((SVnodeModifOpStmt*)pQuery->pRoot)->pDataBlocks); + } + insDestroyBlockArrayList(pVgDataBlocks); + return code; } int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, const char* sTableName, char* tName, diff --git a/source/libs/parser/src/parInsertUtil.c b/source/libs/parser/src/parInsertUtil.c index a0a24d3d31..bc09163753 100644 --- a/source/libs/parser/src/parInsertUtil.c +++ b/source/libs/parser/src/parInsertUtil.c @@ -832,6 +832,10 @@ int32_t insCreateSName(SName* pName, SToken* pTableName, int32_t acctId, const c } } + if (NULL != strchr(pName->tname, '.')) { + code = generateSyntaxErrMsgExt(pMsgBuf, TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME, "The table name cannot contain '.'"); + } + return code; } @@ -930,24 +934,24 @@ static void buildMsgHeader(STableDataBlocks* src, SVgDataBlocks* blocks) { } } -int32_t insBuildOutput(SVnodeModifOpStmt* pStmt) { - size_t numOfVg = taosArrayGetSize(pStmt->pVgDataBlocks); - pStmt->pDataBlocks = taosArrayInit(numOfVg, POINTER_BYTES); - if (NULL == pStmt->pDataBlocks) { +int32_t insBuildOutput(SHashObj* pVgroupsHashObj, SArray* pVgDataBlocks, SArray** pDataBlocks) { + size_t numOfVg = taosArrayGetSize(pVgDataBlocks); + *pDataBlocks = taosArrayInit(numOfVg, POINTER_BYTES); + if (NULL == *pDataBlocks) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } for (size_t i = 0; i < numOfVg; ++i) { - STableDataBlocks* src = taosArrayGetP(pStmt->pVgDataBlocks, i); + STableDataBlocks* src = taosArrayGetP(pVgDataBlocks, i); SVgDataBlocks* dst = taosMemoryCalloc(1, sizeof(SVgDataBlocks)); if (NULL == dst) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } - taosHashGetDup(pStmt->pVgroupsHashObj, (const char*)&src->vgId, sizeof(src->vgId), &dst->vg); + taosHashGetDup(pVgroupsHashObj, (const char*)&src->vgId, sizeof(src->vgId), &dst->vg); dst->numOfTables = src->numOfTables; dst->size = src->size; TSWAP(dst->pData, src->pData); buildMsgHeader(src, dst); - taosArrayPush(pStmt->pDataBlocks, &dst); + taosArrayPush(*pDataBlocks, &dst); } return TSDB_CODE_SUCCESS; } diff --git a/tests/system-test/output.txt b/tests/system-test/output.txt new file mode 100644 index 0000000000..df83ae9acb --- /dev/null +++ b/tests/system-test/output.txt @@ -0,0 +1,48 @@ +[11/06 18:32:29.554116] INFO: start creating 100 table(s) with 8 thread(s) +[11/06 18:32:29.649584] SUCC: Spent 0.0950 seconds to create 100 table(s) with 8 thread(s), already exist 0 table(s), actual 100 table(s) pre created, 0 table(s) will be auto created +[11/06 18:32:29.687312] SUCC: thread[3] completed total inserted rows: 1300, 48030.74 records/second +[11/06 18:32:29.690040] SUCC: thread[0] completed total inserted rows: 1300, 43537.96 records/second +[11/06 18:32:29.691061] SUCC: thread[5] completed total inserted rows: 1200, 39417.93 records/second +[11/06 18:32:29.708308] SUCC: thread[4] completed total inserted rows: 1200, 26671.41 records/second +[11/06 18:32:29.709035] SUCC: thread[6] completed total inserted rows: 1200, 26515.82 records/second +[11/06 18:32:29.709355] SUCC: thread[2] completed total inserted rows: 1300, 27370.73 records/second +[11/06 18:32:29.711754] SUCC: thread[7] completed total inserted rows: 1200, 25266.35 records/second +[11/06 18:32:29.713149] SUCC: thread[1] completed total inserted rows: 1300, 24847.57 records/second +[11/06 18:32:29.714356] SUCC: Spent 0.052822 seconds to insert rows: 10000 with 8 thread(s) into test 189315.06 records/second +[11/06 18:32:29.714381] SUCC: insert delay, min: 0.78ms, avg: 3.25ms, p90: 7.37ms, p95: 10.48ms, p99: 11.98ms, max: 11.98ms +[11/06 18:32:58.075364] INFO: start creating 100 table(s) with 8 thread(s) +[11/06 18:32:58.140297] SUCC: Spent 0.0650 seconds to create 100 table(s) with 8 thread(s), already exist 0 table(s), actual 100 table(s) pre created, 0 table(s) will be auto created +[11/06 18:32:58.212640] SUCC: thread[2] completed total inserted rows: 13000, 214904.45 records/second +[11/06 18:32:58.235314] SUCC: thread[5] completed total inserted rows: 12000, 144468.66 records/second +[11/06 18:32:58.238604] SUCC: thread[4] completed total inserted rows: 12000, 142203.68 records/second +[11/06 18:32:58.239432] SUCC: thread[3] completed total inserted rows: 13000, 148244.44 records/second +[11/06 18:32:58.240457] SUCC: thread[1] completed total inserted rows: 13000, 146629.22 records/second +[11/06 18:32:58.242997] SUCC: thread[6] completed total inserted rows: 12000, 135484.53 records/second +[11/06 18:32:58.244407] SUCC: thread[0] completed total inserted rows: 13000, 139774.43 records/second +[11/06 18:32:58.244679] SUCC: thread[7] completed total inserted rows: 12000, 133324.45 records/second +[11/06 18:32:58.245536] SUCC: Spent 0.093401 seconds to insert rows: 100000 with 8 thread(s) into test 1070652.35 records/second +[11/06 18:32:58.245574] SUCC: insert delay, min: 1.87ms, avg: 6.76ms, p90: 11.67ms, p95: 16.21ms, p99: 20.30ms, max: 20.30ms +[11/06 18:41:03.922840] INFO: start creating 100 table(s) with 8 thread(s) +[11/06 18:41:03.987301] SUCC: Spent 0.0650 seconds to create 100 table(s) with 8 thread(s), already exist 0 table(s), actual 100 table(s) pre created, 0 table(s) will be auto created +[11/06 18:41:04.035594] SUCC: thread[0] completed total inserted rows: 1300, 34073.34 records/second +[11/06 18:41:04.036945] SUCC: thread[2] completed total inserted rows: 1300, 33197.99 records/second +[11/06 18:41:04.043375] SUCC: thread[5] completed total inserted rows: 1200, 27713.63 records/second +[11/06 18:41:04.054865] SUCC: thread[4] completed total inserted rows: 1200, 21176.72 records/second +[11/06 18:41:04.056026] SUCC: thread[7] completed total inserted rows: 1200, 23637.87 records/second +[11/06 18:41:04.056954] SUCC: thread[6] completed total inserted rows: 1200, 20847.45 records/second +[11/06 18:41:04.058160] SUCC: thread[3] completed total inserted rows: 1300, 21564.24 records/second +[11/06 18:41:04.060392] SUCC: thread[1] completed total inserted rows: 1300, 20688.12 records/second +[11/06 18:41:04.061369] SUCC: Spent 0.063170 seconds to insert rows: 10000 with 8 thread(s) into test 158302.99 records/second +[11/06 18:41:04.061385] SUCC: insert delay, min: 0.62ms, avg: 4.09ms, p90: 13.44ms, p95: 19.25ms, p99: 29.64ms, max: 29.64ms +[11/06 18:41:34.386476] INFO: start creating 100 table(s) with 8 thread(s) +[11/06 18:41:34.465414] SUCC: Spent 0.0790 seconds to create 100 table(s) with 8 thread(s), already exist 0 table(s), actual 100 table(s) pre created, 0 table(s) will be auto created +[11/06 18:41:34.546439] SUCC: thread[2] completed total inserted rows: 13000, 186091.79 records/second +[11/06 18:41:34.549390] SUCC: thread[4] completed total inserted rows: 12000, 164774.05 records/second +[11/06 18:41:34.556956] SUCC: thread[5] completed total inserted rows: 12000, 153440.91 records/second +[11/06 18:41:34.563860] SUCC: thread[6] completed total inserted rows: 12000, 138154.94 records/second +[11/06 18:41:34.564020] SUCC: thread[0] completed total inserted rows: 13000, 148469.62 records/second +[11/06 18:41:34.564693] SUCC: thread[3] completed total inserted rows: 13000, 149214.33 records/second +[11/06 18:41:34.571055] SUCC: thread[1] completed total inserted rows: 13000, 140452.47 records/second +[11/06 18:41:34.571921] SUCC: thread[7] completed total inserted rows: 12000, 129424.71 records/second +[11/06 18:41:34.572778] SUCC: Spent 0.095759 seconds to insert rows: 100000 with 8 thread(s) into test 1044288.27 records/second +[11/06 18:41:34.572797] SUCC: insert delay, min: 1.77ms, avg: 6.68ms, p90: 10.72ms, p95: 14.14ms, p99: 17.69ms, max: 17.69ms From 758b4b93a49e85131620fba6143df3222add96ab Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Mon, 7 Nov 2022 11:31:45 +0800 Subject: [PATCH 7/9] enh: insert optimize --- source/libs/parser/src/parInsertSql.c | 54 +++++++++++++++++++-------- 1 file changed, 39 insertions(+), 15 deletions(-) diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index fe12332840..7a38f48cb2 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -1454,6 +1454,7 @@ static void resetEnvPreTable(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt tdDestroySVCreateTbReq(&pStmt->createTblReq); pCxt->missCache = false; pCxt->usingDuplicateTable = false; + pStmt->pBoundCols = NULL; pStmt->usingTableProcessing = false; pStmt->fileProcessing = false; } @@ -1569,7 +1570,7 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt, SVnodeModifOpStmt* pSt static void destroySubTableHashElem(void* p) { taosMemoryFree(*(STableMeta**)p); } -static int32_t createVnodeModifOpStmt(SParseContext* pCxt, SNode** pOutput) { +static int32_t createVnodeModifOpStmt(SParseContext* pCxt, bool reentry, SNode** pOutput) { SVnodeModifOpStmt* pStmt = (SVnodeModifOpStmt*)nodesMakeNode(QUERY_NODE_VNODE_MODIF_STMT); if (NULL == pStmt) { return TSDB_CODE_OUT_OF_MEMORY; @@ -1582,13 +1583,16 @@ static int32_t createVnodeModifOpStmt(SParseContext* pCxt, SNode** pOutput) { pStmt->freeHashFunc = insDestroyBlockHashmap; pStmt->freeArrayFunc = insDestroyBlockArrayList; - pStmt->pVgroupsHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); - pStmt->pTableBlockHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); + if (!reentry) { + pStmt->pVgroupsHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); + pStmt->pTableBlockHashObj = + taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); + } pStmt->pSubTableHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), true, HASH_NO_LOCK); pStmt->pTableNameHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), true, HASH_NO_LOCK); pStmt->pDbFNameHashObj = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), true, HASH_NO_LOCK); - if (NULL == pStmt->pVgroupsHashObj || NULL == pStmt->pTableBlockHashObj || NULL == pStmt->pSubTableHashObj || - NULL == pStmt->pTableNameHashObj || NULL == pStmt->pDbFNameHashObj) { + if ((!reentry && (NULL == pStmt->pVgroupsHashObj || NULL == pStmt->pTableBlockHashObj)) || + NULL == pStmt->pSubTableHashObj || NULL == pStmt->pTableNameHashObj || NULL == pStmt->pDbFNameHashObj) { nodesDestroyNode((SNode*)pStmt); return TSDB_CODE_OUT_OF_MEMORY; } @@ -1609,7 +1613,7 @@ static int32_t createInsertQuery(SParseContext* pCxt, SQuery** pOutput) { pQuery->haveResultSet = false; pQuery->msgType = TDMT_VND_SUBMIT; - int32_t code = createVnodeModifOpStmt(pCxt, &pQuery->pRoot); + int32_t code = createVnodeModifOpStmt(pCxt, false, &pQuery->pRoot); if (TSDB_CODE_SUCCESS == code) { *pOutput = pQuery; } else { @@ -1679,6 +1683,10 @@ static void destoryTablesReq(void* p) { } static void clearCatalogReq(SCatalogReq* pCatalogReq) { + if (NULL == pCatalogReq) { + return; + } + taosArrayDestroyEx(pCatalogReq->pTableMeta, destoryTablesReq); pCatalogReq->pTableMeta = NULL; taosArrayDestroyEx(pCatalogReq->pTableHash, destoryTablesReq); @@ -1691,7 +1699,19 @@ static int32_t setVnodeModifOpStmt(SParseContext* pCxt, SCatalogReq* pCatalogReq SVnodeModifOpStmt* pStmt) { clearCatalogReq(pCatalogReq); - if (pCxt->pStmtCb) { + if (pStmt->usingTableProcessing) { + return getTableSchemaFromMetaData(pMetaData, pStmt, true); + } + return getTableSchemaFromMetaData(pMetaData, pStmt, false); +} + +static int32_t resetVnodeModifOpStmt(SParseContext* pCxt, SQuery* pQuery) { + nodesDestroyNode(pQuery->pRoot); + + int32_t code = createVnodeModifOpStmt(pCxt, true, &pQuery->pRoot); + if (TSDB_CODE_SUCCESS == code) { + SVnodeModifOpStmt* pStmt = (SVnodeModifOpStmt*)pQuery->pRoot; + (*pCxt->pStmtCb->getExecInfoFn)(pCxt->pStmtCb->pStmt, &pStmt->pVgroupsHashObj, &pStmt->pTableBlockHashObj); if (NULL == pStmt->pVgroupsHashObj) { pStmt->pVgroupsHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); @@ -1700,14 +1720,12 @@ static int32_t setVnodeModifOpStmt(SParseContext* pCxt, SCatalogReq* pCatalogReq pStmt->pTableBlockHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); } - - TSDB_QUERY_SET_TYPE(pStmt->insertType, TSDB_QUERY_TYPE_STMT_INSERT); + if (NULL == pStmt->pVgroupsHashObj || NULL == pStmt->pTableBlockHashObj) { + code = TSDB_CODE_OUT_OF_MEMORY; + } } - if (pStmt->usingTableProcessing) { - return getTableSchemaFromMetaData(pMetaData, pStmt, true); - } - return getTableSchemaFromMetaData(pMetaData, pStmt, false); + return code; } static int32_t initInsertQuery(SParseContext* pCxt, SCatalogReq* pCatalogReq, const SMetaData* pMetaData, @@ -1716,8 +1734,14 @@ static int32_t initInsertQuery(SParseContext* pCxt, SCatalogReq* pCatalogReq, co return createInsertQuery(pCxt, pQuery); } - if (!((SVnodeModifOpStmt*)(*pQuery)->pRoot)->fileProcessing) { - return setVnodeModifOpStmt(pCxt, pCatalogReq, pMetaData, (SVnodeModifOpStmt*)(*pQuery)->pRoot); + if (NULL != pCxt->pStmtCb) { + return resetVnodeModifOpStmt(pCxt, *pQuery); + } + + SVnodeModifOpStmt* pStmt = (SVnodeModifOpStmt*)(*pQuery)->pRoot; + + if (!pStmt->fileProcessing) { + return setVnodeModifOpStmt(pCxt, pCatalogReq, pMetaData, pStmt); } return TSDB_CODE_SUCCESS; From f1265cc7a09ff63d52f22cd8418476284944c51d Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Mon, 7 Nov 2022 16:48:53 +0800 Subject: [PATCH 8/9] enh: insert optimize --- source/client/src/clientMain.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index e78f308b67..71a87a4b54 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -801,8 +801,8 @@ static void doAsyncQueryFromParse(SMetaData *pResultMeta, void *param, int32_t c tstrerror(code), pWrapper->pRequest->requestId); destorySqlCallbackWrapper(pWrapper); terrno = code; - pWrapper->pRequest->code = code; - pWrapper->pRequest->body.queryFp(pWrapper->pRequest->body.param, pWrapper->pRequest, code); + pRequest->code = code; + pRequest->body.queryFp(pRequest->body.param, pRequest, code); } } From a196685d9f999cf37941a924045ed15678432b78 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Mon, 7 Nov 2022 18:29:57 +0800 Subject: [PATCH 9/9] enh: insert optimize --- tests/system-test/output.txt | 48 ------------------------------------ 1 file changed, 48 deletions(-) delete mode 100644 tests/system-test/output.txt diff --git a/tests/system-test/output.txt b/tests/system-test/output.txt deleted file mode 100644 index df83ae9acb..0000000000 --- a/tests/system-test/output.txt +++ /dev/null @@ -1,48 +0,0 @@ -[11/06 18:32:29.554116] INFO: start creating 100 table(s) with 8 thread(s) -[11/06 18:32:29.649584] SUCC: Spent 0.0950 seconds to create 100 table(s) with 8 thread(s), already exist 0 table(s), actual 100 table(s) pre created, 0 table(s) will be auto created -[11/06 18:32:29.687312] SUCC: thread[3] completed total inserted rows: 1300, 48030.74 records/second -[11/06 18:32:29.690040] SUCC: thread[0] completed total inserted rows: 1300, 43537.96 records/second -[11/06 18:32:29.691061] SUCC: thread[5] completed total inserted rows: 1200, 39417.93 records/second -[11/06 18:32:29.708308] SUCC: thread[4] completed total inserted rows: 1200, 26671.41 records/second -[11/06 18:32:29.709035] SUCC: thread[6] completed total inserted rows: 1200, 26515.82 records/second -[11/06 18:32:29.709355] SUCC: thread[2] completed total inserted rows: 1300, 27370.73 records/second -[11/06 18:32:29.711754] SUCC: thread[7] completed total inserted rows: 1200, 25266.35 records/second -[11/06 18:32:29.713149] SUCC: thread[1] completed total inserted rows: 1300, 24847.57 records/second -[11/06 18:32:29.714356] SUCC: Spent 0.052822 seconds to insert rows: 10000 with 8 thread(s) into test 189315.06 records/second -[11/06 18:32:29.714381] SUCC: insert delay, min: 0.78ms, avg: 3.25ms, p90: 7.37ms, p95: 10.48ms, p99: 11.98ms, max: 11.98ms -[11/06 18:32:58.075364] INFO: start creating 100 table(s) with 8 thread(s) -[11/06 18:32:58.140297] SUCC: Spent 0.0650 seconds to create 100 table(s) with 8 thread(s), already exist 0 table(s), actual 100 table(s) pre created, 0 table(s) will be auto created -[11/06 18:32:58.212640] SUCC: thread[2] completed total inserted rows: 13000, 214904.45 records/second -[11/06 18:32:58.235314] SUCC: thread[5] completed total inserted rows: 12000, 144468.66 records/second -[11/06 18:32:58.238604] SUCC: thread[4] completed total inserted rows: 12000, 142203.68 records/second -[11/06 18:32:58.239432] SUCC: thread[3] completed total inserted rows: 13000, 148244.44 records/second -[11/06 18:32:58.240457] SUCC: thread[1] completed total inserted rows: 13000, 146629.22 records/second -[11/06 18:32:58.242997] SUCC: thread[6] completed total inserted rows: 12000, 135484.53 records/second -[11/06 18:32:58.244407] SUCC: thread[0] completed total inserted rows: 13000, 139774.43 records/second -[11/06 18:32:58.244679] SUCC: thread[7] completed total inserted rows: 12000, 133324.45 records/second -[11/06 18:32:58.245536] SUCC: Spent 0.093401 seconds to insert rows: 100000 with 8 thread(s) into test 1070652.35 records/second -[11/06 18:32:58.245574] SUCC: insert delay, min: 1.87ms, avg: 6.76ms, p90: 11.67ms, p95: 16.21ms, p99: 20.30ms, max: 20.30ms -[11/06 18:41:03.922840] INFO: start creating 100 table(s) with 8 thread(s) -[11/06 18:41:03.987301] SUCC: Spent 0.0650 seconds to create 100 table(s) with 8 thread(s), already exist 0 table(s), actual 100 table(s) pre created, 0 table(s) will be auto created -[11/06 18:41:04.035594] SUCC: thread[0] completed total inserted rows: 1300, 34073.34 records/second -[11/06 18:41:04.036945] SUCC: thread[2] completed total inserted rows: 1300, 33197.99 records/second -[11/06 18:41:04.043375] SUCC: thread[5] completed total inserted rows: 1200, 27713.63 records/second -[11/06 18:41:04.054865] SUCC: thread[4] completed total inserted rows: 1200, 21176.72 records/second -[11/06 18:41:04.056026] SUCC: thread[7] completed total inserted rows: 1200, 23637.87 records/second -[11/06 18:41:04.056954] SUCC: thread[6] completed total inserted rows: 1200, 20847.45 records/second -[11/06 18:41:04.058160] SUCC: thread[3] completed total inserted rows: 1300, 21564.24 records/second -[11/06 18:41:04.060392] SUCC: thread[1] completed total inserted rows: 1300, 20688.12 records/second -[11/06 18:41:04.061369] SUCC: Spent 0.063170 seconds to insert rows: 10000 with 8 thread(s) into test 158302.99 records/second -[11/06 18:41:04.061385] SUCC: insert delay, min: 0.62ms, avg: 4.09ms, p90: 13.44ms, p95: 19.25ms, p99: 29.64ms, max: 29.64ms -[11/06 18:41:34.386476] INFO: start creating 100 table(s) with 8 thread(s) -[11/06 18:41:34.465414] SUCC: Spent 0.0790 seconds to create 100 table(s) with 8 thread(s), already exist 0 table(s), actual 100 table(s) pre created, 0 table(s) will be auto created -[11/06 18:41:34.546439] SUCC: thread[2] completed total inserted rows: 13000, 186091.79 records/second -[11/06 18:41:34.549390] SUCC: thread[4] completed total inserted rows: 12000, 164774.05 records/second -[11/06 18:41:34.556956] SUCC: thread[5] completed total inserted rows: 12000, 153440.91 records/second -[11/06 18:41:34.563860] SUCC: thread[6] completed total inserted rows: 12000, 138154.94 records/second -[11/06 18:41:34.564020] SUCC: thread[0] completed total inserted rows: 13000, 148469.62 records/second -[11/06 18:41:34.564693] SUCC: thread[3] completed total inserted rows: 13000, 149214.33 records/second -[11/06 18:41:34.571055] SUCC: thread[1] completed total inserted rows: 13000, 140452.47 records/second -[11/06 18:41:34.571921] SUCC: thread[7] completed total inserted rows: 12000, 129424.71 records/second -[11/06 18:41:34.572778] SUCC: Spent 0.095759 seconds to insert rows: 100000 with 8 thread(s) into test 1044288.27 records/second -[11/06 18:41:34.572797] SUCC: insert delay, min: 1.77ms, avg: 6.68ms, p90: 10.72ms, p95: 14.14ms, p99: 17.69ms, max: 17.69ms