From f7c21c45023100a7556a0dd152229c073b1292b9 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 30 May 2022 18:00:03 +0800 Subject: [PATCH 1/7] feat:add schemaless db paras --- include/common/tmsg.h | 2 ++ include/libs/parser/parser.h | 1 + include/util/taoserror.h | 1 + source/client/inc/clientInt.h | 1 + source/client/src/clientEnv.c | 1 + source/client/src/clientImpl.c | 1 + source/client/src/clientSml.c | 33 +++++++++++++++++++++++++ source/common/src/tmsg.c | 2 ++ source/dnode/mnode/impl/inc/mndDef.h | 1 + source/dnode/mnode/impl/src/mndDb.c | 7 ++++++ source/dnode/mnode/impl/src/mndVgroup.c | 1 + source/libs/parser/inc/parInt.h | 1 + source/libs/parser/src/parInsert.c | 24 +++++++++++++++++- source/libs/parser/src/parTranslater.c | 11 +++++++++ source/util/src/terror.c | 1 + tests/test/c/sdbDump.c | 1 + 16 files changed, 88 insertions(+), 1 deletion(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index faf4addb4b..68480e4ffd 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -779,6 +779,7 @@ typedef struct { int8_t cacheLastRow; int32_t numOfRetensions; SArray* pRetensions; + int8_t schemaless; } SDbCfgRsp; int32_t tSerializeSDbCfgRsp(void* buf, int32_t bufLen, const SDbCfgRsp* pRsp); @@ -1009,6 +1010,7 @@ typedef struct { SReplica replicas[TSDB_MAX_REPLICA]; int32_t numOfRetensions; SArray* pRetensions; // SRetention + int8_t schemaless; // for tsma int8_t isTsma; diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index 1466bb7400..774a23ff4b 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -49,6 +49,7 @@ typedef struct SParseContext { const char* pUser; bool isSuperUser; bool async; + int8_t schemalessType; } SParseContext; int32_t qParseSql(SParseContext* pCxt, SQuery** pQuery); diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 65cfe8de0b..b9eae6962c 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -673,6 +673,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_SML_INVALID_PROTOCOL_TYPE TAOS_DEF_ERROR_CODE(0, 0x3000) #define TSDB_CODE_SML_INVALID_PRECISION_TYPE TAOS_DEF_ERROR_CODE(0, 0x3001) #define TSDB_CODE_SML_INVALID_DATA TAOS_DEF_ERROR_CODE(0, 0x3002) +#define TSDB_CODE_SML_INVALID_DB_CONF TAOS_DEF_ERROR_CODE(0, 0x3003) #ifdef __cplusplus } diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index d9f3351008..4c5bac71e4 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -149,6 +149,7 @@ typedef struct STscObj { int32_t numOfReqs; // number of sqlObj bound to this connection SAppInstInfo* pAppInfo; SHashObj* pRequests; + int8_t schemalessType; } STscObj; typedef struct SResultColumn { diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 669b2bc97e..35e354e952 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -161,6 +161,7 @@ void *createTscObj(const char *user, const char *auth, const char *db, int32_t c taosThreadMutexInit(&pObj->mutex, NULL); pObj->id = taosAddRef(clientConnRefPool, pObj); + pObj->schemalessType = 0; tscDebug("connObj created, 0x%" PRIx64, pObj->id); return pObj; diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index eb4c4cb59f..ba25f2f4ca 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -175,6 +175,7 @@ int32_t parseSql(SRequestObj* pRequest, bool topicQuery, SQuery** pQuery, SStmtC .pTransporter = pTscObj->pAppInfo->pTransporter, .pStmtCb = pStmtCb, .pUser = pTscObj->user, + .schemalessType = pTscObj->schemalessType, .isSuperUser = (0 == strcmp(pTscObj->user, TSDB_DEFAULT_USER))}; cxt.mgmtEpSet = getEpSet_s(&pTscObj->pAppInfo->mgmtEp); diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 7d623072d6..29c7608126 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -2318,6 +2318,28 @@ cleanup: return code; } +static int32_t isSchemalessDb(SSmlHandle* info){ + SName name; + tNameSetDbName(&name, info->taos->acctId, info->taos->db, strlen(info->taos->db)); + char dbFname[TSDB_DB_FNAME_LEN] = {0}; + tNameGetFullDbName(&name, dbFname); + SDbCfgInfo pInfo = {0}; + SEpSet ep = getEpSet_s(&info->taos->pAppInfo->mgmtEp); + + int32_t code = catalogGetDBCfg(info->pCatalog, info->taos->pAppInfo->pTransporter, &ep, dbFname, &pInfo); + if (code != TSDB_CODE_SUCCESS) { + info->pRequest->code = code; + smlBuildInvalidDataMsg(&info->msgBuf, "catalogGetDBCfg error, code:", tstrerror(code)); + return code; + } + if (!pInfo.schemaless){ + info->pRequest->code = TSDB_CODE_SML_INVALID_DB_CONF; + smlBuildInvalidDataMsg(&info->msgBuf, "can not insert into schemaless db:", dbFname); + return TSDB_CODE_SML_INVALID_DB_CONF; + } + return TSDB_CODE_SUCCESS; +} + /** * taos_schemaless_insert() parse and insert data points into database according to * different protocol. @@ -2351,6 +2373,17 @@ TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int pr return (TAOS_RES*)request; } + info->taos->schemalessType = 1; + if(request->pDb == NULL){ + request->code = TSDB_CODE_PAR_DB_NOT_SPECIFIED; + smlBuildInvalidDataMsg(&info->msgBuf, "Database not specified", NULL); + goto end; + } + + if(isSchemalessDb(info) != TSDB_CODE_SUCCESS){ + goto end; + } + if (!lines) { request->code = TSDB_CODE_SML_INVALID_DATA; smlBuildInvalidDataMsg(&info->msgBuf, "lines is null", NULL); diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 7615f7b070..b001953ee9 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -2226,6 +2226,7 @@ int32_t tSerializeSDbCfgRsp(void *buf, int32_t bufLen, const SDbCfgRsp *pRsp) { if (tEncodeI8(&encoder, pRetension->freqUnit) < 0) return -1; if (tEncodeI8(&encoder, pRetension->keepUnit) < 0) return -1; } + if (tEncodeI8(&encoder, pRsp->schemaless) < 0) return -1; tEndEncode(&encoder); int32_t tlen = encoder.pos; @@ -2274,6 +2275,7 @@ int32_t tDeserializeSDbCfgRsp(void *buf, int32_t bufLen, SDbCfgRsp *pRsp) { return -1; } } + if (tDecodeI8(&decoder, &pRsp->schemaless) < 0) return -1; tEndDecode(&decoder); tDecoderClear(&decoder); diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 9e036d7b25..6d839ea622 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -302,6 +302,7 @@ typedef struct { int8_t hashMethod; // default is 1 int32_t numOfRetensions; SArray* pRetensions; + int8_t schemaless; } SDbCfg; typedef struct { diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 95d3383ee1..cc884fb18b 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -115,6 +115,7 @@ static SSdbRaw *mndDbActionEncode(SDbObj *pDb) { SDB_SET_INT8(pRaw, dataPos, pRetension->freqUnit, _OVER) SDB_SET_INT8(pRaw, dataPos, pRetension->keepUnit, _OVER) } + SDB_SET_INT8(pRaw, dataPos, pDb->cfg.schemaless, _OVER) SDB_SET_RESERVE(pRaw, dataPos, DB_RESERVE_SIZE, _OVER) SDB_SET_DATALEN(pRaw, dataPos, _OVER) @@ -192,6 +193,7 @@ static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw) { } } } + SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.schemaless, _OVER) SDB_GET_RESERVE(pRaw, dataPos, DB_RESERVE_SIZE, _OVER) taosInitRWLatch(&pDb->lock); @@ -380,6 +382,7 @@ static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg) { if (pCfg->replications < TSDB_MIN_DB_REPLICA || pCfg->replications > TSDB_MAX_DB_REPLICA) return -1; if (pCfg->replications != 1 && pCfg->replications != 3) return -1; if (pCfg->strict < TSDB_DB_STRICT_OFF || pCfg->strict > TSDB_DB_STRICT_ON) return -1; + if (pCfg->schemaless < TSDB_DB_SCHEMALESS_OFF || pCfg->schemaless > TSDB_DB_SCHEMALESS_ON) return -1; if (pCfg->cacheLastRow < TSDB_MIN_DB_CACHE_LAST_ROW || pCfg->cacheLastRow > TSDB_MAX_DB_CACHE_LAST_ROW) return -1; if (pCfg->hashMethod != 1) return -1; if (pCfg->replications > mndGetDnodeSize(pMnode)) { @@ -411,6 +414,8 @@ static void mndSetDefaultDbCfg(SDbCfg *pCfg) { if (pCfg->strict < 0) pCfg->strict = TSDB_DEFAULT_DB_STRICT; if (pCfg->cacheLastRow < 0) pCfg->cacheLastRow = TSDB_DEFAULT_CACHE_LAST_ROW; if (pCfg->numOfRetensions < 0) pCfg->numOfRetensions = 0; + if (pCfg->schemaless < 0) pCfg->schemaless = TSDB_DB_SCHEMALESS_OFF; + } static int32_t mndSetCreateDbRedoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroups) { @@ -521,6 +526,7 @@ static int32_t mndCreateDb(SMnode *pMnode, SRpcMsg *pReq, SCreateDbReq *pCreate, .strict = pCreate->strict, .cacheLastRow = pCreate->cacheLastRow, .hashMethod = 1, + .schemaless = pCreate->schemaless, }; dbObj.cfg.numOfRetensions = pCreate->numOfRetensions; @@ -890,6 +896,7 @@ static int32_t mndProcessGetDbCfgReq(SRpcMsg *pReq) { cfgRsp.cacheLastRow = pDb->cfg.cacheLastRow; cfgRsp.numOfRetensions = pDb->cfg.numOfRetensions; cfgRsp.pRetensions = pDb->cfg.pRetensions; + cfgRsp.schemaless = pDb->cfg.schemaless; int32_t contLen = tSerializeSDbCfgRsp(NULL, 0, &cfgRsp); void *pRsp = rpcMallocCont(contLen); diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index e05b38a7c0..a00841f911 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -218,6 +218,7 @@ void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVg createReq.hashMethod = pDb->cfg.hashMethod; createReq.numOfRetensions = pDb->cfg.numOfRetensions; createReq.pRetensions = pDb->cfg.pRetensions; + createReq.schemaless = pDb->cfg.schemaless; for (int32_t v = 0; v < pVgroup->replica; ++v) { SReplica *pReplica = &createReq.replicas[v]; diff --git a/source/libs/parser/inc/parInt.h b/source/libs/parser/inc/parInt.h index 184ed7d8b2..c74567f1a0 100644 --- a/source/libs/parser/inc/parInt.h +++ b/source/libs/parser/inc/parInt.h @@ -31,6 +31,7 @@ int32_t authenticate(SParseContext* pParseCxt, SQuery* pQuery); int32_t translate(SParseContext* pParseCxt, SQuery* pQuery); int32_t extractResultSchema(const SNode* pRoot, int32_t* numOfCols, SSchema** pSchema); int32_t calculateConstant(SParseContext* pParseCxt, SQuery* pQuery); +int32_t isNotSchemalessDb(SParseContext* pContext); #ifdef __cplusplus } diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index 58a6d1483f..c78733c3c4 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -1204,6 +1204,23 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) { return buildOutput(pCxt); } +int32_t isNotSchemalessDb(SParseContext* pContext){ + SName name; + tNameSetDbName(&name, pContext->acctId, pContext->db, strlen(pContext->db)); + char dbFname[TSDB_DB_FNAME_LEN] = {0}; + tNameGetFullDbName(&name, dbFname); + SDbCfgInfo pInfo = {0}; + int32_t code = catalogGetDBCfg(pContext->pCatalog, pContext->pTransporter, &pContext->mgmtEpSet, dbFname, &pInfo); + if (code != TSDB_CODE_SUCCESS) { + parserError("catalogGetDBCfg error, code:%s, dbFName:%s", tstrerror(code), dbFname); + return code; + } + if (pInfo.schemaless){ + parserError("can not insert into schemaless db:%s", dbFname); + return TSDB_CODE_SML_INVALID_DB_CONF; + } + return TSDB_CODE_SUCCESS; +} // INSERT INTO // tb_name // [USING stb_name [(tag1_name, ...)] TAGS (tag1_value, ...)] @@ -1252,6 +1269,11 @@ int32_t parseInsertSql(SParseContext* pContext, SQuery** pQuery) { (*pQuery)->pRoot = (SNode*)context.pOutput; } + int32_t code = isNotSchemalessDb(pContext); + if(code != TSDB_CODE_SUCCESS){ + return code; + } + if (NULL == (*pQuery)->pTableList) { (*pQuery)->pTableList = taosArrayInit(taosHashGetSize(context.pTableNameHashObj), sizeof(SName)); if (NULL == (*pQuery)->pTableList) { @@ -1261,7 +1283,7 @@ int32_t parseInsertSql(SParseContext* pContext, SQuery** pQuery) { context.pOutput->payloadType = PAYLOAD_TYPE_KV; - int32_t code = skipInsertInto(&context); + code = skipInsertInto(&context); if (TSDB_CODE_SUCCESS == code) { code = parseInsertBody(&context); } diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index d84b005f7f..31363401fe 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -2619,6 +2619,11 @@ static int32_t checkCreateTable(STranslateContext* pCxt, SCreateTableStmt* pStmt if (TSDB_CODE_SUCCESS == code) { code = checkTableSchema(pCxt, pStmt); } + if (TSDB_CODE_SUCCESS == code) { + if(pCxt->pParseCxt->schemalessType == 0 && isNotSchemalessDb(pCxt->pParseCxt) != TSDB_CODE_SUCCESS){ + code = TSDB_CODE_SML_INVALID_DB_CONF; + } + } return code; } @@ -4350,6 +4355,9 @@ static SArray* serializeVgroupsCreateTableBatch(int32_t acctId, SHashObj* pVgrou } static int32_t rewriteCreateMultiTable(STranslateContext* pCxt, SQuery* pQuery) { + if(pCxt->pParseCxt->schemalessType == 0 && isNotSchemalessDb(pCxt->pParseCxt) != TSDB_CODE_SUCCESS){ + return TSDB_CODE_SML_INVALID_DB_CONF; + } SCreateMultiTableStmt* pStmt = (SCreateMultiTableStmt*)pQuery->pRoot; SHashObj* pVgroupHashmap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); @@ -4766,6 +4774,9 @@ static int32_t buildModifyVnodeArray(STranslateContext* pCxt, SAlterTableStmt* p } static int32_t rewriteAlterTable(STranslateContext* pCxt, SQuery* pQuery) { + if(pCxt->pParseCxt->schemalessType == 0 && isNotSchemalessDb(pCxt->pParseCxt) != TSDB_CODE_SUCCESS){ + return TSDB_CODE_SML_INVALID_DB_CONF; + } SAlterTableStmt* pStmt = (SAlterTableStmt*)pQuery->pRoot; STableMeta* pTableMeta = NULL; diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 178d6e8d2b..0ffcebca5b 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -469,6 +469,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_UDF_INVALID_OUTPUT_TYPE, "udf invalid output ty TAOS_DEFINE_ERROR(TSDB_CODE_SML_INVALID_PROTOCOL_TYPE, "Invalid line protocol type") TAOS_DEFINE_ERROR(TSDB_CODE_SML_INVALID_PRECISION_TYPE, "Invalid timestamp precision type") TAOS_DEFINE_ERROR(TSDB_CODE_SML_INVALID_DATA, "Invalid data type") +TAOS_DEFINE_ERROR(TSDB_CODE_SML_INVALID_DB_CONF, "Invalid schemaless db config") #ifdef TAOS_ERROR_C }; diff --git a/tests/test/c/sdbDump.c b/tests/test/c/sdbDump.c index 8be2822c0a..6a7255b33c 100644 --- a/tests/test/c/sdbDump.c +++ b/tests/test/c/sdbDump.c @@ -86,6 +86,7 @@ void dumpDb(SSdb *pSdb, SJson *json) { tjsonAddIntegerToObject(item, "cacheLastRow", pObj->cfg.cacheLastRow); tjsonAddIntegerToObject(item, "hashMethod", pObj->cfg.hashMethod); tjsonAddIntegerToObject(item, "numOfRetensions", pObj->cfg.numOfRetensions); + tjsonAddIntegerToObject(item, "schemaless", pObj->cfg.schemaless); sdbRelease(pSdb, pObj); } From 35635ea68c76727a8d0628a1b9c95875a19af291 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 31 May 2022 10:42:18 +0800 Subject: [PATCH 2/7] feat:add schemaless db paras --- include/common/tmsg.h | 1 - source/common/src/systable.c | 2 ++ source/dnode/mnode/impl/src/mndDb.c | 4 ++++ source/dnode/mnode/impl/src/mndVgroup.c | 1 - 4 files changed, 6 insertions(+), 2 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 68480e4ffd..dbfbfcf0f2 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1010,7 +1010,6 @@ typedef struct { SReplica replicas[TSDB_MAX_REPLICA]; int32_t numOfRetensions; SArray* pRetensions; // SRetention - int8_t schemaless; // for tsma int8_t isTsma; diff --git a/source/common/src/systable.c b/source/common/src/systable.c index 38a6bafe9a..4e2f9b074a 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -89,7 +89,9 @@ static const SSysDbTableSchema userDBSchema[] = { {.name = "cachelast", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT}, {.name = "precision", .bytes = 2 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, {.name = "single_stable", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT}, + {.name = "schemaless", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT}, {.name = "status", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, + // {.name = "update", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT}, // disable update }; diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index cc884fb18b..35dc24dd9b 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -1540,8 +1540,12 @@ static void dumpDbInfoData(SSDataBlock *pBlock, SDbObj *pDb, SShowObj *pShow, in pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.numOfStables, false); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.schemaless, false); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols); colDataAppend(pColInfo, rows, (const char *)statusB, false); + } } diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index a00841f911..e05b38a7c0 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -218,7 +218,6 @@ void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVg createReq.hashMethod = pDb->cfg.hashMethod; createReq.numOfRetensions = pDb->cfg.numOfRetensions; createReq.pRetensions = pDb->cfg.pRetensions; - createReq.schemaless = pDb->cfg.schemaless; for (int32_t v = 0; v < pVgroup->replica; ++v) { SReplica *pReplica = &createReq.replicas[v]; From 6418caa4304b620b763d21c669877e1af17163f8 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 1 Jun 2022 18:53:23 +0800 Subject: [PATCH 3/7] feat:add schemaless param from db --- source/client/src/clientSml.c | 2 ++ source/client/test/smlTest.cpp | 24 +++++++++++++++++++++++- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 29c7608126..8d1924b1c9 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -2381,6 +2381,8 @@ TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int pr } if(isSchemalessDb(info) != TSDB_CODE_SUCCESS){ + request->code = TSDB_CODE_SML_INVALID_DB_CONF; + smlBuildInvalidDataMsg(&info->msgBuf, "Cannot write data to a non schemaless database", NULL); goto end; } diff --git a/source/client/test/smlTest.cpp b/source/client/test/smlTest.cpp index 217699e360..4684df8418 100644 --- a/source/client/test/smlTest.cpp +++ b/source/client/test/smlTest.cpp @@ -1258,4 +1258,26 @@ TEST(testCase, sml_TD15742_Test) { destroyRequest(request); smlDestroyInfo(info); -} \ No newline at end of file +} + +TEST(testCase, sml_params_Test) { + TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + ASSERT_NE(taos, nullptr); + + TAOS_RES* pRes = taos_query(taos, "create database if not exists param"); + taos_free_result(pRes); + + const char *sql[] = { + "test_ms,t0=t c0=f 1626006833641", + }; + TAOS_RES* res = taos_schemaless_insert(taos, (char**)sql, 1, TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_MILLI_SECONDS); + ASSERT_EQ(taos_errno(res), TSDB_CODE_PAR_DB_NOT_SPECIFIED); + taos_free_result(pRes); + + pRes = taos_query(taos, "use param"); + taos_free_result(pRes); + + res = taos_schemaless_insert(taos, (char**)sql, 1, TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_MILLI_SECONDS); + ASSERT_EQ(taos_errno(res), TSDB_CODE_SML_INVALID_DB_CONF); + taos_free_result(pRes); +} From ea51d91bc15a01fa01ab89bfba815b445fb5e3e7 Mon Sep 17 00:00:00 2001 From: jiajingbin Date: Wed, 1 Jun 2022 20:19:11 +0800 Subject: [PATCH 4/7] test: update schemaless-testcases for schemaless-database --- .../system-test/1-insert/influxdb_line_taosc_insert.py | 4 ++-- .../system-test/1-insert/opentsdb_json_taosc_insert.py | 10 +++++----- .../1-insert/opentsdb_telnet_line_taosc_insert.py | 6 +++--- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/system-test/1-insert/influxdb_line_taosc_insert.py b/tests/system-test/1-insert/influxdb_line_taosc_insert.py index 75e0c3d59c..0ddeba4665 100644 --- a/tests/system-test/1-insert/influxdb_line_taosc_insert.py +++ b/tests/system-test/1-insert/influxdb_line_taosc_insert.py @@ -33,10 +33,10 @@ class TDTestCase: def createDb(self, name="test", db_update_tag=0): if db_update_tag == 0: tdSql.execute(f"drop database if exists {name}") - tdSql.execute(f"create database if not exists {name} precision 'ms'") + tdSql.execute(f"create database if not exists {name} precision 'ms' schemaless 1") else: tdSql.execute(f"drop database if exists {name}") - tdSql.execute(f"create database if not exists {name} precision 'ms' update 1") + tdSql.execute(f"create database if not exists {name} precision 'ms' update 1 schemaless 1") tdSql.execute(f'use {name}') def timeTrans(self, time_value, ts_type): diff --git a/tests/system-test/1-insert/opentsdb_json_taosc_insert.py b/tests/system-test/1-insert/opentsdb_json_taosc_insert.py index 360d878dfe..f9bc5bbaf4 100644 --- a/tests/system-test/1-insert/opentsdb_json_taosc_insert.py +++ b/tests/system-test/1-insert/opentsdb_json_taosc_insert.py @@ -36,10 +36,10 @@ class TDTestCase: if db_update_tag == 0: tdSql.execute(f"drop database if exists {name}") - tdSql.execute(f"create database if not exists {name} precision 'ms'") + tdSql.execute(f"create database if not exists {name} precision 'ms' schemaless 1") else: tdSql.execute(f"drop database if exists {name}") - tdSql.execute(f"create database if not exists {name} precision 'ms' update 1") + tdSql.execute(f"create database if not exists {name} precision 'ms' update 1 schemaless 1") tdSql.execute(f'use {name}') def timeTrans(self, time_value): @@ -535,7 +535,7 @@ class TDTestCase: # check result #! bug tdSql.execute(f"drop database if exists test_ts") - tdSql.execute(f"create database if not exists test_ts precision 'ms'") + tdSql.execute(f"create database if not exists test_ts precision 'ms' schemaless 1") tdSql.execute("use test_ts") input_json = [{"metric": "test_ms", "timestamp": {"value": 1626006833640, "type": "ms"}, "value": True, "tags": {"t0": True}}, {"metric": "test_ms", "timestamp": {"value": 1626006833641, "type": "ms"}, "value": False, "tags": {"t0": True}}] @@ -545,7 +545,7 @@ class TDTestCase: tdSql.checkEqual(str(res[1][0]), "2021-07-11 20:33:53.641000") tdSql.execute(f"drop database if exists test_ts") - tdSql.execute(f"create database if not exists test_ts precision 'us'") + tdSql.execute(f"create database if not exists test_ts precision 'us' schemaless 1") tdSql.execute("use test_ts") input_json = [{"metric": "test_us", "timestamp": {"value": 1626006833639000, "type": "us"}, "value": True, "tags": {"t0": True}}, {"metric": "test_us", "timestamp": {"value": 1626006833639001, "type": "us"}, "value": False, "tags": {"t0": True}}] @@ -555,7 +555,7 @@ class TDTestCase: tdSql.checkEqual(str(res[1][0]), "2021-07-11 20:33:53.639001") tdSql.execute(f"drop database if exists test_ts") - tdSql.execute(f"create database if not exists test_ts precision 'ns'") + tdSql.execute(f"create database if not exists test_ts precision 'ns' schemaless 1") tdSql.execute("use test_ts") input_json = [{"metric": "test_ns", "timestamp": {"value": 1626006833639000000, "type": "ns"}, "value": True, "tags": {"t0": True}}, {"metric": "test_ns", "timestamp": {"value": 1626006833639000001, "type": "ns"}, "value": False, "tags": {"t0": True}}] diff --git a/tests/system-test/1-insert/opentsdb_telnet_line_taosc_insert.py b/tests/system-test/1-insert/opentsdb_telnet_line_taosc_insert.py index 6d50dd4fda..23404330ed 100644 --- a/tests/system-test/1-insert/opentsdb_telnet_line_taosc_insert.py +++ b/tests/system-test/1-insert/opentsdb_telnet_line_taosc_insert.py @@ -36,10 +36,10 @@ class TDTestCase: if db_update_tag == 0: tdSql.execute(f"drop database if exists {name}") - tdSql.execute(f"create database if not exists {name} precision 'us'") + tdSql.execute(f"create database if not exists {name} precision 'us' schemaless 1") else: tdSql.execute(f"drop database if exists {name}") - tdSql.execute(f"create database if not exists {name} precision 'ns' update 1") + tdSql.execute(f"create database if not exists {name} precision 'ns' update 1 schemaless 1") tdSql.execute(f'use {name}') def timeTrans(self, time_value, ts_type): @@ -393,7 +393,7 @@ class TDTestCase: self.resCmp(input_sql, stb_name, ts_type=TDSmlTimestampType.SECOND.value) tdSql.execute(f"drop database if exists test_ts") - tdSql.execute(f"create database if not exists test_ts precision 'ms'") + tdSql.execute(f"create database if not exists test_ts precision 'ms' schemaless 1") tdSql.execute("use test_ts") input_sql = ['test_ms 1626006833640 t t0=t', 'test_ms 1626006833641 f t0=t'] self._conn.schemaless_insert(input_sql, TDSmlProtocolType.TELNET.value, None) From 10807abbc8429fea42937240e0468aeba4f3efc8 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 2 Jun 2022 11:43:57 +0800 Subject: [PATCH 5/7] fix:fix error in get db name for schemaless parameters --- source/libs/parser/inc/parInt.h | 2 +- source/libs/parser/src/parInsert.c | 4 ++-- source/libs/parser/src/parTranslater.c | 22 +++++++++++++--------- 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/source/libs/parser/inc/parInt.h b/source/libs/parser/inc/parInt.h index bdef3becf7..8ec20cde5a 100644 --- a/source/libs/parser/inc/parInt.h +++ b/source/libs/parser/inc/parInt.h @@ -32,7 +32,7 @@ int32_t authenticate(SParseContext* pParseCxt, SQuery* pQuery); int32_t translate(SParseContext* pParseCxt, SQuery* pQuery); int32_t extractResultSchema(const SNode* pRoot, int32_t* numOfCols, SSchema** pSchema); int32_t calculateConstant(SParseContext* pParseCxt, SQuery* pQuery); -int32_t isNotSchemalessDb(SParseContext* pContext); +int32_t isNotSchemalessDb(SParseContext* pContext, char *dbName); #ifdef __cplusplus } diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index 7729d4c415..7c179f8ac9 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -1404,9 +1404,9 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) { return buildOutput(pCxt); } -int32_t isNotSchemalessDb(SParseContext* pContext){ +int32_t isNotSchemalessDb(SParseContext* pContext, char *dbName){ SName name; - tNameSetDbName(&name, pContext->acctId, pContext->db, strlen(pContext->db)); + tNameSetDbName(&name, pContext->acctId, dbName, strlen(dbName)); char dbFname[TSDB_DB_FNAME_LEN] = {0}; tNameGetFullDbName(&name, dbFname); SDbCfgInfo pInfo = {0}; diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 18d64905c9..7b9d38e5b3 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -2648,8 +2648,8 @@ static int32_t checkCreateTable(STranslateContext* pCxt, SCreateTableStmt* pStmt code = checkTableSchema(pCxt, pStmt); } if (TSDB_CODE_SUCCESS == code) { - if(pCxt->pParseCxt->schemalessType == 0 && isNotSchemalessDb(pCxt->pParseCxt) != TSDB_CODE_SUCCESS){ - code = TSDB_CODE_SML_INVALID_DB_CONF; + if(pCxt->pParseCxt->schemalessType == 0){ + code = isNotSchemalessDb(pCxt->pParseCxt, pStmt->dbName); } } return code; @@ -4430,9 +4430,7 @@ static SArray* serializeVgroupsCreateTableBatch(int32_t acctId, SHashObj* pVgrou } static int32_t rewriteCreateMultiTable(STranslateContext* pCxt, SQuery* pQuery) { - if(pCxt->pParseCxt->schemalessType == 0 && isNotSchemalessDb(pCxt->pParseCxt) != TSDB_CODE_SUCCESS){ - return TSDB_CODE_SML_INVALID_DB_CONF; - } + SCreateMultiTableStmt* pStmt = (SCreateMultiTableStmt*)pQuery->pRoot; SHashObj* pVgroupHashmap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); @@ -4443,6 +4441,10 @@ static int32_t rewriteCreateMultiTable(STranslateContext* pCxt, SQuery* pQuery) int32_t code = TSDB_CODE_SUCCESS; SNode* pNode; FOREACH(pNode, pStmt->pSubTables) { + if(pCxt->pParseCxt->schemalessType == 0 && + (code = isNotSchemalessDb(pCxt->pParseCxt, ((SCreateSubTableClause*)pNode)->dbName)) != TSDB_CODE_SUCCESS){ + return code; + } code = rewriteCreateSubTable(pCxt, (SCreateSubTableClause*)pNode, pVgroupHashmap); if (TSDB_CODE_SUCCESS != code) { taosHashCleanup(pVgroupHashmap); @@ -4853,13 +4855,15 @@ static int32_t buildModifyVnodeArray(STranslateContext* pCxt, SAlterTableStmt* p } static int32_t rewriteAlterTable(STranslateContext* pCxt, SQuery* pQuery) { - if(pCxt->pParseCxt->schemalessType == 0 && isNotSchemalessDb(pCxt->pParseCxt) != TSDB_CODE_SUCCESS){ - return TSDB_CODE_SML_INVALID_DB_CONF; - } SAlterTableStmt* pStmt = (SAlterTableStmt*)pQuery->pRoot; + int32_t code = TSDB_CODE_SUCCESS; + if(pCxt->pParseCxt->schemalessType == 0 && + (code = isNotSchemalessDb(pCxt->pParseCxt, pStmt->dbName)) != TSDB_CODE_SUCCESS){ + return code; + } STableMeta* pTableMeta = NULL; - int32_t code = getTableMeta(pCxt, pStmt->dbName, pStmt->tableName, &pTableMeta); + code = getTableMeta(pCxt, pStmt->dbName, pStmt->tableName, &pTableMeta); if (TSDB_CODE_SUCCESS != code) { return code; } From 90cdc3601876cfec59849cac077a2f51d8ce2441 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 2 Jun 2022 12:55:51 +0800 Subject: [PATCH 6/7] fix:fix error in get db name for schemaless parameters --- source/client/src/clientSml.c | 11 ++++++----- source/libs/parser/src/parInsert.c | 9 +++------ 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 8d1924b1c9..e77b9a9bd0 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -960,10 +960,10 @@ static int32_t smlParseInfluxString(const char* sql, SSmlLineInfo *elements, SSm return TSDB_CODE_SML_INVALID_DATA; } elements->colsLen = sql - elements->cols; - if(elements->colsLen == 0) { - smlBuildInvalidDataMsg(msg, "cols is empty", NULL); - return TSDB_CODE_SML_INVALID_DATA; - } +// if(elements->colsLen == 0) { +// smlBuildInvalidDataMsg(msg, "cols is empty", NULL); +// return TSDB_CODE_SML_INVALID_DATA; +// } // parse timestamp JUMP_SPACE(sql) @@ -1124,7 +1124,7 @@ static int32_t smlParseTelnetString(SSmlHandle *info, const char* sql, SSmlTable } static int32_t smlParseCols(const char* data, int32_t len, SArray *cols, char *childTableName, bool isTag, SHashObj *dumplicateKey, SSmlMsgBuf *msg){ - if(isTag && len == 0){ + if(len == 0){ return TSDB_CODE_SUCCESS; } @@ -2407,6 +2407,7 @@ TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int pr info->pRequest->code = smlProcess(info, lines, numLines); end: + uDebug("result:%s", info->msgBuf.buf); smlDestroyInfo(info); return (TAOS_RES*)request; } diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index 7c179f8ac9..4973f792ce 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -1329,6 +1329,8 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) { SName name; CHECK_CODE(createSName(&name, &tbnameToken, pCxt->pComCxt->acctId, pCxt->pComCxt->db, &pCxt->msg)); + CHECK_CODE(isNotSchemalessDb(pCxt->pComCxt, name.dbname)); + tNameExtractFullName(&name, tbFName); CHECK_CODE(taosHashPut(pCxt->pTableNameHashObj, tbFName, strlen(tbFName), &name, sizeof(SName))); @@ -1468,11 +1470,6 @@ int32_t parseInsertSql(SParseContext* pContext, SQuery** pQuery) { (*pQuery)->msgType = TDMT_VND_SUBMIT; (*pQuery)->pRoot = (SNode*)context.pOutput; - int32_t code = isNotSchemalessDb(pContext); - if(code != TSDB_CODE_SUCCESS){ - return code; - } - if (NULL == (*pQuery)->pTableList) { (*pQuery)->pTableList = taosArrayInit(taosHashGetSize(context.pTableNameHashObj), sizeof(SName)); if (NULL == (*pQuery)->pTableList) { @@ -1482,7 +1479,7 @@ int32_t parseInsertSql(SParseContext* pContext, SQuery** pQuery) { context.pOutput->payloadType = PAYLOAD_TYPE_KV; - code = skipInsertInto(&context.pSql, &context.msg); + int32_t code = skipInsertInto(&context.pSql, &context.msg); if (TSDB_CODE_SUCCESS == code) { code = parseInsertBody(&context); } From 8cba836611ad7fdea61a1227e371c469c5c4644d Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 2 Jun 2022 14:54:21 +0800 Subject: [PATCH 7/7] fix:fix CI error in schemaless parameters --- source/common/src/systable.c | 2 +- source/dnode/mnode/impl/src/mndDb.c | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/source/common/src/systable.c b/source/common/src/systable.c index d5999411ae..b1baf5a01b 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -89,8 +89,8 @@ static const SSysDbTableSchema userDBSchema[] = { {.name = "cachelast", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT}, {.name = "precision", .bytes = 2 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, {.name = "single_stable", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT}, - {.name = "schemaless", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT}, {.name = "status", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "schemaless", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT}, // {.name = "update", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT}, // disable update }; diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 9bba1a1b2f..6d7a638c30 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -1550,11 +1550,10 @@ static void dumpDbInfoData(SSDataBlock *pBlock, SDbObj *pDb, SShowObj *pShow, in colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.numOfStables, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.schemaless, false); - - pColInfo = taosArrayGet(pBlock->pDataBlock, cols); colDataAppend(pColInfo, rows, (const char *)statusB, false); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols); + colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.schemaless, false); } }