diff --git a/Jenkinsfile2 b/Jenkinsfile2 index fc00c5e2dc..1b2f28908c 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -7,7 +7,8 @@ file_zh_changed = '' file_en_changed = '' file_no_doc_changed = '1' file_only_tdgpt_change_except = '1' -tdgpt_file = "forecastoperator.c\\|anomalywindowoperator.c\\|tanalytics.h\\|tanalytics.c\\|tdgpt_cases.task\\|analytics" +tdgpt_file = "forecastoperator.c|anomalywindowoperator.c|tanalytics.h|tanalytics.c|tdgpt_cases.task|analytics" + def abortPreviousBuilds() { def currentJobName = env.JOB_NAME @@ -69,7 +70,7 @@ def check_docs(){ returnStdout: true ) - file_no_doc_changed = sh ( + def file_no_doc_changed = sh ( script: ''' cd ${WKC} git --no-pager diff --name-only FETCH_HEAD `git merge-base FETCH_HEAD ${CHANGE_TARGET}`|grep -v "^docs/en/"|grep -v "^docs/zh/"|grep -v ".md$" || : @@ -80,7 +81,7 @@ def check_docs(){ file_only_tdgpt_change_except = sh ( script: ''' cd ${WKC} - git --no-pager diff --name-only FETCH_HEAD `git merge-base FETCH_HEAD ${CHANGE_TARGET}`|grep -v "^docs/en/"|grep -v "^docs/zh/"|grep -v ".md$" | grep -v ${tdgpt_file} || : + git --no-pager diff --name-only FETCH_HEAD `git merge-base FETCH_HEAD ${CHANGE_TARGET}`|grep -v "^docs/en/"|grep -v "^docs/zh/"|grep -v ".md$" | grep -Ev "forecastoperator.c|anomalywindowoperator.c|tanalytics.h|tanalytics.c|tdgpt_cases.task|analytics" ||: ''', returnStdout: true ).trim() @@ -572,7 +573,7 @@ pipeline { cd ${WKC}/tests/parallel_test ./run_scan_container.sh -d ${WKDIR} -b ${BRANCH_NAME}_${BUILD_ID} -f ${WKDIR}/tmp/${BRANCH_NAME}_${BUILD_ID}/docs_changed.txt ''' + extra_param + ''' ''' - if ( file_no_doc_changed =~ /orecastoperator.c|anomalywindowoperator.c|tanalytics.h|tanalytics.c|tdgpt_cases.task|analytics/ ) { + if ( file_no_doc_changed =~ /forecastoperator.c|anomalywindowoperator.c|tanalytics.h|tanalytics.c|tdgpt_cases.task|analytics/ ) { sh ''' cd ${WKC}/tests/parallel_test export DEFAULT_RETRY_TIME=2 diff --git a/cmake/taosadapter_CMakeLists.txt.in b/cmake/taosadapter_CMakeLists.txt.in index 13826a1a74..ef6ed4af1d 100644 --- a/cmake/taosadapter_CMakeLists.txt.in +++ b/cmake/taosadapter_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taosadapter ExternalProject_Add(taosadapter GIT_REPOSITORY https://github.com/taosdata/taosadapter.git - GIT_TAG main + GIT_TAG 3.0 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosadapter" BINARY_DIR "" #BUILD_IN_SOURCE TRUE diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index 9bbda8309f..9a6a5329ae 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG main + GIT_TAG 3.0 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE diff --git a/cmake/taosws_CMakeLists.txt.in b/cmake/taosws_CMakeLists.txt.in index b013d45911..17446d184d 100644 --- a/cmake/taosws_CMakeLists.txt.in +++ b/cmake/taosws_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taosws-rs ExternalProject_Add(taosws-rs GIT_REPOSITORY https://github.com/taosdata/taos-connector-rust.git - GIT_TAG main + GIT_TAG 3.0 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosws-rs" BINARY_DIR "" #BUILD_IN_SOURCE TRUE diff --git a/docs/zh/06-advanced/06-TDgpt/06-dev/02-forecast.md b/docs/zh/06-advanced/06-TDgpt/06-dev/02-forecast.md index 5395dc374b..841722c6a2 100644 --- a/docs/zh/06-advanced/06-TDgpt/06-dev/02-forecast.md +++ b/docs/zh/06-advanced/06-TDgpt/06-dev/02-forecast.md @@ -99,7 +99,7 @@ def test_myfc(self): s = loader.get_service("myfc") # 设置用于预测分析的数据 - s.set_input_list(self.get_input_list()) + s.set_input_list(self.get_input_list(), None) # 检查预测结果应该全部为 1 r = s.set_params( {"fc_rows": 10, "start_ts": 171000000, "time_step": 86400 * 30, "start_p": 0} diff --git a/docs/zh/06-advanced/06-TDgpt/06-dev/03-ad.md b/docs/zh/06-advanced/06-TDgpt/06-dev/03-ad.md index 5a9ac20140..7c85d41c50 100644 --- a/docs/zh/06-advanced/06-TDgpt/06-dev/03-ad.md +++ b/docs/zh/06-advanced/06-TDgpt/06-dev/03-ad.md @@ -44,10 +44,10 @@ class _MyAnomalyDetectionService(AbstractAnomalyDetectionService): def set_params(self, params): """该算法无需任何输入参数，直接重载父类该函数，不处理算法参数设置逻辑""" - pass + return super().set_params(params) ``` -将该文件保存在 `./lib/taosanalytics/algo/ad/` 目录下，然后重启 taosanode 服务。在 TDengine 命令行接口 taos 中执行 `SHOW ANODES FULL` 就能够看到新加入的算法，然后应用就可以通过 SQL 语句调用该检测算法。 +将该文件保存在 `./lib/taosanalytics/algo/ad/` 目录下，然后重启 taosanode 服务。在 TDengine 命令行接口 taos 中执行 `SHOW ANODES FULL` 就能够看到新加入的算法，然后就可以通过 SQL 语句调用该算法。 ```SQL --- 对 col 列进行异常检测，通过指定 algo 参数为 myad 来调用新添加的异常检测类 @@ -65,7 +65,7 @@ def test_myad(self): s = loader.get_service("myad") # 设置需要进行检测的输入数据 - s.set_input_list(AnomalyDetectionTest.input_list) + s.set_input_list(AnomalyDetectionTest.input_list, None) r = s.execute() diff --git a/include/libs/function/taosudf.h b/include/libs/function/taosudf.h index 91487e5d1d..fd6b42f61e 100644 --- a/include/libs/function/taosudf.h +++ b/include/libs/function/taosudf.h @@ -237,18 +237,20 @@ static FORCE_INLINE int32_t udfColDataSet(SUdfColumn *pColumn, uint32_t currentR (void)memcpy(data->fixLenCol.data + meta->bytes * currentRow, pData, meta->bytes); } else { int32_t dataLen = varDataTLen(pData); - if (meta->type == TSDB_DATA_TYPE_JSON) { - if (*pData == TSDB_DATA_TYPE_NULL) { - dataLen = 0; - } else if (*pData == TSDB_DATA_TYPE_NCHAR) { - dataLen = varDataTLen(pData + sizeof(char)); - } else if (*pData == TSDB_DATA_TYPE_BIGINT || *pData == TSDB_DATA_TYPE_DOUBLE) { - dataLen = sizeof(int64_t); - } else if (*pData == TSDB_DATA_TYPE_BOOL) { - dataLen = sizeof(char); - } - dataLen += sizeof(char); - } + // This is a piece of code to help users implement udf. It is only called during testing. + // Currently, the json type is not supported and will not be called. + // if (meta->type == TSDB_DATA_TYPE_JSON) { + // if (*pData == TSDB_DATA_TYPE_NULL) { + // dataLen = 0; + // } else if (*pData == TSDB_DATA_TYPE_NCHAR) { + // dataLen = varDataTLen(pData + sizeof(char)); + // } else if (*pData == TSDB_DATA_TYPE_BIGINT || *pData == TSDB_DATA_TYPE_DOUBLE) { + // dataLen = sizeof(int64_t); + // } else if (*pData == TSDB_DATA_TYPE_BOOL) { + // dataLen = sizeof(char); + // } + // dataLen += sizeof(char); + // } if (data->varLenCol.payloadAllocLen < data->varLenCol.payloadLen + dataLen) { uint32_t newSize = data->varLenCol.payloadAllocLen; diff --git a/source/dnode/mnode/impl/src/mndCompact.c b/source/dnode/mnode/impl/src/mndCompact.c index e8f7202986..33a6ddcc5d 100644 --- a/source/dnode/mnode/impl/src/mndCompact.c +++ b/source/dnode/mnode/impl/src/mndCompact.c @@ -348,6 +348,7 @@ static void *mndBuildKillCompactReq(SMnode *pMnode, SVgObj *pVgroup, int32_t *pC req.compactId = compactId; req.vgId = pVgroup->vgId; req.dnodeId = dnodeid; + terrno = 0; mInfo("vgId:%d, build compact vnode config req", pVgroup->vgId); int32_t contLen = tSerializeSVKillCompactReq(NULL, 0, &req); @@ -367,8 +368,10 @@ static void *mndBuildKillCompactReq(SMnode *pMnode, SVgObj *pVgroup, int32_t *pC pHead->contLen = htonl(contLen); pHead->vgId = htonl(pVgroup->vgId); - if ((contLen = tSerializeSVKillCompactReq((char *)pReq + sizeof(SMsgHead), contLen, &req)) < 0) { - terrno = contLen; + mTrace("vgId:%d, build compact vnode config req, contLen:%d", pVgroup->vgId, contLen); + int32_t ret = 0; + if ((ret = tSerializeSVKillCompactReq((char *)pReq + sizeof(SMsgHead), contLen, &req)) < 0) { + terrno = ret; return NULL; } *pContLen = contLen; @@ -401,6 +404,8 @@ static int32_t mndAddKillCompactAction(SMnode *pMnode, STrans *pTrans, SVgObj *p action.contLen = contLen; action.msgType = TDMT_VND_KILL_COMPACT; + mTrace("trans:%d, kill compact msg len:%d", pTrans->id, contLen); + if ((code = mndTransAppendRedoAction(pTrans, &action)) != 0) { taosMemoryFree(pReq); TAOS_RETURN(code); diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 4f1ecf81dd..940116317c 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -160,7 +160,7 @@ int32_t metaAlterSuperTable(SMeta* pMeta, int64_t version, SVCreateStbRe int32_t metaDropSuperTable(SMeta* pMeta, int64_t verison, SVDropStbReq* pReq); int32_t metaCreateTable2(SMeta* pMeta, int64_t version, SVCreateTbReq* pReq, STableMetaRsp** ppRsp); int32_t metaDropTable2(SMeta* pMeta, int64_t version, SVDropTbReq* pReq); -int32_t metaTrimTables(SMeta* pMeta); +int32_t metaTrimTables(SMeta* pMeta, int64_t version); int32_t metaDropMultipleTables(SMeta* pMeta, int64_t version, SArray* tbUids); int metaTtlFindExpired(SMeta* pMeta, int64_t timePointMs, SArray* tbUids, int32_t ttlDropMaxCount); int metaAlterTable(SMeta* pMeta, int64_t version, SVAlterTbReq* pReq, STableMetaRsp* pMetaRsp); diff --git a/source/dnode/vnode/src/meta/metaEntry2.c b/source/dnode/vnode/src/meta/metaEntry2.c index 7aa57edf73..20744577a7 100644 --- a/source/dnode/vnode/src/meta/metaEntry2.c +++ b/source/dnode/vnode/src/meta/metaEntry2.c @@ -1645,7 +1645,6 @@ static int32_t metaHandleSuperTableUpdate(SMeta *pMeta, const SMetaEntry *pEntry metaFetchEntryFree(&pOldEntry); return code; } - // TAOS_CHECK_RETURN(metaGetSubtables(pMeta, pEntry->uid, uids)); TAOS_CHECK_RETURN(tsdbCacheNewSTableColumn(pTsdb, uids, cid, col_type)); } else if (deltaCol == -1) { int16_t cid = -1; @@ -1667,7 +1666,6 @@ static int32_t metaHandleSuperTableUpdate(SMeta *pMeta, const SMetaEntry *pEntry metaFetchEntryFree(&pOldEntry); return code; } - // TAOS_CHECK_RETURN(metaGetSubtables(pMeta, pEntry->uid, uids)); TAOS_CHECK_RETURN(tsdbCacheDropSTableColumn(pTsdb, uids, cid, hasPrimaryKey)); } } diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index ee37dd1f90..2dbc89f58f 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -28,26 +28,14 @@ int32_t metaUpdateTableColCompress2(SMeta *pMeta, int64_t version, SVAlterTbReq int32_t metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSchema *pSchema); -int32_t metaDelJsonVarFromIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSchema *pSchema); -static int metaSaveToTbDb(SMeta *pMeta, const SMetaEntry *pME); -static int metaUpdateUidIdx(SMeta *pMeta, const SMetaEntry *pME); -static int metaUpdateNameIdx(SMeta *pMeta, const SMetaEntry *pME); -static void metaUpdateTtl(SMeta *pMeta, const SMetaEntry *pME); -static int metaUpdateChangeTime(SMeta *pMeta, tb_uid_t uid, int64_t changeTimeMs); -static int metaSaveToSkmDb(SMeta *pMeta, const SMetaEntry *pME); -static int metaUpdateCtbIdx(SMeta *pMeta, const SMetaEntry *pME); -static int metaUpdateSuidIdx(SMeta *pMeta, const SMetaEntry *pME); -static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry); -static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type, tb_uid_t *pSuid, int8_t *pSysTbl); -void metaDestroyTagIdxKey(STagIdxKey *pTagIdxKey); +int32_t metaDelJsonVarFromIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSchema *pSchema); +static int metaUpdateChangeTime(SMeta *pMeta, tb_uid_t uid, int64_t changeTimeMs); +static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type, tb_uid_t *pSuid, int8_t *pSysTbl); +void metaDestroyTagIdxKey(STagIdxKey *pTagIdxKey); // opt ins_tables query -static int metaUpdateBtimeIdx(SMeta *pMeta, const SMetaEntry *pME); static int metaDeleteBtimeIdx(SMeta *pMeta, const SMetaEntry *pME); -static int metaUpdateNcolIdx(SMeta *pMeta, const SMetaEntry *pME); static int metaDeleteNcolIdx(SMeta *pMeta, const SMetaEntry *pME); -int metaHandleEntry(SMeta *pMeta, const SMetaEntry *pME); - int32_t updataTableColCmpr(SColCmprWrapper *pWp, SSchema *pSchema, int8_t add, uint32_t compress) { int32_t nCols = pWp->nCols; int32_t ver = pWp->version; @@ -80,22 +68,6 @@ int32_t updataTableColCmpr(SColCmprWrapper *pWp, SSchema *pSchema, int8_t add, u } return 0; } -static void metaGetEntryInfo(const SMetaEntry *pEntry, SMetaInfo *pInfo) { - pInfo->uid = pEntry->uid; - pInfo->version = pEntry->version; - if (pEntry->type == TSDB_SUPER_TABLE) { - pInfo->suid = pEntry->uid; - pInfo->skmVer = pEntry->stbEntry.schemaRow.version; - } else if (pEntry->type == TSDB_CHILD_TABLE) { - pInfo->suid = pEntry->ctbEntry.suid; - pInfo->skmVer = 0; - } else if (pEntry->type == TSDB_NORMAL_TABLE) { - pInfo->suid = 0; - pInfo->skmVer = pEntry->ntbEntry.schemaRow.version; - } else { - metaError("meta/table: invalide table type: %" PRId8 " get entry info failed.", pEntry->type); - } -} int metaUpdateMetaRsp(tb_uid_t uid, char *tbName, SSchemaWrapper *pSchema, STableMetaRsp *pMetaRsp) { pMetaRsp->pSchemas = taosMemoryMalloc(pSchema->nCols * sizeof(SSchema)); @@ -303,991 +275,6 @@ void metaTimeSeriesNotifyCheck(SMeta *pMeta) { #endif } -static int metaCreateSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { - SMetaEntry me = {0}; - int kLen = 0; - int vLen = 0; - const void *pKey = NULL; - const void *pVal = NULL; - void *pBuf = NULL; - int32_t szBuf = 0; - void *p = NULL; - int32_t code = 0; - - // validate req - void *pData = NULL; - int nData = 0; - if (tdbTbGet(pMeta->pNameIdx, pReq->name, strlen(pReq->name) + 1, &pData, &nData) == 0) { - tb_uid_t uid = *(tb_uid_t *)pData; - tdbFree(pData); - SMetaInfo info; - if (metaGetInfo(pMeta, uid, &info, NULL) == TSDB_CODE_NOT_FOUND) { - return terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST; - } - if (info.uid == info.suid) { - return 0; - } else { - return terrno = TSDB_CODE_TDB_TABLE_ALREADY_EXIST; - } - } - - // set structs - me.version = version; - me.type = TSDB_SUPER_TABLE; - me.uid = pReq->suid; - me.name = pReq->name; - me.stbEntry.schemaRow = pReq->schemaRow; - me.stbEntry.schemaTag = pReq->schemaTag; - if (pReq->rollup) { - TABLE_SET_ROLLUP(me.flags); - me.stbEntry.rsmaParam = pReq->rsmaParam; - } - if (pReq->colCmpred) { - TABLE_SET_COL_COMPRESSED(me.flags); - me.colCmpr = pReq->colCmpr; - } - - code = metaHandleEntry(pMeta, &me); - if (code) goto _err; - - ++pMeta->pVnode->config.vndStats.numOfSTables; - - pMeta->changed = true; - metaDebug("vgId:%d, stb:%s is created, suid:%" PRId64, TD_VID(pMeta->pVnode), pReq->name, pReq->suid); - - return 0; - -_err: - metaError("vgId:%d, failed to create stb:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, pReq->suid, - tstrerror(terrno)); - return code; -} - -static int metaDropSTable(SMeta *pMeta, int64_t verison, SVDropStbReq *pReq, SArray *tbUidList) { - void *pKey = NULL; - int nKey = 0; - void *pData = NULL; - int nData = 0; - int c = 0; - int rc = 0; - int32_t lino; - int32_t ret; - - // check if super table exists - rc = tdbTbGet(pMeta->pNameIdx, pReq->name, strlen(pReq->name) + 1, &pData, &nData); - if (rc < 0 || *(tb_uid_t *)pData != pReq->suid) { - tdbFree(pData); - return terrno = TSDB_CODE_TDB_STB_NOT_EXIST; - } - - // drop all child tables - TBC *pCtbIdxc = NULL; - - rc = tdbTbcOpen(pMeta->pCtbIdx, &pCtbIdxc, NULL); - if (rc) { - return (terrno = rc); - } - - rc = tdbTbcMoveTo(pCtbIdxc, &(SCtbIdxKey){.suid = pReq->suid, .uid = INT64_MIN}, sizeof(SCtbIdxKey), &c); - if (rc < 0) { - tdbTbcClose(pCtbIdxc); - metaWLock(pMeta); - goto _drop_super_table; - } - - for (;;) { - rc = tdbTbcNext(pCtbIdxc, &pKey, &nKey, NULL, NULL); - if (rc < 0) break; - - if (((SCtbIdxKey *)pKey)->suid < pReq->suid) { - continue; - } else if (((SCtbIdxKey *)pKey)->suid > pReq->suid) { - break; - } - - if (taosArrayPush(tbUidList, &(((SCtbIdxKey *)pKey)->uid)) == NULL) { - tdbFree(pKey); - tdbTbcClose(pCtbIdxc); - return terrno; - } - } - - tdbTbcClose(pCtbIdxc); - - ret = tsdbCacheDropSubTables(pMeta->pVnode->pTsdb, tbUidList, pReq->suid); - if (ret < 0) { - metaError("vgId:%d, failed to drop stb:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, pReq->suid, - tstrerror(terrno)); - } - - metaWLock(pMeta); - - for (int32_t iChild = 0; iChild < taosArrayGetSize(tbUidList); iChild++) { - tb_uid_t uid = *(tb_uid_t *)taosArrayGet(tbUidList, iChild); - ret = metaDropTableByUid(pMeta, uid, NULL, NULL, NULL); - if (ret < 0) { - metaError("vgId:%d, failed to drop stb:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, - pReq->suid, tstrerror(terrno)); - } - } - - // drop super table -_drop_super_table: - tdbTbGet(pMeta->pUidIdx, &pReq->suid, sizeof(tb_uid_t), &pData, &nData); - ret = tdbTbDelete(pMeta->pTbDb, &(STbDbKey){.version = ((SUidIdxVal *)pData)[0].version, .uid = pReq->suid}, - sizeof(STbDbKey), pMeta->txn); - if (ret < 0) { - metaError("vgId:%d, failed to drop stb:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, pReq->suid, - tstrerror(terrno)); - } - - ret = tdbTbDelete(pMeta->pNameIdx, pReq->name, strlen(pReq->name) + 1, pMeta->txn); - if (ret < 0) { - metaError("vgId:%d, failed to drop stb:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, pReq->suid, - tstrerror(terrno)); - } - - ret = metaCacheDrop(pMeta, pReq->suid); - if (ret < 0) { - metaError("vgId:%d, failed to drop stb:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, pReq->suid, - tstrerror(terrno)); - } - - ret = tdbTbDelete(pMeta->pUidIdx, &pReq->suid, sizeof(tb_uid_t), pMeta->txn); - if (ret < 0) { - metaError("vgId:%d, failed to drop stb:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, pReq->suid, - tstrerror(terrno)); - } - - ret = tdbTbDelete(pMeta->pSuidIdx, &pReq->suid, sizeof(tb_uid_t), pMeta->txn); - if (ret < 0) { - metaError("vgId:%d, failed to drop stb:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, pReq->suid, - tstrerror(terrno)); - } - - ret = metaStatsCacheDrop(pMeta, pReq->suid); - if (ret < 0) { - metaError("vgId:%d, failed to drop stb:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, pReq->suid, - tstrerror(terrno)); - } - - metaULock(pMeta); - - metaUpdTimeSeriesNum(pMeta); - - pMeta->changed = true; - -_exit: - tdbFree(pKey); - tdbFree(pData); - metaDebug("vgId:%d, super table %s uid:%" PRId64 " is dropped", TD_VID(pMeta->pVnode), pReq->name, pReq->suid); - return 0; -} - -static int32_t metaGetSubtables(SMeta *pMeta, int64_t suid, SArray *uids) { - if (!uids) return TSDB_CODE_INVALID_PARA; - - int c = 0; - void *pKey = NULL; - int nKey = 0; - TBC *pCtbIdxc = NULL; - - TAOS_CHECK_RETURN(tdbTbcOpen(pMeta->pCtbIdx, &pCtbIdxc, NULL)); - int rc = tdbTbcMoveTo(pCtbIdxc, &(SCtbIdxKey){.suid = suid, .uid = INT64_MIN}, sizeof(SCtbIdxKey), &c); - if (rc < 0) { - tdbTbcClose(pCtbIdxc); - metaWLock(pMeta); - return 0; - } - - for (;;) { - rc = tdbTbcNext(pCtbIdxc, &pKey, &nKey, NULL, NULL); - if (rc < 0) break; - - if (((SCtbIdxKey *)pKey)->suid < suid) { - continue; - } else if (((SCtbIdxKey *)pKey)->suid > suid) { - break; - } - - if (taosArrayPush(uids, &(((SCtbIdxKey *)pKey)->uid)) == NULL) { - tdbFree(pKey); - tdbTbcClose(pCtbIdxc); - return terrno; - } - } - - tdbFree(pKey); - - tdbTbcClose(pCtbIdxc); - return 0; -} - -static int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { - SMetaEntry oStbEntry = {0}; - SMetaEntry nStbEntry = {0}; - TBC *pUidIdxc = NULL; - TBC *pTbDbc = NULL; - const void *pData; - int nData; - int64_t oversion; - SDecoder dc = {0}; - int32_t ret; - int32_t c = -2; - - TAOS_CHECK_RETURN(tdbTbcOpen(pMeta->pUidIdx, &pUidIdxc, NULL)); - ret = tdbTbcMoveTo(pUidIdxc, &pReq->suid, sizeof(tb_uid_t), &c); - if (ret < 0 || c) { - tdbTbcClose(pUidIdxc); - - return terrno = TSDB_CODE_TDB_STB_NOT_EXIST; - } - - ret = tdbTbcGet(pUidIdxc, NULL, NULL, &pData, &nData); - if (ret < 0) { - tdbTbcClose(pUidIdxc); - - return terrno = TSDB_CODE_TDB_STB_NOT_EXIST; - } - - oversion = ((SUidIdxVal *)pData)[0].version; - - TAOS_CHECK_RETURN(tdbTbcOpen(pMeta->pTbDb, &pTbDbc, NULL)); - ret = tdbTbcMoveTo(pTbDbc, &((STbDbKey){.uid = pReq->suid, .version = oversion}), sizeof(STbDbKey), &c); - if (!(ret == 0 && c == 0)) { - tdbTbcClose(pUidIdxc); - tdbTbcClose(pTbDbc); - - metaError("meta/table: invalide ret: %" PRId32 " or c: %" PRId32 "alter stb failed.", ret, c); - return terrno = TSDB_CODE_TDB_STB_NOT_EXIST; - } - - ret = tdbTbcGet(pTbDbc, NULL, NULL, &pData, &nData); - if (ret < 0) { - tdbTbcClose(pUidIdxc); - tdbTbcClose(pTbDbc); - - return terrno = TSDB_CODE_TDB_STB_NOT_EXIST; - } - - if ((oStbEntry.pBuf = taosMemoryMalloc(nData)) == NULL) { - tdbTbcClose(pTbDbc); - tdbTbcClose(pUidIdxc); - return terrno; - } - memcpy(oStbEntry.pBuf, pData, nData); - tDecoderInit(&dc, oStbEntry.pBuf, nData); - ret = metaDecodeEntry(&dc, &oStbEntry); - if (ret < 0) { - metaError("vgId:%d, failed to decode stb:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, - pReq->suid, tstrerror(ret)); - tDecoderClear(&dc); - tdbTbcClose(pTbDbc); - tdbTbcClose(pUidIdxc); - return terrno; - } - - nStbEntry.version = version; - nStbEntry.type = TSDB_SUPER_TABLE; - nStbEntry.uid = pReq->suid; - nStbEntry.name = pReq->name; - nStbEntry.stbEntry.schemaRow = pReq->schemaRow; - nStbEntry.stbEntry.schemaTag = pReq->schemaTag; - nStbEntry.colCmpr = pReq->colCmpr; - TABLE_SET_COL_COMPRESSED(nStbEntry.flags); - - int nCols = pReq->schemaRow.nCols; - int onCols = oStbEntry.stbEntry.schemaRow.nCols; - int32_t deltaCol = nCols - onCols; - bool updStat = deltaCol != 0 && !metaTbInFilterCache(pMeta, pReq->name, 1); - - if (!TSDB_CACHE_NO(pMeta->pVnode->config)) { - STsdb *pTsdb = pMeta->pVnode->pTsdb; - SArray *uids = taosArrayInit(8, sizeof(int64_t)); - if (uids == NULL) { - if (oStbEntry.pBuf) taosMemoryFree(oStbEntry.pBuf); - tDecoderClear(&dc); - tdbTbcClose(pTbDbc); - tdbTbcClose(pUidIdxc); - return terrno; - } - if (deltaCol == 1) { - int16_t cid = pReq->schemaRow.pSchema[nCols - 1].colId; - int8_t col_type = pReq->schemaRow.pSchema[nCols - 1].type; - - TAOS_CHECK_RETURN(metaGetSubtables(pMeta, pReq->suid, uids)); - TAOS_CHECK_RETURN(tsdbCacheNewSTableColumn(pTsdb, uids, cid, col_type)); - } else if (deltaCol == -1) { - int16_t cid = -1; - bool hasPrimaryKey = false; - if (onCols >= 2) { - hasPrimaryKey = (oStbEntry.stbEntry.schemaRow.pSchema[1].flags & COL_IS_KEY) ? true : false; - } - for (int i = 0, j = 0; i < nCols && j < onCols; ++i, ++j) { - if (pReq->schemaRow.pSchema[i].colId != oStbEntry.stbEntry.schemaRow.pSchema[j].colId) { - cid = oStbEntry.stbEntry.schemaRow.pSchema[j].colId; - break; - } - } - - if (cid != -1) { - TAOS_CHECK_RETURN(metaGetSubtables(pMeta, pReq->suid, uids)); - TAOS_CHECK_RETURN(tsdbCacheDropSTableColumn(pTsdb, uids, cid, hasPrimaryKey)); - } - } - if (uids) taosArrayDestroy(uids); - - tsdbCacheInvalidateSchema(pTsdb, pReq->suid, -1, pReq->schemaRow.version); - } - - metaWLock(pMeta); - // compare two entry - if (oStbEntry.stbEntry.schemaRow.version != pReq->schemaRow.version) { - ret = metaSaveToSkmDb(pMeta, &nStbEntry); - if (ret < 0) { - metaError("vgId:%d, failed to save skm db:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, - pReq->suid, tstrerror(ret)); - } - } - - // update table.db - ret = metaSaveToTbDb(pMeta, &nStbEntry); - if (ret < 0) { - metaError("vgId:%d, failed to save tb db:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, - pReq->suid, tstrerror(ret)); - } - - // update uid index - ret = metaUpdateUidIdx(pMeta, &nStbEntry); - if (ret < 0) { - metaError("vgId:%d, failed to update uid idx:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, - pReq->suid, tstrerror(ret)); - } - - // metaStatsCacheDrop(pMeta, nStbEntry.uid); - - if (updStat) { - metaUpdateStbStats(pMeta, pReq->suid, 0, deltaCol); - } - metaULock(pMeta); - - if (updStat) { - int64_t ctbNum; - ret = metaGetStbStats(pMeta->pVnode, pReq->suid, &ctbNum, NULL); - if (ret < 0) { - metaError("vgId:%d, failed to get stb stats:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, - pReq->suid, tstrerror(ret)); - } - pMeta->pVnode->config.vndStats.numOfTimeSeries += (ctbNum * deltaCol); - metaTimeSeriesNotifyCheck(pMeta); - } - - pMeta->changed = true; - -_exit: - if (oStbEntry.pBuf) taosMemoryFree(oStbEntry.pBuf); - tDecoderClear(&dc); - tdbTbcClose(pTbDbc); - tdbTbcClose(pUidIdxc); - return 0; -} - -static int metaAddIndexToSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { - SMetaEntry oStbEntry = {0}; - SMetaEntry nStbEntry = {0}; - STbDbKey tbDbKey = {0}; - - TBC *pUidIdxc = NULL; - TBC *pTbDbc = NULL; - void *pData = NULL; - int nData = 0; - int64_t oversion; - SDecoder dc = {0}; - int32_t ret; - int32_t c = -2; - tb_uid_t suid = pReq->suid; - int32_t code = 0; - - // get super table - if ((code = tdbTbGet(pMeta->pUidIdx, &suid, sizeof(tb_uid_t), &pData, &nData)) != 0) { - goto _err; - } - - tbDbKey.uid = suid; - tbDbKey.version = ((SUidIdxVal *)pData)[0].version; - if ((code = tdbTbGet(pMeta->pTbDb, &tbDbKey, sizeof(tbDbKey), &pData, &nData)) != 0) { - goto _err; - } - - tDecoderInit(&dc, pData, nData); - code = metaDecodeEntry(&dc, &oStbEntry); - if (code < 0) { - goto _err; - } - - if (oStbEntry.stbEntry.schemaTag.pSchema == NULL || oStbEntry.stbEntry.schemaTag.pSchema == NULL) { - code = TSDB_CODE_INVALID_PARA; - goto _err; - } - - if (oStbEntry.stbEntry.schemaTag.version == pReq->schemaTag.version) { - code = TSDB_CODE_INVALID_PARA; - goto _err; - } - - if (oStbEntry.stbEntry.schemaTag.nCols != pReq->schemaTag.nCols) { - code = TSDB_CODE_INVALID_PARA; - goto _err; - } - - int diffIdx = -1; - for (int i = 0; i < pReq->schemaTag.nCols; i++) { - SSchema *pNew = pReq->schemaTag.pSchema + i; - SSchema *pOld = oStbEntry.stbEntry.schemaTag.pSchema + i; - if (pNew->type != pOld->type || pNew->colId != pOld->colId || pNew->bytes != pOld->bytes || - strncmp(pOld->name, pNew->name, sizeof(pNew->name))) { - code = TSDB_CODE_INVALID_PARA; - goto _err; - } - if (IS_IDX_ON(pNew) && !IS_IDX_ON(pOld)) { - // if (diffIdx != -1) goto _err; - diffIdx = i; - break; - } - } - - if (diffIdx == -1) { - code = TSDB_CODE_INVALID_PARA; - goto _err; - } - - // Get target schema info - SSchemaWrapper *pTagSchema = &pReq->schemaTag; - if (pTagSchema->nCols == 1 && pTagSchema->pSchema[0].type == TSDB_DATA_TYPE_JSON) { - code = TSDB_CODE_INVALID_PARA; - goto _err; - } - SSchema *pCol = pTagSchema->pSchema + diffIdx; - - /* - * iterator all pTdDbc by uid and version - */ - TBC *pCtbIdxc = NULL; - code = tdbTbcOpen(pMeta->pCtbIdx, &pCtbIdxc, NULL); - if (code != 0) { - goto _err; - } - - code = tdbTbcMoveTo(pCtbIdxc, &(SCtbIdxKey){.suid = suid, .uid = INT64_MIN}, sizeof(SCtbIdxKey), &c); - if (code < 0) { - tdbTbcClose(pCtbIdxc); - goto _err; - } - for (;;) { - void *pKey = NULL, *pVal = NULL; - int nKey = 0, nVal = 0; - code = tdbTbcNext(pCtbIdxc, &pKey, &nKey, &pVal, &nVal); - if (code < 0) { - tdbFree(pKey); - tdbFree(pVal); - tdbTbcClose(pCtbIdxc); - pCtbIdxc = NULL; - break; - } - if (((SCtbIdxKey *)pKey)->suid != suid) { - tdbFree(pKey); - tdbFree(pVal); - continue; - } - STagIdxKey *pTagIdxKey = NULL; - int32_t nTagIdxKey; - - const void *pTagData = NULL; - int32_t nTagData = 0; - - SCtbIdxKey *table = (SCtbIdxKey *)pKey; - STagVal tagVal = {.cid = pCol->colId}; - if (tTagGet((const STag *)pVal, &tagVal)) { - if (IS_VAR_DATA_TYPE(pCol->type)) { - pTagData = tagVal.pData; - nTagData = (int32_t)tagVal.nData; - } else { - pTagData = &(tagVal.i64); - nTagData = tDataTypes[pCol->type].bytes; - } - } else { - if (!IS_VAR_DATA_TYPE(pCol->type)) { - nTagData = tDataTypes[pCol->type].bytes; - } - } - code = metaCreateTagIdxKey(suid, pCol->colId, pTagData, nTagData, pCol->type, table->uid, &pTagIdxKey, &nTagIdxKey); - tdbFree(pKey); - tdbFree(pVal); - if (code < 0) { - metaDestroyTagIdxKey(pTagIdxKey); - tdbTbcClose(pCtbIdxc); - goto _err; - } - - metaWLock(pMeta); - ret = tdbTbUpsert(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, NULL, 0, pMeta->txn); - if (ret < 0) { - metaError("vgId:%d, failed to upsert tag idx key:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, - pReq->suid, tstrerror(ret)); - } - metaULock(pMeta); - - metaDestroyTagIdxKey(pTagIdxKey); - pTagIdxKey = NULL; - } - - nStbEntry.version = version; - nStbEntry.type = TSDB_SUPER_TABLE; - nStbEntry.uid = pReq->suid; - nStbEntry.name = pReq->name; - nStbEntry.stbEntry.schemaRow = pReq->schemaRow; - nStbEntry.stbEntry.schemaTag = pReq->schemaTag; - nStbEntry.colCmpr = pReq->colCmpr; - - metaWLock(pMeta); - // update table.db - ret = metaSaveToTbDb(pMeta, &nStbEntry); - if (ret < 0) { - metaError("vgId:%d, failed to save tb db:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, - pReq->suid, tstrerror(ret)); - } - // update uid index - ret = metaUpdateUidIdx(pMeta, &nStbEntry); - if (ret < 0) { - metaError("vgId:%d, failed to update uid idx:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, - pReq->suid, tstrerror(ret)); - } - metaULock(pMeta); - - if (oStbEntry.pBuf) taosMemoryFree(oStbEntry.pBuf); - tDecoderClear(&dc); - tdbFree(pData); - - tdbTbcClose(pCtbIdxc); - return TSDB_CODE_SUCCESS; -_err: - if (oStbEntry.pBuf) taosMemoryFree(oStbEntry.pBuf); - tDecoderClear(&dc); - tdbFree(pData); - - return code; -} - -static int metaDropIndexFromSTable(SMeta *pMeta, int64_t version, SDropIndexReq *pReq) { - int32_t code = 0; - SMetaEntry oStbEntry = {0}; - SMetaEntry nStbEntry = {0}; - - STbDbKey tbDbKey = {0}; - TBC *pUidIdxc = NULL; - TBC *pTbDbc = NULL; - int ret = 0; - int c = -2; - void *pData = NULL; - int nData = 0; - int64_t oversion; - SDecoder dc = {0}; - - tb_uid_t suid = pReq->stbUid; - - if ((code = tdbTbGet(pMeta->pUidIdx, &suid, sizeof(tb_uid_t), &pData, &nData)) != 0) { - goto _err; - } - - tbDbKey.uid = suid; - tbDbKey.version = ((SUidIdxVal *)pData)[0].version; - if ((code = tdbTbGet(pMeta->pTbDb, &tbDbKey, sizeof(tbDbKey), &pData, &nData)) != 0) { - goto _err; - } - - tDecoderInit(&dc, pData, nData); - code = metaDecodeEntry(&dc, &oStbEntry); - if (code != 0) { - goto _err; - } - - SSchema *pCol = NULL; - int32_t colId = -1; - for (int i = 0; i < oStbEntry.stbEntry.schemaTag.nCols; i++) { - SSchema *schema = oStbEntry.stbEntry.schemaTag.pSchema + i; - if (0 == strncmp(schema->name, pReq->colName, sizeof(pReq->colName))) { - if (IS_IDX_ON(schema)) { - pCol = schema; - } - break; - } - } - - if (pCol == NULL) { - metaError("vgId:%d, failed to drop index on %s.%s,since %s", TD_VID(pMeta->pVnode), pReq->stb, pReq->colName, - tstrerror(TSDB_CODE_VND_COL_NOT_EXISTS)); - code = 0; - - goto _err; - } - - /* - * iterator all pTdDbc by uid and version - */ - TBC *pCtbIdxc = NULL; - code = tdbTbcOpen(pMeta->pCtbIdx, &pCtbIdxc, NULL); - if (code != 0) { - goto _err; - } - - code = tdbTbcMoveTo(pCtbIdxc, &(SCtbIdxKey){.suid = suid, .uid = INT64_MIN}, sizeof(SCtbIdxKey), &c); - if (code < 0) { - tdbTbcClose(pCtbIdxc); - goto _err; - } - for (;;) { - void *pKey = NULL, *pVal = NULL; - int nKey = 0, nVal = 0; - - code = tdbTbcNext(pCtbIdxc, &pKey, &nKey, &pVal, &nVal); - if (code < 0) { - tdbFree(pKey); - tdbFree(pVal); - tdbTbcClose(pCtbIdxc); - pCtbIdxc = NULL; - break; - } - if (((SCtbIdxKey *)pKey)->suid != suid) { - tdbFree(pKey); - tdbFree(pVal); - continue; - } - STagIdxKey *pTagIdxKey = NULL; - int32_t nTagIdxKey; - - const void *pTagData = NULL; - int32_t nTagData = 0; - - SCtbIdxKey *table = (SCtbIdxKey *)pKey; - STagVal tagVal = {.cid = pCol->colId}; - if (tTagGet((const STag *)pVal, &tagVal)) { - if (IS_VAR_DATA_TYPE(pCol->type)) { - pTagData = tagVal.pData; - nTagData = (int32_t)tagVal.nData; - } else { - pTagData = &(tagVal.i64); - nTagData = tDataTypes[pCol->type].bytes; - } - } else { - if (!IS_VAR_DATA_TYPE(pCol->type)) { - nTagData = tDataTypes[pCol->type].bytes; - } - } - - code = metaCreateTagIdxKey(suid, pCol->colId, pTagData, nTagData, pCol->type, table->uid, &pTagIdxKey, &nTagIdxKey); - tdbFree(pKey); - tdbFree(pVal); - if (code < 0) { - metaDestroyTagIdxKey(pTagIdxKey); - tdbTbcClose(pCtbIdxc); - goto _err; - } - - metaWLock(pMeta); - ret = tdbTbDelete(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, pMeta->txn); - if (ret < 0) { - metaError("vgId:%d, failed to delete tag idx key:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->stb, - pReq->stbUid, tstrerror(ret)); - } - metaULock(pMeta); - metaDestroyTagIdxKey(pTagIdxKey); - pTagIdxKey = NULL; - } - - // clear idx flag - SSCHMEA_SET_IDX_OFF(pCol); - - nStbEntry.version = version; - nStbEntry.type = TSDB_SUPER_TABLE; - nStbEntry.uid = oStbEntry.uid; - nStbEntry.name = oStbEntry.name; - - SSchemaWrapper *row = tCloneSSchemaWrapper(&oStbEntry.stbEntry.schemaRow); - SSchemaWrapper *tag = tCloneSSchemaWrapper(&oStbEntry.stbEntry.schemaTag); - SColCmprWrapper *cmpr = tCloneSColCmprWrapper(&oStbEntry.colCmpr); - if (row == NULL || tag == NULL || cmpr == NULL) { - tDeleteSchemaWrapper(row); - tDeleteSchemaWrapper(tag); - tDeleteSColCmprWrapper(cmpr); - code = TSDB_CODE_OUT_OF_MEMORY; - - tdbTbcClose(pCtbIdxc); - goto _err; - } - - nStbEntry.stbEntry.schemaRow = *row; - nStbEntry.stbEntry.schemaTag = *tag; - nStbEntry.stbEntry.rsmaParam = oStbEntry.stbEntry.rsmaParam; - nStbEntry.colCmpr = *cmpr; - - nStbEntry.colCmpr = oStbEntry.colCmpr; - - metaWLock(pMeta); - // update table.db - ret = metaSaveToTbDb(pMeta, &nStbEntry); - if (ret < 0) { - metaError("vgId:%d, failed to save tb db:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->stb, - pReq->stbUid, tstrerror(ret)); - } - // update uid index - ret = metaUpdateUidIdx(pMeta, &nStbEntry); - if (ret < 0) { - metaError("vgId:%d, failed to update uid idx:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->stb, - pReq->stbUid, tstrerror(ret)); - } - metaULock(pMeta); - - tDeleteSchemaWrapper(tag); - tDeleteSchemaWrapper(row); - tDeleteSColCmprWrapper(cmpr); - - if (oStbEntry.pBuf) taosMemoryFree(oStbEntry.pBuf); - tDecoderClear(&dc); - tdbFree(pData); - - tdbTbcClose(pCtbIdxc); - return TSDB_CODE_SUCCESS; -_err: - if (oStbEntry.pBuf) taosMemoryFree(oStbEntry.pBuf); - tDecoderClear(&dc); - tdbFree(pData); - - return code; -} - -static int metaCreateTable(SMeta *pMeta, int64_t ver, SVCreateTbReq *pReq, STableMetaRsp **pMetaRsp) { - SMetaEntry me = {0}; - SMetaReader mr = {0}; - int32_t ret; - - // validate message - if (pReq->type != TSDB_CHILD_TABLE && pReq->type != TSDB_NORMAL_TABLE) { - terrno = TSDB_CODE_INVALID_MSG; - goto _err; - } - - if (pReq->type == TSDB_CHILD_TABLE) { - tb_uid_t suid = metaGetTableEntryUidByName(pMeta, pReq->ctb.stbName); - if (suid != pReq->ctb.suid) { - return terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST; - } - } - - // validate req - metaReaderDoInit(&mr, pMeta, META_READER_LOCK); - if (metaGetTableEntryByName(&mr, pReq->name) == 0) { - if (pReq->type == TSDB_CHILD_TABLE && pReq->ctb.suid != mr.me.ctbEntry.suid) { - metaReaderClear(&mr); - return terrno = TSDB_CODE_TDB_TABLE_IN_OTHER_STABLE; - } - pReq->uid = mr.me.uid; - if (pReq->type == TSDB_CHILD_TABLE) { - pReq->ctb.suid = mr.me.ctbEntry.suid; - } - metaReaderClear(&mr); - return terrno = TSDB_CODE_TDB_TABLE_ALREADY_EXIST; - } else if (terrno == TSDB_CODE_PAR_TABLE_NOT_EXIST) { - terrno = TSDB_CODE_SUCCESS; - } - metaReaderClear(&mr); - - bool sysTbl = (pReq->type == TSDB_CHILD_TABLE) && metaTbInFilterCache(pMeta, pReq->ctb.stbName, 1); - - if (!sysTbl && ((terrno = grantCheck(TSDB_GRANT_TIMESERIES)) < 0)) goto _err; - - // build SMetaEntry - SVnodeStats *pStats = &pMeta->pVnode->config.vndStats; - me.version = ver; - me.type = pReq->type; - me.uid = pReq->uid; - me.name = pReq->name; - if (me.type == TSDB_CHILD_TABLE) { - me.ctbEntry.btime = pReq->btime; - me.ctbEntry.ttlDays = pReq->ttl; - me.ctbEntry.commentLen = pReq->commentLen; - me.ctbEntry.comment = pReq->comment; - me.ctbEntry.suid = pReq->ctb.suid; - me.ctbEntry.pTags = pReq->ctb.pTag; - -#ifdef TAG_FILTER_DEBUG - SArray *pTagVals = NULL; - int32_t code = tTagToValArray((STag *)pReq->ctb.pTag, &pTagVals); - for (int i = 0; i < taosArrayGetSize(pTagVals); i++) { - STagVal *pTagVal = (STagVal *)taosArrayGet(pTagVals, i); - - if (IS_VAR_DATA_TYPE(pTagVal->type)) { - char *buf = taosMemoryCalloc(pTagVal->nData + 1, 1); - memcpy(buf, pTagVal->pData, pTagVal->nData); - metaDebug("metaTag table:%s varchar index:%d cid:%d type:%d value:%s", pReq->name, i, pTagVal->cid, - pTagVal->type, buf); - taosMemoryFree(buf); - } else { - double val = 0; - GET_TYPED_DATA(val, double, pTagVal->type, &pTagVal->i64); - metaDebug("metaTag table:%s number index:%d cid:%d type:%d value:%f", pReq->name, i, pTagVal->cid, - pTagVal->type, val); - } - } -#endif - - ++pStats->numOfCTables; - - if (!sysTbl) { - int32_t nCols = 0; - ret = metaGetStbStats(pMeta->pVnode, me.ctbEntry.suid, 0, &nCols); - if (ret < 0) { - metaError("vgId:%d, failed to get stb stats:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, - pReq->ctb.suid, tstrerror(ret)); - } - pStats->numOfTimeSeries += nCols - 1; - } - - metaWLock(pMeta); - metaUpdateStbStats(pMeta, me.ctbEntry.suid, 1, 0); - ret = metaUidCacheClear(pMeta, me.ctbEntry.suid); - if (ret < 0) { - metaError("vgId:%d, failed to clear uid cache:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, - pReq->ctb.suid, tstrerror(ret)); - } - ret = metaTbGroupCacheClear(pMeta, me.ctbEntry.suid); - if (ret < 0) { - metaError("vgId:%d, failed to clear group cache:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, - pReq->ctb.suid, tstrerror(ret)); - } - metaULock(pMeta); - - if (!TSDB_CACHE_NO(pMeta->pVnode->config)) { - ret = tsdbCacheNewTable(pMeta->pVnode->pTsdb, me.uid, me.ctbEntry.suid, NULL); - if (ret < 0) { - metaError("vgId:%d, failed to create table:%s since %s", TD_VID(pMeta->pVnode), pReq->name, tstrerror(ret)); - goto _err; - } - } - } else { - me.ntbEntry.btime = pReq->btime; - me.ntbEntry.ttlDays = pReq->ttl; - me.ntbEntry.commentLen = pReq->commentLen; - me.ntbEntry.comment = pReq->comment; - me.ntbEntry.schemaRow = pReq->ntb.schemaRow; - me.ntbEntry.ncid = me.ntbEntry.schemaRow.pSchema[me.ntbEntry.schemaRow.nCols - 1].colId + 1; - me.colCmpr = pReq->colCmpr; - TABLE_SET_COL_COMPRESSED(me.flags); - - ++pStats->numOfNTables; - pStats->numOfNTimeSeries += me.ntbEntry.schemaRow.nCols - 1; - - if (!TSDB_CACHE_NO(pMeta->pVnode->config)) { - ret = tsdbCacheNewTable(pMeta->pVnode->pTsdb, me.uid, -1, &me.ntbEntry.schemaRow); - if (ret < 0) { - metaError("vgId:%d, failed to create table:%s since %s", TD_VID(pMeta->pVnode), pReq->name, tstrerror(ret)); - goto _err; - } - } - } - - if (metaHandleEntry(pMeta, &me) < 0) goto _err; - - metaTimeSeriesNotifyCheck(pMeta); - - if (pMetaRsp) { - *pMetaRsp = taosMemoryCalloc(1, sizeof(STableMetaRsp)); - - if (*pMetaRsp) { - if (me.type == TSDB_CHILD_TABLE) { - (*pMetaRsp)->tableType = TSDB_CHILD_TABLE; - (*pMetaRsp)->tuid = pReq->uid; - (*pMetaRsp)->suid = pReq->ctb.suid; - tstrncpy((*pMetaRsp)->tbName, pReq->name, strlen(pReq->name) + 1); - } else { - ret = metaUpdateMetaRsp(pReq->uid, pReq->name, &pReq->ntb.schemaRow, *pMetaRsp); - if (ret < 0) { - metaError("vgId:%d, failed to update meta rsp:%s since %s", TD_VID(pMeta->pVnode), pReq->name, - tstrerror(ret)); - } - for (int32_t i = 0; i < pReq->colCmpr.nCols; i++) { - SColCmpr *p = &pReq->colCmpr.pColCmpr[i]; - (*pMetaRsp)->pSchemaExt[i].colId = p->id; - (*pMetaRsp)->pSchemaExt[i].compress = p->alg; - } - } - } - } - - pMeta->changed = true; - metaDebug("vgId:%d, table:%s uid %" PRId64 " is created, type:%" PRId8, TD_VID(pMeta->pVnode), pReq->name, pReq->uid, - pReq->type); - return 0; - -_err: - metaError("vgId:%d, failed to create table:%s type:%s since %s", TD_VID(pMeta->pVnode), pReq->name, - pReq->type == TSDB_CHILD_TABLE ? "child table" : "normal table", tstrerror(terrno)); - return TSDB_CODE_FAILED; -} - -static int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq, SArray *tbUids, tb_uid_t *tbUid) { - void *pData = NULL; - int nData = 0; - int rc = 0; - tb_uid_t uid = 0; - tb_uid_t suid = 0; - int8_t sysTbl = 0; - int type; - - rc = tdbTbGet(pMeta->pNameIdx, pReq->name, strlen(pReq->name) + 1, &pData, &nData); - if (rc < 0) { - return terrno = TSDB_CODE_TDB_TABLE_NOT_EXIST; - } - uid = *(tb_uid_t *)pData; - - metaWLock(pMeta); - rc = metaDropTableByUid(pMeta, uid, &type, &suid, &sysTbl); - metaULock(pMeta); - - if (rc < 0) goto _exit; - - if (!sysTbl && type == TSDB_CHILD_TABLE) { - int32_t nCols = 0; - SVnodeStats *pStats = &pMeta->pVnode->config.vndStats; - if (metaGetStbStats(pMeta->pVnode, suid, NULL, &nCols) == 0) { - pStats->numOfTimeSeries -= nCols - 1; - } - } - - if ((type == TSDB_CHILD_TABLE || type == TSDB_NORMAL_TABLE) && tbUids) { - if (taosArrayPush(tbUids, &uid) == NULL) { - rc = terrno; - goto _exit; - } - - if (!TSDB_CACHE_NO(pMeta->pVnode->config)) { - int32_t ret = tsdbCacheDropTable(pMeta->pVnode->pTsdb, uid, suid, NULL); - if (ret < 0) { - metaError("vgId:%d, failed to drop table:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, uid, - tstrerror(ret)); - } - } - } - - if ((type == TSDB_CHILD_TABLE) && tbUid) { - *tbUid = uid; - } - - pMeta->changed = true; -_exit: - tdbFree(pData); - return rc; -} - static int32_t metaDropTables(SMeta *pMeta, SArray *tbUids) { int32_t code = 0; if (taosArrayGetSize(tbUids) == 0) return TSDB_CODE_SUCCESS; @@ -1399,7 +386,7 @@ static int32_t metaFilterTableByHash(SMeta *pMeta, SArray *uidList) { return 0; } -int32_t metaTrimTables(SMeta *pMeta) { +int32_t metaTrimTables(SMeta *pMeta, int64_t version) { int32_t code = 0; SArray *tbUids = taosArrayInit(8, sizeof(int64_t)); @@ -1679,17 +666,6 @@ static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type, tb_uid_t *p return 0; } -// opt ins_tables -static int metaUpdateBtimeIdx(SMeta *pMeta, const SMetaEntry *pME) { - SBtimeIdxKey btimeKey = {0}; - if (metaBuildBtimeIdxKey(&btimeKey, pME) < 0) { - return 0; - } - metaTrace("vgId:%d, start to save version:%" PRId64 " uid:%" PRId64 " btime:%" PRId64, TD_VID(pMeta->pVnode), - pME->version, pME->uid, btimeKey.btime); - - return tdbTbUpsert(pMeta->pBtimeIdx, &btimeKey, sizeof(btimeKey), NULL, 0, pMeta->txn); -} static int metaDeleteBtimeIdx(SMeta *pMeta, const SMetaEntry *pME) { SBtimeIdxKey btimeKey = {0}; @@ -1699,14 +675,6 @@ static int metaDeleteBtimeIdx(SMeta *pMeta, const SMetaEntry *pME) { return tdbTbDelete(pMeta->pBtimeIdx, &btimeKey, sizeof(btimeKey), pMeta->txn); } -int metaUpdateNcolIdx(SMeta *pMeta, const SMetaEntry *pME) { - SNcolIdxKey ncolKey = {0}; - if (metaBuildNColIdxKey(&ncolKey, pME) < 0) { - return 0; - } - return tdbTbUpsert(pMeta->pNcolIdx, &ncolKey, sizeof(ncolKey), NULL, 0, pMeta->txn); -} - int metaDeleteNcolIdx(SMeta *pMeta, const SMetaEntry *pME) { SNcolIdxKey ncolKey = {0}; if (metaBuildNColIdxKey(&ncolKey, pME) < 0) { @@ -1715,1143 +683,6 @@ int metaDeleteNcolIdx(SMeta *pMeta, const SMetaEntry *pME) { return tdbTbDelete(pMeta->pNcolIdx, &ncolKey, sizeof(ncolKey), pMeta->txn); } -static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTbReq, STableMetaRsp *pMetaRsp) { - void *pVal = NULL; - int nVal = 0; - const void *pData = NULL; - int nData = 0; - int ret = 0; - tb_uid_t uid; - int64_t oversion; - SSchema *pColumn = NULL; - SMetaEntry entry = {0}; - SSchemaWrapper *pSchema; - int c; - bool freeColCmpr = false; - if (pAlterTbReq->colName == NULL) { - metaError("meta/table: null pAlterTbReq->colName"); - return terrno = TSDB_CODE_INVALID_MSG; - } - - // search name index - ret = tdbTbGet(pMeta->pNameIdx, pAlterTbReq->tbName, strlen(pAlterTbReq->tbName) + 1, &pVal, &nVal); - if (ret < 0) { - return terrno = TSDB_CODE_TDB_TABLE_NOT_EXIST; - } - - uid = *(tb_uid_t *)pVal; - tdbFree(pVal); - pVal = NULL; - - // search uid index - TBC *pUidIdxc = NULL; - - TAOS_CHECK_RETURN(tdbTbcOpen(pMeta->pUidIdx, &pUidIdxc, NULL)); - ret = tdbTbcMoveTo(pUidIdxc, &uid, sizeof(uid), &c); - if (c != 0) { - tdbTbcClose(pUidIdxc); - metaError("meta/table: invalide c: %" PRId32 " alt tb column failed.", c); - return TSDB_CODE_FAILED; - } - - ret = tdbTbcGet(pUidIdxc, NULL, NULL, &pData, &nData); - oversion = ((SUidIdxVal *)pData)[0].version; - - // search table.db - TBC *pTbDbc = NULL; - - TAOS_CHECK_RETURN(tdbTbcOpen(pMeta->pTbDb, &pTbDbc, NULL)); - ret = tdbTbcMoveTo(pTbDbc, &((STbDbKey){.uid = uid, .version = oversion}), sizeof(STbDbKey), &c); - if (c != 0) { - tdbTbcClose(pUidIdxc); - tdbTbcClose(pTbDbc); - metaError("meta/table: invalide c: %" PRId32 " alt tb column failed.", c); - return TSDB_CODE_FAILED; - } - - ret = tdbTbcGet(pTbDbc, NULL, NULL, &pData, &nData); - - // get table entry - SDecoder dc = {0}; - if ((entry.pBuf = taosMemoryMalloc(nData)) == NULL) { - tdbTbcClose(pUidIdxc); - tdbTbcClose(pTbDbc); - return terrno; - } - memcpy(entry.pBuf, pData, nData); - tDecoderInit(&dc, entry.pBuf, nData); - ret = metaDecodeEntry(&dc, &entry); - if (ret != 0) { - tdbTbcClose(pUidIdxc); - tdbTbcClose(pTbDbc); - tDecoderClear(&dc); - metaError("meta/table: invalide ret: %" PRId32 " alt tb column failed.", ret); - return ret; - } - - if (entry.type != TSDB_NORMAL_TABLE) { - terrno = TSDB_CODE_VND_INVALID_TABLE_ACTION; - goto _err; - } - // search the column to add/drop/update - pSchema = &entry.ntbEntry.schemaRow; - - // save old entry - SMetaEntry oldEntry = {.type = TSDB_NORMAL_TABLE, .uid = entry.uid}; - oldEntry.ntbEntry.schemaRow.nCols = pSchema->nCols; - - int32_t rowLen = -1; - if (pAlterTbReq->action == TSDB_ALTER_TABLE_ADD_COLUMN || - pAlterTbReq->action == TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES) { - rowLen = 0; - } - - int32_t iCol = 0, jCol = 0; - SSchema *qColumn = NULL; - for (;;) { - qColumn = NULL; - - if (jCol >= pSchema->nCols) break; - qColumn = &pSchema->pSchema[jCol]; - - if (!pColumn && (strcmp(qColumn->name, pAlterTbReq->colName) == 0)) { - pColumn = qColumn; - iCol = jCol; - if (rowLen < 0) break; - } - rowLen += qColumn->bytes; - ++jCol; - } - - entry.version = version; - int tlen; - SSchema *pNewSchema = NULL; - SSchema tScheam; - switch (pAlterTbReq->action) { - case TSDB_ALTER_TABLE_ADD_COLUMN: - case TSDB_ALTER_TABLE_ADD_COLUMN_WITH_COMPRESS_OPTION: - if (pColumn) { - terrno = TSDB_CODE_VND_COL_ALREADY_EXISTS; - goto _err; - } - if ((terrno = grantCheck(TSDB_GRANT_TIMESERIES)) < 0) { - goto _err; - } - if (rowLen + pAlterTbReq->bytes > TSDB_MAX_BYTES_PER_ROW) { - terrno = TSDB_CODE_PAR_INVALID_ROW_LENGTH; - goto _err; - } - pSchema->version++; - pSchema->nCols++; - pNewSchema = taosMemoryMalloc(sizeof(SSchema) * pSchema->nCols); - if (pNewSchema == NULL) { - goto _err; - } - memcpy(pNewSchema, pSchema->pSchema, sizeof(SSchema) * (pSchema->nCols - 1)); - pSchema->pSchema = pNewSchema; - pSchema->pSchema[entry.ntbEntry.schemaRow.nCols - 1].bytes = pAlterTbReq->bytes; - pSchema->pSchema[entry.ntbEntry.schemaRow.nCols - 1].type = pAlterTbReq->type; - pSchema->pSchema[entry.ntbEntry.schemaRow.nCols - 1].flags = pAlterTbReq->flags; - pSchema->pSchema[entry.ntbEntry.schemaRow.nCols - 1].colId = entry.ntbEntry.ncid++; - tstrncpy(pSchema->pSchema[entry.ntbEntry.schemaRow.nCols - 1].name, pAlterTbReq->colName, - strlen(pAlterTbReq->colName) + 1); - - ++pMeta->pVnode->config.vndStats.numOfNTimeSeries; - metaTimeSeriesNotifyCheck(pMeta); - - if (!TSDB_CACHE_NO(pMeta->pVnode->config)) { - int16_t cid = pSchema->pSchema[entry.ntbEntry.schemaRow.nCols - 1].colId; - int8_t col_type = pSchema->pSchema[entry.ntbEntry.schemaRow.nCols - 1].type; - int32_t ret = tsdbCacheNewNTableColumn(pMeta->pVnode->pTsdb, entry.uid, cid, col_type); - if (ret < 0) { - terrno = ret; - goto _err; - } - } - SSchema *pCol = &pSchema->pSchema[entry.ntbEntry.schemaRow.nCols - 1]; - uint32_t compress = pAlterTbReq->action == TSDB_ALTER_TABLE_ADD_COLUMN ? createDefaultColCmprByType(pCol->type) - : pAlterTbReq->compress; - if (updataTableColCmpr(&entry.colCmpr, pCol, 1, compress) != 0) { - metaError("vgId:%d, failed to update table col cmpr:%s uid:%" PRId64, TD_VID(pMeta->pVnode), entry.name, - entry.uid); - } - freeColCmpr = true; - if (entry.colCmpr.nCols != pSchema->nCols) { - if (pNewSchema) taosMemoryFree(pNewSchema); - if (freeColCmpr) taosMemoryFree(entry.colCmpr.pColCmpr); - terrno = TSDB_CODE_VND_INVALID_TABLE_ACTION; - goto _err; - } - break; - case TSDB_ALTER_TABLE_DROP_COLUMN: - if (pColumn == NULL) { - terrno = TSDB_CODE_VND_COL_NOT_EXISTS; - goto _err; - } - if (pColumn->colId == 0) { - terrno = TSDB_CODE_VND_INVALID_TABLE_ACTION; - goto _err; - } - if (tqCheckColModifiable(pMeta->pVnode->pTq, uid, pColumn->colId) != 0) { - terrno = TSDB_CODE_VND_COL_SUBSCRIBED; - goto _err; - } - bool hasPrimayKey = false; - if (pSchema->nCols >= 2) { - hasPrimayKey = pSchema->pSchema[1].flags & COL_IS_KEY ? true : false; - } - - memcpy(&tScheam, pColumn, sizeof(SSchema)); - pSchema->version++; - tlen = (pSchema->nCols - iCol - 1) * sizeof(SSchema); - if (tlen) { - memmove(pColumn, pColumn + 1, tlen); - } - pSchema->nCols--; - - --pMeta->pVnode->config.vndStats.numOfNTimeSeries; - - if (!TSDB_CACHE_NO(pMeta->pVnode->config)) { - int16_t cid = pColumn->colId; - - if (tsdbCacheDropNTableColumn(pMeta->pVnode->pTsdb, entry.uid, cid, hasPrimayKey) != 0) { - metaError("vgId:%d, failed to drop ntable column:%s uid:%" PRId64, TD_VID(pMeta->pVnode), entry.name, - entry.uid); - } - } - - if (updataTableColCmpr(&entry.colCmpr, &tScheam, 0, 0) != 0) { - metaError("vgId:%d, failed to update table col cmpr:%s uid:%" PRId64, TD_VID(pMeta->pVnode), entry.name, - entry.uid); - } - if (entry.colCmpr.nCols != pSchema->nCols) { - terrno = TSDB_CODE_VND_INVALID_TABLE_ACTION; - goto _err; - } - break; - case TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES: - if (pColumn == NULL) { - terrno = TSDB_CODE_VND_COL_NOT_EXISTS; - goto _err; - } - if (!IS_VAR_DATA_TYPE(pColumn->type) || pColumn->bytes >= pAlterTbReq->colModBytes) { - terrno = TSDB_CODE_VND_INVALID_TABLE_ACTION; - goto _err; - } - if (rowLen + pAlterTbReq->colModBytes - pColumn->bytes > TSDB_MAX_BYTES_PER_ROW) { - terrno = TSDB_CODE_PAR_INVALID_ROW_LENGTH; - goto _err; - } - if (tqCheckColModifiable(pMeta->pVnode->pTq, uid, pColumn->colId) != 0) { - terrno = TSDB_CODE_VND_COL_SUBSCRIBED; - goto _err; - } - pSchema->version++; - pColumn->bytes = pAlterTbReq->colModBytes; - break; - case TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME: - if (pAlterTbReq->colNewName == NULL) { - terrno = TSDB_CODE_INVALID_MSG; - goto _err; - } - if (pColumn == NULL) { - terrno = TSDB_CODE_VND_COL_NOT_EXISTS; - goto _err; - } - if (tqCheckColModifiable(pMeta->pVnode->pTq, uid, pColumn->colId) != 0) { - terrno = TSDB_CODE_VND_COL_SUBSCRIBED; - goto _err; - } - pSchema->version++; - tstrncpy(pColumn->name, pAlterTbReq->colNewName, strlen(pAlterTbReq->colNewName) + 1); - break; - } - - if (!TSDB_CACHE_NO(pMeta->pVnode->config)) { - tsdbCacheInvalidateSchema(pMeta->pVnode->pTsdb, 0, entry.uid, pSchema->version); - } - - entry.version = version; - - // do actual write - metaWLock(pMeta); - - if (metaDeleteNcolIdx(pMeta, &oldEntry) < 0) { - metaError("vgId:%d, failed to delete ncol idx:%s uid:%" PRId64, TD_VID(pMeta->pVnode), entry.name, entry.uid); - } - - if (metaUpdateNcolIdx(pMeta, &entry) < 0) { - metaError("vgId:%d, failed to update ncol idx:%s uid:%" PRId64, TD_VID(pMeta->pVnode), entry.name, entry.uid); - } - - // save to table db - if (metaSaveToTbDb(pMeta, &entry) < 0) { - metaError("vgId:%d, failed to save to tb db:%s uid:%" PRId64, TD_VID(pMeta->pVnode), entry.name, entry.uid); - } - - if (metaUpdateUidIdx(pMeta, &entry) < 0) { - metaError("vgId:%d, failed to update uid idx:%s uid:%" PRId64, TD_VID(pMeta->pVnode), entry.name, entry.uid); - } - - if (metaSaveToSkmDb(pMeta, &entry) < 0) { - metaError("vgId:%d, failed to save to skm db:%s uid:%" PRId64, TD_VID(pMeta->pVnode), entry.name, entry.uid); - } - - if (metaUpdateChangeTime(pMeta, entry.uid, pAlterTbReq->ctimeMs) < 0) { - metaError("vgId:%d, failed to update change time:%s uid:%" PRId64, TD_VID(pMeta->pVnode), entry.name, entry.uid); - } - - metaULock(pMeta); - - if (metaUpdateMetaRsp(uid, pAlterTbReq->tbName, pSchema, pMetaRsp) < 0) { - metaError("vgId:%d, failed to update meta rsp:%s uid:%" PRId64, TD_VID(pMeta->pVnode), entry.name, entry.uid); - } - for (int32_t i = 0; i < entry.colCmpr.nCols; i++) { - SColCmpr *p = &entry.colCmpr.pColCmpr[i]; - pMetaRsp->pSchemaExt[i].colId = p->id; - pMetaRsp->pSchemaExt[i].compress = p->alg; - } - - if (entry.pBuf) taosMemoryFree(entry.pBuf); - if (pNewSchema) taosMemoryFree(pNewSchema); - if (freeColCmpr) taosMemoryFree(entry.colCmpr.pColCmpr); - - tdbTbcClose(pTbDbc); - tdbTbcClose(pUidIdxc); - tDecoderClear(&dc); - - return 0; - -_err: - if (entry.pBuf) taosMemoryFree(entry.pBuf); - tdbTbcClose(pTbDbc); - tdbTbcClose(pUidIdxc); - tDecoderClear(&dc); - - return terrno != 0 ? terrno : TSDB_CODE_FAILED; -} - -static int metaUpdateTableMultiTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTbReq) { - SMetaEntry ctbEntry = {0}; - SMetaEntry stbEntry = {0}; - void *pVal = NULL; - int nVal = 0; - int ret; - int c; - tb_uid_t uid; - int64_t oversion; - const void *pData = NULL; - int nData = 0; - SHashObj *pTagTable = NULL; - - // search name index - ret = tdbTbGet(pMeta->pNameIdx, pAlterTbReq->tbName, strlen(pAlterTbReq->tbName) + 1, &pVal, &nVal); - if (ret < 0) { - return terrno = TSDB_CODE_TDB_TABLE_NOT_EXIST; - } - - uid = *(tb_uid_t *)pVal; - tdbFree(pVal); - pVal = NULL; - - // search uid index - TBC *pUidIdxc = NULL; - - TAOS_CHECK_RETURN(tdbTbcOpen(pMeta->pUidIdx, &pUidIdxc, NULL)); - if (tdbTbcMoveTo(pUidIdxc, &uid, sizeof(uid), &c) < 0) { - metaTrace("meta/table: failed to move to uid index, uid:%" PRId64, uid); - } - if (c != 0) { - tdbTbcClose(pUidIdxc); - metaError("meta/table: invalide c: %" PRId32 " update tb tag val failed.", c); - return terrno = TSDB_CODE_TDB_TABLE_NOT_EXIST; - } - - if (tdbTbcGet(pUidIdxc, NULL, NULL, &pData, &nData) != 0) { - tdbTbcClose(pUidIdxc); - metaError("meta/table: failed to get uid index, uid:%" PRId64, uid); - return terrno = TSDB_CODE_TDB_TABLE_NOT_EXIST; - } - oversion = ((SUidIdxVal *)pData)[0].version; - - // search table.db - TBC *pTbDbc = NULL; - SDecoder dc1 = {0}; - SDecoder dc2 = {0}; - - /* get ctbEntry */ - TAOS_CHECK_RETURN(tdbTbcOpen(pMeta->pTbDb, &pTbDbc, NULL)); - if (tdbTbcMoveTo(pTbDbc, &((STbDbKey){.uid = uid, .version = oversion}), sizeof(STbDbKey), &c) != 0) { - metaError("meta/table: failed to move to tb db, uid:%" PRId64, uid); - } - if (c != 0) { - tdbTbcClose(pUidIdxc); - tdbTbcClose(pTbDbc); - metaError("meta/table: invalide c: %" PRId32 " update tb tag val failed.", c); - return terrno = TSDB_CODE_TDB_TABLE_NOT_EXIST; - } - - if (tdbTbcGet(pTbDbc, NULL, NULL, &pData, &nData) != 0) { - metaError("meta/table: failed to get tb db, uid:%" PRId64, uid); - tdbTbcClose(pUidIdxc); - tdbTbcClose(pTbDbc); - return terrno = TSDB_CODE_TDB_TABLE_NOT_EXIST; - } - - if ((ctbEntry.pBuf = taosMemoryMalloc(nData)) == NULL) { - tdbTbcClose(pUidIdxc); - tdbTbcClose(pTbDbc); - return terrno; - } - memcpy(ctbEntry.pBuf, pData, nData); - tDecoderInit(&dc1, ctbEntry.pBuf, nData); - ret = metaDecodeEntry(&dc1, &ctbEntry); - if (ret < 0) { - terrno = ret; - goto _err; - } - - /* get stbEntry*/ - if (tdbTbGet(pMeta->pUidIdx, &ctbEntry.ctbEntry.suid, sizeof(tb_uid_t), &pVal, &nVal) != 0) { - metaError("meta/table: failed to get uid index, uid:%" PRId64, ctbEntry.ctbEntry.suid); - } - if (!pVal) { - terrno = TSDB_CODE_INVALID_MSG; - goto _err; - } - - if (tdbTbGet(pMeta->pTbDb, &((STbDbKey){.uid = ctbEntry.ctbEntry.suid, .version = ((SUidIdxVal *)pVal)[0].version}), - sizeof(STbDbKey), (void **)&stbEntry.pBuf, &nVal) != 0) { - metaError("meta/table: failed to get tb db, uid:%" PRId64, ctbEntry.ctbEntry.suid); - } - tdbFree(pVal); - tDecoderInit(&dc2, stbEntry.pBuf, nVal); - ret = metaDecodeEntry(&dc2, &stbEntry); - if (ret < 0) { - terrno = ret; - goto _err; - } - - int32_t nTagVals = taosArrayGetSize(pAlterTbReq->pMultiTag); - pTagTable = taosHashInit(nTagVals, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); - if (pTagTable == NULL) { - ret = terrno; - goto _err; - } - - // remove duplicate tag name - for (int i = 0; i < nTagVals; i++) { - SMultiTagUpateVal *pTagVal = taosArrayGet(pAlterTbReq->pMultiTag, i); - ret = taosHashPut(pTagTable, pTagVal->tagName, strlen(pTagVal->tagName), pTagVal, sizeof(*pTagVal)); - if (ret != 0) { - goto _err; - } - } - - SSchemaWrapper *pTagSchema = &stbEntry.stbEntry.schemaTag; - SSchema *pColumn = NULL; - int32_t iCol = 0; - int32_t count = 0; - - for (;;) { - pColumn = NULL; - - if (iCol >= pTagSchema->nCols) break; - pColumn = &pTagSchema->pSchema[iCol]; - if (taosHashGet(pTagTable, pColumn->name, strlen(pColumn->name)) != NULL) { - count++; - } - iCol++; - } - if (count != taosHashGetSize(pTagTable)) { - terrno = TSDB_CODE_VND_COL_NOT_EXISTS; - goto _err; - } - - ctbEntry.version = version; - if (pTagSchema->nCols == 1 && pTagSchema->pSchema[0].type == TSDB_DATA_TYPE_JSON) { - terrno = TSDB_CODE_VND_COL_NOT_EXISTS; - goto _err; - } else { - const STag *pOldTag = (const STag *)ctbEntry.ctbEntry.pTags; - STag *pNewTag = NULL; - SArray *pTagArray = taosArrayInit(pTagSchema->nCols, sizeof(STagVal)); - if (!pTagArray) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - for (int32_t i = 0; i < pTagSchema->nCols; i++) { - SSchema *pCol = &pTagSchema->pSchema[i]; - SMultiTagUpateVal *pTagVal = taosHashGet(pTagTable, pCol->name, strlen(pCol->name)); - if (pTagVal == NULL) { - STagVal val = {.cid = pCol->colId}; - if (tTagGet(pOldTag, &val)) { - if (taosArrayPush(pTagArray, &val) == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - taosArrayDestroy(pTagArray); - goto _err; - } - } - } else { - STagVal val = {0}; - val.type = pCol->type; - val.cid = pCol->colId; - if (pTagVal->isNull) continue; - - if (IS_VAR_DATA_TYPE(pCol->type)) { - val.pData = pTagVal->pTagVal; - val.nData = pTagVal->nTagVal; - } else { - memcpy(&val.i64, pTagVal->pTagVal, pTagVal->nTagVal); - } - if (taosArrayPush(pTagArray, &val) == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - taosArrayDestroy(pTagArray); - goto _err; - } - } - } - if ((terrno = tTagNew(pTagArray, pTagSchema->version, false, &pNewTag)) < 0) { - taosArrayDestroy(pTagArray); - goto _err; - } - ctbEntry.ctbEntry.pTags = (uint8_t *)pNewTag; - taosArrayDestroy(pTagArray); - } - - metaWLock(pMeta); - - // save to table.db - if (metaSaveToTbDb(pMeta, &ctbEntry) < 0) { - metaError("meta/table: failed to save to tb db:%s uid:%" PRId64, ctbEntry.name, ctbEntry.uid); - } - - // save to uid.idx - if (metaUpdateUidIdx(pMeta, &ctbEntry) < 0) { - metaError("meta/table: failed to update uid idx:%s uid:%" PRId64, ctbEntry.name, ctbEntry.uid); - } - - if (metaUpdateTagIdx(pMeta, &ctbEntry) < 0) { - metaError("meta/table: failed to update tag idx:%s uid:%" PRId64, ctbEntry.name, ctbEntry.uid); - } - - SCtbIdxKey ctbIdxKey = {.suid = ctbEntry.ctbEntry.suid, .uid = uid}; - if (tdbTbUpsert(pMeta->pCtbIdx, &ctbIdxKey, sizeof(ctbIdxKey), ctbEntry.ctbEntry.pTags, - ((STag *)(ctbEntry.ctbEntry.pTags))->len, pMeta->txn) < 0) { - metaError("meta/table: failed to upsert ctb idx:%s uid:%" PRId64, ctbEntry.name, ctbEntry.uid); - } - - if (metaUidCacheClear(pMeta, ctbEntry.ctbEntry.suid) < 0) { - metaError("meta/table: failed to clear uid cache:%s uid:%" PRId64, ctbEntry.name, ctbEntry.uid); - } - - if (metaTbGroupCacheClear(pMeta, ctbEntry.ctbEntry.suid) < 0) { - metaError("meta/table: failed to clear group cache:%s uid:%" PRId64, ctbEntry.name, ctbEntry.uid); - } - - if (metaUpdateChangeTime(pMeta, ctbEntry.uid, pAlterTbReq->ctimeMs) < 0) { - metaError("meta/table: failed to update change time:%s uid:%" PRId64, ctbEntry.name, ctbEntry.uid); - } - - metaULock(pMeta); - - tDecoderClear(&dc1); - tDecoderClear(&dc2); - taosMemoryFree((void *)ctbEntry.ctbEntry.pTags); - if (ctbEntry.pBuf) taosMemoryFree(ctbEntry.pBuf); - if (stbEntry.pBuf) tdbFree(stbEntry.pBuf); - tdbTbcClose(pTbDbc); - tdbTbcClose(pUidIdxc); - taosHashCleanup(pTagTable); - return 0; - -_err: - tDecoderClear(&dc1); - tDecoderClear(&dc2); - if (ctbEntry.pBuf) taosMemoryFree(ctbEntry.pBuf); - if (stbEntry.pBuf) tdbFree(stbEntry.pBuf); - tdbTbcClose(pTbDbc); - tdbTbcClose(pUidIdxc); - taosHashCleanup(pTagTable); - return -1; -} -static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTbReq) { - SMetaEntry ctbEntry = {0}; - SMetaEntry stbEntry = {0}; - void *pVal = NULL; - int nVal = 0; - int ret; - int c; - tb_uid_t uid; - int64_t oversion; - const void *pData = NULL; - int nData = 0; - - if (pAlterTbReq->tagName == NULL) { - return terrno = TSDB_CODE_INVALID_MSG; - } - - // search name index - ret = tdbTbGet(pMeta->pNameIdx, pAlterTbReq->tbName, strlen(pAlterTbReq->tbName) + 1, &pVal, &nVal); - if (ret < 0) { - return terrno = TSDB_CODE_TDB_TABLE_NOT_EXIST; - } - - uid = *(tb_uid_t *)pVal; - tdbFree(pVal); - pVal = NULL; - - // search uid index - TBC *pUidIdxc = NULL; - - TAOS_CHECK_RETURN(tdbTbcOpen(pMeta->pUidIdx, &pUidIdxc, NULL)); - if (tdbTbcMoveTo(pUidIdxc, &uid, sizeof(uid), &c) < 0) { - metaTrace("meta/table: failed to move to uid index, uid:%" PRId64, uid); - } - if (c != 0) { - tdbTbcClose(pUidIdxc); - metaError("meta/table: invalide c: %" PRId32 " update tb tag val failed.", c); - return terrno = TSDB_CODE_TDB_TABLE_NOT_EXIST; - } - - if (tdbTbcGet(pUidIdxc, NULL, NULL, &pData, &nData) != 0) { - tdbTbcClose(pUidIdxc); - metaError("meta/table: failed to get uid index, uid:%" PRId64, uid); - return terrno = TSDB_CODE_TDB_TABLE_NOT_EXIST; - } - oversion = ((SUidIdxVal *)pData)[0].version; - - // search table.db - TBC *pTbDbc = NULL; - SDecoder dc1 = {0}; - SDecoder dc2 = {0}; - - /* get ctbEntry */ - TAOS_CHECK_RETURN(tdbTbcOpen(pMeta->pTbDb, &pTbDbc, NULL)); - if (tdbTbcMoveTo(pTbDbc, &((STbDbKey){.uid = uid, .version = oversion}), sizeof(STbDbKey), &c) != 0) { - metaError("meta/table: failed to move to tb db, uid:%" PRId64, uid); - } - if (c != 0) { - tdbTbcClose(pUidIdxc); - tdbTbcClose(pTbDbc); - metaError("meta/table: invalide c: %" PRId32 " update tb tag val failed.", c); - return terrno = TSDB_CODE_TDB_TABLE_NOT_EXIST; - } - - if (tdbTbcGet(pTbDbc, NULL, NULL, &pData, &nData) != 0) { - metaError("meta/table: failed to get tb db, uid:%" PRId64, uid); - tdbTbcClose(pUidIdxc); - tdbTbcClose(pTbDbc); - return terrno = TSDB_CODE_INVALID_MSG; - } - - if ((ctbEntry.pBuf = taosMemoryMalloc(nData)) == NULL) { - tdbTbcClose(pUidIdxc); - tdbTbcClose(pTbDbc); - return terrno; - } - memcpy(ctbEntry.pBuf, pData, nData); - tDecoderInit(&dc1, ctbEntry.pBuf, nData); - ret = metaDecodeEntry(&dc1, &ctbEntry); - if (ret < 0) { - terrno = ret; - goto _err; - } - - /* get stbEntry*/ - if (tdbTbGet(pMeta->pUidIdx, &ctbEntry.ctbEntry.suid, sizeof(tb_uid_t), &pVal, &nVal) != 0) { - metaError("meta/table: failed to get uid index, uid:%" PRId64, ctbEntry.ctbEntry.suid); - } - if (!pVal) { - terrno = TSDB_CODE_INVALID_MSG; - goto _err; - } - - if (tdbTbGet(pMeta->pTbDb, &((STbDbKey){.uid = ctbEntry.ctbEntry.suid, .version = ((SUidIdxVal *)pVal)[0].version}), - sizeof(STbDbKey), (void **)&stbEntry.pBuf, &nVal) != 0) { - metaError("meta/table: failed to get tb db, uid:%" PRId64, ctbEntry.ctbEntry.suid); - } - tdbFree(pVal); - tDecoderInit(&dc2, stbEntry.pBuf, nVal); - ret = metaDecodeEntry(&dc2, &stbEntry); - if (ret < 0) { - terrno = ret; - goto _err; - } - - SSchemaWrapper *pTagSchema = &stbEntry.stbEntry.schemaTag; - SSchema *pColumn = NULL; - int32_t iCol = 0; - - for (;;) { - pColumn = NULL; - - if (iCol >= pTagSchema->nCols) break; - pColumn = &pTagSchema->pSchema[iCol]; - - if (strcmp(pColumn->name, pAlterTbReq->tagName) == 0) break; - iCol++; - } - - if (pColumn == NULL) { - terrno = TSDB_CODE_VND_COL_NOT_EXISTS; - goto _err; - } - - ctbEntry.version = version; - if (pTagSchema->nCols == 1 && pTagSchema->pSchema[0].type == TSDB_DATA_TYPE_JSON) { - ctbEntry.ctbEntry.pTags = taosMemoryMalloc(pAlterTbReq->nTagVal); - if (ctbEntry.ctbEntry.pTags == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - memcpy((void *)ctbEntry.ctbEntry.pTags, pAlterTbReq->pTagVal, pAlterTbReq->nTagVal); - } else { - const STag *pOldTag = (const STag *)ctbEntry.ctbEntry.pTags; - STag *pNewTag = NULL; - SArray *pTagArray = taosArrayInit(pTagSchema->nCols, sizeof(STagVal)); - if (!pTagArray) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - for (int32_t i = 0; i < pTagSchema->nCols; i++) { - SSchema *pCol = &pTagSchema->pSchema[i]; - if (iCol == i) { - if (pAlterTbReq->isNull) { - continue; - } - STagVal val = {0}; - val.type = pCol->type; - val.cid = pCol->colId; - if (IS_VAR_DATA_TYPE(pCol->type)) { - val.pData = pAlterTbReq->pTagVal; - val.nData = pAlterTbReq->nTagVal; - } else { - memcpy(&val.i64, pAlterTbReq->pTagVal, pAlterTbReq->nTagVal); - } - if (taosArrayPush(pTagArray, &val) == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - taosArrayDestroy(pTagArray); - goto _err; - } - } else { - STagVal val = {.cid = pCol->colId}; - if (tTagGet(pOldTag, &val)) { - if (taosArrayPush(pTagArray, &val) == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - taosArrayDestroy(pTagArray); - goto _err; - } - } - } - } - if ((terrno = tTagNew(pTagArray, pTagSchema->version, false, &pNewTag)) < 0) { - taosArrayDestroy(pTagArray); - goto _err; - } - ctbEntry.ctbEntry.pTags = (uint8_t *)pNewTag; - taosArrayDestroy(pTagArray); - } - - metaWLock(pMeta); - - // save to table.db - if (metaSaveToTbDb(pMeta, &ctbEntry) < 0) { - metaError("meta/table: failed to save to tb db:%s uid:%" PRId64, ctbEntry.name, ctbEntry.uid); - } - - // save to uid.idx - if (metaUpdateUidIdx(pMeta, &ctbEntry) < 0) { - metaError("meta/table: failed to update uid idx:%s uid:%" PRId64, ctbEntry.name, ctbEntry.uid); - } - - if (metaUpdateTagIdx(pMeta, &ctbEntry) < 0) { - metaError("meta/table: failed to update tag idx:%s uid:%" PRId64, ctbEntry.name, ctbEntry.uid); - } - - SCtbIdxKey ctbIdxKey = {.suid = ctbEntry.ctbEntry.suid, .uid = uid}; - if (tdbTbUpsert(pMeta->pCtbIdx, &ctbIdxKey, sizeof(ctbIdxKey), ctbEntry.ctbEntry.pTags, - ((STag *)(ctbEntry.ctbEntry.pTags))->len, pMeta->txn) < 0) { - metaError("meta/table: failed to upsert ctb idx:%s uid:%" PRId64, ctbEntry.name, ctbEntry.uid); - } - - if (metaUidCacheClear(pMeta, ctbEntry.ctbEntry.suid) < 0) { - metaError("meta/table: failed to clear uid cache:%s uid:%" PRId64, ctbEntry.name, ctbEntry.uid); - } - - if (metaTbGroupCacheClear(pMeta, ctbEntry.ctbEntry.suid) < 0) { - metaError("meta/table: failed to clear group cache:%s uid:%" PRId64, ctbEntry.name, ctbEntry.uid); - } - - if (metaUpdateChangeTime(pMeta, ctbEntry.uid, pAlterTbReq->ctimeMs) < 0) { - metaError("meta/table: failed to update change time:%s uid:%" PRId64, ctbEntry.name, ctbEntry.uid); - } - - metaULock(pMeta); - - tDecoderClear(&dc1); - tDecoderClear(&dc2); - taosMemoryFree((void *)ctbEntry.ctbEntry.pTags); - if (ctbEntry.pBuf) taosMemoryFree(ctbEntry.pBuf); - if (stbEntry.pBuf) tdbFree(stbEntry.pBuf); - tdbTbcClose(pTbDbc); - tdbTbcClose(pUidIdxc); - return 0; - -_err: - tDecoderClear(&dc1); - tDecoderClear(&dc2); - if (ctbEntry.pBuf) taosMemoryFree(ctbEntry.pBuf); - if (stbEntry.pBuf) tdbFree(stbEntry.pBuf); - tdbTbcClose(pTbDbc); - tdbTbcClose(pUidIdxc); - return -1; -} - -static int metaUpdateTableOptions(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTbReq) { - void *pVal = NULL; - int nVal = 0; - const void *pData = NULL; - int nData = 0; - int ret = 0; - tb_uid_t uid; - int64_t oversion; - SMetaEntry entry = {0}; - int c = 0; - - // search name index - ret = tdbTbGet(pMeta->pNameIdx, pAlterTbReq->tbName, strlen(pAlterTbReq->tbName) + 1, &pVal, &nVal); - if (ret < 0) { - return terrno = TSDB_CODE_TDB_TABLE_NOT_EXIST; - } - - uid = *(tb_uid_t *)pVal; - tdbFree(pVal); - pVal = NULL; - - // search uid index - TBC *pUidIdxc = NULL; - - TAOS_CHECK_RETURN(tdbTbcOpen(pMeta->pUidIdx, &pUidIdxc, NULL)); - if (tdbTbcMoveTo(pUidIdxc, &uid, sizeof(uid), &c) < 0) { - metaError("meta/table: failed to move to uid index, uid:%" PRId64, uid); - } - if (c != 0) { - tdbTbcClose(pUidIdxc); - metaError("meta/table: invalide c: %" PRId32 " update tb options failed.", c); - return TSDB_CODE_FAILED; - } - - if (tdbTbcGet(pUidIdxc, NULL, NULL, &pData, &nData) < 0) { - metaError("meta/table: failed to get uid index, uid:%" PRId64, uid); - } - oversion = ((SUidIdxVal *)pData)[0].version; - - // search table.db - TBC *pTbDbc = NULL; - - TAOS_CHECK_RETURN(tdbTbcOpen(pMeta->pTbDb, &pTbDbc, NULL)); - if (tdbTbcMoveTo(pTbDbc, &((STbDbKey){.uid = uid, .version = oversion}), sizeof(STbDbKey), &c) < 0) { - metaError("meta/table: failed to move to tb db, uid:%" PRId64, uid); - } - if (c != 0) { - tdbTbcClose(pUidIdxc); - tdbTbcClose(pTbDbc); - metaError("meta/table: invalide c: %" PRId32 " update tb options failed.", c); - return TSDB_CODE_FAILED; - } - - if (tdbTbcGet(pTbDbc, NULL, NULL, &pData, &nData) < 0) { - metaError("meta/table: failed to get tb db, uid:%" PRId64, uid); - } - - // get table entry - SDecoder dc = {0}; - if ((entry.pBuf = taosMemoryMalloc(nData)) == NULL) { - tdbTbcClose(pUidIdxc); - tdbTbcClose(pTbDbc); - return terrno; - } - memcpy(entry.pBuf, pData, nData); - tDecoderInit(&dc, entry.pBuf, nData); - ret = metaDecodeEntry(&dc, &entry); - if (ret != 0) { - tDecoderClear(&dc); - tdbTbcClose(pUidIdxc); - tdbTbcClose(pTbDbc); - metaError("meta/table: invalide ret: %" PRId32 " alt tb options failed.", ret); - return TSDB_CODE_FAILED; - } - - entry.version = version; - metaWLock(pMeta); - // build SMetaEntry - if (entry.type == TSDB_CHILD_TABLE) { - if (pAlterTbReq->updateTTL) { - metaDeleteTtl(pMeta, &entry); - entry.ctbEntry.ttlDays = pAlterTbReq->newTTL; - metaUpdateTtl(pMeta, &entry); - } - if (pAlterTbReq->newCommentLen >= 0) { - entry.ctbEntry.commentLen = pAlterTbReq->newCommentLen; - entry.ctbEntry.comment = pAlterTbReq->newComment; - } - } else { - if (pAlterTbReq->updateTTL) { - metaDeleteTtl(pMeta, &entry); - entry.ntbEntry.ttlDays = pAlterTbReq->newTTL; - metaUpdateTtl(pMeta, &entry); - } - if (pAlterTbReq->newCommentLen >= 0) { - entry.ntbEntry.commentLen = pAlterTbReq->newCommentLen; - entry.ntbEntry.comment = pAlterTbReq->newComment; - } - } - - // save to table db - if (metaSaveToTbDb(pMeta, &entry) < 0) { - metaError("meta/table: failed to save to tb db:%s uid:%" PRId64, entry.name, entry.uid); - } - - if (metaUpdateUidIdx(pMeta, &entry) < 0) { - metaError("meta/table: failed to update uid idx:%s uid:%" PRId64, entry.name, entry.uid); - } - - if (metaUpdateChangeTime(pMeta, entry.uid, pAlterTbReq->ctimeMs) < 0) { - metaError("meta/table: failed to update change time:%s uid:%" PRId64, entry.name, entry.uid); - } - - metaULock(pMeta); - - tdbTbcClose(pTbDbc); - tdbTbcClose(pUidIdxc); - tDecoderClear(&dc); - if (entry.pBuf) taosMemoryFree(entry.pBuf); - return 0; -} - -typedef struct SMetaPair { - void *key; - int nkey; -} SMetaPair; - -static int metaDropTagIndex(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTbReq) { - SMetaEntry stbEntry = {0}; - void *pVal = NULL; - int nVal = 0; - int ret; - int c; - tb_uid_t suid; - int64_t oversion; - const void *pData = NULL; - int nData = 0; - SDecoder dc = {0}; - - if (pAlterTbReq->tagName == NULL) { - return terrno = TSDB_CODE_INVALID_MSG; - } - - // search name index - ret = tdbTbGet(pMeta->pNameIdx, pAlterTbReq->tbName, strlen(pAlterTbReq->tbName) + 1, &pVal, &nVal); - if (ret < 0) { - return terrno = TSDB_CODE_TDB_TABLE_NOT_EXIST; - } - suid = *(tb_uid_t *)pVal; - tdbFree(pVal); - pVal = NULL; - - if (tdbTbGet(pMeta->pUidIdx, &suid, sizeof(tb_uid_t), &pVal, &nVal) == -1) { - ret = -1; - goto _err; - } - - STbDbKey tbDbKey = {0}; - tbDbKey.uid = suid; - tbDbKey.version = ((SUidIdxVal *)pVal)[0].version; - ret = tdbTbGet(pMeta->pTbDb, &tbDbKey, sizeof(tbDbKey), &pVal, &nVal); - if (ret < 0) { - goto _err; - } - - tDecoderInit(&dc, pVal, nVal); - ret = metaDecodeEntry(&dc, &stbEntry); - if (ret < 0) { - tDecoderClear(&dc); - goto _err; - } - - // Get targe schema info - SSchemaWrapper *pTagSchema = &stbEntry.stbEntry.schemaTag; - if (pTagSchema->nCols == 1 && pTagSchema->pSchema[0].type == TSDB_DATA_TYPE_JSON) { - terrno = TSDB_CODE_VND_COL_ALREADY_EXISTS; - goto _err; - } - SSchema *pCol = NULL; - int32_t iCol = 0; - for (;;) { - pCol = NULL; - if (iCol >= pTagSchema->nCols) break; - pCol = &pTagSchema->pSchema[iCol]; - if (strcmp(pCol->name, pAlterTbReq->tagName) == 0) break; - iCol++; - } - if (iCol == 0) { - // cannot drop 1th tag index - terrno = -1; - goto _err; - } - if (pCol == NULL) { - terrno = TSDB_CODE_VND_COL_NOT_EXISTS; - goto _err; - } - - if (IS_IDX_ON(pCol)) { - terrno = TSDB_CODE_VND_COL_ALREADY_EXISTS; - goto _err; - } - - SArray *tagIdxList = taosArrayInit(512, sizeof(SMetaPair)); - if (tagIdxList == NULL) { - goto _err; - } - - TBC *pTagIdxc = NULL; - TAOS_CHECK_RETURN(tdbTbcOpen(pMeta->pTagIdx, &pTagIdxc, NULL)); - int rc = - tdbTbcMoveTo(pTagIdxc, &(STagIdxKey){.suid = suid, .cid = INT32_MIN, .type = pCol->type}, sizeof(STagIdxKey), &c); - for (;;) { - void *pKey, *pVal; - int nKey, nVal; - rc = tdbTbcNext(pTagIdxc, &pKey, &nKey, &pVal, &nVal); - STagIdxKey *pIdxKey = (STagIdxKey *)pKey; - if (pIdxKey->suid != suid || pIdxKey->cid != pCol->colId) { - tdbFree(pKey); - tdbFree(pVal); - continue; - } - - SMetaPair pair = {.key = pKey, nKey = nKey}; - if (taosArrayPush(tagIdxList, &pair) == NULL) { - goto _err; - } - } - tdbTbcClose(pTagIdxc); - - metaWLock(pMeta); - for (int i = 0; i < taosArrayGetSize(tagIdxList); i++) { - SMetaPair *pair = taosArrayGet(tagIdxList, i); - ret = tdbTbDelete(pMeta->pTagIdx, pair->key, pair->nkey, pMeta->txn); - if (ret < 0) { - metaError("meta/table: failed to delete tag idx:%s uid:%" PRId64, stbEntry.name, stbEntry.uid); - } - } - metaULock(pMeta); - - taosArrayDestroy(tagIdxList); - - // set pCol->flags; INDEX_ON - return 0; -_err: - return TSDB_CODE_FAILED; -} -static int32_t metaUpdateTableColCompress(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq) { - // impl later - SMetaEntry tbEntry = {0}; - void *pVal = NULL; - int nVal = 0; - int ret; - int c; - tb_uid_t suid; - int64_t oversion; - const void *pData = NULL; - int nData = 0; - SDecoder dc = {0}; - ret = tdbTbGet(pMeta->pNameIdx, pReq->tbName, strlen(pReq->tbName) + 1, &pVal, &nVal); - if (ret < 0) { - return terrno = TSDB_CODE_TDB_TABLE_NOT_EXIST; - } - suid = *(tb_uid_t *)pVal; - tdbFree(pVal); - pVal = NULL; - - if (tdbTbGet(pMeta->pUidIdx, &suid, sizeof(tb_uid_t), &pVal, &nVal) == -1) { - terrno = TSDB_CODE_INVALID_MSG; - ret = -1; - goto _err; - } - - STbDbKey tbDbKey = {0}; - tbDbKey.uid = suid; - tbDbKey.version = ((SUidIdxVal *)pVal)[0].version; - if (tdbTbGet(pMeta->pTbDb, &tbDbKey, sizeof(tbDbKey), &pVal, &nVal) < 0) { - terrno = TSDB_CODE_INVALID_MSG; - tdbFree(pVal); - goto _err; - } - - tDecoderInit(&dc, pVal, nVal); - ret = metaDecodeEntry(&dc, &tbEntry); - if (ret < 0) { - terrno = TSDB_CODE_INVALID_MSG; - tdbFree(pVal); - tDecoderClear(&dc); - goto _err; - } - if (tbEntry.type != TSDB_NORMAL_TABLE && tbEntry.type != TSDB_SUPER_TABLE) { - terrno = TSDB_CODE_INVALID_MSG; - tdbFree(pVal); - tDecoderClear(&dc); - goto _err; - } - int8_t updated = 0; - SColCmprWrapper *wp = &tbEntry.colCmpr; - for (int32_t i = 0; i < wp->nCols; i++) { - SColCmpr *p = &wp->pColCmpr[i]; - if (p->id == pReq->colId) { - uint32_t dst = 0; - updated = tUpdateCompress(p->alg, pReq->compress, TSDB_COLVAL_COMPRESS_DISABLED, TSDB_COLVAL_LEVEL_DISABLED, - TSDB_COLVAL_LEVEL_MEDIUM, &dst); - if (updated > 0) { - p->alg = dst; - } - } - } - if (updated == 0) { - tdbFree(pVal); - tDecoderClear(&dc); - terrno = TSDB_CODE_VND_COLUMN_COMPRESS_ALREADY_EXIST; - goto _err; - } else if (updated < 0) { - tdbFree(pVal); - tDecoderClear(&dc); - terrno = TSDB_CODE_TSC_COMPRESS_LEVEL_ERROR; - goto _err; - } - tbEntry.version = version; - - metaWLock(pMeta); - if (metaSaveToTbDb(pMeta, &tbEntry) < 0) { - metaError("meta/table: failed to save to tb db:%s uid:%" PRId64, tbEntry.name, tbEntry.uid); - } - - if (metaUpdateUidIdx(pMeta, &tbEntry) < 0) { - metaError("meta/table: failed to update uid idx:%s uid:%" PRId64, tbEntry.name, tbEntry.uid); - } - - if (metaUpdateChangeTime(pMeta, suid, pReq->ctimeMs) < 0) { - metaError("meta/table: failed to update change time:%s uid:%" PRId64, tbEntry.name, tbEntry.uid); - } - - metaULock(pMeta); - - tdbFree(pVal); - tDecoderClear(&dc); - - return 0; -_err: - return TSDB_CODE_FAILED; -} - int metaAlterTable(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq, STableMetaRsp *pMetaRsp) { pMeta->changed = true; switch (pReq->action) { @@ -2878,102 +709,6 @@ int metaAlterTable(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq, STableMeta } } -static int metaSaveToTbDb(SMeta *pMeta, const SMetaEntry *pME) { - STbDbKey tbDbKey; - void *pKey = NULL; - void *pVal = NULL; - int kLen = 0; - int vLen = 0; - SEncoder coder = {0}; - - // set key and value - tbDbKey.version = pME->version; - tbDbKey.uid = pME->uid; - - metaDebug("vgId:%d, start to save table version:%" PRId64 " uid:%" PRId64, TD_VID(pMeta->pVnode), pME->version, - pME->uid); - - pKey = &tbDbKey; - kLen = sizeof(tbDbKey); - - int32_t ret = 0; - tEncodeSize(metaEncodeEntry, pME, vLen, ret); - if (ret < 0) { - goto _err; - } - - pVal = taosMemoryMalloc(vLen); - if (pVal == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - - tEncoderInit(&coder, pVal, vLen); - - if (metaEncodeEntry(&coder, pME) < 0) { - goto _err; - } - - tEncoderClear(&coder); - - // write to table.db - if (tdbTbInsert(pMeta->pTbDb, pKey, kLen, pVal, vLen, pMeta->txn) < 0) { - goto _err; - } - - taosMemoryFree(pVal); - return 0; - -_err: - metaError("vgId:%d, failed to save table version:%" PRId64 "uid:%" PRId64 " %s", TD_VID(pMeta->pVnode), pME->version, - pME->uid, tstrerror(terrno)); - - taosMemoryFree(pVal); - return TSDB_CODE_FAILED; -} - -static int metaUpdateUidIdx(SMeta *pMeta, const SMetaEntry *pME) { - // upsert cache - SMetaInfo info; - metaGetEntryInfo(pME, &info); - int32_t ret = metaCacheUpsert(pMeta, &info); - if (ret < 0) { - metaError("vgId:%d, failed to upsert cache, uid: %" PRId64 " %s", TD_VID(pMeta->pVnode), pME->uid, tstrerror(ret)); - } - - SUidIdxVal uidIdxVal = {.suid = info.suid, .version = info.version, .skmVer = info.skmVer}; - - return tdbTbUpsert(pMeta->pUidIdx, &pME->uid, sizeof(tb_uid_t), &uidIdxVal, sizeof(uidIdxVal), pMeta->txn); -} - -static int metaUpdateSuidIdx(SMeta *pMeta, const SMetaEntry *pME) { - return tdbTbUpsert(pMeta->pSuidIdx, &pME->uid, sizeof(tb_uid_t), NULL, 0, pMeta->txn); -} - -static int metaUpdateNameIdx(SMeta *pMeta, const SMetaEntry *pME) { - return tdbTbUpsert(pMeta->pNameIdx, pME->name, strlen(pME->name) + 1, &pME->uid, sizeof(tb_uid_t), pMeta->txn); -} - -static void metaUpdateTtl(SMeta *pMeta, const SMetaEntry *pME) { - if (pME->type != TSDB_CHILD_TABLE && pME->type != TSDB_NORMAL_TABLE) return; - - STtlUpdTtlCtx ctx = {.uid = pME->uid, .pTxn = pMeta->txn}; - if (pME->type == TSDB_CHILD_TABLE) { - ctx.ttlDays = pME->ctbEntry.ttlDays; - ctx.changeTimeMs = pME->ctbEntry.btime; - } else { - ctx.ttlDays = pME->ntbEntry.ttlDays; - ctx.changeTimeMs = pME->ntbEntry.btime; - } - - int32_t ret = ttlMgrInsertTtl(pMeta->pTtlMgr, &ctx); - if (ret < 0) { - metaError("vgId:%d, failed to insert ttl, uid: %" PRId64 " %s", TD_VID(pMeta->pVnode), pME->uid, tstrerror(ret)); - } - - return; -} - static int metaUpdateChangeTime(SMeta *pMeta, tb_uid_t uid, int64_t changeTimeMs) { if (!tsTtlChangeOnWrite) return 0; @@ -2996,13 +731,6 @@ int metaUpdateChangeTimeWithLock(SMeta *pMeta, tb_uid_t uid, int64_t changeTimeM return ret; } -static int metaUpdateCtbIdx(SMeta *pMeta, const SMetaEntry *pME) { - SCtbIdxKey ctbIdxKey = {.suid = pME->ctbEntry.suid, .uid = pME->uid}; - - return tdbTbUpsert(pMeta->pCtbIdx, &ctbIdxKey, sizeof(ctbIdxKey), pME->ctbEntry.pTags, - ((STag *)(pME->ctbEntry.pTags))->len, pMeta->txn); -} - int metaCreateTagIdxKey(tb_uid_t suid, int32_t cid, const void *pTagData, int32_t nTagData, int8_t type, tb_uid_t uid, STagIdxKey **ppTagIdxKey, int32_t *nTagIdxKey) { if (IS_VAR_DATA_TYPE(type)) { @@ -3038,212 +766,6 @@ void metaDestroyTagIdxKey(STagIdxKey *pTagIdxKey) { if (pTagIdxKey) taosMemoryFree(pTagIdxKey); } -static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) { - void *pData = NULL; - int nData = 0; - STbDbKey tbDbKey = {0}; - SMetaEntry stbEntry = {0}; - STagIdxKey *pTagIdxKey = NULL; - int32_t nTagIdxKey; - const SSchema *pTagColumn; - const void *pTagData = NULL; - int32_t nTagData = 0; - SDecoder dc = {0}; - int32_t ret = 0; - // get super table - if (tdbTbGet(pMeta->pUidIdx, &pCtbEntry->ctbEntry.suid, sizeof(tb_uid_t), &pData, &nData) != 0) { - metaError("vgId:%d, failed to get stable suid for update. version:%" PRId64, TD_VID(pMeta->pVnode), - pCtbEntry->version); - ret = TSDB_CODE_TDB_INVALID_TABLE_ID; - goto end; - } - tbDbKey.uid = pCtbEntry->ctbEntry.suid; - tbDbKey.version = ((SUidIdxVal *)pData)[0].version; - ret = tdbTbGet(pMeta->pTbDb, &tbDbKey, sizeof(tbDbKey), &pData, &nData); - if (ret < 0) { - metaError("vgId:%d, failed to get stable for update. version:%" PRId64, TD_VID(pMeta->pVnode), pCtbEntry->version); - goto end; - } - - tDecoderInit(&dc, pData, nData); - ret = metaDecodeEntry(&dc, &stbEntry); - if (ret < 0) { - goto end; - } - - if (stbEntry.stbEntry.schemaTag.pSchema == NULL) { - ret = TSDB_CODE_INVALID_PARA; - goto end; - } - - SSchemaWrapper *pTagSchema = &stbEntry.stbEntry.schemaTag; - if (pTagSchema->nCols == 1 && pTagSchema->pSchema[0].type == TSDB_DATA_TYPE_JSON) { - pTagColumn = &stbEntry.stbEntry.schemaTag.pSchema[0]; - STagVal tagVal = {.cid = pTagColumn->colId}; - - pTagData = pCtbEntry->ctbEntry.pTags; - nTagData = ((const STag *)pCtbEntry->ctbEntry.pTags)->len; - ret = metaSaveJsonVarToIdx(pMeta, pCtbEntry, pTagColumn); - goto end; - } else { - for (int i = 0; i < pTagSchema->nCols; i++) { - pTagData = NULL; - nTagData = 0; - pTagColumn = &pTagSchema->pSchema[i]; - if (!IS_IDX_ON(pTagColumn)) continue; - - STagVal tagVal = {.cid = pTagColumn->colId}; - if (tTagGet((const STag *)pCtbEntry->ctbEntry.pTags, &tagVal)) { - if (IS_VAR_DATA_TYPE(pTagColumn->type)) { - pTagData = tagVal.pData; - nTagData = (int32_t)tagVal.nData; - } else { - pTagData = &(tagVal.i64); - nTagData = tDataTypes[pTagColumn->type].bytes; - } - } else { - if (!IS_VAR_DATA_TYPE(pTagColumn->type)) { - nTagData = tDataTypes[pTagColumn->type].bytes; - } - } - if (metaCreateTagIdxKey(pCtbEntry->ctbEntry.suid, pTagColumn->colId, pTagData, nTagData, pTagColumn->type, - pCtbEntry->uid, &pTagIdxKey, &nTagIdxKey) < 0) { - ret = -1; - goto end; - } - if (tdbTbUpsert(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, NULL, 0, pMeta->txn) < 0) { - metaError("vgId:%d, failed to update tag index. version:%" PRId64, TD_VID(pMeta->pVnode), pCtbEntry->version); - } - metaDestroyTagIdxKey(pTagIdxKey); - pTagIdxKey = NULL; - } - } -end: - tDecoderClear(&dc); - tdbFree(pData); - return ret; -} - -static int metaSaveToSkmDb(SMeta *pMeta, const SMetaEntry *pME) { - SEncoder coder = {0}; - void *pVal = NULL; - int vLen = 0; - int rcode = 0; - SSkmDbKey skmDbKey = {0}; - const SSchemaWrapper *pSW; - - if (pME->type == TSDB_SUPER_TABLE) { - pSW = &pME->stbEntry.schemaRow; - } else if (pME->type == TSDB_NORMAL_TABLE) { - pSW = &pME->ntbEntry.schemaRow; - } else { - metaError("meta/table: invalide table type: %" PRId8 " save skm db failed.", pME->type); - return TSDB_CODE_FAILED; - } - - skmDbKey.uid = pME->uid; - skmDbKey.sver = pSW->version; - - // if receive tmq meta message is: create stable1 then delete stable1 then create stable1 with multi vgroups - if (tdbTbGet(pMeta->pSkmDb, &skmDbKey, sizeof(skmDbKey), NULL, NULL) == 0) { - return rcode; - } - - // encode schema - int32_t ret = 0; - tEncodeSize(tEncodeSSchemaWrapper, pSW, vLen, ret); - if (ret < 0) return -1; - pVal = taosMemoryMalloc(vLen); - if (pVal == NULL) { - rcode = -1; - terrno = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; - } - - tEncoderInit(&coder, pVal, vLen); - ret = tEncodeSSchemaWrapper(&coder, pSW); - if (ret < 0) { - rcode = -1; - goto _exit; - } - - if (tdbTbInsert(pMeta->pSkmDb, &skmDbKey, sizeof(skmDbKey), pVal, vLen, pMeta->txn) < 0) { - rcode = -1; - goto _exit; - } - - metaDebug("vgId:%d, set schema:(%" PRId64 ") sver:%d since %s", TD_VID(pMeta->pVnode), pME->uid, pSW->version, - tstrerror(terrno)); - -_exit: - taosMemoryFree(pVal); - tEncoderClear(&coder); - return rcode; -} - -int metaHandleEntry(SMeta *pMeta, const SMetaEntry *pME) { - int32_t code = 0; - int32_t line = 0; - metaWLock(pMeta); - - // save to table.db - code = metaSaveToTbDb(pMeta, pME); - VND_CHECK_CODE(code, line, _err); - - // update uid.idx - code = metaUpdateUidIdx(pMeta, pME); - VND_CHECK_CODE(code, line, _err); - - // update name.idx - code = metaUpdateNameIdx(pMeta, pME); - VND_CHECK_CODE(code, line, _err); - - if (pME->type == TSDB_CHILD_TABLE) { - // update ctb.idx - code = metaUpdateCtbIdx(pMeta, pME); - VND_CHECK_CODE(code, line, _err); - - // update tag.idx - code = metaUpdateTagIdx(pMeta, pME); - VND_CHECK_CODE(code, line, _err); - } else { - // update schema.db - code = metaSaveToSkmDb(pMeta, pME); - VND_CHECK_CODE(code, line, _err); - - if (pME->type == TSDB_SUPER_TABLE) { - code = metaUpdateSuidIdx(pMeta, pME); - VND_CHECK_CODE(code, line, _err); - } - } - - code = metaUpdateBtimeIdx(pMeta, pME); - VND_CHECK_CODE(code, line, _err); - - if (pME->type == TSDB_NORMAL_TABLE) { - code = metaUpdateNcolIdx(pMeta, pME); - VND_CHECK_CODE(code, line, _err); - } - - if (pME->type != TSDB_SUPER_TABLE) { - metaUpdateTtl(pMeta, pME); - } - - if (pME->type == TSDB_SUPER_TABLE || pME->type == TSDB_NORMAL_TABLE) { - } - - metaULock(pMeta); - metaDebug("vgId:%d, handle meta entry, ver:%" PRId64 ", uid:%" PRId64 ", name:%s", TD_VID(pMeta->pVnode), - pME->version, pME->uid, pME->name); - return 0; - -_err: - metaULock(pMeta); - metaError("vgId:%d, failed to handle meta entry since %s at line:%d, ver:%" PRId64 ", uid:%" PRId64 ", name:%s", - TD_VID(pMeta->pVnode), terrstr(), line, pME->version, pME->uid, pME->name); - return TSDB_CODE_FAILED; -} - static void colCompressDebug(SHashObj *pColCmprObj) { void *p = taosHashIterate(pColCmprObj, NULL); while (p) { @@ -3261,6 +783,7 @@ static void colCompressDebug(SHashObj *pColCmprObj) { } return; } + int32_t metaGetColCmpr(SMeta *pMeta, tb_uid_t uid, SHashObj **ppColCmprObj) { int rc = 0; diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index e82209e03f..f8b75eb15c 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -607,9 +607,9 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t ver, SRpcMsg } vDebug("vgId:%d, start to process write request %s, index:%" PRId64 ", applied:%" PRId64 ", state.applyTerm:%" PRId64 - ", conn.applyTerm:%" PRId64, + ", conn.applyTerm:%" PRId64 ", contLen:%d", TD_VID(pVnode), TMSG_INFO(pMsg->msgType), ver, pVnode->state.applied, pVnode->state.applyTerm, - pMsg->info.conn.applyTerm); + pMsg->info.conn.applyTerm, pMsg->contLen); if (!(pVnode->state.applyTerm <= pMsg->info.conn.applyTerm)) { return terrno = TSDB_CODE_INTERNAL_ERROR; @@ -843,7 +843,7 @@ int32_t vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) { SReadHandle handle = {.vnode = pVnode, .pMsgCb = &pVnode->msgCb, .pWorkerCb = pInfo->workerCb}; initStorageAPI(&handle.api); int32_t code = TSDB_CODE_SUCCESS; - bool redirected = false; + bool redirected = false; switch (pMsg->msgType) { case TDMT_SCH_QUERY: @@ -2145,7 +2145,7 @@ static int32_t vnodeConsolidateAlterHashRange(SVnode *pVnode, int64_t ver) { pVnode->config.hashBegin, pVnode->config.hashEnd, ver); // TODO: trim meta of tables from TDB per hash range [pVnode->config.hashBegin, pVnode->config.hashEnd] - code = metaTrimTables(pVnode->pMeta); + code = metaTrimTables(pVnode->pMeta, ver); return code; } diff --git a/source/libs/function/CMakeLists.txt b/source/libs/function/CMakeLists.txt index 6891653981..fad0a749d5 100644 --- a/source/libs/function/CMakeLists.txt +++ b/source/libs/function/CMakeLists.txt @@ -142,6 +142,50 @@ target_link_libraries( udf2_dup PUBLIC os ${LINK_JEMALLOC} ) +set(TARGET_NAMES + change_udf_normal + change_udf_no_init + change_udf_no_process + change_udf_no_destroy + change_udf_init_failed + change_udf_process_failed + change_udf_destory_failed +) + +set(COMPILE_DEFINITIONS + CHANGE_UDF_NORMAL + CHANGE_UDF_NO_INIT + CHANGE_UDF_NO_PROCESS + CHANGE_UDF_NO_DESTROY + CHANGE_UDF_INIT_FAILED + CHANGE_UDF_PROCESS_FAILED + CHANGE_UDF_DESTORY_FAILED +) + +foreach(index RANGE 0 6) + list(GET TARGET_NAMES ${index} target_name) + list(GET COMPILE_DEFINITIONS ${index} compile_def) + + add_library(${target_name} STATIC MODULE test/change_udf.c) + target_include_directories( + ${target_name} + PUBLIC + "${TD_SOURCE_DIR}/include/libs/function" + "${TD_SOURCE_DIR}/include/util" + "${TD_SOURCE_DIR}/include/common" + "${TD_SOURCE_DIR}/include/client" + "${TD_SOURCE_DIR}/include/os" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" + ) + target_compile_definitions(${target_name} PRIVATE ${compile_def}) + IF(TD_LINUX_64 AND JEMALLOC_ENABLED) + ADD_DEPENDENCIES(${target_name} jemalloc) + ENDIF() + target_link_libraries( + ${target_name} PUBLIC os ${LINK_JEMALLOC} + ) +endforeach() + # SET(EXECUTABLE_OUTPUT_PATH ${CMAKE_BINARY_DIR}/build/bin) add_executable(udfd src/udfd.c) diff --git a/source/libs/function/test/change_udf.c b/source/libs/function/test/change_udf.c new file mode 100644 index 0000000000..f623a3194d --- /dev/null +++ b/source/libs/function/test/change_udf.c @@ -0,0 +1,172 @@ +#include +#include +#include +#ifdef LINUX +#include +#endif +#ifdef WINDOWS +#include +#endif +#include "taosudf.h" + +// rename function name +#ifdef CHANGE_UDF_NORMAL +#define UDFNAME change_udf_normal +#define UDFNAMEINIT change_udf_normal_init +#define UDFNAMEDESTROY change_udf_normal_destroy +#elif defined(CHANGE_UDF_NO_INIT) +#define UDFNAME change_udf_no_init +#define UDFNAMEINIT change_udf_no_init_init +#define UDFNAMEDESTROY change_udf_no_init_destroy +#elif defined(CHANGE_UDF_NO_PROCESS) +#define UDFNAME change_udf_no_process +#define UDFNAMEINIT change_udf_no_process_init +#define UDFNAMEDESTROY change_udf_no_process_destroy +#elif defined(CHANGE_UDF_NO_DESTROY) +#define UDFNAME change_udf_no_destroy +#define UDFNAMEINIT change_udf_no_destroy_init +#define UDFNAMEDESTROY change_udf_no_destroy_destroy +#elif defined(CHANGE_UDF_INIT_FAILED) +#define UDFNAME change_udf_init_failed +#define UDFNAMEINIT change_udf_init_failed_init +#define UDFNAMEDESTROY change_udf_init_failed_destroy +#elif defined(CHANGE_UDF_PROCESS_FAILED) +#define UDFNAME change_udf_process_failed +#define UDFNAMEINIT change_udf_process_failed_init +#define UDFNAMEDESTROY change_udf_process_failed_destroy +#elif defined(CHANGE_UDF_DESTORY_FAILED) +#define UDFNAME change_udf_destory_failed +#define UDFNAMEINIT change_udf_destory_failed_init +#define UDFNAMEDESTROY change_udf_destory_failed_destroy +#else +#define UDFNAME change_udf_normal +#define UDFNAMEINIT change_udf_normal_init +#define UDFNAMEDESTROY change_udf_normal_destroy +#endif + + +#ifdef CHANGE_UDF_NO_INIT +#else +DLL_EXPORT int32_t UDFNAMEINIT() { + #ifdef CHANGE_UDF_INIT_FAILED + return -1; + #else + return 0; + #endif // ifdef CHANGE_UDF_INIT_FAILED +} +#endif // ifdef CHANGE_UDF_NO_INIT + +#ifdef CHANGE_UDF_NO_DESTROY +#else +DLL_EXPORT int32_t UDFNAMEDESTROY() { + #ifdef CHANGE_UDF_DESTORY_FAILED + return -1; + #else + return 0; + #endif // ifdef CHANGE_UDF_DESTORY_FAILED + } +#endif // ifdef CHANGE_UDF_NO_DESTROY + +#ifdef CHANGE_UDF_NO_PROCESS +#else +DLL_EXPORT int32_t UDFNAME(SUdfDataBlock *block, SUdfColumn *resultCol) { + #ifdef CHANGE_UDF_PROCESS_FAILED + return -1; + #else + int32_t code = 0; + SUdfColumnData *resultData = &resultCol->colData; + for (int32_t i = 0; i < block->numOfRows; ++i) { + int j = 0; + for (; j < block->numOfCols; ++j) { + if (udfColDataIsNull(block->udfCols[j], i)) { + code = udfColDataSetNull(resultCol, i); + if (code != 0) { + return code; + } + break; + } + } + if (j == block->numOfCols) { + int32_t luckyNum = 1; + code = udfColDataSet(resultCol, i, (char *)&luckyNum, false); + if (code != 0) { + return code; + } + } + } + // to simulate actual processing delay by udf +#ifdef LINUX + usleep(1 * 1000); // usleep takes sleep time in us (1 millionth of a second) +#endif // ifdef LINUX +#ifdef WINDOWS + Sleep(1); +#endif // ifdef WINDOWS + resultData->numOfRows = block->numOfRows; + return 0; + #endif // ifdef CHANGE_UDF_PROCESS_FAILED +} +#endif // ifdef CHANGE_UDF_NO_PROCESS + + + + +/********************************************************************************************************************/ +// udf revert functions +/********************************************************************************************************************/ +DLL_EXPORT int32_t udf_reverse_init() { return 0; } + +DLL_EXPORT int32_t udf_reverse_destroy() { return 0; } + +static void reverse_data(char* data, size_t len) { + size_t i, j; + char temp; + for (i = 0, j = len - 1; i < j; i++, j--) { + temp = data[i]; + data[i] = data[j]; + data[j] = temp; + } +} + +DLL_EXPORT int32_t udf_reverse(SUdfDataBlock *block, SUdfColumn *resultCol) { + int32_t code = 0; + SUdfColumnData *resultData = &resultCol->colData; + for (int32_t i = 0; i < block->numOfRows; ++i) { + int j = 0; + for (; j < block->numOfCols; ++j) { + if (udfColDataIsNull(block->udfCols[j], i)) { + code = udfColDataSetNull(resultCol, i); + if (code != 0) { + return code; + } + break; + } else { + int32_t oldLen = udfColDataGetDataLen(block->udfCols[j], i); + char *pOldData = udfColDataGetData(block->udfCols[j], i); + + + char *buff = malloc(sizeof(VarDataLenT) + oldLen); + if (buff == NULL) { + return -1; + } + ((VarDataLenT *)buff)[0] = (VarDataLenT)oldLen; + memcpy(buff, pOldData, oldLen + sizeof(VarDataLenT)); + reverse_data(buff + sizeof(VarDataLenT), oldLen); + code = udfColDataSet(resultCol, i, buff, false); + if (code != 0) { + free(buff); + return code; + } + } + } + } + // to simulate actual processing delay by udf +#ifdef LINUX + usleep(1 * 1000); // usleep takes sleep time in us (1 millionth of a second) +#endif +#ifdef WINDOWS + Sleep(1); +#endif + resultData->numOfRows = block->numOfRows; + return 0; +} + diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index 52794af4dd..547cdb6cdf 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -603,11 +603,11 @@ int32_t checkItemDyn(SConfigItem *pItem, bool isServer) { return TSDB_CODE_SUCCESS; } if (isServer) { - if (pItem->dynScope == CFG_DYN_ENT_CLIENT || pItem->dynScope == CFG_DYN_ENT_CLIENT_LAZY) { + if (pItem->dynScope == CFG_DYN_CLIENT || pItem->dynScope == CFG_DYN_CLIENT_LAZY) { return TSDB_CODE_INVALID_CFG; } } else { - if (pItem->dynScope == CFG_DYN_ENT_SERVER || pItem->dynScope == CFG_DYN_ENT_SERVER_LAZY) { + if (pItem->dynScope == CFG_DYN_SERVER || pItem->dynScope == CFG_DYN_SERVER_LAZY) { return TSDB_CODE_INVALID_CFG; } } diff --git a/tests/army/cluster/arbitrator.py b/tests/army/cluster/arbitrator.py new file mode 100644 index 0000000000..9fd8e7b1f3 --- /dev/null +++ b/tests/army/cluster/arbitrator.py @@ -0,0 +1,81 @@ +import taos +import sys +import os +import subprocess +import glob +import shutil +import time + +from frame.log import * +from frame.cases import * +from frame.sql import * +from frame.srvCtl import * +from frame.caseBase import * +from frame import * +from frame.autogen import * +from frame import epath +# from frame.server.dnodes import * +# from frame.server.cluster import * + + +class TDTestCase(TBase): + + def init(self, conn, logSql, replicaVar=1): + updatecfgDict = {'dDebugFlag':131} + super(TDTestCase, self).init(conn, logSql, replicaVar=1, checkColName="c1") + + self.valgrind = 0 + self.db = "test" + self.stb = "meters" + self.childtable_count = 10 + tdSql.init(conn.cursor(), logSql) + + def run(self): + tdSql.execute('CREATE DATABASE db vgroups 1 replica 2;') + + time.sleep(1) + + count = 0 + + while count < 100: + tdSql.query("show arbgroups;") + + if tdSql.getData(0, 4) == 1: + break + + tdLog.info("wait 1 seconds for is sync") + time.sleep(1) + + count += 1 + + + tdSql.query("show db.vgroups;") + + if(tdSql.getData(0, 4) == "follower") and (tdSql.getData(0, 6) == "leader"): + tdLog.info("stop dnode2") + sc.dnodeStop(2) + + if(tdSql.getData(0, 6) == "follower") and (tdSql.getData(0, 4) == "leader"): + tdLog.info("stop dnode 3") + sc.dnodeStop(3) + + + count = 0 + while count < 100: + tdSql.query("show db.vgroups;") + + if(tdSql.getData(0, 4) == "assigned ") or (tdSql.getData(0, 6) == "assigned "): + break + + tdLog.info("wait 1 seconds for set assigned") + time.sleep(1) + + count += 1 + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index c7d777f59a..1d2c65cf2a 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -11,6 +11,7 @@ # ,,y,army,./pytest.sh python3 ./test.py -f multi-level/mlevel_basic.py -N 3 -L 3 -D 2 ,,y,army,./pytest.sh python3 ./test.py -f db-encrypt/basic.py -N 3 -M 3 +,,y,army,./pytest.sh python3 ./test.py -f cluster/arbitrator.py -N 3 ,,n,army,python3 ./test.py -f storage/s3/s3Basic.py -N 3 ,,y,army,./pytest.sh python3 ./test.py -f cluster/snapshot.py -N 3 -L 3 -D 2 ,,y,army,./pytest.sh python3 ./test.py -f query/function/test_func_elapsed.py @@ -365,6 +366,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/telemetry.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/backquote_check.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/taosdMonitor.py +,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/taosdNewMonitor.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/taosd_audit.py ,,n,system-test,python3 ./test.py -f 0-others/taosdlog.py ,,n,system-test,python3 ./test.py -f 0-others/taosdShell.py -N 5 -M 3 -Q 3 @@ -407,7 +409,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/persisit_config.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/qmemCtrl.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/compact_vgroups.py - +,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/compact.py -N 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/composite_primary_key_create.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/composite_primary_key_insert.py diff --git a/tests/pytest/util/sql.py b/tests/pytest/util/sql.py index 46b7e1f795..bb7b8411f9 100644 --- a/tests/pytest/util/sql.py +++ b/tests/pytest/util/sql.py @@ -155,6 +155,9 @@ class TDSql: try: self.cursor.execute(sql) + self.queryResult = self.cursor.fetchall() + self.queryRows = len(self.queryResult) + self.queryCols = len(self.cursor.description) except BaseException as e: tdLog.info("err:%s" % (e)) expectErrNotOccured = False @@ -165,10 +168,6 @@ class TDSql: if expectErrNotOccured: tdLog.exit("%s(%d) failed: sql:%s, expect error not occured" % (caller.filename, caller.lineno, sql)) else: - self.queryRows = 0 - self.queryCols = 0 - self.queryResult = None - if fullMatched: if expectedErrno != None: expectedErrno_rest = expectedErrno & 0x0000ffff diff --git a/tests/pytest/util/tserror.py b/tests/pytest/util/tserror.py new file mode 100644 index 0000000000..35d74153c3 --- /dev/null +++ b/tests/pytest/util/tserror.py @@ -0,0 +1,13 @@ +import ctypes + +TAOS_SYSTEM_ERROR = ctypes.c_int32(0x80ff0000).value +TAOS_DEF_ERROR_CODE = ctypes.c_int32(0x80000000).value + + +TSDB_CODE_MND_FUNC_NOT_EXIST = (TAOS_DEF_ERROR_CODE | 0x0374) + + +TSDB_CODE_UDF_FUNC_EXEC_FAILURE = (TAOS_DEF_ERROR_CODE | 0x290A) + + +TSDB_CODE_TSC_INTERNAL_ERROR = (TAOS_DEF_ERROR_CODE | 0x02FF) diff --git a/tests/system-test/0-others/compact.py b/tests/system-test/0-others/compact.py new file mode 100644 index 0000000000..eb2938e399 --- /dev/null +++ b/tests/system-test/0-others/compact.py @@ -0,0 +1,85 @@ +from util.log import * +from util.cases import * +from util.dnodes import * +from util.sql import * + +import socket +import taos + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + tdLog.debug(f"start to excute {__file__}") + self.replicaVar = int(replicaVar) + + def run(self): + + tdSql.query("CREATE DATABASE power KEEP 365 DURATION 10 BUFFER 16 WAL_LEVEL 1 vgroups 1 replica 1;") + + tdSql.query("CREATE DATABASE power1 KEEP 365 DURATION 10 BUFFER 16 WAL_LEVEL 1 vgroups 1 replica 1;") + + #first + tdSql.query("compact database power;") + + tdLog.info("compact id:%d"%tdSql.queryResult[0][1]) + + tdSql.query("show compact %d;"%tdSql.queryResult[0][1]) + + tdLog.info("detail:%d"%tdSql.queryRows) + + #second + tdSql.query("compact database power1;") + + tdLog.info("compact id:%d"%tdSql.queryResult[0][1]) + + tdSql.query("show compact %d;"%tdSql.queryResult[0][1]) + + tdLog.info("detail:%d"%tdSql.queryRows) + + + #kill + tdSql.query("show compacts;") + number1 = tdSql.queryResult[0][0] + number2 = tdSql.queryResult[1][0] + + #first + tdLog.info("kill compact %d;"%number1) + tdSql.query("kill compact %d;"%number1) + + #second + tdLog.info("kill compact %d;"%number2) + tdSql.query("kill compact %d;"%number2) + + + #show + count = 0 + tdLog.info("query progress") + while count < 50: + tdSql.query("show compact %d;"%number1) + + row1 = tdSql.queryRows + + tdSql.query("show compact %d;"%number2) + + row2 = tdSql.queryRows + + tdLog.info("compact%d:detail count:%d"%(number1, row1)) + tdLog.info("compact%d:detail count:%d"%(number2, row2)) + + if row1 == 0 and row2 == 0 : + break + + time.sleep(1) + + count +=1 + #tdLog.info("loop%d"%count) + + if row1 != 0 or row2 != 0: + tdLog.exit("compact failed") + + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/system-test/0-others/taosdNewMonitor.py b/tests/system-test/0-others/taosdNewMonitor.py new file mode 100644 index 0000000000..a3ced155c5 --- /dev/null +++ b/tests/system-test/0-others/taosdNewMonitor.py @@ -0,0 +1,243 @@ +import taos +import sys +import time +import socket +# import pexpect +import os +import http.server +import gzip +import threading +import json +import pickle + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * + +telemetryPort = '6043' +serverPort = '7080' +hostname = socket.gethostname() + +class RequestHandlerImpl(http.server.BaseHTTPRequestHandler): + hostPort = hostname + ":" + serverPort + + def telemetryInfoCheck(self, infoDict=''): + if len(infoDict) == 0: + return + + if "ts" not in infoDict[0] or len(infoDict[0]["ts"]) == 0: + tdLog.exit("ts is null!") + + if "protocol" not in infoDict[0] or infoDict[0]["protocol"] != 2: + tdLog.exit("protocol is null!") + + if "tables" not in infoDict[0]: + tdLog.exit("tables is null!") + + if infoDict[0]["tables"][0]["name"] != "taosd_dnodes_info": + tdLog.exit("taosd_dnodes_info is null!") + + # dnode_info ==================================== + + dnode_infos = ['disk_engine', 'system_net_in', 'vnodes_num', 'system_net_out', 'uptime', 'has_mnode', 'io_read_disk', 'error_log_count', + 'io_read', 'cpu_cores', 'has_qnode', 'has_snode', 'disk_total', 'mem_engine', 'info_log_count', 'cpu_engine', 'io_write_disk', + 'debug_log_count', 'disk_used', 'mem_total', 'io_write', 'masters', 'cpu_system', + 'trace_log_count', 'mem_free'] + index = 0 + for elem in dnode_infos: + tdLog.debug(f"elem: {index},{elem}") + if infoDict[0]["tables"][0]["metric_groups"][0]["metrics"][index]["name"] != elem: + tdLog.exit(f"{elem} is null!") + index += 1 + + if infoDict[0]["tables"][1]["name"] != "taosd_dnodes_log_dirs": + tdLog.exit("taosd_dnodes_log_dirs is null!") + + # logdir + if infoDict[0]["tables"][1]["metric_groups"][0]["tags"][3]["name"] != "data_dir_name": + tdLog.exit("data_dir_name is null!") + + if infoDict[0]["tables"][1]["metric_groups"][0]["metrics"][0]["name"] != "total": + tdLog.exit("total is null!") + + if infoDict[0]["tables"][1]["metric_groups"][0]["metrics"][1]["name"] != "used": + tdLog.exit("used is null!") + + if infoDict[0]["tables"][1]["metric_groups"][0]["metrics"][2]["name"] != "avail": + tdLog.exit("avail is null!") + + if infoDict[0]["tables"][2]["name"] != "taosd_dnodes_data_dirs": + tdLog.exit("taosd_dnodes_data_dirs is null!") + + # data_dir + if infoDict[0]["tables"][2]["metric_groups"][0]["tags"][3]["name"] != "data_dir_name": + tdLog.exit("data_dir_name is null!") + + if infoDict[0]["tables"][2]["metric_groups"][0]["metrics"][0]["name"] != "avail": + tdLog.exit("total is null!") + + if infoDict[0]["tables"][2]["metric_groups"][0]["metrics"][1]["name"] != "total": + tdLog.exit("used is null!") + + if infoDict[0]["tables"][2]["metric_groups"][0]["metrics"][2]["name"] != "used": + tdLog.exit("avail is null!") + + if infoDict[0]["tables"][3]["name"] != "taosd_cluster_info": + tdLog.exit("taosd_cluster_info is null!") + + # cluster_info ==================================== + + if infoDict[0]["tables"][3]["metric_groups"][0]["metrics"][0]["name"] != "cluster_uptime": + tdLog.exit("cluster_uptime is null!") + + if infoDict[0]["tables"][3]["metric_groups"][0]["metrics"][1]["name"] != "dbs_total": + tdLog.exit("dbs_total is null!") + + if infoDict[0]["tables"][3]["metric_groups"][0]["metrics"][4]["name"] != "vgroups_total": + tdLog.exit("vgroups_total is null!") + + if infoDict[0]["tables"][3]["metric_groups"][0]["metrics"][5]["name"] != "vgroups_alive": + tdLog.exit("vgroups_alive is null!") + + if infoDict[0]["tables"][3]["metric_groups"][0]["metrics"][10]["name"] != "connections_total": + tdLog.exit("connections_total is null!") + + if infoDict[0]["tables"][3]["metric_groups"][0]["metrics"][13]["name"] != "dnodes_total": + tdLog.exit("dnodes_total is null!") + + # grant_info ==================================== + if infoDict[0]["tables"][3]["metric_groups"][0]["metrics"][15]["name"] != "grants_expire_time": + tdLog.exit("grants_expire_time is null!") + + if infoDict[0]["tables"][3]["metric_groups"][0]["metrics"][16]["name"] != "grants_timeseries_used": + tdLog.exit("grants_timeseries_used is null!") + + if infoDict[0]["tables"][3]["metric_groups"][0]["metrics"][17]["name"] != "grants_timeseries_total": + tdLog.exit("grants_timeseries_total is null!") + + # vgroup_infos ==================================== + + vgroup_infos_nums = len(infoDict[0]["tables"][4]["metric_groups"]) + + for index in range(vgroup_infos_nums): + if infoDict[0]["tables"][4]["metric_groups"][index]["metrics"][0]["name"] != "tables_num": + tdLog.exit("tables_num is null!") + + if infoDict[0]["tables"][4]["metric_groups"][index]["metrics"][1]["name"] != "status": + tdLog.exit("status is null!") + + if infoDict[0]["tables"][5]["name"] != "taosd_dnodes_status": + tdLog.exit("taosd_dnodes_status is null!") + + if infoDict[0]["tables"][6]["name"] != "taosd_mnodes_info": + tdLog.exit("taosd_mnodes_info is null!") + + if infoDict[0]["tables"][7]["name"] != "taosd_vnodes_info": + tdLog.exit("taosd_vnodes_info is null!") + + def do_GET(self): + """ + process GET request + """ + + def do_POST(self): + """ + process POST request + """ + contentEncoding = self.headers["Content-Encoding"] + + if contentEncoding == 'gzip': + req_body = self.rfile.read(int(self.headers["Content-Length"])) + plainText = gzip.decompress(req_body).decode() + else: + plainText = self.rfile.read(int(self.headers["Content-Length"])).decode() + + print(plainText) + # 1. send response code and header + self.send_response(200) + self.send_header("Content-Type", "text/html; charset=utf-8") + self.end_headers() + + # 2. send response content + #self.wfile.write(("Hello World: " + req_body + "\n").encode("utf-8")) + + # 3. check request body info + infoDict = json.loads(plainText) + #print("================") + # print(infoDict) + self.telemetryInfoCheck(infoDict) + + # 4. shutdown the server and exit case + assassin = threading.Thread(target=self.server.shutdown) + assassin.daemon = True + assassin.start() + print ("==== shutdown http server ====") + +class TDTestCase: + global hostname + global serverPort + if (platform.system().lower() == 'windows' and not tdDnodes.dnodes[0].remoteIP == ""): + try: + config = eval(tdDnodes.dnodes[0].remoteIP ) + hostname = config["host"] + except Exception: + hostname = tdDnodes.dnodes[0].remoteIP + rpcDebugFlagVal = '143' + clientCfgDict = {'serverPort': '', 'firstEp': '', 'secondEp':'', 'rpcDebugFlag':'135', 'fqdn':''} + clientCfgDict["serverPort"] = serverPort + clientCfgDict["firstEp"] = hostname + ':' + serverPort + clientCfgDict["secondEp"] = hostname + ':' + serverPort + clientCfgDict["rpcDebugFlag"] = rpcDebugFlagVal + clientCfgDict["fqdn"] = hostname + + updatecfgDict = {'clientCfg': {}, 'serverPort': '', 'firstEp': '', 'secondEp':'', 'rpcDebugFlag':'135', 'fqdn':''} + updatecfgDict["clientCfg"] = clientCfgDict + updatecfgDict["serverPort"] = serverPort + updatecfgDict["firstEp"] = hostname + ':' + serverPort + updatecfgDict["secondEp"] = hostname + ':' + serverPort + updatecfgDict["fqdn"] = hostname + + updatecfgDict["monitorFqdn"] = hostname + updatecfgDict["monitorPort"] = '6043' + updatecfgDict["monitor"] = '1' + updatecfgDict["monitorInterval"] = "5" + updatecfgDict["monitorMaxLogs"] = "10" + updatecfgDict["monitorComp"] = "1" + updatecfgDict["monitorForceV2"] = "1" + + updatecfgDict["audit"] = '0' + + print ("===================: ", updatecfgDict) + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + + def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring + tdSql.prepare() + # time.sleep(2) + vgroups = "4" + sql = "create database db3 vgroups " + vgroups + tdSql.query(sql) + sql = "create table db3.stb (ts timestamp, f int) tags (t int)" + tdSql.query(sql) + sql = "create table db3.tb using db3.stb tags (1)" + tdSql.query(sql) + + # create http server: bing ip/port , and request processor + if (platform.system().lower() == 'windows' and not tdDnodes.dnodes[0].remoteIP == ""): + RequestHandlerImplStr = base64.b64encode(pickle.dumps(RequestHandlerImpl)).decode() + cmdStr = "import pickle\nimport http\nRequestHandlerImpl=pickle.loads(base64.b64decode(\"%s\".encode()))\nclass NewRequestHandlerImpl(RequestHandlerImpl):\n hostPort = \'%s\'\nhttp.server.HTTPServer((\"\", %s), NewRequestHandlerImpl).serve_forever()"%(RequestHandlerImplStr,hostname+":"+serverPort,telemetryPort) + tdDnodes.dnodes[0].remoteExec({}, cmdStr) + else: + serverAddress = ("", int(telemetryPort)) + http.server.HTTPServer(serverAddress, RequestHandlerImpl).serve_forever() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/0-others/udfTest.py b/tests/system-test/0-others/udfTest.py index d3efa61e04..8134327b41 100644 --- a/tests/system-test/0-others/udfTest.py +++ b/tests/system-test/0-others/udfTest.py @@ -11,6 +11,7 @@ from util.log import * from util.sql import * from util.cases import * from util.dnodes import * +from util.tserror import * import subprocess class TDTestCase: @@ -56,8 +57,32 @@ class TDTestCase: else: self.libudf1 = subprocess.Popen('find %s -name "libudf1.so"|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") self.libudf2 = subprocess.Popen('find %s -name "libudf2.so"|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") + self.libchange_udf_no_init = subprocess.Popen('find %s -name "libchange_udf_no_init.so"|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") + self.libchange_udf_no_process = subprocess.Popen('find %s -name "libchange_udf_no_process.so"|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") + self.libchange_udf_no_destroy = subprocess.Popen('find %s -name "libchange_udf_no_destroy.so"|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") + self.libchange_udf_init_failed = subprocess.Popen('find %s -name "libchange_udf_init_failed.so"|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") + self.libchange_udf_process_failed = subprocess.Popen('find %s -name "libchange_udf_process_failed.so"|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") + self.libchange_udf_destroy_failed = subprocess.Popen('find %s -name "libchange_udf_destory_failed.so"|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") + self.libchange_udf_normal = subprocess.Popen('find %s -name "libchange_udf_normal.so"|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") + self.libudf1 = self.libudf1.replace('\r','').replace('\n','') self.libudf2 = self.libudf2.replace('\r','').replace('\n','') + self.libchange_udf_no_init = self.libchange_udf_no_init.replace('\r','').replace('\n','') + self.libchange_udf_no_process = self.libchange_udf_no_process.replace('\r','').replace('\n','') + self.libchange_udf_normal = self.libchange_udf_normal.replace('\r','').replace('\n','') + self.libchange_udf_no_destroy = self.libchange_udf_no_destroy.replace('\r','').replace('\n','') + self.libchange_udf_init_failed = self.libchange_udf_init_failed.replace('\r','').replace('\n','') + self.libchange_udf_process_failed = self.libchange_udf_process_failed.replace('\r','').replace('\n','') + self.libchange_udf_destroy_failed = self.libchange_udf_destroy_failed.replace('\r','').replace('\n','') + tdLog.info(f"udf1 so path is {self.libudf1}") + tdLog.info(f"udf2 so path is {self.libudf2}") + tdLog.info(f"change_udf_no_init so path is {self.libchange_udf_no_init}") + tdLog.info(f"change_udf_no_process so path is {self.libchange_udf_no_process}") + tdLog.info(f"change_udf_no_destroy so path is {self.libchange_udf_no_destroy}") + tdLog.info(f"change_udf_init_failed so path is {self.libchange_udf_init_failed}") + tdLog.info(f"change_udf_process_failed so path is {self.libchange_udf_process_failed}") + tdLog.info(f"change_udf_destroy_failed so path is {self.libchange_udf_destroy_failed}") + tdLog.info(f"change_udf_normal so path is {self.libchange_udf_normal}") def prepare_data(self): @@ -664,13 +689,118 @@ class TDTestCase: path = ''.join(random.choice(letters) for i in range(5000)) os.system(f"udfd -c {path}") + + def test_change_udf_normal(self, func_name): + # create function with normal file + tdSql.execute(f"create function {func_name} as '%s' outputtype int"%self.libchange_udf_normal) + functions = tdSql.getResult("show functions") + for function in functions: + if f"{func_name}" in function[0]: + tdLog.info(f"create {func_name} functions success, using {self.libchange_udf_normal}") + break + tdSql.query(f"select num1 , {func_name}(num1) ,num2 ,{func_name}(num2),num3 ,{func_name}(num3),num4 ,{func_name}(num4) from db.tb", TSDB_CODE_UDF_FUNC_EXEC_FAILURE) + tdSql.checkData(0,0,None) + tdSql.checkData(0,1,None) + tdSql.checkData(0,2,1) + tdSql.checkData(0,3,1) + tdSql.checkData(0,4,1.000000000) + tdSql.checkData(0,5,1) + tdSql.checkData(0,6,"binary1") + tdSql.checkData(0,7,1) + tdSql.query(f"select {func_name}(num1) from db.tb", TSDB_CODE_UDF_FUNC_EXEC_FAILURE) + tdSql.execute(f"drop function {func_name}") + tdSql.error(f"select {func_name}(num1) from db.tb", TSDB_CODE_MND_FUNC_NOT_EXIST) + tdLog.info(f"change udf test finished, using {self.libchange_udf_normal}") + + def test_change_udf_failed(self, func_name, lib_name): + tdLog.info(f"test change udf start: using {lib_name}") + tdSql.error(f"select num1 , {func_name}(num1) ,num2 ,{func_name}(num2),num3 ,{func_name}(num3),num4 ,{func_name}(num4) from db.tb", TSDB_CODE_MND_FUNC_NOT_EXIST) + tdSql.execute(f"create function {func_name} as '{lib_name}' outputtype int") + functions = tdSql.getResult("show functions") + for function in functions: + if f"{func_name}" in function[0]: + tdLog.info(f"create {func_name} functions success, using {lib_name}") + break + + tdSql.error(f"select num1 , {func_name}(num1) ,num2 ,{func_name}(num2),num3 ,{func_name}(num3),num4 ,{func_name}(num4) from db.tb", TSDB_CODE_UDF_FUNC_EXEC_FAILURE) + tdSql.error(f"select {func_name}(num1) from db.tb", TSDB_CODE_UDF_FUNC_EXEC_FAILURE) + tdSql.execute(f"drop function {func_name}") + tdSql.error(f"select {func_name}(num1) from db.tb", TSDB_CODE_MND_FUNC_NOT_EXIST) + tdLog.info(f"change udf test finished, using {lib_name}") + + def test_change_udf_reverse(self): + tdSql.execute("create database if not exists db duration 100") + tdSql.execute("use db") + + func_name = "udf_reverse" + tdSql.execute(f"create function {func_name} as '%s' outputtype nchar(256)"%self.libchange_udf_normal) + functions = tdSql.getResult("show functions") + for function in functions: + if f"{func_name}" in function[0]: + tdLog.info(f"create {func_name} functions success, using {self.libchange_udf_normal}") + break + + tdSql.error(f"select {func_name}(c8) from db.t1", TSDB_CODE_TSC_INTERNAL_ERROR) + tdSql.execute(f"drop function {func_name}") + tdSql.error(f"select {func_name}(num1) from db.tb", TSDB_CODE_MND_FUNC_NOT_EXIST) + + self.test_change_udf_normal("change_udf_normal") + tdSql.execute(f"create function {func_name} as '%s' outputtype varchar(256)"%self.libchange_udf_normal) + functions = tdSql.getResult("show functions") + for function in functions: + if f"{func_name}" in function[0]: + tdLog.info(f"create {func_name} functions success, using {self.libchange_udf_normal}") + break + + tdSql.query(f"select {func_name}(c8) from db.t1 order by ts") + tdSql.checkData(0,0, None) + tdSql.checkData(1,0, "1yranib") + tdSql.checkData(2,0, "2yranib") + tdSql.checkData(3,0, "3yranib") + + + + def unexpected_using_test(self): + tdSql.execute("use db ") + + # create function without wrong file path + tdSql.error("create function udf1 as '%s_wrongpath' outputtype int;"%self.libudf1, TAOS_SYSTEM_ERROR|2) + tdSql.error("create aggregate function udf2 as '%s_wrongpath' outputtype double;"%self.libudf2, TAOS_SYSTEM_ERROR|2) + + tdSql.execute("create function udf1 as '%s' outputtype int;"%self.libudf1) + tdSql.query("select num1 , udf1(num1) ,num2 ,udf1(num2),num3 ,udf1(num3),num4 ,udf1(num4) from tb") + + self.test_change_udf_normal("change_udf_normal") + self.test_change_udf_failed("change_udf_no_init", self.libchange_udf_no_init) + + self.test_change_udf_normal("change_udf_normal") + self.test_change_udf_failed("change_udf_no_process", self.libchange_udf_no_process) + + self.test_change_udf_normal("change_udf_normal") + self.test_change_udf_failed("change_udf_no_destroy", self.libchange_udf_no_destroy) + + self.test_change_udf_normal("change_udf_normal") + self.test_change_udf_failed("change_udf_init_failed", self.libchange_udf_init_failed) + + self.test_change_udf_normal("change_udf_normal") + self.test_change_udf_failed("change_udf_process_failed", self.libchange_udf_process_failed) + + self.test_change_udf_normal("change_udf_normal") + self.test_change_udf_failed("libchange_udf_destroy_failed", self.libchange_udf_destroy_failed) + + tdSql.query("select num1 , udf1(num1) ,num2 ,udf1(num2),num3 ,udf1(num3),num4 ,udf1(num4) from tb") + tdSql.execute(f"drop function udf1") + tdSql.error("select num1 , udf1(num1) ,num2 ,udf1(num2),num3 ,udf1(num3),num4 ,udf1(num4) from tb", TSDB_CODE_MND_FUNC_NOT_EXIST) def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring print(" env is ok for all ") self.test_udfd_cmd() self.prepare_udf_so() self.prepare_data() + + self.unexpected_using_test() self.create_udf_function() + self.test_change_udf_reverse() self.basic_udf_query() self.loop_kill_udfd() tdSql.execute(" drop function udf1 ") diff --git a/tests/system-test/6-cluster/replica2.py b/tests/system-test/6-cluster/replica2.py new file mode 100644 index 0000000000..1cda2bb4e8 --- /dev/null +++ b/tests/system-test/6-cluster/replica2.py @@ -0,0 +1,50 @@ +################################################################### + # Copyright (c) 2016 by TAOS Technologies, Inc. + # All rights reserved. + # + # This file is proprietary and confidential to TAOS Technologies. + # No part of this file may be reproduced, stored, transmitted, + # disclosed or used in any form or by any means other than as + # expressly provided by the written permission from Jianhui Tao + # +################################################################### +from util.cases import * +from util.sql import * +from util.dnodes import * +from util.log import * + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + tdLog.debug(f"start to init {__file__}") + self.replicaVar = int(replicaVar) + tdSql.init(conn.cursor(), logSql) + + def run(self): + tdSql.execute('CREATE DATABASE db vgroups 1 replica 2;') + + time.sleep(1) + + tdSql.query("show db.vgroups;") + + if(tdSql.queryResult[0][4] == "follower") and (tdSql.queryResult[0][6] == "leader"): + tdLog.info("stop dnode2") + sc.dnodeStop(2) + + if(tdSql.queryResult[0][6] == "follower") and (tdSql.queryResult[0][4] == "leader"): + tdLog.info("stop dnode 3") + sc.dnodeStop(3) + + tdLog.info("wait 10 seconds") + time.sleep(10) + + tdSql.query("show db.vgroups;") + + if(tdSql.queryResult[0][4] != "assigned") and (tdSql.queryResult[0][6] != "assigned"): + tdLog.exit("failed to set aasigned") + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file