From 8991e50eeebcda967a5e8e1b95b906240c758fc1 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 21 Nov 2024 15:52:20 +0800 Subject: [PATCH 01/45] enh: (TD-29367-1)Add delete entry encode and decode --- source/dnode/vnode/src/meta/metaEntry.c | 178 ++++++++++++------------ 1 file changed, 91 insertions(+), 87 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaEntry.c b/source/dnode/vnode/src/meta/metaEntry.c index 3b6eaf45d3..49f9dfa85a 100644 --- a/source/dnode/vnode/src/meta/metaEntry.c +++ b/source/dnode/vnode/src/meta/metaEntry.c @@ -71,44 +71,46 @@ int metaEncodeEntry(SEncoder *pCoder, const SMetaEntry *pME) { TAOS_CHECK_RETURN(tEncodeI8(pCoder, pME->type)); TAOS_CHECK_RETURN(tEncodeI64(pCoder, pME->uid)); - if (pME->name == NULL) { - return TSDB_CODE_INVALID_PARA; - } + if (pME->type < 0) { + if (pME->name == NULL) { + return TSDB_CODE_INVALID_PARA; + } - TAOS_CHECK_RETURN(tEncodeCStr(pCoder, pME->name)); + TAOS_CHECK_RETURN(tEncodeCStr(pCoder, pME->name)); - if (pME->type == TSDB_SUPER_TABLE) { - TAOS_CHECK_RETURN(tEncodeI8(pCoder, pME->flags)); - TAOS_CHECK_RETURN(tEncodeSSchemaWrapper(pCoder, &pME->stbEntry.schemaRow)); - TAOS_CHECK_RETURN(tEncodeSSchemaWrapper(pCoder, &pME->stbEntry.schemaTag)); - if (TABLE_IS_ROLLUP(pME->flags)) { - TAOS_CHECK_RETURN(tEncodeSRSmaParam(pCoder, &pME->stbEntry.rsmaParam)); + if (pME->type == TSDB_SUPER_TABLE) { + TAOS_CHECK_RETURN(tEncodeI8(pCoder, pME->flags)); + TAOS_CHECK_RETURN(tEncodeSSchemaWrapper(pCoder, &pME->stbEntry.schemaRow)); + TAOS_CHECK_RETURN(tEncodeSSchemaWrapper(pCoder, &pME->stbEntry.schemaTag)); + if (TABLE_IS_ROLLUP(pME->flags)) { + TAOS_CHECK_RETURN(tEncodeSRSmaParam(pCoder, &pME->stbEntry.rsmaParam)); + } + } else if (pME->type == TSDB_CHILD_TABLE) { + TAOS_CHECK_RETURN(tEncodeI64(pCoder, pME->ctbEntry.btime)); + TAOS_CHECK_RETURN(tEncodeI32(pCoder, pME->ctbEntry.ttlDays)); + TAOS_CHECK_RETURN(tEncodeI32v(pCoder, pME->ctbEntry.commentLen)); + if (pME->ctbEntry.commentLen > 0) { + TAOS_CHECK_RETURN(tEncodeCStr(pCoder, pME->ctbEntry.comment)); + } + TAOS_CHECK_RETURN(tEncodeI64(pCoder, pME->ctbEntry.suid)); + TAOS_CHECK_RETURN(tEncodeTag(pCoder, (const STag *)pME->ctbEntry.pTags)); + } else if (pME->type == TSDB_NORMAL_TABLE) { + TAOS_CHECK_RETURN(tEncodeI64(pCoder, pME->ntbEntry.btime)); + TAOS_CHECK_RETURN(tEncodeI32(pCoder, pME->ntbEntry.ttlDays)); + TAOS_CHECK_RETURN(tEncodeI32v(pCoder, pME->ntbEntry.commentLen)); + if (pME->ntbEntry.commentLen > 0) { + TAOS_CHECK_RETURN(tEncodeCStr(pCoder, pME->ntbEntry.comment)); + } + TAOS_CHECK_RETURN(tEncodeI32v(pCoder, pME->ntbEntry.ncid)); + TAOS_CHECK_RETURN(tEncodeSSchemaWrapper(pCoder, &pME->ntbEntry.schemaRow)); + } else if (pME->type == TSDB_TSMA_TABLE) { + TAOS_CHECK_RETURN(tEncodeTSma(pCoder, pME->smaEntry.tsma)); + } else { + metaError("meta/entry: invalide table type: %" PRId8 " encode failed.", pME->type); + return TSDB_CODE_INVALID_PARA; } - } else if (pME->type == TSDB_CHILD_TABLE) { - TAOS_CHECK_RETURN(tEncodeI64(pCoder, pME->ctbEntry.btime)); - TAOS_CHECK_RETURN(tEncodeI32(pCoder, pME->ctbEntry.ttlDays)); - TAOS_CHECK_RETURN(tEncodeI32v(pCoder, pME->ctbEntry.commentLen)); - if (pME->ctbEntry.commentLen > 0) { - TAOS_CHECK_RETURN(tEncodeCStr(pCoder, pME->ctbEntry.comment)); - } - TAOS_CHECK_RETURN(tEncodeI64(pCoder, pME->ctbEntry.suid)); - TAOS_CHECK_RETURN(tEncodeTag(pCoder, (const STag *)pME->ctbEntry.pTags)); - } else if (pME->type == TSDB_NORMAL_TABLE) { - TAOS_CHECK_RETURN(tEncodeI64(pCoder, pME->ntbEntry.btime)); - TAOS_CHECK_RETURN(tEncodeI32(pCoder, pME->ntbEntry.ttlDays)); - TAOS_CHECK_RETURN(tEncodeI32v(pCoder, pME->ntbEntry.commentLen)); - if (pME->ntbEntry.commentLen > 0) { - TAOS_CHECK_RETURN(tEncodeCStr(pCoder, pME->ntbEntry.comment)); - } - TAOS_CHECK_RETURN(tEncodeI32v(pCoder, pME->ntbEntry.ncid)); - TAOS_CHECK_RETURN(tEncodeSSchemaWrapper(pCoder, &pME->ntbEntry.schemaRow)); - } else if (pME->type == TSDB_TSMA_TABLE) { - TAOS_CHECK_RETURN(tEncodeTSma(pCoder, pME->smaEntry.tsma)); - } else { - metaError("meta/entry: invalide table type: %" PRId8 " encode failed.", pME->type); - return TSDB_CODE_INVALID_PARA; + TAOS_CHECK_RETURN(meteEncodeColCmprEntry(pCoder, pME)); } - TAOS_CHECK_RETURN(meteEncodeColCmprEntry(pCoder, pME)); tEndEncode(pCoder); return 0; @@ -119,66 +121,68 @@ int metaDecodeEntry(SDecoder *pCoder, SMetaEntry *pME) { TAOS_CHECK_RETURN(tDecodeI64(pCoder, &pME->version)); TAOS_CHECK_RETURN(tDecodeI8(pCoder, &pME->type)); TAOS_CHECK_RETURN(tDecodeI64(pCoder, &pME->uid)); - TAOS_CHECK_RETURN(tDecodeCStr(pCoder, &pME->name)); + if (pME->type < 0) { + TAOS_CHECK_RETURN(tDecodeCStr(pCoder, &pME->name)); - if (pME->type == TSDB_SUPER_TABLE) { - TAOS_CHECK_RETURN(tDecodeI8(pCoder, &pME->flags)); - TAOS_CHECK_RETURN(tDecodeSSchemaWrapperEx(pCoder, &pME->stbEntry.schemaRow)); - TAOS_CHECK_RETURN(tDecodeSSchemaWrapperEx(pCoder, &pME->stbEntry.schemaTag)); - if (TABLE_IS_ROLLUP(pME->flags)) { - TAOS_CHECK_RETURN(tDecodeSRSmaParam(pCoder, &pME->stbEntry.rsmaParam)); - } - } else if (pME->type == TSDB_CHILD_TABLE) { - TAOS_CHECK_RETURN(tDecodeI64(pCoder, &pME->ctbEntry.btime)); - TAOS_CHECK_RETURN(tDecodeI32(pCoder, &pME->ctbEntry.ttlDays)); - TAOS_CHECK_RETURN(tDecodeI32v(pCoder, &pME->ctbEntry.commentLen)); - if (pME->ctbEntry.commentLen > 0) { - TAOS_CHECK_RETURN(tDecodeCStr(pCoder, &pME->ctbEntry.comment)); - } - TAOS_CHECK_RETURN(tDecodeI64(pCoder, &pME->ctbEntry.suid)); - TAOS_CHECK_RETURN(tDecodeTag(pCoder, (STag **)&pME->ctbEntry.pTags)); - } else if (pME->type == TSDB_NORMAL_TABLE) { - TAOS_CHECK_RETURN(tDecodeI64(pCoder, &pME->ntbEntry.btime)); - TAOS_CHECK_RETURN(tDecodeI32(pCoder, &pME->ntbEntry.ttlDays)); - TAOS_CHECK_RETURN(tDecodeI32v(pCoder, &pME->ntbEntry.commentLen)); - if (pME->ntbEntry.commentLen > 0) { - TAOS_CHECK_RETURN(tDecodeCStr(pCoder, &pME->ntbEntry.comment)); - } - TAOS_CHECK_RETURN(tDecodeI32v(pCoder, &pME->ntbEntry.ncid)); - TAOS_CHECK_RETURN(tDecodeSSchemaWrapperEx(pCoder, &pME->ntbEntry.schemaRow)); - } else if (pME->type == TSDB_TSMA_TABLE) { - pME->smaEntry.tsma = tDecoderMalloc(pCoder, sizeof(STSma)); - if (!pME->smaEntry.tsma) { - return terrno; - } - TAOS_CHECK_RETURN(tDecodeTSma(pCoder, pME->smaEntry.tsma, true)); - } else { - metaError("meta/entry: invalide table type: %" PRId8 " decode failed.", pME->type); - return TSDB_CODE_INVALID_PARA; - } - if (pME->type == TSDB_SUPER_TABLE) { - if (TABLE_IS_COL_COMPRESSED(pME->flags)) { - TAOS_CHECK_RETURN(meteDecodeColCmprEntry(pCoder, pME)); - - if (pME->colCmpr.nCols == 0) { - TAOS_CHECK_RETURN(metatInitDefaultSColCmprWrapper(pCoder, &pME->colCmpr, &pME->stbEntry.schemaRow)); + if (pME->type == TSDB_SUPER_TABLE) { + TAOS_CHECK_RETURN(tDecodeI8(pCoder, &pME->flags)); + TAOS_CHECK_RETURN(tDecodeSSchemaWrapperEx(pCoder, &pME->stbEntry.schemaRow)); + TAOS_CHECK_RETURN(tDecodeSSchemaWrapperEx(pCoder, &pME->stbEntry.schemaTag)); + if (TABLE_IS_ROLLUP(pME->flags)) { + TAOS_CHECK_RETURN(tDecodeSRSmaParam(pCoder, &pME->stbEntry.rsmaParam)); } + } else if (pME->type == TSDB_CHILD_TABLE) { + TAOS_CHECK_RETURN(tDecodeI64(pCoder, &pME->ctbEntry.btime)); + TAOS_CHECK_RETURN(tDecodeI32(pCoder, &pME->ctbEntry.ttlDays)); + TAOS_CHECK_RETURN(tDecodeI32v(pCoder, &pME->ctbEntry.commentLen)); + if (pME->ctbEntry.commentLen > 0) { + TAOS_CHECK_RETURN(tDecodeCStr(pCoder, &pME->ctbEntry.comment)); + } + TAOS_CHECK_RETURN(tDecodeI64(pCoder, &pME->ctbEntry.suid)); + TAOS_CHECK_RETURN(tDecodeTag(pCoder, (STag **)&pME->ctbEntry.pTags)); + } else if (pME->type == TSDB_NORMAL_TABLE) { + TAOS_CHECK_RETURN(tDecodeI64(pCoder, &pME->ntbEntry.btime)); + TAOS_CHECK_RETURN(tDecodeI32(pCoder, &pME->ntbEntry.ttlDays)); + TAOS_CHECK_RETURN(tDecodeI32v(pCoder, &pME->ntbEntry.commentLen)); + if (pME->ntbEntry.commentLen > 0) { + TAOS_CHECK_RETURN(tDecodeCStr(pCoder, &pME->ntbEntry.comment)); + } + TAOS_CHECK_RETURN(tDecodeI32v(pCoder, &pME->ntbEntry.ncid)); + TAOS_CHECK_RETURN(tDecodeSSchemaWrapperEx(pCoder, &pME->ntbEntry.schemaRow)); + } else if (pME->type == TSDB_TSMA_TABLE) { + pME->smaEntry.tsma = tDecoderMalloc(pCoder, sizeof(STSma)); + if (!pME->smaEntry.tsma) { + return terrno; + } + TAOS_CHECK_RETURN(tDecodeTSma(pCoder, pME->smaEntry.tsma, true)); } else { - TAOS_CHECK_RETURN(metatInitDefaultSColCmprWrapper(pCoder, &pME->colCmpr, &pME->stbEntry.schemaRow)); - TABLE_SET_COL_COMPRESSED(pME->flags); + metaError("meta/entry: invalide table type: %" PRId8 " decode failed.", pME->type); + return TSDB_CODE_INVALID_PARA; } - } else if (pME->type == TSDB_NORMAL_TABLE) { - if (!tDecodeIsEnd(pCoder)) { - uDebug("set type: %d, tableName:%s", pME->type, pME->name); - TAOS_CHECK_RETURN(meteDecodeColCmprEntry(pCoder, pME)); - if (pME->colCmpr.nCols == 0) { + if (pME->type == TSDB_SUPER_TABLE) { + if (TABLE_IS_COL_COMPRESSED(pME->flags)) { + TAOS_CHECK_RETURN(meteDecodeColCmprEntry(pCoder, pME)); + + if (pME->colCmpr.nCols == 0) { + TAOS_CHECK_RETURN(metatInitDefaultSColCmprWrapper(pCoder, &pME->colCmpr, &pME->stbEntry.schemaRow)); + } + } else { + TAOS_CHECK_RETURN(metatInitDefaultSColCmprWrapper(pCoder, &pME->colCmpr, &pME->stbEntry.schemaRow)); + TABLE_SET_COL_COMPRESSED(pME->flags); + } + } else if (pME->type == TSDB_NORMAL_TABLE) { + if (!tDecodeIsEnd(pCoder)) { + uDebug("set type: %d, tableName:%s", pME->type, pME->name); + TAOS_CHECK_RETURN(meteDecodeColCmprEntry(pCoder, pME)); + if (pME->colCmpr.nCols == 0) { + TAOS_CHECK_RETURN(metatInitDefaultSColCmprWrapper(pCoder, &pME->colCmpr, &pME->ntbEntry.schemaRow)); + } + } else { + uDebug("set default type: %d, tableName:%s", pME->type, pME->name); TAOS_CHECK_RETURN(metatInitDefaultSColCmprWrapper(pCoder, &pME->colCmpr, &pME->ntbEntry.schemaRow)); } - } else { - uDebug("set default type: %d, tableName:%s", pME->type, pME->name); - TAOS_CHECK_RETURN(metatInitDefaultSColCmprWrapper(pCoder, &pME->colCmpr, &pME->ntbEntry.schemaRow)); + TABLE_SET_COL_COMPRESSED(pME->flags); } - TABLE_SET_COL_COMPRESSED(pME->flags); } tEndDecode(pCoder); From 4d6faa8f13ad49c610ba842b59e4558ec6a0efd8 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 21 Nov 2024 16:09:55 +0800 Subject: [PATCH 02/45] enh: (TD-29367-2) add a new meta handle function called `metaHandleEntry2` --- source/dnode/vnode/src/meta/metaEntry2.c | 46 ++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 source/dnode/vnode/src/meta/metaEntry2.c diff --git a/source/dnode/vnode/src/meta/metaEntry2.c b/source/dnode/vnode/src/meta/metaEntry2.c new file mode 100644 index 0000000000..dbdc10a94a --- /dev/null +++ b/source/dnode/vnode/src/meta/metaEntry2.c @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2023 Hongze Cheng . + * All rights reserved. + * + * This code is the intellectual property of Hongze Cheng. + * Any reproduction or distribution, in whole or in part, + * without the express written permission of Hongze Cheng is + * strictly prohibited. + */ + +#include "meta.h" + +#define metaErrLog(ERRNO) metaError("%s failed at %s:%d since %s", __func__, __FILE__, __LINE__, tstrerror(ERRNO)); + +static int32_t metaHandleEntryDrop(SMeta *pMeta, const SMetaEntry *pEntry) { + int32_t code = TSDB_CODE_SUCCESS; + + // TODO + + return code; +} + +static int32_t metaHandleEntryUpsert(SMeta *pMeta, const SMetaEntry *pEntry) { + int32_t code = TSDB_CODE_SUCCESS; + + // TODO + + return code; +} + +int32_t metaHandleEntry2(SMeta *pMeta, const SMetaEntry *pEntry) { + int32_t code = TSDB_CODE_SUCCESS; + + if (pMeta == NULL || pEntry == NULL) { + metaErrLog(TSDB_CODE_INVALID_PARA); + return TSDB_CODE_INVALID_PARA; + } + + if (pEntry->type < 0) { + code = metaHandleEntryDrop(pMeta, pEntry); + } else { + code = metaHandleEntryUpsert(pMeta, pEntry); + } + return code; + ; +} \ No newline at end of file From 3d2a5be1b6d5697b2b10792cffd7497faa54cb7c Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 3 Dec 2024 15:20:27 +0800 Subject: [PATCH 03/45] more code --- source/dnode/vnode/CMakeLists.txt | 2 + source/dnode/vnode/src/meta/metaEntry2.c | 121 +++++++++++++++++++++-- source/dnode/vnode/src/meta/metaTable2.c | 11 +++ 3 files changed, 127 insertions(+), 7 deletions(-) create mode 100644 source/dnode/vnode/src/meta/metaTable2.c diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index c377e69f03..8f63cc8779 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -32,6 +32,8 @@ set( "src/meta/metaSnapshot.c" "src/meta/metaCache.c" "src/meta/metaTtl.c" + "src/meta/metaEntry2.c" + "src/meta/metaTable2.c" # sma "src/sma/smaEnv.c" diff --git a/source/dnode/vnode/src/meta/metaEntry2.c b/source/dnode/vnode/src/meta/metaEntry2.c index dbdc10a94a..812314c8c5 100644 --- a/source/dnode/vnode/src/meta/metaEntry2.c +++ b/source/dnode/vnode/src/meta/metaEntry2.c @@ -10,20 +10,127 @@ #include "meta.h" -#define metaErrLog(ERRNO) metaError("%s failed at %s:%d since %s", __func__, __FILE__, __LINE__, tstrerror(ERRNO)); +#define metaErrLog(ERRNO, INFO) \ + do { \ + if (INFO) { \ + metaError("%s failed at %s:%d since %s, info:%s", __func__, __FILE__, __LINE__, tstrerror(ERRNO), INFO); \ + } else { \ + metaError("%s failed at %s:%d since %s", __func__, __FILE__, __LINE__, tstrerror(ERRNO)); \ + } \ + } while (0) + +static int32_t metaHandleNormalTableDrop(SMeta *pMeta, const SMetaEntry *pEntry) { + int32_t code = TSDB_CODE_SUCCESS; + // TODO + return code; +} + +static int32_t metaHandleChildTableDrop(SMeta *pMeta, const SMetaEntry *pEntry) { + int32_t code = TSDB_CODE_SUCCESS; + // TODO + return code; +} + +static int32_t metaHandleSuperTableDrop(SMeta *pMeta, const SMetaEntry *pEntry) { + int32_t code = TSDB_CODE_SUCCESS; + // TODO + return code; +} static int32_t metaHandleEntryDrop(SMeta *pMeta, const SMetaEntry *pEntry) { int32_t code = TSDB_CODE_SUCCESS; + int8_t type = -pEntry->type; + + if (type == TSDB_NORMAL_TABLE) { + code = metaHandleNormalTableDrop(pMeta, pEntry); + } else if (type == TSDB_CHILD_TABLE) { + code = metaHandleChildTableDrop(pMeta, pEntry); + } else if (type == TSDB_SUPER_TABLE) { + code = metaHandleSuperTableDrop(pMeta, pEntry); + } else { + code = TSDB_CODE_INVALID_PARA; + metaErrLog(code, NULL); + } + + if (TSDB_CODE_SUCCESS == code) { + } else { + } + + return code; +} + +static int32_t metaHandleNormalTableEntryUpdate(SMeta *pMeta, const SMetaEntry *pEntry) { + int32_t code = TSDB_CODE_SUCCESS; + // TODO: + return code; +} + +static int32_t metaHandleNormalTableEntryCreateImpl(SMeta *pMeta, const SMetaEntry *pEntry) { + int32_t code = TSDB_CODE_SUCCESS; // TODO + // code = metaSaveToTbDb(pMeta, pEntry); + // code = metaUpdateUidIdx(pMeta, pEntry); + // code = metaUpdatenameIdx(pMeta, pEntry); + // code = metaSaveToSkmDb(pMeta, pEntry); + // code = metaUpdateBtimeIdx(pMeta, pEntry); + // code = metaUpdateNcolIdx(pMeta, pEntry); + // code = metaUpdateTtl(pMeta, pEntry); return code; } +static int32_t metaHandleNormalTableEntryCreate(SMeta *pMeta, const SMetaEntry *pEntry) { + int32_t code = TSDB_CODE_SUCCESS; + + // TODO: do neccessary check + + // do handle entry + metaWLock(pMeta); + code = metaHandleNormalTableEntryCreateImpl(pMeta, pEntry); + metaULock(pMeta); + + return code; +} + +static int32_t metaHandleNormalTableEntryUpsert(SMeta *pMeta, const SMetaEntry *pEntry) { + int32_t code = TSDB_CODE_SUCCESS; + SMetaInfo metaInfo = {0}; + + if (TSDB_CODE_SUCCESS == metaGetInfo(pMeta, pEntry->uid, &metaInfo, NULL)) { + code = metaHandleNormalTableEntryUpdate(pMeta, pEntry); + } else { + code = metaHandleNormalTableEntryCreate(pMeta, pEntry); + } + + return code; +} + +static int32_t metaHandleSuperTableEntryUpsert(SMeta *pMeta, const SMetaEntry *pEntry) { + int32_t code = TSDB_CODE_SUCCESS; + // TODO + return code; +} + +static int32_t metaHandleChildTableEntryUpsert(SMeta *pMeta, const SMetaEntry *pEntry) { + int32_t code = TSDB_CODE_SUCCESS; + // TODO + return code; +} + static int32_t metaHandleEntryUpsert(SMeta *pMeta, const SMetaEntry *pEntry) { int32_t code = TSDB_CODE_SUCCESS; - // TODO + if (pEntry->type == TSDB_NORMAL_TABLE) { + code = metaHandleNormalTableEntryUpsert(pMeta, pEntry); + } else if (pEntry->type == TSDB_SUPER_TABLE) { + code = metaHandleSuperTableEntryUpsert(pMeta, pEntry); + } else if (pEntry->type == TSDB_CHILD_TABLE) { + code = metaHandleChildTableEntryUpsert(pMeta, pEntry); + } else { + code = TSDB_CODE_INVALID_PARA; + metaErrLog(code, NULL); + } return code; } @@ -32,15 +139,15 @@ int32_t metaHandleEntry2(SMeta *pMeta, const SMetaEntry *pEntry) { int32_t code = TSDB_CODE_SUCCESS; if (pMeta == NULL || pEntry == NULL) { - metaErrLog(TSDB_CODE_INVALID_PARA); + metaErrLog(TSDB_CODE_INVALID_PARA, NULL); return TSDB_CODE_INVALID_PARA; } - if (pEntry->type < 0) { - code = metaHandleEntryDrop(pMeta, pEntry); - } else { + if (pEntry->type > 0) { code = metaHandleEntryUpsert(pMeta, pEntry); + } else { + code = metaHandleEntryDrop(pMeta, pEntry); } + return code; - ; } \ No newline at end of file diff --git a/source/dnode/vnode/src/meta/metaTable2.c b/source/dnode/vnode/src/meta/metaTable2.c new file mode 100644 index 0000000000..bb44899d8e --- /dev/null +++ b/source/dnode/vnode/src/meta/metaTable2.c @@ -0,0 +1,11 @@ +/* + * Copyright (c) 2023 Hongze Cheng . + * All rights reserved. + * + * This code is the intellectual property of Hongze Cheng. + * Any reproduction or distribution, in whole or in part, + * without the express written permission of Hongze Cheng is + * strictly prohibited. + */ + +#include "meta.h" \ No newline at end of file From 8120765b24393af74defc6d4b75c4da3ae3ffa88 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 3 Dec 2024 17:00:37 +0800 Subject: [PATCH 04/45] Make a copy of old code --- source/dnode/vnode/src/meta/metaTable2.c | 3459 +++++++++++++++++++++- 1 file changed, 3452 insertions(+), 7 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaTable2.c b/source/dnode/vnode/src/meta/metaTable2.c index bb44899d8e..65e520bb4a 100644 --- a/source/dnode/vnode/src/meta/metaTable2.c +++ b/source/dnode/vnode/src/meta/metaTable2.c @@ -1,11 +1,3456 @@ /* - * Copyright (c) 2023 Hongze Cheng . - * All rights reserved. + * Copyright (c) 2019 TAOS Data, Inc. * - * This code is the intellectual property of Hongze Cheng. - * Any reproduction or distribution, in whole or in part, - * without the express written permission of Hongze Cheng is - * strictly prohibited. + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . */ -#include "meta.h" \ No newline at end of file +#include "meta.h" + +extern SDmNotifyHandle dmNotifyHdl; + +static int metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSchema *pSchema); +static int metaDelJsonVarFromIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSchema *pSchema); +static int metaSaveToTbDb(SMeta *pMeta, const SMetaEntry *pME); +static int metaUpdateUidIdx(SMeta *pMeta, const SMetaEntry *pME); +static int metaUpdateNameIdx(SMeta *pMeta, const SMetaEntry *pME); +static void metaUpdateTtl(SMeta *pMeta, const SMetaEntry *pME); +static int metaUpdateChangeTime(SMeta *pMeta, tb_uid_t uid, int64_t changeTimeMs); +static int metaSaveToSkmDb(SMeta *pMeta, const SMetaEntry *pME); +static int metaUpdateCtbIdx(SMeta *pMeta, const SMetaEntry *pME); +static int metaUpdateSuidIdx(SMeta *pMeta, const SMetaEntry *pME); +static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry); +static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type, tb_uid_t *pSuid, int8_t *pSysTbl); +static void metaDestroyTagIdxKey(STagIdxKey *pTagIdxKey); +// opt ins_tables query +static int metaUpdateBtimeIdx(SMeta *pMeta, const SMetaEntry *pME); +static int metaDeleteBtimeIdx(SMeta *pMeta, const SMetaEntry *pME); +static int metaUpdateNcolIdx(SMeta *pMeta, const SMetaEntry *pME); +static int metaDeleteNcolIdx(SMeta *pMeta, const SMetaEntry *pME); + +static int32_t updataTableColCmpr(SColCmprWrapper *pWp, SSchema *pSchema, int8_t add, uint32_t compress) { + int32_t nCols = pWp->nCols; + int32_t ver = pWp->version; + if (add) { + SColCmpr *p = taosMemoryCalloc(1, sizeof(SColCmpr) * (nCols + 1)); + if (p == NULL) { + return terrno; + } + + memcpy(p, pWp->pColCmpr, sizeof(SColCmpr) * nCols); + + SColCmpr *pCol = p + nCols; + pCol->id = pSchema->colId; + pCol->alg = compress; + pWp->nCols = nCols + 1; + pWp->version = ver; + pWp->pColCmpr = p; + } else { + for (int32_t i = 0; i < nCols; i++) { + SColCmpr *pOCmpr = &pWp->pColCmpr[i]; + if (pOCmpr->id == pSchema->colId) { + int32_t left = (nCols - i - 1) * sizeof(SColCmpr); + if (left) { + memmove(pWp->pColCmpr + i, pWp->pColCmpr + i + 1, left); + } + nCols--; + break; + } + } + pWp->nCols = nCols; + pWp->version = ver; + } + return 0; +} +static void metaGetEntryInfo(const SMetaEntry *pEntry, SMetaInfo *pInfo) { + pInfo->uid = pEntry->uid; + pInfo->version = pEntry->version; + if (pEntry->type == TSDB_SUPER_TABLE) { + pInfo->suid = pEntry->uid; + pInfo->skmVer = pEntry->stbEntry.schemaRow.version; + } else if (pEntry->type == TSDB_CHILD_TABLE) { + pInfo->suid = pEntry->ctbEntry.suid; + pInfo->skmVer = 0; + } else if (pEntry->type == TSDB_NORMAL_TABLE) { + pInfo->suid = 0; + pInfo->skmVer = pEntry->ntbEntry.schemaRow.version; + } else { + metaError("meta/table: invalide table type: %" PRId8 " get entry info failed.", pEntry->type); + } +} + +static int metaUpdateMetaRsp(tb_uid_t uid, char *tbName, SSchemaWrapper *pSchema, STableMetaRsp *pMetaRsp) { + pMetaRsp->pSchemas = taosMemoryMalloc(pSchema->nCols * sizeof(SSchema)); + if (NULL == pMetaRsp->pSchemas) { + return terrno; + } + + pMetaRsp->pSchemaExt = taosMemoryMalloc(pSchema->nCols * sizeof(SSchemaExt)); + if (pMetaRsp->pSchemaExt == NULL) { + taosMemoryFree(pMetaRsp->pSchemas); + return terrno; + } + + tstrncpy(pMetaRsp->tbName, tbName, TSDB_TABLE_NAME_LEN); + pMetaRsp->numOfColumns = pSchema->nCols; + pMetaRsp->tableType = TSDB_NORMAL_TABLE; + pMetaRsp->sversion = pSchema->version; + pMetaRsp->tuid = uid; + + memcpy(pMetaRsp->pSchemas, pSchema->pSchema, pSchema->nCols * sizeof(SSchema)); + + return 0; +} + +static int metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSchema *pSchema) { + int32_t code = 0; + +#ifdef USE_INVERTED_INDEX + if (pMeta->pTagIvtIdx == NULL || pCtbEntry == NULL) { + return TSDB_CODE_INVALID_PARA; + } + void *data = pCtbEntry->ctbEntry.pTags; + const char *tagName = pSchema->name; + + tb_uid_t suid = pCtbEntry->ctbEntry.suid; + tb_uid_t tuid = pCtbEntry->uid; + const void *pTagData = pCtbEntry->ctbEntry.pTags; + int32_t nTagData = 0; + + SArray *pTagVals = NULL; + code = tTagToValArray((const STag *)data, &pTagVals); + if (code) { + return code; + } + + SIndexMultiTerm *terms = indexMultiTermCreate(); + if (terms == NULL) { + return terrno; + } + + int16_t nCols = taosArrayGetSize(pTagVals); + for (int i = 0; i < nCols; i++) { + STagVal *pTagVal = (STagVal *)taosArrayGet(pTagVals, i); + char type = pTagVal->type; + + char *key = pTagVal->pKey; + int32_t nKey = strlen(key); + + SIndexTerm *term = NULL; + if (type == TSDB_DATA_TYPE_NULL) { + term = indexTermCreate(suid, ADD_VALUE, TSDB_DATA_TYPE_VARCHAR, key, nKey, NULL, 0); + } else if (type == TSDB_DATA_TYPE_NCHAR) { + if (pTagVal->nData > 0) { + char *val = taosMemoryCalloc(1, pTagVal->nData + VARSTR_HEADER_SIZE); + if (val == NULL) { + TAOS_CHECK_GOTO(terrno, NULL, _exception); + } + int32_t len = taosUcs4ToMbs((TdUcs4 *)pTagVal->pData, pTagVal->nData, val + VARSTR_HEADER_SIZE); + if (len < 0) { + TAOS_CHECK_GOTO(len, NULL, _exception); + } + memcpy(val, (uint16_t *)&len, VARSTR_HEADER_SIZE); + type = TSDB_DATA_TYPE_VARCHAR; + term = indexTermCreate(suid, ADD_VALUE, type, key, nKey, val, len); + taosMemoryFree(val); + } else if (pTagVal->nData == 0) { + term = indexTermCreate(suid, ADD_VALUE, TSDB_DATA_TYPE_VARCHAR, key, nKey, pTagVal->pData, 0); + } + } else if (type == TSDB_DATA_TYPE_DOUBLE) { + double val = *(double *)(&pTagVal->i64); + int len = sizeof(val); + term = indexTermCreate(suid, ADD_VALUE, type, key, nKey, (const char *)&val, len); + } else if (type == TSDB_DATA_TYPE_BOOL) { + int val = *(int *)(&pTagVal->i64); + int len = sizeof(val); + term = indexTermCreate(suid, ADD_VALUE, TSDB_DATA_TYPE_BOOL, key, nKey, (const char *)&val, len); + } + + if (term != NULL) { + int32_t ret = indexMultiTermAdd(terms, term); + if (ret < 0) { + metaError("vgId:%d, failed to add term to multi term, uid: %" PRId64 ", key: %s, type: %d, ret: %d", + TD_VID(pMeta->pVnode), tuid, key, type, ret); + } + } else { + code = terrno; + goto _exception; + } + } + code = indexJsonPut(pMeta->pTagIvtIdx, terms, tuid); + indexMultiTermDestroy(terms); + + taosArrayDestroy(pTagVals); +#endif + return code; +_exception: + indexMultiTermDestroy(terms); + taosArrayDestroy(pTagVals); + return code; +} +int metaDelJsonVarFromIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSchema *pSchema) { +#ifdef USE_INVERTED_INDEX + if (pMeta->pTagIvtIdx == NULL || pCtbEntry == NULL) { + return TSDB_CODE_INVALID_PARA; + } + void *data = pCtbEntry->ctbEntry.pTags; + const char *tagName = pSchema->name; + + tb_uid_t suid = pCtbEntry->ctbEntry.suid; + tb_uid_t tuid = pCtbEntry->uid; + const void *pTagData = pCtbEntry->ctbEntry.pTags; + int32_t nTagData = 0; + + SArray *pTagVals = NULL; + int32_t code = tTagToValArray((const STag *)data, &pTagVals); + if (code) { + return code; + } + + SIndexMultiTerm *terms = indexMultiTermCreate(); + if (terms == NULL) { + return terrno; + } + + int16_t nCols = taosArrayGetSize(pTagVals); + for (int i = 0; i < nCols; i++) { + STagVal *pTagVal = (STagVal *)taosArrayGet(pTagVals, i); + char type = pTagVal->type; + + char *key = pTagVal->pKey; + int32_t nKey = strlen(key); + + SIndexTerm *term = NULL; + if (type == TSDB_DATA_TYPE_NULL) { + term = indexTermCreate(suid, DEL_VALUE, TSDB_DATA_TYPE_VARCHAR, key, nKey, NULL, 0); + } else if (type == TSDB_DATA_TYPE_NCHAR) { + if (pTagVal->nData > 0) { + char *val = taosMemoryCalloc(1, pTagVal->nData + VARSTR_HEADER_SIZE); + if (val == NULL) { + TAOS_CHECK_GOTO(terrno, NULL, _exception); + } + int32_t len = taosUcs4ToMbs((TdUcs4 *)pTagVal->pData, pTagVal->nData, val + VARSTR_HEADER_SIZE); + if (len < 0) { + TAOS_CHECK_GOTO(len, NULL, _exception); + } + memcpy(val, (uint16_t *)&len, VARSTR_HEADER_SIZE); + type = TSDB_DATA_TYPE_VARCHAR; + term = indexTermCreate(suid, DEL_VALUE, type, key, nKey, val, len); + taosMemoryFree(val); + } else if (pTagVal->nData == 0) { + term = indexTermCreate(suid, DEL_VALUE, TSDB_DATA_TYPE_VARCHAR, key, nKey, pTagVal->pData, 0); + } + } else if (type == TSDB_DATA_TYPE_DOUBLE) { + double val = *(double *)(&pTagVal->i64); + int len = sizeof(val); + term = indexTermCreate(suid, DEL_VALUE, type, key, nKey, (const char *)&val, len); + } else if (type == TSDB_DATA_TYPE_BOOL) { + int val = *(int *)(&pTagVal->i64); + int len = sizeof(val); + term = indexTermCreate(suid, DEL_VALUE, TSDB_DATA_TYPE_BOOL, key, nKey, (const char *)&val, len); + } + if (term != NULL) { + int32_t ret = indexMultiTermAdd(terms, term); + if (ret < 0) { + metaError("vgId:%d, failed to add term to multi term, uid: %" PRId64 ", key: %s, type: %d, ret: %d", + TD_VID(pMeta->pVnode), tuid, key, type, ret); + } + } else { + code = terrno; + goto _exception; + } + } + code = indexJsonPut(pMeta->pTagIvtIdx, terms, tuid); + indexMultiTermDestroy(terms); + taosArrayDestroy(pTagVals); +#endif + return code; +_exception: + indexMultiTermDestroy(terms); + taosArrayDestroy(pTagVals); + return code; +} + +static inline void metaTimeSeriesNotifyCheck(SMeta *pMeta) { +#if defined(TD_ENTERPRISE) + int64_t nTimeSeries = metaGetTimeSeriesNum(pMeta, 0); + int64_t deltaTS = nTimeSeries - pMeta->pVnode->config.vndStats.numOfReportedTimeSeries; + if (deltaTS > tsTimeSeriesThreshold) { + if (0 == atomic_val_compare_exchange_8(&dmNotifyHdl.state, 1, 2)) { + if (tsem_post(&dmNotifyHdl.sem) != 0) { + metaError("vgId:%d, failed to post semaphore, errno:%d", TD_VID(pMeta->pVnode), errno); + } + } + } +#endif +} + +int metaCreateSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { + SMetaEntry me = {0}; + int kLen = 0; + int vLen = 0; + const void *pKey = NULL; + const void *pVal = NULL; + void *pBuf = NULL; + int32_t szBuf = 0; + void *p = NULL; + int32_t code = 0; + + // validate req + void *pData = NULL; + int nData = 0; + if (tdbTbGet(pMeta->pNameIdx, pReq->name, strlen(pReq->name) + 1, &pData, &nData) == 0) { + tb_uid_t uid = *(tb_uid_t *)pData; + tdbFree(pData); + SMetaInfo info; + if (metaGetInfo(pMeta, uid, &info, NULL) == TSDB_CODE_NOT_FOUND) { + return terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST; + } + if (info.uid == info.suid) { + return 0; + } else { + return terrno = TSDB_CODE_TDB_TABLE_ALREADY_EXIST; + } + } + + // set structs + me.version = version; + me.type = TSDB_SUPER_TABLE; + me.uid = pReq->suid; + me.name = pReq->name; + me.stbEntry.schemaRow = pReq->schemaRow; + me.stbEntry.schemaTag = pReq->schemaTag; + if (pReq->rollup) { + TABLE_SET_ROLLUP(me.flags); + me.stbEntry.rsmaParam = pReq->rsmaParam; + } + if (pReq->colCmpred) { + TABLE_SET_COL_COMPRESSED(me.flags); + me.colCmpr = pReq->colCmpr; + } + + code = metaHandleEntry(pMeta, &me); + if (code) goto _err; + + ++pMeta->pVnode->config.vndStats.numOfSTables; + + pMeta->changed = true; + metaDebug("vgId:%d, stb:%s is created, suid:%" PRId64, TD_VID(pMeta->pVnode), pReq->name, pReq->suid); + + return 0; + +_err: + metaError("vgId:%d, failed to create stb:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, pReq->suid, + tstrerror(terrno)); + return code; +} + +int metaDropSTable(SMeta *pMeta, int64_t verison, SVDropStbReq *pReq, SArray *tbUidList) { + void *pKey = NULL; + int nKey = 0; + void *pData = NULL; + int nData = 0; + int c = 0; + int rc = 0; + int32_t lino; + int32_t ret; + + // check if super table exists + rc = tdbTbGet(pMeta->pNameIdx, pReq->name, strlen(pReq->name) + 1, &pData, &nData); + if (rc < 0 || *(tb_uid_t *)pData != pReq->suid) { + tdbFree(pData); + return terrno = TSDB_CODE_TDB_STB_NOT_EXIST; + } + + // drop all child tables + TBC *pCtbIdxc = NULL; + + rc = tdbTbcOpen(pMeta->pCtbIdx, &pCtbIdxc, NULL); + if (rc) { + return (terrno = rc); + } + + rc = tdbTbcMoveTo(pCtbIdxc, &(SCtbIdxKey){.suid = pReq->suid, .uid = INT64_MIN}, sizeof(SCtbIdxKey), &c); + if (rc < 0) { + tdbTbcClose(pCtbIdxc); + metaWLock(pMeta); + goto _drop_super_table; + } + + for (;;) { + rc = tdbTbcNext(pCtbIdxc, &pKey, &nKey, NULL, NULL); + if (rc < 0) break; + + if (((SCtbIdxKey *)pKey)->suid < pReq->suid) { + continue; + } else if (((SCtbIdxKey *)pKey)->suid > pReq->suid) { + break; + } + + if (taosArrayPush(tbUidList, &(((SCtbIdxKey *)pKey)->uid)) == NULL) { + tdbFree(pKey); + tdbTbcClose(pCtbIdxc); + return terrno; + } + } + + tdbTbcClose(pCtbIdxc); + + ret = tsdbCacheDropSubTables(pMeta->pVnode->pTsdb, tbUidList, pReq->suid); + if (ret < 0) { + metaError("vgId:%d, failed to drop stb:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, pReq->suid, + tstrerror(terrno)); + } + + metaWLock(pMeta); + + for (int32_t iChild = 0; iChild < taosArrayGetSize(tbUidList); iChild++) { + tb_uid_t uid = *(tb_uid_t *)taosArrayGet(tbUidList, iChild); + ret = metaDropTableByUid(pMeta, uid, NULL, NULL, NULL); + if (ret < 0) { + metaError("vgId:%d, failed to drop stb:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, + pReq->suid, tstrerror(terrno)); + } + } + + // drop super table +_drop_super_table: + tdbTbGet(pMeta->pUidIdx, &pReq->suid, sizeof(tb_uid_t), &pData, &nData); + ret = tdbTbDelete(pMeta->pTbDb, &(STbDbKey){.version = ((SUidIdxVal *)pData)[0].version, .uid = pReq->suid}, + sizeof(STbDbKey), pMeta->txn); + if (ret < 0) { + metaError("vgId:%d, failed to drop stb:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, pReq->suid, + tstrerror(terrno)); + } + + ret = tdbTbDelete(pMeta->pNameIdx, pReq->name, strlen(pReq->name) + 1, pMeta->txn); + if (ret < 0) { + metaError("vgId:%d, failed to drop stb:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, pReq->suid, + tstrerror(terrno)); + } + + ret = tdbTbDelete(pMeta->pUidIdx, &pReq->suid, sizeof(tb_uid_t), pMeta->txn); + if (ret < 0) { + metaError("vgId:%d, failed to drop stb:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, pReq->suid, + tstrerror(terrno)); + } + + ret = tdbTbDelete(pMeta->pSuidIdx, &pReq->suid, sizeof(tb_uid_t), pMeta->txn); + if (ret < 0) { + metaError("vgId:%d, failed to drop stb:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, pReq->suid, + tstrerror(terrno)); + } + + ret = metaStatsCacheDrop(pMeta, pReq->suid); + if (ret < 0) { + metaError("vgId:%d, failed to drop stb:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, pReq->suid, + tstrerror(terrno)); + } + + metaULock(pMeta); + + metaUpdTimeSeriesNum(pMeta); + + pMeta->changed = true; + +_exit: + tdbFree(pKey); + tdbFree(pData); + metaDebug("vgId:%d, super table %s uid:%" PRId64 " is dropped", TD_VID(pMeta->pVnode), pReq->name, pReq->suid); + return 0; +} + +static int32_t metaGetSubtables(SMeta *pMeta, int64_t suid, SArray *uids) { + if (!uids) return TSDB_CODE_INVALID_PARA; + + int c = 0; + void *pKey = NULL; + int nKey = 0; + TBC *pCtbIdxc = NULL; + + TAOS_CHECK_RETURN(tdbTbcOpen(pMeta->pCtbIdx, &pCtbIdxc, NULL)); + int rc = tdbTbcMoveTo(pCtbIdxc, &(SCtbIdxKey){.suid = suid, .uid = INT64_MIN}, sizeof(SCtbIdxKey), &c); + if (rc < 0) { + tdbTbcClose(pCtbIdxc); + metaWLock(pMeta); + return 0; + } + + for (;;) { + rc = tdbTbcNext(pCtbIdxc, &pKey, &nKey, NULL, NULL); + if (rc < 0) break; + + if (((SCtbIdxKey *)pKey)->suid < suid) { + continue; + } else if (((SCtbIdxKey *)pKey)->suid > suid) { + break; + } + + if (taosArrayPush(uids, &(((SCtbIdxKey *)pKey)->uid)) == NULL) { + tdbFree(pKey); + tdbTbcClose(pCtbIdxc); + return terrno; + } + } + + tdbFree(pKey); + + tdbTbcClose(pCtbIdxc); + return 0; +} + +int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { + SMetaEntry oStbEntry = {0}; + SMetaEntry nStbEntry = {0}; + TBC *pUidIdxc = NULL; + TBC *pTbDbc = NULL; + const void *pData; + int nData; + int64_t oversion; + SDecoder dc = {0}; + int32_t ret; + int32_t c = -2; + + TAOS_CHECK_RETURN(tdbTbcOpen(pMeta->pUidIdx, &pUidIdxc, NULL)); + ret = tdbTbcMoveTo(pUidIdxc, &pReq->suid, sizeof(tb_uid_t), &c); + if (ret < 0 || c) { + tdbTbcClose(pUidIdxc); + + return terrno = TSDB_CODE_TDB_STB_NOT_EXIST; + } + + ret = tdbTbcGet(pUidIdxc, NULL, NULL, &pData, &nData); + if (ret < 0) { + tdbTbcClose(pUidIdxc); + + return terrno = TSDB_CODE_TDB_STB_NOT_EXIST; + } + + oversion = ((SUidIdxVal *)pData)[0].version; + + TAOS_CHECK_RETURN(tdbTbcOpen(pMeta->pTbDb, &pTbDbc, NULL)); + ret = tdbTbcMoveTo(pTbDbc, &((STbDbKey){.uid = pReq->suid, .version = oversion}), sizeof(STbDbKey), &c); + if (!(ret == 0 && c == 0)) { + tdbTbcClose(pUidIdxc); + tdbTbcClose(pTbDbc); + + metaError("meta/table: invalide ret: %" PRId32 " or c: %" PRId32 "alter stb failed.", ret, c); + return terrno = TSDB_CODE_TDB_STB_NOT_EXIST; + } + + ret = tdbTbcGet(pTbDbc, NULL, NULL, &pData, &nData); + if (ret < 0) { + tdbTbcClose(pUidIdxc); + tdbTbcClose(pTbDbc); + + return terrno = TSDB_CODE_TDB_STB_NOT_EXIST; + } + + if ((oStbEntry.pBuf = taosMemoryMalloc(nData)) == NULL) { + tdbTbcClose(pTbDbc); + tdbTbcClose(pUidIdxc); + return terrno; + } + memcpy(oStbEntry.pBuf, pData, nData); + tDecoderInit(&dc, oStbEntry.pBuf, nData); + ret = metaDecodeEntry(&dc, &oStbEntry); + if (ret < 0) { + metaError("vgId:%d, failed to decode stb:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, + pReq->suid, tstrerror(ret)); + tDecoderClear(&dc); + tdbTbcClose(pTbDbc); + tdbTbcClose(pUidIdxc); + return terrno; + } + + nStbEntry.version = version; + nStbEntry.type = TSDB_SUPER_TABLE; + nStbEntry.uid = pReq->suid; + nStbEntry.name = pReq->name; + nStbEntry.stbEntry.schemaRow = pReq->schemaRow; + nStbEntry.stbEntry.schemaTag = pReq->schemaTag; + nStbEntry.colCmpr = pReq->colCmpr; + TABLE_SET_COL_COMPRESSED(nStbEntry.flags); + + int nCols = pReq->schemaRow.nCols; + int onCols = oStbEntry.stbEntry.schemaRow.nCols; + int32_t deltaCol = nCols - onCols; + bool updStat = deltaCol != 0 && !metaTbInFilterCache(pMeta, pReq->name, 1); + + if (!TSDB_CACHE_NO(pMeta->pVnode->config)) { + STsdb *pTsdb = pMeta->pVnode->pTsdb; + SArray *uids = taosArrayInit(8, sizeof(int64_t)); + if (uids == NULL) { + if (oStbEntry.pBuf) taosMemoryFree(oStbEntry.pBuf); + tDecoderClear(&dc); + tdbTbcClose(pTbDbc); + tdbTbcClose(pUidIdxc); + return terrno; + } + if (deltaCol == 1) { + int16_t cid = pReq->schemaRow.pSchema[nCols - 1].colId; + int8_t col_type = pReq->schemaRow.pSchema[nCols - 1].type; + + TAOS_CHECK_RETURN(metaGetSubtables(pMeta, pReq->suid, uids)); + TAOS_CHECK_RETURN(tsdbCacheNewSTableColumn(pTsdb, uids, cid, col_type)); + } else if (deltaCol == -1) { + int16_t cid = -1; + bool hasPrimaryKey = false; + if (onCols >= 2) { + hasPrimaryKey = (oStbEntry.stbEntry.schemaRow.pSchema[1].flags & COL_IS_KEY) ? true : false; + } + for (int i = 0, j = 0; i < nCols && j < onCols; ++i, ++j) { + if (pReq->schemaRow.pSchema[i].colId != oStbEntry.stbEntry.schemaRow.pSchema[j].colId) { + cid = oStbEntry.stbEntry.schemaRow.pSchema[j].colId; + break; + } + } + + if (cid != -1) { + TAOS_CHECK_RETURN(metaGetSubtables(pMeta, pReq->suid, uids)); + TAOS_CHECK_RETURN(tsdbCacheDropSTableColumn(pTsdb, uids, cid, hasPrimaryKey)); + } + } + if (uids) taosArrayDestroy(uids); + + tsdbCacheInvalidateSchema(pTsdb, pReq->suid, -1, pReq->schemaRow.version); + } + + metaWLock(pMeta); + // compare two entry + if (oStbEntry.stbEntry.schemaRow.version != pReq->schemaRow.version) { + ret = metaSaveToSkmDb(pMeta, &nStbEntry); + if (ret < 0) { + metaError("vgId:%d, failed to save skm db:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, + pReq->suid, tstrerror(ret)); + } + } + + // update table.db + ret = metaSaveToTbDb(pMeta, &nStbEntry); + if (ret < 0) { + metaError("vgId:%d, failed to save tb db:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, + pReq->suid, tstrerror(ret)); + } + + // update uid index + ret = metaUpdateUidIdx(pMeta, &nStbEntry); + if (ret < 0) { + metaError("vgId:%d, failed to update uid idx:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, + pReq->suid, tstrerror(ret)); + } + + // metaStatsCacheDrop(pMeta, nStbEntry.uid); + + if (updStat) { + metaUpdateStbStats(pMeta, pReq->suid, 0, deltaCol); + } + metaULock(pMeta); + + if (updStat) { + int64_t ctbNum; + ret = metaGetStbStats(pMeta->pVnode, pReq->suid, &ctbNum, NULL); + if (ret < 0) { + metaError("vgId:%d, failed to get stb stats:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, + pReq->suid, tstrerror(ret)); + } + pMeta->pVnode->config.vndStats.numOfTimeSeries += (ctbNum * deltaCol); + metaTimeSeriesNotifyCheck(pMeta); + } + + pMeta->changed = true; + +_exit: + if (oStbEntry.pBuf) taosMemoryFree(oStbEntry.pBuf); + tDecoderClear(&dc); + tdbTbcClose(pTbDbc); + tdbTbcClose(pUidIdxc); + return 0; +} +int metaAddIndexToSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { + SMetaEntry oStbEntry = {0}; + SMetaEntry nStbEntry = {0}; + STbDbKey tbDbKey = {0}; + + TBC *pUidIdxc = NULL; + TBC *pTbDbc = NULL; + void *pData = NULL; + int nData = 0; + int64_t oversion; + SDecoder dc = {0}; + int32_t ret; + int32_t c = -2; + tb_uid_t suid = pReq->suid; + int32_t code = 0; + + // get super table + if ((code = tdbTbGet(pMeta->pUidIdx, &suid, sizeof(tb_uid_t), &pData, &nData)) != 0) { + goto _err; + } + + tbDbKey.uid = suid; + tbDbKey.version = ((SUidIdxVal *)pData)[0].version; + if ((code = tdbTbGet(pMeta->pTbDb, &tbDbKey, sizeof(tbDbKey), &pData, &nData)) != 0) { + goto _err; + } + + tDecoderInit(&dc, pData, nData); + code = metaDecodeEntry(&dc, &oStbEntry); + if (code < 0) { + goto _err; + } + + if (oStbEntry.stbEntry.schemaTag.pSchema == NULL || oStbEntry.stbEntry.schemaTag.pSchema == NULL) { + code = TSDB_CODE_INVALID_PARA; + goto _err; + } + + if (oStbEntry.stbEntry.schemaTag.version == pReq->schemaTag.version) { + code = TSDB_CODE_INVALID_PARA; + goto _err; + } + + if (oStbEntry.stbEntry.schemaTag.nCols != pReq->schemaTag.nCols) { + code = TSDB_CODE_INVALID_PARA; + goto _err; + } + + int diffIdx = -1; + for (int i = 0; i < pReq->schemaTag.nCols; i++) { + SSchema *pNew = pReq->schemaTag.pSchema + i; + SSchema *pOld = oStbEntry.stbEntry.schemaTag.pSchema + i; + if (pNew->type != pOld->type || pNew->colId != pOld->colId || pNew->bytes != pOld->bytes || + strncmp(pOld->name, pNew->name, sizeof(pNew->name))) { + code = TSDB_CODE_INVALID_PARA; + goto _err; + } + if (IS_IDX_ON(pNew) && !IS_IDX_ON(pOld)) { + // if (diffIdx != -1) goto _err; + diffIdx = i; + break; + } + } + + if (diffIdx == -1) { + code = TSDB_CODE_INVALID_PARA; + goto _err; + } + + // Get target schema info + SSchemaWrapper *pTagSchema = &pReq->schemaTag; + if (pTagSchema->nCols == 1 && pTagSchema->pSchema[0].type == TSDB_DATA_TYPE_JSON) { + code = TSDB_CODE_INVALID_PARA; + goto _err; + } + SSchema *pCol = pTagSchema->pSchema + diffIdx; + + /* + * iterator all pTdDbc by uid and version + */ + TBC *pCtbIdxc = NULL; + code = tdbTbcOpen(pMeta->pCtbIdx, &pCtbIdxc, NULL); + if (code != 0) { + goto _err; + } + + code = tdbTbcMoveTo(pCtbIdxc, &(SCtbIdxKey){.suid = suid, .uid = INT64_MIN}, sizeof(SCtbIdxKey), &c); + if (code < 0) { + tdbTbcClose(pCtbIdxc); + goto _err; + } + for (;;) { + void *pKey = NULL, *pVal = NULL; + int nKey = 0, nVal = 0; + code = tdbTbcNext(pCtbIdxc, &pKey, &nKey, &pVal, &nVal); + if (code < 0) { + tdbFree(pKey); + tdbFree(pVal); + tdbTbcClose(pCtbIdxc); + pCtbIdxc = NULL; + break; + } + if (((SCtbIdxKey *)pKey)->suid != suid) { + tdbFree(pKey); + tdbFree(pVal); + continue; + } + STagIdxKey *pTagIdxKey = NULL; + int32_t nTagIdxKey; + + const void *pTagData = NULL; + int32_t nTagData = 0; + + SCtbIdxKey *table = (SCtbIdxKey *)pKey; + STagVal tagVal = {.cid = pCol->colId}; + if (tTagGet((const STag *)pVal, &tagVal)) { + if (IS_VAR_DATA_TYPE(pCol->type)) { + pTagData = tagVal.pData; + nTagData = (int32_t)tagVal.nData; + } else { + pTagData = &(tagVal.i64); + nTagData = tDataTypes[pCol->type].bytes; + } + } else { + if (!IS_VAR_DATA_TYPE(pCol->type)) { + nTagData = tDataTypes[pCol->type].bytes; + } + } + code = metaCreateTagIdxKey(suid, pCol->colId, pTagData, nTagData, pCol->type, table->uid, &pTagIdxKey, &nTagIdxKey); + tdbFree(pKey); + tdbFree(pVal); + if (code < 0) { + metaDestroyTagIdxKey(pTagIdxKey); + tdbTbcClose(pCtbIdxc); + goto _err; + } + + metaWLock(pMeta); + ret = tdbTbUpsert(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, NULL, 0, pMeta->txn); + if (ret < 0) { + metaError("vgId:%d, failed to upsert tag idx key:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, + pReq->suid, tstrerror(ret)); + } + metaULock(pMeta); + + metaDestroyTagIdxKey(pTagIdxKey); + pTagIdxKey = NULL; + } + + nStbEntry.version = version; + nStbEntry.type = TSDB_SUPER_TABLE; + nStbEntry.uid = pReq->suid; + nStbEntry.name = pReq->name; + nStbEntry.stbEntry.schemaRow = pReq->schemaRow; + nStbEntry.stbEntry.schemaTag = pReq->schemaTag; + nStbEntry.colCmpr = pReq->colCmpr; + + metaWLock(pMeta); + // update table.db + ret = metaSaveToTbDb(pMeta, &nStbEntry); + if (ret < 0) { + metaError("vgId:%d, failed to save tb db:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, + pReq->suid, tstrerror(ret)); + } + // update uid index + ret = metaUpdateUidIdx(pMeta, &nStbEntry); + if (ret < 0) { + metaError("vgId:%d, failed to update uid idx:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, + pReq->suid, tstrerror(ret)); + } + metaULock(pMeta); + + if (oStbEntry.pBuf) taosMemoryFree(oStbEntry.pBuf); + tDecoderClear(&dc); + tdbFree(pData); + + tdbTbcClose(pCtbIdxc); + return TSDB_CODE_SUCCESS; +_err: + if (oStbEntry.pBuf) taosMemoryFree(oStbEntry.pBuf); + tDecoderClear(&dc); + tdbFree(pData); + + return code; +} +int metaDropIndexFromSTable(SMeta *pMeta, int64_t version, SDropIndexReq *pReq) { + int32_t code = 0; + SMetaEntry oStbEntry = {0}; + SMetaEntry nStbEntry = {0}; + + STbDbKey tbDbKey = {0}; + TBC *pUidIdxc = NULL; + TBC *pTbDbc = NULL; + int ret = 0; + int c = -2; + void *pData = NULL; + int nData = 0; + int64_t oversion; + SDecoder dc = {0}; + + tb_uid_t suid = pReq->stbUid; + + if ((code = tdbTbGet(pMeta->pUidIdx, &suid, sizeof(tb_uid_t), &pData, &nData)) != 0) { + goto _err; + } + + tbDbKey.uid = suid; + tbDbKey.version = ((SUidIdxVal *)pData)[0].version; + if ((code = tdbTbGet(pMeta->pTbDb, &tbDbKey, sizeof(tbDbKey), &pData, &nData)) != 0) { + goto _err; + } + + tDecoderInit(&dc, pData, nData); + code = metaDecodeEntry(&dc, &oStbEntry); + if (code != 0) { + goto _err; + } + + SSchema *pCol = NULL; + int32_t colId = -1; + for (int i = 0; i < oStbEntry.stbEntry.schemaTag.nCols; i++) { + SSchema *schema = oStbEntry.stbEntry.schemaTag.pSchema + i; + if (0 == strncmp(schema->name, pReq->colName, sizeof(pReq->colName))) { + if (IS_IDX_ON(schema)) { + pCol = schema; + } + break; + } + } + + if (pCol == NULL) { + metaError("vgId:%d, failed to drop index on %s.%s,since %s", TD_VID(pMeta->pVnode), pReq->stb, pReq->colName, + tstrerror(TSDB_CODE_VND_COL_NOT_EXISTS)); + code = 0; + + goto _err; + } + + /* + * iterator all pTdDbc by uid and version + */ + TBC *pCtbIdxc = NULL; + code = tdbTbcOpen(pMeta->pCtbIdx, &pCtbIdxc, NULL); + if (code != 0) { + goto _err; + } + + code = tdbTbcMoveTo(pCtbIdxc, &(SCtbIdxKey){.suid = suid, .uid = INT64_MIN}, sizeof(SCtbIdxKey), &c); + if (code < 0) { + tdbTbcClose(pCtbIdxc); + goto _err; + } + for (;;) { + void *pKey = NULL, *pVal = NULL; + int nKey = 0, nVal = 0; + + code = tdbTbcNext(pCtbIdxc, &pKey, &nKey, &pVal, &nVal); + if (code < 0) { + tdbFree(pKey); + tdbFree(pVal); + tdbTbcClose(pCtbIdxc); + pCtbIdxc = NULL; + break; + } + if (((SCtbIdxKey *)pKey)->suid != suid) { + tdbFree(pKey); + tdbFree(pVal); + continue; + } + STagIdxKey *pTagIdxKey = NULL; + int32_t nTagIdxKey; + + const void *pTagData = NULL; + int32_t nTagData = 0; + + SCtbIdxKey *table = (SCtbIdxKey *)pKey; + STagVal tagVal = {.cid = pCol->colId}; + if (tTagGet((const STag *)pVal, &tagVal)) { + if (IS_VAR_DATA_TYPE(pCol->type)) { + pTagData = tagVal.pData; + nTagData = (int32_t)tagVal.nData; + } else { + pTagData = &(tagVal.i64); + nTagData = tDataTypes[pCol->type].bytes; + } + } else { + if (!IS_VAR_DATA_TYPE(pCol->type)) { + nTagData = tDataTypes[pCol->type].bytes; + } + } + + code = metaCreateTagIdxKey(suid, pCol->colId, pTagData, nTagData, pCol->type, table->uid, &pTagIdxKey, &nTagIdxKey); + tdbFree(pKey); + tdbFree(pVal); + if (code < 0) { + metaDestroyTagIdxKey(pTagIdxKey); + tdbTbcClose(pCtbIdxc); + goto _err; + } + + metaWLock(pMeta); + ret = tdbTbDelete(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, pMeta->txn); + if (ret < 0) { + metaError("vgId:%d, failed to delete tag idx key:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->stb, + pReq->stbUid, tstrerror(ret)); + } + metaULock(pMeta); + metaDestroyTagIdxKey(pTagIdxKey); + pTagIdxKey = NULL; + } + + // clear idx flag + SSCHMEA_SET_IDX_OFF(pCol); + + nStbEntry.version = version; + nStbEntry.type = TSDB_SUPER_TABLE; + nStbEntry.uid = oStbEntry.uid; + nStbEntry.name = oStbEntry.name; + + SSchemaWrapper *row = tCloneSSchemaWrapper(&oStbEntry.stbEntry.schemaRow); + SSchemaWrapper *tag = tCloneSSchemaWrapper(&oStbEntry.stbEntry.schemaTag); + SColCmprWrapper *cmpr = tCloneSColCmprWrapper(&oStbEntry.colCmpr); + if (row == NULL || tag == NULL || cmpr == NULL) { + tDeleteSchemaWrapper(row); + tDeleteSchemaWrapper(tag); + tDeleteSColCmprWrapper(cmpr); + code = TSDB_CODE_OUT_OF_MEMORY; + + tdbTbcClose(pCtbIdxc); + goto _err; + } + + nStbEntry.stbEntry.schemaRow = *row; + nStbEntry.stbEntry.schemaTag = *tag; + nStbEntry.stbEntry.rsmaParam = oStbEntry.stbEntry.rsmaParam; + nStbEntry.colCmpr = *cmpr; + + nStbEntry.colCmpr = oStbEntry.colCmpr; + + metaWLock(pMeta); + // update table.db + ret = metaSaveToTbDb(pMeta, &nStbEntry); + if (ret < 0) { + metaError("vgId:%d, failed to save tb db:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->stb, + pReq->stbUid, tstrerror(ret)); + } + // update uid index + ret = metaUpdateUidIdx(pMeta, &nStbEntry); + if (ret < 0) { + metaError("vgId:%d, failed to update uid idx:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->stb, + pReq->stbUid, tstrerror(ret)); + } + metaULock(pMeta); + + tDeleteSchemaWrapper(tag); + tDeleteSchemaWrapper(row); + tDeleteSColCmprWrapper(cmpr); + + if (oStbEntry.pBuf) taosMemoryFree(oStbEntry.pBuf); + tDecoderClear(&dc); + tdbFree(pData); + + tdbTbcClose(pCtbIdxc); + return TSDB_CODE_SUCCESS; +_err: + if (oStbEntry.pBuf) taosMemoryFree(oStbEntry.pBuf); + tDecoderClear(&dc); + tdbFree(pData); + + return code; +} + +int metaCreateTable(SMeta *pMeta, int64_t ver, SVCreateTbReq *pReq, STableMetaRsp **pMetaRsp) { + SMetaEntry me = {0}; + SMetaReader mr = {0}; + int32_t ret; + + // validate message + if (pReq->type != TSDB_CHILD_TABLE && pReq->type != TSDB_NORMAL_TABLE) { + terrno = TSDB_CODE_INVALID_MSG; + goto _err; + } + + if (pReq->type == TSDB_CHILD_TABLE) { + tb_uid_t suid = metaGetTableEntryUidByName(pMeta, pReq->ctb.stbName); + if (suid != pReq->ctb.suid) { + return terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST; + } + } + + // validate req + metaReaderDoInit(&mr, pMeta, META_READER_LOCK); + if (metaGetTableEntryByName(&mr, pReq->name) == 0) { + if (pReq->type == TSDB_CHILD_TABLE && pReq->ctb.suid != mr.me.ctbEntry.suid) { + metaReaderClear(&mr); + return terrno = TSDB_CODE_TDB_TABLE_IN_OTHER_STABLE; + } + pReq->uid = mr.me.uid; + if (pReq->type == TSDB_CHILD_TABLE) { + pReq->ctb.suid = mr.me.ctbEntry.suid; + } + metaReaderClear(&mr); + return terrno = TSDB_CODE_TDB_TABLE_ALREADY_EXIST; + } else if (terrno == TSDB_CODE_PAR_TABLE_NOT_EXIST) { + terrno = TSDB_CODE_SUCCESS; + } + metaReaderClear(&mr); + + bool sysTbl = (pReq->type == TSDB_CHILD_TABLE) && metaTbInFilterCache(pMeta, pReq->ctb.stbName, 1); + + if (!sysTbl && ((terrno = grantCheck(TSDB_GRANT_TIMESERIES)) < 0)) goto _err; + + // build SMetaEntry + SVnodeStats *pStats = &pMeta->pVnode->config.vndStats; + me.version = ver; + me.type = pReq->type; + me.uid = pReq->uid; + me.name = pReq->name; + if (me.type == TSDB_CHILD_TABLE) { + me.ctbEntry.btime = pReq->btime; + me.ctbEntry.ttlDays = pReq->ttl; + me.ctbEntry.commentLen = pReq->commentLen; + me.ctbEntry.comment = pReq->comment; + me.ctbEntry.suid = pReq->ctb.suid; + me.ctbEntry.pTags = pReq->ctb.pTag; + +#ifdef TAG_FILTER_DEBUG + SArray *pTagVals = NULL; + int32_t code = tTagToValArray((STag *)pReq->ctb.pTag, &pTagVals); + for (int i = 0; i < taosArrayGetSize(pTagVals); i++) { + STagVal *pTagVal = (STagVal *)taosArrayGet(pTagVals, i); + + if (IS_VAR_DATA_TYPE(pTagVal->type)) { + char *buf = taosMemoryCalloc(pTagVal->nData + 1, 1); + memcpy(buf, pTagVal->pData, pTagVal->nData); + metaDebug("metaTag table:%s varchar index:%d cid:%d type:%d value:%s", pReq->name, i, pTagVal->cid, + pTagVal->type, buf); + taosMemoryFree(buf); + } else { + double val = 0; + GET_TYPED_DATA(val, double, pTagVal->type, &pTagVal->i64); + metaDebug("metaTag table:%s number index:%d cid:%d type:%d value:%f", pReq->name, i, pTagVal->cid, + pTagVal->type, val); + } + } +#endif + + ++pStats->numOfCTables; + + if (!sysTbl) { + int32_t nCols = 0; + ret = metaGetStbStats(pMeta->pVnode, me.ctbEntry.suid, 0, &nCols); + if (ret < 0) { + metaError("vgId:%d, failed to get stb stats:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, + pReq->ctb.suid, tstrerror(ret)); + } + pStats->numOfTimeSeries += nCols - 1; + } + + metaWLock(pMeta); + metaUpdateStbStats(pMeta, me.ctbEntry.suid, 1, 0); + ret = metaUidCacheClear(pMeta, me.ctbEntry.suid); + if (ret < 0) { + metaError("vgId:%d, failed to clear uid cache:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, + pReq->ctb.suid, tstrerror(ret)); + } + ret = metaTbGroupCacheClear(pMeta, me.ctbEntry.suid); + if (ret < 0) { + metaError("vgId:%d, failed to clear group cache:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, + pReq->ctb.suid, tstrerror(ret)); + } + metaULock(pMeta); + + if (!TSDB_CACHE_NO(pMeta->pVnode->config)) { + ret = tsdbCacheNewTable(pMeta->pVnode->pTsdb, me.uid, me.ctbEntry.suid, NULL); + if (ret < 0) { + metaError("vgId:%d, failed to create table:%s since %s", TD_VID(pMeta->pVnode), pReq->name, tstrerror(ret)); + goto _err; + } + } + } else { + me.ntbEntry.btime = pReq->btime; + me.ntbEntry.ttlDays = pReq->ttl; + me.ntbEntry.commentLen = pReq->commentLen; + me.ntbEntry.comment = pReq->comment; + me.ntbEntry.schemaRow = pReq->ntb.schemaRow; + me.ntbEntry.ncid = me.ntbEntry.schemaRow.pSchema[me.ntbEntry.schemaRow.nCols - 1].colId + 1; + me.colCmpr = pReq->colCmpr; + TABLE_SET_COL_COMPRESSED(me.flags); + + ++pStats->numOfNTables; + pStats->numOfNTimeSeries += me.ntbEntry.schemaRow.nCols - 1; + + if (!TSDB_CACHE_NO(pMeta->pVnode->config)) { + ret = tsdbCacheNewTable(pMeta->pVnode->pTsdb, me.uid, -1, &me.ntbEntry.schemaRow); + if (ret < 0) { + metaError("vgId:%d, failed to create table:%s since %s", TD_VID(pMeta->pVnode), pReq->name, tstrerror(ret)); + goto _err; + } + } + } + + if (metaHandleEntry(pMeta, &me) < 0) goto _err; + + metaTimeSeriesNotifyCheck(pMeta); + + if (pMetaRsp) { + *pMetaRsp = taosMemoryCalloc(1, sizeof(STableMetaRsp)); + + if (*pMetaRsp) { + if (me.type == TSDB_CHILD_TABLE) { + (*pMetaRsp)->tableType = TSDB_CHILD_TABLE; + (*pMetaRsp)->tuid = pReq->uid; + (*pMetaRsp)->suid = pReq->ctb.suid; + strcpy((*pMetaRsp)->tbName, pReq->name); + } else { + ret = metaUpdateMetaRsp(pReq->uid, pReq->name, &pReq->ntb.schemaRow, *pMetaRsp); + if (ret < 0) { + metaError("vgId:%d, failed to update meta rsp:%s since %s", TD_VID(pMeta->pVnode), pReq->name, + tstrerror(ret)); + } + for (int32_t i = 0; i < pReq->colCmpr.nCols; i++) { + SColCmpr *p = &pReq->colCmpr.pColCmpr[i]; + (*pMetaRsp)->pSchemaExt[i].colId = p->id; + (*pMetaRsp)->pSchemaExt[i].compress = p->alg; + } + } + } + } + + pMeta->changed = true; + metaDebug("vgId:%d, table:%s uid %" PRId64 " is created, type:%" PRId8, TD_VID(pMeta->pVnode), pReq->name, pReq->uid, + pReq->type); + return 0; + +_err: + metaError("vgId:%d, failed to create table:%s type:%s since %s", TD_VID(pMeta->pVnode), pReq->name, + pReq->type == TSDB_CHILD_TABLE ? "child table" : "normal table", tstrerror(terrno)); + return TSDB_CODE_FAILED; +} + +int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq, SArray *tbUids, tb_uid_t *tbUid) { + void *pData = NULL; + int nData = 0; + int rc = 0; + tb_uid_t uid = 0; + tb_uid_t suid = 0; + int8_t sysTbl = 0; + int type; + + rc = tdbTbGet(pMeta->pNameIdx, pReq->name, strlen(pReq->name) + 1, &pData, &nData); + if (rc < 0) { + return terrno = TSDB_CODE_TDB_TABLE_NOT_EXIST; + } + uid = *(tb_uid_t *)pData; + + metaWLock(pMeta); + rc = metaDropTableByUid(pMeta, uid, &type, &suid, &sysTbl); + metaULock(pMeta); + + if (rc < 0) goto _exit; + + if (!sysTbl && type == TSDB_CHILD_TABLE) { + int32_t nCols = 0; + SVnodeStats *pStats = &pMeta->pVnode->config.vndStats; + if (metaGetStbStats(pMeta->pVnode, suid, NULL, &nCols) == 0) { + pStats->numOfTimeSeries -= nCols - 1; + } + } + + if ((type == TSDB_CHILD_TABLE || type == TSDB_NORMAL_TABLE) && tbUids) { + if (taosArrayPush(tbUids, &uid) == NULL) { + rc = terrno; + goto _exit; + } + + if (!TSDB_CACHE_NO(pMeta->pVnode->config)) { + int32_t ret = tsdbCacheDropTable(pMeta->pVnode->pTsdb, uid, suid, NULL); + if (ret < 0) { + metaError("vgId:%d, failed to drop table:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, uid, + tstrerror(ret)); + } + } + } + + if ((type == TSDB_CHILD_TABLE) && tbUid) { + *tbUid = uid; + } + + pMeta->changed = true; +_exit: + tdbFree(pData); + return rc; +} + +int32_t metaDropTables(SMeta *pMeta, SArray *tbUids) { + int32_t code = 0; + if (taosArrayGetSize(tbUids) == 0) return TSDB_CODE_SUCCESS; + + int64_t nCtbDropped = 0; + SSHashObj *suidHash = tSimpleHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT)); + if (suidHash == NULL) { + return terrno; + } + + metaWLock(pMeta); + for (int i = 0; i < taosArrayGetSize(tbUids); ++i) { + tb_uid_t uid = *(tb_uid_t *)taosArrayGet(tbUids, i); + tb_uid_t suid = 0; + int8_t sysTbl = 0; + int type; + code = metaDropTableByUid(pMeta, uid, &type, &suid, &sysTbl); + if (code) return code; + if (!sysTbl && type == TSDB_CHILD_TABLE && suid != 0 && suidHash) { + int64_t *pVal = tSimpleHashGet(suidHash, &suid, sizeof(tb_uid_t)); + if (pVal) { + nCtbDropped = *pVal + 1; + } else { + nCtbDropped = 1; + } + code = tSimpleHashPut(suidHash, &suid, sizeof(tb_uid_t), &nCtbDropped, sizeof(int64_t)); + if (code) return code; + } + /* + if (!TSDB_CACHE_NO(pMeta->pVnode->config)) { + tsdbCacheDropTable(pMeta->pVnode->pTsdb, uid, suid, NULL); + } + */ + metaDebug("batch drop table:%" PRId64, uid); + } + metaULock(pMeta); + + // update timeseries + void *pCtbDropped = NULL; + int32_t iter = 0; + while ((pCtbDropped = tSimpleHashIterate(suidHash, pCtbDropped, &iter))) { + tb_uid_t *pSuid = tSimpleHashGetKey(pCtbDropped, NULL); + int32_t nCols = 0; + SVnodeStats *pStats = &pMeta->pVnode->config.vndStats; + if (metaGetStbStats(pMeta->pVnode, *pSuid, NULL, &nCols) == 0) { + pStats->numOfTimeSeries -= *(int64_t *)pCtbDropped * (nCols - 1); + } + } + tSimpleHashCleanup(suidHash); + + pMeta->changed = true; + return 0; +} + +static int32_t metaFilterTableByHash(SMeta *pMeta, SArray *uidList) { + int32_t code = 0; + // 1, tranverse table's + // 2, validate table name using vnodeValidateTableHash + // 3, push invalidated table's uid into uidList + + TBC *pCur; + code = tdbTbcOpen(pMeta->pTbDb, &pCur, NULL); + if (code < 0) { + return code; + } + + code = tdbTbcMoveToFirst(pCur); + if (code) { + tdbTbcClose(pCur); + return code; + } + + void *pData = NULL, *pKey = NULL; + int nData = 0, nKey = 0; + + while (1) { + int32_t ret = tdbTbcNext(pCur, &pKey, &nKey, &pData, &nData); + if (ret < 0) { + break; + } + + SMetaEntry me = {0}; + SDecoder dc = {0}; + tDecoderInit(&dc, pData, nData); + code = metaDecodeEntry(&dc, &me); + if (code < 0) { + tDecoderClear(&dc); + return code; + } + + if (me.type != TSDB_SUPER_TABLE) { + char tbFName[TSDB_TABLE_FNAME_LEN + 1]; + snprintf(tbFName, sizeof(tbFName), "%s.%s", pMeta->pVnode->config.dbname, me.name); + tbFName[TSDB_TABLE_FNAME_LEN] = '\0'; + int32_t ret = vnodeValidateTableHash(pMeta->pVnode, tbFName); + if (ret < 0 && terrno == TSDB_CODE_VND_HASH_MISMATCH) { + if (taosArrayPush(uidList, &me.uid) == NULL) { + code = terrno; + break; + } + } + } + tDecoderClear(&dc); + } + tdbFree(pData); + tdbFree(pKey); + tdbTbcClose(pCur); + + return 0; +} + +int32_t metaTrimTables(SMeta *pMeta) { + int32_t code = 0; + + SArray *tbUids = taosArrayInit(8, sizeof(int64_t)); + if (tbUids == NULL) { + return terrno; + } + + code = metaFilterTableByHash(pMeta, tbUids); + if (code != 0) { + goto end; + } + if (TARRAY_SIZE(tbUids) == 0) { + goto end; + } + + metaInfo("vgId:%d, trim %ld tables", TD_VID(pMeta->pVnode), taosArrayGetSize(tbUids)); + code = metaDropTables(pMeta, tbUids); + if (code) goto end; + +end: + taosArrayDestroy(tbUids); + + return code; +} + +int metaTtlFindExpired(SMeta *pMeta, int64_t timePointMs, SArray *tbUids, int32_t ttlDropMaxCount) { + metaRLock(pMeta); + + int ret = ttlMgrFindExpired(pMeta->pTtlMgr, timePointMs, tbUids, ttlDropMaxCount); + + metaULock(pMeta); + + if (ret != 0) { + metaError("ttl failed to find expired table, ret:%d", ret); + } + + return ret; +} + +static int metaBuildBtimeIdxKey(SBtimeIdxKey *btimeKey, const SMetaEntry *pME) { + int64_t btime; + if (pME->type == TSDB_CHILD_TABLE) { + btime = pME->ctbEntry.btime; + } else if (pME->type == TSDB_NORMAL_TABLE) { + btime = pME->ntbEntry.btime; + } else { + return TSDB_CODE_FAILED; + } + + btimeKey->btime = btime; + btimeKey->uid = pME->uid; + return 0; +} + +static int metaBuildNColIdxKey(SNcolIdxKey *ncolKey, const SMetaEntry *pME) { + if (pME->type == TSDB_NORMAL_TABLE) { + ncolKey->ncol = pME->ntbEntry.schemaRow.nCols; + ncolKey->uid = pME->uid; + } else { + return TSDB_CODE_FAILED; + } + return 0; +} + +static void metaDeleteTtl(SMeta *pMeta, const SMetaEntry *pME) { + if (pME->type != TSDB_CHILD_TABLE && pME->type != TSDB_NORMAL_TABLE) return; + + STtlDelTtlCtx ctx = {.uid = pME->uid, .pTxn = pMeta->txn}; + if (pME->type == TSDB_CHILD_TABLE) { + ctx.ttlDays = pME->ctbEntry.ttlDays; + } else { + ctx.ttlDays = pME->ntbEntry.ttlDays; + } + + int32_t ret = ttlMgrDeleteTtl(pMeta->pTtlMgr, &ctx); + if (ret < 0) { + metaError("vgId:%d, failed to delete ttl for table:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pME->name, + pME->uid, tstrerror(ret)); + } + return; +} + +static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type, tb_uid_t *pSuid, int8_t *pSysTbl) { + void *pData = NULL; + int nData = 0; + int rc = 0; + SMetaEntry e = {0}; + SDecoder dc = {0}; + int32_t ret = 0; + + rc = tdbTbGet(pMeta->pUidIdx, &uid, sizeof(uid), &pData, &nData); + if (rc < 0) { + return rc; + } + int64_t version = ((SUidIdxVal *)pData)[0].version; + + rc = tdbTbGet(pMeta->pTbDb, &(STbDbKey){.version = version, .uid = uid}, sizeof(STbDbKey), &pData, &nData); + if (rc < 0) { + tdbFree(pData); + return rc; + } + + tDecoderInit(&dc, pData, nData); + rc = metaDecodeEntry(&dc, &e); + if (rc < 0) { + tDecoderClear(&dc); + return rc; + } + + if (type) *type = e.type; + + if (e.type == TSDB_CHILD_TABLE) { + if (pSuid) *pSuid = e.ctbEntry.suid; + void *tData = NULL; + int tLen = 0; + + if (tdbTbGet(pMeta->pUidIdx, &e.ctbEntry.suid, sizeof(tb_uid_t), &tData, &tLen) == 0) { + STbDbKey tbDbKey = {.uid = e.ctbEntry.suid, .version = ((SUidIdxVal *)tData)[0].version}; + if (tdbTbGet(pMeta->pTbDb, &tbDbKey, sizeof(tbDbKey), &tData, &tLen) == 0) { + SDecoder tdc = {0}; + SMetaEntry stbEntry = {0}; + + tDecoderInit(&tdc, tData, tLen); + int32_t ret = metaDecodeEntry(&tdc, &stbEntry); + if (ret < 0) { + tDecoderClear(&tdc); + metaError("vgId:%d, failed to decode child table:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), e.name, + e.ctbEntry.suid, tstrerror(ret)); + return ret; + } + + if (pSysTbl) *pSysTbl = metaTbInFilterCache(pMeta, stbEntry.name, 1) ? 1 : 0; + + SSchema *pTagColumn = NULL; + SSchemaWrapper *pTagSchema = &stbEntry.stbEntry.schemaTag; + if (pTagSchema->nCols == 1 && pTagSchema->pSchema[0].type == TSDB_DATA_TYPE_JSON) { + pTagColumn = &stbEntry.stbEntry.schemaTag.pSchema[0]; + ret = metaDelJsonVarFromIdx(pMeta, &e, pTagColumn); + if (ret < 0) { + metaError("vgId:%d, failed to delete json var from idx:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), + e.name, e.uid, tstrerror(ret)); + } + } else { + for (int i = 0; i < pTagSchema->nCols; i++) { + pTagColumn = &stbEntry.stbEntry.schemaTag.pSchema[i]; + if (!IS_IDX_ON(pTagColumn)) continue; + STagIdxKey *pTagIdxKey = NULL; + int32_t nTagIdxKey; + + const void *pTagData = NULL; + int32_t nTagData = 0; + + STagVal tagVal = {.cid = pTagColumn->colId}; + if (tTagGet((const STag *)e.ctbEntry.pTags, &tagVal)) { + if (IS_VAR_DATA_TYPE(pTagColumn->type)) { + pTagData = tagVal.pData; + nTagData = (int32_t)tagVal.nData; + } else { + pTagData = &(tagVal.i64); + nTagData = tDataTypes[pTagColumn->type].bytes; + } + } else { + if (!IS_VAR_DATA_TYPE(pTagColumn->type)) { + nTagData = tDataTypes[pTagColumn->type].bytes; + } + } + + if (metaCreateTagIdxKey(e.ctbEntry.suid, pTagColumn->colId, pTagData, nTagData, pTagColumn->type, uid, + &pTagIdxKey, &nTagIdxKey) == 0) { + ret = tdbTbDelete(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, pMeta->txn); + if (ret < 0) { + metaError("vgId:%d, failed to delete tag idx key:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), + e.name, e.uid, tstrerror(ret)); + } + } + metaDestroyTagIdxKey(pTagIdxKey); + pTagIdxKey = NULL; + } + } + tDecoderClear(&tdc); + } + tdbFree(tData); + } + } + + ret = tdbTbDelete(pMeta->pTbDb, &(STbDbKey){.version = version, .uid = uid}, sizeof(STbDbKey), pMeta->txn); + if (ret < 0) { + metaError("vgId:%d, failed to delete table:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), e.name, e.uid, + tstrerror(ret)); + } + ret = tdbTbDelete(pMeta->pNameIdx, e.name, strlen(e.name) + 1, pMeta->txn); + if (ret < 0) { + metaError("vgId:%d, failed to delete name idx:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), e.name, e.uid, + tstrerror(ret)); + } + ret = tdbTbDelete(pMeta->pUidIdx, &uid, sizeof(uid), pMeta->txn); + if (ret < 0) { + metaError("vgId:%d, failed to delete uid idx:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), e.name, e.uid, + tstrerror(ret)); + } + + if (e.type == TSDB_CHILD_TABLE || e.type == TSDB_NORMAL_TABLE) metaDeleteBtimeIdx(pMeta, &e); + if (e.type == TSDB_NORMAL_TABLE) metaDeleteNcolIdx(pMeta, &e); + + if (e.type != TSDB_SUPER_TABLE) metaDeleteTtl(pMeta, &e); + + if (e.type == TSDB_CHILD_TABLE) { + ret = + tdbTbDelete(pMeta->pCtbIdx, &(SCtbIdxKey){.suid = e.ctbEntry.suid, .uid = uid}, sizeof(SCtbIdxKey), pMeta->txn); + if (ret < 0) { + metaError("vgId:%d, failed to delete ctb idx:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), e.name, e.uid, + tstrerror(ret)); + } + + --pMeta->pVnode->config.vndStats.numOfCTables; + metaUpdateStbStats(pMeta, e.ctbEntry.suid, -1, 0); + ret = metaUidCacheClear(pMeta, e.ctbEntry.suid); + if (ret < 0) { + metaError("vgId:%d, failed to clear uid cache:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), e.name, + e.ctbEntry.suid, tstrerror(ret)); + } + ret = metaTbGroupCacheClear(pMeta, e.ctbEntry.suid); + if (ret < 0) { + metaError("vgId:%d, failed to clear group cache:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), e.name, + e.ctbEntry.suid, tstrerror(ret)); + } + /* + if (!TSDB_CACHE_NO(pMeta->pVnode->config)) { + tsdbCacheDropTable(pMeta->pVnode->pTsdb, e.uid, e.ctbEntry.suid, NULL); + } + */ + } else if (e.type == TSDB_NORMAL_TABLE) { + // drop schema.db (todo) + + --pMeta->pVnode->config.vndStats.numOfNTables; + pMeta->pVnode->config.vndStats.numOfNTimeSeries -= e.ntbEntry.schemaRow.nCols - 1; + + /* + if (!TSDB_CACHE_NO(pMeta->pVnode->config)) { + tsdbCacheDropTable(pMeta->pVnode->pTsdb, e.uid, -1, &e.ntbEntry.schemaRow); + } + */ + } else if (e.type == TSDB_SUPER_TABLE) { + ret = tdbTbDelete(pMeta->pSuidIdx, &e.uid, sizeof(tb_uid_t), pMeta->txn); + if (ret < 0) { + metaError("vgId:%d, failed to delete suid idx:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), e.name, e.uid, + tstrerror(ret)); + } + // drop schema.db (todo) + + ret = metaStatsCacheDrop(pMeta, uid); + if (ret < 0) { + metaError("vgId:%d, failed to drop stats cache:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), e.name, e.uid, + tstrerror(ret)); + } + ret = metaUidCacheClear(pMeta, uid); + if (ret < 0) { + metaError("vgId:%d, failed to clear uid cache:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), e.name, e.uid, + tstrerror(ret)); + } + ret = metaTbGroupCacheClear(pMeta, uid); + if (ret < 0) { + metaError("vgId:%d, failed to clear group cache:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), e.name, + e.uid, tstrerror(ret)); + } + --pMeta->pVnode->config.vndStats.numOfSTables; + } + + ret = metaCacheDrop(pMeta, uid); + if (ret < 0) { + metaError("vgId:%d, failed to drop cache:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), e.name, e.uid, + tstrerror(ret)); + } + + tDecoderClear(&dc); + tdbFree(pData); + + return 0; +} +// opt ins_tables +int metaUpdateBtimeIdx(SMeta *pMeta, const SMetaEntry *pME) { + SBtimeIdxKey btimeKey = {0}; + if (metaBuildBtimeIdxKey(&btimeKey, pME) < 0) { + return 0; + } + metaTrace("vgId:%d, start to save version:%" PRId64 " uid:%" PRId64 " btime:%" PRId64, TD_VID(pMeta->pVnode), + pME->version, pME->uid, btimeKey.btime); + + return tdbTbUpsert(pMeta->pBtimeIdx, &btimeKey, sizeof(btimeKey), NULL, 0, pMeta->txn); +} + +int metaDeleteBtimeIdx(SMeta *pMeta, const SMetaEntry *pME) { + SBtimeIdxKey btimeKey = {0}; + if (metaBuildBtimeIdxKey(&btimeKey, pME) < 0) { + return 0; + } + return tdbTbDelete(pMeta->pBtimeIdx, &btimeKey, sizeof(btimeKey), pMeta->txn); +} +int metaUpdateNcolIdx(SMeta *pMeta, const SMetaEntry *pME) { + SNcolIdxKey ncolKey = {0}; + if (metaBuildNColIdxKey(&ncolKey, pME) < 0) { + return 0; + } + return tdbTbUpsert(pMeta->pNcolIdx, &ncolKey, sizeof(ncolKey), NULL, 0, pMeta->txn); +} + +int metaDeleteNcolIdx(SMeta *pMeta, const SMetaEntry *pME) { + SNcolIdxKey ncolKey = {0}; + if (metaBuildNColIdxKey(&ncolKey, pME) < 0) { + return 0; + } + return tdbTbDelete(pMeta->pNcolIdx, &ncolKey, sizeof(ncolKey), pMeta->txn); +} + +static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTbReq, STableMetaRsp *pMetaRsp) { + void *pVal = NULL; + int nVal = 0; + const void *pData = NULL; + int nData = 0; + int ret = 0; + tb_uid_t uid; + int64_t oversion; + SSchema *pColumn = NULL; + SMetaEntry entry = {0}; + SSchemaWrapper *pSchema; + int c; + bool freeColCmpr = false; + if (pAlterTbReq->colName == NULL) { + metaError("meta/table: null pAlterTbReq->colName"); + return terrno = TSDB_CODE_INVALID_MSG; + } + + // search name index + ret = tdbTbGet(pMeta->pNameIdx, pAlterTbReq->tbName, strlen(pAlterTbReq->tbName) + 1, &pVal, &nVal); + if (ret < 0) { + return terrno = TSDB_CODE_TDB_TABLE_NOT_EXIST; + } + + uid = *(tb_uid_t *)pVal; + tdbFree(pVal); + pVal = NULL; + + // search uid index + TBC *pUidIdxc = NULL; + + TAOS_CHECK_RETURN(tdbTbcOpen(pMeta->pUidIdx, &pUidIdxc, NULL)); + ret = tdbTbcMoveTo(pUidIdxc, &uid, sizeof(uid), &c); + if (c != 0) { + tdbTbcClose(pUidIdxc); + metaError("meta/table: invalide c: %" PRId32 " alt tb column failed.", c); + return TSDB_CODE_FAILED; + } + + ret = tdbTbcGet(pUidIdxc, NULL, NULL, &pData, &nData); + oversion = ((SUidIdxVal *)pData)[0].version; + + // search table.db + TBC *pTbDbc = NULL; + + TAOS_CHECK_RETURN(tdbTbcOpen(pMeta->pTbDb, &pTbDbc, NULL)); + ret = tdbTbcMoveTo(pTbDbc, &((STbDbKey){.uid = uid, .version = oversion}), sizeof(STbDbKey), &c); + if (c != 0) { + tdbTbcClose(pUidIdxc); + tdbTbcClose(pTbDbc); + metaError("meta/table: invalide c: %" PRId32 " alt tb column failed.", c); + return TSDB_CODE_FAILED; + } + + ret = tdbTbcGet(pTbDbc, NULL, NULL, &pData, &nData); + + // get table entry + SDecoder dc = {0}; + if ((entry.pBuf = taosMemoryMalloc(nData)) == NULL) { + tdbTbcClose(pUidIdxc); + tdbTbcClose(pTbDbc); + return terrno; + } + memcpy(entry.pBuf, pData, nData); + tDecoderInit(&dc, entry.pBuf, nData); + ret = metaDecodeEntry(&dc, &entry); + if (ret != 0) { + tdbTbcClose(pUidIdxc); + tdbTbcClose(pTbDbc); + tDecoderClear(&dc); + metaError("meta/table: invalide ret: %" PRId32 " alt tb column failed.", ret); + return ret; + } + + if (entry.type != TSDB_NORMAL_TABLE) { + terrno = TSDB_CODE_VND_INVALID_TABLE_ACTION; + goto _err; + } + // search the column to add/drop/update + pSchema = &entry.ntbEntry.schemaRow; + + // save old entry + SMetaEntry oldEntry = {.type = TSDB_NORMAL_TABLE, .uid = entry.uid}; + oldEntry.ntbEntry.schemaRow.nCols = pSchema->nCols; + + int32_t rowLen = -1; + if (pAlterTbReq->action == TSDB_ALTER_TABLE_ADD_COLUMN || + pAlterTbReq->action == TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES) { + rowLen = 0; + } + + int32_t iCol = 0, jCol = 0; + SSchema *qColumn = NULL; + for (;;) { + qColumn = NULL; + + if (jCol >= pSchema->nCols) break; + qColumn = &pSchema->pSchema[jCol]; + + if (!pColumn && (strcmp(qColumn->name, pAlterTbReq->colName) == 0)) { + pColumn = qColumn; + iCol = jCol; + if (rowLen < 0) break; + } + rowLen += qColumn->bytes; + ++jCol; + } + + entry.version = version; + int tlen; + SSchema *pNewSchema = NULL; + SSchema tScheam; + switch (pAlterTbReq->action) { + case TSDB_ALTER_TABLE_ADD_COLUMN: + case TSDB_ALTER_TABLE_ADD_COLUMN_WITH_COMPRESS_OPTION: + if (pColumn) { + terrno = TSDB_CODE_VND_COL_ALREADY_EXISTS; + goto _err; + } + if ((terrno = grantCheck(TSDB_GRANT_TIMESERIES)) < 0) { + goto _err; + } + if (rowLen + pAlterTbReq->bytes > TSDB_MAX_BYTES_PER_ROW) { + terrno = TSDB_CODE_PAR_INVALID_ROW_LENGTH; + goto _err; + } + pSchema->version++; + pSchema->nCols++; + pNewSchema = taosMemoryMalloc(sizeof(SSchema) * pSchema->nCols); + if (pNewSchema == NULL) { + goto _err; + } + memcpy(pNewSchema, pSchema->pSchema, sizeof(SSchema) * (pSchema->nCols - 1)); + pSchema->pSchema = pNewSchema; + pSchema->pSchema[entry.ntbEntry.schemaRow.nCols - 1].bytes = pAlterTbReq->bytes; + pSchema->pSchema[entry.ntbEntry.schemaRow.nCols - 1].type = pAlterTbReq->type; + pSchema->pSchema[entry.ntbEntry.schemaRow.nCols - 1].flags = pAlterTbReq->flags; + pSchema->pSchema[entry.ntbEntry.schemaRow.nCols - 1].colId = entry.ntbEntry.ncid++; + strcpy(pSchema->pSchema[entry.ntbEntry.schemaRow.nCols - 1].name, pAlterTbReq->colName); + + ++pMeta->pVnode->config.vndStats.numOfNTimeSeries; + metaTimeSeriesNotifyCheck(pMeta); + + if (!TSDB_CACHE_NO(pMeta->pVnode->config)) { + int16_t cid = pSchema->pSchema[entry.ntbEntry.schemaRow.nCols - 1].colId; + int8_t col_type = pSchema->pSchema[entry.ntbEntry.schemaRow.nCols - 1].type; + int32_t ret = tsdbCacheNewNTableColumn(pMeta->pVnode->pTsdb, entry.uid, cid, col_type); + if (ret < 0) { + terrno = ret; + goto _err; + } + } + SSchema *pCol = &pSchema->pSchema[entry.ntbEntry.schemaRow.nCols - 1]; + uint32_t compress = pAlterTbReq->action == TSDB_ALTER_TABLE_ADD_COLUMN ? createDefaultColCmprByType(pCol->type) + : pAlterTbReq->compress; + if (updataTableColCmpr(&entry.colCmpr, pCol, 1, compress) != 0) { + metaError("vgId:%d, failed to update table col cmpr:%s uid:%" PRId64, TD_VID(pMeta->pVnode), entry.name, + entry.uid); + } + freeColCmpr = true; + if (entry.colCmpr.nCols != pSchema->nCols) { + if (pNewSchema) taosMemoryFree(pNewSchema); + if (freeColCmpr) taosMemoryFree(entry.colCmpr.pColCmpr); + terrno = TSDB_CODE_VND_INVALID_TABLE_ACTION; + goto _err; + } + break; + case TSDB_ALTER_TABLE_DROP_COLUMN: + if (pColumn == NULL) { + terrno = TSDB_CODE_VND_COL_NOT_EXISTS; + goto _err; + } + if (pColumn->colId == 0) { + terrno = TSDB_CODE_VND_INVALID_TABLE_ACTION; + goto _err; + } + if (tqCheckColModifiable(pMeta->pVnode->pTq, uid, pColumn->colId) != 0) { + terrno = TSDB_CODE_VND_COL_SUBSCRIBED; + goto _err; + } + bool hasPrimayKey = false; + if (pSchema->nCols >= 2) { + hasPrimayKey = pSchema->pSchema[1].flags & COL_IS_KEY ? true : false; + } + + memcpy(&tScheam, pColumn, sizeof(SSchema)); + pSchema->version++; + tlen = (pSchema->nCols - iCol - 1) * sizeof(SSchema); + if (tlen) { + memmove(pColumn, pColumn + 1, tlen); + } + pSchema->nCols--; + + --pMeta->pVnode->config.vndStats.numOfNTimeSeries; + + if (!TSDB_CACHE_NO(pMeta->pVnode->config)) { + int16_t cid = pColumn->colId; + + if (tsdbCacheDropNTableColumn(pMeta->pVnode->pTsdb, entry.uid, cid, hasPrimayKey) != 0) { + metaError("vgId:%d, failed to drop ntable column:%s uid:%" PRId64, TD_VID(pMeta->pVnode), entry.name, + entry.uid); + } + } + + if (updataTableColCmpr(&entry.colCmpr, &tScheam, 0, 0) != 0) { + metaError("vgId:%d, failed to update table col cmpr:%s uid:%" PRId64, TD_VID(pMeta->pVnode), entry.name, + entry.uid); + } + if (entry.colCmpr.nCols != pSchema->nCols) { + terrno = TSDB_CODE_VND_INVALID_TABLE_ACTION; + goto _err; + } + break; + case TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES: + if (pColumn == NULL) { + terrno = TSDB_CODE_VND_COL_NOT_EXISTS; + goto _err; + } + if (!IS_VAR_DATA_TYPE(pColumn->type) || pColumn->bytes >= pAlterTbReq->colModBytes) { + terrno = TSDB_CODE_VND_INVALID_TABLE_ACTION; + goto _err; + } + if (rowLen + pAlterTbReq->colModBytes - pColumn->bytes > TSDB_MAX_BYTES_PER_ROW) { + terrno = TSDB_CODE_PAR_INVALID_ROW_LENGTH; + goto _err; + } + if (tqCheckColModifiable(pMeta->pVnode->pTq, uid, pColumn->colId) != 0) { + terrno = TSDB_CODE_VND_COL_SUBSCRIBED; + goto _err; + } + pSchema->version++; + pColumn->bytes = pAlterTbReq->colModBytes; + break; + case TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME: + if (pAlterTbReq->colNewName == NULL) { + terrno = TSDB_CODE_INVALID_MSG; + goto _err; + } + if (pColumn == NULL) { + terrno = TSDB_CODE_VND_COL_NOT_EXISTS; + goto _err; + } + if (tqCheckColModifiable(pMeta->pVnode->pTq, uid, pColumn->colId) != 0) { + terrno = TSDB_CODE_VND_COL_SUBSCRIBED; + goto _err; + } + pSchema->version++; + strcpy(pColumn->name, pAlterTbReq->colNewName); + break; + } + + if (!TSDB_CACHE_NO(pMeta->pVnode->config)) { + tsdbCacheInvalidateSchema(pMeta->pVnode->pTsdb, 0, entry.uid, pSchema->version); + } + + entry.version = version; + + // do actual write + metaWLock(pMeta); + + if (metaDeleteNcolIdx(pMeta, &oldEntry) < 0) { + metaError("vgId:%d, failed to delete ncol idx:%s uid:%" PRId64, TD_VID(pMeta->pVnode), entry.name, entry.uid); + } + + if (metaUpdateNcolIdx(pMeta, &entry) < 0) { + metaError("vgId:%d, failed to update ncol idx:%s uid:%" PRId64, TD_VID(pMeta->pVnode), entry.name, entry.uid); + } + + // save to table db + if (metaSaveToTbDb(pMeta, &entry) < 0) { + metaError("vgId:%d, failed to save to tb db:%s uid:%" PRId64, TD_VID(pMeta->pVnode), entry.name, entry.uid); + } + + if (metaUpdateUidIdx(pMeta, &entry) < 0) { + metaError("vgId:%d, failed to update uid idx:%s uid:%" PRId64, TD_VID(pMeta->pVnode), entry.name, entry.uid); + } + + if (metaSaveToSkmDb(pMeta, &entry) < 0) { + metaError("vgId:%d, failed to save to skm db:%s uid:%" PRId64, TD_VID(pMeta->pVnode), entry.name, entry.uid); + } + + if (metaUpdateChangeTime(pMeta, entry.uid, pAlterTbReq->ctimeMs) < 0) { + metaError("vgId:%d, failed to update change time:%s uid:%" PRId64, TD_VID(pMeta->pVnode), entry.name, entry.uid); + } + + metaULock(pMeta); + + if (metaUpdateMetaRsp(uid, pAlterTbReq->tbName, pSchema, pMetaRsp) < 0) { + metaError("vgId:%d, failed to update meta rsp:%s uid:%" PRId64, TD_VID(pMeta->pVnode), entry.name, entry.uid); + } + for (int32_t i = 0; i < entry.colCmpr.nCols; i++) { + SColCmpr *p = &entry.colCmpr.pColCmpr[i]; + pMetaRsp->pSchemaExt[i].colId = p->id; + pMetaRsp->pSchemaExt[i].compress = p->alg; + } + + if (entry.pBuf) taosMemoryFree(entry.pBuf); + if (pNewSchema) taosMemoryFree(pNewSchema); + if (freeColCmpr) taosMemoryFree(entry.colCmpr.pColCmpr); + + tdbTbcClose(pTbDbc); + tdbTbcClose(pUidIdxc); + tDecoderClear(&dc); + + return 0; + +_err: + if (entry.pBuf) taosMemoryFree(entry.pBuf); + tdbTbcClose(pTbDbc); + tdbTbcClose(pUidIdxc); + tDecoderClear(&dc); + + return terrno != 0 ? terrno : TSDB_CODE_FAILED; +} + +static int metaUpdateTableMultiTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTbReq) { + SMetaEntry ctbEntry = {0}; + SMetaEntry stbEntry = {0}; + void *pVal = NULL; + int nVal = 0; + int ret; + int c; + tb_uid_t uid; + int64_t oversion; + const void *pData = NULL; + int nData = 0; + SHashObj *pTagTable = NULL; + + // search name index + ret = tdbTbGet(pMeta->pNameIdx, pAlterTbReq->tbName, strlen(pAlterTbReq->tbName) + 1, &pVal, &nVal); + if (ret < 0) { + return terrno = TSDB_CODE_TDB_TABLE_NOT_EXIST; + } + + uid = *(tb_uid_t *)pVal; + tdbFree(pVal); + pVal = NULL; + + // search uid index + TBC *pUidIdxc = NULL; + + TAOS_CHECK_RETURN(tdbTbcOpen(pMeta->pUidIdx, &pUidIdxc, NULL)); + if (tdbTbcMoveTo(pUidIdxc, &uid, sizeof(uid), &c) < 0) { + metaTrace("meta/table: failed to move to uid index, uid:%" PRId64, uid); + } + if (c != 0) { + tdbTbcClose(pUidIdxc); + metaError("meta/table: invalide c: %" PRId32 " update tb tag val failed.", c); + return terrno = TSDB_CODE_TDB_TABLE_NOT_EXIST; + } + + if (tdbTbcGet(pUidIdxc, NULL, NULL, &pData, &nData) != 0) { + tdbTbcClose(pUidIdxc); + metaError("meta/table: failed to get uid index, uid:%" PRId64, uid); + return terrno = TSDB_CODE_TDB_TABLE_NOT_EXIST; + } + oversion = ((SUidIdxVal *)pData)[0].version; + + // search table.db + TBC *pTbDbc = NULL; + SDecoder dc1 = {0}; + SDecoder dc2 = {0}; + + /* get ctbEntry */ + TAOS_CHECK_RETURN(tdbTbcOpen(pMeta->pTbDb, &pTbDbc, NULL)); + if (tdbTbcMoveTo(pTbDbc, &((STbDbKey){.uid = uid, .version = oversion}), sizeof(STbDbKey), &c) != 0) { + metaError("meta/table: failed to move to tb db, uid:%" PRId64, uid); + } + if (c != 0) { + tdbTbcClose(pUidIdxc); + tdbTbcClose(pTbDbc); + metaError("meta/table: invalide c: %" PRId32 " update tb tag val failed.", c); + return terrno = TSDB_CODE_TDB_TABLE_NOT_EXIST; + } + + if (tdbTbcGet(pTbDbc, NULL, NULL, &pData, &nData) != 0) { + metaError("meta/table: failed to get tb db, uid:%" PRId64, uid); + tdbTbcClose(pUidIdxc); + tdbTbcClose(pTbDbc); + return terrno = TSDB_CODE_TDB_TABLE_NOT_EXIST; + } + + if ((ctbEntry.pBuf = taosMemoryMalloc(nData)) == NULL) { + tdbTbcClose(pUidIdxc); + tdbTbcClose(pTbDbc); + return terrno; + } + memcpy(ctbEntry.pBuf, pData, nData); + tDecoderInit(&dc1, ctbEntry.pBuf, nData); + ret = metaDecodeEntry(&dc1, &ctbEntry); + if (ret < 0) { + terrno = ret; + goto _err; + } + + /* get stbEntry*/ + if (tdbTbGet(pMeta->pUidIdx, &ctbEntry.ctbEntry.suid, sizeof(tb_uid_t), &pVal, &nVal) != 0) { + metaError("meta/table: failed to get uid index, uid:%" PRId64, ctbEntry.ctbEntry.suid); + } + if (!pVal) { + terrno = TSDB_CODE_INVALID_MSG; + goto _err; + } + + if (tdbTbGet(pMeta->pTbDb, &((STbDbKey){.uid = ctbEntry.ctbEntry.suid, .version = ((SUidIdxVal *)pVal)[0].version}), + sizeof(STbDbKey), (void **)&stbEntry.pBuf, &nVal) != 0) { + metaError("meta/table: failed to get tb db, uid:%" PRId64, ctbEntry.ctbEntry.suid); + } + tdbFree(pVal); + tDecoderInit(&dc2, stbEntry.pBuf, nVal); + ret = metaDecodeEntry(&dc2, &stbEntry); + if (ret < 0) { + terrno = ret; + goto _err; + } + + int32_t nTagVals = taosArrayGetSize(pAlterTbReq->pMultiTag); + pTagTable = taosHashInit(nTagVals, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + if (pTagTable == NULL) { + ret = terrno; + goto _err; + } + + // remove duplicate tag name + for (int i = 0; i < nTagVals; i++) { + SMultiTagUpateVal *pTagVal = taosArrayGet(pAlterTbReq->pMultiTag, i); + ret = taosHashPut(pTagTable, pTagVal->tagName, strlen(pTagVal->tagName), pTagVal, sizeof(*pTagVal)); + if (ret != 0) { + goto _err; + } + } + + SSchemaWrapper *pTagSchema = &stbEntry.stbEntry.schemaTag; + SSchema *pColumn = NULL; + int32_t iCol = 0; + int32_t count = 0; + + for (;;) { + pColumn = NULL; + + if (iCol >= pTagSchema->nCols) break; + pColumn = &pTagSchema->pSchema[iCol]; + if (taosHashGet(pTagTable, pColumn->name, strlen(pColumn->name)) != NULL) { + count++; + } + iCol++; + } + if (count != taosHashGetSize(pTagTable)) { + terrno = TSDB_CODE_VND_COL_NOT_EXISTS; + goto _err; + } + + ctbEntry.version = version; + if (pTagSchema->nCols == 1 && pTagSchema->pSchema[0].type == TSDB_DATA_TYPE_JSON) { + terrno = TSDB_CODE_VND_COL_NOT_EXISTS; + goto _err; + } else { + const STag *pOldTag = (const STag *)ctbEntry.ctbEntry.pTags; + STag *pNewTag = NULL; + SArray *pTagArray = taosArrayInit(pTagSchema->nCols, sizeof(STagVal)); + if (!pTagArray) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + for (int32_t i = 0; i < pTagSchema->nCols; i++) { + SSchema *pCol = &pTagSchema->pSchema[i]; + SMultiTagUpateVal *pTagVal = taosHashGet(pTagTable, pCol->name, strlen(pCol->name)); + if (pTagVal == NULL) { + STagVal val = {.cid = pCol->colId}; + if (tTagGet(pOldTag, &val)) { + if (taosArrayPush(pTagArray, &val) == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + taosArrayDestroy(pTagArray); + goto _err; + } + } + } else { + STagVal val = {0}; + val.type = pCol->type; + val.cid = pCol->colId; + if (pTagVal->isNull) continue; + + if (IS_VAR_DATA_TYPE(pCol->type)) { + val.pData = pTagVal->pTagVal; + val.nData = pTagVal->nTagVal; + } else { + memcpy(&val.i64, pTagVal->pTagVal, pTagVal->nTagVal); + } + if (taosArrayPush(pTagArray, &val) == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + taosArrayDestroy(pTagArray); + goto _err; + } + } + } + if ((terrno = tTagNew(pTagArray, pTagSchema->version, false, &pNewTag)) < 0) { + taosArrayDestroy(pTagArray); + goto _err; + } + ctbEntry.ctbEntry.pTags = (uint8_t *)pNewTag; + taosArrayDestroy(pTagArray); + } + + metaWLock(pMeta); + + // save to table.db + if (metaSaveToTbDb(pMeta, &ctbEntry) < 0) { + metaError("meta/table: failed to save to tb db:%s uid:%" PRId64, ctbEntry.name, ctbEntry.uid); + } + + // save to uid.idx + if (metaUpdateUidIdx(pMeta, &ctbEntry) < 0) { + metaError("meta/table: failed to update uid idx:%s uid:%" PRId64, ctbEntry.name, ctbEntry.uid); + } + + if (metaUpdateTagIdx(pMeta, &ctbEntry) < 0) { + metaError("meta/table: failed to update tag idx:%s uid:%" PRId64, ctbEntry.name, ctbEntry.uid); + } + + SCtbIdxKey ctbIdxKey = {.suid = ctbEntry.ctbEntry.suid, .uid = uid}; + if (tdbTbUpsert(pMeta->pCtbIdx, &ctbIdxKey, sizeof(ctbIdxKey), ctbEntry.ctbEntry.pTags, + ((STag *)(ctbEntry.ctbEntry.pTags))->len, pMeta->txn) < 0) { + metaError("meta/table: failed to upsert ctb idx:%s uid:%" PRId64, ctbEntry.name, ctbEntry.uid); + } + + if (metaUidCacheClear(pMeta, ctbEntry.ctbEntry.suid) < 0) { + metaError("meta/table: failed to clear uid cache:%s uid:%" PRId64, ctbEntry.name, ctbEntry.uid); + } + + if (metaTbGroupCacheClear(pMeta, ctbEntry.ctbEntry.suid) < 0) { + metaError("meta/table: failed to clear group cache:%s uid:%" PRId64, ctbEntry.name, ctbEntry.uid); + } + + if (metaUpdateChangeTime(pMeta, ctbEntry.uid, pAlterTbReq->ctimeMs) < 0) { + metaError("meta/table: failed to update change time:%s uid:%" PRId64, ctbEntry.name, ctbEntry.uid); + } + + metaULock(pMeta); + + tDecoderClear(&dc1); + tDecoderClear(&dc2); + taosMemoryFree((void *)ctbEntry.ctbEntry.pTags); + if (ctbEntry.pBuf) taosMemoryFree(ctbEntry.pBuf); + if (stbEntry.pBuf) tdbFree(stbEntry.pBuf); + tdbTbcClose(pTbDbc); + tdbTbcClose(pUidIdxc); + taosHashCleanup(pTagTable); + return 0; + +_err: + tDecoderClear(&dc1); + tDecoderClear(&dc2); + if (ctbEntry.pBuf) taosMemoryFree(ctbEntry.pBuf); + if (stbEntry.pBuf) tdbFree(stbEntry.pBuf); + tdbTbcClose(pTbDbc); + tdbTbcClose(pUidIdxc); + taosHashCleanup(pTagTable); + return -1; +} +static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTbReq) { + SMetaEntry ctbEntry = {0}; + SMetaEntry stbEntry = {0}; + void *pVal = NULL; + int nVal = 0; + int ret; + int c; + tb_uid_t uid; + int64_t oversion; + const void *pData = NULL; + int nData = 0; + + if (pAlterTbReq->tagName == NULL) { + return terrno = TSDB_CODE_INVALID_MSG; + } + + // search name index + ret = tdbTbGet(pMeta->pNameIdx, pAlterTbReq->tbName, strlen(pAlterTbReq->tbName) + 1, &pVal, &nVal); + if (ret < 0) { + return terrno = TSDB_CODE_TDB_TABLE_NOT_EXIST; + } + + uid = *(tb_uid_t *)pVal; + tdbFree(pVal); + pVal = NULL; + + // search uid index + TBC *pUidIdxc = NULL; + + TAOS_CHECK_RETURN(tdbTbcOpen(pMeta->pUidIdx, &pUidIdxc, NULL)); + if (tdbTbcMoveTo(pUidIdxc, &uid, sizeof(uid), &c) < 0) { + metaTrace("meta/table: failed to move to uid index, uid:%" PRId64, uid); + } + if (c != 0) { + tdbTbcClose(pUidIdxc); + metaError("meta/table: invalide c: %" PRId32 " update tb tag val failed.", c); + return terrno = TSDB_CODE_TDB_TABLE_NOT_EXIST; + } + + if (tdbTbcGet(pUidIdxc, NULL, NULL, &pData, &nData) != 0) { + tdbTbcClose(pUidIdxc); + metaError("meta/table: failed to get uid index, uid:%" PRId64, uid); + return terrno = TSDB_CODE_TDB_TABLE_NOT_EXIST; + } + oversion = ((SUidIdxVal *)pData)[0].version; + + // search table.db + TBC *pTbDbc = NULL; + SDecoder dc1 = {0}; + SDecoder dc2 = {0}; + + /* get ctbEntry */ + TAOS_CHECK_RETURN(tdbTbcOpen(pMeta->pTbDb, &pTbDbc, NULL)); + if (tdbTbcMoveTo(pTbDbc, &((STbDbKey){.uid = uid, .version = oversion}), sizeof(STbDbKey), &c) != 0) { + metaError("meta/table: failed to move to tb db, uid:%" PRId64, uid); + } + if (c != 0) { + tdbTbcClose(pUidIdxc); + tdbTbcClose(pTbDbc); + metaError("meta/table: invalide c: %" PRId32 " update tb tag val failed.", c); + return terrno = TSDB_CODE_TDB_TABLE_NOT_EXIST; + } + + if (tdbTbcGet(pTbDbc, NULL, NULL, &pData, &nData) != 0) { + metaError("meta/table: failed to get tb db, uid:%" PRId64, uid); + tdbTbcClose(pUidIdxc); + tdbTbcClose(pTbDbc); + return terrno = TSDB_CODE_INVALID_MSG; + } + + if ((ctbEntry.pBuf = taosMemoryMalloc(nData)) == NULL) { + tdbTbcClose(pUidIdxc); + tdbTbcClose(pTbDbc); + return terrno; + } + memcpy(ctbEntry.pBuf, pData, nData); + tDecoderInit(&dc1, ctbEntry.pBuf, nData); + ret = metaDecodeEntry(&dc1, &ctbEntry); + if (ret < 0) { + terrno = ret; + goto _err; + } + + /* get stbEntry*/ + if (tdbTbGet(pMeta->pUidIdx, &ctbEntry.ctbEntry.suid, sizeof(tb_uid_t), &pVal, &nVal) != 0) { + metaError("meta/table: failed to get uid index, uid:%" PRId64, ctbEntry.ctbEntry.suid); + } + if (!pVal) { + terrno = TSDB_CODE_INVALID_MSG; + goto _err; + } + + if (tdbTbGet(pMeta->pTbDb, &((STbDbKey){.uid = ctbEntry.ctbEntry.suid, .version = ((SUidIdxVal *)pVal)[0].version}), + sizeof(STbDbKey), (void **)&stbEntry.pBuf, &nVal) != 0) { + metaError("meta/table: failed to get tb db, uid:%" PRId64, ctbEntry.ctbEntry.suid); + } + tdbFree(pVal); + tDecoderInit(&dc2, stbEntry.pBuf, nVal); + ret = metaDecodeEntry(&dc2, &stbEntry); + if (ret < 0) { + terrno = ret; + goto _err; + } + + SSchemaWrapper *pTagSchema = &stbEntry.stbEntry.schemaTag; + SSchema *pColumn = NULL; + int32_t iCol = 0; + + for (;;) { + pColumn = NULL; + + if (iCol >= pTagSchema->nCols) break; + pColumn = &pTagSchema->pSchema[iCol]; + + if (strcmp(pColumn->name, pAlterTbReq->tagName) == 0) break; + iCol++; + } + + if (pColumn == NULL) { + terrno = TSDB_CODE_VND_COL_NOT_EXISTS; + goto _err; + } + + ctbEntry.version = version; + if (pTagSchema->nCols == 1 && pTagSchema->pSchema[0].type == TSDB_DATA_TYPE_JSON) { + ctbEntry.ctbEntry.pTags = taosMemoryMalloc(pAlterTbReq->nTagVal); + if (ctbEntry.ctbEntry.pTags == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + memcpy((void *)ctbEntry.ctbEntry.pTags, pAlterTbReq->pTagVal, pAlterTbReq->nTagVal); + } else { + const STag *pOldTag = (const STag *)ctbEntry.ctbEntry.pTags; + STag *pNewTag = NULL; + SArray *pTagArray = taosArrayInit(pTagSchema->nCols, sizeof(STagVal)); + if (!pTagArray) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + for (int32_t i = 0; i < pTagSchema->nCols; i++) { + SSchema *pCol = &pTagSchema->pSchema[i]; + if (iCol == i) { + if (pAlterTbReq->isNull) { + continue; + } + STagVal val = {0}; + val.type = pCol->type; + val.cid = pCol->colId; + if (IS_VAR_DATA_TYPE(pCol->type)) { + val.pData = pAlterTbReq->pTagVal; + val.nData = pAlterTbReq->nTagVal; + } else { + memcpy(&val.i64, pAlterTbReq->pTagVal, pAlterTbReq->nTagVal); + } + if (taosArrayPush(pTagArray, &val) == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + taosArrayDestroy(pTagArray); + goto _err; + } + } else { + STagVal val = {.cid = pCol->colId}; + if (tTagGet(pOldTag, &val)) { + if (taosArrayPush(pTagArray, &val) == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + taosArrayDestroy(pTagArray); + goto _err; + } + } + } + } + if ((terrno = tTagNew(pTagArray, pTagSchema->version, false, &pNewTag)) < 0) { + taosArrayDestroy(pTagArray); + goto _err; + } + ctbEntry.ctbEntry.pTags = (uint8_t *)pNewTag; + taosArrayDestroy(pTagArray); + } + + metaWLock(pMeta); + + // save to table.db + if (metaSaveToTbDb(pMeta, &ctbEntry) < 0) { + metaError("meta/table: failed to save to tb db:%s uid:%" PRId64, ctbEntry.name, ctbEntry.uid); + } + + // save to uid.idx + if (metaUpdateUidIdx(pMeta, &ctbEntry) < 0) { + metaError("meta/table: failed to update uid idx:%s uid:%" PRId64, ctbEntry.name, ctbEntry.uid); + } + + if (metaUpdateTagIdx(pMeta, &ctbEntry) < 0) { + metaError("meta/table: failed to update tag idx:%s uid:%" PRId64, ctbEntry.name, ctbEntry.uid); + } + + SCtbIdxKey ctbIdxKey = {.suid = ctbEntry.ctbEntry.suid, .uid = uid}; + if (tdbTbUpsert(pMeta->pCtbIdx, &ctbIdxKey, sizeof(ctbIdxKey), ctbEntry.ctbEntry.pTags, + ((STag *)(ctbEntry.ctbEntry.pTags))->len, pMeta->txn) < 0) { + metaError("meta/table: failed to upsert ctb idx:%s uid:%" PRId64, ctbEntry.name, ctbEntry.uid); + } + + if (metaUidCacheClear(pMeta, ctbEntry.ctbEntry.suid) < 0) { + metaError("meta/table: failed to clear uid cache:%s uid:%" PRId64, ctbEntry.name, ctbEntry.uid); + } + + if (metaTbGroupCacheClear(pMeta, ctbEntry.ctbEntry.suid) < 0) { + metaError("meta/table: failed to clear group cache:%s uid:%" PRId64, ctbEntry.name, ctbEntry.uid); + } + + if (metaUpdateChangeTime(pMeta, ctbEntry.uid, pAlterTbReq->ctimeMs) < 0) { + metaError("meta/table: failed to update change time:%s uid:%" PRId64, ctbEntry.name, ctbEntry.uid); + } + + metaULock(pMeta); + + tDecoderClear(&dc1); + tDecoderClear(&dc2); + taosMemoryFree((void *)ctbEntry.ctbEntry.pTags); + if (ctbEntry.pBuf) taosMemoryFree(ctbEntry.pBuf); + if (stbEntry.pBuf) tdbFree(stbEntry.pBuf); + tdbTbcClose(pTbDbc); + tdbTbcClose(pUidIdxc); + return 0; + +_err: + tDecoderClear(&dc1); + tDecoderClear(&dc2); + if (ctbEntry.pBuf) taosMemoryFree(ctbEntry.pBuf); + if (stbEntry.pBuf) tdbFree(stbEntry.pBuf); + tdbTbcClose(pTbDbc); + tdbTbcClose(pUidIdxc); + return -1; +} + +static int metaUpdateTableOptions(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTbReq) { + void *pVal = NULL; + int nVal = 0; + const void *pData = NULL; + int nData = 0; + int ret = 0; + tb_uid_t uid; + int64_t oversion; + SMetaEntry entry = {0}; + int c = 0; + + // search name index + ret = tdbTbGet(pMeta->pNameIdx, pAlterTbReq->tbName, strlen(pAlterTbReq->tbName) + 1, &pVal, &nVal); + if (ret < 0) { + return terrno = TSDB_CODE_TDB_TABLE_NOT_EXIST; + } + + uid = *(tb_uid_t *)pVal; + tdbFree(pVal); + pVal = NULL; + + // search uid index + TBC *pUidIdxc = NULL; + + TAOS_CHECK_RETURN(tdbTbcOpen(pMeta->pUidIdx, &pUidIdxc, NULL)); + if (tdbTbcMoveTo(pUidIdxc, &uid, sizeof(uid), &c) < 0) { + metaError("meta/table: failed to move to uid index, uid:%" PRId64, uid); + } + if (c != 0) { + tdbTbcClose(pUidIdxc); + metaError("meta/table: invalide c: %" PRId32 " update tb options failed.", c); + return TSDB_CODE_FAILED; + } + + if (tdbTbcGet(pUidIdxc, NULL, NULL, &pData, &nData) < 0) { + metaError("meta/table: failed to get uid index, uid:%" PRId64, uid); + } + oversion = ((SUidIdxVal *)pData)[0].version; + + // search table.db + TBC *pTbDbc = NULL; + + TAOS_CHECK_RETURN(tdbTbcOpen(pMeta->pTbDb, &pTbDbc, NULL)); + if (tdbTbcMoveTo(pTbDbc, &((STbDbKey){.uid = uid, .version = oversion}), sizeof(STbDbKey), &c) < 0) { + metaError("meta/table: failed to move to tb db, uid:%" PRId64, uid); + } + if (c != 0) { + tdbTbcClose(pUidIdxc); + tdbTbcClose(pTbDbc); + metaError("meta/table: invalide c: %" PRId32 " update tb options failed.", c); + return TSDB_CODE_FAILED; + } + + if (tdbTbcGet(pTbDbc, NULL, NULL, &pData, &nData) < 0) { + metaError("meta/table: failed to get tb db, uid:%" PRId64, uid); + } + + // get table entry + SDecoder dc = {0}; + if ((entry.pBuf = taosMemoryMalloc(nData)) == NULL) { + tdbTbcClose(pUidIdxc); + tdbTbcClose(pTbDbc); + return terrno; + } + memcpy(entry.pBuf, pData, nData); + tDecoderInit(&dc, entry.pBuf, nData); + ret = metaDecodeEntry(&dc, &entry); + if (ret != 0) { + tDecoderClear(&dc); + tdbTbcClose(pUidIdxc); + tdbTbcClose(pTbDbc); + metaError("meta/table: invalide ret: %" PRId32 " alt tb options failed.", ret); + return TSDB_CODE_FAILED; + } + + entry.version = version; + metaWLock(pMeta); + // build SMetaEntry + if (entry.type == TSDB_CHILD_TABLE) { + if (pAlterTbReq->updateTTL) { + metaDeleteTtl(pMeta, &entry); + entry.ctbEntry.ttlDays = pAlterTbReq->newTTL; + metaUpdateTtl(pMeta, &entry); + } + if (pAlterTbReq->newCommentLen >= 0) { + entry.ctbEntry.commentLen = pAlterTbReq->newCommentLen; + entry.ctbEntry.comment = pAlterTbReq->newComment; + } + } else { + if (pAlterTbReq->updateTTL) { + metaDeleteTtl(pMeta, &entry); + entry.ntbEntry.ttlDays = pAlterTbReq->newTTL; + metaUpdateTtl(pMeta, &entry); + } + if (pAlterTbReq->newCommentLen >= 0) { + entry.ntbEntry.commentLen = pAlterTbReq->newCommentLen; + entry.ntbEntry.comment = pAlterTbReq->newComment; + } + } + + // save to table db + if (metaSaveToTbDb(pMeta, &entry) < 0) { + metaError("meta/table: failed to save to tb db:%s uid:%" PRId64, entry.name, entry.uid); + } + + if (metaUpdateUidIdx(pMeta, &entry) < 0) { + metaError("meta/table: failed to update uid idx:%s uid:%" PRId64, entry.name, entry.uid); + } + + if (metaUpdateChangeTime(pMeta, entry.uid, pAlterTbReq->ctimeMs) < 0) { + metaError("meta/table: failed to update change time:%s uid:%" PRId64, entry.name, entry.uid); + } + + metaULock(pMeta); + + tdbTbcClose(pTbDbc); + tdbTbcClose(pUidIdxc); + tDecoderClear(&dc); + if (entry.pBuf) taosMemoryFree(entry.pBuf); + return 0; +} + +static int metaAddTagIndex(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTbReq) { + SMetaEntry stbEntry = {0}; + void *pVal = NULL; + int nVal = 0; + int ret; + int c; + tb_uid_t uid, suid; + int64_t oversion; + const void *pData = NULL; + int nData = 0; + SDecoder dc = {0}; + + if (pAlterTbReq->tagName == NULL) { + return terrno = TSDB_CODE_INVALID_MSG; + } + + // search name index + ret = tdbTbGet(pMeta->pNameIdx, pAlterTbReq->tbName, strlen(pAlterTbReq->tbName) + 1, &pVal, &nVal); + if (ret < 0) { + return terrno = TSDB_CODE_TDB_TABLE_NOT_EXIST; + } else { + uid = *(tb_uid_t *)pVal; + tdbFree(pVal); + pVal = NULL; + } + + if (tdbTbGet(pMeta->pUidIdx, &uid, sizeof(tb_uid_t), &pVal, &nVal) == -1) { + ret = -1; + goto _err; + } + suid = ((SUidIdxVal *)pVal)[0].suid; + + STbDbKey tbDbKey = {0}; + tbDbKey.uid = suid; + tbDbKey.version = ((SUidIdxVal *)pVal)[0].version; + ret = tdbTbGet(pMeta->pTbDb, &tbDbKey, sizeof(tbDbKey), &pVal, &nVal); + if (ret < 0) { + goto _err; + } + tDecoderInit(&dc, pVal, nVal); + ret = metaDecodeEntry(&dc, &stbEntry); + if (ret < 0) { + tDecoderClear(&dc); + goto _err; + } + + // Get target schema info + SSchemaWrapper *pTagSchema = &stbEntry.stbEntry.schemaTag; + if (pTagSchema->nCols == 1 && pTagSchema->pSchema[0].type == TSDB_DATA_TYPE_JSON) { + terrno = TSDB_CODE_VND_COL_ALREADY_EXISTS; + goto _err; + } + SSchema *pCol = NULL; + int32_t iCol = 0; + for (;;) { + pCol = NULL; + if (iCol >= pTagSchema->nCols) break; + pCol = &pTagSchema->pSchema[iCol]; + if (strcmp(pCol->name, pAlterTbReq->tagName) == 0) break; + iCol++; + } + + if (iCol == 0) { + terrno = TSDB_CODE_VND_COL_ALREADY_EXISTS; + goto _err; + } + if (pCol == NULL) { + terrno = TSDB_CODE_VND_COL_NOT_EXISTS; + goto _err; + } + + /* + * iterator all pTdDbc by uid and version + */ + TBC *pCtbIdxc = NULL; + TAOS_CHECK_RETURN(tdbTbcOpen(pMeta->pCtbIdx, &pCtbIdxc, NULL)); + int rc = tdbTbcMoveTo(pCtbIdxc, &(SCtbIdxKey){.suid = suid, .uid = INT64_MIN}, sizeof(SCtbIdxKey), &c); + if (rc < 0) { + tdbTbcClose(pCtbIdxc); + goto _err; + } + for (;;) { + void *pKey, *pVal; + int nKey, nVal; + rc = tdbTbcNext(pCtbIdxc, &pKey, &nKey, &pVal, &nVal); + if (rc < 0) break; + if (((SCtbIdxKey *)pKey)->suid != uid) { + tdbFree(pKey); + tdbFree(pVal); + continue; + } + STagIdxKey *pTagIdxKey = NULL; + int32_t nTagIdxKey; + + const void *pTagData = NULL; + int32_t nTagData = 0; + + STagVal tagVal = {.cid = pCol->colId}; + if (tTagGet((const STag *)pVal, &tagVal)) { + if (IS_VAR_DATA_TYPE(pCol->type)) { + pTagData = tagVal.pData; + nTagData = (int32_t)tagVal.nData; + } else { + pTagData = &(tagVal.i64); + nTagData = tDataTypes[pCol->type].bytes; + } + } else { + if (!IS_VAR_DATA_TYPE(pCol->type)) { + nTagData = tDataTypes[pCol->type].bytes; + } + } + if (metaCreateTagIdxKey(suid, pCol->colId, pTagData, nTagData, pCol->type, uid, &pTagIdxKey, &nTagIdxKey) < 0) { + tdbFree(pKey); + tdbFree(pVal); + metaDestroyTagIdxKey(pTagIdxKey); + tdbTbcClose(pCtbIdxc); + goto _err; + } + ret = tdbTbUpsert(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, NULL, 0, pMeta->txn); + if (ret < 0) { + metaError("meta/table: failed to upsert tag idx:%s uid:%" PRId64, stbEntry.name, stbEntry.uid); + } + metaDestroyTagIdxKey(pTagIdxKey); + pTagIdxKey = NULL; + } + tdbTbcClose(pCtbIdxc); + return 0; + +_err: + // tDecoderClear(&dc1); + // tDecoderClear(&dc2); + // if (ctbEntry.pBuf) taosMemoryFree(ctbEntry.pBuf); + // if (stbEntry.pBuf) tdbFree(stbEntry.pBuf); + // tdbTbcClose(pTbDbc); + // tdbTbcClose(pUidIdxc); + return TSDB_CODE_FAILED; +} + +typedef struct SMetaPair { + void *key; + int nkey; +} SMetaPair; + +static int metaDropTagIndex(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTbReq) { + SMetaEntry stbEntry = {0}; + void *pVal = NULL; + int nVal = 0; + int ret; + int c; + tb_uid_t suid; + int64_t oversion; + const void *pData = NULL; + int nData = 0; + SDecoder dc = {0}; + + if (pAlterTbReq->tagName == NULL) { + return terrno = TSDB_CODE_INVALID_MSG; + } + + // search name index + ret = tdbTbGet(pMeta->pNameIdx, pAlterTbReq->tbName, strlen(pAlterTbReq->tbName) + 1, &pVal, &nVal); + if (ret < 0) { + return terrno = TSDB_CODE_TDB_TABLE_NOT_EXIST; + } + suid = *(tb_uid_t *)pVal; + tdbFree(pVal); + pVal = NULL; + + if (tdbTbGet(pMeta->pUidIdx, &suid, sizeof(tb_uid_t), &pVal, &nVal) == -1) { + ret = -1; + goto _err; + } + + STbDbKey tbDbKey = {0}; + tbDbKey.uid = suid; + tbDbKey.version = ((SUidIdxVal *)pVal)[0].version; + ret = tdbTbGet(pMeta->pTbDb, &tbDbKey, sizeof(tbDbKey), &pVal, &nVal); + if (ret < 0) { + goto _err; + } + + tDecoderInit(&dc, pVal, nVal); + ret = metaDecodeEntry(&dc, &stbEntry); + if (ret < 0) { + tDecoderClear(&dc); + goto _err; + } + + // Get targe schema info + SSchemaWrapper *pTagSchema = &stbEntry.stbEntry.schemaTag; + if (pTagSchema->nCols == 1 && pTagSchema->pSchema[0].type == TSDB_DATA_TYPE_JSON) { + terrno = TSDB_CODE_VND_COL_ALREADY_EXISTS; + goto _err; + } + SSchema *pCol = NULL; + int32_t iCol = 0; + for (;;) { + pCol = NULL; + if (iCol >= pTagSchema->nCols) break; + pCol = &pTagSchema->pSchema[iCol]; + if (strcmp(pCol->name, pAlterTbReq->tagName) == 0) break; + iCol++; + } + if (iCol == 0) { + // cannot drop 1th tag index + terrno = -1; + goto _err; + } + if (pCol == NULL) { + terrno = TSDB_CODE_VND_COL_NOT_EXISTS; + goto _err; + } + + if (IS_IDX_ON(pCol)) { + terrno = TSDB_CODE_VND_COL_ALREADY_EXISTS; + goto _err; + } + + SArray *tagIdxList = taosArrayInit(512, sizeof(SMetaPair)); + if (tagIdxList == NULL) { + goto _err; + } + + TBC *pTagIdxc = NULL; + TAOS_CHECK_RETURN(tdbTbcOpen(pMeta->pTagIdx, &pTagIdxc, NULL)); + int rc = + tdbTbcMoveTo(pTagIdxc, &(STagIdxKey){.suid = suid, .cid = INT32_MIN, .type = pCol->type}, sizeof(STagIdxKey), &c); + for (;;) { + void *pKey, *pVal; + int nKey, nVal; + rc = tdbTbcNext(pTagIdxc, &pKey, &nKey, &pVal, &nVal); + STagIdxKey *pIdxKey = (STagIdxKey *)pKey; + if (pIdxKey->suid != suid || pIdxKey->cid != pCol->colId) { + tdbFree(pKey); + tdbFree(pVal); + continue; + } + + SMetaPair pair = {.key = pKey, nKey = nKey}; + if (taosArrayPush(tagIdxList, &pair) == NULL) { + goto _err; + } + } + tdbTbcClose(pTagIdxc); + + metaWLock(pMeta); + for (int i = 0; i < taosArrayGetSize(tagIdxList); i++) { + SMetaPair *pair = taosArrayGet(tagIdxList, i); + ret = tdbTbDelete(pMeta->pTagIdx, pair->key, pair->nkey, pMeta->txn); + if (ret < 0) { + metaError("meta/table: failed to delete tag idx:%s uid:%" PRId64, stbEntry.name, stbEntry.uid); + } + } + metaULock(pMeta); + + taosArrayDestroy(tagIdxList); + + // set pCol->flags; INDEX_ON + return 0; +_err: + return TSDB_CODE_FAILED; +} +int32_t metaUpdateTableColCompress(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq) { + // impl later + SMetaEntry tbEntry = {0}; + void *pVal = NULL; + int nVal = 0; + int ret; + int c; + tb_uid_t suid; + int64_t oversion; + const void *pData = NULL; + int nData = 0; + SDecoder dc = {0}; + ret = tdbTbGet(pMeta->pNameIdx, pReq->tbName, strlen(pReq->tbName) + 1, &pVal, &nVal); + if (ret < 0) { + return terrno = TSDB_CODE_TDB_TABLE_NOT_EXIST; + } + suid = *(tb_uid_t *)pVal; + tdbFree(pVal); + pVal = NULL; + + if (tdbTbGet(pMeta->pUidIdx, &suid, sizeof(tb_uid_t), &pVal, &nVal) == -1) { + terrno = TSDB_CODE_INVALID_MSG; + ret = -1; + goto _err; + } + + STbDbKey tbDbKey = {0}; + tbDbKey.uid = suid; + tbDbKey.version = ((SUidIdxVal *)pVal)[0].version; + if (tdbTbGet(pMeta->pTbDb, &tbDbKey, sizeof(tbDbKey), &pVal, &nVal) < 0) { + terrno = TSDB_CODE_INVALID_MSG; + tdbFree(pVal); + goto _err; + } + + tDecoderInit(&dc, pVal, nVal); + ret = metaDecodeEntry(&dc, &tbEntry); + if (ret < 0) { + terrno = TSDB_CODE_INVALID_MSG; + tdbFree(pVal); + tDecoderClear(&dc); + goto _err; + } + if (tbEntry.type != TSDB_NORMAL_TABLE && tbEntry.type != TSDB_SUPER_TABLE) { + terrno = TSDB_CODE_INVALID_MSG; + tdbFree(pVal); + tDecoderClear(&dc); + goto _err; + } + int8_t updated = 0; + SColCmprWrapper *wp = &tbEntry.colCmpr; + for (int32_t i = 0; i < wp->nCols; i++) { + SColCmpr *p = &wp->pColCmpr[i]; + if (p->id == pReq->colId) { + uint32_t dst = 0; + updated = tUpdateCompress(p->alg, pReq->compress, TSDB_COLVAL_COMPRESS_DISABLED, TSDB_COLVAL_LEVEL_DISABLED, + TSDB_COLVAL_LEVEL_MEDIUM, &dst); + if (updated > 0) { + p->alg = dst; + } + } + } + if (updated == 0) { + tdbFree(pVal); + tDecoderClear(&dc); + terrno = TSDB_CODE_VND_COLUMN_COMPRESS_ALREADY_EXIST; + goto _err; + } else if (updated < 0) { + tdbFree(pVal); + tDecoderClear(&dc); + terrno = TSDB_CODE_TSC_COMPRESS_LEVEL_ERROR; + goto _err; + } + tbEntry.version = version; + + metaWLock(pMeta); + if (metaSaveToTbDb(pMeta, &tbEntry) < 0) { + metaError("meta/table: failed to save to tb db:%s uid:%" PRId64, tbEntry.name, tbEntry.uid); + } + + if (metaUpdateUidIdx(pMeta, &tbEntry) < 0) { + metaError("meta/table: failed to update uid idx:%s uid:%" PRId64, tbEntry.name, tbEntry.uid); + } + + if (metaUpdateChangeTime(pMeta, suid, pReq->ctimeMs) < 0) { + metaError("meta/table: failed to update change time:%s uid:%" PRId64, tbEntry.name, tbEntry.uid); + } + + metaULock(pMeta); + + tdbFree(pVal); + tDecoderClear(&dc); + + return 0; +_err: + return TSDB_CODE_FAILED; +} + +int metaAlterTable(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq, STableMetaRsp *pMetaRsp) { + pMeta->changed = true; + switch (pReq->action) { + case TSDB_ALTER_TABLE_ADD_COLUMN: + case TSDB_ALTER_TABLE_ADD_COLUMN_WITH_COMPRESS_OPTION: + case TSDB_ALTER_TABLE_DROP_COLUMN: + case TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES: + case TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME: + return metaAlterTableColumn(pMeta, version, pReq, pMetaRsp); + case TSDB_ALTER_TABLE_UPDATE_TAG_VAL: + return metaUpdateTableTagVal(pMeta, version, pReq); + case TSDB_ALTER_TABLE_UPDATE_MULTI_TAG_VAL: + return metaUpdateTableMultiTagVal(pMeta, version, pReq); + return terrno = TSDB_CODE_VND_INVALID_TABLE_ACTION; + case TSDB_ALTER_TABLE_UPDATE_OPTIONS: + return metaUpdateTableOptions(pMeta, version, pReq); + case TSDB_ALTER_TABLE_ADD_TAG_INDEX: + return metaAddTagIndex(pMeta, version, pReq); + case TSDB_ALTER_TABLE_DROP_TAG_INDEX: + return metaDropTagIndex(pMeta, version, pReq); + case TSDB_ALTER_TABLE_UPDATE_COLUMN_COMPRESS: + return metaUpdateTableColCompress(pMeta, version, pReq); + default: + return terrno = TSDB_CODE_VND_INVALID_TABLE_ACTION; + break; + } +} + +static int metaSaveToTbDb(SMeta *pMeta, const SMetaEntry *pME) { + STbDbKey tbDbKey; + void *pKey = NULL; + void *pVal = NULL; + int kLen = 0; + int vLen = 0; + SEncoder coder = {0}; + + // set key and value + tbDbKey.version = pME->version; + tbDbKey.uid = pME->uid; + + metaDebug("vgId:%d, start to save table version:%" PRId64 " uid:%" PRId64, TD_VID(pMeta->pVnode), pME->version, + pME->uid); + + pKey = &tbDbKey; + kLen = sizeof(tbDbKey); + + int32_t ret = 0; + tEncodeSize(metaEncodeEntry, pME, vLen, ret); + if (ret < 0) { + goto _err; + } + + pVal = taosMemoryMalloc(vLen); + if (pVal == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + + tEncoderInit(&coder, pVal, vLen); + + if (metaEncodeEntry(&coder, pME) < 0) { + goto _err; + } + + tEncoderClear(&coder); + + // write to table.db + if (tdbTbInsert(pMeta->pTbDb, pKey, kLen, pVal, vLen, pMeta->txn) < 0) { + goto _err; + } + + taosMemoryFree(pVal); + return 0; + +_err: + metaError("vgId:%d, failed to save table version:%" PRId64 "uid:%" PRId64 " %s", TD_VID(pMeta->pVnode), pME->version, + pME->uid, tstrerror(terrno)); + + taosMemoryFree(pVal); + return TSDB_CODE_FAILED; +} + +static int metaUpdateUidIdx(SMeta *pMeta, const SMetaEntry *pME) { + // upsert cache + SMetaInfo info; + metaGetEntryInfo(pME, &info); + int32_t ret = metaCacheUpsert(pMeta, &info); + if (ret < 0) { + metaError("vgId:%d, failed to upsert cache, uid: %" PRId64 " %s", TD_VID(pMeta->pVnode), pME->uid, tstrerror(ret)); + } + + SUidIdxVal uidIdxVal = {.suid = info.suid, .version = info.version, .skmVer = info.skmVer}; + + return tdbTbUpsert(pMeta->pUidIdx, &pME->uid, sizeof(tb_uid_t), &uidIdxVal, sizeof(uidIdxVal), pMeta->txn); +} + +static int metaUpdateSuidIdx(SMeta *pMeta, const SMetaEntry *pME) { + return tdbTbUpsert(pMeta->pSuidIdx, &pME->uid, sizeof(tb_uid_t), NULL, 0, pMeta->txn); +} + +static int metaUpdateNameIdx(SMeta *pMeta, const SMetaEntry *pME) { + return tdbTbUpsert(pMeta->pNameIdx, pME->name, strlen(pME->name) + 1, &pME->uid, sizeof(tb_uid_t), pMeta->txn); +} + +static void metaUpdateTtl(SMeta *pMeta, const SMetaEntry *pME) { + if (pME->type != TSDB_CHILD_TABLE && pME->type != TSDB_NORMAL_TABLE) return; + + STtlUpdTtlCtx ctx = {.uid = pME->uid, .pTxn = pMeta->txn}; + if (pME->type == TSDB_CHILD_TABLE) { + ctx.ttlDays = pME->ctbEntry.ttlDays; + ctx.changeTimeMs = pME->ctbEntry.btime; + } else { + ctx.ttlDays = pME->ntbEntry.ttlDays; + ctx.changeTimeMs = pME->ntbEntry.btime; + } + + int32_t ret = ttlMgrInsertTtl(pMeta->pTtlMgr, &ctx); + if (ret < 0) { + metaError("vgId:%d, failed to insert ttl, uid: %" PRId64 " %s", TD_VID(pMeta->pVnode), pME->uid, tstrerror(ret)); + } + + return; +} + +static int metaUpdateChangeTime(SMeta *pMeta, tb_uid_t uid, int64_t changeTimeMs) { + if (!tsTtlChangeOnWrite) return 0; + + if (changeTimeMs <= 0) { + metaWarn("Skip to change ttl deletetion time on write, uid: %" PRId64, uid); + return TSDB_CODE_VERSION_NOT_COMPATIBLE; + } + + STtlUpdCtimeCtx ctx = {.uid = uid, .changeTimeMs = changeTimeMs, .pTxn = pMeta->txn}; + + return ttlMgrUpdateChangeTime(pMeta->pTtlMgr, &ctx); +} + +int metaUpdateChangeTimeWithLock(SMeta *pMeta, tb_uid_t uid, int64_t changeTimeMs) { + if (!tsTtlChangeOnWrite) return 0; + + metaWLock(pMeta); + int ret = metaUpdateChangeTime(pMeta, uid, changeTimeMs); + metaULock(pMeta); + return ret; +} + +static int metaUpdateCtbIdx(SMeta *pMeta, const SMetaEntry *pME) { + SCtbIdxKey ctbIdxKey = {.suid = pME->ctbEntry.suid, .uid = pME->uid}; + + return tdbTbUpsert(pMeta->pCtbIdx, &ctbIdxKey, sizeof(ctbIdxKey), pME->ctbEntry.pTags, + ((STag *)(pME->ctbEntry.pTags))->len, pMeta->txn); +} + +int metaCreateTagIdxKey(tb_uid_t suid, int32_t cid, const void *pTagData, int32_t nTagData, int8_t type, tb_uid_t uid, + STagIdxKey **ppTagIdxKey, int32_t *nTagIdxKey) { + if (IS_VAR_DATA_TYPE(type)) { + *nTagIdxKey = sizeof(STagIdxKey) + nTagData + VARSTR_HEADER_SIZE + sizeof(tb_uid_t); + } else { + *nTagIdxKey = sizeof(STagIdxKey) + nTagData + sizeof(tb_uid_t); + } + + *ppTagIdxKey = (STagIdxKey *)taosMemoryMalloc(*nTagIdxKey); + if (*ppTagIdxKey == NULL) { + return terrno; + } + + (*ppTagIdxKey)->suid = suid; + (*ppTagIdxKey)->cid = cid; + (*ppTagIdxKey)->isNull = (pTagData == NULL) ? 1 : 0; + (*ppTagIdxKey)->type = type; + + // refactor + if (IS_VAR_DATA_TYPE(type)) { + memcpy((*ppTagIdxKey)->data, (uint16_t *)&nTagData, VARSTR_HEADER_SIZE); + if (pTagData != NULL) memcpy((*ppTagIdxKey)->data + VARSTR_HEADER_SIZE, pTagData, nTagData); + *(tb_uid_t *)((*ppTagIdxKey)->data + VARSTR_HEADER_SIZE + nTagData) = uid; + } else { + if (pTagData != NULL) memcpy((*ppTagIdxKey)->data, pTagData, nTagData); + *(tb_uid_t *)((*ppTagIdxKey)->data + nTagData) = uid; + } + + return 0; +} + +static void metaDestroyTagIdxKey(STagIdxKey *pTagIdxKey) { + if (pTagIdxKey) taosMemoryFree(pTagIdxKey); +} + +static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) { + void *pData = NULL; + int nData = 0; + STbDbKey tbDbKey = {0}; + SMetaEntry stbEntry = {0}; + STagIdxKey *pTagIdxKey = NULL; + int32_t nTagIdxKey; + const SSchema *pTagColumn; + const void *pTagData = NULL; + int32_t nTagData = 0; + SDecoder dc = {0}; + int32_t ret = 0; + // get super table + if (tdbTbGet(pMeta->pUidIdx, &pCtbEntry->ctbEntry.suid, sizeof(tb_uid_t), &pData, &nData) != 0) { + metaError("vgId:%d, failed to get stable suid for update. version:%" PRId64, TD_VID(pMeta->pVnode), + pCtbEntry->version); + ret = TSDB_CODE_TDB_INVALID_TABLE_ID; + goto end; + } + tbDbKey.uid = pCtbEntry->ctbEntry.suid; + tbDbKey.version = ((SUidIdxVal *)pData)[0].version; + ret = tdbTbGet(pMeta->pTbDb, &tbDbKey, sizeof(tbDbKey), &pData, &nData); + if (ret < 0) { + metaError("vgId:%d, failed to get stable for update. version:%" PRId64, TD_VID(pMeta->pVnode), pCtbEntry->version); + goto end; + } + + tDecoderInit(&dc, pData, nData); + ret = metaDecodeEntry(&dc, &stbEntry); + if (ret < 0) { + goto end; + } + + if (stbEntry.stbEntry.schemaTag.pSchema == NULL) { + ret = TSDB_CODE_INVALID_PARA; + goto end; + } + + SSchemaWrapper *pTagSchema = &stbEntry.stbEntry.schemaTag; + if (pTagSchema->nCols == 1 && pTagSchema->pSchema[0].type == TSDB_DATA_TYPE_JSON) { + pTagColumn = &stbEntry.stbEntry.schemaTag.pSchema[0]; + STagVal tagVal = {.cid = pTagColumn->colId}; + + pTagData = pCtbEntry->ctbEntry.pTags; + nTagData = ((const STag *)pCtbEntry->ctbEntry.pTags)->len; + ret = metaSaveJsonVarToIdx(pMeta, pCtbEntry, pTagColumn); + goto end; + } else { + for (int i = 0; i < pTagSchema->nCols; i++) { + pTagData = NULL; + nTagData = 0; + pTagColumn = &pTagSchema->pSchema[i]; + if (!IS_IDX_ON(pTagColumn)) continue; + + STagVal tagVal = {.cid = pTagColumn->colId}; + if (tTagGet((const STag *)pCtbEntry->ctbEntry.pTags, &tagVal)) { + if (IS_VAR_DATA_TYPE(pTagColumn->type)) { + pTagData = tagVal.pData; + nTagData = (int32_t)tagVal.nData; + } else { + pTagData = &(tagVal.i64); + nTagData = tDataTypes[pTagColumn->type].bytes; + } + } else { + if (!IS_VAR_DATA_TYPE(pTagColumn->type)) { + nTagData = tDataTypes[pTagColumn->type].bytes; + } + } + if (metaCreateTagIdxKey(pCtbEntry->ctbEntry.suid, pTagColumn->colId, pTagData, nTagData, pTagColumn->type, + pCtbEntry->uid, &pTagIdxKey, &nTagIdxKey) < 0) { + ret = -1; + goto end; + } + if (tdbTbUpsert(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, NULL, 0, pMeta->txn) < 0) { + metaError("vgId:%d, failed to update tag index. version:%" PRId64, TD_VID(pMeta->pVnode), pCtbEntry->version); + } + metaDestroyTagIdxKey(pTagIdxKey); + pTagIdxKey = NULL; + } + } +end: + tDecoderClear(&dc); + tdbFree(pData); + return ret; +} + +static int metaSaveToSkmDb(SMeta *pMeta, const SMetaEntry *pME) { + SEncoder coder = {0}; + void *pVal = NULL; + int vLen = 0; + int rcode = 0; + SSkmDbKey skmDbKey = {0}; + const SSchemaWrapper *pSW; + + if (pME->type == TSDB_SUPER_TABLE) { + pSW = &pME->stbEntry.schemaRow; + } else if (pME->type == TSDB_NORMAL_TABLE) { + pSW = &pME->ntbEntry.schemaRow; + } else { + metaError("meta/table: invalide table type: %" PRId8 " save skm db failed.", pME->type); + return TSDB_CODE_FAILED; + } + + skmDbKey.uid = pME->uid; + skmDbKey.sver = pSW->version; + + // if receive tmq meta message is: create stable1 then delete stable1 then create stable1 with multi vgroups + if (tdbTbGet(pMeta->pSkmDb, &skmDbKey, sizeof(skmDbKey), NULL, NULL) == 0) { + return rcode; + } + + // encode schema + int32_t ret = 0; + tEncodeSize(tEncodeSSchemaWrapper, pSW, vLen, ret); + if (ret < 0) return -1; + pVal = taosMemoryMalloc(vLen); + if (pVal == NULL) { + rcode = -1; + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + + tEncoderInit(&coder, pVal, vLen); + ret = tEncodeSSchemaWrapper(&coder, pSW); + if (ret < 0) { + rcode = -1; + goto _exit; + } + + if (tdbTbInsert(pMeta->pSkmDb, &skmDbKey, sizeof(skmDbKey), pVal, vLen, pMeta->txn) < 0) { + rcode = -1; + goto _exit; + } + + metaDebug("vgId:%d, set schema:(%" PRId64 ") sver:%d since %s", TD_VID(pMeta->pVnode), pME->uid, pSW->version, + tstrerror(terrno)); + +_exit: + taosMemoryFree(pVal); + tEncoderClear(&coder); + return rcode; +} + +int metaHandleEntry(SMeta *pMeta, const SMetaEntry *pME) { + int32_t code = 0; + int32_t line = 0; + metaWLock(pMeta); + + // save to table.db + code = metaSaveToTbDb(pMeta, pME); + VND_CHECK_CODE(code, line, _err); + + // update uid.idx + code = metaUpdateUidIdx(pMeta, pME); + VND_CHECK_CODE(code, line, _err); + + // update name.idx + code = metaUpdateNameIdx(pMeta, pME); + VND_CHECK_CODE(code, line, _err); + + if (pME->type == TSDB_CHILD_TABLE) { + // update ctb.idx + code = metaUpdateCtbIdx(pMeta, pME); + VND_CHECK_CODE(code, line, _err); + + // update tag.idx + code = metaUpdateTagIdx(pMeta, pME); + VND_CHECK_CODE(code, line, _err); + } else { + // update schema.db + code = metaSaveToSkmDb(pMeta, pME); + VND_CHECK_CODE(code, line, _err); + + if (pME->type == TSDB_SUPER_TABLE) { + code = metaUpdateSuidIdx(pMeta, pME); + VND_CHECK_CODE(code, line, _err); + } + } + + code = metaUpdateBtimeIdx(pMeta, pME); + VND_CHECK_CODE(code, line, _err); + + if (pME->type == TSDB_NORMAL_TABLE) { + code = metaUpdateNcolIdx(pMeta, pME); + VND_CHECK_CODE(code, line, _err); + } + + if (pME->type != TSDB_SUPER_TABLE) { + metaUpdateTtl(pMeta, pME); + } + + if (pME->type == TSDB_SUPER_TABLE || pME->type == TSDB_NORMAL_TABLE) { + } + + metaULock(pMeta); + metaDebug("vgId:%d, handle meta entry, ver:%" PRId64 ", uid:%" PRId64 ", name:%s", TD_VID(pMeta->pVnode), + pME->version, pME->uid, pME->name); + return 0; + +_err: + metaULock(pMeta); + metaError("vgId:%d, failed to handle meta entry since %s at line:%d, ver:%" PRId64 ", uid:%" PRId64 ", name:%s", + TD_VID(pMeta->pVnode), terrstr(), line, pME->version, pME->uid, pME->name); + return TSDB_CODE_FAILED; +} + +static void colCompressDebug(SHashObj *pColCmprObj) { + void *p = taosHashIterate(pColCmprObj, NULL); + while (p) { + uint32_t cmprAlg = *(uint32_t *)p; + col_id_t colId = *(col_id_t *)taosHashGetKey(p, NULL); + p = taosHashIterate(pColCmprObj, p); + + uint8_t l1, l2, lvl; + tcompressDebug(cmprAlg, &l1, &l2, &lvl); + + const char *l1str = columnEncodeStr(l1); + const char *l2str = columnCompressStr(l2); + const char *lvlstr = columnLevelStr(lvl); + metaDebug("colId: %d, encode:%s, compress:%s,level:%s", colId, l1str, l2str, lvlstr); + } + return; +} +int32_t metaGetColCmpr(SMeta *pMeta, tb_uid_t uid, SHashObj **ppColCmprObj) { + int rc = 0; + + SHashObj *pColCmprObj = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_SMALLINT), false, HASH_NO_LOCK); + if (pColCmprObj == NULL) { + pColCmprObj = NULL; + return TSDB_CODE_OUT_OF_MEMORY; + } + + void *pData = NULL; + int nData = 0; + SMetaEntry e = {0}; + SDecoder dc = {0}; + + *ppColCmprObj = NULL; + + metaRLock(pMeta); + rc = tdbTbGet(pMeta->pUidIdx, &uid, sizeof(uid), &pData, &nData); + if (rc < 0) { + taosHashClear(pColCmprObj); + metaULock(pMeta); + return TSDB_CODE_FAILED; + } + int64_t version = ((SUidIdxVal *)pData)[0].version; + rc = tdbTbGet(pMeta->pTbDb, &(STbDbKey){.version = version, .uid = uid}, sizeof(STbDbKey), &pData, &nData); + if (rc < 0) { + metaULock(pMeta); + taosHashClear(pColCmprObj); + metaError("failed to get table entry"); + return rc; + } + + tDecoderInit(&dc, pData, nData); + rc = metaDecodeEntry(&dc, &e); + if (rc < 0) { + tDecoderClear(&dc); + tdbFree(pData); + metaULock(pMeta); + taosHashClear(pColCmprObj); + return rc; + } + if (useCompress(e.type)) { + SColCmprWrapper *p = &e.colCmpr; + for (int32_t i = 0; i < p->nCols; i++) { + SColCmpr *pCmpr = &p->pColCmpr[i]; + rc = taosHashPut(pColCmprObj, &pCmpr->id, sizeof(pCmpr->id), &pCmpr->alg, sizeof(pCmpr->alg)); + if (rc < 0) { + tDecoderClear(&dc); + tdbFree(pData); + metaULock(pMeta); + taosHashClear(pColCmprObj); + return rc; + } + } + } else { + tDecoderClear(&dc); + tdbFree(pData); + metaULock(pMeta); + taosHashClear(pColCmprObj); + return 0; + } + tDecoderClear(&dc); + tdbFree(pData); + metaULock(pMeta); + + *ppColCmprObj = pColCmprObj; + colCompressDebug(pColCmprObj); + + return 0; +} +// refactor later +void *metaGetIdx(SMeta *pMeta) { return pMeta->pTagIdx; } +void *metaGetIvtIdx(SMeta *pMeta) { return pMeta->pTagIvtIdx; } From 3e06712493160d219df87acbf15c3414c04f1411 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 4 Dec 2024 17:39:28 +0800 Subject: [PATCH 05/45] refact: create super table code --- source/dnode/vnode/inc/vnode.h | 38 +- source/dnode/vnode/src/inc/vnodeInt.h | 1 + source/dnode/vnode/src/meta/metaEntry2.c | 406 ++- source/dnode/vnode/src/meta/metaTable2.c | 3512 +--------------------- source/dnode/vnode/src/vnd/vnodeSvr.c | 11 +- 5 files changed, 417 insertions(+), 3551 deletions(-) diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 68a7766370..f61a3c970e 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -162,25 +162,25 @@ typedef struct STsdbReader STsdbReader; #define CACHESCAN_RETRIEVE_LAST_ROW 0x4 #define CACHESCAN_RETRIEVE_LAST 0x8 -int32_t tsdbReaderOpen2(void *pVnode, SQueryTableDataCond *pCond, void *pTableList, int32_t numOfTables, - SSDataBlock *pResBlock, void **ppReader, const char *idstr, SHashObj **pIgnoreTables); -int32_t tsdbSetTableList2(STsdbReader *pReader, const void *pTableList, int32_t num); -int32_t tsdbReaderSetId(void *pReader, const char *idstr); -void tsdbReaderClose2(STsdbReader *pReader); -int32_t tsdbNextDataBlock2(STsdbReader *pReader, bool *hasNext); -int32_t tsdbRetrieveDatablockSMA2(STsdbReader *pReader, SSDataBlock *pDataBlock, bool *allHave, bool *hasNullSMA); -void tsdbReleaseDataBlock2(STsdbReader *pReader); -int32_t tsdbRetrieveDataBlock2(STsdbReader *pReader, SSDataBlock **pBlock, SArray *pIdList); -int32_t tsdbReaderReset2(STsdbReader *pReader, SQueryTableDataCond *pCond); -int32_t tsdbGetFileBlocksDistInfo2(STsdbReader *pReader, STableBlockDistInfo *pTableBlockInfo); -int64_t tsdbGetNumOfRowsInMemTable2(STsdbReader *pHandle, uint32_t *rows); -void *tsdbGetIdx2(SMeta *pMeta); -void *tsdbGetIvtIdx2(SMeta *pMeta); -uint64_t tsdbGetReaderMaxVersion2(STsdbReader *pReader); -void tsdbReaderSetCloseFlag(STsdbReader *pReader); -int64_t tsdbGetLastTimestamp2(SVnode *pVnode, void *pTableList, int32_t numOfTables, const char *pIdStr); -void tsdbSetFilesetDelimited(STsdbReader *pReader); -void tsdbReaderSetNotifyCb(STsdbReader *pReader, TsdReaderNotifyCbFn notifyFn, void *param); +int32_t tsdbReaderOpen2(void *pVnode, SQueryTableDataCond *pCond, void *pTableList, int32_t numOfTables, + SSDataBlock *pResBlock, void **ppReader, const char *idstr, SHashObj **pIgnoreTables); +int32_t tsdbSetTableList2(STsdbReader *pReader, const void *pTableList, int32_t num); +int32_t tsdbReaderSetId(void *pReader, const char *idstr); +void tsdbReaderClose2(STsdbReader *pReader); +int32_t tsdbNextDataBlock2(STsdbReader *pReader, bool *hasNext); +int32_t tsdbRetrieveDatablockSMA2(STsdbReader *pReader, SSDataBlock *pDataBlock, bool *allHave, bool *hasNullSMA); +void tsdbReleaseDataBlock2(STsdbReader *pReader); +int32_t tsdbRetrieveDataBlock2(STsdbReader *pReader, SSDataBlock **pBlock, SArray *pIdList); +int32_t tsdbReaderReset2(STsdbReader *pReader, SQueryTableDataCond *pCond); +int32_t tsdbGetFileBlocksDistInfo2(STsdbReader *pReader, STableBlockDistInfo *pTableBlockInfo); +int64_t tsdbGetNumOfRowsInMemTable2(STsdbReader *pHandle, uint32_t *rows); +void *tsdbGetIdx2(SMeta *pMeta); +void *tsdbGetIvtIdx2(SMeta *pMeta); +uint64_t tsdbGetReaderMaxVersion2(STsdbReader *pReader); +void tsdbReaderSetCloseFlag(STsdbReader *pReader); +int64_t tsdbGetLastTimestamp2(SVnode *pVnode, void *pTableList, int32_t numOfTables, const char *pIdStr); +void tsdbSetFilesetDelimited(STsdbReader *pReader); +void tsdbReaderSetNotifyCb(STsdbReader *pReader, TsdReaderNotifyCbFn notifyFn, void *param); int32_t tsdbReuseCacherowsReader(void *pReader, void *pTableIdList, int32_t numOfTables); int32_t tsdbCacherowsReaderOpen(void *pVnode, int32_t type, void *pTableIdList, int32_t numOfTables, int32_t numOfCols, diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 0ac475f41b..43164da8a7 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -156,6 +156,7 @@ int metaFinishCommit(SMeta* pMeta, TXN* txn); int metaPrepareAsyncCommit(SMeta* pMeta); int metaAbort(SMeta* pMeta); int metaCreateSTable(SMeta* pMeta, int64_t version, SVCreateStbReq* pReq); +int metaCreateSuperTable(SMeta* pMeta, int64_t version, SVCreateStbReq* pReq); int metaAlterSTable(SMeta* pMeta, int64_t version, SVCreateStbReq* pReq); int metaDropSTable(SMeta* pMeta, int64_t verison, SVDropStbReq* pReq, SArray* tbUidList); int metaCreateTable(SMeta* pMeta, int64_t version, SVCreateTbReq* pReq, STableMetaRsp** pMetaRsp); diff --git a/source/dnode/vnode/src/meta/metaEntry2.c b/source/dnode/vnode/src/meta/metaEntry2.c index 812314c8c5..27680950dc 100644 --- a/source/dnode/vnode/src/meta/metaEntry2.c +++ b/source/dnode/vnode/src/meta/metaEntry2.c @@ -10,15 +10,242 @@ #include "meta.h" -#define metaErrLog(ERRNO, INFO) \ - do { \ - if (INFO) { \ - metaError("%s failed at %s:%d since %s, info:%s", __func__, __FILE__, __LINE__, tstrerror(ERRNO), INFO); \ - } else { \ - metaError("%s failed at %s:%d since %s", __func__, __FILE__, __LINE__, tstrerror(ERRNO)); \ - } \ +#define metaErr(VGID, ERRNO) \ + do { \ + metaError("vgId:%d, %s failed at %s:%d since %s, version:%" PRId64 " type:%d uid:%" PRId64 " name:%s", VGID, \ + __func__, __FILE__, __LINE__, tstrerror(ERRNO), pEntry->version, pEntry->type, pEntry->uid, \ + pEntry->type > 0 ? pEntry->name : NULL); \ } while (0) +// Entry Table +static int32_t metaEntryTableInsert(SMeta *pMeta, const SMetaEntry *pEntry) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t vgId = TD_VID(pMeta->pVnode); + void *value = NULL; + int32_t valueSize = 0; + SEncoder encoder = {0}; + STbDbKey key = { + .version = pEntry->version, + .uid = pEntry->uid, + }; + + // encode entry + tEncodeSize(metaEncodeEntry, pEntry, valueSize, code); + if (code != 0) { + metaErr(vgId, code); + return code; + } + + value = taosMemoryMalloc(valueSize); + if (NULL == value) { + metaErr(vgId, terrno); + return terrno; + } + + tEncoderInit(&encoder, value, valueSize); + code = metaEncodeEntry(&encoder, pEntry); + if (code) { + metaErr(vgId, code); + tEncoderClear(&encoder); + taosMemoryFree(value); + return code; + } + tEncoderClear(&encoder); + + // put to tdb + code = tdbTbInsert(pMeta->pTbDb, &key, sizeof(key), value, valueSize, pMeta->txn); + if (TSDB_CODE_SUCCESS != code) { + metaErr(vgId, code); + } + taosMemoryFree(value); + return code; +} + +static int32_t metaEntryTableDrop(SMeta *pMeta, const SMetaEntry *pEntry) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t vgId = TD_VID(pMeta->pVnode); + + // TODO + + return code; +} + +// Schema Table +static int32_t metaSchemaTableInsert(SMeta *pMeta, const SMetaEntry *pEntry) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t vgId = TD_VID(pMeta->pVnode); + SEncoder encoder = {0}; + void *value = NULL; + int32_t valueSize = 0; + + const SSchemaWrapper *pSchema = NULL; + if (pEntry->type == TSDB_SUPER_TABLE) { + pSchema = &pEntry->stbEntry.schemaRow; + } else if (pEntry->type == TSDB_NORMAL_TABLE) { + pSchema = &pEntry->ntbEntry.schemaRow; + } else { + return TSDB_CODE_INVALID_PARA; + } + SSkmDbKey key = { + .uid = pEntry->uid, + .sver = pSchema->version, + }; + + // encode schema + tEncodeSize(tEncodeSSchemaWrapper, pSchema, valueSize, code); + if (TSDB_CODE_SUCCESS != code) { + metaErr(vgId, code); + return code; + } + + tEncoderInit(&encoder, value, valueSize); + code = tEncodeSSchemaWrapper(&encoder, pSchema); + if (TSDB_CODE_SUCCESS != code) { + metaErr(vgId, code); + tEncoderClear(&encoder); + taosMemoryFree(value); + return code; + } + tEncoderClear(&encoder); + + // put to tdb + code = tdbTbInsert(pMeta->pSkmDb, &key, sizeof(key), value, valueSize, pMeta->txn); + if (TSDB_CODE_SUCCESS != code) { + metaErr(vgId, code); + } + taosMemoryFree(value); + return code; +} + +// Uid Index +static void metaBuildEntryInfo(const SMetaEntry *pEntry, SMetaInfo *pInfo) { + pInfo->uid = pEntry->uid; + pInfo->version = pEntry->version; + if (pEntry->type == TSDB_SUPER_TABLE) { + pInfo->suid = pEntry->uid; + pInfo->skmVer = pEntry->stbEntry.schemaRow.version; + } else if (pEntry->type == TSDB_CHILD_TABLE) { + pInfo->suid = pEntry->ctbEntry.suid; + pInfo->skmVer = 0; + } else if (pEntry->type == TSDB_NORMAL_TABLE) { + pInfo->suid = 0; + pInfo->skmVer = pEntry->ntbEntry.schemaRow.version; + } +} +static int32_t metaUidIdxUpsert(SMeta *pMeta, const SMetaEntry *pEntry) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t vgId = TD_VID(pMeta->pVnode); + + // update cache + SMetaInfo info = {0}; + metaBuildEntryInfo(pEntry, &info); + code = metaCacheUpsert(pMeta, &info); + if (code) { + metaErr(vgId, code); + } + + // put to tdb + SUidIdxVal value = { + .suid = 0, + .skmVer = 0, + .version = pEntry->version, + }; + code = tdbTbUpsert(pMeta->pUidIdx, &pEntry->uid, sizeof(pEntry->uid), &value, sizeof(value), pMeta->txn); + return code; +} + +// Name Index +static int32_t metaNameIdxInsert(SMeta *pMeta, const SMetaEntry *pEntry) { + int32_t code = tdbTbInsert(pMeta->pNameIdx, pEntry->name, strlen(pEntry->name) + 1, &pEntry->uid, sizeof(pEntry->uid), + pMeta->txn); + if (code) { + metaErr(TD_VID(pMeta->pVnode), code); + } + return code; +} + +static int32_t metaNameIdxDelete(SMeta *pMeta, const SMetaEntry *pEntry) { + int32_t code = tdbTbDelete(pMeta->pNameIdx, pEntry->name, strlen(pEntry->name) + 1, pMeta->txn); + if (code) { + metaErr(TD_VID(pMeta->pVnode), code); + } + return code; +} + +// Suid Index +static int32_t metaSUidIdxInsert(SMeta *pMeta, const SMetaEntry *pEntry) { + int32_t code = tdbTbInsert(pMeta->pSuidIdx, &pEntry->uid, sizeof(pEntry->uid), NULL, 0, pMeta->txn); + if (code) { + metaErr(TD_VID(pMeta->pVnode), code); + } + return code; +} + +// Tag Index + +static int32_t metaHandleSuperTableCreateImpl(SMeta *pMeta, const SMetaEntry *pEntry) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t vgId = TD_VID(pMeta->pVnode); + + // Insert into Entry Table + code = metaEntryTableInsert(pMeta, pEntry); + if (TSDB_CODE_SUCCESS != code) { + metaErr(vgId, code); + return code; + } + + // Insert into Schema Table + code = metaSchemaTableInsert(pMeta, pEntry); + if (TSDB_CODE_SUCCESS != code) { + metaErr(vgId, code); + return code; + } + + // Insert to Uid Index + code = metaUidIdxUpsert(pMeta, pEntry); + if (TSDB_CODE_SUCCESS != code) { + metaErr(vgId, code); + return code; + } + + // Insert to Name Index + code = metaNameIdxInsert(pMeta, pEntry); + if (TSDB_CODE_SUCCESS != code) { + metaErr(vgId, code); + return code; + } + + // Insert to Suid Index + code = metaSUidIdxInsert(pMeta, pEntry); + if (TSDB_CODE_SUCCESS != code) { + metaErr(vgId, code); + return code; + } + + return code; +} + +static int32_t metaHandleSuperTableCreate(SMeta *pMeta, const SMetaEntry *pEntry) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t vgId = TD_VID(pMeta->pVnode); + + metaWLock(pMeta); + code = metaHandleSuperTableCreateImpl(pMeta, pEntry); + metaULock(pMeta); + + if (TSDB_CODE_SUCCESS == code) { + pMeta->pVnode->config.vndStats.numOfSTables++; + pMeta->changed = true; + + metaInfo("vgId:%d, %s success, version:%" PRId64 " type:%d uid:%" PRId64 " name:%s", vgId, __func__, + pEntry->version, pEntry->type, pEntry->uid, pEntry->name); + } else { + metaError("vgId:%d, %s failed at %s:%d since %s, version" PRId64 " type:%d uid:%" PRId64 " name:%s", vgId, __func__, + __FILE__, __LINE__, tstrerror(code), pEntry->version, pEntry->type, pEntry->uid, pEntry->name); + } + return code; +} + static int32_t metaHandleNormalTableDrop(SMeta *pMeta, const SMetaEntry *pEntry) { int32_t code = TSDB_CODE_SUCCESS; // TODO @@ -37,117 +264,74 @@ static int32_t metaHandleSuperTableDrop(SMeta *pMeta, const SMetaEntry *pEntry) return code; } -static int32_t metaHandleEntryDrop(SMeta *pMeta, const SMetaEntry *pEntry) { - int32_t code = TSDB_CODE_SUCCESS; - int8_t type = -pEntry->type; - - if (type == TSDB_NORMAL_TABLE) { - code = metaHandleNormalTableDrop(pMeta, pEntry); - } else if (type == TSDB_CHILD_TABLE) { - code = metaHandleChildTableDrop(pMeta, pEntry); - } else if (type == TSDB_SUPER_TABLE) { - code = metaHandleSuperTableDrop(pMeta, pEntry); - } else { - code = TSDB_CODE_INVALID_PARA; - metaErrLog(code, NULL); - } - - if (TSDB_CODE_SUCCESS == code) { - } else { - } - - return code; -} - -static int32_t metaHandleNormalTableEntryUpdate(SMeta *pMeta, const SMetaEntry *pEntry) { - int32_t code = TSDB_CODE_SUCCESS; - // TODO: - return code; -} - -static int32_t metaHandleNormalTableEntryCreateImpl(SMeta *pMeta, const SMetaEntry *pEntry) { - int32_t code = TSDB_CODE_SUCCESS; - - // TODO - // code = metaSaveToTbDb(pMeta, pEntry); - // code = metaUpdateUidIdx(pMeta, pEntry); - // code = metaUpdatenameIdx(pMeta, pEntry); - // code = metaSaveToSkmDb(pMeta, pEntry); - // code = metaUpdateBtimeIdx(pMeta, pEntry); - // code = metaUpdateNcolIdx(pMeta, pEntry); - // code = metaUpdateTtl(pMeta, pEntry); - - return code; -} - -static int32_t metaHandleNormalTableEntryCreate(SMeta *pMeta, const SMetaEntry *pEntry) { - int32_t code = TSDB_CODE_SUCCESS; - - // TODO: do neccessary check - - // do handle entry - metaWLock(pMeta); - code = metaHandleNormalTableEntryCreateImpl(pMeta, pEntry); - metaULock(pMeta); - - return code; -} - -static int32_t metaHandleNormalTableEntryUpsert(SMeta *pMeta, const SMetaEntry *pEntry) { - int32_t code = TSDB_CODE_SUCCESS; - SMetaInfo metaInfo = {0}; - - if (TSDB_CODE_SUCCESS == metaGetInfo(pMeta, pEntry->uid, &metaInfo, NULL)) { - code = metaHandleNormalTableEntryUpdate(pMeta, pEntry); - } else { - code = metaHandleNormalTableEntryCreate(pMeta, pEntry); - } - - return code; -} - -static int32_t metaHandleSuperTableEntryUpsert(SMeta *pMeta, const SMetaEntry *pEntry) { - int32_t code = TSDB_CODE_SUCCESS; - // TODO - return code; -} - -static int32_t metaHandleChildTableEntryUpsert(SMeta *pMeta, const SMetaEntry *pEntry) { - int32_t code = TSDB_CODE_SUCCESS; - // TODO - return code; -} - -static int32_t metaHandleEntryUpsert(SMeta *pMeta, const SMetaEntry *pEntry) { - int32_t code = TSDB_CODE_SUCCESS; - - if (pEntry->type == TSDB_NORMAL_TABLE) { - code = metaHandleNormalTableEntryUpsert(pMeta, pEntry); - } else if (pEntry->type == TSDB_SUPER_TABLE) { - code = metaHandleSuperTableEntryUpsert(pMeta, pEntry); - } else if (pEntry->type == TSDB_CHILD_TABLE) { - code = metaHandleChildTableEntryUpsert(pMeta, pEntry); - } else { - code = TSDB_CODE_INVALID_PARA; - metaErrLog(code, NULL); - } - - return code; -} - int32_t metaHandleEntry2(SMeta *pMeta, const SMetaEntry *pEntry) { - int32_t code = TSDB_CODE_SUCCESS; + int32_t code = TSDB_CODE_SUCCESS; + int32_t vgId = TD_VID(pMeta->pVnode); + SMetaInfo info = {0}; + int8_t type = pEntry->type > 0 ? pEntry->type : -pEntry->type; - if (pMeta == NULL || pEntry == NULL) { - metaErrLog(TSDB_CODE_INVALID_PARA, NULL); + if (NULL == pMeta || NULL == pEntry || type != TSDB_SUPER_TABLE || type != TSDB_CHILD_TABLE || + type != TSDB_NORMAL_TABLE) { + metaError("%s failed at %s:%d since invalid parameter", __func__, __FILE__, __LINE__); return TSDB_CODE_INVALID_PARA; } if (pEntry->type > 0) { - code = metaHandleEntryUpsert(pMeta, pEntry); + bool isExist = false; + if (TSDB_CODE_SUCCESS == metaGetInfo(pMeta, pEntry->uid, &info, NULL)) { + isExist = true; + } + + switch (type) { + case TSDB_SUPER_TABLE: + if (isExist) { + // code = metaHandleSuperTableUpdate(pMeta, pEntry); + } else { + code = metaHandleSuperTableCreate(pMeta, pEntry); + } + break; + case TSDB_CHILD_TABLE: + if (isExist) { + // code = metaHandleChildTableUpdate(pMeta, pEntry); + } else { + // code = metaHandleChildTableCreate(pMeta, pEntry); + } + break; + case TSDB_NORMAL_TABLE: + if (isExist) { + // code = metaHandleNormalTableUpdate(pMeta, pEntry); + } else { + // code = metaHandleNormalTableCreate(pMeta, pEntry); + } + break; + default: + code = TSDB_CODE_INVALID_PARA; + break; + } } else { - code = metaHandleEntryDrop(pMeta, pEntry); + switch (type) { + case TSDB_SUPER_TABLE: + // code = metaHandleSuperTableDrop(pMeta, pEntry); + break; + case TSDB_CHILD_TABLE: + // code = metaHandleChildTableDrop(pMeta, pEntry); + break; + case TSDB_NORMAL_TABLE: + // code = metaHandleNormalTableDrop(pMeta, pEntry); + break; + default: + code = TSDB_CODE_INVALID_PARA; + break; + } } - return code; + if (TSDB_CODE_SUCCESS == code) { + metaDebug("vgId:%d, %s success, version:%" PRId64 " type:%d uid:%" PRId64 " name:%s", vgId, __func__, + pEntry->version, pEntry->type, pEntry->uid, pEntry->type > 0 ? pEntry->name : ""); + } else { + metaError("vgId:%d, %s failed at %s:%d since %s, version" PRId64 " type:%d uid:%" PRId64 " name:%s", vgId, __func__, + __FILE__, __LINE__, tstrerror(code), pEntry->version, pEntry->type, pEntry->uid, + pEntry->type > 0 ? pEntry->name : ""); + } + TAOS_RETURN(code); } \ No newline at end of file diff --git a/source/dnode/vnode/src/meta/metaTable2.c b/source/dnode/vnode/src/meta/metaTable2.c index 65e520bb4a..4e0b41fb93 100644 --- a/source/dnode/vnode/src/meta/metaTable2.c +++ b/source/dnode/vnode/src/meta/metaTable2.c @@ -15,3442 +15,124 @@ #include "meta.h" -extern SDmNotifyHandle dmNotifyHdl; +extern int32_t metaHandleEntry2(SMeta *pMeta, const SMetaEntry *pEntry); -static int metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSchema *pSchema); -static int metaDelJsonVarFromIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSchema *pSchema); -static int metaSaveToTbDb(SMeta *pMeta, const SMetaEntry *pME); -static int metaUpdateUidIdx(SMeta *pMeta, const SMetaEntry *pME); -static int metaUpdateNameIdx(SMeta *pMeta, const SMetaEntry *pME); -static void metaUpdateTtl(SMeta *pMeta, const SMetaEntry *pME); -static int metaUpdateChangeTime(SMeta *pMeta, tb_uid_t uid, int64_t changeTimeMs); -static int metaSaveToSkmDb(SMeta *pMeta, const SMetaEntry *pME); -static int metaUpdateCtbIdx(SMeta *pMeta, const SMetaEntry *pME); -static int metaUpdateSuidIdx(SMeta *pMeta, const SMetaEntry *pME); -static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry); -static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type, tb_uid_t *pSuid, int8_t *pSysTbl); -static void metaDestroyTagIdxKey(STagIdxKey *pTagIdxKey); -// opt ins_tables query -static int metaUpdateBtimeIdx(SMeta *pMeta, const SMetaEntry *pME); -static int metaDeleteBtimeIdx(SMeta *pMeta, const SMetaEntry *pME); -static int metaUpdateNcolIdx(SMeta *pMeta, const SMetaEntry *pME); -static int metaDeleteNcolIdx(SMeta *pMeta, const SMetaEntry *pME); +static int32_t metaCheckCreateSuperTableReq(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { + int32_t vgId = TD_VID(pMeta->pVnode); + void *value = NULL; + int32_t valueSize = 0; + SMetaInfo info; -static int32_t updataTableColCmpr(SColCmprWrapper *pWp, SSchema *pSchema, int8_t add, uint32_t compress) { - int32_t nCols = pWp->nCols; - int32_t ver = pWp->version; - if (add) { - SColCmpr *p = taosMemoryCalloc(1, sizeof(SColCmpr) * (nCols + 1)); - if (p == NULL) { - return terrno; + // check name + if (NULL == pReq->name || strlen(pReq->name) == 0) { + metaError("vgId:%d, %s failed at %s:%d since invalid name:%s, version:%" PRId64, vgId, __func__, __FILE__, __LINE__, + pReq->name, version); + return TSDB_CODE_INVALID_MSG; + } + + int32_t r = tdbTbGet(pMeta->pNameIdx, pReq->name, strlen(pReq->name) + 1, &value, &valueSize); + if (r == 0) { // name exists, check uid and type + int64_t uid = *(tb_uid_t *)value; + tdbFree(value); + + if (pReq->suid != uid) { + metaError("vgId:%d, %s failed at %s:%d since table %s uid:%" PRId64 " already exists, request uid:%" PRId64 + " version:%" PRId64, + vgId, __func__, __FILE__, __LINE__, pReq->name, uid, pReq->suid, version); + return TSDB_CODE_TDB_TABLE_ALREADY_EXIST; } - memcpy(p, pWp->pColCmpr, sizeof(SColCmpr) * nCols); - - SColCmpr *pCol = p + nCols; - pCol->id = pSchema->colId; - pCol->alg = compress; - pWp->nCols = nCols + 1; - pWp->version = ver; - pWp->pColCmpr = p; - } else { - for (int32_t i = 0; i < nCols; i++) { - SColCmpr *pOCmpr = &pWp->pColCmpr[i]; - if (pOCmpr->id == pSchema->colId) { - int32_t left = (nCols - i - 1) * sizeof(SColCmpr); - if (left) { - memmove(pWp->pColCmpr + i, pWp->pColCmpr + i + 1, left); - } - nCols--; - break; - } - } - pWp->nCols = nCols; - pWp->version = ver; - } - return 0; -} -static void metaGetEntryInfo(const SMetaEntry *pEntry, SMetaInfo *pInfo) { - pInfo->uid = pEntry->uid; - pInfo->version = pEntry->version; - if (pEntry->type == TSDB_SUPER_TABLE) { - pInfo->suid = pEntry->uid; - pInfo->skmVer = pEntry->stbEntry.schemaRow.version; - } else if (pEntry->type == TSDB_CHILD_TABLE) { - pInfo->suid = pEntry->ctbEntry.suid; - pInfo->skmVer = 0; - } else if (pEntry->type == TSDB_NORMAL_TABLE) { - pInfo->suid = 0; - pInfo->skmVer = pEntry->ntbEntry.schemaRow.version; - } else { - metaError("meta/table: invalide table type: %" PRId8 " get entry info failed.", pEntry->type); - } -} - -static int metaUpdateMetaRsp(tb_uid_t uid, char *tbName, SSchemaWrapper *pSchema, STableMetaRsp *pMetaRsp) { - pMetaRsp->pSchemas = taosMemoryMalloc(pSchema->nCols * sizeof(SSchema)); - if (NULL == pMetaRsp->pSchemas) { - return terrno; - } - - pMetaRsp->pSchemaExt = taosMemoryMalloc(pSchema->nCols * sizeof(SSchemaExt)); - if (pMetaRsp->pSchemaExt == NULL) { - taosMemoryFree(pMetaRsp->pSchemas); - return terrno; - } - - tstrncpy(pMetaRsp->tbName, tbName, TSDB_TABLE_NAME_LEN); - pMetaRsp->numOfColumns = pSchema->nCols; - pMetaRsp->tableType = TSDB_NORMAL_TABLE; - pMetaRsp->sversion = pSchema->version; - pMetaRsp->tuid = uid; - - memcpy(pMetaRsp->pSchemas, pSchema->pSchema, pSchema->nCols * sizeof(SSchema)); - - return 0; -} - -static int metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSchema *pSchema) { - int32_t code = 0; - -#ifdef USE_INVERTED_INDEX - if (pMeta->pTagIvtIdx == NULL || pCtbEntry == NULL) { - return TSDB_CODE_INVALID_PARA; - } - void *data = pCtbEntry->ctbEntry.pTags; - const char *tagName = pSchema->name; - - tb_uid_t suid = pCtbEntry->ctbEntry.suid; - tb_uid_t tuid = pCtbEntry->uid; - const void *pTagData = pCtbEntry->ctbEntry.pTags; - int32_t nTagData = 0; - - SArray *pTagVals = NULL; - code = tTagToValArray((const STag *)data, &pTagVals); - if (code) { - return code; - } - - SIndexMultiTerm *terms = indexMultiTermCreate(); - if (terms == NULL) { - return terrno; - } - - int16_t nCols = taosArrayGetSize(pTagVals); - for (int i = 0; i < nCols; i++) { - STagVal *pTagVal = (STagVal *)taosArrayGet(pTagVals, i); - char type = pTagVal->type; - - char *key = pTagVal->pKey; - int32_t nKey = strlen(key); - - SIndexTerm *term = NULL; - if (type == TSDB_DATA_TYPE_NULL) { - term = indexTermCreate(suid, ADD_VALUE, TSDB_DATA_TYPE_VARCHAR, key, nKey, NULL, 0); - } else if (type == TSDB_DATA_TYPE_NCHAR) { - if (pTagVal->nData > 0) { - char *val = taosMemoryCalloc(1, pTagVal->nData + VARSTR_HEADER_SIZE); - if (val == NULL) { - TAOS_CHECK_GOTO(terrno, NULL, _exception); - } - int32_t len = taosUcs4ToMbs((TdUcs4 *)pTagVal->pData, pTagVal->nData, val + VARSTR_HEADER_SIZE); - if (len < 0) { - TAOS_CHECK_GOTO(len, NULL, _exception); - } - memcpy(val, (uint16_t *)&len, VARSTR_HEADER_SIZE); - type = TSDB_DATA_TYPE_VARCHAR; - term = indexTermCreate(suid, ADD_VALUE, type, key, nKey, val, len); - taosMemoryFree(val); - } else if (pTagVal->nData == 0) { - term = indexTermCreate(suid, ADD_VALUE, TSDB_DATA_TYPE_VARCHAR, key, nKey, pTagVal->pData, 0); - } - } else if (type == TSDB_DATA_TYPE_DOUBLE) { - double val = *(double *)(&pTagVal->i64); - int len = sizeof(val); - term = indexTermCreate(suid, ADD_VALUE, type, key, nKey, (const char *)&val, len); - } else if (type == TSDB_DATA_TYPE_BOOL) { - int val = *(int *)(&pTagVal->i64); - int len = sizeof(val); - term = indexTermCreate(suid, ADD_VALUE, TSDB_DATA_TYPE_BOOL, key, nKey, (const char *)&val, len); - } - - if (term != NULL) { - int32_t ret = indexMultiTermAdd(terms, term); - if (ret < 0) { - metaError("vgId:%d, failed to add term to multi term, uid: %" PRId64 ", key: %s, type: %d, ret: %d", - TD_VID(pMeta->pVnode), tuid, key, type, ret); - } - } else { - code = terrno; - goto _exception; - } - } - code = indexJsonPut(pMeta->pTagIvtIdx, terms, tuid); - indexMultiTermDestroy(terms); - - taosArrayDestroy(pTagVals); -#endif - return code; -_exception: - indexMultiTermDestroy(terms); - taosArrayDestroy(pTagVals); - return code; -} -int metaDelJsonVarFromIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSchema *pSchema) { -#ifdef USE_INVERTED_INDEX - if (pMeta->pTagIvtIdx == NULL || pCtbEntry == NULL) { - return TSDB_CODE_INVALID_PARA; - } - void *data = pCtbEntry->ctbEntry.pTags; - const char *tagName = pSchema->name; - - tb_uid_t suid = pCtbEntry->ctbEntry.suid; - tb_uid_t tuid = pCtbEntry->uid; - const void *pTagData = pCtbEntry->ctbEntry.pTags; - int32_t nTagData = 0; - - SArray *pTagVals = NULL; - int32_t code = tTagToValArray((const STag *)data, &pTagVals); - if (code) { - return code; - } - - SIndexMultiTerm *terms = indexMultiTermCreate(); - if (terms == NULL) { - return terrno; - } - - int16_t nCols = taosArrayGetSize(pTagVals); - for (int i = 0; i < nCols; i++) { - STagVal *pTagVal = (STagVal *)taosArrayGet(pTagVals, i); - char type = pTagVal->type; - - char *key = pTagVal->pKey; - int32_t nKey = strlen(key); - - SIndexTerm *term = NULL; - if (type == TSDB_DATA_TYPE_NULL) { - term = indexTermCreate(suid, DEL_VALUE, TSDB_DATA_TYPE_VARCHAR, key, nKey, NULL, 0); - } else if (type == TSDB_DATA_TYPE_NCHAR) { - if (pTagVal->nData > 0) { - char *val = taosMemoryCalloc(1, pTagVal->nData + VARSTR_HEADER_SIZE); - if (val == NULL) { - TAOS_CHECK_GOTO(terrno, NULL, _exception); - } - int32_t len = taosUcs4ToMbs((TdUcs4 *)pTagVal->pData, pTagVal->nData, val + VARSTR_HEADER_SIZE); - if (len < 0) { - TAOS_CHECK_GOTO(len, NULL, _exception); - } - memcpy(val, (uint16_t *)&len, VARSTR_HEADER_SIZE); - type = TSDB_DATA_TYPE_VARCHAR; - term = indexTermCreate(suid, DEL_VALUE, type, key, nKey, val, len); - taosMemoryFree(val); - } else if (pTagVal->nData == 0) { - term = indexTermCreate(suid, DEL_VALUE, TSDB_DATA_TYPE_VARCHAR, key, nKey, pTagVal->pData, 0); - } - } else if (type == TSDB_DATA_TYPE_DOUBLE) { - double val = *(double *)(&pTagVal->i64); - int len = sizeof(val); - term = indexTermCreate(suid, DEL_VALUE, type, key, nKey, (const char *)&val, len); - } else if (type == TSDB_DATA_TYPE_BOOL) { - int val = *(int *)(&pTagVal->i64); - int len = sizeof(val); - term = indexTermCreate(suid, DEL_VALUE, TSDB_DATA_TYPE_BOOL, key, nKey, (const char *)&val, len); - } - if (term != NULL) { - int32_t ret = indexMultiTermAdd(terms, term); - if (ret < 0) { - metaError("vgId:%d, failed to add term to multi term, uid: %" PRId64 ", key: %s, type: %d, ret: %d", - TD_VID(pMeta->pVnode), tuid, key, type, ret); - } - } else { - code = terrno; - goto _exception; - } - } - code = indexJsonPut(pMeta->pTagIvtIdx, terms, tuid); - indexMultiTermDestroy(terms); - taosArrayDestroy(pTagVals); -#endif - return code; -_exception: - indexMultiTermDestroy(terms); - taosArrayDestroy(pTagVals); - return code; -} - -static inline void metaTimeSeriesNotifyCheck(SMeta *pMeta) { -#if defined(TD_ENTERPRISE) - int64_t nTimeSeries = metaGetTimeSeriesNum(pMeta, 0); - int64_t deltaTS = nTimeSeries - pMeta->pVnode->config.vndStats.numOfReportedTimeSeries; - if (deltaTS > tsTimeSeriesThreshold) { - if (0 == atomic_val_compare_exchange_8(&dmNotifyHdl.state, 1, 2)) { - if (tsem_post(&dmNotifyHdl.sem) != 0) { - metaError("vgId:%d, failed to post semaphore, errno:%d", TD_VID(pMeta->pVnode), errno); - } - } - } -#endif -} - -int metaCreateSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { - SMetaEntry me = {0}; - int kLen = 0; - int vLen = 0; - const void *pKey = NULL; - const void *pVal = NULL; - void *pBuf = NULL; - int32_t szBuf = 0; - void *p = NULL; - int32_t code = 0; - - // validate req - void *pData = NULL; - int nData = 0; - if (tdbTbGet(pMeta->pNameIdx, pReq->name, strlen(pReq->name) + 1, &pData, &nData) == 0) { - tb_uid_t uid = *(tb_uid_t *)pData; - tdbFree(pData); - SMetaInfo info; if (metaGetInfo(pMeta, uid, &info, NULL) == TSDB_CODE_NOT_FOUND) { - return terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST; + metaError("vgId:%d, %s failed at %s:%d since table %s uid:%" PRId64 + " not found, this is an internal error in meta, version:%" PRId64, + vgId, __func__, __FILE__, __LINE__, pReq->name, uid, version); + return TSDB_CODE_PAR_TABLE_NOT_EXIST; } + if (info.uid == info.suid) { - return 0; + return TSDB_CODE_TDB_STB_ALREADY_EXIST; } else { - return terrno = TSDB_CODE_TDB_TABLE_ALREADY_EXIST; + metaError("vgId:%d, %s failed at %s:%d since table %s uid:%" PRId64 + " already exists but not a super table, version:%" PRId64, + vgId, __func__, __FILE__, __LINE__, pReq->name, uid, version); + return TSDB_CODE_TDB_TABLE_ALREADY_EXIST; } } - // set structs - me.version = version; - me.type = TSDB_SUPER_TABLE; - me.uid = pReq->suid; - me.name = pReq->name; - me.stbEntry.schemaRow = pReq->schemaRow; - me.stbEntry.schemaTag = pReq->schemaTag; + // check suid + if (metaGetInfo(pMeta, pReq->suid, &info, NULL) != TSDB_CODE_NOT_FOUND) { + metaError("vgId:%d, %s failed at %s:%d since table with uid:%" PRId64 " already exist, name:%s version:%" PRId64, + vgId, __func__, __FILE__, __LINE__, pReq->suid, pReq->name, version); + return TSDB_CODE_INVALID_MSG; + } + + // other fields + if (pReq->schemaRow.nCols <= 0) { + metaError("vgId:%d, %s failed at %s:%d since invalid row schema, version:%" PRId64, vgId, __func__, __FILE__, + __LINE__, version); + return TSDB_CODE_INVALID_MSG; + } + + if (pReq->schemaTag.nCols <= 0) { + metaError("vgId:%d, %s failed at %s:%d since invalid tag schema, version:%" PRId64, vgId, __func__, __FILE__, + __LINE__, version); + return TSDB_CODE_INVALID_MSG; + } + + return TSDB_CODE_SUCCESS; +} + +// Create Super Table +int32_t metaCreateSuperTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { + int32_t code = TSDB_CODE_SUCCESS; + + // check request + code = metaCheckCreateSuperTableReq(pMeta, version, pReq); + if (code != TSDB_CODE_SUCCESS) { + if (code == TSDB_CODE_TDB_STB_ALREADY_EXIST) { + metaWarn("vgId:%d, super table %s uid:%" PRId64 " already exists, version:%" PRId64, TD_VID(pMeta->pVnode), + pReq->name, pReq->suid, version); + TAOS_RETURN(TSDB_CODE_SUCCESS); + } else { + TAOS_RETURN(code); + } + } + + // handle entry + SMetaEntry entry = { + .version = version, + .type = TSDB_SUPER_TABLE, + .uid = pReq->suid, + .name = pReq->name, + .stbEntry.schemaRow = pReq->schemaRow, + .stbEntry.schemaTag = pReq->schemaTag, + }; if (pReq->rollup) { - TABLE_SET_ROLLUP(me.flags); - me.stbEntry.rsmaParam = pReq->rsmaParam; + TABLE_SET_ROLLUP(entry.flags); + entry.stbEntry.rsmaParam = pReq->rsmaParam; } if (pReq->colCmpred) { - TABLE_SET_COL_COMPRESSED(me.flags); - me.colCmpr = pReq->colCmpr; + TABLE_SET_COL_COMPRESSED(entry.flags); + entry.colCmpr = pReq->colCmpr; } - code = metaHandleEntry(pMeta, &me); - if (code) goto _err; - - ++pMeta->pVnode->config.vndStats.numOfSTables; - - pMeta->changed = true; - metaDebug("vgId:%d, stb:%s is created, suid:%" PRId64, TD_VID(pMeta->pVnode), pReq->name, pReq->suid); - - return 0; - -_err: - metaError("vgId:%d, failed to create stb:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, pReq->suid, - tstrerror(terrno)); - return code; -} - -int metaDropSTable(SMeta *pMeta, int64_t verison, SVDropStbReq *pReq, SArray *tbUidList) { - void *pKey = NULL; - int nKey = 0; - void *pData = NULL; - int nData = 0; - int c = 0; - int rc = 0; - int32_t lino; - int32_t ret; - - // check if super table exists - rc = tdbTbGet(pMeta->pNameIdx, pReq->name, strlen(pReq->name) + 1, &pData, &nData); - if (rc < 0 || *(tb_uid_t *)pData != pReq->suid) { - tdbFree(pData); - return terrno = TSDB_CODE_TDB_STB_NOT_EXIST; - } - - // drop all child tables - TBC *pCtbIdxc = NULL; - - rc = tdbTbcOpen(pMeta->pCtbIdx, &pCtbIdxc, NULL); - if (rc) { - return (terrno = rc); - } - - rc = tdbTbcMoveTo(pCtbIdxc, &(SCtbIdxKey){.suid = pReq->suid, .uid = INT64_MIN}, sizeof(SCtbIdxKey), &c); - if (rc < 0) { - tdbTbcClose(pCtbIdxc); - metaWLock(pMeta); - goto _drop_super_table; - } - - for (;;) { - rc = tdbTbcNext(pCtbIdxc, &pKey, &nKey, NULL, NULL); - if (rc < 0) break; - - if (((SCtbIdxKey *)pKey)->suid < pReq->suid) { - continue; - } else if (((SCtbIdxKey *)pKey)->suid > pReq->suid) { - break; - } - - if (taosArrayPush(tbUidList, &(((SCtbIdxKey *)pKey)->uid)) == NULL) { - tdbFree(pKey); - tdbTbcClose(pCtbIdxc); - return terrno; - } - } - - tdbTbcClose(pCtbIdxc); - - ret = tsdbCacheDropSubTables(pMeta->pVnode->pTsdb, tbUidList, pReq->suid); - if (ret < 0) { - metaError("vgId:%d, failed to drop stb:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, pReq->suid, - tstrerror(terrno)); - } - - metaWLock(pMeta); - - for (int32_t iChild = 0; iChild < taosArrayGetSize(tbUidList); iChild++) { - tb_uid_t uid = *(tb_uid_t *)taosArrayGet(tbUidList, iChild); - ret = metaDropTableByUid(pMeta, uid, NULL, NULL, NULL); - if (ret < 0) { - metaError("vgId:%d, failed to drop stb:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, - pReq->suid, tstrerror(terrno)); - } - } - - // drop super table -_drop_super_table: - tdbTbGet(pMeta->pUidIdx, &pReq->suid, sizeof(tb_uid_t), &pData, &nData); - ret = tdbTbDelete(pMeta->pTbDb, &(STbDbKey){.version = ((SUidIdxVal *)pData)[0].version, .uid = pReq->suid}, - sizeof(STbDbKey), pMeta->txn); - if (ret < 0) { - metaError("vgId:%d, failed to drop stb:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, pReq->suid, - tstrerror(terrno)); - } - - ret = tdbTbDelete(pMeta->pNameIdx, pReq->name, strlen(pReq->name) + 1, pMeta->txn); - if (ret < 0) { - metaError("vgId:%d, failed to drop stb:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, pReq->suid, - tstrerror(terrno)); - } - - ret = tdbTbDelete(pMeta->pUidIdx, &pReq->suid, sizeof(tb_uid_t), pMeta->txn); - if (ret < 0) { - metaError("vgId:%d, failed to drop stb:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, pReq->suid, - tstrerror(terrno)); - } - - ret = tdbTbDelete(pMeta->pSuidIdx, &pReq->suid, sizeof(tb_uid_t), pMeta->txn); - if (ret < 0) { - metaError("vgId:%d, failed to drop stb:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, pReq->suid, - tstrerror(terrno)); - } - - ret = metaStatsCacheDrop(pMeta, pReq->suid); - if (ret < 0) { - metaError("vgId:%d, failed to drop stb:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, pReq->suid, - tstrerror(terrno)); - } - - metaULock(pMeta); - - metaUpdTimeSeriesNum(pMeta); - - pMeta->changed = true; - -_exit: - tdbFree(pKey); - tdbFree(pData); - metaDebug("vgId:%d, super table %s uid:%" PRId64 " is dropped", TD_VID(pMeta->pVnode), pReq->name, pReq->suid); - return 0; -} - -static int32_t metaGetSubtables(SMeta *pMeta, int64_t suid, SArray *uids) { - if (!uids) return TSDB_CODE_INVALID_PARA; - - int c = 0; - void *pKey = NULL; - int nKey = 0; - TBC *pCtbIdxc = NULL; - - TAOS_CHECK_RETURN(tdbTbcOpen(pMeta->pCtbIdx, &pCtbIdxc, NULL)); - int rc = tdbTbcMoveTo(pCtbIdxc, &(SCtbIdxKey){.suid = suid, .uid = INT64_MIN}, sizeof(SCtbIdxKey), &c); - if (rc < 0) { - tdbTbcClose(pCtbIdxc); - metaWLock(pMeta); - return 0; - } - - for (;;) { - rc = tdbTbcNext(pCtbIdxc, &pKey, &nKey, NULL, NULL); - if (rc < 0) break; - - if (((SCtbIdxKey *)pKey)->suid < suid) { - continue; - } else if (((SCtbIdxKey *)pKey)->suid > suid) { - break; - } - - if (taosArrayPush(uids, &(((SCtbIdxKey *)pKey)->uid)) == NULL) { - tdbFree(pKey); - tdbTbcClose(pCtbIdxc); - return terrno; - } - } - - tdbFree(pKey); - - tdbTbcClose(pCtbIdxc); - return 0; -} - -int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { - SMetaEntry oStbEntry = {0}; - SMetaEntry nStbEntry = {0}; - TBC *pUidIdxc = NULL; - TBC *pTbDbc = NULL; - const void *pData; - int nData; - int64_t oversion; - SDecoder dc = {0}; - int32_t ret; - int32_t c = -2; - - TAOS_CHECK_RETURN(tdbTbcOpen(pMeta->pUidIdx, &pUidIdxc, NULL)); - ret = tdbTbcMoveTo(pUidIdxc, &pReq->suid, sizeof(tb_uid_t), &c); - if (ret < 0 || c) { - tdbTbcClose(pUidIdxc); - - return terrno = TSDB_CODE_TDB_STB_NOT_EXIST; - } - - ret = tdbTbcGet(pUidIdxc, NULL, NULL, &pData, &nData); - if (ret < 0) { - tdbTbcClose(pUidIdxc); - - return terrno = TSDB_CODE_TDB_STB_NOT_EXIST; - } - - oversion = ((SUidIdxVal *)pData)[0].version; - - TAOS_CHECK_RETURN(tdbTbcOpen(pMeta->pTbDb, &pTbDbc, NULL)); - ret = tdbTbcMoveTo(pTbDbc, &((STbDbKey){.uid = pReq->suid, .version = oversion}), sizeof(STbDbKey), &c); - if (!(ret == 0 && c == 0)) { - tdbTbcClose(pUidIdxc); - tdbTbcClose(pTbDbc); - - metaError("meta/table: invalide ret: %" PRId32 " or c: %" PRId32 "alter stb failed.", ret, c); - return terrno = TSDB_CODE_TDB_STB_NOT_EXIST; - } - - ret = tdbTbcGet(pTbDbc, NULL, NULL, &pData, &nData); - if (ret < 0) { - tdbTbcClose(pUidIdxc); - tdbTbcClose(pTbDbc); - - return terrno = TSDB_CODE_TDB_STB_NOT_EXIST; - } - - if ((oStbEntry.pBuf = taosMemoryMalloc(nData)) == NULL) { - tdbTbcClose(pTbDbc); - tdbTbcClose(pUidIdxc); - return terrno; - } - memcpy(oStbEntry.pBuf, pData, nData); - tDecoderInit(&dc, oStbEntry.pBuf, nData); - ret = metaDecodeEntry(&dc, &oStbEntry); - if (ret < 0) { - metaError("vgId:%d, failed to decode stb:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, - pReq->suid, tstrerror(ret)); - tDecoderClear(&dc); - tdbTbcClose(pTbDbc); - tdbTbcClose(pUidIdxc); - return terrno; - } - - nStbEntry.version = version; - nStbEntry.type = TSDB_SUPER_TABLE; - nStbEntry.uid = pReq->suid; - nStbEntry.name = pReq->name; - nStbEntry.stbEntry.schemaRow = pReq->schemaRow; - nStbEntry.stbEntry.schemaTag = pReq->schemaTag; - nStbEntry.colCmpr = pReq->colCmpr; - TABLE_SET_COL_COMPRESSED(nStbEntry.flags); - - int nCols = pReq->schemaRow.nCols; - int onCols = oStbEntry.stbEntry.schemaRow.nCols; - int32_t deltaCol = nCols - onCols; - bool updStat = deltaCol != 0 && !metaTbInFilterCache(pMeta, pReq->name, 1); - - if (!TSDB_CACHE_NO(pMeta->pVnode->config)) { - STsdb *pTsdb = pMeta->pVnode->pTsdb; - SArray *uids = taosArrayInit(8, sizeof(int64_t)); - if (uids == NULL) { - if (oStbEntry.pBuf) taosMemoryFree(oStbEntry.pBuf); - tDecoderClear(&dc); - tdbTbcClose(pTbDbc); - tdbTbcClose(pUidIdxc); - return terrno; - } - if (deltaCol == 1) { - int16_t cid = pReq->schemaRow.pSchema[nCols - 1].colId; - int8_t col_type = pReq->schemaRow.pSchema[nCols - 1].type; - - TAOS_CHECK_RETURN(metaGetSubtables(pMeta, pReq->suid, uids)); - TAOS_CHECK_RETURN(tsdbCacheNewSTableColumn(pTsdb, uids, cid, col_type)); - } else if (deltaCol == -1) { - int16_t cid = -1; - bool hasPrimaryKey = false; - if (onCols >= 2) { - hasPrimaryKey = (oStbEntry.stbEntry.schemaRow.pSchema[1].flags & COL_IS_KEY) ? true : false; - } - for (int i = 0, j = 0; i < nCols && j < onCols; ++i, ++j) { - if (pReq->schemaRow.pSchema[i].colId != oStbEntry.stbEntry.schemaRow.pSchema[j].colId) { - cid = oStbEntry.stbEntry.schemaRow.pSchema[j].colId; - break; - } - } - - if (cid != -1) { - TAOS_CHECK_RETURN(metaGetSubtables(pMeta, pReq->suid, uids)); - TAOS_CHECK_RETURN(tsdbCacheDropSTableColumn(pTsdb, uids, cid, hasPrimaryKey)); - } - } - if (uids) taosArrayDestroy(uids); - - tsdbCacheInvalidateSchema(pTsdb, pReq->suid, -1, pReq->schemaRow.version); - } - - metaWLock(pMeta); - // compare two entry - if (oStbEntry.stbEntry.schemaRow.version != pReq->schemaRow.version) { - ret = metaSaveToSkmDb(pMeta, &nStbEntry); - if (ret < 0) { - metaError("vgId:%d, failed to save skm db:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, - pReq->suid, tstrerror(ret)); - } - } - - // update table.db - ret = metaSaveToTbDb(pMeta, &nStbEntry); - if (ret < 0) { - metaError("vgId:%d, failed to save tb db:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, - pReq->suid, tstrerror(ret)); - } - - // update uid index - ret = metaUpdateUidIdx(pMeta, &nStbEntry); - if (ret < 0) { - metaError("vgId:%d, failed to update uid idx:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, - pReq->suid, tstrerror(ret)); - } - - // metaStatsCacheDrop(pMeta, nStbEntry.uid); - - if (updStat) { - metaUpdateStbStats(pMeta, pReq->suid, 0, deltaCol); - } - metaULock(pMeta); - - if (updStat) { - int64_t ctbNum; - ret = metaGetStbStats(pMeta->pVnode, pReq->suid, &ctbNum, NULL); - if (ret < 0) { - metaError("vgId:%d, failed to get stb stats:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, - pReq->suid, tstrerror(ret)); - } - pMeta->pVnode->config.vndStats.numOfTimeSeries += (ctbNum * deltaCol); - metaTimeSeriesNotifyCheck(pMeta); - } - - pMeta->changed = true; - -_exit: - if (oStbEntry.pBuf) taosMemoryFree(oStbEntry.pBuf); - tDecoderClear(&dc); - tdbTbcClose(pTbDbc); - tdbTbcClose(pUidIdxc); - return 0; -} -int metaAddIndexToSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { - SMetaEntry oStbEntry = {0}; - SMetaEntry nStbEntry = {0}; - STbDbKey tbDbKey = {0}; - - TBC *pUidIdxc = NULL; - TBC *pTbDbc = NULL; - void *pData = NULL; - int nData = 0; - int64_t oversion; - SDecoder dc = {0}; - int32_t ret; - int32_t c = -2; - tb_uid_t suid = pReq->suid; - int32_t code = 0; - - // get super table - if ((code = tdbTbGet(pMeta->pUidIdx, &suid, sizeof(tb_uid_t), &pData, &nData)) != 0) { - goto _err; - } - - tbDbKey.uid = suid; - tbDbKey.version = ((SUidIdxVal *)pData)[0].version; - if ((code = tdbTbGet(pMeta->pTbDb, &tbDbKey, sizeof(tbDbKey), &pData, &nData)) != 0) { - goto _err; - } - - tDecoderInit(&dc, pData, nData); - code = metaDecodeEntry(&dc, &oStbEntry); - if (code < 0) { - goto _err; - } - - if (oStbEntry.stbEntry.schemaTag.pSchema == NULL || oStbEntry.stbEntry.schemaTag.pSchema == NULL) { - code = TSDB_CODE_INVALID_PARA; - goto _err; - } - - if (oStbEntry.stbEntry.schemaTag.version == pReq->schemaTag.version) { - code = TSDB_CODE_INVALID_PARA; - goto _err; - } - - if (oStbEntry.stbEntry.schemaTag.nCols != pReq->schemaTag.nCols) { - code = TSDB_CODE_INVALID_PARA; - goto _err; - } - - int diffIdx = -1; - for (int i = 0; i < pReq->schemaTag.nCols; i++) { - SSchema *pNew = pReq->schemaTag.pSchema + i; - SSchema *pOld = oStbEntry.stbEntry.schemaTag.pSchema + i; - if (pNew->type != pOld->type || pNew->colId != pOld->colId || pNew->bytes != pOld->bytes || - strncmp(pOld->name, pNew->name, sizeof(pNew->name))) { - code = TSDB_CODE_INVALID_PARA; - goto _err; - } - if (IS_IDX_ON(pNew) && !IS_IDX_ON(pOld)) { - // if (diffIdx != -1) goto _err; - diffIdx = i; - break; - } - } - - if (diffIdx == -1) { - code = TSDB_CODE_INVALID_PARA; - goto _err; - } - - // Get target schema info - SSchemaWrapper *pTagSchema = &pReq->schemaTag; - if (pTagSchema->nCols == 1 && pTagSchema->pSchema[0].type == TSDB_DATA_TYPE_JSON) { - code = TSDB_CODE_INVALID_PARA; - goto _err; - } - SSchema *pCol = pTagSchema->pSchema + diffIdx; - - /* - * iterator all pTdDbc by uid and version - */ - TBC *pCtbIdxc = NULL; - code = tdbTbcOpen(pMeta->pCtbIdx, &pCtbIdxc, NULL); - if (code != 0) { - goto _err; - } - - code = tdbTbcMoveTo(pCtbIdxc, &(SCtbIdxKey){.suid = suid, .uid = INT64_MIN}, sizeof(SCtbIdxKey), &c); - if (code < 0) { - tdbTbcClose(pCtbIdxc); - goto _err; - } - for (;;) { - void *pKey = NULL, *pVal = NULL; - int nKey = 0, nVal = 0; - code = tdbTbcNext(pCtbIdxc, &pKey, &nKey, &pVal, &nVal); - if (code < 0) { - tdbFree(pKey); - tdbFree(pVal); - tdbTbcClose(pCtbIdxc); - pCtbIdxc = NULL; - break; - } - if (((SCtbIdxKey *)pKey)->suid != suid) { - tdbFree(pKey); - tdbFree(pVal); - continue; - } - STagIdxKey *pTagIdxKey = NULL; - int32_t nTagIdxKey; - - const void *pTagData = NULL; - int32_t nTagData = 0; - - SCtbIdxKey *table = (SCtbIdxKey *)pKey; - STagVal tagVal = {.cid = pCol->colId}; - if (tTagGet((const STag *)pVal, &tagVal)) { - if (IS_VAR_DATA_TYPE(pCol->type)) { - pTagData = tagVal.pData; - nTagData = (int32_t)tagVal.nData; - } else { - pTagData = &(tagVal.i64); - nTagData = tDataTypes[pCol->type].bytes; - } - } else { - if (!IS_VAR_DATA_TYPE(pCol->type)) { - nTagData = tDataTypes[pCol->type].bytes; - } - } - code = metaCreateTagIdxKey(suid, pCol->colId, pTagData, nTagData, pCol->type, table->uid, &pTagIdxKey, &nTagIdxKey); - tdbFree(pKey); - tdbFree(pVal); - if (code < 0) { - metaDestroyTagIdxKey(pTagIdxKey); - tdbTbcClose(pCtbIdxc); - goto _err; - } - - metaWLock(pMeta); - ret = tdbTbUpsert(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, NULL, 0, pMeta->txn); - if (ret < 0) { - metaError("vgId:%d, failed to upsert tag idx key:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, - pReq->suid, tstrerror(ret)); - } - metaULock(pMeta); - - metaDestroyTagIdxKey(pTagIdxKey); - pTagIdxKey = NULL; - } - - nStbEntry.version = version; - nStbEntry.type = TSDB_SUPER_TABLE; - nStbEntry.uid = pReq->suid; - nStbEntry.name = pReq->name; - nStbEntry.stbEntry.schemaRow = pReq->schemaRow; - nStbEntry.stbEntry.schemaTag = pReq->schemaTag; - nStbEntry.colCmpr = pReq->colCmpr; - - metaWLock(pMeta); - // update table.db - ret = metaSaveToTbDb(pMeta, &nStbEntry); - if (ret < 0) { - metaError("vgId:%d, failed to save tb db:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, - pReq->suid, tstrerror(ret)); - } - // update uid index - ret = metaUpdateUidIdx(pMeta, &nStbEntry); - if (ret < 0) { - metaError("vgId:%d, failed to update uid idx:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, - pReq->suid, tstrerror(ret)); - } - metaULock(pMeta); - - if (oStbEntry.pBuf) taosMemoryFree(oStbEntry.pBuf); - tDecoderClear(&dc); - tdbFree(pData); - - tdbTbcClose(pCtbIdxc); - return TSDB_CODE_SUCCESS; -_err: - if (oStbEntry.pBuf) taosMemoryFree(oStbEntry.pBuf); - tDecoderClear(&dc); - tdbFree(pData); - - return code; -} -int metaDropIndexFromSTable(SMeta *pMeta, int64_t version, SDropIndexReq *pReq) { - int32_t code = 0; - SMetaEntry oStbEntry = {0}; - SMetaEntry nStbEntry = {0}; - - STbDbKey tbDbKey = {0}; - TBC *pUidIdxc = NULL; - TBC *pTbDbc = NULL; - int ret = 0; - int c = -2; - void *pData = NULL; - int nData = 0; - int64_t oversion; - SDecoder dc = {0}; - - tb_uid_t suid = pReq->stbUid; - - if ((code = tdbTbGet(pMeta->pUidIdx, &suid, sizeof(tb_uid_t), &pData, &nData)) != 0) { - goto _err; - } - - tbDbKey.uid = suid; - tbDbKey.version = ((SUidIdxVal *)pData)[0].version; - if ((code = tdbTbGet(pMeta->pTbDb, &tbDbKey, sizeof(tbDbKey), &pData, &nData)) != 0) { - goto _err; - } - - tDecoderInit(&dc, pData, nData); - code = metaDecodeEntry(&dc, &oStbEntry); - if (code != 0) { - goto _err; - } - - SSchema *pCol = NULL; - int32_t colId = -1; - for (int i = 0; i < oStbEntry.stbEntry.schemaTag.nCols; i++) { - SSchema *schema = oStbEntry.stbEntry.schemaTag.pSchema + i; - if (0 == strncmp(schema->name, pReq->colName, sizeof(pReq->colName))) { - if (IS_IDX_ON(schema)) { - pCol = schema; - } - break; - } - } - - if (pCol == NULL) { - metaError("vgId:%d, failed to drop index on %s.%s,since %s", TD_VID(pMeta->pVnode), pReq->stb, pReq->colName, - tstrerror(TSDB_CODE_VND_COL_NOT_EXISTS)); - code = 0; - - goto _err; - } - - /* - * iterator all pTdDbc by uid and version - */ - TBC *pCtbIdxc = NULL; - code = tdbTbcOpen(pMeta->pCtbIdx, &pCtbIdxc, NULL); - if (code != 0) { - goto _err; - } - - code = tdbTbcMoveTo(pCtbIdxc, &(SCtbIdxKey){.suid = suid, .uid = INT64_MIN}, sizeof(SCtbIdxKey), &c); - if (code < 0) { - tdbTbcClose(pCtbIdxc); - goto _err; - } - for (;;) { - void *pKey = NULL, *pVal = NULL; - int nKey = 0, nVal = 0; - - code = tdbTbcNext(pCtbIdxc, &pKey, &nKey, &pVal, &nVal); - if (code < 0) { - tdbFree(pKey); - tdbFree(pVal); - tdbTbcClose(pCtbIdxc); - pCtbIdxc = NULL; - break; - } - if (((SCtbIdxKey *)pKey)->suid != suid) { - tdbFree(pKey); - tdbFree(pVal); - continue; - } - STagIdxKey *pTagIdxKey = NULL; - int32_t nTagIdxKey; - - const void *pTagData = NULL; - int32_t nTagData = 0; - - SCtbIdxKey *table = (SCtbIdxKey *)pKey; - STagVal tagVal = {.cid = pCol->colId}; - if (tTagGet((const STag *)pVal, &tagVal)) { - if (IS_VAR_DATA_TYPE(pCol->type)) { - pTagData = tagVal.pData; - nTagData = (int32_t)tagVal.nData; - } else { - pTagData = &(tagVal.i64); - nTagData = tDataTypes[pCol->type].bytes; - } - } else { - if (!IS_VAR_DATA_TYPE(pCol->type)) { - nTagData = tDataTypes[pCol->type].bytes; - } - } - - code = metaCreateTagIdxKey(suid, pCol->colId, pTagData, nTagData, pCol->type, table->uid, &pTagIdxKey, &nTagIdxKey); - tdbFree(pKey); - tdbFree(pVal); - if (code < 0) { - metaDestroyTagIdxKey(pTagIdxKey); - tdbTbcClose(pCtbIdxc); - goto _err; - } - - metaWLock(pMeta); - ret = tdbTbDelete(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, pMeta->txn); - if (ret < 0) { - metaError("vgId:%d, failed to delete tag idx key:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->stb, - pReq->stbUid, tstrerror(ret)); - } - metaULock(pMeta); - metaDestroyTagIdxKey(pTagIdxKey); - pTagIdxKey = NULL; - } - - // clear idx flag - SSCHMEA_SET_IDX_OFF(pCol); - - nStbEntry.version = version; - nStbEntry.type = TSDB_SUPER_TABLE; - nStbEntry.uid = oStbEntry.uid; - nStbEntry.name = oStbEntry.name; - - SSchemaWrapper *row = tCloneSSchemaWrapper(&oStbEntry.stbEntry.schemaRow); - SSchemaWrapper *tag = tCloneSSchemaWrapper(&oStbEntry.stbEntry.schemaTag); - SColCmprWrapper *cmpr = tCloneSColCmprWrapper(&oStbEntry.colCmpr); - if (row == NULL || tag == NULL || cmpr == NULL) { - tDeleteSchemaWrapper(row); - tDeleteSchemaWrapper(tag); - tDeleteSColCmprWrapper(cmpr); - code = TSDB_CODE_OUT_OF_MEMORY; - - tdbTbcClose(pCtbIdxc); - goto _err; - } - - nStbEntry.stbEntry.schemaRow = *row; - nStbEntry.stbEntry.schemaTag = *tag; - nStbEntry.stbEntry.rsmaParam = oStbEntry.stbEntry.rsmaParam; - nStbEntry.colCmpr = *cmpr; - - nStbEntry.colCmpr = oStbEntry.colCmpr; - - metaWLock(pMeta); - // update table.db - ret = metaSaveToTbDb(pMeta, &nStbEntry); - if (ret < 0) { - metaError("vgId:%d, failed to save tb db:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->stb, - pReq->stbUid, tstrerror(ret)); - } - // update uid index - ret = metaUpdateUidIdx(pMeta, &nStbEntry); - if (ret < 0) { - metaError("vgId:%d, failed to update uid idx:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->stb, - pReq->stbUid, tstrerror(ret)); - } - metaULock(pMeta); - - tDeleteSchemaWrapper(tag); - tDeleteSchemaWrapper(row); - tDeleteSColCmprWrapper(cmpr); - - if (oStbEntry.pBuf) taosMemoryFree(oStbEntry.pBuf); - tDecoderClear(&dc); - tdbFree(pData); - - tdbTbcClose(pCtbIdxc); - return TSDB_CODE_SUCCESS; -_err: - if (oStbEntry.pBuf) taosMemoryFree(oStbEntry.pBuf); - tDecoderClear(&dc); - tdbFree(pData); - - return code; -} - -int metaCreateTable(SMeta *pMeta, int64_t ver, SVCreateTbReq *pReq, STableMetaRsp **pMetaRsp) { - SMetaEntry me = {0}; - SMetaReader mr = {0}; - int32_t ret; - - // validate message - if (pReq->type != TSDB_CHILD_TABLE && pReq->type != TSDB_NORMAL_TABLE) { - terrno = TSDB_CODE_INVALID_MSG; - goto _err; - } - - if (pReq->type == TSDB_CHILD_TABLE) { - tb_uid_t suid = metaGetTableEntryUidByName(pMeta, pReq->ctb.stbName); - if (suid != pReq->ctb.suid) { - return terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST; - } - } - - // validate req - metaReaderDoInit(&mr, pMeta, META_READER_LOCK); - if (metaGetTableEntryByName(&mr, pReq->name) == 0) { - if (pReq->type == TSDB_CHILD_TABLE && pReq->ctb.suid != mr.me.ctbEntry.suid) { - metaReaderClear(&mr); - return terrno = TSDB_CODE_TDB_TABLE_IN_OTHER_STABLE; - } - pReq->uid = mr.me.uid; - if (pReq->type == TSDB_CHILD_TABLE) { - pReq->ctb.suid = mr.me.ctbEntry.suid; - } - metaReaderClear(&mr); - return terrno = TSDB_CODE_TDB_TABLE_ALREADY_EXIST; - } else if (terrno == TSDB_CODE_PAR_TABLE_NOT_EXIST) { - terrno = TSDB_CODE_SUCCESS; - } - metaReaderClear(&mr); - - bool sysTbl = (pReq->type == TSDB_CHILD_TABLE) && metaTbInFilterCache(pMeta, pReq->ctb.stbName, 1); - - if (!sysTbl && ((terrno = grantCheck(TSDB_GRANT_TIMESERIES)) < 0)) goto _err; - - // build SMetaEntry - SVnodeStats *pStats = &pMeta->pVnode->config.vndStats; - me.version = ver; - me.type = pReq->type; - me.uid = pReq->uid; - me.name = pReq->name; - if (me.type == TSDB_CHILD_TABLE) { - me.ctbEntry.btime = pReq->btime; - me.ctbEntry.ttlDays = pReq->ttl; - me.ctbEntry.commentLen = pReq->commentLen; - me.ctbEntry.comment = pReq->comment; - me.ctbEntry.suid = pReq->ctb.suid; - me.ctbEntry.pTags = pReq->ctb.pTag; - -#ifdef TAG_FILTER_DEBUG - SArray *pTagVals = NULL; - int32_t code = tTagToValArray((STag *)pReq->ctb.pTag, &pTagVals); - for (int i = 0; i < taosArrayGetSize(pTagVals); i++) { - STagVal *pTagVal = (STagVal *)taosArrayGet(pTagVals, i); - - if (IS_VAR_DATA_TYPE(pTagVal->type)) { - char *buf = taosMemoryCalloc(pTagVal->nData + 1, 1); - memcpy(buf, pTagVal->pData, pTagVal->nData); - metaDebug("metaTag table:%s varchar index:%d cid:%d type:%d value:%s", pReq->name, i, pTagVal->cid, - pTagVal->type, buf); - taosMemoryFree(buf); - } else { - double val = 0; - GET_TYPED_DATA(val, double, pTagVal->type, &pTagVal->i64); - metaDebug("metaTag table:%s number index:%d cid:%d type:%d value:%f", pReq->name, i, pTagVal->cid, - pTagVal->type, val); - } - } -#endif - - ++pStats->numOfCTables; - - if (!sysTbl) { - int32_t nCols = 0; - ret = metaGetStbStats(pMeta->pVnode, me.ctbEntry.suid, 0, &nCols); - if (ret < 0) { - metaError("vgId:%d, failed to get stb stats:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, - pReq->ctb.suid, tstrerror(ret)); - } - pStats->numOfTimeSeries += nCols - 1; - } - - metaWLock(pMeta); - metaUpdateStbStats(pMeta, me.ctbEntry.suid, 1, 0); - ret = metaUidCacheClear(pMeta, me.ctbEntry.suid); - if (ret < 0) { - metaError("vgId:%d, failed to clear uid cache:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, - pReq->ctb.suid, tstrerror(ret)); - } - ret = metaTbGroupCacheClear(pMeta, me.ctbEntry.suid); - if (ret < 0) { - metaError("vgId:%d, failed to clear group cache:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, - pReq->ctb.suid, tstrerror(ret)); - } - metaULock(pMeta); - - if (!TSDB_CACHE_NO(pMeta->pVnode->config)) { - ret = tsdbCacheNewTable(pMeta->pVnode->pTsdb, me.uid, me.ctbEntry.suid, NULL); - if (ret < 0) { - metaError("vgId:%d, failed to create table:%s since %s", TD_VID(pMeta->pVnode), pReq->name, tstrerror(ret)); - goto _err; - } - } + code = metaHandleEntry2(pMeta, &entry); + if (TSDB_CODE_SUCCESS == code) { + metaInfo("vgId:%d, super table %s suid:%" PRId64 " is created, version:%" PRId64, TD_VID(pMeta->pVnode), pReq->name, + pReq->suid, version); } else { - me.ntbEntry.btime = pReq->btime; - me.ntbEntry.ttlDays = pReq->ttl; - me.ntbEntry.commentLen = pReq->commentLen; - me.ntbEntry.comment = pReq->comment; - me.ntbEntry.schemaRow = pReq->ntb.schemaRow; - me.ntbEntry.ncid = me.ntbEntry.schemaRow.pSchema[me.ntbEntry.schemaRow.nCols - 1].colId + 1; - me.colCmpr = pReq->colCmpr; - TABLE_SET_COL_COMPRESSED(me.flags); - - ++pStats->numOfNTables; - pStats->numOfNTimeSeries += me.ntbEntry.schemaRow.nCols - 1; - - if (!TSDB_CACHE_NO(pMeta->pVnode->config)) { - ret = tsdbCacheNewTable(pMeta->pVnode->pTsdb, me.uid, -1, &me.ntbEntry.schemaRow); - if (ret < 0) { - metaError("vgId:%d, failed to create table:%s since %s", TD_VID(pMeta->pVnode), pReq->name, tstrerror(ret)); - goto _err; - } - } + metaError("vgId:%d, failed to create stb:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, + pReq->suid, tstrerror(code)); } - - if (metaHandleEntry(pMeta, &me) < 0) goto _err; - - metaTimeSeriesNotifyCheck(pMeta); - - if (pMetaRsp) { - *pMetaRsp = taosMemoryCalloc(1, sizeof(STableMetaRsp)); - - if (*pMetaRsp) { - if (me.type == TSDB_CHILD_TABLE) { - (*pMetaRsp)->tableType = TSDB_CHILD_TABLE; - (*pMetaRsp)->tuid = pReq->uid; - (*pMetaRsp)->suid = pReq->ctb.suid; - strcpy((*pMetaRsp)->tbName, pReq->name); - } else { - ret = metaUpdateMetaRsp(pReq->uid, pReq->name, &pReq->ntb.schemaRow, *pMetaRsp); - if (ret < 0) { - metaError("vgId:%d, failed to update meta rsp:%s since %s", TD_VID(pMeta->pVnode), pReq->name, - tstrerror(ret)); - } - for (int32_t i = 0; i < pReq->colCmpr.nCols; i++) { - SColCmpr *p = &pReq->colCmpr.pColCmpr[i]; - (*pMetaRsp)->pSchemaExt[i].colId = p->id; - (*pMetaRsp)->pSchemaExt[i].compress = p->alg; - } - } - } - } - - pMeta->changed = true; - metaDebug("vgId:%d, table:%s uid %" PRId64 " is created, type:%" PRId8, TD_VID(pMeta->pVnode), pReq->name, pReq->uid, - pReq->type); - return 0; - -_err: - metaError("vgId:%d, failed to create table:%s type:%s since %s", TD_VID(pMeta->pVnode), pReq->name, - pReq->type == TSDB_CHILD_TABLE ? "child table" : "normal table", tstrerror(terrno)); - return TSDB_CODE_FAILED; + TAOS_RETURN(code); } -int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq, SArray *tbUids, tb_uid_t *tbUid) { - void *pData = NULL; - int nData = 0; - int rc = 0; - tb_uid_t uid = 0; - tb_uid_t suid = 0; - int8_t sysTbl = 0; - int type; +// Drop Super Table - rc = tdbTbGet(pMeta->pNameIdx, pReq->name, strlen(pReq->name) + 1, &pData, &nData); - if (rc < 0) { - return terrno = TSDB_CODE_TDB_TABLE_NOT_EXIST; - } - uid = *(tb_uid_t *)pData; +// Alter Super Table - metaWLock(pMeta); - rc = metaDropTableByUid(pMeta, uid, &type, &suid, &sysTbl); - metaULock(pMeta); +// Create Normal Table - if (rc < 0) goto _exit; +// Drop Normal Table - if (!sysTbl && type == TSDB_CHILD_TABLE) { - int32_t nCols = 0; - SVnodeStats *pStats = &pMeta->pVnode->config.vndStats; - if (metaGetStbStats(pMeta->pVnode, suid, NULL, &nCols) == 0) { - pStats->numOfTimeSeries -= nCols - 1; - } - } - - if ((type == TSDB_CHILD_TABLE || type == TSDB_NORMAL_TABLE) && tbUids) { - if (taosArrayPush(tbUids, &uid) == NULL) { - rc = terrno; - goto _exit; - } - - if (!TSDB_CACHE_NO(pMeta->pVnode->config)) { - int32_t ret = tsdbCacheDropTable(pMeta->pVnode->pTsdb, uid, suid, NULL); - if (ret < 0) { - metaError("vgId:%d, failed to drop table:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, uid, - tstrerror(ret)); - } - } - } - - if ((type == TSDB_CHILD_TABLE) && tbUid) { - *tbUid = uid; - } - - pMeta->changed = true; -_exit: - tdbFree(pData); - return rc; -} - -int32_t metaDropTables(SMeta *pMeta, SArray *tbUids) { - int32_t code = 0; - if (taosArrayGetSize(tbUids) == 0) return TSDB_CODE_SUCCESS; - - int64_t nCtbDropped = 0; - SSHashObj *suidHash = tSimpleHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT)); - if (suidHash == NULL) { - return terrno; - } - - metaWLock(pMeta); - for (int i = 0; i < taosArrayGetSize(tbUids); ++i) { - tb_uid_t uid = *(tb_uid_t *)taosArrayGet(tbUids, i); - tb_uid_t suid = 0; - int8_t sysTbl = 0; - int type; - code = metaDropTableByUid(pMeta, uid, &type, &suid, &sysTbl); - if (code) return code; - if (!sysTbl && type == TSDB_CHILD_TABLE && suid != 0 && suidHash) { - int64_t *pVal = tSimpleHashGet(suidHash, &suid, sizeof(tb_uid_t)); - if (pVal) { - nCtbDropped = *pVal + 1; - } else { - nCtbDropped = 1; - } - code = tSimpleHashPut(suidHash, &suid, sizeof(tb_uid_t), &nCtbDropped, sizeof(int64_t)); - if (code) return code; - } - /* - if (!TSDB_CACHE_NO(pMeta->pVnode->config)) { - tsdbCacheDropTable(pMeta->pVnode->pTsdb, uid, suid, NULL); - } - */ - metaDebug("batch drop table:%" PRId64, uid); - } - metaULock(pMeta); - - // update timeseries - void *pCtbDropped = NULL; - int32_t iter = 0; - while ((pCtbDropped = tSimpleHashIterate(suidHash, pCtbDropped, &iter))) { - tb_uid_t *pSuid = tSimpleHashGetKey(pCtbDropped, NULL); - int32_t nCols = 0; - SVnodeStats *pStats = &pMeta->pVnode->config.vndStats; - if (metaGetStbStats(pMeta->pVnode, *pSuid, NULL, &nCols) == 0) { - pStats->numOfTimeSeries -= *(int64_t *)pCtbDropped * (nCols - 1); - } - } - tSimpleHashCleanup(suidHash); - - pMeta->changed = true; - return 0; -} - -static int32_t metaFilterTableByHash(SMeta *pMeta, SArray *uidList) { - int32_t code = 0; - // 1, tranverse table's - // 2, validate table name using vnodeValidateTableHash - // 3, push invalidated table's uid into uidList - - TBC *pCur; - code = tdbTbcOpen(pMeta->pTbDb, &pCur, NULL); - if (code < 0) { - return code; - } - - code = tdbTbcMoveToFirst(pCur); - if (code) { - tdbTbcClose(pCur); - return code; - } - - void *pData = NULL, *pKey = NULL; - int nData = 0, nKey = 0; - - while (1) { - int32_t ret = tdbTbcNext(pCur, &pKey, &nKey, &pData, &nData); - if (ret < 0) { - break; - } - - SMetaEntry me = {0}; - SDecoder dc = {0}; - tDecoderInit(&dc, pData, nData); - code = metaDecodeEntry(&dc, &me); - if (code < 0) { - tDecoderClear(&dc); - return code; - } - - if (me.type != TSDB_SUPER_TABLE) { - char tbFName[TSDB_TABLE_FNAME_LEN + 1]; - snprintf(tbFName, sizeof(tbFName), "%s.%s", pMeta->pVnode->config.dbname, me.name); - tbFName[TSDB_TABLE_FNAME_LEN] = '\0'; - int32_t ret = vnodeValidateTableHash(pMeta->pVnode, tbFName); - if (ret < 0 && terrno == TSDB_CODE_VND_HASH_MISMATCH) { - if (taosArrayPush(uidList, &me.uid) == NULL) { - code = terrno; - break; - } - } - } - tDecoderClear(&dc); - } - tdbFree(pData); - tdbFree(pKey); - tdbTbcClose(pCur); - - return 0; -} - -int32_t metaTrimTables(SMeta *pMeta) { - int32_t code = 0; - - SArray *tbUids = taosArrayInit(8, sizeof(int64_t)); - if (tbUids == NULL) { - return terrno; - } - - code = metaFilterTableByHash(pMeta, tbUids); - if (code != 0) { - goto end; - } - if (TARRAY_SIZE(tbUids) == 0) { - goto end; - } - - metaInfo("vgId:%d, trim %ld tables", TD_VID(pMeta->pVnode), taosArrayGetSize(tbUids)); - code = metaDropTables(pMeta, tbUids); - if (code) goto end; - -end: - taosArrayDestroy(tbUids); - - return code; -} - -int metaTtlFindExpired(SMeta *pMeta, int64_t timePointMs, SArray *tbUids, int32_t ttlDropMaxCount) { - metaRLock(pMeta); - - int ret = ttlMgrFindExpired(pMeta->pTtlMgr, timePointMs, tbUids, ttlDropMaxCount); - - metaULock(pMeta); - - if (ret != 0) { - metaError("ttl failed to find expired table, ret:%d", ret); - } - - return ret; -} - -static int metaBuildBtimeIdxKey(SBtimeIdxKey *btimeKey, const SMetaEntry *pME) { - int64_t btime; - if (pME->type == TSDB_CHILD_TABLE) { - btime = pME->ctbEntry.btime; - } else if (pME->type == TSDB_NORMAL_TABLE) { - btime = pME->ntbEntry.btime; - } else { - return TSDB_CODE_FAILED; - } - - btimeKey->btime = btime; - btimeKey->uid = pME->uid; - return 0; -} - -static int metaBuildNColIdxKey(SNcolIdxKey *ncolKey, const SMetaEntry *pME) { - if (pME->type == TSDB_NORMAL_TABLE) { - ncolKey->ncol = pME->ntbEntry.schemaRow.nCols; - ncolKey->uid = pME->uid; - } else { - return TSDB_CODE_FAILED; - } - return 0; -} - -static void metaDeleteTtl(SMeta *pMeta, const SMetaEntry *pME) { - if (pME->type != TSDB_CHILD_TABLE && pME->type != TSDB_NORMAL_TABLE) return; - - STtlDelTtlCtx ctx = {.uid = pME->uid, .pTxn = pMeta->txn}; - if (pME->type == TSDB_CHILD_TABLE) { - ctx.ttlDays = pME->ctbEntry.ttlDays; - } else { - ctx.ttlDays = pME->ntbEntry.ttlDays; - } - - int32_t ret = ttlMgrDeleteTtl(pMeta->pTtlMgr, &ctx); - if (ret < 0) { - metaError("vgId:%d, failed to delete ttl for table:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pME->name, - pME->uid, tstrerror(ret)); - } - return; -} - -static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type, tb_uid_t *pSuid, int8_t *pSysTbl) { - void *pData = NULL; - int nData = 0; - int rc = 0; - SMetaEntry e = {0}; - SDecoder dc = {0}; - int32_t ret = 0; - - rc = tdbTbGet(pMeta->pUidIdx, &uid, sizeof(uid), &pData, &nData); - if (rc < 0) { - return rc; - } - int64_t version = ((SUidIdxVal *)pData)[0].version; - - rc = tdbTbGet(pMeta->pTbDb, &(STbDbKey){.version = version, .uid = uid}, sizeof(STbDbKey), &pData, &nData); - if (rc < 0) { - tdbFree(pData); - return rc; - } - - tDecoderInit(&dc, pData, nData); - rc = metaDecodeEntry(&dc, &e); - if (rc < 0) { - tDecoderClear(&dc); - return rc; - } - - if (type) *type = e.type; - - if (e.type == TSDB_CHILD_TABLE) { - if (pSuid) *pSuid = e.ctbEntry.suid; - void *tData = NULL; - int tLen = 0; - - if (tdbTbGet(pMeta->pUidIdx, &e.ctbEntry.suid, sizeof(tb_uid_t), &tData, &tLen) == 0) { - STbDbKey tbDbKey = {.uid = e.ctbEntry.suid, .version = ((SUidIdxVal *)tData)[0].version}; - if (tdbTbGet(pMeta->pTbDb, &tbDbKey, sizeof(tbDbKey), &tData, &tLen) == 0) { - SDecoder tdc = {0}; - SMetaEntry stbEntry = {0}; - - tDecoderInit(&tdc, tData, tLen); - int32_t ret = metaDecodeEntry(&tdc, &stbEntry); - if (ret < 0) { - tDecoderClear(&tdc); - metaError("vgId:%d, failed to decode child table:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), e.name, - e.ctbEntry.suid, tstrerror(ret)); - return ret; - } - - if (pSysTbl) *pSysTbl = metaTbInFilterCache(pMeta, stbEntry.name, 1) ? 1 : 0; - - SSchema *pTagColumn = NULL; - SSchemaWrapper *pTagSchema = &stbEntry.stbEntry.schemaTag; - if (pTagSchema->nCols == 1 && pTagSchema->pSchema[0].type == TSDB_DATA_TYPE_JSON) { - pTagColumn = &stbEntry.stbEntry.schemaTag.pSchema[0]; - ret = metaDelJsonVarFromIdx(pMeta, &e, pTagColumn); - if (ret < 0) { - metaError("vgId:%d, failed to delete json var from idx:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), - e.name, e.uid, tstrerror(ret)); - } - } else { - for (int i = 0; i < pTagSchema->nCols; i++) { - pTagColumn = &stbEntry.stbEntry.schemaTag.pSchema[i]; - if (!IS_IDX_ON(pTagColumn)) continue; - STagIdxKey *pTagIdxKey = NULL; - int32_t nTagIdxKey; - - const void *pTagData = NULL; - int32_t nTagData = 0; - - STagVal tagVal = {.cid = pTagColumn->colId}; - if (tTagGet((const STag *)e.ctbEntry.pTags, &tagVal)) { - if (IS_VAR_DATA_TYPE(pTagColumn->type)) { - pTagData = tagVal.pData; - nTagData = (int32_t)tagVal.nData; - } else { - pTagData = &(tagVal.i64); - nTagData = tDataTypes[pTagColumn->type].bytes; - } - } else { - if (!IS_VAR_DATA_TYPE(pTagColumn->type)) { - nTagData = tDataTypes[pTagColumn->type].bytes; - } - } - - if (metaCreateTagIdxKey(e.ctbEntry.suid, pTagColumn->colId, pTagData, nTagData, pTagColumn->type, uid, - &pTagIdxKey, &nTagIdxKey) == 0) { - ret = tdbTbDelete(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, pMeta->txn); - if (ret < 0) { - metaError("vgId:%d, failed to delete tag idx key:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), - e.name, e.uid, tstrerror(ret)); - } - } - metaDestroyTagIdxKey(pTagIdxKey); - pTagIdxKey = NULL; - } - } - tDecoderClear(&tdc); - } - tdbFree(tData); - } - } - - ret = tdbTbDelete(pMeta->pTbDb, &(STbDbKey){.version = version, .uid = uid}, sizeof(STbDbKey), pMeta->txn); - if (ret < 0) { - metaError("vgId:%d, failed to delete table:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), e.name, e.uid, - tstrerror(ret)); - } - ret = tdbTbDelete(pMeta->pNameIdx, e.name, strlen(e.name) + 1, pMeta->txn); - if (ret < 0) { - metaError("vgId:%d, failed to delete name idx:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), e.name, e.uid, - tstrerror(ret)); - } - ret = tdbTbDelete(pMeta->pUidIdx, &uid, sizeof(uid), pMeta->txn); - if (ret < 0) { - metaError("vgId:%d, failed to delete uid idx:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), e.name, e.uid, - tstrerror(ret)); - } - - if (e.type == TSDB_CHILD_TABLE || e.type == TSDB_NORMAL_TABLE) metaDeleteBtimeIdx(pMeta, &e); - if (e.type == TSDB_NORMAL_TABLE) metaDeleteNcolIdx(pMeta, &e); - - if (e.type != TSDB_SUPER_TABLE) metaDeleteTtl(pMeta, &e); - - if (e.type == TSDB_CHILD_TABLE) { - ret = - tdbTbDelete(pMeta->pCtbIdx, &(SCtbIdxKey){.suid = e.ctbEntry.suid, .uid = uid}, sizeof(SCtbIdxKey), pMeta->txn); - if (ret < 0) { - metaError("vgId:%d, failed to delete ctb idx:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), e.name, e.uid, - tstrerror(ret)); - } - - --pMeta->pVnode->config.vndStats.numOfCTables; - metaUpdateStbStats(pMeta, e.ctbEntry.suid, -1, 0); - ret = metaUidCacheClear(pMeta, e.ctbEntry.suid); - if (ret < 0) { - metaError("vgId:%d, failed to clear uid cache:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), e.name, - e.ctbEntry.suid, tstrerror(ret)); - } - ret = metaTbGroupCacheClear(pMeta, e.ctbEntry.suid); - if (ret < 0) { - metaError("vgId:%d, failed to clear group cache:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), e.name, - e.ctbEntry.suid, tstrerror(ret)); - } - /* - if (!TSDB_CACHE_NO(pMeta->pVnode->config)) { - tsdbCacheDropTable(pMeta->pVnode->pTsdb, e.uid, e.ctbEntry.suid, NULL); - } - */ - } else if (e.type == TSDB_NORMAL_TABLE) { - // drop schema.db (todo) - - --pMeta->pVnode->config.vndStats.numOfNTables; - pMeta->pVnode->config.vndStats.numOfNTimeSeries -= e.ntbEntry.schemaRow.nCols - 1; - - /* - if (!TSDB_CACHE_NO(pMeta->pVnode->config)) { - tsdbCacheDropTable(pMeta->pVnode->pTsdb, e.uid, -1, &e.ntbEntry.schemaRow); - } - */ - } else if (e.type == TSDB_SUPER_TABLE) { - ret = tdbTbDelete(pMeta->pSuidIdx, &e.uid, sizeof(tb_uid_t), pMeta->txn); - if (ret < 0) { - metaError("vgId:%d, failed to delete suid idx:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), e.name, e.uid, - tstrerror(ret)); - } - // drop schema.db (todo) - - ret = metaStatsCacheDrop(pMeta, uid); - if (ret < 0) { - metaError("vgId:%d, failed to drop stats cache:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), e.name, e.uid, - tstrerror(ret)); - } - ret = metaUidCacheClear(pMeta, uid); - if (ret < 0) { - metaError("vgId:%d, failed to clear uid cache:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), e.name, e.uid, - tstrerror(ret)); - } - ret = metaTbGroupCacheClear(pMeta, uid); - if (ret < 0) { - metaError("vgId:%d, failed to clear group cache:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), e.name, - e.uid, tstrerror(ret)); - } - --pMeta->pVnode->config.vndStats.numOfSTables; - } - - ret = metaCacheDrop(pMeta, uid); - if (ret < 0) { - metaError("vgId:%d, failed to drop cache:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), e.name, e.uid, - tstrerror(ret)); - } - - tDecoderClear(&dc); - tdbFree(pData); - - return 0; -} -// opt ins_tables -int metaUpdateBtimeIdx(SMeta *pMeta, const SMetaEntry *pME) { - SBtimeIdxKey btimeKey = {0}; - if (metaBuildBtimeIdxKey(&btimeKey, pME) < 0) { - return 0; - } - metaTrace("vgId:%d, start to save version:%" PRId64 " uid:%" PRId64 " btime:%" PRId64, TD_VID(pMeta->pVnode), - pME->version, pME->uid, btimeKey.btime); - - return tdbTbUpsert(pMeta->pBtimeIdx, &btimeKey, sizeof(btimeKey), NULL, 0, pMeta->txn); -} - -int metaDeleteBtimeIdx(SMeta *pMeta, const SMetaEntry *pME) { - SBtimeIdxKey btimeKey = {0}; - if (metaBuildBtimeIdxKey(&btimeKey, pME) < 0) { - return 0; - } - return tdbTbDelete(pMeta->pBtimeIdx, &btimeKey, sizeof(btimeKey), pMeta->txn); -} -int metaUpdateNcolIdx(SMeta *pMeta, const SMetaEntry *pME) { - SNcolIdxKey ncolKey = {0}; - if (metaBuildNColIdxKey(&ncolKey, pME) < 0) { - return 0; - } - return tdbTbUpsert(pMeta->pNcolIdx, &ncolKey, sizeof(ncolKey), NULL, 0, pMeta->txn); -} - -int metaDeleteNcolIdx(SMeta *pMeta, const SMetaEntry *pME) { - SNcolIdxKey ncolKey = {0}; - if (metaBuildNColIdxKey(&ncolKey, pME) < 0) { - return 0; - } - return tdbTbDelete(pMeta->pNcolIdx, &ncolKey, sizeof(ncolKey), pMeta->txn); -} - -static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTbReq, STableMetaRsp *pMetaRsp) { - void *pVal = NULL; - int nVal = 0; - const void *pData = NULL; - int nData = 0; - int ret = 0; - tb_uid_t uid; - int64_t oversion; - SSchema *pColumn = NULL; - SMetaEntry entry = {0}; - SSchemaWrapper *pSchema; - int c; - bool freeColCmpr = false; - if (pAlterTbReq->colName == NULL) { - metaError("meta/table: null pAlterTbReq->colName"); - return terrno = TSDB_CODE_INVALID_MSG; - } - - // search name index - ret = tdbTbGet(pMeta->pNameIdx, pAlterTbReq->tbName, strlen(pAlterTbReq->tbName) + 1, &pVal, &nVal); - if (ret < 0) { - return terrno = TSDB_CODE_TDB_TABLE_NOT_EXIST; - } - - uid = *(tb_uid_t *)pVal; - tdbFree(pVal); - pVal = NULL; - - // search uid index - TBC *pUidIdxc = NULL; - - TAOS_CHECK_RETURN(tdbTbcOpen(pMeta->pUidIdx, &pUidIdxc, NULL)); - ret = tdbTbcMoveTo(pUidIdxc, &uid, sizeof(uid), &c); - if (c != 0) { - tdbTbcClose(pUidIdxc); - metaError("meta/table: invalide c: %" PRId32 " alt tb column failed.", c); - return TSDB_CODE_FAILED; - } - - ret = tdbTbcGet(pUidIdxc, NULL, NULL, &pData, &nData); - oversion = ((SUidIdxVal *)pData)[0].version; - - // search table.db - TBC *pTbDbc = NULL; - - TAOS_CHECK_RETURN(tdbTbcOpen(pMeta->pTbDb, &pTbDbc, NULL)); - ret = tdbTbcMoveTo(pTbDbc, &((STbDbKey){.uid = uid, .version = oversion}), sizeof(STbDbKey), &c); - if (c != 0) { - tdbTbcClose(pUidIdxc); - tdbTbcClose(pTbDbc); - metaError("meta/table: invalide c: %" PRId32 " alt tb column failed.", c); - return TSDB_CODE_FAILED; - } - - ret = tdbTbcGet(pTbDbc, NULL, NULL, &pData, &nData); - - // get table entry - SDecoder dc = {0}; - if ((entry.pBuf = taosMemoryMalloc(nData)) == NULL) { - tdbTbcClose(pUidIdxc); - tdbTbcClose(pTbDbc); - return terrno; - } - memcpy(entry.pBuf, pData, nData); - tDecoderInit(&dc, entry.pBuf, nData); - ret = metaDecodeEntry(&dc, &entry); - if (ret != 0) { - tdbTbcClose(pUidIdxc); - tdbTbcClose(pTbDbc); - tDecoderClear(&dc); - metaError("meta/table: invalide ret: %" PRId32 " alt tb column failed.", ret); - return ret; - } - - if (entry.type != TSDB_NORMAL_TABLE) { - terrno = TSDB_CODE_VND_INVALID_TABLE_ACTION; - goto _err; - } - // search the column to add/drop/update - pSchema = &entry.ntbEntry.schemaRow; - - // save old entry - SMetaEntry oldEntry = {.type = TSDB_NORMAL_TABLE, .uid = entry.uid}; - oldEntry.ntbEntry.schemaRow.nCols = pSchema->nCols; - - int32_t rowLen = -1; - if (pAlterTbReq->action == TSDB_ALTER_TABLE_ADD_COLUMN || - pAlterTbReq->action == TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES) { - rowLen = 0; - } - - int32_t iCol = 0, jCol = 0; - SSchema *qColumn = NULL; - for (;;) { - qColumn = NULL; - - if (jCol >= pSchema->nCols) break; - qColumn = &pSchema->pSchema[jCol]; - - if (!pColumn && (strcmp(qColumn->name, pAlterTbReq->colName) == 0)) { - pColumn = qColumn; - iCol = jCol; - if (rowLen < 0) break; - } - rowLen += qColumn->bytes; - ++jCol; - } - - entry.version = version; - int tlen; - SSchema *pNewSchema = NULL; - SSchema tScheam; - switch (pAlterTbReq->action) { - case TSDB_ALTER_TABLE_ADD_COLUMN: - case TSDB_ALTER_TABLE_ADD_COLUMN_WITH_COMPRESS_OPTION: - if (pColumn) { - terrno = TSDB_CODE_VND_COL_ALREADY_EXISTS; - goto _err; - } - if ((terrno = grantCheck(TSDB_GRANT_TIMESERIES)) < 0) { - goto _err; - } - if (rowLen + pAlterTbReq->bytes > TSDB_MAX_BYTES_PER_ROW) { - terrno = TSDB_CODE_PAR_INVALID_ROW_LENGTH; - goto _err; - } - pSchema->version++; - pSchema->nCols++; - pNewSchema = taosMemoryMalloc(sizeof(SSchema) * pSchema->nCols); - if (pNewSchema == NULL) { - goto _err; - } - memcpy(pNewSchema, pSchema->pSchema, sizeof(SSchema) * (pSchema->nCols - 1)); - pSchema->pSchema = pNewSchema; - pSchema->pSchema[entry.ntbEntry.schemaRow.nCols - 1].bytes = pAlterTbReq->bytes; - pSchema->pSchema[entry.ntbEntry.schemaRow.nCols - 1].type = pAlterTbReq->type; - pSchema->pSchema[entry.ntbEntry.schemaRow.nCols - 1].flags = pAlterTbReq->flags; - pSchema->pSchema[entry.ntbEntry.schemaRow.nCols - 1].colId = entry.ntbEntry.ncid++; - strcpy(pSchema->pSchema[entry.ntbEntry.schemaRow.nCols - 1].name, pAlterTbReq->colName); - - ++pMeta->pVnode->config.vndStats.numOfNTimeSeries; - metaTimeSeriesNotifyCheck(pMeta); - - if (!TSDB_CACHE_NO(pMeta->pVnode->config)) { - int16_t cid = pSchema->pSchema[entry.ntbEntry.schemaRow.nCols - 1].colId; - int8_t col_type = pSchema->pSchema[entry.ntbEntry.schemaRow.nCols - 1].type; - int32_t ret = tsdbCacheNewNTableColumn(pMeta->pVnode->pTsdb, entry.uid, cid, col_type); - if (ret < 0) { - terrno = ret; - goto _err; - } - } - SSchema *pCol = &pSchema->pSchema[entry.ntbEntry.schemaRow.nCols - 1]; - uint32_t compress = pAlterTbReq->action == TSDB_ALTER_TABLE_ADD_COLUMN ? createDefaultColCmprByType(pCol->type) - : pAlterTbReq->compress; - if (updataTableColCmpr(&entry.colCmpr, pCol, 1, compress) != 0) { - metaError("vgId:%d, failed to update table col cmpr:%s uid:%" PRId64, TD_VID(pMeta->pVnode), entry.name, - entry.uid); - } - freeColCmpr = true; - if (entry.colCmpr.nCols != pSchema->nCols) { - if (pNewSchema) taosMemoryFree(pNewSchema); - if (freeColCmpr) taosMemoryFree(entry.colCmpr.pColCmpr); - terrno = TSDB_CODE_VND_INVALID_TABLE_ACTION; - goto _err; - } - break; - case TSDB_ALTER_TABLE_DROP_COLUMN: - if (pColumn == NULL) { - terrno = TSDB_CODE_VND_COL_NOT_EXISTS; - goto _err; - } - if (pColumn->colId == 0) { - terrno = TSDB_CODE_VND_INVALID_TABLE_ACTION; - goto _err; - } - if (tqCheckColModifiable(pMeta->pVnode->pTq, uid, pColumn->colId) != 0) { - terrno = TSDB_CODE_VND_COL_SUBSCRIBED; - goto _err; - } - bool hasPrimayKey = false; - if (pSchema->nCols >= 2) { - hasPrimayKey = pSchema->pSchema[1].flags & COL_IS_KEY ? true : false; - } - - memcpy(&tScheam, pColumn, sizeof(SSchema)); - pSchema->version++; - tlen = (pSchema->nCols - iCol - 1) * sizeof(SSchema); - if (tlen) { - memmove(pColumn, pColumn + 1, tlen); - } - pSchema->nCols--; - - --pMeta->pVnode->config.vndStats.numOfNTimeSeries; - - if (!TSDB_CACHE_NO(pMeta->pVnode->config)) { - int16_t cid = pColumn->colId; - - if (tsdbCacheDropNTableColumn(pMeta->pVnode->pTsdb, entry.uid, cid, hasPrimayKey) != 0) { - metaError("vgId:%d, failed to drop ntable column:%s uid:%" PRId64, TD_VID(pMeta->pVnode), entry.name, - entry.uid); - } - } - - if (updataTableColCmpr(&entry.colCmpr, &tScheam, 0, 0) != 0) { - metaError("vgId:%d, failed to update table col cmpr:%s uid:%" PRId64, TD_VID(pMeta->pVnode), entry.name, - entry.uid); - } - if (entry.colCmpr.nCols != pSchema->nCols) { - terrno = TSDB_CODE_VND_INVALID_TABLE_ACTION; - goto _err; - } - break; - case TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES: - if (pColumn == NULL) { - terrno = TSDB_CODE_VND_COL_NOT_EXISTS; - goto _err; - } - if (!IS_VAR_DATA_TYPE(pColumn->type) || pColumn->bytes >= pAlterTbReq->colModBytes) { - terrno = TSDB_CODE_VND_INVALID_TABLE_ACTION; - goto _err; - } - if (rowLen + pAlterTbReq->colModBytes - pColumn->bytes > TSDB_MAX_BYTES_PER_ROW) { - terrno = TSDB_CODE_PAR_INVALID_ROW_LENGTH; - goto _err; - } - if (tqCheckColModifiable(pMeta->pVnode->pTq, uid, pColumn->colId) != 0) { - terrno = TSDB_CODE_VND_COL_SUBSCRIBED; - goto _err; - } - pSchema->version++; - pColumn->bytes = pAlterTbReq->colModBytes; - break; - case TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME: - if (pAlterTbReq->colNewName == NULL) { - terrno = TSDB_CODE_INVALID_MSG; - goto _err; - } - if (pColumn == NULL) { - terrno = TSDB_CODE_VND_COL_NOT_EXISTS; - goto _err; - } - if (tqCheckColModifiable(pMeta->pVnode->pTq, uid, pColumn->colId) != 0) { - terrno = TSDB_CODE_VND_COL_SUBSCRIBED; - goto _err; - } - pSchema->version++; - strcpy(pColumn->name, pAlterTbReq->colNewName); - break; - } - - if (!TSDB_CACHE_NO(pMeta->pVnode->config)) { - tsdbCacheInvalidateSchema(pMeta->pVnode->pTsdb, 0, entry.uid, pSchema->version); - } - - entry.version = version; - - // do actual write - metaWLock(pMeta); - - if (metaDeleteNcolIdx(pMeta, &oldEntry) < 0) { - metaError("vgId:%d, failed to delete ncol idx:%s uid:%" PRId64, TD_VID(pMeta->pVnode), entry.name, entry.uid); - } - - if (metaUpdateNcolIdx(pMeta, &entry) < 0) { - metaError("vgId:%d, failed to update ncol idx:%s uid:%" PRId64, TD_VID(pMeta->pVnode), entry.name, entry.uid); - } - - // save to table db - if (metaSaveToTbDb(pMeta, &entry) < 0) { - metaError("vgId:%d, failed to save to tb db:%s uid:%" PRId64, TD_VID(pMeta->pVnode), entry.name, entry.uid); - } - - if (metaUpdateUidIdx(pMeta, &entry) < 0) { - metaError("vgId:%d, failed to update uid idx:%s uid:%" PRId64, TD_VID(pMeta->pVnode), entry.name, entry.uid); - } - - if (metaSaveToSkmDb(pMeta, &entry) < 0) { - metaError("vgId:%d, failed to save to skm db:%s uid:%" PRId64, TD_VID(pMeta->pVnode), entry.name, entry.uid); - } - - if (metaUpdateChangeTime(pMeta, entry.uid, pAlterTbReq->ctimeMs) < 0) { - metaError("vgId:%d, failed to update change time:%s uid:%" PRId64, TD_VID(pMeta->pVnode), entry.name, entry.uid); - } - - metaULock(pMeta); - - if (metaUpdateMetaRsp(uid, pAlterTbReq->tbName, pSchema, pMetaRsp) < 0) { - metaError("vgId:%d, failed to update meta rsp:%s uid:%" PRId64, TD_VID(pMeta->pVnode), entry.name, entry.uid); - } - for (int32_t i = 0; i < entry.colCmpr.nCols; i++) { - SColCmpr *p = &entry.colCmpr.pColCmpr[i]; - pMetaRsp->pSchemaExt[i].colId = p->id; - pMetaRsp->pSchemaExt[i].compress = p->alg; - } - - if (entry.pBuf) taosMemoryFree(entry.pBuf); - if (pNewSchema) taosMemoryFree(pNewSchema); - if (freeColCmpr) taosMemoryFree(entry.colCmpr.pColCmpr); - - tdbTbcClose(pTbDbc); - tdbTbcClose(pUidIdxc); - tDecoderClear(&dc); - - return 0; - -_err: - if (entry.pBuf) taosMemoryFree(entry.pBuf); - tdbTbcClose(pTbDbc); - tdbTbcClose(pUidIdxc); - tDecoderClear(&dc); - - return terrno != 0 ? terrno : TSDB_CODE_FAILED; -} - -static int metaUpdateTableMultiTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTbReq) { - SMetaEntry ctbEntry = {0}; - SMetaEntry stbEntry = {0}; - void *pVal = NULL; - int nVal = 0; - int ret; - int c; - tb_uid_t uid; - int64_t oversion; - const void *pData = NULL; - int nData = 0; - SHashObj *pTagTable = NULL; - - // search name index - ret = tdbTbGet(pMeta->pNameIdx, pAlterTbReq->tbName, strlen(pAlterTbReq->tbName) + 1, &pVal, &nVal); - if (ret < 0) { - return terrno = TSDB_CODE_TDB_TABLE_NOT_EXIST; - } - - uid = *(tb_uid_t *)pVal; - tdbFree(pVal); - pVal = NULL; - - // search uid index - TBC *pUidIdxc = NULL; - - TAOS_CHECK_RETURN(tdbTbcOpen(pMeta->pUidIdx, &pUidIdxc, NULL)); - if (tdbTbcMoveTo(pUidIdxc, &uid, sizeof(uid), &c) < 0) { - metaTrace("meta/table: failed to move to uid index, uid:%" PRId64, uid); - } - if (c != 0) { - tdbTbcClose(pUidIdxc); - metaError("meta/table: invalide c: %" PRId32 " update tb tag val failed.", c); - return terrno = TSDB_CODE_TDB_TABLE_NOT_EXIST; - } - - if (tdbTbcGet(pUidIdxc, NULL, NULL, &pData, &nData) != 0) { - tdbTbcClose(pUidIdxc); - metaError("meta/table: failed to get uid index, uid:%" PRId64, uid); - return terrno = TSDB_CODE_TDB_TABLE_NOT_EXIST; - } - oversion = ((SUidIdxVal *)pData)[0].version; - - // search table.db - TBC *pTbDbc = NULL; - SDecoder dc1 = {0}; - SDecoder dc2 = {0}; - - /* get ctbEntry */ - TAOS_CHECK_RETURN(tdbTbcOpen(pMeta->pTbDb, &pTbDbc, NULL)); - if (tdbTbcMoveTo(pTbDbc, &((STbDbKey){.uid = uid, .version = oversion}), sizeof(STbDbKey), &c) != 0) { - metaError("meta/table: failed to move to tb db, uid:%" PRId64, uid); - } - if (c != 0) { - tdbTbcClose(pUidIdxc); - tdbTbcClose(pTbDbc); - metaError("meta/table: invalide c: %" PRId32 " update tb tag val failed.", c); - return terrno = TSDB_CODE_TDB_TABLE_NOT_EXIST; - } - - if (tdbTbcGet(pTbDbc, NULL, NULL, &pData, &nData) != 0) { - metaError("meta/table: failed to get tb db, uid:%" PRId64, uid); - tdbTbcClose(pUidIdxc); - tdbTbcClose(pTbDbc); - return terrno = TSDB_CODE_TDB_TABLE_NOT_EXIST; - } - - if ((ctbEntry.pBuf = taosMemoryMalloc(nData)) == NULL) { - tdbTbcClose(pUidIdxc); - tdbTbcClose(pTbDbc); - return terrno; - } - memcpy(ctbEntry.pBuf, pData, nData); - tDecoderInit(&dc1, ctbEntry.pBuf, nData); - ret = metaDecodeEntry(&dc1, &ctbEntry); - if (ret < 0) { - terrno = ret; - goto _err; - } - - /* get stbEntry*/ - if (tdbTbGet(pMeta->pUidIdx, &ctbEntry.ctbEntry.suid, sizeof(tb_uid_t), &pVal, &nVal) != 0) { - metaError("meta/table: failed to get uid index, uid:%" PRId64, ctbEntry.ctbEntry.suid); - } - if (!pVal) { - terrno = TSDB_CODE_INVALID_MSG; - goto _err; - } - - if (tdbTbGet(pMeta->pTbDb, &((STbDbKey){.uid = ctbEntry.ctbEntry.suid, .version = ((SUidIdxVal *)pVal)[0].version}), - sizeof(STbDbKey), (void **)&stbEntry.pBuf, &nVal) != 0) { - metaError("meta/table: failed to get tb db, uid:%" PRId64, ctbEntry.ctbEntry.suid); - } - tdbFree(pVal); - tDecoderInit(&dc2, stbEntry.pBuf, nVal); - ret = metaDecodeEntry(&dc2, &stbEntry); - if (ret < 0) { - terrno = ret; - goto _err; - } - - int32_t nTagVals = taosArrayGetSize(pAlterTbReq->pMultiTag); - pTagTable = taosHashInit(nTagVals, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); - if (pTagTable == NULL) { - ret = terrno; - goto _err; - } - - // remove duplicate tag name - for (int i = 0; i < nTagVals; i++) { - SMultiTagUpateVal *pTagVal = taosArrayGet(pAlterTbReq->pMultiTag, i); - ret = taosHashPut(pTagTable, pTagVal->tagName, strlen(pTagVal->tagName), pTagVal, sizeof(*pTagVal)); - if (ret != 0) { - goto _err; - } - } - - SSchemaWrapper *pTagSchema = &stbEntry.stbEntry.schemaTag; - SSchema *pColumn = NULL; - int32_t iCol = 0; - int32_t count = 0; - - for (;;) { - pColumn = NULL; - - if (iCol >= pTagSchema->nCols) break; - pColumn = &pTagSchema->pSchema[iCol]; - if (taosHashGet(pTagTable, pColumn->name, strlen(pColumn->name)) != NULL) { - count++; - } - iCol++; - } - if (count != taosHashGetSize(pTagTable)) { - terrno = TSDB_CODE_VND_COL_NOT_EXISTS; - goto _err; - } - - ctbEntry.version = version; - if (pTagSchema->nCols == 1 && pTagSchema->pSchema[0].type == TSDB_DATA_TYPE_JSON) { - terrno = TSDB_CODE_VND_COL_NOT_EXISTS; - goto _err; - } else { - const STag *pOldTag = (const STag *)ctbEntry.ctbEntry.pTags; - STag *pNewTag = NULL; - SArray *pTagArray = taosArrayInit(pTagSchema->nCols, sizeof(STagVal)); - if (!pTagArray) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - for (int32_t i = 0; i < pTagSchema->nCols; i++) { - SSchema *pCol = &pTagSchema->pSchema[i]; - SMultiTagUpateVal *pTagVal = taosHashGet(pTagTable, pCol->name, strlen(pCol->name)); - if (pTagVal == NULL) { - STagVal val = {.cid = pCol->colId}; - if (tTagGet(pOldTag, &val)) { - if (taosArrayPush(pTagArray, &val) == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - taosArrayDestroy(pTagArray); - goto _err; - } - } - } else { - STagVal val = {0}; - val.type = pCol->type; - val.cid = pCol->colId; - if (pTagVal->isNull) continue; - - if (IS_VAR_DATA_TYPE(pCol->type)) { - val.pData = pTagVal->pTagVal; - val.nData = pTagVal->nTagVal; - } else { - memcpy(&val.i64, pTagVal->pTagVal, pTagVal->nTagVal); - } - if (taosArrayPush(pTagArray, &val) == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - taosArrayDestroy(pTagArray); - goto _err; - } - } - } - if ((terrno = tTagNew(pTagArray, pTagSchema->version, false, &pNewTag)) < 0) { - taosArrayDestroy(pTagArray); - goto _err; - } - ctbEntry.ctbEntry.pTags = (uint8_t *)pNewTag; - taosArrayDestroy(pTagArray); - } - - metaWLock(pMeta); - - // save to table.db - if (metaSaveToTbDb(pMeta, &ctbEntry) < 0) { - metaError("meta/table: failed to save to tb db:%s uid:%" PRId64, ctbEntry.name, ctbEntry.uid); - } - - // save to uid.idx - if (metaUpdateUidIdx(pMeta, &ctbEntry) < 0) { - metaError("meta/table: failed to update uid idx:%s uid:%" PRId64, ctbEntry.name, ctbEntry.uid); - } - - if (metaUpdateTagIdx(pMeta, &ctbEntry) < 0) { - metaError("meta/table: failed to update tag idx:%s uid:%" PRId64, ctbEntry.name, ctbEntry.uid); - } - - SCtbIdxKey ctbIdxKey = {.suid = ctbEntry.ctbEntry.suid, .uid = uid}; - if (tdbTbUpsert(pMeta->pCtbIdx, &ctbIdxKey, sizeof(ctbIdxKey), ctbEntry.ctbEntry.pTags, - ((STag *)(ctbEntry.ctbEntry.pTags))->len, pMeta->txn) < 0) { - metaError("meta/table: failed to upsert ctb idx:%s uid:%" PRId64, ctbEntry.name, ctbEntry.uid); - } - - if (metaUidCacheClear(pMeta, ctbEntry.ctbEntry.suid) < 0) { - metaError("meta/table: failed to clear uid cache:%s uid:%" PRId64, ctbEntry.name, ctbEntry.uid); - } - - if (metaTbGroupCacheClear(pMeta, ctbEntry.ctbEntry.suid) < 0) { - metaError("meta/table: failed to clear group cache:%s uid:%" PRId64, ctbEntry.name, ctbEntry.uid); - } - - if (metaUpdateChangeTime(pMeta, ctbEntry.uid, pAlterTbReq->ctimeMs) < 0) { - metaError("meta/table: failed to update change time:%s uid:%" PRId64, ctbEntry.name, ctbEntry.uid); - } - - metaULock(pMeta); - - tDecoderClear(&dc1); - tDecoderClear(&dc2); - taosMemoryFree((void *)ctbEntry.ctbEntry.pTags); - if (ctbEntry.pBuf) taosMemoryFree(ctbEntry.pBuf); - if (stbEntry.pBuf) tdbFree(stbEntry.pBuf); - tdbTbcClose(pTbDbc); - tdbTbcClose(pUidIdxc); - taosHashCleanup(pTagTable); - return 0; - -_err: - tDecoderClear(&dc1); - tDecoderClear(&dc2); - if (ctbEntry.pBuf) taosMemoryFree(ctbEntry.pBuf); - if (stbEntry.pBuf) tdbFree(stbEntry.pBuf); - tdbTbcClose(pTbDbc); - tdbTbcClose(pUidIdxc); - taosHashCleanup(pTagTable); - return -1; -} -static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTbReq) { - SMetaEntry ctbEntry = {0}; - SMetaEntry stbEntry = {0}; - void *pVal = NULL; - int nVal = 0; - int ret; - int c; - tb_uid_t uid; - int64_t oversion; - const void *pData = NULL; - int nData = 0; - - if (pAlterTbReq->tagName == NULL) { - return terrno = TSDB_CODE_INVALID_MSG; - } - - // search name index - ret = tdbTbGet(pMeta->pNameIdx, pAlterTbReq->tbName, strlen(pAlterTbReq->tbName) + 1, &pVal, &nVal); - if (ret < 0) { - return terrno = TSDB_CODE_TDB_TABLE_NOT_EXIST; - } - - uid = *(tb_uid_t *)pVal; - tdbFree(pVal); - pVal = NULL; - - // search uid index - TBC *pUidIdxc = NULL; - - TAOS_CHECK_RETURN(tdbTbcOpen(pMeta->pUidIdx, &pUidIdxc, NULL)); - if (tdbTbcMoveTo(pUidIdxc, &uid, sizeof(uid), &c) < 0) { - metaTrace("meta/table: failed to move to uid index, uid:%" PRId64, uid); - } - if (c != 0) { - tdbTbcClose(pUidIdxc); - metaError("meta/table: invalide c: %" PRId32 " update tb tag val failed.", c); - return terrno = TSDB_CODE_TDB_TABLE_NOT_EXIST; - } - - if (tdbTbcGet(pUidIdxc, NULL, NULL, &pData, &nData) != 0) { - tdbTbcClose(pUidIdxc); - metaError("meta/table: failed to get uid index, uid:%" PRId64, uid); - return terrno = TSDB_CODE_TDB_TABLE_NOT_EXIST; - } - oversion = ((SUidIdxVal *)pData)[0].version; - - // search table.db - TBC *pTbDbc = NULL; - SDecoder dc1 = {0}; - SDecoder dc2 = {0}; - - /* get ctbEntry */ - TAOS_CHECK_RETURN(tdbTbcOpen(pMeta->pTbDb, &pTbDbc, NULL)); - if (tdbTbcMoveTo(pTbDbc, &((STbDbKey){.uid = uid, .version = oversion}), sizeof(STbDbKey), &c) != 0) { - metaError("meta/table: failed to move to tb db, uid:%" PRId64, uid); - } - if (c != 0) { - tdbTbcClose(pUidIdxc); - tdbTbcClose(pTbDbc); - metaError("meta/table: invalide c: %" PRId32 " update tb tag val failed.", c); - return terrno = TSDB_CODE_TDB_TABLE_NOT_EXIST; - } - - if (tdbTbcGet(pTbDbc, NULL, NULL, &pData, &nData) != 0) { - metaError("meta/table: failed to get tb db, uid:%" PRId64, uid); - tdbTbcClose(pUidIdxc); - tdbTbcClose(pTbDbc); - return terrno = TSDB_CODE_INVALID_MSG; - } - - if ((ctbEntry.pBuf = taosMemoryMalloc(nData)) == NULL) { - tdbTbcClose(pUidIdxc); - tdbTbcClose(pTbDbc); - return terrno; - } - memcpy(ctbEntry.pBuf, pData, nData); - tDecoderInit(&dc1, ctbEntry.pBuf, nData); - ret = metaDecodeEntry(&dc1, &ctbEntry); - if (ret < 0) { - terrno = ret; - goto _err; - } - - /* get stbEntry*/ - if (tdbTbGet(pMeta->pUidIdx, &ctbEntry.ctbEntry.suid, sizeof(tb_uid_t), &pVal, &nVal) != 0) { - metaError("meta/table: failed to get uid index, uid:%" PRId64, ctbEntry.ctbEntry.suid); - } - if (!pVal) { - terrno = TSDB_CODE_INVALID_MSG; - goto _err; - } - - if (tdbTbGet(pMeta->pTbDb, &((STbDbKey){.uid = ctbEntry.ctbEntry.suid, .version = ((SUidIdxVal *)pVal)[0].version}), - sizeof(STbDbKey), (void **)&stbEntry.pBuf, &nVal) != 0) { - metaError("meta/table: failed to get tb db, uid:%" PRId64, ctbEntry.ctbEntry.suid); - } - tdbFree(pVal); - tDecoderInit(&dc2, stbEntry.pBuf, nVal); - ret = metaDecodeEntry(&dc2, &stbEntry); - if (ret < 0) { - terrno = ret; - goto _err; - } - - SSchemaWrapper *pTagSchema = &stbEntry.stbEntry.schemaTag; - SSchema *pColumn = NULL; - int32_t iCol = 0; - - for (;;) { - pColumn = NULL; - - if (iCol >= pTagSchema->nCols) break; - pColumn = &pTagSchema->pSchema[iCol]; - - if (strcmp(pColumn->name, pAlterTbReq->tagName) == 0) break; - iCol++; - } - - if (pColumn == NULL) { - terrno = TSDB_CODE_VND_COL_NOT_EXISTS; - goto _err; - } - - ctbEntry.version = version; - if (pTagSchema->nCols == 1 && pTagSchema->pSchema[0].type == TSDB_DATA_TYPE_JSON) { - ctbEntry.ctbEntry.pTags = taosMemoryMalloc(pAlterTbReq->nTagVal); - if (ctbEntry.ctbEntry.pTags == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - memcpy((void *)ctbEntry.ctbEntry.pTags, pAlterTbReq->pTagVal, pAlterTbReq->nTagVal); - } else { - const STag *pOldTag = (const STag *)ctbEntry.ctbEntry.pTags; - STag *pNewTag = NULL; - SArray *pTagArray = taosArrayInit(pTagSchema->nCols, sizeof(STagVal)); - if (!pTagArray) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - for (int32_t i = 0; i < pTagSchema->nCols; i++) { - SSchema *pCol = &pTagSchema->pSchema[i]; - if (iCol == i) { - if (pAlterTbReq->isNull) { - continue; - } - STagVal val = {0}; - val.type = pCol->type; - val.cid = pCol->colId; - if (IS_VAR_DATA_TYPE(pCol->type)) { - val.pData = pAlterTbReq->pTagVal; - val.nData = pAlterTbReq->nTagVal; - } else { - memcpy(&val.i64, pAlterTbReq->pTagVal, pAlterTbReq->nTagVal); - } - if (taosArrayPush(pTagArray, &val) == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - taosArrayDestroy(pTagArray); - goto _err; - } - } else { - STagVal val = {.cid = pCol->colId}; - if (tTagGet(pOldTag, &val)) { - if (taosArrayPush(pTagArray, &val) == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - taosArrayDestroy(pTagArray); - goto _err; - } - } - } - } - if ((terrno = tTagNew(pTagArray, pTagSchema->version, false, &pNewTag)) < 0) { - taosArrayDestroy(pTagArray); - goto _err; - } - ctbEntry.ctbEntry.pTags = (uint8_t *)pNewTag; - taosArrayDestroy(pTagArray); - } - - metaWLock(pMeta); - - // save to table.db - if (metaSaveToTbDb(pMeta, &ctbEntry) < 0) { - metaError("meta/table: failed to save to tb db:%s uid:%" PRId64, ctbEntry.name, ctbEntry.uid); - } - - // save to uid.idx - if (metaUpdateUidIdx(pMeta, &ctbEntry) < 0) { - metaError("meta/table: failed to update uid idx:%s uid:%" PRId64, ctbEntry.name, ctbEntry.uid); - } - - if (metaUpdateTagIdx(pMeta, &ctbEntry) < 0) { - metaError("meta/table: failed to update tag idx:%s uid:%" PRId64, ctbEntry.name, ctbEntry.uid); - } - - SCtbIdxKey ctbIdxKey = {.suid = ctbEntry.ctbEntry.suid, .uid = uid}; - if (tdbTbUpsert(pMeta->pCtbIdx, &ctbIdxKey, sizeof(ctbIdxKey), ctbEntry.ctbEntry.pTags, - ((STag *)(ctbEntry.ctbEntry.pTags))->len, pMeta->txn) < 0) { - metaError("meta/table: failed to upsert ctb idx:%s uid:%" PRId64, ctbEntry.name, ctbEntry.uid); - } - - if (metaUidCacheClear(pMeta, ctbEntry.ctbEntry.suid) < 0) { - metaError("meta/table: failed to clear uid cache:%s uid:%" PRId64, ctbEntry.name, ctbEntry.uid); - } - - if (metaTbGroupCacheClear(pMeta, ctbEntry.ctbEntry.suid) < 0) { - metaError("meta/table: failed to clear group cache:%s uid:%" PRId64, ctbEntry.name, ctbEntry.uid); - } - - if (metaUpdateChangeTime(pMeta, ctbEntry.uid, pAlterTbReq->ctimeMs) < 0) { - metaError("meta/table: failed to update change time:%s uid:%" PRId64, ctbEntry.name, ctbEntry.uid); - } - - metaULock(pMeta); - - tDecoderClear(&dc1); - tDecoderClear(&dc2); - taosMemoryFree((void *)ctbEntry.ctbEntry.pTags); - if (ctbEntry.pBuf) taosMemoryFree(ctbEntry.pBuf); - if (stbEntry.pBuf) tdbFree(stbEntry.pBuf); - tdbTbcClose(pTbDbc); - tdbTbcClose(pUidIdxc); - return 0; - -_err: - tDecoderClear(&dc1); - tDecoderClear(&dc2); - if (ctbEntry.pBuf) taosMemoryFree(ctbEntry.pBuf); - if (stbEntry.pBuf) tdbFree(stbEntry.pBuf); - tdbTbcClose(pTbDbc); - tdbTbcClose(pUidIdxc); - return -1; -} - -static int metaUpdateTableOptions(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTbReq) { - void *pVal = NULL; - int nVal = 0; - const void *pData = NULL; - int nData = 0; - int ret = 0; - tb_uid_t uid; - int64_t oversion; - SMetaEntry entry = {0}; - int c = 0; - - // search name index - ret = tdbTbGet(pMeta->pNameIdx, pAlterTbReq->tbName, strlen(pAlterTbReq->tbName) + 1, &pVal, &nVal); - if (ret < 0) { - return terrno = TSDB_CODE_TDB_TABLE_NOT_EXIST; - } - - uid = *(tb_uid_t *)pVal; - tdbFree(pVal); - pVal = NULL; - - // search uid index - TBC *pUidIdxc = NULL; - - TAOS_CHECK_RETURN(tdbTbcOpen(pMeta->pUidIdx, &pUidIdxc, NULL)); - if (tdbTbcMoveTo(pUidIdxc, &uid, sizeof(uid), &c) < 0) { - metaError("meta/table: failed to move to uid index, uid:%" PRId64, uid); - } - if (c != 0) { - tdbTbcClose(pUidIdxc); - metaError("meta/table: invalide c: %" PRId32 " update tb options failed.", c); - return TSDB_CODE_FAILED; - } - - if (tdbTbcGet(pUidIdxc, NULL, NULL, &pData, &nData) < 0) { - metaError("meta/table: failed to get uid index, uid:%" PRId64, uid); - } - oversion = ((SUidIdxVal *)pData)[0].version; - - // search table.db - TBC *pTbDbc = NULL; - - TAOS_CHECK_RETURN(tdbTbcOpen(pMeta->pTbDb, &pTbDbc, NULL)); - if (tdbTbcMoveTo(pTbDbc, &((STbDbKey){.uid = uid, .version = oversion}), sizeof(STbDbKey), &c) < 0) { - metaError("meta/table: failed to move to tb db, uid:%" PRId64, uid); - } - if (c != 0) { - tdbTbcClose(pUidIdxc); - tdbTbcClose(pTbDbc); - metaError("meta/table: invalide c: %" PRId32 " update tb options failed.", c); - return TSDB_CODE_FAILED; - } - - if (tdbTbcGet(pTbDbc, NULL, NULL, &pData, &nData) < 0) { - metaError("meta/table: failed to get tb db, uid:%" PRId64, uid); - } - - // get table entry - SDecoder dc = {0}; - if ((entry.pBuf = taosMemoryMalloc(nData)) == NULL) { - tdbTbcClose(pUidIdxc); - tdbTbcClose(pTbDbc); - return terrno; - } - memcpy(entry.pBuf, pData, nData); - tDecoderInit(&dc, entry.pBuf, nData); - ret = metaDecodeEntry(&dc, &entry); - if (ret != 0) { - tDecoderClear(&dc); - tdbTbcClose(pUidIdxc); - tdbTbcClose(pTbDbc); - metaError("meta/table: invalide ret: %" PRId32 " alt tb options failed.", ret); - return TSDB_CODE_FAILED; - } - - entry.version = version; - metaWLock(pMeta); - // build SMetaEntry - if (entry.type == TSDB_CHILD_TABLE) { - if (pAlterTbReq->updateTTL) { - metaDeleteTtl(pMeta, &entry); - entry.ctbEntry.ttlDays = pAlterTbReq->newTTL; - metaUpdateTtl(pMeta, &entry); - } - if (pAlterTbReq->newCommentLen >= 0) { - entry.ctbEntry.commentLen = pAlterTbReq->newCommentLen; - entry.ctbEntry.comment = pAlterTbReq->newComment; - } - } else { - if (pAlterTbReq->updateTTL) { - metaDeleteTtl(pMeta, &entry); - entry.ntbEntry.ttlDays = pAlterTbReq->newTTL; - metaUpdateTtl(pMeta, &entry); - } - if (pAlterTbReq->newCommentLen >= 0) { - entry.ntbEntry.commentLen = pAlterTbReq->newCommentLen; - entry.ntbEntry.comment = pAlterTbReq->newComment; - } - } - - // save to table db - if (metaSaveToTbDb(pMeta, &entry) < 0) { - metaError("meta/table: failed to save to tb db:%s uid:%" PRId64, entry.name, entry.uid); - } - - if (metaUpdateUidIdx(pMeta, &entry) < 0) { - metaError("meta/table: failed to update uid idx:%s uid:%" PRId64, entry.name, entry.uid); - } - - if (metaUpdateChangeTime(pMeta, entry.uid, pAlterTbReq->ctimeMs) < 0) { - metaError("meta/table: failed to update change time:%s uid:%" PRId64, entry.name, entry.uid); - } - - metaULock(pMeta); - - tdbTbcClose(pTbDbc); - tdbTbcClose(pUidIdxc); - tDecoderClear(&dc); - if (entry.pBuf) taosMemoryFree(entry.pBuf); - return 0; -} - -static int metaAddTagIndex(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTbReq) { - SMetaEntry stbEntry = {0}; - void *pVal = NULL; - int nVal = 0; - int ret; - int c; - tb_uid_t uid, suid; - int64_t oversion; - const void *pData = NULL; - int nData = 0; - SDecoder dc = {0}; - - if (pAlterTbReq->tagName == NULL) { - return terrno = TSDB_CODE_INVALID_MSG; - } - - // search name index - ret = tdbTbGet(pMeta->pNameIdx, pAlterTbReq->tbName, strlen(pAlterTbReq->tbName) + 1, &pVal, &nVal); - if (ret < 0) { - return terrno = TSDB_CODE_TDB_TABLE_NOT_EXIST; - } else { - uid = *(tb_uid_t *)pVal; - tdbFree(pVal); - pVal = NULL; - } - - if (tdbTbGet(pMeta->pUidIdx, &uid, sizeof(tb_uid_t), &pVal, &nVal) == -1) { - ret = -1; - goto _err; - } - suid = ((SUidIdxVal *)pVal)[0].suid; - - STbDbKey tbDbKey = {0}; - tbDbKey.uid = suid; - tbDbKey.version = ((SUidIdxVal *)pVal)[0].version; - ret = tdbTbGet(pMeta->pTbDb, &tbDbKey, sizeof(tbDbKey), &pVal, &nVal); - if (ret < 0) { - goto _err; - } - tDecoderInit(&dc, pVal, nVal); - ret = metaDecodeEntry(&dc, &stbEntry); - if (ret < 0) { - tDecoderClear(&dc); - goto _err; - } - - // Get target schema info - SSchemaWrapper *pTagSchema = &stbEntry.stbEntry.schemaTag; - if (pTagSchema->nCols == 1 && pTagSchema->pSchema[0].type == TSDB_DATA_TYPE_JSON) { - terrno = TSDB_CODE_VND_COL_ALREADY_EXISTS; - goto _err; - } - SSchema *pCol = NULL; - int32_t iCol = 0; - for (;;) { - pCol = NULL; - if (iCol >= pTagSchema->nCols) break; - pCol = &pTagSchema->pSchema[iCol]; - if (strcmp(pCol->name, pAlterTbReq->tagName) == 0) break; - iCol++; - } - - if (iCol == 0) { - terrno = TSDB_CODE_VND_COL_ALREADY_EXISTS; - goto _err; - } - if (pCol == NULL) { - terrno = TSDB_CODE_VND_COL_NOT_EXISTS; - goto _err; - } - - /* - * iterator all pTdDbc by uid and version - */ - TBC *pCtbIdxc = NULL; - TAOS_CHECK_RETURN(tdbTbcOpen(pMeta->pCtbIdx, &pCtbIdxc, NULL)); - int rc = tdbTbcMoveTo(pCtbIdxc, &(SCtbIdxKey){.suid = suid, .uid = INT64_MIN}, sizeof(SCtbIdxKey), &c); - if (rc < 0) { - tdbTbcClose(pCtbIdxc); - goto _err; - } - for (;;) { - void *pKey, *pVal; - int nKey, nVal; - rc = tdbTbcNext(pCtbIdxc, &pKey, &nKey, &pVal, &nVal); - if (rc < 0) break; - if (((SCtbIdxKey *)pKey)->suid != uid) { - tdbFree(pKey); - tdbFree(pVal); - continue; - } - STagIdxKey *pTagIdxKey = NULL; - int32_t nTagIdxKey; - - const void *pTagData = NULL; - int32_t nTagData = 0; - - STagVal tagVal = {.cid = pCol->colId}; - if (tTagGet((const STag *)pVal, &tagVal)) { - if (IS_VAR_DATA_TYPE(pCol->type)) { - pTagData = tagVal.pData; - nTagData = (int32_t)tagVal.nData; - } else { - pTagData = &(tagVal.i64); - nTagData = tDataTypes[pCol->type].bytes; - } - } else { - if (!IS_VAR_DATA_TYPE(pCol->type)) { - nTagData = tDataTypes[pCol->type].bytes; - } - } - if (metaCreateTagIdxKey(suid, pCol->colId, pTagData, nTagData, pCol->type, uid, &pTagIdxKey, &nTagIdxKey) < 0) { - tdbFree(pKey); - tdbFree(pVal); - metaDestroyTagIdxKey(pTagIdxKey); - tdbTbcClose(pCtbIdxc); - goto _err; - } - ret = tdbTbUpsert(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, NULL, 0, pMeta->txn); - if (ret < 0) { - metaError("meta/table: failed to upsert tag idx:%s uid:%" PRId64, stbEntry.name, stbEntry.uid); - } - metaDestroyTagIdxKey(pTagIdxKey); - pTagIdxKey = NULL; - } - tdbTbcClose(pCtbIdxc); - return 0; - -_err: - // tDecoderClear(&dc1); - // tDecoderClear(&dc2); - // if (ctbEntry.pBuf) taosMemoryFree(ctbEntry.pBuf); - // if (stbEntry.pBuf) tdbFree(stbEntry.pBuf); - // tdbTbcClose(pTbDbc); - // tdbTbcClose(pUidIdxc); - return TSDB_CODE_FAILED; -} - -typedef struct SMetaPair { - void *key; - int nkey; -} SMetaPair; - -static int metaDropTagIndex(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTbReq) { - SMetaEntry stbEntry = {0}; - void *pVal = NULL; - int nVal = 0; - int ret; - int c; - tb_uid_t suid; - int64_t oversion; - const void *pData = NULL; - int nData = 0; - SDecoder dc = {0}; - - if (pAlterTbReq->tagName == NULL) { - return terrno = TSDB_CODE_INVALID_MSG; - } - - // search name index - ret = tdbTbGet(pMeta->pNameIdx, pAlterTbReq->tbName, strlen(pAlterTbReq->tbName) + 1, &pVal, &nVal); - if (ret < 0) { - return terrno = TSDB_CODE_TDB_TABLE_NOT_EXIST; - } - suid = *(tb_uid_t *)pVal; - tdbFree(pVal); - pVal = NULL; - - if (tdbTbGet(pMeta->pUidIdx, &suid, sizeof(tb_uid_t), &pVal, &nVal) == -1) { - ret = -1; - goto _err; - } - - STbDbKey tbDbKey = {0}; - tbDbKey.uid = suid; - tbDbKey.version = ((SUidIdxVal *)pVal)[0].version; - ret = tdbTbGet(pMeta->pTbDb, &tbDbKey, sizeof(tbDbKey), &pVal, &nVal); - if (ret < 0) { - goto _err; - } - - tDecoderInit(&dc, pVal, nVal); - ret = metaDecodeEntry(&dc, &stbEntry); - if (ret < 0) { - tDecoderClear(&dc); - goto _err; - } - - // Get targe schema info - SSchemaWrapper *pTagSchema = &stbEntry.stbEntry.schemaTag; - if (pTagSchema->nCols == 1 && pTagSchema->pSchema[0].type == TSDB_DATA_TYPE_JSON) { - terrno = TSDB_CODE_VND_COL_ALREADY_EXISTS; - goto _err; - } - SSchema *pCol = NULL; - int32_t iCol = 0; - for (;;) { - pCol = NULL; - if (iCol >= pTagSchema->nCols) break; - pCol = &pTagSchema->pSchema[iCol]; - if (strcmp(pCol->name, pAlterTbReq->tagName) == 0) break; - iCol++; - } - if (iCol == 0) { - // cannot drop 1th tag index - terrno = -1; - goto _err; - } - if (pCol == NULL) { - terrno = TSDB_CODE_VND_COL_NOT_EXISTS; - goto _err; - } - - if (IS_IDX_ON(pCol)) { - terrno = TSDB_CODE_VND_COL_ALREADY_EXISTS; - goto _err; - } - - SArray *tagIdxList = taosArrayInit(512, sizeof(SMetaPair)); - if (tagIdxList == NULL) { - goto _err; - } - - TBC *pTagIdxc = NULL; - TAOS_CHECK_RETURN(tdbTbcOpen(pMeta->pTagIdx, &pTagIdxc, NULL)); - int rc = - tdbTbcMoveTo(pTagIdxc, &(STagIdxKey){.suid = suid, .cid = INT32_MIN, .type = pCol->type}, sizeof(STagIdxKey), &c); - for (;;) { - void *pKey, *pVal; - int nKey, nVal; - rc = tdbTbcNext(pTagIdxc, &pKey, &nKey, &pVal, &nVal); - STagIdxKey *pIdxKey = (STagIdxKey *)pKey; - if (pIdxKey->suid != suid || pIdxKey->cid != pCol->colId) { - tdbFree(pKey); - tdbFree(pVal); - continue; - } - - SMetaPair pair = {.key = pKey, nKey = nKey}; - if (taosArrayPush(tagIdxList, &pair) == NULL) { - goto _err; - } - } - tdbTbcClose(pTagIdxc); - - metaWLock(pMeta); - for (int i = 0; i < taosArrayGetSize(tagIdxList); i++) { - SMetaPair *pair = taosArrayGet(tagIdxList, i); - ret = tdbTbDelete(pMeta->pTagIdx, pair->key, pair->nkey, pMeta->txn); - if (ret < 0) { - metaError("meta/table: failed to delete tag idx:%s uid:%" PRId64, stbEntry.name, stbEntry.uid); - } - } - metaULock(pMeta); - - taosArrayDestroy(tagIdxList); - - // set pCol->flags; INDEX_ON - return 0; -_err: - return TSDB_CODE_FAILED; -} -int32_t metaUpdateTableColCompress(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq) { - // impl later - SMetaEntry tbEntry = {0}; - void *pVal = NULL; - int nVal = 0; - int ret; - int c; - tb_uid_t suid; - int64_t oversion; - const void *pData = NULL; - int nData = 0; - SDecoder dc = {0}; - ret = tdbTbGet(pMeta->pNameIdx, pReq->tbName, strlen(pReq->tbName) + 1, &pVal, &nVal); - if (ret < 0) { - return terrno = TSDB_CODE_TDB_TABLE_NOT_EXIST; - } - suid = *(tb_uid_t *)pVal; - tdbFree(pVal); - pVal = NULL; - - if (tdbTbGet(pMeta->pUidIdx, &suid, sizeof(tb_uid_t), &pVal, &nVal) == -1) { - terrno = TSDB_CODE_INVALID_MSG; - ret = -1; - goto _err; - } - - STbDbKey tbDbKey = {0}; - tbDbKey.uid = suid; - tbDbKey.version = ((SUidIdxVal *)pVal)[0].version; - if (tdbTbGet(pMeta->pTbDb, &tbDbKey, sizeof(tbDbKey), &pVal, &nVal) < 0) { - terrno = TSDB_CODE_INVALID_MSG; - tdbFree(pVal); - goto _err; - } - - tDecoderInit(&dc, pVal, nVal); - ret = metaDecodeEntry(&dc, &tbEntry); - if (ret < 0) { - terrno = TSDB_CODE_INVALID_MSG; - tdbFree(pVal); - tDecoderClear(&dc); - goto _err; - } - if (tbEntry.type != TSDB_NORMAL_TABLE && tbEntry.type != TSDB_SUPER_TABLE) { - terrno = TSDB_CODE_INVALID_MSG; - tdbFree(pVal); - tDecoderClear(&dc); - goto _err; - } - int8_t updated = 0; - SColCmprWrapper *wp = &tbEntry.colCmpr; - for (int32_t i = 0; i < wp->nCols; i++) { - SColCmpr *p = &wp->pColCmpr[i]; - if (p->id == pReq->colId) { - uint32_t dst = 0; - updated = tUpdateCompress(p->alg, pReq->compress, TSDB_COLVAL_COMPRESS_DISABLED, TSDB_COLVAL_LEVEL_DISABLED, - TSDB_COLVAL_LEVEL_MEDIUM, &dst); - if (updated > 0) { - p->alg = dst; - } - } - } - if (updated == 0) { - tdbFree(pVal); - tDecoderClear(&dc); - terrno = TSDB_CODE_VND_COLUMN_COMPRESS_ALREADY_EXIST; - goto _err; - } else if (updated < 0) { - tdbFree(pVal); - tDecoderClear(&dc); - terrno = TSDB_CODE_TSC_COMPRESS_LEVEL_ERROR; - goto _err; - } - tbEntry.version = version; - - metaWLock(pMeta); - if (metaSaveToTbDb(pMeta, &tbEntry) < 0) { - metaError("meta/table: failed to save to tb db:%s uid:%" PRId64, tbEntry.name, tbEntry.uid); - } - - if (metaUpdateUidIdx(pMeta, &tbEntry) < 0) { - metaError("meta/table: failed to update uid idx:%s uid:%" PRId64, tbEntry.name, tbEntry.uid); - } - - if (metaUpdateChangeTime(pMeta, suid, pReq->ctimeMs) < 0) { - metaError("meta/table: failed to update change time:%s uid:%" PRId64, tbEntry.name, tbEntry.uid); - } - - metaULock(pMeta); - - tdbFree(pVal); - tDecoderClear(&dc); - - return 0; -_err: - return TSDB_CODE_FAILED; -} - -int metaAlterTable(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq, STableMetaRsp *pMetaRsp) { - pMeta->changed = true; - switch (pReq->action) { - case TSDB_ALTER_TABLE_ADD_COLUMN: - case TSDB_ALTER_TABLE_ADD_COLUMN_WITH_COMPRESS_OPTION: - case TSDB_ALTER_TABLE_DROP_COLUMN: - case TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES: - case TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME: - return metaAlterTableColumn(pMeta, version, pReq, pMetaRsp); - case TSDB_ALTER_TABLE_UPDATE_TAG_VAL: - return metaUpdateTableTagVal(pMeta, version, pReq); - case TSDB_ALTER_TABLE_UPDATE_MULTI_TAG_VAL: - return metaUpdateTableMultiTagVal(pMeta, version, pReq); - return terrno = TSDB_CODE_VND_INVALID_TABLE_ACTION; - case TSDB_ALTER_TABLE_UPDATE_OPTIONS: - return metaUpdateTableOptions(pMeta, version, pReq); - case TSDB_ALTER_TABLE_ADD_TAG_INDEX: - return metaAddTagIndex(pMeta, version, pReq); - case TSDB_ALTER_TABLE_DROP_TAG_INDEX: - return metaDropTagIndex(pMeta, version, pReq); - case TSDB_ALTER_TABLE_UPDATE_COLUMN_COMPRESS: - return metaUpdateTableColCompress(pMeta, version, pReq); - default: - return terrno = TSDB_CODE_VND_INVALID_TABLE_ACTION; - break; - } -} - -static int metaSaveToTbDb(SMeta *pMeta, const SMetaEntry *pME) { - STbDbKey tbDbKey; - void *pKey = NULL; - void *pVal = NULL; - int kLen = 0; - int vLen = 0; - SEncoder coder = {0}; - - // set key and value - tbDbKey.version = pME->version; - tbDbKey.uid = pME->uid; - - metaDebug("vgId:%d, start to save table version:%" PRId64 " uid:%" PRId64, TD_VID(pMeta->pVnode), pME->version, - pME->uid); - - pKey = &tbDbKey; - kLen = sizeof(tbDbKey); - - int32_t ret = 0; - tEncodeSize(metaEncodeEntry, pME, vLen, ret); - if (ret < 0) { - goto _err; - } - - pVal = taosMemoryMalloc(vLen); - if (pVal == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - - tEncoderInit(&coder, pVal, vLen); - - if (metaEncodeEntry(&coder, pME) < 0) { - goto _err; - } - - tEncoderClear(&coder); - - // write to table.db - if (tdbTbInsert(pMeta->pTbDb, pKey, kLen, pVal, vLen, pMeta->txn) < 0) { - goto _err; - } - - taosMemoryFree(pVal); - return 0; - -_err: - metaError("vgId:%d, failed to save table version:%" PRId64 "uid:%" PRId64 " %s", TD_VID(pMeta->pVnode), pME->version, - pME->uid, tstrerror(terrno)); - - taosMemoryFree(pVal); - return TSDB_CODE_FAILED; -} - -static int metaUpdateUidIdx(SMeta *pMeta, const SMetaEntry *pME) { - // upsert cache - SMetaInfo info; - metaGetEntryInfo(pME, &info); - int32_t ret = metaCacheUpsert(pMeta, &info); - if (ret < 0) { - metaError("vgId:%d, failed to upsert cache, uid: %" PRId64 " %s", TD_VID(pMeta->pVnode), pME->uid, tstrerror(ret)); - } - - SUidIdxVal uidIdxVal = {.suid = info.suid, .version = info.version, .skmVer = info.skmVer}; - - return tdbTbUpsert(pMeta->pUidIdx, &pME->uid, sizeof(tb_uid_t), &uidIdxVal, sizeof(uidIdxVal), pMeta->txn); -} - -static int metaUpdateSuidIdx(SMeta *pMeta, const SMetaEntry *pME) { - return tdbTbUpsert(pMeta->pSuidIdx, &pME->uid, sizeof(tb_uid_t), NULL, 0, pMeta->txn); -} - -static int metaUpdateNameIdx(SMeta *pMeta, const SMetaEntry *pME) { - return tdbTbUpsert(pMeta->pNameIdx, pME->name, strlen(pME->name) + 1, &pME->uid, sizeof(tb_uid_t), pMeta->txn); -} - -static void metaUpdateTtl(SMeta *pMeta, const SMetaEntry *pME) { - if (pME->type != TSDB_CHILD_TABLE && pME->type != TSDB_NORMAL_TABLE) return; - - STtlUpdTtlCtx ctx = {.uid = pME->uid, .pTxn = pMeta->txn}; - if (pME->type == TSDB_CHILD_TABLE) { - ctx.ttlDays = pME->ctbEntry.ttlDays; - ctx.changeTimeMs = pME->ctbEntry.btime; - } else { - ctx.ttlDays = pME->ntbEntry.ttlDays; - ctx.changeTimeMs = pME->ntbEntry.btime; - } - - int32_t ret = ttlMgrInsertTtl(pMeta->pTtlMgr, &ctx); - if (ret < 0) { - metaError("vgId:%d, failed to insert ttl, uid: %" PRId64 " %s", TD_VID(pMeta->pVnode), pME->uid, tstrerror(ret)); - } - - return; -} - -static int metaUpdateChangeTime(SMeta *pMeta, tb_uid_t uid, int64_t changeTimeMs) { - if (!tsTtlChangeOnWrite) return 0; - - if (changeTimeMs <= 0) { - metaWarn("Skip to change ttl deletetion time on write, uid: %" PRId64, uid); - return TSDB_CODE_VERSION_NOT_COMPATIBLE; - } - - STtlUpdCtimeCtx ctx = {.uid = uid, .changeTimeMs = changeTimeMs, .pTxn = pMeta->txn}; - - return ttlMgrUpdateChangeTime(pMeta->pTtlMgr, &ctx); -} - -int metaUpdateChangeTimeWithLock(SMeta *pMeta, tb_uid_t uid, int64_t changeTimeMs) { - if (!tsTtlChangeOnWrite) return 0; - - metaWLock(pMeta); - int ret = metaUpdateChangeTime(pMeta, uid, changeTimeMs); - metaULock(pMeta); - return ret; -} - -static int metaUpdateCtbIdx(SMeta *pMeta, const SMetaEntry *pME) { - SCtbIdxKey ctbIdxKey = {.suid = pME->ctbEntry.suid, .uid = pME->uid}; - - return tdbTbUpsert(pMeta->pCtbIdx, &ctbIdxKey, sizeof(ctbIdxKey), pME->ctbEntry.pTags, - ((STag *)(pME->ctbEntry.pTags))->len, pMeta->txn); -} - -int metaCreateTagIdxKey(tb_uid_t suid, int32_t cid, const void *pTagData, int32_t nTagData, int8_t type, tb_uid_t uid, - STagIdxKey **ppTagIdxKey, int32_t *nTagIdxKey) { - if (IS_VAR_DATA_TYPE(type)) { - *nTagIdxKey = sizeof(STagIdxKey) + nTagData + VARSTR_HEADER_SIZE + sizeof(tb_uid_t); - } else { - *nTagIdxKey = sizeof(STagIdxKey) + nTagData + sizeof(tb_uid_t); - } - - *ppTagIdxKey = (STagIdxKey *)taosMemoryMalloc(*nTagIdxKey); - if (*ppTagIdxKey == NULL) { - return terrno; - } - - (*ppTagIdxKey)->suid = suid; - (*ppTagIdxKey)->cid = cid; - (*ppTagIdxKey)->isNull = (pTagData == NULL) ? 1 : 0; - (*ppTagIdxKey)->type = type; - - // refactor - if (IS_VAR_DATA_TYPE(type)) { - memcpy((*ppTagIdxKey)->data, (uint16_t *)&nTagData, VARSTR_HEADER_SIZE); - if (pTagData != NULL) memcpy((*ppTagIdxKey)->data + VARSTR_HEADER_SIZE, pTagData, nTagData); - *(tb_uid_t *)((*ppTagIdxKey)->data + VARSTR_HEADER_SIZE + nTagData) = uid; - } else { - if (pTagData != NULL) memcpy((*ppTagIdxKey)->data, pTagData, nTagData); - *(tb_uid_t *)((*ppTagIdxKey)->data + nTagData) = uid; - } - - return 0; -} - -static void metaDestroyTagIdxKey(STagIdxKey *pTagIdxKey) { - if (pTagIdxKey) taosMemoryFree(pTagIdxKey); -} - -static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) { - void *pData = NULL; - int nData = 0; - STbDbKey tbDbKey = {0}; - SMetaEntry stbEntry = {0}; - STagIdxKey *pTagIdxKey = NULL; - int32_t nTagIdxKey; - const SSchema *pTagColumn; - const void *pTagData = NULL; - int32_t nTagData = 0; - SDecoder dc = {0}; - int32_t ret = 0; - // get super table - if (tdbTbGet(pMeta->pUidIdx, &pCtbEntry->ctbEntry.suid, sizeof(tb_uid_t), &pData, &nData) != 0) { - metaError("vgId:%d, failed to get stable suid for update. version:%" PRId64, TD_VID(pMeta->pVnode), - pCtbEntry->version); - ret = TSDB_CODE_TDB_INVALID_TABLE_ID; - goto end; - } - tbDbKey.uid = pCtbEntry->ctbEntry.suid; - tbDbKey.version = ((SUidIdxVal *)pData)[0].version; - ret = tdbTbGet(pMeta->pTbDb, &tbDbKey, sizeof(tbDbKey), &pData, &nData); - if (ret < 0) { - metaError("vgId:%d, failed to get stable for update. version:%" PRId64, TD_VID(pMeta->pVnode), pCtbEntry->version); - goto end; - } - - tDecoderInit(&dc, pData, nData); - ret = metaDecodeEntry(&dc, &stbEntry); - if (ret < 0) { - goto end; - } - - if (stbEntry.stbEntry.schemaTag.pSchema == NULL) { - ret = TSDB_CODE_INVALID_PARA; - goto end; - } - - SSchemaWrapper *pTagSchema = &stbEntry.stbEntry.schemaTag; - if (pTagSchema->nCols == 1 && pTagSchema->pSchema[0].type == TSDB_DATA_TYPE_JSON) { - pTagColumn = &stbEntry.stbEntry.schemaTag.pSchema[0]; - STagVal tagVal = {.cid = pTagColumn->colId}; - - pTagData = pCtbEntry->ctbEntry.pTags; - nTagData = ((const STag *)pCtbEntry->ctbEntry.pTags)->len; - ret = metaSaveJsonVarToIdx(pMeta, pCtbEntry, pTagColumn); - goto end; - } else { - for (int i = 0; i < pTagSchema->nCols; i++) { - pTagData = NULL; - nTagData = 0; - pTagColumn = &pTagSchema->pSchema[i]; - if (!IS_IDX_ON(pTagColumn)) continue; - - STagVal tagVal = {.cid = pTagColumn->colId}; - if (tTagGet((const STag *)pCtbEntry->ctbEntry.pTags, &tagVal)) { - if (IS_VAR_DATA_TYPE(pTagColumn->type)) { - pTagData = tagVal.pData; - nTagData = (int32_t)tagVal.nData; - } else { - pTagData = &(tagVal.i64); - nTagData = tDataTypes[pTagColumn->type].bytes; - } - } else { - if (!IS_VAR_DATA_TYPE(pTagColumn->type)) { - nTagData = tDataTypes[pTagColumn->type].bytes; - } - } - if (metaCreateTagIdxKey(pCtbEntry->ctbEntry.suid, pTagColumn->colId, pTagData, nTagData, pTagColumn->type, - pCtbEntry->uid, &pTagIdxKey, &nTagIdxKey) < 0) { - ret = -1; - goto end; - } - if (tdbTbUpsert(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, NULL, 0, pMeta->txn) < 0) { - metaError("vgId:%d, failed to update tag index. version:%" PRId64, TD_VID(pMeta->pVnode), pCtbEntry->version); - } - metaDestroyTagIdxKey(pTagIdxKey); - pTagIdxKey = NULL; - } - } -end: - tDecoderClear(&dc); - tdbFree(pData); - return ret; -} - -static int metaSaveToSkmDb(SMeta *pMeta, const SMetaEntry *pME) { - SEncoder coder = {0}; - void *pVal = NULL; - int vLen = 0; - int rcode = 0; - SSkmDbKey skmDbKey = {0}; - const SSchemaWrapper *pSW; - - if (pME->type == TSDB_SUPER_TABLE) { - pSW = &pME->stbEntry.schemaRow; - } else if (pME->type == TSDB_NORMAL_TABLE) { - pSW = &pME->ntbEntry.schemaRow; - } else { - metaError("meta/table: invalide table type: %" PRId8 " save skm db failed.", pME->type); - return TSDB_CODE_FAILED; - } - - skmDbKey.uid = pME->uid; - skmDbKey.sver = pSW->version; - - // if receive tmq meta message is: create stable1 then delete stable1 then create stable1 with multi vgroups - if (tdbTbGet(pMeta->pSkmDb, &skmDbKey, sizeof(skmDbKey), NULL, NULL) == 0) { - return rcode; - } - - // encode schema - int32_t ret = 0; - tEncodeSize(tEncodeSSchemaWrapper, pSW, vLen, ret); - if (ret < 0) return -1; - pVal = taosMemoryMalloc(vLen); - if (pVal == NULL) { - rcode = -1; - terrno = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; - } - - tEncoderInit(&coder, pVal, vLen); - ret = tEncodeSSchemaWrapper(&coder, pSW); - if (ret < 0) { - rcode = -1; - goto _exit; - } - - if (tdbTbInsert(pMeta->pSkmDb, &skmDbKey, sizeof(skmDbKey), pVal, vLen, pMeta->txn) < 0) { - rcode = -1; - goto _exit; - } - - metaDebug("vgId:%d, set schema:(%" PRId64 ") sver:%d since %s", TD_VID(pMeta->pVnode), pME->uid, pSW->version, - tstrerror(terrno)); - -_exit: - taosMemoryFree(pVal); - tEncoderClear(&coder); - return rcode; -} - -int metaHandleEntry(SMeta *pMeta, const SMetaEntry *pME) { - int32_t code = 0; - int32_t line = 0; - metaWLock(pMeta); - - // save to table.db - code = metaSaveToTbDb(pMeta, pME); - VND_CHECK_CODE(code, line, _err); - - // update uid.idx - code = metaUpdateUidIdx(pMeta, pME); - VND_CHECK_CODE(code, line, _err); - - // update name.idx - code = metaUpdateNameIdx(pMeta, pME); - VND_CHECK_CODE(code, line, _err); - - if (pME->type == TSDB_CHILD_TABLE) { - // update ctb.idx - code = metaUpdateCtbIdx(pMeta, pME); - VND_CHECK_CODE(code, line, _err); - - // update tag.idx - code = metaUpdateTagIdx(pMeta, pME); - VND_CHECK_CODE(code, line, _err); - } else { - // update schema.db - code = metaSaveToSkmDb(pMeta, pME); - VND_CHECK_CODE(code, line, _err); - - if (pME->type == TSDB_SUPER_TABLE) { - code = metaUpdateSuidIdx(pMeta, pME); - VND_CHECK_CODE(code, line, _err); - } - } - - code = metaUpdateBtimeIdx(pMeta, pME); - VND_CHECK_CODE(code, line, _err); - - if (pME->type == TSDB_NORMAL_TABLE) { - code = metaUpdateNcolIdx(pMeta, pME); - VND_CHECK_CODE(code, line, _err); - } - - if (pME->type != TSDB_SUPER_TABLE) { - metaUpdateTtl(pMeta, pME); - } - - if (pME->type == TSDB_SUPER_TABLE || pME->type == TSDB_NORMAL_TABLE) { - } - - metaULock(pMeta); - metaDebug("vgId:%d, handle meta entry, ver:%" PRId64 ", uid:%" PRId64 ", name:%s", TD_VID(pMeta->pVnode), - pME->version, pME->uid, pME->name); - return 0; - -_err: - metaULock(pMeta); - metaError("vgId:%d, failed to handle meta entry since %s at line:%d, ver:%" PRId64 ", uid:%" PRId64 ", name:%s", - TD_VID(pMeta->pVnode), terrstr(), line, pME->version, pME->uid, pME->name); - return TSDB_CODE_FAILED; -} - -static void colCompressDebug(SHashObj *pColCmprObj) { - void *p = taosHashIterate(pColCmprObj, NULL); - while (p) { - uint32_t cmprAlg = *(uint32_t *)p; - col_id_t colId = *(col_id_t *)taosHashGetKey(p, NULL); - p = taosHashIterate(pColCmprObj, p); - - uint8_t l1, l2, lvl; - tcompressDebug(cmprAlg, &l1, &l2, &lvl); - - const char *l1str = columnEncodeStr(l1); - const char *l2str = columnCompressStr(l2); - const char *lvlstr = columnLevelStr(lvl); - metaDebug("colId: %d, encode:%s, compress:%s,level:%s", colId, l1str, l2str, lvlstr); - } - return; -} -int32_t metaGetColCmpr(SMeta *pMeta, tb_uid_t uid, SHashObj **ppColCmprObj) { - int rc = 0; - - SHashObj *pColCmprObj = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_SMALLINT), false, HASH_NO_LOCK); - if (pColCmprObj == NULL) { - pColCmprObj = NULL; - return TSDB_CODE_OUT_OF_MEMORY; - } - - void *pData = NULL; - int nData = 0; - SMetaEntry e = {0}; - SDecoder dc = {0}; - - *ppColCmprObj = NULL; - - metaRLock(pMeta); - rc = tdbTbGet(pMeta->pUidIdx, &uid, sizeof(uid), &pData, &nData); - if (rc < 0) { - taosHashClear(pColCmprObj); - metaULock(pMeta); - return TSDB_CODE_FAILED; - } - int64_t version = ((SUidIdxVal *)pData)[0].version; - rc = tdbTbGet(pMeta->pTbDb, &(STbDbKey){.version = version, .uid = uid}, sizeof(STbDbKey), &pData, &nData); - if (rc < 0) { - metaULock(pMeta); - taosHashClear(pColCmprObj); - metaError("failed to get table entry"); - return rc; - } - - tDecoderInit(&dc, pData, nData); - rc = metaDecodeEntry(&dc, &e); - if (rc < 0) { - tDecoderClear(&dc); - tdbFree(pData); - metaULock(pMeta); - taosHashClear(pColCmprObj); - return rc; - } - if (useCompress(e.type)) { - SColCmprWrapper *p = &e.colCmpr; - for (int32_t i = 0; i < p->nCols; i++) { - SColCmpr *pCmpr = &p->pColCmpr[i]; - rc = taosHashPut(pColCmprObj, &pCmpr->id, sizeof(pCmpr->id), &pCmpr->alg, sizeof(pCmpr->alg)); - if (rc < 0) { - tDecoderClear(&dc); - tdbFree(pData); - metaULock(pMeta); - taosHashClear(pColCmprObj); - return rc; - } - } - } else { - tDecoderClear(&dc); - tdbFree(pData); - metaULock(pMeta); - taosHashClear(pColCmprObj); - return 0; - } - tDecoderClear(&dc); - tdbFree(pData); - metaULock(pMeta); - - *ppColCmprObj = pColCmprObj; - colCompressDebug(pColCmprObj); - - return 0; -} -// refactor later -void *metaGetIdx(SMeta *pMeta) { return pMeta->pTagIdx; } -void *metaGetIvtIdx(SMeta *pMeta) { return pMeta->pTagIvtIdx; } +// Alter Normal Table diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 328c2c1585..b7ec3c0b6f 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -491,7 +491,7 @@ static int32_t vnodePreProcessArbCheckSyncMsg(SVnode *pVnode, SRpcMsg *pMsg) { return code; } -int32_t vnodePreProcessDropTbMsg(SVnode* pVnode, SRpcMsg* pMsg) { +int32_t vnodePreProcessDropTbMsg(SVnode *pVnode, SRpcMsg *pMsg) { int32_t code = TSDB_CODE_SUCCESS; int32_t lino = 0; int32_t size = 0; @@ -514,14 +514,14 @@ int32_t vnodePreProcessDropTbMsg(SVnode* pVnode, SRpcMsg* pMsg) { } for (int32_t i = 0; i < receivedBatchReqs.nReqs; ++i) { - SVDropTbReq* pReq = receivedBatchReqs.pReqs + i; - tb_uid_t uid = metaGetTableEntryUidByName(pVnode->pMeta, pReq->name); + SVDropTbReq *pReq = receivedBatchReqs.pReqs + i; + tb_uid_t uid = metaGetTableEntryUidByName(pVnode->pMeta, pReq->name); if (uid == 0) { vWarn("vgId:%d, preprocess drop ctb: %s not found", TD_VID(pVnode), pReq->name); continue; } pReq->uid = uid; - vDebug("vgId:%d %s for: %s, uid: %"PRId64, TD_VID(pVnode), __func__, pReq->name, pReq->uid); + vDebug("vgId:%d %s for: %s, uid: %" PRId64, TD_VID(pVnode), __func__, pReq->name, pReq->uid); if (taosArrayPush(sentBatchReqs.pArray, pReq) == NULL) { code = terrno; goto _exit; @@ -1145,7 +1145,7 @@ static int32_t vnodeProcessCreateStbReq(SVnode *pVnode, int64_t ver, void *pReq, goto _err; } - code = metaCreateSTable(pVnode->pMeta, ver, &req); + code = metaCreateSuperTable(pVnode->pMeta, ver, &req); if (code) { pRsp->code = code; goto _err; @@ -2579,4 +2579,3 @@ _OVER: int32_t vnodeAsyncCompact(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) { return 0; } int32_t tsdbAsyncCompact(STsdb *tsdb, const STimeWindow *tw, bool sync) { return 0; } #endif - From 98c7111f13f97bc85af87a78a658359c6b6049fa Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 4 Dec 2024 19:28:35 +0800 Subject: [PATCH 06/45] fix: compile issue --- source/dnode/vnode/src/meta/metaEntry2.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaEntry2.c b/source/dnode/vnode/src/meta/metaEntry2.c index 27680950dc..46117f19d9 100644 --- a/source/dnode/vnode/src/meta/metaEntry2.c +++ b/source/dnode/vnode/src/meta/metaEntry2.c @@ -240,8 +240,7 @@ static int32_t metaHandleSuperTableCreate(SMeta *pMeta, const SMetaEntry *pEntry metaInfo("vgId:%d, %s success, version:%" PRId64 " type:%d uid:%" PRId64 " name:%s", vgId, __func__, pEntry->version, pEntry->type, pEntry->uid, pEntry->name); } else { - metaError("vgId:%d, %s failed at %s:%d since %s, version" PRId64 " type:%d uid:%" PRId64 " name:%s", vgId, __func__, - __FILE__, __LINE__, tstrerror(code), pEntry->version, pEntry->type, pEntry->uid, pEntry->name); + metaErr(vgId, code); } return code; } @@ -329,9 +328,7 @@ int32_t metaHandleEntry2(SMeta *pMeta, const SMetaEntry *pEntry) { metaDebug("vgId:%d, %s success, version:%" PRId64 " type:%d uid:%" PRId64 " name:%s", vgId, __func__, pEntry->version, pEntry->type, pEntry->uid, pEntry->type > 0 ? pEntry->name : ""); } else { - metaError("vgId:%d, %s failed at %s:%d since %s, version" PRId64 " type:%d uid:%" PRId64 " name:%s", vgId, __func__, - __FILE__, __LINE__, tstrerror(code), pEntry->version, pEntry->type, pEntry->uid, - pEntry->type > 0 ? pEntry->name : ""); + metaErr(vgId, code); } TAOS_RETURN(code); } \ No newline at end of file From 96b9e3d20e61bb7e7ea99caf98af4e53cbea8c03 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 5 Dec 2024 13:41:54 +0800 Subject: [PATCH 07/45] fix error --- source/dnode/vnode/src/meta/metaEntry.c | 5 +++-- source/dnode/vnode/src/meta/metaEntry2.c | 6 ++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaEntry.c b/source/dnode/vnode/src/meta/metaEntry.c index 49f9dfa85a..7ec73a6abb 100644 --- a/source/dnode/vnode/src/meta/metaEntry.c +++ b/source/dnode/vnode/src/meta/metaEntry.c @@ -71,7 +71,7 @@ int metaEncodeEntry(SEncoder *pCoder, const SMetaEntry *pME) { TAOS_CHECK_RETURN(tEncodeI8(pCoder, pME->type)); TAOS_CHECK_RETURN(tEncodeI64(pCoder, pME->uid)); - if (pME->type < 0) { + if (pME->type > 0) { if (pME->name == NULL) { return TSDB_CODE_INVALID_PARA; } @@ -121,7 +121,8 @@ int metaDecodeEntry(SDecoder *pCoder, SMetaEntry *pME) { TAOS_CHECK_RETURN(tDecodeI64(pCoder, &pME->version)); TAOS_CHECK_RETURN(tDecodeI8(pCoder, &pME->type)); TAOS_CHECK_RETURN(tDecodeI64(pCoder, &pME->uid)); - if (pME->type < 0) { + + if (pME->type > 0) { TAOS_CHECK_RETURN(tDecodeCStr(pCoder, &pME->name)); if (pME->type == TSDB_SUPER_TABLE) { diff --git a/source/dnode/vnode/src/meta/metaEntry2.c b/source/dnode/vnode/src/meta/metaEntry2.c index 46117f19d9..a369afaad1 100644 --- a/source/dnode/vnode/src/meta/metaEntry2.c +++ b/source/dnode/vnode/src/meta/metaEntry2.c @@ -98,6 +98,12 @@ static int32_t metaSchemaTableInsert(SMeta *pMeta, const SMetaEntry *pEntry) { return code; } + value = taosMemoryMalloc(valueSize); + if (NULL == value) { + metaErr(vgId, terrno); + return terrno; + } + tEncoderInit(&encoder, value, valueSize); code = tEncodeSSchemaWrapper(&encoder, pSchema); if (TSDB_CODE_SUCCESS != code) { From a51be1cee3fcc2e6b667b15ff03225a8dd019e56 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 5 Dec 2024 14:50:00 +0800 Subject: [PATCH 08/45] fix: invalid logic --- source/dnode/vnode/src/meta/metaEntry2.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaEntry2.c b/source/dnode/vnode/src/meta/metaEntry2.c index a369afaad1..b500ff474c 100644 --- a/source/dnode/vnode/src/meta/metaEntry2.c +++ b/source/dnode/vnode/src/meta/metaEntry2.c @@ -275,8 +275,7 @@ int32_t metaHandleEntry2(SMeta *pMeta, const SMetaEntry *pEntry) { SMetaInfo info = {0}; int8_t type = pEntry->type > 0 ? pEntry->type : -pEntry->type; - if (NULL == pMeta || NULL == pEntry || type != TSDB_SUPER_TABLE || type != TSDB_CHILD_TABLE || - type != TSDB_NORMAL_TABLE) { + if (NULL == pMeta || NULL == pEntry) { metaError("%s failed at %s:%d since invalid parameter", __func__, __FILE__, __LINE__); return TSDB_CODE_INVALID_PARA; } From 60ffaddc92da61b2f286cbd81e0d6c5138a682a5 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 5 Dec 2024 16:57:25 +0800 Subject: [PATCH 09/45] fix: CI issue --- source/dnode/vnode/src/meta/metaEntry2.c | 305 ++++++++++++++++++----- source/dnode/vnode/src/meta/metaTable2.c | 218 ++++++++++++++++ 2 files changed, 458 insertions(+), 65 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaEntry2.c b/source/dnode/vnode/src/meta/metaEntry2.c index b500ff474c..33694c6443 100644 --- a/source/dnode/vnode/src/meta/metaEntry2.c +++ b/source/dnode/vnode/src/meta/metaEntry2.c @@ -17,8 +17,32 @@ pEntry->type > 0 ? pEntry->name : NULL); \ } while (0) +typedef enum { + META_ENTRY_TABLE = 0, + META_SCHEMA_TABLE, + META_UID_IDX, + META_NAME_IDX, + META_SUID_IDX, + META_TAG_IDX, + META_BTIME_IDX, + META_TTL_IDX, + META_TABLE_MAX, +} EMetaTable; + +typedef enum { + META_TABLE_OP_INSERT = 0, + META_TABLE_OP_UPDATA, + META_TABLE_OP_DELETE, + META_TABLE_OP_MAX, +} EMetaTableOp; + +typedef struct { + EMetaTable table; + EMetaTableOp op; +} SMetaTableOp; + // Entry Table -static int32_t metaEntryTableInsert(SMeta *pMeta, const SMetaEntry *pEntry) { +static int32_t metaEntryTableUpsert(SMeta *pMeta, const SMetaEntry *pEntry, EMetaTableOp op) { int32_t code = TSDB_CODE_SUCCESS; int32_t vgId = TD_VID(pMeta->pVnode); void *value = NULL; @@ -53,7 +77,13 @@ static int32_t metaEntryTableInsert(SMeta *pMeta, const SMetaEntry *pEntry) { tEncoderClear(&encoder); // put to tdb - code = tdbTbInsert(pMeta->pTbDb, &key, sizeof(key), value, valueSize, pMeta->txn); + if (META_TABLE_OP_INSERT == op) { + code = tdbTbInsert(pMeta->pTbDb, &key, sizeof(key), value, valueSize, pMeta->txn); + } else if (META_TABLE_OP_UPDATA == op) { + code = tdbTbUpsert(pMeta->pTbDb, &key, sizeof(key), value, valueSize, pMeta->txn); + } else { + code = TSDB_CODE_INVALID_PARA; + } if (TSDB_CODE_SUCCESS != code) { metaErr(vgId, code); } @@ -61,7 +91,15 @@ static int32_t metaEntryTableInsert(SMeta *pMeta, const SMetaEntry *pEntry) { return code; } -static int32_t metaEntryTableDrop(SMeta *pMeta, const SMetaEntry *pEntry) { +static int32_t metaEntryTableInsert(SMeta *pMeta, const SMetaEntry *pEntry) { + return metaEntryTableUpsert(pMeta, pEntry, META_TABLE_OP_INSERT); +} + +static int32_t metaEntryTableUpdate(SMeta *pMeta, const SMetaEntry *pEntry) { + return metaEntryTableUpsert(pMeta, pEntry, META_TABLE_OP_UPDATA); +} + +static int32_t metaEntryTableDelete(SMeta *pMeta, const SMetaEntry *pEntry) { int32_t code = TSDB_CODE_SUCCESS; int32_t vgId = TD_VID(pMeta->pVnode); @@ -71,7 +109,7 @@ static int32_t metaEntryTableDrop(SMeta *pMeta, const SMetaEntry *pEntry) { } // Schema Table -static int32_t metaSchemaTableInsert(SMeta *pMeta, const SMetaEntry *pEntry) { +static int32_t metaSchemaTableUpsert(SMeta *pMeta, const SMetaEntry *pEntry, EMetaTableOp op) { int32_t code = TSDB_CODE_SUCCESS; int32_t vgId = TD_VID(pMeta->pVnode); SEncoder encoder = {0}; @@ -115,7 +153,13 @@ static int32_t metaSchemaTableInsert(SMeta *pMeta, const SMetaEntry *pEntry) { tEncoderClear(&encoder); // put to tdb - code = tdbTbInsert(pMeta->pSkmDb, &key, sizeof(key), value, valueSize, pMeta->txn); + if (META_TABLE_OP_INSERT == op) { + code = tdbTbInsert(pMeta->pSkmDb, &key, sizeof(key), value, valueSize, pMeta->txn); + } else if (META_TABLE_OP_UPDATA == op) { + code = tdbTbUpsert(pMeta->pSkmDb, &key, sizeof(key), value, valueSize, pMeta->txn); + } else { + code = TSDB_CODE_INVALID_PARA; + } if (TSDB_CODE_SUCCESS != code) { metaErr(vgId, code); } @@ -123,6 +167,14 @@ static int32_t metaSchemaTableInsert(SMeta *pMeta, const SMetaEntry *pEntry) { return code; } +static int32_t metaSchemaTableInsert(SMeta *pMeta, const SMetaEntry *pEntry) { + return metaSchemaTableUpsert(pMeta, pEntry, META_TABLE_OP_INSERT); +} + +static int32_t metaSchemaTableUpdate(SMeta *pMeta, const SMetaEntry *pEntry) { + return metaSchemaTableUpsert(pMeta, pEntry, META_TABLE_OP_UPDATA); +} + // Uid Index static void metaBuildEntryInfo(const SMetaEntry *pEntry, SMetaInfo *pInfo) { pInfo->uid = pEntry->uid; @@ -138,7 +190,8 @@ static void metaBuildEntryInfo(const SMetaEntry *pEntry, SMetaInfo *pInfo) { pInfo->skmVer = pEntry->ntbEntry.schemaRow.version; } } -static int32_t metaUidIdxUpsert(SMeta *pMeta, const SMetaEntry *pEntry) { + +static int32_t metaUidIdxUpsert(SMeta *pMeta, const SMetaEntry *pEntry, EMetaTableOp op) { int32_t code = TSDB_CODE_SUCCESS; int32_t vgId = TD_VID(pMeta->pVnode); @@ -152,30 +205,48 @@ static int32_t metaUidIdxUpsert(SMeta *pMeta, const SMetaEntry *pEntry) { // put to tdb SUidIdxVal value = { - .suid = 0, - .skmVer = 0, + .suid = info.suid, + .skmVer = info.skmVer, .version = pEntry->version, }; - code = tdbTbUpsert(pMeta->pUidIdx, &pEntry->uid, sizeof(pEntry->uid), &value, sizeof(value), pMeta->txn); + if (META_TABLE_OP_INSERT == op) { + code = tdbTbInsert(pMeta->pUidIdx, &pEntry->uid, sizeof(pEntry->uid), &value, sizeof(value), pMeta->txn); + } else if (META_TABLE_OP_UPDATA == op) { + code = tdbTbUpsert(pMeta->pUidIdx, &pEntry->uid, sizeof(pEntry->uid), &value, sizeof(value), pMeta->txn); + } return code; } +static int32_t metaUidIdxInsert(SMeta *pMeta, const SMetaEntry *pEntry) { + return metaUidIdxUpsert(pMeta, pEntry, META_TABLE_OP_INSERT); +} + +static int32_t metaUidIdxUpdate(SMeta *pMeta, const SMetaEntry *pEntry) { + return metaUidIdxUpsert(pMeta, pEntry, META_TABLE_OP_UPDATA); +} + // Name Index -static int32_t metaNameIdxInsert(SMeta *pMeta, const SMetaEntry *pEntry) { - int32_t code = tdbTbInsert(pMeta->pNameIdx, pEntry->name, strlen(pEntry->name) + 1, &pEntry->uid, sizeof(pEntry->uid), - pMeta->txn); - if (code) { - metaErr(TD_VID(pMeta->pVnode), code); +static int32_t metaNameIdxUpsert(SMeta *pMeta, const SMetaEntry *pEntry, EMetaTableOp op) { + int32_t code = TSDB_CODE_SUCCESS; + if (META_TABLE_OP_INSERT == op) { + code = tdbTbInsert(pMeta->pNameIdx, pEntry->name, strlen(pEntry->name) + 1, &pEntry->uid, sizeof(pEntry->uid), + pMeta->txn); + } else if (META_TABLE_OP_UPDATA == op) { + code = tdbTbUpsert(pMeta->pNameIdx, pEntry->name, strlen(pEntry->name) + 1, &pEntry->uid, sizeof(pEntry->uid), + pMeta->txn); + } else { + code = TSDB_CODE_INVALID_PARA; } return code; } +static int32_t metaNameIdxInsert(SMeta *pMeta, const SMetaEntry *pEntry) { + return metaNameIdxUpsert(pMeta, pEntry, META_TABLE_OP_INSERT); +} + static int32_t metaNameIdxDelete(SMeta *pMeta, const SMetaEntry *pEntry) { - int32_t code = tdbTbDelete(pMeta->pNameIdx, pEntry->name, strlen(pEntry->name) + 1, pMeta->txn); - if (code) { - metaErr(TD_VID(pMeta->pVnode), code); - } - return code; + // TODO + return 0; } // Suid Index @@ -189,51 +260,109 @@ static int32_t metaSUidIdxInsert(SMeta *pMeta, const SMetaEntry *pEntry) { // Tag Index +// Btime Index +static int32_t metaBtimeIdxUpsert(SMeta *pMeta, const SMetaEntry *pEntry, EMetaTableOp op) { + SBtimeIdxKey key = { + .uid = pEntry->uid, + }; + + if (TSDB_CHILD_TABLE == pEntry->type) { + key.btime = pEntry->ctbEntry.btime; + } else if (TSDB_NORMAL_TABLE == pEntry->type) { + key.btime = pEntry->ntbEntry.btime; + } else { + return TSDB_CODE_INVALID_PARA; + } + + if (META_TABLE_OP_INSERT == op) { + return tdbTbInsert(pMeta->pBtimeIdx, &key, sizeof(key), NULL, 0, pMeta->txn); + } else if (META_TABLE_OP_UPDATA == op) { + return tdbTbUpsert(pMeta->pBtimeIdx, &key, sizeof(key), NULL, 0, pMeta->txn); + } else { + return TSDB_CODE_INVALID_PARA; + } +} + +static int32_t metaBtimeIdxInsert(SMeta *pMeta, const SMetaEntry *pEntry) { + return metaBtimeIdxUpsert(pMeta, pEntry, META_TABLE_OP_INSERT); +} + +static int32_t (*metaTableOpFn[META_TABLE_MAX][META_TABLE_OP_MAX])(SMeta *pMeta, const SMetaEntry *pEntry) = { + [META_ENTRY_TABLE] = + { + [META_TABLE_OP_INSERT] = metaEntryTableInsert, + [META_TABLE_OP_UPDATA] = metaEntryTableUpdate, + [META_TABLE_OP_DELETE] = metaEntryTableDelete, + }, + [META_SCHEMA_TABLE] = + { + [META_TABLE_OP_INSERT] = metaSchemaTableInsert, + [META_TABLE_OP_UPDATA] = NULL, + [META_TABLE_OP_DELETE] = NULL, + }, + [META_UID_IDX] = + { + [META_TABLE_OP_INSERT] = metaUidIdxInsert, + [META_TABLE_OP_UPDATA] = metaUidIdxUpdate, + [META_TABLE_OP_DELETE] = NULL, + }, + [META_NAME_IDX] = + { + [META_TABLE_OP_INSERT] = metaNameIdxInsert, + [META_TABLE_OP_UPDATA] = NULL, + [META_TABLE_OP_DELETE] = NULL, + }, + [META_SUID_IDX] = + { + [META_TABLE_OP_INSERT] = metaSUidIdxInsert, + [META_TABLE_OP_UPDATA] = NULL, + [META_TABLE_OP_DELETE] = NULL, + }, + [META_TAG_IDX] = + { + [META_TABLE_OP_INSERT] = NULL, + [META_TABLE_OP_UPDATA] = NULL, + [META_TABLE_OP_DELETE] = NULL, + }, + [META_BTIME_IDX] = + { + [META_TABLE_OP_INSERT] = metaBtimeIdxInsert, + [META_TABLE_OP_UPDATA] = NULL, + [META_TABLE_OP_DELETE] = NULL, + }, + [META_TTL_IDX] = + { + [META_TABLE_OP_INSERT] = NULL, + [META_TABLE_OP_UPDATA] = NULL, + [META_TABLE_OP_DELETE] = NULL, + }, +}; + static int32_t metaHandleSuperTableCreateImpl(SMeta *pMeta, const SMetaEntry *pEntry) { int32_t code = TSDB_CODE_SUCCESS; - int32_t vgId = TD_VID(pMeta->pVnode); - // Insert into Entry Table - code = metaEntryTableInsert(pMeta, pEntry); - if (TSDB_CODE_SUCCESS != code) { - metaErr(vgId, code); - return code; - } + SMetaTableOp ops[] = { + {META_ENTRY_TABLE, META_TABLE_OP_INSERT}, // + {META_SCHEMA_TABLE, META_TABLE_OP_INSERT}, // + {META_UID_IDX, META_TABLE_OP_INSERT}, // + {META_NAME_IDX, META_TABLE_OP_INSERT}, // + {META_SUID_IDX, META_TABLE_OP_INSERT}, // + }; - // Insert into Schema Table - code = metaSchemaTableInsert(pMeta, pEntry); - if (TSDB_CODE_SUCCESS != code) { - metaErr(vgId, code); - return code; - } + for (int i = 0; i < sizeof(ops) / sizeof(ops[0]); i++) { + SMetaTableOp *op = &ops[i]; - // Insert to Uid Index - code = metaUidIdxUpsert(pMeta, pEntry); - if (TSDB_CODE_SUCCESS != code) { - metaErr(vgId, code); - return code; - } - - // Insert to Name Index - code = metaNameIdxInsert(pMeta, pEntry); - if (TSDB_CODE_SUCCESS != code) { - metaErr(vgId, code); - return code; - } - - // Insert to Suid Index - code = metaSUidIdxInsert(pMeta, pEntry); - if (TSDB_CODE_SUCCESS != code) { - metaErr(vgId, code); - return code; + code = metaTableOpFn[op->table][op->op](pMeta, pEntry); + if (TSDB_CODE_SUCCESS != code) { + metaErr(TD_VID(pMeta->pVnode), code); + return code; + } } return code; } - static int32_t metaHandleSuperTableCreate(SMeta *pMeta, const SMetaEntry *pEntry) { int32_t code = TSDB_CODE_SUCCESS; - int32_t vgId = TD_VID(pMeta->pVnode); metaWLock(pMeta); code = metaHandleSuperTableCreateImpl(pMeta, pEntry); @@ -243,14 +372,52 @@ static int32_t metaHandleSuperTableCreate(SMeta *pMeta, const SMetaEntry *pEntry pMeta->pVnode->config.vndStats.numOfSTables++; pMeta->changed = true; - metaInfo("vgId:%d, %s success, version:%" PRId64 " type:%d uid:%" PRId64 " name:%s", vgId, __func__, - pEntry->version, pEntry->type, pEntry->uid, pEntry->name); + metaInfo("vgId:%d, %s success, version:%" PRId64 " type:%d uid:%" PRId64 " name:%s", TD_VID(pMeta->pVnode), + __func__, pEntry->version, pEntry->type, pEntry->uid, pEntry->name); } else { - metaErr(vgId, code); + metaErr(TD_VID(pMeta->pVnode), code); } return code; } +static int32_t metaHandleNormalTableCreateImpl(SMeta *pMeta, const SMetaEntry *pEntry) { + int32_t code = TSDB_CODE_SUCCESS; + + SMetaTableOp ops[] = { + {META_ENTRY_TABLE, META_TABLE_OP_INSERT}, // + {META_SCHEMA_TABLE, META_TABLE_OP_INSERT}, // + {META_UID_IDX, META_TABLE_OP_INSERT}, // + {META_NAME_IDX, META_TABLE_OP_INSERT}, // + {META_BTIME_IDX, META_TABLE_OP_INSERT}, // + {META_TTL_IDX, META_TABLE_OP_INSERT}, // + }; + + for (int i = 0; i < sizeof(ops) / sizeof(ops[0]); i++) { + SMetaTableOp *op = &ops[i]; + + code = metaTableOpFn[op->table][op->op](pMeta, pEntry); + if (TSDB_CODE_SUCCESS != code) { + metaErr(TD_VID(pMeta->pVnode), code); + return code; + } + } + + return code; +} +static int32_t metaHandleNormalTableCreate(SMeta *pMeta, const SMetaEntry *pEntry) { + int32_t code = TSDB_CODE_SUCCESS; + + metaWLock(pMeta); + code = metaHandleNormalTableCreateImpl(pMeta, pEntry); + metaULock(pMeta); + + { + // TODO: other stuff + } + + return code; +} + static int32_t metaHandleNormalTableDrop(SMeta *pMeta, const SMetaEntry *pEntry) { int32_t code = TSDB_CODE_SUCCESS; // TODO @@ -287,45 +454,53 @@ int32_t metaHandleEntry2(SMeta *pMeta, const SMetaEntry *pEntry) { } switch (type) { - case TSDB_SUPER_TABLE: + case TSDB_SUPER_TABLE: { if (isExist) { // code = metaHandleSuperTableUpdate(pMeta, pEntry); } else { code = metaHandleSuperTableCreate(pMeta, pEntry); } break; - case TSDB_CHILD_TABLE: + } + case TSDB_CHILD_TABLE: { if (isExist) { // code = metaHandleChildTableUpdate(pMeta, pEntry); } else { // code = metaHandleChildTableCreate(pMeta, pEntry); } break; - case TSDB_NORMAL_TABLE: + } + case TSDB_NORMAL_TABLE: { if (isExist) { // code = metaHandleNormalTableUpdate(pMeta, pEntry); } else { - // code = metaHandleNormalTableCreate(pMeta, pEntry); + code = metaHandleNormalTableCreate(pMeta, pEntry); } break; - default: + } + default: { code = TSDB_CODE_INVALID_PARA; break; + } } } else { switch (type) { - case TSDB_SUPER_TABLE: + case TSDB_SUPER_TABLE: { // code = metaHandleSuperTableDrop(pMeta, pEntry); break; - case TSDB_CHILD_TABLE: + } + case TSDB_CHILD_TABLE: { // code = metaHandleChildTableDrop(pMeta, pEntry); break; - case TSDB_NORMAL_TABLE: + } + case TSDB_NORMAL_TABLE: { // code = metaHandleNormalTableDrop(pMeta, pEntry); break; - default: + } + default: { code = TSDB_CODE_INVALID_PARA; break; + } } } diff --git a/source/dnode/vnode/src/meta/metaTable2.c b/source/dnode/vnode/src/meta/metaTable2.c index 4e0b41fb93..d1a90455ff 100644 --- a/source/dnode/vnode/src/meta/metaTable2.c +++ b/source/dnode/vnode/src/meta/metaTable2.c @@ -131,8 +131,226 @@ int32_t metaCreateSuperTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq // Alter Super Table +// Create Child Table +static int32_t metaCreateChildTable(SMeta *pMeta, int64_t version, SVCreateTbReq *pReq, STableMetaRsp **ppRsp) { + int32_t code = TSDB_CODE_SUCCESS; + // TODO + return code; + +#if 0 + SMetaEntry me = {0}; + SMetaReader mr = {0}; + int32_t ret; + + if (pReq->type == TSDB_CHILD_TABLE) { + tb_uid_t suid = metaGetTableEntryUidByName(pMeta, pReq->ctb.stbName); + if (suid != pReq->ctb.suid) { + return terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST; + } + } + + // validate req + metaReaderDoInit(&mr, pMeta, META_READER_LOCK); + if (metaGetTableEntryByName(&mr, pReq->name) == 0) { + if (pReq->type == TSDB_CHILD_TABLE && pReq->ctb.suid != mr.me.ctbEntry.suid) { + metaReaderClear(&mr); + return terrno = TSDB_CODE_TDB_TABLE_IN_OTHER_STABLE; + } + pReq->uid = mr.me.uid; + if (pReq->type == TSDB_CHILD_TABLE) { + pReq->ctb.suid = mr.me.ctbEntry.suid; + } + metaReaderClear(&mr); + return terrno = TSDB_CODE_TDB_TABLE_ALREADY_EXIST; + } else if (terrno == TSDB_CODE_PAR_TABLE_NOT_EXIST) { + terrno = TSDB_CODE_SUCCESS; + } + metaReaderClear(&mr); + + bool sysTbl = (pReq->type == TSDB_CHILD_TABLE) && metaTbInFilterCache(pMeta, pReq->ctb.stbName, 1); + + if (!sysTbl && ((terrno = grantCheck(TSDB_GRANT_TIMESERIES)) < 0)) goto _err; + + // build SMetaEntry + SVnodeStats *pStats = &pMeta->pVnode->config.vndStats; + me.version = version; + me.type = pReq->type; + me.uid = pReq->uid; + me.name = pReq->name; + me.ctbEntry.btime = pReq->btime; + me.ctbEntry.ttlDays = pReq->ttl; + me.ctbEntry.commentLen = pReq->commentLen; + me.ctbEntry.comment = pReq->comment; + me.ctbEntry.suid = pReq->ctb.suid; + me.ctbEntry.pTags = pReq->ctb.pTag; + + ++pStats->numOfCTables; + + if (!sysTbl) { + int32_t nCols = 0; + ret = metaGetStbStats(pMeta->pVnode, me.ctbEntry.suid, 0, &nCols); + if (ret < 0) { + metaError("vgId:%d, failed to get stb stats:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, + pReq->ctb.suid, tstrerror(ret)); + } + pStats->numOfTimeSeries += nCols - 1; + } + + metaWLock(pMeta); + metaUpdateStbStats(pMeta, me.ctbEntry.suid, 1, 0); + ret = metaUidCacheClear(pMeta, me.ctbEntry.suid); + if (ret < 0) { + metaError("vgId:%d, failed to clear uid cache:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, + pReq->ctb.suid, tstrerror(ret)); + } + ret = metaTbGroupCacheClear(pMeta, me.ctbEntry.suid); + if (ret < 0) { + metaError("vgId:%d, failed to clear group cache:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, + pReq->ctb.suid, tstrerror(ret)); + } + metaULock(pMeta); + + if (!TSDB_CACHE_NO(pMeta->pVnode->config)) { + ret = tsdbCacheNewTable(pMeta->pVnode->pTsdb, me.uid, me.ctbEntry.suid, NULL); + if (ret < 0) { + metaError("vgId:%d, failed to create table:%s since %s", TD_VID(pMeta->pVnode), pReq->name, tstrerror(ret)); + goto _err; + } + } + + if (metaHandleEntry(pMeta, &me) < 0) goto _err; + + metaTimeSeriesNotifyCheck(pMeta); + + if (pMetaRsp) { + *pMetaRsp = taosMemoryCalloc(1, sizeof(STableMetaRsp)); + + if (*pMetaRsp) { + (*pMetaRsp)->tableType = TSDB_CHILD_TABLE; + (*pMetaRsp)->tuid = pReq->uid; + (*pMetaRsp)->suid = pReq->ctb.suid; + strcpy((*pMetaRsp)->tbName, pReq->name); + } + } + + pMeta->changed = true; + metaDebug("vgId:%d, table:%s uid %" PRId64 " is created, type:%" PRId8, TD_VID(pMeta->pVnode), pReq->name, pReq->uid, + pReq->type); + return 0; + +_err: + metaError("vgId:%d, failed to create table:%s type:%s since %s", TD_VID(pMeta->pVnode), pReq->name, + pReq->type == TSDB_CHILD_TABLE ? "child table" : "normal table", tstrerror(terrno)); + return TSDB_CODE_FAILED; +#endif +} + +// Drop Child Table + +// Alter Child Table + // Create Normal Table +static int32_t metaCreateNormalTable(SMeta *pMeta, int64_t version, SVCreateTbReq *pReq, STableMetaRsp **ppRsp) { + int32_t code = TSDB_CODE_SUCCESS; + +#if 0 + SMetaEntry me = {0}; + SMetaReader mr = {0}; + int32_t ret; + + // validate req + metaReaderDoInit(&mr, pMeta, META_READER_LOCK); + if (metaGetTableEntryByName(&mr, pReq->name) == 0) { + if (pReq->type == TSDB_CHILD_TABLE && pReq->ctb.suid != mr.me.ctbEntry.suid) { + metaReaderClear(&mr); + return terrno = TSDB_CODE_TDB_TABLE_IN_OTHER_STABLE; + } + pReq->uid = mr.me.uid; + if (pReq->type == TSDB_CHILD_TABLE) { + pReq->ctb.suid = mr.me.ctbEntry.suid; + } + metaReaderClear(&mr); + return terrno = TSDB_CODE_TDB_TABLE_ALREADY_EXIST; + } else if (terrno == TSDB_CODE_PAR_TABLE_NOT_EXIST) { + terrno = TSDB_CODE_SUCCESS; + } + metaReaderClear(&mr); + + bool sysTbl = (pReq->type == TSDB_CHILD_TABLE) && metaTbInFilterCache(pMeta, pReq->ctb.stbName, 1); + + if (!sysTbl && ((terrno = grantCheck(TSDB_GRANT_TIMESERIES)) < 0)) goto _err; + + // build SMetaEntry + SVnodeStats *pStats = &pMeta->pVnode->config.vndStats; + me.version = version; + me.type = pReq->type; + me.uid = pReq->uid; + me.name = pReq->name; + me.ntbEntry.btime = pReq->btime; + me.ntbEntry.ttlDays = pReq->ttl; + me.ntbEntry.commentLen = pReq->commentLen; + me.ntbEntry.comment = pReq->comment; + me.ntbEntry.schemaRow = pReq->ntb.schemaRow; + me.ntbEntry.ncid = me.ntbEntry.schemaRow.pSchema[me.ntbEntry.schemaRow.nCols - 1].colId + 1; + me.colCmpr = pReq->colCmpr; + TABLE_SET_COL_COMPRESSED(me.flags); + + ++pStats->numOfNTables; + pStats->numOfNTimeSeries += me.ntbEntry.schemaRow.nCols - 1; + + if (!TSDB_CACHE_NO(pMeta->pVnode->config)) { + ret = tsdbCacheNewTable(pMeta->pVnode->pTsdb, me.uid, -1, &me.ntbEntry.schemaRow); + if (ret < 0) { + metaError("vgId:%d, failed to create table:%s since %s", TD_VID(pMeta->pVnode), pReq->name, tstrerror(ret)); + goto _err; + } + } + + if (metaHandleEntry(pMeta, &me) < 0) goto _err; + + metaTimeSeriesNotifyCheck(pMeta); + + if (pMetaRsp) { + *pMetaRsp = taosMemoryCalloc(1, sizeof(STableMetaRsp)); + + if (*pMetaRsp) { + ret = metaUpdateMetaRsp(pReq->uid, pReq->name, &pReq->ntb.schemaRow, *pMetaRsp); + if (ret < 0) { + metaError("vgId:%d, failed to update meta rsp:%s since %s", TD_VID(pMeta->pVnode), pReq->name, tstrerror(ret)); + } + for (int32_t i = 0; i < pReq->colCmpr.nCols; i++) { + SColCmpr *p = &pReq->colCmpr.pColCmpr[i]; + (*pMetaRsp)->pSchemaExt[i].colId = p->id; + (*pMetaRsp)->pSchemaExt[i].compress = p->alg; + } + } + } + + pMeta->changed = true; + metaDebug("vgId:%d, table:%s uid %" PRId64 " is created, type:%" PRId8, TD_VID(pMeta->pVnode), pReq->name, pReq->uid, + pReq->type); + return 0; + +_err: + metaError("vgId:%d, failed to create table:%s type:%s since %s", TD_VID(pMeta->pVnode), pReq->name, + pReq->type == TSDB_CHILD_TABLE ? "child table" : "normal table", tstrerror(terrno)); + return TSDB_CODE_FAILED; +#endif + return code; +} // Drop Normal Table // Alter Normal Table + +int32_t metaCreateTable2(SMeta *pMeta, int64_t version, SVCreateTbReq *pReq, STableMetaRsp **ppRsp) { + int32_t code = TSDB_CODE_SUCCESS; + if (TSDB_CHILD_TABLE == pReq->type) { + code = metaCreateChildTable(pMeta, version, pReq, ppRsp); + } else if (TSDB_NORMAL_TABLE == pReq->type) { + code = metaCreateNormalTable(pMeta, version, pReq, ppRsp); + } else { + code = TSDB_CODE_INVALID_MSG; + } + return code; +} From 4804ff63cc2db1d904b347b37bac891c08949f61 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 6 Dec 2024 14:46:38 +0800 Subject: [PATCH 10/45] fix: more code --- source/dnode/vnode/src/meta/metaEntry2.c | 2 +- source/dnode/vnode/src/meta/metaTable.c | 1 + source/dnode/vnode/src/meta/metaTable2.c | 13 ------------- 3 files changed, 2 insertions(+), 14 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaEntry2.c b/source/dnode/vnode/src/meta/metaEntry2.c index 33694c6443..0a4622ff1b 100644 --- a/source/dnode/vnode/src/meta/metaEntry2.c +++ b/source/dnode/vnode/src/meta/metaEntry2.c @@ -343,7 +343,7 @@ static int32_t metaHandleSuperTableCreateImpl(SMeta *pMeta, const SMetaEntry *pE SMetaTableOp ops[] = { {META_ENTRY_TABLE, META_TABLE_OP_INSERT}, // - {META_SCHEMA_TABLE, META_TABLE_OP_INSERT}, // + {META_SCHEMA_TABLE, META_TABLE_OP_UPDATA}, // TODO: here should be insert {META_UID_IDX, META_TABLE_OP_INSERT}, // {META_NAME_IDX, META_TABLE_OP_INSERT}, // {META_SUID_IDX, META_TABLE_OP_INSERT}, // diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 65e520bb4a..f28beac5e5 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -437,6 +437,7 @@ _drop_super_table: tstrerror(terrno)); } + metaCacheDrop(pMeta, pReq->suid); ret = tdbTbDelete(pMeta->pUidIdx, &pReq->suid, sizeof(tb_uid_t), pMeta->txn); if (ret < 0) { metaError("vgId:%d, failed to drop stb:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, pReq->suid, diff --git a/source/dnode/vnode/src/meta/metaTable2.c b/source/dnode/vnode/src/meta/metaTable2.c index d1a90455ff..84e031036b 100644 --- a/source/dnode/vnode/src/meta/metaTable2.c +++ b/source/dnode/vnode/src/meta/metaTable2.c @@ -66,19 +66,6 @@ static int32_t metaCheckCreateSuperTableReq(SMeta *pMeta, int64_t version, SVCre return TSDB_CODE_INVALID_MSG; } - // other fields - if (pReq->schemaRow.nCols <= 0) { - metaError("vgId:%d, %s failed at %s:%d since invalid row schema, version:%" PRId64, vgId, __func__, __FILE__, - __LINE__, version); - return TSDB_CODE_INVALID_MSG; - } - - if (pReq->schemaTag.nCols <= 0) { - metaError("vgId:%d, %s failed at %s:%d since invalid tag schema, version:%" PRId64, vgId, __func__, __FILE__, - __LINE__, version); - return TSDB_CODE_INVALID_MSG; - } - return TSDB_CODE_SUCCESS; } From 76d36ab9fb781cfc820804b6de146f792c6561c2 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 6 Dec 2024 16:20:53 +0800 Subject: [PATCH 11/45] more code --- source/dnode/vnode/src/inc/vnodeInt.h | 3 +- source/dnode/vnode/src/meta/metaEntry2.c | 20 +- source/dnode/vnode/src/meta/metaTable.c | 9 +- source/dnode/vnode/src/meta/metaTable2.c | 311 ++++++++++++++--------- source/dnode/vnode/src/tsdb/tsdbCache.c | 2 +- source/dnode/vnode/src/vnd/vnodeSvr.c | 4 +- 6 files changed, 222 insertions(+), 127 deletions(-) diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 43164da8a7..f7135a0977 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -160,6 +160,7 @@ int metaCreateSuperTable(SMeta* pMeta, int64_t version, SVCreateStbR int metaAlterSTable(SMeta* pMeta, int64_t version, SVCreateStbReq* pReq); int metaDropSTable(SMeta* pMeta, int64_t verison, SVDropStbReq* pReq, SArray* tbUidList); int metaCreateTable(SMeta* pMeta, int64_t version, SVCreateTbReq* pReq, STableMetaRsp** pMetaRsp); +int32_t metaCreateTable2(SMeta* pMeta, int64_t version, SVCreateTbReq* pReq, STableMetaRsp** ppRsp); int metaDropTable(SMeta* pMeta, int64_t version, SVDropTbReq* pReq, SArray* tbUids, int64_t* tbUid); int32_t metaTrimTables(SMeta* pMeta); int32_t metaDropTables(SMeta* pMeta, SArray* tbUids); @@ -218,7 +219,7 @@ int32_t tsdbBegin(STsdb* pTsdb); // int32_t tsdbPrepareCommit(STsdb* pTsdb); // int32_t tsdbCommit(STsdb* pTsdb, SCommitInfo* pInfo); int32_t tsdbCacheCommit(STsdb* pTsdb); -int32_t tsdbCacheNewTable(STsdb* pTsdb, int64_t uid, tb_uid_t suid, SSchemaWrapper* pSchemaRow); +int32_t tsdbCacheNewTable(STsdb* pTsdb, int64_t uid, tb_uid_t suid, const SSchemaWrapper* pSchemaRow); int32_t tsdbCacheDropTable(STsdb* pTsdb, int64_t uid, tb_uid_t suid, SSchemaWrapper* pSchemaRow); int32_t tsdbCacheDropSubTables(STsdb* pTsdb, SArray* uids, tb_uid_t suid); int32_t tsdbCacheNewSTableColumn(STsdb* pTsdb, SArray* uids, int16_t cid, int8_t col_type); diff --git a/source/dnode/vnode/src/meta/metaEntry2.c b/source/dnode/vnode/src/meta/metaEntry2.c index 0a4622ff1b..a81f263511 100644 --- a/source/dnode/vnode/src/meta/metaEntry2.c +++ b/source/dnode/vnode/src/meta/metaEntry2.c @@ -385,11 +385,12 @@ static int32_t metaHandleNormalTableCreateImpl(SMeta *pMeta, const SMetaEntry *p SMetaTableOp ops[] = { {META_ENTRY_TABLE, META_TABLE_OP_INSERT}, // - {META_SCHEMA_TABLE, META_TABLE_OP_INSERT}, // + {META_SCHEMA_TABLE, META_TABLE_OP_UPDATA}, // TODO: need to be insert {META_UID_IDX, META_TABLE_OP_INSERT}, // {META_NAME_IDX, META_TABLE_OP_INSERT}, // {META_BTIME_IDX, META_TABLE_OP_INSERT}, // {META_TTL_IDX, META_TABLE_OP_INSERT}, // + // TODO: need ncol idx }; for (int i = 0; i < sizeof(ops) / sizeof(ops[0]); i++) { @@ -407,14 +408,25 @@ static int32_t metaHandleNormalTableCreateImpl(SMeta *pMeta, const SMetaEntry *p static int32_t metaHandleNormalTableCreate(SMeta *pMeta, const SMetaEntry *pEntry) { int32_t code = TSDB_CODE_SUCCESS; + // update TDB metaWLock(pMeta); code = metaHandleNormalTableCreateImpl(pMeta, pEntry); metaULock(pMeta); - { - // TODO: other stuff + // update other stuff + if (TSDB_CODE_SUCCESS != code) { + pMeta->pVnode->config.vndStats.numOfNTables++; + pMeta->pVnode->config.vndStats.numOfNTimeSeries += pEntry->ntbEntry.schemaRow.nCols - 1; + pMeta->changed = true; + if (!TSDB_CACHE_NO(pMeta->pVnode->config)) { + int32_t rc = tsdbCacheNewTable(pMeta->pVnode->pTsdb, pEntry->uid, -1, &pEntry->ntbEntry.schemaRow); + if (rc < 0) { + metaError("vgId:%d, failed to create table:%s since %s", TD_VID(pMeta->pVnode), pEntry->name, tstrerror(rc)); + } + } + } else { + metaErr(TD_VID(pMeta->pVnode), code); } - return code; } diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index f28beac5e5..7987a7648a 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -87,7 +87,7 @@ static void metaGetEntryInfo(const SMetaEntry *pEntry, SMetaInfo *pInfo) { } } -static int metaUpdateMetaRsp(tb_uid_t uid, char *tbName, SSchemaWrapper *pSchema, STableMetaRsp *pMetaRsp) { +int metaUpdateMetaRsp(tb_uid_t uid, char *tbName, SSchemaWrapper *pSchema, STableMetaRsp *pMetaRsp) { pMetaRsp->pSchemas = taosMemoryMalloc(pSchema->nCols * sizeof(SSchema)); if (NULL == pMetaRsp->pSchemas) { return terrno; @@ -437,7 +437,12 @@ _drop_super_table: tstrerror(terrno)); } - metaCacheDrop(pMeta, pReq->suid); + ret = metaCacheDrop(pMeta, pReq->suid); + if (ret < 0) { + metaError("vgId:%d, failed to drop stb:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, pReq->suid, + tstrerror(terrno)); + } + ret = tdbTbDelete(pMeta->pUidIdx, &pReq->suid, sizeof(tb_uid_t), pMeta->txn); if (ret < 0) { metaError("vgId:%d, failed to drop stb:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, pReq->suid, diff --git a/source/dnode/vnode/src/meta/metaTable2.c b/source/dnode/vnode/src/meta/metaTable2.c index 84e031036b..4760d39d37 100644 --- a/source/dnode/vnode/src/meta/metaTable2.c +++ b/source/dnode/vnode/src/meta/metaTable2.c @@ -16,6 +16,7 @@ #include "meta.h" extern int32_t metaHandleEntry2(SMeta *pMeta, const SMetaEntry *pEntry); +extern int32_t metaUpdateMetaRsp(tb_uid_t uid, char *tbName, SSchemaWrapper *pSchema, STableMetaRsp *pMetaRsp); static int32_t metaCheckCreateSuperTableReq(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { int32_t vgId = TD_VID(pMeta->pVnode); @@ -129,6 +130,12 @@ static int32_t metaCreateChildTable(SMeta *pMeta, int64_t version, SVCreateTbReq SMetaReader mr = {0}; int32_t ret; + // validate message + if (pReq->type != TSDB_CHILD_TABLE && pReq->type != TSDB_NORMAL_TABLE) { + terrno = TSDB_CODE_INVALID_MSG; + goto _err; + } + if (pReq->type == TSDB_CHILD_TABLE) { tb_uid_t suid = metaGetTableEntryUidByName(pMeta, pReq->ctb.stbName); if (suid != pReq->ctb.suid) { @@ -160,48 +167,91 @@ static int32_t metaCreateChildTable(SMeta *pMeta, int64_t version, SVCreateTbReq // build SMetaEntry SVnodeStats *pStats = &pMeta->pVnode->config.vndStats; - me.version = version; + me.version = ver; me.type = pReq->type; me.uid = pReq->uid; me.name = pReq->name; - me.ctbEntry.btime = pReq->btime; - me.ctbEntry.ttlDays = pReq->ttl; - me.ctbEntry.commentLen = pReq->commentLen; - me.ctbEntry.comment = pReq->comment; - me.ctbEntry.suid = pReq->ctb.suid; - me.ctbEntry.pTags = pReq->ctb.pTag; + if (me.type == TSDB_CHILD_TABLE) { + me.ctbEntry.btime = pReq->btime; + me.ctbEntry.ttlDays = pReq->ttl; + me.ctbEntry.commentLen = pReq->commentLen; + me.ctbEntry.comment = pReq->comment; + me.ctbEntry.suid = pReq->ctb.suid; + me.ctbEntry.pTags = pReq->ctb.pTag; - ++pStats->numOfCTables; +#ifdef TAG_FILTER_DEBUG + SArray *pTagVals = NULL; + int32_t code = tTagToValArray((STag *)pReq->ctb.pTag, &pTagVals); + for (int i = 0; i < taosArrayGetSize(pTagVals); i++) { + STagVal *pTagVal = (STagVal *)taosArrayGet(pTagVals, i); - if (!sysTbl) { - int32_t nCols = 0; - ret = metaGetStbStats(pMeta->pVnode, me.ctbEntry.suid, 0, &nCols); + if (IS_VAR_DATA_TYPE(pTagVal->type)) { + char *buf = taosMemoryCalloc(pTagVal->nData + 1, 1); + memcpy(buf, pTagVal->pData, pTagVal->nData); + metaDebug("metaTag table:%s varchar index:%d cid:%d type:%d value:%s", pReq->name, i, pTagVal->cid, + pTagVal->type, buf); + taosMemoryFree(buf); + } else { + double val = 0; + GET_TYPED_DATA(val, double, pTagVal->type, &pTagVal->i64); + metaDebug("metaTag table:%s number index:%d cid:%d type:%d value:%f", pReq->name, i, pTagVal->cid, + pTagVal->type, val); + } + } +#endif + + ++pStats->numOfCTables; + + if (!sysTbl) { + int32_t nCols = 0; + ret = metaGetStbStats(pMeta->pVnode, me.ctbEntry.suid, 0, &nCols); + if (ret < 0) { + metaError("vgId:%d, failed to get stb stats:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, + pReq->ctb.suid, tstrerror(ret)); + } + pStats->numOfTimeSeries += nCols - 1; + } + + metaWLock(pMeta); + metaUpdateStbStats(pMeta, me.ctbEntry.suid, 1, 0); + ret = metaUidCacheClear(pMeta, me.ctbEntry.suid); if (ret < 0) { - metaError("vgId:%d, failed to get stb stats:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, + metaError("vgId:%d, failed to clear uid cache:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, pReq->ctb.suid, tstrerror(ret)); } - pStats->numOfTimeSeries += nCols - 1; - } - - metaWLock(pMeta); - metaUpdateStbStats(pMeta, me.ctbEntry.suid, 1, 0); - ret = metaUidCacheClear(pMeta, me.ctbEntry.suid); - if (ret < 0) { - metaError("vgId:%d, failed to clear uid cache:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, - pReq->ctb.suid, tstrerror(ret)); - } - ret = metaTbGroupCacheClear(pMeta, me.ctbEntry.suid); - if (ret < 0) { - metaError("vgId:%d, failed to clear group cache:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, - pReq->ctb.suid, tstrerror(ret)); - } - metaULock(pMeta); - - if (!TSDB_CACHE_NO(pMeta->pVnode->config)) { - ret = tsdbCacheNewTable(pMeta->pVnode->pTsdb, me.uid, me.ctbEntry.suid, NULL); + ret = metaTbGroupCacheClear(pMeta, me.ctbEntry.suid); if (ret < 0) { - metaError("vgId:%d, failed to create table:%s since %s", TD_VID(pMeta->pVnode), pReq->name, tstrerror(ret)); - goto _err; + metaError("vgId:%d, failed to clear group cache:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, + pReq->ctb.suid, tstrerror(ret)); + } + metaULock(pMeta); + + if (!TSDB_CACHE_NO(pMeta->pVnode->config)) { + ret = tsdbCacheNewTable(pMeta->pVnode->pTsdb, me.uid, me.ctbEntry.suid, NULL); + if (ret < 0) { + metaError("vgId:%d, failed to create table:%s since %s", TD_VID(pMeta->pVnode), pReq->name, tstrerror(ret)); + goto _err; + } + } + } else { + me.ntbEntry.btime = pReq->btime; + me.ntbEntry.ttlDays = pReq->ttl; + me.ntbEntry.commentLen = pReq->commentLen; + me.ntbEntry.comment = pReq->comment; + me.ntbEntry.schemaRow = pReq->ntb.schemaRow; + me.ntbEntry.ncid = me.ntbEntry.schemaRow.pSchema[me.ntbEntry.schemaRow.nCols - 1].colId + 1; + me.colCmpr = pReq->colCmpr; + TABLE_SET_COL_COMPRESSED(me.flags); + + ++pStats->numOfNTables; + pStats->numOfNTimeSeries += me.ntbEntry.schemaRow.nCols - 1; + + if (!TSDB_CACHE_NO(pMeta->pVnode->config)) { + ret = tsdbCacheNewTable(pMeta->pVnode->pTsdb, me.uid, -1, &me.ntbEntry.schemaRow); + if (ret < 0) { + metaError("vgId:%d, failed to create table:%s since %s", TD_VID(pMeta->pVnode), pReq->name, tstrerror(ret)); + goto _err; + } } } @@ -213,10 +263,23 @@ static int32_t metaCreateChildTable(SMeta *pMeta, int64_t version, SVCreateTbReq *pMetaRsp = taosMemoryCalloc(1, sizeof(STableMetaRsp)); if (*pMetaRsp) { - (*pMetaRsp)->tableType = TSDB_CHILD_TABLE; - (*pMetaRsp)->tuid = pReq->uid; - (*pMetaRsp)->suid = pReq->ctb.suid; - strcpy((*pMetaRsp)->tbName, pReq->name); + if (me.type == TSDB_CHILD_TABLE) { + (*pMetaRsp)->tableType = TSDB_CHILD_TABLE; + (*pMetaRsp)->tuid = pReq->uid; + (*pMetaRsp)->suid = pReq->ctb.suid; + strcpy((*pMetaRsp)->tbName, pReq->name); + } else { + ret = metaUpdateMetaRsp(pReq->uid, pReq->name, &pReq->ntb.schemaRow, *pMetaRsp); + if (ret < 0) { + metaError("vgId:%d, failed to update meta rsp:%s since %s", TD_VID(pMeta->pVnode), pReq->name, + tstrerror(ret)); + } + for (int32_t i = 0; i < pReq->colCmpr.nCols; i++) { + SColCmpr *p = &pReq->colCmpr.pColCmpr[i]; + (*pMetaRsp)->pSchemaExt[i].colId = p->id; + (*pMetaRsp)->pSchemaExt[i].compress = p->alg; + } + } } } @@ -237,93 +300,106 @@ _err: // Alter Child Table // Create Normal Table +static int32_t metaCheckCreateNormalTableReq(SMeta *pMeta, int64_t version, SVCreateTbReq *pReq) { + int32_t code = 0; + void *value = NULL; + int32_t valueSize = 0; + + if (NULL == pReq->name || strlen(pReq->name) == 0) { + metaError("vgId:%d, %s failed at %s:%d since invalid name:%s, version:%" PRId64, TD_VID(pMeta->pVnode), __func__, + __FILE__, __LINE__, pReq->name, version); + return TSDB_CODE_INVALID_MSG; + } + + // check name + if (tdbTbGet(pMeta->pNameIdx, pReq->name, strlen(pReq->name) + 1, &value, &valueSize) == 0) { + // for auto create table, we return the uid of the existing table + pReq->uid = *(tb_uid_t *)value; + tdbFree(value); + return TSDB_CODE_TDB_TABLE_ALREADY_EXIST; + } + + // grant check + code = grantCheck(TSDB_GRANT_TIMESERIES); + if (code) { + metaError("vgId:%d, %s failed at %s:%d since %s, version:%" PRId64 " name:%s", TD_VID(pMeta->pVnode), __func__, + __FILE__, __LINE__, tstrerror(code), version, pReq->name); + } + return code; +} + +static int32_t metaBuildCreateNormalTableRsp(SMeta *pMeta, SMetaEntry *pEntry, STableMetaRsp **ppRsp) { + int32_t code = TSDB_CODE_SUCCESS; + + if (NULL == ppRsp) { + return code; + } + + *ppRsp = taosMemoryCalloc(1, sizeof(STableMetaRsp)); + if (NULL == *ppRsp) { + return terrno; + } + + code = metaUpdateMetaRsp(pEntry->uid, pEntry->name, &pEntry->ntbEntry.schemaRow, *ppRsp); + if (code) { + taosMemoryFreeClear(*ppRsp); + return code; + } + + for (int32_t i = 0; i < pEntry->colCmpr.nCols; i++) { + SColCmpr *p = &pEntry->colCmpr.pColCmpr[i]; + (*ppRsp)->pSchemaExt[i].colId = p->id; + (*ppRsp)->pSchemaExt[i].compress = p->alg; + } + + return code; +} + static int32_t metaCreateNormalTable(SMeta *pMeta, int64_t version, SVCreateTbReq *pReq, STableMetaRsp **ppRsp) { int32_t code = TSDB_CODE_SUCCESS; -#if 0 - SMetaEntry me = {0}; - SMetaReader mr = {0}; - int32_t ret; - - // validate req - metaReaderDoInit(&mr, pMeta, META_READER_LOCK); - if (metaGetTableEntryByName(&mr, pReq->name) == 0) { - if (pReq->type == TSDB_CHILD_TABLE && pReq->ctb.suid != mr.me.ctbEntry.suid) { - metaReaderClear(&mr); - return terrno = TSDB_CODE_TDB_TABLE_IN_OTHER_STABLE; - } - pReq->uid = mr.me.uid; - if (pReq->type == TSDB_CHILD_TABLE) { - pReq->ctb.suid = mr.me.ctbEntry.suid; - } - metaReaderClear(&mr); - return terrno = TSDB_CODE_TDB_TABLE_ALREADY_EXIST; - } else if (terrno == TSDB_CODE_PAR_TABLE_NOT_EXIST) { - terrno = TSDB_CODE_SUCCESS; - } - metaReaderClear(&mr); - - bool sysTbl = (pReq->type == TSDB_CHILD_TABLE) && metaTbInFilterCache(pMeta, pReq->ctb.stbName, 1); - - if (!sysTbl && ((terrno = grantCheck(TSDB_GRANT_TIMESERIES)) < 0)) goto _err; - - // build SMetaEntry - SVnodeStats *pStats = &pMeta->pVnode->config.vndStats; - me.version = version; - me.type = pReq->type; - me.uid = pReq->uid; - me.name = pReq->name; - me.ntbEntry.btime = pReq->btime; - me.ntbEntry.ttlDays = pReq->ttl; - me.ntbEntry.commentLen = pReq->commentLen; - me.ntbEntry.comment = pReq->comment; - me.ntbEntry.schemaRow = pReq->ntb.schemaRow; - me.ntbEntry.ncid = me.ntbEntry.schemaRow.pSchema[me.ntbEntry.schemaRow.nCols - 1].colId + 1; - me.colCmpr = pReq->colCmpr; - TABLE_SET_COL_COMPRESSED(me.flags); - - ++pStats->numOfNTables; - pStats->numOfNTimeSeries += me.ntbEntry.schemaRow.nCols - 1; - - if (!TSDB_CACHE_NO(pMeta->pVnode->config)) { - ret = tsdbCacheNewTable(pMeta->pVnode->pTsdb, me.uid, -1, &me.ntbEntry.schemaRow); - if (ret < 0) { - metaError("vgId:%d, failed to create table:%s since %s", TD_VID(pMeta->pVnode), pReq->name, tstrerror(ret)); - goto _err; + // check request + code = metaCheckCreateNormalTableReq(pMeta, version, pReq); + if (code) { + if (TSDB_CODE_TDB_TABLE_ALREADY_EXIST != code) { + metaError("vgId:%d, %s failed at %s:%d since %s, version:%" PRId64 " name:%s", TD_VID(pMeta->pVnode), __func__, + __FILE__, __LINE__, tstrerror(code), version, pReq->name); } + TAOS_RETURN(code); } - if (metaHandleEntry(pMeta, &me) < 0) goto _err; + // handle entry + SMetaEntry entry = { + .version = version, + .type = TSDB_NORMAL_TABLE, + .uid = pReq->uid, + .name = pReq->name, + .ntbEntry.btime = pReq->btime, + .ntbEntry.ttlDays = pReq->ttl, + .ntbEntry.commentLen = pReq->commentLen, + .ntbEntry.comment = pReq->comment, + .ntbEntry.schemaRow = pReq->ntb.schemaRow, + .ntbEntry.ncid = pReq->ntb.schemaRow.pSchema[pReq->ntb.schemaRow.nCols - 1].colId + 1, + .colCmpr = pReq->colCmpr, + }; + TABLE_SET_COL_COMPRESSED(entry.flags); - metaTimeSeriesNotifyCheck(pMeta); - - if (pMetaRsp) { - *pMetaRsp = taosMemoryCalloc(1, sizeof(STableMetaRsp)); - - if (*pMetaRsp) { - ret = metaUpdateMetaRsp(pReq->uid, pReq->name, &pReq->ntb.schemaRow, *pMetaRsp); - if (ret < 0) { - metaError("vgId:%d, failed to update meta rsp:%s since %s", TD_VID(pMeta->pVnode), pReq->name, tstrerror(ret)); - } - for (int32_t i = 0; i < pReq->colCmpr.nCols; i++) { - SColCmpr *p = &pReq->colCmpr.pColCmpr[i]; - (*pMetaRsp)->pSchemaExt[i].colId = p->id; - (*pMetaRsp)->pSchemaExt[i].compress = p->alg; - } - } + // build response + code = metaBuildCreateNormalTableRsp(pMeta, &entry, ppRsp); + if (code) { + metaError("vgId:%d, %s failed at %s:%d since %s", TD_VID(pMeta->pVnode), __func__, __FILE__, __LINE__, + tstrerror(code)); } - pMeta->changed = true; - metaDebug("vgId:%d, table:%s uid %" PRId64 " is created, type:%" PRId8, TD_VID(pMeta->pVnode), pReq->name, pReq->uid, - pReq->type); - return 0; - -_err: - metaError("vgId:%d, failed to create table:%s type:%s since %s", TD_VID(pMeta->pVnode), pReq->name, - pReq->type == TSDB_CHILD_TABLE ? "child table" : "normal table", tstrerror(terrno)); - return TSDB_CODE_FAILED; -#endif - return code; + code = metaHandleEntry2(pMeta, &entry); + if (TSDB_CODE_SUCCESS == code) { + metaInfo("vgId:%d, normal table:%s uid %" PRId64 " is created, version:%" PRId64, TD_VID(pMeta->pVnode), pReq->name, + pReq->uid, version); + } else { + metaError("vgId:%d, %s failed at %s:%d since %s, uid:%" PRId64 " name:%s version:%" PRId64, TD_VID(pMeta->pVnode), + __func__, __FILE__, __LINE__, tstrerror(code), pReq->uid, pReq->name, version); + } + TAOS_RETURN(code); } // Drop Normal Table @@ -333,11 +409,12 @@ _err: int32_t metaCreateTable2(SMeta *pMeta, int64_t version, SVCreateTbReq *pReq, STableMetaRsp **ppRsp) { int32_t code = TSDB_CODE_SUCCESS; if (TSDB_CHILD_TABLE == pReq->type) { - code = metaCreateChildTable(pMeta, version, pReq, ppRsp); + // TODO + code = metaCreateTable(pMeta, version, pReq, ppRsp); } else if (TSDB_NORMAL_TABLE == pReq->type) { code = metaCreateNormalTable(pMeta, version, pReq, ppRsp); } else { code = TSDB_CODE_INVALID_MSG; } - return code; + TAOS_RETURN(code); } diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 2cef541cdb..a46da532e5 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -776,7 +776,7 @@ _exit: TAOS_RETURN(code); } -int32_t tsdbCacheNewTable(STsdb *pTsdb, tb_uid_t uid, tb_uid_t suid, SSchemaWrapper *pSchemaRow) { +int32_t tsdbCacheNewTable(STsdb *pTsdb, tb_uid_t uid, tb_uid_t suid, const SSchemaWrapper *pSchemaRow) { int32_t code = 0; (void)taosThreadMutexLock(&pTsdb->lruMutex); diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index b7ec3c0b6f..30b0d04ce9 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -1233,7 +1233,7 @@ static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t ver, void *pReq, } // do create table - if (metaCreateTable(pVnode->pMeta, ver, pCreateReq, &cRsp.pMeta) < 0) { + if (metaCreateTable2(pVnode->pMeta, ver, pCreateReq, &cRsp.pMeta) < 0) { if (pCreateReq->flags & TD_CREATE_IF_NOT_EXISTS && terrno == TSDB_CODE_TDB_TABLE_ALREADY_EXIST) { cRsp.code = TSDB_CODE_SUCCESS; } else { @@ -1961,7 +1961,7 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t ver, void *pReq, in SVCreateTbRsp *pCreateTbRsp = taosArrayReserve(pSubmitRsp->aCreateTbRsp, 1); // create table - if (metaCreateTable(pVnode->pMeta, ver, pSubmitTbData->pCreateTbReq, &pCreateTbRsp->pMeta) == 0) { + if (metaCreateTable2(pVnode->pMeta, ver, pSubmitTbData->pCreateTbReq, &pCreateTbRsp->pMeta) == 0) { // create table success if (newTbUids == NULL && From 9bfdbca1106260bb0e614e33ea7ea28fd0b71174 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 6 Dec 2024 18:29:01 +0800 Subject: [PATCH 12/45] fix: CI problem --- source/dnode/vnode/src/meta/metaEntry2.c | 2 +- source/dnode/vnode/src/meta/metaTable2.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaEntry2.c b/source/dnode/vnode/src/meta/metaEntry2.c index a81f263511..cb385d449a 100644 --- a/source/dnode/vnode/src/meta/metaEntry2.c +++ b/source/dnode/vnode/src/meta/metaEntry2.c @@ -297,7 +297,7 @@ static int32_t (*metaTableOpFn[META_TABLE_MAX][META_TABLE_OP_MAX])(SMeta *pMeta, [META_SCHEMA_TABLE] = { [META_TABLE_OP_INSERT] = metaSchemaTableInsert, - [META_TABLE_OP_UPDATA] = NULL, + [META_TABLE_OP_UPDATA] = metaSchemaTableUpdate, [META_TABLE_OP_DELETE] = NULL, }, [META_UID_IDX] = diff --git a/source/dnode/vnode/src/meta/metaTable2.c b/source/dnode/vnode/src/meta/metaTable2.c index 4760d39d37..b0e4923d22 100644 --- a/source/dnode/vnode/src/meta/metaTable2.c +++ b/source/dnode/vnode/src/meta/metaTable2.c @@ -412,7 +412,7 @@ int32_t metaCreateTable2(SMeta *pMeta, int64_t version, SVCreateTbReq *pReq, STa // TODO code = metaCreateTable(pMeta, version, pReq, ppRsp); } else if (TSDB_NORMAL_TABLE == pReq->type) { - code = metaCreateNormalTable(pMeta, version, pReq, ppRsp); + code = metaCreateTable(pMeta, version, pReq, ppRsp); } else { code = TSDB_CODE_INVALID_MSG; } From 5c8ce60dd02e38b5f3624078638a88173c6eb222 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 8 Dec 2024 16:54:30 +0800 Subject: [PATCH 13/45] more code --- include/util/tutil.h | 4 +- source/dnode/vnode/src/meta/metaEntry2.c | 402 +++++++++++++++++++---- source/dnode/vnode/src/meta/metaTable2.c | 393 ++++++++++++---------- source/libs/tdb/inc/tdb.h | 5 + 4 files changed, 574 insertions(+), 230 deletions(-) diff --git a/include/util/tutil.h b/include/util/tutil.h index c1b6b2505b..07c47009c6 100644 --- a/include/util/tutil.h +++ b/include/util/tutil.h @@ -127,10 +127,10 @@ static FORCE_INLINE int32_t taosGetTbHashVal(const char *tbname, int32_t tblen, * only in very hot code paths. Misuse or abuse can lead to performance degradation. */ #if __GNUC__ >= 3 -#define LIKELY(x) __builtin_expect((x) != 0, 1) +#define LIKELY(x) __builtin_expect((x) != 0, 1) #define UNLIKELY(x) __builtin_expect((x) != 0, 0) #else -#define LIKELY(x) ((x) != 0) +#define LIKELY(x) ((x) != 0) #define UNLIKELY(x) ((x) != 0) #endif diff --git a/source/dnode/vnode/src/meta/metaEntry2.c b/source/dnode/vnode/src/meta/metaEntry2.c index cb385d449a..77effc7e13 100644 --- a/source/dnode/vnode/src/meta/metaEntry2.c +++ b/source/dnode/vnode/src/meta/metaEntry2.c @@ -23,6 +23,7 @@ typedef enum { META_UID_IDX, META_NAME_IDX, META_SUID_IDX, + META_CHILD_IDX, META_TAG_IDX, META_BTIME_IDX, META_TTL_IDX, @@ -101,7 +102,8 @@ static int32_t metaEntryTableUpdate(SMeta *pMeta, const SMetaEntry *pEntry) { static int32_t metaEntryTableDelete(SMeta *pMeta, const SMetaEntry *pEntry) { int32_t code = TSDB_CODE_SUCCESS; - int32_t vgId = TD_VID(pMeta->pVnode); + + // SMetaEntry entry { .version = ; }; // TODO @@ -175,6 +177,11 @@ static int32_t metaSchemaTableUpdate(SMeta *pMeta, const SMetaEntry *pEntry) { return metaSchemaTableUpsert(pMeta, pEntry, META_TABLE_OP_UPDATA); } +static int32_t metaSchemaTableDelete(SMeta *pMeta, const SMetaEntry *pEntry) { + // TODO + return TSDB_CODE_SUCCESS; +} + // Uid Index static void metaBuildEntryInfo(const SMetaEntry *pEntry, SMetaInfo *pInfo) { pInfo->uid = pEntry->uid; @@ -225,6 +232,20 @@ static int32_t metaUidIdxUpdate(SMeta *pMeta, const SMetaEntry *pEntry) { return metaUidIdxUpsert(pMeta, pEntry, META_TABLE_OP_UPDATA); } +static int32_t metaUidIdxDelete(SMeta *pMeta, const SMetaEntry *pEntry) { + int32_t code = 0; + + // delete tdb + code = tdbTbDelete(pMeta->pUidIdx, &pEntry->uid, sizeof(pEntry->uid), pMeta->txn); + if (code) { + metaErr(TD_VID(pMeta->pVnode), code); + } + + // delete cache + (void)metaCacheDrop(pMeta, pEntry->uid); + return code; +} + // Name Index static int32_t metaNameIdxUpsert(SMeta *pMeta, const SMetaEntry *pEntry, EMetaTableOp op) { int32_t code = TSDB_CODE_SUCCESS; @@ -241,12 +262,30 @@ static int32_t metaNameIdxUpsert(SMeta *pMeta, const SMetaEntry *pEntry, EMetaTa } static int32_t metaNameIdxInsert(SMeta *pMeta, const SMetaEntry *pEntry) { - return metaNameIdxUpsert(pMeta, pEntry, META_TABLE_OP_INSERT); + int32_t code = TSDB_CODE_SUCCESS; + code = metaNameIdxUpsert(pMeta, pEntry, META_TABLE_OP_INSERT); + if (code) { + metaErr(TD_VID(pMeta->pVnode), code); + } + return code; +} + +static int32_t metaNameIdxUpdate(SMeta *pMeta, const SMetaEntry *pEntry) { + int32_t code = TSDB_CODE_SUCCESS; + code = metaNameIdxUpsert(pMeta, pEntry, META_TABLE_OP_UPDATA); + if (code) { + metaErr(TD_VID(pMeta->pVnode), code); + } + return code; } static int32_t metaNameIdxDelete(SMeta *pMeta, const SMetaEntry *pEntry) { - // TODO - return 0; + int32_t code = TSDB_CODE_SUCCESS; + code = tdbTbDelete(pMeta->pNameIdx, pEntry->name, strlen(pEntry->name) + 1, pMeta->txn); + if (code) { + metaErr(TD_VID(pMeta->pVnode), code); + } + return code; } // Suid Index @@ -258,7 +297,66 @@ static int32_t metaSUidIdxInsert(SMeta *pMeta, const SMetaEntry *pEntry) { return code; } +static int32_t metaSUidIdxDelete(SMeta *pMeta, const SMetaEntry *pEntry) { + int32_t code = tdbTbDelete(pMeta->pSuidIdx, &pEntry->uid, sizeof(pEntry->uid), pMeta->txn); + if (code) { + metaErr(TD_VID(pMeta->pVnode), code); + } + return code; +} + +// Child Index +static int32_t metaChildIdxUpsert(SMeta *pMeta, const SMetaEntry *pEntry, EMetaTableOp op) { + int32_t code = TSDB_CODE_SUCCESS; + + SCtbIdxKey key = { + .suid = pEntry->ctbEntry.suid, + .uid = pEntry->uid, + }; + + if (META_TABLE_OP_INSERT == op) { + code = tdbTbInsert(pMeta->pCtbIdx, &key, sizeof(key), pEntry->ctbEntry.pTags, + ((STag *)(pEntry->ctbEntry.pTags))->len, pMeta->txn); + } else if (META_TABLE_OP_UPDATA == op) { + code = tdbTbUpsert(pMeta->pCtbIdx, &key, sizeof(key), pEntry->ctbEntry.pTags, + ((STag *)(pEntry->ctbEntry.pTags))->len, pMeta->txn); + } else { + code = TSDB_CODE_INVALID_PARA; + } + return code; +} + +static int32_t metaChildIdxInsert(SMeta *pMeta, const SMetaEntry *pEntry) { + return metaChildIdxUpsert(pMeta, pEntry, META_TABLE_OP_INSERT); +} + +static int32_t metaChildIdxUpdate(SMeta *pMeta, const SMetaEntry *pEntry) { + return metaChildIdxUpsert(pMeta, pEntry, META_TABLE_OP_UPDATA); +} + +static int32_t metaChildIdxDelete(SMeta *pMeta, const SMetaEntry *pEntry) { + SCtbIdxKey key = { + .suid = pEntry->ctbEntry.suid, + .uid = pEntry->uid, + }; + return tdbTbDelete(pMeta->pCtbIdx, &key, sizeof(key), pMeta->txn); +} + // Tag Index +static int32_t metaTagIdxInsert(SMeta *pMeta, const SMetaEntry *pEntry) { + // TODO + return 0; +} + +static int32_t metaTagIdxUpdate(SMeta *pMeta, const SMetaEntry *pEntry) { + // TODO + return 0; +} + +static int32_t metaTagIdxDelete(SMeta *pMeta, const SMetaEntry *pEntry) { + // TODO + return 0; +} // Btime Index static int32_t metaBtimeIdxUpsert(SMeta *pMeta, const SMetaEntry *pEntry, EMetaTableOp op) { @@ -287,55 +385,101 @@ static int32_t metaBtimeIdxInsert(SMeta *pMeta, const SMetaEntry *pEntry) { return metaBtimeIdxUpsert(pMeta, pEntry, META_TABLE_OP_INSERT); } -static int32_t (*metaTableOpFn[META_TABLE_MAX][META_TABLE_OP_MAX])(SMeta *pMeta, const SMetaEntry *pEntry) = { - [META_ENTRY_TABLE] = - { - [META_TABLE_OP_INSERT] = metaEntryTableInsert, - [META_TABLE_OP_UPDATA] = metaEntryTableUpdate, - [META_TABLE_OP_DELETE] = metaEntryTableDelete, - }, - [META_SCHEMA_TABLE] = - { - [META_TABLE_OP_INSERT] = metaSchemaTableInsert, - [META_TABLE_OP_UPDATA] = metaSchemaTableUpdate, - [META_TABLE_OP_DELETE] = NULL, - }, - [META_UID_IDX] = - { - [META_TABLE_OP_INSERT] = metaUidIdxInsert, - [META_TABLE_OP_UPDATA] = metaUidIdxUpdate, - [META_TABLE_OP_DELETE] = NULL, - }, - [META_NAME_IDX] = - { - [META_TABLE_OP_INSERT] = metaNameIdxInsert, - [META_TABLE_OP_UPDATA] = NULL, - [META_TABLE_OP_DELETE] = NULL, - }, - [META_SUID_IDX] = - { - [META_TABLE_OP_INSERT] = metaSUidIdxInsert, - [META_TABLE_OP_UPDATA] = NULL, - [META_TABLE_OP_DELETE] = NULL, - }, - [META_TAG_IDX] = - { - [META_TABLE_OP_INSERT] = NULL, - [META_TABLE_OP_UPDATA] = NULL, - [META_TABLE_OP_DELETE] = NULL, - }, - [META_BTIME_IDX] = - { - [META_TABLE_OP_INSERT] = metaBtimeIdxInsert, - [META_TABLE_OP_UPDATA] = NULL, - [META_TABLE_OP_DELETE] = NULL, - }, - [META_TTL_IDX] = - { - [META_TABLE_OP_INSERT] = NULL, - [META_TABLE_OP_UPDATA] = NULL, - [META_TABLE_OP_DELETE] = NULL, - }, +static int32_t metaBtimeIdxUpdate(SMeta *pMeta, const SMetaEntry *pEntry) { + return metaBtimeIdxUpsert(pMeta, pEntry, META_TABLE_OP_UPDATA); +} + +static int32_t metaBtimeIdxDelete(SMeta *pMeta, const SMetaEntry *pEntry) { + int32_t code = TSDB_CODE_SUCCESS; + SBtimeIdxKey key = { + .uid = pEntry->uid, + }; + + if (TSDB_CHILD_TABLE == pEntry->type) { + key.btime = pEntry->ctbEntry.btime; + } else if (TSDB_NORMAL_TABLE == pEntry->type) { + key.btime = pEntry->ntbEntry.btime; + } else { + return TSDB_CODE_INVALID_PARA; + } + return tdbTbDelete(pMeta->pBtimeIdx, &key, sizeof(key), pMeta->txn); +} + +// TTL Index +static int32_t metaTtlIdxUpsert(SMeta *pMeta, const SMetaEntry *pEntry, EMetaTableOp op) { + // TODO + return 0; +} + +static int32_t metaTtlIdxInsert(SMeta *pMeta, const SMetaEntry *pEntry) { + return metaTtlIdxUpsert(pMeta, pEntry, META_TABLE_OP_INSERT); +} + +static int32_t metaTtlIdxUpdate(SMeta *pMeta, const SMetaEntry *pEntry) { + return metaTtlIdxUpsert(pMeta, pEntry, META_TABLE_OP_UPDATA); +} + +static int32_t metaTtlIdxDelete(SMeta *pMeta, const SMetaEntry *pEntry) { + // TODO + return 0; +} + +static int32_t (*metaTableOpFn[META_TABLE_MAX][META_TABLE_OP_MAX])(SMeta *pMeta, const SMetaEntry *pEntry) = + { + [META_ENTRY_TABLE] = + { + [META_TABLE_OP_INSERT] = metaEntryTableInsert, + [META_TABLE_OP_UPDATA] = metaEntryTableUpdate, + [META_TABLE_OP_DELETE] = metaEntryTableDelete, + }, + [META_SCHEMA_TABLE] = + { + [META_TABLE_OP_INSERT] = metaSchemaTableInsert, + [META_TABLE_OP_UPDATA] = metaSchemaTableUpdate, + [META_TABLE_OP_DELETE] = metaSchemaTableDelete, + }, + [META_UID_IDX] = + { + [META_TABLE_OP_INSERT] = metaUidIdxInsert, + [META_TABLE_OP_UPDATA] = metaUidIdxUpdate, + [META_TABLE_OP_DELETE] = metaUidIdxDelete, + }, + [META_NAME_IDX] = + { + [META_TABLE_OP_INSERT] = metaNameIdxInsert, + [META_TABLE_OP_UPDATA] = metaNameIdxUpdate, + [META_TABLE_OP_DELETE] = metaNameIdxDelete, + }, + [META_SUID_IDX] = + { + [META_TABLE_OP_INSERT] = metaSUidIdxInsert, + [META_TABLE_OP_UPDATA] = NULL, + [META_TABLE_OP_DELETE] = metaSUidIdxDelete, + }, + [META_CHILD_IDX] = + { + [META_TABLE_OP_INSERT] = metaChildIdxInsert, + [META_TABLE_OP_UPDATA] = metaChildIdxUpdate, + [META_TABLE_OP_DELETE] = metaChildIdxDelete, + }, + [META_TAG_IDX] = + { + [META_TABLE_OP_INSERT] = metaTagIdxInsert, + [META_TABLE_OP_UPDATA] = metaTagIdxUpdate, + [META_TABLE_OP_DELETE] = metaTagIdxDelete, + }, + [META_BTIME_IDX] = + { + [META_TABLE_OP_INSERT] = metaBtimeIdxInsert, + [META_TABLE_OP_UPDATA] = metaBtimeIdxUpdate, + [META_TABLE_OP_DELETE] = metaBtimeIdxDelete, + }, + [META_TTL_IDX] = + { + [META_TABLE_OP_INSERT] = metaTtlIdxInsert, + [META_TABLE_OP_UPDATA] = metaTtlIdxUpdate, + [META_TABLE_OP_DELETE] = metaTtlIdxDelete, + }, }; static int32_t metaHandleSuperTableCreateImpl(SMeta *pMeta, const SMetaEntry *pEntry) { @@ -389,8 +533,8 @@ static int32_t metaHandleNormalTableCreateImpl(SMeta *pMeta, const SMetaEntry *p {META_UID_IDX, META_TABLE_OP_INSERT}, // {META_NAME_IDX, META_TABLE_OP_INSERT}, // {META_BTIME_IDX, META_TABLE_OP_INSERT}, // - {META_TTL_IDX, META_TABLE_OP_INSERT}, // - // TODO: need ncol idx + + // {META_TTL_IDX, META_TABLE_OP_INSERT}, // }; for (int i = 0; i < sizeof(ops) / sizeof(ops[0]); i++) { @@ -418,6 +562,7 @@ static int32_t metaHandleNormalTableCreate(SMeta *pMeta, const SMetaEntry *pEntr pMeta->pVnode->config.vndStats.numOfNTables++; pMeta->pVnode->config.vndStats.numOfNTimeSeries += pEntry->ntbEntry.schemaRow.nCols - 1; pMeta->changed = true; + if (!TSDB_CACHE_NO(pMeta->pVnode->config)) { int32_t rc = tsdbCacheNewTable(pMeta->pVnode->pTsdb, pEntry->uid, -1, &pEntry->ntbEntry.schemaRow); if (rc < 0) { @@ -430,9 +575,148 @@ static int32_t metaHandleNormalTableCreate(SMeta *pMeta, const SMetaEntry *pEntr return code; } -static int32_t metaHandleNormalTableDrop(SMeta *pMeta, const SMetaEntry *pEntry) { +static int32_t metaHandleChildTableCreateImpl(SMeta *pMeta, const SMetaEntry *pEntry) { int32_t code = TSDB_CODE_SUCCESS; - // TODO + + SMetaTableOp ops[] = { + {META_ENTRY_TABLE, META_TABLE_OP_INSERT}, // + {META_UID_IDX, META_TABLE_OP_INSERT}, // + {META_NAME_IDX, META_TABLE_OP_INSERT}, // + {META_CHILD_IDX, META_TABLE_OP_INSERT}, // + {META_TAG_IDX, META_TABLE_OP_INSERT}, // + {META_BTIME_IDX, META_TABLE_OP_INSERT}, // + {META_TTL_IDX, META_TABLE_OP_INSERT}, // + }; + + for (int i = 0; i < sizeof(ops) / sizeof(ops[0]); i++) { + SMetaTableOp *op = &ops[i]; + + code = metaTableOpFn[op->table][op->op](pMeta, pEntry); + if (TSDB_CODE_SUCCESS != code) { + metaErr(TD_VID(pMeta->pVnode), code); + return code; + } + } + + return code; +} + +static int32_t metaHandleChildTableCreate(SMeta *pMeta, const SMetaEntry *pEntry) { + int32_t code = TSDB_CODE_SUCCESS; + + // update TDB + metaWLock(pMeta); + code = metaHandleChildTableCreateImpl(pMeta, pEntry); + + if (TSDB_CODE_SUCCESS == code) { + // metaUpdateStbStats(pMeta, me.ctbEntry.suid, 1, 0); + // ret = metaUidCacheClear(pMeta, me.ctbEntry.suid); + // if (ret < 0) { + // metaError("vgId:%d, failed to clear uid cache:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, + // pReq->ctb.suid, tstrerror(ret)); + // } + // ret = metaTbGroupCacheClear(pMeta, me.ctbEntry.suid); + // if (ret < 0) { + // metaError("vgId:%d, failed to clear group cache:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), + // pReq->name, + // pReq->ctb.suid, tstrerror(ret)); + // } + } + metaULock(pMeta); + + // update other stuff + if (TSDB_CODE_SUCCESS != code) { + pMeta->pVnode->config.vndStats.numOfCTables++; + + // if (!metaTbInFilterCache(pMeta, pReq->ctb.stbName, 1)) { + // int32_t nCols = 0; + // ret = metaGetStbStats(pMeta->pVnode, me.ctbEntry.suid, 0, &nCols); + // if (ret < 0) { + // metaError("vgId:%d, failed to get stb stats:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), + // pReq->name, + // pReq->ctb.suid, tstrerror(ret)); + // } + // pStats->numOfTimeSeries += nCols - 1; + // } + + if (!TSDB_CACHE_NO(pMeta->pVnode->config)) { + int32_t rc = tsdbCacheNewTable(pMeta->pVnode->pTsdb, pEntry->uid, pEntry->ctbEntry.suid, NULL); + if (rc < 0) { + metaError("vgId:%d, %s failed at %s:%d since %s", TD_VID(pMeta->pVnode), __func__, __FILE__, __LINE__, + tstrerror(rc)); + } + } + pMeta->changed = true; + } else { + metaErr(TD_VID(pMeta->pVnode), code); + } + return code; +} + +static int32_t metaHandleNormalTableDropImpl(SMeta *pMeta, const SMetaEntry *pEntry) { + int32_t code = TSDB_CODE_SUCCESS; + + SMetaTableOp ops[] = { + {META_ENTRY_TABLE, META_TABLE_OP_DELETE}, // + {META_UID_IDX, META_TABLE_OP_DELETE}, // + {META_NAME_IDX, META_TABLE_OP_DELETE}, // + {META_BTIME_IDX, META_TABLE_OP_DELETE}, // + + // {META_SCHEMA_TABLE, META_TABLE_OP_UPDATA}, // TODO: need to be insert + // {META_TTL_IDX, META_TABLE_OP_INSERT}, // + }; + + for (int32_t i = 0; i < sizeof(ops) / sizeof(ops[0]); i++) { + SMetaTableOp *op = &ops[i]; + code = metaTableOpFn[op->table][op->op](pMeta, pEntry); + if (code) { + metaErr(TD_VID(pMeta->pVnode), code); + } + } + + return code; +} + +static int32_t metaHandleNormalTableDrop(SMeta *pMeta, const SMetaEntry *pEntry) { + int32_t code = TSDB_CODE_SUCCESS; + SMetaEntry entry = {0}; + + // TODO: get entry + + metaWLock(pMeta); + code = metaHandleNormalTableDropImpl(pMeta, &entry); + if (code) { + metaErr(TD_VID(pMeta->pVnode), code); + } else { + code = metaTableOpFn[META_ENTRY_TABLE][META_TABLE_OP_INSERT](pMeta, pEntry); + } + metaULock(pMeta); + + if (TSDB_CODE_SUCCESS == code) { + pMeta->pVnode->config.vndStats.numOfNTables--; + pMeta->pVnode->config.vndStats.numOfNTimeSeries -= (entry.ntbEntry.schemaRow.nCols - 1); + + // if ((type == TSDB_CHILD_TABLE || type == TSDB_NORMAL_TABLE) && tbUids) { + // if (taosArrayPush(tbUids, &uid) == NULL) { + // rc = terrno; + // goto _exit; + // } + + // if (!TSDB_CACHE_NO(pMeta->pVnode->config)) { + // int32_t ret = tsdbCacheDropTable(pMeta->pVnode->pTsdb, uid, suid, NULL); + // if (ret < 0) { + // metaError("vgId:%d, failed to drop table:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, + // uid, + // tstrerror(ret)); + // } + // } + // } + + pMeta->changed = true; + } else { + metaErr(TD_VID(pMeta->pVnode), code); + } + return code; } @@ -478,7 +762,7 @@ int32_t metaHandleEntry2(SMeta *pMeta, const SMetaEntry *pEntry) { if (isExist) { // code = metaHandleChildTableUpdate(pMeta, pEntry); } else { - // code = metaHandleChildTableCreate(pMeta, pEntry); + code = metaHandleChildTableCreate(pMeta, pEntry); } break; } @@ -506,7 +790,7 @@ int32_t metaHandleEntry2(SMeta *pMeta, const SMetaEntry *pEntry) { break; } case TSDB_NORMAL_TABLE: { - // code = metaHandleNormalTableDrop(pMeta, pEntry); + code = metaHandleNormalTableDrop(pMeta, pEntry); break; } default: { diff --git a/source/dnode/vnode/src/meta/metaTable2.c b/source/dnode/vnode/src/meta/metaTable2.c index b0e4923d22..37116f9f89 100644 --- a/source/dnode/vnode/src/meta/metaTable2.c +++ b/source/dnode/vnode/src/meta/metaTable2.c @@ -120,178 +120,149 @@ int32_t metaCreateSuperTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq // Alter Super Table // Create Child Table +static int32_t metaCheckCreateChildTableReq(SMeta *pMeta, int64_t version, SVCreateTbReq *pReq) { + int32_t code = TSDB_CODE_SUCCESS; + void *value = NULL; + int32_t valueSize = 0; + SMetaInfo info; + + if (NULL == pReq->name || strlen(pReq->name) == 0 || NULL == pReq->ctb.stbName || strlen(pReq->ctb.stbName) == 0 || + pReq->ctb.suid == 0) { + metaError("vgId:%d, %s failed at %s:%d since invalid name:%s stb name:%s, version:%" PRId64, TD_VID(pMeta->pVnode), + __func__, __FILE__, __LINE__, pReq->name, pReq->ctb.stbName, version); + return TSDB_CODE_INVALID_MSG; + } + + // check super table existence + if (tdbTbGet(pMeta->pNameIdx, pReq->ctb.stbName, strlen(pReq->ctb.stbName) + 1, &value, &valueSize) == 0) { + int64_t suid = *(int64_t *)value; + tdbFreeClear(value); + if (suid != pReq->ctb.suid) { + metaError("vgId:%d, %s failed at %s:%d since super table %s has uid %" PRId64 " instead of %" PRId64 + ", version:%" PRId64, + TD_VID(pMeta->pVnode), __func__, __FILE__, __LINE__, pReq->ctb.stbName, suid, pReq->ctb.suid, version); + return TSDB_CODE_PAR_TABLE_NOT_EXIST; + } + } else { + metaError("vgId:%d, %s failed at %s:%d since super table %s does not eixst, version:%" PRId64, + TD_VID(pMeta->pVnode), __func__, __FILE__, __LINE__, pReq->ctb.stbName, version); + return TSDB_CODE_PAR_TABLE_NOT_EXIST; + } + + // check super table is a super table + if (metaGetInfo(pMeta, pReq->ctb.suid, &info, NULL) != TSDB_CODE_SUCCESS) { + metaError("vgId:%d, %s failed at %s:%d since cannot find table with uid %" PRId64 + ", which is an internal error, version:%" PRId64, + TD_VID(pMeta->pVnode), __func__, __FILE__, __LINE__, pReq->ctb.suid, version); + return TSDB_CODE_INTERNAL_ERROR; + } else if (info.suid != info.uid) { + metaError("vgId:%d, %s failed at %s:%d since table with uid %" PRId64 " is not a super table, version:%" PRId64, + TD_VID(pMeta->pVnode), __func__, __FILE__, __LINE__, pReq->ctb.suid, version); + return TSDB_CODE_INVALID_MSG; + } + + // check table existence + if (tdbTbGet(pMeta->pNameIdx, pReq->name, strlen(pReq->name) + 1, &value, &valueSize) == 0) { + pReq->uid = *(int64_t *)value; + tdbFreeClear(value); + + if (metaGetInfo(pMeta, pReq->uid, &info, NULL) != 0) { + metaError("vgId:%d, %s failed at %s:%d since cannot find table with uid %" PRId64 + ", which is an internal error, version:%" PRId64, + TD_VID(pMeta->pVnode), __func__, __FILE__, __LINE__, pReq->uid, version); + return TSDB_CODE_INTERNAL_ERROR; + } + + // check table type and suid + if (info.suid != pReq->ctb.suid) { + metaError("vgId:%d, %s failed at %s:%d since table %s uid %" PRId64 " exists in another stable with uid %" PRId64 + " instead of stable with uid %" PRId64 " version:%" PRId64, + TD_VID(pMeta->pVnode), __func__, __FILE__, __LINE__, pReq->name, pReq->uid, info.suid, pReq->ctb.suid, + version); + return TSDB_CODE_TDB_TABLE_IN_OTHER_STABLE; + } + + return TSDB_CODE_TDB_TABLE_ALREADY_EXIST; + } + + // check grant + if (!metaTbInFilterCache(pMeta, pReq->ctb.stbName, 1)) { + code = grantCheck(TSDB_GRANT_TIMESERIES); + if (TSDB_CODE_SUCCESS != code) { + metaError("vgId:%d, %s failed at %s:%d since %s, version:%" PRId64 " name:%s", TD_VID(pMeta->pVnode), __func__, + __FILE__, __LINE__, tstrerror(code), version, pReq->name); + } + } + return code; +} + +static int32_t metaBuildCreateChildTableRsp(SMeta *pMeta, const SMetaEntry *pEntry, STableMetaRsp **ppRsp) { + int32_t code = TSDB_CODE_SUCCESS; + + if (NULL == ppRsp) { + return code; + } + + *ppRsp = taosMemoryCalloc(1, sizeof(STableMetaRsp)); + if (NULL == ppRsp) { + return terrno; + } + + (*ppRsp)->tableType = TSDB_CHILD_TABLE; + (*ppRsp)->tuid = pEntry->uid; + (*ppRsp)->suid = pEntry->ctbEntry.suid; + tstrncpy((*ppRsp)->tbName, pEntry->name, TSDB_TABLE_NAME_LEN); + + return code; +} + static int32_t metaCreateChildTable(SMeta *pMeta, int64_t version, SVCreateTbReq *pReq, STableMetaRsp **ppRsp) { int32_t code = TSDB_CODE_SUCCESS; - // TODO + + // check request + code = metaCheckCreateChildTableReq(pMeta, version, pReq); + if (code) { + if (TSDB_CODE_TDB_TABLE_ALREADY_EXIST != code) { + metaError("vgId:%d, %s failed at %s:%d since %s, version:%" PRId64 " name:%s", TD_VID(pMeta->pVnode), __func__, + __FILE__, __LINE__, tstrerror(code), version, pReq->name); + } + return code; + } + + SMetaEntry entry = { + .version = version, + .type = TSDB_CHILD_TABLE, + .uid = pReq->uid, + .name = pReq->name, + .ctbEntry.btime = pReq->btime, + .ctbEntry.ttlDays = pReq->ttl, + .ctbEntry.commentLen = pReq->commentLen, + .ctbEntry.comment = pReq->comment, + .ctbEntry.suid = pReq->ctb.suid, + .ctbEntry.pTags = pReq->ctb.pTag, + }; + + // build response + code = metaBuildCreateChildTableRsp(pMeta, &entry, ppRsp); + if (code) { + metaError("vgId:%d, %s failed at %s:%d since %s", TD_VID(pMeta->pVnode), __func__, __FILE__, __LINE__, + tstrerror(code)); + } + + // handle entry + code = metaHandleEntry2(pMeta, &entry); + if (TSDB_CODE_SUCCESS == code) { + metaInfo("vgId:%d, child table:%s uid %" PRId64 " suid:%" PRId64 " is created, version:%" PRId64, + TD_VID(pMeta->pVnode), pReq->name, pReq->uid, pReq->ctb.suid, version); + } else { + metaError("vgId:%d, %s failed at %s:%d since %s, uid:%" PRId64 " name:%s suid:%" PRId64 " version:%" PRId64, + TD_VID(pMeta->pVnode), __func__, __FILE__, __LINE__, tstrerror(code), pReq->uid, pReq->name, + pReq->ctb.suid, version); + } return code; #if 0 - SMetaEntry me = {0}; - SMetaReader mr = {0}; - int32_t ret; - - // validate message - if (pReq->type != TSDB_CHILD_TABLE && pReq->type != TSDB_NORMAL_TABLE) { - terrno = TSDB_CODE_INVALID_MSG; - goto _err; - } - - if (pReq->type == TSDB_CHILD_TABLE) { - tb_uid_t suid = metaGetTableEntryUidByName(pMeta, pReq->ctb.stbName); - if (suid != pReq->ctb.suid) { - return terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST; - } - } - - // validate req - metaReaderDoInit(&mr, pMeta, META_READER_LOCK); - if (metaGetTableEntryByName(&mr, pReq->name) == 0) { - if (pReq->type == TSDB_CHILD_TABLE && pReq->ctb.suid != mr.me.ctbEntry.suid) { - metaReaderClear(&mr); - return terrno = TSDB_CODE_TDB_TABLE_IN_OTHER_STABLE; - } - pReq->uid = mr.me.uid; - if (pReq->type == TSDB_CHILD_TABLE) { - pReq->ctb.suid = mr.me.ctbEntry.suid; - } - metaReaderClear(&mr); - return terrno = TSDB_CODE_TDB_TABLE_ALREADY_EXIST; - } else if (terrno == TSDB_CODE_PAR_TABLE_NOT_EXIST) { - terrno = TSDB_CODE_SUCCESS; - } - metaReaderClear(&mr); - - bool sysTbl = (pReq->type == TSDB_CHILD_TABLE) && metaTbInFilterCache(pMeta, pReq->ctb.stbName, 1); - - if (!sysTbl && ((terrno = grantCheck(TSDB_GRANT_TIMESERIES)) < 0)) goto _err; - - // build SMetaEntry - SVnodeStats *pStats = &pMeta->pVnode->config.vndStats; - me.version = ver; - me.type = pReq->type; - me.uid = pReq->uid; - me.name = pReq->name; - if (me.type == TSDB_CHILD_TABLE) { - me.ctbEntry.btime = pReq->btime; - me.ctbEntry.ttlDays = pReq->ttl; - me.ctbEntry.commentLen = pReq->commentLen; - me.ctbEntry.comment = pReq->comment; - me.ctbEntry.suid = pReq->ctb.suid; - me.ctbEntry.pTags = pReq->ctb.pTag; - -#ifdef TAG_FILTER_DEBUG - SArray *pTagVals = NULL; - int32_t code = tTagToValArray((STag *)pReq->ctb.pTag, &pTagVals); - for (int i = 0; i < taosArrayGetSize(pTagVals); i++) { - STagVal *pTagVal = (STagVal *)taosArrayGet(pTagVals, i); - - if (IS_VAR_DATA_TYPE(pTagVal->type)) { - char *buf = taosMemoryCalloc(pTagVal->nData + 1, 1); - memcpy(buf, pTagVal->pData, pTagVal->nData); - metaDebug("metaTag table:%s varchar index:%d cid:%d type:%d value:%s", pReq->name, i, pTagVal->cid, - pTagVal->type, buf); - taosMemoryFree(buf); - } else { - double val = 0; - GET_TYPED_DATA(val, double, pTagVal->type, &pTagVal->i64); - metaDebug("metaTag table:%s number index:%d cid:%d type:%d value:%f", pReq->name, i, pTagVal->cid, - pTagVal->type, val); - } - } -#endif - - ++pStats->numOfCTables; - - if (!sysTbl) { - int32_t nCols = 0; - ret = metaGetStbStats(pMeta->pVnode, me.ctbEntry.suid, 0, &nCols); - if (ret < 0) { - metaError("vgId:%d, failed to get stb stats:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, - pReq->ctb.suid, tstrerror(ret)); - } - pStats->numOfTimeSeries += nCols - 1; - } - - metaWLock(pMeta); - metaUpdateStbStats(pMeta, me.ctbEntry.suid, 1, 0); - ret = metaUidCacheClear(pMeta, me.ctbEntry.suid); - if (ret < 0) { - metaError("vgId:%d, failed to clear uid cache:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, - pReq->ctb.suid, tstrerror(ret)); - } - ret = metaTbGroupCacheClear(pMeta, me.ctbEntry.suid); - if (ret < 0) { - metaError("vgId:%d, failed to clear group cache:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, - pReq->ctb.suid, tstrerror(ret)); - } - metaULock(pMeta); - - if (!TSDB_CACHE_NO(pMeta->pVnode->config)) { - ret = tsdbCacheNewTable(pMeta->pVnode->pTsdb, me.uid, me.ctbEntry.suid, NULL); - if (ret < 0) { - metaError("vgId:%d, failed to create table:%s since %s", TD_VID(pMeta->pVnode), pReq->name, tstrerror(ret)); - goto _err; - } - } - } else { - me.ntbEntry.btime = pReq->btime; - me.ntbEntry.ttlDays = pReq->ttl; - me.ntbEntry.commentLen = pReq->commentLen; - me.ntbEntry.comment = pReq->comment; - me.ntbEntry.schemaRow = pReq->ntb.schemaRow; - me.ntbEntry.ncid = me.ntbEntry.schemaRow.pSchema[me.ntbEntry.schemaRow.nCols - 1].colId + 1; - me.colCmpr = pReq->colCmpr; - TABLE_SET_COL_COMPRESSED(me.flags); - - ++pStats->numOfNTables; - pStats->numOfNTimeSeries += me.ntbEntry.schemaRow.nCols - 1; - - if (!TSDB_CACHE_NO(pMeta->pVnode->config)) { - ret = tsdbCacheNewTable(pMeta->pVnode->pTsdb, me.uid, -1, &me.ntbEntry.schemaRow); - if (ret < 0) { - metaError("vgId:%d, failed to create table:%s since %s", TD_VID(pMeta->pVnode), pReq->name, tstrerror(ret)); - goto _err; - } - } - } - - if (metaHandleEntry(pMeta, &me) < 0) goto _err; - metaTimeSeriesNotifyCheck(pMeta); - - if (pMetaRsp) { - *pMetaRsp = taosMemoryCalloc(1, sizeof(STableMetaRsp)); - - if (*pMetaRsp) { - if (me.type == TSDB_CHILD_TABLE) { - (*pMetaRsp)->tableType = TSDB_CHILD_TABLE; - (*pMetaRsp)->tuid = pReq->uid; - (*pMetaRsp)->suid = pReq->ctb.suid; - strcpy((*pMetaRsp)->tbName, pReq->name); - } else { - ret = metaUpdateMetaRsp(pReq->uid, pReq->name, &pReq->ntb.schemaRow, *pMetaRsp); - if (ret < 0) { - metaError("vgId:%d, failed to update meta rsp:%s since %s", TD_VID(pMeta->pVnode), pReq->name, - tstrerror(ret)); - } - for (int32_t i = 0; i < pReq->colCmpr.nCols; i++) { - SColCmpr *p = &pReq->colCmpr.pColCmpr[i]; - (*pMetaRsp)->pSchemaExt[i].colId = p->id; - (*pMetaRsp)->pSchemaExt[i].compress = p->alg; - } - } - } - } - - pMeta->changed = true; - metaDebug("vgId:%d, table:%s uid %" PRId64 " is created, type:%" PRId8, TD_VID(pMeta->pVnode), pReq->name, pReq->uid, - pReq->type); - return 0; - -_err: - metaError("vgId:%d, failed to create table:%s type:%s since %s", TD_VID(pMeta->pVnode), pReq->name, - pReq->type == TSDB_CHILD_TABLE ? "child table" : "normal table", tstrerror(terrno)); - return TSDB_CODE_FAILED; #endif } @@ -368,7 +339,6 @@ static int32_t metaCreateNormalTable(SMeta *pMeta, int64_t version, SVCreateTbRe TAOS_RETURN(code); } - // handle entry SMetaEntry entry = { .version = version, .type = TSDB_NORMAL_TABLE, @@ -391,6 +361,7 @@ static int32_t metaCreateNormalTable(SMeta *pMeta, int64_t version, SVCreateTbRe tstrerror(code)); } + // handle entry code = metaHandleEntry2(pMeta, &entry); if (TSDB_CODE_SUCCESS == code) { metaInfo("vgId:%d, normal table:%s uid %" PRId64 " is created, version:%" PRId64, TD_VID(pMeta->pVnode), pReq->name, @@ -400,19 +371,103 @@ static int32_t metaCreateNormalTable(SMeta *pMeta, int64_t version, SVCreateTbRe __func__, __FILE__, __LINE__, tstrerror(code), pReq->uid, pReq->name, version); } TAOS_RETURN(code); +#if 0 + metaTimeSeriesNotifyCheck(pMeta); +#endif } // Drop Normal Table // Alter Normal Table +static int32_t metaCheckDropTableReq(SMeta *pMeta, int64_t version, SVDropTbReq *pReq) { + int32_t code = TSDB_CODE_SUCCESS; + void *value = NULL; + int32_t valueSize = 0; + SMetaInfo info; + + if (NULL == pReq->name || strlen(pReq->name) == 0) { + code = TSDB_CODE_INVALID_MSG; + metaError("vgId:%d, %s failed at %s:%d since invalid name:%s, version:%" PRId64, TD_VID(pMeta->pVnode), __func__, + __FILE__, __LINE__, pReq->name, version); + return code; + } + + code = tdbTbGet(pMeta->pNameIdx, pReq->name, strlen(pReq->name) + 1, &value, &valueSize); + if (TSDB_CODE_SUCCESS != code) { + if (pReq->igNotExists) { + metaTrace("vgId:%d, %s success since table %s not found, version:%" PRId64, TD_VID(pMeta->pVnode), __func__, + pReq->name, version); + } else { + metaError("vgId:%d, %s failed at %s:%d since table %s not found, version:%" PRId64, TD_VID(pMeta->pVnode), + __func__, __FILE__, __LINE__, pReq->name, version); + } + return TSDB_CODE_TDB_TABLE_NOT_EXIST; + } + pReq->uid = *(tb_uid_t *)value; + tdbFreeClear(value); + + code = metaGetInfo(pMeta, pReq->uid, &info, NULL); + if (TSDB_CODE_SUCCESS != code) { + metaError("vgId:%d, %s failed at %s:%d since table %s uid %" PRId64 + " not found, this is an internal error, version:%" PRId64, + TD_VID(pMeta->pVnode), __func__, __FILE__, __LINE__, pReq->name, pReq->uid, version); + code = TSDB_CODE_INTERNAL_ERROR; + return code; + } + pReq->suid = info.suid; + + return code; +} + +int32_t metaDropTable2(SMeta *pMeta, int64_t version, SVDropTbReq *pReq, SArray *tbUids, tb_uid_t *tbUid) { + int32_t code = TSDB_CODE_SUCCESS; + + // check request + code = metaCheckDropTableReq(pMeta, version, pReq); + if (code) { + if (TSDB_CODE_TDB_TABLE_NOT_EXIST != code) { + metaError("vgId:%d, %s failed at %s:%d since %s, version:%" PRId64 " name:%s", TD_VID(pMeta->pVnode), __func__, + __FILE__, __LINE__, tstrerror(code), version, pReq->name); + } + TAOS_RETURN(code); + } + + if (pReq->suid == pReq->uid) { + code = TSDB_CODE_INVALID_MSG; + metaError("vgId:%d, %s failed at %s:%d since %s, uid:%" PRId64 " name:%s version:%" PRId64, TD_VID(pMeta->pVnode), + __func__, __FILE__, __LINE__, tstrerror(code), pReq->uid, pReq->name, version); + TAOS_RETURN(code); + } + + SMetaEntry entry = { + .version = version, + .uid = pReq->uid, + }; + + if (pReq->suid == 0) { + entry.type = -TSDB_NORMAL_TABLE; + } else { + entry.type = -TSDB_CHILD_TABLE; + } + code = metaHandleEntry2(pMeta, &entry); + if (code) { + metaError("vgId:%d, %s failed at %s:%d since %s, uid:%" PRId64 " name:%s version:%" PRId64, TD_VID(pMeta->pVnode), + __func__, __FILE__, __LINE__, tstrerror(code), pReq->uid, pReq->name, version); + } else { + metaInfo("vgId:%d, table %s uid %" PRId64 " is dropped, version:%" PRId64, TD_VID(pMeta->pVnode), pReq->name, + pReq->uid, version); + } + + TAOS_RETURN(code); +} + int32_t metaCreateTable2(SMeta *pMeta, int64_t version, SVCreateTbReq *pReq, STableMetaRsp **ppRsp) { int32_t code = TSDB_CODE_SUCCESS; if (TSDB_CHILD_TABLE == pReq->type) { - // TODO code = metaCreateTable(pMeta, version, pReq, ppRsp); } else if (TSDB_NORMAL_TABLE == pReq->type) { - code = metaCreateTable(pMeta, version, pReq, ppRsp); + code = metaCreateNormalTable(pMeta, version, pReq, ppRsp); } else { code = TSDB_CODE_INVALID_MSG; } diff --git a/source/libs/tdb/inc/tdb.h b/source/libs/tdb/inc/tdb.h index 52ff749191..01d5daf9f5 100644 --- a/source/libs/tdb/inc/tdb.h +++ b/source/libs/tdb/inc/tdb.h @@ -88,6 +88,11 @@ void tdbTxnCloseImpl(TXN *pTxn); // other void tdbFree(void *); +#define tdbFreeClear(p) \ + do { \ + tdbFree(p); \ + p = NULL; \ + } while (0) typedef struct hashset_st *hashset_t; From f549bbdeb254e607ee2c6219f91d12631723f2bc Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 9 Dec 2024 15:09:10 +0800 Subject: [PATCH 14/45] fix: ci cases --- source/dnode/vnode/src/meta/metaEntry2.c | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaEntry2.c b/source/dnode/vnode/src/meta/metaEntry2.c index 77effc7e13..9c4836aee5 100644 --- a/source/dnode/vnode/src/meta/metaEntry2.c +++ b/source/dnode/vnode/src/meta/metaEntry2.c @@ -407,8 +407,23 @@ static int32_t metaBtimeIdxDelete(SMeta *pMeta, const SMetaEntry *pEntry) { // TTL Index static int32_t metaTtlIdxUpsert(SMeta *pMeta, const SMetaEntry *pEntry, EMetaTableOp op) { - // TODO - return 0; + STtlUpdTtlCtx ctx = { + .uid = pEntry->uid, + .pTxn = pMeta->txn, + }; + if (pEntry->type == TSDB_CHILD_TABLE) { + ctx.ttlDays = pEntry->ctbEntry.ttlDays; + ctx.changeTimeMs = pEntry->ctbEntry.btime; + } else { + ctx.ttlDays = pEntry->ntbEntry.ttlDays; + ctx.changeTimeMs = pEntry->ntbEntry.btime; + } + + int32_t ret = ttlMgrInsertTtl(pMeta->pTtlMgr, &ctx); + if (ret < 0) { + metaError("vgId:%d, failed to insert ttl, uid: %" PRId64 " %s", TD_VID(pMeta->pVnode), pEntry->uid, tstrerror(ret)); + } + return TSDB_CODE_SUCCESS; } static int32_t metaTtlIdxInsert(SMeta *pMeta, const SMetaEntry *pEntry) { @@ -533,8 +548,7 @@ static int32_t metaHandleNormalTableCreateImpl(SMeta *pMeta, const SMetaEntry *p {META_UID_IDX, META_TABLE_OP_INSERT}, // {META_NAME_IDX, META_TABLE_OP_INSERT}, // {META_BTIME_IDX, META_TABLE_OP_INSERT}, // - - // {META_TTL_IDX, META_TABLE_OP_INSERT}, // + {META_TTL_IDX, META_TABLE_OP_INSERT}, // }; for (int i = 0; i < sizeof(ops) / sizeof(ops[0]); i++) { From 57f16a6f24bef22acecb803721310fb2bb53d570 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 9 Dec 2024 18:28:29 +0800 Subject: [PATCH 15/45] more code --- source/dnode/vnode/src/meta/metaEntry.c | 173 +++++++++ source/dnode/vnode/src/meta/metaEntry2.c | 426 ++++++++++++++++------- source/dnode/vnode/src/meta/metaTable.c | 9 +- source/dnode/vnode/src/meta/metaTable2.c | 2 +- 4 files changed, 484 insertions(+), 126 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaEntry.c b/source/dnode/vnode/src/meta/metaEntry.c index 7ec73a6abb..73a35d27c2 100644 --- a/source/dnode/vnode/src/meta/metaEntry.c +++ b/source/dnode/vnode/src/meta/metaEntry.c @@ -65,6 +65,23 @@ static FORCE_INLINE int32_t metatInitDefaultSColCmprWrapper(SDecoder *pDecoder, return 0; } +static int32_t metaCloneColCmpr(const SColCmprWrapper *pSrc, SColCmprWrapper *pDst) { + pDst->nCols = pSrc->nCols; + pDst->version = pSrc->version; + pDst->pColCmpr = (SColCmpr *)taosMemoryCalloc(pSrc->nCols, sizeof(SColCmpr)); + if (NULL == pDst->pColCmpr) { + return terrno; + } + memcpy(pDst->pColCmpr, pSrc->pColCmpr, pSrc->nCols * sizeof(SColCmpr)); + return 0; +} + +static void metaCloneColCmprFree(SColCmprWrapper *pCmpr) { + if (pCmpr == NULL) { + taosMemoryFreeClear(pCmpr->pColCmpr); + } +} + int metaEncodeEntry(SEncoder *pCoder, const SMetaEntry *pME) { TAOS_CHECK_RETURN(tStartEncode(pCoder)); TAOS_CHECK_RETURN(tEncodeI64(pCoder, pME->version)); @@ -189,3 +206,159 @@ int metaDecodeEntry(SDecoder *pCoder, SMetaEntry *pME) { tEndDecode(pCoder); return 0; } + +static int32_t metaCloneSchema(const SSchemaWrapper *pSrc, SSchemaWrapper *pDst) { + if (pSrc == NULL || pDst == NULL) { + return TSDB_CODE_INVALID_PARA; + } + + pDst->nCols = pSrc->nCols; + pDst->version = pSrc->version; + pDst->pSchema = (SSchema *)taosMemoryMalloc(pSrc->nCols * sizeof(SSchema)); + if (pDst->pSchema == NULL) { + return terrno; + } + memcpy(pDst->pSchema, pSrc->pSchema, pSrc->nCols * sizeof(SSchema)); + return TSDB_CODE_SUCCESS; +} + +static void metaCloneSchemaFree(SSchemaWrapper *pSchema) { + if (pSchema) { + taosMemoryFreeClear(pSchema->pSchema); + } +} + +int32_t metaCloneEntryFree(SMetaEntry **ppEntry) { + if (ppEntry == NULL || *ppEntry == NULL) { + return TSDB_CODE_SUCCESS; + } + + taosMemoryFreeClear((*ppEntry)->name); + + if ((*ppEntry)->type < 0) { + taosMemoryFreeClear(*ppEntry); + return TSDB_CODE_SUCCESS; + } + + if (TSDB_SUPER_TABLE == (*ppEntry)->type) { + metaCloneSchemaFree(&(*ppEntry)->stbEntry.schemaRow); + metaCloneSchemaFree(&(*ppEntry)->stbEntry.schemaTag); + } else if (TSDB_CHILD_TABLE == (*ppEntry)->type) { + taosMemoryFreeClear((*ppEntry)->ctbEntry.comment); + taosMemoryFreeClear((*ppEntry)->ctbEntry.pTags); + } else if (TSDB_CHILD_TABLE == (*ppEntry)->type) { + metaCloneSchemaFree(&(*ppEntry)->ntbEntry.schemaRow); + taosMemoryFreeClear((*ppEntry)->ntbEntry.comment); + } else { + return TSDB_CODE_INVALID_PARA; + } + metaCloneColCmprFree(&(*ppEntry)->colCmpr); + + taosMemoryFreeClear(*ppEntry); + return TSDB_CODE_SUCCESS; +} + +int32_t metaCloneEntry(const SMetaEntry *pEntry, SMetaEntry **ppEntry) { + int32_t code = TSDB_CODE_SUCCESS; + + if (NULL == pEntry || NULL == ppEntry) { + return TSDB_CODE_INVALID_PARA; + } + + *ppEntry = (SMetaEntry *)taosMemoryCalloc(1, sizeof(SMetaEntry)); + if (NULL == *ppEntry) { + return terrno; + } + + (*ppEntry)->version = pEntry->version; + (*ppEntry)->type = pEntry->type; + (*ppEntry)->uid = pEntry->uid; + + if (pEntry->type < 0) { + return TSDB_CODE_SUCCESS; + } + + if (pEntry->name) { + (*ppEntry)->name = tstrdup(pEntry->name); + if (NULL == (*ppEntry)->name) { + code = terrno; + metaCloneEntryFree(ppEntry); + return code; + } + } + + if (pEntry->type == TSDB_SUPER_TABLE) { + (*ppEntry)->flags = pEntry->flags; + + code = metaCloneSchema(&pEntry->stbEntry.schemaRow, &(*ppEntry)->stbEntry.schemaRow); + if (code) { + metaCloneEntryFree(ppEntry); + return code; + } + + code = metaCloneSchema(&pEntry->stbEntry.schemaTag, &(*ppEntry)->stbEntry.schemaTag); + if (code) { + metaCloneEntryFree(ppEntry); + return code; + } + } else if (pEntry->type == TSDB_CHILD_TABLE) { + (*ppEntry)->ctbEntry.btime = pEntry->ctbEntry.btime; + (*ppEntry)->ctbEntry.ttlDays = pEntry->ctbEntry.ttlDays; + (*ppEntry)->ctbEntry.suid = pEntry->ctbEntry.suid; + + // comment + (*ppEntry)->ctbEntry.commentLen = pEntry->ctbEntry.commentLen; + if (pEntry->ctbEntry.commentLen > 0) { + (*ppEntry)->ctbEntry.comment = taosMemoryMalloc(pEntry->ctbEntry.commentLen + 1); + if (NULL == (*ppEntry)->ctbEntry.comment) { + code = terrno; + metaCloneEntryFree(ppEntry); + return code; + } + memcpy((*ppEntry)->ctbEntry.comment, pEntry->ctbEntry.comment, pEntry->ctbEntry.commentLen); + } + + // tags + STag *pTags = (STag *)pEntry->ctbEntry.pTags; + (*ppEntry)->ctbEntry.pTags = taosMemoryCalloc(1, pTags->len); + if (NULL == (*ppEntry)->ctbEntry.pTags) { + code = terrno; + metaCloneEntryFree(ppEntry); + return code; + } + memcpy((*ppEntry)->ctbEntry.pTags, pEntry->ctbEntry.pTags, pTags->len); + } else if (pEntry->type == TSDB_NORMAL_TABLE) { + (*ppEntry)->ntbEntry.btime = pEntry->ntbEntry.btime; + (*ppEntry)->ntbEntry.ttlDays = pEntry->ntbEntry.ttlDays; + (*ppEntry)->ntbEntry.ncid = pEntry->ntbEntry.ncid; + + // schema + code = metaCloneSchema(&pEntry->ntbEntry.schemaRow, &(*ppEntry)->ntbEntry.schemaRow); + if (code) { + metaCloneEntryFree(ppEntry); + return code; + } + + // comment + (*ppEntry)->ntbEntry.commentLen = pEntry->ntbEntry.commentLen; + if (pEntry->ntbEntry.commentLen > 0) { + (*ppEntry)->ntbEntry.comment = taosMemoryMalloc(pEntry->ntbEntry.commentLen + 1); + if (NULL == (*ppEntry)->ntbEntry.comment) { + code = terrno; + metaCloneEntryFree(ppEntry); + return code; + } + memcpy((*ppEntry)->ntbEntry.comment, pEntry->ntbEntry.comment, pEntry->ntbEntry.commentLen); + } + } else { + return TSDB_CODE_INVALID_PARA; + } + + code = metaCloneColCmpr(&pEntry->colCmpr, &(*ppEntry)->colCmpr); + if (code) { + metaCloneEntryFree(ppEntry); + return code; + } + + return code; +} diff --git a/source/dnode/vnode/src/meta/metaEntry2.c b/source/dnode/vnode/src/meta/metaEntry2.c index 9c4836aee5..9faf576183 100644 --- a/source/dnode/vnode/src/meta/metaEntry2.c +++ b/source/dnode/vnode/src/meta/metaEntry2.c @@ -10,6 +10,11 @@ #include "meta.h" +int32_t metaCloneEntry(const SMetaEntry *pEntry, SMetaEntry **ppEntry); +int32_t metaCloneEntryFree(SMetaEntry **ppEntry); +void metaDestroyTagIdxKey(STagIdxKey *pTagIdxKey); +int metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSchema *pSchema); + #define metaErr(VGID, ERRNO) \ do { \ metaError("vgId:%d, %s failed at %s:%d since %s, version:%" PRId64 " type:%d uid:%" PRId64 " name:%s", VGID, \ @@ -37,13 +42,98 @@ typedef enum { META_TABLE_OP_MAX, } EMetaTableOp; +typedef struct { + const SMetaEntry *pEntry; + const SMetaEntry *pSuperEntry; + const SMetaEntry *pOldEntry; +} SMetaHandleParam; + typedef struct { EMetaTable table; EMetaTableOp op; } SMetaTableOp; +static int32_t metaFetchEntryByUid(SMeta *pMeta, int64_t uid, SMetaEntry **ppEntry) { + int32_t code = TSDB_CODE_SUCCESS; + void *value = NULL; + int32_t valueSize = 0; + + // search uid index + code = tdbTbGet(pMeta->pUidIdx, &uid, sizeof(uid), &value, &valueSize); + if (TSDB_CODE_SUCCESS != code) { + metaError("vgId:%d, failed to get entry by uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), uid, tstrerror(code)); + return code; + } + + // search entry table + STbDbKey key = { + .version = ((SUidIdxVal *)value)->version, + .uid = uid, + }; + tdbFreeClear(value); + + code = tdbTbGet(pMeta->pTbDb, &key, sizeof(key), &value, &valueSize); + if (TSDB_CODE_SUCCESS != code) { + metaError("vgId:%d, failed to get entry by uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), uid, tstrerror(code)); + code = TSDB_CODE_INTERNAL_ERROR; + return code; + } + + // decode entry + SDecoder decoder = {0}; + SMetaEntry entry = {0}; + + tDecoderInit(&decoder, value, valueSize); + code = metaDecodeEntry(&decoder, &entry); + if (code) { + metaError("vgId:%d, failed to decode entry by uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), uid, + tstrerror(code)); + tDecoderClear(&decoder); + tdbFreeClear(value); + return code; + } + + code = metaCloneEntry(&entry, ppEntry); + if (code) { + metaError("vgId:%d, failed to clone entry by uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), uid, + tstrerror(code)); + tDecoderClear(&decoder); + tdbFreeClear(value); + return code; + } + + tdbFreeClear(value); + tDecoderClear(&decoder); + return code; +} + +static int32_t metaFetchEntryByName(SMeta *pMeta, const char *name, SMetaEntry **ppEntry) { + int32_t code = TSDB_CODE_SUCCESS; + void *value = NULL; + int32_t valueSize = 0; + + code = tdbTbGet(pMeta->pNameIdx, name, strlen(name) + 1, &value, &valueSize); + if (TSDB_CODE_SUCCESS != code) { + metaError("vgId:%d, failed to get entry by name:%s since %s", TD_VID(pMeta->pVnode), name, tstrerror(code)); + return code; + } + int64_t uid = *(int64_t *)value; + tdbFreeClear(value); + + code = metaFetchEntryByUid(pMeta, uid, ppEntry); + if (TSDB_CODE_SUCCESS != code) { + metaError("vgId:%d, failed to get entry by uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), uid, tstrerror(code)); + code = TSDB_CODE_INTERNAL_ERROR; + } + return code; +} + +static int32_t metaFetchEntryFree(SMetaEntry **ppEntry) { return metaCloneEntryFree(ppEntry); } + // Entry Table -static int32_t metaEntryTableUpsert(SMeta *pMeta, const SMetaEntry *pEntry, EMetaTableOp op) { +static int32_t metaEntryTableUpsert(SMeta *pMeta, const SMetaHandleParam *pParam, EMetaTableOp op) { + const SMetaEntry *pEntry = pParam->pEntry; + int32_t code = TSDB_CODE_SUCCESS; int32_t vgId = TD_VID(pMeta->pVnode); void *value = NULL; @@ -92,15 +182,15 @@ static int32_t metaEntryTableUpsert(SMeta *pMeta, const SMetaEntry *pEntry, EMet return code; } -static int32_t metaEntryTableInsert(SMeta *pMeta, const SMetaEntry *pEntry) { - return metaEntryTableUpsert(pMeta, pEntry, META_TABLE_OP_INSERT); +static int32_t metaEntryTableInsert(SMeta *pMeta, const SMetaHandleParam *pParam) { + return metaEntryTableUpsert(pMeta, pParam, META_TABLE_OP_INSERT); } -static int32_t metaEntryTableUpdate(SMeta *pMeta, const SMetaEntry *pEntry) { - return metaEntryTableUpsert(pMeta, pEntry, META_TABLE_OP_UPDATA); +static int32_t metaEntryTableUpdate(SMeta *pMeta, const SMetaHandleParam *pParam) { + return metaEntryTableUpsert(pMeta, pParam, META_TABLE_OP_UPDATA); } -static int32_t metaEntryTableDelete(SMeta *pMeta, const SMetaEntry *pEntry) { +static int32_t metaEntryTableDelete(SMeta *pMeta, const SMetaHandleParam *pEntry) { int32_t code = TSDB_CODE_SUCCESS; // SMetaEntry entry { .version = ; }; @@ -111,13 +201,14 @@ static int32_t metaEntryTableDelete(SMeta *pMeta, const SMetaEntry *pEntry) { } // Schema Table -static int32_t metaSchemaTableUpsert(SMeta *pMeta, const SMetaEntry *pEntry, EMetaTableOp op) { +static int32_t metaSchemaTableUpsert(SMeta *pMeta, const SMetaHandleParam *pParam, EMetaTableOp op) { int32_t code = TSDB_CODE_SUCCESS; int32_t vgId = TD_VID(pMeta->pVnode); SEncoder encoder = {0}; void *value = NULL; int32_t valueSize = 0; + const SMetaEntry *pEntry = pParam->pEntry; const SSchemaWrapper *pSchema = NULL; if (pEntry->type == TSDB_SUPER_TABLE) { pSchema = &pEntry->stbEntry.schemaRow; @@ -169,15 +260,15 @@ static int32_t metaSchemaTableUpsert(SMeta *pMeta, const SMetaEntry *pEntry, EMe return code; } -static int32_t metaSchemaTableInsert(SMeta *pMeta, const SMetaEntry *pEntry) { - return metaSchemaTableUpsert(pMeta, pEntry, META_TABLE_OP_INSERT); +static int32_t metaSchemaTableInsert(SMeta *pMeta, const SMetaHandleParam *pParam) { + return metaSchemaTableUpsert(pMeta, pParam, META_TABLE_OP_INSERT); } -static int32_t metaSchemaTableUpdate(SMeta *pMeta, const SMetaEntry *pEntry) { - return metaSchemaTableUpsert(pMeta, pEntry, META_TABLE_OP_UPDATA); +static int32_t metaSchemaTableUpdate(SMeta *pMeta, const SMetaHandleParam *pParam) { + return metaSchemaTableUpsert(pMeta, pParam, META_TABLE_OP_UPDATA); } -static int32_t metaSchemaTableDelete(SMeta *pMeta, const SMetaEntry *pEntry) { +static int32_t metaSchemaTableDelete(SMeta *pMeta, const SMetaHandleParam *pEntry) { // TODO return TSDB_CODE_SUCCESS; } @@ -198,10 +289,12 @@ static void metaBuildEntryInfo(const SMetaEntry *pEntry, SMetaInfo *pInfo) { } } -static int32_t metaUidIdxUpsert(SMeta *pMeta, const SMetaEntry *pEntry, EMetaTableOp op) { +static int32_t metaUidIdxUpsert(SMeta *pMeta, const SMetaHandleParam *pParam, EMetaTableOp op) { int32_t code = TSDB_CODE_SUCCESS; int32_t vgId = TD_VID(pMeta->pVnode); + const SMetaEntry *pEntry = pParam->pEntry; + // update cache SMetaInfo info = {0}; metaBuildEntryInfo(pEntry, &info); @@ -224,17 +317,19 @@ static int32_t metaUidIdxUpsert(SMeta *pMeta, const SMetaEntry *pEntry, EMetaTab return code; } -static int32_t metaUidIdxInsert(SMeta *pMeta, const SMetaEntry *pEntry) { - return metaUidIdxUpsert(pMeta, pEntry, META_TABLE_OP_INSERT); +static int32_t metaUidIdxInsert(SMeta *pMeta, const SMetaHandleParam *pParam) { + return metaUidIdxUpsert(pMeta, pParam, META_TABLE_OP_INSERT); } -static int32_t metaUidIdxUpdate(SMeta *pMeta, const SMetaEntry *pEntry) { - return metaUidIdxUpsert(pMeta, pEntry, META_TABLE_OP_UPDATA); +static int32_t metaUidIdxUpdate(SMeta *pMeta, const SMetaHandleParam *pParam) { + return metaUidIdxUpsert(pMeta, pParam, META_TABLE_OP_UPDATA); } -static int32_t metaUidIdxDelete(SMeta *pMeta, const SMetaEntry *pEntry) { +static int32_t metaUidIdxDelete(SMeta *pMeta, const SMetaHandleParam *pParam) { int32_t code = 0; + const SMetaEntry *pEntry = pParam->pEntry; + // delete tdb code = tdbTbDelete(pMeta->pUidIdx, &pEntry->uid, sizeof(pEntry->uid), pMeta->txn); if (code) { @@ -247,8 +342,11 @@ static int32_t metaUidIdxDelete(SMeta *pMeta, const SMetaEntry *pEntry) { } // Name Index -static int32_t metaNameIdxUpsert(SMeta *pMeta, const SMetaEntry *pEntry, EMetaTableOp op) { +static int32_t metaNameIdxUpsert(SMeta *pMeta, const SMetaHandleParam *pParam, EMetaTableOp op) { int32_t code = TSDB_CODE_SUCCESS; + + const SMetaEntry *pEntry = pParam->pEntry; + if (META_TABLE_OP_INSERT == op) { code = tdbTbInsert(pMeta->pNameIdx, pEntry->name, strlen(pEntry->name) + 1, &pEntry->uid, sizeof(pEntry->uid), pMeta->txn); @@ -261,26 +359,18 @@ static int32_t metaNameIdxUpsert(SMeta *pMeta, const SMetaEntry *pEntry, EMetaTa return code; } -static int32_t metaNameIdxInsert(SMeta *pMeta, const SMetaEntry *pEntry) { +static int32_t metaNameIdxInsert(SMeta *pMeta, const SMetaHandleParam *pParam) { int32_t code = TSDB_CODE_SUCCESS; - code = metaNameIdxUpsert(pMeta, pEntry, META_TABLE_OP_INSERT); - if (code) { - metaErr(TD_VID(pMeta->pVnode), code); - } - return code; + return metaNameIdxUpsert(pMeta, pParam, META_TABLE_OP_INSERT); } -static int32_t metaNameIdxUpdate(SMeta *pMeta, const SMetaEntry *pEntry) { - int32_t code = TSDB_CODE_SUCCESS; - code = metaNameIdxUpsert(pMeta, pEntry, META_TABLE_OP_UPDATA); - if (code) { - metaErr(TD_VID(pMeta->pVnode), code); - } - return code; +static int32_t metaNameIdxUpdate(SMeta *pMeta, const SMetaHandleParam *pParam) { + return metaNameIdxUpsert(pMeta, pParam, META_TABLE_OP_UPDATA); } -static int32_t metaNameIdxDelete(SMeta *pMeta, const SMetaEntry *pEntry) { - int32_t code = TSDB_CODE_SUCCESS; +static int32_t metaNameIdxDelete(SMeta *pMeta, const SMetaHandleParam *pParam) { + int32_t code = TSDB_CODE_SUCCESS; + const SMetaEntry *pEntry = pParam->pEntry; code = tdbTbDelete(pMeta->pNameIdx, pEntry->name, strlen(pEntry->name) + 1, pMeta->txn); if (code) { metaErr(TD_VID(pMeta->pVnode), code); @@ -289,7 +379,9 @@ static int32_t metaNameIdxDelete(SMeta *pMeta, const SMetaEntry *pEntry) { } // Suid Index -static int32_t metaSUidIdxInsert(SMeta *pMeta, const SMetaEntry *pEntry) { +static int32_t metaSUidIdxInsert(SMeta *pMeta, const SMetaHandleParam *pParam) { + const SMetaEntry *pEntry = pParam->pEntry; + int32_t code = tdbTbInsert(pMeta->pSuidIdx, &pEntry->uid, sizeof(pEntry->uid), NULL, 0, pMeta->txn); if (code) { metaErr(TD_VID(pMeta->pVnode), code); @@ -297,8 +389,9 @@ static int32_t metaSUidIdxInsert(SMeta *pMeta, const SMetaEntry *pEntry) { return code; } -static int32_t metaSUidIdxDelete(SMeta *pMeta, const SMetaEntry *pEntry) { - int32_t code = tdbTbDelete(pMeta->pSuidIdx, &pEntry->uid, sizeof(pEntry->uid), pMeta->txn); +static int32_t metaSUidIdxDelete(SMeta *pMeta, const SMetaHandleParam *pParam) { + const SMetaEntry *pEntry = pParam->pEntry; + int32_t code = tdbTbDelete(pMeta->pSuidIdx, &pEntry->uid, sizeof(pEntry->uid), pMeta->txn); if (code) { metaErr(TD_VID(pMeta->pVnode), code); } @@ -306,9 +399,11 @@ static int32_t metaSUidIdxDelete(SMeta *pMeta, const SMetaEntry *pEntry) { } // Child Index -static int32_t metaChildIdxUpsert(SMeta *pMeta, const SMetaEntry *pEntry, EMetaTableOp op) { +static int32_t metaChildIdxUpsert(SMeta *pMeta, const SMetaHandleParam *pParam, EMetaTableOp op) { int32_t code = TSDB_CODE_SUCCESS; + const SMetaEntry *pEntry = pParam->pEntry; + SCtbIdxKey key = { .suid = pEntry->ctbEntry.suid, .uid = pEntry->uid, @@ -326,15 +421,17 @@ static int32_t metaChildIdxUpsert(SMeta *pMeta, const SMetaEntry *pEntry, EMetaT return code; } -static int32_t metaChildIdxInsert(SMeta *pMeta, const SMetaEntry *pEntry) { - return metaChildIdxUpsert(pMeta, pEntry, META_TABLE_OP_INSERT); +static int32_t metaChildIdxInsert(SMeta *pMeta, const SMetaHandleParam *pParam) { + return metaChildIdxUpsert(pMeta, pParam, META_TABLE_OP_INSERT); } -static int32_t metaChildIdxUpdate(SMeta *pMeta, const SMetaEntry *pEntry) { - return metaChildIdxUpsert(pMeta, pEntry, META_TABLE_OP_UPDATA); +static int32_t metaChildIdxUpdate(SMeta *pMeta, const SMetaHandleParam *pParam) { + return metaChildIdxUpsert(pMeta, pParam, META_TABLE_OP_UPDATA); } -static int32_t metaChildIdxDelete(SMeta *pMeta, const SMetaEntry *pEntry) { +static int32_t metaChildIdxDelete(SMeta *pMeta, const SMetaHandleParam *pParam) { + const SMetaEntry *pEntry = pParam->pEntry; + SCtbIdxKey key = { .suid = pEntry->ctbEntry.suid, .uid = pEntry->uid, @@ -343,23 +440,89 @@ static int32_t metaChildIdxDelete(SMeta *pMeta, const SMetaEntry *pEntry) { } // Tag Index -static int32_t metaTagIdxInsert(SMeta *pMeta, const SMetaEntry *pEntry) { +static int32_t metaTagIdxInsert(SMeta *pMeta, const SMetaHandleParam *pParam) { + int32_t code = TSDB_CODE_SUCCESS; + + const SMetaEntry *pEntry = pParam->pEntry; + const SMetaEntry *pSuperEntry = pParam->pSuperEntry; + + const SSchemaWrapper *pTagSchema = &pSuperEntry->stbEntry.schemaTag; + if (pTagSchema->nCols == 1 && pTagSchema->pSchema[0].type == TSDB_DATA_TYPE_JSON) { + const SSchema *pTagColumn = &pTagSchema->pSchema[0]; + + STagVal tagVal = { + .cid = pTagColumn->colId, + }; + + const void *pTagData = pEntry->ctbEntry.pTags; + int32_t nTagData = ((const STag *)pEntry->ctbEntry.pTags)->len; + code = metaSaveJsonVarToIdx(pMeta, pEntry, pTagColumn); + if (code) { + metaErr(TD_VID(pMeta->pVnode), code); + } + } else { + STagIdxKey *pTagIdxKey = NULL; + int32_t nTagIdxKey; + + for (int32_t i = 0; i < pTagSchema->nCols; i++) { + const void *pTagData = NULL; + int32_t nTagData = 0; + const SSchema *pTagColumn = &pTagSchema->pSchema[i]; + + if (!IS_IDX_ON(pTagColumn)) { + continue; + } + + STagVal tagVal = { + .cid = pTagColumn->colId, + }; + + if (tTagGet((const STag *)pEntry->ctbEntry.pTags, &tagVal)) { + if (IS_VAR_DATA_TYPE(pTagColumn->type)) { + pTagData = tagVal.pData; + nTagData = (int32_t)tagVal.nData; + } else { + pTagData = &(tagVal.i64); + nTagData = tDataTypes[pTagColumn->type].bytes; + } + } else { + if (!IS_VAR_DATA_TYPE(pTagColumn->type)) { + nTagData = tDataTypes[pTagColumn->type].bytes; + } + } + + code = metaCreateTagIdxKey(pSuperEntry->uid, pTagColumn->colId, pTagData, nTagData, pTagColumn->type, pEntry->uid, + &pTagIdxKey, &nTagIdxKey); + if (code) { + metaErr(TD_VID(pMeta->pVnode), code); + return code; + } + + code = tdbTbInsert(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, NULL, 0, pMeta->txn); + if (code) { + metaErr(TD_VID(pMeta->pVnode), code); + } + metaDestroyTagIdxKey(pTagIdxKey); + pTagIdxKey = NULL; + } + } + return code; +} + +static int32_t metaTagIdxUpdate(SMeta *pMeta, const SMetaHandleParam *pParam) { // TODO return 0; } -static int32_t metaTagIdxUpdate(SMeta *pMeta, const SMetaEntry *pEntry) { - // TODO - return 0; -} - -static int32_t metaTagIdxDelete(SMeta *pMeta, const SMetaEntry *pEntry) { +static int32_t metaTagIdxDelete(SMeta *pMeta, const SMetaHandleParam *pParam) { // TODO return 0; } // Btime Index -static int32_t metaBtimeIdxUpsert(SMeta *pMeta, const SMetaEntry *pEntry, EMetaTableOp op) { +static int32_t metaBtimeIdxUpsert(SMeta *pMeta, const SMetaHandleParam *pParam, EMetaTableOp op) { + const SMetaEntry *pEntry = pParam->pEntry; + SBtimeIdxKey key = { .uid = pEntry->uid, }; @@ -381,32 +544,35 @@ static int32_t metaBtimeIdxUpsert(SMeta *pMeta, const SMetaEntry *pEntry, EMetaT } } -static int32_t metaBtimeIdxInsert(SMeta *pMeta, const SMetaEntry *pEntry) { - return metaBtimeIdxUpsert(pMeta, pEntry, META_TABLE_OP_INSERT); +static int32_t metaBtimeIdxInsert(SMeta *pMeta, const SMetaHandleParam *pParam) { + return metaBtimeIdxUpsert(pMeta, pParam, META_TABLE_OP_INSERT); } -static int32_t metaBtimeIdxUpdate(SMeta *pMeta, const SMetaEntry *pEntry) { - return metaBtimeIdxUpsert(pMeta, pEntry, META_TABLE_OP_UPDATA); +static int32_t metaBtimeIdxUpdate(SMeta *pMeta, const SMetaHandleParam *pParam) { + return metaBtimeIdxUpsert(pMeta, pParam, META_TABLE_OP_UPDATA); } -static int32_t metaBtimeIdxDelete(SMeta *pMeta, const SMetaEntry *pEntry) { - int32_t code = TSDB_CODE_SUCCESS; - SBtimeIdxKey key = { - .uid = pEntry->uid, - }; +static int32_t metaBtimeIdxDelete(SMeta *pMeta, const SMetaHandleParam *pParam) { + // int32_t code = TSDB_CODE_SUCCESS; + // SBtimeIdxKey key = { + // .uid = pEntry->uid, + // }; - if (TSDB_CHILD_TABLE == pEntry->type) { - key.btime = pEntry->ctbEntry.btime; - } else if (TSDB_NORMAL_TABLE == pEntry->type) { - key.btime = pEntry->ntbEntry.btime; - } else { - return TSDB_CODE_INVALID_PARA; - } - return tdbTbDelete(pMeta->pBtimeIdx, &key, sizeof(key), pMeta->txn); + // if (TSDB_CHILD_TABLE == pEntry->type) { + // key.btime = pEntry->ctbEntry.btime; + // } else if (TSDB_NORMAL_TABLE == pEntry->type) { + // key.btime = pEntry->ntbEntry.btime; + // } else { + // return TSDB_CODE_INVALID_PARA; + // } + // return tdbTbDelete(pMeta->pBtimeIdx, &key, sizeof(key), pMeta->txn); + return TSDB_CODE_SUCCESS; } // TTL Index -static int32_t metaTtlIdxUpsert(SMeta *pMeta, const SMetaEntry *pEntry, EMetaTableOp op) { +static int32_t metaTtlIdxUpsert(SMeta *pMeta, const SMetaHandleParam *pParam, EMetaTableOp op) { + const SMetaEntry *pEntry = pParam->pEntry; + STtlUpdTtlCtx ctx = { .uid = pEntry->uid, .pTxn = pMeta->txn, @@ -426,20 +592,20 @@ static int32_t metaTtlIdxUpsert(SMeta *pMeta, const SMetaEntry *pEntry, EMetaTab return TSDB_CODE_SUCCESS; } -static int32_t metaTtlIdxInsert(SMeta *pMeta, const SMetaEntry *pEntry) { - return metaTtlIdxUpsert(pMeta, pEntry, META_TABLE_OP_INSERT); +static int32_t metaTtlIdxInsert(SMeta *pMeta, const SMetaHandleParam *pParam) { + return metaTtlIdxUpsert(pMeta, pParam, META_TABLE_OP_INSERT); } -static int32_t metaTtlIdxUpdate(SMeta *pMeta, const SMetaEntry *pEntry) { - return metaTtlIdxUpsert(pMeta, pEntry, META_TABLE_OP_UPDATA); +static int32_t metaTtlIdxUpdate(SMeta *pMeta, const SMetaHandleParam *pParam) { + return metaTtlIdxUpsert(pMeta, pParam, META_TABLE_OP_UPDATA); } -static int32_t metaTtlIdxDelete(SMeta *pMeta, const SMetaEntry *pEntry) { +static int32_t metaTtlIdxDelete(SMeta *pMeta, const SMetaHandleParam *pParam) { // TODO return 0; } -static int32_t (*metaTableOpFn[META_TABLE_MAX][META_TABLE_OP_MAX])(SMeta *pMeta, const SMetaEntry *pEntry) = +static int32_t (*metaTableOpFn[META_TABLE_MAX][META_TABLE_OP_MAX])(SMeta *pMeta, const SMetaHandleParam *pParam) = { [META_ENTRY_TABLE] = { @@ -509,9 +675,12 @@ static int32_t metaHandleSuperTableCreateImpl(SMeta *pMeta, const SMetaEntry *pE }; for (int i = 0; i < sizeof(ops) / sizeof(ops[0]); i++) { - SMetaTableOp *op = &ops[i]; + SMetaTableOp *op = &ops[i]; + const SMetaHandleParam param = { + .pEntry = pEntry, + }; - code = metaTableOpFn[op->table][op->op](pMeta, pEntry); + code = metaTableOpFn[op->table][op->op](pMeta, ¶m); if (TSDB_CODE_SUCCESS != code) { metaErr(TD_VID(pMeta->pVnode), code); return code; @@ -554,7 +723,11 @@ static int32_t metaHandleNormalTableCreateImpl(SMeta *pMeta, const SMetaEntry *p for (int i = 0; i < sizeof(ops) / sizeof(ops[0]); i++) { SMetaTableOp *op = &ops[i]; - code = metaTableOpFn[op->table][op->op](pMeta, pEntry); + SMetaHandleParam param = { + .pEntry = pEntry, + }; + + code = metaTableOpFn[op->table][op->op](pMeta, ¶m); if (TSDB_CODE_SUCCESS != code) { metaErr(TD_VID(pMeta->pVnode), code); return code; @@ -589,7 +762,7 @@ static int32_t metaHandleNormalTableCreate(SMeta *pMeta, const SMetaEntry *pEntr return code; } -static int32_t metaHandleChildTableCreateImpl(SMeta *pMeta, const SMetaEntry *pEntry) { +static int32_t metaHandleChildTableCreateImpl(SMeta *pMeta, const SMetaEntry *pEntry, const SMetaEntry *pSuperEntry) { int32_t code = TSDB_CODE_SUCCESS; SMetaTableOp ops[] = { @@ -605,53 +778,61 @@ static int32_t metaHandleChildTableCreateImpl(SMeta *pMeta, const SMetaEntry *pE for (int i = 0; i < sizeof(ops) / sizeof(ops[0]); i++) { SMetaTableOp *op = &ops[i]; - code = metaTableOpFn[op->table][op->op](pMeta, pEntry); + SMetaHandleParam param = { + .pEntry = pEntry, + .pSuperEntry = pSuperEntry, + }; + + code = metaTableOpFn[op->table][op->op](pMeta, ¶m); if (TSDB_CODE_SUCCESS != code) { metaErr(TD_VID(pMeta->pVnode), code); return code; } } + if (TSDB_CODE_SUCCESS == code) { + metaUpdateStbStats(pMeta, pSuperEntry->uid, 1, 0); + int32_t ret = metaUidCacheClear(pMeta, pSuperEntry->uid); + if (ret < 0) { + metaErr(TD_VID(pMeta->pVnode), ret); + } + + ret = metaTbGroupCacheClear(pMeta, pSuperEntry->uid); + if (ret < 0) { + metaErr(TD_VID(pMeta->pVnode), ret); + } + } return code; } static int32_t metaHandleChildTableCreate(SMeta *pMeta, const SMetaEntry *pEntry) { - int32_t code = TSDB_CODE_SUCCESS; + int32_t code = TSDB_CODE_SUCCESS; + SMetaEntry *pSuperEntry = NULL; + + // get the super table entry + code = metaFetchEntryByUid(pMeta, pEntry->ctbEntry.suid, &pSuperEntry); + if (code) { + metaErr(TD_VID(pMeta->pVnode), code); + return code; + } // update TDB metaWLock(pMeta); - code = metaHandleChildTableCreateImpl(pMeta, pEntry); - - if (TSDB_CODE_SUCCESS == code) { - // metaUpdateStbStats(pMeta, me.ctbEntry.suid, 1, 0); - // ret = metaUidCacheClear(pMeta, me.ctbEntry.suid); - // if (ret < 0) { - // metaError("vgId:%d, failed to clear uid cache:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, - // pReq->ctb.suid, tstrerror(ret)); - // } - // ret = metaTbGroupCacheClear(pMeta, me.ctbEntry.suid); - // if (ret < 0) { - // metaError("vgId:%d, failed to clear group cache:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), - // pReq->name, - // pReq->ctb.suid, tstrerror(ret)); - // } - } + code = metaHandleChildTableCreateImpl(pMeta, pEntry, pSuperEntry); metaULock(pMeta); // update other stuff if (TSDB_CODE_SUCCESS != code) { pMeta->pVnode->config.vndStats.numOfCTables++; - // if (!metaTbInFilterCache(pMeta, pReq->ctb.stbName, 1)) { - // int32_t nCols = 0; - // ret = metaGetStbStats(pMeta->pVnode, me.ctbEntry.suid, 0, &nCols); - // if (ret < 0) { - // metaError("vgId:%d, failed to get stb stats:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), - // pReq->name, - // pReq->ctb.suid, tstrerror(ret)); - // } - // pStats->numOfTimeSeries += nCols - 1; - // } + if (!metaTbInFilterCache(pMeta, pSuperEntry->name, 1)) { + int32_t nCols = 0; + int32_t ret = metaGetStbStats(pMeta->pVnode, pSuperEntry->uid, 0, &nCols); + if (ret < 0) { + metaErr(TD_VID(pMeta->pVnode), ret); + } + pMeta->pVnode->config.vndStats.numOfNTimeSeries += (nCols - 1); + } if (!TSDB_CACHE_NO(pMeta->pVnode->config)) { int32_t rc = tsdbCacheNewTable(pMeta->pVnode->pTsdb, pEntry->uid, pEntry->ctbEntry.suid, NULL); @@ -660,10 +841,13 @@ static int32_t metaHandleChildTableCreate(SMeta *pMeta, const SMetaEntry *pEntry tstrerror(rc)); } } + pMeta->changed = true; } else { metaErr(TD_VID(pMeta->pVnode), code); } + + metaFetchEntryFree(&pSuperEntry); return code; } @@ -680,13 +864,13 @@ static int32_t metaHandleNormalTableDropImpl(SMeta *pMeta, const SMetaEntry *pEn // {META_TTL_IDX, META_TABLE_OP_INSERT}, // }; - for (int32_t i = 0; i < sizeof(ops) / sizeof(ops[0]); i++) { - SMetaTableOp *op = &ops[i]; - code = metaTableOpFn[op->table][op->op](pMeta, pEntry); - if (code) { - metaErr(TD_VID(pMeta->pVnode), code); - } - } + // for (int32_t i = 0; i < sizeof(ops) / sizeof(ops[0]); i++) { + // SMetaTableOp *op = &ops[i]; + // code = metaTableOpFn[op->table][op->op](pMeta, pEntry); + // if (code) { + // metaErr(TD_VID(pMeta->pVnode), code); + // } + // } return code; } @@ -698,13 +882,13 @@ static int32_t metaHandleNormalTableDrop(SMeta *pMeta, const SMetaEntry *pEntry) // TODO: get entry metaWLock(pMeta); - code = metaHandleNormalTableDropImpl(pMeta, &entry); - if (code) { - metaErr(TD_VID(pMeta->pVnode), code); - } else { - code = metaTableOpFn[META_ENTRY_TABLE][META_TABLE_OP_INSERT](pMeta, pEntry); - } - metaULock(pMeta); + // code = metaHandleNormalTableDropImpl(pMeta, &entry); + // if (code) { + // metaErr(TD_VID(pMeta->pVnode), code); + // } else { + // code = metaTableOpFn[META_ENTRY_TABLE][META_TABLE_OP_INSERT](pMeta, pEntry); + // } + // metaULock(pMeta); if (TSDB_CODE_SUCCESS == code) { pMeta->pVnode->config.vndStats.numOfNTables--; diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 7987a7648a..d3b48ea40b 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -17,7 +17,8 @@ extern SDmNotifyHandle dmNotifyHdl; -static int metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSchema *pSchema); +int metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSchema *pSchema); + static int metaDelJsonVarFromIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSchema *pSchema); static int metaSaveToTbDb(SMeta *pMeta, const SMetaEntry *pME); static int metaUpdateUidIdx(SMeta *pMeta, const SMetaEntry *pME); @@ -29,7 +30,7 @@ static int metaUpdateCtbIdx(SMeta *pMeta, const SMetaEntry *pME); static int metaUpdateSuidIdx(SMeta *pMeta, const SMetaEntry *pME); static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry); static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type, tb_uid_t *pSuid, int8_t *pSysTbl); -static void metaDestroyTagIdxKey(STagIdxKey *pTagIdxKey); +void metaDestroyTagIdxKey(STagIdxKey *pTagIdxKey); // opt ins_tables query static int metaUpdateBtimeIdx(SMeta *pMeta, const SMetaEntry *pME); static int metaDeleteBtimeIdx(SMeta *pMeta, const SMetaEntry *pME); @@ -110,7 +111,7 @@ int metaUpdateMetaRsp(tb_uid_t uid, char *tbName, SSchemaWrapper *pSchema, STabl return 0; } -static int metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSchema *pSchema) { +int metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSchema *pSchema) { int32_t code = 0; #ifdef USE_INVERTED_INDEX @@ -3160,7 +3161,7 @@ int metaCreateTagIdxKey(tb_uid_t suid, int32_t cid, const void *pTagData, int32_ return 0; } -static void metaDestroyTagIdxKey(STagIdxKey *pTagIdxKey) { +void metaDestroyTagIdxKey(STagIdxKey *pTagIdxKey) { if (pTagIdxKey) taosMemoryFree(pTagIdxKey); } diff --git a/source/dnode/vnode/src/meta/metaTable2.c b/source/dnode/vnode/src/meta/metaTable2.c index 37116f9f89..58b24432c0 100644 --- a/source/dnode/vnode/src/meta/metaTable2.c +++ b/source/dnode/vnode/src/meta/metaTable2.c @@ -465,7 +465,7 @@ int32_t metaDropTable2(SMeta *pMeta, int64_t version, SVDropTbReq *pReq, SArray int32_t metaCreateTable2(SMeta *pMeta, int64_t version, SVCreateTbReq *pReq, STableMetaRsp **ppRsp) { int32_t code = TSDB_CODE_SUCCESS; if (TSDB_CHILD_TABLE == pReq->type) { - code = metaCreateTable(pMeta, version, pReq, ppRsp); + code = metaCreateChildTable(pMeta, version, pReq, ppRsp); } else if (TSDB_NORMAL_TABLE == pReq->type) { code = metaCreateNormalTable(pMeta, version, pReq, ppRsp); } else { From 4458956f6a3571fed3e1cd4720071eb13e531742 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 9 Dec 2024 19:16:42 +0800 Subject: [PATCH 16/45] fix: check return code --- source/dnode/vnode/src/meta/metaEntry.c | 10 +++++----- source/dnode/vnode/src/meta/metaEntry2.c | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaEntry.c b/source/dnode/vnode/src/meta/metaEntry.c index 73a35d27c2..739f84f0d9 100644 --- a/source/dnode/vnode/src/meta/metaEntry.c +++ b/source/dnode/vnode/src/meta/metaEntry.c @@ -228,16 +228,16 @@ static void metaCloneSchemaFree(SSchemaWrapper *pSchema) { } } -int32_t metaCloneEntryFree(SMetaEntry **ppEntry) { +void metaCloneEntryFree(SMetaEntry **ppEntry) { if (ppEntry == NULL || *ppEntry == NULL) { - return TSDB_CODE_SUCCESS; + return; } taosMemoryFreeClear((*ppEntry)->name); if ((*ppEntry)->type < 0) { taosMemoryFreeClear(*ppEntry); - return TSDB_CODE_SUCCESS; + return; } if (TSDB_SUPER_TABLE == (*ppEntry)->type) { @@ -250,12 +250,12 @@ int32_t metaCloneEntryFree(SMetaEntry **ppEntry) { metaCloneSchemaFree(&(*ppEntry)->ntbEntry.schemaRow); taosMemoryFreeClear((*ppEntry)->ntbEntry.comment); } else { - return TSDB_CODE_INVALID_PARA; + return; } metaCloneColCmprFree(&(*ppEntry)->colCmpr); taosMemoryFreeClear(*ppEntry); - return TSDB_CODE_SUCCESS; + return; } int32_t metaCloneEntry(const SMetaEntry *pEntry, SMetaEntry **ppEntry) { diff --git a/source/dnode/vnode/src/meta/metaEntry2.c b/source/dnode/vnode/src/meta/metaEntry2.c index 9faf576183..869510c6f9 100644 --- a/source/dnode/vnode/src/meta/metaEntry2.c +++ b/source/dnode/vnode/src/meta/metaEntry2.c @@ -11,7 +11,7 @@ #include "meta.h" int32_t metaCloneEntry(const SMetaEntry *pEntry, SMetaEntry **ppEntry); -int32_t metaCloneEntryFree(SMetaEntry **ppEntry); +void metaCloneEntryFree(SMetaEntry **ppEntry); void metaDestroyTagIdxKey(STagIdxKey *pTagIdxKey); int metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSchema *pSchema); @@ -128,7 +128,7 @@ static int32_t metaFetchEntryByName(SMeta *pMeta, const char *name, SMetaEntry * return code; } -static int32_t metaFetchEntryFree(SMetaEntry **ppEntry) { return metaCloneEntryFree(ppEntry); } +static void metaFetchEntryFree(SMetaEntry **ppEntry) { metaCloneEntryFree(ppEntry); } // Entry Table static int32_t metaEntryTableUpsert(SMeta *pMeta, const SMetaHandleParam *pParam, EMetaTableOp op) { From 71d4274b6b9f45e4f0e42d5fc678461e22fc2a36 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 10 Dec 2024 08:41:16 +0800 Subject: [PATCH 17/45] fix: memory leak --- source/dnode/vnode/src/meta/metaEntry.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/meta/metaEntry.c b/source/dnode/vnode/src/meta/metaEntry.c index 739f84f0d9..74a16733f8 100644 --- a/source/dnode/vnode/src/meta/metaEntry.c +++ b/source/dnode/vnode/src/meta/metaEntry.c @@ -77,7 +77,7 @@ static int32_t metaCloneColCmpr(const SColCmprWrapper *pSrc, SColCmprWrapper *pD } static void metaCloneColCmprFree(SColCmprWrapper *pCmpr) { - if (pCmpr == NULL) { + if (pCmpr) { taosMemoryFreeClear(pCmpr->pColCmpr); } } From 82188ded46a48b68992835059b893cd6c64eaf62 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 11 Dec 2024 09:29:55 +0800 Subject: [PATCH 18/45] fix: ci case --- source/dnode/vnode/src/meta/metaTable2.c | 55 +++++++++++++----------- 1 file changed, 31 insertions(+), 24 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaTable2.c b/source/dnode/vnode/src/meta/metaTable2.c index 58b24432c0..5fc38a7292 100644 --- a/source/dnode/vnode/src/meta/metaTable2.c +++ b/source/dnode/vnode/src/meta/metaTable2.c @@ -133,6 +133,37 @@ static int32_t metaCheckCreateChildTableReq(SMeta *pMeta, int64_t version, SVCre return TSDB_CODE_INVALID_MSG; } + // check table existence + if (tdbTbGet(pMeta->pNameIdx, pReq->name, strlen(pReq->name) + 1, &value, &valueSize) == 0) { + pReq->uid = *(int64_t *)value; + tdbFreeClear(value); + + if (metaGetInfo(pMeta, pReq->uid, &info, NULL) != 0) { + metaError("vgId:%d, %s failed at %s:%d since cannot find table with uid %" PRId64 + ", which is an internal error, version:%" PRId64, + TD_VID(pMeta->pVnode), __func__, __FILE__, __LINE__, pReq->uid, version); + return TSDB_CODE_INTERNAL_ERROR; + } + + // check table type + if (info.suid == info.uid || info.suid == 0) { + metaError("vgId:%d, %s failed at %s:%d since table with uid %" PRId64 " is not a super table, version:%" PRId64, + TD_VID(pMeta->pVnode), __func__, __FILE__, __LINE__, pReq->uid, version); + return TSDB_CODE_TDB_TABLE_IN_OTHER_STABLE; + } + + // check suid + if (info.suid != pReq->ctb.suid) { + metaError("vgId:%d, %s failed at %s:%d since table %s uid %" PRId64 " exists in another stable with uid %" PRId64 + " instead of stable with uid %" PRId64 " version:%" PRId64, + TD_VID(pMeta->pVnode), __func__, __FILE__, __LINE__, pReq->name, pReq->uid, info.suid, pReq->ctb.suid, + version); + return TSDB_CODE_TDB_TABLE_IN_OTHER_STABLE; + } + + return TSDB_CODE_TDB_TABLE_ALREADY_EXIST; + } + // check super table existence if (tdbTbGet(pMeta->pNameIdx, pReq->ctb.stbName, strlen(pReq->ctb.stbName) + 1, &value, &valueSize) == 0) { int64_t suid = *(int64_t *)value; @@ -161,30 +192,6 @@ static int32_t metaCheckCreateChildTableReq(SMeta *pMeta, int64_t version, SVCre return TSDB_CODE_INVALID_MSG; } - // check table existence - if (tdbTbGet(pMeta->pNameIdx, pReq->name, strlen(pReq->name) + 1, &value, &valueSize) == 0) { - pReq->uid = *(int64_t *)value; - tdbFreeClear(value); - - if (metaGetInfo(pMeta, pReq->uid, &info, NULL) != 0) { - metaError("vgId:%d, %s failed at %s:%d since cannot find table with uid %" PRId64 - ", which is an internal error, version:%" PRId64, - TD_VID(pMeta->pVnode), __func__, __FILE__, __LINE__, pReq->uid, version); - return TSDB_CODE_INTERNAL_ERROR; - } - - // check table type and suid - if (info.suid != pReq->ctb.suid) { - metaError("vgId:%d, %s failed at %s:%d since table %s uid %" PRId64 " exists in another stable with uid %" PRId64 - " instead of stable with uid %" PRId64 " version:%" PRId64, - TD_VID(pMeta->pVnode), __func__, __FILE__, __LINE__, pReq->name, pReq->uid, info.suid, pReq->ctb.suid, - version); - return TSDB_CODE_TDB_TABLE_IN_OTHER_STABLE; - } - - return TSDB_CODE_TDB_TABLE_ALREADY_EXIST; - } - // check grant if (!metaTbInFilterCache(pMeta, pReq->ctb.stbName, 1)) { code = grantCheck(TSDB_GRANT_TIMESERIES); From fd858372f6b477c61c27b4620b5b27591ff1ad25 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 11 Dec 2024 15:44:42 +0800 Subject: [PATCH 19/45] more code --- source/dnode/vnode/src/inc/vnodeInt.h | 1 + source/dnode/vnode/src/meta/metaEntry2.c | 385 +++++++++++++++++------ source/dnode/vnode/src/meta/metaTable.c | 2 +- source/dnode/vnode/src/meta/metaTable2.c | 105 ++++--- source/dnode/vnode/src/vnd/vnodeSvr.c | 2 +- 5 files changed, 348 insertions(+), 147 deletions(-) diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index f7135a0977..8152f0e4fd 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -162,6 +162,7 @@ int metaDropSTable(SMeta* pMeta, int64_t verison, SVDropStbReq* pReq int metaCreateTable(SMeta* pMeta, int64_t version, SVCreateTbReq* pReq, STableMetaRsp** pMetaRsp); int32_t metaCreateTable2(SMeta* pMeta, int64_t version, SVCreateTbReq* pReq, STableMetaRsp** ppRsp); int metaDropTable(SMeta* pMeta, int64_t version, SVDropTbReq* pReq, SArray* tbUids, int64_t* tbUid); +int32_t metaDropTable2(SMeta* pMeta, int64_t version, SVDropTbReq* pReq); int32_t metaTrimTables(SMeta* pMeta); int32_t metaDropTables(SMeta* pMeta, SArray* tbUids); int metaTtlFindExpired(SMeta* pMeta, int64_t timePointMs, SArray* tbUids, int32_t ttlDropMaxCount); diff --git a/source/dnode/vnode/src/meta/metaEntry2.c b/source/dnode/vnode/src/meta/metaEntry2.c index 869510c6f9..bbc9a14d02 100644 --- a/source/dnode/vnode/src/meta/metaEntry2.c +++ b/source/dnode/vnode/src/meta/metaEntry2.c @@ -14,6 +14,7 @@ int32_t metaCloneEntry(const SMetaEntry *pEntry, SMetaEntry **ppEntry); void metaCloneEntryFree(SMetaEntry **ppEntry); void metaDestroyTagIdxKey(STagIdxKey *pTagIdxKey); int metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSchema *pSchema); +int metaDelJsonVarFromIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSchema *pSchema); #define metaErr(VGID, ERRNO) \ do { \ @@ -172,6 +173,8 @@ static int32_t metaEntryTableUpsert(SMeta *pMeta, const SMetaHandleParam *pParam code = tdbTbInsert(pMeta->pTbDb, &key, sizeof(key), value, valueSize, pMeta->txn); } else if (META_TABLE_OP_UPDATA == op) { code = tdbTbUpsert(pMeta->pTbDb, &key, sizeof(key), value, valueSize, pMeta->txn); + } else if (META_TABLE_OP_DELETE == op) { + code = tdbTbInsert(pMeta->pTbDb, &key, sizeof(key), value, valueSize, pMeta->txn); } else { code = TSDB_CODE_INVALID_PARA; } @@ -190,14 +193,8 @@ static int32_t metaEntryTableUpdate(SMeta *pMeta, const SMetaHandleParam *pParam return metaEntryTableUpsert(pMeta, pParam, META_TABLE_OP_UPDATA); } -static int32_t metaEntryTableDelete(SMeta *pMeta, const SMetaHandleParam *pEntry) { - int32_t code = TSDB_CODE_SUCCESS; - - // SMetaEntry entry { .version = ; }; - - // TODO - - return code; +static int32_t metaEntryTableDelete(SMeta *pMeta, const SMetaHandleParam *pParam) { + return metaEntryTableUpsert(pMeta, pParam, META_TABLE_OP_DELETE); } // Schema Table @@ -328,7 +325,7 @@ static int32_t metaUidIdxUpdate(SMeta *pMeta, const SMetaHandleParam *pParam) { static int32_t metaUidIdxDelete(SMeta *pMeta, const SMetaHandleParam *pParam) { int32_t code = 0; - const SMetaEntry *pEntry = pParam->pEntry; + const SMetaEntry *pEntry = pParam->pOldEntry; // delete tdb code = tdbTbDelete(pMeta->pUidIdx, &pEntry->uid, sizeof(pEntry->uid), pMeta->txn); @@ -369,8 +366,9 @@ static int32_t metaNameIdxUpdate(SMeta *pMeta, const SMetaHandleParam *pParam) { } static int32_t metaNameIdxDelete(SMeta *pMeta, const SMetaHandleParam *pParam) { - int32_t code = TSDB_CODE_SUCCESS; - const SMetaEntry *pEntry = pParam->pEntry; + int32_t code = TSDB_CODE_SUCCESS; + + const SMetaEntry *pEntry = pParam->pOldEntry; code = tdbTbDelete(pMeta->pNameIdx, pEntry->name, strlen(pEntry->name) + 1, pMeta->txn); if (code) { metaErr(TD_VID(pMeta->pVnode), code); @@ -430,7 +428,7 @@ static int32_t metaChildIdxUpdate(SMeta *pMeta, const SMetaHandleParam *pParam) } static int32_t metaChildIdxDelete(SMeta *pMeta, const SMetaHandleParam *pParam) { - const SMetaEntry *pEntry = pParam->pEntry; + const SMetaEntry *pEntry = pParam->pOldEntry; SCtbIdxKey key = { .suid = pEntry->ctbEntry.suid, @@ -461,10 +459,9 @@ static int32_t metaTagIdxInsert(SMeta *pMeta, const SMetaHandleParam *pParam) { metaErr(TD_VID(pMeta->pVnode), code); } } else { - STagIdxKey *pTagIdxKey = NULL; - int32_t nTagIdxKey; - for (int32_t i = 0; i < pTagSchema->nCols; i++) { + STagIdxKey *pTagIdxKey = NULL; + int32_t nTagIdxKey; const void *pTagData = NULL; int32_t nTagData = 0; const SSchema *pTagColumn = &pTagSchema->pSchema[i]; @@ -501,6 +498,8 @@ static int32_t metaTagIdxInsert(SMeta *pMeta, const SMetaHandleParam *pParam) { code = tdbTbInsert(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, NULL, 0, pMeta->txn); if (code) { metaErr(TD_VID(pMeta->pVnode), code); + metaDestroyTagIdxKey(pTagIdxKey); + return code; } metaDestroyTagIdxKey(pTagIdxKey); pTagIdxKey = NULL; @@ -515,13 +514,79 @@ static int32_t metaTagIdxUpdate(SMeta *pMeta, const SMetaHandleParam *pParam) { } static int32_t metaTagIdxDelete(SMeta *pMeta, const SMetaHandleParam *pParam) { - // TODO - return 0; + int32_t code = TSDB_CODE_SUCCESS; + + const SMetaEntry *pEntry = pParam->pEntry; + const SMetaEntry *pChild = pParam->pOldEntry; + const SMetaEntry *pSuper = pParam->pSuperEntry; + const SSchemaWrapper *pTagSchema = &pSuper->stbEntry.schemaTag; + const SSchema *pTagColumn = NULL; + const STag *pTags = (const STag *)pChild->ctbEntry.pTags; + + if (pTagSchema->nCols == 1 && pTagSchema->pSchema[0].type == TSDB_DATA_TYPE_JSON) { + pTagColumn = &pTagSchema->pSchema[0]; + code = metaDelJsonVarFromIdx(pMeta, pChild, pTagColumn); + if (code) { + metaErr(TD_VID(pMeta->pVnode), code); + } + } else { + for (int32_t i = 0; i < pTagSchema->nCols; i++) { + pTagColumn = &pTagSchema->pSchema[i]; + + if (!IS_IDX_ON(pTagColumn)) { + continue; + } + + STagIdxKey *pTagIdxKey = NULL; + int32_t nTagIdxKey; + + const void *pTagData = NULL; + int32_t nTagData = 0; + + STagVal tagVal = {.cid = pTagColumn->colId}; + if (tTagGet(pTags, &tagVal)) { + if (IS_VAR_DATA_TYPE(pTagColumn->type)) { + pTagData = tagVal.pData; + nTagData = (int32_t)tagVal.nData; + } else { + pTagData = &(tagVal.i64); + nTagData = tDataTypes[pTagColumn->type].bytes; + } + } else { + if (!IS_VAR_DATA_TYPE(pTagColumn->type)) { + nTagData = tDataTypes[pTagColumn->type].bytes; + } + } + code = metaCreateTagIdxKey(pSuper->uid, pTagColumn->colId, pTagData, nTagData, pTagColumn->type, pChild->uid, + &pTagIdxKey, &nTagIdxKey); + if (code) { + metaErr(TD_VID(pMeta->pVnode), code); + return code; + } + + code = tdbTbDelete(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, pMeta->txn); + if (code) { + metaErr(TD_VID(pMeta->pVnode), code); + metaDestroyTagIdxKey(pTagIdxKey); + return code; + } + metaDestroyTagIdxKey(pTagIdxKey); + pTagIdxKey = NULL; + } + } + return code; } // Btime Index static int32_t metaBtimeIdxUpsert(SMeta *pMeta, const SMetaHandleParam *pParam, EMetaTableOp op) { - const SMetaEntry *pEntry = pParam->pEntry; + int32_t code = TSDB_CODE_SUCCESS; + + const SMetaEntry *pEntry; + if (META_TABLE_OP_DELETE == op) { + pEntry = pParam->pOldEntry; + } else { + pEntry = pParam->pEntry; + } SBtimeIdxKey key = { .uid = pEntry->uid, @@ -536,12 +601,18 @@ static int32_t metaBtimeIdxUpsert(SMeta *pMeta, const SMetaHandleParam *pParam, } if (META_TABLE_OP_INSERT == op) { - return tdbTbInsert(pMeta->pBtimeIdx, &key, sizeof(key), NULL, 0, pMeta->txn); + code = tdbTbInsert(pMeta->pBtimeIdx, &key, sizeof(key), NULL, 0, pMeta->txn); } else if (META_TABLE_OP_UPDATA == op) { - return tdbTbUpsert(pMeta->pBtimeIdx, &key, sizeof(key), NULL, 0, pMeta->txn); + code = tdbTbUpsert(pMeta->pBtimeIdx, &key, sizeof(key), NULL, 0, pMeta->txn); + } else if (META_TABLE_OP_DELETE == op) { + code = tdbTbDelete(pMeta->pBtimeIdx, &key, sizeof(key), pMeta->txn); } else { - return TSDB_CODE_INVALID_PARA; + code = TSDB_CODE_INVALID_PARA; } + if (code) { + metaErr(TD_VID(pMeta->pVnode), code); + } + return code; } static int32_t metaBtimeIdxInsert(SMeta *pMeta, const SMetaHandleParam *pParam) { @@ -553,20 +624,7 @@ static int32_t metaBtimeIdxUpdate(SMeta *pMeta, const SMetaHandleParam *pParam) } static int32_t metaBtimeIdxDelete(SMeta *pMeta, const SMetaHandleParam *pParam) { - // int32_t code = TSDB_CODE_SUCCESS; - // SBtimeIdxKey key = { - // .uid = pEntry->uid, - // }; - - // if (TSDB_CHILD_TABLE == pEntry->type) { - // key.btime = pEntry->ctbEntry.btime; - // } else if (TSDB_NORMAL_TABLE == pEntry->type) { - // key.btime = pEntry->ntbEntry.btime; - // } else { - // return TSDB_CODE_INVALID_PARA; - // } - // return tdbTbDelete(pMeta->pBtimeIdx, &key, sizeof(key), pMeta->txn); - return TSDB_CODE_SUCCESS; + return metaBtimeIdxUpsert(pMeta, pParam, META_TABLE_OP_DELETE); } // TTL Index @@ -577,12 +635,14 @@ static int32_t metaTtlIdxUpsert(SMeta *pMeta, const SMetaHandleParam *pParam, EM .uid = pEntry->uid, .pTxn = pMeta->txn, }; - if (pEntry->type == TSDB_CHILD_TABLE) { + if (TSDB_CHILD_TABLE == pEntry->type) { ctx.ttlDays = pEntry->ctbEntry.ttlDays; ctx.changeTimeMs = pEntry->ctbEntry.btime; - } else { + } else if (TSDB_NORMAL_TABLE == pEntry->type) { ctx.ttlDays = pEntry->ntbEntry.ttlDays; ctx.changeTimeMs = pEntry->ntbEntry.btime; + } else { + return TSDB_CODE_INVALID_PARA; } int32_t ret = ttlMgrInsertTtl(pMeta->pTtlMgr, &ctx); @@ -601,8 +661,30 @@ static int32_t metaTtlIdxUpdate(SMeta *pMeta, const SMetaHandleParam *pParam) { } static int32_t metaTtlIdxDelete(SMeta *pMeta, const SMetaHandleParam *pParam) { - // TODO - return 0; + int32_t code = TSDB_CODE_SUCCESS; + + const SMetaEntry *pEntry = pParam->pOldEntry; + STtlDelTtlCtx ctx = { + .uid = pEntry->uid, + .pTxn = pMeta->txn, + }; + + if (TSDB_CHILD_TABLE == pEntry->type) { + ctx.ttlDays = pEntry->ctbEntry.ttlDays; + } else if (TSDB_NORMAL_TABLE == pEntry->type) { + ctx.ttlDays = pEntry->ntbEntry.ttlDays; + } else { + code = TSDB_CODE_INVALID_PARA; + } + + if (TSDB_CODE_SUCCESS == code) { + int32_t ret = ttlMgrDeleteTtl(pMeta->pTtlMgr, &ctx); + if (ret < 0) { + metaError("vgId:%d, failed to delete ttl, uid: %" PRId64 " %s", TD_VID(pMeta->pVnode), pEntry->uid, + tstrerror(ret)); + } + } + return code; } static int32_t (*metaTableOpFn[META_TABLE_MAX][META_TABLE_OP_MAX])(SMeta *pMeta, const SMetaHandleParam *pParam) = @@ -851,7 +933,7 @@ static int32_t metaHandleChildTableCreate(SMeta *pMeta, const SMetaEntry *pEntry return code; } -static int32_t metaHandleNormalTableDropImpl(SMeta *pMeta, const SMetaEntry *pEntry) { +static int32_t metaHandleNormalTableDropImpl(SMeta *pMeta, SMetaHandleParam *pParam) { int32_t code = TSDB_CODE_SUCCESS; SMetaTableOp ops[] = { @@ -859,68 +941,183 @@ static int32_t metaHandleNormalTableDropImpl(SMeta *pMeta, const SMetaEntry *pEn {META_UID_IDX, META_TABLE_OP_DELETE}, // {META_NAME_IDX, META_TABLE_OP_DELETE}, // {META_BTIME_IDX, META_TABLE_OP_DELETE}, // + {META_TTL_IDX, META_TABLE_OP_DELETE}, // - // {META_SCHEMA_TABLE, META_TABLE_OP_UPDATA}, // TODO: need to be insert - // {META_TTL_IDX, META_TABLE_OP_INSERT}, // + // {META_SCHEMA_TABLE, META_TABLE_OP_DELETE}, // }; - // for (int32_t i = 0; i < sizeof(ops) / sizeof(ops[0]); i++) { - // SMetaTableOp *op = &ops[i]; - // code = metaTableOpFn[op->table][op->op](pMeta, pEntry); - // if (code) { - // metaErr(TD_VID(pMeta->pVnode), code); - // } - // } - - return code; -} - -static int32_t metaHandleNormalTableDrop(SMeta *pMeta, const SMetaEntry *pEntry) { - int32_t code = TSDB_CODE_SUCCESS; - SMetaEntry entry = {0}; - - // TODO: get entry - - metaWLock(pMeta); - // code = metaHandleNormalTableDropImpl(pMeta, &entry); - // if (code) { - // metaErr(TD_VID(pMeta->pVnode), code); - // } else { - // code = metaTableOpFn[META_ENTRY_TABLE][META_TABLE_OP_INSERT](pMeta, pEntry); - // } - // metaULock(pMeta); - - if (TSDB_CODE_SUCCESS == code) { - pMeta->pVnode->config.vndStats.numOfNTables--; - pMeta->pVnode->config.vndStats.numOfNTimeSeries -= (entry.ntbEntry.schemaRow.nCols - 1); - - // if ((type == TSDB_CHILD_TABLE || type == TSDB_NORMAL_TABLE) && tbUids) { - // if (taosArrayPush(tbUids, &uid) == NULL) { - // rc = terrno; - // goto _exit; - // } - - // if (!TSDB_CACHE_NO(pMeta->pVnode->config)) { - // int32_t ret = tsdbCacheDropTable(pMeta->pVnode->pTsdb, uid, suid, NULL); - // if (ret < 0) { - // metaError("vgId:%d, failed to drop table:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, - // uid, - // tstrerror(ret)); - // } - // } - // } - - pMeta->changed = true; - } else { - metaErr(TD_VID(pMeta->pVnode), code); + for (int32_t i = 0; i < sizeof(ops) / sizeof(ops[0]); i++) { + SMetaTableOp *op = &ops[i]; + code = metaTableOpFn[op->table][op->op](pMeta, pParam); + if (code) { + const SMetaEntry *pEntry = pParam->pEntry; + metaErr(TD_VID(pMeta->pVnode), code); + } } return code; } -static int32_t metaHandleChildTableDrop(SMeta *pMeta, const SMetaEntry *pEntry) { +static int32_t metaHandleNormalTableDrop(SMeta *pMeta, const SMetaEntry *pEntry) { + int32_t code = TSDB_CODE_SUCCESS; + SMetaEntry *pOldEntry = NULL; + + // fetch the entry + code = metaFetchEntryByUid(pMeta, pEntry->uid, &pOldEntry); + if (code) { + metaErr(TD_VID(pMeta->pVnode), code); + return code; + } + + SMetaHandleParam param = { + .pEntry = pEntry, + .pOldEntry = pOldEntry, + }; + + // do the drop + metaWLock(pMeta); + code = metaHandleNormalTableDropImpl(pMeta, ¶m); + metaULock(pMeta); + if (code) { + metaErr(TD_VID(pMeta->pVnode), code); + metaFetchEntryFree(&pOldEntry); + return code; + } + + // update other stuff + pMeta->pVnode->config.vndStats.numOfNTables--; + pMeta->pVnode->config.vndStats.numOfNTimeSeries -= (pOldEntry->ntbEntry.schemaRow.nCols - 1); + +#if 0 + if (tbUids) { + if (taosArrayPush(tbUids, &uid) == NULL) { + rc = terrno; + goto _exit; + } + } +#endif + + if (!TSDB_CACHE_NO(pMeta->pVnode->config)) { + int32_t ret = tsdbCacheDropTable(pMeta->pVnode->pTsdb, pOldEntry->uid, 0, NULL); + if (ret < 0) { + metaErr(TD_VID(pMeta->pVnode), ret); + } + } + + pMeta->changed = true; + metaFetchEntryFree(&pOldEntry); + return code; +} + +static int32_t metaHandleChildTableDropImpl(SMeta *pMeta, const SMetaHandleParam *pParam) { int32_t code = TSDB_CODE_SUCCESS; - // TODO + + const SMetaEntry *pEntry = pParam->pEntry; + const SMetaEntry *pChild = pParam->pOldEntry; + const SMetaEntry *pSuper = pParam->pSuperEntry; + + SMetaTableOp ops[] = { + {META_ENTRY_TABLE, META_TABLE_OP_DELETE}, // + {META_UID_IDX, META_TABLE_OP_DELETE}, // + {META_NAME_IDX, META_TABLE_OP_DELETE}, // + {META_CHILD_IDX, META_TABLE_OP_DELETE}, // + {META_TAG_IDX, META_TABLE_OP_DELETE}, // + {META_BTIME_IDX, META_TABLE_OP_DELETE}, // + {META_TTL_IDX, META_TABLE_OP_DELETE}, // + }; + + for (int i = 0; i < sizeof(ops) / sizeof(ops[0]); i++) { + SMetaTableOp *op = &ops[i]; + + code = metaTableOpFn[op->table][op->op](pMeta, pParam); + if (code) { + metaErr(TD_VID(pMeta->pVnode), code); + return code; + } + } + + --pMeta->pVnode->config.vndStats.numOfCTables; + metaUpdateStbStats(pMeta, pParam->pSuperEntry->uid, -1, 0); + int32_t ret = metaUidCacheClear(pMeta, pSuper->uid); + if (ret < 0) { + metaErr(TD_VID(pMeta->pVnode), ret); + } + + ret = metaTbGroupCacheClear(pMeta, pSuper->uid); + if (ret < 0) { + metaErr(TD_VID(pMeta->pVnode), ret); + } + return code; +} + +static int32_t metaHandleChildTableDrop(SMeta *pMeta, const SMetaEntry *pEntry) { + int32_t code = TSDB_CODE_SUCCESS; + SMetaEntry *pChild = NULL; + SMetaEntry *pSuper = NULL; + + // fetch old entry + code = metaFetchEntryByUid(pMeta, pEntry->uid, &pChild); + if (code) { + metaErr(TD_VID(pMeta->pVnode), code); + return code; + } + + // fetch super entry + code = metaFetchEntryByUid(pMeta, pChild->ctbEntry.suid, &pSuper); + if (code) { + metaErr(TD_VID(pMeta->pVnode), code); + metaFetchEntryFree(&pChild); + return code; + } + + SMetaHandleParam param = { + .pEntry = pEntry, + .pOldEntry = pChild, + .pSuperEntry = pSuper, + }; + + // do the drop + metaWLock(pMeta); + code = metaHandleChildTableDropImpl(pMeta, ¶m); + metaULock(pMeta); + if (code) { + metaErr(TD_VID(pMeta->pVnode), code); + metaFetchEntryFree(&pChild); + metaFetchEntryFree(&pSuper); + return code; + } + + // do other stuff + if (!metaTbInFilterCache(pMeta, pSuper->name, 1)) { + int32_t nCols = 0; + SVnodeStats *pStats = &pMeta->pVnode->config.vndStats; + if (metaGetStbStats(pMeta->pVnode, pSuper->uid, NULL, &nCols) == 0) { + pStats->numOfTimeSeries -= nCols - 1; + } + } + + if (!TSDB_CACHE_NO(pMeta->pVnode->config)) { + int32_t ret = tsdbCacheDropTable(pMeta->pVnode->pTsdb, pChild->uid, pSuper->uid, NULL); + if (ret < 0) { + metaErr(TD_VID(pMeta->pVnode), ret); + } + } + +#if 0 + if (tbUids) { + if (taosArrayPush(tbUids, &uid) == NULL) { + rc = terrno; + goto _exit; + } + } + + if ((type == TSDB_CHILD_TABLE) && tbUid) { + *tbUid = uid; + } +#endif + pMeta->changed = true; + metaFetchEntryFree(&pChild); + metaFetchEntryFree(&pSuper); return code; } @@ -984,7 +1181,7 @@ int32_t metaHandleEntry2(SMeta *pMeta, const SMetaEntry *pEntry) { break; } case TSDB_CHILD_TABLE: { - // code = metaHandleChildTableDrop(pMeta, pEntry); + code = metaHandleChildTableDrop(pMeta, pEntry); break; } case TSDB_NORMAL_TABLE: { diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 7aff393b55..9e8e02ff39 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -19,7 +19,7 @@ extern SDmNotifyHandle dmNotifyHdl; int metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSchema *pSchema); -static int metaDelJsonVarFromIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSchema *pSchema); +int metaDelJsonVarFromIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSchema *pSchema); static int metaSaveToTbDb(SMeta *pMeta, const SMetaEntry *pME); static int metaUpdateUidIdx(SMeta *pMeta, const SMetaEntry *pME); static int metaUpdateNameIdx(SMeta *pMeta, const SMetaEntry *pME); diff --git a/source/dnode/vnode/src/meta/metaTable2.c b/source/dnode/vnode/src/meta/metaTable2.c index 5fc38a7292..9c20f8c8ba 100644 --- a/source/dnode/vnode/src/meta/metaTable2.c +++ b/source/dnode/vnode/src/meta/metaTable2.c @@ -70,6 +70,45 @@ static int32_t metaCheckCreateSuperTableReq(SMeta *pMeta, int64_t version, SVCre return TSDB_CODE_SUCCESS; } +static int32_t metaCheckDropTableReq(SMeta *pMeta, int64_t version, SVDropTbReq *pReq) { + int32_t code = TSDB_CODE_SUCCESS; + void *value = NULL; + int32_t valueSize = 0; + SMetaInfo info; + + if (NULL == pReq->name || strlen(pReq->name) == 0) { + metaError("vgId:%d, %s failed at %s:%d since invalid name:%s, version:%" PRId64, TD_VID(pMeta->pVnode), __func__, + __FILE__, __LINE__, pReq->name, version); + return TSDB_CODE_INVALID_MSG; + } + + code = tdbTbGet(pMeta->pNameIdx, pReq->name, strlen(pReq->name) + 1, &value, &valueSize); + if (TSDB_CODE_SUCCESS != code) { + if (pReq->igNotExists) { + metaTrace("vgId:%d, %s success since table %s not found, version:%" PRId64, TD_VID(pMeta->pVnode), __func__, + pReq->name, version); + } else { + metaError("vgId:%d, %s failed at %s:%d since table %s not found, version:%" PRId64, TD_VID(pMeta->pVnode), + __func__, __FILE__, __LINE__, pReq->name, version); + } + return TSDB_CODE_TDB_TABLE_NOT_EXIST; + } + pReq->uid = *(tb_uid_t *)value; + tdbFreeClear(value); + + code = metaGetInfo(pMeta, pReq->uid, &info, NULL); + if (TSDB_CODE_SUCCESS != code) { + metaError("vgId:%d, %s failed at %s:%d since table %s uid %" PRId64 + " not found, this is an internal error, version:%" PRId64, + TD_VID(pMeta->pVnode), __func__, __FILE__, __LINE__, pReq->name, pReq->uid, version); + code = TSDB_CODE_INTERNAL_ERROR; + return code; + } + pReq->suid = info.suid; + + return code; +} + // Create Super Table int32_t metaCreateSuperTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { int32_t code = TSDB_CODE_SUCCESS; @@ -116,6 +155,11 @@ int32_t metaCreateSuperTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq } // Drop Super Table +int32_t metaDropSuperTable() { + int32_t code = TSDB_CODE_SUCCESS; + // TODO + return code; +} // Alter Super Table @@ -387,47 +431,19 @@ static int32_t metaCreateNormalTable(SMeta *pMeta, int64_t version, SVCreateTbRe // Alter Normal Table -static int32_t metaCheckDropTableReq(SMeta *pMeta, int64_t version, SVDropTbReq *pReq) { - int32_t code = TSDB_CODE_SUCCESS; - void *value = NULL; - int32_t valueSize = 0; - SMetaInfo info; - - if (NULL == pReq->name || strlen(pReq->name) == 0) { +int32_t metaCreateTable2(SMeta *pMeta, int64_t version, SVCreateTbReq *pReq, STableMetaRsp **ppRsp) { + int32_t code = TSDB_CODE_SUCCESS; + if (TSDB_CHILD_TABLE == pReq->type) { + code = metaCreateChildTable(pMeta, version, pReq, ppRsp); + } else if (TSDB_NORMAL_TABLE == pReq->type) { + code = metaCreateNormalTable(pMeta, version, pReq, ppRsp); + } else { code = TSDB_CODE_INVALID_MSG; - metaError("vgId:%d, %s failed at %s:%d since invalid name:%s, version:%" PRId64, TD_VID(pMeta->pVnode), __func__, - __FILE__, __LINE__, pReq->name, version); - return code; } - - code = tdbTbGet(pMeta->pNameIdx, pReq->name, strlen(pReq->name) + 1, &value, &valueSize); - if (TSDB_CODE_SUCCESS != code) { - if (pReq->igNotExists) { - metaTrace("vgId:%d, %s success since table %s not found, version:%" PRId64, TD_VID(pMeta->pVnode), __func__, - pReq->name, version); - } else { - metaError("vgId:%d, %s failed at %s:%d since table %s not found, version:%" PRId64, TD_VID(pMeta->pVnode), - __func__, __FILE__, __LINE__, pReq->name, version); - } - return TSDB_CODE_TDB_TABLE_NOT_EXIST; - } - pReq->uid = *(tb_uid_t *)value; - tdbFreeClear(value); - - code = metaGetInfo(pMeta, pReq->uid, &info, NULL); - if (TSDB_CODE_SUCCESS != code) { - metaError("vgId:%d, %s failed at %s:%d since table %s uid %" PRId64 - " not found, this is an internal error, version:%" PRId64, - TD_VID(pMeta->pVnode), __func__, __FILE__, __LINE__, pReq->name, pReq->uid, version); - code = TSDB_CODE_INTERNAL_ERROR; - return code; - } - pReq->suid = info.suid; - - return code; + TAOS_RETURN(code); } -int32_t metaDropTable2(SMeta *pMeta, int64_t version, SVDropTbReq *pReq, SArray *tbUids, tb_uid_t *tbUid) { +int32_t metaDropTable2(SMeta *pMeta, int64_t version, SVDropTbReq *pReq) { int32_t code = TSDB_CODE_SUCCESS; // check request @@ -441,7 +457,7 @@ int32_t metaDropTable2(SMeta *pMeta, int64_t version, SVDropTbReq *pReq, SArray } if (pReq->suid == pReq->uid) { - code = TSDB_CODE_INVALID_MSG; + code = TSDB_CODE_INVALID_PARA; metaError("vgId:%d, %s failed at %s:%d since %s, uid:%" PRId64 " name:%s version:%" PRId64, TD_VID(pMeta->pVnode), __func__, __FILE__, __LINE__, tstrerror(code), pReq->uid, pReq->name, version); TAOS_RETURN(code); @@ -465,18 +481,5 @@ int32_t metaDropTable2(SMeta *pMeta, int64_t version, SVDropTbReq *pReq, SArray metaInfo("vgId:%d, table %s uid %" PRId64 " is dropped, version:%" PRId64, TD_VID(pMeta->pVnode), pReq->name, pReq->uid, version); } - - TAOS_RETURN(code); -} - -int32_t metaCreateTable2(SMeta *pMeta, int64_t version, SVCreateTbReq *pReq, STableMetaRsp **ppRsp) { - int32_t code = TSDB_CODE_SUCCESS; - if (TSDB_CHILD_TABLE == pReq->type) { - code = metaCreateChildTable(pMeta, version, pReq, ppRsp); - } else if (TSDB_NORMAL_TABLE == pReq->type) { - code = metaCreateNormalTable(pMeta, version, pReq, ppRsp); - } else { - code = TSDB_CODE_INVALID_MSG; - } TAOS_RETURN(code); } diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 50e88d0f0b..d0ad765f7a 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -1490,7 +1490,7 @@ static int32_t vnodeProcessDropTbReq(SVnode *pVnode, int64_t ver, void *pReq, in tb_uid_t tbUid = 0; /* code */ - ret = metaDropTable(pVnode->pMeta, ver, pDropTbReq, tbUids, &tbUid); + ret = metaDropTable2(pVnode->pMeta, ver, pDropTbReq); if (ret < 0) { if (pDropTbReq->igNotExists && terrno == TSDB_CODE_TDB_TABLE_NOT_EXIST) { dropTbRsp.code = TSDB_CODE_SUCCESS; From e853de4d3f7b4da5a2593da0e0de5919c8d69637 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 11 Dec 2024 16:06:56 +0800 Subject: [PATCH 20/45] fix: typo --- source/dnode/vnode/src/meta/metaEntry.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/meta/metaEntry.c b/source/dnode/vnode/src/meta/metaEntry.c index 74a16733f8..48bf77bb9c 100644 --- a/source/dnode/vnode/src/meta/metaEntry.c +++ b/source/dnode/vnode/src/meta/metaEntry.c @@ -246,7 +246,7 @@ void metaCloneEntryFree(SMetaEntry **ppEntry) { } else if (TSDB_CHILD_TABLE == (*ppEntry)->type) { taosMemoryFreeClear((*ppEntry)->ctbEntry.comment); taosMemoryFreeClear((*ppEntry)->ctbEntry.pTags); - } else if (TSDB_CHILD_TABLE == (*ppEntry)->type) { + } else if (TSDB_NORMAL_TABLE == (*ppEntry)->type) { metaCloneSchemaFree(&(*ppEntry)->ntbEntry.schemaRow); taosMemoryFreeClear((*ppEntry)->ntbEntry.comment); } else { From 4922cade4451ad0a96341a9e9e17c5a0810c108c Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 11 Dec 2024 17:04:26 +0800 Subject: [PATCH 21/45] feat: refact drop super table --- source/dnode/vnode/src/meta/metaEntry2.c | 159 +++++++++++++++++++++-- source/dnode/vnode/src/meta/metaTable2.c | 67 +++++++++- 2 files changed, 215 insertions(+), 11 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaEntry2.c b/source/dnode/vnode/src/meta/metaEntry2.c index bbc9a14d02..879693e7b0 100644 --- a/source/dnode/vnode/src/meta/metaEntry2.c +++ b/source/dnode/vnode/src/meta/metaEntry2.c @@ -388,8 +388,9 @@ static int32_t metaSUidIdxInsert(SMeta *pMeta, const SMetaHandleParam *pParam) { } static int32_t metaSUidIdxDelete(SMeta *pMeta, const SMetaHandleParam *pParam) { - const SMetaEntry *pEntry = pParam->pEntry; - int32_t code = tdbTbDelete(pMeta->pSuidIdx, &pEntry->uid, sizeof(pEntry->uid), pMeta->txn); + const SMetaEntry *pEntry = pParam->pOldEntry; + + int32_t code = tdbTbDelete(pMeta->pSuidIdx, &pEntry->uid, sizeof(pEntry->uid), pMeta->txn); if (code) { metaErr(TD_VID(pMeta->pVnode), code); } @@ -1009,7 +1010,7 @@ static int32_t metaHandleNormalTableDrop(SMeta *pMeta, const SMetaEntry *pEntry) return code; } -static int32_t metaHandleChildTableDropImpl(SMeta *pMeta, const SMetaHandleParam *pParam) { +static int32_t metaHandleChildTableDropImpl(SMeta *pMeta, const SMetaHandleParam *pParam, bool superDropped) { int32_t code = TSDB_CODE_SUCCESS; const SMetaEntry *pEntry = pParam->pEntry; @@ -1029,6 +1030,10 @@ static int32_t metaHandleChildTableDropImpl(SMeta *pMeta, const SMetaHandleParam for (int i = 0; i < sizeof(ops) / sizeof(ops[0]); i++) { SMetaTableOp *op = &ops[i]; + if (op->table == META_ENTRY_TABLE && superDropped) { + continue; + } + code = metaTableOpFn[op->table][op->op](pMeta, pParam); if (code) { metaErr(TD_VID(pMeta->pVnode), code); @@ -1050,7 +1055,7 @@ static int32_t metaHandleChildTableDropImpl(SMeta *pMeta, const SMetaHandleParam return code; } -static int32_t metaHandleChildTableDrop(SMeta *pMeta, const SMetaEntry *pEntry) { +static int32_t metaHandleChildTableDrop(SMeta *pMeta, const SMetaEntry *pEntry, bool superDropped) { int32_t code = TSDB_CODE_SUCCESS; SMetaEntry *pChild = NULL; SMetaEntry *pSuper = NULL; @@ -1078,7 +1083,7 @@ static int32_t metaHandleChildTableDrop(SMeta *pMeta, const SMetaEntry *pEntry) // do the drop metaWLock(pMeta); - code = metaHandleChildTableDropImpl(pMeta, ¶m); + code = metaHandleChildTableDropImpl(pMeta, ¶m, superDropped); metaULock(pMeta); if (code) { metaErr(TD_VID(pMeta->pVnode), code); @@ -1121,9 +1126,145 @@ static int32_t metaHandleChildTableDrop(SMeta *pMeta, const SMetaEntry *pEntry) return code; } -static int32_t metaHandleSuperTableDrop(SMeta *pMeta, const SMetaEntry *pEntry) { +static int32_t metaGetChildUidsOfSuperTable(SMeta *pMeta, tb_uid_t suid, SArray **childList) { int32_t code = TSDB_CODE_SUCCESS; - // TODO + void *key = NULL; + int32_t keySize = 0; + int32_t c; + + *childList = taosArrayInit(64, sizeof(tb_uid_t)); + if (*childList == NULL) { + return terrno; + } + + TBC *cursor = NULL; + code = tdbTbcOpen(pMeta->pCtbIdx, &cursor, NULL); + if (code) { + taosArrayDestroy(*childList); + *childList = NULL; + return code; + } + + int32_t rc = tdbTbcMoveTo(cursor, + &(SCtbIdxKey){ + .suid = suid, + .uid = INT64_MIN, + }, + sizeof(SCtbIdxKey), &c); + if (rc < 0) { + tdbTbcClose(cursor); + return 0; + } + + for (;;) { + if (tdbTbcNext(cursor, &key, &keySize, NULL, NULL) < 0) { + break; + } + + if (((SCtbIdxKey *)key)->suid < suid) { + continue; + } else if (((SCtbIdxKey *)key)->suid > suid) { + break; + } + + if (taosArrayPush(*childList, &(((SCtbIdxKey *)key)->uid)) == NULL) { + tdbFreeClear(key); + tdbTbcClose(cursor); + taosArrayDestroy(*childList); + *childList = NULL; + return terrno; + } + } + + tdbTbcClose(cursor); + tdbFreeClear(key); + return code; +} + +static int32_t metaHandleSuperTableDropImpl(SMeta *pMeta, const SMetaHandleParam *pParam) { + int32_t code = TSDB_CODE_SUCCESS; + const SMetaEntry *pEntry = pParam->pEntry; + + SMetaTableOp ops[] = { + {META_ENTRY_TABLE, META_TABLE_OP_DELETE}, // + {META_UID_IDX, META_TABLE_OP_DELETE}, // + {META_NAME_IDX, META_TABLE_OP_DELETE}, // + {META_SUID_IDX, META_TABLE_OP_DELETE}, // + + // {META_SCHEMA_TABLE, META_TABLE_OP_UPDATA}, // TODO: here should be insert + }; + + for (int i = 0; i < sizeof(ops) / sizeof(ops[0]); i++) { + SMetaTableOp *op = &ops[i]; + + code = metaTableOpFn[op->table][op->op](pMeta, pParam); + if (TSDB_CODE_SUCCESS != code) { + metaErr(TD_VID(pMeta->pVnode), code); + return code; + } + } + + int32_t ret = metaStatsCacheDrop(pMeta, pEntry->uid); + if (ret < 0) { + metaErr(TD_VID(pMeta->pVnode), ret); + } + return code; +} + +static int32_t metaHandleSuperTableDrop(SMeta *pMeta, const SMetaEntry *pEntry) { + int32_t code = TSDB_CODE_SUCCESS; + SArray *childList = NULL; + SMetaEntry *pOldEntry = NULL; + + code = metaFetchEntryByUid(pMeta, pEntry->uid, &pOldEntry); + if (code) { + metaErr(TD_VID(pMeta->pVnode), code); + return code; + } + + code = metaGetChildUidsOfSuperTable(pMeta, pEntry->uid, &childList); + if (code) { + metaErr(TD_VID(pMeta->pVnode), code); + metaFetchEntryFree(&pOldEntry); + return code; + } + + // loop to drop all child tables + for (int32_t i = 0; i < taosArrayGetSize(childList); i++) { + SMetaEntry childEntry = { + .version = pEntry->version, + .uid = *(tb_uid_t *)taosArrayGet(childList, i), + .type = -TSDB_CHILD_TABLE, + }; + + code = metaHandleChildTableDrop(pMeta, &childEntry, true); + if (code) { + metaErr(TD_VID(pMeta->pVnode), code); + } + } + + // do drop super table + SMetaHandleParam param = { + .pEntry = pEntry, + .pOldEntry = pOldEntry, + }; + metaWLock(pMeta); + code = metaHandleSuperTableDropImpl(pMeta, ¶m); + metaULock(pMeta); + if (code) { + metaErr(TD_VID(pMeta->pVnode), code); + taosArrayDestroy(childList); + metaFetchEntryFree(&pOldEntry); + return code; + } + + // do other stuff + metaUpdTimeSeriesNum(pMeta); + pMeta->changed = true; + + // free resource and return + taosArrayDestroy(childList); + metaFetchEntryFree(&pOldEntry); return code; } @@ -1177,11 +1318,11 @@ int32_t metaHandleEntry2(SMeta *pMeta, const SMetaEntry *pEntry) { } else { switch (type) { case TSDB_SUPER_TABLE: { - // code = metaHandleSuperTableDrop(pMeta, pEntry); + code = metaHandleSuperTableDrop(pMeta, pEntry); break; } case TSDB_CHILD_TABLE: { - code = metaHandleChildTableDrop(pMeta, pEntry); + code = metaHandleChildTableDrop(pMeta, pEntry, false); break; } case TSDB_NORMAL_TABLE: { diff --git a/source/dnode/vnode/src/meta/metaTable2.c b/source/dnode/vnode/src/meta/metaTable2.c index 9c20f8c8ba..1798a26a1d 100644 --- a/source/dnode/vnode/src/meta/metaTable2.c +++ b/source/dnode/vnode/src/meta/metaTable2.c @@ -109,6 +109,49 @@ static int32_t metaCheckDropTableReq(SMeta *pMeta, int64_t version, SVDropTbReq return code; } +static int32_t metaCheckDropSuperTableReq(SMeta *pMeta, int64_t version, SVDropStbReq *pReq) { + int32_t code = TSDB_CODE_SUCCESS; + void *value = NULL; + int32_t valueSize = 0; + SMetaInfo info; + + if (NULL == pReq->name || strlen(pReq->name) == 0) { + metaError("vgId:%d, %s failed at %s:%d since invalid name:%s, version:%" PRId64, TD_VID(pMeta->pVnode), __func__, + __FILE__, __LINE__, pReq->name, version); + return TSDB_CODE_INVALID_MSG; + } + + code = tdbTbGet(pMeta->pNameIdx, pReq->name, strlen(pReq->name) + 1, &value, &valueSize); + if (code) { + metaError("vgId:%d, %s failed at %s:%d since table %s not found, version:%" PRId64, TD_VID(pMeta->pVnode), __func__, + __FILE__, __LINE__, pReq->name, version); + return TSDB_CODE_TDB_STB_NOT_EXIST; + } else { + int64_t uid = *(int64_t *)value; + tdbFreeClear(value); + + if (uid != pReq->suid) { + metaError("vgId:%d, %s failed at %s:%d since table %s uid:%" PRId64 " not match, version:%" PRId64, + TD_VID(pMeta->pVnode), __func__, __FILE__, __LINE__, pReq->name, pReq->suid, version); + return TSDB_CODE_TDB_STB_NOT_EXIST; + } + } + + code = metaGetInfo(pMeta, pReq->suid, &info, NULL); + if (code) { + metaError("vgId:%d, %s failed at %s:%d since table %s uid %" PRId64 + " not found, this is an internal error, version:%" PRId64, + TD_VID(pMeta->pVnode), __func__, __FILE__, __LINE__, pReq->name, pReq->suid, version); + return TSDB_CODE_INTERNAL_ERROR; + } + if (info.suid != info.uid) { + metaError("vgId:%d, %s failed at %s:%d since table %s uid %" PRId64 " is not a super table, version:%" PRId64, + TD_VID(pMeta->pVnode), __func__, __FILE__, __LINE__, pReq->name, pReq->suid, version); + return TSDB_CODE_INVALID_MSG; + } + return code; +} + // Create Super Table int32_t metaCreateSuperTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { int32_t code = TSDB_CODE_SUCCESS; @@ -155,9 +198,29 @@ int32_t metaCreateSuperTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq } // Drop Super Table -int32_t metaDropSuperTable() { +int32_t metaDropSuperTable(SMeta *pMeta, int64_t verison, SVDropStbReq *pReq) { int32_t code = TSDB_CODE_SUCCESS; - // TODO + + // check request + code = metaCheckDropSuperTableReq(pMeta, verison, pReq); + if (code) { + TAOS_RETURN(code); + } + + // handle entry + SMetaEntry entry = { + .version = verison, + .type = -TSDB_SUPER_TABLE, + .uid = pReq->suid, + }; + code = metaHandleEntry2(pMeta, &entry); + if (code) { + metaError("vgId:%d, failed to drop stb:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, pReq->suid, + tstrerror(code)); + } else { + metaInfo("vgId:%d, super table %s uid:%" PRId64 " is dropped, version:%" PRId64, TD_VID(pMeta->pVnode), pReq->name, + pReq->suid, verison); + } return code; } From 982c0665f53acf60a0ace08924749efbda0e218e Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 11 Dec 2024 18:47:47 +0800 Subject: [PATCH 22/45] more code --- source/dnode/vnode/src/meta/metaTable.c | 10 +- source/dnode/vnode/src/meta/metaTable2.c | 352 +++++++++++++++++++++++ 2 files changed, 360 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 9e8e02ff39..0b4c19a687 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -17,9 +17,15 @@ extern SDmNotifyHandle dmNotifyHdl; -int metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSchema *pSchema); +int32_t metaAddTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq, STableMetaRsp *pRsp); +int32_t metaDropTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq, STableMetaRsp *pRsp); +int32_t metaAlterTableColumnName(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq, STableMetaRsp *pRsp); +int32_t metaAlterTableColumnBytes(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq, STableMetaRsp *pRsp); +int32_t metaAlterTableColumnCompressOption(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq, STableMetaRsp *pRsp); -int metaDelJsonVarFromIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSchema *pSchema); +int32_t metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSchema *pSchema); + +int32_t metaDelJsonVarFromIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSchema *pSchema); static int metaSaveToTbDb(SMeta *pMeta, const SMetaEntry *pME); static int metaUpdateUidIdx(SMeta *pMeta, const SMetaEntry *pME); static int metaUpdateNameIdx(SMeta *pMeta, const SMetaEntry *pME); diff --git a/source/dnode/vnode/src/meta/metaTable2.c b/source/dnode/vnode/src/meta/metaTable2.c index 1798a26a1d..8c936d1b44 100644 --- a/source/dnode/vnode/src/meta/metaTable2.c +++ b/source/dnode/vnode/src/meta/metaTable2.c @@ -546,3 +546,355 @@ int32_t metaDropTable2(SMeta *pMeta, int64_t version, SVDropTbReq *pReq) { } TAOS_RETURN(code); } + +int32_t metaAddTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq, STableMetaRsp *pRsp) { + int32_t code = TSDB_CODE_SUCCESS; + + // check request + if (NULL == pReq->colName || strlen(pReq->colName) == 0) { + metaError("vgId:%d, %s failed at %s:%d since invalid column name:%s, version:%" PRId64, TD_VID(pMeta->pVnode), + __func__, __FILE__, __LINE__, pReq->colName, version); + TAOS_RETURN(TSDB_CODE_INVALID_MSG); + } + + void *value = NULL; + int32_t valueSize = 0; + code = tdbTbGet(pMeta->pNameIdx, pReq->tbName, strlen(pReq->tbName) + 1, &value, &valueSize); + if (code) { + metaError("vgId:%d, %s failed at %s:%d since table %s not found, version:%" PRId64, TD_VID(pMeta->pVnode), __func__, + __FILE__, __LINE__, pReq->tbName, version); + code = TSDB_CODE_TDB_TABLE_NOT_EXIST; + TAOS_RETURN(code); + } + + int64_t uid = *(int64_t *)value; + tdbFreeClear(value); + + // TODO + return code; + +#if 0 + void *pVal = NULL; + int nVal = 0; + const void *pData = NULL; + int nData = 0; + int ret = 0; + tb_uid_t uid; + int64_t oversion; + SSchema *pColumn = NULL; + SMetaEntry entry = {0}; + SSchemaWrapper *pSchema; + int c; + bool freeColCmpr = false; + + // search uid index + TBC *pUidIdxc = NULL; + + TAOS_CHECK_RETURN(tdbTbcOpen(pMeta->pUidIdx, &pUidIdxc, NULL)); + ret = tdbTbcMoveTo(pUidIdxc, &uid, sizeof(uid), &c); + if (c != 0) { + tdbTbcClose(pUidIdxc); + metaError("meta/table: invalide c: %" PRId32 " alt tb column failed.", c); + return TSDB_CODE_FAILED; + } + + ret = tdbTbcGet(pUidIdxc, NULL, NULL, &pData, &nData); + oversion = ((SUidIdxVal *)pData)[0].version; + + // search table.db + TBC *pTbDbc = NULL; + + TAOS_CHECK_RETURN(tdbTbcOpen(pMeta->pTbDb, &pTbDbc, NULL)); + ret = tdbTbcMoveTo(pTbDbc, &((STbDbKey){.uid = uid, .version = oversion}), sizeof(STbDbKey), &c); + if (c != 0) { + tdbTbcClose(pUidIdxc); + tdbTbcClose(pTbDbc); + metaError("meta/table: invalide c: %" PRId32 " alt tb column failed.", c); + return TSDB_CODE_FAILED; + } + + ret = tdbTbcGet(pTbDbc, NULL, NULL, &pData, &nData); + + // get table entry + SDecoder dc = {0}; + if ((entry.pBuf = taosMemoryMalloc(nData)) == NULL) { + tdbTbcClose(pUidIdxc); + tdbTbcClose(pTbDbc); + return terrno; + } + memcpy(entry.pBuf, pData, nData); + tDecoderInit(&dc, entry.pBuf, nData); + ret = metaDecodeEntry(&dc, &entry); + if (ret != 0) { + tdbTbcClose(pUidIdxc); + tdbTbcClose(pTbDbc); + tDecoderClear(&dc); + metaError("meta/table: invalide ret: %" PRId32 " alt tb column failed.", ret); + return ret; + } + + if (entry.type != TSDB_NORMAL_TABLE) { + terrno = TSDB_CODE_VND_INVALID_TABLE_ACTION; + goto _err; + } + // search the column to add/drop/update + pSchema = &entry.ntbEntry.schemaRow; + + // save old entry + SMetaEntry oldEntry = {.type = TSDB_NORMAL_TABLE, .uid = entry.uid}; + oldEntry.ntbEntry.schemaRow.nCols = pSchema->nCols; + + int32_t rowLen = -1; + if (pAlterTbReq->action == TSDB_ALTER_TABLE_ADD_COLUMN || + pAlterTbReq->action == TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES) { + rowLen = 0; + } + + int32_t iCol = 0, jCol = 0; + SSchema *qColumn = NULL; + for (;;) { + qColumn = NULL; + + if (jCol >= pSchema->nCols) break; + qColumn = &pSchema->pSchema[jCol]; + + if (!pColumn && (strcmp(qColumn->name, pAlterTbReq->colName) == 0)) { + pColumn = qColumn; + iCol = jCol; + if (rowLen < 0) break; + } + rowLen += qColumn->bytes; + ++jCol; + } + + entry.version = version; + int tlen; + SSchema *pNewSchema = NULL; + SSchema tScheam; + switch (pAlterTbReq->action) { + case TSDB_ALTER_TABLE_ADD_COLUMN: + case TSDB_ALTER_TABLE_ADD_COLUMN_WITH_COMPRESS_OPTION: + if (pColumn) { + terrno = TSDB_CODE_VND_COL_ALREADY_EXISTS; + goto _err; + } + if ((terrno = grantCheck(TSDB_GRANT_TIMESERIES)) < 0) { + goto _err; + } + if (rowLen + pAlterTbReq->bytes > TSDB_MAX_BYTES_PER_ROW) { + terrno = TSDB_CODE_PAR_INVALID_ROW_LENGTH; + goto _err; + } + pSchema->version++; + pSchema->nCols++; + pNewSchema = taosMemoryMalloc(sizeof(SSchema) * pSchema->nCols); + if (pNewSchema == NULL) { + goto _err; + } + memcpy(pNewSchema, pSchema->pSchema, sizeof(SSchema) * (pSchema->nCols - 1)); + pSchema->pSchema = pNewSchema; + pSchema->pSchema[entry.ntbEntry.schemaRow.nCols - 1].bytes = pAlterTbReq->bytes; + pSchema->pSchema[entry.ntbEntry.schemaRow.nCols - 1].type = pAlterTbReq->type; + pSchema->pSchema[entry.ntbEntry.schemaRow.nCols - 1].flags = pAlterTbReq->flags; + pSchema->pSchema[entry.ntbEntry.schemaRow.nCols - 1].colId = entry.ntbEntry.ncid++; + strcpy(pSchema->pSchema[entry.ntbEntry.schemaRow.nCols - 1].name, pAlterTbReq->colName); + + ++pMeta->pVnode->config.vndStats.numOfNTimeSeries; + metaTimeSeriesNotifyCheck(pMeta); + + if (!TSDB_CACHE_NO(pMeta->pVnode->config)) { + int16_t cid = pSchema->pSchema[entry.ntbEntry.schemaRow.nCols - 1].colId; + int8_t col_type = pSchema->pSchema[entry.ntbEntry.schemaRow.nCols - 1].type; + int32_t ret = tsdbCacheNewNTableColumn(pMeta->pVnode->pTsdb, entry.uid, cid, col_type); + if (ret < 0) { + terrno = ret; + goto _err; + } + } + SSchema *pCol = &pSchema->pSchema[entry.ntbEntry.schemaRow.nCols - 1]; + uint32_t compress = pAlterTbReq->action == TSDB_ALTER_TABLE_ADD_COLUMN ? createDefaultColCmprByType(pCol->type) + : pAlterTbReq->compress; + if (updataTableColCmpr(&entry.colCmpr, pCol, 1, compress) != 0) { + metaError("vgId:%d, failed to update table col cmpr:%s uid:%" PRId64, TD_VID(pMeta->pVnode), entry.name, + entry.uid); + } + freeColCmpr = true; + if (entry.colCmpr.nCols != pSchema->nCols) { + if (pNewSchema) taosMemoryFree(pNewSchema); + if (freeColCmpr) taosMemoryFree(entry.colCmpr.pColCmpr); + terrno = TSDB_CODE_VND_INVALID_TABLE_ACTION; + goto _err; + } + break; + case TSDB_ALTER_TABLE_DROP_COLUMN: + if (pColumn == NULL) { + terrno = TSDB_CODE_VND_COL_NOT_EXISTS; + goto _err; + } + if (pColumn->colId == 0) { + terrno = TSDB_CODE_VND_INVALID_TABLE_ACTION; + goto _err; + } + if (tqCheckColModifiable(pMeta->pVnode->pTq, uid, pColumn->colId) != 0) { + terrno = TSDB_CODE_VND_COL_SUBSCRIBED; + goto _err; + } + bool hasPrimayKey = false; + if (pSchema->nCols >= 2) { + hasPrimayKey = pSchema->pSchema[1].flags & COL_IS_KEY ? true : false; + } + + memcpy(&tScheam, pColumn, sizeof(SSchema)); + pSchema->version++; + tlen = (pSchema->nCols - iCol - 1) * sizeof(SSchema); + if (tlen) { + memmove(pColumn, pColumn + 1, tlen); + } + pSchema->nCols--; + + --pMeta->pVnode->config.vndStats.numOfNTimeSeries; + + if (!TSDB_CACHE_NO(pMeta->pVnode->config)) { + int16_t cid = pColumn->colId; + + if (tsdbCacheDropNTableColumn(pMeta->pVnode->pTsdb, entry.uid, cid, hasPrimayKey) != 0) { + metaError("vgId:%d, failed to drop ntable column:%s uid:%" PRId64, TD_VID(pMeta->pVnode), entry.name, + entry.uid); + } + } + + if (updataTableColCmpr(&entry.colCmpr, &tScheam, 0, 0) != 0) { + metaError("vgId:%d, failed to update table col cmpr:%s uid:%" PRId64, TD_VID(pMeta->pVnode), entry.name, + entry.uid); + } + if (entry.colCmpr.nCols != pSchema->nCols) { + terrno = TSDB_CODE_VND_INVALID_TABLE_ACTION; + goto _err; + } + break; + case TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES: + if (pColumn == NULL) { + terrno = TSDB_CODE_VND_COL_NOT_EXISTS; + goto _err; + } + if (!IS_VAR_DATA_TYPE(pColumn->type) || pColumn->bytes >= pAlterTbReq->colModBytes) { + terrno = TSDB_CODE_VND_INVALID_TABLE_ACTION; + goto _err; + } + if (rowLen + pAlterTbReq->colModBytes - pColumn->bytes > TSDB_MAX_BYTES_PER_ROW) { + terrno = TSDB_CODE_PAR_INVALID_ROW_LENGTH; + goto _err; + } + if (tqCheckColModifiable(pMeta->pVnode->pTq, uid, pColumn->colId) != 0) { + terrno = TSDB_CODE_VND_COL_SUBSCRIBED; + goto _err; + } + pSchema->version++; + pColumn->bytes = pAlterTbReq->colModBytes; + break; + case TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME: + if (pAlterTbReq->colNewName == NULL) { + terrno = TSDB_CODE_INVALID_MSG; + goto _err; + } + if (pColumn == NULL) { + terrno = TSDB_CODE_VND_COL_NOT_EXISTS; + goto _err; + } + if (tqCheckColModifiable(pMeta->pVnode->pTq, uid, pColumn->colId) != 0) { + terrno = TSDB_CODE_VND_COL_SUBSCRIBED; + goto _err; + } + pSchema->version++; + strcpy(pColumn->name, pAlterTbReq->colNewName); + break; + } + + if (!TSDB_CACHE_NO(pMeta->pVnode->config)) { + tsdbCacheInvalidateSchema(pMeta->pVnode->pTsdb, 0, entry.uid, pSchema->version); + } + + entry.version = version; + + // do actual write + metaWLock(pMeta); + + if (metaDeleteNcolIdx(pMeta, &oldEntry) < 0) { + metaError("vgId:%d, failed to delete ncol idx:%s uid:%" PRId64, TD_VID(pMeta->pVnode), entry.name, entry.uid); + } + + if (metaUpdateNcolIdx(pMeta, &entry) < 0) { + metaError("vgId:%d, failed to update ncol idx:%s uid:%" PRId64, TD_VID(pMeta->pVnode), entry.name, entry.uid); + } + + // save to table db + if (metaSaveToTbDb(pMeta, &entry) < 0) { + metaError("vgId:%d, failed to save to tb db:%s uid:%" PRId64, TD_VID(pMeta->pVnode), entry.name, entry.uid); + } + + if (metaUpdateUidIdx(pMeta, &entry) < 0) { + metaError("vgId:%d, failed to update uid idx:%s uid:%" PRId64, TD_VID(pMeta->pVnode), entry.name, entry.uid); + } + + if (metaSaveToSkmDb(pMeta, &entry) < 0) { + metaError("vgId:%d, failed to save to skm db:%s uid:%" PRId64, TD_VID(pMeta->pVnode), entry.name, entry.uid); + } + + if (metaUpdateChangeTime(pMeta, entry.uid, pAlterTbReq->ctimeMs) < 0) { + metaError("vgId:%d, failed to update change time:%s uid:%" PRId64, TD_VID(pMeta->pVnode), entry.name, entry.uid); + } + + metaULock(pMeta); + + if (metaUpdateMetaRsp(uid, pAlterTbReq->tbName, pSchema, pMetaRsp) < 0) { + metaError("vgId:%d, failed to update meta rsp:%s uid:%" PRId64, TD_VID(pMeta->pVnode), entry.name, entry.uid); + } + for (int32_t i = 0; i < entry.colCmpr.nCols; i++) { + SColCmpr *p = &entry.colCmpr.pColCmpr[i]; + pMetaRsp->pSchemaExt[i].colId = p->id; + pMetaRsp->pSchemaExt[i].compress = p->alg; + } + + if (entry.pBuf) taosMemoryFree(entry.pBuf); + if (pNewSchema) taosMemoryFree(pNewSchema); + if (freeColCmpr) taosMemoryFree(entry.colCmpr.pColCmpr); + + tdbTbcClose(pTbDbc); + tdbTbcClose(pUidIdxc); + tDecoderClear(&dc); + + return 0; + +_err: + if (entry.pBuf) taosMemoryFree(entry.pBuf); + tdbTbcClose(pTbDbc); + tdbTbcClose(pUidIdxc); + tDecoderClear(&dc); + + return terrno != 0 ? terrno : TSDB_CODE_FAILED; +#endif +} + +int32_t metaDropTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq, STableMetaRsp *pRsp) { + int32_t code = TSDB_CODE_SUCCESS; + // TODO + return code; +} + +int32_t metaAlterTableColumnName(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq, STableMetaRsp *pRsp) { + int32_t code = TSDB_CODE_SUCCESS; + // TODO + return code; +} + +int32_t metaAlterTableColumnBytes(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq, STableMetaRsp *pRsp) { + int32_t code = TSDB_CODE_SUCCESS; + // TODO + return code; +} + +int32_t metaAlterTableColumnCompressOption(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq, STableMetaRsp *pRsp) { + int32_t code = TSDB_CODE_SUCCESS; + // TODO + return code; +} \ No newline at end of file From 2763c81f4546fa26ae226f60ae8f7c98582df654 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 11 Dec 2024 21:15:58 +0800 Subject: [PATCH 23/45] fix: memory problem --- source/dnode/vnode/src/meta/metaEntry.c | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaEntry.c b/source/dnode/vnode/src/meta/metaEntry.c index 48bf77bb9c..4ddd7c991d 100644 --- a/source/dnode/vnode/src/meta/metaEntry.c +++ b/source/dnode/vnode/src/meta/metaEntry.c @@ -66,13 +66,15 @@ static FORCE_INLINE int32_t metatInitDefaultSColCmprWrapper(SDecoder *pDecoder, } static int32_t metaCloneColCmpr(const SColCmprWrapper *pSrc, SColCmprWrapper *pDst) { - pDst->nCols = pSrc->nCols; - pDst->version = pSrc->version; - pDst->pColCmpr = (SColCmpr *)taosMemoryCalloc(pSrc->nCols, sizeof(SColCmpr)); - if (NULL == pDst->pColCmpr) { - return terrno; + if (pSrc->nCols > 0) { + pDst->nCols = pSrc->nCols; + pDst->version = pSrc->version; + pDst->pColCmpr = (SColCmpr *)taosMemoryCalloc(pSrc->nCols, sizeof(SColCmpr)); + if (NULL == pDst->pColCmpr) { + return terrno; + } + memcpy(pDst->pColCmpr, pSrc->pColCmpr, pSrc->nCols * sizeof(SColCmpr)); } - memcpy(pDst->pColCmpr, pSrc->pColCmpr, pSrc->nCols * sizeof(SColCmpr)); return 0; } From 3ede729a7c859bcb0aa4a73b6532299fa07370ab Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 12 Dec 2024 08:42:48 +0800 Subject: [PATCH 24/45] feat: replace drop super table api --- source/dnode/vnode/src/inc/vnodeInt.h | 1 + source/dnode/vnode/src/meta/metaTable2.c | 2 +- source/dnode/vnode/src/vnd/vnodeSvr.c | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 8152f0e4fd..acb8de3780 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -159,6 +159,7 @@ int metaCreateSTable(SMeta* pMeta, int64_t version, SVCreateStbReq* int metaCreateSuperTable(SMeta* pMeta, int64_t version, SVCreateStbReq* pReq); int metaAlterSTable(SMeta* pMeta, int64_t version, SVCreateStbReq* pReq); int metaDropSTable(SMeta* pMeta, int64_t verison, SVDropStbReq* pReq, SArray* tbUidList); +int32_t metaDropSuperTable(SMeta* pMeta, int64_t verison, SVDropStbReq* pReq); int metaCreateTable(SMeta* pMeta, int64_t version, SVCreateTbReq* pReq, STableMetaRsp** pMetaRsp); int32_t metaCreateTable2(SMeta* pMeta, int64_t version, SVCreateTbReq* pReq, STableMetaRsp** ppRsp); int metaDropTable(SMeta* pMeta, int64_t version, SVDropTbReq* pReq, SArray* tbUids, int64_t* tbUid); diff --git a/source/dnode/vnode/src/meta/metaTable2.c b/source/dnode/vnode/src/meta/metaTable2.c index 8c936d1b44..2e6daf016b 100644 --- a/source/dnode/vnode/src/meta/metaTable2.c +++ b/source/dnode/vnode/src/meta/metaTable2.c @@ -221,7 +221,7 @@ int32_t metaDropSuperTable(SMeta *pMeta, int64_t verison, SVDropStbReq *pReq) { metaInfo("vgId:%d, super table %s uid:%" PRId64 " is dropped, version:%" PRId64, TD_VID(pMeta->pVnode), pReq->name, pReq->suid, verison); } - return code; + TAOS_RETURN(code); } // Alter Super Table diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index d0ad765f7a..fccd25822e 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -1376,7 +1376,7 @@ static int32_t vnodeProcessDropStbReq(SVnode *pVnode, int64_t ver, void *pReq, i // process request tbUidList = taosArrayInit(8, sizeof(int64_t)); if (tbUidList == NULL) goto _exit; - if (metaDropSTable(pVnode->pMeta, ver, &req, tbUidList) < 0) { + if (metaDropSuperTable(pVnode->pMeta, ver, &req) < 0) { rcode = terrno; goto _exit; } From 01ceb2e90b4bdee62f9b24ce8bc64e2a923f19db Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 12 Dec 2024 11:27:34 +0800 Subject: [PATCH 25/45] more code --- source/dnode/vnode/src/meta/metaEntry2.c | 109 ++++++- source/dnode/vnode/src/meta/metaTable.c | 3 +- source/dnode/vnode/src/meta/metaTable2.c | 377 ++++++----------------- 3 files changed, 206 insertions(+), 283 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaEntry2.c b/source/dnode/vnode/src/meta/metaEntry2.c index 879693e7b0..384dd5abbd 100644 --- a/source/dnode/vnode/src/meta/metaEntry2.c +++ b/source/dnode/vnode/src/meta/metaEntry2.c @@ -15,6 +15,7 @@ void metaCloneEntryFree(SMetaEntry **ppEntry); void metaDestroyTagIdxKey(STagIdxKey *pTagIdxKey); int metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSchema *pSchema); int metaDelJsonVarFromIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSchema *pSchema); +void metaTimeSeriesNotifyCheck(SMeta *pMeta); #define metaErr(VGID, ERRNO) \ do { \ @@ -54,7 +55,7 @@ typedef struct { EMetaTableOp op; } SMetaTableOp; -static int32_t metaFetchEntryByUid(SMeta *pMeta, int64_t uid, SMetaEntry **ppEntry) { +int32_t metaFetchEntryByUid(SMeta *pMeta, int64_t uid, SMetaEntry **ppEntry) { int32_t code = TSDB_CODE_SUCCESS; void *value = NULL; int32_t valueSize = 0; @@ -129,7 +130,7 @@ static int32_t metaFetchEntryByName(SMeta *pMeta, const char *name, SMetaEntry * return code; } -static void metaFetchEntryFree(SMetaEntry **ppEntry) { metaCloneEntryFree(ppEntry); } +void metaFetchEntryFree(SMetaEntry **ppEntry) { metaCloneEntryFree(ppEntry); } // Entry Table static int32_t metaEntryTableUpsert(SMeta *pMeta, const SMetaHandleParam *pParam, EMetaTableOp op) { @@ -262,7 +263,38 @@ static int32_t metaSchemaTableInsert(SMeta *pMeta, const SMetaHandleParam *pPara } static int32_t metaSchemaTableUpdate(SMeta *pMeta, const SMetaHandleParam *pParam) { - return metaSchemaTableUpsert(pMeta, pParam, META_TABLE_OP_UPDATA); + int32_t code = TSDB_CODE_SUCCESS; + + const SMetaEntry *pEntry = pParam->pEntry; + const SMetaEntry *pOldEntry = pParam->pOldEntry; + + if (NULL == pOldEntry) { + return metaSchemaTableUpsert(pMeta, pParam, META_TABLE_OP_UPDATA); + } + + if (pEntry->type == TSDB_NORMAL_TABLE && + pOldEntry->ntbEntry.schemaRow.version != pEntry->ntbEntry.schemaRow.version) { + code = metaSchemaTableUpsert(pMeta, pParam, META_TABLE_OP_UPDATA); + if (code) { + metaErr(TD_VID(pMeta->pVnode), code); + return code; + } + + if (pOldEntry->ntbEntry.schemaRow.nCols != pEntry->ntbEntry.schemaRow.nCols) { + pMeta->pVnode->config.vndStats.numOfNTimeSeries += + (pEntry->ntbEntry.schemaRow.nCols - pOldEntry->ntbEntry.schemaRow.nCols); + } + } + + if (pEntry->type == TSDB_SUPER_TABLE && pOldEntry->stbEntry.schemaRow.version != pEntry->stbEntry.schemaRow.version) { + return metaSchemaTableUpsert(pMeta, pParam, META_TABLE_OP_UPDATA); + } + + if (pParam->pEntry->type == TSDB_CHILD_TABLE) { + return TSDB_CODE_INVALID_MSG; + } + + return 0; } static int32_t metaSchemaTableDelete(SMeta *pMeta, const SMetaHandleParam *pEntry) { @@ -1211,6 +1243,75 @@ static int32_t metaHandleSuperTableDropImpl(SMeta *pMeta, const SMetaHandleParam return code; } +static int32_t metaHandleNormalTableUpdateImpl(SMeta *pMeta, const SMetaHandleParam *pParam) { + int32_t code = TSDB_CODE_SUCCESS; + + const SMetaEntry *pEntry = pParam->pEntry; + + SMetaTableOp ops[] = { + {META_ENTRY_TABLE, META_TABLE_OP_UPDATA}, // + {META_SCHEMA_TABLE, META_TABLE_OP_UPDATA}, // + {META_UID_IDX, META_TABLE_OP_UPDATA}, // + }; + for (int32_t i = 0; i < sizeof(ops) / sizeof(ops[0]); i++) { + SMetaTableOp *op = &ops[i]; + code = metaTableOpFn[op->table][op->op](pMeta, pParam); + if (code) { + metaErr(TD_VID(pMeta->pVnode), code); + return code; + } + } +#if 0 + if (metaUpdateChangeTime(pMeta, entry.uid, pAlterTbReq->ctimeMs) < 0) { + metaError("vgId:%d, failed to update change time:%s uid:%" PRId64, TD_VID(pMeta->pVnode), entry.name, entry.uid); + } +#endif + return code; +} + +static int32_t metaHandleNormalTableUpdate(SMeta *pMeta, const SMetaEntry *pEntry) { + int32_t code = TSDB_CODE_SUCCESS; + SMetaEntry *pOldEntry = NULL; + + // fetch old entry + code = metaFetchEntryByUid(pMeta, pEntry->uid, &pOldEntry); + if (code) { + metaErr(TD_VID(pMeta->pVnode), code); + return code; + } + + // handle update + SMetaHandleParam param = { + .pEntry = pEntry, + .pOldEntry = pOldEntry, + }; + metaWLock(pMeta); + code = metaHandleNormalTableUpdateImpl(pMeta, ¶m); + metaULock(pMeta); + if (code) { + metaErr(TD_VID(pMeta->pVnode), code); + metaFetchEntryFree(&pOldEntry); + return code; + } + + // do other stuff + if (!TSDB_CACHE_NO(pMeta->pVnode->config) && + pEntry->ntbEntry.schemaRow.version != pOldEntry->ntbEntry.schemaRow.version) { +#if 0 + int16_t cid = pSchema->pSchema[entry.ntbEntry.schemaRow.nCols - 1].colId; + int8_t col_type = pSchema->pSchema[entry.ntbEntry.schemaRow.nCols - 1].type; + int32_t ret = tsdbCacheNewNTableColumn(pMeta->pVnode->pTsdb, entry.uid, cid, col_type); + if (ret < 0) { + terrno = ret; + goto _err; + } +#endif + tsdbCacheInvalidateSchema(pMeta->pVnode->pTsdb, 0, pEntry->uid, pEntry->ntbEntry.schemaRow.version); + } + metaTimeSeriesNotifyCheck(pMeta); + return code; +} + static int32_t metaHandleSuperTableDrop(SMeta *pMeta, const SMetaEntry *pEntry) { int32_t code = TSDB_CODE_SUCCESS; SArray *childList = NULL; @@ -1304,7 +1405,7 @@ int32_t metaHandleEntry2(SMeta *pMeta, const SMetaEntry *pEntry) { } case TSDB_NORMAL_TABLE: { if (isExist) { - // code = metaHandleNormalTableUpdate(pMeta, pEntry); + code = metaHandleNormalTableUpdate(pMeta, pEntry); } else { code = metaHandleNormalTableCreate(pMeta, pEntry); } diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index fa152fef6e..81936ed2a4 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -286,7 +286,7 @@ _exception: return code; } -static inline void metaTimeSeriesNotifyCheck(SMeta *pMeta) { +void metaTimeSeriesNotifyCheck(SMeta *pMeta) { #if defined(TD_ENTERPRISE) int64_t nTimeSeries = metaGetTimeSeriesNum(pMeta, 0); int64_t deltaTS = nTimeSeries - pMeta->pVnode->config.vndStats.numOfReportedTimeSeries; @@ -2988,6 +2988,7 @@ int metaAlterTable(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq, STableMeta pMeta->changed = true; switch (pReq->action) { case TSDB_ALTER_TABLE_ADD_COLUMN: + return metaAddTableColumn(pMeta, version, pReq, pMetaRsp); case TSDB_ALTER_TABLE_ADD_COLUMN_WITH_COMPRESS_OPTION: case TSDB_ALTER_TABLE_DROP_COLUMN: case TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES: diff --git a/source/dnode/vnode/src/meta/metaTable2.c b/source/dnode/vnode/src/meta/metaTable2.c index 2e6daf016b..83f9a4b3d2 100644 --- a/source/dnode/vnode/src/meta/metaTable2.c +++ b/source/dnode/vnode/src/meta/metaTable2.c @@ -17,6 +17,8 @@ extern int32_t metaHandleEntry2(SMeta *pMeta, const SMetaEntry *pEntry); extern int32_t metaUpdateMetaRsp(tb_uid_t uid, char *tbName, SSchemaWrapper *pSchema, STableMetaRsp *pMetaRsp); +extern int32_t metaFetchEntryByUid(SMeta *pMeta, int64_t uid, SMetaEntry **ppEntry); +extern void metaFetchEntryFree(SMetaEntry **ppEntry); static int32_t metaCheckCreateSuperTableReq(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { int32_t vgId = TD_VID(pMeta->pVnode); @@ -557,6 +559,7 @@ int32_t metaAddTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq, ST TAOS_RETURN(TSDB_CODE_INVALID_MSG); } + // check name void *value = NULL; int32_t valueSize = 0; code = tdbTbGet(pMeta->pNameIdx, pReq->tbName, strlen(pReq->tbName) + 1, &value, &valueSize); @@ -566,312 +569,130 @@ int32_t metaAddTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq, ST code = TSDB_CODE_TDB_TABLE_NOT_EXIST; TAOS_RETURN(code); } - int64_t uid = *(int64_t *)value; tdbFreeClear(value); - // TODO - return code; - -#if 0 - void *pVal = NULL; - int nVal = 0; - const void *pData = NULL; - int nData = 0; - int ret = 0; - tb_uid_t uid; - int64_t oversion; - SSchema *pColumn = NULL; - SMetaEntry entry = {0}; - SSchemaWrapper *pSchema; - int c; - bool freeColCmpr = false; - - // search uid index - TBC *pUidIdxc = NULL; - - TAOS_CHECK_RETURN(tdbTbcOpen(pMeta->pUidIdx, &pUidIdxc, NULL)); - ret = tdbTbcMoveTo(pUidIdxc, &uid, sizeof(uid), &c); - if (c != 0) { - tdbTbcClose(pUidIdxc); - metaError("meta/table: invalide c: %" PRId32 " alt tb column failed.", c); - return TSDB_CODE_FAILED; + // check table type + SMetaInfo info; + if (metaGetInfo(pMeta, uid, &info, NULL) != 0) { + metaError("vgId:%d, %s failed at %s:%d since table %s uid %" PRId64 + " not found, this is an internal error in meta, version:%" PRId64, + TD_VID(pMeta->pVnode), __func__, __FILE__, __LINE__, pReq->tbName, uid, version); + code = TSDB_CODE_INTERNAL_ERROR; + TAOS_RETURN(code); + } + if (info.suid != 0) { + metaError("vgId:%d, %s failed at %s:%d since table %s uid %" PRId64 " is not a normal table, version:%" PRId64, + TD_VID(pMeta->pVnode), __func__, __FILE__, __LINE__, pReq->tbName, uid, version); + code = TSDB_CODE_VND_INVALID_TABLE_ACTION; + TAOS_RETURN(code); } - ret = tdbTbcGet(pUidIdxc, NULL, NULL, &pData, &nData); - oversion = ((SUidIdxVal *)pData)[0].version; - - // search table.db - TBC *pTbDbc = NULL; - - TAOS_CHECK_RETURN(tdbTbcOpen(pMeta->pTbDb, &pTbDbc, NULL)); - ret = tdbTbcMoveTo(pTbDbc, &((STbDbKey){.uid = uid, .version = oversion}), sizeof(STbDbKey), &c); - if (c != 0) { - tdbTbcClose(pUidIdxc); - tdbTbcClose(pTbDbc); - metaError("meta/table: invalide c: %" PRId32 " alt tb column failed.", c); - return TSDB_CODE_FAILED; + // check grant + code = grantCheck(TSDB_GRANT_TIMESERIES); + if (code) { + metaError("vgId:%d, %s failed at %s:%d since %s, version:%" PRId64 " name:%s", TD_VID(pMeta->pVnode), __func__, + __FILE__, __LINE__, tstrerror(code), version, pReq->tbName); + TAOS_RETURN(code); } - ret = tdbTbcGet(pTbDbc, NULL, NULL, &pData, &nData); - - // get table entry - SDecoder dc = {0}; - if ((entry.pBuf = taosMemoryMalloc(nData)) == NULL) { - tdbTbcClose(pUidIdxc); - tdbTbcClose(pTbDbc); - return terrno; + // fetch old entry + SMetaEntry *pEntry = NULL; + code = metaFetchEntryByUid(pMeta, uid, &pEntry); + if (code) { + metaError("vgId:%d, %s failed at %s:%d since table %s uid %" PRId64 " not found, version:%" PRId64, + TD_VID(pMeta->pVnode), __func__, __FILE__, __LINE__, pReq->tbName, uid, version); + TAOS_RETURN(code); } - memcpy(entry.pBuf, pData, nData); - tDecoderInit(&dc, entry.pBuf, nData); - ret = metaDecodeEntry(&dc, &entry); - if (ret != 0) { - tdbTbcClose(pUidIdxc); - tdbTbcClose(pTbDbc); - tDecoderClear(&dc); - metaError("meta/table: invalide ret: %" PRId32 " alt tb column failed.", ret); - return ret; + if (pEntry->version >= version) { + metaError("vgId:%d, %s failed at %s:%d since table %s uid %" PRId64 " version %" PRId64 + " is not less than %" PRId64, + TD_VID(pMeta->pVnode), __func__, __FILE__, __LINE__, pReq->tbName, uid, pEntry->version, version); + metaFetchEntryFree(&pEntry); + TAOS_RETURN(TSDB_CODE_INVALID_PARA); } - if (entry.type != TSDB_NORMAL_TABLE) { - terrno = TSDB_CODE_VND_INVALID_TABLE_ACTION; - goto _err; - } - // search the column to add/drop/update - pSchema = &entry.ntbEntry.schemaRow; - - // save old entry - SMetaEntry oldEntry = {.type = TSDB_NORMAL_TABLE, .uid = entry.uid}; - oldEntry.ntbEntry.schemaRow.nCols = pSchema->nCols; - - int32_t rowLen = -1; - if (pAlterTbReq->action == TSDB_ALTER_TABLE_ADD_COLUMN || - pAlterTbReq->action == TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES) { - rowLen = 0; - } - - int32_t iCol = 0, jCol = 0; - SSchema *qColumn = NULL; - for (;;) { - qColumn = NULL; - - if (jCol >= pSchema->nCols) break; - qColumn = &pSchema->pSchema[jCol]; - - if (!pColumn && (strcmp(qColumn->name, pAlterTbReq->colName) == 0)) { - pColumn = qColumn; - iCol = jCol; - if (rowLen < 0) break; + // do add column + int32_t rowSize = 0; + SSchemaWrapper *pSchema = &pEntry->ntbEntry.schemaRow; + SSchema *pColumn; + pEntry->version = version; + for (int32_t i = 0; i < pSchema->nCols; i++) { + pColumn = &pSchema->pSchema[i]; + if (strncmp(pColumn->name, pReq->colName, TSDB_COL_NAME_LEN) == 0) { + metaError("vgId:%d, %s failed at %s:%d since column %s already exists in table %s, version:%" PRId64, + TD_VID(pMeta->pVnode), __func__, __FILE__, __LINE__, pReq->colName, pReq->tbName, version); + metaFetchEntryFree(&pEntry); + TAOS_RETURN(TSDB_CODE_VND_COL_ALREADY_EXISTS); } - rowLen += qColumn->bytes; - ++jCol; + rowSize += pColumn->bytes; } - entry.version = version; - int tlen; - SSchema *pNewSchema = NULL; - SSchema tScheam; - switch (pAlterTbReq->action) { - case TSDB_ALTER_TABLE_ADD_COLUMN: - case TSDB_ALTER_TABLE_ADD_COLUMN_WITH_COMPRESS_OPTION: - if (pColumn) { - terrno = TSDB_CODE_VND_COL_ALREADY_EXISTS; - goto _err; - } - if ((terrno = grantCheck(TSDB_GRANT_TIMESERIES)) < 0) { - goto _err; - } - if (rowLen + pAlterTbReq->bytes > TSDB_MAX_BYTES_PER_ROW) { - terrno = TSDB_CODE_PAR_INVALID_ROW_LENGTH; - goto _err; - } - pSchema->version++; - pSchema->nCols++; - pNewSchema = taosMemoryMalloc(sizeof(SSchema) * pSchema->nCols); - if (pNewSchema == NULL) { - goto _err; - } - memcpy(pNewSchema, pSchema->pSchema, sizeof(SSchema) * (pSchema->nCols - 1)); - pSchema->pSchema = pNewSchema; - pSchema->pSchema[entry.ntbEntry.schemaRow.nCols - 1].bytes = pAlterTbReq->bytes; - pSchema->pSchema[entry.ntbEntry.schemaRow.nCols - 1].type = pAlterTbReq->type; - pSchema->pSchema[entry.ntbEntry.schemaRow.nCols - 1].flags = pAlterTbReq->flags; - pSchema->pSchema[entry.ntbEntry.schemaRow.nCols - 1].colId = entry.ntbEntry.ncid++; - strcpy(pSchema->pSchema[entry.ntbEntry.schemaRow.nCols - 1].name, pAlterTbReq->colName); - - ++pMeta->pVnode->config.vndStats.numOfNTimeSeries; - metaTimeSeriesNotifyCheck(pMeta); - - if (!TSDB_CACHE_NO(pMeta->pVnode->config)) { - int16_t cid = pSchema->pSchema[entry.ntbEntry.schemaRow.nCols - 1].colId; - int8_t col_type = pSchema->pSchema[entry.ntbEntry.schemaRow.nCols - 1].type; - int32_t ret = tsdbCacheNewNTableColumn(pMeta->pVnode->pTsdb, entry.uid, cid, col_type); - if (ret < 0) { - terrno = ret; - goto _err; - } - } - SSchema *pCol = &pSchema->pSchema[entry.ntbEntry.schemaRow.nCols - 1]; - uint32_t compress = pAlterTbReq->action == TSDB_ALTER_TABLE_ADD_COLUMN ? createDefaultColCmprByType(pCol->type) - : pAlterTbReq->compress; - if (updataTableColCmpr(&entry.colCmpr, pCol, 1, compress) != 0) { - metaError("vgId:%d, failed to update table col cmpr:%s uid:%" PRId64, TD_VID(pMeta->pVnode), entry.name, - entry.uid); - } - freeColCmpr = true; - if (entry.colCmpr.nCols != pSchema->nCols) { - if (pNewSchema) taosMemoryFree(pNewSchema); - if (freeColCmpr) taosMemoryFree(entry.colCmpr.pColCmpr); - terrno = TSDB_CODE_VND_INVALID_TABLE_ACTION; - goto _err; - } - break; - case TSDB_ALTER_TABLE_DROP_COLUMN: - if (pColumn == NULL) { - terrno = TSDB_CODE_VND_COL_NOT_EXISTS; - goto _err; - } - if (pColumn->colId == 0) { - terrno = TSDB_CODE_VND_INVALID_TABLE_ACTION; - goto _err; - } - if (tqCheckColModifiable(pMeta->pVnode->pTq, uid, pColumn->colId) != 0) { - terrno = TSDB_CODE_VND_COL_SUBSCRIBED; - goto _err; - } - bool hasPrimayKey = false; - if (pSchema->nCols >= 2) { - hasPrimayKey = pSchema->pSchema[1].flags & COL_IS_KEY ? true : false; - } - - memcpy(&tScheam, pColumn, sizeof(SSchema)); - pSchema->version++; - tlen = (pSchema->nCols - iCol - 1) * sizeof(SSchema); - if (tlen) { - memmove(pColumn, pColumn + 1, tlen); - } - pSchema->nCols--; - - --pMeta->pVnode->config.vndStats.numOfNTimeSeries; - - if (!TSDB_CACHE_NO(pMeta->pVnode->config)) { - int16_t cid = pColumn->colId; - - if (tsdbCacheDropNTableColumn(pMeta->pVnode->pTsdb, entry.uid, cid, hasPrimayKey) != 0) { - metaError("vgId:%d, failed to drop ntable column:%s uid:%" PRId64, TD_VID(pMeta->pVnode), entry.name, - entry.uid); - } - } - - if (updataTableColCmpr(&entry.colCmpr, &tScheam, 0, 0) != 0) { - metaError("vgId:%d, failed to update table col cmpr:%s uid:%" PRId64, TD_VID(pMeta->pVnode), entry.name, - entry.uid); - } - if (entry.colCmpr.nCols != pSchema->nCols) { - terrno = TSDB_CODE_VND_INVALID_TABLE_ACTION; - goto _err; - } - break; - case TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES: - if (pColumn == NULL) { - terrno = TSDB_CODE_VND_COL_NOT_EXISTS; - goto _err; - } - if (!IS_VAR_DATA_TYPE(pColumn->type) || pColumn->bytes >= pAlterTbReq->colModBytes) { - terrno = TSDB_CODE_VND_INVALID_TABLE_ACTION; - goto _err; - } - if (rowLen + pAlterTbReq->colModBytes - pColumn->bytes > TSDB_MAX_BYTES_PER_ROW) { - terrno = TSDB_CODE_PAR_INVALID_ROW_LENGTH; - goto _err; - } - if (tqCheckColModifiable(pMeta->pVnode->pTq, uid, pColumn->colId) != 0) { - terrno = TSDB_CODE_VND_COL_SUBSCRIBED; - goto _err; - } - pSchema->version++; - pColumn->bytes = pAlterTbReq->colModBytes; - break; - case TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME: - if (pAlterTbReq->colNewName == NULL) { - terrno = TSDB_CODE_INVALID_MSG; - goto _err; - } - if (pColumn == NULL) { - terrno = TSDB_CODE_VND_COL_NOT_EXISTS; - goto _err; - } - if (tqCheckColModifiable(pMeta->pVnode->pTq, uid, pColumn->colId) != 0) { - terrno = TSDB_CODE_VND_COL_SUBSCRIBED; - goto _err; - } - pSchema->version++; - strcpy(pColumn->name, pAlterTbReq->colNewName); - break; + if (rowSize + pReq->bytes > TSDB_MAX_BYTES_PER_ROW) { + metaError("vgId:%d, %s failed at %s:%d since row size %d + %d > %d, version:%" PRId64, TD_VID(pMeta->pVnode), + __func__, __FILE__, __LINE__, rowSize, pReq->bytes, TSDB_MAX_BYTES_PER_ROW, version); + metaFetchEntryFree(&pEntry); + TAOS_RETURN(TSDB_CODE_PAR_INVALID_ROW_LENGTH); } - if (!TSDB_CACHE_NO(pMeta->pVnode->config)) { - tsdbCacheInvalidateSchema(pMeta->pVnode->pTsdb, 0, entry.uid, pSchema->version); + SSchema *pNewSchema = taosMemoryRealloc(pSchema->pSchema, sizeof(SSchema) * (pSchema->nCols + 1)); + if (NULL == pNewSchema) { + metaError("vgId:%d, %s failed at %s:%d since %s, version:%" PRId64, TD_VID(pMeta->pVnode), __func__, __FILE__, + __LINE__, tstrerror(terrno), version); + metaFetchEntryFree(&pEntry); + TAOS_RETURN(terrno); } + pSchema->pSchema = pNewSchema; + pSchema->version++; + pSchema->nCols++; + pColumn = &pSchema->pSchema[pSchema->nCols - 1]; + pColumn->bytes = pReq->bytes; + pColumn->type = pReq->type; + pColumn->flags = pReq->flags; + pColumn->colId = pEntry->ntbEntry.ncid++; + tstrncpy(pColumn->name, pReq->colName, TSDB_COL_NAME_LEN); - entry.version = version; - - // do actual write - metaWLock(pMeta); - - if (metaDeleteNcolIdx(pMeta, &oldEntry) < 0) { - metaError("vgId:%d, failed to delete ncol idx:%s uid:%" PRId64, TD_VID(pMeta->pVnode), entry.name, entry.uid); + // do handle entry + code = metaHandleEntry2(pMeta, pEntry); + if (code) { + metaError("vgId:%d, %s failed at %s:%d since %s, uid:%" PRId64 " name:%s version:%" PRId64, TD_VID(pMeta->pVnode), + __func__, __FILE__, __LINE__, tstrerror(code), uid, pReq->tbName, version); + } else { + metaInfo("vgId:%d, table %s uid %" PRId64 " is updated, version:%" PRId64, TD_VID(pMeta->pVnode), pReq->tbName, uid, + version); } + metaFetchEntryFree(&pEntry); - if (metaUpdateNcolIdx(pMeta, &entry) < 0) { - metaError("vgId:%d, failed to update ncol idx:%s uid:%" PRId64, TD_VID(pMeta->pVnode), entry.name, entry.uid); - } - - // save to table db - if (metaSaveToTbDb(pMeta, &entry) < 0) { - metaError("vgId:%d, failed to save to tb db:%s uid:%" PRId64, TD_VID(pMeta->pVnode), entry.name, entry.uid); - } - - if (metaUpdateUidIdx(pMeta, &entry) < 0) { - metaError("vgId:%d, failed to update uid idx:%s uid:%" PRId64, TD_VID(pMeta->pVnode), entry.name, entry.uid); - } - - if (metaSaveToSkmDb(pMeta, &entry) < 0) { - metaError("vgId:%d, failed to save to skm db:%s uid:%" PRId64, TD_VID(pMeta->pVnode), entry.name, entry.uid); - } - - if (metaUpdateChangeTime(pMeta, entry.uid, pAlterTbReq->ctimeMs) < 0) { - metaError("vgId:%d, failed to update change time:%s uid:%" PRId64, TD_VID(pMeta->pVnode), entry.name, entry.uid); - } - - metaULock(pMeta); - - if (metaUpdateMetaRsp(uid, pAlterTbReq->tbName, pSchema, pMetaRsp) < 0) { - metaError("vgId:%d, failed to update meta rsp:%s uid:%" PRId64, TD_VID(pMeta->pVnode), entry.name, entry.uid); + if (metaUpdateMetaRsp(uid, pReq->tbName, pSchema, pRsp) < 0) { + metaError("vgId:%d, %s failed at %s:%d since %s, uid:%" PRId64 " name:%s version:%" PRId64, TD_VID(pMeta->pVnode), + __func__, __FILE__, __LINE__, tstrerror(code), uid, pReq->tbName, version); } +#if 0 for (int32_t i = 0; i < entry.colCmpr.nCols; i++) { SColCmpr *p = &entry.colCmpr.pColCmpr[i]; pMetaRsp->pSchemaExt[i].colId = p->id; pMetaRsp->pSchemaExt[i].compress = p->alg; } +#endif - if (entry.pBuf) taosMemoryFree(entry.pBuf); - if (pNewSchema) taosMemoryFree(pNewSchema); - if (freeColCmpr) taosMemoryFree(entry.colCmpr.pColCmpr); + TAOS_RETURN(code); - tdbTbcClose(pTbDbc); - tdbTbcClose(pUidIdxc); - tDecoderClear(&dc); +#if 0 + SSchema *pCol = &pSchema->pSchema[entry.ntbEntry.schemaRow.nCols - 1]; + uint32_t compress = pAlterTbReq->action == TSDB_ALTER_TABLE_ADD_COLUMN ? createDefaultColCmprByType(pCol->type) + : pAlterTbReq->compress; + if (updataTableColCmpr(&entry.colCmpr, pCol, 1, compress) != 0) { + metaError("vgId:%d, failed to update table col cmpr:%s uid:%" PRId64, TD_VID(pMeta->pVnode), entry.name, entry.uid); + } + freeColCmpr = true; + if (entry.colCmpr.nCols != pSchema->nCols) { + if (pNewSchema) taosMemoryFree(pNewSchema); + if (freeColCmpr) taosMemoryFree(entry.colCmpr.pColCmpr); + terrno = TSDB_CODE_VND_INVALID_TABLE_ACTION; + goto _err; + } + break; - return 0; - -_err: - if (entry.pBuf) taosMemoryFree(entry.pBuf); - tdbTbcClose(pTbDbc); - tdbTbcClose(pUidIdxc); - tDecoderClear(&dc); - - return terrno != 0 ? terrno : TSDB_CODE_FAILED; #endif } From 9ce7d0e986bc2f77308006c46b31bdf290186c3d Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 12 Dec 2024 16:38:55 +0800 Subject: [PATCH 26/45] fix: ci cases --- source/dnode/vnode/src/meta/metaEntry2.c | 3 +- source/dnode/vnode/src/meta/metaTable.c | 11 +- source/dnode/vnode/src/meta/metaTable2.c | 339 ++++++++++++++++++++--- 3 files changed, 305 insertions(+), 48 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaEntry2.c b/source/dnode/vnode/src/meta/metaEntry2.c index 384dd5abbd..943cb05fb6 100644 --- a/source/dnode/vnode/src/meta/metaEntry2.c +++ b/source/dnode/vnode/src/meta/metaEntry2.c @@ -109,7 +109,7 @@ int32_t metaFetchEntryByUid(SMeta *pMeta, int64_t uid, SMetaEntry **ppEntry) { return code; } -static int32_t metaFetchEntryByName(SMeta *pMeta, const char *name, SMetaEntry **ppEntry) { +int32_t metaFetchEntryByName(SMeta *pMeta, const char *name, SMetaEntry **ppEntry) { int32_t code = TSDB_CODE_SUCCESS; void *value = NULL; int32_t valueSize = 0; @@ -1309,6 +1309,7 @@ static int32_t metaHandleNormalTableUpdate(SMeta *pMeta, const SMetaEntry *pEntr tsdbCacheInvalidateSchema(pMeta->pVnode->pTsdb, 0, pEntry->uid, pEntry->ntbEntry.schemaRow.version); } metaTimeSeriesNotifyCheck(pMeta); + metaFetchEntryFree(&pOldEntry); return code; } diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 81936ed2a4..7ecfa0fc12 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -43,23 +43,21 @@ static int metaDeleteBtimeIdx(SMeta *pMeta, const SMetaEntry *pME); static int metaUpdateNcolIdx(SMeta *pMeta, const SMetaEntry *pME); static int metaDeleteNcolIdx(SMeta *pMeta, const SMetaEntry *pME); -static int32_t updataTableColCmpr(SColCmprWrapper *pWp, SSchema *pSchema, int8_t add, uint32_t compress) { +int32_t updataTableColCmpr(SColCmprWrapper *pWp, SSchema *pSchema, int8_t add, uint32_t compress) { int32_t nCols = pWp->nCols; int32_t ver = pWp->version; if (add) { - SColCmpr *p = taosMemoryCalloc(1, sizeof(SColCmpr) * (nCols + 1)); + SColCmpr *p = taosMemoryRealloc(pWp->pColCmpr, sizeof(SColCmpr) * (nCols + 1)); if (p == NULL) { return terrno; } - - memcpy(p, pWp->pColCmpr, sizeof(SColCmpr) * nCols); + pWp->pColCmpr = p; SColCmpr *pCol = p + nCols; pCol->id = pSchema->colId; pCol->alg = compress; pWp->nCols = nCols + 1; pWp->version = ver; - pWp->pColCmpr = p; } else { for (int32_t i = 0; i < nCols; i++) { SColCmpr *pOCmpr = &pWp->pColCmpr[i]; @@ -2989,8 +2987,9 @@ int metaAlterTable(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq, STableMeta switch (pReq->action) { case TSDB_ALTER_TABLE_ADD_COLUMN: return metaAddTableColumn(pMeta, version, pReq, pMetaRsp); - case TSDB_ALTER_TABLE_ADD_COLUMN_WITH_COMPRESS_OPTION: case TSDB_ALTER_TABLE_DROP_COLUMN: + // return metaDropTableColumn(pMeta, version, pReq, pMetaRsp); + case TSDB_ALTER_TABLE_ADD_COLUMN_WITH_COMPRESS_OPTION: case TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES: case TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME: return metaAlterTableColumn(pMeta, version, pReq, pMetaRsp); diff --git a/source/dnode/vnode/src/meta/metaTable2.c b/source/dnode/vnode/src/meta/metaTable2.c index 83f9a4b3d2..d31d7b688a 100644 --- a/source/dnode/vnode/src/meta/metaTable2.c +++ b/source/dnode/vnode/src/meta/metaTable2.c @@ -18,7 +18,9 @@ extern int32_t metaHandleEntry2(SMeta *pMeta, const SMetaEntry *pEntry); extern int32_t metaUpdateMetaRsp(tb_uid_t uid, char *tbName, SSchemaWrapper *pSchema, STableMetaRsp *pMetaRsp); extern int32_t metaFetchEntryByUid(SMeta *pMeta, int64_t uid, SMetaEntry **ppEntry); +extern int32_t metaFetchEntryByName(SMeta *pMeta, const char *name, SMetaEntry **ppEntry); extern void metaFetchEntryFree(SMetaEntry **ppEntry); +extern int32_t updataTableColCmpr(SColCmprWrapper *pWp, SSchema *pSchema, int8_t add, uint32_t compress); static int32_t metaCheckCreateSuperTableReq(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { int32_t vgId = TD_VID(pMeta->pVnode); @@ -549,10 +551,9 @@ int32_t metaDropTable2(SMeta *pMeta, int64_t version, SVDropTbReq *pReq) { TAOS_RETURN(code); } -int32_t metaAddTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq, STableMetaRsp *pRsp) { - int32_t code = TSDB_CODE_SUCCESS; +static int32_t metaCheckAlterTableColumnReq(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq) { + int32_t code = 0; - // check request if (NULL == pReq->colName || strlen(pReq->colName) == 0) { metaError("vgId:%d, %s failed at %s:%d since invalid column name:%s, version:%" PRId64, TD_VID(pMeta->pVnode), __func__, __FILE__, __LINE__, pReq->colName, version); @@ -595,19 +596,29 @@ int32_t metaAddTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq, ST __FILE__, __LINE__, tstrerror(code), version, pReq->tbName); TAOS_RETURN(code); } + TAOS_RETURN(code); +} + +int32_t metaAddTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq, STableMetaRsp *pRsp) { + int32_t code = TSDB_CODE_SUCCESS; + + // check request + code = metaCheckAlterTableColumnReq(pMeta, version, pReq); + if (code) { + TAOS_RETURN(code); + } // fetch old entry SMetaEntry *pEntry = NULL; - code = metaFetchEntryByUid(pMeta, uid, &pEntry); + code = metaFetchEntryByName(pMeta, pReq->tbName, &pEntry); if (code) { - metaError("vgId:%d, %s failed at %s:%d since table %s uid %" PRId64 " not found, version:%" PRId64, - TD_VID(pMeta->pVnode), __func__, __FILE__, __LINE__, pReq->tbName, uid, version); + metaError("vgId:%d, %s failed at %s:%d since table %s not found, version:%" PRId64, TD_VID(pMeta->pVnode), __func__, + __FILE__, __LINE__, pReq->tbName, version); TAOS_RETURN(code); } if (pEntry->version >= version) { - metaError("vgId:%d, %s failed at %s:%d since table %s uid %" PRId64 " version %" PRId64 - " is not less than %" PRId64, - TD_VID(pMeta->pVnode), __func__, __FILE__, __LINE__, pReq->tbName, uid, pEntry->version, version); + metaError("vgId:%d, %s failed at %s:%d since table %s version %" PRId64 " is not less than %" PRId64, + TD_VID(pMeta->pVnode), __func__, __FILE__, __LINE__, pReq->tbName, pEntry->version, version); metaFetchEntryFree(&pEntry); TAOS_RETURN(TSDB_CODE_INVALID_PARA); } @@ -651,55 +662,301 @@ int32_t metaAddTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq, ST pColumn->flags = pReq->flags; pColumn->colId = pEntry->ntbEntry.ncid++; tstrncpy(pColumn->name, pReq->colName, TSDB_COL_NAME_LEN); + uint32_t compress = createDefaultColCmprByType(pColumn->type); + code = updataTableColCmpr(&pEntry->colCmpr, pColumn, 1, compress); + if (code) { + metaError("vgId:%d, %s failed at %s:%d since %s, version:%" PRId64, TD_VID(pMeta->pVnode), __func__, __FILE__, + __LINE__, tstrerror(code), version); + metaFetchEntryFree(&pEntry); + TAOS_RETURN(code); + } // do handle entry code = metaHandleEntry2(pMeta, pEntry); if (code) { metaError("vgId:%d, %s failed at %s:%d since %s, uid:%" PRId64 " name:%s version:%" PRId64, TD_VID(pMeta->pVnode), - __func__, __FILE__, __LINE__, tstrerror(code), uid, pReq->tbName, version); + __func__, __FILE__, __LINE__, tstrerror(code), pEntry->uid, pReq->tbName, version); } else { - metaInfo("vgId:%d, table %s uid %" PRId64 " is updated, version:%" PRId64, TD_VID(pMeta->pVnode), pReq->tbName, uid, - version); + metaInfo("vgId:%d, table %s uid %" PRId64 " is updated, version:%" PRId64, TD_VID(pMeta->pVnode), pReq->tbName, + pEntry->uid, version); } - metaFetchEntryFree(&pEntry); - if (metaUpdateMetaRsp(uid, pReq->tbName, pSchema, pRsp) < 0) { + if (metaUpdateMetaRsp(pEntry->uid, pReq->tbName, pSchema, pRsp) < 0) { metaError("vgId:%d, %s failed at %s:%d since %s, uid:%" PRId64 " name:%s version:%" PRId64, TD_VID(pMeta->pVnode), - __func__, __FILE__, __LINE__, tstrerror(code), uid, pReq->tbName, version); + __func__, __FILE__, __LINE__, tstrerror(code), pEntry->uid, pReq->tbName, version); + } else { + for (int32_t i = 0; i < pEntry->colCmpr.nCols; i++) { + SColCmpr *p = &pEntry->colCmpr.pColCmpr[i]; + pRsp->pSchemaExt[i].colId = p->id; + pRsp->pSchemaExt[i].compress = p->alg; + } } + + metaFetchEntryFree(&pEntry); + TAOS_RETURN(code); +} + +int32_t metaDropTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq, STableMetaRsp *pRsp) { + int32_t code = TSDB_CODE_SUCCESS; + + code = metaCheckAlterTableColumnReq(pMeta, version, pReq); + if (code) { + TAOS_RETURN(code); + } + + // TODO + TAOS_RETURN(code); #if 0 + void *pVal = NULL; + int nVal = 0; + const void *pData = NULL; + int nData = 0; + int ret = 0; + tb_uid_t uid; + int64_t oversion; + SSchema *pColumn = NULL; + SMetaEntry entry = {0}; + SSchemaWrapper *pSchema; + int c; + bool freeColCmpr = false; + + // search the column to add/drop/update + pSchema = &entry.ntbEntry.schemaRow; + + // save old entry + SMetaEntry oldEntry = {.type = TSDB_NORMAL_TABLE, .uid = entry.uid}; + oldEntry.ntbEntry.schemaRow.nCols = pSchema->nCols; + + int32_t rowLen = -1; + if (pAlterTbReq->action == TSDB_ALTER_TABLE_ADD_COLUMN || + pAlterTbReq->action == TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES) { + rowLen = 0; + } + + int32_t iCol = 0, jCol = 0; + SSchema *qColumn = NULL; + for (;;) { + qColumn = NULL; + + if (jCol >= pSchema->nCols) break; + qColumn = &pSchema->pSchema[jCol]; + + if (!pColumn && (strcmp(qColumn->name, pAlterTbReq->colName) == 0)) { + pColumn = qColumn; + iCol = jCol; + if (rowLen < 0) break; + } + rowLen += qColumn->bytes; + ++jCol; + } + + entry.version = version; + int tlen; + SSchema *pNewSchema = NULL; + SSchema tScheam; + switch (pAlterTbReq->action) { + case TSDB_ALTER_TABLE_ADD_COLUMN: + case TSDB_ALTER_TABLE_ADD_COLUMN_WITH_COMPRESS_OPTION: + if (pColumn) { + terrno = TSDB_CODE_VND_COL_ALREADY_EXISTS; + goto _err; + } + if ((terrno = grantCheck(TSDB_GRANT_TIMESERIES)) < 0) { + goto _err; + } + if (rowLen + pAlterTbReq->bytes > TSDB_MAX_BYTES_PER_ROW) { + terrno = TSDB_CODE_PAR_INVALID_ROW_LENGTH; + goto _err; + } + pSchema->version++; + pSchema->nCols++; + pNewSchema = taosMemoryMalloc(sizeof(SSchema) * pSchema->nCols); + if (pNewSchema == NULL) { + goto _err; + } + memcpy(pNewSchema, pSchema->pSchema, sizeof(SSchema) * (pSchema->nCols - 1)); + pSchema->pSchema = pNewSchema; + pSchema->pSchema[entry.ntbEntry.schemaRow.nCols - 1].bytes = pAlterTbReq->bytes; + pSchema->pSchema[entry.ntbEntry.schemaRow.nCols - 1].type = pAlterTbReq->type; + pSchema->pSchema[entry.ntbEntry.schemaRow.nCols - 1].flags = pAlterTbReq->flags; + pSchema->pSchema[entry.ntbEntry.schemaRow.nCols - 1].colId = entry.ntbEntry.ncid++; + tstrncpy(pSchema->pSchema[entry.ntbEntry.schemaRow.nCols - 1].name, pAlterTbReq->colName, + strlen(pAlterTbReq->colName) + 1); + + ++pMeta->pVnode->config.vndStats.numOfNTimeSeries; + metaTimeSeriesNotifyCheck(pMeta); + + if (!TSDB_CACHE_NO(pMeta->pVnode->config)) { + int16_t cid = pSchema->pSchema[entry.ntbEntry.schemaRow.nCols - 1].colId; + int8_t col_type = pSchema->pSchema[entry.ntbEntry.schemaRow.nCols - 1].type; + int32_t ret = tsdbCacheNewNTableColumn(pMeta->pVnode->pTsdb, entry.uid, cid, col_type); + if (ret < 0) { + terrno = ret; + goto _err; + } + } + SSchema *pCol = &pSchema->pSchema[entry.ntbEntry.schemaRow.nCols - 1]; + uint32_t compress = pAlterTbReq->action == TSDB_ALTER_TABLE_ADD_COLUMN ? createDefaultColCmprByType(pCol->type) + : pAlterTbReq->compress; + if (updataTableColCmpr(&entry.colCmpr, pCol, 1, compress) != 0) { + metaError("vgId:%d, failed to update table col cmpr:%s uid:%" PRId64, TD_VID(pMeta->pVnode), entry.name, + entry.uid); + } + freeColCmpr = true; + if (entry.colCmpr.nCols != pSchema->nCols) { + if (pNewSchema) taosMemoryFree(pNewSchema); + if (freeColCmpr) taosMemoryFree(entry.colCmpr.pColCmpr); + terrno = TSDB_CODE_VND_INVALID_TABLE_ACTION; + goto _err; + } + break; + case TSDB_ALTER_TABLE_DROP_COLUMN: +if (pColumn == NULL) { + terrno = TSDB_CODE_VND_COL_NOT_EXISTS; + goto _err; + } + if (pColumn->colId == 0) { + terrno = TSDB_CODE_VND_INVALID_TABLE_ACTION; + goto _err; + } + if (tqCheckColModifiable(pMeta->pVnode->pTq, uid, pColumn->colId) != 0) { + terrno = TSDB_CODE_VND_COL_SUBSCRIBED; + goto _err; + } + bool hasPrimayKey = false; + if (pSchema->nCols >= 2) { + hasPrimayKey = pSchema->pSchema[1].flags & COL_IS_KEY ? true : false; + } + + memcpy(&tScheam, pColumn, sizeof(SSchema)); + pSchema->version++; + tlen = (pSchema->nCols - iCol - 1) * sizeof(SSchema); + if (tlen) { + memmove(pColumn, pColumn + 1, tlen); + } + pSchema->nCols--; + + --pMeta->pVnode->config.vndStats.numOfNTimeSeries; + + if (!TSDB_CACHE_NO(pMeta->pVnode->config)) { + int16_t cid = pColumn->colId; + + if (tsdbCacheDropNTableColumn(pMeta->pVnode->pTsdb, entry.uid, cid, hasPrimayKey) != 0) { + metaError("vgId:%d, failed to drop ntable column:%s uid:%" PRId64, TD_VID(pMeta->pVnode), entry.name, + entry.uid); + } + } + + if (updataTableColCmpr(&entry.colCmpr, &tScheam, 0, 0) != 0) { + metaError("vgId:%d, failed to update table col cmpr:%s uid:%" PRId64, TD_VID(pMeta->pVnode), entry.name, + entry.uid); + } + if (entry.colCmpr.nCols != pSchema->nCols) { + terrno = TSDB_CODE_VND_INVALID_TABLE_ACTION; + goto _err; + } + break; + case TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES: + if (pColumn == NULL) { + terrno = TSDB_CODE_VND_COL_NOT_EXISTS; + goto _err; + } + if (!IS_VAR_DATA_TYPE(pColumn->type) || pColumn->bytes >= pAlterTbReq->colModBytes) { + terrno = TSDB_CODE_VND_INVALID_TABLE_ACTION; + goto _err; + } + if (rowLen + pAlterTbReq->colModBytes - pColumn->bytes > TSDB_MAX_BYTES_PER_ROW) { + terrno = TSDB_CODE_PAR_INVALID_ROW_LENGTH; + goto _err; + } + if (tqCheckColModifiable(pMeta->pVnode->pTq, uid, pColumn->colId) != 0) { + terrno = TSDB_CODE_VND_COL_SUBSCRIBED; + goto _err; + } + pSchema->version++; + pColumn->bytes = pAlterTbReq->colModBytes; + break; + case TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME: + if (pAlterTbReq->colNewName == NULL) { + terrno = TSDB_CODE_INVALID_MSG; + goto _err; + } + if (pColumn == NULL) { + terrno = TSDB_CODE_VND_COL_NOT_EXISTS; + goto _err; + } + if (tqCheckColModifiable(pMeta->pVnode->pTq, uid, pColumn->colId) != 0) { + terrno = TSDB_CODE_VND_COL_SUBSCRIBED; + goto _err; + } + pSchema->version++; + tstrncpy(pColumn->name, pAlterTbReq->colNewName, strlen(pAlterTbReq->colNewName) + 1); + break; + } + + if (!TSDB_CACHE_NO(pMeta->pVnode->config)) { + tsdbCacheInvalidateSchema(pMeta->pVnode->pTsdb, 0, entry.uid, pSchema->version); + } + + entry.version = version; + + // do actual write + metaWLock(pMeta); + + if (metaDeleteNcolIdx(pMeta, &oldEntry) < 0) { + metaError("vgId:%d, failed to delete ncol idx:%s uid:%" PRId64, TD_VID(pMeta->pVnode), entry.name, entry.uid); + } + + if (metaUpdateNcolIdx(pMeta, &entry) < 0) { + metaError("vgId:%d, failed to update ncol idx:%s uid:%" PRId64, TD_VID(pMeta->pVnode), entry.name, entry.uid); + } + + // save to table db + if (metaSaveToTbDb(pMeta, &entry) < 0) { + metaError("vgId:%d, failed to save to tb db:%s uid:%" PRId64, TD_VID(pMeta->pVnode), entry.name, entry.uid); + } + + if (metaUpdateUidIdx(pMeta, &entry) < 0) { + metaError("vgId:%d, failed to update uid idx:%s uid:%" PRId64, TD_VID(pMeta->pVnode), entry.name, entry.uid); + } + + if (metaSaveToSkmDb(pMeta, &entry) < 0) { + metaError("vgId:%d, failed to save to skm db:%s uid:%" PRId64, TD_VID(pMeta->pVnode), entry.name, entry.uid); + } + + if (metaUpdateChangeTime(pMeta, entry.uid, pAlterTbReq->ctimeMs) < 0) { + metaError("vgId:%d, failed to update change time:%s uid:%" PRId64, TD_VID(pMeta->pVnode), entry.name, entry.uid); + } + + metaULock(pMeta); + + if (metaUpdateMetaRsp(uid, pAlterTbReq->tbName, pSchema, pMetaRsp) < 0) { + metaError("vgId:%d, failed to update meta rsp:%s uid:%" PRId64, TD_VID(pMeta->pVnode), entry.name, entry.uid); + } for (int32_t i = 0; i < entry.colCmpr.nCols; i++) { SColCmpr *p = &entry.colCmpr.pColCmpr[i]; pMetaRsp->pSchemaExt[i].colId = p->id; pMetaRsp->pSchemaExt[i].compress = p->alg; } + + if (entry.pBuf) taosMemoryFree(entry.pBuf); + if (pNewSchema) taosMemoryFree(pNewSchema); + if (freeColCmpr) taosMemoryFree(entry.colCmpr.pColCmpr); + + tdbTbcClose(pTbDbc); + tdbTbcClose(pUidIdxc); + tDecoderClear(&dc); + + return 0; + +_err: + if (entry.pBuf) taosMemoryFree(entry.pBuf); + tdbTbcClose(pTbDbc); + tdbTbcClose(pUidIdxc); + tDecoderClear(&dc); + + return terrno != 0 ? terrno : TSDB_CODE_FAILED; #endif - - TAOS_RETURN(code); - -#if 0 - SSchema *pCol = &pSchema->pSchema[entry.ntbEntry.schemaRow.nCols - 1]; - uint32_t compress = pAlterTbReq->action == TSDB_ALTER_TABLE_ADD_COLUMN ? createDefaultColCmprByType(pCol->type) - : pAlterTbReq->compress; - if (updataTableColCmpr(&entry.colCmpr, pCol, 1, compress) != 0) { - metaError("vgId:%d, failed to update table col cmpr:%s uid:%" PRId64, TD_VID(pMeta->pVnode), entry.name, entry.uid); - } - freeColCmpr = true; - if (entry.colCmpr.nCols != pSchema->nCols) { - if (pNewSchema) taosMemoryFree(pNewSchema); - if (freeColCmpr) taosMemoryFree(entry.colCmpr.pColCmpr); - terrno = TSDB_CODE_VND_INVALID_TABLE_ACTION; - goto _err; - } - break; - -#endif -} - -int32_t metaDropTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq, STableMetaRsp *pRsp) { - int32_t code = TSDB_CODE_SUCCESS; - // TODO - return code; } int32_t metaAlterTableColumnName(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq, STableMetaRsp *pRsp) { From ff0389d8d90e1b6ad7d06790fe2868622fdde6fb Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 12 Dec 2024 17:19:06 +0800 Subject: [PATCH 27/45] drop table column --- source/dnode/vnode/src/meta/metaEntry2.c | 26 +- source/dnode/vnode/src/meta/metaTable.c | 4 +- source/dnode/vnode/src/meta/metaTable2.c | 329 +++++++---------------- 3 files changed, 112 insertions(+), 247 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaEntry2.c b/source/dnode/vnode/src/meta/metaEntry2.c index 943cb05fb6..15cc2dc525 100644 --- a/source/dnode/vnode/src/meta/metaEntry2.c +++ b/source/dnode/vnode/src/meta/metaEntry2.c @@ -1298,12 +1298,26 @@ static int32_t metaHandleNormalTableUpdate(SMeta *pMeta, const SMetaEntry *pEntr if (!TSDB_CACHE_NO(pMeta->pVnode->config) && pEntry->ntbEntry.schemaRow.version != pOldEntry->ntbEntry.schemaRow.version) { #if 0 - int16_t cid = pSchema->pSchema[entry.ntbEntry.schemaRow.nCols - 1].colId; - int8_t col_type = pSchema->pSchema[entry.ntbEntry.schemaRow.nCols - 1].type; - int32_t ret = tsdbCacheNewNTableColumn(pMeta->pVnode->pTsdb, entry.uid, cid, col_type); - if (ret < 0) { - terrno = ret; - goto _err; + { // for add column + int16_t cid = pSchema->pSchema[entry.ntbEntry.schemaRow.nCols - 1].colId; + int8_t col_type = pSchema->pSchema[entry.ntbEntry.schemaRow.nCols - 1].type; + int32_t ret = tsdbCacheNewNTableColumn(pMeta->pVnode->pTsdb, entry.uid, cid, col_type); + if (ret < 0) { + terrno = ret; + goto _err; + } + } + { // for drop column + + if (!TSDB_CACHE_NO(pMeta->pVnode->config)) { + int16_t cid = pColumn->colId; + + if (tsdbCacheDropNTableColumn(pMeta->pVnode->pTsdb, entry.uid, cid, hasPrimayKey) != 0) { + metaError("vgId:%d, failed to drop ntable column:%s uid:%" PRId64, TD_VID(pMeta->pVnode), entry.name, + entry.uid); + } + tsdbCacheInvalidateSchema(pMeta->pVnode->pTsdb, 0, entry.uid, pSchema->version); + } } #endif tsdbCacheInvalidateSchema(pMeta->pVnode->pTsdb, 0, pEntry->uid, pEntry->ntbEntry.schemaRow.version); diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 7ecfa0fc12..cff067f4bb 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -2986,10 +2986,10 @@ int metaAlterTable(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq, STableMeta pMeta->changed = true; switch (pReq->action) { case TSDB_ALTER_TABLE_ADD_COLUMN: + case TSDB_ALTER_TABLE_ADD_COLUMN_WITH_COMPRESS_OPTION: return metaAddTableColumn(pMeta, version, pReq, pMetaRsp); case TSDB_ALTER_TABLE_DROP_COLUMN: - // return metaDropTableColumn(pMeta, version, pReq, pMetaRsp); - case TSDB_ALTER_TABLE_ADD_COLUMN_WITH_COMPRESS_OPTION: + return metaDropTableColumn(pMeta, version, pReq, pMetaRsp); case TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES: case TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME: return metaAlterTableColumn(pMeta, version, pReq, pMetaRsp); diff --git a/source/dnode/vnode/src/meta/metaTable2.c b/source/dnode/vnode/src/meta/metaTable2.c index d31d7b688a..cfad79161b 100644 --- a/source/dnode/vnode/src/meta/metaTable2.c +++ b/source/dnode/vnode/src/meta/metaTable2.c @@ -662,7 +662,12 @@ int32_t metaAddTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq, ST pColumn->flags = pReq->flags; pColumn->colId = pEntry->ntbEntry.ncid++; tstrncpy(pColumn->name, pReq->colName, TSDB_COL_NAME_LEN); - uint32_t compress = createDefaultColCmprByType(pColumn->type); + uint32_t compress; + if (TSDB_ALTER_TABLE_ADD_COLUMN == pReq->action) { + compress = createDefaultColCmprByType(pColumn->type); + } else { + compress = pReq->compress; + } code = updataTableColCmpr(&pEntry->colCmpr, pColumn, 1, compress); if (code) { metaError("vgId:%d, %s failed at %s:%d since %s, version:%" PRId64, TD_VID(pMeta->pVnode), __func__, __FILE__, @@ -676,6 +681,8 @@ int32_t metaAddTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq, ST if (code) { metaError("vgId:%d, %s failed at %s:%d since %s, uid:%" PRId64 " name:%s version:%" PRId64, TD_VID(pMeta->pVnode), __func__, __FILE__, __LINE__, tstrerror(code), pEntry->uid, pReq->tbName, version); + metaFetchEntryFree(&pEntry); + TAOS_RETURN(code); } else { metaInfo("vgId:%d, table %s uid %" PRId64 " is updated, version:%" PRId64, TD_VID(pMeta->pVnode), pReq->tbName, pEntry->uid, version); @@ -699,264 +706,108 @@ int32_t metaAddTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq, ST int32_t metaDropTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq, STableMetaRsp *pRsp) { int32_t code = TSDB_CODE_SUCCESS; + // check request code = metaCheckAlterTableColumnReq(pMeta, version, pReq); if (code) { TAOS_RETURN(code); } - // TODO - TAOS_RETURN(code); -#if 0 - void *pVal = NULL; - int nVal = 0; - const void *pData = NULL; - int nData = 0; - int ret = 0; - tb_uid_t uid; - int64_t oversion; + // fetch old entry + SMetaEntry *pEntry = NULL; + code = metaFetchEntryByName(pMeta, pReq->tbName, &pEntry); + if (code) { + metaError("vgId:%d, %s failed at %s:%d since table %s not found, version:%" PRId64, TD_VID(pMeta->pVnode), __func__, + __FILE__, __LINE__, pReq->tbName, version); + TAOS_RETURN(code); + } + + if (pEntry->version >= version) { + metaError("vgId:%d, %s failed at %s:%d since table %s version %" PRId64 " is not less than %" PRId64, + TD_VID(pMeta->pVnode), __func__, __FILE__, __LINE__, pReq->tbName, pEntry->version, version); + metaFetchEntryFree(&pEntry); + TAOS_RETURN(TSDB_CODE_INVALID_PARA); + } + + // search the column to drop + SSchemaWrapper *pSchema = &pEntry->ntbEntry.schemaRow; SSchema *pColumn = NULL; - SMetaEntry entry = {0}; - SSchemaWrapper *pSchema; - int c; - bool freeColCmpr = false; - - // search the column to add/drop/update - pSchema = &entry.ntbEntry.schemaRow; - - // save old entry - SMetaEntry oldEntry = {.type = TSDB_NORMAL_TABLE, .uid = entry.uid}; - oldEntry.ntbEntry.schemaRow.nCols = pSchema->nCols; - - int32_t rowLen = -1; - if (pAlterTbReq->action == TSDB_ALTER_TABLE_ADD_COLUMN || - pAlterTbReq->action == TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES) { - rowLen = 0; - } - - int32_t iCol = 0, jCol = 0; - SSchema *qColumn = NULL; - for (;;) { - qColumn = NULL; - - if (jCol >= pSchema->nCols) break; - qColumn = &pSchema->pSchema[jCol]; - - if (!pColumn && (strcmp(qColumn->name, pAlterTbReq->colName) == 0)) { - pColumn = qColumn; - iCol = jCol; - if (rowLen < 0) break; + SSchema tColumn; + int32_t iColumn = 0; + for (int32_t iColumn = 0; iColumn < pSchema->nCols; iColumn++) { + pColumn = &pSchema->pSchema[iColumn]; + if (strncmp(pColumn->name, pReq->colName, TSDB_COL_NAME_LEN) == 0) { + break; } - rowLen += qColumn->bytes; - ++jCol; } - entry.version = version; - int tlen; - SSchema *pNewSchema = NULL; - SSchema tScheam; - switch (pAlterTbReq->action) { - case TSDB_ALTER_TABLE_ADD_COLUMN: - case TSDB_ALTER_TABLE_ADD_COLUMN_WITH_COMPRESS_OPTION: - if (pColumn) { - terrno = TSDB_CODE_VND_COL_ALREADY_EXISTS; - goto _err; - } - if ((terrno = grantCheck(TSDB_GRANT_TIMESERIES)) < 0) { - goto _err; - } - if (rowLen + pAlterTbReq->bytes > TSDB_MAX_BYTES_PER_ROW) { - terrno = TSDB_CODE_PAR_INVALID_ROW_LENGTH; - goto _err; - } - pSchema->version++; - pSchema->nCols++; - pNewSchema = taosMemoryMalloc(sizeof(SSchema) * pSchema->nCols); - if (pNewSchema == NULL) { - goto _err; - } - memcpy(pNewSchema, pSchema->pSchema, sizeof(SSchema) * (pSchema->nCols - 1)); - pSchema->pSchema = pNewSchema; - pSchema->pSchema[entry.ntbEntry.schemaRow.nCols - 1].bytes = pAlterTbReq->bytes; - pSchema->pSchema[entry.ntbEntry.schemaRow.nCols - 1].type = pAlterTbReq->type; - pSchema->pSchema[entry.ntbEntry.schemaRow.nCols - 1].flags = pAlterTbReq->flags; - pSchema->pSchema[entry.ntbEntry.schemaRow.nCols - 1].colId = entry.ntbEntry.ncid++; - tstrncpy(pSchema->pSchema[entry.ntbEntry.schemaRow.nCols - 1].name, pAlterTbReq->colName, - strlen(pAlterTbReq->colName) + 1); - - ++pMeta->pVnode->config.vndStats.numOfNTimeSeries; - metaTimeSeriesNotifyCheck(pMeta); - - if (!TSDB_CACHE_NO(pMeta->pVnode->config)) { - int16_t cid = pSchema->pSchema[entry.ntbEntry.schemaRow.nCols - 1].colId; - int8_t col_type = pSchema->pSchema[entry.ntbEntry.schemaRow.nCols - 1].type; - int32_t ret = tsdbCacheNewNTableColumn(pMeta->pVnode->pTsdb, entry.uid, cid, col_type); - if (ret < 0) { - terrno = ret; - goto _err; - } - } - SSchema *pCol = &pSchema->pSchema[entry.ntbEntry.schemaRow.nCols - 1]; - uint32_t compress = pAlterTbReq->action == TSDB_ALTER_TABLE_ADD_COLUMN ? createDefaultColCmprByType(pCol->type) - : pAlterTbReq->compress; - if (updataTableColCmpr(&entry.colCmpr, pCol, 1, compress) != 0) { - metaError("vgId:%d, failed to update table col cmpr:%s uid:%" PRId64, TD_VID(pMeta->pVnode), entry.name, - entry.uid); - } - freeColCmpr = true; - if (entry.colCmpr.nCols != pSchema->nCols) { - if (pNewSchema) taosMemoryFree(pNewSchema); - if (freeColCmpr) taosMemoryFree(entry.colCmpr.pColCmpr); - terrno = TSDB_CODE_VND_INVALID_TABLE_ACTION; - goto _err; - } - break; - case TSDB_ALTER_TABLE_DROP_COLUMN: -if (pColumn == NULL) { - terrno = TSDB_CODE_VND_COL_NOT_EXISTS; - goto _err; - } - if (pColumn->colId == 0) { - terrno = TSDB_CODE_VND_INVALID_TABLE_ACTION; - goto _err; - } - if (tqCheckColModifiable(pMeta->pVnode->pTq, uid, pColumn->colId) != 0) { - terrno = TSDB_CODE_VND_COL_SUBSCRIBED; - goto _err; - } - bool hasPrimayKey = false; - if (pSchema->nCols >= 2) { - hasPrimayKey = pSchema->pSchema[1].flags & COL_IS_KEY ? true : false; - } - - memcpy(&tScheam, pColumn, sizeof(SSchema)); - pSchema->version++; - tlen = (pSchema->nCols - iCol - 1) * sizeof(SSchema); - if (tlen) { - memmove(pColumn, pColumn + 1, tlen); - } - pSchema->nCols--; - - --pMeta->pVnode->config.vndStats.numOfNTimeSeries; - - if (!TSDB_CACHE_NO(pMeta->pVnode->config)) { - int16_t cid = pColumn->colId; - - if (tsdbCacheDropNTableColumn(pMeta->pVnode->pTsdb, entry.uid, cid, hasPrimayKey) != 0) { - metaError("vgId:%d, failed to drop ntable column:%s uid:%" PRId64, TD_VID(pMeta->pVnode), entry.name, - entry.uid); - } - } - - if (updataTableColCmpr(&entry.colCmpr, &tScheam, 0, 0) != 0) { - metaError("vgId:%d, failed to update table col cmpr:%s uid:%" PRId64, TD_VID(pMeta->pVnode), entry.name, - entry.uid); - } - if (entry.colCmpr.nCols != pSchema->nCols) { - terrno = TSDB_CODE_VND_INVALID_TABLE_ACTION; - goto _err; - } - break; - case TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES: - if (pColumn == NULL) { - terrno = TSDB_CODE_VND_COL_NOT_EXISTS; - goto _err; - } - if (!IS_VAR_DATA_TYPE(pColumn->type) || pColumn->bytes >= pAlterTbReq->colModBytes) { - terrno = TSDB_CODE_VND_INVALID_TABLE_ACTION; - goto _err; - } - if (rowLen + pAlterTbReq->colModBytes - pColumn->bytes > TSDB_MAX_BYTES_PER_ROW) { - terrno = TSDB_CODE_PAR_INVALID_ROW_LENGTH; - goto _err; - } - if (tqCheckColModifiable(pMeta->pVnode->pTq, uid, pColumn->colId) != 0) { - terrno = TSDB_CODE_VND_COL_SUBSCRIBED; - goto _err; - } - pSchema->version++; - pColumn->bytes = pAlterTbReq->colModBytes; - break; - case TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME: - if (pAlterTbReq->colNewName == NULL) { - terrno = TSDB_CODE_INVALID_MSG; - goto _err; - } - if (pColumn == NULL) { - terrno = TSDB_CODE_VND_COL_NOT_EXISTS; - goto _err; - } - if (tqCheckColModifiable(pMeta->pVnode->pTq, uid, pColumn->colId) != 0) { - terrno = TSDB_CODE_VND_COL_SUBSCRIBED; - goto _err; - } - pSchema->version++; - tstrncpy(pColumn->name, pAlterTbReq->colNewName, strlen(pAlterTbReq->colNewName) + 1); - break; + if (iColumn == pSchema->nCols) { + metaError("vgId:%d, %s failed at %s:%d since column %s not found in table %s, version:%" PRId64, + TD_VID(pMeta->pVnode), __func__, __FILE__, __LINE__, pReq->colName, pReq->tbName, version); + metaFetchEntryFree(&pEntry); + TAOS_RETURN(TSDB_CODE_VND_COL_NOT_EXISTS); } - if (!TSDB_CACHE_NO(pMeta->pVnode->config)) { - tsdbCacheInvalidateSchema(pMeta->pVnode->pTsdb, 0, entry.uid, pSchema->version); + if (pColumn->colId == 0 || pColumn->flags & COL_IS_KEY) { + metaError("vgId:%d, %s failed at %s:%d since column %s is primary key, version:%" PRId64, TD_VID(pMeta->pVnode), + __func__, __FILE__, __LINE__, pReq->colName, version); + metaFetchEntryFree(&pEntry); + TAOS_RETURN(TSDB_CODE_VND_INVALID_TABLE_ACTION); } - entry.version = version; + if (tqCheckColModifiable(pMeta->pVnode->pTq, pEntry->uid, pColumn->colId) != 0) { + metaError("vgId:%d, %s failed at %s:%d since column %s is not modifiable, version:%" PRId64, TD_VID(pMeta->pVnode), + __func__, __FILE__, __LINE__, pReq->colName, version); + metaFetchEntryFree(&pEntry); + TAOS_RETURN(TSDB_CODE_VND_INVALID_TABLE_ACTION); + } + tColumn = *pColumn; - // do actual write - metaWLock(pMeta); - - if (metaDeleteNcolIdx(pMeta, &oldEntry) < 0) { - metaError("vgId:%d, failed to delete ncol idx:%s uid:%" PRId64, TD_VID(pMeta->pVnode), entry.name, entry.uid); + // do drop column + pEntry->version = version; + pSchema->nCols--; + pSchema->version++; + if (pSchema->nCols - iColumn - 1 > 0) { + memmove(pColumn, pColumn + 1, (pSchema->nCols - iColumn - 1) * sizeof(SSchema)); + } + code = updataTableColCmpr(&pEntry->colCmpr, &tColumn, 0, 0); + if (code) { + metaError("vgId:%d, %s failed at %s:%d since %s, version:%" PRId64, TD_VID(pMeta->pVnode), __func__, __FILE__, + __LINE__, tstrerror(code), version); + metaFetchEntryFree(&pEntry); + TAOS_RETURN(code); + } + if (pEntry->colCmpr.nCols != pSchema->nCols) { + metaError("vgId:%d, %s failed at %s:%d since column count mismatch, version:%" PRId64, TD_VID(pMeta->pVnode), + __func__, __FILE__, __LINE__, version); + metaFetchEntryFree(&pEntry); + TAOS_RETURN(TSDB_CODE_VND_INVALID_TABLE_ACTION); } - if (metaUpdateNcolIdx(pMeta, &entry) < 0) { - metaError("vgId:%d, failed to update ncol idx:%s uid:%" PRId64, TD_VID(pMeta->pVnode), entry.name, entry.uid); + // do handle entry + code = metaHandleEntry2(pMeta, pEntry); + if (code) { + metaError("vgId:%d, %s failed at %s:%d since %s, uid:%" PRId64 " name:%s version:%" PRId64, TD_VID(pMeta->pVnode), + __func__, __FILE__, __LINE__, tstrerror(code), pEntry->uid, pReq->tbName, version); + metaFetchEntryFree(&pEntry); + } else { + metaInfo("vgId:%d, table %s uid %" PRId64 " is updated, version:%" PRId64, TD_VID(pMeta->pVnode), pReq->tbName, + pEntry->uid, version); } - // save to table db - if (metaSaveToTbDb(pMeta, &entry) < 0) { - metaError("vgId:%d, failed to save to tb db:%s uid:%" PRId64, TD_VID(pMeta->pVnode), entry.name, entry.uid); + // build response + if (metaUpdateMetaRsp(pEntry->uid, pReq->tbName, pSchema, pRsp) < 0) { + metaError("vgId:%d, %s failed at %s:%d since %s, uid:%" PRId64 " name:%s version:%" PRId64, TD_VID(pMeta->pVnode), + __func__, __FILE__, __LINE__, tstrerror(code), pEntry->uid, pReq->tbName, version); + } else { + for (int32_t i = 0; i < pEntry->colCmpr.nCols; i++) { + SColCmpr *p = &pEntry->colCmpr.pColCmpr[i]; + pRsp->pSchemaExt[i].colId = p->id; + pRsp->pSchemaExt[i].compress = p->alg; + } } - if (metaUpdateUidIdx(pMeta, &entry) < 0) { - metaError("vgId:%d, failed to update uid idx:%s uid:%" PRId64, TD_VID(pMeta->pVnode), entry.name, entry.uid); - } - - if (metaSaveToSkmDb(pMeta, &entry) < 0) { - metaError("vgId:%d, failed to save to skm db:%s uid:%" PRId64, TD_VID(pMeta->pVnode), entry.name, entry.uid); - } - - if (metaUpdateChangeTime(pMeta, entry.uid, pAlterTbReq->ctimeMs) < 0) { - metaError("vgId:%d, failed to update change time:%s uid:%" PRId64, TD_VID(pMeta->pVnode), entry.name, entry.uid); - } - - metaULock(pMeta); - - if (metaUpdateMetaRsp(uid, pAlterTbReq->tbName, pSchema, pMetaRsp) < 0) { - metaError("vgId:%d, failed to update meta rsp:%s uid:%" PRId64, TD_VID(pMeta->pVnode), entry.name, entry.uid); - } - for (int32_t i = 0; i < entry.colCmpr.nCols; i++) { - SColCmpr *p = &entry.colCmpr.pColCmpr[i]; - pMetaRsp->pSchemaExt[i].colId = p->id; - pMetaRsp->pSchemaExt[i].compress = p->alg; - } - - if (entry.pBuf) taosMemoryFree(entry.pBuf); - if (pNewSchema) taosMemoryFree(pNewSchema); - if (freeColCmpr) taosMemoryFree(entry.colCmpr.pColCmpr); - - tdbTbcClose(pTbDbc); - tdbTbcClose(pUidIdxc); - tDecoderClear(&dc); - - return 0; - -_err: - if (entry.pBuf) taosMemoryFree(entry.pBuf); - tdbTbcClose(pTbDbc); - tdbTbcClose(pUidIdxc); - tDecoderClear(&dc); - - return terrno != 0 ? terrno : TSDB_CODE_FAILED; -#endif + metaFetchEntryFree(&pEntry); + TAOS_RETURN(code); } int32_t metaAlterTableColumnName(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq, STableMetaRsp *pRsp) { From 823f3b2137890248d1ba4e1973511cb331fd833b Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 12 Dec 2024 18:51:10 +0800 Subject: [PATCH 28/45] fix: ci case --- source/dnode/vnode/src/meta/metaEntry2.c | 6 ++ source/dnode/vnode/src/meta/metaTable.c | 2 +- source/dnode/vnode/src/meta/metaTable2.c | 111 ++++++++++++++++++++--- 3 files changed, 107 insertions(+), 12 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaEntry2.c b/source/dnode/vnode/src/meta/metaEntry2.c index 15cc2dc525..cc0859f883 100644 --- a/source/dnode/vnode/src/meta/metaEntry2.c +++ b/source/dnode/vnode/src/meta/metaEntry2.c @@ -1319,6 +1319,12 @@ static int32_t metaHandleNormalTableUpdate(SMeta *pMeta, const SMetaEntry *pEntr tsdbCacheInvalidateSchema(pMeta->pVnode->pTsdb, 0, entry.uid, pSchema->version); } } + { // for update column bytes + + if (!TSDB_CACHE_NO(pMeta->pVnode->config)) { + tsdbCacheInvalidateSchema(pMeta->pVnode->pTsdb, 0, entry.uid, pSchema->version); + } + } #endif tsdbCacheInvalidateSchema(pMeta->pVnode->pTsdb, 0, pEntry->uid, pEntry->ntbEntry.schemaRow.version); } diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index cff067f4bb..202b0454a9 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -21,7 +21,6 @@ int32_t metaAddTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq, ST int32_t metaDropTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq, STableMetaRsp *pRsp); int32_t metaAlterTableColumnName(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq, STableMetaRsp *pRsp); int32_t metaAlterTableColumnBytes(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq, STableMetaRsp *pRsp); -int32_t metaAlterTableColumnCompressOption(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq, STableMetaRsp *pRsp); int32_t metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSchema *pSchema); @@ -2991,6 +2990,7 @@ int metaAlterTable(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq, STableMeta case TSDB_ALTER_TABLE_DROP_COLUMN: return metaDropTableColumn(pMeta, version, pReq, pMetaRsp); case TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES: + return metaAlterTableColumnBytes(pMeta, version, pReq, pMetaRsp); case TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME: return metaAlterTableColumn(pMeta, version, pReq, pMetaRsp); case TSDB_ALTER_TABLE_UPDATE_TAG_VAL: diff --git a/source/dnode/vnode/src/meta/metaTable2.c b/source/dnode/vnode/src/meta/metaTable2.c index cfad79161b..fce2de7c5d 100644 --- a/source/dnode/vnode/src/meta/metaTable2.c +++ b/source/dnode/vnode/src/meta/metaTable2.c @@ -733,7 +733,7 @@ int32_t metaDropTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq, S SSchema *pColumn = NULL; SSchema tColumn; int32_t iColumn = 0; - for (int32_t iColumn = 0; iColumn < pSchema->nCols; iColumn++) { + for (; iColumn < pSchema->nCols; iColumn++) { pColumn = &pSchema->pSchema[iColumn]; if (strncmp(pColumn->name, pReq->colName, TSDB_COL_NAME_LEN) == 0) { break; @@ -764,11 +764,11 @@ int32_t metaDropTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq, S // do drop column pEntry->version = version; - pSchema->nCols--; - pSchema->version++; if (pSchema->nCols - iColumn - 1 > 0) { memmove(pColumn, pColumn + 1, (pSchema->nCols - iColumn - 1) * sizeof(SSchema)); } + pSchema->nCols--; + pSchema->version++; code = updataTableColCmpr(&pEntry->colCmpr, &tColumn, 0, 0); if (code) { metaError("vgId:%d, %s failed at %s:%d since %s, version:%" PRId64, TD_VID(pMeta->pVnode), __func__, __FILE__, @@ -818,12 +818,101 @@ int32_t metaAlterTableColumnName(SMeta *pMeta, int64_t version, SVAlterTbReq *pR int32_t metaAlterTableColumnBytes(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq, STableMetaRsp *pRsp) { int32_t code = TSDB_CODE_SUCCESS; - // TODO - return code; -} -int32_t metaAlterTableColumnCompressOption(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq, STableMetaRsp *pRsp) { - int32_t code = TSDB_CODE_SUCCESS; - // TODO - return code; -} \ No newline at end of file + // check request + code = metaCheckAlterTableColumnReq(pMeta, version, pReq); + if (code) { + TAOS_RETURN(code); + } + + // fetch old entry + SMetaEntry *pEntry = NULL; + code = metaFetchEntryByName(pMeta, pReq->tbName, &pEntry); + if (code) { + metaError("vgId:%d, %s failed at %s:%d since table %s not found, version:%" PRId64, TD_VID(pMeta->pVnode), __func__, + __FILE__, __LINE__, pReq->tbName, version); + TAOS_RETURN(code); + } + + if (pEntry->version >= version) { + metaError("vgId:%d, %s failed at %s:%d since table %s version %" PRId64 " is not less than %" PRId64, + TD_VID(pMeta->pVnode), __func__, __FILE__, __LINE__, pReq->tbName, pEntry->version, version); + metaFetchEntryFree(&pEntry); + TAOS_RETURN(TSDB_CODE_INVALID_PARA); + } + + // search the column to update + SSchemaWrapper *pSchema = &pEntry->ntbEntry.schemaRow; + SSchema *pColumn = NULL; + int32_t iColumn = 0; + int32_t rowSize = 0; + for (int32_t i = 0; i < pSchema->nCols; i++) { + if (pSchema->pSchema[i].colId == pReq->colId) { + pColumn = &pSchema->pSchema[i]; + iColumn = i; + } + rowSize += pSchema->pSchema[i].bytes; + } + + if (NULL == pColumn) { + metaError("vgId:%d, %s failed at %s:%d since column %s not found in table %s, version:%" PRId64, + TD_VID(pMeta->pVnode), __func__, __FILE__, __LINE__, pReq->colName, pReq->tbName, version); + metaFetchEntryFree(&pEntry); + TAOS_RETURN(TSDB_CODE_VND_COL_NOT_EXISTS); + } + + if (!IS_VAR_DATA_TYPE(pColumn->type) || pColumn->bytes >= pReq->colModBytes) { + metaError("vgId:%d, %s failed at %s:%d since column %s is not var data type or bytes %d >= %d, version:%" PRId64, + TD_VID(pMeta->pVnode), __func__, __FILE__, __LINE__, pReq->colName, pColumn->bytes, pReq->colModBytes, + version); + metaFetchEntryFree(&pEntry); + TAOS_RETURN(TSDB_CODE_VND_INVALID_TABLE_ACTION); + } + + if (tqCheckColModifiable(pMeta->pVnode->pTq, pEntry->uid, pColumn->colId) != 0) { + metaError("vgId:%d, %s failed at %s:%d since column %s is not modifiable, version:%" PRId64, TD_VID(pMeta->pVnode), + __func__, __FILE__, __LINE__, pReq->colName, version); + metaFetchEntryFree(&pEntry); + TAOS_RETURN(TSDB_CODE_VND_COL_SUBSCRIBED); + } + + if (rowSize + pReq->colModBytes - pColumn->bytes > TSDB_MAX_BYTES_PER_ROW) { + metaError("vgId:%d, %s failed at %s:%d since row size %d + %d - %d > %d, version:%" PRId64, TD_VID(pMeta->pVnode), + __func__, __FILE__, __LINE__, rowSize, pReq->colModBytes, pColumn->bytes, TSDB_MAX_BYTES_PER_ROW, + version); + metaFetchEntryFree(&pEntry); + TAOS_RETURN(TSDB_CODE_PAR_INVALID_ROW_LENGTH); + } + + // do change the column bytes + pEntry->version = version; + pSchema->version++; + pColumn->bytes = pReq->colModBytes; + + // do handle entry + code = metaHandleEntry2(pMeta, pEntry); + if (code) { + metaError("vgId:%d, %s failed at %s:%d since %s, uid:%" PRId64 " name:%s version:%" PRId64, TD_VID(pMeta->pVnode), + __func__, __FILE__, __LINE__, tstrerror(code), pEntry->uid, pReq->tbName, version); + metaFetchEntryFree(&pEntry); + TAOS_RETURN(code); + } else { + metaInfo("vgId:%d, table %s uid %" PRId64 " is updated, version:%" PRId64, TD_VID(pMeta->pVnode), pReq->tbName, + pEntry->uid, version); + } + + // build response + if (metaUpdateMetaRsp(pEntry->uid, pReq->tbName, pSchema, pRsp) < 0) { + metaError("vgId:%d, %s failed at %s:%d since %s, uid:%" PRId64 " name:%s version:%" PRId64, TD_VID(pMeta->pVnode), + __func__, __FILE__, __LINE__, tstrerror(code), pEntry->uid, pReq->tbName, version); + } else { + for (int32_t i = 0; i < pEntry->colCmpr.nCols; i++) { + SColCmpr *p = &pEntry->colCmpr.pColCmpr[i]; + pRsp->pSchemaExt[i].colId = p->id; + pRsp->pSchemaExt[i].compress = p->alg; + } + } + + metaFetchEntryFree(&pEntry); + TAOS_RETURN(code); +} From 3aebca5404e80176adcf46135dab7e169023db11 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 12 Dec 2024 19:05:39 +0800 Subject: [PATCH 29/45] feat: update table column name --- source/dnode/vnode/src/meta/metaEntry2.c | 5 -- source/dnode/vnode/src/meta/metaTable.c | 2 +- source/dnode/vnode/src/meta/metaTable2.c | 88 +++++++++++++++++++++++- 3 files changed, 87 insertions(+), 8 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaEntry2.c b/source/dnode/vnode/src/meta/metaEntry2.c index cc0859f883..b180f52927 100644 --- a/source/dnode/vnode/src/meta/metaEntry2.c +++ b/source/dnode/vnode/src/meta/metaEntry2.c @@ -1319,11 +1319,6 @@ static int32_t metaHandleNormalTableUpdate(SMeta *pMeta, const SMetaEntry *pEntr tsdbCacheInvalidateSchema(pMeta->pVnode->pTsdb, 0, entry.uid, pSchema->version); } } - { // for update column bytes - - if (!TSDB_CACHE_NO(pMeta->pVnode->config)) { - tsdbCacheInvalidateSchema(pMeta->pVnode->pTsdb, 0, entry.uid, pSchema->version); - } } #endif tsdbCacheInvalidateSchema(pMeta->pVnode->pTsdb, 0, pEntry->uid, pEntry->ntbEntry.schemaRow.version); diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 202b0454a9..145f506c7e 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -2992,7 +2992,7 @@ int metaAlterTable(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq, STableMeta case TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES: return metaAlterTableColumnBytes(pMeta, version, pReq, pMetaRsp); case TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME: - return metaAlterTableColumn(pMeta, version, pReq, pMetaRsp); + return metaAlterTableColumnName(pMeta, version, pReq, pMetaRsp); case TSDB_ALTER_TABLE_UPDATE_TAG_VAL: return metaUpdateTableTagVal(pMeta, version, pReq); case TSDB_ALTER_TABLE_UPDATE_MULTI_TAG_VAL: diff --git a/source/dnode/vnode/src/meta/metaTable2.c b/source/dnode/vnode/src/meta/metaTable2.c index fce2de7c5d..97d5e8903e 100644 --- a/source/dnode/vnode/src/meta/metaTable2.c +++ b/source/dnode/vnode/src/meta/metaTable2.c @@ -812,8 +812,92 @@ int32_t metaDropTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq, S int32_t metaAlterTableColumnName(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq, STableMetaRsp *pRsp) { int32_t code = TSDB_CODE_SUCCESS; - // TODO - return code; + + // check request + code = metaCheckAlterTableColumnReq(pMeta, version, pReq); + if (code) { + TAOS_RETURN(code); + } + + if (NULL == pReq->colNewName) { + metaError("vgId:%d, %s failed at %s:%d since invalid new column name, version:%" PRId64, TD_VID(pMeta->pVnode), + __func__, __FILE__, __LINE__, version); + TAOS_RETURN(TSDB_CODE_INVALID_MSG); + } + + // fetch old entry + SMetaEntry *pEntry = NULL; + code = metaFetchEntryByName(pMeta, pReq->tbName, &pEntry); + if (code) { + metaError("vgId:%d, %s failed at %s:%d since table %s not found, version:%" PRId64, TD_VID(pMeta->pVnode), __func__, + __FILE__, __LINE__, pReq->tbName, version); + TAOS_RETURN(code); + } + + if (pEntry->version >= version) { + metaError("vgId:%d, %s failed at %s:%d since table %s version %" PRId64 " is not less than %" PRId64, + TD_VID(pMeta->pVnode), __func__, __FILE__, __LINE__, pReq->tbName, pEntry->version, version); + metaFetchEntryFree(&pEntry); + TAOS_RETURN(TSDB_CODE_INVALID_PARA); + } + + // search the column to update + SSchemaWrapper *pSchema = &pEntry->ntbEntry.schemaRow; + SSchema *pColumn = NULL; + int32_t iColumn = 0; + for (int32_t i = 0; i < pSchema->nCols; i++) { + if (pSchema->pSchema[i].colId == pReq->colId) { + pColumn = &pSchema->pSchema[i]; + iColumn = i; + break; + } + } + + if (NULL == pColumn) { + metaError("vgId:%d, %s failed at %s:%d since column id %" PRId64 " not found in table %s, version:%" PRId64, + TD_VID(pMeta->pVnode), __func__, __FILE__, __LINE__, pReq->colId, pReq->tbName, version); + metaFetchEntryFree(&pEntry); + TAOS_RETURN(TSDB_CODE_VND_COL_NOT_EXISTS); + } + + if (tqCheckColModifiable(pMeta->pVnode->pTq, pEntry->uid, pColumn->colId) != 0) { + metaError("vgId:%d, %s failed at %s:%d since column %s is not modifiable, version:%" PRId64, TD_VID(pMeta->pVnode), + __func__, __FILE__, __LINE__, pColumn->name, version); + metaFetchEntryFree(&pEntry); + TAOS_RETURN(TSDB_CODE_VND_COL_SUBSCRIBED); + } + + // do update column name + pEntry->version = version; + tstrncpy(pColumn->name, pReq->colNewName, TSDB_COL_NAME_LEN); + pSchema->version++; + + // do handle entry + code = metaHandleEntry2(pMeta, pEntry); + if (code) { + metaError("vgId:%d, %s failed at %s:%d since %s, uid:%" PRId64 " name:%s version:%" PRId64, TD_VID(pMeta->pVnode), + __func__, __FILE__, __LINE__, tstrerror(code), pEntry->uid, pReq->tbName, version); + metaFetchEntryFree(&pEntry); + TAOS_RETURN(code); + } else { + metaInfo("vgId:%d, table %s uid %" PRId64 " is updated, version:%" PRId64, TD_VID(pMeta->pVnode), pReq->tbName, + pEntry->uid, version); + } + + // build response + if (metaUpdateMetaRsp(pEntry->uid, pReq->tbName, pSchema, pRsp) < 0) { + metaError("vgId:%d, %s failed at %s:%d since %s, uid:%" PRId64 " name:%s version:%" PRId64, TD_VID(pMeta->pVnode), + __func__, __FILE__, __LINE__, tstrerror(code), pEntry->uid, pReq->tbName, version); + } else { + for (int32_t i = 0; i < pEntry->colCmpr.nCols; i++) { + SColCmpr *p = &pEntry->colCmpr.pColCmpr[i]; + pRsp->pSchemaExt[i].colId = p->id; + pRsp->pSchemaExt[i].compress = p->alg; + } + } + + metaFetchEntryFree(&pEntry); + TAOS_RETURN(code); } int32_t metaAlterTableColumnBytes(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq, STableMetaRsp *pRsp) { From 8c7b8a8afa9564487a5307a7d23af3908f0bcabb Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 12 Dec 2024 20:33:55 +0800 Subject: [PATCH 30/45] feat: refact alter child table tag --- source/dnode/vnode/src/meta/metaEntry2.c | 266 ++++++++++++++++++----- source/dnode/vnode/src/meta/metaOpen.c | 4 +- source/dnode/vnode/src/meta/metaTable.c | 3 +- source/dnode/vnode/src/meta/metaTable2.c | 168 ++++++++++++++ 4 files changed, 386 insertions(+), 55 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaEntry2.c b/source/dnode/vnode/src/meta/metaEntry2.c index b180f52927..393c8dda21 100644 --- a/source/dnode/vnode/src/meta/metaEntry2.c +++ b/source/dnode/vnode/src/meta/metaEntry2.c @@ -16,6 +16,7 @@ void metaDestroyTagIdxKey(STagIdxKey *pTagIdxKey); int metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSchema *pSchema); int metaDelJsonVarFromIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSchema *pSchema); void metaTimeSeriesNotifyCheck(SMeta *pMeta); +int tagIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2); #define metaErr(VGID, ERRNO) \ do { \ @@ -457,7 +458,16 @@ static int32_t metaChildIdxInsert(SMeta *pMeta, const SMetaHandleParam *pParam) } static int32_t metaChildIdxUpdate(SMeta *pMeta, const SMetaHandleParam *pParam) { - return metaChildIdxUpsert(pMeta, pParam, META_TABLE_OP_UPDATA); + const SMetaEntry *pEntry = pParam->pEntry; + const SMetaEntry *pOldEntry = pParam->pOldEntry; + const SMetaEntry *pSuperEntry = pParam->pSuperEntry; + + const STag *pNewTags = (const STag *)pEntry->ctbEntry.pTags; + const STag *pOldTags = (const STag *)pOldEntry->ctbEntry.pTags; + if (pNewTags->len != pOldTags->len || memcmp(pNewTags, pOldTags, pNewTags->len)) { + return metaChildIdxUpsert(pMeta, pParam, META_TABLE_OP_UPDATA); + } + return 0; } static int32_t metaChildIdxDelete(SMeta *pMeta, const SMetaHandleParam *pParam) { @@ -471,6 +481,50 @@ static int32_t metaChildIdxDelete(SMeta *pMeta, const SMetaHandleParam *pParam) } // Tag Index +static int32_t metaFetchTagIdxKey(SMeta *pMeta, const SMetaEntry *pEntry, const SSchema *pTagColumn, + STagIdxKey **ppTagIdxKey, int32_t *pTagIdxKeySize) { + int32_t code = TSDB_CODE_SUCCESS; + + STagIdxKey *pTagIdxKey = NULL; + int32_t nTagIdxKey; + const void *pTagData = NULL; + int32_t nTagData = 0; + + STagVal tagVal = { + .cid = pTagColumn->colId, + }; + + if (tTagGet((const STag *)pEntry->ctbEntry.pTags, &tagVal)) { + if (IS_VAR_DATA_TYPE(pTagColumn->type)) { + pTagData = tagVal.pData; + nTagData = (int32_t)tagVal.nData; + } else { + pTagData = &(tagVal.i64); + nTagData = tDataTypes[pTagColumn->type].bytes; + } + } else { + if (!IS_VAR_DATA_TYPE(pTagColumn->type)) { + nTagData = tDataTypes[pTagColumn->type].bytes; + } + } + + code = metaCreateTagIdxKey(pEntry->ctbEntry.suid, pTagColumn->colId, pTagData, nTagData, pTagColumn->type, + pEntry->uid, &pTagIdxKey, &nTagIdxKey); + if (code) { + metaErr(TD_VID(pMeta->pVnode), code); + return code; + } + + *ppTagIdxKey = pTagIdxKey; + *pTagIdxKeySize = nTagIdxKey; + return code; +} + +static void metaFetchTagIdxKeyFree(STagIdxKey **ppTagIdxKey) { + metaDestroyTagIdxKey(*ppTagIdxKey); + *ppTagIdxKey = NULL; +} + static int32_t metaTagIdxInsert(SMeta *pMeta, const SMetaHandleParam *pParam) { int32_t code = TSDB_CODE_SUCCESS; @@ -495,34 +549,13 @@ static int32_t metaTagIdxInsert(SMeta *pMeta, const SMetaHandleParam *pParam) { for (int32_t i = 0; i < pTagSchema->nCols; i++) { STagIdxKey *pTagIdxKey = NULL; int32_t nTagIdxKey; - const void *pTagData = NULL; - int32_t nTagData = 0; const SSchema *pTagColumn = &pTagSchema->pSchema[i]; if (!IS_IDX_ON(pTagColumn)) { continue; } - STagVal tagVal = { - .cid = pTagColumn->colId, - }; - - if (tTagGet((const STag *)pEntry->ctbEntry.pTags, &tagVal)) { - if (IS_VAR_DATA_TYPE(pTagColumn->type)) { - pTagData = tagVal.pData; - nTagData = (int32_t)tagVal.nData; - } else { - pTagData = &(tagVal.i64); - nTagData = tDataTypes[pTagColumn->type].bytes; - } - } else { - if (!IS_VAR_DATA_TYPE(pTagColumn->type)) { - nTagData = tDataTypes[pTagColumn->type].bytes; - } - } - - code = metaCreateTagIdxKey(pSuperEntry->uid, pTagColumn->colId, pTagData, nTagData, pTagColumn->type, pEntry->uid, - &pTagIdxKey, &nTagIdxKey); + code = metaFetchTagIdxKey(pMeta, pEntry, pTagColumn, &pTagIdxKey, &nTagIdxKey); if (code) { metaErr(TD_VID(pMeta->pVnode), code); return code; @@ -531,19 +564,90 @@ static int32_t metaTagIdxInsert(SMeta *pMeta, const SMetaHandleParam *pParam) { code = tdbTbInsert(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, NULL, 0, pMeta->txn); if (code) { metaErr(TD_VID(pMeta->pVnode), code); - metaDestroyTagIdxKey(pTagIdxKey); + metaFetchTagIdxKeyFree(&pTagIdxKey); return code; } - metaDestroyTagIdxKey(pTagIdxKey); - pTagIdxKey = NULL; + metaFetchTagIdxKeyFree(&pTagIdxKey); } } return code; } static int32_t metaTagIdxUpdate(SMeta *pMeta, const SMetaHandleParam *pParam) { - // TODO - return 0; + int32_t code = TSDB_CODE_SUCCESS; + + const SMetaEntry *pEntry = pParam->pEntry; + const SMetaEntry *pOldEntry = pParam->pOldEntry; + const SMetaEntry *pSuperEntry = pParam->pSuperEntry; + const SSchemaWrapper *pTagSchema = &pSuperEntry->stbEntry.schemaTag; + const STag *pNewTags = (const STag *)pEntry->ctbEntry.pTags; + const STag *pOldTags = (const STag *)pOldEntry->ctbEntry.pTags; + + if (pNewTags->len == pOldTags->len && !memcmp(pNewTags, pOldTags, pNewTags->len)) { + return code; + } + + if (pTagSchema->nCols == 1 && pTagSchema->pSchema[0].type == TSDB_DATA_TYPE_JSON) { + code = metaDelJsonVarFromIdx(pMeta, pOldEntry, &pTagSchema->pSchema[0]); + if (code) { + metaErr(TD_VID(pMeta->pVnode), code); + return code; + } + + code = metaSaveJsonVarToIdx(pMeta, pEntry, &pTagSchema->pSchema[0]); + if (code) { + metaErr(TD_VID(pMeta->pVnode), code); + return code; + } + } else { + for (int32_t i = 0; i < pTagSchema->nCols; i++) { + const SSchema *pTagColumn = &pTagSchema->pSchema[i]; + + if (!IS_IDX_ON(pTagColumn)) { + continue; + } + + STagIdxKey *pOldTagIdxKey = NULL; + int32_t oldTagIdxKeySize = 0; + STagIdxKey *pNewTagIdxKey = NULL; + int32_t newTagIdxKeySize = 0; + + code = metaFetchTagIdxKey(pMeta, pOldEntry, pTagColumn, &pOldTagIdxKey, &oldTagIdxKeySize); + if (code) { + metaErr(TD_VID(pMeta->pVnode), code); + return code; + } + + code = metaFetchTagIdxKey(pMeta, pEntry, pTagColumn, &pNewTagIdxKey, &newTagIdxKeySize); + if (code) { + metaErr(TD_VID(pMeta->pVnode), code); + metaFetchTagIdxKeyFree(&pOldTagIdxKey); + return code; + } + + if (tagIdxKeyCmpr(pOldTagIdxKey, oldTagIdxKeySize, pNewTagIdxKey, newTagIdxKeySize)) { + code = tdbTbDelete(pMeta->pTagIdx, pOldTagIdxKey, oldTagIdxKeySize, pMeta->txn); + if (code) { + metaErr(TD_VID(pMeta->pVnode), code); + metaFetchTagIdxKeyFree(&pOldTagIdxKey); + metaFetchTagIdxKeyFree(&pNewTagIdxKey); + return code; + } + + code = tdbTbInsert(pMeta->pTagIdx, pNewTagIdxKey, newTagIdxKeySize, NULL, 0, pMeta->txn); + if (code) { + metaErr(TD_VID(pMeta->pVnode), code); + metaFetchTagIdxKeyFree(&pOldTagIdxKey); + metaFetchTagIdxKeyFree(&pNewTagIdxKey); + return code; + } + } + + metaFetchTagIdxKeyFree(&pOldTagIdxKey); + metaFetchTagIdxKeyFree(&pNewTagIdxKey); + } + } + return code; } static int32_t metaTagIdxDelete(SMeta *pMeta, const SMetaHandleParam *pParam) { @@ -565,7 +669,6 @@ static int32_t metaTagIdxDelete(SMeta *pMeta, const SMetaHandleParam *pParam) { } else { for (int32_t i = 0; i < pTagSchema->nCols; i++) { pTagColumn = &pTagSchema->pSchema[i]; - if (!IS_IDX_ON(pTagColumn)) { continue; } @@ -573,25 +676,7 @@ static int32_t metaTagIdxDelete(SMeta *pMeta, const SMetaHandleParam *pParam) { STagIdxKey *pTagIdxKey = NULL; int32_t nTagIdxKey; - const void *pTagData = NULL; - int32_t nTagData = 0; - - STagVal tagVal = {.cid = pTagColumn->colId}; - if (tTagGet(pTags, &tagVal)) { - if (IS_VAR_DATA_TYPE(pTagColumn->type)) { - pTagData = tagVal.pData; - nTagData = (int32_t)tagVal.nData; - } else { - pTagData = &(tagVal.i64); - nTagData = tDataTypes[pTagColumn->type].bytes; - } - } else { - if (!IS_VAR_DATA_TYPE(pTagColumn->type)) { - nTagData = tDataTypes[pTagColumn->type].bytes; - } - } - code = metaCreateTagIdxKey(pSuper->uid, pTagColumn->colId, pTagData, nTagData, pTagColumn->type, pChild->uid, - &pTagIdxKey, &nTagIdxKey); + code = metaFetchTagIdxKey(pMeta, pChild, pTagColumn, &pTagIdxKey, &nTagIdxKey); if (code) { metaErr(TD_VID(pMeta->pVnode), code); return code; @@ -600,11 +685,10 @@ static int32_t metaTagIdxDelete(SMeta *pMeta, const SMetaHandleParam *pParam) { code = tdbTbDelete(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, pMeta->txn); if (code) { metaErr(TD_VID(pMeta->pVnode), code); - metaDestroyTagIdxKey(pTagIdxKey); + metaFetchTagIdxKeyFree(&pTagIdxKey); return code; } - metaDestroyTagIdxKey(pTagIdxKey); - pTagIdxKey = NULL; + metaFetchTagIdxKeyFree(&pTagIdxKey); } } return code; @@ -1269,6 +1353,84 @@ static int32_t metaHandleNormalTableUpdateImpl(SMeta *pMeta, const SMetaHandlePa return code; } +static int32_t metaHandleChildTableUpdateImpl(SMeta *pMeta, const SMetaHandleParam *pParam) { + int32_t code = TSDB_CODE_SUCCESS; + + const SMetaEntry *pEntry = pParam->pEntry; + const SMetaEntry *pOldEntry = pParam->pOldEntry; + const SMetaEntry *pSuperEntry = pParam->pSuperEntry; + + SMetaTableOp ops[] = { + {META_ENTRY_TABLE, META_TABLE_OP_UPDATA}, // + {META_UID_IDX, META_TABLE_OP_UPDATA}, // + {META_TAG_IDX, META_TABLE_OP_UPDATA}, // + {META_CHILD_IDX, META_TABLE_OP_UPDATA}, // + }; + + for (int i = 0; i < sizeof(ops) / sizeof(ops[0]); i++) { + SMetaTableOp *op = &ops[i]; + code = metaTableOpFn[op->table][op->op](pMeta, pParam); + if (code) { + metaErr(TD_VID(pMeta->pVnode), code); + return code; + } + } + + if (metaUidCacheClear(pMeta, pSuperEntry->uid) < 0) { + metaErr(TD_VID(pMeta->pVnode), code); + } + + if (metaTbGroupCacheClear(pMeta, pSuperEntry->uid) < 0) { + metaErr(TD_VID(pMeta->pVnode), code); + } + return code; +#if 0 + if (metaUpdateChangeTime(pMeta, ctbEntry.uid, pReq->ctimeMs) < 0) { + metaError("meta/table: failed to update change time:%s uid:%" PRId64, ctbEntry.name, ctbEntry.uid); + } +#endif +} + +static int32_t metaHandleChildTableUpdate(SMeta *pMeta, const SMetaEntry *pEntry) { + int32_t code = TSDB_CODE_SUCCESS; + + SMetaEntry *pOldEntry = NULL; + SMetaEntry *pSuperEntry = NULL; + + code = metaFetchEntryByUid(pMeta, pEntry->uid, &pOldEntry); + if (code) { + metaErr(TD_VID(pMeta->pVnode), code); + return code; + } + + code = metaFetchEntryByUid(pMeta, pEntry->ctbEntry.suid, &pSuperEntry); + if (code) { + metaErr(TD_VID(pMeta->pVnode), code); + metaFetchEntryFree(&pOldEntry); + return code; + } + + SMetaHandleParam param = { + .pEntry = pEntry, + .pOldEntry = pOldEntry, + .pSuperEntry = pSuperEntry, + }; + + metaWLock(pMeta); + code = metaHandleChildTableUpdateImpl(pMeta, ¶m); + metaULock(pMeta); + if (code) { + metaErr(TD_VID(pMeta->pVnode), code); + metaFetchEntryFree(&pOldEntry); + metaFetchEntryFree(&pSuperEntry); + return code; + } + + metaFetchEntryFree(&pOldEntry); + metaFetchEntryFree(&pSuperEntry); + return code; +} + static int32_t metaHandleNormalTableUpdate(SMeta *pMeta, const SMetaEntry *pEntry) { int32_t code = TSDB_CODE_SUCCESS; SMetaEntry *pOldEntry = NULL; @@ -1413,7 +1575,7 @@ int32_t metaHandleEntry2(SMeta *pMeta, const SMetaEntry *pEntry) { } case TSDB_CHILD_TABLE: { if (isExist) { - // code = metaHandleChildTableUpdate(pMeta, pEntry); + code = metaHandleChildTableUpdate(pMeta, pEntry); } else { code = metaHandleChildTableCreate(pMeta, pEntry); } diff --git a/source/dnode/vnode/src/meta/metaOpen.c b/source/dnode/vnode/src/meta/metaOpen.c index 5351554631..0bced535cb 100644 --- a/source/dnode/vnode/src/meta/metaOpen.c +++ b/source/dnode/vnode/src/meta/metaOpen.c @@ -19,7 +19,7 @@ static int tbDbKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2); static int skmDbKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2); static int ctbIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2); -static int tagIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2); +int tagIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2); static int uidIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2); static int smaIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2); static int taskIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2); @@ -598,7 +598,7 @@ static int ctbIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kL return 0; } -static int tagIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2) { +int tagIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2) { STagIdxKey *pTagIdxKey1 = (STagIdxKey *)pKey1; STagIdxKey *pTagIdxKey2 = (STagIdxKey *)pKey2; tb_uid_t uid1 = 0, uid2 = 0; diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 145f506c7e..d6917f9ab5 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -21,6 +21,7 @@ int32_t metaAddTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq, ST int32_t metaDropTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq, STableMetaRsp *pRsp); int32_t metaAlterTableColumnName(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq, STableMetaRsp *pRsp); int32_t metaAlterTableColumnBytes(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq, STableMetaRsp *pRsp); +int32_t metaUpdateTableTagValue(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq); int32_t metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSchema *pSchema); @@ -2994,7 +2995,7 @@ int metaAlterTable(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq, STableMeta case TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME: return metaAlterTableColumnName(pMeta, version, pReq, pMetaRsp); case TSDB_ALTER_TABLE_UPDATE_TAG_VAL: - return metaUpdateTableTagVal(pMeta, version, pReq); + return metaUpdateTableTagValue(pMeta, version, pReq); case TSDB_ALTER_TABLE_UPDATE_MULTI_TAG_VAL: return metaUpdateTableMultiTagVal(pMeta, version, pReq); return terrno = TSDB_CODE_VND_INVALID_TABLE_ACTION; diff --git a/source/dnode/vnode/src/meta/metaTable2.c b/source/dnode/vnode/src/meta/metaTable2.c index 97d5e8903e..f5aa5f04bf 100644 --- a/source/dnode/vnode/src/meta/metaTable2.c +++ b/source/dnode/vnode/src/meta/metaTable2.c @@ -1000,3 +1000,171 @@ int32_t metaAlterTableColumnBytes(SMeta *pMeta, int64_t version, SVAlterTbReq *p metaFetchEntryFree(&pEntry); TAOS_RETURN(code); } + +static int32_t metaCheckUpdateTableTagValReq(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq) { + int32_t code = 0; + + // check tag name + if (NULL == pReq->tagName || strlen(pReq->tagName) == 0) { + metaError("vgId:%d, %s failed at %s:%d since invalid tag name:%s, version:%" PRId64, TD_VID(pMeta->pVnode), + __func__, __FILE__, __LINE__, pReq->tagName, version); + TAOS_RETURN(TSDB_CODE_INVALID_MSG); + } + + // check name + void *value = NULL; + int32_t valueSize = 0; + code = tdbTbGet(pMeta->pNameIdx, pReq->tbName, strlen(pReq->tbName) + 1, &value, &valueSize); + if (code) { + metaError("vgId:%d, %s failed at %s:%d since table %s not found, version:%" PRId64, TD_VID(pMeta->pVnode), __func__, + __FILE__, __LINE__, pReq->tbName, version); + code = TSDB_CODE_TDB_TABLE_NOT_EXIST; + TAOS_RETURN(code); + } + + TAOS_RETURN(code); +} + +int32_t metaUpdateTableTagValue(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq) { + int32_t code = TSDB_CODE_SUCCESS; + + // check request + code = metaCheckUpdateTableTagValReq(pMeta, version, pReq); + if (code) { + TAOS_RETURN(code); + } + + // fetch child entry + SMetaEntry *pChild = NULL; + code = metaFetchEntryByName(pMeta, pReq->tbName, &pChild); + if (code) { + metaError("vgId:%d, %s failed at %s:%d since table %s not found, version:%" PRId64, TD_VID(pMeta->pVnode), __func__, + __FILE__, __LINE__, pReq->tbName, version); + TAOS_RETURN(code); + } + + if (pChild->type != TSDB_CHILD_TABLE) { + metaError("vgId:%d, %s failed at %s:%d since table %s is not a child table, version:%" PRId64, + TD_VID(pMeta->pVnode), __func__, __FILE__, __LINE__, pReq->tbName, version); + metaFetchEntryFree(&pChild); + TAOS_RETURN(TSDB_CODE_VND_INVALID_TABLE_ACTION); + } + + // fetch super entry + SMetaEntry *pSuper = NULL; + code = metaFetchEntryByUid(pMeta, pChild->ctbEntry.suid, &pSuper); + if (code) { + metaError("vgId:%d, %s failed at %s:%d since super table uid %" PRId64 " not found, version:%" PRId64, + TD_VID(pMeta->pVnode), __func__, __FILE__, __LINE__, pChild->ctbEntry.suid, version); + metaFetchEntryFree(&pChild); + TAOS_RETURN(TSDB_CODE_INTERNAL_ERROR); + } + + // do change tag value + SSchemaWrapper *pTagSchema = &pSuper->stbEntry.schemaTag; + SSchema *pColumn = NULL; + int32_t iColumn = 0; + for (int32_t i = 0; i < pTagSchema->nCols; i++) { + if (strncmp(pTagSchema->pSchema[i].name, pReq->tagName, TSDB_COL_NAME_LEN) == 0) { + pColumn = &pTagSchema->pSchema[i]; + iColumn = i; + break; + } + } + + if (NULL == pColumn) { + metaError("vgId:%d, %s failed at %s:%d since tag %s not found in table %s, version:%" PRId64, TD_VID(pMeta->pVnode), + __func__, __FILE__, __LINE__, pReq->tagName, pReq->tbName, version); + metaFetchEntryFree(&pChild); + metaFetchEntryFree(&pSuper); + TAOS_RETURN(TSDB_CODE_VND_COL_NOT_EXISTS); + } + + // do change tag value + pChild->version = version; + if (pTagSchema->nCols == 1 && pTagSchema->pSchema[0].type == TSDB_DATA_TYPE_JSON) { + void *pNewTag = taosMemoryRealloc(pChild->ctbEntry.pTags, pReq->nTagVal); + if (NULL == pNewTag) { + metaError("vgId:%d, %s failed at %s:%d since %s, version:%" PRId64, TD_VID(pMeta->pVnode), __func__, __FILE__, + __LINE__, tstrerror(terrno), version); + metaFetchEntryFree(&pChild); + metaFetchEntryFree(&pSuper); + TAOS_RETURN(terrno); + } + pChild->ctbEntry.pTags = pNewTag; + memcpy(pChild->ctbEntry.pTags, pReq->pTagVal, pReq->nTagVal); + } else { + STag *pOldTag = (STag *)pChild->ctbEntry.pTags; + + SArray *pTagArray = taosArrayInit(pTagSchema->nCols, sizeof(STagVal)); + if (NULL == pTagArray) { + metaError("vgId:%d, %s failed at %s:%d since %s, version:%" PRId64, TD_VID(pMeta->pVnode), __func__, __FILE__, + __LINE__, tstrerror(terrno), version); + metaFetchEntryFree(&pChild); + metaFetchEntryFree(&pSuper); + TAOS_RETURN(terrno); + } + + for (int32_t i = 0; i < pTagSchema->nCols; i++) { + STagVal value = { + .type = pTagSchema->pSchema[i].type, + .cid = pTagSchema->pSchema[i].colId, + }; + + if (iColumn == i) { + if (pReq->isNull) { + continue; + } + if (IS_VAR_DATA_TYPE(value.type)) { + value.pData = pReq->pTagVal; + value.nData = pReq->nTagVal; + } else { + memcpy(&value.i64, pReq->pTagVal, pReq->nTagVal); + } + } else if (!tTagGet(pOldTag, &value)) { + continue; + } + + if (NULL == taosArrayPush(pTagArray, &value)) { + metaError("vgId:%d, %s failed at %s:%d since %s, version:%" PRId64, TD_VID(pMeta->pVnode), __func__, __FILE__, + __LINE__, tstrerror(terrno), version); + taosArrayDestroy(pTagArray); + metaFetchEntryFree(&pChild); + metaFetchEntryFree(&pSuper); + TAOS_RETURN(terrno); + } + } + + STag *pNewTag = NULL; + code = tTagNew(pTagArray, pTagSchema->version, false, &pNewTag); + if (code) { + metaError("vgId:%d, %s failed at %s:%d since %s, version:%" PRId64, TD_VID(pMeta->pVnode), __func__, __FILE__, + __LINE__, tstrerror(code), version); + taosArrayDestroy(pTagArray); + metaFetchEntryFree(&pChild); + metaFetchEntryFree(&pSuper); + TAOS_RETURN(code); + } + taosArrayDestroy(pTagArray); + taosMemoryFree(pChild->ctbEntry.pTags); + pChild->ctbEntry.pTags = (uint8_t *)pNewTag; + } + + // do handle entry + code = metaHandleEntry2(pMeta, pChild); + if (code) { + metaError("vgId:%d, %s failed at %s:%d since %s, uid:%" PRId64 " name:%s version:%" PRId64, TD_VID(pMeta->pVnode), + __func__, __FILE__, __LINE__, tstrerror(code), pChild->uid, pReq->tbName, version); + metaFetchEntryFree(&pChild); + metaFetchEntryFree(&pSuper); + TAOS_RETURN(code); + } else { + metaInfo("vgId:%d, table %s uid %" PRId64 " is updated, version:%" PRId64, TD_VID(pMeta->pVnode), pReq->tbName, + pChild->uid, version); + } + + // free resource and return + metaFetchEntryFree(&pChild); + metaFetchEntryFree(&pSuper); + TAOS_RETURN(code); +} \ No newline at end of file From 69efdda495bb5a653028bebb3a73ecfccc992580 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 13 Dec 2024 08:48:30 +0800 Subject: [PATCH 31/45] fix: compile error --- source/dnode/vnode/src/meta/metaTable2.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/meta/metaTable2.c b/source/dnode/vnode/src/meta/metaTable2.c index f5aa5f04bf..3c313108bb 100644 --- a/source/dnode/vnode/src/meta/metaTable2.c +++ b/source/dnode/vnode/src/meta/metaTable2.c @@ -854,7 +854,7 @@ int32_t metaAlterTableColumnName(SMeta *pMeta, int64_t version, SVAlterTbReq *pR } if (NULL == pColumn) { - metaError("vgId:%d, %s failed at %s:%d since column id %" PRId64 " not found in table %s, version:%" PRId64, + metaError("vgId:%d, %s failed at %s:%d since column id %d not found in table %s, version:%" PRId64, TD_VID(pMeta->pVnode), __func__, __FILE__, __LINE__, pReq->colId, pReq->tbName, version); metaFetchEntryFree(&pEntry); TAOS_RETURN(TSDB_CODE_VND_COL_NOT_EXISTS); From 1247fb614c56d0fffecff24a36263d16ed2ccc64 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 13 Dec 2024 10:49:25 +0800 Subject: [PATCH 32/45] free resources --- source/dnode/vnode/src/meta/metaTable2.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/dnode/vnode/src/meta/metaTable2.c b/source/dnode/vnode/src/meta/metaTable2.c index 3c313108bb..cb5a853504 100644 --- a/source/dnode/vnode/src/meta/metaTable2.c +++ b/source/dnode/vnode/src/meta/metaTable2.c @@ -1021,6 +1021,7 @@ static int32_t metaCheckUpdateTableTagValReq(SMeta *pMeta, int64_t version, SVAl code = TSDB_CODE_TDB_TABLE_NOT_EXIST; TAOS_RETURN(code); } + tdbFreeClear(value); TAOS_RETURN(code); } From 13a4dcbc53054660d4041a920b2a83b02b2be03a Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 13 Dec 2024 12:00:27 +0800 Subject: [PATCH 33/45] more code --- source/dnode/vnode/src/meta/metaTable.c | 3 +- source/dnode/vnode/src/meta/metaTable2.c | 206 ++++++++++++++++++++++- 2 files changed, 204 insertions(+), 5 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index d6917f9ab5..dbb9a945af 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -22,6 +22,7 @@ int32_t metaDropTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq, S int32_t metaAlterTableColumnName(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq, STableMetaRsp *pRsp); int32_t metaAlterTableColumnBytes(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq, STableMetaRsp *pRsp); int32_t metaUpdateTableTagValue(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq); +int32_t metaUpdateTableMultiTagValue(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq); int32_t metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSchema *pSchema); @@ -2997,7 +2998,7 @@ int metaAlterTable(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq, STableMeta case TSDB_ALTER_TABLE_UPDATE_TAG_VAL: return metaUpdateTableTagValue(pMeta, version, pReq); case TSDB_ALTER_TABLE_UPDATE_MULTI_TAG_VAL: - return metaUpdateTableMultiTagVal(pMeta, version, pReq); + return metaUpdateTableMultiTagValue(pMeta, version, pReq); return terrno = TSDB_CODE_VND_INVALID_TABLE_ACTION; case TSDB_ALTER_TABLE_UPDATE_OPTIONS: return metaUpdateTableOptions(pMeta, version, pReq); diff --git a/source/dnode/vnode/src/meta/metaTable2.c b/source/dnode/vnode/src/meta/metaTable2.c index cb5a853504..9649f603fa 100644 --- a/source/dnode/vnode/src/meta/metaTable2.c +++ b/source/dnode/vnode/src/meta/metaTable2.c @@ -846,7 +846,7 @@ int32_t metaAlterTableColumnName(SMeta *pMeta, int64_t version, SVAlterTbReq *pR SSchema *pColumn = NULL; int32_t iColumn = 0; for (int32_t i = 0; i < pSchema->nCols; i++) { - if (pSchema->pSchema[i].colId == pReq->colId) { + if (strncmp(pSchema->pSchema[i].name, pReq->colName, TSDB_COL_NAME_LEN) == 0) { pColumn = &pSchema->pSchema[i]; iColumn = i; break; @@ -931,7 +931,7 @@ int32_t metaAlterTableColumnBytes(SMeta *pMeta, int64_t version, SVAlterTbReq *p int32_t iColumn = 0; int32_t rowSize = 0; for (int32_t i = 0; i < pSchema->nCols; i++) { - if (pSchema->pSchema[i].colId == pReq->colId) { + if (strncmp(pSchema->pSchema[i].name, pReq->colName, TSDB_COL_NAME_LEN) == 0) { pColumn = &pSchema->pSchema[i]; iColumn = i; } @@ -1061,7 +1061,7 @@ int32_t metaUpdateTableTagValue(SMeta *pMeta, int64_t version, SVAlterTbReq *pRe TAOS_RETURN(TSDB_CODE_INTERNAL_ERROR); } - // do change tag value + // search the tag to update SSchemaWrapper *pTagSchema = &pSuper->stbEntry.schemaTag; SSchema *pColumn = NULL; int32_t iColumn = 0; @@ -1168,4 +1168,202 @@ int32_t metaUpdateTableTagValue(SMeta *pMeta, int64_t version, SVAlterTbReq *pRe metaFetchEntryFree(&pChild); metaFetchEntryFree(&pSuper); TAOS_RETURN(code); -} \ No newline at end of file +} + +static int32_t metaCheckUpdateTableMultiTagValueReq(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq) { + int32_t code = 0; + + // check tag name + if (NULL == pReq->pMultiTag || taosArrayGetSize(pReq->pMultiTag) == 0) { + metaError("vgId:%d, %s failed at %s:%d since invalid tag name, version:%" PRId64, TD_VID(pMeta->pVnode), __func__, + __FILE__, __LINE__, version); + TAOS_RETURN(TSDB_CODE_INVALID_MSG); + } + + // check name + void *value = NULL; + int32_t valueSize = 0; + code = tdbTbGet(pMeta->pNameIdx, pReq->tbName, strlen(pReq->tbName) + 1, &value, &valueSize); + if (code) { + metaError("vgId:%d, %s failed at %s:%d since table %s not found, version:%" PRId64, TD_VID(pMeta->pVnode), __func__, + __FILE__, __LINE__, pReq->tbName, version); + code = TSDB_CODE_TDB_TABLE_NOT_EXIST; + TAOS_RETURN(code); + } + tdbFreeClear(value); + + if (taosArrayGetSize(pReq->pMultiTag) == 0) { + metaError("vgId:%d, %s failed at %s:%d since invalid tag name, version:%" PRId64, TD_VID(pMeta->pVnode), __func__, + __FILE__, __LINE__, version); + TAOS_RETURN(TSDB_CODE_INVALID_MSG); + } + + TAOS_RETURN(code); +} + +int32_t metaUpdateTableMultiTagValue(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq) { + int32_t code = TSDB_CODE_SUCCESS; + + code = metaCheckUpdateTableMultiTagValueReq(pMeta, version, pReq); + if (code) { + TAOS_RETURN(code); + } + + // fetch child entry + SMetaEntry *pChild = NULL; + code = metaFetchEntryByName(pMeta, pReq->tbName, &pChild); + if (code) { + metaError("vgId:%d, %s failed at %s:%d since table %s not found, version:%" PRId64, TD_VID(pMeta->pVnode), __func__, + __FILE__, __LINE__, pReq->tbName, version); + TAOS_RETURN(code); + } + + if (pChild->type != TSDB_CHILD_TABLE) { + metaError("vgId:%d, %s failed at %s:%d since table %s is not a child table, version:%" PRId64, + TD_VID(pMeta->pVnode), __func__, __FILE__, __LINE__, pReq->tbName, version); + metaFetchEntryFree(&pChild); + TAOS_RETURN(TSDB_CODE_VND_INVALID_TABLE_ACTION); + } + + // fetch super entry + SMetaEntry *pSuper = NULL; + code = metaFetchEntryByUid(pMeta, pChild->ctbEntry.suid, &pSuper); + if (code) { + metaError("vgId:%d, %s failed at %s:%d since super table uid %" PRId64 " not found, version:%" PRId64, + TD_VID(pMeta->pVnode), __func__, __FILE__, __LINE__, pChild->ctbEntry.suid, version); + metaFetchEntryFree(&pChild); + TAOS_RETURN(TSDB_CODE_INTERNAL_ERROR); + } + + // search the tags to update + SSchemaWrapper *pTagSchema = &pSuper->stbEntry.schemaTag; + + if (pTagSchema->nCols == 1 && pTagSchema->pSchema[0].type == TSDB_DATA_TYPE_JSON) { + metaError("vgId:%d, %s failed at %s:%d since table %s has no tag, version:%" PRId64, TD_VID(pMeta->pVnode), + __func__, __FILE__, __LINE__, pReq->tbName, version); + metaFetchEntryFree(&pChild); + metaFetchEntryFree(&pSuper); + TAOS_RETURN(TSDB_CODE_VND_COL_NOT_EXISTS); + } + + // do check if tag name exists + SHashObj *pTagTable = + taosHashInit(pTagSchema->nCols, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + if (pTagTable == NULL) { + metaError("vgId:%d, %s failed at %s:%d since %s, version:%" PRId64, TD_VID(pMeta->pVnode), __func__, __FILE__, + __LINE__, tstrerror(terrno), version); + metaFetchEntryFree(&pChild); + metaFetchEntryFree(&pSuper); + TAOS_RETURN(terrno); + } + + for (int32_t i = 0; i < taosArrayGetSize(pReq->pMultiTag); i++) { + SMultiTagUpateVal *pTagVal = taosArrayGet(pReq->pMultiTag, i); + if (taosHashPut(pTagTable, pTagVal->tagName, strlen(pTagVal->tagName), pTagVal, sizeof(*pTagVal)) != 0) { + metaError("vgId:%d, %s failed at %s:%d since %s, version:%" PRId64, TD_VID(pMeta->pVnode), __func__, __FILE__, + __LINE__, tstrerror(terrno), version); + taosHashCleanup(pTagTable); + metaFetchEntryFree(&pChild); + metaFetchEntryFree(&pSuper); + TAOS_RETURN(terrno); + } + } + + int32_t numOfChangedTags = 0; + for (int32_t i = 0; i < pTagSchema->nCols; i++) { + taosHashGet(pTagTable, pTagSchema->pSchema[i].name, strlen(pTagSchema->pSchema[i].name)) != NULL + ? numOfChangedTags++ + : 0; + } + if (numOfChangedTags < taosHashGetSize(pTagTable)) { + metaError("vgId:%d, %s failed at %s:%d since tag count mismatch, version:%" PRId64, TD_VID(pMeta->pVnode), __func__, + __FILE__, __LINE__, version); + taosHashCleanup(pTagTable); + metaFetchEntryFree(&pChild); + metaFetchEntryFree(&pSuper); + TAOS_RETURN(TSDB_CODE_VND_COL_NOT_EXISTS); + } + + // do change tag value + pChild->version = version; + const STag *pOldTag = (const STag *)pChild->ctbEntry.pTags; + SArray *pTagArray = taosArrayInit(pTagSchema->nCols, sizeof(STagVal)); + if (NULL == pTagArray) { + metaError("vgId:%d, %s failed at %s:%d since %s, version:%" PRId64, TD_VID(pMeta->pVnode), __func__, __FILE__, + __LINE__, tstrerror(terrno), version); + taosHashCleanup(pTagTable); + metaFetchEntryFree(&pChild); + metaFetchEntryFree(&pSuper); + TAOS_RETURN(terrno); + } + + for (int32_t i = 0; i < pTagSchema->nCols; i++) { + SSchema *pCol = &pTagSchema->pSchema[i]; + STagVal value = { + .cid = pCol->colId, + }; + + SMultiTagUpateVal *pTagVal = taosHashGet(pTagTable, pCol->name, strlen(pCol->name)); + if (pTagVal == NULL) { + if (!tTagGet(pOldTag, &value)) { + continue; + } + } else { + value.type = pCol->type; + if (pTagVal->isNull) { + continue; + } + + if (IS_VAR_DATA_TYPE(pCol->type)) { + value.pData = pTagVal->pTagVal; + value.nData = pTagVal->nTagVal; + } else { + memcpy(&value.i64, pTagVal->pTagVal, pTagVal->nTagVal); + } + } + + if (taosArrayPush(pTagArray, &value) == NULL) { + metaError("vgId:%d, %s failed at %s:%d since %s, version:%" PRId64, TD_VID(pMeta->pVnode), __func__, __FILE__, + __LINE__, tstrerror(terrno), version); + taosHashCleanup(pTagTable); + taosArrayDestroy(pTagArray); + metaFetchEntryFree(&pChild); + metaFetchEntryFree(&pSuper); + TAOS_RETURN(terrno); + } + } + + STag *pNewTag = NULL; + code = tTagNew(pTagArray, pTagSchema->version, false, &pNewTag); + if (code) { + metaError("vgId:%d, %s failed at %s:%d since %s, version:%" PRId64, TD_VID(pMeta->pVnode), __func__, __FILE__, + __LINE__, tstrerror(code), version); + taosHashCleanup(pTagTable); + taosArrayDestroy(pTagArray); + metaFetchEntryFree(&pChild); + metaFetchEntryFree(&pSuper); + TAOS_RETURN(code); + } + taosArrayDestroy(pTagArray); + taosMemoryFree(pChild->ctbEntry.pTags); + pChild->ctbEntry.pTags = (uint8_t *)pNewTag; + + // do handle entry + code = metaHandleEntry2(pMeta, pChild); + if (code) { + metaError("vgId:%d, %s failed at %s:%d since %s, uid:%" PRId64 " name:%s version:%" PRId64, TD_VID(pMeta->pVnode), + __func__, __FILE__, __LINE__, tstrerror(code), pChild->uid, pReq->tbName, version); + taosHashCleanup(pTagTable); + metaFetchEntryFree(&pChild); + metaFetchEntryFree(&pSuper); + TAOS_RETURN(code); + } else { + metaInfo("vgId:%d, table %s uid %" PRId64 " is updated, version:%" PRId64, TD_VID(pMeta->pVnode), pReq->tbName, + pChild->uid, version); + } + + taosHashCleanup(pTagTable); + metaFetchEntryFree(&pChild); + metaFetchEntryFree(&pSuper); + TAOS_RETURN(code); +} From 95130d2cac329536ada344053acc0ee570491c54 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 13 Dec 2024 12:20:08 +0800 Subject: [PATCH 34/45] feat: update table options --- source/dnode/vnode/src/meta/metaEntry2.c | 24 ++++++- source/dnode/vnode/src/meta/metaTable.c | 4 +- source/dnode/vnode/src/meta/metaTable2.c | 90 ++++++++++++++++++++++++ 3 files changed, 115 insertions(+), 3 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaEntry2.c b/source/dnode/vnode/src/meta/metaEntry2.c index 393c8dda21..cdc3628102 100644 --- a/source/dnode/vnode/src/meta/metaEntry2.c +++ b/source/dnode/vnode/src/meta/metaEntry2.c @@ -773,8 +773,28 @@ static int32_t metaTtlIdxInsert(SMeta *pMeta, const SMetaHandleParam *pParam) { return metaTtlIdxUpsert(pMeta, pParam, META_TABLE_OP_INSERT); } +static int32_t metaTtlIdxDelete(SMeta *pMeta, const SMetaHandleParam *pParam); + static int32_t metaTtlIdxUpdate(SMeta *pMeta, const SMetaHandleParam *pParam) { - return metaTtlIdxUpsert(pMeta, pParam, META_TABLE_OP_UPDATA); + int32_t code = TSDB_CODE_SUCCESS; + + const SMetaEntry *pEntry = pParam->pEntry; + const SMetaEntry *pOldEntry = pParam->pOldEntry; + + if ((pEntry->type == TSDB_CHILD_TABLE && pOldEntry->ctbEntry.ttlDays != pEntry->ctbEntry.ttlDays) || + (pEntry->type == TSDB_NORMAL_TABLE && pOldEntry->ntbEntry.ttlDays != pEntry->ntbEntry.ttlDays)) { + code = metaTtlIdxDelete(pMeta, pParam); + if (code) { + metaErr(TD_VID(pMeta->pVnode), code); + } + + code = metaTtlIdxInsert(pMeta, pParam); + if (code) { + metaErr(TD_VID(pMeta->pVnode), code); + } + } + + return TSDB_CODE_SUCCESS; } static int32_t metaTtlIdxDelete(SMeta *pMeta, const SMetaHandleParam *pParam) { @@ -1336,6 +1356,7 @@ static int32_t metaHandleNormalTableUpdateImpl(SMeta *pMeta, const SMetaHandlePa {META_ENTRY_TABLE, META_TABLE_OP_UPDATA}, // {META_SCHEMA_TABLE, META_TABLE_OP_UPDATA}, // {META_UID_IDX, META_TABLE_OP_UPDATA}, // + {META_TTL_IDX, META_TABLE_OP_UPDATA}, // }; for (int32_t i = 0; i < sizeof(ops) / sizeof(ops[0]); i++) { SMetaTableOp *op = &ops[i]; @@ -1365,6 +1386,7 @@ static int32_t metaHandleChildTableUpdateImpl(SMeta *pMeta, const SMetaHandlePar {META_UID_IDX, META_TABLE_OP_UPDATA}, // {META_TAG_IDX, META_TABLE_OP_UPDATA}, // {META_CHILD_IDX, META_TABLE_OP_UPDATA}, // + {META_TTL_IDX, META_TABLE_OP_UPDATA}, // }; for (int i = 0; i < sizeof(ops) / sizeof(ops[0]); i++) { diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index dbb9a945af..31713421db 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -23,6 +23,7 @@ int32_t metaAlterTableColumnName(SMeta *pMeta, int64_t version, SVAlterTbReq *pR int32_t metaAlterTableColumnBytes(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq, STableMetaRsp *pRsp); int32_t metaUpdateTableTagValue(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq); int32_t metaUpdateTableMultiTagValue(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq); +int32_t metaUpdateTableOptions2(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq); int32_t metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSchema *pSchema); @@ -2999,9 +3000,8 @@ int metaAlterTable(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq, STableMeta return metaUpdateTableTagValue(pMeta, version, pReq); case TSDB_ALTER_TABLE_UPDATE_MULTI_TAG_VAL: return metaUpdateTableMultiTagValue(pMeta, version, pReq); - return terrno = TSDB_CODE_VND_INVALID_TABLE_ACTION; case TSDB_ALTER_TABLE_UPDATE_OPTIONS: - return metaUpdateTableOptions(pMeta, version, pReq); + return metaUpdateTableOptions2(pMeta, version, pReq); case TSDB_ALTER_TABLE_ADD_TAG_INDEX: return metaAddTagIndex(pMeta, version, pReq); case TSDB_ALTER_TABLE_DROP_TAG_INDEX: diff --git a/source/dnode/vnode/src/meta/metaTable2.c b/source/dnode/vnode/src/meta/metaTable2.c index 9649f603fa..e889526319 100644 --- a/source/dnode/vnode/src/meta/metaTable2.c +++ b/source/dnode/vnode/src/meta/metaTable2.c @@ -1367,3 +1367,93 @@ int32_t metaUpdateTableMultiTagValue(SMeta *pMeta, int64_t version, SVAlterTbReq metaFetchEntryFree(&pSuper); TAOS_RETURN(code); } + +static int32_t metaCheckUpdateTableOptionsReq(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq) { + int32_t code = TSDB_CODE_SUCCESS; + + if (pReq->tbName == NULL || strlen(pReq->tbName) == 0) { + metaError("vgId:%d, %s failed at %s:%d since invalid table name, version:%" PRId64, TD_VID(pMeta->pVnode), __func__, + __FILE__, __LINE__, version); + TAOS_RETURN(TSDB_CODE_INVALID_MSG); + } + + return code; +} + +int32_t metaUpdateTableOptions2(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq) { + int32_t code = 0; + + code = metaCheckUpdateTableOptionsReq(pMeta, version, pReq); + if (code) { + TAOS_RETURN(code); + } + + // fetch entry + SMetaEntry *pEntry = NULL; + code = metaFetchEntryByName(pMeta, pReq->tbName, &pEntry); + if (code) { + metaError("vgId:%d, %s failed at %s:%d since table %s not found, version:%" PRId64, TD_VID(pMeta->pVnode), __func__, + __FILE__, __LINE__, pReq->tbName, version); + TAOS_RETURN(code); + } + + // do change the entry + pEntry->version = version; + if (pEntry->type == TSDB_CHILD_TABLE) { + if (pReq->updateTTL) { + pEntry->ctbEntry.ttlDays = pReq->newTTL; + // metaDeleteTtl(pMeta, &entry); + // entry.ctbEntry.ttlDays = pReq->newTTL; + // metaUpdateTtl(pMeta, &entry); + } + if (pReq->newCommentLen >= 0) { + char *pNewComment = taosMemoryRealloc(pEntry->ctbEntry.comment, pReq->newCommentLen + 1); + if (NULL == pNewComment) { + metaError("vgId:%d, %s failed at %s:%d since %s, version:%" PRId64, TD_VID(pMeta->pVnode), __func__, __FILE__, + __LINE__, tstrerror(terrno), version); + metaFetchEntryFree(&pEntry); + TAOS_RETURN(terrno); + } + pEntry->ctbEntry.comment = pNewComment; + pEntry->ctbEntry.commentLen = pReq->newCommentLen; + } + } else if (pEntry->type == TSDB_NORMAL_TABLE) { + if (pReq->updateTTL) { + pEntry->ntbEntry.ttlDays = pReq->newTTL; + // metaDeleteTtl(pMeta, &entry); + // entry.ntbEntry.ttlDays = pReq->newTTL; + // metaUpdateTtl(pMeta, &entry); + } + if (pReq->newCommentLen >= 0) { + char *pNewComment = taosMemoryRealloc(pEntry->ntbEntry.comment, pReq->newCommentLen + 1); + if (NULL == pNewComment) { + metaError("vgId:%d, %s failed at %s:%d since %s, version:%" PRId64, TD_VID(pMeta->pVnode), __func__, __FILE__, + __LINE__, tstrerror(terrno), version); + metaFetchEntryFree(&pEntry); + TAOS_RETURN(terrno); + } + pEntry->ntbEntry.comment = pNewComment; + pEntry->ntbEntry.commentLen = pReq->newCommentLen; + } + } else { + metaError("vgId:%d, %s failed at %s:%d since table %s type %d is invalid, version:%" PRId64, TD_VID(pMeta->pVnode), + __func__, __FILE__, __LINE__, pReq->tbName, pEntry->type, version); + metaFetchEntryFree(&pEntry); + TAOS_RETURN(TSDB_CODE_VND_INVALID_TABLE_ACTION); + } + + // do handle entry + code = metaHandleEntry2(pMeta, pEntry); + if (code) { + metaError("vgId:%d, %s failed at %s:%d since %s, uid:%" PRId64 " name:%s version:%" PRId64, TD_VID(pMeta->pVnode), + __func__, __FILE__, __LINE__, tstrerror(code), pEntry->uid, pReq->tbName, version); + metaFetchEntryFree(&pEntry); + TAOS_RETURN(code); + } else { + metaInfo("vgId:%d, table %s uid %" PRId64 " is updated, version:%" PRId64, TD_VID(pMeta->pVnode), pReq->tbName, + pEntry->uid, version); + } + + metaFetchEntryFree(&pEntry); + TAOS_RETURN(code); +} \ No newline at end of file From 8525df81a8e75ac1f077d40c03ee4ba0dbf408f2 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 13 Dec 2024 12:27:40 +0800 Subject: [PATCH 35/45] fix: update table column compress --- source/dnode/vnode/src/meta/metaTable.c | 3 +- source/dnode/vnode/src/meta/metaTable2.c | 78 ++++++++++++++++++++++++ 2 files changed, 80 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 31713421db..ae846d860b 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -24,6 +24,7 @@ int32_t metaAlterTableColumnBytes(SMeta *pMeta, int64_t version, SVAlterTbReq *p int32_t metaUpdateTableTagValue(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq); int32_t metaUpdateTableMultiTagValue(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq); int32_t metaUpdateTableOptions2(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq); +int32_t metaUpdateTableColCompress2(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq); int32_t metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSchema *pSchema); @@ -3007,7 +3008,7 @@ int metaAlterTable(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq, STableMeta case TSDB_ALTER_TABLE_DROP_TAG_INDEX: return metaDropTagIndex(pMeta, version, pReq); case TSDB_ALTER_TABLE_UPDATE_COLUMN_COMPRESS: - return metaUpdateTableColCompress(pMeta, version, pReq); + return metaUpdateTableColCompress2(pMeta, version, pReq); default: return terrno = TSDB_CODE_VND_INVALID_TABLE_ACTION; break; diff --git a/source/dnode/vnode/src/meta/metaTable2.c b/source/dnode/vnode/src/meta/metaTable2.c index e889526319..4b55000525 100644 --- a/source/dnode/vnode/src/meta/metaTable2.c +++ b/source/dnode/vnode/src/meta/metaTable2.c @@ -1454,6 +1454,84 @@ int32_t metaUpdateTableOptions2(SMeta *pMeta, int64_t version, SVAlterTbReq *pRe pEntry->uid, version); } + metaFetchEntryFree(&pEntry); + TAOS_RETURN(code); +} + +int32_t metaUpdateTableColCompress2(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq) { + int32_t code = TSDB_CODE_SUCCESS; + + if (NULL == pReq->tbName || strlen(pReq->tbName) == 0) { + metaError("vgId:%d, %s failed at %s:%d since invalid table name, version:%" PRId64, TD_VID(pMeta->pVnode), __func__, + __FILE__, __LINE__, version); + TAOS_RETURN(TSDB_CODE_INVALID_MSG); + } + + SMetaEntry *pEntry = NULL; + code = metaFetchEntryByName(pMeta, pReq->tbName, &pEntry); + if (code) { + metaError("vgId:%d, %s failed at %s:%d since table %s not found, version:%" PRId64, TD_VID(pMeta->pVnode), __func__, + __FILE__, __LINE__, pReq->tbName, version); + TAOS_RETURN(code); + } + + if (pEntry->version >= version) { + metaError("vgId:%d, %s failed at %s:%d since table %s version %" PRId64 " is not less than %" PRId64, + TD_VID(pMeta->pVnode), __func__, __FILE__, __LINE__, pReq->tbName, pEntry->version, version); + metaFetchEntryFree(&pEntry); + TAOS_RETURN(TSDB_CODE_INVALID_PARA); + } + + if (pEntry->type != TSDB_NORMAL_TABLE && pEntry->type != TSDB_SUPER_TABLE) { + metaError("vgId:%d, %s failed at %s:%d since table %s type %d is invalid, version:%" PRId64, TD_VID(pMeta->pVnode), + __func__, __FILE__, __LINE__, pReq->tbName, pEntry->type, version); + metaFetchEntryFree(&pEntry); + TAOS_RETURN(TSDB_CODE_VND_INVALID_TABLE_ACTION); + } + + // do change the entry + int8_t updated = 0; + SColCmprWrapper *wp = &pEntry->colCmpr; + for (int32_t i = 0; i < wp->nCols; i++) { + SColCmpr *p = &wp->pColCmpr[i]; + if (p->id == pReq->colId) { + uint32_t dst = 0; + updated = tUpdateCompress(p->alg, pReq->compress, TSDB_COLVAL_COMPRESS_DISABLED, TSDB_COLVAL_LEVEL_DISABLED, + TSDB_COLVAL_LEVEL_MEDIUM, &dst); + if (updated > 0) { + p->alg = dst; + } + } + } + + if (updated == 0) { + code = TSDB_CODE_VND_COLUMN_COMPRESS_ALREADY_EXIST; + metaError("vgId:%d, %s failed at %s:%d since column %d compress level is not changed, version:%" PRId64, + TD_VID(pMeta->pVnode), __func__, __FILE__, __LINE__, pReq->colId, version); + metaFetchEntryFree(&pEntry); + TAOS_RETURN(code); + } else if (updated < 0) { + code = TSDB_CODE_TSC_COMPRESS_LEVEL_ERROR; + metaError("vgId:%d, %s failed at %s:%d since column %d compress level is invalid, version:%" PRId64, + TD_VID(pMeta->pVnode), __func__, __FILE__, __LINE__, pReq->colId, version); + metaFetchEntryFree(&pEntry); + TAOS_RETURN(code); + } + + pEntry->version = version; + + // do handle entry + code = metaHandleEntry2(pMeta, pEntry); + if (code) { + metaError("vgId:%d, %s failed at %s:%d since %s, uid:%" PRId64 " name:%s version:%" PRId64, TD_VID(pMeta->pVnode), + __func__, __FILE__, __LINE__, tstrerror(code), pEntry->uid, pReq->tbName, version); + metaFetchEntryFree(&pEntry); + TAOS_RETURN(code); + } else { + metaInfo("vgId:%d, table %s uid %" PRId64 " is updated, version:%" PRId64, TD_VID(pMeta->pVnode), pReq->tbName, + pEntry->uid, version); + } + metaFetchEntryFree(&pEntry); TAOS_RETURN(code); } \ No newline at end of file From a6463884f0491b565d908b4851f4d4fa8815f4db Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 13 Dec 2024 16:47:05 +0800 Subject: [PATCH 36/45] more code --- include/libs/executor/storageapi.h | 4 ++-- source/dnode/vnode/src/meta/metaEntry.c | 4 ++-- source/dnode/vnode/src/meta/metaEntry2.c | 4 ++-- source/dnode/vnode/src/meta/metaTable.c | 4 ---- source/dnode/vnode/src/meta/metaTable2.c | 8 ++------ 5 files changed, 8 insertions(+), 16 deletions(-) diff --git a/include/libs/executor/storageapi.h b/include/libs/executor/storageapi.h index 52e740b3df..04ff6b9b3d 100644 --- a/include/libs/executor/storageapi.h +++ b/include/libs/executor/storageapi.h @@ -62,7 +62,7 @@ typedef struct SMetaEntry { struct { int64_t btime; int32_t ttlDays; - int32_t commentLen; + int32_t commentLen; // not include '\0' char* comment; tb_uid_t suid; uint8_t* pTags; @@ -431,7 +431,7 @@ typedef struct SStateStore { int32_t (*streamFileStateInit)(int64_t memSize, uint32_t keySize, uint32_t rowSize, uint32_t selectRowSize, GetTsFun fp, void* pFile, TSKEY delMark, const char* id, int64_t ckId, int8_t type, struct SStreamFileState** ppFileState); - + int32_t (*streamStateGroupPut)(SStreamState* pState, int64_t groupId, void* value, int32_t vLen); SStreamStateCur* (*streamStateGroupGetCur)(SStreamState* pState); void (*streamStateGroupCurNext)(SStreamStateCur* pCur); diff --git a/source/dnode/vnode/src/meta/metaEntry.c b/source/dnode/vnode/src/meta/metaEntry.c index 4ddd7c991d..302a1eb04a 100644 --- a/source/dnode/vnode/src/meta/metaEntry.c +++ b/source/dnode/vnode/src/meta/metaEntry.c @@ -317,7 +317,7 @@ int32_t metaCloneEntry(const SMetaEntry *pEntry, SMetaEntry **ppEntry) { metaCloneEntryFree(ppEntry); return code; } - memcpy((*ppEntry)->ctbEntry.comment, pEntry->ctbEntry.comment, pEntry->ctbEntry.commentLen); + memcpy((*ppEntry)->ctbEntry.comment, pEntry->ctbEntry.comment, pEntry->ctbEntry.commentLen + 1); } // tags @@ -350,7 +350,7 @@ int32_t metaCloneEntry(const SMetaEntry *pEntry, SMetaEntry **ppEntry) { metaCloneEntryFree(ppEntry); return code; } - memcpy((*ppEntry)->ntbEntry.comment, pEntry->ntbEntry.comment, pEntry->ntbEntry.commentLen); + memcpy((*ppEntry)->ntbEntry.comment, pEntry->ntbEntry.comment, pEntry->ntbEntry.commentLen + 1); } } else { return TSDB_CODE_INVALID_PARA; diff --git a/source/dnode/vnode/src/meta/metaEntry2.c b/source/dnode/vnode/src/meta/metaEntry2.c index cdc3628102..bcbd6f275c 100644 --- a/source/dnode/vnode/src/meta/metaEntry2.c +++ b/source/dnode/vnode/src/meta/metaEntry2.c @@ -964,7 +964,7 @@ static int32_t metaHandleNormalTableCreate(SMeta *pMeta, const SMetaEntry *pEntr metaULock(pMeta); // update other stuff - if (TSDB_CODE_SUCCESS != code) { + if (TSDB_CODE_SUCCESS == code) { pMeta->pVnode->config.vndStats.numOfNTables++; pMeta->pVnode->config.vndStats.numOfNTimeSeries += pEntry->ntbEntry.schemaRow.nCols - 1; pMeta->changed = true; @@ -1041,7 +1041,7 @@ static int32_t metaHandleChildTableCreate(SMeta *pMeta, const SMetaEntry *pEntry metaULock(pMeta); // update other stuff - if (TSDB_CODE_SUCCESS != code) { + if (TSDB_CODE_SUCCESS == code) { pMeta->pVnode->config.vndStats.numOfCTables++; if (!metaTbInFilterCache(pMeta, pSuperEntry->name, 1)) { diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index ae846d860b..8bad8ab569 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -3003,10 +3003,6 @@ int metaAlterTable(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq, STableMeta return metaUpdateTableMultiTagValue(pMeta, version, pReq); case TSDB_ALTER_TABLE_UPDATE_OPTIONS: return metaUpdateTableOptions2(pMeta, version, pReq); - case TSDB_ALTER_TABLE_ADD_TAG_INDEX: - return metaAddTagIndex(pMeta, version, pReq); - case TSDB_ALTER_TABLE_DROP_TAG_INDEX: - return metaDropTagIndex(pMeta, version, pReq); case TSDB_ALTER_TABLE_UPDATE_COLUMN_COMPRESS: return metaUpdateTableColCompress2(pMeta, version, pReq); default: diff --git a/source/dnode/vnode/src/meta/metaTable2.c b/source/dnode/vnode/src/meta/metaTable2.c index 4b55000525..1cb714e7bd 100644 --- a/source/dnode/vnode/src/meta/metaTable2.c +++ b/source/dnode/vnode/src/meta/metaTable2.c @@ -1402,9 +1402,6 @@ int32_t metaUpdateTableOptions2(SMeta *pMeta, int64_t version, SVAlterTbReq *pRe if (pEntry->type == TSDB_CHILD_TABLE) { if (pReq->updateTTL) { pEntry->ctbEntry.ttlDays = pReq->newTTL; - // metaDeleteTtl(pMeta, &entry); - // entry.ctbEntry.ttlDays = pReq->newTTL; - // metaUpdateTtl(pMeta, &entry); } if (pReq->newCommentLen >= 0) { char *pNewComment = taosMemoryRealloc(pEntry->ctbEntry.comment, pReq->newCommentLen + 1); @@ -1414,15 +1411,13 @@ int32_t metaUpdateTableOptions2(SMeta *pMeta, int64_t version, SVAlterTbReq *pRe metaFetchEntryFree(&pEntry); TAOS_RETURN(terrno); } + memcpy(pNewComment, pReq->newComment, pReq->newCommentLen + 1); pEntry->ctbEntry.comment = pNewComment; pEntry->ctbEntry.commentLen = pReq->newCommentLen; } } else if (pEntry->type == TSDB_NORMAL_TABLE) { if (pReq->updateTTL) { pEntry->ntbEntry.ttlDays = pReq->newTTL; - // metaDeleteTtl(pMeta, &entry); - // entry.ntbEntry.ttlDays = pReq->newTTL; - // metaUpdateTtl(pMeta, &entry); } if (pReq->newCommentLen >= 0) { char *pNewComment = taosMemoryRealloc(pEntry->ntbEntry.comment, pReq->newCommentLen + 1); @@ -1432,6 +1427,7 @@ int32_t metaUpdateTableOptions2(SMeta *pMeta, int64_t version, SVAlterTbReq *pRe metaFetchEntryFree(&pEntry); TAOS_RETURN(terrno); } + memcpy(pNewComment, pReq->newComment, pReq->newCommentLen + 1); pEntry->ntbEntry.comment = pNewComment; pEntry->ntbEntry.commentLen = pReq->newCommentLen; } From bdaa3228a28741b2eac72afdd1a3c5468658ae75 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sat, 14 Dec 2024 13:12:40 +0800 Subject: [PATCH 37/45] fix: CI case --- source/dnode/vnode/src/meta/metaTable2.c | 38 +++++++++++++++--------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaTable2.c b/source/dnode/vnode/src/meta/metaTable2.c index 1cb714e7bd..34aa61e724 100644 --- a/source/dnode/vnode/src/meta/metaTable2.c +++ b/source/dnode/vnode/src/meta/metaTable2.c @@ -1404,14 +1404,19 @@ int32_t metaUpdateTableOptions2(SMeta *pMeta, int64_t version, SVAlterTbReq *pRe pEntry->ctbEntry.ttlDays = pReq->newTTL; } if (pReq->newCommentLen >= 0) { - char *pNewComment = taosMemoryRealloc(pEntry->ctbEntry.comment, pReq->newCommentLen + 1); - if (NULL == pNewComment) { - metaError("vgId:%d, %s failed at %s:%d since %s, version:%" PRId64, TD_VID(pMeta->pVnode), __func__, __FILE__, - __LINE__, tstrerror(terrno), version); - metaFetchEntryFree(&pEntry); - TAOS_RETURN(terrno); + char *pNewComment = NULL; + if (pReq->newCommentLen) { + pNewComment = taosMemoryRealloc(pEntry->ctbEntry.comment, pReq->newCommentLen + 1); + if (NULL == pNewComment) { + metaError("vgId:%d, %s failed at %s:%d since %s, version:%" PRId64, TD_VID(pMeta->pVnode), __func__, __FILE__, + __LINE__, tstrerror(terrno), version); + metaFetchEntryFree(&pEntry); + TAOS_RETURN(terrno); + } + memcpy(pNewComment, pReq->newComment, pReq->newCommentLen + 1); + } else { + taosMemoryFreeClear(pEntry->ctbEntry.comment); } - memcpy(pNewComment, pReq->newComment, pReq->newCommentLen + 1); pEntry->ctbEntry.comment = pNewComment; pEntry->ctbEntry.commentLen = pReq->newCommentLen; } @@ -1420,14 +1425,19 @@ int32_t metaUpdateTableOptions2(SMeta *pMeta, int64_t version, SVAlterTbReq *pRe pEntry->ntbEntry.ttlDays = pReq->newTTL; } if (pReq->newCommentLen >= 0) { - char *pNewComment = taosMemoryRealloc(pEntry->ntbEntry.comment, pReq->newCommentLen + 1); - if (NULL == pNewComment) { - metaError("vgId:%d, %s failed at %s:%d since %s, version:%" PRId64, TD_VID(pMeta->pVnode), __func__, __FILE__, - __LINE__, tstrerror(terrno), version); - metaFetchEntryFree(&pEntry); - TAOS_RETURN(terrno); + char *pNewComment = NULL; + if (pReq->newCommentLen > 0) { + pNewComment = taosMemoryRealloc(pEntry->ntbEntry.comment, pReq->newCommentLen + 1); + if (NULL == pNewComment) { + metaError("vgId:%d, %s failed at %s:%d since %s, version:%" PRId64, TD_VID(pMeta->pVnode), __func__, __FILE__, + __LINE__, tstrerror(terrno), version); + metaFetchEntryFree(&pEntry); + TAOS_RETURN(terrno); + } + memcpy(pNewComment, pReq->newComment, pReq->newCommentLen + 1); + } else { + taosMemoryFreeClear(pEntry->ntbEntry.comment); } - memcpy(pNewComment, pReq->newComment, pReq->newCommentLen + 1); pEntry->ntbEntry.comment = pNewComment; pEntry->ntbEntry.commentLen = pReq->newCommentLen; } From 582a40874f986380e552474dcf0a5ec5a9753222 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sat, 14 Dec 2024 17:08:44 +0800 Subject: [PATCH 38/45] feat: replace add index --- source/dnode/vnode/src/inc/vnodeInt.h | 4 +- source/dnode/vnode/src/meta/metaEntry2.c | 258 +++++++++++++++++-- source/dnode/vnode/src/meta/metaTable2.c | 312 +++++++++++++++++++++++ source/dnode/vnode/src/vnd/vnodeSvr.c | 2 +- 4 files changed, 554 insertions(+), 22 deletions(-) diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index acb8de3780..975d8b0f8c 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -180,8 +180,8 @@ int metaAlterCache(SMeta* pMeta, int32_t nPage); int32_t metaUidCacheClear(SMeta* pMeta, uint64_t suid); int32_t metaTbGroupCacheClear(SMeta* pMeta, uint64_t suid); -int metaAddIndexToSTable(SMeta* pMeta, int64_t version, SVCreateStbReq* pReq); -int metaDropIndexFromSTable(SMeta* pMeta, int64_t version, SDropIndexReq* pReq); +int32_t metaAddIndexToSuperTable(SMeta* pMeta, int64_t version, SVCreateStbReq* pReq); +int metaDropIndexFromSTable(SMeta* pMeta, int64_t version, SDropIndexReq* pReq); int64_t metaGetTimeSeriesNum(SMeta* pMeta, int type); void metaUpdTimeSeriesNum(SMeta* pMeta); diff --git a/source/dnode/vnode/src/meta/metaEntry2.c b/source/dnode/vnode/src/meta/metaEntry2.c index bcbd6f275c..1b00250aa4 100644 --- a/source/dnode/vnode/src/meta/metaEntry2.c +++ b/source/dnode/vnode/src/meta/metaEntry2.c @@ -18,6 +18,11 @@ int metaDelJsonVarFromIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const S void metaTimeSeriesNotifyCheck(SMeta *pMeta); int tagIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2); +static int32_t metaGetChildUidsOfSuperTable(SMeta *pMeta, tb_uid_t suid, SArray **childList); +static int32_t metaFetchTagIdxKey(SMeta *pMeta, const SMetaEntry *pEntry, const SSchema *pTagColumn, + STagIdxKey **ppTagIdxKey, int32_t *pTagIdxKeySize); +static void metaFetchTagIdxKeyFree(STagIdxKey **ppTagIdxKey); + #define metaErr(VGID, ERRNO) \ do { \ metaError("vgId:%d, %s failed at %s:%d since %s, version:%" PRId64 " type:%d uid:%" PRId64 " name:%s", VGID, \ @@ -263,6 +268,168 @@ static int32_t metaSchemaTableInsert(SMeta *pMeta, const SMetaHandleParam *pPara return metaSchemaTableUpsert(pMeta, pParam, META_TABLE_OP_INSERT); } +static int32_t metaAddOrDropTagIndexOfSuperTable(SMeta *pMeta, const SMetaHandleParam *pParam, + const SSchema *pOldColumn, const SSchema *pNewColumn) { + int32_t code = TSDB_CODE_SUCCESS; + + const SMetaEntry *pEntry = pParam->pEntry; + const SMetaEntry *pOldEntry = pParam->pOldEntry; + enum { ADD_INDEX, DROP_INDEX } action; + + if (pOldColumn && pNewColumn) { + if (IS_IDX_ON(pOldColumn) && IS_IDX_ON(pNewColumn)) { + return TSDB_CODE_SUCCESS; + } else if (IS_IDX_ON(pOldColumn) && !IS_IDX_ON(pNewColumn)) { + action = DROP_INDEX; + } else if (!IS_IDX_ON(pOldColumn) && IS_IDX_ON(pNewColumn)) { + action = ADD_INDEX; + } else { + return TSDB_CODE_SUCCESS; + } + } else if (pOldColumn) { + if (IS_IDX_ON(pOldColumn)) { + action = DROP_INDEX; + } else { + return TSDB_CODE_SUCCESS; + } + } else { + if (IS_IDX_ON(pNewColumn)) { + action = ADD_INDEX; + } else { + return TSDB_CODE_SUCCESS; + } + } + + // fetch all child tables + SArray *childTables = 0; + code = metaGetChildUidsOfSuperTable(pMeta, pEntry->uid, &childTables); + if (code) { + metaErr(TD_VID(pMeta->pVnode), code); + return code; + } + + // do drop or add index + for (int32_t i = 0; i < taosArrayGetSize(childTables); i++) { + int64_t uid = *(int64_t *)taosArrayGet(childTables, i); + + // fetch child entry + SMetaEntry *pChildEntry = NULL; + code = metaFetchEntryByUid(pMeta, uid, &pChildEntry); + if (code) { + metaErr(TD_VID(pMeta->pVnode), code); + taosArrayDestroy(childTables); + return code; + } + + STagIdxKey *pTagIdxKey = NULL; + int32_t tagIdxKeySize = 0; + + if (action == ADD_INDEX) { + code = metaFetchTagIdxKey(pMeta, pChildEntry, pNewColumn, &pTagIdxKey, &tagIdxKeySize); + if (code) { + metaErr(TD_VID(pMeta->pVnode), code); + taosArrayDestroy(childTables); + metaFetchEntryFree(&pChildEntry); + return code; + } + + code = tdbTbInsert(pMeta->pTagIdx, pTagIdxKey, tagIdxKeySize, &pChildEntry->uid, sizeof(pChildEntry->uid), + pMeta->txn); + if (code) { + metaErr(TD_VID(pMeta->pVnode), code); + taosArrayDestroy(childTables); + metaFetchEntryFree(&pChildEntry); + metaFetchTagIdxKeyFree(&pTagIdxKey); + return code; + } + } else { + code = metaFetchTagIdxKey(pMeta, pChildEntry, pOldColumn, &pTagIdxKey, &tagIdxKeySize); + if (code) { + metaErr(TD_VID(pMeta->pVnode), code); + taosArrayDestroy(childTables); + metaFetchEntryFree(&pChildEntry); + return code; + } + + code = tdbTbDelete(pMeta->pTagIdx, pTagIdxKey, tagIdxKeySize, pMeta->txn); + if (code) { + metaErr(TD_VID(pMeta->pVnode), code); + taosArrayDestroy(childTables); + metaFetchEntryFree(&pChildEntry); + metaFetchTagIdxKeyFree(&pTagIdxKey); + return code; + } + } + + metaFetchTagIdxKeyFree(&pTagIdxKey); + metaFetchEntryFree(&pChildEntry); + } + + taosArrayDestroy(childTables); + return code; +} + +static int32_t metaUpdateSuperTableTagSchema(SMeta *pMeta, const SMetaHandleParam *pParam) { + int32_t code = TSDB_CODE_SUCCESS; + const SMetaEntry *pEntry = pParam->pEntry; + const SMetaEntry *pOldEntry = pParam->pOldEntry; + const SSchemaWrapper *pNewTagSchema = &pEntry->stbEntry.schemaTag; + const SSchemaWrapper *pOldTagSchema = &pOldEntry->stbEntry.schemaTag; + + int32_t iOld = 0, iNew = 0; + for (; iOld < pOldTagSchema->nCols && iNew < pNewTagSchema->nCols;) { + SSchema *pOldColumn = pOldTagSchema->pSchema + iOld; + SSchema *pNewColumn = pNewTagSchema->pSchema + iNew; + + if (pOldColumn->colId == pNewColumn->colId) { + code = metaAddOrDropTagIndexOfSuperTable(pMeta, pParam, pOldColumn, pNewColumn); + if (code) { + metaErr(TD_VID(pMeta->pVnode), code); + return code; + } + + iOld++; + iNew++; + } else if (pOldColumn->colId < pNewColumn->colId) { + code = metaAddOrDropTagIndexOfSuperTable(pMeta, pParam, pOldColumn, NULL); + if (code) { + metaErr(TD_VID(pMeta->pVnode), code); + return code; + } + + iOld++; + } else { + code = metaAddOrDropTagIndexOfSuperTable(pMeta, pParam, NULL, pNewColumn); + if (code) { + metaErr(TD_VID(pMeta->pVnode), code); + return code; + } + + iNew++; + } + } + + for (; iOld < pOldTagSchema->nCols; iOld++) { + SSchema *pOldColumn = pOldTagSchema->pSchema + iOld; + code = metaAddOrDropTagIndexOfSuperTable(pMeta, pParam, pOldColumn, NULL); + if (code) { + metaErr(TD_VID(pMeta->pVnode), code); + return code; + } + } + + for (; iNew < pNewTagSchema->nCols; iNew++) { + SSchema *pNewColumn = pNewTagSchema->pSchema + iNew; + code = metaAddOrDropTagIndexOfSuperTable(pMeta, pParam, NULL, pNewColumn); + if (code) { + metaErr(TD_VID(pMeta->pVnode), code); + return code; + } + } + + return code; +} + static int32_t metaSchemaTableUpdate(SMeta *pMeta, const SMetaHandleParam *pParam) { int32_t code = TSDB_CODE_SUCCESS; @@ -273,29 +440,30 @@ static int32_t metaSchemaTableUpdate(SMeta *pMeta, const SMetaHandleParam *pPara return metaSchemaTableUpsert(pMeta, pParam, META_TABLE_OP_UPDATA); } - if (pEntry->type == TSDB_NORMAL_TABLE && - pOldEntry->ntbEntry.schemaRow.version != pEntry->ntbEntry.schemaRow.version) { - code = metaSchemaTableUpsert(pMeta, pParam, META_TABLE_OP_UPDATA); - if (code) { - metaErr(TD_VID(pMeta->pVnode), code); - return code; + if (pEntry->type == TSDB_NORMAL_TABLE) { + // check row schema + if (pOldEntry->ntbEntry.schemaRow.version != pEntry->ntbEntry.schemaRow.version) { + return metaSchemaTableUpsert(pMeta, pParam, META_TABLE_OP_UPDATA); + } + } else if (pEntry->type == TSDB_SUPER_TABLE) { + // check row schema + if (pOldEntry->stbEntry.schemaRow.version != pEntry->stbEntry.schemaRow.version) { + return metaSchemaTableUpsert(pMeta, pParam, META_TABLE_OP_UPDATA); } - if (pOldEntry->ntbEntry.schemaRow.nCols != pEntry->ntbEntry.schemaRow.nCols) { - pMeta->pVnode->config.vndStats.numOfNTimeSeries += - (pEntry->ntbEntry.schemaRow.nCols - pOldEntry->ntbEntry.schemaRow.nCols); + // check tag schema + if (pOldEntry->stbEntry.schemaTag.version != pEntry->stbEntry.schemaTag.version) { + code = metaUpdateSuperTableTagSchema(pMeta, pParam); + if (code) { + metaErr(TD_VID(pMeta->pVnode), code); + return code; + } } + } else { + return TSDB_CODE_INVALID_PARA; } - if (pEntry->type == TSDB_SUPER_TABLE && pOldEntry->stbEntry.schemaRow.version != pEntry->stbEntry.schemaRow.version) { - return metaSchemaTableUpsert(pMeta, pParam, META_TABLE_OP_UPDATA); - } - - if (pParam->pEntry->type == TSDB_CHILD_TABLE) { - return TSDB_CODE_INVALID_MSG; - } - - return 0; + return TSDB_CODE_SUCCESS; } static int32_t metaSchemaTableDelete(SMeta *pMeta, const SMetaHandleParam *pEntry) { @@ -1413,6 +1581,58 @@ static int32_t metaHandleChildTableUpdateImpl(SMeta *pMeta, const SMetaHandlePar #endif } +static int32_t metaHandleSuperTableUpdateImpl(SMeta *pMeta, SMetaHandleParam *pParam) { + int32_t code = TSDB_CODE_SUCCESS; + + const SMetaEntry *pEntry = pParam->pEntry; + const SMetaEntry *pOldEntry = pParam->pOldEntry; + + SMetaTableOp ops[] = { + {META_ENTRY_TABLE, META_TABLE_OP_UPDATA}, // + {META_UID_IDX, META_TABLE_OP_UPDATA}, // + {META_SCHEMA_TABLE, META_TABLE_OP_UPDATA}, // + }; + + for (int i = 0; i < sizeof(ops) / sizeof(ops[0]); i++) { + SMetaTableOp *op = &ops[i]; + code = metaTableOpFn[op->table][op->op](pMeta, pParam); + if (code) { + metaErr(TD_VID(pMeta->pVnode), code); + return code; + } + } + + return code; +} + +static int32_t metaHandleSuperTableUpdate(SMeta *pMeta, const SMetaEntry *pEntry) { + int32_t code = TSDB_CODE_SUCCESS; + + SMetaEntry *pOldEntry = NULL; + + code = metaFetchEntryByUid(pMeta, pEntry->uid, &pOldEntry); + if (code) { + metaErr(TD_VID(pMeta->pVnode), code); + return code; + } + + SMetaHandleParam param = { + .pEntry = pEntry, + .pOldEntry = pOldEntry, + }; + metaWLock(pMeta); + code = metaHandleSuperTableUpdateImpl(pMeta, ¶m); + metaULock(pMeta); + if (code) { + metaErr(TD_VID(pMeta->pVnode), code); + metaFetchEntryFree(&pOldEntry); + return code; + } + + metaFetchEntryFree(&pOldEntry); + return code; +} + static int32_t metaHandleChildTableUpdate(SMeta *pMeta, const SMetaEntry *pEntry) { int32_t code = TSDB_CODE_SUCCESS; @@ -1589,7 +1809,7 @@ int32_t metaHandleEntry2(SMeta *pMeta, const SMetaEntry *pEntry) { switch (type) { case TSDB_SUPER_TABLE: { if (isExist) { - // code = metaHandleSuperTableUpdate(pMeta, pEntry); + code = metaHandleSuperTableUpdate(pMeta, pEntry); } else { code = metaHandleSuperTableCreate(pMeta, pEntry); } diff --git a/source/dnode/vnode/src/meta/metaTable2.c b/source/dnode/vnode/src/meta/metaTable2.c index 34aa61e724..e0980ccf0d 100644 --- a/source/dnode/vnode/src/meta/metaTable2.c +++ b/source/dnode/vnode/src/meta/metaTable2.c @@ -1540,4 +1540,316 @@ int32_t metaUpdateTableColCompress2(SMeta *pMeta, int64_t version, SVAlterTbReq metaFetchEntryFree(&pEntry); TAOS_RETURN(code); +} + +int32_t metaAddIndexToSuperTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { + int32_t code = TSDB_CODE_SUCCESS; + + if (NULL == pReq->name || strlen(pReq->name) == 0) { + metaError("vgId:%d, %s failed at %s:%d since invalid table name, version:%" PRId64, TD_VID(pMeta->pVnode), __func__, + __FILE__, __LINE__, version); + TAOS_RETURN(TSDB_CODE_INVALID_MSG); + } + + SMetaEntry *pEntry = NULL; + code = metaFetchEntryByName(pMeta, pReq->name, &pEntry); + if (code) { + metaError("vgId:%d, %s failed at %s:%d since table %s not found, version:%" PRId64, TD_VID(pMeta->pVnode), __func__, + __FILE__, __LINE__, pReq->name, version); + TAOS_RETURN(code); + } + + if (pEntry->type != TSDB_SUPER_TABLE) { + metaError("vgId:%d, %s failed at %s:%d since table %s type %d is invalid, version:%" PRId64, TD_VID(pMeta->pVnode), + __func__, __FILE__, __LINE__, pReq->name, pEntry->type, version); + metaFetchEntryFree(&pEntry); + TAOS_RETURN(TSDB_CODE_VND_INVALID_TABLE_ACTION); + } + + if (pEntry->uid != pReq->suid) { + metaError("vgId:%d, %s failed at %s:%d since table %s uid %" PRId64 " is not equal to %" PRId64 + ", version:%" PRId64, + TD_VID(pMeta->pVnode), __func__, __FILE__, __LINE__, pReq->name, pEntry->uid, pReq->suid, version); + metaFetchEntryFree(&pEntry); + TAOS_RETURN(TSDB_CODE_INVALID_MSG); + } + + if (pEntry->stbEntry.schemaTag.version >= pReq->schemaTag.version) { + metaError("vgId:%d, %s failed at %s:%d since table %s tag schema version %" PRId64 " is not less than %" PRId64 + ", version:%" PRId64, + TD_VID(pMeta->pVnode), __func__, __FILE__, __LINE__, pReq->name, pEntry->stbEntry.schemaTag.version, + pReq->schemaTag.version, version); + metaFetchEntryFree(&pEntry); + TAOS_RETURN(TSDB_CODE_INVALID_MSG); + } + + // do change the entry + SSchemaWrapper *pOldTagSchema = &pEntry->stbEntry.schemaTag; + SSchemaWrapper *pNewTagSchema = &pReq->schemaTag; + if (pOldTagSchema->nCols == 1 && pOldTagSchema->pSchema[0].type == TSDB_DATA_TYPE_JSON) { + metaError("vgId:%d, %s failed at %s:%d since table %s has no tag, version:%" PRId64, TD_VID(pMeta->pVnode), + __func__, __FILE__, __LINE__, pReq->name, version); + metaFetchEntryFree(&pEntry); + TAOS_RETURN(TSDB_CODE_VND_COL_NOT_EXISTS); + } + + if (pOldTagSchema->nCols != pNewTagSchema->nCols) { + metaError( + "vgId:%d, %s failed at %s:%d since table %s tag schema column count %d is not equal to %d, version:%" PRId64, + TD_VID(pMeta->pVnode), __func__, __FILE__, __LINE__, pReq->name, pOldTagSchema->nCols, pNewTagSchema->nCols, + version); + metaFetchEntryFree(&pEntry); + TAOS_RETURN(TSDB_CODE_INVALID_MSG); + } + + if (pOldTagSchema->version >= pNewTagSchema->version) { + metaError("vgId:%d, %s failed at %s:%d since table %s tag schema version %" PRId64 " is not less than %" PRId64 + ", version:%" PRId64, + TD_VID(pMeta->pVnode), __func__, __FILE__, __LINE__, pReq->name, pOldTagSchema->version, + pNewTagSchema->version, version); + metaFetchEntryFree(&pEntry); + TAOS_RETURN(TSDB_CODE_INVALID_MSG); + } + + int32_t numOfChangedTags = 0; + for (int32_t i = 0; i < pOldTagSchema->nCols; i++) { + SSchema *pOldColumn = pOldTagSchema->pSchema + i; + SSchema *pNewColumn = pNewTagSchema->pSchema + i; + + if (pOldColumn->type != pNewColumn->type || pOldColumn->colId != pNewColumn->colId || + strncmp(pOldColumn->name, pNewColumn->name, sizeof(pNewColumn->name))) { + metaError("vgId:%d, %s failed at %s:%d since table %s tag schema column %d is not equal, version:%" PRId64, + TD_VID(pMeta->pVnode), __func__, __FILE__, __LINE__, pReq->name, i, version); + metaFetchEntryFree(&pEntry); + TAOS_RETURN(TSDB_CODE_INVALID_MSG); + } + + if (IS_IDX_ON(pNewColumn) && !IS_IDX_ON(pOldColumn)) { + numOfChangedTags++; + SSCHMEA_SET_IDX_ON(pOldColumn); + } else if (!IS_IDX_ON(pNewColumn) && IS_IDX_ON(pOldColumn)) { + metaError("vgId:%d, %s failed at %s:%d since table %s tag schema column %d is not equal, version:%" PRId64, + TD_VID(pMeta->pVnode), __func__, __FILE__, __LINE__, pReq->name, i, version); + metaFetchEntryFree(&pEntry); + TAOS_RETURN(TSDB_CODE_INVALID_MSG); + } + } + + if (numOfChangedTags != 1) { + metaError( + "vgId:%d, %s failed at %s:%d since table %s tag schema column count %d is not equal to 1, version:%" PRId64, + TD_VID(pMeta->pVnode), __func__, __FILE__, __LINE__, pReq->name, numOfChangedTags, version); + metaFetchEntryFree(&pEntry); + TAOS_RETURN(TSDB_CODE_INVALID_MSG); + } + + pEntry->version = version; + pEntry->stbEntry.schemaTag.version = pNewTagSchema->version; + + // do handle the entry + code = metaHandleEntry2(pMeta, pEntry); + if (code) { + metaError("vgId:%d, %s failed at %s:%d since %s, uid:%" PRId64 " name:%s version:%" PRId64, TD_VID(pMeta->pVnode), + __func__, __FILE__, __LINE__, tstrerror(code), pEntry->uid, pReq->name, version); + metaFetchEntryFree(&pEntry); + TAOS_RETURN(TSDB_CODE_INVALID_MSG); + } else { + metaInfo("vgId:%d, table %s uid %" PRId64 " is updated, version:%" PRId64, TD_VID(pMeta->pVnode), pReq->name, + pEntry->uid, version); + } + + metaFetchEntryFree(&pEntry); + TAOS_RETURN(code); +} + +int32_t metaDropIndexFromSuperTable(SMeta *pMeta, int64_t version, SDropIndexReq *pReq) { + int32_t code = TSDB_CODE_SUCCESS; + // TODO + TAOS_RETURN(code); +#if 0 + int32_t code = 0; + SMetaEntry oStbEntry = {0}; + SMetaEntry nStbEntry = {0}; + + STbDbKey tbDbKey = {0}; + TBC *pUidIdxc = NULL; + TBC *pTbDbc = NULL; + int ret = 0; + int c = -2; + void *pData = NULL; + int nData = 0; + int64_t oversion; + SDecoder dc = {0}; + + tb_uid_t suid = pReq->stbUid; + + if ((code = tdbTbGet(pMeta->pUidIdx, &suid, sizeof(tb_uid_t), &pData, &nData)) != 0) { + goto _err; + } + + tbDbKey.uid = suid; + tbDbKey.version = ((SUidIdxVal *)pData)[0].version; + if ((code = tdbTbGet(pMeta->pTbDb, &tbDbKey, sizeof(tbDbKey), &pData, &nData)) != 0) { + goto _err; + } + + tDecoderInit(&dc, pData, nData); + code = metaDecodeEntry(&dc, &oStbEntry); + if (code != 0) { + goto _err; + } + + SSchema *pCol = NULL; + int32_t colId = -1; + for (int i = 0; i < oStbEntry.stbEntry.schemaTag.nCols; i++) { + SSchema *schema = oStbEntry.stbEntry.schemaTag.pSchema + i; + if (0 == strncmp(schema->name, pReq->colName, sizeof(pReq->colName))) { + if (IS_IDX_ON(schema)) { + pCol = schema; + } + break; + } + } + + if (pCol == NULL) { + metaError("vgId:%d, failed to drop index on %s.%s,since %s", TD_VID(pMeta->pVnode), pReq->stb, pReq->colName, + tstrerror(TSDB_CODE_VND_COL_NOT_EXISTS)); + code = 0; + + goto _err; + } + + /* + * iterator all pTdDbc by uid and version + */ + TBC *pCtbIdxc = NULL; + code = tdbTbcOpen(pMeta->pCtbIdx, &pCtbIdxc, NULL); + if (code != 0) { + goto _err; + } + + code = tdbTbcMoveTo(pCtbIdxc, &(SCtbIdxKey){.suid = suid, .uid = INT64_MIN}, sizeof(SCtbIdxKey), &c); + if (code < 0) { + tdbTbcClose(pCtbIdxc); + goto _err; + } + for (;;) { + void *pKey = NULL, *pVal = NULL; + int nKey = 0, nVal = 0; + + code = tdbTbcNext(pCtbIdxc, &pKey, &nKey, &pVal, &nVal); + if (code < 0) { + tdbFree(pKey); + tdbFree(pVal); + tdbTbcClose(pCtbIdxc); + pCtbIdxc = NULL; + break; + } + if (((SCtbIdxKey *)pKey)->suid != suid) { + tdbFree(pKey); + tdbFree(pVal); + continue; + } + STagIdxKey *pTagIdxKey = NULL; + int32_t nTagIdxKey; + + const void *pTagData = NULL; + int32_t nTagData = 0; + + SCtbIdxKey *table = (SCtbIdxKey *)pKey; + STagVal tagVal = {.cid = pCol->colId}; + if (tTagGet((const STag *)pVal, &tagVal)) { + if (IS_VAR_DATA_TYPE(pCol->type)) { + pTagData = tagVal.pData; + nTagData = (int32_t)tagVal.nData; + } else { + pTagData = &(tagVal.i64); + nTagData = tDataTypes[pCol->type].bytes; + } + } else { + if (!IS_VAR_DATA_TYPE(pCol->type)) { + nTagData = tDataTypes[pCol->type].bytes; + } + } + + code = metaCreateTagIdxKey(suid, pCol->colId, pTagData, nTagData, pCol->type, table->uid, &pTagIdxKey, &nTagIdxKey); + tdbFree(pKey); + tdbFree(pVal); + if (code < 0) { + metaDestroyTagIdxKey(pTagIdxKey); + tdbTbcClose(pCtbIdxc); + goto _err; + } + + metaWLock(pMeta); + ret = tdbTbDelete(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, pMeta->txn); + if (ret < 0) { + metaError("vgId:%d, failed to delete tag idx key:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->stb, + pReq->stbUid, tstrerror(ret)); + } + metaULock(pMeta); + metaDestroyTagIdxKey(pTagIdxKey); + pTagIdxKey = NULL; + } + + // clear idx flag + SSCHMEA_SET_IDX_OFF(pCol); + + nStbEntry.version = version; + nStbEntry.type = TSDB_SUPER_TABLE; + nStbEntry.uid = oStbEntry.uid; + nStbEntry.name = oStbEntry.name; + + SSchemaWrapper *row = tCloneSSchemaWrapper(&oStbEntry.stbEntry.schemaRow); + SSchemaWrapper *tag = tCloneSSchemaWrapper(&oStbEntry.stbEntry.schemaTag); + SColCmprWrapper *cmpr = tCloneSColCmprWrapper(&oStbEntry.colCmpr); + if (row == NULL || tag == NULL || cmpr == NULL) { + tDeleteSchemaWrapper(row); + tDeleteSchemaWrapper(tag); + tDeleteSColCmprWrapper(cmpr); + code = TSDB_CODE_OUT_OF_MEMORY; + + tdbTbcClose(pCtbIdxc); + goto _err; + } + + nStbEntry.stbEntry.schemaRow = *row; + nStbEntry.stbEntry.schemaTag = *tag; + nStbEntry.stbEntry.rsmaParam = oStbEntry.stbEntry.rsmaParam; + nStbEntry.colCmpr = *cmpr; + + nStbEntry.colCmpr = oStbEntry.colCmpr; + + metaWLock(pMeta); + // update table.db + ret = metaSaveToTbDb(pMeta, &nStbEntry); + if (ret < 0) { + metaError("vgId:%d, failed to save tb db:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->stb, + pReq->stbUid, tstrerror(ret)); + } + // update uid index + ret = metaUpdateUidIdx(pMeta, &nStbEntry); + if (ret < 0) { + metaError("vgId:%d, failed to update uid idx:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->stb, + pReq->stbUid, tstrerror(ret)); + } + metaULock(pMeta); + + tDeleteSchemaWrapper(tag); + tDeleteSchemaWrapper(row); + tDeleteSColCmprWrapper(cmpr); + + if (oStbEntry.pBuf) taosMemoryFree(oStbEntry.pBuf); + tDecoderClear(&dc); + tdbFree(pData); + + tdbTbcClose(pCtbIdxc); + return TSDB_CODE_SUCCESS; +_err: + if (oStbEntry.pBuf) taosMemoryFree(oStbEntry.pBuf); + tDecoderClear(&dc); + tdbFree(pData); + + return code; +#endif } \ No newline at end of file diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 5d65e8ce92..8d757f95a6 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -2433,7 +2433,7 @@ static int32_t vnodeProcessCreateIndexReq(SVnode *pVnode, int64_t ver, void *pRe return terrno = TSDB_CODE_INVALID_MSG; } - code = metaAddIndexToSTable(pVnode->pMeta, ver, &req); + code = metaAddIndexToSuperTable(pVnode->pMeta, ver, &req); if (code) { pRsp->code = code; goto _err; From 161081a0a2a4c89c6e22f020ae97d609e2b73766 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sat, 14 Dec 2024 17:21:17 +0800 Subject: [PATCH 39/45] feat: add drop tag index of super table feature --- source/dnode/vnode/src/inc/vnodeInt.h | 2 +- source/dnode/vnode/src/meta/metaTable2.c | 264 +++++++---------------- source/dnode/vnode/src/vnd/vnodeSvr.c | 2 +- 3 files changed, 79 insertions(+), 189 deletions(-) diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 975d8b0f8c..bb101e762c 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -181,7 +181,7 @@ int32_t metaUidCacheClear(SMeta* pMeta, uint64_t suid); int32_t metaTbGroupCacheClear(SMeta* pMeta, uint64_t suid); int32_t metaAddIndexToSuperTable(SMeta* pMeta, int64_t version, SVCreateStbReq* pReq); -int metaDropIndexFromSTable(SMeta* pMeta, int64_t version, SDropIndexReq* pReq); +int32_t metaDropIndexFromSuperTable(SMeta* pMeta, int64_t version, SDropIndexReq* pReq); int64_t metaGetTimeSeriesNum(SMeta* pMeta, int type); void metaUpdTimeSeriesNum(SMeta* pMeta); diff --git a/source/dnode/vnode/src/meta/metaTable2.c b/source/dnode/vnode/src/meta/metaTable2.c index e0980ccf0d..fd54a3e0d8 100644 --- a/source/dnode/vnode/src/meta/metaTable2.c +++ b/source/dnode/vnode/src/meta/metaTable2.c @@ -1664,192 +1664,82 @@ int32_t metaAddIndexToSuperTable(SMeta *pMeta, int64_t version, SVCreateStbReq * int32_t metaDropIndexFromSuperTable(SMeta *pMeta, int64_t version, SDropIndexReq *pReq) { int32_t code = TSDB_CODE_SUCCESS; - // TODO + + if (strlen(pReq->colName) == 0 || strlen(pReq->stb) == 0) { + metaError("vgId:%d, %s failed at %s:%d since invalid table name or column name, version:%" PRId64, + TD_VID(pMeta->pVnode), __func__, __FILE__, __LINE__, version); + TAOS_RETURN(TSDB_CODE_INVALID_MSG); + } + + SMetaEntry *pEntry = NULL; + code = metaFetchEntryByName(pMeta, pReq->stb, &pEntry); + if (code) { + metaError("vgId:%d, %s failed at %s:%d since table %s not found, version:%" PRId64, TD_VID(pMeta->pVnode), __func__, + __FILE__, __LINE__, pReq->stb, version); + TAOS_RETURN(code); + } + + if (TSDB_SUPER_TABLE != pEntry->type) { + metaError("vgId:%d, %s failed at %s:%d since table %s type %d is invalid, version:%" PRId64, TD_VID(pMeta->pVnode), + __func__, __FILE__, __LINE__, pReq->stb, pEntry->type, version); + metaFetchEntryFree(&pEntry); + TAOS_RETURN(TSDB_CODE_VND_INVALID_TABLE_ACTION); + } + + if (pReq->stbUid != pEntry->uid) { + metaError("vgId:%d, %s failed at %s:%d since table %s uid %" PRId64 " is not equal to %" PRId64 + ", version:%" PRId64, + TD_VID(pMeta->pVnode), __func__, __FILE__, __LINE__, pReq->stb, pEntry->uid, pReq->stbUid, version); + metaFetchEntryFree(&pEntry); + TAOS_RETURN(TSDB_CODE_INVALID_MSG); + } + + SSchemaWrapper *pTagSchema = &pEntry->stbEntry.schemaTag; + if (pTagSchema->nCols == 1 && pTagSchema->pSchema[0].type == TSDB_DATA_TYPE_JSON) { + metaError("vgId:%d, %s failed at %s:%d since table %s has no tag, version:%" PRId64, TD_VID(pMeta->pVnode), + __func__, __FILE__, __LINE__, pReq->stb, version); + metaFetchEntryFree(&pEntry); + TAOS_RETURN(TSDB_CODE_VND_INVALID_TABLE_ACTION); + } + + // search and set the tag index off + int32_t numOfChangedTags = 0; + for (int32_t i = 0; i < pTagSchema->nCols; i++) { + SSchema *pCol = pTagSchema->pSchema + i; + if (0 == strncmp(pCol->name, pReq->colName, sizeof(pReq->colName))) { + if (!IS_IDX_ON(pCol)) { + metaError("vgId:%d, %s failed at %s:%d since table %s column %s is not indexed, version:%" PRId64, + TD_VID(pMeta->pVnode), __func__, __FILE__, __LINE__, pReq->stb, pReq->colName, version); + metaFetchEntryFree(&pEntry); + TAOS_RETURN(TSDB_CODE_VND_INVALID_TABLE_ACTION); + } + numOfChangedTags++; + SSCHMEA_SET_IDX_OFF(pCol); + break; + } + } + + if (numOfChangedTags != 1) { + metaError("vgId:%d, %s failed at %s:%d since table %s column %s is not found, version:%" PRId64, + TD_VID(pMeta->pVnode), __func__, __FILE__, __LINE__, pReq->stb, pReq->colName, version); + metaFetchEntryFree(&pEntry); + TAOS_RETURN(TSDB_CODE_VND_COL_NOT_EXISTS); + } + + // do handle the entry + pEntry->version = version; + pTagSchema->version++; + code = metaHandleEntry2(pMeta, pEntry); + if (code) { + metaError("vgId:%d, %s failed at %s:%d since %s, uid:%" PRId64 " name:%s version:%" PRId64, TD_VID(pMeta->pVnode), + __func__, __FILE__, __LINE__, tstrerror(code), pEntry->uid, pReq->stb, version); + metaFetchEntryFree(&pEntry); + TAOS_RETURN(code); + } else { + metaInfo("vgId:%d, table %s uid %" PRId64 " is updated, version:%" PRId64, TD_VID(pMeta->pVnode), pReq->stb, + pEntry->uid, version); + } + + metaFetchEntryFree(&pEntry); TAOS_RETURN(code); -#if 0 - int32_t code = 0; - SMetaEntry oStbEntry = {0}; - SMetaEntry nStbEntry = {0}; - - STbDbKey tbDbKey = {0}; - TBC *pUidIdxc = NULL; - TBC *pTbDbc = NULL; - int ret = 0; - int c = -2; - void *pData = NULL; - int nData = 0; - int64_t oversion; - SDecoder dc = {0}; - - tb_uid_t suid = pReq->stbUid; - - if ((code = tdbTbGet(pMeta->pUidIdx, &suid, sizeof(tb_uid_t), &pData, &nData)) != 0) { - goto _err; - } - - tbDbKey.uid = suid; - tbDbKey.version = ((SUidIdxVal *)pData)[0].version; - if ((code = tdbTbGet(pMeta->pTbDb, &tbDbKey, sizeof(tbDbKey), &pData, &nData)) != 0) { - goto _err; - } - - tDecoderInit(&dc, pData, nData); - code = metaDecodeEntry(&dc, &oStbEntry); - if (code != 0) { - goto _err; - } - - SSchema *pCol = NULL; - int32_t colId = -1; - for (int i = 0; i < oStbEntry.stbEntry.schemaTag.nCols; i++) { - SSchema *schema = oStbEntry.stbEntry.schemaTag.pSchema + i; - if (0 == strncmp(schema->name, pReq->colName, sizeof(pReq->colName))) { - if (IS_IDX_ON(schema)) { - pCol = schema; - } - break; - } - } - - if (pCol == NULL) { - metaError("vgId:%d, failed to drop index on %s.%s,since %s", TD_VID(pMeta->pVnode), pReq->stb, pReq->colName, - tstrerror(TSDB_CODE_VND_COL_NOT_EXISTS)); - code = 0; - - goto _err; - } - - /* - * iterator all pTdDbc by uid and version - */ - TBC *pCtbIdxc = NULL; - code = tdbTbcOpen(pMeta->pCtbIdx, &pCtbIdxc, NULL); - if (code != 0) { - goto _err; - } - - code = tdbTbcMoveTo(pCtbIdxc, &(SCtbIdxKey){.suid = suid, .uid = INT64_MIN}, sizeof(SCtbIdxKey), &c); - if (code < 0) { - tdbTbcClose(pCtbIdxc); - goto _err; - } - for (;;) { - void *pKey = NULL, *pVal = NULL; - int nKey = 0, nVal = 0; - - code = tdbTbcNext(pCtbIdxc, &pKey, &nKey, &pVal, &nVal); - if (code < 0) { - tdbFree(pKey); - tdbFree(pVal); - tdbTbcClose(pCtbIdxc); - pCtbIdxc = NULL; - break; - } - if (((SCtbIdxKey *)pKey)->suid != suid) { - tdbFree(pKey); - tdbFree(pVal); - continue; - } - STagIdxKey *pTagIdxKey = NULL; - int32_t nTagIdxKey; - - const void *pTagData = NULL; - int32_t nTagData = 0; - - SCtbIdxKey *table = (SCtbIdxKey *)pKey; - STagVal tagVal = {.cid = pCol->colId}; - if (tTagGet((const STag *)pVal, &tagVal)) { - if (IS_VAR_DATA_TYPE(pCol->type)) { - pTagData = tagVal.pData; - nTagData = (int32_t)tagVal.nData; - } else { - pTagData = &(tagVal.i64); - nTagData = tDataTypes[pCol->type].bytes; - } - } else { - if (!IS_VAR_DATA_TYPE(pCol->type)) { - nTagData = tDataTypes[pCol->type].bytes; - } - } - - code = metaCreateTagIdxKey(suid, pCol->colId, pTagData, nTagData, pCol->type, table->uid, &pTagIdxKey, &nTagIdxKey); - tdbFree(pKey); - tdbFree(pVal); - if (code < 0) { - metaDestroyTagIdxKey(pTagIdxKey); - tdbTbcClose(pCtbIdxc); - goto _err; - } - - metaWLock(pMeta); - ret = tdbTbDelete(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, pMeta->txn); - if (ret < 0) { - metaError("vgId:%d, failed to delete tag idx key:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->stb, - pReq->stbUid, tstrerror(ret)); - } - metaULock(pMeta); - metaDestroyTagIdxKey(pTagIdxKey); - pTagIdxKey = NULL; - } - - // clear idx flag - SSCHMEA_SET_IDX_OFF(pCol); - - nStbEntry.version = version; - nStbEntry.type = TSDB_SUPER_TABLE; - nStbEntry.uid = oStbEntry.uid; - nStbEntry.name = oStbEntry.name; - - SSchemaWrapper *row = tCloneSSchemaWrapper(&oStbEntry.stbEntry.schemaRow); - SSchemaWrapper *tag = tCloneSSchemaWrapper(&oStbEntry.stbEntry.schemaTag); - SColCmprWrapper *cmpr = tCloneSColCmprWrapper(&oStbEntry.colCmpr); - if (row == NULL || tag == NULL || cmpr == NULL) { - tDeleteSchemaWrapper(row); - tDeleteSchemaWrapper(tag); - tDeleteSColCmprWrapper(cmpr); - code = TSDB_CODE_OUT_OF_MEMORY; - - tdbTbcClose(pCtbIdxc); - goto _err; - } - - nStbEntry.stbEntry.schemaRow = *row; - nStbEntry.stbEntry.schemaTag = *tag; - nStbEntry.stbEntry.rsmaParam = oStbEntry.stbEntry.rsmaParam; - nStbEntry.colCmpr = *cmpr; - - nStbEntry.colCmpr = oStbEntry.colCmpr; - - metaWLock(pMeta); - // update table.db - ret = metaSaveToTbDb(pMeta, &nStbEntry); - if (ret < 0) { - metaError("vgId:%d, failed to save tb db:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->stb, - pReq->stbUid, tstrerror(ret)); - } - // update uid index - ret = metaUpdateUidIdx(pMeta, &nStbEntry); - if (ret < 0) { - metaError("vgId:%d, failed to update uid idx:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->stb, - pReq->stbUid, tstrerror(ret)); - } - metaULock(pMeta); - - tDeleteSchemaWrapper(tag); - tDeleteSchemaWrapper(row); - tDeleteSColCmprWrapper(cmpr); - - if (oStbEntry.pBuf) taosMemoryFree(oStbEntry.pBuf); - tDecoderClear(&dc); - tdbFree(pData); - - tdbTbcClose(pCtbIdxc); - return TSDB_CODE_SUCCESS; -_err: - if (oStbEntry.pBuf) taosMemoryFree(oStbEntry.pBuf); - tDecoderClear(&dc); - tdbFree(pData); - - return code; -#endif } \ No newline at end of file diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 8d757f95a6..2b6d95b9da 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -2458,7 +2458,7 @@ static int32_t vnodeProcessDropIndexReq(SVnode *pVnode, int64_t ver, void *pReq, return code; } - code = metaDropIndexFromSTable(pVnode->pMeta, ver, &req); + code = metaDropIndexFromSuperTable(pVnode->pMeta, ver, &req); if (code) { pRsp->code = code; return code; From 389f545a53196701230980baefbd78c4637ab7f8 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sat, 14 Dec 2024 18:06:50 +0800 Subject: [PATCH 40/45] more code --- source/dnode/vnode/src/inc/vnodeInt.h | 8 +- source/dnode/vnode/src/meta/metaEntry2.c | 7 +- source/dnode/vnode/src/meta/metaTable.c | 163 ++-------------------- source/dnode/vnode/src/meta/metaTable2.c | 90 ++++++++++++ source/dnode/vnode/src/sma/smaTimeRange.c | 4 +- source/dnode/vnode/src/vnd/vnodeSvr.c | 4 +- 6 files changed, 111 insertions(+), 165 deletions(-) diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index bb101e762c..ef4b233f94 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -155,17 +155,13 @@ int metaCommit(SMeta* pMeta, TXN* txn); int metaFinishCommit(SMeta* pMeta, TXN* txn); int metaPrepareAsyncCommit(SMeta* pMeta); int metaAbort(SMeta* pMeta); -int metaCreateSTable(SMeta* pMeta, int64_t version, SVCreateStbReq* pReq); int metaCreateSuperTable(SMeta* pMeta, int64_t version, SVCreateStbReq* pReq); -int metaAlterSTable(SMeta* pMeta, int64_t version, SVCreateStbReq* pReq); -int metaDropSTable(SMeta* pMeta, int64_t verison, SVDropStbReq* pReq, SArray* tbUidList); +int32_t metaAlterSuperTable(SMeta* pMeta, int64_t version, SVCreateStbReq* pReq); int32_t metaDropSuperTable(SMeta* pMeta, int64_t verison, SVDropStbReq* pReq); -int metaCreateTable(SMeta* pMeta, int64_t version, SVCreateTbReq* pReq, STableMetaRsp** pMetaRsp); int32_t metaCreateTable2(SMeta* pMeta, int64_t version, SVCreateTbReq* pReq, STableMetaRsp** ppRsp); -int metaDropTable(SMeta* pMeta, int64_t version, SVDropTbReq* pReq, SArray* tbUids, int64_t* tbUid); int32_t metaDropTable2(SMeta* pMeta, int64_t version, SVDropTbReq* pReq); int32_t metaTrimTables(SMeta* pMeta); -int32_t metaDropTables(SMeta* pMeta, SArray* tbUids); +int32_t metaDropMultipleTables(SMeta* pMeta, int64_t version, SArray* tbUids); int metaTtlFindExpired(SMeta* pMeta, int64_t timePointMs, SArray* tbUids, int32_t ttlDropMaxCount); int metaAlterTable(SMeta* pMeta, int64_t version, SVAlterTbReq* pReq, STableMetaRsp* pMetaRsp); int metaUpdateChangeTimeWithLock(SMeta* pMeta, tb_uid_t uid, int64_t changeTimeMs); diff --git a/source/dnode/vnode/src/meta/metaEntry2.c b/source/dnode/vnode/src/meta/metaEntry2.c index 1b00250aa4..fd129ec5d5 100644 --- a/source/dnode/vnode/src/meta/metaEntry2.c +++ b/source/dnode/vnode/src/meta/metaEntry2.c @@ -1085,7 +1085,6 @@ static int32_t metaHandleSuperTableCreate(SMeta *pMeta, const SMetaEntry *pEntry if (TSDB_CODE_SUCCESS == code) { pMeta->pVnode->config.vndStats.numOfSTables++; - pMeta->changed = true; metaInfo("vgId:%d, %s success, version:%" PRId64 " type:%d uid:%" PRId64 " name:%s", TD_VID(pMeta->pVnode), __func__, pEntry->version, pEntry->type, pEntry->uid, pEntry->name); @@ -1135,7 +1134,6 @@ static int32_t metaHandleNormalTableCreate(SMeta *pMeta, const SMetaEntry *pEntr if (TSDB_CODE_SUCCESS == code) { pMeta->pVnode->config.vndStats.numOfNTables++; pMeta->pVnode->config.vndStats.numOfNTimeSeries += pEntry->ntbEntry.schemaRow.nCols - 1; - pMeta->changed = true; if (!TSDB_CACHE_NO(pMeta->pVnode->config)) { int32_t rc = tsdbCacheNewTable(pMeta->pVnode->pTsdb, pEntry->uid, -1, &pEntry->ntbEntry.schemaRow); @@ -1229,7 +1227,6 @@ static int32_t metaHandleChildTableCreate(SMeta *pMeta, const SMetaEntry *pEntry } } - pMeta->changed = true; } else { metaErr(TD_VID(pMeta->pVnode), code); } @@ -1309,7 +1306,6 @@ static int32_t metaHandleNormalTableDrop(SMeta *pMeta, const SMetaEntry *pEntry) } } - pMeta->changed = true; metaFetchEntryFree(&pOldEntry); return code; } @@ -1424,7 +1420,6 @@ static int32_t metaHandleChildTableDrop(SMeta *pMeta, const SMetaEntry *pEntry, *tbUid = uid; } #endif - pMeta->changed = true; metaFetchEntryFree(&pChild); metaFetchEntryFree(&pSuper); return code; @@ -1781,7 +1776,6 @@ static int32_t metaHandleSuperTableDrop(SMeta *pMeta, const SMetaEntry *pEntry) // do other stuff metaUpdTimeSeriesNum(pMeta); - pMeta->changed = true; // free resource and return taosArrayDestroy(childList); @@ -1858,6 +1852,7 @@ int32_t metaHandleEntry2(SMeta *pMeta, const SMetaEntry *pEntry) { } if (TSDB_CODE_SUCCESS == code) { + pMeta->changed = true; metaDebug("vgId:%d, %s success, version:%" PRId64 " type:%d uid:%" PRId64 " name:%s", vgId, __func__, pEntry->version, pEntry->type, pEntry->uid, pEntry->type > 0 ? pEntry->name : ""); } else { diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 8bad8ab569..f82fea8fe5 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -301,7 +301,7 @@ void metaTimeSeriesNotifyCheck(SMeta *pMeta) { #endif } -int metaCreateSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { +static int metaCreateSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { SMetaEntry me = {0}; int kLen = 0; int vLen = 0; @@ -361,7 +361,7 @@ _err: return code; } -int metaDropSTable(SMeta *pMeta, int64_t verison, SVDropStbReq *pReq, SArray *tbUidList) { +static int metaDropSTable(SMeta *pMeta, int64_t verison, SVDropStbReq *pReq, SArray *tbUidList) { void *pKey = NULL; int nKey = 0; void *pData = NULL; @@ -521,7 +521,7 @@ static int32_t metaGetSubtables(SMeta *pMeta, int64_t suid, SArray *uids) { return 0; } -int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { +static int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { SMetaEntry oStbEntry = {0}; SMetaEntry nStbEntry = {0}; TBC *pUidIdxc = NULL; @@ -689,7 +689,8 @@ _exit: tdbTbcClose(pUidIdxc); return 0; } -int metaAddIndexToSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { + +static int metaAddIndexToSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { SMetaEntry oStbEntry = {0}; SMetaEntry nStbEntry = {0}; STbDbKey tbDbKey = {0}; @@ -874,7 +875,8 @@ _err: return code; } -int metaDropIndexFromSTable(SMeta *pMeta, int64_t version, SDropIndexReq *pReq) { + +static int metaDropIndexFromSTable(SMeta *pMeta, int64_t version, SDropIndexReq *pReq) { int32_t code = 0; SMetaEntry oStbEntry = {0}; SMetaEntry nStbEntry = {0}; @@ -1061,7 +1063,7 @@ _err: return code; } -int metaCreateTable(SMeta *pMeta, int64_t ver, SVCreateTbReq *pReq, STableMetaRsp **pMetaRsp) { +static int metaCreateTable(SMeta *pMeta, int64_t ver, SVCreateTbReq *pReq, STableMetaRsp **pMetaRsp) { SMetaEntry me = {0}; SMetaReader mr = {0}; int32_t ret; @@ -1230,7 +1232,7 @@ _err: return TSDB_CODE_FAILED; } -int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq, SArray *tbUids, tb_uid_t *tbUid) { +static int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq, SArray *tbUids, tb_uid_t *tbUid) { void *pData = NULL; int nData = 0; int rc = 0; @@ -1284,7 +1286,7 @@ _exit: return rc; } -int32_t metaDropTables(SMeta *pMeta, SArray *tbUids) { +static int32_t metaDropTables(SMeta *pMeta, SArray *tbUids) { int32_t code = 0; if (taosArrayGetSize(tbUids) == 0) return TSDB_CODE_SUCCESS; @@ -1676,7 +1678,7 @@ static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type, tb_uid_t *p return 0; } // opt ins_tables -int metaUpdateBtimeIdx(SMeta *pMeta, const SMetaEntry *pME) { +static int metaUpdateBtimeIdx(SMeta *pMeta, const SMetaEntry *pME) { SBtimeIdxKey btimeKey = {0}; if (metaBuildBtimeIdxKey(&btimeKey, pME) < 0) { return 0; @@ -1687,13 +1689,14 @@ int metaUpdateBtimeIdx(SMeta *pMeta, const SMetaEntry *pME) { return tdbTbUpsert(pMeta->pBtimeIdx, &btimeKey, sizeof(btimeKey), NULL, 0, pMeta->txn); } -int metaDeleteBtimeIdx(SMeta *pMeta, const SMetaEntry *pME) { +static int metaDeleteBtimeIdx(SMeta *pMeta, const SMetaEntry *pME) { SBtimeIdxKey btimeKey = {0}; if (metaBuildBtimeIdxKey(&btimeKey, pME) < 0) { return 0; } return tdbTbDelete(pMeta->pBtimeIdx, &btimeKey, sizeof(btimeKey), pMeta->txn); } + int metaUpdateNcolIdx(SMeta *pMeta, const SMetaEntry *pME) { SNcolIdxKey ncolKey = {0}; if (metaBuildNColIdxKey(&ncolKey, pME) < 0) { @@ -2625,144 +2628,6 @@ static int metaUpdateTableOptions(SMeta *pMeta, int64_t version, SVAlterTbReq *p return 0; } -static int metaAddTagIndex(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTbReq) { - SMetaEntry stbEntry = {0}; - void *pVal = NULL; - int nVal = 0; - int ret; - int c; - tb_uid_t uid, suid; - int64_t oversion; - const void *pData = NULL; - int nData = 0; - SDecoder dc = {0}; - - if (pAlterTbReq->tagName == NULL) { - return terrno = TSDB_CODE_INVALID_MSG; - } - - // search name index - ret = tdbTbGet(pMeta->pNameIdx, pAlterTbReq->tbName, strlen(pAlterTbReq->tbName) + 1, &pVal, &nVal); - if (ret < 0) { - return terrno = TSDB_CODE_TDB_TABLE_NOT_EXIST; - } else { - uid = *(tb_uid_t *)pVal; - tdbFree(pVal); - pVal = NULL; - } - - if (tdbTbGet(pMeta->pUidIdx, &uid, sizeof(tb_uid_t), &pVal, &nVal) == -1) { - ret = -1; - goto _err; - } - suid = ((SUidIdxVal *)pVal)[0].suid; - - STbDbKey tbDbKey = {0}; - tbDbKey.uid = suid; - tbDbKey.version = ((SUidIdxVal *)pVal)[0].version; - ret = tdbTbGet(pMeta->pTbDb, &tbDbKey, sizeof(tbDbKey), &pVal, &nVal); - if (ret < 0) { - goto _err; - } - tDecoderInit(&dc, pVal, nVal); - ret = metaDecodeEntry(&dc, &stbEntry); - if (ret < 0) { - tDecoderClear(&dc); - goto _err; - } - - // Get target schema info - SSchemaWrapper *pTagSchema = &stbEntry.stbEntry.schemaTag; - if (pTagSchema->nCols == 1 && pTagSchema->pSchema[0].type == TSDB_DATA_TYPE_JSON) { - terrno = TSDB_CODE_VND_COL_ALREADY_EXISTS; - goto _err; - } - SSchema *pCol = NULL; - int32_t iCol = 0; - for (;;) { - pCol = NULL; - if (iCol >= pTagSchema->nCols) break; - pCol = &pTagSchema->pSchema[iCol]; - if (strcmp(pCol->name, pAlterTbReq->tagName) == 0) break; - iCol++; - } - - if (iCol == 0) { - terrno = TSDB_CODE_VND_COL_ALREADY_EXISTS; - goto _err; - } - if (pCol == NULL) { - terrno = TSDB_CODE_VND_COL_NOT_EXISTS; - goto _err; - } - - /* - * iterator all pTdDbc by uid and version - */ - TBC *pCtbIdxc = NULL; - TAOS_CHECK_RETURN(tdbTbcOpen(pMeta->pCtbIdx, &pCtbIdxc, NULL)); - int rc = tdbTbcMoveTo(pCtbIdxc, &(SCtbIdxKey){.suid = suid, .uid = INT64_MIN}, sizeof(SCtbIdxKey), &c); - if (rc < 0) { - tdbTbcClose(pCtbIdxc); - goto _err; - } - for (;;) { - void *pKey, *pVal; - int nKey, nVal; - rc = tdbTbcNext(pCtbIdxc, &pKey, &nKey, &pVal, &nVal); - if (rc < 0) break; - if (((SCtbIdxKey *)pKey)->suid != uid) { - tdbFree(pKey); - tdbFree(pVal); - continue; - } - STagIdxKey *pTagIdxKey = NULL; - int32_t nTagIdxKey; - - const void *pTagData = NULL; - int32_t nTagData = 0; - - STagVal tagVal = {.cid = pCol->colId}; - if (tTagGet((const STag *)pVal, &tagVal)) { - if (IS_VAR_DATA_TYPE(pCol->type)) { - pTagData = tagVal.pData; - nTagData = (int32_t)tagVal.nData; - } else { - pTagData = &(tagVal.i64); - nTagData = tDataTypes[pCol->type].bytes; - } - } else { - if (!IS_VAR_DATA_TYPE(pCol->type)) { - nTagData = tDataTypes[pCol->type].bytes; - } - } - if (metaCreateTagIdxKey(suid, pCol->colId, pTagData, nTagData, pCol->type, uid, &pTagIdxKey, &nTagIdxKey) < 0) { - tdbFree(pKey); - tdbFree(pVal); - metaDestroyTagIdxKey(pTagIdxKey); - tdbTbcClose(pCtbIdxc); - goto _err; - } - ret = tdbTbUpsert(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, NULL, 0, pMeta->txn); - if (ret < 0) { - metaError("meta/table: failed to upsert tag idx:%s uid:%" PRId64, stbEntry.name, stbEntry.uid); - } - metaDestroyTagIdxKey(pTagIdxKey); - pTagIdxKey = NULL; - } - tdbTbcClose(pCtbIdxc); - return 0; - -_err: - // tDecoderClear(&dc1); - // tDecoderClear(&dc2); - // if (ctbEntry.pBuf) taosMemoryFree(ctbEntry.pBuf); - // if (stbEntry.pBuf) tdbFree(stbEntry.pBuf); - // tdbTbcClose(pTbDbc); - // tdbTbcClose(pUidIdxc); - return TSDB_CODE_FAILED; -} - typedef struct SMetaPair { void *key; int nkey; @@ -2887,7 +2752,7 @@ static int metaDropTagIndex(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterT _err: return TSDB_CODE_FAILED; } -int32_t metaUpdateTableColCompress(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq) { +static int32_t metaUpdateTableColCompress(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq) { // impl later SMetaEntry tbEntry = {0}; void *pVal = NULL; diff --git a/source/dnode/vnode/src/meta/metaTable2.c b/source/dnode/vnode/src/meta/metaTable2.c index fd54a3e0d8..ec6381c6ed 100644 --- a/source/dnode/vnode/src/meta/metaTable2.c +++ b/source/dnode/vnode/src/meta/metaTable2.c @@ -1742,4 +1742,94 @@ int32_t metaDropIndexFromSuperTable(SMeta *pMeta, int64_t version, SDropIndexReq metaFetchEntryFree(&pEntry); TAOS_RETURN(code); +} + +int32_t metaAlterSuperTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { + int32_t code = TSDB_CODE_SUCCESS; + + if (NULL == pReq->name || strlen(pReq->name) == 0) { + metaError("vgId:%d, %s failed at %s:%d since invalid table name, version:%" PRId64, TD_VID(pMeta->pVnode), __func__, + __FILE__, __LINE__, version); + TAOS_RETURN(TSDB_CODE_INVALID_MSG); + } + + SMetaEntry *pEntry = NULL; + code = metaFetchEntryByName(pMeta, pReq->name, &pEntry); + if (code) { + metaError("vgId:%d, %s failed at %s:%d since table %s not found, version:%" PRId64, TD_VID(pMeta->pVnode), __func__, + __FILE__, __LINE__, pReq->name, version); + TAOS_RETURN(TSDB_CODE_TDB_STB_NOT_EXIST); + } + + if (pEntry->type != TSDB_SUPER_TABLE) { + metaError("vgId:%d, %s failed at %s:%d since table %s type %d is invalid, version:%" PRId64, TD_VID(pMeta->pVnode), + __func__, __FILE__, __LINE__, pReq->name, pEntry->type, version); + metaFetchEntryFree(&pEntry); + TAOS_RETURN(TSDB_CODE_VND_INVALID_TABLE_ACTION); + } + + SMetaEntry entry = { + .version = version, + .type = TSDB_SUPER_TABLE, + .uid = pReq->suid, + .name = pReq->name, + .stbEntry.schemaRow = pReq->schemaRow, + .stbEntry.schemaTag = pReq->schemaTag, + .colCmpr = pReq->colCmpr, + }; + TABLE_SET_COL_COMPRESSED(entry.flags); + + // do handle the entry + code = metaHandleEntry2(pMeta, &entry); + if (code) { + metaError("vgId:%d, %s failed at %s:%d since %s, uid:%" PRId64 " name:%s version:%" PRId64, TD_VID(pMeta->pVnode), + __func__, __FILE__, __LINE__, tstrerror(code), pReq->suid, pReq->name, version); + metaFetchEntryFree(&pEntry); + TAOS_RETURN(code); + } else { + metaInfo("vgId:%d, table %s uid %" PRId64 " is updated, version:%" PRId64, TD_VID(pMeta->pVnode), pReq->name, + pReq->suid, version); + } + + metaFetchEntryFree(&pEntry); + TAOS_RETURN(code); +} + +int32_t metaDropMultipleTables(SMeta *pMeta, int64_t version, SArray *uidArray) { + int32_t code = 0; + + if (taosArrayGetSize(uidArray) == 0) { + return TSDB_CODE_SUCCESS; + } + + for (int32_t i = 0; i < taosArrayGetSize(uidArray); i++) { + tb_uid_t uid = *(tb_uid_t *)taosArrayGet(uidArray, i); + SMetaInfo info; + code = metaGetInfo(pMeta, uid, &info, NULL); + if (code) { + metaError("vgId:%d, %s failed at %s:%d since table uid %" PRId64 " not found, code:%d", TD_VID(pMeta->pVnode), + __func__, __FILE__, __LINE__, uid, code); + return code; + } + + SMetaEntry entry = { + .version = version, + .uid = uid, + }; + + if (info.suid == 0) { + entry.type = -TSDB_NORMAL_TABLE; + } else if (info.suid == uid) { + entry.type = -TSDB_SUPER_TABLE; + } else { + entry.type = -TSDB_CHILD_TABLE; + } + code = metaHandleEntry2(pMeta, &entry); + if (code) { + metaError("vgId:%d, %s failed at %s:%d since %s, uid:%" PRId64 " version:%" PRId64, TD_VID(pMeta->pVnode), + __func__, __FILE__, __LINE__, tstrerror(code), uid, version); + return code; + } + } + return code; } \ No newline at end of file diff --git a/source/dnode/vnode/src/sma/smaTimeRange.c b/source/dnode/vnode/src/sma/smaTimeRange.c index 86246aace1..c9868f0398 100644 --- a/source/dnode/vnode/src/sma/smaTimeRange.c +++ b/source/dnode/vnode/src/sma/smaTimeRange.c @@ -129,7 +129,7 @@ static int32_t tdProcessTSmaCreateImpl(SSma *pSma, int64_t ver, const char *pMsg pReq.schemaRow = pCfg->schemaRow; pReq.schemaTag = pCfg->schemaTag; - TAOS_CHECK_EXIT(metaCreateSTable(SMA_META(pSma), ver, &pReq)); + TAOS_CHECK_EXIT(metaCreateSuperTable(SMA_META(pSma), ver, &pReq)); } else { TAOS_CHECK_EXIT(TSDB_CODE_TSMA_INVALID_STAT); } @@ -204,7 +204,7 @@ int32_t smaBlockToSubmit(SVnode *pVnode, const SArray *pBlocks, const STSchema * continue; } - if( taosArrayPush(pReq->aSubmitTbData, &tbData) == NULL) { + if (taosArrayPush(pReq->aSubmitTbData, &tbData) == NULL) { code = terrno; continue; } diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 2b6d95b9da..1e3580e388 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -1038,7 +1038,7 @@ static int32_t vnodeProcessDropTtlTbReq(SVnode *pVnode, int64_t ver, void *pReq, } if (ttlReq.nUids > 0) { - int32_t code = metaDropTables(pVnode->pMeta, ttlReq.pTbUids); + int32_t code = metaDropMultipleTables(pVnode->pMeta, ver, ttlReq.pTbUids); if (code) return code; code = tqUpdateTbUidList(pVnode->pTq, ttlReq.pTbUids, false); @@ -1344,7 +1344,7 @@ static int32_t vnodeProcessAlterStbReq(SVnode *pVnode, int64_t ver, void *pReq, return code; } - code = metaAlterSTable(pVnode->pMeta, ver, &req); + code = metaAlterSuperTable(pVnode->pMeta, ver, &req); if (code) { pRsp->code = code; tDecoderClear(&dc); From 5cac78d10bd91c967528a4d5ca1218c0fb07d2af Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sat, 14 Dec 2024 20:00:09 +0800 Subject: [PATCH 41/45] fix: compile issue --- source/dnode/vnode/src/meta/metaTable2.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaTable2.c b/source/dnode/vnode/src/meta/metaTable2.c index ec6381c6ed..586c0aa8a9 100644 --- a/source/dnode/vnode/src/meta/metaTable2.c +++ b/source/dnode/vnode/src/meta/metaTable2.c @@ -1575,8 +1575,7 @@ int32_t metaAddIndexToSuperTable(SMeta *pMeta, int64_t version, SVCreateStbReq * } if (pEntry->stbEntry.schemaTag.version >= pReq->schemaTag.version) { - metaError("vgId:%d, %s failed at %s:%d since table %s tag schema version %" PRId64 " is not less than %" PRId64 - ", version:%" PRId64, + metaError("vgId:%d, %s failed at %s:%d since table %s tag schema version %d is not less than %d, version:%" PRId64, TD_VID(pMeta->pVnode), __func__, __FILE__, __LINE__, pReq->name, pEntry->stbEntry.schemaTag.version, pReq->schemaTag.version, version); metaFetchEntryFree(&pEntry); @@ -1603,8 +1602,7 @@ int32_t metaAddIndexToSuperTable(SMeta *pMeta, int64_t version, SVCreateStbReq * } if (pOldTagSchema->version >= pNewTagSchema->version) { - metaError("vgId:%d, %s failed at %s:%d since table %s tag schema version %" PRId64 " is not less than %" PRId64 - ", version:%" PRId64, + metaError("vgId:%d, %s failed at %s:%d since table %s tag schema version %d is not less than %d, version:%" PRId64, TD_VID(pMeta->pVnode), __func__, __FILE__, __LINE__, pReq->name, pOldTagSchema->version, pNewTagSchema->version, version); metaFetchEntryFree(&pEntry); From a0cae0ddca8062bc5d75e35eb8e6b1157280fd19 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 16 Dec 2024 10:59:55 +0800 Subject: [PATCH 42/45] more code --- source/dnode/vnode/src/inc/meta.h | 2 +- source/dnode/vnode/src/meta/metaEntry2.c | 11 ++++---- source/dnode/vnode/src/meta/metaOpen.c | 2 +- source/dnode/vnode/src/meta/metaSnapshot.c | 6 ++-- source/dnode/vnode/src/meta/metaTable.c | 2 ++ source/dnode/vnode/src/meta/metaTable2.c | 32 ++++++++++++---------- 6 files changed, 28 insertions(+), 27 deletions(-) diff --git a/source/dnode/vnode/src/inc/meta.h b/source/dnode/vnode/src/inc/meta.h index 87d7d2150e..c329bc8812 100644 --- a/source/dnode/vnode/src/inc/meta.h +++ b/source/dnode/vnode/src/inc/meta.h @@ -60,7 +60,7 @@ int metaRemoveTableFromIdx(SMeta* pMeta, tb_uid_t uid); static FORCE_INLINE tb_uid_t metaGenerateUid(SMeta* pMeta) { return tGenIdPI64(); } // metaTable ================== -int metaHandleEntry(SMeta* pMeta, const SMetaEntry* pME); +int32_t metaHandleEntry2(SMeta* pMeta, const SMetaEntry* pEntry); // metaCache ================== int32_t metaCacheOpen(SMeta* pMeta); diff --git a/source/dnode/vnode/src/meta/metaEntry2.c b/source/dnode/vnode/src/meta/metaEntry2.c index fd129ec5d5..0600e42708 100644 --- a/source/dnode/vnode/src/meta/metaEntry2.c +++ b/source/dnode/vnode/src/meta/metaEntry2.c @@ -452,13 +452,12 @@ static int32_t metaSchemaTableUpdate(SMeta *pMeta, const SMetaHandleParam *pPara } // check tag schema - if (pOldEntry->stbEntry.schemaTag.version != pEntry->stbEntry.schemaTag.version) { - code = metaUpdateSuperTableTagSchema(pMeta, pParam); - if (code) { - metaErr(TD_VID(pMeta->pVnode), code); - return code; - } + code = metaUpdateSuperTableTagSchema(pMeta, pParam); + if (code) { + metaErr(TD_VID(pMeta->pVnode), code); + return code; } + } else { return TSDB_CODE_INVALID_PARA; } diff --git a/source/dnode/vnode/src/meta/metaOpen.c b/source/dnode/vnode/src/meta/metaOpen.c index 0bced535cb..ebb85ea39d 100644 --- a/source/dnode/vnode/src/meta/metaOpen.c +++ b/source/dnode/vnode/src/meta/metaOpen.c @@ -330,7 +330,7 @@ static int32_t metaGenerateNewMeta(SMeta **ppMeta) { tdbTbGet(pMeta->pUidIdx, &me.ctbEntry.suid, sizeof(me.ctbEntry.suid), NULL, NULL) != 0) { metaError("vgId:%d failed to get super table uid:%" PRId64 " for child table uid:%" PRId64, TD_VID(pVnode), me.ctbEntry.suid, uid); - } else if (metaHandleEntry(pNewMeta, &me) != 0) { + } else if (metaHandleEntry2(pNewMeta, &me) != 0) { metaError("vgId:%d failed to handle entry, uid:%" PRId64, TD_VID(pVnode), uid); } } diff --git a/source/dnode/vnode/src/meta/metaSnapshot.c b/source/dnode/vnode/src/meta/metaSnapshot.c index b2826ec45a..30a20cd68d 100644 --- a/source/dnode/vnode/src/meta/metaSnapshot.c +++ b/source/dnode/vnode/src/meta/metaSnapshot.c @@ -72,7 +72,6 @@ int32_t metaSnapRead(SMetaSnapReader* pReader, uint8_t** ppData) { int32_t nKey = 0; int32_t nData = 0; STbDbKey key; - SMetaInfo info; *ppData = NULL; for (;;) { @@ -85,8 +84,7 @@ int32_t metaSnapRead(SMetaSnapReader* pReader, uint8_t** ppData) { goto _exit; } - if (key.version < pReader->sver // - || metaGetInfo(pReader->pMeta, key.uid, &info, NULL) == TSDB_CODE_NOT_FOUND) { + if (key.version < pReader->sver) { if (tdbTbcMoveToNext(pReader->pTbc) != 0) { metaTrace("vgId:%d, vnode snapshot meta read data done", TD_VID(pReader->pMeta->pVnode)); } @@ -199,7 +197,7 @@ int32_t metaSnapWrite(SMetaSnapWriter* pWriter, uint8_t* pData, uint32_t nData) code = metaDecodeEntry(pDecoder, &metaEntry); TSDB_CHECK_CODE(code, lino, _exit); - code = metaHandleEntry(pMeta, &metaEntry); + code = metaHandleEntry2(pMeta, &metaEntry); TSDB_CHECK_CODE(code, lino, _exit); _exit: diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index f82fea8fe5..ee37dd1f90 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -46,6 +46,8 @@ static int metaDeleteBtimeIdx(SMeta *pMeta, const SMetaEntry *pME); static int metaUpdateNcolIdx(SMeta *pMeta, const SMetaEntry *pME); static int metaDeleteNcolIdx(SMeta *pMeta, const SMetaEntry *pME); +int metaHandleEntry(SMeta *pMeta, const SMetaEntry *pME); + int32_t updataTableColCmpr(SColCmprWrapper *pWp, SSchema *pSchema, int8_t add, uint32_t compress) { int32_t nCols = pWp->nCols; int32_t ver = pWp->version; diff --git a/source/dnode/vnode/src/meta/metaTable2.c b/source/dnode/vnode/src/meta/metaTable2.c index 586c0aa8a9..af902809d6 100644 --- a/source/dnode/vnode/src/meta/metaTable2.c +++ b/source/dnode/vnode/src/meta/metaTable2.c @@ -1574,13 +1574,14 @@ int32_t metaAddIndexToSuperTable(SMeta *pMeta, int64_t version, SVCreateStbReq * TAOS_RETURN(TSDB_CODE_INVALID_MSG); } - if (pEntry->stbEntry.schemaTag.version >= pReq->schemaTag.version) { - metaError("vgId:%d, %s failed at %s:%d since table %s tag schema version %d is not less than %d, version:%" PRId64, - TD_VID(pMeta->pVnode), __func__, __FILE__, __LINE__, pReq->name, pEntry->stbEntry.schemaTag.version, - pReq->schemaTag.version, version); - metaFetchEntryFree(&pEntry); - TAOS_RETURN(TSDB_CODE_INVALID_MSG); - } + // if (pEntry->stbEntry.schemaTag.version >= pReq->schemaTag.version) { + // metaError("vgId:%d, %s failed at %s:%d since table %s tag schema version %d is not less than %d, version:%" + // PRId64, + // TD_VID(pMeta->pVnode), __func__, __FILE__, __LINE__, pReq->name, pEntry->stbEntry.schemaTag.version, + // pReq->schemaTag.version, version); + // metaFetchEntryFree(&pEntry); + // TAOS_RETURN(TSDB_CODE_INVALID_MSG); + // } // do change the entry SSchemaWrapper *pOldTagSchema = &pEntry->stbEntry.schemaTag; @@ -1601,13 +1602,14 @@ int32_t metaAddIndexToSuperTable(SMeta *pMeta, int64_t version, SVCreateStbReq * TAOS_RETURN(TSDB_CODE_INVALID_MSG); } - if (pOldTagSchema->version >= pNewTagSchema->version) { - metaError("vgId:%d, %s failed at %s:%d since table %s tag schema version %d is not less than %d, version:%" PRId64, - TD_VID(pMeta->pVnode), __func__, __FILE__, __LINE__, pReq->name, pOldTagSchema->version, - pNewTagSchema->version, version); - metaFetchEntryFree(&pEntry); - TAOS_RETURN(TSDB_CODE_INVALID_MSG); - } + // if (pOldTagSchema->version >= pNewTagSchema->version) { + // metaError("vgId:%d, %s failed at %s:%d since table %s tag schema version %d is not less than %d, version:%" + // PRId64, + // TD_VID(pMeta->pVnode), __func__, __FILE__, __LINE__, pReq->name, pOldTagSchema->version, + // pNewTagSchema->version, version); + // metaFetchEntryFree(&pEntry); + // TAOS_RETURN(TSDB_CODE_INVALID_MSG); + // } int32_t numOfChangedTags = 0; for (int32_t i = 0; i < pOldTagSchema->nCols; i++) { @@ -1670,7 +1672,7 @@ int32_t metaDropIndexFromSuperTable(SMeta *pMeta, int64_t version, SDropIndexReq } SMetaEntry *pEntry = NULL; - code = metaFetchEntryByName(pMeta, pReq->stb, &pEntry); + code = metaFetchEntryByUid(pMeta, pReq->stbUid, &pEntry); if (code) { metaError("vgId:%d, %s failed at %s:%d since table %s not found, version:%" PRId64, TD_VID(pMeta->pVnode), __func__, __FILE__, __LINE__, pReq->stb, version); From 5b6136ec8280534b9653a7424a77432f466d83c4 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Mon, 16 Dec 2024 11:00:51 +0800 Subject: [PATCH 43/45] fix(meta/tsdb cache): fix drop sub tables for super table --- source/dnode/vnode/src/meta/metaEntry2.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/meta/metaEntry2.c b/source/dnode/vnode/src/meta/metaEntry2.c index fd129ec5d5..2a99796359 100644 --- a/source/dnode/vnode/src/meta/metaEntry2.c +++ b/source/dnode/vnode/src/meta/metaEntry2.c @@ -1745,6 +1745,11 @@ static int32_t metaHandleSuperTableDrop(SMeta *pMeta, const SMetaEntry *pEntry) return code; } + if (tsdbCacheDropSubTables(pMeta->pVnode->pTsdb, childList, pEntry->uid) < 0) { + metaError("vgId:%d, failed to drop stb:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pEntry->name, + pEntry->uid, tstrerror(terrno)); + } + // loop to drop all child tables for (int32_t i = 0; i < taosArrayGetSize(childList); i++) { SMetaEntry childEntry = { @@ -1859,4 +1864,4 @@ int32_t metaHandleEntry2(SMeta *pMeta, const SMetaEntry *pEntry) { metaErr(vgId, code); } TAOS_RETURN(code); -} \ No newline at end of file +} From 7aacaf580e4cade3b4fbc9945229d5e54f6ac0b2 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Mon, 16 Dec 2024 11:27:27 +0800 Subject: [PATCH 44/45] meta/tsdb cache: add/drop column with supertables --- source/dnode/vnode/src/meta/metaEntry2.c | 54 ++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/source/dnode/vnode/src/meta/metaEntry2.c b/source/dnode/vnode/src/meta/metaEntry2.c index 2a99796359..108c6a9ae6 100644 --- a/source/dnode/vnode/src/meta/metaEntry2.c +++ b/source/dnode/vnode/src/meta/metaEntry2.c @@ -1624,6 +1624,60 @@ static int32_t metaHandleSuperTableUpdate(SMeta *pMeta, const SMetaEntry *pEntry return code; } + int nCols = pEntry->stbEntry.schemaRow.nCols; + int onCols = pOldEntry->stbEntry.schemaRow.nCols; + int32_t deltaCol = nCols - onCols; + bool updStat = deltaCol != 0 && !metaTbInFilterCache(pMeta, pEntry->name, 1); + + if (!TSDB_CACHE_NO(pMeta->pVnode->config)) { + STsdb *pTsdb = pMeta->pVnode->pTsdb; + SArray *uids = NULL; /*taosArrayInit(8, sizeof(int64_t)); + if (uids == NULL) { + metaErr(TD_VID(pMeta->pVnode), code); + metaFetchEntryFree(&pOldEntry); + return terrno; + }*/ + if (deltaCol == 1) { + int16_t cid = pEntry->stbEntry.schemaRow.pSchema[nCols - 1].colId; + int8_t col_type = pEntry->stbEntry.schemaRow.pSchema[nCols - 1].type; + + code = metaGetChildUidsOfSuperTable(pMeta, pEntry->uid, &uids); + if (code) { + metaErr(TD_VID(pMeta->pVnode), code); + metaFetchEntryFree(&pOldEntry); + return code; + } + // TAOS_CHECK_RETURN(metaGetSubtables(pMeta, pEntry->uid, uids)); + TAOS_CHECK_RETURN(tsdbCacheNewSTableColumn(pTsdb, uids, cid, col_type)); + } else if (deltaCol == -1) { + int16_t cid = -1; + bool hasPrimaryKey = false; + if (onCols >= 2) { + hasPrimaryKey = (pOldEntry->stbEntry.schemaRow.pSchema[1].flags & COL_IS_KEY) ? true : false; + } + for (int i = 0, j = 0; i < nCols && j < onCols; ++i, ++j) { + if (pEntry->stbEntry.schemaRow.pSchema[i].colId != pOldEntry->stbEntry.schemaRow.pSchema[j].colId) { + cid = pOldEntry->stbEntry.schemaRow.pSchema[j].colId; + break; + } + } + + if (cid != -1) { + code = metaGetChildUidsOfSuperTable(pMeta, pEntry->uid, &uids); + if (code) { + metaErr(TD_VID(pMeta->pVnode), code); + metaFetchEntryFree(&pOldEntry); + return code; + } + // TAOS_CHECK_RETURN(metaGetSubtables(pMeta, pEntry->uid, uids)); + TAOS_CHECK_RETURN(tsdbCacheDropSTableColumn(pTsdb, uids, cid, hasPrimaryKey)); + } + } + if (uids) taosArrayDestroy(uids); + + tsdbCacheInvalidateSchema(pTsdb, pEntry->uid, -1, pEntry->stbEntry.schemaRow.version); + } + metaFetchEntryFree(&pOldEntry); return code; } From dc883e95cedfe7e57dfad32add0ad8acc9cd5dd5 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 16 Dec 2024 17:35:04 +0800 Subject: [PATCH 45/45] fix: ci problem --- source/dnode/vnode/src/meta/metaEntry2.c | 3 +-- source/dnode/vnode/src/meta/metaTable2.c | 8 -------- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaEntry2.c b/source/dnode/vnode/src/meta/metaEntry2.c index 0600e42708..fc60a63ced 100644 --- a/source/dnode/vnode/src/meta/metaEntry2.c +++ b/source/dnode/vnode/src/meta/metaEntry2.c @@ -333,8 +333,7 @@ static int32_t metaAddOrDropTagIndexOfSuperTable(SMeta *pMeta, const SMetaHandle return code; } - code = tdbTbInsert(pMeta->pTagIdx, pTagIdxKey, tagIdxKeySize, &pChildEntry->uid, sizeof(pChildEntry->uid), - pMeta->txn); + code = tdbTbInsert(pMeta->pTagIdx, pTagIdxKey, tagIdxKeySize, NULL, 0, pMeta->txn); if (code) { metaErr(TD_VID(pMeta->pVnode), code); taosArrayDestroy(childTables); diff --git a/source/dnode/vnode/src/meta/metaTable2.c b/source/dnode/vnode/src/meta/metaTable2.c index af902809d6..6ff4cd6fdc 100644 --- a/source/dnode/vnode/src/meta/metaTable2.c +++ b/source/dnode/vnode/src/meta/metaTable2.c @@ -1686,14 +1686,6 @@ int32_t metaDropIndexFromSuperTable(SMeta *pMeta, int64_t version, SDropIndexReq TAOS_RETURN(TSDB_CODE_VND_INVALID_TABLE_ACTION); } - if (pReq->stbUid != pEntry->uid) { - metaError("vgId:%d, %s failed at %s:%d since table %s uid %" PRId64 " is not equal to %" PRId64 - ", version:%" PRId64, - TD_VID(pMeta->pVnode), __func__, __FILE__, __LINE__, pReq->stb, pEntry->uid, pReq->stbUid, version); - metaFetchEntryFree(&pEntry); - TAOS_RETURN(TSDB_CODE_INVALID_MSG); - } - SSchemaWrapper *pTagSchema = &pEntry->stbEntry.schemaTag; if (pTagSchema->nCols == 1 && pTagSchema->pSchema[0].type == TSDB_DATA_TYPE_JSON) { metaError("vgId:%d, %s failed at %s:%d since table %s has no tag, version:%" PRId64, TD_VID(pMeta->pVnode),