diff --git a/cmake/taosadapter_CMakeLists.txt.in b/cmake/taosadapter_CMakeLists.txt.in index 13826a1a74..ef6ed4af1d 100644 --- a/cmake/taosadapter_CMakeLists.txt.in +++ b/cmake/taosadapter_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taosadapter ExternalProject_Add(taosadapter GIT_REPOSITORY https://github.com/taosdata/taosadapter.git - GIT_TAG main + GIT_TAG 3.0 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosadapter" BINARY_DIR "" #BUILD_IN_SOURCE TRUE diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index 9bbda8309f..9a6a5329ae 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG main + GIT_TAG 3.0 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE diff --git a/cmake/taosws_CMakeLists.txt.in b/cmake/taosws_CMakeLists.txt.in index b013d45911..17446d184d 100644 --- a/cmake/taosws_CMakeLists.txt.in +++ b/cmake/taosws_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taosws-rs ExternalProject_Add(taosws-rs GIT_REPOSITORY https://github.com/taosdata/taos-connector-rust.git - GIT_TAG main + GIT_TAG 3.0 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosws-rs" BINARY_DIR "" #BUILD_IN_SOURCE TRUE diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 4f1ecf81dd..940116317c 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -160,7 +160,7 @@ int32_t metaAlterSuperTable(SMeta* pMeta, int64_t version, SVCreateStbRe int32_t metaDropSuperTable(SMeta* pMeta, int64_t verison, SVDropStbReq* pReq); int32_t metaCreateTable2(SMeta* pMeta, int64_t version, SVCreateTbReq* pReq, STableMetaRsp** ppRsp); int32_t metaDropTable2(SMeta* pMeta, int64_t version, SVDropTbReq* pReq); -int32_t metaTrimTables(SMeta* pMeta); +int32_t metaTrimTables(SMeta* pMeta, int64_t version); int32_t metaDropMultipleTables(SMeta* pMeta, int64_t version, SArray* tbUids); int metaTtlFindExpired(SMeta* pMeta, int64_t timePointMs, SArray* tbUids, int32_t ttlDropMaxCount); int metaAlterTable(SMeta* pMeta, int64_t version, SVAlterTbReq* pReq, STableMetaRsp* pMetaRsp); diff --git a/source/dnode/vnode/src/meta/metaEntry2.c b/source/dnode/vnode/src/meta/metaEntry2.c index 7aa57edf73..20744577a7 100644 --- a/source/dnode/vnode/src/meta/metaEntry2.c +++ b/source/dnode/vnode/src/meta/metaEntry2.c @@ -1645,7 +1645,6 @@ static int32_t metaHandleSuperTableUpdate(SMeta *pMeta, const SMetaEntry *pEntry metaFetchEntryFree(&pOldEntry); return code; } - // TAOS_CHECK_RETURN(metaGetSubtables(pMeta, pEntry->uid, uids)); TAOS_CHECK_RETURN(tsdbCacheNewSTableColumn(pTsdb, uids, cid, col_type)); } else if (deltaCol == -1) { int16_t cid = -1; @@ -1667,7 +1666,6 @@ static int32_t metaHandleSuperTableUpdate(SMeta *pMeta, const SMetaEntry *pEntry metaFetchEntryFree(&pOldEntry); return code; } - // TAOS_CHECK_RETURN(metaGetSubtables(pMeta, pEntry->uid, uids)); TAOS_CHECK_RETURN(tsdbCacheDropSTableColumn(pTsdb, uids, cid, hasPrimaryKey)); } } diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index ee37dd1f90..2dbc89f58f 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -28,26 +28,14 @@ int32_t metaUpdateTableColCompress2(SMeta *pMeta, int64_t version, SVAlterTbReq int32_t metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSchema *pSchema); -int32_t metaDelJsonVarFromIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSchema *pSchema); -static int metaSaveToTbDb(SMeta *pMeta, const SMetaEntry *pME); -static int metaUpdateUidIdx(SMeta *pMeta, const SMetaEntry *pME); -static int metaUpdateNameIdx(SMeta *pMeta, const SMetaEntry *pME); -static void metaUpdateTtl(SMeta *pMeta, const SMetaEntry *pME); -static int metaUpdateChangeTime(SMeta *pMeta, tb_uid_t uid, int64_t changeTimeMs); -static int metaSaveToSkmDb(SMeta *pMeta, const SMetaEntry *pME); -static int metaUpdateCtbIdx(SMeta *pMeta, const SMetaEntry *pME); -static int metaUpdateSuidIdx(SMeta *pMeta, const SMetaEntry *pME); -static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry); -static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type, tb_uid_t *pSuid, int8_t *pSysTbl); -void metaDestroyTagIdxKey(STagIdxKey *pTagIdxKey); +int32_t metaDelJsonVarFromIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSchema *pSchema); +static int metaUpdateChangeTime(SMeta *pMeta, tb_uid_t uid, int64_t changeTimeMs); +static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type, tb_uid_t *pSuid, int8_t *pSysTbl); +void metaDestroyTagIdxKey(STagIdxKey *pTagIdxKey); // opt ins_tables query -static int metaUpdateBtimeIdx(SMeta *pMeta, const SMetaEntry *pME); static int metaDeleteBtimeIdx(SMeta *pMeta, const SMetaEntry *pME); -static int metaUpdateNcolIdx(SMeta *pMeta, const SMetaEntry *pME); static int metaDeleteNcolIdx(SMeta *pMeta, const SMetaEntry *pME); -int metaHandleEntry(SMeta *pMeta, const SMetaEntry *pME); - int32_t updataTableColCmpr(SColCmprWrapper *pWp, SSchema *pSchema, int8_t add, uint32_t compress) { int32_t nCols = pWp->nCols; int32_t ver = pWp->version; @@ -80,22 +68,6 @@ int32_t updataTableColCmpr(SColCmprWrapper *pWp, SSchema *pSchema, int8_t add, u } return 0; } -static void metaGetEntryInfo(const SMetaEntry *pEntry, SMetaInfo *pInfo) { - pInfo->uid = pEntry->uid; - pInfo->version = pEntry->version; - if (pEntry->type == TSDB_SUPER_TABLE) { - pInfo->suid = pEntry->uid; - pInfo->skmVer = pEntry->stbEntry.schemaRow.version; - } else if (pEntry->type == TSDB_CHILD_TABLE) { - pInfo->suid = pEntry->ctbEntry.suid; - pInfo->skmVer = 0; - } else if (pEntry->type == TSDB_NORMAL_TABLE) { - pInfo->suid = 0; - pInfo->skmVer = pEntry->ntbEntry.schemaRow.version; - } else { - metaError("meta/table: invalide table type: %" PRId8 " get entry info failed.", pEntry->type); - } -} int metaUpdateMetaRsp(tb_uid_t uid, char *tbName, SSchemaWrapper *pSchema, STableMetaRsp *pMetaRsp) { pMetaRsp->pSchemas = taosMemoryMalloc(pSchema->nCols * sizeof(SSchema)); @@ -303,991 +275,6 @@ void metaTimeSeriesNotifyCheck(SMeta *pMeta) { #endif } -static int metaCreateSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { - SMetaEntry me = {0}; - int kLen = 0; - int vLen = 0; - const void *pKey = NULL; - const void *pVal = NULL; - void *pBuf = NULL; - int32_t szBuf = 0; - void *p = NULL; - int32_t code = 0; - - // validate req - void *pData = NULL; - int nData = 0; - if (tdbTbGet(pMeta->pNameIdx, pReq->name, strlen(pReq->name) + 1, &pData, &nData) == 0) { - tb_uid_t uid = *(tb_uid_t *)pData; - tdbFree(pData); - SMetaInfo info; - if (metaGetInfo(pMeta, uid, &info, NULL) == TSDB_CODE_NOT_FOUND) { - return terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST; - } - if (info.uid == info.suid) { - return 0; - } else { - return terrno = TSDB_CODE_TDB_TABLE_ALREADY_EXIST; - } - } - - // set structs - me.version = version; - me.type = TSDB_SUPER_TABLE; - me.uid = pReq->suid; - me.name = pReq->name; - me.stbEntry.schemaRow = pReq->schemaRow; - me.stbEntry.schemaTag = pReq->schemaTag; - if (pReq->rollup) { - TABLE_SET_ROLLUP(me.flags); - me.stbEntry.rsmaParam = pReq->rsmaParam; - } - if (pReq->colCmpred) { - TABLE_SET_COL_COMPRESSED(me.flags); - me.colCmpr = pReq->colCmpr; - } - - code = metaHandleEntry(pMeta, &me); - if (code) goto _err; - - ++pMeta->pVnode->config.vndStats.numOfSTables; - - pMeta->changed = true; - metaDebug("vgId:%d, stb:%s is created, suid:%" PRId64, TD_VID(pMeta->pVnode), pReq->name, pReq->suid); - - return 0; - -_err: - metaError("vgId:%d, failed to create stb:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, pReq->suid, - tstrerror(terrno)); - return code; -} - -static int metaDropSTable(SMeta *pMeta, int64_t verison, SVDropStbReq *pReq, SArray *tbUidList) { - void *pKey = NULL; - int nKey = 0; - void *pData = NULL; - int nData = 0; - int c = 0; - int rc = 0; - int32_t lino; - int32_t ret; - - // check if super table exists - rc = tdbTbGet(pMeta->pNameIdx, pReq->name, strlen(pReq->name) + 1, &pData, &nData); - if (rc < 0 || *(tb_uid_t *)pData != pReq->suid) { - tdbFree(pData); - return terrno = TSDB_CODE_TDB_STB_NOT_EXIST; - } - - // drop all child tables - TBC *pCtbIdxc = NULL; - - rc = tdbTbcOpen(pMeta->pCtbIdx, &pCtbIdxc, NULL); - if (rc) { - return (terrno = rc); - } - - rc = tdbTbcMoveTo(pCtbIdxc, &(SCtbIdxKey){.suid = pReq->suid, .uid = INT64_MIN}, sizeof(SCtbIdxKey), &c); - if (rc < 0) { - tdbTbcClose(pCtbIdxc); - metaWLock(pMeta); - goto _drop_super_table; - } - - for (;;) { - rc = tdbTbcNext(pCtbIdxc, &pKey, &nKey, NULL, NULL); - if (rc < 0) break; - - if (((SCtbIdxKey *)pKey)->suid < pReq->suid) { - continue; - } else if (((SCtbIdxKey *)pKey)->suid > pReq->suid) { - break; - } - - if (taosArrayPush(tbUidList, &(((SCtbIdxKey *)pKey)->uid)) == NULL) { - tdbFree(pKey); - tdbTbcClose(pCtbIdxc); - return terrno; - } - } - - tdbTbcClose(pCtbIdxc); - - ret = tsdbCacheDropSubTables(pMeta->pVnode->pTsdb, tbUidList, pReq->suid); - if (ret < 0) { - metaError("vgId:%d, failed to drop stb:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, pReq->suid, - tstrerror(terrno)); - } - - metaWLock(pMeta); - - for (int32_t iChild = 0; iChild < taosArrayGetSize(tbUidList); iChild++) { - tb_uid_t uid = *(tb_uid_t *)taosArrayGet(tbUidList, iChild); - ret = metaDropTableByUid(pMeta, uid, NULL, NULL, NULL); - if (ret < 0) { - metaError("vgId:%d, failed to drop stb:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, - pReq->suid, tstrerror(terrno)); - } - } - - // drop super table -_drop_super_table: - tdbTbGet(pMeta->pUidIdx, &pReq->suid, sizeof(tb_uid_t), &pData, &nData); - ret = tdbTbDelete(pMeta->pTbDb, &(STbDbKey){.version = ((SUidIdxVal *)pData)[0].version, .uid = pReq->suid}, - sizeof(STbDbKey), pMeta->txn); - if (ret < 0) { - metaError("vgId:%d, failed to drop stb:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, pReq->suid, - tstrerror(terrno)); - } - - ret = tdbTbDelete(pMeta->pNameIdx, pReq->name, strlen(pReq->name) + 1, pMeta->txn); - if (ret < 0) { - metaError("vgId:%d, failed to drop stb:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, pReq->suid, - tstrerror(terrno)); - } - - ret = metaCacheDrop(pMeta, pReq->suid); - if (ret < 0) { - metaError("vgId:%d, failed to drop stb:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, pReq->suid, - tstrerror(terrno)); - } - - ret = tdbTbDelete(pMeta->pUidIdx, &pReq->suid, sizeof(tb_uid_t), pMeta->txn); - if (ret < 0) { - metaError("vgId:%d, failed to drop stb:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, pReq->suid, - tstrerror(terrno)); - } - - ret = tdbTbDelete(pMeta->pSuidIdx, &pReq->suid, sizeof(tb_uid_t), pMeta->txn); - if (ret < 0) { - metaError("vgId:%d, failed to drop stb:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, pReq->suid, - tstrerror(terrno)); - } - - ret = metaStatsCacheDrop(pMeta, pReq->suid); - if (ret < 0) { - metaError("vgId:%d, failed to drop stb:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, pReq->suid, - tstrerror(terrno)); - } - - metaULock(pMeta); - - metaUpdTimeSeriesNum(pMeta); - - pMeta->changed = true; - -_exit: - tdbFree(pKey); - tdbFree(pData); - metaDebug("vgId:%d, super table %s uid:%" PRId64 " is dropped", TD_VID(pMeta->pVnode), pReq->name, pReq->suid); - return 0; -} - -static int32_t metaGetSubtables(SMeta *pMeta, int64_t suid, SArray *uids) { - if (!uids) return TSDB_CODE_INVALID_PARA; - - int c = 0; - void *pKey = NULL; - int nKey = 0; - TBC *pCtbIdxc = NULL; - - TAOS_CHECK_RETURN(tdbTbcOpen(pMeta->pCtbIdx, &pCtbIdxc, NULL)); - int rc = tdbTbcMoveTo(pCtbIdxc, &(SCtbIdxKey){.suid = suid, .uid = INT64_MIN}, sizeof(SCtbIdxKey), &c); - if (rc < 0) { - tdbTbcClose(pCtbIdxc); - metaWLock(pMeta); - return 0; - } - - for (;;) { - rc = tdbTbcNext(pCtbIdxc, &pKey, &nKey, NULL, NULL); - if (rc < 0) break; - - if (((SCtbIdxKey *)pKey)->suid < suid) { - continue; - } else if (((SCtbIdxKey *)pKey)->suid > suid) { - break; - } - - if (taosArrayPush(uids, &(((SCtbIdxKey *)pKey)->uid)) == NULL) { - tdbFree(pKey); - tdbTbcClose(pCtbIdxc); - return terrno; - } - } - - tdbFree(pKey); - - tdbTbcClose(pCtbIdxc); - return 0; -} - -static int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { - SMetaEntry oStbEntry = {0}; - SMetaEntry nStbEntry = {0}; - TBC *pUidIdxc = NULL; - TBC *pTbDbc = NULL; - const void *pData; - int nData; - int64_t oversion; - SDecoder dc = {0}; - int32_t ret; - int32_t c = -2; - - TAOS_CHECK_RETURN(tdbTbcOpen(pMeta->pUidIdx, &pUidIdxc, NULL)); - ret = tdbTbcMoveTo(pUidIdxc, &pReq->suid, sizeof(tb_uid_t), &c); - if (ret < 0 || c) { - tdbTbcClose(pUidIdxc); - - return terrno = TSDB_CODE_TDB_STB_NOT_EXIST; - } - - ret = tdbTbcGet(pUidIdxc, NULL, NULL, &pData, &nData); - if (ret < 0) { - tdbTbcClose(pUidIdxc); - - return terrno = TSDB_CODE_TDB_STB_NOT_EXIST; - } - - oversion = ((SUidIdxVal *)pData)[0].version; - - TAOS_CHECK_RETURN(tdbTbcOpen(pMeta->pTbDb, &pTbDbc, NULL)); - ret = tdbTbcMoveTo(pTbDbc, &((STbDbKey){.uid = pReq->suid, .version = oversion}), sizeof(STbDbKey), &c); - if (!(ret == 0 && c == 0)) { - tdbTbcClose(pUidIdxc); - tdbTbcClose(pTbDbc); - - metaError("meta/table: invalide ret: %" PRId32 " or c: %" PRId32 "alter stb failed.", ret, c); - return terrno = TSDB_CODE_TDB_STB_NOT_EXIST; - } - - ret = tdbTbcGet(pTbDbc, NULL, NULL, &pData, &nData); - if (ret < 0) { - tdbTbcClose(pUidIdxc); - tdbTbcClose(pTbDbc); - - return terrno = TSDB_CODE_TDB_STB_NOT_EXIST; - } - - if ((oStbEntry.pBuf = taosMemoryMalloc(nData)) == NULL) { - tdbTbcClose(pTbDbc); - tdbTbcClose(pUidIdxc); - return terrno; - } - memcpy(oStbEntry.pBuf, pData, nData); - tDecoderInit(&dc, oStbEntry.pBuf, nData); - ret = metaDecodeEntry(&dc, &oStbEntry); - if (ret < 0) { - metaError("vgId:%d, failed to decode stb:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, - pReq->suid, tstrerror(ret)); - tDecoderClear(&dc); - tdbTbcClose(pTbDbc); - tdbTbcClose(pUidIdxc); - return terrno; - } - - nStbEntry.version = version; - nStbEntry.type = TSDB_SUPER_TABLE; - nStbEntry.uid = pReq->suid; - nStbEntry.name = pReq->name; - nStbEntry.stbEntry.schemaRow = pReq->schemaRow; - nStbEntry.stbEntry.schemaTag = pReq->schemaTag; - nStbEntry.colCmpr = pReq->colCmpr; - TABLE_SET_COL_COMPRESSED(nStbEntry.flags); - - int nCols = pReq->schemaRow.nCols; - int onCols = oStbEntry.stbEntry.schemaRow.nCols; - int32_t deltaCol = nCols - onCols; - bool updStat = deltaCol != 0 && !metaTbInFilterCache(pMeta, pReq->name, 1); - - if (!TSDB_CACHE_NO(pMeta->pVnode->config)) { - STsdb *pTsdb = pMeta->pVnode->pTsdb; - SArray *uids = taosArrayInit(8, sizeof(int64_t)); - if (uids == NULL) { - if (oStbEntry.pBuf) taosMemoryFree(oStbEntry.pBuf); - tDecoderClear(&dc); - tdbTbcClose(pTbDbc); - tdbTbcClose(pUidIdxc); - return terrno; - } - if (deltaCol == 1) { - int16_t cid = pReq->schemaRow.pSchema[nCols - 1].colId; - int8_t col_type = pReq->schemaRow.pSchema[nCols - 1].type; - - TAOS_CHECK_RETURN(metaGetSubtables(pMeta, pReq->suid, uids)); - TAOS_CHECK_RETURN(tsdbCacheNewSTableColumn(pTsdb, uids, cid, col_type)); - } else if (deltaCol == -1) { - int16_t cid = -1; - bool hasPrimaryKey = false; - if (onCols >= 2) { - hasPrimaryKey = (oStbEntry.stbEntry.schemaRow.pSchema[1].flags & COL_IS_KEY) ? true : false; - } - for (int i = 0, j = 0; i < nCols && j < onCols; ++i, ++j) { - if (pReq->schemaRow.pSchema[i].colId != oStbEntry.stbEntry.schemaRow.pSchema[j].colId) { - cid = oStbEntry.stbEntry.schemaRow.pSchema[j].colId; - break; - } - } - - if (cid != -1) { - TAOS_CHECK_RETURN(metaGetSubtables(pMeta, pReq->suid, uids)); - TAOS_CHECK_RETURN(tsdbCacheDropSTableColumn(pTsdb, uids, cid, hasPrimaryKey)); - } - } - if (uids) taosArrayDestroy(uids); - - tsdbCacheInvalidateSchema(pTsdb, pReq->suid, -1, pReq->schemaRow.version); - } - - metaWLock(pMeta); - // compare two entry - if (oStbEntry.stbEntry.schemaRow.version != pReq->schemaRow.version) { - ret = metaSaveToSkmDb(pMeta, &nStbEntry); - if (ret < 0) { - metaError("vgId:%d, failed to save skm db:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, - pReq->suid, tstrerror(ret)); - } - } - - // update table.db - ret = metaSaveToTbDb(pMeta, &nStbEntry); - if (ret < 0) { - metaError("vgId:%d, failed to save tb db:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, - pReq->suid, tstrerror(ret)); - } - - // update uid index - ret = metaUpdateUidIdx(pMeta, &nStbEntry); - if (ret < 0) { - metaError("vgId:%d, failed to update uid idx:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, - pReq->suid, tstrerror(ret)); - } - - // metaStatsCacheDrop(pMeta, nStbEntry.uid); - - if (updStat) { - metaUpdateStbStats(pMeta, pReq->suid, 0, deltaCol); - } - metaULock(pMeta); - - if (updStat) { - int64_t ctbNum; - ret = metaGetStbStats(pMeta->pVnode, pReq->suid, &ctbNum, NULL); - if (ret < 0) { - metaError("vgId:%d, failed to get stb stats:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, - pReq->suid, tstrerror(ret)); - } - pMeta->pVnode->config.vndStats.numOfTimeSeries += (ctbNum * deltaCol); - metaTimeSeriesNotifyCheck(pMeta); - } - - pMeta->changed = true; - -_exit: - if (oStbEntry.pBuf) taosMemoryFree(oStbEntry.pBuf); - tDecoderClear(&dc); - tdbTbcClose(pTbDbc); - tdbTbcClose(pUidIdxc); - return 0; -} - -static int metaAddIndexToSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { - SMetaEntry oStbEntry = {0}; - SMetaEntry nStbEntry = {0}; - STbDbKey tbDbKey = {0}; - - TBC *pUidIdxc = NULL; - TBC *pTbDbc = NULL; - void *pData = NULL; - int nData = 0; - int64_t oversion; - SDecoder dc = {0}; - int32_t ret; - int32_t c = -2; - tb_uid_t suid = pReq->suid; - int32_t code = 0; - - // get super table - if ((code = tdbTbGet(pMeta->pUidIdx, &suid, sizeof(tb_uid_t), &pData, &nData)) != 0) { - goto _err; - } - - tbDbKey.uid = suid; - tbDbKey.version = ((SUidIdxVal *)pData)[0].version; - if ((code = tdbTbGet(pMeta->pTbDb, &tbDbKey, sizeof(tbDbKey), &pData, &nData)) != 0) { - goto _err; - } - - tDecoderInit(&dc, pData, nData); - code = metaDecodeEntry(&dc, &oStbEntry); - if (code < 0) { - goto _err; - } - - if (oStbEntry.stbEntry.schemaTag.pSchema == NULL || oStbEntry.stbEntry.schemaTag.pSchema == NULL) { - code = TSDB_CODE_INVALID_PARA; - goto _err; - } - - if (oStbEntry.stbEntry.schemaTag.version == pReq->schemaTag.version) { - code = TSDB_CODE_INVALID_PARA; - goto _err; - } - - if (oStbEntry.stbEntry.schemaTag.nCols != pReq->schemaTag.nCols) { - code = TSDB_CODE_INVALID_PARA; - goto _err; - } - - int diffIdx = -1; - for (int i = 0; i < pReq->schemaTag.nCols; i++) { - SSchema *pNew = pReq->schemaTag.pSchema + i; - SSchema *pOld = oStbEntry.stbEntry.schemaTag.pSchema + i; - if (pNew->type != pOld->type || pNew->colId != pOld->colId || pNew->bytes != pOld->bytes || - strncmp(pOld->name, pNew->name, sizeof(pNew->name))) { - code = TSDB_CODE_INVALID_PARA; - goto _err; - } - if (IS_IDX_ON(pNew) && !IS_IDX_ON(pOld)) { - // if (diffIdx != -1) goto _err; - diffIdx = i; - break; - } - } - - if (diffIdx == -1) { - code = TSDB_CODE_INVALID_PARA; - goto _err; - } - - // Get target schema info - SSchemaWrapper *pTagSchema = &pReq->schemaTag; - if (pTagSchema->nCols == 1 && pTagSchema->pSchema[0].type == TSDB_DATA_TYPE_JSON) { - code = TSDB_CODE_INVALID_PARA; - goto _err; - } - SSchema *pCol = pTagSchema->pSchema + diffIdx; - - /* - * iterator all pTdDbc by uid and version - */ - TBC *pCtbIdxc = NULL; - code = tdbTbcOpen(pMeta->pCtbIdx, &pCtbIdxc, NULL); - if (code != 0) { - goto _err; - } - - code = tdbTbcMoveTo(pCtbIdxc, &(SCtbIdxKey){.suid = suid, .uid = INT64_MIN}, sizeof(SCtbIdxKey), &c); - if (code < 0) { - tdbTbcClose(pCtbIdxc); - goto _err; - } - for (;;) { - void *pKey = NULL, *pVal = NULL; - int nKey = 0, nVal = 0; - code = tdbTbcNext(pCtbIdxc, &pKey, &nKey, &pVal, &nVal); - if (code < 0) { - tdbFree(pKey); - tdbFree(pVal); - tdbTbcClose(pCtbIdxc); - pCtbIdxc = NULL; - break; - } - if (((SCtbIdxKey *)pKey)->suid != suid) { - tdbFree(pKey); - tdbFree(pVal); - continue; - } - STagIdxKey *pTagIdxKey = NULL; - int32_t nTagIdxKey; - - const void *pTagData = NULL; - int32_t nTagData = 0; - - SCtbIdxKey *table = (SCtbIdxKey *)pKey; - STagVal tagVal = {.cid = pCol->colId}; - if (tTagGet((const STag *)pVal, &tagVal)) { - if (IS_VAR_DATA_TYPE(pCol->type)) { - pTagData = tagVal.pData; - nTagData = (int32_t)tagVal.nData; - } else { - pTagData = &(tagVal.i64); - nTagData = tDataTypes[pCol->type].bytes; - } - } else { - if (!IS_VAR_DATA_TYPE(pCol->type)) { - nTagData = tDataTypes[pCol->type].bytes; - } - } - code = metaCreateTagIdxKey(suid, pCol->colId, pTagData, nTagData, pCol->type, table->uid, &pTagIdxKey, &nTagIdxKey); - tdbFree(pKey); - tdbFree(pVal); - if (code < 0) { - metaDestroyTagIdxKey(pTagIdxKey); - tdbTbcClose(pCtbIdxc); - goto _err; - } - - metaWLock(pMeta); - ret = tdbTbUpsert(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, NULL, 0, pMeta->txn); - if (ret < 0) { - metaError("vgId:%d, failed to upsert tag idx key:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, - pReq->suid, tstrerror(ret)); - } - metaULock(pMeta); - - metaDestroyTagIdxKey(pTagIdxKey); - pTagIdxKey = NULL; - } - - nStbEntry.version = version; - nStbEntry.type = TSDB_SUPER_TABLE; - nStbEntry.uid = pReq->suid; - nStbEntry.name = pReq->name; - nStbEntry.stbEntry.schemaRow = pReq->schemaRow; - nStbEntry.stbEntry.schemaTag = pReq->schemaTag; - nStbEntry.colCmpr = pReq->colCmpr; - - metaWLock(pMeta); - // update table.db - ret = metaSaveToTbDb(pMeta, &nStbEntry); - if (ret < 0) { - metaError("vgId:%d, failed to save tb db:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, - pReq->suid, tstrerror(ret)); - } - // update uid index - ret = metaUpdateUidIdx(pMeta, &nStbEntry); - if (ret < 0) { - metaError("vgId:%d, failed to update uid idx:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, - pReq->suid, tstrerror(ret)); - } - metaULock(pMeta); - - if (oStbEntry.pBuf) taosMemoryFree(oStbEntry.pBuf); - tDecoderClear(&dc); - tdbFree(pData); - - tdbTbcClose(pCtbIdxc); - return TSDB_CODE_SUCCESS; -_err: - if (oStbEntry.pBuf) taosMemoryFree(oStbEntry.pBuf); - tDecoderClear(&dc); - tdbFree(pData); - - return code; -} - -static int metaDropIndexFromSTable(SMeta *pMeta, int64_t version, SDropIndexReq *pReq) { - int32_t code = 0; - SMetaEntry oStbEntry = {0}; - SMetaEntry nStbEntry = {0}; - - STbDbKey tbDbKey = {0}; - TBC *pUidIdxc = NULL; - TBC *pTbDbc = NULL; - int ret = 0; - int c = -2; - void *pData = NULL; - int nData = 0; - int64_t oversion; - SDecoder dc = {0}; - - tb_uid_t suid = pReq->stbUid; - - if ((code = tdbTbGet(pMeta->pUidIdx, &suid, sizeof(tb_uid_t), &pData, &nData)) != 0) { - goto _err; - } - - tbDbKey.uid = suid; - tbDbKey.version = ((SUidIdxVal *)pData)[0].version; - if ((code = tdbTbGet(pMeta->pTbDb, &tbDbKey, sizeof(tbDbKey), &pData, &nData)) != 0) { - goto _err; - } - - tDecoderInit(&dc, pData, nData); - code = metaDecodeEntry(&dc, &oStbEntry); - if (code != 0) { - goto _err; - } - - SSchema *pCol = NULL; - int32_t colId = -1; - for (int i = 0; i < oStbEntry.stbEntry.schemaTag.nCols; i++) { - SSchema *schema = oStbEntry.stbEntry.schemaTag.pSchema + i; - if (0 == strncmp(schema->name, pReq->colName, sizeof(pReq->colName))) { - if (IS_IDX_ON(schema)) { - pCol = schema; - } - break; - } - } - - if (pCol == NULL) { - metaError("vgId:%d, failed to drop index on %s.%s,since %s", TD_VID(pMeta->pVnode), pReq->stb, pReq->colName, - tstrerror(TSDB_CODE_VND_COL_NOT_EXISTS)); - code = 0; - - goto _err; - } - - /* - * iterator all pTdDbc by uid and version - */ - TBC *pCtbIdxc = NULL; - code = tdbTbcOpen(pMeta->pCtbIdx, &pCtbIdxc, NULL); - if (code != 0) { - goto _err; - } - - code = tdbTbcMoveTo(pCtbIdxc, &(SCtbIdxKey){.suid = suid, .uid = INT64_MIN}, sizeof(SCtbIdxKey), &c); - if (code < 0) { - tdbTbcClose(pCtbIdxc); - goto _err; - } - for (;;) { - void *pKey = NULL, *pVal = NULL; - int nKey = 0, nVal = 0; - - code = tdbTbcNext(pCtbIdxc, &pKey, &nKey, &pVal, &nVal); - if (code < 0) { - tdbFree(pKey); - tdbFree(pVal); - tdbTbcClose(pCtbIdxc); - pCtbIdxc = NULL; - break; - } - if (((SCtbIdxKey *)pKey)->suid != suid) { - tdbFree(pKey); - tdbFree(pVal); - continue; - } - STagIdxKey *pTagIdxKey = NULL; - int32_t nTagIdxKey; - - const void *pTagData = NULL; - int32_t nTagData = 0; - - SCtbIdxKey *table = (SCtbIdxKey *)pKey; - STagVal tagVal = {.cid = pCol->colId}; - if (tTagGet((const STag *)pVal, &tagVal)) { - if (IS_VAR_DATA_TYPE(pCol->type)) { - pTagData = tagVal.pData; - nTagData = (int32_t)tagVal.nData; - } else { - pTagData = &(tagVal.i64); - nTagData = tDataTypes[pCol->type].bytes; - } - } else { - if (!IS_VAR_DATA_TYPE(pCol->type)) { - nTagData = tDataTypes[pCol->type].bytes; - } - } - - code = metaCreateTagIdxKey(suid, pCol->colId, pTagData, nTagData, pCol->type, table->uid, &pTagIdxKey, &nTagIdxKey); - tdbFree(pKey); - tdbFree(pVal); - if (code < 0) { - metaDestroyTagIdxKey(pTagIdxKey); - tdbTbcClose(pCtbIdxc); - goto _err; - } - - metaWLock(pMeta); - ret = tdbTbDelete(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, pMeta->txn); - if (ret < 0) { - metaError("vgId:%d, failed to delete tag idx key:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->stb, - pReq->stbUid, tstrerror(ret)); - } - metaULock(pMeta); - metaDestroyTagIdxKey(pTagIdxKey); - pTagIdxKey = NULL; - } - - // clear idx flag - SSCHMEA_SET_IDX_OFF(pCol); - - nStbEntry.version = version; - nStbEntry.type = TSDB_SUPER_TABLE; - nStbEntry.uid = oStbEntry.uid; - nStbEntry.name = oStbEntry.name; - - SSchemaWrapper *row = tCloneSSchemaWrapper(&oStbEntry.stbEntry.schemaRow); - SSchemaWrapper *tag = tCloneSSchemaWrapper(&oStbEntry.stbEntry.schemaTag); - SColCmprWrapper *cmpr = tCloneSColCmprWrapper(&oStbEntry.colCmpr); - if (row == NULL || tag == NULL || cmpr == NULL) { - tDeleteSchemaWrapper(row); - tDeleteSchemaWrapper(tag); - tDeleteSColCmprWrapper(cmpr); - code = TSDB_CODE_OUT_OF_MEMORY; - - tdbTbcClose(pCtbIdxc); - goto _err; - } - - nStbEntry.stbEntry.schemaRow = *row; - nStbEntry.stbEntry.schemaTag = *tag; - nStbEntry.stbEntry.rsmaParam = oStbEntry.stbEntry.rsmaParam; - nStbEntry.colCmpr = *cmpr; - - nStbEntry.colCmpr = oStbEntry.colCmpr; - - metaWLock(pMeta); - // update table.db - ret = metaSaveToTbDb(pMeta, &nStbEntry); - if (ret < 0) { - metaError("vgId:%d, failed to save tb db:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->stb, - pReq->stbUid, tstrerror(ret)); - } - // update uid index - ret = metaUpdateUidIdx(pMeta, &nStbEntry); - if (ret < 0) { - metaError("vgId:%d, failed to update uid idx:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->stb, - pReq->stbUid, tstrerror(ret)); - } - metaULock(pMeta); - - tDeleteSchemaWrapper(tag); - tDeleteSchemaWrapper(row); - tDeleteSColCmprWrapper(cmpr); - - if (oStbEntry.pBuf) taosMemoryFree(oStbEntry.pBuf); - tDecoderClear(&dc); - tdbFree(pData); - - tdbTbcClose(pCtbIdxc); - return TSDB_CODE_SUCCESS; -_err: - if (oStbEntry.pBuf) taosMemoryFree(oStbEntry.pBuf); - tDecoderClear(&dc); - tdbFree(pData); - - return code; -} - -static int metaCreateTable(SMeta *pMeta, int64_t ver, SVCreateTbReq *pReq, STableMetaRsp **pMetaRsp) { - SMetaEntry me = {0}; - SMetaReader mr = {0}; - int32_t ret; - - // validate message - if (pReq->type != TSDB_CHILD_TABLE && pReq->type != TSDB_NORMAL_TABLE) { - terrno = TSDB_CODE_INVALID_MSG; - goto _err; - } - - if (pReq->type == TSDB_CHILD_TABLE) { - tb_uid_t suid = metaGetTableEntryUidByName(pMeta, pReq->ctb.stbName); - if (suid != pReq->ctb.suid) { - return terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST; - } - } - - // validate req - metaReaderDoInit(&mr, pMeta, META_READER_LOCK); - if (metaGetTableEntryByName(&mr, pReq->name) == 0) { - if (pReq->type == TSDB_CHILD_TABLE && pReq->ctb.suid != mr.me.ctbEntry.suid) { - metaReaderClear(&mr); - return terrno = TSDB_CODE_TDB_TABLE_IN_OTHER_STABLE; - } - pReq->uid = mr.me.uid; - if (pReq->type == TSDB_CHILD_TABLE) { - pReq->ctb.suid = mr.me.ctbEntry.suid; - } - metaReaderClear(&mr); - return terrno = TSDB_CODE_TDB_TABLE_ALREADY_EXIST; - } else if (terrno == TSDB_CODE_PAR_TABLE_NOT_EXIST) { - terrno = TSDB_CODE_SUCCESS; - } - metaReaderClear(&mr); - - bool sysTbl = (pReq->type == TSDB_CHILD_TABLE) && metaTbInFilterCache(pMeta, pReq->ctb.stbName, 1); - - if (!sysTbl && ((terrno = grantCheck(TSDB_GRANT_TIMESERIES)) < 0)) goto _err; - - // build SMetaEntry - SVnodeStats *pStats = &pMeta->pVnode->config.vndStats; - me.version = ver; - me.type = pReq->type; - me.uid = pReq->uid; - me.name = pReq->name; - if (me.type == TSDB_CHILD_TABLE) { - me.ctbEntry.btime = pReq->btime; - me.ctbEntry.ttlDays = pReq->ttl; - me.ctbEntry.commentLen = pReq->commentLen; - me.ctbEntry.comment = pReq->comment; - me.ctbEntry.suid = pReq->ctb.suid; - me.ctbEntry.pTags = pReq->ctb.pTag; - -#ifdef TAG_FILTER_DEBUG - SArray *pTagVals = NULL; - int32_t code = tTagToValArray((STag *)pReq->ctb.pTag, &pTagVals); - for (int i = 0; i < taosArrayGetSize(pTagVals); i++) { - STagVal *pTagVal = (STagVal *)taosArrayGet(pTagVals, i); - - if (IS_VAR_DATA_TYPE(pTagVal->type)) { - char *buf = taosMemoryCalloc(pTagVal->nData + 1, 1); - memcpy(buf, pTagVal->pData, pTagVal->nData); - metaDebug("metaTag table:%s varchar index:%d cid:%d type:%d value:%s", pReq->name, i, pTagVal->cid, - pTagVal->type, buf); - taosMemoryFree(buf); - } else { - double val = 0; - GET_TYPED_DATA(val, double, pTagVal->type, &pTagVal->i64); - metaDebug("metaTag table:%s number index:%d cid:%d type:%d value:%f", pReq->name, i, pTagVal->cid, - pTagVal->type, val); - } - } -#endif - - ++pStats->numOfCTables; - - if (!sysTbl) { - int32_t nCols = 0; - ret = metaGetStbStats(pMeta->pVnode, me.ctbEntry.suid, 0, &nCols); - if (ret < 0) { - metaError("vgId:%d, failed to get stb stats:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, - pReq->ctb.suid, tstrerror(ret)); - } - pStats->numOfTimeSeries += nCols - 1; - } - - metaWLock(pMeta); - metaUpdateStbStats(pMeta, me.ctbEntry.suid, 1, 0); - ret = metaUidCacheClear(pMeta, me.ctbEntry.suid); - if (ret < 0) { - metaError("vgId:%d, failed to clear uid cache:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, - pReq->ctb.suid, tstrerror(ret)); - } - ret = metaTbGroupCacheClear(pMeta, me.ctbEntry.suid); - if (ret < 0) { - metaError("vgId:%d, failed to clear group cache:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, - pReq->ctb.suid, tstrerror(ret)); - } - metaULock(pMeta); - - if (!TSDB_CACHE_NO(pMeta->pVnode->config)) { - ret = tsdbCacheNewTable(pMeta->pVnode->pTsdb, me.uid, me.ctbEntry.suid, NULL); - if (ret < 0) { - metaError("vgId:%d, failed to create table:%s since %s", TD_VID(pMeta->pVnode), pReq->name, tstrerror(ret)); - goto _err; - } - } - } else { - me.ntbEntry.btime = pReq->btime; - me.ntbEntry.ttlDays = pReq->ttl; - me.ntbEntry.commentLen = pReq->commentLen; - me.ntbEntry.comment = pReq->comment; - me.ntbEntry.schemaRow = pReq->ntb.schemaRow; - me.ntbEntry.ncid = me.ntbEntry.schemaRow.pSchema[me.ntbEntry.schemaRow.nCols - 1].colId + 1; - me.colCmpr = pReq->colCmpr; - TABLE_SET_COL_COMPRESSED(me.flags); - - ++pStats->numOfNTables; - pStats->numOfNTimeSeries += me.ntbEntry.schemaRow.nCols - 1; - - if (!TSDB_CACHE_NO(pMeta->pVnode->config)) { - ret = tsdbCacheNewTable(pMeta->pVnode->pTsdb, me.uid, -1, &me.ntbEntry.schemaRow); - if (ret < 0) { - metaError("vgId:%d, failed to create table:%s since %s", TD_VID(pMeta->pVnode), pReq->name, tstrerror(ret)); - goto _err; - } - } - } - - if (metaHandleEntry(pMeta, &me) < 0) goto _err; - - metaTimeSeriesNotifyCheck(pMeta); - - if (pMetaRsp) { - *pMetaRsp = taosMemoryCalloc(1, sizeof(STableMetaRsp)); - - if (*pMetaRsp) { - if (me.type == TSDB_CHILD_TABLE) { - (*pMetaRsp)->tableType = TSDB_CHILD_TABLE; - (*pMetaRsp)->tuid = pReq->uid; - (*pMetaRsp)->suid = pReq->ctb.suid; - tstrncpy((*pMetaRsp)->tbName, pReq->name, strlen(pReq->name) + 1); - } else { - ret = metaUpdateMetaRsp(pReq->uid, pReq->name, &pReq->ntb.schemaRow, *pMetaRsp); - if (ret < 0) { - metaError("vgId:%d, failed to update meta rsp:%s since %s", TD_VID(pMeta->pVnode), pReq->name, - tstrerror(ret)); - } - for (int32_t i = 0; i < pReq->colCmpr.nCols; i++) { - SColCmpr *p = &pReq->colCmpr.pColCmpr[i]; - (*pMetaRsp)->pSchemaExt[i].colId = p->id; - (*pMetaRsp)->pSchemaExt[i].compress = p->alg; - } - } - } - } - - pMeta->changed = true; - metaDebug("vgId:%d, table:%s uid %" PRId64 " is created, type:%" PRId8, TD_VID(pMeta->pVnode), pReq->name, pReq->uid, - pReq->type); - return 0; - -_err: - metaError("vgId:%d, failed to create table:%s type:%s since %s", TD_VID(pMeta->pVnode), pReq->name, - pReq->type == TSDB_CHILD_TABLE ? "child table" : "normal table", tstrerror(terrno)); - return TSDB_CODE_FAILED; -} - -static int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq, SArray *tbUids, tb_uid_t *tbUid) { - void *pData = NULL; - int nData = 0; - int rc = 0; - tb_uid_t uid = 0; - tb_uid_t suid = 0; - int8_t sysTbl = 0; - int type; - - rc = tdbTbGet(pMeta->pNameIdx, pReq->name, strlen(pReq->name) + 1, &pData, &nData); - if (rc < 0) { - return terrno = TSDB_CODE_TDB_TABLE_NOT_EXIST; - } - uid = *(tb_uid_t *)pData; - - metaWLock(pMeta); - rc = metaDropTableByUid(pMeta, uid, &type, &suid, &sysTbl); - metaULock(pMeta); - - if (rc < 0) goto _exit; - - if (!sysTbl && type == TSDB_CHILD_TABLE) { - int32_t nCols = 0; - SVnodeStats *pStats = &pMeta->pVnode->config.vndStats; - if (metaGetStbStats(pMeta->pVnode, suid, NULL, &nCols) == 0) { - pStats->numOfTimeSeries -= nCols - 1; - } - } - - if ((type == TSDB_CHILD_TABLE || type == TSDB_NORMAL_TABLE) && tbUids) { - if (taosArrayPush(tbUids, &uid) == NULL) { - rc = terrno; - goto _exit; - } - - if (!TSDB_CACHE_NO(pMeta->pVnode->config)) { - int32_t ret = tsdbCacheDropTable(pMeta->pVnode->pTsdb, uid, suid, NULL); - if (ret < 0) { - metaError("vgId:%d, failed to drop table:%s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, uid, - tstrerror(ret)); - } - } - } - - if ((type == TSDB_CHILD_TABLE) && tbUid) { - *tbUid = uid; - } - - pMeta->changed = true; -_exit: - tdbFree(pData); - return rc; -} - static int32_t metaDropTables(SMeta *pMeta, SArray *tbUids) { int32_t code = 0; if (taosArrayGetSize(tbUids) == 0) return TSDB_CODE_SUCCESS; @@ -1399,7 +386,7 @@ static int32_t metaFilterTableByHash(SMeta *pMeta, SArray *uidList) { return 0; } -int32_t metaTrimTables(SMeta *pMeta) { +int32_t metaTrimTables(SMeta *pMeta, int64_t version) { int32_t code = 0; SArray *tbUids = taosArrayInit(8, sizeof(int64_t)); @@ -1679,17 +666,6 @@ static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type, tb_uid_t *p return 0; } -// opt ins_tables -static int metaUpdateBtimeIdx(SMeta *pMeta, const SMetaEntry *pME) { - SBtimeIdxKey btimeKey = {0}; - if (metaBuildBtimeIdxKey(&btimeKey, pME) < 0) { - return 0; - } - metaTrace("vgId:%d, start to save version:%" PRId64 " uid:%" PRId64 " btime:%" PRId64, TD_VID(pMeta->pVnode), - pME->version, pME->uid, btimeKey.btime); - - return tdbTbUpsert(pMeta->pBtimeIdx, &btimeKey, sizeof(btimeKey), NULL, 0, pMeta->txn); -} static int metaDeleteBtimeIdx(SMeta *pMeta, const SMetaEntry *pME) { SBtimeIdxKey btimeKey = {0}; @@ -1699,14 +675,6 @@ static int metaDeleteBtimeIdx(SMeta *pMeta, const SMetaEntry *pME) { return tdbTbDelete(pMeta->pBtimeIdx, &btimeKey, sizeof(btimeKey), pMeta->txn); } -int metaUpdateNcolIdx(SMeta *pMeta, const SMetaEntry *pME) { - SNcolIdxKey ncolKey = {0}; - if (metaBuildNColIdxKey(&ncolKey, pME) < 0) { - return 0; - } - return tdbTbUpsert(pMeta->pNcolIdx, &ncolKey, sizeof(ncolKey), NULL, 0, pMeta->txn); -} - int metaDeleteNcolIdx(SMeta *pMeta, const SMetaEntry *pME) { SNcolIdxKey ncolKey = {0}; if (metaBuildNColIdxKey(&ncolKey, pME) < 0) { @@ -1715,1143 +683,6 @@ int metaDeleteNcolIdx(SMeta *pMeta, const SMetaEntry *pME) { return tdbTbDelete(pMeta->pNcolIdx, &ncolKey, sizeof(ncolKey), pMeta->txn); } -static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTbReq, STableMetaRsp *pMetaRsp) { - void *pVal = NULL; - int nVal = 0; - const void *pData = NULL; - int nData = 0; - int ret = 0; - tb_uid_t uid; - int64_t oversion; - SSchema *pColumn = NULL; - SMetaEntry entry = {0}; - SSchemaWrapper *pSchema; - int c; - bool freeColCmpr = false; - if (pAlterTbReq->colName == NULL) { - metaError("meta/table: null pAlterTbReq->colName"); - return terrno = TSDB_CODE_INVALID_MSG; - } - - // search name index - ret = tdbTbGet(pMeta->pNameIdx, pAlterTbReq->tbName, strlen(pAlterTbReq->tbName) + 1, &pVal, &nVal); - if (ret < 0) { - return terrno = TSDB_CODE_TDB_TABLE_NOT_EXIST; - } - - uid = *(tb_uid_t *)pVal; - tdbFree(pVal); - pVal = NULL; - - // search uid index - TBC *pUidIdxc = NULL; - - TAOS_CHECK_RETURN(tdbTbcOpen(pMeta->pUidIdx, &pUidIdxc, NULL)); - ret = tdbTbcMoveTo(pUidIdxc, &uid, sizeof(uid), &c); - if (c != 0) { - tdbTbcClose(pUidIdxc); - metaError("meta/table: invalide c: %" PRId32 " alt tb column failed.", c); - return TSDB_CODE_FAILED; - } - - ret = tdbTbcGet(pUidIdxc, NULL, NULL, &pData, &nData); - oversion = ((SUidIdxVal *)pData)[0].version; - - // search table.db - TBC *pTbDbc = NULL; - - TAOS_CHECK_RETURN(tdbTbcOpen(pMeta->pTbDb, &pTbDbc, NULL)); - ret = tdbTbcMoveTo(pTbDbc, &((STbDbKey){.uid = uid, .version = oversion}), sizeof(STbDbKey), &c); - if (c != 0) { - tdbTbcClose(pUidIdxc); - tdbTbcClose(pTbDbc); - metaError("meta/table: invalide c: %" PRId32 " alt tb column failed.", c); - return TSDB_CODE_FAILED; - } - - ret = tdbTbcGet(pTbDbc, NULL, NULL, &pData, &nData); - - // get table entry - SDecoder dc = {0}; - if ((entry.pBuf = taosMemoryMalloc(nData)) == NULL) { - tdbTbcClose(pUidIdxc); - tdbTbcClose(pTbDbc); - return terrno; - } - memcpy(entry.pBuf, pData, nData); - tDecoderInit(&dc, entry.pBuf, nData); - ret = metaDecodeEntry(&dc, &entry); - if (ret != 0) { - tdbTbcClose(pUidIdxc); - tdbTbcClose(pTbDbc); - tDecoderClear(&dc); - metaError("meta/table: invalide ret: %" PRId32 " alt tb column failed.", ret); - return ret; - } - - if (entry.type != TSDB_NORMAL_TABLE) { - terrno = TSDB_CODE_VND_INVALID_TABLE_ACTION; - goto _err; - } - // search the column to add/drop/update - pSchema = &entry.ntbEntry.schemaRow; - - // save old entry - SMetaEntry oldEntry = {.type = TSDB_NORMAL_TABLE, .uid = entry.uid}; - oldEntry.ntbEntry.schemaRow.nCols = pSchema->nCols; - - int32_t rowLen = -1; - if (pAlterTbReq->action == TSDB_ALTER_TABLE_ADD_COLUMN || - pAlterTbReq->action == TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES) { - rowLen = 0; - } - - int32_t iCol = 0, jCol = 0; - SSchema *qColumn = NULL; - for (;;) { - qColumn = NULL; - - if (jCol >= pSchema->nCols) break; - qColumn = &pSchema->pSchema[jCol]; - - if (!pColumn && (strcmp(qColumn->name, pAlterTbReq->colName) == 0)) { - pColumn = qColumn; - iCol = jCol; - if (rowLen < 0) break; - } - rowLen += qColumn->bytes; - ++jCol; - } - - entry.version = version; - int tlen; - SSchema *pNewSchema = NULL; - SSchema tScheam; - switch (pAlterTbReq->action) { - case TSDB_ALTER_TABLE_ADD_COLUMN: - case TSDB_ALTER_TABLE_ADD_COLUMN_WITH_COMPRESS_OPTION: - if (pColumn) { - terrno = TSDB_CODE_VND_COL_ALREADY_EXISTS; - goto _err; - } - if ((terrno = grantCheck(TSDB_GRANT_TIMESERIES)) < 0) { - goto _err; - } - if (rowLen + pAlterTbReq->bytes > TSDB_MAX_BYTES_PER_ROW) { - terrno = TSDB_CODE_PAR_INVALID_ROW_LENGTH; - goto _err; - } - pSchema->version++; - pSchema->nCols++; - pNewSchema = taosMemoryMalloc(sizeof(SSchema) * pSchema->nCols); - if (pNewSchema == NULL) { - goto _err; - } - memcpy(pNewSchema, pSchema->pSchema, sizeof(SSchema) * (pSchema->nCols - 1)); - pSchema->pSchema = pNewSchema; - pSchema->pSchema[entry.ntbEntry.schemaRow.nCols - 1].bytes = pAlterTbReq->bytes; - pSchema->pSchema[entry.ntbEntry.schemaRow.nCols - 1].type = pAlterTbReq->type; - pSchema->pSchema[entry.ntbEntry.schemaRow.nCols - 1].flags = pAlterTbReq->flags; - pSchema->pSchema[entry.ntbEntry.schemaRow.nCols - 1].colId = entry.ntbEntry.ncid++; - tstrncpy(pSchema->pSchema[entry.ntbEntry.schemaRow.nCols - 1].name, pAlterTbReq->colName, - strlen(pAlterTbReq->colName) + 1); - - ++pMeta->pVnode->config.vndStats.numOfNTimeSeries; - metaTimeSeriesNotifyCheck(pMeta); - - if (!TSDB_CACHE_NO(pMeta->pVnode->config)) { - int16_t cid = pSchema->pSchema[entry.ntbEntry.schemaRow.nCols - 1].colId; - int8_t col_type = pSchema->pSchema[entry.ntbEntry.schemaRow.nCols - 1].type; - int32_t ret = tsdbCacheNewNTableColumn(pMeta->pVnode->pTsdb, entry.uid, cid, col_type); - if (ret < 0) { - terrno = ret; - goto _err; - } - } - SSchema *pCol = &pSchema->pSchema[entry.ntbEntry.schemaRow.nCols - 1]; - uint32_t compress = pAlterTbReq->action == TSDB_ALTER_TABLE_ADD_COLUMN ? createDefaultColCmprByType(pCol->type) - : pAlterTbReq->compress; - if (updataTableColCmpr(&entry.colCmpr, pCol, 1, compress) != 0) { - metaError("vgId:%d, failed to update table col cmpr:%s uid:%" PRId64, TD_VID(pMeta->pVnode), entry.name, - entry.uid); - } - freeColCmpr = true; - if (entry.colCmpr.nCols != pSchema->nCols) { - if (pNewSchema) taosMemoryFree(pNewSchema); - if (freeColCmpr) taosMemoryFree(entry.colCmpr.pColCmpr); - terrno = TSDB_CODE_VND_INVALID_TABLE_ACTION; - goto _err; - } - break; - case TSDB_ALTER_TABLE_DROP_COLUMN: - if (pColumn == NULL) { - terrno = TSDB_CODE_VND_COL_NOT_EXISTS; - goto _err; - } - if (pColumn->colId == 0) { - terrno = TSDB_CODE_VND_INVALID_TABLE_ACTION; - goto _err; - } - if (tqCheckColModifiable(pMeta->pVnode->pTq, uid, pColumn->colId) != 0) { - terrno = TSDB_CODE_VND_COL_SUBSCRIBED; - goto _err; - } - bool hasPrimayKey = false; - if (pSchema->nCols >= 2) { - hasPrimayKey = pSchema->pSchema[1].flags & COL_IS_KEY ? true : false; - } - - memcpy(&tScheam, pColumn, sizeof(SSchema)); - pSchema->version++; - tlen = (pSchema->nCols - iCol - 1) * sizeof(SSchema); - if (tlen) { - memmove(pColumn, pColumn + 1, tlen); - } - pSchema->nCols--; - - --pMeta->pVnode->config.vndStats.numOfNTimeSeries; - - if (!TSDB_CACHE_NO(pMeta->pVnode->config)) { - int16_t cid = pColumn->colId; - - if (tsdbCacheDropNTableColumn(pMeta->pVnode->pTsdb, entry.uid, cid, hasPrimayKey) != 0) { - metaError("vgId:%d, failed to drop ntable column:%s uid:%" PRId64, TD_VID(pMeta->pVnode), entry.name, - entry.uid); - } - } - - if (updataTableColCmpr(&entry.colCmpr, &tScheam, 0, 0) != 0) { - metaError("vgId:%d, failed to update table col cmpr:%s uid:%" PRId64, TD_VID(pMeta->pVnode), entry.name, - entry.uid); - } - if (entry.colCmpr.nCols != pSchema->nCols) { - terrno = TSDB_CODE_VND_INVALID_TABLE_ACTION; - goto _err; - } - break; - case TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES: - if (pColumn == NULL) { - terrno = TSDB_CODE_VND_COL_NOT_EXISTS; - goto _err; - } - if (!IS_VAR_DATA_TYPE(pColumn->type) || pColumn->bytes >= pAlterTbReq->colModBytes) { - terrno = TSDB_CODE_VND_INVALID_TABLE_ACTION; - goto _err; - } - if (rowLen + pAlterTbReq->colModBytes - pColumn->bytes > TSDB_MAX_BYTES_PER_ROW) { - terrno = TSDB_CODE_PAR_INVALID_ROW_LENGTH; - goto _err; - } - if (tqCheckColModifiable(pMeta->pVnode->pTq, uid, pColumn->colId) != 0) { - terrno = TSDB_CODE_VND_COL_SUBSCRIBED; - goto _err; - } - pSchema->version++; - pColumn->bytes = pAlterTbReq->colModBytes; - break; - case TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME: - if (pAlterTbReq->colNewName == NULL) { - terrno = TSDB_CODE_INVALID_MSG; - goto _err; - } - if (pColumn == NULL) { - terrno = TSDB_CODE_VND_COL_NOT_EXISTS; - goto _err; - } - if (tqCheckColModifiable(pMeta->pVnode->pTq, uid, pColumn->colId) != 0) { - terrno = TSDB_CODE_VND_COL_SUBSCRIBED; - goto _err; - } - pSchema->version++; - tstrncpy(pColumn->name, pAlterTbReq->colNewName, strlen(pAlterTbReq->colNewName) + 1); - break; - } - - if (!TSDB_CACHE_NO(pMeta->pVnode->config)) { - tsdbCacheInvalidateSchema(pMeta->pVnode->pTsdb, 0, entry.uid, pSchema->version); - } - - entry.version = version; - - // do actual write - metaWLock(pMeta); - - if (metaDeleteNcolIdx(pMeta, &oldEntry) < 0) { - metaError("vgId:%d, failed to delete ncol idx:%s uid:%" PRId64, TD_VID(pMeta->pVnode), entry.name, entry.uid); - } - - if (metaUpdateNcolIdx(pMeta, &entry) < 0) { - metaError("vgId:%d, failed to update ncol idx:%s uid:%" PRId64, TD_VID(pMeta->pVnode), entry.name, entry.uid); - } - - // save to table db - if (metaSaveToTbDb(pMeta, &entry) < 0) { - metaError("vgId:%d, failed to save to tb db:%s uid:%" PRId64, TD_VID(pMeta->pVnode), entry.name, entry.uid); - } - - if (metaUpdateUidIdx(pMeta, &entry) < 0) { - metaError("vgId:%d, failed to update uid idx:%s uid:%" PRId64, TD_VID(pMeta->pVnode), entry.name, entry.uid); - } - - if (metaSaveToSkmDb(pMeta, &entry) < 0) { - metaError("vgId:%d, failed to save to skm db:%s uid:%" PRId64, TD_VID(pMeta->pVnode), entry.name, entry.uid); - } - - if (metaUpdateChangeTime(pMeta, entry.uid, pAlterTbReq->ctimeMs) < 0) { - metaError("vgId:%d, failed to update change time:%s uid:%" PRId64, TD_VID(pMeta->pVnode), entry.name, entry.uid); - } - - metaULock(pMeta); - - if (metaUpdateMetaRsp(uid, pAlterTbReq->tbName, pSchema, pMetaRsp) < 0) { - metaError("vgId:%d, failed to update meta rsp:%s uid:%" PRId64, TD_VID(pMeta->pVnode), entry.name, entry.uid); - } - for (int32_t i = 0; i < entry.colCmpr.nCols; i++) { - SColCmpr *p = &entry.colCmpr.pColCmpr[i]; - pMetaRsp->pSchemaExt[i].colId = p->id; - pMetaRsp->pSchemaExt[i].compress = p->alg; - } - - if (entry.pBuf) taosMemoryFree(entry.pBuf); - if (pNewSchema) taosMemoryFree(pNewSchema); - if (freeColCmpr) taosMemoryFree(entry.colCmpr.pColCmpr); - - tdbTbcClose(pTbDbc); - tdbTbcClose(pUidIdxc); - tDecoderClear(&dc); - - return 0; - -_err: - if (entry.pBuf) taosMemoryFree(entry.pBuf); - tdbTbcClose(pTbDbc); - tdbTbcClose(pUidIdxc); - tDecoderClear(&dc); - - return terrno != 0 ? terrno : TSDB_CODE_FAILED; -} - -static int metaUpdateTableMultiTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTbReq) { - SMetaEntry ctbEntry = {0}; - SMetaEntry stbEntry = {0}; - void *pVal = NULL; - int nVal = 0; - int ret; - int c; - tb_uid_t uid; - int64_t oversion; - const void *pData = NULL; - int nData = 0; - SHashObj *pTagTable = NULL; - - // search name index - ret = tdbTbGet(pMeta->pNameIdx, pAlterTbReq->tbName, strlen(pAlterTbReq->tbName) + 1, &pVal, &nVal); - if (ret < 0) { - return terrno = TSDB_CODE_TDB_TABLE_NOT_EXIST; - } - - uid = *(tb_uid_t *)pVal; - tdbFree(pVal); - pVal = NULL; - - // search uid index - TBC *pUidIdxc = NULL; - - TAOS_CHECK_RETURN(tdbTbcOpen(pMeta->pUidIdx, &pUidIdxc, NULL)); - if (tdbTbcMoveTo(pUidIdxc, &uid, sizeof(uid), &c) < 0) { - metaTrace("meta/table: failed to move to uid index, uid:%" PRId64, uid); - } - if (c != 0) { - tdbTbcClose(pUidIdxc); - metaError("meta/table: invalide c: %" PRId32 " update tb tag val failed.", c); - return terrno = TSDB_CODE_TDB_TABLE_NOT_EXIST; - } - - if (tdbTbcGet(pUidIdxc, NULL, NULL, &pData, &nData) != 0) { - tdbTbcClose(pUidIdxc); - metaError("meta/table: failed to get uid index, uid:%" PRId64, uid); - return terrno = TSDB_CODE_TDB_TABLE_NOT_EXIST; - } - oversion = ((SUidIdxVal *)pData)[0].version; - - // search table.db - TBC *pTbDbc = NULL; - SDecoder dc1 = {0}; - SDecoder dc2 = {0}; - - /* get ctbEntry */ - TAOS_CHECK_RETURN(tdbTbcOpen(pMeta->pTbDb, &pTbDbc, NULL)); - if (tdbTbcMoveTo(pTbDbc, &((STbDbKey){.uid = uid, .version = oversion}), sizeof(STbDbKey), &c) != 0) { - metaError("meta/table: failed to move to tb db, uid:%" PRId64, uid); - } - if (c != 0) { - tdbTbcClose(pUidIdxc); - tdbTbcClose(pTbDbc); - metaError("meta/table: invalide c: %" PRId32 " update tb tag val failed.", c); - return terrno = TSDB_CODE_TDB_TABLE_NOT_EXIST; - } - - if (tdbTbcGet(pTbDbc, NULL, NULL, &pData, &nData) != 0) { - metaError("meta/table: failed to get tb db, uid:%" PRId64, uid); - tdbTbcClose(pUidIdxc); - tdbTbcClose(pTbDbc); - return terrno = TSDB_CODE_TDB_TABLE_NOT_EXIST; - } - - if ((ctbEntry.pBuf = taosMemoryMalloc(nData)) == NULL) { - tdbTbcClose(pUidIdxc); - tdbTbcClose(pTbDbc); - return terrno; - } - memcpy(ctbEntry.pBuf, pData, nData); - tDecoderInit(&dc1, ctbEntry.pBuf, nData); - ret = metaDecodeEntry(&dc1, &ctbEntry); - if (ret < 0) { - terrno = ret; - goto _err; - } - - /* get stbEntry*/ - if (tdbTbGet(pMeta->pUidIdx, &ctbEntry.ctbEntry.suid, sizeof(tb_uid_t), &pVal, &nVal) != 0) { - metaError("meta/table: failed to get uid index, uid:%" PRId64, ctbEntry.ctbEntry.suid); - } - if (!pVal) { - terrno = TSDB_CODE_INVALID_MSG; - goto _err; - } - - if (tdbTbGet(pMeta->pTbDb, &((STbDbKey){.uid = ctbEntry.ctbEntry.suid, .version = ((SUidIdxVal *)pVal)[0].version}), - sizeof(STbDbKey), (void **)&stbEntry.pBuf, &nVal) != 0) { - metaError("meta/table: failed to get tb db, uid:%" PRId64, ctbEntry.ctbEntry.suid); - } - tdbFree(pVal); - tDecoderInit(&dc2, stbEntry.pBuf, nVal); - ret = metaDecodeEntry(&dc2, &stbEntry); - if (ret < 0) { - terrno = ret; - goto _err; - } - - int32_t nTagVals = taosArrayGetSize(pAlterTbReq->pMultiTag); - pTagTable = taosHashInit(nTagVals, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); - if (pTagTable == NULL) { - ret = terrno; - goto _err; - } - - // remove duplicate tag name - for (int i = 0; i < nTagVals; i++) { - SMultiTagUpateVal *pTagVal = taosArrayGet(pAlterTbReq->pMultiTag, i); - ret = taosHashPut(pTagTable, pTagVal->tagName, strlen(pTagVal->tagName), pTagVal, sizeof(*pTagVal)); - if (ret != 0) { - goto _err; - } - } - - SSchemaWrapper *pTagSchema = &stbEntry.stbEntry.schemaTag; - SSchema *pColumn = NULL; - int32_t iCol = 0; - int32_t count = 0; - - for (;;) { - pColumn = NULL; - - if (iCol >= pTagSchema->nCols) break; - pColumn = &pTagSchema->pSchema[iCol]; - if (taosHashGet(pTagTable, pColumn->name, strlen(pColumn->name)) != NULL) { - count++; - } - iCol++; - } - if (count != taosHashGetSize(pTagTable)) { - terrno = TSDB_CODE_VND_COL_NOT_EXISTS; - goto _err; - } - - ctbEntry.version = version; - if (pTagSchema->nCols == 1 && pTagSchema->pSchema[0].type == TSDB_DATA_TYPE_JSON) { - terrno = TSDB_CODE_VND_COL_NOT_EXISTS; - goto _err; - } else { - const STag *pOldTag = (const STag *)ctbEntry.ctbEntry.pTags; - STag *pNewTag = NULL; - SArray *pTagArray = taosArrayInit(pTagSchema->nCols, sizeof(STagVal)); - if (!pTagArray) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - for (int32_t i = 0; i < pTagSchema->nCols; i++) { - SSchema *pCol = &pTagSchema->pSchema[i]; - SMultiTagUpateVal *pTagVal = taosHashGet(pTagTable, pCol->name, strlen(pCol->name)); - if (pTagVal == NULL) { - STagVal val = {.cid = pCol->colId}; - if (tTagGet(pOldTag, &val)) { - if (taosArrayPush(pTagArray, &val) == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - taosArrayDestroy(pTagArray); - goto _err; - } - } - } else { - STagVal val = {0}; - val.type = pCol->type; - val.cid = pCol->colId; - if (pTagVal->isNull) continue; - - if (IS_VAR_DATA_TYPE(pCol->type)) { - val.pData = pTagVal->pTagVal; - val.nData = pTagVal->nTagVal; - } else { - memcpy(&val.i64, pTagVal->pTagVal, pTagVal->nTagVal); - } - if (taosArrayPush(pTagArray, &val) == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - taosArrayDestroy(pTagArray); - goto _err; - } - } - } - if ((terrno = tTagNew(pTagArray, pTagSchema->version, false, &pNewTag)) < 0) { - taosArrayDestroy(pTagArray); - goto _err; - } - ctbEntry.ctbEntry.pTags = (uint8_t *)pNewTag; - taosArrayDestroy(pTagArray); - } - - metaWLock(pMeta); - - // save to table.db - if (metaSaveToTbDb(pMeta, &ctbEntry) < 0) { - metaError("meta/table: failed to save to tb db:%s uid:%" PRId64, ctbEntry.name, ctbEntry.uid); - } - - // save to uid.idx - if (metaUpdateUidIdx(pMeta, &ctbEntry) < 0) { - metaError("meta/table: failed to update uid idx:%s uid:%" PRId64, ctbEntry.name, ctbEntry.uid); - } - - if (metaUpdateTagIdx(pMeta, &ctbEntry) < 0) { - metaError("meta/table: failed to update tag idx:%s uid:%" PRId64, ctbEntry.name, ctbEntry.uid); - } - - SCtbIdxKey ctbIdxKey = {.suid = ctbEntry.ctbEntry.suid, .uid = uid}; - if (tdbTbUpsert(pMeta->pCtbIdx, &ctbIdxKey, sizeof(ctbIdxKey), ctbEntry.ctbEntry.pTags, - ((STag *)(ctbEntry.ctbEntry.pTags))->len, pMeta->txn) < 0) { - metaError("meta/table: failed to upsert ctb idx:%s uid:%" PRId64, ctbEntry.name, ctbEntry.uid); - } - - if (metaUidCacheClear(pMeta, ctbEntry.ctbEntry.suid) < 0) { - metaError("meta/table: failed to clear uid cache:%s uid:%" PRId64, ctbEntry.name, ctbEntry.uid); - } - - if (metaTbGroupCacheClear(pMeta, ctbEntry.ctbEntry.suid) < 0) { - metaError("meta/table: failed to clear group cache:%s uid:%" PRId64, ctbEntry.name, ctbEntry.uid); - } - - if (metaUpdateChangeTime(pMeta, ctbEntry.uid, pAlterTbReq->ctimeMs) < 0) { - metaError("meta/table: failed to update change time:%s uid:%" PRId64, ctbEntry.name, ctbEntry.uid); - } - - metaULock(pMeta); - - tDecoderClear(&dc1); - tDecoderClear(&dc2); - taosMemoryFree((void *)ctbEntry.ctbEntry.pTags); - if (ctbEntry.pBuf) taosMemoryFree(ctbEntry.pBuf); - if (stbEntry.pBuf) tdbFree(stbEntry.pBuf); - tdbTbcClose(pTbDbc); - tdbTbcClose(pUidIdxc); - taosHashCleanup(pTagTable); - return 0; - -_err: - tDecoderClear(&dc1); - tDecoderClear(&dc2); - if (ctbEntry.pBuf) taosMemoryFree(ctbEntry.pBuf); - if (stbEntry.pBuf) tdbFree(stbEntry.pBuf); - tdbTbcClose(pTbDbc); - tdbTbcClose(pUidIdxc); - taosHashCleanup(pTagTable); - return -1; -} -static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTbReq) { - SMetaEntry ctbEntry = {0}; - SMetaEntry stbEntry = {0}; - void *pVal = NULL; - int nVal = 0; - int ret; - int c; - tb_uid_t uid; - int64_t oversion; - const void *pData = NULL; - int nData = 0; - - if (pAlterTbReq->tagName == NULL) { - return terrno = TSDB_CODE_INVALID_MSG; - } - - // search name index - ret = tdbTbGet(pMeta->pNameIdx, pAlterTbReq->tbName, strlen(pAlterTbReq->tbName) + 1, &pVal, &nVal); - if (ret < 0) { - return terrno = TSDB_CODE_TDB_TABLE_NOT_EXIST; - } - - uid = *(tb_uid_t *)pVal; - tdbFree(pVal); - pVal = NULL; - - // search uid index - TBC *pUidIdxc = NULL; - - TAOS_CHECK_RETURN(tdbTbcOpen(pMeta->pUidIdx, &pUidIdxc, NULL)); - if (tdbTbcMoveTo(pUidIdxc, &uid, sizeof(uid), &c) < 0) { - metaTrace("meta/table: failed to move to uid index, uid:%" PRId64, uid); - } - if (c != 0) { - tdbTbcClose(pUidIdxc); - metaError("meta/table: invalide c: %" PRId32 " update tb tag val failed.", c); - return terrno = TSDB_CODE_TDB_TABLE_NOT_EXIST; - } - - if (tdbTbcGet(pUidIdxc, NULL, NULL, &pData, &nData) != 0) { - tdbTbcClose(pUidIdxc); - metaError("meta/table: failed to get uid index, uid:%" PRId64, uid); - return terrno = TSDB_CODE_TDB_TABLE_NOT_EXIST; - } - oversion = ((SUidIdxVal *)pData)[0].version; - - // search table.db - TBC *pTbDbc = NULL; - SDecoder dc1 = {0}; - SDecoder dc2 = {0}; - - /* get ctbEntry */ - TAOS_CHECK_RETURN(tdbTbcOpen(pMeta->pTbDb, &pTbDbc, NULL)); - if (tdbTbcMoveTo(pTbDbc, &((STbDbKey){.uid = uid, .version = oversion}), sizeof(STbDbKey), &c) != 0) { - metaError("meta/table: failed to move to tb db, uid:%" PRId64, uid); - } - if (c != 0) { - tdbTbcClose(pUidIdxc); - tdbTbcClose(pTbDbc); - metaError("meta/table: invalide c: %" PRId32 " update tb tag val failed.", c); - return terrno = TSDB_CODE_TDB_TABLE_NOT_EXIST; - } - - if (tdbTbcGet(pTbDbc, NULL, NULL, &pData, &nData) != 0) { - metaError("meta/table: failed to get tb db, uid:%" PRId64, uid); - tdbTbcClose(pUidIdxc); - tdbTbcClose(pTbDbc); - return terrno = TSDB_CODE_INVALID_MSG; - } - - if ((ctbEntry.pBuf = taosMemoryMalloc(nData)) == NULL) { - tdbTbcClose(pUidIdxc); - tdbTbcClose(pTbDbc); - return terrno; - } - memcpy(ctbEntry.pBuf, pData, nData); - tDecoderInit(&dc1, ctbEntry.pBuf, nData); - ret = metaDecodeEntry(&dc1, &ctbEntry); - if (ret < 0) { - terrno = ret; - goto _err; - } - - /* get stbEntry*/ - if (tdbTbGet(pMeta->pUidIdx, &ctbEntry.ctbEntry.suid, sizeof(tb_uid_t), &pVal, &nVal) != 0) { - metaError("meta/table: failed to get uid index, uid:%" PRId64, ctbEntry.ctbEntry.suid); - } - if (!pVal) { - terrno = TSDB_CODE_INVALID_MSG; - goto _err; - } - - if (tdbTbGet(pMeta->pTbDb, &((STbDbKey){.uid = ctbEntry.ctbEntry.suid, .version = ((SUidIdxVal *)pVal)[0].version}), - sizeof(STbDbKey), (void **)&stbEntry.pBuf, &nVal) != 0) { - metaError("meta/table: failed to get tb db, uid:%" PRId64, ctbEntry.ctbEntry.suid); - } - tdbFree(pVal); - tDecoderInit(&dc2, stbEntry.pBuf, nVal); - ret = metaDecodeEntry(&dc2, &stbEntry); - if (ret < 0) { - terrno = ret; - goto _err; - } - - SSchemaWrapper *pTagSchema = &stbEntry.stbEntry.schemaTag; - SSchema *pColumn = NULL; - int32_t iCol = 0; - - for (;;) { - pColumn = NULL; - - if (iCol >= pTagSchema->nCols) break; - pColumn = &pTagSchema->pSchema[iCol]; - - if (strcmp(pColumn->name, pAlterTbReq->tagName) == 0) break; - iCol++; - } - - if (pColumn == NULL) { - terrno = TSDB_CODE_VND_COL_NOT_EXISTS; - goto _err; - } - - ctbEntry.version = version; - if (pTagSchema->nCols == 1 && pTagSchema->pSchema[0].type == TSDB_DATA_TYPE_JSON) { - ctbEntry.ctbEntry.pTags = taosMemoryMalloc(pAlterTbReq->nTagVal); - if (ctbEntry.ctbEntry.pTags == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - memcpy((void *)ctbEntry.ctbEntry.pTags, pAlterTbReq->pTagVal, pAlterTbReq->nTagVal); - } else { - const STag *pOldTag = (const STag *)ctbEntry.ctbEntry.pTags; - STag *pNewTag = NULL; - SArray *pTagArray = taosArrayInit(pTagSchema->nCols, sizeof(STagVal)); - if (!pTagArray) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - for (int32_t i = 0; i < pTagSchema->nCols; i++) { - SSchema *pCol = &pTagSchema->pSchema[i]; - if (iCol == i) { - if (pAlterTbReq->isNull) { - continue; - } - STagVal val = {0}; - val.type = pCol->type; - val.cid = pCol->colId; - if (IS_VAR_DATA_TYPE(pCol->type)) { - val.pData = pAlterTbReq->pTagVal; - val.nData = pAlterTbReq->nTagVal; - } else { - memcpy(&val.i64, pAlterTbReq->pTagVal, pAlterTbReq->nTagVal); - } - if (taosArrayPush(pTagArray, &val) == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - taosArrayDestroy(pTagArray); - goto _err; - } - } else { - STagVal val = {.cid = pCol->colId}; - if (tTagGet(pOldTag, &val)) { - if (taosArrayPush(pTagArray, &val) == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - taosArrayDestroy(pTagArray); - goto _err; - } - } - } - } - if ((terrno = tTagNew(pTagArray, pTagSchema->version, false, &pNewTag)) < 0) { - taosArrayDestroy(pTagArray); - goto _err; - } - ctbEntry.ctbEntry.pTags = (uint8_t *)pNewTag; - taosArrayDestroy(pTagArray); - } - - metaWLock(pMeta); - - // save to table.db - if (metaSaveToTbDb(pMeta, &ctbEntry) < 0) { - metaError("meta/table: failed to save to tb db:%s uid:%" PRId64, ctbEntry.name, ctbEntry.uid); - } - - // save to uid.idx - if (metaUpdateUidIdx(pMeta, &ctbEntry) < 0) { - metaError("meta/table: failed to update uid idx:%s uid:%" PRId64, ctbEntry.name, ctbEntry.uid); - } - - if (metaUpdateTagIdx(pMeta, &ctbEntry) < 0) { - metaError("meta/table: failed to update tag idx:%s uid:%" PRId64, ctbEntry.name, ctbEntry.uid); - } - - SCtbIdxKey ctbIdxKey = {.suid = ctbEntry.ctbEntry.suid, .uid = uid}; - if (tdbTbUpsert(pMeta->pCtbIdx, &ctbIdxKey, sizeof(ctbIdxKey), ctbEntry.ctbEntry.pTags, - ((STag *)(ctbEntry.ctbEntry.pTags))->len, pMeta->txn) < 0) { - metaError("meta/table: failed to upsert ctb idx:%s uid:%" PRId64, ctbEntry.name, ctbEntry.uid); - } - - if (metaUidCacheClear(pMeta, ctbEntry.ctbEntry.suid) < 0) { - metaError("meta/table: failed to clear uid cache:%s uid:%" PRId64, ctbEntry.name, ctbEntry.uid); - } - - if (metaTbGroupCacheClear(pMeta, ctbEntry.ctbEntry.suid) < 0) { - metaError("meta/table: failed to clear group cache:%s uid:%" PRId64, ctbEntry.name, ctbEntry.uid); - } - - if (metaUpdateChangeTime(pMeta, ctbEntry.uid, pAlterTbReq->ctimeMs) < 0) { - metaError("meta/table: failed to update change time:%s uid:%" PRId64, ctbEntry.name, ctbEntry.uid); - } - - metaULock(pMeta); - - tDecoderClear(&dc1); - tDecoderClear(&dc2); - taosMemoryFree((void *)ctbEntry.ctbEntry.pTags); - if (ctbEntry.pBuf) taosMemoryFree(ctbEntry.pBuf); - if (stbEntry.pBuf) tdbFree(stbEntry.pBuf); - tdbTbcClose(pTbDbc); - tdbTbcClose(pUidIdxc); - return 0; - -_err: - tDecoderClear(&dc1); - tDecoderClear(&dc2); - if (ctbEntry.pBuf) taosMemoryFree(ctbEntry.pBuf); - if (stbEntry.pBuf) tdbFree(stbEntry.pBuf); - tdbTbcClose(pTbDbc); - tdbTbcClose(pUidIdxc); - return -1; -} - -static int metaUpdateTableOptions(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTbReq) { - void *pVal = NULL; - int nVal = 0; - const void *pData = NULL; - int nData = 0; - int ret = 0; - tb_uid_t uid; - int64_t oversion; - SMetaEntry entry = {0}; - int c = 0; - - // search name index - ret = tdbTbGet(pMeta->pNameIdx, pAlterTbReq->tbName, strlen(pAlterTbReq->tbName) + 1, &pVal, &nVal); - if (ret < 0) { - return terrno = TSDB_CODE_TDB_TABLE_NOT_EXIST; - } - - uid = *(tb_uid_t *)pVal; - tdbFree(pVal); - pVal = NULL; - - // search uid index - TBC *pUidIdxc = NULL; - - TAOS_CHECK_RETURN(tdbTbcOpen(pMeta->pUidIdx, &pUidIdxc, NULL)); - if (tdbTbcMoveTo(pUidIdxc, &uid, sizeof(uid), &c) < 0) { - metaError("meta/table: failed to move to uid index, uid:%" PRId64, uid); - } - if (c != 0) { - tdbTbcClose(pUidIdxc); - metaError("meta/table: invalide c: %" PRId32 " update tb options failed.", c); - return TSDB_CODE_FAILED; - } - - if (tdbTbcGet(pUidIdxc, NULL, NULL, &pData, &nData) < 0) { - metaError("meta/table: failed to get uid index, uid:%" PRId64, uid); - } - oversion = ((SUidIdxVal *)pData)[0].version; - - // search table.db - TBC *pTbDbc = NULL; - - TAOS_CHECK_RETURN(tdbTbcOpen(pMeta->pTbDb, &pTbDbc, NULL)); - if (tdbTbcMoveTo(pTbDbc, &((STbDbKey){.uid = uid, .version = oversion}), sizeof(STbDbKey), &c) < 0) { - metaError("meta/table: failed to move to tb db, uid:%" PRId64, uid); - } - if (c != 0) { - tdbTbcClose(pUidIdxc); - tdbTbcClose(pTbDbc); - metaError("meta/table: invalide c: %" PRId32 " update tb options failed.", c); - return TSDB_CODE_FAILED; - } - - if (tdbTbcGet(pTbDbc, NULL, NULL, &pData, &nData) < 0) { - metaError("meta/table: failed to get tb db, uid:%" PRId64, uid); - } - - // get table entry - SDecoder dc = {0}; - if ((entry.pBuf = taosMemoryMalloc(nData)) == NULL) { - tdbTbcClose(pUidIdxc); - tdbTbcClose(pTbDbc); - return terrno; - } - memcpy(entry.pBuf, pData, nData); - tDecoderInit(&dc, entry.pBuf, nData); - ret = metaDecodeEntry(&dc, &entry); - if (ret != 0) { - tDecoderClear(&dc); - tdbTbcClose(pUidIdxc); - tdbTbcClose(pTbDbc); - metaError("meta/table: invalide ret: %" PRId32 " alt tb options failed.", ret); - return TSDB_CODE_FAILED; - } - - entry.version = version; - metaWLock(pMeta); - // build SMetaEntry - if (entry.type == TSDB_CHILD_TABLE) { - if (pAlterTbReq->updateTTL) { - metaDeleteTtl(pMeta, &entry); - entry.ctbEntry.ttlDays = pAlterTbReq->newTTL; - metaUpdateTtl(pMeta, &entry); - } - if (pAlterTbReq->newCommentLen >= 0) { - entry.ctbEntry.commentLen = pAlterTbReq->newCommentLen; - entry.ctbEntry.comment = pAlterTbReq->newComment; - } - } else { - if (pAlterTbReq->updateTTL) { - metaDeleteTtl(pMeta, &entry); - entry.ntbEntry.ttlDays = pAlterTbReq->newTTL; - metaUpdateTtl(pMeta, &entry); - } - if (pAlterTbReq->newCommentLen >= 0) { - entry.ntbEntry.commentLen = pAlterTbReq->newCommentLen; - entry.ntbEntry.comment = pAlterTbReq->newComment; - } - } - - // save to table db - if (metaSaveToTbDb(pMeta, &entry) < 0) { - metaError("meta/table: failed to save to tb db:%s uid:%" PRId64, entry.name, entry.uid); - } - - if (metaUpdateUidIdx(pMeta, &entry) < 0) { - metaError("meta/table: failed to update uid idx:%s uid:%" PRId64, entry.name, entry.uid); - } - - if (metaUpdateChangeTime(pMeta, entry.uid, pAlterTbReq->ctimeMs) < 0) { - metaError("meta/table: failed to update change time:%s uid:%" PRId64, entry.name, entry.uid); - } - - metaULock(pMeta); - - tdbTbcClose(pTbDbc); - tdbTbcClose(pUidIdxc); - tDecoderClear(&dc); - if (entry.pBuf) taosMemoryFree(entry.pBuf); - return 0; -} - -typedef struct SMetaPair { - void *key; - int nkey; -} SMetaPair; - -static int metaDropTagIndex(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTbReq) { - SMetaEntry stbEntry = {0}; - void *pVal = NULL; - int nVal = 0; - int ret; - int c; - tb_uid_t suid; - int64_t oversion; - const void *pData = NULL; - int nData = 0; - SDecoder dc = {0}; - - if (pAlterTbReq->tagName == NULL) { - return terrno = TSDB_CODE_INVALID_MSG; - } - - // search name index - ret = tdbTbGet(pMeta->pNameIdx, pAlterTbReq->tbName, strlen(pAlterTbReq->tbName) + 1, &pVal, &nVal); - if (ret < 0) { - return terrno = TSDB_CODE_TDB_TABLE_NOT_EXIST; - } - suid = *(tb_uid_t *)pVal; - tdbFree(pVal); - pVal = NULL; - - if (tdbTbGet(pMeta->pUidIdx, &suid, sizeof(tb_uid_t), &pVal, &nVal) == -1) { - ret = -1; - goto _err; - } - - STbDbKey tbDbKey = {0}; - tbDbKey.uid = suid; - tbDbKey.version = ((SUidIdxVal *)pVal)[0].version; - ret = tdbTbGet(pMeta->pTbDb, &tbDbKey, sizeof(tbDbKey), &pVal, &nVal); - if (ret < 0) { - goto _err; - } - - tDecoderInit(&dc, pVal, nVal); - ret = metaDecodeEntry(&dc, &stbEntry); - if (ret < 0) { - tDecoderClear(&dc); - goto _err; - } - - // Get targe schema info - SSchemaWrapper *pTagSchema = &stbEntry.stbEntry.schemaTag; - if (pTagSchema->nCols == 1 && pTagSchema->pSchema[0].type == TSDB_DATA_TYPE_JSON) { - terrno = TSDB_CODE_VND_COL_ALREADY_EXISTS; - goto _err; - } - SSchema *pCol = NULL; - int32_t iCol = 0; - for (;;) { - pCol = NULL; - if (iCol >= pTagSchema->nCols) break; - pCol = &pTagSchema->pSchema[iCol]; - if (strcmp(pCol->name, pAlterTbReq->tagName) == 0) break; - iCol++; - } - if (iCol == 0) { - // cannot drop 1th tag index - terrno = -1; - goto _err; - } - if (pCol == NULL) { - terrno = TSDB_CODE_VND_COL_NOT_EXISTS; - goto _err; - } - - if (IS_IDX_ON(pCol)) { - terrno = TSDB_CODE_VND_COL_ALREADY_EXISTS; - goto _err; - } - - SArray *tagIdxList = taosArrayInit(512, sizeof(SMetaPair)); - if (tagIdxList == NULL) { - goto _err; - } - - TBC *pTagIdxc = NULL; - TAOS_CHECK_RETURN(tdbTbcOpen(pMeta->pTagIdx, &pTagIdxc, NULL)); - int rc = - tdbTbcMoveTo(pTagIdxc, &(STagIdxKey){.suid = suid, .cid = INT32_MIN, .type = pCol->type}, sizeof(STagIdxKey), &c); - for (;;) { - void *pKey, *pVal; - int nKey, nVal; - rc = tdbTbcNext(pTagIdxc, &pKey, &nKey, &pVal, &nVal); - STagIdxKey *pIdxKey = (STagIdxKey *)pKey; - if (pIdxKey->suid != suid || pIdxKey->cid != pCol->colId) { - tdbFree(pKey); - tdbFree(pVal); - continue; - } - - SMetaPair pair = {.key = pKey, nKey = nKey}; - if (taosArrayPush(tagIdxList, &pair) == NULL) { - goto _err; - } - } - tdbTbcClose(pTagIdxc); - - metaWLock(pMeta); - for (int i = 0; i < taosArrayGetSize(tagIdxList); i++) { - SMetaPair *pair = taosArrayGet(tagIdxList, i); - ret = tdbTbDelete(pMeta->pTagIdx, pair->key, pair->nkey, pMeta->txn); - if (ret < 0) { - metaError("meta/table: failed to delete tag idx:%s uid:%" PRId64, stbEntry.name, stbEntry.uid); - } - } - metaULock(pMeta); - - taosArrayDestroy(tagIdxList); - - // set pCol->flags; INDEX_ON - return 0; -_err: - return TSDB_CODE_FAILED; -} -static int32_t metaUpdateTableColCompress(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq) { - // impl later - SMetaEntry tbEntry = {0}; - void *pVal = NULL; - int nVal = 0; - int ret; - int c; - tb_uid_t suid; - int64_t oversion; - const void *pData = NULL; - int nData = 0; - SDecoder dc = {0}; - ret = tdbTbGet(pMeta->pNameIdx, pReq->tbName, strlen(pReq->tbName) + 1, &pVal, &nVal); - if (ret < 0) { - return terrno = TSDB_CODE_TDB_TABLE_NOT_EXIST; - } - suid = *(tb_uid_t *)pVal; - tdbFree(pVal); - pVal = NULL; - - if (tdbTbGet(pMeta->pUidIdx, &suid, sizeof(tb_uid_t), &pVal, &nVal) == -1) { - terrno = TSDB_CODE_INVALID_MSG; - ret = -1; - goto _err; - } - - STbDbKey tbDbKey = {0}; - tbDbKey.uid = suid; - tbDbKey.version = ((SUidIdxVal *)pVal)[0].version; - if (tdbTbGet(pMeta->pTbDb, &tbDbKey, sizeof(tbDbKey), &pVal, &nVal) < 0) { - terrno = TSDB_CODE_INVALID_MSG; - tdbFree(pVal); - goto _err; - } - - tDecoderInit(&dc, pVal, nVal); - ret = metaDecodeEntry(&dc, &tbEntry); - if (ret < 0) { - terrno = TSDB_CODE_INVALID_MSG; - tdbFree(pVal); - tDecoderClear(&dc); - goto _err; - } - if (tbEntry.type != TSDB_NORMAL_TABLE && tbEntry.type != TSDB_SUPER_TABLE) { - terrno = TSDB_CODE_INVALID_MSG; - tdbFree(pVal); - tDecoderClear(&dc); - goto _err; - } - int8_t updated = 0; - SColCmprWrapper *wp = &tbEntry.colCmpr; - for (int32_t i = 0; i < wp->nCols; i++) { - SColCmpr *p = &wp->pColCmpr[i]; - if (p->id == pReq->colId) { - uint32_t dst = 0; - updated = tUpdateCompress(p->alg, pReq->compress, TSDB_COLVAL_COMPRESS_DISABLED, TSDB_COLVAL_LEVEL_DISABLED, - TSDB_COLVAL_LEVEL_MEDIUM, &dst); - if (updated > 0) { - p->alg = dst; - } - } - } - if (updated == 0) { - tdbFree(pVal); - tDecoderClear(&dc); - terrno = TSDB_CODE_VND_COLUMN_COMPRESS_ALREADY_EXIST; - goto _err; - } else if (updated < 0) { - tdbFree(pVal); - tDecoderClear(&dc); - terrno = TSDB_CODE_TSC_COMPRESS_LEVEL_ERROR; - goto _err; - } - tbEntry.version = version; - - metaWLock(pMeta); - if (metaSaveToTbDb(pMeta, &tbEntry) < 0) { - metaError("meta/table: failed to save to tb db:%s uid:%" PRId64, tbEntry.name, tbEntry.uid); - } - - if (metaUpdateUidIdx(pMeta, &tbEntry) < 0) { - metaError("meta/table: failed to update uid idx:%s uid:%" PRId64, tbEntry.name, tbEntry.uid); - } - - if (metaUpdateChangeTime(pMeta, suid, pReq->ctimeMs) < 0) { - metaError("meta/table: failed to update change time:%s uid:%" PRId64, tbEntry.name, tbEntry.uid); - } - - metaULock(pMeta); - - tdbFree(pVal); - tDecoderClear(&dc); - - return 0; -_err: - return TSDB_CODE_FAILED; -} - int metaAlterTable(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq, STableMetaRsp *pMetaRsp) { pMeta->changed = true; switch (pReq->action) { @@ -2878,102 +709,6 @@ int metaAlterTable(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq, STableMeta } } -static int metaSaveToTbDb(SMeta *pMeta, const SMetaEntry *pME) { - STbDbKey tbDbKey; - void *pKey = NULL; - void *pVal = NULL; - int kLen = 0; - int vLen = 0; - SEncoder coder = {0}; - - // set key and value - tbDbKey.version = pME->version; - tbDbKey.uid = pME->uid; - - metaDebug("vgId:%d, start to save table version:%" PRId64 " uid:%" PRId64, TD_VID(pMeta->pVnode), pME->version, - pME->uid); - - pKey = &tbDbKey; - kLen = sizeof(tbDbKey); - - int32_t ret = 0; - tEncodeSize(metaEncodeEntry, pME, vLen, ret); - if (ret < 0) { - goto _err; - } - - pVal = taosMemoryMalloc(vLen); - if (pVal == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - - tEncoderInit(&coder, pVal, vLen); - - if (metaEncodeEntry(&coder, pME) < 0) { - goto _err; - } - - tEncoderClear(&coder); - - // write to table.db - if (tdbTbInsert(pMeta->pTbDb, pKey, kLen, pVal, vLen, pMeta->txn) < 0) { - goto _err; - } - - taosMemoryFree(pVal); - return 0; - -_err: - metaError("vgId:%d, failed to save table version:%" PRId64 "uid:%" PRId64 " %s", TD_VID(pMeta->pVnode), pME->version, - pME->uid, tstrerror(terrno)); - - taosMemoryFree(pVal); - return TSDB_CODE_FAILED; -} - -static int metaUpdateUidIdx(SMeta *pMeta, const SMetaEntry *pME) { - // upsert cache - SMetaInfo info; - metaGetEntryInfo(pME, &info); - int32_t ret = metaCacheUpsert(pMeta, &info); - if (ret < 0) { - metaError("vgId:%d, failed to upsert cache, uid: %" PRId64 " %s", TD_VID(pMeta->pVnode), pME->uid, tstrerror(ret)); - } - - SUidIdxVal uidIdxVal = {.suid = info.suid, .version = info.version, .skmVer = info.skmVer}; - - return tdbTbUpsert(pMeta->pUidIdx, &pME->uid, sizeof(tb_uid_t), &uidIdxVal, sizeof(uidIdxVal), pMeta->txn); -} - -static int metaUpdateSuidIdx(SMeta *pMeta, const SMetaEntry *pME) { - return tdbTbUpsert(pMeta->pSuidIdx, &pME->uid, sizeof(tb_uid_t), NULL, 0, pMeta->txn); -} - -static int metaUpdateNameIdx(SMeta *pMeta, const SMetaEntry *pME) { - return tdbTbUpsert(pMeta->pNameIdx, pME->name, strlen(pME->name) + 1, &pME->uid, sizeof(tb_uid_t), pMeta->txn); -} - -static void metaUpdateTtl(SMeta *pMeta, const SMetaEntry *pME) { - if (pME->type != TSDB_CHILD_TABLE && pME->type != TSDB_NORMAL_TABLE) return; - - STtlUpdTtlCtx ctx = {.uid = pME->uid, .pTxn = pMeta->txn}; - if (pME->type == TSDB_CHILD_TABLE) { - ctx.ttlDays = pME->ctbEntry.ttlDays; - ctx.changeTimeMs = pME->ctbEntry.btime; - } else { - ctx.ttlDays = pME->ntbEntry.ttlDays; - ctx.changeTimeMs = pME->ntbEntry.btime; - } - - int32_t ret = ttlMgrInsertTtl(pMeta->pTtlMgr, &ctx); - if (ret < 0) { - metaError("vgId:%d, failed to insert ttl, uid: %" PRId64 " %s", TD_VID(pMeta->pVnode), pME->uid, tstrerror(ret)); - } - - return; -} - static int metaUpdateChangeTime(SMeta *pMeta, tb_uid_t uid, int64_t changeTimeMs) { if (!tsTtlChangeOnWrite) return 0; @@ -2996,13 +731,6 @@ int metaUpdateChangeTimeWithLock(SMeta *pMeta, tb_uid_t uid, int64_t changeTimeM return ret; } -static int metaUpdateCtbIdx(SMeta *pMeta, const SMetaEntry *pME) { - SCtbIdxKey ctbIdxKey = {.suid = pME->ctbEntry.suid, .uid = pME->uid}; - - return tdbTbUpsert(pMeta->pCtbIdx, &ctbIdxKey, sizeof(ctbIdxKey), pME->ctbEntry.pTags, - ((STag *)(pME->ctbEntry.pTags))->len, pMeta->txn); -} - int metaCreateTagIdxKey(tb_uid_t suid, int32_t cid, const void *pTagData, int32_t nTagData, int8_t type, tb_uid_t uid, STagIdxKey **ppTagIdxKey, int32_t *nTagIdxKey) { if (IS_VAR_DATA_TYPE(type)) { @@ -3038,212 +766,6 @@ void metaDestroyTagIdxKey(STagIdxKey *pTagIdxKey) { if (pTagIdxKey) taosMemoryFree(pTagIdxKey); } -static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) { - void *pData = NULL; - int nData = 0; - STbDbKey tbDbKey = {0}; - SMetaEntry stbEntry = {0}; - STagIdxKey *pTagIdxKey = NULL; - int32_t nTagIdxKey; - const SSchema *pTagColumn; - const void *pTagData = NULL; - int32_t nTagData = 0; - SDecoder dc = {0}; - int32_t ret = 0; - // get super table - if (tdbTbGet(pMeta->pUidIdx, &pCtbEntry->ctbEntry.suid, sizeof(tb_uid_t), &pData, &nData) != 0) { - metaError("vgId:%d, failed to get stable suid for update. version:%" PRId64, TD_VID(pMeta->pVnode), - pCtbEntry->version); - ret = TSDB_CODE_TDB_INVALID_TABLE_ID; - goto end; - } - tbDbKey.uid = pCtbEntry->ctbEntry.suid; - tbDbKey.version = ((SUidIdxVal *)pData)[0].version; - ret = tdbTbGet(pMeta->pTbDb, &tbDbKey, sizeof(tbDbKey), &pData, &nData); - if (ret < 0) { - metaError("vgId:%d, failed to get stable for update. version:%" PRId64, TD_VID(pMeta->pVnode), pCtbEntry->version); - goto end; - } - - tDecoderInit(&dc, pData, nData); - ret = metaDecodeEntry(&dc, &stbEntry); - if (ret < 0) { - goto end; - } - - if (stbEntry.stbEntry.schemaTag.pSchema == NULL) { - ret = TSDB_CODE_INVALID_PARA; - goto end; - } - - SSchemaWrapper *pTagSchema = &stbEntry.stbEntry.schemaTag; - if (pTagSchema->nCols == 1 && pTagSchema->pSchema[0].type == TSDB_DATA_TYPE_JSON) { - pTagColumn = &stbEntry.stbEntry.schemaTag.pSchema[0]; - STagVal tagVal = {.cid = pTagColumn->colId}; - - pTagData = pCtbEntry->ctbEntry.pTags; - nTagData = ((const STag *)pCtbEntry->ctbEntry.pTags)->len; - ret = metaSaveJsonVarToIdx(pMeta, pCtbEntry, pTagColumn); - goto end; - } else { - for (int i = 0; i < pTagSchema->nCols; i++) { - pTagData = NULL; - nTagData = 0; - pTagColumn = &pTagSchema->pSchema[i]; - if (!IS_IDX_ON(pTagColumn)) continue; - - STagVal tagVal = {.cid = pTagColumn->colId}; - if (tTagGet((const STag *)pCtbEntry->ctbEntry.pTags, &tagVal)) { - if (IS_VAR_DATA_TYPE(pTagColumn->type)) { - pTagData = tagVal.pData; - nTagData = (int32_t)tagVal.nData; - } else { - pTagData = &(tagVal.i64); - nTagData = tDataTypes[pTagColumn->type].bytes; - } - } else { - if (!IS_VAR_DATA_TYPE(pTagColumn->type)) { - nTagData = tDataTypes[pTagColumn->type].bytes; - } - } - if (metaCreateTagIdxKey(pCtbEntry->ctbEntry.suid, pTagColumn->colId, pTagData, nTagData, pTagColumn->type, - pCtbEntry->uid, &pTagIdxKey, &nTagIdxKey) < 0) { - ret = -1; - goto end; - } - if (tdbTbUpsert(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, NULL, 0, pMeta->txn) < 0) { - metaError("vgId:%d, failed to update tag index. version:%" PRId64, TD_VID(pMeta->pVnode), pCtbEntry->version); - } - metaDestroyTagIdxKey(pTagIdxKey); - pTagIdxKey = NULL; - } - } -end: - tDecoderClear(&dc); - tdbFree(pData); - return ret; -} - -static int metaSaveToSkmDb(SMeta *pMeta, const SMetaEntry *pME) { - SEncoder coder = {0}; - void *pVal = NULL; - int vLen = 0; - int rcode = 0; - SSkmDbKey skmDbKey = {0}; - const SSchemaWrapper *pSW; - - if (pME->type == TSDB_SUPER_TABLE) { - pSW = &pME->stbEntry.schemaRow; - } else if (pME->type == TSDB_NORMAL_TABLE) { - pSW = &pME->ntbEntry.schemaRow; - } else { - metaError("meta/table: invalide table type: %" PRId8 " save skm db failed.", pME->type); - return TSDB_CODE_FAILED; - } - - skmDbKey.uid = pME->uid; - skmDbKey.sver = pSW->version; - - // if receive tmq meta message is: create stable1 then delete stable1 then create stable1 with multi vgroups - if (tdbTbGet(pMeta->pSkmDb, &skmDbKey, sizeof(skmDbKey), NULL, NULL) == 0) { - return rcode; - } - - // encode schema - int32_t ret = 0; - tEncodeSize(tEncodeSSchemaWrapper, pSW, vLen, ret); - if (ret < 0) return -1; - pVal = taosMemoryMalloc(vLen); - if (pVal == NULL) { - rcode = -1; - terrno = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; - } - - tEncoderInit(&coder, pVal, vLen); - ret = tEncodeSSchemaWrapper(&coder, pSW); - if (ret < 0) { - rcode = -1; - goto _exit; - } - - if (tdbTbInsert(pMeta->pSkmDb, &skmDbKey, sizeof(skmDbKey), pVal, vLen, pMeta->txn) < 0) { - rcode = -1; - goto _exit; - } - - metaDebug("vgId:%d, set schema:(%" PRId64 ") sver:%d since %s", TD_VID(pMeta->pVnode), pME->uid, pSW->version, - tstrerror(terrno)); - -_exit: - taosMemoryFree(pVal); - tEncoderClear(&coder); - return rcode; -} - -int metaHandleEntry(SMeta *pMeta, const SMetaEntry *pME) { - int32_t code = 0; - int32_t line = 0; - metaWLock(pMeta); - - // save to table.db - code = metaSaveToTbDb(pMeta, pME); - VND_CHECK_CODE(code, line, _err); - - // update uid.idx - code = metaUpdateUidIdx(pMeta, pME); - VND_CHECK_CODE(code, line, _err); - - // update name.idx - code = metaUpdateNameIdx(pMeta, pME); - VND_CHECK_CODE(code, line, _err); - - if (pME->type == TSDB_CHILD_TABLE) { - // update ctb.idx - code = metaUpdateCtbIdx(pMeta, pME); - VND_CHECK_CODE(code, line, _err); - - // update tag.idx - code = metaUpdateTagIdx(pMeta, pME); - VND_CHECK_CODE(code, line, _err); - } else { - // update schema.db - code = metaSaveToSkmDb(pMeta, pME); - VND_CHECK_CODE(code, line, _err); - - if (pME->type == TSDB_SUPER_TABLE) { - code = metaUpdateSuidIdx(pMeta, pME); - VND_CHECK_CODE(code, line, _err); - } - } - - code = metaUpdateBtimeIdx(pMeta, pME); - VND_CHECK_CODE(code, line, _err); - - if (pME->type == TSDB_NORMAL_TABLE) { - code = metaUpdateNcolIdx(pMeta, pME); - VND_CHECK_CODE(code, line, _err); - } - - if (pME->type != TSDB_SUPER_TABLE) { - metaUpdateTtl(pMeta, pME); - } - - if (pME->type == TSDB_SUPER_TABLE || pME->type == TSDB_NORMAL_TABLE) { - } - - metaULock(pMeta); - metaDebug("vgId:%d, handle meta entry, ver:%" PRId64 ", uid:%" PRId64 ", name:%s", TD_VID(pMeta->pVnode), - pME->version, pME->uid, pME->name); - return 0; - -_err: - metaULock(pMeta); - metaError("vgId:%d, failed to handle meta entry since %s at line:%d, ver:%" PRId64 ", uid:%" PRId64 ", name:%s", - TD_VID(pMeta->pVnode), terrstr(), line, pME->version, pME->uid, pME->name); - return TSDB_CODE_FAILED; -} - static void colCompressDebug(SHashObj *pColCmprObj) { void *p = taosHashIterate(pColCmprObj, NULL); while (p) { @@ -3261,6 +783,7 @@ static void colCompressDebug(SHashObj *pColCmprObj) { } return; } + int32_t metaGetColCmpr(SMeta *pMeta, tb_uid_t uid, SHashObj **ppColCmprObj) { int rc = 0; diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 6291e0c9c7..f8b75eb15c 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -843,7 +843,7 @@ int32_t vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) { SReadHandle handle = {.vnode = pVnode, .pMsgCb = &pVnode->msgCb, .pWorkerCb = pInfo->workerCb}; initStorageAPI(&handle.api); int32_t code = TSDB_CODE_SUCCESS; - bool redirected = false; + bool redirected = false; switch (pMsg->msgType) { case TDMT_SCH_QUERY: @@ -2145,7 +2145,7 @@ static int32_t vnodeConsolidateAlterHashRange(SVnode *pVnode, int64_t ver) { pVnode->config.hashBegin, pVnode->config.hashEnd, ver); // TODO: trim meta of tables from TDB per hash range [pVnode->config.hashBegin, pVnode->config.hashEnd] - code = metaTrimTables(pVnode->pMeta); + code = metaTrimTables(pVnode->pMeta, ver); return code; } diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 199bac5246..bc1462176a 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -424,6 +424,7 @@ int32_t ctgGetTbCfg(SCatalog* pCtg, SRequestConnInfo* pConn, SName* pTableName, CTG_RET(TSDB_CODE_SUCCESS); } +#if 0 int32_t ctgGetTbTag(SCatalog* pCtg, SRequestConnInfo* pConn, SName* pTableName, SArray** pRes) { SVgroupInfo vgroupInfo = {0}; STableCfg* pCfg = NULL; @@ -474,6 +475,7 @@ _return: CTG_RET(code); } +#endif int32_t ctgGetTbDistVgInfo(SCatalog* pCtg, SRequestConnInfo* pConn, SName* pTableName, SArray** pVgList) { STableMeta* tbMeta = NULL; @@ -1695,6 +1697,7 @@ _return: CTG_API_LEAVE(code); } +#if 0 int32_t catalogGetTableTag(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, SArray** pRes) { CTG_API_ENTER(); @@ -1709,6 +1712,7 @@ _return: CTG_API_LEAVE(code); } +#endif int32_t catalogRefreshGetTableCfg(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, STableCfg** pCfg) { CTG_API_ENTER(); @@ -1845,7 +1849,7 @@ _return: CTG_API_LEAVE(code); } - +#if 0 int32_t catalogAsyncUpdateViewMeta(SCatalog* pCtg, SViewMetaRsp* pMsg) { CTG_API_ENTER(); @@ -1860,6 +1864,7 @@ _return: CTG_API_LEAVE(code); } +#endif int32_t catalogGetViewMeta(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pViewName, STableMeta** pTableMeta) { CTG_API_ENTER(); diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index 7b0504504d..c2889f096b 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -2992,6 +2992,7 @@ TEST(apiTest, catalogGetTableIndex_test) { catalogDestroy(); } + TEST(apiTest, catalogGetDBCfg_test) { struct SCatalog *pCtg = NULL; SRequestConnInfo connInfo = {0}; diff --git a/source/libs/executor/test/joinTests.cpp b/source/libs/executor/test/joinTests.cpp index 6dd42c3a93..efbe1fcc83 100755 --- a/source/libs/executor/test/joinTests.cpp +++ b/source/libs/executor/test/joinTests.cpp @@ -41,6 +41,7 @@ #include "tvariant.h" #include "stub.h" #include "querytask.h" +#include "hashjoin.h" namespace { @@ -3766,6 +3767,21 @@ TEST(leftWinJoin, noCondProjectionTest) { #endif +#if 1 +TEST(functionsTest, branch) { + struct SOperatorInfo op = {0}; + SHJoinOperatorInfo join; + SBufRowInfo bufrow = {0}; + SSDataBlock blk = {0}; + + op.info = &join; + memset(&join, 0, sizeof(join)); + join.ctx.pBuildRow = &bufrow; + blk.info.rows = 1; + join.finBlk = &blk; + hInnerJoinDo(&op); +} +#endif int main(int argc, char** argv) { diff --git a/source/libs/planner/src/planner.c b/source/libs/planner/src/planner.c index c3aa95f5b7..ee460e2610 100644 --- a/source/libs/planner/src/planner.c +++ b/source/libs/planner/src/planner.c @@ -161,6 +161,10 @@ int32_t qSubPlanToString(const SSubplan* pSubplan, char** pStr, int32_t* pLen) { int32_t qStringToSubplan(const char* pStr, SSubplan** pSubplan) { return nodesStringToNode(pStr, (SNode**)pSubplan); } int32_t qSubPlanToMsg(const SSubplan* pSubplan, char** pStr, int32_t* pLen) { + if (NULL == pSubplan) { + return terrno = TSDB_CODE_INVALID_PARA; + } + if (SUBPLAN_TYPE_MODIFY == pSubplan->subplanType && NULL == pSubplan->pNode) { SDataInserterNode* insert = (SDataInserterNode*)pSubplan->pDataSink; *pLen = insert->size; diff --git a/source/libs/qworker/src/qwMem.c b/source/libs/qworker/src/qwMem.c index 69d4093221..6451e073c8 100644 --- a/source/libs/qworker/src/qwMem.c +++ b/source/libs/qworker/src/qwMem.c @@ -1,6 +1,7 @@ #include "qwInt.h" #include "qworker.h" +#if 0 void qwSetConcurrentTaskNumCb(int32_t taskNum) { int32_t finTaskNum = TMIN(taskNum, tsNumOfQueryThreads * QW_DEFAULT_THREAD_TASK_NUM); @@ -33,6 +34,7 @@ void qwIncConcurrentTaskNumCb(void) { //TODO } +#endif int32_t qwInitJobInfo(QW_FPARAMS_DEF, SQWJobInfo* pJob) { pJob->pSessions= taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); diff --git a/source/libs/scheduler/inc/schInt.h b/source/libs/scheduler/inc/schInt.h index 607f43a06f..cdbeb2a2e0 100644 --- a/source/libs/scheduler/inc/schInt.h +++ b/source/libs/scheduler/inc/schInt.h @@ -149,11 +149,11 @@ typedef struct SSchedulerMgmt { bool exit; int32_t jobRef; int32_t jobNum; - SSchStat stat; void *timer; SRWLatch hbLock; SHashObj *hbConnections; void *queryMgmt; + SSchStat stat; } SSchedulerMgmt; typedef struct SSchCallbackParamHeader { diff --git a/source/libs/scheduler/src/schFlowCtrl.c b/source/libs/scheduler/src/schFlowCtrl.c index d9a0b88e40..c66f1dafe6 100644 --- a/source/libs/scheduler/src/schFlowCtrl.c +++ b/source/libs/scheduler/src/schFlowCtrl.c @@ -50,11 +50,6 @@ int32_t schChkJobNeedFlowCtrl(SSchJob *pJob, SSchLevel *pLevel) { int32_t taskNum = taosArrayGetSize(pJob->dataSrcTasks); for (int32_t i = 0; i < taskNum; ++i) { SSchTask *pTask = *(SSchTask **)taosArrayGet(pJob->dataSrcTasks, i); - if (NULL == pTask) { - SCH_JOB_DLOG("fail to get the %dth task", i); - SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); - } - sum += pTask->plan->execNodeStat.tableNum; } diff --git a/source/libs/scheduler/src/schTask.c b/source/libs/scheduler/src/schTask.c index b31353e97f..0e76e21aff 100644 --- a/source/libs/scheduler/src/schTask.c +++ b/source/libs/scheduler/src/schTask.c @@ -273,18 +273,18 @@ int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) { if (taskDone < pTask->level->taskNum) { SCH_TASK_DLOG("wait all tasks, done:%d, all:%d", taskDone, pTask->level->taskNum); return TSDB_CODE_SUCCESS; - } else if (taskDone > pTask->level->taskNum) { - SCH_TASK_ELOG("taskDone number invalid, done:%d, total:%d", taskDone, pTask->level->taskNum); } + + SCH_TASK_DLOG("taskDone number reach level task number, done:%d, total:%d", taskDone, pTask->level->taskNum); if (pTask->level->taskFailed > 0) { SCH_RET(schHandleJobFailure(pJob, pJob->errCode)); - } else { - SCH_RET(schSwitchJobStatus(pJob, JOB_TASK_STATUS_PART_SUCC, NULL)); } - } else { - pJob->resNode = pTask->succeedAddr; + + SCH_RET(schSwitchJobStatus(pJob, JOB_TASK_STATUS_PART_SUCC, NULL)); } + + pJob->resNode = pTask->succeedAddr; pJob->fetchTask = pTask; @@ -600,12 +600,12 @@ int32_t schPushTaskToExecList(SSchJob *pJob, SSchTask *pTask) { int32_t code = taosHashPut(pJob->execTasks, &pTask->taskId, sizeof(pTask->taskId), &pTask, POINTER_BYTES); if (0 != code) { if (HASH_NODE_EXIST(code)) { - SCH_TASK_DLOG("task already in execTask list, code:%x", code); + SCH_TASK_DLOG("task already in execTask list, code:0x%x", code); return TSDB_CODE_SUCCESS; } - SCH_TASK_ELOG("taosHashPut task to execTask list failed, errno:0x%x", errno); - SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + SCH_TASK_ELOG("taosHashPut task to execTask list failed, code:0x%x", code); + SCH_ERR_RET(code); } SCH_TASK_DLOG("task added to execTask list, numOfTasks:%d", taosHashGetSize(pJob->execTasks)); @@ -800,11 +800,6 @@ int32_t schSetAddrsFromNodeList(SSchJob *pJob, SSchTask *pTask) { for (int32_t i = 0; i < nodeNum; ++i) { SQueryNodeLoad *nload = taosArrayGet(pJob->nodeList, i); - if (NULL == nload) { - SCH_TASK_ELOG("fail to get the %dth node in nodeList, nodeNum:%d", i, nodeNum); - SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); - } - SQueryNodeAddr *naddr = &nload->addr; if (NULL == taosArrayPush(pTask->candidateAddrs, naddr)) { diff --git a/source/libs/scheduler/test/schedulerTests.cpp b/source/libs/scheduler/test/schedulerTests.cpp index c13ea913f5..9d70227c48 100644 --- a/source/libs/scheduler/test/schedulerTests.cpp +++ b/source/libs/scheduler/test/schedulerTests.cpp @@ -60,6 +60,19 @@ extern "C" int32_t schHandleCallback(void *param, const SDataBuf *pMsg, int32_t extern "C" int32_t schHandleNotifyCallback(void *param, SDataBuf *pMsg, int32_t code); extern "C" int32_t schHandleLinkBrokenCallback(void *param, SDataBuf *pMsg, int32_t code); extern "C" int32_t schRescheduleTask(SSchJob *pJob, SSchTask *pTask); +extern "C" int32_t schValidateRspMsgType(SSchJob *pJob, SSchTask *pTask, int32_t msgType); +//extern "C" int32_t schProcessFetchRsp(SSchJob *pJob, SSchTask *pTask, char *msg, int32_t rspCode); +extern "C" int32_t schProcessResponseMsg(SSchJob *pJob, SSchTask *pTask, SDataBuf *pMsg, int32_t rspCode); +extern "C" void schInitTaskRetryTimes(SSchJob *pJob, SSchTask *pTask, SSchLevel *pLevel); +extern "C" int32_t schRecordTaskSucceedNode(SSchJob *pJob, SSchTask *pTask); +extern "C" int32_t schDropTaskExecNode(SSchJob *pJob, SSchTask *pTask, void *handle, int32_t execId); +extern "C" int32_t schPushTaskToExecList(SSchJob *pJob, SSchTask *pTask); +extern "C" int32_t schSetAddrsFromNodeList(SSchJob *pJob, SSchTask *pTask); +extern "C" int32_t schNotifyTaskOnExecNode(SSchJob *pJob, SSchTask *pTask, ETaskNotifyType type); +extern "C" int32_t schLaunchRemoteTask(SSchJob *pJob, SSchTask *pTask); +extern "C" int32_t schLaunchTaskImpl(void *param); +extern "C" void schHandleTimerEvent(void *param, void *tmrId); + int64_t insertJobRefId = 0; int64_t queryJobRefId = 0; @@ -1473,6 +1486,263 @@ TEST(otherTest, function) { schMgmt.jobRef = -1; } +TEST(otherTest, branch) { + SSchJob job = {0}; + SSchTask task = {0}; + memset(&schMgmt, 0, sizeof(schMgmt)); + + schValidateRspMsgType(&job, &task, TDMT_SCH_MERGE_FETCH_RSP); + + task.lastMsgType = TDMT_SCH_MERGE_FETCH_RSP - 1; + schValidateRspMsgType(&job, &task, TDMT_SCH_MERGE_FETCH_RSP); + + schValidateRspMsgType(&job, &task, 0); + + schValidateRspMsgType(&job, &task, TDMT_SCH_QUERY_RSP); + + task.lastMsgType = TDMT_SCH_QUERY_RSP - 1; + schValidateRspMsgType(&job, &task, TDMT_SCH_QUERY_RSP); + + schProcessFetchRsp(&job, &task, NULL, -1); + schProcessFetchRsp(&job, &task, NULL, 0); + + job.fetchRes = (void*)0x1; + schProcessFetchRsp(&job, &task, (char*)taosMemoryMalloc(0), 0); + job.fetchRes = NULL; + + SDataBuf databuf = {0}; + databuf.msgType = TDMT_VND_ALTER_TABLE_RSP; + job.status = JOB_TASK_STATUS_FAIL; + schProcessResponseMsg(&job, &task, &databuf, 0); + + databuf.msgType = TDMT_VND_SUBMIT_RSP; + databuf.pData = taosMemoryMalloc(0); + job.status = JOB_TASK_STATUS_FAIL; + schProcessResponseMsg(&job, &task, &databuf, 0); + + databuf.msgType = TDMT_VND_DELETE_RSP; + databuf.pData = taosMemoryMalloc(0); + job.status = JOB_TASK_STATUS_FAIL; + schProcessResponseMsg(&job, &task, &databuf, 0); + + databuf.msgType = TDMT_SCH_QUERY_RSP; + job.status = JOB_TASK_STATUS_FAIL; + schProcessResponseMsg(&job, &task, &databuf, 0); + + databuf.msgType = TDMT_SCH_QUERY_RSP; + databuf.pData = taosMemoryMalloc(0); + job.status = JOB_TASK_STATUS_FAIL; + schProcessResponseMsg(&job, &task, &databuf, 0); + + + databuf.msgType = TDMT_SCH_EXPLAIN_RSP; + job.status = JOB_TASK_STATUS_FAIL; + schProcessResponseMsg(&job, &task, &databuf, 0); + + databuf.msgType = TDMT_SCH_EXPLAIN_RSP; + databuf.pData = taosMemoryMalloc(0); + job.status = JOB_TASK_STATUS_FAIL; + schProcessResponseMsg(&job, &task, &databuf, 0); + job.status = 0; + + job.attr.explainMode = EXPLAIN_MODE_ANALYZE; + databuf.msgType = TDMT_SCH_EXPLAIN_RSP; + databuf.pData = taosMemoryMalloc(0); + job.status = JOB_TASK_STATUS_FAIL; + job.fetchRes = (void*)0x1; + schProcessResponseMsg(&job, &task, &databuf, 0); + job.fetchRes = NULL; + + job.attr.explainMode = EXPLAIN_MODE_ANALYZE; + databuf.msgType = TDMT_SCH_EXPLAIN_RSP; + databuf.pData = taosMemoryMalloc(0); + job.status = JOB_TASK_STATUS_FAIL; + schProcessResponseMsg(&job, &task, &databuf, 0); + + databuf.msgType = TDMT_SCH_DROP_TASK_RSP; + job.status = JOB_TASK_STATUS_FAIL; + schProcessResponseMsg(&job, &task, &databuf, 0); + + databuf.msgType = TDMT_SCH_LINK_BROKEN; + job.status = JOB_TASK_STATUS_FAIL; + schProcessResponseMsg(&job, &task, &databuf, 0); + + databuf.msgType = 0; + job.status = JOB_TASK_STATUS_FAIL; + schProcessResponseMsg(&job, &task, &databuf, 0); + + databuf.pData = taosMemoryMalloc(0); + schHandleHbCallback(NULL, &databuf, 0); + + __async_send_cb_fn_t fp = NULL; + schGetCallbackFp(TDMT_SCH_TASK_NOTIFY, &fp); + schGetCallbackFp(0, &fp); + + SQueryNodeEpId ep = {0}; + schBuildAndSendHbMsg(&ep, NULL); + + schBuildAndSendMsg(&job, &task, NULL, 0, NULL); + + SSchLevel level = {0}; + SSubplan subplan; + memset(&subplan, 0, sizeof(subplan)); + job.attr.queryJob = true; + schMgmt.cfg.schPolicy = SCH_ALL; + task.plan = &subplan; + schInitTaskRetryTimes(&job, &task, &level); + job.attr.queryJob = false; + memset(&schMgmt.cfg, 0, sizeof(schMgmt.cfg)); + memset(&level, 0, sizeof(level)); + + schRecordTaskSucceedNode(&job, &task); + + schDropTaskExecNode(&job, &task, NULL, 0); + + task.execNodes = taosHashInit(1, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); + schDropTaskExecNode(&job, &task, NULL, 0); + + int32_t execId = 0; + task.execId = 1; + (void)taosHashPut(task.execNodes, &execId, sizeof(execId), &execId, sizeof(execId)); + schDropTaskExecNode(&job, &task, NULL, execId); + task.execId = 0; + taosHashCleanup(task.execNodes); + task.execNodes = NULL; + + job.status = JOB_TASK_STATUS_FAIL; + schProcessOnTaskFailure(&job, &task, 0); + job.status = 0; + + task.status = JOB_TASK_STATUS_FAIL; + schProcessOnTaskFailure(&job, &task, 0); + task.status = 0; + + task.level = &level; + schProcessOnTaskFailure(&job, &task, TSDB_CODE_SCH_TIMEOUT_ERROR); + memset(&level, 0, sizeof(level)); + task.level = NULL; + + subplan.subplanType = SUBPLAN_TYPE_SCAN; + task.plan = &subplan; + SEpSet epset = {0}; + epset.numOfEps = 127; + schChkUpdateRedirectCtx(&job, &task, &epset, 0); + + schChkUpdateRedirectCtx(&job, &task, NULL, 0); + task.plan = NULL; + + schPushTaskToExecList(&job, &task); + + job.execTasks = taosHashInit(1, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_ENTRY_LOCK); + taosHashPut(job.execTasks, &task.taskId, sizeof(task.taskId), &task, POINTER_BYTES); + schPushTaskToExecList(&job, &task); + taosHashCleanup(job.execTasks); + job.execTasks = NULL; + + bool needRetry = false; + task.timeoutUsec = SCH_MAX_TASK_TIMEOUT_USEC / 2 + 1; + task.retryTimes = 0; + task.maxRetryTimes = 0; + schTaskCheckSetRetry(&job, &task, TSDB_CODE_SCH_TIMEOUT_ERROR, &needRetry); + + task.execId = 0; + task.retryTimes = 0; + task.maxRetryTimes = 100; + task.maxExecTimes = 1; + schTaskCheckSetRetry(&job, &task, TSDB_CODE_SCH_TIMEOUT_ERROR, &needRetry); + + + task.execId = 0; + task.retryTimes = 0; + task.maxRetryTimes = 100; + task.maxExecTimes = 100; + task.lastMsgType = TDMT_SCH_LINK_BROKEN; + schTaskCheckSetRetry(&job, &task, TSDB_CODE_SCH_TIMEOUT_ERROR, &needRetry); + + schSetAddrsFromNodeList(&job, &task); + + schSwitchTaskCandidateAddr(&job, &task); + + + task.candidateAddrs = taosArrayInit(SCHEDULE_DEFAULT_MAX_NODE_NUM, sizeof(SQueryNodeAddr)); + SQueryNodeAddr addr = {0}; + taosArrayPush(task.candidateAddrs, &addr); + taosArrayPush(task.candidateAddrs, &addr); + schMgmt.cfg.schPolicy = SCH_LOAD_SEQ; + task.candidateIdx = 1; + schSwitchTaskCandidateAddr(&job, &task); + + schMgmt.cfg.schPolicy = SCH_RANDOM; + schSwitchTaskCandidateAddr(&job, &task); + taosArrayDestroy(task.candidateAddrs); + task.candidateAddrs = NULL; + memset(&schMgmt.cfg, 0, sizeof(schMgmt.cfg)); + task.candidateIdx = 0; + + schDropTaskOnExecNode(&job, &task); + + schNotifyTaskOnExecNode(&job, &task, TASK_NOTIFY_FINISHED); + + schLaunchRemoteTask(&job, &task); + + SSchTaskCtx* pCtx = (SSchTaskCtx*)taosMemoryCalloc(1, sizeof(SSchTaskCtx)); + pCtx->jobRid = -1; + schLaunchTaskImpl((void*)pCtx); + + task.plan = &subplan; + subplan.subplanType = SUBPLAN_TYPE_SCAN; + job.attr.needFlowCtrl = true; + level.taskNum = 1000; + task.level = &level; + schLaunchTask(&job, &task); + task.plan = NULL; + task.level = NULL; + job.attr.needFlowCtrl = false; + + SSchTimerParam param = {0}; + param.rId = -1; + schHandleTimerEvent(¶m, NULL); + + job.execTasks = taosHashInit(1, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_ENTRY_LOCK); + task.delayExecMs = 1; + schMgmt.timer = NULL; + schDelayLaunchTask(&job, &task); + task.delayExecMs = 0; + taosHashCleanup(job.execTasks); + job.execTasks = NULL; + + job.fetchRes = (void*)0x1; + schLaunchFetchTask(&job); + job.fetchRes = NULL; + + job.fetchTask = &task; + job.attr.localExec = true; + job.attr.queryJob = true; + subplan.subplanType = SUBPLAN_TYPE_MERGE; + task.plan = &subplan; + void* p = taosMemoryCalloc(1, 1024); + schMgmt.queryMgmt = p; + schLaunchFetchTask(&job); + memset(&job, 0, sizeof(job)); + memset(&subplan, 0, sizeof(subplan)); + task.plan = NULL; + taosMemoryFreeClear(schMgmt.queryMgmt); + + // flow ctrl + + job.flowCtrl = taosHashInit(1, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); + SEp sep = {0}; + SSchFlowControl nctrl = {0}; + nctrl.taskList = taosArrayInit(1, POINTER_BYTES); + taosHashPut(job.flowCtrl, &sep, sizeof(SEp), &nctrl, sizeof(nctrl)); + schFreeFlowCtrl(&job); + + + + schMgmt.jobRef = -1; +} + + void schtReset() { insertJobRefId = 0; queryJobRefId = 0; diff --git a/source/libs/wal/inc/walInt.h b/source/libs/wal/inc/walInt.h index 1886541d62..14f6503941 100644 --- a/source/libs/wal/inc/walInt.h +++ b/source/libs/wal/inc/walInt.h @@ -157,7 +157,7 @@ int32_t walLoadMeta(SWal* pWal); int32_t walSaveMeta(SWal* pWal); int32_t walRemoveMeta(SWal* pWal); int32_t walRollFileInfo(SWal* pWal); - +int32_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx, int64_t* lastVer); int32_t walCheckAndRepairMeta(SWal* pWal); int32_t walCheckAndRepairIdx(SWal* pWal); diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index 3faeb53499..470a6b3f40 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -46,7 +46,7 @@ static FORCE_INLINE int walBuildTmpMetaName(SWal* pWal, char* buf) { return snprintf(buf, WAL_FILE_LEN, "%s/meta-ver.tmp", pWal->path); } -static FORCE_INLINE int32_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx, int64_t* lastVer) { +FORCE_INLINE int32_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx, int64_t* lastVer) { int32_t code = 0, lino = 0; int32_t sz = taosArrayGetSize(pWal->fileInfoSet); int64_t retVer = -1; diff --git a/source/libs/wal/test/walMetaTest.cpp b/source/libs/wal/test/walMetaTest.cpp index a958ad74e0..b2875bdca1 100644 --- a/source/libs/wal/test/walMetaTest.cpp +++ b/source/libs/wal/test/walMetaTest.cpp @@ -127,7 +127,7 @@ class WalRetentionEnv : public ::testing::Test { SWalCfg cfg; cfg.rollPeriod = -1; cfg.segSize = -1; - cfg.committed =-1; + cfg.committed = -1; cfg.retentionPeriod = -1; cfg.retentionSize = 0; cfg.rollPeriod = 0; @@ -146,6 +146,83 @@ class WalRetentionEnv : public ::testing::Test { const char* pathName = TD_TMP_DIR_PATH "wal_test"; }; +class WalSkipLevel : public ::testing::Test { + protected: + static void SetUpTestCase() { + int code = walInit(NULL); + ASSERT(code == 0); + } + + static void TearDownTestCase() { walCleanUp(); } + + void walResetEnv() { + TearDown(); + taosRemoveDir(pathName); + SetUp(); + } + + void SetUp() override { + SWalCfg cfg; + cfg.rollPeriod = -1; + cfg.segSize = -1; + cfg.committed = -1; + cfg.retentionPeriod = -1; + cfg.retentionSize = 0; + cfg.rollPeriod = 0; + cfg.vgId = 1; + cfg.level = TAOS_WAL_SKIP; + pWal = walOpen(pathName, &cfg); + ASSERT(pWal != NULL); + } + + void TearDown() override { + walClose(pWal); + pWal = NULL; + } + + SWal* pWal = NULL; + const char* pathName = TD_TMP_DIR_PATH "wal_test"; +}; + +class WalEncrypted : public ::testing::Test { + protected: + static void SetUpTestCase() { + int code = walInit(NULL); + ASSERT(code == 0); + } + + static void TearDownTestCase() { walCleanUp(); } + + void walResetEnv() { + TearDown(); + taosRemoveDir(pathName); + SetUp(); + } + + void SetUp() override { + SWalCfg cfg; + cfg.rollPeriod = -1; + cfg.segSize = -1; + cfg.committed = -1; + cfg.retentionPeriod = -1; + cfg.retentionSize = 0; + cfg.rollPeriod = 0; + cfg.vgId = 0; + cfg.level = TAOS_WAL_FSYNC; + cfg.encryptAlgorithm = 1; + pWal = walOpen(pathName, &cfg); + ASSERT(pWal != NULL); + } + + void TearDown() override { + walClose(pWal); + pWal = NULL; + } + + SWal* pWal = NULL; + const char* pathName = TD_TMP_DIR_PATH "wal_test"; +}; + TEST_F(WalCleanEnv, createNew) { walRollFileInfo(pWal); ASSERT(pWal->fileInfoSet != NULL); @@ -373,6 +450,183 @@ TEST_F(WalKeepEnv, readHandleRead) { walCloseReader(pRead); } +TEST_F(WalKeepEnv, walLogExist) { + walResetEnv(); + int code; + SWalReader* pRead = walOpenReader(pWal, NULL, 0); + ASSERT(pRead != NULL); + + int i; + for (i = 0; i < 100; i++) { + char newStr[100]; + sprintf(newStr, "%s-%d", ranStr, i); + int len = strlen(newStr); + code = walAppendLog(pWal, i, 0, syncMeta, newStr, len); + ASSERT_EQ(code, 0); + } + walLogExist(pWal, 0); + ASSERT_EQ(code, 0); + walCloseReader(pRead); +} + +TEST_F(WalKeepEnv, walScanLogGetLastVerHeadMissMatch) { + walResetEnv(); + int code; + do { + char newStr[100]; + sprintf(newStr, "%s-%d", ranStr, 0); + sprintf(newStr, "%s-%d", ranStr, 0); + int len = strlen(newStr); + code = walAppendLog(pWal, 0, 0, syncMeta, newStr, len); + } while (0); + + int i = 0; + char newStr[100]; + sprintf(newStr, "%s-%d", ranStr, i); + int len = strlen(newStr); + int64_t offset = walGetCurFileOffset(pWal); + SWalFileInfo* pFileInfo = walGetCurFileInfo(pWal); + + pWal->writeHead.head.version = i; + pWal->writeHead.head.bodyLen = len; + pWal->writeHead.head.msgType = 0; + pWal->writeHead.head.ingestTs = taosGetTimestampUs(); + + pWal->writeHead.head.syncMeta = syncMeta; + + pWal->writeHead.cksumHead = 1; + pWal->writeHead.cksumBody = walCalcBodyCksum(newStr, len); + taosWriteFile(pWal->pLogFile, &pWal->writeHead, sizeof(SWalCkHead)); + taosWriteFile(pWal->pLogFile, newStr, len); + + int64_t lastVer = 0; + code = walScanLogGetLastVer(pWal, 0, &lastVer); + ASSERT_EQ(code, TSDB_CODE_WAL_CHKSUM_MISMATCH); +} + +TEST_F(WalKeepEnv, walScanLogGetLastVerBodyMissMatch) { + walResetEnv(); + int code; + do { + char newStr[100]; + sprintf(newStr, "%s-%d", ranStr, 0); + sprintf(newStr, "%s-%d", ranStr, 0); + int len = strlen(newStr); + code = walAppendLog(pWal, 0, 0, syncMeta, newStr, len); + } while (0); + + int i = 0; + char newStr[100]; + sprintf(newStr, "%s-%d", ranStr, i); + int len = strlen(newStr); + int64_t offset = walGetCurFileOffset(pWal); + SWalFileInfo* pFileInfo = walGetCurFileInfo(pWal); + + pWal->writeHead.head.version = i; + pWal->writeHead.head.bodyLen = len; + pWal->writeHead.head.msgType = 0; + pWal->writeHead.head.ingestTs = taosGetTimestampUs(); + + pWal->writeHead.head.syncMeta = syncMeta; + + pWal->writeHead.cksumHead = walCalcHeadCksum(&pWal->writeHead); + pWal->writeHead.cksumBody = 1; + taosWriteFile(pWal->pLogFile, &pWal->writeHead, sizeof(SWalCkHead)); + taosWriteFile(pWal->pLogFile, newStr, len); + + int64_t lastVer = 0; + code = walScanLogGetLastVer(pWal, 0, &lastVer); + ASSERT_EQ(code, TSDB_CODE_WAL_CHKSUM_MISMATCH); +} + +TEST_F(WalKeepEnv, walCheckAndRepairIdxFile) { + walResetEnv(); + int code; + do { + char newStr[100]; + sprintf(newStr, "%s-%d", ranStr, 0); + sprintf(newStr, "%s-%d", ranStr, 0); + int len = strlen(newStr); + code = walAppendLog(pWal, 0, 0, syncMeta, newStr, len); + } while (0); + SWalFileInfo* pFileInfo = walGetCurFileInfo(pWal); + for (int i = 1; i < 100; i++) { + char newStr[100]; + sprintf(newStr, "%s-%d", ranStr, i); + int len = strlen(newStr); + pWal->writeHead.head.version = i; + pWal->writeHead.head.bodyLen = len; + pWal->writeHead.head.msgType = 0; + pWal->writeHead.head.ingestTs = taosGetTimestampUs(); + pWal->writeHead.head.syncMeta = syncMeta; + pWal->writeHead.cksumHead = walCalcHeadCksum(&pWal->writeHead); + pWal->writeHead.cksumBody = walCalcBodyCksum(newStr, len); + taosWriteFile(pWal->pLogFile, &pWal->writeHead, sizeof(SWalCkHead)); + taosWriteFile(pWal->pLogFile, newStr, len); + } + pWal->vers.lastVer = 99; + pFileInfo->lastVer = 99; + code = walCheckAndRepairIdx(pWal); + ASSERT_EQ(code, 0); +} + +TEST_F(WalKeepEnv, walRestoreFromSnapshot1) { + walResetEnv(); + int code; + + int i; + for (i = 0; i < 100; i++) { + char newStr[100]; + sprintf(newStr, "%s-%d", ranStr, i); + int len = strlen(newStr); + code = walAppendLog(pWal, i, 0, syncMeta, newStr, len); + ASSERT_EQ(code, 0); + } + code = walRestoreFromSnapshot(pWal, 50); + ASSERT_EQ(code, 0); +} + +TEST_F(WalKeepEnv, walRestoreFromSnapshot2) { + walResetEnv(); + int code; + + int i; + for (i = 0; i < 100; i++) { + char newStr[100]; + sprintf(newStr, "%s-%d", ranStr, i); + int len = strlen(newStr); + code = walAppendLog(pWal, i, 0, syncMeta, newStr, len); + ASSERT_EQ(code, 0); + } + SWalRef* ref = walOpenRef(pWal); + ref->refVer = 10; + code = walRestoreFromSnapshot(pWal, 99); + ASSERT_EQ(code, -1); +} + +TEST_F(WalKeepEnv, walRollback) { + walResetEnv(); + int code; + + int i; + for (i = 0; i < 100; i++) { + char newStr[100]; + sprintf(newStr, "%s-%d", ranStr, i); + int len = strlen(newStr); + code = walAppendLog(pWal, i, 0, syncMeta, newStr, len); + ASSERT_EQ(code, 0); + } + code = walRollback(pWal, -1); + ASSERT_EQ(code, TSDB_CODE_WAL_INVALID_VER); + pWal->vers.lastVer = 50; + pWal->vers.commitVer = 40; + pWal->vers.snapshotVer = 40; + SWalFileInfo* fileInfo = walGetCurFileInfo(pWal); + + code = walRollback(pWal, 48); + ASSERT_EQ(code, 0); +} + TEST_F(WalRetentionEnv, repairMeta1) { walResetEnv(); int code; @@ -456,44 +710,6 @@ TEST_F(WalRetentionEnv, repairMeta1) { walCloseReader(pRead); } -class WalSkipLevel : public ::testing::Test { - protected: - static void SetUpTestCase() { - int code = walInit(NULL); - ASSERT(code == 0); - } - - static void TearDownTestCase() { walCleanUp(); } - - void walResetEnv() { - TearDown(); - taosRemoveDir(pathName); - SetUp(); - } - - void SetUp() override { - SWalCfg cfg; - cfg.rollPeriod = -1; - cfg.segSize = -1; - cfg.committed =-1; - cfg.retentionPeriod = -1; - cfg.retentionSize = 0; - cfg.rollPeriod = 0; - cfg.vgId = 1; - cfg.level = TAOS_WAL_SKIP; - pWal = walOpen(pathName, &cfg); - ASSERT(pWal != NULL); - } - - void TearDown() override { - walClose(pWal); - pWal = NULL; - } - - SWal* pWal = NULL; - const char* pathName = TD_TMP_DIR_PATH "wal_test"; -}; - TEST_F(WalSkipLevel, restart) { walResetEnv(); int code; @@ -533,4 +749,15 @@ TEST_F(WalSkipLevel, roll) { ASSERT_EQ(code, 0); code = walEndSnapshot(pWal); ASSERT_EQ(code, 0); +} + +TEST_F(WalEncrypted, write) { + int code; + for (int i = 0; i < 100; i++) { + code = walAppendLog(pWal, i, i + 1, syncMeta, (void*)ranStr, ranStrLen); + ASSERT_EQ(code, 0); + ASSERT_EQ(pWal->vers.lastVer, i); + } + code = walSaveMeta(pWal); + ASSERT_EQ(code, 0); } \ No newline at end of file diff --git a/source/os/src/osTime.c b/source/os/src/osTime.c index 75cb2b91a2..3bab3e7e25 100644 --- a/source/os/src/osTime.c +++ b/source/os/src/osTime.c @@ -454,7 +454,8 @@ struct tm *taosGmTimeR(const time_t *timep, struct tm *result) { return NULL; } #ifdef WINDOWS - return gmtime_s(result, timep); + errno_t code = gmtime_s(result, timep); + return (code == 0) ? result : NULL; #else return gmtime_r(timep, result); #endif diff --git a/source/os/test/osTimeTests.cpp b/source/os/test/osTimeTests.cpp index 1d34587ad8..d1ce654a76 100644 --- a/source/os/test/osTimeTests.cpp +++ b/source/os/test/osTimeTests.cpp @@ -94,6 +94,35 @@ TEST(osTimeTests, taosLocalTime) { #endif } +TEST(osTimeTests, taosGmTimeR) { + // Test 1: Test when both timep and result are not NULL + time_t timep = 1617531000; // 2021-04-04 18:10:00 + struct tm tmInfo; + ASSERT_NE(taosGmTimeR(&timep, &tmInfo), nullptr); + + char buf[128]; + taosStrfTime(buf, sizeof(buf), "%Y-%m-%dT%H:%M:%S", &tmInfo); + ASSERT_STREQ(buf, "2021-04-04T10:10:00"); +} + +TEST(osTimeTests, taosTimeGm) { + char *timestr= "2021-04-04T18:10:00"; + struct tm tm = {0}; + + taosStrpTime(timestr, "%Y-%m-%dT%H:%M:%S", &tm); + int64_t seconds = taosTimeGm(&tm); + ASSERT_EQ(seconds, 1617559800); +} + +TEST(osTimeTests, taosMktime) { + char *timestr= "2021-04-04T18:10:00"; + struct tm tm = {0}; + + taosStrpTime(timestr, "%Y-%m-%dT%H:%M:%S", &tm); + time_t seconds = taosMktime(&tm, NULL); + ASSERT_EQ(seconds, 1617531000); +} + TEST(osTimeTests, invalidParameter) { void *retp = NULL; int32_t reti = 0; diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index 547cdb6cdf..9ca53c8202 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -519,7 +519,7 @@ int32_t cfgSetItemVal(SConfigItem *pItem, const char *name, const char *value, E int32_t code = TSDB_CODE_SUCCESS; if (pItem == NULL) { - TAOS_RETURN(TSDB_CODE_INVALID_CFG); + TAOS_RETURN(TSDB_CODE_CFG_NOT_FOUND); } switch (pItem->dtype) { case CFG_DTYPE_BOOL: { @@ -629,6 +629,7 @@ int32_t cfgCheckRangeForDynUpdate(SConfig *pCfg, const char *name, const char *p cfgUnLock(pCfg); TAOS_RETURN(code); } + if ((pItem->category == CFG_CATEGORY_GLOBAL) && alterType == CFG_ALTER_DNODE) { uError("failed to config:%s, not support update global config on only one dnode", name); cfgUnLock(pCfg); diff --git a/source/util/test/cfgTest.cpp b/source/util/test/cfgTest.cpp index a5812d375b..74c34f5c91 100644 --- a/source/util/test/cfgTest.cpp +++ b/source/util/test/cfgTest.cpp @@ -12,6 +12,10 @@ #include #include "tconfig.h" +#ifndef WINDOWS +#include "osFile.h" +#endif + class CfgTest : public ::testing::Test { protected: static void SetUpTestSuite() {} @@ -35,6 +39,9 @@ TEST_F(CfgTest, 01_Str) { EXPECT_STREQ(cfgStypeStr(CFG_STYPE_ENV_CMD), "env_cmd"); EXPECT_STREQ(cfgStypeStr(CFG_STYPE_APOLLO_URL), "apollo_url"); EXPECT_STREQ(cfgStypeStr(CFG_STYPE_ARG_LIST), "arg_list"); + EXPECT_STREQ(cfgStypeStr(CFG_STYPE_TAOS_OPTIONS), "taos_options"); + EXPECT_STREQ(cfgStypeStr(CFG_STYPE_ALTER_CLIENT_CMD), "alter_client_cmd"); + EXPECT_STREQ(cfgStypeStr(CFG_STYPE_ALTER_SERVER_CMD), "alter_server_cmd"); EXPECT_STREQ(cfgStypeStr(ECfgSrcType(1024)), "invalid"); EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_NONE), "none"); @@ -47,6 +54,10 @@ TEST_F(CfgTest, 01_Str) { EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_DIR), "dir"); EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_DIR), "dir"); EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_DIR), "dir"); + EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_DOUBLE), "double"); + EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_LOCALE), "locale"); + EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_CHARSET), "charset"); + EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_TIMEZONE), "timezone"); EXPECT_STREQ(cfgDtypeStr(ECfgDataType(1024)), "invalid"); } @@ -57,24 +68,30 @@ TEST_F(CfgTest, 02_Basic) { ASSERT_EQ(code, TSDB_CODE_SUCCESS); ASSERT_NE(pConfig, nullptr); - EXPECT_EQ(cfgAddBool(pConfig, "test_bool", 0, 0, 0, 0), 0); - EXPECT_EQ(cfgAddInt32(pConfig, "test_int32", 1, 0, 16, 0, 0, 0), 0); - EXPECT_EQ(cfgAddInt64(pConfig, "test_int64", 2, 0, 16, 0, 0, 0), 0); - EXPECT_EQ(cfgAddFloat(pConfig, "test_float", 3, 0, 16, 0, 0, 0), 0); - EXPECT_EQ(cfgAddString(pConfig, "test_string", "4", 0, 0, 0), 0); - EXPECT_EQ(cfgAddDir(pConfig, "test_dir", TD_TMP_DIR_PATH, 0, 0, 0), 0); + EXPECT_EQ(cfgAddBool(pConfig, "test_bool", 0, 0, 6, 0), 0); + + EXPECT_EQ(cfgAddInt32(pConfig, "test_int32", 21, 0, 16, 0, 1, 0), TSDB_CODE_OUT_OF_RANGE); + EXPECT_EQ(cfgAddInt32(pConfig, "test_int32", 1, 0, 16, 0, 1, 0), 0); + + EXPECT_EQ(cfgAddInt64(pConfig, "test_int64", 21, 0, 16, 0, 2, 0), TSDB_CODE_OUT_OF_RANGE); + EXPECT_EQ(cfgAddInt64(pConfig, "test_int64", 2, 0, 16, 0, 2, 0), 0); + + EXPECT_EQ(cfgAddFloat(pConfig, "test_float", 21, 0, 16, 0, 6, 0), TSDB_CODE_OUT_OF_RANGE); + EXPECT_EQ(cfgAddFloat(pConfig, "test_float", 3, 0, 16, 0, 6, 0), 0); + EXPECT_EQ(cfgAddString(pConfig, "test_string", "4", 0, 6, 0), 0); + EXPECT_EQ(cfgAddDir(pConfig, "test_dir", TD_TMP_DIR_PATH, 0, 6, 0), 0); EXPECT_EQ(cfgGetSize(pConfig), 6); int32_t size = cfgGetSize(pConfig); - SConfigItem* pItem = NULL; + SConfigItem *pItem = NULL; SConfigIter *pIter = NULL; code = cfgCreateIter(pConfig, &pIter); ASSERT_EQ(code, TSDB_CODE_SUCCESS); ASSERT_NE(pIter, nullptr); - while((pItem = cfgNextIter(pIter)) != NULL) { + while ((pItem = cfgNextIter(pIter)) != NULL) { switch (pItem->dtype) { case CFG_DTYPE_BOOL: printf("index:%d, cfg:%s value:%d\n", size, pItem->name, pItem->bval); @@ -115,12 +132,16 @@ TEST_F(CfgTest, 02_Basic) { EXPECT_EQ(pItem->dtype, CFG_DTYPE_INT32); EXPECT_STREQ(pItem->name, "test_int32"); EXPECT_EQ(pItem->i32, 1); + code = cfgSetItem(pConfig, "test_int32", "21", CFG_STYPE_DEFAULT, true); + ASSERT_EQ(code, TSDB_CODE_OUT_OF_RANGE); pItem = cfgGetItem(pConfig, "test_int64"); EXPECT_EQ(pItem->stype, CFG_STYPE_DEFAULT); EXPECT_EQ(pItem->dtype, CFG_DTYPE_INT64); EXPECT_STREQ(pItem->name, "test_int64"); EXPECT_EQ(pItem->i64, 2); + code = cfgSetItem(pConfig, "test_int64", "21", CFG_STYPE_DEFAULT, true); + ASSERT_EQ(code, TSDB_CODE_OUT_OF_RANGE); pItem = cfgGetItem(pConfig, "test_float"); EXPECT_EQ(pItem->stype, CFG_STYPE_DEFAULT); @@ -140,5 +161,213 @@ TEST_F(CfgTest, 02_Basic) { EXPECT_STREQ(pItem->name, "test_dir"); EXPECT_STREQ(pItem->str, TD_TMP_DIR_PATH); + code = cfgGetAndSetItem(pConfig, &pItem, "err_cfg", "err_val", CFG_STYPE_DEFAULT, true); + ASSERT_EQ(code, TSDB_CODE_CFG_NOT_FOUND); + + code = cfgCheckRangeForDynUpdate(pConfig, "test_int32", "4", false, CFG_ALTER_LOCAL); + ASSERT_EQ(code, TSDB_CODE_INVALID_CFG); + + code = cfgCheckRangeForDynUpdate(pConfig, "test_int64", "4", true, CFG_ALTER_LOCAL); + ASSERT_EQ(code, TSDB_CODE_INVALID_CFG); + + code = cfgCheckRangeForDynUpdate(pConfig, "test_bool", "3", false, CFG_ALTER_LOCAL); + ASSERT_EQ(code, TSDB_CODE_OUT_OF_RANGE); + + code = cfgCheckRangeForDynUpdate(pConfig, "test_int32", "74", true, CFG_ALTER_LOCAL); + ASSERT_EQ(code, TSDB_CODE_OUT_OF_RANGE); + + code = cfgCheckRangeForDynUpdate(pConfig, "test_int64", "74", false, CFG_ALTER_LOCAL); + ASSERT_EQ(code, TSDB_CODE_OUT_OF_RANGE); + + code = cfgCheckRangeForDynUpdate(pConfig, "test_float", "74", false, CFG_ALTER_LOCAL); + ASSERT_EQ(code, TSDB_CODE_OUT_OF_RANGE); + cfgCleanup(pConfig); } + +TEST_F(CfgTest, initWithArray) { + SConfig *pConfig = NULL; + int32_t code = cfgInit(&pConfig); + + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + ASSERT_NE(pConfig, nullptr); + + EXPECT_EQ(cfgAddBool(pConfig, "test_bool", 0, 0, 0, 0), 0); + EXPECT_EQ(cfgAddInt32(pConfig, "test_int32", 1, 0, 16, 0, 0, 0), 0); + EXPECT_EQ(cfgAddInt64(pConfig, "test_int64", 2, 0, 16, 0, 0, 0), 0); + EXPECT_EQ(cfgAddFloat(pConfig, "test_float", 3, 0, 16, 0, 0, 0), 0); + EXPECT_EQ(cfgAddString(pConfig, "test_string", "4", 0, 0, 0), 0); + EXPECT_EQ(cfgAddDir(pConfig, "test_dir", TD_TMP_DIR_PATH, 0, 0, 0), 0); + + SArray *pArgs = taosArrayInit(6, sizeof(SConfigPair)); + SConfigPair *pPair = (SConfigPair *)taosMemoryMalloc(sizeof(SConfigPair)); + pPair->name = "test_bool"; + pPair->value = "1"; + taosArrayPush(pArgs, pPair); + SConfigPair *pPair1 = (SConfigPair *)taosMemoryMalloc(sizeof(SConfigPair)); + pPair1->name = "test_int32"; + pPair1->value = "2"; + taosArrayPush(pArgs, pPair1); + SConfigPair *pPair2 = (SConfigPair *)taosMemoryMalloc(sizeof(SConfigPair)); + pPair2->name = "test_int64"; + pPair2->value = "3"; + taosArrayPush(pArgs, pPair2); + SConfigPair *pPair3 = (SConfigPair *)taosMemoryMalloc(sizeof(SConfigPair)); + pPair3->name = "test_float"; + pPair3->value = "4"; + taosArrayPush(pArgs, pPair3); + SConfigPair *pPair4 = (SConfigPair *)taosMemoryMalloc(sizeof(SConfigPair)); + pPair4->name = "test_string"; + pPair4->value = "5"; + taosArrayPush(pArgs, pPair4); + SConfigPair *pPair5 = (SConfigPair *)taosMemoryMalloc(sizeof(SConfigPair)); + pPair5->name = "test_dir"; + pPair5->value = TD_TMP_DIR_PATH; + taosArrayPush(pArgs, pPair5); + code = cfgLoadFromArray(pConfig, pArgs); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); +} + +TEST_F(CfgTest, cfgDumpItemCategory) { + SConfig *pConfig = NULL; + int32_t code = cfgInit(&pConfig); + + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + ASSERT_NE(pConfig, nullptr); + + EXPECT_EQ(cfgAddBool(pConfig, "test_bool", 0, 0, 6, 100), 0); + + SConfigItem *pItem = NULL; + pItem = cfgGetItem(pConfig, "test_bool"); + EXPECT_EQ(pItem->stype, CFG_STYPE_DEFAULT); + EXPECT_EQ(pItem->dtype, CFG_DTYPE_BOOL); + EXPECT_STREQ(pItem->name, "test_bool"); + EXPECT_EQ(pItem->bval, 0); + + EXPECT_EQ(cfgDumpItemCategory(pItem, NULL, 0, 0), TSDB_CODE_INVALID_CFG); +} + +TEST_F(CfgTest, cfgDumpCfgS3) { + SConfig *pConfig = NULL; + int32_t code = cfgInit(&pConfig); + + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + ASSERT_NE(pConfig, nullptr); + + cfgAddInt32(pConfig, "s3MigrateIntervalSec", 60 * 60, 600, 100000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER, + CFG_CATEGORY_GLOBAL); + cfgAddBool(pConfig, "s3MigrateEnabled", 60 * 60, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER, CFG_CATEGORY_GLOBAL); + cfgAddString(pConfig, "s3Accesskey", "", CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER_LAZY, CFG_CATEGORY_GLOBAL); + cfgAddString(pConfig, "s3Endpoint", "", CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER_LAZY, CFG_CATEGORY_GLOBAL); + cfgAddString(pConfig, "s3BucketName", "", CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER_LAZY, CFG_CATEGORY_GLOBAL); + cfgAddInt32(pConfig, "s3PageCacheSize", 10, 4, 1024 * 1024 * 1024, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER_LAZY, + CFG_CATEGORY_GLOBAL); + cfgAddInt32(pConfig, "s3UploadDelaySec", 10, 1, 60 * 60 * 24 * 30, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER, + CFG_CATEGORY_GLOBAL); + cfgAddDir(pConfig, "scriptDir", configDir, CFG_SCOPE_BOTH, CFG_DYN_NONE, CFG_CATEGORY_LOCAL); + + cfgDumpCfgS3(pConfig, false, false); + + cfgDumpCfgS3(pConfig, true, true); + + cfgDumpCfgS3(pConfig, false, true); + + cfgDumpCfgS3(pConfig, true, false); +} + +#ifndef WINDOWS +TEST_F(CfgTest, cfgLoadFromEnvVar) { + SConfig *pConfig = NULL; + int32_t code = cfgInit(&pConfig); + + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + ASSERT_NE(pConfig, nullptr); + + EXPECT_EQ(cfgAddBool(pConfig, "test_bool", 0, 0, 6, 0), 0); + + EXPECT_EQ(cfgAddInt32(pConfig, "test_int32", 21, 0, 16, 0, 1, 0), TSDB_CODE_OUT_OF_RANGE); + EXPECT_EQ(cfgAddInt32(pConfig, "test_int32", 1, 0, 16, 0, 1, 0), 0); + + EXPECT_EQ(cfgAddInt64(pConfig, "test_int64", 21, 0, 16, 0, 2, 0), TSDB_CODE_OUT_OF_RANGE); + EXPECT_EQ(cfgAddInt64(pConfig, "test_int64", 2, 0, 16, 0, 2, 0), 0); + + EXPECT_EQ(cfgAddFloat(pConfig, "test_float", 21, 0, 16, 0, 6, 0), TSDB_CODE_OUT_OF_RANGE); + EXPECT_EQ(cfgAddFloat(pConfig, "test_float", 3, 0, 16, 0, 6, 0), 0); + EXPECT_EQ(cfgAddString(pConfig, "test_string", "4", 0, 6, 0), 0); + EXPECT_EQ(cfgAddDir(pConfig, "test_dir", TD_TMP_DIR_PATH, 0, 6, 0), 0); + + setenv("test_bool", "1", 1); + setenv("test_int32", "2", 1); + setenv("test_int64", "3", 1); + setenv("test_float", "4", 1); + setenv("test_string", "5", 1); + setenv("test_dir", TD_TMP_DIR_PATH, 1); + + ASSERT_EQ(cfgLoad(pConfig, CFG_STYPE_ENV_VAR, "test_bool"), TSDB_CODE_SUCCESS); +} + +TEST_F(CfgTest, cfgLoadFromEnvCmd) { + SConfig *pConfig = NULL; + int32_t code = cfgInit(&pConfig); + + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + ASSERT_NE(pConfig, nullptr); + + EXPECT_EQ(cfgAddBool(pConfig, "test_bool", 0, 0, 6, 0), 0); + + EXPECT_EQ(cfgAddInt32(pConfig, "test_int32", 21, 0, 16, 0, 1, 0), TSDB_CODE_OUT_OF_RANGE); + EXPECT_EQ(cfgAddInt32(pConfig, "test_int32", 1, 0, 16, 0, 1, 0), 0); + + EXPECT_EQ(cfgAddInt64(pConfig, "test_int64", 21, 0, 16, 0, 2, 0), TSDB_CODE_OUT_OF_RANGE); + EXPECT_EQ(cfgAddInt64(pConfig, "test_int64", 2, 0, 16, 0, 2, 0), 0); + + EXPECT_EQ(cfgAddFloat(pConfig, "test_float", 21, 0, 16, 0, 6, 0), TSDB_CODE_OUT_OF_RANGE); + EXPECT_EQ(cfgAddFloat(pConfig, "test_float", 3, 0, 16, 0, 6, 0), 0); + EXPECT_EQ(cfgAddString(pConfig, "test_string", "4", 0, 6, 0), 0); + + const char *envCmd[] = {"test_bool=1", "test_int32=2", "test_int64=3", "test_float=4", "test_string=5", NULL}; + + ASSERT_EQ(cfgLoad(pConfig, CFG_STYPE_ENV_CMD, envCmd), TSDB_CODE_SUCCESS); +} + +TEST_F(CfgTest, cfgLoadFromEnvFile) { + SConfig *pConfig = NULL; + int32_t code = cfgInit(&pConfig); + + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + ASSERT_NE(pConfig, nullptr); + + TdFilePtr envFile = NULL; + const char *envFilePath = TD_TMP_DIR_PATH "envFile"; + envFile = taosOpenFile(envFilePath, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND); + const char *buf = "test_bool=1\ntest_int32=2\ntest_int64=3\ntest_float=4\ntest_string=5\n"; + taosWriteFile(envFile, buf, strlen(buf)); + taosCloseFile(&envFile); + ASSERT_EQ(cfgLoad(pConfig, CFG_STYPE_ENV_FILE, envFilePath), TSDB_CODE_SUCCESS); + + taosRemoveFile(envFilePath); +} + +TEST_F(CfgTest, cfgLoadFromApollUrl) { + SConfig *pConfig = NULL; + int32_t code = cfgInit(&pConfig); + + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + ASSERT_NE(pConfig, nullptr); + + TdFilePtr jsonFile = NULL; + const char *jsonFilePath = TD_TMP_DIR_PATH "envJson.json"; + jsonFile = taosOpenFile(jsonFilePath, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND); + const char *buf = + "{\"test_bool\":\"1\",\"test_int32\":\"2\",\"test_int64\":\"3\",\"test_float\":\"4\",\"test_string\":\"5\"}"; + taosWriteFile(jsonFile, buf, strlen(buf)); + taosCloseFile(&jsonFile); + + char str[256]; + snprintf(str, sizeof(str), "jsonFile:%s", jsonFilePath); + ASSERT_EQ(cfgLoad(pConfig, CFG_STYPE_APOLLO_URL, str), 0); + + taosRemoveFile(jsonFilePath); +} + +#endif \ No newline at end of file diff --git a/source/util/test/memPoolTest.cpp b/source/util/test/memPoolTest.cpp index d0c08b1318..a8da96711d 100644 --- a/source/util/test/memPoolTest.cpp +++ b/source/util/test/memPoolTest.cpp @@ -2048,7 +2048,252 @@ TEST(DisablePoolFuncTest, MultiThreadTest) { } #endif +#if 1 +TEST(functionsTest, internalFunc) { + char* caseName = "functionsTest:internalFunc"; + int32_t code = 0; + int64_t msize = 10; + void* pSession = NULL; + void* pJob = NULL; + + mptInitPool(); + + memset(mptCtx.jobCtxs, 0, sizeof(*mptCtx.jobCtxs)); + + assert(0 == taosMemPoolCallocJob(0, 0, (void**)&pJob)); + assert(0 == taosMemPoolInitSession(gMemPoolHandle, &pSession, pJob, "id")); + + int32_t loopTimes = 1; + int64_t st = 0; + void **addrList = (void**)taosMemCalloc(loopTimes, POINTER_BYTES); + + + // MALLOC + + for (int32_t i = 0; i < loopTimes; ++i) { + addrList[i] = (char*)mptMemoryMalloc(msize); + } + mptFreeAddrList(addrList, loopTimes); + + + + tsMemPoolFullFunc = 0; + mptEnableMemoryPoolUsage(gMemPoolHandle, pSession); + for (int32_t i = 0; i < loopTimes; ++i) { + addrList[i] = (char*)mptMemoryMalloc(msize); + } + mptDisableMemoryPoolUsage(); + mptFreeAddrList(addrList, loopTimes); + + + tsMemPoolFullFunc = 1; + mptEnableMemoryPoolUsage(gMemPoolHandle, pSession); + for (int32_t i = 0; i < loopTimes; ++i) { + addrList[i] = (char*)mptMemoryMalloc(msize); + } + mptDisableMemoryPoolUsage(); + mptFreeAddrList(addrList, loopTimes); + + + // CALLOC + + tsMemPoolFullFunc = 0; + mptEnableMemoryPoolUsage(gMemPoolHandle, pSession); + for (int32_t i = 0; i < loopTimes; ++i) { + addrList[i] = (char*)mptMemoryCalloc(1, msize); + } + mptDisableMemoryPoolUsage(); + mptFreeAddrList(addrList, loopTimes); + + + tsMemPoolFullFunc = 1; + mptEnableMemoryPoolUsage(gMemPoolHandle, pSession); + for (int32_t i = 0; i < loopTimes; ++i) { + addrList[i] = (char*)mptMemoryCalloc(1, msize); + } + mptDisableMemoryPoolUsage(); + mptFreeAddrList(addrList, loopTimes); + + + for (int32_t i = 0; i < loopTimes; ++i) { + addrList[i] = (char*)mptMemoryCalloc(1, msize); + } + //mptFreeAddrList(addrList, loopTimes); NO FREE FOR REALLOC + + // REALLOC + + tsMemPoolFullFunc = 0; + mptEnableMemoryPoolUsage(gMemPoolHandle, pSession); + for (int32_t i = 0; i < loopTimes; ++i) { + addrList[i] = (char*)mptMemoryRealloc(addrList[i], msize); + } + mptDisableMemoryPoolUsage(); + + + tsMemPoolFullFunc = 1; + mptEnableMemoryPoolUsage(gMemPoolHandle, pSession); + for (int32_t i = 0; i < loopTimes; ++i) { + addrList[i] = (char*)mptMemoryRealloc(addrList[i], msize); + } + mptDisableMemoryPoolUsage(); + + + for (int32_t i = 0; i < loopTimes; ++i) { + addrList[i] = (char*)mptMemoryRealloc(addrList[i], msize); + } + mptFreeAddrList(addrList, loopTimes); + + + // STRDUP + + tsMemPoolFullFunc = 0; + mptEnableMemoryPoolUsage(gMemPoolHandle, pSession); + for (int32_t i = 0; i < loopTimes; ++i) { + addrList[i] = (char*)mptStrdup("abc"); + } + mptDisableMemoryPoolUsage(); + mptFreeAddrList(addrList, loopTimes); + + + tsMemPoolFullFunc = 1; + mptEnableMemoryPoolUsage(gMemPoolHandle, pSession); + for (int32_t i = 0; i < loopTimes; ++i) { + addrList[i] = (char*)mptStrdup("abc"); + } + mptDisableMemoryPoolUsage(); + mptFreeAddrList(addrList, loopTimes); + + + for (int32_t i = 0; i < loopTimes; ++i) { + addrList[i] = (char*)mptStrdup("abc"); + } + mptFreeAddrList(addrList, loopTimes); + + // STRNDUP + + tsMemPoolFullFunc = 0; + mptEnableMemoryPoolUsage(gMemPoolHandle, pSession); + for (int32_t i = 0; i < loopTimes; ++i) { + addrList[i] = (char*)mptStrndup("abc", 3); + } + mptDisableMemoryPoolUsage(); + mptFreeAddrList(addrList, loopTimes); + + + tsMemPoolFullFunc = 1; + mptEnableMemoryPoolUsage(gMemPoolHandle, pSession); + for (int32_t i = 0; i < loopTimes; ++i) { + addrList[i] = (char*)mptStrndup("abc", 3); + } + mptDisableMemoryPoolUsage(); + mptFreeAddrList(addrList, loopTimes); + + + for (int32_t i = 0; i < loopTimes; ++i) { + addrList[i] = (char*)mptStrndup("abc", 3); + } + mptFreeAddrList(addrList, loopTimes); + + // ALIGNALLOC + + tsMemPoolFullFunc = 0; + mptEnableMemoryPoolUsage(gMemPoolHandle, pSession); + for (int32_t i = 0; i < loopTimes; ++i) { + addrList[i] = (char*)mptMemoryMallocAlign(8, msize); + } + mptDisableMemoryPoolUsage(); + mptFreeAddrList(addrList, loopTimes); + + + tsMemPoolFullFunc = 1; + mptEnableMemoryPoolUsage(gMemPoolHandle, pSession); + for (int32_t i = 0; i < loopTimes; ++i) { + addrList[i] = (char*)mptMemoryMallocAlign(8, msize); + } + mptDisableMemoryPoolUsage(); + mptFreeAddrList(addrList, loopTimes); + + + for (int32_t i = 0; i < loopTimes; ++i) { + addrList[i] = (char*)mptMemoryMallocAlign(8, msize); + } + //mptFreeAddrList(addrList, loopTimes); NO FREE FOR GETSIZE + + + // GETSIZE + + tsMemPoolFullFunc = 0; + mptEnableMemoryPoolUsage(gMemPoolHandle, pSession); + for (int32_t i = 0; i < loopTimes; ++i) { + mptMemorySize(addrList[i]); + } + mptDisableMemoryPoolUsage(); + + + tsMemPoolFullFunc = 1; + mptEnableMemoryPoolUsage(gMemPoolHandle, pSession); + for (int32_t i = 0; i < loopTimes; ++i) { + mptMemorySize(addrList[i]); + } + mptDisableMemoryPoolUsage(); + + + for (int32_t i = 0; i < loopTimes; ++i) { + mptMemorySize(addrList[i]); + } + + // FREE + + tsMemPoolFullFunc = 0; + mptEnableMemoryPoolUsage(gMemPoolHandle, pSession); + for (int32_t i = 0; i < loopTimes; ++i) { + mptMemoryFree(addrList[i]); + } + mptDisableMemoryPoolUsage(); + + + for (int32_t i = 0; i < loopTimes; ++i) { + addrList[i] = (char*)mptMemoryMalloc(msize); + } + tsMemPoolFullFunc = 1; + mptEnableMemoryPoolUsage(gMemPoolHandle, pSession); + for (int32_t i = 0; i < loopTimes; ++i) { + mptMemoryFree(addrList[i]); + } + mptDisableMemoryPoolUsage(); + + + for (int32_t i = 0; i < loopTimes; ++i) { + addrList[i] = (char*)mptMemoryMalloc(msize); + } + for (int32_t i = 0; i < loopTimes; ++i) { + mptMemoryFree(addrList[i]); + } + + // TRIM + + bool trimed = false; + tsMemPoolFullFunc = 0; + mptEnableMemoryPoolUsage(gMemPoolHandle, pSession); + for (int32_t i = 0; i < loopTimes; ++i) { + mptMemoryTrim(0, NULL); + mptMemoryTrim(0, &trimed); + } + mptDisableMemoryPoolUsage(); + + + tsMemPoolFullFunc = 1; + mptEnableMemoryPoolUsage(gMemPoolHandle, pSession); + for (int32_t i = 0; i < loopTimes; ++i) { + mptMemoryTrim(0, NULL); + mptMemoryTrim(0, &trimed); + } + mptDisableMemoryPoolUsage(); + + +} +#endif #endif diff --git a/tests/army/cluster/arbitrator.py b/tests/army/cluster/arbitrator.py new file mode 100644 index 0000000000..9fd8e7b1f3 --- /dev/null +++ b/tests/army/cluster/arbitrator.py @@ -0,0 +1,81 @@ +import taos +import sys +import os +import subprocess +import glob +import shutil +import time + +from frame.log import * +from frame.cases import * +from frame.sql import * +from frame.srvCtl import * +from frame.caseBase import * +from frame import * +from frame.autogen import * +from frame import epath +# from frame.server.dnodes import * +# from frame.server.cluster import * + + +class TDTestCase(TBase): + + def init(self, conn, logSql, replicaVar=1): + updatecfgDict = {'dDebugFlag':131} + super(TDTestCase, self).init(conn, logSql, replicaVar=1, checkColName="c1") + + self.valgrind = 0 + self.db = "test" + self.stb = "meters" + self.childtable_count = 10 + tdSql.init(conn.cursor(), logSql) + + def run(self): + tdSql.execute('CREATE DATABASE db vgroups 1 replica 2;') + + time.sleep(1) + + count = 0 + + while count < 100: + tdSql.query("show arbgroups;") + + if tdSql.getData(0, 4) == 1: + break + + tdLog.info("wait 1 seconds for is sync") + time.sleep(1) + + count += 1 + + + tdSql.query("show db.vgroups;") + + if(tdSql.getData(0, 4) == "follower") and (tdSql.getData(0, 6) == "leader"): + tdLog.info("stop dnode2") + sc.dnodeStop(2) + + if(tdSql.getData(0, 6) == "follower") and (tdSql.getData(0, 4) == "leader"): + tdLog.info("stop dnode 3") + sc.dnodeStop(3) + + + count = 0 + while count < 100: + tdSql.query("show db.vgroups;") + + if(tdSql.getData(0, 4) == "assigned ") or (tdSql.getData(0, 6) == "assigned "): + break + + tdLog.info("wait 1 seconds for set assigned") + time.sleep(1) + + count += 1 + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index ec10338642..7d093c80d1 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -11,6 +11,7 @@ # ,,y,army,./pytest.sh python3 ./test.py -f multi-level/mlevel_basic.py -N 3 -L 3 -D 2 ,,y,army,./pytest.sh python3 ./test.py -f db-encrypt/basic.py -N 3 -M 3 +,,y,army,./pytest.sh python3 ./test.py -f cluster/arbitrator.py -N 3 ,,n,army,python3 ./test.py -f storage/s3/s3Basic.py -N 3 ,,y,army,./pytest.sh python3 ./test.py -f cluster/snapshot.py -N 3 -L 3 -D 2 ,,y,army,./pytest.sh python3 ./test.py -f query/function/test_func_elapsed.py @@ -365,6 +366,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/telemetry.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/backquote_check.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/taosdMonitor.py +,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/taosdNewMonitor.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/taosd_audit.py ,,n,system-test,python3 ./test.py -f 0-others/taosdlog.py ,,n,system-test,python3 ./test.py -f 0-others/taosdShell.py -N 5 -M 3 -Q 3 @@ -407,8 +409,10 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/persisit_config.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/qmemCtrl.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/compact_vgroups.py +,,n,system-test,python3 ./test.py -f 0-others/dumpsdb.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/compact.py -N 3 + ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/composite_primary_key_create.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/composite_primary_key_insert.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/composite_primary_key_delete.py @@ -782,6 +786,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_querys.py -N 4 -M 1 ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_vgroups.py -N 4 -M 1 ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/compactDBConflict.py -N 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/mnodeEncrypt.py 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/between.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distinct.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/varchar.py -Q 2 diff --git a/tests/system-test/0-others/dumpsdb.py b/tests/system-test/0-others/dumpsdb.py new file mode 100644 index 0000000000..fd0fd2fcd9 --- /dev/null +++ b/tests/system-test/0-others/dumpsdb.py @@ -0,0 +1,98 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import os +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def caseDescription(self): + """ + dump sdb taosd -s + """ + + def init(self, conn, logSql, replicaVar=1): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + self.tmpdir = "tmp" + + def getPath(self, tool="taosd"): + if (platform.system().lower() == 'windows'): + tool = tool + ".exe" + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + paths = [] + for root, dirs, files in os.walk(projPath): + if ((tool) in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + paths.append(os.path.join(root, tool)) + break + if (len(paths) == 0): + tdLog.exit("%s not found!"%tool) + return + else: + tdLog.info("%s found in %s" % (tool, paths[0])) + return paths[0] + + def run(self): + tdSql.execute("create database db keep 3649 ") + + tdSql.execute("use db") + tdSql.execute( + "create table st(ts timestamp, c1 INT, c2 BOOL, c3 TINYINT, c4 SMALLINT, c5 BIGINT, c6 FLOAT, c7 DOUBLE, c8 TIMESTAMP, c9 BINARY(10), c10 NCHAR(10), c11 TINYINT UNSIGNED, c12 SMALLINT UNSIGNED, c13 INT UNSIGNED, c14 BIGINT UNSIGNED) tags(n1 INT, w2 BOOL, t3 TINYINT, t4 SMALLINT, t5 BIGINT, t6 FLOAT, t7 DOUBLE, t8 TIMESTAMP, t9 BINARY(10), t10 NCHAR(10), t11 TINYINT UNSIGNED, t12 SMALLINT UNSIGNED, t13 INT UNSIGNED, t14 BIGINT UNSIGNED)" + ) + tdSql.execute( + "create table t1 using st tags(1, true, 1, 1, 1, 1.0, 1.0, 1, '1', '一', 1, 1, 1, 1)" + ) + tdSql.execute( + "insert into t1 values(1640000000000, 1, true, 1, 1, 1, 1.0, 1.0, 1, '1', '一', 1, 1, 1, 1)" + ) + tdSql.execute( + "create table t2 using st tags(NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL)" + ) + tdSql.execute( + "insert into t2 values(1640000000000, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL)" + ) + + # sys.exit(1) + + binPath = self.getPath() + if binPath == "": + tdLog.exit("taosd not found!") + else: + tdLog.info("taosd found in %s" % binPath) + + if os.path.exists("sdb.json"): + os.system("rm -f sdb.json") + + os.system("%s -s" % binPath) + + if not os.path.exists("sdb.json"): + tdLog.exit("taosd -s failed!") + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/system-test/0-others/taosdNewMonitor.py b/tests/system-test/0-others/taosdNewMonitor.py new file mode 100644 index 0000000000..a3ced155c5 --- /dev/null +++ b/tests/system-test/0-others/taosdNewMonitor.py @@ -0,0 +1,243 @@ +import taos +import sys +import time +import socket +# import pexpect +import os +import http.server +import gzip +import threading +import json +import pickle + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * + +telemetryPort = '6043' +serverPort = '7080' +hostname = socket.gethostname() + +class RequestHandlerImpl(http.server.BaseHTTPRequestHandler): + hostPort = hostname + ":" + serverPort + + def telemetryInfoCheck(self, infoDict=''): + if len(infoDict) == 0: + return + + if "ts" not in infoDict[0] or len(infoDict[0]["ts"]) == 0: + tdLog.exit("ts is null!") + + if "protocol" not in infoDict[0] or infoDict[0]["protocol"] != 2: + tdLog.exit("protocol is null!") + + if "tables" not in infoDict[0]: + tdLog.exit("tables is null!") + + if infoDict[0]["tables"][0]["name"] != "taosd_dnodes_info": + tdLog.exit("taosd_dnodes_info is null!") + + # dnode_info ==================================== + + dnode_infos = ['disk_engine', 'system_net_in', 'vnodes_num', 'system_net_out', 'uptime', 'has_mnode', 'io_read_disk', 'error_log_count', + 'io_read', 'cpu_cores', 'has_qnode', 'has_snode', 'disk_total', 'mem_engine', 'info_log_count', 'cpu_engine', 'io_write_disk', + 'debug_log_count', 'disk_used', 'mem_total', 'io_write', 'masters', 'cpu_system', + 'trace_log_count', 'mem_free'] + index = 0 + for elem in dnode_infos: + tdLog.debug(f"elem: {index},{elem}") + if infoDict[0]["tables"][0]["metric_groups"][0]["metrics"][index]["name"] != elem: + tdLog.exit(f"{elem} is null!") + index += 1 + + if infoDict[0]["tables"][1]["name"] != "taosd_dnodes_log_dirs": + tdLog.exit("taosd_dnodes_log_dirs is null!") + + # logdir + if infoDict[0]["tables"][1]["metric_groups"][0]["tags"][3]["name"] != "data_dir_name": + tdLog.exit("data_dir_name is null!") + + if infoDict[0]["tables"][1]["metric_groups"][0]["metrics"][0]["name"] != "total": + tdLog.exit("total is null!") + + if infoDict[0]["tables"][1]["metric_groups"][0]["metrics"][1]["name"] != "used": + tdLog.exit("used is null!") + + if infoDict[0]["tables"][1]["metric_groups"][0]["metrics"][2]["name"] != "avail": + tdLog.exit("avail is null!") + + if infoDict[0]["tables"][2]["name"] != "taosd_dnodes_data_dirs": + tdLog.exit("taosd_dnodes_data_dirs is null!") + + # data_dir + if infoDict[0]["tables"][2]["metric_groups"][0]["tags"][3]["name"] != "data_dir_name": + tdLog.exit("data_dir_name is null!") + + if infoDict[0]["tables"][2]["metric_groups"][0]["metrics"][0]["name"] != "avail": + tdLog.exit("total is null!") + + if infoDict[0]["tables"][2]["metric_groups"][0]["metrics"][1]["name"] != "total": + tdLog.exit("used is null!") + + if infoDict[0]["tables"][2]["metric_groups"][0]["metrics"][2]["name"] != "used": + tdLog.exit("avail is null!") + + if infoDict[0]["tables"][3]["name"] != "taosd_cluster_info": + tdLog.exit("taosd_cluster_info is null!") + + # cluster_info ==================================== + + if infoDict[0]["tables"][3]["metric_groups"][0]["metrics"][0]["name"] != "cluster_uptime": + tdLog.exit("cluster_uptime is null!") + + if infoDict[0]["tables"][3]["metric_groups"][0]["metrics"][1]["name"] != "dbs_total": + tdLog.exit("dbs_total is null!") + + if infoDict[0]["tables"][3]["metric_groups"][0]["metrics"][4]["name"] != "vgroups_total": + tdLog.exit("vgroups_total is null!") + + if infoDict[0]["tables"][3]["metric_groups"][0]["metrics"][5]["name"] != "vgroups_alive": + tdLog.exit("vgroups_alive is null!") + + if infoDict[0]["tables"][3]["metric_groups"][0]["metrics"][10]["name"] != "connections_total": + tdLog.exit("connections_total is null!") + + if infoDict[0]["tables"][3]["metric_groups"][0]["metrics"][13]["name"] != "dnodes_total": + tdLog.exit("dnodes_total is null!") + + # grant_info ==================================== + if infoDict[0]["tables"][3]["metric_groups"][0]["metrics"][15]["name"] != "grants_expire_time": + tdLog.exit("grants_expire_time is null!") + + if infoDict[0]["tables"][3]["metric_groups"][0]["metrics"][16]["name"] != "grants_timeseries_used": + tdLog.exit("grants_timeseries_used is null!") + + if infoDict[0]["tables"][3]["metric_groups"][0]["metrics"][17]["name"] != "grants_timeseries_total": + tdLog.exit("grants_timeseries_total is null!") + + # vgroup_infos ==================================== + + vgroup_infos_nums = len(infoDict[0]["tables"][4]["metric_groups"]) + + for index in range(vgroup_infos_nums): + if infoDict[0]["tables"][4]["metric_groups"][index]["metrics"][0]["name"] != "tables_num": + tdLog.exit("tables_num is null!") + + if infoDict[0]["tables"][4]["metric_groups"][index]["metrics"][1]["name"] != "status": + tdLog.exit("status is null!") + + if infoDict[0]["tables"][5]["name"] != "taosd_dnodes_status": + tdLog.exit("taosd_dnodes_status is null!") + + if infoDict[0]["tables"][6]["name"] != "taosd_mnodes_info": + tdLog.exit("taosd_mnodes_info is null!") + + if infoDict[0]["tables"][7]["name"] != "taosd_vnodes_info": + tdLog.exit("taosd_vnodes_info is null!") + + def do_GET(self): + """ + process GET request + """ + + def do_POST(self): + """ + process POST request + """ + contentEncoding = self.headers["Content-Encoding"] + + if contentEncoding == 'gzip': + req_body = self.rfile.read(int(self.headers["Content-Length"])) + plainText = gzip.decompress(req_body).decode() + else: + plainText = self.rfile.read(int(self.headers["Content-Length"])).decode() + + print(plainText) + # 1. send response code and header + self.send_response(200) + self.send_header("Content-Type", "text/html; charset=utf-8") + self.end_headers() + + # 2. send response content + #self.wfile.write(("Hello World: " + req_body + "\n").encode("utf-8")) + + # 3. check request body info + infoDict = json.loads(plainText) + #print("================") + # print(infoDict) + self.telemetryInfoCheck(infoDict) + + # 4. shutdown the server and exit case + assassin = threading.Thread(target=self.server.shutdown) + assassin.daemon = True + assassin.start() + print ("==== shutdown http server ====") + +class TDTestCase: + global hostname + global serverPort + if (platform.system().lower() == 'windows' and not tdDnodes.dnodes[0].remoteIP == ""): + try: + config = eval(tdDnodes.dnodes[0].remoteIP ) + hostname = config["host"] + except Exception: + hostname = tdDnodes.dnodes[0].remoteIP + rpcDebugFlagVal = '143' + clientCfgDict = {'serverPort': '', 'firstEp': '', 'secondEp':'', 'rpcDebugFlag':'135', 'fqdn':''} + clientCfgDict["serverPort"] = serverPort + clientCfgDict["firstEp"] = hostname + ':' + serverPort + clientCfgDict["secondEp"] = hostname + ':' + serverPort + clientCfgDict["rpcDebugFlag"] = rpcDebugFlagVal + clientCfgDict["fqdn"] = hostname + + updatecfgDict = {'clientCfg': {}, 'serverPort': '', 'firstEp': '', 'secondEp':'', 'rpcDebugFlag':'135', 'fqdn':''} + updatecfgDict["clientCfg"] = clientCfgDict + updatecfgDict["serverPort"] = serverPort + updatecfgDict["firstEp"] = hostname + ':' + serverPort + updatecfgDict["secondEp"] = hostname + ':' + serverPort + updatecfgDict["fqdn"] = hostname + + updatecfgDict["monitorFqdn"] = hostname + updatecfgDict["monitorPort"] = '6043' + updatecfgDict["monitor"] = '1' + updatecfgDict["monitorInterval"] = "5" + updatecfgDict["monitorMaxLogs"] = "10" + updatecfgDict["monitorComp"] = "1" + updatecfgDict["monitorForceV2"] = "1" + + updatecfgDict["audit"] = '0' + + print ("===================: ", updatecfgDict) + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + + def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring + tdSql.prepare() + # time.sleep(2) + vgroups = "4" + sql = "create database db3 vgroups " + vgroups + tdSql.query(sql) + sql = "create table db3.stb (ts timestamp, f int) tags (t int)" + tdSql.query(sql) + sql = "create table db3.tb using db3.stb tags (1)" + tdSql.query(sql) + + # create http server: bing ip/port , and request processor + if (platform.system().lower() == 'windows' and not tdDnodes.dnodes[0].remoteIP == ""): + RequestHandlerImplStr = base64.b64encode(pickle.dumps(RequestHandlerImpl)).decode() + cmdStr = "import pickle\nimport http\nRequestHandlerImpl=pickle.loads(base64.b64decode(\"%s\".encode()))\nclass NewRequestHandlerImpl(RequestHandlerImpl):\n hostPort = \'%s\'\nhttp.server.HTTPServer((\"\", %s), NewRequestHandlerImpl).serve_forever()"%(RequestHandlerImplStr,hostname+":"+serverPort,telemetryPort) + tdDnodes.dnodes[0].remoteExec({}, cmdStr) + else: + serverAddress = ("", int(telemetryPort)) + http.server.HTTPServer(serverAddress, RequestHandlerImpl).serve_forever() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/6-cluster/mnodeEncrypt.py b/tests/system-test/6-cluster/mnodeEncrypt.py new file mode 100644 index 0000000000..e878611d32 --- /dev/null +++ b/tests/system-test/6-cluster/mnodeEncrypt.py @@ -0,0 +1,69 @@ +from ssl import ALERT_DESCRIPTION_CERTIFICATE_UNOBTAINABLE +from numpy import row_stack +import taos +import sys +import time +import os + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import TDDnodes +from util.dnodes import TDDnode +from util.cluster import * +sys.path.append("./6-cluster") +from clusterCommonCreate import * +from clusterCommonCheck import clusterComCheck + +import time +import socket +import subprocess +from multiprocessing import Process +import threading +import time +import inspect +import ctypes + +class TDTestCase: + + def init(self, conn, logSql, replicaVar=1): + tdLog.debug(f"start to excute {__file__}") + self.TDDnodes = None + tdSql.init(conn.cursor()) + self.host = socket.gethostname() + + + def run(self): + tdSql.execute('create database if not exists db'); + tdSql.execute('use db') + tdSql.execute('create table st (ts timestamp, i int, j float, k double) tags(a int)') + + for i in range(0, 2): + tdSql.execute("create table if not exists db.t%d using db.st tags(%d)" % (i, i)) + + + for i in range(2, 4): + tdSql.execute("create table if not exists db.t%d using db.st tags(%d)" % (i, i)) + + sql = "show db.tables" + tdSql.query(sql) + tdSql.checkRows(4) + + timestamp = 1530374400000 + for i in range (4) : + val = i + sql = "insert into db.t%d values(%d, %d, %d, %d)" % (i, timestamp, val, val, val) + tdSql.execute(sql) + + for i in range ( 4) : + val = i + sql = "select * from db.t%d" % (i) + tdSql.query(sql) + tdSql.checkRows(1) + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/6-cluster/replica2.py b/tests/system-test/6-cluster/replica2.py new file mode 100644 index 0000000000..1cda2bb4e8 --- /dev/null +++ b/tests/system-test/6-cluster/replica2.py @@ -0,0 +1,50 @@ +################################################################### + # Copyright (c) 2016 by TAOS Technologies, Inc. + # All rights reserved. + # + # This file is proprietary and confidential to TAOS Technologies. + # No part of this file may be reproduced, stored, transmitted, + # disclosed or used in any form or by any means other than as + # expressly provided by the written permission from Jianhui Tao + # +################################################################### +from util.cases import * +from util.sql import * +from util.dnodes import * +from util.log import * + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + tdLog.debug(f"start to init {__file__}") + self.replicaVar = int(replicaVar) + tdSql.init(conn.cursor(), logSql) + + def run(self): + tdSql.execute('CREATE DATABASE db vgroups 1 replica 2;') + + time.sleep(1) + + tdSql.query("show db.vgroups;") + + if(tdSql.queryResult[0][4] == "follower") and (tdSql.queryResult[0][6] == "leader"): + tdLog.info("stop dnode2") + sc.dnodeStop(2) + + if(tdSql.queryResult[0][6] == "follower") and (tdSql.queryResult[0][4] == "leader"): + tdLog.info("stop dnode 3") + sc.dnodeStop(3) + + tdLog.info("wait 10 seconds") + time.sleep(10) + + tdSql.query("show db.vgroups;") + + if(tdSql.queryResult[0][4] != "assigned") and (tdSql.queryResult[0][6] != "assigned"): + tdLog.exit("failed to set aasigned") + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file diff --git a/tools/keeper/go.mod b/tools/keeper/go.mod index 00a399768d..f8edf2709b 100644 --- a/tools/keeper/go.mod +++ b/tools/keeper/go.mod @@ -70,10 +70,10 @@ require ( github.com/ugorji/go/codec v1.2.12 // indirect github.com/yusufpapurcu/wmi v1.2.2 // indirect golang.org/x/arch v0.7.0 // indirect - golang.org/x/crypto v0.31.0 // indirect + golang.org/x/crypto v0.21.0 // indirect golang.org/x/net v0.23.0 // indirect - golang.org/x/sys v0.28.0 // indirect - golang.org/x/text v0.21.0 // indirect + golang.org/x/sys v0.24.0 // indirect + golang.org/x/text v0.14.0 // indirect google.golang.org/protobuf v1.33.0 // indirect gopkg.in/ini.v1 v1.66.4 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/tools/keeper/go.sum b/tools/keeper/go.sum index f879c731ba..8f6e9bd13a 100644 --- a/tools/keeper/go.sum +++ b/tools/keeper/go.sum @@ -424,8 +424,8 @@ golang.org/x/crypto v0.0.0-20210314154223-e6e6c4f2bb5b/go.mod h1:T9bdIzuCu7OtxOm golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= -golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= +golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= +golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -573,8 +573,8 @@ golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= -golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg= +golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -583,8 +583,8 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= -golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=